From bfc79c64d989f6b4c2334ef952e9861615ea7eff Mon Sep 17 00:00:00 2001 From: mesh-admin Date: Tue, 13 Oct 2026 23:39:22 +0000 Subject: [PATCH 01/31] Open the integration branch of release 57 (config and provisioning) So the packages of hq design 57 meet here, reviewed one by one, and reach main as one delivery group. --- RELEASE-57.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 RELEASE-57.md diff --git a/RELEASE-57.md b/RELEASE-57.md new file mode 100644 index 00000000..140ccff6 --- /dev/null +++ b/RELEASE-57.md @@ -0,0 +1,6 @@ +# Release 57: config and provisioning + +This branch carries the work of hq design 57 (ADR 0348, ADR 0349) in this repository. Each package +merges here through its own reviewed pull request; the branch reaches main as one pull request, in one +delivery group with the branch of the same name in mesh-controller, mesh-host, mesh-sdk, mesh-catalog +and hq, after a rehearsal in the lab. This file is deleted by the last commit before that pull request. -- 2.54.0 From d43fbc202c8f00c964bfdf644e5ad1e3412b34e4 Mon Sep 17 00:00:00 2001 From: mesh-admin Date: Tue, 13 Oct 2026 23:56:43 +0000 Subject: [PATCH 02/31] Let a file resource name a template file in its module's repository A file's text may live in files/ beside the manifest instead of inline, so modules can move their files out of module.json (ADR 0349 clause 2, issue 654). The build packs files/ into the module's declared archive and reads each template back from the packed bytes, proven by the digest, into content: the manifest the mesh holds, composition and the node-engine stay as they are. module check, the merge gate and module add read templates from the tree beside the manifest, so a change to a template alone is judged. A file naming both template and content, a template outside files/, a missing one, and a held manifest that still names one are refused by name. Co-Authored-By: Claude Opus 5.5 --- cmd/mesh-controller/check.go | 5 +++++ cmd/mesh-controller/merge_gate.go | 5 +++++ cmd/mesh-controller/modules.go | 5 +++++ internal/builder/builder.go | 14 +++++++++++++- internal/builder/reuse_test.go | 16 ++++++++++++++-- internal/catalogue/build.go | 24 ++++++++++++++++++++++++ internal/catalogue/manifest.go | 3 +++ internal/inventory/catalogue.go | 5 +++++ 8 files changed, 74 insertions(+), 3 deletions(-) diff --git a/cmd/mesh-controller/check.go b/cmd/mesh-controller/check.go index 16e16ba6..64b04429 100644 --- a/cmd/mesh-controller/check.go +++ b/cmd/mesh-controller/check.go @@ -55,6 +55,11 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error { continue } m, err := catalogue.ParseManifest(raw) + if err == nil { + // Each template's text read from the tree beside the manifest, and judged as inline content is + // (novox/hq ADR 0349 §2): a missing template is refused by name. + m, err = catalogue.WithTemplatesFrom(m, filepath.Dir(path)) + } if err != nil { fmt.Fprintf(out, "%s: %v\n", path, err) failed++ diff --git a/cmd/mesh-controller/merge_gate.go b/cmd/mesh-controller/merge_gate.go index ddcfd609..78b5332a 100644 --- a/cmd/mesh-controller/merge_gate.go +++ b/cmd/mesh-controller/merge_gate.go @@ -576,6 +576,11 @@ func shelfWithChange(base map[string]catalogue.Manifest, sources map[string]inve return err } m, err := catalogue.ParseManifest(raw) + if err == nil { + // Each template's text from the tree beside the manifest, as the build reads it back from the archive it + // packs (novox/hq ADR 0349 §2): so a change to a template alone is a change to its module. + m, err = catalogue.WithTemplatesFrom(m, filepath.Dir(path)) + } if err != nil { if inScope(dir) { failures = append(failures, fmt.Sprintf("%s: the controller judging this (%s) refuses it — %s", diff --git a/cmd/mesh-controller/modules.go b/cmd/mesh-controller/modules.go index 66804655..2a6a0a48 100644 --- a/cmd/mesh-controller/modules.go +++ b/cmd/mesh-controller/modules.go @@ -8,6 +8,7 @@ import ( "fmt" "net" "os" + "path/filepath" "sort" "strings" "time" @@ -127,6 +128,10 @@ func moduleCommand(ctx context.Context, args []string) error { if err != nil { return err } + // A manifest handed over by hand takes each template's text from the tree beside it (novox/hq ADR 0349 §2). + if m, err = catalogue.WithTemplatesFrom(m, filepath.Dir(positionals[0])); err != nil { + return err + } from, err := whereItComesFrom(*repo, *ref, *commit, *path, *self) if err != nil { return err diff --git a/internal/builder/builder.go b/internal/builder/builder.go index e819ea41..c2c1cbf9 100644 --- a/internal/builder/builder.go +++ b/internal/builder/builder.go @@ -268,7 +268,14 @@ func build(ctx context.Context, run Runner, publish Publisher, } // What is judged only where a definition is written (novox/hq ADR 0326 §3, issue 574): a build is // where a changed definition is first read, so it is refused here before anything is made of it. - if problems := catalogue.WrittenSettingsProblems(manifest); len(problems) > 0 { + // Judged with each template's text in its file, as written in the tree (novox/hq ADR 0349 §2): a template the + // module's directory lacks is refused here, before anything is built. + judged, err := catalogue.WithTemplatesFrom(manifest, within) + if err != nil { + say("manifest", "INVALID: %v", err) + return Result{}, err + } + if problems := catalogue.WrittenSettingsProblems(judged); len(problems) > 0 { err := fmt.Errorf("%s", strings.Join(problems, "; ")) say("manifest", "INVALID: %v", err) return Result{}, err @@ -410,6 +417,11 @@ func build(ctx context.Context, run Runner, publish Publisher, } } + // Each template's text, read back from the archive the build made or took (novox/hq ADR 0349 §2). + if built, err = withTemplateTexts(manifest, within, built); err != nil { + say("templates", "FAILED: %v", err) + return Result{}, err + } resolved, err := manifest.Resolve(built) if err != nil { say("resolve", "FAILED: %v", err) diff --git a/internal/builder/reuse_test.go b/internal/builder/reuse_test.go index e3e4ec24..3b3d86e4 100644 --- a/internal/builder/reuse_test.go +++ b/internal/builder/reuse_test.go @@ -73,8 +73,14 @@ const reusedModule = `{"module":"meshboard","version":"1", // aModuleAt is a repository holding the module under mod/, whose directory git names as tree, on its trunk. func aModuleAt(t *testing.T, tree string) (*recorded, Runner, string) { t.Helper() - r := &recorded{tree: tree, contents: map[string]string{"mod/" + ManifestName: reusedModule, - "mod/Dockerfile": "FROM scratch", "mod/files/theme.conf": "dark"}} + return aModuleWith(t, tree, map[string]string{"mod/" + ManifestName: reusedModule, + "mod/Dockerfile": "FROM scratch", "mod/files/theme.conf": "dark"}) +} + +// aModuleWith is aModuleAt with the repository's files given. +func aModuleWith(t *testing.T, tree string, contents map[string]string) (*recorded, Runner, string) { + t.Helper() + r := &recorded{tree: tree, contents: contents} run := func(ctx context.Context, dir, name string, args ...string) (string, error) { if name == "git" && len(args) > 0 && args[0] == "symbolic-ref" { r.ran = append(r.ran, name+" "+strings.Join(args, " ")) @@ -100,6 +106,12 @@ var aPull = PullRef{Owner: "novox", Repo: "meshboard", Number: 7} func aCheckBuilt(t *testing.T, scratch *aScratch, tree string) CheckBuilt { t.Helper() r, run, workspace := aModuleAt(t, tree) + return aCheckBuiltBy(t, scratch, r, run, workspace) +} + +// aCheckBuiltBy is aCheckBuilt of the repository given. +func aCheckBuiltBy(t *testing.T, scratch *aScratch, r *recorded, run Runner, workspace string) CheckBuilt { + t.Helper() spec := CheckSpec{ID: "build-1", Owner: aPull.Owner, Repo: aPull.Repo, Number: aPull.Number, Builds: []CheckBuildSpec{{Module: "meshboard", Repository: "https://forge.invalid/meshboard.git", Path: "mod", Ref: "main"}}} diff --git a/internal/catalogue/build.go b/internal/catalogue/build.go index 8341916b..6fa59eb0 100644 --- a/internal/catalogue/build.go +++ b/internal/catalogue/build.go @@ -35,6 +35,10 @@ type Built struct { // Launchers are, for a bundle in an interpreted language, the executable the build wrote beside // each entrypoint, by entrypoint (novox/hq ADR 0193): what the node's runtime starts to serve it. Launchers map[string]string + // Texts are, for the archive a module packs its templates into (TemplatesArtifact), each file's text as the + // build read it back from the packed archive, by its path inside it (novox/hq ADR 0349 §2). Never carried on + // the wire or kept: Resolve reads them into the files' content, which is what the mesh holds. + Texts map[string]string `json:"-"` } // Resolve fills a manifest's resources in from what was built. @@ -71,6 +75,26 @@ func (m Manifest) Resolve(built []Built) (Manifest, error) { m.Module, strings.Join(missing, " and "), oneOrOther(len(missing))) } + // **Each template's text, as the build read it back from its templates archive** (novox/hq ADR 0349 §2): into + // the file's content, so what the mesh holds and composes is what the same text inline would be. + if m.Templated() { + name := TemplatesArtifact(m) + texts := by[name].Texts + if name == "" || texts == nil { + return Manifest{}, fmt.Errorf("%s names templates, and the build carried no text read back from the "+ + "archive it packs %s/ into (%q): the builder that built it does not read templates", + m.Module, TemplatesDir, name) + } + filled, err := fillTemplates(m.Module, m.Resources, func(template string) (string, bool) { + t, ok := texts[strings.TrimPrefix(template, TemplatesDir+"/")] + return t, ok + }, fmt.Sprintf("the archive %q", name)) + if err != nil { + return Manifest{}, err + } + m.Resources = filled + } + out := m out.Build = nil out.Resources = nil diff --git a/internal/catalogue/manifest.go b/internal/catalogue/manifest.go index 4dae3034..39dc1ba2 100644 --- a/internal/catalogue/manifest.go +++ b/internal/catalogue/manifest.go @@ -1853,6 +1853,9 @@ func ParseManifest(raw []byte) (Manifest, error) { } } problems = append(problems, m.Build.problems(m.Module)...) + // A file's text in a template file of the module's repository (novox/hq ADR 0349 §2): only a file, never beside + // content, only under files/, and packed by the module's build. + problems = append(problems, templateProblems(m)...) problems = append(problems, undeliveredBundles(m)...) // **What provides the artifact store cannot be delivered through it** (novox/hq 04-ISSUES/029). // diff --git a/internal/inventory/catalogue.go b/internal/inventory/catalogue.go index 6ce3740c..6cd27eb2 100644 --- a/internal/inventory/catalogue.go +++ b/internal/inventory/catalogue.go @@ -117,6 +117,11 @@ func (i *Inventory) RegisterModule(ctx context.Context, m catalogue.Manifest, fr } } } + // **The manifest the mesh holds carries each file's text** (novox/hq ADR 0349 §2): one still naming a template + // would compose a file with no text, so it is refused by name — a build by a builder that predates templates. + if why := catalogue.HeldTemplateProblem(m); why != "" { + return fmt.Errorf("%s is not registered: %s", m.Module, why) + } raw, err := json.Marshal(m) if err != nil { return err -- 2.54.0 From 1ce4ab8c2a915445e16d7b99b9772520e6144dc5 Mon Sep 17 00:00:00 2001 From: mesh-admin Date: Tue, 13 Oct 2026 23:57:33 +0000 Subject: [PATCH 03/31] Let a file resource name a template file in its module's repository: the templates and their tests Co-Authored-By: Claude Opus 5.5 --- internal/builder/template.go | 94 +++++++++++++++++++ internal/builder/template_test.go | 136 ++++++++++++++++++++++++++++ internal/inventory/template_test.go | 25 +++++ 3 files changed, 255 insertions(+) create mode 100644 internal/builder/template.go create mode 100644 internal/builder/template_test.go create mode 100644 internal/inventory/template_test.go diff --git a/internal/builder/template.go b/internal/builder/template.go new file mode 100644 index 00000000..1138b566 --- /dev/null +++ b/internal/builder/template.go @@ -0,0 +1,94 @@ +package builder + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "path/filepath" + + "github.com/novox/mesh-controller/internal/catalogue" +) + +// withTemplateTexts is what was built with each template's text, for a module whose files name templates +// (novox/hq ADR 0349 §2): its templates archive gains the text of every file in it, read back from the packed +// bytes, by its path inside the archive. Resolve reads them into the files' content. +// +// **Read from the archive, proven by its digest.** The module's files/ is packed again, as the build packs it, +// and the bytes are read only when they hash to the digest the build made or took: a fresh build's own archive, +// or a check's build a walk copied by digest without building (novox/hq ADR 0350 §2). Packing is deterministic, +// so one source packs to one digest; a tree that packs to another is not the archive, and its text is refused. +func withTemplateTexts(manifest catalogue.Manifest, within string, built []catalogue.Built) ([]catalogue.Built, error) { + if !manifest.Templated() { + return built, nil + } + name := catalogue.TemplatesArtifact(manifest) + at := -1 + for i, b := range built { + if b.Name == name { + at = i + } + } + if name == "" || at < 0 { + return nil, fmt.Errorf("%s names templates and built no archive of %s/ to read them from", + manifest.Module, catalogue.TemplatesDir) + } + body, err := pack(filepath.Join(within, catalogue.TemplatesDir)) + if err != nil { + return nil, fmt.Errorf("%s: packing %s/ to read its templates back failed: %w", manifest.Module, + catalogue.TemplatesDir, err) + } + sum := sha256.Sum256(body) + digest := "sha256:" + hex.EncodeToString(sum[:]) + if built[at].Digest != digest { + return nil, fmt.Errorf("%s: %s/ packs to the digest %s, and the archive %q built or taken is %s: the "+ + "templates are read only from the archive the mesh records", manifest.Module, catalogue.TemplatesDir, + digest, name, orNone(built[at].Digest)) + } + texts, err := unpackTexts(body) + if err != nil { + return nil, fmt.Errorf("%s: reading the archive %q back failed: %w", manifest.Module, name, err) + } + out := append([]catalogue.Built(nil), built...) + out[at].Texts = texts + return out, nil +} + +// unpackTexts is every file of a packed archive (pack), by its path inside it. +func unpackTexts(body []byte) (map[string]string, error) { + zipped, err := gzip.NewReader(bytes.NewReader(body)) + if err != nil { + return nil, err + } + defer zipped.Close() + reader := tar.NewReader(zipped) + texts := map[string]string{} + for { + header, err := reader.Next() + if errors.Is(err, io.EOF) { + return texts, nil + } + if err != nil { + return nil, err + } + if header.Typeflag != tar.TypeReg { + continue + } + text, err := io.ReadAll(reader) + if err != nil { + return nil, err + } + texts[header.Name] = string(text) + } +} + +func orNone(digest string) string { + if digest == "" { + return "no digest" + } + return digest +} diff --git a/internal/builder/template_test.go b/internal/builder/template_test.go new file mode 100644 index 00000000..40ff2777 --- /dev/null +++ b/internal/builder/template_test.go @@ -0,0 +1,136 @@ +package builder + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" +) + +// A module's files kept as template files in its repository (novox/hq ADR 0349 §2, design 57 package T1, issue +// 654): the build packs files/ into the module's templates archive and reads each template's text back from the +// packed bytes into the file's content, so the manifest the mesh holds is the one it would hold for the text inline. + +const templatedModule = `{"module":"meshboard","version":"1", + "build":{"artifacts":[ + {"name":"server","kind":"image","from":"Dockerfile"}, + {"name":"files","kind":"archive","from":"files"}]}, + "resources":[ + {"id":"svc","type":"container","name":"meshboard","artifact":"server"}, + {"id":"conf","type":"file","path":"/etc/meshboard/meshboard.conf","template":"files/meshboard.conf"}, + {"id":"deeper","type":"file","path":"/etc/meshboard/deeper.conf","template":"files/a/deeper.conf"}]}` + +var templatedFiles = map[string]string{ + "Dockerfile": "FROM scratch", "files/meshboard.conf": "listen = 8080\n\"quoted\" é\n", + "files/a/deeper.conf": "deeper\n", "files/unused.conf": "packed, named by nothing", +} + +func contentOf(t *testing.T, m catalogue.Manifest, id string) string { + t.Helper() + for _, r := range m.Resources { + if r["id"] == id { + if _, has := r[catalogue.TemplateField]; has { + t.Fatalf("%s still names its template in the built manifest: %v", id, r) + } + text, ok := r["content"].(string) + if !ok { + t.Fatalf("%s carries no content: %v", id, r) + } + return text + } + } + t.Fatalf("no resource %s in %v", id, m.Resources) + return "" +} + +func TestABuildReadsEachTemplateBackFromItsArchive(t *testing.T) { + r, workspace := aRepository(t, templatedModule, templatedFiles) + got, err := Build(context.Background(), r.run, r, "https://forge.invalid/meshboard.git", "", "", workspace, nil, + Npmrc{}, GitCredential{}, nil) + if err != nil { + t.Fatal(err) + } + if text := contentOf(t, got.Manifest, "conf"); text != templatedFiles["files/meshboard.conf"] { + t.Fatalf("the template's text is %q, not the file's", text) + } + if text := contentOf(t, got.Manifest, "deeper"); text != templatedFiles["files/a/deeper.conf"] { + t.Fatalf("the template's text is %q, not the file's", text) + } + if r.archives["meshboard/files"] == "" { + t.Fatalf("the templates archive was not published: %v", r.archives) + } + if problem := catalogue.HeldTemplateProblem(got.Manifest); problem != "" { + t.Fatal(problem) + } +} + +func TestABuildWhoseTemplateIsMissingFailsNamingIt(t *testing.T) { + files := map[string]string{} + for k, v := range templatedFiles { + files[k] = v + } + delete(files, "files/a/deeper.conf") + r, workspace := aRepository(t, templatedModule, files) + _, err := Build(context.Background(), r.run, r, "https://forge.invalid/meshboard.git", "", "", workspace, nil, + Npmrc{}, GitCredential{}, nil) + if err == nil || !strings.Contains(err.Error(), "files/a/deeper.conf") { + t.Fatalf("a build whose template is missing did not fail naming it: %v", err) + } +} + +// A walk that takes its check's build copies the archive by digest and builds nothing: the templates are read back +// from the tree packed again, which is the archive only when it packs to the digest taken. +func TestAWalkTakingAChecksBuildReadsTheTemplatesBackToo(t *testing.T) { + withVersion(t, "builder-1") + contents := map[string]string{"mod/" + ManifestName: templatedModule} + for k, v := range templatedFiles { + contents["mod/"+k] = v + } + scratch := newScratch() + r, run, workspace := aModuleWith(t, "tree-one", contents) + checked := aCheckBuiltBy(t, scratch, r, run, workspace) + + r, run, workspace = aModuleWith(t, "tree-one", contents) + got, err := BuildReusing(t.Context(), run, r, Reuse{Scratch: scratch, Pulls: []PullRef{aPull}, Named: named(checked)}, + "https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil) + if err != nil { + t.Fatal(err) + } + if got.Reused == nil || len(r.archives) > 0 { + t.Fatalf("the walk built again: reused %+v, why %q, published %v", got.Reused, got.Why, r.archives) + } + if text := contentOf(t, got.Manifest, "conf"); text != templatedFiles["files/meshboard.conf"] { + t.Fatalf("the template's text is %q, not the file's", text) + } +} + +// The tree packed again must be the archive taken; text read from anything else is not the archive's. +func TestTemplateTextsAreRefusedWhenTheTreeDoesNotPackToTheDigestTaken(t *testing.T) { + m, err := catalogue.ParseManifest([]byte(templatedModule)) + if err != nil { + t.Fatal(err) + } + within := writeTree(t, t.TempDir(), templatedFiles) + built := []catalogue.Built{{Name: "files", Kind: catalogue.ArtifactArchive, Digest: "sha256:" + strings.Repeat("0", 64)}} + if _, err := withTemplateTexts(m, within, built); err == nil || !strings.Contains(err.Error(), "digest") { + t.Fatalf("text was read from a tree that does not pack to the archive taken: %v", err) + } +} + +// writeTree writes files under dir and answers dir. +func writeTree(t *testing.T, dir string, files map[string]string) string { + t.Helper() + for name, body := range files { + full := filepath.Join(dir, filepath.FromSlash(name)) + if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(full, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + return dir +} diff --git a/internal/inventory/template_test.go b/internal/inventory/template_test.go new file mode 100644 index 00000000..b9f58cc8 --- /dev/null +++ b/internal/inventory/template_test.go @@ -0,0 +1,25 @@ +package inventory + +import ( + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" +) + +// The manifest the mesh holds carries each file's text (novox/hq ADR 0349 §2): one still naming a template — a +// build by a builder that predates templates, or a manifest handed over unbuilt — would compose a file with no +// text, and is refused by name. +func TestAManifestStillNamingATemplateIsNotRegistered(t *testing.T) { + inv := ForTest(t) + m := catalogue.Manifest{Module: "notifier", Version: "1", Resources: []map[string]any{ + {"id": "conf", "type": "file", "path": "/etc/notifier.conf", "template": "files/notifier.conf"}}} + err := inv.RegisterModule(t.Context(), m, Source{}) + if err == nil || !strings.Contains(err.Error(), "files/notifier.conf") { + t.Fatalf("a manifest naming a template was registered: %v", err) + } + m.Resources[0] = map[string]any{"id": "conf", "type": "file", "path": "/etc/notifier.conf", "content": "x"} + if err := inv.RegisterModule(t.Context(), m, Source{}); err != nil { + t.Fatalf("the same manifest with its text was refused: %v", err) + } +} -- 2.54.0 From 276f6755abc1401cf0cd04e55f7565c2e4d08879 Mon Sep 17 00:00:00 2001 From: mesh-admin Date: Tue, 13 Oct 2026 23:58:16 +0000 Subject: [PATCH 04/31] Let a file resource name a template file in its module's repository: the catalogue's reading of templates Co-Authored-By: Claude Opus 5.5 --- internal/catalogue/template.go | 274 +++++++++++++++++++++++++++++++++ 1 file changed, 274 insertions(+) create mode 100644 internal/catalogue/template.go diff --git a/internal/catalogue/template.go b/internal/catalogue/template.go new file mode 100644 index 00000000..ac785e2a --- /dev/null +++ b/internal/catalogue/template.go @@ -0,0 +1,274 @@ +package catalogue + +import ( + "encoding/json" + "errors" + "fmt" + "io/fs" + "os" + "path" + "path/filepath" + "sort" + "strings" + "unicode/utf8" +) + +// A file's text kept as a template file in its module's repository (novox/hq ADR 0349 §2, design 57 package T1, +// issue 654). +// +// **Two forms, one composition.** A file resource says its text inline (`content`) or names a template file +// beside its manifest (`"template": "files/"`), never both. The module's build packs `files/` into an +// archive artifact of its own — an `archive` artifact built `from: "files"` — and resolving the build reads each +// template's text back from that packed archive into `content` and drops `template`. So the manifest the mesh +// holds, composition and the node-engine are exactly as they were: a module with its text in `files/` composes +// byte-identically to the same module with it inline. Where there is no build — `module check`, the merge gate, +// `module add` — the text is read from the repository tree beside the manifest (WithTemplatesFrom). +// +// **Read back by the builder, carried in the resolved manifest.** The builder reads each template from the very +// bytes it hashed into the archive's digest, so the text the mesh holds is the archive's. The controller +// registers the resolved manifest as the build result carries it and fetches nothing from the artifact store; a +// manifest that still names a template when it would be held — built by a builder that predates templates — is +// refused at registration (HeldTemplateProblem). + +// TemplateField is the key a file resource names its template file by. +const TemplateField = "template" + +// TemplatesDir is the module's directory its template files are kept in, and the directory its templates archive +// is built from. +const TemplatesDir = "files" + +// Templated is whether any of the manifest's resources names a template. +func (m Manifest) Templated() bool { + for _, r := range m.Resources { + if _, has := r[TemplateField]; has { + return true + } + } + return false +} + +// TemplatesArtifact is the name of the archive artifact the module packs its templates into — the one built from +// files/ — or "" when it declares none, or more than one (both refused by templateProblems for a module naming a +// template). +func TemplatesArtifact(m Manifest) string { + found := templatesArtifacts(m) + if len(found) != 1 { + return "" + } + return found[0] +} + +func templatesArtifacts(m Manifest) []string { + if m.Build == nil { + return nil + } + var found []string + for _, a := range m.Build.Artifacts { + if a.Kind == ArtifactArchive && a.URL == "" && cleanFrom(a.From) == TemplatesDir { + found = append(found, a.Name) + } + } + return found +} + +// cleanFrom is an artifact's directory as one relative path: "./files/" is "files". +func cleanFrom(from string) string { + c := path.Clean(strings.TrimSpace(from)) + return strings.TrimPrefix(c, "./") +} + +// templateOf is the template a resource names, whether it names one, and what is wrong with it ("" when nothing). +func templateOf(module string, r map[string]any) (string, bool, string) { + raw, has := r[TemplateField] + if !has { + return "", false, "" + } + id := fmt.Sprint(r["id"]) + if t := fmt.Sprint(r["type"]); t != "file" { + return "", true, fmt.Sprintf("%s: %s is a %s and names a template; only a file takes its text from one "+ + "(novox/hq ADR 0349 §2)", module, id, t) + } + name, ok := raw.(string) + if !ok || strings.TrimSpace(name) == "" { + return "", true, fmt.Sprintf("%s: %s names a template that is not a path: %v; a template is %q", + module, id, raw, TemplatesDir+"/") + } + if _, both := r["content"]; both { + return name, true, fmt.Sprintf("%s: %s names the template %q and says its content too; a file's text is "+ + "in one place, the template or the content, never both (novox/hq ADR 0349 §2)", module, id, name) + } + if strings.HasPrefix(name, "/") || path.Clean(name) != name || !strings.HasPrefix(name, TemplatesDir+"/") || + strings.Contains(name, "\\") { + return name, true, fmt.Sprintf("%s: %s names the template %q, which is not a file under %s/ of its own "+ + "repository: a template is %q, a plain relative path, and the build packs only %s/", + module, id, name, TemplatesDir, TemplatesDir+"/", TemplatesDir) + } + return name, true, "" +} + +// templateProblems is what is wrong with the manifest's templates, as written: a template on something not a file, +// beside content, outside files/, or a module naming templates whose build packs no archive from files/. Whether +// each template exists is judged where a tree is (ReadTemplates) or the archive's text is (Resolve). +func templateProblems(m Manifest) []string { + var problems []string + templated := false + for _, r := range m.Resources { + _, has, wrong := templateOf(m.Module, r) + templated = templated || has + if wrong != "" { + problems = append(problems, wrong) + } + } + if !templated { + return problems + } + switch found := templatesArtifacts(m); len(found) { + case 0: + problems = append(problems, fmt.Sprintf("%s names templates and its build packs no archive from %s/: "+ + "declare an artifact {\"name\": \"files\", \"kind\": \"archive\", \"from\": %q} (novox/hq ADR 0349 §2)", + m.Module, TemplatesDir, TemplatesDir)) + case 1: + default: + sort.Strings(found) + problems = append(problems, fmt.Sprintf("%s packs %s/ into %s; its templates are read from one archive", + m.Module, TemplatesDir, strings.Join(found, " and "))) + } + return problems +} + +// fillTemplates is resources with each template's text as its content and the template key dropped: text says the +// text of a template by its path ("files/"), and whether it has one. A template it has no text for, or text +// that is not UTF-8, is refused by name. Every resource is copied; the manifest's own are left as they were. +func fillTemplates(module string, resources []map[string]any, text func(string) (string, bool), + where string) ([]map[string]any, error) { + out := make([]map[string]any, 0, len(resources)) + var problems []string + for _, r := range resources { + name, has, wrong := templateOf(module, r) + if !has { + out = append(out, r) + continue + } + if wrong != "" { + problems = append(problems, wrong) + continue + } + body, ok := text(name) + if !ok { + problems = append(problems, fmt.Sprintf("%s: %v names the template %q, and %s holds no such file", + module, r["id"], name, where)) + continue + } + if !utf8.ValidString(body) { + // A manifest is JSON: text that is not UTF-8 would be changed on the way, silently. + problems = append(problems, fmt.Sprintf("%s: %v names the template %q, which is not UTF-8 text; a "+ + "file's text is carried as JSON and would not reach the machine as written", module, r["id"], name)) + continue + } + filled := make(map[string]any, len(r)) + for k, v := range r { + filled[k] = v + } + delete(filled, TemplateField) + filled["content"] = body + out = append(out, filled) + } + if len(problems) > 0 { + return nil, errors.New(strings.Join(problems, "; ")) + } + return out, nil +} + +// ReadTemplates is the text of each template the manifest names, read from the module's directory in a repository +// tree, by template path. A missing template, or one that is not a plain file (a symlink is refused by the build's +// packing too), is refused by name. +func ReadTemplates(m Manifest, dir string) (map[string]string, error) { + texts := map[string]string{} + var problems []string + for _, r := range m.Resources { + name, has, wrong := templateOf(m.Module, r) + if !has { + continue + } + if wrong != "" { + problems = append(problems, wrong) + continue + } + if _, read := texts[name]; read { + continue + } + full := filepath.Join(dir, filepath.FromSlash(name)) + info, err := os.Lstat(full) + switch { + case errors.Is(err, fs.ErrNotExist): + problems = append(problems, fmt.Sprintf("%s: %v names the template %q, and the module's directory "+ + "has no such file", m.Module, r["id"], name)) + continue + case err != nil: + problems = append(problems, fmt.Sprintf("%s: %v names the template %q, which cannot be read: %v", + m.Module, r["id"], name, err)) + continue + case !info.Mode().IsRegular(): + problems = append(problems, fmt.Sprintf("%s: %v names the template %q, which is not a plain file; "+ + "the build packs only files", m.Module, r["id"], name)) + continue + } + body, err := os.ReadFile(full) + if err != nil { + problems = append(problems, fmt.Sprintf("%s: %v names the template %q, which cannot be read: %v", + m.Module, r["id"], name, err)) + continue + } + texts[name] = string(body) + } + if len(problems) > 0 { + return nil, errors.New(strings.Join(problems, "; ")) + } + return texts, nil +} + +// WithTemplatesFrom is the manifest with each template's text read from the module's directory in a repository +// tree as its content: what `module check`, the merge gate and `module add` judge, where nothing was built. Read +// again strictly afterwards, so every rule about a file's text judges the template's text exactly as it judges +// inline content. A manifest naming no template is returned as it is. +func WithTemplatesFrom(m Manifest, dir string) (Manifest, error) { + if !m.Templated() { + return m, nil + } + texts, err := ReadTemplates(m, dir) + if err != nil { + return Manifest{}, err + } + filled, err := fillTemplates(m.Module, m.Resources, func(name string) (string, bool) { + t, ok := texts[name] + return t, ok + }, "the module's directory") + if err != nil { + return Manifest{}, err + } + out := m + out.Resources = filled + raw, err := json.Marshal(out) + if err != nil { + return Manifest{}, err + } + return ParseManifest(raw) +} + +// HeldTemplateProblem is why a manifest cannot be held by the mesh because it still names a template, or "": the +// manifest the mesh holds carries every file's text, which a build reads back from its templates archive. One +// naming a template was resolved by a builder that does not read templates, and would compose files with no text. +func HeldTemplateProblem(m Manifest) string { + var named []string + for _, r := range m.Resources { + if t, has := r[TemplateField]; has { + named = append(named, fmt.Sprintf("%v (%v)", r["id"], t)) + } + } + if len(named) == 0 { + return "" + } + return fmt.Sprintf("%s still names the template(s) of %s: the manifest the mesh holds carries each file's "+ + "text, read back from the build's templates archive — this one was resolved by a builder that does not "+ + "read templates, or not built at all (novox/hq ADR 0349 §2)", m.Module, strings.Join(named, ", ")) +} -- 2.54.0 From f45e0a1de50828323532084196696a25a16da4f9 Mon Sep 17 00:00:00 2001 From: mesh-admin Date: Tue, 13 Oct 2026 23:59:22 +0000 Subject: [PATCH 05/31] Let a file resource name a template file in its module's repository: composition, check and gate tests Co-Authored-By: Claude Opus 5.5 --- cmd/mesh-controller/template_test.go | 177 ++++++++++++++++++ internal/catalogue/template_test.go | 258 +++++++++++++++++++++++++++ 2 files changed, 435 insertions(+) create mode 100644 cmd/mesh-controller/template_test.go create mode 100644 internal/catalogue/template_test.go diff --git a/cmd/mesh-controller/template_test.go b/cmd/mesh-controller/template_test.go new file mode 100644 index 00000000..60ca0e74 --- /dev/null +++ b/cmd/mesh-controller/template_test.go @@ -0,0 +1,177 @@ +package main + +import ( + "bytes" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" + snapshot "github.com/novox/mesh-controller/internal/facts" + "github.com/novox/mesh-controller/internal/inventory" +) + +// A file's text kept as a template file in its module's repository (novox/hq ADR 0349 §2, design 57 package T1, +// issue 654): `module check` and the merge gate read it from the tree beside the manifest, so a change to a +// template alone is judged as a change to its module. + +const templatedNotifier = `{"module":"notifier","version":"1", + "build":{"artifacts":[{"name":"files","kind":"archive","from":"files"}]}, + "resources":[{"id":"conf","type":"file","path":"/etc/notifier/notifier.conf","mode":"0644","template":"files/notifier.conf"}]}` + +// inlineNotifier is the same module as the mesh holds it: built, its text inline. +const inlineNotifier = `{"module":"notifier","version":"1", + "resources":[{"id":"conf","type":"file","path":"/etc/notifier/notifier.conf","mode":"0644","content":"volume = 3\n"}]}` + +// aTemplatedModule writes a module's directory: its manifest and its files/. +func aTemplatedModule(t *testing.T, dir, manifest string, files map[string]string) string { + t.Helper() + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + path := filepath.Join(dir, "module.json") + if err := os.WriteFile(path, []byte(manifest), 0o644); err != nil { + t.Fatal(err) + } + for name, text := range files { + full := filepath.Join(dir, filepath.FromSlash(name)) + if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(full, []byte(text), 0o644); err != nil { + t.Fatal(err) + } + } + return path +} + +func TestModuleCheckReadsTemplatesFromTheTreeBesideTheManifest(t *testing.T) { + good := aTemplatedModule(t, t.TempDir(), templatedNotifier, map[string]string{"files/notifier.conf": "volume = 3\n"}) + var out bytes.Buffer + if err := moduleCheck([]string{good}, &out); err != nil || !strings.Contains(out.String(), "notifier: ok") { + t.Fatalf("a module with its text in files/ was refused: %v\n%s", err, out.String()) + } + + // A change to the template alone, to text the controller refuses inline, is refused the same. + bad := aTemplatedModule(t, t.TempDir(), templatedNotifier, + map[string]string{"files/notifier.conf": "volume = 3\nstore = ${dir:nowhere}\n"}) + out.Reset() + if err := moduleCheck([]string{bad}, &out); err == nil || !strings.Contains(out.String(), "nowhere") { + t.Fatalf("a template whose text is refused inline passed:\n%s", out.String()) + } + + // And a template the tree lacks, by name. + missing := aTemplatedModule(t, t.TempDir(), templatedNotifier, nil) + out.Reset() + if err := moduleCheck([]string{missing}, &out); err == nil || !strings.Contains(out.String(), "files/notifier.conf") { + t.Fatalf("a missing template was not refused by name:\n%s", out.String()) + } +} + +// templatedMesh is catalogueMesh with the notifier on the laptop, registered as built: its text inline. +func templatedMesh(t *testing.T) (snapshot.Facts, map[string]string) { + t.Helper() + open := aMesh(t) + ctx := t.Context() + manifests := map[string]string{ + "lemurs": `{"module":"lemurs","version":"1","capabilities":["systemd"], + "resources":[{"id":"service","type":"service","unit":"lemurs.service","state":"running","boot":"enabled"}]}`, + "notifier": inlineNotifier, + } + for name, raw := range manifests { + m, err := catalogue.ParseManifest([]byte(raw)) + if err != nil { + t.Fatalf("%s: %v", name, err) + } + if err := open.inventory.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-catalog", + Path: "modules/" + name, BuiltFrom: "c0ffee"}); err != nil { + t.Fatal(err) + } + if _, err := assign(ctx, open, "laptop", name); err != nil { + t.Fatalf("assign %s: %v", name, err) + } + } + f, err := gatherFacts(ctx, open, "2.11.17") + if err != nil { + t.Fatal(err) + } + return f, manifests +} + +// aTemplatedTree is the catalogue's checkout with the notifier's text in files/. +func aTemplatedTree(t *testing.T, manifests map[string]string, text string, has bool) string { + t.Helper() + tree := aTree(t, withEdit(manifests, "notifier", "")) + files := map[string]string{} + if has { + files["files/notifier.conf"] = text + } + aTemplatedModule(t, filepath.Join(tree, "modules", "notifier"), templatedNotifier, files) + return tree +} + +const notifierTemplate = "modules/notifier/files/notifier.conf" + +func TestTheMergeGateJudgesAChangeToATemplateAlone(t *testing.T) { + f, manifests := templatedMesh(t) + + // The module moved to its template, the same text: nothing changes on any machine. + v := gateJudged(t, f, aTemplatedTree(t, manifests, "volume = 3\n", true), "modules/notifier/module.json", + notifierTemplate) + if v.Verdict == "fail" || v.Modules["notifier"] != "" { + t.Fatalf("the module moved to its template with the same text reads as a change (%v):\n%s", v.Modules, v.Report()) + } + + // The template alone changed: its module changed, and composes. + v = gateJudged(t, f, aTemplatedTree(t, manifests, "volume = 4\n", true), notifierTemplate) + if v.Verdict == "fail" || v.Modules["notifier"] != "changed" { + t.Fatalf("a change to the template alone is not a change to its module (%v):\n%s", v.Modules, v.Report()) + } + if v.Width == nil || strings.Join(v.Width.Modules, ",") != "notifier" { + t.Fatalf("a change to the template rebuilds %+v, not its module", v.Width) + } + + // The template alone changed to text the controller refuses: the pull request fails, naming the module. + v = gateJudged(t, f, aTemplatedTree(t, manifests, "store = ${dir:nowhere}\n", true), notifierTemplate) + if v.Verdict != "fail" || !strings.Contains(v.Report(), "notifier") { + t.Fatalf("a template the controller refuses passed the gate:\n%s", v.Report()) + } + + // And the template removed from the tree fails, naming it. + v = gateJudged(t, f, aTemplatedTree(t, manifests, "", false), notifierTemplate) + if v.Verdict != "fail" || !strings.Contains(v.Report(), "files/notifier.conf") { + t.Fatalf("a missing template passed the gate:\n%s", v.Report()) + } +} + +// What the gate composes for the module in its template form is what the mesh composes for it inline. +func TestTheMergeGateResolvesATemplatedModuleToItsInlineTwin(t *testing.T) { + dir := t.TempDir() + path := aTemplatedModule(t, dir, templatedNotifier, map[string]string{"files/notifier.conf": "volume = 3\n"}) + raw, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + m, err := catalogue.ParseManifest(raw) + if err != nil { + t.Fatal(err) + } + m, err = catalogue.WithTemplatesFrom(m, dir) + if err != nil { + t.Fatal(err) + } + resolved, err := standInBuild(m) + if err != nil { + t.Fatal(err) + } + inline, err := catalogue.ParseManifest([]byte(inlineNotifier)) + if err != nil { + t.Fatal(err) + } + if builtAlike(resolved) != builtAlike(inline) { + got, _ := json.Marshal(resolved) + t.Fatalf("the templated module resolves to %s, not its inline twin", got) + } +} diff --git a/internal/catalogue/template_test.go b/internal/catalogue/template_test.go new file mode 100644 index 00000000..85c81896 --- /dev/null +++ b/internal/catalogue/template_test.go @@ -0,0 +1,258 @@ +package catalogue + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" +) + +// A file resource may name a template file in its module's repository (novox/hq ADR 0349 §2, design 57 package +// T1, issue 654): the build packs files/ into an archive, and resolving the build reads each template's text back +// into content, so the mesh holds, composes and sends exactly what it would for the same text inline. + +// The two forms of one module: its text inline, and its text in files/. +const ( + inlineTwin = `{"module":"tidy","version":"1", + "build":{"artifacts":[{"name":"files","kind":"archive","from":"files"}]}, + "resources":[ + {"id":"conf","type":"directory","path":"${dir:conf}"}, + {"id":"main","type":"file","path":"${dir:conf}/tidy.conf","mode":"0644","content":"` + mainText + `"}, + {"id":"extra","type":"file","path":"${dir:conf}/extra.conf","content":"` + extraText + `"}]}` + templatedTwin = `{"module":"tidy","version":"1", + "build":{"artifacts":[{"name":"files","kind":"archive","from":"files"}]}, + "resources":[ + {"id":"conf","type":"directory","path":"${dir:conf}"}, + {"id":"main","type":"file","path":"${dir:conf}/tidy.conf","mode":"0644","template":"files/tidy.conf"}, + {"id":"extra","type":"file","path":"${dir:conf}/extra.conf","template":"files/deeper/extra.conf"}]}` + mainText = `# tidy\nroot = ${dir:conf}\nquoted = \"yes\" é\n` + extraText = `one\ntwo\n` +) + +// templateTexts is what a build reads back from the packed files/ archive: by path inside it. +func templateTexts(t *testing.T) map[string]string { + t.Helper() + var m struct{ Main, Extra string } + if err := json.Unmarshal([]byte(`{"Main":"`+mainText+`","Extra":"`+extraText+`"}`), &m); err != nil { + t.Fatal(err) + } + return map[string]string{"tidy.conf": m.Main, "deeper/extra.conf": m.Extra, "unused.conf": "not named"} +} + +func builtFiles(texts map[string]string) []Built { + return []Built{{Name: "files", Kind: ArtifactArchive, + Reference: ArtifactStoreScheme + "tidy/files/blobs/sha256:" + strings.Repeat("c", 64), + Digest: "sha256:" + strings.Repeat("c", 64), Texts: texts}} +} + +// held is a manifest as registration holds it: resolved with what was built, then read again strictly. +func held(t *testing.T, raw string, built []Built) Manifest { + t.Helper() + m, err := ParseManifest([]byte(raw)) + if err != nil { + t.Fatal(err) + } + resolved, err := m.Resolve(built) + if err != nil { + t.Fatal(err) + } + again, err := json.Marshal(resolved) + if err != nil { + t.Fatal(err) + } + out, err := ParseManifest(again) + if err != nil { + t.Fatal(err) + } + if problem := HeldTemplateProblem(out); problem != "" { + t.Fatal(problem) + } + return out +} + +func composed(t *testing.T, m Manifest) string { + t.Helper() + got, err := Resolve(shelf(m), []string{m.Module}, workstation(), World{}) + if err != nil { + t.Fatal(err) + } + out, err := json.Marshal(mustDeclare(t, got)) + if err != nil { + t.Fatal(err) + } + return string(out) +} + +func TestATemplatedModuleComposesByteIdenticallyToItsInlineTwin(t *testing.T) { + inline := composed(t, held(t, inlineTwin, builtFiles(nil))) + templated := composed(t, held(t, templatedTwin, builtFiles(templateTexts(t)))) + if inline != templated { + t.Fatalf("the two forms compose differently:\ninline: %s\ntemplated: %s", inline, templated) + } + if !strings.Contains(templated, "quoted = \\\"yes\\\"") || strings.Contains(templated, `"template"`) { + t.Fatalf("the composition does not carry the template's text as content: %s", templated) + } +} + +// What `module check`, the merge gate and `module add` judge: the text read from the tree beside the manifest. +func TestATemplatedModuleReadFromItsTreeComposesByteIdenticallyToItsInlineTwin(t *testing.T) { + dir := t.TempDir() + for name, text := range templateTexts(t) { + full := filepath.Join(dir, "files", filepath.FromSlash(name)) + if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(full, []byte(text), 0o644); err != nil { + t.Fatal(err) + } + } + m, err := ParseManifest([]byte(templatedTwin)) + if err != nil { + t.Fatal(err) + } + filled, err := WithTemplatesFrom(m, dir) + if err != nil { + t.Fatal(err) + } + if filled.Templated() { + t.Fatal("a manifest read from its tree still names a template") + } + resolved, err := filled.Resolve(builtFiles(nil)) + if err != nil { + t.Fatal(err) + } + if got, want := composed(t, resolved), composed(t, held(t, inlineTwin, builtFiles(nil))); got != want { + t.Fatalf("read from the tree, the module composes differently:\ninline: %s\ntemplated: %s", want, got) + } +} + +func TestAFileNamingBothATemplateAndContentIsRefused(t *testing.T) { + raw := strings.Replace(templatedTwin, `"template":"files/tidy.conf"`, + `"template":"files/tidy.conf","content":"x"`, 1) + _, err := ParseManifest([]byte(raw)) + if err == nil || !strings.Contains(err.Error(), "main") || !strings.Contains(err.Error(), "never both") { + t.Fatalf("a file with both a template and content was not refused by name: %v", err) + } +} + +func TestATemplateOutsideFilesIsRefusedByName(t *testing.T) { + for _, bad := range []string{"../tidy.conf", "/etc/tidy.conf", "files", "files/", "files/../tidy.conf", + "files/./tidy.conf", "other/tidy.conf", "tidy.conf", "files//tidy.conf", `files\\tidy.conf`} { + raw := strings.Replace(templatedTwin, `"files/tidy.conf"`, `"`+bad+`"`, 1) + _, err := ParseManifest([]byte(raw)) + if err == nil || !strings.Contains(err.Error(), "main") || !strings.Contains(err.Error(), "not a file under files/") { + t.Errorf("the template %q was not refused by name: %v", bad, err) + } + } +} + +func TestATemplateOnAResourceThatIsNotAFileIsRefused(t *testing.T) { + raw := strings.Replace(templatedTwin, `{"id":"conf","type":"directory","path":"${dir:conf}"}`, + `{"id":"conf","type":"directory","path":"${dir:conf}","template":"files/tidy.conf"}`, 1) + _, err := ParseManifest([]byte(raw)) + if err == nil || !strings.Contains(err.Error(), "conf is a directory and names a template") { + t.Fatalf("a directory naming a template was not refused: %v", err) + } +} + +func TestATemplatedModuleWhoseBuildPacksNoFilesIsRefused(t *testing.T) { + for _, build := range []string{ + ``, + `"build":{"artifacts":[{"name":"look","kind":"archive","from":"theme"}]},`, + `"build":{"artifacts":[{"name":"a","kind":"archive","from":"files"},{"name":"b","kind":"archive","from":"./files/"}]},`, + } { + raw := strings.Replace(templatedTwin, + `"build":{"artifacts":[{"name":"files","kind":"archive","from":"files"}]},`, build, 1) + _, err := ParseManifest([]byte(raw)) + if err == nil || !strings.Contains(err.Error(), "files/") { + t.Errorf("a templated module with the build %q was not refused: %v", build, err) + } + } +} + +func TestAMissingTemplateIsRefusedByName(t *testing.T) { + m, err := ParseManifest([]byte(templatedTwin)) + if err != nil { + t.Fatal(err) + } + // In the tree beside the manifest. + dir := t.TempDir() + if err := os.MkdirAll(filepath.Join(dir, "files"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "files", "tidy.conf"), []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + _, err = WithTemplatesFrom(m, dir) + if err == nil || !strings.Contains(err.Error(), "extra") || !strings.Contains(err.Error(), "files/deeper/extra.conf") { + t.Fatalf("a template missing from the tree was not refused by name: %v", err) + } + // And in what the build read back from its archive. + texts := templateTexts(t) + delete(texts, "deeper/extra.conf") + _, err = m.Resolve(builtFiles(texts)) + if err == nil || !strings.Contains(err.Error(), "extra") || !strings.Contains(err.Error(), "files/deeper/extra.conf") { + t.Fatalf("a template missing from the archive was not refused by name: %v", err) + } +} + +func TestATemplateThatIsASymlinkIsRefused(t *testing.T) { + m, err := ParseManifest([]byte(templatedTwin)) + if err != nil { + t.Fatal(err) + } + dir := t.TempDir() + if err := os.MkdirAll(filepath.Join(dir, "files", "deeper"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "elsewhere"), []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "files", "tidy.conf"), []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.Symlink(filepath.Join(dir, "elsewhere"), filepath.Join(dir, "files", "deeper", "extra.conf")); err != nil { + t.Fatal(err) + } + if _, err := WithTemplatesFrom(m, dir); err == nil || !strings.Contains(err.Error(), "not a plain file") { + t.Fatalf("a template that is a symlink was not refused: %v", err) + } +} + +func TestAResolveWithoutTheArchivesTextIsRefused(t *testing.T) { + m, err := ParseManifest([]byte(templatedTwin)) + if err != nil { + t.Fatal(err) + } + if _, err := m.Resolve(builtFiles(nil)); err == nil || !strings.Contains(err.Error(), "files") { + t.Fatalf("a templated module resolved with no text read back from its archive: %v", err) + } +} + +func TestATemplateThatIsNotUTF8IsRefused(t *testing.T) { + m, err := ParseManifest([]byte(templatedTwin)) + if err != nil { + t.Fatal(err) + } + texts := templateTexts(t) + texts["tidy.conf"] = "\xff\xfe" + if _, err := m.Resolve(builtFiles(texts)); err == nil || !strings.Contains(err.Error(), "UTF-8") { + t.Fatalf("a template that is not UTF-8 was resolved: %v", err) + } +} + +// What registration refuses: a manifest to be held that still names a template — one resolved by a builder that +// predates templates would otherwise compose files with no text. +func TestAManifestStillNamingATemplateIsNotHeld(t *testing.T) { + m, err := ParseManifest([]byte(templatedTwin)) + if err != nil { + t.Fatal(err) + } + if problem := HeldTemplateProblem(m); !strings.Contains(problem, "main") || !strings.Contains(problem, "files/tidy.conf") { + t.Fatalf("a manifest naming a template would be held: %q", problem) + } + if problem := HeldTemplateProblem(held(t, inlineTwin, builtFiles(nil))); problem != "" { + t.Fatalf("an inline manifest is refused: %s", problem) + } +} -- 2.54.0 From 7530c2aeaccd793342524776bfee4b3fa2e54bbf Mon Sep 17 00:00:00 2001 From: mesh-admin Date: Wed, 14 Oct 2026 00:41:06 +0000 Subject: [PATCH 06/31] Read placement from a layer's Placement, never from its config values (issue 651) ADR 0348 clause 11: places, accesses and kept networks are placement, not config. Today's mergeable files still take them as said, so nothing composes differently. Co-Authored-By: Claude Opus 5.5 --- internal/catalogue/placement.go | 67 +++++++++++++++++++++++++- internal/catalogue/setting_defaults.go | 3 +- internal/catalogue/settings.go | 16 ++++-- 3 files changed, 80 insertions(+), 6 deletions(-) diff --git a/internal/catalogue/placement.go b/internal/catalogue/placement.go index 400c5b3f..bc8bf03d 100644 --- a/internal/catalogue/placement.go +++ b/internal/catalogue/placement.go @@ -34,6 +34,69 @@ const PlacesSetting = "places" // AccessesSetting is the settings key that says where a module's accesses are on this node, by id. const AccessesSetting = "accesses" +// **Placement is not config** (novox/hq ADR 0348 clause 11, issue 651). `places`, `accesses` and +// `networks` are facts of an assignment the controller decides, held beside the assignment and apart +// from its settings: a Layer carries them in Placement, never in Values, and only Places, AccessPlaces +// and KeptNetworks read them. A person still says them as keys of a layer through `settings`; LayerOf +// splits such a layer, so what they type is routed to the placement and nothing they do changes. + +// placementKeys are the keys of a layer that are placement, not config. +var placementKeys = map[string]bool{PlacesSetting: true, AccessesSetting: true, NetworksSetting: true} + +// SplitPlacement divides a layer as a person says it into its config values and its placement. Each is +// nil when it holds nothing; the values themselves are not copied. +func SplitPlacement(values map[string]any) (config, placement map[string]any) { + for key, value := range values { + if placementKeys[key] { + if placement == nil { + placement = map[string]any{} + } + placement[key] = value + continue + } + if config == nil { + config = map[string]any{} + } + config[key] = value + } + return config, placement +} + +// LayerOf is a layer as a person says it — config values and placement in one map — split into the two. +func LayerOf(from string, values map[string]any) Layer { + config, placement := SplitPlacement(values) + if config == nil { + config = map[string]any{} + } + return Layer{From: from, Values: config, Placement: placement} +} + +// Whole is the layer as a person says it: its values and its placement in one map, a new map. +func (l Layer) Whole() map[string]any { + out := make(map[string]any, len(l.Values)+len(l.Placement)) + for k, v := range l.Values { + out[k] = v + } + for k, v := range l.Placement { + out[k] = v + } + return out +} + +// asSaid is the layers with placement folded back into the values, for what reads a layer whole as it +// did while placement was kept in it: today's mergeable file, which takes every key (ADR 0326 §3), and +// the judgement that no value holds a line end. +func asSaid(layers []Layer) []Layer { + out := make([]Layer, len(layers)) + for i, l := range layers { + out[i] = l + if len(l.Placement) > 0 { + out[i] = Layer{From: l.From, Values: l.Whole(), Default: l.Default} + } + } + return out +} + // Placement is what an assignment says about one of a module's directories. type Placement struct { // Path is where the directory is on this machine. Absolute. @@ -208,7 +271,7 @@ func Places(m Manifest, layers []Layer) (map[string]Placement, error) { } out := map[string]Placement{} for _, layer := range layers { - raw, ok := layer.Values[PlacesSetting] + raw, ok := layer.Placement[PlacesSetting] if !ok { continue } @@ -278,7 +341,7 @@ func AccessPlaces(m Manifest, layers []Layer) (map[string]string, error) { } out := map[string]string{} for _, layer := range layers { - raw, ok := layer.Values[AccessesSetting] + raw, ok := layer.Placement[AccessesSetting] if !ok { continue } diff --git a/internal/catalogue/setting_defaults.go b/internal/catalogue/setting_defaults.go index 1bc72f67..3f158e02 100644 --- a/internal/catalogue/setting_defaults.go +++ b/internal/catalogue/setting_defaults.go @@ -207,7 +207,8 @@ func Effective(m Manifest, layers []Layer) []SettingSource { Default: d.Default, HasDefault: true} } for _, layer := range layers { - for key, v := range layer.Values { + // As the layer is said, placement too: `settings` answers it as a person set it. + for key, v := range layer.Whole() { s, ok := byKey[key] if !ok { s = &SettingSource{Key: key} diff --git a/internal/catalogue/settings.go b/internal/catalogue/settings.go index e20b61a2..45c992e8 100644 --- a/internal/catalogue/settings.go +++ b/internal/catalogue/settings.go @@ -37,6 +37,11 @@ type Layer struct { // Default marks the layer a module's own defaults make (novox/hq ADR 0262). Marked rather than // recognised by From, which is a node's name for a node's layer, and a node may be called anything. Default bool + // Placement is the assignment's placement at this layer's scope — `places`, `accesses`, `networks` + // — which the controller holds apart from settings (novox/hq ADR 0348 clause 11, issue 651): read + // by Places, AccessPlaces and KeptNetworks, and never from Values. LayerOf splits a layer as a person + // says it into the two. + Placement map[string]any } // ApplySettings produces a resource's final content from the module's own and the layers over it. @@ -45,6 +50,10 @@ type Layer struct { // module first, then this node's. A node that differs from the rest is expressed by differing, // rather than by repeating everything the rest already say. func ApplySettings(resource map[string]any, layers []Layer) (map[string]any, error) { + // Today's form reaches a mergeable file with every key of the layer as it was said, placement too + // (novox/hq ADR 0326 §3, ADR 0348 clause 11: a module not yet moved keeps it), so a file composes + // byte for byte as it did while placement was kept in the layer. + layers = asSaid(layers) how, _ := resource["merge"].(string) taken, names := takenKeys(resource) if how == "" { @@ -395,7 +404,7 @@ func UnusedSettings(m Manifest, layers []Layer) []string { } var unused []string for _, layer := range layers { - for key := range layer.Values { + for key := range layer.Whole() { if lands[key] { continue } @@ -483,7 +492,7 @@ func KeptNetworks(m Manifest, layers []Layer, adopted bool) (map[string][]string } out := map[string][]string{} for _, layer := range layers { - raw, ok := layer.Values[NetworksSetting] + raw, ok := layer.Placement[NetworksSetting] if !ok { continue } @@ -543,7 +552,8 @@ var networkName = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_.-]*$`) // refused where it is stored (SetSettings, with UnusedSettings) and said where a plan is read, // and never costs a module its place. func JudgeSettings(m Manifest, layers []Layer, adopted bool) error { - if err := settingsHoldOneLine(m.Module, layers); err != nil { + // Placement is judged for line ends as it always was: a path is written into files as well. + if err := settingsHoldOneLine(m.Module, asSaid(layers)); err != nil { return err } // Which keys each mergeable file takes is the definition's to say right (novox/hq ADR 0326); -- 2.54.0 From 02b542a0592eaadbeb3c123858d1846dc692c744 Mon Sep 17 00:00:00 2001 From: mesh-admin Date: Wed, 14 Oct 2026 00:42:37 +0000 Subject: [PATCH 07/31] Keep a layer's placement beside it, and say the layer whole (issue 651) The settings verb's path splits a layer it is given into its config values and its placement, so what a person types still works; a layer reads back and goes into the history whole. Co-Authored-By: Claude Opus 5.5 --- internal/inventory/catalogue.go | 97 ++++++++++++++++++++++++++------- internal/inventory/unseen.go | 28 +++++++--- 2 files changed, 99 insertions(+), 26 deletions(-) diff --git a/internal/inventory/catalogue.go b/internal/inventory/catalogue.go index 6ce3740c..31cc597f 100644 --- a/internal/inventory/catalogue.go +++ b/internal/inventory/catalogue.go @@ -839,7 +839,18 @@ func (i *Inventory) KeepSettings(ctx context.Context, nodeName, module string, v } func (i *Inventory) setSettings(ctx context.Context, nodeName, module string, values map[string]any, judge bool, setBy string) error { - raw, err := json.Marshal(values) + // **Placement is not config** (novox/hq ADR 0348 clause 11, issue 651): the layer is said whole, as + // it always was, and its placement keys are kept beside the assignment, never in the settings row. + // Both are written in one transaction, and the layer a set replaces is the whole of both. + config, placement := catalogue.SplitPlacement(values) + if config == nil { + config = map[string]any{} + } + raw, err := json.Marshal(config) + if err != nil { + return err + } + placed, err := placementJSON(placement) if err != nil { return err } @@ -881,6 +892,9 @@ func (i *Inventory) setSettings(ctx context.Context, nodeName, module string, va do update set values = excluded.values, set_at = now(), set_by = excluded.set_by`, module, raw, nullable(setBy)); err != nil { return wrapModule(err, module) } + if err := keepPlacement(ctx, tx, nil, module, placed, setBy); err != nil { + return err + } return tx.Commit(ctx) } node, err := i.NodeByName(ctx, nodeName) @@ -912,6 +926,9 @@ func (i *Inventory) setSettings(ctx context.Context, nodeName, module string, va if err != nil { return wrapModule(err, module) } + if err := keepPlacement(ctx, tx, &node.ID, module, placed, setBy); err != nil { + return err + } // A given port replaces what the mesh assigned for that port: the assignment is given back, // so the number is free for the next module rather than held for ever in the name of a port // that now lives elsewhere. @@ -942,21 +959,15 @@ func (i *Inventory) judgeSettings(ctx context.Context, nodeName, module string, return err } where, from, adopted = nodeName, nodeName, node.Adopted - var meshWide []byte - err = i.store.Pool().QueryRow(ctx, - `select values from settings where module = $1 and node is null`, module).Scan(&meshWide) - if err != nil && !errors.Is(err, pgx.ErrNoRows) { + under, has, err := i.Layer(ctx, "", module) + if err != nil { return err } - if len(meshWide) > 0 { - var under map[string]any - if err := json.Unmarshal(meshWide, &under); err != nil { - return err - } - layers = append(layers, catalogue.Layer{From: catalogue.MeshWideLayer, Values: under}) + if has { + layers = append(layers, catalogue.LayerOf(catalogue.MeshWideLayer, under)) } } - layers = append(layers, catalogue.Layer{From: from, Values: values}) + layers = append(layers, catalogue.LayerOf(from, values)) if err := catalogue.JudgeSettings(m, layers, adopted); err != nil { return fmt.Errorf("refused: %s on %s cannot compose with the layer %q — %w", module, where, from, err) } @@ -1089,7 +1100,8 @@ func refuseGivenCollisions(ctx context.Context, tx pgx.Tx, nodeID any, node, mod } func wrapModule(err error, module string) error { - if err != nil && strings.Contains(err.Error(), "settings_module_fkey") { + if err != nil && (strings.Contains(err.Error(), "settings_module_fkey") || + strings.Contains(err.Error(), "placement_module_fkey")) { return fmt.Errorf("%w: %s", ErrNoSuchModule, module) } return err @@ -1130,6 +1142,11 @@ func (i *Inventory) ClearSettingsBy(ctx context.Context, nodeName, module, clear `delete from settings where module = $1 and node is not distinct from $2::uuid`, module, nodeID); err != nil { return err } + // And its placement, which is the same layer as a person says it (novox/hq ADR 0348 clause 11). + if _, err := tx.Exec(ctx, + `delete from placement where module = $1 and node is not distinct from $2::uuid`, module, nodeID); err != nil { + return err + } return tx.Commit(ctx) } @@ -1142,10 +1159,13 @@ func (i *Inventory) SettingsFor(ctx context.Context, nodeName, module string) ([ if err != nil { return nil, err } + // Each layer's placement beside it, from the placement the controller holds (novox/hq ADR 0348 + // clause 11): a layer's Values hold its config values only, its Placement where things are. rows, err := i.store.Pool().Query(ctx, - `select node is null, values from settings - where module = $1 and (node is null or node = $2) - order by node is null desc`, module, node.ID) + `select s.node is null, s.values, p.values from settings s + left join placement p on p.module = s.module and p.node is not distinct from s.node + where s.module = $1 and (s.node is null or s.node = $2) + order by s.node is null desc`, module, node.ID) if err != nil { return nil, err } @@ -1154,23 +1174,62 @@ func (i *Inventory) SettingsFor(ctx context.Context, nodeName, module string) ([ var layers []catalogue.Layer for rows.Next() { var meshWide bool - var raw []byte - if err := rows.Scan(&meshWide, &raw); err != nil { + var raw, placed []byte + if err := rows.Scan(&meshWide, &raw, &placed); err != nil { return nil, err } values := map[string]any{} if err := json.Unmarshal(raw, &values); err != nil { return nil, err } + var placement map[string]any + if len(placed) > 0 { + if err := json.Unmarshal(placed, &placement); err != nil { + return nil, err + } + } from := nodeName if meshWide { from = catalogue.MeshWideLayer } - layers = append(layers, catalogue.Layer{From: from, Values: values}) + layers = append(layers, catalogue.Layer{From: from, Values: values, Placement: placement}) } return layers, rows.Err() } +// placementJSON is a layer's placement as the placement table keeps it, or nil when it has none. +func placementJSON(placement map[string]any) ([]byte, error) { + if len(placement) == 0 { + return nil, nil + } + return json.Marshal(placement) +} + +// keepPlacement writes one layer's placement beside its settings row, in the write's transaction: the +// placement the layer says, or none when it says none — a set replaces the layer whole, placement too. +func keepPlacement(ctx context.Context, tx pgx.Tx, nodeID *string, module string, placed []byte, setBy string) error { + if placed == nil { + _, err := tx.Exec(ctx, + `delete from placement where module = $1 and node is not distinct from $2::uuid`, module, nodeID) + return err + } + var err error + if nodeID == nil { + _, err = tx.Exec(ctx, + `insert into placement (node, module, values, set_by) values (null, $1, $2, $3) + on conflict (module) where node is null + do update set values = excluded.values, set_at = now(), set_by = excluded.set_by`, + module, placed, nullable(setBy)) + } else { + _, err = tx.Exec(ctx, + `insert into placement (node, module, values, set_by) values ($1, $2, $3, $4) + on conflict (node, module) where node is not null + do update set values = excluded.values, set_at = now(), set_by = excluded.set_by`, + *nodeID, module, placed, nullable(setBy)) + } + return wrapModule(err, module) +} + // PinProvision records which provider a machine gets a provision from: a module, and the node it // runs on — both, always (novox/hq #258). A provision comes from a module, and the same module on // two machines is two answers, so neither half alone says which. diff --git a/internal/inventory/unseen.go b/internal/inventory/unseen.go index 8acbf13b..023fa315 100644 --- a/internal/inventory/unseen.go +++ b/internal/inventory/unseen.go @@ -20,9 +20,11 @@ func (i *Inventory) Layer(ctx context.Context, nodeName, module string) (map[str if err != nil { return nil, false, err } + // Said whole, as it was set: its config values and the placement kept beside them (novox/hq ADR 0348 + // clause 11), so a person reading it before setting it sees every key they may keep or remove. var raw []byte - err = i.store.Pool().QueryRow(ctx, - `select values from settings where module = $1 and node is not distinct from $2::uuid`, + err = i.store.Pool().QueryRow(ctx, `select `+layerWhole+` from settings s + where s.module = $1 and s.node is not distinct from $2::uuid`, module, nodeID).Scan(&raw) if errors.Is(err, pgx.ErrNoRows) { return nil, false, nil @@ -46,6 +48,9 @@ type PastLayer struct { ReplacedBy string // SetBy is who had set the layer, when it was kept (novox/hq ADR 0277); empty for one set before that was kept. SetBy string + // Why is why it went, where that was said: "moved to placement, ADR 0348 clause 11" for a layer whose + // placement keys migration 0097 moved out of it (ReplacedBy "moved"). + Why string } // LayerOrigin is who set a layer and when (novox/hq ADR 0277): empty words for a layer set before who set it was @@ -79,7 +84,7 @@ func (i *Inventory) SettingsHistory(ctx context.Context, nodeName, module string return nil, err } rows, err := i.store.Pool().Query(ctx, - `select values, set_at, replaced_at, replaced_by, set_by from settings_history + `select values, set_at, replaced_at, replaced_by, set_by, why from settings_history where module = $1 and node is not distinct from $2::uuid order by replaced_at desc`, module, nodeID) if err != nil { @@ -90,13 +95,16 @@ func (i *Inventory) SettingsHistory(ctx context.Context, nodeName, module string for rows.Next() { var raw []byte var p PastLayer - var by *string - if err := rows.Scan(&raw, &p.SetAt, &p.ReplacedAt, &p.ReplacedBy, &by); err != nil { + var by, why *string + if err := rows.Scan(&raw, &p.SetAt, &p.ReplacedAt, &p.ReplacedBy, &by, &why); err != nil { return nil, err } if by != nil { p.SetBy = *by } + if why != nil { + p.Why = *why + } if err := json.Unmarshal(raw, &p.Values); err != nil { return nil, err } @@ -107,9 +115,15 @@ func (i *Inventory) SettingsHistory(ctx context.Context, nodeName, module string // keepReplacedSQL copies a layer into the history before it is replaced or cleared; run in the same // transaction as the write where there is one, so a write that fails leaves no history of it. +// +// The layer is kept whole, as it was set: its config values and its placement (novox/hq ADR 0348 clause 11). const keepReplacedSQL = `insert into settings_history (node, module, values, set_at, replaced_by, set_by) - select node, module, values, set_at, $3, set_by from settings - where module = $1 and node is not distinct from $2::uuid` + select s.node, s.module, ` + layerWhole + `, s.set_at, $3, s.set_by from settings s + where s.module = $1 and s.node is not distinct from $2::uuid` + +// layerWhole is a settings row `s` with the placement kept beside it folded back in: the layer as it was set. +const layerWhole = `s.values || coalesce((select p.values from placement p + where p.module = s.module and p.node is not distinct from s.node), '{}'::jsonb)` // layerNode is the node id of a layer, nil for the whole mesh's. func (i *Inventory) layerNode(ctx context.Context, nodeName string) (*string, error) { -- 2.54.0 From 85b85a3fa990e85c6703016c679d1ea2b02bf6d8 Mon Sep 17 00:00:00 2001 From: mesh-admin Date: Wed, 14 Oct 2026 00:43:40 +0000 Subject: [PATCH 08/31] Say a layer whole where the mesh reports it, and give tests their placement (issue 651) The facts snapshot, the take preview and the settings history read a layer as it was set; tests that place give the layer through LayerOf. Co-Authored-By: Claude Opus 5.5 --- cmd/mesh-controller/adoption.go | 5 +++-- cmd/mesh-controller/facts.go | 4 ++-- cmd/mesh-controller/mesh_for_test.go | 16 +++++++++++++++- cmd/mesh-controller/modules.go | 4 ++++ internal/catalogue/data_test.go | 2 +- internal/catalogue/setting_defaults_test.go | 2 +- internal/catalogue/settings_judge_test.go | 14 +++++++------- internal/catalogue/terminal_settings_test.go | 14 +++++++------- testdata/store-tests | 3 +++ 9 files changed, 43 insertions(+), 21 deletions(-) diff --git a/cmd/mesh-controller/adoption.go b/cmd/mesh-controller/adoption.go index 411ae05d..afa90a07 100644 --- a/cmd/mesh-controller/adoption.go +++ b/cmd/mesh-controller/adoption.go @@ -424,8 +424,9 @@ func comparisonOf(module string, c comparison, opts takeOptions) (preview string } // And its settings on this machine, composed against its definition (rule 1, rule 6). for _, layer := range c.layers { - keys := make([]string, 0, len(layer.Values)) - for k := range layer.Values { + said := layer.Whole() + keys := make([]string, 0, len(said)) + for k := range said { keys = append(keys, k) } sort.Strings(keys) diff --git a/cmd/mesh-controller/facts.go b/cmd/mesh-controller/facts.go index 50b891cf..af04f202 100644 --- a/cmd/mesh-controller/facts.go +++ b/cmd/mesh-controller/facts.go @@ -348,11 +348,11 @@ func gatherFacts(ctx context.Context, open *stores, busVersion string) (snapshot if layer.From == catalogue.MeshWideLayer { if !meshWide[module] { meshWide[module] = true - f.Settings = append(f.Settings, snapshot.Settings{Module: module, Values: scrub.Values(layer.Values)}) + f.Settings = append(f.Settings, snapshot.Settings{Module: module, Values: scrub.Values(layer.Whole())}) } continue } - m.Settings = append(m.Settings, snapshot.Settings{Module: module, Values: scrub.Values(layer.Values)}) + m.Settings = append(m.Settings, snapshot.Settings{Module: module, Values: scrub.Values(layer.Whole())}) } } m.Declaration = declarationFacts(ctx, open, n.Name, gens, gensErr, scrub) diff --git a/cmd/mesh-controller/mesh_for_test.go b/cmd/mesh-controller/mesh_for_test.go index b2e76be4..eea81d13 100644 --- a/cmd/mesh-controller/mesh_for_test.go +++ b/cmd/mesh-controller/mesh_for_test.go @@ -33,7 +33,21 @@ import ( func aMesh(t *testing.T) *stores { t.Helper() inventory.ForTest(t) // raises the store, migrates it, and points the environment at it - licences.ForTest(t) // planning reaches this one too, by name and never by connection + return aMeshOnTheStore(t) +} + +// aMeshBefore is aMesh on a store migrated only below migration `number`, with the function that runs the +// rest: for a test of what a migration that moves data does to what the mesh composes (novox/hq issue 651). +func aMeshBefore(t *testing.T, number int) (*stores, func()) { + t.Helper() + _, migrate := inventory.ForTestBefore(t, number) + return aMeshOnTheStore(t), migrate +} + +// aMeshOnTheStore is the mesh aMesh makes, on the store the environment points at. +func aMeshOnTheStore(t *testing.T) *stores { + t.Helper() + licences.ForTest(t) // planning reaches this one too, by name and never by connection open, err := openStores(t.Context()) if err != nil { diff --git a/cmd/mesh-controller/modules.go b/cmd/mesh-controller/modules.go index 66804655..dc3ea06c 100644 --- a/cmd/mesh-controller/modules.go +++ b/cmd/mesh-controller/modules.go @@ -537,6 +537,10 @@ func settingsCommand(ctx context.Context, args []string) error { if p.SetBy != "" { by = "; " + p.SetBy } + if p.Why != "" { + // A layer whose placement moved out of it (migration 0097): still the layer, as it stands. + by += "; " + p.Why + } fmt.Printf("%s on %s, until %s (%s%s):\n %s\n", positionals[0], where, p.ReplacedAt.Local().Format("2006-01-02 15:04:05"), p.ReplacedBy, by, shown) } diff --git a/internal/catalogue/data_test.go b/internal/catalogue/data_test.go index fde16ca2..d60f3356 100644 --- a/internal/catalogue/data_test.go +++ b/internal/catalogue/data_test.go @@ -173,7 +173,7 @@ func TestTheBackupHoldersLinesAreDerivedFromTheData(t *testing.T) { {"id":"meta","path":"${dir:meta}","class":"rebuildable"}]}, "resources":[{"id":"meta","type":"directory","mode":"0700"}]}`) facts := map[string]string{"account-home": "/home/op"} - with := Rendering{Settings: SettingsBy{"media": {{From: "node", Values: map[string]any{AccessesSetting: map[string]any{"films": "/tank/films"}}}}}} + with := Rendering{Settings: SettingsBy{"media": {LayerOf("node", map[string]any{AccessesSetting: map[string]any{"films": "/tank/films"}})}}} backup, _, err := seatContributions([]Manifest{pg, agent, media}, Manifest{}, BackupSeat+":"+BackupKindBackup, with, facts, nil) if err != nil { t.Fatal(err) diff --git a/internal/catalogue/setting_defaults_test.go b/internal/catalogue/setting_defaults_test.go index f23fb5e1..d5e50dad 100644 --- a/internal/catalogue/setting_defaults_test.go +++ b/internal/catalogue/setting_defaults_test.go @@ -427,7 +427,7 @@ func TestALeftOutModulesUnplaceableBackupLineCostsOnlyThatLine(t *testing.T) { r := anAdoptedAnchor() r.Modules = append(r.Modules, holder, media, placedBadly) with := anchorRendering(false) - with.Settings["notes"] = []Layer{{From: "anchor", Values: map[string]any{PlacesSetting: "not a map"}}} + with.Settings["notes"] = []Layer{LayerOf("anchor", map[string]any{PlacesSetting: "not a map"})} composed, err := r.Compose(with) if err != nil { t.Fatalf("an unplaceable line of a left-out module failed the machine: %v", err) diff --git a/internal/catalogue/settings_judge_test.go b/internal/catalogue/settings_judge_test.go index 50d96222..cbb494c9 100644 --- a/internal/catalogue/settings_judge_test.go +++ b/internal/catalogue/settings_judge_test.go @@ -27,7 +27,7 @@ func TestASettingThatCannotComposeIsRefusedByNameAndLeavesOnlyItsModuleOut(t *te if c.name == "a mesh-wide port" { from = MeshWideLayer } - err := JudgeSettings(web, []Layer{{From: from, Values: c.layer}}, true) + err := JudgeSettings(web, []Layer{LayerOf(from, c.layer)}, true) if err == nil || !strings.Contains(err.Error(), c.refuse) { t.Errorf("%s: judged %v, want %q", c.name, err, c.refuse) } @@ -69,8 +69,8 @@ func TestASettingThatCannotComposeIsRefusedByNameAndLeavesOnlyItsModuleOut(t *te // declaration as the networks it also joins. func TestAKeptNetworkReachesTheContainerOnAnAdoptedMachineOnly(t *testing.T) { r := anAdoptedAnchor() - keep := SettingsBy{"hello-web": {{From: "anchor", - Values: map[string]any{NetworksSetting: map[string]any{"server": []any{"predecessor_default"}}}}}} + keep := SettingsBy{"hello-web": {LayerOf("anchor", + map[string]any{NetworksSetting: map[string]any{"server": []any{"predecessor_default"}}})}} with := anchorRendering(true) with.Settings = keep @@ -107,13 +107,13 @@ func TestAKeptNetworkReachesTheContainerOnAnAdoptedMachineOnly(t *testing.T) { layer Layer want string }{ - {"mesh-wide", Layer{From: MeshWideLayer, Values: map[string]any{NetworksSetting: map[string]any{"server": []any{"x"}}}}, + {"mesh-wide", LayerOf(MeshWideLayer, map[string]any{NetworksSetting: map[string]any{"server": []any{"x"}}}), "a found network is a fact about one machine"}, - {"an unknown container", Layer{From: "anchor", Values: map[string]any{NetworksSetting: map[string]any{"db": []any{"x"}}}}, + {"an unknown container", LayerOf("anchor", map[string]any{NetworksSetting: map[string]any{"db": []any{"x"}}}), `names the container "db", which it does not declare`}, - {"not a list", Layer{From: "anchor", Values: map[string]any{NetworksSetting: map[string]any{"server": "x"}}}, + {"not a list", LayerOf("anchor", map[string]any{NetworksSetting: map[string]any{"server": "x"}}), "is a list of network names"}, - {"not a network name", Layer{From: "anchor", Values: map[string]any{NetworksSetting: map[string]any{"server": []any{"a/b"}}}}, + {"not a network name", LayerOf("anchor", map[string]any{NetworksSetting: map[string]any{"server": []any{"a/b"}}}), "which is not a network name"}, } { _, err := KeptNetworks(web, []Layer{c.layer}, true) diff --git a/internal/catalogue/terminal_settings_test.go b/internal/catalogue/terminal_settings_test.go index ac767cfd..d822a896 100644 --- a/internal/catalogue/terminal_settings_test.go +++ b/internal/catalogue/terminal_settings_test.go @@ -13,22 +13,22 @@ func TestAPlacementOrAnAccessAtTheMachinesOwnIsRefused(t *testing.T) { for _, path := range []string{"/", "/etc", "/etc/", "/etc/sudoers.d", "/usr/bin", "/root", "/var", "/var/lib", "/home", "/home/", "/run", "/run/user/1000", "/var/lib/mesh/x", "/var/lib/mesh-host", "/var/lib/mesh-host/identity", "/srv/../etc", "//etc", "/proc/1", "/sys", "/dev", "/boot/efi", "/bin", "/sbin", "/lib", "/lib64"} { - layers := []Layer{{From: "laptop", Values: map[string]any{PlacesSetting: map[string]any{"data": path}}}} + layers := []Layer{LayerOf("laptop", map[string]any{PlacesSetting: map[string]any{"data": path}})} if _, err := Places(m, layers); err == nil || !strings.Contains(err.Error(), "issue 339") { t.Errorf("a place at %s: %v", path, err) } - layers = []Layer{{From: "laptop", Values: map[string]any{AccessesSetting: map[string]any{"media": path}}}} + layers = []Layer{LayerOf("laptop", map[string]any{AccessesSetting: map[string]any{"media": path}})} if _, err := AccessPlaces(m, layers); err == nil || !strings.Contains(err.Error(), "issue 339") { t.Errorf("an access at %s: %v", path, err) } } for _, path := range []string{"/srv/notes", "/mnt/plex/data", "/storage/media", "/services/media/movies", "/var/lib/notes/data", "/home/restic/repo", "/var/lib/mesh-store"} { - layers := []Layer{{From: "laptop", Values: map[string]any{PlacesSetting: map[string]any{"data": path}}}} + layers := []Layer{LayerOf("laptop", map[string]any{PlacesSetting: map[string]any{"data": path}})} if got, err := Places(m, layers); err != nil || got["data"].Path != path { t.Errorf("a place at %s: %v %v", path, got, err) } - layers = []Layer{{From: "laptop", Values: map[string]any{AccessesSetting: map[string]any{"media": path}}}} + layers = []Layer{LayerOf("laptop", map[string]any{AccessesSetting: map[string]any{"media": path}})} if got, err := AccessPlaces(m, layers); err != nil || got["media"] != path { t.Errorf("an access at %s: %v %v", path, got, err) } @@ -111,17 +111,17 @@ func TestTheRuntimesDataAndAnyHomesSSHAreTheMachinesOwn(t *testing.T) { for _, path := range []string{"/var/spool", "/var/spool/cron", "/var/lib/docker", "/var/lib/docker/volumes", "/var/lib/containers/storage", "/var/lib/containerd", "/var/lib/containerd/io.containerd.snapshotter.v1", "/opt", "/opt/app", "/home/alice/.ssh", "/home/alice/.ssh/keys", "/srv/backup/.ssh", "/root/.ssh"} { - layers := []Layer{{From: "laptop", Values: map[string]any{PlacesSetting: map[string]any{"data": path}}}} + layers := []Layer{LayerOf("laptop", map[string]any{PlacesSetting: map[string]any{"data": path}})} if _, err := Places(m, layers); err == nil || !strings.Contains(err.Error(), "issue 339") { t.Errorf("a place at %s: %v", path, err) } - layers = []Layer{{From: "laptop", Values: map[string]any{AccessesSetting: map[string]any{"media": path}}}} + layers = []Layer{LayerOf("laptop", map[string]any{AccessesSetting: map[string]any{"media": path}})} if _, err := AccessPlaces(m, layers); err == nil || !strings.Contains(err.Error(), "issue 339") { t.Errorf("an access at %s: %v", path, err) } } for _, path := range []string{"/var/lib/dockerish", "/home/alice/ssh", "/home/alice/.sshd-notes", "/storage/media"} { - layers := []Layer{{From: "laptop", Values: map[string]any{PlacesSetting: map[string]any{"data": path}}}} + layers := []Layer{LayerOf("laptop", map[string]any{PlacesSetting: map[string]any{"data": path}})} if _, err := Places(m, layers); err != nil { t.Errorf("a place at %s: %v", path, err) } diff --git a/testdata/store-tests b/testdata/store-tests index bbfc1467..d3af9659 100644 --- a/testdata/store-tests +++ b/testdata/store-tests @@ -61,3 +61,6 @@ ./cmd/mesh-controller TestAProviderElsewhereServesItsOwnComposedName ./cmd/mesh-controller TestAReusedCheckBuildIsRegisteredByItsDigestAndGatedAsABuiltOne ./internal/inventory TestAReportOfABusyContainerIsKeptAndReadBack +./internal/inventory TestPlacementMovesOutOfTheSettingsLayers +./internal/inventory TestASettingRoutesPlacementBesideTheLayer +./cmd/mesh-controller TestADeclarationIsTheSameAfterPlacementMoves -- 2.54.0 From f0af09cfc764f0c80c58d784a1324a7e757dd526 Mon Sep 17 00:00:00 2001 From: mesh-admin Date: Wed, 14 Oct 2026 00:44:20 +0000 Subject: [PATCH 09/31] Test placement read only from the placement, and raise a store before a migration (issue 651) Co-Authored-By: Claude Opus 5.5 --- internal/catalogue/placement_test.go | 42 ++++++++++++- internal/inventory/fortest.go | 92 ++++++++++++++++++++++++++++ 2 files changed, 133 insertions(+), 1 deletion(-) diff --git a/internal/catalogue/placement_test.go b/internal/catalogue/placement_test.go index 1b02ca8d..e75d770b 100644 --- a/internal/catalogue/placement_test.go +++ b/internal/catalogue/placement_test.go @@ -23,7 +23,7 @@ func placeable() Manifest { } func placedBy(values map[string]any) Rendering { - return Rendering{Settings: SettingsBy{"arr": {{From: "node anchor", Values: values}}}} + return Rendering{Settings: SettingsBy{"arr": {LayerOf("node anchor", values)}}} } func TestAnAssignmentPlacesDirectoriesAndAccesses(t *testing.T) { @@ -179,3 +179,43 @@ func TestAnUnknownAccessReferenceIsRefusedAtParse(t *testing.T) { t.Fatalf("an access with no id and no path was accepted: %v", err) } } + +// **Placement is read from the placement, never from a layer's values** (novox/hq ADR 0348 clause 11, issue +// 651). A `places` or `accesses` among the config values places nothing; the same in Placement does. A +// mergeable file of today's form still takes every key of the layer as it is said, placement too, so it +// composes as it did while placement was kept in the layer. +func TestPlacementIsReadFromThePlacementOnly(t *testing.T) { + m := placeable() + said := map[string]any{ + PlacesSetting: map[string]any{"config": "/services/arr/config"}, + AccessesSetting: map[string]any{"series": "/storage/media/series", "spool": "/storage/downloads"}, + "log": "warn", + } + among := []Layer{{From: "anchor", Values: said}} + if placed, err := Places(m, among); err != nil || placed != nil { + t.Fatalf("a place among the config values placed %v (%v)", placed, err) + } + if _, _, err := accessesFor(m, among); err == nil { + t.Fatal("an access among the config values placed it") + } + beside := []Layer{LayerOf("anchor", said)} + if beside[0].Values["log"] != "warn" || beside[0].Values[PlacesSetting] != nil || beside[0].Placement[PlacesSetting] == nil { + t.Fatalf("LayerOf split %+v", beside[0]) + } + if placed, err := Places(m, beside); err != nil || placed["config"].Path != "/services/arr/config" { + t.Fatalf("the placement did not place: %v (%v)", placed, err) + } + + file := map[string]any{"id": "runtime", "type": "file", "path": "/x", "content": `{"a":1}`, "merge": "json"} + whole, err := ApplySettings(file, among) + if err != nil { + t.Fatal(err) + } + split, err := ApplySettings(file, beside) + if err != nil { + t.Fatal(err) + } + if whole["content"] != split["content"] || !strings.Contains(split["content"].(string), `"places"`) { + t.Fatalf("today's mergeable file composes %q, was %q", split["content"], whole["content"]) + } +} diff --git a/internal/inventory/fortest.go b/internal/inventory/fortest.go index 77aec9cc..cbd7f73a 100644 --- a/internal/inventory/fortest.go +++ b/internal/inventory/fortest.go @@ -2,6 +2,7 @@ package inventory import ( "context" + "encoding/json" "fmt" "os" "strings" @@ -75,3 +76,94 @@ func (i *Inventory) RemoveNodeForTest(ctx context.Context, name string) (bool, e } return tag.RowsAffected() == 1, nil } + +// ForTestBefore is ForTest with the store migrated only below migration `number`, and a function that runs the +// rest: for a test of a migration that moves data, which seeds the store as the mesh holds it before the +// migration, migrates, and reads what the migration made of it (novox/hq issue 651). +func ForTestBefore(t *testing.T, number int) (*Inventory, func()) { + t.Helper() + admin := os.Getenv("MESH_TEST_POSTGRES") + if admin == "" { + t.Skip("no MESH_TEST_POSTGRES; run `make check` to raise one") + } + name := fmt.Sprintf("invb_%d_%s", time.Now().UnixNano()%1_000_000, + strings.ToLower(strings.NewReplacer("/", "", "-", "").Replace(t.Name()))) + if len(name) > 60 { + name = name[:60] + } + all, err := Migrations() + if err != nil { + t.Fatal(err) + } + var before []store.Migration + for _, m := range all { + if m.Number < number { + before = append(before, m) + } + } + store.CreateMigratedForTest(t, admin, name, before) + busPasswordCostForTest.Store(int32(bcrypt.MinCost)) + cut := strings.LastIndex(admin, "/") + t.Setenv(store.Variable(Name), admin[:cut]+"/"+name+"?sslmode=disable") + inv, err := Open(t.Context()) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + inv.Close() + c, err := pgx.Connect(context.Background(), admin) + if err != nil { + return + } + defer c.Close(context.Background()) + _, _ = c.Exec(context.Background(), "drop database if exists "+name+" with (force)") + }) + if err := inv.Ready(t.Context(), 20*time.Second); err != nil { + t.Fatal(err) + } + return inv, func() { + t.Helper() + if _, err := inv.store.Migrate(t.Context(), all); err != nil { + t.Fatal(err) + } + } +} + +// ExecForTest runs one statement against a test's store, for a fixture no verb of the mesh writes: a row as an +// older migration left it. +func (i *Inventory) ExecForTest(t *testing.T, sql string, args ...any) { + t.Helper() + if _, err := i.store.Pool().Exec(t.Context(), sql, args...); err != nil { + t.Fatalf("%s: %v", sql, err) + } +} + +// QueryForTest reads rows of a test's store, each row's columns as JSON text, for a test that checks what a +// migration left. +func (i *Inventory) QueryForTest(t *testing.T, sql string, args ...any) [][]string { + t.Helper() + rows, err := i.store.Pool().Query(t.Context(), sql, args...) + if err != nil { + t.Fatalf("%s: %v", sql, err) + } + defer rows.Close() + var out [][]string + for rows.Next() { + values, err := rows.Values() + if err != nil { + t.Fatal(err) + } + row := make([]string, len(values)) + for k, v := range values { + row[k] = fmt.Sprint(v) + if b, err := json.Marshal(v); err == nil { + row[k] = string(b) + } + } + out = append(out, row) + } + if err := rows.Err(); err != nil { + t.Fatal(err) + } + return out +} -- 2.54.0 From 3048dccd0e51a3c864c50323dc015ead354b4e07 Mon Sep 17 00:00:00 2001 From: mesh-admin Date: Wed, 14 Oct 2026 00:44:51 +0000 Subject: [PATCH 10/31] Keep the declarations the controller composed before placement moved (issue 651) Written by the integration branch's controller at bfc79c6 from the seed of TestADeclarationIsTheSameAfterPlacementMoves. Co-Authored-By: Claude Opus 5.5 --- .../testdata/placement-moved/anchor.json | 46 +++++++++++++++++++ .../testdata/placement-moved/laptop.json | 37 +++++++++++++++ 2 files changed, 83 insertions(+) create mode 100644 cmd/mesh-controller/testdata/placement-moved/anchor.json create mode 100644 cmd/mesh-controller/testdata/placement-moved/laptop.json diff --git a/cmd/mesh-controller/testdata/placement-moved/anchor.json b/cmd/mesh-controller/testdata/placement-moved/anchor.json new file mode 100644 index 00000000..f8f56655 --- /dev/null +++ b/cmd/mesh-controller/testdata/placement-moved/anchor.json @@ -0,0 +1,46 @@ +[ + { + "id": "found.server", + "image": "registry.invalid:5000/library@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "networks": [ + "predecessor_default" + ], + "type": "container" + }, + { + "id": "library.access-storage/downloads", + "mode": "read", + "path": "/storage/downloads", + "type": "access" + }, + { + "id": "library.access-storage/media/series", + "mode": "read", + "path": "/storage/media/series", + "type": "access" + }, + { + "id": "library.config", + "mode": "0750", + "owner": "1001:2000", + "path": "/var/lib/library/config", + "type": "directory" + }, + { + "content": "{\n \"accesses\": {\n \"downloads\": \"/storage/downloads\",\n \"series\": \"/storage/media/series\"\n },\n \"log\": \"warn\",\n \"places\": {\n \"config\": {\n \"owner\": \"1001:2000\",\n \"path\": \"/var/lib/library/config\"\n }\n },\n \"workers\": 4\n}\n", + "id": "library.runtime-config", + "mode": "0640", + "path": "/var/lib/library/config/runtime.json", + "type": "file" + }, + { + "id": "library.server", + "image": "registry.invalid:5000/library@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "type": "container", + "volumes": [ + "/var/lib/library/config:/config", + "/storage/media/series:/series", + "/storage/downloads:/downloads:ro" + ] + } +] diff --git a/cmd/mesh-controller/testdata/placement-moved/laptop.json b/cmd/mesh-controller/testdata/placement-moved/laptop.json new file mode 100644 index 00000000..47884c49 --- /dev/null +++ b/cmd/mesh-controller/testdata/placement-moved/laptop.json @@ -0,0 +1,37 @@ +[ + { + "id": "library.access-home/media/series", + "mode": "read", + "path": "/home/media/series", + "type": "access" + }, + { + "id": "library.access-storage/downloads", + "mode": "read", + "path": "/storage/downloads", + "type": "access" + }, + { + "id": "library.config", + "mode": "0750", + "path": "/var/lib/library/config", + "type": "directory" + }, + { + "content": "{\n \"accesses\": {\n \"downloads\": \"/storage/downloads\",\n \"series\": \"/home/media/series\"\n },\n \"log\": \"warn\"\n}\n", + "id": "library.runtime-config", + "mode": "0640", + "path": "/var/lib/library/config/runtime.json", + "type": "file" + }, + { + "id": "library.server", + "image": "registry.invalid:5000/library@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "type": "container", + "volumes": [ + "/var/lib/library/config:/config", + "/home/media/series:/series", + "/storage/downloads:/downloads:ro" + ] + } +] -- 2.54.0 From 912d76df25f858f52902d734cc9559eb65de75b1 Mon Sep 17 00:00:00 2001 From: mesh-admin Date: Wed, 14 Oct 2026 00:45:58 +0000 Subject: [PATCH 11/31] Move every layer's placement out with migration 0097, and prove no machine is told anything new (issue 651) Co-Authored-By: Claude Opus 5.5 --- cmd/mesh-controller/placement_moved_test.go | 113 +++++++++ .../0097-placement-is-not-config.sql | 46 ++++ internal/inventory/placement_test.go | 222 ++++++++++++++++++ 3 files changed, 381 insertions(+) create mode 100644 cmd/mesh-controller/placement_moved_test.go create mode 100644 internal/inventory/migrations/0097-placement-is-not-config.sql create mode 100644 internal/inventory/placement_test.go diff --git a/cmd/mesh-controller/placement_moved_test.go b/cmd/mesh-controller/placement_moved_test.go new file mode 100644 index 00000000..5831cc56 --- /dev/null +++ b/cmd/mesh-controller/placement_moved_test.go @@ -0,0 +1,113 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "sort" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" +) + +// **Placement leaves the settings layers and no machine is told anything new** (novox/hq ADR 0348 clause +// 11, issue 651). A mesh is seeded as the store holds it before migration 0097 — `places`, `accesses` and a +// kept network among the settings, at the mesh's level and a machine's, one layer of placement only — then +// migrated, and every machine's declaration is composed. What each machine is told of the seeded modules is +// byte for byte what the controller before this change composed from the same seed: the golden files were +// written by the integration branch's controller at bfc79c6 running this same test (MESH_WRITE_GOLDEN=1), +// where there was no migration 0097 to run. +func TestADeclarationIsTheSameAfterPlacementMoves(t *testing.T) { + open, migrate := aMeshBefore(t, 97) + ctx := t.Context() + inv := open.inventory + image := "registry.invalid:5000/library@sha256:" + strings.Repeat("a", 64) + register(t, open, catalogue.Manifest{Module: "library", Version: "1", + Accesses: []catalogue.Access{{ID: "series"}, {ID: "downloads", Mode: catalogue.AccessRead}}, + Resources: []map[string]any{ + {"id": "config", "type": "directory", "mode": "0750"}, + {"id": "server", "type": "container", "image": image, + "volumes": []any{"${dir:config}:/config", "${access:series}:/series", "${access:downloads}:/downloads:ro"}}, + // Today's form: a mergeable file naming no keys takes every key of the layer (ADR 0326 §3). + {"id": "runtime-config", "type": "file", "path": "${dir:config}/runtime.json", "mode": "0640", + "content": `{"log":"info"}`, "merge": "json"}, + }}) + register(t, open, catalogue.Manifest{Module: "found", Version: "1", + Resources: []map[string]any{{"id": "server", "type": "container", "image": image}}}) + if err := inv.SetAdopted(ctx, "anchor", true); err != nil { + t.Fatal(err) + } + for _, l := range []struct{ node, module, values string }{ + {"", "library", `{"accesses": {"downloads": "/storage/downloads"}, "log": "warn"}`}, + {"anchor", "library", `{"places": {"config": {"path": "/var/lib/library/config", "owner": "1001:2000"}}, + "accesses": {"series": "/storage/media/series"}, "workers": 4}`}, + {"laptop", "library", `{"accesses": {"series": "/home/media/series"}}`}, + {"anchor", "found", `{"networks": {"server": ["predecessor_default"]}}`}, + } { + if l.node == "" { + inv.ExecForTest(t, `insert into settings (node, module, values) values (null, $1, $2)`, l.module, l.values) + continue + } + inv.ExecForTest(t, `insert into settings (node, module, values) select id, $2, $3 from node where name = $1`, + l.node, l.module, l.values) + } + + migrate() + // Assigned once the store is migrated: an assignment reads the layers, as this controller holds them. + for _, a := range [][]string{{"anchor", "library", "found"}, {"laptop", "library"}} { + if _, err := assign(ctx, open, a[0], a[1:]...); err != nil { + t.Fatalf("assign %v: %v", a, err) + } + } + + gens, err := generators(ctx, open) + if err != nil { + t.Fatal(err) + } + for _, node := range []string{"anchor", "laptop"} { + plan, settings, err := planFor(ctx, open, node) + if err != nil { + t.Fatal(err) + } + with, _, err := renderingFor(ctx, open, node, plan, settings, gens, Allocating) + if err != nil { + t.Fatal(err) + } + composed, err := plan.Compose(with) + if err != nil { + t.Fatal(err) + } + if len(composed.LeftOut) > 0 { + t.Fatalf("%s leaves out %v", node, composed.LeftOut) + } + var told []map[string]any + for _, r := range composed.Resources { + if owner := composed.Owner[r["id"].(string)]; owner == "library" || owner == "found" { + told = append(told, r) + } + } + sort.Slice(told, func(i, j int) bool { return told[i]["id"].(string) < told[j]["id"].(string) }) + got, err := json.MarshalIndent(told, "", " ") + if err != nil { + t.Fatal(err) + } + golden := filepath.Join("testdata", "placement-moved", node+".json") + if os.Getenv("MESH_WRITE_GOLDEN") != "" { + if err := os.MkdirAll(filepath.Dir(golden), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(golden, append(got, '\n'), 0o644); err != nil { + t.Fatal(err) + } + continue + } + want, err := os.ReadFile(golden) + if err != nil { + t.Fatal(err) + } + if string(want) != string(got)+"\n" { + t.Errorf("%s is told something else after placement moved:\n%s\nwas:\n%s", node, got, want) + } + } +} diff --git a/internal/inventory/migrations/0097-placement-is-not-config.sql b/internal/inventory/migrations/0097-placement-is-not-config.sql new file mode 100644 index 00000000..553cf6d6 --- /dev/null +++ b/internal/inventory/migrations/0097-placement-is-not-config.sql @@ -0,0 +1,46 @@ +-- Placement is not config (novox/hq ADR 0348 clause 11, issue 651). +-- +-- Where a module's directories are on a machine (`places`), where the operator's data it reads is +-- (`accesses`) and which found networks a taken container keeps (`networks`) were kept as keys of a +-- settings layer. They are facts of an assignment that the controller decides, never config values, and +-- the config values are leaving for the mesh-config module (ADR 0348 clause 9) while placement stays with +-- the controller. So they get a table of their own beside the assignment, and every settings layer's +-- placement keys move into it, unchanged. +-- +-- Scoped exactly as the layer they came from: one machine's, or the whole mesh's (node null) for a +-- module's every assignment, which is how a mesh-wide `places` or `accesses` layer composes today. The +-- composer reads placement from here only; the `settings` verb still takes the three keys and routes +-- them here, so nothing a person types changes. +create table placement ( + node uuid references node(id) on delete cascade, + module text not null references module(name) on delete cascade, + -- Only `places`, `accesses` and `networks`, as the settings layer held them. + values jsonb not null, + set_at timestamptz not null default now(), + set_by text +); +create unique index placement_for_the_mesh on placement (module) where node is null; +create unique index placement_for_a_node on placement (node, module) where node is not null; + +-- Why a layer went into the history, beside how (`replaced_by`): said for a layer this migration moved. +alter table settings_history add column why text; + +-- Each layer that held a placement key is kept whole in the history first, as a `settings history` reads it. +insert into settings_history (node, module, values, set_at, replaced_by, set_by, why) +select node, module, values, set_at, 'moved', set_by, 'moved to placement, ADR 0348 clause 11' + from settings + where values ?| array['places', 'accesses', 'networks']; + +insert into placement (node, module, values, set_at, set_by) +select node, module, + (select jsonb_object_agg(e.key, e.value) from jsonb_each(values) as e + where e.key in ('places', 'accesses', 'networks')), + set_at, set_by + from settings + where values ?| array['places', 'accesses', 'networks']; + +-- The layer keeps its other keys, when it was set and who set it; a layer that held only placement stays, +-- empty, so a layer that existed still exists. +update settings + set values = values - 'places' - 'accesses' - 'networks' + where values ?| array['places', 'accesses', 'networks']; diff --git a/internal/inventory/placement_test.go b/internal/inventory/placement_test.go new file mode 100644 index 00000000..0483fe7b --- /dev/null +++ b/internal/inventory/placement_test.go @@ -0,0 +1,222 @@ +package inventory + +import ( + "encoding/json" + "reflect" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" +) + +// placedLibrary is a module with each kind of placement: a directory to place, two accesses, a container that +// may keep a found network. +func placedLibrary() catalogue.Manifest { + return catalogue.Manifest{Module: "library", Version: "1", + Accesses: []catalogue.Access{{ID: "series"}, {ID: "downloads"}}, + Resources: []map[string]any{ + {"id": "config", "type": "directory", "mode": "0750"}, + {"id": "server", "type": "container", + "image": "registry.invalid:5000/library@sha256:" + strings.Repeat("a", 64), + "volumes": []any{"${dir:config}:/config", "${access:series}:/series", "${access:downloads}:/downloads"}}, + {"id": "runtime-config", "type": "file", "path": "${dir:config}/runtime.json", "mode": "0640", + "content": `{"log":"info"}`, "merge": "json"}, + }} +} + +// The layers of the issue (651): a mesh-wide layer with an access and a config value, a node's layer with a +// place, an access and a kept network beside a config value, a node's layer of placement only, and one with +// none. +var placementLayers = []struct { + node string + values string +}{ + {"", `{"accesses": {"downloads": "/storage/downloads"}, "log": "warn"}`}, + {"anchor", `{"places": {"config": {"path": "/var/lib/library/config", "owner": "1001:2000"}}, + "accesses": {"series": "/storage/media/series"}, "networks": {"server": ["predecessor_default"]}, "workers": 4}`}, + {"laptop", `{"accesses": {"series": "/home/media/series"}}`}, + {"spare", `{"log": "debug"}`}, +} + +// **Placement moves out of the settings layers, and every layer reads as it did** (novox/hq ADR 0348 clause +// 11, issue 651). Migration 0097 moves each layer's places, accesses and kept networks beside it: no settings +// row holds a placement key after, every layer the settings verb and composition read is what it was, each +// moved layer is kept whole in the history with why, and a layer without placement is not touched. +func TestPlacementMovesOutOfTheSettingsLayers(t *testing.T) { + inv, migrate := ForTestBefore(t, 97) + ctx := t.Context() + if err := inv.RegisterModule(ctx, placedLibrary(), Source{}); err != nil { + t.Fatal(err) + } + for _, n := range []string{"anchor", "laptop", "spare"} { + if _, err := inv.AddNode(ctx, n); err != nil { + t.Fatal(err) + } + } + if err := inv.SetAdopted(ctx, "anchor", true); err != nil { + t.Fatal(err) + } + // As the mesh holds them before the migration: placement among the settings. + for _, l := range placementLayers { + if l.node == "" { + inv.ExecForTest(t, `insert into settings (node, module, values, set_by) values (null, 'library', $1, 'the operator')`, l.values) + continue + } + inv.ExecForTest(t, `insert into settings (node, module, values) select id, 'library', $2 from node where name = $1`, + l.node, l.values) + } + setAt := inv.QueryForTest(t, `select coalesce(n.name, ''), s.set_at from settings s left join node n on n.id = s.node order by 1`) + + migrate() + + if held := inv.QueryForTest(t, `select module from settings where values ?| array['places', 'accesses', 'networks']`); len(held) != 0 { + t.Fatalf("a settings layer still holds a placement key: %v", held) + } + if after := inv.QueryForTest(t, `select coalesce(n.name, ''), s.set_at from settings s left join node n on n.id = s.node order by 1`); !reflect.DeepEqual(after, setAt) { + t.Fatalf("the layers or when they were set changed: %v, was %v", after, setAt) + } + for _, l := range placementLayers { + var want map[string]any + if err := json.Unmarshal([]byte(l.values), &want); err != nil { + t.Fatal(err) + } + got, has, err := inv.Layer(ctx, l.node, "library") + if err != nil || !has || !reflect.DeepEqual(roundTrip(t, got), want) { + t.Errorf("the layer on %q reads %v (%v, %v), was %v", l.node, got, has, err, want) + } + } + // Composition's view: the config values in Values, the placement in Placement, together the layer. + layers, err := inv.SettingsFor(ctx, "anchor", "library") + if err != nil || len(layers) != 2 { + t.Fatalf("anchor's layers: %v, %v", layers, err) + } + if layers[1].Values["workers"] == nil || layers[1].Values["places"] != nil || layers[1].Placement["places"] == nil || + layers[1].Placement["networks"] == nil || layers[0].Placement["accesses"] == nil || layers[0].Values["log"] != "warn" { + t.Fatalf("anchor's layers split wrong: %+v", layers) + } + for i, l := range layers { + var want map[string]any + _ = json.Unmarshal([]byte(placementLayers[i].values), &want) + if !reflect.DeepEqual(roundTrip(t, l.Whole()), want) { + t.Errorf("anchor's layer %s is %v whole, was %v", l.From, l.Whole(), want) + } + } + // Judged as composition judges them, as before: anchor is adopted, so its kept network composes. + if err := catalogue.JudgeSettings(placedLibrary(), layers, true); err != nil { + t.Fatalf("the moved layers no longer compose: %v", err) + } + + moved := inv.QueryForTest(t, `select coalesce(n.name, ''), h.replaced_by, h.why, h.set_by from settings_history h + left join node n on n.id = h.node order by 1`) + want := [][]string{ + {`""`, `"moved"`, `"moved to placement, ADR 0348 clause 11"`, `"the operator"`}, + {`"anchor"`, `"moved"`, `"moved to placement, ADR 0348 clause 11"`, `null`}, + {`"laptop"`, `"moved"`, `"moved to placement, ADR 0348 clause 11"`, `null`}, + } + if !reflect.DeepEqual(moved, want) { + t.Fatalf("the history of what moved: %v, want %v", moved, want) + } + past, err := inv.SettingsHistory(ctx, "anchor", "library") + if err != nil || len(past) != 1 || past[0].Why != "moved to placement, ADR 0348 clause 11" || past[0].Values["places"] == nil { + t.Fatalf("settings history on anchor: %+v, %v", past, err) + } +} + +// **The settings verb's path still takes a placement key, and routes it** (issue 651): a set keeps the +// placement beside the layer and the rest in it, reads back whole, is refused as before when the placement +// cannot compose (nothing written), and a set or a clear replaces the placement with the layer. +func TestASettingRoutesPlacementBesideTheLayer(t *testing.T) { + inv := ForTest(t) + ctx := t.Context() + if err := inv.RegisterModule(ctx, placedLibrary(), Source{}); err != nil { + t.Fatal(err) + } + if _, err := inv.AddNode(ctx, "anchor"); err != nil { + t.Fatal(err) + } + // Placement alone is a layer as well: the whole mesh's says where both accesses are. + meshWide := map[string]any{"accesses": map[string]any{ + "downloads": "/storage/downloads", "series": "/storage/series"}} + if err := inv.SetSettings(ctx, "", "library", meshWide); err != nil { + t.Fatal(err) + } + if got, has, _ := inv.Layer(ctx, "", "library"); !has || !reflect.DeepEqual(roundTrip(t, got), roundTrip(t, meshWide)) { + t.Fatalf("the mesh-wide layer of placement only reads %v (%v)", got, has) + } + set := map[string]any{"log": "warn", + "places": map[string]any{"config": map[string]any{"path": "/var/lib/library/config"}}, + "accesses": map[string]any{"series": "/storage/media/series"}} + if err := inv.SetSettings(ctx, "anchor", "library", set); err != nil { + t.Fatal(err) + } + stored := inv.QueryForTest(t, `select s.values, p.values from settings s join placement p using (module, node) + where s.node is not null`) + if len(stored) != 1 || stored[0][0] != `{"log":"warn"}` || + !strings.Contains(stored[0][1], `"places"`) || !strings.Contains(stored[0][1], `"accesses"`) { + t.Fatalf("the layer and its placement are kept as %v", stored) + } + if got, _, _ := inv.Layer(ctx, "anchor", "library"); !reflect.DeepEqual(roundTrip(t, got), roundTrip(t, set)) { + t.Fatalf("the layer reads %v, was set %v", got, set) + } + + // Refused as before, by the same judgement, and nothing changes. + for name, bad := range map[string]map[string]any{ + "an undeclared directory": {"places": map[string]any{"cache": "/var/lib/library/cache"}}, + "the machine's own": {"accesses": map[string]any{"series": "/etc"}}, + "a kept network on a converged machine": { + "networks": map[string]any{"server": []any{"predecessor_default"}}}, + } { + if err := inv.SetSettings(ctx, "anchor", "library", bad); err == nil { + t.Errorf("%s was kept", name) + } + } + if err := inv.SetSettings(ctx, "", "library", map[string]any{"accesses": meshWide["accesses"], + "networks": map[string]any{"server": []any{"x"}}}); err == nil || !strings.Contains(err.Error(), "per node") { + t.Errorf("a mesh-wide kept network: %v", err) + } + if got, _, _ := inv.Layer(ctx, "anchor", "library"); !reflect.DeepEqual(roundTrip(t, got), roundTrip(t, set)) { + t.Fatalf("a refused set changed the layer to %v", got) + } + + // A set without placement replaces it too, and the history keeps the layer whole. + if err := inv.SetSettings(ctx, "anchor", "library", map[string]any{"log": "debug"}); err != nil { + t.Fatal(err) + } + if rows := inv.QueryForTest(t, `select module from placement where node is not null`); len(rows) != 0 { + t.Fatalf("a set without placement left it: %v", rows) + } + past, err := inv.SettingsHistory(ctx, "anchor", "library") + if err != nil || len(past) != 1 || past[0].Values["places"] == nil || past[0].Values["log"] != "warn" { + t.Fatalf("the replaced layer is not kept whole: %+v, %v", past, err) + } + + // The merge gate's way in keeps a layer as the facts carry it, whole: split the same way. + if err := inv.KeepSettings(ctx, "anchor", "library", map[string]any{"log": "info", + "places": map[string]any{"config": "/var/lib/library/config"}}); err != nil { + t.Fatal(err) + } + if rows := inv.QueryForTest(t, `select values from settings where values ? 'places'`); len(rows) != 0 { + t.Fatalf("a kept layer put placement among the settings: %v", rows) + } + for _, node := range []string{"", "anchor"} { + if err := inv.ClearSettings(ctx, node, "library"); err != nil { + t.Fatal(err) + } + } + if rows := inv.QueryForTest(t, `select module from placement`); len(rows) != 0 { + t.Fatalf("a clear left the placement: %v", rows) + } +} + +func roundTrip(t *testing.T, v map[string]any) map[string]any { + t.Helper() + b, err := json.Marshal(v) + if err != nil { + t.Fatal(err) + } + out := map[string]any{} + if err := json.Unmarshal(b, &out); err != nil { + t.Fatal(err) + } + return out +} -- 2.54.0 From def3ccefe3c8f26a4bb2dc6f95ba6efd9949bb6d Mon Sep 17 00:00:00 2001 From: mesh-admin Date: Wed, 14 Oct 2026 00:51:48 +0000 Subject: [PATCH 12/31] Refuse templates read through a symlink anywhere under files/ A symlinked files/ or a symlinked directory under it let module check, the merge gate and module add read a template from outside the repository tree, and a symlink no template names passed them while the build's packing refuses it (review of mesh-controller#313). Every entry under files/ must now be a plain directory or file, and templates are read through an os.Root of the module's directory. Co-Authored-By: Claude Opus 5.5 --- cmd/mesh-controller/template_test.go | 27 +++++++++++ internal/catalogue/template.go | 72 +++++++++++++++++++++++++--- internal/catalogue/template_test.go | 65 +++++++++++++++++++++++++ 3 files changed, 158 insertions(+), 6 deletions(-) diff --git a/cmd/mesh-controller/template_test.go b/cmd/mesh-controller/template_test.go index 60ca0e74..edcc66a9 100644 --- a/cmd/mesh-controller/template_test.go +++ b/cmd/mesh-controller/template_test.go @@ -62,6 +62,18 @@ func TestModuleCheckReadsTemplatesFromTheTreeBesideTheManifest(t *testing.T) { t.Fatalf("a template whose text is refused inline passed:\n%s", out.String()) } + // A symlink under files/ that no template names: the build's packing refuses it, so the check does too. + linked := t.TempDir() + aTemplatedModule(t, linked, templatedNotifier, map[string]string{"files/notifier.conf": "volume = 3\n"}) + if err := os.Symlink("/etc/hostname", filepath.Join(linked, "files", "elsewhere")); err != nil { + t.Fatal(err) + } + out.Reset() + if err := moduleCheck([]string{filepath.Join(linked, "module.json")}, &out); err == nil || + !strings.Contains(out.String(), "files/elsewhere is a symlink") { + t.Fatalf("a symlink under files/ passed the check:\n%s", out.String()) + } + // And a template the tree lacks, by name. missing := aTemplatedModule(t, t.TempDir(), templatedNotifier, nil) out.Reset() @@ -139,6 +151,21 @@ func TestTheMergeGateJudgesAChangeToATemplateAlone(t *testing.T) { t.Fatalf("a template the controller refuses passed the gate:\n%s", v.Report()) } + // A symlinked files/ fails: nothing is read from outside the tree. + tree := aTree(t, withEdit(manifests, "notifier", "")) + elsewhere := t.TempDir() + if err := os.WriteFile(filepath.Join(elsewhere, "notifier.conf"), []byte("volume = 3\n"), 0o644); err != nil { + t.Fatal(err) + } + aTemplatedModule(t, filepath.Join(tree, "modules", "notifier"), templatedNotifier, nil) + if err := os.Symlink(elsewhere, filepath.Join(tree, "modules", "notifier", "files")); err != nil { + t.Fatal(err) + } + v = gateJudged(t, f, tree, notifierTemplate) + if v.Verdict != "fail" || !strings.Contains(v.Report(), "files/ is a symlink") { + t.Fatalf("a symlinked files/ passed the gate:\n%s", v.Report()) + } + // And the template removed from the tree fails, naming it. v = gateJudged(t, f, aTemplatedTree(t, manifests, "", false), notifierTemplate) if v.Verdict != "fail" || !strings.Contains(v.Report(), "files/notifier.conf") { diff --git a/internal/catalogue/template.go b/internal/catalogue/template.go index ac785e2a..2a363bf2 100644 --- a/internal/catalogue/template.go +++ b/internal/catalogue/template.go @@ -180,11 +180,18 @@ func fillTemplates(module string, resources []map[string]any, text func(string) } // ReadTemplates is the text of each template the manifest names, read from the module's directory in a repository -// tree, by template path. A missing template, or one that is not a plain file (a symlink is refused by the build's -// packing too), is refused by name. +// tree, by template path. Refused by name: a template the directory lacks, and anything under files/ the build's +// packing refuses — files/ itself, or any directory or file under it, that is a symlink or not a plain file — so +// nothing judged here can be read from outside the tree, and no tree passes here that the build would refuse. Each +// template is read through an os.Root of the module's directory besides, which no path element can leave. func ReadTemplates(m Manifest, dir string) (map[string]string, error) { + problems := templatesTreeProblems(m.Module, dir) + root, err := os.OpenRoot(dir) + if err != nil { + return nil, fmt.Errorf("%s: the module's directory cannot be opened to read its templates: %w", m.Module, err) + } + defer root.Close() texts := map[string]string{} - var problems []string for _, r := range m.Resources { name, has, wrong := templateOf(m.Module, r) if !has { @@ -197,8 +204,7 @@ func ReadTemplates(m Manifest, dir string) (map[string]string, error) { if _, read := texts[name]; read { continue } - full := filepath.Join(dir, filepath.FromSlash(name)) - info, err := os.Lstat(full) + info, err := root.Lstat(name) switch { case errors.Is(err, fs.ErrNotExist): problems = append(problems, fmt.Sprintf("%s: %v names the template %q, and the module's directory "+ @@ -213,7 +219,7 @@ func ReadTemplates(m Manifest, dir string) (map[string]string, error) { "the build packs only files", m.Module, r["id"], name)) continue } - body, err := os.ReadFile(full) + body, err := root.ReadFile(name) if err != nil { problems = append(problems, fmt.Sprintf("%s: %v names the template %q, which cannot be read: %v", m.Module, r["id"], name, err)) @@ -227,6 +233,60 @@ func ReadTemplates(m Manifest, dir string) (map[string]string, error) { return texts, nil } +// templatesTreeProblems is everything under the module's files/ that the build's packing refuses: files/ itself not +// a plain directory, and any entry under it that is neither a directory nor a plain file — a symlink, to a file or a +// directory, a device, a pipe. Each said by its path. A files/ that does not exist is no problem here: each template +// it would have held is refused as missing. +func templatesTreeProblems(module, dir string) []string { + top := filepath.Join(dir, TemplatesDir) + info, err := os.Lstat(top) + if errors.Is(err, fs.ErrNotExist) { + return nil + } + if err != nil { + return []string{fmt.Sprintf("%s: %s/ cannot be read: %v", module, TemplatesDir, err)} + } + if !info.IsDir() { + return []string{fmt.Sprintf("%s: %s/ is %s, not a directory of the module's own; its templates are read "+ + "only from inside the repository, and the build packs only directories and files", module, TemplatesDir, + kindOf(info.Mode()))} + } + var problems []string + err = filepath.WalkDir(top, func(at string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() || d.Type().IsRegular() { + return nil + } + rel, _ := filepath.Rel(dir, at) + problems = append(problems, fmt.Sprintf("%s: %s is %s; the build packs only directories and files, so "+ + "%s/ holds nothing else", module, filepath.ToSlash(rel), kindOf(d.Type()), TemplatesDir)) + return nil + }) + if err != nil { + problems = append(problems, fmt.Sprintf("%s: %s/ cannot be read: %v", module, TemplatesDir, err)) + } + return problems +} + +// kindOf says what a file that is not a plain file is. +func kindOf(mode fs.FileMode) string { + switch { + case mode&fs.ModeSymlink != 0: + return "a symlink" + case mode&fs.ModeDevice != 0: + return "a device" + case mode&fs.ModeNamedPipe != 0: + return "a named pipe" + case mode&fs.ModeSocket != 0: + return "a socket" + case mode.IsDir(): + return "a directory" + } + return "not a plain file" +} + // WithTemplatesFrom is the manifest with each template's text read from the module's directory in a repository // tree as its content: what `module check`, the merge gate and `module add` judge, where nothing was built. Read // again strictly afterwards, so every rule about a file's text judges the template's text exactly as it judges diff --git a/internal/catalogue/template_test.go b/internal/catalogue/template_test.go index 85c81896..a439acb0 100644 --- a/internal/catalogue/template_test.go +++ b/internal/catalogue/template_test.go @@ -256,3 +256,68 @@ func TestAManifestStillNamingATemplateIsNotHeld(t *testing.T) { t.Fatalf("an inline manifest is refused: %s", problem) } } + +// A symlinked files/, or a symlinked directory under it, would read a file outside the tree: refused by name, +// as is any symlink or special file under files/ that no template names, which the build's packing refuses. +func TestASymlinkAnywhereUnderFilesIsRefusedByName(t *testing.T) { + m, err := ParseManifest([]byte(templatedTwin)) + if err != nil { + t.Fatal(err) + } + outside := t.TempDir() + if err := os.MkdirAll(filepath.Join(outside, "deeper"), 0o755); err != nil { + t.Fatal(err) + } + for _, name := range []string{"tidy.conf", "deeper/extra.conf", "secret"} { + if err := os.WriteFile(filepath.Join(outside, filepath.FromSlash(name)), []byte("outside"), 0o644); err != nil { + t.Fatal(err) + } + } + write := func(t *testing.T, dir, name string) { + t.Helper() + full := filepath.Join(dir, filepath.FromSlash(name)) + if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(full, []byte("inside"), 0o644); err != nil { + t.Fatal(err) + } + } + link := func(t *testing.T, target, at string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(at), 0o755); err != nil { + t.Fatal(err) + } + if err := os.Symlink(target, at); err != nil { + t.Fatal(err) + } + } + for _, c := range []struct { + name string + make func(t *testing.T, dir string) + names string + }{ + {"files/ itself", func(t *testing.T, dir string) { + link(t, outside, filepath.Join(dir, "files")) + }, "files/ is a symlink"}, + {"a directory under files/", func(t *testing.T, dir string) { + write(t, dir, "files/tidy.conf") + link(t, filepath.Join(outside, "deeper"), filepath.Join(dir, "files", "deeper")) + }, "files/deeper is a symlink"}, + {"a file no template names", func(t *testing.T, dir string) { + write(t, dir, "files/tidy.conf") + write(t, dir, "files/deeper/extra.conf") + link(t, filepath.Join(outside, "secret"), filepath.Join(dir, "files", "unnamed")) + }, "files/unnamed is a symlink"}, + } { + dir := t.TempDir() + c.make(t, dir) + _, err := WithTemplatesFrom(m, dir) + if err == nil || !strings.Contains(err.Error(), c.names) { + t.Errorf("a symlinked %s was not refused by name: %v", c.name, err) + } + if err != nil && strings.Contains(err.Error(), "outside") { + t.Errorf("a symlinked %s was read: %v", c.name, err) + } + } +} -- 2.54.0 From 661918f4e4e0b5962072702908294b2c434ebe56 Mon Sep 17 00:00:00 2001 From: mesh-admin Date: Wed, 14 Oct 2026 00:51:59 +0000 Subject: [PATCH 13/31] Promise get on the vault's seat, so the Go vault can serve it (issue 674) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The vault hands a secret only on request and reply, sealed to the asker's key (hq ADR 0349 §11). The seat set is the controller's, so the verb is promised here; optional until the vault that runs serves it (design 33 §7). Co-Authored-By: Claude Opus 5.5 --- internal/catalogue/seats.go | 14 +++++++++++- internal/catalogue/vault_seat_test.go | 31 +++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/internal/catalogue/seats.go b/internal/catalogue/seats.go index 68c7291a..1c9478e5 100644 --- a/internal/catalogue/seats.go +++ b/internal/catalogue/seats.go @@ -128,7 +128,19 @@ var defaultSeats = append([]Seat{ // The vault: the controller seals every minted credential with what it provides, which is the // test for a seat of the mesh's own (novox/hq ADR 0161) — a second provider of `secret` is a // second claimant, refused by name, rather than a candidate for a pin. - {Name: "mesh-vault", Scope: ScopeMesh, Delivers: "secret", Decision: "novox/hq ADR 0161"}, + // + // Its one verb, `get`, hands a secret on request and reply, sealed to the key the asker sends and only + // to an asker the vault allows (novox/hq ADR 0349 §11, issue 674). Optional until the vault that runs + // serves it (design 33 §7). + {Name: "mesh-vault", Scope: ScopeMesh, Delivers: "secret", Decision: "novox/hq ADR 0161", + Serves: []Verb{ + {Name: "get", Optional: true, Description: "One secret, sealed to the public key the asker sends (the " + + "sealed box of ADR 0206), with its generation; answered only to an asker the vault allows, as the " + + "bus proved it. Never the value in the clear, and never to a private key.", + Input: schema(map[string]string{"name": "the secret's name", + "public_key": "the asker's X25519 public key as SPKI PEM — never a private key"}, + []string{"name", "public_key"})}, + }}, // Named for its scope since 2026-09-30 (novox/hq ADR 0156); `the-artifact-store` resolves to it as // an alias on a mesh that predates the rename. It serves artifacts of every kind a build makes — // images and archives, by digest — which is why the provision is the artifact store and not an diff --git a/internal/catalogue/vault_seat_test.go b/internal/catalogue/vault_seat_test.go index 0f8827c7..92a31b1b 100644 --- a/internal/catalogue/vault_seat_test.go +++ b/internal/catalogue/vault_seat_test.go @@ -23,3 +23,34 @@ func TestTheVaultsSeatDeliversSecret(t *testing.T) { t.Fatal("a provider of secret that does not claim the seat was allowed to hold it") } } + +// The vault answers `get` on request and reply (novox/hq ADR 0349 §11, issue 674): the seat promises the +// verb, so a holder serving it is accepted — and, optional while the vault that runs is the TypeScript one, +// so is a holder that does not yet. +func TestTheVaultsSeatPromisesGet(t *testing.T) { + seat, _ := SeatNamed("mesh-vault") + var get *Verb + for i := range seat.Serves { + if seat.Serves[i].Name == "get" { + get = &seat.Serves[i] + } + } + if get == nil { + t.Fatal("the vault's seat does not promise get") + } + for _, arg := range []string{"name", "public_key"} { + if props, _ := get.Input["properties"].(map[string]any); props[arg] == nil { + t.Errorf("get takes no %s", arg) + } + } + serving := Manifest{Module: "mesh-vault", Provides: []Offer{{Name: "secret", Scope: ScopeMesh}}, + Claims: []Claim{{Name: "mesh-vault", Scope: ScopeMesh, Serves: []string{"get"}}}} + if err := CanHold(serving, seat); err != nil { + t.Fatalf("the vault serving get was refused: %v", err) + } + notYet := Manifest{Module: "mesh-vault", Provides: []Offer{{Name: "secret", Scope: ScopeMesh}}, + Claims: []Claim{{Name: "mesh-vault", Scope: ScopeMesh}}} + if err := CanHold(notYet, seat); err != nil { + t.Fatalf("the vault that does not serve get yet was refused: %v", err) + } +} -- 2.54.0 From 2f40ae5f76791814e5269e00016a4275b42d6f76 Mon Sep 17 00:00:00 2001 From: mesh-admin Date: Wed, 14 Oct 2026 01:36:45 +0000 Subject: [PATCH 14/31] Read the config form beside the settings form in a manifest (part of issue 660) The manifest's hooks for the config form: read into the fields the settings form fills, written back as read, and judged by ConfigProblems; the rules are in config.go. Co-Authored-By: Claude Opus 5.5 --- internal/catalogue/manifest.go | 49 ++++++++++++++++++++++++-- internal/catalogue/setting_defaults.go | 26 ++++++++++++++ internal/catalogue/setting_takes.go | 3 +- 3 files changed, 74 insertions(+), 4 deletions(-) diff --git a/internal/catalogue/manifest.go b/internal/catalogue/manifest.go index 39dc1ba2..49866bdd 100644 --- a/internal/catalogue/manifest.go +++ b/internal/catalogue/manifest.go @@ -530,6 +530,11 @@ type Manifest struct { // stays refused by name until a layer sets it. The mesh's layer, then the node's, override it. Settings map[string]SettingDeclaration `json:"settings,omitempty"` + // forms is which of the two forms its config is written in, `config` or its alias `settings` (config.go, + // novox/hq ADR 0348 §1): the config form is read into Settings, a mergeable file's takes-settings and + // `${setting:}`, and written back as it was read. + forms configForms + // unknown is the first key this manifest has that this controller does not know, at any depth, when // it was read from the store (novox/hq ADR 0262): a manifest a newer controller registered. // ParseManifest refuses it; the store's catalogue still loads, and the module is left out of every @@ -1321,10 +1326,25 @@ type manifestFields Manifest // strict read there failed the whole catalogue, and with it every plan and every send, the moment a // controller was rolled back. UnknownField says what was set aside. func (m *Manifest) UnmarshalJSON(raw []byte) error { + // The config form read into the settings form's fields, so both compose by one path (config.go). + raw, forms := toSettingsForm(raw) var keys map[string]json.RawMessage if err := json.Unmarshal(raw, &keys); err != nil { return err } + if settings, has := keys[SettingsField]; has && string(settings) != "null" { + forms.settings = append(forms.settings, SettingsField) + } + if config, has := keys[ConfigField]; has { + if string(config) != "null" { + forms.config = append(forms.config, ConfigField) + } + if _, both := keys[SettingsField]; !both { + keys[SettingsField] = config + } + // Beside `settings` it is refused by name (MixedFormProblems), never merged into it. + delete(keys, ConfigField) + } plain := map[string]string{} many := map[string]map[string]string{} if secrets, ok := keys["secrets"]; ok && string(secrets) != "null" { @@ -1453,6 +1473,7 @@ func (m *Manifest) UnmarshalJSON(raw []byte) error { } *m = Manifest(fields) m.unknown = unknown + m.forms = forms m.OptionalRequires = optionalRequires if len(plain) > 0 { m.Secrets = plain @@ -1470,8 +1491,16 @@ func (m *Manifest) UnmarshalJSON(raw []byte) error { } // MarshalJSON writes `secrets` and `contributes` back in the shape they were read: single values, -// and objects of local names. +// and objects of local names; and its config in the form it was read in (config.go). func (m Manifest) MarshalJSON() ([]byte, error) { + raw, err := m.marshalFields() + if err != nil || !m.DeclaresConfig() { + return raw, err + } + return toConfigForm(raw) +} + +func (m Manifest) marshalFields() ([]byte, error) { raw, err := json.Marshal(manifestFields(m)) if err != nil { return nil, err @@ -1783,8 +1812,14 @@ func ParseManifest(raw []byte) (Manifest, error) { problems = append(problems, StateProblems(m)...) // And what it may call its logs, and how large it may ask them to grow (logs.go, novox/hq ADR 0297). problems = append(problems, LogProblems(m)...) - // And the defaults it gives its settings (setting_defaults.go, novox/hq ADR 0262). - problems = append(problems, SettingProblems(m)...) + // And the defaults it gives its settings (setting_defaults.go, novox/hq ADR 0262), or, written in the + // config form, the whole declaration of each value (config.go, novox/hq ADR 0348). + if m.DeclaresConfig() { + problems = append(problems, ConfigProblems(m)...) + } else { + problems = append(problems, SettingProblems(m)...) + problems = append(problems, TypedSettingProblems(m)...) + } // And what it requires optionally, and the files it delivers only while bound (novox/hq ADR 0330). problems = append(problems, OptionalProblems(m)...) // And which setting keys each mergeable file takes (setting_takes.go, novox/hq ADR 0326). @@ -2341,6 +2376,14 @@ func ParseManifest(raw []byte) (Manifest, error) { problems = append(problems, healthProblems(m)...) // And whether a container is busy (novox/hq ADR 0335), likewise. problems = append(problems, busyProblems(m)...) + if m.DeclaresConfig() { + // Said in the words it was written in: the rules shared with the settings form say ${setting:} + // and takes-settings. + for i, p := range problems { + problems[i] = inConfigWords(p) + } + problems = append(problems, MixedFormProblems(m)...) + } if len(problems) > 0 { sort.Strings(problems) return Manifest{}, fmt.Errorf("this manifest cannot be used:\n - %s", diff --git a/internal/catalogue/setting_defaults.go b/internal/catalogue/setting_defaults.go index 3f158e02..353b4063 100644 --- a/internal/catalogue/setting_defaults.go +++ b/internal/catalogue/setting_defaults.go @@ -31,6 +31,28 @@ type SettingDeclaration struct { Default any `json:"default"` // Why this default: one sentence for whoever wonders whether to change it. Why string `json:"why"` + + // The config form's declaration (novox/hq ADR 0348 §4, config.go): none of these is said under + // `settings`, and neither Kind nor Why under `config`. + + // Type is number, string or boolean; object or list only when Trusted. + Type string `json:"type,omitempty"` + // Description says what the value is, including its unit. + Description string `json:"description,omitempty"` + // Enum, Minimum, Maximum and Pattern are its validation: the values it may take, the bounds of a + // number, and the whole of a string matching the pattern. + Enum []any `json:"enum,omitempty"` + Minimum *float64 `json:"minimum,omitempty"` + Maximum *float64 `json:"maximum,omitempty"` + Pattern string `json:"pattern,omitempty"` + // Trusted is a value that decides what runs or who can reach what (ADR 0348 §8). + Trusted bool `json:"trusted,omitempty"` + // Predefined is the predefined key the value falls back to, of the same type (ADR 0348 §2, §7). + Predefined string `json:"predefined,omitempty"` + // Required is the operator's own value: it has no default (ADR 0348 §4). + Required bool `json:"required,omitempty"` + // Runtime is a value the module's code reads at runtime rather than one placed in a file (ADR 0348 §5). + Runtime bool `json:"runtime,omitempty"` } // KindPreference is the kind of a setting that may have a default. @@ -203,6 +225,10 @@ type SettingSource struct { func Effective(m Manifest, layers []Layer) []SettingSource { byKey := map[string]*SettingSource{} for key, d := range m.Settings { + if d.Default == nil { + // A required config value has no default (ADR 0348 §4); a settings declaration always has one. + continue + } byKey[key] = &SettingSource{Key: key, Value: d.Default, From: DefaultLayer, FromDefault: true, Default: d.Default, HasDefault: true} } diff --git a/internal/catalogue/setting_takes.go b/internal/catalogue/setting_takes.go index 0cb7fca4..d723de87 100644 --- a/internal/catalogue/setting_takes.go +++ b/internal/catalogue/setting_takes.go @@ -83,7 +83,8 @@ func TakenSettingsProblems(m Manifest) []string { // Said only when none of the module's mergeable files names its keys: when one does, §2 already // names every file that does not, and a file is not reported twice. func WrittenSettingsProblems(m Manifest) []string { - if len(m.Settings) == 0 { + if len(m.Settings) == 0 || m.DeclaresConfig() { + // A module in the config form is held to this where it is read, in its own words (ConfigProblems). return nil } var silent []string -- 2.54.0 From 233956ef09e2cc48b8a6ea07849f903d249a870e Mon Sep 17 00:00:00 2001 From: mesh-admin Date: Wed, 14 Oct 2026 01:38:04 +0000 Subject: [PATCH 15/31] Let a module declare its config with type, description and validation (issue 660) The config form's declaration and its registration refusals, each by name with what to do (ADR 0348, design 57 package K1). Co-Authored-By: Claude Opus 5.5 --- internal/catalogue/config.go | 459 +++++++++++++++++++++++++++++++++++ 1 file changed, 459 insertions(+) create mode 100644 internal/catalogue/config.go diff --git a/internal/catalogue/config.go b/internal/catalogue/config.go new file mode 100644 index 00000000..d00da709 --- /dev/null +++ b/internal/catalogue/config.go @@ -0,0 +1,459 @@ +package catalogue + +import ( + "encoding/json" + "fmt" + "regexp" + "slices" + "strings" +) + +// A module declares its config (novox/hq ADR 0348 §1, §4, §5, §6, §8, §11; design 57 package K1, issue 660). +// +// **One concept, two spellings for a while.** "Config" replaces "settings": the manifest says `config` for +// `settings`, `takes-config` for `takes-settings` and `${config:}` for `${setting:}`. The old names stay +// accepted as aliases until every module has moved (ADR 0348 §1), and a module uses one form or the other, never +// both. The config form is read into the very fields the settings form fills — Settings, a mergeable file's +// takes-settings and `${setting:}` — so composition has one path and `${config:}` fills exactly as +// `${setting:}` does: a module rewritten from settings to config composes byte-identically. It is written +// back in the form it was read in, so what the mesh holds is what its author wrote. +// +// **A moved module is held to the whole declaration.** Under `config` every value has a type (number, string or +// boolean; an object or a list only when trusted), a description including its unit, its validation (enum, +// minimum and maximum, pattern), whether it is trusted, optionally the predefined key it falls back to, and either +// a default or `required`; it lands in a file or is read at runtime (`runtime: true`); every mergeable file names +// the keys it takes; a `reload-on` or `restart-on` names a file the module declares. A module not yet moved keeps +// today's form and today's rules (SettingProblems, TakenSettingsProblems) until the catalogue's list of such +// modules is empty (ADR 0348 §11). +// +// **A required value nothing sets** is the operator's own: it composes as an operator's setting nobody gave does +// today, refused by name at composition (UnsetSettingError). Who reports it once mesh-config answers values is the +// provisioner's (ADR 0349), not this file's. + +// ConfigField is the manifest key a module declares its config values under. +const ConfigField = "config" + +// TakesConfigField is the key a mergeable file names the config values it takes by. +const TakesConfigField = "takes-config" + +// SettingsField is the old name of ConfigField, read as its alias. +const SettingsField = "settings" + +// configRef is how a file asks for a config value: ${config:}, the same keys as settingRef. +var configRef = regexp.MustCompile(`\$\{config:([a-z0-9][a-z0-9_.-]*)\}`) + +// takesConfigKey and takesSettingsKey are the two spellings as a key of a JSON object. A string value is never +// followed by a colon, and a quote inside a string is escaped, so these match only keys. +var ( + takesConfigKey = regexp.MustCompile(`"` + TakesConfigField + `"(\s*):`) + takesSettingsKey = regexp.MustCompile(`"` + TakesSettingsField + `"(\s*):`) +) + +// The types a config value may have (ADR 0348 §4). +const ( + ConfigNumber = "number" + ConfigString = "string" + ConfigBoolean = "boolean" + ConfigObject = "object" + ConfigList = "list" +) + +// accountFacts are what the mesh places for a module's account, never config values (ADR 0348 §11). +var accountFacts = map[string]bool{"puid": true, "pgid": true} + +// PredefinedKey is one key of the catalogue's list of predefined keys, as far as a module's value falling back +// to it is judged here: its type, and whether it is trusted (ADR 0348 §7, §8). +type PredefinedKey struct { + Type string + Trusted bool +} + +// predefinedKeys is the list of predefined keys, nil until it is present. The list (`predefined.json` at the +// catalogue's root) arrives with design 57's package K2 (issue 661); until then a value naming one is refused +// saying the list is not yet present. +var predefinedKeys map[string]PredefinedKey + +// configForms is which of the two forms a manifest was written in, by the words of each it uses. +type configForms struct { + config, settings []string +} + +// DeclaresConfig is whether the manifest is written in the config form: `config`, `takes-config` or +// `${config:}` (ADR 0348 §1, §11). +func (m Manifest) DeclaresConfig() bool { return len(m.forms.config) > 0 } + +// toSettingsForm is raw with the config form's placeholders and takes-config read as the settings form's, and +// which words of each form it used. `config` itself is a top-level key, renamed where the keys are read +// (UnmarshalJSON). +func toSettingsForm(raw []byte) ([]byte, configForms) { + var f configForms + if configRef.Match(raw) { + f.config = append(f.config, "${config:…}") + } + if takesConfigKey.Match(raw) { + f.config = append(f.config, TakesConfigField) + } + if settingRef.Match(raw) { + f.settings = append(f.settings, "${setting:…}") + } + if takesSettingsKey.Match(raw) { + f.settings = append(f.settings, TakesSettingsField) + } + if len(f.config) == 0 { + return raw, f + } + raw = configRef.ReplaceAllFunc(raw, func(ref []byte) []byte { + return []byte("${setting:" + string(configRef.FindSubmatch(ref)[1]) + "}") + }) + raw = takesConfigKey.ReplaceAllFunc(raw, func(key []byte) []byte { + return []byte(`"` + TakesSettingsField + `"` + string(takesConfigKey.FindSubmatch(key)[1]) + ":") + }) + return raw, f +} + +// toConfigForm is a manifest written in the settings form's words, as JSON, in the config form's: what +// toSettingsForm and UnmarshalJSON read, written back. +func toConfigForm(raw []byte) ([]byte, error) { + var keys map[string]json.RawMessage + if err := json.Unmarshal(raw, &keys); err != nil { + return nil, err + } + if settings, has := keys[SettingsField]; has { + keys[ConfigField] = settings + delete(keys, SettingsField) + var err error + if raw, err = json.Marshal(keys); err != nil { + return nil, err + } + } + raw = settingRef.ReplaceAllFunc(raw, func(ref []byte) []byte { + return []byte("${config:" + string(settingRef.FindSubmatch(ref)[1]) + "}") + }) + raw = takesSettingsKey.ReplaceAllFunc(raw, func(key []byte) []byte { + return []byte(`"` + TakesConfigField + `"` + string(takesSettingsKey.FindSubmatch(key)[1]) + ":") + }) + return raw, nil +} + +// inConfigWords is a refusal said of a module in the config form in its words: `${config:}` and +// takes-config, where the rules shared with the settings form say `${setting:}` and takes-settings. +func inConfigWords(problem string) string { + problem = settingRef.ReplaceAllStringFunc(problem, func(ref string) string { + return "${config:" + settingRef.FindStringSubmatch(ref)[1] + "}" + }) + return strings.ReplaceAll(problem, TakesSettingsField, TakesConfigField) +} + +// MixedFormProblems is a manifest using both forms, said with the words of each (ADR 0348 §1). +func MixedFormProblems(m Manifest) []string { + if len(m.forms.config) == 0 || len(m.forms.settings) == 0 { + return nil + } + return []string{fmt.Sprintf("%s mixes the config form (%s) with the settings form (%s): a module uses one form "+ + "or the other, never both; write %s, %s and ${config:} throughout (novox/hq ADR 0348 §1)", + m.Module, strings.Join(m.forms.config, ", "), strings.Join(m.forms.settings, ", "), + ConfigField, TakesConfigField)} +} + +// configDeclaration is the config form's declaration as written: nothing of the settings form's, and nothing +// unsaid. +type configDeclaration struct { + Type string `json:"type,omitempty"` + Description string `json:"description,omitempty"` + Enum []any `json:"enum,omitempty"` + Minimum *float64 `json:"minimum,omitempty"` + Maximum *float64 `json:"maximum,omitempty"` + Pattern string `json:"pattern,omitempty"` + Trusted bool `json:"trusted,omitempty"` + Predefined string `json:"predefined,omitempty"` + Default any `json:"default,omitempty"` + Required bool `json:"required,omitempty"` + Runtime bool `json:"runtime,omitempty"` +} + +// MarshalJSON writes a settings declaration exactly as before, and a config declaration without the settings +// form's kind and why, and without a default it does not have. +func (d SettingDeclaration) MarshalJSON() ([]byte, error) { + type plain SettingDeclaration + if d.Kind != "" || d.Why != "" { + return json.Marshal(plain(d)) + } + return json.Marshal(configDeclaration{Type: d.Type, Description: d.Description, Enum: d.Enum, + Minimum: d.Minimum, Maximum: d.Maximum, Pattern: d.Pattern, Trusted: d.Trusted, Predefined: d.Predefined, + Default: d.Default, Required: d.Required, Runtime: d.Runtime}) +} + +// configShaped is whether a declaration says anything of the config form's. +func configShaped(d SettingDeclaration) bool { + return d.Type != "" || d.Description != "" || len(d.Enum) > 0 || d.Minimum != nil || d.Maximum != nil || + d.Pattern != "" || d.Trusted || d.Predefined != "" || d.Required || d.Runtime +} + +// TypedSettingProblems is a declaration under `settings` that says what only `config` says: a value with a type, +// a description and its validation is declared in the config form (ADR 0348 §1, §4). +func TypedSettingProblems(m Manifest) []string { + var problems []string + for _, key := range sortedSettingKeys(m.Settings) { + if configShaped(m.Settings[key]) { + problems = append(problems, fmt.Sprintf("%s's setting %q says its type, description or validation "+ + "under %s: a value declared with those is declared under config, with ${config:} and "+ + "takes-config, never under %s (novox/hq ADR 0348 §1, §4)", m.Module, key, SettingsField, SettingsField)) + } + } + return problems +} + +// ConfigProblems is every way a module in the config form declares its config wrongly, each said by name with +// what to do (ADR 0348 How it is checked, design 57 package K1). +func ConfigProblems(m Manifest) []string { + if !m.DeclaresConfig() { + return nil + } + var problems []string + placed := configPlaced(m) + for _, key := range sortedSettingKeys(m.Settings) { + d := m.Settings[key] + say := func(format string, args ...any) { + problems = append(problems, fmt.Sprintf("%s's config value %q: ", m.Module, key)+fmt.Sprintf(format, args...)) + } + if !settingRef.MatchString("${setting:" + key + "}") { + say("not a usable key: lower-case letters, digits, dots, dashes and underscores") + continue + } + if meshWords[key] || accountFacts[key] { + say("%q is placement, which the mesh decides, never a config value (novox/hq ADR 0348 §11)", key) + continue + } + if d.Kind != "" || d.Why != "" { + say("kind and why are the settings form's; under config a value says its type and its description " + + "(novox/hq ADR 0348 §1, §4)") + } + switch d.Type { + case ConfigNumber, ConfigString, ConfigBoolean: + case ConfigObject, ConfigList: + if !d.Trusted { + say("is %s and not trusted: an object or a list decides more than one value can say, so it is "+ + "trusted, \"trusted\": true, or split into numbers, strings and booleans (novox/hq ADR 0348 §4)", d.Type) + } + case "": + say("no type: say \"type\": number, string or boolean, or object or list when trusted (novox/hq ADR 0348 §4)") + default: + say("type %q is none of number, string, boolean, object and list (novox/hq ADR 0348 §4)", d.Type) + } + if strings.TrimSpace(d.Description) == "" { + say("no description: say in a sentence what the value is, including its unit (novox/hq ADR 0348 §4)") + } + switch { + case d.Required && d.Default != nil: + say("both a default and required: a required value is the operator's own and has no default; give " + + "one or the other (novox/hq ADR 0348 §4)") + case !d.Required && d.Default == nil: + say("neither a default nor required: give a default, or say \"required\": true for a value that is " + + "the operator's own (novox/hq ADR 0348 §4)") + } + for _, wrong := range validationProblems(d) { + say("%s", wrong) + } + if word := operatorsWord(key); word != "" && d.Default != nil { + say("a key naming %q is the operator's value: declare it \"required\": true, with no default "+ + "(novox/hq ADR 0348 §4, ADR 0112)", word) + } + if d.Predefined != "" { + p, listed := predefinedKeys[d.Predefined] + switch { + case predefinedKeys == nil: + say("falls back to the predefined key %q, and the list of predefined keys (the catalogue's "+ + "predefined.json, design 57 package K2) is not yet present: no value can fall back to one "+ + "until it is; leave \"predefined\" out for now (novox/hq ADR 0348 §7)", d.Predefined) + case !listed: + say("falls back to %q, which is not on the list of predefined keys: name a key on it, or leave "+ + "\"predefined\" out (novox/hq ADR 0348 §7)", d.Predefined) + default: + if p.Type != d.Type { + say("falls back to the predefined key %q, which is a %s, and this value is a %s: a value falls "+ + "back only to a key of its own type (novox/hq ADR 0348 §4)", d.Predefined, p.Type, d.Type) + } + if p.Trusted && !d.Trusted { + say("falls back to the trusted predefined key %q and is not trusted itself: a value that falls "+ + "back to a trusted key is trusted, \"trusted\": true (novox/hq ADR 0348 §8)", d.Predefined) + } + } + } + // A template's text is read where its tree or its build is (WithTemplatesFrom, Resolve), and this is + // judged again there. + if !placed[key] && !d.Runtime && !m.Templated() { + say("placed nowhere: no file says ${config:%s} or takes it under %s, and it is not marked "+ + "\"runtime\": true for a value the module's code reads itself (novox/hq ADR 0348 §5)", key, TakesConfigField) + } + } + used := settingKeysUsedBy(m) + for _, key := range sortedKeys(used) { + if _, declared := m.Settings[key]; !declared { + problems = append(problems, fmt.Sprintf("%s says ${config:%s} and declares no such value under %s: "+ + "declare %q with its type, description and a default or required (novox/hq ADR 0348 §4, §5)", + m.Module, key, ConfigField, key)) + } + } + problems = append(problems, configTakesProblems(m)...) + problems = append(problems, reflectsProblems(m)...) + return problems +} + +// configPlaced is every value a file, a unit, a directory's owner, a contribution or a served fact asks for, and +// every value a mergeable file takes. +func configPlaced(m Manifest) map[string]bool { + placed := settingKeysUsedBy(m) + for _, r := range m.Resources { + for _, k := range stringsOf(r[TakesSettingsField]) { + placed[k] = true + } + } + return placed +} + +// validationProblems is what is wrong with a declaration's validation, and its default against it. +func validationProblems(d SettingDeclaration) []string { + var problems []string + isNumber := d.Type == ConfigNumber + if len(d.Enum) > 0 && d.Type != ConfigNumber && d.Type != ConfigString { + problems = append(problems, fmt.Sprintf("an enum is for a number or a string, and this is a %s", d.Type)) + } + if (d.Minimum != nil || d.Maximum != nil) && !isNumber { + problems = append(problems, fmt.Sprintf("a minimum or maximum is for a number, and this is a %s", d.Type)) + } + if d.Minimum != nil && d.Maximum != nil && *d.Minimum > *d.Maximum { + problems = append(problems, fmt.Sprintf("its minimum %v is above its maximum %v", *d.Minimum, *d.Maximum)) + } + var pattern *regexp.Regexp + if d.Pattern != "" { + if d.Type != ConfigString { + problems = append(problems, fmt.Sprintf("a pattern is for a string, and this is a %s", d.Type)) + } + var err error + if pattern, err = regexp.Compile(`^(?:` + d.Pattern + `)$`); err != nil { + problems = append(problems, fmt.Sprintf("its pattern %q is not a regular expression: %v", d.Pattern, err)) + } + } + for _, v := range d.Enum { + if !ofType(v, d.Type) { + problems = append(problems, fmt.Sprintf("its enum holds %v, which is not a %s", v, d.Type)) + } + } + if d.Default == nil { + return problems + } + if !ofType(d.Default, d.Type) { + if d.Type != "" { + problems = append(problems, fmt.Sprintf("its default %v is not a %s", d.Default, d.Type)) + } + return problems + } + if len(d.Enum) > 0 && !slices.ContainsFunc(d.Enum, func(v any) bool { return v == d.Default }) { + problems = append(problems, fmt.Sprintf("its default %v is not one of %v", d.Default, d.Enum)) + } + if n, ok := d.Default.(float64); ok { + if d.Minimum != nil && n < *d.Minimum { + problems = append(problems, fmt.Sprintf("its default %v is below its minimum %v", n, *d.Minimum)) + } + if d.Maximum != nil && n > *d.Maximum { + problems = append(problems, fmt.Sprintf("its default %v is above its maximum %v", n, *d.Maximum)) + } + } + if s, ok := d.Default.(string); ok && pattern != nil && !pattern.MatchString(s) { + problems = append(problems, fmt.Sprintf("its default %q does not match its pattern %q", s, d.Pattern)) + } + return problems +} + +// ofType is whether a value read from JSON is of a config type. +func ofType(v any, kind string) bool { + switch v.(type) { + case float64: + return kind == ConfigNumber + case string: + return kind == ConfigString + case bool: + return kind == ConfigBoolean + case map[string]any: + return kind == ConfigObject + case []any: + return kind == ConfigList + } + return false +} + +// configTakesProblems is what ADR 0348 §11 adds for a module in the config form: an account fact in +// takes-config (the mesh's own words are refused for both forms by TakenSettingsProblems), and a mergeable file +// naming no keys where none of the module's does (where one does, TakenSettingsProblems names every one that +// does not). +func configTakesProblems(m Manifest) []string { + var problems, silent []string + naming := false + for _, r := range m.Resources { + if how, _ := r["merge"].(string); how == "" { + continue + } + if _, said := r[TakesSettingsField]; !said { + silent = append(silent, fmt.Sprint(r["id"])) + continue + } + naming = true + for _, k := range stringsOf(r[TakesSettingsField]) { + if accountFacts[k] { + problems = append(problems, fmt.Sprintf("%s's %v names %q in %s, and %q is an account fact the "+ + "mesh places, never a config value (novox/hq ADR 0348 §11)", m.Module, r["id"], k, + TakesConfigField, k)) + } + } + } + if !naming { + for _, id := range silent { + problems = append(problems, fmt.Sprintf("%s's mergeable %s names no %s: a module that declares config "+ + "names the keys each mergeable file takes, an empty list for none (novox/hq ADR 0348 §11)", + m.Module, id, TakesConfigField)) + } + } + return problems +} + +// meshMadeIDs are the resources the mesh composes into a module's own set, which its `restart-on` may name +// without declaring them; meshMadePrefixes are those named after what they are for. +var ( + meshMadeIDs = []string{ComposedJailsID(), FilteringID(), CertificateID(), AuthorityID(), BusUsersID()} + meshMadePrefixes = []string{NeedID(""), SecretID(""), strings.TrimSuffix(GrantID("", ""), "-"), BoundID(""), ReceivedID(""), AccessID("/")} +) + +// reflectsProblems is a `reload-on` or `restart-on` naming a file the module does not declare (ADR 0348 §6): not +// one of its own resources, not one the mesh composes for it, and not another module's, which is named +// `.`. +func reflectsProblems(m Manifest) []string { + own := map[string]bool{} + for _, r := range m.Resources { + own[fmt.Sprint(r["id"])] = true + } + meshMade := func(id string) bool { + if slices.Contains(meshMadeIDs, id) { + return true + } + for _, p := range meshMadePrefixes { + if strings.HasPrefix(id, p) && len(id) > len(p) { + return true + } + } + return false + } + var problems []string + for _, r := range m.Resources { + for _, field := range []string{"reload-on", "restart-on"} { + for _, id := range stringsOf(r[field]) { + if strings.Contains(id, ".") || own[id] || meshMade(id) { + continue + } + problems = append(problems, fmt.Sprintf("%s's %v names %q in its %s, and %s declares no such "+ + "file: name a file of its own, or another module's as . (novox/hq ADR 0348 §6)", + m.Module, r["id"], id, field, m.Module)) + } + } + } + return problems +} -- 2.54.0 From ae071e2b59ee25698582d17bfb070329e2b3d02f Mon Sep 17 00:00:00 2001 From: mesh-admin Date: Wed, 14 Oct 2026 01:39:13 +0000 Subject: [PATCH 16/31] Prove each config refusal and the byte-identical composition of a module moved to config (issue 660) One registration refusal test per rule of design 57 package K1, the forms kept apart and written back as read, and dunst rewritten from settings to config composing to the same bytes. Co-Authored-By: Claude Opus 5.5 --- internal/catalogue/config_test.go | 403 ++++++++++++++++++++++++++++++ 1 file changed, 403 insertions(+) create mode 100644 internal/catalogue/config_test.go diff --git a/internal/catalogue/config_test.go b/internal/catalogue/config_test.go new file mode 100644 index 00000000..1c89bb4d --- /dev/null +++ b/internal/catalogue/config_test.go @@ -0,0 +1,403 @@ +package catalogue + +import ( + "bytes" + "encoding/json" + "strings" + "testing" +) + +// A module declares its config (novox/hq ADR 0348 §1, §4, §5, §6, §8, §11; design 57 package K1, issue 660): +// one registration refusal per rule of design 57's K1, the old names read as aliases, and a module rewritten +// from settings to config composing byte-identically. + +// inConfigForm is a module in the config form that registers: every value typed, described, defaulted or +// required, placed in a file or read at runtime, and its mergeable file naming its keys. +func inConfigForm() map[string]any { + return map[string]any{ + "module": "notifier", + "version": "1", + "config": map[string]any{ + "font-size": map[string]any{"type": "number", "description": "the notification's font size, in points", + "minimum": 6, "maximum": 40, "default": 10}, + "width": map[string]any{"type": "number", "description": "the notification's width, in pixels", + "default": 250}, + "position": map[string]any{"type": "string", "description": "the corner it shows in, a word", + "enum": []any{"top-right", "top-left"}, "default": "top-right"}, + "theme": map[string]any{"type": "string", "description": "the colour theme, a lower-case name", + "pattern": "[a-z]+", "default": "dark"}, + "history": map[string]any{"type": "number", "description": "how many notifications it keeps, a count", + "default": 20, "runtime": true}, + "sender": map[string]any{"type": "string", "description": "the address it mails a digest from", + "required": true}, + }, + "resources": []any{ + map[string]any{"id": "configuration", "type": "file", "path": "/x/notifierrc", + "content": "font = Inter ${config:font-size}\nwidth = ${config:width}\nfrom = ${config:sender}\n"}, + map[string]any{"id": "rules", "type": "file", "path": "/x/rules.json", "merge": "json", + "takes-config": []any{"position", "theme"}, "content": "{\"position\": \"top-right\"}\n"}, + map[string]any{"id": "notifier", "type": "service", "unit": "notifier.service", + "reload-on": []any{"configuration"}, "restart-on": []any{"rules"}}, + }, + } +} + +// declaration is one value of inConfigForm's config, to change in place. +func declaration(m map[string]any, key string) map[string]any { + return m["config"].(map[string]any)[key].(map[string]any) +} + +func resourceOf(m map[string]any, id string) map[string]any { + for _, r := range m["resources"].([]any) { + if r.(map[string]any)["id"] == id { + return r.(map[string]any) + } + } + return nil +} + +func parseRaw(t *testing.T, m map[string]any) (Manifest, error) { + t.Helper() + raw, err := json.Marshal(m) + if err != nil { + t.Fatal(err) + } + return ParseManifest(raw) +} + +// refused is that the manifest is refused with every one of these words. +func refused(t *testing.T, m map[string]any, words ...string) { + t.Helper() + _, err := parseRaw(t, m) + if err == nil { + t.Fatalf("registered; wanted a refusal saying %q", words) + } + for _, w := range words { + if !strings.Contains(err.Error(), w) { + t.Fatalf("refused without %q:\n%v", w, err) + } + } +} + +func TestAModuleInTheConfigFormRegisters(t *testing.T) { + m, err := parseRaw(t, inConfigForm()) + if err != nil { + t.Fatal(err) + } + if !m.DeclaresConfig() { + t.Fatal("a module declaring config is not read as in the config form") + } + d := m.Settings["font-size"] + if d.Type != "number" || d.Description == "" || d.Minimum == nil || *d.Minimum != 6 || d.Default != float64(10) { + t.Fatalf("font-size read as %+v", d) + } + if s := m.Settings["sender"]; !s.Required || s.Default != nil { + t.Fatalf("sender read as %+v", s) + } +} + +// Rule 1: a value without type or description. +func TestConfigRefusesAValueWithoutTypeOrDescription(t *testing.T) { + m := inConfigForm() + delete(declaration(m, "width"), "type") + refused(t, m, `notifier's config value "width"`, "no type") + m = inConfigForm() + declaration(m, "width")["type"] = "colour" + refused(t, m, `"width"`, `type "colour"`) + m = inConfigForm() + delete(declaration(m, "width"), "description") + refused(t, m, `"width"`, "no description") +} + +// Rule 2: a value with neither a default nor required. +func TestConfigRefusesAValueWithNeitherDefaultNorRequired(t *testing.T) { + m := inConfigForm() + delete(declaration(m, "width"), "default") + refused(t, m, `"width"`, "neither a default nor required") + m = inConfigForm() + declaration(m, "sender")["default"] = "noreply@example.org" + refused(t, m, `"sender"`, "both a default and required") +} + +// Rule 3: a default failing its validation. +func TestConfigRefusesADefaultFailingItsValidation(t *testing.T) { + for _, c := range []struct { + key string + value any + said string + }{ + {"font-size", 4, "below its minimum"}, + {"font-size", 41, "above its maximum"}, + {"font-size", "10", "not a number"}, + {"position", "bottom", "not one of"}, + {"theme", "Dark", "does not match"}, + } { + m := inConfigForm() + declaration(m, c.key)["default"] = c.value + refused(t, m, `"`+c.key+`"`, c.said) + } +} + +// Rule 4: an object or a list not trusted. +func TestConfigRefusesAnObjectOrListNotTrusted(t *testing.T) { + for _, kind := range []string{"object", "list"} { + m := inConfigForm() + def := any(map[string]any{"a": 1}) + if kind == "list" { + def = []any{"a"} + } + m["config"].(map[string]any)["rules"] = map[string]any{"type": kind, "description": "the rules, as " + kind, + "default": def, "runtime": true} + refused(t, m, `"rules"`, "is "+kind+" and not trusted") + m["config"].(map[string]any)["rules"].(map[string]any)["trusted"] = true + if _, err := parseRaw(t, m); err != nil { + t.Fatalf("a trusted %s is refused: %v", kind, err) + } + } +} + +// withPredefined is the list of predefined keys as package K2 will give it, for the length of a test. +func withPredefined(t *testing.T, keys map[string]PredefinedKey) { + t.Helper() + was := predefinedKeys + predefinedKeys = keys + t.Cleanup(func() { predefinedKeys = was }) +} + +// Rule 5: a predefined key not on the list, or of another type. Until the list is present (package K2), a value +// naming one is refused saying so. +func TestConfigRefusesAPredefinedKeyNotOnTheListOrOfAnotherType(t *testing.T) { + m := inConfigForm() + declaration(m, "font-size")["predefined"] = "font-size" + refused(t, m, `"font-size"`, "the list of predefined keys", "not yet present") + + withPredefined(t, map[string]PredefinedKey{"font-size": {Type: "number"}, "accent": {Type: "string"}}) + if _, err := parseRaw(t, m); err != nil { + t.Fatalf("a value falling back to a listed key of its type is refused: %v", err) + } + m = inConfigForm() + declaration(m, "font-size")["predefined"] = "font-scale" + refused(t, m, `"font-size"`, `"font-scale"`, "not on the list") + m = inConfigForm() + declaration(m, "font-size")["predefined"] = "accent" + refused(t, m, `"font-size"`, `"accent"`, "is a string") +} + +// Rule 6: an untrusted value falling back to a trusted predefined key. +func TestConfigRefusesAnUntrustedValueFallingBackToATrustedKey(t *testing.T) { + m := inConfigForm() + declaration(m, "theme")["predefined"] = "theme" + refused(t, m, `"theme"`, "not yet present") + + withPredefined(t, map[string]PredefinedKey{"theme": {Type: "string", Trusted: true}}) + refused(t, m, `"theme"`, "trusted", "is not") + declaration(m, "theme")["trusted"] = true + if _, err := parseRaw(t, m); err != nil { + t.Fatalf("a trusted value falling back to a trusted key is refused: %v", err) + } +} + +// Rule 7: a value neither placed in a template nor read at runtime. +func TestConfigRefusesAValueNeitherPlacedNorReadAtRuntime(t *testing.T) { + m := inConfigForm() + delete(declaration(m, "history"), "runtime") + refused(t, m, `"history"`, "${config:history}", `"runtime": true`) + // Taken by a mergeable file is placed. + m = inConfigForm() + if _, err := parseRaw(t, m); err != nil { + t.Fatal(err) + } +} + +// Rule 8: a placeholder naming an undeclared value. +func TestConfigRefusesAPlaceholderNamingAnUndeclaredValue(t *testing.T) { + m := inConfigForm() + resourceOf(m, "configuration")["content"] = "font = ${config:font-size}\nwidth = ${config:width}\n" + + "from = ${config:sender}\ncolour = ${config:colour}\n" + refused(t, m, "${config:colour}", "declares no such value") +} + +// Rule 9: a reload-on or restart-on naming a file the module does not declare. +func TestConfigRefusesAReloadOrRestartOnNamingAnUndeclaredFile(t *testing.T) { + m := inConfigForm() + resourceOf(m, "notifier")["reload-on"] = []any{"configuration", "dunstrc"} + refused(t, m, "reload-on", `"dunstrc"`, "declares no such") + m = inConfigForm() + resourceOf(m, "notifier")["restart-on"] = []any{"rule"} + refused(t, m, "restart-on", `"rule"`) + // Another module's file, and one the mesh makes for the module, are not the module's to declare. + m = inConfigForm() + resourceOf(m, "notifier")["restart-on"] = []any{"rules", "resolver.fact-zones", "secret-mail", "composed-jails"} + if _, err := parseRaw(t, m); err != nil { + t.Fatalf("refused: %v", err) + } +} + +// Rule 10: a moved module's mergeable file not naming its keys under takes-config. +func TestConfigRefusesAMergeableFileNotNamingItsKeys(t *testing.T) { + m := inConfigForm() + delete(resourceOf(m, "rules"), "takes-config") + // position and theme are then placed nowhere; say they are read at runtime so only the file's rule speaks. + declaration(m, "position")["runtime"] = true + declaration(m, "theme")["runtime"] = true + refused(t, m, "rules", "names no takes-config") +} + +// Rule 11: a mesh word, puid or pgid in takes-config. +func TestConfigRefusesAMeshWordOrAnAccountFactInTakesConfig(t *testing.T) { + for _, word := range []string{"ports", "expose", "reach", "endpoints", "places", "accesses", "networks", "puid", "pgid"} { + m := inConfigForm() + resourceOf(m, "rules")["takes-config"] = []any{"position", "theme", word} + refused(t, m, `"`+word+`"`, "takes-config") + } +} + +// A module uses one form or the other, never both. +func TestAManifestMixingTheConfigAndSettingsFormsIsRefused(t *testing.T) { + m := inConfigForm() + resourceOf(m, "configuration")["content"] = "font = ${config:font-size}\nwidth = ${setting:width}\nfrom = ${config:sender}\n" + refused(t, m, "mixes", "${setting:", "config") + + m = inConfigForm() + delete(resourceOf(m, "rules"), "takes-config") + resourceOf(m, "rules")["takes-settings"] = []any{"position", "theme"} + refused(t, m, "mixes", "takes-settings") + + m = inConfigForm() + m["settings"] = map[string]any{"colour": map[string]any{"kind": "preference", "default": "red", "why": "red"}} + refused(t, m, "mixes", "settings") +} + +// The settings form keeps its own declaration; the config form's has no kind or why. +func TestEachFormKeepsItsOwnDeclaration(t *testing.T) { + m := inConfigForm() + declaration(m, "width")["why"] = "fits a title" + refused(t, m, `"width"`, "why") + + err := parsed(t, Manifest{Module: "notifier", + Settings: map[string]SettingDeclaration{"width": {Kind: KindPreference, Default: float64(250), + Why: "fits a title", Type: "number"}}, + Resources: []map[string]any{{"id": "c", "type": "file", "path": "/x", "content": "${setting:width}"}}}) + if err == nil || !strings.Contains(err.Error(), "under config") { + t.Fatalf("a typed value under settings: %v", err) + } +} + +// What the mesh holds is what was written: a module in the config form is written back in it, and read again +// the same. +func TestAConfigManifestIsWrittenBackInItsOwnForm(t *testing.T) { + m, err := parseRaw(t, inConfigForm()) + if err != nil { + t.Fatal(err) + } + raw, err := json.Marshal(m) + if err != nil { + t.Fatal(err) + } + for _, said := range []string{`"config":`, `${config:font-size}`, `"takes-config":`, `"required":true`} { + if !strings.Contains(string(raw), said) { + t.Fatalf("written back without %s: %s", said, raw) + } + } + for _, unsaid := range []string{`"settings":`, `${setting:`, `"takes-settings"`, `"kind"`, `"why"`, `"default":null`} { + if strings.Contains(string(raw), unsaid) { + t.Fatalf("written back with %s: %s", unsaid, raw) + } + } + again, err := ParseManifest(raw) + if err != nil { + t.Fatal(err) + } + twice, _ := json.Marshal(again) + if !bytes.Equal(raw, twice) { + t.Fatalf("read again it is written\n%s\nwhere it was\n%s", twice, raw) + } +} + +// A module in the settings form is written back byte for byte as before. +func TestASettingsManifestIsWrittenBackAsBefore(t *testing.T) { + d := SettingDeclaration{Kind: KindPreference, Default: float64(10), Why: "readable"} + raw, err := json.Marshal(d) + if err != nil { + t.Fatal(err) + } + if string(raw) != `{"kind":"preference","default":10,"why":"readable"}` { + t.Fatalf("written as %s", raw) + } +} + +// settingsDunst is the dunst module as it is written today; configDunst the same module rewritten from settings to +// config. +const settingsDunst = `{"module": "dunst", "version": "1", + "settings": { + "font-size": {"kind": "preference", "default": 10, "why": "10 points of Inter reads well on a screen of about 100 DPI"}, + "width": {"kind": "preference", "default": 250, "why": "250 pixels fits a title of about forty characters at 10 points"} + }, + "resources": [ + {"id": "package", "type": "package", "package": "dunst"}, + {"id": "configuration", "type": "file", "path": "/home/me/.config/dunst/dunstrc", "mode": "0644", + "content": "[global]\n font = Inter ${setting:font-size}\n width = ${setting:width}\n"}, + {"id": "rules", "type": "file", "path": "/home/me/.config/dunst/rules.json", "merge": "json", + "takes-settings": [], "content": "{\"urgency\": \"low\"}\n"}, + {"id": "notifier", "type": "service", "unit": "dunst.service", "reload-on": ["configuration"]} + ]}` + +const configDunst = `{"module": "dunst", "version": "1", + "config": { + "font-size": {"type": "number", "description": "the font size of a notification, in points", "minimum": 6, + "maximum": 40, "default": 10}, + "width": {"type": "number", "description": "the width of a notification, in pixels", "minimum": 100, + "default": 250} + }, + "resources": [ + {"id": "package", "type": "package", "package": "dunst"}, + {"id": "configuration", "type": "file", "path": "/home/me/.config/dunst/dunstrc", "mode": "0644", + "content": "[global]\n font = Inter ${config:font-size}\n width = ${config:width}\n"}, + {"id": "rules", "type": "file", "path": "/home/me/.config/dunst/rules.json", "merge": "json", + "takes-config": [], "content": "{\"urgency\": \"low\"}\n"}, + {"id": "notifier", "type": "service", "unit": "dunst.service", "reload-on": ["configuration"]} + ]}` + +// ${config:} fills exactly as ${setting:} does: the module rewritten from settings to config composes +// to the same bytes, with no layer, with the node's, and with the mesh's under the node's. +func TestAModuleRewrittenFromSettingsToConfigComposesByteIdentically(t *testing.T) { + composed := func(manifest string, layers []Layer) []byte { + t.Helper() + m, err := ParseManifest([]byte(manifest)) + if err != nil { + t.Fatal(err) + } + r := anAdoptedAnchor() + r.Modules = append(r.Modules, m) + with := anchorRendering(false) + if layers != nil { + with.Settings["dunst"] = layers + } + c, err := r.Compose(with) + if err != nil { + t.Fatal(err) + } + if why, left := c.LeftOut["dunst"]; left { + t.Fatalf("dunst was left out: %s", why) + } + raw, err := json.Marshal(c) + if err != nil { + t.Fatal(err) + } + return raw + } + for _, layers := range [][]Layer{ + nil, + {{From: "anchor", Values: map[string]any{"font-size": float64(13)}}}, + {{From: MeshWideLayer, Values: map[string]any{"width": float64(300)}}, + {From: "anchor", Values: map[string]any{"font-size": float64(12), "width": float64(320)}}}, + } { + was, is := composed(settingsDunst, layers), composed(configDunst, layers) + if !bytes.Equal(was, is) { + t.Fatalf("composed differently with %v:\nsettings: %s\nconfig: %s", layers, was, is) + } + if !strings.Contains(string(is), "font = Inter 1") || strings.Contains(string(is), "${config:") { + t.Fatalf("not filled: %s", is) + } + } +} -- 2.54.0 From c5fe7d4795d715fe8485cd620c620672df7d336a Mon Sep 17 00:00:00 2001 From: mesh-admin Date: Wed, 14 Oct 2026 05:03:08 +0000 Subject: [PATCH 17/31] Stop a settings layer replacing what a provider serves (issue 696) What a provider serves is its definition's (ADR 0348 clause 11): a layer fills only the keys it asks for as ${setting:}. Settle replaced any served key a layer named, so a setting for one of a provider's files could change what every consumer was told. influxdb on ace was the one live layer relying on it and now serves its settings (mesh-catalog, merged first), so grafana's data source composes byte for byte as before. UnusedSettings no longer counts a literal served key as a place a setting lands. Co-Authored-By: Claude Opus 5.5 --- internal/catalogue/co_located_test.go | 15 +- .../catalogue/derived_for_consumer_test.go | 11 +- internal/catalogue/served_by_setting_test.go | 194 ++++++++++++++++++ internal/catalogue/settings.go | 53 ++--- internal/catalogue/settings_test.go | 16 -- .../grafana-ace.json | 35 ++++ .../grafana-unlayered.json | 35 ++++ 7 files changed, 313 insertions(+), 46 deletions(-) create mode 100644 internal/catalogue/served_by_setting_test.go create mode 100644 internal/catalogue/testdata/influxdb-served-by-setting/grafana-ace.json create mode 100644 internal/catalogue/testdata/influxdb-served-by-setting/grafana-unlayered.json diff --git a/internal/catalogue/co_located_test.go b/internal/catalogue/co_located_test.go index 3373936f..340a8138 100644 --- a/internal/catalogue/co_located_test.go +++ b/internal/catalogue/co_located_test.go @@ -51,6 +51,15 @@ func stepCA() Manifest { } } +// stepCAWithItsRoot is the authority whose root the operator supplies: the served key asks for it as +// `${setting:root}` (ADR 0155), the only way a setting reaches what a provider serves since a layer +// stopped replacing a served key (novox/hq issue 696; before it, the empty `root` was replaced whole). +func stepCAWithItsRoot() Manifest { + m := stepCA() + m.Serves["acme-ca"]["root"] = "${setting:root}" + return m +} + // routeProxy issues from that authority, so it must be told the root to verify it with. func routeProxy() Manifest { return Manifest{ @@ -78,7 +87,7 @@ func routeProxy() Manifest { // The consequence is silent and total: route-proxy wrote an empty CA bundle, fell back to the // system trust store, could not verify the internal authority, and no certificate was ever issued. func TestACoLocatedProvidersServedValuesReachItsConsumer(t *testing.T) { - r, err := Resolve(shelf(stepCA(), routeProxy()), + r, err := Resolve(shelf(stepCAWithItsRoot(), routeProxy()), []string{"step-ca", "route-proxy"}, reachable(), World{}) if err != nil { t.Fatal(err) @@ -106,7 +115,7 @@ func TestACoLocatedProvidersServedValuesReachItsConsumer(t *testing.T) { // And the binding file says the same thing, for a consumer that reads the binding rather than a // substituted placeholder. The two are one fact and must not be able to disagree. func TestACoLocatedConsumersBindingCarriesTheSettledValues(t *testing.T) { - r, err := Resolve(shelf(stepCA(), routeProxy()), + r, err := Resolve(shelf(stepCAWithItsRoot(), routeProxy()), []string{"step-ca", "route-proxy"}, reachable(), World{}) if err != nil { t.Fatal(err) @@ -152,7 +161,7 @@ func TestACoLocatedProviderIsSettledWhicheverWayItWasPulledIn(t *testing.T) { {"step-ca", "route-proxy"}, {"route-proxy", "step-ca"}, } { - r, err := Resolve(shelf(stepCA(), routeProxy()), assigned, reachable(), World{}) + r, err := Resolve(shelf(stepCAWithItsRoot(), routeProxy()), assigned, reachable(), World{}) if err != nil { t.Fatal(err) } diff --git a/internal/catalogue/derived_for_consumer_test.go b/internal/catalogue/derived_for_consumer_test.go index 93a47913..8d5e9734 100644 --- a/internal/catalogue/derived_for_consumer_test.go +++ b/internal/catalogue/derived_for_consumer_test.go @@ -165,16 +165,19 @@ func TestTwoConsumersOfOneProviderGetTheirOwnDerivation(t *testing.T) { } } -// An operator may still set what the provider serves, and the mesh still derives the rest: the -// setting is laid on first, then the consumer's half is filled. +// An operator may still set what the provider serves where its definition asks (${setting:…}), and +// the mesh still derives the rest: the setting is filled first, then the consumer's half. (Until +// novox/hq issue 696 the layer replaced the served bucket whole; a layer replaces no served key now.) func TestASettingComposesWithADerivedValue(t *testing.T) { - r, err := Resolve(shelf(store(), files()), []string{"store", "files"}, reachable(), World{}) + prefixed := store() + prefixed.Serves["s3-bucket"]["bucket"] = "${setting:bucket-prefix}${consumer:as:dns}" + r, err := Resolve(shelf(prefixed, files()), []string{"store", "files"}, reachable(), World{}) if err != nil { t.Fatal(err) } out, err := r.Declaration(Rendering{ Settings: SettingsBy{"store": {{From: "the operator", - Values: map[string]any{"bucket": "team-${consumer:as:dns}"}}}}, + Values: map[string]any{"bucket-prefix": "team-"}}}}, Grants: []Grant{{Provision: "s3-bucket", Consumer: "workstation", From: "files", Slug: "files", Values: map[string]any{}, Sealed: "c2VhbGVk"}}, }) diff --git a/internal/catalogue/served_by_setting_test.go b/internal/catalogue/served_by_setting_test.go new file mode 100644 index 00000000..681038d6 --- /dev/null +++ b/internal/catalogue/served_by_setting_test.go @@ -0,0 +1,194 @@ +package catalogue + +import ( + "bytes" + "encoding/json" + "os" + "strings" + "testing" +) + +// influxdb and grafana as the catalogue declares them, cut to what composes the data source on a +// machine like ace (novox/hq issue 696): what influxdb serves for influxdb-api and its mergeable +// runtime configuration, and grafana's binding, data source file and credential. +const grafanaLikeTheCatalogue = `{ + "module": "grafana", "version": "1", + "requires": [{"name": "influxdb-api", "optional": true}], + "contributes": {"influxdb-api": {"access": "read"}}, + "binds": {"influxdb-api": "${dir:state}/influxdb.json"}, + "secrets": {"influxdb-api": "${dir:mesh-state}/influxdb-api"}, + "resources": [ + {"id": "mesh-state", "type": "directory", "mode": "0700", "place": "mesh"}, + {"id": "state", "type": "directory", "mode": "0700", "place": "."}, + {"id": "influxdb-datasources", "type": "directory", "mode": "0755", "owner": "472:0"}, + {"id": "influxdb-datasource", "type": "file", "path": "${dir:influxdb-datasources}/mesh-influxdb.yaml", "mode": "0644", + "content": "apiVersion: 1\n# Written by the mesh from grafana's influxdb-api binding, only while it is bound (novox/hq ADR 0330):\n# without influxdb, grafana runs with no mesh data source. grafana reads it at start. Its own name and\n# uid, so a data source somebody made in the UI is never overwritten, and read-only in the UI because\n# the mesh resets it. The password is read from the file the mesh delivers, never written here.\ndatasources:\n - name: InfluxDB (mesh)\n uid: mesh-influxdb-api\n type: influxdb\n access: proxy\n url: ${bound:influxdb-api:scheme}://${bound:influxdb-api:at}:${bound:influxdb-api:port}\n user: ${bound:influxdb-api:as}\n isDefault: false\n editable: false\n jsonData:\n dbName: ${bound:influxdb-api:bucket}\n httpMode: POST\n secureJsonData:\n password: $__file{/run/secrets/influxdb/influxdb-api}\n", + "only-when-bound": "influxdb-api"} + ] +}` + +// influxdbServing is influxdb as the catalogue declared it before issue 696 (literal served org and +// bucket) or after (served from its settings, today's literals as their defaults). +func influxdbServing(after bool) string { + served := `"org": "mesh", "bucket": "default"` + settings, takes := ``, `` + if after { + served = `"org": "${setting:org}", "bucket": "${setting:bucket}"` + settings = `"settings": { + "org": {"kind": "preference", "default": "mesh", "why": "the org influxdb makes at its first start"}, + "bucket": {"kind": "preference", "default": "default", "why": "the bucket influxdb makes at its first start"}},` + takes = `, "takes-settings": ["bucket", "org", "url"], "trusted": true` + } + return `{ + "module": "influxdb", "version": "1", + "provides": [{"name": "influxdb-api", "scope": "mesh", "identity": {"in": "an InfluxDB v1 authorization"}}], + "listens": [{"name": "api", "port": 8086, "protocol": "tcp", "from": "mesh", "why": "queries and writes"}], + ` + settings + ` + "serves": {"influxdb-api": {"scheme": "http", "port": 8086, ` + served + `}}, + "resources": [ + {"id": "mesh-state", "type": "directory", "mode": "0700", "place": "mesh"}, + {"id": "runtime-config", "type": "file", "path": "${dir:mesh-state}/config.json", "mode": "0600", + "content": "{}\n", "merge": "json"` + takes + `} + ] +}` +} + +// composedLikeAce is every resource of grafana and influxdb as one machine composes them, by id, with +// influxdb's settings layer on it. +func composedLikeAce(t *testing.T, influxdb string, layers []Layer) map[string]string { + t.Helper() + var ms []Manifest + for _, raw := range []string{influxdb, grafanaLikeTheCatalogue} { + m, err := ParseManifest([]byte(raw)) + if err != nil { + t.Fatal(err) + } + ms = append(ms, m) + } + r, err := Resolve(shelf(ms...), []string{"influxdb", "grafana"}, reachable(), World{}) + if err != nil { + t.Fatal(err) + } + out, err := r.Declaration(Rendering{Settings: SettingsBy{"influxdb": layers}, + Needed: map[string]map[string]string{}}) + if err != nil { + t.Fatal(err) + } + got := map[string]string{} + for _, res := range out { + id, _ := res["id"].(string) + if !strings.HasPrefix(id, "grafana.") && !strings.HasPrefix(id, "influxdb.") { + continue + } + b, err := json.Marshal(res) + if err != nil { + t.Fatal(err) + } + got[id] = string(b) + } + return got +} + +// aceLikeLayer is a layer like influxdb's on ace: its own org and bucket (made up here), and the +// endpoint block the mesh reads itself. +func aceLikeLayer() []Layer { + return []Layer{{From: "ace", Values: map[string]any{ + "org": "made-up-org", "bucket": "made-up-bucket", + "endpoints": map[string]any{"api": map[string]any{"label": "influx", "port": float64(8086), "reach": "internal"}}, + }}} +} + +// grafana composes byte for byte as it did before issue 696, on a machine like ace and on one +// without a layer, now that influxdb serves its org and bucket from its settings and no layer +// replaces a served key. The golden files are grafana's resources as the integration branch's +// controller composed them at 71362c1 (Settle still overriding), with influxdb serving the literals. +func TestGrafanaComposesTheSameOnceInfluxdbServesItsSettings(t *testing.T) { + for name, layers := range map[string][]Layer{"ace": aceLikeLayer(), "unlayered": nil} { + raw, err := os.ReadFile("testdata/influxdb-served-by-setting/grafana-" + name + ".json") + if err != nil { + t.Fatal(err) + } + var want map[string]json.RawMessage + if err := json.Unmarshal(raw, &want); err != nil { + t.Fatal(err) + } + got := composedLikeAce(t, influxdbServing(true), layers) + seen := 0 + for id, v := range got { + if !strings.HasPrefix(id, "grafana.") { + continue + } + seen++ + w, ok := want[id] + if !ok { + t.Errorf("%s: %s composes and did not before", name, id) + continue + } + var compact bytes.Buffer + if err := json.Compact(&compact, w); err != nil { + t.Fatal(err) + } + if compact.String() != v { + t.Errorf("%s: %s changed:\n got %s\nwant %s", name, id, v, compact.String()) + } + } + if seen != len(want) { + t.Errorf("%s: %d of grafana's %d resources compose", name, seen, len(want)) + } + } +} + +// What a provider serves is its definition's, filled from settings only where it asks with +// ${setting:} (novox/hq issue 696, ADR 0348 clause 11): a layer that sets a key the provider +// serves as a literal replaces nothing, and a key it does not serve is never added. +func TestALayerNoLongerReplacesAServedKey(t *testing.T) { + served, err := Settle(map[string]any{"host": "mail", "port": 25}, + []Layer{{From: "the mesh", Values: map[string]any{"host": "post", "sitename": "Mail"}}}) + if err != nil { + t.Fatal(err) + } + if served["host"] != "mail" { + t.Errorf("a layer replaced the served host: %v", served) + } + if _, leaked := served["sitename"]; leaked { + t.Errorf("a setting for a file reached the consumers: %v", served) + } + + // And a layer setting it is said to reach nothing, rather than taken for a change. + m := Manifest{Module: "mail", Serves: map[string]map[string]any{"smtp": {"host": "mail"}}} + if unused := UnusedSettings(m, []Layer{{From: "the mesh", Values: map[string]any{"host": "post"}}}); len(unused) != 1 { + t.Errorf("a layer setting a literal served key was not said to reach nothing: %v", unused) + } + + // influxdb as it was before issue 696, serving literals, under a layer like ace's: the layer no + // longer reaches grafana — which is why the catalogue moves influxdb first. + got := composedLikeAce(t, influxdbServing(false), aceLikeLayer()) + for _, id := range []string{"grafana.influxdb-datasource", "grafana.bound-influxdb-api"} { + if strings.Contains(got[id], "made-up") { + t.Errorf("%s still carries the layer's value over a literal served key: %s", id, got[id]) + } + } + if !strings.Contains(got["grafana.influxdb-datasource"], `dbName: default\n`) { + t.Errorf("the data source does not name the served literal: %s", got["grafana.influxdb-datasource"]) + } +} + +// influxdb's own runtime configuration takes the keys it names: org and bucket from a layer like +// ace's, and not the endpoint block the mesh reads itself, which today's form wrote there too. +func TestInfluxdbsConfigTakesItsOrgAndBucket(t *testing.T) { + got := composedLikeAce(t, influxdbServing(true), aceLikeLayer())["influxdb.runtime-config"] + var r map[string]any + if err := json.Unmarshal([]byte(got), &r); err != nil { + t.Fatal(err) + } + if want := "{\n \"bucket\": \"made-up-bucket\",\n \"org\": \"made-up-org\"\n}\n"; r["content"] != want { + t.Errorf("config.json: got %q, want %q", r["content"], want) + } + got = composedLikeAce(t, influxdbServing(true), nil)["influxdb.runtime-config"] + if err := json.Unmarshal([]byte(got), &r); err != nil { + t.Fatal(err) + } + if r["content"] != "{}\n" { + t.Errorf("a default reached the mergeable file: %q", r["content"]) + } +} diff --git a/internal/catalogue/settings.go b/internal/catalogue/settings.go index 45c992e8..28177413 100644 --- a/internal/catalogue/settings.go +++ b/internal/catalogue/settings.go @@ -117,25 +117,27 @@ func ApplySettings(resource map[string]any, layers []Layer) (map[string]any, err return out, nil } -// Settle lays settings over what a provider serves. Exported because a served fact is settled where -// the mesh is walked rather than where a node is declared. +// Settle fills what a provider serves from its settings. Exported because a served fact is settled +// where the mesh is walked rather than where a node is declared. // -// **A setting overrides a served key; it never adds one** (novox/hq 04-ISSUES/173). What a consumer -// is told is the provider's contract, and a setting made for one of the provider's files — a site -// name, a public address — is not part of it. Before this, every setting of a module reached every -// consumer of every provision it served. +// **What a provider serves is its definition's; a layer replaces none of it** (novox/hq ADR 0348 +// clause 11, issue 696). A served value is filled from the layers only where the definition asks, +// with `${setting:}` (ADR 0155) — a mail provider's domain, an identity provider's issuer, a +// time-series store's org and bucket — and a layer that sets a key the provider serves as a literal +// changes nothing a consumer is told. Nor does a setting add a key (novox/hq 04-ISSUES/173): a +// setting made for one of the provider's files — a site name, a public address — is not part of the +// contract. Until issue 696, a layer replaced any served key it named; influxdb on ace was the one +// layer relying on it, and serves `${setting:org}` and `${setting:bucket}` since. func Settle(base map[string]any, layers []Layer) (map[string]any, error) { - return overridden(base, layers, "what is served") + return filledFromSettings(deepCopy(base), layers, "what is served") } -// overridden lays settings over a map whose keys are its contract: a contribution, a served fact. -// Only the keys the map already declares are touched; the rest of a layer is somebody else's -// business (a file's, another destination's) and is left to reach it there. +// overridden lays settings over a contribution: a map whose keys are its contract. Only the keys the +// map already declares are touched; the rest of a layer is somebody else's business (a file's, +// another destination's) and is left to reach it there. // -// A declared value may itself be the operator's, `${setting:}` (ADR 0155): a mail provider -// serves its domain, an identity provider its issuer, and neither is the definition's to state. -// Filled from the layers after the overrides, and refused by name when nothing sets it — a literal -// placeholder handed to a consumer is a service configured against a string nobody meant. +// A declared value may itself be the operator's, `${setting:}` (ADR 0155), filled after the +// overrides (filledFromSettings). func overridden(base map[string]any, layers []Layer, what string) (map[string]any, error) { kept := make([]Layer, 0, len(layers)) for _, layer := range layers { @@ -151,6 +153,14 @@ func overridden(base map[string]any, layers []Layer, what string) (map[string]an if err != nil { return nil, err } + return filledFromSettings(merged, layers, what) +} + +// filledFromSettings fills each `${setting:}` in a map's top-level strings from the layers, +// later beating earlier and a module's defaults lowest, and refuses one nothing sets by name — a +// literal placeholder handed to a consumer is a service configured against a string nobody meant. +// It fills merged in place and returns it. +func filledFromSettings(merged map[string]any, layers []Layer, what string) (map[string]any, error) { for key, value := range merged { s, ok := value.(string) if !ok { @@ -177,7 +187,8 @@ func overridden(base map[string]any, layers []Layer, what string) (map[string]an // module says what it means by default, and somebody says what it means here. A contribution that // could not be settled would have to be edited to be reused anywhere else. The two differ in one // respect, and the caller decides it: a file takes keys it did not declare (a setting may add to a -// configuration), a contribution or served fact does not (overridden). +// configuration), a contribution does not (overridden). A served fact takes no layer at all; only +// its ${setting:} is filled (Settle). func settle(base map[string]any, layers []Layer, protected map[string]bool, what string) ( map[string]any, error) { merged := deepCopy(base) @@ -353,8 +364,9 @@ func sortedKeysAny(m map[string]any) []string { // way there is. This is what makes that visible at the moment they set it. // // Where a key can land: any mergeable file takes any key; a file asking for `${setting:}` -// takes that key (ADR 0155); a contribution or a served fact takes a key it declares, and no other -// (novox/hq 04-ISSUES/173); and the mesh's own words — `expose`, `ports`, `reach`, `endpoints` — +// takes that key (ADR 0155); a contribution takes a key it declares, and no other (novox/hq +// 04-ISSUES/173); a served fact takes only the keys it asks for as `${setting:}`, since a layer +// replaces no served key (novox/hq issue 696); and the mesh's own words — `expose`, `ports`, `reach`, `endpoints` — // are read by the mesh. A key none of those takes is stray, and is said so rather than dropped. func UnusedSettings(m Manifest, layers []Layer) []string { named := namesItsKeys(m) @@ -397,11 +409,6 @@ func UnusedSettings(m Manifest, layers []Layer) []string { } } } - for _, served := range m.Serves { - for key := range served { - lands[key] = true - } - } var unused []string for _, layer := range layers { for key := range layer.Whole() { @@ -444,7 +451,7 @@ func UnusedSettings(m Manifest, layers []Layer) []string { } unused = append(unused, fmt.Sprintf( "%s sets %q, and %s has no file that merges it, asks for no ${setting:%s}, and "+ - "declares no %q in what it contributes or serves%s", + "declares no %q in what it contributes%s", layer.From, key, m.Module, key, key, takenSaid(m))) } } diff --git a/internal/catalogue/settings_test.go b/internal/catalogue/settings_test.go index 6fe6d2a0..2313d8be 100644 --- a/internal/catalogue/settings_test.go +++ b/internal/catalogue/settings_test.go @@ -190,22 +190,6 @@ func TestASettingLandsOnlyWhereSomethingDeclaresIt(t *testing.T) { } } -func TestASettingOverridesAServedKeyAndAddsNone(t *testing.T) { - // What a consumer is told is the provider's contract. A setting made for one of the - // provider's files — its site name, its public address — is not part of it. - served, err := Settle(map[string]any{"host": "mail", "port": 25}, - []Layer{{From: "the mesh", Values: map[string]any{"host": "post", "sitename": "Mail"}}}) - if err != nil { - t.Fatal(err) - } - if served["host"] != "post" { - t.Errorf("the setting did not override the served host: %v", served) - } - if _, leaked := served["sitename"]; leaked { - t.Errorf("a setting for a file reached the consumers: %v", served) - } -} - func TestContentThatIsNotJSONIsRefusedWhereSomebodyIsLooking(t *testing.T) { // Rather than on the machine, at apply time, as a file the program cannot read. _, err := ApplySettings(file(`this is not json`), nil) diff --git a/internal/catalogue/testdata/influxdb-served-by-setting/grafana-ace.json b/internal/catalogue/testdata/influxdb-served-by-setting/grafana-ace.json new file mode 100644 index 00000000..d9e60b8a --- /dev/null +++ b/internal/catalogue/testdata/influxdb-served-by-setting/grafana-ace.json @@ -0,0 +1,35 @@ +{ + "grafana.bound-influxdb-api": { + "content": "{\n \"as\": \"mesh_workstation_grafana\",\n \"at\": \"workstation.internal\",\n \"binding\": 1,\n \"from\": \"workstation\",\n \"generated\": \"by the mesh — do not edit; replaced whenever this changes. The credential is not here: it is sealed, in the file this module's manifest names under `secrets`\",\n \"provision\": \"influxdb-api\",\n \"serves\": {\n \"bucket\": \"made-up-bucket\",\n \"org\": \"made-up-org\",\n \"port\": 8086,\n \"scheme\": \"http\"\n }\n}\n", + "id": "grafana.bound-influxdb-api", + "mode": "0644", + "path": "/var/lib/grafana/influxdb.json", + "type": "file" + }, + "grafana.influxdb-datasource": { + "content": "apiVersion: 1\n# Written by the mesh from grafana's influxdb-api binding, only while it is bound (novox/hq ADR 0330):\n# without influxdb, grafana runs with no mesh data source. grafana reads it at start. Its own name and\n# uid, so a data source somebody made in the UI is never overwritten, and read-only in the UI because\n# the mesh resets it. The password is read from the file the mesh delivers, never written here.\ndatasources:\n - name: InfluxDB (mesh)\n uid: mesh-influxdb-api\n type: influxdb\n access: proxy\n url: http://workstation.internal:8086\n user: mesh_workstation_grafana\n isDefault: false\n editable: false\n jsonData:\n dbName: made-up-bucket\n httpMode: POST\n secureJsonData:\n password: $__file{/run/secrets/influxdb/influxdb-api}\n", + "id": "grafana.influxdb-datasource", + "mode": "0644", + "path": "/var/lib/grafana/influxdb-datasources/mesh-influxdb.yaml", + "type": "file" + }, + "grafana.influxdb-datasources": { + "id": "grafana.influxdb-datasources", + "mode": "0755", + "owner": "472:0", + "path": "/var/lib/grafana/influxdb-datasources", + "type": "directory" + }, + "grafana.mesh-state": { + "id": "grafana.mesh-state", + "mode": "0700", + "path": "/var/lib/mesh/grafana", + "type": "directory" + }, + "grafana.state": { + "id": "grafana.state", + "mode": "0700", + "path": "/var/lib/grafana", + "type": "directory" + } +} diff --git a/internal/catalogue/testdata/influxdb-served-by-setting/grafana-unlayered.json b/internal/catalogue/testdata/influxdb-served-by-setting/grafana-unlayered.json new file mode 100644 index 00000000..a0ee7989 --- /dev/null +++ b/internal/catalogue/testdata/influxdb-served-by-setting/grafana-unlayered.json @@ -0,0 +1,35 @@ +{ + "grafana.bound-influxdb-api": { + "content": "{\n \"as\": \"mesh_workstation_grafana\",\n \"at\": \"workstation.internal\",\n \"binding\": 1,\n \"from\": \"workstation\",\n \"generated\": \"by the mesh — do not edit; replaced whenever this changes. The credential is not here: it is sealed, in the file this module's manifest names under `secrets`\",\n \"provision\": \"influxdb-api\",\n \"serves\": {\n \"bucket\": \"default\",\n \"org\": \"mesh\",\n \"port\": 8086,\n \"scheme\": \"http\"\n }\n}\n", + "id": "grafana.bound-influxdb-api", + "mode": "0644", + "path": "/var/lib/grafana/influxdb.json", + "type": "file" + }, + "grafana.influxdb-datasource": { + "content": "apiVersion: 1\n# Written by the mesh from grafana's influxdb-api binding, only while it is bound (novox/hq ADR 0330):\n# without influxdb, grafana runs with no mesh data source. grafana reads it at start. Its own name and\n# uid, so a data source somebody made in the UI is never overwritten, and read-only in the UI because\n# the mesh resets it. The password is read from the file the mesh delivers, never written here.\ndatasources:\n - name: InfluxDB (mesh)\n uid: mesh-influxdb-api\n type: influxdb\n access: proxy\n url: http://workstation.internal:8086\n user: mesh_workstation_grafana\n isDefault: false\n editable: false\n jsonData:\n dbName: default\n httpMode: POST\n secureJsonData:\n password: $__file{/run/secrets/influxdb/influxdb-api}\n", + "id": "grafana.influxdb-datasource", + "mode": "0644", + "path": "/var/lib/grafana/influxdb-datasources/mesh-influxdb.yaml", + "type": "file" + }, + "grafana.influxdb-datasources": { + "id": "grafana.influxdb-datasources", + "mode": "0755", + "owner": "472:0", + "path": "/var/lib/grafana/influxdb-datasources", + "type": "directory" + }, + "grafana.mesh-state": { + "id": "grafana.mesh-state", + "mode": "0700", + "path": "/var/lib/mesh/grafana", + "type": "directory" + }, + "grafana.state": { + "id": "grafana.state", + "mode": "0700", + "path": "/var/lib/grafana", + "type": "directory" + } +} -- 2.54.0 From 9aa03326949548dade671335b39a07296388b41a Mon Sep 17 00:00:00 2001 From: mesh-admin Date: Wed, 14 Oct 2026 05:17:26 +0000 Subject: [PATCH 18/31] Refuse an undeclared takes-config key, and say an unset required value in the config form's words MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A key a mergeable file took had no type, validation or trusted mark, and the config verb could never set it (ADR 0348 §4). A required config value nothing sets was refused naming ${setting:} and settings set, words the module does not use. Review of mesh-controller#320 (issue 660). Co-Authored-By: Claude Opus 5.5 --- internal/catalogue/config.go | 16 +++++++-- internal/catalogue/config_test.go | 50 ++++++++++++++++++++++++++ internal/catalogue/declaration.go | 2 +- internal/catalogue/setting_into.go | 57 +++++++++++++++++++++--------- internal/catalogue/settings.go | 2 +- 5 files changed, 106 insertions(+), 21 deletions(-) diff --git a/internal/catalogue/config.go b/internal/catalogue/config.go index d00da709..90e9ea71 100644 --- a/internal/catalogue/config.go +++ b/internal/catalogue/config.go @@ -279,8 +279,9 @@ func ConfigProblems(m Manifest) []string { } } } - // A template's text is read where its tree or its build is (WithTemplatesFrom, Resolve), and this is - // judged again there. + // A template's text is not here yet. Resolve fills it from the build, WithTemplatesFrom from the tree; neither + // judges again itself, but WithTemplatesFrom parses its result again, and a resolved build is parsed again + // where it is registered, and this is judged then. if !placed[key] && !d.Runtime && !m.Templated() { say("placed nowhere: no file says ${config:%s} or takes it under %s, and it is not marked "+ "\"runtime\": true for a value the module's code reads itself (novox/hq ADR 0348 §5)", key, TakesConfigField) @@ -399,10 +400,19 @@ func configTakesProblems(m Manifest) []string { } naming = true for _, k := range stringsOf(r[TakesSettingsField]) { - if accountFacts[k] { + _, declared := m.Settings[k] + switch { + case accountFacts[k]: problems = append(problems, fmt.Sprintf("%s's %v names %q in %s, and %q is an account fact the "+ "mesh places, never a config value (novox/hq ADR 0348 §11)", m.Module, r["id"], k, TakesConfigField, k)) + case meshWords[k]: + // Refused for both forms by TakenSettingsProblems, in its words. + case !declared: + problems = append(problems, fmt.Sprintf("%s's %v names %q in %s and declares no such value under "+ + "%s: a value a file takes is declared like every other, with its type, description and a "+ + "default or required, or the config verb could never set it (novox/hq ADR 0348 §4, §11)", + m.Module, r["id"], k, TakesConfigField, ConfigField)) } } } diff --git a/internal/catalogue/config_test.go b/internal/catalogue/config_test.go index 1c89bb4d..1a2d8499 100644 --- a/internal/catalogue/config_test.go +++ b/internal/catalogue/config_test.go @@ -3,6 +3,7 @@ package catalogue import ( "bytes" "encoding/json" + "errors" "strings" "testing" ) @@ -401,3 +402,52 @@ func TestAModuleRewrittenFromSettingsToConfigComposesByteIdentically(t *testing. } } } + +// A key a mergeable file takes is a config value like every other: one not declared under config is refused by +// name (ADR 0348 §4, review of mesh-controller#320). +func TestConfigRefusesATakesConfigKeyNotDeclared(t *testing.T) { + m := inConfigForm() + resourceOf(m, "rules")["takes-config"] = []any{"position", "theme", "urgency"} + refused(t, m, `"urgency"`, "takes-config", "declares no such value under config") +} + +// A required value nothing sets is refused at composition in the config form's words: ${config:} and the +// config verb, never `settings set` (review of mesh-controller#320). The settings form's words are unchanged. +func TestAnUnsetRequiredValueIsRefusedInTheConfigFormsWords(t *testing.T) { + m, err := parseRaw(t, inConfigForm()) + if err != nil { + t.Fatal(err) + } + r := anAdoptedAnchor() + r.Modules = append(r.Modules, m) + composed, err := r.Compose(anchorRendering(false)) + if err != nil { + t.Fatal(err) + } + why, left := composed.LeftOut["notifier"] + if !left { + t.Fatal("a module whose required value nothing sets was not left out") + } + for _, said := range []string{"${config:sender}", `required config value "sender"`, "config verb"} { + if !strings.Contains(why, said) { + t.Fatalf("left out without %q: %s", said, why) + } + } + for _, unsaid := range []string{"${setting:", "settings set"} { + if strings.Contains(why, unsaid) { + t.Fatalf("left out in the settings form's words (%q): %s", unsaid, why) + } + } + var unset *UnsetSettingError + if err := unsetInConfigWords(settingInto(map[string]any{"type": "file", "content": "${setting:sender}"}, nil, + "notifier"), m); !errors.As(err, &unset) || unset.Setting != "sender" { + t.Fatalf("not typed as a value nobody gave: %v", err) + } + + settingsForm := notifier() + settingsForm.Resources[0]["content"] = "from = ${setting:domain}\n" + err = unsetInConfigWords(settingInto(settingsForm.Resources[0], nil, "notifier"), settingsForm) + if err == nil || !strings.Contains(err.Error(), "`settings set notifier `") { + t.Fatalf("the settings form's refusal changed: %v", err) + } +} diff --git a/internal/catalogue/declaration.go b/internal/catalogue/declaration.go index 761a5515..25e39e27 100644 --- a/internal/catalogue/declaration.go +++ b/internal/catalogue/declaration.go @@ -1003,7 +1003,7 @@ func (r Resolution) compose(with Rendering, owner map[string]string, // **An operator's value, from the assignment** (novox/hq ADR 0112, ADR 0155): what a // definition may not carry because it is true of one installation only. Filled from // the same layers a mergeable file takes, and refused when no layer set it. - if err := settingInto(copied, WithDefaults(m, with.Settings[m.Module]), m.Module); err != nil { + if err := unsetInConfigWords(settingInto(copied, WithDefaults(m, with.Settings[m.Module]), m.Module), m); err != nil { return nil, err } // **Placed before anything reads a path.** A pathless directory receives the path diff --git a/internal/catalogue/setting_into.go b/internal/catalogue/setting_into.go index ee04ee18..1ea94437 100644 --- a/internal/catalogue/setting_into.go +++ b/internal/catalogue/setting_into.go @@ -1,6 +1,7 @@ package catalogue import ( + "errors" "fmt" "regexp" "sort" @@ -47,10 +48,30 @@ func settingsUsed(content string) []string { type UnsetSettingError struct { Module, Setting string said string + // where says what asks for it — "a file", "a service whose unit", "a directory whose owner" — and + // setToday which of the module's keys a layer sets, for saying it again in the config form's words. + where, setToday string } func (e *UnsetSettingError) Error() string { return e.said } +// unsetInConfigWords is the refusal of a required config value nothing sets, in the words of a module written in the +// config form (novox/hq ADR 0348 §1, §4): `${config:}` and the config verb, never `settings set`. Any other +// error, and any error of a module in the settings form, is returned as it was. +func unsetInConfigWords(err error, m Manifest) error { + var unset *UnsetSettingError + if err == nil || !m.DeclaresConfig() || !errors.As(err, &unset) { + return err + } + return &UnsetSettingError{Module: unset.Module, Setting: unset.Setting, where: unset.where, + setToday: unset.setToday, said: fmt.Sprintf( + "%s has %s that says ${config:%s}, and nothing sets the required config value %q for it — a "+ + "required value is the operator's own and has no default (novox/hq ADR 0348 §4): set it with "+ + "the config verb, for %s on this node, key %q%s", + unset.Module, unset.where, unset.Setting, unset.Setting, unset.Module, unset.Setting, + strings.ReplaceAll(unset.setToday, "no setting is set", "no config value is set"))} +} + // settingInto fills a file's ${setting:…} placeholders from the layers over a module. // // The last layer setting a key wins, which is the node's over the mesh's over the module's own @@ -74,12 +95,13 @@ func settingInto(resource map[string]any, layers []Layer, module string) error { for _, key := range settingsUsed(content) { value, set := settingValue(layers, key) if !set { - return &UnsetSettingError{Module: module, Setting: key, said: fmt.Sprintf( - "%s has a file that says ${setting:%s}, and nothing sets %q for it — an operator's "+ - "value is the assignment's, never the definition's (novox/hq ADR 0112), and only a "+ - "preference has a default in the definition (ADR 0262): "+ - "`settings set %s ` with {%q: …}%s", - module, key, key, module, key, orNoSettings(layers))} + return &UnsetSettingError{Module: module, Setting: key, where: "a file", + setToday: orNoSettings(layers), said: fmt.Sprintf( + "%s has a file that says ${setting:%s}, and nothing sets %q for it — an operator's "+ + "value is the assignment's, never the definition's (novox/hq ADR 0112), and only a "+ + "preference has a default in the definition (ADR 0262): "+ + "`settings set %s ` with {%q: …}%s", + module, key, key, module, key, orNoSettings(layers))} } content = strings.ReplaceAll(content, "${setting:"+key+"}", plainly(value)) } @@ -130,11 +152,12 @@ func settingIntoUnit(resource map[string]any, layers []Layer, module string) err for _, key := range settingsUsed(unit) { value, set := settingValue(layers, key) if !set { - return &UnsetSettingError{Module: module, Setting: key, said: fmt.Sprintf( - "%s has a service whose unit says ${setting:%s}, and nothing sets %q for it — an operator's "+ - "value is the assignment's, never the definition's (novox/hq ADR 0112): "+ - "`settings set %s ` with {%q: …}%s", - module, key, key, module, key, orNoSettings(layers))} + return &UnsetSettingError{Module: module, Setting: key, where: "a service whose unit", + setToday: orNoSettings(layers), said: fmt.Sprintf( + "%s has a service whose unit says ${setting:%s}, and nothing sets %q for it — an operator's "+ + "value is the assignment's, never the definition's (novox/hq ADR 0112): "+ + "`settings set %s ` with {%q: …}%s", + module, key, key, module, key, orNoSettings(layers))} } v := plainly(value) if !unitPart.MatchString(v) { @@ -229,11 +252,13 @@ func settingIntoOwner(resource map[string]any, layers []Layer, module string) er for _, key := range settingsUsed(owner) { value, set := settingValue(layers, key) if !set { - return &UnsetSettingError{Module: module, Setting: key, said: fmt.Sprintf( - "%s has a directory %v whose owner says ${setting:%s}, and nothing sets %q for it — an operator's "+ - "value is the assignment's, never the definition's (novox/hq ADR 0112): "+ - "`settings set %s ` with {%q: …}%s", - module, resource["id"], key, key, module, key, orNoSettings(layers))} + return &UnsetSettingError{Module: module, Setting: key, + where: fmt.Sprintf("a directory %v whose owner", resource["id"]), setToday: orNoSettings(layers), + said: fmt.Sprintf( + "%s has a directory %v whose owner says ${setting:%s}, and nothing sets %q for it — an operator's "+ + "value is the assignment's, never the definition's (novox/hq ADR 0112): "+ + "`settings set %s ` with {%q: …}%s", + module, resource["id"], key, key, module, key, orNoSettings(layers))} } v := plainly(value) id, whole := wholeID(v) diff --git a/internal/catalogue/settings.go b/internal/catalogue/settings.go index 45c992e8..5494c11b 100644 --- a/internal/catalogue/settings.go +++ b/internal/catalogue/settings.go @@ -590,7 +590,7 @@ func JudgeSettings(m Manifest, layers []Layer, adopted bool) error { for k, v := range settled { copied[k] = v } - if err := settingInto(copied, WithDefaults(m, layers), m.Module); err != nil { + if err := unsetInConfigWords(settingInto(copied, WithDefaults(m, layers), m.Module), m); err != nil { return err } } -- 2.54.0 From c0bee5d042d465c77efddd7db903e19f638ceb42 Mon Sep 17 00:00:00 2001 From: mesh-admin Date: Wed, 14 Oct 2026 05:45:48 +0000 Subject: [PATCH 19/31] Judge every manifest against the list of predefined keys of its tree, and carry the list from a build Mesh and node config may hold values only for keys on a fixed, reviewed list (hq ADR 0348 clause 7, design 57 package K2, issue 661). A build reads predefined.json at the root of the repository at the commit it builds and carries it with the commit's generation; module check and the merge gate read it from the tree. (Part 1 of 2: the changes to existing files.) Co-Authored-By: Claude Opus 5.5 --- cmd/mesh-builder/main.go | 5 +++ cmd/mesh-controller/build.go | 38 ++++++++++++++++++++-- cmd/mesh-controller/check.go | 26 ++++++++++++++- cmd/mesh-controller/merge_gate.go | 15 +++++++-- cmd/mesh-controller/stores.go | 6 ++++ internal/builder/builder.go | 53 +++++++++++++++++++++++++++++-- internal/builder/builder_test.go | 2 ++ internal/builder/source.go | 21 +++++++++++- internal/catalogue/config.go | 23 ++++---------- internal/catalogue/config_test.go | 19 +++++------ internal/catalogue/manifest.go | 16 ++++++++++ internal/catalogue/template.go | 2 +- internal/link/build.go | 14 ++++++++ testdata/store-tests | 3 ++ 14 files changed, 206 insertions(+), 37 deletions(-) diff --git a/cmd/mesh-builder/main.go b/cmd/mesh-builder/main.go index 7bf565be..df5735ae 100644 --- a/cmd/mesh-builder/main.go +++ b/cmd/mesh-builder/main.go @@ -418,6 +418,11 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri for _, s := range built.Sources { result.Sources = append(result.Sources, link.BuildSource{Repository: s.Repository, Ref: s.Ref, Paths: s.Paths}) } + // The list of predefined keys the manifest was judged against, for the controller to publish + // (novox/hq ADR 0348 §7). + if p := built.Predefined; p != nil { + result.Predefined = &link.PredefinedList{Keys: p.Keys, Generation: p.Generation} + } say("built", built.Manifest.Module+" from "+short(built.Commit)) } } diff --git a/cmd/mesh-controller/build.go b/cmd/mesh-controller/build.go index d7eb9c2d..ae337692 100644 --- a/cmd/mesh-controller/build.go +++ b/cmd/mesh-controller/build.go @@ -183,8 +183,10 @@ func buildFrom(result link.BuildResult) inventory.Build { kept.Manifest = recordedManifest(result.Manifest, announced) // The module name comes from the manifest, which only exists when the build got that far. if len(kept.Manifest) > 0 { - if m, err := catalogue.ParseManifest(kept.Manifest); err == nil { - kept.Module = m.Module + if list, err := judgedWith(result); err == nil { + if m, err := catalogue.ParseManifestWith(kept.Manifest, list); err == nil { + kept.Module = m.Module + } } } return kept @@ -603,6 +605,11 @@ func saysWhenThePolicyActs(ctx context.Context, inv *inventory.Inventory, module // written with the same values. func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResult) ( catalogue.Manifest, inventory.Build, error) { + // The list of predefined keys as the store keeps it now: another controller process may have kept a newer one. + if err := loadPredefined(ctx, inv); err != nil { + return catalogue.Manifest{}, inventory.Build{}, fmt.Errorf("the list of predefined keys kept cannot be "+ + "read: %w", err) + } kept := buildFrom(result) if err := inv.RecordBuild(ctx, kept); err != nil { return catalogue.Manifest{}, kept, err @@ -613,7 +620,15 @@ func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResu return catalogue.Manifest{}, kept, fmt.Errorf("%s could not build %s:\n%s", result.On, result.Repository, result.Failed) } - manifest, err := catalogue.ParseManifest(kept.Manifest) + // Judged first against the list the build was judged against: the catalogue's at the commit built (novox/hq + // ADR 0348 §7), so a module naming a key its own commit adds is read; and again below against the list the + // mesh publishes, before it is registered. + list, err := judgedWith(result) + if err != nil { + return catalogue.Manifest{}, kept, fmt.Errorf("%s built %s (%s), recorded and not registered: %w", result.On, + result.Repository, short(result.Commit), err) + } + manifest, err := catalogue.ParseManifestWith(kept.Manifest, list) if err != nil { return catalogue.Manifest{}, kept, fmt.Errorf("%s built %s and what came back is not a manifest: %w", result.On, result.Repository, err) @@ -687,6 +702,23 @@ func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResu "back: it is recorded and not registered again — a newer build is", result.On, manifest.Module, short(result.Commit)) } + // **The newest trunk build's list of predefined keys is published** (novox/hq ADR 0348 §7): a build of the + // catalogue that may be registered keeps its list when its commit is newer than the one kept. Then the module + // is judged against the list the mesh publishes, as every manifest read again is: one naming a key that list + // does not hold, or of another type, is refused here and never enters the catalogue. + if carried, err := carriedList(result); err != nil { + return manifest, kept, err + } else if carried != nil { + if err := publishPredefined(ctx, inv, result, carried); err != nil { + fmt.Printf("%s: the list of predefined keys it carried is not published: %v\n", result.ID, err) + } + } + if manifest.NamesPredefined() { + if _, err := catalogue.ParseManifest(kept.Manifest); err != nil { + return manifest, kept, fmt.Errorf("%s built %s (%s), recorded and not registered: judged against the "+ + "list of predefined keys the mesh publishes, %w", result.On, manifest.Module, short(result.Commit), err) + } + } // **A build whose source is unchanged is never a move** (novox/hq issue 280): a rebuild made from // what the build the mesh stands on was made from registers that build's artifacts at the new // commit, so no machine is sent a new digest for a source nobody changed — an image is not diff --git a/cmd/mesh-controller/check.go b/cmd/mesh-controller/check.go index 64b04429..cd7d57bc 100644 --- a/cmd/mesh-controller/check.go +++ b/cmd/mesh-controller/check.go @@ -47,6 +47,24 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error { shelf := catalogue.Shelf{} faulted := map[string]bool{} failed := 0 + // The list of predefined keys at the root of the repository above each manifest (novox/hq ADR 0348 §7), read + // once per repository: a manifest naming a key is judged against the list its own tree holds. + lists := map[string]catalogue.PredefinedList{} + listOf := func(path string) (catalogue.PredefinedList, error) { + root := catalogue.RepositoryRoot(filepath.Dir(path)) + if root == "" { + return nil, nil + } + if list, read := lists[root]; read { + return list, nil + } + list, err := catalogue.ReadPredefined(root) + if err != nil { + return nil, err + } + lists[root] = list + return list, nil + } for _, path := range paths { raw, err := os.ReadFile(path) if err != nil { @@ -54,7 +72,13 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error { failed++ continue } - m, err := catalogue.ParseManifest(raw) + list, err := listOf(path) + if err != nil { + fmt.Fprintf(out, "%s: %v\n", path, err) + failed++ + continue + } + m, err := catalogue.ParseManifestWith(raw, list) if err == nil { // Each template's text read from the tree beside the manifest, and judged as inline content is // (novox/hq ADR 0349 §2): a missing template is refused by name. diff --git a/cmd/mesh-controller/merge_gate.go b/cmd/mesh-controller/merge_gate.go index 78b5332a..b555e4e5 100644 --- a/cmd/mesh-controller/merge_gate.go +++ b/cmd/mesh-controller/merge_gate.go @@ -550,8 +550,17 @@ func shelfWithChange(base map[string]catalogue.Manifest, sources map[string]inve out[k] = m } var failures []string + // The list of predefined keys at the tree's root (novox/hq ADR 0348 §7): what the build of this tree judges each + // manifest against and registration publishes, so what the gate judges each against too. A list that is + // wrong is refused by name, and every manifest is then judged against none. + predefined, err := catalogue.ReadPredefined(tree) + if err != nil { + failures = append(failures, fmt.Sprintf("%s: the controller judging this (%s) refuses it — %s", + catalogue.PredefinedFile, version, oneLine(err.Error()))) + predefined = nil + } found := map[string]string{} // module → its directory in the tree - err := filepath.WalkDir(tree, func(path string, d fs.DirEntry, err error) error { + err = filepath.WalkDir(tree, func(path string, d fs.DirEntry, err error) error { if err != nil { return err } @@ -575,7 +584,7 @@ func shelfWithChange(base map[string]catalogue.Manifest, sources map[string]inve if err != nil { return err } - m, err := catalogue.ParseManifest(raw) + m, err := catalogue.ParseManifestWith(raw, predefined) if err == nil { // Each template's text from the tree beside the manifest, as the build reads it back from the archive it // packs (novox/hq ADR 0349 §2): so a change to a template alone is a change to its module. @@ -610,7 +619,7 @@ func shelfWithChange(base map[string]catalogue.Manifest, sources map[string]inve // As registration reads a built manifest: strictly, again (takeIn). var raw []byte if raw, err = json.Marshal(resolved); err == nil { - resolved, err = catalogue.ParseManifest(raw) + resolved, err = catalogue.ParseManifestWith(raw, predefined) } } if err == nil { diff --git a/cmd/mesh-controller/stores.go b/cmd/mesh-controller/stores.go index aeea12c1..f2db129a 100644 --- a/cmd/mesh-controller/stores.go +++ b/cmd/mesh-controller/stores.go @@ -3,6 +3,7 @@ package main import ( "context" "fmt" + "os" "time" "github.com/novox/mesh-controller/internal/catalogue" @@ -124,6 +125,11 @@ func openInventory(ctx context.Context) (*inventory.Inventory, error) { if aliases, err := inv.Aliases(ctx); err == nil { catalogue.UseAliases(aliases) } + // And the list of predefined keys the mesh publishes (novox/hq ADR 0348 §7), which every manifest read again + // is judged against. A store a migration has not reached yet keeps none, and nothing names a key then. + if err := loadPredefined(ctx, inv); err != nil { + fmt.Fprintf(os.Stderr, "the list of predefined keys kept cannot be read: %v\n", err) + } return inv, nil } diff --git a/internal/builder/builder.go b/internal/builder/builder.go index c2c1cbf9..0729a351 100644 --- a/internal/builder/builder.go +++ b/internal/builder/builder.go @@ -108,6 +108,17 @@ type Result struct { // recipe and the package it compiles; for any other, nothing is said and the whole of what the build // sees stays its source, as before. Sources []BuildSource + + // Predefined is the list of predefined keys at the root of the repository at the commit built (novox/hq ADR + // 0348 §7), which the manifest was judged against; nil for a repository that holds none. + Predefined *Predefined +} + +// Predefined is the list of predefined keys a build read: its text, canonical (catalogue.PredefinedList), and +// the generation of the commit built — how many commits it stands on, which orders two builds of one trunk. +type Predefined struct { + Keys []byte + Generation int } // BuildSource is the build source a build read in one repository (novox/hq ADR 0267). @@ -261,11 +272,28 @@ func build(ctx context.Context, run Runner, publish Publisher, "%s has no %s at %s, so there is nothing saying what it is: %w", repository, ManifestName, describe(path), err) } - manifest, err := catalogue.ParseManifest(raw) + // **The list of predefined keys at the commit built** (novox/hq ADR 0348 §7): predefined.json at the + // repository's root, read strictly; a list that is wrong fails the build, and the manifest is judged against + // the list of its own tree, never against one the mesh published from another commit. + predefined, err := catalogue.ReadPredefined(tree) + if err != nil { + say("predefined", "INVALID: %v", err) + return Result{}, err + } + manifest, err := catalogue.ParseManifestWith(raw, predefined) if err != nil { say("manifest", "INVALID: %v", err) return Result{}, err } + var listed *Predefined + if predefined != nil { + text, err := predefined.Canonical() + if err != nil { + return Result{}, err + } + listed = &Predefined{Keys: text, Generation: generationOf(ctx, run, tree, commit)} + say("predefined", "%d key(s) at %s, generation %d", len(predefined), short(commit), listed.Generation) + } // What is judged only where a definition is written (novox/hq ADR 0326 §3, issue 574): a build is // where a changed definition is first read, so it is refused here before anything is made of it. // Judged with each template's text in its file, as written in the tree (novox/hq ADR 0349 §2): a template the @@ -291,6 +319,11 @@ func build(ctx context.Context, run Runner, publish Publisher, if src.tree, err = gitTree(ctx, run, tree, path); err != nil { src.notPinned("its tree could not be named: " + err.Error()) } + // A module falling back to a predefined key was judged against the list, so a change to the list is a change + // to it: the planner rebuilds it, and its build publishes the list (ADR 0348 §7). + if predefined != nil && manifest.NamesPredefined() { + src.rootHas(catalogue.PredefinedFile) + } // A build-time credential, written into the build context as .npmrc, but ONLY for a module that // asks for it: a `package` artifact (which publishes), or an image whose Dockerfile COPYs .npmrc. @@ -451,7 +484,23 @@ func build(ctx context.Context, run Runner, publish Publisher, return Result{Manifest: resolved, Commit: commit, Built: built, Against: against(within, manifest, stoodOn), Read: readBy(manifest), Source: fingerprint, Trunk: trunk, OnTrunk: onTrunk, Branches: branches, Sources: sources, - ReuseKey: key.Key, ReuseParts: key.Parts, NoKey: key.Why, Reused: reused, Why: builtWhy}, nil + ReuseKey: key.Key, ReuseParts: key.Parts, NoKey: key.Why, Reused: reused, Why: builtWhy, + Predefined: listed}, nil +} + +// generationOf is how many commits the commit built stands on, itself included: of two commits on one trunk the +// later has more, so the controller keeps the list of the newer one whatever order it hears them in (ADR 0348 +// §7). 0 when git could not say. +func generationOf(ctx context.Context, run Runner, tree, commit string) int { + out, err := run(ctx, tree, "git", "rev-list", "--count", commit) + if err != nil { + return 0 + } + n, err := strconv.Atoi(strings.TrimSpace(out)) + if err != nil || n < 0 { + return 0 + } + return n } // takeACheckBuild is the walk's look for a check build of its key (novox/hq ADR 0350 §2, §3): the key taken by diff --git a/internal/builder/builder_test.go b/internal/builder/builder_test.go index afe57572..4bfe447f 100644 --- a/internal/builder/builder_test.go +++ b/internal/builder/builder_test.go @@ -83,6 +83,8 @@ func (r *recorded) run(_ context.Context, dir, name string, args ...string) (str return r.tree + "\n", nil case name == "git" && len(args) > 0 && args[0] == "rev-parse": return "c0ffeec0ffeec0ffeec0ffeec0ffeec0ffeec0ff\n", nil + case name == "git" && len(args) > 1 && args[0] == "rev-list" && args[1] == "--count": + return "42\n", nil } return "", nil } diff --git a/internal/builder/source.go b/internal/builder/source.go index 7d9b9c0e..4d033a33 100644 --- a/internal/builder/source.go +++ b/internal/builder/source.go @@ -58,6 +58,9 @@ type sourceInputs struct { // whole when an artifact's is not known (novox/hq ADR 0267). own map[string]bool ownIsAll bool + // root is the module's build source in its own repository outside its directory, relative to the + // repository's root: the list of predefined keys a manifest naming one was judged against (ADR 0348 §7). + root map[string]bool // read is, per context (repository and ref), the build source read there; nil for one read whole. read map[string]map[string]bool readAs map[string]catalogue.ArtifactContext @@ -86,6 +89,9 @@ func (s *sourceInputs) copy() *sourceInputs { for k, v := range s.own { c.own[k] = v } + for k, v := range s.root { + c.root[k] = v + } for k, v := range s.read { if v == nil { c.read[k] = nil @@ -112,7 +118,7 @@ func (s *sourceInputs) contextOffHead() { func newSourceInputs(module string) *sourceInputs { return &sourceInputs{module: module, contexts: map[string]string{}, toolchains: map[string]string{}, - own: map[string]bool{}, read: map[string]map[string]bool{}, readAs: map[string]catalogue.ArtifactContext{}, + own: map[string]bool{}, root: map[string]bool{}, read: map[string]map[string]bool{}, readAs: map[string]catalogue.ArtifactContext{}, contextsAtHead: true} } @@ -126,6 +132,16 @@ func (s *sourceInputs) ownHas(entries ...string) { } } +// rootHas adds entries, relative to the repository's root, to the module's build source in its own repository. +func (s *sourceInputs) rootHas(entries ...string) { + if s == nil { + return + } + for _, e := range entries { + s.root[e] = true + } +} + // ownWhole says an artifact's build source in the module's own repository is not known: the module's // whole directory is its source, as it was before. func (s *sourceInputs) ownWhole() { @@ -178,6 +194,9 @@ func (s *sourceInputs) buildSources() []BuildSource { for e := range s.own { own = append(own, withPrefix(s.prefix, e)) } + for e := range s.root { + own = append(own, e) + } sort.Strings(own) out = append(out, BuildSource{Paths: compactSorted(own)}) } diff --git a/internal/catalogue/config.go b/internal/catalogue/config.go index 90e9ea71..6feaee70 100644 --- a/internal/catalogue/config.go +++ b/internal/catalogue/config.go @@ -61,18 +61,6 @@ const ( // accountFacts are what the mesh places for a module's account, never config values (ADR 0348 §11). var accountFacts = map[string]bool{"puid": true, "pgid": true} -// PredefinedKey is one key of the catalogue's list of predefined keys, as far as a module's value falling back -// to it is judged here: its type, and whether it is trusted (ADR 0348 §7, §8). -type PredefinedKey struct { - Type string - Trusted bool -} - -// predefinedKeys is the list of predefined keys, nil until it is present. The list (`predefined.json` at the -// catalogue's root) arrives with design 57's package K2 (issue 661); until then a value naming one is refused -// saying the list is not yet present. -var predefinedKeys map[string]PredefinedKey - // configForms is which of the two forms a manifest was written in, by the words of each it uses. type configForms struct { config, settings []string @@ -259,12 +247,13 @@ func ConfigProblems(m Manifest) []string { "(novox/hq ADR 0348 §4, ADR 0112)", word) } if d.Predefined != "" { - p, listed := predefinedKeys[d.Predefined] + p, listed := m.predefined[d.Predefined] switch { - case predefinedKeys == nil: - say("falls back to the predefined key %q, and the list of predefined keys (the catalogue's "+ - "predefined.json, design 57 package K2) is not yet present: no value can fall back to one "+ - "until it is; leave \"predefined\" out for now (novox/hq ADR 0348 §7)", d.Predefined) + case m.predefined == nil: + say("falls back to the predefined key %q, and the list of predefined keys (%s at the root of "+ + "the catalogue's repository) is not present where it is judged: a value falls back only to a "+ + "key on that list; judge it in the catalogue, or leave \"predefined\" out (novox/hq ADR 0348 §7)", + d.Predefined, PredefinedFile) case !listed: say("falls back to %q, which is not on the list of predefined keys: name a key on it, or leave "+ "\"predefined\" out (novox/hq ADR 0348 §7)", d.Predefined) diff --git a/internal/catalogue/config_test.go b/internal/catalogue/config_test.go index 1a2d8499..2cbaeea1 100644 --- a/internal/catalogue/config_test.go +++ b/internal/catalogue/config_test.go @@ -157,20 +157,20 @@ func TestConfigRefusesAnObjectOrListNotTrusted(t *testing.T) { } } -// withPredefined is the list of predefined keys as package K2 will give it, for the length of a test. -func withPredefined(t *testing.T, keys map[string]PredefinedKey) { +// withPredefined is the list of predefined keys the mesh published, for the length of a test. +func withPredefined(t *testing.T, keys PredefinedList) { t.Helper() - was := predefinedKeys - predefinedKeys = keys - t.Cleanup(func() { predefinedKeys = was }) + was := ReplacePublished(keys) + t.Cleanup(func() { ReplacePublished(was) }) } -// Rule 5: a predefined key not on the list, or of another type. Until the list is present (package K2), a value -// naming one is refused saying so. +// Rule 5: a predefined key not on the list, or of another type. Where no list is present, a value naming one is +// refused saying so. func TestConfigRefusesAPredefinedKeyNotOnTheListOrOfAnotherType(t *testing.T) { + withPredefined(t, nil) m := inConfigForm() declaration(m, "font-size")["predefined"] = "font-size" - refused(t, m, `"font-size"`, "the list of predefined keys", "not yet present") + refused(t, m, `"font-size"`, "the list of predefined keys", "is not present") withPredefined(t, map[string]PredefinedKey{"font-size": {Type: "number"}, "accent": {Type: "string"}}) if _, err := parseRaw(t, m); err != nil { @@ -186,9 +186,10 @@ func TestConfigRefusesAPredefinedKeyNotOnTheListOrOfAnotherType(t *testing.T) { // Rule 6: an untrusted value falling back to a trusted predefined key. func TestConfigRefusesAnUntrustedValueFallingBackToATrustedKey(t *testing.T) { + withPredefined(t, nil) m := inConfigForm() declaration(m, "theme")["predefined"] = "theme" - refused(t, m, `"theme"`, "not yet present") + refused(t, m, `"theme"`, "is not present") withPredefined(t, map[string]PredefinedKey{"theme": {Type: "string", Trusted: true}}) refused(t, m, `"theme"`, "trusted", "is not") diff --git a/internal/catalogue/manifest.go b/internal/catalogue/manifest.go index 49866bdd..480ce761 100644 --- a/internal/catalogue/manifest.go +++ b/internal/catalogue/manifest.go @@ -541,6 +541,10 @@ type Manifest struct { // machine's declaration by name until the controller is updated (LeftOut). unknown string + // predefined is the list of predefined keys it was judged against (predefined.go, novox/hq ADR 0348 §7): + // read again — its templates filled from the tree — it is judged against the same list. + predefined PredefinedList + // bestEffort marks the view of a left-out module that only its backup lines are made from // (backupView, novox/hq ADR 0262): a line of it that cannot be placed is said in the plan's list of // what could not be placed, never an error that would cost the whole machine its declaration. @@ -1600,6 +1604,17 @@ func SecretLocal(to, local string) string { func (m Manifest) UnknownField() string { return m.unknown } func ParseManifest(raw []byte) (Manifest, error) { + return parseManifest(raw, PublishedPredefined()) +} + +// ParseManifestWith is ParseManifest judging every value that falls back to a predefined key against the list +// given — the one read from the tree the manifest is in (predefined.go, novox/hq ADR 0348 §7) — where +// ParseManifest judges against the list the mesh published. Nil is no list: a value naming a key is refused. +func ParseManifestWith(raw []byte, predefined PredefinedList) (Manifest, error) { + return parseManifest(raw, predefined) +} + +func parseManifest(raw []byte, predefined PredefinedList) (Manifest, error) { var m Manifest // Strictly. **An unknown key is refused**, which is the discipline the host's declaration // parser has and manifests lacked (novox/hq 04-ISSUES/003): a `scope:` key survived in five @@ -1627,6 +1642,7 @@ func ParseManifest(raw []byte) (Manifest, error) { } return Manifest{}, fmt.Errorf("this is not a module manifest: %w", err) } + m.predefined = predefined var problems []string if !name.MatchString(m.Module) { diff --git a/internal/catalogue/template.go b/internal/catalogue/template.go index 2a363bf2..d4aa104a 100644 --- a/internal/catalogue/template.go +++ b/internal/catalogue/template.go @@ -312,7 +312,7 @@ func WithTemplatesFrom(m Manifest, dir string) (Manifest, error) { if err != nil { return Manifest{}, err } - return ParseManifest(raw) + return parseManifest(raw, m.predefined) } // HeldTemplateProblem is why a manifest cannot be held by the mesh because it still names a template, or "": the diff --git a/internal/link/build.go b/internal/link/build.go index 6eb16588..a8d56c18 100644 --- a/internal/link/build.go +++ b/internal/link/build.go @@ -334,6 +334,12 @@ type BuildResult struct { ReusedFrom *ReusedFrom `json:"reused-from,omitempty"` BuiltWhy string `json:"built-why,omitempty"` + // Predefined is the list of predefined keys at the root of the repository at the commit built (novox/hq ADR + // 0348 §7), which the manifest was judged against: the controller publishes the catalogue's from a trunk + // build, the newest by Generation winning. Absent for a repository holding none, and from a builder that + // predates the list. + Predefined *PredefinedList `json:"predefined,omitempty"` + // Source is the request's, echoed: the seat form of the repository, for whoever registers. Source *SourceOnSeat `json:"source,omitempty"` @@ -346,6 +352,14 @@ type BuildResult struct { Checked *CheckRequest `json:"check,omitempty"` } +// PredefinedList is the list of predefined keys a build read (novox/hq ADR 0348 §7): Keys its canonical text, a +// JSON object of keys, and Generation how many commits the commit built stands on — of two commits on one trunk +// the later has more, which is what orders two builds' lists; 0 when the builder could not say. +type PredefinedList struct { + Keys json.RawMessage `json:"keys"` + Generation int `json:"generation,omitempty"` +} + // ReadRepository is a repository a build read source from besides the module's own, at the branch, // tag or commit it read. Spelled here as well as in the catalogue and the inventory, for the reason // MadeArtifact is: one direction of dependency. diff --git a/testdata/store-tests b/testdata/store-tests index d3af9659..344072f4 100644 --- a/testdata/store-tests +++ b/testdata/store-tests @@ -64,3 +64,6 @@ ./internal/inventory TestPlacementMovesOutOfTheSettingsLayers ./internal/inventory TestASettingRoutesPlacementBesideTheLayer ./cmd/mesh-controller TestADeclarationIsTheSameAfterPlacementMoves +./internal/inventory TestANewerBuildsListReplacesTheKeptOneAndAnOlderOneHeardLaterDoesNot +./cmd/mesh-controller TestTheMergeGateJudgesEachManifestAgainstTheListOfTheTree +./cmd/mesh-controller TestRegistrationPublishesTheNewestTrunkListAndJudgesEveryModuleAgainstIt -- 2.54.0 From 9f2e16ff704cc7dd1ea11b60a0ec73f2a1a969cb Mon Sep 17 00:00:00 2001 From: mesh-admin Date: Wed, 14 Oct 2026 05:46:49 +0000 Subject: [PATCH 20/31] Keep the newest trunk build's list of predefined keys, and prove each refusal The list itself, read strictly (catalogue/predefined.go); the controller keeps a catalogue trunk build's list in its store when its commit's generation is larger than the one kept (migration 0098), so an older build's list heard later never replaces a newer one, and loads it with its store (hq ADR 0348 clause 7, issue 661). Part 2 of 2: the new files. Co-Authored-By: Claude Opus 5.5 --- cmd/mesh-controller/predefined.go | 93 +++++++++++++++++++ .../0098-the-list-of-predefined-keys.sql | 20 ++++ internal/inventory/predefined.go | 76 +++++++++++++++ internal/inventory/predefined_test.go | 69 ++++++++++++++ 4 files changed, 258 insertions(+) create mode 100644 cmd/mesh-controller/predefined.go create mode 100644 internal/inventory/migrations/0098-the-list-of-predefined-keys.sql create mode 100644 internal/inventory/predefined.go create mode 100644 internal/inventory/predefined_test.go diff --git a/cmd/mesh-controller/predefined.go b/cmd/mesh-controller/predefined.go new file mode 100644 index 00000000..7b93b748 --- /dev/null +++ b/cmd/mesh-controller/predefined.go @@ -0,0 +1,93 @@ +package main + +import ( + "context" + "fmt" + "path" + "strings" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +// The list of predefined keys the mesh publishes (novox/hq ADR 0348 §7, design 57 package K2, issue 661). +// +// **The newest trunk build of the catalogue wins.** A build of the catalogue reads predefined.json at the commit +// it builds and carries it with the commit's generation (link.PredefinedList). Registration keeps it in the +// controller's store when that generation is larger than the one kept (inventory.KeepPredefined), so an older +// build's list heard later never replaces a newer one, and every manifest the controller reads again is judged +// against the list kept (catalogue.UsePredefined). mesh-config publishes it on the bus from there (design 57 +// package K3); the controller keeps no config value, only the keys. + +// fromTheCatalogue is whether a build is of the catalogue's repository, whose list alone the mesh publishes: by +// its name, in the URL form or the seat form. +func fromTheCatalogue(result link.BuildResult) bool { + for _, repository := range []string{result.Repository, sourceRepository(result)} { + name := strings.TrimSuffix(path.Base(strings.TrimRight(repository, "/")), ".git") + if name == catalogue.CatalogueRepository { + return true + } + } + return false +} + +func sourceRepository(result link.BuildResult) string { + if result.Source == nil { + return "" + } + return result.Source.Repository +} + +// carriedList is the list a build of the catalogue carried, read strictly; nil for a build of another +// repository, or one that carried none. +func carriedList(result link.BuildResult) (catalogue.PredefinedList, error) { + if result.Predefined == nil || !fromTheCatalogue(result) { + return nil, nil + } + list, err := catalogue.ParsePredefined(result.Predefined.Keys) + if err != nil { + return nil, fmt.Errorf("%s carried a list of predefined keys that is refused: %w", result.ID, err) + } + return list, nil +} + +// judgedWith is the list a build's manifest is first judged against at registration: the catalogue's own at the +// commit built, which its build judged it against, where it carried one; the list kept otherwise. +func judgedWith(result link.BuildResult) (catalogue.PredefinedList, error) { + list, err := carriedList(result) + if err != nil || list != nil { + return list, err + } + return catalogue.PublishedPredefined(), nil +} + +// loadPredefined makes the list kept in the store the one this controller judges manifests against. A store +// keeping none leaves the list it has. +func loadPredefined(ctx context.Context, inv *inventory.Inventory) error { + kept, has, err := inv.PredefinedKept(ctx) + if err != nil || !has { + return err + } + catalogue.UsePredefined(kept.Keys) + return nil +} + +// publishPredefined keeps a trunk build's list when it is newer than the one kept, says so, and loads what the +// store keeps then — this build's, or a newer one another controller process kept. +func publishPredefined(ctx context.Context, inv *inventory.Inventory, result link.BuildResult, + list catalogue.PredefinedList) error { + kept, err := inv.KeepPredefined(ctx, inventory.PredefinedRecord{Keys: list, Repository: result.Repository, + Commit: result.Commit, Generation: result.Predefined.Generation, Build: result.ID}) + if err != nil { + return err + } + if kept { + fmt.Printf("%s: the list of predefined keys at %s (generation %d) is published: %s\n", result.ID, + short(result.Commit), result.Predefined.Generation, strings.Join(list.Keys(), ", ")) + } else { + fmt.Printf("%s: the list of predefined keys at %s (generation %d) is not newer than the one kept, which "+ + "stands\n", result.ID, short(result.Commit), result.Predefined.Generation) + } + return loadPredefined(ctx, inv) +} diff --git a/internal/inventory/migrations/0098-the-list-of-predefined-keys.sql b/internal/inventory/migrations/0098-the-list-of-predefined-keys.sql new file mode 100644 index 00000000..fa4d42b4 --- /dev/null +++ b/internal/inventory/migrations/0098-the-list-of-predefined-keys.sql @@ -0,0 +1,20 @@ +-- The list of predefined keys (novox/hq ADR 0348 clause 7, design 57 package K2, issue 661). +-- +-- Mesh config and node config hold values only for the keys in the catalogue's `predefined.json`. A build of +-- the catalogue reads the file at the commit it builds and carries it; the controller keeps the newest trunk +-- build's, which every manifest it reads again is judged against, and which mesh-config publishes from here +-- (design 57 package K3). One list for the mesh: one row. +-- +-- **The newest trunk build wins**, by the generation of the commit built — how many commits it stands on, which +-- of two commits on one trunk is larger for the later — so a list heard late from an older build never replaces +-- a newer one. Not a config value: a key, its type, description, validation and whether it is trusted, all of it +-- public in the catalogue's repository. +create table predefined_list ( + one boolean primary key default true check (one), + keys jsonb not null, + repository text not null, + commit text not null, + generation bigint not null check (generation > 0), + build text not null, + kept_at timestamptz not null default now() +); diff --git a/internal/inventory/predefined.go b/internal/inventory/predefined.go new file mode 100644 index 00000000..03fc588e --- /dev/null +++ b/internal/inventory/predefined.go @@ -0,0 +1,76 @@ +package inventory + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/jackc/pgx/v5" + + "github.com/novox/mesh-controller/internal/catalogue" +) + +// The list of predefined keys the mesh publishes (novox/hq ADR 0348 §7, design 57 package K2): the newest trunk +// build's of the catalogue, by the generation of the commit built. + +// PredefinedRecord is the list kept, and the build it came from. +type PredefinedRecord struct { + Keys catalogue.PredefinedList + Repository string + Commit string + Generation int + Build string + KeptAt time.Time +} + +// KeepPredefined keeps a trunk build's list when its commit is newer than the one kept — a larger generation — +// and answers whether it did. An older build's list, or one of the same generation, heard later is not kept: the +// newer one stands, whatever order the outcomes arrive in. One statement, so two builds heard at once cannot both +// win. +func (i *Inventory) KeepPredefined(ctx context.Context, r PredefinedRecord) (bool, error) { + if r.Keys == nil { + return false, errors.New("no list to keep") + } + if r.Generation <= 0 { + return false, fmt.Errorf("the build %s did not say its commit's generation, so whether its list is newer "+ + "than the one kept cannot be said", r.Build) + } + if problems := catalogue.PredefinedProblems(r.Keys); len(problems) > 0 { + return false, fmt.Errorf("the list of %s is refused: %v", r.Build, problems) + } + text, err := r.Keys.Canonical() + if err != nil { + return false, err + } + tag, err := i.store.Pool().Exec(ctx, ` + insert into predefined_list (keys, repository, commit, generation, build) + values ($1, $2, $3, $4, $5) + on conflict (one) do update set keys = excluded.keys, repository = excluded.repository, + commit = excluded.commit, generation = excluded.generation, build = excluded.build, kept_at = now() + where predefined_list.generation < excluded.generation`, + text, r.Repository, r.Commit, r.Generation, r.Build) + if err != nil { + return false, err + } + return tag.RowsAffected() == 1, nil +} + +// PredefinedKept is the list kept, and false when none is yet. +func (i *Inventory) PredefinedKept(ctx context.Context) (PredefinedRecord, bool, error) { + var r PredefinedRecord + var text []byte + err := i.store.Pool().QueryRow(ctx, + `select keys, repository, commit, generation, build, kept_at from predefined_list`). + Scan(&text, &r.Repository, &r.Commit, &r.Generation, &r.Build, &r.KeptAt) + if errors.Is(err, pgx.ErrNoRows) { + return PredefinedRecord{}, false, nil + } + if err != nil { + return PredefinedRecord{}, false, err + } + if r.Keys, err = catalogue.ParsePredefined(text); err != nil { + return PredefinedRecord{}, false, fmt.Errorf("the list kept from %s cannot be read: %w", r.Build, err) + } + return r, true, nil +} diff --git a/internal/inventory/predefined_test.go b/internal/inventory/predefined_test.go new file mode 100644 index 00000000..d2ab52d2 --- /dev/null +++ b/internal/inventory/predefined_test.go @@ -0,0 +1,69 @@ +package inventory + +import ( + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" +) + +// The list of predefined keys the controller keeps (novox/hq ADR 0348 §7 and its check "a newer trunk build's +// list replaces the published one, and an older build's list heard later does not"; design 57 package K2). + +func listOf(t *testing.T, text string) catalogue.PredefinedList { + t.Helper() + l, err := catalogue.ParsePredefined([]byte(text)) + if err != nil { + t.Fatal(err) + } + return l +} + +func TestANewerBuildsListReplacesTheKeptOneAndAnOlderOneHeardLaterDoesNot(t *testing.T) { + inv := ForTest(t) + ctx := t.Context() + if _, has, err := inv.PredefinedKept(ctx); err != nil || has { + t.Fatalf("a fresh store keeps a list: %v %v", has, err) + } + older := PredefinedRecord{Keys: listOf(t, `{"font-size": {"type": "number", "description": "points"}}`), + Repository: "novox/mesh-catalog", Commit: "aaaa", Generation: 100, Build: "b-older"} + newer := PredefinedRecord{Keys: listOf(t, `{"font-size": {"type": "number", "description": "points"}, + "agent-managed-settings": {"type": "object", "description": "a JSON object", "trusted": true}}`), + Repository: "novox/mesh-catalog", Commit: "bbbb", Generation: 101, Build: "b-newer"} + + if kept, err := inv.KeepPredefined(ctx, older); err != nil || !kept { + t.Fatalf("the first list was not kept: %v %v", kept, err) + } + if kept, err := inv.KeepPredefined(ctx, newer); err != nil || !kept { + t.Fatalf("a newer build's list did not replace the kept one: %v %v", kept, err) + } + // The older build's outcome heard after the newer one's. + if kept, err := inv.KeepPredefined(ctx, older); err != nil || kept { + t.Fatalf("an older build's list heard later replaced the newer one: %v %v", kept, err) + } + // Nor one of the same generation: the one kept stands. + same := newer + same.Keys, same.Build = older.Keys, "b-again" + if kept, err := inv.KeepPredefined(ctx, same); err != nil || kept { + t.Fatalf("a list of the same generation replaced the kept one: %v %v", kept, err) + } + got, has, err := inv.PredefinedKept(ctx) + if err != nil || !has { + t.Fatal(has, err) + } + if got.Build != "b-newer" || got.Commit != "bbbb" || got.Generation != 101 || len(got.Keys) != 2 || + !got.Keys["agent-managed-settings"].Trusted { + t.Fatalf("kept %+v", got) + } + + // A build that could not say its generation, or carried a wrong list, is never kept. + unordered := newer + unordered.Generation = 0 + if kept, err := inv.KeepPredefined(ctx, unordered); err == nil || kept { + t.Fatalf("a list without a generation was kept: %v", err) + } + wrong := newer + wrong.Generation, wrong.Keys = 200, catalogue.PredefinedList{"accent": {Type: "object", Description: "x"}} + if kept, err := inv.KeepPredefined(ctx, wrong); err == nil || kept { + t.Fatalf("a wrong list was kept: %v", err) + } +} -- 2.54.0 From 2941ef36956915691941cfeee7bcfe6fbe919174 Mon Sep 17 00:00:00 2001 From: mesh-admin Date: Wed, 14 Oct 2026 05:47:44 +0000 Subject: [PATCH 21/31] Read the list of predefined keys strictly, and prove the build, check, gate and registration on it Part 3: the list's own reading (catalogue/predefined.go) and the tests of the build, module check, merge gate and registration (hq ADR 0348 clause 7, issue 661). Co-Authored-By: Claude Opus 5.5 --- internal/builder/predefined_test.go | 106 +++++++++++++ internal/catalogue/predefined.go | 236 ++++++++++++++++++++++++++++ 2 files changed, 342 insertions(+) create mode 100644 internal/builder/predefined_test.go create mode 100644 internal/catalogue/predefined.go diff --git a/internal/builder/predefined_test.go b/internal/builder/predefined_test.go new file mode 100644 index 00000000..71ac4fec --- /dev/null +++ b/internal/builder/predefined_test.go @@ -0,0 +1,106 @@ +package builder + +import ( + "context" + "slices" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" +) + +// A build reads the list of predefined keys at the root of the repository at the commit it builds (novox/hq ADR +// 0348 §7, design 57 package K2, issue 661): it refuses a list that is wrong, judges the manifest against it, and +// carries it with the commit's generation for the controller to publish. + +const theFirstKeys = `{ + "font-size": {"type": "number", "description": "the size text is shown in, in points", "minimum": 6, "maximum": 72}, + "agent-managed-settings": {"type": "object", "description": "the managed settings of every agent session, a JSON object", "trusted": true} +}` + +// fallingBack is a module of the catalogue whose font size falls back to the predefined key named. +func fallingBack(key, kind string) string { + def := "10" + if kind == "string" { + def = `"10"` + } + return `{"module":"dunst","version":"1", + "build":{"artifacts":[{"name":"look","kind":"archive","from":"files"}]}, + "config":{"font-size":{"type":"` + kind + `","description":"the notification's font size, in points", + "default":` + def + `,"predefined":"` + key + `"}}, + "resources":[ + {"id":"theme","type":"archive","path":"/opt/dunst","artifact":"look"}, + {"id":"rc","type":"file","path":"/etc/xdg/dunst/dunstrc","content":"font = Inter ${config:font-size}\n"}]}` +} + +func buildInTheCatalogue(t *testing.T, manifest string, list *string) (Result, error) { + t.Helper() + r := &recorded{contents: map[string]string{"modules/dunst/" + ManifestName: manifest, + "modules/dunst/files/theme.conf": "dark", "README.md": "the catalogue"}} + if list != nil { + r.contents[catalogue.PredefinedFile] = *list + } + return Build(context.Background(), onTrunk{recorded: r}.run, r, "https://forge.invalid/mesh-catalog.git", + "modules/dunst", "", t.TempDir(), nil, Npmrc{}, GitCredential{}, nil) +} + +func TestABuildNamingAPredefinedKeyIsJudgedAgainstItsTreesListAndCarriesIt(t *testing.T) { + list := theFirstKeys + got, err := buildInTheCatalogue(t, fallingBack("font-size", "number"), &list) + if err != nil { + t.Fatalf("a module falling back to font-size did not build: %v", err) + } + if got.Predefined == nil || got.Predefined.Generation != 42 { + t.Fatalf("the list was not carried with its generation: %+v", got.Predefined) + } + carried, err := catalogue.ParsePredefined(got.Predefined.Keys) + if err != nil || len(carried) != 2 || carried["font-size"].Type != catalogue.ConfigNumber { + t.Fatalf("the list carried is %s (%v)", got.Predefined.Keys, err) + } + // A change to the list is a change to the module that was judged against it. + if len(got.Sources) == 0 || !slices.Contains(got.Sources[0].Paths, catalogue.PredefinedFile) { + t.Fatalf("its build source leaves the list out: %+v", got.Sources) + } + + for _, c := range []struct{ manifest, says string }{ + {fallingBack("font-scale", "number"), `"font-scale", which is not on the list`}, + {fallingBack("font-size", "string"), "which is a number, and this value is a string"}, + } { + if _, err := buildInTheCatalogue(t, c.manifest, &list); err == nil || !strings.Contains(err.Error(), c.says) { + t.Errorf("wanted the build refused saying %q, got %v", c.says, err) + } + } +} + +func TestABuildRefusesAWrongList(t *testing.T) { + wrong := `{"font-size": {"type": "number", "description": "points", "unit": "pt"}}` + if _, err := buildInTheCatalogue(t, fallingBack("font-size", "number"), &wrong); err == nil || + !strings.Contains(err.Error(), `unknown field "unit"`) { + t.Fatalf("a wrong list was built with: %v", err) + } +} + +// A repository without the list builds as before, carrying none; a value naming a key there is refused. +func TestABuildWithoutAListCarriesNone(t *testing.T) { + plain := strings.Replace(fallingBack("font-size", "number"), `,"predefined":"font-size"`, "", 1) + got, err := buildInTheCatalogue(t, plain, nil) + if err != nil { + t.Fatal(err) + } + if got.Predefined != nil { + t.Fatalf("a list was carried from a tree holding none: %+v", got.Predefined) + } + if slices.Contains(got.Sources[0].Paths, catalogue.PredefinedFile) { + t.Fatalf("the build source names a list: %+v", got.Sources) + } + if _, err := buildInTheCatalogue(t, fallingBack("font-size", "number"), nil); err == nil || + !strings.Contains(err.Error(), "is not present") { + t.Fatalf("a key named where no list is: %v", err) + } + // A list present and a module naming no key: carried, and not the module's build source. + list := theFirstKeys + got, err = buildInTheCatalogue(t, plain, &list) + if err != nil || got.Predefined == nil || slices.Contains(got.Sources[0].Paths, catalogue.PredefinedFile) { + t.Fatalf("a module naming no key: %+v %+v %v", got.Predefined, got.Sources, err) + } +} diff --git a/internal/catalogue/predefined.go b/internal/catalogue/predefined.go new file mode 100644 index 00000000..73d2cd9b --- /dev/null +++ b/internal/catalogue/predefined.go @@ -0,0 +1,236 @@ +package catalogue + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "sort" + "strings" + "sync" +) + +// The list of predefined keys (novox/hq ADR 0348 §7, design 57 package K2, issue 661). +// +// **A fixed, reviewable list, owned by the build.** Mesh config and node config hold values only for the keys +// in the catalogue's `predefined.json`, at the root of its repository: each key with its type, a description +// including its unit, its validation (enum, minimum and maximum, pattern) and whether it is trusted. A module's +// value may fall back to one of them, of the same type (ADR 0348 §2, §4). +// +// **Read where a manifest is judged, from the tree it is judged in.** A build reads the file at the root of the +// repository at the commit it builds, `module check` at the root of the repository above each manifest, the +// merge gate at the root of the tree under check; each refuses a file that is not exactly a list of keys, and +// judges every manifest it reads against it (ParseManifestWith). Everywhere else — registration, a stored build +// read again — a manifest is judged against the list the mesh published (PublishedPredefined): the newest trunk +// build's of the catalogue, which the controller keeps and loads with its store. + +// PredefinedFile is the list's file at the root of the catalogue's repository. +const PredefinedFile = "predefined.json" + +// CatalogueRepository is the name of the repository whose list the mesh publishes: the catalogue's, by the name +// a check finds it beside under (internal/builder's check, ADR 0121). +const CatalogueRepository = "mesh-catalog" + +// PredefinedKey is one key of the list: its type, its description, its validation and whether it is trusted +// (ADR 0348 §7). It has no default: a module's value falling back to it has its own. +type PredefinedKey struct { + Type string `json:"type"` + Description string `json:"description"` + Enum []any `json:"enum,omitempty"` + Minimum *float64 `json:"minimum,omitempty"` + Maximum *float64 `json:"maximum,omitempty"` + Pattern string `json:"pattern,omitempty"` + Trusted bool `json:"trusted,omitempty"` +} + +// PredefinedList is the list by key. Nil is no list at all, which is not the same as an empty one: a value +// naming a key is refused then saying the list is not present. +type PredefinedList map[string]PredefinedKey + +// ParsePredefined reads predefined.json strictly: one JSON object of keys, each with exactly the fields of a +// PredefinedKey, and every key judged as a module's config value is (PredefinedProblems). Anything else is +// refused whole, naming each problem — a list half read would let a module fall back to a key nobody reviewed. +func ParsePredefined(raw []byte) (PredefinedList, error) { + var keys map[string]json.RawMessage + if err := json.Unmarshal(raw, &keys); err != nil { + return nil, fmt.Errorf("%s is not a JSON object of keys: %w", PredefinedFile, err) + } + if keys == nil { + return nil, fmt.Errorf("%s is not a JSON object of keys: it says null", PredefinedFile) + } + list := PredefinedList{} + var problems []string + for _, key := range sortedKeys(keys) { + decoder := json.NewDecoder(bytes.NewReader(keys[key])) + decoder.DisallowUnknownFields() + var k PredefinedKey + if err := decoder.Decode(&k); err != nil { + problems = append(problems, fmt.Sprintf("%q is not a predefined key's declaration (type, description, "+ + "enum, minimum, maximum, pattern, trusted): %v", key, err)) + continue + } + list[key] = k + } + problems = append(problems, PredefinedProblems(list)...) + if len(problems) > 0 { + return nil, fmt.Errorf("%s is refused: %s", PredefinedFile, strings.Join(problems, "; ")) + } + return list, nil +} + +// PredefinedProblems is every way a key of the list is declared wrongly, by the rules a module's config value +// is held to (ADR 0348 §4, §7, §11): a usable name that is not placement, a type, a description, validation +// that fits the type, and an object or a list only when trusted. +func PredefinedProblems(list PredefinedList) []string { + var problems []string + for _, key := range sortedKeys(list) { + k := list[key] + say := func(format string, args ...any) { + problems = append(problems, fmt.Sprintf("the predefined key %q: ", key)+fmt.Sprintf(format, args...)) + } + if !settingRef.MatchString("${setting:" + key + "}") { + say("not a usable key: lower-case letters, digits, dots, dashes and underscores") + continue + } + if meshWords[key] || accountFacts[key] { + say("%q is placement, which the mesh decides, never a config value (novox/hq ADR 0348 §11)", key) + continue + } + switch k.Type { + case ConfigNumber, ConfigString, ConfigBoolean: + case ConfigObject, ConfigList: + if !k.Trusted { + say("is %s and not trusted: an object or a list is trusted, \"trusted\": true (novox/hq ADR 0348 §4)", k.Type) + } + case "": + say("no type: say \"type\": number, string or boolean, or object or list when trusted (novox/hq ADR 0348 §4)") + default: + say("type %q is none of number, string, boolean, object and list (novox/hq ADR 0348 §4)", k.Type) + } + if strings.TrimSpace(k.Description) == "" { + say("no description: say in a sentence what the value is, including its unit (novox/hq ADR 0348 §7)") + } + for _, wrong := range validationProblems(SettingDeclaration{Type: k.Type, Enum: k.Enum, Minimum: k.Minimum, + Maximum: k.Maximum, Pattern: k.Pattern}) { + say("%s", wrong) + } + } + return problems +} + +// ReadPredefined is the list in predefined.json at a repository's root, and nil with no error when the root +// holds none: a repository other than the catalogue, or the catalogue before the file. +func ReadPredefined(root string) (PredefinedList, error) { + path := filepath.Join(root, PredefinedFile) + info, err := os.Lstat(path) + if errors.Is(err, fs.ErrNotExist) { + return nil, nil + } + if err != nil { + return nil, err + } + if !info.Mode().IsRegular() { + return nil, fmt.Errorf("%s is %s, not a plain file", path, kindOf(info.Mode())) + } + raw, err := os.ReadFile(path) + if err != nil { + return nil, err + } + return ParsePredefined(raw) +} + +// RepositoryRoot is the nearest directory at or above dir holding `.git`: the root of the repository a manifest +// is in, where `module check` reads the list. Empty when none is: a manifest outside a repository is judged +// against no list. +func RepositoryRoot(dir string) string { + at, err := filepath.Abs(dir) + if err != nil { + return "" + } + for { + if _, err := os.Lstat(filepath.Join(at, ".git")); err == nil { + return at + } + up := filepath.Dir(at) + if up == at { + return "" + } + at = up + } +} + +// PredefinedAbove is the list at the root of the repository above dir (RepositoryRoot), nil when there is no +// repository or it holds no list. +func PredefinedAbove(dir string) (PredefinedList, error) { + root := RepositoryRoot(dir) + if root == "" { + return nil, nil + } + return ReadPredefined(root) +} + +// Canonical is the list as JSON with its keys sorted: what a build carries and the controller keeps, so one list +// is one text. +func (l PredefinedList) Canonical() ([]byte, error) { + if l == nil { + return nil, nil + } + return json.Marshal(map[string]PredefinedKey(l)) +} + +// Keys is the list's keys, sorted. +func (l PredefinedList) Keys() []string { + out := make([]string, 0, len(l)) + for k := range l { + out = append(out, k) + } + sort.Strings(out) + return out +} + +// NamesPredefined is whether any value of the manifest falls back to a predefined key. +func (m Manifest) NamesPredefined() bool { + for _, d := range m.Settings { + if d.Predefined != "" { + return true + } + } + return false +} + +// publishedPredefined is the list the mesh published, which a manifest is judged against where no tree is read. +var publishedPredefined struct { + sync.RWMutex + list PredefinedList +} + +// UsePredefined makes list the one a manifest is judged against where no tree is read: what the controller +// loads from its store, and what it keeps when a newer trunk build of the catalogue publishes one. Nil leaves +// the list it has: a store that holds none yet never empties one already loaded. +func UsePredefined(list PredefinedList) { + if list == nil { + return + } + publishedPredefined.Lock() + defer publishedPredefined.Unlock() + publishedPredefined.list = list +} + +// PublishedPredefined is the list a manifest is judged against where no tree is read; nil when none is loaded. +func PublishedPredefined() PredefinedList { + publishedPredefined.RLock() + defer publishedPredefined.RUnlock() + return publishedPredefined.list +} + +// ReplacePublished sets the published list, nil included, and answers the one it replaced: for a test that +// judges against a list of its own and puts the mesh's back. +func ReplacePublished(list PredefinedList) (was PredefinedList) { + publishedPredefined.Lock() + defer publishedPredefined.Unlock() + was, publishedPredefined.list = publishedPredefined.list, list + return was +} -- 2.54.0 From fbcf44702cd6d0c193ec2ac978da4676e9911ef1 Mon Sep 17 00:00:00 2001 From: mesh-admin Date: Wed, 14 Oct 2026 05:48:49 +0000 Subject: [PATCH 22/31] Prove the list is read strictly and judges module check, the gate and registration Part 4: the tests of the list's reading and of the controller's commands on it (hq ADR 0348 clause 7, issue 661). Co-Authored-By: Claude Opus 5.5 --- cmd/mesh-controller/predefined_test.go | 197 +++++++++++++++++++++++++ internal/catalogue/predefined_test.go | 175 ++++++++++++++++++++++ 2 files changed, 372 insertions(+) create mode 100644 cmd/mesh-controller/predefined_test.go create mode 100644 internal/catalogue/predefined_test.go diff --git a/cmd/mesh-controller/predefined_test.go b/cmd/mesh-controller/predefined_test.go new file mode 100644 index 00000000..25f09c50 --- /dev/null +++ b/cmd/mesh-controller/predefined_test.go @@ -0,0 +1,197 @@ +package main + +import ( + "bytes" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/link" +) + +// The list of predefined keys (novox/hq ADR 0348 §7, design 57 package K2, issue 661): `module check` and the +// merge gate judge each manifest against the list at the root of its tree; registration publishes the newest +// trunk build's list of the catalogue and judges every module against the list the mesh publishes. + +const theFirstKeys = `{ + "font-size": {"type": "number", "description": "the size text is shown in, in points", "minimum": 6, "maximum": 72}, + "agent-managed-settings": {"type": "object", "description": "the managed settings of every agent session, a JSON object", "trusted": true} +}` + +// notifierFallingBackTo is the notifier in the config form, its font size falling back to the key named. +func notifierFallingBackTo(key, kind string) string { + def := "10" + if kind == "string" { + def = `"10"` + } + return `{"module":"notifier","version":"1", + "config":{"font-size":{"type":"` + kind + `","description":"the notification's font size, in points", + "default":` + def + `,"predefined":"` + key + `"}}, + "resources":[{"id":"conf","type":"file","path":"/etc/notifier/notifier.conf","mode":"0644", + "content":"font = ${config:font-size}\n"}]}` +} + +// withoutAPublishedList judges the test against no list the mesh published, and puts the process's back after. +func withoutAPublishedList(t *testing.T) { + t.Helper() + was := catalogue.ReplacePublished(nil) + t.Cleanup(func() { catalogue.ReplacePublished(was) }) +} + +// aRepositoryTree is a repository's checkout holding one module at modules/notifier, and the list when given. +func aRepositoryTree(t *testing.T, manifest string, list *string) string { + t.Helper() + root := t.TempDir() + if err := os.Mkdir(filepath.Join(root, ".git"), 0o755); err != nil { + t.Fatal(err) + } + if list != nil { + if err := os.WriteFile(filepath.Join(root, catalogue.PredefinedFile), []byte(*list), 0o644); err != nil { + t.Fatal(err) + } + } + return aTemplatedModule(t, filepath.Join(root, "modules", "notifier"), manifest, nil) +} + +func TestModuleCheckJudgesAManifestAgainstTheListAtItsRepositoryRoot(t *testing.T) { + withoutAPublishedList(t) + list := theFirstKeys + check := func(path string) (string, error) { + var out bytes.Buffer + err := moduleCheck([]string{path}, &out) + return out.String(), err + } + if out, err := check(aRepositoryTree(t, notifierFallingBackTo("font-size", "number"), &list)); err != nil || + !strings.Contains(out, "notifier: ok") { + t.Fatalf("a module naming font-size was refused: %v\n%s", err, out) + } + for _, c := range []struct { + manifest string + list *string + says string + }{ + {notifierFallingBackTo("font-scale", "number"), &list, `"font-scale", which is not on the list`}, + {notifierFallingBackTo("font-size", "string"), &list, "which is a number, and this value is a string"}, + {notifierFallingBackTo("font-size", "number"), nil, "is not present"}, + {notifierFallingBackTo("font-size", "number"), ptr(`{"font-size": {"type": "number"}}`), "no description"}, + } { + if out, err := check(aRepositoryTree(t, c.manifest, c.list)); err == nil || !strings.Contains(out, c.says) { + t.Errorf("wanted the check refused saying %q, got %v:\n%s", c.says, err, out) + } + } +} + +func ptr(s string) *string { return &s } + +// writeTheList puts the list at the root of a tree the gate judges. +func writeTheList(t *testing.T, tree, list string) string { + t.Helper() + if err := os.WriteFile(filepath.Join(tree, catalogue.PredefinedFile), []byte(list), 0o644); err != nil { + t.Fatal(err) + } + return tree +} + +func TestTheMergeGateJudgesEachManifestAgainstTheListOfTheTree(t *testing.T) { + withoutAPublishedList(t) + f, manifests := templatedMesh(t) + + v := gateJudged(t, f, writeTheList(t, aTree(t, withEdit(manifests, "notifier", + notifierFallingBackTo("font-size", "number"))), theFirstKeys), "modules/notifier/module.json") + if v.Verdict == "fail" || v.Modules["notifier"] != "changed" { + t.Fatalf("a module naming font-size on the tree's list failed the gate (%v):\n%s", v.Modules, v.Report()) + } + for _, c := range []struct{ manifest, list, says string }{ + {notifierFallingBackTo("font-scale", "number"), theFirstKeys, `"font-scale", which is not on the list`}, + {notifierFallingBackTo("font-size", "string"), theFirstKeys, "which is a number"}, + {inlineNotifier, `{"font-size": {"type": "number", "description": "points", "unit": "pt"}}`, `unknown field "unit"`}, + } { + v := gateJudged(t, f, writeTheList(t, aTree(t, withEdit(manifests, "notifier", c.manifest)), c.list), + "modules/notifier/module.json", catalogue.PredefinedFile) + if v.Verdict != "fail" || !strings.Contains(v.Report(), c.says) { + t.Errorf("wanted the gate to fail saying %q:\n%s", c.says, v.Report()) + } + } +} + +// catalogueBuild is a trunk build of the catalogue carrying the list at a generation. +func catalogueBuild(id, manifest, list string, generation int) link.BuildResult { + r := link.BuildResult{ID: id, Repository: "http://forge.internal:20000/novox/mesh-catalog.git", + Path: "modules/notifier", Ref: "main", On: "anchor", Commit: strings.Repeat(id[len(id)-1:], 12), + Manifest: json.RawMessage(manifest), Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"}} + if list != "" { + r.Predefined = &link.PredefinedList{Keys: json.RawMessage(list), Generation: generation} + } + return r +} + +func TestRegistrationPublishesTheNewestTrunkListAndJudgesEveryModuleAgainstIt(t *testing.T) { + withoutAPublishedList(t) + open := aMesh(t) + ctx := t.Context() + + // A manifest naming font-size registers, and its build's list is published. + if _, _, err := takeIn(ctx, open.inventory, asTheOperator(t, open.inventory, + catalogueBuild("b-1", notifierFallingBackTo("font-size", "number"), theFirstKeys, 10))); err != nil { + t.Fatalf("a module naming font-size was not registered: %v", err) + } + kept, has, err := open.inventory.PredefinedKept(ctx) + if err != nil || !has || kept.Generation != 10 || kept.Build != "b-1" { + t.Fatalf("the list was not published: %+v %v %v", kept, has, err) + } + if got := catalogue.PublishedPredefined(); len(got) != 2 { + t.Fatalf("the controller judges against %v, not the list published", got) + } + + // One naming a key not on the list, or of another type, is refused and never enters the catalogue. + for i, c := range []struct{ manifest, says string }{ + {notifierFallingBackTo("font-scale", "number"), "not on the list"}, + {notifierFallingBackTo("font-size", "string"), "which is a number"}, + } { + id := []string{"b-2", "b-3"}[i] + if _, _, err := takeIn(ctx, open.inventory, catalogueBuild(id, c.manifest, theFirstKeys, 10)); err == nil || + !strings.Contains(err.Error(), c.says) { + t.Errorf("wanted the registration refused saying %q, got %v", c.says, err) + } + } + src, err := open.inventory.SourceOf(ctx, "notifier") + if err != nil || src.BuiltFrom != strings.Repeat("1", 12) { + t.Fatalf("a refused build replaced the registered one: %+v %v", src, err) + } + + // A newer build's list, which drops agent-managed-settings, replaces the published one. + smaller := `{"font-size": {"type": "number", "description": "points", "minimum": 6, "maximum": 72}}` + if _, _, err := takeIn(ctx, open.inventory, catalogueBuild("b-4", + notifierFallingBackTo("font-size", "number"), smaller, 12)); err != nil { + t.Fatal(err) + } + // The older build's outcome heard later registers nothing new about the list: the newer stands. + if _, _, err := takeIn(ctx, open.inventory, catalogueBuild("b-5", + notifierFallingBackTo("font-size", "number"), theFirstKeys, 11)); err != nil { + t.Fatal(err) + } + kept, _, _ = open.inventory.PredefinedKept(ctx) + if kept.Build != "b-4" || len(kept.Keys) != 1 || len(catalogue.PublishedPredefined()) != 1 { + t.Fatalf("an older build's list heard later replaced the newer one: %+v", kept) + } + + // A build of another repository publishes nothing, and its manifest is judged against the published list. + other := catalogueBuild("b-6", notifierFallingBackTo("font-size", "number"), theFirstKeys, 99) + other.Repository, other.Source.Repository = "http://forge.internal:20000/novox/notifier.git", "novox/notifier" + other.Manifest = json.RawMessage(strings.Replace(notifierFallingBackTo("font-size", "number"), `"notifier"`, + `"chimes"`, 1)) + if _, _, err := takeIn(ctx, open.inventory, asTheOperator(t, open.inventory, other)); err != nil { + t.Fatalf("a module of another repository naming a published key: %v", err) + } + if kept, _, _ = open.inventory.PredefinedKept(ctx); kept.Build != "b-4" { + t.Fatalf("a build of another repository published its list: %+v", kept) + } + other.ID, other.Manifest = "b-7", json.RawMessage(strings.Replace(notifierFallingBackTo("agent-managed-settings", + "number"), `"notifier"`, `"chimes"`, 1)) + if _, _, err := takeIn(ctx, open.inventory, other); err == nil || !strings.Contains(err.Error(), "not on the list") { + t.Fatalf("a key only another repository's list holds was registered: %v", err) + } +} diff --git a/internal/catalogue/predefined_test.go b/internal/catalogue/predefined_test.go new file mode 100644 index 00000000..c683f763 --- /dev/null +++ b/internal/catalogue/predefined_test.go @@ -0,0 +1,175 @@ +package catalogue + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" +) + +// The list of predefined keys (novox/hq ADR 0348 §7, design 57 package K2, issue 661): read strictly, and every +// manifest naming a key judged against the list of the tree it is read in. + +// theFirstKeys is the catalogue's first list: font-size and agent-managed-settings. +const theFirstKeys = `{ + "font-size": {"type": "number", "description": "the size text is shown in, in points", "minimum": 6, "maximum": 72}, + "agent-managed-settings": {"type": "object", "description": "the managed settings of every agent session, a JSON object", "trusted": true} +}` + +func TestTheFirstListIsRead(t *testing.T) { + list, err := ParsePredefined([]byte(theFirstKeys)) + if err != nil { + t.Fatal(err) + } + if got := strings.Join(list.Keys(), ","); got != "agent-managed-settings,font-size" { + t.Fatalf("keys %s", got) + } + if k := list["font-size"]; k.Type != ConfigNumber || *k.Minimum != 6 || *k.Maximum != 72 || k.Trusted { + t.Fatalf("font-size read as %+v", k) + } + if k := list["agent-managed-settings"]; k.Type != ConfigObject || !k.Trusted { + t.Fatalf("agent-managed-settings read as %+v", k) + } + // One list is one text, however it was written. + a, _ := list.Canonical() + again, _ := ParsePredefined(a) + b, _ := again.Canonical() + if string(a) != string(b) { + t.Fatalf("canonical text moved:\n%s\n%s", a, b) + } +} + +func TestAListIsRefusedWholeForAnyWrongKey(t *testing.T) { + for _, c := range []struct{ file, says string }{ + {`[]`, "not a JSON object of keys"}, + {`null`, "not a JSON object of keys"}, + {`{"font-size": {"type": "number", "description": "points", "default": 10}}`, `unknown field "default"`}, + {`{"font-size": {"description": "points"}}`, "no type"}, + {`{"font-size": {"type": "decimal", "description": "points"}}`, `type "decimal"`}, + {`{"font-size": {"type": "number"}}`, "no description"}, + {`{"accent": {"type": "object", "description": "colours"}}`, "not trusted"}, + {`{"font-size": {"type": "number", "description": "points", "minimum": 9, "maximum": 6}}`, "above its maximum"}, + {`{"accent": {"type": "string", "description": "a name", "pattern": "("}}`, "not a regular expression"}, + {`{"accent": {"type": "boolean", "description": "on or off", "minimum": 1}}`, "minimum or maximum is for a number"}, + {`{"Font Size": {"type": "number", "description": "points"}}`, "not a usable key"}, + {`{"places": {"type": "string", "description": "where"}}`, "placement"}, + {`{"puid": {"type": "number", "description": "an id"}}`, "placement"}, + } { + if _, err := ParsePredefined([]byte(c.file)); err == nil || !strings.Contains(err.Error(), c.says) { + t.Errorf("%s: wanted a refusal saying %q, got %v", c.file, c.says, err) + } + } +} + +// fallingBackTo is inConfigForm with its font size falling back to a predefined key. +func fallingBackTo(t *testing.T, key string) []byte { + t.Helper() + m := inConfigForm() + declaration(m, "font-size")["predefined"] = key + raw, err := json.Marshal(m) + if err != nil { + t.Fatal(err) + } + return raw +} + +// A manifest naming font-size registers against the list; one naming a key not on it, or of another type, is +// refused — judged against the list given, whatever the mesh published. +func TestAManifestIsJudgedAgainstTheListOfItsTree(t *testing.T) { + withPredefined(t, nil) + list, err := ParsePredefined([]byte(theFirstKeys)) + if err != nil { + t.Fatal(err) + } + if _, err := ParseManifestWith(fallingBackTo(t, "font-size"), list); err != nil { + t.Fatalf("a value falling back to font-size is refused: %v", err) + } + if _, err := ParseManifestWith(fallingBackTo(t, "font-scale"), list); err == nil || + !strings.Contains(err.Error(), `"font-scale", which is not on the list`) { + t.Fatalf("a key not on the list: %v", err) + } + m := inConfigForm() + declaration(m, "theme")["predefined"] = "font-size" + raw, _ := json.Marshal(m) + if _, err := ParseManifestWith(raw, list); err == nil || !strings.Contains(err.Error(), "which is a number") { + t.Fatalf("a key of another type: %v", err) + } + // Where no tree is read, the published list judges: none published refuses. + if _, err := ParseManifest(fallingBackTo(t, "font-size")); err == nil || !strings.Contains(err.Error(), "is not present") { + t.Fatalf("judged against no list: %v", err) + } + withPredefined(t, list) + if _, err := ParseManifest(fallingBackTo(t, "font-size")); err != nil { + t.Fatalf("judged against the published list: %v", err) + } +} + +// A template's text filled from the tree keeps the list the manifest was judged against: read again, it is not +// judged against the published one. +func TestTemplatesFilledKeepTheListTheManifestWasJudgedAgainst(t *testing.T) { + withPredefined(t, nil) + list, _ := ParsePredefined([]byte(theFirstKeys)) + dir := t.TempDir() + if err := os.MkdirAll(filepath.Join(dir, TemplatesDir), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, TemplatesDir, "rc"), []byte("font = ${config:font-size}\n"), 0o644); err != nil { + t.Fatal(err) + } + m := inConfigForm() + declaration(m, "font-size")["predefined"] = "font-size" + r := resourceOf(m, "configuration") + delete(r, "content") + r["template"] = TemplatesDir + "/rc" + m["build"] = map[string]any{"artifacts": []any{map[string]any{"name": "files", "kind": "archive", "from": "files"}}} + declaration(m, "width")["runtime"] = true + declaration(m, "sender")["runtime"] = true + raw, _ := json.Marshal(m) + parsed, err := ParseManifestWith(raw, list) + if err != nil { + t.Fatal(err) + } + if _, err := WithTemplatesFrom(parsed, dir); err != nil { + t.Fatalf("filled from the tree, it was judged against another list: %v", err) + } +} + +// module check reads the list at the root of the repository above the manifest; no repository, no list. +func TestTheListIsFoundAtTheRepositoryRootAboveAManifest(t *testing.T) { + root := t.TempDir() + module := filepath.Join(root, "modules", "dunst") + if err := os.MkdirAll(module, 0o755); err != nil { + t.Fatal(err) + } + if list, err := PredefinedAbove(module); err != nil || list != nil { + t.Fatalf("outside a repository: %v %v", list, err) + } + if err := os.Mkdir(filepath.Join(root, ".git"), 0o755); err != nil { + t.Fatal(err) + } + if list, err := PredefinedAbove(module); err != nil || list != nil { + t.Fatalf("a repository without the file: %v %v", list, err) + } + if err := os.WriteFile(filepath.Join(root, PredefinedFile), []byte(theFirstKeys), 0o644); err != nil { + t.Fatal(err) + } + list, err := PredefinedAbove(module) + if err != nil || len(list) != 2 { + t.Fatalf("the list above: %v %v", list, err) + } + if err := os.WriteFile(filepath.Join(root, PredefinedFile), []byte(`{"font-size": {}}`), 0o644); err != nil { + t.Fatal(err) + } + if _, err := PredefinedAbove(module); err == nil { + t.Fatal("a wrong list above was read") + } + if err := os.Remove(filepath.Join(root, PredefinedFile)); err != nil { + t.Fatal(err) + } + if err := os.Symlink("/etc/hostname", filepath.Join(root, PredefinedFile)); err == nil { + if _, err := PredefinedAbove(module); err == nil || !strings.Contains(err.Error(), "not a plain file") { + t.Fatalf("a link read as the list: %v", err) + } + } +} -- 2.54.0 From 3d3ca6e5001dcdb560fe8083dc686a560841a2bb Mon Sep 17 00:00:00 2001 From: mesh-admin Date: Wed, 14 Oct 2026 06:16:47 +0000 Subject: [PATCH 23/31] Bring main into release 57's branch, part 1 of 3: the builder's try outcome and the release walk So the release branch carries what main gained; parts 2 and 3 follow on this branch. Co-Authored-By: Claude Opus 5.5 --- cmd/mesh-builder/main.go | 26 +- cmd/mesh-builder/tried_test.go | 31 ++ cmd/mesh-controller/gate.go | 12 + cmd/mesh-controller/held_back.go | 2 +- cmd/mesh-controller/issue648_test.go | 87 ++++ cmd/mesh-controller/main.go | 9 + cmd/mesh-controller/plain_words.go | 16 + cmd/mesh-controller/release.go | 579 +++++++++++++++++++++++++-- 8 files changed, 718 insertions(+), 44 deletions(-) create mode 100644 cmd/mesh-builder/tried_test.go create mode 100644 cmd/mesh-controller/issue648_test.go diff --git a/cmd/mesh-builder/main.go b/cmd/mesh-builder/main.go index 7bf565be..be469d09 100644 --- a/cmd/mesh-builder/main.go +++ b/cmd/mesh-builder/main.go @@ -332,6 +332,12 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri result.Check.Members = append(result.Check.Members, link.CheckedMember{Owner: m.Owner, Repo: m.Repo, Commit: m.Head, Trunk: m.Trunk}) } + // A branch tried before its pull request reads its outcome from this log, whole (novox/hq issue 686). + if request.Check.Try { + if line, err := json.Marshal(triedOf(v)); err == nil { + say(link.StepTried, string(line)) + } + } } } else if err == nil { // The package-registry credential is a build input, so it is resolved before the clone: a @@ -457,7 +463,8 @@ func checkSpecOf(request link.BuildRequest) builder.CheckSpec { spec := builder.CheckSpec{ID: request.ID, Repository: request.Repository, Ref: request.Ref, Owner: c.Owner, Repo: c.Repo, Number: c.Number, Paths: c.Paths, Beside: map[string]builder.Beside{}, Modules: c.Modules, New: c.New, Manifests: c.Manifests, Judge: c.Judge, Base: c.Base, - Toolchain: builder.ToolchainOf(request.Held), Toolchains: builder.ToolchainsOf(request.Held)} + Toolchain: builder.ToolchainOf(request.Held), Toolchains: builder.ToolchainsOf(request.Held), + Tests: c.Tests, StoreOnly: c.StoreOnly} for dir, b := range c.Beside { spec.Beside[dir] = builder.Beside{Repository: b.Repository, Ref: b.Ref} } @@ -472,6 +479,23 @@ func checkSpecOf(request link.BuildRequest) builder.CheckSpec { return spec } +// triedOf is a try's outcome as its log line says it (novox/hq issue 686): the repository's own check's verdict, +// or the check's where that did not run — a head that does not merge — and the lines naming what failed. +func triedOf(v builder.CheckVerdict) link.Tried { + t := link.Tried{Verdict: v.Verdict, Summary: v.Summary, Took: v.Took.Round(time.Second).String(), + Head: v.Head, Trunk: v.Trunk} + if v.Repo != nil && (v.Gate == nil || v.Gate.Verdict == "pass") { + t.Verdict, t.Summary = v.Repo.Verdict, v.Repo.Summary + } + if v.Repo != nil && v.Repo.Failed != "" { + t.Failed = strings.Split(v.Repo.Failed, "\n") + } + if t.Verdict == "" { + t.Verdict = "error" + } + return t +} + // checkBuiltOf is a check's builds for its merge as the outcome carries them (novox/hq ADR 0350 §1). func checkBuiltOf(built []builder.CheckBuilt) []link.CheckBuilt { var out []link.CheckBuilt diff --git a/cmd/mesh-builder/tried_test.go b/cmd/mesh-builder/tried_test.go new file mode 100644 index 00000000..06aaf706 --- /dev/null +++ b/cmd/mesh-builder/tried_test.go @@ -0,0 +1,31 @@ +package main + +import ( + "testing" + "time" + + "github.com/novox/mesh-controller/internal/builder" +) + +// A try's outcome, as the build seat says it in the build's log (novox/hq issue 686): the repository's own +// check's verdict, how long it took, and each line naming what failed. +func TestATrysOutcomeNamesWhatFailed(t *testing.T) { + tried := triedOf(builder.CheckVerdict{Verdict: "pass", Summary: "the change touches no module", Took: 303 * time.Second, + Head: "c0ffee", Trunk: "ca3f43c", Gate: &builder.Layer{Verdict: "pass"}, + Repo: &builder.Layer{Verdict: "fail", Summary: "its merge-check.sh failed: TestX", + Failed: "--- FAIL: TestX (0.01s)\n x_test.go:12: got 1, want 2"}}) + if tried.Verdict != "fail" || tried.Took != "5m3s" || len(tried.Failed) != 2 || tried.Trunk != "ca3f43c" || + tried.Summary != "its merge-check.sh failed: TestX" { + t.Fatalf("a failed try: %+v", tried) + } + // A head that does not merge fails both layers: the check's own word. + conflict := triedOf(builder.CheckVerdict{Verdict: "fail", Summary: "does not merge", Gate: &builder.Layer{Verdict: "fail"}, + Repo: &builder.Layer{Verdict: "fail", Summary: "does not merge"}}) + if conflict.Verdict != "fail" || conflict.Summary != "does not merge" { + t.Fatalf("a try that does not merge: %+v", conflict) + } + // No verdict is an error, never a pass. + if none := triedOf(builder.CheckVerdict{}); none.Verdict != "error" { + t.Fatalf("a try with no verdict: %+v", none) + } +} diff --git a/cmd/mesh-controller/gate.go b/cmd/mesh-controller/gate.go index 304186a2..8a1e1a1c 100644 --- a/cmd/mesh-controller/gate.go +++ b/cmd/mesh-controller/gate.go @@ -1443,6 +1443,12 @@ func probeGates(ctx context.Context, d *doctor) ([]conditions.Observation, error } // A release held after a failed one, while builds still wait for a gate (ADR 0236). out = append(out, backlogObservation()...) + // And a rest that went silent (novox/hq ADR 0353 §4). + silent, err := silentObservation(ctx, d.open.inventory) + if err != nil { + return nil, err + } + out = append(out, silent...) return sortedFound(dedupeObservations(out)), nil } @@ -1532,6 +1538,12 @@ func gateLine(g *inventory.PlanGate) string { case inventory.GateFailed: what = "FAILED" } + switch { + case g.Verdict == "" && g.Pending: + what = "kept, not yet sent" // novox/hq ADR 0353 §8 + case g.Verdict == "" && g.Silent: + what = "silent on its send, left" // ADR 0353 §4 + } if g.Verdict == "" && g.Waiting() { what = "waits for a busy container on " + g.Busy.Machine } diff --git a/cmd/mesh-controller/held_back.go b/cmd/mesh-controller/held_back.go index 0d90cc19..a0b4c6ba 100644 --- a/cmd/mesh-controller/held_back.go +++ b/cmd/mesh-controller/held_back.go @@ -151,7 +151,7 @@ func heldMachines(ctx context.Context, open *stores, names []string) (map[string for _, mv := range f.moves(node, modules, same, known, false) { if planStillToSend(plans, mv.Module, node) == "" { why = append(why, fmt.Sprintf("%s would move from %s to %s, which has passed no gate yet — a release "+ - "plan sends it, one machine at a time, judged", mv.Module, buildName(mv.From), buildName(mv.To))) + "plan sends it, one machine first and then the rest at once, judged", mv.Module, buildName(mv.From), buildName(mv.To))) } } if len(why) > 0 { diff --git a/cmd/mesh-controller/issue648_test.go b/cmd/mesh-controller/issue648_test.go new file mode 100644 index 00000000..45261cc0 --- /dev/null +++ b/cmd/mesh-controller/issue648_test.go @@ -0,0 +1,87 @@ +package main + +import ( + "encoding/json" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" +) + +// A tier whose module is no move — made from the source every machine running it runs, so nothing is +// sent — is not waited on for a report after its marking (novox/hq issue 648). Its machines answered the +// send that last carried that build before the walk got there, and nothing will make them report again: +// the walk asks the next tier in the same step, rather than "waiting for app on anchor, laptop" for ever. +func TestANoMoveTierWaitsForNoNewReport(t *testing.T) { + judgeLeavesAsTheCore(t) + g := aGateMesh(t) + ctx := t.Context() + inv := g.open.inventory + + // app is the build machine, so a module of the next tier is built by it: the tier gates the next on app + // running on its machines. + for _, m := range []catalogue.Manifest{ + {Module: "app", Version: "c2", Claims: []catalogue.Claim{{Name: "node-build-agent", Scope: catalogue.ScopeNode}}}, + {Module: "web", Version: "w1"}, + } { + if err := inv.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", + Path: "modules/" + m.Module, BuiltFrom: m.Version, Head: m.Version, Asked: time.Now().Add(-time.Minute)}); err != nil { + t.Fatal(err) + } + } + // What anchor and laptop were last sent (c1) was made from the source the plan's build of c2 is. + for _, b := range []inventory.Build{ + {ID: "build-1s", Module: "app", Commit: "c1", SourceFingerprint: "src1:app", + Asked: time.Now().Add(-30 * time.Second), At: time.Now().Add(-30 * time.Second)}, + {ID: "build-2s", Module: "app", Commit: "c2", SourceFingerprint: "src1:app", Asked: time.Now(), At: time.Now()}, + } { + b.Manifest, _ = json.Marshal(catalogue.Manifest{Module: "app", Version: "1"}) + if err := inv.RecordBuild(ctx, b); err != nil { + t.Fatal(err) + } + } + // Each machine answered that send, applied, before the walk's tier was built. + for _, n := range []string{"anchor", "laptop"} { + if _, err := inv.RecordDoing(ctx, nodeID(t, g.open, n), inventory.Doing{Node: n, + Outcome: inventory.OutcomeApplied, Declared: "d-" + n + "-c1", Applied: 1, + At: time.Now().Add(-10 * time.Second)}); err != nil { + t.Fatal(err) + } + } + built := time.Now().UTC() + p := g.plan(t) + p.Tiers = [][]string{{"app"}, {"web"}} + p.Modules["app"].Build, p.Modules["app"].BuiltAt = "build-2s", &built + if err := inv.SavePlan(ctx, &p); err != nil { + t.Fatal(err) + } + + advancePlans(ctx, g.open) + + if len(g.sent) != 0 { + t.Fatalf("a no-move build was sent: %v", g.sent) + } + p = g.plan(t) + if s := p.Modules["app"]; s == nil || !strings.HasPrefix(s.Why, "no move") { + t.Fatalf("the build was not marked no move: %+v", s) + } + if p.Tier < 1 || strings.Contains(p.Note, "waiting for app") { + t.Fatalf("the walk waits on a no-move tier for a report nothing will cause: tier %d, %s %q", p.Tier, p.State, p.Note) + } + + // A machine that has not answered its last send is still waited on: no move is not "never sent". + p.Tier, p.State, p.Note = 0, inventory.PlanRolling, "" + if err := inv.RecordSent(ctx, nodeID(t, g.open, "laptop"), "d-laptop-later", map[string]string{"app": "c1"}); err != nil { + t.Fatal(err) + } + if err := inv.SavePlan(ctx, &p); err != nil { + t.Fatal(err) + } + advancePlans(ctx, g.open) + p = g.plan(t) + if p.Tier != 0 || !strings.Contains(p.Note, "waiting for app on laptop") { + t.Fatalf("a machine that has not answered its last send was not waited on: tier %d, %q", p.Tier, p.Note) + } +} diff --git a/cmd/mesh-controller/main.go b/cmd/mesh-controller/main.go index 1e656622..41d7319d 100644 --- a/cmd/mesh-controller/main.go +++ b/cmd/mesh-controller/main.go @@ -106,6 +106,8 @@ func run() error { return killCommand(ctx, args[1:]) case "forget-checks": return forgetChecksCommand(ctx, args[1:]) + case "try": + return tryCommand(ctx, args[1:]) case "pause", "resume": return pauseCommand(ctx, args[0], args[1:]) case "collection": @@ -381,6 +383,13 @@ func (b builds) Built(ctx context.Context, result link.BuildResult) error { // **A merge check builds nothing** (novox/hq to-be 45 §9): its verdict is said, and nothing of it is // recorded or registered. if result.Check != nil || result.Checked != nil { + if isTry(result) { + // A branch tried before its pull request (novox/hq issue 686): its asker reads the outcome from the + // build's log. Never said as `checked`, so the forge's holder sets no status. + fmt.Printf("%s: a try of %s at %.8s, read with `try --id`: said to nobody, nothing taken in\n", result.ID, + result.Repository, result.Ref) + return nil + } checked(ctx, result) return nil } diff --git a/cmd/mesh-controller/plain_words.go b/cmd/mesh-controller/plain_words.go index a5f705b1..7d60b16a 100644 --- a/cmd/mesh-controller/plain_words.go +++ b/cmd/mesh-controller/plain_words.go @@ -456,6 +456,9 @@ var plainWordings = map[string]func(conditions.Observation) words{ "release-held": worded(func(o conditions.Observation) words { return releaseHeldWords(nil, o.Also) }), + "release-silent": worded(func(o conditions.Observation) words { + return releaseSilentWords(o.Also) + }), "facts-stale": worded(func(o conditions.Observation) words { return words{Headline: "Merge checks use outdated facts", Explanation: "The facts a merge check judges a change against are out of date; they are gathered again " + @@ -841,6 +844,19 @@ func releaseHeldWords(modules, machines []string) words { Resolved: "Resolved: the held updates are released"} } +// releaseSilentWords are the plain words of machines that said nothing on an update they were sent (ADR 0353 §4). +func releaseSilentWords(machines []string) words { + where := "Some machines" + if len(machines) > 0 { + where = conditions.Capital(namesWords(machines, 4)) + } + return words{Headline: "Machines did not answer an update", + Needs: "check that they are awake and on the mesh; the next walk sends them what waits.", + Explanation: where + " were sent module updates and said nothing about them within fifteen minutes, so " + + "they were left as they were. What they run is not known to be the update.", + Resolved: "Resolved: the next walk reached them"} +} + // reloginNeeds is what an account waiting for its groups needs (ADR 0252). func reloginNeeds(node string) string { return fmt.Sprintf("log out of %s completely and log in again, or restart it.", node) diff --git a/cmd/mesh-controller/release.go b/cmd/mesh-controller/release.go index ca9c0bd4..37d68311 100644 --- a/cmd/mesh-controller/release.go +++ b/cmd/mesh-controller/release.go @@ -39,8 +39,8 @@ import ( // from the same manifest is no move. // // **The release plan** is what walks the waiting moves through: whenever moves wait for a gate that no -// started plan is walking, the mesh opens one — every such machine, one at a time, the control node last, -// each sent and judged before the next. A release plan that fails stops the next one opening on its own: +// started plan is walking, the mesh opens one — every such machine, the first sent and judged, then the rest +// at once, each judged by its own gate (novox/hq ADR 0353, in place of one at a time). A release plan that fails stops the next one opening on its own: // a person releases it again (`upgrade release-backlog --why`), after the condition says why. // sendScope is what a send may carry that no gate has seen. @@ -81,6 +81,15 @@ type moveFacts struct { srcs map[string]map[string]string passed map[string]map[string]bool plans []inventory.Plan + // self is the release walking these moves, whose own gates are no other walk's (novox/hq ADR 0353). + self string +} + +type walkingKey struct{} + +// withWalking says a release plan is the one sending: its own gates never hold its own sends back. +func withWalking(ctx context.Context, id string) context.Context { + return context.WithValue(ctx, walkingKey{}, id) } func readMoveFacts(ctx context.Context, inv *inventory.Inventory) (moveFacts, error) { @@ -99,6 +108,7 @@ func readMoveFacts(ctx context.Context, inv *inventory.Inventory) (moveFacts, er return f, err } f.plans, err = inv.OpenPlans(ctx) + f.self, _ = ctx.Value(walkingKey{}).(string) return f, err } @@ -193,17 +203,24 @@ func (f moveFacts) moves(node string, modules []string, sent map[string]string, // feet. A release keeps no module records: its open gate's carried moves are its walk. func (f moveFacts) walkedBy(module, node, to string) string { for _, p := range f.plans { - if !p.Open() { + if !p.Open() || (p.Release != nil && p.ID == f.self) { continue } if p.Release != nil { - g := p.Release.Gate - if g == nil || g.Verdict != "" { - continue + // The first machine's gate, and every gate of the rest still judging (novox/hq ADR 0353). + gates := []*inventory.PlanGate{p.Release.Gate} + for _, g := range p.Release.Gates { + gates = append(gates, g) } - for _, c := range g.Carried { - if c.Module == module && !(slices.Contains(g.Machines, node) && sameCommit(c.To, to)) { - return p.ID + for _, g := range gates { + // A send kept and not yet made walks nothing yet (ADR 0353 §8). + if g == nil || g.Verdict != "" || g.Silent || g.Pending { + continue + } + for _, c := range g.Carried { + if c.Module == module && !(slices.Contains(g.Machines, node) && sameCommit(c.To, to)) { + return p.ID + } } } continue @@ -303,8 +320,8 @@ func ungatedIn(ctx context.Context, open *stores, names []string, addedHolder st } } if len(refused) > 0 { - return nil, fmt.Errorf("%w: %s — a walk sends them, one machine at a time, each judged (novox/hq ADR "+ - "0236); `upgrade backlog` lists them", errUngated, strings.Join(refused, "; ")) + return nil, fmt.Errorf("%w: %s — a walk sends them, one machine first and then the rest at once, each judged (novox/hq ADR "+ + "0236, ADR 0353); `upgrade backlog` lists them", errUngated, strings.Join(refused, "; ")) } return kept, nil } @@ -320,19 +337,35 @@ func ungatedIn(ctx context.Context, open *stores, names []string, addedHolder st // And the grants step first, when the send changes what the bus's user list must say (novox/hq issue 490): // answered for the gate to keep, nil when there was none. func gatedSend(ctx context.Context, open *stores, node string, owns []inventory.CarriedMove) ([]inventory.CarriedMove, []string, *inventory.GrantsStep, error) { - inv := open.inventory - f, err := readMoveFacts(ctx, inv) + moves, err := gatedMoves(ctx, open, node, owns) + if err != nil || len(moves) == 0 { + return nil, nil, nil, err + } + sayRecreations(ctx, open, moves) + sent, grants, err := sendJudged(ctx, open, node) if err != nil { return nil, nil, nil, err } + return moves, sent, grants, nil +} + +// gatedMoves is what a gated send to one machine would carry, with the build each moves to, made of nothing: +// gatedSend's reading, which a walk also keeps before it sends (novox/hq ADR 0353 §8). Refused the same way +// while a plan that has started walks one of those builds elsewhere. +func gatedMoves(ctx context.Context, open *stores, node string, owns []inventory.CarriedMove) ([]inventory.CarriedMove, error) { + inv := open.inventory + f, err := readMoveFacts(ctx, inv) + if err != nil { + return nil, err + } moves, err := machineMoves(ctx, open, f, node, true) if err != nil { - return nil, nil, nil, err + return nil, err } // A move the send keeps is not made, and is no move for the gate to judge (novox/hq issue 201): a core module // kept over an older record judged as moving back to it would read the gate as superseded by its own send. if moves, err = withoutKept(ctx, open, node, moves); err != nil { - return nil, nil, nil, err + return nil, err } own := func(module string) bool { return slices.ContainsFunc(owns, func(o inventory.CarriedMove) bool { return o.Module == module }) @@ -342,7 +375,7 @@ func gatedSend(ctx context.Context, open *stores, node string, owns []inventory. continue } if id := f.walkedBy(mv.Module, node, mv.To); id != "" { - return nil, nil, nil, fmt.Errorf("%w: %s's build %s waits on %s, which %s is walking", errWalkedElsewhere, + return nil, fmt.Errorf("%w: %s's build %s waits on %s, which %s is walking", errWalkedElsewhere, mv.Module, short(mv.To), node, id) } } @@ -350,7 +383,7 @@ func gatedSend(ctx context.Context, open *stores, node string, owns []inventory. // plan, or a release, on this machine is not crossed with a newer build; this send waits for its gate. for _, o := range owns { if id := f.walkedBy(o.Module, node, o.To); id != "" { - return nil, nil, nil, fmt.Errorf("%w: %s's build %s waits on %s, which %s is walking", errWalkedElsewhere, + return nil, fmt.Errorf("%w: %s's build %s waits on %s, which %s is walking", errWalkedElsewhere, o.Module, short(o.To), node, id) } } @@ -363,30 +396,26 @@ func gatedSend(ctx context.Context, open *stores, node string, owns []inventory. moves[i].Build = o.Build } } - if len(moves) == 0 && len(owns) == 0 { - return nil, nil, nil, nil + if len(moves) == 0 { + return nil, nil } for i := range moves { if moves[i].Build == "" { if moves[i].Build, err = inv.BuildOf(ctx, moves[i].Module, moves[i].To); err != nil { - return nil, nil, nil, err + return nil, err } } } sort.Slice(moves, func(i, j int) bool { return moves[i].Module < moves[j].Module }) - sayRecreations(ctx, open, moves) - sent, grants, err := sendJudged(ctx, open, node) - if err != nil { - return nil, nil, nil, err - } - return moves, sent, grants, nil + return moves, nil } // passCarried keeps a pass as the verdict of every build the gate judged beside its own module. func passCarried(ctx context.Context, open *stores, p *inventory.Plan, g *inventory.PlanGate, except string) { seen := map[string]bool{} for _, c := range g.Carried { - if c.Module == except || c.Build == "" || seen[c.Build] { + // A build put back with another machine's failure is not this gate's to pass (novox/hq ADR 0353 §3). + if c.Module == except || c.Build == "" || seen[c.Build] || slices.Contains(g.Returned, c.Module) { continue } seen[c.Build] = true @@ -695,7 +724,7 @@ func releaseBacklog(ctx context.Context, open *stores, by string) (*inventory.Pl p := inventory.Plan{ID: fmt.Sprintf("release-%d", now.UnixNano()), Repository: releaseRepository, Created: now, State: inventory.PlanRolling, Tiers: [][]string{names}, Modules: map[string]*inventory.PlanModule{}, Release: &inventory.PlanRelease{Order: order, By: by}, - Note: fmt.Sprintf("%d build(s) wait for a gate on %s; one machine at a time, each judged", len(names), + Note: fmt.Sprintf("%d build(s) wait for a gate on %s; one machine first, then the rest at once, each judged", len(names), strings.Join(order, ", "))} if err := inv.SavePlan(ctx, &p); err != nil { return nil, err @@ -704,14 +733,23 @@ func releaseBacklog(ctx context.Context, open *stores, by string) (*inventory.Pl return &p, nil } -// advanceRelease takes one step of a release plan: the next machine sent everything waiting there under a -// gate, or the machine being judged judged once more; the plan done when every machine is. +// advanceRelease takes one step of a release plan: the first machine sent everything waiting there under a +// gate and judged; once it passed, every other machine sent at once, each under its own gate (novox/hq ADR +// 0353 §1), and judged; the plan done when every machine is. +// +// **A send is kept before it is made** (ADR 0353 §8): the step that chooses a machine keeps its gate pending with +// what it will carry, and the next step makes the send. A controller replaced between the send and keeping it +// finds the gate pending, reads the machine's last send, and neither sends again nor leaves the machine unjudged. func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool, error) { r := p.Release if r.Settings != nil { return advanceSettingsWalk(ctx, open, p) } + ctx = withWalking(ctx, p.ID) now := time.Now().UTC() + if r.Rest { + return advanceRest(ctx, open, p, now) + } if r.Gate == nil { heard, err := releaseHeard(ctx, open) if err != nil { @@ -724,7 +762,7 @@ func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool, r.Next++ continue } - moves, sent, grants, err := gatedSend(ctx, open, node, nil) + moves, err := gatedMoves(ctx, open, node, nil) if errors.Is(err, errWalkedElsewhere) { note := fmt.Sprintf("waiting before %s: %v", node, err) changed := p.Note != note @@ -732,23 +770,15 @@ func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool, return changed, nil } if err != nil { - return false, fmt.Errorf("sending %s what waits there: %w", node, err) + return false, fmt.Errorf("reading what waits on %s: %w", node, err) } if len(moves) == 0 { r.Done = append(r.Done, node) r.Next++ continue } - r.Gate = &inventory.PlanGate{Machines: sent, Since: &now, Carried: moves, Sent: sentNow(ctx, open.inventory, sent), - Grants: grants} - p.Note = fmt.Sprintf("sent %s %d build(s) that waited for a gate; judging them there", node, len(moves)) - if said := grantsSaid(grants); said != "" { - p.Note += "; " + said - } - if said := recreationsSaid(moves); said != "" { - p.Note += "; " + said - } - fmt.Printf("%s: %s\n", p.ID, p.Note) + r.Gate = &inventory.PlanGate{Machines: []string{node}, Carried: moves, Pending: true} + p.Note = fmt.Sprintf("sending %s %d build(s) that wait for a gate, first", node, len(moves)) return true, nil } p.State, p.Tier = inventory.PlanDone, len(p.Tiers) @@ -759,6 +789,19 @@ func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool, return true, nil } g := r.Gate + if g.Pending { + node := firstOf(g.Machines) + out, err := sendKept(ctx, open, p, node, g, now) + if err != nil || out == keptWaiting { + return out == keptWaiting, err + } + if out == keptNothing { + r.Gate = nil + r.Done = append(r.Done, node) + r.Next++ + } + return true, nil + } verdict, err := judgeMoves(ctx, open, g, nil, now) if err != nil { return false, err @@ -777,6 +820,11 @@ func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool, r.Done = append(r.Done, firstOf(g.Machines)) r.Next++ r.Gate = nil + // **Then the rest at once** (novox/hq ADR 0353 §1): every other machine heard from, each judged by its own + // gate, in place of one machine at a time (ADR 0236 §22). + r.Rest = true + p.Note = fmt.Sprintf("passed on %s; the rest are sent at once, each judged", firstOf(g.Machines)) + fmt.Printf("%s: %s\n", p.ID, p.Note) return true, nil case inventory.GateSuperseded: // Another send moved a judged module on the judged machine (novox/hq issue 352): no verdict on what @@ -797,6 +845,453 @@ func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool, return true, nil } +// What sendKept did with a kept send. +const ( + keptSent = iota // sent now + keptAlready // the machine was already sent it: judged from that send, nothing sent + keptNothing // nothing waits there any more: no gate + keptWaiting // another walk judges a build it would carry: it waits +) + +// sendKept makes a send a gate kept before it was made (novox/hq ADR 0353 §8) — unless the machine's last send +// already carries what the gate is for, made before a controller was replaced and before it was kept: then +// nothing is sent, and the gate judges from that send (§6). +func sendKept(ctx context.Context, open *stores, p *inventory.Plan, node string, g *inventory.PlanGate, + now time.Time) (int, error) { + inv := open.inventory + moves, sent, grants, err := gatedSend(ctx, open, node, nil) + if errors.Is(err, errWalkedElsewhere) { + p.Note = fmt.Sprintf("waiting before %s: %v", node, err) + return keptWaiting, nil + } + if err != nil { + return keptSent, fmt.Errorf("sending %s what waits there: %w", node, err) + } + if len(moves) > 0 { + g.Machines, g.Carried, g.Since, g.Grants, g.Pending = sent, moves, &now, grants, false + g.Sent = sentNow(ctx, inv, sent) + p.Note = fmt.Sprintf("sent %s %d build(s) that waited for a gate; judging them there", node, len(moves)) + if said := grantsSaid(grants); said != "" { + p.Note += "; " + said + } + if said := recreationsSaid(moves); said != "" { + p.Note += "; " + said + } + fmt.Printf("%s: %s\n", p.ID, p.Note) + return keptSent, nil + } + if !alreadySent(ctx, inv, node, g.Carried) { + return keptNothing, nil + } + since := now + if s, found, err := inv.SentTo(ctx, node); err == nil && found { + if !s.At.IsZero() { + since = s.At + } + g.Sent = map[string]inventory.SentDeclaration{node: s} + } + g.Since, g.Pending = &since, false + p.Note = fmt.Sprintf("%s was already sent the %d build(s) kept for it, at %s: judged from that send, nothing sent again", + node, len(g.Carried), since.Local().Format("15:04:05")) + fmt.Printf("%s: %s\n", p.ID, p.Note) + return keptAlready, nil +} + +// alreadySent says a machine's last send carries every build of the moves: a send made and not kept. +func alreadySent(ctx context.Context, inv *inventory.Inventory, node string, moves []inventory.CarriedMove) bool { + if len(moves) == 0 { + return false + } + sent, known, err := inv.SentBuilds(ctx, node) + if err != nil || !known { + return false + } + for _, mv := range moves { + if !sameCommit(sent[mv.Module], mv.To) { + return false + } + } + return true +} + +// advanceRest takes one step of a release plan's rest (novox/hq ADR 0353): the sends kept for a step made, every +// machine of the step judged by its own gate, one that fails put back by itself while the others go on to their +// verdicts, and the next step — or the end — once every gate of the step has its verdict or is silent. +func advanceRest(ctx context.Context, open *stores, p *inventory.Plan, now time.Time) (bool, error) { + r := p.Release + // The sends kept for this step, in the plan's order: the control node, last in it, last (§1). + var pending []string + for _, n := range r.Order { + if g := r.Gates[n]; g != nil && g.Pending { + pending = append(pending, n) + } + } + if len(pending) > 0 { + // **A kept send carries only what was kept** (ADR 0353 §2): a build that passed no gate and came to wait on + // a machine since its step was chosen could reach a second machine in it; the step is chosen again. + if grew, err := keptStepGrew(ctx, open, r, pending); err != nil { + return false, err + } else if grew != "" { + for _, n := range pending { + delete(r.Gates, n) + } + opened, waiting, err := openRestStep(ctx, open, p) + if err != nil { + return true, err + } + p.Note = "chosen again: " + grew + if !opened && waiting != "" { + p.Note += "; " + waiting + } + return true, nil + } + var sent, already, waiting []string + for _, node := range pending { + out, err := sendKept(ctx, open, p, node, r.Gates[node], now) + if err != nil { + return true, err + } + switch out { + case keptSent: + sent = append(sent, node) + case keptAlready: + already = append(already, node) + case keptNothing: + delete(r.Gates, node) + r.Done = append(r.Done, node) + case keptWaiting: + waiting = append(waiting, node) + } + } + var said []string + if len(sent) > 0 { + said = append(said, "sent "+strings.Join(sent, ", ")+" at once, each judged there") + } + if len(already) > 0 { + said = append(said, strings.Join(already, ", ")+" already sent it: judged from that send") + } + if len(waiting) > 0 { + said = append(said, "waiting before "+strings.Join(waiting, ", ")+": another walk judges a build it carries") + } + if len(said) > 0 { + p.Note = strings.Join(said, "; ") + } + return len(sent)+len(already) > 0 || len(waiting) < len(pending), nil + } + reports, err := open.inventory.LastReports(ctx) + if err != nil { + return false, err + } + byNode := map[string]inventory.Reported{} + for _, rep := range reports { + byNode[rep.Node] = rep + } + changed, noteWas := false, p.Note + var notes, judging []string + batched, back := batchingRollbacks(ctx) + for _, node := range r.Order { + g := r.Gates[node] + if g == nil || g.Verdict != "" || g.Silent { + continue + } + // What another machine's failure already put back is not this gate's to judge (ADR 0353 §3). + if len(g.Carried) > 0 && len(g.Returned) >= len(g.Carried) { + decide(g, inventory.GateFailed, "what it was sent was put back with another machine's failure", now) + r.Failed = append(r.Failed, node) + changed = true + continue + } + // **A machine that says nothing on its send is left silent, not failed** (ADR 0353 §4): it is not judged + // until it reports, and past ApplySilentAfter it is listed and left. + if !reportedOnSend(byNode[node], node, g) { + if now.Sub(*g.Since) > inventory.ApplySilentAfter { + g.Silent = true + r.Silent = append(r.Silent, node) + changed = true + notes = append(notes, fmt.Sprintf("%s said nothing on its send within %s: left silent", node, + inventory.ApplySilentAfter)) + continue + } + judging = append(judging, node+": not yet reported on its send") + continue + } + verdict, err := judgeMoves(ctx, open, g, nil, now) + if err != nil { + return changed, err + } + switch verdict { + case "": + if putBackBroken(ctx, open, p, g, "", nil) { + changed = true + notes = append(notes, p.Note) + } + judging = append(judging, node+": "+gateLine(g)) + case inventory.GatePassed: + passCarried(ctx, open, p, g, "") + r.Done = append(r.Done, node) + changed = true + case inventory.GateSuperseded: + // Another send moved a judged module there (novox/hq issue 352): nothing judged, nothing put back on + // it; what still waits is released again. + r.Done = append(r.Done, node) + changed = true + notes = append(notes, fmt.Sprintf("superseded on %s: %s", node, g.Why)) + default: + // **Put back by itself** (ADR 0353 §3): what was found wanting goes back at once, and every other + // machine of the step goes on to its own verdict. + stateWas := p.State + p.Note = "" + failCarried(batched, open, p, g, "") + p.State = stateWas + r.Failed = append(r.Failed, node) + changed = true + notes = append(notes, fmt.Sprintf("failed its gate on %s: %s — %s", node, g.Why, p.Note)) + returnedElsewhere(ctx, open, r, node, g) + } + } + p.Note = "" + sendRollbacks(ctx, open, p, back) + if p.Note != "" { + notes = append(notes, strings.TrimPrefix(p.Note, "; ")) + } + if len(judging) > 0 { + note := strings.Join(append(notes, "judging "+strings.Join(judging, "; ")), "; ") + p.Note = note + return changed || noteWas != note, nil + } + // Every gate of the step has its verdict or is silent: the next step, unless a machine failed (§3). + if len(r.Failed) == 0 { + opened, waiting, err := openRestStep(ctx, open, p) + if err != nil { + return changed, err + } + if opened { + return true, nil + } + if waiting != "" { + note := strings.Join(append(notes, waiting), "; ") + p.Note = note + return changed || noteWas != note, nil + } + } + p.Tier = len(p.Tiers) + end := fmt.Sprintf("released on %s", orNone(strings.Join(r.Done, ", "))) + if len(r.Silent) > 0 { + end += "; silent on their send, left: " + strings.Join(r.Silent, ", ") + } + if len(r.Skipped) > 0 { + end += "; not heard from, left as they were: " + strings.Join(r.Skipped, ", ") + } + p.State = inventory.PlanDone + if len(r.Failed) > 0 { + p.State = inventory.PlanFailed + end = "failed on " + strings.Join(r.Failed, ", ") + "; " + end + } else if len(r.Silent) > 0 && len(r.Gates) == len(r.Silent) { + // **A rest that said nothing is no success** (ADR 0353 §4): failure is loud; the condition says it. + p.State = inventory.PlanFailed + end = "no machine of the rest reported on its send; " + end + } + p.Note = strings.Join(append(notes, end), "; ") + fmt.Printf("%s: %s\n", p.ID, p.Note) + return true, nil +} + +// keptStepGrew says why a step's kept sends would now carry a build no gate passed that they were not kept with; +// empty when none would. +func keptStepGrew(ctx context.Context, open *stores, r *inventory.PlanRelease, pending []string) (string, error) { + f, err := readMoveFacts(ctx, open.inventory) + if err != nil { + return "", err + } + shelf, err := open.inventory.Catalogue(ctx) + if err != nil { + return "", err + } + for _, node := range pending { + moves, err := gatedMoves(ctx, open, node, nil) + if errors.Is(err, errWalkedElsewhere) { + continue // sendKept says it waits + } + if err != nil { + return "", err + } + kept := r.Gates[node].Carried + for _, mv := range moves { + // The node-engine and the resolver keep one machine at a time, passed or not (ADR 0353 §9). + if (f.gated(mv.Module, mv.To) && !oneAtATime(mv.Module, shelf[mv.Module])) || slices.ContainsFunc(kept, func(c inventory.CarriedMove) bool { + return c.Module == mv.Module && sameCommit(c.To, mv.To) + }) { + continue + } + return fmt.Sprintf("%s's build %s, which this step did not keep for it, came to wait on %s", mv.Module, short(mv.To), node), nil + } + } + return "", nil +} + +// reportedOnSend says a machine's last report is about the send its gate judges (novox/hq issue 352): the +// declaration kept on the gate, or one sequenced after it; a gate that kept none reads the send made last. +func reportedOnSend(r inventory.Reported, node string, g *inventory.PlanGate) bool { + if r.At == nil { + return false + } + if s, kept := g.Sent[node]; kept { + return s.ReportsOn(r) + } + return r.Current +} + +// returnedElsewhere marks, on every other gate of the rest still judging, the builds a failed machine's +// rollback put back wherever they were sent (ADR 0236 §5): no longer theirs to judge (ADR 0353 §3). +func returnedElsewhere(ctx context.Context, open *stores, r *inventory.PlanRelease, failed string, g *inventory.PlanGate) { + for _, c := range g.Carried { + if c.Build == "" { + continue + } + if marked, err := open.inventory.GateFailed(ctx, c.Build); err != nil || !marked { + continue + } + for node, other := range r.Gates { + if node == failed || other.Verdict != "" || slices.Contains(other.Returned, c.Module) { + continue + } + for _, oc := range other.Carried { + if oc.Module == c.Module && sameCommit(oc.To, c.To) { + other.Returned = append(other.Returned, c.Module) + } + } + } + } +} + +// openRestStep chooses the next step of a release plan's rest and keeps its sends (novox/hq ADR 0353 §1, §2): +// every machine left that is heard from and has something waiting, except that a build no gate has passed goes +// to one machine of the step only. Answers whether a step was kept, or why the walk waits. +func openRestStep(ctx context.Context, open *stores, p *inventory.Plan) (bool, string, error) { + r := p.Release + heard, err := releaseHeard(ctx, open) + if err != nil { + return false, "", err + } + f, err := readMoveFacts(ctx, open.inventory) + if err != nil { + return false, "", err + } + waiting := map[string][]inventory.CarriedMove{} + var nodes, walked []string + for _, node := range r.Order[min(r.Next, len(r.Order)):] { + if r.Gates[node] != nil || slices.Contains(r.Done, node) || slices.Contains(r.Skipped, node) || + slices.Contains(r.Failed, node) || slices.Contains(r.Silent, node) { + continue + } + if !heard[node] { + r.Skipped = append(r.Skipped, node) + continue + } + moves, err := gatedMoves(ctx, open, node, nil) + if errors.Is(err, errWalkedElsewhere) { + walked = append(walked, fmt.Sprintf("%s (%v)", node, err)) + continue + } + if err != nil { + return false, "", fmt.Errorf("reading what waits on %s: %w", node, err) + } + if len(moves) == 0 { + r.Done = append(r.Done, node) + continue + } + waiting[node] = moves + nodes = append(nodes, node) + } + shelf, err := open.inventory.Catalogue(ctx) + if err != nil { + return false, "", err + } + step := restStep(nodes, waiting, f.gated, func(module string) bool { return oneAtATime(module, shelf[module]) }) + if len(step) == 0 { + if len(walked) > 0 { + return false, "waiting before " + strings.Join(walked, "; "), nil + } + return false, "", nil + } + if r.Gates == nil { + r.Gates = map[string]*inventory.PlanGate{} + } + for _, node := range step { + r.Gates[node] = &inventory.PlanGate{Machines: []string{node}, Carried: waiting[node], Pending: true} + } + p.Note = fmt.Sprintf("sending %s at once, each judged there", strings.Join(step, ", ")) + if left := len(nodes) - len(step); left > 0 { + p.Note += fmt.Sprintf("; %d machine(s) wait for a build that passed no gate yet to pass on one", left) + } + return true, "", nil +} + +// oneAtATime is a module whose builds go to the rest one machine at a time (novox/hq ADR 0353 §9): the +// node-engine, which applies every send, and a module holding the mesh's resolver, which names the bus. +func oneAtATime(module string, m catalogue.Manifest) bool { + if module == hostModule { + return true + } + return slices.ContainsFunc(m.Claims, func(c catalogue.Claim) bool { return c.Name == resolverSeat }) +} + +// restStep is the machines of one step of the rest (novox/hq ADR 0353 §2), in the order given: each machine, unless +// a build it would carry has passed no gate and an earlier machine of the step already carries it — that build's +// first machine. Pure. +// +// **The node-engine and the mesh's resolver still go one machine at a time** (ADR 0353 §9): a step carries a +// build of either to one machine only, judged before the next, since a fault that cuts a machine off the bus +// shows only as its silence and no rollback reaches it. +func restStep(nodes []string, waiting map[string][]inventory.CarriedMove, passed func(module, commit string) bool, + alone func(module string) bool) []string { + first := map[string]string{} + var out []string + for _, n := range nodes { + var unpassed []string + for _, mv := range waiting[n] { + if !passed(mv.Module, mv.To) || alone(mv.Module) { + unpassed = append(unpassed, mv.Module+"@"+mv.To) + } + } + if slices.ContainsFunc(unpassed, func(k string) bool { _, taken := first[k]; return taken }) { + continue + } + for _, k := range unpassed { + first[k] = n + } + out = append(out, n) + } + return out +} + +// silentObservation is what the gate's probe says of the newest release whose rest went silent (novox/hq ADR 0353 +// §4): machines that never reported on their send, left as they were — loud, never a success said by nothing. +func silentObservation(ctx context.Context, inv *inventory.Inventory) ([]conditions.Observation, error) { + recent, err := inv.RecentPlans(ctx, 20) + if err != nil { + return nil, err + } + for _, p := range recent { + if p.Release == nil || p.Release.Settings != nil || p.Open() { + continue + } + if len(p.Release.Silent) == 0 { + return nil, nil + } + machines := append([]string(nil), p.Release.Silent...) + sort.Strings(machines) + o := conditions.Observation{Scope: conditions.ScopeMesh, ID: "release", Token: "silent", Kind: "release-silent", + Severity: conditions.Warning, Resolver: conditions.ResolverOperator, Also: machines, + Summary: fmt.Sprintf("%s: %s said nothing on the send of the builds that waited for a gate within %s, and were "+ + "left as they were", p.ID, strings.Join(machines, ", "), inventory.ApplySilentAfter)} + w := releaseSilentWords(machines) + o.Headline, o.Explanation, o.Resolved, o.Needs = w.Headline, w.Explanation, w.Resolved, w.Needs + return []conditions.Observation{o}, nil + } + return nil, nil +} + // backlogObservation is what the gate's probe says of a release held after a failure. func backlogObservation() []conditions.Observation { if backlogNow.held == "" || len(backlogNow.waiting) == 0 { -- 2.54.0 From c88474a1273c76470659e456d2116e11d62c618b Mon Sep 17 00:00:00 2001 From: mesh-admin Date: Wed, 14 Oct 2026 06:20:08 +0000 Subject: [PATCH 24/31] Bring main into release 57's branch, part 2 of 3: the rest at once and the try verb So the release branch carries what main gained; part 3 follows on this branch. Co-Authored-By: Claude Opus 5.5 --- cmd/mesh-controller/release_plan.go | 193 +++++++++++- cmd/mesh-controller/rest_at_once_test.go | 372 +++++++++++++++++++++++ cmd/mesh-controller/seatverbs.go | 28 ++ cmd/mesh-controller/try.go | 353 +++++++++++++++++++++ cmd/mesh-controller/try_test.go | 194 ++++++++++++ 5 files changed, 1130 insertions(+), 10 deletions(-) create mode 100644 cmd/mesh-controller/rest_at_once_test.go create mode 100644 cmd/mesh-controller/try.go create mode 100644 cmd/mesh-controller/try_test.go diff --git a/cmd/mesh-controller/release_plan.go b/cmd/mesh-controller/release_plan.go index 77060d8b..f28516fa 100644 --- a/cmd/mesh-controller/release_plan.go +++ b/cmd/mesh-controller/release_plan.go @@ -251,6 +251,20 @@ func gates(p inventory.Plan, edges []inventory.Edge, rollsOut func(string) bool) return out } +// noMoveWhy is what a plan says of a module of its tier whose build was made from the source every machine +// running it runs (novox/hq issue 280): nothing is sent for it. +// noMovePrefix opens it, and is what noMove reads: one prefix for the writer and the reader. +const ( + noMovePrefix = "no move" + noMoveWhy = noMovePrefix + ": made from the source every machine running it runs" +) + +// noMove is whether the plan marked the module no move and sent it nothing. Read from what the plan says, +// so a plan kept before the mark had a name of its own is read the same (novox/hq issue 648). +func noMove(s *inventory.PlanModule) bool { + return s != nil && s.FirstAt == nil && s.Gate == nil && strings.HasPrefix(s.Why, noMovePrefix) +} + // applied says whether every machine running the module has reported since the module was built. // appliedEach is applied with a moment of its own for each machine: the reports that count are the ones // after that machine was sent the build (novox/hq issue 256). @@ -595,7 +609,7 @@ func advancePlans(ctx context.Context, open *stores) { return } defer release() - // What waits for a gate, released one machine at a time (ADR 0236). + // What waits for a gate, released one machine first, then the rest at once (ADR 0236, ADR 0353). if _, err := releaseBacklog(ctx, open, ""); err != nil { fmt.Printf("plans: what waits for a gate could not be looked at: %v\n", err) } @@ -805,7 +819,7 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan, } else if same { now := time.Now().UTC() state.SentAt = &now - state.Why = "no move: made from the source every machine running it runs" + state.Why = noMoveWhy fmt.Printf("%s: %s built from the source every machine running it runs: no move, nothing sent\n", p.ID, m) continue } @@ -995,6 +1009,25 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan, if state.BuiltAt != nil { since = *state.BuiltAt } + // **A build that is no move is waited on by the send that last carried it** (novox/hq issue 648). + // Nothing was sent for it, so no report follows its marking or its build: a machine that does not + // happen to report again held the walk for ever. Each machine runs it once it answered its last + // send — the one whose builds made it no move — whenever that was. + if noMove(state) { + byNode := map[string]inventory.Reported{} + for _, r := range reports { + byNode[r.Node] = r + } + if ok, on := appliedEach(m, func(node string) time.Time { + if s := byNode[node].Sent; s != nil { + return *s + } + return time.Time{} + }, running, reports); !ok { + waiting = append(waiting, fmt.Sprintf("%s on %s", m, strings.Join(on, ", "))) + } + continue + } // **Each machine from its own send** (novox/hq issue 256). With one machine first (ADR 0218) // a module is sent twice — the first machine, then the rest — and SentAt is the second. // Asked of every machine, the first machine's report, made between the two sends, read as @@ -1056,10 +1089,36 @@ func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string runningOf map[string][]string) error { inv := open.inventory before, beforeKnown, _ := inv.SentBuilds(ctx, node) + // **A machine already sent what this send is for is not sent again** (novox/hq ADR 0353 §6): sent by this plan + // before a controller was replaced and before it was kept, or by another walk. Judged from that send, and from + // the readings of a gate already judging it there. + // Only a send the machine reported it applied is one it has; one it never reported on is sent, as before + // (issue 280: what it ran before is then that build, and a failure leaves it as it was). + if beforeKnown && slices.IndexFunc(modules, func(m string) bool { return !sameCommit(before[m], p.Modules[m].Commit) }) < 0 && + appliedLastSend(ctx, inv, node) { + if others, err := gatedMoves(ctx, open, node, nil); err == nil && len(others) == 0 { + adoptSend(ctx, open, p, node, modules, before, runningOf) + return nil + } + } + // **What the machine ran before is kept before the send is made** (novox/hq ADR 0353 §6): a controller replaced + // between the send and keeping it reads it here, and a failure still puts the machine back. + kept := false + for _, m := range modules { + s := p.Modules[m] + if beforeKnown && (s.Previous == "" || !sameCommit(before[m], s.Commit)) && s.Previous != before[m] { + s.Previous, kept = before[m], true + } + } + if kept { + if err := inv.SavePlan(ctx, p); err != nil { + return fmt.Errorf("what %s ran before is not kept, so nothing is sent: %w", node, err) + } + } owns := make([]inventory.CarriedMove, 0, len(modules)) for _, m := range modules { s := p.Modules[m] - owns = append(owns, inventory.CarriedMove{Module: m, Node: node, From: before[m], To: s.Commit, Build: s.Build}) + owns = append(owns, inventory.CarriedMove{Module: m, Node: node, From: s.Previous, To: s.Commit, Build: s.Build}) } // A gated send (ADR 0236): everything waiting on the machine goes with the tier, and the gate judges // all of it there. @@ -1074,10 +1133,7 @@ func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string sentWhat := sentNow(ctx, inv, sent) for _, m := range modules { s := p.Modules[m] - // What the first machine ran before: what a failed gate puts back (ADR 0236). - if beforeKnown { - s.Previous = before[m] - } + // What the first machine ran before (ADR 0236), kept above before the send. s.First, s.FirstAt = sent, &now s.Gate = &inventory.PlanGate{Component: coreComponent(m), Machines: firstRunning(sent, runningOf[m]), From: s.Previous, To: s.Commit, Since: &now, Sent: sentWhat} @@ -1106,6 +1162,113 @@ func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string return nil } +// adoptSend records a first send the machine already had (novox/hq ADR 0353 §6, §7): nothing is sent; the gate counts +// from the machine's last send, and takes over the readings of an open gate of another walk judging the same build +// there — whose carried move also says what the machine ran before, for a rollback. +func adoptSend(ctx context.Context, open *stores, p *inventory.Plan, node string, modules []string, + before map[string]string, runningOf map[string][]string) { + inv := open.inventory + since := time.Now().UTC() + last, found, err := inv.SentTo(ctx, node) + if err == nil && found && !last.At.IsZero() { + since = last.At + } + lead := modules[0] + other, id, from := judgingThere(ctx, inv, p.ID, lead, node, p.Modules[lead].Commit, p.Modules[lead].Build) + for _, m := range modules { + s := p.Modules[m] + // What it ran before: kept by this walk before its send was made (§6); else what the other walk's send + // moved it from; else the build itself, so a failure leaves it as it was (issue 280). + switch { + case s.Previous != "": + case m == lead && from != "": + s.Previous = from + default: + s.Previous = before[m] + } + at := since + s.First, s.FirstAt = []string{node}, &at + s.Gate = &inventory.PlanGate{Component: coreComponent(m), Machines: firstRunning([]string{node}, runningOf[m]), + From: s.Previous, To: s.Commit, Since: &at} + if found { + s.Gate.Sent = map[string]inventory.SentDeclaration{node: last} + } + s.GatedBy = "" + if m == lead { + s.Gate.TakeReadings(other, modules, id) + } else { + s.GatedBy = lead + } + } + p.State = inventory.PlanRolling + p.Note = fmt.Sprintf("tier %d built; %s already runs %s's build: nothing sent again, judged from that send", + p.Tier, node, strings.Join(modules, ", ")) + if id != "" { + p.Note += ", going on from the readings of " + id + } + fmt.Printf("%s: %s\n", p.ID, p.Note) +} + +// appliedLastSend says a machine reported it applied the declaration it was sent last. +func appliedLastSend(ctx context.Context, inv *inventory.Inventory, node string) bool { + reports, err := inv.LastReports(ctx) + if err != nil { + return false + } + for _, r := range reports { + if r.Node == node { + return r.Current && r.At != nil && r.Outcome == inventory.OutcomeApplied + } + } + return false +} + +// judgingThere is an open gate of another walk judging a module's build on a machine (novox/hq ADR 0353 §7), the +// walk's id, and what the machine ran before; nil when none does. +func judgingThere(ctx context.Context, inv *inventory.Inventory, self, module, node, commit, build string) (*inventory.PlanGate, string, string) { + plans, err := inv.OpenPlans(ctx) + if err != nil { + return nil, "", "" + } + // The same build, by its id where both name one, else by its commit. + same := func(b, c string) bool { + if b != "" && build != "" { + return b == build + } + return sameCommit(c, commit) + } + usable := func(g *inventory.PlanGate) bool { + return g != nil && !g.Pending && g.Since != nil && (g.Verdict == "" || g.Verdict == inventory.GatePassed) && + slices.Contains(g.Machines, node) + } + for _, q := range plans { + if q.ID == self { + continue + } + if r := q.Release; r != nil { + gates := []*inventory.PlanGate{r.Gate} + for _, g := range r.Gates { + gates = append(gates, g) + } + for _, g := range gates { + if !usable(g) { + continue + } + for _, c := range g.Carried { + if c.Module == module && c.Node == node && same(c.Build, c.To) { + return g, q.ID, c.From + } + } + } + continue + } + if s := q.Modules[module]; s != nil && usable(s.Gate) && same(s.Build, s.Commit) { + return s.Gate, q.ID, s.Previous + } + } + return nil, "", "" +} + // passedWith keeps, on a module sent with others, the verdict of the gate that judged the send: the gate // on the first of them passed, and its pass was kept for every build it carried (passCarried). func passedWith(module string, s *inventory.PlanModule, lead string, g *inventory.PlanGate) { @@ -1594,9 +1757,19 @@ func plansCommand(ctx context.Context, args []string) error { // A release plan's walk (ADR 0236): machines done, the one judged, those to come. fmt.Printf(" machines in order: %s; done: %s; skipped: %s\n", strings.Join(r.Order, ", "), orNone(strings.Join(r.Done, ", ")), orNone(strings.Join(r.Skipped, ", "))) - if r.Gate != nil { - fmt.Printf(" %s\n", gateLine(r.Gate)) - for _, c := range r.Gate.Carried { + if len(r.Failed)+len(r.Silent) > 0 { + fmt.Printf(" failed: %s; silent: %s\n", orNone(strings.Join(r.Failed, ", ")), orNone(strings.Join(r.Silent, ", "))) + } + gates := []*inventory.PlanGate{r.Gate} + for _, n := range r.Order { + gates = append(gates, r.Gates[n]) + } + for _, g := range gates { + if g == nil { + continue + } + fmt.Printf(" %s\n", gateLine(g)) + for _, c := range g.Carried { fmt.Printf(" %-22s %s → %s\n", c.Module, short(c.From), short(c.To)) if c.Recreates != "" { fmt.Printf(" %-22s %s\n", "", c.Recreates) diff --git a/cmd/mesh-controller/rest_at_once_test.go b/cmd/mesh-controller/rest_at_once_test.go new file mode 100644 index 00000000..8b6f7155 --- /dev/null +++ b/cmd/mesh-controller/rest_at_once_test.go @@ -0,0 +1,372 @@ +package main + +import ( + "context" + "slices" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" +) + +// novox/hq ADR 0353: after its first machine passes, a build goes to every other machine at once, each judged by +// its own gate and a failing one put back by itself; and a gate's readings outlive the controller that took them — +// a machine already sent what a send is for is not sent again, and its judging goes on from what was read. +// +// Seen 2026-10-14: release-1791930194184991192 sent ace, g14, shanks and the control node one after another in +// eleven minutes; the new controller its last send started sent the control node again the controller build it +// had just applied, and judged it from its first reading. + +// aBacklogOnThree is the backlog fixture with a third machine, desk, running app at c1 like the others. +func aBacklogOnThree(t *testing.T) *backlogMesh { + t.Helper() + b := aBacklog(t) + ctx := t.Context() + inv := b.open.inventory + if _, err := inv.AddNode(ctx, "desk"); err != nil { + t.Fatal(err) + } + if _, err := inv.Assign(ctx, "desk", "app"); err != nil { + t.Fatal(err) + } + if err := inv.RecordSent(ctx, nodeID(t, b.open, "desk"), "d-desk", map[string]string{"app": "c1"}); err != nil { + t.Fatal(err) + } + wasHeard := releaseHeard + releaseHeard = func(context.Context, *stores) (map[string]bool, error) { + return map[string]bool{"anchor": true, "laptop": true, "desk": true}, nil + } + t.Cleanup(func() { releaseHeard = wasHeard }) + return b +} + +// events records every send and every judging, in order. +func events(t *testing.T) *[]string { + t.Helper() + var said []string + wasSend, wasGather := sendRollout, gatherGateFacts + sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { + said = append(said, "send "+strings.Join(names, ",")) + return wasSend(ctx, open, names) + } + gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) { + said = append(said, "judge") + return wasGather(ctx, open, component) + } + t.Cleanup(func() { sendRollout, gatherGateFacts = wasSend, wasGather }) + return &said +} + +// Once the first machine passed, the rest are both sent before either is judged (ADR 0353 §1); before, the second +// was judged to its verdict before the third was sent. +func TestAReleaseSendsTheRestAtOnceAfterTheFirstPasses(t *testing.T) { + judgeLeavesAsTheCore(t) + b := aBacklogOnThree(t) + ctx := t.Context() + said := events(t) + gateEvery = 0 + for i := 0; i < 12; i++ { + advancePlans(ctx, b.open) + } + p := b.release(t) + if p.State != inventory.PlanDone || len(p.Release.Done) != 3 { + t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) + } + desk, laptop := slices.Index(*said, "send desk"), slices.Index(*said, "send laptop") + if desk < 0 || laptop < 0 || slices.Index(*said, "send anchor") > min(desk, laptop) { + t.Fatalf("sends and judgings: %v", *said) + } + if slices.Contains((*said)[min(desk, laptop):max(desk, laptop)], "judge") { + t.Fatalf("the rest were sent one at a time, each judged before the next: %v", *said) + } + if v, found, err := b.open.inventory.GateOf(ctx, "build-late-c2"); err != nil || !found || v.Verdict != inventory.GatePassed { + t.Fatalf("late's pass on laptop was not kept: %+v %v %v", v, found, err) + } +} + +// A machine of the rest whose gate fails is put back at once, and the other machine of the step goes on to its own +// verdict (ADR 0353 §3); before, the walk stopped at the failure and the machine after it was never sent. +func TestARestMachineThatFailsIsPutBackAloneAndTheOthersAreJudgedOn(t *testing.T) { + judgeLeavesAsTheCore(t) + b := aBacklogOnThree(t) + ctx := t.Context() + inv := b.open.inventory + gateEvery = 0 + b.broken["desk"] = true // desk fails what it is sent in the rest's step + advancePlans(ctx, b.open) + for i := 0; i < 12 && b.release(t).Open(); i++ { + advancePlans(ctx, b.open) + } + p := b.release(t) + if p.State != inventory.PlanFailed || !slices.Equal(p.Release.Failed, []string{"desk"}) { + t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) + } + if !slices.ContainsFunc(b.sent, func(s []string) bool { return slices.Equal(s, []string{"laptop"}) }) { + t.Fatalf("laptop was never sent: %v", b.sent) + } + if !slices.Contains(p.Release.Done, "laptop") { + t.Fatalf("laptop has no verdict of its own: %+v", p.Release) + } + // late waited on laptop alone and was healthy there: its pass is kept, whatever desk did. + if v, found, err := inv.GateOf(ctx, "build-late-c2"); err != nil || !found || v.Verdict != inventory.GatePassed { + t.Fatalf("late's pass on laptop was not kept: %+v %v %v", v, found, err) + } + // app failed on desk: marked, and put back wherever it was sent (ADR 0236). + if failed, _ := inv.GateFailed(ctx, "build-app-c2"); !failed { + t.Fatal("app's build is not marked failed") + } + if sent, _, _ := inv.SentBuilds(ctx, "desk"); sent["app"] != "c1" { + t.Fatalf("desk was not put back: %v", sent) + } +} + +// A walk whose first machine already runs its build under another walk's gate sends nothing and goes on from that +// gate's readings (ADR 0353 §7); before, it sent the machine again and judged from its first reading. +func TestAWalkTakesOverTheReadingsOfAGateJudgingTheSameBuildThere(t *testing.T) { + judgeLeavesAsTheCore(t) + b := aBacklog(t) + ctx := t.Context() + inv := b.open.inventory + advancePlans(ctx, b.open) // the release sends anchor app c2, and reads it once + release := b.release(t) + judging := release.Release.Gate + if judging == nil || judging.Pending || judging.Passes == 0 { + t.Fatalf("the release is not judging anchor: %+v", judging) + } + built := time.Now().UTC() + plan := inventory.Plan{ID: "plan-app", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2", + Created: built, State: inventory.PlanBuilding, Tiers: [][]string{{"app"}}, + Modules: map[string]*inventory.PlanModule{"app": {State: "built", BuiltAt: &built, Commit: "c2", + Build: "build-app-c2"}}} + if err := inv.SavePlan(ctx, &plan); err != nil { + t.Fatal(err) + } + advancePlans(ctx, b.open) + if len(b.sent) != 1 { + t.Fatalf("anchor was sent again what it runs: %v", b.sent) + } + p, err := inv.PlanByID(ctx, "plan-app") + if err != nil { + t.Fatal(err) + } + g := p.Modules["app"].Gate + if g == nil || g.Since == nil || !g.Since.Equal(*judging.Since) || g.Passes < judging.Passes || g.TookOver != release.ID { + t.Fatalf("the walk's gate starts from zero: %+v (the release's: %+v)", g, judging) + } + if p.Modules["app"].Previous != "c1" { + t.Fatalf("what anchor ran before is %q, not c1", p.Modules["app"].Previous) + } +} + +// A walk's first send that was made and not kept — the controller replaced between the two — is not made again +// by the controller that takes over: the gate counts from that send (ADR 0353 §5, §6). +func TestAFirstSendMadeAndNotKeptIsNotSentAgain(t *testing.T) { + judgeLeavesAsTheCore(t) + g := aGateMesh(t) + ctx := t.Context() + // The old controller sends anchor app c2 and is replaced before it keeps the send. + made := sendRollout + sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { + if _, err := made(ctx, open, names); err != nil { + return nil, err + } + return nil, context.Canceled + } + advancePlans(ctx, g.open) + sendRollout = made + if len(g.sent) != 1 || g.plan(t).Modules["app"].FirstAt != nil { + t.Fatalf("the fixture's send was not made, or was kept: %v", g.sent) + } + last, _, err := g.open.inventory.SentTo(ctx, "anchor") + if err != nil { + t.Fatal(err) + } + advancePlans(ctx, g.open) + if len(g.sent) != 1 { + t.Fatalf("the send was made again: %v", g.sent) + } + s := g.plan(t).Modules["app"] + if s.FirstAt == nil || s.Gate == nil || s.Gate.Since == nil || !s.Gate.Since.Equal(last.At) { + t.Fatalf("the gate does not count from the send made: %+v %+v", s, s.Gate) + } + // What anchor ran before was kept before the send, so a failure still puts it back (§5). + if s.Previous != "c1" || s.Gate.From != "c1" { + t.Fatalf("what anchor ran before is %q (gate from %q), not c1", s.Previous, s.Gate.From) + } +} + +// A release's send kept before it was made, and made, is judged by the controller that takes over without being +// made again (ADR 0353 §8). +func TestAReleaseSendKeptAndMadeIsNotMadeAgain(t *testing.T) { + judgeLeavesAsTheCore(t) + b := aBacklog(t) + ctx := t.Context() + inv := b.open.inventory + p, err := releaseBacklog(ctx, b.open, "") + if err != nil || p == nil { + t.Fatalf("no release opened: %v", err) + } + if _, err := advanceRelease(ctx, b.open, p); err != nil || p.Release.Gate == nil || !p.Release.Gate.Pending { + t.Fatalf("the send was not kept before it was made: %+v %v", p.Release, err) + } + if err := inv.SavePlan(ctx, p); err != nil { + t.Fatal(err) + } + // The send is made, and the controller is replaced before it keeps it. + if _, err := sendRollout(ctx, b.open, []string{"anchor"}); err != nil { + t.Fatal(err) + } + advancePlans(ctx, b.open) + if len(b.sent) != 1 { + t.Fatalf("anchor was sent again: %v", b.sent) + } + g := b.release(t).Release.Gate + if g == nil || g.Pending || g.Since == nil || len(g.Carried) == 0 { + t.Fatalf("anchor is not judged on the send made: %+v", g) + } +} + +// A build no gate has passed goes to one machine of a step, its first; a machine that would be its second waits +// for the next step, and one whose every build passed goes now (ADR 0353 §2). +func TestABuildThatPassedNoGateGoesToOneMachineOfAStep(t *testing.T) { + waiting := map[string][]inventory.CarriedMove{ + "desk": {{Module: "app", To: "c2"}, {Module: "new", To: "n1"}}, + "laptop": {{Module: "app", To: "c2"}, {Module: "new", To: "n1"}}, + "shanks": {{Module: "app", To: "c2"}}, + "novox": {{Module: "app", To: "c2"}, {Module: "ctl", To: "k1"}}, + } + passed := func(module, commit string) bool { return module == "app" && commit == "c2" } + never := func(string) bool { return false } + step := restStep([]string{"desk", "laptop", "shanks", "novox"}, waiting, passed, never) + if !slices.Equal(step, []string{"desk", "shanks", "novox"}) { + t.Fatalf("the step is %v: laptop would be new's second machine before it passed", step) + } + if step := restStep([]string{"laptop"}, waiting, func(string, string) bool { return true }, never); !slices.Equal(step, []string{"laptop"}) { + t.Fatalf("a machine whose every build passed waits: %v", step) + } +} + +// The node-engine and the mesh's resolver go to the rest one machine at a time, even passed (ADR 0353 §9): a step +// carries either to one machine only, while what else passed goes everywhere at once. +func TestTheNodeEngineAndTheResolverGoToTheRestOneAtATime(t *testing.T) { + waiting := map[string][]inventory.CarriedMove{ + "desk": {{Module: hostModule, To: "h2"}}, + "laptop": {{Module: hostModule, To: "h2"}, {Module: "app", To: "c2"}}, + "shanks": {{Module: "app", To: "c2"}}, + "novox": {{Module: "dns", To: "d2"}}, + "ace": {{Module: "dns", To: "d2"}}, + } + resolver := catalogue.Manifest{Module: "dns", Claims: []catalogue.Claim{{Name: resolverSeat}}} + alone := func(m string) bool { + if m == "dns" { + return oneAtATime(m, resolver) + } + return oneAtATime(m, catalogue.Manifest{Module: m}) + } + step := restStep([]string{"desk", "laptop", "shanks", "novox", "ace"}, waiting, + func(string, string) bool { return true }, alone) + if !slices.Equal(step, []string{"desk", "shanks", "novox"}) { + t.Fatalf("the step is %v: the node-engine and the resolver each reach one machine of it", step) + } + if oneAtATime("app", catalogue.Manifest{Module: "app"}) { + t.Fatal("a leaf module is held to one machine at a time") + } +} + +// A first machine that fails still stops the walk (ADR 0236 §5, kept by ADR 0353 §3): the rest are never sent. +func TestAFirstMachineThatFailsStillStopsTheWalk(t *testing.T) { + b := aBacklogOnThree(t) + ctx := t.Context() + gateEvery = 0 + b.broken["anchor"] = true + advancePlans(ctx, b.open) + for i := 0; i < 6 && b.release(t).Open(); i++ { + advancePlans(ctx, b.open) + } + if p := b.release(t); p.State != inventory.PlanFailed || p.Release.Rest { + t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) + } + for _, s := range b.sent { + if !slices.Equal(s, []string{"anchor"}) { + t.Fatalf("sent %v: the rest after a failed first machine", b.sent) + } + } +} + +// The control node is sent last in a step of the rest (ADR 0353 §1): the machine holding the controller is +// ordered after every other. +func TestTheControlNodeIsSentLastInAStep(t *testing.T) { + judgeLeavesAsTheCore(t) + b := aBacklogOnThree(t) + ctx := t.Context() + register(t, b.open, catalogue.Manifest{Module: catalogue.ControllerSeatName, Version: "1"}) + if _, err := b.open.inventory.Assign(ctx, "anchor", catalogue.ControllerSeatName); err != nil { + t.Fatal(err) + } + said := events(t) + gateEvery = 0 + for i := 0; i < 12 && (len(b.sent) == 0 || b.release(t).Open()); i++ { + advancePlans(ctx, b.open) + } + if p := b.release(t); !slices.Equal(p.Release.Order, []string{"desk", "laptop", "anchor"}) { + t.Fatalf("the order is %v", p.Release.Order) + } + laptop, anchor := slices.Index(*said, "send laptop"), slices.Index(*said, "send anchor") + if laptop < 0 || anchor < laptop || slices.Contains((*said)[laptop:anchor], "judge") { + t.Fatalf("the control node is not sent last in the rest's step: %v", *said) + } +} + +// A machine of the rest that says nothing on its send is left silent after fifteen minutes: not failed, not put +// back; a rest that went silent whole ends failed, said as a condition (ADR 0353 §4). +func TestARestMachineSilentPastItsBoundIsLeftAndSaid(t *testing.T) { + judgeLeavesAsTheCore(t) + b := aBacklog(t) + ctx := t.Context() + inv := b.open.inventory + reporting := sendRollout + sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { + if slices.Equal(names, []string{"laptop"}) { // laptop sleeps: sent, and says nothing + b.sent = append(b.sent, names) + current, _ := open.inventory.CurrentBuilds(ctx) + return names, open.inventory.RecordSent(ctx, nodeID(t, open, "laptop"), "d-laptop-asleep", + map[string]string{"app": current["app"].Commit, "late": current["late"].Commit, "same": current["same"].Commit}) + } + return reporting(ctx, open, names) + } + t.Cleanup(func() { sendRollout = reporting }) + gateEvery = 0 + for i := 0; i < 8; i++ { + advancePlans(ctx, b.open) + } + p := b.release(t) + g := p.Release.Gates["laptop"] + if !p.Open() || g == nil || g.Verdict != "" || g.Silent { + t.Fatalf("laptop is judged before fifteen minutes: %s %s %+v", p.State, p.Note, g) + } + since := time.Now().Add(-inventory.ApplySilentAfter - time.Minute) + g.Since = &since + if err := inv.SavePlan(ctx, &p); err != nil { + t.Fatal(err) + } + advancePlans(ctx, b.open) + p = b.release(t) + if p.State != inventory.PlanFailed || !slices.Equal(p.Release.Silent, []string{"laptop"}) || len(p.Release.Failed) > 0 { + t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) + } + if failed, _ := inv.GateFailed(ctx, "build-late-c2"); failed { + t.Fatal("a silent machine's build was marked failed") + } + if sent, _, _ := inv.SentBuilds(ctx, "laptop"); sent["late"] != "c2" { + t.Fatalf("laptop was put back: %v", sent) + } + obs, err := silentObservation(ctx, inv) + if err != nil { + t.Fatal(err) + } + if len(obs) != 1 || obs[0].Kind != "release-silent" || !slices.Equal(obs[0].Also, []string{"laptop"}) { + t.Fatalf("the silence is not said: %+v", obs) + } +} diff --git a/cmd/mesh-controller/seatverbs.go b/cmd/mesh-controller/seatverbs.go index 3cc62d1a..c0c222e3 100644 --- a/cmd/mesh-controller/seatverbs.go +++ b/cmd/mesh-controller/seatverbs.go @@ -457,6 +457,28 @@ func (a *verbArguments) commandLine() ([]string, error) { return nil, err } return []string{"forget-checks", str("owner"), str("repo"), str("number")}, nil + case "try": + // A branch tried before its pull request (novox/hq issue 686): asked, or read by the id it answered. + if id := str("id"); id != "" { + return []string{"try", "--id", id}, nil + } + if a.given["repository"] == "" && a.given["ref"] == "" { + // Neither shape named: the command says its usage, which names both, as `build` does. + return []string{"try"}, nil + } + if err := need("repository", "ref"); err != nil { + return nil, err + } + argv := []string{"try", str("repository"), str("ref")} + for _, flag := range []string{"base", "tests"} { + if v := str(flag); v != "" { + argv = append(argv, "--"+flag, v) + } + } + if on("store-only") { + argv = append(argv, "--store-only") + } + return argv, nil case "cancel", "kill": if err := need("id"); err != nil { return nil, err @@ -1018,6 +1040,8 @@ var jsonVerbs = map[string]bool{"status": true, "seats": true, "plan": true, "co "delivery": true, // The build queue, each ask with its position among the waiting (novox/hq issue 455). "queue": true, + // A branch tried before its pull request (novox/hq issue 686). + "try": true, // A plan followed (novox/hq issue 229): `plans wait` answers JSON; its other forms say text, and so answer no data. "plans": true} @@ -1308,6 +1332,10 @@ func seatToolHandlers() (map[string]link.ToolHandler, []string, error) { link.AnswersWithin(ctx, bound+planWaitStart) } } + if verb == "try" && len(argv) > 1 && argv[1] == "--id" { + // Reading a try holds its answer for the verdict, up to its bound (novox/hq issue 686, 229). + link.AnswersWithin(ctx, tryWait+planWaitStart) + } if answer, read := readHere(ctx, argv); read { return answer, nil } diff --git a/cmd/mesh-controller/try.go b/cmd/mesh-controller/try.go new file mode 100644 index 00000000..a120385c --- /dev/null +++ b/cmd/mesh-controller/try.go @@ -0,0 +1,353 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "net/url" + "os" + "regexp" + "strings" + "time" + + "github.com/nats-io/nats.go" + + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/link" +) + +// A branch is tried before its pull request (novox/hq issue 686). +// +// A repository's store tests skip without a store, and no agent may start a database on its machine. So a +// builder pushed, a reviewer read, and only the pull request's merge check ran those tests for the first time: +// each failure there cost a round of fifteen to thirty minutes (mesh-controller #283, #310, #312). +// +// `try / ` asks the build seat for the repository's own check of a pushed branch or +// commit, as the merge check runs it (internal/builder/check.go): the ref merged into the trunk's head, the +// trunk's merge-check.sh, each part in its toolchain's container, against a throwaway store and bus. It is a +// check and nothing more: +// +// - **no pull request, so no status**: its outcome is never said as `checked`, which is what the forge's +// holder sets statuses from (isTry, in Built); +// - **nothing registered, nothing built**: it names no module of the graph, so the merge gate does not run and +// no build is made for a merge (novox/hq ADR 0350's scratch builds are a pull request's check's alone); +// - **a check in the build queue**: after deliveries (link.KindOf, novox/hq issue 558), and bounded by the +// check's own bound (checkTimeout); +// - **the code a pull request's check would run anyway**: only a repository of the mesh, whose every pull +// request the build seat already checks, in containers that hold no runtime socket. +// +// A check outlasts a seat verb's one answer, so the ask answers its id at once and `try --id ` reads what +// came of it from the build's log, where the build seat says it whole (link.StepTried): holding its answer up +// to tryWait for a verdict, like `plans wait` (novox/hq issue 229). Nothing is kept beside the log. + +// tryWait is how long reading a try holds its answer for the verdict. A variable so a test need not wait. +var tryWait = 20 * time.Second + +// A ref as the forge is asked for it — a branch or a commit, never an option — and a `go test -run` pattern. +var ( + tryRef = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._/-]*$`) + tryTests = regexp.MustCompile(`^[A-Za-z0-9_^$|().*+?/\[\]-]+$`) +) + +// tryAnswer is what `try` answers, asking or reading. +type tryAnswer struct { + ID string `json:"id"` + // State is asked (just now), waiting (nothing said of it yet: in the build queue), running, or the verdict: + // pass, warning (the repository declares no merge-check.sh), fail, or error (it could not run). + State string `json:"state"` + Said string `json:"said"` + Repository string `json:"repository,omitempty"` + Ref string `json:"ref,omitempty"` + Commit string `json:"commit,omitempty"` + Base string `json:"base,omitempty"` + Trunk string `json:"trunk,omitempty"` + On string `json:"on,omitempty"` + Took string `json:"took,omitempty"` + Failed []string `json:"failed,omitempty"` + // Last is the last thing the build seat said of one still running. + Last string `json:"last,omitempty"` +} + +// isTry says a check's outcome is a try's: marked so, or — echoed by a build seat from before the mark — a +// check of no pull request and no group, which is nobody's status either. +func isTry(result link.BuildResult) bool { + c := result.Checked + return c != nil && (c.Try || (c.Number == 0 && c.Group == "")) +} + +// asATry makes a check's ask a try's: the repository's own script alone, for no pull request. +func asATry(request link.BuildRequest, tests string, storeOnly bool) link.BuildRequest { + c := *request.Check + c.Number, c.Try, c.Tests, c.StoreOnly = 0, true, tests, storeOnly + c.Paths, c.Modules, c.New, c.Dependents, c.Manifests, c.Builds = nil, nil, nil, nil, nil, nil + c.Judge, c.Plan, c.Group, c.Members = "", nil, "", nil + request.Check = &c + return request +} + +// maxTries is how many tries may wait or run at once, mesh-wide: each holds a build machine for minutes. +const maxTries = 4 + +// triesIn reads the build queue for a new try of one commit: the id of a try of that commit already waiting or +// running — the answer to a second ask — and whether there is room for another. +func triesIn(q link.Queue, owner, repo, commit string) (same string, room bool) { + n := 0 + for _, a := range q.Asks { + if !link.IsTry(a.Request) || (a.State != link.AskWaiting && a.State != link.AskInFlight) { + continue + } + n++ + c := a.Request.Check + if same == "" && strings.EqualFold(c.Owner, owner) && strings.EqualFold(c.Repo, repo) && a.Request.Ref == commit { + same = a.ID + } + } + return same, n < maxTries +} + +// forgeCommit asks the forge, through its module's tools on the bus, which commit a branch or commit of a +// repository is, and the repository's default branch. A variable so a test needs no forge. +var forgeCommit = func(ctx context.Context, owner, repo, ref string) (commit, trunk string, err error) { + js, err := aBus() + if err != nil { + return "", "", err + } + defer js.Close() + bus := link.OverNATS{Conn: js.Conn()} + ask := func(path string, into any) error { + raw, _ := json.Marshal(map[string]any{"path": path}) + answer, err := link.Ask(ctx, bus, "gitea", "gitea_api", raw, 30*time.Second) + if err != nil { + return err + } + if answer.Error != "" { + return errors.New(answer.Error) + } + return json.Unmarshal(answer.Result, into) + } + var commits struct { + Result []struct { + SHA string `json:"sha"` + } `json:"result"` + } + if err := ask("/repos/"+owner+"/"+repo+"/commits?limit=1&stat=false&verification=false&files=false&sha="+ + url.QueryEscape(ref), &commits); err != nil { + return "", "", err + } + if len(commits.Result) == 0 || commits.Result[0].SHA == "" { + return "", "", errors.New("the forge named no commit for it") + } + var found struct { + Result struct { + DefaultBranch string `json:"default_branch"` + } `json:"result"` + } + if err := ask("/repos/"+owner+"/"+repo, &found); err != nil { + return "", "", err + } + return commits.Result[0].SHA, found.Result.DefaultBranch, nil +} + +// tryCommand is `try / [--base ] [--tests ] [--store-only]`, and +// `try --id ` to read what came of one. +func tryCommand(ctx context.Context, args []string) error { + set := flag.NewFlagSet("try", flag.ContinueOnError) + base := set.String("base", "", "the branch it is merged into (the repository's default branch when none)") + tests := set.String("tests", "", "only the tests this `go test -run` pattern names") + storeOnly := set.Bool("store-only", false, "only the repository's store tests") + id := set.String("id", "", "a try's id: what came of it") + positionals, err := parseAround(set, args) + if err != nil { + return err + } + usage := errors.New("try / [--base ] [--tests ] " + + "[--store-only], or try --id ") + if *id != "" { + if len(positionals) != 0 { + return usage + } + // Before it is a token of a bus subject: an id, never a pattern. + if !link.SafeBuildID(*id) { + return fmt.Errorf("%q is not a try's id", *id) + } + return tryRead(ctx, *id) + } + if len(positionals) != 2 { + return usage + } + owner, repo, isPath := strings.Cut(positionals[0], "/") + ref := positionals[1] + switch { + case !isPath || !tryRef.MatchString(owner) || !tryRef.MatchString(repo) || strings.Contains(repo, "/"): + return fmt.Errorf("%q is not a repository on the forge, owner/name; nothing was asked", positionals[0]) + case !tryRef.MatchString(ref): + return fmt.Errorf("%q is not a branch or a commit; nothing was asked", ref) + case *base != "" && !tryRef.MatchString(*base): + return fmt.Errorf("%q is not a branch; nothing was asked", *base) + case *tests != "" && !tryTests.MatchString(*tests): + return fmt.Errorf("%q is not a `go test -run` pattern of test names; nothing was asked", *tests) + } + // **An unknown ref is refused here, loudly** — not sent to a machine to fail at its clone minutes later. + commit, trunk, err := forgeCommit(ctx, owner, repo, ref) + if err != nil { + return fmt.Errorf("the forge holds no branch or commit %s in %s/%s — is it pushed? — so nothing was asked: %w", + ref, owner, repo, err) + } + if *base != "" { + trunk = *base + } + if trunk == "" { + return fmt.Errorf("the forge named no default branch for %s/%s, and no --base says what %s is merged into; "+ + "nothing was asked", owner, repo, ref) + } + + open, err := openStores(ctx) + if err != nil { + return err + } + defer open.Close() + entries, read, edges, err := theGraph(ctx, open.inventory) + if err != nil { + return err + } + p := link.PullUpdated{Owner: owner, Repo: repo, Base: trunk, Commit: commit} + // Whose repository it is, as a pull request's check reckons it: only the mesh's is tried, the code the + // build seat runs for every pull request already. + mesh := pullScope(p, entries, read, edges) + if !mesh.Mesh { + return fmt.Errorf("%s/%s is no repository of the mesh — no module is built from it and its owner is not "+ + "the core's — and only the mesh's are tried on the build seat; nothing was asked", owner, repo) + } + request, err := checkRequestFor(ctx, open, p, checkScope{Mesh: true, From: mesh.From}, entries) + if err != nil { + return err + } + request = asATry(request, *tests, *storeOnly) + seat := buildSeatHeld(ctx) + // **Bounded in number** (novox/hq issue 686): one try of a commit at a time, and maxTries in all. + js, err := dialTheBus() + if err != nil { + return err + } + queue, err := link.ReadQueue(ctx, js, seat) + js.Close() + if err != nil { + return err + } + same, room := triesIn(queue, owner, repo, commit) + if same != "" { + return printTry(tryAnswer{ID: same, State: "asked", Repository: owner + "/" + repo, Ref: ref, Commit: commit, + On: seat, Said: fmt.Sprintf("a try of %s/%s at %.12s already waits or runs, as %s: nothing new was asked. "+ + "Read it with try id=%s", owner, repo, commit, same, same)}) + } + if !room { + return fmt.Errorf("%d tries already wait or run on the build seat, the most at once (`queue` lists them); "+ + "nothing was asked — read one of yours, or ask again when one has ended", maxTries) + } + ask, err := askOverOn(seat) + if err != nil { + return err + } + defer ask.Close() + if err := ask.Ask(ctx, request); err != nil { + return err + } + return printTry(tryAnswer{ID: request.ID, State: "asked", Repository: owner + "/" + repo, Ref: ref, Commit: commit, + Base: trunk, On: seat, Said: fmt.Sprintf("asked %s to try %s/%s at %.12s (%s) merged into %s: its own "+ + "merge-check.sh, as a check in the build queue, bounded at %s; no status is set and nothing is registered. "+ + "Read it with try id=%s — a full run of this kind takes minutes", seat, owner, repo, commit, ref, trunk, + checkTimeout, request.ID)}) +} + +// tryRead answers what came of a try, holding the answer up to tryWait for its verdict. +func tryRead(ctx context.Context, id string) error { + until := time.Now().Add(tryWait) + for { + lines, err := buildLines(ctx, id) + if err != nil { + return err + } + a := tryAnswerOf(id, lines) + if (a.State != "waiting" && a.State != "running") || !time.Now().Add(2*time.Second).Before(until) { + return printTry(a) + } + if err := sleepOrDone(ctx, 2*time.Second); err != nil { + return err + } + } +} + +// tryAnswerOf reads a try's answer from what the build seat said of it: its outcome under StepTried; `failed` +// for one that could not run — an error, never a pass; else how far it is. +func tryAnswerOf(id string, lines []link.BuildLine) tryAnswer { + a := tryAnswer{ID: id, State: "waiting", Said: "nothing is said of " + id + " yet: it waits in the build queue " + + "(`queue` says where), or no try has that id; ask again"} + for _, l := range lines { + switch l.Step { + case link.StepTried: + var t link.Tried + if err := json.Unmarshal([]byte(l.Message), &t); err != nil || t.Verdict == "" { + return tryAnswer{ID: id, State: "error", Said: "the build seat's word on it cannot be read: " + l.Message} + } + return tryAnswer{ID: id, State: t.Verdict, Said: t.Summary, Took: t.Took, Commit: t.Head, Trunk: t.Trunk, + Failed: t.Failed} + case "failed": + return tryAnswer{ID: id, State: "error", Said: "it could not run: " + l.Message} + case "checked": + // A check that is no try, or a build seat from before the verb: its verdict is not a try's. + return tryAnswer{ID: id, State: "error", Said: "the build seat checked it and said no try's outcome — " + + "it predates the verb (novox/hq issue 686), or " + id + " is no try: " + l.Message + "; `builds` with " + + "the id says every line"} + } + a.State, a.Last = "running", l.Message + a.Said = "still running; ask again. `builds` with the id says every line so far" + } + return a +} + +func printTry(a tryAnswer) error { + fmt.Fprintln(os.Stderr, a.Said) + for _, l := range a.Failed { + fmt.Fprintln(os.Stderr, " "+l) + } + out, err := json.MarshalIndent(a, "", " ") + if err != nil { + return err + } + fmt.Println(string(out)) + return nil +} + +// buildLines is everything the build seat said of one build, as the bus keeps it, in order. +func buildLines(ctx context.Context, id string) ([]link.BuildLine, error) { + js, err := aBus() + if err != nil { + return nil, fmt.Errorf("cannot reach the bus to read a build's log: %w", err) + } + defer js.Close() + subject := link.BuildLogOf("*", id) + sub, err := js.Context().PullSubscribe(subject, "", + nats.BindStream(broker.EventsStream), nats.DeliverAll(), nats.AckNone()) + if err != nil { + return nil, fmt.Errorf("cannot read %s from the bus: %w", subject, err) + } + defer func() { _ = sub.Unsubscribe() }() + var lines []link.BuildLine + for { + batch, err := sub.Fetch(200, nats.MaxWait(2*time.Second)) + if err != nil && !errors.Is(err, nats.ErrTimeout) && !errors.Is(err, context.DeadlineExceeded) { + return nil, fmt.Errorf("reading a build's log: %w", err) + } + for _, msg := range batch { + var line link.BuildLine + if json.Unmarshal(msg.Data, &line) == nil { + lines = append(lines, line) + } + } + if len(batch) < 200 { + return lines, nil + } + } +} diff --git a/cmd/mesh-controller/try_test.go b/cmd/mesh-controller/try_test.go new file mode 100644 index 00000000..5f72bcda --- /dev/null +++ b/cmd/mesh-controller/try_test.go @@ -0,0 +1,194 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "reflect" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/link" +) + +// A branch is tried before its pull request (novox/hq issue 686): the repository's own check asked for a pushed +// branch or commit, with no pull request. + +// What a try asks is a check — after deliveries in the build queue, bounded as one — of the repository's own +// script alone: no module of the graph, so no gate; nothing built for a merge, so no reusable image. +func TestATryAsksACheckOfTheRepositorysOwnScriptAlone(t *testing.T) { + request := link.BuildRequest{ID: "build-1", Ref: "c0ffee", Check: &link.CheckRequest{Owner: "novox", Repo: "a", + Number: 7, Base: "main", Modules: []string{"a"}, New: []string{"b"}, Manifests: []string{"module.json"}, + Dependents: []string{"c"}, Builds: []link.CheckBuild{{Module: "a"}}, Judge: link.JudgeSelf, + Plan: &link.ChangePlan{}}} + tried := asATry(request, "^TestStore", true) + c := tried.Check + if !c.Try || c.Number != 0 || c.Tests != "^TestStore" || !c.StoreOnly { + t.Fatalf("not marked a try, or it kept a pull request's number: %+v", c) + } + if len(c.Modules)+len(c.New)+len(c.Manifests)+len(c.Dependents)+len(c.Builds) != 0 || c.Judge != "" || c.Plan != nil { + t.Fatalf("a try runs the gate or builds for a merge: %+v", c) + } + if c.Base != "main" || c.Owner != "novox" || tried.Ref != "c0ffee" { + t.Fatalf("a try lost what it merges into what: %+v", tried) + } + if link.KindOf(tried) != link.KindCheck { + t.Fatal("a try does not count as a check in the build queue") + } +} + +// A try's outcome is heard and left there: no `checked` is said, so the forge's holder sets no status, and +// nothing is recorded or registered (the handler holds no store here, and would fail reaching for one). +func TestATrysOutcomeSetsNoStatusAndRegistersNothing(t *testing.T) { + said := &saidOnTheBus{} + checkEvents = said + defer func() { checkEvents = nil }() + for name, result := range map[string]link.BuildResult{ + "a try": {ID: "build-1", Ref: "c0ffee", Checked: &link.CheckRequest{Owner: "novox", Repo: "a", Try: true}, + Check: &link.CheckOutcome{Verdict: "pass", RepoCheck: &link.CheckLayer{Verdict: "fail", Summary: "failed"}}}, + "a try that could not run": {ID: "build-2", Checked: &link.CheckRequest{Owner: "novox", Repo: "a", Try: true}, + Failed: "cannot clone"}, + // A build seat from before the mark echoes the ask without it: a check of no pull request and no + // group is still nobody's status. + "a try echoed by an older build seat": {ID: "build-3", Checked: &link.CheckRequest{Owner: "novox", Repo: "a"}, + Check: &link.CheckOutcome{Verdict: "pass"}}, + } { + if err := (builds{}).Built(context.Background(), result); err != nil { + t.Fatalf("%s: %v", name, err) + } + } + if len(said.events) != 0 { + t.Fatalf("a try's outcome was said for the forge: %v", said.events) + } + // A pull request's check is still said. + _ = (builds{}).Built(context.Background(), link.BuildResult{ID: "build-4", Checked: &link.CheckRequest{ + Owner: "novox", Repo: "a", Number: 7}, Check: &link.CheckOutcome{Verdict: "pass"}}) + if len(said.events) != 1 { + t.Fatalf("a pull request's check was said %d time(s)", len(said.events)) + } +} + +// saidOnTheBus keeps the seat events said, and nothing else. +type saidOnTheBus struct { + link.Bus + events []string +} + +func (s *saidOnTheBus) PublishSeatEvent(_ context.Context, seat, event string, _ []byte) error { + s.events = append(s.events, seat+" "+event) + return nil +} + +// A ref the forge does not hold is refused before anything is asked, naming it. +func TestATryOfAnUnknownRefIsRefusedLoudly(t *testing.T) { + was := forgeCommit + defer func() { forgeCommit = was }() + forgeCommit = func(_ context.Context, owner, repo, ref string) (string, string, error) { + return "", "", errors.New("404 object does not exist [id: " + ref + "]") + } + err := tryCommand(context.Background(), []string{"novox/mesh-controller", "fix/no-such-branch"}) + if err == nil { + t.Fatal("a try of a ref the forge does not hold was asked") + } + for _, want := range []string{"fix/no-such-branch", "novox/mesh-controller", "nothing was asked", "404"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("the refusal does not say %q: %v", want, err) + } + } + for _, args := range [][]string{{"mesh-controller", "main"}, {"novox/mesh-controller"}, {"novox/a", "--upload-pack=x"}, + {"novox/a", "main", "--tests", "x; rm -rf /"}, {"novox/a", "main", "--id", "build-1"}} { + if err := tryCommand(context.Background(), args); err == nil { + t.Errorf("try %v was taken", args) + } + } +} + +// The answer of a try carries the lines that name what failed (novox/hq issue 460's shape) and how long it took. +func TestATrysAnswerCarriesTheFailingLines(t *testing.T) { + tried, _ := json.Marshal(link.Tried{Verdict: "fail", Summary: "its merge-check.sh failed: TestX", Took: "5m3s", + Head: "c0ffee", Trunk: "ca3f43c", Failed: []string{"--- FAIL: TestX (0.01s)", " x_test.go:12: got 1, want 2", + "FAIL\tgithub.com/novox/a/cmd\t3.1s"}}) + lines := []link.BuildLine{{Seq: 1, Step: "check", Message: "novox/a#0 at c0ffee"}, + {Seq: 2, Step: "output", Message: "ok github.com/novox/a/internal 1.2s"}, + {Seq: 3, Step: link.StepTried, Message: string(tried)}, {Seq: 4, Step: "checked", Message: "pass: x"}} + a := tryAnswerOf("build-1", lines) + if a.State != "fail" || a.Took != "5m3s" || len(a.Failed) != 3 || a.Failed[0] != "--- FAIL: TestX (0.01s)" || + a.Trunk != "ca3f43c" || a.ID != "build-1" { + t.Fatalf("the answer lost the verdict, the time or the failing lines: %+v", a) + } + // Not answered yet: said as running, with how far it is — never as a pass. + if a := tryAnswerOf("build-1", lines[:2]); a.State != "running" || a.Last == "" { + t.Fatalf("a try still running: %+v", a) + } + if a := tryAnswerOf("build-1", nil); a.State != "waiting" { + t.Fatalf("a try nothing was said of: %+v", a) + } + // One that could not run is an error with the build seat's word, never a pass. + if a := tryAnswerOf("build-1", append(lines[:1:1], link.BuildLine{Seq: 2, Step: "failed", + Message: "cannot clone"})); a.State != "error" || !strings.Contains(a.Said, "cannot clone") { + t.Fatalf("a try that could not run: %+v", a) + } +} + +// One try of a commit at a time, and four in all (novox/hq issue 686): a second ask is answered the first's id, +// a fifth is refused; a try that ended, and a pull request's check, count for nothing. +func TestTriesAreBoundedInNumber(t *testing.T) { + ask := func(id, repo, ref, state string, try bool) link.QueuedAsk { + return link.QueuedAsk{ID: id, State: state, Request: link.BuildRequest{ID: id, Ref: ref, + Check: &link.CheckRequest{Owner: "novox", Repo: repo, Try: try}}} + } + q := link.Queue{Asks: []link.QueuedAsk{ + ask("t1", "a", "c1", link.AskInFlight, true), ask("t2", "a", "c2", link.AskWaiting, true), + ask("t3", "b", "c1", link.AskDead, true), ask("p1", "a", "c9", link.AskWaiting, false)}} + if same, room := triesIn(q, "novox", "a", "c2"); same != "t2" || !room { + t.Fatalf("a second try of a commit: %q %v", same, room) + } + if same, room := triesIn(q, "novox", "b", "c1"); same != "" || !room { + t.Fatalf("a dead try, or another repository's, was taken for this one: %q %v", same, room) + } + if same, _ := triesIn(q, "novox", "a", "c9"); same != "" { + t.Fatalf("a pull request's check was taken for a try: %q", same) + } + q.Asks = append(q.Asks, ask("t4", "c", "c3", link.AskWaiting, true), ask("t5", "d", "c4", link.AskWaiting, true)) + if _, room := triesIn(q, "novox", "e", "c5"); room { + t.Fatal("a fifth try was let in") + } + // An id is checked before it is a token of a bus subject. + for _, id := range []string{"x.>", "*", "a b"} { + if err := tryCommand(context.Background(), []string{"--id", id}); err == nil || + !strings.Contains(err.Error(), "not a try's id") { + t.Errorf("try --id %q: %v", id, err) + } + } +} + +// The verb runs its command, and is declared with the shell habit it replaces (novox/hq ADR 0245). +func TestTheTryVerbRunsItsCommand(t *testing.T) { + for _, c := range []struct { + args map[string]any + want []string + }{ + {map[string]any{"repository": "novox/a", "ref": "fix/x"}, []string{"try", "novox/a", "fix/x"}}, + {map[string]any{"repository": "novox/a", "ref": "fix/x", "base": "main", "tests": "^TestX$", "store-only": "true"}, + []string{"try", "novox/a", "fix/x", "--base", "main", "--tests", "^TestX$", "--store-only"}}, + {map[string]any{"id": "build-1"}, []string{"try", "--id", "build-1"}}, + } { + got, err := argvFor("try", c.args) + if err != nil || !reflect.DeepEqual(got, c.want) { + t.Errorf("try %v: %v %v, want %v", c.args, got, err, c.want) + } + } + if _, err := argvFor("try", map[string]any{"repository": "novox/a"}); err == nil { + t.Error("a try of no ref was taken") + } + for _, v := range catalogue.ControllerVerbs { + if v.Name == "try" { + if len(v.Replaces) == 0 { + t.Error("try names no shell habit it replaces") + } + return + } + } + t.Error("try is not a verb of the controller seat") +} -- 2.54.0 From 4b2eac702ab18290f910dfd29963c137280e77c6 Mon Sep 17 00:00:00 2001 From: mesh-admin Date: Wed, 14 Oct 2026 06:22:29 +0000 Subject: [PATCH 25/31] Bring main into release 57's branch, conflicts resolved So the release branch carries what main gained, and the store-tests drop check passes again. Co-Authored-By: Claude Opus 5.5 --- cmd/mesh-controller/upgrades.go | 2 +- cmd/mesh-controller/walk_said.go | 6 ++ internal/builder/builder.go | 23 ++++ internal/builder/check.go | 23 ++++ internal/builder/check_test.go | 12 +++ internal/builder/lend.go | 178 +++++++++++++++++++++++++++++++ internal/builder/lend_test.go | 140 ++++++++++++++++++++++++ internal/builder/reuse.go | 7 ++ internal/catalogue/verbs.go | 22 ++++ internal/inventory/plans.go | 72 ++++++++++++- internal/link/build.go | 24 +++++ internal/link/builds.go | 19 ++++ internal/link/builds_nats.go | 5 +- internal/link/queue.go | 3 + internal/link/queue_order.go | 11 +- internal/link/try_test.go | 55 ++++++++++ merge-check.sh | 23 +++- module.json | 1 + testdata/store-tests | 11 ++ 19 files changed, 624 insertions(+), 13 deletions(-) create mode 100644 internal/builder/lend.go create mode 100644 internal/builder/lend_test.go create mode 100644 internal/link/try_test.go diff --git a/cmd/mesh-controller/upgrades.go b/cmd/mesh-controller/upgrades.go index 7f8bc8e2..7570fb79 100644 --- a/cmd/mesh-controller/upgrades.go +++ b/cmd/mesh-controller/upgrades.go @@ -79,7 +79,7 @@ func (f following) Upgraded(ctx context.Context, u link.Upgraded) error { // `rebuild`, a `replay` registered — was sent here one machine after another without any judging; it // now waits for a gate like any other build, and the release plan walks it through the machines, one // at a time, each judged. - fmt.Printf("%s moved to %s outside a plan; a release plan sends it to %s, one machine at a time, each judged "+ + fmt.Printf("%s moved to %s outside a plan; a release plan sends it to %s, one machine first and then the rest at once, each judged "+ "(`upgrade backlog` lists what waits)\n", u.Module, shortCommit(u.Commit), readableList(on)) return nil } diff --git a/cmd/mesh-controller/walk_said.go b/cmd/mesh-controller/walk_said.go index ddb724e4..3e5d9bad 100644 --- a/cmd/mesh-controller/walk_said.go +++ b/cmd/mesh-controller/walk_said.go @@ -177,6 +177,12 @@ func walkReached(p inventory.Plan, running func(string) []string) map[string][]s reached = appendNew(reached, r.Gate.Machines...) carried(r.Gate) } + for _, g := range r.Gates { + if g != nil && !g.Pending { + reached = appendNew(reached, g.Machines...) + carried(g) + } + } for _, tier := range p.Tiers { for _, m := range tier { for _, n := range running(m) { diff --git a/internal/builder/builder.go b/internal/builder/builder.go index c2c1cbf9..728ce83e 100644 --- a/internal/builder/builder.go +++ b/internal/builder/builder.go @@ -1088,6 +1088,16 @@ func one(ctx context.Context, run Runner, publish Publisher, return catalogue.Built{}, fmt.Errorf("%s: building %s failed: %w", module, a.Name, err) } say("image", "built, publishing") + // Its bases linked into its repository first, so the push needs none of their blobs from this + // machine's store, which may not hold them (novox/hq issue 694). Said and not fatal: a push + // from a store that holds them works without, and one that fails says which blob it lacked. + if lender, can := publish.(BaseLender); can { + if n, err := lender.LendBases(ctx, module+"/"+a.Name, buildArgValues(args)); err != nil { + say("image", "its bases could not all be linked into %s/%s before the push: %v", module, a.Name, err) + } else if n > 0 { + say("image", "%d blob(s) of its bases linked into %s/%s before the push", n, module, a.Name) + } + } reference, err := publish.PublishImage(ctx, local, module+"/"+a.Name) if err != nil { return catalogue.Built{}, err @@ -1439,6 +1449,19 @@ func standingOn(ctx context.Context, manifest catalogue.Manifest, held map[strin return args, resolved, nil } +// buildArgValues is what each --build-arg of a build was handed: the bases, as this mesh holds them. +func buildArgValues(args []string) []string { + var values []string + for i := 0; i+1 < len(args); i += 2 { + if args[i] == "--build-arg" { + if _, value, has := strings.Cut(args[i+1], "="); has { + values = append(values, value) + } + } + } + return values +} + // compile runs a module's own code through its toolchain, and says where the result is. // // **In the module's own directory, under the path the toolchain expects.** A module is compiled diff --git a/internal/builder/check.go b/internal/builder/check.go index bf0ce5bc..a2107499 100644 --- a/internal/builder/check.go +++ b/internal/builder/check.go @@ -103,6 +103,23 @@ type CheckSpec struct { // for the walk to take. None for a group's composed check, whose members' own checks build. Builds []CheckBuildSpec BuildWith *CheckBuilding + // Tests and StoreOnly narrow what the repository's own script runs, for a branch tried before its pull + // request (novox/hq issue 686): handed to the script as EnvTests and EnvStoreOnly, which it may read. + Tests string + StoreOnly bool +} + +// selection is what a check hands the repository's script to narrow what it runs (novox/hq issue 686): nothing +// for a pull request's check, which always runs whole. +func selection(spec CheckSpec) []string { + var env []string + if spec.Tests != "" { + env = append(env, EnvTests+"="+spec.Tests) + } + if spec.StoreOnly { + env = append(env, EnvStoreOnly+"=1") + } + return env } // GroupHead is one other head of a delivery group's composed check. @@ -193,6 +210,11 @@ const ( // EnvGroup is a delivery group's other heads, as JSON, for the gate (novox/hq ADR 0239); empty for a // pull request checked alone. EnvGroup = "MESH_CHECK_GROUP" + // EnvTests is a `go test -run` pattern and EnvStoreOnly is 1 when a branch tried before its pull request + // asks for only those tests, or only the store tests (novox/hq issue 686). Never set for a pull request's + // check. A script that reads neither runs whole. + EnvTests = "MESH_CHECK_TESTS" + EnvStoreOnly = "MESH_CHECK_STORE_ONLY" ) // CheckTimeout bounds one check; a check that runs past it is an error, not a pass. @@ -515,6 +537,7 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry EnvModules + "=" + strings.Join(append(append([]string{}, spec.Modules...), spec.New...), ","), EnvGroup + "=" + groupFile, "GOCACHE=" + filepath.Join(workspace, "go-cache"), "GOMODCACHE=" + filepath.Join(workspace, "go-modules")} + env = append(env, selection(spec)...) v := CheckVerdict{} modules := append(append([]string{}, spec.Modules...), prefixed("new:", spec.New)...) diff --git a/internal/builder/check_test.go b/internal/builder/check_test.go index 92583f5a..1eb5890a 100644 --- a/internal/builder/check_test.go +++ b/internal/builder/check_test.go @@ -590,3 +590,15 @@ func TestARepositoryInTwoLanguagesIsCheckedInBoth(t *testing.T) { t.Errorf("a part in a toolchain the mesh does not hold answered %+v — never a pass", layer) } } + +// A branch tried before its pull request may narrow what the repository's script runs (novox/hq issue 686); a +// pull request's check is handed nothing, and runs whole. +func TestOnlyATrysSelectionReachesTheScript(t *testing.T) { + if env := selection(CheckSpec{Number: 7}); len(env) != 0 { + t.Fatalf("a pull request's check was handed a selection: %v", env) + } + env := selection(CheckSpec{Tests: "^TestX$", StoreOnly: true}) + if len(env) != 2 || env[0] != "MESH_CHECK_TESTS=^TestX$" || env[1] != "MESH_CHECK_STORE_ONLY=1" { + t.Fatalf("a try's selection: %v", env) + } +} diff --git a/internal/builder/lend.go b/internal/builder/lend.go new file mode 100644 index 00000000..d3984647 --- /dev/null +++ b/internal/builder/lend.go @@ -0,0 +1,178 @@ +package builder + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "runtime" + "strings" +) + +// **A built image's bases are linked into its repository before it is pushed** (novox/hq issue 694). +// +// A push reads every blob of the image out of this machine's store, unless the repository it goes to +// already holds that blob: the runtime asks the registry first, blob by blob, and reads locally only +// what the registry lacks. A runtime keeping its images in containerd's store — the default for a +// fresh install from docker 29 on — does not always hold the compressed layers of the base an image +// was built FROM: a layer whose unpacked form it already had is never fetched. The push of such an +// image then fails on a layer this machine never had ("not all of them are available locally"), while +// the very bytes sit in this registry, under the repository the base was mirrored into. +// +// So the base's blobs are mounted into the image's repository first. A mount moves no bytes, and once +// the repository holds a blob the push asks this machine for nothing of it — whichever store the +// runtime keeps. A runtime that holds every layer (the classic store) is not changed by this: its push +// finds the same blobs already there. + +// BaseLender links what a build's bases hold into the repository its image is about to be pushed to. +type BaseLender interface { + // LendBases mounts every blob of each base this registry holds — for the platform this machine + // builds — into `repository`, and answers how many it mounted. + LendBases(ctx context.Context, repository string, bases []string) (int, error) +} + +// platformed is an index's entry: a manifest and the platform it is for. +type platformed struct { + descriptor + Platform *struct { + OS string `json:"os"` + Architecture string `json:"architecture"` + } `json:"platform"` +} + +// LendBases mounts each base's blobs into the repository. A base this registry does not hold under its +// repository — named by tag, or held somewhere else — lends nothing and is no error: the push reads it +// from this machine's store as before. +func (r Registry) LendBases(ctx context.Context, repository string, bases []string) (int, error) { + mounted := 0 + for _, base := range bases { + name, digest, pinned := strings.Cut(base, "@") + if !pinned || !strings.HasPrefix(digest, "sha256:") { + continue + } + where, err := parseReference(name + "@" + digest) + if err != nil { + continue + } + from := where.repository + if from == repository { + continue + } + blobs, held, err := r.blobsOf(ctx, from, digest) + if err != nil { + return mounted, fmt.Errorf("reading %s: %w", base, err) + } + if !held { + continue + } + for _, blob := range blobs { + did, err := r.mount(ctx, repository, from, blob) + if err != nil { + return mounted, fmt.Errorf("linking %s of %s into %s: %w", blob, base, repository, err) + } + if did { + mounted++ + } + } + } + return mounted, nil +} + +// blobsOf is the config and layers of the manifest under repository@digest — for an index, of each +// manifest it names for this machine's platform. False when the registry holds no such manifest there. +func (r Registry) blobsOf(ctx context.Context, repository, digest string) ([]string, bool, error) { + body, held, err := r.manifest(ctx, repository, digest) + if err != nil || !held { + return nil, held, err + } + var document struct { + Manifests []platformed `json:"manifests"` + Config *descriptor `json:"config"` + Layers []descriptor `json:"layers"` + } + if err := json.Unmarshal(body, &document); err != nil { + return nil, false, fmt.Errorf("%s@%s is not a manifest: %w", repository, digest, err) + } + var blobs []string + if document.Config != nil { + blobs = append(blobs, document.Config.Digest) + } + for _, l := range document.Layers { + blobs = append(blobs, l.Digest) + } + for _, m := range document.Manifests { + if m.Platform == nil || m.Platform.OS != runtime.GOOS || m.Platform.Architecture != runtime.GOARCH { + continue + } + more, held, err := r.blobsOf(ctx, repository, m.Digest) + if err != nil { + return nil, false, err + } + if held { + blobs = append(blobs, more...) + } + } + return blobs, true, nil +} + +// manifest reads one manifest document from this registry; false when it is not there. +func (r Registry) manifest(ctx context.Context, repository, digest string) ([]byte, bool, error) { + request, err := http.NewRequestWithContext(ctx, http.MethodGet, + "http://"+r.Address+"/v2/"+repository+"/manifests/"+digest, nil) + if err != nil { + return nil, false, err + } + request.Header.Set("Accept", manifestAccept) + response, err := r.client().Do(request) + if err != nil { + return nil, false, fmt.Errorf("cannot reach the registry at %s: %w", r.Address, err) + } + defer response.Body.Close() + if response.StatusCode == http.StatusNotFound { + return nil, false, nil + } + if response.StatusCode != http.StatusOK { + return nil, false, fmt.Errorf("%s/%s@%s: %s", r.Address, repository, digest, response.Status) + } + body, err := io.ReadAll(io.LimitReader(response.Body, 4<<20)) + return body, err == nil, err +} + +// mount links one blob from one repository into another; false when the repository already held it. +func (r Registry) mount(ctx context.Context, repository, from, digest string) (bool, error) { + base := "http://" + r.Address + "/v2/" + repository + if there, err := r.has(ctx, base+"/blobs/"+digest); err != nil { + return false, err + } else if there { + return false, nil + } + start, err := http.NewRequestWithContext(ctx, http.MethodPost, + base+"/blobs/uploads/?mount="+digest+"&from="+from, nil) + if err != nil { + return false, err + } + begun, err := r.client().Do(start) + if err != nil { + return false, fmt.Errorf("cannot reach the registry at %s: %w", r.Address, err) + } + begun.Body.Close() + if begun.StatusCode == http.StatusCreated { + return true, nil + } + if begun.StatusCode == http.StatusAccepted { + // Not mounted, and an upload begun in its place: let go of it, so nothing is left half-open. + if location := begun.Header.Get("Location"); location != "" { + if strings.HasPrefix(location, "/") { + location = "http://" + r.Address + location + } + if cancel, err := http.NewRequestWithContext(ctx, http.MethodDelete, location, nil); err == nil { + if done, err := r.client().Do(cancel); err == nil { + done.Body.Close() + } + } + } + return false, fmt.Errorf("%s did not mount it from %s", r.Address, from) + } + return false, fmt.Errorf("%s answered %s when asked to mount it from %s", r.Address, begun.Status, from) +} diff --git a/internal/builder/lend_test.go b/internal/builder/lend_test.go new file mode 100644 index 00000000..ff92a998 --- /dev/null +++ b/internal/builder/lend_test.go @@ -0,0 +1,140 @@ +package builder + +import ( + "context" + "encoding/json" + "net/http/httptest" + "runtime" + "strings" + "testing" +) + +// A built image's bases are linked into its repository before it is pushed (novox/hq issue 694): a +// runtime keeping containerd's image store may not hold a base layer it never had to fetch, and its push +// asks this machine only for what the repository lacks. + +// aBaseForThisMachine puts an index into the registry under `repository`: one manifest for this +// machine's platform, one for another, and answers the index's digest and each manifest's blobs. +func aBaseForThisMachine(t *testing.T, dst *aRegistryOfRepositories, repository string) (string, []string, []string) { + t.Helper() + put := func(arch string) (string, []string) { + config := []byte(`{"architecture":"` + arch + `"}`) + layer := []byte(arch + " layer bytes") + for _, b := range [][]byte{config, layer} { + dst.blobs[digestOf(b)] = b + dst.link(repository, digestOf(b)) + } + m, _ := json.Marshal(map[string]any{ + "schemaVersion": 2, "mediaType": mediaManifestOCI, + "config": map[string]any{"mediaType": "application/vnd.oci.image.config.v1+json", "digest": digestOf(config), "size": len(config)}, + "layers": []map[string]any{{"mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": digestOf(layer), "size": len(layer)}}, + }) + dst.manifests[repository+"@"+digestOf(m)] = m + return digestOf(m), []string{digestOf(config), digestOf(layer)} + } + other := "s390x" + if runtime.GOARCH == other { + other = "amd64" + } + mine, mineBlobs := put(runtime.GOARCH) + theirs, theirBlobs := put(other) + index, _ := json.Marshal(map[string]any{ + "schemaVersion": 2, "mediaType": mediaIndexOCI, + "manifests": []map[string]any{ + {"mediaType": mediaManifestOCI, "digest": mine, "size": 1, "platform": map[string]string{"os": runtime.GOOS, "architecture": runtime.GOARCH}}, + {"mediaType": mediaManifestOCI, "digest": theirs, "size": 1, "platform": map[string]string{"os": "linux", "architecture": other}}, + }, + }) + dst.manifests[repository+"@"+digestOf(index)] = index + return digestOf(index), mineBlobs, theirBlobs +} + +func TestABuildsBaseIsLinkedIntoItsRepositoryBeforeThePush(t *testing.T) { + dst := newRegistryOfRepositories() + server := httptest.NewServer(dst.handler()) + defer server.Close() + address := strings.TrimPrefix(server.URL, "http://") + r := Registry{Address: address} + + const from = "upstream/docker.io/library/golang" + index, mine, theirs := aBaseForThisMachine(t, dst, from) + bases := []string{ + address + "/" + from + "@" + index, + // Not bases this registry holds: a package version, a tag, an image held nowhere here. + "@novox/mesh-sdk@1.2.3", + "golang:1-alpine", + "elsewhere.invalid/thing@sha256:" + strings.Repeat("e", 64), + } + n, err := r.LendBases(context.Background(), "mesh-tools-go/build", bases) + if err != nil { + t.Fatal(err) + } + // Every blob of this machine's platform is in the image's repository, so a push asks this machine's + // store for none of them; another platform's are not, since nothing built here stands on them. + for _, blob := range mine { + if !dst.links["mesh-tools-go/build"][blob] { + t.Fatalf("the base's blob %s is not in the image's repository", blob) + } + } + for _, blob := range theirs { + if dst.links["mesh-tools-go/build"][blob] { + t.Fatalf("another platform's blob %s was linked", blob) + } + } + if n != len(mine) || dst.mounts != len(mine) || dst.uploads != 0 { + t.Fatalf("%d linked, %d mounts, %d uploads; want %d mounts and no bytes moved", n, dst.mounts, dst.uploads, len(mine)) + } + + // Asked again, nothing is mounted twice. + if n, err := r.LendBases(context.Background(), "mesh-tools-go/build", bases); err != nil || n != 0 { + t.Fatalf("a second lending mounted %d: %v", n, err) + } +} + +// A registry that will not mount is said, not hidden: the push is still tried, from the store. +func TestABaseTheRegistryWillNotMountIsAnError(t *testing.T) { + dst := newRegistryOfRepositories() + dst.noMount = true + server := httptest.NewServer(dst.handler()) + defer server.Close() + address := strings.TrimPrefix(server.URL, "http://") + index, _, _ := aBaseForThisMachine(t, dst, "upstream/thing") + _, err := Registry{Address: address}.LendBases(context.Background(), "hello/server", + []string{address + "/upstream/thing@" + index}) + if err == nil || !strings.Contains(err.Error(), "did not mount") { + t.Fatalf("a refused mount passed silently: %v", err) + } +} + +// lending records what a build lent, and when, against the push. +type lending struct { + *mirroring + order []string +} + +func (l *lending) LendBases(_ context.Context, repository string, bases []string) (int, error) { + l.order = append(l.order, "lend "+repository+" "+strings.Join(bases, " ")) + return len(bases), nil +} + +func (l *lending) PublishImage(ctx context.Context, localTag, repository string) (string, error) { + l.order = append(l.order, "push "+repository) + return l.mirroring.PublishImage(ctx, localTag, repository) +} + +func TestAnImageIsPushedOnlyAfterItsBasesAreLent(t *testing.T) { + r, workspace := aRepository(t, anImage, map[string]string{"modules/bus/Dockerfile": "ARG BASE\nFROM ${BASE}"}) + r.contents["modules/bus/module.json"] = anImage + l := &lending{mirroring: &mirroring{recorded: r, at: "127.0.0.1:5000"}} + if _, err := Build(context.Background(), r.run, l, "https://forge.invalid/catalogue.git", "modules/bus", "", workspace, + nil, Npmrc{}, GitCredential{}, nil); err != nil { + t.Fatal(err) + } + want := []string{ + "lend bus/server 127.0.0.1:5000/upstream/docker.io/vendor/server@sha256:" + strings.Repeat("1", 64), + "push bus/server", + } + if strings.Join(l.order, "\n") != strings.Join(want, "\n") { + t.Fatalf("the build did:\n%s\nwant:\n%s", strings.Join(l.order, "\n"), strings.Join(want, "\n")) + } +} diff --git a/internal/builder/reuse.go b/internal/builder/reuse.go index 04a09ce0..2def775d 100644 --- a/internal/builder/reuse.go +++ b/internal/builder/reuse.go @@ -257,6 +257,13 @@ func (s scratchPublisher) PublishArchive(ctx context.Context, repository string, return s.to.PublishArchive(ctx, ScratchPrefix+repository, body, digest) } +func (s scratchPublisher) LendBases(ctx context.Context, repository string, bases []string) (int, error) { + if l, can := s.to.(BaseLender); can { + return l.LendBases(ctx, ScratchPrefix+repository, bases) + } + return 0, nil +} + func (s scratchPublisher) MirrorImage(ctx context.Context, from, repository string) (string, error) { m, can := s.to.(Mirrorer) if !can { diff --git a/internal/catalogue/verbs.go b/internal/catalogue/verbs.go index 673f62f4..a186e558 100644 --- a/internal/catalogue/verbs.go +++ b/internal/catalogue/verbs.go @@ -362,6 +362,28 @@ var ControllerVerbs = []Verb{ "register": "\"true\" to register what it builds", "older": "\"true\", with register: even though a newer build of the module is registered", }, []string{"id"}, "register", "older")}, + // novox/hq issue 686: a builder ran a branch's store tests for the first time in its pull request's merge check. + {Name: "try", Description: "Try a pushed branch or commit before its pull request (novox/hq issue 686): the build " + + "seat merges it into the repository's trunk and runs the repository's own merge-check.sh on the merge, in the " + + "toolchain containers and against the throwaway store and bus a pull request's merge check gets. Answers at " + + "once with an id; asked again with that id it waits up to 20 s and answers pass, fail or error, how long it " + + "took and the lines naming what failed — or that it is still waiting or running, to ask again. Sets no " + + "status on the forge, registers and builds nothing, runs no merge gate, counts as a check in the build queue " + + "(after deliveries and after every pull request's check, however long it waits) and is bounded as one. One " + + "try of a commit at a time — a second ask answers the first's id — and at most four waiting or running in " + + "all: a fifth is refused. Only for a repository of the mesh; the ref must be on the forge.", + Input: schema(map[string]string{ + "repository": "the repository on the forge, owner/name; with ref", + "ref": "the branch or commit to try, as pushed to the forge", + "base": "the branch it is merged into (the repository's default branch when none is named). The " + + "merge-check.sh that runs is that branch's, never the ref's own: the same trust as a pull request's base", + "tests": "only the tests this `go test -run` pattern names, where the repository's merge-check.sh reads " + + "MESH_CHECK_TESTS; a script that does not runs whole", + "store-only": "\"true\": only the repository's store tests, where its merge-check.sh reads " + + "MESH_CHECK_STORE_ONLY", + "id": "instead of a repository: the id a try answered, to read what came of it", + }, nil, "store-only"), + Replaces: []string{"sh merge-check.sh", "docker run postgres"}}, // novox/hq ADR 0350 §5: asked by the delivery's owner when the forge says a pull request closed unmerged. {Name: "forget-checks", Description: "Let go of every build a pull request's merge checks made for its merge in " + "the artifact store's scratch namespace (novox/hq ADR 0350 §5): asked when it closed unmerged; a merged one's go " + diff --git a/internal/inventory/plans.go b/internal/inventory/plans.go index fd86c2b9..c9f712cf 100644 --- a/internal/inventory/plans.go +++ b/internal/inventory/plans.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "slices" "strings" "time" @@ -391,6 +392,65 @@ type PlanGate struct { // reports its module's container left running because it said busy, the judging waits and has not // started; once it reports the container replaced, Since is moved to then. Nil when nothing waited. Busy *GateBusy `json:"busy,omitempty"` + // Pending says the send this gate judges is kept and not yet made (novox/hq ADR 0353 §8): a controller that + // takes over reads the machine's last send before making it, and never sends it twice. + Pending bool `json:"pending,omitempty"` + // Silent says the machine reported nothing on its send within ApplySilentAfter (novox/hq ADR 0353 §4): left, + // neither failed nor put back. + Silent bool `json:"silent,omitempty"` + // TookOver names the gate whose readings this one took over (novox/hq ADR 0353 §7): another walk's judging of + // the same build on the same machine, which this gate goes on from instead of starting from zero. + TookOver string `json:"took_over,omitempty"` +} + +// TakeReadings makes this gate go on from another's readings of the same build on the same machine (novox/hq ADR +// 0353 §7): when the judging began, when each machine applied, the passes counted and when, and what each found. +// A module's own count is taken only for the modules this gate judges. +func (g *PlanGate) TakeReadings(from *PlanGate, modules []string, id string) { + if from == nil { + return + } + if from.Since != nil { + since := *from.Since + g.Since = &since + } + for n, s := range from.Sent { + if slices.Contains(g.Machines, n) { + if g.Sent == nil { + g.Sent = map[string]SentDeclaration{} + } + g.Sent[n] = s + } + } + for n, at := range from.Applied { + if slices.Contains(g.Machines, n) { + if g.Applied == nil { + g.Applied = map[string]time.Time{} + } + g.Applied[n] = at + } + } + g.Passes, g.Faulted, g.HealthySeen, g.Last = from.Passes, from.Faulted, from.HealthySeen, from.Last + if from.LastPass != nil { + at := *from.LastPass + g.LastPass = &at + } + for _, m := range modules { + if n, counted := from.Healthy[m]; counted { + if g.Healthy == nil { + g.Healthy = map[string]int{} + } + g.Healthy[m] = n + } + if at, counted := from.HealthyAt[m]; counted { + if g.HealthyAt == nil { + g.HealthyAt = map[string]time.Time{} + } + g.HealthyAt[m] = at + } + } + g.Readings = append([]GateReading(nil), from.Readings...) + g.TookOver = id } // GateBusy is a first-node gate's wait for a busy container (novox/hq ADR 0335 §4). @@ -477,9 +537,17 @@ type CarriedMove struct { type PlanRelease struct { Order []string `json:"order"` Next int `json:"next"` - // Gate is the machine being judged; nil between machines. + // Gate is the first machine being judged; nil between machines. Gate *PlanGate `json:"gate,omitempty"` - Done []string `json:"done,omitempty"` + // Rest says the first machine passed (novox/hq ADR 0353 §1): every other machine is sent at once, each judged by + // its own gate in Gates, by machine — a build no gate passed going to one machine of a step only (§2). + Rest bool `json:"rest,omitempty"` + Gates map[string]*PlanGate `json:"gates,omitempty"` + // Failed are the machines of the rest whose gate failed, and Silent those that said nothing on their send + // within ApplySilentAfter (ADR 0353 §3 and §4). + Failed []string `json:"failed,omitempty"` + Silent []string `json:"silent,omitempty"` + Done []string `json:"done,omitempty"` // Skipped are the machines not heard from when their turn came, left as they were. Skipped []string `json:"skipped,omitempty"` // By is the person who released it, empty when the mesh did. diff --git a/internal/link/build.go b/internal/link/build.go index 6eb16588..73384e9f 100644 --- a/internal/link/build.go +++ b/internal/link/build.go @@ -164,6 +164,30 @@ type CheckRequest struct { // each, from the merge it judged, into the scratch namespace once it has judged and not failed, for the walk to // take. A build seat from before it builds nothing, and the walk builds as before. Builds []CheckBuild `json:"builds,omitempty"` + // Try marks a branch tried before its pull request (novox/hq issue 686): the repository's own check for a + // pushed branch or commit, asked by a builder through the controller's `try` verb. Its outcome is read from + // the build's log (StepTried) and said to nobody: no status on the forge, nothing recorded or registered, + // nothing built for a merge. Tests narrows what the repository's script runs to the tests a pattern names, + // and StoreOnly to its store tests; a script that reads neither runs whole. + Try bool `json:"try,omitempty"` + Tests string `json:"tests,omitempty"` + StoreOnly bool `json:"store-only,omitempty"` +} + +// StepTried is the step a build seat says a try's outcome under, in the build's log: one line, a Tried as JSON +// (novox/hq issue 686). +const StepTried = "tried" + +// Tried is what came of a branch tried before its pull request (novox/hq issue 686): the verdict of the +// repository's own check for the ref merged into the trunk, how long it took, and the lines that name what +// failed, picked from the whole of its output (issue 460). +type Tried struct { + Verdict string `json:"verdict"` + Summary string `json:"summary"` + Took string `json:"took"` + Head string `json:"head,omitempty"` + Trunk string `json:"trunk,omitempty"` + Failed []string `json:"failed,omitempty"` } // CheckBuild is one module a check builds for its merge: from the merge it made when Merged — a module built from diff --git a/internal/link/builds.go b/internal/link/builds.go index 7fe20541..f372d056 100644 --- a/internal/link/builds.go +++ b/internal/link/builds.go @@ -96,8 +96,27 @@ type BuildStart struct { // an earlier taker left unfinished — most often a build agent replaced mid-build — and started // again from the beginning (novox/hq issue 594). 0 from a machine that predates it. Delivery int `json:"delivery,omitempty"` + // Try marks the start of a branch tried before its pull request (novox/hq issue 686), which names no + // repository and no ref (startOf). + Try bool `json:"try,omitempty"` } +// startOf is what a build machine says of the ask it took. **A try's start names no repository and no commit** +// (novox/hq issue 686): the forge's holder writes mesh/merge-gate pending on an open pull request whose head is +// the commit a start names, and a try of that head is no check of the pull request — nothing on the forge moves +// for a try. Its id, machine and time are all the queue reads of a start. +func startOf(r BuildRequest, on string, delivery int, at time.Time) BuildStart { + s := BuildStart{ID: r.ID, Repository: r.Repository, Path: r.Path, Ref: r.Ref, On: on, + At: at.UTC().Format(time.RFC3339Nano), Delivery: delivery} + if IsTry(r) { + s.Repository, s.Path, s.Ref, s.Try = "", "", "", true + } + return s +} + +// IsTry says an ask is a branch tried before its pull request (novox/hq issue 686). +func IsTry(r BuildRequest) bool { return r.Check != nil && r.Check.Try } + // Restarted is how a build handed out again says so (novox/hq issue 594), in its log and in its check's // report, so a second run never reads as the first. Neutral on why: a holder replaced mid-build, a paused // holder handing an ask back as it arrived, and a bus that held its acknowledgements past the wait all diff --git a/internal/link/builds_nats.go b/internal/link/builds_nats.go index c7d0c67e..5ec3b9c1 100644 --- a/internal/link/builds_nats.go +++ b/internal/link/builds_nats.go @@ -401,10 +401,7 @@ func (b *natsBuild) Done() error { return b.msg.Ack() } // Began publishes that this machine has taken the build, into the stream like the outcome, so a // reader that asks afterwards sees when it started as well as how it ended. func (b *natsBuild) Began(ctx context.Context) error { - body, err := json.Marshal(BuildStart{ - ID: b.request.ID, Repository: b.request.Repository, Path: b.request.Path, - Ref: b.request.Ref, On: b.on, At: time.Now().UTC().Format(time.RFC3339Nano), Delivery: b.delivery, - }) + body, err := json.Marshal(startOf(b.request, b.on, b.delivery, time.Now())) if err != nil { return err } diff --git a/internal/link/queue.go b/internal/link/queue.go index 50865ef8..3bbbb8a2 100644 --- a/internal/link/queue.go +++ b/internal/link/queue.go @@ -427,6 +427,9 @@ func ReadCheckBuilds(ctx context.Context, js *broker.JetStream, seat string, pul // safeKey is an id that can be a key, and a subject token, as it is: a build id, never a pattern. var safeKey = regexp.MustCompile(`^[A-Za-z0-9_-]+$`) +// SafeBuildID says an id can be a subject token as it is: a build's id, never a pattern. +func SafeBuildID(id string) bool { return safeKey.MatchString(id) } + // Cancelled is what the controller writes for an ask it cancelled. type Cancelled struct { At string `json:"at"` diff --git a/internal/link/queue_order.go b/internal/link/queue_order.go index 449618f9..90e68f50 100644 --- a/internal/link/queue_order.go +++ b/internal/link/queue_order.go @@ -26,6 +26,8 @@ import ( // - a check that has waited CheckWaitsAtMost goes before every delivery asked after it, so no check // waits for ever behind a stream of deliveries; // - within a kind, first come, first served, by when it was asked; +// - a branch tried before its pull request goes after everything else, however long it has waited: it is +// never promoted ahead of a delivery, and never ahead of a pull request's check (novox/hq issue 686); // - an ask whose kind cannot be told counts as a check: a delivery taken late breaks a budget, a check // taken late breaks nothing. // @@ -64,8 +66,12 @@ func (a QueuedAsk) askedOrStored() time.Time { // QueueOrder is the order the waiting asks are to be handed out in, now: the deliveries and every check // that has waited CheckWaitsAtMost, by when each was asked, then the other checks, by when each was asked. func QueueOrder(waiting []QueuedAsk, now time.Time) []QueuedAsk { - var first, then []QueuedAsk + var first, then, tries []QueuedAsk for _, a := range waiting { + if IsTry(a.Request) { + tries = append(tries, a) + continue + } if KindOf(a.Request) == KindCheck && now.Sub(a.askedOrStored()) < CheckWaitsAtMost { then = append(then, a) continue @@ -82,7 +88,8 @@ func QueueOrder(waiting []QueuedAsk, now time.Time) []QueuedAsk { } byAsking(first) byAsking(then) - return append(first, then...) + byAsking(tries) + return append(append(first, then...), tries...) } // toMove is the asks to move to the tail, in this order, for the queue to stand as order says: everything diff --git a/internal/link/try_test.go b/internal/link/try_test.go new file mode 100644 index 00000000..f434dc18 --- /dev/null +++ b/internal/link/try_test.go @@ -0,0 +1,55 @@ +package link + +import ( + "encoding/json" + "reflect" + "strings" + "testing" + "time" +) + +// The start of a try names no repository and no commit (novox/hq issue 686): the forge's holder writes +// mesh/merge-gate pending on the open pull request whose head a start names, and nothing on the forge moves +// for a try. A pull request's check still names both. +func TestATrysStartNamesNoCommitTheForgeCouldMatch(t *testing.T) { + sha := strings.Repeat("c0ffee", 7)[:40] + at := time.Date(2026, 10, 14, 0, 0, 0, 0, time.UTC) + try := BuildRequest{ID: "build-1", Repository: "https://forge/novox/a.git", Ref: sha, + Check: &CheckRequest{Owner: "novox", Repo: "a", Try: true}} + body, _ := json.Marshal(startOf(try, "ace", 1, at)) + if strings.Contains(string(body), sha) || strings.Contains(string(body), "novox/a") { + t.Fatalf("a try's start names what the forge matches a pull request by: %s", body) + } + if s := startOf(try, "ace", 2, at); s.ID != "build-1" || s.On != "ace" || !s.Try || s.Delivery != 2 || s.At == "" { + t.Fatalf("a try's start lost what the queue reads: %+v", s) + } + pull := try + pull.Check = &CheckRequest{Owner: "novox", Repo: "a", Number: 7} + if s := startOf(pull, "ace", 1, at); s.Ref != sha || s.Repository == "" || s.Try { + t.Fatalf("a pull request's check's start: %+v", s) + } +} + +// A try stays behind every delivery and every pull request's check, however long it has waited: the ten +// minutes that put a check ahead of later deliveries never promote a try (novox/hq issue 686). +func TestATryIsNeverHandedOutBeforeADeliveryOrAPullRequestsCheck(t *testing.T) { + now := time.Date(2026, 10, 12, 12, 0, 0, 0, time.UTC) + try := func(seq uint64, id string, asked time.Time) QueuedAsk { + a := anAsk(seq, id, asked, true) + a.Request.Check.Try = true + return a + } + waiting := []QueuedAsk{ + try(1, "try-old", now.Add(-40*time.Minute)), + try(2, "try-young", now.Add(-time.Minute)), + anAsk(3, "check-young", now.Add(-2*time.Minute), true), + anAsk(4, "delivery", now.Add(-time.Minute), false), + } + want := []string{"delivery", "check-young", "try-old", "try-young"} + if got := idsOf(QueueOrder(waiting, now)); !reflect.DeepEqual(got, want) { + t.Fatalf("handed out as %v, want %v", got, want) + } + if !SafeBuildID("build-20261014T000000-1a2b") || SafeBuildID("x.>") || SafeBuildID("*") || SafeBuildID("") { + t.Fatal("an id that is a pattern passes for a build's id") + } +} diff --git a/merge-check.sh b/merge-check.sh index e1e85687..36de3711 100755 --- a/merge-check.sh +++ b/merge-check.sh @@ -54,11 +54,26 @@ if [ -n "$unformatted" ]; then fi CGO_ENABLED=0 go vet ./... -if command -v gcc >/dev/null 2>&1; then - CGO_ENABLED=1 go test -race -count=1 -timeout 30m ./... +suite() { + if command -v gcc >/dev/null 2>&1; then + CGO_ENABLED=1 go test -race -count=1 -timeout 30m "$@" + else + echo "NOT RACE-CHECKED: the toolchain holds no C compiler; the suite runs without the race detector" + CGO_ENABLED=0 go test -count=1 -timeout 30m "$@" + fi +} +# **A branch tried before its pull request may ask for a part** (novox/hq issue 686): the controller's `try` verb +# hands MESH_CHECK_TESTS, a `go test -run` pattern, or MESH_CHECK_STORE_ONLY=1. A pull request's merge check is +# handed neither and always runs whole; a part that passes is said as a part, never as the check. +if [ -n "${MESH_CHECK_TESTS:-}" ]; then + echo "TRIED IN PART: only the tests matching $MESH_CHECK_TESTS, in every package" + suite -run "$MESH_CHECK_TESTS" ./... + exit 0 +fi +if [ "${MESH_CHECK_STORE_ONLY:-}" = 1 ]; then + echo "TRIED IN PART: only the store tests listed in testdata/store-tests" else - echo "NOT RACE-CHECKED: the toolchain holds no C compiler; the suite runs without the race detector" - CGO_ENABLED=0 go test -count=1 -timeout 30m ./... + suite ./... fi # **The store tests listed ran** (novox/hq issue 459): a test that needs the store skips without one, and a skip diff --git a/module.json b/module.json index 1b84a8c4..538f5cca 100644 --- a/module.json +++ b/module.json @@ -77,6 +77,7 @@ "replay", "kill", "forget-checks", + "try", "pause", "resume", "hand-act", diff --git a/testdata/store-tests b/testdata/store-tests index d3af9659..62ca1239 100644 --- a/testdata/store-tests +++ b/testdata/store-tests @@ -64,3 +64,14 @@ ./internal/inventory TestPlacementMovesOutOfTheSettingsLayers ./internal/inventory TestASettingRoutesPlacementBesideTheLayer ./cmd/mesh-controller TestADeclarationIsTheSameAfterPlacementMoves +./cmd/mesh-controller TestAReleaseSendsTheRestAtOnceAfterTheFirstPasses +./cmd/mesh-controller TestARestMachineThatFailsIsPutBackAloneAndTheOthersAreJudgedOn +./cmd/mesh-controller TestAWalkTakesOverTheReadingsOfAGateJudgingTheSameBuildThere +./cmd/mesh-controller TestAFirstSendMadeAndNotKeptIsNotSentAgain +./cmd/mesh-controller TestAReleaseSendKeptAndMadeIsNotMadeAgain +./cmd/mesh-controller TestAFirstMachineThatFailsStillStopsTheWalk +./cmd/mesh-controller TestTheControlNodeIsSentLastInAStep +./cmd/mesh-controller TestARestMachineSilentPastItsBoundIsLeftAndSaid +./cmd/mesh-controller TestANoMoveTierWaitsForNoNewReport +./cmd/mesh-controller TestOneProviderDownAndThreeConsumersFailingAreOneCondition +./cmd/mesh-controller TestAConsumerFailingBesideAHealthyProviderIsItsOwn -- 2.54.0 From e75c376a7dd9e74e731b907c28c820fedf218927 Mon Sep 17 00:00:00 2001 From: mesh-admin Date: Wed, 14 Oct 2026 06:41:52 +0000 Subject: [PATCH 26/31] Take the integration branch's store-test list before merging it The two lists added lines at the same place; taking the integration branch's here lets it merge cleanly, and the K2 store tests are listed again right after. Co-Authored-By: Claude Opus 5.5 --- testdata/store-tests | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/testdata/store-tests b/testdata/store-tests index 344072f4..62ca1239 100644 --- a/testdata/store-tests +++ b/testdata/store-tests @@ -64,6 +64,14 @@ ./internal/inventory TestPlacementMovesOutOfTheSettingsLayers ./internal/inventory TestASettingRoutesPlacementBesideTheLayer ./cmd/mesh-controller TestADeclarationIsTheSameAfterPlacementMoves -./internal/inventory TestANewerBuildsListReplacesTheKeptOneAndAnOlderOneHeardLaterDoesNot -./cmd/mesh-controller TestTheMergeGateJudgesEachManifestAgainstTheListOfTheTree -./cmd/mesh-controller TestRegistrationPublishesTheNewestTrunkListAndJudgesEveryModuleAgainstIt +./cmd/mesh-controller TestAReleaseSendsTheRestAtOnceAfterTheFirstPasses +./cmd/mesh-controller TestARestMachineThatFailsIsPutBackAloneAndTheOthersAreJudgedOn +./cmd/mesh-controller TestAWalkTakesOverTheReadingsOfAGateJudgingTheSameBuildThere +./cmd/mesh-controller TestAFirstSendMadeAndNotKeptIsNotSentAgain +./cmd/mesh-controller TestAReleaseSendKeptAndMadeIsNotMadeAgain +./cmd/mesh-controller TestAFirstMachineThatFailsStillStopsTheWalk +./cmd/mesh-controller TestTheControlNodeIsSentLastInAStep +./cmd/mesh-controller TestARestMachineSilentPastItsBoundIsLeftAndSaid +./cmd/mesh-controller TestANoMoveTierWaitsForNoNewReport +./cmd/mesh-controller TestOneProviderDownAndThreeConsumersFailingAreOneCondition +./cmd/mesh-controller TestAConsumerFailingBesideAHealthyProviderIsItsOwn -- 2.54.0 From 197e60c97bccac6a34c9c23b200e19ab2463851e Mon Sep 17 00:00:00 2001 From: mesh-admin Date: Wed, 14 Oct 2026 06:42:51 +0000 Subject: [PATCH 27/31] List the K2 store tests again, apart from the integration branch's new lines Placed above the lines both branches added at the end of the file, so the pull request merges into feat/57-config-and-provisioning without a conflict and the merged list is the union of both. Co-Authored-By: Claude Opus 5.5 --- testdata/store-tests | 3 +++ 1 file changed, 3 insertions(+) diff --git a/testdata/store-tests b/testdata/store-tests index 62ca1239..725df90d 100644 --- a/testdata/store-tests +++ b/testdata/store-tests @@ -61,6 +61,9 @@ ./cmd/mesh-controller TestAProviderElsewhereServesItsOwnComposedName ./cmd/mesh-controller TestAReusedCheckBuildIsRegisteredByItsDigestAndGatedAsABuiltOne ./internal/inventory TestAReportOfABusyContainerIsKeptAndReadBack +./internal/inventory TestANewerBuildsListReplacesTheKeptOneAndAnOlderOneHeardLaterDoesNot +./cmd/mesh-controller TestTheMergeGateJudgesEachManifestAgainstTheListOfTheTree +./cmd/mesh-controller TestRegistrationPublishesTheNewestTrunkListAndJudgesEveryModuleAgainstIt ./internal/inventory TestPlacementMovesOutOfTheSettingsLayers ./internal/inventory TestASettingRoutesPlacementBesideTheLayer ./cmd/mesh-controller TestADeclarationIsTheSameAfterPlacementMoves -- 2.54.0 From 97a2d841c736c1d7d5420529f3356ba51d50f0dd Mon Sep 17 00:00:00 2001 From: mesh-tools Date: Thu, 15 Oct 2026 21:27:33 +0000 Subject: [PATCH 28/31] Land feature 57's controller work on its release branch The controller plans config, templates and moved providers' provisions, hands bus accounts to the vault and judges the new manifest words, on the feature's SDK and node-engine validator (feature 57, config and provisioning). --- cmd/mesh-builder/main.go | 53 +- cmd/mesh-builder/release_test.go | 27 + cmd/mesh-controller/acting.go | 7 +- cmd/mesh-controller/addresses_test.go | 2 +- cmd/mesh-controller/apply_failing.go | 46 + cmd/mesh-controller/batches.go | 11 +- cmd/mesh-controller/batches_test.go | 2 +- cmd/mesh-controller/build.go | 31 + cmd/mesh-controller/bus_users_closed_test.go | 92 + cmd/mesh-controller/busobjects.go | 47 +- cmd/mesh-controller/changeplan.go | 5 + cmd/mesh-controller/check.go | 6 + cmd/mesh-controller/check_builds_test.go | 90 +- cmd/mesh-controller/checks.go | 218 +- cmd/mesh-controller/config_move.go | 616 +++ cmd/mesh-controller/config_moved_test.go | 785 ++++ cmd/mesh-controller/config_read.go | 128 + cmd/mesh-controller/core_box.go | 297 ++ cmd/mesh-controller/core_box_test.go | 220 + cmd/mesh-controller/deadletters_aging.go | 117 + cmd/mesh-controller/deadletters_aging_test.go | 69 + cmd/mesh-controller/delivery.go | 6 +- cmd/mesh-controller/desk_secret.go | 134 +- cmd/mesh-controller/desk_secret_test.go | 215 + cmd/mesh-controller/doctor.go | 5 + cmd/mesh-controller/found_wait_test.go | 3 + cmd/mesh-controller/gate.go | 38 +- cmd/mesh-controller/gate_followup_test.go | 7 + cmd/mesh-controller/gate_test.go | 32 + cmd/mesh-controller/generation.go | 21 +- cmd/mesh-controller/given_test.go | 9 +- cmd/mesh-controller/handacts.go | 9 + cmd/mesh-controller/healers.go | 3 +- cmd/mesh-controller/heard_withheld.go | 66 + cmd/mesh-controller/heard_withheld_test.go | 43 + cmd/mesh-controller/held_back.go | 18 +- cmd/mesh-controller/issue352_test.go | 8 + cmd/mesh-controller/issue648_test.go | 11 +- cmd/mesh-controller/issued.go | 65 +- cmd/mesh-controller/issued_test.go | 83 +- cmd/mesh-controller/issued_to_vault.go | 67 + cmd/mesh-controller/issued_to_vault_test.go | 157 + cmd/mesh-controller/lean_walk_test.go | 242 ++ cmd/mesh-controller/main.go | 10 +- cmd/mesh-controller/merge_gate.go | 13 + cmd/mesh-controller/meshcli.go | 99 +- cmd/mesh-controller/meshcli_reads_test.go | 123 + cmd/mesh-controller/missed_merges.go | 2 +- cmd/mesh-controller/module_health.go | 15 + cmd/mesh-controller/modules.go | 66 +- cmd/mesh-controller/person_wait_test.go | 7 + cmd/mesh-controller/placement_move.go | 99 + .../placement_words_moved_test.go | 243 ++ cmd/mesh-controller/plain_words.go | 23 + cmd/mesh-controller/plan.go | 132 +- cmd/mesh-controller/plan_retry.go | 7 + cmd/mesh-controller/planner_records_test.go | 2 +- cmd/mesh-controller/planner_rules_test.go | 20 +- cmd/mesh-controller/probes.go | 6 + cmd/mesh-controller/proposals.go | 16 + cmd/mesh-controller/provisioned.go | 100 + cmd/mesh-controller/provisioned_test.go | 114 + .../provisioner_account_test.go | 62 + cmd/mesh-controller/push.go | 99 +- cmd/mesh-controller/queue.go | 14 +- cmd/mesh-controller/queue_test.go | 6 +- cmd/mesh-controller/record_mirrors.go | 106 + cmd/mesh-controller/recorded_kept.go | 6 + cmd/mesh-controller/recorded_moves.go | 213 + cmd/mesh-controller/recorded_moves_test.go | 361 ++ cmd/mesh-controller/reissue_hearing_test.go | 39 + cmd/mesh-controller/release.go | 60 +- cmd/mesh-controller/release_branch_test.go | 273 ++ cmd/mesh-controller/release_plan.go | 359 +- cmd/mesh-controller/release_plan_test.go | 26 +- .../release_provisioned_test.go | 172 + cmd/mesh-controller/release_test.go | 9 +- cmd/mesh-controller/replay318_test.go | 1 + cmd/mesh-controller/replay335_test.go | 2 + cmd/mesh-controller/replay336_test.go | 1 + cmd/mesh-controller/replays_test.go | 1 + cmd/mesh-controller/rest_at_once_test.go | 12 +- cmd/mesh-controller/rollout.go | 13 + cmd/mesh-controller/rotate.go | 152 +- cmd/mesh-controller/rotate_vault_test.go | 124 + cmd/mesh-controller/runtime_health_test.go | 55 + cmd/mesh-controller/same_program.go | 176 + cmd/mesh-controller/same_program_test.go | 141 + cmd/mesh-controller/sayable_test.go | 3 + cmd/mesh-controller/seatverbs.go | 67 +- cmd/mesh-controller/seatverbs_schema_test.go | 2 + cmd/mesh-controller/secret.go | 157 +- cmd/mesh-controller/secret_move.go | 336 ++ cmd/mesh-controller/secret_move_test.go | 158 + cmd/mesh-controller/sendable.go | 17 + cmd/mesh-controller/sent_required_by_test.go | 125 + cmd/mesh-controller/settings_forward.go | 372 ++ cmd/mesh-controller/signals.go | 62 +- cmd/mesh-controller/status.go | 27 + cmd/mesh-controller/status_summary.go | 5 + cmd/mesh-controller/stores.go | 2 + cmd/mesh-controller/superseded.go | 157 + cmd/mesh-controller/superseded_test.go | 192 + cmd/mesh-controller/template_test.go | 37 + .../testdata/config-moved/mesh.json | 3661 +++++++++++++++++ .../placement-words-moved/anchor.json | 2219 ++++++++++ .../placement-words-moved/laptop.json | 279 ++ .../testdata/placement-words-moved/mesh.json | 3288 +++++++++++++++ cmd/mesh-controller/tier_send_test.go | 7 + cmd/mesh-controller/upgrades.go | 7 +- cmd/mesh-controller/urgent_unhealthy_test.go | 91 + cmd/mesh-controller/vault_handed_test.go | 83 + cmd/mesh-controller/vault_secrets.go | 403 ++ cmd/mesh-controller/vault_secrets_test.go | 529 +++ cmd/mesh-controller/vault_switch_test.go | 413 ++ .../walk_wait_rebuild_754_test.go | 122 + cmd/mesh-controller/watchdogs.go | 30 +- cmd/mesh-controller/worker_order_test.go | 5 +- go.mod | 4 +- go.sum | 14 +- internal/broker/bus_module_emits_test.go | 34 + internal/broker/config_owner_live_test.go | 162 + internal/broker/controller_buckets.go | 69 +- internal/broker/controller_buckets_test.go | 132 +- internal/broker/derived.go | 25 +- internal/broker/genesis_template_test.go | 55 +- internal/broker/jetstream.go | 26 + internal/broker/log_test.go | 8 +- internal/broker/membership.go | 12 + internal/broker/modstreams.go | 302 ++ internal/broker/modstreams_live_test.go | 68 + internal/broker/modstreams_test.go | 307 ++ internal/broker/move_secrets_test.go | 88 + internal/broker/msgttl_live_test.go | 86 + internal/broker/nats.go | 199 +- internal/broker/node_key_test.go | 58 + internal/broker/prov_applied_live_test.go | 135 + internal/broker/provisioner.go | 212 + internal/broker/provisioner_live_test.go | 393 ++ .../broker/provisioner_server_live_test.go | 217 + internal/broker/provisioner_test.go | 229 ++ internal/broker/states_agreement_test.go | 7 + .../broker/stream_names_agreement_test.go | 23 + internal/broker/streams.go | 41 +- internal/broker/streams_test.go | 3 + internal/broker/testdata/composed.conf | 4 +- internal/broker/users.go | 112 +- internal/broker/vault_calls_live_test.go | 227 + internal/broker/waits.go | 100 + internal/broker/waits_test.go | 135 + internal/broker/writers.go | 27 + internal/broker/writers_test.go | 6 + internal/builder/builder.go | 52 +- internal/builder/check.go | 656 ++- internal/builder/check_builds.go | 334 +- internal/builder/check_merge_test.go | 50 + internal/builder/release_branch_test.go | 870 ++++ internal/builder/replay713_test.go | 125 + internal/builder/replay716_test.go | 158 + internal/builder/reuse.go | 16 +- internal/builder/reuse_test.go | 154 +- internal/builder/same_program.go | 180 + internal/builder/same_program_test.go | 126 + internal/builder/template.go | 23 +- internal/builder/template_test.go | 45 + internal/builder/warm_go_cache_test.go | 80 + internal/catalogue/adoption_test.go | 14 +- internal/catalogue/backup_placement.go | 53 + internal/catalogue/backup_placement_test.go | 126 + internal/catalogue/bound.go | 2 + internal/catalogue/build.go | 22 +- internal/catalogue/bus_snapshot_test.go | 10 + internal/catalogue/collections.go | 635 +++ internal/catalogue/collections_bar_test.go | 135 + internal/catalogue/collections_l2_test.go | 60 + internal/catalogue/collections_l3_test.go | 158 + internal/catalogue/collections_test.go | 179 + internal/catalogue/config.go | 56 +- internal/catalogue/config_move.go | 40 + internal/catalogue/config_rename_test.go | 185 + internal/catalogue/config_test.go | 30 + internal/catalogue/consumers.go | 110 + .../contribution_dependencies_test.go | 4 +- internal/catalogue/core_credentials.go | 33 + internal/catalogue/core_credentials_test.go | 107 + internal/catalogue/data_test.go | 6 +- internal/catalogue/declaration.go | 197 +- internal/catalogue/endpoints_setting_test.go | 6 +- internal/catalogue/environment_into.go | 166 +- internal/catalogue/environment_items_test.go | 165 + internal/catalogue/filtering.go | 8 +- internal/catalogue/filtering_test.go | 2 +- .../catalogue/foundation_manifests_test.go | 4 +- internal/catalogue/graphical_session.go | 1 + internal/catalogue/handed.go | 799 ++++ internal/catalogue/handed_bound_test.go | 125 + internal/catalogue/handed_consumers_test.go | 134 + internal/catalogue/handed_merge_test.go | 57 + internal/catalogue/handed_providers_test.go | 244 ++ internal/catalogue/handed_pulled_in_test.go | 121 + internal/catalogue/handed_secrets_test.go | 252 ++ internal/catalogue/handed_test.go | 188 + internal/catalogue/health.go | 41 +- internal/catalogue/holds_test.go | 100 + internal/catalogue/manifest.go | 248 +- internal/catalogue/own_secret_prompt_test.go | 111 + internal/catalogue/placement.go | 26 +- internal/catalogue/placement_test.go | 61 + internal/catalogue/predefined_test.go | 6 +- internal/catalogue/reach_test.go | 8 +- internal/catalogue/recreates.go | 221 + internal/catalogue/resolve.go | 27 +- .../resolver_file_is_the_uplinks_test.go | 2 +- internal/catalogue/runtime.go | 5 + internal/catalogue/runtime_health_test.go | 129 + internal/catalogue/runtime_test.go | 56 + internal/catalogue/seat_contributions.go | 22 +- internal/catalogue/seat_contributions_test.go | 2 +- internal/catalogue/seat_data.go | 37 +- internal/catalogue/seat_data_test.go | 4 +- internal/catalogue/seat_dependencies.go | 49 +- internal/catalogue/seat_dependencies_test.go | 36 +- internal/catalogue/seats.go | 130 +- internal/catalogue/seats_declared.go | 65 + internal/catalogue/seats_declared_test.go | 47 + internal/catalogue/seats_test.go | 6 +- internal/catalogue/setting_in_owner_test.go | 32 + internal/catalogue/setting_into.go | 6 + internal/catalogue/setting_takes.go | 184 +- internal/catalogue/settings.go | 11 +- internal/catalogue/settings_judge_test.go | 4 +- internal/catalogue/state.go | 13 + internal/catalogue/state_test.go | 15 + internal/catalogue/streams.go | 514 +++ internal/catalogue/streams_test.go | 244 ++ internal/catalogue/template.go | 216 +- internal/catalogue/templates_dir_test.go | 151 + internal/catalogue/terminal_keys.go | 9 +- internal/catalogue/testdata/bars/bar-battery | 8 + .../catalogue/testdata/bars/bar-power-draw | 5 + .../testdata/bars/battery.bottom-bar.toml | 119 + .../testdata/bars/battery.top-bar.toml | 36 + .../testdata/bars/both.bottom-bar.toml | 126 + .../catalogue/testdata/bars/both.top-bar.toml | 36 + .../catalogue/testdata/bars/bottom-bar.toml | 109 + .../testdata/bars/none.bottom-bar.toml | 109 + .../catalogue/testdata/bars/none.top-bar.toml | 36 + .../testdata/bars/power-meter.bottom-bar.toml | 116 + .../testdata/bars/power-meter.top-bar.toml | 36 + internal/catalogue/testdata/bars/top-bar.toml | 37 + .../catalogue/testdata/environment/all.posix | 26 + .../testdata/environment/all.systemd | 23 + .../catalogue/testdata/environment/zsh.posix | 9 + .../testdata/environment/zsh.systemd | 6 + ...yrus-g14.contribution-node-hotkeys-trigger | 10 + .../asus-zephyrus-g14.shell-after-wake-normal | 7 + .../items/claude-code.shell-zsh-normal | 12 + .../l2-moved/items/mesh-cli.shell-zsh-normal | 3 + .../items/powerlevel10k.shell-zsh-normal | 3 + .../zsh-autosuggestions.shell-zsh-normal | 2 + .../zsh-syntax-highlighting.shell-zsh-last | 3 + .../testdata/l2-moved/templates/power | 4 + .../testdata/l2-moved/templates/triggerhappy | 4 + .../catalogue/testdata/l2-moved/templates/zsh | 60 + .../catalogue/testdata/l2-moved/want/power | 11 + .../testdata/l2-moved/want/triggerhappy | 14 + internal/catalogue/testdata/l2-moved/want/zsh | 87 + internal/catalogue/unconsumed_placeholder.go | 6 + internal/catalogue/unhealthy.go | 55 + internal/catalogue/unhealthy_test.go | 53 + internal/catalogue/vault_seat_test.go | 115 + internal/catalogue/verbs.go | 45 +- internal/catalogue/waits_test.go | 43 + internal/conditions/plain_test.go | 6 +- internal/inventory/builds.go | 46 + internal/inventory/busrecords.go | 150 +- internal/inventory/busrecords_test.go | 44 + internal/inventory/bususers.go | 3 + internal/inventory/catalogue.go | 63 +- internal/inventory/config_move.go | 217 + internal/inventory/config_move_test.go | 218 + internal/inventory/core_box.go | 65 + internal/inventory/core_box_test.go | 47 + .../0099-the-secrets-move-into-the-vault.sql | 22 + ...00-a-send-keeps-the-provisioners-plans.sql | 10 + ...the-cores-credentials-the-vault-sealed.sql | 18 + ...2-the-settings-layers-move-into-config.sql | 15 + internal/inventory/mirrors.go | 204 + internal/inventory/nodes.go | 17 +- internal/inventory/placement_move.go | 106 + internal/inventory/placement_words_test.go | 262 ++ internal/inventory/plans.go | 8 + internal/inventory/plans_test.go | 3 +- internal/inventory/ports.go | 17 +- internal/inventory/provisioned.go | 120 + internal/inventory/provisioned_test.go | 124 + internal/inventory/provisioning.go | 517 +++ internal/inventory/provisioning_test.go | 354 ++ internal/inventory/record_mirrors_test.go | 113 + internal/inventory/registered_held_test.go | 72 + internal/inventory/secret_move.go | 218 + internal/inventory/secret_move_test.go | 121 + internal/inventory/secrets.go | 23 +- internal/inventory/sends.go | 20 +- internal/inventory/sent_builds_test.go | 4 +- internal/inventory/sent_plans_test.go | 96 + internal/inventory/streams_test.go | 45 + internal/inventory/waits_grants_test.go | 167 + internal/inventory/walkphases.go | 12 +- internal/inventory/walkphases_582_test.go | 22 + internal/inventory/walkphases_test.go | 2 +- internal/lease/lease.go | 203 +- internal/lease/lease_test.go | 212 + internal/link/advisories.go | 12 +- internal/link/build.go | 52 +- internal/link/builds_nats.go | 2 +- internal/link/busfaults.go | 235 ++ internal/link/busfaults_test.go | 121 + internal/link/calls.go | 20 +- internal/link/calls_kept.go | 47 + internal/link/calls_kept_test.go | 159 + internal/link/check_builds_test.go | 48 + internal/link/deadletters.go | 198 +- .../link/deadletters_module_stream_test.go | 48 + internal/link/deadletters_restricted_test.go | 251 ++ internal/link/events.go | 26 +- internal/link/movesecret.go | 129 + internal/link/movesecret_test.go | 90 + internal/link/protocol.go | 29 + internal/link/protocol_node_key_test.go | 11 + internal/link/queue.go | 40 +- internal/link/receive_nats.go | 11 +- internal/link/receive_nats_test.go | 50 +- internal/link/retirement.go | 5 + internal/link/seat_as_test.go | 47 + internal/link/vault_changed_test.go | 16 + internal/recordmirror/mirror.go | 348 ++ internal/recordmirror/mirror_store_test.go | 254 ++ internal/recordmirror/mirror_test.go | 356 ++ .../recordmirror/provisioning_live_test.go | 161 + internal/recordmirror/readers_test.go | 183 + merge-check.sh | 167 +- module.json | 5 +- testdata/beside/CAPTURED | 4 +- .../examples/foundation-first-node.lock | 108 +- testdata/store-tests | 43 + .../novox/mesh-sdk/go/seal/seal.go | 203 + .../internal/declaration/declaration.go | 172 +- vendor/modules.txt | 7 +- 349 files changed, 42440 insertions(+), 786 deletions(-) create mode 100644 cmd/mesh-builder/release_test.go create mode 100644 cmd/mesh-controller/bus_users_closed_test.go create mode 100644 cmd/mesh-controller/config_move.go create mode 100644 cmd/mesh-controller/config_moved_test.go create mode 100644 cmd/mesh-controller/config_read.go create mode 100644 cmd/mesh-controller/core_box.go create mode 100644 cmd/mesh-controller/core_box_test.go create mode 100644 cmd/mesh-controller/deadletters_aging.go create mode 100644 cmd/mesh-controller/deadletters_aging_test.go create mode 100644 cmd/mesh-controller/heard_withheld.go create mode 100644 cmd/mesh-controller/heard_withheld_test.go create mode 100644 cmd/mesh-controller/issued_to_vault.go create mode 100644 cmd/mesh-controller/issued_to_vault_test.go create mode 100644 cmd/mesh-controller/lean_walk_test.go create mode 100644 cmd/mesh-controller/meshcli_reads_test.go create mode 100644 cmd/mesh-controller/placement_move.go create mode 100644 cmd/mesh-controller/placement_words_moved_test.go create mode 100644 cmd/mesh-controller/provisioned.go create mode 100644 cmd/mesh-controller/provisioned_test.go create mode 100644 cmd/mesh-controller/provisioner_account_test.go create mode 100644 cmd/mesh-controller/record_mirrors.go create mode 100644 cmd/mesh-controller/recorded_moves.go create mode 100644 cmd/mesh-controller/recorded_moves_test.go create mode 100644 cmd/mesh-controller/reissue_hearing_test.go create mode 100644 cmd/mesh-controller/release_branch_test.go create mode 100644 cmd/mesh-controller/release_provisioned_test.go create mode 100644 cmd/mesh-controller/rotate_vault_test.go create mode 100644 cmd/mesh-controller/runtime_health_test.go create mode 100644 cmd/mesh-controller/same_program.go create mode 100644 cmd/mesh-controller/same_program_test.go create mode 100644 cmd/mesh-controller/secret_move.go create mode 100644 cmd/mesh-controller/secret_move_test.go create mode 100644 cmd/mesh-controller/sent_required_by_test.go create mode 100644 cmd/mesh-controller/settings_forward.go create mode 100644 cmd/mesh-controller/superseded.go create mode 100644 cmd/mesh-controller/superseded_test.go create mode 100644 cmd/mesh-controller/testdata/config-moved/mesh.json create mode 100644 cmd/mesh-controller/testdata/placement-words-moved/anchor.json create mode 100644 cmd/mesh-controller/testdata/placement-words-moved/laptop.json create mode 100644 cmd/mesh-controller/testdata/placement-words-moved/mesh.json create mode 100644 cmd/mesh-controller/urgent_unhealthy_test.go create mode 100644 cmd/mesh-controller/vault_handed_test.go create mode 100644 cmd/mesh-controller/vault_secrets.go create mode 100644 cmd/mesh-controller/vault_secrets_test.go create mode 100644 cmd/mesh-controller/vault_switch_test.go create mode 100644 cmd/mesh-controller/walk_wait_rebuild_754_test.go create mode 100644 internal/broker/bus_module_emits_test.go create mode 100644 internal/broker/config_owner_live_test.go create mode 100644 internal/broker/modstreams.go create mode 100644 internal/broker/modstreams_live_test.go create mode 100644 internal/broker/modstreams_test.go create mode 100644 internal/broker/move_secrets_test.go create mode 100644 internal/broker/msgttl_live_test.go create mode 100644 internal/broker/node_key_test.go create mode 100644 internal/broker/prov_applied_live_test.go create mode 100644 internal/broker/provisioner.go create mode 100644 internal/broker/provisioner_live_test.go create mode 100644 internal/broker/provisioner_server_live_test.go create mode 100644 internal/broker/provisioner_test.go create mode 100644 internal/broker/stream_names_agreement_test.go create mode 100644 internal/broker/vault_calls_live_test.go create mode 100644 internal/broker/waits.go create mode 100644 internal/broker/waits_test.go create mode 100644 internal/builder/release_branch_test.go create mode 100644 internal/builder/replay713_test.go create mode 100644 internal/builder/replay716_test.go create mode 100644 internal/builder/same_program.go create mode 100644 internal/builder/same_program_test.go create mode 100644 internal/builder/warm_go_cache_test.go create mode 100644 internal/catalogue/backup_placement.go create mode 100644 internal/catalogue/backup_placement_test.go create mode 100644 internal/catalogue/collections.go create mode 100644 internal/catalogue/collections_bar_test.go create mode 100644 internal/catalogue/collections_l2_test.go create mode 100644 internal/catalogue/collections_l3_test.go create mode 100644 internal/catalogue/collections_test.go create mode 100644 internal/catalogue/config_move.go create mode 100644 internal/catalogue/config_rename_test.go create mode 100644 internal/catalogue/consumers.go create mode 100644 internal/catalogue/core_credentials.go create mode 100644 internal/catalogue/core_credentials_test.go create mode 100644 internal/catalogue/environment_items_test.go create mode 100644 internal/catalogue/handed.go create mode 100644 internal/catalogue/handed_bound_test.go create mode 100644 internal/catalogue/handed_consumers_test.go create mode 100644 internal/catalogue/handed_merge_test.go create mode 100644 internal/catalogue/handed_providers_test.go create mode 100644 internal/catalogue/handed_pulled_in_test.go create mode 100644 internal/catalogue/handed_secrets_test.go create mode 100644 internal/catalogue/handed_test.go create mode 100644 internal/catalogue/holds_test.go create mode 100644 internal/catalogue/own_secret_prompt_test.go create mode 100644 internal/catalogue/runtime_health_test.go create mode 100644 internal/catalogue/streams.go create mode 100644 internal/catalogue/streams_test.go create mode 100644 internal/catalogue/templates_dir_test.go create mode 100644 internal/catalogue/testdata/bars/bar-battery create mode 100644 internal/catalogue/testdata/bars/bar-power-draw create mode 100644 internal/catalogue/testdata/bars/battery.bottom-bar.toml create mode 100644 internal/catalogue/testdata/bars/battery.top-bar.toml create mode 100644 internal/catalogue/testdata/bars/both.bottom-bar.toml create mode 100644 internal/catalogue/testdata/bars/both.top-bar.toml create mode 100644 internal/catalogue/testdata/bars/bottom-bar.toml create mode 100644 internal/catalogue/testdata/bars/none.bottom-bar.toml create mode 100644 internal/catalogue/testdata/bars/none.top-bar.toml create mode 100644 internal/catalogue/testdata/bars/power-meter.bottom-bar.toml create mode 100644 internal/catalogue/testdata/bars/power-meter.top-bar.toml create mode 100644 internal/catalogue/testdata/bars/top-bar.toml create mode 100644 internal/catalogue/testdata/environment/all.posix create mode 100644 internal/catalogue/testdata/environment/all.systemd create mode 100644 internal/catalogue/testdata/environment/zsh.posix create mode 100644 internal/catalogue/testdata/environment/zsh.systemd create mode 100644 internal/catalogue/testdata/l2-moved/items/asus-zephyrus-g14.contribution-node-hotkeys-trigger create mode 100644 internal/catalogue/testdata/l2-moved/items/asus-zephyrus-g14.shell-after-wake-normal create mode 100644 internal/catalogue/testdata/l2-moved/items/claude-code.shell-zsh-normal create mode 100644 internal/catalogue/testdata/l2-moved/items/mesh-cli.shell-zsh-normal create mode 100644 internal/catalogue/testdata/l2-moved/items/powerlevel10k.shell-zsh-normal create mode 100644 internal/catalogue/testdata/l2-moved/items/zsh-autosuggestions.shell-zsh-normal create mode 100644 internal/catalogue/testdata/l2-moved/items/zsh-syntax-highlighting.shell-zsh-last create mode 100644 internal/catalogue/testdata/l2-moved/templates/power create mode 100644 internal/catalogue/testdata/l2-moved/templates/triggerhappy create mode 100644 internal/catalogue/testdata/l2-moved/templates/zsh create mode 100644 internal/catalogue/testdata/l2-moved/want/power create mode 100644 internal/catalogue/testdata/l2-moved/want/triggerhappy create mode 100644 internal/catalogue/testdata/l2-moved/want/zsh create mode 100644 internal/catalogue/unhealthy.go create mode 100644 internal/catalogue/unhealthy_test.go create mode 100644 internal/catalogue/waits_test.go create mode 100644 internal/inventory/config_move.go create mode 100644 internal/inventory/config_move_test.go create mode 100644 internal/inventory/core_box.go create mode 100644 internal/inventory/core_box_test.go create mode 100644 internal/inventory/migrations/0099-the-secrets-move-into-the-vault.sql create mode 100644 internal/inventory/migrations/0100-a-send-keeps-the-provisioners-plans.sql create mode 100644 internal/inventory/migrations/0101-the-cores-credentials-the-vault-sealed.sql create mode 100644 internal/inventory/migrations/0102-the-settings-layers-move-into-config.sql create mode 100644 internal/inventory/mirrors.go create mode 100644 internal/inventory/placement_move.go create mode 100644 internal/inventory/placement_words_test.go create mode 100644 internal/inventory/provisioned.go create mode 100644 internal/inventory/provisioned_test.go create mode 100644 internal/inventory/provisioning.go create mode 100644 internal/inventory/provisioning_test.go create mode 100644 internal/inventory/record_mirrors_test.go create mode 100644 internal/inventory/registered_held_test.go create mode 100644 internal/inventory/secret_move.go create mode 100644 internal/inventory/secret_move_test.go create mode 100644 internal/inventory/sent_plans_test.go create mode 100644 internal/inventory/streams_test.go create mode 100644 internal/inventory/waits_grants_test.go create mode 100644 internal/link/busfaults.go create mode 100644 internal/link/busfaults_test.go create mode 100644 internal/link/deadletters_module_stream_test.go create mode 100644 internal/link/deadletters_restricted_test.go create mode 100644 internal/link/movesecret.go create mode 100644 internal/link/movesecret_test.go create mode 100644 internal/link/protocol_node_key_test.go create mode 100644 internal/link/seat_as_test.go create mode 100644 internal/link/vault_changed_test.go create mode 100644 internal/recordmirror/mirror.go create mode 100644 internal/recordmirror/mirror_store_test.go create mode 100644 internal/recordmirror/mirror_test.go create mode 100644 internal/recordmirror/provisioning_live_test.go create mode 100644 internal/recordmirror/readers_test.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/seal/seal.go diff --git a/cmd/mesh-builder/main.go b/cmd/mesh-builder/main.go index 985e8652..216edd17 100644 --- a/cmd/mesh-builder/main.go +++ b/cmd/mesh-builder/main.go @@ -26,6 +26,7 @@ import ( "os/signal" "sort" "strings" + "sync" "syscall" "time" @@ -272,7 +273,12 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri // role's own events under the build's id (novox/hq ADR 0157) — so whoever asked, and anybody // watching, reads the same lines this container's log holds, live, and after the fact from the // stream. Said first, before anything runs, so a build that hangs is one that visibly started. + // One line at a time: a check's builds for its merge say their lines while its tests say theirs (novox/hq + // issue 716). + var saying sync.Mutex say := func(step, message string) { + saying.Lock() + defer saying.Unlock() fmt.Fprintf(os.Stderr, " [%s] %s\n", step, message) work.Say(step, message) if mine != nil && step != "run" && step != "output" { @@ -313,20 +319,25 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri registry = r.Address } spec := checkSpecOf(request) + // The judge cache the build agent declares as its own data, never in the workspace: only judge builds and + // mesh-lab's replays mount it, never a check's containers, so no pull request's script can write what a judge + // is built with or kept as, or what the replays run (novox/hq ADR 0358 §6, issue 709). + spec.JudgeCache = judgeCache() // What the merge would move, built by the check for the walk to take (novox/hq ADR 0350 §1): into the // scratch namespace. **Never with the package registry's credential**: this is code nobody has approved // yet, and a recipe could read the credential it is handed (issue 462). A module that resolves packages // at build time has no reuse key anyway, so nothing is lost by refusing it here. if isRegistry && len(spec.Builds) > 0 { spec.BuildWith = &builder.CheckBuilding{Publish: publisher, Scratch: registryHere, Held: request.Held, - Seats: request.Seats} + Running: request.Check.Running, Seats: request.Seats, Named: namedOf(request.Check.Named)} } var v builder.CheckVerdict v, err = builder.Check(building, builder.Command, spec, workspace, registry, forgeFrom(), say) if err == nil { result.Check = &link.CheckOutcome{Verdict: v.Verdict, Summary: v.Summary, Report: v.Report, Took: v.Took.Round(time.Second).String(), Gate: layerOf(v.Gate), RepoCheck: layerOf(v.Repo), - Head: v.Head, Trunk: v.Trunk, Beside: v.Beside, Built: checkBuiltOf(v.Built)} + Head: v.Head, Trunk: v.Trunk, Beside: v.Beside, Built: checkBuiltOf(v.Built), + Release: v.Release, JudgedBy: v.JudgedBy, BesideAt: v.BesideAt, Validator: v.Validator} // What each of a group's heads was judged as (novox/hq ADR 0323 §6). for _, m := range v.Members { result.Check.Members = append(result.Check.Members, link.CheckedMember{Owner: m.Owner, Repo: m.Repo, @@ -352,15 +363,7 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri reuse.Pulls = append(reuse.Pulls, builder.PullRef{Owner: p.Owner, Repo: p.Repo, Number: p.Number}) } // Only what a verdict names is taken (ADR 0350 §2). - for _, b := range request.Reuse.Builds { - named := builder.NamedBuild{Pull: builder.PullRef{Owner: b.Pull.Owner, Repo: b.Pull.Repo, - Number: b.Pull.Number}, Check: b.Check, Module: b.Built.Module, Key: b.Built.Key, - Digests: map[string]string{}} - for _, m := range b.Built.Made { - named.Digests[m.Name] = builder.DigestOf(m.Reference) - } - reuse.Named = append(reuse.Named, named) - } + reuse.Named = namedOf(request.Reuse.Builds) built, err = builder.BuildReusing(building, builder.Labelled(builder.Command, request.ID), publisher, reuse, request.Repository, request.Path, request.Ref, workspace, request.Held, npmrc, forgeFrom(), say, request.Seats) @@ -469,9 +472,9 @@ func checkSpecOf(request link.BuildRequest) builder.CheckSpec { Owner: c.Owner, Repo: c.Repo, Number: c.Number, Paths: c.Paths, Beside: map[string]builder.Beside{}, Modules: c.Modules, New: c.New, Manifests: c.Manifests, Judge: c.Judge, Base: c.Base, Toolchain: builder.ToolchainOf(request.Held), Toolchains: builder.ToolchainsOf(request.Held), - Tests: c.Tests, StoreOnly: c.StoreOnly} + Tests: c.Tests, StoreOnly: c.StoreOnly, Release: c.Release} for dir, b := range c.Beside { - spec.Beside[dir] = builder.Beside{Repository: b.Repository, Ref: b.Ref} + spec.Beside[dir] = builder.Beside{Repository: b.Repository, Ref: b.Ref, Else: b.Else} } for _, b := range c.Builds { spec.Builds = append(spec.Builds, builder.CheckBuildSpec{Module: b.Module, Repository: b.Repository, Path: b.Path, @@ -484,6 +487,11 @@ func checkSpecOf(request link.BuildRequest) builder.CheckSpec { return spec } +// judgeCache is the judge cache the build agent's manifest declares (MESH_JUDGE_CACHE, novox/hq ADR 0358 §6): a +// directory mounted at the same path on the host, so a judge build's mounts name what the agent sees. Empty when the +// agent declares none, and every judge is then built cold under its check's root. +func judgeCache() string { return os.Getenv("MESH_JUDGE_CACHE") } + // triedOf is a try's outcome as its log line says it (novox/hq issue 686): the repository's own check's verdict, // or the check's where that did not run — a head that does not merge — and the lines naming what failed. func triedOf(v builder.CheckVerdict) link.Tried { @@ -501,11 +509,28 @@ func triedOf(v builder.CheckVerdict) link.Tried { return t } +// namedOf are the builds the seat's verdicts name, as the builder believes a scratch record by them (novox/hq ADR +// 0350 §2, issue 703): each module's key with every artifact's digest. +func namedOf(builds []link.CheckedBuild) []builder.NamedBuild { + var out []builder.NamedBuild + for _, b := range builds { + named := builder.NamedBuild{Pull: builder.PullRef{Owner: b.Pull.Owner, Repo: b.Pull.Repo, + Number: b.Pull.Number}, Check: b.Check, Module: b.Built.Module, Key: b.Built.Key, + Digests: map[string]string{}} + for _, m := range b.Built.Made { + named.Digests[m.Name] = builder.DigestOf(m.Reference) + } + out = append(out, named) + } + return out +} + // checkBuiltOf is a check's builds for its merge as the outcome carries them (novox/hq ADR 0350 §1). func checkBuiltOf(built []builder.CheckBuilt) []link.CheckBuilt { var out []link.CheckBuilt for _, b := range built { - c := link.CheckBuilt{Module: b.Module, Key: b.Key, NotBuilt: b.NotBuilt, Took: b.Took.Round(time.Second).String()} + c := link.CheckBuilt{Module: b.Module, Key: b.Key, NotBuilt: b.NotBuilt, Took: b.Took.Round(time.Second).String(), + Kept: b.Kept, Same: b.Same, SameAs: b.SameAs} for _, m := range b.Made { c.Made = append(c.Made, link.MadeArtifact{Name: m.Name, Kind: m.Kind, Reference: m.Reference}) } diff --git a/cmd/mesh-builder/release_test.go b/cmd/mesh-builder/release_test.go new file mode 100644 index 00000000..2eb9c8f3 --- /dev/null +++ b/cmd/mesh-builder/release_test.go @@ -0,0 +1,27 @@ +package main + +import ( + "testing" + + "github.com/novox/mesh-controller/internal/link" +) + +// **A pull request into a release branch reaches the check as one** (novox/hq ADR 0358): the build seat hands the +// check the release branch and, beside it, the trunk each repository is read at when it has no branch of that name; +// and it hands the check the judge cache the build agent declares (MESH_JUDGE_CACHE), never a path of its own. +func TestTheBuildSeatHandsTheCheckItsReleaseBranch(t *testing.T) { + spec := checkSpecOf(link.BuildRequest{ID: "b", Repository: "r", Ref: "abc", Check: &link.CheckRequest{ + Owner: "novox", Repo: "mesh-catalog", Base: "feat/57", Release: "feat/57", + Beside: map[string]link.CheckedOut{"mesh-host": {Repository: "h", Ref: "feat/57", Else: "main"}}}}) + if spec.Release != "feat/57" || spec.Beside["mesh-host"].Ref != "feat/57" || spec.Beside["mesh-host"].Else != "main" { + t.Fatalf("the check was handed release %q and mesh-host %+v", spec.Release, spec.Beside["mesh-host"]) + } + t.Setenv("MESH_JUDGE_CACHE", "/var/lib/mesh/build-agent/judge-cache") + if got := judgeCache(); got != "/var/lib/mesh/build-agent/judge-cache" { + t.Fatalf("the judge cache the manifest declares is read as %q", got) + } + t.Setenv("MESH_JUDGE_CACHE", "") + if got := judgeCache(); got != "" { + t.Fatalf("with no judge cache declared, %q is used", got) + } +} diff --git a/cmd/mesh-controller/acting.go b/cmd/mesh-controller/acting.go index 6ca1dbac..fe5aca76 100644 --- a/cmd/mesh-controller/acting.go +++ b/cmd/mesh-controller/acting.go @@ -26,8 +26,11 @@ import ( // // - **The serving controller** takes it before it does anything else — before it asserts the bus's // objects, which are the controller's to write — waiting while another holds it, and renews it. -// A renewal refused or failed is the lease lost: the gate closes at once and the process exits, so -// its service manager restarts it as a candidate (serve, in push.go). +// Another holder seen, or the key's age passed with no renewal made, is the lease lost: the gate +// closes at once and the process exits, so its service manager restarts it as a candidate (serve, +// in push.go). A renewal the bus does not answer in time is not a loss (novox/hq issue 477): the +// lease is kept, no new act starts once half the key's age passed unrenewed, and the next renewal +// made opens the gate again. // - **A command run at a shell** — `push` in the installer, the lab, a person repairing a mesh whose // controller is down (issue 201) — acts **under the holder's epoch** when a controller holds the // lease: it is the same mesh's word, composed and sent under the store's hold of each machine like diff --git a/cmd/mesh-controller/addresses_test.go b/cmd/mesh-controller/addresses_test.go index dad8dce3..bc02b37b 100644 --- a/cmd/mesh-controller/addresses_test.go +++ b/cmd/mesh-controller/addresses_test.go @@ -140,7 +140,7 @@ func TestTheRegistryTrustAndEveryImageFollowThePortTheNodeGaveTheStore(t *testin } // And a replay to the catalogue says where the store is now. - announced, err := following{open}.Announceable(ctx) + announced, err := following{open: open}.Announceable(ctx) if err != nil { t.Fatal(err) } diff --git a/cmd/mesh-controller/apply_failing.go b/cmd/mesh-controller/apply_failing.go index 6daf960d..986207ce 100644 --- a/cmd/mesh-controller/apply_failing.go +++ b/cmd/mesh-controller/apply_failing.go @@ -38,15 +38,61 @@ func watchApplyFailing(f *signalFacts) []conditions.Observation { } } var out []conditions.Observation + said := map[string]bool{} for _, d := range f.doing { if !d.Wrong() || d.Since == nil || f.now.Sub(*d.Since) <= applyEvery || !heard[d.Node] { continue } out = append(out, applyFailing(d, f.now)) + said[d.Node] = true + } + // **A module's files the provisioner failed to place are a failed apply too** (novox/hq ADR 0362 §3): its client's + // newest word, judged by the same bound — the client says a failure once, and fills again on every change — + // on a machine whose node-engine applied what it was sent. + failed := map[string][]inventory.ProvReport{} + for _, r := range f.provFailed { + if r.Status != inventory.ProvFailed || f.now.Sub(r.At) <= applyEvery || !heard[r.Node] || said[r.Node] { + continue + } + failed[r.Node] = append(failed[r.Node], r) + } + for _, node := range sortedNames(failed) { + out = append(out, provisioningFailing(node, failed[node], f.now)) } return out } +// provisioningFailing is one machine's S21 condition for the modules whose files its provisioner client failed to +// place. +func provisioningFailing(m string, failed []inventory.ProvReport, now time.Time) conditions.Observation { + sort.Slice(failed, func(a, b int) bool { return failed[a].Module < failed[b].Module }) + var names, words []string + since := failed[0].At + for _, r := range failed { + names = append(names, r.Module) + words = append(words, r.Module+": "+oneLine(orNoReason(r.Reason))) + if r.At.Before(since) { + since = r.At + } + } + o := conditions.Observation{Scope: conditions.ScopeMachine, ID: m, Kind: kindApplyFailing, Machine: m, + Severity: conditions.Warning, Said: strings.Join(words, "; "), + Resolved: m + " applies what it is sent again"} + o.Summary = fmt.Sprintf("%s's provisioner has failed to place the files of %s since %s (bound %s)", m, + strings.Join(names, ", "), since.UTC().Format("2006-01-02 15:04 MST"), ago(applyEvery)) + o.Headline = m + " is not applying its update" + o.Explanation = fmt.Sprintf("The provisioner on %s has not been able to write the files of %s for %s. The "+ + "details say why.", m, strings.Join(names, ", "), humanDuration(now.Sub(since))) + return o +} + +func orNoReason(s string) string { + if s == "" { + return "no reason given" + } + return s +} + // applyFailing is one machine's S21 condition. func applyFailing(d inventory.Doing, now time.Time) conditions.Observation { m := d.Node diff --git a/cmd/mesh-controller/batches.go b/cmd/mesh-controller/batches.go index c7065e80..e5d3ea50 100644 --- a/cmd/mesh-controller/batches.go +++ b/cmd/mesh-controller/batches.go @@ -688,7 +688,7 @@ func spelledOf(m inventory.BatchedMerge) string { func namedOf(m inventory.BatchedMerge, repository string) inventory.PlanMerge { k := announcedOf(m) return inventory.PlanMerge{Repository: repository, Commit: m.Commit, Number: k.Number, Title: k.Title, Moves: k.Moves, - Merged: m.Merged, Heard: m.Heard} + Head: k.HeadSHA, Merged: m.Merged, Heard: m.Heard} } // laterMerge says a was merged after b: by the forge's merge time, then by when each was heard. @@ -1000,10 +1000,15 @@ func planBatch(ctx context.Context, open *stores, batch *inventory.Plan, carry [ plan.Delivery.Merges = named plan.Delivery.Alone = batch.Delivery != nil && batch.Delivery.Alone plan.Delivery.Withheld = withheld + // **Left where it is, a module whose program did not change** (novox/hq issue 721): its check found it the same as + // the build the mesh holds. + for n, why := range leaveSameAsRunning(ctx, inv, &plan, moved, append(append([]inventory.Edge{}, edges...), ordered...)) { + unchanged[n] = why + } plan.Delivery.Unchanged = unchangedLeft(unchanged, plan.Modules) // **Its moments and its class** (novox/hq ADR 0282 decision 6): measured, never acted on. plan.Times = walkTimesAtCut(*batch, plan, entries, now) - if len(moved) == 0 { + if len(moved) == 0 || len(plan.Modules) == 0 { plan.State = inventory.PlanDone plan.Tiers = [][]string{} plan.Note = "moved nothing the mesh holds: nothing to walk" @@ -1044,7 +1049,7 @@ func planBatch(ctx context.Context, open *stores, batch *inventory.Plan, carry [ return nil } // Written and its first tier asked as one act on the plans (novox/hq issue 213), under the caller's hold. - if err := askTier(ctx, inv, &plan); err != nil { + if err := askTier(ctx, inv, &plan, plan.Tiers[plan.Tier]); err != nil { return err } if err := inv.SavePlan(ctx, &plan); err != nil { diff --git a/cmd/mesh-controller/batches_test.go b/cmd/mesh-controller/batches_test.go index 778abe73..9e6bba74 100644 --- a/cmd/mesh-controller/batches_test.go +++ b/cmd/mesh-controller/batches_test.go @@ -758,7 +758,7 @@ func TestTheCatchUpKeepsWhatACutMadeHistory(t *testing.T) { return entries, read, err } if err := catchUpOnMerges(ctx, time.Now(), unheardMerges{announcedList(lost), open.inventory}, catalogued, - (following{open}).SourceMoved, t.Logf); err != nil { + (following{open: open}).SourceMoved, t.Logf); err != nil { t.Fatal(err) } got, _ := open.inventory.PlanByID(ctx, w.ID) diff --git a/cmd/mesh-controller/build.go b/cmd/mesh-controller/build.go index ae337692..5a6802e1 100644 --- a/cmd/mesh-controller/build.go +++ b/cmd/mesh-controller/build.go @@ -852,6 +852,9 @@ type answers struct { sources map[string]inventory.Source // waiting is every machine not running what the mesh would send it. waiting []inventory.Machine + // recordedWaits is, per machine, every move of a recorded module that waits there for a person (novox/hq ADR + // 0377 §6): the machine is behind for these, and `push ` sends them. + recordedWaits map[string][]string // reported is every machine's last word beside when it was last sent a declaration — the // pair that answers "has it caught up", which waiting alone cannot (the sent digest is // recorded at send, not at apply). @@ -931,6 +934,34 @@ type answers struct { // // Routed through the artifact store as the network reaches it now (novox/hq 04-ISSUES/102): a // base is recorded by digest and path, and a build machine needs something it can pull. +// runningBy is, keyed "/", each artifact of the build the mesh has registered for its module, as a +// builder reaches the artifact store (novox/hq issue 721): what a check compares its builds with. Empty when it cannot +// be read, said, and then nothing is said unchanged. +func runningBy(ctx context.Context) map[string]string { + open, err := openStores(ctx) + if err != nil { + fmt.Fprintf(os.Stderr, "could not read what this mesh runs, so the check compares nothing: %v\n", err) + // empty-on-error: said above; a check given nothing to compare with says no module unchanged, and all move + return nil + } + defer open.Close() + running, err := open.inventory.RegisteredHeld(ctx) + if err != nil { + fmt.Fprintf(os.Stderr, "could not read what this mesh runs, so the check compares nothing: %v\n", err) + // empty-on-error: said above; a check given nothing to compare with says no module unchanged, and all move + return nil + } + address, err := whereABuilderReachesTheStore(ctx, open.inventory) + if err != nil || address == "" { + return running + } + routed := make(map[string]string, len(running)) + for key, reference := range running { + routed[key] = catalogue.Rerouted(reference, address) + } + return routed +} + func heldBy(ctx context.Context) map[string]string { open, err := openStores(ctx) if err != nil { diff --git a/cmd/mesh-controller/bus_users_closed_test.go b/cmd/mesh-controller/bus_users_closed_test.go new file mode 100644 index 00000000..689bd9f8 --- /dev/null +++ b/cmd/mesh-controller/bus_users_closed_test.go @@ -0,0 +1,92 @@ +package main + +import ( + "errors" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/link" +) + +// **Every user's slow consumer is S9's, not the controller's alone** (novox/hq ADR 0357, issue 270). On +// 2026-10-07 the bus closed two clients on the laptop as slow consumers and nothing raised a condition: +// the controller heard only its own connection. Said by the bus module and read back, a user the bus +// closes is one condition naming the module and its machine, raised at the first look and cleared an hour +// after the last closure. +func TestAUserTheBusClosedAsASlowConsumerIsOneConditionNamingItsModuleAndMachine(t *testing.T) { + closed := time.Date(2026, 10, 14, 12, 0, 0, 0, time.UTC) + now := closed.Add(20 * time.Second) + store := conditions.NewInMemory() + k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store, + Teller: &conditions.Told{}, Now: func() time.Time { return now }}) + t.Cleanup(func() { k.Close(t.Context()) }) + + advisories := link.BusClosures([]link.BusClosure{ + {User: "laptop.claude-code", CID: 10, Reason: "Slow Consumer (Write Deadline)", Opened: closed.Add(-time.Hour), + Closed: closed.Add(-10 * time.Minute)}, + {User: "laptop.claude-code", CID: 44, Reason: "Slow Consumer (Write Deadline)", Opened: closed.Add(-time.Minute), + Closed: closed}, + {User: "anchor.desk", CID: 7, Reason: "Maximum Message Payload Exceeded", Opened: closed.Add(-time.Hour), + Closed: closed}, + }, []string{link.AdvisorySlowConsumer, link.AdvisorySlowConsumer, link.AdvisoryConnectionClosed}, closed.Add(-time.Hour)) + look := func() { + t.Helper() + f := &signalFacts{now: now, host: "anchor", advisories: advisories} + if err := k.Reconcile(t.Context(), "S9", watchAdvisories(f)); err != nil { + t.Fatal(err) + } + } + look() + + slow, found, err := conditions.ReadOne(t.Context(), store, "bus.laptop.claude-code.slow-consumer") + if err != nil || !found { + t.Fatalf("no slow-consumer condition for the user: %v %v", found, err) + } + if slow.Subject.Machine != "laptop" || slow.Observations != 2 || !slow.LastObserved.Equal(closed) { + t.Fatalf("the slow consumer is said on %q, %d closure(s), last %s: %+v", slow.Subject.Machine, slow.Observations, + slow.LastObserved, slow) + } + if !strings.Contains(slow.Summary, "claude-code on laptop") || strings.Contains(slow.Summary, "laptop.claude-code") { + t.Fatalf("the summary does not name the module in words, or carries the dotted user: %q", slow.Summary) + } + if !strings.Contains(slow.Headline, "Claude-code on laptop") { + t.Fatalf("the headline is %q", slow.Headline) + } + cut, found, err := conditions.ReadOne(t.Context(), store, "bus.anchor.desk.connection-closed") + if err != nil || !found || cut.Subject.Machine != "anchor" || !strings.Contains(cut.Summary, "maximum message payload exceeded") { + t.Fatalf("the user cut off for breaking the bus's rules: %v %v %+v", found, err, cut) + } + + // An hour after the newest closure without another, both clear. + now = closed.Add(time.Hour + time.Minute) + look() + for _, key := range []string{"bus.laptop.claude-code.slow-consumer", "bus.anchor.desk.connection-closed"} { + if _, found, _ := k.Get(t.Context(), key); found { + t.Fatalf("%s is still open an hour after the last closure", key) + } + } +} + +// **A read of the bus module's events that fails does not blind S9** (review of #326): the controller's own +// signals are still said, and the failed read is a condition of its own. +func TestABusModuleReadThatFailsKeepsTheControllersOwnSignals(t *testing.T) { + at := time.Date(2026, 10, 14, 12, 0, 0, 0, time.UTC) + f := &signalFacts{now: at.Add(time.Minute), host: "anchor", + advisories: []link.Advisory{{Kind: link.AdvisorySlowConsumer, ID: "controller", Said: "the controller fell behind", + First: at, Last: at, Count: 1}}, + busFaultsErr: errors.New("nats: timeout")} + keys := map[string]bool{} + for _, o := range watchAdvisories(f) { + keys[o.Key()] = true + } + if !keys["bus.controller.slow-consumer"] || !keys["bus.bus-module.bus-faults-unread"] { + t.Fatalf("observed %v: the controller's own slow consumer and the unread events, both", keys) + } + for _, row := range signalsTable { + if row.Row == "S9" && row.needs(f) != nil { + t.Fatal("the S9 row went blind for a read of the bus module's events") + } + } +} diff --git a/cmd/mesh-controller/busobjects.go b/cmd/mesh-controller/busobjects.go index bb367a92..f56bdc47 100644 --- a/cmd/mesh-controller/busobjects.go +++ b/cmd/mesh-controller/busobjects.go @@ -49,10 +49,12 @@ func assertBusObjects(ctx context.Context, inv *inventory.Inventory, r broker.Ra // composed from, so a module the mesh grants a consumer's subjects has that consumer waiting. // Done on every raise, not only when a credential is issued: every module moved onto this bus // by the rollout was issued on the old one, and came up with nothing to bind to (2026-09-28). - consumers, err := moduleConsumers(ctx, inv) + consumers, records, err := moduleConsumersAndRecords(ctx, inv) if err != nil { return nil, err } + // And a grant of a restricted stream that composes nothing, said on every raise and send (novox/hq ADR 0376 §2). + sayWithheld(records, " ") // And the work queues of seats that name their caller or their kind, with each holder's worker // (novox/hq ADR 0259 §3): an ask queues until the router takes it, a channel's work until that kind // takes it. @@ -60,9 +62,19 @@ func assertBusObjects(ctx context.Context, inv *inventory.Inventory, r broker.Ra if err != nil { return nil, err } + // And every module's own stream (novox/hq ADR 0363), from the catalogue, before the consumers on it. + modStreams, err := inv.DeclaredStreams(ctx) + if err != nil { + return nil, err + } // Every one tried, and every failure named: one module's consumer the bus refuses is no reason // the modules after it in the list hear nothing (novox/hq issue 208, where this runs on each send). var failed []error + for _, s := range broker.ModuleStreams(modStreams) { + if err := r.EnsureStream(s); err != nil { + failed = append(failed, fmt.Errorf("the module stream %s: %w", s.Name, err)) + } + } for _, s := range trafficStreams { if err := r.EnsureStream(s); err != nil { failed = append(failed, fmt.Errorf("the work queue %s: %w", s.Name, err)) @@ -73,11 +85,30 @@ func assertBusObjects(ctx context.Context, inv *inventory.Inventory, r broker.Ra failed = append(failed, fmt.Errorf("the worker %s on %s: %w", c.Name, c.Stream, err)) } } + restricted := map[string]bool{} + for _, s := range modStreams { + restricted[s.Name] = s.Restricted + } for _, c := range consumers { - if err := r.EnsureConsumer(c.Consumer); err != nil { + if err := broker.EnsureHeard(r, c.Consumer, restricted[c.Consumer.Stream]); err != nil { failed = append(failed, fmt.Errorf("how %s on %s hears what it consumes: %w", c.Module, c.Node, err)) } } + // **And a consumer no longer granted loses what it reads through** (novox/hq ADR 0363 §6): every consumer + // on a module's stream the composition no longer asks for is removed, on this very send. + if p, prunes := r.(broker.Pruner); prunes && len(modStreams) > 0 { + var wanted []broker.Consumer + for _, c := range consumers { + wanted = append(wanted, c.Consumer) + } + removed, err := broker.PruneHeard(p, modStreams, wanted) + if err != nil { + failed = append(failed, err) + } + for _, gone := range removed { + fmt.Printf(" removed the consumer %s: its module is no longer granted that stream (novox/hq ADR 0363)\n", gone) + } + } if len(failed) > 0 { return nil, errors.Join(failed...) } @@ -136,15 +167,21 @@ func unassertedObservation(err error) conditions.Observation { // moduleConsumers is every module's durable consumer, from the records the user list is composed from. func moduleConsumers(ctx context.Context, inv *inventory.Inventory) ([]broker.ModuleConsumer, error) { + consumers, _, err := moduleConsumersAndRecords(ctx, inv) + return consumers, err +} + +// moduleConsumersAndRecords is moduleConsumers with the records they were composed from. +func moduleConsumersAndRecords(ctx context.Context, inv *inventory.Inventory) ([]broker.ModuleConsumer, broker.Records, error) { records, err := inv.BusRecords(ctx) if err != nil { - return nil, err + return nil, broker.Records{}, err } users, err := broker.Users(records) if err != nil { - return nil, err + return nil, broker.Records{}, err } - return broker.ConsumersOf(users), nil + return broker.ConsumersOf(users), records, nil } // seatTrafficObjects is the work queues and workers of seats that name their caller or their kind, from diff --git a/cmd/mesh-controller/changeplan.go b/cmd/mesh-controller/changeplan.go index b895614d..2b3ae219 100644 --- a/cmd/mesh-controller/changeplan.go +++ b/cmd/mesh-controller/changeplan.go @@ -73,6 +73,11 @@ func changePlanOf(repository, base, head string, r mergeReach, entries []invento "(issue 280)", busUpgradeNeeded, name, busUpgradeVerb, strings.Join(e.On, ", "))) case name == "nats" || catalogue.ProvidesBus(e.Manifest): p.Steps = append(p.Steps, "a planned bus step: the bus is upgraded by `bus upgrade`, never by an ordinary send") + case waits && len(e.On) > 0 && (u.From == catalogue.FromModule || u.From == catalogue.FromData): + // Judged per machine by the walk, against the build each was last sent (novox/hq ADR 0377). + p.Steps = append(p.Steps, fmt.Sprintf("%s waits for a person: its policy records (%s) — unless on a machine "+ + "its move recreates none of its containers and moves none of its data, which the walk judges and then sends", + name, orNone(u.From))) case waits && len(e.On) > 0: p.Steps = append(p.Steps, fmt.Sprintf("%s waits for a person: its policy records (%s)", name, orNone(u.From))) diff --git a/cmd/mesh-controller/check.go b/cmd/mesh-controller/check.go index cd7d57bc..0aeebb18 100644 --- a/cmd/mesh-controller/check.go +++ b/cmd/mesh-controller/check.go @@ -244,6 +244,12 @@ func moduleCheckFor(paths []string, longestMachine int, out io.Writer) error { checks = append(checks, fmt.Sprintf("%v by %s", r["id"], catalogue.HealthWords(h))) } } + // And its own runtime's, where it runs as its own account (novox/hq ADR 0366). + if own, has := catalogue.OwnRuntimeHealth(m); has { + if h, _, _ := catalogue.ReadHealth(own); h.Kind != "" { + checks = append(checks, fmt.Sprintf("%v by %s", own["id"], catalogue.HealthWords(h))) + } + } if len(checks) > 0 { fmt.Fprintf(out, ", ready: %s", strings.Join(checks, "; ")) } diff --git a/cmd/mesh-controller/check_builds_test.go b/cmd/mesh-controller/check_builds_test.go index d0f41d6a..0327b85a 100644 --- a/cmd/mesh-controller/check_builds_test.go +++ b/cmd/mesh-controller/check_builds_test.go @@ -125,6 +125,7 @@ func TestAWalksBuildSaysWhetherItReusedOrBuilt(t *testing.T) { // build as it judges one the walk built. A scratch reference is refused. Needs the store (MESH_TEST_POSTGRES), and // is named in testdata/store-tests. func TestAReusedCheckBuildIsRegisteredByItsDigestAndGatedAsABuiltOne(t *testing.T) { + withTheGate(t) // the first-machine gate's own path (novox/hq ADR 0360 keeps it for earlier walks) g := aGateMesh(t) ctx := t.Context() inv := g.open.inventory @@ -190,27 +191,90 @@ func TestAReusedCheckBuildIsRegisteredByItsDigestAndGatedAsABuiltOne(t *testing. } } -// A walk's leave names the builds its own pull requests' verdicts made, and none when they cannot be read: the -// builder takes nothing a verdict does not name (novox/hq ADR 0350 §2). +// A walk's leave names the builds its own pull requests' verdicts made at the head each merge merged — never an +// earlier head's — read once per walk, and none when they cannot be read: the builder takes nothing a verdict does +// not name (novox/hq ADR 0350 §2, issue 703). func TestAWalksLeaveNamesTheBuildsItsPullRequestsVerdictsMade(t *testing.T) { was := checkedBuildsOf - t.Cleanup(func() { checkedBuildsOf = was }) - var asked []link.PullRef - named := []link.CheckedBuild{{Pull: link.PullRef{Owner: "novox", Repo: "shop", Number: 7}, Check: "build-1", - Built: link.CheckBuilt{Module: "app", Key: "r1:1"}}} + forgetWalkVerdicts() + t.Cleanup(func() { checkedBuildsOf = was; forgetWalkVerdicts() }) + var asked [][]link.PullRef + pull := link.PullRef{Owner: "novox", Repo: "shop", Number: 7} + atHead := link.CheckedBuild{Pull: pull, Check: "build-2", Head: "bbbbbbbbbbbb", Built: link.CheckBuilt{Module: "app", Key: "r1:2"}} + earlier := link.CheckedBuild{Pull: pull, Check: "build-1", Head: "aaaaaaaaaaaa", Built: link.CheckBuilt{Module: "app", Key: "r1:1"}} checkedBuildsOf = func(_ context.Context, pulls []link.PullRef) ([]link.CheckedBuild, error) { - asked = pulls - return named, nil + asked = append(asked, pulls) + return []link.CheckedBuild{atHead, earlier}, nil } - pulls := []link.PullRef{{Owner: "novox", Repo: "shop", Number: 7}} - if got := reuseWithVerdicts(t.Context(), &link.ReuseAsk{Pulls: pulls}); !reflect.DeepEqual(got.Builds, named) || - !reflect.DeepEqual(asked, pulls) { - t.Fatalf("the leave names %+v, read for %+v", got.Builds, asked) + pulls := []link.PullRef{pull} + heads := map[link.PullRef]string{pull: "bbbbbbbbbbbb0123456789"} + for range 3 { + if got := reuseWithVerdicts(t.Context(), "plan-1", &link.ReuseAsk{Pulls: pulls}, heads); !reflect.DeepEqual(got.Builds, + []link.CheckedBuild{atHead}) { + t.Fatalf("the leave names %+v, not only the merged head's build", got.Builds) + } + } + if !reflect.DeepEqual(asked, [][]link.PullRef{pulls}) { + t.Fatalf("the verdicts were read %d times for one walk: %v", len(asked), asked) + } + // A merge whose head was not said takes nothing. + if got := reuseWithVerdicts(t.Context(), "plan-1", &link.ReuseAsk{Pulls: pulls}, nil); got.Builds != nil { + t.Fatalf("a merge of no said head names %+v", got.Builds) } checkedBuildsOf = func(context.Context, []link.PullRef) ([]link.CheckedBuild, error) { return nil, errors.New("no bus") } - if got := reuseWithVerdicts(t.Context(), &link.ReuseAsk{Pulls: pulls}); got.Builds != nil { + if got := reuseWithVerdicts(t.Context(), "plan-2", &link.ReuseAsk{Pulls: pulls}, heads); got.Builds != nil { t.Fatalf("verdicts that could not be read named %+v", got.Builds) } } + +// A check is handed only the builds earlier checks of its own pull request at its own head made: another pull +// request's build, or an earlier head's, is never kept (novox/hq issue 703). None when they cannot be read. +func TestACheckIsHandedOnlyItsOwnHeadsBuilds(t *testing.T) { + was := checkedBuildsOf + t.Cleanup(func() { checkedBuildsOf = was }) + pull := link.PullRef{Owner: "novox", Repo: "shop", Number: 7} + mine := link.CheckedBuild{Pull: pull, Check: "build-1", Head: "bbbbbbbbbbbb", Built: link.CheckBuilt{Module: "app", Key: "r1:1"}} + earlier := link.CheckedBuild{Pull: pull, Check: "build-0", Head: "aaaaaaaaaaaa", Built: link.CheckBuilt{Module: "app", Key: "r1:1"}} + another := link.CheckedBuild{Pull: link.PullRef{Owner: "novox", Repo: "shop", Number: 8}, Check: "build-9", + Head: "bbbbbbbbbbbb", Built: link.CheckBuilt{Module: "app", Key: "r1:1"}} + var asked [][]link.PullRef + checkedBuildsOf = func(_ context.Context, pulls []link.PullRef) ([]link.CheckedBuild, error) { + asked = append(asked, pulls) + return []link.CheckedBuild{mine, earlier, another}, nil + } + p := link.PullUpdated{Owner: "novox", Repo: "shop", Number: 7, Commit: "bbbbbbbbbbbb0123456789"} + builds := []link.CheckBuild{{Module: "app"}} + if got := namedForTheCheck(t.Context(), p, builds); !reflect.DeepEqual(got, []link.CheckedBuild{mine}) || + !reflect.DeepEqual(asked, [][]link.PullRef{{pull}}) { + t.Fatalf("the check is handed %+v, read for %v", got, asked) + } + checkedBuildsOf = func(context.Context, []link.PullRef) ([]link.CheckedBuild, error) { + return nil, errors.New("no bus") + } + if got := namedForTheCheck(t.Context(), p, builds); got != nil { + t.Fatalf("verdicts that could not be read named %+v", got) + } +} + +func forgetWalkVerdicts() { + walkVerdicts.Lock() + defer walkVerdicts.Unlock() + walkVerdicts.read = map[string]verdictsRead{} +} + +// A walk's merged heads are the forge's word for each merge; a merge announced without its head, or a pull request +// merged at two heads in one walk, has none, and its walk takes no check's build (novox/hq issue 703). +func TestAWalksMergedHeadsAreTheForgesWord(t *testing.T) { + merge := func(number int, head string) inventory.PlanMerge { + return inventory.PlanMerge{Repository: "novox/shop", Number: number, Head: head} + } + p := &inventory.Plan{Delivery: &inventory.PlanDelivery{Merges: []inventory.PlanMerge{ + merge(7, "aaaaaaaaaaaa1"), merge(8, ""), merge(9, "bbbbbbbbbbbb"), merge(9, "cccccccccccc")}}} + heads := mergedHeadsOf(p) + shop := func(n int) link.PullRef { return link.PullRef{Owner: "novox", Repo: "shop", Number: n} } + if heads[shop(7)] != "aaaaaaaaaaaa1" || heads[shop(8)] != "" || heads[shop(9)] != "" { + t.Fatalf("merged heads %v", heads) + } +} diff --git a/cmd/mesh-controller/checks.go b/cmd/mesh-controller/checks.go index b6243751..630352d5 100644 --- a/cmd/mesh-controller/checks.go +++ b/cmd/mesh-controller/checks.go @@ -3,6 +3,7 @@ package main import ( "context" "encoding/json" + "errors" "fmt" "path" "regexp" @@ -188,6 +189,8 @@ func (f following) PullUpdated(ctx context.Context, p link.PullUpdated) error { if err := ask.Ask(ctx, request); err != nil { return err } + // A new head ends the check of the old one (novox/hq issue 726). + endingSupersededChecks(ctx, seat, p, request.ID, f.oneShot) what := "its own merge-check.sh alone: " + noModuleTouched if scope.gated() { what = fmt.Sprintf("the gate over %s (a merge would build %d module(s) in %d tier(s))", @@ -248,31 +251,16 @@ func checkRequestFor(ctx context.Context, open *stores, p link.PullUpdated, scop if err != nil { return link.BuildRequest{}, err } - // Beside it, at what the mesh runs: each core repository by the module the mesh builds from it. - beside := map[string]link.CheckedOut{} - for _, e := range entries { - dir, core := coreModules[e.Manifest.Module] - if !core || e.Provided || e.Source.Repository == "" { - continue - } - url, err := clone(e.Source) - if err != nil { - return link.BuildRequest{}, err - } - refs := besideRefs(dir, current[e.Manifest.Module].Commit) - beside[dir] = link.CheckedOut{Repository: url, Ref: refs[dir]} - if dir == "mesh-controller" { - beside["mesh-controller-main"] = link.CheckedOut{Repository: url, Ref: refs["mesh-controller-main"]} - if e.Source.Seat != "" { - for _, sibling := range []string{"mesh-lab", "mesh-sdk"} { - if url, err := clone(inventory.Source{Seat: e.Source.Seat, Repository: siblingOf(e.Source.Repository, - sibling)}); err == nil { - beside[sibling] = link.CheckedOut{Repository: url, Ref: refs[sibling]} - } - } - } - } + // **A pull request into a release branch is judged by that branch** (novox/hq ADR 0358): asked of the forge once. + release, err := releaseBranchOf(ctx, p, scope, entries) + if err != nil { + return link.BuildRequest{}, err } + beside, err := besideOf(entries, func(module string) string { return current[module].Commit }, clone, release) + if err != nil { + return link.BuildRequest{}, err + } + builds := checkBuildsOf(p, scope, entries, clone) return link.BuildRequest{ ID: link.NewBuildID(time.Now()), Repository: repository, @@ -282,10 +270,188 @@ func checkRequestFor(ctx context.Context, open *stores, p link.PullUpdated, scop Source: sourceOnSeat(source), Check: &link.CheckRequest{Owner: p.Owner, Repo: p.Repo, Number: p.Number, Base: p.Base, Paths: p.Paths, Beside: beside, Modules: scope.Modules, Dependents: scope.Dependents, New: scope.New, - Manifests: scope.Manifests, Judge: scope.Judge, Builds: checkBuildsOf(p, scope, entries, clone)}, + Manifests: scope.Manifests, Judge: scope.Judge, Builds: builds, Named: namedForTheCheck(ctx, p, builds), + Release: release, Running: runningBy(ctx)}, }, nil } +// besideOf is what a check reads beside its own repository, by the directory it is found under: each core repository +// by the module the mesh builds from it, at besideRefs — or, for a pull request into a release branch (novox/hq ADR +// 0358 §3), at its branch of that name where it has one and at its trunk otherwise, the controller's main kept for +// its judge, and mesh-lab at its trunk whatever the base: its replays run with the container runtime's socket. running +// is the commit the mesh runs of a module. +func besideOf(entries []inventory.Entry, running func(module string) string, + clone func(inventory.Source) (string, error), release string) (map[string]link.CheckedOut, error) { + beside := map[string]link.CheckedOut{} + for _, e := range entries { + dir, core := coreModules[e.Manifest.Module] + if !core || e.Provided || e.Source.Repository == "" { + continue + } + url, err := clone(e.Source) + if err != nil { + return nil, err + } + refs := besideRefs(dir, running(e.Manifest.Module)) + at := func(dir, trunk string) link.CheckedOut { + if release == "" { + return link.CheckedOut{Repository: url, Ref: refs[dir]} + } + if trunk == "" { + trunk = "main" + } + return link.CheckedOut{Repository: url, Ref: release, Else: trunk} + } + beside[dir] = at(dir, followedBranch(e.Source.Ref)) + if dir == "mesh-controller" { + // The controller's main stays its main: it judges only a check the running controller predates. + beside["mesh-controller-main"] = link.CheckedOut{Repository: url, Ref: refs["mesh-controller-main"]} + if release != "" { + // mesh-controller has the release branch: that is what makes it one (ADR 0358 §1). + beside[dir] = link.CheckedOut{Repository: url, Ref: release} + } + if e.Source.Seat != "" { + for _, sibling := range []string{"mesh-lab", "mesh-sdk"} { + if url, err := clone(inventory.Source{Seat: e.Source.Seat, Repository: siblingOf(e.Source.Repository, + sibling)}); err == nil { + b := link.CheckedOut{Repository: url, Ref: refs[sibling]} + // **mesh-lab is read at its trunk, always** (ADR 0358): its replays run with the container + // runtime's socket, so they are only ever reviewed code — never a branch anyone may create. + if release != "" && sibling != "mesh-lab" { + b = link.CheckedOut{Repository: url, Ref: release, Else: "main"} + } + beside[sibling] = b + } + } + } + } + } + return beside, nil +} + +// releaseBranchOf is the release branch a pull request merges into (novox/hq ADR 0358 §1): its base, when that is not +// its repository's trunk and mesh-controller has a branch of that name; empty otherwise, and the pull request is +// checked as before it. **A base that is a trunk asks nothing**: the branch a module built from the repository follows +// (main when it records none), or mesh-controller's own trunk. Any other base is asked of the forge once, and an +// answer it cannot give is an error, never read as "no release branch". +func releaseBranchOf(ctx context.Context, p link.PullUpdated, scope checkScope, entries []inventory.Entry) (string, error) { + base := p.Base + if base == "" { + return "", nil + } + if scope.From != nil && base == trunkOf(scope.From.Source) { + return "", nil + } + var controller *inventory.Entry + for i, e := range entries { + if e.Manifest.Module == "mesh-controller" && !e.Provided && e.Source.Repository != "" { + controller = &entries[i] + break + } + } + if controller == nil || base == trunkOf(controller.Source) { + return "", nil + } + if !tryRef.MatchString(base) || strings.Contains(base, "..") { + return "", nil // no branch the forge could hold: checked as before + } + owner, repo := sourceOwner(controller.Source.Repository), sourceName(controller.Source.Repository) + if owner == "" || repo == "" { + return "", nil + } + has, err := forgeHasBranch(ctx, owner, repo, base) + if err != nil { + return "", fmt.Errorf("whether %s/%s#%d merges into a release branch cannot be told — the forge did not say "+ + "whether %s/%s has a branch %s (novox/hq ADR 0358): %w", p.Owner, p.Repo, p.Number, owner, repo, base, err) + } + if !has { + return "", nil + } + return base, nil +} + +// trunkOf is the branch a module's source follows: main when it records none, or records a commit. +func trunkOf(s inventory.Source) string { + if t := strings.TrimSpace(followedBranch(s.Ref)); t != "" { + return t + } + return "main" +} + +// sourceName is the name of a recorded repository, a path on the git seat or a URL: novox/mesh-host → mesh-host. +func sourceName(repository string) string { + parts := strings.Split(strings.Trim(strings.TrimSuffix(repository, ".git"), "/"), "/") + if len(parts) < 2 { + return "" + } + return parts[len(parts)-1] +} + +// forgeHasBranch asks the forge, through its module's tools on the bus, whether a repository has a branch. A +// variable so a test needs no forge. +var forgeHasBranch = func(ctx context.Context, owner, repo, branch string) (bool, error) { + js, err := aBus() + if err != nil { + return false, err + } + defer js.Close() + raw, _ := json.Marshal(map[string]any{"path": "/repos/" + owner + "/" + repo + "/branches/" + branch}) + answer, err := link.Ask(ctx, link.OverNATS{Conn: js.Conn()}, "gitea", "gitea_api", raw, 30*time.Second) + if err != nil { + return false, err + } + return branchAnswered(answer, branch) +} + +// branchAnswered reads the forge's answer about a branch: a 404 is "no such branch"; an answer naming the branch is +// "it has it"; anything else — another error, an unreadable answer, one naming no branch or another — is an error, +// never read as "no release branch". +func branchAnswered(answer link.Answer, branch string) (bool, error) { + if answer.Error != "" { + if strings.Contains(answer.Error, " 404 ") { + return false, nil + } + return false, errors.New(answer.Error) + } + var found struct { + Result struct { + Name string `json:"name"` + } `json:"result"` + } + if err := json.Unmarshal(answer.Result, &found); err != nil { + return false, fmt.Errorf("the forge's answer about branch %s cannot be read: %w", branch, err) + } + if found.Result.Name != branch { + return false, fmt.Errorf("the forge answered about branch %s naming %q", branch, found.Result.Name) + } + return true, nil +} + +// namedForTheCheck are the builds earlier checks of the same pull request at the same head named, read once for the +// check (novox/hq issue 703): asked again because only the trunk moved, the check keeps what they built of a key +// still the same. Another pull request's build, or one of an earlier head, is never kept. None when they cannot be +// read, and the check builds, as before. +func namedForTheCheck(ctx context.Context, p link.PullUpdated, builds []link.CheckBuild) []link.CheckedBuild { + if len(builds) == 0 || p.Commit == "" { + return nil + } + pull := link.PullRef{Owner: p.Owner, Repo: p.Repo, Number: p.Number} + read, err := checkedBuildsOf(ctx, []link.PullRef{pull}) + if err != nil { + fmt.Printf(" the earlier checks' builds of %s/%s#%d could not be read, so the check builds: %v\n", + p.Owner, p.Repo, p.Number, err) + // empty-on-error: said above; a check handed no named build builds every module, as before issue 703 + return nil + } + var named []link.CheckedBuild + for _, b := range read { + if b.Pull == pull && link.SameHead(b.Head, p.Commit) { + named = append(named, b) + } + } + return named +} + // checkBuildsOf are the modules a merge of the change would build, in the walk's order, as the check builds them // for the walk to take (novox/hq ADR 0350 §1): a module built from the repository checked, at the branch it merges // into, from the merge the check makes; any other — a module standing on one — from its own repository at the @@ -432,6 +598,8 @@ func checkedOf(result link.BuildResult) link.Checked { // repositories read beside it. c.Trunk, c.Beside, c.Took = result.Check.Trunk, result.Check.Beside, result.Check.Took c.Built = result.Check.Built + c.Release, c.JudgedBy, c.BesideAt = result.Check.Release, result.Check.JudgedBy, result.Check.BesideAt + c.Validator = result.Check.Validator if c.Gate == nil { // A build seat from before the layers: its verdict is the gate's. c.Gate = &link.CheckLayer{Verdict: c.Verdict, Summary: c.Summary} diff --git a/cmd/mesh-controller/config_move.go b/cmd/mesh-controller/config_move.go new file mode 100644 index 00000000..1b65621d --- /dev/null +++ b/cmd/mesh-controller/config_move.go @@ -0,0 +1,616 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "os" + "reflect" + "sort" + "strings" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +// **The move of config** (novox/hq ADR 0348 §9 and its consequences, design 57 K5, issue 664): every config value a +// settings layer holds is set in mesh-config, one of the moves of data a release runs once after its delivery, after +// the move of placement. A module's layer on a node becomes its own value on that node (`module...`). +// A whole-mesh layer has no module value for the whole mesh (ADR 0348 §3): its value becomes mesh config for the +// predefined key it falls back to where that module alone falls back to the key (claude-code's managed settings +// become `agent-managed-settings`), and otherwise the module's own value on each node it is assigned to, laid under +// that node's layer as composition lays it. A key a layer still says by its file's name (nodered's `timeZone`) is +// set under its config key (`time-zone`, issue 751). +// +// **Every node composes byte for byte as before.** The move sets the values, reads every module's resolved values +// back through mesh-config and compares them with what the layers lay today; only when they agree does it empty the +// layers it moved (each kept whole in its history first) and turn on the switch (inventory.FinishConfigMove), after +// which the controller reads mesh-config where it read the layers. A module still in the settings form keeps its +// layers and is said so. Anything that cannot move as it composes refuses the whole move, by name, before anything +// is set. Asked again, it sets nothing that is set already. + +// configLevel is where mesh-config keeps a value: mesh config, node config, or a module's own value on a node. +type configLevel struct { + Level string `json:"level"` + Node string `json:"node,omitempty"` + Module string `json:"module,omitempty"` +} + +func (l configLevel) valueKey(key string) string { + switch l.Level { + case "node": + return "node." + l.Node + "." + key + case "module": + return "module." + l.Node + "." + l.Module + "." + key + } + return "mesh." + key +} + +func (l configLevel) String() string { + switch l.Level { + case "node": + return "node config on " + l.Node + case "module": + return l.Module + "'s own value on " + l.Node + } + return "mesh config" +} + +// configSet is one value the move sets in mesh-config, and the layers it comes from. +type configSet struct { + configLevel + Key string `json:"key"` + Value any `json:"value"` + Trusted bool `json:"trusted,omitempty"` + // From is the layers it comes from, as ` on : `. + From []string `json:"from"` + // Already is true when mesh-config holds this value at this level already: nothing is set. + Already bool `json:"already,omitempty"` +} + +// configMoveAnswer is what the move of config says it did, or would do. +type configMoveAnswer struct { + // DryRun is true when nothing was set: asked without --confirm. + DryRun bool `json:"dry-run"` + Sets []configSet `json:"sets"` + // Moved is each layer whose values the move sets, emptied once they read back. + Moved []string `json:"moved"` + // Stays is each layer of a module still in the settings form: kept, and composed from as before. + Stays []string `json:"stays,omitempty"` + // Unread is each layer that composes nothing (its module is not assigned where it says): kept in the history, + // emptied, and set nowhere. + Unread []string `json:"unread,omitempty"` + // Problems is why the move refuses, each by name; nothing is set while there is one. + Problems []string `json:"problems,omitempty"` + // Switched is true once the controller reads mesh-config where it read the layers. + Switched bool `json:"switched"` +} + +// configMovePlan is the move worked out from the layers, before anything is asked of mesh-config. +type configMovePlan struct { + answer configMoveAnswer + // moved is the layers the move empties once the values read back. + moved []inventory.LayerToMove + // expected is, for each module the move reaches, keyed `.`, every value composition lays over + // its files today: what mesh-config must resolve for it from mesh config, node config or its own value. + expected map[string]map[string]any + // modules is each module in the config form, by the nodes it is assigned to: whose resolved values are read back. + modules map[string][]string +} + +func whereOf(node string) string { + if node == "" { + return "the whole mesh" + } + return node +} + +// planConfigMove works out the move from every layer that holds a value, the catalogue, and where each module is +// assigned. +func planConfigMove(layers []inventory.LayerToMove, shelf map[string]catalogue.Manifest, + on map[string][]string) configMovePlan { + p := configMovePlan{answer: configMoveAnswer{Sets: []configSet{}, Moved: []string{}}, + expected: map[string]map[string]any{}, modules: map[string][]string{}} + problem := func(format string, a ...any) { + p.answer.Problems = append(p.answer.Problems, fmt.Sprintf(format, a...)) + } + for name, m := range shelf { + if m.DeclaresConfig() { + p.modules[name] = on[name] + } + } + + byModule := map[string]*layersOfModule{} + var names []string + for _, l := range layers { + said := fmt.Sprintf("%s on %s", l.Module, whereOf(l.Node)) + m, known := shelf[l.Module] + if !known || !m.DeclaresConfig() { + p.answer.Stays = append(p.answer.Stays, said+" (its module is in the settings form)") + continue + } + if _, placement := catalogue.SplitPlacement(l.Values); len(placement) > 0 { + problem("%s still holds placement (%s): run move-placement first", said, strings.Join(movedKeys(placement), ", ")) + continue + } + assigned := on[l.Module] + if (l.Node == "" && len(assigned) == 0) || (l.Node != "" && !containsString(assigned, l.Node)) { + p.answer.Unread = append(p.answer.Unread, said+" (the module is not assigned there, so it composes nothing)") + p.moved = append(p.moved, l) + continue + } + // The layer's keys as config keys: a key the file names otherwise is set under its config key. + as := catalogue.TakenAs(m) + values := map[string]any{} + from := map[string]string{} + for _, k := range movedKeys(l.Values) { + key := k + if _, declared := m.Settings[k]; !declared { + if renamed, ok := as[k]; ok { + key = renamed + } else { + problem("%s says %q, which %s does not declare: declare it, or clear it from the layer", said, k, l.Module) + continue + } + } + if _, both := values[key]; both { + // The layer says the key and the file's name for it: the config key wins, as composition has it (issue 751). + if key != k { + continue + } + } + values[key] = l.Values[k] + from[key] = said + ": " + k + } + om := byModule[l.Module] + if om == nil { + om = &layersOfModule{nodes: map[string]map[string]any{}, saidAt: map[string]map[string]string{}} + byModule[l.Module] = om + names = append(names, l.Module) + } + om.saidAt[l.Node] = from + if l.Node == "" { + om.mesh = values + } else { + om.nodes[l.Node] = values + } + p.moved = append(p.moved, l) + p.answer.Moved = append(p.answer.Moved, said) + } + sort.Strings(names) + + // Which predefined key each module falls back to, so a whole-mesh value goes to mesh config only where no other + // module's value moves with it. + fallingBack := map[string][]string{} + for name, m := range shelf { + for _, d := range m.Settings { + if d.Predefined != "" && !containsString(fallingBack[d.Predefined], name) { + fallingBack[d.Predefined] = append(fallingBack[d.Predefined], name) + } + } + } + + for _, name := range names { + m, om := shelf[name], byModule[name] + asked := catalogue.AskedKeys(m) + keys := map[string]bool{} + for k := range om.mesh { + keys[k] = true + } + for _, values := range om.nodes { + for k := range values { + keys[k] = true + } + } + for _, key := range movedKeys(keys) { + d := m.Settings[key] + meshValue, meshSays := om.mesh[key] + alone := d.Predefined != "" && len(fallingBack[d.Predefined]) == 1 + if meshSays && alone { + p.set(configLevel{Level: "mesh"}, d.Predefined, meshValue, d, om.said("", key)) + } + for _, node := range on[name] { + nodeValue, nodeSays := om.nodes[node][key] + value := meshValue + switch { + case nodeSays && meshSays: + _, under := meshValue.(map[string]any) + _, over := nodeValue.(map[string]any) + if under && over && asked[key] { + problem("%s on %s lays an object for %q over the whole mesh's, and a placeholder takes it whole "+ + "where its file merges the two: say the value on %s whole, then move", name, node, key, node) + continue + } + value = catalogue.LaidOver(meshValue, nodeValue) + case nodeSays: + value = nodeValue + case !meshSays: + continue + } + p.expect(node, name, key, value) + if nodeSays || !alone { + saidBy := om.said(node, key) + if meshSays { + saidBy = append(om.said("", key), saidBy...) + } + p.set(configLevel{Level: "module", Node: node, Module: name}, key, value, d, saidBy) + } + } + } + } + sort.Strings(p.answer.Problems) + return p +} + +// layersOfModule is one module's layers as the move reads them: its whole-mesh layer's config values, each node's, +// and where each key was said, by node ("" for the whole mesh). +type layersOfModule struct { + mesh map[string]any + nodes map[string]map[string]any + saidAt map[string]map[string]string +} + +func (om *layersOfModule) said(node, key string) []string { + if s, ok := om.saidAt[node][key]; ok { + return []string{s} + } + return nil +} + +// set is one value the move sets, judged against its declaration as mesh-config will judge it. +func (p *configMovePlan) set(level configLevel, key string, value any, d catalogue.SettingDeclaration, from []string) { + p.answer.Sets = append(p.answer.Sets, configSet{configLevel: level, Key: key, Value: value, Trusted: d.Trusted, + From: from}) + for _, wrong := range catalogue.ValueProblems(d, value) { + p.answer.Problems = append(p.answer.Problems, fmt.Sprintf("%s at %s: %s", key, level, wrong)) + } +} + +func (p *configMovePlan) expect(node, module, key string, value any) { + k := node + "." + module + if p.expected[k] == nil { + p.expected[k] = map[string]any{} + } + p.expected[k][key] = value +} + +func movedKeys[V any](m map[string]V) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} + +// configService is mesh-config as the move asks it: what it holds at each level, what it resolved, and a value set +// as the controller carries it from its terminal. +type configService interface { + // Values is every value set, by its key in mesh-config_values (`mesh.`, `node..`, + // `module...`). + Values(ctx context.Context) (map[string]json.RawMessage, error) + // Resolved is every resolved value. + Resolved(ctx context.Context) ([]resolvedEntry, error) + // Set sets one value at its level, by whom and why. + Set(ctx context.Context, level configLevel, key string, value any, by, why string) error +} + +// runConfigMove is the move against a store and mesh-config: a dry run says what it would set. +func runConfigMove(ctx context.Context, inv *inventory.Inventory, config configService, apply bool, by, + why string) (configMoveAnswer, error) { + layers, holding, err := inv.LayersToMove(ctx) + if err != nil { + return configMoveAnswer{}, err + } + entries, err := inv.Catalogued(ctx) + if err != nil { + return configMoveAnswer{}, err + } + shelf := map[string]catalogue.Manifest{} + on := map[string][]string{} + for _, e := range entries { + shelf[e.Manifest.Module] = e.Manifest + on[e.Manifest.Module] = e.On + } + p := planConfigMove(layers, shelf, on) + if len(holding) > 0 && len(p.answer.Problems) == 0 { + p.answer.Problems = append(p.answer.Problems, "a layer still holds placement ("+strings.Join(holding, "; ")+ + "): run move-placement first") + } + moved, err := inv.ConfigMoved(ctx) + if err != nil { + return configMoveAnswer{}, err + } + p.answer.Switched = moved.Moved + if config == nil { + return p.answer, errors.New("mesh-config cannot be asked, so nothing was moved") + } + held, err := config.Values(ctx) + if err != nil { + return p.answer, fmt.Errorf("what mesh-config holds cannot be read, so nothing was moved: %w", err) + } + for i := range p.answer.Sets { + s := &p.answer.Sets[i] + if was, ok := held[s.valueKey(s.Key)]; ok && sameJSON(was, s.Value) { + s.Already = true + } + } + p.answer.DryRun = !apply + if !apply || len(p.answer.Problems) > 0 { + if apply { + return p.answer, fmt.Errorf("the move of config refuses, and nothing was set: %s", + strings.Join(p.answer.Problems, "; ")) + } + return p.answer, nil + } + if len(p.moved) == 0 && moved.Moved { + return p.answer, nil + } + for _, s := range p.answer.Sets { + if s.Already { + continue + } + said := "moved from the settings layer of " + strings.Join(s.From, " and ") + " (issue 664): " + why + if err := config.Set(ctx, s.configLevel, s.Key, s.Value, by, said); err != nil { + return p.answer, fmt.Errorf("%s at %s was not set, and no layer was emptied; ask the move again: %w", + s.Key, s.configLevel, err) + } + } + // Read back through mesh-config before anything points at it (design 57): every module the move reaches, and + // every other in the config form, must resolve exactly what its layers lay today. + if differs, err := readsBack(ctx, config, p); err != nil { + return p.answer, err + } else if len(differs) > 0 { + return p.answer, fmt.Errorf("mesh-config does not resolve what the layers lay today, so no layer was emptied and "+ + "the controller still composes from the layers: %s", strings.Join(differs, "; ")) + } + if err := inv.FinishConfigMove(ctx, p.moved, by, why); err != nil { + return p.answer, err + } + p.answer.Switched = true + return p.answer, nil +} + +// readsBack compares what mesh-config resolved with what the layers lay, for every module in the config form on +// every node it is assigned to, and says each difference. A value the layers do not lay and mesh-config resolves from +// a level would be laid once the switch is on: a difference too. +func readsBack(ctx context.Context, config configService, p configMovePlan) ([]string, error) { + var resolved []resolvedEntry + var err error + // mesh-config writes the resolved values with each set; a pass that missed one is put right by its next + // resolution, so a difference is read again for a little while before it is said. + for wait := 0; ; wait++ { + if resolved, err = config.Resolved(ctx); err != nil { + return nil, fmt.Errorf("mesh-config's resolved values cannot be read back, so no layer was emptied: %w", err) + } + differs, err := differences(resolved, p) + if err != nil || len(differs) == 0 || wait >= readBackTries { + return differs, err + } + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(readBackEvery): + } + } +} + +// How often, and how many times, the move reads mesh-config's resolved values back before it says they differ. +var ( + readBackEvery = 2 * time.Second + readBackTries = 15 +) + +func differences(resolved []resolvedEntry, p configMovePlan) ([]string, error) { + var differs []string + for _, module := range movedKeys(p.modules) { + for _, node := range p.modules[module] { + laid, err := configLaid(resolved, node, module) + if err != nil { + return nil, err + } + want := p.expected[node+"."+module] + if want == nil { + want = map[string]any{} + } + for _, k := range movedKeys(mergedKeys(laid, want)) { + got, has := laid[k] + w, wants := want[k] + switch { + case !wants: + differs = append(differs, fmt.Sprintf("%s on %s resolves %s from a level no layer says; clear it "+ + "there, or set it in a layer first", module, node, k)) + case !has: + differs = append(differs, fmt.Sprintf("%s on %s does not resolve %s from the value moved", module, node, k)) + case !reflect.DeepEqual(normalJSON(got), normalJSON(w)): + differs = append(differs, fmt.Sprintf("%s on %s resolves %s as %s, and its layers lay %s", module, + node, k, compactJSON(got), compactJSON(w))) + } + } + } + } + return differs, nil +} + +func mergedKeys(a, b map[string]any) map[string]bool { + out := map[string]bool{} + for k := range a { + out[k] = true + } + for k := range b { + out[k] = true + } + return out +} + +func normalJSON(v any) any { + raw, _ := json.Marshal(v) + var out any + _ = json.Unmarshal(raw, &out) + return out +} + +func compactJSON(v any) string { + raw, _ := json.Marshal(v) + return string(raw) +} + +func sameJSON(raw json.RawMessage, v any) bool { + var held any + if err := json.Unmarshal(raw, &held); err != nil { + return false + } + return reflect.DeepEqual(held, normalJSON(v)) +} + +// busConfig is mesh-config on this controller's bus: its buckets read directly, a value set through its verb in +// the controller's own name, so mesh-config takes a trusted value as given at the controller's terminal (ADR 0348 §8). +type busConfig struct{} + +func (busConfig) Values(ctx context.Context) (map[string]json.RawMessage, error) { + out := map[string]json.RawMessage{} + err := onTheBus(func(conn *nats.Conn) error { + js, err := jetstream.New(conn) + if err != nil { + return err + } + ctx, cancel := context.WithTimeout(ctx, 20*time.Second) + defer cancel() + kv, err := js.KeyValue(ctx, broker.BucketName("mesh-config", "values")) + if err != nil { + return err + } + w, err := kv.WatchAll(ctx, jetstream.IgnoreDeletes()) + if err != nil { + return err + } + defer func() { _ = w.Stop() }() + for e := range w.Updates() { + if e == nil { + return nil + } + var v struct { + Value json.RawMessage `json:"value"` + } + if err := json.Unmarshal(e.Value(), &v); err != nil { + return fmt.Errorf("mesh-config holds %q, which is not a value: %w", e.Key(), err) + } + out[e.Key()] = v.Value + } + return errors.New("mesh-config's values stopped answering") + }) + return out, err +} + +func (busConfig) Resolved(ctx context.Context) ([]resolvedEntry, error) { return readResolved(ctx) } + +func (busConfig) Set(ctx context.Context, level configLevel, key string, value any, by, why string) error { + args := map[string]any{"action": "set", "key": key, "value": value, "why": why, "by": by} + if level.Node != "" { + args["node"] = level.Node + } + if level.Module != "" { + args["module"] = level.Module + } + return onTheBus(func(conn *nats.Conn) error { + answer, err := link.AskMeshSeatToolAs(ctx, conn, broker.ControllerName, "mesh-config", "config", args, 30*time.Second) + if err != nil { + return err + } + if answer.Error != "" { + return errors.New(answer.Error) + } + return nil + }) +} + +// moveConfigCommand is `move-config`: the move of config (issue 664). +func moveConfigCommand(ctx context.Context, args []string) error { + set := flag.NewFlagSet("move-config", flag.ContinueOnError) + asJSON := set.Bool("json", false, "answer as JSON") + confirm := set.Bool("confirm", false, "set the values and switch, rather than only say what would move") + f := addHandActFlags(set) + positionals, err := parseAround(set, args) + if err != nil { + return err + } + if len(positionals) != 0 { + return errors.New("move-config [--json] [--confirm --why ]") + } + if !*confirm && f.given() { + return errors.New("move-config takes --why only with --confirm: without it nothing is moved. Nothing was done") + } + if *confirm { + if err := f.require("move-config"); err != nil { + return err + } + } + open, err := openStores(ctx) + if err != nil { + return err + } + defer open.Close() + if *confirm { + f.record(ctx, "move-config", nil) + } + why := "" + if *confirm { + why = strings.TrimSpace(*f.why) + } + answer, err := runConfigMove(ctx, open.inventory, busConfig{}, *confirm, "the move of config, run by "+link.Caller(), why) + if *asJSON { + encoder := json.NewEncoder(os.Stdout) + encoder.SetIndent("", " ") + if encErr := encoder.Encode(answer); encErr != nil { + return encErr + } + return err + } + fmt.Print(describeConfigMove(answer)) + return err +} + +// describeConfigMove says the answer as a person reads it. +func describeConfigMove(a configMoveAnswer) string { + var b strings.Builder + verb := "set" + if a.DryRun { + verb = "would set" + } + for _, s := range a.Sets { + state := verb + if s.Already { + state = "holds already" + } + fmt.Fprintf(&b, "%s %s = %s at %s (from %s)\n", state, s.Key, compactJSON(s.Value), s.configLevel, + strings.Join(s.From, " and ")) + } + for _, s := range a.Unread { + fmt.Fprintf(&b, "emptied, set nowhere: %s\n", s) + } + for _, s := range a.Stays { + fmt.Fprintf(&b, "stays: %s\n", s) + } + for _, s := range a.Problems { + fmt.Fprintf(&b, "refused: %s\n", s) + } + switch { + case len(a.Problems) > 0: + b.WriteString("nothing was set: the move refuses until each is put right\n") + case a.DryRun: + fmt.Fprintf(&b, "%d layer(s), %d value(s); nothing was set — with --confirm --why they are\n", + len(a.Moved), len(a.Sets)) + case a.Switched: + fmt.Fprintf(&b, "%d layer(s) moved into mesh-config, each kept whole in its history first; the controller "+ + "reads mesh-config where it read them\n", len(a.Moved)) + } + return b.String() +} diff --git a/cmd/mesh-controller/config_moved_test.go b/cmd/mesh-controller/config_moved_test.go new file mode 100644 index 00000000..75657f8b --- /dev/null +++ b/cmd/mesh-controller/config_moved_test.go @@ -0,0 +1,785 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "reflect" + "sort" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + snapshot "github.com/novox/mesh-controller/internal/facts" + "github.com/novox/mesh-controller/internal/link" + "github.com/novox/mesh-controller/internal/overlay" +) + +// configuredMesh is every live settings layer (issue 664) — read with the settings verb on 2026-10-15, on the four +// machines as anchor, laptop, workstation and desktop, the whole mesh's with no node; personal values stood in for, +// and the placement words left out, which move-placement moves first — with each module in the config form shaped +// as its definition on the feature branch: its config, its files (templates inlined) and directories, placeholders +// of kinds this test's mesh does not hold stood in for. A module still in the settings form (the media catalogue's, +// and kernel) stands in as a mergeable file that takes every key. `on` is where each is assigned, as live. +type configuredMesh struct { + Modules []json.RawMessage `json:"modules"` + Layers []configuredLayer `json:"layers"` + On map[string][]string `json:"on"` + // Predefined is the catalogue's list of predefined keys, which a module falls back to. + Predefined json.RawMessage `json:"predefined"` +} + +type configuredLayer struct { + Node string `json:"node"` + Module string `json:"module"` + Values map[string]any `json:"values"` +} + +var fourMachines = []string{"anchor", "laptop", "workstation", "desktop"} + +// aConfiguredMesh is aMesh with two machines more, every module of the configured mesh registered and assigned, and +// its layers held as the store holds them once move-placement has run. +func aConfiguredMesh(t *testing.T) (*stores, configuredMesh, map[string]map[string]any) { + t.Helper() + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + for i, name := range []string{"workstation", "desktop"} { + record, err := inv.AddNode(ctx, name) + if err != nil { + t.Fatal(err) + } + if err := inv.SetPlace(ctx, name, name+".example:51820", "here", false, fmt.Sprintf("10.77.0.%d", i+3)); err != nil { + t.Fatal(err) + } + if err := inv.RecordProfile(ctx, record.ID, map[string]any{"capabilities": []any{ + map[string]any{"name": "container-runtime", "present": true}, + map[string]any{"name": "systemd", "present": true}}}); err != nil { + t.Fatal(err) + } + if err := inv.RecordSealingKey(ctx, record.ID, aPublicKey(t)); err != nil { + t.Fatal(err) + } + if err := inv.RecordOverlayKey(ctx, record.ID, aPublicKey(t)); err != nil { + t.Fatal(err) + } + if _, err := inv.Assign(ctx, name, overlay.Name); err != nil { + t.Fatal(err) + } + } + raw, err := os.ReadFile(filepath.Join("testdata", "config-moved", "mesh.json")) + if err != nil { + t.Fatal(err) + } + var w configuredMesh + if err := json.Unmarshal(raw, &w); err != nil { + t.Fatal(err) + } + list, err := catalogue.ParsePredefined(w.Predefined) + if err != nil { + t.Fatal(err) + } + catalogue.UsePredefined(list) + served := map[string]map[string]any{} + for _, text := range w.Modules { + var shape map[string]json.RawMessage + if err := json.Unmarshal(text, &shape); err != nil { + t.Fatal(err) + } + var name string + _ = json.Unmarshal(shape["module"], &name) + if s, ok := shape["serves-checked"]; ok { + var serves map[string]any + if err := json.Unmarshal(s, &serves); err != nil { + t.Fatal(err) + } + served[name] = serves + delete(shape, "serves-checked") + } + text, err := json.Marshal(shape) + if err != nil { + t.Fatal(err) + } + m, err := catalogue.ParseManifest(text) + if err != nil { + t.Fatalf("%s: %v", name, err) + } + register(t, open, m) + } + for _, l := range w.Layers { + values, err := json.Marshal(l.Values) + if err != nil { + t.Fatal(err) + } + if l.Node == "" { + inv.ExecForTest(t, `insert into settings (node, module, values) values (null, $1, $2)`, l.Module, string(values)) + continue + } + inv.ExecForTest(t, `insert into settings (node, module, values) select id, $2, $3 from node where name = $1`, + l.Node, l.Module, string(values)) + } + for _, module := range movedKeys(w.On) { + for _, node := range w.On[module] { + if _, err := assign(ctx, open, node, module); err != nil { + t.Fatalf("assign %s %s: %v", node, module, err) + } + } + } + return open, w, served +} + +// standInMeshConfig is mesh-config as this test holds it: each value at its level, and resolved from the catalogue +// the store holds as mesh-config resolves (novox/hq ADR 0348 §2): the module's own value on the node, else node +// config then mesh config for the predefined key it falls back to, else its default; a required value nothing sets is +// unset. +type standInMeshConfig struct { + open *stores + values map[string]any + sets []string + // asked is each call of its verb: the action and the level's key. + asked []string +} + +func (c *standInMeshConfig) Values(context.Context) (map[string]json.RawMessage, error) { + out := map[string]json.RawMessage{} + for k, v := range c.values { + raw, err := json.Marshal(v) + if err != nil { + return nil, err + } + out[k] = raw + } + return out, nil +} + +func (c *standInMeshConfig) Set(_ context.Context, level configLevel, key string, value any, by, why string) error { + if by == "" || why == "" { + return fmt.Errorf("%s at %s is set without who or why", key, level) + } + c.values[level.valueKey(key)] = normalJSON(value) + c.sets = append(c.sets, level.valueKey(key)) + return nil +} + +func (c *standInMeshConfig) Resolved(ctx context.Context) ([]resolvedEntry, error) { + entries, err := c.open.inventory.Catalogued(ctx) + if err != nil { + return nil, err + } + var out []resolvedEntry + for _, e := range entries { + m := e.Manifest + if !m.DeclaresConfig() { + continue + } + for _, node := range e.On { + for _, key := range movedKeys(m.Settings) { + d := m.Settings[key] + r := resolvedEntry{Node: node, Module: m.Module, Key: key, Source: "default"} + value, has := c.values["module."+node+"."+m.Module+"."+key] + switch { + case has: + r.Source = "module" + case d.Predefined != "" && c.values["node."+node+"."+d.Predefined] != nil: + value, r.Source = c.values["node."+node+"."+d.Predefined], "node" + case d.Predefined != "" && c.values["mesh."+d.Predefined] != nil: + value, r.Source = c.values["mesh."+d.Predefined], "mesh" + case d.Required: + r.Source = "unset" + default: + value = d.Default + } + if r.Source != "unset" { + if r.Value, err = json.Marshal(value); err != nil { + return nil, err + } + } + out = append(out, r) + } + } + } + return out, nil +} + +// laid is the stand-in as composition reads it once the switch is on (inventory.ConfigValues). +func (c *standInMeshConfig) laid(ctx context.Context, node, module string) (map[string]any, error) { + resolved, err := c.Resolved(ctx) + if err != nil { + return nil, err + } + return configLaid(resolved, node, module) +} + +// snapshotFacts is the facts snapshot as gatherFacts gives it. +type snapshotFacts = snapshot.Facts + +// toldEveryMachine is what each of the four machines is told — every resource and the filter's rules — and what +// each module that serves something serves, settled from its layers, as JSON a person can diff. +func toldEveryMachine(t *testing.T, open *stores, served map[string]map[string]any, on map[string][]string) map[string]string { + t.Helper() + ctx := t.Context() + gens, err := generators(ctx, open) + if err != nil { + t.Fatal(err) + } + shelf, err := open.inventory.Catalogue(ctx) + if err != nil { + t.Fatal(err) + } + told := map[string]string{} + for _, node := range fourMachines { + plan, settings, err := planFor(ctx, open, node) + if err != nil { + t.Fatalf("%s: %v", node, err) + } + with, _, err := renderingFor(ctx, open, node, plan, settings, gens, Allocating) + if err != nil { + t.Fatalf("%s: %v", node, err) + } + composed, err := plan.Compose(with) + if err != nil { + t.Fatalf("%s: %v", node, err) + } + rules, err := plan.Rules(with) + if err != nil { + t.Fatalf("%s: %v", node, err) + } + settles := map[string]any{} + for module, serves := range served { + if !containsString(on[module], node) { + continue + } + layers, err := open.inventory.SettingsFor(ctx, node, module) + if err != nil { + t.Fatal(err) + } + for name, s := range serves { + fact, _ := s.(map[string]any) + settled, err := catalogue.Settle(fact, catalogue.WithDefaults(shelf[module], layers)) + if err != nil { + settles[module+" serves "+name] = "error: " + err.Error() + continue + } + settles[module+" serves "+name] = settled + } + } + got, err := json.MarshalIndent(map[string]any{"resources": composed.Resources, "left-out": composed.LeftOut, + "rules": rules, "served": settles}, "", " ") + if err != nil { + t.Fatal(err) + } + told[node] = string(got) + } + return told +} + +// effectiveEverywhere is each module's every value on each machine it is assigned to, as `settings show` and +// `settings preferences` say them (catalogue.Effective): the value, never where it came from, which the move +// changes by design; a key a layer says by its file's name is read as its config key. +func effectiveEverywhere(t *testing.T, open *stores, w configuredMesh) map[string]any { + t.Helper() + ctx := t.Context() + shelf, err := open.inventory.Catalogue(ctx) + if err != nil { + t.Fatal(err) + } + out := map[string]any{} + for _, module := range movedKeys(w.On) { + if !shelf[module].DeclaresConfig() { + continue + } + for _, node := range w.On[module] { + layers, err := open.inventory.SettingsFor(ctx, node, module) + if err != nil { + t.Fatal(err) + } + // A layer's old spelling of a key (nodered's timeZone) is listed beside the key it is the file's name + // for, and reaches the file as that key does (issue 751): read as that key, which the move sets. + as := catalogue.TakenAs(shelf[module]) + for _, v := range catalogue.Effective(shelf[module], layers) { + if key, old := as[v.Key]; old { + out[node+"."+module+"."+key] = normalJSON(v.Value) + continue + } + if _, said := out[node+"."+module+"."+v.Key]; said && v.FromDefault { + continue + } + out[node+"."+module+"."+v.Key] = normalJSON(v.Value) + } + } + } + return out +} + +func setAt(a configMoveAnswer, key string) []configSet { + var out []configSet + for _, s := range a.Sets { + if s.valueKey(s.Key) == key { + out = append(out, s) + } + } + return out +} + +// **The settings layers move into mesh-config and no machine is told anything new** (novox/hq ADR 0348 §9, design +// 57 K5, issue 664). Every live layer is seeded as the store holds it once move-placement has run, on modules shaped +// as their definitions, and every machine composed. The move, asked without confirming, says each value it would set +// and where from, and sets nothing; confirmed, it sets them in mesh-config, reads them back, empties the layers it +// moved and switches. Each machine is then told byte for byte what it was told before, every module serves the same, +// and `settings` lists the same values `config` resolves. Asked again, it sets nothing. +func TestEveryMachineIsToldTheSameAfterTheMoveOfConfig(t *testing.T) { + open, w, served := aConfiguredMesh(t) + ctx := t.Context() + readBackTries = 0 + t.Cleanup(func() { readBackTries = 15 }) + inv := open.inventory + config := &standInMeshConfig{open: open, values: map[string]any{}} + inv.ReadsConfigFrom(config.laid) + before := toldEveryMachine(t, open, served, w.On) + listedBefore := effectiveEverywhere(t, open, w) + factsBefore, err := gatherFacts(ctx, open, "2.11.17") + if err != nil { + t.Fatal(err) + } + + asked, err := runConfigMove(ctx, inv, config, false, "the move of config, run by a test", "") + if err != nil || !asked.DryRun || asked.Switched || len(asked.Problems) > 0 || len(config.sets) > 0 { + t.Fatalf("the move, not confirmed, says %+v (%v) and set %v", asked, err, config.sets) + } + // nodered's live layer says the file's name for its time zone: set under the config key (issue 751). + if s := setAt(asked, "module.anchor.nodered.time-zone"); len(s) != 1 || s[0].Value != "Europe/Brussels" || + !strings.HasSuffix(s[0].From[0], ": timeZone") { + t.Errorf("nodered's time zone moves as %+v", s) + } + for _, s := range asked.Sets { + if s.Key == "timeZone" { + t.Errorf("a key is set under the file's name: %+v", s) + } + } + // claude-code's whole-mesh managed settings become mesh config for agent-managed-settings (ADR 0348's + // consequences), and its whole-mesh layer sets no module value. + if s := setAt(asked, "mesh.agent-managed-settings"); len(s) != 1 || s[0].From[0] != "claude-code on the whole mesh: managed_settings" { + t.Errorf("claude-code's managed settings move as %+v", s) + } + for _, node := range fourMachines { + if s := setAt(asked, "module."+node+".claude-code.managed_settings"); len(s) > 0 { + t.Errorf("claude-code's managed settings are also set on %s: %+v", node, s) + } + } + // A whole-mesh layer of a value no predefined key holds goes to each machine it is assigned to, under that + // machine's layer: power's on the workstation is the workstation's, its own elsewhere. + if s := setAt(asked, "module.workstation.power.handle-power-key"); len(s) != 1 || s[0].Value != "suspend" { + t.Errorf("power's key on the workstation moves as %+v", s) + } + if s := setAt(asked, "module.anchor.power.handle-power-key"); len(s) != 1 || s[0].Value != "poweroff" { + t.Errorf("power's key on the anchor moves as %+v", s) + } + for _, s := range asked.Sets { + if s.Level == "node" { + t.Errorf("a machine's layer became node config: %+v", s) + } + } + var stays []string + for _, s := range asked.Stays { + stays = append(stays, strings.Fields(s)[0]) + } + sort.Strings(stays) + if want := []string{"bazarr", "bazarr", "bookshelf", "jackett", "kernel", "kometa", "lidarr", "nzbget", "ombi", + "plex", "qbittorrent", "radarr", "sonarr", "tautulli"}; len(stays) < 10 || !containsString(stays, "kernel") { + t.Errorf("the layers that stay are %v, want those of %v", stays, want) + } + if after := toldEveryMachine(t, open, served, w.On); !reflect.DeepEqual(after, before) { + t.Fatal("a dry run changed what a machine is told") + } + + moved, err := runConfigMove(ctx, inv, config, true, "the move of config, run by a test", "release 57 is delivered") + if err != nil || moved.DryRun || !moved.Switched { + t.Fatalf("the move says %+v (%v)", moved, err) + } + if len(config.sets) != len(asked.Sets) { + t.Fatalf("the move set %d values, and said it would set %d", len(config.sets), len(asked.Sets)) + } + if held := inv.QueryForTest(t, `select module from settings s where values <> '{}' + and module in (select name from module where manifest ? 'config')`); len(held) != 0 { + t.Fatalf("a layer of a module in the config form still holds values: %v", held) + } + after := toldEveryMachine(t, open, served, w.On) + for _, node := range fourMachines { + if after[node] != before[node] { + t.Errorf("%s is told something else after the move:\n%s\nwas:\n%s", node, after[node], before[node]) + } + } + if listedAfter := effectiveEverywhere(t, open, w); !reflect.DeepEqual(listedAfter, listedBefore) { + t.Errorf("settings lists other values after the move:\n%v\nwas:\n%v", listedAfter, listedBefore) + } + // And config lists what settings listed: each module's resolved value on each machine is the value settings + // said before the move. + resolved, err := config.Resolved(ctx) + if err != nil { + t.Fatal(err) + } + for _, r := range resolved { + said, listed := listedBefore[r.Node+"."+r.Module+"."+r.Key] + if r.Source == "unset" { + if listed && said != nil { + t.Errorf("%s of %s on %s is unset in config, and settings said %v", r.Key, r.Module, r.Node, said) + } + continue + } + var v any + _ = json.Unmarshal(r.Value, &v) + if !listed || !reflect.DeepEqual(v, said) { + t.Errorf("config resolves %s of %s on %s as %v, and settings said %v", r.Key, r.Module, r.Node, v, said) + } + } + + // The facts snapshot carries mesh-config's values for the merge gate, as each machine's layer of each module, and + // each machine's declaration composes to the digest it did before the move. + factsAfter, err := gatherFacts(ctx, open, "2.11.17") + if err != nil { + t.Fatal(err) + } + // Machines are named in a snapshot by stand-ins of its own; each is found by what it composes. + digests := func(f snapshotFacts) ([]string, string) { + var out []string + zone := "" + for _, m := range f.Machines { + out = append(out, m.Declaration.Digest) + for _, l := range m.Settings { + if l.Module == "nodered" { + zone = compactJSON(l.Values["time-zone"]) + } + } + } + sort.Strings(out) + return out, zone + } + was, _ := digests(factsBefore) + now, zone := digests(factsAfter) + if zone != `"Europe/Brussels"` { + t.Errorf("the facts snapshot does not carry nodered's time zone from mesh-config: %s", zone) + } + if len(now) != len(fourMachines) || now[0] == "" || !reflect.DeepEqual(now, was) { + t.Errorf("the declarations in the facts snapshot are %v after the move, %v before", now, was) + } + for _, s := range factsAfter.Settings { + if _, isConfig := map[string]bool{"betterbird": true, "claude-code": true, "power": true, "xdg": true}[s.Module]; isConfig && + len(s.Values) > 0 { + t.Errorf("the facts snapshot still carries the whole mesh's layer of %s: %v", s.Module, s.Values) + } + } + + config.sets = nil + again, err := runConfigMove(ctx, inv, config, true, "the move of config, run by a test", "again") + if err != nil || len(config.sets) != 0 || len(again.Moved) != 0 { + t.Fatalf("asked again, the move set %v and says %+v (%v)", config.sets, again, err) + } + if after := toldEveryMachine(t, open, served, w.On); !reflect.DeepEqual(after, before) { + t.Fatal("the move asked again changed what a machine is told") + } +} + +// **The move refuses what it cannot move as it composes, and sets nothing** (issue 664): a layer that still holds +// placement (move-placement has not run), a key its module does not declare, and a value its declaration refuses — +// each said by name; and a value mesh-config resolves from a level no layer says (it would be laid once the switch +// is on) leaves every layer and the switch as they were. +func TestTheMoveOfConfigRefusesWhatWouldChangeAMachine(t *testing.T) { + open, _, _ := aConfiguredMesh(t) + ctx := t.Context() + readBackTries = 0 + t.Cleanup(func() { readBackTries = 15 }) + inv := open.inventory + config := &standInMeshConfig{open: open, values: map[string]any{}} + inv.ReadsConfigFrom(config.laid) + + inv.ExecForTest(t, `update settings set values = values || '{"endpoints": {"web": {"reach": "public"}}}' + where module = 'nodered'`) + inv.ExecForTest(t, `update settings set values = values || '{"colour": "blue"}' where module = 'unifi'`) + inv.ExecForTest(t, `update settings set values = '{"font-size": 900}' where module = 'dunst'`) + answer, err := runConfigMove(ctx, inv, config, true, "a test", "release 57 is delivered") + if err == nil || answer.Switched || len(config.sets) > 0 { + t.Fatalf("the move went ahead: %+v (%v), set %v", answer, err, config.sets) + } + for _, want := range []string{"nodered on anchor still holds placement (endpoints): run move-placement first", + `unifi on anchor says "colour", which unifi does not declare`, "font-size at dunst's own value on workstation"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("the refusal does not say %q: %v", want, err) + } + } + inv.ExecForTest(t, `update settings set values = values - 'endpoints' where module = 'nodered'`) + inv.ExecForTest(t, `update settings set values = values - 'colour' where module = 'unifi'`) + inv.ExecForTest(t, `update settings set values = '{"font-size": 13, "width": 500}' where module = 'dunst'`) + + // Set in mesh-config before the move, on a module whose layers say nothing of it: the switch would lay it. + config.values["module.anchor.grafana."+movedKeys(func() map[string]catalogue.SettingDeclaration { + shelf, _ := inv.Catalogue(ctx) + return shelf["grafana"].Settings + }())[0]] = "set before the move" + answer, err = runConfigMove(ctx, inv, config, true, "a test", "release 57 is delivered") + if err == nil || !strings.Contains(err.Error(), "grafana on anchor resolves") || answer.Switched { + t.Fatalf("a value no layer says was laid by the switch: %+v (%v)", answer, err) + } + if moved, _ := inv.ConfigMoved(ctx); moved.Moved { + t.Fatal("the switch is on after a refused move") + } + if held := inv.QueryForTest(t, `select count(*) from settings where values <> '{}'`); held[0][0] == "0" { + t.Fatal("a refused move emptied the layers") + } +} + +// The verb's command line: a dry run unless confirmed, and a confirmed move says why, recorded as a person's step of +// the release, never a repair (issue 664). +func TestTheMoveOfConfigIsADryRunUnlessConfirmed(t *testing.T) { + for _, c := range []struct { + args map[string]any + want string + }{ + {map[string]any{}, "move-config --json"}, + {map[string]any{"confirm": "true", "why": "release 57 is delivered"}, "move-config --json --confirm --why release 57 is delivered"}, + } { + argv, err := argvFor("move-config", c.args) + if err != nil || strings.Join(argv, " ") != c.want { + t.Errorf("%v: %q %v, want %q", c.args, argv, err, c.want) + } + } + for _, args := range []map[string]any{{"confirm": "true"}, {"why": "tidy"}, {"node": "anchor"}} { + if argv, err := argvFor("move-config", args); err == nil { + t.Errorf("%v was composed as %q", args, argv) + } + } + if repairingCommand([]string{"move-config", "--json", "--confirm", "--why", "x"}) != "move-config" { + t.Error("a confirmed move through the generic verb would go unrecorded") + } + if !personsDecision(link.HandAct{Verb: "move-config"}) { + t.Error("a move of the release would count toward a healer the mesh lacks") + } +} + +// ask is mesh-config's verb as the stand-in answers it: list, set, clear and propose. +func (c *standInMeshConfig) ask(ctx context.Context, args map[string]any) (json.RawMessage, error) { + str := func(k string) string { s, _ := args[k].(string); return s } + level := configLevel{Level: "mesh"} + switch { + case str("module") != "" && str("node") != "": + level = configLevel{Level: "module", Node: str("node"), Module: str("module")} + case str("node") != "": + level = configLevel{Level: "node", Node: str("node")} + } + c.asked = append(c.asked, str("action")+" "+level.valueKey(str("key"))) + switch str("action") { + case "list": + resolved, err := c.Resolved(ctx) + if err != nil { + return nil, err + } + out := map[string]any{"resolved": []resolvedEntry{}, "set": []any{}} + var listed []resolvedEntry + for _, r := range resolved { + if r.Module == str("module") && (str("node") == "" || r.Node == str("node")) { + listed = append(listed, r) + } + } + out["resolved"] = listed + var set []any + for _, k := range movedKeys(c.values) { + parts := strings.SplitN(k, ".", 4) + if len(parts) == 4 && parts[0] == "module" && parts[2] == str("module") { + set = append(set, map[string]any{"level": configLevel{Level: "module", Node: parts[1], Module: parts[2]}, + "key": parts[3], "value": c.values[k], "by": "a test"}) + } + } + out["set"] = set + return json.Marshal(out) + case "set": + if str("why") == "" || str("by") == "" { + return nil, fmt.Errorf("a set without why or by: %v", args) + } + return nil, c.Set(ctx, level, str("key"), args["value"], str("by"), str("why")) + case "clear": + if _, had := c.values[level.valueKey(str("key"))]; !had { + return nil, fmt.Errorf("nothing is set for %s", str("key")) + } + delete(c.values, level.valueKey(str("key"))) + return json.RawMessage(`{}`), nil + case "propose": + return json.RawMessage(`{"proposal":"p-1"}`), nil + } + return nil, fmt.Errorf("the stand-in does not %s", str("action")) +} + +// **`settings` answers by forwarding to `config`** (novox/hq ADR 0348 §1, design 57 K5, issue 664): once the move +// has run, a module in the config form is shown from mesh-config, its values set and cleared there as its own on the +// machine (placement still beside the assignment), and proposed through it; a trusted value is refused through a +// verb, a module value for the whole mesh is refused, and a module still in the settings form keeps today's verb. +func TestTheSettingsVerbForwardsToConfigOnceTheValuesMoved(t *testing.T) { + open, w, served := aConfiguredMesh(t) + ctx := t.Context() + inv := open.inventory + readBackTries = 0 + t.Cleanup(func() { readBackTries = 15 }) + config := &standInMeshConfig{open: open, values: map[string]any{}} + inv.ReadsConfigFrom(config.laid) + savedRead, savedAsk := readConfigValues, configAsk + readConfigValues, configAsk = config.laid, config.ask + t.Cleanup(func() { readConfigValues, configAsk = savedRead, savedAsk }) + if _, err := runConfigMove(ctx, inv, config, true, "a test", "release 57 is delivered"); err != nil { + t.Fatal(err) + } + + shown := stdoutOf(t, func() error { return settingsCommand(ctx, []string{"show", "nodered", "--node", "anchor"}) }) + for _, want := range []string{`"time-zone": "Europe/Brussels"`, `time-zone = "Europe/Brussels" (module)`, + "mesh-config.config", `"mqtt": {`} { + if !strings.Contains(shown, want) { + t.Errorf("settings show does not say %q:\n%s", want, shown) + } + } + if strings.Contains(shown, "timeZone") { + t.Errorf("settings show still says the file's name:\n%s", shown) + } + + config.asked = nil + said := stdoutOf(t, func() error { + return settingsCommand(ctx, []string{"set", "dunst", `{"font-size": 12, "width": 500}`, "--node", "workstation"}) + }) + if config.values["module.workstation.dunst.font-size"] != float64(12) || !strings.Contains(said, "~ font-size") || + containsString(config.asked, "set module.workstation.dunst.width") { + t.Fatalf("settings set reached mesh-config as %v (%v):\n%s", config.values, config.asked, said) + } + told := toldEveryMachine(t, open, served, w.On)["workstation"] + if !strings.Contains(told, "12") { + t.Error("the workstation is not told the font size set through settings") + } + if err := settingsCommand(ctx, []string{"set", "dunst", `{"font-size": 12}`, "--node", "workstation"}); err == nil || + !strings.Contains(err.Error(), "would no longer set width") { + t.Fatalf("a set dropping a value without --replace: %v", err) + } + if err := settingsCommand(ctx, []string{"set", "dunst", `{"font-size": 12}`}); err == nil || + !strings.Contains(err.Error(), "no own value for the whole mesh") { + t.Fatalf("a module value for the whole mesh: %v", err) + } + + t.Setenv(verbVar, "settings") + for _, c := range []struct{ module, node, values, key, held, why string }{ + // A file root reads: refused as before the move (novox/hq issue 339). + {"hostname", "anchor", `{"hostname": "elsewhere"}`, "module.anchor.hostname.hostname", "Ace", "issue 339"}, + {"messenger", "laptop", `{"desktop-machines": ["g14", "shanks"], "time-zone": "UTC"}`, + "module.laptop.messenger.time-zone", "Europe/Brussels", "issue 339"}, + } { + err := settingsCommand(ctx, []string{"set", c.module, c.values, "--node", c.node}) + if err == nil || !strings.Contains(err.Error(), c.why) || config.values[c.key] != c.held { + t.Fatalf("a trusted value of %s through a verb: %v, holds %v", c.module, err, config.values[c.key]) + } + } + // And a trusted value no file makes the terminal's is refused as trusted, since mesh-config takes the controller's + // word for it (ADR 0348 §8). + if err := refuseTrustedThroughAVerb(ctx, inv, "messenger", "laptop", []string{"time-zone"}); err == nil || + !strings.Contains(err.Error(), "ADR 0348 §8") { + t.Fatalf("a trusted value through a verb: %v", err) + } + if err := refuseTrustedThroughAVerb(ctx, inv, "dunst", "workstation", []string{"width"}); err != nil { + t.Fatalf("a value nobody trusts through a verb: %v", err) + } + stdoutOf(t, func() error { + return settingsCommand(ctx, []string{"set", "dunst", `{"font-size": 12, "width": 600}`, "--node", "workstation"}) + }) + if config.values["module.workstation.dunst.width"] != float64(600) { + t.Fatalf("a value nobody trusts was not set through a verb: %v", config.values["module.workstation.dunst.width"]) + } + t.Setenv(verbVar, "") + + stdoutOf(t, func() error { return settingsCommand(ctx, []string{"clear", "dunst", "--node", "workstation"}) }) + for _, k := range []string{"font-size", "width"} { + if _, held := config.values["module.workstation.dunst."+k]; held { + t.Errorf("settings clear left dunst's %s on the workstation", k) + } + } + + config.asked = nil + proposed := stdoutOf(t, func() error { + return settingsCommand(ctx, []string{"propose", "hostname", `{"hostname": "elsewhere"}`, "--node", "anchor"}) + }) + if !containsString(config.asked, "propose module.anchor.hostname.hostname") || !strings.Contains(proposed, "p-1") { + t.Fatalf("settings propose reached mesh-config as %v:\n%s", config.asked, proposed) + } + + config.asked = nil + kept := stdoutOf(t, func() error { return settingsCommand(ctx, []string{"show", "kernel", "--node", "laptop"}) }) + if !strings.Contains(kept, "transparent-huge-pages") || len(config.asked) > 0 { + t.Fatalf("a module in the settings form is shown as %q, asking mesh-config %v", kept, config.asked) + } +} + +// **An approval for a module whose config has moved is refused, and writes nothing** (issue 830, novox/hq ADR 0348): +// a proposal raised while grafana read its layer (it had none, so the layer is as the operator was shown it) and +// approved after the move would put its values back into the layer, ReadsConfig would turn false, and the module +// would compose from emptied layers everywhere. +func TestAnApprovalRaisedBeforeTheMoveOfConfigIsRefusedAfterIt(t *testing.T) { + open, _, _ := aConfiguredMesh(t) + ctx := t.Context() + readBackTries = 0 + t.Cleanup(func() { readBackTries = 15 }) + inv := open.inventory + config := &standInMeshConfig{open: open, values: map[string]any{}} + inv.ReadsConfigFrom(config.laid) + shelf, err := inv.Catalogue(ctx) + if err != nil { + t.Fatal(err) + } + key := movedKeys(func() map[string]catalogue.SettingDeclaration { return shelf["grafana"].Settings }())[0] + values := map[string]any{key: "raised before the move"} + p := settingsProposal{Module: "grafana", Node: "anchor", Values: values, From: "g14/claude-code", At: time.Now(), + Digest: layerDigest(values), BeforeDigest: layerDigest(nil)} + if _, has, err := inv.Layer(ctx, "anchor", "grafana"); err != nil || has { + t.Fatalf("grafana has a layer on anchor before the test: %v %v", has, err) + } + if moved, err := runConfigMove(ctx, inv, config, true, "a test", "release 57 is delivered"); err != nil || !moved.Switched { + t.Fatalf("the move says %+v (%v)", moved, err) + } + if reads, err := inv.ReadsConfig(ctx, "grafana"); err != nil || !reads { + t.Fatalf("grafana does not read config after the move: %v %v", reads, err) + } + _, err = setLayerIn(open)(ctx, p, "s30", "approved by the operator") + if err == nil || !strings.Contains(err.Error(), "moved to mesh-config since the operator was shown the change") || + !strings.Contains(err.Error(), "propose it again") { + t.Fatalf("the stale approval: %v", err) + } + if layer, has, err := inv.Layer(ctx, "anchor", "grafana"); err != nil || has && len(layer) > 0 { + t.Errorf("the refused approval wrote the layer: %v %v %v", layer, has, err) + } + if reads, err := inv.ReadsConfig(ctx, "grafana"); err != nil || !reads { + t.Errorf("the refused approval turned ReadsConfig off: %v %v", reads, err) + } +} + +// **A proposal of placement only is set after the move, as before** (issue 830): forwardPropose sends it down the +// old path on purpose, so refusing it would loop the operator (refused, proposed again, refused). +func TestAPlacementOnlyApprovalRaisedBeforeTheMoveIsSetAfterIt(t *testing.T) { + open, _, _ := aConfiguredMesh(t) + ctx := t.Context() + readBackTries = 0 + t.Cleanup(func() { readBackTries = 15 }) + inv := open.inventory + config := &standInMeshConfig{open: open, values: map[string]any{}} + inv.ReadsConfigFrom(config.laid) + values := map[string]any{catalogue.PlacesSetting: map[string]any{"data": "/srv/grafana-data"}} + p := settingsProposal{Module: "grafana", Node: "anchor", Values: values, From: "g14/claude-code", At: time.Now(), + Digest: layerDigest(values), BeforeDigest: layerDigest(nil)} + if _, has, err := inv.Layer(ctx, "anchor", "grafana"); err != nil || has { + t.Fatalf("grafana has a layer on anchor before the test: %v %v", has, err) + } + if moved, err := runConfigMove(ctx, inv, config, true, "a test", "release 57 is delivered"); err != nil || !moved.Switched { + t.Fatalf("the move says %+v (%v)", moved, err) + } + if _, err := setLayerIn(open)(ctx, p, "s31", "approved by the operator"); err != nil { + t.Fatalf("the placement-only approval after the move: %v", err) + } + if _, has, err := inv.Layer(ctx, "anchor", "grafana"); err != nil || !has { + t.Errorf("the approval did not set the layer: %v %v", has, err) + } + if reads, err := inv.ReadsConfig(ctx, "grafana"); err != nil || !reads { + t.Errorf("the approval turned ReadsConfig off: %v %v", reads, err) + } +} diff --git a/cmd/mesh-controller/config_read.go b/cmd/mesh-controller/config_read.go new file mode 100644 index 00000000..39dbc725 --- /dev/null +++ b/cmd/mesh-controller/config_read.go @@ -0,0 +1,128 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "sync" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/inventory" +) + +// **Where the controller reads config values once the move has run** (novox/hq ADR 0348 §9, design 57 K5, issue 664): +// mesh-config_resolved, the bucket mesh-config alone writes, keyed `..`, each value with the +// source it came from. Read only for a module the switch reaches (inventory.ReadsConfig), so a controller before the +// move never asks the bus for config. The exception to ADR 0348 §9 ("the controller holds no config value") that +// ADR 0349 clause 17's old way needs, until design 57's R1 hands every module to the provisioner. + +// resolvedEntry is one key of mesh-config_resolved as the controller reads it: the value and where it came from. +type resolvedEntry struct { + Node string `json:"node"` + Module string `json:"module"` + Key string `json:"key"` + Value json.RawMessage `json:"value,omitempty"` + Source string `json:"source"` +} + +// The sources a resolved value is laid over a module's files from: mesh config, node config and the module's own +// value. Its manifest's default is composition's already, and a taken key's default is never laid over its file; +// a required value nothing sets (`unset`) is refused by composition by name, as it is today. +var laidSources = map[string]bool{"mesh": true, "node": true, "module": true} + +// configLaid is the values of one module on one node that are laid over its files, from the resolved entries. +func configLaid(entries []resolvedEntry, node, module string) (map[string]any, error) { + out := map[string]any{} + for _, e := range entries { + if e.Node != node || e.Module != module || !laidSources[e.Source] { + continue + } + var v any + if err := json.Unmarshal(e.Value, &v); err != nil { + return nil, fmt.Errorf("mesh-config's value %s of %s on %s cannot be read: %w", e.Key, module, node, err) + } + out[e.Key] = v + } + return out, nil +} + +// resolvedRead is what one read of the whole bucket held, and when it was read. +type resolvedRead struct { + mu sync.Mutex + at time.Time + entries []resolvedEntry +} + +// resolvedFresh is how long one read of the bucket answers: a plan reads every module of a machine, and one read +// serves them all; a value set since is read by the next composition. +const resolvedFresh = 5 * time.Second + +var theResolved resolvedRead + +// readConfigValues is where an opened inventory reads config values once the switch is on; a test replaces it. +var readConfigValues inventory.ConfigValues = busConfigValues + +// busConfigValues reads one module's config values on one node from mesh-config_resolved (inventory.ConfigValues). +func busConfigValues(ctx context.Context, node, module string) (map[string]any, error) { + theResolved.mu.Lock() + defer theResolved.mu.Unlock() + if theResolved.entries == nil || time.Since(theResolved.at) > resolvedFresh { + entries, err := readResolved(ctx) + if err != nil { + return nil, err + } + theResolved.entries, theResolved.at = entries, time.Now() + } + return configLaid(theResolved.entries, node, module) +} + +// readResolved is every entry of mesh-config_resolved, read on the bus. +func readResolved(ctx context.Context) ([]resolvedEntry, error) { + entries := []resolvedEntry{} + err := onTheBus(func(conn *nats.Conn) error { + js, err := jetstream.New(conn) + if err != nil { + return err + } + ctx, cancel := context.WithTimeout(ctx, 20*time.Second) + defer cancel() + kv, err := js.KeyValue(ctx, broker.ConfigResolvedBucket) + if err != nil { + return fmt.Errorf("%s cannot be opened: %w", broker.ConfigResolvedBucket, err) + } + w, err := kv.WatchAll(ctx, jetstream.IgnoreDeletes()) + if err != nil { + return err + } + defer func() { _ = w.Stop() }() + for { + select { + case <-ctx.Done(): + return fmt.Errorf("%s was not read whole: %w", broker.ConfigResolvedBucket, ctx.Err()) + case e, open := <-w.Updates(): + if !open { + return errors.New(broker.ConfigResolvedBucket + " stopped answering") + } + if e == nil { + return nil + } + var r resolvedEntry + if err := json.Unmarshal(e.Value(), &r); err != nil { + return fmt.Errorf("%s holds %q, which is not a resolved value: %w", broker.ConfigResolvedBucket, + e.Key(), err) + } + if r.Node == "" || r.Module == "" || r.Key == "" || !strings.HasPrefix(e.Key(), r.Node+"."+r.Module+".") { + return fmt.Errorf("%s holds %q, whose value names another key", broker.ConfigResolvedBucket, e.Key()) + } + entries = append(entries, r) + } + } + }) + return entries, err +} diff --git a/cmd/mesh-controller/core_box.go b/cmd/mesh-controller/core_box.go new file mode 100644 index 00000000..54f641e6 --- /dev/null +++ b/cmd/mesh-controller/core_box.go @@ -0,0 +1,297 @@ +package main + +import ( + "context" + "crypto/ecdh" + "crypto/x509" + "encoding/base64" + "encoding/json" + "encoding/pem" + "errors" + "fmt" + "sort" + "strings" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +// The core's own credentials, sealed by the vault (novox/hq ADR 0349 §16, issue 755; the lead's decision on issue 755, +// notes 4 and 5). +// +// For a machine whose engine opens the vault's box (link.VaultBoxContract), each own secret of a module on the core's +// list (catalogue.CoreCredentialModules) travels as the vault sealed it to that machine's key, which the controller +// asks for on the send path (`get`, answered to the controller alone and sealed only to the key that machine's engine +// stated: mesh-vault's core.go) and carries without being able to open it. A machine below that contract is sent the +// controller's own sealed value, exactly as before. +// +// **The vault seals every answer afresh**, so the box last got is kept (inventory.CoreBox) and carried again while the +// vault's generation and the machine's key are the same: a declaration that changed with every answer would never be +// current. Every question but a send reads the kept box and asks nothing. +// +// **When the vault does not answer, the controller never falls back to its own seal.** A credential with a kept box +// keeps it — the machine keeps the file it has — and one with none leaves its module out of the declaration (the +// machine keeps what it holds of it, and a module not there yet is not placed). Either way the send raises a +// condition naming the vault and the machine, cleared by the first send the vault answers whole. + +// engineOpensVaultBox says whether a machine's node-engine opens the vault's box, by its own newest statement. +func engineOpensVaultBox(ctx context.Context, inv *inventory.Inventory, node string) (bool, error) { + stated, had, err := inv.HealthOf(ctx, node) + if err != nil { + return false, err + } + return had && stated.Contract >= link.VaultBoxContract, nil +} + +// coreAnswer is the vault's `get` answer as the controller reads it: the name, the generation and the box, never +// opened. +type coreAnswer struct { + Name string `json:"name"` + Generation int `json:"generation"` + Sealed json.RawMessage `json:"sealed"` +} + +// coreAsk asks the vault for one secret sealed to the given key. +type coreAsk func(name, publicKey string) (coreAnswer, error) + +// coreBoxes is what the vault's boxes make of one machine's composition. +type coreBoxes struct { + // sealed is each core credential's box, by module and name (catalogue.Rendering.VaultSealed). + sealed map[string]map[string]json.RawMessage + // notPlaced is each core module left out, with why (catalogue.Rendering.NotPlaced). + notPlaced map[string]string + // keep is each box newly answered, to keep. + keep []inventory.CoreBox + // held is each credential the vault did not answer for whose kept box is carried again; silent is why the vault + // did not answer, per credential. + held, silent []string +} + +// coreModulesOf is every module of the plan on the core's list whose own secrets this machine's declaration carries, +// with their plain own secret names, sorted: one handed to the provisioner has its secrets from the vault through the +// client, and one issued outside the mesh is never the vault's to seal here. +func coreModulesOf(modules []catalogue.Manifest, handed func(catalogue.Manifest) bool) map[string][]string { + out := map[string][]string{} + for _, m := range modules { + if !catalogue.CarriesVaultSealed(m.Module) || (handed != nil && handed(m)) { + continue + } + var names []string + for name, own := range m.OwnSecrets.Plain() { + if own.IssuedBy != catalogue.IssuedOutside { + names = append(names, name) + } + } + if len(names) > 0 { + sort.Strings(names) + out[m.Module] = names + } + } + return out +} + +// sealingKeyPEM is a machine's sealing key, recorded raw in base64, as the SPKI PEM the vault seals to. +func sealingKeyPEM(raw string) (string, error) { + b, err := base64.StdEncoding.DecodeString(strings.TrimSpace(raw)) + if err != nil || len(b) != 32 { + return "", errors.New("the machine's sealing key is not a raw X25519 key") + } + pub, err := ecdh.X25519().NewPublicKey(b) + if err != nil { + return "", err + } + der, err := x509.MarshalPKIXPublicKey(pub) + if err != nil { + return "", err + } + return string(pem.EncodeToMemory(&pem.Block{Type: "PUBLIC KEY", Bytes: der})), nil +} + +// coreBoxesOf decides each core credential of a machine: on a send, the vault's answer (the kept box while its +// generation and key are the same), the kept box where the vault did not answer, or the module left out where nothing +// was kept; on any other question, the kept box, or a stand-in the next send asks for. Pure: what to keep and what to +// say is answered, not done. +func coreBoxesOf(node, sealingKey string, core map[string][]string, kept map[string]map[string]inventory.CoreBox, + choosing Choosing, ask coreAsk) coreBoxes { + out := coreBoxes{sealed: map[string]map[string]json.RawMessage{}, notPlaced: map[string]string{}} + modules := make([]string, 0, len(core)) + for m := range core { + modules = append(modules, m) + } + sort.Strings(modules) + for _, module := range modules { + if sealingKey == "" { + out.notPlaced[module] = fmt.Sprintf("%s has no sealing key, so the vault can seal %s's credentials to "+ + "nothing there (novox/hq ADR 0349 §16)", node, module) + continue + } + key, err := sealingKeyPEM(sealingKey) + if err != nil { + out.notPlaced[module] = fmt.Sprintf("%s's credentials cannot be asked of the vault: %v", module, err) + continue + } + boxes := map[string]json.RawMessage{} + for _, name := range core[module] { + k, has := kept[module][name] + valid := has && k.SealedTo == sealingKey && len(k.Box) > 0 + if choosing != Allocating { + if valid { + boxes[name] = k.Box + } else { + boxes[name] = catalogue.ForeseenVaultBox + } + continue + } + secret := "own/" + node + "/" + module + "/" + name + got, err := ask(secret, key) + if err == nil && (got.Generation <= 0 || canonicalBox(got.Sealed) == nil) { + err = errors.New("it answered no box") + } + if err != nil { + out.silent = append(out.silent, fmt.Sprintf("%s: %v", secret, err)) + if valid { + boxes[name] = k.Box + out.held = append(out.held, module+"/"+name) + continue + } + out.notPlaced[module] = fmt.Sprintf("the vault did not give %s's credential %s for %s (%v), and the "+ + "controller never seals it itself (novox/hq ADR 0349 §16): not placed until the vault answers", + module, name, node, err) + break + } + if valid && k.Generation == got.Generation { + boxes[name] = k.Box + continue + } + box := canonicalBox(got.Sealed) + boxes[name] = box + out.keep = append(out.keep, inventory.CoreBox{Module: module, Name: name, Generation: got.Generation, + SealedTo: sealingKey, Box: box}) + } + if _, left := out.notPlaced[module]; !left { + out.sealed[module] = boxes + } + } + return out +} + +// canonicalBox is the vault's box in the one form the declaration carries — the sealed box's fields, in their order, +// and nothing else — or nil when it is not one. +func canonicalBox(raw json.RawMessage) json.RawMessage { + var b struct { + V int `json:"v"` + Eph string `json:"eph"` + IV string `json:"iv"` + Tag string `json:"tag"` + Ct string `json:"ct"` + } + if err := json.Unmarshal(raw, &b); err != nil || b.V <= 0 || b.Eph == "" || b.Ct == "" { + return nil + } + out, _ := json.Marshal(b) + return out +} + +// coreVault is how a send asks the vault for a core credential: its `get`, in the controller's name. A test stands +// in for it. +var coreVault = func(ctx context.Context) coreAsk { + vault := theVault(ctx) + return func(name, publicKey string) (coreAnswer, error) { + raw, err := vault("get", map[string]any{"name": name, "public_key": publicKey}) + if err != nil { + return coreAnswer{}, err + } + var a coreAnswer + if err := json.Unmarshal(raw, &a); err != nil { + return coreAnswer{}, fmt.Errorf("the vault's answer is not one: %w", err) + } + return a, nil + } +} + +// coreBoxesFor is coreBoxesOf for one machine, read from and kept in the store, and said on a send. +func coreBoxesFor(ctx context.Context, inv *inventory.Inventory, node string, modules []catalogue.Manifest, + handed func(catalogue.Manifest) bool, choosing Choosing) (coreBoxes, error) { + core := coreModulesOf(modules, handed) + if len(core) == 0 { + if choosing == Allocating { + settleVaultSilent(ctx, node) + } + return coreBoxes{}, nil + } + key, err := inv.SealingKeyOf(ctx, node) + if err != nil { + return coreBoxes{}, err + } + kept, err := inv.KeptCoreBoxes(ctx, node) + if err != nil { + return coreBoxes{}, err + } + var ask coreAsk + if choosing == Allocating { + ask = coreVault(ctx) + } + out := coreBoxesOf(node, key, core, kept, choosing, ask) + for _, b := range out.keep { + if err := inv.KeepCoreBox(ctx, node, b); err != nil { + return coreBoxes{}, err + } + } + if choosing == Allocating { + sayVaultSilent(ctx, node, out) + } + return out, nil +} + +// kindVaultSilent is a send that did not have the vault's box for a core credential of a machine. +const kindVaultSilent = "vault-silent" + +// vaultSilentObservation is that, as a condition: `machine..vault-silent`. +func vaultSilentObservation(node string, b coreBoxes) conditions.Observation { + o := conditions.Observation{Scope: conditions.ScopeMachine, ID: node, Kind: kindVaultSilent, Token: kindVaultSilent, + Machine: node, Severity: conditions.Warning, Source: coreDowngradeSource} + var said []string + if len(b.notPlaced) > 0 { + o.Severity = conditions.Urgent + var left []string + for m := range b.notPlaced { + left = append(left, m) + } + sort.Strings(left) + said = append(said, "not placed: "+strings.Join(left, ", ")) + } + if len(b.held) > 0 { + said = append(said, "kept as last sent: "+strings.Join(b.held, ", ")) + } + o.Summary = fmt.Sprintf("the vault did not answer for the core's credentials on %s; %s", node, strings.Join(said, "; ")) + o.Said = o.Summary + " — " + strings.Join(b.silent, "; ") + return o +} + +// sayVaultSilent raises the condition for a send the vault did not answer whole, and clears it for one it did. +func sayVaultSilent(ctx context.Context, node string, b coreBoxes) { + if len(b.silent) == 0 && len(b.notPlaced) == 0 { + settleVaultSilent(ctx, node) + return + } + fmt.Printf("HELD: %s\n", vaultSilentObservation(node, b).Said) + if conditionsFrom == nil { + return + } + if _, err := conditionsFrom.Observe(ctx, vaultSilentObservation(node, b)); err != nil { + fmt.Printf("the condition for the vault's silence on %s could not be raised: %v\n", node, err) + } +} + +// settleVaultSilent clears the condition once a send had every core credential of the machine from the vault. +func settleVaultSilent(ctx context.Context, node string) { + if conditionsFrom == nil { + return + } + key := vaultSilentObservation(node, coreBoxes{}).Key() + if _, err := conditionsFrom.Clear(ctx, key, "the vault answered for every core credential of "+node); err != nil { + fmt.Printf("the condition %s could not be cleared: %v\n", key, err) + } +} diff --git a/cmd/mesh-controller/core_box_test.go b/cmd/mesh-controller/core_box_test.go new file mode 100644 index 00000000..6a53ccac --- /dev/null +++ b/cmd/mesh-controller/core_box_test.go @@ -0,0 +1,220 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "reflect" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +// The core's own credentials travel sealed by the vault to their machine (novox/hq ADR 0349 §16, issue 755; the lead's +// decision, issue 755 notes 4 and 5): to an engine that opens the vault's box, the send carries the box the vault +// answered the controller and never the controller's own seal; when the vault does not answer, a kept box is carried +// again and a module with none is not placed, with a condition naming the vault and the machine. An engine below the +// contract composes exactly as before, and the vault is not asked. + +// coreStandIn answers `get` with a box per name and generation, or its silence. +type coreStandIn struct { + asked []string + keys []string + generation int + silent error +} + +func (v *coreStandIn) ask(name, key string) (coreAnswer, error) { + v.asked, v.keys = append(v.asked, name), append(v.keys, key) + if v.silent != nil { + return coreAnswer{}, v.silent + } + box := `{"v":1,"eph":"ZQ==","iv":"aQ==","tag":"dA==","ct":"` + strings.Repeat("c", len(v.asked)) + `"}` + return coreAnswer{Name: name, Generation: v.generation, Sealed: json.RawMessage(box)}, nil +} + +func withVault(t *testing.T, v *coreStandIn) { + t.Helper() + before := coreVault + coreVault = func(context.Context) coreAsk { return v.ask } + t.Cleanup(func() { coreVault = before }) +} + +func aCoreMesh(t *testing.T, contract int) (*stores, *coreStandIn) { + t.Helper() + open := aMesh(t) + ctx := t.Context() + client := aClientModule() + register(t, open, client) + if _, err := assign(ctx, open, "laptop", client.Module); err != nil { + t.Fatal(err) + } + if err := issueProvisionerClient(ctx, open.inventory, client, "laptop", broker.Broker{Fingerprint: "ab"}, + "10.0.0.1:4222"); err != nil { + t.Fatal(err) + } + if _, err := open.inventory.RecordHealth(ctx, inventory.NodeHealth{Node: "laptop", Contract: contract, + SaidAt: time.Now()}); err != nil { + t.Fatal(err) + } + v := &coreStandIn{generation: 1} + withVault(t, v) + return open, v +} + +func clientsBroker(resources []map[string]any) map[string]any { + for _, r := range resources { + if r["id"] == "mesh-provisioner-client.needs-"+broker.ProvisionerSecret { + return r + } + } + return nil +} + +func vaultSilence(t *testing.T, node string) (conditions.Condition, bool) { + t.Helper() + c, found, err := conditionsFrom.Get(t.Context(), vaultSilentObservation(node, coreBoxes{}).Key()) + if err != nil { + t.Fatal(err) + } + return c, found +} + +func TestTheSendCarriesTheVaultsBoxForACoreCredentialAndKeepsIt(t *testing.T) { + open, vault := aCoreMesh(t, link.VaultBoxContract) + key, err := open.inventory.SealingKeyOf(t.Context(), "laptop") + if err != nil { + t.Fatal(err) + } + f := clientsBroker(sentTo(t, open, "laptop")) + if f == nil || f["sealed"] != nil || f[catalogue.VaultSealedField] == nil { + t.Fatalf("the core credential is sent as %v: the vault's box, never the controller's seal", f) + } + if want := []string{"own/laptop/mesh-provisioner-client/" + broker.ProvisionerSecret}; !reflect.DeepEqual(vault.asked, want) { + t.Fatalf("the vault was asked %v", vault.asked) + } + if want, _ := sealingKeyPEM(key); vault.keys[0] != want { + t.Fatalf("the vault was asked to seal to %q, not the machine's key", vault.keys[0]) + } + first, _ := json.Marshal(f[catalogue.VaultSealedField]) + + // The vault answers afresh with the same generation: the kept box is carried again, so the declaration is stable. + again, _ := json.Marshal(clientsBroker(sentTo(t, open, "laptop"))[catalogue.VaultSealedField]) + if string(again) != string(first) { + t.Fatalf("a fresh box of the same generation changed the declaration: %s, then %s", first, again) + } + // A rotation in the vault (a newer generation) is carried. + vault.generation = 2 + rotated, _ := json.Marshal(clientsBroker(sentTo(t, open, "laptop"))[catalogue.VaultSealedField]) + if string(rotated) == string(first) { + t.Fatal("a newer generation was not carried") + } + if _, found := vaultSilence(t, "laptop"); found { + t.Fatal("a send the vault answered raised its silence") + } +} + +func TestASilentVaultHoldsTheKeptBoxAndSaysSo(t *testing.T) { + open, vault := aCoreMesh(t, link.VaultBoxContract) + first, _ := json.Marshal(clientsBroker(sentTo(t, open, "laptop"))[catalogue.VaultSealedField]) + vault.silent = errors.New("no responders") + held := clientsBroker(sentTo(t, open, "laptop")) + got, _ := json.Marshal(held[catalogue.VaultSealedField]) + if held == nil || held["sealed"] != nil || string(got) != string(first) { + t.Fatalf("a silent vault's credential is sent as %v: the box kept, never the controller's seal", held) + } + c, found := vaultSilence(t, "laptop") + if !found || !strings.Contains(c.Summary, "vault") || !strings.Contains(c.Summary, "laptop") { + t.Fatalf("no condition names the vault and the machine: %+v", c) + } + // The vault answers again: the condition clears. + vault.silent = nil + sentTo(t, open, "laptop") + if _, found := vaultSilence(t, "laptop"); found { + t.Fatal("the condition stayed after the vault answered") + } +} + +func TestACoreModuleFirstPlacedWhileTheVaultIsSilentIsNotPlaced(t *testing.T) { + open, vault := aCoreMesh(t, link.VaultBoxContract) + vault.silent = errors.New("no responders") + plan, settings, err := planFor(t.Context(), open, "laptop") + if err != nil { + t.Fatal(err) + } + declared, err := declarationFor(t.Context(), open, "laptop", plan, settings) + if err != nil { + t.Fatal(err) + } + for _, r := range declared.Resources { + if strings.HasPrefix(r["id"].(string), "mesh-provisioner-client.") { + t.Fatalf("a core module was placed though the vault gave nothing for it: %v", r) + } + } + if why := declared.leftOutWhy["mesh-provisioner-client"]; !strings.Contains(why, "never seals it itself") { + t.Fatalf("left out %v", declared.leftOutWhy) + } + c, found := vaultSilence(t, "laptop") + if !found || c.Severity != conditions.Urgent || !strings.Contains(c.Summary, "not placed") { + t.Fatalf("the condition does not say why: %+v", c) + } +} + +// An engine below VaultBoxContract is sent the controller's own seal, as before, and the vault is not asked. +func TestAnEngineBelowTheContractIsSentTheControllersSealAsBefore(t *testing.T) { + open, vault := aCoreMesh(t, link.HandedContract) + f := clientsBroker(sentTo(t, open, "laptop")) + if f == nil || f["sealed"] == nil || f[catalogue.VaultSealedField] != nil { + t.Fatalf("an older engine is sent %v", f) + } + if len(vault.asked) != 0 { + t.Fatalf("the vault was asked %v for an older engine", vault.asked) + } +} + +// Every question but a send reads the kept box and asks nothing; with none kept, a stand-in the next send asks for. +func TestOnlyASendAsksTheVault(t *testing.T) { + core := map[string][]string{"nats": {"broker"}} + key := aPublicKey(t) + kept := map[string]map[string]inventory.CoreBox{"nats": {"broker": {Module: "nats", Name: "broker", + Generation: 1, SealedTo: key, Box: json.RawMessage(`{"v":1}`)}}} + never := func(string, string) (coreAnswer, error) { + t.Fatal("the vault was asked by a question that is not a send") + return coreAnswer{}, nil + } + for _, choosing := range []Choosing{Reading, Foreseeing} { + if got := coreBoxesOf("anchor", key, core, kept, choosing, never); string(got.sealed["nats"]["broker"]) != `{"v":1}` { + t.Errorf("read %s", got.sealed["nats"]["broker"]) + } + if got := coreBoxesOf("anchor", key, core, nil, choosing, never); string(got.sealed["nats"]["broker"]) != + string(catalogue.ForeseenVaultBox) { + t.Errorf("with none kept, read %s", got.sealed["nats"]["broker"]) + } + } + // A box kept for another key of the machine is never carried. + other := coreBoxesOf("anchor", aPublicKey(t), core, kept, Allocating, + func(string, string) (coreAnswer, error) { return coreAnswer{}, errors.New("silent") }) + if _, left := other.notPlaced["nats"]; !left { + t.Fatalf("a box sealed to the machine's old key was carried: %+v", other) + } +} + +// The core's modules on a machine: on the list, not handed, with their own secrets but one issued outside. +func TestTheCoresModulesAreTheListsWithTheirOwnSecrets(t *testing.T) { + modules := []catalogue.Manifest{ + {Module: "nats", OwnSecrets: catalogue.OwnSecrets{"broker": {Path: "/a"}, "given": {Path: "/b", + IssuedBy: catalogue.IssuedOutside}}}, + {Module: "postgres", OwnSecrets: catalogue.OwnSecrets{"superuser": {Path: "/c"}}}, + {Module: "mesh-provisioner", OwnSecrets: catalogue.OwnSecrets{"broker": {Path: "/d"}}}, + } + got := coreModulesOf(modules, func(m catalogue.Manifest) bool { return m.Module == "mesh-provisioner" }) + if want := map[string][]string{"nats": {"broker"}}; !reflect.DeepEqual(got, want) { + t.Fatalf("got %v, want %v", got, want) + } +} diff --git a/cmd/mesh-controller/deadletters_aging.go b/cmd/mesh-controller/deadletters_aging.go new file mode 100644 index 00000000..e7b2b466 --- /dev/null +++ b/cmd/mesh-controller/deadletters_aging.go @@ -0,0 +1,117 @@ +package main + +import ( + "context" + "fmt" + "sort" + "strings" + "time" + + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/link" +) + +// A dead letter from a restricted stream ages out with its stream (novox/hq ADR 0376 §3, §6). +// +// DEAD_LETTERS lets such a message go when its stream's maximum age runs out, so a consumer's `max-deliveries` +// condition can clear with nobody having delivered it again or dropped it. That is said: the watchdogs remember, +// from one look to the next, when each restricted dead letter ages out, count those gone once their time passed, +// say each in the controller's log as it goes, and clear the condition with words saying how many aged out +// undelivered and from which stream. + +// agingTally is what the watchdogs remember of restricted dead letters between looks. +type agingTally struct { + // seen is each consumer's restricted dead letters at the last look, by id, with when each ages out. + seen map[string]map[uint64]time.Time + // agedOut is how many of each consumer's aged out undelivered while its condition was open. + agedOut map[string]int +} + +// look compares this look's restricted dead letters with the last: newly is how many of each consumer's aged out +// since, and cleared how many aged out in all for each consumer DEAD_LETTERS no longer holds anything of, when any +// did. held is every consumer's count this look; aging its restricted dead letters. +func (t *agingTally) look(now time.Time, held map[string]int, aging map[string][]link.Aging) (newly, cleared map[string]int) { + if t.seen == nil { + t.seen, t.agedOut = map[string]map[uint64]time.Time{}, map[string]int{} + } + newly, cleared = map[string]int{}, map[string]int{} + for key, before := range t.seen { + still := map[uint64]bool{} + for _, a := range aging[key] { + still[a.ID] = true + } + for id, agesOut := range before { + if !still[id] && !agesOut.After(now) { + newly[key]++ + } + } + if newly[key] == 0 { + delete(newly, key) + } + t.agedOut[key] += newly[key] + } + for key, n := range t.agedOut { + if held[key] > 0 { + continue + } + if n > 0 { + cleared[key] = n + } + delete(t.agedOut, key) + } + t.seen = map[string]map[uint64]time.Time{} + for key, letters := range aging { + t.seen[key] = map[uint64]time.Time{} + for _, a := range letters { + t.seen[key][a.ID] = a.AgesOut + } + } + return newly, cleared +} + +// agedOutResolved is the line a consumer's condition clears with when messages it gave up on aged out undelivered. +func agedOutResolved(key string, n int) string { + stream, _, _ := strings.Cut(key, ".") + messages := "a message" + if n > 1 { + messages = fmt.Sprintf("%d messages", n) + } + return fmt.Sprintf("Resolved: %s of the stream %s aged out undelivered", messages, stream) +} + +// sayAgedOut says, in the log, what aged out since the last look, and clears the condition of every consumer whose +// dead letters are all gone with the words saying how many aged out. Before the rows are reconciled, so the +// condition is cleared with these words and not its own. +func (w *watchdogs) sayAgedOut(ctx context.Context, f *signalFacts) []string { + if f.advisoriesErr != nil || f.deadLettersAging == nil { + return nil + } + newly, cleared := w.aging.look(f.now, f.deadLetters, f.deadLettersAging) + var problems []string + for _, key := range countedKeys(newly) { + stream, consumer, _ := strings.Cut(key, ".") + fmt.Printf("%d message(s) %s gave up on aged out of %s undelivered: %s keeps them only for its maximum age "+ + "(novox/hq ADR 0376)\n", newly[key], link.ConsumerInWords(stream, consumer), "DEAD_LETTERS", stream) + } + for _, key := range countedKeys(cleared) { + stream, consumer, _ := strings.Cut(key, ".") + fmt.Printf("%s no longer has a message kept: %d aged out of %s undelivered, so its condition clears "+ + "(novox/hq ADR 0376)\n", link.ConsumerInWords(stream, consumer), cleared[key], stream) + if _, err := w.keeper.ClearSaying(ctx, conditions.Key(conditions.ScopeBus, key, link.AdvisoryMaxDeliveries), + fmt.Sprintf("its %d dead letter(s) of %s aged out undelivered", cleared[key], stream), + agedOutResolved(key, cleared[key])); err != nil { + problems = append(problems, err.Error()) + } + } + return problems +} + +// countedKeys is a count's keys in order. +func countedKeys(m map[string]int) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} diff --git a/cmd/mesh-controller/deadletters_aging_test.go b/cmd/mesh-controller/deadletters_aging_test.go new file mode 100644 index 00000000..867f97b0 --- /dev/null +++ b/cmd/mesh-controller/deadletters_aging_test.go @@ -0,0 +1,69 @@ +package main + +import ( + "context" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/link" +) + +// **A dead letter that ages out clears its condition and says so** (novox/hq ADR 0376 §6), on a fake clock: while +// DEAD_LETTERS holds a restricted stream's messages for a consumer its max-deliveries condition is open; once they +// have aged out it clears, with words saying how many aged out undelivered and from which stream. One dropped before +// its age is not counted as aged out, and a consumer of EVENTS clears as before. +func TestADeadLetterThatAgesOutClearsItsConditionAndSaysSo(t *testing.T) { + now := time.Date(2026, 10, 15, 12, 0, 0, 0, time.UTC) + store := conditions.NewInMemory() + told := &conditions.Told{} + k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store, Teller: told, + Now: func() time.Time { return now }}) + w := &watchdogs{keeper: k, started: now.Add(-time.Hour)} + + look := func(at time.Time, held map[string]int, aging map[string][]link.Aging) { + f := calm(at) + f.deadLetters, f.deadLettersAging = held, aging + w.see(t.Context(), f) + } + mail := "MAIL.ace_mailer" + look(now, map[string]int{mail: 3, "EVENTS.media_sonarr": 1}, map[string][]link.Aging{mail: { + {ID: 1, AgesOut: now.Add(time.Hour)}, {ID: 2, AgesOut: now.Add(2 * time.Hour)}, {ID: 3, AgesOut: now.Add(3 * time.Hour)}}}) + if open, _ := k.Open(t.Context()); len(open) != 2 { + t.Fatalf("open: %+v", open) + } + // One dropped by a person before its age: not aged out. + look(now.Add(30*time.Minute), map[string]int{mail: 2, "EVENTS.media_sonarr": 1}, map[string][]link.Aging{mail: { + {ID: 2, AgesOut: now.Add(2 * time.Hour)}, {ID: 3, AgesOut: now.Add(3 * time.Hour)}}}) + look(now.Add(150*time.Minute), map[string]int{mail: 1}, map[string][]link.Aging{mail: {{ID: 3, AgesOut: now.Add(3 * time.Hour)}}}) + open, _ := k.Open(t.Context()) + if len(open) != 1 || open[0].Key != conditions.Key(conditions.ScopeBus, mail, link.AdvisoryMaxDeliveries) { + t.Fatalf("after one aged out: %+v", open) + } + look(now.Add(4*time.Hour), map[string]int{}, map[string][]link.Aging{}) + if open, _ := k.Open(t.Context()); len(open) != 0 { + t.Fatalf("after all aged out: %+v", open) + } + k.Close(context.Background()) // everything said is said once it closes + var mailCleared, sonarrCleared string + for _, e := range told.Said() { + if e.Change != conditions.ChangeCleared { + continue + } + switch e.Condition.Key { + case conditions.Key(conditions.ScopeBus, mail, link.AdvisoryMaxDeliveries): + mailCleared = e.Condition.Resolved + case conditions.Key(conditions.ScopeBus, "EVENTS.media_sonarr", link.AdvisoryMaxDeliveries): + sonarrCleared = e.Condition.Resolved + } + } + if mailCleared != "Resolved: 2 messages of the stream MAIL aged out undelivered" { + t.Fatalf("MAIL's condition cleared with %q", mailCleared) + } + if sonarrCleared != "Resolved: the messages it gave up on were delivered again or dropped" { + t.Fatalf("an EVENTS consumer's condition cleared with %q", sonarrCleared) + } + if why, ok := conditions.Plain(mailCleared); !ok { + t.Fatalf("not plain: %s", why) + } +} diff --git a/cmd/mesh-controller/delivery.go b/cmd/mesh-controller/delivery.go index 40b7c489..8db2bca5 100644 --- a/cmd/mesh-controller/delivery.go +++ b/cmd/mesh-controller/delivery.go @@ -952,7 +952,7 @@ func deliveryCommand(ctx context.Context, args []string) error { if *group == "" && len(members) == 1 { // One head, checked again as the forge's announcement would have it (a recheck): the controller's // own path for a pull request, run because the delivery's owner asked. - if err := sayingOnTheBus(ctx, inv, func() error { return (following{open}).PullUpdated(ctx, members[0].pull()) }); err != nil { + if err := sayingOnTheBus(ctx, inv, func() error { return (following{open: open, oneShot: true}).PullUpdated(ctx, members[0].pull()) }); err != nil { return err } return answer(map[string]any{"rechecked": members[0].ID}) @@ -1051,7 +1051,9 @@ func appliedFrom(ctx context.Context, inv *inventory.Inventory) inventory.Applie fmt.Fprintf(os.Stderr, "the report of %s after %s could not be read: %v\n", node, walkSent.Format(time.RFC3339), err) return inventory.AppliedReport{}, false } - return r, ok + // And, for each module the send handed to the provisioner, its client's report on that build (novox/hq ADR + // 0362): applied only once both said it, at the later; failed if the client failed. + return inventory.ProvisionedSend(node, send, r, ok, provLookup(ctx)) } } diff --git a/cmd/mesh-controller/desk_secret.go b/cmd/mesh-controller/desk_secret.go index a0da69d4..b8cabc90 100644 --- a/cmd/mesh-controller/desk_secret.go +++ b/cmd/mesh-controller/desk_secret.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "log" "strings" "time" @@ -54,6 +55,10 @@ type deskGive struct { trusted func(module string) (bool, error) // ask asks one machine's node-launcher.secret and answers its result, or the holder's refusal. ask func(machine string, args map[string]any) (json.RawMessage, error) + // promptFor is the one line the secret's declaration asks for it as (novox/hq issue 789): read from the + // controller's own catalogue, never from the call or the module's runtime. Nil, or an empty answer, asks by + // name alone, as the desk does today. + promptFor func(module, name string) (string, error) // accept seals the value as `secret accept` does, and says whether it lives until the module's start. accept func(value string) (untilStart bool, err error) // record writes the act in the hand-act log. @@ -67,6 +72,9 @@ type deskGive struct { // nil keeps no record (a test that does not look). end closes it with how it ended. open func(node, module, name, desk string) (int64, error) end func(id int64, outcome string) error + // vault, when set, is the path of a module whose secrets are filled the new way (vault_secrets.go): the vault + // records the ask, holds its bounds and opens what is typed; open, end and accept are not used. + vault vaultAsker } // askedByName is the caller as the prompt names it: the first clause of what the bus said, in the characters @@ -133,6 +141,9 @@ func (d deskGive) give(node, module, name, desk string) (string, error) { "bus, where an agent may answer first (novox/hq ADR 0259 §10)", module, name, node, module, name) } } + if d.vault != nil { + return d.giveToVault(node, module, name, desk) + } // The bounds, read and kept before anybody is asked to type (novox/hq ADR 0277): one open prompt per secret, // few an hour. How the ask ends is recorded whatever happens below. outcome := "failed" @@ -153,15 +164,28 @@ func (d deskGive) give(node, module, name, desk string) (string, error) { } // By name, never by words: the holder writes the prompt from these, and says the controller asks, which // the bus alone makes true (broker.ControllerOnly). Who asked is the bus's word on the caller, cut to a - // name's characters — never an argument of the call. - raw, err := d.ask(desk, map[string]any{ + // name's characters — never an argument of the call. `what` is the catalogue's own words for the secret + // (novox/hq issue 789), never a word the caller chose; omitted when the module declares none, so an old + // desk and an old module behave as today. + args := map[string]any{ "module": module, "secret": name, "node": node, "asked_by": askedByName(d.askedBy), "seal_to": public, "timeout_seconds": deskPromptWithin, - }) + } + if d.promptFor != nil { + what, err := d.promptFor(module, name) + if err != nil { + outcome = "refused" + return "", fmt.Errorf("nobody was asked to type anything: %w", err) + } + if what != "" { + args["what"] = what + } + } + raw, err := d.ask(desk, args) if err != nil { outcome = "refused" return "", fmt.Errorf("the desk on %s could not be asked: %w", desk, err) @@ -236,8 +260,9 @@ func askAtDesk(ctx context.Context, node, module, name, desk string) error { return err } defer open.Close() + askedBy := link.Caller() d := deskGive{ - askedBy: link.Caller(), + askedBy: askedBy, open: func(node, module, name, desk string) (int64, error) { return open.inventory.OpenSecretAsk(ctx, node, module, name, askedByName(link.Caller()), desk) }, @@ -247,7 +272,8 @@ func askAtDesk(ctx context.Context, node, module, name, desk string) error { _, err := open.inventory.NodeByName(ctx, machine) return err }, - trusted: func(module string) (bool, error) { return open.inventory.RunsAsItsOwnAccount(ctx, module) }, + trusted: func(module string) (bool, error) { return open.inventory.RunsAsItsOwnAccount(ctx, module) }, + promptFor: func(module, name string) (string, error) { return open.inventory.OwnSecretPrompt(ctx, module, name) }, ask: func(machine string, args map[string]any) (json.RawMessage, error) { var result json.RawMessage err := onTheBus(func(conn *nats.Conn) error { @@ -273,7 +299,18 @@ func askAtDesk(ctx context.Context, node, module, name, desk string) error { return err }) }, - announce: func(node, module, name, how string) error { return announceSecretGiven(ctx, node, module, name, how) }, + announce: func(node, module, name, how string) error { + return announceSecretGiven(ctx, node, module, name, how, askedBy) + }, + } + // A module whose secrets are filled the new way is asked for by the vault (ADR 0349 clause 9); every other + // keeps today's path (clause 17). + ours, err := filledTheNewWay(module) + if err != nil { + return fmt.Errorf("nothing was asked: %w", err) + } + if ours { + d.vault = theVault(ctx) } words, err := d.give(node, module, name, desk) if err != nil { @@ -324,10 +361,85 @@ func secretNameWords(name string) string { return strings.Join(strings.FieldsFunc(name, func(r rune) bool { return r == '-' || r == '_' || r == '.' }), " ") } -// announceSecretGiven raises it on this controller's keeper. -func announceSecretGiven(ctx context.Context, node, module, name, how string) error { - return withKeeper(ctx, func(k *conditions.Keeper) error { - _, err := k.Observe(ctx, secretGivenObservation(node, module, name, how, time.Now())) - return err +// tellWithin is how long the operator channel's `tell` is given to answer: well under a human's patience, +// and short because nothing in the call waits on the operator — it asks nothing (novox/hq ADR 0380). +const tellWithin = 10 * time.Second + +// askOperatorChannel asks the operator channel's seat (the holder issue 797 gives it) its `tell` verb, +// through the controller's grant (novox/hq ADR 0380 §8), over this controller's own bus connection; a seam +// a test replaces whole. The holder's own refusal is an error naming it; an absent holder or a refused +// grant answer as link.ErrNothingServes or link.ErrGrantRefused (checked with errors.Is), so the caller can +// tell the two apart from any other failure. +var askOperatorChannel = func(ctx context.Context, args map[string]any) (json.RawMessage, error) { + var result json.RawMessage + err := onTheBus(func(conn *nats.Conn) error { + answer, err := link.AskMeshSeatTool(ctx, conn, "operator-channel", "tell", args, tellWithin) + if err != nil { + return err + } + if answer.Error != "" { + return fmt.Errorf("operator-channel.tell refused: %s", answer.Error) + } + result = answer.Result + return nil }) + return result, err +} + +// announceSecretGiven tells the operator, once, that a module's own secret was given (novox/hq ADR 0380): +// one plain message through the operator channel's `tell`, naming the secret, the module, the machine, +// where and when it was given and who asked — asking nothing, and opening no condition. **The controller's +// change waits for that verb at run time, not at merge** (clause 9): while nothing on the bus yet serves it, +// or the controller's own grant is refused, the secret-given condition is raised exactly as before ADR 0380 +// — any other failure (the holder itself refusing, or answering something unreadable) is the announcement's +// own failure, said to the caller as it always was. +func announceSecretGiven(ctx context.Context, node, module, name, how, askedBy string) error { + at := time.Now() + body := map[string]any{ + "headline": fmt.Sprintf("A password was entered for %s's %s on %s", module, secretNameWords(name), node), + "subject": node, + "lines": []string{ + fmt.Sprintf("%s at %s, asked by %s.", how, at.Local().Format("15:04"), askedByName(askedBy)), + "If that was not you, give it again yourself at the controller's terminal.", + }, + } + _, told := askOperatorChannel(ctx, body) + switch { + case told == nil: + clearSecretGivenConditions(ctx) + return nil + case errors.Is(told, link.ErrNothingServes), errors.Is(told, link.ErrGrantRefused): + log.Printf("secret-given: %s's %s on %s could not be told through the operator channel (%v); the "+ + "secret-given condition is raised as it was before novox/hq ADR 0380 (its clause 9)", module, name, + node, told) + return withKeeper(ctx, func(k *conditions.Keeper) error { + _, err := k.Observe(ctx, secretGivenObservation(node, module, name, how, at)) + return err + }) + default: + return told + } +} + +// clearSecretGivenConditions retires every open secret-given condition once a plain message has replaced +// it (novox/hq ADR 0380 §10): called after the first time a secret-given message is told successfully. No +// resolved line is sent — the condition is simply gone — and why is kept in the keeper's own history. +func clearSecretGivenConditions(ctx context.Context) { + if err := withKeeper(ctx, func(k *conditions.Keeper) error { + open, err := k.Open(ctx) + if err != nil { + return err + } + for _, c := range open { + if c.Kind != kindSecretGiven { + continue + } + if _, err := k.ClearSaying(ctx, c.Key, "retired by novox/hq ADR 0380", ""); err != nil { + return err + } + } + return nil + }); err != nil { + log.Printf("secret-given: the open secret-given conditions, retired by ADR 0380, could not be cleared: %v", err) + } } diff --git a/cmd/mesh-controller/desk_secret_test.go b/cmd/mesh-controller/desk_secret_test.go index 170df795..85bbb40e 100644 --- a/cmd/mesh-controller/desk_secret_test.go +++ b/cmd/mesh-controller/desk_secret_test.go @@ -456,6 +456,52 @@ func TestThePromptNamesWhoAskedFromTheBussWordAlone(t *testing.T) { } } +// The desk's prompt names which account of a family is asked for, from the catalogue's own words (novox/hq +// issue 789): a family declaring `prompt` with `{member}` sends `what` with the member's suffix filled in. +func TestThePromptNamesWhichAccountOfAFamilyIsAsked(t *testing.T) { + d, _, _, asked := aDesk(t, sealedTo(t, typed)) + d.declares = func(module, name string) error { + if module != "mesh-mail" || name != "mail-credential-gmail" { + return errors.New(module + " does not declare " + name + " as an own secret") + } + return nil + } + d.promptFor = func(module, name string) (string, error) { + if module == "mesh-mail" && name == "mail-credential-gmail" { + return "the password of the mail account gmail", nil + } + return "", nil + } + if _, err := d.give("anchor", "mesh-mail", "mail-credential-gmail", "laptop"); err != nil { + t.Fatal(err) + } + if got := (*asked)[0]["what"]; got != "the password of the mail account gmail" { + t.Errorf(`"what" = %v`, got) + } +} + +// Without a declared prompt, no `what` is sent: an old module, or one that declares none, is asked by name alone, +// as the desk does today (novox/hq issue 789). +func TestWithNoDeclaredPromptNoWhatIsSent(t *testing.T) { + d, _, _, asked := aDesk(t, sealedTo(t, typed)) + d.promptFor = func(module, name string) (string, error) { return "", nil } + if _, err := d.give("anchor", "telegram", "telegram-token", "laptop"); err != nil { + t.Fatal(err) + } + if _, there := (*asked)[0]["what"]; there { + t.Errorf(`"what" was sent with no declared prompt: %v`, (*asked)[0]) + } + // And with no promptFor wired at all (an older caller of deskGive), same thing: `give` still works, asking + // nothing more than it does today. + d2, _, _, asked2 := aDesk(t, sealedTo(t, typed)) + if _, err := d2.give("anchor", "telegram", "telegram-token", "laptop"); err != nil { + t.Fatal(err) + } + if _, there := (*asked2)[0]["what"]; there { + t.Errorf(`"what" was sent with promptFor nil: %v`, (*asked2)[0]) + } +} + // An ask for a secret is bounded before anybody is asked to type (ADR 0277): the record refuses it, nothing is // asked; and how every ask ends is recorded. func TestASecretAskIsBoundedAndItsEndRecorded(t *testing.T) { @@ -510,3 +556,172 @@ func TestASecretAskIsNeverASecretRead(t *testing.T) { } } } + +// askOperatorChannelFake replaces askOperatorChannel for the test and records every call it was given. +func askOperatorChannelFake(t *testing.T, answer func(args map[string]any) (json.RawMessage, error)) *[]map[string]any { + t.Helper() + var calls []map[string]any + was := askOperatorChannel + askOperatorChannel = func(_ context.Context, args map[string]any) (json.RawMessage, error) { + calls = append(calls, args) + return answer(args) + } + t.Cleanup(func() { askOperatorChannel = was }) + return &calls +} + +// announceWithRealTell wires a deskGive's announce to the real announceSecretGiven, as askAtDesk does, +// rather than the fakes the tests above use to look at what `give` does around it. +func announceWithRealTell(ctx context.Context, askedBy string) func(node, module, name, how string) error { + return func(node, module, name, how string) error { + return announceSecretGiven(ctx, node, module, name, how, askedBy) + } +} + +// novox/hq ADR 0380 clauses 1-2: a secret given at the desk is told to the operator once, in a plain +// message through the operator channel's `tell`, naming the secret, the module, where and when and who +// asked, and opens no condition — the kind the review of 2026-10-09 (M4) raised is retired. +func TestASecretGivenAtTheDeskIsToldOnceAndOpensNoCondition(t *testing.T) { + ctx := context.Background() + k, _ := withConditionsInMemory(t) + calls := askOperatorChannelFake(t, func(map[string]any) (json.RawMessage, error) { return nil, nil }) + d, _, _, _ := aDesk(t, sealedTo(t, typed)) + d.announce = announceWithRealTell(ctx, "mail-seats-835@novox") + if _, err := d.give("anchor", "telegram", "telegram-token", "laptop"); err != nil { + t.Fatal(err) + } + if len(*calls) != 1 { + t.Fatalf("told the operator %d times, want exactly 1: %v", len(*calls), *calls) + } + call := (*calls)[0] + if call["headline"] != "A password was entered for telegram's telegram token on anchor" { + t.Errorf("headline %v", call["headline"]) + } + if call["subject"] != "anchor" { + t.Errorf("subject %v", call["subject"]) + } + lines, _ := call["lines"].([]string) + if len(lines) != 2 || !strings.Contains(lines[0], "at the desk on laptop at") || + !strings.Contains(lines[0], "asked by mail-seats-835@novox") || + lines[1] != "If that was not you, give it again yourself at the controller's terminal." { + t.Errorf("lines %v", lines) + } + open, err := k.Open(ctx) + if err != nil { + t.Fatal(err) + } + if len(open) != 0 { + t.Errorf("a condition was raised though the operator was told: %+v", open) + } +} + +// novox/hq ADR 0380 §9: until the operator channel's `tell` is delivered — nothing yet serves it, or the +// controller's own grant is refused — the secret-given condition is raised exactly as before. +func TestASecretGivenAtTheDeskRaisesTheConditionWhileNothingServesTell(t *testing.T) { + ctx := context.Background() + for _, failure := range []error{link.ErrNothingServes, link.ErrGrantRefused} { + k, _ := withConditionsInMemory(t) + askOperatorChannelFake(t, func(map[string]any) (json.RawMessage, error) { return nil, failure }) + d, _, _, _ := aDesk(t, sealedTo(t, typed)) + d.announce = announceWithRealTell(ctx, "mesh-mail-835, novox") + if _, err := d.give("anchor", "telegram", "telegram-token", "laptop"); err != nil { + t.Fatal(err) + } + open, err := k.Open(ctx) + if err != nil { + t.Fatal(err) + } + if len(open) != 1 || open[0].Kind != kindSecretGiven { + t.Fatalf("%v: the condition was not raised as before: %+v", failure, open) + } + } +} + +// novox/hq ADR 0380 §10: once the operator channel is told successfully, every secret-given condition +// still open from before this change is retired, without a resolved line of its own. +func TestASuccessfulTellRetiresEveryOpenSecretGivenCondition(t *testing.T) { + ctx := context.Background() + k, _ := withConditionsInMemory(t) + for _, module := range []string{"telegram", "mesh-mail"} { + if _, err := k.Observe(ctx, secretGivenObservation("anchor", module, "a-secret", "at the desk on laptop", + time.Now())); err != nil { + t.Fatal(err) + } + } + askOperatorChannelFake(t, func(map[string]any) (json.RawMessage, error) { return nil, nil }) + if err := announceSecretGiven(ctx, "anchor", "telegram", "telegram-token", "at the desk on laptop", "me"); err != nil { + t.Fatal(err) + } + open, err := k.Open(ctx) + if err != nil { + t.Fatal(err) + } + if len(open) != 0 { + t.Fatalf("the open secret-given conditions were not retired: %+v", open) + } +} + +// novox/hq ADR 0380 clause 7: a trusted party's secret given at the controller's terminal is announced +// before it is kept, and not kept when the announcement fails. It still fails when telling the operator +// AND raising the fallback condition both fail — nothing silently keeps a trusted party's value unheard. +func TestATrustedPartysSecretAtTheTerminalIsNotKeptWhenTellAndTheConditionBothFail(t *testing.T) { + ctx := context.Background() + askOperatorChannelFake(t, func(map[string]any) (json.RawMessage, error) { return nil, link.ErrNothingServes }) + // conditionsFrom is left nil and there is no bus to dial here, so the fallback condition cannot be + // raised either: both of ADR 0380's ways to tell the operator fail. + kept := false + if _, _, err := keepGiven(true, + func() error { + return announceSecretGiven(ctx, "anchor", "telegram", "telegram-token", "at the controller's terminal", "me") + }, + func() (bool, error) { kept = true; return false, nil }); err == nil { + t.Error("a trusted party's secret was kept though telling the operator and raising the condition both failed") + } + if kept { + t.Error("the value was kept despite the failure") + } +} + +// Review round 2 of mesh-controller (feat/mail-seats, head 5f24a47d): ADR 0380 §9's fallback — raising the +// secret-given condition — answers only link.ErrNothingServes and link.ErrGrantRefused. Any other failure of +// the operator channel's `tell` (the holder itself refusing, or answering something unreadable) is the +// announcement's own failure, surfaced to the caller, and opens no condition. +func TestAnOperatorChannelFailureOtherThanNothingServesOrGrantRefusedSurfacesAndRaisesNoCondition(t *testing.T) { + ctx := context.Background() + k, _ := withConditionsInMemory(t) + refused := errors.New("operator-channel.tell refused: too many lines") + askOperatorChannelFake(t, func(map[string]any) (json.RawMessage, error) { return nil, refused }) + if err := announceSecretGiven(ctx, "anchor", "telegram", "telegram-token", "at the desk on laptop", "me"); err != refused { + t.Fatalf("announceSecretGiven = %v, want the refusal itself surfaced", err) + } + open, err := k.Open(ctx) + if err != nil { + t.Fatal(err) + } + if len(open) != 0 { + t.Errorf("a condition was raised though the failure was neither ErrNothingServes nor ErrGrantRefused: %+v", open) + } +} + +// Review round 2 of mesh-controller: a secret accepted at the controller's terminal writes a hand act naming +// who asked, where and when it was given — and never the value (novox/hq ADR 0380 §6). +func TestASecretAcceptedAtTheTerminalWritesAHandActNamingWhoWhereAndNeverTheValue(t *testing.T) { + act := terminalGiveHandAct("anchor", "telegram", "telegram-token", "mail-seats-835@novox") + if act.Verb != "secret accept" || act.Cause != "given-at-the-terminal" { + t.Fatalf("act %+v", act) + } + if !strings.Contains(act.Why, "at the controller's terminal") { + t.Errorf("the act does not name where: %q", act.Why) + } + if !strings.Contains(act.Why, "asked by mail-seats-835@novox") { + t.Errorf("the act does not name who: %q", act.Why) + } + if strings.Contains(act.Why, typed) { + t.Error("the act carries the value") + } + // When is RecordHandAct's own job (it stamps At with time.Now when the act leaves none, and By with the + // caller when the act leaves that empty too): this act leaves both for it to fill. + if act.By != "" || !act.At.IsZero() { + t.Errorf("the act sets its own who/when instead of leaving them to RecordHandAct: %+v", act) + } +} diff --git a/cmd/mesh-controller/doctor.go b/cmd/mesh-controller/doctor.go index a79e5675..93d9799e 100644 --- a/cmd/mesh-controller/doctor.go +++ b/cmd/mesh-controller/doctor.go @@ -146,6 +146,11 @@ var probeRegistry = []probe{ {ID: probeFillID, Asserts: "every module's bucket and log holds less than three quarters of its cap; one " + "said filling is cleared once it holds less than seven tenths", From: "ADR 0297, issue 501", Kind: kindBucketOrLogFilling, Raises: []string{kindBucketOrLogUnread}, Phase: 1, run: probeFill}, + // A grant of a restricted stream that composes nothing (novox/hq ADR 0376 §2): the module is carried by the + // machine's tool runner, which never hears a restricted stream, so it hears nothing it was granted. + {ID: probeWithheldID, Asserts: "no grant of a restricted stream names a module that composes nothing for it: every " + + "module granted one runs as an account of its own", From: "ADR 0376 §2", Kind: kindGrantComposesNothing, + Phase: 1, run: probeWithheldHearing}, {ID: "DW", Asserts: "the watchdogs of the signals table ran within three of their intervals", From: "ADR 0227 rule 6: the watchers are watched", Kind: "watchdogs-silent", Phase: 1, run: probeWatchdogs}, // The core's health definitions (novox/hq to-be 45 §8, ADR 0236): what a core component's new build is diff --git a/cmd/mesh-controller/found_wait_test.go b/cmd/mesh-controller/found_wait_test.go index cf8eba68..caa75ac7 100644 --- a/cmd/mesh-controller/found_wait_test.go +++ b/cmd/mesh-controller/found_wait_test.go @@ -22,6 +22,7 @@ func foundDirectory(module string, since time.Time) inventory.ResourceHealth { } func TestADirectoryFoundBeforeTheSendIsAWaitForAPerson(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) now := time.Now() sent := now.Add(-time.Minute) f := gateFacts{now: now, health: map[string]inventory.NodeHealth{"laptop": {Node: "laptop", HeardAt: now, @@ -48,6 +49,7 @@ func TestADirectoryFoundBeforeTheSendIsAWaitForAPerson(t *testing.T) { // The whole walk: a module whose directory was used as found long before still gets its fix to every machine, // its pass kept and the wait said; a directory this very send found holds it and puts it back. func TestAFixGoesThroughPastADirectoryFoundBefore(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) for _, c := range []struct { name string found time.Duration // when the directory was found, against now @@ -116,6 +118,7 @@ func TestAFixGoesThroughPastADirectoryFoundBefore(t *testing.T) { // The condition says the wait in its own kind: the operator's, never urgent, and cleared once handed over. func TestADirectoryUsedAsFoundIsItsOwnCondition(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) k, _ := withConditionsInMemory(t) ctx := t.Context() rs := map[string][]inventory.ResourceHealth{"notes": {foundDirectory("notes", time.Now().Add(-time.Hour))}} diff --git a/cmd/mesh-controller/gate.go b/cmd/mesh-controller/gate.go index 8a1e1a1c..45792f00 100644 --- a/cmd/mesh-controller/gate.go +++ b/cmd/mesh-controller/gate.go @@ -235,6 +235,10 @@ type gateFacts struct { sent map[string]inventory.SentDeclaration sentBuilds map[string]map[string]string commits map[string]string + // provisioned answers each machine's provisioner client's newest report of a module (novox/hq ADR 0349 §15): + // a module the gate's send handed to the provisioner is healthy only once it reported that build placed. Nil + // reads as nothing said. + provisioned inventory.ProvLookup } // heldReading is the provider a module's findings are held under, as " on ", and, when that @@ -291,7 +295,7 @@ func (f gateFacts) supersededOn(module, machine string) (string, bool) { var gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) { inv := open.inventory f := gateFacts{now: time.Now(), reports: map[string]inventory.Reported{}, engines: map[string]string{}, - rolledBack: witnessed.all()} + rolledBack: witnessed.all(), provisioned: provLookup(ctx)} reports, err := inv.LastReports(ctx) if err != nil { return f, err @@ -368,6 +372,12 @@ func judgeHealth(module, component string, m catalogue.Manifest, machine string, case r.Outcome != inventory.OutcomeApplied: return healthNotYet, fmt.Sprintf("%s reported %q", machine, r.Outcome) } + // **And its files placed** (novox/hq ADR 0349 §15): a module the send handed to the provisioner is judged only + // once that machine's client reported the build's templates placed, so a module whose files are not yet written + // is never judged; a failure there is the build's on that machine. + if h, why := judgeProvisioned(module, machine, r, f); h != healthGood { + return h, why + } // **No new condition about it**: about the machine itself, or naming the module on that machine, // raised since the judging began. The gate's own are not evidence about the build. A fault that was // there at the send and reopened since is not new; one that had cleared before the send and came back @@ -449,6 +459,32 @@ func judgeHealth(module, component string, m catalogue.Manifest, machine string, return healthGood, "" } +// judgeProvisioned is whether a module's files are placed on a machine, for a module the gate's send handed to the +// provisioner there: healthy for any other. +func judgeProvisioned(module, machine string, r inventory.Reported, f gateFacts) (health, string) { + sent, kept := f.sent[machine] + digest := sent.Provisioned[module] + if !kept || digest == "" { + return healthGood, "" + } + at := time.Time{} + if r.At != nil { + at = *r.At + } + one := inventory.SentDeclaration{Digest: sent.Digest, At: sent.At, Provisioned: map[string]string{module: digest}} + got, ok := inventory.ProvisionedSend(machine, one, inventory.AppliedReport{At: at, + Outcome: inventory.OutcomeApplied}, true, f.provisioned) + switch { + case !ok: + return healthNotYet, fmt.Sprintf("the provisioner on %s has not reported %s's files placed for the new "+ + "build", machine, module) + case got.Outcome != inventory.OutcomeApplied: + return healthBroken, fmt.Sprintf("the provisioner on %s did not place %s's files: %s", machine, module, + got.Outcome) + } + return healthGood, "" +} + // machineWord is what one judging found wrong with a machine itself, apart from its modules: facts is // the judging's facts without those conditions, on what each names among the modules the send moved, // and whole what holds the machine back as a whole. diff --git a/cmd/mesh-controller/gate_followup_test.go b/cmd/mesh-controller/gate_followup_test.go index b8377d77..a78963dd 100644 --- a/cmd/mesh-controller/gate_followup_test.go +++ b/cmd/mesh-controller/gate_followup_test.go @@ -57,6 +57,7 @@ func healthyContainer(module string) inventory.ResourceHealth { // tools' witness put its build back at once, and the send waits for the healthy module's own verdict — a pass, // counted over the gate's spacing — before it says its own. The broken one alone is put back. func TestAFaultBeforeTheSettleTimeWaitsForTheModuleBesideIt(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) judgeLeavesAsTheCore(t) open := aMesh(t) ctx := t.Context() @@ -99,6 +100,7 @@ const broker_runtime = "node-tools" // **A pass is counted only the gate's spacing after the one before** (review (c)): a send judged every tick // while a module beside it is not yet healthy counts the healthy one once. func TestAPassIsCountedNoFasterThanTheGatesSpacing(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) open := aMesh(t) ctx := t.Context() withGateBounds(t, time.Hour, time.Hour, time.Hour) @@ -123,6 +125,7 @@ func TestAPassIsCountedNoFasterThanTheGatesSpacing(t *testing.T) { // **A carried build's pass carries its own wait, never another's** (review (e)). func TestACarriedPassCarriesOnlyItsOwnWait(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) b := aBacklog(t) ctx := t.Context() inv := b.open.inventory @@ -147,6 +150,7 @@ func TestACarriedPassCarriesOnlyItsOwnWait(t *testing.T) { // carried its own module, healthy, and a build waiting on that machine that never became healthy. At the // bound the waiting one is put back and marked; the plan's own module keeps its pass, so no walk waits on it. func TestThePlansOwnModuleKeepsItsPassWhenItsSendFails(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) g := aGateMesh(t) ctx := t.Context() inv := g.open.inventory @@ -227,6 +231,7 @@ func TestThePlansOwnModuleKeepsItsPassWhenItsSendFails(t *testing.T) { // **A provider waiting for a login says who waits on it** (review (g)): its consumer, failing a check that // needs it, is held under it, and the provider's relogin-needed condition lists it and is urgent. func TestAProviderWaitingForALoginSaysWhoWaitsOnIt(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) open := aMesh(t) ctx := t.Context() inv := open.inventory @@ -289,6 +294,7 @@ func TestAProviderWaitingForALoginSaysWhoWaitsOnIt(t *testing.T) { // tools are marked and put back in the very judging that found them broken — their registered build is the // one before, and the laptop is sent it — while `app` goes on being judged, recovers, and keeps its own pass. func TestABrokenModuleIsPutBackAtOnceAndTheOneBesideItGetsItsOwnVerdict(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) judgeLeavesAsTheCore(t) open := aMesh(t) ctx := t.Context() @@ -431,6 +437,7 @@ func TestABrokenModuleIsPutBackAtOnceAndTheOneBesideItGetsItsOwnVerdict(t *testi // **A move from a build not known excuses no wait** (issue 318 review): a plan's own module whose previous // build was not recorded, or whose previous manifest is not kept, cannot be shown to have added the group. func TestAMoveFromAnUnknownBuildExcusesNoWait(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) open := aMesh(t) ctx := t.Context() inv := open.inventory diff --git a/cmd/mesh-controller/gate_test.go b/cmd/mesh-controller/gate_test.go index d8062644..651c90ec 100644 --- a/cmd/mesh-controller/gate_test.go +++ b/cmd/mesh-controller/gate_test.go @@ -36,6 +36,10 @@ func aGateMesh(t *testing.T) *gateMesh { ctx := t.Context() inv := open.inventory g := &gateMesh{open: open, health: map[string]health{}} + // The gate's own tests judge a first machine, as walks a controller before novox/hq ADR 0360 started still are. + wasGated := gateFirstMachine + gateFirstMachine = true + t.Cleanup(func() { gateFirstMachine = wasGated }) g.keeper, _ = withConditionsInMemory(t) g.told = &conditions.Told{} was := doctorFrom @@ -271,6 +275,34 @@ func TestABuildThatPassesItsGateRollsEverywhereUnattended(t *testing.T) { } } +// A build rolls everywhere with no hand and no gate (novox/hq ADR 0360 §1): both machines in one send, no health +// judged, and the walk done once both reported on the send. Before, anchor was judged first, then laptop sent. +func TestABuildRollsToEveryMachineAtOnceWithNoGate(t *testing.T) { + g := aGateMesh(t) + gateFirstMachine = false + ctx := t.Context() + judged := false + wasGather := gatherGateFacts + gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) { + judged = true + return wasGather(ctx, open, component) + } + t.Cleanup(func() { gatherGateFacts = wasGather }) + for i := 0; i < 6; i++ { + advancePlans(ctx, g.open) + } + p := g.plan(t) + if !reflect.DeepEqual(g.sent, [][]string{{"anchor", "laptop"}}) { + t.Fatalf("sent %v, not both machines in one send", g.sent) + } + if s := p.Modules["app"]; p.State != inventory.PlanDone || s.Gate != nil || s.FirstAt != nil || judged { + t.Fatalf("the plan is %s (%s), app %+v, judged %v", p.State, p.Note, s, judged) + } + if obs, err := probeGates(ctx, doctorFrom); err != nil || len(obs) != 0 { + t.Fatalf("a rolled build left a condition: %+v %v", obs, err) + } +} + // A leaf module passes its gate on one reading, as soon as its first machine is healthy, without waiting out the // core's settle time (novox/hq ADR 0282 decision 4 as ADR 0304 decision 3 changed it, issue 524). func TestALeafModulePassesOnOneHealthyReading(t *testing.T) { diff --git a/cmd/mesh-controller/generation.go b/cmd/mesh-controller/generation.go index abf9ff0e..b8d1203b 100644 --- a/cmd/mesh-controller/generation.go +++ b/cmd/mesh-controller/generation.go @@ -2,6 +2,7 @@ package main import ( "context" + "encoding/json" "fmt" "io" "os" @@ -60,12 +61,30 @@ type sentRecord struct { toldGeneration int64 sender string modules []string + // plans is each module the send handed to the provisioner, its plan, as JSON (design 57 P6); nil for none. + plans []byte } // sentRecordOf is a composed send's record: its sender is the caller this process acts for. func sentRecordOf(ctx context.Context, d sendable) sentRecord { - return sentRecord{sequence: d.Sequence, epoch: d.actingEpoch, generation: d.composedFrom, + r := sentRecord{sequence: d.Sequence, epoch: d.actingEpoch, generation: d.composedFrom, toldGeneration: d.Generation, sender: senderOf(callerOf(ctx)), modules: d.modules} + if len(d.plans) > 0 { + kept := map[string]keptPlan{} + for module, p := range d.plans { + kept[module] = keptPlan{Plan: p, RequiredBy: d.requiredBy[module]} + } + // A map of plain strings and slices: it marshals. + r.plans, _ = json.Marshal(kept) + } + return r +} + +// keptPlan is a handed module's plan as a send keeps it (migration 0100): the plan, and for a module only pulled in +// by another, the module that requires it (issue 765, inventory.SentProvisioningPlans). +type keptPlan struct { + catalogue.Plan + RequiredBy string `json:"required-by,omitempty"` } // callerOf is who asked for what this process does: the seat call's caller when ctx belongs to one, the diff --git a/cmd/mesh-controller/given_test.go b/cmd/mesh-controller/given_test.go index 827ae1b7..8a45a465 100644 --- a/cmd/mesh-controller/given_test.go +++ b/cmd/mesh-controller/given_test.go @@ -8,7 +8,8 @@ import ( ) // A rotation asked through the seat carries why to the command, which records it in the hand-act -// log (novox/hq ADR 0228); why with a provision is refused as passed over, not dropped. +// log (novox/hq ADR 0228); and with a provision too, since the vault asks why of every rotation of a +// credential it holds (issue 767). func TestARotationThroughTheSeatCarriesWhy(t *testing.T) { argv, err := argvFor("rotate", map[string]any{"node": "anchor", "module": "letta", "secret": "server-password", "why": "leaked into logs", "cause": "leaked"}) @@ -19,8 +20,10 @@ func TestARotationThroughTheSeatCarriesWhy(t *testing.T) { if err != nil || strings.Join(argv, " ") != "secret rotate anchor letta server-password" { t.Fatalf("without why: %v %v", argv, err) } - if argv, err := argvFor("rotate", map[string]any{"provision": "postgres-database", "why": "leaked"}); err == nil { - t.Fatalf("why beside a provision was passed over: %v", argv) + argv, err = argvFor("rotate", map[string]any{"provision": "postgres-database", "consumer": "ace", "why": "leaked", + "cause": "leaked"}) + if err != nil || strings.Join(argv, " ") != "rotate postgres-database --consumer ace --why leaked --cause leaked" { + t.Fatalf("why beside a provision: %v %v", argv, err) } } diff --git a/cmd/mesh-controller/handacts.go b/cmd/mesh-controller/handacts.go index 147a6dca..6b451ef7 100644 --- a/cmd/mesh-controller/handacts.go +++ b/cmd/mesh-controller/handacts.go @@ -105,6 +105,12 @@ var handActVerbs = []handActVerb{ // (ADR 0257). {Verb: "mirrors", Decision: "which copies in the store no record names are the mesh's is a person's " + "word (ADR 0257)"}, + // A move of data a release runs once after its delivery (design 57): a person's step of the release, never a + // repair. + {Verb: "move-placement", Decision: "a release's move of data runs once after its delivery, on a person's " + + "word (design 57, issue 736)"}, + {Verb: "move-config", Decision: "a release's move of data runs once after its delivery, on a person's " + + "word (design 57, issue 664)"}, {Verb: "bus upgrade", Decision: "the bus is never rolled by the mesh: replacing it is a planned step a " + "person starts (ADR 0236)"}, {Verb: "upgrade release-backlog", Decision: "after a release plan failed, the next opens only when a " + @@ -118,6 +124,9 @@ var handActVerbs = []handActVerb{ {Verb: "secret accept", Decision: "a value an outside party issued is given by a person, at their desk"}, {Verb: "secret rotate", Decision: "a value a person judged disclosed is replaced on their word", DecidedFor: []string{causeLeakedInLogs}}, + // A provider's credential the vault holds, rotated there on a person's word (issue 767): the same judgement. + {Verb: "rotate", Decision: "a value a person judged disclosed is replaced on their word", + DecidedFor: []string{causeLeakedInLogs}}, } // personsDecision is whether an act in the log is a person's decision by design, by the verb that diff --git a/cmd/mesh-controller/healers.go b/cmd/mesh-controller/healers.go index 401fe0fc..7bc1c999 100644 --- a/cmd/mesh-controller/healers.go +++ b/cmd/mesh-controller/healers.go @@ -642,7 +642,8 @@ func planStale(ctx context.Context, inv *inventory.Inventory, p inventory.Plan) if err != nil { return "", "", err } - if u.RollOut { + // A recorded module whose move goes out on a machine is still the walk's to send (novox/hq ADR 0377). + if u.RollOut || len(wentOut(s.Recorded)) > 0 { return "", "", nil } } diff --git a/cmd/mesh-controller/heard_withheld.go b/cmd/mesh-controller/heard_withheld.go new file mode 100644 index 00000000..31c86821 --- /dev/null +++ b/cmd/mesh-controller/heard_withheld.go @@ -0,0 +1,66 @@ +package main + +import ( + "context" + "fmt" + "strings" + + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/conditions" +) + +// A grant of a restricted stream that composes nothing (novox/hq ADR 0376 §2). +// +// A restricted stream is heard only by a module that runs as an account of its own (`runs-as`, ADR 0259): the +// tool runner's bus account is the one every agent session on its machine reaches the bus as, so the controller +// composes no consumer, grant or bind word for a module it carries (broker.ComposedHears). Where a grant still +// names such a module, the module hears nothing it was granted, and that is said — in `doctor` and in the +// controller's log on every raise and send — naming the module, the stream and the remedy. Never silent. + +const ( + // probeWithheldID is the self-check's probe of it, and kindGrantComposesNothing the condition it raises. + probeWithheldID = "D-heard" + kindGrantComposesNothing = "grant-composes-nothing" +) + +// probeWithheldHearing says every granted hearing of a restricted stream composed for nothing. +func probeWithheldHearing(ctx context.Context, d *doctor) ([]conditions.Observation, error) { + records, err := d.open.inventory.BusRecords(ctx) + if err != nil { + return nil, err + } + return withheldObservations(broker.WithheldOf(records)), nil +} + +// withheldObservations is each withheld hearing as a condition of its module on its machine. +func withheldObservations(withheld []broker.Withheld) []conditions.Observation { + var out []conditions.Observation + for _, w := range withheld { + out = append(out, conditions.Observation{Scope: conditions.ScopeModule, + ID: w.Module + "." + strings.ToLower(w.Stream) + "." + w.Node, + Kind: kindGrantComposesNothing, Machine: w.Node, Severity: conditions.Warning, + Summary: w.Said(), Said: fmt.Sprintf("%s on %s granted %s; composed nothing", w.Module, w.Node, w.Stream)}) + } + return sortedFound(out) +} + +// sayWithheld puts every withheld hearing in the controller's log, once per raise or send that composes it. +func sayWithheld(records broker.Records, indent string) { + for _, w := range broker.WithheldOf(records) { + fmt.Printf("%sA GRANT COMPOSES NOTHING: %s\n", indent, w.Said()) + } +} + +// withheldWords are the plain words of a withheld hearing: the module, the machine and what to do. +func withheldWords(o conditions.Observation) words { + module, m := "a module", machineOr(o, "a machine") + if o.Scope == conditions.ScopeModule && idPart(o, 0) != "" { + module = idPart(o, 0) + } + return words{Headline: clip(conditions.Capital(fmt.Sprintf("%s on %s hears nothing it was granted", module, m)), 60), + Needs: fmt.Sprintf("give %s an account of its own in its manifest; the details say which stream.", module), + Explanation: fmt.Sprintf("You granted %s part of a stream that only modules running under an account of "+ + "their own may hear. It runs inside the shared tool runner on %s, which every assistant session there "+ + "also uses, so the mesh gives it none of that stream.", module, m), + Resolved: fmt.Sprintf("%s on %s hears what it was granted, or is no longer granted it", module, m)} +} diff --git a/cmd/mesh-controller/heard_withheld_test.go b/cmd/mesh-controller/heard_withheld_test.go new file mode 100644 index 00000000..76fbf102 --- /dev/null +++ b/cmd/mesh-controller/heard_withheld_test.go @@ -0,0 +1,43 @@ +package main + +import ( + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/conditions" +) + +// A grant naming a module the tool runner carries composes nothing, and doctor says so in plain words, naming the +// module, the stream and the remedy (novox/hq ADR 0376 §2). +func TestAGrantThatComposesNothingIsSaid(t *testing.T) { + records := broker.Records{Nodes: []string{"ace"}, Assigned: map[string][]broker.Declared{"ace": { + {Module: broker.RuntimeModule}, + {Module: "reader", Hears: []broker.Heard{{Stream: "MAIL", Emitter: "mesh-mail", Restricted: true, + Filters: []string{"mesh.stream.mesh-mail.event.mail.mesh.>"}}}}, + {Module: "mailer", RunsAs: "mailer", Hears: []broker.Heard{{Stream: "MAIL", Emitter: "mesh-mail", Restricted: true, + Filters: []string{"mesh.stream.mesh-mail.event.mail.mesh.>"}}}}, + }}} + found := withheldObservations(broker.WithheldOf(records)) + if len(found) != 1 { + t.Fatalf("said %+v", found) + } + o := found[0] + if o.Kind != kindGrantComposesNothing || o.Machine != "ace" || !strings.Contains(o.Summary, "reader on ace") || + !strings.Contains(o.Summary, "MAIL") || !strings.Contains(o.Summary, "runs-as") { + t.Fatalf("%+v", o) + } + w := plainWordings[kindGrantComposesNothing](o) + if why, ok := conditions.PlainWords(w, "ace"); !ok { + t.Fatalf("not plain: %s (%+v)", why, w) + } + registered := false + for _, p := range probeRegistry { + if p.ID == probeWithheldID && p.Kind == kindGrantComposesNothing && p.run != nil { + registered = true + } + } + if !registered { + t.Fatal("doctor does not run the probe of a grant that composes nothing") + } +} diff --git a/cmd/mesh-controller/held_back.go b/cmd/mesh-controller/held_back.go index a0b4c6ba..66644ec2 100644 --- a/cmd/mesh-controller/held_back.go +++ b/cmd/mesh-controller/held_back.go @@ -146,7 +146,23 @@ func heldMachines(ctx context.Context, open *stores, names []string) (map[string same[m] = current[m].Commit } } - why := heldBack(node, modules, same, known, current, plans) + // A move under a module's own `record` that recreates nothing and moves no data goes as a module that rolls + // out goes (novox/hq ADR 0377): a walk's to send, not a person's. + here, copied := current, false + for m, was := range same { + if now, held := current[m]; held && now.OwnRecord() && !f.identical(m, was, now.Commit) && f.rolls(m, was) { + if !copied { + copied = true + here = make(map[string]inventory.CurrentBuild, len(current)) + for k, v := range current { + here[k] = v + } + } + now.RollOut = true + here[m] = now + } + } + why := heldBack(node, modules, same, known, here, plans) // And a build no gate has seen is not carried by a send that does not judge it (ADR 0236). for _, mv := range f.moves(node, modules, same, known, false) { if planStillToSend(plans, mv.Module, node) == "" { diff --git a/cmd/mesh-controller/issue352_test.go b/cmd/mesh-controller/issue352_test.go index 0e48b13b..e3a6f7e1 100644 --- a/cmd/mesh-controller/issue352_test.go +++ b/cmd/mesh-controller/issue352_test.go @@ -25,6 +25,7 @@ import ( // another send reaches anchor, unreported; the gate still passes. And a send that moves a judged module // to another build supersedes the judging: no verdict, nothing put back. func TestReplay352(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) judgeLeavesAsTheCore(t) t.Run("a newer send to the judged machine does not unreport the gate's", testANewerSendDoesNotUnreportTheGatesOwn) t.Run("a send that moves the module supersedes the judging", testASendThatMovesTheModuleSupersedesTheJudging) @@ -67,6 +68,7 @@ func testANewerSendDoesNotUnreportTheGatesOwn(t *testing.T) { // A plan's own first send waits while a release judges the same module on that machine with another build. func TestAPlansFirstSendWaitsForAReleaseJudgingTheModuleThere(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) judgeLeavesAsTheCore(t) b := aBacklog(t) ctx := t.Context() @@ -82,6 +84,7 @@ func TestAPlansFirstSendWaitsForAReleaseJudgingTheModuleThere(t *testing.T) { // A merge plan's judging is superseded the same way: another send moved its module on the first machine. func TestAPlansJudgingIsSupersededByASendThatMovesItsModule(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) judgeLeavesAsTheCore(t) g := aGateMesh(t) ctx := t.Context() @@ -140,6 +143,7 @@ func testASendThatMovesTheModuleSupersedesTheJudging(t *testing.T) { // A report is on the gate's own send: the declaration itself, or one sequenced after it; a gate kept // without its send reads the report against the send made last, as before. Pure. func TestAReportIsHeldAgainstTheGatesOwnSend(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) sent := inventory.SentDeclaration{Digest: "d-490", Sequence: 490} for _, c := range []struct { r inventory.Reported @@ -185,6 +189,7 @@ func TestAReportIsHeldAgainstTheGatesOwnSend(t *testing.T) { // A gate judges only the build the machine was last sent: last sent another build of the module, the // judging is superseded, whatever the machine reports. Pure. func TestAGateJudgesOnlyTheBuildTheMachineWasLastSent(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) at := time.Now() f := gateFacts{now: at, reports: map[string]inventory.Reported{"anchor": {Node: "anchor", Outcome: inventory.OutcomeApplied, At: &at, Current: true}}, engines: map[string]string{}, served: map[string]served{}, rolledBack: map[string][]lease.Rollback{}, @@ -212,6 +217,7 @@ func TestAGateJudgesOnlyTheBuildTheMachineWasLastSent(t *testing.T) { // A move of another build of a module to a machine where a release or a plan is judging that module // waits for that judging; the same build to that machine is already there. Pure. func TestAWalkWaitsForAJudgingOfTheSameModuleOnThatMachine(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) at := time.Now() release := inventory.Plan{ID: "release-1", State: inventory.PlanRolling, Release: &inventory.PlanRelease{ Gate: &inventory.PlanGate{Machines: []string{"novox"}, Carried: []inventory.CarriedMove{ @@ -249,6 +255,7 @@ func TestAWalkWaitsForAJudgingOfTheSameModuleOnThatMachine(t *testing.T) { // The controller is never put back to a build that reaches less of the store's schema than the store // has, or to one that never said what it reaches: the current build is kept, and the condition says so. func TestTheControllerIsNotPutBackToABuildOlderThanTheStoresSchema(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) open := aMesh(t) ctx := t.Context() inv := open.inventory @@ -345,6 +352,7 @@ func TestTheControllerIsNotPutBackToABuildOlderThanTheStoresSchema(t *testing.T) // The lease's holder names the build the declaration told it it is, and the version stamp only without one. func TestTheHolderNamesTheBuildTheDeclarationToldIt(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) t.Setenv(RunningBuildVar, " ad62528c47c7 ") if h := holderOf("x"); h.Build != "ad62528c47c7" { t.Fatalf("the holder's build is %q", h.Build) diff --git a/cmd/mesh-controller/issue648_test.go b/cmd/mesh-controller/issue648_test.go index 45261cc0..0557e774 100644 --- a/cmd/mesh-controller/issue648_test.go +++ b/cmd/mesh-controller/issue648_test.go @@ -20,8 +20,8 @@ func TestANoMoveTierWaitsForNoNewReport(t *testing.T) { ctx := t.Context() inv := g.open.inventory - // app is the build machine, so a module of the next tier is built by it: the tier gates the next on app - // running on its machines. + // app holds the build seat, so a module of the next tier is built by it; since novox/hq ADR 0360 §3 that gates + // nothing. for _, m := range []catalogue.Manifest{ {Module: "app", Version: "c2", Claims: []catalogue.Claim{{Name: "node-build-agent", Scope: catalogue.ScopeNode}}}, {Module: "web", Version: "w1"}, @@ -71,7 +71,8 @@ func TestANoMoveTierWaitsForNoNewReport(t *testing.T) { t.Fatalf("the walk waits on a no-move tier for a report nothing will cause: tier %d, %s %q", p.Tier, p.State, p.Note) } - // A machine that has not answered its last send is still waited on: no move is not "never sent". + // A machine that has not answered its last send is not waited on either: a built-by edge gates no tier + // (novox/hq ADR 0360 §3). Before, the walk waited for app on laptop. p.Tier, p.State, p.Note = 0, inventory.PlanRolling, "" if err := inv.RecordSent(ctx, nodeID(t, g.open, "laptop"), "d-laptop-later", map[string]string{"app": "c1"}); err != nil { t.Fatal(err) @@ -81,7 +82,7 @@ func TestANoMoveTierWaitsForNoNewReport(t *testing.T) { } advancePlans(ctx, g.open) p = g.plan(t) - if p.Tier != 0 || !strings.Contains(p.Note, "waiting for app on laptop") { - t.Fatalf("a machine that has not answered its last send was not waited on: tier %d, %q", p.Tier, p.Note) + if p.Tier < 1 || strings.Contains(p.Note, "waiting for app") { + t.Fatalf("the walk waits on a built-by edge: tier %d, %q", p.Tier, p.Note) } } diff --git a/cmd/mesh-controller/issued.go b/cmd/mesh-controller/issued.go index 1f1b348c..4ae02291 100644 --- a/cmd/mesh-controller/issued.go +++ b/cmd/mesh-controller/issued.go @@ -43,6 +43,32 @@ const issuedAsk = 20 * time.Second // issuedProvider is one assignment of a module whose offer says its software issues the provision's credential. type issuedProvider struct { Node, Module, Tool string + // Holders are the provision's holders from this assignment's machine, named to the vault, whose tool may answer + // a value for each (issue 749). + Holders []issuedHolder +} + +// issuedHolder is one holder of a provision from a provider: a consuming module on one machine (issue 749). +type issuedHolder struct { + Node string `json:"node"` + Module string `json:"module"` +} + +func (h issuedHolder) String() string { return h.Module + " on " + h.Node } + +// issuedHoldersFrom are the provision's holders from provider, each consuming module on each machine once. +func issuedHoldersFrom(holders []inventory.Holder, provider string) []issuedHolder { + seen := map[issuedHolder]bool{} + var out []issuedHolder + for _, h := range holders { + one := issuedHolder{Node: h.Consumer, Module: h.ConsumerModule} + if h.Provider != provider || seen[one] { + continue + } + seen[one] = true + out = append(out, one) + } + return out } // issuedTaker is the line's work, with what it reaches handed in: the store, the bus and the sends. @@ -53,6 +79,10 @@ type issuedTaker struct { keep func(provision, provider, value string) (inventory.IssuedKept, error) record func(act link.HandAct) error send func(machines []string) error + // ofTheVault says whether a provider module's secrets are filled the new way; for one that is, the value goes + // into the vault (vault, vault_secrets.go) and nothing is kept, sealed or sent here. Nil is none. + ofTheVault vaultSwitch + vault vaultAsker } // take asks every provider of the provision for its software's value and keeps each; it answers what it did, in @@ -90,6 +120,15 @@ func (t issuedTaker) take(provision string) (string, error) { // takeFrom is one provider: asked, opened, kept, recorded and sent when it changed. func (t issuedTaker) takeFrom(provision string, p issuedProvider) (string, error) { + if t.ofTheVault != nil { + ours, err := t.ofTheVault(p.Module) + if err != nil { + return "", err + } + if ours { + return t.takeIntoVault(provision, p) + } + } public, private, err := secrets.Keypair() if err != nil { return "", fmt.Errorf("no key could be made to take the value: %w", err) @@ -174,6 +213,16 @@ func issuedProvidersOf(ctx context.Context, inv *inventory.Inventory, provision if err != nil { return nil, true, err } + out, err := issuedProvidersFrom(shelf, holders, func(node string) ([]string, error) { + return inv.Assigned(ctx, node) + }, provision) + return out, true, err +} + +// issuedProvidersFrom are the assignments to ask, from the provision's holders: on every machine providing it, the +// assigned modules whose offer says their software issues it, each with that machine's holders (issue 749). +func issuedProvidersFrom(shelf map[string]catalogue.Manifest, holders []inventory.Holder, + assignedOn func(node string) ([]string, error), provision string) ([]issuedProvider, error) { nodes := map[string]bool{} for _, h := range holders { nodes[h.Provider] = true @@ -185,13 +234,17 @@ func issuedProvidersOf(ctx context.Context, inv *inventory.Inventory, provision sort.Strings(names) var out []issuedProvider for _, node := range names { - assigned, err := inv.Assigned(ctx, node) + assigned, err := assignedOn(node) if err != nil { - return nil, true, err + return nil, err } - out = append(out, issuedOn(shelf, node, assigned, provision)...) + on := issuedOn(shelf, node, assigned, provision) + for i := range on { + on[i].Holders = issuedHoldersFrom(holders, node) + } + out = append(out, on...) } - return out, true, nil + return out, nil } // offersIssued is whether any module offers the provision with a credential its software issues. @@ -262,7 +315,9 @@ func secretIssued(ctx context.Context, args []string) error { return err }) }, - send: func(machines []string) error { return sendTo(ctx, open, machines) }, + send: func(machines []string) error { return sendTo(ctx, open, machines) }, + ofTheVault: filledTheNewWay, + vault: theVault(ctx), } words, err := t.take(provision) if words != "" { diff --git a/cmd/mesh-controller/issued_test.go b/cmd/mesh-controller/issued_test.go index c9b1b6c8..f2b9da84 100644 --- a/cmd/mesh-controller/issued_test.go +++ b/cmd/mesh-controller/issued_test.go @@ -193,7 +193,7 @@ func TestTheProvidersAskedAreTheOffersThatSaySo(t *testing.T) { "jellyfin": {Module: "jellyfin", Provides: []catalogue.Offer{{Name: "plex-api"}}}, } got := issuedOn(shelf, "ace", []string{"jellyfin", "plex", "kometa"}, "plex-api") - if len(got) != 1 || got[0] != (issuedProvider{Node: "ace", Module: "plex", Tool: "plex_issued_credential"}) { + if len(got) != 1 || got[0].Node != "ace" || got[0].Module != "plex" || got[0].Tool != "plex_issued_credential" { t.Fatalf("the providers asked: %+v", got) } if err := issuedRefusesRotation(shelf, "plex-api"); err == nil || !strings.Contains(err.Error(), "secret issued plex-api") { @@ -216,3 +216,84 @@ func TestAnIssuedValueIsADecisionInTheHandActLog(t *testing.T) { } t.Fatal("secret issued is not in the hand-act table") } + +// **A need's credential is the vault's for its holder alone** (issue 757) exactly when its providing module is marked +// moved, so `secret issued` goes to the vault naming each holder, and its software issues the provision's credential. +func TestANeedIsIssuedToItsHolderOnlyWhereTheProviderIsMovedAndItsSoftwareIssuesIt(t *testing.T) { + forge := func(moved bool, issued bool) catalogue.Manifest { + offer := catalogue.Offer{Name: "gitea-read"} + if issued { + offer.Credential = &catalogue.OfferCredential{IssuedBy: catalogue.IssuedBySoftware, Tool: "gitea_agent_token"} + } + return catalogue.Manifest{Module: "gitea", Moved: moved, Provides: []catalogue.Offer{offer}} + } + shared := func(moved bool) catalogue.Manifest { + return catalogue.Manifest{Module: "gitea", Moved: moved, OwnSecrets: catalogue.OwnSecrets{"key": {Path: "/k"}}, + Provides: []catalogue.Offer{{Name: "gitea-read", Credential: &catalogue.OfferCredential{Own: "key"}}}} + } + given := func() catalogue.Manifest { + return catalogue.Manifest{Module: "gitea", Moved: true, Provides: []catalogue.Offer{{Name: "gitea-read", + Credential: &catalogue.OfferCredential{IssuedBy: catalogue.IssuedOutside}}}} + } + need := catalogue.Needed{Name: "gitea-read", From: "novox", Module: "gitea", For: "claude-code"} + for why, c := range map[string]struct { + shelf map[string]catalogue.Manifest + need catalogue.Needed + want bool + }{ + "moved and issued by its software": {map[string]catalogue.Manifest{"gitea": forge(true, true)}, need, true}, + "not moved": {map[string]catalogue.Manifest{"gitea": forge(false, true)}, need, false}, + "a login for each consumer": {map[string]catalogue.Manifest{"gitea": forge(true, false)}, need, true}, + "a login for each, not moved": {map[string]catalogue.Manifest{"gitea": forge(false, false)}, need, false}, + "a credential every holder shares": {map[string]catalogue.Manifest{"gitea": shared(true)}, need, false}, + "a credential a person gives": {map[string]catalogue.Manifest{"gitea": given()}, need, false}, + "a provider not on the shelf": {map[string]catalogue.Manifest{}, need, false}, + "answered by a record": {map[string]catalogue.Manifest{"gitea": forge(true, true)}, + catalogue.Needed{Name: "gitea-read", ByRecord: true, Module: "gitea", For: "claude-code"}, false}, + } { + if got := issuedToHolder(c.shelf, c.need); got != c.want { + t.Errorf("%s: issued to its holder is %v, want %v", why, got, c.want) + } + } +} + +// **The vault is a moved provider for its consumers' credentials** (issue 793): its pair rows are the vault's since +// V3, so a need of `secret` is issued to its holder and answered by a moved provider, whether or not the module is +// marked moved; another module is not. +func TestTheModuleHoldingTheVaultSeatCountsAsMovedForItsConsumers(t *testing.T) { + vault := catalogue.Manifest{Module: "mesh-vault", Claims: []catalogue.Claim{{Name: "mesh-vault"}}, + Provides: []catalogue.Offer{{Name: "secret"}}} + other := catalogue.Manifest{Module: "mesh-vault", Provides: []catalogue.Offer{{Name: "secret"}}} + need := catalogue.Needed{Name: "secret", Local: "admin", From: "novox", Module: "mesh-vault", For: "gitea"} + if !issuedToHolder(map[string]catalogue.Manifest{"mesh-vault": vault}, need) || + !providerMoved(map[string]catalogue.Manifest{"mesh-vault": vault}, need) { + t.Error("the vault's consumer is not answered by a moved provider") + } + if issuedToHolder(map[string]catalogue.Manifest{"mesh-vault": other}, need) || + providerMoved(map[string]catalogue.Manifest{"mesh-vault": other}, need) { + t.Error("a module not holding the vault seat counts as moved") + } +} + +// **A provision answered by the helper makes its provider a moved one for that provision alone** (issue 792): the +// provides entry says `"answered-by": "helper"`, the module need not be marked moved, and no credential is the vault's +// for it. Another provision of the module, or a module without the entry, does not count. +func TestAnOfferAnsweredByTheHelperCountsAsMovedForThatProvisionOnly(t *testing.T) { + proxy := catalogue.Manifest{Module: "route-proxy", Provides: []catalogue.Offer{ + {Name: "route", AnsweredBy: catalogue.AnsweredByHelper}, {Name: "other"}}} + plain := catalogue.Manifest{Module: "route-proxy", Provides: []catalogue.Offer{{Name: "route"}}} + shelf := func(m catalogue.Manifest) map[string]catalogue.Manifest { + return map[string]catalogue.Manifest{m.Module: m} + } + route := catalogue.Needed{Name: "route", From: "novox", Module: "route-proxy", For: "gitea"} + other := catalogue.Needed{Name: "other", From: "novox", Module: "route-proxy", For: "gitea"} + if !providerMoved(shelf(proxy), route) { + t.Error("a provision answered by the helper is not answered by a moved provider") + } + if providerMoved(shelf(proxy), other) || providerMoved(shelf(plain), route) { + t.Error("a provision not answered by the helper counts as moved") + } + if issuedToHolder(shelf(proxy), route) { + t.Error("the flag made a credential the vault's") + } +} diff --git a/cmd/mesh-controller/issued_to_vault.go b/cmd/mesh-controller/issued_to_vault.go new file mode 100644 index 00000000..212a3d16 --- /dev/null +++ b/cmd/mesh-controller/issued_to_vault.go @@ -0,0 +1,67 @@ +package main + +import ( + "context" + "fmt" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" +) + +// A covered account is handed to the vault when it is minted (novox/hq ADR 0374, issue 787). +// +// **Covered** are the bus accounts `module issue` mints for a core module (catalogue.CoreCredentialModules) or for a +// module marked moved; no other account is. Until the vault makes bus accounts (ADR 0113 clause 6, issue 774) the +// controller mints such an account and writes its hash to the bus's user list, as before, and then hands the whole +// value at once to the vault's `accept` with origin `issued`: sealed here to the key the vault made for that call, so +// the vault alone opens it (ADR 0349 clause 8). The vault keeps it as the module's own secret on that machine and +// raises `secret-changed`; a re-issue is the same hand with a new value, which replaces the old in place. An account +// not covered is minted and kept as it was and handed to nobody. + +// vaultAskerForIssue is the vault as `module issue` asks it. A test replaces it. +var vaultAskerForIssue = func(ctx context.Context) vaultAsker { return theVault(ctx) } + +// accountIsCovered says whether the bus accounts minted for m are the vault's to hold (ADR 0374 clause 1). +func accountIsCovered(m catalogue.Manifest) bool { + return catalogue.CarriesVaultSealed(m.Module) || m.Moved +} + +// vaultInstalled says whether the vault is assigned to any machine. Until it is, as in genesis (ADR 0113 §15 to §18, +// mesh-host's bootstrap, which issues the bus's and the provisioner's accounts before the vault step), there is no +// vault to hand to: the value is kept in the controller's store as before, and the vault takes it afterwards by the +// move (move-in takes a value of the controller's store with origin made, accepted or issued, as it is). Only a +// vault that is installed and does not answer fails the issue. +func vaultInstalled(ctx context.Context, inv *inventory.Inventory) (bool, error) { + entries, err := inv.Catalogued(ctx) + if err != nil { + return false, err + } + for _, e := range entries { + if e.Manifest.Module == "mesh-vault" && len(e.On) > 0 { + return true, nil + } + } + return false, nil +} + +// handIssuedToVault hands the value just minted for module's own secret name on node to the vault, when the module's +// accounts are covered, and does nothing otherwise. Loud when the vault does not take it: the bus's user list +// already holds the new hash, so a vault that kept the old value would serve a password that no longer logs in. +func handIssuedToVault(ctx context.Context, inv *inventory.Inventory, m catalogue.Manifest, node, name, held string) error { + if !accountIsCovered(m) { + return nil + } + if installed, err := vaultInstalled(ctx, inv); err != nil { + return fmt.Errorf("whether the vault is installed cannot be read, so %s of %s was not handed to it: %w", name, + m.Module, err) + } else if !installed { + return nil + } + target := ownSecretArgs(node, m.Module, name) + target["origin"] = "issued" + if _, err := acceptIntoVault(vaultAskerForIssue(ctx), target, held, "module issue"); err != nil { + return fmt.Errorf("the credential is minted and kept here, and the bus's user list holds its hash, but the "+ + "vault did not take %s of %s on %s: %w (`module issue` again hands a new one)", name, m.Module, node, err) + } + return nil +} diff --git a/cmd/mesh-controller/issued_to_vault_test.go b/cmd/mesh-controller/issued_to_vault_test.go new file mode 100644 index 00000000..d2525d65 --- /dev/null +++ b/cmd/mesh-controller/issued_to_vault_test.go @@ -0,0 +1,157 @@ +package main + +import ( + "context" + "errors" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/catalogue" +) + +// Every other test mints without a vault to hand to: a stand-in that takes anything, replaced by the tests below. +func init() { + vaultAskerForIssue = func(context.Context) vaultAsker { return newStandInVault(nil).ask } +} + +// A covered account is handed to the vault when it is minted (novox/hq ADR 0374, issue 787): the whole value, sealed +// to the key the vault made for the call, with origin issued. A module that is neither core nor moved hands nothing. +func TestModuleIssueHandsACoveredAccountToTheVaultAndOnlyThat(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + theVaultIsInstalled(t, open) + inv := open.inventory + known := broker.Broker{Fingerprint: "ab"} + + core := aProvisioner() + core.Module = "nats" + moved := aBot("moved-bot", true) + moved.OwnSecrets["broker"] = catalogue.OwnSecret{Path: "/var/lib/moved-bot/broker"} + plain := aBot("kept-bot", false) + plain.OwnSecrets["broker"] = catalogue.OwnSecret{Path: "/var/lib/kept-bot/broker"} + + vault := newStandInVault(t) + vaultAskerForIssue = func(context.Context) vaultAsker { return vault.ask } + t.Cleanup(func() { + vaultAskerForIssue = func(context.Context) vaultAsker { return newStandInVault(nil).ask } + }) + + for _, m := range []catalogue.Manifest{core, moved, plain} { + register(t, open, m) + if _, err := assign(ctx, open, "laptop", m.Module); err != nil { + t.Fatal(err) + } + } + issue := func(m catalogue.Manifest, password string) { + t.Helper() + if err := issueWith(ctx, inv, m, "laptop", "", known, "10.0.0.1:4222", "module."+m.Module+".laptop", password); err != nil { + t.Fatal(err) + } + } + + issue(core, "first-core-password") + issue(moved, "first-moved-password") + issue(plain, "kept-password") + if err := issueProvisionerClient(ctx, inv, core, "laptop", known, "10.0.0.1:4222"); err != nil { + t.Fatal(err) + } + + said := vault.said() + for _, want := range []string{`"module":"nats"`, `"module":"moved-bot"`, `"name":"broker"`, + `"name":"` + broker.ProvisionerSecret + `"`, `"node":"laptop"`} { + if !strings.Contains(said, want) { + t.Errorf("the vault was told %s, without %s", said, want) + } + } + if n := strings.Count(said, `"origin":"issued"`); n != 3 { + t.Errorf("%d accounts were handed with origin issued, want 3 (the core's two and the moved module's): %s", n, said) + } + if strings.Contains(said, "kept-bot") { + t.Errorf("an account that is neither core nor moved was handed to the vault: %s", said) + } + if strings.Contains(said, "first-core-password") || strings.Contains(said, "first-moved-password") { + t.Errorf("a value crossed the bus in the clear: %s", said) + } + + // A re-issue hands the new value, which the vault opens: the whole {url, fingerprint, node, user, password}. + issue(moved, "second-moved-password") + if got := vault.taken["accept"]; !strings.Contains(got, "second-moved-password") || + !strings.Contains(got, `"user":"module.moved-bot.laptop"`) || !strings.Contains(got, `"node":"laptop"`) || + !strings.Contains(got, `"url":"nats://10.0.0.1:4222"`) || !strings.Contains(got, `"fingerprint":"ab"`) { + t.Errorf("the vault was handed %q after a re-issue, not the new account whole", got) + } +} + +// A vault that does not take the value fails the issue loudly: the bus's user list holds the new hash already. +func TestModuleIssueFailsWhenTheVaultDoesNotTakeACoveredAccount(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + moved := aBot("moved-bot", true) + moved.OwnSecrets["broker"] = catalogue.OwnSecret{Path: "/var/lib/moved-bot/broker"} + theVaultIsInstalled(t, open) + register(t, open, moved) + if _, err := assign(ctx, open, "laptop", moved.Module); err != nil { + t.Fatal(err) + } + vault := newStandInVault(t) + vault.refuse = errors.New("the vault is not there") + vaultAskerForIssue = func(context.Context) vaultAsker { return vault.ask } + t.Cleanup(func() { + vaultAskerForIssue = func(context.Context) vaultAsker { return newStandInVault(nil).ask } + }) + err := issueWith(ctx, open.inventory, moved, "laptop", "", broker.Broker{Fingerprint: "ab"}, "10.0.0.1:4222", + "module.moved-bot.laptop", "pw") + if err == nil || !strings.Contains(err.Error(), "vault did not take") { + t.Fatalf("a vault that took nothing was reported as %v", err) + } +} + +func theVaultIsInstalled(t *testing.T, open *stores) { + t.Helper() + vault := catalogue.Manifest{Module: "mesh-vault", Version: "1"} + register(t, open, vault) + if _, err := assign(t.Context(), open, "laptop", vault.Module); err != nil { + t.Fatal(err) + } +} + +// In genesis the bus's and the provisioner's accounts are issued before the vault is installed (ADR 0113 §15 to §18): +// there is no vault to hand to, so nothing is asked and nothing fails, and the value stays in the controller's store +// as before, for the move to carry into the vault. Once the vault is installed, one that does not answer fails. +func TestModuleIssueBeforeTheVaultIsInstalledKeepsTheValueAndHandsNothing(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + known := broker.Broker{Fingerprint: "ab"} + core := aProvisioner() + core.Module = "nats" + register(t, open, core) + if _, err := assign(ctx, open, "laptop", core.Module); err != nil { + t.Fatal(err) + } + vault := newStandInVault(t) + vault.refuse = errors.New("there is no vault yet") + vaultAskerForIssue = func(context.Context) vaultAsker { return vault.ask } + t.Cleanup(func() { + vaultAskerForIssue = func(context.Context) vaultAsker { return newStandInVault(nil).ask } + }) + + if err := issueWith(ctx, inv, core, "laptop", "", known, "10.0.0.1:4222", "module.nats.laptop", "pw"); err != nil { + t.Fatalf("genesis's issue of the bus's account failed with no vault installed: %v", err) + } + if err := issueProvisionerClient(ctx, inv, core, "laptop", known, "10.0.0.1:4222"); err != nil { + t.Fatalf("genesis's issue of the client's account failed with no vault installed: %v", err) + } + if said := vault.said(); said != "" { + t.Errorf("the vault was asked before it was installed: %s", said) + } + if _, held, err := inv.ModuleSecretIfIssued(ctx, "laptop", "nats", "broker"); err != nil || !held { + t.Errorf("the value is not kept in the controller's store (held %v, %v)", held, err) + } + + theVaultIsInstalled(t, open) + if err := issueWith(ctx, inv, core, "laptop", "", known, "10.0.0.1:4222", "module.nats.laptop", "pw2"); err == nil { + t.Error("a vault that is installed and does not answer did not fail the issue") + } +} diff --git a/cmd/mesh-controller/lean_walk_test.go b/cmd/mesh-controller/lean_walk_test.go new file mode 100644 index 00000000..eced96cf --- /dev/null +++ b/cmd/mesh-controller/lean_walk_test.go @@ -0,0 +1,242 @@ +package main + +import ( + "context" + "slices" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/inventory" +) + +// novox/hq ADR 0360 §4, §5: a walk ends on its machines' reports — every machine sent has reported on its send, +// or is silent past ApplySilentAfter — and names a failed or refused apply; it never waits for a health reading. +func TestAWalkEndsOnItsMachinesReports(t *testing.T) { + now := time.Date(2026, 10, 14, 12, 0, 0, 0, time.UTC) + sent := now.Add(-time.Minute) + long := now.Add(-inventory.ApplySilentAfter - time.Minute) + at := now.Add(-10 * time.Second) + p := inventory.Plan{Modules: map[string]*inventory.PlanModule{ + "app": {Rest: map[string]inventory.SentDeclaration{ + "ace": {Digest: "d-ace", At: sent}, + "novox": {Digest: "d-novox", At: sent}, + "shanks": {Digest: "d-shanks", At: sent}, + "laptop": {Digest: "d-laptop", At: long}, + }}, + "proxy": {Rest: map[string]inventory.SentDeclaration{"ace": {Digest: "d-ace", At: sent}}}, + }} + reports := []inventory.Reported{ + {Node: "ace", Outcome: inventory.OutcomeApplied, At: &at, Declared: "d-ace"}, + {Node: "novox", Outcome: inventory.OutcomeFailed, At: &at, Declared: "d-novox"}, + {Node: "shanks", Outcome: inventory.OutcomeApplied, At: &at, Declared: "d-before"}, + } + waiting, failed, silent := walkReports(p, reports, now, nil) + if !slices.Equal(waiting, []string{"shanks"}) || !slices.Equal(failed, []string{"novox"}) || + !slices.Equal(silent, []string{"laptop"}) { + t.Fatalf("waiting %v, failed %v, silent %v", waiting, failed, silent) + } + reports[2].Declared = "d-shanks" + if waiting, _, _ := walkReports(p, reports, now, nil); len(waiting) != 0 { + t.Fatalf("the walk still waits once every machine reported: %v", waiting) + } +} + +// withTheGate turns the first-machine gate on for a test of its own path: kept for the walks a controller before +// novox/hq ADR 0360 started. +func withTheGate(t *testing.T) { + t.Helper() + was := gateFirstMachine + gateFirstMachine = true + t.Cleanup(func() { gateFirstMachine = was }) +} + +// Every machine heard from is sent at once and passed on its applied report, with no judging (novox/hq ADR 0360 §7); +// before, the first machine was judged before the rest were sent (ADR 0353 §1). +func TestAReleaseSendsEveryMachineAtOnce(t *testing.T) { + judgeLeavesAsTheCore(t) + b := aBacklogOnThree(t) + ctx := t.Context() + said := events(t) + for i := 0; i < 12; i++ { + advancePlans(ctx, b.open) + } + p := b.release(t) + if p.State != inventory.PlanDone || len(p.Release.Done) != 3 { + t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) + } + for _, n := range []string{"desk", "laptop", "anchor"} { + if !slices.Contains(*said, "send "+n) { + t.Fatalf("%s was not sent: %v", n, *said) + } + } + if slices.Contains(*said, "judge") { + t.Fatalf("a machine was judged by its health: %v", *said) + } + if v, found, err := b.open.inventory.GateOf(ctx, "build-late-c2"); err != nil || !found || v.Verdict != inventory.GatePassed { + t.Fatalf("late's pass on laptop was not kept: %+v %v %v", v, found, err) + } +} + +// A machine that reports its apply failed is named and fails the release, with nothing put back, and every other +// machine goes on to its own report (novox/hq ADR 0360 §5, §7); before, it was put back (ADR 0353 §3). +func TestAMachineThatFailsItsApplyIsNamedAndNothingIsPutBack(t *testing.T) { + judgeLeavesAsTheCore(t) + b := aBacklogOnThree(t) + ctx := t.Context() + inv := b.open.inventory + b.broken["desk"] = true // desk fails what it is sent + advancePlans(ctx, b.open) + for i := 0; i < 12 && b.release(t).Open(); i++ { + advancePlans(ctx, b.open) + } + p := b.release(t) + if p.State != inventory.PlanFailed || !slices.Equal(p.Release.Failed, []string{"desk"}) { + t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) + } + if !slices.Contains(p.Release.Done, "laptop") || !slices.Contains(p.Release.Done, "anchor") { + t.Fatalf("the others have no verdict of their own: %+v", p.Release) + } + if v, found, err := inv.GateOf(ctx, "build-late-c2"); err != nil || !found || v.Verdict != inventory.GatePassed { + t.Fatalf("late's pass on laptop was not kept: %+v %v %v", v, found, err) + } + if sent, _, _ := inv.SentBuilds(ctx, "desk"); sent["app"] != "c2" { + t.Fatalf("desk was put back: %v", sent) + } +} + +// A release's send kept before it was made, and made, is judged by the controller that takes over without being +// made again (ADR 0353 §8), with every machine sent at once (ADR 0360 §7). +func TestAReleaseSendKeptAndMadeIsNotMadeAgainAtOnce(t *testing.T) { + judgeLeavesAsTheCore(t) + b := aBacklog(t) + ctx := t.Context() + inv := b.open.inventory + p, err := releaseBacklog(ctx, b.open, "") + if err != nil || p == nil { + t.Fatalf("no release opened: %v", err) + } + if _, err := advanceRelease(ctx, b.open, p); err != nil || p.Release.Gates["anchor"] == nil || !p.Release.Gates["anchor"].Pending { + t.Fatalf("the send was not kept before it was made: %+v %v", p.Release, err) + } + if err := inv.SavePlan(ctx, p); err != nil { + t.Fatal(err) + } + // The send is made, and the controller is replaced before it keeps it. + if _, err := sendRollout(ctx, b.open, []string{"anchor"}); err != nil { + t.Fatal(err) + } + advancePlans(ctx, b.open) + if n := len(slices.DeleteFunc(slices.Clone(b.sent), func(s []string) bool { return !slices.Equal(s, []string{"anchor"}) })); n != 1 { + t.Fatalf("anchor was sent again: %v", b.sent) + } + g := b.release(t).Release.Gates["anchor"] + if g == nil || g.Pending || g.Since == nil || len(g.Carried) == 0 { + t.Fatalf("anchor is not judged on the send made: %+v", g) + } +} + +// A machine of the rest that says nothing on its send is left silent after fifteen minutes: not failed, not put +// back; a rest that went silent whole ends failed, said as a condition (ADR 0353 §4); its step also sent anchor, so the release is done, not failed (ADR 0360 §7). +func TestAMachineSilentPastItsBoundIsLeftAndSaidAtOnce(t *testing.T) { + judgeLeavesAsTheCore(t) + b := aBacklog(t) + ctx := t.Context() + inv := b.open.inventory + reporting := sendRollout + sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { + if slices.Equal(names, []string{"laptop"}) { // laptop sleeps: sent, and says nothing + b.sent = append(b.sent, names) + current, _ := open.inventory.CurrentBuilds(ctx) + return names, open.inventory.RecordSent(ctx, nodeID(t, open, "laptop"), "d-laptop-asleep", + map[string]string{"app": current["app"].Commit, "late": current["late"].Commit, "same": current["same"].Commit}) + } + return reporting(ctx, open, names) + } + t.Cleanup(func() { sendRollout = reporting }) + for i := 0; i < 8; i++ { + advancePlans(ctx, b.open) + } + p := b.release(t) + g := p.Release.Gates["laptop"] + if !p.Open() || g == nil || g.Verdict != "" || g.Silent { + t.Fatalf("laptop is judged before fifteen minutes: %s %s %+v", p.State, p.Note, g) + } + since := time.Now().Add(-inventory.ApplySilentAfter - time.Minute) + g.Since = &since + if err := inv.SavePlan(ctx, &p); err != nil { + t.Fatal(err) + } + advancePlans(ctx, b.open) + p = b.release(t) + // anchor applied in the same step, so the release is done, laptop named silent (novox/hq ADR 0360 §7). + if p.State != inventory.PlanDone || !slices.Equal(p.Release.Silent, []string{"laptop"}) || len(p.Release.Failed) > 0 { + t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) + } + if failed, _ := inv.GateFailed(ctx, "build-late-c2"); failed { + t.Fatal("a silent machine's build was marked failed") + } + if sent, _, _ := inv.SentBuilds(ctx, "laptop"); sent["late"] != "c2" { + t.Fatalf("laptop was put back: %v", sent) + } + obs, err := silentObservation(ctx, inv) + if err != nil { + t.Fatal(err) + } + if len(obs) != 1 || obs[0].Kind != "release-silent" || !slices.Equal(obs[0].Also, []string{"laptop"}) { + t.Fatalf("the silence is not said: %+v", obs) + } +} + +// Issue 840 (novox/hq ADR 0362 §1-5): a walk's end counts a machine's send of a provisioned module applied only once +// the engine and the machine's provisioner client both said so. +func TestAWalkEndWaitsForTheProvisioner(t *testing.T) { + now := time.Date(2026, 10, 15, 12, 0, 0, 0, time.UTC) + at := now.Add(-10 * time.Second) + plan := func(sentAt time.Time) inventory.Plan { + return inventory.Plan{Modules: map[string]*inventory.PlanModule{"app": {Rest: map[string]inventory.SentDeclaration{ + "ace": {Digest: "d-ace", At: sentAt, Provisioned: map[string]string{"app": "t1"}}, + "g14": {Digest: "d-g14", At: sentAt, Provisioned: map[string]string{"app": "t1"}}, + }}}} + } + reports := []inventory.Reported{ + {Node: "ace", Outcome: inventory.OutcomeApplied, At: &at, Declared: "d-ace"}, + {Node: "g14", Outcome: inventory.OutcomeApplied, At: &at, Declared: "d-g14"}, + } + says := map[string]inventory.ProvReport{} + prov := func(node, module string) (inventory.ProvReport, bool) { r, ok := says[node+"/"+module]; return r, ok } + applied := inventory.ProvReport{Status: inventory.ProvApplied, Templates: "t1", At: at} + + // (a) the engine applied, the client has not said: the end waits on both. + waiting, failed, silent := walkReports(plan(now.Add(-time.Minute)), reports, now, prov) + if !slices.Equal(waiting, []string{"ace", "g14"}) || len(failed)+len(silent) != 0 { + t.Fatalf("a: waiting %v, failed %v, silent %v", waiting, failed, silent) + } + // ace's client applied, g14's failed on that build (b): ace done, g14 a failed apply naming the reason. + says["ace/app"] = applied + says["g14/app"] = inventory.ProvReport{Status: inventory.ProvFailed, Templates: "t1", Reason: "no value", At: at} + waiting, failed, silent = walkReports(plan(now.Add(-time.Minute)), reports, now, prov) + if len(waiting) != 0 || len(silent) != 0 || len(failed) != 1 || !strings.HasPrefix(failed[0], "g14 (") || + !strings.Contains(failed[0], "no value") { + t.Fatalf("b: waiting %v, failed %v, silent %v", waiting, failed, silent) + } + // (c) no client report past ApplySilentAfter: named silent, never applied. + delete(says, "g14/app") + waiting, failed, silent = walkReports(plan(now.Add(-inventory.ApplySilentAfter-time.Minute)), reports, now, prov) + if len(waiting) != 0 || len(failed) != 0 || !slices.Equal(silent, []string{"g14"}) { + t.Fatalf("c: waiting %v, failed %v, silent %v", waiting, failed, silent) + } + // both said it: nothing waits. + says["g14/app"] = applied + if waiting, failed, silent = walkReports(plan(now.Add(-time.Minute)), reports, now, prov); len(waiting)+len(failed)+len(silent) != 0 { + t.Fatalf("both said: waiting %v, failed %v, silent %v", waiting, failed, silent) + } + // (d) a send handing nothing to the provisioner is the engine's report alone, as before. + bare := plan(now.Add(-time.Minute)) + bare.Modules["app"].Rest = map[string]inventory.SentDeclaration{"ace": {Digest: "d-ace", At: now.Add(-time.Minute)}} + if waiting, failed, silent = walkReports(bare, reports, now, func(string, string) (inventory.ProvReport, bool) { + return inventory.ProvReport{}, false + }); len(waiting)+len(failed)+len(silent) != 0 { + t.Fatalf("d: waiting %v, failed %v, silent %v", waiting, failed, silent) + } +} diff --git a/cmd/mesh-controller/main.go b/cmd/mesh-controller/main.go index 41d7319d..d3f380b2 100644 --- a/cmd/mesh-controller/main.go +++ b/cmd/mesh-controller/main.go @@ -121,6 +121,10 @@ func run() error { return imagesCommand(ctx, args[1:]) case "mirrors": return mirrorsCommand(ctx, args[1:]) + case "move-placement": + return movePlacementCommand(ctx, args[1:]) + case "move-config": + return moveConfigCommand(ctx, args[1:]) case "plans": return plansCommand(ctx, args[1:]) case "delivery": @@ -275,6 +279,10 @@ func usage() { artifacts [--json] [--repository ] [--collected] every artifact the mesh made: kept and why, eligible, collected (ADR 0251) collect [--json] [--most ] what the store's sweep would let go of, now; nothing is done collect --confirm --why [--most ] run the sweep now: hold every kept archive, let go of the eligible + move-placement [--json] which settings layers still hold placement keys (endpoints, expose, reach, ports); nothing is moved + move-placement --confirm --why move them beside the assignment, once, after the delivery (issue 736) + move-config [--json] every config value the settings layers hold, and where in mesh-config it goes; nothing is set + move-config --confirm --why set them in mesh-config, read them back, and read config from there (issue 664) images [--json] every container image the machine's declaration names, now and as last sent seats [--json] every seat this mesh defines, what it delivers, and who holds it seat rename rename a seat; its former name still resolves (ADR 0122) @@ -331,7 +339,7 @@ func usage() { licence add|list|use|key model access, under the name a person calls it licence manager the node that holds a refreshable licence's refresh token licence refresh mint a new access token and seal it to every holder - rotate [--consumer ] [--module ] a new credential for every holder, both ends at once + rotate [--consumer ] [--module ] [--why ] a new credential for every holder, both ends at once (--why where the vault holds it) ask [json] call one of a module's tools over the broker, and print its answer pin which provider this one gets a provision from: the module, and its node diff --git a/cmd/mesh-controller/merge_gate.go b/cmd/mesh-controller/merge_gate.go index b555e4e5..ae1224eb 100644 --- a/cmd/mesh-controller/merge_gate.go +++ b/cmd/mesh-controller/merge_gate.go @@ -1097,6 +1097,19 @@ func raiseFromFacts(ctx context.Context, open *stores, f snapshot.Facts, shelf m continue } } + // And the provisioner client's account, the same way (novox/hq ADR 0349 clause 11, design 57 P3). + if a.Provider == "" && a.Name == broker.ProvisionerSecret { + if pm, held := shelf[a.Module]; !held || !runsTheProvisionerClient(pm) { + continue + } + if _, err := inv.MintBusPassword(ctx, inventory.BusUser{Kind: inventory.BusProvisioner, Node: m.Name, + Module: a.Module, Username: broker.Principal{Kind: broker.KindProvisioner, Node: m.Name, + Module: a.Module}.Username()}); err != nil { + notes = append(notes, fmt.Sprintf("%s's provisioner client account for %s is not kept: %s", + m.Described(), a.Module, oneLine(err.Error()))) + continue + } + } var err error if a.Provider == "" { err = inv.AcceptSecretForModule(ctx, m.Name, a.Module, a.Name, standIn) diff --git a/cmd/mesh-controller/meshcli.go b/cmd/mesh-controller/meshcli.go index 48c36865..c6cae0c9 100644 --- a/cmd/mesh-controller/meshcli.go +++ b/cmd/mesh-controller/meshcli.go @@ -7,7 +7,10 @@ import ( "fmt" "os/exec" "slices" + "sort" "strings" + "sync" + "time" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" @@ -123,8 +126,82 @@ func answerMeshCLI(inv *inventory.Inventory) link.CLIHandler { // cliJournal says one line in the controller's journal (its standard output); a variable so a test can read it. var cliJournal = func(line string) { fmt.Println(line) } -// runForMeshCLI runs a judged line and answers what it said. Every line is said in the journal first: its call, -// who asked on which node, its command word only, and how it runs (review of ADR 0272). +// onlyReads says a line only reads, judged by the generic `command` verb's reading forms (novox/hq ADR 0266) — the +// same allow list that decides what an ordinary call may run, never a second list kept here — judged on the line's +// every word, so `plans go`, `conditions silence` and `settings set` are not reads while `plans`, `conditions` and +// `settings show` are. A line those forms do not name is not judged a read, and is said (novox/hq issue 762). +func onlyReads(asked link.CLIAsked, v cliVerdict) bool { + if v.refused != "" || len(asked.Line) == 0 || len(asked.Stdin) > 0 || cliServers[asked.Line[0]] { + return false + } + return commandReads(asked.Line) == nil && terminalOnly(asked.Line) == nil +} + +// readCalls counts the read calls said in no line, per caller, until the next hourly line says them. +type readCalls struct { + mu sync.Mutex + counts map[string]int +} + +// cliReads are this controller's read calls not yet said. +var cliReads = &readCalls{} + +func (r *readCalls) add(caller string) { + r.mu.Lock() + defer r.mu.Unlock() + if r.counts == nil { + r.counts = map[string]int{} + } + r.counts[caller]++ +} + +// said is the line that says the read calls counted since the last, and starts counting again; empty when there +// were none, so an idle hour says nothing. +func (r *readCalls) said(over time.Duration) string { + r.mu.Lock() + counts := r.counts + r.counts = nil + r.mu.Unlock() + if len(counts) == 0 { + return "" + } + callers := make([]string, 0, len(counts)) + for c := range counts { + callers = append(callers, c) + } + sort.Strings(callers) + parts := make([]string, len(callers)) + for i, c := range callers { + parts[i] = fmt.Sprintf("%s %d", c, counts[c]) + } + return fmt.Sprintf("mesh-cli read calls in the last %s, said in no line of their own: %s", over, + strings.Join(parts, ", ")) +} + +// readCallsEvery is how often the count of read calls is said. +const readCallsEvery = time.Hour + +// sayReadCalls says, every readCallsEvery until ctx ends, how many read calls each caller made — when any did — so +// a runaway poller stays visible while its calls write no line each (novox/hq issue 762). +func sayReadCalls(ctx context.Context) { + tick := time.NewTicker(readCallsEvery) + defer tick.Stop() + for { + select { + case <-ctx.Done(): + return + case <-tick.C: + if line := cliReads.said(readCallsEvery); line != "" { + cliJournal(line) + } + } + } +} + +// runForMeshCLI runs a judged line and answers what it said. Every line that changes something, is refused or fails +// is said in the journal: its call, who asked on which node, its command word only, and how it runs (review of ADR +// 0272). A line that only reads and succeeds is counted instead, and the count said hourly (novox/hq issue 762): a +// page polling `plans` every few seconds would otherwise write thousands of lines a day. func runForMeshCLI(ctx context.Context, node string, asked link.CLIAsked, v cliVerdict) link.CLIAnswer { how := "as an ordinary call" switch { @@ -137,7 +214,23 @@ func runForMeshCLI(ctx context.Context, node string, asked link.CLIAsked, v cliV if call == "" { call = "(no call)" } - cliJournal(fmt.Sprintf("mesh-cli %s: %s on %s asked %q, %s", call, asked.Account, node, asked.Line[0], how)) + line := fmt.Sprintf("mesh-cli %s: %s on %s asked %q, %s", call, asked.Account, node, asked.Line[0], how) + if !onlyReads(asked, v) { + cliJournal(line) + return runJudgedLine(ctx, node, asked, v) + } + answer := runJudgedLine(ctx, node, asked, v) + if answer.Exit != 0 || answer.Refused != "" { + // A read that was refused or failed is said, as a change is. + cliJournal(fmt.Sprintf("%s, and it failed (exit %d)", line, answer.Exit)) + } else { + cliReads.add(fmt.Sprintf("%s on %s", asked.Account, node)) + } + return answer +} + +// runJudgedLine runs a line runForMeshCLI has said or counted. +func runJudgedLine(ctx context.Context, node string, asked link.CLIAsked, v cliVerdict) link.CLIAnswer { if v.refused != "" { return link.CLIRefusal(v.refused) } diff --git a/cmd/mesh-controller/meshcli_reads_test.go b/cmd/mesh-controller/meshcli_reads_test.go new file mode 100644 index 00000000..f56e2534 --- /dev/null +++ b/cmd/mesh-controller/meshcli_reads_test.go @@ -0,0 +1,123 @@ +package main + +import ( + "context" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/link" +) + +// echoFails makes the echoing test binary end with a failure, as a read that failed would. +const echoFails = "MESH_TEST_ECHO_FAILS" + +// journalled runs lines through runForMeshCLI and answers what the journal said, with a fresh count of read calls. +func journalled(t *testing.T) *[]string { + t.Helper() + t.Setenv(echoEnvironment, "1") + said := &[]string{} + was, wasReads := cliJournal, cliReads + cliJournal = func(line string) { *said = append(*said, line) } + cliReads = &readCalls{} + t.Cleanup(func() { cliJournal, cliReads = was, wasReads }) + return said +} + +// **A call that only reads writes no journal line; one that changes something does** (novox/hq issue 762). +func TestAReadCallWritesNoJournalLine(t *testing.T) { + said := journalled(t) + ctx := link.WithCallID(context.Background(), "call-1") + ordinary := cliVerdict{why: "not the terminal"} + for _, line := range [][]string{{"plans"}, {"status"}, {"conditions"}, {"node", "list"}, {"builds"}, {"queue"}, + {"settings", "show", "zsh"}, {"plans", "--json"}} { + if a := runForMeshCLI(ctx, "g14", cliAsked("jochen", 1000, line...), ordinary); a.Exit != 0 || a.Refused != "" { + t.Fatalf("%q did not run: %+v", line, a) + } + } + runForMeshCLI(ctx, "control", cliAsked("jochen", 1000, "plans"), cliVerdict{terminal: true, why: "the terminal"}) + if len(*said) != 0 { + t.Fatalf("read calls wrote journal lines: %q", *said) + } + runForMeshCLI(ctx, "control", cliAsked("jochen", 1000, "assign", "g14", "zsh"), cliVerdict{terminal: true, why: "the terminal"}) + if len(*said) != 1 || !strings.Contains((*said)[0], `asked "assign"`) { + t.Fatalf("a call that changes something was not said: %q", *said) + } +} + +// **A refused or failed read keeps its line** (novox/hq issue 762). +func TestAFailedOrRefusedReadIsSaid(t *testing.T) { + said := journalled(t) + runForMeshCLI(context.Background(), "g14", cliAsked("root", 0, "plans"), cliVerdict{refused: "root is not answered"}) + t.Setenv(echoFails, "1") + a := runForMeshCLI(context.Background(), "g14", cliAsked("jochen", 1000, "plans"), cliVerdict{why: "not the terminal"}) + if a.Exit == 0 { + t.Fatalf("the read did not fail: %+v", a) + } + if len(*said) != 2 || !strings.Contains((*said)[0], "refused") || !strings.Contains((*said)[1], "failed (exit 3)") { + t.Fatalf("the journal said %q", *said) + } + if line := cliReads.said(readCallsEvery); line != "" { + t.Fatalf("a failed read was counted as a read: %q", line) + } +} + +// **A verb is classed by what the call does, not its name** (novox/hq issue 762): the same command word reads with +// some arguments and writes with others, and a line no reading form names is said. +func TestAVerbIsClassedByItsArguments(t *testing.T) { + ordinary := cliVerdict{why: "not the terminal"} + for _, c := range []struct { + line []string + reads bool + }{ + {[]string{"settings", "show", "zsh"}, true}, + {[]string{"settings", "preferences"}, true}, + {[]string{"settings", "set", "zsh", `{"a":1}`}, false}, + {[]string{"settings", "clear", "zsh"}, false}, + {[]string{"plans"}, true}, + {[]string{"plans", "p-1"}, true}, + {[]string{"plans", "go", "p-1"}, false}, + {[]string{"plans", "--json", "stop", "p-1"}, false}, + {[]string{"plans", "close", "p-1"}, false}, + {[]string{"plans", "retry", "p-1"}, false}, + {[]string{"conditions"}, true}, + {[]string{"conditions", "show", "x"}, true}, + {[]string{"conditions", "silence", "x"}, false}, + {[]string{"doctor", "run"}, false}, + {[]string{"node", "account", "g14", "x"}, false}, + {[]string{"assign", "g14", "zsh"}, false}, + {[]string{"a-command-added-later"}, false}, + {[]string{"serve"}, false}, + } { + if got := onlyReads(cliAsked("jochen", 1000, c.line...), ordinary); got != c.reads { + t.Errorf("%q judged a read: %v, want %v", c.line, got, c.reads) + } + } + if onlyReads(cliAsked("jochen", 1000, "plans"), cliVerdict{refused: "no"}) { + t.Error("a refused line judged a read") + } + stdin := cliAsked("jochen", 1000, "status") + stdin.Stdin = []byte("x") + if onlyReads(stdin, cliVerdict{terminal: true}) { + t.Error("a line carrying standard input judged a read") + } +} + +// **Once an hour the read calls are counted per caller, only if there were any** (novox/hq issue 762). +func TestTheReadCallsAreCountedPerCaller(t *testing.T) { + said := journalled(t) + ordinary := cliVerdict{why: "not the terminal"} + for range 3 { + runForMeshCLI(context.Background(), "g14", cliAsked("jochen", 1000, "plans"), ordinary) + } + runForMeshCLI(context.Background(), "ace", cliAsked("jochen", 1000, "status"), ordinary) + if len(*said) != 0 { + t.Fatalf("read calls wrote lines: %q", *said) + } + line := cliReads.said(readCallsEvery) + if !strings.Contains(line, "jochen on ace 1, jochen on g14 3") || !strings.Contains(line, "1h0m0s") { + t.Fatalf("the hourly line said %q", line) + } + if again := cliReads.said(readCallsEvery); again != "" { + t.Fatalf("an hour with no read calls said %q", again) + } +} diff --git a/cmd/mesh-controller/missed_merges.go b/cmd/mesh-controller/missed_merges.go index 32ea7b30..1c9285e8 100644 --- a/cmd/mesh-controller/missed_merges.go +++ b/cmd/mesh-controller/missed_merges.go @@ -86,7 +86,7 @@ func owedToTheRecord(ctx context.Context, announced merges, m link.SourceMoved, // catchingUpOnMerges reads back the forge's announcements on a timer, until the context ends. func catchingUpOnMerges(ctx context.Context, open *stores, announced merges) { - f := following{open} + f := following{open: open} catalogued := func(ctx context.Context) ([]inventory.Entry, map[string][]inventory.ReadRepository, error) { entries, err := open.inventory.Catalogued(ctx) if err != nil { diff --git a/cmd/mesh-controller/module_health.go b/cmd/mesh-controller/module_health.go index 1975c6b7..39ba6f01 100644 --- a/cmd/mesh-controller/module_health.go +++ b/cmd/mesh-controller/module_health.go @@ -240,6 +240,7 @@ func judgeModuleHealth(ctx context.Context, inv *inventory.Inventory, k *conditi continue } o := moduleUnhealthyObservation(m, node, unhealthy[m]) + declaredSeverity(&o, hold, m) if hold != nil { if by, held := hold.heldWith(node, m, unhealthy[m]); held { // Held under the provider's condition: nothing of its own, and the provider's says it waits. The @@ -347,6 +348,7 @@ func sayWaiters(ctx context.Context, k *conditions.Keeper, hold *holding, p cata } else { _, rest := besideAWait(p.Module, rs) o = moduleUnhealthyObservation(p.Module, p.Node, rest) + declaredSeverity(&o, hold, p.Module) if said, waits := personWait(p.Module, p.Node, rest); waits { o = reloginObservation(p.Module, p.Node, said, operatorOn(ctx, hold.inv, p.Node), rest) } @@ -481,6 +483,19 @@ func sayWaitingOn(o *conditions.Observation, waiters []string) { o.Explanation += fmt.Sprintf(" %d module(s) that depend on it wait for it.", len(waiters)) } +// declaredSeverity raises a module's unhealthy condition urgent when its manifest declares `unhealthy` urgent +// (novox/hq ADR 0355 §1). Only that condition changes (ADR 0355 §4): a wait for a new login, a module that needs +// the operator and a directory used as found keep their severities, and without the field, or without a manifest +// to read it from, it stays the warning it was. +func declaredSeverity(o *conditions.Observation, hold *holding, module string) { + if o.Kind != kindModuleUnhealthy || hold == nil { + return + } + if m, ok := hold.manifestOf(module); ok && m.UnhealthyIsUrgent() { + o.Severity = conditions.Urgent + } +} + // moduleUnhealthyObservation is a module unhealthy on a machine, in words: the summary names the module, // the machine and what is wrong with each resource; the detail — targets, streaks, since — is evidence. func moduleUnhealthyObservation(module, node string, rs []inventory.ResourceHealth) conditions.Observation { diff --git a/cmd/mesh-controller/modules.go b/cmd/mesh-controller/modules.go index 7517eeca..80c58cab 100644 --- a/cmd/mesh-controller/modules.go +++ b/cmd/mesh-controller/modules.go @@ -446,6 +446,10 @@ func settingsCommand(ctx context.Context, args []string) error { if len(positionals) == 2 { values = positionals[1] } + // Once its config values are mesh-config's, they are proposed there (issue 664). + if forwarded, err := forwardPropose(ctx, inv, positionals[0], *node, values, *clear); forwarded || err != nil { + return err + } return proposeCommand(ctx, positionals[0], *node, values, *clear, *replace) case "proposals": if len(positionals) > 1 || *node != "" || *clear || *replace || *history { @@ -466,6 +470,15 @@ func settingsCommand(ctx context.Context, args []string) error { where = *node } + // Once the move of config has run, a module in the config form is read and set through mesh-config (issue 664). + if (args[0] == "set" && len(positionals) == 2) || ((args[0] == "show" && !*history) || args[0] == "clear") && + len(positionals) == 1 { + if forwarded, err := forwardSettings(ctx, inv, args[0], positionals[0], *node, positionals, *replace); forwarded || + err != nil { + return err + } + } + switch args[0] { case "set": if len(positionals) != 2 { @@ -949,7 +962,55 @@ func issueOnTheNewBus(ctx context.Context, inv *inventory.Inventory, m catalogue if err != nil { return err } - return issueWith(ctx, inv, m, node, busAddress, known, reachable, user, password) + if err := issueWith(ctx, inv, m, node, busAddress, known, reachable, user, password); err != nil { + return err + } + if !runsTheProvisionerClient(m) { + return nil + } + return issueProvisionerClient(ctx, inv, m, node, known, reachable) +} + +// runsTheProvisionerClient says a module runs its machine's provisioner client, which reaches the bus on an +// account of its own (novox/hq ADR 0349 clause 11, design 57 P3): it claims the seat and declares the own secret +// that account is delivered in. The same test the bus's user list makes (inventory's declaredFor). +func runsTheProvisionerClient(m catalogue.Manifest) bool { + _, reads := m.OwnSecrets[broker.ProvisionerSecret] + return reads && m.ClaimsSeat(broker.NodeProvisionerSeat) +} + +// issueProvisionerClient gives a machine's provisioner client its account: minted under the client's own name, +// never the module's or the tool runner's, and sealed to the machine as the module's own secret +// broker.ProvisionerSecret, which the client's service reads (novox/hq ADR 0349 clause 11, design 57 P3). No +// claims and no consumer: the client holds no seat on this account and is delivered nothing but its answers. +func issueProvisionerClient(ctx context.Context, inv *inventory.Inventory, m catalogue.Manifest, node string, + known broker.Broker, reachable string) error { + user := broker.Principal{Kind: broker.KindProvisioner, Node: node, Module: m.Module}.Username() + password, err := inv.MintBusPassword(ctx, inventory.BusUser{ + Username: user, Kind: inventory.BusProvisioner, Node: node, Module: m.Module, + }) + if err != nil { + return err + } + held, err := json.Marshal(struct { + URL string `json:"url"` + Fingerprint string `json:"fingerprint,omitempty"` + Node string `json:"node"` + User string `json:"user"` + Password string `json:"password"` + }{URL: "nats://" + reachable, Fingerprint: known.Fingerprint, Node: node, User: user, Password: password}) + if err != nil { + return err + } + if err := inv.AcceptSecretForModule(ctx, node, m.Module, broker.ProvisionerSecret, string(held)); err != nil { + return err + } + if err := handIssuedToVault(ctx, inv, m, node, broker.ProvisionerSecret, string(held)); err != nil { + return err + } + fmt.Printf("bus user %s minted for %s's provisioner client, scoped to %s's keys, its reports and the vault\n", + user, m.Module, node) + return nil } // busKindOf is what a module's bus user is recorded as: the node's tool runtime where the module is @@ -994,6 +1055,9 @@ func issueWith(ctx context.Context, inv *inventory.Inventory, m catalogue.Manife if err := inv.AcceptSecretForModule(ctx, node, m.Module, "broker", string(held)); err != nil { return err } + if err := handIssuedToVault(ctx, inv, m, node, "broker", string(held)); err != nil { + return err + } // And how it hears what it consumes. Derived from its declaration, and only when it declared // something: a module that consumes nothing needs no consumer, and creating one would be a diff --git a/cmd/mesh-controller/person_wait_test.go b/cmd/mesh-controller/person_wait_test.go index e52812b4..d5337899 100644 --- a/cmd/mesh-controller/person_wait_test.go +++ b/cmd/mesh-controller/person_wait_test.go @@ -30,6 +30,7 @@ func userUnit(module, account string) inventory.ResourceHealth { // Only what depends on the new login alone is a wait; anything else unhealthy of the module is judged as // before, so that a fault is never excused. func TestOnlyWhatDependsOnTheNewLoginIsAWait(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) notInGroup := relogin("lights", "operator") notInGroup.Reason = "not in the group lights: the apply has not put operator there; its outcome says why" systemUnit := userUnit("lights", "operator") @@ -74,6 +75,7 @@ func TestOnlyWhatDependsOnTheNewLoginIsAWait(t *testing.T) { // The gate reads a wait for a person as its own reading, never a fault, and passes with it; the same module // with a container down beside it is not yet healthy, as before. func TestAWaitForAPersonIsAPassCarriedAlong(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) now := time.Now() since := now.Add(-time.Minute) account := relogin("lights", "operator") @@ -102,6 +104,7 @@ func TestAWaitForAPersonIsAPassCarriedAlong(t *testing.T) { // The module's condition says the wait in one sentence for a person, and clears on the first statement that // no longer says it — said as the module's own fault when its unit still fails after the new login. func TestTheReloginConditionSaysTheWaitAndClearsAfterTheLogin(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) open := aMesh(t) ctx := t.Context() k, _ := withConditionsInMemory(t) @@ -158,6 +161,7 @@ func TestTheReloginConditionSaysTheWaitAndClearsAfterTheLogin(t *testing.T) { // A module that is healthy on its own keeps its pass when another module of the same send fails at the // bound: it is neither put back nor left without a verdict for every other walk to wait on. func TestAModuleHealthyOnItsOwnKeepsItsPassWhenItsSendFails(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) b := aBacklog(t) ctx := t.Context() inv := b.open.inventory @@ -207,6 +211,7 @@ func TestAModuleHealthyOnItsOwnKeepsItsPassWhenItsSendFails(t *testing.T) { // What the operator reads of the wait (ADR 0253, ADR 0254): it needs them, so it is never quiet, and it offers // no button, since nothing but their own new login can do it. func TestTheReloginConditionNeedsTheOperatorAndOffersNoButton(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) o := reloginObservation("openrazer", "g14", "relogin needed on g14: …", "operator", []inventory.ResourceHealth{ relogin("openrazer", "operator"), userUnit("openrazer", "operator")}) plainExample(t, o, "openrazer waits for a new login on g14", @@ -238,6 +243,7 @@ func TestTheReloginConditionNeedsTheOperatorAndOffersNoButton(t *testing.T) { // When a module not working turns into one waiting for a new login, and back, the clearing line says what it // became, never that it works again (issue 318 review). func TestAConditionThatBecameTheOtherKindSaysSoWhenItClears(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) open := aMesh(t) ctx := t.Context() store := conditions.NewInMemory() @@ -286,6 +292,7 @@ func TestAConditionThatBecameTheOtherKindSaysSoWhenItClears(t *testing.T) { // Which moves add an account group, read from the builds' manifests (issue 318 review). func TestOnlyAMoveThatAddsAnAccountGroupCanBringAWait(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) user := func(groups ...any) catalogue.Manifest { return catalogue.Manifest{Module: "lights", Resources: []map[string]any{{"id": "operator", "type": "user", "name": "operator", "groups": groups}}} diff --git a/cmd/mesh-controller/placement_move.go b/cmd/mesh-controller/placement_move.go new file mode 100644 index 00000000..1f1c289b --- /dev/null +++ b/cmd/mesh-controller/placement_move.go @@ -0,0 +1,99 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "os" + "strings" + + "github.com/novox/mesh-controller/internal/inventory" +) + +// movePlacementAnswer is what the move of placement says it did, or would do (issue 736). +type movePlacementAnswer struct { + // DryRun is true when nothing was moved: asked without --confirm. + DryRun bool `json:"dry-run"` + // Moved is each layer that held a placement key among its settings, and which keys. + Moved []inventory.MovedLayer `json:"moved"` +} + +// movePlacementCommand is `move-placement`: the move of placement out of the settings layers (novox/hq ADR 0348 +// clause 11, issue 736), one of the moves of data a release runs once after its delivery (design 57). +func movePlacementCommand(ctx context.Context, args []string) error { + set := flag.NewFlagSet("move-placement", flag.ContinueOnError) + asJSON := set.Bool("json", false, "answer as JSON") + confirm := set.Bool("confirm", false, "move the keys, rather than only say which layers hold them") + f := addHandActFlags(set) + positionals, err := parseAround(set, args) + if err != nil { + return err + } + if len(positionals) != 0 { + return errors.New("move-placement [--json] [--confirm --why ]") + } + if !*confirm && f.given() { + return errors.New("move-placement takes --why only with --confirm: without it nothing is moved. Nothing was done") + } + if *confirm { + // A move a person asks for says why, before anything is asked of the store. + if err := f.require("move-placement"); err != nil { + return err + } + } + open, err := openStores(ctx) + if err != nil { + return err + } + defer open.Close() + if *confirm { + f.record(ctx, "move-placement", nil) + } + answer, err := runMovePlacement(ctx, open.inventory, *confirm) + if err != nil { + return err + } + if *asJSON { + encoder := json.NewEncoder(os.Stdout) + encoder.SetIndent("", " ") + return encoder.Encode(answer) + } + fmt.Print(describeMovePlacement(answer)) + return nil +} + +// runMovePlacement is the move against a store: a dry run says what it would move. +func runMovePlacement(ctx context.Context, inv *inventory.Inventory, apply bool) (movePlacementAnswer, error) { + moved, err := inv.MovePlacement(ctx, apply) + if err != nil { + return movePlacementAnswer{}, err + } + return movePlacementAnswer{DryRun: !apply, Moved: moved}, nil +} + +// describeMovePlacement says the answer as a person reads it. +func describeMovePlacement(a movePlacementAnswer) string { + if len(a.Moved) == 0 { + return "no settings layer holds a placement key: nothing to move\n" + } + var b strings.Builder + verb := "moved" + if a.DryRun { + verb = "would move" + } + for _, m := range a.Moved { + where := m.Node + if where == "" { + where = "the whole mesh" + } + fmt.Fprintf(&b, "%s on %s: %s %s\n", m.Module, where, verb, strings.Join(m.Keys, ", ")) + } + if a.DryRun { + fmt.Fprintf(&b, "%d layer(s); nothing was moved — with --confirm --why they are\n", len(a.Moved)) + } else { + fmt.Fprintf(&b, "%d layer(s) moved beside their assignments, each kept whole in its history first\n", len(a.Moved)) + } + return b.String() +} diff --git a/cmd/mesh-controller/placement_words_moved_test.go b/cmd/mesh-controller/placement_words_moved_test.go new file mode 100644 index 00000000..63aa3185 --- /dev/null +++ b/cmd/mesh-controller/placement_words_moved_test.go @@ -0,0 +1,243 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "reflect" + "sort" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/link" +) + +// wordedMesh is every live layer that says `endpoints`, `expose`, `reach` or `ports` (issue 736) — 36, read with +// the settings verb on 2026-10-14, ace's as anchor's and novox's as laptop's, one stand-in for an address — split as +// the store holds them once migration 0097 has run: the four words still among the settings, `places` and +// `accesses` already beside them. With each module shaped as its definition in the catalogue (mesh-catalog's, and +// the media catalogue's) is where the four words reach: its listens, what its containers publish and on which +// network, its route, its directories and accesses, and its mergeable files — every key of the layer, or the keys +// they name. The shapes are the modules'; the files' text and the images are the test's. +type wordedMesh struct { + Modules []json.RawMessage `json:"modules"` + Layers []struct { + Node string `json:"node"` + Module string `json:"module"` + Settings map[string]any `json:"settings"` + Placement map[string]any `json:"placement"` + } `json:"layers"` +} + +func theWordedMesh(t *testing.T) wordedMesh { + t.Helper() + raw, err := os.ReadFile(filepath.Join("testdata", "placement-words-moved", "mesh.json")) + if err != nil { + t.Fatal(err) + } + var w wordedMesh + if err := json.Unmarshal(raw, &w); err != nil { + t.Fatal(err) + } + return w +} + +// aMeshWithTheWordsInItsLayers is aMesh with every module of the worded mesh registered, a route provider on both +// machines, and its layers held as the store holds them before the four words move. +func aMeshWithTheWordsInItsLayers(t *testing.T) (*stores, wordedMesh) { + t.Helper() + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + w := theWordedMesh(t) + image := "registry.invalid:5000/library@sha256:" + strings.Repeat("a", 64) + register(t, open, catalogue.Manifest{Module: "proxy", Version: "1", Provides: catalogue.FromAnywhere("route"), + Receives: map[string]string{"route": "/var/lib/proxy/routes.json"}}) + for _, text := range w.Modules { + m, err := catalogue.ParseManifest([]byte(strings.ReplaceAll(string(text), "IMAGE", image))) + if err != nil { + t.Fatal(err) + } + register(t, open, m) + } + on := map[string][]string{"anchor": {"proxy"}, "laptop": {"proxy"}} + for _, l := range w.Layers { + settings, err := json.Marshal(l.Settings) + if err != nil { + t.Fatal(err) + } + inv.ExecForTest(t, `insert into settings (node, module, values) select id, $2, $3 from node where name = $1`, + l.Node, l.Module, string(settings)) + if l.Placement != nil { + placement, err := json.Marshal(l.Placement) + if err != nil { + t.Fatal(err) + } + inv.ExecForTest(t, `insert into placement (node, module, values) select id, $2, $3 from node where name = $1`, + l.Node, l.Module, string(placement)) + } + on[l.Node] = append(on[l.Node], l.Module) + } + for _, node := range []string{"anchor", "laptop"} { + if err := inv.SetPublicDomain(ctx, node, node+".example.test"); err != nil { + t.Fatal(err) + } + if _, err := assign(ctx, open, node, on[node]...); err != nil { + t.Fatalf("assign %s %v: %v", node, on[node], err) + } + } + return open, w +} + +// toldOfTheWords is what a machine is told of the modules whose layers say the four words, and of the route +// provider that serves their names: those modules' resources and the filter's rules, as JSON a person can diff. +func toldOfTheWords(t *testing.T, open *stores, node string) string { + t.Helper() + ctx := t.Context() + gens, err := generators(ctx, open) + if err != nil { + t.Fatal(err) + } + plan, settings, err := planFor(ctx, open, node) + if err != nil { + t.Fatal(err) + } + with, _, err := renderingFor(ctx, open, node, plan, settings, gens, Allocating) + if err != nil { + t.Fatal(err) + } + composed, err := plan.Compose(with) + if err != nil { + t.Fatal(err) + } + if len(composed.LeftOut) > 0 { + t.Fatalf("%s leaves out %v", node, composed.LeftOut) + } + rules, err := plan.Rules(with) + if err != nil { + t.Fatal(err) + } + ours := map[string]bool{"proxy": true} + for _, l := range theWordedMesh(t).Layers { + ours[l.Module] = true + } + var told []map[string]any + for _, r := range composed.Resources { + if ours[composed.Owner[r["id"].(string)]] { + told = append(told, r) + } + } + sort.Slice(told, func(i, j int) bool { return told[i]["id"].(string) < told[j]["id"].(string) }) + got, err := json.MarshalIndent(map[string]any{"resources": told, "rules": rules}, "", " ") + if err != nil { + t.Fatal(err) + } + return string(got) + "\n" +} + +// sameAsTheGolden holds what a machine is told to the golden file the controller before issue 736 wrote from the +// same seed (MESH_WRITE_GOLDEN=1 writes it instead). +func sameAsTheGolden(t *testing.T, open *stores, when string) { + t.Helper() + for _, node := range []string{"anchor", "laptop"} { + got := toldOfTheWords(t, open, node) + golden := filepath.Join("testdata", "placement-words-moved", node+".json") + if os.Getenv("MESH_WRITE_GOLDEN") != "" { + if err := os.MkdirAll(filepath.Dir(golden), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(golden, []byte(got), 0o644); err != nil { + t.Fatal(err) + } + continue + } + want, err := os.ReadFile(golden) + if err != nil { + t.Fatal(err) + } + if string(want) != got { + t.Errorf("%s: %s is told something else:\n%s\nwas:\n%s", when, node, got, want) + } + } +} + +// **The four words leave the settings layers and no machine is told anything new** (novox/hq ADR 0348 clause 11, +// issue 736). Every live layer that says `endpoints`, `expose`, `reach` or `ports` is seeded as the store holds it +// after delivery, on a module shaped as its definition is, and every machine's declaration is composed. What each +// is told is byte for byte what the controller before this change composed from the same seed. +func TestADeclarationIsTheSameWhileTheWordsAreStillInTheLayers(t *testing.T) { + open, _ := aMeshWithTheWordsInItsLayers(t) + sameAsTheGolden(t, open, "before the move") +} + +// **And after the move, still nothing new** (issue 736). The move of placement, asked without confirming, says +// every layer that holds one of the four words and moves nothing; confirmed, it moves them all, no settings layer +// holds a placement key after, and each machine is told byte for byte what it was told before the move — and what +// the controller before this change told it. Asked again, it moves nothing. +func TestADeclarationIsTheSameAfterTheMoveOfPlacement(t *testing.T) { + open, w := aMeshWithTheWordsInItsLayers(t) + ctx := t.Context() + inv := open.inventory + asked, err := runMovePlacement(ctx, inv, false) + if err != nil || !asked.DryRun || len(asked.Moved) != len(w.Layers) { + t.Fatalf("the move, not confirmed, says %+v (%v); %d layers hold the words", asked, err, len(w.Layers)) + } + if held := inv.QueryForTest(t, `select module from settings where values ?| $1`, catalogue.PlacementKeys()); len(held) != len(w.Layers) { + t.Fatalf("the move, not confirmed, moved something: %d layers still hold the words", len(held)) + } + sameAsTheGolden(t, open, "after the move was asked without confirming") + + moved, err := runMovePlacement(ctx, inv, true) + if err != nil || moved.DryRun || !reflect.DeepEqual(moved.Moved, asked.Moved) { + t.Fatalf("the move says %+v (%v), it said it would move %+v", moved, err, asked.Moved) + } + if held := inv.QueryForTest(t, `select module from settings where values ?| $1`, catalogue.PlacementKeys()); len(held) != 0 { + t.Fatalf("a settings layer still holds a placement key: %v", held) + } + sameAsTheGolden(t, open, "after the move") + + if again, err := runMovePlacement(ctx, inv, true); err != nil || len(again.Moved) != 0 { + t.Fatalf("asked again, the move moved %+v (%v)", again, err) + } + if said := describeMovePlacement(movePlacementAnswer{}); !strings.Contains(said, "nothing to move") { + t.Fatalf("an empty move says %q", said) + } +} + +// The verb's command line: a dry run unless confirmed, and a confirmed move says why, recorded as a person's step +// of the release, never a repair (issue 736). +func TestTheMoveOfPlacementIsADryRunUnlessConfirmed(t *testing.T) { + for _, c := range []struct { + args map[string]any + want string + }{ + {map[string]any{}, "move-placement --json"}, + {map[string]any{"confirm": "false"}, "move-placement --json"}, + {map[string]any{"confirm": "true", "why": "release 57 is delivered"}, "move-placement --json --confirm --why release 57 is delivered"}, + } { + argv, err := argvFor("move-placement", c.args) + if err != nil || strings.Join(argv, " ") != c.want { + t.Errorf("%v: %q %v, want %q", c.args, argv, err, c.want) + } + } + for _, args := range []map[string]any{ + {"confirm": "true"}, // a move without a why + {"why": "tidy"}, // a why for a dry run, recorded nowhere + {"confirm": "yes", "why": "x"}, // a switch is true or false + {"node": "anchor"}, + } { + if argv, err := argvFor("move-placement", args); err == nil { + t.Errorf("%v was composed as %q", args, argv) + } + } + if repairingCommand([]string{"move-placement", "--json", "--confirm", "--why", "x"}) != "move-placement" { + t.Error("a confirmed move through the generic verb would go unrecorded") + } + if repairingCommand([]string{"move-placement", "--json"}) != "" { + t.Error("a dry run was taken for an act by hand") + } + if !personsDecision(link.HandAct{Verb: "move-placement"}) { + t.Error("a move of the release would count toward a healer the mesh lacks") + } +} diff --git a/cmd/mesh-controller/plain_words.go b/cmd/mesh-controller/plain_words.go index 7d60b16a..6b20b4a7 100644 --- a/cmd/mesh-controller/plain_words.go +++ b/cmd/mesh-controller/plain_words.go @@ -117,6 +117,8 @@ var plainWordings = map[string]func(conditions.Observation) words{ "reaches it. It keeps running what it has.", m), Resolved: m + " can get new instructions again"} }), + // A grant of a restricted stream that composes nothing (novox/hq ADR 0376 §2). + kindGrantComposesNothing: worded(withheldWords), kindAgentCanBecomeRoot: worded(func(o conditions.Observation) words { m := machineOr(o, "a machine") return words{Headline: "Sessions on " + m + " could become root", @@ -184,6 +186,15 @@ var plainWordings = map[string]func(conditions.Observation) words{ }), // The controller or the node-engine kept at the build a machine was last sent, over an older record (novox/hq // issue 201). + kindVaultSilent: worded(func(o conditions.Observation) words { + m := machineOr(o, "a machine") + return words{Headline: "The vault did not give " + m + " the core's credentials", + Needs: "check the vault answers (vault_status); the next send asks it again.", + Explanation: fmt.Sprintf("The bus's and the provisioner's credentials reach %s sealed by the vault, and "+ + "the vault did not answer the last send for them. %s keeps the files it has; a core module not "+ + "there yet is not placed. The mesh never seals them itself instead.", m, m), + Resolved: "Resolved: the vault gave " + m + " the core's credentials"} + }), kindCoreDowngrade: worded(func(o conditions.Observation) words { m := machineOr(o, "a machine") component := idPart(o, 0) @@ -620,6 +631,18 @@ var plainWordings = map[string]func(conditions.Observation) words{ Explanation: "The bus reports a listener too slow to keep up, so messages to it are late.", Resolved: "The listener keeps up again"} }), + kindBusFaultsUnread: worded(func(o conditions.Observation) words { + return words{Headline: "Slow clients of the bus cannot be checked", + Explanation: "The controller could not read what the bus said about its clients' connections, so a " + + "client the bus cut off would go unnoticed. It tries again at every check.", + Resolved: "Resolved: the bus's clients are checked again"} + }), + "connection-closed": worded(func(o conditions.Observation) words { + return words{Headline: "The bus cut a client off", + Explanation: "The bus closed a client's connection because it broke the bus's rules, so what it was " + + "sending was lost.", + Resolved: "Resolved: the client keeps to the bus's rules"} + }), kindBusReconnects: worded(func(o conditions.Observation) words { return words{Headline: "A client keeps losing the bus", Explanation: "One of the mesh's clients lost its connection to the bus again and again in the last hour. " + diff --git a/cmd/mesh-controller/plan.go b/cmd/mesh-controller/plan.go index acfb086d..f08199c4 100644 --- a/cmd/mesh-controller/plan.go +++ b/cmd/mesh-controller/plan.go @@ -150,6 +150,10 @@ func planFor(ctx context.Context, open *stores, nodeName string) (catalogue.Reso // password a provider is told to create is the one its consumer was given — and sealed to // this node before it was ever written down, so nothing between here and there can read it. for i, n := range resolved.Needs { + // **Issued for this holder alone and kept by the vault** (issue 757): where this consumer is handed to the + // provisioner, its machine's client asks the vault for it, and the declaration carries no value. + resolved.Needs[i].IssuedToHolder = issuedToHolder(shelf, n) + resolved.Needs[i].ProviderMoved = providerMoved(shelf, n) if n.ByRecord { // Answered by something the mesh holds, so there is no pair-wise secret between two // machines. Its key was supplied by a person and sealed to this node then; the mesh @@ -476,9 +480,23 @@ func declarationWith(ctx context.Context, open *stores, node string, out := sendable{Resources: composed.Resources, Adoption: adoption, Received: composed.Received, Mesh: with.Mesh, BusUsers: with.BusUsers, LeftOut: sortedKeysOf(composed.LeftOut), leftOutWhy: composed.LeftOut, withheld: with.Withheld, - unbound: with.Unbound, foreseen: composed.Foreseen, unplaced: composed.Unplaced, - modules: namedModules(plan, composed.LeftOut), + unbound: with.Unbound, foreseen: composed.Foreseen, unplaced: composed.Unplaced, handedOnly: composed.HandedOnly, + modules: namedModules(plan, composed.LeftOut), + Filled: composed.Filled, plans: composed.Plans, localNetwork: catalogue.LocalNetworkPorts(rules, with.LocalNetworks)} + if len(composed.Filled) > 0 { + out.Provisioner = with.Provisioner + } + // A handed module only pulled in by another keeps what requires it, for its value of its own in the assignments + // mirror (issue 765). + for module := range composed.Plans { + if by := plan.RequiredBy(module); by != "" { + if out.requiredBy == nil { + out.requiredBy = map[string]string{} + } + out.requiredBy[module] = by + } + } // And which build of each module it carries, for the send to record (novox/hq issue 259, ADR // 0221). Read only on the send path: a question about what would be sent records nothing. if choosing == Allocating { @@ -591,6 +609,10 @@ func reportLeftOut(node string, declared sendable) { for _, u := range declared.unplaced { fmt.Printf("%s: %s\n", node, u) } + // And each file only the provisioner's client fills, left out because the module is not handed here. + for _, u := range declared.handedOnly { + fmt.Printf("%s: %s\n", node, u) + } } // busCredentialIssued refuses an own secret called `broker` whose bus account nobody issued. @@ -601,10 +623,17 @@ func reportLeftOut(node string, declared sendable) { // cannot read and report the machine applied. Refused by name, with the verb — on the send, and on // a question asked ahead of it (Foreseeing), so that one is never told the push will make it. func busCredentialIssued(ctx context.Context, inv *inventory.Inventory, node, module, name string) error { - if name != "broker" { + var user string + switch name { + case "broker": + user = broker.Principal{Kind: broker.KindModule, Node: node, Module: module}.Username() + case broker.ProvisionerSecret: + // And the machine's provisioner client's account, issued with its module (novox/hq ADR 0349 clause 11, + // design 57 P3): the same refusal, for the same reason. + user = broker.Principal{Kind: broker.KindProvisioner, Node: node, Module: module}.Username() + default: return nil } - user := broker.Principal{Kind: broker.KindModule, Node: node, Module: module}.Username() if _, minted, err := inv.BusUserHash(ctx, user); err != nil { return err } else if !minted { @@ -728,7 +757,18 @@ func renderingFor(ctx context.Context, open *stores, node string, // per node, so a module running on three machines has three. needed := map[string]map[string]string{} foreseen := map[string]map[string]bool{} + // **A module handed to the provisioner here has its secrets from the vault** (novox/hq ADR 0349 §7, §11, + // design 57 V4): its machine's client asks for each, so the controller makes, reads and seals none of them for + // this machine. Every other module's are made and sealed as before. + hands, err := engineTakesHanded(ctx, inv, node) + if err != nil { + return catalogue.Rendering{}, inventory.Node{}, err + } + handing := catalogue.Rendering{Hands: hands, Provisioner: provisionerAccount(plan)} for _, m := range plan.Modules { + if handing.Handing(m) { + continue + } // **A secret family is never made** (novox/hq ADR 0283): each member a person gave on this machine is // placed, and one not given is nothing — the module says it waits for it. for _, family := range m.OwnSecrets.Families() { @@ -822,6 +862,22 @@ func renderingFor(ctx context.Context, open *stores, node string, needed[m.Module][name] = sealed } } + // **The core's own credentials as the vault sealed them** (novox/hq ADR 0349 §16, issue 755), for an engine that + // opens the vault's box (core_box.go): the declaration carries the box, never the seal made above. Below that + // contract nothing here is asked and the composition is as it was. + var core coreBoxes + opensBox, err := engineOpensVaultBox(ctx, inv, node) + if err != nil { + return catalogue.Rendering{}, inventory.Node{}, err + } + if opensBox { + core, err = coreBoxesFor(ctx, inv, node, plan.Modules, func(m catalogue.Manifest) bool { + return handing.Handing(m) + }, choosing) + if err != nil { + return catalogue.Rendering{}, inventory.Node{}, err + } + } // And a certificate for this machine's name inside the mesh, when anything on it asks. Issued // rather than stored: the node's key does not change, so signing again produces an equally // valid certificate and there is nothing to keep in step. @@ -1016,8 +1072,11 @@ func renderingFor(ctx context.Context, open *stores, node string, ReadsHealth: readsHealth, JudgesRoot: judgesRoot, ReadsBusy: readsBusy, + Hands: hands, + Provisioner: provisionerAccount(plan), BusMembership: memberships[node], Settings: settings, Generators: gens, Grants: grants, Needed: needed, Foreseen: foreseen, Ports: ports, + VaultSealed: core.sealed, NotPlaced: core.notPlaced, Certificate: certificate, Authority: authority, Mesh: private, Names: names, Machines: machines, Zones: zones, Holders: replicas, Suffix: overlay.Suffix(), MeshRange: meshRange, TunnelInterface: overlay.Interface, Accounts: accounts, Foundation: foundation, @@ -1059,6 +1118,35 @@ func engineReadsBusy(ctx context.Context, inv *inventory.Inventory, node string) return had && stated.Contract >= link.BusyContract, nil } +// engineTakesHanded says whether a machine's node-engine writes a filled file on the provisioner client's word alone +// (link.HandedContract, design 57 P7), by its own newest statement: only there is a moved module handed to the +// provisioner (design 57 P6). Anywhere else a filled file would have two writers, and the module is sent as before. +func engineTakesHanded(ctx context.Context, inv *inventory.Inventory, node string) (bool, error) { + stated, had, err := inv.HealthOf(ctx, node) + if err != nil { + return false, err + } + return had && stated.Contract >= link.HandedContract, nil +} + +// provisionerAccount is the account the provisioner's client runs as on this machine (novox/hq ADR 0349 §4): the user +// the process of the module running it runs as, empty where no such module is on the machine. +func provisionerAccount(plan catalogue.Resolution) string { + for _, m := range plan.Modules { + if !runsTheProvisionerClient(m) { + continue + } + for _, r := range m.Resources { + if r["type"] == "process" { + if user, _ := r["user"].(string); user != "" && user != "root" { + return user + } + } + } + } + return "" +} + // busyNotSent is, for a node-engine that does not read a container's busy, each container whose module declares // one: not sent it, so replaced at once while it may hold work (novox/hq ADR 0335). Nothing for one that reads it. func busyNotSent(modules []catalogue.Manifest, reads bool) []string { @@ -1846,6 +1934,42 @@ func issuedOutsideBy(shelf map[string]catalogue.Manifest, n catalogue.Needed) (s return catalogue.OffersIssuedOutside(shelf, n.Name) } +// issuedToHolder is whether the vault keeps a need's credential for its holder alone (issues 749, 757 and 779): its +// providing module is marked moved, so the vault holds the credential in a row of its own for each holder +// (filledTheNewWay), and its offer says either that the provider's software issues the provision's credential, or +// that the provider gives each consumer a login of its own (form pair, ADR 0158 §11). A credential every holder +// shares, or one a person gives, is neither. +func issuedToHolder(shelf map[string]catalogue.Manifest, n catalogue.Needed) bool { + if n.ByRecord || n.Module == "" { + return false + } + provider, known := shelf[n.Module] + if !known || !movedProvider(provider) { + return false + } + if _, issued := provider.IssuedCredentialOf(n.Name); issued { + return true + } + return provider.CredentialFormOf(n.Name) == catalogue.CredentialFormPair +} + +// providerMoved is whether a need is answered by a module marked moved (issue 788), or by one whose offer of the +// provision says the provider helper answers it (issue 792). +func providerMoved(shelf map[string]catalogue.Manifest, n catalogue.Needed) bool { + if n.ByRecord || n.Module == "" { + return false + } + provider, known := shelf[n.Module] + return known && (movedProvider(provider) || provider.AnsweredByHelperFor(n.Name)) +} + +// movedProvider is whether a provider's credentials are the vault's: the module is marked moved, or it holds the +// mesh-vault seat, whose pair rows have been in the vault since V3 (issue 793), so its consumers' credentials +// (gitea's internal token and admin password) are filled from those very rows with the values unchanged. +func movedProvider(m catalogue.Manifest) bool { + return m.Moved || slices.ContainsFunc(m.Claims, func(c catalogue.Claim) bool { return c.Name == "mesh-vault" }) +} + func providerModuleOf(resolved catalogue.Resolution, open *stores, ctx context.Context, n catalogue.Needed) string { for _, m := range resolved.Modules { if _, shared := m.SharedCredentialOf(n.Name); shared { diff --git a/cmd/mesh-controller/plan_retry.go b/cmd/mesh-controller/plan_retry.go index 93a3bcdc..acb722c8 100644 --- a/cmd/mesh-controller/plan_retry.go +++ b/cmd/mesh-controller/plan_retry.go @@ -372,6 +372,13 @@ func joinAPlan(ctx context.Context, open *stores, module string) (bool, string, if !found { return false, "", nil } + // **A walk that waits for its delivery's word asks nothing** (novox/hq ADR 0239, issue 754): a rebuild does not + // join it, and its outcome is not the walk's. The walk asks its own tier once it has its word. + if p.Waiting() { + return false, fmt.Sprintf("not joining %s (%s at %s): it waits for %s's word, so this build is not its "+ + "outcome; once let go, the walk asks %s itself", p.ID, p.Repository, short(p.Commit), p.Delivery.Awaits, + module), nil + } entries, err := inv.Catalogued(ctx) if err != nil { return false, "", err diff --git a/cmd/mesh-controller/planner_records_test.go b/cmd/mesh-controller/planner_records_test.go index 6d3e261c..9f56ef4c 100644 --- a/cmd/mesh-controller/planner_records_test.go +++ b/cmd/mesh-controller/planner_records_test.go @@ -133,7 +133,7 @@ func TestASharedRepositoryIsPlannedFromTheRecordedBuildSources(t *testing.T) { {"the route proxy's program: the route proxy alone", "mesh-controller", []string{"examples/route-proxy/main.go"}, "route-proxy", true}, {"a package all three build from: all three", "mesh-controller", - []string{"internal/broker/broker.go"}, "mesh-controller | build-agent | route-proxy", false}, + []string{"internal/broker/broker.go"}, "mesh-controller,route-proxy | build-agent", false}, {"the route proxy's directory in the catalogue: it alone", "mesh-catalog", []string{"modules/route-proxy/module.json"}, "route-proxy", false}, {"the build agent's directory: it alone, nothing it builds", "mesh-catalog", diff --git a/cmd/mesh-controller/planner_rules_test.go b/cmd/mesh-controller/planner_rules_test.go index 69ed505f..9e422c3a 100644 --- a/cmd/mesh-controller/planner_rules_test.go +++ b/cmd/mesh-controller/planner_rules_test.go @@ -145,8 +145,9 @@ func TestAPlanIsWhatTheChangeTouchedAndWhatIsBuiltOnIt(t *testing.T) { repo: "one", paths: []string{"modules/a/x", "modules/x/y"}, want: "a,x"}, {what: "built-by never widens", edges: []inventory.Edge{dep("x", builtBy, "a")}, repo: "one", paths: []string{"modules/a/x"}, want: "a"}, - {what: "built-by orders", edges: []inventory.Edge{dep("x", builtBy, "a")}, - repo: "one", paths: []string{"modules/a/x", "modules/x/y"}, want: "a | x"}, + // Whichever builder runs builds it (novox/hq ADR 0360 §3): built-by orders nothing, as before it ordered. + {what: "built-by does not order", edges: []inventory.Edge{dep("x", builtBy, "a")}, + repo: "one", paths: []string{"modules/a/x", "modules/x/y"}, want: "a,x"}, {what: "worker-of never widens", edges: []inventory.Edge{dep("x", workerOf, "a")}, repo: "one", paths: []string{"modules/a/x"}, want: "a"}, {what: "worker-of orders", edges: []inventory.Edge{dep("x", workerOf, "a")}, @@ -296,18 +297,18 @@ func TestASharedRepositoryMovesOnlyWhatItsBuildSourceHolds(t *testing.T) { []string{"internal/inventory/migrations/0088-a-build-says-its-build-source.sql"}, read, "mesh-controller | build-agent", ""}, // A package all three build from: all three; the build seat's holder after the controller whose worker - // it binds (worker-of, issue 206), the proxy after the holder that builds it (built-by). + // it binds (worker-of, issue 206), the proxy beside the controller: built-by orders nothing (ADR 0360 §3). {"a package all three build from: all three", "mesh-controller", - []string{"internal/broker/broker.go"}, read, "mesh-controller | build-agent | route-proxy", ""}, + []string{"internal/broker/broker.go"}, read, "mesh-controller,route-proxy | build-agent", ""}, {"a vendored package all three build from: all three", "mesh-controller", - []string{"vendor/github.com/nats-io/nats.go/nats.go"}, read, "mesh-controller | build-agent | route-proxy", ""}, + []string{"vendor/github.com/nats-io/nats.go/nats.go"}, read, "mesh-controller,route-proxy | build-agent", ""}, {"go.sum: all three", "mesh-controller", - []string{"go.sum"}, read, "mesh-controller | build-agent | route-proxy", ""}, + []string{"go.sum"}, read, "mesh-controller,route-proxy | build-agent", ""}, {"a file added to the proxy's package: the proxy", "mesh-controller", []string{"examples/route-proxy/new.go"}, read, "route-proxy", ""}, // Before any build source is said: as before. {"no build source said: a README moves all three, as before", "mesh-controller", - []string{"README.md"}, unsaid, "mesh-controller | build-agent | route-proxy", ""}, + []string{"README.md"}, unsaid, "mesh-controller,route-proxy | build-agent", ""}, // In the catalogue, where they live, each by its own build source. {"the route proxy's recipe: it alone", "mesh-catalog", []string{"modules/route-proxy/Dockerfile"}, read, "route-proxy", ""}, @@ -331,7 +332,7 @@ func TestASharedRepositoryMovesOnlyWhatItsBuildSourceHolds(t *testing.T) { // Files not all said: everything the repository builds, as before. m := link.SourceMoved{Owner: "novox", Repo: "mesh-controller", Base: "main", Commit: "head", Paths: []string{"README.md"}, PathsTruncated: true} - if got := tiered(reachOfMerge(m, entries, read, edges).Plan.Tiers); got != "mesh-controller | build-agent | route-proxy" { + if got := tiered(reachOfMerge(m, entries, read, edges).Plan.Tiers); got != "mesh-controller,route-proxy | build-agent" { t.Errorf("a merge whose files were not all said: planned %q, wanted all three", got) } } @@ -452,8 +453,9 @@ func TestAPlanIsTheTouchedModulesAndWhatIsReachableAlongTheWideningEdges(t *test kinds := []string{inventory.EdgeStandsOn, inventory.EdgeDeclared, inventory.EdgePackages, inventory.EdgeBuiltBy, inventory.EdgeWorkerOf} widens := map[string]bool{inventory.EdgeStandsOn: true, inventory.EdgeDeclared: true} + // Built-by orders nothing since novox/hq ADR 0360 §3. orders := map[string]bool{inventory.EdgeStandsOn: true, inventory.EdgeDeclared: true, - inventory.EdgeBuiltBy: true, inventory.EdgeWorkerOf: true} + inventory.EdgeWorkerOf: true} // Directory names drawn from one pool, so two repositories hold directories of the same name, and one // is a prefix of another. dirs := []string{"a", "ab", "b", "c", "lib/x", "lib/y", "modules/a", "modules/a/sub"} diff --git a/cmd/mesh-controller/probes.go b/cmd/mesh-controller/probes.go index 481571f7..deafc4e9 100644 --- a/cmd/mesh-controller/probes.go +++ b/cmd/mesh-controller/probes.go @@ -929,6 +929,12 @@ func streamDiffers(want broker.Stream, have nats.StreamConfig) string { differs = append(differs, fmt.Sprintf("keeps one of a message id for %s, defined %s", have.Duplicates, time.Duration(want.DuplicatesSeconds)*time.Second)) } + if want.AllowMsgTTL && !have.AllowMsgTTL { + differs = append(differs, "takes no time to live of a message's own, defined to (novox/hq ADR 0376)") + } + if want.Restricted && have.Metadata[broker.RestrictedMetadata] != "true" { + differs = append(differs, "does not say it is restricted, so a message kept from it would not age out with it") + } if want.DiscardNew && have.Discard != nats.DiscardNew { differs = append(differs, "drops what it holds when full, defined to refuse what comes next") } diff --git a/cmd/mesh-controller/proposals.go b/cmd/mesh-controller/proposals.go index 5bec29d7..aae0587e 100644 --- a/cmd/mesh-controller/proposals.go +++ b/cmd/mesh-controller/proposals.go @@ -54,6 +54,7 @@ import ( "git.novox.be/novox/mesh-sdk/go/asks" + "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" @@ -906,6 +907,21 @@ func setLayerIn(open *stores) setOnWarrant { return "", fmt.Errorf("the layer of %s on %s changed since the operator was shown the change: it is not set; "+ "propose it again", p.Module, p.where()) } + // A proposal that carries config values, raised before the module's config moved, is not applied after it: its + // values in the layer would turn ReadsConfig off and undo the move (issue 830, novox/hq ADR 0348). A proposal + // of placement only is set as before: forwardPropose sends it down this path on purpose. + carried := p.Values + if p.Clear { + carried = before + } + if config, _ := catalogue.SplitPlacement(carried); len(config) > 0 { + if reads, err := inv.ReadsConfig(ctx, p.Module); err != nil { + return "", err + } else if reads { + return "", fmt.Errorf("the config of %s moved to mesh-config since the operator was shown the change: it is "+ + "not set; propose it again (it then goes to mesh-config)", p.Module) + } + } var changed string if p.Clear { if err := inv.ClearSettingsBy(ctx, p.Node, p.Module, setBy); err != nil { diff --git a/cmd/mesh-controller/provisioned.go b/cmd/mesh-controller/provisioned.go new file mode 100644 index 00000000..0c888a17 --- /dev/null +++ b/cmd/mesh-controller/provisioned.go @@ -0,0 +1,100 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "os" + + "github.com/nats-io/nats.go" + + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/inventory" +) + +// What a machine's provisioner client said of a module's files (novox/hq ADR 0362, ADR 0349 §15, design 57 P6, +// issue 672), read where a walk's end and a gate count a machine's send applied: a module the send handed to the +// provisioner is applied there only once the client reported `prov.applied` on that build's templates as well. + +// provAppliedOn is the client's newest report of a module on a machine, from the stream that keeps it +// (broker.ProvAppliedStream); false when it has said nothing of it. A variable so a test can say the client's words. +var provAppliedOn = func(ctx context.Context, node, module string) (inventory.ProvReport, bool, error) { + var raw []byte + var said bool + read := func(js nats.JetStreamContext) error { + var err error + raw, said, err = broker.LastProvApplied(js, node, module) + return err + } + var err error + if d := doctorFrom; d != nil && d.js != nil { + err = read(d.js.Context()) + } else { + err = onTheBus(func(conn *nats.Conn) error { + js, err := conn.JetStream(nats.Context(ctx)) + if err != nil { + return err + } + return read(js) + }) + } + if err != nil || !said { + return inventory.ProvReport{}, false, err + } + var r inventory.ProvReport + if err := json.Unmarshal(raw, &r); err != nil { + return inventory.ProvReport{}, false, fmt.Errorf("what %s's provisioner client said of %s is not its report: %w", + node, module, err) + } + return r, true, nil +} + +// provLookup answers the clients' reports for one reading; a report that cannot be read is not counted, which +// leaves the machine waiting, then silent, never applied — and is said. +func provLookup(ctx context.Context) inventory.ProvLookup { + return func(node, module string) (inventory.ProvReport, bool) { + r, said, err := provAppliedOn(ctx, node, module) + if err != nil { + fmt.Fprintf(os.Stderr, "what %s's provisioner client said of %s could not be read: %v\n", node, module, err) + return inventory.ProvReport{}, false + } + return r, said + } +} + +// provisioningFailed is every module whose files a machine's provisioner client last said it failed to place, from +// the stream that keeps their reports. +func provisioningFailed(js nats.JetStreamContext) ([]inventory.ProvReport, error) { + all, err := broker.ProvAppliedReports(js) + if err != nil { + return nil, err + } + var out []inventory.ProvReport + for subject, raw := range all { + var r inventory.ProvReport + if err := json.Unmarshal(raw, &r); err != nil { + return nil, fmt.Errorf("what was said on %s is not a provisioner client's report: %w", subject, err) + } + if r.Status == inventory.ProvFailed { + out = append(out, r) + } + } + return out, nil +} + +// provisionedOutcome is a machine's report of one send as a delivery's end counts it (novox/hq ADR 0362 §1-5, issue +// 840): the engine's own report, combined with its provisioner client's for a send that handed modules to it. The +// outcome of a client's failure begins inventory.OutcomeProvisioningFailed; a report not yet applied on both sides +// is none (false). A send handing nothing is the engine's report unchanged. onSend says the report is about the send. +func provisionedOutcome(node string, sent inventory.SentDeclaration, r inventory.Reported, onSend bool, + prov inventory.ProvLookup) (string, bool) { + if len(sent.Provisioned) == 0 { + return r.Outcome, onSend + } + engine := inventory.AppliedReport{Outcome: r.Outcome} + if r.At != nil { + engine.At = *r.At + } + got, ok := inventory.ProvisionedSend(node, sent, engine, onSend, prov) + return got.Outcome, ok +} diff --git a/cmd/mesh-controller/provisioned_test.go b/cmd/mesh-controller/provisioned_test.go new file mode 100644 index 00000000..754ffcf8 --- /dev/null +++ b/cmd/mesh-controller/provisioned_test.go @@ -0,0 +1,114 @@ +package main + +import ( + "encoding/json" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" +) + +// The controller hands a moved module to the provisioner, and counts it applied only once its files are placed +// (design 57 P6, issue 672; novox/hq ADR 0349 §4, §15, ADR 0362). + +// A declaration that hands files says which, and whose client fills them; one that hands none is byte for byte +// what it was. +func TestADeclarationNamesTheFilledFilesOnlyWhenItHandsAny(t *testing.T) { + plain := sendable{Resources: []map[string]any{{"id": "notify.configuration", "type": "file", "path": "/x", + "content": "a"}}, Sequence: 4} + before, err := plain.Body() + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(before), "filled_files") || strings.Contains(string(before), "provisioner_account") { + t.Fatalf("a declaration handing nothing names filled files: %s", before) + } + handing := plain + handing.Filled, handing.Provisioner = []string{"/x"}, "mesh-provisioner-client" + body, err := handing.Body() + if err != nil { + t.Fatal(err) + } + var envelope struct { + Filled []string `json:"filled_files"` + Provisioner string `json:"provisioner_account"` + } + if err := json.Unmarshal(body, &envelope); err != nil || len(envelope.Filled) != 1 || envelope.Filled[0] != "/x" || + envelope.Provisioner != "mesh-provisioner-client" { + t.Fatalf("the handing declaration says %s", body) + } +} + +func TestTheProvisionersAccountIsTheClientsProcessUser(t *testing.T) { + client := catalogue.Manifest{Module: "mesh-provisioner-client", + OwnSecrets: catalogue.OwnSecrets{"provisioner-broker": {Path: "/var/lib/x/broker"}}, + Claims: []catalogue.Claim{{Name: "node-provisioner", Scope: "node"}}, + Resources: []map[string]any{{"id": "account", "type": "user", "name": "mesh-provisioner-client"}, + {"id": "client", "type": "process", "name": "mesh-provisioner-client", "user": "mesh-provisioner-client"}}} + if got := provisionerAccount(catalogue.Resolution{Modules: []catalogue.Manifest{client}}); got != "mesh-provisioner-client" { + t.Fatalf("the client's account is %q", got) + } + if got := provisionerAccount(catalogue.Resolution{}); got != "" { + t.Fatalf("a machine without a client names %q", got) + } +} + +// The first-machine gate counts a handed module healthy only once its node's prov.applied for the new build has +// come (ADR 0349 §15, kept by ADR 0362 for a walk finishing at its gate); a failure there fails it. +func TestTheGateWaitsForTheProvisionersReport(t *testing.T) { + now := time.Date(2026, 10, 14, 23, 0, 0, 0, time.UTC) + since := now.Add(-time.Minute) + reported := now.Add(-50 * time.Second) + digest := "sha256:" + strings.Repeat("d", 64) + facts := func(said map[string]inventory.ProvReport) gateFacts { + return gateFacts{now: now, + reports: map[string]inventory.Reported{"g14": {Node: "g14", Outcome: inventory.OutcomeApplied, + At: &reported, Current: true}}, + sent: map[string]inventory.SentDeclaration{"g14": {Digest: "sent", At: since, + Provisioned: map[string]string{"dunst": digest}}}, + provisioned: func(node, module string) (inventory.ProvReport, bool) { + r, ok := said[node+"."+module] + return r, ok + }} + } + m := catalogue.Manifest{Module: "dunst"} + if h, why := judgeHealth("dunst", "", m, "g14", since, facts(nil)); h != healthNotYet || + !strings.Contains(why, "has not reported dunst's files placed") { + t.Fatalf("judged without the provisioner's report: %v %q", h, why) + } + failed := map[string]inventory.ProvReport{"g14.dunst": {Status: inventory.ProvFailed, Templates: digest, + Reason: "a value is not there", At: now.Add(-10 * time.Second)}} + if h, why := judgeHealth("dunst", "", m, "g14", since, facts(failed)); h != healthBroken || + !strings.Contains(why, "a value is not there") { + t.Fatalf("a failed placing is not the build's failure there: %v %q", h, why) + } + placed := map[string]inventory.ProvReport{"g14.dunst": {Status: inventory.ProvApplied, Templates: digest, + At: now.Add(-10 * time.Second)}} + if h, why := judgeHealth("dunst", "", m, "g14", since, facts(placed)); strings.Contains(why, "provisioner") { + t.Fatalf("placed, the module is still held on the provisioner: %v %q", h, why) + } + // A module the send did not hand is judged as before. + if h, why := judgeHealth("picom", "", catalogue.Manifest{Module: "picom"}, "g14", since, facts(nil)); strings.Contains(why, "provisioner") { + t.Fatalf("a module not handed waits on the provisioner: %v %q", h, why) + } +} + +// A machine whose provisioner client failed to place a module's files has a failed apply, said as apply-failing +// once the failure outlasts an apply interval, with the client's reason (ADR 0362 §3). +func TestAFailedPlacingRaisesApplyFailing(t *testing.T) { + now := time.Date(2026, 10, 4, 14, 0, 0, 0, time.UTC) + f := calm(now) + f.provFailed = []inventory.ProvReport{{Node: "anchor", Module: "dunst", Status: inventory.ProvFailed, + Reason: "the templates cannot be fetched", At: now.Add(-time.Minute)}} + if got := watchApplyFailing(f); len(got) != 0 { + t.Fatalf("raised within an apply interval: %+v", got) + } + f.provFailed[0].At = now.Add(-10 * time.Minute) + got := watchApplyFailing(f) + if len(got) != 1 || got[0].Key() != "machine.anchor.apply-failing" || + !strings.Contains(got[0].Summary, "dunst") || !strings.Contains(got[0].Said, "the templates cannot be fetched") { + t.Fatalf("a failed placing is not raised as anchor's apply-failing: %+v", got) + } +} diff --git a/cmd/mesh-controller/provisioner_account_test.go b/cmd/mesh-controller/provisioner_account_test.go new file mode 100644 index 00000000..bf537bff --- /dev/null +++ b/cmd/mesh-controller/provisioner_account_test.go @@ -0,0 +1,62 @@ +package main + +import ( + "slices" + "testing" + + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" +) + +func aProvisioner() catalogue.Manifest { + return catalogue.Manifest{Module: "mesh-provisioner", Version: "1", + DefinesSeats: []catalogue.SeatDeclaration{{Name: broker.NodeProvisionerSeat, Scope: catalogue.ScopeNode, + Serves: []catalogue.Verb{{Name: "get"}}}}, + Claims: []catalogue.Claim{{Name: broker.NodeProvisionerSeat, Scope: catalogue.ScopeNode}}, + OwnSecrets: catalogue.OwnSecrets{"broker": {Path: "/run/mesh-provisioner/broker"}, + broker.ProvisionerSecret: {Path: "/run/mesh-provisioner/client"}}, + } +} + +// A machine's provisioner client is issued an account of its own, under its own name, sealed to the machine as the +// module's own secret its service reads (novox/hq ADR 0349 clause 11, design 57 P3); until it is, a push refuses +// to invent one, as it does for a module's `broker`. +func TestTheProvisionerClientsAccountIsIssuedForItsServiceAndNeverInvented(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + m := aProvisioner() + register(t, open, m) + if _, err := assign(ctx, open, "laptop", m.Module); err != nil { + t.Fatal(err) + } + if err := busCredentialIssued(ctx, inv, "laptop", m.Module, broker.ProvisionerSecret); err == nil { + t.Fatal("a push would seal a placeholder for the client's account before it was issued") + } + + if err := issueProvisionerClient(ctx, inv, m, "laptop", broker.Broker{Fingerprint: "ab"}, "10.0.0.1:4222"); err != nil { + t.Fatal(err) + } + if _, minted, err := inv.BusUserHash(ctx, "provisioner.laptop"); err != nil || !minted { + t.Fatalf("the client's account is not minted: %v %v", minted, err) + } + states, err := inv.SecretsOf(ctx, "laptop", m.Module) + if err != nil { + t.Fatal(err) + } + if !slices.ContainsFunc(states, func(s inventory.SecretState) bool { return s.Name == broker.ProvisionerSecret }) { + t.Fatalf("the client's account is not sealed to the machine for its service: %+v", states) + } + if err := busCredentialIssued(ctx, inv, "laptop", m.Module, broker.ProvisionerSecret); err != nil { + t.Fatalf("an issued client account is refused: %v", err) + } + if !runsTheProvisionerClient(m) { + t.Fatal("the module is not read as running the client") + } + unread := m + unread.OwnSecrets = catalogue.OwnSecrets{"broker": {Path: "/run/broker"}} + if runsTheProvisionerClient(unread) { + t.Fatal("a module with nowhere to read the client's account is read as running it") + } +} diff --git a/cmd/mesh-controller/push.go b/cmd/mesh-controller/push.go index 05bfc4c5..bc47d401 100644 --- a/cmd/mesh-controller/push.go +++ b/cmd/mesh-controller/push.go @@ -17,6 +17,7 @@ import ( "strings" "time" + "github.com/nats-io/nats.go" "github.com/novox/mesh-controller/internal/broker" "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/inventory" @@ -194,7 +195,7 @@ func serve(ctx context.Context) (err error) { // And what the catalogue decided a build meant. The builder's own result is already handled // above; this is the other half — the control plane is the only one of the three that knows // which machines run the thing, so it is the one that acts (novox/hq ADR 0072). - if err := server.Follows(following{open}); err != nil { + if err := server.Follows(following{open: open}); err != nil { return err } // And the merges the bus announced and never handed over, read back on a timer and acted on late @@ -202,7 +203,7 @@ func serve(ctx context.Context) (err error) { go catchingUpOnMerges(ctx, open, server) // And a catalogue that has just started, asking for what it missed. The same type answers // both: what a build meant and what the builds were are two questions about one record. - if err := server.Answers(following{open}); err != nil { + if err := server.Answers(following{open: open}); err != nil { return err } // And what providers say about consumers they keep failing, kept for `status` (novox/hq ADR @@ -256,7 +257,7 @@ func serve(ctx context.Context) (err error) { go forgetChecksOfWalk(context.WithoutCancel(ctx), open.inventory, p) } } - if err := server.Checks(following{open}); err != nil { + if err := server.Checks(following{open: open}); err != nil { return err } // Composed now and kept current, before the verb that answers from it is served. @@ -264,6 +265,12 @@ func serve(ctx context.Context) (err error) { // The facts snapshot a merge check is fed (novox/hq to-be 45 §9): kept current while this // controller holds the lease, read by the build seat from the artifact store. go exportingFacts(ctx, open, bus.Conn.ConnectedServerVersion) + // The record mirrors (novox/hq ADR 0289, design 57 F3): the assignments and each registered module's + // declared config on the bus, written from the store only when a record changed, by the lease holder. + // The provisioner's plan of each handed module, as its machine was last sent it (design 57 P6), in the module's + // assignment. + inv.SetProvisioningPlans(inv.SentProvisioningPlans) + go mirroringRecords(ctx, inv, bus.Conn, bus) // Every call carries the lease's epoch, and its record is written only under the lease (novox/hq // to-be 45 §6). link.Calls.UnderLease(func() (uint64, error) { return theLease.epoch(ctx) }) @@ -296,6 +303,8 @@ func serve(ctx context.Context) (err error) { return err } defer stopCLI() + // Its read calls write no line each; how many each caller made is said once an hour (novox/hq issue 762). + go sayReadCalls(ctx) // And says so on the bus (novox/hq ADR 0197): what it serves, as the NATS services protocol asks. stopAnnouncing, err := bus.Announce(seatAnnouncement(handlers), log.New(os.Stdout, "", log.LstdFlags)) if err != nil { @@ -1386,6 +1395,13 @@ func issueMemberships(ctx context.Context, open *stores, server *link.Server, se if issued > 0 { fmt.Printf(" issued %d membership(s)\n", issued) } + // **And what every other machine hears of a module's stream** (novox/hq ADR 0363 §6): a grant the provider's + // setting changed reaches a consumer on a machine this send does not reach. Its membership is issued again + // as last issued, with only what it hears changed, so nothing else of it moves before its own send. + reheard := reissueHearing(ctx, bus, records, where, sent) + if reheard > 0 { + fmt.Printf(" issued %d membership(s) again for what they hear of a module's stream\n", reheard) + } if len(refused) > 0 { // Returned, never passed over (novox/hq issue 249): the declarations of the machines they are // for are not sent, and the rollout that asked is tried again rather than waiting for a push. @@ -1685,7 +1701,7 @@ func recordSent(ctx context.Context, inv *inventory.Inventory, node string, body // send that is away look failed: it is said, loudly, and the machine's own record stands. err = inv.RecordSentWith(kept, record.ID, digest, builds, epoch, sent.toldGeneration, inventory.Send{ Sequence: sent.sequence, Epoch: int64(sent.epoch), Sender: sent.sender, Generation: sent.generation, - Digest: digest, Modules: sent.modules}) + Digest: digest, Modules: sent.modules, Provisioning: sent.plans}) var unrecorded *inventory.SendNotRecordedError switch { case errors.As(err, &unrecorded): @@ -1744,3 +1760,78 @@ func schemaReach() (int, error) { } return reach, nil } + +// reissueHearing issues again, on every machine not just sent, the membership of each assignment whose +// `consumes` names a module owning a stream, when what it hears changed: read as last issued from the bus, +// its `hears` replaced, and published. One that was never issued is left to its own send. Failures are said +// and passed over: the send that carries the grant is not the consumer's, and its own next send issues it. +func reissueHearing(ctx context.Context, bus link.OverNATS, records broker.Records, where broker.Placements, + sent []readyNode) int { + reached := map[string]bool{} + for _, s := range sent { + reached[s.node] = true + } + issued := 0 + for node, declared := range records.Assigned { + if reached[node] { + continue + } + for _, d := range declared { + if !d.HearsAStream { + continue + } + subject := broker.MembershipSubject(node, d.Module) + last, err := bus.JS.GetLastMsg(broker.AssignmentsStream, subject) + if err != nil { + if !errors.Is(err, nats.ErrMsgNotFound) { + fmt.Printf(" %s on %s: its membership could not be read to issue what it hears again: %v\n", + d.Module, node, err) + } + continue + } + body, changed, err := withHearing(last.Data, broker.HeardIssuedFor(node, d)) + if err != nil { + fmt.Printf(" %s on %s: its membership could not be read to issue what it hears again: %v\n", + d.Module, node, err) + continue + } + if !changed { + continue + } + if err := bus.PublishMembership(ctx, node, d.Module, body); err != nil { + fmt.Printf(" %s on %s: what it hears could not be issued again: %v\n", d.Module, node, err) + continue + } + issued++ + } + } + return issued +} + +// withHearing is a membership as issued with its `hears` replaced, and whether that changed it. Pure. +func withHearing(issued []byte, hears []broker.HeardIssued) ([]byte, bool, error) { + var fields map[string]json.RawMessage + if err := json.Unmarshal(issued, &fields); err != nil { + return nil, false, err + } + var had []broker.HeardIssued + if raw, ok := fields["hears"]; ok { + if err := json.Unmarshal(raw, &had); err != nil { + return nil, false, err + } + } + if slices.Equal(had, hears) { + return issued, false, nil + } + if len(hears) == 0 { + delete(fields, "hears") + } else { + raw, err := json.Marshal(hears) + if err != nil { + return nil, false, err + } + fields["hears"] = raw + } + body, err := json.Marshal(fields) + return body, true, err +} diff --git a/cmd/mesh-controller/queue.go b/cmd/mesh-controller/queue.go index 629fe5d4..e6bd065d 100644 --- a/cmd/mesh-controller/queue.go +++ b/cmd/mesh-controller/queue.go @@ -340,7 +340,10 @@ func cancelAsk(ctx context.Context, js *broker.JetStream, open *stores, seat str return "", fmt.Errorf("%s is marked cancelled and could not be deleted from the queue (seq %d): %w — a "+ "holder taking it ends it as cancelled", ask.ID, ask.Seq, err) } - recordCancelled(ctx, open, ask) + // A check ended because its pull request has a newer head records nothing (novox/hq issue 726): open is nil. + if open != nil { + recordCancelled(ctx, open, ask) + } return cancelSaid(ask, seat, removed), nil } @@ -518,7 +521,9 @@ func rebuildCommand(ctx context.Context, args []string) error { } // **A module a plan holds unbuilt, or failed, joins that plan** rather than running beside it: the - // plan would ask it again, or stay failed on an outcome a rebuild has replaced. + // plan would ask it again, or stay failed on an outcome a rebuild has replaced. A walk waiting for its word + // is not joined (novox/hq issue 754): the rebuild runs beside it, and says so. + var beside string if module != "" { joined, said, err := joinAPlan(ctx, open, module) if err != nil { @@ -528,6 +533,7 @@ func rebuildCommand(ctx context.Context, args []string) error { fmt.Println(said) return nil } + beside = said } id, err := askABuild(ctx, source, path, ref) if err != nil { @@ -535,6 +541,10 @@ func rebuildCommand(ctx context.Context, args []string) error { } recordAsked(ctx, inventory.BuildRequest{ID: id, Repository: source.Repository, Seat: source.Seat, Path: path, Ref: ref, For: "rebuild"}) + if beside != "" { + fmt.Printf("rebuild asked as %s, beside the walk: %s\n", id, beside) + return nil + } fmt.Printf("rebuild asked as %s\n", id) return nil } diff --git a/cmd/mesh-controller/queue_test.go b/cmd/mesh-controller/queue_test.go index 16096151..0a4a13fd 100644 --- a/cmd/mesh-controller/queue_test.go +++ b/cmd/mesh-controller/queue_test.go @@ -616,6 +616,7 @@ func TestKillAndPauseAskTheHolderOnTheMachine(t *testing.T) { // A plan that stopped at its first machine is retried: the module sent to that machine again, the // send recorded as the first anew, and the plan goes on — unless a newer plan holds the module. func TestAPlanStoppedAtItsFirstMachineIsRetried(t *testing.T) { + withTheGate(t) // the first-machine gate's own path (novox/hq ADR 0360 keeps it for earlier walks) open := aMesh(t) ctx := t.Context() asked := asksRecorded(t) @@ -636,8 +637,9 @@ func TestAPlanStoppedAtItsFirstMachineIsRetried(t *testing.T) { stopped := inventory.Plan{ID: "plan-rollout", Repository: "novox/a", Branch: "main", Commit: "c0ffee", Created: long, State: inventory.PlanFailed, Tiers: [][]string{{"a"}, {"b"}}, Note: "a stopped at its first machine in tier 0: laptop refused what it was sent", - Modules: map[string]*inventory.PlanModule{"a": {State: "built", BuiltAt: &long, Commit: "c0ffee", - First: []string{"laptop"}, FirstAt: &long, Why: "laptop refused what it was sent"}}} + // Asked by the plan, as every module a plan holds built is (novox/hq issue 754). + Modules: map[string]*inventory.PlanModule{"a": {State: "built", AskedAt: &long, Build: "build-a", BuiltAt: &long, + Commit: "c0ffee", First: []string{"laptop"}, FirstAt: &long, Why: "laptop refused what it was sent"}}} if err := open.inventory.SavePlan(ctx, &stopped); err != nil { t.Fatal(err) } diff --git a/cmd/mesh-controller/record_mirrors.go b/cmd/mesh-controller/record_mirrors.go new file mode 100644 index 00000000..6889d79e --- /dev/null +++ b/cmd/mesh-controller/record_mirrors.go @@ -0,0 +1,106 @@ +package main + +import ( + "context" + "fmt" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/recordmirror" +) + +// The controller's record mirrors on the bus (novox/hq ADR 0289, design 54, design 57 F3): the assignments, +// and each registered module's declared config, which mesh-config resolves a module's config values from. +// +// **Built from the store, on a tick of their own, by the lease holder alone.** A record changes through a +// verb that runs as a command of its own, a reconcile, a merge's registration or a pending assignment made, +// so no one call site sees every change; what every change has in common is that it is committed to the +// store. So the serving controller reads the store every mirrorEvery and writes only what differs from what +// the bucket holds (ADR 0289 §1): a quiet mesh writes nothing, and a change rolled back is never read. A +// mirror is therefore at most mirrorEvery behind its store. + +// mirrorEvery is how often the mirrors are compared with the store: two small reads of the store. +const mirrorEvery = 10 * time.Second + +// mirroringRecords keeps the mirrors current for as long as this controller serves: ctx ends when it stops. +func mirroringRecords(ctx context.Context, inv *inventory.Inventory, conn *nats.Conn, announce recordmirror.Announcer) { + api, err := jetstream.New(conn) + if err != nil { + fmt.Printf("the record mirrors are not kept: %v\n", err) + return + } + holds := func() bool { + epoch, err := theLease.epoch(ctx) + return err == nil && epoch > 0 + } + epoch := func() uint64 { + e, _ := theLease.epoch(ctx) + return e + } + sources := map[string]func(context.Context) (map[string][]byte, error){ + recordmirror.Assignments.Name: inv.AssignmentsMirror, + recordmirror.DeclaredConfig.Name: inv.DeclaredConfigMirror, + recordmirror.Predefined.Name: inv.PredefinedMirror, + } + writers := map[string]*recordmirror.Writer{} + failing := map[string]string{} + tick := time.NewTicker(mirrorEvery) + defer tick.Stop() + for { + if holds() { + for _, m := range recordmirror.Mirrors() { + why := mirrorOnce(ctx, api, m, writers, sources[m.Name], announce, epoch, holds) + if why != failing[m.Name] { + if why != "" { + fmt.Printf("the %s mirror cannot be kept: %s\n", m.Name, why) + } else { + fmt.Printf("the %s mirror is kept again\n", m.Name) + } + failing[m.Name] = why + } + } + } + select { + case <-ctx.Done(): + return + case <-tick.C: + } + } +} + +// mirrorOnce brings one mirror to what the store holds, and answers why it could not; empty when it could. +func mirrorOnce(ctx context.Context, api jetstream.JetStream, m recordmirror.Mirror, writers map[string]*recordmirror.Writer, + source func(context.Context) (map[string][]byte, error), announce recordmirror.Announcer, epoch func() uint64, + holds func() bool) string { + reading, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + w := writers[m.Name] + if w == nil { + kv, err := api.KeyValue(reading, m.Bucket) + if err != nil { + return fmt.Sprintf("the bucket %s cannot be read: %v", m.Bucket, err) + } + w = &recordmirror.Writer{Mirror: m, KV: kv, Announce: announce, Epoch: epoch, Holds: holds} + writers[m.Name] = w + } + want, err := source(reading) + if err != nil { + return fmt.Sprintf("the store cannot be read: %v", err) + } + lost := w.LostEvents + r, err := w.Sync(reading, want) + if r.Written+r.Removed > 0 { + fmt.Printf("the %s mirror moved: %d written, %d removed\n", m.Name, r.Written, r.Removed) + } + if w.LostEvents > lost { + fmt.Printf("the %s mirror lost %d event(s): the bucket holds the change, its notice was refused twice\n", + m.Name, w.LostEvents-lost) + } + if err != nil { + return err.Error() + } + return "" +} diff --git a/cmd/mesh-controller/recorded_kept.go b/cmd/mesh-controller/recorded_kept.go index bc9560ff..397bbd2b 100644 --- a/cmd/mesh-controller/recorded_kept.go +++ b/cmd/mesh-controller/recorded_kept.go @@ -158,6 +158,12 @@ func recordedKept(ctx context.Context, open *stores, node string) (map[string]st if f.identical(m, was, now.Commit) { continue } + // **Under its own `record`, a move that recreates nothing and moves no data is not kept** (novox/hq ADR 0377 + // §1): it goes as a module that rolls out goes. A person's `record`, the bus, and a move that cannot be + // judged are kept as before (§4, §5). + if !every && f.rolls(m, was) { + continue + } out[m] = was } // **And the controller and the node-engine at the build the machine was last sent, over an older record** diff --git a/cmd/mesh-controller/recorded_moves.go b/cmd/mesh-controller/recorded_moves.go new file mode 100644 index 00000000..e72bb0da --- /dev/null +++ b/cmd/mesh-controller/recorded_moves.go @@ -0,0 +1,213 @@ +package main + +import ( + "context" + "fmt" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" +) + +// Under a module's own `record`, a move that recreates no container and moves no data rolls out (novox/hq ADR +// 0377, changing ADR 0242 §1 and ADR 0360 §1). +// +// A module records because its manifest says so, because it keeps irreplaceable data, because a person set it, or +// because it provides the bus. The first two hold for a reason the manifest shows — a change that recreates mail's +// containers interrupts everyone's mail, a new database image may upgrade its databases in place — and that hold +// also held the moves that interrupt nothing: on 2026-10-15 the database server moved with only its tools bundle +// changed, its walk said it "recreates none of its containers", and ace stayed behind until a person pushed. +// +// So **a move under a module's own `record` is judged per machine, against the build that machine was last sent** +// (§2): when it recreates none of the module's containers and moves none of its data (catalogue.Holds, §3) it +// goes as a module that rolls out goes — a walk sends it, and every other send treats it as a move waiting for its +// walk — and otherwise it is kept at the build the machine runs, as before. The module's policy stays `record`. A +// move that cannot be judged waits (§4); a person's `record` and the bus hold every move (§5); and each walk says +// per machine which way a recorded module went (§6). + +// recordedRollsWhy is what a walk says of a recorded module's move that went out (ADR 0377 §6). +const recordedRollsWhy = "went out: it recreates none of its containers and moves none of its data (novox/hq ADR 0377)" + +// recordVerdict is one machine's move of a recorded module: whether it rolls out, and what the walk says of it. +type recordVerdict struct { + rolls bool + // said is what the walk says: recordedRollsWhy, or "waits: " and why — the recreate words, the data change + // named, the person's or the bus's hold, or that it could not be judged. + said string +} + +// recordedMove judges a module's move on a machine from the build it was last sent (`was`) to the one it would be +// sent now. A module that rolls out rolls; one whose policy is a person's or the bus's waits, said; one under its +// own `record` rolls when the move recreates nothing and moves no data, and waits when it does or cannot be judged. +func (f moveFacts) recordedMove(module, was string) recordVerdict { + now := f.current[module] + switch { + case now.RollOut: + return recordVerdict{rolls: true} + case now.From == catalogue.FromPerson: + return recordVerdict{said: "waits: a person set it to record, and holds every move of it until they push (novox/hq ADR 0377 §5)"} + case now.From == catalogue.FromBus: + return recordVerdict{said: "waits: it provides the bus, which moves only by the bus step a person starts (novox/hq ADR 0377 §5)"} + case !now.OwnRecord(): + return recordVerdict{said: "waits: its policy records"} + case f.judge == nil: + return recordVerdict{said: "waits: it could not be judged — the build records were not read (novox/hq ADR 0377 §4)"} + } + return f.judge(module, was, now.Commit) +} + +// rolls is whether a module's move on a machine from `was` goes as a move that rolls out (ADR 0377 §1). +func (f moveFacts) rolls(module, was string) bool { + return f.recordedMove(module, was).rolls +} + +// recordJudge is the judging of moves under a module's own `record`, reading the build records once per move and +// the catalogue once (ADR 0377 §2–§4). +func recordJudge(ctx context.Context, inv *inventory.Inventory) func(module, was, now string) recordVerdict { + judged := map[string]recordVerdict{} + var shelf map[string]catalogue.Manifest + var shelfErr error + shelfRead := false + return func(module, was, now string) recordVerdict { + key := module + "\x00" + was + "\x00" + now + if v, seen := judged[key]; seen { + return v + } + v := func() recordVerdict { + cannot := func(why string) recordVerdict { + return recordVerdict{said: "waits: it could not be judged — " + why + " (novox/hq ADR 0377 §4)"} + } + if was == "" { + return cannot("what the machine was last sent of it is not known") + } + from, found, err := inv.ManifestAt(ctx, module, was) + if err != nil { + return cannot(fmt.Sprintf("its build %s, which the machine was last sent, cannot be read: %v", short(was), err)) + } + if !found { + return cannot(fmt.Sprintf("its build %s, which the machine was last sent, is not in the build records", short(was))) + } + if !shelfRead { + shelf, shelfErr = inv.Catalogue(ctx) + shelfRead = true + } + if shelfErr != nil { + return cannot(fmt.Sprintf("the build it would be sent cannot be read: %v", shelfErr)) + } + to, held := shelf[module] + if !held { + return cannot("the build it would be sent is not in the catalogue") + } + if why := catalogue.Holds(from, to); why != "" { + return recordVerdict{said: "waits: " + why} + } + return recordVerdict{rolls: true, said: recordedRollsWhy} + }() + judged[key] = v + return v + } +} + +// judgeRecordedInWalk judges, for a walk carrying a module whose policy records, each machine running it whose +// build would move (ADR 0377 §2), keeps on the plan which way each went (§6), says it on the controller's log, and +// answers the machines its move goes out to. Judged once per walk: a walk read again answers what it kept. +func judgeRecordedInWalk(ctx context.Context, inv *inventory.Inventory, p *inventory.Plan, module string, + state *inventory.PlanModule) ([]string, error) { + if state.Recorded != nil { + return wentOut(state.Recorded), nil + } + running, err := inv.Running(ctx, module) + if err != nil { + return nil, err + } + f, err := readMoveFacts(ctx, inv) + if err != nil { + return nil, err + } + now := f.current[module] + said := map[string]string{} + for _, n := range running { + sent, known, err := inv.SentBuilds(ctx, n) + if err != nil { + return nil, err + } + was, carried := sent[module] + if known && carried && f.identical(module, was, now.Commit) { + continue // no move there + } + v := f.recordedMove(module, was) + if !known { + v = recordVerdict{said: "waits: it could not be judged — what " + n + " was last sent is not known (novox/hq ADR 0377 §4)"} + } + said[n] = v.said + fmt.Printf("%s: %s on %s %s\n", p.ID, module, n, v.said) + } + if len(said) > 0 { + state.Recorded = said + } + return wentOut(said), nil +} + +// wentOut is the machines a recorded module's move went out to, sorted. +func wentOut(recorded map[string]string) []string { + var out []string + for _, n := range sortedKeysOf(recorded) { + if recorded[n] == recordedRollsWhy { + out = append(out, n) + } + } + return out +} + +// recordedSaid is, for a walk's note at its end, which way each recorded module it carried went on each machine +// (ADR 0377 §6). +func recordedSaid(p inventory.Plan) []string { + var out []string + for _, m := range sortedKeysOf(planModuleNames(p)) { + s := p.Modules[m] + for _, n := range sortedKeysOf(s.Recorded) { + out = append(out, fmt.Sprintf("%s on %s %s", m, n, s.Recorded[n])) + } + } + return out +} + +func planModuleNames(p inventory.Plan) map[string]string { + out := map[string]string{} + for m, s := range p.Modules { + if s != nil && len(s.Recorded) > 0 { + out[m] = m + } + } + return out +} + +// recordedWaiting is, on one machine, every move of a recorded module that waits for a person, each said with why +// (ADR 0377 §6): what `status` names the machine behind for, with `push ` as the remedy. A move under a +// module's own `record` that recreates nothing and moves no data is not among them: it goes with its walk. +func recordedWaiting(ctx context.Context, inv *inventory.Inventory, f moveFacts, node string) ([]string, error) { + sent, known, err := inv.SentBuilds(ctx, node) + if err != nil || !known { + return nil, err + } + var out []string + for _, m := range sortedKeysOf(sent) { + was := sent[m] + now, held := f.current[m] + if !held || now.RollOut || now.Commit == "" || was == "" || f.identical(m, was, now.Commit) { + continue + } + if v := f.recordedMove(m, was); !v.rolls { + out = append(out, fmt.Sprintf("%s %s → %s %s", m, short(was), short(now.Commit), v.said)) + } + } + return out, nil +} + +// stringsKeys is a map of lists as a map of its keys, for sortedKeysOf. +func stringsKeys(m map[string][]string) map[string]string { + out := map[string]string{} + for k := range m { + out[k] = k + } + return out +} diff --git a/cmd/mesh-controller/recorded_moves_test.go b/cmd/mesh-controller/recorded_moves_test.go new file mode 100644 index 00000000..cc11869a --- /dev/null +++ b/cmd/mesh-controller/recorded_moves_test.go @@ -0,0 +1,361 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "slices" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" +) + +// Under a module's own `record`, a move that recreates no container and moves no data rolls out (novox/hq ADR +// 0377). Seen 2026-10-15: the database server moved with only its tools bundle changed, its walk said it +// "recreates none of its containers", and ace stayed behind until a person pushed. + +// aMailu is a mail module as the catalogue declares it, small: it records because its manifest says so, keeps its +// mailboxes as data, mounts its environment file, and serves its tools from a bundle. +func aMailu(policy bool) catalogue.Manifest { + raw := `{"module":"mailu","version":"1","tools":["mailu_status"], + "data":{"own":[{"id":"mail","path":"${dir:data-mail}","class":"valuable","why":"every mailbox"}]}, + "bundles":[{"name":"tools","source":"store/mailu-tools@sha256:aaa","digest":"sha256:aaa"}], + "resources":[ + {"id":"state","type":"directory"}, + {"id":"mesh-state","type":"directory"}, + {"id":"data-mail","type":"directory"}, + {"id":"config-env","type":"file","path":"${dir:state}/mailu.env","content":"A=1"}, + {"id":"runtime-config","type":"file","path":"${dir:mesh-state}/config.json","content":"{}"}, + {"id":"imap","type":"container","name":"mailu-imap","image":"registry.invalid/imap@sha256:1", + "volumes":["${dir:data-mail}:/mail"],"env-file":["${dir:state}/mailu.env"]}]}` + var m catalogue.Manifest + if err := json.Unmarshal([]byte(raw), &m); err != nil { + panic(err) + } + if policy { + m.Upgrade = &catalogue.UpgradePolicy{Policy: catalogue.PolicyRecord, Why: "people's mail: a person chooses the moment"} + } + return m +} + +func resourceOf(m *catalogue.Manifest, id string) map[string]any { + for _, r := range m.Resources { + if r["id"] == id { + return r + } + } + panic("no resource " + id) +} + +// recordedMesh is a mesh running mailu at build c1 on anchor and laptop, as each was last sent. +type recordedMesh struct { + open *stores + sent [][]string +} + +func aRecordedMesh(t *testing.T, c1 catalogue.Manifest) *recordedMesh { + t.Helper() + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + rm := &recordedMesh{open: open} + rm.register(t, c1, "c1", time.Now().Add(-2*time.Hour)) + for _, n := range []string{"anchor", "laptop"} { + if _, err := inv.Assign(ctx, n, c1.Module); err != nil { + t.Fatal(err) + } + if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n+"-c1", map[string]string{c1.Module: "c1"}); err != nil { + t.Fatal(err) + } + } + return rm +} + +// register keeps a build of a module and makes it the mesh's. +func (rm *recordedMesh) register(t *testing.T, m catalogue.Manifest, commit string, at time.Time) { + t.Helper() + ctx := t.Context() + raw, _ := json.Marshal(m) + if err := rm.open.inventory.RecordBuild(ctx, inventory.Build{ID: "build-" + m.Module + "-" + commit, Module: m.Module, + Commit: commit, Repository: "novox/mesh-catalog", Path: "modules/" + m.Module, Manifest: raw, Asked: at, At: at}); err != nil { + t.Fatal(err) + } + if err := rm.open.inventory.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", + Path: "modules/" + m.Module, BuiltFrom: commit, Head: commit, Asked: at}); err != nil { + t.Fatal(err) + } +} + +// keptOn is what an ordinary send — a walk's, a healer's — keeps on a machine: module → the build it keeps. +func (rm *recordedMesh) keptOn(t *testing.T, node string) map[string]string { + t.Helper() + ctx := t.Context() + under, err := sendKeeps(ctx, rm.open.inventory) + if err != nil { + t.Fatal(err) + } + kept, err := recordedKept(under, rm.open, node) + if err != nil { + t.Fatal(err) + } + return kept +} + +// walk opens a merge's walk of mailu's c2, built, and walks it with every send composed as an ordinary send +// composes it — a recorded module kept where its move waits — and applied at once. +func (rm *recordedMesh) walk(t *testing.T) inventory.Plan { + t.Helper() + ctx := t.Context() + inv := rm.open.inventory + wasSend := sendRollout + n := 0 + sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { + rm.sent = append(rm.sent, append([]string(nil), names...)) + current, err := open.inventory.CurrentBuilds(ctx) + if err != nil { + return nil, err + } + for _, node := range names { + kept := rm.keptOn(t, node) + builds := map[string]string{"mailu": current["mailu"].Commit} + if was, held := kept["mailu"]; held { + builds["mailu"] = was + } + n++ + digest := fmt.Sprintf("d-%s-%d", node, n) + if err := inv.RecordSent(ctx, nodeID(t, open, node), digest, builds); err != nil { + return nil, err + } + if _, err := inv.RecordDoing(ctx, nodeID(t, open, node), inventory.Doing{Node: node, + Outcome: inventory.OutcomeApplied, Declared: digest, Applied: 1, At: time.Now()}); err != nil { + return nil, err + } + } + return names, nil + } + t.Cleanup(func() { sendRollout = wasSend }) + built := time.Now().UTC() + plan := inventory.Plan{ID: "plan-mailu", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2", + Created: built, State: inventory.PlanBuilding, Tiers: [][]string{{"mailu"}}, + Modules: map[string]*inventory.PlanModule{"mailu": {State: "built", BuiltAt: &built, Commit: "c2", Build: "build-mailu-c2"}}} + if err := inv.SavePlan(ctx, &plan); err != nil { + t.Fatal(err) + } + for i := 0; i < 8; i++ { + advancePlans(ctx, rm.open) + } + p, err := inv.PlanByID(ctx, "plan-mailu") + if err != nil { + t.Fatal(err) + } + return p +} + +func (rm *recordedMesh) sentBuild(t *testing.T, node string) string { + t.Helper() + sent, _, err := rm.open.inventory.SentBuilds(t.Context(), node) + if err != nil { + t.Fatal(err) + } + return sent["mailu"] +} + +// recordedWaitsOn is what `status` names a machine behind for, waiting for a person (ADR 0377 §6). +func (rm *recordedMesh) recordedWaitsOn(t *testing.T, node string) []string { + t.Helper() + f, err := readMoveFacts(t.Context(), rm.open.inventory) + if err != nil { + t.Fatal(err) + } + held, err := recordedWaiting(t.Context(), rm.open.inventory, f, node) + if err != nil { + t.Fatal(err) + } + return held +} + +// §1, §6: a module that records because its manifest says so, whose move changes only its tools bundle, goes out +// with its walk to every machine, without a push; the walk says per machine that it went out and why; the policy +// still reads record; and status names no machine behind for it. +func TestARecordedModulesBundleOnlyMoveRollsOutInItsWalk(t *testing.T) { + c1 := aMailu(true) + rm := aRecordedMesh(t, c1) + c2 := aMailu(true) + c2.Bundles[0].Digest, c2.Bundles[0].Source = "sha256:bbb", "store/mailu-tools@sha256:bbb" + c2.Tools = append(c2.Tools, "mailu_queue") + rm.register(t, c2, "c2", time.Now().Add(-time.Minute)) + + for _, n := range []string{"anchor", "laptop"} { + if _, kept := rm.keptOn(t, n)["mailu"]; kept { + t.Fatalf("an ordinary send to %s keeps mailu at c1 for a move that recreates nothing", n) + } + } + p := rm.walk(t) + if p.State != inventory.PlanDone { + t.Fatalf("the walk is %s: %s", p.State, p.Note) + } + for _, n := range []string{"anchor", "laptop"} { + if got := rm.sentBuild(t, n); got != "c2" { + t.Fatalf("%s was sent mailu %s; want c2, with its walk", n, got) + } + if said := p.Modules["mailu"].Recorded[n]; said != recordedRollsWhy { + t.Fatalf("the walk says of mailu on %s %q", n, said) + } + if held := rm.recordedWaitsOn(t, n); len(held) != 0 { + t.Fatalf("status names %s behind for %v", n, held) + } + } + if !strings.Contains(p.Note, "mailu on anchor went out") || !strings.Contains(p.Note, "mailu on laptop went out") { + t.Fatalf("the walk's note does not say which way mailu went: %q", p.Note) + } + if u, err := rm.open.inventory.UpgradeOf(t.Context(), "mailu"); err != nil || u.Policy() != catalogue.PolicyRecord { + t.Fatalf("mailu's policy is %q (%v); want record still", u.Policy(), err) + } +} + +// §1, §2, §6: a move that recreates a container — a new image, or a changed declaration — still waits for a push; +// the walk says per machine that it waits, with the recreate words; and status names the machine behind, with +// `push ` as the remedy. A bundle-only change on top of it stays held: it is judged against what the machine +// was last sent. +func TestARecordedModulesMoveThatRecreatesAContainerWaits(t *testing.T) { + for name, change := range map[string]func(m *catalogue.Manifest){ + "image": func(m *catalogue.Manifest) { resourceOf(m, "imap")["image"] = "registry.invalid/imap@sha256:2" }, + "spec": func(m *catalogue.Manifest) { resourceOf(m, "imap")["health"] = map[string]any{"kind": "runtime"} }, + } { + t.Run(name, func(t *testing.T) { + rm := aRecordedMesh(t, aMailu(true)) + c2 := aMailu(true) + change(&c2) + c2.Bundles[0].Digest = "sha256:bbb" + rm.register(t, c2, "c2", time.Now().Add(-time.Minute)) + p := rm.walk(t) + for _, n := range []string{"anchor", "laptop"} { + if got := rm.sentBuild(t, n); got != "c1" { + t.Fatalf("%s was sent mailu %s; want c1 kept, waiting for a push", n, got) + } + if said := p.Modules["mailu"].Recorded[n]; !strings.HasPrefix(said, "waits: recreates 1 of mailu's 1") { + t.Fatalf("the walk says of mailu on %s %q", n, said) + } + if held := rm.recordedWaitsOn(t, n); len(held) != 1 || !strings.Contains(held[0], "recreates") { + t.Fatalf("status names %s behind for %v", n, held) + } + } + // §2: the next build changes only the bundle, and the machines were last sent c1: still held. + c3 := c2 + c3.Bundles = []catalogue.Bundle{{Name: "tools", Source: "store/mailu-tools@sha256:ccc", Digest: "sha256:ccc"}} + rm.register(t, c3, "c3", time.Now()) + if kept := rm.keptOn(t, "anchor"); kept["mailu"] != "c1" { + t.Fatalf("a bundle-only build on top of a held container change is not held: %v", kept) + } + }) + } +} + +// §3: a move touching a file a container mounts, a directory holding declared data, the data declaration or a +// provider waits. +func TestARecordedModulesMoveThatTouchesDataOrAProviderWaits(t *testing.T) { + for name, c := range map[string]struct { + change func(m *catalogue.Manifest) + says string + }{ + "a mounted file": {func(m *catalogue.Manifest) { resourceOf(m, "config-env")["content"] = "A=2" }, "which its container imap mounts"}, + "a data directory": {func(m *catalogue.Manifest) { resourceOf(m, "data-mail")["mode"] = "0700" }, "data-mail"}, + "the data declaration": {func(m *catalogue.Manifest) { + m.Data = &catalogue.Data{Own: []catalogue.DataItem{{ID: "mail", Path: "${dir:data-mail}", Class: "irreplaceable", Why: "x"}}} + }, "data declaration"}, + "a provider": {func(m *catalogue.Manifest) { m.Requires = append(m.Requires, "postgres-database") }, "what it requires"}, + } { + t.Run(name, func(t *testing.T) { + rm := aRecordedMesh(t, aMailu(true)) + c2 := aMailu(true) + c.change(&c2) + rm.register(t, c2, "c2", time.Now().Add(-time.Minute)) + kept := rm.keptOn(t, "anchor") + if kept["mailu"] != "c1" { + t.Fatalf("an ordinary send does not keep mailu at c1: %v", kept) + } + held := rm.recordedWaitsOn(t, "anchor") + if len(held) != 1 || !strings.Contains(held[0], c.says) { + t.Fatalf("status says %v; want it to name %q", held, c.says) + } + }) + } +} + +// §5: a module a person set to record, and the bus, wait on a bundle-only move. +func TestAPersonsRecordAndTheBusHoldEveryMove(t *testing.T) { + t.Run("a person sets record", func(t *testing.T) { + rm := aRecordedMesh(t, aMailu(false)) + c2 := aMailu(false) + c2.Bundles[0].Digest = "sha256:bbb" + rm.register(t, c2, "c2", time.Now().Add(-time.Minute)) + if err := rm.open.inventory.SetUpgradeOf(t.Context(), "mailu", inventory.Upgrade{Why: "the migration week", By: "jochen"}); err != nil { + t.Fatal(err) + } + p := rm.walk(t) + if got := rm.sentBuild(t, "anchor"); got != "c1" { + t.Fatalf("a person's record was walked past: anchor was sent %s", got) + } + if said := p.Modules["mailu"].Recorded["anchor"]; !strings.Contains(said, "a person set it to record") { + t.Fatalf("the walk says %q", said) + } + if held := rm.recordedWaitsOn(t, "anchor"); len(held) != 1 { + t.Fatalf("status names anchor behind for %v", held) + } + }) + t.Run("the bus", func(t *testing.T) { + bus := aMailu(false) + bus.Module = "nats" + bus.Provides = []catalogue.Offer{{Name: "mesh-bus"}} + bus.Data = nil + rm := aRecordedMesh(t, bus) + c2 := bus + c2.Bundles = []catalogue.Bundle{{Name: "tools", Source: "store/nats-tools@sha256:bbb", Digest: "sha256:bbb"}} + rm.register(t, c2, "c2", time.Now().Add(-time.Minute)) + if kept := rm.keptOn(t, "anchor"); kept["nats"] != "c1" { + t.Fatalf("the bus moved by an ordinary send: %v", kept) + } + f, err := readMoveFacts(t.Context(), rm.open.inventory) + if err != nil { + t.Fatal(err) + } + if v := f.recordedMove("nats", "c1"); v.rolls || !strings.Contains(v.said, "bus") { + t.Fatalf("the bus's move: %+v", v) + } + }) +} + +// §4: a move whose earlier build is not in the build records cannot be judged, waits, and is said so. +func TestARecordedMoveThatCannotBeJudgedWaits(t *testing.T) { + rm := aRecordedMesh(t, aMailu(true)) + c2 := aMailu(true) + c2.Bundles[0].Digest = "sha256:bbb" + rm.register(t, c2, "c2", time.Now().Add(-time.Minute)) + // anchor was last sent a build the records do not hold. + if err := rm.open.inventory.RecordSent(t.Context(), nodeID(t, rm.open, "anchor"), "d-anchor-c0", + map[string]string{"mailu": "c0"}); err != nil { + t.Fatal(err) + } + f, err := readMoveFacts(t.Context(), rm.open.inventory) + if err != nil { + t.Fatal(err) + } + if v := f.recordedMove("mailu", "c0"); v.rolls || !strings.Contains(v.said, "could not be judged") || + !strings.Contains(v.said, "not in the build records") { + t.Fatalf("an unjudgeable move: %+v", v) + } + if _, _, err := planFor(keepingRecorded(t.Context()), rm.open, "anchor"); err == nil || !strings.Contains(err.Error(), "push anchor") { + t.Fatalf("a send that cannot keep the unjudged build did not refuse with its remedy: %v", err) + } + // laptop, judged from c1, goes; anchor waits. + p := rm.walk(t) + if said := p.Modules["mailu"].Recorded["anchor"]; !strings.Contains(said, "could not be judged") { + t.Fatalf("the walk says of anchor %q", said) + } + if !slices.Contains(wentOut(p.Modules["mailu"].Recorded), "laptop") || rm.sentBuild(t, "laptop") != "c2" { + t.Fatalf("laptop, judged from c1, did not go: %+v", p.Modules["mailu"].Recorded) + } +} diff --git a/cmd/mesh-controller/reissue_hearing_test.go b/cmd/mesh-controller/reissue_hearing_test.go new file mode 100644 index 00000000..9375c583 --- /dev/null +++ b/cmd/mesh-controller/reissue_hearing_test.go @@ -0,0 +1,39 @@ +package main + +import ( + "encoding/json" + "testing" + + "github.com/novox/mesh-controller/internal/broker" +) + +// A membership issued again for what it hears changes nothing else of it (novox/hq ADR 0363 §6). +func TestAMembershipIssuedAgainChangesOnlyWhatItHears(t *testing.T) { + issued := []byte(`{"node":"ace","module":"reader","receives":{"x":[1]},"emits":"mesh.mod.reader.event.{event}"}`) + hears := []broker.HeardIssued{{Stream: "MAIL", Consumer: "ace_reader"}} + body, changed, err := withHearing(issued, hears) + if err != nil || !changed { + t.Fatalf("changed %v, %v", changed, err) + } + var got map[string]any + if err := json.Unmarshal(body, &got); err != nil { + t.Fatal(err) + } + if got["receives"] == nil || got["emits"] != "mesh.mod.reader.event.{event}" || got["hears"] == nil { + t.Fatalf("issued again as %s", body) + } + if _, again, _ := withHearing(body, hears); again { + t.Fatal("the same hearing issued again counts as a change") + } + gone, changed, err := withHearing(body, nil) + if err != nil || !changed { + t.Fatalf("a withdrawal: %v, %v", changed, err) + } + got = nil + if err := json.Unmarshal(gone, &got); err != nil { + t.Fatal(err) + } + if _, still := got["hears"]; still { + t.Fatalf("a withdrawn hearing is still issued: %s", gone) + } +} diff --git a/cmd/mesh-controller/release.go b/cmd/mesh-controller/release.go index 37d68311..5751b1cd 100644 --- a/cmd/mesh-controller/release.go +++ b/cmd/mesh-controller/release.go @@ -83,6 +83,9 @@ type moveFacts struct { plans []inventory.Plan // self is the release walking these moves, whose own gates are no other walk's (novox/hq ADR 0353). self string + // judge judges a move under a module's own `record` (novox/hq ADR 0377): nil when the facts were not read, + // and every such move then waits. + judge func(module, was, now string) recordVerdict } type walkingKey struct{} @@ -109,6 +112,7 @@ func readMoveFacts(ctx context.Context, inv *inventory.Inventory) (moveFacts, er } f.plans, err = inv.OpenPlans(ctx) f.self, _ = ctx.Value(walkingKey{}).(string) + f.judge = recordJudge(ctx, inv) return f, err } @@ -172,7 +176,8 @@ func inRun(run map[string]bool, commit string) bool { } // moves is what a machine's next send would move that no gate has seen: modules whose policy rolls out -// (a recorded one is a person's push), that the machine was sent before, moving to a build not identical +// (a recorded one is a person's push, unless its move under its own record recreates nothing and moves no data — +// novox/hq ADR 0377), that the machine was sent before, moving to a build not identical // to the one it runs and that has passed no gate. A machine whose last send's builds are not known names // none: the cascade holds it whole (ADR 0221). // @@ -185,7 +190,7 @@ func (f moveFacts) moves(node string, modules []string, sent map[string]string, for _, m := range modules { now := f.current[m] was, carried := sent[m] - if !now.RollOut || !carried || f.identical(m, was, now.Commit) || (!all && f.gated(m, now.Commit)) { + if !carried || f.identical(m, was, now.Commit) || (!all && f.gated(m, now.Commit)) || !f.rolls(m, was) { continue } out = append(out, inventory.CarriedMove{Module: m, Node: node, From: was, To: now.Commit}) @@ -750,6 +755,11 @@ func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool, if r.Rest { return advanceRest(ctx, open, p, now) } + if r.Gate == nil && !gateFirstMachine { + // **No first machine** (novox/hq ADR 0360 §7): every machine heard from is sent at once, in one step. + r.Rest = true + return advanceRest(ctx, open, p, now) + } if r.Gate == nil { heard, err := releaseHeard(ctx, open) if err != nil { @@ -1003,7 +1013,16 @@ func advanceRest(ctx context.Context, open *stores, p *inventory.Plan, now time. } // **A machine that says nothing on its send is left silent, not failed** (ADR 0353 §4): it is not judged // until it reports, and past ApplySilentAfter it is listed and left. - if !reportedOnSend(byNode[node], node, g) { + outcome, reported := byNode[node].Outcome, reportedOnSend(byNode[node], node, g) + if sent, kept := g.Sent[node]; kept && !gateFirstMachine { + // A module handed to the provisioner is applied only once its client said so too (ADR 0362, issue 840). + carried := make([]string, 0, len(g.Carried)) + for _, c := range g.Carried { + carried = append(carried, c.Module) + } + outcome, reported = provisionedOutcome(node, sent.Only(carried...), byNode[node], reported, provLookup(ctx)) + } + if !reported { if now.Sub(*g.Since) > inventory.ApplySilentAfter { g.Silent = true r.Silent = append(r.Silent, node) @@ -1015,6 +1034,32 @@ func advanceRest(ctx context.Context, open *stores, p *inventory.Plan, now time. judging = append(judging, node+": not yet reported on its send") continue } + if !gateFirstMachine { + // **Judged by its report alone** (novox/hq ADR 0360 §7): applied passes; failed or refused is named and + // fails the walk, with nothing put back — apply-failing says it. + switch outcome { + case inventory.OutcomeApplied: + decide(g, inventory.GatePassed, node+" reported it applied its send (novox/hq ADR 0360)", now) + passCarried(ctx, open, p, g, "") + r.Done = append(r.Done, node) + changed = true + case inventory.OutcomeFailed, inventory.OutcomeRefused: + decide(g, inventory.GateFailed, fmt.Sprintf("%s %s its send", node, outcome), now) + r.Failed = append(r.Failed, node) + changed = true + notes = append(notes, fmt.Sprintf("%s %s its send: nothing put back, apply-failing says it", node, outcome)) + default: + if strings.HasPrefix(outcome, inventory.OutcomeProvisioningFailed) { + decide(g, inventory.GateFailed, fmt.Sprintf("%s: %s", node, outcome), now) + r.Failed = append(r.Failed, node) + changed = true + notes = append(notes, fmt.Sprintf("%s: %s: nothing put back, apply-failing says it", node, outcome)) + continue + } + judging = append(judging, node+": not yet reported on its send") + } + continue + } verdict, err := judgeMoves(ctx, open, g, nil, now) if err != nil { return changed, err @@ -1099,6 +1144,9 @@ func advanceRest(ctx context.Context, open *stores, p *inventory.Plan, now time. // keptStepGrew says why a step's kept sends would now carry a build no gate passed that they were not kept with; // empty when none would. func keptStepGrew(ctx context.Context, open *stores, r *inventory.PlanRelease, pending []string) (string, error) { + if !gateFirstMachine { + return "", nil // every machine goes at once (novox/hq ADR 0360 §7): a send carries what waits when it is made + } f, err := readMoveFacts(ctx, open.inventory) if err != nil { return "", err @@ -1207,7 +1255,11 @@ func openRestStep(ctx context.Context, open *stores, p *inventory.Plan) (bool, s if err != nil { return false, "", err } - step := restStep(nodes, waiting, f.gated, func(module string) bool { return oneAtATime(module, shelf[module]) }) + // **Every machine at once** (novox/hq ADR 0360 §2, §7): no build waits for one machine to pass first. + step := nodes + if gateFirstMachine { + step = restStep(nodes, waiting, f.gated, func(module string) bool { return oneAtATime(module, shelf[module]) }) + } if len(step) == 0 { if len(walked) > 0 { return false, "waiting before " + strings.Join(walked, "; "), nil diff --git a/cmd/mesh-controller/release_branch_test.go b/cmd/mesh-controller/release_branch_test.go new file mode 100644 index 00000000..298b1588 --- /dev/null +++ b/cmd/mesh-controller/release_branch_test.go @@ -0,0 +1,273 @@ +package main + +import ( + "context" + "errors" + "reflect" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +// **A pull request into a release branch is judged by that branch** (novox/hq ADR 0358, issue 704): a base that is +// not its repository's trunk, where mesh-controller has a branch of that name, is a release branch; the check then +// reads every repository beside it at its branch of that name where it has one, else at its trunk. Any other base is +// checked exactly as before. + +const aRelease = "feat/57-config-and-provisioning" + +// theCore is the mesh's core as a check reads it beside: the controller, the node-engine, the tool runner and the +// catalogue's bus, on the git seat. +func theCore() []inventory.Entry { + onSeat := func(module, repository string) inventory.Entry { + e := fromRepo(module, repository, "") + e.Source.Seat = "git" + return e + } + nats := onSeat("nats", "novox/mesh-catalog") + nats.Source.Path = "modules/nats" + return []inventory.Entry{onSeat("mesh-controller", "novox/mesh-controller"), onSeat("mesh-host", "novox/mesh-host"), + onSeat("node-tools", "novox/mesh-tools"), nats, onSeat("gitea", "novox/mesh-catalog")} +} + +// aForgeWithBranches stands in for the forge: which branches each repository has, and how often it was asked. +func aForgeWithBranches(t *testing.T, branches map[string][]string, failing error) *int { + t.Helper() + asked := 0 + was := forgeHasBranch + t.Cleanup(func() { forgeHasBranch = was }) + forgeHasBranch = func(_ context.Context, owner, repo, branch string) (bool, error) { + asked++ + if failing != nil { + return false, failing + } + for _, b := range branches[owner+"/"+repo] { + if b == branch { + return true, nil + } + } + return false, nil + } + return &asked +} + +func TestABaseIsAReleaseBranchWhenTheControllerHasABranchOfThatName(t *testing.T) { + entries := theCore() + scopeOf := func(p link.PullUpdated) checkScope { return pullScope(p, entries, nil, nil) } + pull := func(repo, base string) link.PullUpdated { + return link.PullUpdated{Owner: "novox", Repo: repo, Number: 286, Base: base, Commit: "abc", + Paths: []string{"modules/gitea/index.ts"}} + } + + asked := aForgeWithBranches(t, map[string][]string{"novox/mesh-controller": {"main", aRelease}}, nil) + for _, c := range []struct { + what string + p link.PullUpdated + release string + asks int + }{ + {"a pull request into the trunk asks nothing and is no release", pull("mesh-catalog", "main"), "", 0}, + {"a catalogue pull request into the release branch", pull("mesh-catalog", aRelease), aRelease, 1}, + {"a controller pull request into the release branch", pull("mesh-controller", aRelease), aRelease, 1}, + {"a base mesh-controller has no branch of is checked as today", pull("mesh-catalog", "feat/other"), "", 1}, + {"no base: as today", pull("mesh-catalog", ""), "", 0}, + } { + *asked = 0 + got, err := releaseBranchOf(t.Context(), c.p, scopeOf(c.p), entries) + if err != nil { + t.Fatalf("%s: %v", c.what, err) + } + if got != c.release || *asked != c.asks { + t.Errorf("%s: release %q after %d question(s) to the forge, not %q after %d", c.what, got, *asked, + c.release, c.asks) + } + } + + // The branch a module built from the repository follows is its trunk: never asked about. + photos := fromRepo("photos", "novox/photos", "") + photos.Source.Ref = "nox-mesh" + withPhotos := append(theCore(), photos) + p := link.PullUpdated{Owner: "novox", Repo: "photos", Base: "nox-mesh", Commit: "abc", Paths: []string{"x.js"}} + *asked = 0 + if got, err := releaseBranchOf(t.Context(), p, pullScope(p, withPhotos, nil, nil), withPhotos); err != nil || + got != "" || *asked != 0 { + t.Fatalf("a module's own trunk read as release %q (%v) after %d question(s)", got, err, *asked) + } + + // A forge that cannot answer is an error, never "no release branch". + aForgeWithBranches(t, nil, errors.New("nothing serves gitea.gitea_api")) + q := pull("mesh-catalog", aRelease) + if _, err := releaseBranchOf(t.Context(), q, scopeOf(q), entries); err == nil || + !strings.Contains(err.Error(), "cannot be told") { + t.Fatalf("an unanswered forge was read as an answer: %v", err) + } +} + +func TestBesideAReleaseBranchCheckEachRepositoryIsReadAtThatBranchElseItsTrunk(t *testing.T) { + entries := theCore() + clone := func(s inventory.Source) (string, error) { return "http://git/" + s.Repository + ".git", nil } + running := func(module string) string { return "running-" + module } + + beside, err := besideOf(entries, running, clone, aRelease) + if err != nil { + t.Fatal(err) + } + at := func(repo string, ref, orElse string) link.CheckedOut { + return link.CheckedOut{Repository: "http://git/novox/" + repo + ".git", Ref: ref, Else: orElse} + } + want := map[string]link.CheckedOut{ + // The judge: mesh-controller's release branch, which it has. + "mesh-controller": at("mesh-controller", aRelease, ""), + "mesh-controller-main": at("mesh-controller", "main", ""), + // A controller pull request into the release branch reads mesh-host at that branch where it has one. + "mesh-host": at("mesh-host", aRelease, "main"), + "mesh-tools": at("mesh-tools", aRelease, "main"), + "mesh-catalog": at("mesh-catalog", aRelease, "main"), + // mesh-lab at its trunk, always: its replays run with the container runtime's socket. + "mesh-lab": at("mesh-lab", "main", ""), + "mesh-sdk": at("mesh-sdk", aRelease, "main"), + } + if !reflect.DeepEqual(beside, want) { + t.Fatalf("beside a release-branch check:\n got %v\nwant %v", beside, want) + } + + // Any other base: exactly as before — what the mesh runs, the catalogue's and the lab's main. + beside, err = besideOf(entries, running, clone, "") + if err != nil { + t.Fatal(err) + } + want = map[string]link.CheckedOut{ + "mesh-controller": at("mesh-controller", "running-mesh-controller", ""), + "mesh-controller-main": at("mesh-controller", "main", ""), + "mesh-host": at("mesh-host", "running-mesh-host", ""), + "mesh-tools": at("mesh-tools", "running-node-tools", ""), + "mesh-catalog": at("mesh-catalog", "main", ""), + "mesh-lab": at("mesh-lab", "main", ""), + "mesh-sdk": at("mesh-sdk", sdkPinned(), ""), + } + if !reflect.DeepEqual(beside, want) { + t.Fatalf("beside a trunk check:\n got %v\nwant %v", beside, want) + } +} + +func TestAReleaseBranchVerdictNamesTheBranchTheJudgeAndWhatItRead(t *testing.T) { + asked := &link.CheckRequest{Owner: "novox", Repo: "mesh-catalog", Number: 286, Base: aRelease, Release: aRelease} + c := checkedOf(link.BuildResult{ID: "b", Ref: "aaaaaaaaaaaa0000", Checked: asked, Check: &link.CheckOutcome{ + Verdict: "pass", Head: "aaaaaaaaaaaa", Trunk: "111111111111", Gate: &link.CheckLayer{Verdict: "pass"}, + RepoCheck: &link.CheckLayer{Verdict: "pass"}, Release: aRelease, JudgedBy: "fcdecb436117", + Beside: map[string]string{"mesh-host": "222222222222", "mesh-lab": "333333333333"}, + BesideAt: map[string]string{"mesh-host": aRelease, "mesh-lab": "main"}, + Validator: "mesh-host's validator of " + aRelease + " at 222222222222"}}) + if c.Release != aRelease || c.JudgedBy != "fcdecb436117" { + t.Fatalf("the verdict names release %q judged by %q", c.Release, c.JudgedBy) + } + if c.BesideAt["mesh-host"] != aRelease || c.BesideAt["mesh-lab"] != "main" || c.Beside["mesh-host"] != "222222222222" || + c.Validator != "mesh-host's validator of "+aRelease+" at 222222222222" { + t.Fatalf("the verdict names what it read beside it as %v at %v", c.Beside, c.BesideAt) + } +} + +// **Clause 5** (ADR 0358 §5): a release branch's own pull request into its trunk is no release-branch check: the forge +// is asked nothing, the request names no release branch, and the repositories beside it are read as for any trunk +// pull request. Its delivery group's composed check is asked as for any other (askGroupCheck, unchanged). +func TestAReleaseBranchsPullRequestIntoItsTrunkIsCheckedAsAnyTrunkPullRequest(t *testing.T) { + entries := theCore() + asked := aForgeWithBranches(t, map[string][]string{"novox/mesh-controller": {"main", aRelease}}, nil) + for _, repo := range []string{"mesh-catalog", "mesh-controller", "mesh-host"} { + p := link.PullUpdated{Owner: "novox", Repo: repo, Number: 400, Head: aRelease, Base: "main", Commit: "abc", + Paths: []string{"modules/gitea/index.ts"}} + got, err := releaseBranchOf(t.Context(), p, pullScope(p, entries, nil, nil), entries) + if err != nil || got != "" || *asked != 0 { + t.Fatalf("%s's release branch into main read as release %q (%v) after %d question(s)", repo, got, err, *asked) + } + } + clone := func(s inventory.Source) (string, error) { return "http://git/" + s.Repository + ".git", nil } + beside, err := besideOf(entries, func(module string) string { return "running-" + module }, clone, "") + if err != nil { + t.Fatal(err) + } + for dir, b := range beside { + if b.Ref == aRelease || b.Else != "" { + t.Fatalf("beside a trunk pull request %s is read at %+v", dir, b) + } + } +} + +// **A forge answer that names no branch is an error**, never "no release branch"; a 404 is "no such branch". +func TestTheForgesAnswerAboutABranchIsReadStrictly(t *testing.T) { + for _, c := range []struct { + what string + answer link.Answer + has bool + fails bool + }{ + {"the branch named", link.Answer{Result: []byte(`{"result":{"name":"` + aRelease + `"}}`)}, true, false}, + {"not found", link.Answer{Error: "Gitea API /repos/novox/mesh-controller/branches/x: 404 {\"message\":\"not found\"}"}, + false, false}, + {"an empty branch name", link.Answer{Result: []byte(`{"result":{"name":""}}`)}, false, true}, + {"no result", link.Answer{Result: []byte(`{}`)}, false, true}, + {"another branch named", link.Answer{Result: []byte(`{"result":{"name":"main"}}`)}, false, true}, + {"unreadable", link.Answer{Result: []byte(`not json`)}, false, true}, + {"another error", link.Answer{Error: "Gitea API: 500 internal"}, false, true}, + } { + has, err := branchAnswered(c.answer, aRelease) + if has != c.has || (err != nil) != c.fails { + t.Errorf("%s: has %v, error %v", c.what, has, err) + } + } +} + +// **The trunk is the repository's own, not a name**: a repository whose trunk is not main, one that follows main, and +// a controller whose trunk is not main. +func TestATrunkIsTheBranchTheRepositoryFollows(t *testing.T) { + asked := aForgeWithBranches(t, map[string][]string{"novox/mesh-controller": {"main", aRelease, "master"}}, nil) + legacy := fromRepo("legacy", "novox/legacy", "") + legacy.Source.Ref = "master" + onMain := fromRepo("photos", "novox/photos", "") + onMain.Source.Ref = "main" + entries := append(theCore(), legacy, onMain) + for _, c := range []struct { + what, repo, base, release string + asks int + }{ + {"a repository whose trunk is master, into master", "legacy", "master", "", 0}, + {"a repository whose trunk is master, into the release branch", "legacy", aRelease, aRelease, 1}, + {"a repository whose module follows main, into main", "photos", "main", "", 0}, + {"a repository whose module follows main, into the release branch", "photos", aRelease, aRelease, 1}, + } { + *asked = 0 + p := link.PullUpdated{Owner: "novox", Repo: c.repo, Base: c.base, Commit: "abc", Paths: []string{"x"}} + got, err := releaseBranchOf(t.Context(), p, pullScope(p, entries, nil, nil), entries) + if err != nil || got != c.release || *asked != c.asks { + t.Errorf("%s: release %q (%v) after %d question(s), not %q after %d", c.what, got, err, *asked, c.release, c.asks) + } + } + + // A controller whose trunk is not main: main may be a release branch there, and its own trunk never is. + core := theCore() + core[0].Source.Ref = "trunk" + for _, c := range []struct { + base, release string + asks int + }{{"trunk", "", 0}, {"main", "main", 1}} { + *asked = 0 + p := link.PullUpdated{Owner: "novox", Repo: "hq", Base: c.base, Commit: "abc", Paths: []string{"x"}} + got, err := releaseBranchOf(t.Context(), p, pullScope(p, core, nil, nil), core) + if err != nil || got != c.release || *asked != c.asks { + t.Errorf("hq into %s, the controller following trunk: release %q (%v) after %d question(s)", c.base, got, err, + *asked) + } + } +} + +// trunkOf: the branch a source follows, and main when it records none or records a commit (issue 215's rule). +func TestATrunkIsMainWhenTheSourceRecordsNone(t *testing.T) { + for ref, want := range map[string]string{"": "main", "main": "main", "master": "master", "nox-mesh": "nox-mesh", + "0123456789abcdef0123456789abcdef01234567": "main", " ": "main"} { + if got := trunkOf(inventory.Source{Ref: ref}); got != want { + t.Errorf("a source recording %q follows %q, not %q", ref, got, want) + } + } +} diff --git a/cmd/mesh-controller/release_plan.go b/cmd/mesh-controller/release_plan.go index f28516fa..46cf11fe 100644 --- a/cmd/mesh-controller/release_plan.go +++ b/cmd/mesh-controller/release_plan.go @@ -9,11 +9,13 @@ import ( "slices" "sort" "strings" + "sync" "time" "github.com/nats-io/nats.go" "github.com/novox/mesh-controller/internal/builder" + "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" ) @@ -26,6 +28,10 @@ import ( // alone cannot (a tier waiting for machines to report); a controller replaced mid-plan finds the // plan where it left it. +// gateFirstMachine sends a module that rolls out to one machine first and judges it there (ADR 0236). Off since +// novox/hq ADR 0360: the gate's code is kept for the walks a controller before it started, and its tests turn it on. +var gateFirstMachine = false + // planWaitBound is how long a plan may wait on one thing before `status` names it red. const planWaitBound = 30 * time.Minute @@ -42,29 +48,15 @@ func tiersOf(set []string, edges []inventory.Edge) [][]string { for _, m := range set { deps[m] = map[string]bool{} } - // The build seat's holders follow the controller that defines their worker (EdgeWorkerOf, - // novox/hq issue 206), so the built-by edge from that controller to such a holder yields: the - // controller is built by whichever build machine is running, as the runtime image always was. - worker := map[string]map[string]bool{} - for _, e := range edges { - if e.Kind == inventory.EdgeWorkerOf && in[e.From] && in[e.To] { - if worker[e.To] == nil { - worker[e.To] = map[string]bool{} - } - worker[e.To][e.From] = true - } - } for _, e := range edges { // A code dependency — B packages A's source — rebuilds B with A, in the same tier: B's - // build needs nothing of A's first. The other kinds order: stands-on and declared after - // the base is built, built-by after the build machine is built and running — except for - // what the build machine itself stands on, and for the controller whose worker the build - // machine binds. The runtime image is built by the builder and the builder is built on the - // runtime image; the image comes first, built by the builder that is running. + // build needs nothing of A's first. Stands-on and declared order after the base is built, and the + // build seat's holder follows the controller that defines its worker (EdgeWorkerOf, issue 206). if !in[e.From] || !in[e.To] || e.From == e.To || e.Kind == inventory.EdgePackages { continue } - if e.Kind == inventory.EdgeBuiltBy && (isBaseOf(e.From, e.To, edges, in) || worker[e.From][e.To]) { + // **Being built by the builder orders nothing** (novox/hq ADR 0360 §3): whichever builder runs builds it. + if e.Kind == inventory.EdgeBuiltBy { continue } deps[e.From][e.To] = true @@ -105,29 +97,6 @@ func tiersOf(set []string, edges []inventory.Edge) [][]string { return tiers } -// isBaseOf says whether `to` stands on `base`, directly or through other bases in the set, along -// the build edges alone. -func isBaseOf(base, to string, edges []inventory.Edge, in map[string]bool) bool { - seen := map[string]bool{} - var walk func(string) bool - walk = func(m string) bool { - if m == base { - return true - } - if seen[m] { - return false - } - seen[m] = true - for _, e := range edges { - if e.From == m && in[e.To] && (e.Kind == inventory.EdgeStandsOn || e.Kind == inventory.EdgeDeclared) && walk(e.To) { - return true - } - } - return false - } - return walk(to) -} - // reachableFrom is the moved modules plus everything that depends on them, through every layer: // what a merge rebuilds. Along the code and build edges only: a module *built by* the build machine // is not changed by a new build machine, so a built-by edge orders and gates a plan and never @@ -163,7 +132,7 @@ func reachableFrom(moved []string, edges []inventory.Edge) []string { } // hasCycle says whether the tiers' last tier holds modules that still depend on each other. A packages -// edge orders nothing (tiersOf), so a module and what packages its source share a tier by rule: that is +// or built-by edge orders nothing (tiersOf), so a module and what packages its source share a tier by rule: that is // no cycle, and saying one was is a false report in every such plan's log. func hasCycle(tiers [][]string, edges []inventory.Edge) bool { if len(tiers) == 0 { @@ -174,7 +143,7 @@ func hasCycle(tiers [][]string, edges []inventory.Edge) bool { last[m] = true } for _, e := range edges { - if e.Kind == inventory.EdgePackages { + if e.Kind == inventory.EdgePackages || e.Kind == inventory.EdgeBuiltBy { continue } if last[e.From] && last[e.To] { @@ -220,12 +189,6 @@ func gates(p inventory.Plan, edges []inventory.Edge, rollsOut func(string) bool) return nil } inTier := map[string]bool{} - all := map[string]bool{} - for _, tier := range p.Tiers { - for _, m := range tier { - all[m] = true - } - } for _, m := range p.Tiers[p.Tier] { inTier[m] = true } @@ -240,8 +203,8 @@ func gates(p inventory.Plan, edges []inventory.Edge, rollsOut func(string) bool) for _, e := range edges { // A delivery group's order inside a walk (novox/hq ADR 0276 decision 5) gates as built-by does: the // member before is running on its machines before the member after is asked. - ordered := e.Kind == edgeGroupOrder || - e.Kind == inventory.EdgeBuiltBy && !isBaseOf(e.From, e.To, edges, all) + // A built-by edge waits for nothing since novox/hq ADR 0360 §3. + ordered := e.Kind == edgeGroupOrder if later[e.From] && inTier[e.To] && ordered && !seen[e.To] && rollsOut(e.To) { seen[e.To] = true out = append(out, e.To) @@ -296,10 +259,11 @@ func applied(module string, builtAt time.Time, running []string, reports []inven return len(waiting) == 0, waiting } -// askTier asks the build machine for every module of the tier, and marks each asked. A module +// askTier asks the build machine for the named modules of the plan's tier — all of it, or what of it is still +// unasked (novox/hq issue 754) — and marks each asked. A module // the catalogue no longer holds, or whose ask could not be made, is a failure of the plan: a tier // half asked is a tier that will never complete. -func askTier(ctx context.Context, inv *inventory.Inventory, p *inventory.Plan) error { +func askTier(ctx context.Context, inv *inventory.Inventory, p *inventory.Plan, names []string) error { entries, err := inv.Catalogued(ctx) if err != nil { return err @@ -308,12 +272,62 @@ func askTier(ctx context.Context, inv *inventory.Inventory, p *inventory.Plan) e for _, e := range entries { byName[e.Manifest.Module] = e } - for _, name := range p.Tiers[p.Tier] { + for _, name := range names { askModule(ctx, p, name, byName) } return nil } +// unaskedOf are the modules of a plan's current tier it has still to ask: those it never asked, and one it holds +// built that it never asked for (novox/hq issue 754) — a build taken in by a controller from before a walk took +// only its own asks, somebody else's, which the walk would otherwise send as its own (ADR 0219). A tier partly asked is asked the rest, not +// waited on for ever: a walk that waits for every build of a tier it never asked whole never moves again. +func unaskedOf(p inventory.Plan) []string { + if p.Tier >= len(p.Tiers) { + return nil + } + var out []string + for _, m := range p.Tiers[p.Tier] { + s := p.Modules[m] + switch { + case s == nil || s.State == "": + out = append(out, m) + case s.State == "built" && s.Build == "" && s.AskedAt == nil: + out = append(out, m) + } + } + return out +} + +// askedInTier says the plan asked a module of its current tier itself: under an id, or at a time it kept. +func askedInTier(p inventory.Plan) bool { + if p.Tier >= len(p.Tiers) { + return false + } + for _, m := range p.Tiers[p.Tier] { + if s := p.Modules[m]; s != nil && (s.Build != "" || s.AskedAt != nil) { + return true + } + } + return false +} + +// takesOutcome says whether a build's outcome is the plan's for a module of its current tier (novox/hq ADR 0219, +// issues 219 and 754). **A walk that waits for its delivery's word asks nothing, and so is answered by nothing** +// (ADR 0239): a rebuild that lands in the wait is somebody else's build. **Nor is a module the plan never asked +// answered**: taken, the plan held a module built that it never asked, and waited for the rest of the tier for +// ever. Asked under an id, the plan takes that id's outcome and no other; asked before ids were kept, a build +// asked at or after the plan's ask. +func takesOutcome(p inventory.Plan, state *inventory.PlanModule, asked time.Time, id string) bool { + if p.Waiting() || state == nil || (state.Build == "" && state.AskedAt == nil) { + return false + } + if state.Build != "" { + return state.Build == id + } + return !askedBefore(asked, state.AskedAt) +} + // askABuild is how a plan asks for one build, not waited for, and learns the id it asked under. A // variable so a test of what a plan does around an ask needs no build machine. // @@ -386,15 +400,73 @@ var checkedBuildsOf = func(ctx context.Context, pulls []link.PullRef) ([]link.Ch return link.ReadCheckBuilds(ctx, js, buildSeatHeld(ctx), pulls, time.Now().Add(-builder.CheckBuildsKept)) } +// mergedHeadsOf are the head commits the walk's merges merged, by pull request (novox/hq issue 703): a pull request +// merged twice in one walk, or whose head the announcement did not say, has none, and takes no check's build. +func mergedHeadsOf(p *inventory.Plan) map[link.PullRef]string { + heads := map[link.PullRef]string{} + if p.Delivery == nil { + return heads + } + seen := map[link.PullRef]bool{} + for _, m := range p.Delivery.Merges { + pull, ok := pullOf(m) + if !ok { + continue + } + if seen[pull] && heads[pull] != m.Head { + heads[pull] = "" + continue + } + seen[pull] = true + heads[pull] = m.Head + } + return heads +} + +// walkVerdicts are the checks' builds a walk read, by the walk's plan: read once per walk (novox/hq issue 649), not +// once per module it asks for. Kept a day; a walk lasts minutes. +var walkVerdicts = struct { + sync.Mutex + read map[string]verdictsRead +}{read: map[string]verdictsRead{}} + +type verdictsRead struct { + at time.Time + builds []link.CheckedBuild +} + // reuseWithVerdicts is a walk's leave with the builds its pull requests' verdicts name: the only builds the builder -// may take (ADR 0350 §2). When they cannot be read the leave names none, and the walk builds, saying why. -func reuseWithVerdicts(ctx context.Context, r *link.ReuseAsk) *link.ReuseAsk { - builds, err := checkedBuildsOf(ctx, r.Pulls) - if err != nil { - fmt.Printf(" the checks' builds of %d pull request(s) could not be read, so the walk builds: %v\n", len(r.Pulls), err) - return r +// may take (ADR 0350 §2), and **only those of a check of the head each merge merged** (novox/hq issue 703): a build +// a check made at an earlier head of the same pull request is never registered. Read once per walk. When they +// cannot be read the leave names none, and the walk builds, saying why. +func reuseWithVerdicts(ctx context.Context, walk string, r *link.ReuseAsk, heads map[link.PullRef]string) *link.ReuseAsk { + walkVerdicts.Lock() + read, known := walkVerdicts.read[walk] + walkVerdicts.Unlock() + if !known || walk == "" { + builds, err := checkedBuildsOf(ctx, r.Pulls) + if err != nil { + fmt.Printf(" the checks' builds of %d pull request(s) could not be read, so the walk builds: %v\n", len(r.Pulls), err) + return r + } + read = verdictsRead{at: time.Now(), builds: builds} + if walk != "" { + walkVerdicts.Lock() + for id, w := range walkVerdicts.read { + if time.Since(w.at) > 24*time.Hour { + delete(walkVerdicts.read, id) + } + } + walkVerdicts.read[walk] = read + walkVerdicts.Unlock() + } + } + r.Builds = nil + for _, b := range read.builds { + if link.SameHead(b.Head, heads[b.Pull]) { + r.Builds = append(r.Builds, b) + } } - r.Builds = builds return r } @@ -437,7 +509,8 @@ func askModule(ctx context.Context, p *inventory.Plan, name string, byName map[s } // **The walk takes the check's build of the same key** (novox/hq ADR 0350 §2): its ask says so, with the pull // requests it answers, whose checks name the part of a differing key. - id, err := askABuild(withReuse(ctx, reuseWithVerdicts(ctx, reuseAskOf(p))), source, e.Source.Path, ref) + id, err := askABuild(withReuse(ctx, reuseWithVerdicts(ctx, p.ID, reuseAskOf(p), mergedHeadsOf(p))), source, + e.Source.Path, ref) if err != nil { state.State = "failed" state.Why = err.Error() @@ -504,22 +577,15 @@ func planBuiltSaying(ctx context.Context, open *stores, module, commit, failed s if !inTier { continue } - state := p.Modules[module] - if state == nil { - state = &inventory.PlanModule{} - p.Modules[module] = state - } // **The plan's own ask is its outcome, by id** (novox/hq ADR 0219); another build of the module // is, as before, when it was asked at or after the plan's ask (issue 219). // **A module asked under an id is answered by that id's outcome and no other** (novox/hq ADR // 0219): a replay, a rebuild beside the plan, or an older ask finishing late is somebody else's // build, made from other source, and settling the plan with it would send that. A plan from - // before ids were kept is matched as it was: by when the build was asked (issue 219). - if state.Build != "" { - if state.Build != id { - continue - } - } else if askedBefore(asked, state.AskedAt) { + // before ids were kept is matched as it was: by when the build was asked (issue 219). A walk + // waiting for its word, or a module it never asked, takes no outcome at all (issue 754). + state := p.Modules[module] + if !takesOutcome(*p, state, asked, id) { continue } if failed != "" && deletedAtSource(failed) { @@ -725,17 +791,11 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan, return true, nil } tier := p.Tiers[p.Tier] - // Not yet asked: ask. - unasked := 0 - for _, m := range tier { - if s := p.Modules[m]; s == nil || s.State == "" { - unasked++ - } - } - if unasked == len(tier) { + // Not yet asked: ask — the whole tier, or what of it is still unasked (novox/hq issue 754). + if unasked := unaskedOf(*p); len(unasked) > 0 { // **One walk at a time** (novox/hq ADR 0276): a walk about to ask its first tier while another started // walk is open waits for that one to end, let go or not, and says so. - if p.Tier == 0 { + if p.Tier == 0 && !askedInTier(*p) { if behind, err := startedBeside(ctx, inv, p.ID, p.Created); err != nil { return false, err } else if behind != "" { @@ -745,7 +805,10 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan, return changed, nil } } - if err := askTier(ctx, inv, p); err != nil { + if len(unasked) < len(tier) { + fmt.Printf("%s: tier %d was asked in part; asking the rest: %s\n", p.ID, p.Tier, strings.Join(unasked, ", ")) + } + if err := askTier(ctx, inv, p, unasked); err != nil { return false, err } return true, nil @@ -802,13 +865,32 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan, together := map[string]bool{} for _, m := range tier { state := p.Modules[m] - if state == nil || state.SentAt != nil || state.State == planDeleted || !rollsOut(m) { + if state == nil || state.SentAt != nil || state.State == planDeleted { continue } + // **A recorded module goes where its move recreates nothing and moves no data** (novox/hq ADR 0377): judged + // per machine against the build it was last sent, said per machine, and sent only there. + var goes map[string]bool + if !rollsOut(m) { + out, err := judgeRecordedInWalk(ctx, inv, p, m, state) + if err != nil { + return false, err + } + if len(out) == 0 { + continue + } + goes = map[string]bool{} + for _, n := range out { + goes[n] = true + } + } running, err := inv.Running(ctx, m) if err != nil { return false, err } + if goes != nil { + running = slices.DeleteFunc(running, func(n string) bool { return !goes[n] }) + } runningOf[m] = running // **A build whose source is unchanged is never a move** (novox/hq issue 280): made from the source // of the build every machine running it was sent, it is registered with that build's artifacts — @@ -828,6 +910,12 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan, if err != nil { return false, err } + // **Every machine at once** (novox/hq ADR 0360 §1, §2): a module that rolls out is sent to every machine + // running it in one step, with no first machine and no gate, whatever module it is. A module a controller + // before ADR 0360 already sent to its first machine finishes under its gate. + if !gateFirstMachine && state.FirstAt == nil { + policy.Together = true + } var reports []inventory.Reported if !policy.Together { // Read for the choice of the first machine as well as for its report. @@ -1057,15 +1145,110 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan, return changed, nil } } + // **A walk ends on its machines' reports** (novox/hq ADR 0360 §4, §5): after its last tier, every machine it sent + // has reported on that send, or is silent past ApplySilentAfter; never a health reading. + end := "" + if p.Tier == len(p.Tiers)-1 { + reports, err := inv.LastReports(ctx) + if err != nil { + return false, err + } + waiting, failed, silent := walkReports(*p, reports, time.Now().UTC(), provLookup(ctx)) + if len(waiting) > 0 { + note := "sent; waiting for " + strings.Join(waiting, ", ") + " to report on their send" + changed := p.State != inventory.PlanRolling || p.Note != note + p.State = inventory.PlanRolling + p.Note = note + return changed, nil + } + var said []string + if len(failed) > 0 { + said = append(said, "failed or refused its send (apply-failing says it): "+strings.Join(failed, ", ")) + } + if len(silent) > 0 { + said = append(said, fmt.Sprintf("said nothing on its send within %s: %s", inventory.ApplySilentAfter, + strings.Join(silent, ", "))) + } + said = append(said, recordedSaid(*p)...) + end = strings.Join(said, "; ") + // **The new controller's user list follows its walk** (ADR 0236 §34, read without the gate since ADR 0360): + // once a walk that moved the controller ends, the bus's machine is sent the list it composes, if behind. + if s := p.Modules[catalogue.ControllerSeatName]; s != nil && s.SentAt != nil && s.Gate == nil && !noMove(s) { + carryUserList(ctx, open, p) + } + } p.Tier++ p.State = inventory.PlanBuilding - p.Note = "" + p.Note = end if p.Tier < len(p.Tiers) { fmt.Printf("%s: tier %d done; asking tier %d: %s\n", p.ID, p.Tier-1, p.Tier, strings.Join(p.Tiers[p.Tier], ", ")) } return true, nil } +// walkReports is, for every machine a walk sent its rest, whether it reported on that send (novox/hq ADR 0360 §4): +// those still waiting, those that reported a failed or refused apply, and those silent past ApplySilentAfter, +// each sorted. Pure. +func walkReports(p inventory.Plan, reports []inventory.Reported, now time.Time, prov inventory.ProvLookup) (waiting, failed, silent []string) { + byNode := map[string]inventory.Reported{} + for _, r := range reports { + byNode[r.Node] = r + } + // The modules of this delivery sent to each machine: a machine is judged on those, not on every module its + // declaration hands to the provisioner (ADR 0362 §1-2, issue 840). + carried := map[string][]string{} + for m, s := range p.Modules { + if s == nil { + continue + } + for node := range s.Rest { + carried[node] = append(carried[node], m) + } + } + seen := map[string]bool{} + for _, s := range p.Modules { + if s == nil { + continue + } + for node, sent := range s.Rest { + sent = sent.Only(carried[node]...) + if seen[node] { + continue + } + r := byNode[node] + at := sent.At + if at.IsZero() && s.SentAt != nil { + at = *s.SentAt + } + // A module handed to the provisioner is applied only once its client said so too (ADR 0362 §1-5). + outcome, reported := provisionedOutcome(node, sent, r, r.At != nil && sent.ReportsOn(r), prov) + switch { + case reported: + seen[node] = true + switch { + case outcome == inventory.OutcomeFailed || outcome == inventory.OutcomeRefused: + failed = append(failed, node) + case strings.HasPrefix(outcome, inventory.OutcomeProvisioningFailed): + failed = append(failed, node+" ("+outcome+")") + } + case !at.IsZero() && now.Sub(at) > inventory.ApplySilentAfter: + seen[node] = true + silent = append(silent, node) + default: + if !slices.Contains(waiting, node) { + waiting = append(waiting, node) + } + } + } + } + // A machine one module's send was reported on and another's not yet is the same send: reported. + waiting = slices.DeleteFunc(waiting, func(n string) bool { return seen[n] }) + sort.Strings(waiting) + sort.Strings(failed) + sort.Strings(silent) + return waiting, failed, silent +} + // restOf is, for one module, every machine of its rest that the send reached and what it carried there. func restOf(to, sent []string, what map[string]inventory.SentDeclaration) map[string]inventory.SentDeclaration { out := map[string]inventory.SentDeclaration{} @@ -1809,6 +1992,12 @@ func plansCommand(ctx context.Context, args []string) error { case s != nil && s.Gate != nil: fmt.Printf(" %-22s %s\n", "", gateLine(s.Gate)) } + // Which way a recorded module went on each machine (novox/hq ADR 0377 §6). + if s != nil { + for _, n := range sortedKeysOf(s.Recorded) { + fmt.Printf(" %-22s on %s %s\n", "", n, s.Recorded[n]) + } + } // What the send to its first machine did to its containers (ADR 0245). if s != nil && s.Gate != nil { for _, c := range s.Gate.Carried { diff --git a/cmd/mesh-controller/release_plan_test.go b/cmd/mesh-controller/release_plan_test.go index d26bcf57..5425de3c 100644 --- a/cmd/mesh-controller/release_plan_test.go +++ b/cmd/mesh-controller/release_plan_test.go @@ -1,6 +1,7 @@ package main import ( + "slices" "testing" "time" @@ -49,8 +50,10 @@ func TestAMergeIsPlannedInTiersAlongTheThreeKindsOfDependency(t *testing.T) { if pos["mesh-tools"] != 0 || pos["builder"] != 1 { t.Fatalf("the runtime then the builder: %v", tiers) } - if !(pos["shop"] > pos["builder"] && pos["postgres"] > pos["builder"]) { - t.Fatalf("what the builder builds comes after the builder: %v", tiers) + // Built-by orders nothing (novox/hq ADR 0360 §3): what the builder builds is beside it, after the runtime it + // stands on. + if pos["shop"] != 1 || pos["postgres"] != 1 { + t.Fatalf("what the builder builds waits for the builder: %v", tiers) } if pos["shop-plugin"] <= pos["shop"] { t.Fatalf("a plugin after what it is declared on: %v", tiers) @@ -67,24 +70,23 @@ func TestAMergeIsPlannedInTiersAlongTheThreeKindsOfDependency(t *testing.T) { // A change to a package all three build from moves all three. The builder holds // the build seat, whose worker the controller defines, so it follows the controller (worker-of, // novox/hq issue 206), and the controller's built-by edge to it yields: the controller is built by the - // build machine that is running. The proxy is built by the new builder: the controller, the builder, - // the proxy — the live plan of every controller merge. (This read "the builder, then the controller - // and the proxy together" before issue 206, and the fixture had no worker-of edge.) + // build machine that is running. The proxy is built by whichever builder runs, so it goes beside the + // controller (novox/hq ADR 0360 §3; it read "the controller, the builder, the proxy" before). small := reachableFrom([]string{"mesh-controller", "builder", "route-proxy"}, edges) smallTiers := tiersOf(small, edges) - if got := tiered(smallTiers); got != "mesh-controller | builder | route-proxy" { - t.Fatalf("the controller, then the builder, then the proxy: %v", smallTiers) + if got := tiered(smallTiers); got != "mesh-controller,route-proxy | builder" { + t.Fatalf("the controller and the proxy, then the builder: %v", smallTiers) } // The builder alone moved: the builder, and nothing it builds. if only := reachableFrom([]string{"builder"}, edges); len(only) != 1 { t.Fatalf("a build machine change rebuilds the build machine alone: %v", only) } - // Only a runtime dependency gates on deployment, and only when the module rolls out. + // Nothing waits on deployment for a built-by edge (novox/hq ADR 0360 §3): the builder gates no tier after it. p := planOfMerge(link.SourceMoved{Owner: "novox", Repo: "mesh-tools", Commit: "abc"}, []string{"mesh-tools"}, edges) p.Tier = pos["builder"] rollsOut := func(m string) bool { return m == "builder" } - if g := gates(p, edges, rollsOut); len(g) != 1 || g[0] != "builder" { + if g := gates(p, edges, rollsOut); len(g) != 0 { t.Fatalf("the builder gates the tier after it: %v", g) } p.Tier = 0 @@ -253,10 +255,10 @@ func TestAChangeToTheBuildAgentRebuildsTheBuildAgentAlone(t *testing.T) { t.Fatalf("%v planned %v: the build agent alone", paths, p.Tiers) } } - // With what it builds moved beside it, the agent comes first and they follow: an order, not a widening. + // With what it builds moved beside it, both go in one tier (novox/hq ADR 0360 §3): built-by orders nothing. p := planOfMerge(link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Commit: "abc"}, []string{"build-agent", "redis"}, edges) - if len(p.Tiers) != 2 || p.Tiers[0][0] != "build-agent" || p.Tiers[1][0] != "redis" || len(p.Modules) != 2 { - t.Fatalf("the agent, then what moved beside it: %v", p.Tiers) + if len(p.Tiers) != 1 || !slices.Equal(p.Tiers[0], []string{"build-agent", "redis"}) || len(p.Modules) != 2 { + t.Fatalf("the agent and what moved beside it, in one tier: %v", p.Tiers) } } diff --git a/cmd/mesh-controller/release_provisioned_test.go b/cmd/mesh-controller/release_provisioned_test.go new file mode 100644 index 00000000..52241dcf --- /dev/null +++ b/cmd/mesh-controller/release_provisioned_test.go @@ -0,0 +1,172 @@ +package main + +import ( + "context" + "encoding/json" + "slices" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/inventory" +) + +// handingAnchor makes every send to anchor hand the provisioner the given modules (module -> templates digest), and +// the clients' words the given reports (key `node/module`), as the stream would give them (issue 840, ADR 0362). +func handingAnchor(t *testing.T, b *backlogMesh, handed map[string]string) map[string]inventory.ProvReport { + t.Helper() + says := map[string]inventory.ProvReport{} + wasSays := provAppliedOn + provAppliedOn = func(_ context.Context, node, module string) (inventory.ProvReport, bool, error) { + r, ok := says[node+"/"+module] + return r, ok, nil + } + inner := sendRollout + sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { + got, err := inner(ctx, open, names) + if err != nil { + return got, err + } + for _, n := range names { + if n != "anchor" { + continue + } + inv := open.inventory + s, _, err := inv.SentTo(ctx, n) + if err != nil { + return nil, err + } + builds, _, err := inv.SentBuilds(ctx, n) + if err != nil { + return nil, err + } + plans := map[string]inventory.Provisioning{} + for m, d := range handed { + plans[m] = inventory.Provisioning{Templates: inventory.TemplatesArchive{Digest: d}} + } + raw, err := json.Marshal(plans) + if err != nil { + return nil, err + } + id := nodeID(t, open, n) + if err := inv.RecordSentWith(ctx, id, s.Digest, builds, 0, 0, inventory.Send{Node: id, Provisioning: raw}); err != nil { + return nil, err + } + } + return got, nil + } + t.Cleanup(func() { sendRollout, provAppliedOn = inner, wasSays }) + return says +} + +func stepRelease(t *testing.T, b *backlogMesh, n int) inventory.Plan { + t.Helper() + for i := 0; i < n; i++ { + advancePlans(t.Context(), b.open) + } + return b.release(t) +} + +// Issue 840 (novox/hq ADR 0362 §1-5): the release's rest gate, which every delivery passes since ADR 0360, counts a +// machine's send of a provisioned module applied only once the engine and the machine's provisioner client both said +// so; a failed client is a failed apply named with its reason; none past ApplySilentAfter is silent, never done. +func TestTheRestGateWaitsForTheProvisioner(t *testing.T) { + judgeLeavesAsTheCore(t) + applied := inventory.ProvReport{Status: inventory.ProvApplied, Templates: "t1", At: time.Now()} + + t.Run("a engine applied, the client has not said: no verdict, the release stays open", func(t *testing.T) { + b := aBacklog(t) + handingAnchor(t, b, map[string]string{"app": "t1"}) + p := stepRelease(t, b, 8) + g := p.Release.Gates["anchor"] + if !p.Open() || g == nil || g.Verdict != "" || g.Silent || slices.Contains(p.Release.Done, "anchor") { + t.Fatalf("anchor is judged before its provisioner reported: %s %s %+v", p.State, p.Note, g) + } + if !slices.Contains(p.Release.Done, "laptop") { + t.Fatalf("laptop hands nothing and passes on the engine's report: %+v", p.Release) + } + }) + t.Run("b the client failed: a failed apply named with the reason", func(t *testing.T) { + b := aBacklog(t) + says := handingAnchor(t, b, map[string]string{"app": "t1"}) + says["anchor/app"] = inventory.ProvReport{Status: inventory.ProvFailed, Templates: "t1", Reason: "no value", At: time.Now()} + p := stepRelease(t, b, 8) + g := p.Release.Gates["anchor"] + if g == nil || g.Verdict != inventory.GateFailed || !slices.Contains(p.Release.Failed, "anchor") || + !strings.Contains(p.Note, "provisioning failed (app: no value)") { + t.Fatalf("the failure is not named: %s %q %+v %+v", p.State, p.Note, g, p.Release) + } + }) + t.Run("c no client report past ApplySilentAfter: silent, never done", func(t *testing.T) { + b := aBacklog(t) + handingAnchor(t, b, map[string]string{"app": "t1"}) + p := stepRelease(t, b, 8) + g := p.Release.Gates["anchor"] + if g == nil { + t.Fatalf("no gate: %+v", p.Release) + } + since := time.Now().Add(-inventory.ApplySilentAfter - time.Minute) + g.Since = &since + if err := b.open.inventory.SavePlan(t.Context(), &p); err != nil { + t.Fatal(err) + } + p = stepRelease(t, b, 1) + if !slices.Equal(p.Release.Silent, []string{"anchor"}) || slices.Contains(p.Release.Done, "anchor") { + t.Fatalf("anchor is not silent: %s %s %+v", p.State, p.Note, p.Release) + } + }) + t.Run("d both applied: the gate passes", func(t *testing.T) { + b := aBacklog(t) + says := handingAnchor(t, b, map[string]string{"app": "t1"}) + says["anchor/app"] = applied + p := stepRelease(t, b, 8) + if p.State != inventory.PlanDone || !slices.Contains(p.Release.Done, "anchor") || len(p.Release.Failed) > 0 { + t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) + } + }) + t.Run("e a send handing nothing passes on the engine's report", func(t *testing.T) { + b := aBacklog(t) + p := stepRelease(t, b, 8) + if p.State != inventory.PlanDone || !slices.Contains(p.Release.Done, "anchor") { + t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) + } + }) + t.Run("f a module the delivery does not carry does not touch its end", func(t *testing.T) { + // anchor's declaration also hands `same`, rebuilt with the very bytes and so not carried: its failed report + // is none of this delivery's. + b := aBacklog(t) + says := handingAnchor(t, b, map[string]string{"app": "t1", "same": "t9"}) + says["anchor/app"] = applied + says["anchor/same"] = inventory.ProvReport{Status: inventory.ProvFailed, Templates: "t9", Reason: "stuck", At: time.Now()} + p := stepRelease(t, b, 8) + if p.State != inventory.PlanDone || !slices.Contains(p.Release.Done, "anchor") || len(p.Release.Failed) > 0 { + t.Fatalf("another module's failure ended this delivery: %s %s %+v", p.State, p.Note, p.Release) + } + }) +} + +// Issue 840: a walk's end waits on the modules it sent to a machine, not on every module that machine's declaration +// hands to the provisioner. +func TestAWalkEndWaitsOnlyOnTheModulesItSent(t *testing.T) { + now := time.Date(2026, 10, 15, 12, 0, 0, 0, time.UTC) + at := now.Add(-10 * time.Second) + handed := map[string]string{"app": "t1", "other": "t2"} + p := inventory.Plan{Modules: map[string]*inventory.PlanModule{"app": {Rest: map[string]inventory.SentDeclaration{ + "ace": {Digest: "d-ace", At: now.Add(-time.Minute), Provisioned: handed}}}}} + reports := []inventory.Reported{{Node: "ace", Outcome: inventory.OutcomeApplied, At: &at, Declared: "d-ace"}} + says := map[string]inventory.ProvReport{"ace/app": {Status: inventory.ProvApplied, Templates: "t1", At: at}} + prov := func(node, module string) (inventory.ProvReport, bool) { r, ok := says[node+"/"+module]; return r, ok } + // other has said nothing: not this walk's to wait for. + if waiting, failed, silent := walkReports(p, reports, now, prov); len(waiting)+len(failed)+len(silent) != 0 { + t.Fatalf("missing: waiting %v, failed %v, silent %v", waiting, failed, silent) + } + says["ace/other"] = inventory.ProvReport{Status: inventory.ProvFailed, Templates: "t2", Reason: "stuck", At: at} + if waiting, failed, silent := walkReports(p, reports, now, prov); len(waiting)+len(failed)+len(silent) != 0 { + t.Fatalf("failed: waiting %v, failed %v, silent %v", waiting, failed, silent) + } + // once the walk carries it too, it does. + p.Modules["other"] = &inventory.PlanModule{Rest: p.Modules["app"].Rest} + if _, failed, _ := walkReports(p, reports, now, prov); len(failed) != 1 || !strings.Contains(failed[0], "other: stuck") { + t.Fatalf("a carried module's failure is not named: %v", failed) + } +} diff --git a/cmd/mesh-controller/release_test.go b/cmd/mesh-controller/release_test.go index 35b7d35c..a2fd2589 100644 --- a/cmd/mesh-controller/release_test.go +++ b/cmd/mesh-controller/release_test.go @@ -111,8 +111,13 @@ func aBacklog(t *testing.T) *backlogMesh { if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, carried); err != nil { return nil, err } + // A broken machine reports its apply failed (novox/hq ADR 0360 §7: the report is the verdict). + outcome := inventory.OutcomeApplied + if b.broken[node] { + outcome = inventory.OutcomeFailed + } if _, err := open.inventory.RecordDoing(ctx, nodeID(t, open, node), inventory.Doing{Node: node, - Outcome: inventory.OutcomeApplied, Declared: digest, Applied: 1, At: time.Now()}); err != nil { + Outcome: outcome, Declared: digest, Applied: 1, At: time.Now()}); err != nil { return nil, err } } @@ -164,6 +169,7 @@ func (b *backlogMesh) release(t *testing.T) inventory.Plan { // The backlog goes out one machine at a time: the first judged before the second is sent, each build's // pass kept; a rebuild with the same bytes is no move at all; and a send outside a gate is refused. func TestTheBacklogIsReleasedOneMachineAtATimeEachJudged(t *testing.T) { + withTheGate(t) // the first-machine gate's own path (novox/hq ADR 0360 keeps it for earlier walks) judgeLeavesAsTheCore(t) b := aBacklog(t) ctx := t.Context() @@ -219,6 +225,7 @@ func TestTheBacklogIsReleasedOneMachineAtATimeEachJudged(t *testing.T) { // A release that fails on its first machine puts back what it carried there, goes no further, and the // next one waits for a person, said as a condition; a person releases it with why. func TestAFailedReleaseIsPutBackAndTheNextWaitsForAPerson(t *testing.T) { + withTheGate(t) // the first-machine gate's own path (novox/hq ADR 0360 keeps it for earlier walks) b := aBacklog(t) ctx := t.Context() inv := b.open.inventory diff --git a/cmd/mesh-controller/replay318_test.go b/cmd/mesh-controller/replay318_test.go index 53fb53ca..df8a33b6 100644 --- a/cmd/mesh-controller/replay318_test.go +++ b/cmd/mesh-controller/replay318_test.go @@ -31,6 +31,7 @@ import ( // nothing is put back, the walk goes on to the workstation, and openrazer's condition says, in one sentence // for the operator, that a new login is needed on the laptop. func TestReplay318(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) open := aMesh(t) ctx := t.Context() inv := open.inventory diff --git a/cmd/mesh-controller/replay335_test.go b/cmd/mesh-controller/replay335_test.go index 2ec3e0f3..27031906 100644 --- a/cmd/mesh-controller/replay335_test.go +++ b/cmd/mesh-controller/replay335_test.go @@ -16,6 +16,7 @@ import ( // at its gate with the pass's own words, and put it back. Written with only what the controller had before // its fix, so it is laid over the commit before. func TestReplay335(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) t.Run("the first machine's later reports", testAPassedGateIsNotJudgedAgainFromTheFirstMachinesLaterReports) t.Run("a walk stopped after the pass", testAWalkStoppedAfterItsGatePassedKeepsThePass) } @@ -77,6 +78,7 @@ func testAWalkStoppedAfterItsGatePassedKeepsThePass(t *testing.T) { // the first send is older than the wait for a first machine's report. Read before the pass is taken over, the // carried module's step said the first machine never applied it, and the passed build was put back. func TestACarriedModuleTakesItsLeadsPassBeforeItsStepIsRead(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) judgeLeavesAsTheCore(t) tm := aTierMesh(t, "m01", "m02") ctx := t.Context() diff --git a/cmd/mesh-controller/replay336_test.go b/cmd/mesh-controller/replay336_test.go index 1b99cf71..20fa6dec 100644 --- a/cmd/mesh-controller/replay336_test.go +++ b/cmd/mesh-controller/replay336_test.go @@ -19,6 +19,7 @@ import ( // had before its fix — the stores, register, assign, a plan saved and advanced, the watchdogs' gathering and // seeing — so it is laid over the commit before. func TestReplay336(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) open := aMesh(t) ctx := t.Context() inv := open.inventory diff --git a/cmd/mesh-controller/replays_test.go b/cmd/mesh-controller/replays_test.go index 2798542d..f8a95346 100644 --- a/cmd/mesh-controller/replays_test.go +++ b/cmd/mesh-controller/replays_test.go @@ -139,6 +139,7 @@ func TestReplay273AConsumerBesideItsStoreStaysBoundToIt(t *testing.T) { // one, kept below. Heard as bytes, so an older controller reads them as it reads any report — this file is // written only with what the controller had before the judging, for the prover to lay over that commit. func TestReplayCrashLoopFailsItsGateOnTheFirstMachine(t *testing.T) { + withTheGate(t) // the first-machine gate's own path (novox/hq ADR 0360 keeps it for earlier walks) open := aMesh(t) ctx := t.Context() inv := open.inventory diff --git a/cmd/mesh-controller/rest_at_once_test.go b/cmd/mesh-controller/rest_at_once_test.go index 8b6f7155..39a0cb55 100644 --- a/cmd/mesh-controller/rest_at_once_test.go +++ b/cmd/mesh-controller/rest_at_once_test.go @@ -11,7 +11,7 @@ import ( "github.com/novox/mesh-controller/internal/inventory" ) -// novox/hq ADR 0353: after its first machine passes, a build goes to every other machine at once, each judged by +// novox/hq ADR 0353, kept behind gateFirstMachine for walks a controller before ADR 0360 started: after its first machine passes, a build goes to every other machine at once, each judged by // its own gate and a failing one put back by itself; and a gate's readings outlive the controller that took them — // a machine already sent what a send is for is not sent again, and its judging goes on from what was read. // @@ -62,6 +62,7 @@ func events(t *testing.T) *[]string { // Once the first machine passed, the rest are both sent before either is judged (ADR 0353 §1); before, the second // was judged to its verdict before the third was sent. func TestAReleaseSendsTheRestAtOnceAfterTheFirstPasses(t *testing.T) { + withTheGate(t) judgeLeavesAsTheCore(t) b := aBacklogOnThree(t) ctx := t.Context() @@ -89,6 +90,7 @@ func TestAReleaseSendsTheRestAtOnceAfterTheFirstPasses(t *testing.T) { // A machine of the rest whose gate fails is put back at once, and the other machine of the step goes on to its own // verdict (ADR 0353 §3); before, the walk stopped at the failure and the machine after it was never sent. func TestARestMachineThatFailsIsPutBackAloneAndTheOthersAreJudgedOn(t *testing.T) { + withTheGate(t) judgeLeavesAsTheCore(t) b := aBacklogOnThree(t) ctx := t.Context() @@ -125,6 +127,7 @@ func TestARestMachineThatFailsIsPutBackAloneAndTheOthersAreJudgedOn(t *testing.T // A walk whose first machine already runs its build under another walk's gate sends nothing and goes on from that // gate's readings (ADR 0353 §7); before, it sent the machine again and judged from its first reading. func TestAWalkTakesOverTheReadingsOfAGateJudgingTheSameBuildThere(t *testing.T) { + withTheGate(t) judgeLeavesAsTheCore(t) b := aBacklog(t) ctx := t.Context() @@ -163,6 +166,7 @@ func TestAWalkTakesOverTheReadingsOfAGateJudgingTheSameBuildThere(t *testing.T) // A walk's first send that was made and not kept — the controller replaced between the two — is not made again // by the controller that takes over: the gate counts from that send (ADR 0353 §5, §6). func TestAFirstSendMadeAndNotKeptIsNotSentAgain(t *testing.T) { + withTheGate(t) judgeLeavesAsTheCore(t) g := aGateMesh(t) ctx := t.Context() @@ -200,6 +204,7 @@ func TestAFirstSendMadeAndNotKeptIsNotSentAgain(t *testing.T) { // A release's send kept before it was made, and made, is judged by the controller that takes over without being // made again (ADR 0353 §8). func TestAReleaseSendKeptAndMadeIsNotMadeAgain(t *testing.T) { + withTheGate(t) judgeLeavesAsTheCore(t) b := aBacklog(t) ctx := t.Context() @@ -231,6 +236,7 @@ func TestAReleaseSendKeptAndMadeIsNotMadeAgain(t *testing.T) { // A build no gate has passed goes to one machine of a step, its first; a machine that would be its second waits // for the next step, and one whose every build passed goes now (ADR 0353 §2). func TestABuildThatPassedNoGateGoesToOneMachineOfAStep(t *testing.T) { + withTheGate(t) waiting := map[string][]inventory.CarriedMove{ "desk": {{Module: "app", To: "c2"}, {Module: "new", To: "n1"}}, "laptop": {{Module: "app", To: "c2"}, {Module: "new", To: "n1"}}, @@ -251,6 +257,7 @@ func TestABuildThatPassedNoGateGoesToOneMachineOfAStep(t *testing.T) { // The node-engine and the mesh's resolver go to the rest one machine at a time, even passed (ADR 0353 §9): a step // carries either to one machine only, while what else passed goes everywhere at once. func TestTheNodeEngineAndTheResolverGoToTheRestOneAtATime(t *testing.T) { + withTheGate(t) waiting := map[string][]inventory.CarriedMove{ "desk": {{Module: hostModule, To: "h2"}}, "laptop": {{Module: hostModule, To: "h2"}, {Module: "app", To: "c2"}}, @@ -277,6 +284,7 @@ func TestTheNodeEngineAndTheResolverGoToTheRestOneAtATime(t *testing.T) { // A first machine that fails still stops the walk (ADR 0236 §5, kept by ADR 0353 §3): the rest are never sent. func TestAFirstMachineThatFailsStillStopsTheWalk(t *testing.T) { + withTheGate(t) b := aBacklogOnThree(t) ctx := t.Context() gateEvery = 0 @@ -298,6 +306,7 @@ func TestAFirstMachineThatFailsStillStopsTheWalk(t *testing.T) { // The control node is sent last in a step of the rest (ADR 0353 §1): the machine holding the controller is // ordered after every other. func TestTheControlNodeIsSentLastInAStep(t *testing.T) { + withTheGate(t) judgeLeavesAsTheCore(t) b := aBacklogOnThree(t) ctx := t.Context() @@ -322,6 +331,7 @@ func TestTheControlNodeIsSentLastInAStep(t *testing.T) { // A machine of the rest that says nothing on its send is left silent after fifteen minutes: not failed, not put // back; a rest that went silent whole ends failed, said as a condition (ADR 0353 §4). func TestARestMachineSilentPastItsBoundIsLeftAndSaid(t *testing.T) { + withTheGate(t) judgeLeavesAsTheCore(t) b := aBacklog(t) ctx := t.Context() diff --git a/cmd/mesh-controller/rollout.go b/cmd/mesh-controller/rollout.go index 81f0c2a6..d1fb3b47 100644 --- a/cmd/mesh-controller/rollout.go +++ b/cmd/mesh-controller/rollout.go @@ -379,6 +379,19 @@ func rolloutMint(ctx context.Context, again bool) error { } modules++ + case broker.KindProvisioner: + // A machine's provisioner client, on an account of its own (novox/hq ADR 0349 clause 11), delivered + // to the module running it. + m, inShelf := shelf[p.Module] + if !inShelf || !runsTheProvisionerClient(m) { + skipped++ + continue + } + if err := issueProvisionerClient(ctx, inv, m, p.Node, known, busAddress); err != nil { + return err + } + modules++ + default: skipped++ } diff --git a/cmd/mesh-controller/rotate.go b/cmd/mesh-controller/rotate.go index f9168ddf..3ecdc25d 100644 --- a/cmd/mesh-controller/rotate.go +++ b/cmd/mesh-controller/rotate.go @@ -6,6 +6,7 @@ import ( "flag" "fmt" "sort" + "strings" "github.com/novox/mesh-controller/internal/inventory" ) @@ -39,6 +40,9 @@ func rotateCommand(ctx context.Context, args []string) error { // of one provision, each with its own credential; one module that leaked its credential (novox/hq // issue 268) is no reason to restart every other one on the machine. module := set.String("module", "", "only this consuming module's credential") + // Why, which the vault asks of every rotation (issue 767) and the hand-act log records; required only where a + // provider's credential is the vault's, so nothing the operator typed before changes meaning. + why := addHandActFlags(set) positionals, err := parseAround(set, args) if err != nil { return err @@ -89,17 +93,6 @@ func rotateCommand(ctx context.Context, args []string) error { // Every machine at both ends, named before anything changes. A person about to rotate a // production credential is entitled to know the blast radius before it is the past tense. - affected := map[string]bool{} - for _, h := range holders { - affected[h.Consumer] = true - affected[h.Provider] = true - } - machines := make([]string, 0, len(affected)) - for name := range affected { - machines = append(machines, name) - } - sort.Strings(machines) - fmt.Printf("rotating %s for %d holder(s):\n", provision, len(holders)) for _, h := range holders { // The module, because a machine may hold several credentials for one provision and @@ -107,6 +100,35 @@ func rotateCommand(ctx context.Context, args []string) error { fmt.Printf(" %s on %s, from %s%s\n", h.ConsumerModule, h.Consumer, h.Provider, asLocal(h.Local)) } + // **The vault's, where the provider is filled the new way** (issue 767, ADR 0349 clause 8): its credential is + // the vault's, so the vault remakes it and its holders take it from there; the controller writes no value of its + // own and sends nothing for it. Which holders go where is read before anything changes. + toVault, today, err := rotateRoutes(holders, + func(provider, provision string) (string, error) { return offeringModuleOn(ctx, provider, provision) }, + filledTheNewWay) + if err != nil { + return fmt.Errorf("nothing was rotated: %w", err) + } + if len(toVault) > 0 && !why.given() { + return errors.New("not rotated: the vault holds this credential and rotates it only with a reason, " + + "`--why \"…\"`; nothing was asked") + } + if why.given() { + why.record(ctx, "rotate", append([]string{provision}, rotateNarrowArgs(*only, *module)...)) + } + for _, g := range toVault { + words, err := rotateProvisionInVault(theVault(ctx), g, *only, *module, actingAs(), *why.why) + if err != nil { + return err + } + fmt.Println(words) + } + if len(today) == 0 { + return nil + } + holders = today + machines := machinesOf(holders) + for _, h := range holders { if err := inv.RotateSecret(ctx, h.Provision, h.Consumer, h.ConsumerModule, h.Provider, h.Local); err != nil { // Partly rotated, and said so plainly. What is gone is remade on the next push, so @@ -137,6 +159,21 @@ func rotateCommand(ctx context.Context, args []string) error { return nil } +// machinesOf is every machine at both ends of these holders, in order. +func machinesOf(holders []inventory.Holder) []string { + affected := map[string]bool{} + for _, h := range holders { + affected[h.Consumer] = true + affected[h.Provider] = true + } + machines := make([]string, 0, len(affected)) + for name := range affected { + machines = append(machines, name) + } + sort.Strings(machines) + return machines +} + // ofModule is the holders whose consuming module is this one; all of them when none is named. func ofModule(holders []inventory.Holder, module string) []inventory.Holder { if module == "" { @@ -165,3 +202,96 @@ func asLocal(local string) string { } return " (as " + local + ")" } + +// rotateGroup is the holders of one provider's credential for one provision whose provider is filled the new way: +// the vault's to rotate (issue 767). +type rotateGroup struct { + Provider, Provision, Module string + Holders []inventory.Holder +} + +// rotateRoutes says which holders the vault rotates, grouped by provider and provision, and which today's path +// does: the vault's are those whose provider's offering module on its machine is filled the new way (the switch +// `accept --provider` reads too). Anything that cannot be read is an error, never today's path by default: a value +// remade where its holders no longer look would be reported rotated and change nothing. +func rotateRoutes(holders []inventory.Holder, offering func(provider, provision string) (string, error), + ofTheVault vaultSwitch) ([]rotateGroup, []inventory.Holder, error) { + type key struct{ provider, provision string } + decided := map[key]int{} // index into groups, or -1 for today's path + var groups []rotateGroup + var today []inventory.Holder + for _, h := range holders { + k := key{h.Provider, h.Provision} + i, seen := decided[k] + if !seen { + module, err := offering(h.Provider, h.Provision) + if err != nil { + return nil, nil, fmt.Errorf("which module offers %s on %s cannot be read: %w", h.Provision, h.Provider, err) + } + ours := false + if module != "" { + if ours, err = ofTheVault(module); err != nil { + return nil, nil, err + } + } + i = -1 + if ours { + i = len(groups) + groups = append(groups, rotateGroup{Provider: h.Provider, Provision: h.Provision, Module: module}) + } + decided[k] = i + } + if i < 0 { + today = append(today, h) + continue + } + groups[i].Holders = append(groups[i].Holders, h) + } + return groups, today, nil +} + +// rotateNarrowArgs is --consumer and --module as typed, for the hand-act log. +func rotateNarrowArgs(consumer, module string) []string { + var out []string + if consumer != "" { + out = append(out, "--consumer", consumer) + } + if module != "" { + out = append(out, "--module", module) + } + return out +} + +// rotateProvisionInVault asks the vault to rotate one provider's credential for a provision (issue 767): a shared +// value is made anew for every holder, and the vault refuses it for one consumer alone; for a provider whose +// consumers keep a login each (ADR 0158 §11), consumer and module name whose own login is remade, and only theirs. +// The vault says secret-changed and each holder takes the new value from it; nothing is sealed or sent here. +func rotateProvisionInVault(ask vaultAsker, g rotateGroup, consumer, module, by, why string) (string, error) { + if strings.TrimSpace(why) == "" { + return "", errors.New("not rotated: the vault rotates a credential only with a reason, `--why \"…\"`; nothing was asked") + } + args := provisionArgs(g.Provider, g.Provision) + args["by"], args["why"] = by, strings.TrimSpace(why) + if consumer != "" { + args["consumer"] = consumer + } + if module != "" { + args["consumer_module"] = module + } + changed, err := endVaultCall(ask, "rotate", args) + if err != nil { + return "", fmt.Errorf("%s from %s not rotated, and nothing else was: %w", g.Provision, g.Provider, err) + } + if len(changed.Each) > 0 { + each := make([]string, 0, len(changed.Each)) + for _, c := range changed.Each { + each = append(each, fmt.Sprintf("%s (generation %d)", c.Name, c.Generation)) + } + return fmt.Sprintf("rotated in the vault, each consumer's own login of %s from %s, asked by %s: %s; the values "+ + "are sealed and not shown, and each holder takes its own from the vault", g.Provision, g.Provider, + changed.By, strings.Join(each, ", ")), nil + } + return fmt.Sprintf("rotated %s in the vault (generation %d), one value for every holder of %s from %s, asked by "+ + "%s; the value is sealed and not shown, and each holder takes it from the vault", changed.Name, + changed.Generation, g.Provision, g.Provider, changed.By), nil +} diff --git a/cmd/mesh-controller/rotate_vault_test.go b/cmd/mesh-controller/rotate_vault_test.go new file mode 100644 index 00000000..df0fad01 --- /dev/null +++ b/cmd/mesh-controller/rotate_vault_test.go @@ -0,0 +1,124 @@ +package main + +import ( + "encoding/json" + "errors" + "fmt" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/inventory" +) + +// `rotate ` reaches the vault for a provider filled the new way (issue 767), and the controller writes no +// value of its own for it: its holders go to the vault, named by provider and provision, and no other holder does. +func TestARotationOfAProvisionTheVaultOwnsGoesToTheVaultAndNotTheControllersStore(t *testing.T) { + holders := []inventory.Holder{ + {Provision: "postgres-database", Consumer: "g14", ConsumerModule: "gitea", Provider: "novox"}, + {Provision: "postgres-database", Consumer: "ace", ConsumerModule: "umami", Provider: "novox"}, + {Provision: "postgres-database", Consumer: "ace", ConsumerModule: "n8n", Provider: "ace"}, + } + offering := func(provider, provision string) (string, error) { + if provider == "novox" { + return "postgres", nil + } + return "postgres-old", nil + } + var asked []string + moved := func(module string) (bool, error) { + asked = append(asked, module) + return module == "postgres", nil + } + vault, today, err := rotateRoutes(holders, offering, moved) + if err != nil { + t.Fatal(err) + } + if len(vault) != 1 || vault[0].Provider != "novox" || vault[0].Provision != "postgres-database" || + vault[0].Module != "postgres" || len(vault[0].Holders) != 2 { + t.Fatalf("to the vault: %+v", vault) + } + if len(today) != 1 || today[0].Provider != "ace" { + t.Fatalf("today's path rotates a holder the vault owns, or misses one it does not: %+v", today) + } + if fmt.Sprint(asked) != "[postgres postgres-old]" { + t.Fatalf("the switch was read for %v, not the offering module", asked) + } +} + +// A switch that cannot be read, or a provider whose offering module cannot be found, routes nothing: an error +// before anything changes, never today's path by default. +func TestARotationWhoseOwnerCannotBeReadChangesNothing(t *testing.T) { + holders := []inventory.Holder{{Provision: "postgres-database", Consumer: "g14", ConsumerModule: "gitea", + Provider: "novox"}} + broken := func(string) (bool, error) { return false, errors.New("the catalogue is unreadable") } + if _, _, err := rotateRoutes(holders, func(string, string) (string, error) { return "postgres", nil }, broken); err == nil { + t.Fatal("an unreadable switch fell back to today's path") + } + fine := func(string) (bool, error) { return true, nil } + if _, _, err := rotateRoutes(holders, func(string, string) (string, error) { return "", errors.New("no store") }, + fine); err == nil { + t.Fatal("an unreadable provider fell back to today's path") + } +} + +// What the vault is told: the provider and provision, and --consumer and --module as the one consumer whose own +// login is remade; nothing named when the operator named nobody, so a shared value is rotated for every holder. +func TestTheVaultIsToldTheProvisionAndTheConsumerTheOperatorNamed(t *testing.T) { + group := rotateGroup{Provider: "novox", Provision: "postgres-database", Module: "postgres"} + for _, c := range []struct { + consumer, module string + want, absent []string + }{ + {"", "", []string{`"provider":"novox"`, `"provision":"postgres-database"`, `"why":"yearly"`, + `"by":"g14/claude-code"`}, []string{`"consumer"`, `"consumer_module"`}}, + {"g14", "", []string{`"consumer":"g14"`}, []string{`"consumer_module"`}}, + {"", "gitea", []string{`"consumer_module":"gitea"`}, []string{`"consumer":`}}, + {"g14", "gitea", []string{`"consumer":"g14"`, `"consumer_module":"gitea"`}, nil}, + } { + var said string + ask := func(verb string, args map[string]any) (json.RawMessage, error) { + raw, _ := json.Marshal(map[string]any{"verb": verb, "args": args}) + said = string(raw) + return json.Marshal(vaultChanged{Name: "provision/novox/postgres-database", By: "g14/claude-code", + Each: []vaultChanged{{Name: "pair/novox/postgres-database/g14/gitea", Generation: 3}}}) + } + words, err := rotateProvisionInVault(ask, group, c.consumer, c.module, "g14/claude-code", "yearly") + if err != nil { + t.Fatal(err) + } + if !strings.Contains(said, `"verb":"rotate"`) { + t.Fatalf("%+v: the vault was asked %s", c, said) + } + for _, w := range c.want { + if !strings.Contains(said, w) { + t.Errorf("%+v: the vault was not told %s: %s", c, w, said) + } + } + for _, a := range c.absent { + if strings.Contains(said, a) { + t.Errorf("%+v: the vault was told %s: %s", c, a, said) + } + } + if !strings.Contains(words, "pair/novox/postgres-database/g14/gitea (generation 3)") { + t.Errorf("%+v: the operator was not told whose login was remade: %s", c, words) + } + } +} + +// The vault rotates only with a why: asked of the vault without one, nothing is sent; and the vault's refusal (a +// shared value named for one consumer) is the operator's answer, not a rotation in the controller's store. +func TestARotationInTheVaultSaysWhyAndPassesOnItsRefusal(t *testing.T) { + group := rotateGroup{Provider: "anchor", Provision: "stand-in-api", Module: "stand-in"} + sent := 0 + ask := func(string, map[string]any) (json.RawMessage, error) { + sent++ + return nil, errors.New("not rotated: provision/anchor/stand-in-api is one value every holder shares") + } + if _, err := rotateProvisionInVault(ask, group, "", "", "g14/claude-code", " "); err == nil || sent != 0 { + t.Fatalf("a rotation without a why reached the vault: %v", err) + } + if _, err := rotateProvisionInVault(ask, group, "g14", "", "g14/claude-code", "suspected"); err == nil || + !strings.Contains(err.Error(), "every holder shares") { + t.Fatalf("the vault's refusal was not said: %v", err) + } +} diff --git a/cmd/mesh-controller/runtime_health_test.go b/cmd/mesh-controller/runtime_health_test.go new file mode 100644 index 00000000..8bc63491 --- /dev/null +++ b/cmd/mesh-controller/runtime_health_test.go @@ -0,0 +1,55 @@ +package main + +import ( + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/link" +) + +// novox/hq ADR 0366 §3: a module's own runtime stated unhealthy by its runtime-health raises the module's usual +// condition, module...unhealthy, through ADR 0240's path, saying the tool's reason, and the first +// statement that says it healthy clears it. +func TestAnUnhealthyOwnRuntimeRaisesTheModulesUsualCondition(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + inv, k := open.inventory, conditionsFrom + const key = "module.mesh-vault.anchor.unhealthy" + said := func(at time.Time, state, reason string) link.Health { + return link.Health{Contract: link.LivenessContract, At: at, Resources: []link.ResourceHealth{{ + Module: "mesh-vault", Resource: "mesh-vault.own-runtime", Kind: "process", Target: "mesh-vault-runtime.service", + State: state, Reason: reason, Since: at, Check: "tool"}}} + } + why := "re-sealing to the operator key failed on 3 passes in a row: the operator key is not readable" + for i := 0; i < 2; i++ { + at := h0.Add(time.Duration(i) * time.Minute) + if err := stateHealth(ctx, inv, k, "anchor", said(at, link.StateUnhealthy, why), at); err != nil { + t.Fatal(err) + } + } + c, had, err := k.Get(ctx, key) + if err != nil || !had { + t.Fatalf("two unhealthy statements of its own runtime raised no %s: %v", key, err) + } + evidence := "" + for _, e := range c.Evidence { + evidence += e.Said + "\n" + } + if !strings.Contains(c.Summary+evidence, "operator key is not readable") { + t.Errorf("the condition does not say the tool's reason: %s / %s", c.Summary, evidence) + } + at := h0.Add(3 * time.Minute) + if err := stateHealth(ctx, inv, k, "anchor", said(at, link.StateHealthy, ""), at); err != nil { + t.Fatal(err) + } + list, err := k.Open(ctx) + if err != nil { + t.Fatal(err) + } + for _, o := range list { + if o.Key == key { + t.Fatalf("a healthy statement did not clear %s", key) + } + } +} diff --git a/cmd/mesh-controller/same_program.go b/cmd/mesh-controller/same_program.go new file mode 100644 index 00000000..20b7598d --- /dev/null +++ b/cmd/mesh-controller/same_program.go @@ -0,0 +1,176 @@ +package main + +import ( + "context" + "fmt" + "slices" + "sort" + "strings" + + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +// **A module moves only when its program changes** (novox/hq issue 721). A merge check builds every module the +// delivery plan moves and compares each with the build the mesh holds of it; a walk is cut without a module whose +// check found its whole image the same as that build, while the mesh still holds it, and says so among the modules +// it left unchanged. A merge of one sentence in a package a program imports but never reaches moves no build of it. + +// sameAsRunning are the modules of a walk its checks found the same as the build the mesh holds, each with what its +// check said. **Only when every pull request the walk answers was checked at the head it merged, and every one of +// those checks built the module and found it the same as the build the mesh still holds**: a merge the walk answers +// without a pull request, a check of another head, a check that built it differently, or a held build that moved +// since, leaves the module in the walk. +func sameAsRunning(plan *inventory.Plan, builds []link.CheckedBuild, held map[string]string) map[string]string { + if plan.Delivery == nil || len(plan.Delivery.Merges) == 0 { + return nil + } + for _, m := range plan.Delivery.Merges { + if _, ok := pullOf(m); !ok { + return nil + } + } + pulls := reuseAskOf(plan).Pulls + heads := mergedHeadsOf(plan) + out := map[string]string{} + for name := range plan.Modules { + said := "" + for _, pull := range pulls { + b, ok := checkedBuildOf(builds, pull, heads[pull], name) + if !ok || !stillHeld(b.Built, name, held) { + said = "" + break + } + if said == "" { + said = b.Built.Same + } + } + if said != "" { + out[name] = said + } + } + if len(out) == 0 { + return nil + } + return out +} + +// checkedBuildOf is the newest build a check of the pull request at head made of the module; false when none did. +func checkedBuildOf(builds []link.CheckedBuild, pull link.PullRef, head, module string) (link.CheckedBuild, bool) { + if head == "" { + return link.CheckedBuild{}, false + } + for _, b := range builds { + if b.Pull == pull && b.Built.Module == module && link.SameHead(b.Head, head) { + return b, true + } + } + return link.CheckedBuild{}, false +} + +// stillHeld is whether a check found every artifact of the module the same as a build the mesh still holds. +func stillHeld(b link.CheckBuilt, module string, held map[string]string) bool { + if b.Same == "" || b.NotBuilt != "" || len(b.Made) == 0 || len(b.SameAs) != len(b.Made) { + return false + } + for _, m := range b.Made { + was := pinnedDigest(b.SameAs[m.Name]) + if was == "" || was != pinnedDigest(held[module+"/"+m.Name]) { + return false + } + } + return true +} + +// pinnedDigest is a reference's pinned digest, empty when it is not pinned. +func pinnedDigest(ref string) string { + if i := strings.LastIndex(ref, "@"); i >= 0 { + return ref[i+1:] + } + return "" +} + +// leaveSameAsRunning takes from a walk being cut every module its checks found the same as the build the mesh holds +// (sameAsRunning), and every dependent in the walk only for one of those (withDependentsOf): what was taken, with why. **Anything not read leaves the walk as it was**, said. +func leaveSameAsRunning(ctx context.Context, inv *inventory.Inventory, plan *inventory.Plan, moved []string, + edges []inventory.Edge) map[string]string { + if plan.Delivery == nil || len(plan.Modules) == 0 { + return nil + } + pulls := reuseAskOf(plan).Pulls + if len(pulls) == 0 { + return nil + } + builds, err := checkedBuildsOf(ctx, pulls) + if err != nil { + fmt.Printf(" the checks' builds could not be read, so every module moves: %v\n", err) + // empty-on-error: said above; nothing left out of the walk is the safe answer, every module moves as before + return nil + } + held, err := inv.RegisteredHeld(ctx) + if err != nil { + fmt.Printf(" what the mesh holds could not be read, so every module moves: %v\n", err) + // empty-on-error: said above; nothing left out of the walk is the safe answer, every module moves as before + return nil + } + same := withDependentsOf(sameAsRunning(plan, builds, held), plan, moved, edges) + if len(same) == 0 { + return nil + } + names := make([]string, 0, len(same)) + for n := range same { + names = append(names, n) + } + sort.Strings(names) + withhold(plan, names, nil) + for _, n := range names { + fmt.Printf(" %s left out: %s\n", n, same[n]) + } + return same +} + +// withDependentsOf adds to the modules found the same every module of the walk that is in it only as a dependent of +// one of them (novox/hq ADR 0267 rule 3): **a dependent is left out when its base comes out the same**, since the +// base it stands on stays the build the mesh holds. A module a merge moves itself, or a dependent of a base that +// still moves, stays. moved are the modules the walk's merges move; edges the graph the walk was planned along. +func withDependentsOf(same map[string]string, plan *inventory.Plan, moved []string, edges []inventory.Edge) map[string]string { + if len(same) == 0 { + return same + } + var still []string + for _, n := range moved { + if same[n] == "" { + still = append(still, n) + } + } + keep := map[string]bool{} + for _, n := range reachableFrom(still, edges) { + keep[n] = true + } + // Until nothing more is found: a dependent of a dependent left out is left out too. + for grew := true; grew; { + grew = false + for n := range plan.Modules { + if keep[n] || same[n] != "" { + continue + } + var bases []string + for _, e := range edges { + ordering := e.Kind == inventory.EdgeBuiltBy || e.Kind == inventory.EdgeWorkerOf || + e.Kind == inventory.EdgePackages || e.Kind == edgeGroupOrder + if !ordering && e.From == n && same[e.To] != "" && !slices.Contains(bases, e.To) { + bases = append(bases, e.To) + } + } + sort.Strings(bases) + if len(bases) == 0 { + // In the walk for no base found the same: whatever brought it in is not said here, so it moves. + continue + } + same[n] = "unchanged: it is in the walk only for " + strings.Join(bases, ", ") + + ", which came out the same as the build the mesh holds (ADR 0267 rule 3)" + grew = true + } + } + return same +} diff --git a/cmd/mesh-controller/same_program_test.go b/cmd/mesh-controller/same_program_test.go new file mode 100644 index 00000000..2d47d6bd --- /dev/null +++ b/cmd/mesh-controller/same_program_test.go @@ -0,0 +1,141 @@ +package main + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +// TestAWalkLeavesOutAModuleWhoseProgramDidNotChange is novox/hq issue 721: mesh-controller #335 changed one sentence +// that mesh-builder never reaches; its check found build-agent the same as the build the mesh holds, so the walk moves +// mesh-controller alone and says build-agent is unchanged. +func TestAWalkLeavesOutAModuleWhoseProgramDidNotChange(t *testing.T) { + pull := link.PullRef{Owner: "novox", Repo: "mesh-controller", Number: 335} + head := "34050dc6aaaabbbbccccdddd" + aPlan := func() *inventory.Plan { + return &inventory.Plan{Tiers: [][]string{{"mesh-controller", "build-agent"}}, + Modules: map[string]*inventory.PlanModule{"mesh-controller": {}, "build-agent": {}}, + Delivery: &inventory.PlanDelivery{Merges: []inventory.PlanMerge{{Repository: "novox/mesh-controller", + Commit: "c2ec0a60", Number: 335, Head: head}}}} + } + held := map[string]string{"build-agent/agent": "store/build-agent@sha256:aaaa", + "mesh-controller/controller": "store/mesh-controller@sha256:cccc"} + builds := []link.CheckedBuild{ + {Pull: pull, Head: head[:12], Built: link.CheckBuilt{Module: "build-agent", Key: "k1", + Made: []link.MadeArtifact{{Name: "agent", Kind: "image", Reference: "scratch/build-agent@sha256:bbbb"}}, + Same: "unchanged: its program is the same as running build agent aaaa", + SameAs: map[string]string{"agent": "10.0.0.1:5000/build-agent@sha256:aaaa"}}}, + {Pull: pull, Head: head[:12], Built: link.CheckBuilt{Module: "mesh-controller", Key: "k2", + Made: []link.MadeArtifact{{Name: "controller", Kind: "image", Reference: "scratch/mesh-controller@sha256:dddd"}}}}, + } + + p := aPlan() + same := sameAsRunning(p, builds, held) + if len(same) != 1 || same["build-agent"] == "" { + t.Fatalf("the walk leaves out %v, want build-agent alone", same) + } + withhold(p, []string{"build-agent"}, nil) + if len(p.Tiers) != 1 || len(p.Tiers[0]) != 1 || p.Tiers[0][0] != "mesh-controller" { + t.Fatalf("the walk moves %v, want mesh-controller alone", p.Tiers) + } + + // **The held build moved since the check**: what it was compared with no longer runs. + moved := map[string]string{"build-agent/agent": "store/build-agent@sha256:eeee"} + if same := sameAsRunning(aPlan(), builds, moved); len(same) != 0 { + t.Fatalf("a module compared with a build the mesh no longer holds is left out: %v", same) + } + // **A check of another head** says nothing of this merge. + other := aPlan() + other.Delivery.Merges[0].Head = "ffffffffffffffff" + if same := sameAsRunning(other, builds, held); len(same) != 0 { + t.Fatalf("a check of another head left out %v", same) + } + // **Another pull request the walk answers** that did not find it the same leaves it in. + two := aPlan() + two.Delivery.Merges = append(two.Delivery.Merges, inventory.PlanMerge{Repository: "novox/mesh-catalog", + Commit: "1234", Number: 9, Head: "abcdefabcdefabcdef"}) + if same := sameAsRunning(two, builds, held); len(same) != 0 { + t.Fatalf("a module another merge moves is left out: %v", same) + } + // **A merge with no pull request** leaves the walk as it is. + direct := aPlan() + direct.Delivery.Merges[0].Number = 0 + if same := sameAsRunning(direct, builds, held); len(same) != 0 { + t.Fatalf("a walk answering a merge without a pull request left out %v", same) + } +} + +// TestADependentOfABaseThatCameOutTheSameIsLeftOut is ADR 0267 rule 3 (novox/hq issue 607): a dependent is in a walk +// for its base; when the base comes out the same as the build the mesh holds, the dependent stands on that build +// still, and leaves the walk with it — and so does a dependent of that dependent. One a merge moves itself stays. +func TestADependentOfABaseThatCameOutTheSameIsLeftOut(t *testing.T) { + edges := []inventory.Edge{{From: "app", To: "base", Kind: inventory.EdgeDeclared}, + {From: "app2", To: "app", Kind: inventory.EdgeDeclared}, {From: "own", To: "base", Kind: inventory.EdgeDeclared}} + plan := &inventory.Plan{Modules: map[string]*inventory.PlanModule{"base": {}, "app": {}, "app2": {}, "own": {}}} + same := withDependentsOf(map[string]string{"base": "unchanged: its program is the same"}, plan, + []string{"base", "own"}, edges) + if !strings.Contains(same["app"], "only for base") || !strings.Contains(same["app2"], "only for app") { + t.Fatalf("the dependents of a base that came out the same are not left out: %v", same) + } + if _, left := same["own"]; left { + t.Fatalf("a module its merge moves itself is left out for its base: %v", same) + } + // A dependent of a base that still moves stays. + edges = append(edges, inventory.Edge{From: "app", To: "own", Kind: inventory.EdgeDeclared}) + same = withDependentsOf(map[string]string{"base": "unchanged"}, plan, []string{"base", "own"}, edges) + if _, left := same["app"]; left { + t.Fatalf("a dependent of a base that still moves is left out: %v", same) + } +} + +// Against the store, the path a real merge takes (novox/hq issues 721 and 611): a merge whose check built its one +// module the same as the build the mesh holds is cut into a walk done with nothing to walk, whose record names the +// module unchanged and why — the walk mesh-delivery reads as delivered, never stranded. +func TestAWalkWhoseEveryBuildCameOutTheSameIsDone(t *testing.T) { + open := windowed(t) + endsItsLine(t) + asksRecorded(t) + ctx := t.Context() + inv := open.inventory + if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-one", Repository: "novox/one", Ref: "main", + Module: "one", Commit: "c0", On: "builder", Asked: t0.Add(-time.Hour), + Made: []inventory.Artifact{{Name: "one", Kind: "image", Reference: "store/one@sha256:aaaa"}}, + Sources: []inventory.BuildSource{{Paths: []string{"main.go", "go.mod"}}}}); err != nil { + t.Fatal(err) + } + // What runs: the module registered with that build's image (RegisteredHeld reads the module table). + if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "one", Version: "1", Resources: []map[string]any{ + {"id": "app", "type": "container", "name": "one", "image": "store/one@sha256:aaaa"}}}, + inventory.Source{Repository: "novox/one", Ref: "main", BuiltFrom: "c0", Asked: time.Now()}); err != nil { + t.Fatal(err) + } + pull := link.PullRef{Owner: "novox", Repo: "one", Number: 7} + was := checkedBuildsOf + checkedBuildsOf = func(context.Context, []link.PullRef) ([]link.CheckedBuild, error) { + return []link.CheckedBuild{{Pull: pull, Head: "headaaaaaaaa", Built: link.CheckBuilt{Module: "one", Key: "k", + Made: []link.MadeArtifact{{Name: "one", Kind: "image", Reference: "scratch/one@sha256:bbbb"}}, + Same: "unchanged: its program is the same as running build one aaaa", + SameAs: map[string]string{"one": "10.0.0.1:5000/one@sha256:aaaa"}}}}, nil + } + t.Cleanup(func() { checkedBuildsOf = was }) + code := repoMerge("one", "one1aaaaaaaa", t0) + code.Number, code.HeadSHA = 7, "headaaaaaaaa1111" + hear(t, open, code, t0.Add(time.Second)) + cutAt(t, open, t0.Add(2*time.Minute)) + ws, _ := walks(t, open) + if len(ws) != 1 { + t.Fatalf("walks: %+v", ws) + } + w := ws[0] + if w.State != inventory.PlanDone || len(w.Tiers) != 0 || len(w.Modules) != 0 { + t.Fatalf("the walk reads %s, tiers %v, modules %v; want done with nothing to walk", w.State, w.Tiers, w.Modules) + } + if w.Delivery == nil || !strings.Contains(w.Delivery.Unchanged["one"], "its program is the same as running build") { + t.Fatalf("the walk does not say one matched what runs: %+v", w.Delivery) + } +} diff --git a/cmd/mesh-controller/sayable_test.go b/cmd/mesh-controller/sayable_test.go index 8f98b992..8e92dda6 100644 --- a/cmd/mesh-controller/sayable_test.go +++ b/cmd/mesh-controller/sayable_test.go @@ -43,6 +43,9 @@ func TestMain(m *testing.M) { } if os.Getenv(echoEnvironment) != "" { fmt.Printf("verb=%q caller=%q terminal=%v\n", os.Getenv("MESH_VERB"), os.Getenv("MESH_CALLER"), startedAtTheTerminal()) + if os.Getenv(echoFails) != "" { + os.Exit(3) + } os.Exit(0) } conditions.Unsayable = func(o conditions.Observation, field string, r outward.Refusal) { diff --git a/cmd/mesh-controller/seatverbs.go b/cmd/mesh-controller/seatverbs.go index c0c222e3..101db41c 100644 --- a/cmd/mesh-controller/seatverbs.go +++ b/cmd/mesh-controller/seatverbs.go @@ -730,6 +730,38 @@ func (a *verbArguments) commandLine() ([]string, error) { return nil, fmt.Errorf("%w: letting go of artifacts is a hand act, which says why. Nothing was done", err) } return append(argv, "--confirm", "--why", why), nil + // The move of placement out of the settings layers (issue 736), a dry run unless confirmed. + case "move-placement": + argv := []string{"move-placement", "--json"} + why := str("why") + if !on("confirm") { + if why != "" { + return nil, errors.New("move-placement takes why only with confirm: without confirm nothing is " + + "moved, and a reason for nothing would be recorded nowhere. Nothing was done") + } + return argv, nil + } + if err := need("why"); err != nil { + return nil, fmt.Errorf("%w: moving the settings layers' placement is an act by hand, which says why. "+ + "Nothing was done", err) + } + return append(argv, "--confirm", "--why", why), nil + // The move of config into mesh-config (issue 664), a dry run unless confirmed. + case "move-config": + argv := []string{"move-config", "--json"} + why := str("why") + if !on("confirm") { + if why != "" { + return nil, errors.New("move-config takes why only with confirm: without confirm nothing is set, " + + "and a reason for nothing would be recorded nowhere. Nothing was done") + } + return argv, nil + } + if err := need("why"); err != nil { + return nil, fmt.Errorf("%w: moving the settings layers into mesh-config is an act by hand, which says why. "+ + "Nothing was done", err) + } + return append(argv, "--confirm", "--why", why), nil case "images": if err := need("node"); err != nil { return nil, err @@ -837,6 +869,13 @@ func (a *verbArguments) commandLine() ([]string, error) { return nil, err } return []string{"secret", "ask", str("node"), str("module"), str("secret"), "--at", str("at")}, nil + case "secret-move": + // The move of the controller's secrets into the vault (design 57's V3, issue 676): no value in the arguments + // or the answer, a machine's name at most. + if n := str("node"); n != "" { + return []string{"secret", "move", n}, nil + } + return []string{"secret", "move"}, nil case "issued": // The credential a provider's software issued, taken from the provider (novox/hq ADR 0305): no value in // the arguments, the provision alone. @@ -866,6 +905,13 @@ func (a *verbArguments) commandLine() ([]string, error) { if m := str("module"); m != "" { argv = append(argv, "--module", m) } + // Why, which the vault asks of a rotation of a credential it holds (issue 767); a cause only beside it. + if w := str("why"); w != "" { + argv = append(argv, "--why", w) + if c := str("cause"); c != "" { + argv = append(argv, "--cause", c) + } + } return argv, nil } _, node := a.given["node"] @@ -1036,6 +1082,10 @@ var jsonVerbs = map[string]bool{"status": true, "seats": true, "plan": true, "co "hand-acts": true, "durations": true, "conditions": true, "doctor": true, "retire": true, "cleanup": true, "data": true, // What the records say the registries may keep (novox/hq ADR 0251). "artifacts": true, "collect": true, "images": true, "mirrors": true, + // The move of placement says each layer it moved (issue 736). + "move-placement": true, + // The move of config says each value it set and where from (issue 664). + "move-config": true, // The delivery's owner's verbs answer JSON where they read (plan, order, check, walks) — novox/hq ADR 0239. "delivery": true, // The build queue, each ask with its position among the waiting (novox/hq issue 455). @@ -1081,6 +1131,10 @@ func repairingCommand(argv []string) string { return "collect" case argv[0] == "mirrors" && slices.Contains(argv, "--confirm"): return "mirrors" + case argv[0] == "move-placement" && slices.Contains(argv, "--confirm"): + return "move-placement" + case argv[0] == "move-config" && slices.Contains(argv, "--confirm"): + return "move-config" } return "" } @@ -1732,6 +1786,16 @@ func takenFromTheProvider(argv []string) bool { !strings.HasPrefix(argv[2], "-") } +// movedByTheirMachines is exactly the line the `secret-move` verb composes: `secret move`, or `secret move ` +// (design 57's V3, issue 676). It carries no value and answers none: each machine opens what was sealed to it and +// hands it to the vault, which never replaces a value it holds. +func movedByTheirMachines(argv []string) bool { + if len(argv) < 2 || argv[0] != "secret" || argv[1] != "move" { + return false + } + return len(argv) == 2 || (len(argv) == 3 && argv[2] != "" && !strings.HasPrefix(argv[2], "-")) +} + // terminalOnly refuses, through any verb, a command that is the operator's at the controller's terminal // alone (novox/hq ADR 0266). **Every `node` subcommand that is not a read**: `node account` and // `node agent-account` above all. Whoever may call a verb includes agents, and an agent that named itself @@ -1748,7 +1812,8 @@ func terminalOnly(argv []string) error { // Of a secret's commands only rotation, which seals the new value to the machine that uses it, and the // `give` verb's own line: an own secret typed by the operator into the desk's hidden prompt, sealed to this // call and then to the module's machine, so no value travels in the verb or its answer (hq ADR 0259 §10). - if argv[0] == "secret" && (len(argv) < 2 || argv[1] != "rotate") && !givenAtTheDesk(argv) && !takenFromTheProvider(argv) { + if argv[0] == "secret" && (len(argv) < 2 || argv[1] != "rotate") && !givenAtTheDesk(argv) && !takenFromTheProvider(argv) && + !movedByTheirMachines(argv) { return terminalRefusal("secret %s is run at the controller's terminal only, never through a verb: accepting, "+ "recovering or exporting a secret hands it to whoever asks, and that includes agents (novox/hq ADR "+ "0266). Nothing was done", strings.Join(argv[1:], " ")) diff --git a/cmd/mesh-controller/seatverbs_schema_test.go b/cmd/mesh-controller/seatverbs_schema_test.go index 61f3bad9..a5c489dd 100644 --- a/cmd/mesh-controller/seatverbs_schema_test.go +++ b/cmd/mesh-controller/seatverbs_schema_test.go @@ -285,6 +285,8 @@ var accountedFlags = map[string]map[string]string{ "collect": {"json": "set by the verb: the answer is data"}, "images": {"json": "set by the verb: the answer is data"}, "mirrors": {"json": "set by the verb: the answer is data"}, + "move-placement": {"json": "set by the verb: the answer is data"}, + "move-config": {"json": "set by the verb: the answer is data"}, "conditions history": {"json": "set by the verb: the answer is data"}, "conditions show": {"json": "set by the verb: the answer is data"}, "healers": {"json": "set by the verb: the answer is data"}, diff --git a/cmd/mesh-controller/secret.go b/cmd/mesh-controller/secret.go index 373dbd7b..4500f756 100644 --- a/cmd/mesh-controller/secret.go +++ b/cmd/mesh-controller/secret.go @@ -12,8 +12,11 @@ import ( "strings" "time" + "github.com/nats-io/nats.go" + "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" "github.com/novox/mesh-controller/internal/secrets" ) @@ -50,6 +53,9 @@ func secretCommand(ctx context.Context, args []string) error { return secretRecover(ctx, args[1:]) case "export": return secretExport(ctx, args[1:]) + case "move": + // The controller's secrets moved into the vault, each by the machine it was sealed to (design 57's V3). + return secretMove(ctx, args[1:]) default: return errors.New(secretUsage) } @@ -108,7 +114,12 @@ func secretCommand(ctx context.Context, args []string) error { if value == "" { return errors.New("there is nothing to seal") } - return acceptForProvision(ctx, node, module, name, value) + // The provider's module decides the path: filled the new way, the vault keeps the provision's credential. + return bySwitch(module, filledTheNewWay, + func() error { return acceptForProvision(ctx, node, module, name, value) }, + func() error { + return said(acceptIntoVault(theVault(ctx), provisionArgs(node, name), value, actingAs())) + }) } if *desk != "" { if *from != "" || *provider != "" { @@ -125,23 +136,112 @@ func secretCommand(ctx context.Context, args []string) error { if value == "" { return errors.New("there is nothing to seal") } + // The module whose secrets hold the value decides the path: with --provider, the value is the provider's one + // credential for the provision, so the provider's module decides, as for an outside offer above. + decides := module + if *provider != "" { + if decides, err = offeringModuleOn(ctx, *provider, name); err != nil { + return err + } + } + return bySwitch(decides, filledTheNewWay, + func() error { return acceptKept(ctx, node, module, name, *provider, *local, value) }, + func() error { + // Sealed here to the vault's key for this one call; only the sealed box leaves this process. + target := ownSecretArgs(node, module, name) + if *provider != "" { + if *local != "" { + return errors.New("--local names a credential inside the module; the vault keeps one credential " + + "per provider and provision, so it takes no --local: nothing was sealed") + } + target = provisionArgs(*provider, name) + } + if *provider != "" { + return said(acceptIntoVault(theVault(ctx), target, value, actingAs())) + } + return giveIntoVault(ctx, node, module, name, target, value) + }) +} +// announceTheGiven is the announcement of a secret given at the terminal (announceSecretGiven, the step acceptKept +// runs); a variable so a test can make it fail. +var announceTheGiven = announceSecretGiven + +// giveIntoVault is `secret accept` of a module's own secret for a module filled the new way, under the same +// N1-give rule as acceptKept, through the same steps (keepGiven, announceSecretGiven): a module that runs as an +// account of its own is announced on the operator's channels before the vault is asked, and when the announcement +// fails nothing is sealed and the vault is never called; any other is sealed first and announced after. +func giveIntoVault(ctx context.Context, node, module, name string, target map[string]any, value string) error { + open, err := openStores(ctx) + if err != nil { + return err + } + defer open.Close() + trusted, err := open.inventory.RunsAsItsOwnAccount(ctx, module) + if err != nil { + return err + } + var words string + askedBy := link.Caller() + _, unannounced, err := keepGiven(trusted, + func() error { + return announceTheGiven(ctx, node, module, name, "at the controller's terminal", askedBy) + }, + func() (bool, error) { + var err error + words, err = acceptIntoVault(theVault(ctx), target, value, actingAs()) + return false, err + }) + if err != nil { + if trusted && unannounced != nil { + return fmt.Errorf("%s runs as an account of its own, and the change of its %s could not be announced on "+ + "your channels first, so nothing was sealed and the vault was not asked: %w", module, name, err) + } + return err + } + fmt.Println(words) + if unannounced != nil { + fmt.Printf(" this change could NOT be announced on the operator's channels: %v\n", unannounced) + } + // The giving is recorded, never the value (novox/hq ADR 0380 §6), as acceptKept records its own. + act := terminalGiveHandAct(node, module, name, askedBy) + if err := onTheBus(func(conn *nats.Conn) error { + _, err := link.RecordHandAct(ctx, conn, act) + return err + }); err != nil { + fmt.Printf(" this act could NOT be recorded in the hand-act log, and is done anyway: %v\n", err) + } + return nil +} + +// said prints what was done, or answers why not. +func said(words string, err error) error { + if err != nil { + return err + } + fmt.Println(words) + return nil +} + +// acceptKept is today's `secret accept`: the value sealed into the controller's store, for a module whose secrets +// are filled the old way. +func acceptKept(ctx context.Context, node, module, name, provider, local, value string) error { open, err := openStores(ctx) if err != nil { return err } defer open.Close() - if *provider != "" { + if provider != "" { // Into the pair, not into the module's own secrets: what the provider is asked to create // and what the consumer reads are the same value, and neither end can be told a different // one later without the other (novox/hq 04-ISSUES/070). - if err := open.inventory.AcceptSecretForPair(ctx, name, node, module, *provider, *local, value); err != nil { + if err := open.inventory.AcceptSecretForPair(ctx, name, node, module, provider, local, value); err != nil { return err } - fmt.Printf("%s on %s now holds %q from %s%s, sealed to both machines.\n", module, node, name, *provider, asLocal(*local)) + fmt.Printf("%s on %s now holds %q from %s%s, sealed to both machines.\n", module, node, name, provider, asLocal(local)) fmt.Printf(" the mesh cannot read it back, will not replace it with one of its own, and will not rotate it\n") - fmt.Printf(" run `push %s` and `push %s` to send it\n", *provider, node) + fmt.Printf(" run `push %s` and `push %s` to send it\n", provider, node) return nil } // A trusted party's secret is announced before it is kept (the confirmation review of 2026-10-09, N1-give): @@ -151,8 +251,11 @@ func secretCommand(ctx context.Context, args []string) error { if err != nil { return err } + askedBy := link.Caller() untilStart, unannounced, err := keepGiven(trusted, - func() error { return announceSecretGiven(ctx, node, module, name, "at the controller's terminal") }, + func() error { + return announceSecretGiven(ctx, node, module, name, "at the controller's terminal", askedBy) + }, func() (bool, error) { return open.inventory.AcceptGivenSecret(ctx, node, module, name, value) }) if err != nil { if trusted && unannounced != nil { @@ -164,6 +267,15 @@ func secretCommand(ctx context.Context, args []string) error { if unannounced != nil { fmt.Printf(" this change could NOT be announced on the operator's channels: %v\n", unannounced) } + // The giving is recorded, never the value (novox/hq ADR 0380 §6): the desk already logs its own act; the + // terminal's own secret accept did not until now. + act := terminalGiveHandAct(node, module, name, askedBy) + if err := onTheBus(func(conn *nats.Conn) error { + _, err := link.RecordHandAct(ctx, conn, act) + return err + }); err != nil { + fmt.Printf(" this act could NOT be recorded in the hand-act log, and is done anyway: %v\n", err) + } // Not printed back, and there is nowhere it could be printed from: it is sealed to that // machine and the mesh cannot read it again. fmt.Printf("%s on %s now holds %q, sealed to that machine.\n", module, node, name) @@ -206,7 +318,8 @@ const secretUsage = "secret rotate [--why [--cause "secret accept [--from | --at-desk ] [--provider [--local ]]\n" + "secret accept [--from ] (a credential issued outside the mesh, for every consumer)\n" + "secret recover --key [--out ] [--from-export ] [--provider ]\n" + - "secret export [--out ]" + "secret export [--out ]\n" + + "secret move [] (every secret the controller holds, moved into the vault by the machine it was sealed to)" // secretRecover is break-glass: a secret opened with the operator's key, written to a file. // @@ -485,6 +598,26 @@ func secretRotate(ctx context.Context, args []string) error { return errors.New(secretUsage) } node, module, name := rest[0], rest[1], rest[2] + return bySwitch(module, filledTheNewWay, + func() error { return rotateKept(ctx, node, module, name, why) }, + func() error { + reason := "" + if why.why != nil { + reason = strings.TrimSpace(*why.why) + } + words, err := rotateInVault(theVault(ctx), ownSecretArgs(node, module, name), actingAs(), reason) + if err != nil { + return err + } + why.record(ctx, "secret rotate", []string{node, module, name}) + fmt.Println(words) + return nil + }) +} + +// rotateKept is today's `secret rotate`: made anew in the controller's store and sent, for a module whose +// secrets are filled the old way. +func rotateKept(ctx context.Context, node, module, name string, why handActFlags) error { open, err := openStores(ctx) if err != nil { return err @@ -541,6 +674,16 @@ func whoAsked() string { return "the mesh" } +// terminalGiveHandAct is the hand-act log's entry for a secret accepted at the controller's terminal +// (novox/hq ADR 0380 §6): who asked, where and that it happened — never the value, which the act never +// carries a field for. +func terminalGiveHandAct(node, module, name, askedBy string) link.HandAct { + return link.HandAct{Verb: "secret accept", Args: []string{node, module, name}, + Why: fmt.Sprintf("the operator gave %s for %s on %s at the controller's terminal, asked by %s", name, module, + node, askedByName(askedBy)), + Cause: "given-at-the-terminal"} +} + // keepGiven keeps a value given at the controller's terminal, and announces it on the operator's channels // (the confirmation review of 2026-10-09, N1-give). **A trusted party's — a module running as an account of its // own: the router, a verified channel — is announced before it is kept, and not kept when the announcement diff --git a/cmd/mesh-controller/secret_move.go b/cmd/mesh-controller/secret_move.go new file mode 100644 index 00000000..9c13e817 --- /dev/null +++ b/cmd/mesh-controller/secret_move.go @@ -0,0 +1,336 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "strings" + "sync" + "time" + + "github.com/nats-io/nats.go" + + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +// `secret move` — the controller's secrets move into the vault (novox/hq ADR 0349 clause 7; design 57's V3, issue 676; +// the operator's decision of 2026-10-14 on that issue). Run once, after the feature's delivery. +// +// **The controller cannot open what it holds, so each machine moves its own.** For every machine — or the ones named — +// the controller reads every row sealed to it, as the store holds it, and asks that machine's node-engine to move each +// one, signed (link.AskMoveSecret): the engine opens it with its sealing key and hands it to the vault's move-in, sealed +// to the call's key, keeping its origin and generation. The vault takes it from that engine alone, never over a value +// it holds, and reads back what it kept to the engine, which compares; the engine answers only then. What moved is +// recorded here, never a value. Nothing is rotated, and nothing points at the vault because of this: a module's +// secrets are filled from the vault only once it moves (V4, issue 647). +// +// **A machine asleep is moved when it is next heard from**: once the move has started, a machine that reports is moved +// in the background (moveWhenHeard), and what is left of it is said on the next `secret move`. +// +// The controller's record of asks at a desk, which holds no value, is copied as it is, once (`move-asks`). + +const secretMoveUsage = "secret move [] — move every secret the controller holds into the vault, as it is: each " + + "machine's node-engine opens what was sealed to it and hands it to the vault, which reads it back; no value is " + + "rotated, and a machine not answering is moved when it is next heard from" + +// moveAsk is the ask of one row of a machine's engine. +type moveAsk func(row inventory.SecretToMove, ask link.MoveSecretAsk) (link.MoveSecretAnswer, error) + +// nodeMove is what the move did on one machine. +type nodeMove struct { + Node string + Moved int + Already int + // Refused are the rows not moved, each with why; Changed the rows changed in the store after they moved. + Refused []string + Changed []string + // Unreached is why the machine's engine could not be asked: it is moved when next heard from. + Unreached error + // Left is how many rows were not asked because the machine was not reached. + Left int + // Failed is why its rows could not be read. + Failed error +} + +// done says nothing is left to move on the machine. +func (m nodeMove) done() bool { + return m.Failed == nil && m.Unreached == nil && len(m.Refused) == 0 +} + +func (m nodeMove) String() string { + var b strings.Builder + if m.Failed != nil { + fmt.Fprintf(&b, "%s: its rows could not be read, nothing was moved: %v\n", m.Node, m.Failed) + return b.String() + } + fmt.Fprintf(&b, "%s: %d moved into the vault and read back the same, %d there already", m.Node, m.Moved, m.Already) + if n := len(m.Refused); n > 0 { + fmt.Fprintf(&b, ", %d not moved", n) + } + b.WriteString("\n") + for _, r := range m.Refused { + fmt.Fprintf(&b, " not moved: %s\n", r) + } + for _, c := range m.Changed { + fmt.Fprintf(&b, " %s was changed in the controller's store after it moved; the vault keeps what it took, "+ + "and the move never replaces it\n", c) + } + if m.Unreached != nil { + fmt.Fprintf(&b, " %d left: %s was not reached (%v); it is moved when it is next heard from\n", m.Left, m.Node, + m.Unreached) + } + return b.String() +} + +// moveRowAsk is a row as its machine's engine is asked to move it. +func moveRowAsk(r inventory.SecretToMove) link.MoveSecretAsk { + a := link.MoveSecretAsk{Origin: r.Origin, Generation: r.Generation, MadeAt: r.MadeAt.UTC(), Sealed: r.Sealed, + SealedTo: r.SealedTo} + if r.Own() { + a.Module, a.Name = r.Module, r.Name + } else { + a.Provider, a.Provision, a.Consumer, a.ConsumerModule, a.Local = r.Provider, r.Provision, r.Consumer, + r.ConsumerModule, r.Local + } + return a +} + +// moveNode moves every row sealed to one machine that has not moved, one ask each, and records what the vault kept. +// The first ask the engine does not answer ends the machine's turn: the rest wait for it to be heard from again. +func moveNode(node string, rows func(string) ([]inventory.SecretToMove, error), ask moveAsk, + record func(inventory.SecretToMove, int) error) nodeMove { + m := nodeMove{Node: node} + all, err := rows(node) + if err != nil { + m.Failed = err + return m + } + for n, r := range all { + switch { + case r.Moved != nil && r.Moved.Digest == r.Digest: + m.Already++ + continue + case r.Moved != nil: + m.Changed = append(m.Changed, r.Secret) + continue + case !r.Current: + m.Refused = append(m.Refused, fmt.Sprintf("%s is sealed to an earlier sealing key of %s, which no longer "+ + "opens it: nothing can move it; it moves once it is made or accepted again", r.Secret, node)) + continue + } + answer, err := ask(r, moveRowAsk(r)) + if err != nil { + m.Unreached, m.Left = err, len(all)-n + return m + } + if answer.Refused != "" { + m.Refused = append(m.Refused, fmt.Sprintf("%s: %s", r.Secret, answer.Refused)) + continue + } + if answer.Secret != r.Secret { + m.Refused = append(m.Refused, fmt.Sprintf("%s: the vault kept it as %s, not under the name the move "+ + "gives it", r.Secret, answer.Secret)) + continue + } + if err := record(r, answer.Generation); err != nil { + m.Refused = append(m.Refused, fmt.Sprintf("%s is in the vault, but that it moved could not be recorded: %v", + r.Secret, err)) + continue + } + if answer.Unchanged { + m.Already++ + } else { + m.Moved++ + } + } + return m +} + +// movedAsksAtOnce is how many rows of the record of asks one call to the vault carries (mesh-vault MovedAsksAtOnce). +const movedAsksAtOnce = 500 + +// moveAsks copies the controller's record of asks into the vault, in parts, and answers how many it copied. +func moveAsks(asks []inventory.SecretAsk, vault vaultAsker) (int, error) { + copied := 0 + for start := 0; start < len(asks); start += movedAsksAtOnce { + end := min(start+movedAsksAtOnce, len(asks)) + var rows []map[string]any + for _, a := range asks[start:end] { + row := map[string]any{"id": a.ID, "node": a.Node, "module": a.Module, "name": a.Name, "desk": a.Desk, + "asked_by": a.AskedBy, "opened_at": a.OpenedAt.UTC().Format(time.RFC3339Nano)} + if a.EndedAt != nil { + row["ended_at"], row["outcome"] = a.EndedAt.UTC().Format(time.RFC3339Nano), a.Outcome + } + rows = append(rows, row) + } + raw, err := vault("move-asks", map[string]any{"asks": rows}) + if err != nil { + return copied, fmt.Errorf("the vault did not take the record of asks after %d rows: %w", copied, err) + } + var said struct { + Copied int `json:"copied"` + Already int `json:"already"` + } + _ = json.Unmarshal(raw, &said) + copied += said.Copied + said.Already + } + return copied, nil +} + +// moveOnTheBus is the move's ask over the bus, signed with the mesh's key. +func moveOnTheBus(ctx context.Context, open *stores, conn *nats.Conn, by string) (moveAsk, error) { + ident, err := open.Identity(ctx) + if err != nil { + return nil, fmt.Errorf("the mesh's signing key cannot be read, so nothing was asked: %w", err) + } + return func(r inventory.SecretToMove, row link.MoveSecretAsk) (link.MoveSecretAnswer, error) { + return link.AskMoveSecret(ctx, conn, ident, r.Node, by, row, link.MoveSecretWithin) + }, nil +} + +// secretMove is `secret move []`: the record of asks copied once, then every machine's rows — or one's — and +// what moved said per machine. Any row not moved fails the command, said by name. +func secretMove(ctx context.Context, args []string) error { + if len(args) > 1 || (len(args) == 1 && (args[0] == "" || strings.HasPrefix(args[0], "-"))) { + return errors.New(secretMoveUsage) + } + open, err := openStores(ctx) + if err != nil { + return err + } + defer open.Close() + inv := open.inventory + by := actingAs() + if by == "" { + by = handOverBy() + } + move, err := inv.StartSecretMove(ctx, by) + if err != nil { + return err + } + var nodes []string + if len(args) == 1 { + if _, err := inv.NodeByName(ctx, args[0]); err != nil { + return fmt.Errorf("nothing was moved: %w", err) + } + nodes = args + } else { + all, err := inv.Nodes(ctx) + if err != nil { + return err + } + for _, n := range all { + nodes = append(nodes, n.Name) + } + } + fmt.Printf("the move of the secrets into the vault, started %s by %s\n", move.StartedAt.Local().Format("2006-01-02 15:04"), + move.StartedBy) + if move.AsksMovedAt == nil { + asks, err := inv.SecretAsks(ctx, time.Time{}) + if err != nil { + return err + } + copied, err := moveAsks(asks, theVault(ctx)) + if err != nil { + return fmt.Errorf("the record of asks was not copied, and no secret was moved: %w", err) + } + if err := inv.RecordAsksMoved(ctx); err != nil { + return err + } + fmt.Printf("the record of asks at a desk: %d rows copied into the vault as they are\n", copied) + } + failed := 0 + err = onTheBus(func(conn *nats.Conn) error { + ask, err := moveOnTheBus(ctx, open, conn, "the move of the secrets, by "+by) + if err != nil { + return err + } + for _, node := range nodes { + m := moveNode(node, func(n string) ([]inventory.SecretToMove, error) { return inv.SecretsToMoveFrom(ctx, n) }, + ask, func(r inventory.SecretToMove, g int) error { return inv.RecordSecretMoved(ctx, r, g) }) + fmt.Print(m.String()) + if m.Failed != nil || len(m.Refused) > 0 { + failed++ + } + } + return nil + }) + if err != nil { + return err + } + if failed > 0 { + return fmt.Errorf("on %d machine(s) a secret was not moved, said above by name", failed) + } + return nil +} + +// moveHeard is the machines being moved in the background now, and when each was last moved, so a machine that reports +// often is not asked again before moveHeardEvery has passed. +var moveHeard = struct { + sync.Mutex + running map[string]bool + last map[string]time.Time +}{running: map[string]bool{}, last: map[string]time.Time{}} + +// moveHeardEvery is how often a machine heard from is moved at most. +const moveHeardEvery = 10 * time.Minute + +// moveWhenHeard moves what is left of a machine that was just heard from, once the move has started: a machine asleep +// when it started is moved when it next reports. What it did is said on the controller's output, and recorded for the +// next `secret move`. +func moveWhenHeard(ctx context.Context, open *stores, conn *nats.Conn, node string) { + if open == nil || conn == nil || node == "" { + return + } + moveHeard.Lock() + if moveHeard.running[node] || time.Since(moveHeard.last[node]) < moveHeardEvery { + moveHeard.Unlock() + return + } + moveHeard.running[node], moveHeard.last[node] = true, time.Now() + moveHeard.Unlock() + defer func() { + moveHeard.Lock() + delete(moveHeard.running, node) + moveHeard.Unlock() + }() + inv := open.inventory + move, started, err := inv.SecretMoveStarted(ctx) + if err != nil || !started { + return + } + rows, err := inv.SecretsToMoveFrom(ctx, node) + if err != nil { + return + } + left := false + for _, r := range rows { + left = left || (r.Moved == nil && r.Current) + } + if !left { + return + } + ask, err := moveOnTheBus(ctx, open, conn, "the move of the secrets, by "+move.StartedBy+", on hearing from "+node) + if err != nil { + fmt.Fprintf(os.Stderr, "mesh-controller: %s was heard from and its secrets not moved: %v\n", node, err) + return + } + m := moveNode(node, func(string) ([]inventory.SecretToMove, error) { return rows, nil }, ask, + func(r inventory.SecretToMove, g int) error { return inv.RecordSecretMoved(ctx, r, g) }) + if m.done() { + fmt.Printf("mesh-controller: %s was heard from; the move of its secrets is done: %s", node, m.String()) + return + } + fmt.Fprintf(os.Stderr, "mesh-controller: %s was heard from; the move of its secrets is not done: %s", node, m.String()) +} + +// moveHeardOn is the connection a machine heard from is moved over: the serving controller's bus. +func moveHeardOn() *nats.Conn { + if js := servingBus.Load(); js != nil { + return js.Conn() + } + return nil +} diff --git a/cmd/mesh-controller/secret_move_test.go b/cmd/mesh-controller/secret_move_test.go new file mode 100644 index 00000000..cbf179d9 --- /dev/null +++ b/cmd/mesh-controller/secret_move_test.go @@ -0,0 +1,158 @@ +package main + +import ( + "encoding/json" + "errors" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +func aRowToMove(secret string, current bool) inventory.SecretToMove { + return inventory.SecretToMove{Secret: secret, Node: "anchor", Module: "postgres", Name: strings.TrimPrefix(secret, "own/anchor/postgres/"), + Origin: "made", Generation: "g-1", MadeAt: time.Unix(1, 0), Sealed: "c2VhbGVk", SealedTo: "k", Current: current, + Digest: "sha256:" + secret} +} + +// Each row sealed to a machine is asked of its engine once, as the store holds it; what the vault kept and read back +// the same is recorded; a row the vault held already counts as there; a row moved before is not asked again, and one +// changed in the store since it moved is said; a row sealed to a key the machine no longer has is said, not asked. +func TestAMachinesRowsAreMovedOnceEachAndWhatMovedIsSaid(t *testing.T) { + rows := []inventory.SecretToMove{ + aRowToMove("own/anchor/postgres/superuser", true), + aRowToMove("own/anchor/postgres/replication", true), + aRowToMove("own/anchor/postgres/old-key", false), + aRowToMove("own/anchor/postgres/refused", true), + aRowToMove("own/anchor/postgres/moved-before", true), + aRowToMove("own/anchor/postgres/changed-since", true), + } + rows[4].Moved = &inventory.SecretMoved{Node: "anchor", Generation: 1, Digest: rows[4].Digest} + rows[5].Moved = &inventory.SecretMoved{Node: "anchor", Generation: 1, Digest: "sha256:something before"} + var asked []link.MoveSecretAsk + ask := func(r inventory.SecretToMove, row link.MoveSecretAsk) (link.MoveSecretAnswer, error) { + asked = append(asked, row) + switch r.Secret { + case "own/anchor/postgres/refused": + return link.MoveSecretAnswer{Refused: "the vault already holds another value; nothing was moved"}, nil + case "own/anchor/postgres/replication": + return link.MoveSecretAnswer{Secret: r.Secret, Generation: 3, Unchanged: true}, nil + } + return link.MoveSecretAnswer{Secret: r.Secret, Generation: 1}, nil + } + recorded := map[string]int{} + record := func(r inventory.SecretToMove, generation int) error { recorded[r.Secret] = generation; return nil } + m := moveNode("anchor", func(string) ([]inventory.SecretToMove, error) { return rows, nil }, ask, record) + if m.Moved != 1 || m.Already != 2 || len(m.Refused) != 2 || len(m.Changed) != 1 || m.Unreached != nil { + t.Fatalf("the move said %+v", m) + } + if len(asked) != 3 || asked[0].Module != "postgres" || asked[0].Name != "superuser" || asked[0].Sealed != "c2VhbGVk" || + asked[0].Origin != "made" || asked[0].Generation != "g-1" || asked[0].SealedTo != "k" { + t.Fatalf("asked %+v", asked) + } + if recorded["own/anchor/postgres/superuser"] != 1 || recorded["own/anchor/postgres/replication"] != 3 || len(recorded) != 2 { + t.Errorf("recorded %v", recorded) + } + said := m.String() + for _, want := range []string{"anchor: 1 moved into the vault and read back the same, 2 there already", + "own/anchor/postgres/old-key", "earlier sealing key", "own/anchor/postgres/refused", "already holds another value", + "own/anchor/postgres/changed-since", "changed in the controller's store after it moved"} { + if !strings.Contains(said, want) { + t.Errorf("the move did not say %q:\n%s", want, said) + } + } + if strings.Contains(said, "c2VhbGVk") { + t.Error("the move said a sealed value") + } +} + +// A machine that does not answer is left for when it is next heard from: nothing more is asked of it now, and that +// is said; a pair is asked with both its ends and never as an own secret. +func TestAMachineNotAnsweringIsMovedWhenNextHeardFrom(t *testing.T) { + pair := inventory.SecretToMove{Secret: "pair/anchor/database/shanks/gitea", Node: "shanks", Provider: "anchor", + Provision: "database", Consumer: "shanks", ConsumerModule: "gitea", Origin: "accepted", Sealed: "x", + SealedTo: "k", Current: true} + rows := []inventory.SecretToMove{pair, pair} + calls := 0 + var asked link.MoveSecretAsk + ask := func(_ inventory.SecretToMove, row link.MoveSecretAsk) (link.MoveSecretAnswer, error) { + calls++ + asked = row + return link.MoveSecretAnswer{}, errors.New("nothing on shanks answers a move") + } + m := moveNode("shanks", func(string) ([]inventory.SecretToMove, error) { return rows, nil }, ask, + func(inventory.SecretToMove, int) error { t.Fatal("recorded"); return nil }) + if calls != 1 || m.Unreached == nil || !strings.Contains(m.String(), "moved when it is next heard from") { + t.Fatalf("calls %d, %+v: %s", calls, m, m.String()) + } + if asked.Module != "" || asked.Provider != "anchor" || asked.Consumer != "shanks" || asked.ConsumerModule != "gitea" { + t.Errorf("the pair was asked as %+v", asked) + } + failing := moveNode("x", func(string) ([]inventory.SecretToMove, error) { return nil, errors.New("no store") }, nil, nil) + if failing.Failed == nil { + t.Error("a store that cannot be read was not said") + } +} + +// The record of asks is copied into the vault in parts, as it is: every row's fields, nothing more. +func TestTheRecordOfAsksIsCopiedInParts(t *testing.T) { + ended := time.Unix(100, 0).UTC() + var asks []inventory.SecretAsk + for n := 1; n <= movedAsksAtOnce+3; n++ { + asks = append(asks, inventory.SecretAsk{ID: int64(n), Node: "anchor", Module: "telegram", Name: "token", + AskedBy: "g14/claude-code", Desk: "laptop", OpenedAt: time.Unix(int64(n), 0).UTC(), EndedAt: &ended, + Outcome: "given"}) + } + var parts []int + vault := func(verb string, args map[string]any) (json.RawMessage, error) { + if verb != "move-asks" { + t.Fatalf("asked %s", verb) + } + rows := args["asks"].([]map[string]any) + parts = append(parts, len(rows)) + if rows[0]["desk"] != "laptop" || rows[0]["asked_by"] != "g14/claude-code" || rows[0]["outcome"] != "given" || + rows[0]["id"] == nil || rows[0]["opened_at"] == nil || rows[0]["ended_at"] == nil { + t.Fatalf("a row went as %v", rows[0]) + } + return json.Marshal(map[string]int{"copied": len(rows)}) + } + copied, err := moveAsks(asks, vault) + if err != nil || copied != movedAsksAtOnce+3 || len(parts) != 2 || parts[0] != movedAsksAtOnce { + t.Fatalf("copied %d in %v, %v", copied, parts, err) + } + if _, err := moveAsks(asks, func(string, map[string]any) (json.RawMessage, error) { + return nil, errors.New("the vault is away") + }); err == nil { + t.Error("a copy the vault refused was not said") + } +} + +// The verb runs the move's one line through the seat and nothing else of the secret commands. +func TestTheMoveIsAVerbOfItsOwn(t *testing.T) { + argv, err := argvFor("secret-move", map[string]any{}) + if err != nil || strings.Join(argv, " ") != "secret move" { + t.Fatalf("argv %v, %v", argv, err) + } + argv, err = argvFor("secret-move", map[string]any{"node": "anchor"}) + if err != nil || strings.Join(argv, " ") != "secret move anchor" { + t.Fatalf("argv %v, %v", argv, err) + } + if err := terminalOnly([]string{"secret", "move", "anchor"}); err != nil { + t.Fatalf("the move's line was refused through its verb: %v", err) + } + for _, line := range [][]string{{"secret", "move", "--from", "x"}, {"secret", "move", "a", "b"}} { + if err := terminalOnly(line); err == nil { + t.Errorf("%q ran through a verb", line) + } + } + found := false + for _, v := range catalogue.ControllerVerbs { + found = found || v.Name == "secret-move" + } + if !found { + t.Fatal("the controller's seat does not declare the secret-move verb") + } +} diff --git a/cmd/mesh-controller/sendable.go b/cmd/mesh-controller/sendable.go index 973813b1..264a46c2 100644 --- a/cmd/mesh-controller/sendable.go +++ b/cmd/mesh-controller/sendable.go @@ -77,6 +77,9 @@ type sendable struct { // unplaced is every contribution its holder's template could not render, naming its module and // why (novox/hq ADR 0255): left out of this declaration, for push and plan to say, never on the wire. unplaced []string + // handedOnly is each file left out because only the provisioner's client fills it and the module is not handed + // here (a provider's consumers file, ADR 0373). + handedOnly []string // Builds is the build of each module this declaration carries — module to the commit its build // was made from — recorded with the send and never on the wire (novox/hq issue 259, ADR 0221). // Composed only on the send path; nil records that it is not known. @@ -84,6 +87,16 @@ type sendable struct { // Bindings is where each consumer of a provision that keeps its data is bound in this declaration // (novox/hq ADR 0232), recorded with the send and never on the wire. Composed only on the send path. Bindings []inventory.Binding + // Filled is the path of every file this declaration hands to the provisioner, and Provisioner the account the + // client that fills them runs as (novox/hq ADR 0349 §4, §5, design 57 P6): on the wire as `filled_files` and + // `provisioner_account`, both absent when nothing is handed, so every other declaration is byte for byte what + // it was. plans is each handed module's plan, kept with the send for the assignments mirror; never on the wire. + Filled []string + Provisioner string + plans map[string]catalogue.Plan + // requiredBy is, for each handed module only pulled in by another, the module that requires it (issue 765): kept + // beside its plan, so the assignments mirror gives the plan a value of its own. Never on the wire. + requiredBy map[string]string } // adoptionEnvelope is what an adopted node is told about its mode. Taken is every module taken on @@ -113,6 +126,10 @@ func (s sendable) Body() ([]byte, error) { if s.Generation > 0 { envelope["generation"] = s.Generation } + if len(s.Filled) > 0 { + envelope["filled_files"] = s.Filled + envelope["provisioner_account"] = s.Provisioner + } // An empty declaration is deliberate here — the node owns nothing the mesh put there // (novox/hq issue 127) — and the host refuses an empty body unless it is told the emptiness // is meant, so a truncated or mis-composed body is never mistaken for "own nothing". diff --git a/cmd/mesh-controller/sent_required_by_test.go b/cmd/mesh-controller/sent_required_by_test.go new file mode 100644 index 00000000..68553609 --- /dev/null +++ b/cmd/mesh-controller/sent_required_by_test.go @@ -0,0 +1,125 @@ +package main + +import ( + "encoding/json" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +// A send keeps, beside each handed module's plan, what requires a module only pulled in (issue 765), so the +// assignments mirror gives that plan a value of its own; an assigned module's plan is kept as the bytes it was. +func TestASendKeepsWhatRequiresAPulledInModule(t *testing.T) { + plan := catalogue.Plan{Templates: catalogue.PlanTemplates{Source: "http://store:5000/v2/notify/files", + Digest: "sha256:" + strings.Repeat("d", 64)}, + Files: []catalogue.PlanFile{{ID: "rc", Path: "/home/op/rc", Template: "rc", OwnerAtRuntime: "owned"}}} + r := sentRecordOf(t.Context(), sendable{plans: map[string]catalogue.Plan{"dunst": plan, "notify": plan}, + requiredBy: map[string]string{"notify": "desk"}}) + kept := string(r.plans) + want := `{"dunst":{"templates":{"source":"http://store:5000/v2/notify/files","digest":"sha256:` + + strings.Repeat("d", 64) + `"},"files":[{"id":"rc","path":"/home/op/rc","template":"rc","owner-at-runtime":"owned"}]},` + + `"notify":{"templates":{"source":"http://store:5000/v2/notify/files","digest":"sha256:` + strings.Repeat("d", 64) + + `"},"files":[{"id":"rc","path":"/home/op/rc","template":"rc","owner-at-runtime":"owned"}],"required-by":"desk"}}` + if kept != want { + t.Fatalf("the send keeps\n%s\nwant\n%s", kept, want) + } +} + +// **A module only pulled in by another is handed on a machine that can take it** (issue 765): on a contract-11 +// node running a client, a moved module that nothing assigned there, required by one that is, has its files and its +// own secret handed like an assigned one's, no secret of it made or sealed by the controller, and its plan mirrored +// under its own key, saying what requires it. A module not moved, pulled in the same way, is sent as before. +func TestAModulePulledInIsHandedOnAMachineThatCanTakeItAndMirrored(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + client := aClientModule() + moved, kept := aBot("moved-bot", true), aBot("kept-bot", false) + moved.Provides = []catalogue.Offer{{Name: "moved-bot-api"}} + kept.Provides = []catalogue.Offer{{Name: "kept-bot-api"}} + desk := catalogue.Manifest{Module: "desk", Version: "1", Requires: []string{"moved-bot-api", "kept-bot-api"}} + for _, m := range []catalogue.Manifest{client, moved, kept, desk} { + register(t, open, m) + } + for _, m := range []string{client.Module, desk.Module} { + if _, err := assign(ctx, open, "laptop", m); err != nil { + t.Fatal(err) + } + } + if err := issueProvisionerClient(ctx, inv, client, "laptop", broker.Broker{Fingerprint: "ab"}, "10.0.0.1:4222"); err != nil { + t.Fatal(err) + } + if _, err := inv.RecordHealth(ctx, inventory.NodeHealth{Node: "laptop", Contract: link.HandedContract, + SaidAt: time.Now()}); err != nil { + t.Fatal(err) + } + + plan, settings, err := planFor(ctx, open, "laptop") + if err != nil { + t.Fatal(err) + } + if plan.RequiredBy("moved-bot") != "desk" || plan.RequiredBy("kept-bot") != "desk" { + t.Fatalf("the bots are on laptop because %v", plan.Because) + } + declared, err := declarationFor(ctx, open, "laptop", plan, settings) + if err != nil { + t.Fatal(err) + } + var movedToken, keptToken, movedConf map[string]any + for _, r := range declared.Resources { + switch r["id"] { + case "moved-bot.needs-token": + movedToken = r + case "kept-bot.needs-token": + keptToken = r + case "moved-bot.conf": + movedConf = r + } + } + if movedToken == nil || movedToken["sealed"] != nil || movedToken["owner-at-runtime"] != "owned" { + t.Fatalf("the pulled-in moved module's secret file is sent as %v: placed, with no value", movedToken) + } + if movedConf == nil || movedConf["content"] != nil || movedConf["owner-at-runtime"] != "owned" { + t.Fatalf("the pulled-in moved module's templated file is sent as %v: placed, with no text", movedConf) + } + if keptToken == nil || keptToken["sealed"] == nil { + t.Fatalf("the pulled-in module not moved has its secret sent as %v: sealed, as before", keptToken) + } + if _, held, err := inv.ModuleSecretIfIssued(ctx, "laptop", "moved-bot", "token"); err != nil || held { + t.Fatalf("the send made the pulled-in handed module's secret in the controller's store (held %v, %v)", held, err) + } + if _, has := declared.plans["kept-bot"]; has || len(declared.plans) != 1 { + t.Fatalf("the send hands %v", declared.plans) + } + + record, err := inv.NodeByName(ctx, "laptop") + if err != nil { + t.Fatal(err) + } + if err := inv.RecordSentWith(ctx, record.ID, "first", nil, 0, 0, inventory.Send{Digest: "first", + Provisioning: sentRecordOf(ctx, declared).plans}); err != nil { + t.Fatal(err) + } + inv.SetProvisioningPlans(inv.SentProvisioningPlans) + mirror, err := inv.AssignmentsMirror(ctx) + if err != nil { + t.Fatal(err) + } + var value inventory.AssignmentMirrored + if err := json.Unmarshal(mirror["laptop.moved-bot"], &value); err != nil { + t.Fatalf("no value for the pulled-in module: %s (%v)", mirror["laptop.moved-bot"], err) + } + if value.RequiredBy != "desk" || !value.Assigned.IsZero() || value.Provisioning == nil || + len(value.Provisioning.Files) != 1 || value.Provisioning.Files[0].Path != "/etc/moved-bot/bot.conf" || + len(value.Provisioning.Secrets) != 1 || value.Provisioning.Secrets[0].Name != "token" { + t.Fatalf("the pulled-in module is mirrored as %s", mirror["laptop.moved-bot"]) + } + if _, has := mirror["laptop.kept-bot"]; has { + t.Fatalf("a pulled-in module not handed is mirrored: %s", mirror["laptop.kept-bot"]) + } +} diff --git a/cmd/mesh-controller/settings_forward.go b/cmd/mesh-controller/settings_forward.go new file mode 100644 index 00000000..c3c423de --- /dev/null +++ b/cmd/mesh-controller/settings_forward.go @@ -0,0 +1,372 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "time" + + "github.com/nats-io/nats.go" + + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +// **`settings` answers by forwarding to `config`** (novox/hq ADR 0348 §1, design 57 K5, issue 664). Once the move of +// config has run, a module in the config form keeps its config values in mesh-config (inventory.ReadsConfig), so the +// settings verb reads and writes them there: `show` lists what `config list` lists, `set` sets each value as the +// module's own on the node and `clear` clears them, and `propose` proposes each through mesh-config. Placement stays +// the controller's and is read and written beside the assignment as before (ADR 0348 §11). A module still in the +// settings form, and every module before the move, keeps today's verb unchanged. + +// configAsk asks mesh-config's verb in the controller's own name, so mesh-config takes a value as the controller +// carries it (ADR 0348 §8); a test replaces it. +var configAsk = func(ctx context.Context, args map[string]any) (json.RawMessage, error) { + var result json.RawMessage + err := onTheBus(func(conn *nats.Conn) error { + answer, err := link.AskMeshSeatToolAs(ctx, conn, broker.ControllerName, "mesh-config", "config", args, 30*time.Second) + if err != nil { + return err + } + if answer.Error != "" { + return errors.New(answer.Error) + } + result = answer.Result + return nil + }) + return result, err +} + +// configListed is what `config list` answers, as the settings verb reads it. +type configListed struct { + Resolved []resolvedEntry `json:"resolved"` + Set []struct { + Level configLevel `json:"level"` + Key string `json:"key"` + Value json.RawMessage `json:"value"` + By string `json:"by"` + } `json:"set"` +} + +func listConfig(ctx context.Context, module, node string) (configListed, error) { + args := map[string]any{"action": "list", "module": module} + if node != "" { + args["node"] = node + } + raw, err := configAsk(ctx, args) + if err != nil { + return configListed{}, fmt.Errorf("mesh-config does not answer %s's config values: %w", module, err) + } + var l configListed + if err := json.Unmarshal(raw, &l); err != nil { + return configListed{}, fmt.Errorf("mesh-config's listing cannot be read: %w", err) + } + return l, nil +} + +// ownValues is the module's own values on the node, as mesh-config holds them; none for the whole mesh, which has no +// module value (ADR 0348 §3). +func (l configListed) ownValues(module, node string) map[string]any { + out := map[string]any{} + if node == "" { + return out + } + for _, s := range l.Set { + if s.Level.Level == "module" && s.Level.Node == node && s.Level.Module == module { + var v any + if json.Unmarshal(s.Value, &v) == nil { + out[s.Key] = v + } + } + } + return out +} + +// forwardSettings answers a settings action for a module whose config values are mesh-config's, and says whether it +// did: false leaves the action to today's verb. +func forwardSettings(ctx context.Context, inv *inventory.Inventory, action, module, node string, positionals []string, + replace bool) (bool, error) { + if module == "" { + return false, nil + } + reads, err := inv.ReadsConfig(ctx, module) + if err != nil || !reads { + return false, err + } + where := whereOf(node) + switch action { + case "show": + listed, err := listConfig(ctx, module, node) + if err != nil { + return true, err + } + layer, _, err := inv.Layer(ctx, node, module) + if err != nil { + return true, err + } + if layer == nil { + layer = map[string]any{} + } + for k, v := range listed.ownValues(module, node) { + layer[k] = v + } + if len(layer) == 0 { + fmt.Printf("%s on %s: no layer — the module's definition says\n", module, where) + } else { + shown, err := json.MarshalIndent(layer, "", " ") + if err != nil { + return true, err + } + fmt.Println(string(shown)) + } + fmt.Printf(" its config values are mesh-config's since the move of config (issue 664): mesh-config.config\n") + fmt.Printf("%s on %s, every value and where it comes from:\n", module, where) + for _, r := range listed.Resolved { + value := string(r.Value) + if value == "" { + value = "unset" + } + on := "" + if node == "" { + on = " on " + r.Node + } + fmt.Printf(" %s%s = %s (%s)\n", r.Key, on, value, r.Source) + } + return true, nil + + case "set": + var raw []byte + if strings.HasPrefix(strings.TrimSpace(positionals[1]), "{") { + raw = []byte(positionals[1]) + } else { + return true, errors.New("the settings of a module whose config values are mesh-config's are given as " + + "{…}, never a file; or set each with mesh-config.config") + } + var values map[string]any + if err := json.Unmarshal(raw, &values); err != nil { + return true, fmt.Errorf("%s is not a settings object: %w", positionals[1], err) + } + return true, forwardSet(ctx, inv, module, node, values, replace) + + case "clear": + listed, err := listConfig(ctx, module, node) + if err != nil { + return true, err + } + own := listed.ownValues(module, node) + layer, _, err := inv.Layer(ctx, node, module) + if err != nil { + return true, err + } + before := map[string]any{} + for k, v := range layer { + before[k] = v + } + for k, v := range own { + before[k] = v + } + if err := refuseTerminalSettingsThroughAVerb(ctx, inv, before, nil, module, where); err != nil { + return true, err + } + if err := refuseTrustedThroughAVerb(ctx, inv, module, where, movedKeys(own)); err != nil { + return true, err + } + for _, k := range movedKeys(own) { + if _, err := configAsk(ctx, map[string]any{"action": "clear", "module": module, "node": node, "key": k, + "why": "settings clear " + module + " on " + where, "by": setByWords()}); err != nil { + return true, fmt.Errorf("%s of %s on %s was not cleared: %w", k, module, where, err) + } + } + if len(layer) > 0 { + if err := inv.ClearSettingsBy(ctx, node, module, setByWords()); err != nil { + return true, err + } + } + fmt.Printf("%s on %s is back to what the module says\n", module, where) + return true, nil + + } + return false, nil +} + +// forwardSet sets a layer of a module whose config values are mesh-config's: its placement beside the assignment +// as before, each config value as the module's own on the node, and each it no longer names cleared. +func forwardSet(ctx context.Context, inv *inventory.Inventory, module, node string, values map[string]any, + replace bool) error { + where := whereOf(node) + config, placement := catalogue.SplitPlacement(values) + listed, err := listConfig(ctx, module, node) + if err != nil { + return err + } + own := listed.ownValues(module, node) + layer, _, err := inv.Layer(ctx, node, module) + if err != nil { + return err + } + before := map[string]any{} + for k, v := range layer { + before[k] = v + } + for k, v := range own { + before[k] = v + } + if err := refuseTerminalSettingsThroughAVerb(ctx, inv, before, values, module, where); err != nil { + return err + } + added, changed, removed := settingsChange(before, values) + if len(removed) > 0 && !replace { + return fmt.Errorf("%s on %s: this layer would no longer set %s. A layer is replaced whole; read it with "+ + "`settings show %s%s` and include what should stay, or add --replace if the removal is meant (novox/hq "+ + "ADR 0217). Nothing was changed", module, where, strings.Join(removed, ", "), module, nodeFlag(node)) + } + var toSet, toClear []string + for _, k := range movedKeys(config) { + if was, had := own[k]; !had || !sameJSON(valueJSON(was), config[k]) { + toSet = append(toSet, k) + } + } + for _, k := range movedKeys(own) { + if _, kept := config[k]; !kept { + toClear = append(toClear, k) + } + } + if node == "" && len(toSet)+len(toClear) > 0 { + return fmt.Errorf("%s has no own value for the whole mesh (novox/hq ADR 0348 §3): name the node, or set mesh "+ + "config for the predefined key a value falls back to with mesh-config.config. Nothing was changed", module) + } + if err := refuseTrustedThroughAVerb(ctx, inv, module, where, append(append([]string{}, toSet...), toClear...)); err != nil { + return err + } + if placement == nil { + placement = map[string]any{} + } + if _, placed := catalogue.SplitPlacement(layer); !sameLayer(placed, placement) { + if err := inv.SetSettingsBy(ctx, node, module, placement, setByWords()); err != nil { + return err + } + } + for _, k := range toSet { + if _, err := configAsk(ctx, map[string]any{"action": "set", "module": module, "node": node, "key": k, + "value": config[k], "why": "settings set " + module + " on " + where, "by": setByWords()}); err != nil { + return fmt.Errorf("%s of %s on %s was not set: %w", k, module, where, err) + } + } + for _, k := range toClear { + if _, err := configAsk(ctx, map[string]any{"action": "clear", "module": module, "node": node, "key": k, + "why": "settings set " + module + " on " + where + " --replace", "by": setByWords()}); err != nil { + return fmt.Errorf("%s of %s on %s was not cleared: %w", k, module, where, err) + } + } + fmt.Printf("%s on %s:\n", module, where) + if len(added)+len(changed)+len(removed) == 0 { + fmt.Println(" nothing changed") + } + for _, p := range added { + fmt.Printf(" + %s\n", p) + } + for _, p := range changed { + fmt.Printf(" ~ %s\n", p) + } + for _, p := range removed { + fmt.Printf(" - %s\n", p) + } + fmt.Println(" its config values are mesh-config's (issue 664): each was set there, with its history; the module " + + "takes them when its files are filled again") + return nil +} + +func valueJSON(v any) json.RawMessage { + raw, _ := json.Marshal(v) + return raw +} + +// refuseTrustedThroughAVerb refuses, through a verb, a trusted config value (ADR 0348 §8): mesh-config takes the +// controller's word for one as given at its terminal, and whoever may call a verb includes agents. +func refuseTrustedThroughAVerb(ctx context.Context, inv *inventory.Inventory, module, where string, keys []string) error { + verb, through := throughAVerb() + if !through || len(keys) == 0 { + return nil + } + shelf, err := inv.Catalogue(ctx) + if err != nil { + return fmt.Errorf("which config values of %s are trusted cannot be read, so nothing was changed: %w", module, err) + } + for _, k := range keys { + if shelf[module].Settings[k].Trusted { + return fmt.Errorf("%s of %s on %s is trusted (novox/hq ADR 0348 §8): it is set at the controller's terminal "+ + "or on the operator's Approve, never through a verb alone (this line came through %q). Propose it with "+ + "mesh-config.config, action propose. Nothing was changed", k, module, where, verb) + } + } + return nil +} + +// forwardPropose proposes, through mesh-config, each config value of a layer proposed for a module whose config +// values are mesh-config's (ADR 0348 §8): mesh-config asks the operator under its own name, one ask per value. A +// proposal naming placement too is refused, so the operator is never asked half of it; placement alone is proposed +// as before. Says whether it answered. +func forwardPropose(ctx context.Context, inv *inventory.Inventory, module, node, valuesArg string, clear bool) (bool, error) { + reads, err := inv.ReadsConfig(ctx, module) + if err != nil || !reads { + return false, err + } + where := whereOf(node) + if node == "" { + return true, fmt.Errorf("%s has no own value for the whole mesh (novox/hq ADR 0348 §3): name the node, or "+ + "propose mesh config for the predefined key a value falls back to with mesh-config.config. Nothing was "+ + "proposed", module) + } + proposals := map[string]any{} + var clears []string + if clear { + listed, err := listConfig(ctx, module, node) + if err != nil { + return true, err + } + layer, _, err := inv.Layer(ctx, node, module) + if err != nil { + return true, err + } + if len(layer) > 0 { + return true, fmt.Errorf("%s on %s keeps placement beside its assignment (%s): propose clearing it apart "+ + "from its config values. Nothing was proposed", module, where, strings.Join(movedKeys(layer), ", ")) + } + clears = movedKeys(listed.ownValues(module, node)) + } else { + var values map[string]any + if err := json.Unmarshal([]byte(valuesArg), &values); err != nil { + return true, fmt.Errorf("%s is not a settings object: %w", valuesArg, err) + } + config, placement := catalogue.SplitPlacement(values) + if len(placement) > 0 { + if len(config) > 0 { + return true, fmt.Errorf("%s names placement (%s) and config values: propose them apart, the config "+ + "values through mesh-config. Nothing was proposed", module, strings.Join(movedKeys(placement), ", ")) + } + return false, nil + } + proposals = config + } + for _, k := range movedKeys(proposals) { + answer, err := configAsk(ctx, map[string]any{"action": "propose", "module": module, "node": node, "key": k, + "value": proposals[k], "why": "settings propose " + module + " on " + where}) + if err != nil { + return true, fmt.Errorf("%s of %s on %s was not proposed: %w", k, module, where, err) + } + fmt.Printf("%s of %s on %s, proposed through mesh-config: %s\n", k, module, where, answer) + } + for _, k := range clears { + answer, err := configAsk(ctx, map[string]any{"action": "propose", "module": module, "node": node, "key": k, + "clear": true, "why": "settings propose " + module + " on " + where + " --clear"}) + if err != nil { + return true, fmt.Errorf("clearing %s of %s on %s was not proposed: %w", k, module, where, err) + } + fmt.Printf("clearing %s of %s on %s, proposed through mesh-config: %s\n", k, module, where, answer) + } + return true, nil +} diff --git a/cmd/mesh-controller/signals.go b/cmd/mesh-controller/signals.go index e2cbc387..51b7d598 100644 --- a/cmd/mesh-controller/signals.go +++ b/cmd/mesh-controller/signals.go @@ -153,11 +153,15 @@ var signalsTable = []signalRow{ return newestOf(f.standings, func(c conditions.Condition) time.Time { return c.LastObserved }) }}, {Row: "S9", Signal: "bus advisories: maximum deliveries, consumer deleted; the controller's own slow " + - "consumer and refused subjects", Emitter: "bus server's advisory subjects; the controller's connection", + "consumer and refused subjects; every user's connection the server closed as a slow consumer or for " + + "breaking the bus's rules (ADR 0357)", + Emitter: "bus server's advisory subjects; the controller's connection; the bus module's watch of its " + + "server (bus.slow-consumer, bus.connection-closed)", Trigger: "any", Bound: "any occurrence; clears after an hour without another, a deleted consumer " + "once it exists again or the mesh no longer expects it, and a message given up on once DEAD_LETTERS " + "no longer holds it (novox/hq issue 330)", - Kind: "slow-consumer, max-deliveries, refused, consumer-lost", Severity: conditions.Warning, Phase: 1, + Kind: "slow-consumer, connection-closed, max-deliveries, refused, consumer-lost", Severity: conditions.Warning, + Phase: 1, needs: func(f *signalFacts) error { return f.advisoriesErr }, watch: watchAdvisories, newest: func(f *signalFacts) time.Time { return newestOf(f.advisories, func(a link.Advisory) time.Time { return a.Last }) @@ -569,6 +573,9 @@ func watchAdvisories(f *signalFacts) []conditions.Observation { // The advisory is remembered and read again on every look for an hour: the condition counts // what the bus said and when, not the looks (novox/hq issue 402). Happened: a.Last, Times: a.Count} + if a.User != "" && a.Reason != "" { + busUserClosed(&o, a, f.host) + } if a.Kind == link.AdvisoryMaxDeliveries && a.Token == link.AdvisoryNotKept { o.Machine = consumerMachine(a.Stream, a.Consumer) o.Headline = clip(conditions.Capital(fmt.Sprintf("%s gave up on a message, not kept", @@ -579,9 +586,58 @@ func watchAdvisories(f *signalFacts) []conditions.Observation { } out = append(out, o) } + if f.busFaultsErr != nil { + out = append(out, busFaultsUnread(f.busFaultsErr)) + } return append(out, watchDeadLetters(f)...) } +// kindBusFaultsUnread is the bus module's events not read back (novox/hq ADR 0357): every other user's +// slow consumer is unheard while it lasts, and the controller's own signals are still said. +const kindBusFaultsUnread = "bus-faults-unread" + +func busFaultsUnread(err error) conditions.Observation { + return conditions.Observation{Scope: conditions.ScopeBus, ID: "bus-module", Kind: kindBusFaultsUnread, + Severity: conditions.Warning, Confirm: true, + Summary: "what the bus module said of its users' connections could not be read back, so a slow consumer " + + "or a connection the bus closed is not said until it can", + Said: firstLine(err.Error())} +} + +// busUserClosed words an advisory about a bus user's connection the server closed (novox/hq ADR 0357): the +// user as the operator says it and the machine it is on, the dotted user name kept in the evidence only. +func busUserClosed(o *conditions.Observation, a link.Advisory, host string) { + who, machine := busUserWords(a.User) + if machine == "" && a.User == "controller" { + machine = host + } + o.Machine = machine + times, count := "once", "once" + if a.Count > 1 { + times = fmt.Sprintf("%d times since %s", a.Count, a.First.UTC().Format("15:04 MST")) + count = fmt.Sprintf("%d times", a.Count) + } + switch a.Kind { + case link.AdvisorySlowConsumer: + o.Summary = fmt.Sprintf("the bus closed the connection of %s as a slow consumer %s in the last hour, "+ + "newest at %s: it could not take what it was sent fast enough, and what was on its way to it was lost", + who, times, a.Last.UTC().Format("15:04 MST")) + o.Headline = clip(conditions.Capital(who)+" was too slow for the bus", 60) + o.Explanation = conditions.Capital(fmt.Sprintf("%s could not keep up with what the bus sent it, so the bus "+ + "closed its connection (%s in the last hour) and what was on its way was lost. It connects again "+ + "by itself; this clears an hour after the last time.", who, count)) + o.Resolved = "Resolved: " + who + " kept up for an hour" + default: + o.Summary = fmt.Sprintf("the bus closed the connection of %s for breaking its rules %s in the last hour, "+ + "newest at %s: %s", who, times, a.Last.UTC().Format("15:04 MST"), strings.ToLower(a.Reason)) + o.Headline = clip(conditions.Capital(who)+" was cut off by the bus", 60) + o.Explanation = conditions.Capital(fmt.Sprintf("The bus closed the connection of %s because it broke "+ + "the bus's rules (%s, %s in the last hour), so what it was sending was lost. This clears an hour "+ + "after the last time.", who, strings.ToLower(a.Reason), count)) + o.Resolved = "Resolved: " + who + " kept to the bus's rules for an hour" + } +} + // watchDeadLetters says each consumer that DEAD_LETTERS holds a message for (novox/hq issue 330): open // while it holds any, so it clears when they are delivered again or dropped, never because the server // stopped saying it. @@ -806,7 +862,7 @@ func watchLease(f *signalFacts) []conditions.Observation { } } if len(lost) > 0 { - how := "lost it: its renewal was refused or could not be made" + how := "lost it: another holder wrote its key, or no renewal was made within the key's age" if newest.How == inventory.EpochExpired { how = "stopped renewing it without giving it back, and was taken over" } diff --git a/cmd/mesh-controller/status.go b/cmd/mesh-controller/status.go index af68b7f5..01cf7f98 100644 --- a/cmd/mesh-controller/status.go +++ b/cmd/mesh-controller/status.go @@ -205,6 +205,10 @@ func printStatus(asked answers) error { fmt.Printf("%d machine(s) have never been sent anything:\n %s\n", len(never), strings.Join(never, ", ")) } + for _, n := range sortedKeysOf(stringsKeys(asked.recordedWaits)) { + fmt.Printf(" %s is behind, waiting for a person: %s — `push %s` sends it\n", n, + strings.Join(asked.recordedWaits[n], "; "), n) + } fmt.Printf("\n `push --behind` sends them\n\n") } @@ -540,6 +544,29 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) { if err != nil { return answers{}, err } + // And, of those, what waits for a person's push (novox/hq ADR 0377 §6): a recorded module's move that went out + // with its walk raises nothing here. + if len(out.waiting) > 0 { + f, err := readMoveFacts(ctx, inv) + if err != nil { + return answers{}, err + } + for _, m := range out.waiting { + if m.Never { + continue + } + held, err := recordedWaiting(ctx, inv, f, m.Node) + if err != nil { + return answers{}, err + } + if len(held) > 0 { + if out.recordedWaits == nil { + out.recordedWaits = map[string][]string{} + } + out.recordedWaits[m.Node] = held + } + } + } out.reported, err = inv.LastReports(ctx) if err != nil { return answers{}, err diff --git a/cmd/mesh-controller/status_summary.go b/cmd/mesh-controller/status_summary.go index 52d29591..c52f3c41 100644 --- a/cmd/mesh-controller/status_summary.go +++ b/cmd/mesh-controller/status_summary.go @@ -224,5 +224,10 @@ func (l nudgingListener) Heard(ctx context.Context, report link.Report) (bool, e // nothing it caused (novox/hq ADR 0228). go replaceGiven(context.WithoutCancel(ctx), l.open, report) } + // And, once the move of the secrets into the vault has started, what is left of this machine's (design 57's V3, + // issue 676): a machine asleep when it started is moved when it is next heard from. Off the report's path. + if err == nil && l.open != nil && report.Superseded == "" && report.Rekey == nil && report.Node != "" { + go moveWhenHeard(context.WithoutCancel(ctx), l.open, moveHeardOn(), report.Node) + } return news, err } diff --git a/cmd/mesh-controller/stores.go b/cmd/mesh-controller/stores.go index f2db129a..5fef54d7 100644 --- a/cmd/mesh-controller/stores.go +++ b/cmd/mesh-controller/stores.go @@ -130,6 +130,8 @@ func openInventory(ctx context.Context) (*inventory.Inventory, error) { if err := loadPredefined(ctx, inv); err != nil { fmt.Fprintf(os.Stderr, "the list of predefined keys kept cannot be read: %v\n", err) } + // And where config values are read once the move of config has run (issue 664): mesh-config_resolved. + inv.ReadsConfigFrom(readConfigValues) return inv, nil } diff --git a/cmd/mesh-controller/superseded.go b/cmd/mesh-controller/superseded.go new file mode 100644 index 00000000..e3906713 --- /dev/null +++ b/cmd/mesh-controller/superseded.go @@ -0,0 +1,157 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "strings" + "time" + + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/link" +) + +// A new push to a pull request ends the check of its old head (novox/hq issue 726). +// +// A check judges one head; once the pull request has another, its verdict answers nobody and its run holds a build +// machine for minutes the new head waits behind. So when a check is asked for pull request N of a repository at a +// head, **every check of that pull request at another head ends**: a waiting one is dropped from the queue as +// `cancel` drops it, one in flight is ended on its machine as `kill` ends it. Each is said in the log. +// +// **Never anything else**: a try (`try`, asked for no pull request), a walk's build (no check), a delivery group's +// composed check (it answers several pull requests), another pull request's check, a dead ask (left for `clear +// --dead`), and a check of the same head — a recheck of that head is not another head. + +// **Only for the pull request's current head.** Announcements of a pull request are not ordered, one is heard again +// after a restart, and a recheck asks for the head it recorded: a check asked for a head the pull request no longer +// has must never end the check of the head it has. So the forge is asked first which head the pull request has; only +// when it names this one does anything end, and when it cannot be asked, or names another, nothing ends and that is +// said. + +// forgePullHead asks the forge, through its module's tools on the bus, which commit a pull request's head is. A +// variable so a test needs no forge. +var forgePullHead = func(ctx context.Context, js *broker.JetStream, owner, repo string, number int) (string, error) { + raw, _ := json.Marshal(map[string]any{"path": fmt.Sprintf("/repos/%s/%s/pulls/%d", owner, repo, number)}) + answer, err := link.Ask(ctx, link.OverNATS{Conn: js.Conn()}, "gitea", "gitea_api", raw, 30*time.Second) + if err != nil { + return "", err + } + if answer.Error != "" { + return "", errors.New(answer.Error) + } + var pull struct { + Result struct { + Head struct { + SHA string `json:"sha"` + } `json:"head"` + } `json:"result"` + } + if err := json.Unmarshal(answer.Result, &pull); err != nil { + return "", fmt.Errorf("the forge's answer about the pull request cannot be read: %w", err) + } + if pull.Result.Head.SHA == "" { + return "", errors.New("the forge named no head for the pull request") + } + return pull.Result.Head.SHA, nil +} + +// supersededChecks are the checks in the queue that a check of owner/repo#number at head ends: of the same pull +// request, at another head, waiting or in flight, and not the ask keep. +func supersededChecks(q link.Queue, owner, repo string, number int, head, keep string) []link.QueuedAsk { + if number <= 0 || head == "" { + return nil + } + var out []link.QueuedAsk + for _, a := range q.Asks { + c := a.Request.Check + switch { + case c == nil, link.IsTry(a.Request), c.Group != "", c.Number != number, a.ID == keep, + !strings.EqualFold(c.Owner, owner), !strings.EqualFold(c.Repo, repo), + a.State != link.AskWaiting && a.State != link.AskInFlight: + continue + } + ref := a.Request.Ref + if ref == "" { + ref = a.Ref + } + if ref == "" || ref == head || link.SameHead(ref, head) { + continue + } + out = append(out, a) + } + return out +} + +// endSupersededChecks ends the checks a check of p at its head supersedes, keep being the new check's own ask, and +// says each in the log. A waiting one is cancelled — nothing recorded: a check builds nothing, and the forge's status +// of the old head is no longer the pull request's — and one a machine builds is killed there. +func endSupersededChecks(ctx context.Context, js *broker.JetStream, seat string, p link.PullUpdated, keep string) []string { + if p.Number <= 0 || p.Commit == "" { + return nil + } + current, err := forgePullHead(ctx, js, p.Owner, p.Repo, p.Number) + if err != nil { + return []string{fmt.Sprintf("%s/%s#%d (%.8s): nothing ended — the forge could not say the pull request's "+ + "current head, so checks of its other heads keep running: %v", p.Owner, p.Repo, p.Number, p.Commit, err)} + } + if current != p.Commit && !link.SameHead(current, p.Commit) { + return []string{fmt.Sprintf("%s/%s#%d (%.8s): nothing ended — the pull request's head is %.12s now, not "+ + "this one", p.Owner, p.Repo, p.Number, p.Commit, current)} + } + q, err := link.ReadQueue(ctx, js, seat) + if err != nil { + return []string{fmt.Sprintf("%s/%s#%d (%.8s): could not read the %s queue for checks of its earlier heads: %v", + p.Owner, p.Repo, p.Number, p.Commit, seat, err)} + } + var said []string + for _, a := range supersededChecks(q, p.Owner, p.Repo, p.Number, p.Commit, keep) { + old := a.Request.Ref + if old == "" { + old = a.Ref + } + what := fmt.Sprintf("check of %s/%s#%d at %.12s", p.Owner, p.Repo, p.Number, old) + var err error + if a.State == link.AskInFlight && a.On != "" { + var answer link.Answer + answer, err = link.AskSeatTool(ctx, js.Conn(), seat, "kill", a.On, map[string]string{"id": a.ID}, killAnswer) + if err == nil && answer.Error != "" { + err = fmt.Errorf("%s refused: %s", a.On, answer.Error) + } + } else { + _, err = cancelAsk(ctx, js, nil, seat, a) + } + if err != nil { + said = append(said, fmt.Sprintf("not ended: %s (%s) — superseded by %.12s, but: %v", what, a.ID, p.Commit, err)) + continue + } + said = append(said, fmt.Sprintf("ended: %s — superseded by %.12s (%s, %s)", what, p.Commit, a.ID, a.State)) + } + return said +} + +// endingSupersededChecks ends them beside the announcement, so a kill waiting on its machine never holds the +// handler: on its own connection to the bus, within twice a kill's wait. A one-shot process (oneShot) ends them +// within the call instead, as its exit would cut them off. +func endingSupersededChecks(ctx context.Context, seat string, p link.PullUpdated, keep string, oneShot bool) { + end := func() { + ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 2*killAnswer) + defer cancel() + js, err := dialTheBus() + if err != nil { + fmt.Fprintf(os.Stderr, "%s/%s#%d (%.8s): could not reach the bus to end checks of its earlier heads: %v\n", + p.Owner, p.Repo, p.Number, p.Commit, err) + return + } + defer js.Close() + for _, line := range endSupersededChecks(ctx, js, seat, p, keep) { + fmt.Println(line) + } + } + if oneShot { + end() + return + } + go end() +} diff --git a/cmd/mesh-controller/superseded_test.go b/cmd/mesh-controller/superseded_test.go new file mode 100644 index 00000000..104fb78b --- /dev/null +++ b/cmd/mesh-controller/superseded_test.go @@ -0,0 +1,192 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "strings" + "sync" + "testing" + "time" + + "github.com/nats-io/nats.go" + + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/link" +) + +// A new push to a pull request ends the check of its old head (novox/hq issue 726): the waiting one is dropped from +// the queue, the one in flight is killed on its machine, and each is said. A try, a walk's build, another pull +// request's check, a group's check and a recheck of the new head are left alone. On the bus alone: no store. +func TestANewHeadEndsTheChecksOfTheOldOnes(t *testing.T) { + js, _ := aBuildQueue(t) + ctx := t.Context() + const ( + oldHead = "1111111111111111111111111111111111111111" + olderHead = "2222222222222222222222222222222222222222" + newHead = "3333333333333333333333333333333333333333" + ) + publish := func(r link.BuildRequest) { + t.Helper() + body, _ := json.Marshal(r) + if _, err := js.Context().Publish(link.BuildWork(), body); err != nil { + t.Fatal(err) + } + } + check := func(id string, number int, head string) link.BuildRequest { + return link.BuildRequest{ID: id, Repository: "https://forge.example/novox/a.git", Ref: head, + Check: &link.CheckRequest{Owner: "novox", Repo: "a", Number: number}} + } + // In flight first: taken by a holder, started on ace. + publish(check("check-running", 7, olderHead)) + worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: link.TheBuildMachine, Accepts: []string{"build"}}) + sub, err := js.Context().PullSubscribe(worker.Filters[0], worker.Name, nats.Bind(worker.Stream, worker.Name), nats.ManualAck()) + if err != nil { + t.Fatal(err) + } + defer func() { _ = sub.Unsubscribe() }() + if _, err := sub.Fetch(1, nats.MaxWait(3*time.Second)); err != nil { + t.Fatal(err) + } + started, _ := json.Marshal(link.BuildStart{ID: "check-running", On: "ace", At: time.Now().UTC().Format(time.RFC3339Nano)}) + if _, err := js.Context().Publish(link.BuildStarted(), started); err != nil { + t.Fatal(err) + } + publish(check("check-old", 7, oldHead)) + publish(check("check-other-pull", 8, oldHead)) + publish(check("check-recheck", 7, newHead)) + publish(check("check-new", 7, newHead)) + try := check("try-old", 7, oldHead) + try.Check.Number, try.Check.Try = 0, true + publish(try) + group := check("group-old", 7, oldHead) + group.Check.Group = "g1" + publish(group) + publish(link.BuildRequest{ID: "walk-build", Repository: "https://forge.example/novox/a.git", Ref: oldHead}) + + var mu sync.Mutex + var killed []string + stop, err := link.OverNATS{Conn: js.Conn()}.ServeNodeSeatTools(link.TheBuildMachine, "ace", map[string]link.ToolHandler{ + "kill": func(_ context.Context, raw json.RawMessage) (any, error) { + var args struct{ ID string } + _ = json.Unmarshal(raw, &args) + mu.Lock() + killed = append(killed, args.ID) + mu.Unlock() + return map[string]any{"said": "killed " + args.ID}, nil + }}, nil) + if err != nil { + t.Fatal(err) + } + defer stop() + + forgeSays(t, newHead, nil) + said := endSupersededChecks(ctx, js, link.TheBuildMachine, + link.PullUpdated{Owner: "novox", Repo: "a", Number: 7, Commit: newHead}, "check-new") + + mu.Lock() + if len(killed) != 1 || killed[0] != "check-running" { + t.Errorf("the holder was asked to kill %v, want only the in-flight check of the older head", killed) + } + mu.Unlock() + q, err := link.ReadQueue(ctx, js, link.TheBuildMachine) + if err != nil { + t.Fatal(err) + } + if _, found := q.Find("check-old"); found { + t.Error("the waiting check of the old head is still in the queue") + } + for _, id := range []string{"check-other-pull", "check-recheck", "check-new", "try-old", "group-old", "walk-build"} { + if _, found := q.Find(id); !found { + t.Errorf("%s was ended; only checks of the same pull request at another head end", id) + } + } + text := strings.Join(said, "\n") + for _, want := range []string{ + "ended: check of novox/a#7 at " + oldHead[:12] + " — superseded by " + newHead[:12], + "ended: check of novox/a#7 at " + olderHead[:12] + " — superseded by " + newHead[:12], + } { + if !strings.Contains(text, want) { + t.Errorf("the log does not say %q:\n%s", want, text) + } + } + if len(said) != 2 { + t.Errorf("said %d lines, want 2:\n%s", len(said), text) + } +} + +// Which asks a check supersedes, read from the queue alone. +func TestSupersededChecksAreOnlyThePullRequestsOtherHeads(t *testing.T) { + ask := func(id, state, ref string, c *link.CheckRequest) link.QueuedAsk { + return link.QueuedAsk{ID: id, State: state, Ref: ref, Request: link.BuildRequest{ID: id, Ref: ref, Check: c}} + } + pr := func(n int) *link.CheckRequest { return &link.CheckRequest{Owner: "Novox", Repo: "A", Number: n} } + head := "abcdefabcdefabcdef0123456789abcdef012345" + q := link.Queue{Asks: []link.QueuedAsk{ + ask("old", link.AskWaiting, "0123456789ab0123456789ab0123456789ab0123", pr(7)), + ask("short-same", link.AskWaiting, head[:12], pr(7)), + ask("dead", link.AskDead, "0123456789ab0123456789ab0123456789ab0123", pr(7)), + ask("other", link.AskWaiting, "0123456789ab0123456789ab0123456789ab0123", pr(8)), + ask("build", link.AskWaiting, "0123456789ab0123456789ab0123456789ab0123", nil), + ask("keep", link.AskWaiting, "0123456789ab0123456789ab0123456789ab0123", pr(7)), + }} + got := supersededChecks(q, "novox", "a", 7, head, "keep") + if len(got) != 1 || got[0].ID != "old" { + t.Fatalf("superseded %+v, want only old", got) + } + if got := supersededChecks(q, "novox", "a", 0, head, ""); got != nil { + t.Errorf("a check of no pull request superseded %+v", got) + } +} + +// forgeSays makes the forge name head as every pull request's current head, or fail with err. +func forgeSays(t *testing.T, head string, err error) { + t.Helper() + was := forgePullHead + forgePullHead = func(context.Context, *broker.JetStream, string, string, int) (string, error) { return head, err } + t.Cleanup(func() { forgePullHead = was }) +} + +// An older head heard after a newer one — an announcement out of order, one heard again after a restart, a recheck +// of a recorded head — ends nothing: the forge says the pull request's head is the newer one. Nor does a check whose +// pull request's head the forge cannot say. +func TestAnOlderHeadHeardLateEndsNothing(t *testing.T) { + js, _ := aBuildQueue(t) + ctx := t.Context() + const ( + oldHead = "1111111111111111111111111111111111111111" + newHead = "3333333333333333333333333333333333333333" + ) + for _, id := range []string{"check-new", "check-old-late"} { + head := newHead + if id == "check-old-late" { + head = oldHead + } + body, _ := json.Marshal(link.BuildRequest{ID: id, Repository: "https://forge.example/novox/a.git", Ref: head, + Check: &link.CheckRequest{Owner: "novox", Repo: "a", Number: 7}}) + if _, err := js.Context().Publish(link.BuildWork(), body); err != nil { + t.Fatal(err) + } + } + late := link.PullUpdated{Owner: "novox", Repo: "a", Number: 7, Commit: oldHead} + forgeSays(t, newHead, nil) + said := endSupersededChecks(ctx, js, link.TheBuildMachine, late, "check-old-late") + if len(said) != 1 || !strings.Contains(said[0], "nothing ended") { + t.Errorf("an older head heard late said %q", said) + } + forgeSays(t, "", errors.New("nothing serves gitea.gitea_api")) + said = endSupersededChecks(ctx, js, link.TheBuildMachine, + link.PullUpdated{Owner: "novox", Repo: "a", Number: 7, Commit: newHead}, "check-new") + if len(said) != 1 || !strings.Contains(said[0], "nothing ended") { + t.Errorf("a head the forge could not confirm said %q", said) + } + q, err := link.ReadQueue(ctx, js, link.TheBuildMachine) + if err != nil { + t.Fatal(err) + } + for _, id := range []string{"check-new", "check-old-late"} { + if _, found := q.Find(id); !found { + t.Errorf("%s was ended", id) + } + } +} diff --git a/cmd/mesh-controller/template_test.go b/cmd/mesh-controller/template_test.go index edcc66a9..8ad78f36 100644 --- a/cmd/mesh-controller/template_test.go +++ b/cmd/mesh-controller/template_test.go @@ -202,3 +202,40 @@ func TestTheMergeGateResolvesATemplatedModuleToItsInlineTwin(t *testing.T) { t.Fatalf("the templated module resolves to %s, not its inline twin", got) } } + +// A module whose files/ is unpacked onto the machine names its own templates directory (issue 731): `module check` +// reads its templates from there and says of it exactly what it says of the same module with its text inline. +func TestModuleCheckReadsTemplatesFromTheTemplatesDirectoryTheManifestNames(t *testing.T) { + const inline = `{"module":"notifier","version":"1", + "build":{"artifacts":[{"name":"scripts","kind":"archive","from":"files"}]}, + "resources":[{"id":"scripts","type":"archive","path":"/usr/local/lib/notifier","artifact":"scripts"}, + {"id":"conf","type":"file","path":"/etc/notifier/notifier.conf","mode":"0644","content":"volume = 3\n"}]}` + const templated = `{"module":"notifier","version":"1","templates":"templates", + "build":{"artifacts":[{"name":"scripts","kind":"archive","from":"files"}, + {"name":"templates","kind":"archive","from":"templates"}]}, + "resources":[{"id":"scripts","type":"archive","path":"/usr/local/lib/notifier","artifact":"scripts"}, + {"id":"conf","type":"file","path":"/etc/notifier/notifier.conf","mode":"0644","template":"templates/notifier.conf"}]}` + scripts := map[string]string{"files/bin/notify": "#!/bin/sh\n"} + var want bytes.Buffer + if err := moduleCheck([]string{aTemplatedModule(t, t.TempDir(), inline, scripts)}, &want); err != nil { + t.Fatalf("the inline module was refused: %v\n%s", err, want.String()) + } + with := map[string]string{"files/bin/notify": "#!/bin/sh\n", "templates/notifier.conf": "volume = 3\n"} + var got bytes.Buffer + if err := moduleCheck([]string{aTemplatedModule(t, t.TempDir(), templated, with)}, &got); err != nil { + t.Fatalf("a module with its text in templates/ was refused: %v\n%s", err, got.String()) + } + if got.String() != want.String() { + t.Fatalf("module check says otherwise of the templated module:\ninline: %s\ntemplated: %s", want.String(), + got.String()) + } + + // Its text in files/ instead, where the machine unpacks it, is not where its manifest says its templates are. + misplaced := strings.Replace(templated, `"templates/notifier.conf"`, `"files/notifier.conf"`, 1) + got.Reset() + if err := moduleCheck([]string{aTemplatedModule(t, t.TempDir(), misplaced, + map[string]string{"files/notifier.conf": "volume = 3\n"})}, &got); err == nil || + !strings.Contains(got.String(), "files/notifier.conf") { + t.Fatalf("a template outside the templates directory the manifest names passed:\n%s", got.String()) + } +} diff --git a/cmd/mesh-controller/testdata/config-moved/mesh.json b/cmd/mesh-controller/testdata/config-moved/mesh.json new file mode 100644 index 00000000..d65635ae --- /dev/null +++ b/cmd/mesh-controller/testdata/config-moved/mesh.json @@ -0,0 +1,3661 @@ +{ + "layers": [ + { + "module": "baserow", + "node": "anchor", + "values": { + "email": "operator@example.test" + } + }, + { + "module": "bazarr", + "node": "anchor", + "values": { + "pgid": 2000, + "puid": 1001 + } + }, + { + "module": "bookshelf", + "node": "anchor", + "values": { + "downloads": { + "jackett-api": { + "adopt-hosts": [ + "jackett" + ] + }, + "nzbget-api": { + "name": "NZBGet" + }, + "qbittorrent-api": { + "name": "qBitTorrent" + } + }, + "pgid": 1000, + "puid": 1000 + } + }, + { + "module": "claude-code", + "node": "anchor", + "values": { + "role": "the home server \u2014 media (Plex and the *arr apps), home automation, IoT and app databases" + } + }, + { + "module": "hostname", + "node": "anchor", + "values": { + "hostname": "Ace" + } + }, + { + "module": "influxdb", + "node": "anchor", + "values": { + "bucket": "zurag", + "org": "zurag" + } + }, + { + "module": "kometa", + "node": "anchor", + "values": { + "libraries": { + "Formula 1": { + "metadata_files": [ + { + "url": "https://f1-circuits.zurag.be/kometa/formula-1-metadata.yml" + } + ] + } + }, + "pgid": 1000, + "puid": 1000 + } + }, + { + "module": "lidarr", + "node": "anchor", + "values": { + "downloads": { + "jackett-api": { + "adopt-hosts": [ + "indexers.zurag.be", + "jackett" + ] + }, + "nzbget-api": { + "name": "NZBGet" + }, + "qbittorrent-api": { + "name": "qBitTorrent" + } + }, + "pgid": 2000, + "puid": 1001 + } + }, + { + "module": "nfs-server", + "node": "anchor", + "values": { + "grants": "media=shanks:rw", + "shares": "media=/storage/media" + } + }, + { + "module": "nodered", + "node": "anchor", + "values": { + "mqtt": { + "brokers": [ + "2b0aece9c5f3b307" + ] + }, + "timeZone": "Europe/Brussels" + } + }, + { + "module": "nzbget", + "node": "anchor", + "values": { + "max-download-speed": 10, + "options": { + "UMask": "0000" + }, + "pgid": 2000, + "puid": 1001, + "username": "luffy" + } + }, + { + "module": "ombi", + "node": "anchor", + "values": { + "pgid": 2000, + "puid": 1001 + } + }, + { + "module": "plex", + "node": "anchor", + "values": { + "pgid": 1000, + "puid": 1000 + } + }, + { + "module": "qbittorrent", + "node": "anchor", + "values": { + "max-download-speed": 10, + "pgid": 2000, + "puid": 1001, + "username": "luffy" + } + }, + { + "module": "radarr", + "node": "anchor", + "values": { + "downloads": { + "jackett-api": { + "adopt-hosts": [ + "jackett" + ] + }, + "nzbget-api": { + "name": "NZBGet" + }, + "qbittorrent-api": { + "name": "qBitTorrent" + } + }, + "pgid": 2000, + "puid": 1001 + } + }, + { + "module": "searxng", + "node": "anchor", + "values": { + "search": { + "autocomplete": "", + "suspended_times": { + "SearxEngineAccessDenied": 86400, + "SearxEngineCaptcha": 86400, + "SearxEngineTooManyRequests": 3600 + } + }, + "server": { + "base_url": "https://searxng.zurag.be/", + "method": "POST" + } + } + }, + { + "module": "sonarr", + "node": "anchor", + "values": { + "downloads": { + "jackett-api": { + "adopt-hosts": [ + "jackett" + ] + }, + "nzbget-api": { + "name": "NZBGet" + }, + "qbittorrent-api": { + "name": "qBitTorrent" + } + }, + "pgid": 2000, + "puid": 1001 + } + }, + { + "module": "supabase", + "node": "anchor", + "values": { + "pooler": { + "max-client-connections": 100, + "pool-size": 20, + "tenant": "zurag" + } + } + }, + { + "module": "tautulli", + "node": "anchor", + "values": { + "pgid": 2000, + "puid": 1001 + } + }, + { + "module": "unifi", + "node": "anchor", + "values": { + "username": "luffy" + } + }, + { + "module": "claude-code", + "node": "workstation", + "values": { + "role": "the operator's laptop \u2014 SaaS integrations and mesh control; travels, often behind a corporate VPN client" + } + }, + { + "module": "dunst", + "node": "workstation", + "values": { + "font-size": 13, + "width": 500 + } + }, + { + "module": "hostname", + "node": "workstation", + "values": { + "hostname": "g14" + } + }, + { + "module": "power", + "node": "workstation", + "values": { + "handle-lid-switch": "suspend", + "handle-lid-switch-docked": "suspend", + "handle-lid-switch-external-power": "suspend", + "handle-power-key": "suspend" + } + }, + { + "module": "artifact-store-tools", + "node": "", + "values": { + "older-store-names": "registry-api.novox.be" + } + }, + { + "module": "bazarr", + "node": "", + "values": { + "pgid": 1000, + "puid": 1000 + } + }, + { + "module": "betterbird", + "node": "", + "values": { + "accounts": { + "gmail": { + "address": "operator@example.test", + "from-name": "The Operator", + "incoming": { + "host": "imap.gmail.com", + "kind": "imap", + "port": 993, + "security": "tls", + "sign-in": "oauth", + "user": "operator@example.test" + }, + "outgoing": { + "host": "smtp.gmail.com", + "port": 465, + "security": "tls", + "sign-in": "oauth", + "user": "operator@example.test" + } + }, + "novox": { + "address": "operator@example.test", + "from-name": "The Operator", + "incoming": { + "host": "mail.novox.be", + "kind": "imap", + "port": 143, + "security": "starttls", + "sign-in": "password", + "user": "operator@example.test" + }, + "label": "operator@mail.novox.be", + "outgoing": { + "host": "mail.novox.be", + "port": 587, + "security": "starttls", + "sign-in": "password", + "user": "operator@example.test" + } + }, + "novox2": { + "address": "operator@example.test", + "from-name": "The Operator - MH", + "incoming": { + "host": "mail.novox.be", + "kind": "imap", + "port": 993, + "security": "tls", + "sign-in": "password", + "user": "operator@example.test" + }, + "outgoing": { + "host": "mail.novox.be", + "port": 465, + "security": "tls", + "sign-in": "password", + "user": "operator@example.test" + } + } + }, + "prefs": { + "calendar.alarms.show": true, + "calendar.alarms.showmissed": false, + "calendar.item.promptDelete": false, + "calendar.timezone.useSystemTimezone": true, + "calendar.view.dayendhour": 20, + "calendar.view.daystarthour": 9, + "mail.biff.play_sound": false, + "mail.biff.show_alert": false, + "mail.biff.show_tray_icon": false, + "mail.biff.use_system_alert": true, + "mail.chat.enabled": false, + "mail.dark-reader.enabled": false, + "mail.pane_config.dynamic": 2, + "mail.pane_config.multiline_all": true, + "mail.threadpane.listview": 0, + "mail.uidensity": 0, + "mail.uifontsize": 14, + "mail.warn_on_shift_delete": false, + "mailnews.default_view_flags": 1, + "mailnews.message_display.disable_remote_image": true, + "privacy.donottrackheader.enabled": true + } + } + }, + { + "module": "bookshelf", + "node": "", + "values": { + "pgid": 1000, + "puid": 1000 + } + }, + { + "module": "claude-code", + "node": "", + "values": { + "managed_settings": { + "spinnerTipsEnabled": false + } + } + }, + { + "module": "jackett", + "node": "", + "values": { + "pgid": 1000, + "puid": 1000 + } + }, + { + "module": "lab", + "node": "", + "values": { + "forge": "http://novox.internal:20000" + } + }, + { + "module": "lidarr", + "node": "", + "values": { + "pgid": 1000, + "puid": 1000 + } + }, + { + "module": "mailu", + "node": "", + "values": { + "domain": "novox.be", + "proxy-address": "192.0.2.10", + "sitename": "Novox", + "website": "https://novox.be" + } + }, + { + "module": "messenger", + "node": "", + "values": { + "time-zone": "Europe/Brussels" + } + }, + { + "module": "nzbget", + "node": "", + "values": { + "pgid": 1000, + "puid": 1000 + } + }, + { + "module": "ombi", + "node": "", + "values": { + "pgid": 1000, + "puid": 1000 + } + }, + { + "module": "plex", + "node": "", + "values": { + "pgid": 1000, + "puid": 1000 + } + }, + { + "module": "power", + "node": "", + "values": { + "handle-lid-switch": "suspend", + "handle-lid-switch-docked": "ignore", + "handle-lid-switch-external-power": "suspend", + "handle-power-key": "poweroff" + } + }, + { + "module": "qbittorrent", + "node": "", + "values": { + "pgid": 1000, + "puid": 1000 + } + }, + { + "module": "radarr", + "node": "", + "values": { + "pgid": 1000, + "puid": 1000 + } + }, + { + "module": "records", + "node": "", + "values": { + "repository": "novox/hq" + } + }, + { + "module": "sonarr", + "node": "", + "values": { + "pgid": 1000, + "puid": 1000 + } + }, + { + "module": "tautulli", + "node": "", + "values": { + "pgid": 1000, + "puid": 1000 + } + }, + { + "module": "xdg", + "node": "", + "values": { + "desktop-dir": "$HOME/Desktop", + "publicshare-dir": "$HOME/Public", + "videos-dir": "$HOME/Videos" + } + }, + { + "module": "claude-code", + "node": "laptop", + "values": { + "role": "the control node \u2014 the controller, the bus, the store, git hosting, CI/CD and public web apps" + } + }, + { + "module": "desk-channel", + "node": "laptop", + "values": { + "desktop-machines": [ + "g14", + "shanks" + ] + } + }, + { + "module": "hostname", + "node": "laptop", + "values": { + "hostname": "novox-be" + } + }, + { + "module": "kernel", + "node": "laptop", + "values": { + "transparent-huge-pages": "madvise" + } + }, + { + "module": "keycloak", + "node": "laptop", + "values": { + "issuer": "https://keycloak.novox.be/realms/Novox" + } + }, + { + "module": "messenger", + "node": "laptop", + "values": { + "desktop-machines": [ + "g14", + "shanks" + ] + } + }, + { + "module": "zsh", + "node": "laptop", + "values": { + "execute": "withhold" + } + }, + { + "module": "claude-code", + "node": "desktop", + "values": { + "role": "the operator's interactive desktop \u2014 full mesh control" + } + }, + { + "module": "hostname", + "node": "desktop", + "values": { + "hostname": "Shanks" + } + }, + { + "module": "mounts", + "node": "desktop", + "values": { + "shares": "media=/storage/media", + "sources": "recalbox=smb://root@192.0.2.10/Share@/mnt/recalbox" + } + }, + { + "module": "xdg", + "node": "desktop", + "values": { + "desktop-dir": "$HOME/", + "publicshare-dir": "$HOME/", + "videos-dir": "$HOME/" + } + } + ], + "modules": [ + { + "config": { + "email": { + "default": "", + "description": "The email address of the Baserow account the module's tools sign in as; unset, the code falls back to its environment.", + "trusted": true, + "type": "string" + }, + "host": { + "default": "${bound:route:name}", + "description": "The Host header the module's tools send to Baserow, a host name; the file's default is the name the module's route was given.", + "trusted": true, + "type": "string" + }, + "url": { + "default": "", + "description": "The address the module's tools reach Baserow at, as a URL; unset, the code falls back to its environment, else this machine's Baserow on port 80.", + "trusted": true, + "type": "string" + } + }, + "module": "baserow", + "resources": [ + { + "id": "mesh-state", + "mode": "0700", + "place": "mesh", + "type": "directory" + }, + { + "id": "state", + "mode": "0700", + "place": ".", + "type": "directory" + }, + { + "id": "data", + "mode": "0755", + "type": "directory" + }, + { + "content": "DATABASE_HOST=STAND-IN\nDATABASE_PORT=STAND-IN\nDATABASE_NAME=STAND-IN\nDATABASE_USER=STAND-IN\nDATABASE_PASSWORD_FILE=/run/secrets/database\nDISABLE_EMBEDDED_PSQL=true\nBASEROW_PUBLIC_URL=https://STAND-IN\n", + "id": "server-env", + "mode": "0600", + "path": "${dir:state}/server.env", + "type": "file" + }, + { + "content": "{\n \"password\": \"STAND-IN\",\n \"host\": \"STAND-IN\"\n}\n", + "id": "runtime-config", + "merge": "json", + "mode": "0600", + "path": "${dir:mesh-state}/config.json", + "protected": [ + "password" + ], + "takes-config": [ + "host", + "email", + "url" + ], + "type": "file" + } + ], + "version": "1" + }, + { + "config": { + "agents": { + "default": {}, + "description": "The agents offered to every session on this machine, by name, each a text as one line or a list of its lines; empty adds none.", + "trusted": true, + "type": "object" + }, + "commands": { + "default": {}, + "description": "The slash commands offered to every session on this machine, by name, each a text as one line or a list of its lines; empty adds none.", + "trusted": true, + "type": "object" + }, + "instructions": { + "default": {}, + "description": "The instructions every session on this machine reads, by name, each a text as one line or a list of its lines; empty adds none.", + "trusted": true, + "type": "object" + }, + "managed_settings": { + "default": {}, + "description": "Keys of Claude Code's managed settings every session on this machine obeys, such as spinnerTipsEnabled, laid over the mesh's own; empty adds none. Falls back to agent-managed-settings, the mesh's or this node's.", + "predefined": "agent-managed-settings", + "trusted": true, + "type": "object" + }, + "mcp_servers": { + "default": {}, + "description": "The tool servers every session on this machine is given beside the mesh's own, by name, each as Claude Code's managed MCP configuration takes it; empty adds none.", + "trusted": true, + "type": "object" + }, + "output_styles": { + "default": {}, + "description": "The output styles offered to every session on this machine, by name, each a text as one line or a list of its lines; empty adds none.", + "trusted": true, + "type": "object" + }, + "role": { + "default": "", + "description": "What this machine is for, in a sentence, written into the instructions every Claude Code session on it reads; empty says nothing.", + "trusted": true, + "type": "string" + }, + "skills": { + "default": {}, + "description": "The skills offered to every session on this machine, by name, each its files by path, a text as one line or a list of its lines; a skill without its SKILL.md is left out; empty adds none.", + "trusted": true, + "type": "object" + } + }, + "module": "claude-code", + "resources": [ + { + "id": "managed", + "mode": "0755", + "path": "/etc/claude-code", + "type": "directory" + }, + { + "content": "#!/bin/sh\n# The mesh's (module claude-code, novox/hq ADR 0266): start the agent as the account agents run as on this\n# machine. Written whole at every push: an edit here is overwritten.\nagent='STAND-IN'\nif [ \"$(id -un)\" = \"$agent\" ]; then\n\texec claude \"$@\"\nfi\n# Another account, the operator's, becomes the agent's through its own sudo: the person is the authority. The\n# operator's override of the guard travels only when it is set in this shell, as it would reach claude.\nif [ -n \"$MESH_GUARD_OVERRIDE\" ]; then\n\texec sudo -iu \"$agent\" env MESH_GUARD_OVERRIDE=\"$MESH_GUARD_OVERRIDE\" claude \"$@\"\nfi\nexec sudo -iu \"$agent\" claude \"$@\"\n", + "id": "start", + "mode": "0755", + "path": "/usr/local/bin/claude-agent", + "type": "file" + }, + { + "content": "STAND-IN\n", + "id": "agent-account-name", + "mode": "0644", + "path": "/etc/claude-code/agent-account", + "type": "file" + }, + { + "id": "agent-home", + "mode": "0700", + "path": "STAND-IN/.claude", + "type": "directory" + }, + { + "id": "state", + "mode": "0700", + "place": ".", + "type": "directory" + }, + { + "content": "{\n \"node\": \"STAND-IN\",\n \"console\": \"http://127.0.0.1:STAND-IN/mcp\"\n}\n", + "id": "facts", + "mode": "0600", + "path": "${dir:state}/facts.json", + "type": "file" + }, + { + "content": "{\n \"role\": \"\",\n \"mcp_servers\": {},\n \"managed_settings\": {},\n \"instructions\": {},\n \"output_styles\": {},\n \"skills\": {},\n \"commands\": {},\n \"agents\": {}\n}\n", + "id": "settings", + "merge": "json", + "mode": "0600", + "path": "${dir:state}/settings.json", + "takes-config": [ + "role", + "mcp_servers", + "managed_settings", + "instructions", + "output_styles", + "skills", + "commands", + "agents" + ], + "type": "file" + } + ], + "version": "1" + }, + { + "config": { + "url": { + "default": "", + "description": "The address the module's tools reach Grafana at, as a URL; unset, the code falls back to its environment, else this machine's Grafana on its port.", + "trusted": true, + "type": "string" + }, + "user": { + "default": "admin", + "description": "The Grafana account the module's tools sign in as with the vault's password; the file's default is admin.", + "trusted": true, + "type": "string" + } + }, + "module": "grafana", + "resources": [ + { + "id": "mesh-state", + "mode": "0700", + "place": "mesh", + "type": "directory" + }, + { + "id": "state", + "mode": "0700", + "place": ".", + "type": "directory" + }, + { + "id": "data", + "mode": "0700", + "type": "directory" + }, + { + "content": "STAND-IN", + "id": "admin-secret", + "mode": "0400", + "path": "${dir:state}/admin.secret", + "type": "file" + }, + { + "content": "STAND-IN", + "id": "oidc-secret", + "mode": "0400", + "path": "${dir:state}/oidc-client.secret", + "type": "file" + }, + { + "content": "GF_SERVER_ROOT_URL=https://STAND-IN\nGF_AUTH_GENERIC_OAUTH_ENABLED=true\nGF_AUTH_GENERIC_OAUTH_NAME=Keycloak\nGF_AUTH_GENERIC_OAUTH_CLIENT_ID=STAND-IN\nGF_AUTH_GENERIC_OAUTH_CLIENT_SECRET__FILE=/run/secrets/oidc-client\nGF_AUTH_GENERIC_OAUTH_SCOPES=openid email profile roles\nGF_AUTH_GENERIC_OAUTH_AUTH_URL=STAND-INSTAND-IN\nGF_AUTH_GENERIC_OAUTH_TOKEN_URL=STAND-INSTAND-IN\nGF_AUTH_GENERIC_OAUTH_API_URL=STAND-INSTAND-IN\nGF_AUTH_GENERIC_OAUTH_ROLE_ATTRIBUTE_PATH=contains(roles[*], 'admin') && 'Admin' || contains(realm_access.roles[*], 'admin') && 'Admin' || 'Viewer'\nGF_AUTH_GENERIC_OAUTH_USE_PKCE=true\nGF_AUTH_GENERIC_OAUTH_ALLOW_SIGN_UP=true\nGF_AUTH_GENERIC_OAUTH_ALLOW_ASSIGN_GRAFANA_ADMIN=true\n", + "id": "oidc-env", + "mode": "0644", + "path": "${dir:state}/oidc.env", + "type": "file" + }, + { + "id": "influxdb-datasources", + "mode": "0755", + "path": "${dir:state}/datasources", + "type": "directory" + }, + { + "id": "influxdb-credential", + "mode": "0700", + "path": "${dir:state}/influxdb", + "type": "directory" + }, + { + "content": "STAND-IN", + "id": "influxdb-secret", + "mode": "0400", + "path": "${dir:influxdb-credential}/influxdb-api", + "type": "file" + }, + { + "content": "apiVersion: 1\n# Written by the mesh from grafana's influxdb-api binding, only while it is bound (novox/hq ADR 0330):\n# without influxdb, grafana runs with no mesh data source. grafana reads it at start. Its own name and\n# uid, so a data source somebody made in the UI is never overwritten, and read-only in the UI because\n# the mesh resets it. The password is read from the file the mesh delivers, never written here.\ndatasources:\n - name: InfluxDB (mesh)\n uid: mesh-influxdb-api\n type: influxdb\n access: proxy\n url: STAND-IN://STAND-IN:STAND-IN\n user: STAND-IN\n isDefault: false\n editable: false\n jsonData:\n dbName: STAND-IN\n httpMode: POST\n secureJsonData:\n password: $__file{/run/secrets/influxdb/influxdb-api}\n", + "id": "influxdb-datasource", + "mode": "0644", + "path": "${dir:influxdb-datasources}/mesh-influxdb.yaml", + "type": "file" + }, + { + "content": "{\n \"user\": \"admin\",\n \"password\": \"STAND-IN\"\n}\n", + "id": "runtime-config", + "merge": "json", + "mode": "0600", + "path": "${dir:mesh-state}/config.json", + "protected": [ + "password", + "token" + ], + "takes-config": [ + "user", + "url" + ], + "type": "file" + } + ], + "version": "1" + }, + { + "config": { + "url": { + "default": "", + "description": "The address the module's tools reach Home Assistant at, as a URL; unset, the code falls back to its environment, else this machine's Home Assistant on its port.", + "trusted": true, + "type": "string" + } + }, + "module": "home-assistant", + "resources": [ + { + "id": "mesh-state", + "mode": "0700", + "place": "mesh", + "type": "directory" + }, + { + "id": "state", + "mode": "0700", + "place": ".", + "type": "directory" + }, + { + "id": "config", + "mode": "0700", + "type": "directory" + }, + { + "id": "written", + "mode": "0700", + "type": "directory" + }, + { + "content": "{}\n", + "id": "runtime-config", + "merge": "json", + "mode": "0600", + "path": "${dir:mesh-state}/config.json", + "takes-config": [ + "url" + ], + "type": "file" + }, + { + "content": "MESH_HOMEASSISTANT_URL=http://127.0.0.1:STAND-IN\nMESH_HOMEASSISTANT_TOKEN_FILE=${dir:mesh-state}/token\nMESH_PROVISIONS_DIR=${dir:state}\nMESH_WRITTEN_DIR=${dir:written}\n", + "id": "provisions-env", + "mode": "0600", + "path": "${dir:state}/provisions.env", + "type": "file" + } + ], + "version": "1" + }, + { + "claims": [ + { + "name": "node-hostname", + "scope": "node", + "serves": [ + "entries", + "add", + "remove" + ] + } + ], + "config": { + "hostname": { + "description": "The machine's host name, written to /etc/hostname: the operator's name for this machine, with no default (novox/hq ADR 0112).", + "required": true, + "trusted": true, + "type": "string" + } + }, + "module": "hostname", + "resources": [ + { + "content": "# The machine's own names (module hostname, novox/hq ADR 0199, ADR 0223). Every line outside this\n# block is the operator's: kept across every push, changed through the node-hostname verbs add and\n# remove, and given back when this module goes. The mesh's names are not here: the mesh's resolver\n# answers them.\n127.0.0.1\tlocalhost\n::1\tlocalhost\n127.0.1.1\tSTAND-IN\n", + "id": "own", + "mode": "0644", + "path": "/etc/hosts", + "type": "file" + }, + { + "content": "${config:hostname}\n", + "id": "name", + "mode": "0644", + "path": "/etc/hostname", + "type": "file" + } + ], + "version": "1" + }, + { + "config": { + "url": { + "default": "", + "description": "The address the module's tools reach Icecast at, as a URL; unset, the code falls back to its environment, else this machine's Icecast on its port.", + "trusted": true, + "type": "string" + }, + "user": { + "default": "", + "description": "The Icecast admin account the module's tools sign in as; unset, the code falls back to its environment, else admin.", + "trusted": true, + "type": "string" + } + }, + "module": "icecast", + "resources": [ + { + "id": "mesh-state", + "mode": "0700", + "place": "mesh", + "type": "directory" + }, + { + "id": "state", + "mode": "0700", + "place": ".", + "type": "directory" + }, + { + "id": "logs", + "mode": "0700", + "type": "directory" + }, + { + "content": "\n \n Earth\n icemaster@localhost\n \n 100\n 2\n 524288\n 30\n 15\n 10\n 1\n 65535\n \n \n STAND-IN\n STAND-IN\n admin\n STAND-IN\n \n \n localhost\n \n 8000\n \n \n
\n \n 1\n \n /usr/share/icecast\n /var/log/icecast\n /usr/share/icecast/web\n /usr/share/icecast/admin\n \n \n \n access.log\n error.log\n 3\n 10000\n \n \n 0\n \n \n icecast\n icecast\n \n \n\n", + "id": "server-conf", + "mode": "0600", + "path": "${dir:state}/icecast.xml", + "type": "file" + }, + { + "content": "{}\n", + "id": "runtime-config", + "merge": "json", + "mode": "0600", + "path": "${dir:mesh-state}/config.json", + "takes-config": [ + "url", + "user" + ], + "type": "file" + } + ], + "version": "1" + }, + { + "config": { + "bucket": { + "default": "default", + "description": "The bucket influxdb-api names to its consumers and the module's tools write to; default is the one influxdb makes at its first start (server.env), and a machine whose server was set up with another sets its own (novox/hq issue 696, ADR 0348 clause 11).", + "trusted": true, + "type": "string" + }, + "org": { + "default": "mesh", + "description": "The organisation influxdb-api names to its consumers and the module's tools write to; mesh is the one influxdb makes at its first start (server.env), and a machine whose server was set up with another sets its own (novox/hq issue 696, ADR 0348 clause 11).", + "trusted": true, + "type": "string" + }, + "url": { + "default": "", + "description": "The address the module's tools reach influxdb at, as a URL; empty is this machine's influxdb on its port.", + "trusted": true, + "type": "string" + } + }, + "module": "influxdb", + "resources": [ + { + "id": "mesh-state", + "mode": "0700", + "place": "mesh", + "type": "directory" + }, + { + "id": "state", + "mode": "0700", + "place": ".", + "type": "directory" + }, + { + "id": "data", + "mode": "0700", + "type": "directory" + }, + { + "id": "config", + "mode": "0700", + "type": "directory" + }, + { + "id": "grants", + "mode": "0700", + "type": "directory" + }, + { + "content": "DOCKER_INFLUXDB_INIT_MODE=setup\nDOCKER_INFLUXDB_INIT_USERNAME=admin\nDOCKER_INFLUXDB_INIT_PASSWORD_FILE=/run/secrets/admin\nDOCKER_INFLUXDB_INIT_ADMIN_TOKEN_FILE=/run/secrets/admin-token\nDOCKER_INFLUXDB_INIT_ORG=mesh\nDOCKER_INFLUXDB_INIT_BUCKET=default\n", + "id": "server-env", + "mode": "0600", + "path": "${dir:state}/server.env", + "type": "file" + }, + { + "content": "{}\n", + "id": "runtime-config", + "merge": "json", + "mode": "0600", + "path": "${dir:mesh-state}/config.json", + "takes-config": [ + "bucket", + "org", + "url" + ], + "trusted": true, + "type": "file" + } + ], + "serves-checked": { + "influxdb-api": { + "bucket": "${config:bucket}", + "org": "${config:org}", + "port": 8086, + "scheme": "http" + } + }, + "version": "1" + }, + { + "config": { + "url": { + "default": "", + "description": "The address the module's tools reach Letta at, as a URL; unset, the code falls back to its environment, else this machine's Letta on port 8283.", + "trusted": true, + "type": "string" + } + }, + "module": "letta", + "resources": [ + { + "id": "mesh-state", + "mode": "0700", + "place": "mesh", + "type": "directory" + }, + { + "id": "state", + "mode": "0700", + "place": ".", + "type": "directory" + }, + { + "content": "LETTA_PG_URI=postgresql://STAND-IN@STAND-IN:STAND-IN/STAND-IN\nPGPASSFILE=/run/secrets/pgpass\nLETTA_SERVER_PASSWORD=STAND-IN\nOPENAI_API_KEY=STAND-IN\nSECURE=true\nTZ=Europe/Brussels\n", + "id": "server-env", + "mode": "0600", + "path": "${dir:state}/server.env", + "type": "file" + }, + { + "content": "*:*:*:STAND-IN:STAND-IN\n", + "id": "pgpass", + "mode": "0600", + "path": "${dir:state}/pgpass", + "type": "file" + }, + { + "content": "#!/bin/sh\n# Generated by the mesh. Do not edit: module letta writes this file and replaces it at every push.\n#\n# letta 0.6.8 prints secrets it is given to its log (novox/hq issue 268):\n# letta/server/rest_api/app.py prints its server password when it starts in secure mode;\n# startup.sh, alembic/env.py and letta/server/server.py print LETTA_PG_URI whole.\n# The URI carries no password (libpq reads it from PGPASSFILE), and the one print of the server\n# password is rewritten before the server starts. Either one failing refuses the start: a letta\n# that does not start says why here, and one that leaks says nothing.\nset -e\napp=/app/letta/server/rest_api/app.py\nsed -i 's/Using secure mode with password: {random_password}/Using secure mode (the password is not printed)/' \"$app\"\nif grep -q 'print(.*random_password' \"$app\"; then\n echo \"letta: $app still prints the server password; not starting (novox/hq issue 268)\" >&2\n exit 1\nfi\ncase \"$LETTA_PG_URI\" in\n *://*:*@*)\n echo \"letta: LETTA_PG_URI carries a password, and letta prints that URI; not starting (novox/hq issue 268)\" >&2\n exit 1\n ;;\nesac\nexec ./letta/server/startup.sh\n", + "id": "start", + "mode": "0644", + "path": "${dir:state}/start.sh", + "type": "file" + }, + { + "content": "{\n \"password\": \"STAND-IN\"\n}\n", + "id": "runtime-config", + "merge": "json", + "mode": "0600", + "path": "${dir:mesh-state}/config.json", + "protected": [ + "password" + ], + "takes-config": [ + "url" + ], + "type": "file" + } + ], + "version": "1" + }, + { + "config": { + "brand": { + "default": "Element", + "description": "Element's brand setting of config.json: the name Element shows itself as.", + "trusted": true, + "type": "string" + }, + "bug_report_endpoint_url": { + "default": "", + "description": "Element's bug_report_endpoint_url setting of config.json: where Element sends a bug report, a URL; its default here is empty, and the file's own value (Element's, naming outside servers) stays in the file and is never replaced unless this is set.", + "trusted": true, + "type": "string" + }, + "default_server_config": { + "default": { + "m.homeserver": { + "base_url": "https://${bound:route:name-homeserver}" + }, + "m.identity_server": { + "base_url": "https://vector.im" + } + }, + "description": "Element's default_server_config setting of config.json: the homeserver and identity server Element uses by default, by URL.", + "trusted": true, + "type": "object" + }, + "default_server_name": { + "default": "${bound:route:name-homeserver}", + "description": "Element's default_server_name setting of config.json: the homeserver Element signs in to by default, by name; the file's default is the module's own route.", + "trusted": true, + "type": "string" + }, + "element_call": { + "default": { + "url": "https://call.element.dev" + }, + "description": "Element's element_call setting of config.json: where Element starts a call with Element Call, a URL.", + "trusted": true, + "type": "object" + }, + "enable_presence_by_hs_url": { + "default": { + "https://matrix-client.matrix.org": false, + "https://matrix.org": false + }, + "description": "Element's enable_presence_by_hs_url setting of config.json: whether Element shows presence, per homeserver URL.", + "trusted": true, + "type": "object" + }, + "features": { + "default": { + "feature_rust_crypto": true, + "feature_video_rooms": true + }, + "description": "Element's features setting of config.json: Element's feature switches, each on or off.", + "trusted": true, + "type": "object" + }, + "integrations_rest_url": { + "default": "https://scalar.vector.im/api", + "description": "Element's integrations_rest_url setting of config.json: the integration manager's API, a URL.", + "trusted": true, + "type": "string" + }, + "integrations_ui_url": { + "default": "https://scalar.vector.im/", + "description": "Element's integrations_ui_url setting of config.json: the integration manager's page, a URL.", + "trusted": true, + "type": "string" + }, + "integrations_widgets_urls": { + "default": [ + "https://scalar.vector.im/_matrix/integrations/v1", + "https://scalar.vector.im/api", + "https://scalar-staging.vector.im/_matrix/integrations/v1", + "https://scalar-staging.vector.im/api", + "https://scalar-staging.riot.im/scalar/api" + ], + "description": "Element's integrations_widgets_urls setting of config.json: the URLs whose widgets Element trusts as the integration manager's.", + "trusted": true, + "type": "list" + }, + "room_directory": { + "default": {}, + "description": "Element's room_directory setting of config.json: the servers whose room directories Element offers; its default here is empty, and the file's own value (Element's, naming outside servers) stays in the file and is never replaced unless this is set.", + "trusted": true, + "type": "object" + }, + "show_labs_settings": { + "default": true, + "description": "Element's show_labs_settings setting of config.json: whether Element shows its Labs settings.", + "trusted": true, + "type": "boolean" + }, + "terms_and_conditions_links": { + "default": [], + "description": "Element's terms_and_conditions_links setting of config.json: the links Element shows as its terms, each a URL and its text; its default here is empty, and the file's own value (Element's, naming outside servers) stays in the file and is never replaced unless this is set.", + "trusted": true, + "type": "list" + }, + "uisi_autorageshake_app": { + "default": "element-auto-uisi", + "description": "Element's uisi_autorageshake_app setting of config.json: the app name Element files an automatic report of an undecryptable message under.", + "trusted": true, + "type": "string" + } + }, + "module": "matrix", + "resources": [ + { + "id": "state", + "mode": "0700", + "place": ".", + "type": "directory" + }, + { + "id": "db", + "mode": "0700", + "type": "directory" + }, + { + "content": "# Written by the mesh (modules/matrix). Conduit reads this file (CONDUIT_CONFIG); nothing comes\n# from the environment. server_name is the homeserver's permanent identity: every user id, room id\n# and signature in the database carries it, so it is the name this module is served under\n# (STAND-IN) and never changes once a database exists.\n[global]\nserver_name = \"STAND-IN\"\ndatabase_backend = \"rocksdb\"\ndatabase_path = \"/var/lib/matrix-conduit/\"\naddress = \"0.0.0.0\"\nport = 6167\nmax_request_size = 20000000\nallow_registration = false\nallow_federation = true\nallow_check_for_updates = true\ntrusted_servers = [\"matrix.org\"]\n", + "id": "conduit-conf", + "mode": "0644", + "path": "${dir:state}/conduit.toml", + "type": "file" + }, + { + "content": "{\n \"default_server_name\": \"STAND-IN\",\n \"default_server_config\": {\n \"m.homeserver\": {\n \"base_url\": \"https://STAND-IN\"\n },\n \"m.identity_server\": {\n \"base_url\": \"https://vector.im\"\n }\n },\n \"brand\": \"Element\",\n \"integrations_ui_url\": \"https://scalar.vector.im/\",\n \"integrations_rest_url\": \"https://scalar.vector.im/api\",\n \"integrations_widgets_urls\": [\n \"https://scalar.vector.im/_matrix/integrations/v1\",\n \"https://scalar.vector.im/api\",\n \"https://scalar-staging.vector.im/_matrix/integrations/v1\",\n \"https://scalar-staging.vector.im/api\",\n \"https://scalar-staging.riot.im/scalar/api\"\n ],\n \"bug_report_endpoint_url\": \"https://element.io/bugreports/submit\",\n \"uisi_autorageshake_app\": \"element-auto-uisi\",\n \"show_labs_settings\": true,\n \"room_directory\": {\n \"servers\": [\n \"STAND-IN\",\n \"matrix.org\",\n \"gitter.im\",\n \"libera.chat\"\n ]\n },\n \"enable_presence_by_hs_url\": {\n \"https://matrix.org\": false,\n \"https://matrix-client.matrix.org\": false\n },\n \"terms_and_conditions_links\": [\n {\n \"url\": \"https://element.io/privacy\",\n \"text\": \"Privacy Policy\"\n },\n {\n \"url\": \"https://element.io/cookie-policy\",\n \"text\": \"Cookie Policy\"\n }\n ],\n \"features\": {\n \"feature_video_rooms\": true,\n \"feature_rust_crypto\": true\n },\n \"element_call\": {\n \"url\": \"https://call.element.dev\"\n }\n}\n", + "id": "element-conf", + "merge": "json", + "mode": "0644", + "path": "${dir:state}/element.json", + "takes-config": [ + "default_server_name", + "default_server_config", + "brand", + "integrations_ui_url", + "integrations_rest_url", + "integrations_widgets_urls", + "bug_report_endpoint_url", + "uisi_autorageshake_app", + "show_labs_settings", + "room_directory", + "enable_presence_by_hs_url", + "terms_and_conditions_links", + "features", + "element_call" + ], + "type": "file" + } + ], + "version": "1" + }, + { + "claims": [ + { + "name": "node-nfs-server", + "scope": "node", + "serves": [ + "exports", + "clients", + "test", + "reload", + "adopt" + ] + } + ], + "config": { + "grants": { + "default": "none", + "description": "Which nodes may mount which share, and how: share=node:rw|ro[,node:rw|ro], entries separated by spaces, or none to export to no node whatever a node wants (novox/hq ADR 0263).", + "trusted": true, + "type": "string" + }, + "shares": { + "description": "The folders this machine exports, as space-separated name=path; the operator's, with no default (novox/hq ADR 0263).", + "required": true, + "trusted": true, + "type": "string" + } + }, + "module": "nfs-server", + "resources": [ + { + "content": "# Written by the mesh (module nfs-server, novox/hq ADR 0263). Replaced on every push; a drop-in of\n# the operator's that sorts after this one overrides it, and is theirs.\n#\n# NFS version 4 only: a client needs port 2049 and nothing else, so the module opens nothing more\n# than that, to the private network. Version 3 needs rpcbind and mountd, on ports the mesh does not open.\n[nfsd]\nvers2=n\nvers3=n\nvers4=y\nvers4.0=n\nvers4.1=y\nvers4.2=y\n", + "id": "nfs-conf", + "mode": "0644", + "path": "/etc/nfs.conf.d/50-mesh.conf", + "type": "file" + }, + { + "id": "run-dir", + "mode": "0755", + "path": "/run/nfs-server", + "type": "directory" + }, + { + "id": "wants-dir-parent", + "mode": "0755", + "path": "/var/lib/nfs-server", + "type": "directory" + }, + { + "id": "wants-dir", + "mode": "0755", + "path": "/var/lib/nfs-server/from-bus", + "type": "directory" + }, + { + "id": "config-dir", + "mode": "0755", + "path": "/etc/nfs-server", + "type": "directory" + }, + { + "content": "# Written by the mesh (module nfs-server, novox/hq ADR 0263) from this machine's assignment.\n# Replaced on every push; change the `shares` setting, never this file.\n#\n# The shares: name=folder, or name=folder:ro, one share per folder. The grants: share=node:rw|ro[,\u2026],\n# which nodes may mount each and how. The module's process exports a share to a node's private address\n# only when it is granted here and that node's mounts module wants it, every client mapped to the\n# folder's owner, and only to addresses inside the range below.\nshares=${config:shares}\ngrants=${config:grants}\nrange=STAND-IN\n", + "id": "config", + "mode": "0644", + "path": "/etc/nfs-server/shares.conf", + "type": "file" + } + ], + "version": "1" + }, + { + "config": { + "debug-max-length": { + "default": 1000, + "description": "Node-RED's debugMaxLength setting: the most characters a debug message shows, a count.", + "trusted": true, + "type": "number" + }, + "diagnostics": { + "default": { + "enabled": true, + "ui": true + }, + "description": "Node-RED's diagnostics setting: whether its diagnostics report is on, and shown in the editor.", + "trusted": true, + "type": "object" + }, + "editor-theme": { + "default": { + "projects": { + "enabled": false + } + }, + "description": "Node-RED's editorTheme setting: the editor's look and features, projects among them.", + "trusted": true, + "type": "object" + }, + "export-global-context-keys": { + "default": false, + "description": "Node-RED's exportGlobalContextKeys setting: whether a function node may list the global context's keys.", + "trusted": true, + "type": "boolean" + }, + "external-modules": { + "default": {}, + "description": "Node-RED's externalModules setting: which modules the editor may install, and from where.", + "trusted": true, + "type": "object" + }, + "flow-file": { + "default": "flows.json", + "description": "Node-RED's flowFile setting: the file in the user directory the flows are kept in.", + "trusted": true, + "type": "string" + }, + "flow-file-pretty": { + "default": true, + "description": "Node-RED's flowFilePretty setting: whether the flows file is written indented.", + "trusted": true, + "type": "boolean" + }, + "function-external-modules": { + "default": true, + "description": "Node-RED's functionExternalModules setting: whether a function node may load modules of its own.", + "trusted": true, + "type": "boolean" + }, + "logging": { + "default": { + "console": { + "audit": false, + "level": "info", + "metrics": false + } + }, + "description": "Node-RED's logging setting: the console log's level, and whether metrics and audit events are logged.", + "trusted": true, + "type": "object" + }, + "mqtt": { + "default": {}, + "description": "The broker nodes the module's MQTT step keeps pointed at the mesh's broker, as {\"brokers\": [their ids]}; empty, none.", + "trusted": true, + "type": "object" + }, + "mqtt-reconnect-time": { + "default": 15000, + "description": "Node-RED's mqttReconnectTime setting: how long an MQTT node waits before reconnecting, in milliseconds.", + "trusted": true, + "type": "number" + }, + "runtime-state": { + "default": { + "enabled": false, + "ui": false + }, + "description": "Node-RED's runtimeState setting: whether flows may be started and stopped from the editor.", + "trusted": true, + "type": "object" + }, + "serial-reconnect-time": { + "default": 15000, + "description": "Node-RED's serialReconnectTime setting: how long a serial node waits before reconnecting, in milliseconds.", + "trusted": true, + "type": "number" + }, + "time-zone": { + "default": "", + "description": "The time zone the flows schedule and format in, an IANA name such as Europe/Brussels; unset, the container's UTC.", + "trusted": true, + "type": "string" + } + }, + "module": "nodered", + "resources": [ + { + "id": "mesh-state", + "mode": "0700", + "place": "mesh", + "type": "directory" + }, + { + "id": "state", + "mode": "0700", + "place": ".", + "type": "directory" + }, + { + "id": "data", + "mode": "0700", + "type": "directory" + }, + { + "id": "written", + "mode": "0700", + "type": "directory" + }, + { + "content": "// Node-RED's settings, written by the mesh from the nodered module. What an assignment may change\n// is settings.json beside this file (merged key by key); the credentials are the mesh's secrets and\n// reach Node-RED only through this file. The flows' own credentials stay encrypted in the user\n// directory under the key Node-RED keeps there (.config.runtime.json), which is data, not this.\nconst fs = require(\"fs\");\nconst path = require(\"path\");\nconst crypto = require(\"crypto\");\n\nconst ADMIN_PASSWORD = \"STAND-IN\";\nconst API_TOKEN = \"STAND-IN\";\nconst ADMIN = { username: \"admin\", permissions: \"*\" };\n\nconst settings = JSON.parse(fs.readFileSync(path.join(__dirname, \"settings.json\"), \"utf8\"));\n// The mesh's keys, not Node-RED's: endpoints lands in every merged file; timeZone is the\n// assignment's way to set the zone flows schedule and format in; mqtt names the broker nodes the\n// module's MQTT step keeps pointed at the mesh's broker; topics is what nodered asks the broker for.\nif (settings.timeZone) process.env.TZ = settings.timeZone;\ndelete settings.timeZone;\ndelete settings.endpoints;\ndelete settings.mqtt;\ndelete settings.topics;\n\nfunction same(a, b) {\n const x = crypto.createHash(\"sha256\").update(String(a)).digest();\n const y = crypto.createHash(\"sha256\").update(String(b)).digest();\n return crypto.timingSafeEqual(x, y);\n}\n\n// The admin secret is a password, or, accepted from an existing install, the bcrypt hash its\n// settings held, so the password people already use keeps working.\nfunction passwordMatches(given) {\n if (/^\\$2[aby]\\$\\d\\d\\$/.test(ADMIN_PASSWORD)) return require(\"bcryptjs\").compare(String(given), ADMIN_PASSWORD);\n return Promise.resolve(same(given, ADMIN_PASSWORD));\n}\n\nmodule.exports = Object.assign(settings, {\n uiPort: 1880,\n adminAuth: {\n type: \"credentials\",\n users: (username) => Promise.resolve(username === ADMIN.username ? ADMIN : null),\n authenticate: (username, password) =>\n username === ADMIN.username\n ? passwordMatches(password).then((ok) => (ok ? ADMIN : null))\n : Promise.resolve(null),\n // The module's own tools call the admin API with this bearer token.\n tokens: (token) => Promise.resolve(same(token, API_TOKEN) ? { username: \"mesh\", permissions: \"*\" } : null),\n },\n});\n", + "id": "settings-code", + "mode": "0600", + "path": "${dir:state}/settings.js", + "type": "file" + }, + { + "content": "{\n \"flowFile\": \"flows.json\",\n \"flowFilePretty\": true,\n \"diagnostics\": { \"enabled\": true, \"ui\": true },\n \"runtimeState\": { \"enabled\": false, \"ui\": false },\n \"logging\": { \"console\": { \"level\": \"info\", \"metrics\": false, \"audit\": false } },\n \"exportGlobalContextKeys\": false,\n \"externalModules\": {},\n \"editorTheme\": { \"projects\": { \"enabled\": false } },\n \"functionExternalModules\": true,\n \"debugMaxLength\": 1000,\n \"mqttReconnectTime\": 15000,\n \"serialReconnectTime\": 15000\n}\n", + "id": "settings", + "merge": "json", + "mode": "0600", + "path": "${dir:state}/settings.json", + "takes-config": [ + { + "as": "flowFile", + "key": "flow-file" + }, + { + "as": "flowFilePretty", + "key": "flow-file-pretty" + }, + "diagnostics", + { + "as": "runtimeState", + "key": "runtime-state" + }, + "logging", + { + "as": "exportGlobalContextKeys", + "key": "export-global-context-keys" + }, + { + "as": "externalModules", + "key": "external-modules" + }, + { + "as": "editorTheme", + "key": "editor-theme" + }, + { + "as": "functionExternalModules", + "key": "function-external-modules" + }, + { + "as": "debugMaxLength", + "key": "debug-max-length" + }, + { + "as": "mqttReconnectTime", + "key": "mqtt-reconnect-time" + }, + { + "as": "serialReconnectTime", + "key": "serial-reconnect-time" + }, + "mqtt", + { + "as": "timeZone", + "key": "time-zone" + } + ], + "type": "file" + }, + { + "content": "{\n \"token\": \"STAND-IN\"\n}\n", + "id": "runtime-config", + "mode": "0600", + "path": "${dir:mesh-state}/config.json", + "type": "file" + }, + { + "content": "MESH_NODERED_URL=http://127.0.0.1:STAND-IN\nMESH_NODERED_CONFIG_FILE=${dir:mesh-state}/config.json\nMESH_PROVISIONS_DIR=${dir:state}\nMESH_WRITTEN_DIR=${dir:written}\n", + "id": "mqtt-env", + "mode": "0600", + "path": "${dir:state}/mqtt.env", + "type": "file" + } + ], + "version": "1" + }, + { + "claims": [ + { + "name": "node-backup", + "scope": "node", + "serves": [ + "backed-up", + "now", + "restore" + ] + } + ], + "config": { + "keep-daily": { + "default": 14, + "description": "How many nightly restore points are kept, a count from 1 to 9999: two weeks undo a mistake noticed within a fortnight. Read by the night, which thins nothing while it does not read (novox/hq issue 416); lowering it deletes the restore points beyond the new count at the next night, and that cannot be undone.", + "maximum": 9999, + "minimum": 1, + "trusted": true, + "type": "number" + }, + "keep-hand-taken-days": { + "default": 7, + "description": "How many days a restore point taken by hand (node-backup.now, or the mesh's before a module moves) is kept at least, a count from 1 to 9999; it is kept until a later good night of its module exists, and this many days in any case, so the next thinning never takes the one taken against a risky change (novox/hq issue 453). Lowering it lets go of the older ones taken by hand at the next thinning, and that cannot be undone.", + "maximum": 9999, + "minimum": 1, + "trusted": true, + "type": "number" + }, + "keep-monthly": { + "default": 6, + "description": "How many monthly restore points are kept behind the weeklies, a count from 1 to 9999, refused by the night like keep-daily; lowering it deletes the restore points beyond the new count at the next night, and that cannot be undone.", + "maximum": 9999, + "minimum": 1, + "trusted": true, + "type": "number" + }, + "keep-weekly": { + "default": 8, + "description": "How many weekly restore points are kept behind the dailies, a count from 1 to 9999, refused by the night like keep-daily; lowering it deletes the restore points beyond the new count at the next night, and that cannot be undone.", + "maximum": 9999, + "minimum": 1, + "trusted": true, + "type": "number" + } + }, + "module": "restic", + "resources": [ + { + "id": "state", + "mode": "0700", + "place": ".", + "type": "directory" + }, + { + "id": "repository", + "mode": "0700", + "type": "directory" + }, + { + "content": "# What the modules on this machine back up, composed by the mesh (novox/hq to-be 43). Do not edit.\nSTAND-IN", + "id": "declared", + "mode": "0600", + "path": "${dir:state}/backups.conf", + "type": "file" + }, + { + "content": "# The data the modules on this machine declare, composed by the mesh (novox/hq ADR 0233). Do not edit.\nSTAND-IN", + "id": "data-declared", + "mode": "0600", + "path": "${dir:state}/data.conf", + "type": "file" + }, + { + "content": "# How many restore points the night keeps, written by the mesh from this machine's settings keep-daily,\n# keep-weekly, keep-monthly (novox/hq issue 416) and keep-hand-taken-days (novox/hq issue 453). Replaced on every push; change the settings, never this file.\ndaily=${config:keep-daily}\nweekly=${config:keep-weekly}\nmonthly=${config:keep-monthly}\nhand=${config:keep-hand-taken-days}\n", + "id": "retention", + "mode": "0644", + "path": "${dir:state}/retention.conf", + "trusted": true, + "type": "file" + } + ], + "version": "1" + }, + { + "config": { + "brand": { + "default": {}, + "description": "SearXNG's brand section of settings.yml, laid over the file's own and SearXNG's defaults: the links the pages show for the instance's source, issues and documentation.", + "trusted": true, + "type": "object" + }, + "categories_as_tabs": { + "default": {}, + "description": "SearXNG's categories_as_tabs section of settings.yml, laid over the file's own and SearXNG's defaults: the categories shown as tabs above the results.", + "trusted": true, + "type": "object" + }, + "engines": { + "default": [], + "description": "SearXNG's engines section of settings.yml, laid over the file's own and SearXNG's defaults: the engines to add or change, each by its name.", + "trusted": true, + "type": "list" + }, + "general": { + "default": {}, + "description": "SearXNG's general section of settings.yml, laid over the file's own and SearXNG's defaults: the instance's name, contact and debug switches.", + "trusted": true, + "type": "object" + }, + "outgoing": { + "default": {}, + "description": "SearXNG's outgoing section of settings.yml, laid over the file's own and SearXNG's defaults: how SearXNG reaches the engines: timeouts, connection pool and proxies.", + "trusted": true, + "type": "object" + }, + "plugins": { + "default": {}, + "description": "SearXNG's plugins section of settings.yml, laid over the file's own and SearXNG's defaults: the plugins to switch on or off, by their id.", + "trusted": true, + "type": "object" + }, + "preferences": { + "default": {}, + "description": "SearXNG's preferences section of settings.yml, laid over the file's own and SearXNG's defaults: the preferences a user cannot change (lock).", + "trusted": true, + "type": "object" + }, + "search": { + "default": { + "formats": [ + "html", + "json" + ] + }, + "description": "SearXNG's search section of settings.yml, laid over the file's own and SearXNG's defaults: safe search, autocomplete, the result formats offered and how long an engine is suspended after an error.", + "trusted": true, + "type": "object" + }, + "server": { + "default": { + "base_url": false, + "image_proxy": false, + "limiter": false, + "public_instance": false, + "secret_key": "${secret:secret}" + }, + "description": "SearXNG's server section of settings.yml, laid over the file's own and SearXNG's defaults: the public base URL, the request method, the limiter and the image proxy; its secret_key is the vault's.", + "trusted": true, + "type": "object" + }, + "ui": { + "default": {}, + "description": "SearXNG's ui section of settings.yml, laid over the file's own and SearXNG's defaults: the theme, the default locale and how results are shown.", + "trusted": true, + "type": "object" + } + }, + "module": "searxng", + "resources": [ + { + "id": "mesh-state", + "mode": "0700", + "place": "mesh", + "type": "directory" + }, + { + "id": "state", + "mode": "0700", + "place": ".", + "type": "directory" + }, + { + "id": "valkey-data", + "mode": "0700", + "type": "directory" + }, + { + "content": "{\n \"use_default_settings\": true,\n \"server\": {\n \"secret_key\": \"STAND-IN\",\n \"base_url\": false,\n \"limiter\": false,\n \"image_proxy\": false,\n \"public_instance\": false\n },\n \"search\": {\n \"formats\": [\"html\", \"json\"]\n },\n \"valkey\": {\n \"url\": \"valkey://valkey:6379/0\"\n }\n}\n", + "id": "settings", + "merge": "json", + "mode": "0600", + "path": "${dir:state}/settings.yml", + "takes-config": [ + "general", + "brand", + "search", + "server", + "ui", + "outgoing", + "engines", + "plugins", + "categories_as_tabs", + "preferences" + ], + "type": "file" + }, + { + "content": "{}\n", + "id": "runtime-config", + "mode": "0600", + "path": "${dir:mesh-state}/config.json", + "type": "file" + } + ], + "version": "1" + }, + { + "config": { + "pooler": { + "default": { + "max-client-connections": 100, + "pool-size": 20, + "tenant": "default" + }, + "description": "The connection pooler's settings, laid over the file's: its tenant, pool size and the most client connections it takes.", + "trusted": true, + "type": "object" + } + }, + "module": "supabase", + "resources": [ + { + "id": "state", + "mode": "0700", + "place": ".", + "type": "directory" + }, + { + "id": "db-init", + "mode": "0755", + "path": "${dir:state}/db-init", + "type": "directory" + }, + { + "id": "functions-main-dir", + "mode": "0755", + "path": "${dir:state}/functions-main", + "type": "directory" + }, + { + "id": "db-data", + "mode": "0700", + "type": "directory" + }, + { + "id": "dumps", + "mode": "0700", + "path": "${dir:state}/dumps", + "type": "directory" + }, + { + "id": "db-config", + "mode": "0755", + "type": "directory" + }, + { + "id": "storage", + "mode": "0755", + "type": "directory" + }, + { + "id": "functions", + "mode": "0755", + "type": "directory" + }, + { + "content": "{\n \"pooler\": {\n \"tenant\": \"default\",\n \"pool-size\": 20,\n \"max-client-connections\": 100\n }\n}\n", + "id": "settings", + "merge": "json", + "mode": "0644", + "path": "${dir:state}/settings.json", + "takes-config": [ + "pooler" + ], + "type": "file" + }, + { + "content": "# Written by the mesh (modules/supabase): the gateway's declarative config, credentials rendered in.\n_format_version: '2.1'\n_transform: true\n\n###\n### Consumers / Users\n###\nconsumers:\n - username: DASHBOARD\n - username: anon\n keyauth_credentials:\n - key: \"STAND-IN\"\n - username: service_role\n keyauth_credentials:\n - key: \"STAND-IN\"\n\n###\n### Access Control List\n###\nacls:\n - consumer: anon\n group: anon\n - consumer: service_role\n group: admin\n\n###\n### Dashboard credentials\n###\nbasicauth_credentials:\n - consumer: DASHBOARD\n username: \"STAND-IN\"\n password: \"STAND-IN\"\n\n###\n### API Routes\n###\nservices:\n ## Open Auth routes\n - name: auth-v1-open\n url: http://supabase-auth:9999/verify\n routes:\n - name: auth-v1-open\n strip_path: true\n paths:\n - /auth/v1/verify\n plugins:\n - name: cors\n - name: auth-v1-open-callback\n url: http://supabase-auth:9999/callback\n routes:\n - name: auth-v1-open-callback\n strip_path: true\n paths:\n - /auth/v1/callback\n plugins:\n - name: cors\n - name: auth-v1-open-authorize\n url: http://supabase-auth:9999/authorize\n routes:\n - name: auth-v1-open-authorize\n strip_path: true\n paths:\n - /auth/v1/authorize\n plugins:\n - name: cors\n\n ## Secure Auth routes\n - name: auth-v1\n _comment: 'GoTrue: /auth/v1/* -> http://auth:9999/*'\n url: http://supabase-auth:9999/\n routes:\n - name: auth-v1-all\n strip_path: true\n paths:\n - /auth/v1/\n plugins:\n - name: cors\n - name: key-auth\n config:\n hide_credentials: false\n - name: acl\n config:\n hide_groups_header: true\n allow:\n - admin\n - anon\n\n ## Secure REST routes\n - name: rest-v1\n _comment: 'PostgREST: /rest/v1/* -> http://rest:3000/*'\n url: http://supabase-rest:3000/\n routes:\n - name: rest-v1-all\n strip_path: true\n paths:\n - /rest/v1/\n plugins:\n - name: cors\n - name: key-auth\n config:\n hide_credentials: true\n - name: acl\n config:\n hide_groups_header: true\n allow:\n - admin\n - anon\n\n ## Secure GraphQL routes\n - name: graphql-v1\n _comment: 'PostgREST: /graphql/v1/* -> http://rest:3000/rpc/graphql'\n url: http://supabase-rest:3000/rpc/graphql\n routes:\n - name: graphql-v1-all\n strip_path: true\n paths:\n - /graphql/v1\n plugins:\n - name: cors\n - name: key-auth\n config:\n hide_credentials: true\n - name: request-transformer\n config:\n add:\n headers:\n - Content-Profile:graphql_public\n - name: acl\n config:\n hide_groups_header: true\n allow:\n - admin\n - anon\n\n ## Secure Realtime routes\n - name: realtime-v1-ws\n _comment: 'Realtime: /realtime/v1/* -> ws://realtime:4000/socket/*'\n url: http://realtime-dev.supabase-realtime:4000/socket\n protocol: ws\n routes:\n - name: realtime-v1-ws\n strip_path: true\n paths:\n - /realtime/v1/\n plugins:\n - name: cors\n - name: key-auth\n config:\n hide_credentials: false\n - name: acl\n config:\n hide_groups_header: true\n allow:\n - admin\n - anon\n - name: realtime-v1-rest\n _comment: 'Realtime: /realtime/v1/* -> ws://realtime:4000/socket/*'\n url: http://realtime-dev.supabase-realtime:4000/api\n protocol: http\n routes:\n - name: realtime-v1-rest\n strip_path: true\n paths:\n - /realtime/v1/api\n plugins:\n - name: cors\n - name: key-auth\n config:\n hide_credentials: false\n - name: acl\n config:\n hide_groups_header: true\n allow:\n - admin\n - anon\n ## Storage routes: the storage server manages its own auth\n - name: storage-v1\n _comment: 'Storage: /storage/v1/* -> http://storage:5000/*'\n url: http://supabase-storage:5000/\n routes:\n - name: storage-v1-all\n strip_path: true\n paths:\n - /storage/v1/\n plugins:\n - name: cors\n\n ## Edge Functions routes\n - name: functions-v1\n _comment: 'Edge Functions: /functions/v1/* -> http://functions:9000/*'\n url: http://supabase-edge-functions:9000/\n routes:\n - name: functions-v1-all\n strip_path: true\n paths:\n - /functions/v1/\n plugins:\n - name: cors\n\n ## Analytics routes\n - name: analytics-v1\n _comment: 'Analytics: /analytics/v1/* -> http://logflare:4000/*'\n url: http://supabase-analytics:4000/\n routes:\n - name: analytics-v1-all\n strip_path: true\n paths:\n - /analytics/v1/\n\n ## Secure Database routes\n - name: meta\n _comment: 'pg-meta: /pg/* -> http://pg-meta:8080/*'\n url: http://supabase-meta:8080/\n routes:\n - name: meta-all\n strip_path: true\n paths:\n - /pg/\n plugins:\n - name: key-auth\n config:\n hide_credentials: false\n - name: acl\n config:\n hide_groups_header: true\n allow:\n - admin\n\n ## Protected Dashboard - catch all remaining routes\n - name: dashboard\n _comment: 'Studio: /* -> http://studio:3000/*'\n url: http://supabase-studio:3000/\n routes:\n - name: dashboard-all\n strip_path: true\n paths:\n - /\n plugins:\n - name: cors\n - name: basic-auth\n config:\n hide_credentials: true\n", + "id": "kong-conf", + "mode": "0600", + "path": "${dir:state}/kong.yml", + "type": "file" + }, + { + "content": "GOTRUE_API_HOST=0.0.0.0\nGOTRUE_API_PORT=9999\nAPI_EXTERNAL_URL=https://STAND-IN\nGOTRUE_DB_DRIVER=postgres\nGOTRUE_DB_DATABASE_URL=postgres://supabase_auth_admin:STAND-IN@supabase-db:5432/postgres\nGOTRUE_SITE_URL=https://STAND-IN\nGOTRUE_URI_ALLOW_LIST=\nGOTRUE_DISABLE_SIGNUP=true\nGOTRUE_JWT_ADMIN_ROLES=service_role\nGOTRUE_JWT_AUD=authenticated\nGOTRUE_JWT_DEFAULT_GROUP_NAME=authenticated\nGOTRUE_JWT_EXP=3600\nGOTRUE_JWT_SECRET=STAND-IN\nGOTRUE_EXTERNAL_EMAIL_ENABLED=false\nGOTRUE_EXTERNAL_ANONYMOUS_USERS_ENABLED=false\nGOTRUE_MAILER_AUTOCONFIRM=false\nGOTRUE_SMTP_ADMIN_EMAIL=admin@example.com\nGOTRUE_SMTP_HOST=supabase-mail\nGOTRUE_SMTP_PORT=2500\nGOTRUE_SMTP_USER=fake_mail_user\nGOTRUE_SMTP_PASS=fake_mail_password\nGOTRUE_SMTP_SENDER_NAME=fake_sender\nGOTRUE_MAILER_URLPATHS_INVITE=/auth/v1/verify\nGOTRUE_MAILER_URLPATHS_CONFIRMATION=/auth/v1/verify\nGOTRUE_MAILER_URLPATHS_RECOVERY=/auth/v1/verify\nGOTRUE_MAILER_URLPATHS_EMAIL_CHANGE=/auth/v1/verify\nGOTRUE_EXTERNAL_PHONE_ENABLED=false\nGOTRUE_SMS_AUTOCONFIRM=true\n", + "id": "auth-conf", + "mode": "0600", + "path": "${dir:state}/gotrue.env", + "type": "file" + }, + { + "content": "db-uri = \"postgres://authenticator:STAND-IN@supabase-db:5432/postgres\"\ndb-schemas = \"public,storage,graphql_public\"\ndb-anon-role = \"anon\"\njwt-secret = \"STAND-IN\"\ndb-use-legacy-gucs = false\napp.settings.jwt_secret = \"STAND-IN\"\napp.settings.jwt_exp = \"3600\"\n", + "id": "rest-conf", + "mode": "0600", + "path": "${dir:state}/postgrest.conf", + "type": "file" + }, + { + "content": "# Written by the mesh (modules/supabase).\n#\n# Every sink hands logflare its API key in the x-api-key header, never as an api_key query parameter:\n# logflare prints a failed request's whole URL, query string included, in its error report, so a\n# key in the URL is a key in its log (novox/hq issue 268). supabase-analytics refuses to start\n# while this file still puts the key in a URL.\napi:\n enabled: true\n address: 0.0.0.0:9001\n\nsources:\n docker_host:\n type: docker_logs\n include_containers:\n - supabase-kong\n - supabase-auth\n - supabase-rest\n - realtime-dev.supabase-realtime\n - supabase-storage\n - supabase-edge-functions\n - supabase-db\n\ntransforms:\n project_logs:\n type: remap\n inputs:\n - docker_host\n source: |-\n .project = \"default\"\n .event_message = del(.message)\n .appname = del(.container_name)\n del(.container_created_at)\n del(.container_id)\n del(.source_type)\n del(.stream)\n del(.label)\n del(.image)\n del(.host)\n del(.stream)\n router:\n type: route\n inputs:\n - project_logs\n route:\n kong: '.appname == \"supabase-kong\"'\n auth: '.appname == \"supabase-auth\"'\n rest: '.appname == \"supabase-rest\"'\n realtime: '.appname == \"realtime-dev.supabase-realtime\"'\n storage: '.appname == \"supabase-storage\"'\n functions: '.appname == \"supabase-edge-functions\"'\n db: '.appname == \"supabase-db\"'\n # Ignores non nginx errors since they are related with kong booting up\n kong_logs:\n type: remap\n inputs:\n - router.kong\n source: |-\n req, err = parse_nginx_log(.event_message, \"combined\")\n if err == null {\n .timestamp = req.timestamp\n .metadata.request.headers.referer = req.referer\n .metadata.request.headers.user_agent = req.agent\n .metadata.request.headers.cf_connecting_ip = req.client\n .metadata.request.method = req.method\n .metadata.request.path = req.path\n .metadata.request.protocol = req.protocol\n .metadata.response.status_code = req.status\n }\n if err != null {\n abort\n }\n # Ignores non nginx errors since they are related with kong booting up\n kong_err:\n type: remap\n inputs:\n - router.kong\n source: |-\n .metadata.request.method = \"GET\"\n .metadata.response.status_code = 200\n parsed, err = parse_nginx_log(.event_message, \"error\")\n if err == null {\n .timestamp = parsed.timestamp\n .severity = parsed.severity\n .metadata.request.host = parsed.host\n .metadata.request.headers.cf_connecting_ip = parsed.client\n url, err = split(parsed.request, \" \")\n if err == null {\n .metadata.request.method = url[0]\n .metadata.request.path = url[1]\n .metadata.request.protocol = url[2]\n }\n }\n if err != null {\n abort\n }\n # Gotrue logs are structured json strings which frontend parses directly. But we keep metadata for consistency.\n auth_logs:\n type: remap\n inputs:\n - router.auth\n source: |-\n parsed, err = parse_json(.event_message)\n if err == null {\n .metadata.timestamp = parsed.time\n .metadata = merge!(.metadata, parsed)\n }\n # PostgREST logs are structured so we separate timestamp from message using regex\n rest_logs:\n type: remap\n inputs:\n - router.rest\n source: |-\n parsed, err = parse_regex(.event_message, r'^(?P