From 1203f96a24c8d2e0370f782859cef6d8a2dd4af8 Mon Sep 17 00:00:00 2001 From: mesh-admin Date: Tue, 13 Oct 2026 23:39:24 +0000 Subject: [PATCH 01/11] Open the integration branch of release 57 (config and provisioning) So the packages of hq design 57 meet here, reviewed one by one, and reach main as one delivery group. --- RELEASE-57.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 RELEASE-57.md diff --git a/RELEASE-57.md b/RELEASE-57.md new file mode 100644 index 0000000..140ccff --- /dev/null +++ b/RELEASE-57.md @@ -0,0 +1,6 @@ +# Release 57: config and provisioning + +This branch carries the work of hq design 57 (ADR 0348, ADR 0349) in this repository. Each package +merges here through its own reviewed pull request; the branch reaches main as one pull request, in one +delivery group with the branch of the same name in mesh-controller, mesh-host, mesh-sdk, mesh-catalog +and hq, after a rehearsal in the lab. This file is deleted by the last commit before that pull request. -- 2.54.0 From 1cc588ebc0606413cafed86ba4ef471203855a05 Mon Sep 17 00:00:00 2001 From: mesh-admin Date: Tue, 13 Oct 2026 23:47:45 +0000 Subject: [PATCH 02/11] Say who owns a placed file at runtime, and which files the provisioner fills MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The provisioner (ADR 0349 §5) needs each file it places to say whether it is owned, a seed or managed, and the node-engine needs to know which paths it fills. The words map onto today's forms (drift written back, create-once, into), so no write changes; the reader accepts the fields and states OwnershipContract before any controller sends them. Issue 667. Co-Authored-By: Claude Opus 5.5 --- cmd/mesh-host/main.go | 7 +- cmd/mesh-host/units_test.go | 8 +- internal/apply/apply.go | 2 +- internal/apply/ownership_test.go | 129 ++++++++++++++++++++++++ internal/declaration/declaration.go | 126 +++++++++++++++++++++++- internal/declaration/ownership_test.go | 130 +++++++++++++++++++++++++ internal/link/messages.go | 7 ++ validate/validate_test.go | 7 ++ 8 files changed, 407 insertions(+), 9 deletions(-) create mode 100644 internal/apply/ownership_test.go create mode 100644 internal/declaration/ownership_test.go diff --git a/cmd/mesh-host/main.go b/cmd/mesh-host/main.go index eb779bd..0837e9a 100644 --- a/cmd/mesh-host/main.go +++ b/cmd/mesh-host/main.go @@ -1647,7 +1647,7 @@ func declaredResolvConf(d *declaration.Declaration) (string, string, bool) { } for _, r := range d.Resources { f, ok := r.(*declaration.File) - if !ok || f.Path != network.ResolvConf || f.CreateOnce || f.Into != "" { + if !ok || f.Path != network.ResolvConf || f.Seeded() || f.Into != "" { continue } module, ok := liveness.ModuleOf(f.ID) @@ -1769,8 +1769,9 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Queue, sa func healthAsReported(st liveness.Statement, ns *network.Statement) *link.Health { // LibraryContract: this engine reads a resource's declared `health` and judges it (ADR 0240 Phase B), judges a // user's declared `root` (novox/hq ADR 0266), and reads a declaration's identities, groups and libraries - // (novox/hq ADR 0328), which is what tells the controller it may be sent any of those fields. - h := &link.Health{Contract: link.BusyContract, At: st.At.UTC(), Resources: []link.ResourceHealth{}} + // (novox/hq ADR 0328), a container's busy (ADR 0335) and a file's owner-at-runtime and the filled files (ADR + // 0349 §5), which is what tells the controller it may be sent any of those fields. + h := &link.Health{Contract: link.OwnershipContract, At: st.At.UTC(), Resources: []link.ResourceHealth{}} for _, r := range st.Resources { h.Resources = append(h.Resources, link.ResourceHealth{Module: r.Module, Resource: r.ID, Kind: r.Kind, Target: r.Target, State: r.State, Reason: r.Reason, Since: r.Since.UTC(), Streak: r.Streak, diff --git a/cmd/mesh-host/units_test.go b/cmd/mesh-host/units_test.go index 61e98a0..a81df0e 100644 --- a/cmd/mesh-host/units_test.go +++ b/cmd/mesh-host/units_test.go @@ -87,10 +87,10 @@ func TestTheStatementSaysAnAccountJudgedForRoot(t *testing.T) { Groups: []string{"openrazer"}}, State: accounts.Healthy, Since: at}, }} h := withAccounts(healthAsReported(liveness.Statement{At: at}, nil), as) - // At RootContract or later: an engine that also reads a container's busy says BusyContract (novox/hq ADR - // 0335), one above ADR 0334's OwnGroupContract. - if h.Contract < link.RootContract || link.RootContract != 3 || h.Contract != link.BusyContract || - link.BusyContract != link.OwnGroupContract+1 { + // At RootContract or later: an engine that also reads a file's owner-at-runtime and the filled files says + // OwnershipContract (novox/hq ADR 0349 §5), one above ADR 0335's BusyContract. + if h.Contract < link.RootContract || link.RootContract != 3 || h.Contract != link.OwnershipContract || + link.OwnershipContract != link.BusyContract+1 || link.BusyContract != link.OwnGroupContract+1 { t.Fatalf("contract %d", h.Contract) } got := map[string]string{} diff --git a/internal/apply/apply.go b/internal/apply/apply.go index a750c84..0305047 100644 --- a/internal/apply/apply.go +++ b/internal/apply/apply.go @@ -1617,7 +1617,7 @@ func applyFile(r *declaration.File, previous store.Applied, unseal Unseal, keepF // A seed that is already there is left exactly as it is — whatever has grown in it since is // not the mesh's to put back (novox/hq issue 035). What this host records is what it once // wrote, so a later declaration that changes the seed is not mistaken for drift either. - if r.CreateOnce && existed { + if r.Seeded() && existed { out.wrote = previous.Wrote if out.wrote == "" { out.wrote = digestOf(string(existing)) diff --git a/internal/apply/ownership_test.go b/internal/apply/ownership_test.go new file mode 100644 index 0000000..774cdd3 --- /dev/null +++ b/internal/apply/ownership_test.go @@ -0,0 +1,129 @@ +package apply + +import ( + "context" + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/novox/mesh-host/internal/store" +) + +// Defends novox/hq ADR 0349 §5 (issue 667): a file that says who owns it at runtime is written as the +// form it names always was — owned as a whole file whose drift is written back, seed as create-once, +// managed as only its keys or its marked block. Each is applied beside its old form, through the same +// drift, and the two must end the same: the same actions, the same bytes on disk. + +// ownershipCase is one ownership word and the old form it maps onto. +type ownershipCase struct { + name string + // said and old are the resource's fields after id, type and path: with the word, and in the old form. + said, old string + // before is what is at the path before the first apply; "" leaves the path absent. + before string + // drift is what the machine writes there between the two applies. + drift string + // first and second are the actions the old form is known to report. + first, second string +} + +func TestEachOwnershipBehavesAsItsOldForm(t *testing.T) { + for _, c := range []ownershipCase{ + { + name: "owned is a whole file whose drift is written back", + said: `"content":"correct\n","mode":"0644","owner-at-runtime":"owned"`, + old: `"content":"correct\n","mode":"0644"`, + drift: "someone edited this\n", + first: "created", second: "corrected", + }, + { + name: "seed is written once and then the program's", + said: `"content":"user default on\n","mode":"0600","owner-at-runtime":"seed"`, + old: `"content":"user default on\n","mode":"0600","create-once":true`, + drift: "user default on\nuser app-one on\n", + first: "created", second: "kept", + }, + { + name: "managed json sets only its keys", + said: `"into":"json","content":"{\"insecure-registries\":[\"10.42.0.1:5000\"]}","owner-at-runtime":"managed"`, + old: `"into":"json","content":"{\"insecure-registries\":[\"10.42.0.1:5000\"]}"`, + before: machinesOwn, + drift: `{"data-root":"/srv/other","log-opts":{"max-size":"10m"},"insecure-registries":["192.0.2.7:5000"]}`, + first: "updated", second: "corrected", + }, + { + name: "managed block holds only its region", + said: `"into":"block","content":"10.42.0.1 ace\n","owner-at-runtime":"managed"`, + old: `"into":"block","content":"10.42.0.1 ace\n"`, + before: workstationHosts, + drift: workstationHosts + "\n" + marked("f", "6.6.6.6 evil\n") + "192.0.2.9 the-machines-own\n", + first: "updated", second: "corrected", + }, + } { + t.Run(c.name, func(t *testing.T) { + saidActions, saidBytes := applyThroughDrift(t, c, c.said) + oldActions, oldBytes := applyThroughDrift(t, c, c.old) + if saidActions != oldActions { + t.Errorf("with the word the actions were %s; the old form's were %s", saidActions, oldActions) + } + if want := c.first + " " + c.second; oldActions != want { + t.Errorf("the old form's actions were %s; known to be %s", oldActions, want) + } + if saidBytes != oldBytes { + t.Errorf("with the word the file ends\n%q\nthe old form's ends\n%q", saidBytes, oldBytes) + } + }) + } +} + +// applyThroughDrift applies one file, lets the machine write the case's drift into it, applies again, +// and says the two actions and what the file holds at the end. +func applyThroughDrift(t *testing.T, c ownershipCase, fields string) (string, string) { + t.Helper() + path := filepath.Join(t.TempDir(), "the-file") + if c.before != "" { + if err := os.WriteFile(path, []byte(c.before), 0o644); err != nil { + t.Fatal(err) + } + } + d := parse(t, fmt.Sprintf(`{"declaration":1,"resources":[{"id":"f","type":"file","path":%q,%s}]}`, path, fields)) + first, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, noServices, nil, nil) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(c.drift), 0o644); err != nil { + t.Fatal(err) + } + second, _, err := Apply(context.Background(), archHost(t), d, state, store.OriginDeclared, noServices, nil, nil) + if err != nil { + t.Fatal(err) + } + return first.Outcomes[0].Action + " " + second.Outcomes[0].Action, readText(t, path) +} + +// A seed said by the word alone, with no create-once beside it, is still kept once present: the word is +// enough, and the file the program grew is not written back. +func TestASeedByTheWordAloneKeepsWhatGrew(t *testing.T) { + path := filepath.Join(t.TempDir(), "acl.conf") + d := parse(t, fmt.Sprintf(`{"declaration":1,"resources":[ + {"id":"acl","type":"file","path":%q,"content":"user default on\n","owner-at-runtime":"seed"} + ]}`, path)) + _, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, noServices, nil, nil) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte("grown\n"), 0o600); err != nil { + t.Fatal(err) + } + report, _, err := Apply(context.Background(), archHost(t), d, state, store.OriginDeclared, noServices, nil, nil) + if err != nil { + t.Fatal(err) + } + if got := report.Outcomes[0].Action; got != "kept" { + t.Errorf("a seed was %q", got) + } + if got := readText(t, path); got != "grown\n" { + t.Errorf("what grew in the seed was written over: %q", got) + } +} diff --git a/internal/declaration/declaration.go b/internal/declaration/declaration.go index a265afa..0c165c3 100644 --- a/internal/declaration/declaration.go +++ b/internal/declaration/declaration.go @@ -237,8 +237,45 @@ type File struct { // Owner is the user this belongs to, by name. Absent means root. Owner string `json:"owner,omitempty"` + + // OwnerAtRuntime says who owns the file once it is placed (novox/hq ADR 0349 §5): "owned", the mesh's + // whole, written back whenever it differs; "seed", written once and then the program's; or "managed", + // only its named keys or its marked block the mesh's, the program keeping the rest. + // + // **Three words for what the forms above already do.** Owned is an ordinary file, seed is create-once + // and managed is a file written into, so the word changes no write: a file that says it is applied + // exactly as its old form is, and one that does not say is owned by its old form (Ownership). The word + // is what the provisioner's files say of themselves, which the old forms only implied. Seed is enough + // alone; managed still needs `into`, which says the format. A word that contradicts the form is refused. + OwnerAtRuntime string `json:"owner-at-runtime,omitempty"` } +// The words a file says who owns it at runtime with (novox/hq ADR 0349 §5). +const ( + RuntimeOwned = "owned" + RuntimeSeed = "seed" + RuntimeManaged = "managed" +) + +// Ownership is who owns the file at runtime: the word it says, or the one its old form implies — a file +// written into is managed, one created once is a seed, and any other is owned. +func (f *File) Ownership() string { + switch { + case f.OwnerAtRuntime != "": + return f.OwnerAtRuntime + case f.Into != "": + return RuntimeManaged + case f.CreateOnce: + return RuntimeSeed + default: + return RuntimeOwned + } +} + +// Seeded reports whether the file is written only when absent: created once, by create-once or by the +// word seed. +func (f *File) Seeded() bool { return f.CreateOnce || f.OwnerAtRuntime == RuntimeSeed } + // Secret reports whether this file arrived sealed, which is what decides both that it must be // opened before writing and that its contents must never appear in a report. func (f *File) Secret() bool { return f.Sealed != "" } @@ -320,6 +357,7 @@ func (f *File) validate(where string, _ bool) []string { "secrets or create-once") } } + problems = append(problems, f.checkOwnership(where)...) var said []string for name, value := range map[string]string{ "content": f.Content, "sealed": f.Sealed, "bytes": f.Bytes, @@ -361,6 +399,42 @@ func (f *File) validate(where string, _ bool) []string { return append(problems, checkMode(where, f.Mode)...) } +// checkOwnership refuses a word that contradicts the file's form (novox/hq ADR 0349 §5): owned is a whole +// file written back, so not created once nor written into; a seed is a whole file created once, so not +// written into; managed is a file written into, which says its format. Managed and created once together +// is already refused by the rule that a file written into is not created once. +func (f *File) checkOwnership(where string) []string { + switch f.OwnerAtRuntime { + case "": + case RuntimeOwned: + var also []string + if f.CreateOnce { + also = append(also, "create-once") + } + if f.Into != "" { + also = append(also, "into") + } + if len(also) > 0 { + return []string{fmt.Sprintf("%s: owned at runtime is the mesh's whole file, written back whenever it "+ + "differs, and this also says %s", where, strings.Join(also, " and "))} + } + case RuntimeSeed: + if f.Into != "" { + return []string{where + ": a seed is a whole file written once, and this says into; a file the " + + "program shares with the mesh is managed"} + } + case RuntimeManaged: + if f.Into == "" { + return []string{where + ": a managed file names the format its keys or block are written into, " + + "with into \"json\" or \"block\""} + } + default: + return []string{fmt.Sprintf("%s: owner-at-runtime %q; a placed file is %q, %q or %q, or omits it to be "+ + "owned by its form", where, f.OwnerAtRuntime, RuntimeOwned, RuntimeSeed, RuntimeManaged)} + } + return nil +} + // User is a login on the machine. // // The thing that makes a shell, a chat client or a desktop expressible at all: each is a package @@ -1565,6 +1639,52 @@ type Declaration struct { // this is its home, whether the operator's warrant to take it was carried out, the sub-paths its modules ask for, // and the tool runner's account to put in its group (§11). Absent is a controller older than ADR 0328. Libraries []Library + + // Filled is the paths of the files the provisioner fills on this machine (novox/hq ADR 0349 §4, §5): each a + // file this declaration declares, which says who owns it at runtime. The provisioner's client has the + // node-engine write only these. Absent is a controller that hands no file to the provisioner, and every + // file is written from the declaration as before. + Filled []string +} + +// Fills reports whether the provisioner fills the file at path on this machine. +func (d *Declaration) Fills(path string) bool { return slices.Contains(d.Filled, path) } + +// checkFilled refuses a filled path that is not a declared file saying who owns it at runtime, or that is named +// twice or empty. Every file declared at the path must say it: the provisioner writes the path, and a file there +// the mesh writes some other way would be a second writer of it (ADR 0349 §5). +func checkFilled(filled []string, resources []Resource) []string { + var problems []string + files := map[string][]*File{} + for _, r := range resources { + if f, ok := r.(*File); ok { + files[f.Path] = append(files[f.Path], f) + } + } + seen := map[string]bool{} + for _, path := range filled { + switch { + case path == "": + problems = append(problems, "filled_files names a file with no path") + continue + case seen[path]: + problems = append(problems, fmt.Sprintf("filled_files names %q twice", path)) + continue + } + seen[path] = true + if len(files[path]) == 0 { + problems = append(problems, fmt.Sprintf("filled_files names %q, and this declaration declares no "+ + "file there", path)) + continue + } + for _, f := range files[path] { + if f.OwnerAtRuntime == "" { + problems = append(problems, fmt.Sprintf("resource %q: the provisioner fills %q, and a file it "+ + "fills says who owns it at runtime with owner-at-runtime", f.ID, path)) + } + } + } + return problems } // Identity is one module's identity, as the controller allocated it (ADR 0328 §4): its user `mesh-` with @@ -1904,6 +2024,9 @@ type envelope struct { // sends them only to a host whose health says link.LibraryContract. Groups []Group `json:"groups,omitempty"` Libraries []Library `json:"libraries,omitempty"` + // Filled is optional too (novox/hq ADR 0349 §5), as is a file's `owner-at-runtime`. **An older host refuses + // these keys**; a controller sends them only to a host whose health says link.OwnershipContract. + Filled []string `json:"filled_files,omitempty"` } func parse(raw []byte, allowActions bool) (*Declaration, error) { @@ -1922,7 +2045,7 @@ func parse(raw []byte, allowActions bool) (*Declaration, error) { d := &Declaration{Version: env.Version, For: env.For, Adoption: env.Adoption, Sequence: env.Sequence, Epoch: env.Epoch, LeftOut: env.LeftOut, Generation: env.Generation, PutBack: env.PutBack, - Identities: env.Identities, Groups: env.Groups, Libraries: env.Libraries} + Identities: env.Identities, Groups: env.Groups, Libraries: env.Libraries, Filled: env.Filled} problems := checkIdentities(env.Identities, env.Groups) problems = append(problems, checkLibraries(env.Groups, env.Libraries)...) if env.Generation < 0 { @@ -2015,6 +2138,7 @@ func parse(raw []byte, allowActions bool) (*Declaration, error) { problems = append(problems, checkWhileStopped(d.Resources)...) problems = append(problems, checkAdoption(env.Adoption, d.Resources, allowActions)...) problems = append(problems, checkReplaces(d.Resources)...) + problems = append(problems, checkFilled(env.Filled, d.Resources)...) if env.Adoption == nil { for _, r := range d.Resources { if r.Kind() == TypeOpening { diff --git a/internal/declaration/ownership_test.go b/internal/declaration/ownership_test.go new file mode 100644 index 0000000..76b0674 --- /dev/null +++ b/internal/declaration/ownership_test.go @@ -0,0 +1,130 @@ +package declaration + +import ( + "strings" + "testing" +) + +// Defends novox/hq ADR 0349 §5 (issue 667): a file the provisioner places says who owns it at runtime +// — owned, seed or managed — and the declaration names the files the provisioner fills. The words are +// read and judged here before any controller sends them. + +func TestAFileSaysWhoOwnsItAtRuntime(t *testing.T) { + d, err := Parse([]byte(`{"declaration":1,"resources":[ + {"id":"o","type":"file","path":"/etc/o","content":"a","owner-at-runtime":"owned"}, + {"id":"s","type":"file","path":"/etc/s","content":"a","owner-at-runtime":"seed"}, + {"id":"s2","type":"file","path":"/etc/s2","content":"a","owner-at-runtime":"seed","create-once":true}, + {"id":"mj","type":"file","path":"/etc/mj","into":"json","content":"{\"k\":1}","owner-at-runtime":"managed"}, + {"id":"mb","type":"file","path":"/etc/mb","into":"block","content":"a\n","owner-at-runtime":"managed"} + ]}`)) + if err != nil { + t.Fatal(err) + } + want := map[string]string{"o": RuntimeOwned, "s": RuntimeSeed, "s2": RuntimeSeed, "mj": RuntimeManaged, + "mb": RuntimeManaged} + for _, r := range d.Resources { + f := r.(*File) + if f.OwnerAtRuntime != want[f.ID] || f.Ownership() != want[f.ID] { + t.Errorf("%s: read as %q, ownership %q; want %q", f.ID, f.OwnerAtRuntime, f.Ownership(), want[f.ID]) + } + } +} + +// A file that does not say is owned by its old form, so the word means exactly what that form does. +func TestAFileThatDoesNotSayIsOwnedByItsOldForm(t *testing.T) { + d, err := Parse([]byte(`{"declaration":1,"resources":[ + {"id":"o","type":"file","path":"/etc/o","content":"a"}, + {"id":"s","type":"file","path":"/etc/s","content":"a","create-once":true}, + {"id":"mj","type":"file","path":"/etc/mj","into":"json","content":"{}"}, + {"id":"mb","type":"file","path":"/etc/mb","into":"block","content":"a\n"} + ]}`)) + if err != nil { + t.Fatal(err) + } + want := map[string]string{"o": RuntimeOwned, "s": RuntimeSeed, "mj": RuntimeManaged, "mb": RuntimeManaged} + for _, r := range d.Resources { + f := r.(*File) + if f.OwnerAtRuntime != "" || f.Ownership() != want[f.ID] { + t.Errorf("%s: said %q, ownership %q; want unsaid and %q", f.ID, f.OwnerAtRuntime, f.Ownership(), want[f.ID]) + } + if f.Seeded() != (want[f.ID] == RuntimeSeed) { + t.Errorf("%s: seeded %v", f.ID, f.Seeded()) + } + } +} + +func TestAnOwnershipThatContradictsTheFileIsRefused(t *testing.T) { + for name, c := range map[string]struct{ resource, refusal string }{ + "another word": {`{"id":"f","type":"file","path":"/etc/x","content":"a","owner-at-runtime":"mine"}`, + `owner-at-runtime "mine"`}, + "owned and create-once": {`{"id":"f","type":"file","path":"/etc/x","content":"a","owner-at-runtime":"owned","create-once":true}`, + "create-once"}, + "owned and into": {`{"id":"f","type":"file","path":"/etc/x","into":"json","content":"{}","owner-at-runtime":"owned"}`, + "into"}, + "seed and into": {`{"id":"f","type":"file","path":"/etc/x","into":"block","content":"a","owner-at-runtime":"seed"}`, + "into"}, + "managed and whole": {`{"id":"f","type":"file","path":"/etc/x","content":"a","owner-at-runtime":"managed"}`, + "into"}, + "managed and create-once": {`{"id":"f","type":"file","path":"/etc/x","into":"json","content":"{}","owner-at-runtime":"managed","create-once":true}`, + "create-once"}, + // The rule that was there before stays: a file written into is never sealed or created once. + "managed and sealed": {`{"id":"f","type":"file","path":"/etc/x","into":"json","content":"{}","sealed":"x","owner-at-runtime":"managed"}`, + "sealed"}, + } { + _, err := Parse([]byte(`{"declaration":1,"resources":[` + c.resource + `]}`)) + if err == nil || !strings.Contains(err.Error(), c.refusal) { + t.Errorf("%s: want a refusal naming %q, got %v", name, c.refusal, err) + } + } +} + +func TestTheDeclarationNamesTheFilesTheProvisionerFills(t *testing.T) { + d, err := Parse([]byte(`{"declaration":1,"filled_files":["/etc/a","/etc/b"],"resources":[ + {"id":"a","type":"file","path":"/etc/a","content":"a","owner-at-runtime":"owned"}, + {"id":"b","type":"file","path":"/etc/b","into":"block","content":"b\n","owner-at-runtime":"managed"}, + {"id":"c","type":"file","path":"/etc/c","content":"c"} + ]}`)) + if err != nil { + t.Fatal(err) + } + if len(d.Filled) != 2 || d.Filled[0] != "/etc/a" || d.Filled[1] != "/etc/b" { + t.Fatalf("filled files read as %v", d.Filled) + } + if !d.Fills("/etc/a") || !d.Fills("/etc/b") || d.Fills("/etc/c") { + t.Errorf("Fills: a %v, b %v, c %v", d.Fills("/etc/a"), d.Fills("/etc/b"), d.Fills("/etc/c")) + } + + // A declaration that names none fills none, as every declaration before this did. + d, err = Parse([]byte(`{"declaration":1,"resources":[{"id":"c","type":"file","path":"/etc/c","content":"c"}]}`)) + if err != nil { + t.Fatal(err) + } + if d.Filled != nil || d.Fills("/etc/c") { + t.Errorf("a declaration naming no filled files fills %v", d.Filled) + } +} + +func TestAFilledFileMustBeADeclaredFileThatSaysItsOwner(t *testing.T) { + for name, c := range map[string]struct{ filled, resources, refusal string }{ + "not declared": {`["/etc/none"]`, + `{"id":"a","type":"file","path":"/etc/a","content":"a","owner-at-runtime":"owned"}`, + `"/etc/none"`}, + "a directory, not a file": {`["/etc/d"]`, + `{"id":"d","type":"directory","path":"/etc/d"}`, + `"/etc/d"`}, + "says no owner": {`["/etc/a"]`, + `{"id":"a","type":"file","path":"/etc/a","content":"a"}`, + "owner-at-runtime"}, + "named twice": {`["/etc/a","/etc/a"]`, + `{"id":"a","type":"file","path":"/etc/a","content":"a","owner-at-runtime":"owned"}`, + "twice"}, + "no path": {`[""]`, + `{"id":"a","type":"file","path":"/etc/a","content":"a","owner-at-runtime":"owned"}`, + "no path"}, + } { + _, err := Parse([]byte(`{"declaration":1,"filled_files":` + c.filled + `,"resources":[` + c.resources + `]}`)) + if err == nil || !strings.Contains(err.Error(), c.refusal) { + t.Errorf("%s: want a refusal naming %q, got %v", name, c.refusal, err) + } + } +} diff --git a/internal/link/messages.go b/internal/link/messages.go index 0dc192e..5bc1828 100644 --- a/internal/link/messages.go +++ b/internal/link/messages.go @@ -295,6 +295,13 @@ const OwnGroupContract = 7 // for it. const BusyContract = 8 +// OwnershipContract is the statement of a host that also reads a file's `owner-at-runtime` and a declaration's +// `filled_files` (novox/hq ADR 0349 §5): it writes an owned file back on drift, a seed once and a managed file's +// keys or block only, exactly as their old forms, and refuses a filled path that is not a declared file saying +// its owner. Like the fields before it, its presence is what tells the controller this host may be sent those +// fields; an older host refuses the whole declaration for them. +const OwnershipContract = 9 + // Health is one statement of every long-running resource's state on this machine (to-be 48 §4). Said in // every report, as an event on each change, and again every minute while anything is not healthy — so a // lost statement is not a lost fault. diff --git a/validate/validate_test.go b/validate/validate_test.go index dba68e4..433850c 100644 --- a/validate/validate_test.go +++ b/validate/validate_test.go @@ -18,7 +18,14 @@ func TestTheValidatorIsTheHosts(t *testing.T) { if problems := Declaration([]byte(ordered)); problems != nil { t.Fatalf("a declaration carrying its order was refused: %v", problems) } + // And one carrying a file's owner-at-runtime and the files the provisioner fills (novox/hq ADR 0349 §5): the + // reader takes them before any controller sends them. + owned := `{"declaration": 1, "filled_files": ["/etc/x"], "resources": [{"id": "f", "type": "file", "path": "/etc/x", "content": "a", "owner-at-runtime": "owned"}]}` + if problems := Declaration([]byte(owned)); problems != nil { + t.Fatalf("a declaration saying who owns a file at runtime was refused: %v", problems) + } for _, bad := range []string{ + `{"declaration": 1, "filled_files": ["/etc/none"], "resources": [{"id": "f", "type": "file", "path": "/etc/x", "content": "a", "owner-at-runtime": "owned"}]}`, `{"declaration": 1, "resources": []}`, `{"declaration": 99, "resources": [{"id": "d", "type": "directory", "path": "/x"}]}`, `{"declaration": 1, "resources": [{"id": "d", "type": "nothing-the-host-knows"}]}`, -- 2.54.0 From c8920622516548978d92e734f3991a262f98a742 Mon Sep 17 00:00:00 2001 From: mesh-admin Date: Tue, 13 Oct 2026 23:58:05 +0000 Subject: [PATCH 03/11] Let genesis's controller write its record mirrors The controller now keeps the assignments and each registered module's declared config on the bus (hq issue 652, design 57 F3), so its composed grant gains the two buckets and their two events. The installer's first user list must equal that composition, or the controller is refused at genesis; mesh-controller's genesis template test holds the two to each other. Co-Authored-By: Claude Opus 5.5 --- examples/foundation-first-node.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/foundation-first-node.lock b/examples/foundation-first-node.lock index 9f59df6..5e05f7f 100644 --- a/examples/foundation-first-node.lock +++ b/examples/foundation-first-node.lock @@ -221,7 +221,7 @@ "path": "/var/lib/nats-module/conf/accounts.conf", "mode": "0600", "create-once": true, - "content": "// The first user list, carried by the installer because at genesis there is no mesh to\n// compose one. A bootstrap credential, rotated with the store's and replaced by the\n// controller's own composition from its first start onward.\naccounts {\n MESH {\n jetstream: enabled\n users = [\n { user: \"controller\", password: \"$2a$10$AHqJgOifIVbU41KmATiMhuXFs8xa7Wl2HuN4UVBCXdN2jIQzjqApy\", permissions: {\n publish: { allow: [\"$JS.ACK.CONTROL.controller.>\", \"$JS.ACK.EVENTS.controller.>\", \"$JS.ACK.DEAD_LETTER_NOTICES.controller.>\", \"$JS.API.>\", \"$KV.mesh-controller_asked.>\", \"$KV.mesh-controller_calls.>\", \"$KV.mesh-controller_hand-acts.>\", \"$KV.mesh-controller_conditions.>\", \"$KV.mesh-controller_condition-history.>\", \"$KV.mesh-controller_lease.>\", \"$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>\", \"$KV.SEAT_NODE_BUILD_AGENT_cancelled.>\", \"_INBOX.enrol.>\", \"mesh.assignment.>\", \"mesh.mod.*.tool.>\", \"mesh.node.>\", \"mesh.seat.mesh-build-machine.accept.>\", \"mesh.seat.node-build-agent.accept.>\", \"mesh.seat.mesh-build-machine.tool.>\", \"mesh.seat.node-build-agent.tool.>\", \"mesh.seat.mesh-controller.event.applied\", \"mesh.seat.mesh-controller.event.built-before\", \"mesh.seat.mesh-controller.event.refused\", \"mesh.seat.mesh-controller.event.rolled-back\", \"mesh.seat.mesh-controller.event.checked\", \"mesh.seat.mesh-controller.event.plan-moved\", \"mesh.seat.mesh-controller.event.condition-raised\", \"mesh.seat.mesh-controller.event.condition-changed\", \"mesh.seat.mesh-controller.event.condition-cleared\", \"mesh.seat.mesh-controller.event.doctor-heartbeat\", \"mesh.seat.mesh-controller.event.secret-replaced\", \"mesh.seat.mesh-controller.event.healer-acted\", \"$SRV.INFO\", \"mesh.seat.node-intrusion-prevention.tool.banned.*\", \"mesh.seat.node-backup.tool.backed-up.*\", \"mesh.seat.node-backup.tool.now.*\", \"mesh.seat.mesh-delivery.tool.stalled\", \"mesh.seat.mesh-delivery.tool.close\", \"mesh.seat.node-launcher.tool.secret.*\", \"mesh.seat.mesh-delivery.tool.release\", \"mesh.seat.mesh-delivery.tool.stop\", \"mesh.seat.mesh-delivery.tool.times\", \"mesh.seat.mesh-delivery.tool.groups\", \"mesh.seat.node-service-manager.tool.restart.*\", \"mesh.seat.mesh-controller.tool.plans\", \"mesh.events.dead.>\", \"mesh.again.>\"] }\n subscribe: { allow: [\"$JS.API.>\", \"$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>\", \"$JS.EVENT.ADVISORY.CONSUMER.DELETED.>\", \"_DELIVER.controller\", \"_DELIVER.controller.>\", \"_INBOX.controller.>\", \"mesh.control.>\", \"mesh.mod.*.event.provisioner.failing\", \"mesh.mod.*.event.provisioner.recovered\", \"mesh.mod.*.event.provisioner.retirement\", \"mesh.mod.gitea.event.pull.merged\", \"mesh.mod.gitea.event.pull.updated\", \"mesh.mod.mesh-catalog.event.catching-up\", \"mesh.mod.mesh-catalog.event.upgraded\", \"mesh.seat.mesh-build-machine.event.built\", \"mesh.seat.node-build-agent.event.built\", \"mesh.seat.operator-channel.event.decided.mesh-controller\", \"mesh.seat.mesh-controller.tool.>\", \"$SRV.PING\", \"$SRV.INFO\", \"$SRV.PING.mesh-controller\", \"$SRV.PING.mesh-controller.>\", \"$SRV.INFO.mesh-controller\", \"$SRV.INFO.mesh-controller.>\", \"$SRV.STATS\", \"$SRV.STATS.mesh-controller\", \"$SRV.STATS.mesh-controller.>\"] }\n allow_responses: { max: 1, ttl: \"1m\" }\n } }\n ]\n }\n}\n" + "content": "// The first user list, carried by the installer because at genesis there is no mesh to\n// compose one. A bootstrap credential, rotated with the store's and replaced by the\n// controller's own composition from its first start onward.\naccounts {\n MESH {\n jetstream: enabled\n users = [\n { user: \"controller\", password: \"$2a$10$AHqJgOifIVbU41KmATiMhuXFs8xa7Wl2HuN4UVBCXdN2jIQzjqApy\", permissions: {\n publish: { allow: [\"$JS.ACK.CONTROL.controller.>\", \"$JS.ACK.EVENTS.controller.>\", \"$JS.ACK.DEAD_LETTER_NOTICES.controller.>\", \"$JS.API.>\", \"$KV.mesh-controller_asked.>\", \"$KV.mesh-controller_assignments.>\", \"$KV.mesh-controller_calls.>\", \"$KV.mesh-controller_hand-acts.>\", \"$KV.mesh-controller_conditions.>\", \"$KV.mesh-controller_condition-history.>\", \"$KV.mesh-controller_declared-config.>\", \"$KV.mesh-controller_lease.>\", \"$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>\", \"$KV.SEAT_NODE_BUILD_AGENT_cancelled.>\", \"_INBOX.enrol.>\", \"mesh.assignment.>\", \"mesh.mod.*.tool.>\", \"mesh.node.>\", \"mesh.seat.mesh-build-machine.accept.>\", \"mesh.seat.node-build-agent.accept.>\", \"mesh.seat.mesh-build-machine.tool.>\", \"mesh.seat.node-build-agent.tool.>\", \"mesh.seat.mesh-controller.event.applied\", \"mesh.seat.mesh-controller.event.built-before\", \"mesh.seat.mesh-controller.event.refused\", \"mesh.seat.mesh-controller.event.rolled-back\", \"mesh.seat.mesh-controller.event.checked\", \"mesh.seat.mesh-controller.event.plan-moved\", \"mesh.seat.mesh-controller.event.assignment-changed\", \"mesh.seat.mesh-controller.event.declared-config-changed\", \"mesh.seat.mesh-controller.event.condition-raised\", \"mesh.seat.mesh-controller.event.condition-changed\", \"mesh.seat.mesh-controller.event.condition-cleared\", \"mesh.seat.mesh-controller.event.doctor-heartbeat\", \"mesh.seat.mesh-controller.event.secret-replaced\", \"mesh.seat.mesh-controller.event.healer-acted\", \"$SRV.INFO\", \"mesh.seat.node-intrusion-prevention.tool.banned.*\", \"mesh.seat.node-backup.tool.backed-up.*\", \"mesh.seat.node-backup.tool.now.*\", \"mesh.seat.mesh-delivery.tool.stalled\", \"mesh.seat.mesh-delivery.tool.close\", \"mesh.seat.node-launcher.tool.secret.*\", \"mesh.seat.mesh-delivery.tool.release\", \"mesh.seat.mesh-delivery.tool.stop\", \"mesh.seat.mesh-delivery.tool.times\", \"mesh.seat.mesh-delivery.tool.groups\", \"mesh.seat.node-service-manager.tool.restart.*\", \"mesh.seat.mesh-controller.tool.plans\", \"mesh.events.dead.>\", \"mesh.again.>\"] }\n subscribe: { allow: [\"$JS.API.>\", \"$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>\", \"$JS.EVENT.ADVISORY.CONSUMER.DELETED.>\", \"_DELIVER.controller\", \"_DELIVER.controller.>\", \"_INBOX.controller.>\", \"mesh.control.>\", \"mesh.mod.*.event.provisioner.failing\", \"mesh.mod.*.event.provisioner.recovered\", \"mesh.mod.*.event.provisioner.retirement\", \"mesh.mod.gitea.event.pull.merged\", \"mesh.mod.gitea.event.pull.updated\", \"mesh.mod.mesh-catalog.event.catching-up\", \"mesh.mod.mesh-catalog.event.upgraded\", \"mesh.seat.mesh-build-machine.event.built\", \"mesh.seat.node-build-agent.event.built\", \"mesh.seat.operator-channel.event.decided.mesh-controller\", \"mesh.seat.mesh-controller.tool.>\", \"$SRV.PING\", \"$SRV.INFO\", \"$SRV.PING.mesh-controller\", \"$SRV.PING.mesh-controller.>\", \"$SRV.INFO.mesh-controller\", \"$SRV.INFO.mesh-controller.>\", \"$SRV.STATS\", \"$SRV.STATS.mesh-controller\", \"$SRV.STATS.mesh-controller.>\"] }\n allow_responses: { max: 1, ttl: \"1m\" }\n } }\n ]\n }\n}\n" }, // The upstream server, until the broker step adopts it: the `nats` module's own image, built by the // mesh, takes its place there once, on these same directories (ADR 0333 §3), as the bus's planned -- 2.54.0 From fddcddfaf720da89c399c10d725420574c058653 Mon Sep 17 00:00:00 2001 From: mesh-admin Date: Wed, 14 Oct 2026 00:33:48 +0000 Subject: [PATCH 04/11] Name the provisioner client's account in the declaration, and share the apply's record MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The local write of a filled file (ADR 0349 §4) accepts only the provisioner client's account, which the declaration names; the reader takes provisioner_account before any controller sends it (contract 10). The apply's record and its rule for what moved become helpers the write uses too. Issue 668. Co-Authored-By: Claude Opus 5.5 --- cmd/mesh-host/main.go | 8 +++- cmd/mesh-host/units_test.go | 9 ++-- internal/apply/apply.go | 72 +++++++++++++++++------------ internal/declaration/declaration.go | 26 ++++++++++- internal/link/messages.go | 6 +++ validate/validate_test.go | 6 +++ 6 files changed, 91 insertions(+), 36 deletions(-) diff --git a/cmd/mesh-host/main.go b/cmd/mesh-host/main.go index 0837e9a..f7d7dc1 100644 --- a/cmd/mesh-host/main.go +++ b/cmd/mesh-host/main.go @@ -1280,6 +1280,9 @@ func runLink(ctx context.Context, opts options) error { // And mesh-cli is answered on this machine: the account from the kernel, the line asked of the controller // (novox/hq ADR 0272). go serveMeshCLI(aside, queue, say) + // And a filled file is written on the provisioner client's request, one at a time with the apply (novox/hq ADR + // 0349 §4). + go serveFilling(aside, opts, mine.Membership.Signer, say) held := link.HoldRoused(aside, membership, queue, say, opts.timeout, rousedBySignal(ctx)) // The act in hand finishes and is kept before this process exits: an apply that stood aside with @@ -1770,8 +1773,9 @@ func healthAsReported(st liveness.Statement, ns *network.Statement) *link.Health // LibraryContract: this engine reads a resource's declared `health` and judges it (ADR 0240 Phase B), judges a // user's declared `root` (novox/hq ADR 0266), and reads a declaration's identities, groups and libraries // (novox/hq ADR 0328), a container's busy (ADR 0335) and a file's owner-at-runtime and the filled files (ADR - // 0349 §5), which is what tells the controller it may be sent any of those fields. - h := &link.Health{Contract: link.OwnershipContract, At: st.At.UTC(), Resources: []link.ResourceHealth{}} + // 0349 §5) and the provisioner client's account (ADR 0349 §4), which is what tells the controller it may be sent + // any of those fields. + h := &link.Health{Contract: link.ProvisionerContract, At: st.At.UTC(), Resources: []link.ResourceHealth{}} for _, r := range st.Resources { h.Resources = append(h.Resources, link.ResourceHealth{Module: r.Module, Resource: r.ID, Kind: r.Kind, Target: r.Target, State: r.State, Reason: r.Reason, Since: r.Since.UTC(), Streak: r.Streak, diff --git a/cmd/mesh-host/units_test.go b/cmd/mesh-host/units_test.go index a81df0e..423c286 100644 --- a/cmd/mesh-host/units_test.go +++ b/cmd/mesh-host/units_test.go @@ -87,10 +87,11 @@ func TestTheStatementSaysAnAccountJudgedForRoot(t *testing.T) { Groups: []string{"openrazer"}}, State: accounts.Healthy, Since: at}, }} h := withAccounts(healthAsReported(liveness.Statement{At: at}, nil), as) - // At RootContract or later: an engine that also reads a file's owner-at-runtime and the filled files says - // OwnershipContract (novox/hq ADR 0349 §5), one above ADR 0335's BusyContract. - if h.Contract < link.RootContract || link.RootContract != 3 || h.Contract != link.OwnershipContract || - link.OwnershipContract != link.BusyContract+1 || link.BusyContract != link.OwnGroupContract+1 { + // At RootContract or later: an engine that also reads the provisioner client's account says ProvisionerContract + // (novox/hq ADR 0349 §4), one above ADR 0349 §5's OwnershipContract. + if h.Contract < link.RootContract || link.RootContract != 3 || h.Contract != link.ProvisionerContract || + link.ProvisionerContract != link.OwnershipContract+1 || link.OwnershipContract != link.BusyContract+1 || + link.BusyContract != link.OwnGroupContract+1 { t.Fatalf("contract %d", h.Contract) } got := map[string]string{} diff --git a/internal/apply/apply.go b/internal/apply/apply.go index 0305047..9b98d0b 100644 --- a/internal/apply/apply.go +++ b/internal/apply/apply.go @@ -999,34 +999,7 @@ func ApplyMindingWindows( kept, keptMode, keptOwner = from.Kept, from.KeptMode, from.KeptOwner } // Only now. The record follows the fact, never leads it. - known.Record(store.Applied{ - Origin: origin, - ID: resource.Identity(), Type: string(resource.Kind()), - Target: outcome.Target, AppliedAt: time.Now().UTC(), - Wrote: outcome.wrote, - Into: outcome.into, - Kept: kept, - KeptMode: keptMode, - KeptOwner: keptOwner, - Reads: outcome.reads, - BusySince: outcome.busySince, - Stateless: outcome.stateless, - Scope: outcome.scope, - User: outcome.user, - Found: outcome.found, - DeclaredUnit: outcome.declared, - Shell: outcome.shell, - Linger: outcome.linger, - Groups: outcome.groups, - Unpacked: outcome.unpacked, - AsFound: outcome.asFound, - Declared: outcome.asFoundDeclared, - FoundAt: outcome.asFoundAt, - Holds: holds(resource), - }) - if outcome.found != nil { - known.DropFound(resource.Identity()) - } + recordApplied(&known, origin, resource, outcome, kept, keptMode, keptOwner) // Its module has been taken, and what was held for it is now the mesh's. A file written // into, or a service whose lifecycle is the machine's, replaced nothing that was found, so // its outcome says what the apply did, not that a cutover happened; a hold from when it was @@ -1075,7 +1048,7 @@ func ApplyMindingWindows( if outcome.Action == heldStill || outcome.Action == waitsBusy { // Not changed: nothing about it moved, so nothing that restarts on it restarts. log(fmt.Sprintf(" %s %s (%s): %s", outcome.Action, outcome.ID, outcome.Target, outcome.Detail)) - } else if outcome.Action != "unchanged" { + } else if moved(outcome) { changed[resource.Identity()] = true // With the detail, when there is one: "updated app" says a container was replaced; // which file made that happen is what somebody reading the log at the time needs @@ -1168,6 +1141,47 @@ func ApplyMindingWindows( return report, known, nil } +// recordApplied records what applying a resource did, once it is done: the record follows the fact, never leads it. +// kept, keptMode and keptOwner are where the original of what it replaced was kept, and how it was found. What an +// earlier apply found its unit as goes with it. +func recordApplied(known *store.State, origin string, resource declaration.Resource, outcome Outcome, + kept, keptMode, keptOwner string) { + known.Record(store.Applied{ + Origin: origin, + ID: resource.Identity(), Type: string(resource.Kind()), + Target: outcome.Target, AppliedAt: time.Now().UTC(), + Wrote: outcome.wrote, + Into: outcome.into, + Kept: kept, + KeptMode: keptMode, + KeptOwner: keptOwner, + Reads: outcome.reads, + BusySince: outcome.busySince, + Stateless: outcome.stateless, + Scope: outcome.scope, + User: outcome.user, + Found: outcome.found, + DeclaredUnit: outcome.declared, + Shell: outcome.shell, + Linger: outcome.linger, + Groups: outcome.groups, + Unpacked: outcome.unpacked, + AsFound: outcome.asFound, + Declared: outcome.asFoundDeclared, + FoundAt: outcome.asFoundAt, + Holds: holds(resource), + }) + if outcome.found != nil { + known.DropFound(resource.Identity()) + } +} + +// moved reports whether applying a resource moved it, so what restarts or reloads on it does: anything but +// unchanged, and not a container held still or waiting for its busy one to say idle. +func moved(outcome Outcome) bool { + return outcome.Action != "unchanged" && outcome.Action != heldStill && outcome.Action != waitsBusy +} + // afterWhatTheyRead is the resources in declared order, except that **a service comes after every // resource it names under restart-on or reload-on** (novox/hq issue 260). // diff --git a/internal/declaration/declaration.go b/internal/declaration/declaration.go index 0c165c3..6ef97ec 100644 --- a/internal/declaration/declaration.go +++ b/internal/declaration/declaration.go @@ -1645,6 +1645,25 @@ type Declaration struct { // node-engine write only these. Absent is a controller that hands no file to the provisioner, and every // file is written from the declaration as before. Filled []string + + // Provisioner is the account the provisioner's client runs as on this machine, by name (novox/hq ADR 0349 §4): + // the only account whose write of a filled file the node-engine's local write socket accepts. Absent is a + // controller that names none, and every such write is refused. + Provisioner string +} + +// checkProvisioner refuses a provisioner client's account that is no account name, or is root: the account whose +// writes the node-engine accepts must be one that could not write the file without it. +func checkProvisioner(name string) []string { + switch { + case name == "": + return nil + case !accountName.MatchString(name): + return []string{fmt.Sprintf("provisioner_account %q is not an account name", name)} + case name == "root": + return []string{"provisioner_account is root; the provisioner's client runs as an account of its own"} + } + return nil } // Fills reports whether the provisioner fills the file at path on this machine. @@ -2027,6 +2046,9 @@ type envelope struct { // Filled is optional too (novox/hq ADR 0349 §5), as is a file's `owner-at-runtime`. **An older host refuses // these keys**; a controller sends them only to a host whose health says link.OwnershipContract. Filled []string `json:"filled_files,omitempty"` + // Provisioner is optional too (novox/hq ADR 0349 §4). **An older host refuses this key**; a controller sends it + // only to a host whose health says link.ProvisionerContract. + Provisioner string `json:"provisioner_account,omitempty"` } func parse(raw []byte, allowActions bool) (*Declaration, error) { @@ -2045,8 +2067,10 @@ func parse(raw []byte, allowActions bool) (*Declaration, error) { d := &Declaration{Version: env.Version, For: env.For, Adoption: env.Adoption, Sequence: env.Sequence, Epoch: env.Epoch, LeftOut: env.LeftOut, Generation: env.Generation, PutBack: env.PutBack, - Identities: env.Identities, Groups: env.Groups, Libraries: env.Libraries, Filled: env.Filled} + Identities: env.Identities, Groups: env.Groups, Libraries: env.Libraries, Filled: env.Filled, + Provisioner: env.Provisioner} problems := checkIdentities(env.Identities, env.Groups) + problems = append(problems, checkProvisioner(env.Provisioner)...) problems = append(problems, checkLibraries(env.Groups, env.Libraries)...) if env.Generation < 0 { // Below zero would read as "none claimed" and pass every refusal of an older generation. diff --git a/internal/link/messages.go b/internal/link/messages.go index 5bc1828..3a849a1 100644 --- a/internal/link/messages.go +++ b/internal/link/messages.go @@ -302,6 +302,12 @@ const BusyContract = 8 // fields; an older host refuses the whole declaration for them. const OwnershipContract = 9 +// ProvisionerContract is the statement of a host that also reads a declaration's `provisioner_account` (novox/hq ADR +// 0349 §4) and serves the local write socket: it accepts a write of a filled file from that account alone. Its +// presence is what tells the controller this host may be sent the field; an older host refuses the whole +// declaration for it. +const ProvisionerContract = 10 + // Health is one statement of every long-running resource's state on this machine (to-be 48 §4). Said in // every report, as an event on each change, and again every minute while anything is not healthy — so a // lost statement is not a lost fault. diff --git a/validate/validate_test.go b/validate/validate_test.go index 433850c..51ade73 100644 --- a/validate/validate_test.go +++ b/validate/validate_test.go @@ -24,7 +24,13 @@ func TestTheValidatorIsTheHosts(t *testing.T) { if problems := Declaration([]byte(owned)); problems != nil { t.Fatalf("a declaration saying who owns a file at runtime was refused: %v", problems) } + // And one naming the provisioner client's account (novox/hq ADR 0349 §4). + named := `{"declaration": 1, "provisioner_account": "mesh-node-provisioner", "filled_files": ["/etc/x"], "resources": [{"id": "f", "type": "file", "path": "/etc/x", "content": "a", "owner-at-runtime": "owned"}]}` + if problems := Declaration([]byte(named)); problems != nil { + t.Fatalf("a declaration naming the provisioner client's account was refused: %v", problems) + } for _, bad := range []string{ + `{"declaration": 1, "provisioner_account": "root", "resources": [{"id": "f", "type": "file", "path": "/etc/x", "content": "a"}]}`, `{"declaration": 1, "filled_files": ["/etc/none"], "resources": [{"id": "f", "type": "file", "path": "/etc/x", "content": "a", "owner-at-runtime": "owned"}]}`, `{"declaration": 1, "resources": []}`, `{"declaration": 99, "resources": [{"id": "d", "type": "directory", "path": "/x"}]}`, -- 2.54.0 From eb2ec23679609abdcac55a71f4ffd7ec79a1ca47 Mon Sep 17 00:00:00 2001 From: mesh-admin Date: Wed, 14 Oct 2026 00:34:54 +0000 Subject: [PATCH 05/11] Write a filled file on the provisioner client's request, and nothing else MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The provisioner's client (ADR 0349 §4) fills a module's templates and needs the node-engine to write them: it must not hold the right to write the machine's files itself. A second local socket accepts a write only from the account the declaration names as the provisioner client's (SO_PEERCRED, same namespaces), only for a clean path the declaration names as filled, under the apply's own locks, and writes it through the apply as the declared file would be, firing restart-on, reload-on and container reads. Issue 668. Co-Authored-By: Claude Opus 5.5 --- cmd/mesh-host/filling.go | 74 ++++++++++++++++ cmd/mesh-host/filling_test.go | 105 +++++++++++++++++++++++ internal/declaration/provisioner_test.go | 37 ++++++++ internal/meshcli/peer.go | 19 ++++ 4 files changed, 235 insertions(+) create mode 100644 cmd/mesh-host/filling.go create mode 100644 cmd/mesh-host/filling_test.go create mode 100644 internal/declaration/provisioner_test.go create mode 100644 internal/meshcli/peer.go diff --git a/cmd/mesh-host/filling.go b/cmd/mesh-host/filling.go new file mode 100644 index 0000000..22f6dcc --- /dev/null +++ b/cmd/mesh-host/filling.go @@ -0,0 +1,74 @@ +package main + +import ( + "context" + "crypto/ed25519" + "errors" + "path/filepath" + + "github.com/novox/mesh-host/internal/apply" + "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/filling" + "github.com/novox/mesh-host/internal/store" + "github.com/novox/mesh-host/internal/system" +) + +// serveFilling writes filled files on the provisioner client's request for as long as this engine serves (novox/hq +// ADR 0349 §4). A socket that cannot be served is said once, and this engine goes on applying. +func serveFilling(ctx context.Context, opts options, signer ed25519.PublicKey, say func(string)) { + if err := filling.Serve(ctx, filling.Socket, fillingEngine(opts, signer, say), say); err != nil { + say("filled files are not written on request on this machine: " + err.Error()) + } +} + +// fillingEngine is the write of a filled file as this engine does it: the declaration it keeps, verified, read under +// the apply's lock — the same two locks every apply takes, the process's and the one beside the state — and the file +// written through the apply (apply.WriteFilled), its outcome recorded in the node's state as an apply's is. +func fillingEngine(opts options, signer ed25519.PublicKey, say func(string)) *filling.Engine { + return &filling.Engine{ + Declared: func() (*declaration.Declaration, error) { + raw, err := store.LoadDeclared(store.DeclaredPath(opts.state), signer) + if err != nil { + return nil, err + } + return declaration.Parse(raw) + }, + UIDOf: filling.UIDOf, + Lock: func() (func(), error) { + applying.Lock() + unlock, err := store.Lock(opts.state, nil) + if err != nil { + applying.Unlock() + return nil, err + } + return func() { unlock(); applying.Unlock() }, nil + }, + Place: func(ctx context.Context, d *declaration.Declaration, path string, content []byte) (string, string, error) { + built, err := system.For(builtFor) + if err != nil { + return "", "", err + } + known, err := store.Load(opts.state) + if err != nil { + return "", "", err + } + report, updated, writeErr := apply.WriteFilled(ctx, built, d, known, path, content, apply.ExecRunner, + announceOr(say), apply.KeepIn(filepath.Dir(opts.state)), apply.WindowsIn(filepath.Dir(opts.state))) + // Saved whichever way it went, as an apply's: what was written is on the machine either way. + if err := store.Save(opts.state, updated); err != nil { + return "", "", errors.Join(writeErr, errors.New("written, and the node's state could not be saved: "+ + err.Error())) + } + action, digest := "", "" + for _, o := range report.Outcomes { + if o.Target == path && o.Type == string(declaration.TypeFile) { + action = o.Action + } + } + if rec, ok := updated.At(string(declaration.TypeFile), path); ok { + digest = rec.Wrote + } + return action, digest, writeErr + }, + } +} diff --git a/cmd/mesh-host/filling_test.go b/cmd/mesh-host/filling_test.go new file mode 100644 index 0000000..828682a --- /dev/null +++ b/cmd/mesh-host/filling_test.go @@ -0,0 +1,105 @@ +package main + +import ( + "context" + "crypto/ed25519" + "encoding/json" + "os" + "os/user" + "path/filepath" + "testing" + "time" + + "github.com/novox/mesh-host/internal/filling" + "github.com/novox/mesh-host/internal/store" +) + +// Defends novox/hq ADR 0349 §4 (issue 668): a write of a filled file and an apply are one at a time — the write +// takes the lock the apply holds, and writes only once the apply let it go — and the write is recorded in the +// node's state as an apply's outcome is. +func TestAWriteAndAnApplyOfTheSameFileSerialise(t *testing.T) { + was := builtFor + builtFor = "arch" + t.Cleanup(func() { builtFor = was }) + me, err := user.Current() + if err != nil { + t.Fatal(err) + } + if me.Uid == "0" { + t.Skip("run as root, whose account is never the provisioner's") + } + dir := t.TempDir() + opts := options{state: filepath.Join(dir, "state.json")} + public, private, err := ed25519.GenerateKey(nil) + if err != nil { + t.Fatal(err) + } + path := filepath.Join(dir, "m.conf") + raw, err := json.Marshal(map[string]any{"declaration": 1, "provisioner_account": me.Username, + "filled_files": []string{path}, "resources": []any{map[string]any{"id": "m.conf", "type": "file", + "path": path, "content": "from the declaration\n", "owner-at-runtime": "owned", "owner": me.Username}}}) + if err != nil { + t.Fatal(err) + } + if err := store.SaveDeclared(store.DeclaredPath(opts.state), + store.Declared{Declaration: raw, Signature: ed25519.Sign(private, raw)}); err != nil { + t.Fatal(err) + } + e := fillingEngine(opts, public, func(string) {}) + uid := uint32(os.Getuid()) + if err := e.Accepts(uid); err != nil { + t.Fatal(err) + } + + // An apply holds the lock. + applying.Lock() + done := make(chan filling.Answer) + go func() { + done <- e.Write(context.Background(), uid, filling.Request{Path: path, Content: []byte("filled\n")}) + }() + select { + case a := <-done: + applying.Unlock() + t.Fatalf("written while an apply held the lock: %+v", a) + case <-time.After(200 * time.Millisecond): + } + if _, err := os.Stat(path); err == nil { + applying.Unlock() + t.Fatal("the file was written while an apply held the lock") + } + applying.Unlock() + a := <-done + if a.Refused != "" || a.Failed != "" || a.Action != "created" { + t.Fatalf("the write after the apply: %+v", a) + } + got, err := os.ReadFile(path) + if err != nil || string(got) != "filled\n" { + t.Fatalf("the file holds %q (%v)", got, err) + } + known, err := store.Load(opts.state) + if err != nil { + t.Fatal(err) + } + if rec, ok := known.Find("m.conf"); !ok || rec.Wrote != a.Digest || rec.Target != path { + t.Errorf("the write is not recorded in the node's state: %+v, answered %q", rec, a.Digest) + } + + // And the cross-process lock beside the state too: a write waits for a `reconcile` run by hand. + unlock, err := store.Lock(opts.state, nil) + if err != nil { + t.Fatal(err) + } + go func() { + done <- e.Write(context.Background(), uid, filling.Request{Path: path, Content: []byte("again\n")}) + }() + select { + case a := <-done: + unlock() + t.Fatalf("written while another process held the apply's lock: %+v", a) + case <-time.After(200 * time.Millisecond): + } + unlock() + if a := <-done; a.Action != "updated" { + t.Fatalf("the second write: %+v", a) + } +} diff --git a/internal/declaration/provisioner_test.go b/internal/declaration/provisioner_test.go new file mode 100644 index 0000000..233aa40 --- /dev/null +++ b/internal/declaration/provisioner_test.go @@ -0,0 +1,37 @@ +package declaration + +import ( + "strings" + "testing" +) + +// Defends novox/hq ADR 0349 §4 (issue 668): the declaration names the account the provisioner's client runs as, +// the only one whose write of a filled file the node-engine accepts. Read here before any controller sends it. + +func TestTheDeclarationNamesTheProvisionersAccount(t *testing.T) { + d, err := Parse([]byte(`{"declaration":1,"provisioner_account":"mesh-node-provisioner","filled_files":["/etc/a"], + "resources":[{"id":"a","type":"file","path":"/etc/a","content":"a","owner-at-runtime":"owned"}]}`)) + if err != nil { + t.Fatal(err) + } + if d.Provisioner != "mesh-node-provisioner" { + t.Errorf("the provisioner's account read as %q", d.Provisioner) + } + d, err = Parse([]byte(`{"declaration":1,"resources":[{"id":"a","type":"file","path":"/etc/a","content":"a"}]}`)) + if err != nil { + t.Fatal(err) + } + if d.Provisioner != "" { + t.Errorf("a declaration naming no account names %q", d.Provisioner) + } +} + +func TestAProvisionersAccountThatIsNoAccountIsRefused(t *testing.T) { + for name, account := range map[string]string{"root": "root", "not a name": "a b", "a path": "../x"} { + _, err := Parse([]byte(`{"declaration":1,"provisioner_account":"` + account + `", + "resources":[{"id":"a","type":"file","path":"/etc/a","content":"a"}]}`)) + if err == nil || !strings.Contains(err.Error(), "provisioner_account") { + t.Errorf("%s: want a refusal naming provisioner_account, got %v", name, err) + } + } +} diff --git a/internal/meshcli/peer.go b/internal/meshcli/peer.go new file mode 100644 index 0000000..88397cd --- /dev/null +++ b/internal/meshcli/peer.go @@ -0,0 +1,19 @@ +package meshcli + +import "net" + +// The kernel's word on who is at the other end of a socket, shared with the node-engine's other local socket, the +// write of a filled file (internal/filling, novox/hq ADR 0349 §4): one reading of it, so the two never judge a peer +// differently. + +// PeerCredentials is the user id and process id of the process at the other end, as the kernel recorded them when +// it connected (SO_PEERCRED). Nothing the process says is read. +func PeerCredentials(conn *net.UnixConn) (uint32, int32, error) { return peerCredentials(conn) } + +// SameNamespaces refuses a process whose mount or user namespace is not the engine's: a user id is this machine's +// account only in this machine's namespaces. +func SameNamespaces(pid int32) error { return namespacesOf(pid) } + +// NotOwnDirectory says what is wrong with a socket's directory, or nothing: it must be a directory, not a link, +// owned by the account the engine runs as. +func NotOwnDirectory(dir string) string { return notOwnDirectory(dir) } -- 2.54.0 From 1d57bcd6f94a68cb380993e7aacffdb1eee18936 Mon Sep 17 00:00:00 2001 From: mesh-admin Date: Wed, 14 Oct 2026 00:36:04 +0000 Subject: [PATCH 06/11] Write a filled file through the apply, and move what reads it A filled file is the declared file with the client's content: applied through applyFile, applyInto or applyBlock by who owns it at runtime, with the apply's guards, Keep and record, then restart-on, reload-on and container reads applied as an apply does. Issue 668. Co-Authored-By: Claude Opus 5.5 --- internal/apply/filled.go | 203 ++++++++++++++++++++++++++++ internal/apply/filled_test.go | 247 ++++++++++++++++++++++++++++++++++ 2 files changed, 450 insertions(+) create mode 100644 internal/apply/filled.go create mode 100644 internal/apply/filled_test.go diff --git a/internal/apply/filled.go b/internal/apply/filled.go new file mode 100644 index 0000000..a0d750f --- /dev/null +++ b/internal/apply/filled.go @@ -0,0 +1,203 @@ +package apply + +import ( + "context" + "fmt" + "slices" + + "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/store" + "github.com/novox/mesh-host/internal/system" +) + +// WriteFilled writes what the provisioner's client filled into one file the declaration names as filled (novox/hq +// ADR 0349 §4), and makes what reads the file take the change, exactly as an apply that changed it would. +// +// **The declared file says everything but the content.** Its path, mode, owner and who owns it at runtime are the +// declaration's; the client's request carries the content alone. So the file is applied as the declared one with +// that content in place of the declaration's: an owned file whole through applyFile, a seed only when absent, a +// managed file's keys through applyInto or its block through applyBlock — the same guards on where nothing is +// placed, the same Keep of an original found there, the same record. What the declaration seals or fills with +// `${secret:}` the client has already filled, so the content is written as it came; a file that carried a secret +// keeps the mode a secret is written with when the declaration gives none. +// +// **Then what reads it, in the apply's order** (afterWhatTheyRead): a service naming it under restart-on is +// restarted and under reload-on reloaded, a process or container naming it under restart-on is restarted, and a +// running container that reads it at creation — an env-file, a file mounted into it — is recreated, because what +// it read is recorded by digest and this write changed the digest. Each of those is applied and recorded as an +// apply does, and what moves goes on moving what names it. Nothing else of the declaration is touched. +// +// The caller holds the apply's lock: a write and an apply never write one file at once. log is said the path, the +// action and the digest, never the content. +func WriteFilled( + ctx context.Context, + sys system.System, + d *declaration.Declaration, + known store.State, + path string, + content []byte, + run Runner, + log func(string), + keep Keep, + windows *Windows, +) (report Report, _ store.State, _ error) { + if log == nil { + log = func(string) {} + } + if !d.Fills(path) { + return report, known, &Error{Resource: path, Err: fmt.Errorf("this node's declaration does not name %s "+ + "as a file the provisioner fills, so nothing was written", path)} + } + var declared []*declaration.File + for _, r := range d.Resources { + if f, ok := r.(*declaration.File); ok && f.Path == path { + declared = append(declared, f) + } + } + if len(declared) != 1 { + // The parser refuses a filled path no file declares; two at one path would need the request to say whose + // keys or block it carries, and it carries the content only. + return report, known, &Error{Resource: path, Err: fmt.Errorf("%d resources declare %s, and a filled file "+ + "is written for exactly one, so nothing was written", len(declared), path)} + } + file := filledAs(declared[0], content) + if held, isHeld := known.HeldAt(file.ID); isHeld { + return report, known, &Error{Resource: file.ID, Err: fmt.Errorf("%s is held as found for a module not "+ + "yet taken (%s), so nothing was written", path, held.Why)} + } + + in := inputs{declares: map[string]string{}, known: &known, windows: windows, away: newStoreAway(windows, log), + groups: groupsWanted(d.Resources), handed: Handed{}} + refreshRuntimeRoots(ctx) + in.refused = refusedPlaces(d.Resources) + in.agentHomes = rootNeverHomes(d) + in.dirsBefore = map[string]bool{} + for _, r := range d.Resources { + in.declares[r.Identity()] = declaredDigest(r) + } + changed := map[string]bool{} + + // The file, as the apply loop applies it. + was, _ := known.Find(file.ID) + var keepFound Keep + if (was.ID == "" || was.Target != file.Path) && !known.Recorded(string(declaration.TypeFile), file.Path) { + keepFound = keep + } + outcome, err := applyOne(ctx, sys, file, run, changed, in, was, nil, keepFound) + if err != nil { + log(fmt.Sprintf(" failed %s (%s): %v", file.ID, file.Path, err)) + return report, known, &Error{Resource: file.ID, Err: err, Done: report} + } + recordKeeping(&known, file, outcome, was) + report.Outcomes = append(report.Outcomes, outcome) + if !moved(outcome) { + return report, known, nil + } + changed[file.ID] = true + line := fmt.Sprintf(" %s %s (%s), filled: %s", outcome.Action, file.ID, file.Path, outcome.wrote) + if outcome.Detail != "" { + line += ": " + outcome.Detail + } + log(line) + + // What reads it, in the apply's order, each moving what names it in turn. + var failures []*Error + for _, r := range afterWhatTheyRead(d.Resources) { + if r.Identity() == file.ID || !readsChange(r, path, changed) { + continue + } + if _, isHeld := known.HeldAt(r.Identity()); isHeld { + log(fmt.Sprintf(" held %s (%s): held as found for a module not yet taken, so it was not restarted "+ + "for %s", r.Identity(), r.Target(), file.ID)) + continue + } + was, _ := known.Find(r.Identity()) + o, err := applyOne(ctx, sys, r, run, changed, in, was, nil, nil) + if err != nil { + if o.found != nil { + known.KeepFound(r.Identity(), store.OriginDeclared, *o.found) + } + failures = append(failures, &Error{Resource: r.Identity(), Err: err}) + log(fmt.Sprintf(" failed %s (%s): %v", r.Identity(), o.Target, err)) + continue + } + if o.Action == "waiting" { + if o.found != nil { + known.KeepFound(r.Identity(), store.OriginDeclared, *o.found) + } + report.Outcomes = append(report.Outcomes, o) + log(fmt.Sprintf(" waiting %s (%s): %s", o.ID, o.Target, o.Detail)) + continue + } + recordKeeping(&known, r, o, was) + report.Outcomes = append(report.Outcomes, o) + if o.busy != nil { + report.Busy = append(report.Busy, *o.busy) + } + if moved(o) { + changed[r.Identity()] = true + line := fmt.Sprintf(" %s %s (%s)", o.Action, o.ID, o.Target) + if o.Detail != "" { + line += ": " + o.Detail + } + log(line) + } else if o.Action != "unchanged" { + log(fmt.Sprintf(" %s %s (%s): %s", o.Action, o.ID, o.Target, o.Detail)) + } + } + if len(failures) > 0 { + first := failures[0] + first.Done = report + first.Others = len(failures) - 1 + return report, known, first + } + return report, known, nil +} + +// filledAs is the declared file with the filled content in place of the declaration's. +func filledAs(declared *declaration.File, content []byte) *declaration.File { + f := *declared + f.Content, f.Sealed, f.Bytes, f.Secrets = string(content), "", "", nil + if f.Mode == "" && (declared.Secret() || len(declared.Secrets) > 0) { + // Written as the declaration would have written it: a file carrying a credential is not world-readable. + f.Mode = "0600" + } + return &f +} + +// readsChange reports whether applying r is what makes it take a change of what changed: a service's restart-on or +// reload-on, a process's or container's restart-on, or a running container that reads path at creation. +func readsChange(r declaration.Resource, path string, changed map[string]bool) bool { + switch res := r.(type) { + case *declaration.Service: + return len(restartedBy(res.RestartOn, changed)) > 0 || len(restartedBy(res.ReloadOn, changed)) > 0 + case *declaration.Process: + return len(restartedBy(res.RestartOn, changed)) > 0 + case *declaration.Container: + if len(restartedBy(res.RestartOn, changed)) > 0 { + return true + } + if res.RunOnce || res.Schedule != "" { + return false + } + if slices.Contains(res.EnvFile, path) { + return true + } + for _, v := range res.Volumes { + if mountSource(v) == path { + return true + } + } + } + return false +} + +// recordKeeping records r as applied, carrying where the original it replaced was kept from its record when this +// write kept none — for as long as the resource is recorded at the same target, as the apply loop carries it. +func recordKeeping(known *store.State, r declaration.Resource, o Outcome, was store.Applied) { + kept, keptMode, keptOwner := o.kept, o.keptMode, o.keptOwner + if kept == "" && was.Target == o.Target { + kept, keptMode, keptOwner = was.Kept, was.KeptMode, was.KeptOwner + } + recordApplied(known, store.OriginDeclared, r, o, kept, keptMode, keptOwner) +} diff --git a/internal/apply/filled_test.go b/internal/apply/filled_test.go new file mode 100644 index 0000000..b2e9003 --- /dev/null +++ b/internal/apply/filled_test.go @@ -0,0 +1,247 @@ +package apply + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/store" +) + +// Defends novox/hq ADR 0349 §4 (issue 668): the provisioner's client has the node-engine write a file it filled, +// and the node-engine writes it as an apply of the declared file with that content would — through applyFile, +// applyInto or applyBlock by who owns it at runtime — and makes what reads it take the change as an apply does. + +// filledDecl is a declaration naming path as filled, with the file's fields after id, type and path, and more +// resources after it. +func filledDecl(t *testing.T, path, fields, more string) *declaration.Declaration { + t.Helper() + if more != "" { + more = "," + more + } + return parseTrusted(t, fmt.Sprintf(`{"declaration":1,"provisioner_account":"mesh-node-provisioner", + "filled_files":[%q],"resources":[{"id":"m.conf","type":"file","path":%q,%s}%s]}`, path, path, fields, more)) +} + +func TestAWriteOfAPathNotFilledIsRefused(t *testing.T) { + dir := t.TempDir() + filled := filepath.Join(dir, "filled.conf") + other := filepath.Join(dir, "other.conf") + d := parseTrusted(t, fmt.Sprintf(`{"declaration":1,"filled_files":[%q],"resources":[ + {"id":"m.conf","type":"file","path":%q,"content":"x\n","owner-at-runtime":"owned"}, + {"id":"m.other","type":"file","path":%q,"content":"y\n"} + ]}`, filled, filled, other)) + for _, path := range []string{other, filepath.Join(dir, "nothing.conf")} { + _, state, err := WriteFilled(context.Background(), archHost(t), d, store.State{}, path, []byte("evil\n"), + noServices, nil, nil, nil) + if err == nil || !strings.Contains(err.Error(), "does not name") { + t.Errorf("%s: a write of a path the declaration does not fill was not refused: %v", path, err) + } + if _, err := os.Stat(path); err == nil && path != other { + t.Errorf("%s was written", path) + } + if len(state.Resources) != 0 { + t.Errorf("a refused write recorded something: %+v", state.Resources) + } + } + if _, err := os.Stat(other); err == nil { + t.Errorf("%s was written", other) + } +} + +// Each ownership is written by the write verb exactly as an apply of the declared file with the same content: the +// same action, the same bytes on disk, the same mode. +func TestAFilledFileIsWrittenAsItsOwnershipSays(t *testing.T) { + for _, c := range []struct { + name, fields, content, before, action string + }{ + {"owned, absent", `"content":"","mode":"0640","owner-at-runtime":"owned"`, "port = 8080\n", "", "created"}, + {"owned, over what is there", `"content":"","mode":"0640","owner-at-runtime":"owned"`, "port = 8080\n", + "port = 1\n", "updated"}, + {"seed, absent", `"content":"","mode":"0600","owner-at-runtime":"seed"`, "user default on\n", "", "created"}, + {"seed, present", `"content":"","mode":"0600","owner-at-runtime":"seed"`, "user default on\n", + "user default on\nuser app-one on\n", "kept"}, + {"managed json", `"into":"json","content":"{}","owner-at-runtime":"managed"`, + `{"insecure-registries":["10.42.0.1:5000"]}`, machinesOwn, "updated"}, + {"managed block", `"into":"block","content":"x\n","owner-at-runtime":"managed"`, "10.42.0.1 ace\n", + workstationHosts, "updated"}, + } { + t.Run(c.name, func(t *testing.T) { + at := func() string { + path := filepath.Join(t.TempDir(), "the-file") + if c.before != "" { + if err := os.WriteFile(path, []byte(c.before), 0o644); err != nil { + t.Fatal(err) + } + } + return path + } + written := at() + report, state, err := WriteFilled(context.Background(), archHost(t), filledDecl(t, written, c.fields, ""), + store.State{}, written, []byte(c.content), noServices, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + applied := at() + withContent := strings.Replace(c.fields, fmt.Sprintf(`"content":%q`, contentOf(t, c.fields)), + fmt.Sprintf(`"content":%q`, c.content), 1) + want, _, err := Apply(context.Background(), archHost(t), filledDecl(t, applied, withContent, ""), + store.State{}, store.OriginDeclared, noServices, nil, nil) + if err != nil { + t.Fatal(err) + } + if got := report.Outcomes[0].Action; got != c.action || got != want.Outcomes[0].Action { + t.Errorf("written %s; an apply of the same content %s; want %s", got, want.Outcomes[0].Action, c.action) + } + if got, apply := readText(t, written), readText(t, applied); got != apply { + t.Errorf("written, the file holds\n%q\nan apply of the same content leaves\n%q", got, apply) + } + if a, b := modeAt(t, written), modeAt(t, applied); a != b { + t.Errorf("written mode %o, an apply's %o", a, b) + } + if rec, ok := state.Find("m.conf"); !ok || rec.Target != written || rec.Origin != store.OriginDeclared { + t.Errorf("the write is not recorded as the file's: %+v", rec) + } + }) + } +} + +func contentOf(t *testing.T, fields string) string { + t.Helper() + var holder struct { + Content string `json:"content"` + } + if err := json.Unmarshal([]byte("{"+fields+"}"), &holder); err != nil { + t.Fatal(err) + } + return holder.Content +} + +func modeAt(t *testing.T, path string) os.FileMode { + t.Helper() + info, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + return info.Mode().Perm() +} + +// A file that carried a secret keeps the mode a secret is written with: the client filled the value, and the file +// is no more readable for it. +func TestAFilledFileThatCarriedASecretIsNotWorldReadable(t *testing.T) { + path := filepath.Join(t.TempDir(), "token.env") + d := filledDecl(t, path, `"content":"TOKEN=${secret:token}\n","secrets":{"token":"c2VhbGVk"},"owner-at-runtime":"owned"`, "") + if _, _, err := WriteFilled(context.Background(), archHost(t), d, store.State{}, path, []byte("TOKEN=abc\n"), + noServices, nil, nil, nil); err != nil { + t.Fatal(err) + } + if m := modeAt(t, path); m != 0o600 { + t.Errorf("a filled file that carried a secret is mode %o", m) + } +} + +func TestAWriteUnderReloadOnReloadsItsProgramOnce(t *testing.T) { + path := filepath.Join(t.TempDir(), "daemon.conf") + d := filledDecl(t, path, `"content":"","owner-at-runtime":"owned"`, + `{"id":"m.daemon","type":"service","unit":"m.service","state":"running","reload-on":["m.conf"]}, + {"id":"m.other","type":"service","unit":"other.service","state":"running"}`) + var commands []string + var said []string + _, state, err := WriteFilled(context.Background(), archHost(t), d, store.State{}, path, []byte("level = 1\n"), + recordingServices(&commands), func(line string) { said = append(said, line) }, nil, nil) + if err != nil { + t.Fatal(err) + } + if n := count(commands, "systemctl reload m.service"); n != 1 { + t.Errorf("the program was reloaded %d times; commands were %v", n, commands) + } + joined := strings.Join(commands, "\n") + if strings.Contains(joined, "restart m.service") || strings.Contains(joined, "stop m.service") { + t.Errorf("the program was restarted, not reloaded: %v", commands) + } + if strings.Contains(joined, "other.service") { + t.Errorf("a service that does not read the file was touched: %v", commands) + } + if _, ok := state.Find("m.daemon"); !ok { + t.Errorf("the reloaded service was not recorded") + } + // The log says the path and the digest, never the content. + for _, line := range said { + if strings.Contains(line, "level = 1") { + t.Errorf("the content was said: %q", line) + } + } + if !strings.Contains(strings.Join(said, "\n"), digestOf("level = 1\n")) { + t.Errorf("the digest was not said: %q", said) + } + + // The same content again moves nothing, and reloads nothing. + commands = nil + report, _, err := WriteFilled(context.Background(), archHost(t), d, state, path, []byte("level = 1\n"), + recordingServices(&commands), nil, nil, nil) + if err != nil { + t.Fatal(err) + } + if report.Outcomes[0].Action != "unchanged" || count(commands, "reload") != 0 { + t.Errorf("an unchanged write reloaded: %v %+v", commands, report.Outcomes) + } +} + +func TestAWriteUnderRestartOnRestartsItsProgram(t *testing.T) { + path := filepath.Join(t.TempDir(), "daemon.conf") + d := filledDecl(t, path, `"content":"","owner-at-runtime":"owned"`, + `{"id":"m.daemon","type":"service","unit":"m.service","state":"running","restart-on":["m.conf"]}`) + var commands []string + if _, _, err := WriteFilled(context.Background(), archHost(t), d, store.State{}, path, []byte("level = 1\n"), + recordingServices(&commands), nil, nil, nil); err != nil { + t.Fatal(err) + } + // A restart is a stop and a start, once. + stops, starts := count(commands, "systemctl stop m.service"), count(commands, "systemctl start m.service") + if stops != 1 || starts != 1 { + t.Errorf("the program was stopped %d and started %d times; commands were %v", stops, starts, commands) + } +} + +// A running container that reads the file at creation is recreated, as an apply recreates it for a changed file. +func TestAWriteOfAFileAContainerReadsRecreatesIt(t *testing.T) { + env := filepath.Join(t.TempDir(), "forge.env") + d := filledDecl(t, env, `"content":"","mode":"0600","owner-at-runtime":"owned"`, + `{"id":"forge.server","type":"container","name":"forge","image":"`+pinned+`","env-file":["`+env+`"]}`) + m := &machine{containers: map[string]*fakeContainer{}} + _, state, err := WriteFilled(context.Background(), archHost(t), d, store.State{}, env, []byte("PORT=5432\n"), + m.run, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + if !m.did("docker run") { + t.Fatalf("the container was not created: %v", m.asked) + } + m.asked = nil + report, _, err := WriteFilled(context.Background(), archHost(t), d, state, env, []byte("PORT=5433\n"), m.run, + nil, nil, nil) + if err != nil { + t.Fatal(err) + } + if !m.removed("forge") || !m.did("docker run") { + t.Fatalf("the container kept what it read before the write: %v", m.asked) + } + if o := outcomeOf(report, "forge.server"); o.Action != "updated" || !strings.Contains(o.Detail, env) { + t.Errorf("the recreate did not say which file changed: %+v", o) + } +} + +func count(lines []string, prefix string) int { + n := 0 + for _, l := range lines { + if strings.Contains(l, prefix) { + n++ + } + } + return n +} -- 2.54.0 From 499a687149beabb115c596e1d77eb1e30f6b2864 Mon Sep 17 00:00:00 2001 From: mesh-admin Date: Wed, 14 Oct 2026 00:37:29 +0000 Subject: [PATCH 07/11] Serve the local write of a filled file on its own socket A second unix socket beside mesh-cli's: the kernel's word on the asking account (SO_PEERCRED, same namespaces), that account judged against the declaration before its request is read, the path judged clean, absolute and filled, and the write done under the apply's lock. The journal has the path and digest, never the content. Issue 668. Co-Authored-By: Claude Opus 5.5 --- internal/filling/filling.go | 347 +++++++++++++++++++++++++++++++ internal/filling/filling_test.go | 263 +++++++++++++++++++++++ 2 files changed, 610 insertions(+) create mode 100644 internal/filling/filling.go create mode 100644 internal/filling/filling_test.go diff --git a/internal/filling/filling.go b/internal/filling/filling.go new file mode 100644 index 0000000..8e13902 --- /dev/null +++ b/internal/filling/filling.go @@ -0,0 +1,347 @@ +// Package filling is the node-engine's narrow local write verb (novox/hq ADR 0349 §4, design 57 P2): a socket on +// this machine through which the provisioner's client has a file it filled written, and nothing else. +// +// **The client fills; the node-engine writes.** The client renders a module's template with the values each domain +// owns and hands the node-engine the content; the node-engine writes it as the declared file, with the declared +// file's path, owner, mode and ownership at runtime, and makes what reads it take the change. So the one process that +// writes the machine's files is still the node-engine, and the client needs no right to write any of them. +// +// **Narrow by three checks, each from something the client cannot say for itself.** The account asking, from the +// kernel (SO_PEERCRED), in this machine's own namespaces, must be the account the node's current declaration names +// as the provisioner's client's; the path, clean and absolute, must be one the declaration names as filled; and the +// write takes the apply's lock, so a write and an apply never write one file at once. While the declaration names no +// account — every declaration before the provisioner's client has one — every write is refused. +// +// **No listening control surface** (novox/hq ADR 0004): the socket has no network address. And the content is never +// said: the journal has the path, the account and the content's digest. +package filling + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "os" + "os/user" + "path/filepath" + "strconv" + "sync" + "time" + + "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/meshcli" +) + +// Socket is where the node-engine accepts a write of a filled file, beside mesh-cli's socket. +const Socket = "/run/nox-mesh-host/fill.sock" + +// MaxRequest bounds one request, its content encoded in it: a configuration file, never an archive. +const MaxRequest = 8 << 20 + +// Request is one write, sent as one JSON object on one line: the path of a filled file and what the client filled +// into it (base64 in the JSON). The content only: the declaration says everything else about the file. +type Request struct { + Path string `json:"path"` + Content []byte `json:"content"` +} + +// Answer is what came of one request. Refused is a write that was not attempted, with why; Failed one that was +// attempted and did not complete, with why — the file, or something that reads it. Otherwise Action is what the +// write did to the file (created, updated, corrected, unchanged, kept) and Digest what the file holds now, as the +// node-engine records it. +type Answer struct { + Path string `json:"path,omitempty"` + Action string `json:"action,omitempty"` + Digest string `json:"digest,omitempty"` + Refused string `json:"refused,omitempty"` + Failed string `json:"failed,omitempty"` +} + +// Refusal is an answer that nothing was written, and why. +func Refusal(path, why string) Answer { return Answer{Path: path, Refused: why} } + +// Writer is the node-engine's side of the socket. +type Writer interface { + // Accepts judges the asking account before its request is read: nil when the current declaration names it as + // the provisioner's client's. + Accepts(uid uint32) error + // Write judges the request and writes it, holding the apply's lock. + Write(ctx context.Context, uid uint32, req Request) Answer +} + +// Engine is the Writer the node-engine serves: the declaration it applies now, read again under the apply's lock. +type Engine struct { + // Declared reads the declaration this node applies now, verified. + Declared func() (*declaration.Declaration, error) + // UIDOf is the user id of an account, by name, from this machine's user database. + UIDOf func(account string) (uint32, error) + // Lock takes the apply's lock and returns what releases it: the same the apply takes, so a write and an apply + // are one at a time. + Lock func() (func(), error) + // Place writes the content into the filled file at path, as the declared file, and makes what reads it take the + // change. Called holding the lock, with the declaration read under it. + Place func(ctx context.Context, d *declaration.Declaration, path string, content []byte) (action, digest string, + err error) +} + +// Accepts reads the declaration and judges the account. +func (e *Engine) Accepts(uid uint32) error { + d, err := e.Declared() + if err != nil { + return fmt.Errorf("this node's declaration cannot be read, so no write is accepted: %w", err) + } + return JudgeAccount(d, uid, e.UIDOf) +} + +// Write takes the apply's lock, reads the declaration under it, judges the account and the path again — the +// declaration may have changed since the request was accepted — and writes. +func (e *Engine) Write(ctx context.Context, uid uint32, req Request) Answer { + if err := JudgeShape(req.Path); err != nil { + return Refusal(req.Path, err.Error()) + } + unlock, err := e.Lock() + if err != nil { + return Refusal(req.Path, "the apply's lock could not be taken, so nothing was written: "+err.Error()) + } + defer unlock() + d, err := e.Declared() + if err != nil { + return Refusal(req.Path, "this node's declaration cannot be read, so nothing was written: "+err.Error()) + } + if err := JudgeAccount(d, uid, e.UIDOf); err != nil { + return Refusal(req.Path, err.Error()) + } + if err := JudgePath(d, req.Path); err != nil { + return Refusal(req.Path, err.Error()) + } + action, digest, err := e.Place(ctx, d, req.Path, req.Content) + if err != nil { + return Answer{Path: req.Path, Action: action, Digest: digest, Failed: err.Error()} + } + return Answer{Path: req.Path, Action: action, Digest: digest} +} + +// JudgeAccount refuses every account but the one the declaration names as the provisioner's client's, by its user +// id here — and every account while the declaration names none. +func JudgeAccount(d *declaration.Declaration, uid uint32, uidOf func(string) (uint32, error)) error { + if d.Provisioner == "" { + return errors.New("this node's declaration names no provisioner client's account, so no write is accepted") + } + want, err := uidOf(d.Provisioner) + if err != nil { + return fmt.Errorf("the provisioner client's account %s is not on this machine, so no write is accepted: %w", + d.Provisioner, err) + } + if want == 0 { + return fmt.Errorf("the provisioner client's account %s is root here, so no write is accepted", d.Provisioner) + } + if uid != want { + return fmt.Errorf("user id %d is not the provisioner client's account %s (user id %d), so nothing was written", + uid, d.Provisioner, want) + } + return nil +} + +// JudgeShape refuses a path that is not clean and absolute: a relative path, `..`, a doubled or trailing slash. +// Compared as written with the declaration's, a path that is not clean could name a file the declaration does not. +func JudgeShape(path string) error { + switch { + case path == "": + return errors.New("the request names no path, so nothing was written") + case !filepath.IsAbs(path): + return fmt.Errorf("%q is not an absolute path, so nothing was written", path) + case filepath.Clean(path) != path: + return fmt.Errorf("%q is not a clean path (it is %q cleaned), so nothing was written", path, + filepath.Clean(path)) + } + return nil +} + +// JudgePath refuses a path that is not clean and absolute, or that the declaration does not name as filled. Where it +// leads — a link on the way, a place nothing is written — is judged by the write, by the apply's own rules. +func JudgePath(d *declaration.Declaration, path string) error { + if err := JudgeShape(path); err != nil { + return err + } + if !d.Fills(path) { + return fmt.Errorf("%s is not a file this node's declaration names as filled, so nothing was written", path) + } + return nil +} + +// UIDOf is the user id of an account by name, from this machine's user database. +func UIDOf(account string) (uint32, error) { + u, err := user.Lookup(account) + if err != nil { + return 0, err + } + id, err := strconv.ParseUint(u.Uid, 10, 32) + if err != nil { + return 0, fmt.Errorf("user id %q: %w", u.Uid, err) + } + return uint32(id), nil +} + +// peerOf and sameNamespaces are the kernel's word on the asking process, as mesh-cli's socket reads it. Variables so +// a test can say. +var ( + peerOf = meshcli.PeerCredentials + sameNamespaces = meshcli.SameNamespaces +) + +// within bounds how long a connection may take to say its request, and refusedWithin how long a refused one is +// read and dropped before its refusal is written (so the client is not answered with a broken pipe). Variables so a +// test need not wait. +var ( + within = 30 * time.Second + refusedWithin = 1 * time.Second +) + +// atOnce bounds the connections served at the same time: one client, writing one file at a time. One over is +// refused busy at once. +var atOnce = 4 + +// Serve listens on socket until ctx ends, and answers each connection: one request, one answer. +// +// The directory is made 0755 and the socket 0666, as mesh-cli's: who may write is judged per connection from the +// kernel's word, against the current declaration, so the socket needs no account that may not exist yet. A file at +// the socket's path that is not a socket is never removed: the engine says so and serves nothing. +func Serve(ctx context.Context, socket string, w Writer, say func(string)) error { + if say == nil { + say = func(string) {} + } + dir := filepath.Dir(socket) + if err := os.MkdirAll(dir, 0o755); err != nil { + return fmt.Errorf("the filled files' socket directory: %w", err) + } + if problem := meshcli.NotOwnDirectory(dir); problem != "" { + return fmt.Errorf("filled files are not written on request on this machine: %s", problem) + } + if info, err := os.Lstat(socket); err == nil { + if info.Mode()&os.ModeSocket == 0 { + return fmt.Errorf("%s is there and is not a socket, and the node-engine did not put it there; filled "+ + "files are not written on request until it is moved", socket) + } + if err := os.Remove(socket); err != nil { + return fmt.Errorf("the socket a previous engine left at %s: %w", socket, err) + } + } + l, err := net.ListenUnix("unix", &net.UnixAddr{Name: socket, Net: "unix"}) + if err != nil { + return fmt.Errorf("listening for filled files at %s: %w", socket, err) + } + if err := os.Chmod(socket, 0o666); err != nil { + l.Close() + return fmt.Errorf("opening the filled files' socket: %w", err) + } + go func() { + <-ctx.Done() + l.Close() + }() + slots := make(chan struct{}, atOnce) + var wg sync.WaitGroup + defer wg.Wait() + for { + conn, err := l.AcceptUnix() + if err != nil { + if ctx.Err() != nil { + return nil + } + var ne net.Error + if errors.As(err, &ne) && ne.Timeout() { + continue + } + return fmt.Errorf("the filled files' socket: %w", err) + } + readBy := time.Now().Add(within) + _ = conn.SetDeadline(readBy) + wg.Add(1) + go func() { + defer wg.Done() + select { + case slots <- struct{}{}: + defer func() { <-slots }() + answer(ctx, conn, readBy, w, say) + default: + refuse(conn, readBy, Refusal("", "busy: other writes are being served; ask again"), say) + } + }() + } +} + +// answer serves one connection. +func answer(ctx context.Context, conn *net.UnixConn, readBy time.Time, w Writer, say func(string)) { + uid, pid, err := peerOf(conn) + if err != nil { + say("a filled file's write was refused: the asking account could not be read: " + err.Error()) + refuse(conn, readBy, Refusal("", "the node-engine could not read which account is asking, so nothing was "+ + "written: "+err.Error()), say) + return + } + if err := sameNamespaces(pid); err != nil { + say(fmt.Sprintf("a filled file's write from user id %d was refused: not in this machine's namespaces: %v", + uid, err)) + refuse(conn, readBy, Refusal("", "the asking process does not run in this machine's own namespaces, so "+ + "nothing was written: "+err.Error()), say) + return + } + if err := w.Accepts(uid); err != nil { + say(fmt.Sprintf("a filled file's write from user id %d was refused: %v", uid, err)) + refuse(conn, readBy, Refusal("", err.Error()), say) + return + } + var req Request + if err := json.NewDecoder(io.LimitReader(conn, MaxRequest)).Decode(&req); err != nil { + reply(conn, Refusal("", "the request could not be read, so nothing was written: "+err.Error()), say) + return + } + // The write waits for the apply's lock as long as an apply takes, and the client waits with it. + _ = conn.SetDeadline(time.Time{}) + a := w.Write(ctx, uid, req) + switch { + case a.Refused != "": + say(fmt.Sprintf("a write of %s from user id %d was refused: %s", req.Path, uid, a.Refused)) + case a.Failed != "": + say(fmt.Sprintf("a write of %s from user id %d failed: %s", req.Path, uid, a.Failed)) + default: + say(fmt.Sprintf("wrote %s for the provisioner's client (user id %d): %s, %s", req.Path, uid, a.Action, + a.Digest)) + } + _ = conn.SetWriteDeadline(time.Now().Add(within)) + reply(conn, a, say) +} + +// refuse reads and drops what a refused client wrote, briefly, and answers it: nothing it sent is kept. +func refuse(conn *net.UnixConn, readBy time.Time, a Answer, say func(string)) { + by := time.Now().Add(refusedWithin) + if readBy.Before(by) { + by = readBy + } + _ = conn.SetReadDeadline(by) + drain(conn) + _ = conn.SetWriteDeadline(time.Now().Add(within)) + reply(conn, a, say) +} + +// drain reads and drops a request up to its end — the newline after it, the end of the stream or the deadline — +// and at most MaxRequest bytes, keeping none of it. +func drain(r io.Reader) { + var buf [4096]byte + for read := 0; read < MaxRequest; { + n, err := r.Read(buf[:]) + read += n + if bytes.IndexByte(buf[:n], '\n') >= 0 || err != nil { + return + } + } +} + +func reply(conn *net.UnixConn, a Answer, say func(string)) { + defer conn.Close() + if err := json.NewEncoder(conn).Encode(a); err != nil { + say("a filled file's answer could not be written back: " + err.Error()) + } +} diff --git a/internal/filling/filling_test.go b/internal/filling/filling_test.go new file mode 100644 index 0000000..52a6f95 --- /dev/null +++ b/internal/filling/filling_test.go @@ -0,0 +1,263 @@ +package filling + +import ( + "context" + "encoding/json" + "errors" + "net" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/novox/mesh-host/internal/declaration" +) + +// Defends novox/hq ADR 0349 §4 (issue 668): the node-engine writes a filled file on request only for the provisioner +// client's account, only for a clean path the current declaration names as filled, and one at a time with the apply. + +const provisionerUID = 4242 + +func declared(t *testing.T, raw string) *declaration.Declaration { + t.Helper() + d, err := declaration.Parse([]byte(raw)) + if err != nil { + t.Fatal(err) + } + return d +} + +const withAccount = `{"declaration":1,"provisioner_account":"mesh-node-provisioner","filled_files":["/etc/m/m.conf"], + "resources":[{"id":"m.conf","type":"file","path":"/etc/m/m.conf","content":"x","owner-at-runtime":"owned"}, + {"id":"m.other","type":"file","path":"/etc/m/other.conf","content":"y"}]}` + +const withoutAccount = `{"declaration":1,"filled_files":["/etc/m/m.conf"], + "resources":[{"id":"m.conf","type":"file","path":"/etc/m/m.conf","content":"x","owner-at-runtime":"owned"}]}` + +// engine is an Engine over a fixed declaration, recording what it was asked to place. +type engine struct { + Engine + mu sync.Mutex + placed []string +} + +func newEngine(t *testing.T, raw string, lock sync.Locker) *engine { + d := declared(t, raw) + e := &engine{} + e.Engine = Engine{ + Declared: func() (*declaration.Declaration, error) { return d, nil }, + UIDOf: func(name string) (uint32, error) { + if name == "mesh-node-provisioner" { + return provisionerUID, nil + } + return 0, errors.New("no such account") + }, + Lock: func() (func(), error) { lock.Lock(); return lock.Unlock, nil }, + Place: func(_ context.Context, _ *declaration.Declaration, path string, content []byte) (string, string, error) { + e.mu.Lock() + defer e.mu.Unlock() + e.placed = append(e.placed, path) + return "updated", "digest", nil + }, + } + return e +} + +func TestAWriteFromTheProvisionersAccountOfAFilledPathIsPlaced(t *testing.T) { + e := newEngine(t, withAccount, &sync.Mutex{}) + if err := e.Accepts(provisionerUID); err != nil { + t.Fatal(err) + } + a := e.Write(context.Background(), provisionerUID, Request{Path: "/etc/m/m.conf", Content: []byte("x")}) + if a.Refused != "" || a.Failed != "" || a.Action != "updated" || len(e.placed) != 1 { + t.Fatalf("answer %+v, placed %v", a, e.placed) + } +} + +func TestAnUndeclaredPathIsRefused(t *testing.T) { + e := newEngine(t, withAccount, &sync.Mutex{}) + for _, path := range []string{"/etc/m/other.conf", "/etc/m/none.conf", "/etc/passwd"} { + a := e.Write(context.Background(), provisionerUID, Request{Path: path, Content: []byte("evil")}) + if a.Refused == "" || !strings.Contains(a.Refused, "not a file this node's declaration names as filled") { + t.Errorf("%s: %+v", path, a) + } + } + if len(e.placed) != 0 { + t.Errorf("placed %v", e.placed) + } +} + +func TestAnotherAccountIsRefused(t *testing.T) { + e := newEngine(t, withAccount, &sync.Mutex{}) + for _, uid := range []uint32{0, 1000, provisionerUID + 1} { + if err := e.Accepts(uid); err == nil { + t.Errorf("user id %d was accepted", uid) + } + a := e.Write(context.Background(), uid, Request{Path: "/etc/m/m.conf", Content: []byte("evil")}) + if a.Refused == "" || !strings.Contains(a.Refused, "is not the provisioner client's account") { + t.Errorf("user id %d: %+v", uid, a) + } + } + if len(e.placed) != 0 { + t.Errorf("placed %v", e.placed) + } +} + +func TestEveryWriteIsRefusedWhileTheDeclarationNamesNoAccount(t *testing.T) { + e := newEngine(t, withoutAccount, &sync.Mutex{}) + if err := e.Accepts(provisionerUID); err == nil || !strings.Contains(err.Error(), "names no provisioner") { + t.Errorf("accepted with no account named: %v", err) + } + a := e.Write(context.Background(), provisionerUID, Request{Path: "/etc/m/m.conf", Content: []byte("x")}) + if a.Refused == "" || len(e.placed) != 0 { + t.Errorf("written with no account named: %+v", a) + } +} + +func TestAnAccountThatIsRootHereIsRefused(t *testing.T) { + e := newEngine(t, withAccount, &sync.Mutex{}) + e.UIDOf = func(string) (uint32, error) { return 0, nil } + if err := e.Accepts(0); err == nil || !strings.Contains(err.Error(), "root") { + t.Errorf("an account that is root here was accepted: %v", err) + } +} + +func TestAnUncleanPathIsRefused(t *testing.T) { + e := newEngine(t, withAccount, &sync.Mutex{}) + for _, path := range []string{"", "etc/m/m.conf", "m.conf", "/etc//m/m.conf", "/etc/m/../m/m.conf", + "/etc/m/./m.conf", "/etc/m/m.conf/"} { + a := e.Write(context.Background(), provisionerUID, Request{Path: path, Content: []byte("x")}) + if a.Refused == "" { + t.Errorf("%q was not refused: %+v", path, a) + } + } + if len(e.placed) != 0 { + t.Errorf("placed %v", e.placed) + } +} + +// A write waits for an apply holding the apply's lock, and writes only once it is let go. +func TestAWriteWaitsForTheApply(t *testing.T) { + var applying sync.Mutex + e := newEngine(t, withAccount, &applying) + applying.Lock() + done := make(chan Answer) + go func() { + done <- e.Write(context.Background(), provisionerUID, Request{Path: "/etc/m/m.conf", Content: []byte("x")}) + }() + select { + case a := <-done: + t.Fatalf("written while the apply held the lock: %+v", a) + case <-time.After(100 * time.Millisecond): + } + e.mu.Lock() + n := len(e.placed) + e.mu.Unlock() + if n != 0 { + t.Fatalf("placed while the apply held the lock") + } + applying.Unlock() + if a := <-done; a.Action != "updated" { + t.Fatalf("after the apply: %+v", a) + } +} + +// fixed is a Writer that accepts one user id and answers every write. +type fixed struct { + uid uint32 + mu sync.Mutex + writes []Request +} + +func (f *fixed) Accepts(uid uint32) error { + if uid != f.uid { + return errors.New("not the provisioner client's account") + } + return nil +} + +func (f *fixed) Write(_ context.Context, _ uint32, req Request) Answer { + f.mu.Lock() + defer f.mu.Unlock() + f.writes = append(f.writes, req) + return Answer{Path: req.Path, Action: "updated", Digest: "d"} +} + +// serve serves w on a socket in a fresh directory and returns its path, and what was said. +func serve(t *testing.T, w Writer) (string, func() []string) { + t.Helper() + dir, err := os.MkdirTemp("", "fill") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { os.RemoveAll(dir) }) + socket := filepath.Join(dir, "fill.sock") + ctx, cancel := context.WithCancel(context.Background()) + var mu sync.Mutex + var said []string + stopped := make(chan error) + go func() { + stopped <- Serve(ctx, socket, w, func(s string) { mu.Lock(); said = append(said, s); mu.Unlock() }) + }() + t.Cleanup(func() { + cancel() + if err := <-stopped; err != nil { + t.Error(err) + } + }) + for i := 0; i < 100; i++ { + if _, err := os.Stat(socket); err == nil { + break + } + time.Sleep(10 * time.Millisecond) + } + return socket, func() []string { mu.Lock(); defer mu.Unlock(); return append([]string(nil), said...) } +} + +func ask(t *testing.T, socket string, req Request) Answer { + t.Helper() + conn, err := net.Dial("unix", socket) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + if err := json.NewEncoder(conn).Encode(req); err != nil { + t.Fatal(err) + } + var a Answer + if err := json.NewDecoder(conn).Decode(&a); err != nil { + t.Fatal(err) + } + return a +} + +// Over a real socket, the kernel says who is asking: this test's own account is the provisioner's here. +func TestTheSocketWritesForTheAccountTheKernelSays(t *testing.T) { + w := &fixed{uid: uint32(os.Getuid())} + socket, said := serve(t, w) + a := ask(t, socket, Request{Path: "/etc/m/m.conf", Content: []byte("secret value")}) + if a.Refused != "" || a.Action != "updated" || len(w.writes) != 1 || string(w.writes[0].Content) != "secret value" { + t.Fatalf("answer %+v, writes %+v", a, w.writes) + } + for _, line := range said() { + if strings.Contains(line, "secret value") { + t.Errorf("the content was said: %q", line) + } + } +} + +func TestTheSocketRefusesAnotherAccountBeforeReadingItsRequest(t *testing.T) { + w := &fixed{uid: uint32(os.Getuid()) + 1} + socket, said := serve(t, w) + a := ask(t, socket, Request{Path: "/etc/m/m.conf", Content: []byte("secret value")}) + if a.Refused == "" || len(w.writes) != 0 { + t.Fatalf("another account was written for: %+v", a) + } + for _, line := range said() { + if strings.Contains(line, "secret value") { + t.Errorf("the content was said: %q", line) + } + } +} -- 2.54.0 From 76dc93c421bf9738a130dd3b87dd7b49366e9878 Mon Sep 17 00:00:00 2001 From: mesh-admin Date: Wed, 14 Oct 2026 00:46:39 +0000 Subject: [PATCH 08/11] Hold a library's containers through the write verb, as a full apply does MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review 11959 of mesh-host#111: a write through the verb recreated a container that reads the filled file over a library whose disk was missing (ADR 0328 §9). Also refuses a filled path two resources declare, with a test. Issue 668. Co-Authored-By: Claude Opus 5.5 --- internal/apply/filled.go | 3 ++ internal/apply/filled_library_test.go | 57 +++++++++++++++++++++++++++ internal/apply/filled_test.go | 21 ++++++++++ internal/apply/library.go | 18 ++++++--- 4 files changed, 93 insertions(+), 6 deletions(-) create mode 100644 internal/apply/filled_library_test.go diff --git a/internal/apply/filled.go b/internal/apply/filled.go index a0d750f..f3f4800 100644 --- a/internal/apply/filled.go +++ b/internal/apply/filled.go @@ -72,6 +72,9 @@ func WriteFilled( in.refused = refusedPlaces(d.Resources) in.agentHomes = rootNeverHomes(d) in.dirsBefore = map[string]bool{} + // A container that reads the file and mounts a library whose disk or mount is not there is held, as a full apply + // holds it: not created over the missing disk (novox/hq ADR 0328 §9). + in.held = heldLibraries(ctx, d.Libraries) for _, r := range d.Resources { in.declares[r.Identity()] = declaredDigest(r) } diff --git a/internal/apply/filled_library_test.go b/internal/apply/filled_library_test.go new file mode 100644 index 0000000..a378c93 --- /dev/null +++ b/internal/apply/filled_library_test.go @@ -0,0 +1,57 @@ +//go:build linux + +package apply + +import ( + "context" + "fmt" + "path/filepath" + "strings" + "testing" + + "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/store" +) + +// A container that reads a filled file and mounts a library whose disk is not there is held by the write verb as a +// full apply holds it (novox/hq ADR 0328 §9, issue 668): never created or recreated over the missing disk. +func TestAWriteDoesNotRecreateAContainerHeldForItsLibrary(t *testing.T) { + // No folder is given the group: the library's root is absent, and nothing is made. + gid := 61000 + libraries(t, nil, nil) + base := t.TempDir() + root := filepath.Join(base, "gone", "media") + env := filepath.Join(base, "radarr.env") + raw := fmt.Sprintf(`{"declaration":1,"provisioner_account":"mesh-node-provisioner","filled_files":[%q], + "groups":[{"name":"mesh-media","gid":%d}],"libraries":[%s],"resources":[ + {"id":"radarr.env","type":"file","path":%q,"content":"","mode":"0600","owner-at-runtime":"owned"}, + {"id":"radarr.app","type":"container","name":"radarr","image":%q,"env-file":[%q],"volumes":[%q]}]}`, + env, gid, libJSON("media", root, gid, true, false, "movies"), env, pinned, env, + filepath.Join(root, "movies")+":/movies") + d, err := declaration.Parse([]byte(raw)) + if err != nil { + t.Fatal(err) + } + var ran []string + run := func(_ context.Context, name string, args ...string) (string, error) { + ran = append(ran, name+" "+strings.Join(args, " ")) + return "", nil + } + var state store.State + for _, content := range []string{"PORT=1\n", "PORT=2\n"} { + var report Report + report, state, err = WriteFilled(context.Background(), archHost(t), d, state, env, []byte(content), run, + nil, nil, nil) + if err != nil { + t.Fatal(err) + } + if o := outcomeOf(report, "radarr.app"); o.Action != "waiting" { + t.Errorf("after writing %q the container is %+v, not held for its library", content, o) + } + } + for _, line := range ran { + if strings.HasPrefix(line, "docker run") || strings.HasPrefix(line, "docker rm") { + t.Errorf("the container was created or removed over a missing disk: %v", ran) + } + } +} diff --git a/internal/apply/filled_test.go b/internal/apply/filled_test.go index b2e9003..6166d31 100644 --- a/internal/apply/filled_test.go +++ b/internal/apply/filled_test.go @@ -245,3 +245,24 @@ func count(lines []string, prefix string) int { } return n } + +// Two resources declaring one filled path — two modules' blocks in one file — are refused: the request carries the +// content only, so it cannot say whose block it is, and nothing is written. +func TestAFilledPathTwoResourcesDeclareIsRefused(t *testing.T) { + path := filepath.Join(t.TempDir(), "hosts") + if err := os.WriteFile(path, []byte(workstationHosts), 0o644); err != nil { + t.Fatal(err) + } + d := parseTrusted(t, fmt.Sprintf(`{"declaration":1,"filled_files":[%q],"resources":[ + {"id":"a.hosts","type":"file","path":%q,"into":"block","content":"10.42.0.1 ace\n","owner-at-runtime":"managed"}, + {"id":"b.hosts","type":"file","path":%q,"into":"block","content":"10.42.0.2 g14\n","owner-at-runtime":"managed"} + ]}`, path, path, path)) + _, state, err := WriteFilled(context.Background(), archHost(t), d, store.State{}, path, []byte("6.6.6.6 evil\n"), + noServices, nil, nil, nil) + if err == nil || !strings.Contains(err.Error(), "2 resources declare") { + t.Fatalf("a path two resources declare was not refused: %v", err) + } + if got := readText(t, path); got != workstationHosts || len(state.Resources) != 0 { + t.Errorf("something was written or recorded: %q %+v", got, state.Resources) + } +} diff --git a/internal/apply/library.go b/internal/apply/library.go index a198813..36a53b3 100644 --- a/internal/apply/library.go +++ b/internal/apply/library.go @@ -318,6 +318,17 @@ func heldRoots(verdicts map[string]libraryVerdict, libs []declaration.Library) m return out } +// heldLibraries is the root of every library whose disk or mount is not there, with its name, judged now and read +// only: what a full apply holds its containers for (ADR 0328 §9), for one that judges without applying libraries. +func heldLibraries(ctx context.Context, libs []declaration.Library) map[string]string { + verdicts := map[string]libraryVerdict{} + for _, l := range libs { + rec, _ := libraryRecord(l.Library) + verdicts[l.Library] = assessLibrary(ctx, l, rec) + } + return heldRoots(verdicts, libs) +} + // mountsHeld is the library a container waits for: one whose root it mounts, or mounts below, is held. func mountsHeld(c *declaration.Container, held map[string]string) string { for _, v := range c.Volumes { @@ -341,12 +352,7 @@ func WaitingOnLibrary(d *declaration.Declaration) map[string]bool { if d == nil || len(d.Libraries) == 0 { return out } - verdicts := map[string]libraryVerdict{} - for _, l := range d.Libraries { - rec, _ := libraryRecord(l.Library) - verdicts[l.Library] = assessLibrary(context.Background(), l, rec) - } - held := heldRoots(verdicts, d.Libraries) + held := heldLibraries(context.Background(), d.Libraries) for _, r := range d.Resources { if c, ok := r.(*declaration.Container); ok && mountsHeld(c, held) != "" { out[c.ID] = true -- 2.54.0 From 0c17c76493f774cafd82394ab78a02f0f35db226 Mon Sep 17 00:00:00 2001 From: mesh-admin Date: Wed, 14 Oct 2026 00:47:48 +0000 Subject: [PATCH 09/11] Judge a caller before it holds the socket, and keep accepting after a failed accept Review 11959 of mesh-host#111: any account could hold the four slots for a second each and starve the provisioner's client, and one failed accept ended the socket until the engine restarted. A caller is judged from the kernel's word before it holds a slot and, refused, is answered and closed at once; busy refusals that wait are bounded as mesh-cli's are; a failed accept is said, backed off and retried. The request refuses unknown fields, and the socket's directory is given its mode. Issue 668. Co-Authored-By: Claude Opus 5.5 --- internal/filling/filling.go | 166 ++++++++++++++++++++++-------- internal/filling/filling_test.go | 170 ++++++++++++++++++++++++++++++- 2 files changed, 295 insertions(+), 41 deletions(-) diff --git a/internal/filling/filling.go b/internal/filling/filling.go index 8e13902..e14da2a 100644 --- a/internal/filling/filling.go +++ b/internal/filling/filling.go @@ -192,23 +192,42 @@ var ( sameNamespaces = meshcli.SameNamespaces ) -// within bounds how long a connection may take to say its request, and refusedWithin how long a refused one is -// read and dropped before its refusal is written (so the client is not answered with a broken pipe). Variables so a -// test need not wait. +// within bounds how long an accepted connection may take to say its request, and refusedWithin how long a busy one +// is read and dropped before its refusal is written (so the client is not answered with a broken pipe). Variables so +// a test need not wait. var ( within = 30 * time.Second refusedWithin = 1 * time.Second ) -// atOnce bounds the connections served at the same time: one client, writing one file at a time. One over is -// refused busy at once. -var atOnce = 4 +// atOnce bounds the accepted connections served at the same time: one client, writing one file at a time. One over +// is refused busy. busyWaiting bounds how many busy refusals wait for their request at once; over it a busy refusal +// is written at once, as mesh-cli's socket does. Variables so a test can say. +var ( + atOnce = 4 + busyWaiting = make(chan struct{}, 8) + judgingAtOnce = 16 +) + +// acceptBackoff is how long the socket waits after an accept that failed before it accepts again, at first and at +// most: a socket that stopped at the first such failure (too many open files, say) would stop the client's writes +// until the engine restarted. Variables so a test need not wait. +var ( + acceptBackoff = 50 * time.Millisecond + acceptBackoffMax = 5 * time.Second +) + +// acceptFrom accepts one connection. A variable so a test can make it fail. +var acceptFrom = func(l *net.UnixListener) (*net.UnixConn, error) { return l.AcceptUnix() } // Serve listens on socket until ctx ends, and answers each connection: one request, one answer. // // The directory is made 0755 and the socket 0666, as mesh-cli's: who may write is judged per connection from the -// kernel's word, against the current declaration, so the socket needs no account that may not exist yet. A file at -// the socket's path that is not a socket is never removed: the engine says so and serves nothing. +// kernel's word, against the current declaration, so the socket needs no account that may not exist yet. **The +// caller is judged before it holds anything**: one the kernel does not say is the provisioner client's account is +// answered and closed at once, and only the client's connections are served or wait as busy, so no other account can +// hold the socket from it. A file at the socket's path that is not a socket is never removed: the engine says so and +// serves nothing. func Serve(ctx context.Context, socket string, w Writer, say func(string)) error { if say == nil { say = func(string) {} @@ -220,6 +239,9 @@ func Serve(ctx context.Context, socket string, w Writer, say func(string)) error if problem := meshcli.NotOwnDirectory(dir); problem != "" { return fmt.Errorf("filled files are not written on request on this machine: %s", problem) } + if err := os.Chmod(dir, 0o755); err != nil { + return fmt.Errorf("the filled files' socket directory: %w", err) + } if info, err := os.Lstat(socket); err == nil { if info.Mode()&os.ModeSocket == 0 { return fmt.Errorf("%s is there and is not a socket, and the node-engine did not put it there; filled "+ @@ -242,59 +264,117 @@ func Serve(ctx context.Context, socket string, w Writer, say func(string)) error l.Close() }() slots := make(chan struct{}, atOnce) + // Judging reads the declaration: at most judging callers are judged at once, and the rest wait their turn — a + // few milliseconds each — rather than being refused, so a flood delays the client's write and never stops it. + judging := make(chan struct{}, judgingAtOnce) + refusals := &refusalWords{say: say, last: map[uint32]time.Time{}} var wg sync.WaitGroup defer wg.Wait() + backoff := time.Duration(0) for { - conn, err := l.AcceptUnix() + conn, err := acceptFrom(l) if err != nil { if ctx.Err() != nil { return nil } - var ne net.Error - if errors.As(err, &ne) && ne.Timeout() { - continue + if errors.Is(err, net.ErrClosed) { + return fmt.Errorf("the filled files' socket was closed: %w", err) } - return fmt.Errorf("the filled files' socket: %w", err) + // Said, waited for, and accepted again: the failure is the machine's for a moment, not the socket's. + backoff = min(max(2*backoff, acceptBackoff), acceptBackoffMax) + say(fmt.Sprintf("the filled files' socket could not accept a connection (%v); accepting again in %s", + err, backoff)) + select { + case <-ctx.Done(): + return nil + case <-time.After(backoff): + } + continue } - readBy := time.Now().Add(within) - _ = conn.SetDeadline(readBy) + backoff = 0 wg.Add(1) go func() { defer wg.Done() + // Judged before it holds a slot or is waited for. + judging <- struct{}{} + uid, ok := judgePeer(conn, w, refusals) + <-judging + if !ok { + return + } + readBy := time.Now().Add(within) + _ = conn.SetDeadline(readBy) select { case slots <- struct{}{}: defer func() { <-slots }() - answer(ctx, conn, readBy, w, say) + answer(ctx, conn, uid, w, say) default: - refuse(conn, readBy, Refusal("", "busy: other writes are being served; ask again"), say) + busy(conn, readBy, say) } }() } } -// answer serves one connection. -func answer(ctx context.Context, conn *net.UnixConn, readBy time.Time, w Writer, say func(string)) { +// judgePeer reads the asking account from the kernel and judges it. A caller that is not the provisioner client's +// account is answered with why and closed at once, its request never read: false. +func judgePeer(conn *net.UnixConn, w Writer, refusals *refusalWords) (uint32, bool) { + closeWith := func(uid uint32, why string) (uint32, bool) { + refusals.said(uid, why) + _ = conn.SetWriteDeadline(time.Now().Add(refusedWithin)) + _ = json.NewEncoder(conn).Encode(Refusal("", why)) + conn.Close() + return uid, false + } uid, pid, err := peerOf(conn) if err != nil { - say("a filled file's write was refused: the asking account could not be read: " + err.Error()) - refuse(conn, readBy, Refusal("", "the node-engine could not read which account is asking, so nothing was "+ - "written: "+err.Error()), say) - return + return closeWith(^uint32(0), "the node-engine could not read which account is asking, so nothing was "+ + "written: "+err.Error()) } if err := sameNamespaces(pid); err != nil { - say(fmt.Sprintf("a filled file's write from user id %d was refused: not in this machine's namespaces: %v", - uid, err)) - refuse(conn, readBy, Refusal("", "the asking process does not run in this machine's own namespaces, so "+ - "nothing was written: "+err.Error()), say) - return + return closeWith(uid, "the asking process does not run in this machine's own namespaces, so nothing was "+ + "written: "+err.Error()) } if err := w.Accepts(uid); err != nil { - say(fmt.Sprintf("a filled file's write from user id %d was refused: %v", uid, err)) - refuse(conn, readBy, Refusal("", err.Error()), say) + return closeWith(uid, err.Error()) + } + return uid, true +} + +// refusalWords says a refused caller in the journal at most once a minute for each user id, with how many more +// were refused meanwhile: a flood from another account must not flood the journal either. +type refusalWords struct { + mu sync.Mutex + say func(string) + last map[uint32]time.Time + unsaid map[uint32]int +} + +func (r *refusalWords) said(uid uint32, why string) { + r.mu.Lock() + defer r.mu.Unlock() + if r.unsaid == nil { + r.unsaid = map[uint32]int{} + } + if at, ok := r.last[uid]; ok && time.Since(at) < time.Minute { + r.unsaid[uid]++ return } + more := "" + if n := r.unsaid[uid]; n > 0 { + more = fmt.Sprintf(" (and %d more from it since it was last said)", n) + } + r.last[uid], r.unsaid[uid] = time.Now(), 0 + r.say(fmt.Sprintf("a filled file's write from user id %d was refused%s: %s", uid, more, why)) +} + +// answer serves one connection from the provisioner client's account. +func answer(ctx context.Context, conn *net.UnixConn, uid uint32, w Writer, say func(string)) { var req Request - if err := json.NewDecoder(io.LimitReader(conn, MaxRequest)).Decode(&req); err != nil { + dec := json.NewDecoder(io.LimitReader(conn, MaxRequest)) + dec.DisallowUnknownFields() + if err := dec.Decode(&req); err != nil { + // Answered even when the deadline to say it passed: the deadline covered the write too. + _ = conn.SetWriteDeadline(time.Now().Add(refusedWithin)) reply(conn, Refusal("", "the request could not be read, so nothing was written: "+err.Error()), say) return } @@ -314,16 +394,22 @@ func answer(ctx context.Context, conn *net.UnixConn, readBy time.Time, w Writer, reply(conn, a, say) } -// refuse reads and drops what a refused client wrote, briefly, and answers it: nothing it sent is kept. -func refuse(conn *net.UnixConn, readBy time.Time, a Answer, say func(string)) { - by := time.Now().Add(refusedWithin) - if readBy.Before(by) { - by = readBy +// busy answers the client's connection over atOnce: its request read and dropped first, briefly, so the client reads +// the answer rather than a broken pipe — while no more than busyWaiting such refusals wait at once; over that, at once. +func busy(conn *net.UnixConn, readBy time.Time, say func(string)) { + select { + case busyWaiting <- struct{}{}: + by := time.Now().Add(refusedWithin) + if readBy.Before(by) { + by = readBy + } + _ = conn.SetReadDeadline(by) + drain(conn) + <-busyWaiting + default: } - _ = conn.SetReadDeadline(by) - drain(conn) - _ = conn.SetWriteDeadline(time.Now().Add(within)) - reply(conn, a, say) + _ = conn.SetWriteDeadline(time.Now().Add(refusedWithin)) + reply(conn, Refusal("", "busy: other writes are being served; ask again"), say) } // drain reads and drops a request up to its end — the newline after it, the end of the stream or the deadline — diff --git a/internal/filling/filling_test.go b/internal/filling/filling_test.go index 52a6f95..2c198ad 100644 --- a/internal/filling/filling_test.go +++ b/internal/filling/filling_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "errors" + "fmt" "net" "os" "path/filepath" @@ -251,7 +252,17 @@ func TestTheSocketWritesForTheAccountTheKernelSays(t *testing.T) { func TestTheSocketRefusesAnotherAccountBeforeReadingItsRequest(t *testing.T) { w := &fixed{uid: uint32(os.Getuid()) + 1} socket, said := serve(t, w) - a := ask(t, socket, Request{Path: "/etc/m/m.conf", Content: []byte("secret value")}) + conn, err := net.Dial("unix", socket) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + // Closed at once, its request never read: the write may meet a closed socket, and the refusal is there to read. + _ = json.NewEncoder(conn).Encode(Request{Path: "/etc/m/m.conf", Content: []byte("secret value")}) + var a Answer + if err := json.NewDecoder(conn).Decode(&a); err != nil { + t.Fatal(err) + } if a.Refused == "" || len(w.writes) != 0 { t.Fatalf("another account was written for: %+v", a) } @@ -261,3 +272,160 @@ func TestTheSocketRefusesAnotherAccountBeforeReadingItsRequest(t *testing.T) { } } } + +// floodersAreAnother has the kernel's word say another account for every connection a test binds to a name holding +// "flood": the test runs as one account, and the socket must tell the client from everyone else. +func floodersAreAnother(t *testing.T) { + t.Helper() + was := peerOf + peerOf = func(conn *net.UnixConn) (uint32, int32, error) { + uid, pid, err := was(conn) + if a, ok := conn.RemoteAddr().(*net.UnixAddr); ok && a != nil && strings.Contains(a.Name, "flood") { + uid++ + } + return uid, pid, err + } + t.Cleanup(func() { peerOf = was }) +} + +// A flood of connections from another account, each saying nothing and held open, does not stop the client's write: +// each is refused and closed before it holds anything. +func TestAFloodFromAnotherAccountDoesNotStopTheClient(t *testing.T) { + floodersAreAnother(t) + w := &fixed{uid: uint32(os.Getuid())} + socket, _ := serve(t, w) + from := t.TempDir() + var flood []*net.UnixConn + t.Cleanup(func() { + for _, c := range flood { + c.Close() + } + }) + for i := 0; i < 3*atOnce+busyWaitingCap(); i++ { + c, err := net.DialUnix("unix", &net.UnixAddr{Name: filepath.Join(from, fmt.Sprintf("flood-%d", i)), Net: "unix"}, + &net.UnixAddr{Name: socket, Net: "unix"}) + if err != nil { + t.Fatal(err) + } + flood = append(flood, c) + } + start := time.Now() + a := ask(t, socket, Request{Path: "/etc/m/m.conf", Content: []byte("x")}) + if a.Refused != "" || a.Action != "updated" { + t.Fatalf("the client's write during a flood: %+v", a) + } + if took := time.Since(start); took > refusedWithin/2 { + t.Errorf("the client's write waited %s behind the flood", took) + } + // And every flooder was closed: it reads its refusal and then the end. + for _, c := range flood { + _ = c.SetReadDeadline(time.Now().Add(2 * time.Second)) + var got Answer + if err := json.NewDecoder(c).Decode(&got); err != nil || got.Refused == "" { + t.Fatalf("a flooder was not refused at once: %+v %v", got, err) + } + } +} + +func busyWaitingCap() int { return cap(busyWaiting) } + +// A client that connects and says nothing holds one slot only until within, and another write goes on meanwhile. +func TestASilentClientDoesNotHoldTheSocket(t *testing.T) { + was := within + within = 300 * time.Millisecond + t.Cleanup(func() { within = was }) + w := &fixed{uid: uint32(os.Getuid())} + socket, _ := serve(t, w) + silent, err := net.Dial("unix", socket) + if err != nil { + t.Fatal(err) + } + defer silent.Close() + if a := ask(t, socket, Request{Path: "/etc/m/m.conf", Content: []byte("x")}); a.Action != "updated" { + t.Fatalf("a write beside a silent client: %+v", a) + } + _ = silent.SetReadDeadline(time.Now().Add(3 * time.Second)) + var got Answer + if err := json.NewDecoder(silent).Decode(&got); err != nil || got.Refused == "" { + t.Errorf("the silent client was not let go after within: %+v %v", got, err) + } +} + +// A request carrying more than the path and the content is refused: it says what the declaration says. +func TestARequestWithAnUnknownFieldIsRefused(t *testing.T) { + w := &fixed{uid: uint32(os.Getuid())} + socket, _ := serve(t, w) + conn, err := net.Dial("unix", socket) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + if _, err := conn.Write([]byte(`{"path":"/etc/m/m.conf","content":"eA==","mode":"0777"}` + "\n")); err != nil { + t.Fatal(err) + } + var a Answer + if err := json.NewDecoder(conn).Decode(&a); err != nil { + t.Fatal(err) + } + if a.Refused == "" || !strings.Contains(a.Refused, "mode") || len(w.writes) != 0 { + t.Errorf("a request with an unknown field: %+v, writes %v", a, w.writes) + } +} + +// An accept that fails is said, waited for and tried again: the socket goes on serving. +func TestTheSocketGoesOnAfterAnAcceptFails(t *testing.T) { + wasAccept, wasBackoff := acceptFrom, acceptBackoff + failures := 3 + var mu sync.Mutex + acceptFrom = func(l *net.UnixListener) (*net.UnixConn, error) { + mu.Lock() + defer mu.Unlock() + if failures > 0 { + failures-- + return nil, errors.New("accept: too many open files") + } + return l.AcceptUnix() + } + acceptBackoff = time.Millisecond + t.Cleanup(func() { acceptFrom, acceptBackoff = wasAccept, wasBackoff }) + w := &fixed{uid: uint32(os.Getuid())} + socket, said := serve(t, w) + if a := ask(t, socket, Request{Path: "/etc/m/m.conf", Content: []byte("x")}); a.Action != "updated" { + t.Fatalf("after failed accepts: %+v", a) + } + if !strings.Contains(strings.Join(said(), "\n"), "too many open files") { + t.Errorf("the failed accepts were not said: %v", said()) + } +} + +// The socket's directory is given the mode the socket needs, as mesh-cli's is. +func TestTheSocketsDirectoryIsOpenedToEveryAccount(t *testing.T) { + dir, err := os.MkdirTemp("", "fill") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { os.RemoveAll(dir) }) + if err := os.Chmod(dir, 0o700); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + stopped := make(chan error) + go func() { stopped <- Serve(ctx, filepath.Join(dir, "fill.sock"), &fixed{}, nil) }() + for i := 0; i < 100; i++ { + if _, err := os.Stat(filepath.Join(dir, "fill.sock")); err == nil { + break + } + time.Sleep(10 * time.Millisecond) + } + cancel() + if err := <-stopped; err != nil { + t.Fatal(err) + } + info, err := os.Stat(dir) + if err != nil { + t.Fatal(err) + } + if info.Mode().Perm() != 0o755 { + t.Errorf("the socket's directory is %o", info.Mode().Perm()) + } +} -- 2.54.0 From 2767278228bc3812004c16a3b55dd08fa5a7f791 Mon Sep 17 00:00:00 2001 From: mesh-admin Date: Wed, 14 Oct 2026 01:06:47 +0000 Subject: [PATCH 10/11] Say every refusal kind at once and never lose a count; bound the waiting and the busy per socket Review 12004 of mesh-host#111: refusals counted after a said one reached the journal only if the same account was refused again a minute later, and a refusal of another kind hid behind it. The first of each (account, kind) is said at once, and what is counted is said on a timer and when the socket stops. Connections waiting to be judged are bounded below the open-file limit, the busy refusals belong to one Serve, and the tests' ask has a deadline. Issue 668. Co-Authored-By: Claude Opus 5.5 --- internal/filling/filling.go | 137 ++++++++++++++++++++++++------- internal/filling/filling_test.go | 132 ++++++++++++++++++++++++++++- 2 files changed, 239 insertions(+), 30 deletions(-) diff --git a/internal/filling/filling.go b/internal/filling/filling.go index e14da2a..fc26c5e 100644 --- a/internal/filling/filling.go +++ b/internal/filling/filling.go @@ -27,8 +27,10 @@ import ( "os" "os/user" "path/filepath" + "sort" "strconv" "sync" + "sync/atomic" "time" "github.com/novox/mesh-host/internal/declaration" @@ -201,14 +203,22 @@ var ( ) // atOnce bounds the accepted connections served at the same time: one client, writing one file at a time. One over -// is refused busy. busyWaiting bounds how many busy refusals wait for their request at once; over it a busy refusal -// is written at once, as mesh-cli's socket does. Variables so a test can say. +// is refused busy. busyWaitingAtOnce bounds how many busy refusals wait for their request at once, per Serve; over it +// a busy refusal is written at once, as mesh-cli's socket does. judgingAtOnce bounds the callers judged at once, and +// judgeWaiting the connections waiting to be judged — each an open file, so far below the engine's open-file limit; +// one over is closed at once, unread. Variables so a test can say. var ( - atOnce = 4 - busyWaiting = make(chan struct{}, 8) - judgingAtOnce = 16 + atOnce = 4 + busyWaitingAtOnce = 8 + judgingAtOnce = 16 + judgeWaiting = 256 ) +// refusalsSaidEvery is how often the refusals counted and not yet said are said: each kind of refusal of each account +// is said at once the first time, and what follows it within this time is counted and said at its end, or when the +// socket stops. A variable so a test need not wait. +var refusalsSaidEvery = time.Minute + // acceptBackoff is how long the socket waits after an accept that failed before it accepts again, at first and at // most: a socket that stopped at the first such failure (too many open files, say) would stop the client's writes // until the engine restarted. Variables so a test need not wait. @@ -267,9 +277,27 @@ func Serve(ctx context.Context, socket string, w Writer, say func(string)) error // Judging reads the declaration: at most judging callers are judged at once, and the rest wait their turn — a // few milliseconds each — rather than being refused, so a flood delays the client's write and never stops it. judging := make(chan struct{}, judgingAtOnce) - refusals := &refusalWords{say: say, last: map[uint32]time.Time{}} + var waiting atomic.Int64 + busyWaiting := make(chan struct{}, busyWaitingAtOnce) + refusals := &refusalWords{say: say} + // Said once everything has stopped: what was counted is never lost with the socket. + defer refusals.flush() var wg sync.WaitGroup defer wg.Wait() + wg.Add(1) + go func() { + defer wg.Done() + tick := time.NewTicker(refusalsSaidEvery) + defer tick.Stop() + for { + select { + case <-ctx.Done(): + return + case <-tick.C: + refusals.flush() + } + } + }() backoff := time.Duration(0) for { conn, err := acceptFrom(l) @@ -292,6 +320,13 @@ func Serve(ctx context.Context, socket string, w Writer, say func(string)) error continue } backoff = 0 + if waiting.Add(1) > int64(judgeWaiting) { + // Too many waiting to be judged already: this one is closed unread, and counted. + waiting.Add(-1) + refusals.said(^uint32(0), "overflow", "too many connections are waiting to be judged; closed unread") + conn.Close() + continue + } wg.Add(1) go func() { defer wg.Done() @@ -299,6 +334,7 @@ func Serve(ctx context.Context, socket string, w Writer, say func(string)) error judging <- struct{}{} uid, ok := judgePeer(conn, w, refusals) <-judging + waiting.Add(-1) if !ok { return } @@ -309,7 +345,7 @@ func Serve(ctx context.Context, socket string, w Writer, say func(string)) error defer func() { <-slots }() answer(ctx, conn, uid, w, say) default: - busy(conn, readBy, say) + busy(conn, readBy, busyWaiting, say) } }() } @@ -318,8 +354,8 @@ func Serve(ctx context.Context, socket string, w Writer, say func(string)) error // judgePeer reads the asking account from the kernel and judges it. A caller that is not the provisioner client's // account is answered with why and closed at once, its request never read: false. func judgePeer(conn *net.UnixConn, w Writer, refusals *refusalWords) (uint32, bool) { - closeWith := func(uid uint32, why string) (uint32, bool) { - refusals.said(uid, why) + closeWith := func(uid uint32, kind, why string) (uint32, bool) { + refusals.said(uid, kind, why) _ = conn.SetWriteDeadline(time.Now().Add(refusedWithin)) _ = json.NewEncoder(conn).Encode(Refusal("", why)) conn.Close() @@ -327,44 +363,87 @@ func judgePeer(conn *net.UnixConn, w Writer, refusals *refusalWords) (uint32, bo } uid, pid, err := peerOf(conn) if err != nil { - return closeWith(^uint32(0), "the node-engine could not read which account is asking, so nothing was "+ + return closeWith(^uint32(0), "unread", "the node-engine could not read which account is asking, so nothing was "+ "written: "+err.Error()) } if err := sameNamespaces(pid); err != nil { - return closeWith(uid, "the asking process does not run in this machine's own namespaces, so nothing was "+ + return closeWith(uid, "namespace", "the asking process does not run in this machine's own namespaces, so nothing was "+ "written: "+err.Error()) } if err := w.Accepts(uid); err != nil { - return closeWith(uid, err.Error()) + return closeWith(uid, "account", err.Error()) } return uid, true } -// refusalWords says a refused caller in the journal at most once a minute for each user id, with how many more -// were refused meanwhile: a flood from another account must not flood the journal either. +// refusalWords says refused callers in the journal without letting a flood flood it: the first refusal of each kind +// for each account at once, and those that follow it counted and said by flush — on a timer and when the socket stops +// — so an account that stops still has its count said, and a refusal of another kind never hides behind one said. type refusalWords struct { - mu sync.Mutex - say func(string) - last map[uint32]time.Time - unsaid map[uint32]int + mu sync.Mutex + say func(string) + pending map[refusalKey]*refusalCount } -func (r *refusalWords) said(uid uint32, why string) { +type refusalKey struct { + uid uint32 + kind string +} + +type refusalCount struct { + n int + why string +} + +func (r *refusalWords) said(uid uint32, kind, why string) { r.mu.Lock() defer r.mu.Unlock() - if r.unsaid == nil { - r.unsaid = map[uint32]int{} + if r.pending == nil { + r.pending = map[refusalKey]*refusalCount{} } - if at, ok := r.last[uid]; ok && time.Since(at) < time.Minute { - r.unsaid[uid]++ + k := refusalKey{uid, kind} + if c, ok := r.pending[k]; ok { + c.n++ + c.why = why return } - more := "" - if n := r.unsaid[uid]; n > 0 { - more = fmt.Sprintf(" (and %d more from it since it was last said)", n) + r.pending[k] = &refusalCount{} + r.say(fmt.Sprintf("a filled file's write from %s was refused (%s): %s", who(k), kind, why)) +} + +// flush says what was counted since each was last said, and forgets every key, so the next refusal of each is said at +// once again. +func (r *refusalWords) flush() { + r.mu.Lock() + defer r.mu.Unlock() + keys := make([]refusalKey, 0, len(r.pending)) + for k := range r.pending { + keys = append(keys, k) } - r.last[uid], r.unsaid[uid] = time.Now(), 0 - r.say(fmt.Sprintf("a filled file's write from user id %d was refused%s: %s", uid, more, why)) + sort.Slice(keys, func(i, j int) bool { + if keys[i].uid != keys[j].uid { + return keys[i].uid < keys[j].uid + } + return keys[i].kind < keys[j].kind + }) + for _, k := range keys { + if c := r.pending[k]; c.n > 0 { + r.say(fmt.Sprintf("%d more writes of a filled file from %s were refused (%s) since it was last said; the last: %s", + c.n, who(k), k.kind, c.why)) + } + } + r.pending = nil +} + +// who names a caller by its user id, or says why it has none. +func who(k refusalKey) string { + switch { + case k.kind == "overflow": + return "a caller not yet judged" + case k.uid == ^uint32(0): + return "an account that could not be read" + } + return fmt.Sprintf("user id %d", k.uid) } // answer serves one connection from the provisioner client's account. @@ -396,7 +475,7 @@ func answer(ctx context.Context, conn *net.UnixConn, uid uint32, w Writer, say f // busy answers the client's connection over atOnce: its request read and dropped first, briefly, so the client reads // the answer rather than a broken pipe — while no more than busyWaiting such refusals wait at once; over that, at once. -func busy(conn *net.UnixConn, readBy time.Time, say func(string)) { +func busy(conn *net.UnixConn, readBy time.Time, busyWaiting chan struct{}, say func(string)) { select { case busyWaiting <- struct{}{}: by := time.Now().Add(refusedWithin) diff --git a/internal/filling/filling_test.go b/internal/filling/filling_test.go index 2c198ad..0e7cff3 100644 --- a/internal/filling/filling_test.go +++ b/internal/filling/filling_test.go @@ -224,6 +224,8 @@ func ask(t *testing.T, socket string, req Request) Answer { t.Fatal(err) } defer conn.Close() + // Bounded: a socket that never answers fails the test instead of hanging it. + _ = conn.SetDeadline(time.Now().Add(5 * time.Second)) if err := json.NewEncoder(conn).Encode(req); err != nil { t.Fatal(err) } @@ -327,7 +329,7 @@ func TestAFloodFromAnotherAccountDoesNotStopTheClient(t *testing.T) { } } -func busyWaitingCap() int { return cap(busyWaiting) } +func busyWaitingCap() int { return busyWaitingAtOnce } // A client that connects and says nothing holds one slot only until within, and another write goes on meanwhile. func TestASilentClientDoesNotHoldTheSocket(t *testing.T) { @@ -429,3 +431,131 @@ func TestTheSocketsDirectoryIsOpenedToEveryAccount(t *testing.T) { t.Errorf("the socket's directory is %o", info.Mode().Perm()) } } + +// The first refusal of each kind for each account is said at once; what follows within the time is counted, and the +// count is said when the time is up, even when the account has stopped. +func TestRefusalsAreSaidOnceAndTheirCountIsNeverLost(t *testing.T) { + var mu sync.Mutex + var said []string + r := &refusalWords{say: func(s string) { mu.Lock(); said = append(said, s); mu.Unlock() }} + for i := 0; i < 3; i++ { + r.said(1000, "account", "not the provisioner client's account") + } + // Another kind from the same account is said at once, not hidden behind the first. + r.said(1000, "namespace", "another namespace") + if len(said) != 2 || !strings.Contains(said[0], "(account)") || !strings.Contains(said[1], "(namespace)") { + t.Fatalf("said at once: %q", said) + } + // The account stops; the time is up; the two counted are said. + r.flush() + if len(said) != 3 || !strings.Contains(said[2], "2 more") || !strings.Contains(said[2], "user id 1000") { + t.Fatalf("after the time: %q", said) + } + // And the next refusal is said at once again. + r.said(1000, "account", "not the provisioner client's account") + if len(said) != 4 { + t.Fatalf("after the count was said: %q", said) + } +} + +// refusingSocket serves a socket that refuses this test's account, with counted refusals said every every. It returns +// how to be refused n times, everything said so far, and how to stop it. +func refusingSocket(t *testing.T, every time.Duration) (func(int), func() string, func()) { + t.Helper() + was := refusalsSaidEvery + refusalsSaidEvery = every + dir, err := os.MkdirTemp("", "fill") + if err != nil { + t.Fatal(err) + } + socket := filepath.Join(dir, "fill.sock") + ctx, cancel := context.WithCancel(context.Background()) + var mu sync.Mutex + var said []string + all := func() string { mu.Lock(); defer mu.Unlock(); return strings.Join(said, "\n") } + stopped := make(chan error) + go func() { + stopped <- Serve(ctx, socket, &fixed{uid: uint32(os.Getuid()) + 1}, + func(s string) { mu.Lock(); said = append(said, s); mu.Unlock() }) + }() + for i := 0; i < 100; i++ { + if _, err := os.Stat(socket); err == nil { + break + } + time.Sleep(10 * time.Millisecond) + } + refuse := func(n int) { + for i := 0; i < n; i++ { + conn, err := net.Dial("unix", socket) + if err != nil { + t.Fatal(err) + } + _ = conn.SetDeadline(time.Now().Add(5 * time.Second)) + var a Answer + _ = json.NewDecoder(conn).Decode(&a) + conn.Close() + } + } + var once sync.Once + stop := func() { + once.Do(func() { + cancel() + if err := <-stopped; err != nil { + t.Error(err) + } + refusalsSaidEvery = was + os.RemoveAll(dir) + }) + } + t.Cleanup(stop) + return refuse, all, stop +} + +// Refusals counted while the socket serves are said on its timer, though the account has stopped asking. +func TestTheSocketSaysItsCountedRefusalsOnATimer(t *testing.T) { + refuse, all, _ := refusingSocket(t, 200*time.Millisecond) + refuse(3) + // "more writes", not a number: a tick may fall between two refusals and split the count. + deadline := time.Now().Add(3 * time.Second) + for !strings.Contains(all(), "more writes") && time.Now().Before(deadline) { + time.Sleep(20 * time.Millisecond) + } + if !strings.Contains(all(), "more writes") { + t.Fatalf("the counted refusals were not said on the timer: %s", all()) + } +} + +// Refusals counted when the socket stops are said as it stops. +func TestTheSocketSaysItsCountedRefusalsWhenItStops(t *testing.T) { + refuse, all, stop := refusingSocket(t, time.Hour) + refuse(5) + if strings.Contains(all(), "4 more") { + t.Fatalf("said before the socket stopped: %s", all()) + } + stop() + if !strings.Contains(all(), "4 more") { + t.Errorf("the refusals counted when the socket stopped were lost: %s", all()) + } +} + +// A connection over the bound of those waiting to be judged is closed unread, and said. +func TestAConnectionOverTheWaitingBoundIsClosedUnread(t *testing.T) { + was := judgeWaiting + judgeWaiting = 0 + t.Cleanup(func() { judgeWaiting = was }) + w := &fixed{uid: uint32(os.Getuid())} + socket, said := serve(t, w) + conn, err := net.Dial("unix", socket) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + _ = conn.SetDeadline(time.Now().Add(5 * time.Second)) + var a Answer + if err := json.NewDecoder(conn).Decode(&a); err == nil { + t.Fatalf("answered over the bound: %+v", a) + } + if len(w.writes) != 0 || !strings.Contains(strings.Join(said(), "\n"), "(overflow)") { + t.Errorf("writes %v, said %q", w.writes, said()) + } +} -- 2.54.0 From 5a7ec3775b988e315a22959e640a9ace299a5cef Mon Sep 17 00:00:00 2001 From: mesh-tools Date: Thu, 15 Oct 2026 21:13:28 +0000 Subject: [PATCH 11/11] Land feature 57's node-engine work on its release branch The node-engine renders the feature's templates, collections and moved providers' files, on the SDK the feature's modules build on (feature 57, config and provisioning). --- cmd/mesh-host/main.go | 24 ++- cmd/mesh-host/movesecret.go | 131 +++++++++++++++++ cmd/mesh-host/movesecret_test.go | 134 +++++++++++++++++ cmd/mesh-host/nodekey.go | 62 ++++++++ cmd/mesh-host/nodekey_test.go | 112 ++++++++++++++ cmd/mesh-host/units_test.go | 10 +- cmd/mesh-host/vaultbox_nats_test.go | 154 ++++++++++++++++++++ examples/foundation-first-node.lock | 2 +- go.mod | 2 +- go.sum | 8 +- internal/accounts/accounts.go | 2 +- internal/apply/apply.go | 26 +++- internal/apply/filled.go | 26 +++- internal/apply/filled_apply_test.go | 218 ++++++++++++++++++++++++++++ internal/apply/filled_test.go | 7 +- internal/apply/vault_sealed_test.go | 112 ++++++++++++++ internal/declaration/declaration.go | 22 ++- internal/identity/sealing.go | 64 ++++++++ internal/link/handover.go | 9 +- internal/link/hearing_nats.go | 23 +++ internal/link/messages.go | 22 +++ internal/link/movesecret.go | 209 ++++++++++++++++++++++++++ internal/link/movesecret_test.go | 141 ++++++++++++++++++ internal/link/queue.go | 7 +- internal/link/run.go | 4 + internal/liveness/liveness_test.go | 53 +++++++ internal/liveness/runtime.go | 21 ++- internal/system/system.go | 21 +++ internal/units/exec.go | 14 +- internal/units/units.go | 60 ++++++-- internal/units/units_test.go | 117 +++++++++++++++ 31 files changed, 1774 insertions(+), 43 deletions(-) create mode 100644 cmd/mesh-host/movesecret.go create mode 100644 cmd/mesh-host/movesecret_test.go create mode 100644 cmd/mesh-host/nodekey.go create mode 100644 cmd/mesh-host/nodekey_test.go create mode 100644 cmd/mesh-host/vaultbox_nats_test.go create mode 100644 internal/apply/filled_apply_test.go create mode 100644 internal/apply/vault_sealed_test.go create mode 100644 internal/link/movesecret.go create mode 100644 internal/link/movesecret_test.go diff --git a/cmd/mesh-host/main.go b/cmd/mesh-host/main.go index f7d7dc1..e8b3b3f 100644 --- a/cmd/mesh-host/main.go +++ b/cmd/mesh-host/main.go @@ -1130,7 +1130,7 @@ func runLink(ctx context.Context, opts options) error { // The one liveness judge on this machine (novox/hq ADR 0240), made before anything applies: every // report this host makes carries what it says. if j, err := liveness.Open(filepath.Join(filepath.Dir(opts.state), liveness.FileName), - &liveness.Exec{Run: apply.ExecRunner}); j != nil { + &liveness.Exec{Run: apply.ExecRunner, System: builtFor}); j != nil { if err != nil { say(err.Error()) } @@ -1242,6 +1242,16 @@ func runLink(ctx context.Context, opts options) error { Say: say, Timeout: opts.timeout, } + // And the move of a value sealed to this machine into the vault, asked by the controller (design 57's V3, + // issue 676): the vault is asked over this queue's link, so the queue is named once it exists. + queue.MoveSecret = moveSecretFrom(opts.state, queue) + // And this machine's sealing key stated to the vault in its own name, at start and whenever it changes (ADR 0372, + // issue 786), whether or not any value was ever moved in. + go keepNodeKeyStated(ctx, func() (identity.SealingKey, error) { + return identity.LoadSealingKey(identity.SealingKeyPath(opts.state)) + }, func(verb string, args any) (json.RawMessage, error) { + return queue.AskVault(ctx, verb, args) + }, say, nodeKeyRetryFrom, nodeKeyRetryUpTo) worker := make(chan struct{}) go func() { defer close(worker) @@ -1774,8 +1784,11 @@ func healthAsReported(st liveness.Statement, ns *network.Statement) *link.Health // user's declared `root` (novox/hq ADR 0266), and reads a declaration's identities, groups and libraries // (novox/hq ADR 0328), a container's busy (ADR 0335) and a file's owner-at-runtime and the filled files (ADR // 0349 §5) and the provisioner client's account (ADR 0349 §4), which is what tells the controller it may be sent - // any of those fields. - h := &link.Health{Contract: link.ProvisionerContract, At: st.At.UTC(), Resources: []link.ResourceHealth{}} + // any of those fields; and it writes a filled file only on the client's word (HandedContract, design 57 P7), + // which is what tells it it may hand a moved module's files to the provisioner here; and it opens the vault's box + // for a core credential (VaultBoxContract, ADR 0349 §16), which is what tells it it may send that box here; and it states its sealing key to the vault (NodeKeyContract, ADR + // 0372, issue 786). + h := &link.Health{Contract: link.NodeKeyContract, At: st.At.UTC(), Resources: []link.ResourceHealth{}} for _, r := range st.Resources { h.Resources = append(h.Resources, link.ResourceHealth{Module: r.Module, Resource: r.ID, Kind: r.Kind, Target: r.Target, State: r.State, Reason: r.Reason, Since: r.Since.UTC(), Streak: r.Streak, @@ -1799,7 +1812,8 @@ func withUnits(h *link.Health, us *units.Statement) *link.Health { if us == nil || us.State == "" { return h } - h.Units = &link.UnitsHealth{State: us.State, Failed: []link.FailedUnit{}, Unread: us.Unread} + h.Units = &link.UnitsHealth{State: us.State, Failed: []link.FailedUnit{}, Unread: us.Unread, + Away: us.Away} for _, f := range us.Failed { if f.Module == "" { h.Units.Failed = append(h.Units.Failed, link.FailedUnit{Unit: f.Unit, Scope: f.Scope, Load: f.Load, @@ -2353,7 +2367,7 @@ func sealOpener(statePath string) apply.Unseal { if err != nil { return nil, err } - return key.Unseal(sealed) + return key.Open(sealed) } } diff --git a/cmd/mesh-host/movesecret.go b/cmd/mesh-host/movesecret.go new file mode 100644 index 0000000..7bbc2ec --- /dev/null +++ b/cmd/mesh-host/movesecret.go @@ -0,0 +1,131 @@ +package main + +import ( + "context" + "crypto/rand" + "crypto/subtle" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "os" + "time" + + "golang.org/x/crypto/nacl/box" + + "github.com/novox/mesh-host/internal/identity" + "github.com/novox/mesh-host/internal/link" +) + +// The move of a value sealed to this machine into the vault (design 57's V3, issue 676; internal/link +// movesecret.go). The controller cannot open what it holds; this machine can open what was sealed to it, and so +// hands it to the vault, sealed to a key the vault made for the call. The value is in this process for that one +// call and written nowhere. + +// vaultAsk asks one of the vault's verbs in this engine's own name and answers its result. +type vaultAsk func(verb string, args any) (json.RawMessage, error) + +// moveSecretFrom is the engine's move: this machine's sealing key, read for each ask, and the vault asked over +// the link open now. +func moveSecretFrom(statePath string, queue *link.Queue) link.MoveSecretFunc { + return func(m link.MoveSecret) (link.MovedSecret, error) { + key, err := identity.LoadSealingKey(identity.SealingKeyPath(statePath)) + if err != nil { + return link.MovedSecret{}, err + } + ask := func(verb string, args any) (json.RawMessage, error) { + return queue.AskVault(context.Background(), verb, args) + } + moved, err := moveIntoTheVault(ask, key, m) + if err != nil { + fmt.Fprintf(os.Stderr, "mesh-host: a value was not moved into the vault: %v\n", err) + } + return moved, err + } +} + +// moveIntoTheVault opens the row's value with this machine's sealing key, hands it to the vault's move-in in its two +// steps, and answers what the vault kept once its read-back — sealed to a key made here for this one call — opens +// to the same value. +func moveIntoTheVault(ask vaultAsk, key identity.SealingKey, m link.MoveSecret) (link.MovedSecret, error) { + if m.SealedTo != "" && m.SealedTo != key.Public { + return link.MovedSecret{}, errors.New("the controller's store says this value was sealed to a sealing key this " + + "machine no longer has, so it cannot be opened here") + } + value, err := key.Unseal(m.Sealed) + if err != nil { + return link.MovedSecret{}, fmt.Errorf("the value does not open with this machine's sealing key: %w", err) + } + defer wipe(value) + + // This machine's sealing key, stated in its own name: the key the vault seals the core's credentials to for the + // controller to carry here (novox/hq ADR 0349 §16, issue 763), never a key the controller names. + first := map[string]any{"origin": m.Origin, "generation": m.Generation, "sealing_key": key.Public} + if !m.MadeAt.IsZero() { + first["made_at"] = m.MadeAt.UTC().Format(time.RFC3339Nano) + } + if m.Own() { + first["node"], first["module"], first["name"] = m.Node, m.Module, m.Name + } else { + first["provider"], first["provision"] = m.Provider, m.Provision + first["consumer"], first["consumer_module"] = m.Consumer, m.ConsumerModule + if m.Local != "" { + first["local"] = m.Local + } + } + raw, err := ask("move-in", first) + if err != nil { + return link.MovedSecret{}, fmt.Errorf("the vault did not open the move: %w", err) + } + var opened struct { + Call string `json:"call"` + SealTo string `json:"seal_to"` + Secret string `json:"secret"` + } + if err := json.Unmarshal(raw, &opened); err != nil || opened.Call == "" || opened.SealTo == "" { + return link.MovedSecret{}, errors.New("the vault answered no call and no key to seal to") + } + sealed, err := identity.Seal(opened.SealTo, value) + if err != nil { + return link.MovedSecret{}, fmt.Errorf("the vault's key for the call is not a sealing key: %w", err) + } + backPub, backPriv, err := box.GenerateKey(rand.Reader) + if err != nil { + return link.MovedSecret{}, err + } + defer wipe(backPriv[:]) + raw, err = ask("move-in", map[string]any{"call": opened.Call, "sealed": sealed, + "read_back_to": base64.StdEncoding.EncodeToString(backPub[:])}) + if err != nil { + return link.MovedSecret{}, fmt.Errorf("the vault did not take %s: %w", opened.Secret, err) + } + var kept struct { + Name string `json:"name"` + Generation int `json:"generation"` + Unchanged bool `json:"unchanged"` + ReadBack string `json:"read_back"` + } + if err := json.Unmarshal(raw, &kept); err != nil || kept.Name == "" || kept.ReadBack == "" { + return link.MovedSecret{}, fmt.Errorf("the vault answered %s with no read-back", opened.Secret) + } + blob, err := base64.StdEncoding.DecodeString(kept.ReadBack) + if err != nil { + return link.MovedSecret{}, fmt.Errorf("the vault's read-back of %s is not sealed", kept.Name) + } + back, ok := box.OpenAnonymous(nil, blob, backPub, backPriv) + if !ok { + return link.MovedSecret{}, fmt.Errorf("the vault's read-back of %s does not open with the key made for it", kept.Name) + } + defer wipe(back) + if subtle.ConstantTimeCompare(back, value) != 1 { + return link.MovedSecret{}, fmt.Errorf("the vault keeps %s, but it read back another value than the one handed "+ + "in: the move of it did not hold", kept.Name) + } + return link.MovedSecret{Secret: kept.Name, Generation: kept.Generation, Unchanged: kept.Unchanged}, nil +} + +func wipe(b []byte) { + for i := range b { + b[i] = 0 + } +} diff --git a/cmd/mesh-host/movesecret_test.go b/cmd/mesh-host/movesecret_test.go new file mode 100644 index 0000000..e362c77 --- /dev/null +++ b/cmd/mesh-host/movesecret_test.go @@ -0,0 +1,134 @@ +package main + +import ( + "crypto/rand" + "encoding/base64" + "encoding/json" + "errors" + "strings" + "testing" + + "golang.org/x/crypto/nacl/box" + + "github.com/novox/mesh-host/internal/identity" + "github.com/novox/mesh-host/internal/link" +) + +// standInVault answers move-in's two steps as the vault does: a key made for the call, the value opened with it, +// and the value kept read back sealed to the key the second step sent. lie makes its read-back another value. +type standInVault struct { + t *testing.T + pub *[32]byte + priv *[32]byte + opened map[string]any + kept string + lie string + refuse string + carried []string +} + +func (v *standInVault) ask(verb string, args any) (json.RawMessage, error) { + raw, _ := json.Marshal(args) + v.carried = append(v.carried, string(raw)) + if verb != "move-in" { + v.t.Fatalf("asked %s", verb) + } + if v.refuse != "" { + return nil, errors.New(v.refuse) + } + var a map[string]any + _ = json.Unmarshal(raw, &a) + if a["call"] == nil { + v.opened = a + pub, priv, _ := box.GenerateKey(rand.Reader) + v.pub, v.priv = pub, priv + return json.Marshal(map[string]string{"call": "c1", "seal_to": base64.StdEncoding.EncodeToString(pub[:]), + "secret": "own/anchor/postgres/superuser-password"}) + } + blob, _ := base64.StdEncoding.DecodeString(a["sealed"].(string)) + value, ok := box.OpenAnonymous(nil, blob, v.pub, v.priv) + if !ok { + v.t.Fatal("what the engine sealed does not open with the call's key") + } + v.kept = string(value) + back := v.kept + if v.lie != "" { + back = v.lie + } + key, _ := base64.StdEncoding.DecodeString(a["read_back_to"].(string)) + var to [32]byte + copy(to[:], key) + boxed, _ := box.SealAnonymous(nil, []byte(back), &to, rand.Reader) + return json.Marshal(map[string]any{"name": "own/anchor/postgres/superuser-password", "generation": 1, + "read_back": base64.StdEncoding.EncodeToString(boxed)}) +} + +const theValue = "the-superusers-password" + +func aMoveOf(t *testing.T, key identity.SealingKey) link.MoveSecret { + t.Helper() + sealed, err := identity.Seal(key.Public, []byte(theValue)) + if err != nil { + t.Fatal(err) + } + return link.MoveSecret{Node: "anchor", By: "the move", MoveSecretAsk: link.MoveSecretAsk{Module: "postgres", + Name: "superuser-password", Origin: "made", Generation: "g-1", Sealed: sealed, SealedTo: key.Public}} +} + +// The engine opens what was sealed to its machine, hands it to the vault sealed to the call's key — never in the +// clear — and answers what the vault kept only once the read-back opens to the same value. +func TestTheEngineMovesAValueSealedToItAndChecksTheReadBack(t *testing.T) { + key, err := identity.GenerateSealingKey() + if err != nil { + t.Fatal(err) + } + vault := &standInVault{t: t} + moved, err := moveIntoTheVault(vault.ask, key, aMoveOf(t, key)) + if err != nil { + t.Fatal(err) + } + if moved.Secret != "own/anchor/postgres/superuser-password" || moved.Generation != 1 || vault.kept != theValue { + t.Fatalf("moved %+v, the vault kept %q", moved, vault.kept) + } + if vault.opened["module"] != "postgres" || vault.opened["name"] != "superuser-password" || vault.opened["node"] != "anchor" || + vault.opened["origin"] != "made" || vault.opened["generation"] != "g-1" || + vault.opened["sealing_key"] != key.Public { + t.Errorf("the first step named %v", vault.opened) + } + if strings.Contains(strings.Join(vault.carried, " "), theValue) { + t.Fatal("the value crossed the bus in the clear") + } +} + +// A read-back that is not the value handed in is a failed move, said; so is a value this machine cannot open, and +// the vault's refusal. +func TestTheEngineSaysAMoveThatDidNotHold(t *testing.T) { + key, _ := identity.GenerateSealingKey() + vault := &standInVault{t: t, lie: "something-else"} + if _, err := moveIntoTheVault(vault.ask, key, aMoveOf(t, key)); err == nil || + !strings.Contains(err.Error(), "read back") { + t.Fatalf("a read-back of another value: %v", err) + } + other, _ := identity.GenerateSealingKey() + if _, err := moveIntoTheVault((&standInVault{t: t}).ask, key, aMoveOf(t, other)); err == nil || + !strings.Contains(err.Error(), "sealing key") { + t.Fatalf("a value sealed to another key: %v", err) + } + refusing := &standInVault{t: t, refuse: "the move never replaces a value the vault holds"} + if _, err := moveIntoTheVault(refusing.ask, key, aMoveOf(t, key)); err == nil || + !strings.Contains(err.Error(), "never replaces") { + t.Fatalf("the vault's refusal: %v", err) + } + // A pair credential names both its ends. + pair := aMoveOf(t, key) + pair.Module, pair.Name = "", "" + pair.Provider, pair.Provision, pair.Consumer, pair.ConsumerModule = "anchor", "database", "shanks", "gitea" + ends := &standInVault{t: t} + if _, err := moveIntoTheVault(ends.ask, key, pair); err != nil { + t.Fatal(err) + } + if ends.opened["provider"] != "anchor" || ends.opened["consumer"] != "shanks" || ends.opened["consumer_module"] != "gitea" || + ends.opened["node"] != nil { + t.Errorf("the pair's first step named %v", ends.opened) + } +} diff --git a/cmd/mesh-host/nodekey.go b/cmd/mesh-host/nodekey.go new file mode 100644 index 0000000..f00c2f8 --- /dev/null +++ b/cmd/mesh-host/nodekey.go @@ -0,0 +1,62 @@ +package main + +import ( + "context" + "fmt" + "time" + + "github.com/novox/mesh-host/internal/identity" +) + +// This machine's sealing key, stated to the vault in the engine's own name (novox/hq ADR 0372, issue 786). The vault +// records it from the machine's own account alone and, when it is a first or a changed key, says the machine's core +// secrets changed so the controller asks for them again sealed to it. Without this a machine that never moved a +// secret in (adopted after feature 57) has no key at the vault. Nothing but the public half crosses. + +const ( + // nodeKeyRetryFrom and nodeKeyRetryUpTo bound the wait between tries while the vault does not answer: it starts + // after the bus and may not serve the verb yet. + nodeKeyRetryFrom = 5 * time.Second + nodeKeyRetryUpTo = 5 * time.Minute + // nodeKeyLook is how often the key file is read to see whether it changed. + nodeKeyLook = 30 * time.Second +) + +// keepNodeKeyStated states the sealing key load answers to the vault at start, retrying with a growing wait from +// retryFrom to retryUpTo while it does not answer, and states it again whenever the key changes. It says one line +// when a key is stated and one when a try fails for another reason than the last, and returns when ctx ends. +func keepNodeKeyStated(ctx context.Context, load func() (identity.SealingKey, error), ask vaultAsk, say func(string), + retryFrom, retryUpTo time.Duration) { + stated, lastFailure := "", "" + wait := retryFrom + fail := func(why string) { + if why != lastFailure { + say("mesh-host: the sealing key was not stated to the vault, trying again: " + why) + lastFailure = why + } + wait = min(wait*2, retryUpTo) + } + look := min(nodeKeyLook, retryFrom) + for { + delay := look + key, err := load() + switch { + case err != nil: + fail(err.Error()) + delay = wait + case key.Public != stated: + if _, err := ask("node-key", map[string]string{"key": key.Public}); err != nil { + fail(err.Error()) + delay = wait + } else { + stated, lastFailure, wait = key.Public, "", retryFrom + say(fmt.Sprintf("mesh-host: this machine's sealing key %s is stated to the vault", key.Public)) + } + } + select { + case <-ctx.Done(): + return + case <-time.After(delay): + } + } +} diff --git a/cmd/mesh-host/nodekey_test.go b/cmd/mesh-host/nodekey_test.go new file mode 100644 index 0000000..80f85ce --- /dev/null +++ b/cmd/mesh-host/nodekey_test.go @@ -0,0 +1,112 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "sync" + "testing" + "time" + + "github.com/novox/mesh-host/internal/identity" + "github.com/novox/mesh-host/internal/link" +) + +// Issue 786 (novox/hq ADR 0372): the engine states its own sealing key to the vault's node-key at start, retries +// while the vault does not answer, and states it again when the key changes; the same key is not stated twice. +type keyVault struct { + mu sync.Mutex + stated []string + fail int + verbs []string +} + +func (v *keyVault) ask(verb string, args any) (json.RawMessage, error) { + v.mu.Lock() + defer v.mu.Unlock() + v.verbs = append(v.verbs, verb) + if v.fail > 0 { + v.fail-- + return nil, errors.New("the vault does not answer") + } + raw, _ := json.Marshal(args) + var a struct { + Key string `json:"key"` + } + _ = json.Unmarshal(raw, &a) + v.stated = append(v.stated, a.Key) + return json.RawMessage(`{"recorded":true}`), nil +} + +func (v *keyVault) said() []string { + v.mu.Lock() + defer v.mu.Unlock() + return append([]string(nil), v.stated...) +} + +func waitFor(t *testing.T, what string, ok func() bool) { + t.Helper() + for i := 0; i < 400; i++ { + if ok() { + return + } + time.Sleep(5 * time.Millisecond) + } + t.Fatalf("timed out: %s", what) +} + +func TestTheEngineStatesItsSealingKeyToTheVaultAtStartRetriesAndRestatesAChange(t *testing.T) { + first, _ := identity.GenerateSealingKey() + second, _ := identity.GenerateSealingKey() + var mu sync.Mutex + current := first + load := func() (identity.SealingKey, error) { + mu.Lock() + defer mu.Unlock() + return current, nil + } + vault := &keyVault{fail: 2} + var lines []string + var lmu sync.Mutex + say := func(l string) { lmu.Lock(); lines = append(lines, l); lmu.Unlock() } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go keepNodeKeyStated(ctx, load, vault.ask, say, time.Millisecond, 5*time.Millisecond) + + waitFor(t, "the first key stated after two refusals", func() bool { return len(vault.said()) == 1 }) + if vault.said()[0] != first.Public { + t.Errorf("stated %q, want the engine's own key %q", vault.said()[0], first.Public) + } + vault.mu.Lock() + verbs := append([]string(nil), vault.verbs...) + vault.mu.Unlock() + for _, v := range verbs { + if v != "node-key" { + t.Errorf("asked %s", v) + } + } + // The same key again is not stated again. + time.Sleep(60 * time.Millisecond) + if n := len(vault.said()); n != 1 { + t.Errorf("the same key was stated %d times", n) + } + mu.Lock() + current = second + mu.Unlock() + waitFor(t, "the changed key stated", func() bool { return len(vault.said()) == 2 }) + if vault.said()[1] != second.Public { + t.Errorf("stated %q after the change, want %q", vault.said()[1], second.Public) + } + lmu.Lock() + defer lmu.Unlock() + if len(lines) == 0 { + t.Error("the engine said nothing") + } +} + +// The statement a host makes names this contract (issue 786), one above the vault's box. +func TestTheNodeKeyContractIsThirteen(t *testing.T) { + if link.NodeKeyContract != 13 { + t.Errorf("NodeKeyContract %d", link.NodeKeyContract) + } +} diff --git a/cmd/mesh-host/units_test.go b/cmd/mesh-host/units_test.go index 423c286..b079b1c 100644 --- a/cmd/mesh-host/units_test.go +++ b/cmd/mesh-host/units_test.go @@ -87,10 +87,12 @@ func TestTheStatementSaysAnAccountJudgedForRoot(t *testing.T) { Groups: []string{"openrazer"}}, State: accounts.Healthy, Since: at}, }} h := withAccounts(healthAsReported(liveness.Statement{At: at}, nil), as) - // At RootContract or later: an engine that also reads the provisioner client's account says ProvisionerContract - // (novox/hq ADR 0349 §4), one above ADR 0349 §5's OwnershipContract. - if h.Contract < link.RootContract || link.RootContract != 3 || h.Contract != link.ProvisionerContract || - link.ProvisionerContract != link.OwnershipContract+1 || link.OwnershipContract != link.BusyContract+1 || + // At RootContract or later: an engine that states its sealing key to the vault says NodeKeyContract (ADR 0372, issue 786), one above + // VaultBoxContract (it opens the vault's box for the core's credentials, ADR 0349 §16, issue 755), one above HandedContract (design 57 P7), one above ProvisionerContract (novox/hq ADR + // 0349 §4), one above ADR 0349 §5's OwnershipContract. + if h.Contract < link.RootContract || link.RootContract != 3 || h.Contract != link.NodeKeyContract || + link.NodeKeyContract != link.VaultBoxContract+1 || link.VaultBoxContract != link.HandedContract+1 || + link.HandedContract != link.ProvisionerContract+1 || link.ProvisionerContract != link.OwnershipContract+1 || link.OwnershipContract != link.BusyContract+1 || link.BusyContract != link.OwnGroupContract+1 { t.Fatalf("contract %d", h.Contract) } diff --git a/cmd/mesh-host/vaultbox_nats_test.go b/cmd/mesh-host/vaultbox_nats_test.go new file mode 100644 index 0000000..74be00e --- /dev/null +++ b/cmd/mesh-host/vaultbox_nats_test.go @@ -0,0 +1,154 @@ +package main + +// Issue 755's proof on a real bus (novox/hq ADR 0349 §16): a core credential the vault sealed to this machine crosses +// the bus inside the declaration, which the controller carries and cannot open, and this engine opens it with its +// sealing key and writes the value. A listener on every subject keeps every byte that crossed; the value is never +// among them. Run against the throwaway bus merge-check.sh starts (MESH_TEST_NATS). + +import ( + "context" + "encoding/base64" + "encoding/json" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/seal" + "github.com/nats-io/nats.go" + + "github.com/novox/mesh-host/internal/apply" + "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/identity" + "github.com/novox/mesh-host/internal/link" + "github.com/novox/mesh-host/internal/store" + "github.com/novox/mesh-host/internal/system" +) + +const coreCredential = "the-provisioner-clients-bus-password-2025" + +func TestNatsTheEngineFillsACoreCredentialFromTheVaultsBoxWithNoValueInTheClear(t *testing.T) { + url := os.Getenv("MESH_TEST_NATS") + if url == "" { + t.Skip("MESH_TEST_NATS unset") + } + // This machine's sealing key, where the engine reads it. + state := filepath.Join(t.TempDir(), "state.json") + key, err := identity.GenerateSealingKey() + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(identity.SealingKeyPath(state), []byte(key.Private), 0o600); err != nil { + t.Fatal(err) + } + + // Every byte that crosses. + tap, err := nats.Connect(url) + if err != nil { + t.Fatal(err) + } + defer tap.Close() + var mu sync.Mutex + var crossed []string + if _, err := tap.Subscribe(">", func(m *nats.Msg) { + mu.Lock() + crossed = append(crossed, m.Subject+" "+string(m.Data)) + mu.Unlock() + }); err != nil { + t.Fatal(err) + } + _ = tap.Flush() + + // The engine hears its declaration on its own subject. + const node = "vaultbox" + engine, err := nats.Connect(url) + if err != nil { + t.Fatal(err) + } + defer engine.Close() + heard, err := engine.SubscribeSync(link.DeclareSubject(node)) + if err != nil { + t.Fatal(err) + } + _ = engine.Flush() + + // The vault's answer to the controller: the value in the ADR 0206 box, sealed to this machine's key. The + // controller carries it in the declaration as it is. + pub, err := key.PublicPEM() + if err != nil { + t.Fatal(err) + } + box, err := seal.Seal(coreCredential, pub) + if err != nil { + t.Fatal(err) + } + path := filepath.Join(t.TempDir(), "broker") + body, _ := json.Marshal(map[string]any{"declaration": 1, "resources": []map[string]any{ + {"id": "need-provisioner-broker", "type": "file", "path": path, "vault-sealed": box}}}) + controller, err := nats.Connect(url) + if err != nil { + t.Fatal(err) + } + defer controller.Close() + if err := controller.Publish(link.DeclareSubject(node), body); err != nil { + t.Fatal(err) + } + _ = controller.Flush() + + msg, err := heard.NextMsg(5 * time.Second) + if err != nil { + t.Fatalf("the declaration did not reach the engine: %v", err) + } + d, err := declaration.Parse(msg.Data) + if err != nil { + t.Fatal(err) + } + sys, err := system.For("arch") + if err != nil { + t.Fatal(err) + } + if _, _, err := apply.Apply(context.Background(), sys, d, store.State{}, store.OriginCarried, + func(context.Context, string, ...string) (string, error) { return "", nil }, nil, sealOpener(state)); err != nil { + t.Fatal(err) + } + on, err := os.ReadFile(path) + if err != nil || string(on) != coreCredential { + t.Fatalf("the engine wrote %q, %v", on, err) + } + + _ = tap.Flush() + time.Sleep(100 * time.Millisecond) + mu.Lock() + all := strings.Join(crossed, "\n") + mu.Unlock() + if !strings.Contains(all, link.DeclareSubject(node)) || !strings.Contains(all, "vault-sealed") { + t.Fatalf("the listener heard nothing of the declaration:\n%s", all) + } + for _, form := range []string{coreCredential, base64.StdEncoding.EncodeToString([]byte(coreCredential))} { + if strings.Contains(all, form) { + t.Fatal("the core credential crossed the bus in the clear") + } + } +} + +// The engine opens a value the controller sealed (the NaCl box) as before, and the vault's box, with one key. +func TestTheEnginesOpenerOpensBothBoxes(t *testing.T) { + state := filepath.Join(t.TempDir(), "state.json") + key, _ := identity.GenerateSealingKey() + if err := os.WriteFile(identity.SealingKeyPath(state), []byte(key.Private), 0o600); err != nil { + t.Fatal(err) + } + open := sealOpener(state) + nacl, _ := identity.Seal(key.Public, []byte("old")) + if got, err := open(nacl); err != nil || string(got) != "old" { + t.Errorf("the controller's box opened to %q, %v", got, err) + } + pub, _ := key.PublicPEM() + box, _ := seal.Seal("new", pub) + raw, _ := json.Marshal(box) + if got, err := open(string(raw)); err != nil || string(got) != "new" { + t.Errorf("the vault's box opened to %q, %v", got, err) + } +} diff --git a/examples/foundation-first-node.lock b/examples/foundation-first-node.lock index 5e05f7f..142231b 100644 --- a/examples/foundation-first-node.lock +++ b/examples/foundation-first-node.lock @@ -221,7 +221,7 @@ "path": "/var/lib/nats-module/conf/accounts.conf", "mode": "0600", "create-once": true, - "content": "// The first user list, carried by the installer because at genesis there is no mesh to\n// compose one. A bootstrap credential, rotated with the store's and replaced by the\n// controller's own composition from its first start onward.\naccounts {\n MESH {\n jetstream: enabled\n users = [\n { user: \"controller\", password: \"$2a$10$AHqJgOifIVbU41KmATiMhuXFs8xa7Wl2HuN4UVBCXdN2jIQzjqApy\", permissions: {\n publish: { allow: [\"$JS.ACK.CONTROL.controller.>\", \"$JS.ACK.EVENTS.controller.>\", \"$JS.ACK.DEAD_LETTER_NOTICES.controller.>\", \"$JS.API.>\", \"$KV.mesh-controller_asked.>\", \"$KV.mesh-controller_assignments.>\", \"$KV.mesh-controller_calls.>\", \"$KV.mesh-controller_hand-acts.>\", \"$KV.mesh-controller_conditions.>\", \"$KV.mesh-controller_condition-history.>\", \"$KV.mesh-controller_declared-config.>\", \"$KV.mesh-controller_lease.>\", \"$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>\", \"$KV.SEAT_NODE_BUILD_AGENT_cancelled.>\", \"_INBOX.enrol.>\", \"mesh.assignment.>\", \"mesh.mod.*.tool.>\", \"mesh.node.>\", \"mesh.seat.mesh-build-machine.accept.>\", \"mesh.seat.node-build-agent.accept.>\", \"mesh.seat.mesh-build-machine.tool.>\", \"mesh.seat.node-build-agent.tool.>\", \"mesh.seat.mesh-controller.event.applied\", \"mesh.seat.mesh-controller.event.built-before\", \"mesh.seat.mesh-controller.event.refused\", \"mesh.seat.mesh-controller.event.rolled-back\", \"mesh.seat.mesh-controller.event.checked\", \"mesh.seat.mesh-controller.event.plan-moved\", \"mesh.seat.mesh-controller.event.assignment-changed\", \"mesh.seat.mesh-controller.event.declared-config-changed\", \"mesh.seat.mesh-controller.event.condition-raised\", \"mesh.seat.mesh-controller.event.condition-changed\", \"mesh.seat.mesh-controller.event.condition-cleared\", \"mesh.seat.mesh-controller.event.doctor-heartbeat\", \"mesh.seat.mesh-controller.event.secret-replaced\", \"mesh.seat.mesh-controller.event.healer-acted\", \"$SRV.INFO\", \"mesh.seat.node-intrusion-prevention.tool.banned.*\", \"mesh.seat.node-backup.tool.backed-up.*\", \"mesh.seat.node-backup.tool.now.*\", \"mesh.seat.mesh-delivery.tool.stalled\", \"mesh.seat.mesh-delivery.tool.close\", \"mesh.seat.node-launcher.tool.secret.*\", \"mesh.seat.mesh-delivery.tool.release\", \"mesh.seat.mesh-delivery.tool.stop\", \"mesh.seat.mesh-delivery.tool.times\", \"mesh.seat.mesh-delivery.tool.groups\", \"mesh.seat.node-service-manager.tool.restart.*\", \"mesh.seat.mesh-controller.tool.plans\", \"mesh.events.dead.>\", \"mesh.again.>\"] }\n subscribe: { allow: [\"$JS.API.>\", \"$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>\", \"$JS.EVENT.ADVISORY.CONSUMER.DELETED.>\", \"_DELIVER.controller\", \"_DELIVER.controller.>\", \"_INBOX.controller.>\", \"mesh.control.>\", \"mesh.mod.*.event.provisioner.failing\", \"mesh.mod.*.event.provisioner.recovered\", \"mesh.mod.*.event.provisioner.retirement\", \"mesh.mod.gitea.event.pull.merged\", \"mesh.mod.gitea.event.pull.updated\", \"mesh.mod.mesh-catalog.event.catching-up\", \"mesh.mod.mesh-catalog.event.upgraded\", \"mesh.seat.mesh-build-machine.event.built\", \"mesh.seat.node-build-agent.event.built\", \"mesh.seat.operator-channel.event.decided.mesh-controller\", \"mesh.seat.mesh-controller.tool.>\", \"$SRV.PING\", \"$SRV.INFO\", \"$SRV.PING.mesh-controller\", \"$SRV.PING.mesh-controller.>\", \"$SRV.INFO.mesh-controller\", \"$SRV.INFO.mesh-controller.>\", \"$SRV.STATS\", \"$SRV.STATS.mesh-controller\", \"$SRV.STATS.mesh-controller.>\"] }\n allow_responses: { max: 1, ttl: \"1m\" }\n } }\n ]\n }\n}\n" + "content": "// The first user list, carried by the installer because at genesis there is no mesh to\n// compose one. A bootstrap credential, rotated with the store's and replaced by the\n// controller's own composition from its first start onward.\naccounts {\n MESH {\n jetstream: enabled\n users = [\n { user: \"controller\", password: \"$2a$10$AHqJgOifIVbU41KmATiMhuXFs8xa7Wl2HuN4UVBCXdN2jIQzjqApy\", permissions: {\n publish: { allow: [\"$JS.ACK.CONTROL.controller.>\", \"$JS.ACK.EVENTS.controller.>\", \"$JS.ACK.DEAD_LETTER_NOTICES.controller.>\", \"$JS.API.>\", \"$KV.mesh-controller_asked.>\", \"$KV.mesh-controller_assignments.>\", \"$KV.mesh-controller_calls.>\", \"$KV.mesh-controller_hand-acts.>\", \"$KV.mesh-controller_conditions.>\", \"$KV.mesh-controller_condition-history.>\", \"$KV.mesh-controller_declared-config.>\", \"$KV.mesh-controller_lease.>\", \"$KV.mesh-controller_predefined.>\", \"$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>\", \"$KV.SEAT_NODE_BUILD_AGENT_cancelled.>\", \"_INBOX.enrol.>\", \"mesh.assignment.>\", \"mesh.mod.*.tool.>\", \"mesh.node.>\", \"mesh.seat.mesh-build-machine.accept.>\", \"mesh.seat.node-build-agent.accept.>\", \"mesh.seat.mesh-build-machine.tool.>\", \"mesh.seat.node-build-agent.tool.>\", \"mesh.seat.mesh-controller.event.applied\", \"mesh.seat.mesh-controller.event.built-before\", \"mesh.seat.mesh-controller.event.refused\", \"mesh.seat.mesh-controller.event.rolled-back\", \"mesh.seat.mesh-controller.event.checked\", \"mesh.seat.mesh-controller.event.plan-moved\", \"mesh.seat.mesh-controller.event.assignment-changed\", \"mesh.seat.mesh-controller.event.declared-config-changed\", \"mesh.seat.mesh-controller.event.predefined-changed\", \"mesh.seat.mesh-controller.event.condition-raised\", \"mesh.seat.mesh-controller.event.condition-changed\", \"mesh.seat.mesh-controller.event.condition-cleared\", \"mesh.seat.mesh-controller.event.doctor-heartbeat\", \"mesh.seat.mesh-controller.event.secret-replaced\", \"mesh.seat.mesh-controller.event.healer-acted\", \"$SRV.INFO\", \"mesh.seat.node-intrusion-prevention.tool.banned.*\", \"mesh.seat.node-backup.tool.backed-up.*\", \"mesh.seat.node-backup.tool.now.*\", \"mesh.seat.mesh-delivery.tool.stalled\", \"mesh.seat.mesh-delivery.tool.close\", \"mesh.seat.node-launcher.tool.secret.*\", \"mesh.seat.mesh-delivery.tool.release\", \"mesh.seat.mesh-delivery.tool.stop\", \"mesh.seat.mesh-delivery.tool.times\", \"mesh.seat.mesh-delivery.tool.groups\", \"mesh.seat.node-service-manager.tool.restart.*\", \"mesh.seat.operator-channel.tool.tell\", \"mesh.seat.mesh-controller.tool.plans\", \"mesh.events.dead.>\", \"mesh.again.>\"] }\n subscribe: { allow: [\"$JS.API.>\", \"$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>\", \"$JS.EVENT.ADVISORY.CONSUMER.DELETED.>\", \"_DELIVER.controller\", \"_DELIVER.controller.>\", \"_INBOX.controller.>\", \"mesh.control.>\", \"mesh.mod.*.event.provisioner.failing\", \"mesh.mod.*.event.provisioner.recovered\", \"mesh.mod.*.event.provisioner.retirement\", \"mesh.mod.gitea.event.pull.merged\", \"mesh.mod.gitea.event.pull.updated\", \"mesh.mod.mesh-catalog.event.catching-up\", \"mesh.mod.mesh-catalog.event.upgraded\", \"mesh.seat.mesh-build-machine.event.built\", \"mesh.seat.node-build-agent.event.built\", \"mesh.seat.operator-channel.event.decided.mesh-controller\", \"mesh.seat.mesh-controller.tool.>\", \"$SRV.PING\", \"$SRV.INFO\", \"$SRV.PING.mesh-controller\", \"$SRV.PING.mesh-controller.>\", \"$SRV.INFO.mesh-controller\", \"$SRV.INFO.mesh-controller.>\", \"$SRV.STATS\", \"$SRV.STATS.mesh-controller\", \"$SRV.STATS.mesh-controller.>\"] }\n allow_responses: { max: 1, ttl: \"1m\" }\n } }\n ]\n }\n}\n" }, // The upstream server, until the broker step adopts it: the `nats` module's own image, built by the // mesh, takes its place there once, on these same directories (ADR 0333 §3), as the bus's planned diff --git a/go.mod b/go.mod index 627b0d9..406e5ba 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/novox/mesh-host go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261015210729-d3e182503c19 github.com/nats-io/nats.go v1.54.0 golang.org/x/crypto v0.57.0 golang.org/x/sys v0.48.0 diff --git a/go.sum b/go.sum index 315ddeb..007020d 100644 --- a/go.sum +++ b/go.sum @@ -1,9 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.11-0.20261009143344-f047d0a4a970 h1:9tFDQsgmI+4X7/BpZGXIr+HemPKE7YddYGqWV0lINAI= -git.novox.be/novox/mesh-sdk/go v0.1.11-0.20261009143344-f047d0a4a970/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010190644-622b6b4f556d h1:BR8uwv4BqyLaBwITuLylrd9DRUnu0D5DjJw0RuQSXbk= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010190644-622b6b4f556d/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c h1:l5onJwoIeH8yE/PjVlEeoPyXBsHp2NQiWLllz2fwX7s= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261015210729-d3e182503c19 h1:mAMt3KFmrHzDiBv4MLL94opWcG9y73ISMrBUyisapQE= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261015210729-d3e182503c19/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA= diff --git a/internal/accounts/accounts.go b/internal/accounts/accounts.go index 1316527..34efd9f 100644 --- a/internal/accounts/accounts.go +++ b/internal/accounts/accounts.go @@ -119,7 +119,7 @@ func secretsOf(d *declaration.Declaration, account string) []string { var out []string for _, r := range d.Resources { f, ok := r.(*declaration.File) - if !ok || f.Path == "" || (f.Sealed == "" && len(f.Secrets) == 0) || f.Owner == account { + if !ok || f.Path == "" || (!f.Secret() && len(f.Secrets) == 0) || f.Owner == account { continue } out = append(out, f.Path) diff --git a/internal/apply/apply.go b/internal/apply/apply.go index 9b98d0b..1c409a7 100644 --- a/internal/apply/apply.go +++ b/internal/apply/apply.go @@ -15,6 +15,7 @@ import ( "crypto/sha256" "encoding/base64" "encoding/hex" + "encoding/json" "errors" "fmt" "os" @@ -781,6 +782,16 @@ func ApplyMindingWindows( } } + // **A filled file is the provisioner client's to write** (novox/hq ADR 0349 §5, design 57 P7): one writer per + // file. The declaration says where it goes and how it is written, and only the write verb (WriteFilled) + // writes it, on the client's word. This apply — the controller's send and the five-minute re-apply alike — + // neither writes it nor records it, and nothing that reads it is moved for it; so only what the verb placed + // is recorded, and only that is removed as an orphan when the file leaves the declaration. + if f, isFile := resource.(*declaration.File); isFile && d.Fills(f.Path) { + report.Outcomes = append(report.Outcomes, leftToTheClient(f, &known, handedFiles, &report, log)) + continue + } + // The private network takes over the tunnel it found, ahead of the service that replaces // it (novox/hq ADR 0105): its configuration kept, its unit stopped and disabled, never // flushed. A failure here fails the service too — the mesh's interface is not started on a @@ -1250,7 +1261,9 @@ func afterWhatTheyRead(resources []declaration.Resource) []declaration.Resource const guardPrefix = declaration.AdoptionPrefix + "guard" // Unseal opens a value the mesh sealed to this node. Nil when the node has no sealing key, which -// makes every sealed file an error rather than a silently skipped one. +// makes every sealed file an error rather than a silently skipped one. A value the vault sealed (a core +// credential, novox/hq ADR 0349 §16) is given as its box's JSON, which begins with `{`; one the controller sealed +// is base64, which never does (identity.SealingKey.Open). type Unseal func(sealed string) ([]byte, error) func applyOne(ctx context.Context, sys system.System, r declaration.Resource, run Runner, @@ -1590,7 +1603,16 @@ func applyFile(r *declaration.File, previous store.Applied, unseal Unseal, keepF return out, fmt.Errorf( "%s is sealed to this node and this node has no sealing key", r.Path) } - opened, err := unseal(r.Sealed) + sealed := r.Sealed + if r.VaultSealed != nil { + // The vault's box, handed to the opener as its JSON (Unseal). + raw, err := json.Marshal(r.VaultSealed) + if err != nil { + return out, err + } + sealed = string(raw) + } + opened, err := unseal(sealed) if err != nil { return out, fmt.Errorf("cannot open %s: %w", r.Path, err) } diff --git a/internal/apply/filled.go b/internal/apply/filled.go index f3f4800..d95edd3 100644 --- a/internal/apply/filled.go +++ b/internal/apply/filled.go @@ -160,7 +160,7 @@ func WriteFilled( // filledAs is the declared file with the filled content in place of the declaration's. func filledAs(declared *declaration.File, content []byte) *declaration.File { f := *declared - f.Content, f.Sealed, f.Bytes, f.Secrets = string(content), "", "", nil + f.Content, f.Sealed, f.VaultSealed, f.Bytes, f.Secrets = string(content), "", nil, "", nil if f.Mode == "" && (declared.Secret() || len(declared.Secrets) > 0) { // Written as the declaration would have written it: a file carrying a credential is not world-readable. f.Mode = "0600" @@ -204,3 +204,27 @@ func recordKeeping(known *store.State, r declaration.Resource, o Outcome, was st } recordApplied(known, store.OriginDeclared, r, o, kept, keptMode, keptOwner) } + +// leftToTheClient is a filled file as an apply leaves it (novox/hq ADR 0349 §5, design 57 P7): untouched, and said +// so. A file the mesh wrote at the same path under another id, going in this apply, is handed to it as an apply hands +// a whole file (novox/hq ADR 0223): its record — what the mesh wrote there, and the original it kept — is the filled +// file's now, so the verb judges what is there against it, and the file is neither removed nor written. +func leftToTheClient(f *declaration.File, known *store.State, handedFiles map[string]store.Applied, report *Report, + log func(string)) Outcome { + if from, handed := handedFiles[f.ID]; handed { + if _, recorded := known.Find(f.ID); !recorded { + rec := from + rec.ID = f.ID + known.Record(rec) + } + delete(handedFiles, f.ID) + known.Forget(from.ID) + detail := "no longer declared; " + f.ID + " declares the file now, filled by the provisioner's client, so " + + "it was handed to it, not removed" + report.Outcomes = append(report.Outcomes, Outcome{ID: from.ID, Type: from.Type, Target: from.Target, + Action: "forgotten", Detail: detail}) + log(fmt.Sprintf(" forgotten %s (%s): %s", from.ID, from.Target, detail)) + } + return Outcome{ID: f.ID, Type: string(declaration.TypeFile), Target: f.Path, Action: "unchanged", + Detail: "filled by the provisioner's client, which alone has it written"} +} diff --git a/internal/apply/filled_apply_test.go b/internal/apply/filled_apply_test.go new file mode 100644 index 0000000..de1be82 --- /dev/null +++ b/internal/apply/filled_apply_test.go @@ -0,0 +1,218 @@ +package apply + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/store" +) + +// Defends novox/hq ADR 0349 §5, one writer per file (design 57 P7, issue 673): a file the declaration names as +// filled is written only through the write verb (WriteFilled), on the provisioner client's word. An apply — the +// controller's send and the five-minute re-apply alike — never writes it, never records it, and removes it as an +// orphan only when the verb placed it. + +// An apply leaves a filled file alone, whatever who owns it at runtime says: absent stays absent and is not +// recorded, and what the verb wrote stays as it wrote it through every re-apply. +func TestAnApplyNeverWritesAFilledFile(t *testing.T) { + for _, c := range []struct{ name, fields, content string }{ + {"owned", `"content":"","mode":"0640","owner-at-runtime":"owned"`, "port = 8080\n"}, + {"seed", `"content":"","mode":"0600","owner-at-runtime":"seed"`, "user default on\n"}, + {"managed block", `"into":"block","content":"","owner-at-runtime":"managed"`, "10.42.0.1 ace\n"}, + } { + t.Run(c.name, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "the-file") + d := filledDecl(t, path, c.fields, "") + report, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, + noServices, nil, nil) + if err != nil { + t.Fatal(err) + } + if _, err := os.Stat(path); err == nil { + t.Fatalf("an apply wrote a filled file the client has not filled: %q", readText(t, path)) + } + if _, ok := state.Find("m.conf"); ok { + t.Errorf("an apply recorded a filled file it did not write") + } + if o := outcomeOf(report, "m.conf"); o.Action != "unchanged" || !strings.Contains(o.Detail, "provisioner") { + t.Errorf("an apply did not say it left the file to the provisioner's client: %+v", o) + } + + _, state, err = WriteFilled(context.Background(), archHost(t), d, state, path, []byte(c.content), + noServices, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + placed, _ := state.Find("m.conf") + want, wantMode := readText(t, path), modeAt(t, path) + + // The re-apply, twice over: nothing written, the verb's record kept as it wrote it. + for i := 0; i < 2; i++ { + var said []string + report, state, err = Apply(context.Background(), archHost(t), d, state, store.OriginDeclared, + noServices, func(line string) { said = append(said, line) }, nil) + if err != nil { + t.Fatal(err) + } + if got := readText(t, path); got != want || modeAt(t, path) != wantMode { + t.Fatalf("re-apply %d wrote the filled file again: %q, want %q", i+1, got, want) + } + if o := outcomeOf(report, "m.conf"); o.Action != "unchanged" { + t.Errorf("re-apply %d: %+v", i+1, o) + } + if rec, _ := state.Find("m.conf"); rec.Wrote != placed.Wrote || rec.Target != path { + t.Errorf("re-apply %d changed the verb's record: %+v, was %+v", i+1, rec, placed) + } + if len(said) != 0 { + t.Errorf("re-apply %d said something of an untouched file: %q", i+1, said) + } + } + }) + } +} + +// What reads a filled file is moved by the verb's write, once, never again by an apply: a service under +// restart-on is not restarted and a container reading it is not recreated by the re-apply. +func TestAnApplyMovesNothingForAFilledFile(t *testing.T) { + env := filepath.Join(t.TempDir(), "forge.env") + d := filledDecl(t, env, `"content":"","mode":"0600","owner-at-runtime":"owned"`, + `{"id":"forge.server","type":"container","name":"forge","image":"`+pinned+`","env-file":["`+env+`"]}`) + m := &machine{containers: map[string]*fakeContainer{}} + _, state, err := WriteFilled(context.Background(), archHost(t), d, store.State{}, env, []byte("PORT=5432\n"), + m.run, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + m.asked = nil + if _, _, err := Apply(context.Background(), archHost(t), d, state, store.OriginDeclared, m.run, nil, + nil); err != nil { + t.Fatal(err) + } + if m.removed("forge") || m.did("docker run") { + t.Errorf("the re-apply recreated the container reading the filled file: %v", m.asked) + } + if got := readText(t, env); got != "PORT=5432\n" { + t.Errorf("the re-apply wrote the filled file: %q", got) + } + + conf := filepath.Join(t.TempDir(), "daemon.conf") + d = filledDecl(t, conf, `"content":"","owner-at-runtime":"owned"`, + `{"id":"m.daemon","type":"service","unit":"m.service","state":"running","restart-on":["m.conf"]}`) + var commands []string + _, state, err = WriteFilled(context.Background(), archHost(t), d, store.State{}, conf, []byte("level = 1\n"), + recordingServices(&commands), nil, nil, nil) + if err != nil { + t.Fatal(err) + } + commands = nil + if _, _, err := Apply(context.Background(), archHost(t), d, state, store.OriginDeclared, + recordingServices(&commands), nil, nil); err != nil { + t.Fatal(err) + } + if n := count(commands, "systemctl stop m.service"); n != 0 { + t.Errorf("the re-apply restarted what reads the filled file: %v", commands) + } +} + +// A filled file the verb never placed is not the mesh's to remove: one found there, or put there by hand, stays +// when the file leaves the declaration. +func TestAFilledFileTheVerbNeverPlacedIsNotRemoved(t *testing.T) { + path := filepath.Join(t.TempDir(), "the-file") + if err := os.WriteFile(path, []byte("the machine's own\n"), 0o644); err != nil { + t.Fatal(err) + } + d := filledDecl(t, path, `"content":"","owner-at-runtime":"owned"`, "") + _, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, noServices, + nil, nil) + if err != nil { + t.Fatal(err) + } + gone := elsewhere(t) + report, state, err := Apply(context.Background(), archHost(t), gone, state, store.OriginDeclared, noServices, + nil, nil) + if err != nil { + t.Fatal(err) + } + if got := readText(t, path); got != "the machine's own\n" { + t.Errorf("a filled file the verb never placed was touched when it left: %q", got) + } + if o := outcomeOf(report, "m.conf"); o.Action == "removed" { + t.Errorf("a filled file the verb never placed was removed: %+v", o) + } + if _, ok := state.Find("m.conf"); ok { + t.Errorf("a record of the filled file was left") + } +} + +// What the verb placed is the mesh's: when the file leaves the declaration it is removed as an orphan, as a file +// an apply wrote is. +func TestAFilledFileTheVerbPlacedIsRemovedWhenItLeaves(t *testing.T) { + path := filepath.Join(t.TempDir(), "the-file") + d := filledDecl(t, path, `"content":"","owner-at-runtime":"owned"`, "") + _, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, noServices, + nil, nil) + if err != nil { + t.Fatal(err) + } + _, state, err = WriteFilled(context.Background(), archHost(t), d, state, path, []byte("port = 8080\n"), + noServices, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + gone := elsewhere(t) + report, state, err := Apply(context.Background(), archHost(t), gone, state, store.OriginDeclared, noServices, + nil, nil) + if err != nil { + t.Fatal(err) + } + if _, err := os.Stat(path); err == nil { + t.Errorf("a filled file the verb placed stayed when it left the declaration: %q", readText(t, path)) + } + if o := outcomeOf(report, "m.conf"); o.Action != "removed" { + t.Errorf("not removed: %+v", o) + } + if _, ok := state.Find("m.conf"); ok { + t.Errorf("its record stayed") + } +} + +// A file the mesh wrote the old way and that is now filled keeps its record for the verb to judge against, and is +// not written over by the apply that hands it to the client; a file at the same path recorded under another id +// is handed to the filled one, not removed. +func TestAFileThatMovesToTheClientIsHandedNotRewritten(t *testing.T) { + path := filepath.Join(t.TempDir(), "the-file") + old := parseTrusted(t, fmt.Sprintf(`{"declaration":1,"resources":[{"id":"m.old","type":"file","path":%q,`+ + `"content":"port = 1\n"}]}`, path)) + _, state, err := Apply(context.Background(), archHost(t), old, store.State{}, store.OriginDeclared, noServices, + nil, nil) + if err != nil { + t.Fatal(err) + } + was, _ := state.Find("m.old") + d := filledDecl(t, path, `"content":"","owner-at-runtime":"owned"`, "") + _, state, err = Apply(context.Background(), archHost(t), d, state, store.OriginDeclared, noServices, nil, nil) + if err != nil { + t.Fatal(err) + } + if got := readText(t, path); got != "port = 1\n" { + t.Errorf("the file handed to the client was rewritten or removed: %q", got) + } + if _, ok := state.Find("m.old"); ok { + t.Errorf("the record going was kept beside the filled file's") + } + if rec, ok := state.Find("m.conf"); !ok || rec.Wrote != was.Wrote || rec.Target != path { + t.Errorf("the filled file did not take the record of what the mesh wrote there: %+v", rec) + } +} + +// elsewhere is a declaration with nothing at the filled path: one unrelated file. +func elsewhere(t *testing.T) *declaration.Declaration { + t.Helper() + return parseTrusted(t, fmt.Sprintf(`{"declaration":1,"resources":[{"id":"n.other","type":"file","path":%q,`+ + `"content":"x\n"}]}`, filepath.Join(t.TempDir(), "other"))) +} diff --git a/internal/apply/filled_test.go b/internal/apply/filled_test.go index 6166d31..2f7ff74 100644 --- a/internal/apply/filled_test.go +++ b/internal/apply/filled_test.go @@ -90,8 +90,11 @@ func TestAFilledFileIsWrittenAsItsOwnershipSays(t *testing.T) { applied := at() withContent := strings.Replace(c.fields, fmt.Sprintf(`"content":%q`, contentOf(t, c.fields)), fmt.Sprintf(`"content":%q`, c.content), 1) - want, _, err := Apply(context.Background(), archHost(t), filledDecl(t, applied, withContent, ""), - store.State{}, store.OriginDeclared, noServices, nil, nil) + // The same file declared the old way: an apply writes a filled file only on the client's word (P7). + plain := parseTrusted(t, fmt.Sprintf(`{"declaration":1,"resources":[{"id":"m.conf","type":"file",`+ + `"path":%q,%s}]}`, applied, withContent)) + want, _, err := Apply(context.Background(), archHost(t), plain, store.State{}, store.OriginDeclared, + noServices, nil, nil) if err != nil { t.Fatal(err) } diff --git a/internal/apply/vault_sealed_test.go b/internal/apply/vault_sealed_test.go new file mode 100644 index 0000000..0652c4a --- /dev/null +++ b/internal/apply/vault_sealed_test.go @@ -0,0 +1,112 @@ +package apply + +// A core credential the vault sealed to this machine (novox/hq ADR 0349 §16, issue 755): the controller carries the +// vault's box (ADR 0206) in the declaration and cannot open it; this machine opens it with its sealing key and writes +// the value, as it writes a value the controller sealed. + +import ( + "context" + "encoding/json" + "os" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/seal" + + "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/identity" + "github.com/novox/mesh-host/internal/store" +) + +// vaultSealedTo is what the vault answers the controller: the value in the ADR 0206 box, sealed to the machine's +// sealing key as SPKI PEM. +func vaultSealedTo(t *testing.T, key identity.SealingKey, value string) seal.Box { + t.Helper() + pub, err := key.PublicPEM() + if err != nil { + t.Fatal(err) + } + box, err := seal.Seal(value, pub) + if err != nil { + t.Fatal(err) + } + return box +} + +func vaultSealedFile(t *testing.T, path string, box seal.Box) *declaration.Declaration { + t.Helper() + raw := map[string]any{"declaration": 1, "resources": []map[string]any{ + {"id": "broker", "type": "file", "path": path, "vault-sealed": box}, + }} + body, _ := json.Marshal(raw) + d, err := declaration.Parse(body) + if err != nil { + t.Fatal(err) + } + return d +} + +func boxOpener(key identity.SealingKey) Unseal { + return func(sealed string) ([]byte, error) { return key.Open(sealed) } +} + +func TestAFileTheVaultSealedToThisMachineIsOpenedAndWrittenUnreadableToOthers(t *testing.T) { + key, err := identity.GenerateSealingKey() + if err != nil { + t.Fatal(err) + } + path := t.TempDir() + "/broker" + d := vaultSealedFile(t, path, vaultSealedTo(t, key, "the-provisioners-bus-password")) + report, state, err := Apply(context.Background(), archHost(t), d, store.State{}, + store.OriginCarried, noServices, nil, boxOpener(key)) + if err != nil { + t.Fatal(err) + } + on, err := os.ReadFile(path) + if err != nil || string(on) != "the-provisioners-bus-password" { + t.Fatalf("the file holds %q, %v", on, err) + } + if info, _ := os.Stat(path); info.Mode().Perm() != 0o600 { + t.Errorf("a credential landed mode %o", info.Mode().Perm()) + } + said, _ := json.Marshal(report) + kept, _ := json.Marshal(state) + for what, blob := range map[string][]byte{"the report": said, "the node's state": kept} { + if strings.Contains(string(blob), "the-provisioners-bus-password") { + t.Fatalf("%s carries the value in the clear", what) + } + } +} + +func TestAVaultBoxSealedToAnotherMachineIsRefusedAndNothingWritten(t *testing.T) { + mine, _ := identity.GenerateSealingKey() + theirs, _ := identity.GenerateSealingKey() + path := t.TempDir() + "/broker" + d := vaultSealedFile(t, path, vaultSealedTo(t, theirs, "not for you")) + if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, + store.OriginCarried, noServices, nil, boxOpener(mine)); err == nil { + t.Fatal("a box sealed to another machine was applied") + } + if _, err := os.Stat(path); err == nil { + t.Fatal("something was written before the failure") + } +} + +func TestAVaultSealedFileSaysWhatIsInItOnce(t *testing.T) { + key, _ := identity.GenerateSealingKey() + box, _ := json.Marshal(vaultSealedTo(t, key, "x")) + for what, extra := range map[string]string{ + "content": `"content":"a"`, "sealed": `"sealed":"b"`, "bytes": `"bytes":"Yw=="`, + "secrets": `"content":"${secret:x}","secrets":{"x":"c"}`, "into": `"into":"json","content":"{}"`, + } { + _, err := declaration.Parse([]byte(`{"declaration":1,"resources":[{"id":"f","type":"file","path":"/etc/x",` + + `"vault-sealed":` + string(box) + `,` + extra + `}]}`)) + if err == nil { + t.Errorf("a vault-sealed file that also says %s was accepted", what) + } + } + if _, err := declaration.Parse([]byte(`{"declaration":1,"resources":[{"id":"f","type":"file","path":"/etc/x",` + + `"vault-sealed":{"v":2,"eph":"a","iv":"b","tag":"c","ct":"d"}}]}`)); err == nil { + t.Error("a box of a version the SDK does not open was accepted") + } +} diff --git a/internal/declaration/declaration.go b/internal/declaration/declaration.go index 6ef97ec..c66e9e7 100644 --- a/internal/declaration/declaration.go +++ b/internal/declaration/declaration.go @@ -19,6 +19,8 @@ import ( "sort" "strconv" "strings" + + "git.novox.be/novox/mesh-sdk/go/seal" ) // Version is the vocabulary this host speaks. A declaration naming any other version is @@ -205,6 +207,12 @@ type File struct { // by looking rather than by knowing which field won. Sealed string `json:"sealed,omitempty"` + // VaultSealed is content the vault sealed to this node's sealing key, in the mesh's sealed box (novox/hq ADR + // 0206): a core module's own credential, which the controller carries in the declaration and cannot open + // (ADR 0349 §16, issue 755). The same promise as Sealed, from the vault rather than the controller; sent only to + // an engine that states VaultBoxContract. Exclusive with Content, Sealed and Bytes. + VaultSealed *seal.Box `json:"vault-sealed,omitempty"` + // Trusted is the catalogue's word on whether the settings this file's content took are trusted (novox/hq // issue 339). It is the controller's to act on, and a controller takes it out before it sends a declaration. // Accepted here, and nothing is done with it, so that a controller that passes it on — an older one, or one @@ -278,7 +286,7 @@ func (f *File) Seeded() bool { return f.CreateOnce || f.OwnerAtRuntime == Runtim // Secret reports whether this file arrived sealed, which is what decides both that it must be // opened before writing and that its contents must never appear in a report. -func (f *File) Secret() bool { return f.Sealed != "" } +func (f *File) Secret() bool { return f.Sealed != "" || f.VaultSealed != nil } func (f *File) Identity() string { return f.ID } func (f *File) Kind() Type { return TypeFile } @@ -351,7 +359,7 @@ func (f *File) validate(where string, _ bool) []string { where, f.At)) } if f.Into == IntoJSON || f.Into == IntoBlock { - if f.Sealed != "" || f.Bytes != "" || len(f.Secrets) > 0 || f.CreateOnce { + if f.Secret() || f.Bytes != "" || len(f.Secrets) > 0 || f.CreateOnce { problems = append(problems, where+ ": a file written into says only its part, in content — not sealed, bytes, "+ "secrets or create-once") @@ -366,6 +374,16 @@ func (f *File) validate(where string, _ bool) []string { said = append(said, name) } } + if f.VaultSealed != nil { + said = append(said, "vault-sealed") + if f.VaultSealed.V != seal.Version { + problems = append(problems, fmt.Sprintf("%s: vault-sealed is a sealed box of version %d, and this "+ + "node-engine opens version %d", where, f.VaultSealed.V, seal.Version)) + } + if len(f.Secrets) > 0 { + problems = append(problems, where+": a file the vault sealed whole has no holes for secrets") + } + } if len(said) > 1 { sort.Strings(said) problems = append(problems, where+ diff --git a/internal/identity/sealing.go b/internal/identity/sealing.go index 218f8bc..15794c1 100644 --- a/internal/identity/sealing.go +++ b/internal/identity/sealing.go @@ -3,11 +3,15 @@ package identity import ( "crypto/ecdh" "crypto/rand" + "crypto/x509" "encoding/base64" + "encoding/json" + "encoding/pem" "fmt" "os" "strings" + "git.novox.be/novox/mesh-sdk/go/seal" "golang.org/x/crypto/nacl/box" ) @@ -141,3 +145,63 @@ func Seal(publicKey string, value []byte) (string, error) { } return base64.StdEncoding.EncodeToString(sealed), nil } + +// **The vault's box, opened with the same key** (novox/hq ADR 0349 §16, ADR 0206, issue 755). The core's own +// credentials travel in the declaration as values the vault sealed to this machine, which the controller carries and +// cannot open. The vault seals in the mesh's sealed box (mesh-sdk go/seal), whose keys travel as PEM: the public half +// as SPKI, the private as PKCS #8. They are this key's two halves in another wrapping, never another key. + +// PublicPEM is this machine's sealing key's public half as SPKI PEM, the form the vault seals to. +func (s SealingKey) PublicPEM() (string, error) { + raw, err := base64.StdEncoding.DecodeString(s.Public) + if err != nil || len(raw) != 32 { + return "", fmt.Errorf("this node's sealing key is unusable") + } + pub, err := ecdh.X25519().NewPublicKey(raw) + if err != nil { + return "", fmt.Errorf("this node's sealing key is unusable: %w", err) + } + der, err := x509.MarshalPKIXPublicKey(pub) + if err != nil { + return "", err + } + return string(pem.EncodeToMemory(&pem.Block{Type: "PUBLIC KEY", Bytes: der})), nil +} + +// OpenVaultBox opens a value the vault sealed to this machine. It fails for a box sealed to another key, or changed. +func (s SealingKey) OpenVaultBox(b seal.Box) ([]byte, error) { + raw, err := base64.StdEncoding.DecodeString(s.Private) + if err != nil || len(raw) != 32 { + return nil, fmt.Errorf("this node's sealing key is unusable") + } + priv, err := ecdh.X25519().NewPrivateKey(raw) + if err != nil { + return nil, fmt.Errorf("this node's sealing key is unusable: %w", err) + } + der, err := x509.MarshalPKCS8PrivateKey(priv) + if err != nil { + return nil, err + } + block := &pem.Block{Type: "PRIVATE KEY", Bytes: der} + opened, err := seal.Open(b, string(pem.EncodeToMemory(block))) + for i := range block.Bytes { + block.Bytes[i] = 0 + } + if err != nil { + return nil, fmt.Errorf("this was not sealed by the vault to this node's current key") + } + return []byte(opened), nil +} + +// Open opens anything the mesh sealed to this machine: the vault's box, given as its JSON (it begins with `{`), or +// the controller's sealed value, base64 (which never does). +func (s SealingKey) Open(sealed string) ([]byte, error) { + if strings.HasPrefix(strings.TrimSpace(sealed), "{") { + var b seal.Box + if err := json.Unmarshal([]byte(sealed), &b); err != nil { + return nil, fmt.Errorf("this is not the vault's sealed box: %w", err) + } + return s.OpenVaultBox(b) + } + return s.Unseal(sealed) +} diff --git a/internal/link/handover.go b/internal/link/handover.go index 12f5d14..1c5188b 100644 --- a/internal/link/handover.go +++ b/internal/link/handover.go @@ -143,7 +143,14 @@ func believed(check HandOverCheck, context string, withPath bool, body []byte) ( // believedBytes is believed with the ask's bytes as signed, for an ask that carries more than a hand-over's // fields: read out of them only once they are believed. func believedBytes(check HandOverCheck, context string, withPath bool, body []byte) (HandOverAsk, []byte, string) { - if len(body) > MaxHandOverAsk { + return believedBytesWithin(check, context, withPath, body, MaxHandOverAsk) +} + +// believedBytesWithin is believedBytes for an ask bounded at max bytes: a move carries a sealed value, more than +// a hand-over's few hundred bytes (movesecret.go). +func believedBytesWithin(check HandOverCheck, context string, withPath bool, body []byte, max int) (HandOverAsk, + []byte, string) { + if len(body) > max { return HandOverAsk{}, nil, fmt.Sprintf("the ask is %d bytes, more than an ask", len(body)) } var signed SignedHandOver diff --git a/internal/link/hearing_nats.go b/internal/link/hearing_nats.go index a9aace5..d8b945f 100644 --- a/internal/link/hearing_nats.go +++ b/internal/link/hearing_nats.go @@ -68,6 +68,8 @@ type natsLink struct { judging *nats.Subscription // acting hears the controller asking for the take act on a library (novox/hq ADR 0328 §7). acting *nats.Subscription + // moving hears the controller asking to move a value sealed to this machine into the vault (issue 676). + moving *nats.Subscription node string // signer is the mesh's key, which a hand-over ask is verified with as a declaration is (issue 356). signer ed25519.PublicKey @@ -279,6 +281,24 @@ func (l *natsLink) HearAccessActs(handle AccessActFunc) { } } +// HearMoveSecrets listens for the controller asking this node to move one value sealed to it into the vault (design +// 57's V3, issue 676), on this machine's own subject, verified as a hand-over is (AnswerMoveSecret). A bus whose +// user list is older than the grant refuses the subscription; the controller then hears no answer. +func (l *natsLink) HearMoveSecrets(handle MoveSecretFunc) { + if handle == nil { + return + } + check := HandOverCheck{Signer: l.signer, Node: l.node} + moving, err := l.conn.Subscribe(AskMoveSecretSubject(l.node), func(msg *nats.Msg) { + if body, answer := MoveSecretReply(check, handle, msg.Reply, msg.Data); answer { + _ = msg.Respond(body) + } + }) + if err == nil { + l.moving = moving + } +} + func (l *natsLink) Declarations() <-chan Declaration { return l.arrived } func (l *natsLink) AskedToReport() <-chan struct{} { return l.asked } func (l *natsLink) Lost() <-chan error { return l.lost } @@ -299,6 +319,9 @@ func (l *natsLink) Close() { if l.acting != nil { _ = l.acting.Unsubscribe() } + if l.moving != nil { + _ = l.moving.Unsubscribe() + } if l.judging != nil { _ = l.judging.Unsubscribe() } diff --git a/internal/link/messages.go b/internal/link/messages.go index 3a849a1..319fbd7 100644 --- a/internal/link/messages.go +++ b/internal/link/messages.go @@ -308,6 +308,26 @@ const OwnershipContract = 9 // declaration for it. const ProvisionerContract = 10 +// HandedContract is the statement of a host that writes a filled file only on the provisioner client's word (novox/hq +// ADR 0349 §5, design 57 P7): an apply neither writes nor records a file the declaration names as filled, and removes +// one as an orphan only when the write verb placed it. Its presence is what tells the controller it may hand a moved +// module's files to the provisioner here (mesh-controller internal/link/protocol.go's HandedContract): to an older +// host a filled file would have two writers. +const HandedContract = 11 + +// VaultBoxContract is the statement of a host that also reads a file's `vault-sealed` (novox/hq ADR 0349 §16, issue +// 755): a core module's own credential the vault sealed to this machine in the mesh's sealed box (ADR 0206), which +// the controller carries and cannot open, opened here with this machine's sealing key and written as a sealed file +// is. Its presence is what tells the controller it may send the vault's box here (mesh-controller internal/link +// protocol.go's VaultBoxContract); an older host refuses the whole declaration for the field, and is sent the +// controller's sealed value as before. +const VaultBoxContract = 12 + +// NodeKeyContract is the statement of a host that states its own sealing key to the vault's node-key, in its own name, +// at its start and whenever the key changes (novox/hq ADR 0372, issue 786). The same number as mesh-controller +// internal/link/protocol.go's NodeKeyContract. +const NodeKeyContract = 13 + // Health is one statement of every long-running resource's state on this machine (to-be 48 §4). Said in // every report, as an event on each change, and again every minute while anything is not healthy — so a // lost statement is not a lost fault. @@ -372,6 +392,8 @@ type UnitsHealth struct { Failed []FailedUnit `json:"failed"` // Unread names a manager that could not be read, with why. Unread []string `json:"unread,omitempty"` + // Away names an account's own manager not running, so not read: nothing in it is failed (issue 764). + Away []string `json:"away,omitempty"` } // FailedUnit is one failed unit no module places. diff --git a/internal/link/movesecret.go b/internal/link/movesecret.go new file mode 100644 index 0000000..fdfee75 --- /dev/null +++ b/internal/link/movesecret.go @@ -0,0 +1,209 @@ +package link + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "time" +) + +// The move of a secret into the vault, asked over the bus (design 57's V3, issue 676; the operator's decision of +// 2026-10-14 on that issue). +// +// **Only the machine a value was sealed to can open it**, so the controller, which cannot, asks each machine's +// engine to move what was sealed to it: one row of its store at a time, on AskMoveSecretSubject, signed as a +// hand-over is and held to the same checks (believed), under a signing context of its own, naming the row — a +// module's own secret on this machine, or a pair credential with this machine at one end — its origin, the +// controller's generation, when it was made, and the value as the store holds it, sealed to this machine. The +// engine opens it with its sealing key, hands it to the vault's `move-in` in the vault's two steps, and answers +// what the vault kept only once the vault's read-back opens to the same value. Nothing is rotated; the engine +// writes nothing on this machine. The controller holds the same shapes in its own link code, with a test on each +// side holding the field names. + +// AskMoveSecretSubject is where the controller asks this node's engine to move one value sealed to it into the vault. +func AskMoveSecretSubject(node string) string { return "mesh.node." + node + ".ask.move-secret" } + +// MoveSecretContext is prefixed to a move ask's bytes before signing: never another ask's, nor a declaration's. +const MoveSecretContext = "novox-mesh move-secret v1\n" + +// MoveSecretAsk is a move's own fields, beside a hand-over's (node, by, expires, nonce; no path): the row — module +// and name for a module's own secret on the ask's node; provider, provision, consumer, consumer module and local +// name for a pair credential — where its value came from, the controller's generation, when it was made, the value +// sealed to this machine as the store holds it, and the sealing key the store says it was sealed to. +type MoveSecretAsk struct { + Module string `json:"module,omitempty"` + Name string `json:"name,omitempty"` + Provider string `json:"provider,omitempty"` + Provision string `json:"provision,omitempty"` + Consumer string `json:"consumer,omitempty"` + ConsumerModule string `json:"consumer_module,omitempty"` + Local string `json:"local,omitempty"` + Origin string `json:"origin"` + Generation string `json:"generation,omitempty"` + MadeAt time.Time `json:"made_at"` + Sealed string `json:"sealed"` + SealedTo string `json:"sealed_to,omitempty"` +} + +// MoveSecretAnswer is the engine's answer: what the vault kept, or why nothing was moved — never both. +type MoveSecretAnswer struct { + Said string `json:"said,omitempty"` + Refused string `json:"refused,omitempty"` + Secret string `json:"secret,omitempty"` + Generation int `json:"generation,omitempty"` + Unchanged bool `json:"unchanged,omitempty"` +} + +// MoveSecret is one move asked and believed, as the handler is given it: the row, this machine, and who asked. +type MoveSecret struct { + MoveSecretAsk + Node, By string +} + +// Own says the row is a module's own secret rather than a pair credential. +func (m MoveSecret) Own() bool { return m.Module != "" || m.Name != "" } + +// MovedSecret is what the vault kept: its name, its generation, and whether it held the value already. +type MovedSecret struct { + Secret string + Generation int + Unchanged bool +} + +// MoveSecretFunc moves one value into the vault: what it kept, or why not. +type MoveSecretFunc func(MoveSecret) (MovedSecret, error) + +// MaxMoveSecretAsk bounds a move's request: a row's names, a time, a value sealed and a signature. +const MaxMoveSecretAsk = 64 << 10 + +// movedOrigins are the origins the controller's store holds. +var movedOrigins = map[string]bool{"made": true, "accepted": true, "issued": true} + +func refusedMove(why string) MoveSecretAnswer { + return MoveSecretAnswer{Refused: why + "; nothing was moved"} +} + +// oneName is a short name of one line. +func oneName(s string) bool { + return s != "" && len(s) <= MaxActField && !strings.ContainsAny(s, "\r\n\x00/ ") +} + +// AnswerMoveSecret is the engine's judgement of one move ask's bytes: believed as a hand-over is, then the row — +// one shape, this machine's, a value to open and an origin — and only then the handler. +func AnswerMoveSecret(check HandOverCheck, handle MoveSecretFunc, body []byte) MoveSecretAnswer { + if handle == nil { + return refusedMove("this engine moves no secret into the vault") + } + ask, signed, refused := believedBytesWithin(check, MoveSecretContext, false, body, MaxMoveSecretAsk) + if refused != "" { + return refusedMove(refused) + } + var move MoveSecretAsk + if err := json.Unmarshal(signed, &move); err != nil { + return refusedMove("the signed ask is not a move's ask: " + err.Error()) + } + own := move.Module != "" || move.Name != "" + pair := move.Provider != "" || move.Provision != "" || move.Consumer != "" || move.ConsumerModule != "" || + move.Local != "" + switch { + case own == pair: + return refusedMove("the ask names neither one module's own secret nor one pair credential") + case own && (!oneName(move.Module) || !oneName(move.Name)): + return refusedMove("the ask's module and name are not short names") + case pair && (!oneName(move.Provider) || !oneName(move.Provision) || !oneName(move.Consumer) || + !oneName(move.ConsumerModule) || (move.Local != "" && !oneName(move.Local))): + return refusedMove("the ask's provider, provision, consumer, consuming module and local name are not short names") + case pair && move.Provider != ask.Node && move.Consumer != ask.Node: + return refusedMove(fmt.Sprintf("the pair is between %s and %s, and this is %s", move.Provider, move.Consumer, + ask.Node)) + case !movedOrigins[move.Origin]: + return refusedMove(fmt.Sprintf("the origin %q is not made, accepted or issued", move.Origin)) + case strings.TrimSpace(move.Sealed) == "": + return refusedMove("the ask carries no value sealed to this machine") + case len(move.Generation) > MaxActField || strings.ContainsAny(move.Generation, "\r\n\x00"): + return refusedMove("the ask's generation is not one short line") + } + moved, err := handle(MoveSecret{MoveSecretAsk: move, Node: ask.Node, By: ask.By}) + if err != nil { + return refusedMove(err.Error()) + } + said := fmt.Sprintf("%s is in the vault at generation %d, read back the same", moved.Secret, moved.Generation) + if moved.Unchanged { + said = fmt.Sprintf("the vault held %s already, the same value (generation %d)", moved.Secret, moved.Generation) + } + return MoveSecretAnswer{Said: said, Secret: moved.Secret, Generation: moved.Generation, Unchanged: moved.Unchanged} +} + +// MoveSecretReply is what a move request is answered with, and whether to answer, as HandOverReply. +func MoveSecretReply(check HandOverCheck, handle MoveSecretFunc, reply string, body []byte) ([]byte, bool) { + if reply == "" { + return nil, false + } + answer, err := json.Marshal(AnswerMoveSecret(check, handle, body)) + if err != nil { + return nil, false + } + return answer, true +} + +// MoveSecrets is a link that hears the controller asking this node to move a value into the vault. Optional. +type MoveSecrets interface { + HearMoveSecrets(MoveSecretFunc) +} + +// ---- asking the vault ------------------------------------------------------------------------------------------ + +// VaultSeat is the vault's seat; its verbs are asked on `mesh.seat..call..`, the subject that +// names its caller (novox/hq issue 365), which the bus lets only that user publish. +const VaultSeat = "mesh-vault" + +// NodeEngineUser is the bus user of a machine's node-engine, as the controller's broker names it. +func NodeEngineUser(node string) string { return "node." + node } + +// CalledVaultSubject is where this node's engine asks one of the vault's verbs in its own name: each dot of its +// user's name written `~`, as the controller's broker writes a caller's token. +func CalledVaultSubject(verb, node string) string { + return "mesh.seat." + VaultSeat + ".call." + verb + "." + strings.ReplaceAll(NodeEngineUser(node), ".", "~") +} + +// VaultWithin is how long the engine gives the vault to answer one step. +const VaultWithin = 15 * time.Second + +// AskVault asks one of the vault's verbs in this node's engine's own name over the link open now, and answers its +// result, or its refusal as an error. +func (q *Queue) AskVault(ctx context.Context, verb string, args any) (json.RawMessage, error) { + q.init() + q.mu.Lock() + bus := q.bus + q.mu.Unlock() + if bus == nil { + return nil, fmt.Errorf("%w: this machine has no link to the bus open now", ErrNotAsked) + } + asker, ok := bus.(ToolBus) + if !ok { + return nil, fmt.Errorf("%w: the link to the bus open now cannot ask the vault", ErrNotAsked) + } + body, err := json.Marshal(args) + if err != nil { + return nil, err + } + asking, cancel := context.WithTimeout(ctx, VaultWithin) + defer cancel() + reply, err := asker.Ask(asking, CalledVaultSubject(verb, q.Membership.Node), body) + if err != nil { + return nil, fmt.Errorf("the vault's %s could not be asked: %w", verb, err) + } + var env struct { + Result json.RawMessage `json:"result"` + Error string `json:"error"` + } + if err := json.Unmarshal(reply, &env); err != nil { + return nil, fmt.Errorf("the vault answered %s with something that is not a tool's answer", verb) + } + if env.Error != "" { + return nil, errors.New(env.Error) + } + return env.Result, nil +} diff --git a/internal/link/movesecret_test.go b/internal/link/movesecret_test.go new file mode 100644 index 0000000..5cb244a --- /dev/null +++ b/internal/link/movesecret_test.go @@ -0,0 +1,141 @@ +package link + +import ( + "crypto/ed25519" + "encoding/json" + "errors" + "fmt" + "strings" + "testing" + "time" + + "github.com/nats-io/nats.go" +) + +// The move's ask (design 57's V3, issue 676): a hand-over's fields and the row's own in one signed object, under a +// signing context of its own. The controller's side holds the same names and words. +func TestTheMovesAskKeepsItsFieldNamesAndContext(t *testing.T) { + body, _ := json.Marshal(MoveSecretAsk{Module: "m", Name: "n", Provider: "p", Provision: "q", Consumer: "c", + ConsumerModule: "cm", Local: "l", Origin: "made", Generation: "g", MadeAt: time.Unix(1, 0).UTC(), + Sealed: "s", SealedTo: "k"}) + if got := keysOf(t, body); got != "consumer consumer_module generation local made_at module name origin provider "+ + "provision sealed sealed_to" { + t.Fatalf("the move's fields are %q", got) + } + if AskMoveSecretSubject("ace") != "mesh.node.ace.ask.move-secret" { + t.Fatalf("the subject is %q", AskMoveSecretSubject("ace")) + } + if MoveSecretContext != "novox-mesh move-secret v1\n" { + t.Fatalf("the signing context is %q", MoveSecretContext) + } + answer, _ := json.Marshal(MoveSecretAnswer{Said: "s", Refused: "r", Secret: "own/a/b/c", Generation: 1, Unchanged: true}) + if got := keysOf(t, answer); got != "generation refused said secret unchanged" { + t.Fatalf("the answer's fields are %q", got) + } +} + +// signedMove is a move's ask as the controller makes it. +func signedMove(t *testing.T, key ed25519.PrivateKey, ask HandOverAsk, move MoveSecretAsk) []byte { + t.Helper() + raw, _ := json.Marshal(struct { + HandOverAsk + MoveSecretAsk + }{ask, move}) + body, _ := json.Marshal(SignedHandOver{Ask: raw, Signature: ed25519.Sign(key, append([]byte(MoveSecretContext), raw...))}) + return body +} + +// A move is believed as every signed ask is, then names one row this machine holds — its own secret, or a pair +// credential with this machine at one end — with a value sealed to it and an origin the store has; only then is the +// handler given it. A hand-over's signature is not a move's. +func TestAMoveIsTakenOnlyForARowThisMachineHolds(t *testing.T) { + m, key := aMember(t) + now := time.Now() + var given []MoveSecret + handle := func(s MoveSecret) (MovedSecret, error) { + given = append(given, s) + return MovedSecret{Secret: "own/" + m.Node + "/postgres/superuser-password", Generation: 1}, nil + } + check := HandOverCheck{Signer: m.Signer, Node: m.Node, Nonces: &Nonces{}} + ask := func(nonce string) HandOverAsk { + return HandOverAsk{Node: m.Node, By: "the move", Expires: now.Add(HandOverGood), Nonce: nonce} + } + own := MoveSecretAsk{Module: "postgres", Name: "superuser-password", Origin: "made", Sealed: "c2VhbGVk"} + for why, body := range map[string][]byte{ + "a hand-over's": signedAsk(t, key, ask(aNonce+"01")), + "no value": signedMove(t, key, ask(aNonce+"02"), MoveSecretAsk{Module: "postgres", Name: "x", Origin: "made"}), + "no origin": signedMove(t, key, ask(aNonce+"03"), MoveSecretAsk{Module: "postgres", Name: "x", Sealed: "c2Vh"}), + "both shapes": signedMove(t, key, ask(aNonce+"04"), MoveSecretAsk{Module: "postgres", Name: "x", Provider: m.Node, + Provision: "database", Consumer: "shanks", ConsumerModule: "gitea", Origin: "made", Sealed: "c2Vh"}), + "neither end": signedMove(t, key, ask(aNonce+"05"), MoveSecretAsk{Provider: "anchor", Provision: "database", + Consumer: "shanks", ConsumerModule: "gitea", Origin: "made", Sealed: "c2Vh"}), + "a path": signedMove(t, key, HandOverAsk{Node: m.Node, Path: "/etc", By: "x", Expires: now.Add(HandOverGood), + Nonce: aNonce + "06"}, own), + "two lines": signedMove(t, key, ask(aNonce+"07"), MoveSecretAsk{Module: "postgres", Name: "x\ny", Origin: "made", + Sealed: "c2Vh"}), + } { + if a := AnswerMoveSecret(check, handle, body); a.Refused == "" || !strings.HasSuffix(a.Refused, "nothing was moved") { + t.Errorf("%s: not refused, or refused without saying nothing was moved: %+v", why, a) + } + } + if len(given) != 0 { + t.Fatalf("a refused ask reached the move: %+v", given) + } + a := AnswerMoveSecret(check, handle, signedMove(t, key, ask(aNonce+"08"), own)) + if a.Refused != "" || a.Secret != "own/"+m.Node+"/postgres/superuser-password" || a.Generation != 1 || a.Said == "" || + len(given) != 1 || given[0].Name != "superuser-password" || given[0].By != "the move" { + t.Fatalf("a good ask: %+v %+v", a, given) + } + pair := MoveSecretAsk{Provider: "anchor", Provision: "database", Consumer: m.Node, ConsumerModule: "gitea", + Origin: "accepted", Sealed: "c2Vh"} + if a := AnswerMoveSecret(check, handle, signedMove(t, key, ask(aNonce+"09"), pair)); a.Refused != "" { + t.Fatalf("a pair this machine consumes was refused: %+v", a) + } + // What the move refuses is said, and nothing else. + refusing := func(MoveSecret) (MovedSecret, error) { return MovedSecret{}, errors.New("the vault said no") } + if a := AnswerMoveSecret(check, refusing, signedMove(t, key, ask(aNonce+"10"), own)); !strings.Contains(a.Refused, + "the vault said no") || a.Secret != "" { + t.Fatalf("a refused move: %+v", a) + } +} + +// **Against a real server**: the move's signed ask on the node's own subject is answered with what the handler +// moved, and the engine asks the vault on the subject that names it, `node~` last. +func TestNatsAMoveIsAskedOfTheNodeAndTheVaultIsAskedInItsName(t *testing.T) { + conn, js := aBus(t) + m, key := aMember(t) + vault, err := conn.Subscribe("mesh.seat.mesh-vault.call.move-in.node~"+m.Node, func(msg *nats.Msg) { + _ = msg.Respond([]byte(`{"result":{"call":"c1","seal_to":"k"}}`)) + }) + if err != nil { + t.Fatal(err) + } + defer vault.Unsubscribe() + l := &natsLink{conn: conn, js: js, node: m.Node, signer: m.Signer} + q := &Queue{Membership: m} + q.attach(t.Context(), l) + l.HearMoveSecrets(func(s MoveSecret) (MovedSecret, error) { + raw, err := q.AskVault(t.Context(), "move-in", map[string]string{"module": s.Module}) + if err != nil || !strings.Contains(string(raw), `"call":"c1"`) { + return MovedSecret{}, fmt.Errorf("the vault answered %s, %v", raw, err) + } + return MovedSecret{Secret: "own/" + s.Node + "/" + s.Module + "/" + s.Name, Generation: 1}, nil + }) + defer l.Close() + if err := conn.Flush(); err != nil { + t.Fatal(err) + } + ask := HandOverAsk{Node: m.Node, By: "the move", Expires: time.Now().Add(HandOverGood), Nonce: strings.Repeat("ab", 16)} + reply, err := conn.Request(AskMoveSecretSubject(m.Node), signedMove(t, key, ask, + MoveSecretAsk{Module: "postgres", Name: "superuser-password", Origin: "made", Sealed: "c2Vh"}), 5*time.Second) + if err != nil { + t.Fatal(err) + } + var said MoveSecretAnswer + if err := json.Unmarshal(reply.Data, &said); err != nil || said.Secret != "own/"+m.Node+"/postgres/superuser-password" { + t.Fatalf("answered %s, %v", reply.Data, err) + } + if _, err := (&Queue{Membership: Membership{Node: "nowhere"}}).AskVault(t.Context(), "move-in", nil); !errors.Is(err, ErrNotAsked) { + t.Fatalf("a queue with no link asked the vault: %v", err) + } +} diff --git a/internal/link/queue.go b/internal/link/queue.go index 1c345ee..9d064a6 100644 --- a/internal/link/queue.go +++ b/internal/link/queue.go @@ -65,8 +65,11 @@ type Queue struct { // AccessAct judges and runs the take act on an access's folder, asked by the controller on an approved // warrant (novox/hq ADR 0328 §7). Nil answers every such ask with a refusal. AccessAct AccessActFunc - Say Announce - Timeout time.Duration + // MoveSecret moves one value sealed to this machine into the vault, asked by the controller during the move of + // the secrets (design 57's V3, issue 676). Nil answers every such ask with a refusal. + MoveSecret MoveSecretFunc + Say Announce + Timeout time.Duration once sync.Once wake chan struct{} diff --git a/internal/link/run.go b/internal/link/run.go index eb46c48..72028b9 100644 --- a/internal/link/run.go +++ b/internal/link/run.go @@ -256,6 +256,10 @@ func serve(ctx context.Context, link Link, m Membership, queue *Queue, say Annou if s, ok := link.(AccessActs); ok { s.HearAccessActs(queue.AccessAct) } + // And asking to move a value sealed to this machine into the vault (design 57's V3, issue 676). + if s, ok := link.(MoveSecrets); ok { + s.HearMoveSecrets(queue.MoveSecret) + } for { // Asked to stop — by standing aside, say — nothing further is taken in. diff --git a/internal/liveness/liveness_test.go b/internal/liveness/liveness_test.go index b304706..aa39729 100644 --- a/internal/liveness/liveness_test.go +++ b/internal/liveness/liveness_test.go @@ -330,3 +330,56 @@ func TestOneLookIsOneReadOfEveryContainer(t *testing.T) { t.Fatalf("a container the inspect found, beside one it did not: %+v", s) } } + +// **A look starts no account's manager** (novox/hq issue 764, ADR 0177): a user-scoped unit's manager is +// asked of the machine's first; one not running is not read, nothing is started, and its unit is said +// unknown with why — away, never down; one running is read as before. +func TestAUserUnitsManagerNotRunningIsNotReadAndNotDown(t *testing.T) { + var asked []string + manager := "inactive" + run := func(_ context.Context, name string, args ...string) (string, error) { + asked = append(asked, name+" "+strings.Join(args, " ")) + switch { + case name == "getent": + return "operator:x:1000:1000::/home/operator:/bin/zsh\n", nil + case name == "systemctl" && len(args) > 0 && args[0] == "is-active": + if manager != "active" { + return manager + "\n", errors.New("systemctl exited 3") + } + return "active\n", nil + case name == "systemctl": + return "Id=dunst.service\nLoadState=loaded\nActiveState=active\nSubState=running\nNRestarts=0\nInvocationID=i1\n", nil + } + return "", errors.New("not a command the judge may run") + } + c := &clock{now: t0} + j := aJudge(t, &Exec{Run: run, System: "arch"}, c, "") + j.Set([]Resource{{Module: "dunst", ID: "dunst.daemon", Kind: KindService, Target: "dunst.service", + Scope: "user", User: "operator"}}) + var st Statement + for i := 0; i < 4; i++ { + c.now = t0.Add(time.Duration(i) * time.Minute) + st, _ = j.Look(t.Context()) + } + if s := stateOf(t, st, "dunst.daemon"); s.State != Unknown || !strings.Contains(s.Reason, "not logged in") { + t.Fatalf("a unit in a manager not running is away, not down: %+v", s) + } + for _, a := range asked { + if strings.Contains(a, "--machine=") { + t.Fatalf("the account's own manager was asked while it was not running: %q", a) + } + if a != "getent passwd operator" && a != "systemctl is-active user@1000.service" { + t.Errorf("asked something but whether the manager runs: %q", a) + } + } + // Logged in: read in the account's own manager, as before. + manager, asked = "active", nil + c.now = t0.Add(5 * time.Minute) + st, _ = j.Look(t.Context()) + if s := stateOf(t, st, "dunst.daemon"); s.State == Unknown || s.State == Unhealthy { + t.Fatalf("a unit in a running manager is read: %+v", s) + } + if last := asked[len(asked)-1]; !strings.HasPrefix(last, "systemctl --user --machine=operator@ show ") { + t.Fatalf("the running manager is read in the account's own: %q", last) + } +} diff --git a/internal/liveness/runtime.go b/internal/liveness/runtime.go index eccdc5a..93d101a 100644 --- a/internal/liveness/runtime.go +++ b/internal/liveness/runtime.go @@ -8,6 +8,8 @@ import ( "strconv" "strings" "sync" + + "github.com/novox/mesh-host/internal/system" ) // Runner runs a command and answers what it printed — the apply's own (apply.ExecRunner), so a look @@ -16,9 +18,13 @@ type Runner func(ctx context.Context, name string, args ...string) (string, erro // Exec reads the container runtime and the service manager through their command lines: one inspect of // every container, one show of every unit per manager. **Reads only** — the whole of what this file may -// ask either of them is `container inspect`, `version` and `show` (ADR 0240 rule 6, and a test holds it). +// ask either of them is `container inspect`, `version` and `show`, and of the machine's manager whether an +// account's own is active (ADR 0240 rule 6, and a test holds it). type Exec struct { Run Runner + // System is what the host was built for: it says how the machine's manager is asked whether an + // account's own runs (system.ManagerRunning). + System string mu sync.Mutex // cri is the runtime that answered, once one has. @@ -146,6 +152,19 @@ func (e *Exec) Units(ctx context.Context, scope, user string, units []string) (m } args := []string{"show", "--property=" + unitProperties, "--"} if scope == "user" && user != "" { + // **A look starts no account's manager** (novox/hq issue 764, ADR 0177): reading it is a login, + // which starts it when none runs. The machine's manager is asked first; a manager not running is + // away — its units unread, said unknown with why, never down — as the apply says them waiting. + running, exists, err := system.ManagerRunning(ctx, e.System, system.Runner(e.Run), user) + switch { + case err != nil: + return nil, fmt.Errorf("whether %s's own service manager runs could not be read: %w", user, err) + case !exists: + return nil, fmt.Errorf("the machine has no account %q, whose own manager these units are in", user) + case !running: + return nil, fmt.Errorf("%s's own service manager is not running: the account is not logged in "+ + "and does not linger", user) + } args = append([]string{"--user", "--machine=" + user + "@"}, args...) } said, err := e.Run(ctx, "systemctl", append(args, units...)...) diff --git a/internal/system/system.go b/internal/system/system.go index e0e8435..e361dff 100644 --- a/internal/system/system.go +++ b/internal/system/system.go @@ -304,6 +304,27 @@ func For(name string) (System, error) { name, strings.Join(names, ", ")) } +// userManagers is a service manager that runs a manager per account (novox/hq ADR 0177). +type userManagers interface { + UserManagerRunning(ctx context.Context, run Runner, account string) (running, exists bool, err error) +} + +// ManagerRunning is whether account's own manager runs on a machine built for name, asked of the +// machine's manager through run — never of the account's, which a question would start (see +// arch.UserManagerRunning). A look asks it before it reads anything in an account's manager (novox/hq +// issue 764). A system with no manager per account is an error: nothing can be read there. +func ManagerRunning(ctx context.Context, name string, run Runner, account string) (running, exists bool, err error) { + sys, err := For(name) + if err != nil { + return false, false, err + } + um, ok := sys.(userManagers) + if !ok { + return false, false, fmt.Errorf("this machine's service manager (%s) has no manager per account", sys.Name()) + } + return um.UserManagerRunning(ctx, run, account) +} + // All is every system the host can be built for. func All() []System { return []System{arch{}, alpine{}, android{}} diff --git a/internal/units/exec.go b/internal/units/exec.go index 8d8f73e..91c29a8 100644 --- a/internal/units/exec.go +++ b/internal/units/exec.go @@ -5,6 +5,8 @@ import ( "errors" "fmt" "strings" + + "github.com/novox/mesh-host/internal/system" ) // Runner runs a command and answers what it printed — the apply's own (apply.ExecRunner), so a look reads @@ -12,8 +14,9 @@ import ( type Runner func(ctx context.Context, name string, args ...string) (string, error) // Exec reads the service managers through systemctl and the package database through the system's own -// owner query. **Reads only**: `list-units`, `show` and the owner query are the whole of what it asks -// (ADR 0240 rule 6, and a test holds it). +// owner query. **Reads only**: `list-units`, `show`, the owner query, and — of the machine's manager and +// user database — whether an account's manager is active, are the whole of what it asks (ADR 0240 rule 6, +// and a test holds it). type Exec struct { Run Runner // System is what the host was built for; it says how a file's package is asked. A system with no @@ -103,3 +106,10 @@ func (e Exec) PackageOwning(ctx context.Context, path string) (string, bool, err } return "", false, nil } + +// UserManagerRunning asks the machine's manager whether an account's own runs, as the apply does before it +// touches a user-scoped unit (system.UserManagerRunning; novox/hq ADR 0177, issue 764). A system with no +// manager per account cannot say, and is an error: that manager is said unread. +func (e Exec) UserManagerRunning(ctx context.Context, account string) (running, exists bool, err error) { + return system.ManagerRunning(ctx, e.System, system.Runner(e.Run), account) +} diff --git a/internal/units/units.go b/internal/units/units.go index f796b5f..3abb2be 100644 --- a/internal/units/units.go +++ b/internal/units/units.go @@ -9,8 +9,8 @@ // own mount table, and a unit a removed package left behind, failed beside it, said by nobody. // // On every look the engine asks each service manager the mesh places units in — the machine's, and the -// account manager of every account the declaration names — which units failed, and says each one's -// owner: +// account manager of every account the declaration places a user-scoped unit for — which units failed, +// and says each one's owner: // // 1. a service or process the declaration states, by its unit and manager; // 2. a file the declaration writes, when the unit's file is that file; @@ -23,6 +23,12 @@ // // **It reads; it never acts** (ADR 0240 rule 6): `list-units`, `show` and the package manager's owner // query are the whole of what it asks. It neither resets nor restarts anything. +// +// **A look starts no account's manager** (novox/hq issue 764, ADR 0177). Reading an account's manager is +// a login, and a login starts the manager when none runs; so the machine's own manager is asked first +// whether user@.service is active, and the account's is read only then. A manager that is not +// running is away: no unit in it is failed, none is read, and the statement says it away — the apply's +// word for the same absence ("waiting" for its manager). package units import ( @@ -72,7 +78,9 @@ type Owned struct { // Files is keyed by path; Packages by package name, only those declared present. Files map[string]owner Packages map[string]owner - // Accounts are the accounts whose own managers are read, sorted. + // Accounts are the accounts whose own managers are read, sorted: each one the declaration places a + // user-scoped unit for, and no other (novox/hq issue 764). An account the declaration only creates + // has no unit of the mesh's in its manager to fail, and asking that manager starts it. Accounts []string } @@ -123,8 +131,6 @@ func OwnedBy(d *declaration.Declaration, held map[string]bool) Owned { if !v.Absent { o.Packages[v.Package] = own } - case *declaration.User: - accounts[v.Name] = true } } for a := range accounts { @@ -169,6 +175,10 @@ type Reader interface { Show(ctx context.Context, scope, user string, units []string) (map[string]Shown, error) // PackageOwning is the package a file belongs to; false when none does or the machine cannot say. PackageOwning(ctx context.Context, path string) (string, bool, error) + // UserManagerRunning is whether an account's own manager runs now, asked of the machine's manager + // and never of the account's, which a question would start (novox/hq issue 764); exists is false + // for an account the machine does not have. + UserManagerRunning(ctx context.Context, account string) (running, exists bool, err error) } // Failed is one failed unit as the statement says it. @@ -198,6 +208,9 @@ type Statement struct { Failed []Failed // Unread names each manager that could not be read, with why. Unread []string + // Away names each account manager not running, so not read: its account is not logged in and does + // not linger. Nothing in it is failed (novox/hq issue 764). + Away []string } // Owned answers the failures a module owns; Unowned those no module does. @@ -290,16 +303,39 @@ func (j *Judge) Look(ctx context.Context) (Statement, bool) { st := Statement{At: now, State: Unknown} read := 0 failedNow := map[string]bool{} + keep := func(m manager) { + // What it held is neither cleared nor counted while it cannot be read. + for k := range j.seen { + if strings.HasPrefix(k, key(m.scope, m.user, "")) { + failedNow[k] = true + } + } + } for _, m := range managers { + if m.scope == ScopeUser { + // Asked of the machine's manager first: reading the account's own would start it. + running, exists, err := j.reader.UserManagerRunning(ctx, m.user) + switch { + case err != nil: + st.Unread = append(st.Unread, fmt.Sprintf("%s could not be read: %s", m, firstLine(err.Error()))) + keep(m) + continue + case !exists: + st.Unread = append(st.Unread, fmt.Sprintf("%s could not be read: the machine has no account %q", + m, m.user)) + keep(m) + continue + case !running: + // Away: nothing in it runs, so nothing in it is failed; what it held is cleared. + st.Away = append(st.Away, m.user+"'s own service manager is not running: the account is not "+ + "logged in and does not linger, so its units are not read") + continue + } + } listed, err := j.reader.Failed(ctx, m.scope, m.user) if err != nil { st.Unread = append(st.Unread, fmt.Sprintf("%s could not be read: %s", m, firstLine(err.Error()))) - // What it held is neither cleared nor counted while it cannot be read. - for k := range j.seen { - if strings.HasPrefix(k, key(m.scope, m.user, "")) { - failedNow[k] = true - } - } + keep(m) continue } read++ @@ -353,7 +389,7 @@ func (j *Judge) Look(ctx context.Context) (Statement, bool) { if read == 0 { st.State = Unknown } - word := st.State + word := st.State + "|away:" + strings.Join(st.Away, ",") for _, f := range st.Failed { word += "|" + f.Scope + "/" + f.Unit + "/" + f.Module } diff --git a/internal/units/units_test.go b/internal/units/units_test.go index 71f7cf5..e5b6adb 100644 --- a/internal/units/units_test.go +++ b/internal/units/units_test.go @@ -21,6 +21,11 @@ type fakeReader struct { owners map[string]string // by path unread map[string]error ownerQs int + // away is each account whose own manager is not running; noAccount each the machine lacks. + away map[string]bool + noAccount map[string]bool + // read is every manager whose failed units were asked, in order. + read []string } func mgr(scope, user string) string { @@ -31,6 +36,7 @@ func mgr(scope, user string) string { } func (f *fakeReader) Failed(_ context.Context, scope, user string) ([]Listed, error) { + f.read = append(f.read, mgr(scope, user)) if err := f.unread[mgr(scope, user)]; err != nil { return nil, err } @@ -45,6 +51,13 @@ func (f *fakeReader) Show(_ context.Context, _, _ string, units []string) (map[s return out, nil } +func (f *fakeReader) UserManagerRunning(_ context.Context, account string) (bool, bool, error) { + if f.noAccount[account] { + return false, false, nil + } + return !f.away[account], true, nil +} + func (f *fakeReader) PackageOwning(_ context.Context, path string) (string, bool, error) { f.ownerQs++ p, ok := f.owners[path] @@ -75,6 +88,11 @@ func declared() *declaration.Declaration { &declaration.Package{ID: "openrazer.daemon", Type: declaration.TypePackage, Package: "openrazer-daemon"}, &declaration.Package{ID: "clipmenu.greenclip", Type: declaration.TypePackage, Package: "rofi-greenclip", Absent: true}, &declaration.User{ID: "zsh.account", Type: declaration.TypeUser, Name: "operator"}, + // The operator's own manager is read because a module places a unit in it (dunst, on a desktop). + &declaration.Service{ID: "dunst.reload", Type: declaration.TypeService, Unit: "dunst.service", + Scope: declaration.ScopeUser, User: "operator"}, + // A module account the declaration only creates: its manager is never asked (issue 764). + &declaration.User{ID: "telegram.account", Type: declaration.TypeUser, Name: "telegram"}, &declaration.Service{ID: "sshd.run", Type: declaration.TypeService, Unit: "sshd.service", State: "running"}, &declaration.Service{ID: "power.after-boot", Type: declaration.TypeService, Unit: "mesh-power-after-boot.service"}, &declaration.File{ID: "watcher.unit", Type: declaration.TypeFile, Path: "/home/operator/.config/systemd/user/watcher.service"}, @@ -283,3 +301,102 @@ func TestNothingIsReadBeforeADeclaration(t *testing.T) { t.Fatalf("before a declaration every unit would read as the machine's: %+v", st) } } + +// Issue 764: a look starts no account's manager. Only an account the declaration places a user-scoped +// unit for is read at all, and its manager only while the machine's says it runs. + +func TestAnAccountWithNoUserUnitsIsNeverRead(t *testing.T) { + o := OwnedBy(declared(), nil) + if strings.Join(o.Accounts, " ") != "operator" { + t.Fatalf("only an account with a user-scoped unit is read: %v", o.Accounts) + } + r := shanks() + j := New(r) + j.Set(o) + lookTwice(t, j) + for _, m := range r.read { + if m == "user:telegram" { + t.Fatalf("a module account the declaration only creates had its manager read: %v", r.read) + } + } +} + +func TestAManagerNotRunningIsAwayAndNotRead(t *testing.T) { + r := shanks() + j := New(r) + j.Set(OwnedBy(declared(), nil)) + lookTwice(t, j) + if len(j.Last().Owned()) != 1 { + t.Fatalf("logged in, the Razer daemon is said: %+v", j.Last().Failed) + } + // The operator logs out: the manager is not asked, nothing in it is failed, it is said away. + r.away = map[string]bool{"operator": true} + r.read = nil + st, changed := j.Look(context.Background()) + if strings.Join(r.read, " ") != "system" { + t.Fatalf("a manager not running is not read: %v", r.read) + } + if !changed || len(st.Away) != 1 || !strings.Contains(st.Away[0], "operator") || len(st.Unread) != 0 { + t.Fatalf("away, said, and not unread: %+v", st) + } + for _, f := range st.Failed { + if f.Scope == ScopeUser { + t.Fatalf("a manager not running has no failed units: %+v", f) + } + } + if st.State != Degraded { + t.Fatalf("the machine's own failed mounts still say degraded: %q", st.State) + } + // Back in: read as before, and a failure is new again (two looks). + r.away = nil + st, _ = j.Look(context.Background()) + if len(st.Away) != 0 || len(st.Owned()) != 0 { + t.Fatalf("a manager running again is read, its failure on its first look not yet said: %+v", st) + } + st, _ = j.Look(context.Background()) + if len(st.Owned()) != 1 { + t.Fatalf("and said on the second: %+v", st.Failed) + } +} + +func TestAnAccountTheMachineLacksIsUnread(t *testing.T) { + r := shanks() + r.noAccount = map[string]bool{"operator": true} + j := New(r) + j.Set(OwnedBy(declared(), nil)) + st, _ := j.Look(context.Background()) + if strings.Join(r.read, " ") != "system" || len(st.Unread) != 1 { + t.Fatalf("an account the machine lacks is unread and its manager never asked: %v %+v", r.read, st) + } +} + +func TestWhetherAManagerRunsIsAskedOfTheMachines(t *testing.T) { + var asked []string + state := "inactive" + run := func(_ context.Context, name string, args ...string) (string, error) { + asked = append(asked, name+" "+strings.Join(args, " ")) + switch name { + case "getent": + return "telegram:x:10007:10007::/var/lib/telegram:/usr/bin/nologin\n", nil + case "systemctl": + if state != "active" { + return state + "\n", errors.New("systemctl exited 3") + } + return state + "\n", nil + } + return "", errors.New("unexpected") + } + e := Exec{Run: run, System: "arch"} + if running, exists, err := e.UserManagerRunning(context.Background(), "telegram"); running || !exists || err != nil { + t.Fatalf("an inactive manager is not running: %v %v %v", running, exists, err) + } + state = "active" + if running, _, err := e.UserManagerRunning(context.Background(), "telegram"); !running || err != nil { + t.Fatalf("an active manager runs: %v %v", running, err) + } + for _, a := range asked { + if a != "getent passwd telegram" && a != "systemctl is-active user@10007.service" { + t.Errorf("asked something but the user database and the machine's manager: %q", a) + } + } +} -- 2.54.0