From 499a687149beabb115c596e1d77eb1e30f6b2864 Mon Sep 17 00:00:00 2001 From: mesh-admin Date: Wed, 14 Oct 2026 00:37:29 +0000 Subject: [PATCH] Serve the local write of a filled file on its own socket A second unix socket beside mesh-cli's: the kernel's word on the asking account (SO_PEERCRED, same namespaces), that account judged against the declaration before its request is read, the path judged clean, absolute and filled, and the write done under the apply's lock. The journal has the path and digest, never the content. Issue 668. Co-Authored-By: Claude Opus 5.5 --- internal/filling/filling.go | 347 +++++++++++++++++++++++++++++++ internal/filling/filling_test.go | 263 +++++++++++++++++++++++ 2 files changed, 610 insertions(+) create mode 100644 internal/filling/filling.go create mode 100644 internal/filling/filling_test.go diff --git a/internal/filling/filling.go b/internal/filling/filling.go new file mode 100644 index 0000000..8e13902 --- /dev/null +++ b/internal/filling/filling.go @@ -0,0 +1,347 @@ +// Package filling is the node-engine's narrow local write verb (novox/hq ADR 0349 §4, design 57 P2): a socket on +// this machine through which the provisioner's client has a file it filled written, and nothing else. +// +// **The client fills; the node-engine writes.** The client renders a module's template with the values each domain +// owns and hands the node-engine the content; the node-engine writes it as the declared file, with the declared +// file's path, owner, mode and ownership at runtime, and makes what reads it take the change. So the one process that +// writes the machine's files is still the node-engine, and the client needs no right to write any of them. +// +// **Narrow by three checks, each from something the client cannot say for itself.** The account asking, from the +// kernel (SO_PEERCRED), in this machine's own namespaces, must be the account the node's current declaration names +// as the provisioner's client's; the path, clean and absolute, must be one the declaration names as filled; and the +// write takes the apply's lock, so a write and an apply never write one file at once. While the declaration names no +// account — every declaration before the provisioner's client has one — every write is refused. +// +// **No listening control surface** (novox/hq ADR 0004): the socket has no network address. And the content is never +// said: the journal has the path, the account and the content's digest. +package filling + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "os" + "os/user" + "path/filepath" + "strconv" + "sync" + "time" + + "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/meshcli" +) + +// Socket is where the node-engine accepts a write of a filled file, beside mesh-cli's socket. +const Socket = "/run/nox-mesh-host/fill.sock" + +// MaxRequest bounds one request, its content encoded in it: a configuration file, never an archive. +const MaxRequest = 8 << 20 + +// Request is one write, sent as one JSON object on one line: the path of a filled file and what the client filled +// into it (base64 in the JSON). The content only: the declaration says everything else about the file. +type Request struct { + Path string `json:"path"` + Content []byte `json:"content"` +} + +// Answer is what came of one request. Refused is a write that was not attempted, with why; Failed one that was +// attempted and did not complete, with why — the file, or something that reads it. Otherwise Action is what the +// write did to the file (created, updated, corrected, unchanged, kept) and Digest what the file holds now, as the +// node-engine records it. +type Answer struct { + Path string `json:"path,omitempty"` + Action string `json:"action,omitempty"` + Digest string `json:"digest,omitempty"` + Refused string `json:"refused,omitempty"` + Failed string `json:"failed,omitempty"` +} + +// Refusal is an answer that nothing was written, and why. +func Refusal(path, why string) Answer { return Answer{Path: path, Refused: why} } + +// Writer is the node-engine's side of the socket. +type Writer interface { + // Accepts judges the asking account before its request is read: nil when the current declaration names it as + // the provisioner's client's. + Accepts(uid uint32) error + // Write judges the request and writes it, holding the apply's lock. + Write(ctx context.Context, uid uint32, req Request) Answer +} + +// Engine is the Writer the node-engine serves: the declaration it applies now, read again under the apply's lock. +type Engine struct { + // Declared reads the declaration this node applies now, verified. + Declared func() (*declaration.Declaration, error) + // UIDOf is the user id of an account, by name, from this machine's user database. + UIDOf func(account string) (uint32, error) + // Lock takes the apply's lock and returns what releases it: the same the apply takes, so a write and an apply + // are one at a time. + Lock func() (func(), error) + // Place writes the content into the filled file at path, as the declared file, and makes what reads it take the + // change. Called holding the lock, with the declaration read under it. + Place func(ctx context.Context, d *declaration.Declaration, path string, content []byte) (action, digest string, + err error) +} + +// Accepts reads the declaration and judges the account. +func (e *Engine) Accepts(uid uint32) error { + d, err := e.Declared() + if err != nil { + return fmt.Errorf("this node's declaration cannot be read, so no write is accepted: %w", err) + } + return JudgeAccount(d, uid, e.UIDOf) +} + +// Write takes the apply's lock, reads the declaration under it, judges the account and the path again — the +// declaration may have changed since the request was accepted — and writes. +func (e *Engine) Write(ctx context.Context, uid uint32, req Request) Answer { + if err := JudgeShape(req.Path); err != nil { + return Refusal(req.Path, err.Error()) + } + unlock, err := e.Lock() + if err != nil { + return Refusal(req.Path, "the apply's lock could not be taken, so nothing was written: "+err.Error()) + } + defer unlock() + d, err := e.Declared() + if err != nil { + return Refusal(req.Path, "this node's declaration cannot be read, so nothing was written: "+err.Error()) + } + if err := JudgeAccount(d, uid, e.UIDOf); err != nil { + return Refusal(req.Path, err.Error()) + } + if err := JudgePath(d, req.Path); err != nil { + return Refusal(req.Path, err.Error()) + } + action, digest, err := e.Place(ctx, d, req.Path, req.Content) + if err != nil { + return Answer{Path: req.Path, Action: action, Digest: digest, Failed: err.Error()} + } + return Answer{Path: req.Path, Action: action, Digest: digest} +} + +// JudgeAccount refuses every account but the one the declaration names as the provisioner's client's, by its user +// id here — and every account while the declaration names none. +func JudgeAccount(d *declaration.Declaration, uid uint32, uidOf func(string) (uint32, error)) error { + if d.Provisioner == "" { + return errors.New("this node's declaration names no provisioner client's account, so no write is accepted") + } + want, err := uidOf(d.Provisioner) + if err != nil { + return fmt.Errorf("the provisioner client's account %s is not on this machine, so no write is accepted: %w", + d.Provisioner, err) + } + if want == 0 { + return fmt.Errorf("the provisioner client's account %s is root here, so no write is accepted", d.Provisioner) + } + if uid != want { + return fmt.Errorf("user id %d is not the provisioner client's account %s (user id %d), so nothing was written", + uid, d.Provisioner, want) + } + return nil +} + +// JudgeShape refuses a path that is not clean and absolute: a relative path, `..`, a doubled or trailing slash. +// Compared as written with the declaration's, a path that is not clean could name a file the declaration does not. +func JudgeShape(path string) error { + switch { + case path == "": + return errors.New("the request names no path, so nothing was written") + case !filepath.IsAbs(path): + return fmt.Errorf("%q is not an absolute path, so nothing was written", path) + case filepath.Clean(path) != path: + return fmt.Errorf("%q is not a clean path (it is %q cleaned), so nothing was written", path, + filepath.Clean(path)) + } + return nil +} + +// JudgePath refuses a path that is not clean and absolute, or that the declaration does not name as filled. Where it +// leads — a link on the way, a place nothing is written — is judged by the write, by the apply's own rules. +func JudgePath(d *declaration.Declaration, path string) error { + if err := JudgeShape(path); err != nil { + return err + } + if !d.Fills(path) { + return fmt.Errorf("%s is not a file this node's declaration names as filled, so nothing was written", path) + } + return nil +} + +// UIDOf is the user id of an account by name, from this machine's user database. +func UIDOf(account string) (uint32, error) { + u, err := user.Lookup(account) + if err != nil { + return 0, err + } + id, err := strconv.ParseUint(u.Uid, 10, 32) + if err != nil { + return 0, fmt.Errorf("user id %q: %w", u.Uid, err) + } + return uint32(id), nil +} + +// peerOf and sameNamespaces are the kernel's word on the asking process, as mesh-cli's socket reads it. Variables so +// a test can say. +var ( + peerOf = meshcli.PeerCredentials + sameNamespaces = meshcli.SameNamespaces +) + +// within bounds how long a connection may take to say its request, and refusedWithin how long a refused one is +// read and dropped before its refusal is written (so the client is not answered with a broken pipe). Variables so a +// test need not wait. +var ( + within = 30 * time.Second + refusedWithin = 1 * time.Second +) + +// atOnce bounds the connections served at the same time: one client, writing one file at a time. One over is +// refused busy at once. +var atOnce = 4 + +// Serve listens on socket until ctx ends, and answers each connection: one request, one answer. +// +// The directory is made 0755 and the socket 0666, as mesh-cli's: who may write is judged per connection from the +// kernel's word, against the current declaration, so the socket needs no account that may not exist yet. A file at +// the socket's path that is not a socket is never removed: the engine says so and serves nothing. +func Serve(ctx context.Context, socket string, w Writer, say func(string)) error { + if say == nil { + say = func(string) {} + } + dir := filepath.Dir(socket) + if err := os.MkdirAll(dir, 0o755); err != nil { + return fmt.Errorf("the filled files' socket directory: %w", err) + } + if problem := meshcli.NotOwnDirectory(dir); problem != "" { + return fmt.Errorf("filled files are not written on request on this machine: %s", problem) + } + if info, err := os.Lstat(socket); err == nil { + if info.Mode()&os.ModeSocket == 0 { + return fmt.Errorf("%s is there and is not a socket, and the node-engine did not put it there; filled "+ + "files are not written on request until it is moved", socket) + } + if err := os.Remove(socket); err != nil { + return fmt.Errorf("the socket a previous engine left at %s: %w", socket, err) + } + } + l, err := net.ListenUnix("unix", &net.UnixAddr{Name: socket, Net: "unix"}) + if err != nil { + return fmt.Errorf("listening for filled files at %s: %w", socket, err) + } + if err := os.Chmod(socket, 0o666); err != nil { + l.Close() + return fmt.Errorf("opening the filled files' socket: %w", err) + } + go func() { + <-ctx.Done() + l.Close() + }() + slots := make(chan struct{}, atOnce) + var wg sync.WaitGroup + defer wg.Wait() + for { + conn, err := l.AcceptUnix() + if err != nil { + if ctx.Err() != nil { + return nil + } + var ne net.Error + if errors.As(err, &ne) && ne.Timeout() { + continue + } + return fmt.Errorf("the filled files' socket: %w", err) + } + readBy := time.Now().Add(within) + _ = conn.SetDeadline(readBy) + wg.Add(1) + go func() { + defer wg.Done() + select { + case slots <- struct{}{}: + defer func() { <-slots }() + answer(ctx, conn, readBy, w, say) + default: + refuse(conn, readBy, Refusal("", "busy: other writes are being served; ask again"), say) + } + }() + } +} + +// answer serves one connection. +func answer(ctx context.Context, conn *net.UnixConn, readBy time.Time, w Writer, say func(string)) { + uid, pid, err := peerOf(conn) + if err != nil { + say("a filled file's write was refused: the asking account could not be read: " + err.Error()) + refuse(conn, readBy, Refusal("", "the node-engine could not read which account is asking, so nothing was "+ + "written: "+err.Error()), say) + return + } + if err := sameNamespaces(pid); err != nil { + say(fmt.Sprintf("a filled file's write from user id %d was refused: not in this machine's namespaces: %v", + uid, err)) + refuse(conn, readBy, Refusal("", "the asking process does not run in this machine's own namespaces, so "+ + "nothing was written: "+err.Error()), say) + return + } + if err := w.Accepts(uid); err != nil { + say(fmt.Sprintf("a filled file's write from user id %d was refused: %v", uid, err)) + refuse(conn, readBy, Refusal("", err.Error()), say) + return + } + var req Request + if err := json.NewDecoder(io.LimitReader(conn, MaxRequest)).Decode(&req); err != nil { + reply(conn, Refusal("", "the request could not be read, so nothing was written: "+err.Error()), say) + return + } + // The write waits for the apply's lock as long as an apply takes, and the client waits with it. + _ = conn.SetDeadline(time.Time{}) + a := w.Write(ctx, uid, req) + switch { + case a.Refused != "": + say(fmt.Sprintf("a write of %s from user id %d was refused: %s", req.Path, uid, a.Refused)) + case a.Failed != "": + say(fmt.Sprintf("a write of %s from user id %d failed: %s", req.Path, uid, a.Failed)) + default: + say(fmt.Sprintf("wrote %s for the provisioner's client (user id %d): %s, %s", req.Path, uid, a.Action, + a.Digest)) + } + _ = conn.SetWriteDeadline(time.Now().Add(within)) + reply(conn, a, say) +} + +// refuse reads and drops what a refused client wrote, briefly, and answers it: nothing it sent is kept. +func refuse(conn *net.UnixConn, readBy time.Time, a Answer, say func(string)) { + by := time.Now().Add(refusedWithin) + if readBy.Before(by) { + by = readBy + } + _ = conn.SetReadDeadline(by) + drain(conn) + _ = conn.SetWriteDeadline(time.Now().Add(within)) + reply(conn, a, say) +} + +// drain reads and drops a request up to its end — the newline after it, the end of the stream or the deadline — +// and at most MaxRequest bytes, keeping none of it. +func drain(r io.Reader) { + var buf [4096]byte + for read := 0; read < MaxRequest; { + n, err := r.Read(buf[:]) + read += n + if bytes.IndexByte(buf[:n], '\n') >= 0 || err != nil { + return + } + } +} + +func reply(conn *net.UnixConn, a Answer, say func(string)) { + defer conn.Close() + if err := json.NewEncoder(conn).Encode(a); err != nil { + say("a filled file's answer could not be written back: " + err.Error()) + } +} diff --git a/internal/filling/filling_test.go b/internal/filling/filling_test.go new file mode 100644 index 0000000..52a6f95 --- /dev/null +++ b/internal/filling/filling_test.go @@ -0,0 +1,263 @@ +package filling + +import ( + "context" + "encoding/json" + "errors" + "net" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/novox/mesh-host/internal/declaration" +) + +// Defends novox/hq ADR 0349 §4 (issue 668): the node-engine writes a filled file on request only for the provisioner +// client's account, only for a clean path the current declaration names as filled, and one at a time with the apply. + +const provisionerUID = 4242 + +func declared(t *testing.T, raw string) *declaration.Declaration { + t.Helper() + d, err := declaration.Parse([]byte(raw)) + if err != nil { + t.Fatal(err) + } + return d +} + +const withAccount = `{"declaration":1,"provisioner_account":"mesh-node-provisioner","filled_files":["/etc/m/m.conf"], + "resources":[{"id":"m.conf","type":"file","path":"/etc/m/m.conf","content":"x","owner-at-runtime":"owned"}, + {"id":"m.other","type":"file","path":"/etc/m/other.conf","content":"y"}]}` + +const withoutAccount = `{"declaration":1,"filled_files":["/etc/m/m.conf"], + "resources":[{"id":"m.conf","type":"file","path":"/etc/m/m.conf","content":"x","owner-at-runtime":"owned"}]}` + +// engine is an Engine over a fixed declaration, recording what it was asked to place. +type engine struct { + Engine + mu sync.Mutex + placed []string +} + +func newEngine(t *testing.T, raw string, lock sync.Locker) *engine { + d := declared(t, raw) + e := &engine{} + e.Engine = Engine{ + Declared: func() (*declaration.Declaration, error) { return d, nil }, + UIDOf: func(name string) (uint32, error) { + if name == "mesh-node-provisioner" { + return provisionerUID, nil + } + return 0, errors.New("no such account") + }, + Lock: func() (func(), error) { lock.Lock(); return lock.Unlock, nil }, + Place: func(_ context.Context, _ *declaration.Declaration, path string, content []byte) (string, string, error) { + e.mu.Lock() + defer e.mu.Unlock() + e.placed = append(e.placed, path) + return "updated", "digest", nil + }, + } + return e +} + +func TestAWriteFromTheProvisionersAccountOfAFilledPathIsPlaced(t *testing.T) { + e := newEngine(t, withAccount, &sync.Mutex{}) + if err := e.Accepts(provisionerUID); err != nil { + t.Fatal(err) + } + a := e.Write(context.Background(), provisionerUID, Request{Path: "/etc/m/m.conf", Content: []byte("x")}) + if a.Refused != "" || a.Failed != "" || a.Action != "updated" || len(e.placed) != 1 { + t.Fatalf("answer %+v, placed %v", a, e.placed) + } +} + +func TestAnUndeclaredPathIsRefused(t *testing.T) { + e := newEngine(t, withAccount, &sync.Mutex{}) + for _, path := range []string{"/etc/m/other.conf", "/etc/m/none.conf", "/etc/passwd"} { + a := e.Write(context.Background(), provisionerUID, Request{Path: path, Content: []byte("evil")}) + if a.Refused == "" || !strings.Contains(a.Refused, "not a file this node's declaration names as filled") { + t.Errorf("%s: %+v", path, a) + } + } + if len(e.placed) != 0 { + t.Errorf("placed %v", e.placed) + } +} + +func TestAnotherAccountIsRefused(t *testing.T) { + e := newEngine(t, withAccount, &sync.Mutex{}) + for _, uid := range []uint32{0, 1000, provisionerUID + 1} { + if err := e.Accepts(uid); err == nil { + t.Errorf("user id %d was accepted", uid) + } + a := e.Write(context.Background(), uid, Request{Path: "/etc/m/m.conf", Content: []byte("evil")}) + if a.Refused == "" || !strings.Contains(a.Refused, "is not the provisioner client's account") { + t.Errorf("user id %d: %+v", uid, a) + } + } + if len(e.placed) != 0 { + t.Errorf("placed %v", e.placed) + } +} + +func TestEveryWriteIsRefusedWhileTheDeclarationNamesNoAccount(t *testing.T) { + e := newEngine(t, withoutAccount, &sync.Mutex{}) + if err := e.Accepts(provisionerUID); err == nil || !strings.Contains(err.Error(), "names no provisioner") { + t.Errorf("accepted with no account named: %v", err) + } + a := e.Write(context.Background(), provisionerUID, Request{Path: "/etc/m/m.conf", Content: []byte("x")}) + if a.Refused == "" || len(e.placed) != 0 { + t.Errorf("written with no account named: %+v", a) + } +} + +func TestAnAccountThatIsRootHereIsRefused(t *testing.T) { + e := newEngine(t, withAccount, &sync.Mutex{}) + e.UIDOf = func(string) (uint32, error) { return 0, nil } + if err := e.Accepts(0); err == nil || !strings.Contains(err.Error(), "root") { + t.Errorf("an account that is root here was accepted: %v", err) + } +} + +func TestAnUncleanPathIsRefused(t *testing.T) { + e := newEngine(t, withAccount, &sync.Mutex{}) + for _, path := range []string{"", "etc/m/m.conf", "m.conf", "/etc//m/m.conf", "/etc/m/../m/m.conf", + "/etc/m/./m.conf", "/etc/m/m.conf/"} { + a := e.Write(context.Background(), provisionerUID, Request{Path: path, Content: []byte("x")}) + if a.Refused == "" { + t.Errorf("%q was not refused: %+v", path, a) + } + } + if len(e.placed) != 0 { + t.Errorf("placed %v", e.placed) + } +} + +// A write waits for an apply holding the apply's lock, and writes only once it is let go. +func TestAWriteWaitsForTheApply(t *testing.T) { + var applying sync.Mutex + e := newEngine(t, withAccount, &applying) + applying.Lock() + done := make(chan Answer) + go func() { + done <- e.Write(context.Background(), provisionerUID, Request{Path: "/etc/m/m.conf", Content: []byte("x")}) + }() + select { + case a := <-done: + t.Fatalf("written while the apply held the lock: %+v", a) + case <-time.After(100 * time.Millisecond): + } + e.mu.Lock() + n := len(e.placed) + e.mu.Unlock() + if n != 0 { + t.Fatalf("placed while the apply held the lock") + } + applying.Unlock() + if a := <-done; a.Action != "updated" { + t.Fatalf("after the apply: %+v", a) + } +} + +// fixed is a Writer that accepts one user id and answers every write. +type fixed struct { + uid uint32 + mu sync.Mutex + writes []Request +} + +func (f *fixed) Accepts(uid uint32) error { + if uid != f.uid { + return errors.New("not the provisioner client's account") + } + return nil +} + +func (f *fixed) Write(_ context.Context, _ uint32, req Request) Answer { + f.mu.Lock() + defer f.mu.Unlock() + f.writes = append(f.writes, req) + return Answer{Path: req.Path, Action: "updated", Digest: "d"} +} + +// serve serves w on a socket in a fresh directory and returns its path, and what was said. +func serve(t *testing.T, w Writer) (string, func() []string) { + t.Helper() + dir, err := os.MkdirTemp("", "fill") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { os.RemoveAll(dir) }) + socket := filepath.Join(dir, "fill.sock") + ctx, cancel := context.WithCancel(context.Background()) + var mu sync.Mutex + var said []string + stopped := make(chan error) + go func() { + stopped <- Serve(ctx, socket, w, func(s string) { mu.Lock(); said = append(said, s); mu.Unlock() }) + }() + t.Cleanup(func() { + cancel() + if err := <-stopped; err != nil { + t.Error(err) + } + }) + for i := 0; i < 100; i++ { + if _, err := os.Stat(socket); err == nil { + break + } + time.Sleep(10 * time.Millisecond) + } + return socket, func() []string { mu.Lock(); defer mu.Unlock(); return append([]string(nil), said...) } +} + +func ask(t *testing.T, socket string, req Request) Answer { + t.Helper() + conn, err := net.Dial("unix", socket) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + if err := json.NewEncoder(conn).Encode(req); err != nil { + t.Fatal(err) + } + var a Answer + if err := json.NewDecoder(conn).Decode(&a); err != nil { + t.Fatal(err) + } + return a +} + +// Over a real socket, the kernel says who is asking: this test's own account is the provisioner's here. +func TestTheSocketWritesForTheAccountTheKernelSays(t *testing.T) { + w := &fixed{uid: uint32(os.Getuid())} + socket, said := serve(t, w) + a := ask(t, socket, Request{Path: "/etc/m/m.conf", Content: []byte("secret value")}) + if a.Refused != "" || a.Action != "updated" || len(w.writes) != 1 || string(w.writes[0].Content) != "secret value" { + t.Fatalf("answer %+v, writes %+v", a, w.writes) + } + for _, line := range said() { + if strings.Contains(line, "secret value") { + t.Errorf("the content was said: %q", line) + } + } +} + +func TestTheSocketRefusesAnotherAccountBeforeReadingItsRequest(t *testing.T) { + w := &fixed{uid: uint32(os.Getuid()) + 1} + socket, said := serve(t, w) + a := ask(t, socket, Request{Path: "/etc/m/m.conf", Content: []byte("secret value")}) + if a.Refused == "" || len(w.writes) != 0 { + t.Fatalf("another account was written for: %+v", a) + } + for _, line := range said() { + if strings.Contains(line, "secret value") { + t.Errorf("the content was said: %q", line) + } + } +}