// Package meshcli is the node-engine's side of `mesh-cli` (novox/hq ADR 0272): a socket on this machine that any // account may connect to, the account read from the kernel, and the line asked of the controller. // // **The node-engine vouches; it does not decide.** It adds what only it can know — which account on this machine // is asking, from the kernel's peer credentials, never from anything the program says — and sends the request out // over its own link, on its own node's subject. The controller judges whether the line runs as its terminal, as an // ordinary call, or not at all (ADR 0272 §4). Nothing received here is run or applied on the machine. // // **No listening control surface** (novox/hq ADR 0004, ADR 0272 §3): the socket has no network address, so // nothing on another machine reaches it, and nothing sent to it acts on this one. package meshcli import ( "bytes" "context" "errors" "fmt" "io" "net" "os" "os/user" "path/filepath" "strconv" "strings" "sync" "time" "git.novox.be/novox/mesh-sdk/go/cli" ) // Asked is what the node-engine sends the controller on its own node's subject: the line, and the account the // kernel says is asking. The controller's side holds the same field names (a test on each side). type Asked struct { Line []string `json:"line,omitempty"` // Account is the login of the user id the kernel gave, from this machine's user database. Account string `json:"account"` // UID is the user id itself, so a renamed or duplicated login is still the same id. UID uint32 `json:"uid"` // Follow is a call a line runs as, asked about again by the account that asked it (AskTheController). Follow string `json:"follow,omitempty"` // Session is the login session the asking process runs in — its systemd scope, `session-.scope` under the // account's own slice — as the kernel's cgroup says it; empty for anything else: a system service, a user unit, // a process another account's session started. Only a login session is a person at a terminal (novox/hq ADR // 0272): the controller makes nothing else the terminal. Session string `json:"session,omitempty"` // Stdin is what mesh-cli's standard input held, at most cli.MaxStdin: carried to the controller with the line // and nowhere else. The engine says it in no journal, keeps it in no state, and never sends it again: a follow // carries none. The controller gives it to a line that runs as its terminal alone. Stdin []byte `json:"stdin,omitempty"` } // Forward asks the controller and follows the line to what came of it (AskTheController): what it printed, that // nothing ran and why, or that it is not known. type Forward func(ctx context.Context, asked Asked) cli.Answer // readWithin bounds how long a connection may take to say its line: a program that connects and says nothing // holds nothing for long. A variable so a test need not wait. var readWithin = 30 * time.Second // refusedWithin bounds how long the engine waits for the request of a line it refuses before it is asked: a client // that has connected writes its request within microseconds, so a refused connection holds nothing for long. Never // longer than readWithin. A variable so a test can say. var refusedWithin = 1 * time.Second // refusalsWaiting bounds how many refusals wait for their request at the same time, on the whole engine; drainCap // bounds how much of a refused request is read, and it is read into a small buffer and dropped. The socket is open // to every account, so a refusal must cost the engine little however many connections ask (review of ADR 0272). // Variables so a test can say. var ( refusalsWaiting = make(chan struct{}, 32) drainCap = 4 << 10 ) // drain reads and drops what a refused client wrote, up to the end of its request line, the end of the stream, the // connection's deadline or drainCap bytes, whichever comes first. Nothing is kept. func drain(r io.Reader) { var buf [512]byte for read := 0; read < drainCap; { n, err := r.Read(buf[:min(len(buf), drainCap-read)]) read += n if bytes.IndexByte(buf[:n], '\n') >= 0 || err != nil { return } } } // atOnce bounds how many lines are asked at the same time, and perAccount how many of them one account's: a person // types one at a time, so more is a program on the machine. One over either bound is answered busy at once — the // engine never stops accepting for it, so one account cannot hold the socket from every other (review of ADR 0272). var ( atOnce = 8 perAccount = 2 ) // slots counts the lines being asked, in all and per user id. type slots struct { mu sync.Mutex all int byUID map[uint32]int } func (s *slots) take(uid uint32) bool { s.mu.Lock() defer s.mu.Unlock() if s.all >= atOnce || s.byUID[uid] >= perAccount { return false } s.all++ s.byUID[uid]++ return true } func (s *slots) give(uid uint32) { s.mu.Lock() defer s.mu.Unlock() s.all-- if s.byUID[uid]--; s.byUID[uid] <= 0 { delete(s.byUID, uid) } } // peerOf reads the connecting process's credentials from the kernel: its user id and process id. A variable so a // test can say what the kernel would; it is the only source of the account. var peerOf = peerCredentials // sessionOf reads the login session a process runs in, from the kernel. A variable so a test can say. var sessionOf = func(pid int32, uid uint32) string { raw, err := os.ReadFile(fmt.Sprintf("/proc/%d/cgroup", pid)) if err != nil { return "" } return userSessionIn(string(raw), uid) } // sessionsDir is where logind keeps a record of each session it holds. A variable so a test can say. var sessionsDir = "/run/systemd/sessions" // userSessionIn is the login session a cgroup names, when logind's own record of it says it is a person's: this // account's (UID), of class `user` — not a greeter's or the user manager's — and of this very scope. The cgroup // alone names a scope; only the record says whose session it is. No record, or one that does not say so, is no // session. func userSessionIn(cgroup string, uid uint32) string { scope := loginSessionIn(cgroup, uid) id, ok := strings.CutPrefix(strings.TrimSuffix(scope, ".scope"), "session-") if !ok || id == "" || strings.ContainsAny(id, "/.") { return "" } raw, err := os.ReadFile(filepath.Join(sessionsDir, id)) if err != nil { return "" } // **The record is systemd's private format**, not an interface it promises: `KEY=value` lines logind writes for // itself. Only UID, CLASS and SCOPE are read, each at its first occurrence — logind writes each once, and a later // line saying otherwise is not the record's word. A systemd that changes the format, renames a key or stops // writing the file fails closed here: no session is read, so nobody is the terminal until this reads it again. record := map[string]string{} for _, line := range strings.Split(string(raw), "\n") { k, v, ok := strings.Cut(line, "=") if !ok { continue } if _, said := record[k]; !said { record[k] = v } } if record["UID"] != strconv.FormatUint(uint64(uid), 10) || record["CLASS"] != "user" || record["SCOPE"] != scope { return "" } return scope } // loginSessionIn is the login session scope a process's cgroup names, for its own account: the unified hierarchy's // line, `0::/user.slice/user-.slice/session-.scope`, and nothing below or beside it. Anything else is no // login session — a service under system.slice, a unit under the account's user manager (user@.service), a // scope of another account's. func loginSessionIn(cgroup string, uid uint32) string { for _, line := range strings.Split(cgroup, "\n") { path, ok := strings.CutPrefix(line, "0::") if !ok { continue } prefix := fmt.Sprintf("/user.slice/user-%d.slice/", uid) scope, ok := strings.CutPrefix(path, prefix) if !ok || strings.Contains(scope, "/") || !strings.HasPrefix(scope, "session-") || !strings.HasSuffix(scope, ".scope") { return "" } return scope } return "" } // sameNamespaces refuses a peer whose mount or user namespace is not the engine's. A variable so a test can say. var sameNamespaces = namespacesOf // lookupAccount names a user id from this machine's user database. var lookupAccount = func(uid uint32) (string, error) { u, err := user.LookupId(strconv.FormatUint(uint64(uid), 10)) if err != nil { return "", err } return u.Username, nil } // Serve listens on socket until ctx ends, and answers each connection: one request, one answer. // // The directory is made 0755 and the socket 0666: every account on the machine may ask, because the account is // what is vouched for and the controller decides what it may do. A file at the socket's path that is not a socket // is never removed: the engine says so and serves nothing. func Serve(ctx context.Context, socket string, forward Forward, say func(string)) error { dir := filepath.Dir(socket) if err := os.MkdirAll(dir, 0o755); err != nil { return fmt.Errorf("mesh-cli's socket directory: %w", err) } // The directory is the engine's: a link, or one another account owns, could put the socket where that // account replaces it with its own. Said, and nothing served. if problem := notOwnDirectory(dir); problem != "" { return fmt.Errorf("mesh-cli is not served on this machine: %s", problem) } if err := os.Chmod(dir, 0o755); err != nil { return fmt.Errorf("mesh-cli's socket directory: %w", err) } if info, err := os.Lstat(socket); err == nil { if info.Mode()&os.ModeSocket == 0 { return fmt.Errorf("%s is there and is not a socket, and the node-engine did not put it there; "+ "mesh-cli is not served on this machine until it is moved", socket) } // A socket left by the engine before this one: nothing listens on it any more. if err := os.Remove(socket); err != nil { return fmt.Errorf("the socket a previous engine left at %s: %w", socket, err) } } l, err := net.ListenUnix("unix", &net.UnixAddr{Name: socket, Net: "unix"}) if err != nil { return fmt.Errorf("listening for mesh-cli at %s: %w", socket, err) } if err := os.Chmod(socket, 0o666); err != nil { l.Close() return fmt.Errorf("opening mesh-cli's socket to every account: %w", err) } go func() { <-ctx.Done() l.Close() }() held := &slots{byUID: map[uint32]int{}} var wg sync.WaitGroup defer wg.Wait() for { conn, err := l.AcceptUnix() if err != nil { if ctx.Err() != nil { return nil } var ne net.Error if errors.As(err, &ne) && ne.Timeout() { continue } return fmt.Errorf("mesh-cli's socket: %w", err) } // Bounded from the first byte: nothing below may wait on a connection for longer than this before the // line is read, whatever the peer does (review of ADR 0272). readBy := time.Now().Add(readWithin) _ = conn.SetDeadline(readBy) wg.Add(1) go func() { defer wg.Done() answer(ctx, conn, readBy, held, forward, say) }() } } // answer serves one connection; readBy is the deadline set when it was accepted, for reading its line. func answer(ctx context.Context, conn *net.UnixConn, readBy time.Time, held *slots, forward Forward, say func(string)) { defer conn.Close() reply := func(a cli.Answer) { _ = conn.SetWriteDeadline(time.Now().Add(readWithin)) if err := cli.WriteAnswer(conn, a); err != nil && say != nil { say(fmt.Sprintf("mesh-cli: the answer could not be written back: %v", err)) } } // A refusal is written only after the request it answers is read (novox/hq issue 434). The client writes its // request first and reads the answer after; an engine that answers and closes before that write has landed // makes the write fail with a broken pipe, and the client never reads the refusal waiting for it — under load a // line beyond its account's bound read as an error instead of busy. The wait ends at refusedWithin, or earlier // at the deadline set when the connection was accepted; what was written is drained and never asked. Only // refusalsWaiting refusals wait at once: over that the refusal is written at once, as before, and only then can a // client still find its write broken. refuse := func(why string) { select { case refusalsWaiting <- struct{}{}: by := time.Now().Add(refusedWithin) if readBy.Before(by) { by = readBy } _ = conn.SetReadDeadline(by) drain(conn) <-refusalsWaiting default: } reply(cli.Refusal(why)) } uid, pid, err := peerOf(conn) if err != nil { // Never sent as anyone: an account the kernel did not give is no account at all. refuse("the node-engine could not read which account is asking, so nothing was asked: " + err.Error()) return } // The user id is this machine's account only in this machine's namespaces: a container running with the // operator's id shares the number and nothing else (review of ADR 0272). if err := sameNamespaces(pid); err != nil { refuse("the asking process does not run in this machine's own namespaces, so it is not this " + "machine's account and nothing was asked: " + err.Error()) return } // And the login session, now: judged as the peer was when it connected, before it can exec or exit while its // line is read (review of ADR 0272). session := sessionOf(pid, uid) // Looked up before a slot is taken, so a refused account never holds one while it is refused. account, err := lookupAccount(uid) if err != nil || account == "" { refuse(fmt.Sprintf("user id %d is not in this machine's user database, so nothing was asked", uid)) return } if !held.take(uid) { refuse(fmt.Sprintf("busy: user id %d has %d lines being asked already, or the machine %d; ask "+ "again when one has ended. Nothing was asked", uid, perAccount, atOnce)) return } defer held.give(uid) req, err := cli.ReadRequest(conn) if err != nil { reply(cli.Refusal("the request could not be read, so nothing was asked: " + err.Error())) return } // From here the wait is the controller's: as long as mesh-cli waits, and no longer. _ = conn.SetDeadline(time.Now().Add(cli.AskWithin)) asking, cancel := context.WithTimeout(ctx, cli.AskWithin-30*time.Second) defer cancel() reply(forward(asking, Asked{Line: req.Line, Account: account, UID: uid, Session: session, Stdin: req.Stdin})) }