ssh-client keeps ~/.ssh to what it knows: sweep every other path into a kept folder (issue 1017, ADR 0415)

ssh-client owns ~/.ssh, but treated every file it did not place as found, so a
predecessor's drop-in and old copies stayed there for weeks. Its process now
sweeps at start and every ten minutes: keys at any depth (and links to them),
authorized_keys, authorized_keys2 and sockets stay, and a folder holding one is
gone into; everything else moves to ~/.local/state/ssh-client/retired/<UTC
time>/, a folder never shared with another sweep, with a list beside it, which
ssh_client_check shows. Nothing is deleted.

modules/ssh-client/cmd/ssh-client-tools/sweep.go

@@ -0,0 +1,269 @@
+package main
+
+// The sweep (novox/hq ADR 0415, issue 1017): ssh-client owns ~/.ssh, so it keeps it to what it knows.
+// Every path it does not know is moved, whole, into ~/.local/state/ssh-client/retired/<UTC time>/,
+// keeping its path relative to ~/.ssh. Nothing is deleted. A key (private, public or a certificate),
+// at any depth and whatever its name, and authorized_keys are never moved (design 29 §3: moving one
+// can lock the operator out); a folder holding a key is gone into rather than moved whole. A socket
+// is ssh's own runtime and stays. The process sweeps when it starts and every KeyRound (main.go).
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"regexp"
+	"sort"
+	"strings"
+	"time"
+)
+
+// Declared are the paths under ~/.ssh this module declares (module.json) or its own process makes
+// (keykeeper.go). A test checks the manifest against this list.
+var Declared = map[string]bool{
+	"config":            true,
+	"known_hosts":       true,
+	"config.d":          true,
+	"config.d/00-mesh":  true,
+	"config.d/10-forge": true,
+	KeyFile:             true,
+	KeyFile + ".pub":    true,
+	"authorized_keys":   true,
+	"authorized_keys2":  true, // sshd's second AuthorizedKeysFile by default: moving it can lock the operator out
+}
+
+// copyOf is a copy of authorized_keys or known_hosts (authorized_keys.revoked-*, known_hosts.old): it
+// carries key lines but is no key, and sshd and ssh never read it.
+var copyOf = regexp.MustCompile(`^(authorized_keys|known_hosts)[^/]+$`)
+
+// RetiredDir is where the sweep moves what it does not know, under the account's home.
+const RetiredDir = ".local/state/ssh-client/retired"
+
+// Moved is one path a sweep moved out of ~/.ssh.
+type Moved struct {
+	From string `json:"from"`
+	To   string `json:"to"`
+}
+
+// Sweep is what one sweep did: what it moved, and what it left because it could not tell what it is.
+type Sweep struct {
+	At    string   `json:"at"`
+	Moved []Moved  `json:"moved"`
+	Left  []string `json:"left,omitempty"`
+}
+
+// Sweep moves every path under ~/.ssh this module does not know into a retired folder named by now,
+// and writes beside that folder the list of what it moved (<time>.json). It moves nothing when
+// everything is known, and makes nothing then.
+func (c *Client) Sweep(now time.Time) (Sweep, error) {
+	dir := c.ssh("")
+	s := Sweep{At: now.UTC().Format("20060102T150405Z"), Moved: []Moved{}}
+	if _, err := os.Lstat(dir); err != nil {
+		if os.IsNotExist(err) {
+			return s, nil
+		}
+		return s, err
+	}
+	var unknown []string
+	err := filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error {
+		if err != nil {
+			if p == dir {
+				return err
+			}
+			s.Left = append(s.Left, p) // a folder that cannot be read may hold a key
+			return fs.SkipDir
+		}
+		rel, _ := filepath.Rel(dir, p)
+		switch {
+		case rel == "." || Declared[rel] || d.Type()&fs.ModeSocket != 0:
+			return nil
+		case d.IsDir():
+			if holdsKey(p) {
+				return nil // gone into: the key stays where it is, the rest around it goes
+			}
+			unknown = append(unknown, rel)
+			return fs.SkipDir // moved whole
+		case d.Type()&fs.ModeSymlink != 0:
+			// A link to a key is a key's name: it stays. Any other link is moved as the link, never
+			// what it points at.
+			if st, err := os.Stat(p); err == nil && st.Mode().IsRegular() && keyKind(p) == "key" {
+				return nil
+			}
+			unknown = append(unknown, rel)
+			return nil
+		case !d.Type().IsRegular():
+			unknown = append(unknown, rel)
+			return nil
+		case copyOf.MatchString(rel):
+			unknown = append(unknown, rel)
+			return nil
+		}
+		switch k := keyKind(p); k {
+		case "key":
+			return nil
+		case "unreadable":
+			s.Left = append(s.Left, p)
+			return nil
+		}
+		unknown = append(unknown, rel)
+		return nil
+	})
+	if err != nil {
+		return s, err
+	}
+	if len(unknown) == 0 {
+		return s, nil
+	}
+	root, err := c.claim(s.At)
+	if err != nil {
+		return s, err
+	}
+	s.At = filepath.Base(root)
+	defer func() {
+		if len(s.Moved) > 0 {
+			raw, _ := json.MarshalIndent(s.Moved, "", "  ")
+			os.WriteFile(root+".json", raw, 0o600)
+		}
+	}()
+	for _, rel := range unknown {
+		to := filepath.Join(root, rel)
+		if err := os.MkdirAll(filepath.Dir(to), 0o700); err != nil {
+			return s, err
+		}
+		if _, err := os.Lstat(to); err == nil {
+			return s, fmt.Errorf("could not move %s out of ~/.ssh: %s is already there", rel, to) // never over a kept copy
+		}
+		if err := os.Rename(filepath.Join(dir, rel), to); err != nil {
+			return s, fmt.Errorf("could not move %s out of ~/.ssh: %v", rel, err)
+		}
+		s.Moved = append(s.Moved, Moved{From: filepath.Join(dir, rel), To: to})
+	}
+	return s, nil
+}
+
+// claim makes this sweep's own retired folder, named by its time; a folder of that name already there
+// (a sweep in the same second) is never shared: the name takes -2, -3, … instead.
+func (c *Client) claim(at string) (string, error) {
+	base := filepath.Join(c.Home, RetiredDir)
+	if err := os.MkdirAll(base, 0o700); err != nil {
+		return "", err
+	}
+	for n := 1; n <= 1000; n++ {
+		root := filepath.Join(base, at)
+		if n > 1 {
+			root = fmt.Sprintf("%s-%d", root, n)
+		}
+		err := os.Mkdir(root, 0o700)
+		if err == nil {
+			return root, nil
+		}
+		if !os.IsExist(err) {
+			return "", err
+		}
+	}
+	return "", fmt.Errorf("no free name for the sweep's folder under %s", base)
+}
+
+// keyKind says whether a regular file is a key ("key": a private key, a public key or a certificate),
+// cannot be read to tell ("unreadable"), or is neither ("").
+func keyKind(path string) string {
+	if st, err := os.Stat(path); err != nil {
+		return "unreadable"
+	} else if !st.Mode().IsRegular() {
+		return "" // a pipe or a device is no key, and opening one can block
+	}
+	f, err := os.Open(path)
+	if err != nil {
+		return "unreadable"
+	}
+	f.Close()
+	if header(path) || publicKey(path) {
+		return "key"
+	}
+	return ""
+}
+
+// holdsKey is whether a folder holds, at any depth, a key, a socket or a file that cannot be read to
+// tell.
+func holdsKey(dir string) bool {
+	found := false
+	filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error {
+		if err != nil {
+			found = true // a folder that cannot be read may hold one
+			return fs.SkipAll
+		}
+		rel, _ := filepath.Rel(dir, p)
+		if d.Type()&fs.ModeSocket != 0 {
+			found = true
+			return fs.SkipAll
+		}
+		if (d.Type().IsRegular() || d.Type()&fs.ModeSymlink != 0) && !copyOf.MatchString(d.Name()) && keyKind(p) != "" && rel != "." {
+			found = true
+			return fs.SkipAll
+		}
+		return nil
+	})
+	return found
+}
+
+// publicKey reads a small file's first line and says whether it is a public key or a certificate.
+func publicKey(path string) bool {
+	st, err := os.Stat(path)
+	if err != nil || st.Size() > 16<<10 {
+		return false
+	}
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		return false
+	}
+	line, _, _ := strings.Cut(string(raw), "\n")
+	if _, err := ParseKeyLine(line); err == nil {
+		return true
+	}
+	f := strings.Fields(line)
+	return len(f) > 1 && strings.HasSuffix(f[0], "-cert-v01@openssh.com")
+}
+
+// Retired lists the sweeps that moved something, newest first, from the lists each sweep wrote.
+func (c *Client) Retired() ([]Sweep, error) {
+	root := filepath.Join(c.Home, RetiredDir)
+	entries, err := os.ReadDir(root)
+	if os.IsNotExist(err) {
+		return []Sweep{}, nil
+	} else if err != nil {
+		return nil, err
+	}
+	out := []Sweep{}
+	for _, e := range entries {
+		at, ok := strings.CutSuffix(e.Name(), ".json")
+		if !ok || e.IsDir() {
+			continue
+		}
+		raw, err := os.ReadFile(filepath.Join(root, e.Name()))
+		if err != nil {
+			return nil, err
+		}
+		s := Sweep{At: at}
+		if err := json.Unmarshal(raw, &s.Moved); err != nil {
+			return nil, fmt.Errorf("%s: %v", filepath.Join(root, e.Name()), err)
+		}
+		out = append(out, s)
+	}
+	sort.Slice(out, func(a, b int) bool { return out[a].At > out[b].At })
+	return out, nil
+}
+
+// SweepOnce sweeps and says each move, each path left, and a failure through say.
+func (c *Client) SweepOnce(now time.Time, say func(string, ...any)) {
+	s, err := c.Sweep(now)
+	for _, m := range s.Moved {
+		say("moved %s, which this module does not know, to %s", m.From, m.To)
+	}
+	for _, l := range s.Left {
+		say("left %s: it cannot be read to tell whether it is a key", l)
+	}
+	if err != nil {
+		say("the sweep of ~/.ssh: %v", err)
+	}
+}