Merge pull request 'The seat node-files: write, copy, move and delete a file on a node through the mesh, with trash and owned paths (issue 900, ADR 0394)' (#360) from feat/900-node-files into main

internal/catalogue/files.go

@@ -0,0 +1,147 @@
+package catalogue
+
+import (
+	"encoding/json"
+	"fmt"
+	"path"
+	"strings"
+)
+
+// A node's files, read and managed (novox/hq ADR 0394, issue 900; superseding ADR 0310; hq to-be 41 WP9).
+//
+// **One seat for every file on a node.** The holder runs as the tool runner's account, the node's operator
+// account (ADR 0394 §2), and reaches further through `sudo -n` where that account's rights do not. It refuses by
+// name the places a secret is kept (reads only, ADR 0394 §7) and the paths a module owns, unless a write names a reason (`override`).
+// node-reads stays beside it until mesh-files is assigned on every node (ADR 0394 §9); it goes in the next
+// delivery.
+
+// FilesSeat is the role that reads and manages a node's files and accounts (novox/hq ADR 0394).
+const FilesSeat = "node-files"
+
+// FilesOwnedPaths is the word the holder of node-files is given beside the secret places: a JSON object from
+// each absolute path a module owns on its node to "module (tool to use)" (novox/hq ADR 0394 §5).
+const FilesOwnedPaths = "MESH_OWNED_PATHS"
+
+const (
+	filesAsAccount = "as the node's operator account, through sudo -n where its rights do not reach; a place a " +
+		"secret is kept, or a known credential file, is refused by name"
+	overrideWord = "a reason, to change a path a module owns anyway (optional)"
+)
+
+// filesVerbs is the contract every holder of node-files serves (novox/hq ADR 0394 §1): the reads of node-reads,
+// with their descriptions restated for the operator's account, and the writes.
+func filesVerbs() []Verb {
+	var verbs []Verb
+	for _, v := range readsVerbs() {
+		v.Description = strings.Replace(v.Description,
+			"Read as the holder's own account, which holds no secret: a component it may not reach says so.",
+			"", 1)
+		v.Description = strings.TrimSpace(v.Description) + " Read " + filesAsAccount + "."
+		verbs = append(verbs, v)
+	}
+	write := func(name, description string, in map[string]string, required []string, optional []string, replaces ...string) Verb {
+		in["reason"] = "why the file is changed, for the audit trail"
+		in["override"] = overrideWord
+		return Verb{Name: name, Description: description + " Refuses a path a module owns " +
+			"unless override names a reason; done as the node's operator account, through sudo -n where its rights do not reach.",
+			Input: schema(in, append(required, "reason"), append(optional, "override")...), Replaces: replaces}
+	}
+	return append(verbs,
+		write("write", "Write one text file whole, replacing what it holds; the old content goes to the node's trash.",
+			map[string]string{"path": "an absolute path", "content": "the text to write",
+				"mode": "the file's mode, such as 0644 (optional)"},
+			[]string{"path", "content"}, []string{"mode"}, "tee", ">"),
+		write("copy", "Copy a file or directory to a path on this node. A target on another node (<node>:/path) is "+
+			"refused: copying between nodes comes in step 2 (issue 903). An existing target is overwritten, the old one "+
+			"kept in the node's trash for 7 days.",
+			map[string]string{"from": "an absolute path", "to": "an absolute path on this node"},
+			[]string{"from", "to"}, nil, "cp", "cp -a", "ssh <node> cp"),
+		write("move", "Move or rename a file or directory. A target on another node is refused: copying between nodes "+
+			"comes in step 2 (issue 903). An existing target is overwritten, the old one kept in the node's trash for 7 days.",
+			map[string]string{"from": "an absolute path", "to": "an absolute path"},
+			[]string{"from", "to"}, nil, "mv"),
+		write("delete", "Delete a file or directory into the node's trash, where restore can bring it back.",
+			map[string]string{"path": "an absolute path", "recursive": "\"true\": delete a directory with what it holds"},
+			[]string{"path"}, []string{"recursive"}, "rm", "rm -r"),
+		write("mkdir", "Make a directory.",
+			map[string]string{"path": "an absolute path", "parents": "\"true\": make the missing parents too"},
+			[]string{"path"}, []string{"parents"}, "mkdir", "mkdir -p"),
+		write("chmod", "Change the mode of a file or directory.",
+			map[string]string{"path": "an absolute path", "mode": "the mode, such as 0644 or u+x",
+				"recursive": "\"true\": also everything under a directory"},
+			[]string{"path", "mode"}, []string{"recursive"}, "chmod"),
+		write("chown", "Change the owner and group of a file or directory.",
+			map[string]string{"path": "an absolute path", "owner": "an account's name or number",
+				"group": "a group's name or number (optional)", "recursive": "\"true\": also everything under a directory"},
+			[]string{"path", "owner"}, []string{"group", "recursive"}, "chown", "chgrp"),
+		Verb{Name: "trash", Description: "What the node's trash holds: each deleted or replaced item with its id, " +
+			"original path and when it went.",
+			Input: schema(map[string]string{}, nil), Replaces: []string{"ls ~/.local/share/Trash"}},
+		write("restore", "Put an item of the node's trash back where it was; refuses when something is there now.",
+			map[string]string{"id": "an id from trash"}, []string{"id"}, nil, "trash-restore"),
+	)
+}
+
+// FilesHolderProblems is what a holder of node-files is held to (novox/hq ADR 0394 §2): it runs as the tool
+// runner's account, the node's operator account, so it declares no runs-as.
+func FilesHolderProblems(m Manifest) []string {
+	if !m.ClaimsSeat(FilesSeat) || m.RunsAs == "" {
+		return nil
+	}
+	return []string{fmt.Sprintf("%s claims %s and declares runs-as %q: its holder runs as the tool runner's account, "+
+		"the node's operator account (novox/hq ADR 0394 §2)", m.Module, FilesSeat, m.RunsAs)}
+}
+
+// ownedPaths is every path a module owns on this node, as the holder is told (novox/hq ADR 0394 §5): the
+// paths every seat declares (ADR 0287) with the module holding the seat here, and the paths and files each
+// module's manifest places (declaredPaths). A seat's path wins where both name one.
+func (r Resolution) ownedPaths() map[string]string {
+	owned := map[string]string{}
+	for _, m := range r.Modules {
+		for _, p := range sortedKeys(declaredPaths(m)) {
+			owned[p] = m.Module + " (its own tools)"
+		}
+	}
+	for _, s := range Seats() {
+		for _, sp := range s.Paths {
+			for _, m := range r.Modules {
+				if m.ClaimsSeat(s.Name) && declaresPath(m, path.Clean(sp.Path)) {
+					owned[sp.Path] = m.Module + " (" + s.Name + ")"
+				}
+			}
+		}
+	}
+	return owned
+}
+
+// filesWords is what the holder of node-files is given beside its own words: the node's secret places and the
+// paths modules own here.
+func (r Resolution) filesWords(m Manifest, words map[string]string, with Rendering) (map[string]string, error) {
+	if !m.ClaimsSeat(FilesSeat) {
+		return words, nil
+	}
+	places, err := r.secretPlaces(with)
+	if err != nil {
+		return nil, err
+	}
+	places2, err := json.Marshal(places)
+	if err != nil {
+		return nil, err
+	}
+	owned, err := json.Marshal(r.ownedPaths())
+	if err != nil {
+		return nil, err
+	}
+	if words == nil {
+		words = map[string]string{}
+	}
+	for _, w := range []string{ReadsSecretPlaces, FilesOwnedPaths} {
+		if _, has := words[w]; has {
+			return nil, fmt.Errorf("%s gives its own %s; the node's secret places and owned paths are the mesh's to "+
+				"say (novox/hq ADR 0394 §5)", m.Module, w)
+		}
+	}
+	words[ReadsSecretPlaces] = string(places2)
+	words[FilesOwnedPaths] = string(owned)
+	return words, nil
+}

internal/catalogue/files_test.go

@@ -0,0 +1,111 @@
+package catalogue
+
+import (
+	"encoding/json"
+	"strings"
+	"testing"
+)
+
+// issue 900, novox/hq ADR 0394 §1: node-files is a node seat whose verbs say the commands they replace.
+func TestNodeFilesIsANodeSeatWithEveryVerbOfADR0394(t *testing.T) {
+	s, ok := SeatNamed(FilesSeat)
+	if !ok {
+		t.Fatalf("the mesh defines no seat %s", FilesSeat)
+	}
+	if s.Scope != ScopeNode || s.Delivers != "" || !strings.Contains(s.Decision, "ADR 0394") {
+		t.Errorf("%s: scope %q, delivers %q, decision %q", FilesSeat, s.Scope, s.Delivers, s.Decision)
+	}
+	have := map[string]bool{}
+	replaced := map[string]string{}
+	for _, v := range s.Serves {
+		have[v.Name] = true
+		if v.Optional {
+			t.Errorf("%s.%s is optional", FilesSeat, v.Name)
+		}
+		if len(v.Replaces) == 0 {
+			t.Errorf("%s.%s says nothing it replaces", FilesSeat, v.Name)
+		}
+		for _, c := range v.Replaces {
+			replaced[c] = v.Name
+		}
+	}
+	for _, name := range []string{"stat", "readlink", "list", "read", "search", "account", "write", "copy", "move",
+		"delete", "mkdir", "chmod", "chown", "trash", "restore"} {
+		if !have[name] {
+			t.Errorf("%s serves no verb %s", FilesSeat, name)
+		}
+	}
+	for command, verb := range map[string]string{"cat": "read", "tee": "write", "cp": "copy", "mv": "move",
+		"rm": "delete", "mkdir -p": "mkdir", "chmod": "chmod", "chgrp": "chown"} {
+		if replaced[command] != verb {
+			t.Errorf("%q is replaced by %q, want %s.%s", command, replaced[command], FilesSeat, verb)
+		}
+	}
+}
+
+// issue 900, ADR 0394 §2: a holder of node-files runs as the tool runner's account, so it declares no runs-as.
+func TestAHolderOfNodeFilesDeclaringRunsAsIsRefused(t *testing.T) {
+	p := FilesHolderProblems(Manifest{Module: "files", RunsAs: "mesh-files", Claims: []Claim{{Name: FilesSeat}}})
+	if len(p) != 1 || !strings.Contains(p[0], "0394") || !strings.Contains(p[0], "runs-as") {
+		t.Errorf("a holder declaring runs-as was not refused: %v", p)
+	}
+	if p := FilesHolderProblems(Manifest{Module: "files", Claims: []Claim{{Name: FilesSeat}}}); len(p) != 0 {
+		t.Errorf("a holder declaring none was refused: %v", p)
+	}
+	if p := FilesHolderProblems(Manifest{Module: "other", RunsAs: "x"}); len(p) != 0 {
+		t.Errorf("a non-holder was judged: %v", p)
+	}
+}
+
+// issue 900, ADR 0394 §4: the holder is told the secret places and the paths modules own on its node.
+func TestTheHolderOfNodeFilesIsToldSecretPlacesAndOwnedPaths(t *testing.T) {
+	files := Manifest{Module: "files", Version: "1", Claims: []Claim{{Name: FilesSeat}},
+		OwnSecrets: OwnSecrets{"broker": {Path: "/var/lib/files/broker"}}}
+	host := Manifest{Module: "hostnames", Version: "1", Claims: []Claim{{Name: "node-hostname"}},
+		Resources: []map[string]any{{"id": "hosts", "type": "file", "path": "/etc/hosts"}}}
+	app := Manifest{Module: "app", Version: "1",
+		Resources: []map[string]any{{"id": "conf", "type": "file", "path": "/etc/app/app.conf"}}}
+	r := Resolution{Node: "anchor", Modules: []Manifest{files, host, app}}
+	words, err := r.filesWords(files, nil, Rendering{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	var places []string
+	if err := json.Unmarshal([]byte(words[ReadsSecretPlaces]), &places); err != nil || len(places) == 0 {
+		t.Errorf("no secret places: %v %v", err, words)
+	}
+	var owned map[string]string
+	if err := json.Unmarshal([]byte(words[FilesOwnedPaths]), &owned); err != nil {
+		t.Fatalf("no owned paths: %v (%v)", err, words)
+	}
+	if !strings.HasPrefix(owned["/etc/hosts"], "hostnames") {
+		t.Errorf("/etc/hosts is owned by %q, want hostnames", owned["/etc/hosts"])
+	}
+	if !strings.HasPrefix(owned["/etc/app/app.conf"], "app") {
+		t.Errorf("a placed file is owned by %q, want app", owned["/etc/app/app.conf"])
+	}
+	if w, _ := r.filesWords(app, nil, Rendering{}); w[FilesOwnedPaths] != "" {
+		t.Error("a module not holding node-files is told the owned paths")
+	}
+}
+
+// issue 900: restore, like every write verb, takes a reason and an override.
+func TestNodeFilesRestoreTakesIdAndReason(t *testing.T) {
+	s, _ := SeatNamed(FilesSeat)
+	for _, v := range s.Serves {
+		if v.Name != "restore" {
+			continue
+		}
+		props, _ := v.Input["properties"].(map[string]any)
+		for _, p := range []string{"id", "reason", "override"} {
+			if _, has := props[p]; !has {
+				t.Errorf("restore has no input %s", p)
+			}
+		}
+		if req, _ := v.Input["required"].([]string); strings.Join(req, ",") != "id,reason" {
+			t.Errorf("restore requires %v, want id and reason", req)
+		}
+		return
+	}
+	t.Error("no restore verb")
+}

internal/catalogue/manifest.go

@@ -1984,6 +1984,7 @@ func parseManifest(raw []byte, predefined PredefinedList) (Manifest, error) {
 	problems = append(problems, declaredSeatProblems(m)...)
 	problems = append(problems, RunsAsProblems(m)...)
 	problems = append(problems, ReadsHolderProblems(m)...)
+	problems = append(problems, FilesHolderProblems(m)...)
 	if m.Computed != "" && len(m.Resources) > 0 {
 		// One or the other. A module that both ships files and has them computed would leave
 		// nobody able to say where a given file came from.

internal/catalogue/runtime.go

@@ -178,6 +178,10 @@ func (r Resolution) runtimeProcess(with Rendering) (map[string]any, error) {
 		if err != nil {
 			return nil, err
 		}
+		// The holder of node-files is told the same, and the paths modules own here (novox/hq ADR 0394 §4).
+		if words, err = r.filesWords(m, words, with); err != nil {
+			return nil, err
+		}
 		if len(words) > 0 {
 			given[m.Module] = words
 		}

internal/catalogue/seats.go

@@ -421,6 +421,9 @@ var defaultSeats = append([]Seat{
 	// A node's files and accounts, read (novox/hq ADR 0310, reads.go): by an account of the holder's own that holds
 	// no secret, on every node, so the reads ADR 0268 §5 takes over from the login shell's execute have a tool.
 	{Name: ReadsSeat, Scope: ScopeNode, Decision: "novox/hq ADR 0310", Serves: readsVerbs()},
+	// A node's files, read and managed (novox/hq ADR 0394, files.go): beside node-reads until mesh-files is assigned
+	// on every node, then node-reads goes (ADR 0394 §9).
+	{Name: FilesSeat, Scope: ScopeNode, Decision: "novox/hq ADR 0394", Serves: filesVerbs()},
 },
 	// The graphical session's roles (novox/hq ADR 0208), last because they are a workstation's.
 	graphicalSessionSeats()...)

internal/catalogue/seats_test.go

@@ -48,6 +48,7 @@ func TestTheSeatsAreAClosedSetAndEachNamesItsDecision(t *testing.T) {
 			delivered[s.Delivers] = s.Name
 		}
 	}
+	// Forty-five with node-files (novox/hq ADR 0394); forty-four again once node-reads goes.
 	// Forty-four with node-provisioner (novox/hq ADR 0349 clause 3); forty-three with mesh-provisioner (novox/hq ADR 0349 clause 3); forty-two with mesh-config (novox/hq ADR 0348 §9); forty-one with node-reads (novox/hq ADR 0310); forty with node-nfs-server and node-mounts (novox/hq ADR 0263); thirty-eight with node-resolver (novox/hq ADR 0247); thirty-seven with mesh-delivery (novox/hq ADR 0239); thirty-six since node-resolver-config retired
 	// into node-uplink (novox/hq ADR 0223); thirty-seven
 	// since the retired node-dns-resolver went (novox/hq ADR 0220); thirty-eight with
@@ -59,8 +60,8 @@ func TestTheSeatsAreAClosedSetAndEachNamesItsDecision(t *testing.T) {
 	// node-container-runtime (ADR 0207); nineteen with node-environment and node-login-shell (ADR 0203,
 	// ADR 0204); seventeen with node-build-agent (ADR 0190). One fewer once the retired
 	// mesh-build-machine row goes, when no registered manifest claims it.
-	if len(Seats()) != 44 {
-		t.Errorf("the mesh defines %d seats rather than 44; the set is closed, so a change here is "+
+	if len(Seats()) != 45 {
+		t.Errorf("the mesh defines %d seats rather than 45; the set is closed, so a change here is "+
 			"a decision (novox/hq ADR 0110): %s", len(Seats()), seatNames())
 	}
 }