diff --git a/modules/flatpak/README.md b/modules/flatpak/README.md new file mode 100644 index 0000000..6b4d64f --- /dev/null +++ b/modules/flatpak/README.md @@ -0,0 +1,80 @@ +# flatpak + +Flatpak applications on the two workstations (novox/hq research 027/02: "`snapd` and `flatpak` are +modules, on the two workstations only"; to-be 42 phase 2 step 9). + +## Owns + +| what | where | +|---|---| +| flatpak | package `flatpak` (official repositories) | + +**The Flathub remote comes with the package.** The package ships `flathub.flatpakrepo` in +`/usr/share/flatpak/remotes.d`, and flatpak adds every file there to the system installation as a +remote. The module therefore declares no remote file of its own. `flatpak_remotes` checks that the +system installation has Flathub, and says so when it does not. + +The installed applications and runtimes, and their data, are the operator's: found (ADR 0182). + +## Improves + +- **The laptop gains flatpak**, which only the desktop had, with Flathub. +- **Unused runtimes become visible and removable.** On the desktop on 2026-10-04, `/var/lib/flatpak` + held 5.1 GB for three applications. They are Nextcloud on KDE 6.7, Warehouse on GNOME 46, and Plex on + Freedesktop 23.08. Beside them sat the whole Freedesktop 22.08 platform, both of its GL extensions, + and older codec and style extensions that nothing uses. `flatpak_unused` lists them read-only; + `flatpak_remove_unused` lets flatpak remove them. +- **A duplicate remote is named.** The desktop has Flathub configured twice, once in the system + installation and once in the account's, which has nothing installed from it. `flatpak_remotes` + reports it. + +## Tools + +All answer JSON; `(r)` reads, `(a)` acts. Reads run as the operator account. An act on the system +installation (the default) goes through `sudo -n`; one on the account's own installation does not. + +| tool | what | +|---|---| +| `flatpak_list` (r) | applications: id, name, version, branch, origin, installation, size in bytes | +| `flatpak_runtimes` (r) | runtimes and extensions, the same way | +| `flatpak_remotes` (r) | both installations' remotes, how many refs come from each, findings | +| `flatpak_updates` (r) | refs with a newer commit on their remote (needs the network) | +| `flatpak_unused` (r) | runtimes nothing needs, computed without changing anything (below), with the space each takes | +| `flatpak_disk_usage` (r) | each installation's directory on disk, and the 50 largest refs | +| `flatpak_install` (a) | install from a remote (default `flathub`) into the system or the account installation | +| `flatpak_remove` (a) | uninstall, keeping the application's data unless `delete_data` | +| `flatpak_update` (a) | update one ref, or everything in an installation | +| `flatpak_remove_unused` (a) | `uninstall --unused`: flatpak decides | +| `flatpak_job` (r) | a long act's state and the end of its output | + +**Unused, computed.** `flatpak uninstall --unused` has no dry run: it asks, and answering *yes* removes. +So the read tool works it out from what flatpak already says. A runtime is in use when one of these +holds: + +- an installed application names it as its runtime or SDK; +- it fills an extension point declared in the metadata of something in use, matched by id + (subdirectories included) and accepted version; +- it is pinned. + +The SDK is counted as used to err on the safe side. flatpak's own `--unused` may also remove an +application's SDK. + +**Acts are jobs.** An install or update downloads hundreds of megabytes. Each act runs inside the +tool's process for up to 15 minutes, and is waited on for 18 s. A finished act is answered, and a +failed one as an error. One still running is answered with a job id for `flatpak_job`. + +## What changes when it is assigned + +- **laptop:** `flatpak` is installed, with Flathub as the system remote. Nothing else. +- **desktop:** nothing on disk; the package, installed by hand, becomes the mesh's. + +## The one-off step for the operator (ADR 0182) + +On the desktop, the account's own Flathub remote duplicates the system's and has nothing installed +from it. If it is not wanted: `flatpak remote-delete --user flathub`, once. The mesh did not make it, +so it does not remove it. + +## Leaves as found + +The applications and runtimes, the account installation under `~/.local/share/flatpak`, application +data under `~/.var/app`, and any remote other than the package's Flathub. diff --git a/modules/flatpak/cmd/flatpak-tools/flatpak.go b/modules/flatpak/cmd/flatpak-tools/flatpak.go new file mode 100644 index 0000000..1fdf87c --- /dev/null +++ b/modules/flatpak/cmd/flatpak-tools/flatpak.go @@ -0,0 +1,501 @@ +package main + +import ( + "fmt" + "os" + "path/filepath" + "regexp" + "sort" + "strconv" + "strings" +) + +// refs are an application or runtime id, optionally with arch and branch: org.gimp.GIMP, +// org.freedesktop.Platform/x86_64/23.08, app/org.gimp.GIMP/x86_64/stable. +var refs = regexp.MustCompile(`^((app|runtime)/)?[A-Za-z][A-Za-z0-9_-]*(\.[A-Za-z0-9_-]+)+(/[A-Za-z0-9_]*(/[A-Za-z0-9._-]*)?)?$`) + +func checkRef(r string) error { + if !refs.MatchString(r) { + return fmt.Errorf("%q is not a flatpak id or ref", r) + } + return nil +} + +var remoteName = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*$`) + +// Ref is one installed application or runtime. +type Ref struct { + ID string `json:"id"` + Name string `json:"name,omitempty"` + Version string `json:"version,omitempty"` + Branch string `json:"branch"` + Arch string `json:"arch"` + Ref string `json:"ref"` + Origin string `json:"origin"` + Installation string `json:"installation"` + Size string `json:"size"` + Bytes int64 `json:"bytes"` +} + +// sizeBytes reads flatpak's human size ("275.8 MB", with a no-break space): decimal units. +func sizeBytes(s string) int64 { + f := strings.Fields(strings.ReplaceAll(s, " ", " ")) + if len(f) == 0 { + return 0 + } + n, err := strconv.ParseFloat(f[0], 64) + if err != nil { + return 0 + } + unit := map[string]float64{"bytes": 1, "byte": 1, "B": 1, "kB": 1e3, "KB": 1e3, "MB": 1e6, "GB": 1e9, "TB": 1e12} + if len(f) > 1 { + if m, ok := unit[f[1]]; ok { + n *= m + } + } + return int64(n) +} + +func installFlag(inst string) []string { + if inst == "" { + return nil + } + return []string{"--" + inst} +} + +// listRefs runs flatpak list for apps or runtimes. +func listRefs(kind, inst string) ([]Ref, error) { + args := append([]string{"list", "--" + kind, "--columns=application,name,version,branch,arch,origin,installation,size,ref"}, installFlag(inst)...) + r, err := call(Cmd{Name: "flatpak", Args: args}) + if err != nil { + return nil, err + } + out := []Ref{} + for _, l := range lines(r.Stdout) { + f := strings.Split(l, "\t") + if len(f) < 9 { + continue + } + size := strings.ReplaceAll(f[7], " ", " ") + out = append(out, Ref{ID: f[0], Name: f[1], Version: f[2], Branch: f[3], Arch: f[4], Origin: f[5], Installation: f[6], Size: size, Bytes: sizeBytes(size), Ref: f[8]}) + } + return out, nil +} + +// ListAnswer is what flatpak_list and flatpak_runtimes answer. +type ListAnswer struct { + Count int `json:"count"` + Bytes int64 `json:"bytes"` + Refs []Ref `json:"refs"` +} + +// List answers the applications or runtimes. +func List(kind, inst string) (ListAnswer, error) { + got, err := listRefs(kind, inst) + if err != nil { + return ListAnswer{}, err + } + out := ListAnswer{Count: len(got), Refs: got} + for _, r := range got { + out.Bytes += r.Bytes + } + return out, nil +} + +// Remote is one remote of one installation. +type Remote struct { + Name string `json:"name"` + URL string `json:"url"` + Installation string `json:"installation"` + Priority string `json:"priority"` + Disabled bool `json:"disabled"` + Installed int `json:"installed_from"` +} + +// RemotesAnswer is what flatpak_remotes answers. +type RemotesAnswer struct { + Remotes []Remote `json:"remotes"` + Flathub bool `json:"flathub_system"` + Findings []string `json:"findings"` +} + +// Remotes answers both installations' remotes and what is wrong with them. +func Remotes() (RemotesAnswer, error) { + r, err := call(Cmd{Name: "flatpak", Args: []string{"remotes", "--show-disabled", "--columns=name,url,options,priority"}}) + if err != nil { + return RemotesAnswer{}, err + } + all, err := listRefs("app", "") + if err != nil { + return RemotesAnswer{}, err + } + rt, err := listRefs("runtime", "") + if err != nil { + return RemotesAnswer{}, err + } + from := map[string]int{} + for _, x := range append(all, rt...) { + from[x.Origin+"\x00"+x.Installation]++ + } + out := RemotesAnswer{Remotes: []Remote{}, Findings: []string{}} + byURL := map[string][]string{} + for _, l := range lines(r.Stdout) { + f := strings.Split(l, "\t") + if len(f) < 4 { + continue + } + inst := "system" + disabled := false + for _, o := range strings.Split(f[2], ",") { + switch strings.TrimSpace(o) { + case "user": + inst = "user" + case "disabled": + disabled = true + } + if strings.HasPrefix(strings.TrimSpace(o), "system") { + inst = strings.TrimSpace(o) + } + } + rem := Remote{Name: f[0], URL: f[1], Installation: inst, Priority: f[3], Disabled: disabled, Installed: from[f[0]+"\x00"+inst]} + out.Remotes = append(out.Remotes, rem) + byURL[strings.TrimRight(rem.URL, "/")] = append(byURL[strings.TrimRight(rem.URL, "/")], rem.Name+" ("+inst+")") + if inst == "system" && rem.Name == "flathub" && !disabled { + out.Flathub = true + } + if rem.Installed == 0 { + out.Findings = append(out.Findings, fmt.Sprintf("remote %s in the %s installation has nothing installed from it", rem.Name, inst)) + } + } + if !out.Flathub { + out.Findings = append(out.Findings, "the system installation has no enabled flathub remote: the flatpak package ships one in /usr/share/flatpak/remotes.d, so it was removed or disabled by hand") + } + for url, names := range byURL { + if len(names) > 1 { + sort.Strings(names) + out.Findings = append(out.Findings, fmt.Sprintf("%s is configured %d times: %s", url, len(names), strings.Join(names, ", "))) + } + } + sort.Strings(out.Findings) + return out, nil +} + +// Updates answers what an update would change. +func Updates(inst string) (map[string]any, error) { + args := append([]string{"remote-ls", "--updates", "--columns=application,branch,version,origin,ref"}, installFlag(inst)...) + r, err := call(Cmd{Name: "flatpak", Args: args}) + if err != nil { + return nil, err + } + out := []map[string]string{} + for _, l := range lines(r.Stdout) { + f := strings.Split(l, "\t") + if len(f) >= 5 { + out = append(out, map[string]string{"id": f[0], "branch": f[1], "version": f[2], "origin": f[3], "ref": f[4]}) + } + } + return map[string]any{"count": len(out), "updates": out}, nil +} + +// Extension is one extension point a ref's metadata declares. +type Extension struct { + ID string + Versions []string + Subdirs bool +} + +// extensionsOf reads [Extension …] groups from a ref's metadata, with the versions each accepts: +// versions, else version, else the branch of the ref declaring it. +func extensionsOf(metadata, branch string) []Extension { + out := []Extension{} + var cur *Extension + flush := func() { + if cur != nil { + if len(cur.Versions) == 0 { + cur.Versions = []string{branch} + } + out = append(out, *cur) + } + cur = nil + } + var version, versions string + for _, l := range strings.Split(metadata, "\n") { + l = strings.TrimSpace(l) + if strings.HasPrefix(l, "[") { + if cur != nil { + cur.Versions = pickVersions(versions, version) + } + flush() + version, versions = "", "" + if strings.HasPrefix(l, "[Extension ") && strings.HasSuffix(l, "]") { + cur = &Extension{ID: strings.TrimSuffix(strings.TrimPrefix(l, "[Extension "), "]")} + } + continue + } + if cur == nil { + continue + } + k, v, found := strings.Cut(l, "=") + if !found { + continue + } + switch strings.TrimSpace(k) { + case "version": + version = strings.TrimSpace(v) + case "versions": + versions = strings.TrimSpace(v) + case "subdirectories": + cur.Subdirs = strings.TrimSpace(v) == "true" + } + } + if cur != nil { + cur.Versions = pickVersions(versions, version) + } + flush() + return out +} + +func pickVersions(versions, version string) []string { + if versions != "" { + out := []string{} + for _, v := range strings.Split(versions, ";") { + if v = strings.TrimSpace(v); v != "" { + out = append(out, v) + } + } + return out + } + if version != "" { + return []string{version} + } + return nil +} + +// fills says whether a runtime fills an extension point. +func (e Extension) fills(r Ref) bool { + if !(r.ID == e.ID || (e.Subdirs && strings.HasPrefix(r.ID, e.ID+"."))) { + return false + } + for _, v := range e.Versions { + if v == r.Branch { + return true + } + } + return false +} + +// UnusedAnswer is what flatpak_unused answers. +type UnusedAnswer struct { + Count int `json:"count"` + Bytes int64 `json:"bytes"` + Unused []Ref `json:"unused"` + Pinned []string `json:"pinned"` + Note string `json:"note"` +} + +func info(inst string, extra ...string) (string, error) { + r, err := call(Cmd{Name: "flatpak", Args: append(append([]string{"info"}, installFlag(inst)...), extra...)}) + return strings.TrimSpace(r.Stdout), err +} + +// Unused computes what no installed application needs. +func Unused() (UnusedAnswer, error) { + apps, err := listRefs("app", "") + if err != nil { + return UnusedAnswer{}, err + } + runtimes, err := listRefs("runtime", "") + if err != nil { + return UnusedAnswer{}, err + } + key := func(inst, ref string) string { + return inst + "\x00" + strings.TrimPrefix(strings.TrimPrefix(ref, "runtime/"), "app/") + } + installed := map[string]Ref{} + for _, r := range runtimes { + installed[key(r.Installation, r.Ref)] = r + } + used := map[string]bool{} + type item struct { + inst, ref, branch string + } + queue := []item{} + // A user installation's application may use a system runtime; a system one only system runtimes. + mark := func(inst, ref string) { + for _, where := range []string{inst, "system"} { + k := key(where, ref) + if r, ok := installed[k]; ok && !used[k] { + used[k] = true + queue = append(queue, item{where, r.Ref, r.Branch}) + return + } + } + } + for _, a := range apps { + for _, flag := range []string{"--show-runtime", "--show-sdk"} { + ref, err := info(a.Installation, flag, a.Ref) + if err != nil { + return UnusedAnswer{}, err + } + if ref != "" { + mark(a.Installation, ref) + } + } + queue = append(queue, item{a.Installation, a.Ref, a.Branch}) + } + pinned := []string{} + for _, inst := range []string{"system", "user"} { + r := run(Cmd{Name: "flatpak", Args: []string{"pin", "--" + inst}}) + if r.Status != 0 || r.Error != "" { + continue + } + for _, l := range lines(r.Stdout) { + p := strings.TrimSpace(l) + if strings.HasPrefix(p, "runtime/") { + pinned = append(pinned, p) + mark(inst, p) + } + } + } + for len(queue) > 0 { + it := queue[0] + queue = queue[1:] + meta, err := info(it.inst, "--show-metadata", it.ref) + if err != nil { + return UnusedAnswer{}, err + } + for _, e := range extensionsOf(meta, it.branch) { + for k, r := range installed { + if !used[k] && e.fills(r) && (r.Installation == it.inst || r.Installation == "system") { + used[k] = true + queue = append(queue, item{r.Installation, r.Ref, r.Branch}) + } + } + } + } + out := UnusedAnswer{Unused: []Ref{}, Pinned: pinned, + Note: "Computed without changing anything. flatpak_remove_unused lets flatpak decide, which may differ in detail (a locale or debug extension listed here by nothing)."} + for k, r := range installed { + if !used[k] { + out.Unused = append(out.Unused, r) + out.Bytes += r.Bytes + } + } + sort.Slice(out.Unused, func(i, k int) bool { return out.Unused[i].Ref < out.Unused[k].Ref }) + out.Count = len(out.Unused) + return out, nil +} + +// Where each installation lives. +var installDirs = map[string]string{"system": "/var/lib/flatpak", "user": ".local/share/flatpak"} + +// DiskAnswer is what flatpak_disk_usage answers. +type DiskAnswer struct { + Installations map[string]int64 `json:"installation_bytes"` + Largest []Ref `json:"largest"` + Note string `json:"note,omitempty"` +} + +// DiskUsage measures each installation's directory and lists the largest refs. +func DiskUsage() (DiskAnswer, error) { + out := DiskAnswer{Installations: map[string]int64{}, Largest: []Ref{}} + for inst, dir := range installDirs { + if !filepath.IsAbs(dir) { + dir = filepath.Join(accountHome(), dir) + } + // du exits 1 when a file is unreadable and still prints the total of what it could read. + r := run(Cmd{Name: "du", Args: []string{"-sb", dir}}) + if r.Error != "" { + return DiskAnswer{}, failure(Cmd{Name: "du", Args: []string{"-sb", dir}}, r) + } + f := strings.Fields(r.Stdout) + if len(f) == 0 { + if strings.Contains(r.Stderr, "No such file") { + out.Installations[inst] = 0 + continue + } + return DiskAnswer{}, failure(Cmd{Name: "du", Args: []string{"-sb", dir}}, r) + } + n, _ := strconv.ParseInt(f[0], 10, 64) + out.Installations[inst] = n + if r.Status != 0 { + out.Note = "some files were unreadable to the account, so a total is a lower bound" + } + } + apps, err := listRefs("app", "") + if err != nil { + return DiskAnswer{}, err + } + rts, err := listRefs("runtime", "") + if err != nil { + return DiskAnswer{}, err + } + all := append(apps, rts...) + sort.Slice(all, func(i, k int) bool { return all[i].Bytes > all[k].Bytes }) + if len(all) > 50 { + all = all[:50] + } + out.Largest = all + return out, nil +} + +func accountHome() string { + return homeFrom(getenv) +} + +// ActAnswer is what an act answers. +type ActAnswer struct { + Act string `json:"act"` + Ref string `json:"ref,omitempty"` + Installation string `json:"installation"` + Job Job `json:"job"` +} + +// act runs flatpak as a job; the system installation needs root, the account's does not. +func act(verb, ref, inst string, args ...string) (ActAnswer, error) { + c := Cmd{Name: "flatpak", Args: append([]string{verb, "--" + inst, "--noninteractive", "-y"}, args...), Root: inst == "system"} + j, err := actAsJob(c) + if err != nil { + return ActAnswer{}, err + } + return ActAnswer{Act: verb, Ref: ref, Installation: inst, Job: j}, nil +} + +// Install installs a ref from a remote. +func Install(ref, remote, inst string) (ActAnswer, error) { + if !remoteName.MatchString(remote) { + return ActAnswer{}, fmt.Errorf("%q is not a remote's name", remote) + } + return act("install", ref, inst, remote, ref) +} + +// Remove uninstalls a ref. +func Remove(ref, inst string, deleteData bool) (ActAnswer, error) { + if deleteData { + return act("uninstall", ref, inst, "--delete-data", ref) + } + return act("uninstall", ref, inst, ref) +} + +// Update updates one ref, or everything. +func Update(ref, inst string) (ActAnswer, error) { + if ref == "" { + return act("update", "", inst) + } + return act("update", ref, inst, ref) +} + +// RemoveUnused lets flatpak uninstall what it finds unused. +func RemoveUnused(inst string) (ActAnswer, error) { + return act("uninstall", "", inst, "--unused") +} + +// getenv and homeFrom read the account's home the way the runtime gives it. +var getenv = func(k string) string { return strings.TrimSpace(os.Getenv(k)) } + +func homeFrom(env func(string) string) string { + if h := env("MESH_OPERATOR_HOME"); h != "" { + return h + } + return env("HOME") +} diff --git a/modules/flatpak/cmd/flatpak-tools/flatpak_test.go b/modules/flatpak/cmd/flatpak-tools/flatpak_test.go new file mode 100644 index 0000000..22e7bf2 --- /dev/null +++ b/modules/flatpak/cmd/flatpak-tools/flatpak_test.go @@ -0,0 +1,249 @@ +package main + +import ( + "strings" + "testing" +) + +func TestTheManifestIsFlatpaksPackageAndFlathubComesWithIt(t *testing.T) { + m := readManifest(t) + holdsTheBundle(t, m, "flatpak") + if got := strings.Join(m.packages(), ","); got != "flatpak" { + t.Errorf("packages %s", got) + } + // The package ships flathub in /usr/share/flatpak/remotes.d: the module declares no remote file. + if len(m.Resources) != 1 { + t.Errorf("one resource: %v", m.Resources) + } +} + +const nb = " " + +// The desktop's installation on 2026-10-04, as flatpak list prints it. +var apps = "com.nextcloud.desktopclient.nextcloud\tNextcloud Desktop\t3.14\tstable\tx86_64\tflathub\tsystem\t275.8" + nb + "MB\tcom.nextcloud.desktopclient.nextcloud/x86_64/stable\n" + + "tv.plex.PlexDesktop\tPlex\t1.9\tstable\tx86_64\tflathub\tsystem\t368.0" + nb + "MB\ttv.plex.PlexDesktop/x86_64/stable\n" + +var runtimes = strings.Join([]string{ + "org.freedesktop.Platform\tFreedesktop Platform\t22.08.1\t22.08\tx86_64\tflathub\tsystem\t576.7" + nb + "MB\torg.freedesktop.Platform/x86_64/22.08", + "org.freedesktop.Platform\tFreedesktop Platform\t23.08.1\t23.08\tx86_64\tflathub\tsystem\t598.3" + nb + "MB\torg.freedesktop.Platform/x86_64/23.08", + "org.freedesktop.Platform.GL.default\tMesa\t\t22.08\tx86_64\tflathub\tsystem\t442.1" + nb + "MB\torg.freedesktop.Platform.GL.default/x86_64/22.08", + "org.freedesktop.Platform.GL.default\tMesa\t\t23.08\tx86_64\tflathub\tsystem\t533.8" + nb + "MB\torg.freedesktop.Platform.GL.default/x86_64/23.08", + "org.freedesktop.Platform.GL.default\tMesa\t\t23.08-extra\tx86_64\tflathub\tsystem\t533.8" + nb + "MB\torg.freedesktop.Platform.GL.default/x86_64/23.08-extra", + "org.freedesktop.Platform.openh264\topenh264\t\t2.2.0\tx86_64\tflathub\tsystem\t790.0" + nb + "kB\torg.freedesktop.Platform.openh264/x86_64/2.2.0", + "org.freedesktop.Sdk\tSdk\t\t23.08\tx86_64\tflathub\tsystem\t1.6" + nb + "GB\torg.freedesktop.Sdk/x86_64/23.08", + "org.kde.Platform\tKDE\t\t6.7\tx86_64\tflathub\tsystem\t931.4" + nb + "MB\torg.kde.Platform/x86_64/6.7", + "org.kde.KStyle.Adwaita\tAdwaita\t\t5.15-21.08\tx86_64\tflathub\tsystem\t16.3" + nb + "MB\torg.kde.KStyle.Adwaita/x86_64/5.15-21.08", + "org.kde.KStyle.Adwaita\tAdwaita\t\t6.7\tx86_64\tflathub\tsystem\t20.6" + nb + "MB\torg.kde.KStyle.Adwaita/x86_64/6.7", +}, "\n") + "\n" + +const platformMeta = `[Runtime] +name=org.freedesktop.Platform + +[Extension org.freedesktop.Platform.GL] +versions = 23.08;23.08-extra;1.4 +version = 1.4 +directory = lib/x86_64-linux-gnu/GL +subdirectories = true + +[Extension org.freedesktop.Platform.openh264] +directory = lib/openh264 +version = 2.2.0 + +[Extension org.freedesktop.Platform.Timezones] +directory = share/zoneinfo +` + +const kdeMeta = `[Runtime] +name=org.kde.Platform + +[Extension org.kde.KStyle] +directory = lib/plugins/styles +subdirectories = true +version = 6.7 + +[Extension org.freedesktop.Platform.GL] +versions = 23.08;1.4 +subdirectories = true +` + +func theDesktop(t *testing.T) *fake { + return using(t, func(line string, c Cmd) Result { + switch { + case strings.HasPrefix(line, "flatpak list --app"): + return ok(apps) + case strings.HasPrefix(line, "flatpak list --runtime"): + return ok(runtimes) + case strings.Contains(line, "--show-runtime com.nextcloud"): + return ok("org.kde.Platform/x86_64/6.7\n") + case strings.Contains(line, "--show-runtime tv.plex"): + return ok("org.freedesktop.Platform/x86_64/23.08\n") + case strings.Contains(line, "--show-sdk tv.plex"): + return ok("org.freedesktop.Sdk/x86_64/23.08\n") + case strings.Contains(line, "--show-sdk"): + return ok("org.kde.Sdk/x86_64/6.7\n") + case strings.Contains(line, "--show-metadata org.freedesktop.Platform/x86_64/23.08"): + return ok(platformMeta) + case strings.Contains(line, "--show-metadata org.kde.Platform"): + return ok(kdeMeta) + case strings.Contains(line, "--show-metadata"): + return ok("[Application]\nname=x\n") + case strings.HasPrefix(line, "flatpak pin"): + return ok("") + case strings.HasPrefix(line, "flatpak remotes"): + return ok("flathub\thttps://dl.flathub.org/repo/\tsystem\t1\nflathub\thttps://dl.flathub.org/repo/\tuser\t1\n") + } + return Result{Status: 9, Stderr: "unexpected " + line} + }) +} + +func TestUnusedIsWhatNoApplicationNeedsDirectlyOrThroughAnExtensionPoint(t *testing.T) { + f := theDesktop(t) + got, err := Unused() + if err != nil { + t.Fatal(err) + } + names := []string{} + for _, r := range got.Unused { + names = append(names, r.Ref) + } + want := "org.freedesktop.Platform.GL.default/x86_64/22.08,org.freedesktop.Platform/x86_64/22.08,org.kde.KStyle.Adwaita/x86_64/5.15-21.08" + if strings.Join(names, ",") != want { + t.Errorf("unused\n%s\nwant\n%s", strings.Join(names, ","), want) + } + if got.Bytes != 576700000+442100000+16300000 { + t.Errorf("bytes %d", got.Bytes) + } + for _, l := range f.lines() { + if strings.Contains(l, "uninstall") || strings.HasPrefix(l, "sudo") { + t.Errorf("unused only reads: %s", l) + } + } +} + +func TestAnExtensionPointAcceptsItsVersionsOrTheDeclaringBranch(t *testing.T) { + ext := extensionsOf(platformMeta, "23.08") + if len(ext) != 3 { + t.Fatalf("%+v", ext) + } + gl, h264, tz := ext[0], ext[1], ext[2] + if !gl.Subdirs || strings.Join(gl.Versions, ";") != "23.08;23.08-extra;1.4" { + t.Errorf("versions win over version: %+v", gl) + } + if h264.Subdirs || strings.Join(h264.Versions, ";") != "2.2.0" { + t.Errorf("%+v", h264) + } + if strings.Join(tz.Versions, ";") != "23.08" { + t.Errorf("no version is the declaring branch: %+v", tz) + } + if !gl.fills(Ref{ID: "org.freedesktop.Platform.GL.default", Branch: "23.08-extra"}) || gl.fills(Ref{ID: "org.freedesktop.Platform.GL.default", Branch: "22.08"}) { + t.Error("a subdirectory extension, by branch") + } + if h264.fills(Ref{ID: "org.freedesktop.Platform.openh264.x", Branch: "2.2.0"}) { + t.Error("without subdirectories only the id itself") + } +} + +func TestRemotesFindTheDuplicateAndTheEmptyOne(t *testing.T) { + theDesktop(t) + got, err := Remotes() + if err != nil || len(got.Remotes) != 2 || !got.Flathub { + t.Fatalf("%+v %v", got, err) + } + if got.Remotes[0].Installed != 12 || got.Remotes[1].Installed != 0 { + t.Errorf("%+v", got.Remotes) + } + all := strings.Join(got.Findings, ";") + if !strings.Contains(all, "flathub in the user installation has nothing installed") || !strings.Contains(all, "configured 2 times") { + t.Errorf("%s", all) + } +} + +func TestListReadsSizesAsBytes(t *testing.T) { + theDesktop(t) + got, err := List("app", "") + if err != nil || got.Count != 2 || got.Refs[0].Bytes != 275800000 || got.Refs[0].Size != "275.8 MB" || got.Bytes != 643800000 { + t.Fatalf("%+v %v", got, err) + } + for in, want := range map[string]int64{"1.6" + nb + "GB": 1600000000, "790.0 kB": 790000, "12 bytes": 12, "": 0} { + if b := sizeBytes(in); b != want { + t.Errorf("%q: %d", in, b) + } + } +} + +func TestActsOnTheSystemInstallationEscalateAndTheAccountsDoNot(t *testing.T) { + f := using(t, func(string, Cmd) Result { return ok("done") }) + if got, err := Install("org.gimp.GIMP", "flathub", "system"); err != nil || got.Job.Running { + t.Fatalf("%+v %v", got, err) + } + if _, err := Install("org.gimp.GIMP", "flathub", "user"); err != nil { + t.Fatal(err) + } + if _, err := Remove("org.gimp.GIMP", "system", true); err != nil { + t.Fatal(err) + } + if _, err := Update("", "user"); err != nil { + t.Fatal(err) + } + if _, err := RemoveUnused("system"); err != nil { + t.Fatal(err) + } + want := []string{ + "sudo -n flatpak install --system --noninteractive -y flathub org.gimp.GIMP", + "flatpak install --user --noninteractive -y flathub org.gimp.GIMP", + "sudo -n flatpak uninstall --system --noninteractive -y --delete-data org.gimp.GIMP", + "flatpak update --user --noninteractive -y", + "sudo -n flatpak uninstall --system --noninteractive -y --unused", + } + if got := strings.Join(f.lines(), "\n"); got != strings.Join(want, "\n") { + t.Errorf("asked\n%s", got) + } + if _, err := Install("org.gimp.GIMP", "--from=x", "system"); err == nil { + t.Error("an option as a remote") + } + for _, bad := range []string{"--assumeyes", "gimp", "org.gimp.GIMP; rm", "org/../x"} { + if checkRef(bad) == nil { + t.Errorf("%q accepted as a ref", bad) + } + } + for _, good := range []string{"org.gimp.GIMP", "org.freedesktop.Platform/x86_64/23.08", "app/org.gimp.GIMP/x86_64/stable", "org.freedesktop.Platform.GL.default//23.08-extra"} { + if err := checkRef(good); err != nil { + t.Errorf("%v", err) + } + } +} + +func TestAFailedInstallIsAnErrorAndAMissingFlatpakIsSaid(t *testing.T) { + using(t, func(string, Cmd) Result { + return Result{Status: 1, Stderr: "error: Nothing matches org.nope.Nope in remote flathub"} + }) + if _, err := Install("org.nope.Nope", "flathub", "user"); err == nil || !strings.Contains(err.Error(), "Nothing matches") { + t.Fatalf("%v", err) + } + using(t, func(string, Cmd) Result { return Result{Status: 127, Error: "not-found"} }) + if _, err := List("app", ""); err == nil || !strings.Contains(err.Error(), "not installed") { + t.Fatalf("%v", err) + } +} + +func TestDiskUsageReadsEachInstallationAndToleratesAnUnreadableFile(t *testing.T) { + t.Setenv("MESH_OPERATOR_HOME", "/home/op") + using(t, func(line string, c Cmd) Result { + switch { + case line == "du -sb /var/lib/flatpak": + return Result{Status: 1, Stdout: "5094728394\t/var/lib/flatpak\n", Stderr: "du: cannot read directory 'x': Permission denied"} + case line == "du -sb /home/op/.local/share/flatpak": + return Result{Status: 1, Stderr: "du: cannot access '/home/op/.local/share/flatpak': No such file or directory"} + case strings.Contains(line, "--app"): + return ok(apps) + } + return ok(runtimes) + }) + got, err := DiskUsage() + if err != nil || got.Installations["system"] != 5094728394 || got.Installations["user"] != 0 || got.Note == "" { + t.Fatalf("%+v %v", got, err) + } + if got.Largest[0].ID != "org.freedesktop.Sdk" { + t.Errorf("largest first: %+v", got.Largest[0]) + } +} diff --git a/modules/flatpak/cmd/flatpak-tools/jobs.go b/modules/flatpak/cmd/flatpak-tools/jobs.go new file mode 100644 index 0000000..3885479 --- /dev/null +++ b/modules/flatpak/cmd/flatpak-tools/jobs.go @@ -0,0 +1,151 @@ +package main + +// jobs.go is the same file in the bundles whose acts can outlast one call (flatpak, docker-compose): +// an install or an `up` that pulls images takes minutes, and the runtime gives a call 30 s. Such an +// act is started as a job inside this process, waited on for a while, and answered either finished +// or with the job's id for the module's `_job` tool to follow. A job ends with this process: if the +// runtime restarts the bundle, a running job is cut off, and its id is then unknown. + +import ( + "fmt" + "sort" + "strings" + "sync" + "time" +) + +// JobLimit is the longest a job may run; JobWait how long an act waits before answering a job id. +const ( + JobLimit = 15 * time.Minute + JobWait = 18 * time.Second + keptJobs = 50 +) + +// Job is one long act, as its tool answers it. +type Job struct { + ID string `json:"job"` + Command string `json:"command"` + Started time.Time `json:"started"` + Finished *time.Time `json:"finished,omitempty"` + Running bool `json:"running"` + Status *int `json:"status,omitempty"` + Error string `json:"error,omitempty"` + Output string `json:"output,omitempty"` + Truncated bool `json:"truncated,omitempty"` + done chan struct{} +} + +type jobBook struct { + mu sync.Mutex + seq int + jobs map[string]*Job +} + +var jobs = &jobBook{jobs: map[string]*Job{}} + +// startJob runs c in the background, held to JobLimit. +func startJob(c Cmd) *Job { + c.Timeout = JobLimit + name, args := argv(c) + jobs.mu.Lock() + jobs.seq++ + j := &Job{ID: fmt.Sprintf("%d-%d", time.Now().Unix(), jobs.seq), Command: strings.TrimSpace(name + " " + strings.Join(args, " ")), + Started: time.Now().UTC(), Running: true, done: make(chan struct{})} + jobs.jobs[j.ID] = j + jobs.forgetOldest() + jobs.mu.Unlock() + go func() { + r := run(c) + var err error + if r.Status != 0 || r.Error != "" { + err = failure(c, r) + } + jobs.mu.Lock() + now := time.Now().UTC() + j.Finished, j.Running = &now, false + status := r.Status + j.Status = &status + if err != nil { + j.Error = err.Error() + } + j.Output = tail(strings.TrimSpace(r.Stdout+"\n"+r.Stderr), 16<<10) + j.Truncated = r.Truncated || len(r.Stdout)+len(r.Stderr) > 16<<10 + jobs.mu.Unlock() + close(j.done) + }() + return j +} + +// forgetOldest keeps the book bounded; finished jobs go first. Called with the lock held. +func (b *jobBook) forgetOldest() { + if len(b.jobs) <= keptJobs { + return + } + all := make([]*Job, 0, len(b.jobs)) + for _, j := range b.jobs { + all = append(all, j) + } + sort.Slice(all, func(i, k int) bool { return all[i].Started.Before(all[k].Started) }) + for _, j := range all { + if len(b.jobs) <= keptJobs { + return + } + if !j.Running { + delete(b.jobs, j.ID) + } + } +} + +// awaitJob waits up to d for a job to finish and answers a copy of it as it then stands. +func awaitJob(j *Job, d time.Duration) Job { + select { + case <-j.done: + case <-time.After(d): + } + return snapshot(j) +} + +func snapshot(j *Job) Job { + jobs.mu.Lock() + defer jobs.mu.Unlock() + c := *j + c.done = nil + return c +} + +// jobByID answers a job by its id, or says it is not known to this process. +func jobByID(id string) (Job, error) { + jobs.mu.Lock() + j, ok := jobs.jobs[id] + jobs.mu.Unlock() + if !ok { + return Job{}, fmt.Errorf("no job %s in this process: it was never started here, was forgotten after %d newer ones, or the bundle has restarted since", id, keptJobs) + } + return snapshot(j), nil +} + +// listJobs answers every job this process knows, newest first. +func listJobs() []Job { + jobs.mu.Lock() + all := make([]*Job, 0, len(jobs.jobs)) + for _, j := range jobs.jobs { + all = append(all, j) + } + jobs.mu.Unlock() + sort.Slice(all, func(i, k int) bool { return all[i].Started.After(all[k].Started) }) + out := make([]Job, 0, len(all)) + for _, j := range all { + out = append(out, snapshot(j)) + } + return out +} + +// actAsJob starts c and answers the job once it finishes or JobWait passes, whichever is first. +// A finished job that failed is answered as an error, so a failed act is never read as success. +func actAsJob(c Cmd) (Job, error) { + j := awaitJob(startJob(c), JobWait) + if !j.Running && j.Error != "" { + return j, fmt.Errorf("%s (job %s)", j.Error, j.ID) + } + return j, nil +} diff --git a/modules/flatpak/cmd/flatpak-tools/jobs_test.go b/modules/flatpak/cmd/flatpak-tools/jobs_test.go new file mode 100644 index 0000000..46e1725 --- /dev/null +++ b/modules/flatpak/cmd/flatpak-tools/jobs_test.go @@ -0,0 +1,51 @@ +package main + +import ( + "strings" + "testing" + "time" +) + +func TestJobsAFastActIsAnsweredFinishedAndAFailedOneAsAnError(t *testing.T) { + using(t, func(line string, c Cmd) Result { + if c.Timeout != JobLimit { + t.Errorf("a job is held to JobLimit, not %s", c.Timeout) + } + if strings.Contains(line, "bad") { + return Result{Status: 2, Stderr: "it broke"} + } + return ok("done") + }) + j, err := actAsJob(Cmd{Name: "good"}) + if err != nil || j.Running || j.Status == nil || *j.Status != 0 || j.Output != "done" { + t.Fatalf("%+v %v", j, err) + } + if _, err := actAsJob(Cmd{Name: "bad"}); err == nil || !strings.Contains(err.Error(), "it broke") { + t.Fatalf("a failed job: %v", err) + } + got, err := jobByID(j.ID) + if err != nil || got.ID != j.ID { + t.Fatalf("by id: %+v %v", got, err) + } + if _, err := jobByID("nope"); err == nil { + t.Fatal("an unknown job") + } + if len(listJobs()) < 2 { + t.Fatal("listed") + } +} + +func TestJobsASlowActIsAnsweredRunningWithItsID(t *testing.T) { + release := make(chan struct{}) + using(t, func(line string, c Cmd) Result { <-release; return ok("") }) + j := awaitJob(startJob(Cmd{Name: "slow"}), 50*time.Millisecond) + if !j.Running || j.ID == "" { + t.Fatalf("%+v", j) + } + close(release) + time.Sleep(50 * time.Millisecond) + got, _ := jobByID(j.ID) + if got.Running { + t.Fatalf("finished afterwards: %+v", got) + } +} diff --git a/modules/flatpak/cmd/flatpak-tools/kit.go b/modules/flatpak/cmd/flatpak-tools/kit.go new file mode 100644 index 0000000..adc5aac --- /dev/null +++ b/modules/flatpak/cmd/flatpak-tools/kit.go @@ -0,0 +1,352 @@ +package main + +// kit.go is the same file in each of the workstations' tool bundles (fonts, docker-compose, snapd, +// flatpak, cups, bluetooth, xclip, dmenu): how a tool runs a command, escalates, bounds what it +// keeps, and names a failure. A module is built from its own directory, so the file is copied rather +// than shared; a change to one copy is made to all eight. +// +// The rules it holds (novox/hq research 026/05, to-be 38 WP4): +// - the node's tool runtime runs as the operator account, not root (ADR 0175 §4); a command that +// needs root goes through `sudo -n`, never a prompt, and a refusal is named as such; +// - one command gets 20 s, below the runtime's 30 s call limit, and is ended with everything it +// started when it takes longer; +// - each stream is kept to 256 KiB, and the answer says when it was cut; +// - a failure is an error with what went wrong in it, never an empty answer. + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "os" + "os/exec" + "strings" + "syscall" + "time" +) + +// Bounds every command is held to. +const ( + CallTimeout = 20 * time.Second + MostOutput = 256 << 10 +) + +// Cmd is one command a tool runs. +type Cmd struct { + Name string + Args []string + // Stdin is written to the command's standard input when not empty. + Stdin string + // Env is added to this process's own environment. + Env []string + // Root says the command needs root: it is run through `sudo -n` when this process is not root. + Root bool + // Timeout replaces CallTimeout; only a background job (jobs.go) asks for longer. + Timeout time.Duration + // Detached is for a program that forks a child which outlives it, as xclip does to keep the + // selection: its streams go to files, because a pipe the child inherits would hold the call open + // until the child exits. + Detached bool +} + +// Result is what a command did. +type Result struct { + Stdout string `json:"stdout"` + Stderr string `json:"stderr"` + Status int `json:"status"` + // Error is why it did not run to an answer: "not-found" when the program is not there, + // "timeout" when it was ended for taking too long, else the spawn error. + Error string `json:"error,omitempty"` + Truncated bool `json:"truncated,omitempty"` +} + +// Runner runs a command. Tests replace it; nothing else does. +type Runner func(Cmd) Result + +var ( + run Runner = execRun + euid = os.Geteuid +) + +// argv is the command as it is run: through sudo without a prompt when it needs root and this +// process is not root. +func argv(c Cmd) (string, []string) { + if c.Root && euid() != 0 { + return "sudo", append([]string{"-n", c.Name}, c.Args...) + } + return c.Name, c.Args +} + +// bounded keeps the first MostOutput bytes written to it and notes that more came. +type bounded struct { + b bytes.Buffer + cut bool +} + +func (w *bounded) Write(p []byte) (int, error) { + room := MostOutput - w.b.Len() + if room <= 0 { + w.cut = w.cut || len(p) > 0 + return len(p), nil + } + if len(p) > room { + w.b.Write(p[:room]) + w.cut = true + return len(p), nil + } + return w.b.Write(p) +} + +func execRun(c Cmd) Result { + timeout := c.Timeout + if timeout <= 0 { + timeout = CallTimeout + } + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + name, args := argv(c) + cmd := exec.CommandContext(ctx, name, args...) + cmd.Env = append(append(os.Environ(), "LC_ALL=C"), c.Env...) + if !c.Detached { + // Its own process group, so that ending it on a timeout ends what it started too. + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + cmd.Cancel = func() error { + if cmd.Process != nil { + _ = syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) + } + return nil + } + } + cmd.WaitDelay = 2 * time.Second + if c.Stdin != "" { + cmd.Stdin = strings.NewReader(c.Stdin) + } + var out, errs bounded + var outFile, errFile *os.File + if c.Detached { + var err error + if outFile, err = os.CreateTemp("", "mesh-tool-out-*"); err != nil { + return Result{Status: 127, Error: err.Error()} + } + defer os.Remove(outFile.Name()) + defer outFile.Close() + if errFile, err = os.CreateTemp("", "mesh-tool-err-*"); err != nil { + return Result{Status: 127, Error: err.Error()} + } + defer os.Remove(errFile.Name()) + defer errFile.Close() + cmd.Stdout, cmd.Stderr = outFile, errFile + } else { + cmd.Stdout, cmd.Stderr = &out, &errs + } + err := cmd.Run() + if c.Detached { + for _, f := range []struct { + file *os.File + into *bounded + }{{outFile, &out}, {errFile, &errs}} { + if _, e := f.file.Seek(0, io.SeekStart); e == nil { + _, _ = io.Copy(f.into, f.file) + } + } + } + r := Result{Stdout: out.b.String(), Stderr: errs.b.String(), Truncated: out.cut || errs.cut} + var exit *exec.ExitError + switch { + case err == nil: + case ctx.Err() == context.DeadlineExceeded: + r.Status, r.Error = 124, "timeout" + case errors.Is(err, exec.ErrNotFound) || errors.Is(err, os.ErrNotExist): + r.Status, r.Error = 127, "not-found" + case errors.As(err, &exit): + r.Status = exit.ExitCode() + default: + r.Status, r.Error = 127, err.Error() + } + return r +} + +// call runs a command and answers its result, or an error naming what went wrong. +func call(c Cmd) (Result, error) { + r := run(c) + if r.Status == 0 && r.Error == "" { + return r, nil + } + return r, failure(c, r) +} + +// failure names how a command failed: not installed, refused escalation, too slow, or its exit +// status with the end of what it said. +func failure(c Cmd, r Result) error { + program, _ := argv(c) + switch { + case r.Error == "not-found" && program == "sudo": + return fmt.Errorf("%s needs root, and sudo is not installed here for the runtime's account to escalate with", c.Name) + case r.Error == "not-found": + if hint, ok := providedBy[c.Name]; ok { + return fmt.Errorf("%s is not installed on this machine (%s)", c.Name, hint) + } + return fmt.Errorf("%s is not installed on this machine", c.Name) + case r.Error == "timeout": + limit := c.Timeout + if limit <= 0 { + limit = CallTimeout + } + return fmt.Errorf("%s gave no answer within %s and was ended", c.Name, limit) + case r.Error != "": + return fmt.Errorf("%s did not run: %s", c.Name, r.Error) + case program == "sudo" && strings.Contains(r.Stderr, "command not found"): + if hint, ok := providedBy[c.Name]; ok { + return fmt.Errorf("%s is not installed on this machine (%s)", c.Name, hint) + } + return fmt.Errorf("%s is not installed on this machine", c.Name) + case program == "sudo" && strings.HasPrefix(strings.TrimSpace(r.Stderr), "sudo:"): + return fmt.Errorf("%s needs root, and sudo -n refused the runtime's account: %s (the escalation is the sudo module's to declare)", + c.Name, firstLine(r.Stderr)) + } + said := tail(strings.TrimSpace(r.Stderr), 2000) + if said == "" { + said = tail(strings.TrimSpace(r.Stdout), 2000) + } + if said == "" { + said = "and said nothing" + } + return fmt.Errorf("%s %s exited %d: %s", c.Name, strings.Join(c.Args, " "), r.Status, said) +} + +func firstLine(s string) string { + s = strings.TrimSpace(s) + if i := strings.IndexByte(s, '\n'); i >= 0 { + return s[:i] + } + return s +} + +func tail(s string, n int) string { + if len(s) <= n { + return s + } + return "…" + s[len(s)-n:] +} + +// lines are a command's output lines, blank ones dropped. +func lines(s string) []string { + out := []string{} + for _, l := range strings.Split(s, "\n") { + if strings.TrimSpace(l) != "" { + out = append(out, strings.TrimRight(l, "\r")) + } + } + return out +} + +// Arguments, read the way a tool's JSON arguments arrive. + +func text(args map[string]any, key string) (string, error) { + v, ok := args[key] + if !ok || v == nil { + return "", fmt.Errorf("%s is required", key) + } + s, ok := v.(string) + if !ok { + return "", fmt.Errorf("%s must be a string", key) + } + if strings.TrimSpace(s) == "" { + return "", fmt.Errorf("%s must not be empty", key) + } + return s, nil +} + +func optText(args map[string]any, key, def string) (string, error) { + v, ok := args[key] + if !ok || v == nil { + return def, nil + } + s, ok := v.(string) + if !ok { + return "", fmt.Errorf("%s must be a string", key) + } + if strings.TrimSpace(s) == "" { + return def, nil + } + return s, nil +} + +// optWhole reads a whole number, defaulted, refused below least and held to most. +func optWhole(args map[string]any, key string, def, least, most int) (int, error) { + v, ok := args[key] + if !ok || v == nil { + return def, nil + } + f, ok := v.(float64) + if !ok { + if i, isInt := v.(int); isInt { + f = float64(i) + } else { + return 0, fmt.Errorf("%s must be a number", key) + } + } + if f != float64(int(f)) { + return 0, fmt.Errorf("%s must be a whole number", key) + } + n := int(f) + if n < least { + return 0, fmt.Errorf("%s must be at least %d", key, least) + } + if n > most { + n = most + } + return n, nil +} + +func optFlag(args map[string]any, key string, def bool) (bool, error) { + v, ok := args[key] + if !ok || v == nil { + return def, nil + } + b, ok := v.(bool) + if !ok { + return false, fmt.Errorf("%s must be true or false", key) + } + return b, nil +} + +func optList(args map[string]any, key string) ([]string, error) { + v, ok := args[key] + if !ok || v == nil { + return nil, nil + } + items, ok := v.([]any) + if !ok { + return nil, fmt.Errorf("%s must be a list of strings", key) + } + out := make([]string, 0, len(items)) + for _, it := range items { + s, ok := it.(string) + if !ok || strings.TrimSpace(s) == "" { + return nil, fmt.Errorf("%s must be a list of non-empty strings", key) + } + out = append(out, s) + } + return out, nil +} + +// oneOf refuses a value outside a closed set. +func oneOf(key, value string, allowed ...string) error { + for _, a := range allowed { + if value == a { + return nil + } + } + return fmt.Errorf("%s must be one of %s, not %q", key, strings.Join(allowed, ", "), value) +} + +// plainName refuses a name that could be read as an option or carries a path or a space: package, +// snap, application and printer names never do. +func plainName(key, value string) error { + if strings.HasPrefix(value, "-") || strings.ContainsAny(value, " \t\n/\\") { + return fmt.Errorf("%s %q is not a plain name", key, value) + } + return nil +} diff --git a/modules/flatpak/cmd/flatpak-tools/kit_test.go b/modules/flatpak/cmd/flatpak-tools/kit_test.go new file mode 100644 index 0000000..c5d3557 --- /dev/null +++ b/modules/flatpak/cmd/flatpak-tools/kit_test.go @@ -0,0 +1,147 @@ +package main + +// Tests of kit.go, the same in each workstation module. + +import ( + "strings" + "testing" + "time" +) + +// fake records the commands asked and answers each from a function of the command line. +type fake struct { + asked []Cmd + answer func(line string, c Cmd) Result +} + +func (f *fake) runner() Runner { + return func(c Cmd) Result { + f.asked = append(f.asked, c) + name, args := argv(c) + line := strings.TrimSpace(name + " " + strings.Join(args, " ")) + if f.answer == nil { + return Result{} + } + return f.answer(line, c) + } +} + +func (f *fake) lines() []string { + out := []string{} + for _, c := range f.asked { + name, args := argv(c) + out = append(out, strings.TrimSpace(name+" "+strings.Join(args, " "))) + } + return out +} + +// using installs a fake runner and a non-root uid for one test. +func using(t *testing.T, answer func(line string, c Cmd) Result) *fake { + t.Helper() + f := &fake{answer: answer} + wasRun, wasUID := run, euid + run, euid = f.runner(), func() int { return 1000 } + t.Cleanup(func() { run, euid = wasRun, wasUID }) + return f +} + +func ok(stdout string) Result { return Result{Stdout: stdout} } + +func TestKitAnActThatNeedsRootGoesThroughSudoWithoutAPromptUnlessAlreadyRoot(t *testing.T) { + was := euid + defer func() { euid = was }() + euid = func() int { return 1000 } + if name, args := argv(Cmd{Name: "x", Args: []string{"a"}, Root: true}); name != "sudo" || strings.Join(args, " ") != "-n x a" { + t.Fatalf("not root: %s %v", name, args) + } + if name, _ := argv(Cmd{Name: "x"}); name != "x" { + t.Fatalf("a read is run as the account: %s", name) + } + euid = func() int { return 0 } + if name, _ := argv(Cmd{Name: "x", Root: true}); name != "x" { + t.Fatalf("as root no sudo: %s", name) + } +} + +func TestKitAFailureIsNamedByHowItFailed(t *testing.T) { + was := euid + defer func() { euid = was }() + euid = func() int { return 1000 } + cases := []struct { + c Cmd + r Result + want string + }{ + {Cmd{Name: "nothere"}, Result{Status: 127, Error: "not-found"}, "not installed"}, + {Cmd{Name: "x", Root: true}, Result{Status: 127, Error: "not-found"}, "sudo is not installed"}, + {Cmd{Name: "x", Root: true}, Result{Status: 1, Stderr: "sudo: a password is required\n"}, "sudo -n refused"}, + {Cmd{Name: "x", Root: true}, Result{Status: 1, Stderr: "sudo: x: command not found\n"}, "x is not installed"}, + {Cmd{Name: "x"}, Result{Status: 124, Error: "timeout"}, "within 20s"}, + {Cmd{Name: "x", Args: []string{"y"}}, Result{Status: 3, Stderr: "boom\n"}, "x y exited 3: boom"}, + {Cmd{Name: "x"}, Result{Status: 3}, "said nothing"}, + } + for _, k := range cases { + err := failure(k.c, k.r) + if err == nil || !strings.Contains(err.Error(), k.want) { + t.Errorf("%+v: %v, want %q", k.r, err, k.want) + } + } +} + +func TestKitOutputIsBoundedAndSaysSo(t *testing.T) { + var w bounded + big := strings.Repeat("a", MostOutput+10) + n, _ := w.Write([]byte(big)) + if n != len(big) || w.b.Len() != MostOutput || !w.cut { + t.Fatalf("kept %d of %d, cut %v", w.b.Len(), len(big), w.cut) + } +} + +func TestKitTheRealRunnerRunsEndsAndReportsAMissingProgram(t *testing.T) { + r := execRun(Cmd{Name: "sh", Args: []string{"-c", "echo out; echo err >&2; exit 3"}}) + if r.Status != 3 || strings.TrimSpace(r.Stdout) != "out" || strings.TrimSpace(r.Stderr) != "err" { + t.Fatalf("%+v", r) + } + r = execRun(Cmd{Name: "sh", Args: []string{"-c", "sleep 5 & sleep 5"}, Timeout: 200 * time.Millisecond}) + if r.Error != "timeout" { + t.Fatalf("a slow command: %+v", r) + } + r = execRun(Cmd{Name: "no-such-program-anywhere"}) + if r.Error != "not-found" { + t.Fatalf("a missing program: %+v", r) + } + r = execRun(Cmd{Name: "cat", Stdin: "given"}) + if r.Stdout != "given" { + t.Fatalf("stdin: %+v", r) + } + start := time.Now() + r = execRun(Cmd{Name: "sh", Args: []string{"-c", "echo kept; (sleep 3 &) ; exit 0"}, Detached: true}) + if r.Status != 0 || strings.TrimSpace(r.Stdout) != "kept" || time.Since(start) > 2*time.Second { + t.Fatalf("a detached command returns when it exits, not when its child does: %+v after %s", r, time.Since(start)) + } +} + +func TestKitArgumentsAreReadStrictly(t *testing.T) { + args := map[string]any{"s": "x", "n": float64(5), "f": 1.5, "b": true, "l": []any{"a", "b"}} + if _, err := text(args, "missing"); err == nil { + t.Error("a missing required string") + } + if n, _ := optWhole(args, "n", 1, 1, 3); n != 3 { + t.Errorf("held to most: %d", n) + } + if _, err := optWhole(args, "n", 1, 6, 9); err == nil { + t.Error("below least") + } + if _, err := optWhole(args, "f", 1, 0, 9); err == nil { + t.Error("a fraction") + } + if l, _ := optList(args, "l"); len(l) != 2 { + t.Errorf("list: %v", l) + } + if b, _ := optFlag(args, "b", false); !b { + t.Error("flag") + } + if err := plainName("name", "--all"); err == nil { + t.Error("an option as a name") + } +} diff --git a/modules/flatpak/cmd/flatpak-tools/main.go b/modules/flatpak/cmd/flatpak-tools/main.go new file mode 100644 index 0000000..b5278dd --- /dev/null +++ b/modules/flatpak/cmd/flatpak-tools/main.go @@ -0,0 +1,213 @@ +// The flatpak module's tools (novox/hq research 027/02, 026/05): the applications and runtimes in +// both installations, the remotes, pending updates, what nothing uses any more and the space it all +// takes; and installing, removing and updating. A Go bundle the node's runtime launches over stdio +// (ADR 0188, ADR 0193); it runs as the operator account. An act on the system installation goes +// through `sudo -n`; one on the account's own installation does not. +package main + +import ( + "fmt" + "os" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +var providedBy = map[string]string{ + "flatpak": "the flatpak package, which this module installs", + "du": "the coreutils package", +} + +func main() { + if err := stdio.Serve("", tools()); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +var installationArg = map[string]any{"type": "string", "enum": []string{"system", "user"}, + "description": "the system installation (default), or the account's own"} + +func installationOf(args map[string]any, def string) (string, error) { + i, err := optText(args, "installation", def) + if err != nil { + return "", err + } + if i == "" { + return "", nil + } + return i, oneOf("installation", i, "system", "user") +} + +func refArg(args map[string]any, key string) (string, error) { + r, err := text(args, key) + if err != nil { + return "", err + } + return r, checkRef(r) +} + +func tools() []stdio.Tool { + return []stdio.Tool{ + { + Name: "flatpak_list", + Description: "The installed applications, in both installations or one: id, name, version, branch, origin, " + + "installation and size. (r)", + Input: map[string]any{"installation": installationArg}, + Run: func(args map[string]any) (any, error) { + inst, err := installationOf(args, "") + if err != nil { + return nil, err + } + return List("app", inst) + }, + }, + { + Name: "flatpak_runtimes", + Description: "The installed runtimes and extensions, in both installations or one: id, branch, origin, installation and size. (r)", + Input: map[string]any{"installation": installationArg}, + Run: func(args map[string]any) (any, error) { + inst, err := installationOf(args, "") + if err != nil { + return nil, err + } + return List("runtime", inst) + }, + }, + { + Name: "flatpak_remotes", + Description: "The remotes of both installations, with how many installed refs come from each, and findings: " + + "Flathub missing from the system installation, the same remote in both installations, a remote nothing " + + "is installed from. (r)", + Input: map[string]any{}, + Run: func(map[string]any) (any, error) { return Remotes() }, + }, + { + Name: "flatpak_updates", + Description: "What an update would change: each ref with a newer commit on its remote. Asks the remotes, so it needs the network. (r)", + Input: map[string]any{"installation": installationArg}, + Run: func(args map[string]any) (any, error) { + inst, err := installationOf(args, "") + if err != nil { + return nil, err + } + return Updates(inst) + }, + }, + { + Name: "flatpak_unused", + Description: "The runtimes and extensions no installed application needs any more, computed without changing " + + "anything: a runtime is used when an application names it as its runtime or SDK, when it fills an " + + "extension point of something used, or when it is pinned. With the space each takes. (r)", + Input: map[string]any{}, + Run: func(map[string]any) (any, error) { return Unused() }, + }, + { + Name: "flatpak_disk_usage", + Description: "The space flatpak takes: each installation's directory on disk, and the applications and " + + "runtimes by size, largest first. (r)", + Input: map[string]any{}, + Run: func(map[string]any) (any, error) { return DiskUsage() }, + }, + { + Name: "flatpak_install", + Description: "Install an application or runtime from a remote (default flathub), in the system installation " + + "(default) or the account's. Answers when finished, or after 18 s with a job to follow with flatpak_job. (a)", + Input: map[string]any{ + "ref": map[string]any{"type": "string", "description": "an application id such as org.gimp.GIMP, or a full ref"}, + "remote": map[string]any{"type": "string", "description": "the remote (default flathub)"}, + "installation": installationArg, + }, + Run: func(args map[string]any) (any, error) { + ref, err := refArg(args, "ref") + if err != nil { + return nil, err + } + remote, err := optText(args, "remote", "flathub") + if err != nil { + return nil, err + } + inst, err := installationOf(args, "system") + if err != nil { + return nil, err + } + return Install(ref, remote, inst) + }, + }, + { + Name: "flatpak_remove", + Description: "Uninstall an application or runtime, keeping its data unless delete_data is set. Answers when finished, or with a job to follow. (a)", + Input: map[string]any{ + "ref": map[string]any{"type": "string", "description": "the application id or ref"}, + "installation": installationArg, + "delete_data": map[string]any{"type": "boolean", "description": "remove the application's data in the home too"}, + }, + Run: func(args map[string]any) (any, error) { + ref, err := refArg(args, "ref") + if err != nil { + return nil, err + } + inst, err := installationOf(args, "system") + if err != nil { + return nil, err + } + del, err := optFlag(args, "delete_data", false) + if err != nil { + return nil, err + } + return Remove(ref, inst, del) + }, + }, + { + Name: "flatpak_update", + Description: "Update one ref, or everything in an installation when no ref is given. Answers when finished, or with a job to follow. (a)", + Input: map[string]any{ + "ref": map[string]any{"type": "string", "description": "the application id or ref (default all)"}, + "installation": installationArg, + }, + Run: func(args map[string]any) (any, error) { + ref, err := optText(args, "ref", "") + if err != nil { + return nil, err + } + if ref != "" { + if err := checkRef(ref); err != nil { + return nil, err + } + } + inst, err := installationOf(args, "system") + if err != nil { + return nil, err + } + return Update(ref, inst) + }, + }, + { + Name: "flatpak_remove_unused", + Description: "Uninstall what flatpak itself finds unused in one installation (system by default). " + + "flatpak_unused shows what that is beforehand. Answers when finished, or with a job to follow. (a)", + Input: map[string]any{"installation": installationArg}, + Run: func(args map[string]any) (any, error) { + inst, err := installationOf(args, "system") + if err != nil { + return nil, err + } + return RemoveUnused(inst) + }, + }, + { + Name: "flatpak_job", + Description: "A long act this module started: running or finished, its exit status and the end of its output. Without job, every act this process knows. (r)", + Input: map[string]any{"job": map[string]any{"type": "string", "description": "the job id an act answered"}}, + Run: func(args map[string]any) (any, error) { + id, err := optText(args, "job", "") + if err != nil { + return nil, err + } + if id == "" { + return map[string]any{"jobs": listJobs()}, nil + } + return jobByID(id) + }, + }, + } +} diff --git a/modules/flatpak/cmd/flatpak-tools/manifest_kit_test.go b/modules/flatpak/cmd/flatpak-tools/manifest_kit_test.go new file mode 100644 index 0000000..3e675b4 --- /dev/null +++ b/modules/flatpak/cmd/flatpak-tools/manifest_kit_test.go @@ -0,0 +1,107 @@ +package main + +// manifest_kit_test.go is the same file in each workstation module: it reads the module's +// definition so the module's own tests can hold it to what it says. + +import ( + "encoding/json" + "os" + "path/filepath" + "sort" + "strings" + "testing" +) + +type manifest struct { + Module string `json:"module"` + Capabilities []string `json:"capabilities"` + Claims []any `json:"claims"` + Seats []any `json:"seats"` + Tools []string `json:"tools"` + Resources []map[string]any `json:"resources"` + Build struct { + Artifacts []map[string]any `json:"artifacts"` + } `json:"build"` +} + +func readManifest(t *testing.T) manifest { + t.Helper() + raw, err := os.ReadFile(filepath.Join("..", "..", "module.json")) + if err != nil { + t.Fatal(err) + } + var m manifest + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatalf("module.json: %v", err) + } + return m +} + +func (m manifest) resource(id string) map[string]any { + for _, r := range m.Resources { + if r["id"] == id { + return r + } + } + return nil +} + +// packages are the packages the module installs, sorted. +func (m manifest) packages() []string { + out := []string{} + for _, r := range m.Resources { + if r["type"] == "package" && r["absent"] != true { + out = append(out, r["package"].(string)) + } + } + sort.Strings(out) + return out +} + +// services are the units the module declares, by unit name. +func (m manifest) services() map[string]map[string]any { + out := map[string]map[string]any{} + for _, r := range m.Resources { + if r["type"] == "service" { + out[r["unit"].(string)] = r + } + } + return out +} + +// holdsTheBundle holds the manifest to the Go bundle this directory builds: every tool registered +// is listed and nothing else, each named _…, and the artifact builds this command. +func holdsTheBundle(t *testing.T, m manifest, prefix string) { + t.Helper() + registered := []string{} + for _, tool := range tools() { + registered = append(registered, tool.Name) + if !strings.HasPrefix(tool.Name, prefix+"_") { + t.Errorf("tool %s is not named %s_…", tool.Name, prefix) + } + if tool.Description == "" || tool.Run == nil || tool.Input == nil { + t.Errorf("tool %s is not described, runnable and given an input schema", tool.Name) + } + } + if strings.Join(registered, ",") != strings.Join(m.Tools, ",") { + t.Errorf("registered %v, listed %v", registered, m.Tools) + } + if len(m.Build.Artifacts) != 1 { + t.Fatalf("one artifact, got %d", len(m.Build.Artifacts)) + } + cwd, _ := os.Getwd() + binary := filepath.Base(cwd) + a := m.Build.Artifacts[0] + want := map[string]any{"kind": "bundle", "language": "go", "system": "arch", "from": "cmd/" + binary, "binary": binary} + for k, v := range want { + if a[k] != v { + t.Errorf("artifact %s = %v, want %v", k, a[k], v) + } + } + if loads, _ := a["loads"].([]any); len(loads) != 1 || loads[0] != binary { + t.Errorf("artifact loads %v, want [%s]", a["loads"], binary) + } + if m.Claims != nil || m.Seats != nil { + t.Errorf("claims %v, seats %v: this module holds no seat", m.Claims, m.Seats) + } +} diff --git a/modules/flatpak/go.mod b/modules/flatpak/go.mod new file mode 100644 index 0000000..7f34546 --- /dev/null +++ b/modules/flatpak/go.mod @@ -0,0 +1,5 @@ +module flatpak + +go 1.22 + +require git.novox.be/novox/mesh-sdk/go v0.1.6 diff --git a/modules/flatpak/go.sum b/modules/flatpak/go.sum new file mode 100644 index 0000000..0dd6061 --- /dev/null +++ b/modules/flatpak/go.sum @@ -0,0 +1,2 @@ +git.novox.be/novox/mesh-sdk/go v0.1.6 h1:9qzdYONYbJdWcu6sxQcq9v1LI0JxcfkiKYkMUzJSkVQ= +git.novox.be/novox/mesh-sdk/go v0.1.6/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY= diff --git a/modules/flatpak/module.json b/modules/flatpak/module.json new file mode 100644 index 0000000..76d4721 --- /dev/null +++ b/modules/flatpak/module.json @@ -0,0 +1,42 @@ +{ + "module": "flatpak", + "version": "1", + "capabilities": [ + "package-manager" + ], + "tools": [ + "flatpak_list", + "flatpak_runtimes", + "flatpak_remotes", + "flatpak_updates", + "flatpak_unused", + "flatpak_disk_usage", + "flatpak_install", + "flatpak_remove", + "flatpak_update", + "flatpak_remove_unused", + "flatpak_job" + ], + "resources": [ + { + "id": "package", + "type": "package", + "package": "flatpak" + } + ], + "build": { + "artifacts": [ + { + "name": "tools", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/flatpak-tools", + "binary": "flatpak-tools", + "loads": [ + "flatpak-tools" + ] + } + ] + } +}