From 62437f8964128f1e88d09be947095cdfc7173b4a Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 14:36:11 +0200 Subject: [PATCH 1/3] issue 1840/S8: answer the package view from the bus, so every node is read at once and none is asked pacman_packages answers one package on every node, or each node's counts, its explicit packages no module declares and its foreign ones apart, the versions that differ and the packages on some nodes only, with a node over the bus's cap marked not current. pacman_updates, pacman_installed and pacman_foreign read the keeper's states instead of the machine; pacman_updates says when each node's list last changed. --- modules/pacman/cmd/pacman-tools/main.go | 81 +++- .../pacman/cmd/pacman-tools/pacman_test.go | 39 +- modules/pacman/cmd/pacman-tools/query.go | 90 +---- modules/pacman/cmd/pacman-tools/view.go | 377 ++++++++++++++++++ modules/pacman/cmd/pacman-tools/view_test.go | 160 ++++++++ modules/pacman/module.json | 1 + 6 files changed, 627 insertions(+), 121 deletions(-) create mode 100644 modules/pacman/cmd/pacman-tools/view.go create mode 100644 modules/pacman/cmd/pacman-tools/view_test.go diff --git a/modules/pacman/cmd/pacman-tools/main.go b/modules/pacman/cmd/pacman-tools/main.go index 931d625f..f6ce4d70 100644 --- a/modules/pacman/cmd/pacman-tools/main.go +++ b/modules/pacman/cmd/pacman-tools/main.go @@ -10,6 +10,7 @@ import ( "context" "fmt" "os" + "regexp" stdio "git.novox.be/novox/mesh-sdk/go" ) @@ -37,8 +38,22 @@ var ( return map[string]any{"type": "integer", "description": fmt.Sprintf("at most this many (default %d, at most %d)", def, most)} } sinceArg = map[string]any{"type": "string", "description": "a day, YYYY-MM-DD"} + nodeArg = map[string]any{"type": "string", "description": "a node's name"} + nodeName = regexp.MustCompile(`^[A-Za-z0-9_-]+$`) ) +// nodeOf is the node argument, empty when absent; a node's name is one key token. +func nodeOf(args map[string]any) (string, error) { + n, err := text(args, "node", false) + if err != nil { + return "", err + } + if n != "" && !nodeName.MatchString(n) { + return "", fmt.Errorf("node %q is not a node's name", n) + } + return n, nil +} + func tools(m *Machine) []stdio.Tool { return []stdio.Tool{ { @@ -70,15 +85,22 @@ func tools(m *Machine) []stdio.Tool { }, }, { - Name: "pacman_installed", - Description: "Installed packages with version, why each is installed (explicit or dependency) and whether it is foreign (in no repository); narrowed by name, reason or foreign; with totals.", + Name: "pacman_installed", + Description: "A node's installed packages, from the bus (ADR 0507 §2): version, why each is installed (explicit or dependency), its repository, " + + "or foreign (in no repository); narrowed by name, reason or foreign; with totals, the time of the transaction they describe and whether they are current. " + + "This node's by default. (r)", Input: schema(map[string]any{ + "node": nodeArg, "match": map[string]any{"type": "string", "description": "only names holding this"}, "reason": map[string]any{"type": "string", "enum": []string{"explicit", "dependency"}}, "foreign": map[string]any{"type": "boolean", "description": "only foreign packages"}, "limit": limitArg(5000, 20000), }), Run: func(args map[string]any) (any, error) { + node, err := nodeOf(args) + if err != nil { + return nil, err + } match, err := text(args, "match", false) if err != nil { return nil, err @@ -98,7 +120,29 @@ func tools(m *Machine) []stdio.Tool { if err != nil { return nil, err } - return m.Installed(match, reason, foreign, n) + return m.InstalledFrom(node, match, reason, foreign, n) + }, + }, + { + Name: "pacman_packages", + Description: "Every node's packages at once, from the bus (ADR 0507 §6): with package, that package on every node (version, reason, repository " + + "or foreign, the modules declaring it); without, per node its counts, the packages installed explicitly that no module assigned to it declares, " + + "and its foreign packages apart, then the packages whose versions differ between nodes and the packages on some nodes only. " + + "A node whose packages are over the bus's cap is marked not current. No machine is asked. (r)", + Input: schema(map[string]any{ + "package": map[string]any{"type": "string", "description": "one package, on every node"}, + "limit": limitArg(500, 20000), + }), + Run: func(args map[string]any) (any, error) { + p, err := text(args, "package", false) + if err != nil { + return nil, err + } + n, err := whole(args, "limit", 500, 1, 20000) + if err != nil { + return nil, err + } + return m.Packages(p, n) }, }, { @@ -130,10 +174,18 @@ func tools(m *Machine) []stdio.Tool { }, }, { - Name: "pacman_updates", - Description: "What a full upgrade would change, each package from and to (checkupdates: the repositories are asked into a copy of their databases, so asking never sets up a partial upgrade). Needs the network.", - Input: schema(map[string]any{}), - Run: func(map[string]any) (any, error) { return m.Updates() }, + Name: "pacman_updates", + Description: "What a full upgrade would change, from the bus (ADR 0507 §4), on one node or on every node: each package from and to, whether the " + + "node reached its mirrors (false: the list is the last it could confirm), and when the node's list last changed. The keeper reads it with " + + "checkupdates at its start, after every transaction, after every mirror refresh and once a day. (r)", + Input: schema(map[string]any{"node": nodeArg}), + Run: func(args map[string]any) (any, error) { + node, err := nodeOf(args) + if err != nil { + return nil, err + } + return m.UpdatesFor(node) + }, }, { Name: "pacman_upgrade", @@ -295,10 +347,17 @@ func tools(m *Machine) []stdio.Tool { }, }, { - Name: "pacman_foreign", - Description: "Installed packages that no repository this machine syncs carries (pacman -Qm): built from the AUR or by hand, which the mesh cannot install.", - Input: schema(map[string]any{}), - Run: func(map[string]any) (any, error) { return m.Foreign() }, + Name: "pacman_foreign", + Description: "A node's packages that no repository it syncs carries, from the bus: built from the AUR or by hand, which the mesh neither " + + "upgrades nor removes (ADR 0507 §7). This node's by default. (r)", + Input: schema(map[string]any{"node": nodeArg}), + Run: func(args map[string]any) (any, error) { + node, err := nodeOf(args) + if err != nil { + return nil, err + } + return m.ForeignFrom(node) + }, }, { Name: "pacman_news", diff --git a/modules/pacman/cmd/pacman-tools/pacman_test.go b/modules/pacman/cmd/pacman-tools/pacman_test.go index 4ca7f5bb..d6ca6b64 100644 --- a/modules/pacman/cmd/pacman-tools/pacman_test.go +++ b/modules/pacman/cmd/pacman-tools/pacman_test.go @@ -78,31 +78,20 @@ func TestInfoReadsListsAsListsAndFallsBackToTheRepositories(t *testing.T) { } } -func TestInstalledSaysWhyAndWhatIsForeign(t *testing.T) { +func TestReadInstalledSaysWhyAndWhichRepositoryOrForeign(t *testing.T) { m := machine(byLine(map[string]Ran{ - "pacman -Q": {Stdout: "glibc 2.42-1\nyay 12.0-1\nzsh 5.9-1\n"}, + "pacman -Q": {Stdout: "zsh 5.9-1\nglibc 2.42-1\nyay 12.0-1\n"}, "pacman -Qeq": {Stdout: "yay\nzsh\n"}, - "pacman -Qmq": {Stdout: "yay\n"}, + "pacman -Sl": {Stdout: "core glibc 2.42-1 [installed]\nextra zsh 5.9-1 [installed]\nextra-testing zsh 5.9-2\n"}, }, nil), 1000) - r, err := m.Installed("", "", false, 10) - if err != nil { - t.Fatal(err) + pk, skipped, err := m.ReadInstalled() + if err != nil || skipped != 0 { + t.Fatal(err, skipped) } - pk := r["packages"].([]Package) - if pk[0].Reason != "dependency" || pk[1].Reason != "explicit" || !pk[1].Foreign || pk[2].Foreign { + want := []InstalledPackage{{"glibc", "2.42-1", false, "core"}, {"yay", "12.0-1", true, RepoForeign}, {"zsh", "5.9-1", true, "extra"}} + if !samePackages(pk, want) { t.Fatalf("%+v", pk) } - if tot := r["totals"].(map[string]int); tot["explicit"] != 2 || tot["dependency"] != 1 || tot["foreign"] != 1 { - t.Fatalf("%v", tot) - } - r, _ = m.Installed("", "", true, 10) - if r["count"] != 1 { - t.Fatalf("foreign only: %v", r) - } - r, _ = m.Installed("", "explicit", false, 1) - if r["count"] != 2 || r["truncated"] != true { - t.Fatalf("bounded: %v", r) - } } func TestOwnsAnswersNoOwnerAsAnAnswer(t *testing.T) { @@ -125,18 +114,18 @@ func TestOwnsAnswersNoOwnerAsAnAnswer(t *testing.T) { } } -func TestUpdatesReadsCheckupdatesAndItsNothingToDo(t *testing.T) { - m := machine(byLine(map[string]Ran{"checkupdates": {Stdout: "linux 6.1-1 -> 6.2-1\nzsh 5.9-1 -> 5.9-2\n"}}, nil), 1000) - r, err := m.Updates() - if err != nil || r["count"] != 2 || r["updates"].([]Update)[0] != (Update{"linux", "6.1-1", "6.2-1"}) { +func TestUpdateListReadsCheckupdatesAndItsNothingToDo(t *testing.T) { + m := machine(byLine(map[string]Ran{"checkupdates": {Stdout: "zsh 5.9-1 -> 5.9-2\nlinux 6.1-1 -> 6.2-1\n"}}, nil), 1000) + r, err := m.UpdateList() + if err != nil || len(r) != 2 || r[0] != (VersionChange{"linux", "6.1-1", "6.2-1"}) { t.Fatalf("%v %v", r, err) } m = machine(byLine(map[string]Ran{"checkupdates": {Status: 2}}, nil), 1000) - if r, err := m.Updates(); err != nil || r["count"] != 0 { + if r, err := m.UpdateList(); err != nil || r == nil || len(r) != 0 { t.Fatalf("%v %v", r, err) } m = machine(byLine(map[string]Ran{"checkupdates": {Status: 1, Stderr: "==> ERROR: Cannot fetch updates\n"}}, nil), 1000) - if _, err := m.Updates(); err == nil || !strings.Contains(err.Error(), "Cannot fetch updates") { + if _, err := m.UpdateList(); err == nil || !strings.Contains(err.Error(), "Cannot fetch updates") { t.Fatalf("%v", err) } } diff --git a/modules/pacman/cmd/pacman-tools/query.go b/modules/pacman/cmd/pacman-tools/query.go index 4c8a4f14..ae8c6626 100644 --- a/modules/pacman/cmd/pacman-tools/query.go +++ b/modules/pacman/cmd/pacman-tools/query.go @@ -8,7 +8,6 @@ import ( "fmt" "path" "regexp" - "sort" "strings" ) @@ -177,10 +176,11 @@ func (m *Machine) Info(name string) (map[string]any, error) { // Package is an installed package and why it is installed. type Package struct { - Name string `json:"name"` - Version string `json:"version"` - Reason string `json:"reason"` - Foreign bool `json:"foreign"` + Name string `json:"name"` + Version string `json:"version"` + Reason string `json:"reason"` + Repository string `json:"repository,omitempty"` + Foreign bool `json:"foreign"` } func nameVersions(out string) [][2]string { @@ -193,51 +193,6 @@ func nameVersions(out string) [][2]string { return nv } -func nameSet(out string) map[string]bool { - s := map[string]bool{} - for _, l := range lines(out) { - s[strings.TrimSpace(l)] = true - } - return s -} - -// Installed is every installed package with its version, whether it was installed explicitly or as -// a dependency, and whether it is foreign (in no repository this machine syncs). -func (m *Machine) Installed(match, reason string, foreignOnly bool, limit int) (map[string]any, error) { - all, err := m.query("-Q") - if err != nil { - return nil, err - } - explicit, err := m.query("-Qeq") - if err != nil { - return nil, err - } - foreign, err := m.query("-Qmq") - if err != nil { - return nil, err - } - ex, fo := nameSet(explicit), nameSet(foreign) - pkgs := []Package{} - counts := map[string]int{"explicit": 0, "dependency": 0, "foreign": 0} - for _, nv := range nameVersions(all) { - p := Package{Name: nv[0], Version: nv[1], Reason: "dependency", Foreign: fo[nv[0]]} - if ex[p.Name] { - p.Reason = "explicit" - } - counts[p.Reason]++ - if p.Foreign { - counts["foreign"]++ - } - if match != "" && !strings.Contains(p.Name, match) || reason != "" && p.Reason != reason || foreignOnly && !p.Foreign { - continue - } - pkgs = append(pkgs, p) - } - out := bound("packages", pkgs, limit) - out["totals"] = counts - return out, nil -} - var ownedBy = regexp.MustCompile(`^(.*) is owned by (\S+) (\S+)$`) // Owns is which package owns a path. @@ -292,41 +247,6 @@ func (m *Machine) Orphans() (map[string]any, error) { return map[string]any{"count": len(pkgs), "orphans": pkgs}, nil } -// Foreign is every installed package no repository this machine syncs carries: built from the AUR -// or by hand, which the host's `package` shape cannot install (research 027, question 1). -func (m *Machine) Foreign() (map[string]any, error) { - out, err := m.query("-Qm") - if err != nil { - return nil, err - } - pkgs := []map[string]string{} - for _, nv := range nameVersions(out) { - pkgs = append(pkgs, map[string]string{"name": nv[0], "version": nv[1]}) - } - sort.Slice(pkgs, func(i, j int) bool { return pkgs[i]["name"] < pkgs[j]["name"] }) - return map[string]any{"count": len(pkgs), "packages": pkgs}, nil -} - -// Update is one package an upgrade would change. -type Update struct { - Name string `json:"name"` - From string `json:"from"` - To string `json:"to"` -} - -// Updates is what a full upgrade would change, from checkupdates (UpdateList). -func (m *Machine) Updates() (map[string]any, error) { - list, err := m.UpdateList() - if err != nil { - return nil, err - } - ups := []Update{} - for _, u := range list { - ups = append(ups, Update{u.Name, u.From, u.To}) - } - return map[string]any{"count": len(ups), "updates": ups}, nil -} - // Config is the configuration pacman runs with, as pacman-conf resolves it. type Config struct { Options map[string][]string `json:"options"` diff --git a/modules/pacman/cmd/pacman-tools/view.go b/modules/pacman/cmd/pacman-tools/view.go new file mode 100644 index 00000000..1d7918b5 --- /dev/null +++ b/modules/pacman/cmd/pacman-tools/view.go @@ -0,0 +1,377 @@ +package main + +// The package view (novox/hq ADR 0507 §6, §7; acceptance 1840/AC6): verbs that read the module's states and the +// controller's `declared-packages`, so any instance answers for every node and no machine is asked. A node that is +// off keeps its last values, shown with the time they describe; a node whose packages are over the bus's cap is +// marked not current. A foreign package is listed apart in every answer. + +import ( + "encoding/json" + "fmt" + "sort" + "strings" + "time" +) + +// DeclaredPackagesState is the controller's state of the packages each node's modules declare (ADR 0507 §5), +// read through the manifest's `reads`. +const DeclaredPackagesState = "mesh-controller.declared-packages" + +// declaredValue is a node's value of `declared-packages`, as the controller writes it. +type declaredValue struct { + Node string `json:"node"` + Packages []struct { + Package string `json:"package"` + Modules []string `json:"modules"` + Absent bool `json:"absent"` + } `json:"packages"` +} + +// nodeInstalled is one node's key of `installed` with its not-current mark. +type nodeInstalled struct { + Node string + Value InstalledValue + NotCurrent *NotCurrent +} + +func readValue(r StateReader, key string, v any) (bool, error) { + e, err := r.Get(key) + if err != nil || e == nil { + return false, err + } + if err := json.Unmarshal(e.Value, v); err != nil { + return false, fmt.Errorf("%s is not readable: %w", key, err) + } + return true, nil +} + +// nodeKeys are the node keys of a state (``), sorted, and the marks (`.not-current`) apart. +func nodeKeys(r StateReader) (nodes []string, marks map[string]bool, err error) { + keys, err := r.Keys() + if err != nil { + return nil, nil, err + } + marks = map[string]bool{} + for _, k := range keys { + if node, ok := strings.CutSuffix(k, ".not-current"); ok { + marks[node] = true + continue + } + if !strings.Contains(k, ".") { + nodes = append(nodes, k) + } + } + sort.Strings(nodes) + return nodes, marks, nil +} + +func (m *Machine) states() error { + if m.States == nil { + return fmt.Errorf("the module's states cannot be read here") + } + return nil +} + +// installedOn is the installed packages of one node, or of every node when node is empty. +func (m *Machine) installedOn(node string) ([]nodeInstalled, error) { + if err := m.states(); err != nil { + return nil, err + } + r := m.States(StateInstalled) + nodes, marks, err := nodeKeys(r) + if err != nil { + return nil, err + } + if node != "" { + nodes = []string{node} + } + var out []nodeInstalled + for _, n := range nodes { + ni := nodeInstalled{Node: n} + found, err := readValue(r, n, &ni.Value) + if err != nil { + return nil, err + } + if !found { + if node != "" { + return nil, fmt.Errorf("no packages of %s are on the bus: its keeper has not written them", node) + } + continue + } + if marks[n] || node != "" { + var nc NotCurrent + if ok, err := readValue(r, NotCurrentKey(n), &nc); err != nil { + return nil, err + } else if ok { + ni.NotCurrent = &nc + } + } + out = append(out, ni) + } + return out, nil +} + +// declaredOn is what each node's modules declare present, by node and package, with the declaring modules; a +// node the controller holds nothing for is missing. +func (m *Machine) declaredOn(nodes []string) (map[string]map[string][]string, error) { + r := m.States(DeclaredPackagesState) + out := map[string]map[string][]string{} + for _, n := range nodes { + var v declaredValue + found, err := readValue(r, n, &v) + if err != nil { + return nil, err + } + if !found { + continue + } + out[n] = map[string][]string{} + for _, p := range v.Packages { + if !p.Absent { + out[n][p.Package] = p.Modules + } + } + } + return out, nil +} + +// NodeView is one node in the view. +type NodeView struct { + Node string `json:"node"` + // At is the time of the transaction that left the node's packages as shown. + At time.Time `json:"at"` + Packages int `json:"packages"` + Explicit int `json:"explicit"` + // Current is false while the node's packages are over the bus's cap: what is shown is older. + Current bool `json:"current"` + NotCurrent *NotCurrent `json:"not_current,omitempty"` + // Undeclared is the packages installed explicitly from a repository that no module assigned to the node + // declares; null when the controller holds no declaration of the node. + Undeclared []PackageVersion `json:"undeclared"` + Foreign []PackageVersion `json:"foreign"` +} + +// Spread is one package across the nodes: its version on each node that has it. +type Spread struct { + Name string `json:"name"` + Versions map[string]string `json:"versions"` +} + +// PackageOnNode is one package on one node. +type PackageOnNode struct { + Node string `json:"node"` + Installed bool `json:"installed"` + Version string `json:"version,omitempty"` + Explicit bool `json:"explicit,omitempty"` + Repository string `json:"repository,omitempty"` + Foreign bool `json:"foreign,omitempty"` + DeclaredBy []string `json:"declared_by"` + Current bool `json:"current"` +} + +// Packages is pacman_packages: one package on every node, or the whole view: per node its counts, its explicit +// undeclared packages and its foreign ones; the packages whose versions differ between nodes, and the packages +// on some nodes only, among the packages from a repository (each node's foreign ones are in its own list). +func (m *Machine) Packages(name string, limit int) (map[string]any, error) { + all, err := m.installedOn("") + if err != nil { + return nil, err + } + nodes := make([]string, 0, len(all)) + for _, ni := range all { + nodes = append(nodes, ni.Node) + } + declared, err := m.declaredOn(nodes) + if err != nil { + return nil, err + } + if name != "" { + on := []PackageOnNode{} + for _, ni := range all { + p := PackageOnNode{Node: ni.Node, Current: ni.NotCurrent == nil, DeclaredBy: declared[ni.Node][name]} + if p.DeclaredBy == nil { + p.DeclaredBy = []string{} + } + for _, ip := range ni.Value.Packages { + if ip.Name == name { + p.Installed, p.Version, p.Explicit = true, ip.Version, ip.Explicit + p.Foreign = ip.Repo == RepoForeign + if !p.Foreign { + p.Repository = ip.Repo + } + } + } + on = append(on, p) + } + return map[string]any{"package": name, "nodes": on}, nil + } + views := []NodeView{} + versions := map[string]map[string]string{} + for _, ni := range all { + v := NodeView{Node: ni.Node, At: ni.Value.At, Packages: len(ni.Value.Packages), Current: ni.NotCurrent == nil, + NotCurrent: ni.NotCurrent, Foreign: []PackageVersion{}} + d, known := declared[ni.Node] + if known { + v.Undeclared = []PackageVersion{} + } + for _, p := range ni.Value.Packages { + if p.Explicit { + v.Explicit++ + } + if p.Repo == RepoForeign { + v.Foreign = append(v.Foreign, PackageVersion{p.Name, p.Version}) + continue + } + if versions[p.Name] == nil { + versions[p.Name] = map[string]string{} + } + versions[p.Name][ni.Node] = p.Version + if known && p.Explicit && d[p.Name] == nil { + v.Undeclared = append(v.Undeclared, PackageVersion{p.Name, p.Version}) + } + } + views = append(views, v) + } + names := make([]string, 0, len(versions)) + for n := range versions { + names = append(names, n) + } + sort.Strings(names) + differing, some := []Spread{}, []Spread{} + for _, n := range names { + vs := versions[n] + if len(vs) < len(all) { + some = append(some, Spread{n, vs}) + } + seen := map[string]bool{} + for _, ver := range vs { + seen[ver] = true + } + if len(seen) > 1 { + differing = append(differing, Spread{n, vs}) + } + } + out := map[string]any{"nodes": views} + for key, list := range map[string][]Spread{"differing": differing, "some_nodes": some} { + b := bound(key, list, limit) + out[key] = b[key] + out[key+"_count"] = b["count"] + out[key+"_truncated"] = b["truncated"] + } + return out, nil +} + +// UpdatesView is one node's `updates` with the time its key last changed. +type UpdatesView struct { + Node string `json:"node"` + Reachable bool `json:"reachable"` + Count int `json:"count"` + Packages []VersionChange `json:"packages"` + Changed *time.Time `json:"changed,omitempty"` +} + +// UpdatesFor is pacman_updates: what a full upgrade would change, on one node or on every node, each with when its +// key last changed (the bus's time of the write; the value holds no time of a read, ADR 0289 §4). +func (m *Machine) UpdatesFor(node string) (map[string]any, error) { + if err := m.states(); err != nil { + return nil, err + } + r := m.States(StateUpdates) + nodes := []string{node} + if node == "" { + var err error + if nodes, _, err = nodeKeys(r); err != nil { + return nil, err + } + } + views := []UpdatesView{} + for _, n := range nodes { + var v UpdatesValue + found, err := readValue(r, n, &v) + if err != nil { + return nil, err + } + if !found { + if node != "" { + return nil, fmt.Errorf("no updates of %s are on the bus: its keeper has not written them", node) + } + continue + } + uv := UpdatesView{Node: n, Reachable: v.Reachable, Count: len(v.Packages), Packages: v.Packages} + if uv.Packages == nil { + uv.Packages = []VersionChange{} + } + if h, err := r.History(n); err == nil && len(h) > 0 { + at := h[len(h)-1].At + uv.Changed = &at + } + views = append(views, uv) + } + return map[string]any{"nodes": views}, nil +} + +// InstalledFrom is pacman_installed from the bus: a node's installed packages with version, reason, repository +// and whether foreign, narrowed by name, reason or foreign, with totals, the time they describe and whether +// they are current. +func (m *Machine) InstalledFrom(node, match, reason string, foreignOnly bool, limit int) (map[string]any, error) { + ni, err := m.oneNode(node) + if err != nil { + return nil, err + } + pkgs := []Package{} + counts := map[string]int{"explicit": 0, "dependency": 0, "foreign": 0} + for _, ip := range ni.Value.Packages { + p := Package{Name: ip.Name, Version: ip.Version, Reason: "dependency", Foreign: ip.Repo == RepoForeign} + if ip.Explicit { + p.Reason = "explicit" + } + if !p.Foreign { + p.Repository = ip.Repo + } + counts[p.Reason]++ + if p.Foreign { + counts["foreign"]++ + } + if match != "" && !strings.Contains(p.Name, match) || reason != "" && p.Reason != reason || foreignOnly && !p.Foreign { + continue + } + pkgs = append(pkgs, p) + } + out := bound("packages", pkgs, limit) + out["totals"] = counts + out["node"], out["at"], out["current"] = ni.Node, ni.Value.At, ni.NotCurrent == nil + if ni.NotCurrent != nil { + out["not_current"] = ni.NotCurrent + } + return out, nil +} + +// ForeignFrom is pacman_foreign from the bus: a node's packages from outside the repositories. +func (m *Machine) ForeignFrom(node string) (map[string]any, error) { + ni, err := m.oneNode(node) + if err != nil { + return nil, err + } + pkgs := []PackageVersion{} + for _, ip := range ni.Value.Packages { + if ip.Repo == RepoForeign { + pkgs = append(pkgs, PackageVersion{ip.Name, ip.Version}) + } + } + return map[string]any{"node": ni.Node, "count": len(pkgs), "packages": pkgs, "at": ni.Value.At, "current": ni.NotCurrent == nil}, nil +} + +// oneNode is the named node's packages, this node's when none is named. +func (m *Machine) oneNode(node string) (nodeInstalled, error) { + if node == "" { + node = m.Node + } + if node == "" { + return nodeInstalled{}, fmt.Errorf("node is required: this bundle does not know its own") + } + all, err := m.installedOn(node) + if err != nil { + return nodeInstalled{}, err + } + return all[0], nil +} diff --git a/modules/pacman/cmd/pacman-tools/view_test.go b/modules/pacman/cmd/pacman-tools/view_test.go new file mode 100644 index 00000000..8a823c55 --- /dev/null +++ b/modules/pacman/cmd/pacman-tools/view_test.go @@ -0,0 +1,160 @@ +package main + +// The package view over the bus (novox/hq ADR 0507 §6, §7; acceptance 1840/AC6): two nodes' `installed` keys and +// their `declared-packages`, answered without asking any machine. + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "testing" + "time" +) + +// viewStates is two nodes' packages on the bus: node-a current and declared, node-b over the cap since a while, +// with a foreign package, and no declaration the controller holds. +func viewStates(t *testing.T) (*fakeStates, *Machine) { + t.Helper() + states := &fakeStates{} + inst := states.store(StateInstalled) + inst.Put("node-a", InstalledValue{At: t0, Packages: []InstalledPackage{ + {"glibc", "2.42-1", false, "core"}, {"tmux", "3.5-1", true, "extra"}, {"vim", "9.1-1", true, "extra"}, + {"zsh", "5.9-2", true, "extra"}, + }}) + inst.Put("node-b", InstalledValue{At: t0.Add(-time.Hour), Packages: []InstalledPackage{ + {"glibc", "2.42-1", false, "core"}, {"yay", "12.0-1", true, RepoForeign}, {"zsh", "5.9-1", false, "extra"}, + }}) + inst.Put("node-b.not-current", NotCurrent{Bytes: 300000, Since: t0.Add(-30 * time.Minute)}) + decl := states.store(DeclaredPackagesState) + var d declaredValue + json.Unmarshal([]byte(`{"node":"node-a","packages":[{"package":"vim","modules":["editor"],"absent":false},`+ + `{"package":"tmux","modules":["shell"],"absent":true},{"package":"zsh","modules":["login","shell"],"absent":false}]}`), &d) + decl.Put("node-a", d) + m := machine(func(_ context.Context, name string, args ...string) Ran { + t.Errorf("a machine was asked: %s %v", name, args) + return Ran{Status: 99} + }, 1000) + m.Node, m.States = "node-a", readerOf(states) + return states, m +} + +func TestOnePackageOnEveryNodeWithWhoDeclaresIt(t *testing.T) { + _, m := viewStates(t) + r, err := m.Packages("zsh", 500) + if err != nil { + t.Fatal(err) + } + on := r["nodes"].([]PackageOnNode) + if len(on) != 2 { + t.Fatalf("%+v", on) + } + a, b := on[0], on[1] + if a.Node != "node-a" || !a.Installed || a.Version != "5.9-2" || !a.Explicit || a.Repository != "extra" || + strings.Join(a.DeclaredBy, ",") != "login,shell" || !a.Current { + t.Errorf("node-a: %+v", a) + } + if b.Node != "node-b" || b.Version != "5.9-1" || b.Explicit || len(b.DeclaredBy) != 0 || b.Current { + t.Errorf("node-b: %+v", b) + } + r, _ = m.Packages("yay", 500) + on = r["nodes"].([]PackageOnNode) + if on[0].Installed || !on[1].Installed || !on[1].Foreign || on[1].Repository != "" { + t.Errorf("yay: %+v", on) + } +} + +func TestTheViewSaysWhatDiffersWhatIsOnSomeNodesWhatNoModuleDeclaresAndTheForeignApart(t *testing.T) { + _, m := viewStates(t) + r, err := m.Packages("", 500) + if err != nil { + t.Fatal(err) + } + nodes := r["nodes"].([]NodeView) + if len(nodes) != 2 { + t.Fatalf("%+v", nodes) + } + a, b := nodes[0], nodes[1] + if a.Node != "node-a" || a.Packages != 4 || a.Explicit != 3 || !a.Current || fmt.Sprint(a.Undeclared) != "[{tmux 3.5-1}]" || + len(a.Foreign) != 0 || !a.At.Equal(t0) { + t.Errorf("node-a: %+v", a) + } + if b.Current || b.NotCurrent == nil || b.NotCurrent.Bytes != 300000 || b.Undeclared != nil || + fmt.Sprint(b.Foreign) != "[{yay 12.0-1}]" || !b.At.Equal(t0.Add(-time.Hour)) { + t.Errorf("node-b (not current, no declaration, a foreign package): %+v", b) + } + if d := r["differing"].([]Spread); len(d) != 1 || d[0].Name != "zsh" || d[0].Versions["node-a"] != "5.9-2" || d[0].Versions["node-b"] != "5.9-1" { + t.Errorf("differing: %+v", d) + } + some := r["some_nodes"].([]Spread) + var names []string + for _, s := range some { + names = append(names, s.Name) + } + if strings.Join(names, ",") != "tmux,vim" { + t.Errorf("on some nodes only, the foreign apart: %v", names) + } + r, _ = m.Packages("", 1) + if r["some_nodes_count"] != 2 || r["some_nodes_truncated"] != true || len(r["some_nodes"].([]Spread)) != 1 { + t.Errorf("bounded: %v", r) + } +} + +func TestInstalledAndForeignAnswerFromTheBus(t *testing.T) { + _, m := viewStates(t) + r, err := m.InstalledFrom("", "", "", false, 10) + if err != nil || r["node"] != "node-a" || r["count"] != 4 || r["current"] != true { + t.Fatalf("this node's: %v %v", r, err) + } + pk := r["packages"].([]Package) + if pk[0].Reason != "dependency" || pk[0].Repository != "core" || pk[1].Reason != "explicit" { + t.Errorf("%+v", pk) + } + r, err = m.InstalledFrom("node-b", "", "", true, 10) + if err != nil || r["count"] != 1 || r["current"] != false || r["not_current"] == nil { + t.Fatalf("node-b's foreign: %v %v", r, err) + } + if tot := r["totals"].(map[string]int); tot["explicit"] != 1 || tot["dependency"] != 2 || tot["foreign"] != 1 { + t.Errorf("totals: %v", tot) + } + f, err := m.ForeignFrom("node-b") + if err != nil || f["count"] != 1 || fmt.Sprint(f["packages"]) != "[{yay 12.0-1}]" { + t.Errorf("%v %v", f, err) + } + if _, err := m.InstalledFrom("node-c", "", "", false, 10); err == nil || !strings.Contains(err.Error(), "node-c") { + t.Errorf("a node with no packages on the bus: %v", err) + } +} + +func TestUpdatesAnswerEveryNodeAndWhenItsListLastChanged(t *testing.T) { + states, m := viewStates(t) + up := states.store(StateUpdates) + up.now = func() time.Time { return t0.Add(-2 * time.Hour) } + up.Put("node-a", UpdatesValue{Reachable: true, Packages: []VersionChange{{"zsh", "5.9-2", "5.9-3"}}}) + up.now = func() time.Time { return t0.Add(-3 * time.Hour) } + up.Put("node-b", UpdatesValue{Reachable: false, Packages: []VersionChange{}}) + r, err := m.UpdatesFor("") + if err != nil { + t.Fatal(err) + } + v := r["nodes"].([]UpdatesView) + if len(v) != 2 || v[0].Count != 1 || !v[0].Reachable || v[0].Changed == nil || !v[0].Changed.Equal(t0.Add(-2*time.Hour)) || + v[1].Reachable || !v[1].Changed.Equal(t0.Add(-3*time.Hour)) { + t.Fatalf("%+v", v) + } + r, _ = m.UpdatesFor("node-b") + if v := r["nodes"].([]UpdatesView); len(v) != 1 || v[0].Node != "node-b" { + t.Errorf("one node: %+v", v) + } +} + +func TestANodeArgumentIsOneKeyToken(t *testing.T) { + for _, bad := range []string{"node-a.not-current", "node-a.*", "a b"} { + if _, err := nodeOf(map[string]any{"node": bad}); err == nil { + t.Errorf("%q was taken as a node", bad) + } + } + if n, err := nodeOf(map[string]any{"node": "node-a"}); err != nil || n != "node-a" { + t.Errorf("%q %v", n, err) + } +} diff --git a/modules/pacman/module.json b/modules/pacman/module.json index e80c7836..2c9a6250 100644 --- a/modules/pacman/module.json +++ b/modules/pacman/module.json @@ -66,6 +66,7 @@ "pacman_search", "pacman_info", "pacman_installed", + "pacman_packages", "pacman_owns", "pacman_files", "pacman_updates", From 9651a8b2da127ed7130fcee8bd5584c17c7f8177 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 14:37:00 +0200 Subject: [PATCH 2/3] issue 1840/S9: prove pacman's output puts no command line or environment on the bus A fixture of pacman and checkupdates output holding command lines, environment variables and messages leaves only names, versions and repositories in installed, updates and package.changed, and no command the keeper runs names a package, a foreign one least of all. --- .../cmd/pacman-tools/keeper_bus_test.go | 97 +++++++++++++++++++ .../checkupdates | 4 + .../pacman-Q | 9 ++ .../pacman-Qeq | 3 + .../pacman-Sl | 3 + 5 files changed, 116 insertions(+) create mode 100644 modules/pacman/cmd/pacman-tools/keeper_bus_test.go create mode 100644 modules/pacman/cmd/pacman-tools/testdata/pacman-output-with-command-lines-and-environment/checkupdates create mode 100644 modules/pacman/cmd/pacman-tools/testdata/pacman-output-with-command-lines-and-environment/pacman-Q create mode 100644 modules/pacman/cmd/pacman-tools/testdata/pacman-output-with-command-lines-and-environment/pacman-Qeq create mode 100644 modules/pacman/cmd/pacman-tools/testdata/pacman-output-with-command-lines-and-environment/pacman-Sl diff --git a/modules/pacman/cmd/pacman-tools/keeper_bus_test.go b/modules/pacman/cmd/pacman-tools/keeper_bus_test.go new file mode 100644 index 00000000..f337c7e3 --- /dev/null +++ b/modules/pacman/cmd/pacman-tools/keeper_bus_test.go @@ -0,0 +1,97 @@ +package main + +// What the keeper puts on the bus is only its named fields (novox/hq ADR 0507 §1; acceptance 1840/AC7): pacman's +// output holding command lines, environment variables and messages leaves none of them in a value or an event, +// and no command the keeper runs names a package, a foreign one least of all. (busshapes_test.go holds every +// shape to its named fields.) + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestPacmanOutputHoldingCommandLinesAndEnvironmentPutsNeitherOnTheBus(t *testing.T) { + dir := filepath.Join("testdata", "pacman-output-with-command-lines-and-environment") + read := func(name string) string { + b, err := os.ReadFile(filepath.Join(dir, name)) + if err != nil { + t.Fatal(err) + } + return string(b) + } + db := &pacmanDB{sl: read("pacman-Sl"), checkup: Ran{Stdout: read("checkupdates")}} + db.set(read("pacman-Q"), read("pacman-Qeq")) + k, states, ev, clock, w := newTestKeeper(db) + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- k.Run(ctx) }() + for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(time.Millisecond) { + if e, _ := states.store(StateUpdates).Get("node-a"); e != nil { + break + } + } + clock.Advance(time.Minute) + w.c <- LockReleased + w.c <- MirrorsRewritten + time.Sleep(20 * time.Millisecond) + cancel() + <-done + + var onBus []string + for _, name := range []string{StateInstalled, StateUpdates} { + s := states.store(name) + keys, _ := s.Keys() + for _, key := range keys { + e, _ := s.Get(key) + onBus = append(onBus, name+"."+key+" "+string(e.Value)) + } + } + events := ev.Take() + if len(events) == 0 { + t.Fatal("the first write said nothing") + } + for _, e := range events { + body, _ := json.Marshal(e.body) + onBus = append(onBus, e.name+" "+string(body)) + } + if len(onBus) < 3 { + t.Fatalf("on the bus: %v", onBus) + } + whole := strings.Join(onBus, "\n") + for _, never := range []string{"=", "sudo", "--noconfirm", "/", "rm", "warning", "hunter2", "Synchronizing", "HOME", "TOKEN", "SECRET"} { + if strings.Contains(whole, never) { + t.Errorf("%q reached the bus:\n%s", never, whole) + } + } + var inst InstalledValue + e, _ := states.store(StateInstalled).Get("node-a") + json.Unmarshal(e.Value, &inst) + if !samePackages(inst.Packages, []InstalledPackage{{"glibc", "2.42-1", false, "core"}, {"yay", "12.0-1", true, RepoForeign}, {"zsh", "5.9-1", true, "extra"}}) { + t.Errorf("the packages are read, the rest left out: %+v", inst.Packages) + } + var up UpdatesValue + e, _ = states.store(StateUpdates).Get("node-a") + json.Unmarshal(e.Value, &up) + if len(up.Packages) != 1 || up.Packages[0] != (VersionChange{"glibc", "2.42-1", "2.42-2"}) { + t.Errorf("updates: %+v", up) + } + + // no command the keeper ran names a package; the foreign one least of all. + db.mu.Lock() + defer db.mu.Unlock() + for _, c := range db.commands { + switch c { + case "pacman -Q", "pacman -Qeq", "pacman -Sl", "checkupdates": + default: + t.Errorf("the keeper ran %q", c) + } + if strings.Contains(c, "yay") { + t.Errorf("a command named the foreign package: %q", c) + } + } +} diff --git a/modules/pacman/cmd/pacman-tools/testdata/pacman-output-with-command-lines-and-environment/checkupdates b/modules/pacman/cmd/pacman-tools/testdata/pacman-output-with-command-lines-and-environment/checkupdates new file mode 100644 index 00000000..40c952dc --- /dev/null +++ b/modules/pacman/cmd/pacman-tools/testdata/pacman-output-with-command-lines-and-environment/checkupdates @@ -0,0 +1,4 @@ +glibc 2.42-1 -> 2.42-2 +zsh 5.9-1 -> 5.9-2 PASSWORD=hunter2 +env FOO=bar -> /bin/sh +:: Synchronizing package databases... diff --git a/modules/pacman/cmd/pacman-tools/testdata/pacman-output-with-command-lines-and-environment/pacman-Q b/modules/pacman/cmd/pacman-tools/testdata/pacman-output-with-command-lines-and-environment/pacman-Q new file mode 100644 index 00000000..a8c96383 --- /dev/null +++ b/modules/pacman/cmd/pacman-tools/testdata/pacman-output-with-command-lines-and-environment/pacman-Q @@ -0,0 +1,9 @@ +glibc 2.42-1 +sudo pacman -Syu --noconfirm +yay 12.0-1 +LD_PRELOAD=/tmp/hook.so 1.0-1 +zsh 5.9-1 +bash 5.2-1 HOME=/root +evil 1.0;rm-rf +PATH=/usr/bin:/bin +warning: could not get file information for etc/secret diff --git a/modules/pacman/cmd/pacman-tools/testdata/pacman-output-with-command-lines-and-environment/pacman-Qeq b/modules/pacman/cmd/pacman-tools/testdata/pacman-output-with-command-lines-and-environment/pacman-Qeq new file mode 100644 index 00000000..815d6570 --- /dev/null +++ b/modules/pacman/cmd/pacman-tools/testdata/pacman-output-with-command-lines-and-environment/pacman-Qeq @@ -0,0 +1,3 @@ +yay +zsh +TOKEN=abc diff --git a/modules/pacman/cmd/pacman-tools/testdata/pacman-output-with-command-lines-and-environment/pacman-Sl b/modules/pacman/cmd/pacman-tools/testdata/pacman-output-with-command-lines-and-environment/pacman-Sl new file mode 100644 index 00000000..269f745e --- /dev/null +++ b/modules/pacman/cmd/pacman-tools/testdata/pacman-output-with-command-lines-and-environment/pacman-Sl @@ -0,0 +1,3 @@ +core glibc 2.42-1 [installed] +extra zsh 5.9-1 [installed] +extra SECRET=x 1-1 From 9511074f406fc077d0308c9f055dadfd9cf5f854 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 14:37:28 +0200 Subject: [PATCH 3/3] Take the SDK whose process can delete a key (issue 1840/S1) --- modules/pacman/go.mod | 2 +- modules/pacman/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/pacman/go.mod b/modules/pacman/go.mod index 1fac27ba..2ad98047 100644 --- a/modules/pacman/go.mod +++ b/modules/pacman/go.mod @@ -2,7 +2,7 @@ module pacman go 1.26.0 -require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022121659-076e1f7947db +require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022123624-5ea6c215a48b require ( github.com/klauspost/compress v1.20.0 // indirect diff --git a/modules/pacman/go.sum b/modules/pacman/go.sum index 27e3d79b..61e2d7d8 100644 --- a/modules/pacman/go.sum +++ b/modules/pacman/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022121659-076e1f7947db h1:WiBM7p87qsINU+0ipvchRL7ZUFSJHGLG41ZA7vMSR18= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022121659-076e1f7947db/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022123624-5ea6c215a48b h1:5bYfSahHJbfPWV2D4h1ejmRyvUevJtrWSpbtiVCsaOQ= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022123624-5ea6c215a48b/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=