dbus: hold node-message-bus, and never restart the bus live (hq ADR 0215)
A live restart of the system bus during an upgrade hung every login on a workstation until a reboot. The module owns the bus's packages, declares the bus running with no restart or reload trigger, publishes only curated events (health, services, denials; never traffic) and serves tools to look at both buses.
This commit is contained in:
jochen
@@ -0,0 +1,290 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Packages are the bus implementation's packages, as the manifest declares them: dbus-broker (the
|
||||
// bus), its units (the dbus.service alias), and the reference package, which ships the bus's
|
||||
// configuration, the socket that starts it at boot, and libdbus.
|
||||
var Packages = []string{"dbus", "dbus-broker", "dbus-broker-units"}
|
||||
|
||||
// RunningPackages are the packages whose files the running bus loaded when it started: a newer one
|
||||
// installed since takes effect only at the next boot (novox/hq ADR 0215 §2).
|
||||
var RunningPackages = []string{"dbus-broker", "dbus"}
|
||||
|
||||
// SystemUnit is the bus's real unit; dbus.service is its alias.
|
||||
const SystemUnit = "dbus-broker.service"
|
||||
|
||||
// ServiceDirs are where activatable system services are described.
|
||||
var ServiceDirs = []string{"/usr/share/dbus-1/system-services", "/usr/local/share/dbus-1/system-services",
|
||||
"/usr/lib/dbus-1/system-services"}
|
||||
|
||||
// Package is one installed package as the package manager's local database records it.
|
||||
type Package struct {
|
||||
Name string
|
||||
Version string
|
||||
Installed time.Time
|
||||
}
|
||||
|
||||
// ParseDesc reads one package's desc file in the local database.
|
||||
func ParseDesc(desc string) Package {
|
||||
var p Package
|
||||
lines := strings.Split(desc, "\n")
|
||||
for i := 0; i+1 < len(lines); i++ {
|
||||
v := strings.TrimSpace(lines[i+1])
|
||||
switch strings.TrimSpace(lines[i]) {
|
||||
case "%NAME%":
|
||||
p.Name = v
|
||||
case "%VERSION%":
|
||||
p.Version = v
|
||||
case "%INSTALLDATE%":
|
||||
if n, err := strconv.ParseInt(v, 10, 64); err == nil {
|
||||
p.Installed = time.Unix(n, 0)
|
||||
}
|
||||
}
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// InstalledPackage reads a package from the local database, without running the package manager.
|
||||
func (m *Machine) InstalledPackage(name string) (Package, bool) {
|
||||
for _, d := range m.glob("/var/lib/pacman/local/" + name + "-*/desc") {
|
||||
if p := ParseDesc(m.read(d) + "\n"); p.Name == name {
|
||||
return p, true
|
||||
}
|
||||
}
|
||||
return Package{}, false
|
||||
}
|
||||
|
||||
// ParseShow reads `systemctl show` blocks: one map per unit, in the order asked.
|
||||
func ParseShow(out string) []map[string]string {
|
||||
var blocks []map[string]string
|
||||
cur := map[string]string{}
|
||||
for _, line := range strings.Split(out, "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" {
|
||||
if len(cur) > 0 {
|
||||
blocks = append(blocks, cur)
|
||||
cur = map[string]string{}
|
||||
}
|
||||
continue
|
||||
}
|
||||
if k, v, ok := strings.Cut(line, "="); ok {
|
||||
cur[k] = v
|
||||
}
|
||||
}
|
||||
if len(cur) > 0 {
|
||||
blocks = append(blocks, cur)
|
||||
}
|
||||
return blocks
|
||||
}
|
||||
|
||||
// unixStamp reads systemd's "@<seconds>" timestamp.
|
||||
func unixStamp(s string) (time.Time, bool) {
|
||||
n, err := strconv.ParseInt(strings.TrimPrefix(s, "@"), 10, 64)
|
||||
if err != nil || n == 0 {
|
||||
return time.Time{}, false
|
||||
}
|
||||
return time.Unix(n, 0), true
|
||||
}
|
||||
|
||||
// BusUnit is the system bus's unit as the service manager has it.
|
||||
type BusUnit struct {
|
||||
Unit string `json:"unit"`
|
||||
Active string `json:"active"`
|
||||
PID uint32 `json:"pid,omitempty"`
|
||||
Since time.Time `json:"-"`
|
||||
Started string `json:"started,omitempty"`
|
||||
}
|
||||
|
||||
// SystemBusUnit asks the service manager about the system bus through its alias, so the answer is
|
||||
// the implementation's unit whichever it is.
|
||||
func (m *Machine) SystemBusUnit(ctx context.Context) (BusUnit, error) {
|
||||
out, err := m.Run(ctx, "systemctl", "show", "dbus.service", "-p", "Id,ActiveState,MainPID,ActiveEnterTimestamp",
|
||||
"--timestamp=unix")
|
||||
if err != nil {
|
||||
return BusUnit{}, err
|
||||
}
|
||||
b := ParseShow(out)
|
||||
if len(b) == 0 {
|
||||
return BusUnit{}, fmt.Errorf("systemctl show answered nothing for dbus.service")
|
||||
}
|
||||
u := BusUnit{Unit: b[0]["Id"], Active: b[0]["ActiveState"], PID: parsePID(b[0]["MainPID"])}
|
||||
if t, ok := unixStamp(b[0]["ActiveEnterTimestamp"]); ok {
|
||||
u.Since, u.Started = t, t.UTC().Format(time.RFC3339)
|
||||
}
|
||||
return u, nil
|
||||
}
|
||||
|
||||
// ServiceFile is one activatable system service's description.
|
||||
type ServiceFile struct {
|
||||
File string
|
||||
Name string
|
||||
Unit string
|
||||
}
|
||||
|
||||
// ParseServiceFile reads the Name and SystemdService of a D-Bus service file.
|
||||
func ParseServiceFile(file, content string) ServiceFile {
|
||||
s := ServiceFile{File: file}
|
||||
for _, line := range strings.Split(content, "\n") {
|
||||
k, v, ok := strings.Cut(strings.TrimSpace(line), "=")
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
switch strings.TrimSpace(k) {
|
||||
case "Name":
|
||||
s.Name = strings.TrimSpace(v)
|
||||
case "SystemdService":
|
||||
s.Unit = strings.TrimSpace(v)
|
||||
}
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// Check is one thing the module expects of the machine.
|
||||
type Check struct {
|
||||
Name string `json:"name"`
|
||||
OK bool `json:"ok"`
|
||||
Detail string `json:"detail"`
|
||||
}
|
||||
|
||||
// RebootDue says, per package the running bus loaded, whether a newer one was installed after the bus
|
||||
// started: the bus is never restarted live, so that package waits for a boot.
|
||||
func RebootDue(started time.Time, pkgs []Package) (bool, string) {
|
||||
var newer []string
|
||||
for _, p := range pkgs {
|
||||
if !p.Installed.IsZero() && p.Installed.After(started) {
|
||||
newer = append(newer, fmt.Sprintf("%s %s installed %s", p.Name, p.Version, p.Installed.UTC().Format(time.RFC3339)))
|
||||
}
|
||||
}
|
||||
if len(newer) == 0 {
|
||||
return false, "the running bus started " + started.UTC().Format(time.RFC3339) + ", after every package it loaded was installed"
|
||||
}
|
||||
return true, "the running bus started " + started.UTC().Format(time.RFC3339) + " and is older than " +
|
||||
strings.Join(newer, ", ") + ": a reboot is due (the bus is never restarted live, novox/hq ADR 0215)"
|
||||
}
|
||||
|
||||
// Check says what this module expects and whether the machine meets it.
|
||||
func (m *Machine) Check(ctx context.Context, w *Watcher) map[string]any {
|
||||
var checks []Check
|
||||
var notes []string
|
||||
add := func(name string, ok bool, format string, args ...any) {
|
||||
checks = append(checks, Check{Name: name, OK: ok, Detail: fmt.Sprintf(format, args...)})
|
||||
}
|
||||
var running []Package
|
||||
for _, name := range Packages {
|
||||
p, ok := m.InstalledPackage(name)
|
||||
add("package "+name, ok, "installed: %v %s", ok, p.Version)
|
||||
for _, r := range RunningPackages {
|
||||
if ok && r == name {
|
||||
running = append(running, p)
|
||||
}
|
||||
}
|
||||
}
|
||||
unit, err := m.SystemBusUnit(ctx)
|
||||
if err != nil {
|
||||
add("system bus", false, "%v", err)
|
||||
} else {
|
||||
add("system bus", unit.Unit == SystemUnit && unit.Active == "active",
|
||||
"dbus.service is %s, %s, pid %d, since %s (want %s active)", orWord(unit.Unit, "unknown"),
|
||||
orWord(unit.Active, "unknown"), unit.PID, orWord(unit.Started, "unknown"), SystemUnit)
|
||||
if !unit.Since.IsZero() {
|
||||
due, detail := RebootDue(unit.Since, running)
|
||||
add("running bus is the installed one", !due, "%s", detail)
|
||||
}
|
||||
}
|
||||
|
||||
var files []ServiceFile
|
||||
for _, dir := range ServiceDirs {
|
||||
for _, f := range m.glob(dir + "/*.service") {
|
||||
if s := ParseServiceFile(f, m.read(f)); s.Unit != "" {
|
||||
files = append(files, s)
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Slice(files, func(i, j int) bool { return files[i].Name < files[j].Name })
|
||||
if len(files) > 0 {
|
||||
args := []string{"show", "-p", "Id,LoadState"}
|
||||
for _, f := range files {
|
||||
args = append(args, f.Unit)
|
||||
}
|
||||
out, err := m.Run(ctx, "systemctl", args...)
|
||||
blocks := ParseShow(out)
|
||||
switch {
|
||||
case err != nil:
|
||||
add("activatable services", false, "systemctl show: %v", err)
|
||||
case len(blocks) != len(files):
|
||||
add("activatable services", false, "systemctl show answered %d units for %d service files", len(blocks), len(files))
|
||||
default:
|
||||
var broken, disabled []string
|
||||
for i, f := range files {
|
||||
if blocks[i]["LoadState"] != "not-found" {
|
||||
continue
|
||||
}
|
||||
// systemd's own bus services are reached through a dbus-org.* alias that exists only
|
||||
// while the service is enabled: a disabled one is a choice, not a fault.
|
||||
if strings.HasPrefix(f.Unit, "dbus-org.") {
|
||||
disabled = append(disabled, f.Name+" → "+f.Unit)
|
||||
} else {
|
||||
broken = append(broken, f.Name+" → "+f.Unit+" ("+f.File+")")
|
||||
}
|
||||
}
|
||||
add("activatable services", len(broken) == 0, "%d service files; whose unit does not exist: %s",
|
||||
len(files), orWord(strings.Join(broken, ", "), "none"))
|
||||
if len(disabled) > 0 {
|
||||
notes = append(notes, "activation fails for "+strings.Join(disabled, ", ")+
|
||||
": the service is not enabled, so its dbus-org alias does not exist")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
denials, _, err := m.Denials(ctx, "", m.Now().Add(-time.Hour))
|
||||
if err != nil {
|
||||
add("policy denials in the last hour", false, "reading the journal: %v", err)
|
||||
} else {
|
||||
seen := map[string]bool{}
|
||||
var ex []string
|
||||
for _, d := range denials {
|
||||
k := strings.TrimSpace(d.Type + " " + d.Interface + "." + d.Member + " to " + d.Destination)
|
||||
if !seen[k] && len(ex) < DenialExamples {
|
||||
seen[k] = true
|
||||
ex = append(ex, k)
|
||||
}
|
||||
}
|
||||
add("policy denials in the last hour", len(denials) == 0, "%d: %s", len(denials), orWord(strings.Join(ex, "; "), "none"))
|
||||
}
|
||||
|
||||
if a, err := m.SessionAddress(); err == nil {
|
||||
notes = append(notes, "this account's session bus: "+a)
|
||||
} else {
|
||||
notes = append(notes, err.Error())
|
||||
}
|
||||
|
||||
if w != nil {
|
||||
s := w.Snapshot()
|
||||
add("watcher", s.Connected && !s.Stalled, "connected %v, stalled %v%s, last ping %.1f ms, %d well-known names",
|
||||
s.Connected, s.Stalled, orNote(s.StallReason), s.LastPingMS, s.Services)
|
||||
add("events reach the mesh's bus", s.Pending == 0 && s.Problem == "", "%d event(s) waiting, %d dropped%s",
|
||||
s.Pending, s.Dropped, orNote(s.Problem))
|
||||
}
|
||||
failing := 0
|
||||
for _, c := range checks {
|
||||
if !c.OK {
|
||||
failing++
|
||||
}
|
||||
}
|
||||
return map[string]any{"checks": checks, "failing": failing, "notes": notes}
|
||||
}
|
||||
|
||||
func orNote(s string) string {
|
||||
if s == "" {
|
||||
return ""
|
||||
}
|
||||
return " (" + s + ")"
|
||||
}
|
||||
Reference in New Issue
Block a user