From ec983bfc3d1b52443f08095f611c19e04c8d201c Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 12 Oct 2026 17:21:36 +0200 Subject: [PATCH 1/2] Have a machine say what it is made of, and say it once a day unasked (issue 160) The mesh knew a machine's architecture and kernel family and nothing it is made of, and heard even that only when it sent the machine something. The profile now carries the machine's memory, each local filesystem's size and free space, processor model, cores and threads, distribution and version, kernel release, virtual or physical, boot time and timezone; a fact it cannot read is named unknown with why, never zeroed. The reconcile says the profile unasked once the mesh last heard it 23 hours ago, so on a machine nobody pushed to nothing the mesh holds of it is older than a day. --- cmd/mesh-host/machine_refresh_test.go | 64 ++++ cmd/mesh-host/main.go | 33 +- internal/profile/machine.go | 433 ++++++++++++++++++++++++++ internal/profile/machine_test.go | 161 ++++++++++ internal/profile/profile.go | 7 + 5 files changed, 697 insertions(+), 1 deletion(-) create mode 100644 cmd/mesh-host/machine_refresh_test.go create mode 100644 internal/profile/machine.go create mode 100644 internal/profile/machine_test.go diff --git a/cmd/mesh-host/machine_refresh_test.go b/cmd/mesh-host/machine_refresh_test.go new file mode 100644 index 0000000..3bcab8e --- /dev/null +++ b/cmd/mesh-host/machine_refresh_test.go @@ -0,0 +1,64 @@ +package main + +import ( + "testing" + "time" + + "github.com/novox/mesh-host/internal/link" +) + +// Defends novox/hq issue 160: a machine nobody pushed to still says what it is made of once a day, so the +// newest of it the mesh holds is no older than a day. Before, the reconcile said it only with something +// else (a hold, a firewall, an outward link changed), and a converged machine's picture was as old as the +// last delivery to it. +func TestAMachineNobodyPushedToSaysWhatItIsOnceADay(t *testing.T) { + at := time.Date(2026, 10, 12, 9, 0, 0, 0, time.UTC) + w := &adoptionWatch{now: func() time.Time { return at }} + quiet := link.Report{Outward: []string{"wlan0"}, + Profile: map[string]any{"machine": map[string]any{"observed_at": at}}} + + if !w.news(quiet) { + t.Fatal("the first reconcile after the engine started did not say what the machine is") + } + w.said(quiet) + + for _, after := range []time.Duration{5 * time.Minute, 12 * time.Hour, 22*time.Hour + 55*time.Minute} { + at = time.Date(2026, 10, 12, 9, 0, 0, 0, time.UTC).Add(after) + if w.news(quiet) { + t.Errorf("%s after the mesh heard it, an unchanged reconcile was said again", after) + } + } + // The reconcile that falls past 23 hours says it, so it reaches the mesh inside the day. + at = time.Date(2026, 10, 12, 9, 0, 0, 0, time.UTC).Add(23*time.Hour + 5*time.Minute) + if !w.news(quiet) { + t.Fatal("a day on, the reconcile did not say what the machine is") + } + w.said(quiet) + at = at.Add(5 * time.Minute) + if w.news(quiet) { + t.Error("once said, it was said again at the next reconcile") + } +} + +// A delivery's account carries the profile too: heard, it starts the day again. +func TestADeliveryHeardStartsTheDayAgain(t *testing.T) { + at := time.Date(2026, 10, 12, 9, 0, 0, 0, time.UTC) + w := &adoptionWatch{now: func() time.Time { return at }} + report := link.Report{Outward: []string{"eth0"}, Profile: map[string]any{"capabilities": []any{}}} + w.said(report) + at = at.Add(20 * time.Hour) + w.said(report) // a delivery's apply + at = at.Add(20 * time.Hour) + if w.news(report) { + t.Error("said again unasked 20 hours after a delivery told the mesh") + } +} + +// A refusal carries nothing about the machine, so it is never said for being a day old. +func TestARefusalIsNotSaidForTheMachinesSake(t *testing.T) { + at := time.Date(2026, 10, 12, 9, 0, 0, 0, time.UTC) + w := &adoptionWatch{now: func() time.Time { return at }} + if w.machineDue(link.Report{Refused: "no"}) { + t.Error("a refusal was due for what the machine is made of") + } +} diff --git a/cmd/mesh-host/main.go b/cmd/mesh-host/main.go index feb1da7..9638706 100644 --- a/cmd/mesh-host/main.go +++ b/cmd/mesh-host/main.go @@ -1352,6 +1352,29 @@ type adoptionWatch struct { last string // faulty is whether the last report the mesh heard said an apply refused or failed (novox/hq issue 233). faulty bool + // machineSaid is when the mesh last heard what this machine is made of (novox/hq issue 160), and now the + // clock it is judged by (time.Now when nil). + machineSaid time.Time + now func() time.Time +} + +// machineEvery is how old what a machine said of itself may grow before a reconcile says it again unasked +// (novox/hq issue 160): a machine nobody pushed to still says it once a day, so no fact the mesh holds of it +// is older than a day. Under a day by more than one reconcile interval, so the reconcile that says it falls +// inside the day rather than just past it. +const machineEvery = 23 * time.Hour + +func (w *adoptionWatch) clock() time.Time { + if w.now != nil { + return w.now() + } + return time.Now() +} + +// machineDue is whether a report carries what the machine is made of and the mesh last heard it a day ago +// or more. A refusal carries none, so it never is. +func (w *adoptionWatch) machineDue(r link.Report) bool { + return r.Refused == "" && len(r.Profile) > 0 && w.clock().Sub(w.machineSaid) >= machineEvery } // fingerprint is what a report says about adoption: each hold and whether it changed, the @@ -1416,7 +1439,8 @@ func applyState(r link.Report) string { // faulty is whether a report says an apply refused or failed. func faulty(r link.Report) bool { return r.Refused != "" || len(r.Failed) > 0 } -// news is whether a reconcile's report is worth saying unasked: it says something the last one the mesh heard +// news is whether a reconcile's report is worth saying unasked: what the machine is made of last reached the +// mesh a day ago or more (novox/hq issue 160); or it says something the last one the mesh heard // did not, and it is about the machine (worthSaying) — or about how the apply went, refused or failed now, or // clean after the last one the mesh heard was not (novox/hq issue 233). A refusal says nothing about the // machine, but that the machine refuses is what the mesh raises a condition from, and a clean apply after it is @@ -1424,6 +1448,10 @@ func faulty(r link.Report) bool { return r.Refused != "" || len(r.Failed) > 0 } func (w *adoptionWatch) news(r link.Report) bool { w.mu.Lock() defer w.mu.Unlock() + // What the machine is made of, once a day whatever else changed (novox/hq issue 160). + if w.machineDue(r) { + return true + } if adoptionFingerprint(r) == w.last { return false } @@ -1444,6 +1472,9 @@ func (w *adoptionWatch) said(r link.Report) { defer w.mu.Unlock() w.last = adoptionFingerprint(r) w.faulty = faulty(r) + if len(r.Profile) > 0 { + w.machineSaid = w.clock() + } } // changed is differs and said together, for a report published as it is made. diff --git a/internal/profile/machine.go b/internal/profile/machine.go new file mode 100644 index 0000000..db09a3e --- /dev/null +++ b/internal/profile/machine.go @@ -0,0 +1,433 @@ +package profile + +import ( + "context" + "fmt" + "os" + "sort" + "strconv" + "strings" + "syscall" + "time" +) + +// Machine is what this machine is made of, as it says itself (novox/hq issue 160): what somebody +// deciding where a module should go wants to know, beside what it can do. +// +// **A fact it could not determine is said, never zeroed.** Every fact below is either present or +// named in Unknown with why; a machine reporting 0 bytes of memory would be read as a machine with +// none. So a field is omitted only when Unknown says why. +type Machine struct { + // ObservedAt is when this was read: the age every reader shows beside it. + ObservedAt time.Time `json:"observed_at"` + + // KernelRelease is the running kernel's release, as /proc/sys/kernel/osrelease says it. (The + // profile's own `kernel` is the kernel's family, `linux`.) + KernelRelease string `json:"kernel_release,omitempty"` + // Distribution and its Version, from /etc/os-release: VERSION_ID, or BUILD_ID for a rolling + // distribution that has no version (Arch Linux says `rolling`). + Distribution string `json:"distribution,omitempty"` + Version string `json:"version,omitempty"` + + // Processor is its model name; Cores the physical cores and Threads the logical processors. + Processor string `json:"processor,omitempty"` + Cores int `json:"cores,omitempty"` + Threads int `json:"threads,omitempty"` + + MemoryBytes int64 `json:"memory_bytes,omitempty"` + + // Virtual is `physical`, or the kind of virtual machine or container it is (`kvm`, `lxc`), as + // systemd-detect-virt names it. + Virtual string `json:"virtual,omitempty"` + + // BootedAt is when it booted: its uptime is the time since, so the fact does not change between + // two reports of the same boot. + BootedAt *time.Time `json:"booted_at,omitempty"` + + Timezone string `json:"timezone,omitempty"` + + // Filesystems is each local filesystem mounted, by where, with its size and what is free. + Filesystems []Filesystem `json:"filesystems,omitempty"` + + // Unknown names each fact that could not be determined, and why. + Unknown map[string]string `json:"unknown,omitempty"` +} + +// Filesystem is one mounted local filesystem. +type Filesystem struct { + Mount string `json:"mount"` + Device string `json:"device"` + Type string `json:"type"` + // SizeBytes and FreeBytes are what statfs says: free is what an unprivileged writer may still + // use (f_bavail), the number a person deciding where data goes wants. + SizeBytes int64 `json:"size_bytes"` + FreeBytes int64 `json:"free_bytes"` +} + +// The facts, by the names Unknown says them under. +const ( + FactKernelRelease = "kernel release" + FactDistribution = "distribution" + FactVersion = "version" + FactProcessor = "processor" + FactCores = "cores" + FactThreads = "threads" + FactMemory = "memory" + FactVirtual = "virtual or physical" + FactUptime = "uptime" + FactTimezone = "timezone" + FactFilesystems = "filesystems" + FactPublicAddress = "public address" +) + +// MachineFacts is every fact Unknown may name, in the order a reader shows them. +var MachineFacts = []string{FactKernelRelease, FactDistribution, FactVersion, FactProcessor, FactCores, + FactThreads, FactMemory, FactVirtual, FactUptime, FactTimezone, FactFilesystems, FactPublicAddress} + +// publicAddressUnknown is why the public uplink address is not said. A machine behind a NAT cannot see +// its public address from itself, and asking a service outside the mesh is a dependency the mesh does not +// take on unasked; how a machine learns it is not decided yet, so it is said as unknown rather than +// guessed from a private address. +const publicAddressUnknown = "a machine behind a NAT cannot see its public address from itself, and how it " + + "learns it is not decided yet" + +// Sources is where the facts are read from. Replaced in tests only for the parsing layer; the real +// sources are exercised against this machine as well (novox/hq ADR 0017). +type Sources struct { + Read func(path string) ([]byte, error) + Readlink func(path string) (string, error) + Run Runner + // Statfs is (size, free) in bytes of the filesystem mounted at a path. + Statfs func(path string) (size, free int64, err error) + Now func() time.Time +} + +// RealSources reads this machine. +func RealSources() Sources { + return Sources{Read: os.ReadFile, Readlink: os.Readlink, Run: ExecRunner, Statfs: statfs, Now: time.Now} +} + +func statfs(path string) (int64, int64, error) { + var s syscall.Statfs_t + if err := syscall.Statfs(path, &s); err != nil { + return 0, 0, err + } + return int64(s.Blocks) * int64(s.Bsize), int64(s.Bavail) * int64(s.Bsize), nil +} + +// DescribeMachine reads what this machine is made of. It never fails: a fact that cannot be read is +// named in Unknown with why, and the rest are still said. +func DescribeMachine(ctx context.Context, src Sources) Machine { + if src.Now == nil { + src.Now = time.Now + } + m := Machine{ObservedAt: src.Now().UTC(), Unknown: map[string]string{}} + unknown := func(fact string, why string) { m.Unknown[fact] = why } + + if b, err := src.Read("/proc/sys/kernel/osrelease"); err != nil { + unknown(FactKernelRelease, err.Error()) + } else if r := strings.TrimSpace(string(b)); r == "" { + unknown(FactKernelRelease, "/proc/sys/kernel/osrelease is empty") + } else { + m.KernelRelease = r + } + + if b, err := src.Read("/etc/os-release"); err != nil { + unknown(FactDistribution, err.Error()) + unknown(FactVersion, err.Error()) + } else { + name, version, whyName, whyVersion := distributionOf(string(b)) + m.Distribution, m.Version = name, version + if whyName != "" { + unknown(FactDistribution, whyName) + } + if whyVersion != "" { + unknown(FactVersion, whyVersion) + } + } + + if b, err := src.Read("/proc/cpuinfo"); err != nil { + unknown(FactProcessor, err.Error()) + unknown(FactCores, err.Error()) + unknown(FactThreads, err.Error()) + } else { + c := processorOf(string(b)) + m.Processor, m.Cores, m.Threads = c.model, c.cores, c.threads + for fact, why := range c.unknown { + unknown(fact, why) + } + } + + if b, err := src.Read("/proc/meminfo"); err != nil { + unknown(FactMemory, err.Error()) + } else if kb, ok := memTotalKB(string(b)); !ok { + unknown(FactMemory, "MemTotal not found in /proc/meminfo") + } else { + m.MemoryBytes = kb * 1024 + } + + m.Virtual, _ = virtualOf(ctx, src.Run, unknown) + + if b, err := src.Read("/proc/stat"); err != nil { + unknown(FactUptime, err.Error()) + } else if booted, ok := bootedAt(string(b)); !ok { + unknown(FactUptime, "btime not found in /proc/stat") + } else { + m.BootedAt = &booted + } + + if tz, why := timezoneOf(src); why != "" { + unknown(FactTimezone, why) + } else { + m.Timezone = tz + } + + if b, err := src.Read("/proc/self/mounts"); err != nil { + unknown(FactFilesystems, err.Error()) + } else { + fs, why := filesystemsOf(ctx, string(b), src.Statfs) + m.Filesystems = fs + if why != "" { + unknown(FactFilesystems, why) + } + } + + unknown(FactPublicAddress, publicAddressUnknown) + return m +} + +// distributionOf is the distribution's name and version from an os-release file, with why either is +// not said. NAME first, then ID; VERSION_ID first, then BUILD_ID. +func distributionOf(osRelease string) (name, version, whyName, whyVersion string) { + fields := map[string]string{} + for _, line := range strings.Split(osRelease, "\n") { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + key, value, found := strings.Cut(line, "=") + if !found { + continue + } + fields[strings.TrimSpace(key)] = strings.Trim(strings.TrimSpace(value), `"'`) + } + name = firstOf(fields["NAME"], fields["ID"]) + version = firstOf(fields["VERSION_ID"], fields["BUILD_ID"]) + if name == "" { + whyName = "/etc/os-release names neither NAME nor ID" + } + if version == "" { + whyVersion = "/etc/os-release names neither VERSION_ID nor BUILD_ID" + } + return name, version, whyName, whyVersion +} + +func firstOf(values ...string) string { + for _, v := range values { + if v != "" { + return v + } + } + return "" +} + +type processor struct { + model string + cores, threads int + unknown map[string]string +} + +// processorOf reads /proc/cpuinfo: the model from `model name` (or `Model`, `Hardware`, `cpu model`, +// which other architectures use), threads from its `processor` entries, and cores from the distinct +// (physical id, core id) pairs — which an architecture that does not say them leaves unknown rather than +// guessed equal to the threads. +func processorOf(cpuinfo string) processor { + p := processor{unknown: map[string]string{}} + pairs := map[string]bool{} + physical := "0" + var models = map[string]string{} + for _, line := range strings.Split(cpuinfo, "\n") { + key, value, found := strings.Cut(line, ":") + if !found { + continue + } + key, value = strings.TrimSpace(key), strings.TrimSpace(value) + switch key { + case "processor": + p.threads++ + case "physical id": + physical = value + case "core id": + pairs[physical+"/"+value] = true + case "model name", "Model", "Hardware", "cpu model": + if _, had := models[key]; !had { + models[key] = value + } + } + } + p.model = firstOf(models["model name"], models["cpu model"], models["Model"], models["Hardware"]) + p.cores = len(pairs) + if p.model == "" { + p.unknown[FactProcessor] = "/proc/cpuinfo names no model" + } + if p.threads == 0 { + p.unknown[FactThreads] = "/proc/cpuinfo lists no processor" + } + if p.cores == 0 { + p.unknown[FactCores] = "/proc/cpuinfo says no core id" + } + return p +} + +func memTotalKB(meminfo string) (int64, bool) { + for _, line := range strings.Split(meminfo, "\n") { + if !strings.HasPrefix(line, "MemTotal:") { + continue + } + parts := strings.Fields(line) + if len(parts) < 2 { + return 0, false + } + kb, err := strconv.ParseInt(parts[1], 10, 64) + if err != nil || kb <= 0 { + return 0, false + } + return kb, true + } + return 0, false +} + +// virtualOf asks systemd-detect-virt. It says `none` on a physical machine, and exits 1 doing so. +func virtualOf(ctx context.Context, run Runner, unknown func(string, string)) (string, bool) { + if run == nil { + unknown(FactVirtual, "nothing here runs commands") + return "", false + } + out, err := run(ctx, "systemd-detect-virt") + switch said := strings.TrimSpace(out); { + case said == "none": + return "physical", true + case said != "" && err == nil: + return said, true + case err != nil: + unknown(FactVirtual, "systemd-detect-virt: "+err.Error()) + default: + unknown(FactVirtual, "systemd-detect-virt said nothing") + } + return "", false +} + +func bootedAt(stat string) (time.Time, bool) { + for _, line := range strings.Split(stat, "\n") { + parts := strings.Fields(line) + if len(parts) == 2 && parts[0] == "btime" { + secs, err := strconv.ParseInt(parts[1], 10, 64) + if err != nil || secs <= 0 { + return time.Time{}, false + } + return time.Unix(secs, 0).UTC(), true + } + } + return time.Time{}, false +} + +// timezoneOf is the zone /etc/localtime links into the zoneinfo tree, or /etc/timezone's. +func timezoneOf(src Sources) (string, string) { + var whys []string + if src.Readlink != nil { + target, err := src.Readlink("/etc/localtime") + if err == nil { + if _, zone, found := strings.Cut(target, "zoneinfo/"); found && zone != "" { + return zone, "" + } + whys = append(whys, "/etc/localtime links to "+target+", outside a zoneinfo tree") + } else { + whys = append(whys, "/etc/localtime: "+err.Error()) + } + } + if b, err := src.Read("/etc/timezone"); err == nil { + if zone := strings.TrimSpace(string(b)); zone != "" { + return zone, "" + } + whys = append(whys, "/etc/timezone is empty") + } else { + whys = append(whys, err.Error()) + } + return "", strings.Join(whys, "; ") +} + +// localTypes are the filesystem types said: those on a local device. A network filesystem is left out on +// purpose — statfs on a server that stopped answering blocks, and this runs on every apply. +var localTypes = map[string]bool{"ext2": true, "ext3": true, "ext4": true, "xfs": true, "btrfs": true, + "zfs": true, "f2fs": true, "vfat": true, "exfat": true, "ntfs": true, "ntfs3": true, "fuseblk": true, + "jfs": true, "reiserfs": true, "bcachefs": true} + +// filesystemsOf lists each local filesystem once — a device mounted at several places (a btrfs +// subvolume, a bind mount) under its shortest mount — with its size and what is free. Bounded by the +// context, so a filesystem whose statfs hangs cannot hang the apply that reports it. +func filesystemsOf(ctx context.Context, mounts string, stat func(string) (int64, int64, error)) ([]Filesystem, string) { + if stat == nil { + return nil, "nothing here can ask a filesystem its size" + } + byDevice := map[string]Filesystem{} + for _, line := range strings.Split(mounts, "\n") { + f := strings.Fields(line) + if len(f) < 3 || !localTypes[f[2]] { + continue + } + mount := unescapeMount(f[1]) + if had, ok := byDevice[f[0]]; ok && len(had.Mount) <= len(mount) { + continue + } + byDevice[f[0]] = Filesystem{Mount: mount, Device: f[0], Type: f[2]} + } + if len(byDevice) == 0 { + return nil, "/proc/self/mounts lists no local filesystem" + } + + type answer struct { + fs []Filesystem + why []string + } + done := make(chan answer, 1) + go func() { + var a answer + for _, fs := range byDevice { + size, free, err := stat(fs.Mount) + if err != nil { + a.why = append(a.why, fmt.Sprintf("%s: %v", fs.Mount, err)) + continue + } + fs.SizeBytes, fs.FreeBytes = size, free + a.fs = append(a.fs, fs) + } + done <- a + }() + select { + case a := <-done: + sort.Slice(a.fs, func(i, j int) bool { return a.fs[i].Mount < a.fs[j].Mount }) + sort.Strings(a.why) + return a.fs, strings.Join(a.why, "; ") + case <-ctx.Done(): + return nil, "asking the filesystems their size did not finish: " + ctx.Err().Error() + } +} + +// unescapeMount undoes the octal escapes /proc/self/mounts writes a space or tab in a path as. +func unescapeMount(path string) string { + if !strings.Contains(path, `\`) { + return path + } + var b strings.Builder + for i := 0; i < len(path); i++ { + if path[i] == '\\' && i+3 < len(path) { + if n, err := strconv.ParseUint(path[i+1:i+4], 8, 8); err == nil { + b.WriteByte(byte(n)) + i += 3 + continue + } + } + b.WriteByte(path[i]) + } + return b.String() +} diff --git a/internal/profile/machine_test.go b/internal/profile/machine_test.go new file mode 100644 index 0000000..fe32e64 --- /dev/null +++ b/internal/profile/machine_test.go @@ -0,0 +1,161 @@ +package profile + +import ( + "context" + "encoding/json" + "errors" + "os" + "runtime" + "strings" + "testing" + "time" +) + +// The parsing layer, with the machine's files given (novox/hq ADR 0017 allows a fake here and only here); +// the real machine is asked below. + +var files = map[string]string{ + "/proc/sys/kernel/osrelease": "7.2.8-arch1-2\n", + "/etc/os-release": "NAME=\"Arch Linux\"\nPRETTY_NAME=\"Arch Linux\"\nID=arch\nBUILD_ID=rolling\n", + "/proc/cpuinfo": "processor\t: 0\nmodel name\t: AMD Ryzen 9 5900HS\nphysical id\t: 0\ncore id\t\t: 0\n\n" + + "processor\t: 1\nmodel name\t: AMD Ryzen 9 5900HS\nphysical id\t: 0\ncore id\t\t: 0\n\n" + + "processor\t: 2\nmodel name\t: AMD Ryzen 9 5900HS\nphysical id\t: 0\ncore id\t\t: 1\n", + "/proc/meminfo": "MemTotal: 32000000 kB\nMemFree: 1 kB\n", + "/proc/stat": "cpu 1 2 3\nbtime 1791800000\n", + "/proc/self/mounts": "/dev/nvme0n1p2 / btrfs rw 0 0\n/dev/nvme0n1p2 /home btrfs rw 0 0\n" + + "proc /proc proc rw 0 0\ntank/data /srv/my\\040data zfs rw 0 0\n" + + "server:/x /mnt/x nfs4 rw 0 0\n/dev/loop3 /snap/core/1 squashfs ro 0 0\n", +} + +func given(read map[string]string) Sources { + return Sources{ + Read: func(p string) ([]byte, error) { + if s, ok := read[p]; ok { + return []byte(s), nil + } + return nil, &os.PathError{Op: "open", Path: p, Err: os.ErrNotExist} + }, + Readlink: func(string) (string, error) { return "/usr/share/zoneinfo/Europe/Brussels", nil }, + Run: func(context.Context, string, ...string) (string, error) { return "none\n", errors.New("exited 1") }, + Statfs: func(p string) (int64, int64, error) { return 1000, 400, nil }, + Now: func() time.Time { return time.Date(2026, 10, 12, 15, 0, 0, 0, time.UTC) }, + } +} + +func TestMachine_saysEveryFactItCanRead(t *testing.T) { + m := DescribeMachine(context.Background(), given(files)) + + if m.KernelRelease != "7.2.8-arch1-2" || m.Distribution != "Arch Linux" || m.Version != "rolling" { + t.Errorf("kernel %q, distribution %q %q", m.KernelRelease, m.Distribution, m.Version) + } + if m.Processor != "AMD Ryzen 9 5900HS" || m.Cores != 2 || m.Threads != 3 { + t.Errorf("processor %q, %d cores, %d threads", m.Processor, m.Cores, m.Threads) + } + if m.MemoryBytes != 32000000*1024 { + t.Errorf("memory %d", m.MemoryBytes) + } + if m.Virtual != "physical" { + t.Errorf("systemd-detect-virt said none and exited 1; read as %q", m.Virtual) + } + if m.BootedAt == nil || m.BootedAt.Unix() != 1791800000 { + t.Errorf("booted at %v", m.BootedAt) + } + if m.Timezone != "Europe/Brussels" { + t.Errorf("timezone %q", m.Timezone) + } + // One btrfs device under its shortest mount, the zfs dataset with its escaped space, and no network or + // read-only image filesystem. + var mounts []string + for _, f := range m.Filesystems { + mounts = append(mounts, f.Mount) + if f.SizeBytes != 1000 || f.FreeBytes != 400 { + t.Errorf("%s: size %d free %d", f.Mount, f.SizeBytes, f.FreeBytes) + } + } + if got := strings.Join(mounts, ","); got != "/,/srv/my data" { + t.Errorf("filesystems %q", got) + } + // Everything read is not unknown; the public address is, with why. + for fact := range m.Unknown { + if fact != FactPublicAddress { + t.Errorf("%s said unknown though it was readable: %s", fact, m.Unknown[fact]) + } + } + if m.Unknown[FactPublicAddress] == "" { + t.Error("the public address is not said, and not said unknown either") + } +} + +// The acceptance of issue 160: a machine that cannot determine a fact says so rather than reporting a zero. +func TestMachine_aFactItCannotReadIsSaidUnknownNeverZero(t *testing.T) { + src := given(map[string]string{}) + src.Readlink = func(string) (string, error) { return "", os.ErrNotExist } + src.Run = func(context.Context, string, ...string) (string, error) { return "", errors.New("not found") } + m := DescribeMachine(context.Background(), src) + + raw, _ := json.Marshal(m) + var said map[string]any + _ = json.Unmarshal(raw, &said) + for key, v := range said { + if key == "observed_at" || key == "unknown" { + continue + } + t.Errorf("%s is said as %v from a machine where nothing could be read", key, v) + } + for _, fact := range MachineFacts { + if strings.TrimSpace(m.Unknown[fact]) == "" { + t.Errorf("%s is neither said nor said unknown with why", fact) + } + } +} + +func TestMachine_aHungFilesystemCannotHangTheReport(t *testing.T) { + src := given(files) + block := make(chan struct{}) + defer close(block) + src.Statfs = func(string) (int64, int64, error) { <-block; return 0, 0, nil } + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + m := DescribeMachine(ctx, src) + if len(m.Filesystems) != 0 || !strings.Contains(m.Unknown[FactFilesystems], "did not finish") { + t.Errorf("filesystems %v, unknown %q", m.Filesystems, m.Unknown[FactFilesystems]) + } +} + +func TestMachine_coresUnsaidAreUnknownNotGuessed(t *testing.T) { + p := processorOf("processor\t: 0\nModel\t: Raspberry Pi 4\nprocessor\t: 1\n") + if p.model != "Raspberry Pi 4" || p.threads != 2 || p.cores != 0 || p.unknown[FactCores] == "" { + t.Errorf("%+v", p) + } +} + +// Against the real machine (ADR 0017): what it says agrees with what this process sees. +func TestAgainstThisMachine_machineSaysWhatItIs(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skip("the facts are read from Linux's own files") + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + m := DescribeMachine(ctx, RealSources()) + if m.MemoryBytes <= 0 && m.Unknown[FactMemory] == "" { + t.Error("memory neither said nor unknown") + } + if m.Threads != 0 && m.Threads < runtime.NumCPU() { + t.Errorf("%d threads said, and this process alone may use %d", m.Threads, runtime.NumCPU()) + } + if m.BootedAt != nil && m.BootedAt.After(time.Now()) { + t.Errorf("booted in the future: %v", m.BootedAt) + } + for _, f := range m.Filesystems { + if f.SizeBytes <= 0 || f.FreeBytes > f.SizeBytes { + t.Errorf("%s: size %d free %d", f.Mount, f.SizeBytes, f.FreeBytes) + } + } + for _, fact := range MachineFacts { + if why, unknown := m.Unknown[fact]; unknown && strings.TrimSpace(why) == "" { + t.Errorf("%s unknown with no why", fact) + } + } + raw, _ := json.MarshalIndent(m, "", " ") + t.Logf("%s", raw) +} diff --git a/internal/profile/profile.go b/internal/profile/profile.go index ace9572..333d515 100644 --- a/internal/profile/profile.go +++ b/internal/profile/profile.go @@ -68,6 +68,9 @@ type Profile struct { Architecture string `json:"architecture"` Kernel string `json:"kernel"` Capabilities []Verdict `json:"capabilities"` + // Machine is what it is made of (novox/hq issue 160), said with every profile: at enrolment, after + // every apply, and by the reconcile once a day when nothing else made it speak. + Machine *Machine `json:"machine,omitempty"` } // Has reports whether a named capability is present. Unknown names are absent, not an error: @@ -114,5 +117,9 @@ func Detect(ctx context.Context, detectors []Detector, timeout time.Duration) Pr sort.Slice(p.Capabilities, func(i, j int) bool { return p.Capabilities[i].Name < p.Capabilities[j].Name }) + describeCtx, cancel := context.WithTimeout(ctx, timeout) + m := DescribeMachine(describeCtx, RealSources()) + cancel() + p.Machine = &m return p } From c7a4bf5c11d6201af3797aa80fe38794f1624f83 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 12 Oct 2026 17:42:18 +0200 Subject: [PATCH 2/2] Judge the daily report by wall time, and keep one filesystem check at a time (issue 160) Go's monotonic clock stops while a laptop is suspended, so a day judged by it could pass a day and a half of wall time before the machine said what it is. And a hung statfs left a goroutine and thread blocked per apply; now one check runs at a time and reports after it say the sizes unknown until it ends. --- cmd/mesh-host/machine_refresh_test.go | 43 +++++++++++++++++++++++++++ cmd/mesh-host/main.go | 7 +++-- internal/profile/machine.go | 12 ++++++++ internal/profile/machine_test.go | 28 +++++++++++++++-- 4 files changed, 86 insertions(+), 4 deletions(-) diff --git a/cmd/mesh-host/machine_refresh_test.go b/cmd/mesh-host/machine_refresh_test.go index 3bcab8e..91c2dda 100644 --- a/cmd/mesh-host/machine_refresh_test.go +++ b/cmd/mesh-host/machine_refresh_test.go @@ -1,6 +1,7 @@ package main import ( + "strings" "testing" "time" @@ -62,3 +63,45 @@ func TestARefusalIsNotSaidForTheMachinesSake(t *testing.T) { t.Error("a refusal was due for what the machine is made of") } } + +// A laptop that slept still says what it is within a day of wall time (review of mesh-host#92). Go's monotonic +// clock stops while a machine is suspended, and Sub between two times that both carry a monotonic reading uses +// it, so a watch that kept one would count a day asleep as almost nothing. Simulated with a clock that first +// reads now (a monotonic reading), and after the sleep reads the wall time a day on: the watch must keep no +// monotonic reading, and judge the day as the wall clock says it. +func TestALaptopThatSleptStillSaysWhatItIsWithinADay(t *testing.T) { + before := time.Now() // carries a monotonic reading + asleep := before.Add(3 * time.Second) + if asleep.Sub(before) != 3*time.Second { + t.Fatal("the monotonic reading is not what this test simulates with") + } + calls := 0 + w := &adoptionWatch{now: func() time.Time { + calls++ + if calls == 1 { + return before + } + // The wall clock a day on, as a machine's clock reads after waking. + return asleep.Round(0).Add(24 * time.Hour) + }} + report := link.Report{Outward: []string{"wlan0"}, Profile: map[string]any{"capabilities": []any{}}} + w.said(report) + if strings.Contains(w.machineSaid.String(), "m=") { + t.Fatalf("the watch kept a monotonic reading (%s): a sleep would not count toward the day", w.machineSaid) + } + if !w.news(report) { + t.Error("a day of wall time on, a laptop that slept did not say what it is") + } +} + +// And the watch's own clock never carries a monotonic reading. +func TestTheWatchClockIsWallTime(t *testing.T) { + w := &adoptionWatch{} + if s := w.clock().String(); strings.Contains(s, "m=") { + t.Errorf("the watch's clock carries a monotonic reading: %s", s) + } + w = &adoptionWatch{now: time.Now} + if s := w.clock().String(); strings.Contains(s, "m=") { + t.Errorf("a given clock's time kept its monotonic reading: %s", s) + } +} diff --git a/cmd/mesh-host/main.go b/cmd/mesh-host/main.go index 9638706..1f4746c 100644 --- a/cmd/mesh-host/main.go +++ b/cmd/mesh-host/main.go @@ -1364,11 +1364,14 @@ type adoptionWatch struct { // inside the day rather than just past it. const machineEvery = 23 * time.Hour +// clock is the wall time, its monotonic reading stripped (Round(0)). **Wall, because a laptop sleeps**: Go's +// monotonic clock does not advance while the machine is suspended, so a day judged by it on a laptop that slept +// for half of it is a day and a half of wall time, and what the mesh holds of the machine is older than a day. func (w *adoptionWatch) clock() time.Time { if w.now != nil { - return w.now() + return w.now().Round(0) } - return time.Now() + return time.Now().Round(0) } // machineDue is whether a report carries what the machine is made of and the mesh last heard it a day ago diff --git a/internal/profile/machine.go b/internal/profile/machine.go index db09a3e..f67a168 100644 --- a/internal/profile/machine.go +++ b/internal/profile/machine.go @@ -7,6 +7,7 @@ import ( "sort" "strconv" "strings" + "sync" "syscall" "time" ) @@ -362,6 +363,9 @@ var localTypes = map[string]bool{"ext2": true, "ext3": true, "ext4": true, "xfs" "zfs": true, "f2fs": true, "vfat": true, "exfat": true, "ntfs": true, "ntfs3": true, "fuseblk": true, "jfs": true, "reiserfs": true, "bcachefs": true} +// sizing is held while a check of the filesystems' sizes runs. +var sizing sync.Mutex + // filesystemsOf lists each local filesystem once — a device mounted at several places (a btrfs // subvolume, a bind mount) under its shortest mount — with its size and what is free. Bounded by the // context, so a filesystem whose statfs hangs cannot hang the apply that reports it. @@ -385,12 +389,20 @@ func filesystemsOf(ctx context.Context, mounts string, stat func(string) (int64, return nil, "/proc/self/mounts lists no local filesystem" } + // One check at a time, across every report: a filesystem whose statfs hangs keeps the one check that + // asked it, and every report after it says the sizes unknown until it finishes, rather than leaving + // another goroutine and thread blocked behind it on every apply. + if !sizing.TryLock() { + return nil, "the last check of the filesystems' sizes has not finished yet" + } + type answer struct { fs []Filesystem why []string } done := make(chan answer, 1) go func() { + defer sizing.Unlock() var a answer for _, fs := range byDevice { size, free, err := stat(fs.Mount) diff --git a/internal/profile/machine_test.go b/internal/profile/machine_test.go index fe32e64..9cf9e17 100644 --- a/internal/profile/machine_test.go +++ b/internal/profile/machine_test.go @@ -7,6 +7,7 @@ import ( "os" "runtime" "strings" + "sync/atomic" "testing" "time" ) @@ -112,14 +113,37 @@ func TestMachine_aFactItCannotReadIsSaidUnknownNeverZero(t *testing.T) { func TestMachine_aHungFilesystemCannotHangTheReport(t *testing.T) { src := given(files) block := make(chan struct{}) - defer close(block) - src.Statfs = func(string) (int64, int64, error) { <-block; return 0, 0, nil } + var asked atomic.Int32 + src.Statfs = func(string) (int64, int64, error) { asked.Add(1); <-block; return 0, 0, nil } + defer func() { + close(block) + // The check that hung ends once its filesystem answers, and lets the next one run. + for !sizing.TryLock() { + time.Sleep(time.Millisecond) + } + sizing.Unlock() + }() ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) defer cancel() m := DescribeMachine(ctx, src) if len(m.Filesystems) != 0 || !strings.Contains(m.Unknown[FactFilesystems], "did not finish") { t.Errorf("filesystems %v, unknown %q", m.Filesystems, m.Unknown[FactFilesystems]) } + stuck := asked.Load() + + // Every report while it hangs says so at once, and asks no filesystem again: no goroutine is left + // blocked behind the hung one per apply. + for i := 0; i < 3; i++ { + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + m = DescribeMachine(ctx, src) + cancel() + if !strings.Contains(m.Unknown[FactFilesystems], "has not finished yet") { + t.Errorf("a report while the check hangs said %q", m.Unknown[FactFilesystems]) + } + } + if asked.Load() != stuck { + t.Errorf("filesystems asked %d times while one check hung, was %d", asked.Load(), stuck) + } } func TestMachine_coresUnsaidAreUnknownNotGuessed(t *testing.T) {