Count each processor package once, never read a sleeping card, and give no reading for a reset counter (hq ADR 0255)
Intel shows some packages twice (MSR and MMIO), an AMD card's sensors wake it, and a counter reset after resume read as thousands of watts. The loop's cadence and the bar script's staleness are now tested, and the sampler running on every machine is said.
This commit is contained in:
@@ -60,11 +60,22 @@ type counter struct {
|
||||
uj, max int64
|
||||
}
|
||||
|
||||
// packageCounters is every processor package's energy counter, by its directory.
|
||||
// MaxPackageWatts is more than any processor package draws. A rate above it is a counter that was
|
||||
// reset — after a resume, say — not power, and that sample gives no reading.
|
||||
const MaxPackageWatts = 1000
|
||||
|
||||
// packageCounters is every processor package's energy counter, by the package's name, each counted
|
||||
// once. Intel exposes some packages twice, through MSRs (`intel-rapl:N`) and through MMIO
|
||||
// (`intel-rapl-mmio:N`), under one name; the MSR one is read, and the second never adds to it.
|
||||
func (m *Machine) packageCounters() map[string]counter {
|
||||
out := map[string]counter{}
|
||||
from := map[string]string{}
|
||||
for _, dir := range m.glob("/sys/class/powercap/*") {
|
||||
if !strings.HasPrefix(m.read(dir+"/name"), "package") {
|
||||
name := m.read(dir + "/name")
|
||||
if !strings.HasPrefix(name, "package") {
|
||||
continue
|
||||
}
|
||||
if was, seen := from[name]; seen && !strings.Contains(filepath.Base(was), "mmio") {
|
||||
continue
|
||||
}
|
||||
uj, ok := m.readInt(dir + "/energy_uj")
|
||||
@@ -72,13 +83,15 @@ func (m *Machine) packageCounters() map[string]counter {
|
||||
continue
|
||||
}
|
||||
max, _ := m.readInt(dir + "/max_energy_range_uj")
|
||||
out[dir] = counter{uj: uj, max: max}
|
||||
out[name] = counter{uj: uj, max: max}
|
||||
from[name] = dir
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// cpuWatts is the packages' power between two readings `elapsed` apart; nil when no counter was read
|
||||
// both times. A counter that wrapped is counted across its range.
|
||||
// both times. A counter that wrapped is counted across its range; one that gives a rate no package
|
||||
// draws was reset, and the sample gives nothing rather than a number that is not power.
|
||||
func cpuWatts(before, after map[string]counter, elapsed time.Duration) *float64 {
|
||||
if elapsed <= 0 {
|
||||
return nil
|
||||
@@ -97,6 +110,9 @@ func cpuWatts(before, after map[string]counter, elapsed time.Duration) *float64
|
||||
}
|
||||
delta += a.max
|
||||
}
|
||||
if float64(delta)/1e6/elapsed.Seconds() > MaxPackageWatts {
|
||||
return nil
|
||||
}
|
||||
joules += float64(delta) / 1e6
|
||||
read = true
|
||||
}
|
||||
@@ -156,6 +172,10 @@ func (m *Machine) gpuWatts(ctx context.Context) *float64 {
|
||||
if strings.Contains(filepath.Base(card), "-") || !m.exists(card+"/device/mem_info_vram_vendor") {
|
||||
continue
|
||||
}
|
||||
// Asleep, it is not read: reading its sensors wakes it, as asking nvidia-smi would.
|
||||
if status := m.read(card + "/device/power/runtime_status"); status != "" && status != "active" {
|
||||
continue
|
||||
}
|
||||
for _, hw := range m.glob(card + "/device/hwmon/hwmon*") {
|
||||
uw, ok := m.readInt(hw + "/power1_average")
|
||||
if !ok {
|
||||
@@ -217,9 +237,14 @@ func (m *Machine) Publish(line string) error {
|
||||
return os.Rename(tmp, path)
|
||||
}
|
||||
|
||||
// RunDraw samples and publishes until the context ends. A battery reading is instant, so it waits
|
||||
// the interval itself; a counter reading waits inside its sample.
|
||||
// RunDraw samples and publishes until the context ends.
|
||||
func RunDraw(ctx context.Context, m *Machine) error {
|
||||
return runDraw(ctx, m, SampleEvery, m.Publish)
|
||||
}
|
||||
|
||||
// runDraw publishes one reading every `every`. A battery reading is instant, so it waits the interval
|
||||
// itself; a counter reading waits inside its sample. It returns as soon as the context ends.
|
||||
func runDraw(ctx context.Context, m *Machine, every time.Duration, publish func(string) error) error {
|
||||
wait := func(d time.Duration) bool {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
@@ -229,14 +254,14 @@ func RunDraw(ctx context.Context, m *Machine) error {
|
||||
}
|
||||
}
|
||||
for ctx.Err() == nil {
|
||||
d := m.Sample(ctx, SampleEvery, wait)
|
||||
d := m.Sample(ctx, every, wait)
|
||||
if ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
if err := m.Publish(d.Line()); err != nil {
|
||||
if err := publish(d.Line()); err != nil {
|
||||
return fmt.Errorf("publishing the draw: %w", err)
|
||||
}
|
||||
if d.Machine != nil && !wait(SampleEvery) {
|
||||
if d.Machine != nil && !wait(every) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -61,8 +62,10 @@ func TestOnMainsTheProcessorIsMeasuredAndCalledTheProcessor(t *testing.T) {
|
||||
"/sys/class/drm/card1/device/hwmon/hwmon5/power1_average": "19000000",
|
||||
})
|
||||
counter := filepath.Join(m.Root, "/sys/class/powercap/intel-rapl:0/energy_uj")
|
||||
// 1 J over 50 ms: 20 W, well inside what a package draws.
|
||||
d := m.Sample(context.Background(), time.Second, func(time.Duration) bool {
|
||||
return os.WriteFile(counter, []byte("11000000\n"), 0o644) == nil
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
return os.WriteFile(counter, []byte("2000000\n"), 0o644) == nil
|
||||
})
|
||||
if d.Machine != nil || d.CPU == nil || d.GPU != nil || *d.CPU <= 0 {
|
||||
t.Fatalf("%+v", d)
|
||||
@@ -135,3 +138,102 @@ func TestOnlyTheRateIsPublishedReadableByAnyone(t *testing.T) {
|
||||
t.Fatalf("%q", b)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPackageSeenTwiceIsCountedOnce(t *testing.T) {
|
||||
m := tree(t, map[string]string{
|
||||
"/sys/class/powercap/intel-rapl:0/name": "package-0",
|
||||
"/sys/class/powercap/intel-rapl:0/energy_uj": "100",
|
||||
"/sys/class/powercap/intel-rapl-mmio:0/name": "package-0",
|
||||
"/sys/class/powercap/intel-rapl-mmio:0/energy_uj": "999",
|
||||
"/sys/class/powercap/intel-rapl:1/name": "package-1",
|
||||
"/sys/class/powercap/intel-rapl:1/energy_uj": "5",
|
||||
})
|
||||
got := m.packageCounters()
|
||||
if len(got) != 2 || got["package-0"].uj != 100 || got["package-1"].uj != 5 {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestACounterResetGivesNoReading(t *testing.T) {
|
||||
// After a resume the counter starts again near nothing: across its range that would be thousands
|
||||
// of watts, which is no reading at all.
|
||||
before := map[string]counter{"package-0": {uj: 40_000_000_000, max: 65_532_610_987}}
|
||||
after := map[string]counter{"package-0": {uj: 1_000_000, max: 65_532_610_987}}
|
||||
if w := cpuWatts(before, after, 5*time.Second); w != nil {
|
||||
t.Fatalf("a reset counter read as %.0f W", *w)
|
||||
}
|
||||
// A true wrap, near the end of the range, is still counted.
|
||||
before = map[string]counter{"package-0": {uj: 65_532_610_987 - 10_000_000, max: 65_532_610_987}}
|
||||
after = map[string]counter{"package-0": {uj: 40_000_000, max: 65_532_610_987}}
|
||||
if w := cpuWatts(before, after, 5*time.Second); w == nil || *w != 10 {
|
||||
t.Fatalf("a wrapped counter read as %v", w)
|
||||
}
|
||||
}
|
||||
|
||||
func TestASleepingAMDCardIsNotRead(t *testing.T) {
|
||||
m := tree(t, map[string]string{
|
||||
"/sys/class/drm/card0/device/mem_info_vram_vendor": "samsung",
|
||||
"/sys/class/drm/card0/device/power/runtime_status": "suspended",
|
||||
"/sys/class/drm/card0/device/hwmon/hwmon3/power1_average": "30000000",
|
||||
})
|
||||
if w := m.gpuWatts(context.Background()); w != nil {
|
||||
t.Fatalf("a suspended card was read: %v", *w)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheLoopPublishesEveryIntervalAndStopsWhenCancelled(t *testing.T) {
|
||||
m := tree(t, map[string]string{
|
||||
"/sys/class/power_supply/BAT0/type": "Battery",
|
||||
"/sys/class/power_supply/BAT0/status": "Discharging",
|
||||
"/sys/class/power_supply/BAT0/power_now": "12000000",
|
||||
})
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
var lines []string
|
||||
done := make(chan error)
|
||||
go func() {
|
||||
done <- runDraw(ctx, m, 20*time.Millisecond, func(l string) error { lines = append(lines, l); return nil })
|
||||
}()
|
||||
time.Sleep(210 * time.Millisecond)
|
||||
cancel()
|
||||
select {
|
||||
case err := <-done:
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("the loop did not stop when cancelled")
|
||||
}
|
||||
// One at once, then one every 20 ms: about eleven in 210 ms, never a burst, never a stall.
|
||||
if len(lines) < 6 || len(lines) > 12 || lines[0] != "12.0 W machine" {
|
||||
t.Fatalf("%d lines: %v", len(lines), lines)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheBarsScriptShowsOnlyAFreshLine(t *testing.T) {
|
||||
file := filepath.Join(t.TempDir(), "draw")
|
||||
run := func() string {
|
||||
cmd := exec.Command("sh", "../../files/bin/power-draw")
|
||||
cmd.Env = append(os.Environ(), "MESH_POWER_DRAW="+file)
|
||||
out, err := cmd.Output()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return strings.TrimSpace(string(out))
|
||||
}
|
||||
if got := run(); got != "power not measured" {
|
||||
t.Fatalf("no file: %q", got)
|
||||
}
|
||||
if err := os.WriteFile(file, []byte("8.0 W CPU\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := run(); got != "8.0 W CPU" {
|
||||
t.Fatalf("fresh: %q", got)
|
||||
}
|
||||
old := time.Now().Add(-31 * time.Second)
|
||||
if err := os.Chtimes(file, old, old); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := run(); got != "power not measured" {
|
||||
t.Fatalf("31 s old: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user