From 9254180cab157ee97522c43c3cf53de5ec26f89c Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 26 Oct 2026 14:21:24 +0100 Subject: [PATCH] A yielding process runs in the node's one yielding slice, which holds the shared quota (ADR 0553, issue 1806) Review round 1: one quota per unit let three build modules take three shares. The engine now writes meshyield.slice, top-level so its idle weight binds against the operator's session, with the node's share as its CPUQuota; each yielding unit joins it at nice 19. Comments say the IO controls bind only under bfq or io.cost, not under the none scheduler NVMe disks use. --- go.mod | 2 +- go.sum | 4 +- internal/apply/process.go | 73 +++++++++++++++++++++++++++++----- internal/apply/process_test.go | 48 ++++++++++++++++------ 4 files changed, 102 insertions(+), 25 deletions(-) diff --git a/go.mod b/go.mod index b8a3104..1aa503c 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/novox/mesh-host go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026124909-76deab6c67c2 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026132005-852ba81c5af1 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 golang.org/x/crypto v0.57.0 diff --git a/go.sum b/go.sum index 6ca7475..1679d42 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026124909-76deab6c67c2 h1:DugsX7h46z+1FOiNhtDE43Vf1WInz2vgwi0mn/1VZDg= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026124909-76deab6c67c2/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026132005-852ba81c5af1 h1:6O9WKnlVR1uHV5DDQueHGbMAQzfVS06J7DBKjUPBG4E= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026132005-852ba81c5af1/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/internal/apply/process.go b/internal/apply/process.go index c1a89a0..ef07f4c 100644 --- a/internal/apply/process.go +++ b/internal/apply/process.go @@ -95,6 +95,14 @@ func applyProcess(ctx context.Context, r *declaration.Process, run Runner, if r.Schedule != "" { unit = r.Name + ".timer" } + // The yielding slice is checked even then: a slice file removed by hand would leave the build unbound. + if wrote, err := placeYieldSlice(r); err != nil { + return out, err + } else if wrote { + if _, err := run(ctx, "systemctl", "daemon-reload"); err != nil { + return out, err + } + } if active, err := run(ctx, "systemctl", "is-active", "--quiet", unit); err == nil { _ = active return out, nil @@ -190,6 +198,9 @@ func applyProcess(ctx context.Context, r *declaration.Process, run Runner, if err := os.WriteFile(unit, []byte(unitFor(r)), 0o644); err != nil { return out, err } + if _, err := placeYieldSlice(r); err != nil { + return out, err + } if _, err := run(ctx, "systemctl", "daemon-reload"); err != nil { return out, err } @@ -299,30 +310,72 @@ func unitFor(r *declaration.Process) string { // fix it. var machineCPUs = runtime.NumCPU -// writeYields writes how a process gives way to everything else on its machine (novox/hq issue 1806): a build seat's -// holder, whose builds must never slow the operator's desktop. +// writeYields writes how a process gives way to everything else on its machine (novox/hq ADR 0553, issue 1806): a +// build seat's holder, whose builds must never slow the operator's desktop. // -// **Idle priority** is four lines, because one alone leaves a door open: CPUWeight=idle runs its cgroup only on cycles -// its siblings leave, Nice=19 holds every process it starts below the machine's own within the group, IOWeight=1 and -// the lowest best-effort IO level put its disk work behind everyone's. Not the idle IO class: a disk that is never idle -// would starve a build for good, and a build that never ends holds a check for ever. +// **Into the node's one yielding slice**, where the CPU weight and the quota are (yieldSlice): every yielding process +// of the machine shares them, so three build modules take one share together, never one each. **Per process**, what a +// slice cannot say: Nice=19 for every process it starts, and the lowest best-effort IO level. The IO level binds only +// where the disk's scheduler honours it (bfq); under `none`, the usual one for NVMe, it does nothing. Not the idle IO +// class: a disk that is never idle would starve a build for good. // -// **The share** is a hard bound: CPUQuota counts in percent of one processor, so a share of the machine is written for -// the processors it has (half of 24 is 1200%), at least one processor. +// The share is written into the unit as a comment, so a changed share changes the unit, the daemon's identity, and the +// slice is written again. func writeYields(b *strings.Builder, y *declaration.Yields) { if y == nil { return } + fmt.Fprintf(b, "# Yields: %s, %d%% of the machine shared in %s\n", y.Priority, y.CPUShare, declaration.YieldSlice) + fmt.Fprintf(b, "Slice=%s\n", declaration.YieldSlice) if y.Priority == declaration.YieldIdle { - b.WriteString("CPUWeight=idle\nIOWeight=1\nNice=19\nIOSchedulingClass=best-effort\nIOSchedulingPriority=7\n") + b.WriteString("Nice=19\nIOSchedulingClass=best-effort\nIOSchedulingPriority=7\n") + } +} + +// yieldSlice is the node's one yielding slice (declaration.YieldSlice) for a process that yields: idle CPU weight and +// IO weight 1 for an idle priority, and CPUQuota for its share of the processors this machine has (systemd counts a +// quota in percent of one processor, so half of 24 is 1200%), at least one processor. At the top of the hierarchy, so +// its weight and quota bind against the operator's user slice and the system's. +// +// The IO weight binds only where the disk's scheduler honours it (bfq, or the io.cost controller); under `none` it does +// nothing, and the CPU bound is what holds. +// +// Every yielding process of a machine carries the node's one share (the composer's, ADR 0553 §3), so whichever writes +// it writes the same file. Left in place when nothing yields any more: an empty slice binds nothing. +func yieldSlice(y *declaration.Yields) string { + var b strings.Builder + b.WriteString("# Generated by the mesh. Do not edit — this file is replaced whenever a yielding\n") + b.WriteString("# process is declared again (novox/hq ADR 0553).\n") + b.WriteString("[Unit]\nDescription=The mesh's yielding processes: builds, at idle priority under one share\n\n") + b.WriteString("[Slice]\n") + if y.Priority == declaration.YieldIdle { + b.WriteString("CPUWeight=idle\nIOWeight=1\n") } if y.CPUShare > 0 { quota := y.CPUShare * machineCPUs() if quota < 100 { quota = 100 } - fmt.Fprintf(b, "CPUQuota=%d%%\n", quota) + fmt.Fprintf(&b, "CPUQuota=%d%%\n", quota) } + return b.String() +} + +// placeYieldSlice writes the yielding slice for a process that yields, when what is there differs, and says whether it +// wrote: the caller reloads the service manager before it starts anything. +func placeYieldSlice(r *declaration.Process) (bool, error) { + if r.Yields == nil { + return false, nil + } + path := filepath.Join(unitDir, declaration.YieldSlice) + want := yieldSlice(r.Yields) + if have, err := os.ReadFile(path); err == nil && string(have) == want { + return false, nil + } + if err := os.WriteFile(path, []byte(want), 0o644); err != nil { + return false, fmt.Errorf("the yielding slice %s: %w", path, err) + } + return true, nil } // timerFor is the cadence a scheduled process runs on. diff --git a/internal/apply/process_test.go b/internal/apply/process_test.go index f341e0c..c99eef7 100644 --- a/internal/apply/process_test.go +++ b/internal/apply/process_test.go @@ -84,34 +84,58 @@ func TestAProcessRunsAsWhoItSaid(t *testing.T) { t.Fatalf("a process that named no user had one written for it:\n%s", unitFor(aProcess())) } - // A process that yields (issue 1806) is written at idle priority under its share of the machine's processors; one - // that does not has none of it. + // A process that yields (ADR 0553) runs in the node's one yielding slice at nice 19; the slice holds the idle weight + // and the quota for its share of the machine's processors; one that does not yield has none of it. defer func(n func() int) { machineCPUs = n }(machineCPUs) for _, c := range []struct { - cpus int - y *declaration.Yields - want []string - never []string + cpus int + y *declaration.Yields + unit, slice, not []string }{ {24, &declaration.Yields{Priority: declaration.YieldIdle, CPUShare: 50}, - []string{"CPUWeight=idle\n", "IOWeight=1\n", "Nice=19\n", "IOSchedulingPriority=7\n", "CPUQuota=1200%\n"}, nil}, - {1, &declaration.Yields{CPUShare: 25}, []string{"CPUQuota=100%\n"}, []string{"Nice="}}, - {24, nil, nil, []string{"CPUWeight=", "Nice=", "CPUQuota=", "IOWeight="}}, + []string{"Slice=meshyield.slice\n", "Nice=19\n", "IOSchedulingPriority=7\n", "50%"}, + []string{"CPUWeight=idle\n", "IOWeight=1\n", "CPUQuota=1200%\n"}, []string{"CPUQuota=", "CPUWeight="}}, + {1, &declaration.Yields{CPUShare: 25}, []string{"Slice=meshyield.slice\n"}, []string{"CPUQuota=100%\n"}, + []string{"Nice="}}, + {24, nil, nil, nil, []string{"Slice=", "Nice=", "CPUQuota="}}, } { machineCPUs = func() int { return c.cpus } p := aProcess() p.Yields = c.y unit := unitFor(p) - for _, w := range c.want { + for _, w := range c.unit { if !strings.Contains(unit, w) { - t.Errorf("yields %+v on %d processors: the unit does not say %q:\n%s", c.y, c.cpus, w, unit) + t.Errorf("yields %+v: the unit does not say %q:\n%s", c.y, w, unit) } } - for _, n := range c.never { + for _, n := range c.not { if strings.Contains(unit, n) { t.Errorf("yields %+v: the unit says %q:\n%s", c.y, n, unit) } } + if c.y == nil { + continue + } + slice := yieldSlice(c.y) + for _, w := range c.slice { + if !strings.Contains(slice, w) { + t.Errorf("yields %+v on %d processors: the slice does not say %q:\n%s", c.y, c.cpus, w, slice) + } + } + } + // The slice is placed beside the units, once, and again only when it differs. + was := unitDir + unitDir = t.TempDir() + defer func() { unitDir = was }() + p := aProcess() + p.Yields = &declaration.Yields{Priority: declaration.YieldIdle, CPUShare: 50} + for i, want := range []bool{true, false} { + if wrote, err := placeYieldSlice(p); err != nil || wrote != want { + t.Errorf("placing %d: wrote %v (%v), want %v", i, wrote, err, want) + } + } + if b, err := os.ReadFile(filepath.Join(unitDir, "meshyield.slice")); err != nil || !strings.Contains(string(b), "[Slice]") { + t.Errorf("the slice file: %q %v", b, err) } }