Kept composers and programs get a list tool, a drop tool and a sweep (issue 1921, seams must 2)

Claude-Session: https://claude.ai/code/session_01NcbgfFSnn24M3GcqttoU7i

modules/mesh-build-go/README.md

@@ -49,6 +49,10 @@ already gone is no error, one a running step holds is refused); its caller calls
 failed fetch removes its own workspace. A workspace no caller released is removed two hours after its last use (ADR
 0509), and the removal is said in the log as the caller's fault.
 
+The programs this module builds from a commit and keeps in its cache (composers of compose asks, checking programs of run
+asks) are read by `build_go_kept` (kind, key, size, last use) and removed by `build_go_kept_drop {kind, key}`; one no ask used
+for 30 days is swept with the workspaces (issue 1921).
+
 A pin on a pull request's head (issue 1148): when `go mod download` fails on a `git.novox.be/novox/*` pseudo-version, the
 fetch looks for that commit among the forge's `refs/pull/*/head`, places the repository at it under the workspace's
 `pinned/beside/` as a plain directory, and hands it to Go (the fetch's commands, and the build with `GOPROXY=off`) through a

modules/mesh-build-go/cmd/mesh-build-go/compose.go

@@ -38,7 +38,7 @@ var composerLocks sync.Map
 
 // composerPath is where the composer of key is kept: the cache directory, beside Go's own caches.
 func (m *Module) composerPath(key string) string {
-	return filepath.Join(m.Cache, "composers", key, "mesh-compose")
+	return m.programPath("composers", key)
 }
 
 // compose runs a compose ask: the composer of its commit, built once, over its request with the manifests filled.
@@ -99,10 +99,11 @@ func (m *Module) manifestOf(digest string) ([]byte, error) {
 // composerOf is the composer kept under key, or, when none is, the one built now from the fetched commit (built true).
 func (m *Module) composerOf(ctx context.Context, fetch buildstep.StepFunc, job delivery.ComposeJob, key string) (string, bool, error) {
 	exe := m.composerPath(key)
-	mu, _ := composerLocks.LoadOrStore(exe, &sync.Mutex{})
-	mu.(*sync.Mutex).Lock()
-	defer mu.(*sync.Mutex).Unlock()
+	mu := lockOf(exe)
+	mu.Lock()
+	defer mu.Unlock()
 	if st, err := os.Stat(exe); err == nil && st.Mode().IsRegular() {
+		touch(exe, m.clock())
 		return exe, false, nil
 	}
 	if fetch == nil {

modules/mesh-build-go/cmd/mesh-build-go/fetch.go

@@ -348,6 +348,7 @@ func (m *Module) goEnvFor(dir string) []string {
 // net for a caller that never released one (issue 1839, ADR 0509).
 func (m *Module) CleanWorkspaces(now time.Time) {
 	m.sweepSaid(m.spaces().Sweep(now, buildstep.UnheldBound))
+	m.SweepKept(now, KeptBound)
 }
 
 // clock is the time now: Now when a test set it, else the wall clock.

modules/mesh-build-go/cmd/mesh-build-go/kept.go

@@ -0,0 +1,130 @@
+package main
+
+// **Kept programs have tools and a sweep** (novox/hq issue 1921, the seams review's must 2): the programs this module builds
+// from a commit and keeps under its cache so each commit is built once on a node, the composers of compose asks and the
+// programs of run asks, are read by build_go_kept, removed one by one by build_go_kept_drop, and swept when unused for
+// KeptBound. A use touches the program's file, so its modification time is its last use.
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"sort"
+	"sync"
+	"time"
+
+	stdio "git.novox.be/novox/mesh-sdk/go"
+)
+
+// The tools of the kept programs.
+const (
+	KeptTool     = "build_go_kept"
+	KeptDropTool = "build_go_kept_drop"
+)
+
+// KeptBound is how long a kept program that no ask used is kept: a month, longer than any group lives, so only the build
+// of a commit nobody asks about any more goes.
+const KeptBound = 30 * 24 * time.Hour
+
+// keptKinds are the cache directories of kept programs and the name of the program in each key's directory.
+var keptKinds = map[string]string{"composers": "mesh-compose", "programs": "program"}
+
+// KeptProgram is one kept program: its kind (the cache directory), its key, its size and when an ask last used it.
+type KeptProgram struct {
+	Kind     string    `json:"kind"`
+	Key      string    `json:"key"`
+	Bytes    int64     `json:"bytes"`
+	LastUsed time.Time `json:"last-used"`
+}
+
+// programPath is where the program of kind and key is kept.
+func (m *Module) programPath(kind, key string) string {
+	return filepath.Join(m.Cache, kind, key, keptKinds[kind])
+}
+
+// lockOf is the mutex that guards the program at path against a build or a use while it is dropped.
+func lockOf(path string) *sync.Mutex {
+	l, _ := composerLocks.LoadOrStore(path, &sync.Mutex{})
+	return l.(*sync.Mutex)
+}
+
+// touch records a use of the kept program at path.
+func touch(path string, at time.Time) { _ = os.Chtimes(path, at, at) }
+
+// Kept lists the kept programs, composers then programs, each kind by key.
+func (m *Module) Kept() []KeptProgram {
+	out := []KeptProgram{}
+	for _, kind := range []string{"composers", "programs"} {
+		entries, _ := os.ReadDir(filepath.Join(m.Cache, kind))
+		for _, e := range entries {
+			if !e.IsDir() {
+				continue
+			}
+			st, err := os.Stat(m.programPath(kind, e.Name()))
+			if err != nil || !st.Mode().IsRegular() {
+				continue
+			}
+			out = append(out, KeptProgram{Kind: kind, Key: e.Name(), Bytes: st.Size(), LastUsed: st.ModTime().UTC()})
+		}
+	}
+	return out
+}
+
+// DropKept removes the kept program of kind and key; one already gone is no error.
+func (m *Module) DropKept(kind, key string) error {
+	if _, ok := keptKinds[kind]; !ok {
+		return fmt.Errorf("a kept program is of kind composers or programs, not %q", kind)
+	}
+	if key == "" || key != filepath.Base(key) || key == "." || key == ".." {
+		return fmt.Errorf("%q is no kept program's key", key)
+	}
+	exe := m.programPath(kind, key)
+	l := lockOf(exe)
+	l.Lock()
+	defer l.Unlock()
+	return os.RemoveAll(filepath.Dir(exe))
+}
+
+// SweepKept removes the kept programs no ask used within bound before now, and answers what it removed.
+func (m *Module) SweepKept(now time.Time, bound time.Duration) []KeptProgram {
+	var gone []KeptProgram
+	for _, k := range m.Kept() {
+		if now.Sub(k.LastUsed) <= bound {
+			continue
+		}
+		if m.DropKept(k.Kind, k.Key) == nil {
+			gone = append(gone, k)
+		}
+	}
+	sort.Slice(gone, func(i, j int) bool { return gone[i].Key < gone[j].Key })
+	if len(gone) > 0 && m.Say != nil {
+		m.Say("removed %d kept programs no ask used within %s", len(gone), bound)
+	}
+	return gone
+}
+
+// keptTools are the tools that read and drop the kept programs: no guard and no floor, since they only free room.
+func (m *Module) keptTools() []stdio.Tool {
+	return []stdio.Tool{
+		{Name: KeptTool,
+			Description: "The programs this node keeps, built from a commit so each commit is built once: the composers of compose asks and " +
+				"the checking programs of run asks, each with its kind, key, size in bytes and when an ask last used it. One no ask used " +
+				"for 30 days is removed. Read-only.",
+			Run: func(map[string]any) (any, error) { return map[string]any{"kept": m.Kept()}, nil }},
+		{Name: KeptDropTool,
+			Description: "Remove one kept program by its kind (composers or programs) and key, as build_go_kept names them: the next ask " +
+				"for that commit builds it again. One already gone is no error.",
+			Input: map[string]any{"type": "object", "properties": map[string]any{
+				"kind": map[string]any{"type": "string", "enum": []string{"composers", "programs"}},
+				"key":  map[string]any{"type": "string", "description": "the key build_go_kept names"},
+			}, "required": []string{"kind", "key"}},
+			Run: func(args map[string]any) (any, error) {
+				kind, _ := args["kind"].(string)
+				key, _ := args["key"].(string)
+				if err := m.DropKept(kind, key); err != nil {
+					return nil, err
+				}
+				return map[string]any{"dropped": key, "kind": kind}, nil
+			}},
+	}
+}

modules/mesh-build-go/cmd/mesh-build-go/kept_test.go

@@ -0,0 +1,71 @@
+package main
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+)
+
+func keepProgram(t *testing.T, m *Module, kind, key string, size int, used time.Time) {
+	t.Helper()
+	exe := m.programPath(kind, key)
+	if err := os.MkdirAll(filepath.Dir(exe), 0o700); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(exe, make([]byte, size), 0o700); err != nil {
+		t.Fatal(err)
+	}
+	touch(exe, used)
+}
+
+func toolOf(t *testing.T, m *Module, name string) func(map[string]any) (any, error) {
+	t.Helper()
+	for _, tool := range m.Tools(m.StepTools()) {
+		if tool.Name == name {
+			return tool.Run
+		}
+	}
+	t.Fatalf("no tool %s", name)
+	return nil
+}
+
+// Seams review must 2 (issue 1921): the kept composers and programs are listed with their size and last use, dropped one
+// by one, and swept when unused past KeptBound; a use keeps a program.
+func TestKeptProgramsAreListedDroppedAndSwept(t *testing.T) {
+	m, _ := fetchModule(t, &fakeStep{})
+	now := time.Now().UTC().Truncate(time.Second)
+	keepProgram(t, m, "composers", "old", 10, now.Add(-KeptBound-time.Hour))
+	keepProgram(t, m, "programs", "recent", 20, now.Add(-time.Hour))
+	keepProgram(t, m, "programs", "gone", 30, now)
+
+	out, err := toolOf(t, m, KeptTool)(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	list := out.(map[string]any)["kept"].([]KeptProgram)
+	if len(list) != 3 || list[0].Kind != "composers" || list[0].Key != "old" || list[0].Bytes != 10 || !list[0].LastUsed.Equal(now.Add(-KeptBound-time.Hour)) {
+		t.Fatalf("the kept programs = %+v", list)
+	}
+
+	for _, bad := range []map[string]any{{"kind": "nope", "key": "gone"}, {"kind": "programs", "key": "../composers"}, {"kind": "programs", "key": ""}} {
+		if _, err := toolOf(t, m, KeptDropTool)(bad); err == nil {
+			t.Errorf("dropped with %v", bad)
+		}
+	}
+	for i := 0; i < 2; i++ { // a second drop is no error
+		if _, err := toolOf(t, m, KeptDropTool)(map[string]any{"kind": "programs", "key": "gone"}); err != nil {
+			t.Fatalf("drop %d: %v", i, err)
+		}
+	}
+	if _, err := os.Stat(m.programPath("programs", "gone")); !os.IsNotExist(err) {
+		t.Error("the dropped program is still kept")
+	}
+
+	if swept := m.SweepKept(now, KeptBound); len(swept) != 1 || swept[0].Key != "old" {
+		t.Fatalf("swept %+v, want the old composer only", swept)
+	}
+	if got := m.Kept(); len(got) != 1 || got[0].Key != "recent" {
+		t.Fatalf("kept after the sweep = %+v", got)
+	}
+}

modules/mesh-build-go/cmd/mesh-build-go/module_test.go

@@ -115,7 +115,7 @@ func TestAGoThatCannotBeRunRefusesEveryStepAndHealthIsUnhealthy(t *testing.T) {
 		{Name: "short_probe", Run: run},
 	}, m.StepTools()...)
 	for _, tool := range m.Tools(steps) {
-		if tool.Name == StatusTool || tool.Name == HealthTool || tool.Name == StepReadTool || tool.Name == StepsReadTool || tool.Name == ToolchainsReadTool || tool.Name == ReleaseTool || tool.Name == StepEndTool {
+		if tool.Name == StatusTool || tool.Name == HealthTool || tool.Name == StepReadTool || tool.Name == StepsReadTool || tool.Name == ToolchainsReadTool || tool.Name == ReleaseTool || tool.Name == KeptTool || tool.Name == KeptDropTool || tool.Name == StepEndTool {
 			continue
 		}
 		var reason string

modules/mesh-build-go/cmd/mesh-build-go/tools.go

@@ -45,6 +45,7 @@ func (m *Module) Tools(steps []StepTool) []stdio.Tool {
 	tools = append(tools, m.ReadTools()...)
 	tools = append(tools, buildstep.StepEndTool(StepEndTool, m.Node, m.stepScan(), m.Held))
 	tools = append(tools, m.releaseTool())
+	tools = append(tools, m.keptTools()...)
 	return append(tools,
 		stdio.Tool{Name: StatusTool,
 			Description: "What the Go build module is on this machine: the Go version its recipe builds with and the one installed, " +

modules/mesh-build-go/module.json

@@ -50,6 +50,8 @@
     "build_go_build",
     "build_go_imports",
     "build_go_release",
+    "build_go_kept",
+    "build_go_kept_drop",
     "build_go_step",
     "build_go_steps",
     "build_go_step_end",