Kept composers and programs get a list tool, a drop tool and a sweep (issue 1921, seams must 2)
Claude-Session: https://claude.ai/code/session_01NcbgfFSnn24M3GcqttoU7i
This commit is contained in:
@@ -49,6 +49,10 @@ already gone is no error, one a running step holds is refused); its caller calls
|
|||||||
failed fetch removes its own workspace. A workspace no caller released is removed two hours after its last use (ADR
|
failed fetch removes its own workspace. A workspace no caller released is removed two hours after its last use (ADR
|
||||||
0509), and the removal is said in the log as the caller's fault.
|
0509), and the removal is said in the log as the caller's fault.
|
||||||
|
|
||||||
|
The programs this module builds from a commit and keeps in its cache (composers of compose asks, checking programs of run
|
||||||
|
asks) are read by `build_go_kept` (kind, key, size, last use) and removed by `build_go_kept_drop {kind, key}`; one no ask used
|
||||||
|
for 30 days is swept with the workspaces (issue 1921).
|
||||||
|
|
||||||
A pin on a pull request's head (issue 1148): when `go mod download` fails on a `git.novox.be/novox/*` pseudo-version, the
|
A pin on a pull request's head (issue 1148): when `go mod download` fails on a `git.novox.be/novox/*` pseudo-version, the
|
||||||
fetch looks for that commit among the forge's `refs/pull/*/head`, places the repository at it under the workspace's
|
fetch looks for that commit among the forge's `refs/pull/*/head`, places the repository at it under the workspace's
|
||||||
`pinned/beside/` as a plain directory, and hands it to Go (the fetch's commands, and the build with `GOPROXY=off`) through a
|
`pinned/beside/` as a plain directory, and hands it to Go (the fetch's commands, and the build with `GOPROXY=off`) through a
|
||||||
|
|||||||
@@ -38,7 +38,7 @@ var composerLocks sync.Map
|
|||||||
|
|
||||||
// composerPath is where the composer of key is kept: the cache directory, beside Go's own caches.
|
// composerPath is where the composer of key is kept: the cache directory, beside Go's own caches.
|
||||||
func (m *Module) composerPath(key string) string {
|
func (m *Module) composerPath(key string) string {
|
||||||
return filepath.Join(m.Cache, "composers", key, "mesh-compose")
|
return m.programPath("composers", key)
|
||||||
}
|
}
|
||||||
|
|
||||||
// compose runs a compose ask: the composer of its commit, built once, over its request with the manifests filled.
|
// compose runs a compose ask: the composer of its commit, built once, over its request with the manifests filled.
|
||||||
@@ -99,10 +99,11 @@ func (m *Module) manifestOf(digest string) ([]byte, error) {
|
|||||||
// composerOf is the composer kept under key, or, when none is, the one built now from the fetched commit (built true).
|
// composerOf is the composer kept under key, or, when none is, the one built now from the fetched commit (built true).
|
||||||
func (m *Module) composerOf(ctx context.Context, fetch buildstep.StepFunc, job delivery.ComposeJob, key string) (string, bool, error) {
|
func (m *Module) composerOf(ctx context.Context, fetch buildstep.StepFunc, job delivery.ComposeJob, key string) (string, bool, error) {
|
||||||
exe := m.composerPath(key)
|
exe := m.composerPath(key)
|
||||||
mu, _ := composerLocks.LoadOrStore(exe, &sync.Mutex{})
|
mu := lockOf(exe)
|
||||||
mu.(*sync.Mutex).Lock()
|
mu.Lock()
|
||||||
defer mu.(*sync.Mutex).Unlock()
|
defer mu.Unlock()
|
||||||
if st, err := os.Stat(exe); err == nil && st.Mode().IsRegular() {
|
if st, err := os.Stat(exe); err == nil && st.Mode().IsRegular() {
|
||||||
|
touch(exe, m.clock())
|
||||||
return exe, false, nil
|
return exe, false, nil
|
||||||
}
|
}
|
||||||
if fetch == nil {
|
if fetch == nil {
|
||||||
|
|||||||
@@ -348,6 +348,7 @@ func (m *Module) goEnvFor(dir string) []string {
|
|||||||
// net for a caller that never released one (issue 1839, ADR 0509).
|
// net for a caller that never released one (issue 1839, ADR 0509).
|
||||||
func (m *Module) CleanWorkspaces(now time.Time) {
|
func (m *Module) CleanWorkspaces(now time.Time) {
|
||||||
m.sweepSaid(m.spaces().Sweep(now, buildstep.UnheldBound))
|
m.sweepSaid(m.spaces().Sweep(now, buildstep.UnheldBound))
|
||||||
|
m.SweepKept(now, KeptBound)
|
||||||
}
|
}
|
||||||
|
|
||||||
// clock is the time now: Now when a test set it, else the wall clock.
|
// clock is the time now: Now when a test set it, else the wall clock.
|
||||||
|
|||||||
@@ -0,0 +1,130 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
// **Kept programs have tools and a sweep** (novox/hq issue 1921, the seams review's must 2): the programs this module builds
|
||||||
|
// from a commit and keeps under its cache so each commit is built once on a node, the composers of compose asks and the
|
||||||
|
// programs of run asks, are read by build_go_kept, removed one by one by build_go_kept_drop, and swept when unused for
|
||||||
|
// KeptBound. A use touches the program's file, so its modification time is its last use.
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"sort"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The tools of the kept programs.
|
||||||
|
const (
|
||||||
|
KeptTool = "build_go_kept"
|
||||||
|
KeptDropTool = "build_go_kept_drop"
|
||||||
|
)
|
||||||
|
|
||||||
|
// KeptBound is how long a kept program that no ask used is kept: a month, longer than any group lives, so only the build
|
||||||
|
// of a commit nobody asks about any more goes.
|
||||||
|
const KeptBound = 30 * 24 * time.Hour
|
||||||
|
|
||||||
|
// keptKinds are the cache directories of kept programs and the name of the program in each key's directory.
|
||||||
|
var keptKinds = map[string]string{"composers": "mesh-compose", "programs": "program"}
|
||||||
|
|
||||||
|
// KeptProgram is one kept program: its kind (the cache directory), its key, its size and when an ask last used it.
|
||||||
|
type KeptProgram struct {
|
||||||
|
Kind string `json:"kind"`
|
||||||
|
Key string `json:"key"`
|
||||||
|
Bytes int64 `json:"bytes"`
|
||||||
|
LastUsed time.Time `json:"last-used"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// programPath is where the program of kind and key is kept.
|
||||||
|
func (m *Module) programPath(kind, key string) string {
|
||||||
|
return filepath.Join(m.Cache, kind, key, keptKinds[kind])
|
||||||
|
}
|
||||||
|
|
||||||
|
// lockOf is the mutex that guards the program at path against a build or a use while it is dropped.
|
||||||
|
func lockOf(path string) *sync.Mutex {
|
||||||
|
l, _ := composerLocks.LoadOrStore(path, &sync.Mutex{})
|
||||||
|
return l.(*sync.Mutex)
|
||||||
|
}
|
||||||
|
|
||||||
|
// touch records a use of the kept program at path.
|
||||||
|
func touch(path string, at time.Time) { _ = os.Chtimes(path, at, at) }
|
||||||
|
|
||||||
|
// Kept lists the kept programs, composers then programs, each kind by key.
|
||||||
|
func (m *Module) Kept() []KeptProgram {
|
||||||
|
out := []KeptProgram{}
|
||||||
|
for _, kind := range []string{"composers", "programs"} {
|
||||||
|
entries, _ := os.ReadDir(filepath.Join(m.Cache, kind))
|
||||||
|
for _, e := range entries {
|
||||||
|
if !e.IsDir() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
st, err := os.Stat(m.programPath(kind, e.Name()))
|
||||||
|
if err != nil || !st.Mode().IsRegular() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out = append(out, KeptProgram{Kind: kind, Key: e.Name(), Bytes: st.Size(), LastUsed: st.ModTime().UTC()})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// DropKept removes the kept program of kind and key; one already gone is no error.
|
||||||
|
func (m *Module) DropKept(kind, key string) error {
|
||||||
|
if _, ok := keptKinds[kind]; !ok {
|
||||||
|
return fmt.Errorf("a kept program is of kind composers or programs, not %q", kind)
|
||||||
|
}
|
||||||
|
if key == "" || key != filepath.Base(key) || key == "." || key == ".." {
|
||||||
|
return fmt.Errorf("%q is no kept program's key", key)
|
||||||
|
}
|
||||||
|
exe := m.programPath(kind, key)
|
||||||
|
l := lockOf(exe)
|
||||||
|
l.Lock()
|
||||||
|
defer l.Unlock()
|
||||||
|
return os.RemoveAll(filepath.Dir(exe))
|
||||||
|
}
|
||||||
|
|
||||||
|
// SweepKept removes the kept programs no ask used within bound before now, and answers what it removed.
|
||||||
|
func (m *Module) SweepKept(now time.Time, bound time.Duration) []KeptProgram {
|
||||||
|
var gone []KeptProgram
|
||||||
|
for _, k := range m.Kept() {
|
||||||
|
if now.Sub(k.LastUsed) <= bound {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if m.DropKept(k.Kind, k.Key) == nil {
|
||||||
|
gone = append(gone, k)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Slice(gone, func(i, j int) bool { return gone[i].Key < gone[j].Key })
|
||||||
|
if len(gone) > 0 && m.Say != nil {
|
||||||
|
m.Say("removed %d kept programs no ask used within %s", len(gone), bound)
|
||||||
|
}
|
||||||
|
return gone
|
||||||
|
}
|
||||||
|
|
||||||
|
// keptTools are the tools that read and drop the kept programs: no guard and no floor, since they only free room.
|
||||||
|
func (m *Module) keptTools() []stdio.Tool {
|
||||||
|
return []stdio.Tool{
|
||||||
|
{Name: KeptTool,
|
||||||
|
Description: "The programs this node keeps, built from a commit so each commit is built once: the composers of compose asks and " +
|
||||||
|
"the checking programs of run asks, each with its kind, key, size in bytes and when an ask last used it. One no ask used " +
|
||||||
|
"for 30 days is removed. Read-only.",
|
||||||
|
Run: func(map[string]any) (any, error) { return map[string]any{"kept": m.Kept()}, nil }},
|
||||||
|
{Name: KeptDropTool,
|
||||||
|
Description: "Remove one kept program by its kind (composers or programs) and key, as build_go_kept names them: the next ask " +
|
||||||
|
"for that commit builds it again. One already gone is no error.",
|
||||||
|
Input: map[string]any{"type": "object", "properties": map[string]any{
|
||||||
|
"kind": map[string]any{"type": "string", "enum": []string{"composers", "programs"}},
|
||||||
|
"key": map[string]any{"type": "string", "description": "the key build_go_kept names"},
|
||||||
|
}, "required": []string{"kind", "key"}},
|
||||||
|
Run: func(args map[string]any) (any, error) {
|
||||||
|
kind, _ := args["kind"].(string)
|
||||||
|
key, _ := args["key"].(string)
|
||||||
|
if err := m.DropKept(kind, key); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return map[string]any{"dropped": key, "kind": kind}, nil
|
||||||
|
}},
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,71 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func keepProgram(t *testing.T, m *Module, kind, key string, size int, used time.Time) {
|
||||||
|
t.Helper()
|
||||||
|
exe := m.programPath(kind, key)
|
||||||
|
if err := os.MkdirAll(filepath.Dir(exe), 0o700); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(exe, make([]byte, size), 0o700); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
touch(exe, used)
|
||||||
|
}
|
||||||
|
|
||||||
|
func toolOf(t *testing.T, m *Module, name string) func(map[string]any) (any, error) {
|
||||||
|
t.Helper()
|
||||||
|
for _, tool := range m.Tools(m.StepTools()) {
|
||||||
|
if tool.Name == name {
|
||||||
|
return tool.Run
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.Fatalf("no tool %s", name)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Seams review must 2 (issue 1921): the kept composers and programs are listed with their size and last use, dropped one
|
||||||
|
// by one, and swept when unused past KeptBound; a use keeps a program.
|
||||||
|
func TestKeptProgramsAreListedDroppedAndSwept(t *testing.T) {
|
||||||
|
m, _ := fetchModule(t, &fakeStep{})
|
||||||
|
now := time.Now().UTC().Truncate(time.Second)
|
||||||
|
keepProgram(t, m, "composers", "old", 10, now.Add(-KeptBound-time.Hour))
|
||||||
|
keepProgram(t, m, "programs", "recent", 20, now.Add(-time.Hour))
|
||||||
|
keepProgram(t, m, "programs", "gone", 30, now)
|
||||||
|
|
||||||
|
out, err := toolOf(t, m, KeptTool)(nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
list := out.(map[string]any)["kept"].([]KeptProgram)
|
||||||
|
if len(list) != 3 || list[0].Kind != "composers" || list[0].Key != "old" || list[0].Bytes != 10 || !list[0].LastUsed.Equal(now.Add(-KeptBound-time.Hour)) {
|
||||||
|
t.Fatalf("the kept programs = %+v", list)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, bad := range []map[string]any{{"kind": "nope", "key": "gone"}, {"kind": "programs", "key": "../composers"}, {"kind": "programs", "key": ""}} {
|
||||||
|
if _, err := toolOf(t, m, KeptDropTool)(bad); err == nil {
|
||||||
|
t.Errorf("dropped with %v", bad)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for i := 0; i < 2; i++ { // a second drop is no error
|
||||||
|
if _, err := toolOf(t, m, KeptDropTool)(map[string]any{"kind": "programs", "key": "gone"}); err != nil {
|
||||||
|
t.Fatalf("drop %d: %v", i, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(m.programPath("programs", "gone")); !os.IsNotExist(err) {
|
||||||
|
t.Error("the dropped program is still kept")
|
||||||
|
}
|
||||||
|
|
||||||
|
if swept := m.SweepKept(now, KeptBound); len(swept) != 1 || swept[0].Key != "old" {
|
||||||
|
t.Fatalf("swept %+v, want the old composer only", swept)
|
||||||
|
}
|
||||||
|
if got := m.Kept(); len(got) != 1 || got[0].Key != "recent" {
|
||||||
|
t.Fatalf("kept after the sweep = %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -115,7 +115,7 @@ func TestAGoThatCannotBeRunRefusesEveryStepAndHealthIsUnhealthy(t *testing.T) {
|
|||||||
{Name: "short_probe", Run: run},
|
{Name: "short_probe", Run: run},
|
||||||
}, m.StepTools()...)
|
}, m.StepTools()...)
|
||||||
for _, tool := range m.Tools(steps) {
|
for _, tool := range m.Tools(steps) {
|
||||||
if tool.Name == StatusTool || tool.Name == HealthTool || tool.Name == StepReadTool || tool.Name == StepsReadTool || tool.Name == ToolchainsReadTool || tool.Name == ReleaseTool || tool.Name == StepEndTool {
|
if tool.Name == StatusTool || tool.Name == HealthTool || tool.Name == StepReadTool || tool.Name == StepsReadTool || tool.Name == ToolchainsReadTool || tool.Name == ReleaseTool || tool.Name == KeptTool || tool.Name == KeptDropTool || tool.Name == StepEndTool {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
var reason string
|
var reason string
|
||||||
|
|||||||
@@ -45,6 +45,7 @@ func (m *Module) Tools(steps []StepTool) []stdio.Tool {
|
|||||||
tools = append(tools, m.ReadTools()...)
|
tools = append(tools, m.ReadTools()...)
|
||||||
tools = append(tools, buildstep.StepEndTool(StepEndTool, m.Node, m.stepScan(), m.Held))
|
tools = append(tools, buildstep.StepEndTool(StepEndTool, m.Node, m.stepScan(), m.Held))
|
||||||
tools = append(tools, m.releaseTool())
|
tools = append(tools, m.releaseTool())
|
||||||
|
tools = append(tools, m.keptTools()...)
|
||||||
return append(tools,
|
return append(tools,
|
||||||
stdio.Tool{Name: StatusTool,
|
stdio.Tool{Name: StatusTool,
|
||||||
Description: "What the Go build module is on this machine: the Go version its recipe builds with and the one installed, " +
|
Description: "What the Go build module is on this machine: the Go version its recipe builds with and the one installed, " +
|
||||||
|
|||||||
@@ -50,6 +50,8 @@
|
|||||||
"build_go_build",
|
"build_go_build",
|
||||||
"build_go_imports",
|
"build_go_imports",
|
||||||
"build_go_release",
|
"build_go_release",
|
||||||
|
"build_go_kept",
|
||||||
|
"build_go_kept_drop",
|
||||||
"build_go_step",
|
"build_go_step",
|
||||||
"build_go_steps",
|
"build_go_steps",
|
||||||
"build_go_step_end",
|
"build_go_step_end",
|
||||||
|
|||||||
Reference in New Issue
Block a user