diff --git a/modules/mesh-build-go/README.md b/modules/mesh-build-go/README.md index ff16837f8..ce1d97989 100644 --- a/modules/mesh-build-go/README.md +++ b/modules/mesh-build-go/README.md @@ -49,6 +49,10 @@ already gone is no error, one a running step holds is refused); its caller calls failed fetch removes its own workspace. A workspace no caller released is removed two hours after its last use (ADR 0509), and the removal is said in the log as the caller's fault. +The programs this module builds from a commit and keeps in its cache (composers of compose asks, checking programs of run +asks) are read by `build_go_kept` (kind, key, size, last use) and removed by `build_go_kept_drop {kind, key}`; one no ask used +for 30 days is swept with the workspaces (issue 1921). + A pin on a pull request's head (issue 1148): when `go mod download` fails on a `git.novox.be/novox/*` pseudo-version, the fetch looks for that commit among the forge's `refs/pull/*/head`, places the repository at it under the workspace's `pinned/beside/` as a plain directory, and hands it to Go (the fetch's commands, and the build with `GOPROXY=off`) through a diff --git a/modules/mesh-build-go/cmd/mesh-build-go/compose.go b/modules/mesh-build-go/cmd/mesh-build-go/compose.go index 1d55fbf9d..a8f52804f 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/compose.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/compose.go @@ -38,7 +38,7 @@ var composerLocks sync.Map // composerPath is where the composer of key is kept: the cache directory, beside Go's own caches. func (m *Module) composerPath(key string) string { - return filepath.Join(m.Cache, "composers", key, "mesh-compose") + return m.programPath("composers", key) } // compose runs a compose ask: the composer of its commit, built once, over its request with the manifests filled. @@ -99,10 +99,11 @@ func (m *Module) manifestOf(digest string) ([]byte, error) { // composerOf is the composer kept under key, or, when none is, the one built now from the fetched commit (built true). func (m *Module) composerOf(ctx context.Context, fetch buildstep.StepFunc, job delivery.ComposeJob, key string) (string, bool, error) { exe := m.composerPath(key) - mu, _ := composerLocks.LoadOrStore(exe, &sync.Mutex{}) - mu.(*sync.Mutex).Lock() - defer mu.(*sync.Mutex).Unlock() + mu := lockOf(exe) + mu.Lock() + defer mu.Unlock() if st, err := os.Stat(exe); err == nil && st.Mode().IsRegular() { + touch(exe, m.clock()) return exe, false, nil } if fetch == nil { diff --git a/modules/mesh-build-go/cmd/mesh-build-go/fetch.go b/modules/mesh-build-go/cmd/mesh-build-go/fetch.go index b733df269..a1988224a 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/fetch.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/fetch.go @@ -348,6 +348,7 @@ func (m *Module) goEnvFor(dir string) []string { // net for a caller that never released one (issue 1839, ADR 0509). func (m *Module) CleanWorkspaces(now time.Time) { m.sweepSaid(m.spaces().Sweep(now, buildstep.UnheldBound)) + m.SweepKept(now, KeptBound) } // clock is the time now: Now when a test set it, else the wall clock. diff --git a/modules/mesh-build-go/cmd/mesh-build-go/kept.go b/modules/mesh-build-go/cmd/mesh-build-go/kept.go new file mode 100644 index 000000000..7e6bc2344 --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/kept.go @@ -0,0 +1,130 @@ +package main + +// **Kept programs have tools and a sweep** (novox/hq issue 1921, the seams review's must 2): the programs this module builds +// from a commit and keeps under its cache so each commit is built once on a node, the composers of compose asks and the +// programs of run asks, are read by build_go_kept, removed one by one by build_go_kept_drop, and swept when unused for +// KeptBound. A use touches the program's file, so its modification time is its last use. + +import ( + "fmt" + "os" + "path/filepath" + "sort" + "sync" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// The tools of the kept programs. +const ( + KeptTool = "build_go_kept" + KeptDropTool = "build_go_kept_drop" +) + +// KeptBound is how long a kept program that no ask used is kept: a month, longer than any group lives, so only the build +// of a commit nobody asks about any more goes. +const KeptBound = 30 * 24 * time.Hour + +// keptKinds are the cache directories of kept programs and the name of the program in each key's directory. +var keptKinds = map[string]string{"composers": "mesh-compose", "programs": "program"} + +// KeptProgram is one kept program: its kind (the cache directory), its key, its size and when an ask last used it. +type KeptProgram struct { + Kind string `json:"kind"` + Key string `json:"key"` + Bytes int64 `json:"bytes"` + LastUsed time.Time `json:"last-used"` +} + +// programPath is where the program of kind and key is kept. +func (m *Module) programPath(kind, key string) string { + return filepath.Join(m.Cache, kind, key, keptKinds[kind]) +} + +// lockOf is the mutex that guards the program at path against a build or a use while it is dropped. +func lockOf(path string) *sync.Mutex { + l, _ := composerLocks.LoadOrStore(path, &sync.Mutex{}) + return l.(*sync.Mutex) +} + +// touch records a use of the kept program at path. +func touch(path string, at time.Time) { _ = os.Chtimes(path, at, at) } + +// Kept lists the kept programs, composers then programs, each kind by key. +func (m *Module) Kept() []KeptProgram { + out := []KeptProgram{} + for _, kind := range []string{"composers", "programs"} { + entries, _ := os.ReadDir(filepath.Join(m.Cache, kind)) + for _, e := range entries { + if !e.IsDir() { + continue + } + st, err := os.Stat(m.programPath(kind, e.Name())) + if err != nil || !st.Mode().IsRegular() { + continue + } + out = append(out, KeptProgram{Kind: kind, Key: e.Name(), Bytes: st.Size(), LastUsed: st.ModTime().UTC()}) + } + } + return out +} + +// DropKept removes the kept program of kind and key; one already gone is no error. +func (m *Module) DropKept(kind, key string) error { + if _, ok := keptKinds[kind]; !ok { + return fmt.Errorf("a kept program is of kind composers or programs, not %q", kind) + } + if key == "" || key != filepath.Base(key) || key == "." || key == ".." { + return fmt.Errorf("%q is no kept program's key", key) + } + exe := m.programPath(kind, key) + l := lockOf(exe) + l.Lock() + defer l.Unlock() + return os.RemoveAll(filepath.Dir(exe)) +} + +// SweepKept removes the kept programs no ask used within bound before now, and answers what it removed. +func (m *Module) SweepKept(now time.Time, bound time.Duration) []KeptProgram { + var gone []KeptProgram + for _, k := range m.Kept() { + if now.Sub(k.LastUsed) <= bound { + continue + } + if m.DropKept(k.Kind, k.Key) == nil { + gone = append(gone, k) + } + } + sort.Slice(gone, func(i, j int) bool { return gone[i].Key < gone[j].Key }) + if len(gone) > 0 && m.Say != nil { + m.Say("removed %d kept programs no ask used within %s", len(gone), bound) + } + return gone +} + +// keptTools are the tools that read and drop the kept programs: no guard and no floor, since they only free room. +func (m *Module) keptTools() []stdio.Tool { + return []stdio.Tool{ + {Name: KeptTool, + Description: "The programs this node keeps, built from a commit so each commit is built once: the composers of compose asks and " + + "the checking programs of run asks, each with its kind, key, size in bytes and when an ask last used it. One no ask used " + + "for 30 days is removed. Read-only.", + Run: func(map[string]any) (any, error) { return map[string]any{"kept": m.Kept()}, nil }}, + {Name: KeptDropTool, + Description: "Remove one kept program by its kind (composers or programs) and key, as build_go_kept names them: the next ask " + + "for that commit builds it again. One already gone is no error.", + Input: map[string]any{"type": "object", "properties": map[string]any{ + "kind": map[string]any{"type": "string", "enum": []string{"composers", "programs"}}, + "key": map[string]any{"type": "string", "description": "the key build_go_kept names"}, + }, "required": []string{"kind", "key"}}, + Run: func(args map[string]any) (any, error) { + kind, _ := args["kind"].(string) + key, _ := args["key"].(string) + if err := m.DropKept(kind, key); err != nil { + return nil, err + } + return map[string]any{"dropped": key, "kind": kind}, nil + }}, + } +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/kept_test.go b/modules/mesh-build-go/cmd/mesh-build-go/kept_test.go new file mode 100644 index 000000000..615f8b36a --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/kept_test.go @@ -0,0 +1,71 @@ +package main + +import ( + "os" + "path/filepath" + "testing" + "time" +) + +func keepProgram(t *testing.T, m *Module, kind, key string, size int, used time.Time) { + t.Helper() + exe := m.programPath(kind, key) + if err := os.MkdirAll(filepath.Dir(exe), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(exe, make([]byte, size), 0o700); err != nil { + t.Fatal(err) + } + touch(exe, used) +} + +func toolOf(t *testing.T, m *Module, name string) func(map[string]any) (any, error) { + t.Helper() + for _, tool := range m.Tools(m.StepTools()) { + if tool.Name == name { + return tool.Run + } + } + t.Fatalf("no tool %s", name) + return nil +} + +// Seams review must 2 (issue 1921): the kept composers and programs are listed with their size and last use, dropped one +// by one, and swept when unused past KeptBound; a use keeps a program. +func TestKeptProgramsAreListedDroppedAndSwept(t *testing.T) { + m, _ := fetchModule(t, &fakeStep{}) + now := time.Now().UTC().Truncate(time.Second) + keepProgram(t, m, "composers", "old", 10, now.Add(-KeptBound-time.Hour)) + keepProgram(t, m, "programs", "recent", 20, now.Add(-time.Hour)) + keepProgram(t, m, "programs", "gone", 30, now) + + out, err := toolOf(t, m, KeptTool)(nil) + if err != nil { + t.Fatal(err) + } + list := out.(map[string]any)["kept"].([]KeptProgram) + if len(list) != 3 || list[0].Kind != "composers" || list[0].Key != "old" || list[0].Bytes != 10 || !list[0].LastUsed.Equal(now.Add(-KeptBound-time.Hour)) { + t.Fatalf("the kept programs = %+v", list) + } + + for _, bad := range []map[string]any{{"kind": "nope", "key": "gone"}, {"kind": "programs", "key": "../composers"}, {"kind": "programs", "key": ""}} { + if _, err := toolOf(t, m, KeptDropTool)(bad); err == nil { + t.Errorf("dropped with %v", bad) + } + } + for i := 0; i < 2; i++ { // a second drop is no error + if _, err := toolOf(t, m, KeptDropTool)(map[string]any{"kind": "programs", "key": "gone"}); err != nil { + t.Fatalf("drop %d: %v", i, err) + } + } + if _, err := os.Stat(m.programPath("programs", "gone")); !os.IsNotExist(err) { + t.Error("the dropped program is still kept") + } + + if swept := m.SweepKept(now, KeptBound); len(swept) != 1 || swept[0].Key != "old" { + t.Fatalf("swept %+v, want the old composer only", swept) + } + if got := m.Kept(); len(got) != 1 || got[0].Key != "recent" { + t.Fatalf("kept after the sweep = %+v", got) + } +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/module_test.go b/modules/mesh-build-go/cmd/mesh-build-go/module_test.go index a780172fd..eba3f02cc 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/module_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/module_test.go @@ -115,7 +115,7 @@ func TestAGoThatCannotBeRunRefusesEveryStepAndHealthIsUnhealthy(t *testing.T) { {Name: "short_probe", Run: run}, }, m.StepTools()...) for _, tool := range m.Tools(steps) { - if tool.Name == StatusTool || tool.Name == HealthTool || tool.Name == StepReadTool || tool.Name == StepsReadTool || tool.Name == ToolchainsReadTool || tool.Name == ReleaseTool || tool.Name == StepEndTool { + if tool.Name == StatusTool || tool.Name == HealthTool || tool.Name == StepReadTool || tool.Name == StepsReadTool || tool.Name == ToolchainsReadTool || tool.Name == ReleaseTool || tool.Name == KeptTool || tool.Name == KeptDropTool || tool.Name == StepEndTool { continue } var reason string diff --git a/modules/mesh-build-go/cmd/mesh-build-go/tools.go b/modules/mesh-build-go/cmd/mesh-build-go/tools.go index dbc20eab0..de6bc5a65 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/tools.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/tools.go @@ -45,6 +45,7 @@ func (m *Module) Tools(steps []StepTool) []stdio.Tool { tools = append(tools, m.ReadTools()...) tools = append(tools, buildstep.StepEndTool(StepEndTool, m.Node, m.stepScan(), m.Held)) tools = append(tools, m.releaseTool()) + tools = append(tools, m.keptTools()...) return append(tools, stdio.Tool{Name: StatusTool, Description: "What the Go build module is on this machine: the Go version its recipe builds with and the one installed, " + diff --git a/modules/mesh-build-go/module.json b/modules/mesh-build-go/module.json index 2c6289c6a..ee94e020f 100644 --- a/modules/mesh-build-go/module.json +++ b/modules/mesh-build-go/module.json @@ -50,6 +50,8 @@ "build_go_build", "build_go_imports", "build_go_release", + "build_go_kept", + "build_go_kept_drop", "build_go_step", "build_go_steps", "build_go_step_end",