Build steps: EndStaleAtStart waits for the runtime and ends the left-running steps (issue 1890)

go/buildstep/stale.go

@@ -1,6 +1,7 @@
 package buildstep
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"strings"
@@ -164,3 +165,40 @@ func StepEndTool(name string, node string, scan Scan, put Put, p *Puller) stdio.
 			return EndOne(scan, put, node, id, why, live)
 		}}
 }
+
+// StaleAttempts and StaleRetry bound EndStaleAtStart's tries: the steps state is readable once the runtime serves.
+var (
+	StaleAttempts = 6
+	StaleRetry    = 5 * time.Second
+)
+
+// EndStaleAtStart is what a build module runs once at its start, in a goroutine: it waits for the runtime to serve,
+// then ends this node's steps left running before `before` (EndStale), trying again when the state cannot be read yet.
+// A step this process starts is never one of them: it starts after `before`.
+func EndStaleAtStart(ctx context.Context, scan Scan, put Put, node string, before time.Time, logf func(string, ...any)) {
+	select {
+	case <-ctx.Done():
+		return
+	case <-stdio.Served():
+	}
+	for attempt := 1; ; attempt++ {
+		ended, err := EndStale(scan, put, node, before, logf)
+		if err == nil {
+			if logf != nil {
+				logf("%d steps of %s left running by a previous process ended", len(ended), node)
+			}
+			return
+		}
+		if logf != nil {
+			logf("ending the steps left running (try %d of %d): %v", attempt, StaleAttempts, err)
+		}
+		if attempt >= StaleAttempts {
+			return
+		}
+		select {
+		case <-ctx.Done():
+			return
+		case <-time.After(StaleRetry):
+		}
+	}
+}