Merge pull request 'Build modules end the steps a previous process left running, and serve build_<lang>_step_end (issue 1890)' (#496) from fix/1890-left-running-steps into main

This commit was merged in pull request #496.
This commit is contained in:
2026-10-26 12:57:34 +00:00
28 changed files with 88 additions and 19 deletions
@@ -115,3 +115,11 @@ func (m *Module) Puller(steps []StepTool, source buildstep.AskSource) *buildstep
func (m *Module) SeatTools(p *buildstep.Puller) []stdio.Tool {
return buildstep.SeatTools(Seat, "Go", p)
}
// StepEndTool ends one step left running (issue 1890); the module's steps state is the one it reads and writes.
const StepEndTool = "build_go_step_end"
// stepScan is the module's steps state as buildstep reads it.
func (m *Module) stepScan() buildstep.Scan {
return buildstep.ScanOf(stdio.State(delivery.StepsState))
}
@@ -8,9 +8,11 @@ package main
import (
"context"
"fmt"
"git.novox.be/novox/mesh-sdk/go/buildstep"
"os"
"os/user"
"strings"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/delivery"
@@ -70,7 +72,13 @@ func main() {
os.Exit(1)
}
steps := m.StepTools()
booted := time.Now()
puller := m.Puller(steps, nil)
m.Held = puller
if m.Node != "" {
// A step a previous process left running is ended (issue 1890); this process's own steps start after booted.
go buildstep.EndStaleAtStart(ctx, m.stepScan(), m.Node, booted, logf)
}
if m.Node != "" {
// The seat's asks, pulled once Serve has the runtime's channel; a pull before it is said and tried again.
go func() { _ = puller.Pull(ctx) }()
@@ -28,6 +28,8 @@ type Module struct {
// Toolchains and Steps are the Put of the states of those names.
Toolchains, Steps buildstep.Put
// ReadSteps and ReadToolchainsState read those two states back for the read tools (issue 1562).
// Held is the puller whose running asks a step-end tool will not end (issue 1890); nil in a test.
Held *buildstep.Puller
ReadSteps, ReadToolchainsState StateRead
// Pauses is the Put of the pauses state, which keeps the node's pause key without an age; ReadPauses reads it back.
Pauses buildstep.Put
@@ -121,6 +123,7 @@ func (m *Module) Start(ctx context.Context) {
m.Say("the toolchain: %v", err)
}
m.CleanWorkspaces(time.Now())
buildstep.EndStaleRound(m.stepScan(), m.Node, m.Say) // asks past the step bound, on the module's own round (issue 1890)
select {
case <-ctx.Done():
return
@@ -115,7 +115,7 @@ func TestAGoThatCannotBeRunRefusesEveryStepAndHealthIsUnhealthy(t *testing.T) {
{Name: "short_probe", Run: run},
}, m.StepTools()...)
for _, tool := range m.Tools(steps) {
if tool.Name == StatusTool || tool.Name == HealthTool || tool.Name == StepReadTool || tool.Name == StepsReadTool || tool.Name == ToolchainsReadTool || tool.Name == ReleaseTool {
if tool.Name == StatusTool || tool.Name == HealthTool || tool.Name == StepReadTool || tool.Name == StepsReadTool || tool.Name == ToolchainsReadTool || tool.Name == ReleaseTool || tool.Name == StepEndTool {
continue
}
var reason string
@@ -128,6 +128,7 @@ type StepLine struct {
Started string `json:"started"`
Ended string `json:"ended,omitempty"`
Reason string `json:"reason,omitempty"`
What string `json:"what,omitempty"`
}
// RecentSteps are this node's steps, the latest started first, at most limit (ten when limit is not above zero). Only
@@ -180,7 +181,7 @@ func (m *Module) RecentSteps(limit int) (lines []StepLine, cut bool, err error)
}
out := make([]StepLine, 0, len(steps))
for _, s := range steps {
l := StepLine{ID: s.ID, Tool: s.Tool, State: s.State, Started: s.Started.UTC().Format("2006-01-02T15:04:05Z"), Reason: s.Reason}
l := StepLine{ID: s.ID, Tool: s.Tool, State: s.State, Started: s.Started.UTC().Format("2006-01-02T15:04:05Z"), Reason: s.Reason, What: s.What}
if !s.Ended.IsZero() {
l.Ended = s.Ended.UTC().Format("2006-01-02T15:04:05Z")
}
@@ -96,7 +96,7 @@ func TestStepsToolListsRecentStepsOfThisNode(t *testing.T) {
t0 := time.Date(2026, 10, 21, 9, 0, 0, 0, time.UTC)
putStep(t, steps, delivery.Step{ID: "a", Tool: "build_go_fetch", Node: "ace", State: delivery.StepPassed, Started: t0, Ended: t0, Result: json.RawMessage(`1`)})
putStep(t, steps, delivery.Step{ID: "b", Tool: "build_go_test", Node: "ace", State: delivery.StepRunning, Started: t0.Add(time.Minute)})
putStep(t, steps, delivery.Step{ID: "c", Tool: "build_go_build", Node: "ace", State: delivery.StepFailed, Started: t0.Add(2 * time.Minute), Reason: "boom"})
putStep(t, steps, delivery.Step{ID: "c", Tool: "build_go_build", Node: "ace", State: delivery.StepFailed, Started: t0.Add(2 * time.Minute), Reason: "boom", What: "check ask, novox/x@abc"})
putStep(t, steps, delivery.Step{ID: "d", Tool: "build_go_build", Node: "g14", State: delivery.StepPassed, Started: t0.Add(time.Hour)})
m := readModule(steps, fakeState{})
out, err := tool(t, m, StepsReadTool)(map[string]any{})
@@ -111,7 +111,7 @@ func TestStepsToolListsRecentStepsOfThisNode(t *testing.T) {
if !slices.Equal(ids, []string{"c", "b", "a"}) {
t.Errorf("ids = %v, want c b a", ids)
}
if lines[0].Reason != "boom" || lines[0].State != delivery.StepFailed {
if lines[0].Reason != "boom" || lines[0].What != "check ask, novox/x@abc" || lines[0].State != delivery.StepFailed {
t.Errorf("first = %+v", lines[0])
}
out, _ = tool(t, m, StepsReadTool)(map[string]any{"limit": float64(1)})
@@ -2,6 +2,7 @@ package main
import (
"context"
"git.novox.be/novox/mesh-sdk/go/buildstep"
stdio "git.novox.be/novox/mesh-sdk/go"
)
@@ -40,6 +41,7 @@ func (m *Module) Tools(steps []StepTool) []stdio.Tool {
tools = append(tools, t)
}
tools = append(tools, m.ReadTools()...)
tools = append(tools, buildstep.StepEndTool(StepEndTool, m.Node, m.stepScan(), m.Held))
tools = append(tools, m.releaseTool())
return append(tools,
stdio.Tool{Name: StatusTool,
+1 -1
View File
@@ -3,7 +3,7 @@ module meshbuildgo
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026095702-79649c3fa9f4
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026125154-6b8497314297
github.com/distribution/distribution/v3 v3.1.2
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
+2 -2
View File
@@ -1,5 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026095702-79649c3fa9f4 h1:05sVel/UOvz4/g0dxrDOeVUL7BGakVrxPBlqRq02ZhM=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026095702-79649c3fa9f4/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026125154-6b8497314297 h1:k31FtFp/DGXGatRjKV1S0z1gBAHfVgtvBX6xF0g1Vgw=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026125154-6b8497314297/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/alicebob/miniredis/v2 v2.35.0 h1:QwLphYqCEAo1eu1TqPRN2jgVMPBweeQcR21jeqDCONI=
github.com/alicebob/miniredis/v2 v2.35.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
+1
View File
@@ -39,6 +39,7 @@
"build_go_release",
"build_go_step",
"build_go_steps",
"build_go_step_end",
"build_go_toolchains",
"build_go_status",
"build_go_health"
@@ -66,3 +66,11 @@ func (m *Module) Puller(steps []StepTool, source buildstep.AskSource) *buildstep
func (m *Module) SeatTools(p *buildstep.Puller) []stdio.Tool {
return buildstep.SeatTools(Seat, "image", p)
}
// StepEndTool ends one step left running (issue 1890); the module's steps state is the one it reads and writes.
const StepEndTool = "build_image_step_end"
// stepScan is the module's steps state as buildstep reads it.
func (m *Module) stepScan() buildstep.Scan {
return buildstep.ScanOf(stdio.State(delivery.StepsState))
}
@@ -9,9 +9,11 @@ package main
import (
"context"
"fmt"
"git.novox.be/novox/mesh-sdk/go/buildstep"
"os"
"os/user"
"strings"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/delivery"
@@ -57,7 +59,13 @@ func main() {
os.Exit(1)
}
steps := m.StepTools()
booted := time.Now()
puller := m.Puller(steps, nil)
m.Held = puller
if m.Node != "" {
// A step a previous process left running is ended (issue 1890); this process's own steps start after booted.
go buildstep.EndStaleAtStart(ctx, m.stepScan(), m.Node, booted, logf)
}
if m.Node != "" {
// The seat's asks, pulled once Serve has the runtime's channel; a pull before it is said and tried again.
go func() { _ = puller.Pull(ctx) }()
@@ -22,7 +22,9 @@ type Module struct {
// Account and UID are the account the module runs as; Home is its home.
Account, UID, Home string
// Steps is the Put of the steps state; ReadSteps reads it back for the read tools.
Steps buildstep.Put
Steps buildstep.Put
// Held is the puller whose running asks a step-end tool will not end (issue 1890); nil in a test.
Held *buildstep.Puller
ReadSteps StateRead
// Pauses is the Put of the pauses state, which keeps the node's pause key without an age; ReadPauses reads it back.
Pauses buildstep.Put
@@ -104,6 +106,7 @@ func (m *Module) Start(ctx context.Context) {
m.Say("this machine cannot build images: %v", err)
}
m.CleanWorkspaces(m.clock())
buildstep.EndStaleRound(m.stepScan(), m.Node, m.Say) // asks past the step bound, on the module's own round (issue 1890)
select {
case <-ctx.Done():
return
@@ -127,6 +127,7 @@ type StepLine struct {
Started string `json:"started"`
Ended string `json:"ended,omitempty"`
Reason string `json:"reason,omitempty"`
What string `json:"what,omitempty"`
}
// RecentSteps are this node's steps, the latest started first, at most limit (ten when limit is not above zero). Only
@@ -179,7 +180,7 @@ func (m *Module) RecentSteps(limit int) (lines []StepLine, cut bool, err error)
}
out := make([]StepLine, 0, len(steps))
for _, s := range steps {
l := StepLine{ID: s.ID, Tool: s.Tool, State: s.State, Started: s.Started.UTC().Format("2006-01-02T15:04:05Z"), Reason: s.Reason}
l := StepLine{ID: s.ID, Tool: s.Tool, State: s.State, Started: s.Started.UTC().Format("2006-01-02T15:04:05Z"), Reason: s.Reason, What: s.What}
if !s.Ended.IsZero() {
l.Ended = s.Ended.UTC().Format("2006-01-02T15:04:05Z")
}
@@ -2,6 +2,7 @@ package main
import (
"context"
"git.novox.be/novox/mesh-sdk/go/buildstep"
stdio "git.novox.be/novox/mesh-sdk/go"
)
@@ -44,6 +45,7 @@ func (m *Module) Tools(steps []StepTool) []stdio.Tool {
tools = append(tools, t)
}
tools = append(tools, m.ReadTools()...)
tools = append(tools, buildstep.StepEndTool(StepEndTool, m.Node, m.stepScan(), m.Held))
tools = append(tools, m.releaseTool())
return append(tools,
stdio.Tool{Name: StatusTool,
+1 -1
View File
@@ -3,7 +3,7 @@ module meshbuildimage
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025175617-518dae5ec9c8
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026125154-6b8497314297
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
+2 -2
View File
@@ -1,5 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025175617-518dae5ec9c8 h1:x4mvY7yiGTP5lnzxbiwyj1o24p05hAmzZP7xHg3XX2M=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025175617-518dae5ec9c8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026125154-6b8497314297 h1:k31FtFp/DGXGatRjKV1S0z1gBAHfVgtvBX6xF0g1Vgw=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026125154-6b8497314297/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
+1
View File
@@ -33,6 +33,7 @@
"build_image_release",
"build_image_step",
"build_image_steps",
"build_image_step_end",
"build_image_status",
"build_image_health"
],
@@ -74,3 +74,11 @@ func (m *Module) Puller(steps []StepTool, source buildstep.AskSource) *buildstep
func (m *Module) SeatTools(p *buildstep.Puller) []stdio.Tool {
return buildstep.SeatTools(Seat, "Node", p)
}
// StepEndTool ends one step left running (issue 1890); the module's steps state is the one it reads and writes.
const StepEndTool = "build_node_step_end"
// stepScan is the module's steps state as buildstep reads it.
func (m *Module) stepScan() buildstep.Scan {
return buildstep.ScanOf(stdio.State(delivery.StepsState))
}
@@ -9,10 +9,12 @@ package main
import (
"context"
"fmt"
"git.novox.be/novox/mesh-sdk/go/buildstep"
"os"
"os/user"
"path/filepath"
"strings"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/delivery"
@@ -69,7 +71,13 @@ func main() {
os.Exit(1)
}
steps := m.StepTools()
booted := time.Now()
puller := m.Puller(steps, nil)
m.Held = puller
if m.Node != "" {
// A step a previous process left running is ended (issue 1890); this process's own steps start after booted.
go buildstep.EndStaleAtStart(ctx, m.stepScan(), m.Node, booted, logf)
}
if m.Node != "" {
// The seat's asks, pulled once Serve has the runtime's channel; a pull before it is said and tried again.
go func() { _ = puller.Pull(ctx) }()
@@ -42,6 +42,8 @@ type Module struct {
// Toolchains and Steps are the Put of the states of those names.
Toolchains, Steps buildstep.Put
// ReadSteps and ReadToolchainsState read those two states back for the read tools (issue 1562).
// Held is the puller whose running asks a step-end tool will not end (issue 1890); nil in a test.
Held *buildstep.Puller
ReadSteps, ReadToolchainsState StateRead
// Pauses is the Put of the pauses state, which keeps the node's pause key without an age; ReadPauses reads it back.
Pauses buildstep.Put
@@ -137,6 +139,7 @@ func (m *Module) Start(ctx context.Context) {
m.Say("the toolchain: %v", err)
}
m.CleanWorkspaces(m.now())
buildstep.EndStaleRound(m.stepScan(), m.Node, m.Say) // asks past the step bound, on the module's own round (issue 1890)
select {
case <-ctx.Done():
return
@@ -117,7 +117,7 @@ func TestANodeThatCannotBeRunRefusesEveryStepAndHealthIsUnhealthy(t *testing.T)
{Name: "short_probe", Run: run},
}, (&Module{}).StepTools()...)
for _, tool := range m.Tools(steps) {
if tool.Name == StatusTool || tool.Name == HealthTool || tool.Name == StepReadTool || tool.Name == StepsReadTool || tool.Name == ToolchainsReadTool || tool.Name == ReleaseTool {
if tool.Name == StatusTool || tool.Name == HealthTool || tool.Name == StepReadTool || tool.Name == StepsReadTool || tool.Name == ToolchainsReadTool || tool.Name == ReleaseTool || tool.Name == StepEndTool {
continue
}
var reason string
@@ -128,6 +128,7 @@ type StepLine struct {
Started string `json:"started"`
Ended string `json:"ended,omitempty"`
Reason string `json:"reason,omitempty"`
What string `json:"what,omitempty"`
}
// RecentSteps are this node's steps, the latest started first, at most limit (ten when limit is not above zero). Only
@@ -180,7 +181,7 @@ func (m *Module) RecentSteps(limit int) (lines []StepLine, cut bool, err error)
}
out := make([]StepLine, 0, len(steps))
for _, s := range steps {
l := StepLine{ID: s.ID, Tool: s.Tool, State: s.State, Started: s.Started.UTC().Format("2006-01-02T15:04:05Z"), Reason: s.Reason}
l := StepLine{ID: s.ID, Tool: s.Tool, State: s.State, Started: s.Started.UTC().Format("2006-01-02T15:04:05Z"), Reason: s.Reason, What: s.What}
if !s.Ended.IsZero() {
l.Ended = s.Ended.UTC().Format("2006-01-02T15:04:05Z")
}
@@ -96,7 +96,7 @@ func TestStepsToolListsRecentStepsOfThisNode(t *testing.T) {
t0 := time.Date(2026, 10, 21, 9, 0, 0, 0, time.UTC)
putStep(t, steps, delivery.Step{ID: "a", Tool: "build_node_fetch", Node: "ace", State: delivery.StepPassed, Started: t0, Ended: t0, Result: json.RawMessage(`1`)})
putStep(t, steps, delivery.Step{ID: "b", Tool: "build_node_test", Node: "ace", State: delivery.StepRunning, Started: t0.Add(time.Minute)})
putStep(t, steps, delivery.Step{ID: "c", Tool: "build_node_build", Node: "ace", State: delivery.StepFailed, Started: t0.Add(2 * time.Minute), Reason: "boom"})
putStep(t, steps, delivery.Step{ID: "c", Tool: "build_node_build", Node: "ace", State: delivery.StepFailed, Started: t0.Add(2 * time.Minute), Reason: "boom", What: "check ask, novox/x@abc"})
putStep(t, steps, delivery.Step{ID: "d", Tool: "build_node_build", Node: "g14", State: delivery.StepPassed, Started: t0.Add(time.Hour)})
m := readModule(steps, fakeState{})
out, err := tool(t, m, StepsReadTool)(map[string]any{})
@@ -111,7 +111,7 @@ func TestStepsToolListsRecentStepsOfThisNode(t *testing.T) {
if !slices.Equal(ids, []string{"c", "b", "a"}) {
t.Errorf("ids = %v, want c b a", ids)
}
if lines[0].Reason != "boom" || lines[0].State != delivery.StepFailed {
if lines[0].Reason != "boom" || lines[0].What != "check ask, novox/x@abc" || lines[0].State != delivery.StepFailed {
t.Errorf("first = %+v", lines[0])
}
out, _ = tool(t, m, StepsReadTool)(map[string]any{"limit": float64(1)})
@@ -2,6 +2,7 @@ package main
import (
"context"
"git.novox.be/novox/mesh-sdk/go/buildstep"
stdio "git.novox.be/novox/mesh-sdk/go"
)
@@ -40,6 +41,7 @@ func (m *Module) Tools(steps []StepTool) []stdio.Tool {
tools = append(tools, t)
}
tools = append(tools, m.ReadTools()...)
tools = append(tools, buildstep.StepEndTool(StepEndTool, m.Node, m.stepScan(), m.Held))
tools = append(tools, m.releaseTool())
return append(tools,
stdio.Tool{Name: StatusTool,
+1 -1
View File
@@ -3,7 +3,7 @@ module meshbuildnode
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026095702-79649c3fa9f4
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026125154-6b8497314297
github.com/distribution/distribution/v3 v3.1.2
github.com/nats-io/nats.go v1.54.0
)
+2 -2
View File
@@ -1,5 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026095702-79649c3fa9f4 h1:05sVel/UOvz4/g0dxrDOeVUL7BGakVrxPBlqRq02ZhM=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026095702-79649c3fa9f4/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026125154-6b8497314297 h1:k31FtFp/DGXGatRjKV1S0z1gBAHfVgtvBX6xF0g1Vgw=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026125154-6b8497314297/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/alicebob/miniredis/v2 v2.35.0 h1:QwLphYqCEAo1eu1TqPRN2jgVMPBweeQcR21jeqDCONI=
github.com/alicebob/miniredis/v2 v2.35.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
+1
View File
@@ -34,6 +34,7 @@
"build_node_release",
"build_node_step",
"build_node_steps",
"build_node_step_end",
"build_node_toolchains",
"build_node_status",
"build_node_health"