Take no rebuild into a waiting rollout, and ask what a tier left unasked (hq issue 754) #340
@@ -1049,7 +1049,7 @@ func planBatch(ctx context.Context, open *stores, batch *inventory.Plan, carry [
|
||||
return nil
|
||||
}
|
||||
// Written and its first tier asked as one act on the plans (novox/hq issue 213), under the caller's hold.
|
||||
if err := askTier(ctx, inv, &plan); err != nil {
|
||||
if err := askTier(ctx, inv, &plan, plan.Tiers[plan.Tier]); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := inv.SavePlan(ctx, &plan); err != nil {
|
||||
|
||||
@@ -372,6 +372,13 @@ func joinAPlan(ctx context.Context, open *stores, module string) (bool, string,
|
||||
if !found {
|
||||
return false, "", nil
|
||||
}
|
||||
// **A walk that waits for its delivery's word asks nothing** (novox/hq ADR 0239, issue 754): a rebuild does not
|
||||
// join it, and its outcome is not the walk's. The walk asks its own tier once it has its word.
|
||||
if p.Waiting() {
|
||||
return false, fmt.Sprintf("not joining %s (%s at %s): it waits for %s's word, so this build is not its "+
|
||||
"outcome; once let go, the walk asks %s itself", p.ID, p.Repository, short(p.Commit), p.Delivery.Awaits,
|
||||
module), nil
|
||||
}
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return false, "", err
|
||||
|
||||
@@ -521,7 +521,9 @@ func rebuildCommand(ctx context.Context, args []string) error {
|
||||
}
|
||||
|
||||
// **A module a plan holds unbuilt, or failed, joins that plan** rather than running beside it: the
|
||||
// plan would ask it again, or stay failed on an outcome a rebuild has replaced.
|
||||
// plan would ask it again, or stay failed on an outcome a rebuild has replaced. A walk waiting for its word
|
||||
// is not joined (novox/hq issue 754): the rebuild runs beside it, and says so.
|
||||
var beside string
|
||||
if module != "" {
|
||||
joined, said, err := joinAPlan(ctx, open, module)
|
||||
if err != nil {
|
||||
@@ -531,6 +533,7 @@ func rebuildCommand(ctx context.Context, args []string) error {
|
||||
fmt.Println(said)
|
||||
return nil
|
||||
}
|
||||
beside = said
|
||||
}
|
||||
id, err := askABuild(ctx, source, path, ref)
|
||||
if err != nil {
|
||||
@@ -538,6 +541,10 @@ func rebuildCommand(ctx context.Context, args []string) error {
|
||||
}
|
||||
recordAsked(ctx, inventory.BuildRequest{ID: id, Repository: source.Repository, Seat: source.Seat, Path: path,
|
||||
Ref: ref, For: "rebuild"})
|
||||
if beside != "" {
|
||||
fmt.Printf("rebuild asked as %s, beside the walk: %s\n", id, beside)
|
||||
return nil
|
||||
}
|
||||
fmt.Printf("rebuild asked as %s\n", id)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -637,8 +637,9 @@ func TestAPlanStoppedAtItsFirstMachineIsRetried(t *testing.T) {
|
||||
stopped := inventory.Plan{ID: "plan-rollout", Repository: "novox/a", Branch: "main", Commit: "c0ffee",
|
||||
Created: long, State: inventory.PlanFailed, Tiers: [][]string{{"a"}, {"b"}},
|
||||
Note: "a stopped at its first machine in tier 0: laptop refused what it was sent",
|
||||
Modules: map[string]*inventory.PlanModule{"a": {State: "built", BuiltAt: &long, Commit: "c0ffee",
|
||||
First: []string{"laptop"}, FirstAt: &long, Why: "laptop refused what it was sent"}}}
|
||||
// Asked by the plan, as every module a plan holds built is (novox/hq issue 754).
|
||||
Modules: map[string]*inventory.PlanModule{"a": {State: "built", AskedAt: &long, Build: "build-a", BuiltAt: &long,
|
||||
Commit: "c0ffee", First: []string{"laptop"}, FirstAt: &long, Why: "laptop refused what it was sent"}}}
|
||||
if err := open.inventory.SavePlan(ctx, &stopped); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -259,10 +259,11 @@ func applied(module string, builtAt time.Time, running []string, reports []inven
|
||||
return len(waiting) == 0, waiting
|
||||
}
|
||||
|
||||
// askTier asks the build machine for every module of the tier, and marks each asked. A module
|
||||
// askTier asks the build machine for the named modules of the plan's tier — all of it, or what of it is still
|
||||
// unasked (novox/hq issue 754) — and marks each asked. A module
|
||||
// the catalogue no longer holds, or whose ask could not be made, is a failure of the plan: a tier
|
||||
// half asked is a tier that will never complete.
|
||||
func askTier(ctx context.Context, inv *inventory.Inventory, p *inventory.Plan) error {
|
||||
func askTier(ctx context.Context, inv *inventory.Inventory, p *inventory.Plan, names []string) error {
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -271,12 +272,62 @@ func askTier(ctx context.Context, inv *inventory.Inventory, p *inventory.Plan) e
|
||||
for _, e := range entries {
|
||||
byName[e.Manifest.Module] = e
|
||||
}
|
||||
for _, name := range p.Tiers[p.Tier] {
|
||||
for _, name := range names {
|
||||
askModule(ctx, p, name, byName)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// unaskedOf are the modules of a plan's current tier it has still to ask: those it never asked, and one it holds
|
||||
// built that it never asked for (novox/hq issue 754) — a build taken in by a controller from before a walk took
|
||||
// only its own asks, somebody else's, which the walk would otherwise send as its own (ADR 0219). A tier partly asked is asked the rest, not
|
||||
// waited on for ever: a walk that waits for every build of a tier it never asked whole never moves again.
|
||||
func unaskedOf(p inventory.Plan) []string {
|
||||
if p.Tier >= len(p.Tiers) {
|
||||
return nil
|
||||
}
|
||||
var out []string
|
||||
for _, m := range p.Tiers[p.Tier] {
|
||||
s := p.Modules[m]
|
||||
switch {
|
||||
case s == nil || s.State == "":
|
||||
out = append(out, m)
|
||||
case s.State == "built" && s.Build == "" && s.AskedAt == nil:
|
||||
out = append(out, m)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// askedInTier says the plan asked a module of its current tier itself: under an id, or at a time it kept.
|
||||
func askedInTier(p inventory.Plan) bool {
|
||||
if p.Tier >= len(p.Tiers) {
|
||||
return false
|
||||
}
|
||||
for _, m := range p.Tiers[p.Tier] {
|
||||
if s := p.Modules[m]; s != nil && (s.Build != "" || s.AskedAt != nil) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// takesOutcome says whether a build's outcome is the plan's for a module of its current tier (novox/hq ADR 0219,
|
||||
// issues 219 and 754). **A walk that waits for its delivery's word asks nothing, and so is answered by nothing**
|
||||
// (ADR 0239): a rebuild that lands in the wait is somebody else's build. **Nor is a module the plan never asked
|
||||
// answered**: taken, the plan held a module built that it never asked, and waited for the rest of the tier for
|
||||
// ever. Asked under an id, the plan takes that id's outcome and no other; asked before ids were kept, a build
|
||||
// asked at or after the plan's ask.
|
||||
func takesOutcome(p inventory.Plan, state *inventory.PlanModule, asked time.Time, id string) bool {
|
||||
if p.Waiting() || state == nil || (state.Build == "" && state.AskedAt == nil) {
|
||||
return false
|
||||
}
|
||||
if state.Build != "" {
|
||||
return state.Build == id
|
||||
}
|
||||
return !askedBefore(asked, state.AskedAt)
|
||||
}
|
||||
|
||||
// askABuild is how a plan asks for one build, not waited for, and learns the id it asked under. A
|
||||
// variable so a test of what a plan does around an ask needs no build machine.
|
||||
//
|
||||
@@ -526,22 +577,15 @@ func planBuiltSaying(ctx context.Context, open *stores, module, commit, failed s
|
||||
if !inTier {
|
||||
continue
|
||||
}
|
||||
state := p.Modules[module]
|
||||
if state == nil {
|
||||
state = &inventory.PlanModule{}
|
||||
p.Modules[module] = state
|
||||
}
|
||||
// **The plan's own ask is its outcome, by id** (novox/hq ADR 0219); another build of the module
|
||||
// is, as before, when it was asked at or after the plan's ask (issue 219).
|
||||
// **A module asked under an id is answered by that id's outcome and no other** (novox/hq ADR
|
||||
// 0219): a replay, a rebuild beside the plan, or an older ask finishing late is somebody else's
|
||||
// build, made from other source, and settling the plan with it would send that. A plan from
|
||||
// before ids were kept is matched as it was: by when the build was asked (issue 219).
|
||||
if state.Build != "" {
|
||||
if state.Build != id {
|
||||
continue
|
||||
}
|
||||
} else if askedBefore(asked, state.AskedAt) {
|
||||
// before ids were kept is matched as it was: by when the build was asked (issue 219). A walk
|
||||
// waiting for its word, or a module it never asked, takes no outcome at all (issue 754).
|
||||
state := p.Modules[module]
|
||||
if !takesOutcome(*p, state, asked, id) {
|
||||
continue
|
||||
}
|
||||
if failed != "" && deletedAtSource(failed) {
|
||||
@@ -747,17 +791,11 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
||||
return true, nil
|
||||
}
|
||||
tier := p.Tiers[p.Tier]
|
||||
// Not yet asked: ask.
|
||||
unasked := 0
|
||||
for _, m := range tier {
|
||||
if s := p.Modules[m]; s == nil || s.State == "" {
|
||||
unasked++
|
||||
}
|
||||
}
|
||||
if unasked == len(tier) {
|
||||
// Not yet asked: ask — the whole tier, or what of it is still unasked (novox/hq issue 754).
|
||||
if unasked := unaskedOf(*p); len(unasked) > 0 {
|
||||
// **One walk at a time** (novox/hq ADR 0276): a walk about to ask its first tier while another started
|
||||
// walk is open waits for that one to end, let go or not, and says so.
|
||||
if p.Tier == 0 {
|
||||
if p.Tier == 0 && !askedInTier(*p) {
|
||||
if behind, err := startedBeside(ctx, inv, p.ID, p.Created); err != nil {
|
||||
return false, err
|
||||
} else if behind != "" {
|
||||
@@ -767,7 +805,10 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
||||
return changed, nil
|
||||
}
|
||||
}
|
||||
if err := askTier(ctx, inv, p); err != nil {
|
||||
if len(unasked) < len(tier) {
|
||||
fmt.Printf("%s: tier %d was asked in part; asking the rest: %s\n", p.ID, p.Tier, strings.Join(unasked, ", "))
|
||||
}
|
||||
if err := askTier(ctx, inv, p, unasked); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, nil
|
||||
|
||||
@@ -0,0 +1,122 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// novox/hq issue 754: a walk waiting for its delivery's word takes no build's outcome, nor does a walk take one for
|
||||
// a module it never asked; asked under an id, it takes that id's alone.
|
||||
func TestAWalkTakesOnlyTheOutcomesOfWhatItAsked(t *testing.T) {
|
||||
asked := time.Date(2026, 10, 14, 23, 30, 0, 0, time.UTC)
|
||||
before := asked.Add(-time.Minute)
|
||||
waiting := inventory.Plan{Delivery: &inventory.PlanDelivery{Awaits: "mesh-delivery"}}
|
||||
letGo := inventory.Plan{Delivery: &inventory.PlanDelivery{Awaits: "mesh-delivery", Go: &asked}}
|
||||
own := &inventory.PlanModule{State: "asked", AskedAt: &asked, Build: "build-own"}
|
||||
old := &inventory.PlanModule{State: "asked", AskedAt: &asked}
|
||||
for _, c := range []struct {
|
||||
what string
|
||||
p inventory.Plan
|
||||
state *inventory.PlanModule
|
||||
at time.Time
|
||||
id string
|
||||
takes bool
|
||||
}{
|
||||
{"a rebuild while the walk waits", waiting, nil, asked, "build-rebuild", false},
|
||||
{"its own id while it waits", waiting, own, asked, "build-own", false},
|
||||
{"a module it never asked", letGo, nil, asked, "build-rebuild", false},
|
||||
{"a module it never asked, state kept empty", letGo, &inventory.PlanModule{}, asked, "build-rebuild", false},
|
||||
{"another id", letGo, own, asked, "build-rebuild", false},
|
||||
{"its own id", letGo, own, asked, "build-own", true},
|
||||
{"a plan from before ids, asked after it", letGo, old, asked.Add(time.Second), "x", true},
|
||||
{"a plan from before ids, asked before it", letGo, old, before, "x", false},
|
||||
{"a plan with no delivery, its own id", inventory.Plan{}, own, asked, "build-own", true},
|
||||
} {
|
||||
if got := takesOutcome(c.p, c.state, c.at, c.id); got != c.takes {
|
||||
t.Errorf("%s: takes %v, want %v", c.what, got, c.takes)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq issue 754: a tier partly asked asks the rest; a module held built that the plan never asked — the live
|
||||
// shape, a hand rebuild taken in during the wait, built after the plan was made — is asked again; one asked is kept.
|
||||
func TestATierPartlyAskedAsksTheRest(t *testing.T) {
|
||||
created := time.Date(2026, 10, 14, 23, 30, 0, 0, time.UTC)
|
||||
earlier, later := created.Add(-time.Hour), created.Add(7*time.Minute)
|
||||
p := inventory.Plan{Created: created, Tiers: [][]string{{"build-agent", "gitea", "route-proxy", "stale", "kept", "own"}},
|
||||
Modules: map[string]*inventory.PlanModule{
|
||||
"gitea": {},
|
||||
"route-proxy": {State: "built", BuiltAt: &later, Commit: "c51357f8"},
|
||||
"stale": {State: "built", BuiltAt: &earlier},
|
||||
"kept": {State: "asked", AskedAt: &created, Build: "build-kept"},
|
||||
"own": {State: "built", AskedAt: &created, BuiltAt: &later, Build: "build-own"},
|
||||
}}
|
||||
if got, want := unaskedOf(p), []string{"build-agent", "gitea", "route-proxy", "stale"}; !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("unasked %v, want %v", got, want)
|
||||
}
|
||||
if !askedInTier(p) {
|
||||
t.Fatal("a tier with a module asked under an id reads as never asked")
|
||||
}
|
||||
p.Modules["kept"], p.Modules["own"] = nil, nil
|
||||
if askedInTier(p) {
|
||||
t.Fatal("a tier the plan never asked reads as asked")
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq issue 754, as seen on 2026-10-14: a rebuild lands while the walk waits for its word; the walk is let go;
|
||||
// it asks every module of its tier, and completes it once those builds come in.
|
||||
func TestARebuildInTheWaitIsNotTheWalksBuild(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
asked := asksRecorded(t)
|
||||
twoTiers(t, open)
|
||||
created := time.Now().UTC().Add(-10 * time.Minute)
|
||||
plan := inventory.Plan{ID: "plan-754", Repository: "novox/a", Branch: "main", Commit: "c0ffee", Created: created,
|
||||
State: inventory.PlanBuilding, Tiers: [][]string{{"a", "b"}}, Modules: map[string]*inventory.PlanModule{},
|
||||
Delivery: &inventory.PlanDelivery{Awaits: "mesh-delivery"}}
|
||||
if err := open.inventory.SavePlan(ctx, &plan); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// A rebuild of a, beside the waiting walk: not joined, and its outcome not taken.
|
||||
joined, said, err := joinAPlan(ctx, open, "a")
|
||||
if err != nil || joined || said == "" {
|
||||
t.Fatalf("a rebuild joined a waiting walk: %v %q %v", joined, said, err)
|
||||
}
|
||||
planBuilt(ctx, open, "a", "c0ffee", "", time.Now().UTC(), "build-rebuild")
|
||||
p, _ := open.inventory.PlanByID(ctx, plan.ID)
|
||||
if s := p.Modules["a"]; s != nil && s.State != "" {
|
||||
t.Fatalf("the waiting walk took a rebuild as its build: %+v", s)
|
||||
}
|
||||
if len(*asked) != 0 {
|
||||
t.Fatalf("a waiting walk asked %v", *asked)
|
||||
}
|
||||
|
||||
// Let go: both are asked, and their outcomes complete the tier.
|
||||
if _, err := letGo(ctx, open.inventory, plan.ID, "operator", "test"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
advancePlans(ctx, open)
|
||||
p, _ = open.inventory.PlanByID(ctx, plan.ID)
|
||||
for _, m := range []string{"a", "b"} {
|
||||
if s := p.Modules[m]; s == nil || s.State != "asked" || s.Build == "" {
|
||||
t.Fatalf("let go, %s is %+v", m, s)
|
||||
}
|
||||
}
|
||||
if len(*asked) != 2 {
|
||||
t.Fatalf("asked %v", *asked)
|
||||
}
|
||||
for _, m := range []string{"a", "b"} {
|
||||
id := p.Modules[m].Build
|
||||
at, _ := link.BuildAskedAt(id)
|
||||
planBuilt(ctx, open, m, "c0ffee", "", at, id)
|
||||
}
|
||||
advancePlans(ctx, open)
|
||||
if p, _ = open.inventory.PlanByID(ctx, plan.ID); p.Open() && p.Tier == 0 {
|
||||
t.Fatalf("the walk did not get past its tier: %s at %d, %q", p.State, p.Tier, p.Note)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user