From 315c6223867b394c9c712e7b647142416051220a Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 18 Oct 2026 14:36:04 +0200 Subject: [PATCH 1/2] Lay the builds out before the start reads them, so the controller does not exit on its own start (issue 1121) Delivery 2 (#381) gated every chain read on the layout marker and laid the builds out beside the start; a read of the start then met the gate, the controller exited, and systemd started it again every five seconds from 14:34 on 2026-10-18. The layout now runs once inside attachTheChain, before anything reads; it skips keys laid out before, so it takes seconds, and a failure is retried beside the serving as before. --- cmd/mesh-controller/chain.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/cmd/mesh-controller/chain.go b/cmd/mesh-controller/chain.go index 9af1cc87..0f648a9d 100644 --- a/cmd/mesh-controller/chain.go +++ b/cmd/mesh-controller/chain.go @@ -43,8 +43,14 @@ func attachTheChain(ctx context.Context, inv *inventory.Inventory, server *link. } inv.UseChain(c) inv.RequireLayout() - // And the builds laid out under their modules, once, before they are read (issue 1121). - go layBuildsOutSoon(ctx, inv) + // And the builds laid out under their modules, once, before they are read (issue 1121) — **before this returns**, so + // nothing the start goes on to read meets a chain not laid out yet: on 2026-10-18 the layout ran beside the start, a + // read of the start was refused, and the controller exited and was started again every five seconds. The layout of + // keys laid out before skips them, so this takes seconds; one that fails is tried again beside the serving, loudly. + if err := inv.LayBuildsOut(ctx, func(format string, args ...any) { fmt.Printf("chain: "+format+"\n", args...) }); err != nil { + fmt.Printf("chain: the builds are not laid out under their modules yet, and none is read until they are: %v\n", err) + go layBuildsOutSoon(ctx, inv) + } // And the requests the move of delivery 1 found never handed over, asked again once (askAgain), once it is serving. go askAgainSoon(ctx, inv) // And what delivery 1 recorded of what each machine was sent, copied onto the keys of the assignments (CopySentBuilds). From a825a4c904466af58af90b5e8325dd0bfc70e8fb Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 18 Oct 2026 14:39:43 +0200 Subject: [PATCH 2/2] The first read of a chain not laid out lays it out, so no read of the start is refused (issue 1121) The layout gate is switched on when the stores open (UseChainOpener), before attachTheChain; a read of the start came first, was refused, and the controller exited. A read that finds no marker now lays the builds out once, under a lock, and is refused only when that fails (a process without the lease). --- internal/inventory/chain.go | 18 ++++++++++++- .../inventory/chain_builds_layout_test.go | 16 ++++++------ .../chain_first_read_lays_out_test.go | 25 +++++++++++++++++++ 3 files changed, 50 insertions(+), 9 deletions(-) create mode 100644 internal/inventory/chain_first_read_lays_out_test.go diff --git a/internal/inventory/chain.go b/internal/inventory/chain.go index 1a0fbd7e..df09bfa0 100644 --- a/internal/inventory/chain.go +++ b/internal/inventory/chain.go @@ -41,6 +41,16 @@ func (i *Inventory) RequireLayout() { i.state.mu.Unlock() } +// layingOut lets one read at a time lay the builds out; the others wait for it and then find the marker. +var layingOut sync.Mutex + +// layOutOnce lays the builds out under their modules unless another read did so meanwhile. +func (i *Inventory) layOutOnce(ctx context.Context) error { + layingOut.Lock() + defer layingOut.Unlock() + return i.LayBuildsOut(ctx, func(format string, args ...any) { fmt.Printf("chain: "+format+"\n", args...) }) +} + func (i *Inventory) markLaidOut() { i.state.mu.Lock() i.state.laid = true @@ -107,7 +117,13 @@ func (i *Inventory) theChain(ctx context.Context) (*chain.Chain, error) { if _, found, err := c.Builds.GetRaw(ctx, chain.LayoutMarkerKey); err != nil { return nil, err } else if !found { - return nil, ErrNotLaidOut + // **The first read lays them out** (issue 1121): on 2026-10-18 a read of the controller's start came before + // the layout and was refused, and the controller exited and was started again every five seconds. A + // process that may write (the lease holder) lays them out here, once; one that may not is refused, loudly. + if err := i.layOutOnce(ctx); err != nil { + return nil, fmt.Errorf("%w (laying them out now failed: %v)", ErrNotLaidOut, err) + } + return c, nil } i.markLaidOut() } diff --git a/internal/inventory/chain_builds_layout_test.go b/internal/inventory/chain_builds_layout_test.go index 26e30f3a..591f8300 100644 --- a/internal/inventory/chain_builds_layout_test.go +++ b/internal/inventory/chain_builds_layout_test.go @@ -3,7 +3,6 @@ package inventory import ( "context" "encoding/json" - "errors" "fmt" "reflect" "sort" @@ -398,9 +397,10 @@ func TestAnOlderBuildFinishingLastLeavesThePointersOnTheNewer(t *testing.T) { } } -// Nothing reads the chain on the bus until the controller's start has laid the builds out and written the marker; the layout -// is done once. -func TestTheChainIsRefusedUntilTheBuildsAreLaidOutAndTheLayoutIsDoneOnce(t *testing.T) { +// Nothing reads the chain on the bus before the builds are laid out and the marker written: the first read of a process that +// may write lays them out itself (issue 1121: a refused read of the start made the controller exit), and the layout is done +// once. +func TestTheFirstReadLaysTheBuildsOutAndTheLayoutIsDoneOnce(t *testing.T) { mem := chain.NewMemory() inv, c := layoutChain(mem) inv.RequireLayout() @@ -409,11 +409,11 @@ func TestTheChainIsRefusedUntilTheBuildsAreLaidOutAndTheLayoutIsDoneOnce(t *test if _, err := c.Builds.Create(ctx, old.ID, old); err != nil { t.Fatal(err) } - if _, err := inv.theChain(ctx); !errors.Is(err, ErrNotLaidOut) { - t.Fatalf("a read before the layout answered %v", err) + if _, err := inv.theChain(ctx); err != nil { + t.Fatalf("the first read was refused, not laid out: %v", err) } - if err := inv.LayBuildsOut(ctx, nil); err != nil { - t.Fatal(err) + if !containsKey(keysOf(t, mem), chain.BuildKey("alpha", old.ID)) { + t.Fatalf("the first read did not lay %s out under its module: %v", old.ID, keysOf(t, mem)) } if _, err := inv.theChain(ctx); err != nil { t.Fatalf("a read after the layout answered %v", err) diff --git a/internal/inventory/chain_first_read_lays_out_test.go b/internal/inventory/chain_first_read_lays_out_test.go new file mode 100644 index 00000000..7b80c80f --- /dev/null +++ b/internal/inventory/chain_first_read_lays_out_test.go @@ -0,0 +1,25 @@ +package inventory + +import ( + "context" + "testing" + + "github.com/novox/mesh-controller/internal/chain" +) + +// The first read of a chain that requires the layout lays the builds out itself when it may write, instead of refusing +// the controller's own start (issue 1121, 2026-10-18: the controller exited every five seconds). +func TestTheFirstReadOfAChainNotLaidOutLaysItOut(t *testing.T) { + ctx := context.Background() + inv, c := anInventoryOnTheChain(t) + inv.RequireLayout() + if _, found, _ := c.Builds.GetRaw(ctx, chain.LayoutMarkerKey); found { + t.Fatal("a fresh chain holds the marker already; the test proves nothing") + } + if _, err := inv.theChain(ctx); err != nil { + t.Fatalf("the first read was refused: %v", err) + } + if _, found, err := c.Builds.GetRaw(ctx, chain.LayoutMarkerKey); err != nil || !found { + t.Fatalf("the read did not lay the builds out: found %v, %v", found, err) + } +}