An older build request never replaces a newer one's artifact

Builds of one module in flight together finish in any order, and the mesh
took whatever it heard last as what the module is: RegisterModule overwrote
the module's manifest unconditionally, and Held/BuiltAgainst/ReadRepositories
ordered builds by when they were recorded. A postgres build asked before the
mesh-tools runtime fix finished after the one asked after it, and the next
push deployed the stale image (novox/hq issue 219).

A build is now ordered by when it was asked, read from the build-<nanos> id
the controller writes: build.asked and module.built_asked (migration 0055).
A registration from an earlier request than the module's current one is
recorded and refused as superseded. A plan takes as its outcome only a build
asked at or after its own ask, so an earlier plan's leftover build cannot
settle a later plan. Ids of any other shape keep the old order.

internal/inventory/builds.go

@@ -42,9 +42,29 @@ type Build struct {
 	// Failed is the builder's own words, empty when it worked.
 	Failed string
 	Made   []Artifact
-	At     time.Time
+	// Asked is when the build was requested, zero when that is not known (an id of another shape,
+	// or a build recorded before the mesh kept it). **What orders one build of a module against
+	// another** (novox/hq 04-ISSUES/219): builds in flight together finish in any order, and the
+	// one asked last stood on the newest bases.
+	Asked time.Time
+	// At is when the outcome was recorded — when it finished, not when it was asked.
+	At time.Time
 }
 
+// AskedOrAt is when the build was asked, or when it was recorded when that is not known — the
+// order the mesh had before it kept the request time.
+func (b Build) AskedOrAt() time.Time {
+	if !b.Asked.IsZero() {
+		return b.Asked
+	}
+	return b.At
+}
+
+// newestRequestFirst is the ordering every "what a module currently is" question uses: the newest
+// request wins, whenever it finished (novox/hq 04-ISSUES/219). A build whose request time is not
+// known is placed at the moment it was recorded, which is the rule that held before.
+const newestRequestFirst = `coalesce(asked, at) desc, at desc`
+
 // ReadRepository is a repository a build read source from besides the module's own.
 type ReadRepository struct {
 	Repository string `json:"repository"`
@@ -83,13 +103,17 @@ func (i *Inventory) RecordBuild(ctx context.Context, b Build) error {
 	if b.Module != "" {
 		module = &b.Module
 	}
+	var asked *time.Time
+	if !b.Asked.IsZero() {
+		asked = &b.Asked
+	}
 	_, err = i.store.Pool().Exec(ctx,
 		`insert into build (id, repository, ref, module, commit_hash, built_on, failed, made,
-		                    source_path, manifest, built_against, built_contexts)
-		 values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)
+		                    source_path, manifest, built_against, built_contexts, asked)
+		 values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)
 		 on conflict (id) do nothing`,
 		b.ID, b.Repository, b.Ref, module, b.Commit, b.On, b.Failed, made,
-		b.Path, manifestOrNil(b.Manifest), against, read)
+		b.Path, manifestOrNil(b.Manifest), against, read, asked)
 	return err
 }
 
@@ -102,11 +126,11 @@ func (i *Inventory) Builds(ctx context.Context, module string, limit int) ([]Bui
 	if limit <= 0 {
 		limit = 20
 	}
-	query := `select id, repository, ref, coalesce(module,''), commit_hash, built_on, failed, made, at
+	query := `select id, repository, ref, coalesce(module,''), commit_hash, built_on, failed, made, asked, at
 	          from build order by at desc limit $1`
 	args := []any{limit}
 	if module != "" {
-		query = `select id, repository, ref, coalesce(module,''), commit_hash, built_on, failed, made, at
+		query = `select id, repository, ref, coalesce(module,''), commit_hash, built_on, failed, made, asked, at
 		         from build where module = $2 order by at desc limit $1`
 		args = append(args, module)
 	}
@@ -121,10 +145,14 @@ func (i *Inventory) Builds(ctx context.Context, module string, limit int) ([]Bui
 	for rows.Next() {
 		var b Build
 		var made []byte
+		var asked *time.Time
 		if err := rows.Scan(&b.ID, &b.Repository, &b.Ref, &b.Module, &b.Commit,
-			&b.On, &b.Failed, &made, &b.At); err != nil {
+			&b.On, &b.Failed, &made, &asked, &b.At); err != nil {
 			return nil, err
 		}
+		if asked != nil {
+			b.Asked = *asked
+		}
 		if err := json.Unmarshal(made, &b.Made); err != nil {
 			return nil, err
 		}
@@ -135,8 +163,9 @@ func (i *Inventory) Builds(ctx context.Context, module string, limit int) ([]Bui
 
 // Held is every artifact this mesh has built, keyed "<module>/<artifact>".
 //
-// **The newest successful build of each module wins**, which is the same rule the rest of the mesh
-// uses for what a module currently is. A module rebuilt to something broken and then rebuilt again
+// **The successful build of each module asked last wins**, which is the same rule the rest of the
+// mesh uses for what a module currently is — asked last, not finished last (novox/hq
+// 04-ISSUES/219): an older request that finishes later stood on older bases. A module rebuilt to something broken and then rebuilt again
 // is at the second one; a module whose last build failed is at the last one that worked, because a
 // failure published nothing and the thing it published before is still what exists.
 //
@@ -147,7 +176,7 @@ func (i *Inventory) Held(ctx context.Context) (map[string]string, error) {
 		`select distinct on (module) module, made
 		   from build
 		  where module is not null and module <> '' and failed = ''
-		  order by module, at desc`)
+		  order by module, `+newestRequestFirst)
 	if err != nil {
 		return nil, err
 	}
@@ -185,7 +214,7 @@ func (i *Inventory) BuiltAgainst(ctx context.Context) (map[string][]string, erro
 		`select distinct on (module) module, built_against
 		   from build
 		  where module is not null and module <> '' and failed = ''
-		  order by module, at desc`)
+		  order by module, `+newestRequestFirst)
 	if err != nil {
 		return nil, err
 	}
@@ -223,7 +252,7 @@ func (i *Inventory) ReadRepositories(ctx context.Context) (map[string][]ReadRepo
 		`select distinct on (module) module, built_contexts
 		   from build
 		  where module is not null and module <> '' and failed = ''
-		  order by module, at desc`)
+		  order by module, `+newestRequestFirst)
 	if err != nil {
 		return nil, err
 	}
@@ -274,7 +303,7 @@ func manifestOrNil(raw []byte) any {
 // follows when it decides whether to announce at all.
 //
 // One row per module and commit: a module built twice at the same commit is one fact, and the
-// latest row is the one whose artifacts are current.
+// row asked last is the one whose artifacts are current (novox/hq 04-ISSUES/219).
 func (i *Inventory) Announceable(ctx context.Context) ([]Build, error) {
 	rows, err := i.store.Pool().Query(ctx,
 		`select distinct on (module, commit_hash)
@@ -282,7 +311,7 @@ func (i *Inventory) Announceable(ctx context.Context) ([]Build, error) {
 		        source_path, manifest, built_against, at
 		   from build
 		  where failed = '' and module is not null and module <> '' and commit_hash <> ''
-		  order by module, commit_hash, at desc`)
+		  order by module, commit_hash, `+newestRequestFirst)
 	if err != nil {
 		return nil, err
 	}

internal/inventory/catalogue.go

@@ -17,6 +17,10 @@ import (
 // ErrNoSuchModule is what the mesh says about a module it has never been told about.
 var ErrNoSuchModule = errors.New("no module of that name")
 
+// ErrSuperseded is a registration from a build asked before the one the module is already at
+// (novox/hq 04-ISSUES/219). The build is recorded; what the module is does not change.
+var ErrSuperseded = errors.New("a build asked later is already what the module is")
+
 // ErrStillAssigned is why a module cannot be forgotten.
 //
 // Its own error because it is not a fault: it means a machine is running that module now, and
@@ -47,6 +51,10 @@ type Source struct {
 	// itself no longer carries its build (novox/hq to-be 38 WP2.4). Empty for a manifest handed over
 	// by hand, which carries its `build.on` itself.
 	Against []string
+	// Asked is when the build this manifest came from was requested (novox/hq 04-ISSUES/219). Zero
+	// is a manifest handed over by hand, or a build whose request time is not known: either is
+	// taken as asked at the moment it is registered.
+	Asked time.Time
 }
 
 // Current reports whether what the mesh holds is what the source last had.
@@ -97,13 +105,23 @@ func (i *Inventory) RegisterModule(ctx context.Context, m catalogue.Manifest, fr
 		return err
 	}
 
+	asked := from.Asked
+	if asked.IsZero() {
+		asked = time.Now()
+	}
+
 	// A module registered without provenance keeps whatever it had. Handing over a manifest by
 	// hand is a legitimate way to fix something in a hurry, and it should not silently erase the
 	// record of where the module normally comes from — which is the only thing that would say,
 	// afterwards, that the machine is running something nobody can rebuild.
-	_, err = i.store.Pool().Exec(ctx,
-		`insert into module (name, manifest, version, source, source_path, source_seat, ref, built_from, source_head)
-		 values ($1, $2, nullif($3,''), nullif($4,''), $7, $8, nullif($5,''), nullif($6,''), nullif($6,''))
+	//
+	// **An older request never replaces a newer one** (novox/hq 04-ISSUES/219). Builds of one
+	// module in flight together finish in any order, and each stood on the bases the mesh held when
+	// it was asked; the one asked later is what the module is, whichever is heard last. An outcome
+	// of an earlier request is kept in the build records and changes nothing here.
+	tag, err := i.store.Pool().Exec(ctx,
+		`insert into module (name, manifest, version, source, source_path, source_seat, ref, built_from, source_head, built_asked)
+		 values ($1, $2, nullif($3,''), nullif($4,''), $7, $8, nullif($5,''), nullif($6,''), nullif($6,''), $9)
 		 on conflict (name) do update set
 		     manifest   = excluded.manifest,
 		     version    = excluded.version,
@@ -115,9 +133,23 @@ func (i *Inventory) RegisterModule(ctx context.Context, m catalogue.Manifest, fr
 		                        else excluded.source_seat end,
 		     ref        = coalesce(excluded.ref, module.ref),
 		     built_from = coalesce(excluded.built_from, module.built_from),
-		     source_head = coalesce(excluded.built_from, module.source_head)`,
-		m.Module, raw, m.Version, from.Repository, from.Ref, from.BuiltFrom, from.Path, from.Seat)
-	return err
+		     source_head = coalesce(excluded.built_from, module.source_head),
+		     built_asked = excluded.built_asked
+		 where module.built_asked is null or module.built_asked <= excluded.built_asked`,
+		m.Module, raw, m.Version, from.Repository, from.Ref, from.BuiltFrom, from.Path, from.Seat, asked)
+	if err != nil {
+		return err
+	}
+	if tag.RowsAffected() == 0 {
+		var current time.Time
+		if err := i.store.Pool().QueryRow(ctx,
+			`select built_asked from module where name = $1`, m.Module).Scan(&current); err != nil {
+			return err
+		}
+		return fmt.Errorf("%w: %s is at a build asked %s, and this one was asked %s",
+			ErrSuperseded, m.Module, current.UTC().Format(time.RFC3339), asked.UTC().Format(time.RFC3339))
+	}
+	return nil
 }
 
 // registeredInThatShape is whether the catalogue already holds this module as a tools container on

internal/inventory/migrations/0055-an-older-build-never-replaces-a-newer.sql

@@ -0,0 +1,23 @@
+-- A build is ordered by when it was asked, not when it finished (novox/hq 04-ISSUES/219).
+--
+-- Two builds of one module can be in flight together — two merge plans a few minutes apart, each
+-- asking for everything standing on what it changed — and they finish in any order. Each build
+-- stands on the bases the mesh held when it was *asked*, so the one asked later is the newer one.
+-- The mesh ordered builds by `at`, which is when the outcome was recorded, and registered whatever
+-- it heard last: an older request that took longer replaced a newer one as what the module is, and
+-- the next push sent machines an image built on a base the mesh had already replaced.
+--
+-- `build.asked` is when the build was requested, read from the correlation id the controller wrote
+-- (`build-<unix nanoseconds>`). Nullable: an id of any other shape says no request time, and such a
+-- build is placed where it was recorded, which is the order the mesh had before this.
+alter table build add column asked timestamptz;
+
+update build
+   set asked = to_timestamp((substring(id from '^build-([0-9]{19})$'))::numeric / 1000000000)
+ where id ~ '^build-[0-9]{19}$';
+
+-- `module.built_asked` is when the build the module's registered manifest came from was asked, so
+-- a later-heard outcome of an earlier request is recorded and not registered. A manifest handed over
+-- by hand is a request made when it is handed over. Null for a module registered before this was
+-- kept: its next registration, whichever it is, sets it.
+alter table module add column built_asked timestamptz;

internal/inventory/older_build_test.go

@@ -0,0 +1,124 @@
+package inventory
+
+import (
+	"context"
+	"errors"
+	"testing"
+	"time"
+
+	"github.com/novox/mesh-controller/internal/catalogue"
+)
+
+// novox/hq 04-ISSUES/219: two builds of one module in flight together, the one asked first heard
+// last. The newer request stood on the newer base; the older one's late outcome is recorded and is
+// not what the module is.
+
+func postgresBuild(id string, asked time.Time, image string) Build {
+	b := aBuild(id, "postgres", "")
+	b.Asked = asked
+	b.Against = []string{"mesh-tools/runtime@sha256:" + id}
+	b.Made = []Artifact{{Name: "server", Kind: "image", Reference: "postgres@sha256:" + image}}
+	return b
+}
+
+func TestAnOlderRequestFinishingLaterIsNotWhatTheModuleHolds(t *testing.T) {
+	inv := fresh(t)
+	ctx := context.Background()
+	older := time.Date(2026, 10, 3, 21, 33, 45, 0, time.UTC)
+	newer := time.Date(2026, 10, 3, 21, 51, 57, 0, time.UTC)
+
+	// The newer request finishes first, the older one last — recorded in that order.
+	if err := inv.RecordBuild(ctx, postgresBuild("newer", newer, "4bcd5f73")); err != nil {
+		t.Fatal(err)
+	}
+	if err := inv.RecordBuild(ctx, postgresBuild("older", older, "0ab07fa9")); err != nil {
+		t.Fatal(err)
+	}
+
+	held, err := inv.Held(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := held["postgres/server"]; got != "postgres@sha256:4bcd5f73" {
+		t.Errorf("postgres holds %q; want the newer request's image 4bcd5f73", got)
+	}
+	against, err := inv.BuiltAgainst(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := against["postgres"]; len(got) != 1 || got[0] != "mesh-tools/runtime@sha256:newer" {
+		t.Errorf("postgres stands on %v; want what the newer request stood on", got)
+	}
+
+	// Both are still recorded, the late one first as what happened lately.
+	builds, err := inv.Builds(ctx, "postgres", 5)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(builds) != 2 || builds[0].ID != "older" || !builds[0].Asked.Equal(older) {
+		t.Fatalf("both builds, newest heard first, with when they were asked: %+v", builds)
+	}
+}
+
+func TestABuildWithNoKnownRequestTimeIsOrderedByWhenItWasRecorded(t *testing.T) {
+	// What the mesh did before it kept the request time, so a row from before still answers.
+	inv := fresh(t)
+	ctx := context.Background()
+	for _, id := range []string{"first", "second"} {
+		b := aBuild(id, "shell", "")
+		b.Made = []Artifact{{Name: "config", Kind: "archive", Reference: "…/" + id}}
+		if err := inv.RecordBuild(ctx, b); err != nil {
+			t.Fatal(err)
+		}
+	}
+	held, err := inv.Held(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := held["shell/config"]; got != "…/second" {
+		t.Errorf("shell holds %q; want the one recorded last", got)
+	}
+}
+
+func TestARegistrationFromAnOlderRequestDoesNotReplaceANewerOne(t *testing.T) {
+	inv := fresh(t)
+	ctx := context.Background()
+	older := time.Date(2026, 10, 3, 21, 33, 45, 0, time.UTC)
+	newer := time.Date(2026, 10, 3, 21, 51, 57, 0, time.UTC)
+	from := func(asked time.Time) Source {
+		return Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/postgres",
+			BuiltFrom: "efff5415", Asked: asked}
+	}
+
+	if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "postgres", Version: "fixed"}, from(newer)); err != nil {
+		t.Fatal(err)
+	}
+	err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "postgres", Version: "stale"}, from(older))
+	if !errors.Is(err, ErrSuperseded) {
+		t.Fatalf("an older request's registration was not refused as superseded: %v", err)
+	}
+	shelf, err := inv.Catalogue(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := shelf["postgres"].Version; got != "fixed" {
+		t.Fatalf("postgres is %q; want the newer request's manifest", got)
+	}
+
+	// A later request, and a manifest handed over by hand — asked when it is handed over — both
+	// replace it as before.
+	if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "postgres", Version: "later"},
+		from(newer.Add(time.Minute))); err != nil {
+		t.Fatal(err)
+	}
+	if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "postgres", Version: "by-hand"}, Source{}); err != nil {
+		t.Fatal(err)
+	}
+	if shelf, _ := inv.Catalogue(ctx); shelf["postgres"].Version != "by-hand" {
+		t.Fatalf("postgres is %q; want the manifest handed over by hand", shelf["postgres"].Version)
+	}
+	src, err := inv.SourceOf(ctx, "postgres")
+	if err != nil || src.Repository != "novox/mesh-catalog" {
+		t.Fatalf("a hand registration erased the provenance: %+v %v", src, err)
+	}
+}