Files
mesh-controller/internal/inventory/chain_builds_layout_test.go
T
jschoubben a825a4c904
mesh/merge-gate head a825a4c90446 merged into trunk d3556f452e8c — pass: builds build-agent, mesh-controller → ace, g14, novox, shanks; no bus step; every …
mesh/repo-check head a825a4c90446 merged into trunk d3556f452e8c — pass: its merge-check.sh passed
mesh/delivery delivered
The first read of a chain not laid out lays it out, so no read of the start is refused (issue 1121)
The layout gate is switched on when the stores open (UseChainOpener), before
attachTheChain; a read of the start came first, was refused, and the
controller exited. A read that finds no marker now lays the builds out once,
under a lock, and is refused only when that fails (a process without the
lease).
2026-10-18 14:39:43 +02:00

462 lines
17 KiB
Go

package inventory
import (
"context"
"encoding/json"
"fmt"
"reflect"
"sort"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/novox/mesh-controller/internal/chain"
)
// issue 1121: a build is under its module, so a lookup reads the keys it needs and no others.
type readCounting struct {
chain.Backend
gets, watches atomic.Int32
}
func (r *readCounting) Get(ctx context.Context, key string) (chain.Entry, bool, error) {
r.gets.Add(1)
return r.Backend.Get(ctx, key)
}
func (r *readCounting) Watch(ctx context.Context) (<-chan chain.Entry, error) {
r.watches.Add(1)
return r.Backend.Watch(ctx)
}
func (r *readCounting) WatchFiltered(ctx context.Context, f string) (<-chan chain.Entry, error) {
r.watches.Add(1)
return r.Backend.WatchFiltered(ctx, f)
}
func layoutChain(builds chain.Backend) (*Inventory, *chain.Chain) {
c := chain.New(builds, chain.NewMemory(), chain.NewMemory(), func(context.Context) error { return nil }).
WithManifests(builds)
inv := &Inventory{state: &chainState{}}
inv.UseChain(c)
return inv, c
}
var layoutAt = time.Date(2026, 10, 18, 12, 0, 0, 0, time.UTC)
func layoutBuild(module string, n int, commit, source string) Build {
return Build{ID: fmt.Sprintf("build-%d", n), Repository: "novox/" + module, Module: module, Commit: commit,
SourceFingerprint: source, ReuseKey: "v1:" + source, Made: []Artifact{{Name: module, Kind: "image", Reference: fmt.Sprintf("r/%s@sha256:%d", module, n)}},
Manifest: []byte(fmt.Sprintf(`{"module":%q,"n":%d}`, module, n)), At: layoutAt.Add(time.Duration(n) * time.Minute)}
}
func TestALookupOfOneModuleReadsThatModulesKeysOnly(t *testing.T) {
mem := chain.NewMemory()
inv, c := layoutChain(mem)
ctx := context.Background()
for n := 1; n <= 3; n++ {
for _, m := range []string{"alpha", "beta"} {
if err := inv.chainRecordBuild(ctx, c, layoutBuild(m, n, fmt.Sprintf("c%d", n), fmt.Sprintf("s%d", n))); err != nil {
t.Fatal(err)
}
}
}
got, err := inv.chainBuildList(ctx, c, "alpha", 10)
if err != nil || len(got) != 3 {
t.Fatalf("alpha's builds %+v: %v", got, err)
}
if want := map[string]int{"alpha.build.>": 1}; !reflect.DeepEqual(mem.Filtered(), want) {
t.Fatalf("the list of one module watched %v; want only %v", mem.Filtered(), want)
}
if _, err := inv.chainBuildFingerprints(ctx, c, "alpha"); err != nil {
t.Fatal(err)
}
for f := range mem.Filtered() {
if f != "alpha.build.>" && f != "alpha.commit.*" {
t.Errorf("a lookup of alpha watched %q", f)
}
}
}
func TestReuseBySourceFingerprintAndTheBuildOfACommitAreOneGet(t *testing.T) {
counting := &readCounting{Backend: chain.NewMemory()}
inv, c := layoutChain(counting)
ctx := context.Background()
for n := 1; n <= 20; n++ {
if err := inv.chainRecordBuild(ctx, c, layoutBuild("alpha", n, fmt.Sprintf("c%d", n), fmt.Sprintf("s%d", n))); err != nil {
t.Fatal(err)
}
}
counting.gets.Store(0)
counting.watches.Store(0)
id, err := inv.chainBuildOfSource(ctx, c, "alpha", "v1:s7")
if err != nil || id != "build-7" {
t.Fatalf("source s7 is %q: %v", id, err)
}
if counting.gets.Load() != 2 || counting.watches.Load() != 0 {
t.Fatalf("the reuse lookup made %d gets and %d watches; want 2 (the pointer, the record it names) and 0", counting.gets.Load(), counting.watches.Load())
}
counting.gets.Store(0)
if id, err = inv.chainBuildOf(ctx, c, "alpha", "c19"); err != nil || id != "build-19" {
t.Fatalf("commit c19 is %q: %v", id, err)
}
if counting.gets.Load() != 2 || counting.watches.Load() != 0 {
t.Fatalf("the build of a commit made %d gets and %d watches; want 2 and 0", counting.gets.Load(), counting.watches.Load())
}
}
func TestABuildWrittenIsFoundByCommitSourceIDAndRequest(t *testing.T) {
inv, c := layoutChain(chain.NewMemory())
ctx := context.Background()
ask := BuildRequest{ID: "build-5", Repository: "novox/alpha", Seat: "mesh-build-machine", At: layoutAt}
if err := inv.chainRecordBuildRequest(ctx, c, ask); err != nil {
t.Fatal(err)
}
if _, found, _ := inv.chainBuildByID(ctx, c, "build-5"); found {
t.Fatal("a request nobody answered was read as a build")
}
b := layoutBuild("alpha", 5, "c5", "s5")
if err := inv.chainRecordBuild(ctx, c, b); err != nil {
t.Fatal(err)
}
if id, _ := inv.chainBuildOf(ctx, c, "alpha", "c5"); id != "build-5" {
t.Errorf("by commit: %q", id)
}
if id, _ := inv.chainBuildOfSource(ctx, c, "alpha", "v1:s5"); id != "build-5" {
t.Errorf("by source: %q", id)
}
got, found, err := inv.chainBuildByID(ctx, c, "build-5")
if err != nil || !found || got.Module != "alpha" {
t.Fatalf("by id: %+v %v %v", got, found, err)
}
// The manifest is kept once, apart from the record, and read back whole.
raw, err := inv.chainManifestOf(ctx, c, "build-5")
if err != nil || !json.Valid(raw) || string(raw) != string(canonicalJSON(b.Manifest)) {
t.Fatalf("manifest %q: %v", raw, err)
}
k, _, _ := c.Builds.Get(ctx, "alpha.build.build-5")
if len(k.Value.Manifest) != 0 || k.Value.ManifestDigest == "" {
t.Errorf("the record keeps the manifest itself: %+v", k.Value)
}
if k.Value.Request == nil {
t.Error("the outcome lost the request it answers")
}
reqs, err := inv.chainRequests(ctx, c)
if err != nil || len(reqs) != 1 || !reqs[0].Heard || reqs[0].Module != "alpha" {
t.Fatalf("requests %+v: %v", reqs, err)
}
}
func legacyBuild(n int, module, commit string) chain.Build {
b := toChainBuild(layoutBuild(module, n, commit, "s"+commit))
return b
}
func keysOf(t *testing.T, mem chain.Backend) []string {
t.Helper()
keys, err := mem.Keys(context.Background())
if err != nil {
t.Fatal(err)
}
sort.Strings(keys)
return keys
}
// The layout of the bucket's keys is made once from the keys of before it: run again it changes nothing, the old keys stay,
// and the pointers say the newest.
func TestTheLayoutMigrationIsIdempotentAndKeepsTheOldKeys(t *testing.T) {
mem := chain.NewMemory()
inv, c := layoutChain(mem)
ctx := context.Background()
for n, m := range []string{"alpha", "alpha", "beta", "alpha"} {
old := legacyBuild(n+1, m, fmt.Sprintf("c%d", n+1))
if _, err := c.Builds.Create(ctx, old.ID, old); err != nil {
t.Fatal(err)
}
}
failed := legacyBuild(9, "alpha", "c9")
failed.Failed = "broke"
failed.SourceFingerprint = ""
if _, err := c.Builds.Create(ctx, failed.ID, failed); err != nil {
t.Fatal(err)
}
asked := chain.Build{ID: "build-10", State: chain.BuildAsked, Repository: "novox/alpha", Asked: layoutAt,
Request: &chain.Request{Repository: "novox/alpha", At: layoutAt}}
if _, err := c.Builds.Create(ctx, asked.ID, asked); err != nil {
t.Fatal(err)
}
first, err := inv.layoutBuilds(ctx, c, nil)
if err != nil || first.Written == 0 {
t.Fatalf("first run %+v: %v", first, err)
}
keys := keysOf(t, mem)
second, err := inv.layoutBuilds(ctx, c, nil)
if err != nil || second.Written != 0 {
t.Fatalf("second run %+v: %v", second, err)
}
if again := keysOf(t, mem); !reflect.DeepEqual(keys, again) {
t.Fatalf("keys changed on the second run:\n%v\n%v", keys, again)
}
has := func(k string) bool { return containsKey(keys, k) }
for _, k := range []string{"build-1", "build-9", "alpha.build.build-1", "alpha.build.build-4", "beta.build.build-3", "alpha.build.build-9", "req.build-10",
"id.build-3", "alpha.newest", "alpha.worked", "alpha.held", "alpha.commit.c4", "alpha.source.v1-sc4", "beta.worked"} {
if !has(k) {
t.Errorf("no key %s in %v", k, keys)
}
}
if e, _, _ := mem.Get(ctx, "alpha.newest"); string(e.Value) != "build-9" {
t.Errorf("alpha.newest = %q; want the failed build-9", e.Value)
}
if e, _, _ := mem.Get(ctx, "alpha.worked"); string(e.Value) != "build-4" {
t.Errorf("alpha.worked = %q; want build-4", e.Value)
}
if id, _ := inv.chainBuildOf(ctx, c, "alpha", "c4"); id != "build-4" {
t.Errorf("by commit after the layout: %q", id)
}
// No record is read from an old key.
list, err := inv.chainBuildList(ctx, c, "alpha", 10)
if err != nil || len(list) != 4 {
t.Fatalf("alpha's builds after the layout: %d %v", len(list), err)
}
}
func containsKey(s []string, k string) bool {
for _, x := range s {
if x == k {
return true
}
}
return false
}
// The manifest is kept once under its digest in the same bucket, and the moved record keeps none.
func TestAMovedRecordHoldsNoManifestAndItsManifestIsUnderItsDigest(t *testing.T) {
mem := chain.NewMemory()
inv, c := layoutChain(mem)
ctx := context.Background()
old := legacyBuild(1, "alpha", "c1")
if _, err := c.Builds.Create(ctx, old.ID, old); err != nil {
t.Fatal(err)
}
if _, err := inv.layoutBuilds(ctx, c, nil); err != nil {
t.Fatal(err)
}
k, found, err := c.Builds.Get(ctx, "alpha.build.build-1")
if err != nil || !found || len(k.Value.Manifest) != 0 || k.Value.ManifestDigest == "" {
t.Fatalf("moved record %+v %v %v", k.Value, found, err)
}
if !strings.HasPrefix(k.Value.ManifestDigest, "manifest.sha256-") {
t.Errorf("the digest %q is not a key under manifest.", k.Value.ManifestDigest)
}
e, found, _ := mem.Get(ctx, k.Value.ManifestDigest)
if !found || string(e.Value) != string(canonicalJSON([]byte(old.Manifest))) {
t.Errorf("the manifest under its digest is %q (found %v)", e.Value, found)
}
if got := chain.ManifestKey(e.Value); got != k.Value.ManifestDigest {
t.Errorf("the digest does not name the bytes: %s", got)
}
}
// A pointer is a hint: a record written without its pointers is found by reading the module's records, and the pointers are
// repaired; a pointer that names a record that is not what it was for is not followed.
func TestAMissingOrWrongPointerFallsBackToTheModulesRecordsAndIsRepaired(t *testing.T) {
mem := chain.NewMemory()
inv, c := layoutChain(mem)
ctx := context.Background()
for n := 1; n <= 3; n++ {
if err := inv.chainRecordBuild(ctx, c, layoutBuild("alpha", n, fmt.Sprintf("c%d", n), fmt.Sprintf("s%d", n))); err != nil {
t.Fatal(err)
}
}
// A record whose pointers the writer never made (a crash between the record and its pointers).
lost := toChainBuild(layoutBuild("alpha", 4, "c4", "s4"))
if _, err := c.Builds.Create(ctx, "alpha.build.build-4", lost); err != nil {
t.Fatal(err)
}
if id, err := inv.chainBuildOf(ctx, c, "alpha", "c4"); err != nil || id != "build-4" {
t.Fatalf("the build of c4 is %q: %v", id, err)
}
if e, found, _ := mem.Get(ctx, "alpha.commit.c4"); !found || pointedID(e.Value) != "build-4" {
t.Errorf("the commit pointer was not repaired: %q %v", e.Value, found)
}
if id, _ := inv.chainBuildOfSource(ctx, c, "alpha", "v1:s4"); id != "build-4" {
t.Errorf("the source pointer was not repaired: %q", id)
}
// A pointer to the wrong build is not followed.
e, _, _ := mem.Get(ctx, "alpha.commit.c2")
wrong, _ := json.Marshal(chain.CommitPointer{Build: "build-1"})
if _, err := mem.Update(ctx, "alpha.commit.c2", wrong, e.Revision); err != nil {
t.Fatal(err)
}
if id, err := inv.chainBuildOf(ctx, c, "alpha", "c2"); err != nil || id != "build-2" {
t.Fatalf("with a wrong pointer the build of c2 is %q: %v", id, err)
}
if e, _, _ := mem.Get(ctx, "alpha.commit.c2"); pointedID(e.Value) != "build-2" {
t.Errorf("the wrong pointer stayed: %s", e.Value)
}
// A commit nothing built is none.
if id, err := inv.chainBuildOf(ctx, c, "alpha", "nope"); err != nil || id != "" {
t.Errorf("an unbuilt commit is %q: %v", id, err)
}
}
type touchRecording struct {
chain.Backend
mu sync.Mutex
touched []string
}
func (r *touchRecording) note(s string) { r.mu.Lock(); r.touched = append(r.touched, s); r.mu.Unlock() }
func (r *touchRecording) Get(ctx context.Context, key string) (chain.Entry, bool, error) {
r.note("get " + key)
return r.Backend.Get(ctx, key)
}
func (r *touchRecording) Watch(ctx context.Context) (<-chan chain.Entry, error) {
r.note("watch >")
return r.Backend.Watch(ctx)
}
func (r *touchRecording) WatchFiltered(ctx context.Context, f string) (<-chan chain.Entry, error) {
r.note("watch " + f)
return r.Backend.WatchFiltered(ctx, f)
}
func (r *touchRecording) Keys(ctx context.Context) ([]string, error) {
r.note("keys")
return r.Backend.Keys(ctx)
}
// Once the layout is made, no read touches a key that is a build's id alone, or the bucket whole.
func TestAfterTheLayoutNoReadTouchesABareBuildIDKey(t *testing.T) {
rec := &touchRecording{Backend: chain.NewMemory()}
inv, c := layoutChain(rec)
ctx := context.Background()
old := legacyBuild(1, "alpha", "c1")
if _, err := c.Builds.Create(ctx, old.ID, old); err != nil {
t.Fatal(err)
}
if _, err := inv.layoutBuilds(ctx, c, nil); err != nil {
t.Fatal(err)
}
if err := inv.chainRecordBuild(ctx, c, layoutBuild("alpha", 2, "c2", "s2")); err != nil {
t.Fatal(err)
}
rec.mu.Lock()
rec.touched = nil
rec.mu.Unlock()
_, _ = inv.chainBuildList(ctx, c, "alpha", 5)
_, _ = inv.chainBuildList(ctx, c, "", 5)
_, _, _ = inv.chainBuildByID(ctx, c, "build-1")
_, _, _ = inv.chainBuildByID(ctx, c, "build-2")
_, _ = inv.chainBuildOf(ctx, c, "alpha", "c1")
_, _ = inv.chainBuildFingerprints(ctx, c, "alpha")
_, _ = inv.chainFingerprints(ctx, c)
_, _ = inv.chainManifestOf(ctx, c, "build-1")
_, _, _, _ = inv.chainManifestBuild(ctx, c, "alpha", "c1")
_, _ = chainBuilds(ctx, c)
_, _ = inv.chainRequests(ctx, c)
_, _ = inv.chainBuiltAgainst(ctx, c)
rec.mu.Lock()
defer rec.mu.Unlock()
for _, what := range rec.touched {
verb, key, _ := strings.Cut(what, " ")
if what == "keys" || what == "watch >" || what == "watch *" || (verb == "get" && !strings.Contains(key, ".")) {
t.Errorf("a read touched %q", what)
}
}
}
// A pointer follows the newest build, not the last write: two builds overlap and the older, asked first, finishes last.
func TestAnOlderBuildFinishingLastLeavesThePointersOnTheNewer(t *testing.T) {
mem := chain.NewMemory()
inv, c := layoutChain(mem)
ctx := context.Background()
older := layoutBuild("alpha", 1, "c1", "s1")
older.Asked = layoutAt
older.At = layoutAt.Add(20 * time.Minute) // finished last
newer := layoutBuild("alpha", 2, "c1", "s2")
newer.Asked = layoutAt.Add(5 * time.Minute)
newer.At = layoutAt.Add(10 * time.Minute)
for _, b := range []Build{newer, older} {
if err := inv.chainRecordBuild(ctx, c, b); err != nil {
t.Fatal(err)
}
}
for _, key := range []string{"alpha.newest", "alpha.worked"} {
if e, _, _ := mem.Get(ctx, key); string(e.Value) != "build-2" {
t.Errorf("%s = %q; want the newer build-2", key, e.Value)
}
}
if e, _, _ := mem.Get(ctx, "alpha.commit.c1"); pointedID(e.Value) != "build-2" {
t.Errorf("the commit pointer = %s; want build-2", e.Value)
}
// The older build is still found by what only it has.
if id, _ := inv.chainBuildOfSource(ctx, c, "alpha", "v1:s1"); id != "build-1" {
t.Errorf("the older build's own reuse key finds %q", id)
}
}
// Nothing reads the chain on the bus before the builds are laid out and the marker written: the first read of a process that
// may write lays them out itself (issue 1121: a refused read of the start made the controller exit), and the layout is done
// once.
func TestTheFirstReadLaysTheBuildsOutAndTheLayoutIsDoneOnce(t *testing.T) {
mem := chain.NewMemory()
inv, c := layoutChain(mem)
inv.RequireLayout()
ctx := context.Background()
old := legacyBuild(1, "alpha", "c1")
if _, err := c.Builds.Create(ctx, old.ID, old); err != nil {
t.Fatal(err)
}
if _, err := inv.theChain(ctx); err != nil {
t.Fatalf("the first read was refused, not laid out: %v", err)
}
if !containsKey(keysOf(t, mem), chain.BuildKey("alpha", old.ID)) {
t.Fatalf("the first read did not lay %s out under its module: %v", old.ID, keysOf(t, mem))
}
if _, err := inv.theChain(ctx); err != nil {
t.Fatalf("a read after the layout answered %v", err)
}
keys := keysOf(t, mem)
if !containsKey(keys, chain.LayoutMarkerKey) {
t.Fatalf("no marker among %v", keys)
}
if err := inv.LayBuildsOut(ctx, nil); err != nil || !reflect.DeepEqual(keys, keysOf(t, mem)) {
t.Fatalf("the second layout changed the bucket: %v", err)
}
}
// A layout run again — the live start recorded the first in chain_moved, not in the marker — never sets a pointer back to an
// older build, and does not read a manifest of a record already under its new key.
func TestALayoutRunAgainLeavesThePointersOfTheBuildsMadeSince(t *testing.T) {
counting := &readCounting{Backend: chain.NewMemory()}
inv, c := layoutChain(counting)
ctx := context.Background()
for n := 1; n <= 2; n++ {
old := legacyBuild(n, "alpha", fmt.Sprintf("c%d", n))
if _, err := c.Builds.Create(ctx, old.ID, old); err != nil {
t.Fatal(err)
}
}
if _, err := inv.layoutBuilds(ctx, c, nil); err != nil {
t.Fatal(err)
}
if err := inv.chainRecordBuild(ctx, c, layoutBuild("alpha", 3, "c3", "s3")); err != nil {
t.Fatal(err)
}
counting.gets.Store(0)
r, err := inv.layoutBuilds(ctx, c, nil)
if err != nil || r.Written != 0 {
t.Fatalf("the second layout %+v: %v", r, err)
}
for _, key := range []string{"alpha.newest", "alpha.worked", "alpha.held"} {
if e, _, _ := counting.Get(ctx, key); string(e.Value) != "build-3" {
t.Errorf("%s = %q after the second layout; want build-3", key, e.Value)
}
}
if e, _, _ := counting.Get(ctx, "alpha.commit.c3"); pointedID(e.Value) != "build-3" {
t.Errorf("commit c3 = %s", e.Value)
}
}