Files
mesh-controller/internal/inventory/chain_builds_layout_test.go
T
jschoubben 7f074d9c94 A layout run again sets no pointer back and skips records already moved (issue 1121)
The live start recorded its layout in chain_moved, so the marker is missing and the layout runs once more over the old keys. It wrote newest, worked, held, commit and source over the pointers of every build made since. Pointers are now made only where missing, and a record already under its new key is not read again.
2026-10-18 13:41:25 +02:00

462 lines
17 KiB
Go

package inventory
import (
"context"
"encoding/json"
"errors"
"fmt"
"reflect"
"sort"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/novox/mesh-controller/internal/chain"
)
// issue 1121: a build is under its module, so a lookup reads the keys it needs and no others.
type readCounting struct {
chain.Backend
gets, watches atomic.Int32
}
func (r *readCounting) Get(ctx context.Context, key string) (chain.Entry, bool, error) {
r.gets.Add(1)
return r.Backend.Get(ctx, key)
}
func (r *readCounting) Watch(ctx context.Context) (<-chan chain.Entry, error) {
r.watches.Add(1)
return r.Backend.Watch(ctx)
}
func (r *readCounting) WatchFiltered(ctx context.Context, f string) (<-chan chain.Entry, error) {
r.watches.Add(1)
return r.Backend.WatchFiltered(ctx, f)
}
func layoutChain(builds chain.Backend) (*Inventory, *chain.Chain) {
c := chain.New(builds, chain.NewMemory(), chain.NewMemory(), func(context.Context) error { return nil }).
WithManifests(builds)
inv := &Inventory{state: &chainState{}}
inv.UseChain(c)
return inv, c
}
var layoutAt = time.Date(2026, 10, 18, 12, 0, 0, 0, time.UTC)
func layoutBuild(module string, n int, commit, source string) Build {
return Build{ID: fmt.Sprintf("build-%d", n), Repository: "novox/" + module, Module: module, Commit: commit,
SourceFingerprint: source, ReuseKey: "v1:" + source, Made: []Artifact{{Name: module, Kind: "image", Reference: fmt.Sprintf("r/%s@sha256:%d", module, n)}},
Manifest: []byte(fmt.Sprintf(`{"module":%q,"n":%d}`, module, n)), At: layoutAt.Add(time.Duration(n) * time.Minute)}
}
func TestALookupOfOneModuleReadsThatModulesKeysOnly(t *testing.T) {
mem := chain.NewMemory()
inv, c := layoutChain(mem)
ctx := context.Background()
for n := 1; n <= 3; n++ {
for _, m := range []string{"alpha", "beta"} {
if err := inv.chainRecordBuild(ctx, c, layoutBuild(m, n, fmt.Sprintf("c%d", n), fmt.Sprintf("s%d", n))); err != nil {
t.Fatal(err)
}
}
}
got, err := inv.chainBuildList(ctx, c, "alpha", 10)
if err != nil || len(got) != 3 {
t.Fatalf("alpha's builds %+v: %v", got, err)
}
if want := map[string]int{"alpha.build.>": 1}; !reflect.DeepEqual(mem.Filtered(), want) {
t.Fatalf("the list of one module watched %v; want only %v", mem.Filtered(), want)
}
if _, err := inv.chainBuildFingerprints(ctx, c, "alpha"); err != nil {
t.Fatal(err)
}
for f := range mem.Filtered() {
if f != "alpha.build.>" && f != "alpha.commit.*" {
t.Errorf("a lookup of alpha watched %q", f)
}
}
}
func TestReuseBySourceFingerprintAndTheBuildOfACommitAreOneGet(t *testing.T) {
counting := &readCounting{Backend: chain.NewMemory()}
inv, c := layoutChain(counting)
ctx := context.Background()
for n := 1; n <= 20; n++ {
if err := inv.chainRecordBuild(ctx, c, layoutBuild("alpha", n, fmt.Sprintf("c%d", n), fmt.Sprintf("s%d", n))); err != nil {
t.Fatal(err)
}
}
counting.gets.Store(0)
counting.watches.Store(0)
id, err := inv.chainBuildOfSource(ctx, c, "alpha", "v1:s7")
if err != nil || id != "build-7" {
t.Fatalf("source s7 is %q: %v", id, err)
}
if counting.gets.Load() != 2 || counting.watches.Load() != 0 {
t.Fatalf("the reuse lookup made %d gets and %d watches; want 2 (the pointer, the record it names) and 0", counting.gets.Load(), counting.watches.Load())
}
counting.gets.Store(0)
if id, err = inv.chainBuildOf(ctx, c, "alpha", "c19"); err != nil || id != "build-19" {
t.Fatalf("commit c19 is %q: %v", id, err)
}
if counting.gets.Load() != 2 || counting.watches.Load() != 0 {
t.Fatalf("the build of a commit made %d gets and %d watches; want 2 and 0", counting.gets.Load(), counting.watches.Load())
}
}
func TestABuildWrittenIsFoundByCommitSourceIDAndRequest(t *testing.T) {
inv, c := layoutChain(chain.NewMemory())
ctx := context.Background()
ask := BuildRequest{ID: "build-5", Repository: "novox/alpha", Seat: "mesh-build-machine", At: layoutAt}
if err := inv.chainRecordBuildRequest(ctx, c, ask); err != nil {
t.Fatal(err)
}
if _, found, _ := inv.chainBuildByID(ctx, c, "build-5"); found {
t.Fatal("a request nobody answered was read as a build")
}
b := layoutBuild("alpha", 5, "c5", "s5")
if err := inv.chainRecordBuild(ctx, c, b); err != nil {
t.Fatal(err)
}
if id, _ := inv.chainBuildOf(ctx, c, "alpha", "c5"); id != "build-5" {
t.Errorf("by commit: %q", id)
}
if id, _ := inv.chainBuildOfSource(ctx, c, "alpha", "v1:s5"); id != "build-5" {
t.Errorf("by source: %q", id)
}
got, found, err := inv.chainBuildByID(ctx, c, "build-5")
if err != nil || !found || got.Module != "alpha" {
t.Fatalf("by id: %+v %v %v", got, found, err)
}
// The manifest is kept once, apart from the record, and read back whole.
raw, err := inv.chainManifestOf(ctx, c, "build-5")
if err != nil || !json.Valid(raw) || string(raw) != string(canonicalJSON(b.Manifest)) {
t.Fatalf("manifest %q: %v", raw, err)
}
k, _, _ := c.Builds.Get(ctx, "alpha.build.build-5")
if len(k.Value.Manifest) != 0 || k.Value.ManifestDigest == "" {
t.Errorf("the record keeps the manifest itself: %+v", k.Value)
}
if k.Value.Request == nil {
t.Error("the outcome lost the request it answers")
}
reqs, err := inv.chainRequests(ctx, c)
if err != nil || len(reqs) != 1 || !reqs[0].Heard || reqs[0].Module != "alpha" {
t.Fatalf("requests %+v: %v", reqs, err)
}
}
func legacyBuild(n int, module, commit string) chain.Build {
b := toChainBuild(layoutBuild(module, n, commit, "s"+commit))
return b
}
func keysOf(t *testing.T, mem chain.Backend) []string {
t.Helper()
keys, err := mem.Keys(context.Background())
if err != nil {
t.Fatal(err)
}
sort.Strings(keys)
return keys
}
// The layout of the bucket's keys is made once from the keys of before it: run again it changes nothing, the old keys stay,
// and the pointers say the newest.
func TestTheLayoutMigrationIsIdempotentAndKeepsTheOldKeys(t *testing.T) {
mem := chain.NewMemory()
inv, c := layoutChain(mem)
ctx := context.Background()
for n, m := range []string{"alpha", "alpha", "beta", "alpha"} {
old := legacyBuild(n+1, m, fmt.Sprintf("c%d", n+1))
if _, err := c.Builds.Create(ctx, old.ID, old); err != nil {
t.Fatal(err)
}
}
failed := legacyBuild(9, "alpha", "c9")
failed.Failed = "broke"
failed.SourceFingerprint = ""
if _, err := c.Builds.Create(ctx, failed.ID, failed); err != nil {
t.Fatal(err)
}
asked := chain.Build{ID: "build-10", State: chain.BuildAsked, Repository: "novox/alpha", Asked: layoutAt,
Request: &chain.Request{Repository: "novox/alpha", At: layoutAt}}
if _, err := c.Builds.Create(ctx, asked.ID, asked); err != nil {
t.Fatal(err)
}
first, err := inv.layoutBuilds(ctx, c, nil)
if err != nil || first.Written == 0 {
t.Fatalf("first run %+v: %v", first, err)
}
keys := keysOf(t, mem)
second, err := inv.layoutBuilds(ctx, c, nil)
if err != nil || second.Written != 0 {
t.Fatalf("second run %+v: %v", second, err)
}
if again := keysOf(t, mem); !reflect.DeepEqual(keys, again) {
t.Fatalf("keys changed on the second run:\n%v\n%v", keys, again)
}
has := func(k string) bool { return containsKey(keys, k) }
for _, k := range []string{"build-1", "build-9", "alpha.build.build-1", "alpha.build.build-4", "beta.build.build-3", "alpha.build.build-9", "req.build-10",
"id.build-3", "alpha.newest", "alpha.worked", "alpha.held", "alpha.commit.c4", "alpha.source.v1-sc4", "beta.worked"} {
if !has(k) {
t.Errorf("no key %s in %v", k, keys)
}
}
if e, _, _ := mem.Get(ctx, "alpha.newest"); string(e.Value) != "build-9" {
t.Errorf("alpha.newest = %q; want the failed build-9", e.Value)
}
if e, _, _ := mem.Get(ctx, "alpha.worked"); string(e.Value) != "build-4" {
t.Errorf("alpha.worked = %q; want build-4", e.Value)
}
if id, _ := inv.chainBuildOf(ctx, c, "alpha", "c4"); id != "build-4" {
t.Errorf("by commit after the layout: %q", id)
}
// No record is read from an old key.
list, err := inv.chainBuildList(ctx, c, "alpha", 10)
if err != nil || len(list) != 4 {
t.Fatalf("alpha's builds after the layout: %d %v", len(list), err)
}
}
func containsKey(s []string, k string) bool {
for _, x := range s {
if x == k {
return true
}
}
return false
}
// The manifest is kept once under its digest in the same bucket, and the moved record keeps none.
func TestAMovedRecordHoldsNoManifestAndItsManifestIsUnderItsDigest(t *testing.T) {
mem := chain.NewMemory()
inv, c := layoutChain(mem)
ctx := context.Background()
old := legacyBuild(1, "alpha", "c1")
if _, err := c.Builds.Create(ctx, old.ID, old); err != nil {
t.Fatal(err)
}
if _, err := inv.layoutBuilds(ctx, c, nil); err != nil {
t.Fatal(err)
}
k, found, err := c.Builds.Get(ctx, "alpha.build.build-1")
if err != nil || !found || len(k.Value.Manifest) != 0 || k.Value.ManifestDigest == "" {
t.Fatalf("moved record %+v %v %v", k.Value, found, err)
}
if !strings.HasPrefix(k.Value.ManifestDigest, "manifest.sha256-") {
t.Errorf("the digest %q is not a key under manifest.", k.Value.ManifestDigest)
}
e, found, _ := mem.Get(ctx, k.Value.ManifestDigest)
if !found || string(e.Value) != string(canonicalJSON([]byte(old.Manifest))) {
t.Errorf("the manifest under its digest is %q (found %v)", e.Value, found)
}
if got := chain.ManifestKey(e.Value); got != k.Value.ManifestDigest {
t.Errorf("the digest does not name the bytes: %s", got)
}
}
// A pointer is a hint: a record written without its pointers is found by reading the module's records, and the pointers are
// repaired; a pointer that names a record that is not what it was for is not followed.
func TestAMissingOrWrongPointerFallsBackToTheModulesRecordsAndIsRepaired(t *testing.T) {
mem := chain.NewMemory()
inv, c := layoutChain(mem)
ctx := context.Background()
for n := 1; n <= 3; n++ {
if err := inv.chainRecordBuild(ctx, c, layoutBuild("alpha", n, fmt.Sprintf("c%d", n), fmt.Sprintf("s%d", n))); err != nil {
t.Fatal(err)
}
}
// A record whose pointers the writer never made (a crash between the record and its pointers).
lost := toChainBuild(layoutBuild("alpha", 4, "c4", "s4"))
if _, err := c.Builds.Create(ctx, "alpha.build.build-4", lost); err != nil {
t.Fatal(err)
}
if id, err := inv.chainBuildOf(ctx, c, "alpha", "c4"); err != nil || id != "build-4" {
t.Fatalf("the build of c4 is %q: %v", id, err)
}
if e, found, _ := mem.Get(ctx, "alpha.commit.c4"); !found || pointedID(e.Value) != "build-4" {
t.Errorf("the commit pointer was not repaired: %q %v", e.Value, found)
}
if id, _ := inv.chainBuildOfSource(ctx, c, "alpha", "v1:s4"); id != "build-4" {
t.Errorf("the source pointer was not repaired: %q", id)
}
// A pointer to the wrong build is not followed.
e, _, _ := mem.Get(ctx, "alpha.commit.c2")
wrong, _ := json.Marshal(chain.CommitPointer{Build: "build-1"})
if _, err := mem.Update(ctx, "alpha.commit.c2", wrong, e.Revision); err != nil {
t.Fatal(err)
}
if id, err := inv.chainBuildOf(ctx, c, "alpha", "c2"); err != nil || id != "build-2" {
t.Fatalf("with a wrong pointer the build of c2 is %q: %v", id, err)
}
if e, _, _ := mem.Get(ctx, "alpha.commit.c2"); pointedID(e.Value) != "build-2" {
t.Errorf("the wrong pointer stayed: %s", e.Value)
}
// A commit nothing built is none.
if id, err := inv.chainBuildOf(ctx, c, "alpha", "nope"); err != nil || id != "" {
t.Errorf("an unbuilt commit is %q: %v", id, err)
}
}
type touchRecording struct {
chain.Backend
mu sync.Mutex
touched []string
}
func (r *touchRecording) note(s string) { r.mu.Lock(); r.touched = append(r.touched, s); r.mu.Unlock() }
func (r *touchRecording) Get(ctx context.Context, key string) (chain.Entry, bool, error) {
r.note("get " + key)
return r.Backend.Get(ctx, key)
}
func (r *touchRecording) Watch(ctx context.Context) (<-chan chain.Entry, error) {
r.note("watch >")
return r.Backend.Watch(ctx)
}
func (r *touchRecording) WatchFiltered(ctx context.Context, f string) (<-chan chain.Entry, error) {
r.note("watch " + f)
return r.Backend.WatchFiltered(ctx, f)
}
func (r *touchRecording) Keys(ctx context.Context) ([]string, error) {
r.note("keys")
return r.Backend.Keys(ctx)
}
// Once the layout is made, no read touches a key that is a build's id alone, or the bucket whole.
func TestAfterTheLayoutNoReadTouchesABareBuildIDKey(t *testing.T) {
rec := &touchRecording{Backend: chain.NewMemory()}
inv, c := layoutChain(rec)
ctx := context.Background()
old := legacyBuild(1, "alpha", "c1")
if _, err := c.Builds.Create(ctx, old.ID, old); err != nil {
t.Fatal(err)
}
if _, err := inv.layoutBuilds(ctx, c, nil); err != nil {
t.Fatal(err)
}
if err := inv.chainRecordBuild(ctx, c, layoutBuild("alpha", 2, "c2", "s2")); err != nil {
t.Fatal(err)
}
rec.mu.Lock()
rec.touched = nil
rec.mu.Unlock()
_, _ = inv.chainBuildList(ctx, c, "alpha", 5)
_, _ = inv.chainBuildList(ctx, c, "", 5)
_, _, _ = inv.chainBuildByID(ctx, c, "build-1")
_, _, _ = inv.chainBuildByID(ctx, c, "build-2")
_, _ = inv.chainBuildOf(ctx, c, "alpha", "c1")
_, _ = inv.chainBuildFingerprints(ctx, c, "alpha")
_, _ = inv.chainFingerprints(ctx, c)
_, _ = inv.chainManifestOf(ctx, c, "build-1")
_, _, _, _ = inv.chainManifestBuild(ctx, c, "alpha", "c1")
_, _ = chainBuilds(ctx, c)
_, _ = inv.chainRequests(ctx, c)
_, _ = inv.chainBuiltAgainst(ctx, c)
rec.mu.Lock()
defer rec.mu.Unlock()
for _, what := range rec.touched {
verb, key, _ := strings.Cut(what, " ")
if what == "keys" || what == "watch >" || what == "watch *" || (verb == "get" && !strings.Contains(key, ".")) {
t.Errorf("a read touched %q", what)
}
}
}
// A pointer follows the newest build, not the last write: two builds overlap and the older, asked first, finishes last.
func TestAnOlderBuildFinishingLastLeavesThePointersOnTheNewer(t *testing.T) {
mem := chain.NewMemory()
inv, c := layoutChain(mem)
ctx := context.Background()
older := layoutBuild("alpha", 1, "c1", "s1")
older.Asked = layoutAt
older.At = layoutAt.Add(20 * time.Minute) // finished last
newer := layoutBuild("alpha", 2, "c1", "s2")
newer.Asked = layoutAt.Add(5 * time.Minute)
newer.At = layoutAt.Add(10 * time.Minute)
for _, b := range []Build{newer, older} {
if err := inv.chainRecordBuild(ctx, c, b); err != nil {
t.Fatal(err)
}
}
for _, key := range []string{"alpha.newest", "alpha.worked"} {
if e, _, _ := mem.Get(ctx, key); string(e.Value) != "build-2" {
t.Errorf("%s = %q; want the newer build-2", key, e.Value)
}
}
if e, _, _ := mem.Get(ctx, "alpha.commit.c1"); pointedID(e.Value) != "build-2" {
t.Errorf("the commit pointer = %s; want build-2", e.Value)
}
// The older build is still found by what only it has.
if id, _ := inv.chainBuildOfSource(ctx, c, "alpha", "v1:s1"); id != "build-1" {
t.Errorf("the older build's own reuse key finds %q", id)
}
}
// Nothing reads the chain on the bus until the controller's start has laid the builds out and written the marker; the layout
// is done once.
func TestTheChainIsRefusedUntilTheBuildsAreLaidOutAndTheLayoutIsDoneOnce(t *testing.T) {
mem := chain.NewMemory()
inv, c := layoutChain(mem)
inv.RequireLayout()
ctx := context.Background()
old := legacyBuild(1, "alpha", "c1")
if _, err := c.Builds.Create(ctx, old.ID, old); err != nil {
t.Fatal(err)
}
if _, err := inv.theChain(ctx); !errors.Is(err, ErrNotLaidOut) {
t.Fatalf("a read before the layout answered %v", err)
}
if err := inv.LayBuildsOut(ctx, nil); err != nil {
t.Fatal(err)
}
if _, err := inv.theChain(ctx); err != nil {
t.Fatalf("a read after the layout answered %v", err)
}
keys := keysOf(t, mem)
if !containsKey(keys, chain.LayoutMarkerKey) {
t.Fatalf("no marker among %v", keys)
}
if err := inv.LayBuildsOut(ctx, nil); err != nil || !reflect.DeepEqual(keys, keysOf(t, mem)) {
t.Fatalf("the second layout changed the bucket: %v", err)
}
}
// A layout run again — the live start recorded the first in chain_moved, not in the marker — never sets a pointer back to an
// older build, and does not read a manifest of a record already under its new key.
func TestALayoutRunAgainLeavesThePointersOfTheBuildsMadeSince(t *testing.T) {
counting := &readCounting{Backend: chain.NewMemory()}
inv, c := layoutChain(counting)
ctx := context.Background()
for n := 1; n <= 2; n++ {
old := legacyBuild(n, "alpha", fmt.Sprintf("c%d", n))
if _, err := c.Builds.Create(ctx, old.ID, old); err != nil {
t.Fatal(err)
}
}
if _, err := inv.layoutBuilds(ctx, c, nil); err != nil {
t.Fatal(err)
}
if err := inv.chainRecordBuild(ctx, c, layoutBuild("alpha", 3, "c3", "s3")); err != nil {
t.Fatal(err)
}
counting.gets.Store(0)
r, err := inv.layoutBuilds(ctx, c, nil)
if err != nil || r.Written != 0 {
t.Fatalf("the second layout %+v: %v", r, err)
}
for _, key := range []string{"alpha.newest", "alpha.worked", "alpha.held"} {
if e, _, _ := counting.Get(ctx, key); string(e.Value) != "build-3" {
t.Errorf("%s = %q after the second layout; want build-3", key, e.Value)
}
}
if e, _, _ := counting.Get(ctx, "alpha.commit.c3"); pointedID(e.Value) != "build-3" {
t.Errorf("commit c3 = %s", e.Value)
}
}