jschoubben
c795915315
mesh/merge-gate head c79591531575 merged into trunk aa02e7bf4e78 — pass: builds build-agent, mesh-controller → ace, g14, novox, shanks; no bus step; every …
mesh/repo-check head c79591531575 merged into trunk aa02e7bf4e78 — pass: its merge-check.sh passed
mesh/delivery delivered
Milestone 5 moved 8,599 build records onto the bus under bare build ids, each carrying its module's whole manifest. Every build lookup then read the whole bucket, and a delivery plan's send did that per module per machine while it held the plans and the machine's declaration: the send of build-agent never finished, and every other act and the status waited behind it. Builds now live under <module>.build.<id> (requests under req.<id>, id.<id> names a build's module), with pointer keys newest, worked, held, commit.<sha> and source.<reuse key> that move only to a newer build and are read as hints, repaired when wrong. Manifests are kept once in the same bucket under manifest.sha256-<hex>, so no grant the running bus lacks is needed. The old keys are laid out once at start, idempotently, and left unread. The calls a plan look makes on the silent clocks while it holds the plans are bounded, since the vendored kv.Keys ignores its context.
399 lines
14 KiB
Go
399 lines
14 KiB
Go
package inventory
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"reflect"
|
|
"sort"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/novox/mesh-controller/internal/chain"
|
|
)
|
|
|
|
// issue 1121: a build is under its module, so a lookup reads the keys it needs and no others.
|
|
|
|
type readCounting struct {
|
|
chain.Backend
|
|
gets, watches atomic.Int32
|
|
}
|
|
|
|
func (r *readCounting) Get(ctx context.Context, key string) (chain.Entry, bool, error) {
|
|
r.gets.Add(1)
|
|
return r.Backend.Get(ctx, key)
|
|
}
|
|
|
|
func (r *readCounting) Watch(ctx context.Context) (<-chan chain.Entry, error) {
|
|
r.watches.Add(1)
|
|
return r.Backend.Watch(ctx)
|
|
}
|
|
|
|
func (r *readCounting) WatchFiltered(ctx context.Context, f string) (<-chan chain.Entry, error) {
|
|
r.watches.Add(1)
|
|
return r.Backend.WatchFiltered(ctx, f)
|
|
}
|
|
|
|
func layoutChain(builds chain.Backend) (*Inventory, *chain.Chain) {
|
|
c := chain.New(builds, chain.NewMemory(), chain.NewMemory(), func(context.Context) error { return nil }).
|
|
WithManifests(builds)
|
|
inv := &Inventory{moved: &movedKinds{}}
|
|
inv.UseChain(c)
|
|
return inv, c
|
|
}
|
|
|
|
var layoutAt = time.Date(2026, 10, 18, 12, 0, 0, 0, time.UTC)
|
|
|
|
func layoutBuild(module string, n int, commit, source string) Build {
|
|
return Build{ID: fmt.Sprintf("build-%d", n), Repository: "novox/" + module, Module: module, Commit: commit,
|
|
SourceFingerprint: source, ReuseKey: "v1:" + source, Made: []Artifact{{Name: module, Kind: "image", Reference: fmt.Sprintf("r/%s@sha256:%d", module, n)}},
|
|
Manifest: []byte(fmt.Sprintf(`{"module":%q,"n":%d}`, module, n)), At: layoutAt.Add(time.Duration(n) * time.Minute)}
|
|
}
|
|
|
|
func TestALookupOfOneModuleReadsThatModulesKeysOnly(t *testing.T) {
|
|
mem := chain.NewMemory()
|
|
inv, c := layoutChain(mem)
|
|
ctx := context.Background()
|
|
for n := 1; n <= 3; n++ {
|
|
for _, m := range []string{"alpha", "beta"} {
|
|
if err := inv.chainRecordBuild(ctx, c, layoutBuild(m, n, fmt.Sprintf("c%d", n), fmt.Sprintf("s%d", n))); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
}
|
|
got, err := inv.chainBuildList(ctx, c, "alpha", 10)
|
|
if err != nil || len(got) != 3 {
|
|
t.Fatalf("alpha's builds %+v: %v", got, err)
|
|
}
|
|
if want := map[string]int{"alpha.build.>": 1}; !reflect.DeepEqual(mem.Filtered(), want) {
|
|
t.Fatalf("the list of one module watched %v; want only %v", mem.Filtered(), want)
|
|
}
|
|
if _, err := inv.chainBuildFingerprints(ctx, c, "alpha"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for f := range mem.Filtered() {
|
|
if f != "alpha.build.>" && f != "alpha.commit.*" {
|
|
t.Errorf("a lookup of alpha watched %q", f)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestReuseBySourceFingerprintAndTheBuildOfACommitAreOneGet(t *testing.T) {
|
|
counting := &readCounting{Backend: chain.NewMemory()}
|
|
inv, c := layoutChain(counting)
|
|
ctx := context.Background()
|
|
for n := 1; n <= 20; n++ {
|
|
if err := inv.chainRecordBuild(ctx, c, layoutBuild("alpha", n, fmt.Sprintf("c%d", n), fmt.Sprintf("s%d", n))); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
counting.gets.Store(0)
|
|
counting.watches.Store(0)
|
|
id, err := inv.chainBuildOfSource(ctx, c, "alpha", "v1:s7")
|
|
if err != nil || id != "build-7" {
|
|
t.Fatalf("source s7 is %q: %v", id, err)
|
|
}
|
|
if counting.gets.Load() != 2 || counting.watches.Load() != 0 {
|
|
t.Fatalf("the reuse lookup made %d gets and %d watches; want 2 (the pointer, the record it names) and 0", counting.gets.Load(), counting.watches.Load())
|
|
}
|
|
counting.gets.Store(0)
|
|
if id, err = inv.chainBuildOf(ctx, c, "alpha", "c19"); err != nil || id != "build-19" {
|
|
t.Fatalf("commit c19 is %q: %v", id, err)
|
|
}
|
|
if counting.gets.Load() != 2 || counting.watches.Load() != 0 {
|
|
t.Fatalf("the build of a commit made %d gets and %d watches; want 2 and 0", counting.gets.Load(), counting.watches.Load())
|
|
}
|
|
}
|
|
|
|
func TestABuildWrittenIsFoundByCommitSourceIDAndRequest(t *testing.T) {
|
|
inv, c := layoutChain(chain.NewMemory())
|
|
ctx := context.Background()
|
|
ask := BuildRequest{ID: "build-5", Repository: "novox/alpha", Seat: "mesh-build-machine", At: layoutAt}
|
|
if err := inv.chainRecordBuildRequest(ctx, c, ask); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, found, _ := inv.chainBuildByID(ctx, c, "build-5"); found {
|
|
t.Fatal("a request nobody answered was read as a build")
|
|
}
|
|
b := layoutBuild("alpha", 5, "c5", "s5")
|
|
if err := inv.chainRecordBuild(ctx, c, b); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if id, _ := inv.chainBuildOf(ctx, c, "alpha", "c5"); id != "build-5" {
|
|
t.Errorf("by commit: %q", id)
|
|
}
|
|
if id, _ := inv.chainBuildOfSource(ctx, c, "alpha", "v1:s5"); id != "build-5" {
|
|
t.Errorf("by source: %q", id)
|
|
}
|
|
got, found, err := inv.chainBuildByID(ctx, c, "build-5")
|
|
if err != nil || !found || got.Module != "alpha" {
|
|
t.Fatalf("by id: %+v %v %v", got, found, err)
|
|
}
|
|
// The manifest is kept once, apart from the record, and read back whole.
|
|
raw, err := inv.chainManifestOf(ctx, c, "build-5")
|
|
if err != nil || !json.Valid(raw) || string(raw) != string(canonicalJSON(b.Manifest)) {
|
|
t.Fatalf("manifest %q: %v", raw, err)
|
|
}
|
|
k, _, _ := c.Builds.Get(ctx, "alpha.build.build-5")
|
|
if len(k.Value.Manifest) != 0 || k.Value.ManifestDigest == "" {
|
|
t.Errorf("the record keeps the manifest itself: %+v", k.Value)
|
|
}
|
|
if k.Value.Request == nil {
|
|
t.Error("the outcome lost the request it answers")
|
|
}
|
|
reqs, err := inv.chainRequests(ctx, c)
|
|
if err != nil || len(reqs) != 1 || !reqs[0].Heard || reqs[0].Module != "alpha" {
|
|
t.Fatalf("requests %+v: %v", reqs, err)
|
|
}
|
|
}
|
|
|
|
func legacyBuild(n int, module, commit string) chain.Build {
|
|
b := toChainBuild(layoutBuild(module, n, commit, "s"+commit))
|
|
return b
|
|
}
|
|
|
|
func keysOf(t *testing.T, mem chain.Backend) []string {
|
|
t.Helper()
|
|
keys, err := mem.Keys(context.Background())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
sort.Strings(keys)
|
|
return keys
|
|
}
|
|
|
|
// The layout of the bucket's keys is made once from the keys of before it: run again it changes nothing, the old keys stay,
|
|
// and the pointers say the newest.
|
|
func TestTheLayoutMigrationIsIdempotentAndKeepsTheOldKeys(t *testing.T) {
|
|
mem := chain.NewMemory()
|
|
inv, c := layoutChain(mem)
|
|
ctx := context.Background()
|
|
for n, m := range []string{"alpha", "alpha", "beta", "alpha"} {
|
|
old := legacyBuild(n+1, m, fmt.Sprintf("c%d", n+1))
|
|
if _, err := c.Builds.Create(ctx, old.ID, old); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
failed := legacyBuild(9, "alpha", "c9")
|
|
failed.Failed = "broke"
|
|
failed.SourceFingerprint = ""
|
|
if _, err := c.Builds.Create(ctx, failed.ID, failed); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
asked := chain.Build{ID: "build-10", State: chain.BuildAsked, Repository: "novox/alpha", Asked: layoutAt,
|
|
Request: &chain.Request{Repository: "novox/alpha", At: layoutAt}}
|
|
if _, err := c.Builds.Create(ctx, asked.ID, asked); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
first, err := inv.layoutBuilds(ctx, c, nil)
|
|
if err != nil || first.Written == 0 {
|
|
t.Fatalf("first run %+v: %v", first, err)
|
|
}
|
|
keys := keysOf(t, mem)
|
|
second, err := inv.layoutBuilds(ctx, c, nil)
|
|
if err != nil || second.Written != 0 {
|
|
t.Fatalf("second run %+v: %v", second, err)
|
|
}
|
|
if again := keysOf(t, mem); !reflect.DeepEqual(keys, again) {
|
|
t.Fatalf("keys changed on the second run:\n%v\n%v", keys, again)
|
|
}
|
|
has := func(k string) bool { return containsKey(keys, k) }
|
|
for _, k := range []string{"build-1", "build-9", "alpha.build.build-1", "alpha.build.build-4", "beta.build.build-3", "alpha.build.build-9", "req.build-10",
|
|
"id.build-3", "alpha.newest", "alpha.worked", "alpha.held", "alpha.commit.c4", "alpha.source.v1-sc4", "beta.worked"} {
|
|
if !has(k) {
|
|
t.Errorf("no key %s in %v", k, keys)
|
|
}
|
|
}
|
|
if e, _, _ := mem.Get(ctx, "alpha.newest"); string(e.Value) != "build-9" {
|
|
t.Errorf("alpha.newest = %q; want the failed build-9", e.Value)
|
|
}
|
|
if e, _, _ := mem.Get(ctx, "alpha.worked"); string(e.Value) != "build-4" {
|
|
t.Errorf("alpha.worked = %q; want build-4", e.Value)
|
|
}
|
|
if id, _ := inv.chainBuildOf(ctx, c, "alpha", "c4"); id != "build-4" {
|
|
t.Errorf("by commit after the layout: %q", id)
|
|
}
|
|
// No record is read from an old key.
|
|
list, err := inv.chainBuildList(ctx, c, "alpha", 10)
|
|
if err != nil || len(list) != 4 {
|
|
t.Fatalf("alpha's builds after the layout: %d %v", len(list), err)
|
|
}
|
|
}
|
|
|
|
func containsKey(s []string, k string) bool {
|
|
for _, x := range s {
|
|
if x == k {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// The manifest is kept once under its digest in the same bucket, and the moved record keeps none.
|
|
func TestAMovedRecordHoldsNoManifestAndItsManifestIsUnderItsDigest(t *testing.T) {
|
|
mem := chain.NewMemory()
|
|
inv, c := layoutChain(mem)
|
|
ctx := context.Background()
|
|
old := legacyBuild(1, "alpha", "c1")
|
|
if _, err := c.Builds.Create(ctx, old.ID, old); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := inv.layoutBuilds(ctx, c, nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
k, found, err := c.Builds.Get(ctx, "alpha.build.build-1")
|
|
if err != nil || !found || len(k.Value.Manifest) != 0 || k.Value.ManifestDigest == "" {
|
|
t.Fatalf("moved record %+v %v %v", k.Value, found, err)
|
|
}
|
|
if !strings.HasPrefix(k.Value.ManifestDigest, "manifest.sha256-") {
|
|
t.Errorf("the digest %q is not a key under manifest.", k.Value.ManifestDigest)
|
|
}
|
|
e, found, _ := mem.Get(ctx, k.Value.ManifestDigest)
|
|
if !found || string(e.Value) != string(canonicalJSON([]byte(old.Manifest))) {
|
|
t.Errorf("the manifest under its digest is %q (found %v)", e.Value, found)
|
|
}
|
|
if got := chain.ManifestKey(e.Value); got != k.Value.ManifestDigest {
|
|
t.Errorf("the digest does not name the bytes: %s", got)
|
|
}
|
|
}
|
|
|
|
// A pointer is a hint: a record written without its pointers is found by reading the module's records, and the pointers are
|
|
// repaired; a pointer that names a record that is not what it was for is not followed.
|
|
func TestAMissingOrWrongPointerFallsBackToTheModulesRecordsAndIsRepaired(t *testing.T) {
|
|
mem := chain.NewMemory()
|
|
inv, c := layoutChain(mem)
|
|
ctx := context.Background()
|
|
for n := 1; n <= 3; n++ {
|
|
if err := inv.chainRecordBuild(ctx, c, layoutBuild("alpha", n, fmt.Sprintf("c%d", n), fmt.Sprintf("s%d", n))); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
// A record whose pointers the writer never made (a crash between the record and its pointers).
|
|
lost := toChainBuild(layoutBuild("alpha", 4, "c4", "s4"))
|
|
if _, err := c.Builds.Create(ctx, "alpha.build.build-4", lost); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if id, err := inv.chainBuildOf(ctx, c, "alpha", "c4"); err != nil || id != "build-4" {
|
|
t.Fatalf("the build of c4 is %q: %v", id, err)
|
|
}
|
|
if e, found, _ := mem.Get(ctx, "alpha.commit.c4"); !found || pointedID(e.Value) != "build-4" {
|
|
t.Errorf("the commit pointer was not repaired: %q %v", e.Value, found)
|
|
}
|
|
if id, _ := inv.chainBuildOfSource(ctx, c, "alpha", "v1:s4"); id != "build-4" {
|
|
t.Errorf("the source pointer was not repaired: %q", id)
|
|
}
|
|
// A pointer to the wrong build is not followed.
|
|
e, _, _ := mem.Get(ctx, "alpha.commit.c2")
|
|
wrong, _ := json.Marshal(chain.CommitPointer{Build: "build-1"})
|
|
if _, err := mem.Update(ctx, "alpha.commit.c2", wrong, e.Revision); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if id, err := inv.chainBuildOf(ctx, c, "alpha", "c2"); err != nil || id != "build-2" {
|
|
t.Fatalf("with a wrong pointer the build of c2 is %q: %v", id, err)
|
|
}
|
|
if e, _, _ := mem.Get(ctx, "alpha.commit.c2"); pointedID(e.Value) != "build-2" {
|
|
t.Errorf("the wrong pointer stayed: %s", e.Value)
|
|
}
|
|
// A commit nothing built is none.
|
|
if id, err := inv.chainBuildOf(ctx, c, "alpha", "nope"); err != nil || id != "" {
|
|
t.Errorf("an unbuilt commit is %q: %v", id, err)
|
|
}
|
|
}
|
|
|
|
type touchRecording struct {
|
|
chain.Backend
|
|
mu sync.Mutex
|
|
touched []string
|
|
}
|
|
|
|
func (r *touchRecording) note(s string) { r.mu.Lock(); r.touched = append(r.touched, s); r.mu.Unlock() }
|
|
func (r *touchRecording) Get(ctx context.Context, key string) (chain.Entry, bool, error) {
|
|
r.note("get " + key)
|
|
return r.Backend.Get(ctx, key)
|
|
}
|
|
func (r *touchRecording) Watch(ctx context.Context) (<-chan chain.Entry, error) {
|
|
r.note("watch >")
|
|
return r.Backend.Watch(ctx)
|
|
}
|
|
func (r *touchRecording) WatchFiltered(ctx context.Context, f string) (<-chan chain.Entry, error) {
|
|
r.note("watch " + f)
|
|
return r.Backend.WatchFiltered(ctx, f)
|
|
}
|
|
func (r *touchRecording) Keys(ctx context.Context) ([]string, error) {
|
|
r.note("keys")
|
|
return r.Backend.Keys(ctx)
|
|
}
|
|
|
|
// Once the layout is made, no read touches a key that is a build's id alone, or the bucket whole.
|
|
func TestAfterTheLayoutNoReadTouchesABareBuildIDKey(t *testing.T) {
|
|
rec := &touchRecording{Backend: chain.NewMemory()}
|
|
inv, c := layoutChain(rec)
|
|
ctx := context.Background()
|
|
old := legacyBuild(1, "alpha", "c1")
|
|
if _, err := c.Builds.Create(ctx, old.ID, old); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := inv.layoutBuilds(ctx, c, nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := inv.chainRecordBuild(ctx, c, layoutBuild("alpha", 2, "c2", "s2")); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
rec.mu.Lock()
|
|
rec.touched = nil
|
|
rec.mu.Unlock()
|
|
_, _ = inv.chainBuildList(ctx, c, "alpha", 5)
|
|
_, _ = inv.chainBuildList(ctx, c, "", 5)
|
|
_, _, _ = inv.chainBuildByID(ctx, c, "build-1")
|
|
_, _, _ = inv.chainBuildByID(ctx, c, "build-2")
|
|
_, _ = inv.chainBuildOf(ctx, c, "alpha", "c1")
|
|
_, _ = inv.chainBuildFingerprints(ctx, c, "alpha")
|
|
_, _ = inv.chainFingerprints(ctx, c)
|
|
_, _ = inv.chainManifestOf(ctx, c, "build-1")
|
|
_, _, _, _ = inv.chainManifestBuild(ctx, c, "alpha", "c1")
|
|
_, _ = chainBuilds(ctx, c)
|
|
_, _ = inv.chainRequests(ctx, c)
|
|
_, _ = inv.chainBuiltAgainst(ctx, c)
|
|
rec.mu.Lock()
|
|
defer rec.mu.Unlock()
|
|
for _, what := range rec.touched {
|
|
verb, key, _ := strings.Cut(what, " ")
|
|
if what == "keys" || what == "watch >" || what == "watch *" || (verb == "get" && !strings.Contains(key, ".")) {
|
|
t.Errorf("a read touched %q", what)
|
|
}
|
|
}
|
|
}
|
|
|
|
// A pointer follows the newest build, not the last write: two builds overlap and the older, asked first, finishes last.
|
|
func TestAnOlderBuildFinishingLastLeavesThePointersOnTheNewer(t *testing.T) {
|
|
mem := chain.NewMemory()
|
|
inv, c := layoutChain(mem)
|
|
ctx := context.Background()
|
|
older := layoutBuild("alpha", 1, "c1", "s1")
|
|
older.Asked = layoutAt
|
|
older.At = layoutAt.Add(20 * time.Minute) // finished last
|
|
newer := layoutBuild("alpha", 2, "c1", "s2")
|
|
newer.Asked = layoutAt.Add(5 * time.Minute)
|
|
newer.At = layoutAt.Add(10 * time.Minute)
|
|
for _, b := range []Build{newer, older} {
|
|
if err := inv.chainRecordBuild(ctx, c, b); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
for _, key := range []string{"alpha.newest", "alpha.worked"} {
|
|
if e, _, _ := mem.Get(ctx, key); string(e.Value) != "build-2" {
|
|
t.Errorf("%s = %q; want the newer build-2", key, e.Value)
|
|
}
|
|
}
|
|
if e, _, _ := mem.Get(ctx, "alpha.commit.c1"); pointedID(e.Value) != "build-2" {
|
|
t.Errorf("the commit pointer = %s; want build-2", e.Value)
|
|
}
|
|
// The older build is still found by what only it has.
|
|
if id, _ := inv.chainBuildOfSource(ctx, c, "alpha", "v1:s1"); id != "build-1" {
|
|
t.Errorf("the older build's own reuse key finds %q", id)
|
|
}
|
|
}
|