Files
mesh-delivery/modules/mesh-delivery-indexer/internal/index/kept_trunk_test.go
T
jschoubben abdd70d918 I2b: one catalog revision per take-in, <branch>.<source>, carrying its movers
TakeInSource takes a gate's source-moved in: the trees of each repository it moves at the
named commit through the seat git, the fingerprints of the modules registered for it, the
movers against the branch's newest catalog revision, and one revision <escaped
branch>.<source> for every branch, the trunk included, numbered by the source and written
once by read before write. The revision carries its movers, and a redelivery whose revision
is kept says take-in.taken again from them, recomputing nothing (H-A-92). It is pinned for
the take-in's pipeline as pipeline.<branch>.<source> in the same write when it has a mover,
none for a take-in with no mover. take-in.taken carries branch, source, revision, call and
the movers' entries alone, under a duplicate id of its branch and source (H-A-31). A
source-moved with no call is refused as the old form (§765).

Only a take-in keeps a catalog revision (H-A-49): the recompute, the build ends' imports and
the start's reconcile keep none, and KeepView, keepEntries, keepTrunkRevision and
PinPipelines go. Another branch than the first is taken in on its newest revision's view.

Builds: design 47 H §743, §749 (the pin in the take-in's write), §765 (the old form); H-A-31, H-A-32, H-A-49, H-A-92

Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy
2026-11-20 04:56:27 +01:00

326 lines
14 KiB
Go

package index
import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"sync/atomic"
"testing"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-sdk/go/views"
)
// listedObjects counts the listings of its bucket.
type listedObjects struct {
*memObjects
n atomic.Int64
fail atomic.Bool
}
func (l *listedObjects) Names(ctx context.Context) ([]string, error) {
l.n.Add(1)
if l.fail.Load() {
return nil, errors.New("the bucket does not answer")
}
return l.memObjects.Names(ctx)
}
func keptDigest(name string) string {
return "sha256:" + fmt.Sprintf("%064x", len(name))
}
// A-487, A-487b, A-487c: the view of the trunk is read from the bucket once at the start and once for each heard event of a newer
// trunk revision; 100 reads between events list the bucket zero times.
func TestTheKeptTrunkIsReadAtTheStartAndOncePerHeardRevision(t *testing.T) {
objects := &listedObjects{memObjects: newMemObjects(0)}
keep := func(revision int64, modules ...string) {
t.Helper()
rev, entries := meshcatalog.CatalogRevision{Revision: revision, Kind: "trunk"}, map[string][]byte{}
for _, name := range modules {
data, _ := json.Marshal(meshcatalog.CatalogEntry{Module: name, Repository: catalogueRepo, None: "not built", ManifestDigest: keptDigest(name), Manifest: map[string]any{"module": name}})
digest := fmt.Sprintf("%d-%s", revision, name)
entries[digest] = data
rev.Entries = append(rev.Entries, meshcatalog.EntryRef{Module: name, Digest: digest})
}
if err := KeepRevision(context.Background(), objects, rev, entries); err != nil {
t.Fatal(err)
}
}
said := []string{}
kept := &KeptTrunk{Say: func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }}
modules := TrunkStates{Objects: objects, Under: func(string) views.Source { return nil }, Kept: kept}.State("modules")
keep(1, "avahi")
if err := kept.Reread(context.Background(), objects); err != nil { // the start's read
t.Fatal(err)
}
listed := objects.n.Load()
for i := 0; i < 50; i++ {
if _, _, err := modules.Get("avahi"); err != nil {
t.Fatal(err)
}
if _, err := modules.Keys(); err != nil {
t.Fatal(err)
}
}
if got := objects.n.Load() - listed; got != 0 {
t.Errorf("100 reads listed the bucket %d times, want none", got)
}
// a newer revision kept is no reason to read: only the event says it
keep(2, "avahi", "ca-trust")
listed = objects.n.Load()
if keys, _ := modules.Keys(); len(keys) != 1 {
t.Errorf("the view holds %v before the event, want revision 1's one module", keys)
}
for _, e := range []meshcatalog.CatalogRevisionEvent{{Kind: "candidate", Revision: 9}, {Kind: "trunk", Branch: "other", Revision: 9}, {Kind: "trunk", Revision: 1}} {
if err := kept.Heard(context.Background(), objects, e); err != nil {
t.Errorf("%+v: %v", e, err)
}
}
if got := objects.n.Load() - listed; got != 0 {
t.Errorf("events of a candidate, another branch and the revision held listed the bucket %d times, want none", got)
}
if err := kept.Heard(context.Background(), objects, meshcatalog.CatalogRevisionEvent{Kind: "trunk", Branch: "main", Revision: 2}); err != nil {
t.Fatal(err)
}
if got := objects.n.Load() - listed; got != 1 {
t.Errorf("a heard catalog-revision listed the bucket %d times, want exactly one re-read", got)
}
if e, ok, err := modules.Get("ca-trust"); err != nil || !ok || e.Revision != 2 {
t.Errorf("after the event ca-trust = %+v %v (%v), want it at revision 2", e, ok, err)
}
// A-487c: the manifests by digest and the verb revision asked of the trunk's newest answer from the view held.
listed = objects.n.Load()
if raw, err := kept.Manifest(meshcatalog.BareManifestDigest(keptDigest("ca-trust"))); err != nil || raw != `{"module":"ca-trust"}` {
t.Errorf("the held manifest = %q (%v), want the entry's manifest", raw, err)
}
if a, err := Revision(context.Background(), objects, kept, meshcatalog.RevisionInput{}); err != nil || a.Revision.Revision != 2 || len(a.Entries) != 2 {
t.Errorf("revision of the newest answered %+v (%v), want revision 2 whole", a, err)
}
if got := objects.n.Load() - listed; got != 0 {
t.Errorf("the manifest and the verb revision listed the bucket %d times, want none", got)
}
if len(said) != 2 || !strings.Contains(said[1], "read revision 2 (was 1), 2 modules, in ") {
t.Errorf("said %q, want each read with its revision, module count and time", said)
}
// A-487b: a re-read that fails is said loudly, the view held stays, and the next heard event reads again.
keep(3, "avahi", "ca-trust", "dbus")
objects.fail.Store(true)
if err := kept.Heard(context.Background(), objects, meshcatalog.CatalogRevisionEvent{Kind: "trunk", Revision: 3}); err == nil {
t.Error("a re-read that failed was taken")
}
if last := said[len(said)-1]; !strings.Contains(last, "BEHIND") {
t.Errorf("a failed re-read said %q, want it said loudly", last)
}
if keys, _ := modules.Keys(); len(keys) != 2 {
t.Errorf("the view held after a failed re-read has %v, want revision 2's two modules", keys)
}
objects.fail.Store(false)
if err := kept.Heard(context.Background(), objects, meshcatalog.CatalogRevisionEvent{Kind: "trunk", Revision: 3}); err != nil {
t.Fatal(err)
}
if keys, _ := modules.Keys(); len(keys) != 3 {
t.Errorf("the next heard event left the view at %v, want revision 3's three modules", keys)
}
}
// A-487: the verb module reads the one module it is asked.
func TestTheVerbModuleReadsOneModule(t *testing.T) {
c := CatalogRead{
Modules: func() (map[string]meshcatalog.ModulesByModuleValue, error) {
return nil, errors.New("every module was read")
},
Module: func(name string) (meshcatalog.ModulesByModuleValue, bool, error) {
return meshcatalog.ModulesByModuleValue{Manifest: "d1"}, name == "avahi", nil
},
Manifest: func(string) (string, error) { return `{"module":"avahi"}`, nil },
}
if one, err := c.CatalogModuleAnswer("avahi"); err != nil || one.Module.Module != "avahi" {
t.Errorf("module avahi = %+v (%v)", one, err)
}
if _, err := c.CatalogModuleAnswer("nope"); err == nil || !strings.Contains(err.Error(), "holds no module nope") {
t.Errorf("module nope = %v, want the trunk holds none", err)
}
}
// goneUnderRead is a bucket whose pin judge runs while the first entry of the revision named newest is read: a newer revision
// is kept and the one named is removed, its list and its entries, as the indexer's judge does.
type goneUnderRead struct {
*memObjects
once atomic.Bool
prune func()
on string // when set, only the read of this entry runs the pruning
}
func (g *goneUnderRead) Get(ctx context.Context, name string) ([]byte, error) {
if strings.HasPrefix(name, "entry.") && (g.on == "" || name == g.on) && g.once.CompareAndSwap(false, true) {
g.prune()
}
return g.memObjects.Get(ctx, name)
}
// A revision removed while it is read (take-ins keep one every few seconds, each removing the last) is read again as the newest
// one, never failed: on novox the first view of the catalog half never came while revisions 487 to 493 were kept.
func TestTheKeptTrunkReadsTheNewerRevisionWhenTheOneNamedIsRemovedUnderIt(t *testing.T) {
mem := newMemObjects(0)
keep := func(revision int64) {
data, _ := json.Marshal(meshcatalog.CatalogEntry{Module: "avahi", Repository: catalogueRepo, None: "not built"})
digest := fmt.Sprintf("d%d", revision)
rev := meshcatalog.CatalogRevision{Revision: revision, Kind: "trunk", Entries: []meshcatalog.EntryRef{{Module: "avahi", Digest: digest}}}
if err := KeepRevision(context.Background(), mem, rev, map[string][]byte{digest: data}); err != nil {
t.Fatal(err)
}
}
keep(1)
objects := &goneUnderRead{memObjects: mem}
objects.prune = func() {
keep(2)
_ = mem.Delete(context.Background(), RevisionObject(meshcatalog.CatalogRevision{Revision: 1, Kind: "trunk"}))
_ = mem.Delete(context.Background(), EntryObject("d1"))
}
kept := &KeptTrunk{}
if err := kept.Reread(context.Background(), objects); err != nil {
t.Fatalf("the read failed when the revision named was removed under it: %v", err)
}
if rev, _, ok, err := kept.Newest(context.Background(), objects); err != nil || !ok || rev.Revision != 2 {
t.Errorf("the view holds revision %d (%v, %v), want 2", rev.Revision, ok, err)
}
}
// A-487d: the same revision failing twice is the read's failure, said as it is; a read by number never reads another.
func TestTheKeptTrunkFailsWhenTheSameRevisionIsGoneTwice(t *testing.T) {
mem := newMemObjects(0)
data, _ := json.Marshal(meshcatalog.CatalogEntry{Module: "avahi", Repository: catalogueRepo, None: "not built"})
rev := meshcatalog.CatalogRevision{Revision: 1, Kind: "trunk", Entries: []meshcatalog.EntryRef{{Module: "avahi", Digest: "d1"}}}
if err := KeepRevision(context.Background(), mem, rev, map[string][]byte{"d1": data}); err != nil {
t.Fatal(err)
}
_ = mem.Delete(context.Background(), EntryObject("d1")) // revision 1 stays listed, its entry lost, no newer revision
for name, read := range map[string]func() error{
"kept trunk": func() error { return (&KeptTrunk{}).Reread(context.Background(), mem) },
"TrunkView": func() error { _, _, _, err := TrunkView(context.Background(), mem); return err },
"Outdated": func() error {
_, err := outdatedOfAvahi(mem).Outdated(context.Background(), meshcatalog.OutdatedInput{})
return err
},
} {
if err := read(); !errors.Is(err, ErrNoObject) {
t.Errorf("%s: %v, want the lost entry said (no such object)", name, err)
}
}
}
// A-487d: BranchView and TrunkView name the newest again when the one named is removed under the read.
func TestTrunkViewReadsTheNewerRevisionWhenTheOneNamedIsRemovedUnderIt(t *testing.T) {
mem := newMemObjects(0)
keep := func(revision int64) {
data, _ := json.Marshal(meshcatalog.CatalogEntry{Module: "avahi", Repository: catalogueRepo, None: "not built"})
digest := fmt.Sprintf("d%d", revision)
rev := meshcatalog.CatalogRevision{Revision: revision, Kind: "trunk", Entries: []meshcatalog.EntryRef{{Module: "avahi", Digest: digest}}}
if err := KeepRevision(context.Background(), mem, rev, map[string][]byte{digest: data}); err != nil {
t.Fatal(err)
}
}
keep(1)
objects := &goneUnderRead{memObjects: mem}
objects.prune = func() {
keep(2)
_ = mem.Delete(context.Background(), RevisionObject(meshcatalog.CatalogRevision{Revision: 1, Kind: "trunk"}))
_ = mem.Delete(context.Background(), EntryObject("d1"))
}
if _, _, at, err := TrunkView(context.Background(), objects); err != nil || at != 2 {
t.Errorf("TrunkView read revision %d (%v), want 2", at, err)
}
}
// A-487d: every reader of the newest revision in the indexer program reads the newest again when the revision it named is
// removed under it (ModuleInNewestView, SeedEntryHeld, entryOf, SeedModulesFrom, FillVersions, the tool indexer_catalog_revision
// and the verb revision, trunk's and a watched branch's).
func TestTheNewestRevisionReadersReadAgainWhenTheRevisionIsRemovedUnderThem(t *testing.T) {
ctx := context.Background()
cases := map[string]struct {
branch string
read func(ix *Indexer) (bool, error)
}{
"ModuleInNewestView": {"", func(ix *Indexer) (bool, error) { return ix.ModuleInNewestView(ctx, "avahi") }},
"SeedEntryHeld": {"", func(ix *Indexer) (bool, error) { return ix.SeedEntryHeld(ctx, "avahi", "fp2") }},
"entryOf": {"", func(ix *Indexer) (bool, error) {
e, err := ix.entryOf(ctx, "avahi")
return e.Fp == "fp2", err
}},
"SeedModulesFrom": {"", func(ix *Indexer) (bool, error) { _, err := ix.SeedModulesFrom(ctx, nil, nil); return err == nil, err }},
"FillVersions": {"", func(ix *Indexer) (bool, error) {
ix.Versions, ix.Heard = MemoryStore(), MemoryStore()
err := ix.FillVersions(ctx, func(context.Context, string, string) (BuilderVersionsAnswer, error) {
return BuilderVersionsAnswer{}, nil
})
return err == nil, err
}},
"indexer_catalog_revision": {"", func(ix *Indexer) (bool, error) {
var tool map[string]any
for _, tl := range CatalogRevisionTools(ix.Revisions, nil, time.Now, nil) {
if tl.Name == CatalogRevisionTool {
got, err := tl.Run(map[string]any{})
if err != nil {
return false, err
}
tool = got.(map[string]any)
}
}
return tool != nil && tool["name"] == "main.2", nil
}},
"Outdated": {"", func(ix *Indexer) (bool, error) {
a, err := outdatedOfAvahi(ix.Revisions).Outdated(ctx, meshcatalog.OutdatedInput{})
return len(a.Behind) == 1 && a.Behind[0].Kind == 5 && strings.Contains(a.Behind[0].Newest, "fp2"), err
}},
"Revision": {"", func(ix *Indexer) (bool, error) {
a, err := Revision(ctx, ix.Revisions, nil, meshcatalog.RevisionInput{})
return a.Revision != nil && a.Revision.Revision == 2, err
}},
"Revision of a branch": {"dev", func(ix *Indexer) (bool, error) {
a, err := Revision(ctx, ix.Revisions, nil, meshcatalog.RevisionInput{Branch: "dev"})
return a.Revision != nil && a.Revision.Revision == 2 && len(a.Entries) == 1 && a.Entries[0].Fp == "fp2", err
}},
}
for name, c := range cases {
t.Run(name, func(t *testing.T) {
mem := newMemObjects(0)
keep := func(branch string, revision int64) {
data, _ := json.Marshal(meshcatalog.CatalogEntry{Module: "avahi", Repository: catalogueRepo, None: "not built", Fp: fmt.Sprintf("fp%d", revision)})
digest := fmt.Sprintf("%s-d%d", branch, revision)
rev := meshcatalog.CatalogRevision{Branch: branch, Revision: revision, Kind: "trunk", Entries: []meshcatalog.EntryRef{{Module: "avahi", Digest: digest}}}
if err := KeepRevision(ctx, mem, rev, map[string][]byte{digest: data}); err != nil {
t.Fatal(err)
}
}
keep("", 1)
if c.branch != "" { // the trunk stays at revision 1; the branch's revision 1 is removed under the read
keep("", 2)
keep(c.branch, 1)
}
objects := &goneUnderRead{memObjects: mem}
pruned := c.branch
objects.on = EntryObject(fmt.Sprintf("%s-d1", pruned))
objects.prune = func() {
keep(pruned, 2)
_ = mem.Delete(ctx, RevisionObject(meshcatalog.CatalogRevision{Branch: pruned, Revision: 1, Kind: "trunk"}))
_ = mem.Delete(ctx, objects.on)
}
if ok, err := c.read(&Indexer{Revisions: objects}); err != nil || !ok {
t.Errorf("answered %v (%v), want revision 2's entry", ok, err)
}
})
}
}
// outdatedOfAvahi is an indexer whose state versions holds avahi at a kind-5 edge, so Outdated reads the trunk's entry of avahi.
func outdatedOfAvahi(o Objects) *Indexer {
ix := &Indexer{Versions: MemoryStore(), Heard: MemoryStore(), Revisions: o}
data, _ := json.Marshal(meshcatalog.VersionEntry{V: 1, Module: "avahi", Version: "0.1.1", Edges: []meshcatalog.VersionEdge{{Kind: 5, Part: "actor:program"}}})
_ = ix.Versions.Create(context.Background(), meshcatalog.VersionsByModuleVersionKey("avahi", "0.1.1"), data)
return ix
}