Files
mesh-delivery/modules/mesh-delivery-indexer/internal/index/kept_trunk_test.go
T

326 lines
15 KiB
Go

package index
import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"sync/atomic"
"testing"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-sdk/go/views"
)
// listedObjects counts the listings of its bucket.
type listedObjects struct {
*memObjects
n atomic.Int64
fail atomic.Bool
}
func (l *listedObjects) Names(ctx context.Context) ([]string, error) {
l.n.Add(1)
if l.fail.Load() {
return nil, errors.New("the bucket does not answer")
}
return l.memObjects.Names(ctx)
}
func keptDigest(name string) string {
return "sha256:" + fmt.Sprintf("%064x", len(name))
}
// A-487, A-487b, A-487c: the view of the trunk is read from the bucket once at the start and once for each heard event of a newer
// trunk revision; 100 reads between events list the bucket zero times.
func TestTheKeptTrunkIsReadAtTheStartAndOncePerHeardRevision(t *testing.T) {
objects := &listedObjects{memObjects: newMemObjects(0)}
keep := func(revision int64, modules ...string) {
t.Helper()
rev, entries := meshcatalog.CatalogRevision{Revision: revision, Kind: "trunk"}, map[string][]byte{}
for _, name := range modules {
data, _ := json.Marshal(meshcatalog.CatalogEntry{Module: name, Repository: catalogueRepo, None: "not built", ManifestDigest: keptDigest(name), Manifest: map[string]any{"module": name}})
digest := fmt.Sprintf("%d-%s", revision, name)
entries[digest] = data
rev.Entries = append(rev.Entries, meshcatalog.EntryRef{Module: name, Digest: digest})
}
if err := KeepRevision(context.Background(), objects, rev, entries); err != nil {
t.Fatal(err)
}
}
said := []string{}
kept := &KeptTrunk{Say: func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }}
modules := TrunkStates{Objects: objects, Under: func(string) views.Source { return nil }, Kept: kept}.State("modules")
keep(1, "avahi")
if err := kept.Reread(context.Background(), objects); err != nil { // the start's read
t.Fatal(err)
}
listed := objects.n.Load()
for i := 0; i < 50; i++ {
if _, _, err := modules.Get("avahi"); err != nil {
t.Fatal(err)
}
if _, err := modules.Keys(); err != nil {
t.Fatal(err)
}
}
if got := objects.n.Load() - listed; got != 0 {
t.Errorf("100 reads listed the bucket %d times, want none", got)
}
// a newer revision kept is no reason to read: only the event says it
keep(2, "avahi", "ca-trust")
listed = objects.n.Load()
if keys, _ := modules.Keys(); len(keys) != 1 {
t.Errorf("the view holds %v before the event, want revision 1's one module", keys)
}
for _, e := range []meshcatalog.CatalogRevisionEvent{{Kind: "candidate", Revision: 9}, {Kind: "trunk", Branch: "other", Revision: 9}, {Kind: "trunk", Revision: 1}} {
if err := kept.Heard(context.Background(), objects, e); err != nil {
t.Errorf("%+v: %v", e, err)
}
}
if got := objects.n.Load() - listed; got != 0 {
t.Errorf("events of a candidate, another branch and the revision held listed the bucket %d times, want none", got)
}
if err := kept.Heard(context.Background(), objects, meshcatalog.CatalogRevisionEvent{Kind: "trunk", Branch: "main", Revision: 2}); err != nil {
t.Fatal(err)
}
if got := objects.n.Load() - listed; got != 1 {
t.Errorf("a heard catalog-revision listed the bucket %d times, want exactly one re-read", got)
}
if e, ok, err := modules.Get("ca-trust"); err != nil || !ok || e.Revision != 2 {
t.Errorf("after the event ca-trust = %+v %v (%v), want it at revision 2", e, ok, err)
}
// A-487c: the manifests by digest and the verb revision asked of the trunk's newest answer from the view held.
listed = objects.n.Load()
if raw, err := kept.Manifest(meshcatalog.BareManifestDigest(keptDigest("ca-trust"))); err != nil || raw != `{"module":"ca-trust"}` {
t.Errorf("the held manifest = %q (%v), want the entry's manifest", raw, err)
}
if a, err := Revision(context.Background(), objects, kept, "avahi", "avahi", meshcatalog.RevisionInput{}); err != nil || a.Revision.Revision != 2 || len(a.Entries) != 2 {
t.Errorf("revision of the newest answered %+v (%v), want revision 2 whole", a, err)
}
if got := objects.n.Load() - listed; got != 0 {
t.Errorf("the manifest and the verb revision listed the bucket %d times, want none", got)
}
if len(said) != 2 || !strings.Contains(said[1], "read revision 2 (was 1), 2 modules, in ") {
t.Errorf("said %q, want each read with its revision, module count and time", said)
}
// A-487b: a re-read that fails is said loudly, the view held stays, and the next heard event reads again.
keep(3, "avahi", "ca-trust", "dbus")
objects.fail.Store(true)
if err := kept.Heard(context.Background(), objects, meshcatalog.CatalogRevisionEvent{Kind: "trunk", Revision: 3}); err == nil {
t.Error("a re-read that failed was taken")
}
if last := said[len(said)-1]; !strings.Contains(last, "BEHIND") {
t.Errorf("a failed re-read said %q, want it said loudly", last)
}
if keys, _ := modules.Keys(); len(keys) != 2 {
t.Errorf("the view held after a failed re-read has %v, want revision 2's two modules", keys)
}
objects.fail.Store(false)
if err := kept.Heard(context.Background(), objects, meshcatalog.CatalogRevisionEvent{Kind: "trunk", Revision: 3}); err != nil {
t.Fatal(err)
}
if keys, _ := modules.Keys(); len(keys) != 3 {
t.Errorf("the next heard event left the view at %v, want revision 3's three modules", keys)
}
}
// A-487: the verb module reads the one module it is asked.
func TestTheVerbModuleReadsOneModule(t *testing.T) {
c := CatalogRead{
Modules: func() (map[string]meshcatalog.ModulesByModuleValue, error) {
return nil, errors.New("every module was read")
},
Module: func(name string) (meshcatalog.ModulesByModuleValue, bool, error) {
return meshcatalog.ModulesByModuleValue{Manifest: "d1"}, name == "avahi", nil
},
Manifest: func(string) (string, error) { return `{"module":"avahi"}`, nil },
}
if one, err := c.CatalogModuleAnswer("avahi"); err != nil || one.Module.Module != "avahi" {
t.Errorf("module avahi = %+v (%v)", one, err)
}
if _, err := c.CatalogModuleAnswer("nope"); err == nil || !strings.Contains(err.Error(), "holds no module nope") {
t.Errorf("module nope = %v, want the trunk holds none", err)
}
}
// goneUnderRead is a bucket whose pin judge runs while the first entry of the revision named newest is read: a newer revision
// is kept and the one named is removed, its list and its entries, as the indexer's judge does.
type goneUnderRead struct {
*memObjects
once atomic.Bool
prune func()
on string // when set, only the read of this entry runs the pruning
}
func (g *goneUnderRead) Get(ctx context.Context, name string) ([]byte, error) {
if strings.HasPrefix(name, "entry.") && (g.on == "" || name == g.on) && g.once.CompareAndSwap(false, true) {
g.prune()
}
return g.memObjects.Get(ctx, name)
}
// A revision removed while it is read (take-ins keep one every few seconds, each removing the last) is read again as the newest
// one, never failed: on novox the first view of the catalog half never came while revisions 487 to 493 were kept.
func TestTheKeptTrunkReadsTheNewerRevisionWhenTheOneNamedIsRemovedUnderIt(t *testing.T) {
mem := newMemObjects(0)
keep := func(revision int64) {
data, _ := json.Marshal(meshcatalog.CatalogEntry{Module: "avahi", Repository: catalogueRepo, None: "not built"})
digest := fmt.Sprintf("d%d", revision)
rev := meshcatalog.CatalogRevision{Revision: revision, Kind: "trunk", Entries: []meshcatalog.EntryRef{{Module: "avahi", Digest: digest}}}
if err := KeepRevision(context.Background(), mem, rev, map[string][]byte{digest: data}); err != nil {
t.Fatal(err)
}
}
keep(1)
objects := &goneUnderRead{memObjects: mem}
objects.prune = func() {
keep(2)
_ = mem.Delete(context.Background(), RevisionObject(meshcatalog.CatalogRevision{Revision: 1, Kind: "trunk"}))
_ = mem.Delete(context.Background(), EntryObject("d1"))
}
kept := &KeptTrunk{}
if err := kept.Reread(context.Background(), objects); err != nil {
t.Fatalf("the read failed when the revision named was removed under it: %v", err)
}
if rev, _, ok, err := kept.Newest(context.Background(), objects); err != nil || !ok || rev.Revision != 2 {
t.Errorf("the view holds revision %d (%v, %v), want 2", rev.Revision, ok, err)
}
}
// A-487d: the same revision failing twice is the read's failure, said as it is; a read by number never reads another.
func TestTheKeptTrunkFailsWhenTheSameRevisionIsGoneTwice(t *testing.T) {
mem := newMemObjects(0)
data, _ := json.Marshal(meshcatalog.CatalogEntry{Module: "avahi", Repository: catalogueRepo, None: "not built"})
rev := meshcatalog.CatalogRevision{Revision: 1, Kind: "trunk", Entries: []meshcatalog.EntryRef{{Module: "avahi", Digest: "d1"}}}
if err := KeepRevision(context.Background(), mem, rev, map[string][]byte{"d1": data}); err != nil {
t.Fatal(err)
}
_ = mem.Delete(context.Background(), EntryObject("d1")) // revision 1 stays listed, its entry lost, no newer revision
for name, read := range map[string]func() error{
"kept trunk": func() error { return (&KeptTrunk{}).Reread(context.Background(), mem) },
"TrunkView": func() error { _, _, _, err := TrunkView(context.Background(), mem); return err },
"Outdated": func() error {
_, err := outdatedOfAvahi(mem).Outdated(context.Background(), meshcatalog.OutdatedInput{})
return err
},
} {
if err := read(); !errors.Is(err, ErrNoObject) {
t.Errorf("%s: %v, want the lost entry said (no such object)", name, err)
}
}
}
// A-487d: BranchView and TrunkView name the newest again when the one named is removed under the read.
func TestTrunkViewReadsTheNewerRevisionWhenTheOneNamedIsRemovedUnderIt(t *testing.T) {
mem := newMemObjects(0)
keep := func(revision int64) {
data, _ := json.Marshal(meshcatalog.CatalogEntry{Module: "avahi", Repository: catalogueRepo, None: "not built"})
digest := fmt.Sprintf("d%d", revision)
rev := meshcatalog.CatalogRevision{Revision: revision, Kind: "trunk", Entries: []meshcatalog.EntryRef{{Module: "avahi", Digest: digest}}}
if err := KeepRevision(context.Background(), mem, rev, map[string][]byte{digest: data}); err != nil {
t.Fatal(err)
}
}
keep(1)
objects := &goneUnderRead{memObjects: mem}
objects.prune = func() {
keep(2)
_ = mem.Delete(context.Background(), RevisionObject(meshcatalog.CatalogRevision{Revision: 1, Kind: "trunk"}))
_ = mem.Delete(context.Background(), EntryObject("d1"))
}
if _, _, at, err := TrunkView(context.Background(), objects); err != nil || at != 2 {
t.Errorf("TrunkView read revision %d (%v), want 2", at, err)
}
}
// A-487d: every reader of the newest revision in the indexer program reads the newest again when the revision it named is
// removed under it (ModuleInNewestView, SeedEntryHeld, entryOf, SeedModulesFrom, FillVersions, the tool indexer_catalog_revision
// and the verb revision, trunk's and a watched branch's).
func TestTheNewestRevisionReadersReadAgainWhenTheRevisionIsRemovedUnderThem(t *testing.T) {
ctx := context.Background()
cases := map[string]struct {
branch string
read func(ix *Indexer) (bool, error)
}{
"ModuleInNewestView": {"", func(ix *Indexer) (bool, error) { return ix.ModuleInNewestView(ctx, "avahi") }},
"SeedEntryHeld": {"", func(ix *Indexer) (bool, error) { return ix.SeedEntryHeld(ctx, "avahi", "fp2") }},
"entryOf": {"", func(ix *Indexer) (bool, error) {
e, err := ix.entryOf(ctx, "avahi")
return e.Fp == "fp2", err
}},
"SeedModulesFrom": {"", func(ix *Indexer) (bool, error) { _, err := ix.SeedModulesFrom(ctx, nil, nil); return err == nil, err }},
"FillVersions": {"", func(ix *Indexer) (bool, error) {
ix.Versions, ix.Heard = MemoryStore(), MemoryStore()
err := ix.FillVersions(ctx, func(context.Context, string, string) (BuilderVersionsAnswer, error) {
return BuilderVersionsAnswer{}, nil
})
return err == nil, err
}},
"indexer_catalog_revision": {"", func(ix *Indexer) (bool, error) {
var tool map[string]any
for _, tl := range CatalogRevisionTools(ix.Revisions, nil, time.Now, nil) {
if tl.Name == CatalogRevisionTool {
got, err := tl.Run(map[string]any{})
if err != nil {
return false, err
}
tool = got.(map[string]any)
}
}
return tool != nil && tool["name"] == "trunk.2", nil
}},
"Outdated": {"", func(ix *Indexer) (bool, error) {
a, err := outdatedOfAvahi(ix.Revisions).Outdated(ctx, meshcatalog.OutdatedInput{})
return len(a.Behind) == 1 && a.Behind[0].Kind == 5 && strings.Contains(a.Behind[0].Newest, "fp2"), err
}},
"Revision": {"", func(ix *Indexer) (bool, error) {
a, err := Revision(ctx, ix.Revisions, nil, "avahi", "avahi", meshcatalog.RevisionInput{})
return a.Revision != nil && a.Revision.Revision == 2, err
}},
"Revision of a branch": {"dev", func(ix *Indexer) (bool, error) {
a, err := Revision(ctx, ix.Revisions, nil, "avahi", "avahi", meshcatalog.RevisionInput{Branch: "dev"})
return a.Revision != nil && a.Revision.Revision == 2 && len(a.Entries) == 1 && a.Entries[0].Fp == "fp2", err
}},
}
for name, c := range cases {
t.Run(name, func(t *testing.T) {
mem := newMemObjects(0)
keep := func(branch string, revision int64) {
data, _ := json.Marshal(meshcatalog.CatalogEntry{Module: "avahi", Repository: catalogueRepo, None: "not built", Fp: fmt.Sprintf("fp%d", revision)})
digest := fmt.Sprintf("%s-d%d", branch, revision)
rev := meshcatalog.CatalogRevision{Branch: branch, Revision: revision, Kind: "trunk", Entries: []meshcatalog.EntryRef{{Module: "avahi", Digest: digest}}}
if err := KeepRevision(ctx, mem, rev, map[string][]byte{digest: data}); err != nil {
t.Fatal(err)
}
}
keep("", 1)
if c.branch != "" { // the trunk stays at revision 1; the branch's revision 1 is removed under the read
keep("", 2)
keep(c.branch, 1)
}
objects := &goneUnderRead{memObjects: mem}
pruned := c.branch
objects.on = EntryObject(fmt.Sprintf("%s-d1", pruned))
objects.prune = func() {
keep(pruned, 2)
_ = mem.Delete(ctx, RevisionObject(meshcatalog.CatalogRevision{Branch: pruned, Revision: 1, Kind: "trunk"}))
_ = mem.Delete(ctx, objects.on)
}
if ok, err := c.read(&Indexer{Revisions: objects}); err != nil || !ok {
t.Errorf("answered %v (%v), want revision 2's entry", ok, err)
}
})
}
}
// outdatedOfAvahi is an indexer whose state versions holds avahi at a kind-5 edge, so Outdated reads the trunk's entry of avahi.
func outdatedOfAvahi(o Objects) *Indexer {
ix := &Indexer{Versions: MemoryStore(), Heard: MemoryStore(), Revisions: o}
data, _ := json.Marshal(meshcatalog.VersionEntry{V: 1, Module: "avahi", Version: "0.1.1", Edges: []meshcatalog.VersionEdge{{Kind: 5, Part: "actor:program"}}})
_ = ix.Versions.Create(context.Background(), meshcatalog.VersionsByModuleVersionKey("avahi", "0.1.1"), data)
return ix
}