The tools bundle answered the catalog seat's revision verb from its view of the trunk in memory, which ran one or more revisions behind what the indexer kept until it heard the event, so the runner and the builder waited on revisions the bus held all along. The verb now reads which revision is newest from the store at the time of the ask (a listing and one object) and takes the view's entries only when the view holds that very revision. Builds: design 47 H-A-222 (1), H-A-205, H-A-213
341 lines
16 KiB
Go
341 lines
16 KiB
Go
package index
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"strings"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
|
"git.novox.be/novox/mesh-sdk/go/views"
|
|
)
|
|
|
|
// listedObjects counts the listings of its bucket.
|
|
type listedObjects struct {
|
|
*memObjects
|
|
n atomic.Int64
|
|
fail atomic.Bool
|
|
}
|
|
|
|
func (l *listedObjects) Names(ctx context.Context) ([]string, error) {
|
|
l.n.Add(1)
|
|
if l.fail.Load() {
|
|
return nil, errors.New("the bucket does not answer")
|
|
}
|
|
return l.memObjects.Names(ctx)
|
|
}
|
|
|
|
func keptDigest(name string) string {
|
|
return "sha256:" + fmt.Sprintf("%064x", len(name))
|
|
}
|
|
|
|
// A-487, A-487b, A-487c: the view of the trunk is read from the bucket once at the start and once for each heard event of a newer
|
|
// trunk revision; 100 reads between events list the bucket zero times.
|
|
func TestTheKeptTrunkIsReadAtTheStartAndOncePerHeardRevision(t *testing.T) {
|
|
objects := &listedObjects{memObjects: newMemObjects(0)}
|
|
keep := func(revision int64, modules ...string) {
|
|
t.Helper()
|
|
rev, entries := meshcatalog.CatalogRevision{Revision: revision, Kind: "trunk"}, map[string][]byte{}
|
|
for _, name := range modules {
|
|
data, _ := json.Marshal(meshcatalog.CatalogEntry{Module: name, Repository: catalogueRepo, None: "not built", ManifestDigest: keptDigest(name), Manifest: map[string]any{"module": name}})
|
|
digest := fmt.Sprintf("%d-%s", revision, name)
|
|
entries[digest] = data
|
|
rev.Entries = append(rev.Entries, meshcatalog.EntryRef{Module: name, Digest: digest})
|
|
}
|
|
if err := KeepRevision(context.Background(), objects, rev, entries); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
said := []string{}
|
|
kept := &KeptTrunk{Say: func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }}
|
|
modules := TrunkStates{Objects: objects, Under: func(string) views.Source { return nil }, Kept: kept}.State("modules")
|
|
keep(1, "avahi")
|
|
if err := kept.Reread(context.Background(), objects); err != nil { // the start's read
|
|
t.Fatal(err)
|
|
}
|
|
listed := objects.n.Load()
|
|
for i := 0; i < 50; i++ {
|
|
if _, _, err := modules.Get("avahi"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := modules.Keys(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if got := objects.n.Load() - listed; got != 0 {
|
|
t.Errorf("100 reads listed the bucket %d times, want none", got)
|
|
}
|
|
// a newer revision kept is no reason to read: only the event says it
|
|
keep(2, "avahi", "ca-trust")
|
|
listed = objects.n.Load()
|
|
if keys, _ := modules.Keys(); len(keys) != 1 {
|
|
t.Errorf("the view holds %v before the event, want revision 1's one module", keys)
|
|
}
|
|
for _, e := range []meshcatalog.CatalogRevisionEvent{{Kind: "candidate", Revision: 9}, {Kind: "trunk", Branch: "other", Revision: 9}, {Kind: "trunk", Revision: 1}} {
|
|
if err := kept.Heard(context.Background(), objects, e); err != nil {
|
|
t.Errorf("%+v: %v", e, err)
|
|
}
|
|
}
|
|
if got := objects.n.Load() - listed; got != 0 {
|
|
t.Errorf("events of a candidate, another branch and the revision held listed the bucket %d times, want none", got)
|
|
}
|
|
if err := kept.Heard(context.Background(), objects, meshcatalog.CatalogRevisionEvent{Kind: "trunk", Branch: "main", Revision: 2}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got := objects.n.Load() - listed; got != 1 {
|
|
t.Errorf("a heard catalog-revision listed the bucket %d times, want exactly one re-read", got)
|
|
}
|
|
if e, ok, err := modules.Get("ca-trust"); err != nil || !ok || e.Revision != 2 {
|
|
t.Errorf("after the event ca-trust = %+v %v (%v), want it at revision 2", e, ok, err)
|
|
}
|
|
// A-487c: the manifests by digest answer from the view held; the verb revision asked of the trunk's newest lists the bucket
|
|
// once for which revision is newest (H-A-222) and takes the entries of the view held at that very revision.
|
|
listed = objects.n.Load()
|
|
if raw, err := kept.Manifest(meshcatalogclient.BareManifestDigest(keptDigest("ca-trust"))); err != nil || raw != `{"module":"ca-trust"}` {
|
|
t.Errorf("the held manifest = %q (%v), want the entry's manifest", raw, err)
|
|
}
|
|
if got := objects.n.Load() - listed; got != 0 {
|
|
t.Errorf("the manifest listed the bucket %d times, want none", got)
|
|
}
|
|
if a, err := Revision(context.Background(), objects, kept, meshcatalog.RevisionInput{}); err != nil || a.Revision.Revision != 2 || len(a.Entries) != 2 {
|
|
t.Errorf("revision of the newest answered %+v (%v), want revision 2 whole", a, err)
|
|
}
|
|
if got := objects.n.Load() - listed; got != 1 {
|
|
t.Errorf("the verb revision listed the bucket %d times, want once, for the newest", got)
|
|
}
|
|
if len(said) != 2 || !strings.Contains(said[1], "read revision 2 (was 1), 2 modules, in ") {
|
|
t.Errorf("said %q, want each read with its revision, module count and time", said)
|
|
}
|
|
// A-487b: a re-read that fails is said loudly, the view held stays, and the next heard event reads again.
|
|
keep(3, "avahi", "ca-trust", "dbus")
|
|
objects.fail.Store(true)
|
|
if err := kept.Heard(context.Background(), objects, meshcatalog.CatalogRevisionEvent{Kind: "trunk", Revision: 3}); err == nil {
|
|
t.Error("a re-read that failed was taken")
|
|
}
|
|
if last := said[len(said)-1]; !strings.Contains(last, "BEHIND") {
|
|
t.Errorf("a failed re-read said %q, want it said loudly", last)
|
|
}
|
|
if keys, _ := modules.Keys(); len(keys) != 2 {
|
|
t.Errorf("the view held after a failed re-read has %v, want revision 2's two modules", keys)
|
|
}
|
|
objects.fail.Store(false)
|
|
if err := kept.Heard(context.Background(), objects, meshcatalog.CatalogRevisionEvent{Kind: "trunk", Revision: 3}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if keys, _ := modules.Keys(); len(keys) != 3 {
|
|
t.Errorf("the next heard event left the view at %v, want revision 3's three modules", keys)
|
|
}
|
|
// H-A-222: two revisions kept that the view has not heard are answered by the verb revision from the store, whole; the
|
|
// view held stays a cache at its own revision.
|
|
keep(4, "avahi", "ca-trust", "dbus")
|
|
keep(5, "avahi", "ca-trust", "dbus", "feh")
|
|
if a, err := Revision(context.Background(), objects, kept, meshcatalog.RevisionInput{}); err != nil || a.Revision == nil || a.Revision.Revision != 5 || len(a.Entries) != 4 {
|
|
t.Errorf("revision of the newest answered %+v (%v) while the view held 3, want revision 5 whole from the store", a, err)
|
|
}
|
|
if keys, _ := modules.Keys(); len(keys) != 3 {
|
|
t.Errorf("the verb moved the view held to %v, want it at revision 3 until it hears the event", keys)
|
|
}
|
|
}
|
|
|
|
// A-487: the verb module reads the one module it is asked.
|
|
func TestTheVerbModuleReadsOneModule(t *testing.T) {
|
|
c := CatalogRead{
|
|
Modules: func() (map[string]meshcatalog.ModulesByModuleValue, error) {
|
|
return nil, errors.New("every module was read")
|
|
},
|
|
Module: func(name string) (meshcatalog.ModulesByModuleValue, bool, error) {
|
|
return meshcatalog.ModulesByModuleValue{Manifest: "d1"}, name == "avahi", nil
|
|
},
|
|
Manifest: func(string) (string, error) { return `{"module":"avahi"}`, nil },
|
|
}
|
|
if one, err := c.CatalogModuleAnswer("avahi"); err != nil || one.Module.Module != "avahi" {
|
|
t.Errorf("module avahi = %+v (%v)", one, err)
|
|
}
|
|
if _, err := c.CatalogModuleAnswer("nope"); err == nil || !strings.Contains(err.Error(), "holds no module nope") {
|
|
t.Errorf("module nope = %v, want the trunk holds none", err)
|
|
}
|
|
}
|
|
|
|
// goneUnderRead is a bucket whose pin judge runs while the first entry of the revision named newest is read: a newer revision
|
|
// is kept and the one named is removed, its list and its entries, as the indexer's judge does.
|
|
type goneUnderRead struct {
|
|
*memObjects
|
|
once atomic.Bool
|
|
prune func()
|
|
on string // when set, only the read of this entry runs the pruning
|
|
}
|
|
|
|
func (g *goneUnderRead) Get(ctx context.Context, name string) ([]byte, error) {
|
|
if strings.HasPrefix(name, "entry.") && (g.on == "" || name == g.on) && g.once.CompareAndSwap(false, true) {
|
|
g.prune()
|
|
}
|
|
return g.memObjects.Get(ctx, name)
|
|
}
|
|
|
|
// A revision removed while it is read (take-ins keep one every few seconds, each removing the last) is read again as the newest
|
|
// one, never failed: on novox the first view of the catalog half never came while revisions 487 to 493 were kept.
|
|
func TestTheKeptTrunkReadsTheNewerRevisionWhenTheOneNamedIsRemovedUnderIt(t *testing.T) {
|
|
mem := newMemObjects(0)
|
|
keep := func(revision int64) {
|
|
data, _ := json.Marshal(meshcatalog.CatalogEntry{Module: "avahi", Repository: catalogueRepo, None: "not built"})
|
|
digest := fmt.Sprintf("d%d", revision)
|
|
rev := meshcatalog.CatalogRevision{Revision: revision, Kind: "trunk", Entries: []meshcatalog.EntryRef{{Module: "avahi", Digest: digest}}}
|
|
if err := KeepRevision(context.Background(), mem, rev, map[string][]byte{digest: data}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
keep(1)
|
|
objects := &goneUnderRead{memObjects: mem}
|
|
objects.prune = func() {
|
|
keep(2)
|
|
_ = mem.Delete(context.Background(), RevisionObject(meshcatalog.CatalogRevision{Revision: 1, Kind: "trunk"}))
|
|
_ = mem.Delete(context.Background(), EntryObject("d1"))
|
|
}
|
|
kept := &KeptTrunk{}
|
|
if err := kept.Reread(context.Background(), objects); err != nil {
|
|
t.Fatalf("the read failed when the revision named was removed under it: %v", err)
|
|
}
|
|
if rev, _, ok, err := kept.Newest(context.Background(), objects); err != nil || !ok || rev.Revision != 2 {
|
|
t.Errorf("the view holds revision %d (%v, %v), want 2", rev.Revision, ok, err)
|
|
}
|
|
}
|
|
|
|
// A-487d: the same revision failing twice is the read's failure, said as it is; a read by number never reads another.
|
|
func TestTheKeptTrunkFailsWhenTheSameRevisionIsGoneTwice(t *testing.T) {
|
|
mem := newMemObjects(0)
|
|
data, _ := json.Marshal(meshcatalog.CatalogEntry{Module: "avahi", Repository: catalogueRepo, None: "not built"})
|
|
rev := meshcatalog.CatalogRevision{Revision: 1, Kind: "trunk", Entries: []meshcatalog.EntryRef{{Module: "avahi", Digest: "d1"}}}
|
|
if err := KeepRevision(context.Background(), mem, rev, map[string][]byte{"d1": data}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
_ = mem.Delete(context.Background(), EntryObject("d1")) // revision 1 stays listed, its entry lost, no newer revision
|
|
for name, read := range map[string]func() error{
|
|
"kept trunk": func() error { return (&KeptTrunk{}).Reread(context.Background(), mem) },
|
|
"TrunkView": func() error { _, _, _, err := TrunkView(context.Background(), mem); return err },
|
|
"Outdated": func() error {
|
|
_, err := outdatedOfAvahi(mem).Outdated(context.Background(), meshcatalog.OutdatedInput{})
|
|
return err
|
|
},
|
|
} {
|
|
if err := read(); !errors.Is(err, ErrNoObject) {
|
|
t.Errorf("%s: %v, want the lost entry said (no such object)", name, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// A-487d: BranchView and TrunkView name the newest again when the one named is removed under the read.
|
|
func TestTrunkViewReadsTheNewerRevisionWhenTheOneNamedIsRemovedUnderIt(t *testing.T) {
|
|
mem := newMemObjects(0)
|
|
keep := func(revision int64) {
|
|
data, _ := json.Marshal(meshcatalog.CatalogEntry{Module: "avahi", Repository: catalogueRepo, None: "not built"})
|
|
digest := fmt.Sprintf("d%d", revision)
|
|
rev := meshcatalog.CatalogRevision{Revision: revision, Kind: "trunk", Entries: []meshcatalog.EntryRef{{Module: "avahi", Digest: digest}}}
|
|
if err := KeepRevision(context.Background(), mem, rev, map[string][]byte{digest: data}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
keep(1)
|
|
objects := &goneUnderRead{memObjects: mem}
|
|
objects.prune = func() {
|
|
keep(2)
|
|
_ = mem.Delete(context.Background(), RevisionObject(meshcatalog.CatalogRevision{Revision: 1, Kind: "trunk"}))
|
|
_ = mem.Delete(context.Background(), EntryObject("d1"))
|
|
}
|
|
if _, _, at, err := TrunkView(context.Background(), objects); err != nil || at != 2 {
|
|
t.Errorf("TrunkView read revision %d (%v), want 2", at, err)
|
|
}
|
|
}
|
|
|
|
// A-487d: every reader of the newest revision in the indexer program reads the newest again when the revision it named is
|
|
// removed under it (ModuleInNewestView, SeedEntryHeld, entryOf, SeedModulesFrom, FillVersions, the tool indexer_catalog_revision
|
|
// and the verb revision, trunk's and a watched branch's).
|
|
func TestTheNewestRevisionReadersReadAgainWhenTheRevisionIsRemovedUnderThem(t *testing.T) {
|
|
ctx := context.Background()
|
|
cases := map[string]struct {
|
|
branch string
|
|
read func(ix *Indexer) (bool, error)
|
|
}{
|
|
"ModuleInNewestView": {"", func(ix *Indexer) (bool, error) { return ix.ModuleInNewestView(ctx, "avahi") }},
|
|
"SeedEntryHeld": {"", func(ix *Indexer) (bool, error) { return ix.SeedEntryHeld(ctx, "avahi", "fp2") }},
|
|
"entryOf": {"", func(ix *Indexer) (bool, error) {
|
|
e, err := ix.entryOf(ctx, "avahi")
|
|
return e.Fp == "fp2", err
|
|
}},
|
|
"SeedModulesFrom": {"", func(ix *Indexer) (bool, error) { _, err := ix.SeedModulesFrom(ctx, nil, nil); return err == nil, err }},
|
|
"FillVersions": {"", func(ix *Indexer) (bool, error) {
|
|
ix.Versions, ix.Heard = MemoryStore(), MemoryStore()
|
|
err := ix.FillVersions(ctx, func(context.Context, string, string) (BuilderVersionsAnswer, error) {
|
|
return BuilderVersionsAnswer{}, nil
|
|
})
|
|
return err == nil, err
|
|
}},
|
|
"indexer_catalog_revision": {"", func(ix *Indexer) (bool, error) {
|
|
var tool map[string]any
|
|
for _, tl := range CatalogRevisionTools(ix.Revisions, nil, time.Now, nil) {
|
|
if tl.Name == CatalogRevisionTool {
|
|
got, err := tl.Run(map[string]any{})
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
tool = got.(map[string]any)
|
|
}
|
|
}
|
|
return tool != nil && tool["name"] == "main.2", nil
|
|
}},
|
|
"Outdated": {"", func(ix *Indexer) (bool, error) {
|
|
a, err := outdatedOfAvahi(ix.Revisions).Outdated(ctx, meshcatalog.OutdatedInput{})
|
|
return len(a.Behind) == 1 && a.Behind[0].Kind == 5 && strings.Contains(a.Behind[0].Newest, "fp2"), err
|
|
}},
|
|
"Revision": {"", func(ix *Indexer) (bool, error) {
|
|
a, err := Revision(ctx, ix.Revisions, nil, meshcatalog.RevisionInput{})
|
|
return a.Revision != nil && a.Revision.Revision == 2, err
|
|
}},
|
|
"Revision of a branch": {"dev", func(ix *Indexer) (bool, error) {
|
|
a, err := Revision(ctx, ix.Revisions, nil, meshcatalog.RevisionInput{Branch: "dev"})
|
|
return a.Revision != nil && a.Revision.Revision == 2 && len(a.Entries) == 1 && a.Entries[0].Fp == "fp2", err
|
|
}},
|
|
}
|
|
for name, c := range cases {
|
|
t.Run(name, func(t *testing.T) {
|
|
mem := newMemObjects(0)
|
|
keep := func(branch string, revision int64) {
|
|
data, _ := json.Marshal(meshcatalog.CatalogEntry{Module: "avahi", Repository: catalogueRepo, None: "not built", Fp: fmt.Sprintf("fp%d", revision)})
|
|
digest := fmt.Sprintf("%s-d%d", branch, revision)
|
|
rev := meshcatalog.CatalogRevision{Branch: branch, Revision: revision, Kind: "trunk", Entries: []meshcatalog.EntryRef{{Module: "avahi", Digest: digest}}}
|
|
if err := KeepRevision(ctx, mem, rev, map[string][]byte{digest: data}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
keep("", 1)
|
|
if c.branch != "" { // the trunk stays at revision 1; the branch's revision 1 is removed under the read
|
|
keep("", 2)
|
|
keep(c.branch, 1)
|
|
}
|
|
objects := &goneUnderRead{memObjects: mem}
|
|
pruned := c.branch
|
|
objects.on = EntryObject(fmt.Sprintf("%s-d1", pruned))
|
|
objects.prune = func() {
|
|
keep(pruned, 2)
|
|
_ = mem.Delete(ctx, RevisionObject(meshcatalog.CatalogRevision{Branch: pruned, Revision: 1, Kind: "trunk"}))
|
|
_ = mem.Delete(ctx, objects.on)
|
|
}
|
|
if ok, err := c.read(&Indexer{Revisions: objects}); err != nil || !ok {
|
|
t.Errorf("answered %v (%v), want revision 2's entry", ok, err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// outdatedOfAvahi is an indexer whose state versions holds avahi at a kind-5 edge, so Outdated reads the trunk's entry of avahi.
|
|
func outdatedOfAvahi(o Objects) *Indexer {
|
|
ix := &Indexer{Versions: MemoryStore(), Heard: MemoryStore(), Revisions: o}
|
|
data, _ := json.Marshal(meshcatalog.VersionEntry{V: 1, Module: "avahi", Version: "0.1.1", Edges: []meshcatalog.VersionEdge{{Kind: 5, Part: "actor:program"}}})
|
|
_ = ix.Versions.Create(context.Background(), meshcatalog.VersionsByModuleVersionKey("avahi", "0.1.1"), data)
|
|
return ix
|
|
}
|