diff --git a/modules/mesh-delivery-indexer/internal/index/kept_trunk_test.go b/modules/mesh-delivery-indexer/internal/index/kept_trunk_test.go index badac4e4..0dd1eb1f 100644 --- a/modules/mesh-delivery-indexer/internal/index/kept_trunk_test.go +++ b/modules/mesh-delivery-indexer/internal/index/kept_trunk_test.go @@ -8,6 +8,7 @@ import ( "strings" "sync/atomic" "testing" + "time" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-sdk/go/views" @@ -151,10 +152,11 @@ type goneUnderRead struct { *memObjects once atomic.Bool prune func() + on string // when set, only the read of this entry runs the pruning } func (g *goneUnderRead) Get(ctx context.Context, name string) ([]byte, error) { - if strings.HasPrefix(name, "entry.") && g.once.CompareAndSwap(false, true) { + if strings.HasPrefix(name, "entry.") && (g.on == "" || name == g.on) && g.once.CompareAndSwap(false, true) { g.prune() } return g.memObjects.Get(ctx, name) @@ -200,6 +202,10 @@ func TestTheKeptTrunkFailsWhenTheSameRevisionIsGoneTwice(t *testing.T) { for name, read := range map[string]func() error{ "kept trunk": func() error { return (&KeptTrunk{}).Reread(context.Background(), mem) }, "TrunkView": func() error { _, _, _, err := TrunkView(context.Background(), mem); return err }, + "Outdated": func() error { + _, err := outdatedOfAvahi(mem).Outdated(context.Background(), meshcatalog.OutdatedInput{}) + return err + }, } { if err := read(); !errors.Is(err, ErrNoObject) { t.Errorf("%s: %v, want the lost entry said (no such object)", name, err) @@ -229,3 +235,91 @@ func TestTrunkViewReadsTheNewerRevisionWhenTheOneNamedIsRemovedUnderIt(t *testin t.Errorf("TrunkView read revision %d (%v), want 2", at, err) } } + +// A-487d: every reader of the newest revision in the indexer program reads the newest again when the revision it named is +// removed under it (ModuleInNewestView, SeedEntryHeld, entryOf, SeedModulesFrom, FillVersions, the tool indexer_catalog_revision +// and the verb revision, trunk's and a watched branch's). +func TestTheNewestRevisionReadersReadAgainWhenTheRevisionIsRemovedUnderThem(t *testing.T) { + ctx := context.Background() + cases := map[string]struct { + branch string + read func(ix *Indexer) (bool, error) + }{ + "ModuleInNewestView": {"", func(ix *Indexer) (bool, error) { return ix.ModuleInNewestView(ctx, "avahi") }}, + "SeedEntryHeld": {"", func(ix *Indexer) (bool, error) { return ix.SeedEntryHeld(ctx, "avahi", "fp2") }}, + "entryOf": {"", func(ix *Indexer) (bool, error) { + e, err := ix.entryOf(ctx, "avahi") + return e.Fp == "fp2", err + }}, + "SeedModulesFrom": {"", func(ix *Indexer) (bool, error) { _, err := ix.SeedModulesFrom(ctx, nil, nil); return err == nil, err }}, + "FillVersions": {"", func(ix *Indexer) (bool, error) { + ix.Versions, ix.Heard = MemoryStore(), MemoryStore() + err := ix.FillVersions(ctx, func(context.Context, string, string) (BuilderVersionsAnswer, error) { + return BuilderVersionsAnswer{}, nil + }) + return err == nil, err + }}, + "indexer_catalog_revision": {"", func(ix *Indexer) (bool, error) { + var tool map[string]any + for _, tl := range CatalogRevisionTools(ix.Revisions, nil, time.Now, nil) { + if tl.Name == CatalogRevisionTool { + got, err := tl.Run(map[string]any{}) + if err != nil { + return false, err + } + tool = got.(map[string]any) + } + } + return tool != nil && tool["name"] == "trunk.2", nil + }}, + "Outdated": {"", func(ix *Indexer) (bool, error) { + a, err := outdatedOfAvahi(ix.Revisions).Outdated(ctx, meshcatalog.OutdatedInput{}) + return len(a.Behind) == 1 && a.Behind[0].Kind == 5 && strings.Contains(a.Behind[0].Newest, "fp2"), err + }}, + "Revision": {"", func(ix *Indexer) (bool, error) { + a, err := Revision(ctx, ix.Revisions, nil, "avahi", "avahi", meshcatalog.RevisionInput{}) + return a.Revision != nil && a.Revision.Revision == 2, err + }}, + "Revision of a branch": {"dev", func(ix *Indexer) (bool, error) { + a, err := Revision(ctx, ix.Revisions, nil, "avahi", "avahi", meshcatalog.RevisionInput{Branch: "dev"}) + return a.Revision != nil && a.Revision.Revision == 2 && len(a.Entries) == 1 && a.Entries[0].Fp == "fp2", err + }}, + } + for name, c := range cases { + t.Run(name, func(t *testing.T) { + mem := newMemObjects(0) + keep := func(branch string, revision int64) { + data, _ := json.Marshal(meshcatalog.CatalogEntry{Module: "avahi", Repository: catalogueRepo, None: "not built", Fp: fmt.Sprintf("fp%d", revision)}) + digest := fmt.Sprintf("%s-d%d", branch, revision) + rev := meshcatalog.CatalogRevision{Branch: branch, Revision: revision, Kind: "trunk", Entries: []meshcatalog.EntryRef{{Module: "avahi", Digest: digest}}} + if err := KeepRevision(ctx, mem, rev, map[string][]byte{digest: data}); err != nil { + t.Fatal(err) + } + } + keep("", 1) + if c.branch != "" { // the trunk stays at revision 1; the branch's revision 1 is removed under the read + keep("", 2) + keep(c.branch, 1) + } + objects := &goneUnderRead{memObjects: mem} + pruned := c.branch + objects.on = EntryObject(fmt.Sprintf("%s-d1", pruned)) + objects.prune = func() { + keep(pruned, 2) + _ = mem.Delete(ctx, RevisionObject(meshcatalog.CatalogRevision{Branch: pruned, Revision: 1, Kind: "trunk"})) + _ = mem.Delete(ctx, objects.on) + } + if ok, err := c.read(&Indexer{Revisions: objects}); err != nil || !ok { + t.Errorf("answered %v (%v), want revision 2's entry", ok, err) + } + }) + } +} + +// outdatedOfAvahi is an indexer whose state versions holds avahi at a kind-5 edge, so Outdated reads the trunk's entry of avahi. +func outdatedOfAvahi(o Objects) *Indexer { + ix := &Indexer{Versions: MemoryStore(), Heard: MemoryStore(), Revisions: o} + data, _ := json.Marshal(meshcatalog.VersionEntry{V: 1, Module: "avahi", Version: "0.1.1", Edges: []meshcatalog.VersionEdge{{Kind: 5, Part: "actor:program"}}}) + _ = ix.Versions.Create(context.Background(), meshcatalog.VersionsByModuleVersionKey("avahi", "0.1.1"), data) + return ix +} diff --git a/modules/mesh-delivery-indexer/internal/index/open_asks.go b/modules/mesh-delivery-indexer/internal/index/open_asks.go index 576dc802..684e9e33 100644 --- a/modules/mesh-delivery-indexer/internal/index/open_asks.go +++ b/modules/mesh-delivery-indexer/internal/index/open_asks.go @@ -3,7 +3,6 @@ package index import ( "context" "errors" - "fmt" "slices" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" @@ -51,14 +50,10 @@ func (ix *Indexer) SeedEntryHeld(ctx context.Context, module, fp string) (bool, if ctx.Err() != nil || ix.Revisions == nil { return false, ctx.Err() } - rev, found, err := newestTrunk(ctx, ix.Revisions) + _, entries, found, err := readNewestOf(ctx, ix.Revisions, TrunkBranch) if err != nil || !found { return false, err } - _, entries, err := ReadRevision(ctx, ix.Revisions, fmt.Sprintf("trunk.%d", rev.Revision)) - if err != nil { - return false, err - } return slices.ContainsFunc(entries, func(e meshcatalog.CatalogEntry) bool { return e.Module == module && e.Fp == fp }), nil } @@ -67,14 +62,10 @@ func (ix *Indexer) ModuleInNewestView(ctx context.Context, module string) (bool, if ctx.Err() != nil || ix.Revisions == nil { return false, ctx.Err() } - rev, found, err := newestTrunk(ctx, ix.Revisions) + _, entries, found, err := readNewestOf(ctx, ix.Revisions, TrunkBranch) if err != nil || !found { return false, err } - _, entries, err := ReadRevision(ctx, ix.Revisions, fmt.Sprintf("trunk.%d", rev.Revision)) - if err != nil { - return false, err - } return slices.ContainsFunc(entries, func(e meshcatalog.CatalogEntry) bool { return e.Module == module }), nil } diff --git a/modules/mesh-delivery-indexer/internal/index/revision_tools.go b/modules/mesh-delivery-indexer/internal/index/revision_tools.go index 476e25ca..a21bbd3c 100644 --- a/modules/mesh-delivery-indexer/internal/index/revision_tools.go +++ b/modules/mesh-delivery-indexer/internal/index/revision_tools.go @@ -324,24 +324,28 @@ func CatalogRevisionTools(o Objects, ask func(key string, value []byte) (TakeInA ctx := context.Background() n, _ := args["revision"].(float64) name := "" + var rev meshcatalog.CatalogRevision + var entries []meshcatalog.CatalogEntry switch { case str(args, "group") != "" || str(args, "heads") != "": name = RevisionObject(meshcatalog.CatalogRevision{Kind: "candidate", Group: str(args, "group"), Heads: str(args, "heads")}) case n > 0: name = "trunk." + strconv.FormatInt(int64(n), 10) default: - newest, ok, err := newestTrunk(ctx, o) + newest, newestEntries, ok, err := readNewestOf(ctx, o, TrunkBranch) if err != nil { return nil, fmt.Errorf("the catalog graph revisions cannot be read: %w", err) } if !ok { return nil, ErrNoRevision } - name = RevisionObject(newest) + name, rev, entries = RevisionObject(newest), newest, newestEntries } - rev, entries, err := ReadRevision(ctx, o, name) - if err != nil { - return nil, err + if entries == nil { + var err error + if rev, entries, err = ReadRevision(ctx, o, name); err != nil { + return nil, err + } } return map[string]any{"name": name, "revision": rev, "entries": entries}, nil }}, diff --git a/modules/mesh-delivery-indexer/internal/index/revision_verb.go b/modules/mesh-delivery-indexer/internal/index/revision_verb.go index c3e4a589..f2feabb7 100644 --- a/modules/mesh-delivery-indexer/internal/index/revision_verb.go +++ b/modules/mesh-delivery-indexer/internal/index/revision_verb.go @@ -72,12 +72,12 @@ func Revision(ctx context.Context, o Objects, kept *KeptTrunk, holder, caller st var out meshcatalog.RevisionAnswer var newest meshcatalog.CatalogRevision var ok bool - var trunkEntries []meshcatalog.CatalogEntry + var trunkEntries, branchEntries []meshcatalog.CatalogEntry var err error if kept != nil { newest, trunkEntries, ok, err = kept.Newest(ctx, o) } else { - newest, ok, err = newestTrunk(ctx, o) + newest, trunkEntries, ok, err = readNewestOf(ctx, o, TrunkBranch) } if err != nil { return out, fmt.Errorf("the catalog graph revisions cannot be read: %w", err) @@ -86,11 +86,6 @@ func Revision(ctx context.Context, o Objects, kept *KeptTrunk, holder, caller st // An answer is told by its fields (design 47 G §530): no trunk revision kept yet is `none-yet`, no error. return meshcatalog.RevisionAnswer{NoneYet: true}, nil } - if kept == nil { - if _, trunkEntries, err = ReadRevision(ctx, o, RevisionObject(newest)); err != nil { - return out, err - } - } caller = CallerModule(trunkEntries, caller) if err := views.Refusal(meshcatalog.Seat+"."+meshcatalog.VerbRevision, RevisionCallers(trunkEntries, holder), caller); err != nil { return out, fmt.Errorf("%w (novox/hq ADR 0663 §2: a part reads it by declaring %s in invokes)", err, RevisionInvoke) @@ -102,14 +97,14 @@ func Revision(ctx context.Context, o Objects, kept *KeptTrunk, holder, caller st return out, fmt.Errorf("the branch %q is no escaped name: %w", in.Branch, err) } if !isTrunkBranch(branch) { - other, found, err := newestOf(ctx, o, branch) + other, entries, found, err := readNewestOf(ctx, o, branch) if err != nil { return out, fmt.Errorf("the catalog graph revisions of %s cannot be read: %w", branch, err) } if !found { return meshcatalog.RevisionAnswer{NoneYet: true}, nil } - newest = other + newest, branchEntries = other, entries } name := RevisionObject(newest) switch { @@ -133,7 +128,9 @@ func Revision(ctx context.Context, o Objects, kept *KeptTrunk, holder, caller st } rev, entries := newest, trunkEntries if !isTrunkBranch(branch) && in.Group == "" && in.Heads == "" { - if rev, entries, err = ReadRevision(ctx, o, name); err != nil { + if name == RevisionObject(newest) { + entries = branchEntries // the newest of the branch, read by readNewestOf (A-487d) + } else if rev, entries, err = ReadRevision(ctx, o, name); err != nil { return out, err } } else if name != RevisionObject(newest) { diff --git a/modules/mesh-delivery-indexer/internal/index/seed_steps.go b/modules/mesh-delivery-indexer/internal/index/seed_steps.go index 4fece72d..1aed0f99 100644 --- a/modules/mesh-delivery-indexer/internal/index/seed_steps.go +++ b/modules/mesh-delivery-indexer/internal/index/seed_steps.go @@ -63,17 +63,13 @@ func (ix *Indexer) SeedModulesFrom(ctx context.Context, defs map[string]seatdef. if ix.Revisions == nil { return nil, fmt.Errorf("the catalog graph is not kept here: no trunk revision to seed from") } - rev, found, err := newestTrunk(ctx, ix.Revisions) + _, entries, found, err := readNewestOf(ctx, ix.Revisions, TrunkBranch) if err != nil { return nil, err } if !found { return nil, fmt.Errorf("the indexer keeps no trunk revision yet: the seed waits for its first") } - _, entries, err := ReadRevision(ctx, ix.Revisions, fmt.Sprintf("trunk.%d", rev.Revision)) - if err != nil { - return nil, err - } seatOf := map[string]string{} // artifact reference -> its build seat, all found before any call is made unbuilt := map[string][]string{} // module -> each artifact of it no build seat builds, with its language or kind for _, e := range entries { diff --git a/modules/mesh-delivery-indexer/internal/index/versions.go b/modules/mesh-delivery-indexer/internal/index/versions.go index c2a608ee..5c5db15a 100644 --- a/modules/mesh-delivery-indexer/internal/index/versions.go +++ b/modules/mesh-delivery-indexer/internal/index/versions.go @@ -177,17 +177,13 @@ func (ix *Indexer) entryOf(ctx context.Context, module string) (meshcatalog.Cata if ix.Revisions == nil { return meshcatalog.CatalogEntry{}, errors.New("the catalog graph is not kept here: no entry names the module's artifacts") } - rev, found, err := newestTrunk(ctx, ix.Revisions) + _, entries, found, err := readNewestOf(ctx, ix.Revisions, TrunkBranch) if err != nil { return meshcatalog.CatalogEntry{}, err } if !found { return meshcatalog.CatalogEntry{}, errors.New("the indexer keeps no trunk revision yet") } - _, entries, err := ReadRevision(ctx, ix.Revisions, fmt.Sprintf("trunk.%d", rev.Revision)) - if err != nil { - return meshcatalog.CatalogEntry{}, err - } for _, e := range entries { if e.Module == module { return e, nil diff --git a/modules/mesh-delivery-indexer/internal/index/versions_edges.go b/modules/mesh-delivery-indexer/internal/index/versions_edges.go index a4380010..4865bb2e 100644 --- a/modules/mesh-delivery-indexer/internal/index/versions_edges.go +++ b/modules/mesh-delivery-indexer/internal/index/versions_edges.go @@ -194,11 +194,13 @@ func (ix *Indexer) Outdated(ctx context.Context, in meshcatalog.OutdatedInput) ( } trunk := map[string]meshcatalog.CatalogEntry{} // the newest trunk revision's entries, read where a kind needs them if ix.Revisions != nil { - if rev, found, err := newestTrunk(ctx, ix.Revisions); err == nil && found { - if _, entries, err := ReadRevision(ctx, ix.Revisions, fmt.Sprintf("trunk.%d", rev.Revision)); err == nil { - for _, e := range entries { - trunk[e.Module] = e - } + _, entries, found, err := readNewestOf(ctx, ix.Revisions, TrunkBranch) + if err != nil { + return out, fmt.Errorf("the trunk's entries, which the kinds 3 and 5 are told by, cannot be read: %w", err) + } + if found { + for _, e := range entries { + trunk[e.Module] = e } } } diff --git a/modules/mesh-delivery-indexer/internal/index/versions_fill.go b/modules/mesh-delivery-indexer/internal/index/versions_fill.go index ac12fda6..b29aa4e3 100644 --- a/modules/mesh-delivery-indexer/internal/index/versions_fill.go +++ b/modules/mesh-delivery-indexer/internal/index/versions_fill.go @@ -79,17 +79,13 @@ func (ix *Indexer) FillVersions(ctx context.Context, ask VersionsAsker) error { if ix.Revisions == nil { return errors.New("the catalog graph is not kept here: no entry names the modules' artifacts") } - rev, found, err := newestTrunk(ctx, ix.Revisions) + _, entries, found, err := readNewestOf(ctx, ix.Revisions, TrunkBranch) if err != nil { return err } if !found { return errors.New("the indexer keeps no trunk revision yet: the first fill of the versions waits for its first") } - _, entries, err := ReadRevision(ctx, ix.Revisions, fmt.Sprintf("trunk.%d", rev.Revision)) - if err != nil { - return err - } type found1 struct { v BuilderVersion name string