Every reader of the newest catalog revision reads the newest again when it is removed under it (A-487d) #254

Merged
mesh-tools merged 3 commits from fix/newest-readers-again into main 2026-11-19 11:54:41 +00:00
8 changed files with 123 additions and 47 deletions
@@ -8,6 +8,7 @@ import (
"strings"
"sync/atomic"
"testing"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-sdk/go/views"
@@ -151,10 +152,11 @@ type goneUnderRead struct {
*memObjects
once atomic.Bool
prune func()
on string // when set, only the read of this entry runs the pruning
}
func (g *goneUnderRead) Get(ctx context.Context, name string) ([]byte, error) {
if strings.HasPrefix(name, "entry.") && g.once.CompareAndSwap(false, true) {
if strings.HasPrefix(name, "entry.") && (g.on == "" || name == g.on) && g.once.CompareAndSwap(false, true) {
g.prune()
}
return g.memObjects.Get(ctx, name)
@@ -200,6 +202,10 @@ func TestTheKeptTrunkFailsWhenTheSameRevisionIsGoneTwice(t *testing.T) {
for name, read := range map[string]func() error{
"kept trunk": func() error { return (&KeptTrunk{}).Reread(context.Background(), mem) },
"TrunkView": func() error { _, _, _, err := TrunkView(context.Background(), mem); return err },
"Outdated": func() error {
_, err := outdatedOfAvahi(mem).Outdated(context.Background(), meshcatalog.OutdatedInput{})
return err
},
} {
if err := read(); !errors.Is(err, ErrNoObject) {
t.Errorf("%s: %v, want the lost entry said (no such object)", name, err)
@@ -229,3 +235,91 @@ func TestTrunkViewReadsTheNewerRevisionWhenTheOneNamedIsRemovedUnderIt(t *testin
t.Errorf("TrunkView read revision %d (%v), want 2", at, err)
}
}
// A-487d: every reader of the newest revision in the indexer program reads the newest again when the revision it named is
// removed under it (ModuleInNewestView, SeedEntryHeld, entryOf, SeedModulesFrom, FillVersions, the tool indexer_catalog_revision
// and the verb revision, trunk's and a watched branch's).
func TestTheNewestRevisionReadersReadAgainWhenTheRevisionIsRemovedUnderThem(t *testing.T) {
ctx := context.Background()
cases := map[string]struct {
branch string
read func(ix *Indexer) (bool, error)
}{
"ModuleInNewestView": {"", func(ix *Indexer) (bool, error) { return ix.ModuleInNewestView(ctx, "avahi") }},
"SeedEntryHeld": {"", func(ix *Indexer) (bool, error) { return ix.SeedEntryHeld(ctx, "avahi", "fp2") }},
"entryOf": {"", func(ix *Indexer) (bool, error) {
e, err := ix.entryOf(ctx, "avahi")
return e.Fp == "fp2", err
}},
"SeedModulesFrom": {"", func(ix *Indexer) (bool, error) { _, err := ix.SeedModulesFrom(ctx, nil, nil); return err == nil, err }},
"FillVersions": {"", func(ix *Indexer) (bool, error) {
ix.Versions, ix.Heard = MemoryStore(), MemoryStore()
err := ix.FillVersions(ctx, func(context.Context, string, string) (BuilderVersionsAnswer, error) {
return BuilderVersionsAnswer{}, nil
})
return err == nil, err
}},
"indexer_catalog_revision": {"", func(ix *Indexer) (bool, error) {
var tool map[string]any
for _, tl := range CatalogRevisionTools(ix.Revisions, nil, time.Now, nil) {
if tl.Name == CatalogRevisionTool {
got, err := tl.Run(map[string]any{})
if err != nil {
return false, err
}
tool = got.(map[string]any)
}
}
return tool != nil && tool["name"] == "trunk.2", nil
}},
"Outdated": {"", func(ix *Indexer) (bool, error) {
a, err := outdatedOfAvahi(ix.Revisions).Outdated(ctx, meshcatalog.OutdatedInput{})
return len(a.Behind) == 1 && a.Behind[0].Kind == 5 && strings.Contains(a.Behind[0].Newest, "fp2"), err
}},
"Revision": {"", func(ix *Indexer) (bool, error) {
a, err := Revision(ctx, ix.Revisions, nil, "avahi", "avahi", meshcatalog.RevisionInput{})
return a.Revision != nil && a.Revision.Revision == 2, err
}},
"Revision of a branch": {"dev", func(ix *Indexer) (bool, error) {
a, err := Revision(ctx, ix.Revisions, nil, "avahi", "avahi", meshcatalog.RevisionInput{Branch: "dev"})
return a.Revision != nil && a.Revision.Revision == 2 && len(a.Entries) == 1 && a.Entries[0].Fp == "fp2", err
}},
}
for name, c := range cases {
t.Run(name, func(t *testing.T) {
mem := newMemObjects(0)
keep := func(branch string, revision int64) {
data, _ := json.Marshal(meshcatalog.CatalogEntry{Module: "avahi", Repository: catalogueRepo, None: "not built", Fp: fmt.Sprintf("fp%d", revision)})
digest := fmt.Sprintf("%s-d%d", branch, revision)
rev := meshcatalog.CatalogRevision{Branch: branch, Revision: revision, Kind: "trunk", Entries: []meshcatalog.EntryRef{{Module: "avahi", Digest: digest}}}
if err := KeepRevision(ctx, mem, rev, map[string][]byte{digest: data}); err != nil {
t.Fatal(err)
}
}
keep("", 1)
if c.branch != "" { // the trunk stays at revision 1; the branch's revision 1 is removed under the read
keep("", 2)
keep(c.branch, 1)
}
objects := &goneUnderRead{memObjects: mem}
pruned := c.branch
objects.on = EntryObject(fmt.Sprintf("%s-d1", pruned))
objects.prune = func() {
keep(pruned, 2)
_ = mem.Delete(ctx, RevisionObject(meshcatalog.CatalogRevision{Branch: pruned, Revision: 1, Kind: "trunk"}))
_ = mem.Delete(ctx, objects.on)
}
if ok, err := c.read(&Indexer{Revisions: objects}); err != nil || !ok {
t.Errorf("answered %v (%v), want revision 2's entry", ok, err)
}
})
}
}
// outdatedOfAvahi is an indexer whose state versions holds avahi at a kind-5 edge, so Outdated reads the trunk's entry of avahi.
func outdatedOfAvahi(o Objects) *Indexer {
ix := &Indexer{Versions: MemoryStore(), Heard: MemoryStore(), Revisions: o}
data, _ := json.Marshal(meshcatalog.VersionEntry{V: 1, Module: "avahi", Version: "0.1.1", Edges: []meshcatalog.VersionEdge{{Kind: 5, Part: "actor:program"}}})
_ = ix.Versions.Create(context.Background(), meshcatalog.VersionsByModuleVersionKey("avahi", "0.1.1"), data)
return ix
}
@@ -3,7 +3,6 @@ package index
import (
"context"
"errors"
"fmt"
"slices"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
@@ -51,14 +50,10 @@ func (ix *Indexer) SeedEntryHeld(ctx context.Context, module, fp string) (bool,
if ctx.Err() != nil || ix.Revisions == nil {
return false, ctx.Err()
}
rev, found, err := newestTrunk(ctx, ix.Revisions)
_, entries, found, err := readNewestOf(ctx, ix.Revisions, TrunkBranch)
if err != nil || !found {
return false, err
}
_, entries, err := ReadRevision(ctx, ix.Revisions, fmt.Sprintf("trunk.%d", rev.Revision))
if err != nil {
return false, err
}
return slices.ContainsFunc(entries, func(e meshcatalog.CatalogEntry) bool { return e.Module == module && e.Fp == fp }), nil
}
@@ -67,14 +62,10 @@ func (ix *Indexer) ModuleInNewestView(ctx context.Context, module string) (bool,
if ctx.Err() != nil || ix.Revisions == nil {
return false, ctx.Err()
}
rev, found, err := newestTrunk(ctx, ix.Revisions)
_, entries, found, err := readNewestOf(ctx, ix.Revisions, TrunkBranch)
if err != nil || !found {
return false, err
}
_, entries, err := ReadRevision(ctx, ix.Revisions, fmt.Sprintf("trunk.%d", rev.Revision))
if err != nil {
return false, err
}
return slices.ContainsFunc(entries, func(e meshcatalog.CatalogEntry) bool { return e.Module == module }), nil
}
@@ -324,24 +324,28 @@ func CatalogRevisionTools(o Objects, ask func(key string, value []byte) (TakeInA
ctx := context.Background()
n, _ := args["revision"].(float64)
name := ""
var rev meshcatalog.CatalogRevision
var entries []meshcatalog.CatalogEntry
switch {
case str(args, "group") != "" || str(args, "heads") != "":
name = RevisionObject(meshcatalog.CatalogRevision{Kind: "candidate", Group: str(args, "group"), Heads: str(args, "heads")})
case n > 0:
name = "trunk." + strconv.FormatInt(int64(n), 10)
default:
newest, ok, err := newestTrunk(ctx, o)
newest, newestEntries, ok, err := readNewestOf(ctx, o, TrunkBranch)
if err != nil {
return nil, fmt.Errorf("the catalog graph revisions cannot be read: %w", err)
}
if !ok {
return nil, ErrNoRevision
}
name = RevisionObject(newest)
name, rev, entries = RevisionObject(newest), newest, newestEntries
}
rev, entries, err := ReadRevision(ctx, o, name)
if err != nil {
return nil, err
if entries == nil {
var err error
if rev, entries, err = ReadRevision(ctx, o, name); err != nil {
return nil, err
}
}
return map[string]any{"name": name, "revision": rev, "entries": entries}, nil
}},
@@ -72,12 +72,12 @@ func Revision(ctx context.Context, o Objects, kept *KeptTrunk, holder, caller st
var out meshcatalog.RevisionAnswer
var newest meshcatalog.CatalogRevision
var ok bool
var trunkEntries []meshcatalog.CatalogEntry
var trunkEntries, branchEntries []meshcatalog.CatalogEntry
var err error
if kept != nil {
newest, trunkEntries, ok, err = kept.Newest(ctx, o)
} else {
newest, ok, err = newestTrunk(ctx, o)
newest, trunkEntries, ok, err = readNewestOf(ctx, o, TrunkBranch)
}
if err != nil {
return out, fmt.Errorf("the catalog graph revisions cannot be read: %w", err)
@@ -86,11 +86,6 @@ func Revision(ctx context.Context, o Objects, kept *KeptTrunk, holder, caller st
// An answer is told by its fields (design 47 G §530): no trunk revision kept yet is `none-yet`, no error.
return meshcatalog.RevisionAnswer{NoneYet: true}, nil
}
if kept == nil {
if _, trunkEntries, err = ReadRevision(ctx, o, RevisionObject(newest)); err != nil {
return out, err
}
}
caller = CallerModule(trunkEntries, caller)
if err := views.Refusal(meshcatalog.Seat+"."+meshcatalog.VerbRevision, RevisionCallers(trunkEntries, holder), caller); err != nil {
return out, fmt.Errorf("%w (novox/hq ADR 0663 §2: a part reads it by declaring %s in invokes)", err, RevisionInvoke)
@@ -102,14 +97,14 @@ func Revision(ctx context.Context, o Objects, kept *KeptTrunk, holder, caller st
return out, fmt.Errorf("the branch %q is no escaped name: %w", in.Branch, err)
}
if !isTrunkBranch(branch) {
other, found, err := newestOf(ctx, o, branch)
other, entries, found, err := readNewestOf(ctx, o, branch)
if err != nil {
return out, fmt.Errorf("the catalog graph revisions of %s cannot be read: %w", branch, err)
}
if !found {
return meshcatalog.RevisionAnswer{NoneYet: true}, nil
}
newest = other
newest, branchEntries = other, entries
}
name := RevisionObject(newest)
switch {
@@ -133,7 +128,9 @@ func Revision(ctx context.Context, o Objects, kept *KeptTrunk, holder, caller st
}
rev, entries := newest, trunkEntries
if !isTrunkBranch(branch) && in.Group == "" && in.Heads == "" {
if rev, entries, err = ReadRevision(ctx, o, name); err != nil {
if name == RevisionObject(newest) {
entries = branchEntries // the newest of the branch, read by readNewestOf (A-487d)
} else if rev, entries, err = ReadRevision(ctx, o, name); err != nil {
return out, err
}
} else if name != RevisionObject(newest) {
@@ -63,17 +63,13 @@ func (ix *Indexer) SeedModulesFrom(ctx context.Context, defs map[string]seatdef.
if ix.Revisions == nil {
return nil, fmt.Errorf("the catalog graph is not kept here: no trunk revision to seed from")
}
rev, found, err := newestTrunk(ctx, ix.Revisions)
_, entries, found, err := readNewestOf(ctx, ix.Revisions, TrunkBranch)
if err != nil {
return nil, err
}
if !found {
return nil, fmt.Errorf("the indexer keeps no trunk revision yet: the seed waits for its first")
}
_, entries, err := ReadRevision(ctx, ix.Revisions, fmt.Sprintf("trunk.%d", rev.Revision))
if err != nil {
return nil, err
}
seatOf := map[string]string{} // artifact reference -> its build seat, all found before any call is made
unbuilt := map[string][]string{} // module -> each artifact of it no build seat builds, with its language or kind
for _, e := range entries {
@@ -177,17 +177,13 @@ func (ix *Indexer) entryOf(ctx context.Context, module string) (meshcatalog.Cata
if ix.Revisions == nil {
return meshcatalog.CatalogEntry{}, errors.New("the catalog graph is not kept here: no entry names the module's artifacts")
}
rev, found, err := newestTrunk(ctx, ix.Revisions)
_, entries, found, err := readNewestOf(ctx, ix.Revisions, TrunkBranch)
if err != nil {
return meshcatalog.CatalogEntry{}, err
}
if !found {
return meshcatalog.CatalogEntry{}, errors.New("the indexer keeps no trunk revision yet")
}
_, entries, err := ReadRevision(ctx, ix.Revisions, fmt.Sprintf("trunk.%d", rev.Revision))
if err != nil {
return meshcatalog.CatalogEntry{}, err
}
for _, e := range entries {
if e.Module == module {
return e, nil
@@ -194,11 +194,13 @@ func (ix *Indexer) Outdated(ctx context.Context, in meshcatalog.OutdatedInput) (
}
trunk := map[string]meshcatalog.CatalogEntry{} // the newest trunk revision's entries, read where a kind needs them
if ix.Revisions != nil {
if rev, found, err := newestTrunk(ctx, ix.Revisions); err == nil && found {
if _, entries, err := ReadRevision(ctx, ix.Revisions, fmt.Sprintf("trunk.%d", rev.Revision)); err == nil {
for _, e := range entries {
trunk[e.Module] = e
}
_, entries, found, err := readNewestOf(ctx, ix.Revisions, TrunkBranch)
if err != nil {
return out, fmt.Errorf("the trunk's entries, which the kinds 3 and 5 are told by, cannot be read: %w", err)
}
if found {
for _, e := range entries {
trunk[e.Module] = e
}
}
}
@@ -79,17 +79,13 @@ func (ix *Indexer) FillVersions(ctx context.Context, ask VersionsAsker) error {
if ix.Revisions == nil {
return errors.New("the catalog graph is not kept here: no entry names the modules' artifacts")
}
rev, found, err := newestTrunk(ctx, ix.Revisions)
_, entries, found, err := readNewestOf(ctx, ix.Revisions, TrunkBranch)
if err != nil {
return err
}
if !found {
return errors.New("the indexer keeps no trunk revision yet: the first fill of the versions waits for its first")
}
_, entries, err := ReadRevision(ctx, ix.Revisions, fmt.Sprintf("trunk.%d", rev.Revision))
if err != nil {
return err
}
type found1 struct {
v BuilderVersion
name string