Files
mesh-delivery/modules/mesh-delivery-indexer/internal/index/takein_source_test.go
T
jschoubben f95a164941 indexer: drop the earlier model's trunk and branch revisions at a take-in
catalog-revisions kept trunk.<n> and branch.<branch>.<n> objects beside the
branch-keyed revisions, the newest trunk one marked current, so a reader heard
revision 528 while its picture held 1. A branch's take-in now drops every such
object no record pins, says the count in the journal, and the revisions tool marks
each branch's newest revision current.

Builds: design 47 H §743, H-A-135

Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy
2026-11-20 12:12:23 +01:00

291 lines
14 KiB
Go

package index
import (
"context"
"encoding/json"
"errors"
"fmt"
"reflect"
"testing"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
)
// takeInIndexer is an indexer that takes source revisions in from the catalogue repository's trees at f's commit, keeping its
// revisions and pins, and the take-in events it says (a take-in.pending kept as a taken with its why and skipped set).
func takeInIndexer(t *testing.T) (*Indexer, *memObjects, *[]meshcatalog.TakeInTaken, whole) {
t.Helper()
ix, _, f := newFormIndexerUnmerged(t, mod{"avahi", catalogueManifest(t, "avahi")})
reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1")
objects := newMemObjects(1 << 22)
ix.Revisions, ix.Pins, ix.PinsPending = objects, newMem(), func(context.Context) (uint64, error) { return 0, nil }
ix.Read = func(_ context.Context, repository, sha, _ string) (RepoRead, error) {
if repository != f.Repository || sha != f.Merge {
return RepoRead{}, errors.New("no such commit")
}
return RepoRead{Root: f.Root, Trees: f.Trees, Manifests: f.Manifests, Locks: f.Locks, Packages: f.Packages}, nil
}
ix.ReadFile = func(context.Context, string, string, string) ([]byte, error) { return nil, nil }
said := &[]meshcatalog.TakeInTaken{}
ix.SayTakeIn = func(_ context.Context, event, id string, body any) error {
if p, ok := body.(meshcatalog.TakeInPending); ok && event == "take-in.pending" && id == PendingID(p.Branch, p.Source) {
*said = append(*said, meshcatalog.TakeInTaken{Branch: p.Branch, Source: p.Source, Why: p.Why, Skipped: true})
return nil
}
tk, ok := body.(meshcatalog.TakeInTaken)
if event != "take-in.taken" || !ok || id != TakenID(tk.Branch, tk.Source) {
t.Errorf("said %s %s %+v, want take-in.taken under its branch and source", event, id, body)
}
*said = append(*said, tk)
return nil
}
return ix, objects, said, f
}
// Design 47 H §743, H-A-32, H-A-92: a take-in writes one catalog revision <branch>.<source> carrying its movers, pins it for the
// take-in's pipeline and says take-in.taken once; a redelivery writes nothing again and says the same; a revision with no mover
// pins nothing and says no entries; the old form is refused.
func TestASourceRevisionIsTakenInAsOneRevision(t *testing.T) {
ctx := context.Background()
ix, objects, said, f := takeInIndexer(t)
moved := meshdelivery.SourceMovedEvent{V: 1, Branch: "main", Source: 7, Call: "0123456789abcdef",
Heads: []meshdelivery.SourceMovedEventHeadsItem{{Repository: f.Repository, Commit: f.Merge}}}
taken, err := ix.TakeInSource(ctx, moved)
if err != nil {
t.Fatal(err)
}
if len(taken.Entries) != 1 || taken.Entries[0].Module != "avahi" || taken.Entries[0].Fingerprint == "" || taken.Entries[0].Commit != f.Merge ||
taken.Revision != 7 || taken.Branch != "main" || taken.Source != 7 || taken.Call != moved.Call {
t.Fatalf("taken %+v, want revision 7 of main with avahi as its one mover", taken)
}
rev, _, err := ReadRevision(ctx, objects, "main.7")
if err != nil || !reflect.DeepEqual(rev.Movers, taken.Entries) || rev.Source != 7 {
t.Fatalf("main.7 = %+v (%v), want it kept carrying the movers", rev, err)
}
if v, _, found, _ := ix.Pins.Get(ctx, PinKey("main", 7)); !found {
t.Error("main.7 is not pinned for its pipeline")
} else {
var pv PinValue
if json.Unmarshal(v, &pv) != nil || !reflect.DeepEqual(pv.Records, []string{"pipeline.main.7"}) {
t.Errorf("main.7 is pinned by %s, want pipeline.main.7", v)
}
}
puts := objects.puts
again, err := ix.TakeInSource(ctx, moved)
if err != nil || objects.puts != puts || !reflect.DeepEqual(again, taken) {
t.Errorf("a redelivery wrote %d objects and said %+v (%v), want none and the same", objects.puts-puts, again, err)
}
same := moved
same.Source, same.Call = 8, "fedcba9876543210"
none, err := ix.TakeInSource(ctx, same)
if err != nil || len(none.Entries) != 0 || none.Revision != 8 {
t.Errorf("a revision moving no fingerprint said %+v (%v), want revision 8 with no entries", none, err)
}
if _, _, found, _ := ix.Pins.Get(ctx, PinKey("main", 8)); found {
t.Error("a take-in with no mover pinned its revision")
}
if _, _, err := ReadRevision(ctx, objects, "main.8"); err != nil {
t.Errorf("main.8 is not kept: %v", err)
}
if len(*said) != 3 {
t.Errorf("said %d take-in.taken, want 3", len(*said))
}
old := moved
old.Call = ""
if _, err := ix.TakeInSource(ctx, old); !errors.Is(err, ErrOldForm) {
t.Errorf("a source-moved with no call answered %v, want ErrOldForm", err)
}
}
// Design 47 H §768, §743, H-A-19: a branch's first source revision (start true) is taken in with no mover, says take-in.taken
// with no entries and pins nothing; its entries are the base a later take-in's movers are judged against.
func TestAFirstSourceRevisionIsTakenInWithNoMover(t *testing.T) {
ctx := context.Background()
ix, objects, _, f := takeInIndexer(t)
start := meshdelivery.SourceMovedEvent{V: 1, Branch: "main", Source: 1, Call: "seed.operator.20261120", Start: true,
Heads: []meshdelivery.SourceMovedEventHeadsItem{{Repository: f.Repository, Commit: f.Merge}}}
taken, err := ix.TakeInSource(ctx, start)
if err != nil || len(taken.Entries) != 0 || taken.Revision != 1 {
t.Fatalf("the first revision said %+v (%v), want revision 1 with no entries", taken, err)
}
if keys, _ := ix.Pins.Keys(ctx); len(keys) != 0 {
t.Errorf("the first revision pinned %v", keys)
}
if _, entries, err := ReadRevision(ctx, objects, "main.1"); err != nil || len(entries) != 1 || entries[0].Commit != f.Merge {
t.Fatalf("main.1 holds %+v (%v), want avahi at the head", entries, err)
}
next := start
next.Source, next.Call, next.Start = 2, "0123456789abcdef", false
if again, err := ix.TakeInSource(ctx, next); err != nil || len(again.Entries) != 0 {
t.Errorf("a later take-in of the same head said %+v (%v), want no mover against the first revision", again, err)
}
}
// Design 47 H §743, H-A-135: at a branch's take-in the earlier model's revisions (trunk.<n>, branch.<branch>.<n>) no record
// pins are dropped, one a record pins stays, and the branch's newest revision is the current one, no longer the newest trunk.
func TestATakeInDropsTheEarlierFormsNoRecordPins(t *testing.T) {
ctx := context.Background()
ix, objects, _, f := takeInIndexer(t)
for _, n := range []string{"trunk.527", "trunk.528", "branch.master.1"} {
_, rev, _ := RevisionGroup(n)
raw, _ := json.Marshal(meshcatalog.CatalogRevision{Revision: rev, Kind: "trunk", Source: 1000 + rev})
if err := objects.Put(ctx, n, raw); err != nil {
t.Fatal(err)
}
}
if err := ix.pin(ctx, "trunk.527", "pipeline.old"); err != nil {
t.Fatal(err)
}
start := meshdelivery.SourceMovedEvent{V: 1, Branch: "main", Source: 1, Call: "seed.operator.20261120", Start: true,
Heads: []meshdelivery.SourceMovedEventHeadsItem{{Repository: f.Repository, Commit: f.Merge}}}
if _, err := ix.TakeInSource(ctx, start); err != nil {
t.Fatal(err)
}
a, err := ListRevisions(ctx, objects, false)
if err != nil {
t.Fatal(err)
}
var got []string
for _, r := range a.Revisions {
got = append(got, fmt.Sprintf("%s current=%t", r.Name, r.Current))
}
if want := []string{"main.1 current=true", "trunk.527 current=false"}; !reflect.DeepEqual(got, want) {
t.Errorf("kept %v, want %v: the unpinned earlier forms dropped, the pinned one kept, main.1 current", got, want)
}
}
// Design 47 H §769: the take-in of a source revision moving the catalogue repository reads `predefined` from that commit's
// trees, and one changing a predefined key rewrites it.
func TestATakeInOfTheCatalogsCommitRewritesPredefined(t *testing.T) {
ctx := context.Background()
ix, _, _, f := takeInIndexer(t)
ix.Predefined = newMem()
ix.Contains = func(context.Context, string, string, string) (bool, error) { return false, nil }
ix.Read = func(_ context.Context, _, _, _ string) (RepoRead, error) {
return RepoRead{Root: f.Root, Trees: f.Trees, Manifests: f.Manifests, Locks: f.Locks, Packages: f.Packages}, nil
}
lists := map[string]string{"c1": `{"keys":["a"]}`, "c2": `{"keys":["a","b"]}`}
ix.ReadFile = func(_ context.Context, _, sha, path string) ([]byte, error) {
if path != predefinedFile {
return nil, nil
}
return []byte(lists[sha]), nil
}
for i, commit := range []string{"c1", "c2"} {
moved := meshdelivery.SourceMovedEvent{V: 1, Branch: "main", Source: int64(i + 1), Call: "0123456789abcde" + commit[1:],
Heads: []meshdelivery.SourceMovedEventHeadsItem{{Repository: catalogueRepo, Commit: commit}}}
if _, err := ix.TakeInSource(ctx, moved); err != nil {
t.Fatal(err)
}
raw, _, found, _ := ix.Predefined.Get(ctx, PredefinedListKey)
var kept struct {
Commit string `json:"commit"`
List json.RawMessage `json:"list"`
}
if !found || json.Unmarshal(raw, &kept) != nil || kept.Commit != commit || string(kept.List) != lists[commit] {
t.Errorf("after the take-in of %s predefined holds %s, want its list at %s", commit, raw, commit)
}
}
}
// Design 47 H §744 (acceptance test 116): a take-in the full catalog store cannot write is held, said once per source as
// take-in.pending with branch, source and why, and room is said once the store has room again; taken in then.
func TestATakeInOnAFullStoreIsHeldAndSaidOnce(t *testing.T) {
ctx := context.Background()
ix, objects, said, f := takeInIndexer(t)
objects.capacity = 10
ix.Held = newMem()
moved := meshdelivery.SourceMovedEvent{V: 1, Branch: "main", Source: 3, Call: "0123456789abcdef",
Heads: []meshdelivery.SourceMovedEventHeadsItem{{Repository: f.Repository, Commit: f.Merge}}}
var rooms int
ix.SayFill = func(_ context.Context, event, _ string, _ meshcatalog.Fill) error {
if event == meshcatalog.EventRoom {
rooms++
}
return nil
}
for range 2 {
if _, err := ix.TakeInSource(ctx, moved); !errors.Is(err, ErrObjectsFull) {
t.Fatalf("a full store answered %v, want ErrObjectsFull", err)
}
}
if len(*said) != 1 || (*said)[0].Branch != "main" || (*said)[0].Source != 3 || (*said)[0].Why == "" || !(*said)[0].Skipped {
t.Fatalf("said %+v, want one take-in.pending of main 3 with its why", *said)
}
if held, err := ReadHeld(ctx, ix.Held); err != nil || len(held) != 1 || held[0].Branch != "main" || held[0].Source != 3 || held[0].Why == "" {
t.Errorf("held = %+v (%v), want main's source revision 3 and why", held, err)
}
// H-A-118: the held value carries the event whole, and a later source of the branch is held behind it, in order.
next := moved
next.Source, next.Call = 4, "1123456789abcdef"
if behind, err := ix.HoldBehind(ctx, next); err != nil || !behind {
t.Fatalf("the later source held behind: %v (%v)", behind, err)
}
if held, _ := ReadHeld(ctx, ix.Held); len(held) != 2 || held[0].Source != 3 || held[1].Source != 4 || held[0].Call != moved.Call ||
len(held[0].Heads) != 1 || held[0].Heads[0] != moved.Heads[0] || !reflect.DeepEqual(held[0].Event(), moved) {
t.Fatalf("held = %+v, want 3 then 4, each with its event whole", held)
}
objects.capacity = 1 << 22
if ok, err := ix.RoomAgain(ctx); err != nil || !ok || rooms != 1 {
t.Fatalf("room said %v %d times (%v), want once the store has room", ok, rooms, err)
}
// H-A-114: on room (and at start) the held ones are taken in again from the state held alone, in their order.
ix.TakeHeldIn(ctx)
var taken []int64
for _, e := range *said {
if !e.Skipped {
taken = append(taken, e.Source)
}
}
if !reflect.DeepEqual(taken, []int64{3, 4}) {
t.Errorf("taken again on room: %v of %+v, want 3 then 4", taken, *said)
}
if held, _ := ReadHeld(ctx, ix.Held); len(held) != 0 {
t.Errorf("held = %+v after the take-in, want none", held)
}
// A skip ends a held source revision unwritten: take-in.taken with no entries, skipped, by and why; held is deleted.
ix.hold(ctx, meshdelivery.SourceMovedEvent{Branch: "nox-mesh", Source: 9, Call: "fedcba9876543210"}, "the catalog store is full")
if err := ix.SkipTakeIn(ctx, meshdelivery.SourceMovedEvent{Branch: "nox-mesh", Source: 9, Call: "fedcba9876543210"}, "operator", "a stale source"); err != nil {
t.Fatal(err)
}
if last := (*said)[len(*said)-1]; !last.Skipped || last.By != "operator" || last.Why != "a stale source" || len(last.Entries) != 0 || last.Source != 9 {
t.Errorf("the skip said %+v, want take-in.taken skipped by the operator with its why", last)
}
if held, _ := ReadHeld(ctx, ix.Held); len(held) != 0 {
t.Errorf("held = %+v after the skip, want none", held)
}
}
// Design 47 H §743, §768, H-A-129, H-A-130: the gate's source revision read through its verb source is taken in as a heard
// source-moved of it would be: start true when the view keeps no revision of the branch, every repository it names a head;
// one the view keeps at or above is not taken again; a later one is judged against the newest kept, start false.
func TestASourceRevisionReadFromTheGateIsTakenInOnce(t *testing.T) {
ctx := context.Background()
ix, _, said, f := takeInIndexer(t)
answer := meshdelivery.SourceAnswer{Branch: "main", Revision: 1, Call: "0123456789abcdef",
Repositories: []meshdelivery.SourceHead{{Repository: f.Repository, Trunk: "main", Head: f.Merge}}}
e, take, err := ix.SourceToTake(ctx, "main", answer)
want := meshdelivery.SourceMovedEvent{V: 1, Branch: "main", Source: 1, Call: answer.Call, Start: true,
Heads: []meshdelivery.SourceMovedEventHeadsItem{{Repository: f.Repository, Commit: f.Merge}}}
if err != nil || !take || !reflect.DeepEqual(e, want) {
t.Fatalf("a branch the view keeps none of answered %+v, %t (%v); want %+v taken", e, take, err, want)
}
if taken, err := ix.TakeInSource(ctx, e); err != nil || len(taken.Entries) != 0 || taken.Revision != 1 {
t.Fatalf("the first revision read from the gate said %+v (%v), want revision 1 with no mover", taken, err)
}
if _, take, err := ix.SourceToTake(ctx, "main", answer); take || err != nil {
t.Errorf("a revision the view keeps is taken again (%v)", err)
}
if _, take, _ := ix.SourceToTake(ctx, "nox-mesh", meshdelivery.SourceAnswer{Branch: "nox-mesh"}); take {
t.Error("a branch the gate keeps no revision of is taken in")
}
answer.Revision, answer.Call = 2, "1123456789abcdef"
if e, take, err := ix.SourceToTake(ctx, "main", answer); err != nil || !take || e.Start || e.Source != 2 {
t.Errorf("a later revision answered %+v, %t (%v); want revision 2 taken, start false", e, take, err)
}
if len(*said) != 1 {
t.Errorf("said %d take-in.taken, want 1", len(*said))
}
}