Design 47 H1 (1): a release re-pin keeps the next catalog revision of the same source on every watched branch

A branch's revision number is the indexer's own count (the newest + 1); the gate's source is the revision's field source, which the take-in's dedupe and source-to-take read. A catalog revision carries its call (the seat mesh-catalog's CatalogRevision and catalog-revision gain call, regenerated by seatgen). A re-pin on a module's release runs on every watched branch holding a module that names its package (the first on the modules state, another on its newest revision's picture), and keeps, where a fingerprint moved, the branch's next revision of the same source with the call 'release of <module> <version>' and the moved modules as its movers, pinned for the release's pipeline, and says catalog-revision; no take-in.taken.

Builds: design 47 H S11, S12, S13, S14 (H-pins-spec-and-gaps item 1), H-A-211, H-A-212, H-A-213 (1)-(4: the pin of every revision a pipeline runs on), H-A-147
This commit is contained in:
2026-11-20 23:36:39 +01:00
parent 52f7a5b6f6
commit 47ea130332
18 changed files with 389 additions and 162 deletions
@@ -715,6 +715,8 @@ type CatalogRevision struct {
Base int64 `json:"base,omitempty"`
// the watched branch the revision is of
Branch string `json:"branch,omitempty"`
// what made the revision (design 47 H-A-212, H-A-213): a take-in's the gate's call its source revision names, a re-pin's release of <module> <version>, a take-in again's take-in again of <repository>; absent for a candidate's
Call string `json:"call,omitempty"`
// the seat definitions it was judged with, each by its digest, read whole through seat-definitions
Definitions []CatalogRevisionDefinitionsItem `json:"definitions,omitempty"`
// one per module, by name
@@ -1301,6 +1303,8 @@ type CatalogRevisionEvent struct {
At *time.Time `json:"at,omitempty"`
// the watched branch the revision is of, escaped as a key name is
Branch string `json:"branch,omitempty"`
// what made the revision, as the revision's call names it (design 47 H-A-212, H-A-213); absent for a candidate's
Call string `json:"call,omitempty"`
// the modules whose entries differ from the revision before it of the same kind
Changed []string `json:"changed,omitempty"`
// how many entries it lists
@@ -17,7 +17,7 @@ func TestFixtures(t *testing.T) {
decode func([]byte) (any, error)
subjectIs *regexp.Regexp
}{
{name: "a trunk revision kept, on the seat's subject", subject: "mesh.seat.mesh-catalog.event.catalog-revision.main", given: "{\"at\":\"2026-11-09T19:00:00Z\",\"branch\":\"main\",\"changed\":[\"xclip\"],\"entries\":2,\"kind\":\"trunk\",\"revision\":12,\"source\":3589,\"v\":1}", wire: "{\"at\":\"2026-11-09T19:00:00Z\",\"branch\":\"main\",\"changed\":[\"xclip\"],\"entries\":2,\"kind\":\"trunk\",\"revision\":12,\"source\":3589,\"v\":1}",
{name: "a trunk revision kept, on the seat's subject", subject: "mesh.seat.mesh-catalog.event.catalog-revision.main", given: "{\"at\":\"2026-11-09T19:00:00Z\",\"branch\":\"main\",\"call\":\"release of mesh-sdk 0.1.27\",\"changed\":[\"xclip\"],\"entries\":2,\"kind\":\"trunk\",\"revision\":12,\"source\":3589,\"v\":1}", wire: "{\"at\":\"2026-11-09T19:00:00Z\",\"branch\":\"main\",\"call\":\"release of mesh-sdk 0.1.27\",\"changed\":[\"xclip\"],\"entries\":2,\"kind\":\"trunk\",\"revision\":12,\"source\":3589,\"v\":1}",
decode: func(b []byte) (any, error) { var v CatalogRevisionEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-catalog\\.event\\.catalog-revision\\.[^.*> ]+$")},
{name: "a candidate's revision kept", subject: "mesh.seat.mesh-catalog.event.catalog-revision.main", given: "{\"branch\":\"main\",\"entries\":2,\"group\":\"feat/47-G\",\"heads\":\"0123456789ab\",\"kind\":\"candidate\",\"revision\":13,\"source\":3589,\"v\":1}", wire: "{\"branch\":\"main\",\"entries\":2,\"group\":\"feat/47-G\",\"heads\":\"0123456789ab\",\"kind\":\"candidate\",\"revision\":13,\"source\":3589,\"v\":1}",
decode: func(b []byte) (any, error) { var v CatalogRevisionEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-catalog\\.event\\.catalog-revision\\.[^.*> ]+$")},
@@ -59,7 +59,7 @@ func TestFixtures(t *testing.T) {
decode: func(b []byte) (any, error) { var v ModulesAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil},
{name: "the module verb (captured: xclip)", subject: "", given: "{\"module\":{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"fp\":\"src2:b427bdde1f100e2f52490cfa9df0c5112dd3700226b4892349f0c14a15a4af56\",\"manifest\":{\"build\":{\"artifacts\":[{\"binary\":\"xclip-tools\",\"from\":\"cmd/xclip-tools\",\"kind\":\"bundle\",\"language\":\"go\",\"loads\":[\"xclip-tools\"],\"name\":\"tools\",\"system\":\"arch\"}]},\"capabilities\":[\"package-manager\"],\"module\":\"xclip\",\"resources\":[{\"id\":\"package\",\"package\":\"xclip\",\"type\":\"package\"}],\"tools\":[\"xclip_copy\",\"xclip_paste\",\"xclip_targets\",\"xclip_session\"],\"version\":\"1\"},\"module\":\"xclip\",\"path\":\"modules/xclip\",\"repository\":\"novox/mesh-catalog\"}}", wire: "{\"module\":{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"fp\":\"src2:b427bdde1f100e2f52490cfa9df0c5112dd3700226b4892349f0c14a15a4af56\",\"manifest\":{\"build\":{\"artifacts\":[{\"binary\":\"xclip-tools\",\"from\":\"cmd/xclip-tools\",\"kind\":\"bundle\",\"language\":\"go\",\"loads\":[\"xclip-tools\"],\"name\":\"tools\",\"system\":\"arch\"}]},\"capabilities\":[\"package-manager\"],\"module\":\"xclip\",\"resources\":[{\"id\":\"package\",\"package\":\"xclip\",\"type\":\"package\"}],\"tools\":[\"xclip_copy\",\"xclip_paste\",\"xclip_targets\",\"xclip_session\"],\"version\":\"1\"},\"module\":\"xclip\",\"path\":\"modules/xclip\",\"repository\":\"novox/mesh-catalog\"}}",
decode: func(b []byte) (any, error) { var v ModuleAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil},
{name: "a watched branch's catalog graph revision", subject: "", given: "{\"at\":\"2026-11-09T19:00:00Z\",\"base\":9,\"branch\":\"main\",\"definitions\":[{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"seat\":\"node-build-go\"}],\"entries\":[{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"module\":\"mesh-build-go\"},{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"module\":\"xclip\"}],\"kind\":\"trunk\",\"movers\":[{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"fingerprint\":\"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\",\"module\":\"xclip\",\"repository\":\"novox/mesh-catalog\"}],\"revision\":12,\"source\":3589,\"trunk\":[{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"repository\":\"novox/mesh-catalog\"}],\"v\":1}", wire: "{\"at\":\"2026-11-09T19:00:00Z\",\"base\":9,\"branch\":\"main\",\"definitions\":[{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"seat\":\"node-build-go\"}],\"entries\":[{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"module\":\"mesh-build-go\"},{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"module\":\"xclip\"}],\"kind\":\"trunk\",\"movers\":[{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"fingerprint\":\"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\",\"module\":\"xclip\",\"repository\":\"novox/mesh-catalog\"}],\"revision\":12,\"source\":3589,\"trunk\":[{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"repository\":\"novox/mesh-catalog\"}],\"v\":1}",
{name: "a watched branch's catalog graph revision", subject: "", given: "{\"at\":\"2026-11-09T19:00:00Z\",\"base\":9,\"branch\":\"main\",\"call\":\"release of mesh-sdk 0.1.27\",\"definitions\":[{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"seat\":\"node-build-go\"}],\"entries\":[{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"module\":\"mesh-build-go\"},{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"module\":\"xclip\"}],\"kind\":\"trunk\",\"movers\":[{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"fingerprint\":\"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\",\"module\":\"xclip\",\"repository\":\"novox/mesh-catalog\"}],\"revision\":12,\"source\":3589,\"trunk\":[{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"repository\":\"novox/mesh-catalog\"}],\"v\":1}", wire: "{\"at\":\"2026-11-09T19:00:00Z\",\"base\":9,\"branch\":\"main\",\"call\":\"release of mesh-sdk 0.1.27\",\"definitions\":[{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"seat\":\"node-build-go\"}],\"entries\":[{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"module\":\"mesh-build-go\"},{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"module\":\"xclip\"}],\"kind\":\"trunk\",\"movers\":[{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"fingerprint\":\"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\",\"module\":\"xclip\",\"repository\":\"novox/mesh-catalog\"}],\"revision\":12,\"source\":3589,\"trunk\":[{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"repository\":\"novox/mesh-catalog\"}],\"v\":1}",
decode: func(b []byte) (any, error) {
var v CatalogRevisionsByBranchRevisionValue
err := json.Unmarshal(b, &v)
@@ -8,6 +8,7 @@
"revision": 12,
"kind": "trunk",
"source": 3589,
"call": "release of mesh-sdk 0.1.27",
"entries": 2,
"changed": [
"xclip"
@@ -209,7 +209,8 @@
"repository": "novox/mesh-catalog"
}
],
"base": 9
"base": 9,
"call": "release of mesh-sdk 0.1.27"
}
},
{
@@ -657,6 +657,10 @@
"source": {
"type": "integer"
},
"call": {
"type": "string",
"description": "what made the revision, as the revision's call names it (design 47 H-A-212, H-A-213); absent for a candidate's"
},
"group": {
"type": "string"
},
@@ -1871,6 +1875,10 @@
"type": "integer",
"description": "the source revision it was taken in from; for a candidate, the source revision its heads are merged onto"
},
"call": {
"type": "string",
"description": "what made the revision (design 47 H-A-212, H-A-213): a take-in's the gate's call its source revision names, a re-pin's release of <module> <version>, a take-in again's take-in again of <repository>; absent for a candidate's"
},
"group": {
"type": "string",
"description": "a candidate's group"
@@ -277,6 +277,8 @@ export interface CatalogRevision {
"base"?: number;
/** the watched branch the revision is of */
"branch"?: string;
/** what made the revision (design 47 H-A-212, H-A-213): a take-in's the gate's call its source revision names, a re-pin's release of <module> <version>, a take-in again's take-in again of <repository>; absent for a candidate's */
"call"?: string;
/** the seat definitions it was judged with, each by its digest, read whole through seat-definitions */
"definitions"?: CatalogRevisionDefinitionsItem[];
/** one per module, by name */
@@ -796,6 +798,8 @@ export interface CatalogRevisionEvent {
"at"?: string;
/** the watched branch the revision is of, escaped as a key name is */
"branch"?: string;
/** what made the revision, as the revision's call names it (design 47 H-A-212, H-A-213); absent for a candidate's */
"call"?: string;
/** the modules whose entries differ from the revision before it of the same kind */
"changed"?: string[];
/** how many entries it lists */
@@ -6,6 +6,7 @@ import (
"context"
"encoding/json"
"errors"
"fmt"
"os"
"slices"
"strconv"
@@ -447,37 +448,42 @@ func TestIndexerWhole(t *testing.T) {
// The first revision: no mover, no pin; its entries are the base.
sayMoved(t, ctx, js, "main", 10, c10, true)
if first := heard.next(t, "main", 10); len(first.Entries) != 0 || first.Revision != 10 {
t.Fatalf("the first revision said %+v, want revision 10 with no entries", first)
// A revision's number is the indexer's own count, the branch's newest + 1 (design 47 H-A-213 (1)): the later take-ins of
// sources 11 to 13 are counted on from the number of source 10's.
first := heard.next(t, "main", 10)
r := func(source int64) int64 { return first.Revision + source - 10 }
rn := func(source int64) string { return fmt.Sprintf("main.%d", r(source)) }
if len(first.Entries) != 0 || first.Revision <= 0 || first.Source != 10 {
t.Fatalf("the first revision said %+v, want a revision of source 10 with no entries", first)
}
untilKey(t, ctx, pins, index.PinKey("main", 10), absent)
untilKey(t, ctx, pins, index.PinKey("main", r(10)), absent)
// One take-in, one revision carrying its movers, pinned for its pipeline; said again from the kept movers.
sayMoved(t, ctx, js, "main", 11, c11, false)
taken := heard.next(t, "main", 11)
if len(taken.Entries) != 1 || taken.Entries[0].Module != "avahi" || taken.Entries[0].Commit != c11 || taken.Entries[0].Fingerprint == "" ||
taken.Entries[0].Repository != "novox/mesh-catalog" || taken.Revision != 11 || taken.Call == "" {
taken.Entries[0].Repository != "novox/mesh-catalog" || taken.Revision != r(11) || taken.Call == "" {
t.Fatalf("take-in.taken of main 11 = %+v, want avahi at %s as its one mover", taken, c11)
}
rev, _, err := index.ReadRevision(ctx, objects, "main.11")
rev, _, err := index.ReadRevision(ctx, objects, rn(11))
if err != nil || len(rev.Movers) != 1 || rev.Movers[0] != taken.Entries[0] {
t.Fatalf("main.11 = %+v (%v), want it kept carrying the movers said", rev, err)
}
untilKey(t, ctx, pins, index.PinKey("main", 11), pinned("pipeline.main.11"))
untilKey(t, ctx, pins, index.PinKey("main", 10), pinned("pipeline.main.11")) // validate's base, H-A-147
untilKey(t, ctx, pins, index.PinKey("main", r(11)), pinned("pipeline."+rn(11)))
untilKey(t, ctx, pins, index.PinKey("main", r(10)), pinned("pipeline."+rn(11))) // validate's base, H-A-147
info, err := objects.open(ctx)
if err != nil {
t.Fatal(err)
}
before, err := info.GetInfo(ctx, "main.11")
before, err := info.GetInfo(ctx, rn(11))
if err != nil {
t.Fatal(err)
}
sayMoved(t, ctx, js, "main", 11, c11, false)
if again := heard.next(t, "main", 11); len(again.Entries) != 1 || again.Entries[0] != taken.Entries[0] || again.Revision != 11 {
if again := heard.next(t, "main", 11); len(again.Entries) != 1 || again.Entries[0] != taken.Entries[0] || again.Revision != r(11) {
t.Errorf("the source-moved given again said %+v, want the kept movers %+v", again, taken.Entries)
}
if after, err := info.GetInfo(ctx, "main.11"); err != nil || after.NUID != before.NUID {
if after, err := info.GetInfo(ctx, rn(11)); err != nil || after.NUID != before.NUID {
t.Errorf("main.11 was written again (%v): a source-moved given again keeps no second revision", err)
}
@@ -485,11 +491,11 @@ func TestIndexerWhole(t *testing.T) {
// (H-A-147), which the take-in pinned for it.
sayMoved(t, ctx, js, "main", 12, c12, false)
heard.next(t, "main", 12)
if err := pins.Delete(ctx, index.PinKey("main", 12)); err != nil { // the take-in's own pin, so pipeline-started alone writes it below
if err := pins.Delete(ctx, index.PinKey("main", r(12))); err != nil { // the take-in's own pin, so pipeline-started alone writes it below
t.Fatal(err)
}
sayPipeline(t, ctx, js, false, "main", 12)
untilKey(t, ctx, pins, index.PinKey("main", 12), pinned("pipeline.main.12"))
sayPipeline(t, ctx, js, false, "main", r(12))
untilKey(t, ctx, pins, index.PinKey("main", r(12)), pinned("pipeline."+rn(12)))
// A run on request: its candidate and its key written on run-started, from avahi's files at the run's commit (A-160, A-174).
manifest, err := os.ReadFile("../../internal/index/testdata/catalogue-a67f0196-avahi.module.json")
if err != nil {
@@ -515,7 +521,7 @@ func TestIndexerWhole(t *testing.T) {
heard.acked(t, "main")
}
restart()
for _, name := range []string{"main.10", "main.11"} {
for _, name := range []string{rn(10), rn(11)} {
if _, _, err := index.ReadRevision(ctx, objects, name); err != nil {
t.Fatalf("%s is not kept across a restart while main.11's pipeline runs: %v", name, err)
}
@@ -553,27 +559,27 @@ func TestIndexerWhole(t *testing.T) {
t.Errorf("the held source-moved's consumer %+v (%v), want nothing unacknowledged", info, err)
}
setCap(t, ctx, js, 1<<30) // a larger cap alone wakes nothing: no timer takes a held take-in in
sayPipeline(t, ctx, js, true, "main", 11)
sayPipeline(t, ctx, js, true, "main", r(11))
if got := heard.next(t, "main", 13); len(got.Entries) != 1 || got.Entries[0].Commit != c13 {
t.Errorf("main 13 taken in on room said %+v, want avahi at %s", got, c13)
}
untilKey(t, ctx, held, keys.EscapeName("main"), absent)
// main.12's validate base stays (H-A-147), and main.11's passed pipeline's release pins it (H-A-168, H-A-170).
untilKey(t, ctx, pins, index.PinKey("main", 11), pinned("pipeline.main.12", index.ReleaseRecord("main", "avahi")))
untilRevisionGone(t, ctx, js, "main.10") // the end freed both of main.11's pins
untilKey(t, ctx, pins, index.PinKey("main", r(11)), pinned("pipeline."+rn(12), index.ReleaseRecord("main", "avahi")))
untilRevisionGone(t, ctx, js, rn(10)) // the end freed both of main.11's pins
// main.12's passed pipeline releases avahi again; its pipeline-wanted, the seat mesh-wants' (its wants written to the end),
// frees main.11's older release, and main.11 goes (H-A-168, H-A-170, H-A-173).
sayPipeline(t, ctx, js, true, "main", 12)
untilKey(t, ctx, pins, index.PinKey("main", 12), func(v []byte) bool {
sayPipeline(t, ctx, js, true, "main", r(12))
untilKey(t, ctx, pins, index.PinKey("main", r(12)), func(v []byte) bool {
var p index.PinValue
return v != nil && json.Unmarshal(v, &p) == nil && slices.Contains(p.Records, index.ReleaseRecord("main", "avahi"))
})
wantedRaw, _ := json.Marshal(meshwants.PipelineWanted{V: 1, Pipeline: "main.12", Branch: "main", Source: 12, Outcome: meshwants.PipelineWantedOutcomeRunning})
wantedRaw, _ := json.Marshal(meshwants.PipelineWanted{V: 1, Pipeline: rn(12), Branch: "main", Source: 12, Outcome: meshwants.PipelineWantedOutcomeRunning})
if _, err := js.Publish(ctx, meshwants.SubjectPipelineWanted("main", "12"), wantedRaw); err != nil {
t.Fatal(err)
}
untilKey(t, ctx, pins, index.PinKey("main", 11), absent)
untilRevisionGone(t, ctx, js, "main.11")
untilKey(t, ctx, pins, index.PinKey("main", r(11)), absent)
untilRevisionGone(t, ctx, js, rn(11))
if n := len(heard.pending); n != 0 {
t.Errorf("take-in.pending said %d more times for one held source revision, want once", n)
}
@@ -36,13 +36,13 @@ func TestACatalogRevisionIsKeptPerWatchedBranch(t *testing.T) {
for _, n := range names {
have[n] = true
}
for _, want := range []string{"main.5", "feature/x=2ey.7", "feature/x=2ey.8"} {
for _, want := range []string{"main.1", "feature/x=2ey.1", "feature/x=2ey.2"} {
if !have[want] {
t.Errorf("the objects hold %v, want %s", names, want)
}
}
if len(said) != 3 || said[0].Branch != "main" || said[1].Branch != "feature/x=2ey" || said[2].Revision != 8 {
t.Errorf("said %+v, want main 5, then the escaped branch 7 and 8", said)
if len(said) != 3 || said[0].Branch != "main" || said[1].Branch != "feature/x=2ey" || said[2].Revision != 2 || said[2].Source != 8 {
t.Errorf("said %+v, want main 1 of source 5, then the escaped branch 1 and 2 of sources 7 and 8", said)
}
answer := func(branch string, revision int64) meshcatalog.RevisionAnswer {
a, err := Revision(ctx, ix.Revisions, nil, meshcatalog.RevisionInput{Branch: branch, Revision: revision})
@@ -51,14 +51,14 @@ func TestACatalogRevisionIsKeptPerWatchedBranch(t *testing.T) {
}
return a
}
if a := answer("main", 0); a.Revision.Revision != 5 || a.Entries[0].Commit != "c1" {
if a := answer("main", 0); a.Revision.Revision != 1 || a.Revision.Source != 5 || a.Entries[0].Commit != "c1" {
t.Errorf("main answered %+v", a)
}
if a := answer("feature/x=2ey", 0); a.Revision.Revision != 8 || a.Revision.Branch != "feature/x.y" || a.Entries[0].Commit != "c3" {
t.Errorf("the branch's newest answered %+v, want its revision 8 at c3", a.Revision)
if a := answer("feature/x=2ey", 0); a.Revision.Revision != 2 || a.Revision.Branch != "feature/x.y" || a.Entries[0].Commit != "c3" {
t.Errorf("the branch's newest answered %+v, want its revision 2 at c3", a.Revision)
}
if a := answer("feature/x=2ey", 7); a.Revision.Revision != 7 || a.Entries[0].Commit != "c2" {
t.Errorf("the branch's revision 7 answered %+v", a.Revision)
if a := answer("feature/x=2ey", 1); a.Revision.Revision != 1 || a.Entries[0].Commit != "c2" {
t.Errorf("the branch's revision 1 answered %+v", a.Revision)
}
if a := answer("beta", 0); !a.NoneYet {
t.Errorf("a branch with no revision answered %+v, want none-yet", a)
@@ -68,13 +68,13 @@ func TestACatalogRevisionIsKeptPerWatchedBranch(t *testing.T) {
bySource := func(branch string, source int64) (meshcatalog.RevisionAnswer, error) {
return Revision(ctx, ix.Revisions, nil, meshcatalog.RevisionInput{Branch: branch, Source: source})
}
if a, err := bySource("main", 5); err != nil || a.Revision.Revision != 5 || a.Entries[0].Commit != "c1" {
if a, err := bySource("main", 5); err != nil || a.Revision.Revision != 1 || a.Revision.Source != 5 || a.Entries[0].Commit != "c1" {
t.Errorf("main's source 5 answered %+v (%v)", a.Revision, err)
}
if a, err := bySource("feature/x=2ey", 7); err != nil || a.Revision.Revision != 7 || a.Entries[0].Commit != "c2" {
t.Errorf("the branch's source 7 answered %+v (%v), want its revision 7 at c2", a.Revision, err)
if a, err := bySource("feature/x=2ey", 7); err != nil || a.Revision.Revision != 1 || a.Entries[0].Commit != "c2" {
t.Errorf("the branch's source 7 answered %+v (%v), want its revision 1 at c2", a.Revision, err)
}
if a, err := bySource("feature/x=2ey", 8); err != nil || a.Revision.Revision != 8 {
if a, err := bySource("feature/x=2ey", 8); err != nil || a.Revision.Revision != 2 {
t.Errorf("the branch's source 8 answered %+v (%v)", a.Revision, err)
}
if _, err := bySource("feature/x=2ey", 9); err == nil {
@@ -5,6 +5,7 @@ import (
"encoding/json"
"errors"
"fmt"
"maps"
"slices"
"strconv"
"strings"
@@ -87,6 +88,37 @@ func BranchRevisionOf(branch, name string) (int64, bool) {
return r, err == nil && r > 0
}
// watchedBranches are the watched branches the bucket keeps a revision of, the first included when it keeps one, sorted:
// each name <branch>.<n> but an entry's, a candidate's and the earlier forms' (earlierForm).
func (ix *Indexer) watchedBranches(ctx context.Context) ([]string, error) {
if ix.Revisions == nil {
return nil, nil
}
names, err := ix.Revisions.Names(ctx)
if err != nil {
return nil, fmt.Errorf("the objects %s: %w", CatalogRevisionsObjects, err)
}
seen := map[string]bool{}
for _, n := range names {
if strings.HasPrefix(n, "entry.") || strings.HasPrefix(n, "candidate.") || earlierForm(n) {
continue
}
i := strings.LastIndexByte(n, '.')
if i <= 0 {
continue
}
if r, err := strconv.ParseInt(n[i+1:], 10, 64); err != nil || r <= 0 {
continue
}
branch, err := keys.UnescapeName(n[:i])
if err != nil {
continue
}
seen[branch] = true
}
return slices.Sorted(maps.Keys(seen)), nil
}
// TrunkRevisionOf is the revision number a trunk object's name holds, and whether it is one.
func TrunkRevisionOf(name string) (int64, bool) { return BranchRevisionOf(TrunkBranch, name) }
@@ -314,17 +346,22 @@ func (ix *Indexer) viewEntries(ctx context.Context) (map[string]meshcatalog.Cata
}
// KeepBranchEntries keeps entries as the catalog revision of a watched branch taken in from its source revision src (design
// 47 H §743, H-A-32): <branch>.<src>, the trunk's included, numbered by the source, and says catalog-revision naming the
// branch. movers are the take-in's: kept on the revision (H-A-92), and the revision pinned for the take-in's pipeline,
// pipeline.<branch>.<src>, in the same write when there is one (A-449a), with the branch's previous revision (H-A-147). A revision already kept under that name is not written
// again (read before write, under the branch's one consumer); ErrObjectsFull is a store too full to keep it.
// 47 H §743, H-A-32), as keepBranchEntries does with no call.
func (ix *Indexer) KeepBranchEntries(ctx context.Context, branch string, src int64, entries map[string]meshcatalog.CatalogEntry, movers []meshcatalog.TakeInEntry) error {
return ix.keepBranchEntries(ctx, branch, src, entries, movers)
_, err := ix.keepBranchEntries(ctx, branch, src, "", entries, movers)
return err
}
func (ix *Indexer) keepBranchEntries(ctx context.Context, branch string, src int64, entries map[string]meshcatalog.CatalogEntry, movers []meshcatalog.TakeInEntry) error {
// keepBranchEntries keeps entries as the next catalog revision of a watched branch, <branch>.<n>, the trunk's included: n is
// the indexer's own count, the branch's newest revision + 1, and src the gate's source revision it was read at, its field
// source (design 47 H-A-213 (1)); call is what made it (H-A-212, H-A-213 (3)): a take-in's the gate's call, a re-pin's
// "release of <module> <version>", a take-in again's "take-in again of <repository>". It says catalog-revision naming the
// branch, its source and its call. movers are kept on the revision (H-A-92), and the revision pinned for its one pipeline,
// pipeline.<branch>.<n>, in the same write when there is one (A-449a, H-A-213 (4)), with the branch's previous revision
// (H-A-147). It answers the number it kept; ErrObjectsFull is a store too full to keep it.
func (ix *Indexer) keepBranchEntries(ctx context.Context, branch string, src int64, call string, entries map[string]meshcatalog.CatalogEntry, movers []meshcatalog.TakeInEntry) (int64, error) {
if ix.Revisions == nil {
return nil
return 0, nil
}
if branch == "" {
branch = TrunkBranch
@@ -334,28 +371,23 @@ func (ix *Indexer) keepBranchEntries(ctx context.Context, branch string, src int
newest, _, err := newestOf(ctx, ix.Revisions, branch)
if errors.Is(err, process.ErrNotYet) {
ix.logf("catalog graph: no revision is kept: its objects %s are not on the bus yet", CatalogRevisionsObjects)
return err
return 0, err
}
if err != nil {
ix.logf("catalog graph: no revision is kept: the newest is not read: %v", err)
return fmt.Errorf("the newest catalog graph revision: %w", err)
}
if _, err := ix.Revisions.Get(ctx, BranchObject(branch, src)); err == nil {
ix.logf("catalog graph: revision %s is kept already: not written again", BranchObject(branch, src))
return nil
} else if !errors.Is(err, ErrNoObject) {
return fmt.Errorf("revision %s: %w", BranchObject(branch, src), err)
return 0, fmt.Errorf("the newest catalog graph revision: %w", err)
}
number := newest.Revision + 1
entries = WithEdges(ix.withActs(ctx, entries)) // the acting rules and the edges between artifact references are written here, once, for every reader (A-208)
at := ix.now().UTC()
rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: src, Kind: "trunk", Branch: branch, Source: src, At: &at}, entries)
rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: number, Kind: "trunk", Branch: branch, Source: src, Call: call, At: &at}, entries)
if err != nil {
return err
return 0, err
}
rev.Movers = movers
// The base of the validate step's case before: the newest revision kept before this write, which the take-in pins for its
// pipeline below, named on the revision in this same write (H-A-147, H-A-151, H-A-154); none on the branch's first take-in.
pinsBase := len(movers) > 0 && ix.Pins != nil && newest.Revision > 0 && newest.Revision != src
pinsBase := len(movers) > 0 && ix.Pins != nil && newest.Revision > 0
if pinsBase {
rev.Base = newest.Revision
}
@@ -367,32 +399,32 @@ func (ix *Indexer) keepBranchEntries(ctx context.Context, branch string, src int
refused := fmt.Sprintf("the next revision; source revision %d held pending with the reason %s", src, HeldObjectsFull)
ix.logf("catalog graph: revision %d of source revision %d is not kept: %s is full (%d of %d bytes)", rev.Revision, src, CatalogRevisionsObjects, used, capacity)
ix.sayFill(ctx, meshcatalog.EventFull, fmt.Sprintf("%d", src), meshcatalog.Fill{Bytes: used, Cap: capacity, Refused: refused})
return fmt.Errorf("revision %d of source revision %d: %w", rev.Revision, src, ErrObjectsFull)
return 0, fmt.Errorf("revision %d of source revision %d: %w", rev.Revision, src, ErrObjectsFull)
}
if err := ix.indexEntries(ctx, rev, entries); err != nil {
ix.logf("catalog graph: revision %d of source revision %d is not kept: %v", rev.Revision, src, err)
return err
return 0, err
}
if err := KeepRevision(ctx, ix.Revisions, rev, objects); err != nil {
ix.logf("catalog graph: revision %d of source revision %d is not kept: %v", rev.Revision, src, err)
return err
return 0, err
}
// The take-in's revision pinned for its one pipeline before any judge can remove it (§743, §749, A-449a).
if len(movers) > 0 && ix.Pins != nil {
record := PipelineRecord(runner.PipelinesByBranchSourceKey(keys.EscapeName(branch), strconv.FormatInt(src, 10)))
if err := ix.pin(ctx, PinKey(branch, src), record); err != nil {
ix.logf("pins: the revision %s is not pinned for %s: %v", BranchObject(branch, src), record, err)
return fmt.Errorf("the pin of %s: %w", BranchObject(branch, src), err)
record := PipelineRecord(runner.PipelinesByBranchSourceKey(keys.EscapeName(branch), strconv.FormatInt(number, 10)))
if err := ix.pin(ctx, PinKey(branch, number), record); err != nil {
ix.logf("pins: the revision %s is not pinned for %s: %v", BranchObject(branch, number), record, err)
return 0, fmt.Errorf("the pin of %s: %w", BranchObject(branch, number), err)
}
// The branch's previous revision, the newest kept before this write, pinned for the same pipeline: it is the base of
// the validate step's case before (H-A-108 (3), H-A-147), kept as long as the pipeline that needs it.
if pinsBase {
if err := ix.pin(ctx, PinKey(branch, rev.Base), record); err != nil {
ix.logf("pins: the previous revision %s is not pinned for %s: %v", BranchObject(branch, newest.Revision), record, err)
return fmt.Errorf("the pin of %s: %w", BranchObject(branch, newest.Revision), err)
return 0, fmt.Errorf("the pin of %s: %w", BranchObject(branch, newest.Revision), err)
}
} else {
ix.logf("pins: %s has no revision before it on its branch: none is pinned for %s as its validate base", BranchObject(branch, src), record)
ix.logf("pins: %s has no revision before it on its branch: none is pinned for %s as its validate base", BranchObject(branch, number), record)
}
}
// The revision before this one in its branch ceases to be current (§579): it goes unless a record pins it.
@@ -412,14 +444,14 @@ func (ix *Indexer) keepBranchEntries(ctx context.Context, branch string, src int
if after, capacity, err := ix.Revisions.Fill(ctx); fillErr == nil && err == nil && capacity > 0 && used*5 < capacity*4 && after*5 >= capacity*4 {
ix.sayFill(ctx, meshcatalog.EventFilling, fmt.Sprintf("%d", rev.Revision), meshcatalog.Fill{Bytes: after, Cap: capacity})
}
ix.logf("catalog graph: revision %d of source revision %d kept, %d entries, %d changed", rev.Revision, src, len(rev.Entries), len(changed))
ix.logf("catalog graph: revision %d of source revision %d kept (%s), %d entries, %d changed", rev.Revision, src, call, len(rev.Entries), len(changed))
if ix.SayCatalogRevision != nil {
if err := ix.SayCatalogRevision(ctx, meshcatalog.CatalogRevisionEvent{V: CatalogEntryLayout, Revision: rev.Revision, Kind: rev.Kind,
Branch: keys.EscapeName(branch), Source: src, At: &at, Changed: changed, Entries: int64(len(rev.Entries))}); err != nil {
Branch: keys.EscapeName(branch), Source: src, Call: call, At: &at, Changed: changed, Entries: int64(len(rev.Entries))}); err != nil {
ix.logf("catalog graph: revision %d is not said: %v", rev.Revision, err)
}
}
return nil
return number, nil
}
// HeldObjectsFull is the reason a pending merge is held while the catalog graph revisions are full (Pending.Held).
@@ -49,7 +49,7 @@ func TestAKeptRevisionIndexesItsEntries(t *testing.T) {
ctx := context.Background()
fp := "src2:" + strings.Repeat("d", 64)
ix := &Indexer{Module: testSelf, Revisions: newMemObjects(1 << 20), EntryIndex: newMem(), Now: func() time.Time { return time.Date(2026, 11, 1, 3, 0, 0, 0, time.UTC) }}
if err := ix.keepBranchEntries(ctx, TrunkBranch, 1, map[string]meshcatalog.CatalogEntry{"avahi": {Module: "avahi", Repository: "novox/r", Commit: "c", Fp: fp, Manifest: map[string]any{}}}, nil); err != nil {
if err := ix.KeepBranchEntries(ctx, TrunkBranch, 1, map[string]meshcatalog.CatalogEntry{"avahi": {Module: "avahi", Repository: "novox/r", Commit: "c", Fp: fp, Manifest: map[string]any{}}}, nil); err != nil {
t.Fatal(err)
}
if got, err := Entry(ctx, ix.Revisions, ix.EntryIndex, meshcatalog.EntryInput{Fingerprint: fp}); err != nil || got.Entry.Module != "avahi" {
@@ -122,25 +122,25 @@ func TestTheIndexerRemovesARevisionOnlyWhenNoRecordPinsItAndItIsNotCurrent(t *te
}
ix.JudgePins(ctx)
}
take(10)
end(10, runner.PipelineEndedOutcomePassed, "0.1.10")
take(11)
end(11, runner.PipelineEndedOutcomeFailed, "")
want("a passed release past a later failed pipeline", "main.10", "main.11")
take(12)
end(12, runner.PipelineEndedOutcomePassed, "")
wanted(12, meshwants.PipelineWantedOutcomeRunning)
want("a passed pipeline releasing no version frees no older release", "main.10", "main.12")
take(13)
end(13, runner.PipelineEndedOutcomePassed, "0.1.13")
want("two releases, the unreleased one between them gone", "main.10", "main.13")
wanted(13, meshwants.PipelineWantedOutcomeTierLate) // H-A-174: its later tiers' wants are not written, so it frees nothing
want("a tier-late pipeline-wanted", "main.10", "main.13")
wanted(13, meshwants.PipelineWantedOutcomeRunning)
wanted(13, meshwants.PipelineWantedOutcomeRunning) // heard twice: changes nothing more
want("the newer release written everywhere", "main.13")
if v, _, _, _ := ix.Pins.Get(ctx, "main.13"); !slices.Equal(mustRecords(t, v), []string{ReleaseRecord("main", "reader")}) {
t.Errorf("the pin of main.13 holds %s, want its release alone", v)
take(4)
end(4, runner.PipelineEndedOutcomePassed, "0.1.10")
take(5)
end(5, runner.PipelineEndedOutcomeFailed, "")
want("a passed release past a later failed pipeline", "main.4", "main.5")
take(6)
end(6, runner.PipelineEndedOutcomePassed, "")
wanted(6, meshwants.PipelineWantedOutcomeRunning)
want("a passed pipeline releasing no version frees no older release", "main.4", "main.6")
take(7)
end(7, runner.PipelineEndedOutcomePassed, "0.1.13")
want("two releases, the unreleased one between them gone", "main.4", "main.7")
wanted(7, meshwants.PipelineWantedOutcomeTierLate) // H-A-174: its later tiers' wants are not written, so it frees nothing
want("a tier-late pipeline-wanted", "main.4", "main.7")
wanted(7, meshwants.PipelineWantedOutcomeRunning)
wanted(7, meshwants.PipelineWantedOutcomeRunning) // heard twice: changes nothing more
want("the newer release written everywhere", "main.7")
if v, _, _, _ := ix.Pins.Get(ctx, "main.7"); !slices.Equal(mustRecords(t, v), []string{ReleaseRecord("main", "reader")}) {
t.Errorf("the pin of main.7 holds %s, want its release alone", v)
}
}
@@ -168,21 +168,22 @@ func TestATakeInPinsTheBranchsPreviousRevisionForItsPipeline(t *testing.T) {
before []int64 // sources taken in with no mover before the take-in of source 5
whileRuns []string
afterTheEnd []string
base int64 // the base main.5 names (H-A-154)
at int64 // the number the take-in of source 5 keeps: the branch's newest + 1 (H-A-213 (1))
base int64 // the base that revision names (H-A-154)
passed bool // main.5's pipeline passed: its release pins main.5
unreleased bool // its mover released no version there, so nothing pins main.5 (H-A-173)
then []int64 // sources taken in with no mover after the end
released []string
}{
{name: "the previous revision kept and pinned", before: []int64{3, 4}, whileRuns: []string{"main.4", "main.5"}, afterTheEnd: []string{"main.5"}, base: 4},
{name: "the newest kept before, whatever its number", before: []int64{2}, whileRuns: []string{"main.2", "main.5"}, afterTheEnd: []string{"main.5"}, base: 2},
{name: "no revision before it pins only its own", whileRuns: []string{"main.5"}, afterTheEnd: []string{"main.5"}},
{name: "a passed pipeline's release keeps its revision once no longer current", before: []int64{4}, whileRuns: []string{"main.4", "main.5"},
base: 4, passed: true, then: []int64{6}, afterTheEnd: []string{"main.5", "main.6"}, released: []string{"main.5"}},
{name: "a passed pipeline whose mover released no version pins nothing", before: []int64{4}, whileRuns: []string{"main.4", "main.5"},
base: 4, passed: true, unreleased: true, then: []int64{6}, afterTheEnd: []string{"main.6"}},
{name: "a failed pipeline's revision goes once no longer current", before: []int64{4}, whileRuns: []string{"main.4", "main.5"},
base: 4, then: []int64{6}, afterTheEnd: []string{"main.6"}},
{name: "the previous revision kept and pinned", before: []int64{3, 4}, at: 3, whileRuns: []string{"main.2", "main.3"}, afterTheEnd: []string{"main.3"}, base: 2},
{name: "the newest kept before, whatever its source", before: []int64{2}, at: 2, whileRuns: []string{"main.1", "main.2"}, afterTheEnd: []string{"main.2"}, base: 1},
{name: "no revision before it pins only its own", at: 1, whileRuns: []string{"main.1"}, afterTheEnd: []string{"main.1"}},
{name: "a passed pipeline's release keeps its revision once no longer current", before: []int64{4}, at: 2, whileRuns: []string{"main.1", "main.2"},
base: 1, passed: true, then: []int64{6}, afterTheEnd: []string{"main.2", "main.3"}, released: []string{"main.2"}},
{name: "a passed pipeline whose mover released no version pins nothing", before: []int64{4}, at: 2, whileRuns: []string{"main.1", "main.2"},
base: 1, passed: true, unreleased: true, then: []int64{6}, afterTheEnd: []string{"main.3"}},
{name: "a failed pipeline's revision goes once no longer current", before: []int64{4}, at: 2, whileRuns: []string{"main.1", "main.2"},
base: 1, then: []int64{6}, afterTheEnd: []string{"main.3"}},
} {
t.Run(tc.name, func(t *testing.T) {
ix := &Indexer{Module: testSelf, Revisions: newMemObjects(1 << 20), Pins: newMem(), Findings: newFindings(),
@@ -206,18 +207,19 @@ func TestATakeInPinsTheBranchsPreviousRevisionForItsPipeline(t *testing.T) {
keep(s, nil)
}
keep(5, mover)
if rev, _, err := ReadRevision(ctx, ix.Revisions, "main.5"); err != nil || rev.Base != tc.base {
t.Errorf("main.5 names the base %d (%v), want %d, the revision its take-in pinned (H-A-154)", rev.Base, err, tc.base)
at := fmt.Sprintf("main.%d", tc.at)
if rev, _, err := ReadRevision(ctx, ix.Revisions, at); err != nil || rev.Base != tc.base || rev.Source != 5 {
t.Errorf("%s names the source %d and the base %d (%v), want 5 and %d, the revision its take-in pinned (H-A-154)", at, rev.Source, rev.Base, err, tc.base)
}
if got := kept(); !slices.Equal(got, tc.whileRuns) {
t.Errorf("while the pipeline runs: kept %v, want %v", got, tc.whileRuns)
}
for _, k := range tc.whileRuns {
if v, _, _, _ := ix.Pins.Get(ctx, k); !slices.Equal(mustRecords(t, v), []string{"pipeline.main.5"}) {
if v, _, _, _ := ix.Pins.Get(ctx, k); !slices.Equal(mustRecords(t, v), []string{"pipeline." + at}) {
t.Errorf("the pin of %s holds %s, want the pipeline's record", k, v)
}
}
end := runner.PipelineEnded{Branch: "main", Pipeline: "main.5", Revision: 5, Outcome: runner.PipelineEndedOutcomeFailed,
end := runner.PipelineEnded{Branch: "main", Pipeline: at, Revision: tc.at, Outcome: runner.PipelineEndedOutcomeFailed,
Movers: []runner.MoverEnded{{Module: "reader", Fingerprint: "f", Version: "0.1.5"}}}
if tc.unreleased {
end.Movers[0].Version = ""
@@ -37,11 +37,19 @@ func TestAnEntryNamesEveryRequirementPinnedOrWaiting(t *testing.T) {
// Design 47 H-A-204, H-A-165: a release re-pins every kept entry naming one of its packages, of any repository and
// changed or not, by its pins or by its kept requirements alone: here g, of another repository, holds no pin of mesh-sdk.
// H-A-211, H-A-212, H-A-213: on every watched branch holding g, the re-pin keeps the branch's next revision, of the same
// source as the one before it, with the call "release of mesh-sdk 0.1.27" and g as its mover at its new fingerprint, and
// says catalog-revision with that call.
func TestAReleaseRepinsAnEntryNamingItsPackageWithNoPin(t *testing.T) {
ctx := context.Background()
objects := map[string]string{"modules/b/builds": strings.Repeat("b", 40), "modules/c/client": strings.Repeat("c", 40)}
ix, mods, _ := ptpIndexer(t, objects)
ix.Revisions, ix.Packages = newMemObjects(1<<20), newMem()
reportToolchain(t, ix, "mesh-build-go", "n1", "go", "1.26.0")
var said []meshcatalog.CatalogRevisionEvent
ix.SayCatalogRevision = func(_ context.Context, e meshcatalog.CatalogRevisionEvent) error { said = append(said, e); return nil }
f := withManifests(mergedMsg("novox/mesh-delivery", map[string]string{"modules/g": "gtree"}), map[string]string{"modules/g": ptpProgram}, nil)
if err := ix.Apply(context.Background(), f); err != nil {
if err := ix.Apply(ctx, f); err != nil {
t.Fatal(err)
}
g, _ := moduleOf(t, mods, "g")
@@ -52,7 +60,39 @@ func TestAReleaseRepinsAnEntryNamingItsPackageWithNoPin(t *testing.T) {
mods.put("g", mustEncode(t, g))
mods.put("mesh-sdk", mustEncode(t, meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-sdk", Commit: strings.Repeat("1", 40),
Path: meshcatalogclient.RootPath, None2: None2Pending}))
if n, err := ix.Repin(context.Background(), "mesh-sdk"); err != nil || n != 1 {
t.Errorf("a release of mesh-sdk re-pinned %d modules (%v), want g, which requires it with no pin", n, err)
entries, err := ix.viewEntries(ctx)
if err != nil {
t.Fatal(err)
}
sources := map[string]int64{"main": 5, "feature/x": 9}
for branch, src := range sources {
if err := ix.KeepBranchEntries(ctx, branch, src, entries, nil); err != nil {
t.Fatal(err)
}
}
said = nil
if _, err := ix.HearVersion(ctx, HeardVersion{Package: "mesh-sdk/go", GoModule: "git.novox.be/novox/mesh-sdk/go", Version: "0.1.27", Published: true}); err != nil {
t.Fatal(err)
}
if n, err := ix.Repin(ctx, "mesh-sdk", "0.1.27"); err != nil || n != 2 {
t.Errorf("a release of mesh-sdk re-pinned %d modules (%v), want g on each of the two branches, which requires it with no pin", n, err)
}
now, _ := moduleOf(t, mods, "g")
if now.Fp2 == "" || now.Fp2 == g.Fp2 || now.Packages["git.novox.be/novox/mesh-sdk/go"] != "v0.1.27" {
t.Fatalf("g after the re-pin = %s pinned %v, want a new fingerprint pinned to v0.1.27 (was %s)", now.Fp2, now.Packages, g.Fp2)
}
for branch, src := range sources {
rev, kept, found, err := readNewestOf(ctx, ix.Revisions, branch)
if err != nil || !found || rev.Revision != 2 || rev.Source != src || rev.Call != "release of mesh-sdk 0.1.27" ||
len(rev.Movers) != 1 || rev.Movers[0].Module != "g" || rev.Movers[0].Fingerprint != now.Fp2 {
t.Errorf("%s's newest = %+v (%v), want revision 2 of source %d, its call the release, g its mover at %s", branch, rev, err, src, now.Fp2)
continue
}
if i := slices.IndexFunc(kept, func(e meshcatalog.CatalogEntry) bool { return e.Module == "g" }); i < 0 || kept[i].Fp != now.Fp2 {
t.Errorf("%s's newest holds no entry of g at %s", branch, now.Fp2)
}
}
if len(said) != 2 || said[0].Call != "release of mesh-sdk 0.1.27" || said[1].Call != said[0].Call {
t.Errorf("said %+v, want catalog-revision of each branch with the release's call", said)
}
}
@@ -2,9 +2,11 @@ package index
import (
"context"
"encoding/json"
"errors"
"fmt"
"maps"
"slices"
"strconv"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
@@ -25,9 +27,10 @@ func TakenID(branch string, source int64) string {
// TakeInSource takes one source revision of a watched branch in, as the gate's source-moved names it (design 47 H §743): for each
// repository it moves, the trees at the named commit through the seat git (wholeAt) and the fingerprints of the modules
// registered for it; its movers (moversOf) against the branch's newest catalog revision; one catalog revision of the branch,
// <branch>.<source>, carrying the movers and pinned for the take-in's pipeline when it has one (KeepBranchEntries); and
// the branch's next, <branch>.<n> of source <source>, carrying the movers and pinned for the take-in's pipeline when it has one
// (keepBranchEntries, H-A-213); and
// take-in.taken once, with branch, source, revision, call and one entry per mover; a branch's first revision (start) has no
// mover (§768). A revision already kept under that name (a
// mover (§768). A revision already kept of that source (a
// restart before the acknowledgement) is said again from its kept movers, recomputing nothing (H-A-92). The first watched
// branch is taken in on the indexer's own view; another on its newest revision's, computed and kept only as that revision.
// An error leaves the source-moved held, written into the state held behind its branch's earlier ones (ErrHeldUnwritten when
@@ -57,13 +60,13 @@ func (ix *Indexer) takeInSource(ctx context.Context, e meshdelivery.SourceMovedE
}
ix.pendingMu.Lock()
defer ix.pendingMu.Unlock()
name := BranchObject(e.Branch, e.Source)
if kept, _, err := ReadRevision(ctx, ix.Revisions, name); err == nil {
taken.Revision, taken.Entries = kept.Revision, kept.Movers
ix.logf("take-in: %s is kept already: take-in.taken said again from its %d movers", name, len(kept.Movers))
return taken, ix.sayTaken(ctx, taken)
} else if !errors.Is(err, ErrNoObject) {
// A source revision kept already is found by the revision's field source, never by its number (H-A-213 (1)).
if kept, found, err := takenOfSource(ctx, ix.Revisions, e.Branch, e.Source); err != nil {
return taken, err
} else if found {
taken.Revision, taken.Entries = kept.Revision, kept.Movers
ix.logf("take-in: source revision %d of %s is kept already as %s: take-in.taken said again from its %d movers", e.Source, e.Branch, RevisionObject(kept), len(kept.Movers))
return taken, ix.sayTaken(ctx, taken)
}
_, before, _, err := readNewestOf(ctx, ix.Revisions, e.Branch)
if err != nil {
@@ -106,14 +109,15 @@ func (ix *Indexer) takeInSource(ctx context.Context, e meshdelivery.SourceMovedE
if e.Start {
movers = nil // a branch's first revision moves nothing: its entries are the base later take-ins are held against (§768, H-A-19)
}
if err := ix.KeepBranchEntries(ctx, e.Branch, e.Source, entries, movers); err != nil {
number, err := ix.keepBranchEntries(ctx, e.Branch, e.Source, e.Call, entries, movers)
if err != nil {
if errors.Is(err, ErrObjectsFull) {
ix.heldFull.Store(true) // room is said once the store has room again (RoomAgain), and wakes the take-in (§744)
}
return taken, err
}
taken.Revision, taken.Entries = e.Source, movers
ix.logf("take-in: source revision %d of %s (call %s) taken in as %s, %d repositories, %d movers", e.Source, e.Branch, e.Call, name, len(e.Heads), len(movers))
taken.Revision, taken.Entries = number, movers
ix.logf("take-in: source revision %d of %s (call %s) taken in as %s, %d repositories, %d movers", e.Source, e.Branch, e.Call, BranchObject(e.Branch, number), len(e.Heads), len(movers))
return taken, ix.sayTaken(ctx, taken)
}
@@ -157,22 +161,57 @@ func (ix *Indexer) SourceToTake(ctx context.Context, branch string, a meshdelive
if ix.Revisions == nil {
return e, false, errors.New("the indexer keeps no catalog revisions")
}
names, err := ix.Revisions.Names(ctx)
// The gate's source a branch is taken in at is its newest revision's field source, never the revision's number (H-A-213 (1)).
newest, found, err := newestOf(ctx, ix.Revisions, branch)
if err != nil {
return e, false, fmt.Errorf("the objects %s: %w", CatalogRevisionsObjects, err)
return e, false, fmt.Errorf("the newest catalog revision of %s: %w", branch, err)
}
var kept int64
for _, n := range names {
if r, ok := BranchRevisionOf(branch, n); ok && r > kept {
kept = r
}
}
if kept >= a.Revision {
if found && newest.Source >= a.Revision {
return e, false, nil
}
e.Start = kept == 0
e.Start = !found
for _, r := range a.Repositories {
e.Heads = append(e.Heads, meshdelivery.SourceMovedEventHeadsItem{Repository: r.Repository, Commit: r.Head})
}
return e, true, nil
}
// takenOfSource is the revision a take-in of source src kept on branch: of the branch's revisions whose field source is src,
// the lowest numbered (a re-pin or a take-in again keeps the same source on a later number, H-A-211, H-A-213). The revisions
// are read from the newest down while their source is not below src; none is false.
func takenOfSource(ctx context.Context, o Objects, branch string, src int64) (meshcatalog.CatalogRevision, bool, error) {
names, err := o.Names(ctx)
if err != nil {
return meshcatalog.CatalogRevision{}, false, fmt.Errorf("the objects %s: %w", CatalogRevisionsObjects, err)
}
var numbers []int64
for _, n := range names {
if r, ok := BranchRevisionOf(branch, n); ok {
numbers = append(numbers, r)
}
}
slices.Sort(numbers)
var kept meshcatalog.CatalogRevision
found := false
for i := len(numbers) - 1; i >= 0; i-- {
name := BranchObject(branch, numbers[i])
data, err := o.Get(ctx, name)
if errors.Is(err, ErrNoObject) {
continue // removed under the read by the pin judge
}
if err != nil {
return kept, false, fmt.Errorf("revision %s: %w", name, err)
}
var rev meshcatalog.CatalogRevision
if err := json.Unmarshal(data, &rev); err != nil {
return kept, false, fmt.Errorf("%s: %w", name, err)
}
if rev.Source < src {
break
}
if rev.Source == src {
kept, found = rev, true
}
}
return kept, found, nil
}
@@ -43,7 +43,8 @@ func takeInIndexer(t *testing.T) (*Indexer, *memObjects, *[]meshcatalog.TakeInTa
return ix, objects, said, f
}
// Design 47 H §743, H-A-32, H-A-92: a take-in writes one catalog revision <branch>.<source> carrying its movers, pins it for the
// Design 47 H §743, H-A-32, H-A-92, H-A-213 (1): a take-in writes one catalog revision <branch>.<n>, n the branch's newest + 1
// and its source the gate's, carrying its movers, pins it for the
// take-in's pipeline and says take-in.taken once; a redelivery writes nothing again and says the same; a revision with no mover
// pins nothing and says no entries; the old form is refused.
func TestASourceRevisionIsTakenInAsOneRevision(t *testing.T) {
@@ -56,19 +57,19 @@ func TestASourceRevisionIsTakenInAsOneRevision(t *testing.T) {
t.Fatal(err)
}
if len(taken.Entries) != 1 || taken.Entries[0].Module != "avahi" || taken.Entries[0].Fingerprint == "" || taken.Entries[0].Commit != f.Merge ||
taken.Revision != 7 || taken.Branch != "main" || taken.Source != 7 || taken.Call != moved.Call {
t.Fatalf("taken %+v, want revision 7 of main with avahi as its one mover", taken)
taken.Revision != 1 || taken.Branch != "main" || taken.Source != 7 || taken.Call != moved.Call {
t.Fatalf("taken %+v, want revision 1 (the branch's first) of source 7 of main with avahi as its one mover", taken)
}
rev, _, err := ReadRevision(ctx, objects, "main.7")
rev, _, err := ReadRevision(ctx, objects, "main.1")
if err != nil || !reflect.DeepEqual(rev.Movers, taken.Entries) || rev.Source != 7 {
t.Fatalf("main.7 = %+v (%v), want it kept carrying the movers", rev, err)
t.Fatalf("main.1 = %+v (%v), want it kept carrying the movers", rev, err)
}
if v, _, found, _ := ix.Pins.Get(ctx, PinKey("main", 7)); !found {
t.Error("main.7 is not pinned for its pipeline")
if v, _, found, _ := ix.Pins.Get(ctx, PinKey("main", 1)); !found {
t.Error("main.1 is not pinned for its pipeline")
} else {
var pv PinValue
if json.Unmarshal(v, &pv) != nil || !reflect.DeepEqual(pv.Records, []string{"pipeline.main.7"}) {
t.Errorf("main.7 is pinned by %s, want pipeline.main.7", v)
if json.Unmarshal(v, &pv) != nil || !reflect.DeepEqual(pv.Records, []string{"pipeline.main.1"}) {
t.Errorf("main.1 is pinned by %s, want pipeline.main.1", v)
}
}
puts := objects.puts
@@ -79,14 +80,14 @@ func TestASourceRevisionIsTakenInAsOneRevision(t *testing.T) {
same := moved
same.Source, same.Call = 8, "fedcba9876543210"
none, err := ix.TakeInSource(ctx, same)
if err != nil || len(none.Entries) != 0 || none.Revision != 8 {
t.Errorf("a revision moving no fingerprint said %+v (%v), want revision 8 with no entries", none, err)
if err != nil || len(none.Entries) != 0 || none.Revision != 2 || none.Source != 8 {
t.Errorf("a revision moving no fingerprint said %+v (%v), want revision 2 of source 8 with no entries", none, err)
}
if _, _, found, _ := ix.Pins.Get(ctx, PinKey("main", 8)); found {
if _, _, found, _ := ix.Pins.Get(ctx, PinKey("main", 2)); found {
t.Error("a take-in with no mover pinned its revision")
}
if _, _, err := ReadRevision(ctx, objects, "main.8"); err != nil {
t.Errorf("main.8 is not kept: %v", err)
if _, _, err := ReadRevision(ctx, objects, "main.2"); err != nil {
t.Errorf("main.2 is not kept: %v", err)
}
if len(*said) != 3 {
t.Errorf("said %d take-in.taken, want 3", len(*said))
@@ -598,19 +598,70 @@ func importsOf(pkgs sdkstep.PackageVersions, repository string) bool {
return false
}
// ReleaseCall is the call of the catalog revision a re-pin keeps: the release of a module at a version (design 47 H-A-213 (2)).
func ReleaseCall(module, version string) string { return "release of " + module + " " + version }
// Repin moves the pins of every module whose packages hold a package of module (a held module's directory holds it) to
// module's newest released version, the pass of H-A-161 for those modules alone: a version-released of module starts it
// (design 47 H-A-165), so pins move on the event and never wait for a merge or a hand act. It answers how many it named.
func (ix *Indexer) Repin(ctx context.Context, module string) (int, error) {
// module's newest released version, the pass of H-A-161 for those modules alone: a release of module at version starts it
// (design 47 H-A-165), so pins move on the event and never wait for a merge or a hand act. It runs on every watched branch
// holding such a module: the first on the indexer's own modules state, another on its newest revision's picture (as a
// take-in does), and keeps on each whose re-pinned fingerprints moved the branch's next catalog revision, of the same
// source as its newest, with the call ReleaseCall and those modules as its movers, pinned for the release's pipeline, and
// says catalog-revision; no take-in.taken (H-A-211, H-A-212, H-A-213). It answers how many modules it named.
func (ix *Indexer) Repin(ctx context.Context, module, version string) (int, error) {
if ix.Modules == nil || ix.Manifests == nil {
return 0, nil
}
held, err := ix.heldModules(ctx)
ix.pendingMu.Lock() // one take-in step at a time, so the branch's newest stays the one re-pinned (pending.go)
defer ix.pendingMu.Unlock()
branches, err := ix.watchedBranches(ctx)
if err != nil {
return 0, err
}
// Every kept entry naming a package of module is re-pinned, of any repository, changed or not: by its pins, and by its
// kept requirements, which name a package it holds no pin of yet (design 47 H-A-204, H-A-165, H-A-167).
if !slices.Contains(branches, TrunkBranch) {
branches = append([]string{TrunkBranch}, branches...) // the modules state is re-pinned whether a revision is kept or not
}
named := 0
for _, branch := range branches {
x := ix
if !isTrunkBranch(branch) {
view, nodes, _, err := BranchView(ctx, ix.Revisions, branch)
if err != nil {
return named, err
}
d := ix.dry()
d.ix.Modules, d.ix.Graph, d.ix.Taken = view, nodes, newOverlay(nil)
d.ix.Logf = func(format string, args ...any) { ix.logf("re-pin of "+branch+": "+format, args...) }
x = d.ix
}
only, err := x.repinned(ctx, module)
if err != nil {
return named, err
}
if len(only) == 0 {
continue
}
named += len(only)
x.recompute.Lock()
err = x.recomputeLocked(ctx, nil, nil, "", only)
x.recompute.Unlock()
if err != nil {
return named, err
}
if err := ix.keepRepinned(ctx, x, branch, ReleaseCall(module, version), only); err != nil {
return named, err
}
}
return named, nil
}
// repinned are the modules the indexer's modules state holds whose packages name a package of module: by their pins, and
// by their kept requirements, which name a package they hold no pin of yet (design 47 H-A-204, H-A-165, H-A-167).
func (ix *Indexer) repinned(ctx context.Context, module string) (map[string]bool, error) {
held, err := ix.heldModules(ctx)
if err != nil {
return nil, err
}
only := map[string]bool{}
for key, m := range held {
names := slices.Collect(maps.Keys(m.Packages))
@@ -623,10 +674,48 @@ func (ix *Indexer) Repin(ctx context.Context, module string) (int, error) {
}
}
}
if len(only) == 0 {
return 0, nil
return only, nil
}
// keepRepinned keeps x's view, re-pinned for the modules only names, as branch's next catalog revision of the source of its
// newest, with call, when a fingerprint of them moved against that newest; its movers are those that moved (H-A-211,
// H-A-212, H-A-213). A branch with no revision kept yet keeps none: there is no source to keep it of.
func (ix *Indexer) keepRepinned(ctx context.Context, x *Indexer, branch, call string, only map[string]bool) error {
if ix.Revisions == nil {
return nil
}
ix.recompute.Lock()
defer ix.recompute.Unlock()
return len(only), ix.recomputeLocked(ctx, nil, nil, "", only)
newest, before, found, err := readNewestOf(ctx, ix.Revisions, branch)
if err != nil {
return fmt.Errorf("the newest catalog revision of %s: %w", branch, err)
}
if !found {
ix.logf("re-pin (%s): %s keeps no revision yet, so none is kept for it", call, branch)
return nil
}
entries, err := x.viewEntries(ctx)
if err != nil {
return err
}
was := make(map[string]string, len(before))
for _, e := range before {
was[e.Module] = e.Fp
}
var movers []meshcatalog.TakeInEntry
for _, key := range slices.Sorted(maps.Keys(only)) {
e, ok := entries[key]
if !ok || e.Fp == "" || was[e.Module] == e.Fp {
continue
}
movers = append(movers, meshcatalog.TakeInEntry{Module: e.Module, Commit: e.Commit, Fingerprint: e.Fp, Repository: e.Repository})
}
if len(movers) == 0 {
ix.logf("re-pin (%s): no fingerprint of %s moved, so no revision is kept", call, branch)
return nil
}
number, err := ix.keepBranchEntries(ctx, branch, newest.Source, call, entries, movers)
if err != nil {
return err
}
ix.logf("re-pin (%s): %s kept as revision %d of source %d, %d movers", call, branch, number, newest.Source, len(movers))
return nil
}
@@ -504,7 +504,7 @@ func TestBasesAreComputedFirstAndNoFingerprintTakesTheBuilder(t *testing.T) { //
}
return first.KeepBranchEntries(context.Background(), TrunkBranch, source, entries, nil)
}
if err := takeIn(6); err != nil || len(firstSaid) != 1 || firstSaid[0].Revision != 6 ||
if err := takeIn(6); err != nil || len(firstSaid) != 1 || firstSaid[0].Revision != 1 ||
firstSaid[0].Entries != int64(len(builderModules(t))) {
t.Fatalf("the take-in said %+v (%v), want trunk revision 1 with every module", firstSaid, err)
}
@@ -519,7 +519,7 @@ func TestBasesAreComputedFirstAndNoFingerprintTakesTheBuilder(t *testing.T) { //
if err := takeIn(7); err != nil || len(firstSaid) != 2 || !reflect.DeepEqual(firstSaid[1].Changed, []string{"avahi"}) {
t.Errorf("the next take-in after a write said %+v (%v), want revision 7 naming avahi", firstSaid, err)
}
if view, _, at, err := TrunkView(context.Background(), first.Revisions); err != nil || at != 7 {
if view, _, at, err := TrunkView(context.Background(), first.Revisions); err != nil || at != 2 {
t.Errorf("the view at a restart is read at %d (%v), want 7", at, err)
} else if raw, _, _, _ := view.Get(context.Background(), viewKey); !strings.Contains(string(raw), "a field the judge refuses") {
t.Errorf("the view at a restart lost the verdict: %s", raw)
@@ -536,21 +536,21 @@ func TestBasesAreComputedFirstAndNoFingerprintTakesTheBuilder(t *testing.T) { //
t.Errorf("the first-view tools %+v serve no modules", FirstViewTools)
}
if a, err := views.FirstView(testSelf, ModulesState, trunkStates.State(ModulesState), trunkStates.State(views.MarksState), "", time.Now()); err != nil ||
a.Revision != 7 || len(a.Values) != len(builderModules(t)) {
t.Errorf("the first view modules answered %d values at %d (%v), want every module at revision 7", len(a.Values), a.Revision, err)
a.Revision != 2 || len(a.Values) != len(builderModules(t)) {
t.Errorf("the first view modules answered %d values at %d (%v), want every module at revision 2", len(a.Values), a.Revision, err)
} else if m, err := meshcatalogclient.DecodeModule(a.Values[viewKey]); err != nil || m.Manifest == "" {
t.Errorf("the first view's avahi = %s (%v), want the module's value naming its manifest", a.Values[viewKey], err)
}
if e, ok, err := trunkStates.State(views.MarksState).Get("graph"); err != nil || !ok || !strings.Contains(string(e.Value), `"revision":7`) {
t.Errorf("the bundle's mark of graph = %+v %v (%v), want revision 7", e, ok, err)
if e, ok, err := trunkStates.State(views.MarksState).Get("graph"); err != nil || !ok || !strings.Contains(string(e.Value), `"revision":2`) {
t.Errorf("the bundle's mark of graph = %+v %v (%v), want revision 2", e, ok, err)
}
// A kept view reads the revision once while it stays the newest, and its first view answers in pages of values, never
// one revision read per key (about 11 s a page live, 2026-11-14).
reads := 0
keptView := TrunkStates{Objects: first.Revisions, Under: func(string) views.Source { return nil }, Kept: &KeptTrunk{Say: func(string, ...any) { reads++ }}}
if a, err := views.FirstView(testSelf, ModulesState, keptView.State(ModulesState), keptView.State(views.MarksState), "", time.Now()); err != nil ||
a.Revision != 7 || len(a.Values) != len(builderModules(t)) || a.Next != "" {
t.Errorf("the kept first view modules answered %d values at %d, next %q (%v), want every module at revision 7 in one page", len(a.Values), a.Revision, a.Next, err)
a.Revision != 2 || len(a.Values) != len(builderModules(t)) || a.Next != "" {
t.Errorf("the kept first view modules answered %d values at %d, next %q (%v), want every module at revision 2 in one page", len(a.Values), a.Revision, a.Next, err)
}
if _, _, err := keptView.State("graph").Get("avahi"); err != nil || reads != 1 {
t.Errorf("the kept view read the revision %d times (%v), want once", reads, err)
@@ -822,7 +822,7 @@ func TestAModuleKeepsTheToolchainsItsFingerprintWasComputedWith(t *testing.T) {
if _, err := ix.HearVersion(context.Background(), HeardVersion{Package: "mesh-sdk/go", Seat: "go-package-registry", Version: "0.2.0", Commit: strings.Repeat("1", 40), GoModule: sdk, Published: true}); err != nil {
t.Fatal(err)
}
if _, err := ix.Repin(context.Background(), "mesh-sdk"); err != nil { // the pass for avahi alone: mesh-tools is held stale here
if _, err := ix.Repin(context.Background(), "mesh-sdk", "0.1.27"); err != nil { // the pass for avahi alone: mesh-tools is held stale here
t.Fatal(err)
}
if b, _ := moduleOf(t, mods, "avahi"); b.Fp2 != a.Fp2 || b.Packages[sdk] != "v0.1.18" {
@@ -451,7 +451,7 @@ func PinsOf(packages map[string]string) sdkstep.PackageVersions {
// repin is Repin of a release of module at version, said in the journal; its failure is the caller's error.
func (ix *Indexer) repin(ctx context.Context, module, version string) error {
n, err := ix.Repin(ctx, module)
n, err := ix.Repin(ctx, module, version)
if err != nil {
return fmt.Errorf("the modules pinned to %s are not re-pinned to %s: %w", module, version, err)
}
@@ -102,7 +102,7 @@ func TestAnEntryNoRevisionListsStaysWhileAWantNamesItAndGoesAfterTheWantMovesOn(
n := NewNamed(newMem(), nil)
ix := &Indexer{Module: testSelf, Revisions: newMemObjects(1 << 20), EntryIndex: newMem(), NamedSets: n, Now: func() time.Time { return namedBase }}
n.Changed = ix.RemoveUnlisted
if err := ix.keepBranchEntries(ctx, TrunkBranch, 1, map[string]meshcatalog.CatalogEntry{"avahi": {Module: "avahi", Repository: "novox/r", Commit: "c", Fp: fp, Manifest: map[string]any{}}}, nil); err != nil {
if err := ix.KeepBranchEntries(ctx, TrunkBranch, 1, map[string]meshcatalog.CatalogEntry{"avahi": {Module: "avahi", Repository: "novox/r", Commit: "c", Fp: fp, Manifest: map[string]any{}}}, nil); err != nil {
t.Fatal(err)
}
if err := ix.Revisions.Delete(ctx, "main.1"); err != nil { // the revision that listed it is removed
@@ -213,7 +213,7 @@ func TestNoEntryIsRemovedWhileAStartReadWasEmptyOrAnEventWasNotApplied(t *testin
fp := namedFp("c")
setup := func(n *Named) (*Indexer, func() bool) {
ix := &Indexer{Module: testSelf, Revisions: newMemObjects(1 << 20), EntryIndex: newMem(), NamedSets: n, Now: func() time.Time { return namedBase }}
if err := ix.keepBranchEntries(ctx, TrunkBranch, 1, map[string]meshcatalog.CatalogEntry{"m": {Module: "m", Repository: "novox/r", Commit: "c", Fp: fp, Manifest: map[string]any{}}}, nil); err != nil {
if err := ix.KeepBranchEntries(ctx, TrunkBranch, 1, map[string]meshcatalog.CatalogEntry{"m": {Module: "m", Repository: "novox/r", Commit: "c", Fp: fp, Manifest: map[string]any{}}}, nil); err != nil {
t.Fatal(err)
}
if err := ix.Revisions.Delete(ctx, "main.1"); err != nil {