diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/input_integration_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/input_integration_test.go index 79522cc73..c63110b23 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/input_integration_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/input_integration_test.go @@ -11,6 +11,8 @@ import ( "encoding/hex" "encoding/json" "errors" + "os" + "reflect" "strings" "testing" "time" @@ -19,15 +21,24 @@ import ( "github.com/nats-io/nats.go/jetstream" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer" "git.novox.be/novox/mesh-sdk/go/buscalls" "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/seatlist/seatlisttest" "git.novox.be/novox/mesh-sdk/go/stateobjects" + "mesh-delivery-indexer/internal/index" + "mesh-delivery-indexer/judge" ) // TestIndexerAnswersARunsInputOverOneMiBInChunks (design 47 G §712, A-237, §681): an input of 3 MiB, kept in the state inputs with // its object beside it, is answered by the seat mesh-catalog's verb input as a count of chunks and then sent on the call's chunk // subjects, each acknowledged by the caller; the chunks carry the exact bytes. +// +// Design 47 H2 §814 (H-A-441, H-A-443, H-A-453 (3)): with the seat definitions of the tests' seat list in its state, the indexer +// keeps the seat list as one object of seat-lists named by its digest and hands it by the verb seat-list in chunks; a run by +// hand, which reads that object by its address and gives the judge its request as a file, gives the verdicts the live validate +// gives from the verb's bytes, for the builder's and the runner's manifests. func TestIndexerAnswersARunsInputOverOneMiBInChunks(t *testing.T) { body := make([]byte, 3<<20) if _, err := rand.Read(body); err != nil { @@ -35,13 +46,30 @@ func TestIndexerAnswersARunsInputOverOneMiBInChunks(t *testing.T) { } sum := sha256.Sum256(body) digest := hex.EncodeToString(sum[:]) - js, _, _, _ := startIndexer(t, func(context.Context, buildLog) {}, func(ctx context.Context, js jetstream.JetStream) { + var seatLists jetstream.ObjectStore + js, ctx, _, _ := startIndexer(t, func(context.Context, buildLog) {}, func(ctx context.Context, js jetstream.JetStream) { kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.Bucket(testIndexer, index.InputsState), History: 1}) if err != nil { t.Fatal(err) } - // The indexer's own seat list (A-186), its state seat-definitions: empty, so the bootstrapped mesh-catalog stands. - if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.Bucket(testIndexer, index.SeatDefinitionsState), History: 5}); err != nil { + // The indexer's own seat list (A-186), its state seat-definitions: the trunk's definition of every seat of the tests' + // seat list, as the take-in of the contracts modules keeps them; and its objects seat-lists, which the grants part makes. + defs, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.Bucket(testIndexer, index.SeatDefinitionsState), History: 5}) + if err != nil { + t.Fatal(err) + } + for _, d := range seatlisttest.Definitions() { + raw, err := json.Marshal(d) + if err != nil { + t.Fatal(err) + } + sum := sha256.Sum256(raw) + e, _ := json.Marshal(index.SeatDefinitionEntry{Seat: d.Seat, Source: index.SourceTrunk, Digest: hex.EncodeToString(sum[:]), Definition: raw, At: time.Now().UTC()}) + if _, err := defs.Put(ctx, index.SeatDefinitionKey(d.Seat), e); err != nil { + t.Fatal(err) + } + } + if seatLists, err = js.CreateObjectStore(ctx, jetstream.ObjectStoreConfig{Bucket: newBusObjects(nil, testAccount, index.SeatListsObjects).bucket()}); err != nil { t.Fatal(err) } objs, err := js.CreateObjectStore(ctx, jetstream.ObjectStoreConfig{Bucket: catalogue.BusName(testIndexer, catalogue.StateObjectsName(index.InputsState))}) @@ -103,4 +131,102 @@ func TestIndexerAnswersARunsInputOverOneMiBInChunks(t *testing.T) { if !bytes.Equal(got.Bytes(), body) { t.Errorf("the chunks carry %d bytes that are not the input's %d", got.Len(), len(body)) } + + // §814: the live validate's read, the verb seat-list in chunks (the assigner's seatList), and a run by hand's, the object + // seat-lists/ read by its address. + var list jetstream.ObjectInfo + wctx, cancel := context.WithTimeout(ctx, waitBound) + defer cancel() + w, err := seatLists.Watch(wctx) + if err != nil { + t.Fatal(err) + } + defer func() { _ = w.Stop() }() + for info := range w.Updates() { + if info != nil && !info.Deleted { + list = *info + break + } + } + if list.Name == "" { + t.Fatalf("seat-lists holds no seat list after %s", waitBound) + } + listCall := "fedcba9876543210fedcba9876543210" + listSend := buscalls.SeatChunks{Seat: meshcatalog.Seat, Verb: meshcatalog.VerbSeatList, Call: listCall} + listChunks := make(chan *nats.Msg, 16) + listSub, err := nc.ChanSubscribe(listSend.Chunks(), listChunks) + if err != nil { + t.Fatal(err) + } + defer listSub.Unsubscribe() + listReq, _ := json.Marshal(meshcatalog.SeatListInput{Call: listCall}) + m, err := nc.Request("mesh.seat.mesh-catalog.call.seat-list.agent~dev", listReq, 5*time.Second) + if err != nil { + t.Fatal(err) + } + var listReply struct { + Result meshcatalog.SeatListAnswer `json:"result"` + Error string `json:"error"` + } + if err := json.Unmarshal(m.Data, &listReply); err != nil || listReply.Error != "" || listReply.Result.Digest != list.Name || listReply.Result.Size != int64(list.Size) { + t.Fatalf("seat-list = %+v (%q, %v), want the object %s of %d bytes", listReply.Result, listReply.Error, err, list.Name, list.Size) + } + var viaVerb bytes.Buffer + listNext := func() (string, []byte, error) { + select { + case m := <-listChunks: + return m.Subject, m.Data, nil + case <-time.After(20 * time.Second): + return "", nil, buscalls.ErrSeatChunkMissed + } + } + if _, err := listSend.Receive(listReply.Result.Chunks, listNext, ack, &viaVerb); err != nil { + t.Fatal(err) + } + byAddress, err := seatLists.GetBytes(ctx, listReply.Result.Digest) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(viaVerb.Bytes(), byAddress) { + t.Fatalf("the verb's %d bytes are not the object's %d at its address", viaVerb.Len(), len(byAddress)) + } + // Both judged over the builder's and the runner's manifests as they stand in this repository: the live validate's + // request from the verb's bytes, and the hand run's request written to a file and read again as cmd/mesh-judge reads it. + manifests := map[string]composer.Manifest{} + for name, file := range map[string]string{"mesh-delivery-builder": "../../../mesh-delivery-builder/module.json", "mesh-delivery-pipeline": "../../../mesh-delivery-pipeline/module.json"} { + raw, err := os.ReadFile(file) + if err != nil { + t.Fatal(err) + } + manifests[name] = composer.Manifest{Raw: raw} + } + judged := func(seatList []byte, viaFile bool) composer.Verdict { + t.Helper() + var seats []json.RawMessage + if err := json.Unmarshal(seatList, &seats); err != nil || len(seats) == 0 { + t.Fatalf("the seat list is no array of definitions (%v)", err) + } + r := composer.Request{V: composer.Version, Cases: []composer.Case{{Name: "change", Manifests: manifests}}, Seats: seats, SeatList: listReply.Result.Digest} + if viaFile { + file, err := json.Marshal(r) + if err != nil { + t.Fatal(err) + } + if r, err = composer.Decode(file); err != nil { + t.Fatal(err) + } + } + v, err := judge.Request(r) + if err != nil { + t.Fatal(err) + } + return v + } + live, byHand := judged(viaVerb.Bytes(), false), judged(byAddress, true) + _, builder := live.Cases[0].Manifests["mesh-delivery-builder"] + _, runner := live.Cases[0].Manifests["mesh-delivery-pipeline"] + if !builder || !runner || !reflect.DeepEqual(byHand, live) { + t.Errorf("the run by hand judged %+v, want the live validate's %+v for the builder's and the runner's manifests", byHand, live) + } + t.Logf("the verdicts of the builder's and the runner's manifests against seat list %s: %+v", listReply.Result.Digest, live.Cases) } diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go index 52fdda675..d61bb8161 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go @@ -31,6 +31,7 @@ import ( "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" @@ -395,7 +396,7 @@ func startIndexerRun(t *testing.T, put func(context.Context, buildLog), before . "mesh.seat.mesh-catalog.event.seat-definition-removed", "mesh.seat.mesh-catalog.event.manifest-accepted", "mesh.seat.mesh-catalog.event.version-released", "mesh.seat.mesh-catalog.event.versions-filled", "mesh.seat.mesh-conditions.event.>", "mesh.seat.operator-channel.event.>", "mesh.seat.go-package-registry.event.>", "mesh.seat.npm-package-registry.event.>", - "mesh.seat.mesh-module-assigner.event.>", "mesh.seat.node-build-go.event.>"}}); err != nil { // not the seats kept in streams of their own (wanted and running, above) + "mesh.seat.mesh-module-assigner.event.>", "mesh.seat.node-build-go.event.>", meshbuilds.SubjectVersionReserved}}); err != nil { // not the seats kept in streams of their own (wanted and running, above) t.Fatal(err) } // The indexer's durable on the shared events stream (process.EventHub), which carries its own room (design 47 H §744, H-A-50). @@ -798,8 +799,9 @@ func serveAssignerAssignments(t *testing.T, ctx context.Context, js jetstream.Je t.Fatal(err) } if _, err := js.CreateOrUpdateConsumer(ctx, "EVENTS", jetstream.ConsumerConfig{Durable: "n1_mesh-delivery-indexer", AckPolicy: jetstream.AckExplicitPolicy, - // and the build seat's step-ended the manifest's consumes names (design 47 G §713: the end of a validation ask). - FilterSubjects: []string{"mesh.mod." + assignerHolderModule + ".event.views.>", meshcatalog.SubjectRoom, nodebuildgo.SubjectStepEndedAll}}); err != nil { + // and the build seat's step-ended the manifest's consumes names (design 47 G §713: the end of a validation ask), and the + // builder's version-reserved, which no stream of its own keeps (design 47 H2 §805). + FilterSubjects: []string{"mesh.mod." + assignerHolderModule + ".event.views.>", meshcatalog.SubjectRoom, nodebuildgo.SubjectStepEndedAll, meshbuilds.SubjectVersionReserved}}); err != nil { t.Fatal(err) } em := views.Emitter{Owner: assignerHolderModule, Subject: process.Account{Node: "n1", Module: assignerHolderModule}.EventSubject, diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index 51e04cb50..e48e3cbc6 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -362,8 +362,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro Packages: awaitedStore{b: packages, js: js, name: index.Bucket(cred.Module, index.PackagesState)}, Files: filesVia(calls.Call), GoSum: goSumVia(calls.Call), GoMod: goModVia(calls.Call), Heard: heardStore, Runs: awaitedStore{b: runs}, Graph: graph, // How long a reservation is kept and how long its gathering waits at most (design 47 H2 §805, H-A-424). - ReservedKeep: reservedSetting("reserved-keep", index.DefaultReservedKeep, logger.Printf), - ReservedSettle: reservedSetting("reserved-settle", index.DefaultReservedSettle, logger.Printf), + ReservedKeep: durationSetting("reserved-keep", index.DefaultReservedKeep, logger.Printf), + ReservedSettle: durationSetting("reserved-settle", index.DefaultReservedSettle, logger.Printf), + // How long a take-in waits on another part before it says so, and before it raises take-in-waiting (H2 §803, H-A-461 (2)). + WaitingLine: durationSetting(index.WaitingLineSetting, index.DefaultWaitingLine, logger.Printf), + WaitingBound: durationSetting(index.WaitingBoundSetting, index.DefaultWaitingBound, logger.Printf), SeatDefinitions: awaitedStore{b: seatDefs}, ReadFile: readFileVia(calls.Call), SaySeatDefinition: saySeats, Revisions: revisions, EntryIndex: awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.EntryIndexState), "no entry is found by its fingerprint", logger.Printf)}, // Message a kept call: a read, called again through a transport fault (ADR 0668 §1) // Each catalog revision may bring another holder of a build seat: the build modules are found again (buildmodules.go). @@ -690,8 +693,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } } -// reservedSetting is the setting key read as a duration (stdio.SettingDuration); one that does not read is said and takes def. -func reservedSetting(key string, def time.Duration, logf func(string, ...any)) time.Duration { +// durationSetting is the setting key read as a duration (stdio.SettingDuration); one that does not read is said and takes def. +func durationSetting(key string, def time.Duration, logf func(string, ...any)) time.Duration { d, err := stdio.SettingDuration(key, def) if err != nil { logf("%v: %s is taken", err, def) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/packages_integration_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/packages_integration_test.go index 9976e707b..0fbbe3064 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/packages_integration_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/packages_integration_test.go @@ -6,6 +6,8 @@ import ( "context" "encoding/json" "errors" + "fmt" + "maps" "os" "reflect" "slices" @@ -17,6 +19,7 @@ import ( "github.com/nats-io/nats.go/jetstream" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo" @@ -67,11 +70,23 @@ func untilModule(t *testing.T, ctx context.Context, js jetstream.JetStream, name // and not plain; the packages key of mesh-sdk/go names that commit with a digest per package. Design 47 H2 §804: a key a dry // run left in the state dry-runs is gone at the start, and the take-in of a branch moving mesh-sdk again, a dry run, leaves // the trunk's packages state byte for byte as before and no key in dry-runs once it answered. +// +// Design 47 H2 §802 (H-A-453 (3)): the catalogue holds 36 modules, among them lib-a and lib-b, required by app-a (a require) and +// app-b (a require and a replace by a directory); a source revision moving the catalogue to a commit that changes a test file +// of lib-a and one of lib-b, and so their trees alone, moves lib-a, lib-b and their importers alone, never the other 32 +// modules whose repository's commit moved too. +// +// Design 47 H2 §805 (H-A-453 (3)): the builder's twenty version-reserved of mesh-sdk in that revision's pipeline (twenty +// artifacts of its Go package, as repin_test.go reads them) are kept in heard and gathered, and its pipeline-started keeps one +// re-pin revision of main, importer its mover pinned to the reserved version; the next pipeline's reservation keeps the +// revision after it, so none came between. func TestASourceRevisionMovingMeshSDKNamesItAndItsImporters(t *testing.T) { const ( sdkGo = "git.novox.be/novox/mesh-sdk/go" m1, m2, m3 = "1111111111111111111111111111111111111111", "2222222222222222222222222222222222222222", "4444444444444444444444444444444444444444" catalogMerge = "3333333333333333333333333333333333333333" + catalogTests = "5555555555555555555555555555555555555555" // a test file of lib-a and one of lib-b changed + catalogModule = "git.novox.be/novox/mesh-catalog/modules/" goModContent = "module importer\n\nrequire git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000\n" sdkRoot1, sdk2 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" sdk3 = "cccccccccccccccccccccccccccccccccccccccc" @@ -97,7 +112,7 @@ func TestASourceRevisionMovingMeshSDKNamesItAndItsImporters(t *testing.T) { "package.json": "3333333333333333333333333333333333333333", "package-lock.json": "4444444444444444444444444444444444444444", "tsconfig.json": "5555555555555555555555555555555555555555"}}} } - var packages, dryRuns jetstream.KeyValue + var packages, dryRuns, heardKV jetstream.KeyValue var heard *takeIns js, ctx, _, _ := startIndexer(t, func(context.Context, buildLog) {}, func(ctx context.Context, js jetstream.JetStream) { var err error @@ -124,6 +139,10 @@ func TestASourceRevisionMovingMeshSDKNamesItAndItsImporters(t *testing.T) { forge.put(m1, sdkAt(sdkRoot1, "f5", "6666666666666666666666666666666666666666")) forge.put(m2, sdkAt(sdk2, "f6", "7777777777777777777777777777777777777777")) forge.put(m3, sdkAt(sdk3, "f7", "8888888888888888888888888888888888888888")) + // The state heard, declared key-ttl, where a reservation is kept and its gathering held open (§805). + if heardKV, err = js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.Bucket(testIndexer, index.HeardState), History: 64, LimitMarkerTTL: time.Minute}); err != nil { + t.Fatal(err) + } // A dry run's key left by an indexer that stopped mid-run (§804). if dryRuns, err = js.KeyValue(ctx, index.Bucket(testIndexer, index.DryRunsState)); err != nil { t.Fatal(err) @@ -131,14 +150,35 @@ func TestASourceRevisionMovingMeshSDKNamesItAndItsImporters(t *testing.T) { if _, err := dryRuns.Put(ctx, index.DryRunPrefix("beta", "0123")+index.PackagesState+".left", []byte(`{"value":"e30="}`)); err != nil { t.Fatal(err) } - forge.put(catalogMerge, repoAt{trees: map[string]string{"modules/importer": "t1", "modules/plain": "t2"}, + catalogue := repoAt{trees: map[string]string{"modules/importer": "t1", "modules/plain": "t2"}, files: map[string]string{"modules/importer/module.json": named("importer"), "modules/plain/module.json": named("plain"), "modules/importer/go.mod": goModContent, "modules/importer/go.sum": "golang.org/x/sys v0.1.0 h1:x=\n", - "modules/plain/go.mod": "module plain\n\nrequire golang.org/x/sys v0.1.0\n", "modules/plain/go.sum": "golang.org/x/sys v0.1.0 h1:x=\n"}}) - regs := registerAvahi(t, ctx, js, "main") - for _, r := range []meshcatalog.Registration{{V: 1, Module: "importer", Origin: "novox/mesh-catalog", Path: "modules/importer", Branches: []string{"main"}}, + "modules/plain/go.mod": "module plain\n\nrequire golang.org/x/sys v0.1.0\n", "modules/plain/go.sum": "golang.org/x/sys v0.1.0 h1:x=\n"}, + paths: map[string]map[string]string{"": {"modules/lib-a": "a1", "modules/lib-b": "b1"}}} + registrations := []meshcatalog.Registration{{V: 1, Module: "importer", Origin: "novox/mesh-catalog", Path: "modules/importer", Branches: []string{"main"}}, {V: 1, Module: "plain", Origin: "novox/mesh-catalog", Path: "modules/plain", Branches: []string{"main"}}, - {V: 1, Module: "mesh-sdk", Origin: "novox/mesh-sdk", Path: ".", Branches: []string{"main", "beta"}}} { + {V: 1, Module: "mesh-sdk", Origin: "novox/mesh-sdk", Path: ".", Branches: []string{"main", "beta"}}} + // §802: 34 more modules of the catalogue, 36 with importer and plain: the libraries, their importers and 30 others. + goMods := map[string]string{"lib-a": "module " + catalogModule + "lib-a\n", "lib-b": "module " + catalogModule + "lib-b\n", + "app-a": "module app-a\n\nrequire " + catalogModule + "lib-a v0.0.0-00010101000000-000000000000\n", + "app-b": "module app-b\n\nrequire " + catalogModule + "lib-b v0.0.0-00010101000000-000000000000\n\nreplace " + catalogModule + "lib-b => ../lib-b\n"} + for i := range 30 { + goMods[fmt.Sprintf("m%02d", i)] = fmt.Sprintf("module m%02d\n\nrequire golang.org/x/sys v0.1.0\n", i) + } + for module, goMod := range goMods { + dir := "modules/" + module + catalogue.trees[dir] = "t-" + module + catalogue.files[dir+"/module.json"], catalogue.files[dir+"/go.mod"], catalogue.files[dir+"/go.sum"] = named(module), goMod, "golang.org/x/sys v0.1.0 h1:x=\n" + registrations = append(registrations, meshcatalog.Registration{V: 1, Module: module, Origin: "novox/mesh-catalog", Path: dir, Branches: []string{"main"}}) + } + forge.put(catalogMerge, catalogue) + tests := catalogue + tests.trees = maps.Clone(catalogue.trees) + tests.trees["modules/lib-a"], tests.trees["modules/lib-b"] = "t-lib-a-2", "t-lib-b-2" + tests.paths = map[string]map[string]string{"": {"modules/lib-a": "a2", "modules/lib-b": "b2"}} + forge.put(catalogTests, tests) + regs := registerAvahi(t, ctx, js, "main") + for _, r := range registrations { raw, _ := json.Marshal(r) if _, err := regs.Put(ctx, index.RegistrationKey(r.Module), raw); err != nil { t.Fatal(err) @@ -207,6 +247,72 @@ func TestASourceRevisionMovingMeshSDKNamesItAndItsImporters(t *testing.T) { return v != nil && err == nil && pub.Commit == m2 && len(pub.Packages) == 3 }) + // §802: a commit of the catalogue changing a test file of lib-a and one of lib-b moves those two and their importers alone. + heard.acked(t, "main") + say(12, false, meshdelivery.SourceMovedEventHeadsItem{Repository: "novox/mesh-catalog", Commit: catalogTests}) + movers = nil + taken = heard.next(t, "main", 12) + for _, e := range taken.Entries { + movers = append(movers, e.Module) + } + slices.Sort(movers) + if !slices.Equal(movers, []string{"app-a", "app-b", "lib-a", "lib-b"}) { + t.Errorf("the catalogue's commit changing lib-a's and lib-b's tests moves %v of its 36 modules, want lib-a, lib-b and their importers alone", movers) + } + + // §805: twenty reservations of mesh-sdk's packages in main's revision of source 12, kept and gathered; its pipeline-started + // keeps one re-pin revision of main, and the next pipeline's reservation the one after it. + objects := newBusObjects(js, testAccount, index.CatalogRevisionsObjects) + untilRevision := func(n int64) meshcatalog.CatalogRevision { + t.Helper() + obs, err := js.ObjectStore(ctx, objects.bucket()) + if err != nil { + t.Fatal(err) + } + wctx, cancel := context.WithTimeout(ctx, waitBound) + defer cancel() + w, err := obs.Watch(wctx) + if err != nil { + t.Fatal(err) + } + defer func() { _ = w.Stop() }() + name := index.BranchObject("main", n) + for range w.Updates() { + if rev, _, err := index.ReadRevision(ctx, objects, name); err == nil { + return rev + } + } + t.Fatalf("%s is not kept after %s", name, waitBound) + return meshcatalog.CatalogRevision{} + } + reserve := func(revision int64, version string, artifacts int) { + t.Helper() + for i := range artifacts { + raw, _ := json.Marshal(meshbuilds.VersionReservedEvent{Module: "mesh-sdk", Artifact: fmt.Sprintf("mesh-sdk:p%d", i), Path: sdkGo, Version: version, + Fingerprint: "f-" + version, Revision: index.BranchObject("main", revision)}) + if _, err := js.Publish(ctx, meshbuilds.SubjectVersionReserved, raw); err != nil { + t.Fatal(err) + } + } + untilKey(t, ctx, heardKV, index.ReservedKey(index.BranchObject("main", revision), "mesh-sdk"), func(v []byte) bool { + var r index.Reserved + return v != nil && json.Unmarshal(v, &r) == nil && len(r.Reservations) == artifacts + }) + } + r := taken.Revision + reserve(r, "0.1.28", 20) + sayPipeline(t, ctx, js, false, "main", r) + if rev := untilRevision(r + 1); rev.CallKind != meshcatalog.CatalogRevisionCallKindRelease || !strings.Contains(rev.Call, index.BranchObject("main", r)+"'s") || + len(rev.Movers) != 1 || rev.Movers[0].Module != "importer" { + t.Errorf("main.%d = %+v, want one re-pin revision of the twenty reservations, importer its mover", r+1, rev) + } + untilModule(t, ctx, js, "importer", func(m meshcatalog.ModulesByModuleValue) bool { return m.Packages[sdkGo] == "v0.1.28" }) + reserve(r+1, "0.1.29", 1) + sayPipeline(t, ctx, js, false, "main", r+1) + if rev := untilRevision(r + 2); !strings.Contains(rev.Call, index.BranchObject("main", r+1)+"'s") { + t.Errorf("main.%d = %+v, want the next pipeline's re-pin: the twenty reservations kept more than one revision", r+2, rev) + } + // A branch's take-in moving mesh-sdk to m3 is a dry run: its packages stay in dry-runs while it runs, the trunk's are // untouched, and its keys go once it answered (§804). heard.acked(t, "main") diff --git a/modules/mesh-delivery-indexer/internal/index/candidates.go b/modules/mesh-delivery-indexer/internal/index/candidates.go index bb94bccbe..42e39d950 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidates.go +++ b/modules/mesh-delivery-indexer/internal/index/candidates.go @@ -196,6 +196,10 @@ type Indexer struct { // first reservation its gathering settles at the latest: the settings reserved-keep and reserved-settle (design 47 H2 // §805, H-A-424); zero takes the default. ReservedKeep, ReservedSettle time.Duration + // WaitingLine is how long a take-in waits on another part before it says so and keeps its waiting-on, and WaitingBound + // how long before it raises take-in-waiting: the settings take-in-waiting-line and take-in-waiting-bound (design 47 H2 + // §803, H-A-461 (2)); zero takes the default. + WaitingLine, WaitingBound time.Duration } func (ix *Indexer) logf(format string, args ...any) { diff --git a/modules/mesh-delivery-indexer/internal/index/control_records_test.go b/modules/mesh-delivery-indexer/internal/index/control_records_test.go index 580a4bcc0..2a16ce13a 100644 --- a/modules/mesh-delivery-indexer/internal/index/control_records_test.go +++ b/modules/mesh-delivery-indexer/internal/index/control_records_test.go @@ -513,6 +513,7 @@ func TestTheControlRecords(t *testing.T) { t.Errorf("a state not on the bus yet stopped the message: %v", err) } }) + t.Run("seat-list", seatListIsOneObjectByItsDigest) } // memSource is a state as views.Source, for the indexer's tools that answer from its own states. diff --git a/modules/mesh-delivery-indexer/internal/index/seatlists_test.go b/modules/mesh-delivery-indexer/internal/index/seatlists_test.go index d00f43b52..13e96182c 100644 --- a/modules/mesh-delivery-indexer/internal/index/seatlists_test.go +++ b/modules/mesh-delivery-indexer/internal/index/seatlists_test.go @@ -70,9 +70,10 @@ func (noLine) Subscribe(string, nats.MsgHandler) (*nats.Subscription, error) { return nil, errors.New("no bus in this test") } -// Design 47 H2 §814, §820 (H-A-441, H-A-443): the trunk's seat list is one object of seat-lists named by its sha256, in the -// order of its seats, kept at each new list with the newest two, and handed by the verb seat-list. -func TestTheSeatListIsOneObjectByItsDigest(t *testing.T) { +// seatListIsOneObjectByItsDigest is TestTheControlRecords' row "seat-list" (design 47 H2 §814, §820; H-A-441, H-A-443): the +// trunk's seat list is one object of seat-lists named by its sha256, in the order of its seats, kept at each new list with the +// newest two, and handed by the verb seat-list. A row, not a test of its own: the indexer's unit count does not rise (§819). +func seatListIsOneObjectByItsDigest(t *testing.T) { entry := func(seat, source, def string) *views.Entry { raw, _ := json.Marshal(SeatDefinitionEntry{Seat: seat, Source: source, Digest: "d-" + seat, Definition: json.RawMessage(def)}) return &views.Entry{Value: raw} diff --git a/modules/mesh-delivery-indexer/internal/index/takein_source_test.go b/modules/mesh-delivery-indexer/internal/index/takein_source_test.go index 3386f2592..5765977ef 100644 --- a/modules/mesh-delivery-indexer/internal/index/takein_source_test.go +++ b/modules/mesh-delivery-indexer/internal/index/takein_source_test.go @@ -1,6 +1,7 @@ package index import ( + "cmp" "context" "encoding/json" "errors" @@ -141,11 +142,15 @@ func TestASourceRevisionIsTakenInAsOneRevision(t *testing.T) { now := start timers := make(chan context.CancelFunc) var mu sync.Mutex + var asked []time.Duration // the bounds the wait asked, in order var lines []string wrote := make(chan struct{}, 1) findings := newFindings() w := &Indexer{Module: testSelf, TakeIns: running, Findings: findings, Now: func() time.Time { mu.Lock(); defer mu.Unlock(); return now }, - Bound: func(parent context.Context, _ time.Duration) (context.Context, context.CancelFunc) { + Bound: func(parent context.Context, d time.Duration) (context.Context, context.CancelFunc) { + mu.Lock() + asked = append(asked, d) + mu.Unlock() b, cancel := context.WithCancel(parent) timers <- cancel return b, cancel @@ -156,10 +161,10 @@ func TestASourceRevisionIsTakenInAsOneRevision(t *testing.T) { lines = append(lines, fmt.Sprintf(format, args...)) }} w.Findings = findingsSignal{findings, wrote} - listing := func(answerAfter time.Duration) { + listing := func(answerAfter, line, bound time.Duration) { t.Helper() mu.Lock() - now, lines = start, nil + now, lines, asked = start, nil, nil mu.Unlock() release, res := make(chan struct{}), make(chan error) go func() { @@ -167,7 +172,7 @@ func TestASourceRevisionIsTakenInAsOneRevision(t *testing.T) { res <- err }() first := <-timers - if answerAfter < WaitingLineAfter { + if answerAfter < line { close(release) <-res return @@ -177,7 +182,7 @@ func TestASourceRevisionIsTakenInAsOneRevision(t *testing.T) { mu.Unlock() first() second := <-timers // asked once the line is said and the waiting-on written - if answerAfter >= WaitingFindingAfter { + if answerAfter >= bound { second() <-wrote if h := findings.Held(); len(h) != 1 || h[0].Kind != TakeInWaitingKind || h[0].Scope != mcc.ScopeMerge || h[0].ID != "9" { @@ -189,15 +194,27 @@ func TestASourceRevisionIsTakenInAsOneRevision(t *testing.T) { t.Fatal(err) } } + // The bounds are the settings take-in-waiting-line and take-in-waiting-bound, 1m and 5m when unset (H-A-461 (2)): the + // wait asks the line, then the rest to the bound. for _, c := range []struct { - after time.Duration - lines int - finding bool - }{{30 * time.Second, 0, false}, {70 * time.Second, 1, false}, {6 * time.Minute, 1, false}} { - listing(c.after) + after, line, bound time.Duration // line and bound zero: the settings unset + lines int + finding bool + }{{30 * time.Second, 0, 0, 0, false}, {70 * time.Second, 0, 0, 1, false}, {6 * time.Minute, 0, 0, 1, false}, + {6 * time.Minute, 2 * time.Minute, 10 * time.Minute, 1, false}} { + w.WaitingLine, w.WaitingBound = c.line, c.bound + line, bound := cmp.Or(c.line, time.Minute), cmp.Or(c.bound, 5*time.Minute) + listing(c.after, line, bound) mu.Lock() - got := slices.Clone(lines) + got, gotAsked := slices.Clone(lines), slices.Clone(asked) mu.Unlock() + wantAsked := []time.Duration{line} + if c.after >= line { + wantAsked = append(wantAsked, bound-line) + } + if !slices.Equal(gotAsked, wantAsked) { + t.Errorf("answered after %s (line %s, bound %s): bounds asked %v, want %v", c.after, c.line, c.bound, gotAsked, wantAsked) + } if len(got) != c.lines || (c.lines == 1 && !strings.Contains(got[0], "waits on git trees since 2026-11-22T01:00:00Z")) { t.Errorf("answered after %s: lines %q, want %d naming git trees and since when", c.after, got, c.lines) } diff --git a/modules/mesh-delivery-indexer/internal/index/takein_waiting.go b/modules/mesh-delivery-indexer/internal/index/takein_waiting.go index e732b658f..b29d0f9d2 100644 --- a/modules/mesh-delivery-indexer/internal/index/takein_waiting.go +++ b/modules/mesh-delivery-indexer/internal/index/takein_waiting.go @@ -11,17 +11,19 @@ import ( mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" ) -// A take-in says what it waits on (design 47 H2 §803, H-A-409): a source-moved take-in that waits longer than -// WaitingLineAfter on another part (the seat git's listing or a file, a package's versions, a registry) writes one journal +// A take-in says what it waits on (design 47 H2 §803, H-A-409): a source-moved take-in that waits longer than the +// setting take-in-waiting-line on another part (the seat git's listing or a file, a package's versions, a registry) writes one journal // line naming the part, the verb and since when, and keeps its waiting-on in the state takeins under the key -// RunningPrefix and the source revision, rewritten at each new wait and removed at the take-in's end; past -// WaitingFindingAfter it raises the finding take-in-waiting (scope merge, id the source revision), cleared when the wait -// ends. indexer_pending answers it beside the held take-ins. +// RunningPrefix and the source revision, rewritten at each new wait and removed at the take-in's end; past the setting +// take-in-waiting-bound it raises the finding take-in-waiting (scope merge, id the source revision), cleared when the wait +// ends. indexer_pending answers it beside the held take-ins. The two bounds are settings with these defaults (H-A-461 (2)). const ( TakeInWaitingKind = "take-in-waiting" RunningPrefix = "running." - WaitingLineAfter = time.Minute - WaitingFindingAfter = 5 * time.Minute + WaitingLineSetting = "take-in-waiting-line" + WaitingBoundSetting = "take-in-waiting-bound" + DefaultWaitingLine = time.Minute + DefaultWaitingBound = 5 * time.Minute ) // WaitingOn is what a running take-in waits on: the part, its verb and since when. @@ -95,8 +97,8 @@ func passed(ctx, b context.Context, done <-chan struct{}) bool { return ctx.Err() == nil } -// waited is f, a call on part's verb: within a running take-in (ctx), a wait past WaitingLineAfter is said and kept, one -// past WaitingFindingAfter raises take-in-waiting until it ends. Outside a take-in it is f alone. +// waited is f, a call on part's verb: within a running take-in (ctx), a wait past take-in-waiting-line is said and kept, one +// past take-in-waiting-bound raises take-in-waiting until it ends. Outside a take-in it is f alone. func waited[T any](ctx context.Context, part, verb string, f func() (T, error)) (T, error) { run, ok := ctx.Value(takeInRunKey{}).(*takeInRun) if !ok { @@ -115,7 +117,14 @@ func waited[T any](ctx context.Context, part, verb string, f func() (T, error)) func (ix *Indexer) watchWait(ctx context.Context, run *takeInRun, part, verb string, done <-chan struct{}) { since := ix.now().UTC() - line, cancel := ix.bound(ctx, WaitingLineAfter) + after, bound := ix.WaitingLine, ix.WaitingBound + if after <= 0 { + after = DefaultWaitingLine + } + if bound <= 0 { + bound = DefaultWaitingBound + } + line, cancel := ix.bound(ctx, after) defer cancel() if !passed(ctx, line, done) { return @@ -123,7 +132,7 @@ func (ix *Indexer) watchWait(ctx context.Context, run *takeInRun, part, verb str ix.logf("take-in: source revision %d of %s waits on %s %s since %s", run.source, run.branch, part, verb, since.Format(time.RFC3339)) ix.writeWaiting(ctx, RunningTakeIn{Branch: run.branch, Source: run.source, WaitingOn: WaitingOn{Part: part, Verb: verb, Since: since}}) id := strconv.FormatInt(run.source, 10) - finding, cancelFinding := ix.bound(ctx, WaitingFindingAfter-WaitingLineAfter) + finding, cancelFinding := ix.bound(ctx, max(bound-after, 0)) defer cancelFinding() if !passed(ctx, finding, done) { return diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index c8c49cd36..af28dc77e 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -136,12 +136,13 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build if !slices.Equal(m.Uses, []string{"operator-channel"}) || !slices.Contains(m.Invokes, "seat:mesh-module-assigner.assignments") || !slices.Contains(m.Invokes, "seat:mesh-module-assigner.placement") { t.Errorf("uses = %v, invokes = %v, want the seat operator-channel used and seat:mesh-module-assigner.assignments invoked", m.Uses, m.Invokes) } - // 1126 note 76: heard keeps the last 64 FORGE messages the indexer handled, for indexer_heard; 8 MiB, each reservation - // and each gathering of them a key of its own time to live (design 47 H2 §805, H-A-420 (3), H-A-436). + // 1126 note 76: heard keeps the last 64 FORGE messages the indexer handled, for indexer_heard, each reservation and each + // gathering of them a key of its own time to live (design 47 H2 §805, H-A-420 (3), H-A-436); 32 MiB, its live fill 4.44 MiB + // (novox, 2026-11-22) more than a quarter of 8 (H-A-461 (3), H-A-437). // Design 47 H2 §814, §820 (H-A-411 (2), H-A-437): every state names its cap, 8 MiB where the live fill (novox, 2026-11-22) // is a quarter of it or less, else the smallest power of two above four times the fill: candidates 3.78 MiB at 16, - // manifests 5.21 MiB and seat-definitions 4.19 MiB at 32. - if want := []state{{Name: "candidates", History: 10, MaxMiB: 16}, {Name: "heard", History: 64, MaxMiB: 8, KeyTTL: true}, {Name: "manifests", History: 1, MaxMiB: 32}, {Name: "predefined", History: 5, MaxMiB: 8}, {Name: "taken", History: 1, MaxMiB: 8}, {Name: "takeins", History: 1, MaxMiB: 8}, {Name: "oldsections", History: 1, MaxMiB: 8}, {Name: index.RunsState, History: 1, MaxMiB: 8}, {Name: index.PackagesState, History: index.PackagesHistory, MaxMiB: 8}, + // manifests 5.21 MiB, seat-definitions 4.19 MiB and heard 4.44 MiB at 32. + if want := []state{{Name: "candidates", History: 10, MaxMiB: 16}, {Name: "heard", History: 64, MaxMiB: 32, KeyTTL: true}, {Name: "manifests", History: 1, MaxMiB: 32}, {Name: "predefined", History: 5, MaxMiB: 8}, {Name: "taken", History: 1, MaxMiB: 8}, {Name: "takeins", History: 1, MaxMiB: 8}, {Name: "oldsections", History: 1, MaxMiB: 8}, {Name: index.RunsState, History: 1, MaxMiB: 8}, {Name: index.PackagesState, History: index.PackagesHistory, MaxMiB: 8}, {Name: views.MarksState, History: 1, MaxMiB: 8}, {Name: index.FindingsState, History: 5, MaxMiB: 8}, {Name: views.StatusState, History: 1, MaxMiB: 8}, {Name: index.RequirementsState, History: 5, MaxMiB: 8}, {Name: index.SeatDefinitionsState, History: 5, MaxMiB: 32}, {Name: index.EntryIndexState, History: 1, MaxMiB: 8}, {Name: index.NamedState, History: 1, MaxMiB: 4}, // design 47 G §695 @@ -228,11 +229,11 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build t.Errorf("streams = %+v, want registration-history keeping its history, 64 MiB, and the findings stream at 8 MiB (§387, H2 §783)", m.Streams) } // Design 47 H2 §810, §814 (H-A-394 (1)): it claims a seat, so its class is core. §775: it makes kept calls, so it declares - // the kept-call pace, 1 s to 30 s, read while it runs. + // the kept-call pace, 1 s to 30 s, read while it runs. §803 (H-A-461 (2)): a take-in's waiting line and bound, 1m and 5m. if m.Class != "core" { t.Errorf("class = %q, want core", m.Class) } - for key, def := range map[string]string{busretry.PaceFirstSetting: "1s", busretry.PaceMaxSetting: "30s"} { + for key, def := range map[string]string{busretry.PaceFirstSetting: "1s", busretry.PaceMaxSetting: "30s", index.WaitingLineSetting: "1m", index.WaitingBoundSetting: "5m"} { if c, _ := m.Config[key].(map[string]any); c["default"] != def || c["runtime"] != true || c["type"] != "string" { t.Errorf("config %s = %v, want a runtime string setting defaulting to %s", key, m.Config[key], def) } diff --git a/modules/mesh-delivery-indexer/module.json b/modules/mesh-delivery-indexer/module.json index 965ea9527..e48cd603e 100644 --- a/modules/mesh-delivery-indexer/module.json +++ b/modules/mesh-delivery-indexer/module.json @@ -130,11 +130,25 @@ "trusted": false, "default": "30s", "runtime": true + }, + "take-in-waiting-line": { + "type": "string", + "description": "how long a take-in waits on another part (the seat git, a package's versions, a registry) before it writes one journal line and its waiting-on in takeins, a Go duration (design 47 H2 §803, H-A-461 (2)).", + "trusted": false, + "default": "1m", + "runtime": true + }, + "take-in-waiting-bound": { + "type": "string", + "description": "how long a take-in waits on another part before it raises the finding take-in-waiting, cleared when the wait ends, a Go duration (design 47 H2 §803, H-A-461 (2)).", + "trusted": false, + "default": "5m", + "runtime": true } }, "state": [ { "name": "candidates", "history": 10, "max-mib": 16 }, - { "name": "heard", "history": 64, "max-mib": 8, "key-ttl": true }, + { "name": "heard", "history": 64, "max-mib": 32, "key-ttl": true }, { "name": "manifests", "history": 1, "max-mib": 32 }, { "name": "predefined", "history": 5, "max-mib": 8 }, { "name": "taken", "history": 1, "max-mib": 8 },