From a98b6ba77e89f14c53c3d9552e89a2d92df770f5 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 13 Nov 2026 20:48:04 +0100 Subject: [PATCH 01/24] S6: the bus keeper treats a none-yet catalog half as not yet viewed PlanCatalog plans nothing, deletes no plans key and orphans no entry while the catalog half lists no module before any was seen; the first revision with entries plans as before. Builds: design 47 G A-367 (G-22-lead-answers-2.md), W22/Q4 of start-waits.md Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- modules/nats/internal/keeper/keeper.go | 12 +++++- modules/nats/internal/keeper/keeper_test.go | 42 +++++++++++++++++++-- 2 files changed, 50 insertions(+), 4 deletions(-) diff --git a/modules/nats/internal/keeper/keeper.go b/modules/nats/internal/keeper/keeper.go index 3b1af6d94..68788bed4 100644 --- a/modules/nats/internal/keeper/keeper.go +++ b/modules/nats/internal/keeper/keeper.go @@ -456,7 +456,8 @@ func (k *Keeper) statusWritten(s Status, err error) { // watched branch's newest revision is applied (Heard), a module of the new form that sends gets its sent store, a drift // between branches is said once, and a module in no watched branch's newest revision has its plans key deleted, its // entries then orphans. Before every watched branch has its first view it plans nothing, and says what it waits for in -// its status; the first plan checks what is held. +// its status, and an answer listing no module before one has listed any is the same (none-yet); the first plan checks what +// is held. func (k *Keeper) PlanCatalog() error { at := k.now() view, err := k.Catalog() @@ -466,6 +467,15 @@ func (k *Keeper) PlanCatalog() error { } k.mu.Lock() first := !k.viewed + // A catalog half with no module in it before any has been seen is a trunk with no revision yet (`none-yet`, design 47 + // G A-367): not yet viewed, so nothing is planned, no plan deleted and no entry made an orphan for want of knowledge. + if first && len(view.Modules) == 0 { + k.mu.Unlock() + k.setStatus(func(s *Status) { + s.FirstView.Asked, s.FirstView.Error = at, "the catalog half lists no module yet (none-yet): nothing is planned or orphaned until a revision lists one" + }) + return nil + } k.viewed = true k.mu.Unlock() var revision uint64 diff --git a/modules/nats/internal/keeper/keeper_test.go b/modules/nats/internal/keeper/keeper_test.go index b7c2aa305..500eb3d04 100644 --- a/modules/nats/internal/keeper/keeper_test.go +++ b/modules/nats/internal/keeper/keeper_test.go @@ -450,6 +450,9 @@ func TestTheKeeperMakesWhatAVerdictAsksForAndSaysIt(t *testing.T) { if len(bus.created) != 0 { t.Fatalf("created %v before the first view", bus.created) } + k.Catalog = func() (CatalogView, error) { + return CatalogView{Modules: []CatalogModule{{Accepted: Accepted{Module: "m", Digest: "d1", Revision: 3, BusState: []catalogue.BusStateEntry{drafts}}}}}, nil + } if err := k.PlanCatalog(); err != nil { t.Fatal(err) } @@ -541,6 +544,37 @@ func TestTheKeeperMakesWhatAVerdictAsksForAndSaysIt(t *testing.T) { t.Errorf("old sections %d: 8 MiB counted", k.Status().OldBytes) } }}, + // A-367: a trunk with no revision yet (none-yet) is not yet viewed: nothing is planned, deleted or orphaned. + {"a none-yet catalog half plans nothing and orphans nothing until a module is listed", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) { + k.Heard(Accepted{Module: "m", Revision: 9, BusState: []catalogue.BusStateEntry{drafts}}) + k.Catalog = func() (CatalogView, error) { + return CatalogView{Modules: []CatalogModule{{Accepted: Accepted{Module: "m", Revision: 9, BusState: []catalogue.BusStateEntry{drafts}}}}}, nil + } + _ = k.PlanCatalog() + k.mu.Lock() + k.viewed = false + k.mu.Unlock() + k.Catalog = func() (CatalogView, error) { return CatalogView{}, nil } + if err := k.PlanCatalog(); err != nil { + t.Fatal(err) + } + if _, ok := k.planned("m"); !ok { + t.Error("m's plan is deleted by a none-yet view") + } + if a := k.Held()["m"]; a.Made["drafts"].Orphaned { + t.Errorf("held %+v: drafts orphaned by a none-yet view", a) + } + if k.Viewed() || !strings.Contains(k.Status().FirstView.Error, "none-yet") { + t.Errorf("viewed %v, status %+v: none-yet is not a first view and says so", k.Viewed(), k.Status().FirstView) + } + k.Catalog = func() (CatalogView, error) { + return CatalogView{Modules: []CatalogModule{{Accepted: Accepted{Module: "m", Revision: 10, BusState: []catalogue.BusStateEntry{drafts}}}}}, nil + } + _ = k.PlanCatalog() + if !k.Viewed() || k.Status().FirstView.Error != "" { + t.Error("the first revision with entries is viewed and plans as before") + } + }}, // §538: a module in no watched branch's newest revision has its plans key deleted, its entries then orphans. {"a module no watched branch lists loses its plan, its entries orphans", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) { k.Catalog = func() (CatalogView, error) { @@ -1240,9 +1274,11 @@ func TestTheKeeperMakesWhatAVerdictAsksForAndSaysIt(t *testing.T) { } { t.Run(row.name, func(t *testing.T) { k, bus, _, out := newKeeper(CatalogView{}) - if err := k.PlanCatalog(); err != nil { - t.Fatal(err) - } + // The rows start after the first view: an empty answer is none-yet (A-367), so the view is marked answered here. + k.mu.Lock() + k.viewed = true + k.mu.Unlock() + k.setStatus(func(s *Status) { s.FirstView.Asked, s.FirstView.Answered = k.now(), k.now() }) row.run(t, k, bus, out) }) } From ed536102acfa9f3cdcf307f9c3af686cedcd7dc2 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 13 Nov 2026 23:11:57 +0100 Subject: [PATCH 02/24] A-376: the bus keeper makes the catalog half's own holder before the first views Before the catalog half's first views are in (a read refused, or none-yet), the keeper reads the manifest on the trunk of the holder of the seat mesh-catalog from the seat's states modules and manifests (the holder from the seat that places modules) and makes its kept data as any module's, each entry said as ever; every other module waits as A-367 says. Builds: design 47 G A-376 (G-22-lead-answers-2.md) Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- modules/nats/cmd/nats-tools/catalogholder.go | 66 +++++++++++++++++++ .../nats/cmd/nats-tools/catalogholder_test.go | 45 +++++++++++++ modules/nats/cmd/nats-tools/keep.go | 20 +++++- modules/nats/cmd/nats-tools/manifest_test.go | 8 ++- modules/nats/internal/keeper/keeper.go | 30 ++++++++- modules/nats/internal/keeper/keeper_test.go | 29 ++++++++ modules/nats/module.json | 7 +- 7 files changed, 197 insertions(+), 8 deletions(-) create mode 100644 modules/nats/cmd/nats-tools/catalogholder.go create mode 100644 modules/nats/cmd/nats-tools/catalogholder_test.go diff --git a/modules/nats/cmd/nats-tools/catalogholder.go b/modules/nats/cmd/nats-tools/catalogholder.go new file mode 100644 index 000000000..e8b5df706 --- /dev/null +++ b/modules/nats/cmd/nats-tools/catalogholder.go @@ -0,0 +1,66 @@ +package main + +import ( + "encoding/json" + "fmt" + + catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-sdk/go/seatlist" + + "nats-tools/internal/keeper" +) + +// The keeper makes the kept data of the holder of the seat mesh-catalog before the catalog half's first views are in (design +// 47 G A-376, completing A-367): that holder is the catalog half's own source and needs its kept data to answer them. Its +// manifest on the trunk is read as the holder's first views `modules` (the manifest's digest by module) and `manifests` (the +// manifest by digest) give it, both followed as the seat's states (views.ProcessReader.FollowSeat), and the holder is the +// module the seat that places modules answers for the seat (seatlist.HolderOf): no module is named here. + +// holderAccepted is what the holder's manifest on the trunk asks of the bus: the holder's entry of `modules` names its +// manifest's digest, and `manifests` holds that manifest. An error says what is not yet in. +func holderAccepted(holder string, modules, manifests map[string]json.RawMessage) (keeper.Accepted, error) { + raw, ok := modules[holder] + if !ok { + return keeper.Accepted{}, fmt.Errorf("the first view `modules` holds no entry of %s yet", holder) + } + // Only the manifest's digest is read of the entry, so an entry whose fingerprint is not (yet) there still names it. + var v struct { + Manifest string `json:"manifest"` + } + if err := json.Unmarshal(raw, &v); err != nil { + return keeper.Accepted{}, fmt.Errorf("the entry of %s in `modules` does not read: %w", holder, err) + } + if v.Manifest == "" { + return keeper.Accepted{}, fmt.Errorf("the entry of %s in `modules` names no manifest", holder) + } + m, ok := manifests[v.Manifest] + if !ok { + return keeper.Accepted{}, fmt.Errorf("the first view `manifests` holds no manifest %s of %s yet", v.Manifest, holder) + } + var by catalog.ManifestsByDigestValue + if err := json.Unmarshal(m, &by); err != nil { + return keeper.Accepted{}, fmt.Errorf("the manifest %s of %s does not read: %w", v.Manifest, holder, err) + } + var manifest map[string]any + if err := json.Unmarshal([]byte(by.Manifest), &manifest); err != nil { + return keeper.Accepted{}, fmt.Errorf("the manifest %s of %s does not read: %w", v.Manifest, holder, err) + } + a, err := AsksOf(catalog.CatalogEntry{Module: holder, Manifest: manifest}) + if err != nil { + return keeper.Accepted{}, err + } + return a.Accepted, nil +} + +// holder is the keeper's Holder: the seat's holder through the seat that places modules, its two states followed as the +// seat's, nothing asked on a timer (the follows read the holder again at each change of a view the reader follows). +func (f *catalogFollow) holder() (keeper.Accepted, error) { + if f.holderModules == nil || f.holderManifests == nil { + return keeper.Accepted{}, fmt.Errorf("the seat %s's states are not followed", catalog.Seat) + } + who, err := seatlist.HolderOf(f.call, catalog.Seat) + if err != nil { + return keeper.Accepted{}, err + } + return holderAccepted(who, f.holderModules.Follower().Values(), f.holderManifests.Follower().Values()) +} diff --git a/modules/nats/cmd/nats-tools/catalogholder_test.go b/modules/nats/cmd/nats-tools/catalogholder_test.go new file mode 100644 index 000000000..4fb7dfacf --- /dev/null +++ b/modules/nats/cmd/nats-tools/catalogholder_test.go @@ -0,0 +1,45 @@ +package main + +import ( + "encoding/json" + "strings" + "testing" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" +) + +// TestHolderAcceptedReadsTheHoldersManifestFromItsFirstViews is design 47 G A-376: the entry the keeper makes before the catalog +// half's first views are in is the bus state of the holder's manifest, read through `modules` (its digest) and `manifests`. +func TestHolderAcceptedReadsTheHoldersManifestFromItsFirstViews(t *testing.T) { + digest := meshcatalogclient.ManifestDigest([]byte(`{"module":"idx","bus-state":[{"name":"revisions","storage":"object-store","template":"objects","options":{"max-mib":8}}]}`)) + mod := func(d string) json.RawMessage { return json.RawMessage(`{"v":1,"manifest":"` + d + `"}`) } + man := func(text string) json.RawMessage { + b, _ := json.Marshal(map[string]any{"v": 1, "module": "idx", "manifest": text}) + return b + } + rows := []struct { + name string + modules, manifests map[string]json.RawMessage + err string + module string + }{ + {"read", map[string]json.RawMessage{"idx": mod(digest)}, map[string]json.RawMessage{digest: man(`{"module":"idx","bus-state":[{"name":"revisions","storage":"object-store","template":"objects","options":{"max-mib":8}}]}`)}, "", "idx"}, + {"no entry of the holder yet", map[string]json.RawMessage{}, nil, "holds no entry of idx", ""}, + {"its manifest is not in yet", map[string]json.RawMessage{"idx": mod(digest)}, map[string]json.RawMessage{}, "holds no manifest", ""}, + {"an entry that names no manifest", map[string]json.RawMessage{"idx": mod("")}, nil, "names no manifest", ""}, + } + for _, row := range rows { + t.Run(row.name, func(t *testing.T) { + a, err := holderAccepted("idx", row.modules, row.manifests) + if row.err != "" { + if err == nil || !strings.Contains(err.Error(), row.err) { + t.Fatalf("err %v, want it to contain %q", err, row.err) + } + return + } + if err != nil || a.Module != row.module || len(a.BusState) != 1 || a.BusState[0].Name != "revisions" { + t.Fatalf("got %+v, %v", a, err) + } + }) + } +} diff --git a/modules/nats/cmd/nats-tools/keep.go b/modules/nats/cmd/nats-tools/keep.go index 366aacda6..46a012b52 100644 --- a/modules/nats/cmd/nats-tools/keep.go +++ b/modules/nats/cmd/nats-tools/keep.go @@ -240,7 +240,7 @@ func runKeeper(ctx context.Context, getenv func(string) string, logOut io.Writer if err != nil { return err } - k.Catalog, k.Seats, k.Again = follow.view, follow.seats, follow.again + k.Catalog, k.Seats, k.Again, k.Holder = follow.view, follow.seats, follow.again, follow.holder // The process answers the seat mesh-broker's snapshot verbs itself (design 47 G §681, A-107), with the caller the bus proved. stopAnswers, err := conn.AnswerSeat(meshbroker.Seat, seatlist.Definition(func() *seatlist.Reader { return follow.seatList }, meshbroker.Seat), snapshotAnswers(k, heldRequest(k.Store), keeperSnapshots(nc, logger.Printf))) if err != nil { @@ -259,11 +259,15 @@ type catalogFollow struct { branches *meshcatalogclient.Branches seatList *seatlist.Reader routes *eventRoutes + // call asks a seat verb or a module tool as the keeper's account; holderModules and holderManifests are the seat + // mesh-catalog's states `modules` and `manifests` (design 47 G A-376). + call seatlist.HolderCaller + holderModules, holderManifests *views.Picture } // followCatalog wires the branch follower and the seat list, and asks their first views; moved is told of each change. func followCatalog(ctx context.Context, conn *process.Conn, writer views.WireWriter, moved func(), logf func(string, ...any)) (*catalogFollow, error) { - f := &catalogFollow{ctx: ctx, routes: &eventRoutes{by: map[string]func(jetstream.Msg){}}} + f := &catalogFollow{ctx: ctx, routes: &eventRoutes{by: map[string]func(jetstream.Msg){}}, call: conn.Call} f.reader = &views.ProcessReader{Module: keeper.Module, Call: conn.Call, Logf: logf, Findings: func() views.WireWriter { return writer }} branches, err := meshcatalogclient.FollowBranchesProcess(f.reader, conn.Call) if err != nil { @@ -283,6 +287,18 @@ func followCatalog(ctx context.Context, conn *process.Conn, writer views.WireWri if err != nil { logf("the seat list cannot be followed (%v): a module's sent store waits for it", err) } + // The catalog half's own holder (design 47 G A-376): its states `modules` and `manifests`, whichever module holds the seat. + holders := func(seat string) []string { + if who, err := seatlist.HolderOf(f.call, seat); err == nil { + return []string{who} + } + return nil + } + for state, into := range map[string]**views.Picture{"modules": &f.holderModules, "manifests": &f.holderManifests} { + if *into, err = f.reader.FollowSeat(catalog.Seat, state, holders); err != nil { + logf("the seat %s's state %s cannot be followed (%v): its holder's kept data waits for the catalog half", catalog.Seat, state, err) + } + } f.reader.Ask(ctx, nil) return f, nil } diff --git a/modules/nats/cmd/nats-tools/manifest_test.go b/modules/nats/cmd/nats-tools/manifest_test.go index a6330a88a..b7303f5a9 100644 --- a/modules/nats/cmd/nats-tools/manifest_test.go +++ b/modules/nats/cmd/nats-tools/manifest_test.go @@ -195,14 +195,16 @@ func TestTheBussManifestMountsItsProgramsIntoTheUpstreamImage(t *testing.T) { t.Errorf("claims are %+v: mesh-broker, serving its reading verbs %v and its snapshot verbs %v alone", nm.Claims, ReadingVerbs, SnapshotVerbs) } // The catalog half replaces manifest-accepted and indexer_bus_state (design 47 G §433, §614), and the seat list is read - // through the seat mesh-catalog (§537). + // through the seat mesh-catalog (§537); its holder's states modules and manifests, and the seat that places modules, are + // read before the first views (A-376). catalogHalf := []string{"mesh-catalog.catalog-revision", "mesh-catalog.registration-changed", "mesh-delivery.source-moved", - "mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed"} + "mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", "seat:mesh-catalog.views.modules", "seat:mesh-catalog.views.manifests"} if !slices.Equal(nm.Consumes, append(slices.Clone(catalogHalf), "mesh-conditions.findings-asked")) { t.Errorf("consumes %v: the catalog half's and the seat list's events %v, and the finding writer's findings-asked (design 47 G A-17)", nm.Consumes, catalogHalf) } for _, want := range []string{"seat:mesh-catalog.revision", "seat:mesh-catalog.registrations", "seat:mesh-delivery.source", - "seat:mesh-catalog.seat-definitions", "seat:mesh-hand-acts.hand-act"} { + "seat:mesh-catalog.seat-definitions", "seat:mesh-hand-acts.hand-act", "seat:mesh-catalog.modules", "seat:mesh-catalog.manifests", + "seat:mesh-module-assigner.assignments"} { if !slices.Contains(nm.Invokes, want) { t.Errorf("invokes %v lack %s", nm.Invokes, want) } diff --git a/modules/nats/internal/keeper/keeper.go b/modules/nats/internal/keeper/keeper.go index 68788bed4..7f980eca5 100644 --- a/modules/nats/internal/keeper/keeper.go +++ b/modules/nats/internal/keeper/keeper.go @@ -332,8 +332,12 @@ type Keeper struct { // branch has no first view yet, when the keeper plans nothing. Seats are the seat list's definitions, each resolved // with the parts it includes (§537). Again asks every first view of the catalog half again, by hand. Catalog func() (CatalogView, error) - Seats func() map[string]seatdef.Definition - Again func() error + // Holder reads the manifest on the trunk of the holder of the seat mesh-catalog, the catalog half's own source, as the + // indexer's first views `modules` and `manifests` give it (design 47 G A-376): what the keeper makes before the + // catalog half's first views are in. Nil where nothing reads it; an error while it cannot be read. + Holder func() (Accepted, error) + Seats func() map[string]seatdef.Definition + Again func() error // Act records a hand act through the seat mesh-hand-acts; Warrant reads the router's record of an ask. Act func(what, why, cause string) error Warrant func(askID string) (*oc.Ask, *oc.Warrant, error) @@ -463,6 +467,7 @@ func (k *Keeper) PlanCatalog() error { view, err := k.Catalog() if err != nil { k.setStatus(func(s *Status) { s.FirstView.Asked, s.FirstView.Error = at, err.Error() }) + k.planHolder() return err } k.mu.Lock() @@ -474,6 +479,7 @@ func (k *Keeper) PlanCatalog() error { k.setStatus(func(s *Status) { s.FirstView.Asked, s.FirstView.Error = at, "the catalog half lists no module yet (none-yet): nothing is planned or orphaned until a revision lists one" }) + k.planHolder() return nil } k.viewed = true @@ -530,6 +536,21 @@ func (k *Keeper) PlanCatalog() error { return nil } +// planHolder makes the entries of the holder of the seat mesh-catalog before the catalog half's first views are in (design 47 +// G A-376, which completes A-367): the catalog half's own owner needs its kept data to answer them, so it is read from the +// holder's manifest on the trunk and made as Heard makes any module's, each entry said as ever; every other module waits. +func (k *Keeper) planHolder() { + if k.Holder == nil || k.Viewed() { + return + } + a, err := k.Holder() + if err != nil { + k.logf("the holder of the seat %s is not read yet, so its kept data is not made before the catalog half's first views: %v", FirstViewOwner, err) + return + } + k.hear(a) +} + // drifts says each drift once (design 47 G §532): kept in the module's key, said again only when the branch or the // settings planned change. func (k *Keeper) drifts(ds []Drift) { @@ -588,6 +609,11 @@ func (k *Keeper) Heard(a Accepted) { k.logf("the entry of %s is heard before the catalog half's first views: nothing is planned until they are in", a.Module) return } + k.hear(a) +} + +// hear plans one module's entry (Heard, and planHolder before the first views). +func (k *Keeper) hear(a Accepted) { held, _, found := k.asked(a.Module) if !found { held = Asked{V: 1, Module: a.Module} diff --git a/modules/nats/internal/keeper/keeper_test.go b/modules/nats/internal/keeper/keeper_test.go index 500eb3d04..506b44980 100644 --- a/modules/nats/internal/keeper/keeper_test.go +++ b/modules/nats/internal/keeper/keeper_test.go @@ -575,6 +575,35 @@ func TestTheKeeperMakesWhatAVerdictAsksForAndSaysIt(t *testing.T) { t.Error("the first revision with entries is viewed and plans as before") } }}, + // A-376: before the first views are in (read refused, or none-yet) the holder of the seat mesh-catalog is made from its + // trunk manifest, each entry said as ever; every other module waits. + {"before the first views the catalog half's own holder is made and no other module", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) { + k.mu.Lock() + k.viewed = false + k.mu.Unlock() + k.Heard(Accepted{Module: "other", Revision: 1, BusState: []catalogue.BusStateEntry{drafts}}) + k.Holder = func() (Accepted, error) { + return Accepted{Module: "idx", BusState: []catalogue.BusStateEntry{drafts}}, nil + } + for _, view := range []func() (CatalogView, error){ + func() (CatalogView, error) { return CatalogView{}, errors.New("no first view yet") }, + func() (CatalogView, error) { return CatalogView{}, nil }, + } { + k.Catalog = view + k.PlanCatalog() + if _, ok := k.planned("idx"); !ok || fmt.Sprint(out.events) != "[created idx drafts]" { + t.Fatalf("planned idx %v, said %v: the holder's entry is made once", ok, out.events) + } + if _, ok := k.planned("other"); ok || k.Viewed() { + t.Errorf("another module is planned or the keeper counts as viewed") + } + } + k.Holder = func() (Accepted, error) { return Accepted{}, errors.New("not read") } + k.PlanCatalog() // a holder not read plans nothing and says why in the status + if fmt.Sprint(out.events) != "[created idx drafts]" { + t.Errorf("said %v", out.events) + } + }}, // §538: a module in no watched branch's newest revision has its plans key deleted, its entries then orphans. {"a module no watched branch lists loses its plan, its entries orphans", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) { k.Catalog = func() (CatalogView, error) { diff --git a/modules/nats/module.json b/modules/nats/module.json index ab20bcd8a..549f43114 100644 --- a/modules/nats/module.json +++ b/modules/nats/module.json @@ -65,7 +65,10 @@ "seat:mesh-catalog.revision", "seat:mesh-catalog.registrations", "seat:mesh-delivery.source", - "seat:mesh-catalog.seat-definitions" + "seat:mesh-catalog.seat-definitions", + "seat:mesh-catalog.modules", + "seat:mesh-catalog.manifests", + "seat:mesh-module-assigner.assignments" ], "consumes": [ "mesh-catalog.catalog-revision", @@ -73,6 +76,8 @@ "mesh-delivery.source-moved", "mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", + "seat:mesh-catalog.views.modules", + "seat:mesh-catalog.views.manifests", "mesh-conditions.findings-asked" ], "uses": [ From 5df0fb64f5224bebb153a676fc2fb7efd994a8c9 Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 14 Nov 2026 11:52:23 +0100 Subject: [PATCH 03/24] mesh-build-go: a finding writer that cannot start is said, and the module serves on (A-394) Live on shanks the bundle exited 39 times on 'an ordered consumer of mesh-conditions.findings-asked: context deadline exceeded', so the seat node-build-go never answered steps and the seed waited. Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- modules/mesh-build-go/cmd/mesh-build-go/main.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/modules/mesh-build-go/cmd/mesh-build-go/main.go b/modules/mesh-build-go/cmd/mesh-build-go/main.go index 1c404a996..7886aaca6 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/main.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/main.go @@ -87,8 +87,9 @@ func main() { go func() { <-stdio.Served() if err := findings.Start(ctx); err != nil { - logf("the finding writer: %v", err) - os.Exit(1) + // Said, and the module goes on serving: a finding writer that cannot hear its asks keeps its findings + // in its state alone, as the indexer does (lead answer A-394); its builds and steps never wait on it. + logf("the finding writer cannot start, its findings are kept in the state alone: %v", err) } }() } From 5236c2aa7263bc6591ea6ee5ccdf1d53a3b00b33 Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 14 Nov 2026 14:57:27 +0100 Subject: [PATCH 04/24] gitea: the seat git's branch with no branch named answers the repository's default branch (A-399) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The seed and a registration with no branches read the default branch through the seat git (design 47 G-1 §104) by calling branch with an empty branch; gitea refused it as no branch for a path, and the seed stopped on novox 2026-11-14. --- modules/gitea/seat.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/modules/gitea/seat.ts b/modules/gitea/seat.ts index 8cd08bd95..1c0c97124 100644 --- a/modules/gitea/seat.ts +++ b/modules/gitea/seat.ts @@ -152,7 +152,10 @@ export function seatTools(gitea: GiteaClient, deps: SeatDeps): ToolDefinition[] verb(GIT_VERBS.branch, "The seat git's branch: a branch's head, and whether a protection rule refuses a direct push and admits only merges.", async (args): Promise => { const { owner, repo, full } = repositoryOf(args.repository); - const branch = String(args.branch ?? ""); + // No branch named is the repository's default branch, read from the forge, as the verb file reads it (A-399). + let branch = String(args.branch ?? ""); + if (branch === "") branch = String((await gitea.getRepo(owner, repo)).default_branch ?? ""); + if (branch === "") throw new Error(`the forge names no default branch of ${full}`); // The head first: a branch the forge does not hold is its 404, the kind not-found, never an unprotected branch. const head = await gitea.branchHead(owner, repo, branch); return { repository: full, branch, head, protected: pushRefused(await gitea.appliedProtection(owner, repo, branch)) }; From fb24d384a2ae56a44e39ed1c68c2814f86b38b70 Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 14 Nov 2026 19:00:01 +0100 Subject: [PATCH 05/24] Build seat: says each toolchain-reported it said, not only the ones it could not --- modules/mesh-build-go/cmd/mesh-build-go/toolchainreport.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/mesh-build-go/cmd/mesh-build-go/toolchainreport.go b/modules/mesh-build-go/cmd/mesh-build-go/toolchainreport.go index 4ddfe4fd2..bee85bf34 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/toolchainreport.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/toolchainreport.go @@ -74,6 +74,8 @@ func putReporting(put steprun.Put, r *toolchainReporter, digest func(version str } if eerr := r.Report(key, rev, t); eerr != nil && say != nil { say("toolchain-reported could not be said: %v", eerr) + } else if eerr == nil && say != nil { + say("toolchain-reported said: %s %s (key %s, revision %d)", t.Language, t.Version, key, rev) } return rev, nil } From 8ea8ef9768c0b3042b9d06fcc74296308d0eb0f9 Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 14 Nov 2026 22:08:22 +0100 Subject: [PATCH 06/24] Gate: a watched branch starts from the forge, as a clone does; tool gate_source_start (A-401) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit At its start and on each registration-changed of the seat mesh-catalog, the gate writes source. for every watched branch the registrations name that no source key holds, each registered repository at its head from the seat git's verb branch, group empty, then says source-moved; a repository whose head cannot be read is left out and said; a held key is never rewritten. gate_source_start (branch, why) does it by hand through the hand-act log. The manifest consumes registration-changed and invokes seat:git.branch and seat:mesh-catalog.registrations. Builds: design 47 G A-401, §430, §15, A-34 --- .../cmd/mesh-delivery-gate/deliveries.go | 2 + .../cmd/mesh-delivery-gate/deliveries_test.go | 11 + .../cmd/mesh-delivery-gate/gate.go | 13 +- .../cmd/mesh-delivery-gate/main.go | 22 +- .../cmd/mesh-delivery-gate/manifest_test.go | 2 +- .../cmd/mesh-delivery-gate/sourcestart.go | 200 ++++++++++++++++++ .../cmd/mesh-delivery-gate/sourcetools.go | 10 + .../mesh-delivery-gate/sourcetools_test.go | 109 ++++++++++ modules/mesh-delivery-gate/module.json | 6 +- 9 files changed, 367 insertions(+), 8 deletions(-) create mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/sourcestart.go diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/deliveries.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/deliveries.go index 24780ce18..42f700bd5 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/deliveries.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/deliveries.go @@ -69,6 +69,8 @@ type DeliveryStore interface { type ForgeReads interface { OpenPulls() (gitseat.OpenPullsAnswer, error) Pull(repository string, number int64) (gitseat.PullAnswer, error) + // Branch is a repository branch's head (the seat git's verb branch). + Branch(repository, branch string) (gitseat.BranchAnswer, error) } // kvKept is the store over the runtime. diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/deliveries_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/deliveries_test.go index 7710e6fe1..7fd363538 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/deliveries_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/deliveries_test.go @@ -41,6 +41,7 @@ func (k *memKept) Delete(state, key string) error { delete(k.m[state], key); ret type fakeForge struct { open []gitseat.OpenPull pulls map[string]gitseat.Pull + heads map[string]string // "@" -> head fail bool } @@ -50,6 +51,16 @@ func (f *fakeForge) OpenPulls() (gitseat.OpenPullsAnswer, error) { } return gitseat.OpenPullsAnswer{Pulls: f.open}, nil } +func (f *fakeForge) Branch(repository, branch string) (gitseat.BranchAnswer, error) { + if f.fail { + return gitseat.BranchAnswer{}, errors.New("the forge is away") + } + head, ok := f.heads[repository+"@"+branch] + if !ok { + return gitseat.BranchAnswer{Error: "no such branch"}, nil + } + return gitseat.BranchAnswer{Branch: branch, Repository: repository, Head: head}, nil +} func (f *fakeForge) Pull(repository string, number int64) (gitseat.PullAnswer, error) { p, ok := f.pulls[fmt.Sprintf("%s#%d", repository, number)] if !ok { diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/gate.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/gate.go index 94de1c9b7..05222b46c 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/gate.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/gate.go @@ -23,11 +23,14 @@ type Holder struct { // Merges and Old are the old sections the switch reads (switch.go); Source is the gate's source (merges.go). Merges MergeStore // Kept is the gate's states deliveries and groups, and Forge the seat git's reads (deliveries.go, §716). - Kept DeliveryStore - Forge ForgeReads - dmu sync.Mutex - Old OldState - Source SourceStore + Kept DeliveryStore + Forge ForgeReads + // Catalog reads the seat mesh-catalog's registrations (sourcestart.go); startMu keeps one start of the sources at a time. + Catalog Registered + startMu sync.Mutex + dmu sync.Mutex + Old OldState + Source SourceStore // Owed says what the gate owes the forge as the seat's event owed (owed.go). Owed OwedSayer // Records keeps the delivery records on the gate's ledger (record.go, ADR 0656 §7); nil keeps none. diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go index 60984b018..b9bc97998 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go @@ -32,7 +32,7 @@ func (l *listening) get() string { l.mu.Lock(); defer l.mu.Unlock(); return l.n // The events this owner follows: the forge bridge's word on an act owed, and the step events the delivery records are built // from (consumer.go). -var followed = append(append([]string{owedEndedEvent}, stepPatterns()...), pullUpdatedEvent, pullMergedEvent, pullClosedEvent) +var followed = append(append([]string{owedEndedEvent}, stepPatterns()...), pullUpdatedEvent, pullMergedEvent, pullClosedEvent, registrationChangedEvent) func main() { settingsFile := os.Getenv("MESH_DELIVERY_GATE_SETTINGS") @@ -68,6 +68,7 @@ func main() { h.Old = kvOld{} h.Kept = kvKept{} h.Forge = seatGit{ask: stdio.Ask} + h.Catalog = seatCatalog{ask: stdio.Ask} h.Consumer = &StepConsumer{Records: h.Records, Families: stepFamilies, Now: h.Now, Logf: logf, Taken: h.TakenIn} // The gate's one held writer of its own findings (design 47 G A-39, A-37, A-40): made at once, started once the runtime serves. fw, err := NewFindingWriter() @@ -116,12 +117,27 @@ func run(h *Holder, l *listening) { } l.set("listening") logf("[mesh-delivery] listening: %s", strings.Join(followed, ", ")) + // A watched branch no source key holds starts from the forge, once at each start and at each registration change (A-401). + go startSourcesAtStart(h) // A missed pull event is put right from the forge's own record, once at each start (§716). if err := h.ReadForge(); err != nil { logf("[mesh-delivery] the open pull requests were not read from the forge, they are read again at the next start: %v", err) } } +// startSourcesAtStart starts the sources of the watched branches no key holds, retrying until the registrations can be read. +func startSourcesAtStart(h *Holder) { + for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { + started, err := h.StartSources() + if err == nil { + logf("[mesh-delivery] the sources started from the forge at the start: %v", started) + return + } + logf("[mesh-delivery] the sources cannot be started yet (%v); asking again in %s", err, wait) + time.Sleep(wait) + } +} + // switchToRecords carries the groups in flight into their delivery records (switch.go), retrying until it can: no event is // taken before, so no outcome reaches a record the switch has not opened yet. func switchToRecords(h *Holder, l *listening) { @@ -172,6 +188,10 @@ func take(h *Holder, e stdio.Envelope) error { if e.Key == pullMergedEvent { return errors.Join(h.DeliveryMerged(e.Body), h.pullEnded(e.Body, "merged")) } + if e.Key == registrationChangedEvent { + _, err := h.StartSources() + return err + } if e.Key == pullClosedEvent { return errors.Join(h.DeliveryClosed(e.Body), h.pullEnded(e.Body, "closed")) } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go index a679e1fa7..7e060757d 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go @@ -181,7 +181,7 @@ func TestItClaimsTheDeliverySeatAndFollowsWhatItHandles(t *testing.T) { } // It calls no controller verb and no forge verb: the forge's reads and writes are the bridge's (ADR 0437 §8; design 47 §592). // It uses operator-channel only to say its conditions (ADR 0437 §9, issue 1275): the teller asks no more (§431). - want := []string{"seat:git.open-pulls", "seat:git.pull", "seat:mesh-hand-acts.hand-act", WantsEntryVerb, EngineCompositionVerb} + want := []string{"seat:git.branch", "seat:git.open-pulls", "seat:git.pull", "seat:mesh-catalog.registrations", "seat:mesh-hand-acts.hand-act", WantsEntryVerb, EngineCompositionVerb} if !reflect.DeepEqual(m.Uses, []string{"operator-channel"}) { t.Fatalf("uses %v: operator-channel, for its conditions alone", m.Uses) } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/sourcestart.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/sourcestart.go new file mode 100644 index 000000000..b264da7a7 --- /dev/null +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/sourcestart.go @@ -0,0 +1,200 @@ +package main + +// A watched branch starts from the forge, as a clone does (design 47 G A-401, §430, §15): at its start and on each +// registration-changed of the seat mesh-catalog, the gate writes source. for every watched branch the registrations name, +// each registered repository at the head of that branch, asked of the seat git's verb branch, the group left empty, and says +// source-moved. A repository whose head cannot be read is left out and said. A head a key holds is never rewritten here: from +// then on only a take-in moves it (A-34). + +import ( + "encoding/json" + "errors" + "fmt" + "sort" + "strconv" + "strings" + + "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" + gitseat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git" + meshcatalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" + "git.novox.be/novox/mesh-sdk/go/keys" +) + +// registrationChangedEvent is the seat mesh-catalog's event, as the gate's manifest consumes it. +var registrationChangedEvent = declared(meshcatalog.EventRegistrationChanged) + +// Registered is what the gate reads of the seat mesh-catalog: its registrations. +type Registered interface { + Registrations() (meshcatalog.RegistrationsAnswer, error) +} + +// seatCatalog reads the seat mesh-catalog through the tool runner. +type seatCatalog struct{ ask asking } + +func (s seatCatalog) Registrations() (meshcatalog.RegistrationsAnswer, error) { + var a meshcatalog.RegistrationsAnswer + raw, err := s.ask("seat:"+meshcatalog.Seat+"."+meshcatalog.VerbRegistrations, meshcatalog.RegistrationsInput{}) + if err != nil { + return a, err + } + answer, said, ok := answerOf(raw) + if !ok { + return a, fmt.Errorf("registrations refused: %s", lastLine(said)) + } + if err := json.Unmarshal(answer, &a); err != nil { + return a, fmt.Errorf("the answer of registrations is not readable: %w", err) + } + return a, nil +} + +// Branch is the seat git's verb branch: a repository's branch head. +func (s seatGit) Branch(repository, branch string) (gitseat.BranchAnswer, error) { + var a gitseat.BranchAnswer + return a, s.call(gitseat.VerbBranch, gitseat.BranchInput{Repository: repository, Branch: branch}, &a) +} + +// watchedRepositories are the repositories each watched branch is registered for, each once, sorted. +func watchedRepositories(regs []meshcatalog.Registration) map[string][]string { + seen := map[string]map[string]bool{} + for _, r := range regs { + if strings.TrimSpace(r.Origin) == "" { + continue + } + for _, b := range r.Branches { + if strings.TrimSpace(b) == "" { + continue + } + if seen[b] == nil { + seen[b] = map[string]bool{} + } + seen[b][r.Origin] = true + } + } + out := map[string][]string{} + for b, rs := range seen { + for r := range rs { + out[b] = append(out[b], r) + } + sort.Strings(out[b]) + } + return out +} + +// StartSources writes the source key of every watched branch the registrations name that no key holds yet, and answers the +// branches it wrote. A registrations answer that cannot be read is an error (the event is offered again); a repository whose +// head cannot be read is said and left out. +func (h *Holder) StartSources() ([]string, error) { + started, _, err := h.startSources("") + return started, err +} + +// startSources starts the branches no key holds, or only branch when it is named; it answers the branches written and the +// repositories left out as "@". +func (h *Holder) startSources(only string) (started, leftOut []string, err error) { + if h.Source == nil || h.Catalog == nil || h.Forge == nil { + return nil, nil, errors.New("this gate keeps no source, reads no registrations or reads no forge") + } + h.startMu.Lock() + defer h.startMu.Unlock() + regs, err := h.Catalog.Registrations() + if err != nil { + return nil, nil, fmt.Errorf("the registrations cannot be read: %w", err) + } + watched := watchedRepositories(regs.Registrations) + if only != "" { + if _, ok := watched[only]; !ok { + return nil, nil, fmt.Errorf("no registration names the branch %s", only) + } + } + names := make([]string, 0, len(watched)) + for b := range watched { + if only == "" || b == only { + names = append(names, b) + } + } + sort.Strings(names) + var errs []error + for _, branch := range names { + held, _, err := h.Source.Read(branch) + if err != nil { + errs = append(errs, err) + continue + } + if held != nil { + continue + } + heads := map[string]string{} + for _, repository := range watched[branch] { + a, err := h.Forge.Branch(repository, branch) + if err == nil && a.Error != "" { + err = errors.New(a.Error) + } + if err == nil && a.Head == "" { + err = errors.New("the forge named no head") + } + if err != nil { + leftOut = append(leftOut, repository+"@"+branch) + h.logf("[mesh-delivery] source.%s starts without %s: its head cannot be read (%v); it is added at the next registration change or by gate_source_start", branch, repository, err) + continue + } + heads[repository] = a.Head + } + if len(heads) == 0 { + continue + } + wrote := false + rev, err := h.Source.Set(branch, func(prev *Source) (Source, bool) { + wrote = prev == nil // a key written meanwhile is never rewritten + return Source{Heads: heads}, wrote + }) + if err != nil { + errs = append(errs, fmt.Errorf("source.%s cannot be started: %w", branch, err)) + continue + } + if !wrote { + continue + } + started = append(started, branch) + if h.SaySeat == nil { + continue + } + at := h.Now().UTC() + if err := h.SaySeat(seat.SubjectSourceMoved(keys.EscapeName(branch)), + seat.SourceMovedEvent{Branch: branch, Revision: int64(rev), At: &at}, + "source-moved."+keys.EscapeName(branch)+"."+strconv.FormatUint(rev, 10)); err != nil { + errs = append(errs, err) + } + } + return started, leftOut, errors.Join(errs...) +} + +func (h *Holder) logf(format string, a ...any) { + if h.Logf != nil { + h.Logf(format, a...) + } +} + +// SourceStart starts one watched branch by hand, after the hand-act log took it: the same as the start does, for a branch no +// key holds yet. +func (h *Holder) SourceStart(branch, why, by string, acts handActs) (map[string]any, error) { + if strings.TrimSpace(branch) == "" || strings.TrimSpace(why) == "" { + return nil, errors.New("say the branch and why its source is started") + } + if h.Source == nil || h.Catalog == nil || h.Forge == nil { + return nil, errors.New("this gate keeps no source, reads no registrations or reads no forge") + } + if acts == nil { + return nil, errors.New("the hand-act log cannot be reached here, so nothing was started") + } + if _, err := acts.Record(meshhandactsclient.Request{What: "started the source of the branch " + branch, + Why: why, Cause: "gate_source_start", Verb: "source-start"}); err != nil { + return nil, fmt.Errorf("the hand act was not recorded through mesh-hand-acts, so nothing was started: %w", err) + } + started, leftOut, err := h.startSources(branch) + out := map[string]any{"branch": branch, "started": started, "left-out": leftOut, "by": by} + if err == nil && len(started) == 0 && len(leftOut) == 0 { + out["note"] = "the gate holds a source key of the branch already: only a take-in moves it" + } + return out, err +} diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/sourcetools.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/sourcetools.go index 68c982da0..6d3f7586c 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/sourcetools.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/sourcetools.go @@ -79,5 +79,15 @@ func sourceTools(h *Holder, acts handActs) []stdio.Tool { Run: func(a map[string]any) (any, error) { return h.SourceAgain(strArg(a, "branch"), strArg(a, "why"), stdio.CallerOf(a), acts) }}, + {Name: "gate_source_start", + Description: "Start the source of one watched branch the registrations name and no source key holds yet, from the forge: each " + + "registered repository at the branch's head, the group empty, then source-moved; recorded in the hand-act log of the seat " + + "mesh-hand-acts first. A key the gate holds is never rewritten.", + Input: map[string]any{"type": "object", "properties": map[string]any{ + "branch": str("the watched branch"), "why": str("why it is started by hand, in plain words")}, + "required": []string{"branch", "why"}}, + Run: func(a map[string]any) (any, error) { + return h.SourceStart(strArg(a, "branch"), strArg(a, "why"), stdio.CallerOf(a), acts) + }}, } } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/sourcetools_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/sourcetools_test.go index 2094ff7b5..758645bfc 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/sourcetools_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/sourcetools_test.go @@ -2,7 +2,10 @@ package main import ( "errors" + "fmt" + meshcatalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" + "reflect" "strings" "testing" "time" @@ -103,3 +106,109 @@ func TestTheSourceAnswerNamesTheBranchItWasAskedAndStatusSaysEveryBranch(t *test t.Fatalf("status says every branch's source: %+v", h.SourceAll()) } } + +type fakeCatalog struct { + regs []meshcatalog.Registration + err error +} + +func (f fakeCatalog) Registrations() (meshcatalog.RegistrationsAnswer, error) { + return meshcatalog.RegistrationsAnswer{Registrations: f.regs}, f.err +} + +// Design 47 G A-401: a watched branch starts from the forge as a clone does: each registered repository at the branch's head, +// the group empty, source-moved said; a repository whose head cannot be read is left out; a key held is never rewritten. +func TestAWatchedBranchStartsFromTheForge(t *testing.T) { + regs := []meshcatalog.Registration{ + {Module: "car-hunter", Origin: "o/car-hunter", Branches: []string{"main"}}, + {Module: "mesh-a", Origin: "o/mesh", Branches: []string{"main"}}, + {Module: "mesh-b", Origin: "o/mesh", Branches: []string{"main"}}, // the same repository twice: once + {Module: "f1", Origin: "o/f1-circuits", Branches: []string{"master"}}, + {Module: "l2p", Origin: "o/link2pay", Branches: []string{"nox-mesh"}}, + {Module: "ph", Origin: "o/photos", Branches: []string{"nox-mesh", "main"}}, + } + forge := &fakeForge{heads: map[string]string{ + "o/car-hunter@main": "c1", "o/mesh@main": "c2", "o/f1-circuits@master": "c3", "o/link2pay@nox-mesh": "c4", + "o/photos@main": "c6", // o/photos@nox-mesh cannot be read + }} + for _, c := range []struct { + name string + held map[string]Source + catalog fakeCatalog + only string + started []string + heads map[string]map[string]string + events int + wantErr bool + }{ + {name: "every branch no key holds", catalog: fakeCatalog{regs: regs}, + started: []string{"main", "master", "nox-mesh"}, events: 3, + heads: map[string]map[string]string{"main": {"o/car-hunter": "c1", "o/mesh": "c2", "o/photos": "c6"}, + "master": {"o/f1-circuits": "c3"}, "nox-mesh": {"o/link2pay": "c4"}}}, + {name: "a held key is never rewritten", catalog: fakeCatalog{regs: regs}, + held: map[string]Source{"main": {Group: "g", Heads: map[string]string{"o/mesh": "old"}}}, + started: []string{"master", "nox-mesh"}, events: 2, + heads: map[string]map[string]string{"main": {"o/mesh": "old"}, "master": {"o/f1-circuits": "c3"}, "nox-mesh": {"o/link2pay": "c4"}}}, + {name: "by hand, one branch", catalog: fakeCatalog{regs: regs}, only: "master", + started: []string{"master"}, events: 1, heads: map[string]map[string]string{"master": {"o/f1-circuits": "c3"}}}, + {name: "registrations unreadable", catalog: fakeCatalog{err: errors.New("away")}, wantErr: true}, + {name: "no registration names the branch", catalog: fakeCatalog{regs: regs}, only: "nope", wantErr: true}, + } { + t.Run(c.name, func(t *testing.T) { + src := &branchSource{memSource{vals: map[string]Source{}}} + for b, s := range c.held { + src.vals[b] = s + } + events := 0 + var logged []string + h := &Holder{Now: time.Now, Source: src, Forge: forge, Catalog: c.catalog, + Logf: func(f string, a ...any) { logged = append(logged, fmt.Sprintf(f, a...)) }, + SaySeat: func(subject string, body any, id string) error { + events++ + if ev := body.(seat.SourceMovedEvent); ev.Group != "" { + t.Errorf("the group is empty, got %q", ev.Group) + } + return nil + }} + started, _, err := h.startSources(c.only) + if (err != nil) != c.wantErr { + t.Fatalf("err %v", err) + } + if c.wantErr { + return + } + if !reflect.DeepEqual(started, c.started) || events != c.events { + t.Fatalf("started %v (%d events), want %v (%d)", started, events, c.started, c.events) + } + for b, want := range c.heads { + if got := src.vals[b].Heads; !reflect.DeepEqual(got, want) { + t.Errorf("source.%s heads %v, want %v", b, got, want) + } + } + if c.only == "" && c.held == nil && !strings.Contains(strings.Join(logged, "\n"), "o/photos") { + t.Errorf("the repository left out was not said: %v", logged) + } + }) + } +} + +// Design 47 G A-401: gate_source_start is recorded in the hand-act log first, and starts nothing when the log refuses it. +func TestSourceStartByHandIsRecordedFirst(t *testing.T) { + src := &branchSource{memSource{vals: map[string]Source{}}} + h := &Holder{Now: time.Now, Source: src, Forge: &fakeForge{heads: map[string]string{"o/f1@master": "c3"}}, + Catalog: fakeCatalog{regs: []meshcatalog.Registration{{Module: "f1", Origin: "o/f1", Branches: []string{"master"}}}}} + acts := &handActsFake{refuse: errors.New("log down")} + if _, err := h.SourceStart("master", "newly watched", "me", acts); err == nil || len(src.vals) != 0 { + t.Fatalf("started without the log: %v %v", err, src.vals) + } + acts.refuse = nil + if _, err := h.SourceStart("master", "", "me", acts); err == nil { + t.Fatal("started with no reason") + } + if _, err := h.SourceStart("master", "newly watched", "me", acts); err != nil { + t.Fatal(err) + } + if len(acts.took) != 1 || acts.took[0].Cause != "gate_source_start" || src.vals["master"].Heads["o/f1"] != "c3" { + t.Fatalf("log %+v source %+v", acts.took, src.vals) + } +} diff --git a/modules/mesh-delivery-gate/module.json b/modules/mesh-delivery-gate/module.json index aad02e5cf..2d170922d 100644 --- a/modules/mesh-delivery-gate/module.json +++ b/modules/mesh-delivery-gate/module.json @@ -22,6 +22,7 @@ "git.pull-updated", "git.pull-merged", "git.pull-closed", + "mesh-catalog.registration-changed", "mesh-conditions.findings-asked" ], "emits": [ @@ -351,8 +352,10 @@ "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg": "0.1" }, "invokes": [ + "seat:git.branch", "seat:git.open-pulls", "seat:git.pull", + "seat:mesh-catalog.registrations", "seat:mesh-hand-acts.hand-act", "seat:mesh-wants.entry", "seat:node-engine.composition" @@ -379,7 +382,8 @@ "gate_record_repair", "gate_records", "gate_source", - "gate_source_again" + "gate_source_again", + "gate_source_start" ], "config": { "record-retention": { From 28f616e3794f18d00afa446f18f2034f4026173b Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 14 Nov 2026 22:09:06 +0100 Subject: [PATCH 07/24] Gate: a held source key gets the registered repositories it lacks, heads unchanged (A-403) Builds: design 47 G A-403, A-401, A-34 --- .../cmd/mesh-delivery-gate/sourcestart.go | 39 +++++++++++++------ .../mesh-delivery-gate/sourcetools_test.go | 18 ++++++--- 2 files changed, 40 insertions(+), 17 deletions(-) diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/sourcestart.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/sourcestart.go index b264da7a7..6bca6972e 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/sourcestart.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/sourcestart.go @@ -3,7 +3,7 @@ package main // A watched branch starts from the forge, as a clone does (design 47 G A-401, §430, §15): at its start and on each // registration-changed of the seat mesh-catalog, the gate writes source. for every watched branch the registrations name, // each registered repository at the head of that branch, asked of the seat git's verb branch, the group left empty, and says -// source-moved. A repository whose head cannot be read is left out and said. A head a key holds is never rewritten here: from +// source-moved. A repository whose head cannot be read is left out and said. A head a key holds is never changed here (A-403): from // then on only a take-in moves it (A-34). import ( @@ -81,7 +81,7 @@ func watchedRepositories(regs []meshcatalog.Registration) map[string][]string { return out } -// StartSources writes the source key of every watched branch the registrations name that no key holds yet, and answers the +// StartSources adds to the source key of every watched branch the registrations name each registered repository it lacks, and answers the // branches it wrote. A registrations answer that cannot be read is an error (the event is offered again); a repository whose // head cannot be read is said and left out. func (h *Holder) StartSources() ([]string, error) { @@ -121,11 +121,13 @@ func (h *Holder) startSources(only string) (started, leftOut []string, err error errs = append(errs, err) continue } - if held != nil { - continue - } heads := map[string]string{} for _, repository := range watched[branch] { + if held != nil { + if _, has := held.Heads[repository]; has { + continue // a head a key holds moves by a take-in alone (A-34) + } + } a, err := h.Forge.Branch(repository, branch) if err == nil && a.Error != "" { err = errors.New(a.Error) @@ -143,10 +145,24 @@ func (h *Holder) startSources(only string) (started, leftOut []string, err error if len(heads) == 0 { continue } - wrote := false + wrote, group := false, "" rev, err := h.Source.Set(branch, func(prev *Source) (Source, bool) { - wrote = prev == nil // a key written meanwhile is never rewritten - return Source{Heads: heads}, wrote + next := Source{Heads: map[string]string{}} + if prev != nil { + next.Group = prev.Group + for r, c := range prev.Heads { + next.Heads[r] = c + } + } + wrote = false + for r, c := range heads { + if _, has := next.Heads[r]; !has { // a held head is never changed + next.Heads[r] = c + wrote = true + } + } + group = next.Group + return next, wrote }) if err != nil { errs = append(errs, fmt.Errorf("source.%s cannot be started: %w", branch, err)) @@ -161,7 +177,7 @@ func (h *Holder) startSources(only string) (started, leftOut []string, err error } at := h.Now().UTC() if err := h.SaySeat(seat.SubjectSourceMoved(keys.EscapeName(branch)), - seat.SourceMovedEvent{Branch: branch, Revision: int64(rev), At: &at}, + seat.SourceMovedEvent{Branch: branch, Group: group, Revision: int64(rev), At: &at}, "source-moved."+keys.EscapeName(branch)+"."+strconv.FormatUint(rev, 10)); err != nil { errs = append(errs, err) } @@ -175,8 +191,7 @@ func (h *Holder) logf(format string, a ...any) { } } -// SourceStart starts one watched branch by hand, after the hand-act log took it: the same as the start does, for a branch no -// key holds yet. +// SourceStart starts one watched branch by hand, after the hand-act log took it: the same as the start does. func (h *Holder) SourceStart(branch, why, by string, acts handActs) (map[string]any, error) { if strings.TrimSpace(branch) == "" || strings.TrimSpace(why) == "" { return nil, errors.New("say the branch and why its source is started") @@ -194,7 +209,7 @@ func (h *Holder) SourceStart(branch, why, by string, acts handActs) (map[string] started, leftOut, err := h.startSources(branch) out := map[string]any{"branch": branch, "started": started, "left-out": leftOut, "by": by} if err == nil && len(started) == 0 && len(leftOut) == 0 { - out["note"] = "the gate holds a source key of the branch already: only a take-in moves it" + out["note"] = "the key names every registered repository already: only a take-in moves a head" } return out, err } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/sourcetools_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/sourcetools_test.go index 758645bfc..7a3c52382 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/sourcetools_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/sourcetools_test.go @@ -145,10 +145,18 @@ func TestAWatchedBranchStartsFromTheForge(t *testing.T) { started: []string{"main", "master", "nox-mesh"}, events: 3, heads: map[string]map[string]string{"main": {"o/car-hunter": "c1", "o/mesh": "c2", "o/photos": "c6"}, "master": {"o/f1-circuits": "c3"}, "nox-mesh": {"o/link2pay": "c4"}}}, - {name: "a held key is never rewritten", catalog: fakeCatalog{regs: regs}, + {name: "a held key lacking repositories gets them, its heads unchanged", catalog: fakeCatalog{regs: regs}, held: map[string]Source{"main": {Group: "g", Heads: map[string]string{"o/mesh": "old"}}}, - started: []string{"master", "nox-mesh"}, events: 2, - heads: map[string]map[string]string{"main": {"o/mesh": "old"}, "master": {"o/f1-circuits": "c3"}, "nox-mesh": {"o/link2pay": "c4"}}}, + started: []string{"main", "master", "nox-mesh"}, events: 3, + heads: map[string]map[string]string{"main": {"o/mesh": "old", "o/car-hunter": "c1", "o/photos": "c6"}, "master": {"o/f1-circuits": "c3"}, "nox-mesh": {"o/link2pay": "c4"}}}, + {name: "a held key lacking one repository", catalog: fakeCatalog{regs: regs[:2]}, + held: map[string]Source{"main": {Group: "g", Heads: map[string]string{"o/mesh": "old"}}}, + started: []string{"main"}, events: 1, + heads: map[string]map[string]string{"main": {"o/mesh": "old", "o/car-hunter": "c1"}}}, + {name: "a held key naming all: nothing written or said", catalog: fakeCatalog{regs: regs[:2]}, + held: map[string]Source{"main": {Group: "g", Heads: map[string]string{"o/mesh": "old", "o/car-hunter": "older"}}}, + events: 0, + heads: map[string]map[string]string{"main": {"o/mesh": "old", "o/car-hunter": "older"}}}, {name: "by hand, one branch", catalog: fakeCatalog{regs: regs}, only: "master", started: []string{"master"}, events: 1, heads: map[string]map[string]string{"master": {"o/f1-circuits": "c3"}}}, {name: "registrations unreadable", catalog: fakeCatalog{err: errors.New("away")}, wantErr: true}, @@ -165,8 +173,8 @@ func TestAWatchedBranchStartsFromTheForge(t *testing.T) { Logf: func(f string, a ...any) { logged = append(logged, fmt.Sprintf(f, a...)) }, SaySeat: func(subject string, body any, id string) error { events++ - if ev := body.(seat.SourceMovedEvent); ev.Group != "" { - t.Errorf("the group is empty, got %q", ev.Group) + if ev := body.(seat.SourceMovedEvent); ev.Group != c.held[ev.Branch].Group { + t.Errorf("the group is the key's own (empty for a new key), got %q", ev.Group) } return nil }} From 89397466aa62c9f4736d78537eb90df02271f79e Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 14 Nov 2026 22:16:30 +0100 Subject: [PATCH 08/24] Gate: source-moved for started heads names no group; tests for the registration-changed branch and the start pass (A-401, A-404) Builds: design 47 G A-401, A-404 --- .../cmd/mesh-delivery-gate/sourcestart.go | 5 +-- .../mesh-delivery-gate/sourcetools_test.go | 43 ++++++++++++++++++- 2 files changed, 43 insertions(+), 5 deletions(-) diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/sourcestart.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/sourcestart.go index 6bca6972e..dbd772a1a 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/sourcestart.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/sourcestart.go @@ -145,7 +145,7 @@ func (h *Holder) startSources(only string) (started, leftOut []string, err error if len(heads) == 0 { continue } - wrote, group := false, "" + wrote := false rev, err := h.Source.Set(branch, func(prev *Source) (Source, bool) { next := Source{Heads: map[string]string{}} if prev != nil { @@ -161,7 +161,6 @@ func (h *Holder) startSources(only string) (started, leftOut []string, err error wrote = true } } - group = next.Group return next, wrote }) if err != nil { @@ -177,7 +176,7 @@ func (h *Holder) startSources(only string) (started, leftOut []string, err error } at := h.Now().UTC() if err := h.SaySeat(seat.SubjectSourceMoved(keys.EscapeName(branch)), - seat.SourceMovedEvent{Branch: branch, Group: group, Revision: int64(rev), At: &at}, + seat.SourceMovedEvent{Branch: branch, Revision: int64(rev), At: &at}, "source-moved."+keys.EscapeName(branch)+"."+strconv.FormatUint(rev, 10)); err != nil { errs = append(errs, err) } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/sourcetools_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/sourcetools_test.go index 7a3c52382..0ba9afc6f 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/sourcetools_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/sourcetools_test.go @@ -1,10 +1,12 @@ package main import ( + "encoding/json" "errors" "fmt" meshcatalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" + stdio "git.novox.be/novox/mesh-sdk/go" "reflect" "strings" "testing" @@ -173,8 +175,8 @@ func TestAWatchedBranchStartsFromTheForge(t *testing.T) { Logf: func(f string, a ...any) { logged = append(logged, fmt.Sprintf(f, a...)) }, SaySeat: func(subject string, body any, id string) error { events++ - if ev := body.(seat.SourceMovedEvent); ev.Group != c.held[ev.Branch].Group { - t.Errorf("the group is the key's own (empty for a new key), got %q", ev.Group) + if ev := body.(seat.SourceMovedEvent); ev.Group != "" { + t.Errorf("a source-moved for started heads names no group (A-404), got %q", ev.Group) } return nil }} @@ -193,6 +195,11 @@ func TestAWatchedBranchStartsFromTheForge(t *testing.T) { t.Errorf("source.%s heads %v, want %v", b, got, want) } } + for b, h := range c.held { + if src.vals[b].Group != h.Group { + t.Errorf("source.%s keeps the group %q a take-in wrote, got %q", b, h.Group, src.vals[b].Group) + } + } if c.only == "" && c.held == nil && !strings.Contains(strings.Join(logged, "\n"), "o/photos") { t.Errorf("the repository left out was not said: %v", logged) } @@ -220,3 +227,35 @@ func TestSourceStartByHandIsRecordedFirst(t *testing.T) { t.Fatalf("log %+v source %+v", acts.took, src.vals) } } + +// Design 47 G A-401: the registration-changed branch of take and the start pass both write the new branch's key and say +// source-moved; removing either leaves the branch unstarted. +func TestRegistrationChangedAndTheStartPassStartABranch(t *testing.T) { + regs := []meshcatalog.Registration{{Module: "f1", Origin: "o/f1", Branches: []string{"master"}}} + for _, c := range []struct { + name string + run func(h *Holder) error + }{ + {"registration-changed", func(h *Holder) error { + return take(h, stdio.Envelope{Key: registrationChangedEvent, Body: json.RawMessage(`{}`)}) + }}, + {"the start pass", func(h *Holder) error { startSourcesAtStart(h); return nil }}, + } { + t.Run(c.name, func(t *testing.T) { + if registrationChangedEvent == "" { + t.Fatal("the manifest does not consume registration-changed") + } + src := &branchSource{memSource{vals: map[string]Source{}}} + var subjects []string + h := &Holder{Now: time.Now, Source: src, Forge: &fakeForge{heads: map[string]string{"o/f1@master": "c3"}}, + Catalog: fakeCatalog{regs: regs}, Logf: func(string, ...any) {}, + SaySeat: func(subject string, body any, id string) error { subjects = append(subjects, subject); return nil }} + if err := c.run(h); err != nil { + t.Fatal(err) + } + if src.vals["master"].Heads["o/f1"] != "c3" || len(subjects) != 1 || !strings.HasSuffix(subjects[0], ".event.source-moved.master") { + t.Fatalf("source %+v events %v", src.vals, subjects) + } + }) + } +} From e8204a9c27a5b36f1245d33c50dd303a4b316920 Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 14 Nov 2026 22:34:54 +0100 Subject: [PATCH 09/24] Gate: the start pass asks again until no repository is left out (A-401) The forge's git verbs may serve after the gate does; one pass with every head read refused wrote nothing and was never retried. Builds: design 47 G A-401 --- .../cmd/mesh-delivery-gate/main.go | 13 ------------- .../cmd/mesh-delivery-gate/sourcestart.go | 19 +++++++++++++++++++ .../mesh-delivery-gate/sourcetools_test.go | 17 +++++++++++++++++ 3 files changed, 36 insertions(+), 13 deletions(-) diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go index b9bc97998..1992a61e6 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go @@ -125,19 +125,6 @@ func run(h *Holder, l *listening) { } } -// startSourcesAtStart starts the sources of the watched branches no key holds, retrying until the registrations can be read. -func startSourcesAtStart(h *Holder) { - for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { - started, err := h.StartSources() - if err == nil { - logf("[mesh-delivery] the sources started from the forge at the start: %v", started) - return - } - logf("[mesh-delivery] the sources cannot be started yet (%v); asking again in %s", err, wait) - time.Sleep(wait) - } -} - // switchToRecords carries the groups in flight into their delivery records (switch.go), retrying until it can: no event is // taken before, so no outcome reaches a record the switch has not opened yet. func switchToRecords(h *Holder, l *listening) { diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/sourcestart.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/sourcestart.go index dbd772a1a..2fd8e6145 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/sourcestart.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/sourcestart.go @@ -13,6 +13,7 @@ import ( "sort" "strconv" "strings" + "time" "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" gitseat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git" @@ -89,6 +90,24 @@ func (h *Holder) StartSources() ([]string, error) { return started, err } +// startSleep waits between the start's passes; a test replaces it. +var startSleep = time.Sleep + +// startSourcesAtStart starts the sources at the gate's start, and asks again, as switchToRecords does (2 s doubling to a +// minute), until a pass reads the registrations and leaves no repository out: the forge's verbs may serve after the gate +// does (A-401). +func startSourcesAtStart(h *Holder) { + for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { + started, leftOut, err := h.startSources("") + if err == nil && len(leftOut) == 0 { + h.logf("[mesh-delivery] the sources started from the forge at the start: %v", started) + return + } + h.logf("[mesh-delivery] the sources are not all started yet (started %v, left out %v, %v); asking again in %s", started, leftOut, err, wait) + startSleep(wait) + } +} + // startSources starts the branches no key holds, or only branch when it is named; it answers the branches written and the // repositories left out as "@". func (h *Holder) startSources(only string) (started, leftOut []string, err error) { diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/sourcetools_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/sourcetools_test.go index 0ba9afc6f..892df4ea6 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/sourcetools_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/sourcetools_test.go @@ -259,3 +259,20 @@ func TestRegistrationChangedAndTheStartPassStartABranch(t *testing.T) { }) } } + +// Design 47 G A-401: a start pass that left a repository out is not done: it asks again, doubling its wait, until none is left out. +func TestTheStartAsksAgainWhileARepositoryIsLeftOut(t *testing.T) { + var waits []time.Duration + old := startSleep + startSleep = func(d time.Duration) { waits = append(waits, d) } + defer func() { startSleep = old }() + src := &branchSource{memSource{vals: map[string]Source{}}} + forge := &fakeForge{fail: true, heads: map[string]string{"o/f1@master": "c3"}} + startSleep = func(d time.Duration) { waits = append(waits, d); forge.fail = len(waits) < 2 } + h := &Holder{Now: time.Now, Source: src, Forge: forge, Logf: func(string, ...any) {}, + Catalog: fakeCatalog{regs: []meshcatalog.Registration{{Module: "f1", Origin: "o/f1", Branches: []string{"master"}}}}} + startSourcesAtStart(h) + if !reflect.DeepEqual(waits, []time.Duration{2 * time.Second, 4 * time.Second}) || src.vals["master"].Heads["o/f1"] != "c3" { + t.Fatalf("waits %v source %+v", waits, src.vals) + } +} From 99f3e234927c7d8c85a1d8b6d2249b0d26582aba Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 15 Nov 2026 01:20:04 +0100 Subject: [PATCH 10/24] Distribution: each wait of a publish logged with its time; the window's learning no longer holds the answer Builds: design 47 G (publish path of oci-artifact-registry; no design line covers the log lines) --- .../cmd/distribution-tools/main.go | 17 ++++++++-- .../internal/holder/holder_test.go | 15 +++++++++ .../distribution/internal/holder/transfer.go | 32 +++++++++++++++++-- 3 files changed, 58 insertions(+), 6 deletions(-) diff --git a/modules/distribution/cmd/distribution-tools/main.go b/modules/distribution/cmd/distribution-tools/main.go index b0278ca9f..b928f84b4 100644 --- a/modules/distribution/cmd/distribution-tools/main.go +++ b/modules/distribution/cmd/distribution-tools/main.go @@ -249,10 +249,21 @@ func tools(v *holder.Verbs, windows kept[holder.Window]) []stdio.Tool { if err != nil { return nil, err } - a, err := v.Publish(seat, stdio.CallerOf(args), publishOf(in)) + caller := stdio.CallerOf(args) + start := time.Now() + logf("%s.publish received from %s: %s %s (%s, %d parts)", seat, caller, in.Name, in.Version, in.Kind, len(in.Parts)) + a, err := v.Publish(seat, caller, publishOf(in)) if err == nil { - learnPublishing(windows, stdio.CallerOf(args)) + // The window's learning asks the delivery view (up to 60 s, past the runner's 30 s call bound), + // so it never holds the answer. + go func() { + t := time.Now() + logf("learning the publishing build of %s starts (mesh-delivery-view.node_haves)", caller) + learnPublishing(windows, caller) + logf("learning the publishing build of %s ended after %s", caller, time.Since(t).Round(time.Millisecond)) + }() } + logf("%s.publish from %s answered after %s (error: %v)", seat, caller, time.Since(start).Round(time.Millisecond), err) return publishAnswer(a), err }}, stdio.Tool{Name: seat + "." + holder.VerbPublishPart, Description: "Send one chunk (at most 1 MiB) of one part of an open transfer, from the caller that opened it.", @@ -468,7 +479,7 @@ func main() { }() base, _ := url.Parse("http://127.0.0.1:5001") store := &holder.OCI{Base: base, Upstream: &holder.Upstream{}} - h := &holder.Holder{Objects: parts.BusObjects(holder.IncomingObjects), Builder: holder.Reserved{Ask: stdio.Ask}, Store: store, Say: say, Finding: findings.Say, + h := &holder.Holder{Logf: logf, Objects: parts.BusObjects(holder.IncomingObjects), Builder: holder.Reserved{Ask: stdio.Ask}, Store: store, Say: say, Finding: findings.Say, Transfers: parts.BusTransfers[holder.Opening](holder.TransfersState), Cap: parts.BusCap(holder.IncomingObjects), Record: recordAct, Idle: parts.BusArm(holder.IdleState)} files := &holder.Files{Container: or(os.Getenv("MESH_REGISTRY_CONTAINER"), "mesh-registry"), Run: holder.ExecRunner, UID: os.Getuid()} v := &holder.Verbs{Holder: h, Catalog: store, Holds: kept[holder.Hold]{state: "holds", key: holder.Hold.Key}, Mirrors: mirrors{}, diff --git a/modules/distribution/internal/holder/holder_test.go b/modules/distribution/internal/holder/holder_test.go index 1e202aab9..828bd3797 100644 --- a/modules/distribution/internal/holder/holder_test.go +++ b/modules/distribution/internal/holder/holder_test.go @@ -231,6 +231,21 @@ func TestTheHolderOfTheTwoImageSeats(t *testing.T) { return PublishRequest{Transfer: id, Name: "m/program", Version: "0.1.4", Kind: KindBundle, Digest: part.Digest, Parts: []Part{part}} } + t.Run("a publish says each of its waits with its time", func(t *testing.T) { + b := newBed(true) + var lines []string + b.h.Logf = func(f string, a ...any) { lines = append(lines, fmt.Sprintf(f, a...)) } + if _, err := b.h.Publish(ArtifactSeat, builder, req("tl")); err != nil { + t.Fatal(err) + } + all := strings.Join(lines, "\n") + for _, want := range []string{"the store's read of what it holds (the program's registry) starts", "(the program's registry) ended after", "the opening of the transfer on the bus"} { + if !strings.Contains(all, want) { + t.Errorf("no log line %q in:\n%s", want, all) + } + } + }) + t.Run("a transfer in parts, cut and resumed, hashed, numbered by the builder, published", func(t *testing.T) { b := newBed(true) a, err := b.h.Publish(ArtifactSeat, builder, req("t1")) diff --git a/modules/distribution/internal/holder/transfer.go b/modules/distribution/internal/holder/transfer.go index bbd93b487..857ec5841 100644 --- a/modules/distribution/internal/holder/transfer.go +++ b/modules/distribution/internal/holder/transfer.go @@ -173,6 +173,20 @@ type Holder struct { // Idle writes a transfer's idle key with its own time to live, once (parts.BusArm over IdleState); the receiver // arms it on each open that leaves a transfer open. Nil arms nothing. Idle func(id string, ttl time.Duration) error + // Logf says each wait of a publish with its time (stderr in the bundle); nil says nothing. + Logf func(string, ...any) +} + +// timed runs f and logs its name and time when the holder logs. +func (h *Holder) timed(what string, f func() error) error { + if h.Logf == nil { + return f() + } + start := time.Now() + h.Logf("publish: %s starts", what) + err := f() + h.Logf("publish: %s ended after %s (%v)", what, time.Since(start).Round(time.Millisecond), err) + return err } func (h *Holder) now() time.Time { @@ -249,17 +263,29 @@ func (h *Holder) Publish(seat, caller string, r PublishRequest) (PublishAnswer, } // A rebuild of a held fingerprint answers the held version and stores nothing. if r.Version != "" { - if ref, ok, err := h.Store.Held(r.Name, r.Version, r.Digest); err != nil { + var ref string + var ok bool + err := h.timed("the store's read of what it holds (the program's registry)", func() (err error) { + ref, ok, err = h.Store.Held(r.Name, r.Version, r.Digest) + return err + }) + if err != nil { return PublishAnswer{}, fmt.Errorf("publish: the store cannot say whether it holds %s@%s: %w", r.Name, r.Version, err) } else if ok { - return PublishAnswer{Transfer: r.Transfer, Ended: true}, h.Say(seat, EventPublished, h.eventOf(seat, r, SeatReference(seat, ref), "")) + return PublishAnswer{Transfer: r.Transfer, Ended: true}, h.timed("the published event on the seat", func() error { + return h.Say(seat, EventPublished, h.eventOf(seat, r, SeatReference(seat, ref), "")) + }) } } wants := make([]parts.Want, 0, len(r.Parts)) for _, p := range r.Parts { wants = append(wants, parts.Want{Digest: p.Digest, Size: p.Size}) } - got, err := h.receiver().Open(r.Transfer, caller, wants, Opening{PublishRequest: r, Seat: seat}) + var got parts.Received + err := h.timed("the opening of the transfer on the bus (objects, transfer state, idle key)", func() (err error) { + got, err = h.receiver().Open(r.Transfer, caller, wants, Opening{PublishRequest: r, Seat: seat}) + return err + }) if err != nil { return answerOf(got), fmt.Errorf("publish: %w", err) } From bfc877209f26f6ea4952418c15cecb70169bee31 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 15 Nov 2026 01:30:05 +0100 Subject: [PATCH 11/24] Distribution: each bus step of a publish bounded at 10 s and logged by name Builds: design 47 G (publish path of oci-artifact-registry; no design line covers the bounds or log lines) --- .../cmd/distribution-tools/bounded.go | 102 ++++++++++++++++++ .../cmd/distribution-tools/bounded_test.go | 20 ++++ .../cmd/distribution-tools/main.go | 4 +- 3 files changed, 124 insertions(+), 2 deletions(-) create mode 100644 modules/distribution/cmd/distribution-tools/bounded.go create mode 100644 modules/distribution/cmd/distribution-tools/bounded_test.go diff --git a/modules/distribution/cmd/distribution-tools/bounded.go b/modules/distribution/cmd/distribution-tools/bounded.go new file mode 100644 index 000000000..493424b32 --- /dev/null +++ b/modules/distribution/cmd/distribution-tools/bounded.go @@ -0,0 +1,102 @@ +package main + +import ( + "fmt" + "time" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/parts" + + "distribution/internal/holder" +) + +// stepBound bounds each bus operation of a publish well under the tool runner's 30 s call bound, so a hang is an error +// naming its step and the publish answers with it. +const stepBound = 10 * time.Second + +// step runs f, logs its start and its end with its time, and gives up on it after bound: the error names the step. An +// operation given up on keeps running in its goroutine until the bus answers it. +func step[T any](bound time.Duration, what string, f func() (T, error)) (T, error) { + type res struct { + v T + err error + } + start := time.Now() + logf("step %s starts", what) + ch := make(chan res, 1) + go func() { v, err := f(); ch <- res{v, err} }() + select { + case r := <-ch: + logf("step %s ended after %s (%v)", what, time.Since(start).Round(time.Millisecond), r.err) + return r.v, r.err + case <-time.After(bound): + var zero T + err := fmt.Errorf("%s did not answer within %s", what, bound) + logf("step %s: %v", what, err) + return zero, err + } +} + +func stepErr(bound time.Duration, what string, f func() error) error { + _, err := step(bound, what, func() (struct{}, error) { return struct{}{}, f() }) + return err +} + +// boundedObjects is the incoming object store with each operation bounded and logged. +type boundedObjects struct{ parts.Objects } + +func (b boundedObjects) Used() (uint64, error) { + return step(stepBound, "the incoming object store's fill (objects)", b.Objects.Used) +} +func (b boundedObjects) Put(name string, data []byte) error { + return stepErr(stepBound, "the put of a chunk in the incoming object store (objects)", func() error { return b.Objects.Put(name, data) }) +} +func (b boundedObjects) Get(name string) ([]byte, error) { + return step(stepBound, "the get of a chunk from the incoming object store (objects)", func() ([]byte, error) { return b.Objects.Get(name) }) +} +func (b boundedObjects) Delete(name string) error { + return stepErr(stepBound, "the delete of a chunk in the incoming object store (objects)", func() error { return b.Objects.Delete(name) }) +} + +// boundedTransfers is the transfers state with its reads and writes bounded and logged. +type boundedTransfers struct { + parts.Transfers[holder.Opening] +} + +func (b boundedTransfers) Get(id string) (parts.Transfer[holder.Opening], bool, error) { + type r struct { + t parts.Transfer[holder.Opening] + ok bool + } + x, err := step(stepBound, "the read of the transfer in the transfers state", func() (r, error) { + t, ok, err := b.Transfers.Get(id) + return r{t, ok}, err + }) + return x.t, x.ok, err +} +func (b boundedTransfers) Put(t parts.Transfer[holder.Opening]) error { + return stepErr(stepBound, "the write of the transfer in the transfers state", func() error { return b.Transfers.Put(t) }) +} +func (b boundedTransfers) Delete(id string) error { + return stepErr(stepBound, "the delete of the transfer in the transfers state", func() error { return b.Transfers.Delete(id) }) +} + +// boundedCap is the incoming cap read from its binding, bounded and logged. +func boundedCap(c func() (uint64, error)) func() (uint64, error) { + return func() (uint64, error) { + return step(stepBound, "the read of incoming's cap from its binding (objects)", c) + } +} + +// boundedArm is the idle key's write, bounded and logged. +func boundedArm(a func(string, time.Duration) error) func(string, time.Duration) error { + return func(id string, ttl time.Duration) error { + return stepErr(stepBound, "the write of the idle key in the idle state", func() error { return a(id, ttl) }) + } +} + +// boundedFinding is the findings writer's say, bounded and logged: the receiver says the incoming fill through it on an open. +func boundedFinding(f holder.Finding) holder.Finding { + return func(kind, subject, words string, raised bool) { + _ = stepErr(stepBound, "the finding "+kind+" of "+subject, func() error { f(kind, subject, words, raised); return nil }) + } +} diff --git a/modules/distribution/cmd/distribution-tools/bounded_test.go b/modules/distribution/cmd/distribution-tools/bounded_test.go new file mode 100644 index 000000000..9e75bc1f8 --- /dev/null +++ b/modules/distribution/cmd/distribution-tools/bounded_test.go @@ -0,0 +1,20 @@ +package main + +import ( + "strings" + "testing" + "time" +) + +// A bus operation that never answers becomes an error naming its step, within the bound. +func TestAHungStepAnswersWithItsName(t *testing.T) { + block := make(chan struct{}) + defer close(block) + _, err := step(10*time.Millisecond, "the write of the idle key", func() (int, error) { <-block; return 0, nil }) + if err == nil || !strings.Contains(err.Error(), "the write of the idle key did not answer within") { + t.Fatalf("err = %v", err) + } + if v, err := step(time.Second, "x", func() (int, error) { return 7, nil }); v != 7 || err != nil { + t.Fatalf("%d %v", v, err) + } +} diff --git a/modules/distribution/cmd/distribution-tools/main.go b/modules/distribution/cmd/distribution-tools/main.go index b928f84b4..b20db6f5d 100644 --- a/modules/distribution/cmd/distribution-tools/main.go +++ b/modules/distribution/cmd/distribution-tools/main.go @@ -479,8 +479,8 @@ func main() { }() base, _ := url.Parse("http://127.0.0.1:5001") store := &holder.OCI{Base: base, Upstream: &holder.Upstream{}} - h := &holder.Holder{Logf: logf, Objects: parts.BusObjects(holder.IncomingObjects), Builder: holder.Reserved{Ask: stdio.Ask}, Store: store, Say: say, Finding: findings.Say, - Transfers: parts.BusTransfers[holder.Opening](holder.TransfersState), Cap: parts.BusCap(holder.IncomingObjects), Record: recordAct, Idle: parts.BusArm(holder.IdleState)} + h := &holder.Holder{Logf: logf, Objects: boundedObjects{parts.BusObjects(holder.IncomingObjects)}, Builder: holder.Reserved{Ask: stdio.Ask}, Store: store, Say: say, Finding: boundedFinding(findings.Say), + Transfers: boundedTransfers{parts.BusTransfers[holder.Opening](holder.TransfersState)}, Cap: boundedCap(parts.BusCap(holder.IncomingObjects)), Record: recordAct, Idle: boundedArm(parts.BusArm(holder.IdleState))} files := &holder.Files{Container: or(os.Getenv("MESH_REGISTRY_CONTAINER"), "mesh-registry"), Run: holder.ExecRunner, UID: os.Getuid()} v := &holder.Verbs{Holder: h, Catalog: store, Holds: kept[holder.Hold]{state: "holds", key: holder.Hold.Key}, Mirrors: mirrors{}, Files: files, SweepHolder: func() (string, error) { return seatlist.HolderOf(stdio.Ask, holder.SweepSeat) }} From c0065380643dd2c3f16d7b8794fb0a74deb9bd03 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 15 Nov 2026 01:38:27 +0100 Subject: [PATCH 12/24] Distribution: the incoming-full finding is said off the publish's path Builds: design 47 G (publish path of oci-artifact-registry; no design line covers it) --- .../cmd/distribution-tools/bounded.go | 20 +++++++++++++++++-- .../cmd/distribution-tools/bounded_test.go | 14 +++++++++++++ 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/modules/distribution/cmd/distribution-tools/bounded.go b/modules/distribution/cmd/distribution-tools/bounded.go index 493424b32..14236ff47 100644 --- a/modules/distribution/cmd/distribution-tools/bounded.go +++ b/modules/distribution/cmd/distribution-tools/bounded.go @@ -94,9 +94,25 @@ func boundedArm(a func(string, time.Duration) error) func(string, time.Duration) } } -// boundedFinding is the findings writer's say, bounded and logged: the receiver says the incoming fill through it on an open. +// boundedFinding is the findings writer's say moved off the publish's path: the receiver says the incoming fill on every +// open, and a publish never waits for it. One worker says them in order, each bounded and logged; one said while the worker +// is full is dropped and logged (the next open says the fill again). func boundedFinding(f holder.Finding) holder.Finding { + type said struct { + kind, subject, words string + raised bool + } + queue := make(chan said, 64) + go func() { + for s := range queue { + _ = stepErr(stepBound, "the finding "+s.kind+" of "+s.subject, func() error { f(s.kind, s.subject, s.words, s.raised); return nil }) + } + }() return func(kind, subject, words string, raised bool) { - _ = stepErr(stepBound, "the finding "+kind+" of "+subject, func() error { f(kind, subject, words, raised); return nil }) + select { + case queue <- said{kind, subject, words, raised}: + default: + logf("the finding %s of %s is not said now: the writer is behind, the next open says it again", kind, subject) + } } } diff --git a/modules/distribution/cmd/distribution-tools/bounded_test.go b/modules/distribution/cmd/distribution-tools/bounded_test.go index 9e75bc1f8..bd6c428e4 100644 --- a/modules/distribution/cmd/distribution-tools/bounded_test.go +++ b/modules/distribution/cmd/distribution-tools/bounded_test.go @@ -18,3 +18,17 @@ func TestAHungStepAnswersWithItsName(t *testing.T) { t.Fatalf("%d %v", v, err) } } + +// The finding is said off the caller's path: a writer that never answers does not hold the call that says it. +func TestAFindingIsSaidOffThePath(t *testing.T) { + block := make(chan struct{}) + defer close(block) + say := boundedFinding(func(string, string, string, bool) { <-block }) + done := make(chan struct{}) + go func() { say("incoming-full", "incoming", "w", false); say("incoming-full", "incoming", "w", false); close(done) }() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("the call was held by the finding") + } +} From 7d17441a9a2ffeee72519ce44de8f63ecab8b5cd Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 15 Nov 2026 01:38:29 +0100 Subject: [PATCH 13/24] gofmt --- modules/distribution/cmd/distribution-tools/bounded_test.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/modules/distribution/cmd/distribution-tools/bounded_test.go b/modules/distribution/cmd/distribution-tools/bounded_test.go index bd6c428e4..e02f926ac 100644 --- a/modules/distribution/cmd/distribution-tools/bounded_test.go +++ b/modules/distribution/cmd/distribution-tools/bounded_test.go @@ -25,7 +25,11 @@ func TestAFindingIsSaidOffThePath(t *testing.T) { defer close(block) say := boundedFinding(func(string, string, string, bool) { <-block }) done := make(chan struct{}) - go func() { say("incoming-full", "incoming", "w", false); say("incoming-full", "incoming", "w", false); close(done) }() + go func() { + say("incoming-full", "incoming", "w", false) + say("incoming-full", "incoming", "w", false) + close(done) + }() select { case <-done: case <-time.After(5 * time.Second): From 4a99f7044ee78302c5c54a44c256c58bde2efa4a Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 15 Nov 2026 02:39:28 +0100 Subject: [PATCH 14/24] distribution: a publish with a version and no parts tags the digest the store holds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The builder's publish again of mesh-delivery-pipeline 0.1.6 sends the version and the digest with no parts, as A-73 has it; the build stored the bundle untagged, and the holder answered 'publish: no parts' unless the tag existed already. The holder now checks the digest against the store under the name (the artifact store, A-50), asks the builder's reservation, tags it with the version and says published; a digest it holds none of is refused as not stored, naming the digest. Builds: design 47 G A-417, A-73, A-50, §588 --- .../internal/holder/holder_test.go | 30 +++++++---- modules/distribution/internal/holder/oci.go | 19 +++++++ .../distribution/internal/holder/transfer.go | 51 +++++++++++++++++++ 3 files changed, 89 insertions(+), 11 deletions(-) diff --git a/modules/distribution/internal/holder/holder_test.go b/modules/distribution/internal/holder/holder_test.go index 828bd3797..08d6aee84 100644 --- a/modules/distribution/internal/holder/holder_test.go +++ b/modules/distribution/internal/holder/holder_test.go @@ -71,6 +71,7 @@ func (s *fakeStore) Put(t PublishRequest, parts map[string][]byte) (string, erro return t.Name + "@" + t.Digest, nil } func (s *fakeStore) Held(string, string, string) (string, bool, error) { return "", false, nil } +func (s *fakeStore) Holds(r PublishRequest) (bool, error) { return s.has[r.Digest], nil } type said struct { seat, event string @@ -665,30 +666,37 @@ func TestTheHolderOfTheTwoImageSeats(t *testing.T) { } } }) - t.Run("a late publish names a held bundle with no size: tagged at the store's size, nothing sent", func(t *testing.T) { + t.Run("a publish with a version and no parts tags the held digest at the store's size; one not held is not stored", func(t *testing.T) { + // Design 47 G A-417, A-73, A-50: the build stored the bundle untagged; the builder's publish of its version with that + // digest and no parts tags it and says published, nothing sent; a digest the store holds none of is refused as not + // stored, naming the digest. store := newFakeRegistry() sts := httptest.NewServer(store) defer sts.Close() base, _ := url.Parse(sts.URL) - o := &OCI{Base: base} + b := newBed(true) + b.h.Store = &OCI{Base: base} blob := []byte(strings.Repeat("b", 4321)) store.blobs["m/tools@"+digestOf(blob)] = blob - ref, err := o.Put(PublishRequest{Name: "m/tools", Version: "0.1.4", Kind: KindBundle, Digest: digestOf(blob), - Parts: []Part{{Digest: digestOf(blob)}}}, map[string][]byte{}) - if err != nil || ref != "m/tools@"+digestOf(blob) { - t.Fatalf("Put = %q, %v", ref, err) + got, err := b.h.Publish(ArtifactSeat, "x.mesh-delivery-builder", PublishRequest{Name: "m/tools", Version: "0.1.4", Kind: KindBundle, Digest: digestOf(blob), + Transfer: "republish-m-0-1-4-tools"}) + if err != nil || !got.Ended || len(b.said) != 1 || b.said[0].event != EventPublished || b.said[0].body.Reference != SeatReference(ArtifactSeat, "m/tools@"+digestOf(blob)) { + t.Fatalf("Publish = %+v, %v; said %+v", got, err, b.said) } var man struct { Layers []struct { - Size int64 `json:"size"` + Digest string `json:"digest"` + Size int64 `json:"size"` } `json:"layers"` } - if err := json.Unmarshal(store.manifests["m/tools:0.1.4"], &man); err != nil || len(man.Layers) != 1 || man.Layers[0].Size != int64(len(blob)) { + if err := json.Unmarshal(store.manifests["m/tools:0.1.4"], &man); err != nil || len(man.Layers) != 1 || man.Layers[0].Size != int64(len(blob)) || man.Layers[0].Digest != digestOf(blob) { t.Errorf("the version's manifest %s, want one layer of %d bytes", store.manifests["m/tools:0.1.4"], len(blob)) } - if _, err := o.Put(PublishRequest{Name: "m/tools", Version: "0.1.5", Kind: KindBundle, Digest: "sha256:" + strings.Repeat("e", 64), - Parts: []Part{{Digest: "sha256:" + strings.Repeat("e", 64)}}}, map[string][]byte{}); err == nil { - t.Error("a digest the store does not hold was tagged") + gone := "sha256:" + strings.Repeat("e", 64) + _, err = b.h.Publish(ArtifactSeat, "x.mesh-delivery-builder", PublishRequest{Name: "m/tools", Version: "0.1.5", Kind: KindBundle, Digest: gone, Transfer: "republish-m-0-1-5-tools"}) + if err == nil || !strings.Contains(err.Error(), "not stored: ") || !strings.Contains(err.Error(), gone) || store.manifests["m/tools:0.1.5"] != nil || + len(b.said) != 2 || b.said[1].event != EventRefused || !b.findings["publish-refused m/tools@0.1.5"] { + t.Errorf("a digest not held: %v; said %+v; findings %v", err, b.said, b.findings) } }) t.Run("mirror answers at once and says the copy's end on the seat's events", func(t *testing.T) { diff --git a/modules/distribution/internal/holder/oci.go b/modules/distribution/internal/holder/oci.go index 7b41d9feb..65abad32f 100644 --- a/modules/distribution/internal/holder/oci.go +++ b/modules/distribution/internal/holder/oci.go @@ -219,6 +219,25 @@ func (o *OCI) Held(name, version, digest string) (string, bool, error) { return name + "@" + d, true, nil } +// Holds is whether the store holds what r's digest names under r's name: an image's manifest by its digest, or a bundle's +// or an archive's blob. +func (o *OCI) Holds(r PublishRequest) (bool, error) { + if r.Kind != KindImage && r.Kind != KindUpstream { + return o.Has(r.Name, r.Digest) + } + resp, _, err := o.do(http.MethodHead, "/v2/"+r.Name+"/manifests/"+r.Digest, "", nil, accepted...) + if err != nil { + return false, err + } + switch resp.StatusCode { + case http.StatusOK: + return true, nil + case http.StatusNotFound: + return false, nil + } + return false, fmt.Errorf("the store answered %d for the manifest %s of %s", resp.StatusCode, r.Digest, r.Name) +} + // Versions are name's tags. func (o *OCI) Versions(name string) ([]string, error) { resp, b, err := o.do(http.MethodGet, "/v2/"+name+"/tags/list", "", nil) diff --git a/modules/distribution/internal/holder/transfer.go b/modules/distribution/internal/holder/transfer.go index 857ec5841..7b3bcba67 100644 --- a/modules/distribution/internal/holder/transfer.go +++ b/modules/distribution/internal/holder/transfer.go @@ -143,6 +143,9 @@ type Store interface { // Held is the reference's path of name at version when the store already holds digest under it: a rebuild of a held // fingerprint answers it and stores nothing. Held(name, version, digest string) (reference string, ok bool, err error) + // Holds says whether the store holds what a publish's digest names under its name: an image's manifest, or a bundle's or + // an archive's blob. The store is the artifact store (A-50): `artifact-store:///blobs/` is that blob. + Holds(r PublishRequest) (bool, error) } // SeatReference is a reference in the form the seat answers (novox/hq ADR 0605 §9): the seat's name, then the path the @@ -277,6 +280,9 @@ func (h *Holder) Publish(seat, caller string, r PublishRequest) (PublishAnswer, }) } } + if r.Version != "" && len(r.Parts) == 0 { + return h.tagHeld(seat, r) + } wants := make([]parts.Want, 0, len(r.Parts)) for _, p := range r.Parts { wants = append(wants, parts.Want{Digest: p.Digest, Size: p.Size}) @@ -292,6 +298,51 @@ func (h *Holder) Publish(seat, caller string, r PublishRequest) (PublishAnswer, return answerOf(got), nil } +// tagHeld is a publish with a version, a digest and no parts (design 47 G A-417, A-73, A-50): what the store holds of that +// digest under the name (the build's untagged publish, the artifact store's blob) is checked against the builder's reservation +// and tagged with the version, and published is said; when the store holds none of it, the publish is refused as not stored, +// naming the digest. Nothing travels. +func (h *Holder) tagHeld(seat string, r PublishRequest) (PublishAnswer, error) { + var held bool + err := h.timed("the store's read of the digest it holds (the program's registry)", func() (err error) { + held, err = h.Store.Holds(r) + return err + }) + if err != nil { + return PublishAnswer{}, fmt.Errorf("publish: the store cannot say whether it holds %s of %s: %w", r.Digest, r.Name, err) + } + refuse := func(reason string) (PublishAnswer, error) { + h.Finding("publish-refused", r.Name+"@"+r.Version, reason, true) + if err := h.Say(seat, EventRefused, h.eventOf(seat, r, "", reason)); err != nil { + return PublishAnswer{}, fmt.Errorf("publish: %s, and the refused is not said: %w", reason, err) + } + return PublishAnswer{Transfer: r.Transfer, Ended: true}, errors.New("publish: " + reason) + } + if !held { + return refuse(fmt.Sprintf("not stored: the store holds no %s of %s, so version %s names nothing", r.Digest, r.Name, r.Version)) + } + ok, why, err := h.Builder.Recorded(r.Name, r.Version, r.Digest) + if err != nil { + return PublishAnswer{}, fmt.Errorf("publish: the reservation could not be read, so nothing is tagged: %w", err) + } + if !ok { + return refuse("the builder did not reserve it: " + why) + } + tag := r + tag.Parts = []Part{{Digest: r.Digest}} + var path string + if err := h.timed("the store's tag of the held digest (the program's registry)", func() (err error) { + path, err = h.Store.Put(tag, map[string][]byte{}) + return err + }); err != nil { + return refuse("the store did not tag it: " + err.Error()) + } + h.Finding("publish-refused", r.Name+"@"+r.Version, "", false) + return PublishAnswer{Transfer: r.Transfer, Ended: true}, h.timed("the published event on the seat", func() error { + return h.Say(seat, EventPublished, h.eventOf(seat, r, SeatReference(seat, path), "")) + }) +} + // PublishPart is `publish-part`: one chunk, from the caller that opened the transfer alone. func (h *Holder) PublishPart(seat, caller string, r PartRequest) (PublishAnswer, error) { got, err := h.receiver().Take(r.Transfer, caller, r.Digest, r.Sequence, r.Last, r.Data, func(t Transfer) error { From 59704456c354cf34d9348f956f9cf0b45cf7e226 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 15 Nov 2026 15:52:48 +0100 Subject: [PATCH 15/24] messenger: offer every ask when it is raised; hold none back for room The operator channel kept at most three asks of one asker offered and held its oldest back behind newer ones, so asks waited unseen (the pipeline's proposal behind two mailu asks). It keeps no count any more and holds nothing back: every ask is offered when it is raised, and lapses at its expiry or ends at its answer. An ask held back before the change is offered again at the router's next tick; ask-again offers one so and holds nothing back in its place. Builds: design 47 G A-424 --- .../messenger/cmd/messenger/answers_test.go | 6 +- modules/messenger/cmd/messenger/asks.go | 14 +-- modules/messenger/cmd/messenger/held_test.go | 7 +- modules/messenger/cmd/messenger/holdback.go | 84 ++++++-------- .../messenger/cmd/messenger/holdback_test.go | 104 +++++------------- modules/messenger/cmd/messenger/holder.go | 1 + 6 files changed, 70 insertions(+), 146 deletions(-) diff --git a/modules/messenger/cmd/messenger/answers_test.go b/modules/messenger/cmd/messenger/answers_test.go index 025eba1c4..d15449dfa 100644 --- a/modules/messenger/cmd/messenger/answers_test.go +++ b/modules/messenger/cmd/messenger/answers_test.go @@ -601,14 +601,14 @@ func TestAnAskIsRefusedInWordsToItsAsker(t *testing.T) { } } all := strings.Join(refused, "\n") - // A fourth offered ask is never refused (ADR 0666 §12): it is taken and the asker's oldest is held back. + // A fourth offered ask is never refused and none is held back (design 47 G A-424). for _, want := range []string{"d-1: the ask is refused: an ask that destroys lasts at most", "d-2: its words carry"} { if !strings.Contains(all, want) { t.Errorf("not refused: %q in\n%s", want, all) } } - if strings.Contains(all, "d-6") || store.asks["mesh-delivery.d-6"].State != StateOpen || store.asks["mesh-delivery.d-3"].State != StateHeldBack { - t.Errorf("the fourth ask was refused or the oldest not held back: d-6 %s, d-3 %s", store.asks["mesh-delivery.d-6"].State, store.asks["mesh-delivery.d-3"].State) + if strings.Contains(all, "d-6") || store.asks["mesh-delivery.d-6"].State != StateOpen || store.asks["mesh-delivery.d-3"].State != StateOpen { + t.Errorf("the fourth ask was refused or the oldest held back: d-6 %s, d-3 %s", store.asks["mesh-delivery.d-6"].State, store.asks["mesh-delivery.d-3"].State) } // A refused ask is kept as a closed record with its outcome and words (design 47 G §570), and `asks` lists it refused. if rec, kept := store.asks["mesh-delivery.d-1"]; !kept || rec.State != string(occ.OutcomeRefused) || rec.Warrant == nil || diff --git a/modules/messenger/cmd/messenger/asks.go b/modules/messenger/cmd/messenger/asks.go index 74a94cbf1..729ee7af9 100644 --- a/modules/messenger/cmd/messenger/asks.go +++ b/modules/messenger/cmd/messenger/asks.go @@ -563,9 +563,9 @@ func (h *Holder) TakeAsk(verb, asker string, body json.RawMessage) error { h.refuseAsk(asker, a, "its whole words carry "+refusal.String()+", which may not leave the mesh on any channel: withhold it") return nil } - // A fourth offered ask is taken, never refused: the asker's oldest offered one is held back once this one is recorded - // (holdBackOldest). An ask with the same `about` replaces the open one, a held-back one too. - open, replaced := 0, "" + // Every ask is offered when it is raised: the channel keeps no count of an asker's open asks and holds none back for room + // (design 47 G A-424). An ask with the same `about` replaces the open one, a held-back one of before too. + replaced := "" h.mu.Lock() for k, e := range h.asks { if e.rec.Asker != asker || !liveState(e.rec.State) { @@ -573,14 +573,9 @@ func (h *Holder) TakeAsk(verb, asker string, body json.RawMessage) error { } if a.About != "" && e.rec.Ask.About == a.About { replaced = k - continue - } - if e.rec.State != StateHeldBack { - open++ } } h.mu.Unlock() - holdBack := open >= occ.MostOpen rec := AskRecord{Asker: asker, Ask: a, State: StateOpen, Opened: now, Tickets: h.ticketsFor(a)} if len(rec.Tickets) == 0 { // Held, not refused (ADR 0397): the ask breaks no rule, and the operator is asked when a channel can carry it. @@ -611,9 +606,6 @@ func (h *Holder) TakeAsk(verb, asker string, body json.RawMessage) error { } else { h.keepAsk(key, rec, 0) } - if holdBack { - h.holdBackOldest(asker, key) - } if rec.State == StateHeld { h.Logf("[messenger] %s asks %q (%s): held, not shown: %s", asker, a.Headline, a.ID, rec.HeldWhy) return nil diff --git a/modules/messenger/cmd/messenger/held_test.go b/modules/messenger/cmd/messenger/held_test.go index b04abc95f..d666310e6 100644 --- a/modules/messenger/cmd/messenger/held_test.go +++ b/modules/messenger/cmd/messenger/held_test.go @@ -105,7 +105,8 @@ func TestAHeldAskExpiresAsAnyOpenAskDoes(t *testing.T) { } } -func TestAHeldAskCountsTowardTheOffersOfItsAskerAndTheOldestIsHeldBack(t *testing.T) { +// Design 47 G A-424: four held asks of one asker are taken and held for want of a channel, and none is held back for room. +func TestFourHeldAsksOfOneAskerAreHeldAndNoneHeldBack(t *testing.T) { h, _, _, c, store, w := newRouter(t) withoutVerifiedSender(h) for i := 1; i <= 4; i++ { @@ -114,8 +115,8 @@ func TestAHeldAskCountsTowardTheOffersOfItsAskerAndTheOldestIsHeldBack(t *testin submit(t, h, "mesh-delivery", a) c.pass(time.Second) } - if got := w.of("mesh-delivery"); len(got) != 0 || store.asks["mesh-delivery.d-1"].State != StateHeldBack || store.asks["mesh-delivery.d-4"].State != StateHeld { - t.Fatalf("%+v: a fourth is taken, never refused; the oldest is held back (%s)", got, store.asks["mesh-delivery.d-1"].State) + if got := w.of("mesh-delivery"); len(got) != 0 || store.asks["mesh-delivery.d-1"].State != StateHeld || store.asks["mesh-delivery.d-4"].State != StateHeld { + t.Fatalf("%+v: a fourth is taken, never refused, and none is held back (d-1 %s)", got, store.asks["mesh-delivery.d-1"].State) } } diff --git a/modules/messenger/cmd/messenger/holdback.go b/modules/messenger/cmd/messenger/holdback.go index b9aa3a018..d08d22b6c 100644 --- a/modules/messenger/cmd/messenger/holdback.go +++ b/modules/messenger/cmd/messenger/holdback.go @@ -5,14 +5,12 @@ import ( oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel" "sort" "time" - - occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient" ) -// An ask held back (novox/hq ADR 0666 §12, design 46 § 6): an asker has at most MostOpen asks offered at once; a fourth -// is offered and nothing is refused, so no ask is hidden: the asker's oldest offered ask is held back, its tickets -// withdrawn and its message edited to say it waits behind newer asks, and it is offered again, with new tickets, when -// one of the asker's offered asks ends. `held` is another state, an ask no ready channel can carry (ADR 0397). +// An ask held back (novox/hq ADR 0666 §12, design 46 § 6) was one kept open behind newer asks of its asker, offered again when +// one of them ended. The channel holds none back any more (design 47 G A-424): every ask is offered when it is raised, and one +// held back before the change is offered at the router's start (offerAllHeldBack) and by ask-again. `held` is another state, an +// ask no ready channel can carry (ADR 0397). // StateHeldBack is an ask kept open behind newer asks of its asker. const StateHeldBack = "held-back" @@ -90,50 +88,35 @@ func (h *Holder) freshAsk(key string) (AskRecord, uint64, bool) { return rec, rev, true } -// holdBackOldest holds back the asker's oldest offered ask other than except (the newest, just taken): its tickets -// are withdrawn, it stays open, and its own message is edited to say it waits. One held for want of a channel counts as offered -// and is held back the same way. The work lock is held. -func (h *Holder) holdBackOldest(asker, except string) { - h.mu.Lock() - key := h.pickLocked(asker, except, StateOpen, StateHeld) - h.mu.Unlock() - if key == "" { - return +// offerHeldBack offers the asker's held-back asks of before the change again (A-424): there is no room to wait for. The work +// lock is held. +func (h *Holder) offerHeldBack(asker string) { + for { + h.mu.Lock() + key := h.pickLocked(asker, "", StateHeldBack) + h.mu.Unlock() + if key == "" { + return + } + h.offerOneHeldBack(key) + if after := h.askOf(key); after == nil || after.rec.State == StateHeldBack { + return // not offered (its record changed meanwhile): left for the next start rather than tried forever + } } - rec, rev, ok := h.freshAsk(key) - if !ok || (rec.State != StateOpen && rec.State != StateHeld) { - return - } - now := h.Now() - rec.State, rec.HeldBackSince, rec.Tickets, rec.HeldSince, rec.HeldWhy = StateHeldBack, now, nil, time.Time{}, "" - if !h.saveHeld(key, rec, rev) { - return - } - h.Logf("[messenger] %s's ask %q (%s) is held back behind its newer asks: it is offered again when one ends", asker, rec.Ask.Headline, rec.Ask.ID) - h.mu.Lock() - r := h.open[rec.Record] - h.mu.Unlock() - if r == nil || r.Origin != "ask" { - return // a condition's message is the newer ask's to show - } - h.mu.Lock() - r.Answered = "Waits behind newer questions of " + asker + ".\nIt is offered again, with its answers, when one of them ends." - h.mu.Unlock() - h.edit(r, "answered", false) - h.persist(r) } -// offerHeldBack offers the asker's oldest held-back ask again when a slot is free: new tickets, its message edited back -// to the question. The work lock is held. -func (h *Holder) offerHeldBack(asker string) { - if h.offeredCount(asker) >= occ.MostOpen { - return - } +// offerAllHeldBack offers every ask held back before the change, at each tick of the router (A-424). The work lock is held. +func (h *Holder) offerAllHeldBack() { h.mu.Lock() - key := h.pickLocked(asker, "", StateHeldBack) + askers := map[string]bool{} + for _, e := range h.asks { + if e.rec.State == StateHeldBack { + askers[e.rec.Asker] = true + } + } h.mu.Unlock() - if key != "" { - h.offerOneHeldBack(key) + for asker := range askers { + h.offerHeldBack(asker) } } @@ -172,9 +155,9 @@ func (h *Holder) offerOneHeldBack(key string) { h.showAsk(key, rec, now) } -// AskAgain is the seat operator-channel's verb ask-again (G §394, §450): offer a held-back ask now, holding back the -// asker's oldest offered ask in its place when the asker already has the most offered. It answers the ask's state after it, -// or the error that says why it was not offered. +// AskAgain is the seat operator-channel's verb ask-again (G §394, §450): it offers an ask held back before the channel stopped +// holding any back (A-424), and holds back nothing in its place. It answers the ask's state after it, or the error that says why +// it was not offered. func (h *Holder) AskAgain(key string) oc.AskAgainAnswer { h.work.Lock() defer h.work.Unlock() @@ -186,10 +169,7 @@ func (h *Holder) AskAgain(key string) oc.AskAgainAnswer { case !ok: return oc.AskAgainAnswer{Error: fmt.Sprintf("this router holds no ask %q: its key is ., as operator-channel.asks lists", key)} case rec.State != StateHeldBack: - return oc.AskAgainAnswer{Error: fmt.Sprintf("the ask %s is %s, not held back: only a held-back ask is offered again", key, rec.State)} - } - if h.offeredCount(rec.Asker) >= occ.MostOpen { - h.holdBackOldest(rec.Asker, key) + return oc.AskAgainAnswer{Error: fmt.Sprintf("the ask %s is %s, not held back: only a held-back ask is offered again, and the channel holds none back any more", key, rec.State)} } h.offerOneHeldBack(key) got, _, _ := h.freshAsk(key) diff --git a/modules/messenger/cmd/messenger/holdback_test.go b/modules/messenger/cmd/messenger/holdback_test.go index 5b92f1d2f..9e0d8d4e1 100644 --- a/modules/messenger/cmd/messenger/holdback_test.go +++ b/modules/messenger/cmd/messenger/holdback_test.go @@ -9,100 +9,50 @@ import ( occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient" ) -// ADR 0666 §12, design 46 §6 and test 71: an asker has at most three asks offered at once; a fourth is offered and the -// oldest is held back (its tickets withdrawn, its message edited, its record open), offered again with new tickets -// when one of the three ends, offered now by messenger_ask_again; an ask with the same `about` replaces a held-back -// one, and a Decline ends an ask whose asker may ask again at once. -func TestNoAskIsHiddenTheOldestOfFourIsHeldBackAndOfferedAgain(t *testing.T) { - h, tg, _, c, store, w := newRouter(t) +// Design 47 G A-424: the channel keeps no count of an asker's open asks and holds none back: every ask is offered when it is +// raised, five of one asker at once too, and none is refused. An ask held back before the change is offered again at the +// router's next tick, with new tickets; ask-again offers one now and holds nothing back in its place. +func TestEveryAskIsOfferedAndOneHeldBackBeforeIsOfferedAgain(t *testing.T) { + h, _, _, c, store, w := newRouter(t) link(h, store, c, "42", c.now().Add(-2*time.Hour)) - ask := func(id, about string) { + state := func(id string) string { return store.asks["mesh-delivery."+id].State } + for i, id := range []string{"d-1", "d-2", "d-3", "d-4", "d-5"} { a := ackAsk(c, id) - a.About = about + a.About = "about-" + string(rune('1'+i)) submit(t, h, "mesh-delivery", a) c.pass(time.Second) settle(h, c) } - state := func(id string) string { return store.asks["mesh-delivery."+id].State } - for i, id := range []string{"d-1", "d-2", "d-3"} { - ask(id, "about-"+string(rune('1'+i))) - } - oldTicket := "" - for tk := range store.asks["mesh-delivery.d-1"].Tickets { - oldTicket = tk - } - ask("d-4", "about-4") - // M1: the fourth reaches the channel, the oldest is held back, none is refused. - if len(w.of("mesh-delivery")) != 0 || state("d-4") != StateOpen || state("d-1") != StateHeldBack { - t.Fatalf("refused %+v; d-4 %s, d-1 %s", w.of("mesh-delivery"), state("d-4"), state("d-1")) - } - if rec := store.asks["mesh-delivery.d-1"]; len(rec.Tickets) != 0 || rec.HeldBackSince.IsZero() { - t.Errorf("held back %+v: tickets withdrawn, since set", rec) - } - if m, ok := tg.edits["1"]; !ok || len(m.Actions) != 0 || !strings.Contains(m.Note, "Waits behind newer questions of mesh-delivery") { - t.Errorf("its message was not edited to say it waits: %v %+v", ok, m) - } - if _, err := h.Chosen("telegram", choiceOf(oldTicket, "42", c.now())); err == nil || !strings.Contains(err.Error(), "waits behind newer questions") || len(w.of("mesh-delivery")) != 0 { - t.Errorf("a tap of a withdrawn ticket: %v %+v", err, w.of("mesh-delivery")) - } - // M3: asks lists it with since when and the newer asks it waits behind; open includes it. - q, _ := AsksQueryOf(map[string]any{"key": "mesh-delivery.d-1"}) - raw, _ := json.Marshal(q.Narrow(h.AsksNow())) - for _, want := range []string{`"state":"held-back"`, `"held-since":"`, `"behind":["mesh-delivery.d-2","mesh-delivery.d-3","mesh-delivery.d-4"]`} { - if !strings.Contains(string(raw), want) { - t.Errorf("asks lacks %s: %s", want, raw) + for _, id := range []string{"d-1", "d-2", "d-3", "d-4", "d-5"} { + if state(id) != StateOpen || len(store.asks["mesh-delivery."+id].Tickets) == 0 { + t.Fatalf("%s is %s with %d tickets: every ask is offered", id, state(id), len(store.asks["mesh-delivery."+id].Tickets)) } } - if q, _ = AsksQueryOf(nil); !strings.Contains(string(mustJSON(q.Narrow(h.AsksNow()))), "mesh-delivery.d-1") { - t.Error("the open asks leave out the held-back one") - } - // M5: an ask with the same about replaces a held-back one; the three offered stay three, so the oldest of them is held back. - ask("d-5", "about-1") - if state("d-1") != string(occ.OutcomeReplaced) || state("d-2") != StateHeldBack || state("d-5") != StateOpen || state("d-3") != StateOpen { - t.Fatalf("after d-5 about the held-back ask's thing: d-1 %s d-2 %s d-3 %s d-5 %s", state("d-1"), state("d-2"), state("d-3"), state("d-5")) - } - // M2: when one of the offered ends, the oldest held back is offered again with new tickets and its message asks again. - if a := h.AskAgain("mesh-delivery.d-3"); !strings.Contains(a.Error, "not held back") { - t.Errorf("ask-again of an offered ask: %+v", a) - } - if err := h.TakeAsk("cancel", "mesh-delivery", []byte(`{"id":"d-3"}`)); err != nil { + // One held back before the change, as the store kept it. + key := "mesh-delivery.d-2" + got, rev, _ := store.GetAsk(key) + got.State, got.Tickets, got.HeldBackSince = StateHeldBack, nil, c.now() + rev, err := store.UpdateAsk(key, *got, rev) + if err != nil { t.Fatal(err) } - settle(h, c) - if rec := store.asks["mesh-delivery.d-2"]; rec.State != StateOpen || len(rec.Tickets) == 0 || !rec.HeldBackSince.IsZero() { - t.Fatalf("d-2 after d-3 ended: %+v", rec) - } - if m := tg.edits["2"]; len(m.Actions) == 0 || strings.Contains(m.Note, "Waits behind") { - t.Errorf("its message does not ask again: %+v", m) - } - // M4: ask-again by hand offers a held-back ask now, holding back the oldest offered one; the seat operator-channel's ask-again (G §450). - ask("d-6", "about-6") - ask("d-7", "about-7") - held := "" - for _, id := range []string{"d-2", "d-4", "d-5", "d-6", "d-7"} { - if state(id) == StateHeldBack { - held = id - } - } - if held == "" { - t.Fatal("no ask is held back after two more") + h.keepAsk(key, *got, rev) + if a := h.AskAgain("mesh-delivery.d-1"); !strings.Contains(a.Error, "not held back") { + t.Errorf("ask-again of an offered ask: %+v", a) } if a := h.AskAgain("mesh-delivery.d-none"); a.Error == "" || a.State != "" { t.Errorf("ask-again of an ask the router holds none of: %+v", a) } - if a := h.AskAgain("mesh-delivery." + held); a.Error != "" || a.State != string(StateOpen) || state(held) != StateOpen { - t.Fatalf("%s is %s; ask-again answered %+v", held, state(held), a) + h.Tick() + settle(h, c) + if rec := store.asks[key]; rec.State != StateOpen || len(rec.Tickets) == 0 || !rec.HeldBackSince.IsZero() { + t.Fatalf("the ask held back before the change, after a tick: %+v", rec) } - n := 0 - for _, id := range []string{"d-2", "d-4", "d-5", "d-6", "d-7"} { - if state(id) == StateHeldBack { - n++ + for _, id := range []string{"d-1", "d-3", "d-4", "d-5"} { + if state(id) != StateOpen { + t.Errorf("%s is %s after the held-back one was offered: nothing is held back in its place", id, state(id)) } } - if n != 2 || state("d-5") != StateHeldBack { - t.Errorf("%d asks held back after ask-again (d-5 %s): the oldest offered one took its place", n, state("d-5")) - } - // A Decline ends an ask; its asker may ask again at once, and nothing is refused. for _, wt := range w.of("mesh-delivery") { if wt.Outcome == occ.OutcomeRefused { t.Errorf("refused: %+v", wt) diff --git a/modules/messenger/cmd/messenger/holder.go b/modules/messenger/cmd/messenger/holder.go index d87b60e68..09debb721 100644 --- a/modules/messenger/cmd/messenger/holder.go +++ b/modules/messenger/cmd/messenger/holder.go @@ -778,6 +778,7 @@ func (h *Holder) Tick() { h.mu.Unlock() sort.Slice(recs, func(i, j int) bool { return recs[i].Raised.Before(recs[j].Raised) }) h.sweepAsks(now) + h.offerAllHeldBack() // held back before the channel stopped holding any back, offered at the first tick (design 47 G A-424) h.releaseHeld() h.offerUnsent() h.breakGlass() From 76195e421f0bc3f0b845e50cb9fb1f0e6c158965 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 16 Nov 2026 09:41:45 +0100 Subject: [PATCH 16/24] mesh-delivery-gate: the start pass runs once, asking again only while the registrations cannot be read Builds: design 47 G A-442, A-401 --- .../cmd/mesh-delivery-gate/sourcestart.go | 17 +++--- .../mesh-delivery-gate/sourcetools_test.go | 61 ++++++++++++++----- 2 files changed, 57 insertions(+), 21 deletions(-) diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/sourcestart.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/sourcestart.go index 2fd8e6145..f7538a412 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/sourcestart.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/sourcestart.go @@ -93,17 +93,20 @@ func (h *Holder) StartSources() ([]string, error) { // startSleep waits between the start's passes; a test replaces it. var startSleep = time.Sleep -// startSourcesAtStart starts the sources at the gate's start, and asks again, as switchToRecords does (2 s doubling to a -// minute), until a pass reads the registrations and leaves no repository out: the forge's verbs may serve after the gate -// does (A-401). +// errRegistrationsUnread is a start pass's error when the registrations cannot be read. +var errRegistrationsUnread = errors.New("the registrations cannot be read") + +// startSourcesAtStart runs the start pass once (A-442), and asks again, as switchToRecords does (2 s doubling to a minute), +// only while the registrations cannot be read: the seat mesh-catalog may serve after the gate does. A repository a pass +// leaves out is said by the pass, and added at the next registration change or by gate_source_start (A-401). func startSourcesAtStart(h *Holder) { for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { started, leftOut, err := h.startSources("") - if err == nil && len(leftOut) == 0 { - h.logf("[mesh-delivery] the sources started from the forge at the start: %v", started) + if err == nil || !errors.Is(err, errRegistrationsUnread) { + h.logf("[mesh-delivery] the sources started from the forge at the start: %v; left out %v; %v", started, leftOut, err) return } - h.logf("[mesh-delivery] the sources are not all started yet (started %v, left out %v, %v); asking again in %s", started, leftOut, err, wait) + h.logf("[mesh-delivery] the registrations cannot be read yet (%v); asking again in %s", err, wait) startSleep(wait) } } @@ -118,7 +121,7 @@ func (h *Holder) startSources(only string) (started, leftOut []string, err error defer h.startMu.Unlock() regs, err := h.Catalog.Registrations() if err != nil { - return nil, nil, fmt.Errorf("the registrations cannot be read: %w", err) + return nil, nil, fmt.Errorf("%w: %w", errRegistrationsUnread, err) } watched := watchedRepositories(regs.Registrations) if only != "" { diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/sourcetools_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/sourcetools_test.go index 892df4ea6..aa0b7063c 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/sourcetools_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/sourcetools_test.go @@ -260,19 +260,52 @@ func TestRegistrationChangedAndTheStartPassStartABranch(t *testing.T) { } } -// Design 47 G A-401: a start pass that left a repository out is not done: it asks again, doubling its wait, until none is left out. -func TestTheStartAsksAgainWhileARepositoryIsLeftOut(t *testing.T) { - var waits []time.Duration - old := startSleep - startSleep = func(d time.Duration) { waits = append(waits, d) } - defer func() { startSleep = old }() - src := &branchSource{memSource{vals: map[string]Source{}}} - forge := &fakeForge{fail: true, heads: map[string]string{"o/f1@master": "c3"}} - startSleep = func(d time.Duration) { waits = append(waits, d); forge.fail = len(waits) < 2 } - h := &Holder{Now: time.Now, Source: src, Forge: forge, Logf: func(string, ...any) {}, - Catalog: fakeCatalog{regs: []meshcatalog.Registration{{Module: "f1", Origin: "o/f1", Branches: []string{"master"}}}}} - startSourcesAtStart(h) - if !reflect.DeepEqual(waits, []time.Duration{2 * time.Second, 4 * time.Second}) || src.vals["master"].Heads["o/f1"] != "c3" { - t.Fatalf("waits %v source %+v", waits, src.vals) +// Design 47 G A-442: the start pass runs once; it asks again, doubling its wait, only while the registrations cannot be read, +// and a repository left out is said, not waited for. +func TestTheStartAsksAgainOnlyWhileTheRegistrationsCannotBeRead(t *testing.T) { + regs := []meshcatalog.Registration{{Module: "f1", Origin: "o/f1", Branches: []string{"master"}}} + for _, c := range []struct { + name string + unreadFor int // passes the registrations cannot be read + forgeFail bool // the forge cannot name the head + waits []time.Duration + started bool + }{ + {"registrations unread twice", 2, false, []time.Duration{2 * time.Second, 4 * time.Second}, true}, + {"a repository left out is not waited for", 0, true, nil, false}, + } { + t.Run(c.name, func(t *testing.T) { + var waits []time.Duration + old := startSleep + startSleep = func(d time.Duration) { waits = append(waits, d) } + defer func() { startSleep = old }() + src := &branchSource{memSource{vals: map[string]Source{}}} + forge := &fakeForge{fail: c.forgeFail, heads: map[string]string{"o/f1@master": "c3"}} + cat := &flakyCatalog{fakeCatalog: fakeCatalog{regs: regs}, failFor: c.unreadFor} + var said []string + h := &Holder{Now: time.Now, Source: src, Forge: forge, Catalog: cat, + Logf: func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }} + startSourcesAtStart(h) + if !reflect.DeepEqual(waits, c.waits) || (src.vals["master"].Heads["o/f1"] == "c3") != c.started { + t.Fatalf("waits %v source %+v", waits, src.vals) + } + if c.forgeFail && !strings.Contains(strings.Join(said, "\n"), "o/f1@master") { + t.Fatalf("the left-out repository is not said: %v", said) + } + }) } } + +// flakyCatalog cannot read the registrations for its first failFor asks. +type flakyCatalog struct { + fakeCatalog + failFor int +} + +func (f *flakyCatalog) Registrations() (meshcatalog.RegistrationsAnswer, error) { + if f.failFor > 0 { + f.failFor-- + return meshcatalog.RegistrationsAnswer{}, errors.New("not served yet") + } + return f.fakeCatalog.Registrations() +} From 88ff34e855986c6a806fed5c0124f5fc17fac0a3 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 16 Nov 2026 09:46:12 +0100 Subject: [PATCH 17/24] distribution: a step given up at its bound is said given up, and its call's end is said later Builds: design 47 G A-443, A-443a --- .../cmd/distribution-tools/bounded.go | 11 +++++-- .../cmd/distribution-tools/bounded_test.go | 32 +++++++++++++++++++ .../cmd/distribution-tools/main.go | 11 ++++++- 3 files changed, 50 insertions(+), 4 deletions(-) diff --git a/modules/distribution/cmd/distribution-tools/bounded.go b/modules/distribution/cmd/distribution-tools/bounded.go index 14236ff47..7bf77f0dd 100644 --- a/modules/distribution/cmd/distribution-tools/bounded.go +++ b/modules/distribution/cmd/distribution-tools/bounded.go @@ -13,8 +13,9 @@ import ( // naming its step and the publish answers with it. const stepBound = 10 * time.Second -// step runs f, logs its start and its end with its time, and gives up on it after bound: the error names the step. An -// operation given up on keeps running in its goroutine until the bus answers it. +// step runs f, logs its start and its end with its time, and gives up on it after bound: the error names the step. A step +// given up on is not cancelled (the SDK's bus calls take no context): its call ends at the SDK's own bound of the ask, and +// the log says, by the step's name, that it was given up and later when the call behind it ended and how (A-443, A-443a). func step[T any](bound time.Duration, what string, f func() (T, error)) (T, error) { type res struct { v T @@ -31,7 +32,11 @@ func step[T any](bound time.Duration, what string, f func() (T, error)) (T, erro case <-time.After(bound): var zero T err := fmt.Errorf("%s did not answer within %s", what, bound) - logf("step %s: %v", what, err) + logf("step %s: %v; given up, its call is left to end at the SDK's own bound", what, err) + go func() { + r := <-ch + logf("step %s, given up after %s: its call ended after %s (%v)", what, bound, time.Since(start).Round(time.Millisecond), r.err) + }() return zero, err } } diff --git a/modules/distribution/cmd/distribution-tools/bounded_test.go b/modules/distribution/cmd/distribution-tools/bounded_test.go index e02f926ac..64cec0079 100644 --- a/modules/distribution/cmd/distribution-tools/bounded_test.go +++ b/modules/distribution/cmd/distribution-tools/bounded_test.go @@ -1,7 +1,9 @@ package main import ( + "fmt" "strings" + "sync" "testing" "time" ) @@ -19,6 +21,36 @@ func TestAHungStepAnswersWithItsName(t *testing.T) { } } +// A step given up on is said given up by name, and the log says later when the call behind it ended and how (A-443a). +func TestAGivenUpStepSaysWhenItsCallEnded(t *testing.T) { + var mu sync.Mutex + var lines []string + ended := make(chan struct{}) + w := func(f string, a ...any) { + l := fmt.Sprintf(f, a...) + mu.Lock() + lines = append(lines, l) + mu.Unlock() + if strings.Contains(l, "the put of a chunk") && strings.Contains(l, "its call ended after") { + close(ended) + } + } + old := logTo.Swap(&w) + defer logTo.Store(old) + release := make(chan struct{}) + if _, err := step(10*time.Millisecond, "the put of a chunk", func() (int, error) { <-release; return 0, fmt.Errorf("bus closed") }); err == nil { + t.Fatal("not given up") + } + close(release) + <-ended + mu.Lock() + all := strings.Join(lines, "\n") + mu.Unlock() + if !strings.Contains(all, "given up") || !strings.Contains(all, "bus closed") { + t.Fatalf("log: %s", all) + } +} + // The finding is said off the caller's path: a writer that never answers does not hold the call that says it. func TestAFindingIsSaidOffThePath(t *testing.T) { block := make(chan struct{}) diff --git a/modules/distribution/cmd/distribution-tools/main.go b/modules/distribution/cmd/distribution-tools/main.go index b20db6f5d..ae4a503b4 100644 --- a/modules/distribution/cmd/distribution-tools/main.go +++ b/modules/distribution/cmd/distribution-tools/main.go @@ -16,6 +16,7 @@ import ( "os" "strings" "sync" + "sync/atomic" "time" "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" @@ -29,7 +30,15 @@ import ( "distribution/internal/holder" ) -func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, "[distribution-tools] "+format+"\n", a...) } +// logTo is where logf writes; a test replaces it (atomically: a step given up on logs from its own goroutine). +var logTo atomic.Pointer[func(string, ...any)] + +func init() { + w := func(format string, a ...any) { fmt.Fprintf(os.Stderr, "[distribution-tools] "+format+"\n", a...) } + logTo.Store(&w) +} + +func logf(format string, a ...any) { (*logTo.Load())(format, a...) } // kept is one state of distribution's, through the runtime. type kept[T any] struct { From d2c1a24a24d50694672aba56e7f2789daf50ec11 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 16 Nov 2026 09:51:04 +0100 Subject: [PATCH 18/24] nats: the bus keeper no longer reads the seat mesh-catalog's states before the first views (reverts ed536102) A-379 replaced A-376 for the start, and the seat mesh-catalog offers neither the states modules and manifests as views nor verbs of those names, so the validator refused the manifest's two consumes and two invokes. Builds: design 47 G A-379, A-445 (G-22-lead-answers-2.md) --- modules/nats/cmd/nats-tools/catalogholder.go | 66 ------------------- .../nats/cmd/nats-tools/catalogholder_test.go | 45 ------------- modules/nats/cmd/nats-tools/keep.go | 20 +----- modules/nats/cmd/nats-tools/manifest_test.go | 8 +-- modules/nats/internal/keeper/keeper.go | 30 +-------- modules/nats/internal/keeper/keeper_test.go | 29 -------- modules/nats/module.json | 7 +- 7 files changed, 8 insertions(+), 197 deletions(-) delete mode 100644 modules/nats/cmd/nats-tools/catalogholder.go delete mode 100644 modules/nats/cmd/nats-tools/catalogholder_test.go diff --git a/modules/nats/cmd/nats-tools/catalogholder.go b/modules/nats/cmd/nats-tools/catalogholder.go deleted file mode 100644 index e8b5df706..000000000 --- a/modules/nats/cmd/nats-tools/catalogholder.go +++ /dev/null @@ -1,66 +0,0 @@ -package main - -import ( - "encoding/json" - "fmt" - - catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" - "git.novox.be/novox/mesh-sdk/go/seatlist" - - "nats-tools/internal/keeper" -) - -// The keeper makes the kept data of the holder of the seat mesh-catalog before the catalog half's first views are in (design -// 47 G A-376, completing A-367): that holder is the catalog half's own source and needs its kept data to answer them. Its -// manifest on the trunk is read as the holder's first views `modules` (the manifest's digest by module) and `manifests` (the -// manifest by digest) give it, both followed as the seat's states (views.ProcessReader.FollowSeat), and the holder is the -// module the seat that places modules answers for the seat (seatlist.HolderOf): no module is named here. - -// holderAccepted is what the holder's manifest on the trunk asks of the bus: the holder's entry of `modules` names its -// manifest's digest, and `manifests` holds that manifest. An error says what is not yet in. -func holderAccepted(holder string, modules, manifests map[string]json.RawMessage) (keeper.Accepted, error) { - raw, ok := modules[holder] - if !ok { - return keeper.Accepted{}, fmt.Errorf("the first view `modules` holds no entry of %s yet", holder) - } - // Only the manifest's digest is read of the entry, so an entry whose fingerprint is not (yet) there still names it. - var v struct { - Manifest string `json:"manifest"` - } - if err := json.Unmarshal(raw, &v); err != nil { - return keeper.Accepted{}, fmt.Errorf("the entry of %s in `modules` does not read: %w", holder, err) - } - if v.Manifest == "" { - return keeper.Accepted{}, fmt.Errorf("the entry of %s in `modules` names no manifest", holder) - } - m, ok := manifests[v.Manifest] - if !ok { - return keeper.Accepted{}, fmt.Errorf("the first view `manifests` holds no manifest %s of %s yet", v.Manifest, holder) - } - var by catalog.ManifestsByDigestValue - if err := json.Unmarshal(m, &by); err != nil { - return keeper.Accepted{}, fmt.Errorf("the manifest %s of %s does not read: %w", v.Manifest, holder, err) - } - var manifest map[string]any - if err := json.Unmarshal([]byte(by.Manifest), &manifest); err != nil { - return keeper.Accepted{}, fmt.Errorf("the manifest %s of %s does not read: %w", v.Manifest, holder, err) - } - a, err := AsksOf(catalog.CatalogEntry{Module: holder, Manifest: manifest}) - if err != nil { - return keeper.Accepted{}, err - } - return a.Accepted, nil -} - -// holder is the keeper's Holder: the seat's holder through the seat that places modules, its two states followed as the -// seat's, nothing asked on a timer (the follows read the holder again at each change of a view the reader follows). -func (f *catalogFollow) holder() (keeper.Accepted, error) { - if f.holderModules == nil || f.holderManifests == nil { - return keeper.Accepted{}, fmt.Errorf("the seat %s's states are not followed", catalog.Seat) - } - who, err := seatlist.HolderOf(f.call, catalog.Seat) - if err != nil { - return keeper.Accepted{}, err - } - return holderAccepted(who, f.holderModules.Follower().Values(), f.holderManifests.Follower().Values()) -} diff --git a/modules/nats/cmd/nats-tools/catalogholder_test.go b/modules/nats/cmd/nats-tools/catalogholder_test.go deleted file mode 100644 index 4fb7dfacf..000000000 --- a/modules/nats/cmd/nats-tools/catalogholder_test.go +++ /dev/null @@ -1,45 +0,0 @@ -package main - -import ( - "encoding/json" - "strings" - "testing" - - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" -) - -// TestHolderAcceptedReadsTheHoldersManifestFromItsFirstViews is design 47 G A-376: the entry the keeper makes before the catalog -// half's first views are in is the bus state of the holder's manifest, read through `modules` (its digest) and `manifests`. -func TestHolderAcceptedReadsTheHoldersManifestFromItsFirstViews(t *testing.T) { - digest := meshcatalogclient.ManifestDigest([]byte(`{"module":"idx","bus-state":[{"name":"revisions","storage":"object-store","template":"objects","options":{"max-mib":8}}]}`)) - mod := func(d string) json.RawMessage { return json.RawMessage(`{"v":1,"manifest":"` + d + `"}`) } - man := func(text string) json.RawMessage { - b, _ := json.Marshal(map[string]any{"v": 1, "module": "idx", "manifest": text}) - return b - } - rows := []struct { - name string - modules, manifests map[string]json.RawMessage - err string - module string - }{ - {"read", map[string]json.RawMessage{"idx": mod(digest)}, map[string]json.RawMessage{digest: man(`{"module":"idx","bus-state":[{"name":"revisions","storage":"object-store","template":"objects","options":{"max-mib":8}}]}`)}, "", "idx"}, - {"no entry of the holder yet", map[string]json.RawMessage{}, nil, "holds no entry of idx", ""}, - {"its manifest is not in yet", map[string]json.RawMessage{"idx": mod(digest)}, map[string]json.RawMessage{}, "holds no manifest", ""}, - {"an entry that names no manifest", map[string]json.RawMessage{"idx": mod("")}, nil, "names no manifest", ""}, - } - for _, row := range rows { - t.Run(row.name, func(t *testing.T) { - a, err := holderAccepted("idx", row.modules, row.manifests) - if row.err != "" { - if err == nil || !strings.Contains(err.Error(), row.err) { - t.Fatalf("err %v, want it to contain %q", err, row.err) - } - return - } - if err != nil || a.Module != row.module || len(a.BusState) != 1 || a.BusState[0].Name != "revisions" { - t.Fatalf("got %+v, %v", a, err) - } - }) - } -} diff --git a/modules/nats/cmd/nats-tools/keep.go b/modules/nats/cmd/nats-tools/keep.go index 46a012b52..366aacda6 100644 --- a/modules/nats/cmd/nats-tools/keep.go +++ b/modules/nats/cmd/nats-tools/keep.go @@ -240,7 +240,7 @@ func runKeeper(ctx context.Context, getenv func(string) string, logOut io.Writer if err != nil { return err } - k.Catalog, k.Seats, k.Again, k.Holder = follow.view, follow.seats, follow.again, follow.holder + k.Catalog, k.Seats, k.Again = follow.view, follow.seats, follow.again // The process answers the seat mesh-broker's snapshot verbs itself (design 47 G §681, A-107), with the caller the bus proved. stopAnswers, err := conn.AnswerSeat(meshbroker.Seat, seatlist.Definition(func() *seatlist.Reader { return follow.seatList }, meshbroker.Seat), snapshotAnswers(k, heldRequest(k.Store), keeperSnapshots(nc, logger.Printf))) if err != nil { @@ -259,15 +259,11 @@ type catalogFollow struct { branches *meshcatalogclient.Branches seatList *seatlist.Reader routes *eventRoutes - // call asks a seat verb or a module tool as the keeper's account; holderModules and holderManifests are the seat - // mesh-catalog's states `modules` and `manifests` (design 47 G A-376). - call seatlist.HolderCaller - holderModules, holderManifests *views.Picture } // followCatalog wires the branch follower and the seat list, and asks their first views; moved is told of each change. func followCatalog(ctx context.Context, conn *process.Conn, writer views.WireWriter, moved func(), logf func(string, ...any)) (*catalogFollow, error) { - f := &catalogFollow{ctx: ctx, routes: &eventRoutes{by: map[string]func(jetstream.Msg){}}, call: conn.Call} + f := &catalogFollow{ctx: ctx, routes: &eventRoutes{by: map[string]func(jetstream.Msg){}}} f.reader = &views.ProcessReader{Module: keeper.Module, Call: conn.Call, Logf: logf, Findings: func() views.WireWriter { return writer }} branches, err := meshcatalogclient.FollowBranchesProcess(f.reader, conn.Call) if err != nil { @@ -287,18 +283,6 @@ func followCatalog(ctx context.Context, conn *process.Conn, writer views.WireWri if err != nil { logf("the seat list cannot be followed (%v): a module's sent store waits for it", err) } - // The catalog half's own holder (design 47 G A-376): its states `modules` and `manifests`, whichever module holds the seat. - holders := func(seat string) []string { - if who, err := seatlist.HolderOf(f.call, seat); err == nil { - return []string{who} - } - return nil - } - for state, into := range map[string]**views.Picture{"modules": &f.holderModules, "manifests": &f.holderManifests} { - if *into, err = f.reader.FollowSeat(catalog.Seat, state, holders); err != nil { - logf("the seat %s's state %s cannot be followed (%v): its holder's kept data waits for the catalog half", catalog.Seat, state, err) - } - } f.reader.Ask(ctx, nil) return f, nil } diff --git a/modules/nats/cmd/nats-tools/manifest_test.go b/modules/nats/cmd/nats-tools/manifest_test.go index b7303f5a9..a6330a88a 100644 --- a/modules/nats/cmd/nats-tools/manifest_test.go +++ b/modules/nats/cmd/nats-tools/manifest_test.go @@ -195,16 +195,14 @@ func TestTheBussManifestMountsItsProgramsIntoTheUpstreamImage(t *testing.T) { t.Errorf("claims are %+v: mesh-broker, serving its reading verbs %v and its snapshot verbs %v alone", nm.Claims, ReadingVerbs, SnapshotVerbs) } // The catalog half replaces manifest-accepted and indexer_bus_state (design 47 G §433, §614), and the seat list is read - // through the seat mesh-catalog (§537); its holder's states modules and manifests, and the seat that places modules, are - // read before the first views (A-376). + // through the seat mesh-catalog (§537). catalogHalf := []string{"mesh-catalog.catalog-revision", "mesh-catalog.registration-changed", "mesh-delivery.source-moved", - "mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", "seat:mesh-catalog.views.modules", "seat:mesh-catalog.views.manifests"} + "mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed"} if !slices.Equal(nm.Consumes, append(slices.Clone(catalogHalf), "mesh-conditions.findings-asked")) { t.Errorf("consumes %v: the catalog half's and the seat list's events %v, and the finding writer's findings-asked (design 47 G A-17)", nm.Consumes, catalogHalf) } for _, want := range []string{"seat:mesh-catalog.revision", "seat:mesh-catalog.registrations", "seat:mesh-delivery.source", - "seat:mesh-catalog.seat-definitions", "seat:mesh-hand-acts.hand-act", "seat:mesh-catalog.modules", "seat:mesh-catalog.manifests", - "seat:mesh-module-assigner.assignments"} { + "seat:mesh-catalog.seat-definitions", "seat:mesh-hand-acts.hand-act"} { if !slices.Contains(nm.Invokes, want) { t.Errorf("invokes %v lack %s", nm.Invokes, want) } diff --git a/modules/nats/internal/keeper/keeper.go b/modules/nats/internal/keeper/keeper.go index 7f980eca5..68788bed4 100644 --- a/modules/nats/internal/keeper/keeper.go +++ b/modules/nats/internal/keeper/keeper.go @@ -332,12 +332,8 @@ type Keeper struct { // branch has no first view yet, when the keeper plans nothing. Seats are the seat list's definitions, each resolved // with the parts it includes (§537). Again asks every first view of the catalog half again, by hand. Catalog func() (CatalogView, error) - // Holder reads the manifest on the trunk of the holder of the seat mesh-catalog, the catalog half's own source, as the - // indexer's first views `modules` and `manifests` give it (design 47 G A-376): what the keeper makes before the - // catalog half's first views are in. Nil where nothing reads it; an error while it cannot be read. - Holder func() (Accepted, error) - Seats func() map[string]seatdef.Definition - Again func() error + Seats func() map[string]seatdef.Definition + Again func() error // Act records a hand act through the seat mesh-hand-acts; Warrant reads the router's record of an ask. Act func(what, why, cause string) error Warrant func(askID string) (*oc.Ask, *oc.Warrant, error) @@ -467,7 +463,6 @@ func (k *Keeper) PlanCatalog() error { view, err := k.Catalog() if err != nil { k.setStatus(func(s *Status) { s.FirstView.Asked, s.FirstView.Error = at, err.Error() }) - k.planHolder() return err } k.mu.Lock() @@ -479,7 +474,6 @@ func (k *Keeper) PlanCatalog() error { k.setStatus(func(s *Status) { s.FirstView.Asked, s.FirstView.Error = at, "the catalog half lists no module yet (none-yet): nothing is planned or orphaned until a revision lists one" }) - k.planHolder() return nil } k.viewed = true @@ -536,21 +530,6 @@ func (k *Keeper) PlanCatalog() error { return nil } -// planHolder makes the entries of the holder of the seat mesh-catalog before the catalog half's first views are in (design 47 -// G A-376, which completes A-367): the catalog half's own owner needs its kept data to answer them, so it is read from the -// holder's manifest on the trunk and made as Heard makes any module's, each entry said as ever; every other module waits. -func (k *Keeper) planHolder() { - if k.Holder == nil || k.Viewed() { - return - } - a, err := k.Holder() - if err != nil { - k.logf("the holder of the seat %s is not read yet, so its kept data is not made before the catalog half's first views: %v", FirstViewOwner, err) - return - } - k.hear(a) -} - // drifts says each drift once (design 47 G §532): kept in the module's key, said again only when the branch or the // settings planned change. func (k *Keeper) drifts(ds []Drift) { @@ -609,11 +588,6 @@ func (k *Keeper) Heard(a Accepted) { k.logf("the entry of %s is heard before the catalog half's first views: nothing is planned until they are in", a.Module) return } - k.hear(a) -} - -// hear plans one module's entry (Heard, and planHolder before the first views). -func (k *Keeper) hear(a Accepted) { held, _, found := k.asked(a.Module) if !found { held = Asked{V: 1, Module: a.Module} diff --git a/modules/nats/internal/keeper/keeper_test.go b/modules/nats/internal/keeper/keeper_test.go index 506b44980..500eb3d04 100644 --- a/modules/nats/internal/keeper/keeper_test.go +++ b/modules/nats/internal/keeper/keeper_test.go @@ -575,35 +575,6 @@ func TestTheKeeperMakesWhatAVerdictAsksForAndSaysIt(t *testing.T) { t.Error("the first revision with entries is viewed and plans as before") } }}, - // A-376: before the first views are in (read refused, or none-yet) the holder of the seat mesh-catalog is made from its - // trunk manifest, each entry said as ever; every other module waits. - {"before the first views the catalog half's own holder is made and no other module", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) { - k.mu.Lock() - k.viewed = false - k.mu.Unlock() - k.Heard(Accepted{Module: "other", Revision: 1, BusState: []catalogue.BusStateEntry{drafts}}) - k.Holder = func() (Accepted, error) { - return Accepted{Module: "idx", BusState: []catalogue.BusStateEntry{drafts}}, nil - } - for _, view := range []func() (CatalogView, error){ - func() (CatalogView, error) { return CatalogView{}, errors.New("no first view yet") }, - func() (CatalogView, error) { return CatalogView{}, nil }, - } { - k.Catalog = view - k.PlanCatalog() - if _, ok := k.planned("idx"); !ok || fmt.Sprint(out.events) != "[created idx drafts]" { - t.Fatalf("planned idx %v, said %v: the holder's entry is made once", ok, out.events) - } - if _, ok := k.planned("other"); ok || k.Viewed() { - t.Errorf("another module is planned or the keeper counts as viewed") - } - } - k.Holder = func() (Accepted, error) { return Accepted{}, errors.New("not read") } - k.PlanCatalog() // a holder not read plans nothing and says why in the status - if fmt.Sprint(out.events) != "[created idx drafts]" { - t.Errorf("said %v", out.events) - } - }}, // §538: a module in no watched branch's newest revision has its plans key deleted, its entries then orphans. {"a module no watched branch lists loses its plan, its entries orphans", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) { k.Catalog = func() (CatalogView, error) { diff --git a/modules/nats/module.json b/modules/nats/module.json index 549f43114..ab20bcd8a 100644 --- a/modules/nats/module.json +++ b/modules/nats/module.json @@ -65,10 +65,7 @@ "seat:mesh-catalog.revision", "seat:mesh-catalog.registrations", "seat:mesh-delivery.source", - "seat:mesh-catalog.seat-definitions", - "seat:mesh-catalog.modules", - "seat:mesh-catalog.manifests", - "seat:mesh-module-assigner.assignments" + "seat:mesh-catalog.seat-definitions" ], "consumes": [ "mesh-catalog.catalog-revision", @@ -76,8 +73,6 @@ "mesh-delivery.source-moved", "mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", - "seat:mesh-catalog.views.modules", - "seat:mesh-catalog.views.manifests", "mesh-conditions.findings-asked" ], "uses": [ From 1168376930251c30bb6dda287c80ec73a319e0ed Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 16 Nov 2026 11:09:26 +0100 Subject: [PATCH 19/24] Go build module: a fetch asked with dir chooses that module and its local replaces alone (A-447) Builds: design 47 G (G-22 lead answers 2) A-447 --- .../mesh-build-go/cmd/mesh-build-go/fetch.go | 72 +++++++++++++++++++ .../cmd/mesh-build-go/gomoddirs_test.go | 53 ++++++++++++++ 2 files changed, 125 insertions(+) diff --git a/modules/mesh-build-go/cmd/mesh-build-go/fetch.go b/modules/mesh-build-go/cmd/mesh-build-go/fetch.go index b4d5ab854..ffffd85ed 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/fetch.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/fetch.go @@ -20,6 +20,8 @@ import ( "sync" "time" + "golang.org/x/mod/modfile" + sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" dforgekey "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/forgekey" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/steprun" @@ -165,6 +167,17 @@ func (m *Module) fetchInto(ctx context.Context, req sdkstep.FetchRequest, id str if err != nil { return nil, err } + if dir := cleanDir(req.Dir); dir != "" { + // A fetch asked with a module's directory is of that Go module and what its go.mod replaces with a local path, + // never another Go module of the tree (design 47 G, A-447). Said beside the tree, so every later step lists the + // same modules. + if dirs, err = scopedDirs(tree, dir, dirs); err != nil { + return nil, err + } + if err := os.WriteFile(filepath.Join(root, scopeMark), []byte(strings.Join(dirs, "\n")), 0o600); err != nil { + return nil, err + } + } res.Modules = dirs if base != "" && res.Merge == "" { return res, nil // a conflict: the tree is at the trunk, nothing of the head is built or tested @@ -434,7 +447,11 @@ func (m *Module) downloadAndVerify(ctx context.Context, dir, d string) error { } // goModDirs are the directories of the tree that hold a go.mod, relative to it, in order. +// When the fetch was asked with a module's directory (scopeMark), the steps after it see only the directories it chose. func (m *Module) goModDirs(ctx context.Context, tree string) ([]string, error) { + if b, err := os.ReadFile(filepath.Join(filepath.Dir(tree), scopeMark)); err == nil { + return strings.Fields(string(b)), nil + } r := m.Run(ctx, nil, "git", "-C", tree, "ls-files", "--", "go.mod", "*/go.mod") if r.Status != 0 || r.Err != "" { return nil, fmt.Errorf("git ls-files: %s", dforgekey.Said(r.Err+"\n"+r.Stderr)) @@ -608,3 +625,58 @@ func narrowed(tree string) bool { _, err := os.Stat(filepath.Join(filepath.Dir(tree), narrowedMark)) return err == nil } + +// scopeMark is the file beside a workspace's tree naming the Go module directories a fetch asked with a module's +// directory chose, one to a line (empty: none). +const scopeMark = "gomodules" + +// cleanDir is a request's module directory in the form goModDirs lists, "" for none. +func cleanDir(dir string) string { + if strings.TrimSpace(dir) == "" { + return "" + } + return strings.Trim(filepath.ToSlash(filepath.Clean(dir)), "/") +} + +// scopedDirs are the go.mod directories of all that a module directory dir chooses: dir itself and each directory its +// go.mod replaces with a local path, followed through their own replaces (design 47 G, A-447), in order. A directory +// the tree holds no go.mod for is left out. +func scopedDirs(tree, dir string, all []string) ([]string, error) { + have := map[string]bool{} + for _, d := range all { + have[d] = true + } + chosen := map[string]bool{} + todo := []string{dir} + for len(todo) > 0 { + d := todo[0] + todo = todo[1:] + if chosen[d] || !have[d] { + continue + } + chosen[d] = true + file := filepath.Join(tree, filepath.FromSlash(d), "go.mod") + body, err := os.ReadFile(file) + if err != nil { + return nil, err + } + f, err := modfile.Parse(file, body, nil) // not ParseLax: that one ignores the replaces + if err != nil { + return nil, fmt.Errorf("%s: %w", file, err) + } + for _, r := range f.Replace { + if !modfile.IsDirectoryPath(r.New.Path) { + continue + } + target := filepath.ToSlash(filepath.Clean(filepath.Join(filepath.FromSlash(d), filepath.FromSlash(r.New.Path)))) + todo = append(todo, target) + } + } + out := []string{} + for _, d := range all { + if chosen[d] { + out = append(out, d) + } + } + return out, nil +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/gomoddirs_test.go b/modules/mesh-build-go/cmd/mesh-build-go/gomoddirs_test.go index dfd9bb678..e5d2d2352 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/gomoddirs_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/gomoddirs_test.go @@ -2,6 +2,8 @@ package main import ( "context" + "os" + "path/filepath" "reflect" "testing" @@ -25,3 +27,54 @@ func TestGoModDirsLeaveOutTheFixturesUnderTestdata(t *testing.T) { t.Fatalf("go.mod directories = %v, want %v", got, want) } } + +// A-447: a fetch asked with a module's directory chooses that Go module and the directories its go.mod replaces with a +// local path, through their own replaces; the later steps list the same. The takein module of the indexer is no module +// of the assigner's build (live, 2026-11-16). +func TestAFetchAskedWithADirectoryChoosesItsModuleAndWhatItReplacesAlone(t *testing.T) { + all := []string{"modules/assigner", "modules/indexer", "modules/indexer/takein", "modules/lib", "modules/lib/deep", "modules/other"} + mods := map[string]string{ + "modules/assigner": "module a\nreplace x => ../lib\nreplace (\n\ty => ../../outside\n\tz v1.0.0 => z v1.1.0\n)\n", + "modules/indexer": "module i\nreplace t => ./takein\n", + "modules/indexer/takein": "module t\n", + "modules/lib": "module l\nreplace d => ./deep\n", + "modules/lib/deep": "module d\n", + "modules/other": "module o\n", + } + for _, c := range []struct { + dir string + want []string + }{ + {"modules/assigner", []string{"modules/assigner", "modules/lib", "modules/lib/deep"}}, + {"modules/indexer", []string{"modules/indexer", "modules/indexer/takein"}}, + {"modules/other/", []string{"modules/other"}}, + {"modules/none", []string{}}, + } { + tree := t.TempDir() + for d, body := range mods { + if err := os.MkdirAll(filepath.Join(tree, d), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(tree, d, "go.mod"), []byte(body), 0o600); err != nil { + t.Fatal(err) + } + } + got, err := scopedDirs(tree, cleanDir(c.dir), all) + if err != nil || !reflect.DeepEqual(got, c.want) { + t.Errorf("dir %q chooses %v, %v; want %v", c.dir, got, err, c.want) + } + } + // What the steps after the fetch list is what the fetch chose, beside the tree. + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, scopeMark), []byte("modules/assigner\nmodules/lib"), 0o600); err != nil { + t.Fatal(err) + } + m := &Module{} + got, err := m.goModDirs(context.Background(), filepath.Join(root, "tree")) + if err != nil || !reflect.DeepEqual(got, []string{"modules/assigner", "modules/lib"}) { + t.Errorf("the later steps list %v, %v", got, err) + } + if cleanDir("") != "" || cleanDir(".") != "." { + t.Errorf("cleanDir of nothing = %q, of . = %q", cleanDir(""), cleanDir(".")) + } +} From ccab80d7262a7c96f6b492c1dd85178f2a7f83eb Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 16 Nov 2026 11:16:16 +0100 Subject: [PATCH 20/24] Go fetch with dir: a directory with no go.mod chooses every go.mod under it; none chosen fails naming it (A-447a) Builds: design 47 G (G-22 lead answers 2) A-447a --- .../mesh-build-go/cmd/mesh-build-go/fetch.go | 16 +++++++++++++++- .../cmd/mesh-build-go/gomoddirs_test.go | 19 +++++++++++++++++-- 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/modules/mesh-build-go/cmd/mesh-build-go/fetch.go b/modules/mesh-build-go/cmd/mesh-build-go/fetch.go index ffffd85ed..6b0725251 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/fetch.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/fetch.go @@ -640,7 +640,8 @@ func cleanDir(dir string) string { // scopedDirs are the go.mod directories of all that a module directory dir chooses: dir itself and each directory its // go.mod replaces with a local path, followed through their own replaces (design 47 G, A-447), in order. A directory -// the tree holds no go.mod for is left out. +// the tree holds no go.mod for is left out. A directory holding no go.mod chooses every go.mod under it, and one that +// chooses no Go module at all is an error naming it (A-447a). func scopedDirs(tree, dir string, all []string) ([]string, error) { have := map[string]bool{} for _, d := range all { @@ -648,6 +649,16 @@ func scopedDirs(tree, dir string, all []string) ([]string, error) { } chosen := map[string]bool{} todo := []string{dir} + if !have[dir] { + // A directory with no go.mod of its own (a module whose Go code lies in go/, a contracts module with a go.mod per + // seat) is every go.mod under it; the tree is already narrowed to the module's paths (A-447a). + todo = nil + for _, d := range all { + if dir == "." || strings.HasPrefix(d, dir+"/") { + todo = append(todo, d) + } + } + } for len(todo) > 0 { d := todo[0] todo = todo[1:] @@ -678,5 +689,8 @@ func scopedDirs(tree, dir string, all []string) ([]string, error) { out = append(out, d) } } + if len(out) == 0 { + return nil, fmt.Errorf("the module directory %q holds no Go module (no go.mod in it or under it)", dir) + } return out, nil } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/gomoddirs_test.go b/modules/mesh-build-go/cmd/mesh-build-go/gomoddirs_test.go index e5d2d2352..75782a9fa 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/gomoddirs_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/gomoddirs_test.go @@ -5,6 +5,7 @@ import ( "os" "path/filepath" "reflect" + "strings" "testing" "git.novox.be/novox/mesh-sdk/go/forgekey" @@ -32,7 +33,7 @@ func TestGoModDirsLeaveOutTheFixturesUnderTestdata(t *testing.T) { // local path, through their own replaces; the later steps list the same. The takein module of the indexer is no module // of the assigner's build (live, 2026-11-16). func TestAFetchAskedWithADirectoryChoosesItsModuleAndWhatItReplacesAlone(t *testing.T) { - all := []string{"modules/assigner", "modules/indexer", "modules/indexer/takein", "modules/lib", "modules/lib/deep", "modules/other"} + all := []string{".", "modules/assigner", "modules/indexer", "modules/indexer/takein", "modules/lib", "modules/lib/deep", "modules/other", "rootless/x/go", "seats/a", "seats/b"} mods := map[string]string{ "modules/assigner": "module a\nreplace x => ../lib\nreplace (\n\ty => ../../outside\n\tz v1.0.0 => z v1.1.0\n)\n", "modules/indexer": "module i\nreplace t => ./takein\n", @@ -40,6 +41,10 @@ func TestAFetchAskedWithADirectoryChoosesItsModuleAndWhatItReplacesAlone(t *test "modules/lib": "module l\nreplace d => ./deep\n", "modules/lib/deep": "module d\n", "modules/other": "module o\n", + ".": "module r\n", + "seats/a": "module sa\nreplace q => ../q\n", + "seats/b": "module sb\n", + "rootless/x/go": "module rx\n", } for _, c := range []struct { dir string @@ -48,7 +53,11 @@ func TestAFetchAskedWithADirectoryChoosesItsModuleAndWhatItReplacesAlone(t *test {"modules/assigner", []string{"modules/assigner", "modules/lib", "modules/lib/deep"}}, {"modules/indexer", []string{"modules/indexer", "modules/indexer/takein"}}, {"modules/other/", []string{"modules/other"}}, - {"modules/none", []string{}}, + {"modules/lib/deep/", []string{"modules/lib/deep"}}, + {"seats", []string{"seats/a", "seats/b"}}, // a go.mod per seat, none in seats itself + {".", []string{"."}}, // the root holds a go.mod + {"rootless/x", []string{"rootless/x/go"}}, // Go code lies in go/ + {"modules/none", nil}, // chooses no Go module: an error naming it } { tree := t.TempDir() for d, body := range mods { @@ -60,6 +69,12 @@ func TestAFetchAskedWithADirectoryChoosesItsModuleAndWhatItReplacesAlone(t *test } } got, err := scopedDirs(tree, cleanDir(c.dir), all) + if c.want == nil { + if err == nil || !strings.Contains(err.Error(), c.dir) { + t.Errorf("dir %q chose %v, %v; want an error naming it", c.dir, got, err) + } + continue + } if err != nil || !reflect.DeepEqual(got, c.want) { t.Errorf("dir %q chooses %v, %v; want %v", c.dir, got, err, c.want) } From 666be37916afe4e1bd875ab973be72c974c41ce3 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 16 Nov 2026 11:34:17 +0100 Subject: [PATCH 21/24] Go fetch is refused with no dir, naming its asker; the checking program's fetch names its directory (A-447b) Builds: design 47 G (G-22 lead answers 2) A-447b --- .../mesh-build-go/cmd/mesh-build-go/fetch.go | 15 +++++++++ .../cmd/mesh-build-go/fetch_test.go | 31 +++++++++++++++++-- .../cmd/mesh-build-go/gomoddirs_test.go | 21 +++++++++++++ .../cmd/mesh-build-go/mirror_test.go | 4 +++ .../mesh-build-go/narrow_integration_test.go | 2 +- .../cmd/mesh-build-go/pins_test.go | 10 +++++- .../mesh-build-go/cmd/mesh-build-go/run.go | 2 +- .../cmd/mesh-build-go/verify_once_test.go | 6 ++-- 8 files changed, 83 insertions(+), 8 deletions(-) diff --git a/modules/mesh-build-go/cmd/mesh-build-go/fetch.go b/modules/mesh-build-go/cmd/mesh-build-go/fetch.go index 6b0725251..2ab8fff54 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/fetch.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/fetch.go @@ -22,6 +22,8 @@ import ( "golang.org/x/mod/modfile" + stdio "git.novox.be/novox/mesh-sdk/go" + sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" dforgekey "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/forgekey" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/steprun" @@ -108,6 +110,11 @@ func (m *Module) fetch(ctx context.Context, args map[string]any) (res any, err e if err != nil { return nil, err } + // A build is for one module, never a repository (design 47 G, A-447b): a fetch names its module's directory. + if cleanDir(req.Dir) == "" { + return nil, fmt.Errorf("the fetch of %s at %s names no module directory (dir): a fetch is for one module, never a whole repository; asked by %s", + req.Repository, req.Commit, askerOf(args)) + } siblings, err := siblingsArg(args["beside"]) if err != nil { return nil, err @@ -694,3 +701,11 @@ func scopedDirs(tree, dir string, all []string) ([]string, error) { } return out, nil } + +// askerOf names who asked a tool: the caller the bus proved, else "an unnamed asker". +func askerOf(args map[string]any) string { + if c := stdio.CallerOf(args); c != "" { + return c + } + return "an unnamed asker" +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/fetch_test.go b/modules/mesh-build-go/cmd/mesh-build-go/fetch_test.go index 75c8823c9..aeaa3e44f 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/fetch_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/fetch_test.go @@ -61,6 +61,13 @@ func (f *fakeStep) run(ctx context.Context, env []string, name string, args ...s os.MkdirAll(filepath.Dir(filepath.Join(args[i+1], name)), 0o755) os.WriteFile(filepath.Join(args[i+1], name), []byte(text), 0o644) } + // Every go.mod the fake lists exists, since a fetch asked with a directory reads its go.mod (A-447). + for _, l := range strings.Fields(f.gomods) { + if _, err := os.Stat(filepath.Join(args[i+1], l)); err != nil && f.files[l] == "" { + os.MkdirAll(filepath.Dir(filepath.Join(args[i+1], l)), 0o755) + os.WriteFile(filepath.Join(args[i+1], l), []byte("module fake\n"), 0o644) + } + } for _, v := range f.vendors { os.MkdirAll(filepath.Join(args[i+1], v, "vendor"), 0o755) os.WriteFile(filepath.Join(args[i+1], v, "vendor", "modules.txt"), nil, 0o644) @@ -91,7 +98,24 @@ func fetchModule(t *testing.T, f *fakeStep) (*Module, *keptStates) { return m, states } +// fetchTool is the fetch tool as a build module is asked, always with its module's directory (A-447b): the root of the +// tree unless the args name one. rawFetchTool is the tool as it is. func fetchTool(t *testing.T, m *Module) func(map[string]any) (any, error) { + t.Helper() + raw := rawFetchTool(t, m) + return func(args map[string]any) (any, error) { + if _, ok := args["dir"]; !ok { + named := map[string]any{"dir": "."} + for k, v := range args { + named[k] = v + } + args = named + } + return raw(args) + } +} + +func rawFetchTool(t *testing.T, m *Module) func(map[string]any) (any, error) { t.Helper() for _, tool := range m.Tools(m.StepTools()) { if tool.Name == "build_go_fetch" { @@ -107,7 +131,9 @@ var fetchArgs = map[string]any{"repository": "novox/mesh-sdk", "commit": commitS // 1485/AC1: git is asked through FetchCommit with the ssh remote, and go mod download and verify run in each go.mod // directory (not in one that vendors) with GOTOOLCHAIN=local, GOFLAGS=-mod=mod and the module's cache paths. func TestFetchAsksGitForTheSshRemoteAndDownloadsEachModule(t *testing.T) { - f := &fakeStep{gomods: "go.mod\nsub/dir/go.mod\nvend/go.mod\n", vendors: []string{"vend"}} + // The root go.mod replaces the other two by directory: a fetch asked with a module's directory chooses what it replaces. + f := &fakeStep{gomods: "go.mod\nsub/dir/go.mod\nvend/go.mod\n", vendors: []string{"vend"}, + files: map[string]string{"go.mod": "module fake\nreplace a => ./sub/dir\nreplace b => ./vend\n"}} m, states := fetchModule(t, f) if _, err := fetchTool(t, m)(fetchArgs); err != nil { t.Fatal(err) @@ -270,7 +296,8 @@ func TestAVerifyThatFailsEndsTheStepNamingTheDirectoryAndTheLine(t *testing.T) { return forgekey.Ran{} }} m, states := fetchModule(t, f) - if _, err := fetchTool(t, m)(fetchArgs); err != nil { + // Asked for the module of tools, as a build is for one module (A-447b). + if _, err := fetchTool(t, m)(map[string]any{"repository": "novox/mesh-sdk", "commit": commitSHA, "dir": "tools"}); err != nil { t.Fatal(err) } st := states.waitEnd(t) diff --git a/modules/mesh-build-go/cmd/mesh-build-go/gomoddirs_test.go b/modules/mesh-build-go/cmd/mesh-build-go/gomoddirs_test.go index 75782a9fa..1001dbb0a 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/gomoddirs_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/gomoddirs_test.go @@ -8,6 +8,7 @@ import ( "strings" "testing" + sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" "git.novox.be/novox/mesh-sdk/go/forgekey" ) @@ -93,3 +94,23 @@ func TestAFetchAskedWithADirectoryChoosesItsModuleAndWhatItReplacesAlone(t *test t.Errorf("cleanDir of nothing = %q, of . = %q", cleanDir(""), cleanDir(".")) } } + +// A-447b: a build is for one module, never a repository: a fetch with no module directory is refused, naming its asker +// and the repository, and nothing is fetched. +func TestAFetchWithNoDirectoryIsRefusedNamingItsAsker(t *testing.T) { + f := &fakeStep{gomods: "go.mod\n"} + m, states := fetchModule(t, f) + if _, err := rawFetchTool(t, m)(map[string]any{"repository": "novox/mesh-sdk", "commit": commitSHA}); err != nil { + t.Fatal(err) + } + st := states.waitEnd(t) + if st.State != sdkstep.StepFailed || !strings.Contains(st.Reason, "novox/mesh-sdk") || !strings.Contains(st.Reason, "names no module directory") || + !strings.Contains(st.Reason, "asked by") { + t.Fatalf("a fetch with no dir ended %s: %q", st.State, st.Reason) + } + for _, l := range f.lines() { + if strings.HasPrefix(l, "git ") { + t.Errorf("a refused fetch ran %s", l) + } + } +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/mirror_test.go b/modules/mesh-build-go/cmd/mesh-build-go/mirror_test.go index af3f63d02..a71ca19f0 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/mirror_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/mirror_test.go @@ -176,6 +176,10 @@ func TestTheWorkspaceIsSelfContainedAfterAFetchThroughTheMirror(t *testing.T) { return strings.TrimSpace(string(out)) } git(src, "init", "-q") + // A fetch is for one module (A-447b): the repository holds a module at its root. + if err := os.WriteFile(filepath.Join(src, "go.mod"), []byte("module example.org/x\n\ngo 1.26\n"), 0o644); err != nil { + t.Fatal(err) + } if err := os.WriteFile(filepath.Join(src, "f.txt"), []byte("one\n"), 0o644); err != nil { t.Fatal(err) } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/narrow_integration_test.go b/modules/mesh-build-go/cmd/mesh-build-go/narrow_integration_test.go index 2ce32371e..e7ef09577 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/narrow_integration_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/narrow_integration_test.go @@ -45,7 +45,7 @@ func TestAFetchWithNamedPathsNarrowsTheWorkspace(t *testing.T) { if strings.Join(res.Modules, ",") != "lib" { t.Errorf("the fetch answered the modules %v, want lib alone", res.Modules) } - _, st = fetched(t, m, states, map[string]any{"repository": "novox/narrow", "commit": head, "paths": []any{"cmd"}}) + _, st = fetched(t, m, states, map[string]any{"repository": "novox/narrow", "commit": head, "dir": "", "paths": []any{"cmd"}}) if st.State != sdkstep.StepFailed || !strings.Contains(st.Reason, "no module directory") { t.Errorf("paths without a directory: %s %q", st.State, st.Reason) } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/pins_test.go b/modules/mesh-build-go/cmd/mesh-build-go/pins_test.go index 17b5f20e8..5b957bc42 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/pins_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/pins_test.go @@ -50,7 +50,15 @@ func newPinHarness(t *testing.T, gomod string, vendor bool, pulls string) (*pinH for _, g := range strings.Fields(f.gomods) { d := filepath.Join(args[i+1], filepath.Dir(g)) os.MkdirAll(d, 0o755) - os.WriteFile(filepath.Join(d, "go.mod"), []byte(gomod), 0o644) + body := gomod + if g == "go.mod" { // the root replaces the other directories, so a fetch asked with it chooses them (A-447) + for _, o := range strings.Fields(f.gomods) { + if o != "go.mod" { + body += "\nreplace local/" + filepath.Dir(o) + " => ./" + filepath.Dir(o) + "\n" + } + } + } + os.WriteFile(filepath.Join(d, "go.mod"), []byte(body), 0o644) os.WriteFile(filepath.Join(d, "go.sum"), []byte("sum\n"), 0o644) } if vendor { diff --git a/modules/mesh-build-go/cmd/mesh-build-go/run.go b/modules/mesh-build-go/cmd/mesh-build-go/run.go index cdd204725..24c9e50eb 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/run.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/run.go @@ -36,7 +36,7 @@ func (m *Module) run(ctx context.Context, fetch steprun.StepFunc, a sdkstep.Step return nil, err } key := sdkstep.RunKey(job.Program, sdkstep.ToolchainUsed{Version: installed, Recipe: m.Recipe.Version()}, a.Packages) - fetchArgs := sdkstep.FetchRequest{Repository: job.Program.Repository, Commit: job.Program.Commit, Packages: a.Packages}.Args() + fetchArgs := sdkstep.FetchRequest{Repository: job.Program.Repository, Commit: job.Program.Commit, Dir: job.Program.Dir, Packages: a.Packages}.Args() exe, built, err := m.programOf(ctx, fetch, "programs", "checking program", key, fetchArgs, job.Program.Dir, job.Program.Package) if err != nil { return nil, err diff --git a/modules/mesh-build-go/cmd/mesh-build-go/verify_once_test.go b/modules/mesh-build-go/cmd/mesh-build-go/verify_once_test.go index 3305e5cfd..cf7beee85 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/verify_once_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/verify_once_test.go @@ -53,7 +53,7 @@ func TestGoModVerifyRunsOncePerGoModAndGoSumDigest(t *testing.T) { // Without bubblewrap the module cache is not read-only to a step: a pass is never reused. func TestAPassOfVerifyIsNotReusedWhenStepsAreNotShutOff(t *testing.T) { - f := &fakeStep{gomods: "a/go.mod\nb/go.mod\n", files: map[string]string{"a/go.mod": "m\n", "b/go.mod": "m\n"}} + f := &fakeStep{gomods: "a/go.mod\nb/go.mod\n", files: map[string]string{"a/go.mod": "module m\n", "b/go.mod": "module m\n"}} m, states := fetchModule(t, f) // no Bubblewrap fetchOnce(t, m, states, fetchArgs) fetchOnce(t, m, states, fetchArgs) @@ -65,7 +65,7 @@ func TestAPassOfVerifyIsNotReusedWhenStepsAreNotShutOff(t *testing.T) { // A verify that fails leaves no mark: the next fetch verifies again. func TestAFailedVerifyLeavesNoMark(t *testing.T) { fail := true - f := &fakeStep{gomods: "go.mod\n", files: map[string]string{"go.mod": "m\n"}} + f := &fakeStep{gomods: "go.mod\n", files: map[string]string{"go.mod": "module m\n"}} f.goRule = func(line string) forgekey.Ran { if strings.HasSuffix(line, " mod verify") && fail { return forgekey.Ran{Status: 1, Stdout: "x v1: dir has been modified\n"} @@ -88,7 +88,7 @@ func TestDownloadsRunSideBySideUpToTheBound(t *testing.T) { running, most := 0, 0 release := make(chan struct{}) // closed once two run at once: they overlap however slowly the machine starts them f := &fakeStep{gomods: "a/go.mod\nb/go.mod\nc/go.mod\nd/go.mod\n", files: map[string]string{ - "a/go.mod": "a\n", "b/go.mod": "b\n", "c/go.mod": "c\n", "d/go.mod": "d\n"}} + "a/go.mod": "module a\n", "b/go.mod": "module b\n", "c/go.mod": "module c\n", "d/go.mod": "module d\n"}} f.goRule = func(line string) forgekey.Ran { if strings.HasSuffix(line, " mod download") { mu.Lock() From 928de07c431cfb50812c2eedbb3fc179aaf04e2a Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 16 Nov 2026 11:37:23 +0100 Subject: [PATCH 22/24] Go build module: a step for a module compares its own module.json's generated files alone (A-447c) Builds: design 47 G (G-22 lead answers 2) A-447c --- .../mesh-build-go/cmd/mesh-build-go/fetch.go | 16 +++++++++ .../mesh-build-go/cmd/mesh-build-go/test.go | 8 +++-- .../cmd/mesh-build-go/test_test.go | 33 +++++++++++++++++++ 3 files changed, 55 insertions(+), 2 deletions(-) diff --git a/modules/mesh-build-go/cmd/mesh-build-go/fetch.go b/modules/mesh-build-go/cmd/mesh-build-go/fetch.go index 2ab8fff54..046e9a9b3 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/fetch.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/fetch.go @@ -184,6 +184,9 @@ func (m *Module) fetchInto(ctx context.Context, req sdkstep.FetchRequest, id str if err := os.WriteFile(filepath.Join(root, scopeMark), []byte(strings.Join(dirs, "\n")), 0o600); err != nil { return nil, err } + if err := os.WriteFile(filepath.Join(root, moduleDirMark), []byte(dir), 0o600); err != nil { + return nil, err + } } res.Modules = dirs if base != "" && res.Merge == "" { @@ -637,6 +640,19 @@ func narrowed(tree string) bool { // directory chose, one to a line (empty: none). const scopeMark = "gomodules" +// moduleDirMark is the file beside a workspace's tree naming the module directory the fetch was asked with: the one +// module.json a step compares generated files for (design 47 G, A-447c). +const moduleDirMark = "moduledir" + +// moduleDirOf is the module directory the workspace of tree was fetched for, ok false for a workspace with none. +func moduleDirOf(tree string) (string, bool) { + b, err := os.ReadFile(filepath.Join(filepath.Dir(tree), moduleDirMark)) + if err != nil { + return "", false + } + return string(b), true +} + // cleanDir is a request's module directory in the form goModDirs lists, "" for none. func cleanDir(dir string) string { if strings.TrimSpace(dir) == "" { diff --git a/modules/mesh-build-go/cmd/mesh-build-go/test.go b/modules/mesh-build-go/cmd/mesh-build-go/test.go index 4ec9307df..108c73940 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/test.go @@ -109,18 +109,22 @@ func (m *Module) test(ctx context.Context, args map[string]any) (any, error) { } // compareGenerated regenerates the generated Go files of each contracts module in the tree (every module.json in the tree, in -// any folder: ADR 0634 §8; CAT-BG5) and fails the level on any out of step, Why naming each path. +// any folder: ADR 0634 §8; CAT-BG5; for a workspace fetched for a module's directory, that module's own alone) and fails the level on any out of step, Why naming each path. func compareGenerated(tree string, res *TestResult) error { dirs, err := steprun.ModuleDirs(tree) if err != nil { return err } for _, dir := range dirs { + // A step for a module compares its own module.json alone, never another module's of the repository (A-447c). + rel, _ := filepath.Rel(tree, dir) + if own, ok := moduleDirOf(tree); ok && filepath.ToSlash(rel) != own { + continue + } paths, err := steprun.OutOfStep(dir, steprun.GeneratedRootGo) if err != nil { return err } - rel, _ := filepath.Rel(tree, dir) for _, p := range paths { res.OutOfStep = append(res.OutOfStep, filepath.ToSlash(filepath.Join(rel, p))) } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/test_test.go b/modules/mesh-build-go/cmd/mesh-build-go/test_test.go index 0931f3dcd..779b4dba5 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/test_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/test_test.go @@ -340,3 +340,36 @@ func TestADirectoryThatVendorsIsReadFromItsVendorDirectory(t *testing.T) { t.Errorf("the last GOFLAGS of go test in a vendoring directory is %q", flags) } } + +// A-447c: a workspace fetched for a module's directory compares the generated files of that module's own module.json +// alone, never another module's of the repository. +func TestUnitLevelComparesOnlyTheOwnModulesGeneratedFiles(t *testing.T) { + m, states, ws, _ := workspaceFor(t, "empty") + tree := filepath.Join(m.Work, ws, "tree") + for _, folder := range []string{"domain", "other"} { + for p, text := range map[string]string{ + "module.json": `{"module":"c","build":{"artifacts":[{"name":"go","kind":"package","language":"go","paths":["go"],"generated-from":"."}]}}`, + "seats/one-store/seat.json": `{"format":1,"seat":"one-store","scope":"mesh","grain":"seat-list","domain":"Module","decision":"ADR 1","verbs":[{"name":"query","input":{"type":"object"}}]}`, + "go/go.mod": "module example.org/c/go\n", + "go/stray.go": "package stray\n", + } { + full := filepath.Join(tree, folder, filepath.FromSlash(p)) + _ = os.MkdirAll(filepath.Dir(full), 0o755) + if err := os.WriteFile(full, []byte(text), 0o644); err != nil { + t.Fatal(err) + } + } + } + if err := os.WriteFile(filepath.Join(m.Work, ws, moduleDirMark), []byte("domain"), 0o600); err != nil { + t.Fatal(err) + } + res, _ := runUnit(t, m, states, ws) + if len(res.OutOfStep) == 0 { + t.Fatalf("nothing out of step: %+v", res) + } + for _, p := range res.OutOfStep { + if !strings.HasPrefix(p, "domain/") { + t.Errorf("out of step = %v, want the module's own files under domain/ alone", res.OutOfStep) + } + } +} From ae78fb039f99b2a3a90546806c4176063fc893ce Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 16 Nov 2026 11:40:18 +0100 Subject: [PATCH 23/24] Go fetch with no dir fetches the tree whole again, for the checker until G removes it (A-447e) Builds: design 47 G (G-22 lead answers 2) A-447e --- .../mesh-build-go/cmd/mesh-build-go/fetch.go | 17 ++--------------- .../cmd/mesh-build-go/gomoddirs_test.go | 18 ++++++------------ 2 files changed, 8 insertions(+), 27 deletions(-) diff --git a/modules/mesh-build-go/cmd/mesh-build-go/fetch.go b/modules/mesh-build-go/cmd/mesh-build-go/fetch.go index 046e9a9b3..cd61cc20a 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/fetch.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/fetch.go @@ -22,8 +22,6 @@ import ( "golang.org/x/mod/modfile" - stdio "git.novox.be/novox/mesh-sdk/go" - sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" dforgekey "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/forgekey" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/steprun" @@ -110,11 +108,8 @@ func (m *Module) fetch(ctx context.Context, args map[string]any) (res any, err e if err != nil { return nil, err } - // A build is for one module, never a repository (design 47 G, A-447b): a fetch names its module's directory. - if cleanDir(req.Dir) == "" { - return nil, fmt.Errorf("the fetch of %s at %s names no module directory (dir): a fetch is for one module, never a whole repository; asked by %s", - req.Repository, req.Commit, askerOf(args)) - } + // A fetch with no dir (the checker's check of a pull request, until G removes the checker; A-447e) fetches the tree + // whole: every go.mod of it. siblings, err := siblingsArg(args["beside"]) if err != nil { return nil, err @@ -717,11 +712,3 @@ func scopedDirs(tree, dir string, all []string) ([]string, error) { } return out, nil } - -// askerOf names who asked a tool: the caller the bus proved, else "an unnamed asker". -func askerOf(args map[string]any) string { - if c := stdio.CallerOf(args); c != "" { - return c - } - return "an unnamed asker" -} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/gomoddirs_test.go b/modules/mesh-build-go/cmd/mesh-build-go/gomoddirs_test.go index 1001dbb0a..f08f77218 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/gomoddirs_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/gomoddirs_test.go @@ -95,22 +95,16 @@ func TestAFetchAskedWithADirectoryChoosesItsModuleAndWhatItReplacesAlone(t *test } } -// A-447b: a build is for one module, never a repository: a fetch with no module directory is refused, naming its asker -// and the repository, and nothing is fetched. -func TestAFetchWithNoDirectoryIsRefusedNamingItsAsker(t *testing.T) { - f := &fakeStep{gomods: "go.mod\n"} +// A-447e: a fetch with no dir (the checker's check of a pull request, until G removes the checker) fetches the tree whole: +// it chooses every go.mod of it. +func TestAFetchWithNoDirectoryChoosesEveryGoModOfTheTree(t *testing.T) { + f := &fakeStep{gomods: "go.mod\nsub/go.mod\n"} m, states := fetchModule(t, f) if _, err := rawFetchTool(t, m)(map[string]any{"repository": "novox/mesh-sdk", "commit": commitSHA}); err != nil { t.Fatal(err) } st := states.waitEnd(t) - if st.State != sdkstep.StepFailed || !strings.Contains(st.Reason, "novox/mesh-sdk") || !strings.Contains(st.Reason, "names no module directory") || - !strings.Contains(st.Reason, "asked by") { - t.Fatalf("a fetch with no dir ended %s: %q", st.State, st.Reason) - } - for _, l := range f.lines() { - if strings.HasPrefix(l, "git ") { - t.Errorf("a refused fetch ran %s", l) - } + if st.State != sdkstep.StepPassed || !strings.Contains(string(st.Result), `"modules":[".","sub"]`) { + t.Fatalf("a fetch with no dir ended %s: %q %s", st.State, st.Reason, st.Result) } } From cdf168706ba9164dd52b44529916f40c744b8692 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 16 Nov 2026 11:42:56 +0100 Subject: [PATCH 24/24] Go steps test a module's own directories alone, not what its go.mod replaces from outside (A-447c) Builds: design 47 G (G-22 lead answers 2) A-447c --- .../mesh-build-go/cmd/mesh-build-go/digest.go | 2 +- .../mesh-build-go/cmd/mesh-build-go/fetch.go | 21 ++++++++++ .../cmd/mesh-build-go/gomoddirs_test.go | 41 +++++++++++++++++++ .../cmd/mesh-build-go/integration.go | 2 +- .../mesh-build-go/cmd/mesh-build-go/test.go | 2 +- 5 files changed, 65 insertions(+), 3 deletions(-) diff --git a/modules/mesh-build-go/cmd/mesh-build-go/digest.go b/modules/mesh-build-go/cmd/mesh-build-go/digest.go index fe7c06c79..9aa9c4616 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/digest.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/digest.go @@ -260,7 +260,7 @@ func (m *Module) digestHere(ctx context.Context, workspace, level string, names namesHeader = []string{"node-names " + strings.Join(sortedNames, ",")} } - dirs, err := m.goModDirs(ctx, tree) + dirs, err := m.testDirs(ctx, tree) if err != nil { return DigestResult{}, err } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/fetch.go b/modules/mesh-build-go/cmd/mesh-build-go/fetch.go index cd61cc20a..04a18aadf 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/fetch.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/fetch.go @@ -476,6 +476,27 @@ func (m *Module) goModDirs(ctx context.Context, tree string) ([]string, error) { return dirs, nil } +// testDirs are the directories the unit and integration steps and the digest take: the chosen go.mod directories at or +// under the module's own directory. A directory the module's go.mod replaces from outside its own stays for download and +// resolution alone; its tests are another module's (design 47 G, A-447c). A workspace fetched with no dir keeps every go.mod. +func (m *Module) testDirs(ctx context.Context, tree string) ([]string, error) { + all, err := m.goModDirs(ctx, tree) + if err != nil { + return nil, err + } + own, ok := moduleDirOf(tree) + if !ok || own == "." || own == "" { + return all, nil + } + out := []string{} + for _, d := range all { + if d == own || strings.HasPrefix(d, own+"/") { + out = append(out, d) + } + } + return out, nil +} + // inTestdata says a path lies under a directory named testdata, whose go.mod files are fixtures Go ignores, not modules // of the tree (issue 1810). func inTestdata(path string) bool { diff --git a/modules/mesh-build-go/cmd/mesh-build-go/gomoddirs_test.go b/modules/mesh-build-go/cmd/mesh-build-go/gomoddirs_test.go index f08f77218..1a7a8d429 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/gomoddirs_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/gomoddirs_test.go @@ -108,3 +108,44 @@ func TestAFetchWithNoDirectoryChoosesEveryGoModOfTheTree(t *testing.T) { t.Fatalf("a fetch with no dir ended %s: %q %s", st.State, st.Reason, st.Result) } } + +// A-447c: the unit level of a module whose go.mod replaces ../lib tests the module alone; lib is downloaded and resolved +// for it, and its tests are another module's. +func TestTheUnitLevelTestsOnlyTheModulesOwnDirectoriesNotWhatItReplaces(t *testing.T) { + f := &fakeStep{} + m, _ := fetchModule(t, f) + root := filepath.Join(m.Work, "ws1") + tree := filepath.Join(root, "tree") + for d, body := range map[string]string{"mod": "module example.org/mod\nreplace example.org/lib => ../lib\n", "lib": "module example.org/lib\n"} { + if err := os.MkdirAll(filepath.Join(tree, d), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(tree, d, "go.mod"), []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + for name, body := range map[string]string{scopeMark: "lib\nmod", moduleDirMark: "mod"} { + if err := os.WriteFile(filepath.Join(root, name), []byte(body), 0o600); err != nil { + t.Fatal(err) + } + } + dirs, err := m.testDirs(context.Background(), tree) + if err != nil || !reflect.DeepEqual(dirs, []string{"mod"}) { + t.Fatalf("the test directories = %v, %v; want mod alone", dirs, err) + } + if _, err := m.runLevel(context.Background(), tree, levelOpts{workspace: "ws1", level: LevelUnit}); err != nil { + t.Fatal(err) + } + for _, l := range f.lines() { + if strings.Contains(l, "/tree/lib test") { + t.Errorf("the module's unit level tested lib: %s", l) + } + } + // No dir asked: every go.mod of the tree stays. + if err := os.Remove(filepath.Join(root, moduleDirMark)); err != nil { + t.Fatal(err) + } + if dirs, _ = m.testDirs(context.Background(), tree); len(dirs) != 2 { + t.Errorf("a workspace with no module directory tests %v, want every go.mod", dirs) + } +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/integration.go b/modules/mesh-build-go/cmd/mesh-build-go/integration.go index 86c9d6db5..cfa51ba98 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/integration.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/integration.go @@ -58,7 +58,7 @@ func (m *Module) integrationHere(ctx context.Context, workspace, tree string, pa defer os.RemoveAll(run) began := m.clock() - dirs, err := m.goModDirs(ctx, tree) + dirs, err := m.testDirs(ctx, tree) if err != nil { return nil, err } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/test.go b/modules/mesh-build-go/cmd/mesh-build-go/test.go index 108c73940..e76b9b2c5 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/test.go @@ -256,7 +256,7 @@ type levelOpts struct { // runLevel runs `go test -json` in each go.mod directory of the tree and judges the whole against the bounds. func (m *Module) runLevel(ctx context.Context, tree string, o levelOpts) (*TestResult, error) { - dirs, err := m.goModDirs(ctx, tree) + dirs, err := m.testDirs(ctx, tree) if err != nil { return nil, err }