From 1e826ff0086ee32c33c74699c2b18bcda28a21bc Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 20 Nov 2026 03:30:01 +0100 Subject: [PATCH] I8, H-A-82: the earlier take-in goes; no delivered version is kept MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I8: the take-in fed by the forge bridge's messages goes with B1's forgemsg: the FORGE consumer and its handlers, the group end, the pending state with its keys, claims, dry take-in and hand tools (indexer_pending, indexer_take_in_pending, indexer_drop_pending, indexer_take_in_merge, kept again for the held take-in in I5), the catch-up by comparison (kickBehind, TakeInBehind, sayUntaken, takeInHeads, kickCatchUp, CatchUp, merge-untaken), the source numbering in memory (nextSource), the source verb's follow and Src stamping on a source answer, the bridge's merged message call, the pull request candidates written from pull and open messages, the checker's previews and verdict hearing. A repository's trees at a commit are the internal whole (wholeAt), which ApplyMessage takes in; the dry indexer stays for a run on request's candidate. The manifest drops the consumes of forge.**, the checker's verdict and version-delivered, the reads of the checker's previews and the invokes of open-pulls, bridge_merged_message and mesh-delivery.source. RoomAgain says room when a take-in is held on a full store (heldFull, I5 holds it). H-A-82: the fill of delivered versions into the edges goes: HearVersionDelivered, the delivered. key, the first fill's kinds 2, 4 and 6 from the placement and the stream running, and outdated's kinds 2, 4 and 6; their entries read not yet delivered. The tests of the removed paths go (index_test's candidate groups, source_test, branch_take_in_test, previews_test, candidate_manifests_test, the FORGE handle tests); the versions tests now hold kinds 2, 4 and 6 as declarations. The integration tests still drive FORGE and are rewritten in I10. Builds: design 47 H §743, §744, §764 (the indexer); H-A-48, H-A-82 Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy --- .../cmd/mesh-delivery-indexer-tools/main.go | 1 - .../mesh-delivery-indexer/bridgemessage.go | 46 - .../cmd/mesh-delivery-indexer/main.go | 189 +-- .../cmd/mesh-delivery-indexer/main_test.go | 206 --- .../cmd/mesh-delivery-indexer/named.go | 22 - .../cmd/mesh-delivery-indexer/named_test.go | 21 - .../cmd/mesh-delivery-indexer/previews.go | 66 - .../cmd/mesh-delivery-indexer/seatdefs.go | 27 +- .../cmd/mesh-delivery-indexer/source.go | 135 -- .../cmd/mesh-delivery-indexer/takein.go | 16 - .../cmd/mesh-delivery-indexer/versions.go | 74 - .../internal/index/behind.go | 190 --- .../internal/index/branch_take_in.go | 63 - .../internal/index/branch_take_in_test.go | 228 --- .../index/candidate_manifests_test.go | 109 -- .../internal/index/candidates.go | 287 +--- .../internal/index/catalog_revisions.go | 13 +- .../internal/index/control_records.go | 3 +- .../internal/index/control_records_test.go | 109 +- .../internal/index/dryrun.go | 88 + .../internal/index/fingerprints.go | 6 +- .../internal/index/helpers_test.go | 25 + .../internal/index/imports_test.go | 6 - .../internal/index/index_test.go | 289 ---- .../internal/index/manifests.go | 5 +- .../internal/index/manifests_test.go | 11 +- .../internal/index/modules.go | 17 +- .../internal/index/modules_test.go | 44 +- .../internal/index/moves.go | 18 +- .../internal/index/named_test.go | 23 - .../internal/index/names.go | 3 - .../internal/index/open_asks.go | 31 - .../internal/index/open_asks_test.go | 28 +- .../internal/index/order.go | 9 +- .../internal/index/own.go | 3 +- .../internal/index/pending.go | 1475 ----------------- .../internal/index/pending_tools.go | 109 -- .../internal/index/previews.go | 376 ----- .../internal/index/previews_test.go | 346 ---- .../internal/index/previews_values.go | 75 - .../internal/index/registered.go | 6 +- .../internal/index/registered_test.go | 10 +- .../internal/index/runcandidates.go | 3 +- .../internal/index/seatdefs.go | 40 +- .../internal/index/seatsof.go | 52 +- .../internal/index/series.go | 5 +- .../internal/index/source.go | 249 +-- .../internal/index/source_test.go | 167 -- .../internal/index/taken.go | 7 +- .../internal/index/toolchains.go | 10 +- .../internal/index/toolchains_test.go | 84 +- .../internal/index/unreadcall.go | 67 - .../internal/index/versions_edges.go | 127 +- .../internal/index/versions_fill.go | 13 +- .../internal/index/versions_fill_edges.go | 70 +- .../internal/index/versions_test.go | 59 +- .../internal/index/whole.go | 28 + .../mesh-delivery-indexer/manifest_test.go | 21 +- modules/mesh-delivery-indexer/module.json | 14 +- 59 files changed, 309 insertions(+), 5515 deletions(-) delete mode 100644 modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/bridgemessage.go delete mode 100644 modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/previews.go delete mode 100644 modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/source.go delete mode 100644 modules/mesh-delivery-indexer/internal/index/behind.go delete mode 100644 modules/mesh-delivery-indexer/internal/index/branch_take_in.go delete mode 100644 modules/mesh-delivery-indexer/internal/index/branch_take_in_test.go delete mode 100644 modules/mesh-delivery-indexer/internal/index/candidate_manifests_test.go create mode 100644 modules/mesh-delivery-indexer/internal/index/dryrun.go delete mode 100644 modules/mesh-delivery-indexer/internal/index/pending.go delete mode 100644 modules/mesh-delivery-indexer/internal/index/pending_tools.go delete mode 100644 modules/mesh-delivery-indexer/internal/index/previews.go delete mode 100644 modules/mesh-delivery-indexer/internal/index/previews_test.go delete mode 100644 modules/mesh-delivery-indexer/internal/index/previews_values.go delete mode 100644 modules/mesh-delivery-indexer/internal/index/source_test.go delete mode 100644 modules/mesh-delivery-indexer/internal/index/unreadcall.go create mode 100644 modules/mesh-delivery-indexer/internal/index/whole.go diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go index 011602e1..d6864417 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go @@ -266,7 +266,6 @@ func main() { return err })...) tools = append(tools, viewTools(os.Getenv, trunk.State, time.Now)...) - tools = append(tools, index.PendingTools(keptRead{stdio.State(index.PendingState)}, keptRead{stdio.State(index.OwnBuildStateName)}, asks, time.Now)...) tools = append(tools, index.FingerprintsTools(asks, time.Now, func(what, why string) error { _, err := acts.Record(meshhandactsclient.Request{What: what, Why: why, Cause: index.FingerprintsAgainTool, Verb: "fingerprints-again"}) return err diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/bridgemessage.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/bridgemessage.go deleted file mode 100644 index e56ae60d..00000000 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/bridgemessage.go +++ /dev/null @@ -1,46 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "fmt" - - "mesh-delivery-indexer/internal/index" -) - -// mergedTool is the forge bridge's tool that answers a merge's merged message again (issue 1585): arguments repository, -// merge and, when the indexer knows it, number (the bridge then reads a merge made on the forge itself), answer {message}, the message whole as the bridge writes it on FORGE. The indexer's manifest lists it in -// invokes, which is the grant to call it. -const mergedTool = index.ForgeBridgeModule + ".bridge_merged_message" - -// messageVia asks the forge bridge for a merge's merged message, for a held merge the indexer does not hold in memory (a -// restart, a new build), a kept call asked again only while the commit's take-in is pending (A-190). An error is the -// bridge's own words. -func messageVia(call openCaller, pending func(ctx context.Context, repository, merge string) (bool, error), logf func(string, ...any)) func(ctx context.Context, repository, merge string, number int) ([]byte, error) { - return func(ctx context.Context, repository, merge string, number int) ([]byte, error) { - ask := map[string]any{"repository": repository, "merge": merge} - if number > 0 { - ask["number"] = number - } - raw, err := keptWhileOpen(ctx, call, logf, "message."+repository+"."+merge, mergedTool, ask, - func(ctx context.Context) (bool, error) { return pending(ctx, repository, merge) }) - if err != nil { - return nil, err - } - var a struct { - Message json.RawMessage `json:"message"` - } - if err := json.Unmarshal(raw, &a); err != nil || len(a.Message) == 0 { - return nil, fmt.Errorf("the answer of %s is not {message}: %v", mergedTool, err) - } - return a.Message, nil - } -} - -// unreadAgain reads again, after a message heard on FORGE, a pending merge whose message was unread for a transport fault -// (index.Indexer.UnreadAgain; issue 1966 note 153): the forge is heard again, so the bridge is asked again. A failure is said. -func unreadAgain(ctx context.Context, ix *index.Indexer, logger interface{ Printf(string, ...any) }) { - if err := ix.UnreadAgain(ctx); err != nil { - logger.Printf("take-in: a merge unread for a transport fault is not read again after this message: %v", err) - } -} diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index cd7b175b..c9843751 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -11,7 +11,6 @@ package main import ( "cmp" "context" - "encoding/json" "errors" "fmt" "io" @@ -24,7 +23,6 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo" "git.novox.be/novox/mesh-sdk/go/buscalls" @@ -33,7 +31,6 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" stdio "git.novox.be/novox/mesh-sdk/go" mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" @@ -45,11 +42,6 @@ import ( "mesh-delivery-indexer/internal/index" ) -// forgeStream is the bridge's stream as the bus names it: its local name in capitals (mesh-control -// internal/catalogue/streams.go StreamBusName). The module's consumer on it is the durable _ the controller -// makes (internal/broker/modstreams.go HeardIssued); the indexer binds it, pulls and acks. -const forgeStream = "FORGE" - func main() { ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer stop() @@ -177,119 +169,6 @@ type delivered interface { Metadata() (*jetstream.MsgMetadata, error) } -// groupEndSubject is the whole subject of a delivery group's end on the forge bridge's stream: compared exactly, so a -// repository named forge/group, whose merged message ends the same way, is never taken for one. -var groupEndSubject = process.ModuleStreamSubject(index.ForgeBridgeModule, forgemsg.ForgeGroupEndEvent) - -// handleGroupEnd takes a group's end in (index.EndGroup): a body that does not read is terminated and said; an end -// whose members are not all held yet, or while the build this program runs is not known, is left for redelivery. -func handleGroupEnd(ctx context.Context, m delivered, ix *index.Indexer, h *index.Heard, logger *log.Logger) { - end, err := forgemsg.DecodeForgeGroupEnd(m.Data()) - if err != nil { - logger.Printf("%s: not a group's end, dropped: %v", m.Subject(), err) - h.Outcome, h.Words = index.HeardDropped, err.Error() - _ = m.Term() - return - } - h.Kind, h.Branch = forgemsg.ForgeGroupEndEvent, end.Group - applied, words, err := ix.EndGroup(ctx, end) - h.Written, h.Late = applied.Written, applied.Late - if err != nil { - logger.Printf("%s: the end of the group %q left for redelivery: %v", m.Subject(), end.Group, err) - h.Outcome, h.Words = index.HeardLeft, err.Error() - return - } - h.Outcome, h.Words = index.HeardApplied, words - if err := m.DoubleAck(ctx); err != nil { - logger.Printf("%s: the end of the group %q applied, its acknowledgement was not confirmed: %v", m.Subject(), end.Group, err) - } -} - -// handle takes one message of FORGE into the candidates and acknowledges it. A message it can never read is terminated, -// never redelivered, and the log names its subject and why. A write that failed stays unacknowledged: the server -// delivers it again (the server's ack wait is the only clock). Each message handled is written to the heard state -// (index.Heard; nil, or a state not on the bus yet, writes none), so indexer_heard reads what the indexer heard of FORGE. -func handle(ctx context.Context, m delivered, ix *index.Indexer, heard index.Store, logger *log.Logger) { - h := index.Heard{Subject: m.Subject()} - if md, err := m.Metadata(); err == nil && md != nil { - h.Sequence, h.Delivered, h.Published = md.Sequence.Stream, md.NumDelivered, md.Timestamp.UTC() - } - defer func() { - if heard == nil { - return - } - h.Handled = time.Now().UTC() - if err := index.WriteHeard(ctx, heard, h); err != nil && !errors.Is(err, process.ErrNotYet) { - logger.Printf("%s: sequence %d is not written to the heard state: %v", h.Subject, h.Sequence, err) - } - }() - // A delivery group's end (novox/hq ADR 0633 §4) has a body of its own: told apart by its name, before the decode. - if m.Subject() == groupEndSubject { - handleGroupEnd(ctx, m, ix, &h, logger) - return - } - if kind, known := forgeKind(m.Data()); !known { - // A kind the indexer does not take (the bridge's branch-deleted, design 51) is acknowledged and passed over, never - // terminated as unreadable: a new kind on FORGE is no fault of the message. - h.Kind, h.Outcome, h.Words = kind, index.HeardIgnored, "a "+kind+" message, which the indexer does not take" - _ = m.Ack() - return - } - f, err := forgemsg.DecodeForge(m.Data()) - if err != nil { - logger.Printf("%s: not a forge message, dropped: %v", m.Subject(), err) - h.Outcome, h.Words = index.HeardDropped, err.Error() - _ = m.Term() - return - } - h.Kind, h.Repository, h.Number, h.Branch, h.Commit = f.Kind, f.Repository, f.Number, f.Branch, f.Head - if f.Merge != "" { - h.Commit = f.Merge - } - f = index.RevisionForm(f) // a pushed message is taken in as a merged one is, its head where the merge stands (§509, A-269) - var applied index.Applied - if f.Kind == forgemsg.ForgeMerged && h.Sequence > 0 { - // A merge is taken in only by the indexer its revision names (ADR 0633 §1): kept in pending until then. - applied, h.Words, err = ix.Arrive(ctx, h.Sequence, f) - } else { - applied, err = ix.ApplyMessage(ctx, f) - } - h.Written, h.Late = applied.Written, applied.Late - switch { - case errors.Is(err, index.ErrUnreadable): - logger.Printf("%s: %v; dropped", m.Subject(), err) - h.Outcome, h.Words = index.HeardDropped, err.Error() - _ = m.Term() - case err != nil: - logger.Printf("%s: %s#%d left for redelivery: %v", m.Subject(), f.Repository, f.Number, err) - h.Outcome, h.Words = index.HeardLeft, err.Error() - default: - h.Outcome = index.HeardApplied - // Waits for the server, so the consumer's info no longer counts the message as unacknowledged when the fill that - // follows asks whether the stream is applied (issue 1280); a failed ack is redelivered after the ack wait, which - // kicks the fill again. - if err := m.DoubleAck(ctx); err != nil { - logger.Printf("%s: %s#%d applied, its acknowledgement was not confirmed: %v", m.Subject(), f.Repository, f.Number, err) - } - } -} - -// forgeKind is the kind a FORGE message names, and whether it is one the indexer takes; a body naming no kind is taken, -// so DecodeForge says why it is unreadable. -func forgeKind(data []byte) (string, bool) { - var probe struct { - Kind string `json:"kind"` - } - if json.Unmarshal(data, &probe) != nil || probe.Kind == "" { - return "", true - } - switch probe.Kind { - case forgemsg.ForgePull, forgemsg.ForgeMerged, forgemsg.ForgePushed, forgemsg.ForgeRefused, forgemsg.ForgeOpen: - return probe.Kind, true - } - return probe.Kind, false -} - func opOf(e jetstream.KeyValueEntry) bool { return e.Operation() == jetstream.KeyValuePut } @@ -402,7 +281,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro requirements := process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.RequirementsState), "what consumers require is not kept", logger.Printf) // The module graph (ADR 0603 §2) is held in memory, read with the view: each revision's entries carry its nodes. // The state heard (A-80): the forge messages handled (key forge) and what the indexer heard and compares later (the time a call is - // unanswered since, the newest toolchain-agreed and version-delivered, the artifacts published so far of a version). + // unanswered since, the newest toolchain-agreed, the artifacts published so far of a version). heardStore := awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.HeardState), "what the indexer heard is not kept", logger.Printf)} ownBuildState := process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.OwnBuildStateName), "which build this indexer runs is not kept", logger.Printf) // The runs on request the indexer holds a candidate for (A-174): a state this delivery brings, bound when it appears. @@ -453,8 +332,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro ix := &index.Indexer{Module: cred.Module, NamedSets: named, Findings: writer, OldSections: awaitedStore{b: oldSections}, Requirements: awaitedStore{b: requirements}, Store: kvStore{candidates}, Modules: modules, Manifests: kvStore{manifests}, Taken: awaitedStore{b: taken}, Builds: known, Predefined: awaitedStore{b: predefined}, Logf: logger.Printf, Named: namedVia(calls.Call), Trunk: trunkVia(calls.Call), BranchHead: branchHeadVia(calls.Call), Contains: containsVia(calls.Call), Read: readVia(calls.Call), - ReadOpen: openVia(calls.Call), Packages: awaitedStore{b: packages, js: js, name: index.Bucket(cred.Module, index.PackagesState)}, Files: filesVia(calls.Call), GoSum: goSumVia(calls.Call), GoMod: goModVia(calls.Call), - OwnBuild: ownBuildVia(calls.Call, cred.Module, cred.Node), OwnBuildState: awaitedStore{b: ownBuildState}, Heard: heardStore, Placement: placementVia(calls.CallKept), RunningWhole: runningWholeVia(js), Runs: awaitedStore{b: runs}, Graph: graph, + Packages: awaitedStore{b: packages, js: js, name: index.Bucket(cred.Module, index.PackagesState)}, Files: filesVia(calls.Call), GoSum: goSumVia(calls.Call), GoMod: goModVia(calls.Call), + OwnBuild: ownBuildVia(calls.Call, cred.Module, cred.Node), OwnBuildState: awaitedStore{b: ownBuildState}, Heard: heardStore, Runs: awaitedStore{b: runs}, Graph: graph, SeatDefinitions: awaitedStore{b: seatDefs}, ReadFile: readFileVia(calls.Call), SaySeatDefinition: saySeats, Revisions: revisions, EntryIndex: awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.EntryIndexState), "no entry is found by its fingerprint", logger.Printf)}, // Message a kept call: a read, called again through a transport fault (ADR 0668 §1) // Each catalog revision may bring another holder of a build seat: the build modules are found again (buildmodules.go). @@ -468,9 +347,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro Inputs: inputsStore, InputObjects: inObjects, SayVersionReleased: sayVersionReleased(conn, cred), SayVersionsFilled: sayVersionsFilled(conn, cred), - // The take-in step (ADR 0633): its state pending and its stream take-in, both this delivery's, bound when they appear; - // until then every merge is taken in at once, as before. - Pending: awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.PendingState), "no merge is held for the indexer its revision names: each is taken in at once", logger.Printf)}, SayTakeIn: sayTakeIn(js, cred)} // The registrations and the changes that wait for their warrant (design 47 G §2, §667): their states are bound when they // appear; a warrant is heard on the router's decided event for this module, below. @@ -478,7 +354,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.RegistrationsState), "no registration is kept or written", logger.Printf)}, awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.ProposalsState), "no change of a registration waits for its warrant", logger.Printf)}, routerRecordOf(cred.Module, calls.Call, calls.Ask), ix, logger.Printf) - ix.Message = messageVia(calls.CallKeptWhileOpen, ix.TakeInPending, logger.Printf) // a kept call, called again through a transport fault while the commit's take-in is pending (ADR 0668 §1, A-190) registry.Pruned = ix.PruneBranchBuilds ix.Registry = registry named.Changed = ix.RemoveUnlisted @@ -535,11 +410,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro started.Store(true) done() logger.Printf("start: the build log held %d entries at the start", len(startEnds)) - // The source revision of a watched branch (design 47 G §15, §115, §362): asked of the seat mesh-delivery's verb source as a - // kept call, once at the start, before the handler of its event source-moved reads anything; a source-moved heard - // meanwhile is read after. - source := &sourceFollow{read: sourceVia(ctx, calls.CallKeptWhileOpen, ix.BranchWatched, logger.Printf), apply: ix.ApplySource, applyBranch: ix.ApplyBranchSource, logf: logger.Printf} - go source.catchUp(ctx) // Only the toolchain reports of nodes the catalogue places the build module on count (issue 1940, ADR 0549 §3): the // assignments is settled first, so the toolchains settle through the gate already knowing it. // The build modules are no names (design 47 *A module's registration*): the holders of the seats whose acting rule is @@ -590,12 +460,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } buildModules.watching(tools) buildFollow.Store(buildModules) - // The checker's merge previews give the candidates their fingerprints: the bucket is watched where it exists, and - // otherwise when the checker says it wrote a preview. - previews := newPreviewWatch(ctx, js, ix, logger) - if err := previews.ensure(); err != nil { - return err - } // Only now, with the toolchains read, are the fingerprints computed once (before it every Go module would be written as // having no toolchain reported, and back). // The packages the held modules reach and the state does not hold yet are published at their trunk's taken merge, and @@ -660,12 +524,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // indexer_first_views and as view-waits past five minutes, and read again on the next forge message heard. defs := gitWaits(ctx, cred.Module, ix, writer, cred.Node, process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, views.StatusState), "indexer_first_views does not say the wait for the seat git", logger.Printf), logger.Printf) seatDefs.Ready(func(jetstream.KeyValue) { defs.Start(ctx) }) - // A take-in held while a state was not on the bus is evaluated again at the start (issue 1966 note 154). - go func() { - if err := ix.PendingAgain(ctx); err != nil { - logger.Printf("take-in: pending is not evaluated again at the start: %v", err) - } - }() // A build module says each time it wrote its toolchains key: its stream, and the indexer's durable consumer on it, are // made by the controller as the forge's are. Where the stream is not there yet the indexer says so and is started again // with the build module, as with the gate's source. @@ -695,9 +553,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // The versions each package's series holds (DLV-I5): the registry seats' published, on the // indexer's durable consumer of the shared events stream. Where it is not there yet, no series moves until the indexer // starts again, said. - if err := hub.Handle(meshdelivery.SubjectSourceMovedAll, func(m jetstream.Msg) { source.heard(ctx, m) }); err != nil { - return err - } // Once the state is bound every kept revision is judged against the pins it holds (design 47 H §749: at its start). pinsBucket.Ready(func(jetstream.KeyValue) { go ix.JudgePins(ctx) }) versions := func(m jetstream.Msg) { @@ -711,9 +566,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if err := hub.Handle(meshbuilds.SubjectToolchainAgreed, func(m jetstream.Msg) { heardToolchainAgreed(ctx, ix, m, logger.Printf) }); err != nil { return err } - if err := hub.Handle(meshwants.SubjectVersionDelivered, func(m jetstream.Msg) { heardVersionDelivered(ctx, ix, m, logger.Printf) }); err != nil { - return err - } if err := hub.Handle(meshwants.SubjectWaitEnded, func(m jetstream.Msg) { heardWaitEnded(ctx, ix, m, logger.Printf) }); err != nil { return err } @@ -742,32 +594,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro }) hearNamed(ctx, js, cred, notHeard, named, logger.Printf) hearRuns(ctx, js, cred, notHeard, ix, logger.Printf) - // The checker's verdicts carry a pull request's out-of-step files (DLV-I20): on the indexer's durable consumer of the - // checker's stream asks. Where it is not there yet, no pull request's out-of-step files are kept until the indexer - // starts again, said. - hear(checkerAsksStream, "no pull request's out-of-step files are kept until the indexer starts again", func(m jetstream.Msg) { heardVerdict(ctx, ix, m, logger.Printf) }) if stopped != nil { return stopped } - // The pending revisions that name this build are taken in now, oldest first, before the consumer on the forge bridge's - // stream starts (ADR 0633 §2). When the build this program runs is not known yet, nothing waits for it: the consumer - // starts, and a merged message left unacknowledged asks the build again at its redelivery (clause 1). A hand take-in - // (indexer_take_in_pending) is answered at any time: the takeins watch runs already. - // A take-in that fails (a bus error, or a merge lost because the forge bridge did not answer) does not end the process (issue - // 1585, note 70): what pending holds is evaluated again at the next merged message, toolchain report or hand take-in. - if _, words, err := ix.TakeInAtStart(ctx); err != nil { - logger.Printf("the take-in at start failed and is asked again at the next merged message, toolchain report or hand take-in: %v", err) - } else if words != "" { - logger.Printf("take-in at start: %s", words) - } - cons, err := cred.EventConsumer(ctx, js, forgeStream) - if err != nil { - // Essential: the forge's merges are what the indexer indexes. - return fmt.Errorf("the consumer %s_%s on %s cannot be read (%v): the indexer hears no merge without it, so it stops; where the bus refused it or did not answer, this module's consumes (made by the controller with its grant) have not reached the bus yet", cred.Node, cred.Module, forgeStream, err) - } - // A merge applied while the taken state was missing gets its taken key (issue 1280): when the state appears (Ready runs - // once, now or then), and after each applied message. The gate's source heads never move a key (issue 1126). - taken.Ready(func(jetstream.KeyValue) { ix.KickFill(ctx) }) // What the indexer heard of FORGE (issue 1126 note 76) is bound when its state appears, as every state a delivery adds // (issue 1790): until then no message is written there, and the rest of the indexer runs. heard := heardStore @@ -779,16 +608,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro index.PackagesState: awaited(packages), index.HeardState: awaited(heard.b), "taken": awaited(taken), "findings": awaited(findings), index.SeatDefinitionsState: awaited(seatDefs)}, errs, logger.Printf) - cc, err := process.HearEvents(cons, func(m jetstream.Msg) { - handle(ctx, m, ix, heard, logger) - ix.KickFill(ctx) - unreadAgain(ctx, ix, logger) - defs.Heard(ctx) - }, logger.Printf) - if err != nil { - return err - } - defer cc.Stop() select { case <-ctx.Done(): return nil @@ -798,7 +617,5 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return err case err := <-tools.errs: return err - case err := <-previews.errs: - return err } } diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main_test.go index 8477025c..8ff5d8db 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main_test.go @@ -1,22 +1,13 @@ package main import ( - "bytes" "context" "encoding/json" - "errors" - "fmt" - "git.novox.be/novox/mesh-sdk/go/busretry" - "log" - "reflect" - "strings" "testing" "time" "github.com/nats-io/nats.go/jetstream" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "git.novox.be/novox/mesh-sdk/go/process" "git.novox.be/novox/mesh-sdk/go/views" @@ -83,124 +74,6 @@ const testIndexer = "mesh-delivery-indexer" var testAccount = process.Account{Node: "n1", Module: testIndexer} -func TestAMessageIsIndexedAndAcked(t *testing.T) { // 1238/AC2 (unit part) - // Review round 2: the indexer's views emitter writes its finding view-unsaid (Emitter.Findings set). - if em := viewsEmitter(nil, process.Account{}, nil, noWire{}, t.Logf); em.Findings == nil { - t.Error("the indexer's views emitter writes no view-unsaid finding") - } - // Milestone 47/G: the object buckets are the module's own by the name its account runs as, never a fixed name. - if b := newBusObjects(nil, process.Account{Module: "renamed-indexer"}, index.CatalogRevisionsObjects).bucket(); b != "renamed-indexer_catalog-revisions" { - t.Errorf("the catalog graph revisions open as %s, want the account's module's", b) - } - t.Run("not a forge message: terminated and logged", func(t *testing.T) { // 1238 (never redelivered) - var logs bytes.Buffer - st := &memStore{vals: map[string][]byte{}} - m := &fakeMsg{subject: "mesh.stream.mesh-delivery-forge-bridge.event.forge.x.y.pull", data: []byte(`{"v":1,"kind":"pull"}`)} - heard := &memStore{vals: map[string][]byte{}} - handle(context.Background(), m, &index.Indexer{Store: st, Builds: index.NewBuilds()}, heard, log.New(&logs, "", 0)) - if m.termd != 1 || m.acked != 0 || len(st.vals) != 0 || !strings.Contains(logs.String(), m.subject) { - t.Errorf("terminated %d acked %d keys %d log %q", m.termd, m.acked, len(st.vals), logs.String()) - } - if h := heardOf(t, heard); h.Outcome != index.HeardDropped || h.Sequence != 41 || h.Words == "" { - t.Errorf("heard = %+v", h) - } - }) - t.Run("a kind it does not take (branch-deleted): acknowledged and passed over", func(t *testing.T) { // design 51, DLV-FB1 - st := &memStore{vals: map[string][]byte{}} - m := &fakeMsg{subject: "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-sdk.branch-deleted", - data: []byte(`{"v":1,"kind":"branch-deleted","repository":"novox/mesh-sdk","branch":"feat/x","head":"` + head + `"}`)} - heard := &memStore{vals: map[string][]byte{}} - handle(context.Background(), m, &index.Indexer{Store: st, Builds: index.NewBuilds()}, heard, log.New(&bytes.Buffer{}, "", 0)) - if m.termd != 0 || m.acked != 1 || len(st.vals) != 0 { - t.Errorf("terminated %d acked %d keys %d", m.termd, m.acked, len(st.vals)) - } - if h := heardOf(t, heard); h.Outcome != index.HeardIgnored || h.Kind != "branch-deleted" { - t.Errorf("heard = %+v", h) - } - }) - st := &memStore{vals: map[string][]byte{}} - ix := &index.Indexer{Store: st, Builds: index.NewBuilds()} - data, _ := forgemsg.EncodeForge(forgemsg.Forge{Kind: forgemsg.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head, - Action: forgemsg.PullUpdated, Dirs: []string{"modules/a"}, DirsKnown: true}) - m := &fakeMsg{subject: "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-delivery.pull", data: data} - heard := &memStore{vals: map[string][]byte{}} - handle(context.Background(), m, ix, heard, log.New(&bytes.Buffer{}, "", 0)) - c, err := index.DecodeCandidate(st.vals["feat/x"]) - if err != nil || len(c.Pulls) != 1 || c.Pulls[0].Modules[0] != "a" || m.acked != 1 || m.doubled != 1 || m.termd != 0 { - t.Errorf("candidate %+v err %v acked %d terminated %d", c, err, m.acked, m.termd) - } - // Issue 1126 note 76: what the indexer heard of FORGE is readable (indexer_heard): subject, sequence, deliveries, time, - // repository, commit and what it did. - if h := heardOf(t, heard); h.Subject != m.subject || h.Sequence != 41 || h.Delivered != 2 || h.Published.Hour() != 13 || h.Kind != forgemsg.ForgePull || - h.Repository != "novox/mesh-delivery" || h.Number != 12 || h.Commit != head || h.Outcome != index.HeardApplied || h.Handled.IsZero() { - t.Errorf("heard = %+v", h) - } - t.Run("a group's end is told apart by its name before the forge decode", func(t *testing.T) { - var logs bytes.Buffer - st := &memStore{vals: map[string][]byte{}} - heard := &memStore{vals: map[string][]byte{}} - // 1585/T9 (ADR 0633 §4): a group's end is told apart by its name before the forge decode: one that reads is acked - // (an indexer with no pending state takes the members in one by one), one that does not is dropped as a group's end. - for _, c := range []struct { - data string - termd int - logged string - }{{`{"v":1,"group":"feat/x","members":[{"repository":"novox/mesh-sdk","number":3,"merge":"` + head + `"}]}`, 0, ""}, - {`{"v":1,"group":"feat/x"}`, 1, "not a group's end"}} { - logs.Reset() - m := &fakeMsg{subject: "mesh.stream.mesh-delivery-forge-bridge.event." + forgemsg.ForgeGroupEndEvent, data: []byte(c.data)} - handle(context.Background(), m, &index.Indexer{Store: st, Builds: index.NewBuilds()}, heard, log.New(&logs, "", 0)) - if m.termd != c.termd || m.acked+m.termd != 1 || !strings.Contains(logs.String(), c.logged) { - t.Errorf("group end %s: terminated %d acked %d log %q", c.data, m.termd, m.acked, logs.String()) - } - if h := heardOf(t, heard); h.Kind != forgemsg.ForgeGroupEndEvent && c.termd == 0 { - t.Errorf("heard of a group's end = %+v", h) - } - } - // The repository forge/group's merged message ends as a group's end does: it is a forge message all the same. - logs.Reset() - m := &fakeMsg{subject: "mesh.stream.mesh-delivery-forge-bridge.event.forge.forge.group.merged", data: []byte(`{"v":1,"group":"feat/x","members":[{"repository":"novox/mesh-sdk","number":3,"merge":"` + head + `"}]}`)} - handle(context.Background(), m, &index.Indexer{Store: st, Builds: index.NewBuilds()}, heard, log.New(&logs, "", 0)) - if !strings.Contains(logs.String(), "not a forge message") { - t.Errorf("forge/group's merged message read as a group's end: %q", logs.String()) - } - }) - // Issue 1966: a bucket the bus has not made is "not yet"; one it cannot reach (jetstream not enabled, no responders) is the - // bus's own error, which stops a pass instead of reading as nothing published. - t.Run("a bucket not found is not yet, one the bus cannot reach is its error", func(t *testing.T) { - for _, c := range []struct { - name string - err error - notYet bool - }{{"not found", jetstream.ErrBucketNotFound, true}, {"not reachable", jetstream.ErrJetStreamNotEnabled, false}} { - b := process.AwaitBucket(context.Background(), opener{c.err}, nil, "packages", "none", t.Logf) - _, err := awaitedStore{b: b, js: opener{c.err}, name: "packages"}.Keys(context.Background()) - if errors.Is(err, process.ErrNotYet) != c.notYet || err == nil || (!c.notYet && !errors.Is(err, c.err)) { - t.Errorf("%s: Keys answered %v, want not-yet %v", c.name, err, c.notYet) - } - } - }) - // Design 47 G A-397: the program hears its kept seat events (package-published among them) on their own streams once its - // durable on EVENTS is bound, never when that fails. - for _, startErr := range []error{nil, errors.New("not composed yet")} { - h := &startedHub{startErr: startErr} - if err := startHearing(context.Background(), h, nil, process.Account{}); !errors.Is(err, startErr) || h.kept != (startErr == nil) { - t.Errorf("start error %v: answered %v, kept streams bound %v", startErr, err, h.kept) - } - } - // Design 47 G A-420: a package-published that cannot be kept is left for the bus, and on its last delivery said given up, - // naming its subject, the artifact and version and the last refusal; before it, nothing more is said. - for _, n := range []uint64{4, EventDeliveries} { - var said []string - m := &deliveredMsg{subject: "mesh.seat.mesh-builds.event.package-published", data: []byte(`{"artifact":"m:program","version":"0.1.6"}`), n: n} - heardPackagePublished(context.Background(), &index.Indexer{}, m, func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }) - givenUp := strings.Contains(strings.Join(said, "\n"), "gives up mesh.seat.mesh-builds.event.package-published: the package-published of m:program 0.1.6, delivered 5 times") - if !m.naked || givenUp != (n == EventDeliveries) || (givenUp && !strings.Contains(strings.Join(said, "\n"), "the last refusal: the state versions is not kept here")) { - t.Errorf("delivery %d: naked %v, said %q", n, m.naked, said) - } - } -} - // deliveredMsg is a message delivered n times, which records a Nak. type deliveredMsg struct { jetstream.Msg @@ -222,90 +95,11 @@ type opener struct{ err error } func (o opener) KeyValue(context.Context, string) (jetstream.KeyValue, error) { return nil, o.err } -func TestTheSourceIsReadFromTheSeatAndASourceMovedDuringTheCatchUpIsReadAfterIt(t *testing.T) { // design 47 G §15, §115, §362 - var order []string - revision := int64(4) - var logs []string - var s *sourceFollow - s = &sourceFollow{ - read: func(branch string) (meshdelivery.SourceAnswer, error) { - order = append(order, "read "+branch) - if branch == "" { // given no branch the seat lists the branches it keeps a source of (A-93) - return meshdelivery.SourceAnswer{Branches: []string{"main", "beta", "feat=2e1"}}, nil - } - if revision == 0 { - return meshdelivery.SourceAnswer{}, fmt.Errorf("source.main is not readable") - } - if len(order) == 1 { // a source-moved is heard while the catch-up reads - s.moved(context.Background(), "main") - s.moved(context.Background(), "main") - s.moved(context.Background(), "beta") - } - return meshdelivery.SourceAnswer{Branch: branch, Revision: revision}, nil - }, - apply: func(_ context.Context, a meshdelivery.SourceAnswer) error { - order = append(order, fmt.Sprintf("apply %s %d", a.Branch, a.Revision)) - return nil - }, - applyBranch: func(_ context.Context, branch string, a meshdelivery.SourceAnswer) error { - order = append(order, fmt.Sprintf("branch %s %d", branch, a.Revision)) - return nil - }, - logf: func(f string, a ...any) { logs = append(logs, fmt.Sprintf(f, a...)) }, - } - s.catchUp(context.Background()) - // The trunk, then each other watched branch the seat lists (a name unescaped), then what was heard meanwhile once each, - // beta among them: main through the take-in, every other branch into its own revision (A-93). - want := []string{"read main", "apply main 4", "read ", "read beta", "branch beta 4", "read feat=2e1", "branch feat.1 4", "read beta", "branch beta 4", "read main", "apply main 4"} - if !reflect.DeepEqual(order, want) { - t.Errorf("order = %v, want %v", order, want) - } - s.moved(context.Background(), "main") // after the catch-up: read at once - if len(order) != 13 { - t.Errorf("a source-moved after the catch-up was not read at once: %v", order) - } - order, revision = nil, 0 - s.moved(context.Background(), "main") - if len(order) != 1 || len(logs) != 1 || !strings.Contains(logs[0], "not readable") { - t.Errorf("a source that cannot be read: order %v, said %v, want it said and nothing applied", order, logs) - } -} - -func TestTheSeatsSourceAnswerIsReadAndItsErrorIsAnError(t *testing.T) { // design 47 G §115 - var asked any - read := sourceVia(context.Background(), func(_ busretry.Open, key string, body any) (json.RawMessage, error) { - asked = body - if key != "seat:mesh-delivery.source" { - t.Errorf("asked %q", key) - } - if body.(meshdelivery.SourceInput).Branch == "bad" { - return json.RawMessage(`{"error":"source.bad is not readable"}`), nil - } - return json.RawMessage(`{"branch":"main","revision":7,"repositories":[{"repository":"novox/mesh-delivery","head":"` + head + `","trunk":"main"}]}`), nil - }, func(context.Context, string) (bool, error) { return true, nil }, t.Logf) - a, err := read("main") - if err != nil || a.Revision != 7 || len(a.Repositories) != 1 || asked.(meshdelivery.SourceInput).Branch != "main" { - t.Errorf("source = %+v, %v", a, err) - } - if _, err := read("bad"); err == nil || !strings.Contains(err.Error(), "not readable") { - t.Errorf("an answer with an error was no error: %v", err) - } -} - func ptr[T any](v T) *T { return &v } // noWire is a finding writer that is none: a non-nil WireWriter for the test of which emitter is made. type noWire struct{ views.WireWriter } -// Design 47 G §509, §714, A-269: the indexer takes a pushed message alike with a merged one. -func TestTheIndexerTakesAPushedMessage(t *testing.T) { - for _, kind := range []string{forgemsg.ForgeMerged, forgemsg.ForgePushed} { - if got, known := forgeKind([]byte(`{"kind":"` + kind + `"}`)); !known || got != kind { - t.Errorf("forgeKind(%s) = %q, %v, want it taken", kind, got, known) - } - } -} - // startedHub records which of an event hub's starts were asked. type startedHub struct { startErr error diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named.go index 46f39a95..1dd3ce4a 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named.go @@ -12,7 +12,6 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk" "mesh-delivery-indexer/internal/fingerprint" @@ -240,27 +239,6 @@ func readVia(call toolCaller) func(ctx context.Context, repository, sha, dir str // trunkBranch is the branch pull requests are open into. const trunkBranch = index.TrunkBranch -// openVia asks the forge which pull requests into the trunk are open now (the seat git's open-pulls), every repository's. -// An answer that lacks the list is refused, never read as none. -func openVia(call toolCaller) func(ctx context.Context) ([]forgemsg.OpenPull, error) { - return func(context.Context) ([]forgemsg.OpenPull, error) { - var o git.OpenPullsAnswer - if err := askGit(call, git.VerbOpenPulls, git.OpenPullsInput{}, &o); err != nil { - return nil, err - } - if o.Pulls == nil { - return nil, fmt.Errorf("the answer of %s lists no pulls", gitVerb(git.VerbOpenPulls)) - } - out := []forgemsg.OpenPull{} - for _, p := range o.Pulls { - if p.Pull.Base == trunkBranch && p.Pull.Number > 0 { - out = append(out, forgemsg.OpenPull{Repository: p.Pull.Repository, Number: int(p.Pull.Number)}) - } - } - return out, nil - } -} - // branchHeadVia is the head of a repository's branch through the seat git's verb branch. func branchHeadVia(call toolCaller) func(ctx context.Context, repository, branch string) (string, error) { return func(_ context.Context, repository, branch string) (string, error) { diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go index 1c2091c5..830d145a 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go @@ -47,27 +47,6 @@ func TestNamedViaAsksTheTreesToolWithThePathsAndTakesTheirObjects(t *testing.T) if _, err := namedVia(failed)(context.Background(), "novox/mesh-sdk", "s", nil); err == nil || !strings.Contains(err.Error(), "the forge did not answer") { t.Errorf("an error answer: %v, want its failure", err) } - for _, row := range []struct { - name, answer string - want int - refused bool - }{ - {"one open into the trunk, one into another branch", `{"pulls":[{"pull":{"repository":"novox/a","number":3,"base":"main","branch":"b","head":"h"}},{"pull":{"repository":"novox/a","number":4,"base":"dev","branch":"c","head":"h"}}]}`, 1, false}, - {"none open", `{"pulls":[]}`, 0, false}, - {"no list", `{}`, 0, true}, - {"an error", `{"error":"refused","error-kind":"refused"}`, 0, true}, - } { - open := func(key string, _ any) (json.RawMessage, error) { - if key != "seat:git.open-pulls" { - t.Errorf("called %s", key) - } - return json.RawMessage(row.answer), nil - } - got, err := openVia(open)(context.Background()) - if (err != nil) != row.refused || len(got) != row.want { - t.Errorf("open-pulls %s: %v %v, want %d, refused %v", row.name, got, err, row.want, row.refused) - } - } call = func(string, any) (json.RawMessage, error) { return json.RawMessage(`{"root":"r0","trees":{},"paths":{"":{"src":"1111111111111111111111111111111111111111"}}}`), nil } diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/previews.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/previews.go deleted file mode 100644 index 766e7270..00000000 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/previews.go +++ /dev/null @@ -1,66 +0,0 @@ -package main - -import ( - "context" - "errors" - "log" - "sync" - - "git.novox.be/novox/mesh-sdk/go/process" - - "github.com/nats-io/nats.go/jetstream" - - "mesh-delivery-indexer/internal/index" -) - -// previewWatch reads the merge previews the checker writes (mesh-delivery-checker_previews) and follows them: every change -// writes the candidates' fingerprints again (index.Indexer.ApplyPreview). The checker's bucket is made when it is assigned, -// after the indexer started, and the indexer holds no timer (ADR 0437 §8) and hears no preview.written (design 47 G A-160): a -// bucket absent at the start is not watched until the indexer starts again. -type previewWatch struct { - ctx context.Context - js bucketOpener - ix *index.Indexer - logger *log.Logger - errs chan error - mu sync.Mutex - watched bool -} - -func newPreviewWatch(ctx context.Context, js bucketOpener, ix *index.Indexer, logger *log.Logger) *previewWatch { - return &previewWatch{ctx: ctx, js: js, ix: ix, logger: logger, errs: make(chan error, 1)} -} - -// ensure watches the previews bucket if it is not watched and exists, settling what it holds, then writes the candidates' -// fingerprints from it. -func (w *previewWatch) ensure() error { - w.mu.Lock() - defer w.mu.Unlock() - if w.watched { - return nil - } - kv, err := w.js.KeyValue(w.ctx, index.PreviewsBucket) - switch { - case errors.Is(err, jetstream.ErrBucketNotFound): - w.logger.Printf("the bucket %s does not exist yet: it is watched from the indexer's next start", index.PreviewsBucket) - return nil - case err != nil: - // Refused or not answered (its grant not on the bus yet, issue 1945): said, and opened again at the checker's next preview. - w.logger.Printf("the bucket %s cannot be read (%v): it is watched from the indexer's next start", index.PreviewsBucket, err) - return nil - } - watcher, err := settle(w.ctx, kv, func(e jetstream.KeyValueEntry) { w.ix.PutPreview(e.Key(), e.Value(), !opOf(e)) }) - if err != nil { - process.NotWatched(w.logger.Printf, index.PreviewsBucket, err, "it is watched from the indexer's next start") - return nil - } - w.watched = true - go func() { - w.errs <- follow(w.ctx, watcher, func(e jetstream.KeyValueEntry) { - if err := w.ix.ApplyPreview(w.ctx, e.Key(), e.Value(), !opOf(e)); err != nil { - w.logger.Printf("%s.%s could not be taken into the candidates: %v", index.PreviewsBucket, e.Key(), err) - } - }) - }() - return w.ix.RecomputeCandidates(w.ctx) -} diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/seatdefs.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/seatdefs.go index 2fa8f9e6..19dac869 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/seatdefs.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/seatdefs.go @@ -8,18 +8,13 @@ import ( "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdeliverychecker" "git.novox.be/novox/mesh-sdk/go/process" "mesh-delivery-indexer/internal/index" ) // The seat definitions' wiring (DLV-I17-I20; design 26): the forge's get-file tool reads them, seat-definition-changed -// is said on the seat mesh-catalog's own subject (kept on the shared events stream, seven days), and the checker's verdict event, heard on -// the indexer's durable consumer of the checker's stream asks, carries a pull request's out-of-step files. - -// checkerAsksStream is the checker's stream asks as the bus names it (its manifest's streams; StreamBusName). -const checkerAsksStream = "ASKS" +// is said on the seat mesh-catalog's own subject (kept on the shared events stream, seven days),. // readFileVia reads one file of a repository at a commit through the seat git's file; a path the commit does not // hold is nil with no error. @@ -60,23 +55,3 @@ func saySeatDefinition(js jetstream.JetStream, cred process.Account) func(contex return err } } - -// heardVerdict takes the checker's verdict event: a pull request's out-of-step files are kept (a clean verdict removes -// them). Another event of the stream is acknowledged and passed over; a write that failed is delivered again. -func heardVerdict(ctx context.Context, ix *index.Indexer, m jetstream.Msg, logf func(string, ...any)) { - if m.Subject() != meshdeliverychecker.HolderSubjectVerdict(index.CheckerModule) { - _ = m.Ack() - return - } - var v meshdeliverychecker.VerdictHolderEvent - if err := json.Unmarshal(m.Data(), &v); err != nil { - logf("%s holds what is no verdict, left: %v", m.Subject(), err) - _ = m.Term() - return - } - if err := ix.ApplyVerdict(ctx, v.Repository, int(v.Pull), v.Head, v.OutOfStep); err != nil { - logf("%s#%d: its out-of-step files are not kept, delivered again: %v", v.Repository, v.Pull, err) - return - } - _ = m.Ack() -} diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/source.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/source.go deleted file mode 100644 index c9bc6cc2..00000000 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/source.go +++ /dev/null @@ -1,135 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "maps" - "slices" - "strings" - "sync" - - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" - "git.novox.be/novox/mesh-sdk/go/keys" - "github.com/nats-io/nats.go/jetstream" - "mesh-delivery-indexer/internal/index" -) - -// sourceInvoke is the seat mesh-delivery's verb source as the manifest's invokes names it: a watched branch's source -// revision, asked of the seat and never read from the gate's state (design 47 G §15, §115; ADR 0596 §2). -const sourceInvoke = "seat:" + meshdelivery.Seat + "." + meshdelivery.VerbSource - -// sourceVia asks the seat for one branch's source revision through call (process.Conn.CallKeptWhileOpen, a read asked again while the branch is watched, A-190) and reads its -// declared answer; an answer that carries an error is that error. -func sourceVia(ctx context.Context, call openCaller, watched func(ctx context.Context, branch string) (bool, error), logf func(string, ...any)) func(branch string) (meshdelivery.SourceAnswer, error) { - return func(branch string) (meshdelivery.SourceAnswer, error) { - raw, err := keptWhileOpen(ctx, call, logf, "source."+branch, sourceInvoke, meshdelivery.SourceInput{Branch: branch}, - func(ctx context.Context) (bool, error) { return watched(ctx, branch) }) - if err != nil { - return meshdelivery.SourceAnswer{}, err - } - var a meshdelivery.SourceAnswer - if err := json.Unmarshal(raw, &a); err != nil { - return a, fmt.Errorf("the answer of %s is not a source revision: %w", sourceInvoke, err) - } - if a.Error != "" { - return a, fmt.Errorf("%s: %s", sourceInvoke, a.Error) - } - return a, nil - } -} - -// sourceFollow takes the seat's source revisions into the indexer: first the catch-up of the heads the source names, then -// each source-moved the seat says (design 47 G §362). A source-moved heard before the catch-up ended is applied after it, -// once per branch. -type sourceFollow struct { - read func(branch string) (meshdelivery.SourceAnswer, error) - apply func(context.Context, meshdelivery.SourceAnswer) error - // applyBranch takes a watched branch other than the trunk's in (design 47 G §291, A-93); branch is its name, unescaped. - applyBranch func(ctx context.Context, branch string, a meshdelivery.SourceAnswer) error - logf func(string, ...any) - - mu sync.Mutex - caught bool - waiting map[string]bool -} - -// catchUp reads the trunk branch's source revision and applies it, then every other branch the seat keeps a source of (the -// answer given no branch lists them, A-93), then the branches a source-moved named meanwhile. A read that fails is said; -// the catch-up ends all the same, and the next source-moved reads again. -func (s *sourceFollow) catchUp(ctx context.Context) { - s.take(ctx, trunkBranch) - var others []string - if list, err := s.read(""); err != nil { - s.logf("the branches the source names are not read, and each is read at its next source-moved: %v", err) - } else { - others = list.Branches - } - for _, branch := range others { - if branch != trunkBranch { - s.take(ctx, branch) - } - } - s.mu.Lock() - s.caught = true - branches := s.waiting - s.waiting = nil - s.mu.Unlock() - for _, branch := range slices.Sorted(maps.Keys(branches)) { // a map's order is random: the branches are read in name order - s.take(ctx, branch) - } -} - -// moved is the handler of one source-moved on branch: held until the catch-up ends, read at once after. -func (s *sourceFollow) moved(ctx context.Context, branch string) { - s.mu.Lock() - if !s.caught { - if s.waiting == nil { - s.waiting = map[string]bool{} - } - s.waiting[branch] = true - s.mu.Unlock() - return - } - s.mu.Unlock() - s.take(ctx, branch) -} - -// take reads one branch's source revision (branch as the key name source. has it) and applies it: the trunk's -// through the take-in, another watched branch's into its own revision (A-93). -func (s *sourceFollow) take(ctx context.Context, branch string) { - a, err := s.read(branch) - if errors.Is(err, index.ErrAskClosed) { // the branch is no longer watched: not said unanswered (A-190) - return - } - if err != nil { - s.logf("source.%s is not read, and is read again at its next source-moved: %v", branch, err) - return - } - if branch != trunkBranch { - name, err := keys.UnescapeName(branch) - if err != nil { - s.logf("source.%s names no branch: %v", branch, err) - return - } - if s.applyBranch == nil { - return - } - if err := s.applyBranch(ctx, name, a); err != nil { - s.logf("source.%s at revision %d could not be taken into the branch's revision: %v", branch, a.Revision, err) - } - return - } - if err := s.apply(ctx, a); err != nil { - s.logf("source.%s at revision %d could not be taken into the modules: %v", branch, a.Revision, err) - } -} - -// heard is the hub's handler of source-moved: the branch is the last token of the subject. The message is acknowledged at once. -func (s *sourceFollow) heard(ctx context.Context, m jetstream.Msg) { - defer func() { _ = m.Ack() }() - if branch := m.Subject()[strings.LastIndex(m.Subject(), ".")+1:]; branch != "" { - s.moved(ctx, branch) - } -} diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go index 35c5a45d..6269db40 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go @@ -15,11 +15,7 @@ import ( type takerIn interface { TakeIn(ctx context.Context, module string) index.TakenIn TakeInRepository(ctx context.Context, repository string) index.TakenIn - CatchUp(ctx context.Context) index.TakenIn SeatDefinitionsAgain(ctx context.Context, module string, pull int, head string) index.TakenIn - TakeInPendingByHand(ctx context.Context, key, by, why string) index.TakenIn - DropPending(ctx context.Context, key, by, why string, abandon bool) index.TakenIn - TakeInMerge(ctx context.Context, repository, merge string, seq uint64, by, why string) index.TakenIn FingerprintsAgain(ctx context.Context, module string) index.TakenIn RecomputeByHand(ctx context.Context, by, why string) index.TakenIn DropCandidateGraph(ctx context.Context, group, heads, by, why string) index.TakenIn @@ -69,18 +65,6 @@ func takenIn(ctx context.Context, ix takerIn, kv jetstream.KeyValue, e jetstream a.Module = r.Module } else if a.Fingerprints { asked, r = "computing of fingerprints again", ix.FingerprintsAgain(ctx, a.Module) - } else if a.Merge != "" { - asked, r = "hand take-in of a merge", ix.TakeInMerge(ctx, a.MergeRepository, a.Merge, a.Sequence, a.By, a.Why) - a.Module = r.Module - } else if a.Pending != "" && a.Drop { - asked, r = "drop of a pending merge", ix.DropPending(ctx, a.Pending, a.By, a.Why, a.Abandon) - a.Module = a.Pending - } else if a.Pending != "" { - asked, r = "hand take-in of what waits", ix.TakeInPendingByHand(ctx, a.Pending, a.By, a.Why) - a.Module = a.Pending - } else if a.CatchUp { - asked, r = "catch-up of every repository", ix.CatchUp(ctx) - a.Module = r.Module } else if a.Repository != "" { asked, r = "take-in of a repository", ix.TakeInRepository(ctx, a.Repository) a.Module = a.Repository diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/versions.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/versions.go index b9785e96..d726989f 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/versions.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/versions.go @@ -11,8 +11,6 @@ import ( "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" - "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner" - "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/gopackageregistry" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds" @@ -117,23 +115,6 @@ func heardToolchainAgreed(ctx context.Context, ix *index.Indexer, m jetstream.Ms _ = m.Ack() } -// heardVersionDelivered writes the delivered release's entries of kinds 2, 4 and 6 and keeps the newest delivered version of -// its module (design 47 G §374, A-68), as heardToolchainAgreed keeps its event. -func heardVersionDelivered(ctx context.Context, ix *index.Indexer, m jetstream.Msg, logf func(string, ...any)) { - var d meshwants.VersionDeliveredEvent - if err := json.Unmarshal(m.Data(), &d); err != nil { - logf("%s holds what is no version-delivered, left: %v", m.Subject(), err) - _ = m.Ack() - return - } - if err := ix.HearVersionDelivered(ctx, d); err != nil { - logf("versions: the version-delivered of %s %s could not be kept, so the bus gives it again: %v", d.Module, d.Version, err) - leftForBus(m, fmt.Sprintf("the version-delivered of %s %s", d.Module, d.Version), err, logf) - return - } - _ = m.Ack() -} - // heardWaitEnded keeps the act of an acting part's ended wait for kind 5 (A-78 a), as heardToolchainAgreed keeps its event. func heardWaitEnded(ctx context.Context, ix *index.Indexer, m jetstream.Msg, logf func(string, ...any)) { var w meshwants.WaitEndedEvent @@ -240,61 +221,6 @@ func afterRevision(say func(context.Context, meshcatalog.CatalogRevisionEvent) e } } -// placementVia asks the seat mesh-module-assigner's verb placement for one module (A-62), a read. -func placementVia(call toolCaller) index.PlacementAsker { - return func(_ context.Context, module string) ([]meshmoduleassigner.ModulePlacement, error) { - raw, err := call(meshmoduleassigner.Seat+"."+meshmoduleassigner.VerbPlacement, map[string]any{"module": module}) - if err != nil { - return nil, fmt.Errorf("%s.%s: %w", meshmoduleassigner.Seat, meshmoduleassigner.VerbPlacement, err) - } - var a meshmoduleassigner.PlacementAnswer - if err := json.Unmarshal(raw, &a); err != nil { - return nil, fmt.Errorf("the answer of placement is not a module's placements: %w", err) - } - return a.Placements, nil - } -} - -// runningWholeVia reads every subject's last message of the node-engine seat's stream running through an ordered consumer made for -// that one read (design 47 G §556), by node and module. -func runningWholeVia(js jetstream.JetStream) index.RunningReader { - return func(ctx context.Context) (map[string]map[string]nodeengine.Running, error) { - stream, err := js.Stream(ctx, nodeengine.StreamRunning) - if err != nil { - return nil, fmt.Errorf("the stream %s cannot be read whole: %w", nodeengine.StreamRunning, err) - } - cons, err := stream.OrderedConsumer(ctx, jetstream.OrderedConsumerConfig{DeliverPolicy: jetstream.DeliverLastPerSubjectPolicy}) - if err != nil { - return nil, fmt.Errorf("the ordered consumer for the whole read of %s cannot be made: %w", nodeengine.StreamRunning, err) - } - out := map[string]map[string]nodeengine.Running{} - for { - batch, err := cons.FetchNoWait(256) - if err != nil { - return nil, fmt.Errorf("the whole read of %s failed: %w", nodeengine.StreamRunning, err) - } - n := 0 - for m := range batch.Messages() { - n++ - var r nodeengine.Running - if json.Unmarshal(m.Data(), &r) != nil || r.Node == "" || r.Module == "" { - continue - } - if out[r.Node] == nil { - out[r.Node] = map[string]nodeengine.Running{} - } - out[r.Node][r.Module] = r - } - if err := batch.Error(); err != nil { - return nil, fmt.Errorf("the whole read of %s failed: %w", nodeengine.StreamRunning, err) - } - if n == 0 { - return out, nil - } - } - } -} - // tagsVia reads every tag of a repository that starts with a prefix from the seat git's verb tags as a kept call, page by // page until none is more (design 47 G §412, A-490c: protected or not). func tagsVia(call toolCaller) index.TagsReader { diff --git a/modules/mesh-delivery-indexer/internal/index/behind.go b/modules/mesh-delivery-indexer/internal/index/behind.go deleted file mode 100644 index d7906ae9..00000000 --- a/modules/mesh-delivery-indexer/internal/index/behind.go +++ /dev/null @@ -1,190 +0,0 @@ -package index - -import ( - "context" - "errors" - "fmt" - "maps" - "slices" - - mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" - "git.novox.be/novox/mesh-sdk/go/process" - - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk" -) - -// A merge the indexer never took in (design 47 G A-463, amended by A-464): at its start and on every source revision the -// gate's source names, the indexer compares each repository's `taken` merge with the head that source names. A repository -// whose head is newer than its taken merge (the taken merge is that head's ancestor, or no merge is taken) is taken in as -// its merged message would have been (the catch-up of indexer_take_in, for that repository), said in the log, with the -// finding merge-untaken raised while it stays behind and cleared once it is taken in. A head the taken merge contains (the -// source not yet moved to a merge just taken in) is left, and a head the taken merge neither contains nor is in is said. - -// UntakenMergeKind is the finding of a repository whose head the gate's source names but whose merge the indexer never took in. -const UntakenMergeKind = "merge-untaken" - -func untakenFindingID(own, key string) string { return own + ".untaken." + key } - -// kickBehind runs the comparison in the background, one at a time: a source revision that arrives while one runs makes it -// run once more, on the latest heads. It runs after the first start's catch-up, which takes every head in itself. -func (ix *Indexer) kickBehind(ctx context.Context) { - if ix.Taken == nil || ix.Message == nil || ix.Modules == nil { - return - } - if ix.catchingUp.Load() { - ix.behindWaiting.Store(true) // compared again when that catch-up ends (A-463a) - return - } - if ix.behindRunning.Swap(true) { - ix.behindAgain.Store(true) - return - } - ix.catchUpWG.Add(1) - go func() { - defer ix.catchUpWG.Done() - for { - ix.behindAgain.Store(false) - ix.TakeInBehind(ctx) - if !ix.behindAgain.Load() { - ix.behindRunning.Store(false) - if !ix.behindAgain.Load() || ix.behindRunning.Swap(true) { - return - } - } - } - }() -} - -// TakeInBehind compares each repository's taken merge with the head the latest source names, takes in each repository -// that is behind and answers the repositories it took in. -func (ix *Indexer) TakeInBehind(ctx context.Context) []string { - heads := ix.sourceHeads() - if ix.Taken == nil || ix.Message == nil || ix.Modules == nil || len(heads) == 0 { - return nil - } - behind := map[string]string{} - taken := map[string]string{} - held := ix.pendingHeads(ctx) - for _, repository := range slices.Sorted(maps.Keys(heads)) { - head := heads[repository] - key, err := trunk.RepositoryKey(repository) - if head == "" || err != nil { - continue - } - value, _, found, err := ix.Taken.Get(ctx, key) - if errors.Is(err, process.ErrNotYet) { - return nil // the state is not on the bus yet: the next source revision compares again - } - if err != nil { - ix.logf("merges: taken.%s cannot be read, so %s is not compared with its head: %v", key, repository, err) - continue - } - merge := "" - if found { - t, err := takein.DecodeTaken(value) - if err != nil { - ix.logf("merges: taken.%s holds what is not a taken merge, left: %v", key, err) - continue - } - merge = t.Merge - } - switch { - case held[head]: - // A head the pending holds (a member waiting for its group's end) is taken in by that path, whole with - // its group: it is not behind (A-463a). - ix.clearUntaken(ctx, key, "the head of "+repository+" waits in the indexer's pending") - case merge == head: - ix.clearUntaken(ctx, key, "the indexer took in the head of "+repository) - case merge == "": - behind[repository], taken[repository] = head, "" - case ix.Contains == nil: - ix.logf("merges: %s is taken in at %.12s and its head is %.12s, but the indexer reads no ancestry of the forge, so which is newer is not said", repository, merge, head) - default: - inHead, err := ix.Contains(ctx, repository, merge, head) - if err != nil { - ix.logf("merges: whether %.12s is in %.12s of %s cannot be read: %v", merge, head, repository, err) - continue - } - if inHead { - behind[repository], taken[repository] = head, merge - continue - } - ix.clearUntaken(ctx, key, "the head of "+repository+" is in the merge taken in") - if headIn, err := ix.Contains(ctx, repository, head, merge); err == nil && !headIn { - ix.logf("merges: %s is taken in at %.12s and its head is %.12s, and neither holds the other: left", repository, merge, head) - } - } - } - for _, repository := range slices.Sorted(maps.Keys(behind)) { - ix.sayUntaken(ctx, repository, taken[repository], behind[repository]) - ix.logf("merges: %s is taken in at %.12s but the gate's source names %.12s: taken in now", repository, taken[repository], behind[repository]) - } - if len(behind) == 0 { - return nil - } - unread := ix.takeInHeads(ctx, behind) - var done []string - for _, repository := range slices.Sorted(maps.Keys(behind)) { - if slices.Contains(unread, repository) { - continue - } - done = append(done, repository) - key, _ := trunk.RepositoryKey(repository) - ix.clearUntaken(ctx, key, fmt.Sprintf("%s taken in at %.12s", repository, behind[repository])) - ix.logf("merges: %s taken in at %.12s", repository, behind[repository]) - } - if len(done) > 0 { - if err := ix.KeepView(ctx); err != nil { - ix.logf("merges: the view could not be kept after %v were taken in: %v", done, err) - } - } - return done -} - -func (ix *Indexer) sayUntaken(ctx context.Context, repository, merge, head string) { - if ix.Findings == nil { - return - } - key, _ := trunk.RepositoryKey(repository) - taken := "no merge" - if merge != "" { - taken = fmt.Sprintf("%.12s", merge) - } - f := ix.observed(mcc.ScopeModule, untakenFindingID(ix.Module, key), UntakenMergeKind, - fmt.Sprintf("the indexer took in %s at %s, and the gate's source names the head %.12s", repository, taken, head), - "A merge the indexer never took in", - "A merge into a watched branch was never heard or applied, so no package or module of the repository follows its trunk. The indexer takes the head in itself and clears this; indexer_take_in with catch-up does it by hand.", - UntakenMergeKind) - if err := ix.Findings.Write(ctx, f); err != nil { - ix.logf("merges: the finding %s for %s is not written: %v", UntakenMergeKind, repository, err) - } -} - -func (ix *Indexer) clearUntaken(ctx context.Context, key, why string) { - if ix.Findings == nil { - return - } - if err := ix.clearFinding(ctx, mcc.ScopeModule, untakenFindingID(ix.Module, key), UntakenMergeKind, why); err != nil { - ix.logf("merges: the finding %s for %s is not cleared: %v", UntakenMergeKind, key, err) - } -} - -// pendingHeads are the merge commits the indexer's pending holds (A-463a). -func (ix *Indexer) pendingHeads(ctx context.Context) map[string]bool { - out := map[string]bool{} - if ix.Pending == nil || !ix.pendingOnBus(ctx) { - return out - } - entries, err := ReadPending(ctx, ix.Pending, ix.logf) - if err != nil { - ix.logf("merges: the pending state cannot be read, so every head is compared: %v", err) - return out - } - for _, e := range entries { - if e.Forge.Merge != "" { - out[e.Forge.Merge] = true - } - } - return out -} diff --git a/modules/mesh-delivery-indexer/internal/index/branch_take_in.go b/modules/mesh-delivery-indexer/internal/index/branch_take_in.go deleted file mode 100644 index 6ff91d8d..00000000 --- a/modules/mesh-delivery-indexer/internal/index/branch_take_in.go +++ /dev/null @@ -1,63 +0,0 @@ -package index - -import ( - "context" - "errors" - "maps" - "strings" - - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/graph" - - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" - "git.novox.be/novox/mesh-sdk/go/process" -) - -// ApplyBranchSource takes a watched branch other than the first in from the source revision the seat answered for it -// (design 47 G §13, §14, §291, ADR 0666 §6, A-85, A-93): a pure computation from the merged messages of the heads the source -// names, over a view in memory seeded from the branch's newest revision, written straight into the branch's next revision -// (KeepBranchEntries): each module's fingerprint, inputs and graph node at the branch's heads, computed as the trunk's are -// (applyMoving). Nothing of the trunk is written: its modules, manifests, taken state, graph and packages stay its own, -// and the branch has no bucket of its own. A head whose merged message is not read keeps no revision, said; the next -// source-moved of the branch takes it in again. -func (ix *Indexer) ApplyBranchSource(ctx context.Context, branch string, a meshdelivery.SourceAnswer) error { - if isTrunkBranch(branch) || ix.Revisions == nil || ix.Message == nil || len(a.Repositories) == 0 { - return nil - } - heads := map[string]string{} - for _, r := range a.Repositories { - heads[r.Repository] = r.Head - } - view, nodes, _, err := BranchView(ctx, ix.Revisions, branch) - if errors.Is(err, process.ErrNotYet) { - ix.logf("catalog graph: the branch %s is not taken in yet: its objects %s are not on the bus", branch, CatalogRevisionsObjects) - return nil - } - if err != nil { - return err - } - d := ix.dry() - // The branch's own view: its modules and graph from its revision, no taken merge, and the packages it publishes in memory - // over the trunk's, never written to them. - d.ix.Modules, d.ix.Graph, d.ix.Taken = view, nodes, newOverlay(nil) - if ix.Packages != nil { - d.ix.Packages = newOverlay(ix.Packages) - } - d.ix.source = sourceNow{rev: uint64(a.Revision), heads: maps.Clone(heads)} - d.ix.Logf = func(format string, args ...any) { - ix.logf("branch "+branch+", computed and not written: "+format, args...) - } - if unread := d.ix.takeInHeads(ctx, heads); len(unread) > 0 { - ix.logf("catalog graph: the branch %s at source revision %d keeps no revision: %d repositories were not taken in (%v); taken in again at its next source-moved", branch, a.Revision, len(unread), unread) - return nil - } - entries, err := d.ix.viewEntries(ctx) - if err != nil { - return err - } - // A cycle with a build edge no running version breaks is refused here as at the trunk's take-in (design 47 G §49, A-326). - if cycle := graph.RefusedCycle(WithEdges(ix.withActs(ctx, entries))); cycle != nil { - ix.logf("catalog graph: the branch %s at source revision %d keeps no revision: its catalog graph holds a cycle with a build edge, which no running version breaks (design 47 G §49): %s; taken in again at its next source-moved", branch, a.Revision, strings.Join(cycle, ", ")) - return nil - } - return ix.KeepBranchEntries(ctx, branch, a.Revision, entries) -} diff --git a/modules/mesh-delivery-indexer/internal/index/branch_take_in_test.go b/modules/mesh-delivery-indexer/internal/index/branch_take_in_test.go deleted file mode 100644 index 38170e4a..00000000 --- a/modules/mesh-delivery-indexer/internal/index/branch_take_in_test.go +++ /dev/null @@ -1,228 +0,0 @@ -package index - -import ( - "context" - "encoding/json" - "fmt" - "maps" - "slices" - "strings" - "testing" - - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" - "git.novox.be/novox/mesh-sdk/go/catalogue" -) - -// Design 47 G §13, §14, §291, A-93: a watched branch other than the first is taken in as a pure computation from the merged -// messages of its heads, straight into branch..; the trunk's modules, graph and revision stay the trunk's. -func TestABranchOtherThanTheFirstIsTakenInIntoItsOwnRevisionAndTheTrunkIsLeftAlone(t *testing.T) { - ctx := context.Background() - ix, mods, merged := newFormIndexerUnmerged(t, builderModules(t)...) - ix.Revisions = newMemObjects(0) - ix.Graph = newMem() - ix.Message = func(_ context.Context, repository, head string, _ int) ([]byte, error) { - if repository != catalogueRepo || head != merged.Merge { - return nil, fmt.Errorf("no merge of %s at %s", repository, head) - } - return json.Marshal(merged) - } - var said []meshcatalog.CatalogRevisionEvent - ix.SayCatalogRevision = func(_ context.Context, e meshcatalog.CatalogRevisionEvent) error { - said = append(said, e) - return nil - } - before := map[string]string{} - keys, _ := mods.Keys(ctx) - for _, k := range keys { - v, _, _, _ := mods.Get(ctx, k) - before[k] = string(v) - } - graphBefore, _ := ix.Graph.Keys(ctx) - answer := meshdelivery.SourceAnswer{Branch: "beta", Revision: 9, Repositories: []meshdelivery.SourceHead{{Repository: catalogueRepo, Head: merged.Merge, Trunk: "beta"}}} - if err := ix.ApplyBranchSource(ctx, "beta", answer); err != nil { - t.Fatal(err) - } - if len(said) != 1 || said[0].Revision != 1 || said[0].Branch != "beta" || said[0].Source != 9 || said[0].Entries != int64(len(builderModules(t))) { - t.Fatalf("the branch take-in said %+v, want revision 1 of beta at source 9 with every module", said) - } - rev, entries, err := ReadRevision(ctx, ix.Revisions, BranchObject("beta", 1)) - if err != nil || rev.Branch != "beta" || len(entries) != len(builderModules(t)) { - t.Fatalf("branch.beta.1 = %+v with %d entries (%v)", rev, len(entries), err) - } - for _, e := range entries { - if e.Commit != merged.Merge || e.Inputs == nil { // computed at the branch's head, with the inputs its fingerprint stands on - t.Errorf("the entry of %s is at %q with inputs %v, want the branch's head %s with inputs", e.Module, e.Commit, e.Inputs, merged.Merge) - } - } - if _, any, _ := newestTrunk(ctx, ix.Revisions); any { - t.Error("a branch take-in kept a trunk revision") - } - after := map[string]string{} - keys, _ = mods.Keys(ctx) - for _, k := range keys { - v, _, _, _ := mods.Get(ctx, k) - after[k] = string(v) - } - if !maps.Equal(before, after) { - t.Errorf("a branch take-in wrote the trunk's modules") - } - if graphAfter, _ := ix.Graph.Keys(ctx); !slices.Equal(graphBefore, graphAfter) { - t.Errorf("a branch take-in wrote the trunk's graph: %v, was %v", graphAfter, graphBefore) - } - // Taken in again at the same heads, nothing moved: no second revision. - if err := ix.ApplyBranchSource(ctx, "beta", answer); err != nil || len(said) != 1 { - t.Errorf("the same heads said %d revisions (%v), want one", len(said), err) - } -} - -// Design 47 G §49, A-326: a branch take-in refuses a cycle with a build edge as the trunk's does: the branch keeps no revision -// at those heads and says why, naming every artifact on the cycle. -func TestABranchTakeInKeepsNoRevisionWithARefusedCycle(t *testing.T) { - ctx := context.Background() - mods := builderModules(t) - for i, m := range mods { - switch m.name { - case "avahi": // avahi is built on mesh-tools (kind 3) ... - mods[i].manifest = strings.Replace(catalogueManifest(t, "avahi"), `"build": {`, `"build": { - "on": [{"arg": "RUNTIME_BASE", "module": "mesh-tools", "artifact": "tools"}],`, 1) - case "mesh-tools": // ... and mesh-tools' artifact needs avahi's (kind 2): no build seat breaks the cycle. - mods[i].manifest = strings.Replace(m.manifest, `"name": "tools",`, `"name": "tools", "needs": ["avahi:tools"],`, 1) - } - } - ix, _, merged := newFormIndexerUnmerged(t, mods...) - ix.Revisions = newMemObjects(0) - ix.Graph = newMem() - var logs strings.Builder - ix.Logf = func(format string, args ...any) { fmt.Fprintf(&logs, format+"\n", args...) } - ix.Message = func(context.Context, string, string, int) ([]byte, error) { return json.Marshal(merged) } - var said []meshcatalog.CatalogRevisionEvent - ix.SayCatalogRevision = func(_ context.Context, e meshcatalog.CatalogRevisionEvent) error { - said = append(said, e) - return nil - } - answer := meshdelivery.SourceAnswer{Branch: "beta", Revision: 9, Repositories: []meshdelivery.SourceHead{{Repository: catalogueRepo, Head: merged.Merge, Trunk: "beta"}}} - if err := ix.ApplyBranchSource(ctx, "beta", answer); err != nil { - t.Fatal(err) - } - if _, _, err := ReadRevision(ctx, ix.Revisions, BranchObject("beta", 1)); err == nil || len(said) != 0 { - t.Errorf("the branch kept a revision with a refused cycle (said %+v)", said) - } - if want := "mesh-tools:tools, avahi:tools, mesh-tools:tools"; !strings.Contains(logs.String(), want) { - t.Errorf("the journal %q does not say the cycle %s", logs.String(), want) - } -} - -// Design 47 G A-401, A-93: a head the source names that has no merged message (a branch newly watched) is read whole at that -// commit through the seat git and taken in as a merge of every file of its tree, so the branch keeps its first revision; -// a head with a merged message is taken in as before and the seat git is not asked. -func TestAHeadWithNoMergedMessageIsReadWholeAtItsCommit(t *testing.T) { - for _, row := range []struct { - name string - hasMessage bool - readFails bool - wantRevision bool - wantRead bool - }{ - {name: "no merged message: read whole", wantRevision: true, wantRead: true}, - {name: "a merged message: taken in as before", hasMessage: true, wantRevision: true}, - {name: "no merged message and the seat git cannot read it: no revision", readFails: true, wantRead: true}, - } { - t.Run(row.name, func(t *testing.T) { - ctx := context.Background() - ix, _, merged := newFormIndexerUnmerged(t, builderModules(t)...) - ix.Revisions = newMemObjects(0) - ix.Graph = newMem() - var logs strings.Builder - ix.Logf = func(format string, args ...any) { fmt.Fprintf(&logs, format+"\n", args...) } - ix.Message = func(context.Context, string, string, int) ([]byte, error) { - if !row.hasMessage { - return nil, fmt.Errorf("the forge gives no merged pull request") - } - return json.Marshal(merged) - } - reads := 0 - ix.ReadFile = func(context.Context, string, string, string) ([]byte, error) { return nil, nil } // no predefined list - ix.Read = func(_ context.Context, _, sha, dir string) (RepoRead, error) { - reads++ - if row.readFails || sha != merged.Merge { - return RepoRead{}, fmt.Errorf("the seat git does not answer") - } - r := RepoRead{Root: merged.Root, Trees: merged.Trees, Manifests: map[string]string{}, Locks: map[string]map[string]string{}, Packages: map[string][]string{}} - if _, ok := merged.Trees[dir]; ok { - r.Manifests[dir], r.Locks[dir], r.Packages[dir] = merged.Manifests[dir], merged.Locks[dir], merged.Packages[dir] - } - return r, nil - } - answer := meshdelivery.SourceAnswer{Branch: "beta", Revision: 9, Repositories: []meshdelivery.SourceHead{{Repository: catalogueRepo, Head: merged.Merge, Trunk: "beta"}}} - if err := ix.ApplyBranchSource(ctx, "beta", answer); err != nil { - t.Fatal(err) - } - if (reads > 0) != row.wantRead { - t.Errorf("the seat git was asked %d times, want asked: %v", reads, row.wantRead) - } - rev, entries, err := ReadRevision(ctx, ix.Revisions, BranchObject("beta", 1)) - if (err == nil) != row.wantRevision { - t.Fatalf("branch.beta.1 read: %v, want a revision: %v", err, row.wantRevision) - } - if row.wantRevision && (rev.Branch != "beta" || len(entries) != len(builderModules(t))) { - t.Errorf("branch.beta.1 = %+v with %d entries, want every module", rev, len(entries)) - } - if said := strings.Contains(logs.String(), "has no merged message"); said != row.wantRead { - t.Errorf("the journal %q says the whole read: %v, want %v", logs.String(), said, row.wantRead) - } - }) - } -} - -// Design 47 G A-401, A-402: the message a head read whole gives equals the forge bridge's: a templated module's needs -// include its template's text (ADR 0632 §2), and the catalogue's predefined list is carried, or said absent. -func TestAHeadReadWholeCarriesTheTemplateNeedsAndThePredefinedList(t *testing.T) { - templated := strings.Replace(catalogueManifest(t, "avahi"), `"resources": [`, `"resources": [ - {"id": "conf", "type": "file", "path": "/etc/avahi-x.conf", "template": "files/x.conf"},`, 1) - for _, row := range []struct { - name string - template []byte // nil: the forge has no such file - predefined []byte - wantNeeds bool - wantList string - wantNotFound bool - }{ - {name: "template and predefined list", template: []byte("host=${seat:node-service-manager:reach}"), predefined: []byte(`{"a":1}`), wantNeeds: true, wantList: `{"a":1}`}, - {name: "no template file and no list", wantNotFound: true}, - } { - t.Run(row.name, func(t *testing.T) { - ix, _, merged := newFormIndexerUnmerged(t, mod{name: "avahi", manifest: templated}) - ix.ReadFile = func(_ context.Context, repository, sha, p string) ([]byte, error) { - if repository != catalogueRepo || sha != merged.Merge { - return nil, fmt.Errorf("unexpected read of %s at %s", repository, sha) - } - switch p { - case "modules/avahi/files/x.conf": - return row.template, nil - case "predefined.json": - return row.predefined, nil - } - return nil, fmt.Errorf("unexpected file %s", p) - } - ix.Read = func(_ context.Context, _, _, dir string) (RepoRead, error) { - return RepoRead{Root: merged.Root, Trees: merged.Trees, Manifests: map[string]string{dir: merged.Manifests[dir]}, Locks: map[string]map[string]string{dir: merged.Locks[dir]}}, nil - } - f, err := ix.wholeAt(context.Background(), catalogueRepo, merged.Merge) - if err != nil { - t.Fatal(err) - } - text := map[string]string{} - if row.template != nil { - text["files/x.conf"] = string(row.template) - } - want, _ := catalogue.ModuleNeeds([]byte(templated), text) - if got := f.Needs["modules/avahi"]; !slices.Equal(got, want) || (row.wantNeeds && !slices.Contains(got, catalogue.ReadForm+catalogue.ReferenceFormSeatReach)) { - t.Errorf("needs %v, want %v", got, want) - } - if string(f.Predefined) != row.wantList || f.PredefinedRead != row.wantNotFound { - t.Errorf("predefined %q (read-absent %v), want %q (%v)", f.Predefined, f.PredefinedRead, row.wantList, row.wantNotFound) - } - }) - } -} diff --git a/modules/mesh-delivery-indexer/internal/index/candidate_manifests_test.go b/modules/mesh-delivery-indexer/internal/index/candidate_manifests_test.go deleted file mode 100644 index 613b979d..00000000 --- a/modules/mesh-delivery-indexer/internal/index/candidate_manifests_test.go +++ /dev/null @@ -1,109 +0,0 @@ -package index - -import ( - "context" - "errors" - "reflect" - "strings" - "testing" - - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" - "git.novox.be/novox/mesh-sdk/go/keys" -) - -func manifestPull(t *testing.T, number int, head, manifest string) forgemsg.Forge { - f := forgemsg.Forge{V: keys.Version, Kind: forgemsg.ForgePull, Repository: catalogueRepo, Branch: "feat/x", Number: number, Head: head, - Action: forgemsg.PullUpdated, Dirs: []string{"modules/avahi"}, DirsKnown: true} - if manifest != "" { - f.Manifests = map[string]string{"modules/avahi": manifest} - } - return f -} - -func candidateOf(t *testing.T, st *memStore) meshcatalog.CandidatesByGroupValue { - t.Helper() - key, err := CandidatesKey("feat/x") - if err != nil { - t.Fatal(err) - } - raw, ok := st.vals[key] - if !ok { - t.Fatalf("candidates holds %d keys, none %s", len(st.vals), key) - } - c, err := DecodeCandidate(raw) - if err != nil { - t.Fatal(err) - } - return c -} - -func TestACandidateGivesEachTouchedModuleItsManifestAndTheReasonItHasNoPreview(t *testing.T) { // 1451/AC1 - avahi := catalogueManifest(t, "avahi") - ix, _, _ := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z", aInputs)) - if err := ix.Apply(context.Background(), manifestPull(t, 301, h1, avahi)); err != nil { - t.Fatal(err) - } - c := candidateOf(t, ix.Store.(*memStore)) // DecodeCandidate has run Candidate.check - want := meshcatalog.CandidatesByGroupValuePullsItemFpsValue{Manifest: meshcatalogclient.ManifestDigest([]byte(avahi)), None2: "no merge preview is computed"} - if len(c.Pulls) != 1 || len(c.Pulls[0].Fps) != 1 || !reflect.DeepEqual(c.Pulls[0].Fps["avahi"], want) { - t.Fatalf("pulls = %+v, want Fps[avahi] = %+v", c.Pulls, want) - } - if c.Fpset == "" || c.Fpset != meshcatalogclient.FingerprintSet(c.Pulls) { - t.Errorf("fpset = %q, want %q", c.Fpset, meshcatalogclient.FingerprintSet(c.Pulls)) - } - // The same pull request heard again without its directories (the head unchanged) keeps what it had. - again := manifestPull(t, 301, h1, "") - again.Dirs, again.DirsKnown = nil, false - if err := ix.Apply(context.Background(), again); err != nil { - t.Fatal(err) - } - if c2 := candidateOf(t, ix.Store.(*memStore)); !reflect.DeepEqual(c2.Pulls[0].Fps["avahi"], want) || c2.Fpset != c.Fpset { - t.Errorf("after a message without directories: %+v fpset %q", c2.Pulls[0].Fps, c2.Fpset) - } -} - -func TestAPullWithoutManifestsWritesTheCandidateAsToday(t *testing.T) { // 1451/AC2 - ix, _, _ := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z", aInputs)) - if err := ix.Apply(context.Background(), manifestPull(t, 301, h1, "")); err != nil { - t.Fatal(err) - } - c := candidateOf(t, ix.Store.(*memStore)) - if len(c.Pulls) != 1 || c.Pulls[0].Fps != nil || c.Fpset != "" || len(c.Pulls[0].Modules) != 1 || c.Pulls[0].Modules[0] != "avahi" { - t.Errorf("candidate = %+v, want the module avahi and no Fps and no fpset", c) - } -} - -func TestTwoPullsGivingOneModuleDifferentManifestsAreRefusedNamingBoth(t *testing.T) { // 1451/AC3 - avahi := catalogueManifest(t, "avahi") - // The second pull request carries avahi's manifest changed (a line added at its end) for the same directory. - other := avahi + "\n" - ix, _, _ := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z", aInputs)) - if err := ix.Apply(context.Background(), manifestPull(t, 301, h1, avahi)); err != nil { - t.Fatal(err) - } - before := candidateOf(t, ix.Store.(*memStore)) - writes := ix.Store.(*memStore).writes - err := ix.Apply(context.Background(), manifestPull(t, 302, h2, other)) - if !errors.Is(err, ErrUnreadable) { - t.Fatalf("err = %v, want a refusal", err) - } - for _, w := range []string{catalogueRepo + "#301", catalogueRepo + "#302", `"avahi"`} { - if !strings.Contains(err.Error(), w) { - t.Errorf("the refusal %q does not name %s", err, w) - } - } - // Candidate.check refuses the group too, which is what the refusal stands for. - if _, encErr := EncodeCandidate(meshcatalog.CandidatesByGroupValue{V: keys.Version, Group: "feat.x", Heads: before.Heads, Pulls: []meshcatalog.CandidatesByGroupValuePullsItem{before.Pulls[0], { - Repository: catalogueRepo, Number: 302, Head: h2, Modules: []string{"avahi"}, - Fps: map[string]meshcatalog.CandidatesByGroupValuePullsItemFpsValue{"avahi": {Manifest: meshcatalogclient.ManifestDigest([]byte(other)), None2: meshcatalogclient.NoneNoPreview}}}}}); encErr == nil { - t.Error("the candidate check accepts two manifests for one module") - } - if ix.Store.(*memStore).writes != writes { - t.Error("a partial write was made") - } - if after := candidateOf(t, ix.Store.(*memStore)); len(after.Pulls) != 1 || after.Fpset != before.Fpset { - t.Errorf("the candidate changed: %+v", after) - } -} diff --git a/modules/mesh-delivery-indexer/internal/index/candidates.go b/modules/mesh-delivery-indexer/internal/index/candidates.go index f4170329..ee01742d 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidates.go +++ b/modules/mesh-delivery-indexer/internal/index/candidates.go @@ -2,27 +2,18 @@ package index import ( "bytes" - "cmp" "context" "errors" "fmt" "git.novox.be/novox/mesh-sdk/go/stateobjects" - "reflect" - "slices" - "strings" "sync" "sync/atomic" "time" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdeliverychecker" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein" - "git.novox.be/novox/mesh-sdk/go/keys" ) // ErrConflict is what a Store says when a write found the key changed or created since it was read. @@ -152,27 +143,22 @@ type Indexer struct { // Contains says whether a commit is the second or an ancestor of it (the seat git's contains). Contains func(ctx context.Context, repository, commit, in string) (bool, error) Logf func(format string, args ...any) - // ReadOpen asks the forge which pull requests into the trunk are open now (the seat git's open-pulls). A catalogue pull message - // that finds the candidates of another pull request asks it and deletes those of a pull request no longer open, so a - // closed message the indexer missed heals at the next one (issue 1126); nil asks none. - ReadOpen func(ctx context.Context) ([]forgemsg.OpenPull, error) mu sync.Mutex // guards source, toolchains and previews, which the watches and the forge messages all reach source sourceNow - saidSource map[string]uint64 // the newest source revision a take-in.taken named, by watched branch (nextSource) - previews map[string]meshdeliverychecker.PreviewsByRepositoryHeadTrunkValue // by previews key (previews.go) - toolchains map[string]map[string]nodebuildgo.Toolchain // language, then node (toolchains.go) - buildModules map[string]BuildModule // by language: the build modules whose toolchains are taken in (buildmodules.go) - floors sync.Map // the modules whose floor FillFloor reads now (tag_floor.go), one fill each - versionsFilling atomic.Bool // a FillVersions runs (versions_fill.go) - stepsSince map[string]time.Time // by build seat: since when its steps is unanswered (seed_steps.go) - tried map[string]bool // repository, commit and directory a registered module's manifest was tried at (registered.go) - unread map[string]bool // registered modules whose last read the forge could not answer, read again by RetryUnread - moving map[string]forgemsg.Forge // by modules key: a merge naming a module another repository holds, waiting for that trunk to drop it (moves.go) - retrying atomic.Bool // a RetryUnread runs in the background (kickRetry) - retryAgain atomic.Bool // an event asked for one more RetryUnread after the running one - retryWG sync.WaitGroup // the background RetryUnread, for tests to wait on - fresh map[string]bool // modules whose import form is computed again by hand (FingerprintsAgain), "*" for every one; under freshMu + saidSource map[string]uint64 // the newest source revision a take-in.taken named, by watched branch (nextSource) + toolchains map[string]map[string]nodebuildgo.Toolchain // language, then node (toolchains.go) + buildModules map[string]BuildModule // by language: the build modules whose toolchains are taken in (buildmodules.go) + floors sync.Map // the modules whose floor FillFloor reads now (tag_floor.go), one fill each + versionsFilling atomic.Bool // a FillVersions runs (versions_fill.go) + stepsSince map[string]time.Time // by build seat: since when its steps is unanswered (seed_steps.go) + tried map[string]bool // repository, commit and directory a registered module's manifest was tried at (registered.go) + unread map[string]bool // registered modules whose last read the forge could not answer, read again by RetryUnread + moving map[string]whole // by modules key: a merge naming a module another repository holds, waiting for that trunk to drop it (moves.go) + retrying atomic.Bool // a RetryUnread runs in the background (kickRetry) + retryAgain atomic.Bool // an event asked for one more RetryUnread after the running one + retryWG sync.WaitGroup // the background RetryUnread, for tests to wait on + fresh map[string]bool // modules whose import form is computed again by hand (FingerprintsAgain), "*" for every one; under freshMu freshMu sync.Mutex recompute sync.Mutex // one Recompute at a time untaken map[string]string // repository to the latest merge applied while the taken state was missing (taken.go) @@ -205,28 +191,17 @@ type Indexer struct { // OwnBuildState keeps which build this process knows it runs, or since when and why not (FixOwnBuild); nil keeps none. OwnBuildState Store // Heard keeps what the indexer heard and compares later (A-80): the time a call is unanswered since (steps-since, call-since), - // toolchain., delivered., pending.., acted.., each key part escaped as a name + // toolchain., pending.., acted.., each key part escaped as a name // (§73), and the mark versions-filled; nil keeps none. It is the state heard, whose key forge keeps the forge messages handled. - Heard Store - // Placement asks the assigner's verb placement (A-62) and RunningWhole reads the node-engine's stream running whole (§556), both for the - // edge entries of the first fill (§377); nil reads none, and the entries of kinds 2, 4 and 6 then read "not yet delivered". - Placement PlacementAsker - RunningWhole RunningReader + Heard Store judgePending atomic.Bool // the start's judge pass failed: judged again at the next message (verdicts.go) - // Message asks the forge bridge for the merged message of a merge, repository, merge commit and pull request number in (the number lets the bridge read a merge made on the forge, not through the gate), the message whole out (a - // pending value keeps a reference, not a copy: issue 1585); an error is the bridge's words. Nil asks none. - Message func(ctx context.Context, repository, merge string, number int) ([]byte, error) - heldMu sync.Mutex - held map[uint64]forgemsg.Forge // the merged messages Arrive was handed, by stream sequence, until their merge is taken in or dropped - // Pending is the state of merges held for the indexer their revision names (pending.go, novox/hq ADR 0633 §1); nil - // takes every merge in at once, as before the take-in step. - Pending Store // SayTakeIn publishes a take-in event (take-in.pending, take-in.taken; ADR 0633 §5); nil says none. SayTakeIn func(ctx context.Context, t takein.TakeIn) error // Runs is the state runs: the runs on request the indexer holds a candidate for (runcandidates.go, A-174); nil keeps none. Runs Store - runMu sync.Mutex // one run event at a time (runcandidates.go) - pendingMu sync.Mutex // one take-in step at a time: a merge, a group's end, the start and a hand act (pending.go) + runMu sync.Mutex // one run event at a time (runcandidates.go) + heldFull atomic.Bool // a take-in is held because the catalog store is full: room is said when it has room again (RoomAgain) + pendingMu sync.Mutex // one take-in step at a time: a merge, a group's end, the start and a hand act (pending.go) } func (ix *Indexer) logf(format string, args ...any) { @@ -235,33 +210,19 @@ func (ix *Indexer) logf(format string, args ...any) { } } -// Apply takes one message of the bridge's stream FORGE into the group of its branch. A pull message adds or moves the -// pull request, a closed one or a merged message takes it out; a group left with none is deleted. A change is written -// by compare-and-set on the revision read, and read again on a conflict. A lone pull request is a group of one. A -// merged message also writes the fingerprint of each module of its repository (modules.go). -func (ix *Indexer) Apply(ctx context.Context, f forgemsg.Forge) error { - _, err := ix.ApplyMessage(ctx, f) - return err -} - -// ApplyMessage is Apply, and answers which modules a merged message wrote and kept (for the indexer's heard state). Each -// applied merged message is said in one line: its repository, commit and pull request, and the modules written. -func (ix *Indexer) ApplyMessage(ctx context.Context, f forgemsg.Forge) (Applied, error) { +// ApplyMessage takes one repository's trees at a commit in (design 47 H §743): the fingerprint of each module of the repository +// (modules.go), the taken key, the catalogue's predefined list, the packages' names and the seat definitions it carries. It +// answers which modules it wrote and kept, and says them in one line. +func (ix *Indexer) ApplyMessage(ctx context.Context, f whole) (Applied, error) { var applied Applied - if f.Kind == forgemsg.ForgeOpen { - return applied, ix.applyOpen(ctx, f) - } - if f.Kind != forgemsg.ForgePull && f.Kind != forgemsg.ForgeMerged { - return applied, nil - } lateMerge := false - if f.Kind == forgemsg.ForgeMerged && ix.Modules != nil { + if ix.Modules != nil { w, err := ix.applyMerged(ctx, f) applied = Applied{Written: w.written, Late: w.late} if err != nil { return applied, err } - ix.logf("%s", w.said(f.Number)) + ix.logf("%s", w.said()) if w.late == "" { // a late message leaves the taken key, which names a later merge if err := ix.writeTakenKnown(ctx, f, w.asked); err != nil { return applied, err @@ -270,13 +231,12 @@ func (ix *Indexer) ApplyMessage(ctx context.Context, f forgemsg.Forge) (Applied, if err := ix.writePredefined(ctx, f); err != nil { return applied, err } - // Each package's name in its own ecosystem, which the builder numbers it under (design 32), kept before the - // manifests are judged: a manifest of this merge naming a package this merge brings is judged knowing its name. + // Each package's name in its own ecosystem, which the builder numbers it under (design 32). if err := ix.KeepPackageNames(ctx, f); err != nil { return applied, err } } - ix.kickRetry(ctx) // a registered module the forge could not read is read again on a merge, in the background (issue 1945) + ix.kickRetry(ctx) // a registered module the forge could not read is read again on a take-in, in the background (issue 1945) lateMerge = w.late != "" } // The seat definitions of the contracts modules it carries (DLV-I17, I18): a late merge moves no trunk's. @@ -285,114 +245,7 @@ func (ix *Indexer) ApplyMessage(ctx context.Context, f forgemsg.Forge) (Applied, return applied, err } } - var digests map[string]string - var held map[string]meshcatalog.ModulesByModuleValue - if f.Kind == forgemsg.ForgePull { - var err error - if digests, err = ix.keepManifests(ctx, f); err != nil { - return applied, err - } - if held, err = ix.heldModules(ctx); err != nil { - return applied, err - } - } - return applied, ix.writeCandidate(ctx, f, digests, held) -} - -// removeCandidate takes a merged pull request out of its group's candidate and writes nothing else of its message: for a -// merge a newer merge of the same repository covers (it needs only the repository, number and branch). -func (ix *Indexer) removeCandidate(ctx context.Context, f forgemsg.Forge) error { - return ix.writeCandidate(ctx, f, nil, nil) -} - -// writeCandidate writes the group's candidate with the pull request f names (taken out for a merged or closed one). -func (ix *Indexer) writeCandidate(ctx context.Context, f forgemsg.Forge, digests map[string]string, held map[string]meshcatalog.ModulesByModuleValue) error { - if trunkLike[f.Branch] { - ix.logf("%s#%d: its branch %q is no delivery group, so no candidate", f.Repository, f.Number, f.Branch) - return nil - } - key, err := CandidatesKey(f.Branch) - if err != nil { - return fmt.Errorf("%w: %s#%d: %v", ErrUnreadable, f.Repository, f.Number, err) - } - return write(ctx, ix.Store, key, func(value []byte, found bool) ([]byte, bool, error) { - var cur *meshcatalog.CandidatesByGroupValue - if found { - c, err := DecodeCandidate(value) - if err != nil { - return nil, false, fmt.Errorf("candidates.%s holds what is not a candidate: %w", key, err) - } - cur = &c - } - next := ix.next(cur, f, digests, held) - if next != nil { - ix.withPreviews(ctx, next) - if err := ix.refuseDisagreement(*next); err != nil { - return nil, false, err - } - } - switch { - case next == nil && cur == nil, next != nil && cur != nil && SameCandidate(*cur, *next): - return nil, false, nil - case next == nil: - return nil, true, nil - } - data, err := EncodeCandidate(*next) - return data, false, err - }) -} - -// applyOpen takes an open message into the candidates (novox/hq ADR 0443 §5): every pull request it does not name is -// taken out of its group, and a group left with none is deleted. What the message names is left as it is. A pull request -// opened after the bridge read the forge is heard after this message and added again by its own. -func (ix *Indexer) applyOpen(ctx context.Context, f forgemsg.Forge) error { - named := make(map[forgemsg.OpenPull]bool, len(f.Open)) - for _, p := range f.Open { - named[p] = true - } - keys, err := ix.Store.Keys(ctx) - if err != nil { - return fmt.Errorf("listing the candidates: %w", err) - } - for _, key := range keys { - if meshcatalogclient.IsTryGroup(key) { - continue // a run's candidate is no pull request's: its run owns it (runcandidates.go) - } - err := write(ctx, ix.Store, key, func(value []byte, found bool) ([]byte, bool, error) { - if !found { - return nil, false, nil - } - c, err := DecodeCandidate(value) - if err != nil { - return nil, false, fmt.Errorf("candidates.%s holds what is not a candidate: %w", key, err) - } - kept := slices.DeleteFunc(slices.Clone(c.Pulls), func(p meshcatalog.CandidatesByGroupValuePullsItem) bool { - return !named[forgemsg.OpenPull{Repository: p.Repository, Number: int(p.Number)}] - }) - switch { - case len(kept) == len(c.Pulls): - return nil, false, nil - case len(kept) == 0: - ix.logf("%s: no pull request of the group is open, so no candidate", key) - return nil, true, nil - } - heads := make([]meshdeliveryclient.Head, len(kept)) - for i, p := range kept { - heads[i] = meshdeliveryclient.Head{Repository: p.Repository, Number: int(p.Number), Head: p.Head} - } - hash, err := meshdeliveryclient.HeadsHash(heads) - if err != nil { - return nil, false, fmt.Errorf("%w: candidates.%s: %v", ErrUnreadable, key, err) - } - c.Pulls, c.Heads = kept, hash - data, err := EncodeCandidate(c) - return data, false, err - }) - if err != nil { - return err - } - } - return nil + return applied, nil } // write changes one key of a store by compare-and-set: compute is given the value read (found says there is one) and @@ -440,89 +293,3 @@ func write(ctx context.Context, st Store, key string, compute func(value []byte, func unchanged(next []byte, del bool, held []byte, found bool) bool { return found && !del && next != nil && bytes.Equal(next, held) } - -// next is the group after the message; nil when the group is left with no pull request. -// -// A pull message that carries manifests (issue 1448) gives each module it touches, in the pull request's Fps, the digest of -// its manifest and why it has no fingerprint of the new form (meshcatalogclient.NoneNoPreview), and the candidate its FPSet (issue 1451); a -// pull request whose head is held again without its directories keeps the Fps it had. digests are the manifests' digests by -// directory; held are the modules the modules bucket holds, which name the module of a changed directory (moduleOfDir). -func (ix *Indexer) next(cur *meshcatalog.CandidatesByGroupValue, f forgemsg.Forge, digests map[string]string, held map[string]meshcatalog.ModulesByModuleValue) *meshcatalog.CandidatesByGroupValue { - var pulls []meshcatalog.CandidatesByGroupValuePullsItem - var base int64 - if cur != nil { - pulls, base = slices.Clone(cur.Pulls), cur.Base - } - at := slices.IndexFunc(pulls, func(p meshcatalog.CandidatesByGroupValuePullsItem) bool { - return p.Repository == f.Repository && p.Number == int64(f.Number) - }) - if f.Kind == forgemsg.ForgeMerged || f.Action == forgemsg.PullClosed { - if at < 0 { - return cur - } - pulls = slices.Delete(pulls, at, at+1) - } else { - entry := meshcatalog.CandidatesByGroupValuePullsItem{Repository: f.Repository, Number: int64(f.Number), Head: f.Head, Modules: []string{}} - switch { - case f.DirsKnown: - for _, dir := range f.Dirs { - m := moduleOfDir(held, f, dir) - if !slices.Contains(entry.Modules, m) { - entry.Modules = append(entry.Modules, m) - } - if digest, ok := digests[dir]; ok { - if entry.Fps == nil { - entry.Fps = map[string]meshcatalog.CandidatesByGroupValuePullsItemFpsValue{} - } - entry.Fps[m] = meshcatalog.CandidatesByGroupValuePullsItemFpsValue{Manifest: digest, None2: meshcatalogclient.NoneNoPreview} - } - } - case at >= 0 && pulls[at].Head == f.Head: - // Its directories are not known now, for the head the group already holds: what it knew stands. - entry.Modules, entry.Fps = pulls[at].Modules, pulls[at].Fps - } - if at >= 0 { - pulls[at] = entry - } else { - pulls = append(pulls, entry) - } - } - if len(pulls) == 0 { - return nil - } - slices.SortFunc(pulls, func(a, b meshcatalog.CandidatesByGroupValuePullsItem) int { - if c := strings.Compare(a.Repository, b.Repository); c != 0 { - return c - } - return cmp.Compare(a.Number, b.Number) - }) - heads := make([]meshdeliveryclient.Head, len(pulls)) - for i, p := range pulls { - heads[i] = meshdeliveryclient.Head{Repository: p.Repository, Number: int(p.Number), Head: p.Head} - } - hash, _ := meshdeliveryclient.HeadsHash(heads) - return &meshcatalog.CandidatesByGroupValue{V: keys.Version, Group: f.Branch, Base: base, Heads: hash, Pulls: pulls, Fpset: meshcatalogclient.FingerprintSet(pulls)} -} - -// refuseDisagreement refuses a candidate in which two pull requests give one module different manifests, as the -// candidate's own check does (mesh-sdk Candidate.check), naming both pull requests so the log says which; nothing is -// written, and the message is dropped as unreadable and said in the log (main.go handle): the next message of either pull -// request is judged again. -func (ix *Indexer) refuseDisagreement(c meshcatalog.CandidatesByGroupValue) error { - first := map[string]meshcatalog.CandidatesByGroupValuePullsItem{} - for _, p := range c.Pulls { - for module, m := range p.Fps { - before, ok := first[module] - if !ok { - first[module] = p - continue - } - if !reflect.DeepEqual(before.Fps[module], m) { - err := fmt.Errorf("%w: candidates.%s: %s#%d and %s#%d give the module %q different manifests (%s and %s), so the group is not written", - ErrUnreadable, c.Group, before.Repository, before.Number, p.Repository, p.Number, module, before.Fps[module].Manifest, m.Manifest) - return err - } - } - } - return nil -} diff --git a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go index 4772043a..cda30966 100644 --- a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go +++ b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go @@ -433,22 +433,19 @@ func (ix *Indexer) sayFill(ctx context.Context, event, about string, f meshcatal } } -// RoomAgain says room and evaluates pending again when a merge held with the reason objects full finds the store -// below its filling mark (80 % of its cap): called after anything removes catalog graph revisions (no timer). It answers whether room was said. +// RoomAgain says room when a take-in held with the reason objects full (heldFull) finds the store below its filling mark (80 % +// of its cap): called after anything removes catalog graph revisions (no timer). It answers whether room was said; the held +// take-in hears it (design 47 H §744). func (ix *Indexer) RoomAgain(ctx context.Context) (bool, error) { - if ix.Revisions == nil || ix.Pending == nil { + if ix.Revisions == nil || !ix.heldFull.Load() { return false, nil } - entries, err := ReadPending(ctx, ix.Pending, ix.logf) - if err != nil || !slices.ContainsFunc(entries, func(e PendingEntry) bool { return e.Value.Held == HeldObjectsFull }) { - return false, err - } used, capacity, err := ix.Revisions.Fill(ctx) if err != nil || (capacity > 0 && used*5 >= capacity*4) { return false, err // room is said below the filling mark, 80 %, so one refusal is not followed by another at once } ix.sayFill(ctx, meshcatalog.EventRoom, fmt.Sprintf("%d", used), meshcatalog.Fill{Bytes: used, Cap: capacity}) - return true, ix.PendingAgain(ctx) // room in catalog-revisions: pending evaluated again + return true, nil } // TrunkView is the indexer's view of the trunk read from the newest trunk revision the bucket keeps (the lead's ruling diff --git a/modules/mesh-delivery-indexer/internal/index/control_records.go b/modules/mesh-delivery-indexer/internal/index/control_records.go index 3fe9f329..d1ccb81b 100644 --- a/modules/mesh-delivery-indexer/internal/index/control_records.go +++ b/modules/mesh-delivery-indexer/internal/index/control_records.go @@ -8,7 +8,6 @@ import ( "time" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "git.novox.be/novox/mesh-sdk/go/keys" "git.novox.be/novox/mesh-sdk/go/process" ) @@ -20,7 +19,7 @@ import ( // writePredefined keeps the predefined list a merged message carries, unless the key names a later merge: a merge older // than the one it names never replaces it (ADR 0348 §7). The same list at the same merge writes nothing (ADR 0289 §1). -func (ix *Indexer) writePredefined(ctx context.Context, f forgemsg.Forge) error { +func (ix *Indexer) writePredefined(ctx context.Context, f whole) error { if ix.Predefined == nil || len(f.Predefined) == 0 { return nil } diff --git a/modules/mesh-delivery-indexer/internal/index/control_records_test.go b/modules/mesh-delivery-indexer/internal/index/control_records_test.go index 3ad07418..18796889 100644 --- a/modules/mesh-delivery-indexer/internal/index/control_records_test.go +++ b/modules/mesh-delivery-indexer/internal/index/control_records_test.go @@ -10,10 +10,8 @@ import ( "time" mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" "git.novox.be/novox/mesh-sdk/go/keys" "git.novox.be/novox/mesh-sdk/go/process" @@ -174,7 +172,7 @@ func TestTheControlRecords(t *testing.T) { ctx := context.Background() merged := func(merge string) { t.Helper() - if err := ix.ApplySeatDefinitions(ctx, forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: repo, Number: 1, Merge: merge, + if err := ix.ApplySeatDefinitions(ctx, whole{Repository: repo, Merge: merge, Manifests: map[string]string{meshcatalogclient.RootPath: manifest, "modules/other": `{"module":"other","version":"1.0"}`}}); err != nil { t.Fatal(err) } @@ -230,7 +228,7 @@ func TestTheControlRecords(t *testing.T) { otherManifest := strings.Replace(manifest, `"mesh-contracts"`, `"other-contracts"`, 1) mergedOther := func(merge string) { t.Helper() - if err := ix.ApplySeatDefinitions(ctx, forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: other, Number: 2, Merge: merge, + if err := ix.ApplySeatDefinitions(ctx, whole{Repository: other, Merge: merge, Manifests: map[string]string{meshcatalogclient.RootPath: otherManifest}}); err != nil { t.Fatal(err) } @@ -251,90 +249,14 @@ func TestTheControlRecords(t *testing.T) { if g, ok := entry("git.trunk"); !ok || g.Repository != repo { t.Errorf("the second repository's merge without git removed the first's entry: %+v", g) } - // A pull request's head: its own entry, the trunk's answered unless it is named. - if err := ix.ApplySeatDefinitions(ctx, forgemsg.Forge{Kind: forgemsg.ForgePull, Action: forgemsg.PullUpdated, Repository: repo, Number: 9, Head: head, - Dirs: []string{meshcatalogclient.RootPath}, Manifests: map[string]string{meshcatalogclient.RootPath: manifest}}); err != nil { - t.Fatal(err) - } - pk := SeatDefinitionKey("git", SourcePull, repo, 9) - src := func() memSource { - out := memSource{} - for k, v := range defs.vals { - out[k] = &views.Entry{Value: v} - } - return out - } - if a, err := SeatDefinitionOf(src(), nil, "git", "", 0); err != nil || a.Answered != SourceTrunk || a.Entry.Commit != m3 { - t.Errorf("the trunk's answer %+v, %v", a, err) - } - if a, err := SeatDefinitionOf(src(), nil, "git", repo, 9); err != nil || a.Answered != repo+"#9" || a.Entry.Commit != head || a.Trunk.Commit != m3 { - t.Errorf("the pull request's answer %+v, %v", a, err) - } - // Its check's verdict names out-of-step files; a trunk build end names them too, a failed build clears nothing. - if err := ix.ApplyVerdict(ctx, repo, 9, head, []string{"go/b.go", "go/a.go"}); err != nil { - t.Fatal(err) - } - end := func(outcome string, files ...string) []byte { - raw, _ := json.Marshal(map[string]any{"v": 1, "event": builds.BuildEndedEvent, "module": "mesh-contracts", "ref": "main", - "commit": m3, "outcome": outcome, "out-of-step": files}) - return raw - } - ix.ApplyOutOfStepBuild(ctx, end("failed", "ts/x.ts")) - ix.ApplyOutOfStepBuild(ctx, end("failed")) - fsrc := memSource{} - for k, v := range findings.vals { - fsrc[k] = &views.Entry{Value: v} - } - rows, err := OutOfStepOf(testSelf, src(), fsrc) - if err != nil || len(rows) != 2 || rows[0].Module != "mesh-contracts" || !strings.Contains(rows[0].Summary, "ts/x.ts") || - rows[1].Pull != repo+"#9" || !reflect.DeepEqual(rows[1].Files, []string{"go/a.go", "go/b.go"}) { - t.Errorf("out of step %+v, %v; want the trunk's module and the pull request's sorted files", rows, err) - } - ix.ApplyOutOfStepBuild(ctx, end(builds.BuildDone)) - if len(findings.vals) != 0 { - t.Errorf("findings %v stand after a clean build end", findings.vals) - } - // Ruling (lead, on A2i's finding 3): indexer_seat_definitions_again for a pull request reads the head's own manifest's - // generated-from: a trunk that names none, a head that names it, is read as the contracts module the head is. - ix.Modules, ix.Manifests = newMem(), newMem() - plain := `{"module":"mesh-contracts","version":"1.0"}` - kept, _ := meshcatalogclient.EncodeManifest(meshcatalog.ManifestsByDigestValue{Module: "mesh-contracts", Manifest: plain}) - mkey, _ := meshcatalogclient.ManifestsKey(meshcatalogclient.ManifestDigest([]byte(plain))) - ix.Manifests.(*memStore).vals[mkey] = kept - mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: "src2:" + strings.Repeat("a", 64), Manifest: meshcatalogclient.ManifestDigest([]byte(plain)), Repository: repo, Commit: m3, Path: meshcatalogclient.RootPath}) - ix.Modules.(*memStore).vals["mesh-contracts"] = mod - files["h2"] = map[string]string{"module.json": manifest, "seats/git/seat.json": strings.Replace(git, "ADR 0111", "ADR 0400", 1)} - if r := ix.SeatDefinitionsAgain(ctx, "mesh-contracts", 9, "h2"); !r.Stored { - t.Errorf("again for the pull request's head = %+v, want it read with the head's generated-from", r) - } - if e, ok := entry(pk); !ok || e.Commit != "h2" { - t.Errorf("the pull request's entry after again = %+v (%v)", e, ok) - } - if r := ix.SeatDefinitionsAgain(ctx, "mesh-contracts", 0, ""); r.Stored || !strings.Contains(r.Answer, "trunk's manifest") { - t.Errorf("again for the trunk, whose manifest names no generated-from = %+v", r) - } - // The pull request closes: its entries and its out-of-step files go, said. - if err := ix.ApplySeatDefinitions(ctx, forgemsg.Forge{Kind: forgemsg.ForgePull, Action: forgemsg.PullClosed, Repository: repo, Number: 9, Head: head}); err != nil { - t.Fatal(err) - } - if _, ok := entry(pk); ok { - t.Error("a closed pull request's entry is kept") - } - if _, ok := defs.vals[OutOfStepKey(repo, 9)]; ok { - t.Error("a closed pull request's out-of-step files are kept") - } - for _, c := range said { - if c.Source != SourceTrunk { - t.Errorf("said %+v: only a trunk's definition is said (ADR 0636 §4)", c) - } - } // Design 47 (the indexer reads every definition before it judges): at its start the indexer reads every contracts // module's trunk at its repository's head in the source revision, so a definition merged with no event heard is kept // before any manifest is judged; a repository the source names no head of is read at the module's own commit. - kept, _ = meshcatalogclient.EncodeManifest(meshcatalog.ManifestsByDigestValue{Module: "mesh-contracts", Manifest: manifest}) - mkey, _ = meshcatalogclient.ManifestsKey(meshcatalogclient.ManifestDigest([]byte(manifest))) + ix.Modules, ix.Manifests = newMem(), newMem() + kept, _ := meshcatalogclient.EncodeManifest(meshcatalog.ManifestsByDigestValue{Module: "mesh-contracts", Manifest: manifest}) + mkey, _ := meshcatalogclient.ManifestsKey(meshcatalogclient.ManifestDigest([]byte(manifest))) ix.Manifests.(*memStore).vals[mkey] = kept - mod, _ = meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: "src2:" + strings.Repeat("a", 64), Manifest: meshcatalogclient.ManifestDigest([]byte(manifest)), Repository: repo, Commit: m3, Path: meshcatalogclient.RootPath}) + mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: "src2:" + strings.Repeat("a", 64), Manifest: meshcatalogclient.ManifestDigest([]byte(manifest)), Repository: repo, Commit: m3, Path: meshcatalogclient.RootPath}) ix.Modules.(*memStore).vals["mesh-contracts"] = mod files["h3"] = map[string]string{"seats/git/seat.json": strings.Replace(git, "ADR 0111", "ADR 0500", 1)} ix.source = sourceNow{rev: 7, heads: map[string]string{repo: "h3"}} @@ -381,7 +303,7 @@ func TestTheControlRecords(t *testing.T) { mergeAt := func(repository, merge string, manifests map[string]string) { t.Helper() ix.ReadFile = func(_ context.Context, _, sha, p string) ([]byte, error) { return []byte(files[sha][p]), nil } - if err := ix.ApplySeatDefinitions(ctx, forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: repository, Number: 3, Merge: merge, Manifests: manifests}); err != nil { + if err := ix.ApplySeatDefinitions(ctx, whole{Repository: repository, Merge: merge, Manifests: manifests}); err != nil { t.Fatal(err) } } @@ -427,15 +349,12 @@ func TestTheControlRecords(t *testing.T) { t.Errorf("said %+v: a handover removes nothing", c) } } - // A pull request of mesh-contracts defining a Provisioning seat fails its check as a definition that does not parse. - files["h2"] = map[string]string{"seats/wants/seat.json": seat("wants", "Provisioning")} - ix.ReadFile = func(_ context.Context, _, sha, p string) ([]byte, error) { return []byte(files[sha][p]), nil } - if err := ix.ApplySeatDefinitions(ctx, forgemsg.Forge{Kind: forgemsg.ForgePull, Action: forgemsg.PullUpdated, Repository: repo, Number: 11, Head: "h2", - Dirs: []string{meshcatalogclient.RootPath}, Manifests: map[string]string{meshcatalogclient.RootPath: manifest}}); err != nil { - t.Fatal(err) - } - if e, _ := entry(SeatDefinitionKey("wants", SourcePull, repo, 11)); !strings.Contains(e.Unparsed, "mesh-provisioning-contracts") || e.Definition != nil { - t.Errorf("a pull request's wants from mesh-contracts = %+v, want it refused naming the domain's module", e) + src := func() memSource { + out := memSource{} + for k, v := range defs.vals { + out[k] = &views.Entry{Value: v} + } + return out } // The verb seat-definitions: no seat lists every trunk seat with its digest (never a pull request's); a seat answers it whole. listed, err := SeatDefinitionsOf(src(), nil, "") @@ -463,7 +382,7 @@ func TestTheControlRecords(t *testing.T) { }} sdk := `{"module":"mesh-sdk","version":"1.0","build":{"artifacts":[{"name":"go","kind":"package","language":"go","paths":["go"]},` + `{"name":"ts","kind":"package","language":"typescript","paths":["src","package.json"],"series":"0.3"}]}}` - if err := ix.KeepPackageNames(ctx, forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-sdk", Merge: "m1", + if err := ix.KeepPackageNames(ctx, whole{Repository: "novox/mesh-sdk", Merge: "m1", Manifests: map[string]string{meshcatalogclient.RootPath: sdk}}); err != nil { t.Fatal(err) } diff --git a/modules/mesh-delivery-indexer/internal/index/dryrun.go b/modules/mesh-delivery-indexer/internal/index/dryrun.go new file mode 100644 index 00000000..2f2ba5bc --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/dryrun.go @@ -0,0 +1,88 @@ +package index + +import ( + "fmt" + "maps" + "reflect" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo" +) + +type dryRun struct { + ix *Indexer + modules, graph *overlayStore + candidates, taken, predefined *overlayStore + oldSections *overlayStore +} + +// dry is an indexer over overlays of this one's states (overlay.go), for a run on request's candidate (runcandidates.go): what +// it computes stays in memory and nothing reaches the bus but the manifests and the packages, which clause 1 writes all the +// same. It says no event, kicks nothing in the background, and knows what this one knows of the source and the toolchains. Every field of Indexer is named here (TestDryNamesEveryField holds it so). +func (ix *Indexer) dry() *dryRun { + over := func(st Store) (Store, *overlayStore) { + if st == nil { + return nil, nil + } + o := newOverlay(st) + return o, o + } + d := &dryRun{} + x := &Indexer{Module: ix.Module, Manifests: ix.Manifests, Packages: ix.Packages, JudgeID: ix.JudgeID, Files: ix.Files, GoSum: ix.GoSum, + GoMod: ix.GoMod, Now: ix.Now, Builds: ix.Builds, Named: ix.Named, Trunk: ix.Trunk, BranchHead: ix.BranchHead, Read: ix.Read, Contains: ix.Contains, + OwnBuild: nil, SayTakeIn: nil, + // The registry milestone's fields: what a take-in would write on the bus stays out of a dry run, so the states of the + // requirements, findings, seat definitions, holds and their bounds are not overlaid, nothing is said, and no hold is + // weighed (Holds and Counted nil hold nothing); ReadFile only reads. + Requirements: nil, Findings: nil, SeatDefinitions: nil, ReadFile: ix.ReadFile, SaySeatDefinition: nil, + OwnBuildState: nil, Heard: nil, underSeats: true, + // A dry run keeps no catalog graph revision (catalog_revisions.go): its revision is the take-in's to keep. + Revisions: nil, SayCatalogRevision: nil, SayFill: nil, EntryIndex: nil, NamedSets: nil, Versions: nil, SayVersionReleased: nil, SayVersionsFilled: nil, Inputs: nil, InputObjects: nil, Pins: nil, PinsPending: nil, + // A dry run writes no registration and asks no one. + Registry: nil, SeedModules: nil} + x.Logf = func(format string, args ...any) { ix.logf("take-in, computed and not written: "+format, args...) } + x.Store, d.candidates = over(ix.Store) + x.Modules, d.modules = over(ix.Modules) + x.Graph, d.graph = over(ix.Graph) + x.Taken, d.taken = over(ix.Taken) + x.Predefined, d.predefined = over(ix.Predefined) + x.OldSections, d.oldSections = over(ix.OldSections) + ix.mu.Lock() + x.source = sourceNow{rev: ix.source.rev, heads: maps.Clone(ix.source.heads)} + x.toolchains = map[string]map[string]nodebuildgo.Toolchain{} + for l, by := range ix.toolchains { + x.toolchains[l] = maps.Clone(by) + } + x.buildModules = maps.Clone(ix.buildModules) // the dry run computes with the build modules the running indexer takes + x.moving = maps.Clone(ix.moving) + x.untaken = maps.Clone(ix.untaken) + ix.mu.Unlock() + ix.ownMu.Lock() + x.ownFP = ix.ownFP + ix.ownMu.Unlock() + d.ix = x + return d +} + +// refuseDisagreement refuses a candidate in which two pull requests give one module different manifests, as the +// candidate's own check does (mesh-sdk Candidate.check), naming both pull requests so the log says which; nothing is +// written, and the message is dropped as unreadable and said in the log (main.go handle): the next message of either pull +// request is judged again. +func (ix *Indexer) refuseDisagreement(c meshcatalog.CandidatesByGroupValue) error { + first := map[string]meshcatalog.CandidatesByGroupValuePullsItem{} + for _, p := range c.Pulls { + for module, m := range p.Fps { + before, ok := first[module] + if !ok { + first[module] = p + continue + } + if !reflect.DeepEqual(before.Fps[module], m) { + err := fmt.Errorf("%w: candidates.%s: %s#%d and %s#%d give the module %q different manifests (%s and %s), so the group is not written", + ErrUnreadable, c.Group, before.Repository, before.Number, p.Repository, p.Number, module, before.Fps[module].Manifest, m.Manifest) + return err + } + } + } + return nil +} diff --git a/modules/mesh-delivery-indexer/internal/index/fingerprints.go b/modules/mesh-delivery-indexer/internal/index/fingerprints.go index 14a2be8f..7b93ea1e 100644 --- a/modules/mesh-delivery-indexer/internal/index/fingerprints.go +++ b/modules/mesh-delivery-indexer/internal/index/fingerprints.go @@ -58,8 +58,7 @@ func (ix *Indexer) FingerprintsAgain(ctx context.Context, module string) TakenIn ix.freshMu.Lock() ix.fresh = map[string]bool{key: true} ix.freshMu.Unlock() - // The path of a merge's recompute: the modules, then the candidates their fingerprints reach (recomputeWith), then the - // pending merges a changed fingerprint settles (PendingAgain, as after a toolchain report). + // The path of a merge's recompute: the modules, then the candidates their fingerprints reach (recomputeWith). err = ix.recomputeWith(ctx, nil, nil, "") ix.freshMu.Lock() ix.fresh = nil @@ -68,9 +67,6 @@ func (ix *Indexer) FingerprintsAgain(ctx context.Context, module string) TakenIn out.Answer = "nothing was written: " + err.Error() return out } - if err := ix.PendingAgain(ctx); err != nil { - ix.logf("the pending merges are not evaluated again after the fingerprints were computed again: %v", err) - } after, err := ix.fingerprintsHeld(ctx) if err != nil { out.Answer = "computed, but the modules state cannot be read back: " + err.Error() diff --git a/modules/mesh-delivery-indexer/internal/index/helpers_test.go b/modules/mesh-delivery-indexer/internal/index/helpers_test.go index 8dd5af84..94a3faa9 100644 --- a/modules/mesh-delivery-indexer/internal/index/helpers_test.go +++ b/modules/mesh-delivery-indexer/internal/index/helpers_test.go @@ -123,3 +123,28 @@ func (m *memFindings) Held() []mcseat.Finding { // testSelf is the module the tests' indexer runs as, its credential's: its findings, its own entry and its graph key are named // from it, never from a constant (design 47 G A-296). const testSelf = "acme-indexer" + +// Apply takes one repository's trees in, as a take-in does (ApplyMessage), for the tests. +func (ix *Indexer) Apply(ctx context.Context, f whole) error { + _, err := ix.ApplyMessage(ctx, f) + return err +} + +const otherMerge = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + +// srcT is a source revision's heads as the tests write them; applyS makes it the latest source revision the indexer took in. +type srcT struct { + Group string + Heads map[string]string +} + +func sourceOf(repo, head string) srcT { return srcT{Group: "g", Heads: map[string]string{repo: head}} } + +func applyS(ix *Indexer, _ context.Context, rev uint64, s srcT) error { + ix.mu.Lock() + defer ix.mu.Unlock() + if rev >= ix.source.rev { + ix.source = sourceNow{rev: rev, heads: maps.Clone(s.Heads)} + } + return nil +} diff --git a/modules/mesh-delivery-indexer/internal/index/imports_test.go b/modules/mesh-delivery-indexer/internal/index/imports_test.go index 3f0c3c82..456c2f48 100644 --- a/modules/mesh-delivery-indexer/internal/index/imports_test.go +++ b/modules/mesh-delivery-indexer/internal/index/imports_test.go @@ -10,7 +10,6 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdeliverychecker" "git.novox.be/novox/mesh-sdk/go/process" ) @@ -363,11 +362,6 @@ func anUnreadPackagesStateWritesNoFingerprintAndFingerprintsAreComputedAgain(t * if now, _ := moduleOf(t, mods, "avahi"); now.Fp2 != good.Fp2 || ix.fresh != nil { t.Fatalf("avahi holds %q after it, want %q (fresh %v)", now.Fp2, good.Fp2, ix.fresh) } - // It runs a merge's recompute, candidates included: a candidate store that cannot be listed fails it, said. - ix.Store, ix.previews = &memStore{keysErr: errors.New("no bus")}, map[string]meshdeliverychecker.PreviewsByRepositoryHeadTrunkValue{"k": {}} - if r := ix.FingerprintsAgain(ctx, "avahi"); r.Stored || !strings.Contains(r.Answer, "listing the candidates") { - t.Fatalf("fingerprints again over candidates that cannot be listed: %+v", r) - } if r := ix.FingerprintsAgain(ctx, "nothing"); r.Stored { t.Fatalf("fingerprints again of a module not held answered stored: %+v", r) } diff --git a/modules/mesh-delivery-indexer/internal/index/index_test.go b/modules/mesh-delivery-indexer/internal/index/index_test.go index c69c7c21..6ea19e72 100644 --- a/modules/mesh-delivery-indexer/internal/index/index_test.go +++ b/modules/mesh-delivery-indexer/internal/index/index_test.go @@ -5,11 +5,6 @@ import ( "errors" "strings" "testing" - - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" - "git.novox.be/novox/mesh-sdk/go/keys" ) const ( @@ -121,11 +116,6 @@ func buildRecord(t *testing.T, repo, module, path, at string) []byte { return logged(t, "build-"+module, module, repo, path, "", "", at) } -func pull(repo string, number int, branch, head string, dirs ...string) forgemsg.Forge { - return forgemsg.Forge{V: keys.Version, Kind: forgemsg.ForgePull, Repository: repo, Branch: branch, Number: number, Head: head, - Action: forgemsg.PullUpdated, Dirs: dirs, DirsKnown: dirs != nil} -} - func newIndexer(t *testing.T) (*Indexer, *memStore, *strings.Builder) { t.Helper() st, logs := newMem(), &strings.Builder{} @@ -138,282 +128,3 @@ func newIndexer(t *testing.T) (*Indexer, *memStore, *strings.Builder) { _ = a }}, st, logs } - -func read(t *testing.T, st *memStore, key string) (meshcatalog.CandidatesByGroupValue, bool) { - t.Helper() - v, ok := st.vals[key] - if !ok { - return meshcatalog.CandidatesByGroupValue{}, false - } - c, err := DecodeCandidate(v) - if err != nil { - t.Fatal(err) - } - return c, true -} - -func TestTwoRepositoriesFormOneGroupByBranch(t *testing.T) { // 1238/AC2 (the grouping; the bus is the integration test) - ix, st, _ := newIndexer(t) - ctx := context.Background() - if err := ix.Apply(ctx, pull("novox/mesh-delivery", 12, "feat/x", h1, "modules/a")); err != nil { - t.Fatal(err) - } - if err := ix.Apply(ctx, pull("novox/mesh-catalog", 300, "feat/x", h2, "modules/c")); err != nil { - t.Fatal(err) - } - c, ok := read(t, st, "feat/x") - if !ok { - t.Fatal("no candidate under feat/x") - } - hash, _ := meshdeliveryclient.HeadsHash([]meshdeliveryclient.Head{{Repository: "novox/mesh-delivery", Number: 12, Head: h1}, {Repository: "novox/mesh-catalog", Number: 300, Head: h2}}) - if c.Group != "feat/x" || c.Base != 0 || c.Heads != hash || len(c.Pulls) != 2 { - t.Fatalf("candidate = %+v", c) - } - got := map[string][]string{} - for _, p := range c.Pulls { - got[p.Repository] = p.Modules - } - if strings.Join(got["novox/mesh-delivery"], ",") != "a" || strings.Join(got["novox/mesh-catalog"], ",") != "c" { - t.Errorf("modules = %v", got) - } -} - -func TestModuleOfADirectory(t *testing.T) { // 1238/AC3, as R4e has it: the module of a directory is the indexer's own (I2, ADR 0501 clause 6) - ix, st, _ := newIndexer(t) - mods := newMem() - ix.Modules = mods - ctx := context.Background() - // The trunk holds modules/odd as the module odd-name (its manifest names it so); modules/plain is held by none. - merged := mergedMsg("novox/mesh-delivery", map[string]string{"modules/odd": "t"}) - merged.Manifests["modules/odd"] = `{"module":"odd-name","version":"1.0"}` - if err := ix.Apply(ctx, merged); err != nil { - t.Fatal(err) - } - if err := ix.Apply(ctx, pull("novox/mesh-delivery", 12, "feat/x", h1, "modules/odd", "modules/plain", "modules/odd")); err != nil { - t.Fatal(err) - } - c, _ := read(t, st, "feat/x") - if strings.Join(c.Pulls[0].Modules, ",") != "odd-name,plain" { - t.Errorf("modules = %v, want the held module for modules/odd, the last element for modules/plain, no repeats", c.Pulls[0].Modules) - } - held := map[string]meshcatalog.ModulesByModuleValue{"x": {Repository: "novox/other", Path: "modules/a"}} - if name, ok := moduleAt(held, "novox/mesh-delivery", "modules/a"); ok { - t.Errorf("modules/a of novox/mesh-delivery is %q, held only of novox/other", name) - } - // A manifest the pull message carries names its module, whatever directory it is in. - f := pull("novox/mesh-delivery", 13, "feat/y", h2, "modules/odd") - f.Manifests = map[string]string{"modules/odd": `{"module":"renamed","version":"1.0"}`} - if name := moduleOfDir(held, f, "modules/odd"); name != "renamed" { - t.Errorf("modules/odd = %q, want the module its manifest names", name) - } -} -func TestDirsNotKnownKeepTheModulesOfTheSameHead(t *testing.T) { // 1238/AC4 - ix, st, _ := newIndexer(t) - ctx := context.Background() - _ = ix.Apply(ctx, pull("novox/mesh-delivery", 12, "feat/x", h1, "modules/a")) - unknown := pull("novox/mesh-delivery", 12, "feat/x", h1) - if err := ix.Apply(ctx, unknown); err != nil { - t.Fatal(err) - } - c, _ := read(t, st, "feat/x") - if strings.Join(c.Pulls[0].Modules, ",") != "a" { - t.Errorf("modules = %v, want a kept", c.Pulls[0].Modules) - } - // A new head with directories not known is listed with no modules. - unknown.Head = h3 - if err := ix.Apply(ctx, unknown); err != nil { - t.Fatal(err) - } - c, _ = read(t, st, "feat/x") - if c.Pulls[0].Head != h3 || len(c.Pulls[0].Modules) != 0 { - t.Errorf("pull = %+v, want the new head with no modules", c.Pulls[0]) - } - // Known and empty is no modules, not unknown. - known := pull("novox/mesh-delivery", 12, "feat/x", h3) - known.Dirs, known.DirsKnown = nil, true - if err := ix.Apply(ctx, known); err != nil { - t.Fatal(err) - } -} - -func TestClosedAndMergedTakeAPullOutAndTheLastDeletesTheKey(t *testing.T) { // 1238/AC5 - for _, how := range []string{"closed", "merged"} { - ix, st, _ := newIndexer(t) - ctx := context.Background() - _ = ix.Apply(ctx, pull("novox/mesh-delivery", 12, "feat/x", h1, "modules/a")) - _ = ix.Apply(ctx, pull("novox/mesh-catalog", 300, "feat/x", h2, "modules/c")) - gone := func(repo string, n int, head string) forgemsg.Forge { - if how == "closed" { - f := pull(repo, n, "feat/x", head) - f.Action = forgemsg.PullClosed - return f - } - return forgemsg.Forge{V: keys.Version, Kind: forgemsg.ForgeMerged, Repository: repo, Branch: "feat/x", Number: n, Head: head, Merge: h3, Root: "r", Trees: map[string]string{"a": "t"}} - } - if err := ix.Apply(ctx, gone("novox/mesh-delivery", 12, h1)); err != nil { - t.Fatal(err) - } - c, ok := read(t, st, "feat/x") - if !ok || len(c.Pulls) != 1 || c.Pulls[0].Repository != "novox/mesh-catalog" { - t.Fatalf("%s: candidate = %+v (kept %v)", how, c, ok) - } - hash, _ := meshdeliveryclient.HeadsHash([]meshdeliveryclient.Head{headOf(c.Pulls[0])}) - if c.Heads != hash { - t.Errorf("%s: heads hash %q is not the remaining pull request's %q", how, c.Heads, hash) - } - if err := ix.Apply(ctx, gone("novox/mesh-catalog", 300, h2)); err != nil { - t.Fatal(err) - } - if _, ok := read(t, st, "feat/x"); ok { - t.Errorf("%s: the last pull request left and the key is still there", how) - } - // A pull request the group does not hold changes nothing and writes nothing. - writes := st.writes - if err := ix.Apply(ctx, gone("novox/mesh-catalog", 300, h2)); err != nil || st.writes != writes { - t.Errorf("%s: closing an unheld pull request wrote (%d writes, err %v)", how, st.writes-writes, err) - } - } -} - -func TestTrunkLikeBranchesMakeNoCandidate(t *testing.T) { // 1238/AC6 - for _, branch := range []string{"main", "master", "trunk", ""} { - ix, st, logs := newIndexer(t) - if err := ix.Apply(context.Background(), pull("novox/mesh-delivery", 12, branch, h1, "modules/a")); err != nil { - t.Fatal(err) - } - if len(st.vals) != 0 || st.writes != 0 { - t.Errorf("branch %q: %d keys written", branch, len(st.vals)) - } - if !strings.Contains(logs.String(), "no delivery group") { - t.Errorf("branch %q: log %q does not say why", branch, logs.String()) - } - } -} - -func TestSameMessageWritesNoNewRevision(t *testing.T) { // 1238/AC7 - ix, st, _ := newIndexer(t) - ctx := context.Background() - m := pull("novox/mesh-delivery", 12, "feat/x", h1, "modules/a") - _ = ix.Apply(ctx, m) - rev, writes := st.revs["feat/x"], st.writes - for i := 0; i < 3; i++ { - if err := ix.Apply(ctx, m); err != nil { - t.Fatal(err) - } - } - if st.revs["feat/x"] != rev || st.writes != writes { - t.Errorf("the same message wrote again: revision %d to %d", rev, st.revs["feat/x"]) - } -} - -func TestALonePullRequestIsAGroupOfOne(t *testing.T) { // 1238/AC8 - ix, st, _ := newIndexer(t) - if err := ix.Apply(context.Background(), pull("novox/mesh-delivery", 12, "fix/lone", h1, "modules/a")); err != nil { - t.Fatal(err) - } - c, ok := read(t, st, "fix/lone") - if !ok || c.Group != "fix/lone" || len(c.Pulls) != 1 { - t.Errorf("candidate = %+v (%v)", c, ok) - } -} - -func TestAConflictReadsAgain(t *testing.T) { // 1238 (compare-and-set: Update with the revision read, read again on a conflict) - ix, st, _ := newIndexer(t) - ctx := context.Background() - _ = ix.Apply(ctx, pull("novox/mesh-delivery", 12, "feat/x", h1, "modules/a")) - // Between the indexer's read and its write another writer adds #300 to the group. - st.beforeWrite = func() { - other := meshcatalog.CandidatesByGroupValue{V: keys.Version, Group: "feat/x", Pulls: []meshcatalog.CandidatesByGroupValuePullsItem{ - {Repository: "novox/mesh-delivery", Number: 12, Head: h1, Modules: []string{"a"}}, - {Repository: "novox/mesh-catalog", Number: 300, Head: h2, Modules: []string{"c"}}}} - other.Heads, _ = meshdeliveryclient.HeadsHash([]meshdeliveryclient.Head{headOf(other.Pulls[0]), headOf(other.Pulls[1])}) - data, _ := EncodeCandidate(other) - st.put("feat/x", data) - } - if err := ix.Apply(ctx, pull("novox/mesh-delivery", 12, "feat/x", h3, "modules/a")); err != nil { - t.Fatal(err) - } - c, _ := read(t, st, "feat/x") - if len(c.Pulls) != 2 { - t.Fatalf("pulls = %+v, want #12 moved and #300 kept (the other writer's)", c.Pulls) - } - for _, p := range c.Pulls { - if p.Number == 12 && p.Head != h3 { - t.Errorf("#12 head = %s", p.Head) - } - } -} - -// The SDK escapes a branch name's odd characters into a key (mesh-sdk a6dd698), so a name such as "feat x?" keys and is -// indexed; no branch name is unreadable any more. -func TestAnOddBranchNameKeysEscaped(t *testing.T) { - ix, st, _ := newIndexer(t) - if err := ix.Apply(context.Background(), pull("novox/mesh-delivery", 12, "feat x?", h1, "modules/a")); err != nil { - t.Fatal(err) - } - key, err := CandidatesKey("feat x?") - if err != nil { - t.Fatal(err) - } - if _, ok := st.vals[key]; !ok { - t.Errorf("no candidate under %q", key) - } -} - -func open(pulls ...forgemsg.OpenPull) forgemsg.Forge { - return forgemsg.Forge{V: keys.Version, Kind: forgemsg.ForgeOpen, Open: pulls} -} - -// 1279/AC7 (ADR 0443 §5): on an open message, candidates lose every pull request it does not name, a group left empty is -// deleted, and a pull request added by a later message stays. -func TestAnOpenMessageDropsEveryCandidateItDoesNotName(t *testing.T) { // 1279/AC7 - ix, st, _ := newIndexer(t) - ctx := context.Background() - for _, f := range []forgemsg.Forge{ - pull("novox/mesh-delivery", 12, "feat/x", h1, "modules/a"), pull("novox/mesh-catalog", 300, "feat/x", h2, "modules/c"), - pull("novox/mesh-delivery", 13, "feat/y", h2, "modules/a"), - pull("novox/mesh-catalog", 280, "feat/old", h3, "modules/c"), - } { - if err := ix.Apply(ctx, f); err != nil { - t.Fatal(err) - } - } - before, _ := read(t, st, "feat/x") - - // Open: mesh-delivery#12 only (and a pull request no group holds, which changes nothing). - if err := ix.Apply(ctx, open(forgemsg.OpenPull{Repository: "novox/mesh-delivery", Number: 12}, forgemsg.OpenPull{Repository: "novox/mesh-catalog", Number: 999})); err != nil { - t.Fatal(err) - } - c, ok := read(t, st, "feat/x") - hash, _ := meshdeliveryclient.HeadsHash([]meshdeliveryclient.Head{{Repository: "novox/mesh-delivery", Number: 12, Head: h1}}) - if !ok || len(c.Pulls) != 1 || c.Pulls[0].Number != 12 || c.Pulls[0].Head != h1 || strings.Join(c.Pulls[0].Modules, ",") != "a" || c.Heads != hash || c.Heads == before.Heads { - t.Fatalf("feat/x after the open message = %+v, want only mesh-delivery#12 with its modules and the heads hash of it alone", c) - } - for _, gone := range []string{"feat/y", "feat/old"} { - if _, ok := st.vals[gone]; ok { - t.Errorf("the group %s is left with no open pull request and was not deleted", gone) - } - } - - // A pull request opened after the read is heard as a message after the open message and stays. - if err := ix.Apply(ctx, pull("novox/mesh-catalog", 300, "feat/x", h2, "modules/c")); err != nil { - t.Fatal(err) - } - if c, _ := read(t, st, "feat/x"); len(c.Pulls) != 2 { - t.Errorf("a pull request added after the open message is not kept: %+v", c) - } - // The same open message again changes nothing (no write), and an empty one drops every group. - writes := st.writes - if err := ix.Apply(ctx, open(forgemsg.OpenPull{Repository: "novox/mesh-delivery", Number: 12}, forgemsg.OpenPull{Repository: "novox/mesh-catalog", Number: 300})); err != nil { - t.Fatal(err) - } - if st.writes != writes { - t.Errorf("an open message that names every candidate wrote %d times", st.writes-writes) - } - if err := ix.Apply(ctx, open()); err != nil { - t.Fatal(err) - } - if len(st.vals) != 0 { - t.Errorf("after an empty open message the candidates hold %v", st.vals) - } -} diff --git a/modules/mesh-delivery-indexer/internal/index/manifests.go b/modules/mesh-delivery-indexer/internal/index/manifests.go index f6d5d4ad..2a7e2964 100644 --- a/modules/mesh-delivery-indexer/internal/index/manifests.go +++ b/modules/mesh-delivery-indexer/internal/index/manifests.go @@ -7,7 +7,6 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "git.novox.be/novox/mesh-sdk/go/keys" ) @@ -51,7 +50,7 @@ func (ix *Indexer) keepManifest(ctx context.Context, module, raw string) (string // keepManifests writes the manifest of each directory a message carries, and answers the digest of each by directory. // A manifest that cannot be kept is left out and said in the log, so the module reads as one with no manifest. -func (ix *Indexer) keepManifests(ctx context.Context, f forgemsg.Forge) (map[string]string, error) { +func (ix *Indexer) keepManifests(ctx context.Context, f whole) (map[string]string, error) { digests := make(map[string]string, len(f.Manifests)) for dir, raw := range f.Manifests { module := manifestModule(raw) // the name the manifest gives itself: one digest always has one module @@ -60,7 +59,7 @@ func (ix *Indexer) keepManifests(ctx context.Context, f forgemsg.Forge) (map[str case err == nil: digests[dir] = digest case errors.Is(err, errCannotBeAValue): - ix.logf("%s#%d: the manifest of %q cannot be kept: %v", f.Repository, f.Number, dir, err) + ix.logf("%s@%s: the manifest of %q cannot be kept: %v", f.Repository, f.Merge, dir, err) default: return nil, err } diff --git a/modules/mesh-delivery-indexer/internal/index/manifests_test.go b/modules/mesh-delivery-indexer/internal/index/manifests_test.go index 4e64d16f..f2c92a6f 100644 --- a/modules/mesh-delivery-indexer/internal/index/manifests_test.go +++ b/modules/mesh-delivery-indexer/internal/index/manifests_test.go @@ -6,8 +6,6 @@ import ( "testing" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" - "git.novox.be/novox/mesh-sdk/go/keys" ) // The manifests these tests carry are real files, copied unchanged into testdata: the module.json of modules of @@ -39,7 +37,7 @@ func manifestsIndexer(t *testing.T, recs ...[2]any) (*Indexer, *memStore, *memSt return ix, mods, manifests } -func withManifests(f forgemsg.Forge, manifests map[string]string, locks map[string]map[string]string) forgemsg.Forge { +func withManifests(f whole, manifests map[string]string, locks map[string]map[string]string) whole { f.Manifests, f.Locks = manifests, locks return f } @@ -78,16 +76,11 @@ func sameManifestAppliedAgainIsWrittenOnce(t *testing.T) { // 1449/AC2 if err := ix.Apply(context.Background(), f); err != nil { t.Fatal(err) } - // A second merge, and a pull message carrying it, the digest being the same. + // A second merge, the digest being the same. f.Merge = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" if err := ix.Apply(context.Background(), f); err != nil { t.Errorf("a second merge: %v", err) } - pull := forgemsg.Forge{V: keys.Version, Kind: forgemsg.ForgePull, Repository: catalogueRepo, Branch: "feat/x", Number: 301, Head: h1, - Action: forgemsg.PullUpdated, Dirs: []string{"modules/avahi"}, DirsKnown: true, Manifests: map[string]string{"modules/avahi": avahi}} - if err := ix.Apply(context.Background(), pull); err != nil { - t.Errorf("a pull message: %v", err) - } if manifests.writes != 1 || len(manifests.vals) != 1 { t.Errorf("manifests written %d times into %d keys, want once into one", manifests.writes, len(manifests.vals)) } diff --git a/modules/mesh-delivery-indexer/internal/index/modules.go b/modules/mesh-delivery-indexer/internal/index/modules.go index ec1f9762..ee7ece61 100644 --- a/modules/mesh-delivery-indexer/internal/index/modules.go +++ b/modules/mesh-delivery-indexer/internal/index/modules.go @@ -9,7 +9,6 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/keys" @@ -35,20 +34,20 @@ import ( // // A merge never moves a module backwards (order.go): a merged message older than the repository's taken merge writes and // deletes nothing, and is said; it is asked once, before anything is written. -func (ix *Indexer) applyMerged(ctx context.Context, f forgemsg.Forge) (*mergeWrite, error) { +func (ix *Indexer) applyMerged(ctx context.Context, f whole) (*mergeWrite, error) { return ix.applyManifests(ctx, f, true) } // applyManifests is applyMerged; drop says whether the modules of the repository the message holds no manifest for are // deleted. A message that names only some of a repository's modules (a registered module read on its own, registered.go) // drops none. -func (ix *Indexer) applyManifests(ctx context.Context, f forgemsg.Forge, drop bool) (*mergeWrite, error) { +func (ix *Indexer) applyManifests(ctx context.Context, f whole, drop bool) (*mergeWrite, error) { return ix.applyMoving(ctx, f, drop, nil) } // applyMoving is applyManifests; moves names the modules keys the message may take from the repository that holds them // (a move handed over, moves.go). Every other module held of another repository is left, said and kept waiting. -func (ix *Indexer) applyMoving(ctx context.Context, f forgemsg.Forge, drop bool, moves map[string]bool) (*mergeWrite, error) { +func (ix *Indexer) applyMoving(ctx context.Context, f whole, drop bool, moves map[string]bool) (*mergeWrite, error) { w := newMergeWrite(f) if drop { if err := ix.lateMerge(ctx, w); err != nil || w.late != "" { @@ -190,7 +189,7 @@ func withV(m meshcatalog.ModulesByModuleValue) meshcatalog.ModulesByModuleValue // namedObjects are, by module directory, the git objects of the paths each manifest of the merge names for every artifact // (catalogue Manifest.NamedPaths): those the message carries, else asked of the forge at the merge commit in one call // (Named). A forge that cannot say gives, for each directory asked, its words, so the module's reason names them. -func (ix *Indexer) namedObjects(ctx context.Context, f forgemsg.Forge) (map[string]map[string]string, map[string]string) { +func (ix *Indexer) namedObjects(ctx context.Context, f whole) (map[string]map[string]string, map[string]string) { objects := map[string]map[string]string{} need := map[string][]string{} for dir, raw := range f.Manifests { @@ -230,7 +229,7 @@ func (ix *Indexer) namedObjects(ctx context.Context, f forgemsg.Forge) (map[stri // held(key, module) says whether the trunk holds it. A module of another repository, or of none known, is never touched. func (ix *Indexer) dropRemoved(ctx context.Context, repository string, held map[string]meshcatalog.ModulesByModuleValue, trunkHolds func(key string, m meshcatalog.ModulesByModuleValue) bool) error { var handed []string - elsewhere := map[string]forgemsg.Forge{} + elsewhere := map[string]whole{} for _, key := range slices.Sorted(maps.Keys(held)) { cur := held[key] if cur.Repository == "" || !trunk.SameRepository(cur.Repository, repository) || trunkHolds(key, cur) { @@ -372,7 +371,7 @@ func unreadInputs(tree string, locks, paths map[string]string, packages []string // goSumPackages are the version line's packages of the merge's directory dir (ADR 0573 §2, ADR 0593 §2): the mesh Go // modules of other repositories its go.sum names, and those of its own repository its go.mod replaces by a directory // (siblingPackages), which no go.sum line names. A directory with no go.sum is no Go module and has none. -func (ix *Indexer) goSumPackages(ctx context.Context, f forgemsg.Forge, dir string, cur meshcatalog.ModulesByModuleValue, has bool) []string { +func (ix *Indexer) goSumPackages(ctx context.Context, f whole, dir string, cur meshcatalog.ModulesByModuleValue, has bool) []string { others := ix.otherPackages(ctx, f, dir, cur, has) if f.Locks[dir]["go.sum"] == "" && len(f.Packages[dir]) == 0 { return others @@ -386,7 +385,7 @@ func (ix *Indexer) goSumPackages(ctx context.Context, f forgemsg.Forge, dir stri // siblingPackages are the mesh Go modules of the merge's own repository the go.mod of dir replaces by a directory, read // from the forge (GoMod); where it cannot be read, the held module's own-repository packages stand, said. -func (ix *Indexer) siblingPackages(ctx context.Context, f forgemsg.Forge, dir string, cur meshcatalog.ModulesByModuleValue, has bool) []string { +func (ix *Indexer) siblingPackages(ctx context.Context, f whole, dir string, cur meshcatalog.ModulesByModuleValue, has bool) []string { held := func() []string { if !has || cur.Inputs2 == nil { return nil @@ -413,7 +412,7 @@ func (ix *Indexer) siblingPackages(ctx context.Context, f forgemsg.Forge, dir st // otherPackages are the mesh Go modules of other repositories the go.sum of the merge's directory dir names: those the // message carries (an older bridge), else the held module's while its go.sum's hash is unchanged, else the go.sum read // from the forge (GoSum). -func (ix *Indexer) otherPackages(ctx context.Context, f forgemsg.Forge, dir string, cur meshcatalog.ModulesByModuleValue, has bool) []string { +func (ix *Indexer) otherPackages(ctx context.Context, f whole, dir string, cur meshcatalog.ModulesByModuleValue, has bool) []string { if carried := f.Packages[dir]; len(carried) > 0 { return fingerprint.OtherPackages(carried, f.Repository) // its own repository's come with its tree } diff --git a/modules/mesh-delivery-indexer/internal/index/modules_test.go b/modules/mesh-delivery-indexer/internal/index/modules_test.go index b1e865bf..3d4ba64d 100644 --- a/modules/mesh-delivery-indexer/internal/index/modules_test.go +++ b/modules/mesh-delivery-indexer/internal/index/modules_test.go @@ -15,10 +15,8 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk" "git.novox.be/novox/mesh-sdk/go/catalogue" - "git.novox.be/novox/mesh-sdk/go/keys" "git.novox.be/novox/mesh-sdk/go/process" ) @@ -87,12 +85,12 @@ func rec(t *testing.T, id, module, repo, path, ref, state, at, inputs string) [2 // mergedMsg is a merged message of the repository with these module directories' trees, each with a manifest naming its // module, the directory's last element ("r" for the root), as the forge bridge sends it (one manifest per directory that // holds a module.json). -func mergedMsg(repo string, trees map[string]string) forgemsg.Forge { +func mergedMsg(repo string, trees map[string]string) whole { manifests := map[string]string{} for dir := range trees { manifests[dir] = `{"module":"` + nameOfDir(dir) + `","version":"1.0"}` } - return forgemsg.Forge{V: keys.Version, Kind: forgemsg.ForgeMerged, Repository: repo, Branch: "feat/x", Number: 12, Head: h1, Merge: merge, Root: "r0", Trees: trees, Manifests: manifests} + return whole{Repository: repo, Branch: "feat/x", Head: h1, Merge: merge, Root: "r0", Trees: trees, Manifests: manifests} } func nameOfDir(dir string) string { @@ -138,7 +136,7 @@ func TestAnUnchangedModuleWritesNothing(t *testing.T) { // 1239 (unchanged value {name: "the same merge again", writes: 0, commit: map[string]string{"a": merge, "b": merge, "c": merge}, taken: merge, then: func(ix *Indexer) error { return ix.Apply(context.Background(), mergedMsg(repo, three)) }}, {name: "a merge touching one of three modules", writes: 1, commit: map[string]string{"a": next, "b": merge, "c": merge}, taken: next, - said: "novox/mesh-delivery at cccccccccccc (#12): merged message applied; modules written: a", + said: "novox/mesh-delivery at cccccccccccc: taken in; modules written: a", then: func(ix *Indexer) error { m := mergedMsg(repo, map[string]string{"modules/a": "treeA2", "modules/b": "treeB", "modules/c": "treeC"}) m.Merge = next @@ -157,7 +155,7 @@ func TestAnUnchangedModuleWritesNothing(t *testing.T) { // 1239 (unchanged value m.Merge = next return ix.Apply(context.Background(), m) }}, - {name: "a merge heard while the taken state was missing is filled when the state appears", writes: 2, commit: map[string]string{"a": next, "b": merge, "c": merge}, taken: next, + {name: "a merge heard while the taken state was missing is filled when the state appears", writes: 1, commit: map[string]string{"a": next, "b": merge, "c": merge}, taken: next, then: func(ix *Indexer) error { // issue 1280: the key was missing, the merge was taken in without it on := ix.Taken.(*memStore) key, _ := trunk.RepositoryKey(repo) @@ -364,16 +362,6 @@ func TestAnUnchangedModuleWritesNothing(t *testing.T) { // 1239 (unchanged value } } -func TestMergedMessageRemovesThePullFromItsGroupToo(t *testing.T) { // 1238/AC5 with 1239 (both effects of one message) - ix, _ := modulesIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", aInputs)) - cands := ix.Store.(*memStore) - _ = ix.Apply(context.Background(), pull("novox/mesh-delivery", 12, "feat/x", h1, "modules/a")) - _ = ix.Apply(context.Background(), mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"})) - if _, ok := cands.vals["feat/x"]; ok { - t.Error("the merged pull request is still a candidate") - } -} - func TestAMergeWritesTheModulesDirectoryAndKeepsItThroughSourceAndRecompute(t *testing.T) { // 1606/AC1, AC2 ix, mods := modulesIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", aInputs), @@ -393,8 +381,8 @@ func TestAMergeWritesTheModulesDirectoryAndKeepsItThroughSourceAndRecompute(t *t if err := ix.Recompute(ctx); err != nil { t.Fatal(err) } - if m, _ := moduleOf(t, mods, "a"); m.Path != "modules/a" || m.Src != 7 { - t.Errorf("after the source and a recompute a = %+v, want path modules/a kept and Src 7", m) + if m, _ := moduleOf(t, mods, "a"); m.Path != "modules/a" { + t.Errorf("after the source and a recompute a = %+v, want path modules/a kept", m) } raw := string(mods.vals["a"]) if !strings.Contains(raw, `"path":"modules/a"`) { @@ -475,7 +463,7 @@ func TestAModuleMovesBetweenRepositoriesOnlyWhenItsTrunkDropsIt(t *testing.T) { gone.Trees, gone.Manifests = nil, nil for _, tc := range []struct { name string - merges []forgemsg.Forge + merges []whole repository string // modules.a's repository at the end, "" for none deletedKeyA bool // Issue 1367: a fresh indexer, no wait in memory, the source naming heads; trunk is the module directories the forge @@ -488,20 +476,20 @@ func TestAModuleMovesBetweenRepositoriesOnlyWhenItsTrunkDropsIt(t *testing.T) { stored bool bGoneAtEnd bool }{ - {name: "named while its trunk holds it: left", merges: []forgemsg.Forge{held, named}, repository: from}, - {name: "named, then dropped by its trunk: handed over in place", merges: []forgemsg.Forge{held, named, dropped}, repository: to}, - {name: "dropped by its trunk, then named: deleted, then written new", merges: []forgemsg.Forge{held, dropped, named}, repository: to, deletedKeyA: true}, - {name: "dropped by its trunk with no merge naming it: deleted", merges: []forgemsg.Forge{held, dropped}, deletedKeyA: true}, + {name: "named while its trunk holds it: left", merges: []whole{held, named}, repository: from}, + {name: "named, then dropped by its trunk: handed over in place", merges: []whole{held, named, dropped}, repository: to}, + {name: "dropped by its trunk, then named: deleted, then written new", merges: []whole{held, dropped, named}, repository: to, deletedKeyA: true}, + {name: "dropped by its trunk with no merge naming it: deleted", merges: []whole{held, dropped}, deletedKeyA: true}, {name: "no module.json left, another trunk holds it, nothing waits in memory: written from that trunk, never deleted", - merges: []forgemsg.Forge{held, gone}, heads: true, trunk: []string{"modules/a", "modules/other"}, repository: to, bGoneAtEnd: true}, + merges: []whole{held, gone}, heads: true, trunk: []string{"modules/a", "modules/other"}, repository: to, bGoneAtEnd: true}, {name: "no module.json left, no other trunk holds it: deleted", - merges: []forgemsg.Forge{held, gone}, heads: true, deletedKeyA: true, bGoneAtEnd: true}, + merges: []whole{held, gone}, heads: true, deletedKeyA: true, bGoneAtEnd: true}, {name: "taken in by repository, a head with no module.json: handed over from the other trunk", - merges: []forgemsg.Forge{held}, heads: true, trunk: []string{"modules/a"}, takeIn: from, answer: "handed over", stored: true, repository: to, bGoneAtEnd: true}, + merges: []whole{held}, heads: true, trunk: []string{"modules/a"}, takeIn: from, answer: "handed over", stored: true, repository: to, bGoneAtEnd: true}, {name: "taken in by repository, a head with modules: answered by name", - merges: []forgemsg.Forge{held}, heads: true, trunk: []string{"modules/a"}, takeIn: to, answer: "by name", repository: from}, + merges: []whole{held}, heads: true, trunk: []string{"modules/a"}, takeIn: to, answer: "by name", repository: from}, {name: "taken in by repository, no head named: answered so", - merges: []forgemsg.Forge{held}, heads: true, takeIn: "novox/nobody", answer: "names no head", repository: from}, + merges: []whole{held}, heads: true, takeIn: "novox/nobody", answer: "names no head", repository: from}, } { t.Run(tc.name, func(t *testing.T) { mods := &deletesSeen{memStore: newMem()} diff --git a/modules/mesh-delivery-indexer/internal/index/moves.go b/modules/mesh-delivery-indexer/internal/index/moves.go index 9419dd1e..03c7de2b 100644 --- a/modules/mesh-delivery-indexer/internal/index/moves.go +++ b/modules/mesh-delivery-indexer/internal/index/moves.go @@ -7,18 +7,16 @@ import ( "slices" "strings" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk" - "git.novox.be/novox/mesh-sdk/go/keys" ) // heldByAnotherTrunk reads, for a module the trunk of repository no longer holds, whether the trunk of another repository // holds it at the head the latest source names: the module directories there are asked of the forge (Trunk) and the manifest of // each directory named like the module is read (Read). It answers the merge to hand the module over with, or none: a forge // that cannot say leaves the module to be dropped as it was, said (issue 1367). -func (ix *Indexer) heldByAnotherTrunk(ctx context.Context, module, repository string) (forgemsg.Forge, bool) { +func (ix *Indexer) heldByAnotherTrunk(ctx context.Context, module, repository string) (whole, bool) { if ix.Trunk == nil || ix.Read == nil { - return forgemsg.Forge{}, false + return whole{}, false } ix.mu.Lock() heads := maps.Clone(ix.source.heads) @@ -47,7 +45,7 @@ func (ix *Indexer) heldByAnotherTrunk(ctx context.Context, module, repository st if read.Trees[dir] == "" || manifestModule(raw) != module { continue } - f := forgemsg.Forge{V: keys.Version, Kind: forgemsg.ForgeMerged, Repository: other, Merge: head, Root: read.Root, + f := whole{Repository: other, Merge: head, Root: read.Root, Trees: map[string]string{dir: read.Trees[dir]}, Manifests: map[string]string{dir: raw}} if l := read.Locks[dir]; len(l) > 0 { f.Locks = map[string]map[string]string{dir: l} @@ -59,7 +57,7 @@ func (ix *Indexer) heldByAnotherTrunk(ctx context.Context, module, repository st return f, true } } - return forgemsg.Forge{}, false + return whole{}, false } // nameOfDirectory is the last element of a module directory. @@ -78,8 +76,8 @@ func nameOfDirectory(dir string) string { return dir[strings.LastIndex(dir, "/") // waitToMove keeps, by modules key, the part of a merged message naming one module directory the indexer holds of another // repository. A later merge of the same repository replaces it. -func (ix *Indexer) waitToMove(key string, f forgemsg.Forge, dir string) { - one := forgemsg.Forge{V: f.V, Kind: f.Kind, Repository: f.Repository, Branch: f.Branch, Number: f.Number, Merge: f.Merge, Root: f.Root, +func (ix *Indexer) waitToMove(key string, f whole, dir string) { + one := whole{Repository: f.Repository, Branch: f.Branch, Merge: f.Merge, Head: f.Head, Root: f.Root, Trees: map[string]string{dir: f.Trees[dir]}, Manifests: map[string]string{dir: f.Manifests[dir]}} if l := f.Locks[dir]; len(l) > 0 { one.Locks = map[string]map[string]string{dir: l} @@ -93,7 +91,7 @@ func (ix *Indexer) waitToMove(key string, f forgemsg.Forge, dir string) { ix.mu.Lock() defer ix.mu.Unlock() if ix.moving == nil { - ix.moving = map[string]forgemsg.Forge{} + ix.moving = map[string]whole{} } ix.moving[key] = one } @@ -119,7 +117,7 @@ func (ix *Indexer) handOver(ctx context.Context, key string) error { } // handOverFrom writes the module at key from the merge f, which names it, in place. -func (ix *Indexer) handOverFrom(ctx context.Context, key string, f forgemsg.Forge) error { +func (ix *Indexer) handOverFrom(ctx context.Context, key string, f whole) error { if _, err := ix.applyMoving(ctx, f, false, map[string]bool{key: true}); err != nil { return fmt.Errorf("modules.%s, moving to %s: %w", key, f.Repository, err) } diff --git a/modules/mesh-delivery-indexer/internal/index/named_test.go b/modules/mesh-delivery-indexer/internal/index/named_test.go index a0313027..cfc78a1f 100644 --- a/modules/mesh-delivery-indexer/internal/index/named_test.go +++ b/modules/mesh-delivery-indexer/internal/index/named_test.go @@ -8,7 +8,6 @@ import ( "testing" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdeliverychecker" "mesh-delivery-indexer/internal/fingerprint" ) @@ -72,28 +71,6 @@ func TestAModuleNamingNoPathsKeepsItsDirectorysTree(t *testing.T) { // I7, ADR 0 } } -func TestACandidatesModuleTakesTheNamedPathsObjectsFromThePreview(t *testing.T) { // I8, ADR 0499 clause 2 (the merge preview) - ix, _, _ := manifestsIndexer(t) - raw := fixture(t, "sdk-9cb4d59-mesh-sdk") - digest, err := ix.keepManifest(context.Background(), "mesh-sdk", raw) - if err != nil { - t.Fatal(err) - } - named, _ := fingerprint.NamedTree(digest, sdkObjects) - with := meshdeliverychecker.PreviewsByRepositoryHeadTrunkValue{Modules: map[string]meshdeliverychecker.PreviewsByRepositoryHeadTrunkValueModulesValue{".": {Tree: named, Manifest: digest, Paths: sdkObjects}}} - without := meshdeliverychecker.PreviewsByRepositoryHeadTrunkValue{Modules: map[string]meshdeliverychecker.PreviewsByRepositoryHeadTrunkValueModulesValue{".": {Tree: "roottree", Manifest: digest}}} - if got := ix.candidateModule(context.Background(), "mesh-sdk", ".", without, nil, nil); !strings.Contains(got.None2, "named paths") { - t.Errorf("without the objects: %+v, want none naming the named paths", got) - } - got := ix.candidateModule(context.Background(), "mesh-sdk", ".", with, nil, nil) - if strings.Contains(got.None2, "named paths") || got.Manifest != digest { - t.Errorf("with the objects: %+v, want them taken (none only for what else is missing)", got) - } - if got.Fp2 != "" && !reflect.DeepEqual(got.Paths, sdkObjects) { - t.Errorf("with a fingerprint: %+v, want the named paths' objects carried for the builder's fetch", got) - } -} - func TestTheIndexerAsksTheForgeForTheNamedPathsObjectsOfAMerge(t *testing.T) { // I6 as the lead has it: the indexer is the one reader of paths ix, mods, _ := manifestsIndexer(t) type ask struct { diff --git a/modules/mesh-delivery-indexer/internal/index/names.go b/modules/mesh-delivery-indexer/internal/index/names.go index ddcdf3cd..ad02d983 100644 --- a/modules/mesh-delivery-indexer/internal/index/names.go +++ b/modules/mesh-delivery-indexer/internal/index/names.go @@ -15,11 +15,8 @@ const ( PackagesHistory = 5 TakenState = "taken" PredefinedListKey = "list" - ForgeBridgeModule = "mesh-delivery-forge-bridge" GateModule = "mesh-delivery-gate" CheckerModule = "mesh-delivery-checker" - PreviewsBucket = CheckerModule + "_previews" - PreviewsHistory = 5 toolchainsState = "toolchains" ) diff --git a/modules/mesh-delivery-indexer/internal/index/open_asks.go b/modules/mesh-delivery-indexer/internal/index/open_asks.go index 684e9e33..937a749e 100644 --- a/modules/mesh-delivery-indexer/internal/index/open_asks.go +++ b/modules/mesh-delivery-indexer/internal/index/open_asks.go @@ -13,37 +13,6 @@ import ( // what the ask served no longer stands, so the call stopped and is not said unanswered. var ErrAskClosed = errors.New("the ask is closed: what it served no longer stands") -// takingIn marks a context as the take-in of one commit's merged message, which is pending while that context lives. -type takingIn struct{ repository, merge string } - -type takingInKey struct{} - -// forTakeIn is ctx marked as the take-in of the merged message of repository's commit merge. -func forTakeIn(ctx context.Context, repository, merge string) context.Context { - return context.WithValue(ctx, takingInKey{}, takingIn{repository, merge}) -} - -// TakeInPending says the take-in of the commit merge of repository is still pending (A-190, for the forge bridge's -// merged_message): its take-in is the one ctx is made for, or a pending entry names it. -func (ix *Indexer) TakeInPending(ctx context.Context, repository, merge string) (bool, error) { - if ctx.Err() != nil { - return false, nil - } - if t, ok := ctx.Value(takingInKey{}).(takingIn); ok && t.repository == repository && t.merge == merge { - return true, nil - } - if ix.Pending == nil { - return false, nil - } - entries, err := ReadPending(ctx, ix.Pending, ix.logf) - if err != nil { - return false, err - } - return slices.ContainsFunc(entries, func(e PendingEntry) bool { - return e.Forge.Merge == merge && e.Forge.Repository == repository - }), nil -} - // SeedEntryHeld says the newest trunk revision still holds the entry of module at the fingerprint fp, the candidate the // seed asks steps for (A-190). func (ix *Indexer) SeedEntryHeld(ctx context.Context, module, fp string) (bool, error) { diff --git a/modules/mesh-delivery-indexer/internal/index/open_asks_test.go b/modules/mesh-delivery-indexer/internal/index/open_asks_test.go index 19fa1221..ac954677 100644 --- a/modules/mesh-delivery-indexer/internal/index/open_asks_test.go +++ b/modules/mesh-delivery-indexer/internal/index/open_asks_test.go @@ -13,35 +13,9 @@ import ( "git.novox.be/novox/mesh-sdk/go/seatdef" ) -// A-190: a kept call is asked again only while what it serves still stands; each of the indexer's four has its predicate. +// A-190: a kept call is asked again only while what it serves still stands; each of the indexer's three has its predicate. func TestKeptCallsOpenWhileTheirAskServesSomething(t *testing.T) { ctx := context.Background() - t.Run("the take-in of a commit is pending while an entry names it or the take-in is made", func(t *testing.T) { - pending := newMem() - val, _ := encodePending(Pending{Seq: 1, Ref: &PendingRef{Repository: "novox/r", Merge: "c1"}}) - if err := pending.Create(ctx, "m.1", val); err != nil { - t.Fatal(err) - } - ix := &Indexer{Module: testSelf, Pending: pending} - cancelled, cancel := context.WithCancel(ctx) - cancel() - for _, c := range []struct { - name string - ctx context.Context - repo, commit string - want bool - }{ - {"held", ctx, "novox/r", "c1", true}, - {"taken in or dropped", ctx, "novox/r", "c2", false}, - {"in hand", forTakeIn(ctx, "novox/r", "c2"), "novox/r", "c2", true}, - {"in hand for another commit", forTakeIn(ctx, "novox/r", "c3"), "novox/r", "c2", false}, - {"the take-in's context ended", cancelled, "novox/r", "c1", false}, - } { - if got, err := ix.TakeInPending(c.ctx, c.repo, c.commit); err != nil || got != c.want { - t.Errorf("%s: pending = %v, %v, want %v", c.name, got, err, c.want) - } - } - }) t.Run("the seed's candidate and the versions' module are in the newest trunk revision", func(t *testing.T) { ix := &Indexer{Module: testSelf, Revisions: newMemObjects(1 << 20)} keep := func(n int64, fp string) { diff --git a/modules/mesh-delivery-indexer/internal/index/order.go b/modules/mesh-delivery-indexer/internal/index/order.go index 00344d4a..395cebd9 100644 --- a/modules/mesh-delivery-indexer/internal/index/order.go +++ b/modules/mesh-delivery-indexer/internal/index/order.go @@ -9,7 +9,6 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk" "git.novox.be/novox/mesh-sdk/go/process" ) @@ -49,7 +48,7 @@ func (w *mergeWrite) taken(digest string) { w.fresh[digest] = true } -func newMergeWrite(f forgemsg.Forge) *mergeWrite { +func newMergeWrite(f whole) *mergeWrite { return &mergeWrite{repository: f.Repository, merge: f.Merge} } @@ -121,13 +120,13 @@ func (w *mergeWrite) wrote(key string) { } // said is the line of an applied merged message: its repository, commit and pull request, and the modules written. -func (w *mergeWrite) said(number int) string { +func (w *mergeWrite) said() string { if w.late != "" { - return fmt.Sprintf("%s at %.12s (#%d): merged message applied as late: older than the taken %.12s, nothing written", w.repository, w.merge, number, w.late) + return fmt.Sprintf("%s at %.12s: taken in as late: older than the taken %.12s, nothing written", w.repository, w.merge, w.late) } written := "none" if len(w.written) > 0 { written = strings.Join(slices.Sorted(slices.Values(w.written)), ", ") } - return fmt.Sprintf("%s at %.12s (#%d): merged message applied; modules written: %s", w.repository, w.merge, number, written) + return fmt.Sprintf("%s at %.12s: taken in; modules written: %s", w.repository, w.merge, written) } diff --git a/modules/mesh-delivery-indexer/internal/index/own.go b/modules/mesh-delivery-indexer/internal/index/own.go index 2e903596..c008596f 100644 --- a/modules/mesh-delivery-indexer/internal/index/own.go +++ b/modules/mesh-delivery-indexer/internal/index/own.go @@ -9,7 +9,6 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk" ) @@ -73,7 +72,7 @@ func moduleAt(held map[string]meshcatalog.ModulesByModuleValue, repository, dir // moduleOfDir is the module of a changed directory of a pull request: the module its manifest in the message names, else // the one the modules bucket holds at that directory of the repository, else the directory's last element (the // catalogue's rule for nearly every manifest). -func moduleOfDir(held map[string]meshcatalog.ModulesByModuleValue, f forgemsg.Forge, dir string) string { +func moduleOfDir(held map[string]meshcatalog.ModulesByModuleValue, f whole, dir string) string { if name := manifestModule(f.Manifests[dir]); name != "" { return name } diff --git a/modules/mesh-delivery-indexer/internal/index/pending.go b/modules/mesh-delivery-indexer/internal/index/pending.go deleted file mode 100644 index a06bad2f..00000000 --- a/modules/mesh-delivery-indexer/internal/index/pending.go +++ /dev/null @@ -1,1475 +0,0 @@ -package index - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "maps" - "slices" - "strings" - "time" - - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/graph" - - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo" - - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk" - "git.novox.be/novox/mesh-sdk/go/busretry" - "git.novox.be/novox/mesh-sdk/go/keys" - "git.novox.be/novox/mesh-sdk/go/process" - - "mesh-delivery-indexer/judge" -) - -// The take-in step (novox/hq ADR 0633, A-476): a merged message is kept in the indexer's own state pending, one key per -// merge, until the take-in that ends it; a revision is taken in by the running indexer whatever build of the indexer it names. -const ( - // PendingState is the indexer's state of held merges: the bucket mesh-delivery-indexer_pending, keyed - // takein.PendingKey, so the key order is the merge order. An old `state` entry (the indexer is foundation, ADR - // 0595 §11), history 1, reached by the indexer alone. - PendingState = "pending" - // PendingTool reads it (ADR 0633 §5); TakeInPendingTool and DropPendingTool steer it. - PendingTool = "indexer_pending" - TakeInPendingTool = "indexer_take_in_pending" - DropPendingTool = "indexer_drop_pending" -) - -// errClaimLost says another take-in claimed a pending merge first: this one took nothing in. -var errClaimLost = errors.New("another take-in claimed the merges first, so this one took nothing in") - -// NoBuild opens what a revision waits for when it gives the indexer no fingerprint: the reason follows. No indexer -// runs such a build, so the revision waits until a later merge gives it one or a hand take-in (indexer_take_in_pending). -const NoBuild = "no build: " - -// PendingRef is what a pending value keeps of its merged message instead of a copy (a merge of a repository with many -// modules is about 0.5 MB, over the state's 256 KiB value cap; issue 1585): the small fields the waits are worked out -// from. The message itself is read from the forge bridge's stream by Pending.Seq when a take-in needs it. -type PendingRef struct { - Repository string `json:"repository"` - Number int `json:"number,omitempty"` - Merge string `json:"merge"` - // Branch is the pull request's branch, which names its candidate: a merge covered by a newer one is taken out of it. - Branch string `json:"branch,omitempty"` - Members []meshdeliveryclient.Head `json:"members,omitempty"` - // Holds are the modules the merge's manifests name, which indexer_pending says it holds back. - Holds []string `json:"holds,omitempty"` -} - -// Pending is one value of the pending state: a reference to the merged message (Seq, its sequence on the forge bridge's -// stream, and Ref), or, in a value an earlier indexer wrote, the message whole as the forge bridge encoded it; the group it -// was merged in (Ended once the group's end was heard), since when it is held, the indexer build its revision waits -// for (absent while it waits for nothing yet, or behind a group being gathered), the build of the indexer that last -// wrote it, the wait it was last said under on the take-in stream (Said, SaidAt), and Taking, set by a take-in that -// claimed it: a second indexer that finds it claimed takes nothing in (ADR 0633 §1: never two take-ins of one merge). -type Pending struct { - V int `json:"v"` - Seq uint64 `json:"seq,omitempty"` - Ref *PendingRef `json:"ref,omitempty"` - Forge json.RawMessage `json:"forge,omitempty"` - Lost string `json:"lost,omitempty"` - Refused string `json:"refused,omitempty"` - Group string `json:"group,omitempty"` - Ended bool `json:"ended,omitempty"` - Since time.Time `json:"since"` - WaitsFor string `json:"waits-for,omitempty"` - LastWriter string `json:"last-writer"` - Said string `json:"said,omitempty"` - SaidKind string `json:"said-kind,omitempty"` - SaidAt time.Time `json:"said-at,omitzero"` - Taking string `json:"taking,omitempty"` - // Unread is why the merged message could not be read for a transport fault (the forge bridge restarting, a deadline; - // ADR 0664's classes): not lost, it is read again on the next message heard on the forge, at the start and by - // indexer_take_in_pending (issue 1966 note 153, ADR 0668 §1). - Unread string `json:"unread,omitempty"` - // UnreadSince is when the message was first found unread: past meshconditionsclient.CallUnansweredAfter, with the forge - // heard since, it is the finding call-unanswered (unreadcall.go, ADR 0668 §3). - UnreadSince time.Time `json:"unread-since,omitzero"` - // Held is why a ready merge is not taken in although its revision names this build: objects full (HeldObjectsFull), - // until room is said (RoomAgain). - Held string `json:"held,omitempty"` -} - -// PendingEntry is one key of the pending state as read: its key, revision, value and the merged message decoded. For a -// value that keeps a reference, Forge holds only the fields of the reference (Light) until Indexer.load reads the message -// from the stream; Holds are the modules the merge's manifests name. -type PendingEntry struct { - Key string - Revision uint64 - Value Pending - Forge forgemsg.Forge - Light bool - Holds []string - // Covered says its message is not read or applied: either a newer merge of the same repository is in the same revision - // (applying the newest writes every module whose manifest differs from what is held, modules.go), or its repository's - // taken merge is at or ahead of it (loadNewest, takenAlready), so applying it would write nothing. - Covered bool -} - -// DecodePending reads one pending value and its merged message. -func DecodePending(data []byte) (Pending, forgemsg.Forge, error) { - var p Pending - if err := json.Unmarshal(data, &p); err != nil { - return p, forgemsg.Forge{}, fmt.Errorf("not a pending value: %w", err) - } - if p.V != 1 { - return p, forgemsg.Forge{}, fmt.Errorf("a pending value of version %d, not 1", p.V) - } - if len(p.Forge) == 0 && p.Ref != nil { - r := p.Ref - return p, forgemsg.Forge{V: keys.Version, Kind: forgemsg.ForgeMerged, Repository: r.Repository, Number: r.Number, - Merge: r.Merge, Branch: r.Branch, Group: p.Group, Members: r.Members}, nil - } - f, err := forgemsg.DecodeForge(p.Forge) - if err != nil { - return p, f, fmt.Errorf("its merged message: %w", err) - } - if f.Kind != forgemsg.ForgeMerged && f.Kind != forgemsg.ForgePushed { - return p, f, fmt.Errorf("a pending value holds a %s message, not a merged or pushed one", f.Kind) - } - return p, RevisionForm(f), nil -} - -// RevisionForm is a message the take-in holds as a revision's member (design 47 G §509, §714, A-269): a pushed message's head -// stands where a merged message's merge stands, so pending refs, the newest load and the revision's head key on it alike. -// A merged message is returned as it is. -func RevisionForm(f forgemsg.Forge) forgemsg.Forge { - if f.Kind == forgemsg.ForgePushed { - f.Kind, f.Merge = forgemsg.ForgeMerged, f.Head - } - return f -} - -func encodePending(p Pending) ([]byte, error) { - p.V = 1 - return json.Marshal(p) -} - -// ReadPending is every value the pending state holds, in key order (the merge order); a key that does not read is -// said through logf and left. -func ReadPending(ctx context.Context, st Store, logf func(string, ...any)) ([]PendingEntry, error) { - keys, err := st.Keys(ctx) - if err != nil { - return nil, fmt.Errorf("listing %s: %w", PendingState, err) - } - slices.Sort(keys) - var out []PendingEntry - for _, key := range keys { - value, rev, found, err := st.Get(ctx, key) - if err != nil { - return nil, fmt.Errorf("%s.%s: %w", PendingState, key, err) - } - if !found { - continue - } - p, f, err := DecodePending(value) - if err != nil { - if logf != nil { - logf("%s.%s is left: %v", PendingState, key, err) - } - continue - } - e := PendingEntry{Key: key, Revision: rev, Value: p, Forge: f, Light: len(p.Forge) == 0} - if p.Ref != nil { - e.Holds = p.Ref.Holds - } else { - e.Holds = holdsOf(f) - } - out = append(out, e) - } - return out, nil -} - -// holdsOf are the modules a merged message's manifests name, sorted. -func holdsOf(f forgemsg.Forge) []string { - var out []string - for _, raw := range f.Manifests { - if name := manifestModule(raw); name != "" && !slices.Contains(out, name) { - out = append(out, name) - } - } - slices.Sort(out) - return out -} - -// refOf is what a pending value keeps of the merged message f (PendingRef). -func refOf(f forgemsg.Forge) *PendingRef { - return &PendingRef{Repository: f.Repository, Number: f.Number, Merge: f.Merge, Branch: f.Branch, Members: f.Members, Holds: holdsOf(f)} -} - -// heldMessage is the message Arrive was handed for a sequence, kept in memory until its merge is taken in or dropped. -func (ix *Indexer) heldMessage(seq uint64) (forgemsg.Forge, bool) { - ix.heldMu.Lock() - defer ix.heldMu.Unlock() - f, ok := ix.held[seq] - return f, ok -} - -func (ix *Indexer) hold(seq uint64, f forgemsg.Forge) { - ix.heldMu.Lock() - defer ix.heldMu.Unlock() - if ix.held == nil { - ix.held = map[uint64]forgemsg.Forge{} - } - ix.held[seq] = f -} - -func (ix *Indexer) forget(entries ...PendingEntry) { - ix.heldMu.Lock() - defer ix.heldMu.Unlock() - for _, e := range entries { - delete(ix.held, e.Value.Seq) - } -} - -// load gives the entry the merged message whole: the copy an earlier value holds, the message Arrive was handed (kept in -// memory), else the message read from the forge bridge's stream by the value's sequence. A stream that no longer holds it -// answers ErrForgeGone, a bus that refuses the read its error; a message that is not the merge the value refers to (a -// stream rebuilt under the same sequence) is refused. -func (ix *Indexer) load(ctx context.Context, e PendingEntry) (PendingEntry, error) { - if !e.Light { - return e, nil - } - if f, ok := ix.heldMessage(e.Value.Seq); ok { - e.Forge, e.Light = f, false - return e, nil - } - if ix.Message == nil { - return e, fmt.Errorf("the forge bridge is not asked by this indexer, so %s (%s#%d) cannot be read", e.Key, e.Forge.Repository, e.Forge.Number) - } - data, err := ix.Message(ctx, e.Forge.Repository, e.Forge.Merge, e.Forge.Number) - if err != nil { - return e, fmt.Errorf("the forge bridge could not answer for the merge of %s#%d at %s: %w", e.Forge.Repository, e.Forge.Number, e.Forge.Merge, err) - } - f, err := forgemsg.DecodeForge(data) - if err != nil { - return e, fmt.Errorf("the forge bridge's answer for %s is not a forge message: %w", e.Key, err) - } - f = RevisionForm(f) - if f.Kind != forgemsg.ForgeMerged || f.Repository != e.Forge.Repository || f.Merge != e.Forge.Merge || (e.Forge.Number > 0 && f.Number != e.Forge.Number) { - return e, fmt.Errorf("the forge bridge's answer is not the merge %s#%d at %s that %s refers to", e.Forge.Repository, e.Forge.Number, e.Forge.Merge, e.Key) - } - e.Forge, e.Light = f, false - return e, nil -} - -// loadNewest prepares the ready merges of one revision, in merge order. Of the merges of one repository only the newest is -// loaded: applying it writes every module whose manifest differs from what is held, so it covers the older ones, which are -// marked Covered and never read. A newest merge the forge bridge cannot give is answered in lost with its reason, by key, and its revision is held (holdLost). -func (ix *Indexer) loadNewest(ctx context.Context, entries []PendingEntry) (out, lost []PendingEntry, reasons map[string]string, unread map[string]bool) { - out = slices.Clone(entries) - reasons, unread = map[string]string{}, map[string]bool{} - newest := map[string]bool{} - for i := len(out) - 1; i >= 0; i-- { - repo := strings.ToLower(out[i].Forge.Repository) - if newest[repo] { - out[i].Covered = true - continue - } - newest[repo] = true - // A merge at or behind its repository's taken merge writes nothing: it is ended as covered without reading its - // message, which a bridge that cannot give it would otherwise hold the whole revision for (issue 1966). A merge whose - // ancestry cannot be asked is read as before. - if taken, err := ix.takenAlready(ctx, forgemsg.GroupMerge{Repository: out[i].Forge.Repository, Number: out[i].Forge.Number, Merge: out[i].Forge.Merge}); err == nil && taken { - ix.logf("take-in: %s (%s#%d) is covered by the taken merge of %s, its message is not read", out[i].Key, out[i].Forge.Repository, out[i].Forge.Number, out[i].Forge.Repository) - out[i].Covered = true - continue - } - full, err := ix.load(ctx, out[i]) - if err != nil { - lost = append(lost, out[i]) - reasons[out[i].Key] = err.Error() - // A transport fault is not the bridge's answer: the message is unread, never lost (ADR 0668 §1). - unread[out[i].Key] = busretry.Undelivered(err) || busretry.TimedOut(err) - continue - } - out[i] = full - } - slices.Reverse(lost) - return out, lost, reasons, unread -} - -// Waits is the pending state read as what waits (ADR 0633 §3, §4): Ready are the merges no group still being gathered -// holds back, in merge order, which are one revision together; Groups are the groups still being gathered, each with -// its members held so far; Behind are the merges in a gathered group's member repositories that wait behind it, by the -// group's take-in key. -type Waits struct { - Ready []PendingEntry - Groups map[string][]PendingEntry - Behind map[string][]PendingEntry -} - -// WaitsOf works out what waits from the entries, in key order. A member of a group whose end was not heard is -// gathered; from the first such member on, a later merge in one of the group's members' repositories waits behind it, -// because its tree holds that member's commit. Every other merge is ready. -func WaitsOf(entries []PendingEntry) Waits { - w := Waits{Groups: map[string][]PendingEntry{}, Behind: map[string][]PendingEntry{}} - blocked := map[string]string{} // repository, lower case, to the group key it waits behind - for _, e := range entries { - if e.Value.Group != "" && !e.Value.Ended { - w.Groups[e.Value.Group] = append(w.Groups[e.Value.Group], e) - for _, m := range e.Forge.Members { - if _, ok := blocked[strings.ToLower(m.Repository)]; !ok { - blocked[strings.ToLower(m.Repository)] = takein.GroupTakeInKey(e.Value.Group) - } - } - continue - } - if g, ok := blocked[strings.ToLower(e.Forge.Repository)]; ok { - w.Behind[g] = append(w.Behind[g], e) - continue - } - w.Ready = append(w.Ready, e) - } - return w -} - -// mergesOf are the merges of entries, oldest first. -func mergesOf(entries []PendingEntry) []takein.Merge { - out := make([]takein.Merge, len(entries)) - for i, e := range entries { - out[i] = takein.Merge{Repository: e.Forge.Repository, Number: e.Forge.Number, Merge: e.Forge.Merge} - } - return out -} - -func sinceOf(entries []PendingEntry) time.Time { - var at time.Time - for _, e := range entries { - if at.IsZero() || e.Value.Since.Before(at) { - at = e.Value.Since - } - } - return at -} - -// groupWait is the take-in body of a group being gathered. -func groupWait(kind, group string, held []PendingEntry) takein.TakeIn { - t := takein.TakeIn{Kind: kind, Key: takein.GroupTakeInKey(group), What: takein.TakeInGroup, Group: group, - Merges: mergesOf(held), Since: sinceOf(held)} - if len(held) > 0 { - t.Members = held[len(held)-1].Forge.Members - } - return t -} - -// revisionWait is the take-in body of a revision: its key is its oldest merge's. -func revisionWait(kind string, ready []PendingEntry, waitsFor string) takein.TakeIn { - return takein.TakeIn{Kind: kind, Key: ready[0].Key, What: takein.TakeInRevision, Merges: mergesOf(ready), - WaitsFor: waitsFor, Since: sinceOf(ready)} -} - -// pendingOnBus says whether the pending state is on the bus: a state this delivery brings is bound when it appears -// (issue 1790), and until then every merge is taken in at once, so the build this program runs is not needed. A read -// that fails otherwise counts as on the bus: the step that follows answers the error. -func (ix *Indexer) pendingOnBus(ctx context.Context) bool { - _, _, _, err := ix.Pending.Get(ctx, takein.PendingKey(0)) - return !errors.Is(err, process.ErrNotYet) -} - -// ownBuildNow is the build this program runs, as FixOwnBuild fixed it at the start: empty while unknown, and never asked again. -// It names only the writer of what is written (LastWriter); nothing waits for it (A-476 b). -func (ix *Indexer) ownBuildNow(context.Context) string { - ix.ownMu.Lock() - defer ix.ownMu.Unlock() - return ix.ownFP -} - -// sayTakeIn publishes a take-in event and keeps what it said on the entries, so indexer_pending reads the last event of -// each wait from the state. A publish that fails is said in the log: the next evaluation says it again. -func (ix *Indexer) sayTakeIn(ctx context.Context, t takein.TakeIn) bool { - if ix.SayTakeIn == nil { - return true - } - if err := ix.SayTakeIn(ctx, t); err != nil { - ix.logf("take-in: %s.%s is not said: %v", t.Kind, t.Key, err) - return false - } - return true -} - -// markSaid writes on each entry the wait it was said under, by compare-and-set; a lost one is left (its next -// evaluation says it again). -func (ix *Indexer) markSaid(ctx context.Context, entries []PendingEntry, t *takein.TakeIn, change func(*Pending)) { - for _, e := range entries { - p := e.Value - if t != nil { - p.Said, p.SaidKind, p.SaidAt = t.Key, t.Kind, ix.now() - } - if change != nil { - change(&p) - } - data, err := encodePending(p) - if err != nil { - continue - } - if err := ix.Pending.Update(ctx, e.Key, data, e.Revision); err != nil { - ix.logf("take-in: %s.%s is not written again: %v", PendingState, e.Key, err) - } - } -} - -// Arrive takes one merged message of the forge bridge's stream in under ADR 0633: seq is its stream sequence. It is -// written to pending first (a redelivered message finds its key there and writes no second), and then what pending -// holds is evaluated (Evaluate): taken in whole when its revision names the build this program runs, or held for the -// indexer it names. It answers what was taken in, and a line saying what waits. A message is never written while the -// build this program runs is not known (ErrOwnBuildUnknown). -func (ix *Indexer) Arrive(ctx context.Context, seq uint64, f forgemsg.Forge) (Applied, string, error) { - if ix.Pending == nil || ix.Modules == nil || !ix.pendingOnBus(ctx) { - // The state this delivery brings is not on the bus yet (issue 1790): the merge is taken in at once, as before, - // whether or not the build this program runs is known. - a, err := ix.ApplyMessage(ctx, f) - return a, "", err - } - own := ix.ownBuildNow(ctx) - ix.pendingMu.Lock() - defer ix.pendingMu.Unlock() - key := takein.PendingKey(seq) - ix.hold(seq, f) - data, err := encodePending(Pending{Seq: seq, Ref: refOf(f), Group: f.Group, Since: ix.now(), LastWriter: own}) - if err != nil { - return Applied{}, "", err - } - switch err := ix.Pending.Create(ctx, key, data); { - case errors.Is(err, process.ErrNotYet): - // The state this delivery brings is not on the bus yet (issue 1790): the merge is taken in at once, as before. - ix.logf("take-in: the state %s is not on the bus yet, so %s#%d is taken in at once, as before ADR 0633", PendingState, f.Repository, f.Number) - a, err := ix.ApplyMessage(ctx, f) - return a, "", err - case err != nil && !errors.Is(err, ErrConflict): - return Applied{}, "", fmt.Errorf("%s.%s: %w", PendingState, key, err) - } - if f.Group != "" { - entries, err := ReadPending(ctx, ix.Pending, ix.logf) - if err != nil { - return Applied{}, "", err - } - if held := WaitsOf(entries).Groups[f.Group]; len(held) > 0 { - t := groupWait(takein.TakeInPending, f.Group, held) - if ix.sayTakeIn(ctx, t) { - ix.markSaid(ctx, held, &t, nil) - } - } - } - return ix.evaluate(ctx, own, false) -} - -// EndGroup takes a group's end in (ADR 0633 §4): once every member it names is held, the group's members are marked -// ended, the group's wait is said taken (take-in.taken under its group key), and pending is evaluated, the group now -// one revision. A member named and neither held nor taken in yet answers an error, so the end is delivered again once -// the member arrived. An end of a group taken in already (every member's merge at or behind its repository's taken -// merge) changes nothing and is said: it is never delivered again for ever. -func (ix *Indexer) EndGroup(ctx context.Context, end forgemsg.ForgeGroupEnd) (Applied, string, error) { - if ix.Pending == nil || !ix.pendingOnBus(ctx) { - return Applied{}, "no pending state: the group's merges were taken in one by one", nil - } - own := ix.ownBuildNow(ctx) - ix.pendingMu.Lock() - defer ix.pendingMu.Unlock() - entries, err := ReadPending(ctx, ix.Pending, ix.logf) - if errors.Is(err, process.ErrNotYet) { - return Applied{}, "the pending state is not on the bus yet: the group's merges were taken in one by one", nil - } - if err != nil { - return Applied{}, "", err - } - var held []PendingEntry - for _, e := range entries { - if e.Value.Group == end.Group { - held = append(held, e) - } - } - isHeld := func(m forgemsg.GroupMerge) bool { - return slices.ContainsFunc(held, func(e PendingEntry) bool { - return trunk.SameRepository(e.Forge.Repository, m.Repository) && e.Forge.Number == m.Number - }) - } - var missing []string - for _, m := range end.Members { - if isHeld(m) { - continue - } - taken, err := ix.takenAlready(ctx, m) - if err != nil { - return Applied{}, "", err - } - if !taken { - missing = append(missing, fmt.Sprintf("%s#%d", m.Repository, m.Number)) - } - } - if len(missing) > 0 { - return Applied{}, "", fmt.Errorf("the end of the group %q names %s, not held yet: it waits for their merged messages", end.Group, strings.Join(missing, ", ")) - } - if len(held) == 0 { - ix.logf("take-in: the end of the group %q comes after its take-in: every member is taken in already, nothing changes", end.Group) - return Applied{}, "the group is taken in already", nil - } - if !slices.ContainsFunc(held, func(e PendingEntry) bool { return !e.Value.Ended }) { - return ix.evaluate(ctx, own, false) - } - // The group's wait ends first (take-in.taken under its group key); a revision it then is, pending for another - // indexer, is said pending under its oldest merge's key by the evaluation that follows. - t := groupWait(takein.TakeInTaken, end.Group, held) - said := &t - if !ix.sayTakeIn(ctx, t) { - said = nil // said again by the take-in or the hold that follows (saidWaits) - } - ix.markSaid(ctx, held, said, func(p *Pending) { p.Ended = true }) - ix.logf("take-in: the group %q ended with %d members held: one revision now", end.Group, len(held)) - return ix.evaluate(ctx, own, false) -} - -// endTakenGroups ends every group wait whose members are all taken in (issue 1585, notes 70 and 71): a group's end that -// never came, or a member taken in by hand (indexer_take_in_merge, a m. key), would hold the group and every merge behind -// it for ever. A member held is taken in when its merge is at or behind its repository's taken merge; a member no longer -// held when its head is in that taken merge. The wait is said taken (take-in.taken under the group key), and only then are -// its keys removed: a wait that could not be said stays and is ended at the next evaluation. It answers whether one ended. -func (ix *Indexer) endTakenGroups(ctx context.Context, entries []PendingEntry) (bool, error) { - if ix.Taken == nil { - return false, nil - } - w := WaitsOf(entries) - ended := false - for _, group := range slices.Sorted(maps.Keys(w.Groups)) { - held := w.Groups[group] - all, err := ix.groupTaken(ctx, held) - if err != nil { - // Best effort: a probe that fails keeps this group's wait and stops nothing else; the next evaluation asks again. - ix.logf("take-in: whether the group %q is taken in cannot be read, so its wait is kept: %v", group, err) - continue - } - if !all { - continue - } - if !ix.sayTakeIn(ctx, groupWait(takein.TakeInTaken, group, held)) { - continue - } - for _, e := range held { - if err := ix.Pending.Delete(ctx, e.Key, e.Revision); err != nil { - ix.logf("take-in: %s.%s belongs to a group taken in and is not removed: %v", PendingState, e.Key, err) - } - } - ix.forget(held...) - ix.logf("take-in: the group %q ended: every member is taken in already (%d merges held, removed)", group, len(held)) - ended = true - } - return ended, nil -} - -// groupTaken says whether every member of the group whose entries are held is taken in. -func (ix *Indexer) groupTaken(ctx context.Context, held []PendingEntry) (bool, error) { - isHeld := func(m meshdeliveryclient.Head) bool { - return slices.ContainsFunc(held, func(e PendingEntry) bool { - return trunk.SameRepository(e.Forge.Repository, m.Repository) && e.Forge.Number == m.Number - }) - } - for _, e := range held { - if ok, err := ix.takenAlready(ctx, forgemsg.GroupMerge{Repository: e.Forge.Repository, Number: e.Forge.Number, Merge: e.Forge.Merge}); err != nil || !ok { - return false, err - } - } - for _, m := range held[len(held)-1].Forge.Members { - if isHeld(m) { - continue - } - ok, err := ix.headTaken(ctx, m) - if err != nil || !ok { - return false, err - } - } - return true, nil -} - -// headTaken says whether a group member that is not held was taken in: its head is the taken merge of its repository or -// in it. -func (ix *Indexer) headTaken(ctx context.Context, m meshdeliveryclient.Head) (bool, error) { - key, err := trunk.RepositoryKey(m.Repository) - if err != nil { - return false, err - } - value, _, found, err := ix.Taken.Get(ctx, key) - if errors.Is(err, process.ErrNotYet) || (err == nil && !found) { - return false, nil - } - if err != nil { - return false, fmt.Errorf("taken.%s: %w", key, err) - } - cur, err := takein.DecodeTaken(value) - if err != nil || cur.Merge == "" || ix.Contains == nil { - return false, nil - } - if cur.Merge == m.Head { - return true, nil - } - in, err := ix.Contains(ctx, m.Repository, m.Head, cur.Merge) - if notFound(err) { - return false, nil // the forge does not have it (a rewritten history?): not known to be taken, as older has it - } - return in, err -} - -// takenAlready says whether a merge (a group member's, or the newest pending merge of a repository in loadNewest) is at or -// behind its repository's taken merge. -func (ix *Indexer) takenAlready(ctx context.Context, m forgemsg.GroupMerge) (bool, error) { - if ix.Taken == nil { - return false, nil - } - key, err := trunk.RepositoryKey(m.Repository) - if err != nil { - return false, err - } - value, _, found, err := ix.Taken.Get(ctx, key) - if errors.Is(err, process.ErrNotYet) || (err == nil && !found) { - return false, nil - } - if err != nil { - return false, fmt.Errorf("taken.%s: %w", key, err) - } - cur, err := takein.DecodeTaken(value) - if err != nil || cur.Merge == "" { - return false, nil - } - if cur.Merge == m.Merge { - return true, nil - } - return ix.older(ctx, m.Repository, m.Merge, cur.Merge) -} - -// TakeInAtStart is the take-in at the start (ADR 0633 §2), before the consumer on the forge bridge's stream starts: -// it evaluates pending once, taking in the whole revisions whatever build of the indexer they name (A-476 b); a group -// still being gathered stays. A claim left by an indexer that ran before is void here. The build this program runs, known -// or not, only names the writer. No timer. -func (ix *Indexer) TakeInAtStart(ctx context.Context) (Applied, string, error) { - if ix.Pending == nil || !ix.pendingOnBus(ctx) { - // Nothing can wait in a state not on the bus: the start asks no build and does not wait for one. - return Applied{}, "the pending state is not on the bus yet: nothing waits", nil - } - own := ix.ownBuildNow(ctx) - ix.pendingMu.Lock() - defer ix.pendingMu.Unlock() - a, words, err := ix.evaluate(ctx, own, true) - if errors.Is(err, process.ErrNotYet) { - return a, "the pending state is not on the bus yet: nothing waits", nil - } - if err == nil && len(a.Written) == 0 { - // A start that takes nothing in says so. - ix.logf("take-in: at the start, this program (build %q) took nothing in: %s", own, words) - } - return a, words, err -} - -// UnreadAgain evaluates pending again when a merge in it is unread for a transport fault (Pending.Unread): told after -// every message heard on the forge, of any kind, so a merge whose message the forge bridge could not give while it -// restarted is read again once the forge is heard again, with no timer (issue 1966 note 153, ADR 0668 §4). It reads -// pending once and does nothing when no merge is unread. -func (ix *Indexer) UnreadAgain(ctx context.Context) error { - if ix.Pending == nil || !ix.pendingOnBus(ctx) { - return nil - } - entries, err := ReadPending(ctx, ix.Pending, ix.logf) - if err != nil { - return err - } - if slices.ContainsFunc(entries, func(e PendingEntry) bool { return e.Value.Unread != "" }) { - if err := ix.PendingAgain(ctx); err != nil { - return err - } - if entries, err = ReadPending(ctx, ix.Pending, ix.logf); err != nil { - return err - } - } - // The forge was heard, so the bridge that says it is: a read still failing past the bound is a finding (ADR 0668 §3). - ix.unreadFindings(ctx, entries) - return nil -} - -// PendingAgain evaluates pending again after something a fingerprint is computed from changed, a toolchain report -// (ADR 0633 §3), or the verdicts state bound (issue 1966): what waits is taken in. It does nothing while the pending state is not on the bus. -func (ix *Indexer) PendingAgain(ctx context.Context) error { - if ix.Pending == nil || !ix.pendingOnBus(ctx) { - return nil - } - own := ix.ownBuildNow(ctx) - ix.pendingMu.Lock() - defer ix.pendingMu.Unlock() - _, words, err := ix.evaluate(ctx, own, false) - if errors.Is(err, process.ErrNotYet) { - return nil - } - if err == nil && words != "" { - ix.logf("take-in: evaluated again: %s", words) - } - return err -} - -// evaluate works out what pending's ready merges name, together (ADR 0633 §3), and takes them in or holds them. -func (ix *Indexer) evaluate(ctx context.Context, own string, start bool) (Applied, string, error) { - entries, err := ReadPending(ctx, ix.Pending, ix.logf) - if err != nil { - return Applied{}, "", err - } - // A group wait whose members are all taken in ends first, so what waited behind it is evaluated in this same call. - if ended, err := ix.endTakenGroups(ctx, entries); err != nil { - return Applied{}, "", err - } else if ended { - if entries, err = ReadPending(ctx, ix.Pending, ix.logf); err != nil { - return Applied{}, "", err - } - } - w := WaitsOf(entries) - if len(w.Ready) == 0 { - return Applied{}, ix.waitingLine(w), nil - } - if !start { - for _, e := range w.Ready { - if e.Value.Taking != "" { - ix.logf("take-in: %s.%s is claimed by the take-in %s, so this indexer takes nothing in", PendingState, e.Key, e.Value.Taking) - return Applied{}, "claimed by another take-in", nil - } - } - } - // Of each repository's merges only the newest is read (from memory, else asked of the forge bridge: a pending value - // keeps a reference); the older ones are covered by it. A newest merge that cannot be read is said and - // holds its revision: nothing of it is taken in silently. - ready, lost, reasons, unread := ix.loadNewest(ctx, w.Ready) - if len(lost) > 0 { - return Applied{}, ix.waitingLine(w), ix.holdLost(ctx, w.Ready, lost, reasons, unread) - } - dry, err := ix.computeOver(ctx, ready) - if err != nil { - return Applied{}, "", err - } - if words, refused, err := ix.refuseCycle(ctx, ready, dry); refused || err != nil { - return Applied{}, words, err - } - // The indexer takes in with the build it runs, whatever build of the indexer the revision names (A-476 b). - a, err := ix.takeInWhole(ctx, ready, own, "") - if errors.Is(err, errClaimLost) { - return a, "claimed by another take-in", nil - } - if errors.Is(err, ErrObjectsFull) { - ix.holdFull(ctx, ready) - return a, ix.waitingLine(w) + "; held: " + HeldObjectsFull, nil - } - return a, ix.waitingLine(w), err -} - -// newestOfRevision is the index in ready of the revision's newest merge. -func newestOfRevision(ready []PendingEntry) int { - n := 0 - for i, e := range ready { - if e.Value.Seq > ready[n].Value.Seq { - n = i - } - } - return n -} - -// refuseCycle refuses the revision whose catalog graph would hold a cycle with a build edge the take-in refuses (design 47 G -// §49, A-326; RefusedCycle), over the entries the dry run computed: written as the refusal of its newest value and said in -// take-in.pending with every artifact on the cycle, once, as a refusal by its validation is (§713). It answers whether it refused. -func (ix *Indexer) refuseCycle(ctx context.Context, ready []PendingEntry, d *dryRun) (string, bool, error) { - entries, err := d.ix.viewEntries(ctx) - if err != nil { - return "", false, err - } - cycle := graph.RefusedCycle(WithEdges(ix.withActs(ctx, entries))) - if cycle == nil { - return "", false, nil - } - n := newestOfRevision(ready) - refused := "a cycle with a build edge, which no running version breaks (design 47 G §49): " + strings.Join(cycle, ", ") - words := fmt.Sprintf("the revision at %d is refused: %s", ready[n].Value.Seq, refused) - t := revisionWait(takein.TakeInPending, ready, "") - t.Refused = refused - if e := ready[n].Value; e.Refused == refused && e.Said == t.Key && e.SaidKind == takein.TakeInPending { - return words, true, nil - } - ix.logf("take-in: %s", words) - if !ix.sayTakeIn(ctx, t) { - return words, true, nil - } - ix.markSaid(ctx, ready[n:n+1], &t, func(p *Pending) { p.Refused = refused }) - return words, true, nil -} - -// holdFull marks the ready merges held with the reason objects full (Pending.Held), read again at their revisions now -// (the take-in's claim moved them), so indexer_pending shows why they wait. -func (ix *Indexer) holdFull(ctx context.Context, ready []PendingEntry) { - entries, err := ReadPending(ctx, ix.Pending, ix.logf) - if err != nil { - ix.logf("take-in: the merges held by a full %s are not marked: %v", CatalogRevisionsObjects, err) - return - } - var held []PendingEntry - for _, e := range entries { - if slices.ContainsFunc(ready, func(r PendingEntry) bool { return r.Key == e.Key }) && e.Value.Held != HeldObjectsFull { - held = append(held, e) - } - } - ix.markSaid(ctx, held, nil, func(p *Pending) { p.Held = HeldObjectsFull }) -} - -// holdLost holds a revision whose newest merge of a repository cannot be read (the forge bridge's stream no longer holds -// it, or the bus refuses the read): said once on take-in.pending with the lost keys (TakeIn.Lost), and on each lost entry -// (Pending.Lost, the reason, which indexer_pending shows), until a newer merge of its repository covers it or -// indexer_drop_pending drops it. A merge unread for a transport fault (unread) is kept as Pending.Unread, not Lost, and read -// again on the next message heard on the forge (UnreadAgain). Nothing of the revision is taken in meanwhile. -func (ix *Indexer) holdLost(ctx context.Context, ready, lost []PendingEntry, reasons map[string]string, unread map[string]bool) error { - t := revisionWait(takein.TakeInPending, ready, "") - for _, g := range lost { - t.Lost = append(t.Lost, g.Key) - } - said := true - for _, e := range ready { - isLost := slices.Contains(t.Lost, e.Key) - if e.Value.Said != t.Key || e.Value.SaidKind != takein.TakeInPending || (isLost && (unread[e.Key] && e.Value.Unread == "" || !unread[e.Key] && e.Value.Lost == "")) { - said = false - } - } - if said { - return nil - } - for _, g := range lost { - if unread[g.Key] { - ix.logf("take-in: %s (%s#%d) is not taken in and holds its revision, its message is unread for a transport fault: %s; it is read again on the next message heard on the forge, at the start, and by %s", - g.Key, g.Forge.Repository, g.Forge.Number, reasons[g.Key], TakeInPendingTool) - continue - } - ix.logf("take-in: %s (%s#%d) is not taken in and holds its revision, the forge bridge could not give its message: %s; %s reads it again; indexer_drop_pending drops it once a later merge of %s covers it", - g.Key, g.Forge.Repository, g.Forge.Number, reasons[g.Key], TakeInPendingTool, g.Forge.Repository) - } - if !ix.sayTakeIn(ctx, t) { - return nil - } - var kept []PendingEntry - for _, e := range ready { - if !slices.Contains(t.Lost, e.Key) { - kept = append(kept, e) - } - } - for _, g := range lost { - reason, isUnread := reasons[g.Key], unread[g.Key] - now := ix.now().UTC() - ix.markSaid(ctx, []PendingEntry{g}, &t, func(p *Pending) { - p.Lost, p.Unread = reason, "" - if !isUnread { - p.UnreadSince = time.Time{} - return - } - p.Lost, p.Unread = "", reason - if p.UnreadSince.IsZero() { - p.UnreadSince = now - } - }) - } - ix.markSaid(ctx, kept, &t, nil) - return nil -} - -// waitingLine says what waits after an evaluation, for the heard state. -func (ix *Indexer) waitingLine(w Waits) string { - var parts []string - for _, g := range slices.Sorted(maps.Keys(w.Groups)) { - parts = append(parts, fmt.Sprintf("the group %q gathered (%d held)", g, len(w.Groups[g]))) - } - for _, g := range slices.Sorted(maps.Keys(w.Behind)) { - parts = append(parts, fmt.Sprintf("%d merges behind %s", len(w.Behind[g]), g)) - } - return strings.Join(parts, "; ") -} - -// dryRun is an indexer computing a take-in without writing it, and the overlays of what it would have written. -type dryRun struct { - ix *Indexer - modules, graph, verdicts *overlayStore - candidates, taken, predefined *overlayStore - oldSections *overlayStore -} - -// dry is an indexer over overlays of this one's states (overlay.go): what a take-in would write stays in memory and -// nothing reaches the bus but the manifests and the packages the revision publishes, which clause 1 writes all the -// same. It says no event, kicks nothing in the background, and knows what this one knows of the source, the toolchains -// and the previews. Every field of Indexer is named here (TestDryNamesEveryField holds it so). -func (ix *Indexer) dry() *dryRun { - over := func(st Store) (Store, *overlayStore) { - if st == nil { - return nil, nil - } - o := newOverlay(st) - return o, o - } - d := &dryRun{} - x := &Indexer{Module: ix.Module, Manifests: ix.Manifests, Packages: ix.Packages, JudgeID: ix.JudgeID, Files: ix.Files, GoSum: ix.GoSum, - GoMod: ix.GoMod, Now: ix.Now, Builds: ix.Builds, Named: ix.Named, Trunk: ix.Trunk, BranchHead: ix.BranchHead, Read: ix.Read, Contains: ix.Contains, - ReadOpen: nil, OwnBuild: nil, Message: ix.Message, SayTakeIn: nil, Pending: nil, - // The registry milestone's fields: what a take-in would write on the bus stays out of a dry run, so the states of the - // requirements, findings, seat definitions, holds and their bounds are not overlaid, nothing is said, and no hold is - // weighed (Holds and Counted nil hold nothing); ReadFile only reads. - Requirements: nil, Findings: nil, SeatDefinitions: nil, ReadFile: ix.ReadFile, SaySeatDefinition: nil, - OwnBuildState: nil, Heard: nil, Placement: nil, RunningWhole: nil, underSeats: true, - // A dry run keeps no catalog graph revision (catalog_revisions.go): its revision is the take-in's to keep. - Revisions: nil, SayCatalogRevision: nil, SayFill: nil, EntryIndex: nil, NamedSets: nil, Versions: nil, SayVersionReleased: nil, SayVersionsFilled: nil, Inputs: nil, InputObjects: nil, Pins: nil, PinsPending: nil, - // A dry run writes no registration and asks no one. - Registry: nil, SeedModules: nil} - x.Logf = func(format string, args ...any) { ix.logf("take-in, computed and not written: "+format, args...) } - x.Store, d.candidates = over(ix.Store) - x.Modules, d.modules = over(ix.Modules) - x.Graph, d.graph = over(ix.Graph) - x.Taken, d.taken = over(ix.Taken) - x.Predefined, d.predefined = over(ix.Predefined) - x.OldSections, d.oldSections = over(ix.OldSections) - ix.mu.Lock() - x.source = sourceNow{rev: ix.source.rev, heads: maps.Clone(ix.source.heads)} - x.previews = maps.Clone(ix.previews) - x.toolchains = map[string]map[string]nodebuildgo.Toolchain{} - for l, by := range ix.toolchains { - x.toolchains[l] = maps.Clone(by) - } - x.buildModules = maps.Clone(ix.buildModules) // the dry run computes with the build modules the running indexer takes - x.moving = maps.Clone(ix.moving) - x.untaken = maps.Clone(ix.untaken) - ix.mu.Unlock() - ix.ownMu.Lock() - x.ownFP = ix.ownFP - ix.ownMu.Unlock() - d.ix = x - return d -} - -// computeOver applies the ready merges, in merge order, to a dry indexer, as one overlay: each later merge's trees -// govern (ADR 0633 §3). It answers the dry run, whose overlays hold what the take-in would write. A merge that can never be read -// is left out, as its take-in will drop it. -func (ix *Indexer) computeOver(ctx context.Context, ready []PendingEntry) (*dryRun, error) { - d := ix.dry() - // The manifests are judged against the seat definitions the revision carries (seatsof.go, ADR 0633 §1), so the indexer - // it names is not refused for a verb or an event the built-in list lacks. - seats, err := ix.revisionSeats(ctx, ready) - if err != nil { - return d, fmt.Errorf("reading the seat definitions of %s: %w", ready[0].Key, err) - } - var applyErr error - judge.WithSeats(seats, func() { - for _, e := range ready { - if e.Covered { - continue - } - if _, err := d.ix.ApplyMessage(ctx, e.Forge); err != nil { - if errors.Is(err, ErrUnreadable) { - continue - } - applyErr = fmt.Errorf("computing the take-in of %s (%s#%d): %w", e.Key, e.Forge.Repository, e.Forge.Number, err) - return - } - } - }) - if applyErr != nil { - return d, applyErr - } - return d, nil -} - -// claim marks the entries as taken in by this take-in, by compare-and-set on the revisions read. A lost one (another -// take-in wrote the key, or the store failed) releases every claim this take-in made and answers false: nothing is -// taken in, and no claim of this one is left behind. -func (ix *Indexer) claim(ctx context.Context, entries []PendingEntry, own, by string) ([]PendingEntry, bool) { - id := fmt.Sprintf("%s at %s", own, ix.now().Format(time.RFC3339Nano)) - if by != "" { - id = by + ": " + id - } - claimed := make([]PendingEntry, 0, len(entries)) - for _, e := range entries { - p := e.Value - p.Taking, p.LastWriter = id, own - data, err := encodePending(p) - if err == nil { - err = ix.Pending.Update(ctx, e.Key, data, e.Revision) - } - if err != nil { - ix.logf("take-in: the claim of %s.%s is lost (%v): another take-in took it, so this one takes nothing in", PendingState, e.Key, err) - ix.release(ctx, claimed) - return nil, false - } - e.Value = p - _, rev, _, err := ix.Pending.Get(ctx, e.Key) - if err != nil { - ix.logf("take-in: the claim of %s.%s cannot be read back (%v), so this take-in takes nothing in", PendingState, e.Key, err) - ix.release(ctx, append(claimed, e)) - return nil, false - } - e.Revision = rev - claimed = append(claimed, e) - } - return claimed, true -} - -// release clears this take-in's claim on each entry, so a later merge, a restart or a hand take-in takes it in. It -// reads the key again and clears it by compare-and-set only while the claim is still this take-in's: a key another -// take-in wrote since, or removed, is left to it. A release that fails is said; the start of the next indexer voids -// the claim (TakeInAtStart). -func (ix *Indexer) release(ctx context.Context, claimed []PendingEntry) { - for _, e := range claimed { - value, rev, found, err := ix.Pending.Get(ctx, e.Key) - if err == nil && !found { - continue - } - var cur Pending - if err == nil { - err = json.Unmarshal(value, &cur) - } - if err == nil && cur.Taking != e.Value.Taking { - continue - } - if err == nil { - cur.Taking = "" - var data []byte - if data, err = encodePending(cur); err == nil { - err = ix.Pending.Update(ctx, e.Key, data, rev) - } - } - if err != nil { - ix.logf("take-in: the claim of %s.%s is not released (%v): the next indexer's start voids it", PendingState, e.Key, err) - } - } -} - -// takeInWhole takes the entries in with this indexer, as merges are taken in (ApplyMessage), oldest first, then -// removes them from pending and says each wait they were said under taken. by names a hand act ("" for none). -func (ix *Indexer) takeInWhole(ctx context.Context, entries []PendingEntry, own, by string) (Applied, error) { - return ix.takeInWholeBy(ctx, entries, own, by, "", "") -} - -// takeInWholeBy is takeInWhole; who and why, of a hand act, are carried on the take-in.taken events. -func (ix *Indexer) takeInWholeBy(ctx context.Context, entries []PendingEntry, own, by, who, why string) (Applied, error) { - claimed, ok := ix.claim(ctx, entries, own, by) - if !ok { - return Applied{}, errClaimLost - } - var applied Applied - newest := map[string]PendingEntry{} - for _, e := range claimed { - if !e.Covered { - newest[strings.ToLower(e.Forge.Repository)] = e - } - } - for _, e := range claimed { - if e.Covered { - by := "the taken merge" - if n, ok := newest[strings.ToLower(e.Forge.Repository)]; ok { - by = n.Key + ", the newest merge" - } - ix.logf("take-in: %s (%s#%d) is covered by %s of %s in this revision: its message is not read, and it is ended with it", e.Key, e.Forge.Repository, e.Forge.Number, by, e.Forge.Repository) - // Its pull request leaves its group's candidate as a merged message takes it out; nothing else of it is applied. - if err := ix.removeCandidate(ctx, e.Forge); err != nil { - ix.release(ctx, claimed) - return applied, fmt.Errorf("taking %s (%s#%d) out of its candidate: %w", e.Key, e.Forge.Repository, e.Forge.Number, err) - } - continue - } - a, err := ix.ApplyMessage(ctx, e.Forge) - applied.Written = append(applied.Written, a.Written...) - if a.Late != "" { - applied.Late = a.Late - } - if errors.Is(err, ErrUnreadable) { - ix.logf("take-in: %s (%s#%d) can never be read, dropped: %v", e.Key, e.Forge.Repository, e.Forge.Number, err) - continue - } - if err != nil { - ix.release(ctx, claimed) - return applied, fmt.Errorf("taking %s (%s#%d) in: %w", e.Key, e.Forge.Repository, e.Forge.Number, err) - } - } - // What the judge pass of the take-in carried (the verdicts) is kept with the view too; a store that is full refuses it and - // the take-in holds its merges as it holds them for the revision itself (design 47 G §181). - if err := ix.KeepView(ctx); errors.Is(err, ErrObjectsFull) { - ix.release(ctx, claimed) - return applied, err - } - for _, e := range claimed { - if err := ix.Pending.Delete(ctx, e.Key, e.Revision); err != nil { - ix.logf("take-in: %s.%s is taken in and not removed: %v", PendingState, e.Key, err) - } - } - ix.forget(claimed...) - branch, src, took := ix.takenWhat(ctx, claimed, applied) - ix.logf("take-in: %d merges taken in as one revision by %s%s, oldest %s: source revision %d of %s, %d entries", len(claimed), own, byWords(by), claimed[0].Key, src, branch, len(took)) - // The revision this take-in just kept (A-448), pinned for its pipelines before any later keep can remove it (A-449a). - revision := ix.PinPipelines(ctx, branch, src, took) - for _, t := range takenWaits(claimed) { - t.By, t.Why = who, why - t.Branch, t.Source, t.Entries, t.Revision = branch, src, took, revision - ix.sayTakeIn(ctx, t) - } - return applied, nil -} - -// takenWhat is what one take-in took in (design 47 G §476, §493, §648): the watched branch its merges landed on (the -// trunk for a merge that names none, as before G), the source revision it makes of that branch (nextSource) and one entry -// per module it wrote, with the commit, fingerprint and repository the catalog entry holds now and the pull request of -// that repository's merge. A module with no fingerprint (no build) is no entry; one the view cannot read is said. -func (ix *Indexer) takenWhat(ctx context.Context, claimed []PendingEntry, applied Applied) (string, int64, []meshcatalog.TakeInEntry) { - branch := TrunkBranch - pulls := map[string]int64{} - for _, e := range claimed { - if e.Forge.Base != "" { - branch = e.Forge.Base - } - if e.Forge.Number != 0 { - pulls[strings.ToLower(e.Forge.Repository)] = int64(e.Forge.Number) - } - } - src := ix.nextSource(branch) - written := writtenModules(applied.Written) - if len(written) == 0 { - return branch, src, nil - } - view, err := ix.viewEntries(ctx) - if err != nil { - ix.logf("take-in: the entries of source revision %d of %s cannot be read, so its take-in.taken names none and starts no pipeline: %v", src, branch, err) - return branch, src, nil - } - var out []meshcatalog.TakeInEntry - for _, m := range slices.Sorted(slices.Values(written)) { - e, ok := view[m] - if !ok || e.Fp == "" { - ix.logf("take-in: %s is written in source revision %d of %s with no fingerprint (%s): no pipeline entry", m, src, branch, e.None) - continue - } - out = append(out, meshcatalog.TakeInEntry{Module: m, Commit: e.Commit, Fingerprint: e.Fp, Repository: e.Repository}) - } - return branch, src, out -} - -// nextSource is the source revision a take-in of branch makes: one above the newest the indexer has seen for it (the gate's -// source-moved, the trunk's alone held here) or named in a take-in.taken before, which the gate's source. then moves -// to (A-34; the lead's answer A-431, principle 7). -func (ix *Indexer) nextSource(branch string) int64 { - ix.mu.Lock() - defer ix.mu.Unlock() - n := ix.saidSource[branch] - if isTrunkBranch(branch) && ix.source.rev > n { - n = ix.source.rev - } - if ix.saidSource == nil { - ix.saidSource = map[string]uint64{} - } - ix.saidSource[branch] = n + 1 - return int64(n + 1) -} - -// takenWaits are the taken events of a take-in: every wait the entries were said under (saidWaits), every wait -// indexer_pending names the claimed entries by now (WaitsOf: a revision by its oldest merge's key, a group by its g. key) and -// the key of every claimed entry, since indexer_pending named it by that key earlier when it waited behind a group. So a wait -// the watchman learned of without the indexer marking it (seeded from indexer_pending) ends too. No key is said twice. -func takenWaits(entries []PendingEntry) []takein.TakeIn { - out := saidWaits(entries) - said := map[string]bool{} - for _, t := range out { - said[t.Key] = true - } - w := WaitsOf(entries) - var named []takein.TakeIn - if len(w.Ready) > 0 { - named = append(named, revisionWait(takein.TakeInTaken, w.Ready, w.Ready[len(w.Ready)-1].Value.WaitsFor)) - } - for _, g := range slices.Sorted(maps.Keys(w.Groups)) { - named = append(named, groupWait(takein.TakeInTaken, g, w.Groups[g])) - } - for _, g := range slices.Sorted(maps.Keys(w.Behind)) { - named = append(named, revisionWait(takein.TakeInTaken, w.Behind[g], w.Behind[g][len(w.Behind[g])-1].Value.WaitsFor)) - } - for _, e := range entries { - named = append(named, revisionWait(takein.TakeInTaken, []PendingEntry{e}, e.Value.WaitsFor)) - } - for _, t := range named { - if !said[t.Key] { - said[t.Key] = true - out = append(out, t) - } - } - return out -} - -func byWords(by string) string { - if by == "" { - return "" - } - return " (a hand act: " + by + ")" -} - -// saidWaits are the taken events of every wait the entries were said under that is not ended yet. -func saidWaits(entries []PendingEntry) []takein.TakeIn { - by := map[string][]PendingEntry{} - for _, e := range entries { - if e.Value.Said != "" && e.Value.SaidKind == takein.TakeInPending { - by[e.Value.Said] = append(by[e.Value.Said], e) - } - } - var out []takein.TakeIn - for _, key := range slices.Sorted(maps.Keys(by)) { - held := by[key] - if strings.HasPrefix(key, "g.") { - out = append(out, groupWait(takein.TakeInTaken, held[0].Value.Group, held)) - continue - } - t := revisionWait(takein.TakeInTaken, held, held[0].Value.WaitsFor) - t.Key = key - out = append(out, t) - } - return out -} - -// copyKey writes what the overlay holds of key to the store, when it wrote or deleted it. -func copyKey(ctx context.Context, from *overlayStore, to Store, key string) error { - if from == nil || to == nil { - return nil - } - value, deleted, ok := from.written(key) - if !ok { - return nil - } - return write(ctx, to, key, func(cur []byte, found bool) ([]byte, bool, error) { - switch { - case deleted: - return nil, found, nil - case found && string(cur) == string(value): - return nil, false, nil - } - return value, false, nil - }) -} - -// PendingWaits is what indexer_pending answers of the pending state's entries (ADR 0633 §5): the ready merges as one -// revision, each group being gathered, and the merges behind a group, each with the modules it holds back and the -// last take-in event said of it. -func PendingWaits(entries []PendingEntry) []takein.PendingWait { - w := WaitsOf(entries) - var out []takein.PendingWait - if len(w.Ready) > 0 { - out = append(out, waitOf(w.Ready, takein.TakeInRevision, "")) - } - for _, g := range slices.Sorted(maps.Keys(w.Groups)) { - out = append(out, waitOf(w.Groups[g], takein.TakeInGroup, "")) - } - for _, g := range slices.Sorted(maps.Keys(w.Behind)) { - out = append(out, waitOf(w.Behind[g], takein.TakeInRevision, g)) - } - slices.SortStableFunc(out, func(a, b takein.PendingWait) int { return a.Since.Compare(b.Since) }) - return out -} - -func waitOf(entries []PendingEntry, what, behind string) takein.PendingWait { - pw := takein.PendingWait{Key: entries[0].Key, What: what, Merges: mergesOf(entries), Behind: behind, Since: sinceOf(entries), - WaitsFor: entries[len(entries)-1].Value.WaitsFor, Holds: []string{}} - if what == takein.TakeInGroup { - pw.Group = entries[0].Value.Group - pw.Key = takein.GroupTakeInKey(pw.Group) - pw.Members = entries[len(entries)-1].Forge.Members - pw.WaitsFor = "" - } - for _, e := range entries { - pw.Keys = append(pw.Keys, e.Key) - for _, name := range e.Holds { - if !slices.Contains(pw.Holds, name) { - pw.Holds = append(pw.Holds, name) - } - } - if e.Value.Lost != "" { - pw.Lost = append(pw.Lost, e.Key) - } - if e.Value.Unread != "" { - pw.Unread = append(pw.Unread, e.Key) - } - if e.Value.Refused != "" { - pw.Refused = e.Value.Refused - } - if e.Value.Said != "" && e.Value.SaidAt.After(pw.SaidAt) { - pw.Said, pw.SaidAt = "take-in."+e.Value.SaidKind, e.Value.SaidAt - } - } - slices.Sort(pw.Holds) - return pw -} - -// TakeInPendingByHand is the hand take-in (indexer_take_in_pending, ADR 0633 §5, A-476, principle 4): the pending revision (its -// m. key) or the group gathered so far (its g. key) is taken in now with this indexer and its judge, -// for a revision whose message is unread or lost or held for room, or a group whose end never comes. by and why are -// said in the journal and on the take-in.taken event. What waited behind it is evaluated again after. -func (ix *Indexer) TakeInPendingByHand(ctx context.Context, key, by, why string) TakenIn { - r := TakenIn{Module: key} - if ix.Pending == nil { - r.Answer = "the pending state is not on the bus: nothing waits" - return r - } - own := ix.ownBuildNow(ctx) - ix.pendingMu.Lock() - defer ix.pendingMu.Unlock() - entries, err := ReadPending(ctx, ix.Pending, ix.logf) - if err != nil { - r.Answer = err.Error() - return r - } - w := WaitsOf(entries) - var take []PendingEntry - switch { - case strings.HasPrefix(key, "g."): - for g, held := range w.Groups { - if takein.GroupTakeInKey(g) == key { - take = held - } - } - case len(w.Ready) > 0 && w.Ready[0].Key == key: - take = w.Ready - } - if len(take) == 0 { - r.Answer = fmt.Sprintf("nothing waits under %s: indexer_pending names the waits", key) - return r - } - take, lost, reasons, _ := ix.loadNewest(ctx, take) - if len(lost) > 0 { - r.Answer = fmt.Sprintf("nothing is taken in: the message of %s is not read (%s); asked again, it is read again; indexer_drop_pending drops it once a later merge of its repository covers it", lost[0].Key, reasons[lost[0].Key]) - return r - } - ix.logf("take-in: a hand take-in of %s (%d merges) asked by %s: %s", key, len(take), by, why) - hand := fmt.Sprintf("%s (%s)", by, why) - if _, err := ix.takeInWholeBy(ctx, take, own, hand, by, why); err != nil { - r.Answer = err.Error() - return r - } - r.Stored = true - r.Answer = fmt.Sprintf("%d merges taken in by the running indexer %s", len(take), own) - if _, _, err := ix.evaluate(ctx, own, false); err != nil { - r.Answer += "; what waited behind it is not evaluated: " + err.Error() - } - return r -} - -// DropPending is the hand drop (indexer_drop_pending, ADR 0633 §5): the one pending merge under key, abandoned -// because a later merge replaced it, is removed and nothing of it is taken in. Its wait is said taken, naming the drop; -// the merges left under that wait are said pending again by the evaluation that follows. A merge that no later pending -// merge and no taken merge of its repository covers is refused unless abandon (issue 1966: #650, its repository's head, -// was dropped on the advice of a line that named a drop where nothing covered it; ADR 0668 §5). -func (ix *Indexer) DropPending(ctx context.Context, key, by, why string, abandon bool) TakenIn { - r := TakenIn{Module: key} - if ix.Pending == nil { - r.Answer = "the pending state is not on the bus: nothing waits" - return r - } - own := ix.ownBuildNow(ctx) - ix.pendingMu.Lock() - defer ix.pendingMu.Unlock() - entries, err := ReadPending(ctx, ix.Pending, ix.logf) - if err != nil { - r.Answer = err.Error() - return r - } - at := slices.IndexFunc(entries, func(e PendingEntry) bool { return e.Key == key }) - if at < 0 { - r.Answer = fmt.Sprintf("%s holds no merge under %s", PendingState, key) - return r - } - e := entries[at] - if !abandon { - // Covered: a later merge of the same repository, pending or taken in, has this merge's commit as an ancestor (ADR - // 0668 §6), asked of the forge (older); a later merge whose history does not hold it covers nothing. - covered := false - for _, o := range entries[at+1:] { - if o.Forge.Repository != e.Forge.Repository { - continue - } - in, err := ix.older(ctx, e.Forge.Repository, e.Forge.Merge, o.Forge.Merge) - if err != nil { - r.Answer = fmt.Sprintf("%s is not dropped: whether the later pending merge %s covers it is not read: %v", key, o.Key, err) - return r - } - if in { - covered = true - break - } - } - if !covered { - taken, err := ix.takenAlready(ctx, forgemsg.GroupMerge{Repository: e.Forge.Repository, Number: e.Forge.Number, Merge: e.Forge.Merge}) - if err != nil { - r.Answer = fmt.Sprintf("%s is not dropped: whether a taken merge of %s covers it is not read: %v", key, e.Forge.Repository, err) - return r - } - covered = taken - } - if !covered { - r.Answer = fmt.Sprintf("%s is not dropped: no later merge of %s, pending or taken in, has %s as an ancestor, so its "+ - "changes would never be taken in (ADR 0633 §5); %s reads its message again, or abandon: true drops it all the same", - key, e.Forge.Repository, e.Forge.Merge, TakeInPendingTool) - return r - } - } - if err := ix.Pending.Delete(ctx, e.Key, e.Revision); err != nil { - r.Answer = fmt.Sprintf("%s.%s is not removed: %v", PendingState, key, err) - return r - } - ix.forget(e) - ix.logf("take-in: %s (%s#%d at %s) dropped by hand, asked by %s: %s; nothing of it is taken in", key, e.Forge.Repository, e.Forge.Number, e.Forge.Merge, by, why) - if e.Value.Said != "" && e.Value.SaidKind == takein.TakeInPending { - var same []PendingEntry - for _, o := range entries { - if o.Key != key && o.Value.Said == e.Value.Said { - same = append(same, o) - } - } - for _, t := range saidWaits([]PendingEntry{e}) { - t.By, t.Why, t.Dropped = by, why, true - ix.sayTakeIn(ctx, t) - } - ix.markSaid(ctx, same, nil, func(p *Pending) { p.Said, p.SaidKind = "", "" }) - } - r.Stored, r.Answer = true, fmt.Sprintf("%s dropped: %s#%d at %s is not taken in", key, e.Forge.Repository, e.Forge.Number, e.Forge.Merge) - if own != "" { - if _, _, err := ix.evaluate(ctx, own, false); err != nil { - r.Answer += "; what waits is not evaluated again: " + err.Error() - } - } - return r -} - -// TakeInMerge is the hand take-in of one merge (indexer_take_in_merge, issue 1585): its merged message is asked of the forge -// bridge and handed to the take-in as a delivered merged message is (Arrive, under the stream sequence seq, which keeps the -// merge order), so it is held for the indexer its revision names or taken in, and a merge already held under that -// sequence is not written twice. For a merge the take-in never took in. The bridge's own words answer when it cannot give -// the message. -func (ix *Indexer) TakeInMerge(ctx context.Context, repository, merge string, seq uint64, by, why string) TakenIn { - r := TakenIn{Module: takein.PendingKey(seq)} - if ix.Message == nil { - r.Answer = "this indexer does not ask the forge bridge" - return r - } - data, err := ix.Message(forTakeIn(ctx, repository, merge), repository, merge, 0) - if err != nil { - r.Answer = fmt.Sprintf("the forge bridge could not answer for the merge of %s at %s: %v", repository, merge, err) - return r - } - f, err := forgemsg.DecodeForge(data) - if err != nil { - r.Answer = fmt.Sprintf("the forge bridge's answer for %s at %s is not a forge message: %v", repository, merge, err) - return r - } - if f.Kind != forgemsg.ForgeMerged || f.Merge != merge || f.Repository != repository { - r.Answer = fmt.Sprintf("the forge bridge's answer is not the merge of %s at %s", repository, merge) - return r - } - ix.logf("take-in: a hand take-in of the merge %s#%d at %s (sequence %d) asked by %s: %s", f.Repository, f.Number, f.Merge, seq, by, why) - a, words, err := ix.Arrive(ctx, seq, f) - if err != nil { - r.Answer = err.Error() - return r - } - r.Stored, r.Repository, r.Commit = true, f.Repository, f.Merge - r.Answer = fmt.Sprintf("%s#%d at %s handed to the take-in as a delivered merge (%d keys written)", f.Repository, f.Number, f.Merge, len(a.Written)) - if words != "" { - r.Answer += "; " + words - } - return r -} diff --git a/modules/mesh-delivery-indexer/internal/index/pending_tools.go b/modules/mesh-delivery-indexer/internal/index/pending_tools.go deleted file mode 100644 index ba52e019..00000000 --- a/modules/mesh-delivery-indexer/internal/index/pending_tools.go +++ /dev/null @@ -1,109 +0,0 @@ -package index - -import ( - "context" - "encoding/json" - "fmt" - "time" - - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein" - stdio "git.novox.be/novox/mesh-sdk/go" -) - -// PendingTools are the indexer's tools of what waits for a take-in (novox/hq ADR 0633 §5): indexer_pending reads the -// pending state, and indexer_take_in_pending and indexer_drop_pending steer it by an ask in takeins, as indexer_take_in -// does. pending is the pending state as the tools bundle reads it; ask writes an ask and waits for the indexer's answer. -func PendingTools(pending, ownBuild Store, ask func(key string, value []byte) (TakeInAsk, error), now func() time.Time) []stdio.Tool { - str := func(args map[string]any, k string) string { s, _ := args[k].(string); return s } - hand := func(drop bool) func(map[string]any) (any, error) { - return func(args map[string]any) (any, error) { - key, value, err := NewPendingAsk(str(args, "key"), str(args, "why"), stdio.CallerOf(args), drop, args["abandon"] == true, now()) - if err != nil { - return nil, err - } - a, err := ask(key, value) - if err != nil { - return nil, err - } - return a.Result, nil - } - } - merge := func(args map[string]any) (any, error) { - seq, _ := args["sequence"].(float64) - key, value, err := NewMergeAsk(str(args, "repository"), str(args, "merge"), uint64(seq), str(args, "why"), stdio.CallerOf(args), now()) - if err != nil { - return nil, err - } - a, err := ask(key, value) - if err != nil { - return nil, err - } - return a.Result, nil - } - return []stdio.Tool{ - {Name: PendingTool, - Description: "What waits for a take-in (ADR 0633): merges the indexer holds in its pending state instead of taking them in. " + - "A revision (what revision, its key its oldest merge's m.) waits only while its message is unread or lost or the " + - "objects are full. A group (what group, key g.) is a delivery group whose members are gathered until the " + - "forge bridge says its end; a merge in one of its members' repositories waits behind it (behind). Each names its " + - "merges, since when it waits, the modules it holds back (holds), the merges whose message the forge bridge's stream no " + - "longer holds (lost: they are not taken in; indexer_drop_pending drops them) and the last take-in event said of it. Empty waits: " + - "every merge is taken in. The watchman raises take-in-waits past 60 minutes for a revision, 30 for a group; " + - "indexer_take_in_pending and indexer_drop_pending steer a wait. own-build says which build the indexer knows it runs and own-build-since since when; it only names the writer, and no merge waits for it.", - Input: map[string]any{"type": "object", "properties": map[string]any{}}, - Run: func(map[string]any) (any, error) { - var unread []string - entries, err := ReadPending(context.Background(), pending, func(format string, args ...any) { - unread = append(unread, fmt.Sprintf(format, args...)) - }) - if err != nil { - return nil, err - } - answer := takein.PendingAnswer{Waits: PendingWaits(entries), Unread: unread} - // The build this indexer knows it runs, and since when. - if ownBuild != nil { - if v, _, found, err := ownBuild.Get(context.Background(), OwnBuildKey); err == nil && found { - var o OwnBuild - if json.Unmarshal(v, &o) == nil { - answer.OwnBuildSince = o.Since - if o.Known { - answer.OwnBuild = o.FP - } - } - } - } - return answer, nil - }}, - {Name: TakeInPendingTool, - Description: "Take in now a pending revision (its m. key), for a merge whose message is unread or lost or that is held " + - "for room, or a delivery group (its g. key) with the members gathered so far, for when its end never comes (ADR 0633 §5). " + - "A hand act: who and why are said in the indexer's journal and on the take-in.taken event, which clears take-in-waits. " + - "indexer_pending names the keys.", - Input: map[string]any{"type": "object", "required": []string{"key", "why"}, "properties": map[string]any{ - "key": map[string]any{"type": "string", "description": "the wait's key, m. or g., as indexer_pending names it"}, - "why": map[string]any{"type": "string", "description": "why it is taken in by hand, in plain words"}}}, - Run: hand(false)}, - {Name: DropPendingTool, - Description: "Drop one pending merge by hand, abandoned because a later merge replaced it (ADR 0633 §5): it is removed from " + - "the pending state and nothing of it is taken in; what waited with it is worked out again. A merge no later pending merge " + - "and no taken merge of its repository covers is refused, naming indexer_take_in_pending, which reads its message again; " + - "abandon: true drops it all the same. A hand act: who and why are said in the journal and on the take-in.taken event (dropped).", - Input: map[string]any{"type": "object", "required": []string{"key", "why"}, "properties": map[string]any{ - "key": map[string]any{"type": "string", "description": "the pending merge's key, m., as indexer_pending names its merges' wait"}, - "why": map[string]any{"type": "string", "description": "why it is dropped, in plain words"}, - "abandon": map[string]any{"type": "boolean", "description": "drop it though no later merge covers it; its changes are then never taken in"}}}, - Run: hand(true)}, - {Name: TakeInMergeTool, - Description: "Take one merge in by hand (ADR 0633 §5, issue 1585): the indexer asks the forge bridge for the merge's merged message " + - "(bridge_merged_message) and runs it through the take-in as a delivered merged message, so it is held for the " + - "indexer its revision names or taken in. For a merge the take-in did not take in (indexer_heard names its sequence " + - "and why). A hand act: who and why are said in the indexer's journal. The bridge's own words answer when it cannot " + - "build the message (the gate holds no such merge, the forge does not answer).", - Input: map[string]any{"type": "object", "required": []string{"repository", "merge", "sequence", "why"}, "properties": map[string]any{ - "repository": map[string]any{"type": "string", "description": "the repository, owner/name"}, - "merge": map[string]any{"type": "string", "description": "the merge commit, 40 hex digits"}, - "sequence": map[string]any{"type": "integer", "description": "the merge's sequence on the forge bridge's stream, as indexer_heard names it"}, - "why": map[string]any{"type": "string", "description": "why it is taken in by hand, in plain words"}}}, - Run: merge}, - } -} diff --git a/modules/mesh-delivery-indexer/internal/index/previews.go b/modules/mesh-delivery-indexer/internal/index/previews.go deleted file mode 100644 index 45036e59..00000000 --- a/modules/mesh-delivery-indexer/internal/index/previews.go +++ /dev/null @@ -1,376 +0,0 @@ -package index - -import ( - "context" - "fmt" - "maps" - "reflect" - "slices" - "strings" - - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdeliverychecker" - sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/grouppackages" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk" - - "mesh-delivery-indexer/internal/fingerprint" -) - -// The merge previews the checker writes (mesh-delivery-checker_previews; novox/hq ADR 0484 clause 3, issue 1445): a pull -// request's head merged onto its trunk's head, with the tree, the manifest digest and the lock hashes of each module -// directory. From one the indexer gives the candidate's touched module its new-form fingerprint, by the rules a module of -// the trunk is given its own (Recompute), or says why it has none. - -// The reasons a candidate's touched module has no fingerprint of the new form while a preview exists or is missing. -const ( - // NonePreviewStale is for a module whose preview was made onto a trunk head that is not the trunk's head now. - NonePreviewStale = "the merge preview is out of date: the trunk moved since it was made" - // NonePreviewConflict opens the reason of a module whose pull request's head does not merge into the trunk; the files - // in conflict follow. - NonePreviewConflict = "does not merge: the head conflicts with the trunk in " - // NonePreviewNoModule is for a module whose directory the preview names no manifest for: mesh-sdk's words, which the - // assigner reads as the change removing the module (issue 1297). - NonePreviewNoModule = CandidateNoneNoModule - // NonePreviewNoManifest is for a module whose manifest at the merge the indexer does not hold. - NonePreviewNoManifest = "the manifest of the merge is not kept" - // NoneNoDirectory is for a module whose directory in its repository neither the modules bucket nor the preview tells. - NoneNoDirectory = "the module's directory in its repository is not known" -) - -// PutPreview takes a change of the previews bucket in without writing any candidate: a put of a preview (key -// PreviewKey) or its delete. It answers whether the previews held changed. A key or a value that is not a -// preview is said in the log and left out. -func (ix *Indexer) PutPreview(key string, value []byte, deleted bool) bool { - if _, _, _, err := ParsePreviewKey(key); err != nil { - ix.logf("previews.%s is not a preview key, left out: %v", key, err) - return false - } - ix.mu.Lock() - defer ix.mu.Unlock() - if deleted { - if _, had := ix.previews[key]; !had { - return false - } - delete(ix.previews, key) - return true - } - pv, err := DecodePreview(value) - if err != nil { - ix.logf("previews.%s is not a preview, left out: %v", key, err) - return false - } - if ix.previews == nil { - ix.previews = map[string]meshdeliverychecker.PreviewsByRepositoryHeadTrunkValue{} - } - if cur, had := ix.previews[key]; had && SamePreview(cur, pv) { - return false - } - ix.previews[key] = pv - return true -} - -// ApplyPreview takes one change of the previews in (PutPreview) and, when the previews held changed, writes the -// candidates' fingerprints again (RecomputeCandidates). -func (ix *Indexer) ApplyPreview(ctx context.Context, key string, value []byte, deleted bool) error { - if !ix.PutPreview(key, value, deleted) { - return nil - } - return ix.RecomputeCandidates(ctx) -} - -// RecomputeCandidates writes again every candidate whose pull requests' fingerprints, from the previews held, are no longer -// what it holds. A candidate is written only when it differs, by compare-and-set on the revision read. -func (ix *Indexer) RecomputeCandidates(ctx context.Context) error { - if ix.Store == nil { - return nil - } - ix.mu.Lock() - none := len(ix.previews) == 0 - ix.mu.Unlock() - if none { - return nil // no preview held: every candidate has what the pull messages gave it - } - keys, err := ix.Store.Keys(ctx) - if err != nil { - return fmt.Errorf("listing the candidates: %w", err) - } - for _, key := range keys { - err := write(ctx, ix.Store, key, func(value []byte, found bool) ([]byte, bool, error) { - if !found { - return nil, false, nil - } - c, err := DecodeCandidate(value) - if err != nil { - ix.logf("candidates.%s holds what is not a candidate, left: %v", key, err) - return nil, false, nil - } - if !ix.withPreviews(ctx, &c) { - return nil, false, nil - } - data, err := EncodeCandidate(c) - return data, false, err - }) - if err != nil { - return fmt.Errorf("candidates.%s: %w", key, err) - } - } - return nil -} - -// previewOf is the preview of a head merged onto the trunk head the source heads name: found, or stale when only a preview -// onto another trunk head exists, or neither (found false, stale false) when the source is not known or none exists. -func (ix *Indexer) previewOf(heads map[string]string, repository, head string) (pv meshdeliverychecker.PreviewsByRepositoryHeadTrunkValue, found, stale bool) { - ix.mu.Lock() - defer ix.mu.Unlock() - trunkHead := heads[repository] - if trunkHead == "" { - return meshdeliverychecker.PreviewsByRepositoryHeadTrunkValue{}, false, false - } - if key, err := PreviewKey(repository, head, trunkHead); err == nil { - if pv, ok := ix.previews[key]; ok { - return pv, true, false - } - } - for _, pv := range ix.previews { - if pv.Repository == repository && pv.Head == head { - return meshdeliverychecker.PreviewsByRepositoryHeadTrunkValue{}, false, true - } - } - return meshdeliverychecker.PreviewsByRepositoryHeadTrunkValue{}, false, false -} - -// withPreviews gives each touched module of the candidate's pull requests the fingerprint its preview gives it, or the -// reason it has none, and the candidate its fpset. It answers whether the candidate changed. A module of a pull request with -// no preview keeps what it had. -// -// The candidate's Base is the source revision whose trunk heads the previews were read onto (novox/hq ADR 0437 §5): written -// whenever the fingerprints are, so the builder merges a preview onto the trunk head its fingerprint was computed on (the head -// of the pull request's repository in that revision), never onto a module value's commit, which stays at the commit its -// fingerprint first appeared at. -func (ix *Indexer) withPreviews(ctx context.Context, c *meshcatalog.CandidatesByGroupValue) bool { - ix.mu.Lock() - rev, heads := ix.source.rev, ix.source.heads - ix.mu.Unlock() - changed, read := false, false - // A package one of the candidate's pull requests publishes is built at that pull request's head, every other at its - // latest published version (novox/hq ADR 0573 §4): a group's members build against the group's own package. - members := map[string]string{} - for _, p := range c.Pulls { - members[trunk.RepositoryName(p.Repository)] = p.Head - } - var held map[string]meshcatalog.ModulesByModuleValue - listed := false - for i := range c.Pulls { - p := &c.Pulls[i] - pv, found, stale := ix.previewOf(heads, p.Repository, p.Head) - if !found && !stale { - continue - } - read = true - if !listed { - var err error - if held, err = ix.heldModules(ctx); err != nil { - ix.logf("the modules cannot be listed, so no touched module's directory or package version is known: %v", err) - } - listed = true - } - // A touched module's directory: the one the modules bucket holds of the repository, else the preview's directory - // whose manifest is the one the pull request gave the module (a module the pull request adds; ADR 0501 clause 6). - dirs := map[string]string{} - for name, m := range held { - if trunk.SameRepository(m.Repository, p.Repository) && m.Path != "" { - dirs[name] = m.Path - } - } - for _, m := range p.Modules { - if _, known := dirs[m]; known || p.Fps[m].Manifest == "" { - continue - } - for dir, pm := range pv.Modules { - if pm.Manifest == p.Fps[m].Manifest { - dirs[m] = dir - } - } - } - fps := maps.Clone(p.Fps) - for _, m := range p.Modules { - next := meshcatalog.CandidatesByGroupValuePullsItemFpsValue{Manifest: p.Fps[m].Manifest} - switch { - case stale: - next.None2 = NonePreviewStale - case len(pv.Conflicts) > 0: - next.None2 = NonePreviewConflict + strings.Join(pv.Conflicts, ", ") - default: - dir, known := dirs[m] - if !known { - next.None2 = NoneNoDirectory - break - } - next = ix.candidateModule(ctx, m, dir, pv, held, members) - } - if fps == nil { - fps = map[string]meshcatalog.CandidatesByGroupValuePullsItemFpsValue{} - } - if !reflect.DeepEqual(fps[m], next) { - fps[m] = next - changed = true - } - } - if changed || !maps.EqualFunc(fps, p.Fps, func(a, b meshcatalog.CandidatesByGroupValuePullsItemFpsValue) bool { return reflect.DeepEqual(a, b) }) { - p.Fps = fps - } - } - if changed { - c.Fpset = meshcatalogclient.FingerprintSet(c.Pulls) - } - if read && rev != 0 && c.Base != int64(rev) && (changed || c.Base == 0) { - c.Base, changed = int64(rev), true - } - return changed -} - -// candidateModule is the fingerprint of module m, whose directory is dir, at the preview: the module's inputs from the -// merge (tree, manifest, locks), its bases' and the builder's fingerprints as the trunk's modules hold them, the toolchains -// the build modules agree on, and the version of each mesh package it uses (candidatePackages), by the rules Recompute -// applies to a module of the trunk. -func (ix *Indexer) candidateModule(ctx context.Context, m, dir string, pv meshdeliverychecker.PreviewsByRepositoryHeadTrunkValue, held map[string]meshcatalog.ModulesByModuleValue, members map[string]string) meshcatalog.CandidatesByGroupValuePullsItemFpsValue { - pm, ok := pv.Modules[dir] - if !ok && dir == "" { - pm, ok = pv.Modules[meshcatalogclient.RootPath] - } - if !ok && dir == meshcatalogclient.RootPath { - pm, ok = pv.Modules[""] - } - if !ok { - return meshcatalog.CandidatesByGroupValuePullsItemFpsValue{None2: NonePreviewNoModule} - } - out := meshcatalog.CandidatesByGroupValuePullsItemFpsValue{Manifest: pm.Manifest} - if ix.Manifests == nil { - out.None2 = NonePreviewNoManifest - return out - } - value, _, found, err := ix.Manifests.Get(ctx, pm.Manifest) - if err != nil || !found { - out.None2 = NonePreviewNoManifest - return out - } - kept, err := meshcatalogclient.DecodeManifest(value) - if err != nil { - out.None2 = NonePreviewNoManifest - return out - } - // The preview carries the directory's raw inputs (its tree, the named paths' objects, the lock hashes): the indexer - // computes the named tree (ADR 0593 §2). The version line's packages are the trunk module's, unless an older build - // module still names them. - trunkMod := held[m] - packages := fingerprint.OtherPackages(pm.Packages, pv.Repository) - if trunkMod.Inputs2 != nil { - if len(pm.Packages) == 0 { - packages = trunkMod.Inputs2.Packages - } else { - // The siblings its go.mod replaces by a directory are the trunk module's: no go.sum names them. - for _, p := range trunkMod.Inputs2.Packages { - if repo, _, ok := trunk.PackageRepository(p); ok && trunk.SameRepository(repo, pv.Repository) { - packages = append(packages, p) - } - } - } - } - in := inputsOfNamed(m, kept.Manifest, pm.Tree, pm.Locks, pm.Paths, packages, ix.baseFP(ctx)) - if in.Kept == nil { - out.None2 = in.None2 - return out - } - if in.None2 != "" && in.None2 != sdkstep.ToolchainNoReport && in.None2 != None2Pending { - out.None2 = in.None2 // a context, or a base with no fingerprint - return out - } - tools := ix.agreed() - toolchains := map[string]meshcatalogclient.Toolchain2{} - for _, artifact := range slices.Sorted(maps.Keys(in.Languages)) { - t := tools[in.Languages[artifact]] - if t.none != "" { - out.None2 = t.none - return out - } - toolchains[artifact] = t.tc - } - inputs := *in.Kept // no builder input (ADR 0530 §1) - var fp, none string - var versions sdkstep.PackageVersions - if trunkMod.Inputs2 != nil && len(trunkMod.Inputs2.Imports) > 0 { - inputs.Imports, inputs.ImportForm = trunkMod.Inputs2.Imports, trunkMod.Inputs2.ImportForm - } - // The import form against the group's own heads of the packages it publishes (ADR 0573 §4, ADR 0593 §2), as the trunk - // computes it once merged, the trunk's held fingerprint standing where nothing the build was built from changed. - published := ix.candidatePublished(ctx, members, held) - // A package of a repository outside the group by the series the candidate's manifest accepts (DLV-I6); a member's by the - // group's head. - var pins sdkstep.PackageVersions - if _, man, err := ix.manifestRaw(ctx, pm.Manifest); err == nil { - series := ix.SeriesAll(ctx) - for key, s := range series { - if repo, _, _ := trunk.PackageRepository(s.GoModule); members[repo] != "" { - delete(series, key) - } - } - published, pins = acceptedSeries(series, man, pv.Repository, published) - } - if len(inputs.Imports) > 0 && len(fingerprint.Resolve(pv.Repository, inputs.Imports, published).Missing) == 0 { - f := formOf(m, pv.Repository, &inputs, toolchains, trunkMod, held, published, pins, ix.sourceHeads(), nil) - fp, none, versions = f.fp, f.none, f.versions - } else { - inputs.Imports, inputs.ImportForm = nil, "" - versions = candidatePackages(fingerprint.OtherPackages(inputs.Packages, pv.Repository), held, members, ix.sourceHeads()) - for p, v := range pins { - if _, named := versions[p]; named || slices.Contains(inputs.Packages, p) { - if versions == nil { - versions = sdkstep.PackageVersions{} - } - versions[p] = v - } - } - in := inputsFrom(m, inputs, toolchains) - in.Packages = fingerprint.VersionLine(pv.Repository, inputs.Packages, versions, published) - fp, none = fingerprint.Of(in) - } - if fp == "" { - out.None2 = none - return out - } - // The ask of the candidate's build names every member of the group at its head, also one the module names no package of - // (a member that publishes none yet, or one the module reaches through mesh-sdk): the fingerprint stays the trunk's, so - // the build is the one the merge keeps (grouppackages.Heads, issue 1921). - out.Fp2, out.Packages = fp, grouppackages.Heads(versions, members, pv.Repository) - if len(inputs.Paths) > 0 { - out.Paths = inputs.Paths // the builder's fetch for the candidate names them (ADR 0499 clause 4) - } - return out -} - -// candidatePackages is the version each package of a candidate's module is built against (novox/hq ADR 0573 §2, §4): the -// head of the candidate's pull request whose repository publishes it (members, by repository), else its latest published -// version, as the trunk's modules hold it, else the commit the gate's source names for its repository (heads; design 47 G A-365). A package with none is left out, and the fingerprint is then none. -func candidatePackages(packages []string, held map[string]meshcatalog.ModulesByModuleValue, members, heads map[string]string) sdkstep.PackageVersions { - versions, _ := fingerprint.HeldVersionsOf(packages, held) - for _, p := range packages { - repository, _, _ := trunk.PackageRepository(p) - if head := members[repository]; head != "" { - if versions == nil { - versions = sdkstep.PackageVersions{} - } - versions[p] = head - } else if _, named := versions[p]; !named { - // A package no module publishes, of a repository outside the group: the commit the gate's source names (A-365). - if head := heads[repository]; len(head) == 40 { - if versions == nil { - versions = sdkstep.PackageVersions{} - } - versions[p] = head - } - } - } - return versions -} diff --git a/modules/mesh-delivery-indexer/internal/index/previews_test.go b/modules/mesh-delivery-indexer/internal/index/previews_test.go deleted file mode 100644 index 61e7ee40..00000000 --- a/modules/mesh-delivery-indexer/internal/index/previews_test.go +++ /dev/null @@ -1,346 +0,0 @@ -package index - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "maps" - "os" - "strings" - "testing" - - sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" - - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdeliverychecker" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" - - "mesh-delivery-indexer/internal/fingerprint" -) - -const ( - trunkNow = "5555555555555555555555555555555555555555" - trunkOld = "4444444444444444444444444444444444444444" - mergeSha = "6666666666666666666666666666666666666666" -) - -// A trunk of the catalogue with avahi (and the builder, a base and others) merged, the toolchain agreed, and the gate's source -// naming the catalogue's trunk head: avahi has a fingerprint of the new form at tree-avahi. -func previewBed(t *testing.T, report bool) (*Indexer, *memStore, *memStore) { - t.Helper() - ix, mods := newFormIndexer(t, builderModules(t)...) - if report { - reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1") - } - if err := applyS(ix, context.Background(), 5, srcT{Group: "g", Heads: map[string]string{catalogueRepo: trunkNow}}); err != nil { - t.Fatal(err) - } - return ix, mods, ix.Store.(*memStore) -} - -// avahiOnTools is avahi's manifest as builderModules has it: standing on mesh-tools. -func avahiOnTools(t *testing.T) string { - return standingOn(t, catalogueManifest(t, "avahi"), "mesh-tools") -} - -// avahiPull is the pull message of the catalogue's pull request 301 touching avahi, with its manifest. -func avahiPull(t *testing.T, head string) forgemsg.Forge { - t.Helper() - f := manifestPull(t, 301, head, avahiOnTools(t)) - f.Manifests = map[string]string{"modules/avahi": avahiOnTools(t)} - return f -} - -func avahiPreview(t *testing.T, trunk, head string) meshdeliverychecker.PreviewsByRepositoryHeadTrunkValue { - return meshdeliverychecker.PreviewsByRepositoryHeadTrunkValue{Repository: catalogueRepo, Head: head, Trunk: trunk, Merge: mergeSha, Modules: map[string]meshdeliverychecker.PreviewsByRepositoryHeadTrunkValueModulesValue{ - "modules/avahi": {Tree: "tree-avahi", Manifest: meshcatalogclient.ManifestDigest([]byte(avahiOnTools(t))), Locks: map[string]string{"go.sum": avahiGoSum}}}} -} - -func applyPreview(t *testing.T, ix *Indexer, pv meshdeliverychecker.PreviewsByRepositoryHeadTrunkValue) { - t.Helper() - key, err := PreviewKey(pv.Repository, pv.Head, pv.Trunk) - if err != nil { - t.Fatal(err) - } - data, err := EncodePreview(pv) - if err != nil { - t.Fatal(err) - } - if err := ix.ApplyPreview(context.Background(), key, data, false); err != nil { - t.Fatal(err) - } -} - -func fpOf(t *testing.T, st *memStore) meshcatalog.CandidatesByGroupValuePullsItemFpsValue { - t.Helper() - c := candidateOf(t, st) - if len(c.Pulls) != 1 { - t.Fatalf("candidate = %+v", c) - } - return c.Pulls[0].Fps["avahi"] -} - -func TestACandidatesModuleGetsTheFingerprintATrunkModuleOfTheSameTreeHas(t *testing.T) { // 1445/AC2 - ix, mods, cands := previewBed(t, true) - trunk, _ := moduleOf(t, mods, "avahi") - if trunk.Fp2 == "" { - t.Fatalf("the trunk's avahi has no fingerprint of the new form: %+v", trunk) - } - if err := ix.Apply(context.Background(), avahiPull(t, h1)); err != nil { - t.Fatal(err) - } - if got := fpOf(t, cands); got.None2 != meshcatalogclient.NoneNoPreview { - t.Fatalf("before the preview: %+v", got) - } - applyPreview(t, ix, avahiPreview(t, trunkNow, h1)) - got := fpOf(t, cands) - if got.Fp2 != trunk.Fp2 || got.None2 != "" || got.Manifest != meshcatalogclient.ManifestDigest([]byte(avahiOnTools(t))) { - t.Fatalf("candidate avahi = %+v, want the trunk's fingerprint %s", got, trunk.Fp2) - } - c := candidateOf(t, cands) - if c.Fpset != meshcatalogclient.FingerprintSet(c.Pulls) || c.Fpset == "" { - t.Errorf("fpset = %q", c.Fpset) - } - // The candidate names the source revision the preview was read onto (5, whose catalogue head is the preview's trunk): - // the builder merges the preview onto that head, never onto the module value's commit. - if c.Base != 5 { - t.Errorf("candidate base = %d, want the source revision 5 its preview was read onto", c.Base) - } - written(t, writtenCandidate{Source: sourceRev{Rev: 5, Heads: map[string]string{catalogueRepo: trunkNow}}, Candidate: c}) - // Issue 1921: a candidate's build is asked every member of the group at its head, also one that publishes no package yet - // and one the module names no package of; the fingerprint stays the trunk's, so the build is the one the merge keeps. - for _, tc := range []struct { - name string - members map[string]string // the other pull requests of the group: repository, head - want sdkstep.PackageVersions - }{ - {"a group of one names nothing", nil, nil}, - {"a member that publishes no package yet", map[string]string{"novox/mesh-contracts": h2}, - sdkstep.PackageVersions{"git.novox.be/novox/mesh-contracts": h2}}, - {"mesh-sdk by its go module", map[string]string{"novox/mesh-sdk": h2, "novox/mesh-contracts": h3}, - sdkstep.PackageVersions{"git.novox.be/novox/mesh-sdk/go": h2, "git.novox.be/novox/mesh-contracts": h3}}, - } { - ix, mods, cands := previewBed(t, true) - trunk, _ := moduleOf(t, mods, "avahi") - if err := ix.Apply(context.Background(), avahiPull(t, h1)); err != nil { - t.Fatal(err) - } - for repository, head := range tc.members { - other := forgemsg.Forge{V: forgemsg.Version, Kind: forgemsg.ForgePull, Repository: repository, Branch: "feat/x", Number: 7, Head: head, - Action: forgemsg.PullUpdated} - if err := ix.Apply(context.Background(), other); err != nil { - t.Fatal(err) - } - } - applyPreview(t, ix, avahiPreview(t, trunkNow, h1)) - var got meshcatalog.CandidatesByGroupValuePullsItemFpsValue - for _, p := range candidateOf(t, cands).Pulls { - if p.Repository == catalogueRepo { - got = p.Fps["avahi"] - } - } - if got.Fp2 != trunk.Fp2 || !maps.Equal(got.Packages, tc.want) { - t.Errorf("%s: candidate avahi = %+v, want the trunk's fingerprint %s and the packages %v", tc.name, got, trunk.Fp2, tc.want) - } - } - // Another tree gives another fingerprint. - pv := avahiPreview(t, trunkNow, h1) - m := pv.Modules["modules/avahi"] - m.Tree = "tree-changed" - pv.Modules["modules/avahi"] = m - pv.Merge = "7777777777777777777777777777777777777777" - applyPreview(t, ix, pv) - if other := fpOf(t, cands); other.Fp2 == "" || other.Fp2 == trunk.Fp2 { - t.Errorf("a changed tree: %+v", other) - } - // Issue 1585 (ADR 0573 §2, §4): a candidate's module is built against the head of the candidate's pull request that - // publishes a package it uses, else against the package's latest published version; a preview naming a package of - // mesh-sdk that nothing published gives no fingerprint, and one of its own repository's comes with its tree. - const sdk, own = "git.novox.be/novox/mesh-sdk/go", "git.novox.be/novox/mesh-catalog/contract" - held := map[string]meshcatalog.ModulesByModuleValue{"mesh-sdk": {Repository: "novox/mesh-sdk", Path: meshcatalogclient.RootPath, Commit: trunkOld}} - if got := candidatePackages([]string{sdk}, held, map[string]string{catalogueRepo: h1}, nil); got[sdk] != trunkOld { - t.Errorf("outside a group: %v, want mesh-sdk at its trunk %s", got, trunkOld) - } - if got := candidatePackages([]string{sdk}, held, map[string]string{catalogueRepo: h1, "novox/mesh-sdk": mergeSha}, nil); got[sdk] != mergeSha { - t.Errorf("in a group with mesh-sdk: %v, want its member's head %s", got, mergeSha) - } - // Design 47 G A-365: a package no module publishes, of a repository outside the group, takes the commit the source names. - if got := candidatePackages([]string{sdk}, nil, map[string]string{catalogueRepo: h1}, map[string]string{"novox/mesh-sdk": strings.Repeat("9", 40)}); got[sdk] != strings.Repeat("9", 40) { - t.Errorf("an unpublished package outside the group: %v, want the source's head", got) - } - m.Packages = []string{own, sdk} - pv.Modules["modules/avahi"] = m - pv.Merge = "8888888888888888888888888888888888888888" - applyPreview(t, ix, pv) - if got := fpOf(t, cands); got.Fp2 != "" || got.None2 != fingerprint.NoPackageVersion+sdk { - t.Errorf("a package nothing published: %+v", got) - } - // Through the indexer's wiring (A-365): once the gate's source names mesh-sdk's head, the candidate is built against it. - if err := applyS(ix, context.Background(), 5, srcT{Group: "g", Heads: map[string]string{catalogueRepo: trunkNow, "novox/mesh-sdk": strings.Repeat("9", 40)}}); err != nil { - t.Fatal(err) - } - if got := fpOf(t, cands); got.Fp2 == "" { - t.Errorf("a package nothing published, the source naming its repository: %+v, want a fingerprint", got) - } -} - -func TestAnUnknownToolchainLeavesTheCandidatesModuleNamingIt(t *testing.T) { // 1445/AC2 - ix, _, cands := previewBed(t, false) - if err := ix.Apply(context.Background(), avahiPull(t, h1)); err != nil { - t.Fatal(err) - } - applyPreview(t, ix, avahiPreview(t, trunkNow, h1)) - if got := fpOf(t, cands); got.Fp2 != "" || !strings.Contains(got.None2, sdkstep.ToolchainNoReport) { - t.Errorf("candidate avahi = %+v, want none2 naming %q", got, sdkstep.ToolchainNoReport) - } -} - -func TestAPreviewMadeOntoAnOldTrunkIsOutOfDate(t *testing.T) { // 1445/AC3 - ix, _, cands := previewBed(t, true) - if err := ix.Apply(context.Background(), avahiPull(t, h1)); err != nil { - t.Fatal(err) - } - applyPreview(t, ix, avahiPreview(t, trunkOld, h1)) - if got := fpOf(t, cands); got.Fp2 != "" || got.None2 != NonePreviewStale { - t.Errorf("candidate avahi = %+v, want it out of date", got) - } - // A preview onto the trunk head now makes it good again, and the trunk moving again makes it stale. - applyPreview(t, ix, avahiPreview(t, trunkNow, h1)) - if got := fpOf(t, cands); got.Fp2 == "" { - t.Fatalf("after the right preview: %+v", got) - } - if err := applyS(ix, context.Background(), 6, srcT{Group: "g", Heads: map[string]string{catalogueRepo: "8888888888888888888888888888888888888888"}}); err != nil { - t.Fatal(err) - } - if got := fpOf(t, cands); got.None2 != NonePreviewStale { - t.Errorf("after the trunk moved: %+v", got) - } -} - -func TestAPreviewHeardBeforeItsPullRequestGivesTheFingerprintWhenTheCandidateIsWritten(t *testing.T) { // 1445/AC2 - ix, mods, cands := previewBed(t, true) - applyPreview(t, ix, avahiPreview(t, trunkNow, h1)) - if err := ix.Apply(context.Background(), avahiPull(t, h1)); err != nil { - t.Fatal(err) - } - trunk, _ := moduleOf(t, mods, "avahi") - if got := fpOf(t, cands); got.Fp2 != trunk.Fp2 { - t.Errorf("candidate avahi = %+v, want %s", got, trunk.Fp2) - } - // The same pull request at a new head has no preview of it yet. - if err := ix.Apply(context.Background(), avahiPull(t, h2)); err != nil { - t.Fatal(err) - } - if got := fpOf(t, cands); got.Fp2 != "" || got.None2 != meshcatalogclient.NoneNoPreview { - t.Errorf("a new head: %+v, want no preview", got) - } - // Issue 1921: a head whose tree lacks the directory asked is asked once, however many passes read it, until the head changes; - // any other failure (the forge does not answer) is asked again at every pass and said once per head. - for _, row := range []struct { - name string - err error - askedAfterTwo int // the same head read twice - askedAfterNew int // then another head - linesAfterNew int - }{ - {"the directory is not in the tree", fmt.Errorf("listing: %w", ErrNotInTree), 1, 2, 2}, - {"the forge does not answer", errors.New("the forge does not answer"), 2, 3, 2}, - } { - t.Run("a failing head: "+row.name, func(t *testing.T) { - ix, _, _ := previewBed(t, false) - var logs strings.Builder - ix.Logf = func(format string, args ...any) { logs.WriteString(fmt.Sprintf(format, args...) + "\n") } - asked := 0 - ix.Files = func(context.Context, string, string, []string) (map[string]map[string]string, error) { - asked++ - return nil, row.err - } - held := map[string]meshcatalog.ModulesByModuleValue{"x": {Inputs2: &meshcatalog.ModulesByModuleValueInputs2{Packages: []string{"git.novox.be/novox/mesh-sdk/go"}}}} - for pass, head := range []string{h1, h1, h2} { - ix.candidatePublished(context.Background(), map[string]string{"novox/mesh-sdk": head}, held) - if pass == 1 && (asked != row.askedAfterTwo || strings.Count(logs.String(), "its packages are not read") != 1) { - t.Fatalf("after two passes over %s: asked %d (want %d), journal %q: want one line", head, asked, row.askedAfterTwo, logs.String()) - } - } - if asked != row.askedAfterNew || strings.Count(logs.String(), "its packages are not read") != row.linesAfterNew { - t.Errorf("asked %d (want %d), journal %q (want %d lines)", asked, row.askedAfterNew, logs.String(), row.linesAfterNew) - } - }) - } -} - -func TestAPreviewNamingNoManifestOfTheModulesDirectoryGivesAReason(t *testing.T) { - t.Run("the same preview again", testTheSamePreviewAppliedAgainWritesNothing) - ix, _, cands := previewBed(t, true) - if err := ix.Apply(context.Background(), avahiPull(t, h1)); err != nil { - t.Fatal(err) - } - pv := avahiPreview(t, trunkNow, h1) - pv.Modules = map[string]meshdeliverychecker.PreviewsByRepositoryHeadTrunkValueModulesValue{"modules/other": pv.Modules["modules/avahi"]} - applyPreview(t, ix, pv) - if got := fpOf(t, cands); got.None2 != NonePreviewNoModule || !strings.Contains(NonePreviewNoModule, "manifest") { - t.Errorf("candidate avahi = %+v", got) - } -} - -func testTheSamePreviewAppliedAgainWritesNothing(t *testing.T) { - ix, _, cands := previewBed(t, true) - if err := ix.Apply(context.Background(), avahiPull(t, h1)); err != nil { - t.Fatal(err) - } - applyPreview(t, ix, avahiPreview(t, trunkNow, h1)) - w := cands.writes - applyPreview(t, ix, avahiPreview(t, trunkNow, h1)) - if cands.writes != w { - t.Errorf("candidates written %d times for the same preview, want none", cands.writes-w) - } -} - -func TestAPreviewOfConflictsGivesTheCandidatesModuleTheReasonItDoesNotMerge(t *testing.T) { // 1445/AC5 - ix, _, cands := previewBed(t, true) - if err := ix.Apply(context.Background(), avahiPull(t, h1)); err != nil { - t.Fatal(err) - } - applyPreview(t, ix, meshdeliverychecker.PreviewsByRepositoryHeadTrunkValue{Repository: catalogueRepo, Head: h1, Trunk: trunkNow, Conflicts: []string{"modules/avahi/main.go", "go.mod"}}) - got := fpOf(t, cands) - if got.Fp2 != "" || !strings.HasPrefix(got.None2, "does not merge") || !strings.Contains(got.None2, "go.mod, modules/avahi/main.go") && !strings.Contains(got.None2, "modules/avahi/main.go, go.mod") { - t.Errorf("candidate avahi = %+v, want it not merging and naming the files", got) - } -} - -// writtenCandidate is what the builder's tests read of the indexer (mesh-delivery-builder internal/build): a candidate as the -// indexer writes it and the gate's source revision it names, kept in testdata/candidate-written.json so the builder is tested -// on what the indexer writes, never on a candidate made by hand. MESH_WRITE_TESTDATA=1 writes it again. -type writtenCandidate struct { - Source sourceRev `json:"source"` - Candidate meshcatalog.CandidatesByGroupValue `json:"candidate"` -} - -type sourceRev struct { - Rev uint64 `json:"rev"` - Heads map[string]string `json:"heads"` -} - -func written(t *testing.T, w writtenCandidate) { - t.Helper() - data, err := json.MarshalIndent(w, "", " ") - if err != nil { - t.Fatal(err) - } - data = append(data, '\n') - const file = "testdata/candidate-written.json" - if os.Getenv("MESH_WRITE_TESTDATA") == "1" { - if err := os.WriteFile(file, data, 0o644); err != nil { - t.Fatal(err) - } - } - kept, err := os.ReadFile(file) - if err != nil { - t.Fatal(err) - } - if string(kept) != string(data) { - t.Errorf("%s is not what the indexer writes now (MESH_WRITE_TESTDATA=1 writes it again):\n%s", file, data) - } -} diff --git a/modules/mesh-delivery-indexer/internal/index/previews_values.go b/modules/mesh-delivery-indexer/internal/index/previews_values.go deleted file mode 100644 index c6fa71ec..00000000 --- a/modules/mesh-delivery-indexer/internal/index/previews_values.go +++ /dev/null @@ -1,75 +0,0 @@ -package index - -import ( - "encoding/json" - "fmt" - "reflect" - "strings" - - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliverycheckerclient" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdeliverychecker" - "git.novox.be/novox/mesh-sdk/go/keys" -) - -// A preview as the checker writes it into its state previews, read here (reads: mesh-delivery-checker.previews): the -// seat's type and the client's rules; the key is the form the seat names, `..`, the repository -// as a key part (keys.EscapeName). - -// PreviewKey is the key of a head merged onto a trunk commit. -func PreviewKey(repository, head, trunk string) (string, error) { - if err := meshdeliverycheckerclient.CheckRepository(repository); err != nil { - return "", err - } - if err := keys.CheckName("head", head); err != nil { - return "", err - } - if err := keys.CheckName("trunk commit", trunk); err != nil { - return "", err - } - return meshdeliverychecker.PreviewsByRepositoryHeadTrunkKey(keys.EscapeName(repository), head, trunk), nil -} - -// ParsePreviewKey is the inverse of PreviewKey. -func ParsePreviewKey(key string) (repository, head, trunk string, err error) { - parts := strings.Split(key, ".") - if len(parts) != 3 { - return "", "", "", fmt.Errorf("index: %q is not a preview key, ..", key) - } - repository, err = keys.UnescapeName(parts[0]) - if err != nil { - return "", "", "", fmt.Errorf("index: %q is not a preview key: %w", key, err) - } - if _, err := PreviewKey(repository, parts[1], parts[2]); err != nil { - return "", "", "", fmt.Errorf("index: %q is not a preview key: %w", key, err) - } - return repository, parts[1], parts[2], nil -} - -// DecodePreview reads a preview. -func DecodePreview(data []byte) (meshdeliverychecker.PreviewsByRepositoryHeadTrunkValue, error) { - if err := keys.CheckVersion(data); err != nil { - return meshdeliverychecker.PreviewsByRepositoryHeadTrunkValue{}, err - } - p, err := meshdeliverychecker.DecodePreviewsByRepositoryHeadTrunkValue(data) - if err != nil { - return p, fmt.Errorf("index: a preview does not fit its shape: %w", err) - } - if err := meshdeliverycheckerclient.CheckPreview(p); err != nil { - return meshdeliverychecker.PreviewsByRepositoryHeadTrunkValue{}, err - } - return p, nil -} - -// EncodePreview is p as JSON at the kept values' version (the checker writes previews; the indexer's tests do too). -func EncodePreview(p meshdeliverychecker.PreviewsByRepositoryHeadTrunkValue) ([]byte, error) { - if err := meshdeliverycheckerclient.CheckPreview(p); err != nil { - return nil, err - } - p.V = keys.Version - return json.Marshal(p) -} - -// SamePreview says whether two previews are the same value. -func SamePreview(a, b meshdeliverychecker.PreviewsByRepositoryHeadTrunkValue) bool { - return reflect.DeepEqual(a, b) -} diff --git a/modules/mesh-delivery-indexer/internal/index/registered.go b/modules/mesh-delivery-indexer/internal/index/registered.go index f9c86c41..7f52fad2 100644 --- a/modules/mesh-delivery-indexer/internal/index/registered.go +++ b/modules/mesh-delivery-indexer/internal/index/registered.go @@ -10,9 +10,7 @@ import ( "strings" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk" - "git.novox.be/novox/mesh-sdk/go/keys" ) // A module registered in the catalogue from its own repository (the operator registers it with the repository and the @@ -130,7 +128,7 @@ func (ix *Indexer) indexAt(ctx context.Context, r Registration, force, reread bo t.why = fmt.Sprintf("the manifest at %q of %s at %s names the module %q", dir, repository, commit, name) return t, nil } - f := forgemsg.Forge{V: keys.Version, Kind: forgemsg.ForgeMerged, Repository: repository, Merge: commit, Root: read.Root, + f := whole{Repository: repository, Merge: commit, Root: read.Root, Trees: map[string]string{dir: read.Trees[dir]}, Manifests: map[string]string{dir: raw}} if l := read.Locks[dir]; len(l) > 0 { @@ -410,7 +408,7 @@ func (ix *Indexer) TakeInRepository(ctx context.Context, repository string) Take out.Answer = "the modules state cannot be read: " + err.Error() return out } - f := forgemsg.Forge{V: keys.Version, Kind: forgemsg.ForgeMerged, Repository: out.Repository, Merge: head} + f := whole{Repository: out.Repository, Merge: head} if _, err := ix.ApplyMessage(ctx, f); err != nil { out.Answer = fmt.Sprintf("the merge of %s at %.12s could not be taken in: %v", repository, head, err) return out diff --git a/modules/mesh-delivery-indexer/internal/index/registered_test.go b/modules/mesh-delivery-indexer/internal/index/registered_test.go index d872c3ba..c7f035b8 100644 --- a/modules/mesh-delivery-indexer/internal/index/registered_test.go +++ b/modules/mesh-delivery-indexer/internal/index/registered_test.go @@ -225,8 +225,7 @@ func TestAForgeThatCannotSayLeavesTheModuleUnwrittenAndAsksAgain(t *testing.T) { for _, tc := range []struct{ failure, trigger string }{ {"timeout", "the indexer starts again"}, {"Gitea API /repos/x/git/trees/y: 404 not found", "the indexer starts again"}, - {"Gitea API /repos/x/git/trees/y: 404 not found", "a new source revision"}, - {"Gitea API /repos/x/git/trees/y: 404 not found", "a merged message of another repository"}, + {"Gitea API /repos/x/git/trees/y: 404 not found", "a take-in of another repository"}, {"Gitea API /repos/x/git/trees/y: 404 not found", "a person's take-in"}, {"timeout", "a person's take-in"}, } { @@ -259,12 +258,7 @@ func TestAForgeThatCannotSayLeavesTheModuleUnwrittenAndAsksAgain(t *testing.T) { if err := ix.IndexRegisteredAll(ctx); err != nil { t.Fatal(err) } - case "a new source revision": - if err := applyS(ix, ctx, 2, srcT{Heads: map[string]string{"jschoubben/snake": h2, "novox/other": h1}}); err != nil { - t.Fatal(err) - } - ix.retryWG.Wait() // read again in the background - case "a merged message of another repository": + case "a take-in of another repository": if err := ix.Apply(ctx, mergedMsg("novox/other", map[string]string{"": "t9"})); err != nil { t.Fatal(err) } diff --git a/modules/mesh-delivery-indexer/internal/index/runcandidates.go b/modules/mesh-delivery-indexer/internal/index/runcandidates.go index df18d8d2..bfcfead8 100644 --- a/modules/mesh-delivery-indexer/internal/index/runcandidates.go +++ b/modules/mesh-delivery-indexer/internal/index/runcandidates.go @@ -24,7 +24,6 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk" "git.novox.be/novox/mesh-sdk/go/keys" ) @@ -237,7 +236,7 @@ func (ix *Indexer) writeRunCandidate(ctx context.Context, group string, members if name := manifestModule(raw); name != m.Module { return fmt.Errorf("%w: the run %s: the manifest at %q of %s at %.12s names %q, not %s", ErrUnreadable, group, m.dir, m.repository, m.Commit, name, m.Module) } - f := forgemsg.Forge{V: keys.Version, Kind: forgemsg.ForgeMerged, Repository: m.repository, Merge: m.Commit, Root: read.Root, + f := whole{Repository: m.repository, Merge: m.Commit, Root: read.Root, Trees: map[string]string{m.dir: read.Trees[m.dir]}, Manifests: map[string]string{m.dir: raw}} if l := read.Locks[m.dir]; len(l) > 0 { f.Locks = map[string]map[string]string{m.dir: l} diff --git a/modules/mesh-delivery-indexer/internal/index/seatdefs.go b/modules/mesh-delivery-indexer/internal/index/seatdefs.go index 7823db18..e71095d4 100644 --- a/modules/mesh-delivery-indexer/internal/index/seatdefs.go +++ b/modules/mesh-delivery-indexer/internal/index/seatdefs.go @@ -31,7 +31,6 @@ import ( mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/seatdefs" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk" "git.novox.be/novox/mesh-sdk/go/keys" @@ -149,42 +148,15 @@ func digestOf(d seatdef.Definition) (json.RawMessage, string, error) { return raw, hex.EncodeToString(sum[:]), nil } -// ApplySeatDefinitions keeps the seat definitions a forge message carries (DLV-I17, I18): a merged message's contracts -// modules at its merge as the trunk's, a pull message's at its head as that pull request's; a closed pull request, or -// a merged one, has its entries removed. Nothing is done without the state or a forge reader. -func (ix *Indexer) ApplySeatDefinitions(ctx context.Context, f forgemsg.Forge) error { - if ix.SeatDefinitions == nil { +// ApplySeatDefinitions keeps the seat definitions of the contracts modules a take-in's trees carry (DLV-I17, I18) as the +// trunk's at its commit. Nothing is done without the state or a forge reader. +func (ix *Indexer) ApplySeatDefinitions(ctx context.Context, f whole) error { + if ix.SeatDefinitions == nil || ix.ReadFile == nil { return nil } - if f.Kind == forgemsg.ForgeMerged || (f.Kind == forgemsg.ForgePull && f.Action == forgemsg.PullClosed) { - if f.Number > 0 { - if err := ix.dropPull(ctx, f.Repository, f.Number); err != nil { - return err - } - } - if f.Kind != forgemsg.ForgeMerged { - return nil - } - } - if ix.ReadFile == nil || (f.Kind != forgemsg.ForgeMerged && f.Kind != forgemsg.ForgePull) { - return nil - } - manifests := f.Manifests - if f.Kind == forgemsg.ForgePull && len(f.Dirs) > 0 { - manifests = map[string]string{} - for _, d := range f.Dirs { - if raw, ok := f.Manifests[d]; ok { - manifests[d] = raw - } - } - } var errs []error - for _, c := range contractsModules(manifests) { - if f.Kind == forgemsg.ForgeMerged { - errs = append(errs, ix.keepTrunk(ctx, f.Repository, f.Merge, c)) - } else { - errs = append(errs, ix.keepPull(ctx, f.Repository, f.Number, f.Head, c)) - } + for _, c := range contractsModules(f.Manifests) { + errs = append(errs, ix.keepTrunk(ctx, f.Repository, f.Merge, c)) } return errors.Join(errs...) } diff --git a/modules/mesh-delivery-indexer/internal/index/seatsof.go b/modules/mesh-delivery-indexer/internal/index/seatsof.go index e32fcd5a..1cc99289 100644 --- a/modules/mesh-delivery-indexer/internal/index/seatsof.go +++ b/modules/mesh-delivery-indexer/internal/index/seatsof.go @@ -8,54 +8,4 @@ package index // of seatdefs.go, the same resolution of the parts and the same rule of which module may define a seat), never by a reader // of their own. -import ( - "context" - "maps" - "slices" - - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/seatdefs" - "git.novox.be/novox/mesh-sdk/go/catalogue" - "git.novox.be/novox/mesh-sdk/go/seatdef" -) - -// revisionSeats are the seats of the definitions the ready merges carry, read at their merge commits, the newest merge -// of a repository governing; none where the indexer has no forge reader. A read the forge does not answer is an error -// (the take-in is tried again); a definition that does not parse is said and left out, as the indexer that keeps it -// would say it unparsed. -func (ix *Indexer) revisionSeats(ctx context.Context, ready []PendingEntry) ([]catalogue.Seat, error) { - defs, err := ix.revisionSeatDefinitions(ctx, ready) - if err != nil || defs == nil { - return nil, err - } - return catalogue.SeatsFromDefinitions(slices.Collect(maps.Values(defs))), nil -} - -// revisionSeatDefinitions are the definitions revisionSeats reads, by seat; nil where the indexer has no forge reader. -func (ix *Indexer) revisionSeatDefinitions(ctx context.Context, ready []PendingEntry) (map[string]seatdef.Definition, error) { - if ix.ReadFile == nil || ix.Files == nil { - return nil, nil - } - defs := map[string]seatdef.Definition{} - for _, e := range ready { - if e.Covered || e.Forge.Merge == "" { - continue - } - for _, c := range contractsModules(e.Forge.Manifests) { - got, unparsed, err := ix.readDefinitions(ctx, e.Forge.Repository, e.Forge.Merge, c) - if err != nil { - return nil, err - } - for seat, why := range unparsed { - ix.logf("the revision's definition of %s (%s) does not parse and is left out: %s", seat, c.Module, why) - } - for seat, d := range got { - if why := seatdefs.Refusal(c.Module, d); why != "" { - ix.logf("the revision's definition of %s is left out: %s", seat, why) // as keepTrunk refuses it (ADR 0636 §5) - continue - } - defs[seat] = d - } - } - } - return defs, nil -} +import () diff --git a/modules/mesh-delivery-indexer/internal/index/series.go b/modules/mesh-delivery-indexer/internal/index/series.go index bc54b86c..8b61e482 100644 --- a/modules/mesh-delivery-indexer/internal/index/series.go +++ b/modules/mesh-delivery-indexer/internal/index/series.go @@ -10,7 +10,6 @@ import ( "strings" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk" "git.novox.be/novox/mesh-sdk/go/catalogue" @@ -203,8 +202,8 @@ func acceptedSeries(series map[string]SeriesPackage, m catalogue.Manifest, repos // package.json name, each read through the forge. The key keeps what it held and gains the name and the module and // artifact it is built from, so the builder numbers it under that name before its first version. A name the forge does // not answer is said and left for the next merge; a manifest that does not decode is the judge's to refuse. -func (ix *Indexer) KeepPackageNames(ctx context.Context, f forgemsg.Forge) error { - if ix.Packages == nil || ix.ReadFile == nil || f.Kind != forgemsg.ForgeMerged || f.Merge == "" { +func (ix *Indexer) KeepPackageNames(ctx context.Context, f whole) error { + if ix.Packages == nil || ix.ReadFile == nil || f.Merge == "" { return nil } var errs []error diff --git a/modules/mesh-delivery-indexer/internal/index/source.go b/modules/mesh-delivery-indexer/internal/index/source.go index d7c04e8a..62a65850 100644 --- a/modules/mesh-delivery-indexer/internal/index/source.go +++ b/modules/mesh-delivery-indexer/internal/index/source.go @@ -8,24 +8,15 @@ import ( "maps" "path" "slices" - "strings" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "git.novox.be/novox/mesh-sdk/go/catalogue" - "git.novox.be/novox/mesh-sdk/go/keys" - "git.novox.be/novox/mesh-sdk/go/process" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk" ) -// The gate's source revision of a watched branch (design 47 G §15, §115), read through the seat mesh-delivery's verb source -// and its event source-moved, never the gate's state: it names every repository's merged head, and its revision is the -// mesh's source revision of that branch. The indexer keeps the latest it has seen. A module is stamped -// with it (Module.Src) when the head the source names for its repository is the merge commit the module's fingerprint -// was computed at: from the modules bucket when the merge was heard first, and when the merge is heard after the source. +// The gate's source revision of a watched branch (design 47 H §737, §743), heard on its event source-moved: it names the heads +// of the repositories it moved, and its number is the mesh's source revision of that branch. The indexer keeps the latest it +// took in. // sourceNow is the latest source revision seen. type sourceNow struct { @@ -51,236 +42,10 @@ func (ix *Indexer) sourceHeads() map[string]string { return maps.Clone(ix.source.heads) } -// ApplySource takes the source revision the verb source answered in. An older revision than the latest seen is left. For each -// repository the source names, each module built from it whose fingerprint was computed at the head the source names, and -// that carries no Src yet, is written again with Src the revision (once: a later revision leaves it); a module computed at another commit is left (a merge heard later is stamped -// when it is written, applyMerged), and a repository none of the indexer's modules is built from writes nothing. The -// modules of a repository are those the modules bucket holds of it (ADR 0501 clause 6). -func (ix *Indexer) ApplySource(ctx context.Context, a meshdelivery.SourceAnswer) error { - rev := uint64(a.Revision) - s := struct{ Heads map[string]string }{Heads: map[string]string{}} - for _, r := range a.Repositories { - s.Heads[r.Repository] = r.Head - } - ix.mu.Lock() - if rev < ix.source.rev { - ix.mu.Unlock() - return nil - } - before := ix.source.heads - ix.source = sourceNow{rev: rev, heads: maps.Clone(s.Heads)} - ix.mu.Unlock() - if ix.Modules == nil { - return ix.RecomputeCandidates(ctx) - } - if err := ix.kickCatchUp(ctx, s.Heads); err != nil { - return err - } - // A merge the indexer never took in is taken in at each source revision, and at the start (A-463). - ix.kickBehind(ctx) - // A registered module the forge could not read is read again at each new source revision (issue 1945), in the - // background: a forge that does not answer must not hold this revision. One taken in later is stamped when it is written - // (applyManifests reads srcOf). - ix.kickRetry(ctx) - held, err := ix.heldModules(ctx) - if err != nil { - return err - } - for _, key := range slices.Sorted(maps.Keys(held)) { - repository := "" - for r := range s.Heads { - if trunk.SameRepository(r, held[key].Repository) { - repository = r - } - } - if repository == "" { - continue - } - head := s.Heads[repository] - err = write(ctx, ix.Modules, key, func(value []byte, found bool) ([]byte, bool, error) { - if !found { - return nil, false, nil - } - cur, err := meshcatalogclient.DecodeModule(value) - if err != nil { - ix.logf("modules.%s holds what is not a module, left: %v", key, err) - return nil, false, nil - } - // Stamped once, with the first revision naming the commit as the head: a later revision (a merge of another - // repository) leaves the value as it is, so no revision of the source rewrites a module that did not change. - if cur.Repository != repository || cur.Commit != head || cur.Src != 0 { - return nil, false, nil - } - cur.Src = int64(rev) - if cur.FpSrc == 0 { - cur.FpSrc = int64(rev) // its fingerprint moved at this merge, whose revision is heard only now (ADR 0601 §1) - } - data, err := meshcatalogclient.EncodeModule(cur) - return data, false, err - }) - if err != nil { - return fmt.Errorf("modules.%s: %w", key, err) - } - } - // A module built against the commit the source named as a repository's head (a package no module publishes, A-365) is - // computed again when that head moves: the gate moves the source on the forge bridge's merged message (A-464), before the take-in, and at the take-in for a head no merged moved. - if movedPackages(held, before, s.Heads) { - if err := ix.Recompute(ctx); err != nil { - return err - } - } - // The stamps are kept with the view (KeepView), so a restart reads them back. - if err := ix.KeepView(ctx); err != nil { - return err - } - // A trunk that moved makes the previews made onto the old one out of date. - return ix.RecomputeCandidates(ctx) -} - -// movedPackages is whether a held module's package version is the previous head of a repository whose head the new source moved. -func movedPackages(held map[string]meshcatalog.ModulesByModuleValue, before, now map[string]string) bool { - for _, m := range held { - for p, v := range m.Packages { - repo, _, ok := trunk.PackageRepository(p) - if ok && before[repo] != "" && before[repo] == v && now[repo] != before[repo] { - return true - } - } - } - return false -} - -// kickCatchUp takes the trunk in at a first start, when the objects keep no trunk revision yet (the lead's ruling A on -// milestone 47/G: the catch-up path): the head the source names for every repository, by the merged message the forge -// bridge gives for it, each applied as its merge, and revision 1 kept from them all at its end. Until then the revision -// the tools answer waits, never an empty catalog. The catch-up runs off the source handler, in its own goroutine, said -// when it ends (lead-47's ruling R1 on the rebase): a bridge that does not answer holds the kept call (ADR 0668) to its -// bound, and must not hold this source revision's stamps and the pulls' candidates with it. A repository whose message is -// not read keeps no revision, said: the next source revision runs the catch-up again (no timer). Objects not on the bus -// yet run none, said, until then. -func (ix *Indexer) kickCatchUp(ctx context.Context, heads map[string]string) error { - if ix.Revisions == nil || ix.Message == nil || len(heads) == 0 { - return nil - } - if _, any, err := newestTrunk(ctx, ix.Revisions); errors.Is(err, process.ErrNotYet) { - ix.logf("catalog graph: the trunk is not taken in yet: its objects %s are not on the bus", CatalogRevisionsObjects) - return nil - } else if err != nil || any { - return err - } - if !ix.catchingUp.CompareAndSwap(false, true) { - return nil - } - heads = maps.Clone(heads) - ix.catchUpWG.Add(1) - go func() { - defer ix.catchUpWG.Done() - unread := ix.takeInHeads(ctx, heads) - ix.catchingUp.Store(false) - if ix.behindWaiting.Swap(false) { - defer ix.kickBehind(ctx) - } - if len(unread) > 0 { - ix.logf("catalog graph: the first start's catch-up ended and no first revision is kept: %d repositories were not taken in (%s); taken in again at the next source revision", - len(unread), strings.Join(unread, ", ")) - return - } - if err := ix.KeepView(ctx); err != nil { - ix.logf("catalog graph: the first start's catch-up took in the trunk of %d repositories, but its revision could not be kept: %v; taken in again at the next source revision", len(heads), err) - return - } - ix.logf("catalog graph: the first start's catch-up ended: the trunk of %d repositories taken in", len(heads)) - }() - return nil -} - -// takeInHeads applies the forge bridge's merged message of each repository's head, as its merge, and answers the -// repositories whose message was not read or not applied. -func (ix *Indexer) takeInHeads(ctx context.Context, heads map[string]string) []string { - var unread []string - for _, repository := range slices.Sorted(maps.Keys(heads)) { - head := heads[repository] - if head == "" { - continue - } - f, err := ix.mergedAt(ctx, repository, head) - if err != nil && ctx.Err() == nil && ix.Read != nil { - // A head the source names that has no merged message (a branch newly watched, whose head no merge on it made) is - // read whole at that commit, as a clone does (A-401, A-93). - ix.logf("catalog graph: %s at %.12s has no merged message (%v): read whole at that commit through the seat git, as a merge of every file of its tree", repository, head, err) - f, err = ix.wholeAt(ctx, repository, head) - } - if err == nil { - _, err = ix.ApplyMessage(ctx, f) - } - if err != nil { - unread = append(unread, repository) - ix.logf("catalog graph: the trunk of %s at %.12s is not taken in: %v", repository, head, err) - } - } - return unread -} - -// CatchUp is the indexer's catch-up tool's work (indexer_take_in with no module and no repository, the catch-up its -// manifest names beside the catalog graph revisions): after a restore point of them is restored, the node agent calls it -// with no arguments, and the indexer takes in the head the gate's source names for every repository, as the first -// start's catch-up does, whatever revision the objects keep, then keeps the view as the next revision when an entry -// moved. Its answer names the repositories taken in and those whose merged message could not be read. -func (ix *Indexer) CatchUp(ctx context.Context) TakenIn { - out := TakenIn{Module: "catch-up"} - ix.mu.Lock() - heads := maps.Clone(ix.source.heads) - ix.mu.Unlock() - switch { - case len(heads) == 0: - out.Answer = "the gate's source names no head yet, so nothing is taken in; the next source revision is taken in as it comes" - return out - case ix.Modules == nil || ix.Message == nil: - out.Answer = "the indexer holds no view of the trunk or reads no merged message, so nothing is taken in" - return out - case !ix.catchingUp.CompareAndSwap(false, true): - out.Answer = "a catch-up runs already; its end keeps the revision" - return out - } - unread := ix.takeInHeads(ctx, heads) - ix.catchingUp.Store(false) - if ix.behindWaiting.Swap(false) { - defer ix.kickBehind(ctx) - } - if err := ix.KeepView(ctx); err != nil { - out.Answer = "the repositories were taken in but the revision could not be kept: " + err.Error() - return out - } - out.Stored = len(unread) == 0 - out.Unread = unread - out.Answer = fmt.Sprintf("the heads the gate's source names for %d repositories taken in", len(heads)-len(unread)) - if len(unread) > 0 { - out.Answer += fmt.Sprintf("; %d not (%s): their merged message was not read, and the call may be made again", len(unread), strings.Join(unread, ", ")) - } - return out -} - -// mergedAt is the forge bridge's merged message of a repository's head. -func (ix *Indexer) mergedAt(ctx context.Context, repository, head string) (forgemsg.Forge, error) { - data, err := ix.Message(forTakeIn(ctx, repository, head), repository, head, 0) - if err != nil { - return forgemsg.Forge{}, fmt.Errorf("the forge bridge could not answer: %w", err) - } - f, err := forgemsg.DecodeForge(data) - if err != nil { - return f, fmt.Errorf("the forge bridge's answer is not a forge message: %w", err) - } - if f.Kind != forgemsg.ForgeMerged || f.Merge != head || !trunk.SameRepository(f.Repository, repository) { - return f, fmt.Errorf("the forge bridge's answer is not the merge of %s at %s", repository, head) - } - return f, nil -} - -// wholeAt is the merged message of a head that has none (A-401): the head's tree and files read through the seat git -// (Read), every module directory of it with its manifest, lock files and mesh packages, as a merge whose changed files are -// every file of that tree, so it is computed as any other head (applyMoving; A-93). -func (ix *Indexer) wholeAt(ctx context.Context, repository, head string) (forgemsg.Forge, error) { - f := forgemsg.Forge{V: keys.Version, Kind: forgemsg.ForgeMerged, Repository: repository, Merge: head, Head: head} +// wholeAt reads repository's trees at the commit head through the seat git (design 47 H §743): every module directory's manifest, +// templates' needs, lock files and packages, and the catalogue's predefined list where the repository is the catalogue's (§769). +func (ix *Indexer) wholeAt(ctx context.Context, repository, head string) (whole, error) { + f := whole{Repository: repository, Merge: head, Head: head} file := func(p string) ([]byte, error) { if ix.ReadFile == nil { return nil, errors.New("the indexer reads no file of the forge") diff --git a/modules/mesh-delivery-indexer/internal/index/source_test.go b/modules/mesh-delivery-indexer/internal/index/source_test.go deleted file mode 100644 index 764df2d8..00000000 --- a/modules/mesh-delivery-indexer/internal/index/source_test.go +++ /dev/null @@ -1,167 +0,0 @@ -package index - -import ( - "context" - "reflect" - "strings" - "testing" - - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" -) - -const otherMerge = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" - -// srcT is a source revision's heads as the tests write them; applyS says it as the verb source answers it. -type srcT struct { - Group string - Heads map[string]string -} - -func sourceOf(repo, head string) srcT { return srcT{Group: "g", Heads: map[string]string{repo: head}} } - -func applyS(ix *Indexer, ctx context.Context, rev uint64, s srcT) error { - a := meshdelivery.SourceAnswer{Branch: "main", Revision: int64(rev)} - for r, h := range s.Heads { - a.Repositories = append(a.Repositories, meshdelivery.SourceHead{Repository: r, Head: h, Trunk: "main"}) - } - return ix.ApplySource(ctx, a) -} - -func TestASourceRevisionStampsTheModulesOfTheRepositoriesHeadItNames(t *testing.T) { // 1344/AC1 - ix, mods := modulesIndexer(t) - ctx := context.Background() - if err := ix.Apply(ctx, mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"})); err != nil { - t.Fatal(err) - } - before, _ := moduleOf(t, mods, "a") - if before.Src != 0 { - t.Fatalf("module before the source = %+v, want Src 0", before) - } - // The source names the head merged for the delivery repository, and another repository's head no module of ours is built from. - s := srcT{Group: "g", Heads: map[string]string{"novox/mesh-delivery": merge, "novox/mesh-catalog": otherMerge}} - if err := applyS(ix, ctx, 7, s); err != nil { - t.Fatal(err) - } - m, ok := moduleOf(t, mods, "a") - want := before - want.Src, want.FpSrc = 7, 7 // its fingerprint moved at that merge (ADR 0601 §1) - if !ok || !reflect.DeepEqual(m, want) { - t.Errorf("module a = %+v (%v), want %+v: the fingerprint and the commit as the merge wrote them, with Src 7", m, ok, want) - } - if c, ok := moduleOf(t, mods, "c"); ok { - t.Errorf("module c = %+v: written for a head no merge message of the indexer carried", c) - } - // The same revision again changes nothing, and so does a later one: Src is the first revision naming the commit. - if err := applyS(ix, ctx, 7, s); err != nil { - t.Fatal(err) - } - if err := applyS(ix, ctx, 9, s); err != nil { - t.Fatal(err) - } - if m, _ := moduleOf(t, mods, "a"); m.Src != 7 { - t.Errorf("module a Src = %d, want 7", m.Src) - } -} - -func TestAMergeHeardAfterTheSourceNamesItsHeadIsStampedWhenWritten(t *testing.T) { // 1344/AC1 (the source first) - // A source revision older than the one held is left (merged here from a test of its own, for the test budget). - t.Run("an older source revision is left", func(t *testing.T) { - ix, mods := modulesIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", aInputs)) - ctx := context.Background() - _ = ix.Apply(ctx, mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"})) - _ = applyS(ix, ctx, 9, sourceOf("novox/mesh-delivery", merge)) - _ = applyS(ix, ctx, 7, sourceOf("novox/mesh-delivery", merge)) - if m, _ := moduleOf(t, mods, "a"); m.Src != 9 { - t.Errorf("module a Src = %d, want 9", m.Src) - } - }) - ix, mods := modulesIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", aInputs)) - ctx := context.Background() - if err := applyS(ix, ctx, 7, sourceOf("novox/mesh-delivery", merge)); err != nil { - t.Fatal(err) - } - if _, ok := moduleOf(t, mods, "a"); ok { - t.Fatal("a module was written from a source revision alone: its tree is the merge message's") - } - if err := ix.Apply(ctx, mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"})); err != nil { - t.Fatal(err) - } - if m, ok := moduleOf(t, mods, "a"); !ok || m.Src != 7 || m.Commit != merge || m.Path != "modules/a" { - t.Errorf("module a = %+v (%v), want the merge's fingerprint with Src 7", m, ok) - } -} - -func TestAMergeThatNoSourceRevisionNamesIsWrittenAsToday(t *testing.T) { // 1344/AC2 - ix, mods := modulesIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", aInputs)) - ctx := context.Background() - // No source revision yet. - if err := ix.Apply(ctx, mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"})); err != nil { - t.Fatal(err) - } - if m, ok := moduleOf(t, mods, "a"); !ok || m.Src != 0 || m.Path != "modules/a" { - t.Fatalf("module a = %+v (%v), want the merge written with Src 0", m, ok) - } - // A source revision that names another head of the repository: a merge of the head it does not name keeps Src 0. - if err := applyS(ix, ctx, 7, sourceOf("novox/mesh-delivery", otherMerge)); err != nil { - t.Fatal(err) - } - if m, _ := moduleOf(t, mods, "a"); m.Src != 0 { - t.Errorf("module a Src = %d after a source naming another head, want 0", m.Src) - } - merged := mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newertree"}) - merged.Merge = "cccccccccccccccccccccccccccccccccccccccc" - if err := ix.Apply(ctx, merged); err != nil { - t.Fatal(err) - } - if m, _ := moduleOf(t, mods, "a"); m.Src != 0 || m.Commit != merged.Merge { - t.Errorf("module a = %+v, want the new merge with Src 0", m) - } -} - -func TestAModuleBuiltAgainstASourceHeadIsComputedAgainWhenThatHeadMoves(t *testing.T) { // design 47 G A-365 - held := map[string]meshcatalog.ModulesByModuleValue{"a": {Packages: map[string]string{"git.novox.be/novox/mesh-contracts/go/seats/findings": otherMerge}}} - for _, row := range []struct { - name string - before, now map[string]string - want bool - }{ - {"the head its version came from moved", map[string]string{"novox/mesh-contracts": otherMerge}, map[string]string{"novox/mesh-contracts": merge}, true}, - {"the head stands", map[string]string{"novox/mesh-contracts": otherMerge}, map[string]string{"novox/mesh-contracts": otherMerge}, false}, - {"another repository's head moved", map[string]string{"novox/mesh-contracts": otherMerge}, map[string]string{"novox/mesh-contracts": otherMerge, "novox/mesh-sdk": merge}, false}, - {"no source seen before", nil, map[string]string{"novox/mesh-contracts": merge}, false}, - } { - if got := movedPackages(held, row.before, row.now); got != row.want { - t.Errorf("%s: movedPackages = %v, want %v", row.name, got, row.want) - } - } -} - -// Design 47 G A-365: through ApplySource, a held module whose package no module publishes was built against the head the source -// named; when the source moves that repository's head, the module's fingerprint is computed again against the new head. -func TestApplySourceComputesAgainTheModuleBuiltAgainstTheHeadThatMoved(t *testing.T) { - ctx := context.Background() - const pkg = "git.novox.be/novox/mesh-contracts/go/seats/findings" - ix, mods := newFormIndexer(t, mod{"avahi", catalogueManifest(t, "avahi")}) - reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1") - ix.Packages = newMem() - setPackages(t, mods, "avahi", []string{pkg}) - h1, h2 := strings.Repeat("1", 40), strings.Repeat("2", 40) - if err := applyS(ix, ctx, 1, sourceOf("novox/mesh-contracts", h1)); err != nil { - t.Fatal(err) - } - if err := ix.Recompute(ctx); err != nil { - t.Fatal(err) - } - first, _ := moduleOf(t, mods, "avahi") - if first.Fp2 == "" || first.Packages[pkg] != h1 { - t.Fatalf("before the head moves: fp2 %q none2 %q packages %v, want a fingerprint built against %s", first.Fp2, first.None2, first.Packages, h1) - } - if err := applyS(ix, ctx, 2, sourceOf("novox/mesh-contracts", h2)); err != nil { - t.Fatal(err) - } - moved, _ := moduleOf(t, mods, "avahi") - if moved.Packages[pkg] != h2 || moved.Fp2 == first.Fp2 { - t.Errorf("after the head moves: fp2 %q (was %q) packages %v, want it computed again against %s", moved.Fp2, first.Fp2, moved.Packages, h2) - } -} diff --git a/modules/mesh-delivery-indexer/internal/index/taken.go b/modules/mesh-delivery-indexer/internal/index/taken.go index 433dc4d1..3d5df687 100644 --- a/modules/mesh-delivery-indexer/internal/index/taken.go +++ b/modules/mesh-delivery-indexer/internal/index/taken.go @@ -10,7 +10,6 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk" "git.novox.be/novox/mesh-sdk/go/process" ) @@ -22,13 +21,13 @@ import ( // say keeps the key as it is and says so. // writeTaken writes the repository's taken key for the merge f carries, unless it names that merge or a later one. -func (ix *Indexer) writeTaken(ctx context.Context, f forgemsg.Forge) error { +func (ix *Indexer) writeTaken(ctx context.Context, f whole) error { return ix.writeTakenKnown(ctx, f, "") } // writeTakenKnown is writeTaken where the merge is known not to be older than notOlder (lateMerge asked it, issue 1126): a // key naming that merge is written over without asking the forge again. -func (ix *Indexer) writeTakenKnown(ctx context.Context, f forgemsg.Forge, notOlder string) error { +func (ix *Indexer) writeTakenKnown(ctx context.Context, f whole, notOlder string) error { if ix.Taken == nil || f.Merge == "" { return nil } @@ -104,7 +103,7 @@ func (ix *Indexer) FillTaken(ctx context.Context) error { } } for _, repository := range slices.Sorted(maps.Keys(pending)) { - err := ix.writeTaken(ctx, forgemsg.Forge{Repository: repository, Merge: pending[repository]}) + err := ix.writeTaken(ctx, whole{Repository: repository, Merge: pending[repository]}) if err == nil { ix.mu.Lock() if ix.untaken[repository] == pending[repository] { diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains.go b/modules/mesh-delivery-indexer/internal/index/toolchains.go index 452d3fc7..40019e0f 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains.go @@ -78,13 +78,9 @@ func (ix *Indexer) ApplyToolchain(ctx context.Context, module, key string, value return ix.ToolchainReported(ctx) } -// ToolchainReported is what a changed toolchain moves: the modules written again (Recompute), then pending evaluated -// again (PendingAgain), since the indexer build a pending revision names is a fingerprint the toolchain is part of. +// ToolchainReported is what a changed toolchain moves: the modules written again (Recompute). func (ix *Indexer) ToolchainReported(ctx context.Context) error { - if err := ix.Recompute(ctx); err != nil { - return err - } - return ix.PendingAgain(ctx) + return ix.Recompute(ctx) } // agreed is the toolchain of each language every reporting node agrees on, or why there is none. @@ -144,7 +140,7 @@ func (ix *Indexer) recomputeWith(ctx context.Context, overlay map[string]meshcat return err } // What the modules moved to reaches the candidates' fingerprints too (a base, the builder, a toolchain). - return ix.RecomputeCandidates(ctx) + return nil } func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]meshcatalog.ModulesByModuleValue, w *mergeWrite, repository string) error { diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go index d4a8ff0a..83f6b032 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go @@ -10,7 +10,6 @@ import ( "reflect" "slices" "strings" - "sync" "testing" "time" @@ -19,10 +18,8 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" "git.novox.be/novox/mesh-sdk/go/catalogue" - "git.novox.be/novox/mesh-sdk/go/keys" "git.novox.be/novox/mesh-sdk/go/process" "git.novox.be/novox/mesh-sdk/go/seatdef" "git.novox.be/novox/mesh-sdk/go/views" @@ -105,7 +102,7 @@ func newFormIndexer(t *testing.T, mods ...mod) (*Indexer, *memStore) { } // newFormIndexerUnmerged is the indexer and the catalogue merge of these modules, not yet applied. -func newFormIndexerUnmerged(t *testing.T, mods ...mod) (*Indexer, *memStore, forgemsg.Forge) { +func newFormIndexerUnmerged(t *testing.T, mods ...mod) (*Indexer, *memStore, whole) { t.Helper() var recs [][2]any trees, manifests := map[string]string{}, map[string]string{} @@ -154,15 +151,6 @@ func TestAgreedToolchainChangesAGoModulesReasonWithNoMerge(t *testing.T) { // 14 t.Run("a recipe change builds a Go module at once and again once the agreement moves", aRecipeChangeBuildsAGoModuleAtOnceAndAgainOnceTheAgreementMoves) t.Fatalf("before: none2 = %q", m.None2) } - // The release that last merged avahi: its source revision names avahi's commit, so Src (and FPSrc) are 7. - merged, _ := moduleOf(t, mods, "avahi") - src := srcT{Group: "g", Heads: map[string]string{merged.Repository: merged.Commit}} - if err := applyS(ix, context.Background(), 7, src); err != nil { - t.Fatal(err) - } - if m, _ := moduleOf(t, mods, "avahi"); m.Src != 7 || m.FpSrc != 7 { - t.Fatalf("before: src %d fp-src %d, want 7 and 7", m.Src, m.FpSrc) - } if g := graphOf(t, ix, "avahi"); g.None != sdkstep.ToolchainNoReport || g.Toolchains != nil { t.Errorf("before: graph.avahi = %+v, want no toolchain edge and the reason", g) } @@ -185,16 +173,10 @@ func TestAgreedToolchainChangesAGoModulesReasonWithNoMerge(t *testing.T) { // 14 t.Errorf("none2 = %q fp2 = %q, want a fingerprint: no builder input is waited for (ADR 0530 §1)", m.None2, m.Fp2) } // ADR 0601 §1 (review round 2): a fingerprint moved with no merge has no release that carries it: fp-src 0, not known, - // though the source revision is still the release's at the module's own Src; a source revision heard again stamps nothing. + // though the source revision is still the release's at the module's own Src. if m.FpSrc != 0 { t.Errorf("a fingerprint moved by a recompute: fp-src %d, want 0", m.FpSrc) } - if err := applyS(ix, context.Background(), 8, src); err != nil { - t.Fatal(err) - } - if again, _ := moduleOf(t, mods, "avahi"); again.FpSrc != 0 { - t.Errorf("a source revision stamped the recomputed fingerprint: fp-src %d", again.FpSrc) - } } // Design 47 *A module's registration* ("No part fixes a name") and *What ADR 0656 leaves to this design* (the indexer reads @@ -517,42 +499,16 @@ func TestBasesAreComputedFirstAndNoFingerprintTakesTheBuilder(t *testing.T) { // firstSaid = append(firstSaid, e) return nil } - // Lead-47's ruling R1: the catch-up runs off the source handler, so a bridge that has not answered yet holds no source - // revision (the first message is answered only once ApplySource returned; held in the handler it is answered as failed). - answers, applied := 0, make(chan struct{}) - first.Message = func(_ context.Context, repository, head string, _ int) ([]byte, error) { - if answers++; answers == 1 { - select { - case <-applied: - return nil, errors.New("the forge bridge does not answer") - case <-time.After(10 * time.Second): - return nil, errors.New("the catch-up held the source handler") - } - } - if repository != catalogueRepo || head != merged.Merge { - return nil, fmt.Errorf("no merge of %s at %s", repository, head) - } - return json.Marshal(merged) + // The take-in of the head the source names keeps revision 1 with every module (design 47 H §743). + if err := applyS(first, context.Background(), 6, sourceOf(catalogueRepo, merged.Merge)); err != nil { + t.Fatal(err) } - heads := sourceOf(catalogueRepo, merged.Merge) - var firstLog []string - var logMu sync.Mutex - first.Logf = func(f string, a ...any) { - logMu.Lock() - firstLog = append(firstLog, fmt.Sprintf(f, a...)) - logMu.Unlock() + if err := first.Apply(context.Background(), merged); err != nil { + t.Fatal(err) } - err = applyS(first, context.Background(), 5, heads) - close(applied) - first.catchUpWG.Wait() - if err != nil || len(firstSaid) != 0 || !strings.Contains(strings.Join(firstLog, "\n"), "the forge bridge does not answer") { - t.Fatalf("a catch-up whose message is not read said %+v (%v), log %q, want no revision and the bridge's silence said", firstSaid, err, firstLog) - } - err = applyS(first, context.Background(), 6, heads) - first.catchUpWG.Wait() - if err != nil || len(firstSaid) != 1 || firstSaid[0].Revision != 1 || - firstSaid[0].Entries != int64(len(builderModules(t))) || firstSaid[0].Source != 6 { - t.Fatalf("the first start's catch-up said %+v (%v), want trunk revision 1 of source 6 with every module", firstSaid, err) + if err := first.KeepView(context.Background()); err != nil || len(firstSaid) != 1 || firstSaid[0].Revision != 1 || + firstSaid[0].Entries != int64(len(builderModules(t))) { + t.Fatalf("the take-in said %+v (%v), want trunk revision 1 with every module", firstSaid, err) } // KeepView: a write to the view outside a recompute (here a verdict carried into one module) is the next revision. viewKey, _ := meshcatalogclient.ModulesKey("avahi") @@ -570,24 +526,6 @@ func TestBasesAreComputedFirstAndNoFingerprintTakesTheBuilder(t *testing.T) { // } else if raw, _, _, _ := view.Get(context.Background(), viewKey); !strings.Contains(string(raw), "a field the judge refuses") { t.Errorf("the view at a restart lost the verdict: %s", raw) } - // W4.14: the catch-up tool (indexer_take_in with no module and no repository, as the node agent calls it after a - // restore) takes in the head the source names for every repository again, whatever revision is kept, and keeps none - // when nothing moved; a repository whose message is not read is named in unread. - if key, value, err := NewCatchUp("", "node-engine@ace", time.Now()); err != nil || key != CatchUpKey { - t.Fatalf("NewCatchUp = %q (%v), want key %q", key, err, CatchUpKey) - } else if a, err := DecodeTakeIn(value); err != nil || !a.CatchUp { - t.Fatalf("a catch-up ask decodes to %+v (%v), want a catch-up", a, err) - } - asked := answers - if got := first.CatchUp(context.Background()); !got.Stored || answers != asked+1 || len(firstSaid) != 2 || len(got.Unread) != 0 { - t.Errorf("the catch-up answered %+v after %d asks, said %d revisions, want the one repository taken in, no new revision", got, answers-asked, len(firstSaid)) - } - first.Message = func(context.Context, string, string, int) ([]byte, error) { - return nil, errors.New("the forge bridge does not answer") - } - if got := first.CatchUp(context.Background()); got.Stored || !reflect.DeepEqual(got.Unread, []string{catalogueRepo}) { - t.Errorf("a catch-up whose message is not read answered %+v, want %s unread", got, catalogueRepo) - } // The tools bundle answers modules and graph from the newest trunk revision, their marks at its number; with none kept // it says it waits (view-waits), never an empty catalog. trunkStates := TrunkStates{Objects: first.Revisions, Under: func(string) views.Source { return nil }} @@ -824,7 +762,7 @@ func TestAModuleKeepsTheToolchainsItsFingerprintWasComputedWith(t *testing.T) { tools, _ := moduleOf(t, mods, "mesh-tools") sdkMerge := func(commit, tree string) meshcatalog.ModulesByModuleValue { t.Helper() - f := forgemsg.Forge{V: keys.Version, Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-sdk", Branch: "feat/x", Number: 3, Head: h1, + f := whole{Repository: "novox/mesh-sdk", Branch: "feat/x", Head: h1, Merge: commit, Root: tree, Trees: map[string]string{"": tree}, Manifests: map[string]string{"": `{"module":"mesh-sdk","version":"1.0"}`}} if err := ix.Apply(context.Background(), f); err != nil { t.Fatal(err) diff --git a/modules/mesh-delivery-indexer/internal/index/unreadcall.go b/modules/mesh-delivery-indexer/internal/index/unreadcall.go deleted file mode 100644 index d81e49fb..00000000 --- a/modules/mesh-delivery-indexer/internal/index/unreadcall.go +++ /dev/null @@ -1,67 +0,0 @@ -package index - -import ( - "context" - "fmt" - "strings" - "time" - - mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" - mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" - "git.novox.be/novox/mesh-sdk/go/views" -) - -// **A merged message unread past five minutes, the forge heard, is a finding** (issue 1966 note 153; novox/hq ADR 0668 §3). -// The indexer's call for a merged message is a kept call: a transport fault leaves the merge unread (Pending.Unread) and -// it is read again after every message heard on the forge (UnreadAgain). A message heard on the forge is the forge bridge -// heard, the callee of that call: so after each one, a merge still unread since meshconditionsclient.CallUnansweredAfter -// or longer is the finding call-unanswered in the indexer's findings, which the condition keeper raises, and a merge no -// longer unread (read, taken in, covered or dropped) clears its finding. While nothing is heard on the forge (the bridge's -// machine away) no finding is written: the merge stays unread and is read again when the forge is heard. No timer. - -// UnreadCallee is the call the finding names: the forge bridge's tool for one merged message. -const UnreadCallee = ForgeBridgeModule + ".bridge_merged_message" - -// unreadCallID is the finding's id for the pending key: the merge it waits on. -func unreadCallID(key string) string { return "merged-message." + key } - -// unreadFindings raises the finding of every merge of entries unread past the bound and clears the finding of every other -// that the finding writer holds (design 47 G §689, A-29). -func (ix *Indexer) unreadFindings(ctx context.Context, entries []PendingEntry) { - if ix.Findings == nil { - return - } - unread := map[string]bool{} - for _, e := range entries { - if e.Value.Unread == "" { - continue - } - id := keyID(unreadCallID(e.Key)) - unread[id] = true - since := e.Value.UnreadSince - if since.IsZero() || ix.now().Sub(since) < mcc.CallUnansweredAfter { - continue - } - f := ix.observed(mcc.ScopeCall, unreadCallID(e.Key), mcc.KindCallUnanswered, - fmt.Sprintf("the call %s about %s is unanswered since %s", UnreadCallee, unreadCallID(e.Key), since.UTC().Format(time.RFC3339)), - "A call whose answer is kept is not answered", - fmt.Sprintf("The call %s is made again on every transport fault and on the callee's event; it has not been answered since %s. Its last fault: %s.", - UnreadCallee, since.UTC().Format(time.RFC3339), e.Value.Unread), "indexer."+mcc.KindCallUnanswered) - f.Needs = TakeInPendingTool + " reads the merge again now; " + PendingTool + " shows it unread." - f.Evidence = []mcseat.Evidence{{At: since.UTC(), Said: e.Value.Unread}} - if err := ix.Findings.Write(ctx, f); err != nil { - ix.logf("take-in: the finding %s is not written: %v", views.FindingKey(f.Scope, f.ID, f.Kind), err) - } else { - ix.logf("take-in: %s (%s#%d) is unread since %s and the forge is heard: the finding %s", e.Key, e.Forge.Repository, e.Forge.Number, since.Format("2006-01-02T15:04:05Z07:00"), views.FindingKey(f.Scope, f.ID, f.Kind)) - } - } - prefix := keyID(strings.TrimSuffix(unreadCallID(""), ".")) + "." - for _, f := range ix.Findings.Held() { - if f.Scope != mcc.ScopeCall || f.Kind != mcc.KindCallUnanswered || !strings.HasPrefix(f.ID, prefix) || unread[f.ID] { - continue - } - if err := ix.Findings.Clear(ctx, f.Scope, f.ID, f.Kind, "the merge is no longer unread"); err != nil { - ix.logf("take-in: the finding %s is not cleared: %v", views.FindingKey(f.Scope, f.ID, f.Kind), err) - } - } -} diff --git a/modules/mesh-delivery-indexer/internal/index/versions_edges.go b/modules/mesh-delivery-indexer/internal/index/versions_edges.go index 4865bb2e..25e93887 100644 --- a/modules/mesh-delivery-indexer/internal/index/versions_edges.go +++ b/modules/mesh-delivery-indexer/internal/index/versions_edges.go @@ -17,10 +17,9 @@ import ( ) // What outdated compares against is kept as heard, in the indexer's state heard (design 47 G A-68): toolchain. the newest -// toolchain-agreed per language, delivered. the newest version-delivered per module. +// toolchain-agreed per language. const ( ToolchainPrefix = "toolchain." - DeliveredPrefix = "delivered." // ActedPrefix opens acted.., the newest wait-ended of an acting part's wait for a dependent, ran or released // (A-78 a), each key part escaped as a name (A-80, §73). ActedPrefix = "acted." @@ -100,66 +99,10 @@ func (ix *Indexer) HearToolchainAgreed(ctx context.Context, t meshbuilds.Toolcha return ix.keepNewest(ctx, ToolchainPrefix+keys.EscapeName(t.Language), t, func([]byte) bool { return true }) } -// HearVersionDelivered takes one version-delivered (design 47 G §374, §376, A-68): the entries of kinds 2, 4 and 6 of the -// delivered release, which read "not yet delivered", are written once with each dependency's newest delivered version and -// part digest; then the delivered version is kept as the module's newest. -func (ix *Indexer) HearVersionDelivered(ctx context.Context, d meshwants.VersionDeliveredEvent) error { - if ix.Versions == nil || ix.Heard == nil { - return errors.New("the state versions is not kept here") - } - if _, ok := versionParts(d.Version); !ok { - return fmt.Errorf("the version-delivered of %s names the version %q, which is no MAJOR.MINOR.PATCH", d.Module, d.Version) - } - key := meshcatalog.VersionsByModuleVersionKey(d.Module, d.Version) - err := write(ctx, ix.Versions, key, func(raw []byte, found bool) ([]byte, bool, error) { - if !found { - return nil, false, nil // a version the catalog does not hold (a build of another branch) has no entries to write - } - v, err := meshcatalog.DecodeVersionsByModuleVersionValue(raw) - if err != nil { - return nil, false, err - } - changed := false - for i, e := range v.Edges { - if e.Version != notYetDelivered { - continue - } - dep, ok := ix.deliveredOf(ctx, strings.SplitN(e.Part, ":", 2)[0]) - if !ok { - continue // the dependency is not delivered yet: the entry reads so until this release's own delivery finds it - } - v.Edges[i].Version, v.Edges[i].Digest = dep.Version, dep.Parts[e.Part] - changed = true - } - if !changed { - return nil, false, nil - } - data, err := json.Marshal(v) - return data, false, err - }) - if err != nil { - return err - } - return ix.keepNewest(ctx, DeliveredPrefix+keys.EscapeName(d.Module), d, func(kept []byte) bool { - var k meshwants.VersionDeliveredEvent - return json.Unmarshal(kept, &k) != nil || compareVersions(d.Version, k.Version) >= 0 - }) -} - -// deliveredOf is the newest version-delivered kept of a module. -func (ix *Indexer) deliveredOf(ctx context.Context, module string) (meshwants.VersionDeliveredEvent, bool) { - var d meshwants.VersionDeliveredEvent - raw, _, found, err := ix.Heard.Get(ctx, DeliveredPrefix+keys.EscapeName(module)) - if err != nil || !found || json.Unmarshal(raw, &d) != nil { - return d, false - } - return d, true -} - // Outdated answers the verb outdated (design 47 G §375) for every edge kind: every entry of each module's newest released // version that is behind. Kind 1: the recorded digest differs from the part's digest in the dependency's newest released // version. Kind 3: the recorded toolchain version differs from the version of the newest toolchain-agreed of the dependent -// artifact's language (A-80, toolchain.). Kinds 2 and 4: from the dependency's newest version-delivered. Kind 5: the +// artifact's language (A-80, toolchain.). Kinds 2, 4 and 6 are declarations, compared with nothing (design 47 H H-A-82). Kind 5: the // acting part has not acted for the dependent's current build (acted.. against the newest trunk // entry's fingerprint, A-78 a). Kind 6: the delivered version's catalog entry no longer serves the verb or word (A-78 b). A // stale edge (kinds 1, 3) has the act rebuild, an ordering edge is shown and starts nothing. A dependency's newer version that @@ -235,16 +178,6 @@ func (ix *Indexer) Outdated(ctx context.Context, in meshcatalog.OutdatedInput) ( continue } o.Recorded, o.Newest = e.Version+" "+e.Digest, tc.Version - case 2, 4: - d, ok := ix.deliveredOf(ctx, dep) - if !ok || e.Version == notYetDelivered { - continue - } - digest := d.Parts[e.Part] - if digest == "" || digest == e.Digest { - continue - } - o.Recorded, o.Newest = e.Version+" "+e.Digest, d.Version+" "+digest case 5: cur := trunk[m].Fp if cur == "" { @@ -255,15 +188,6 @@ func (ix *Indexer) Outdated(ctx context.Context, in meshcatalog.OutdatedInput) ( continue } o.Recorded, o.Newest = "acted for "+cmp.Or(acted.Build, "no build"), "the current build "+cur - case 6: - d, ok := ix.deliveredOf(ctx, dep) - if !ok || e.Version == notYetDelivered || d.Version == e.Version { - continue - } - if serves, known := ix.deliveredServes(ctx, dep, d.Version, e.Word); !known || serves { - continue - } - o.Recorded, o.Newest = e.Version+" "+e.Word, d.Version+" no longer serves "+e.Word default: continue } @@ -316,53 +240,6 @@ func languageOf(entry meshcatalog.CatalogEntry, e meshcatalog.VersionEdge) strin return one } -// deliveredServes says whether the catalog entry of a delivered version of module serves word, its verb (seat.verb, as the entry's -// serves lists it, the form `seat:` allowed) or a word of its manifest (A-78 b); known is false where that version's entry cannot be read. -func (ix *Indexer) deliveredServes(ctx context.Context, module, version, word string) (serves, known bool) { - if ix.Revisions == nil { - return false, false - } - vs, err := ModuleVersions(ctx, ix.Versions, module) - if err != nil { - return false, false - } - var source int64 - for _, v := range vs { - if v.Version == version { - source = v.Source - } - } - if source == 0 { - return false, false - } - names, err := ix.Revisions.Names(ctx) - if err != nil { - return false, false - } - var numbers []int64 - for _, n := range names { - if r, ok := TrunkRevisionOf(n); ok { - numbers = append(numbers, r) - } - } - slices.SortFunc(numbers, func(a, b int64) int { return cmp.Compare(b, a) }) - for _, n := range numbers { // the newest revision taken from the source revision the version was built at - rev, entries, err := ReadRevision(ctx, ix.Revisions, fmt.Sprintf("trunk.%d", n)) - if err != nil || rev.Source != source { - continue - } - for _, e := range entries { - if e.Module != module { - continue - } - w := strings.TrimPrefix(word, "seat:") - _, inManifest := e.Manifest[word] - return slices.Contains(e.Serves, w) || slices.Contains(e.Provides, w) || slices.Contains(e.Streams, w) || inManifest, true - } - } - return false, false -} - // firstBranch is the first watched branch of a module's registration, which its released versions are of. func (ix *Indexer) firstBranch(ctx context.Context, module string) string { if ix.Registry != nil { diff --git a/modules/mesh-delivery-indexer/internal/index/versions_fill.go b/modules/mesh-delivery-indexer/internal/index/versions_fill.go index b29aa4e3..a19e1274 100644 --- a/modules/mesh-delivery-indexer/internal/index/versions_fill.go +++ b/modules/mesh-delivery-indexer/internal/index/versions_fill.go @@ -10,7 +10,6 @@ import ( "strconv" "time" - "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-sdk/go/keys" @@ -153,19 +152,11 @@ func (ix *Indexer) FillVersions(ctx context.Context, ask VersionsAsker) error { } built[module] = v } - // The edge entries (§377), each marked filled: where the stream running or the placement cannot be read the entries of kinds 2, 4 - // and 6 read "not yet delivered", as at a release, and the first delivery writes them. - var running map[string]map[string]nodeengine.Running - if ix.RunningWhole != nil { - if running, err = ix.RunningWhole(ctx); err != nil { - ix.logf("versions fill: the stream running is not read whole: %v", err) - running = nil - } - } + // The edge entries (§377), each marked filled; those of kinds 2, 4 and 6 read "not yet delivered", as at a release. answers := map[string]BuilderVersionsAnswer{} for _, module := range slices.Sorted(maps.Keys(built)) { v := built[module] - v.Edges = ix.filledEdges(ctx, trunk[module], claimed[module], ask, trunk, built, running, answers) + v.Edges = ix.filledEdges(ctx, trunk[module], claimed[module], ask, trunk, built, answers) built[module] = v } for _, module := range slices.Sorted(maps.Keys(built)) { diff --git a/modules/mesh-delivery-indexer/internal/index/versions_fill_edges.go b/modules/mesh-delivery-indexer/internal/index/versions_fill_edges.go index be4ba466..9de59789 100644 --- a/modules/mesh-delivery-indexer/internal/index/versions_fill_edges.go +++ b/modules/mesh-delivery-indexer/internal/index/versions_fill_edges.go @@ -8,28 +8,17 @@ import ( "strings" "time" - "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner" - "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" - "git.novox.be/novox/mesh-sdk/go/keys" ) // The edge entries of the first fill (design 47 G §377, A-78 c): each module's newest version gets its entries once, each marked // filled. Kinds 1 and 3 come from the dependency version published latest before the version's build was claimed, read with -// the times of builder_versions, and its part's digest; kinds 2, 4 and 6 from the version every machine placing the dependency -// runs, present or away, the machines from the assigner's placement (A-62) and each node's last running from the node-engine seat's -// stream read whole (§556); kind 5 as done for every acting part's running build. - -// PlacementAsker asks the seat mesh-module-assigner's verb placement for a module (a read, A-62). -type PlacementAsker func(ctx context.Context, module string) ([]meshmoduleassigner.ModulePlacement, error) - -// RunningReader reads the node-engine seat's stream running whole, every subject's last message (§556): by node, then module. -type RunningReader func(ctx context.Context) (map[string]map[string]nodeengine.Running, error) +// the times of builder_versions, and its part's digest; kinds 2, 4 and 6 are declarations and read "not yet delivered" (design 47 H +// H-A-82: no delivered version is kept); kind 5 as done for every acting part's running build. // filledEdges are the edge entries of the entry's newest version, which was filled as built; claimed is when its build was claimed. func (ix *Indexer) filledEdges(ctx context.Context, entry meshcatalog.CatalogEntry, claimed time.Time, ask VersionsAsker, trunk map[string]meshcatalog.CatalogEntry, - built map[string]meshcatalog.VersionEntry, running map[string]map[string]nodeengine.Running, answers map[string]BuilderVersionsAnswer) []meshcatalog.VersionEdge { + built map[string]meshcatalog.VersionEntry, answers map[string]BuilderVersionsAnswer) []meshcatalog.VersionEdge { var out []meshcatalog.VersionEdge seen := func(e meshcatalog.VersionEdge) bool { return slices.ContainsFunc(out, func(x meshcatalog.VersionEdge) bool { return x.Kind == e.Kind && x.Part == e.Part && x.Word == e.Word }) @@ -78,15 +67,7 @@ func (ix *Indexer) filledEdges(ctx context.Context, entry meshcatalog.CatalogEnt } e.Version, e.Digest, e.Stale = best.Version, best.Digest, true case 2, 4, 6: - e.Version = notYetDelivered - if v, ok := ix.runningVersion(ctx, module, trunk, built, running); ok { - e.Version = v.Version - for _, a := range v.Artifacts { - if a.Artifact == part { - e.Digest = a.Digest - } - } - } + e.Version = notYetDelivered // a declaration: no delivered version is kept for it (design 47 H H-A-82) case 5: e.Fact = "acted for the running build " + entry.Fp default: @@ -99,51 +80,10 @@ func (ix *Indexer) filledEdges(ctx context.Context, entry meshcatalog.CatalogEnt return out } -// runningVersion is the version of a module that every machine placing it runs, present or away, when each runs the newest -// build; false where no machine places it, a placed machine runs none or another build, or the placement is not answered. -func (ix *Indexer) runningVersion(ctx context.Context, module string, trunk map[string]meshcatalog.CatalogEntry, built map[string]meshcatalog.VersionEntry, - running map[string]map[string]nodeengine.Running) (meshcatalog.VersionEntry, bool) { - v, ok := built[module] - e, known := trunk[module] - if !ok || !known || e.Fp == "" || ix.Placement == nil { - return v, false - } - placements, err := ix.Placement(ctx, module) - if err != nil { - ix.logf("versions fill: the placement of %s is not answered: %v", module, err) - return v, false - } - nodes := 0 - for _, p := range placements { - for _, n := range p.Nodes { - nodes++ - r, ok := running[n][module] - if !ok || r.Stopped || r.Fingerprint != e.Fp { - return v, false - } - } - } - return v, nodes > 0 -} - -// keepFilledFacts keeps what the first fill derived of a version as heard: for each entry of kinds 2, 4 and 6 read as delivered, -// the dependency's version as its newest delivered one (A-68), and for each kind-5 edge the act for the running build (A-78 a). +// keepFilledFacts keeps what the first fill derived of a version as heard: for each kind-5 edge the act for the running build (A-78 a). func (ix *Indexer) keepFilledFacts(ctx context.Context, v meshcatalog.VersionEntry, entry meshcatalog.CatalogEntry, built map[string]meshcatalog.VersionEntry, at time.Time) { for _, e := range v.Edges { switch e.Kind { - case 2, 4, 6: - dep := built[strings.SplitN(e.Part, ":", 2)[0]] - if e.Version == notYetDelivered || dep.Version != e.Version { - continue - } - parts := map[string]string{} - for _, a := range dep.Artifacts { - parts[a.Artifact] = a.Digest - } - d := meshwants.VersionDeliveredEvent{V: 1, Module: dep.Module, Version: dep.Version, Parts: parts, At: &at} - if err := ix.keepNewest(ctx, DeliveredPrefix+keys.EscapeName(dep.Module), d, func([]byte) bool { return false }); err != nil { - ix.logf("versions fill: the delivered version of %s is not kept: %v", dep.Module, err) - } case 5: data, _ := json.Marshal(Acted{V: 1, Part: e.Part, Build: entry.Fp, How: "filled", At: at}) if err := write(ctx, ix.Heard, ActedKey(e.Part, v.Module), func(_ []byte, found bool) ([]byte, bool, error) { diff --git a/modules/mesh-delivery-indexer/internal/index/versions_test.go b/modules/mesh-delivery-indexer/internal/index/versions_test.go index 9a854d85..4aee9718 100644 --- a/modules/mesh-delivery-indexer/internal/index/versions_test.go +++ b/modules/mesh-delivery-indexer/internal/index/versions_test.go @@ -10,8 +10,6 @@ import ( "testing" "time" - "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner" - "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" @@ -219,28 +217,13 @@ func TestOutdatedAnswersTheEdgesBehindByTheirPartsDigest(t *testing.T) { if b := behind(); len(b) != 1 || b[0].Module != "app" || b[0].Kind != 1 || b[0].Act != "rebuild" || b[0].Dependency.Artifact != "lib:pkg" || !strings.Contains(b[0].Newest, "d3") || !strings.Contains(b[0].Recorded, "d1") { t.Fatalf("behind after lib's new digest: %+v", b) } - // delivery: dep 0.1.1 is delivered, then app 0.1.1 writes its kind 4 entry; dep 0.1.2 delivered later is behind, an ordering edge - deliver := func(module, version string, parts map[string]string) { - t.Helper() - if err := ix.HearVersionDelivered(ctx, meshwants.VersionDeliveredEvent{Module: module, Version: version, Parts: parts}); err != nil { - t.Fatal(err) - } - } - deliver("dep", "0.1.1", map[string]string{"dep:program": "x1"}) - deliver("app", "0.1.1", map[string]string{"app:program": "a1"}) + // H-A-82: no delivered version is kept, so an entry of kind 4 reads "not yet delivered" and is never behind. v, _ = ModuleVersions(ctx, ix.Versions, "app") - if e := v[0].Edges[1]; e.Version != "0.1.1" || e.Digest != "x1" { - t.Fatalf("the kind 4 entry after the first delivery: %+v", e) + if e := v[0].Edges[1]; e.Kind != 4 || e.Version != notYetDelivered { + t.Fatalf("the kind 4 entry: %+v, want it not yet delivered", e) } - deliver("dep", "0.1.2", map[string]string{"dep:program": "x2"}) - deliver("app", "0.1.1", map[string]string{"app:program": "a1"}) // delivered again: written once, not changed - v, _ = ModuleVersions(ctx, ix.Versions, "app") - if e := v[0].Edges[1]; e.Version != "0.1.1" { - t.Errorf("an entry written at the first delivery changed: %+v", e) - } - b := behind() - if len(b) != 2 || b[1].Kind != 4 || b[1].Act != "" || b[1].Dependency.Artifact != "dep:program" { - t.Fatalf("behind after dep's second delivery: %+v", b) + if b := behind(); len(b) != 1 || b[0].Kind != 1 { + t.Fatalf("behind: %+v, want only the kind 1 entry", b) } // the newest toolchain-agreed per language is kept as heard if err := ix.HearToolchainAgreed(ctx, meshbuilds.ToolchainAgreed{Language: "go", Version: "1.25"}); err != nil { @@ -308,18 +291,12 @@ func TestAnInputIsKeptByItsDigestAndSweptAfterADay(t *testing.T) { func TestWhatIsHeardIsKeptEscapedInTheStateHeard(t *testing.T) { ctx := context.Background() ix := &Indexer{Module: testSelf, Versions: newMem(), OwnBuildState: newMem(), Heard: newMem(), Revisions: newMemObjects(1 << 20)} - if err := ix.HearVersionDelivered(ctx, meshwants.VersionDeliveredEvent{Module: "a.b", Version: "1.2.3"}); err != nil { - t.Fatal(err) - } if err := ix.HearToolchainAgreed(ctx, meshbuilds.ToolchainAgreed{Language: "go", Version: "1.26"}); err != nil { t.Fatal(err) } - if _, ok := ix.deliveredOf(ctx, "a.b"); !ok { - t.Error("the delivered version of a dotted module is not read back") - } got, _ := ix.Heard.Keys(ctx) slices.Sort(got) - if want := []string{"delivered.a=2eb", "toolchain.go"}; !slices.Equal(got, want) { + if want := []string{"toolchain.go"}; !slices.Equal(got, want) { t.Errorf("the keys of heard are %v, want %v", got, want) } if own, _ := ix.OwnBuildState.Keys(ctx); len(own) != 0 { @@ -421,29 +398,19 @@ func TestOutdatedAnswersKinds3And5And6(t *testing.T) { if _, ok := behind()[6]; ok { t.Error("kind 6 behind with no newer delivered version") } + // H-A-82: a kind-6 entry is a declaration; with no delivered version kept it is never behind. put("prov", meshcatalog.VersionEntry{Version: "0.1.2", Source: 20}) - if err := ix.HearVersionDelivered(ctx, meshwants.VersionDeliveredEvent{Module: "prov", Version: "0.1.2", Parts: map[string]string{"prov:program": "p2"}}); err != nil { - t.Fatal(err) - } - if o, ok := behind()[6]; !ok || o.Act != "" || !strings.Contains(o.Newest, "no longer serves") { - t.Errorf("kind 6 after a version that serves no more: %+v", o) + if _, ok := behind()[6]; ok { + t.Error("kind 6 behind with no delivered version kept") } } // Design 47 G §377, A-78 c: the first fill gives each module's newest version its edge entries, each marked filled: kind 1 from the -// dependency version published latest before the build was claimed, kinds 2, 4 and 6 from the version every placing machine runs, -// read from the assigner's placement and the stream running, kind 5 as acted for the running build; so the first outdated says only what is truly behind. +// dependency version published latest before the build was claimed, kinds 2, 4 and 6 not yet delivered (H-A-82), kind 5 as acted for the running build; so the first outdated says only what is truly behind. func TestTheFirstFillGivesTheNewestVersionsTheirFilledEdges(t *testing.T) { ctx := context.Background() t0 := time.Date(2026, 11, 1, 3, 0, 0, 0, time.UTC) - placed := map[string][]string{"prov": {"n1", "n2"}} - ix := &Indexer{Module: testSelf, Versions: newMem(), Heard: newMem(), Revisions: newMemObjects(1 << 20), Now: func() time.Time { return t0 }, - Placement: func(_ context.Context, m string) ([]meshmoduleassigner.ModulePlacement, error) { - return []meshmoduleassigner.ModulePlacement{{Branch: "trunk", Nodes: placed[m]}}, nil - }, - RunningWhole: func(context.Context) (map[string]map[string]nodeengine.Running, error) { - return map[string]map[string]nodeengine.Running{"n1": {"prov": {Fingerprint: "fp-prov"}}, "n2": {"prov": {Fingerprint: "fp-prov"}}}, nil - }} + ix := &Indexer{Module: testSelf, Versions: newMem(), Heard: newMem(), Revisions: newMemObjects(1 << 20), Now: func() time.Time { return t0 }} entries := map[string]meshcatalog.CatalogEntry{ "lib": {Module: "lib", Manifest: map[string]any{}, Fp: "fp-lib", Artifacts: []meshcatalog.Artifact{{Ref: "lib:pkg", Package: &meshcatalog.ArtifactPackage{Ecosystem: "go", Name: "x/lib"}}}}, "actor": {Module: "actor", Manifest: map[string]any{}, Fp: "fp-actor", Artifacts: []meshcatalog.Artifact{{Ref: "actor:program"}}}, @@ -490,8 +457,8 @@ func TestTheFirstFillGivesTheNewestVersionsTheirFilledEdges(t *testing.T) { t.Errorf("kind 1 is the version numbered or published before the build was claimed: %+v", e) } case 4: - if e.Version != "0.9.1" || e.Digest != "sha256:prov/program" { - t.Errorf("kind 4 is the version every machine runs: %+v", e) + if e.Version != notYetDelivered || e.Digest != "" { // H-A-82: a declaration, filled with no delivered version + t.Errorf("kind 4 is not yet delivered: %+v", e) } } } diff --git a/modules/mesh-delivery-indexer/internal/index/whole.go b/modules/mesh-delivery-indexer/internal/index/whole.go new file mode 100644 index 00000000..d2a304c3 --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/whole.go @@ -0,0 +1,28 @@ +package index + +import "encoding/json" + +// whole is one repository's trees at one commit, as a take-in reads them through the seat git's trees (wholeAt; design 47 H +// §743): the module directories, their manifests, lock files' digests, reader capabilities, named paths and mesh packages, and +// the catalogue's predefined.json where the repository holds it. It is what a take-in applies (ApplyMessage); nothing else +// carries it. +type whole struct { + Repository string `json:"repository"` + // Branch is the watched branch the commit is the head of at the take-in's source revision. + Branch string `json:"branch,omitempty"` + // Merge is the commit taken in; Head is the same commit. + Merge string `json:"merge,omitempty"` + Head string `json:"head,omitempty"` + Root string `json:"root,omitempty"` + Trees map[string]string `json:"trees,omitempty"` + Dirs []string `json:"dirs,omitempty"` + DirsKnown bool `json:"dirs-known,omitempty"` + Manifests map[string]string `json:"manifests,omitempty"` + Locks map[string]map[string]string `json:"locks,omitempty"` + Needs map[string][]string `json:"needs,omitempty"` + Paths map[string]map[string]string `json:"paths,omitempty"` + Packages map[string][]string `json:"packages,omitempty"` + // Predefined is the catalogue's predefined.json, exact, at the commit; PredefinedRead says the commit was read and holds none. + Predefined json.RawMessage `json:"predefined,omitempty"` + PredefinedRead bool `json:"predefined-read,omitempty"` +} diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index b2e567d6..b8a4bff7 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -135,7 +135,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build if want := []state{{Name: "candidates", History: 10}, {Name: "heard", History: 64}, {Name: "manifests", History: 1}, {Name: "predefined", History: 5}, {Name: "taken", History: 1}, {Name: "takeins", History: 1}, {Name: "oldsections", History: 1}, {Name: index.RunsState, History: 1}, {Name: "own-build", History: 1}, {Name: index.PackagesState, History: index.PackagesHistory}, {Name: views.MarksState, History: 1}, {Name: index.FindingsState, History: 5}, {Name: views.StatusState, History: 1}, {Name: index.RequirementsState, History: 5}, {Name: index.SeatDefinitionsState, History: 5}, - {Name: index.PendingState, History: 1}, {Name: index.EntryIndexState, History: 1, MaxMiB: 8}, {Name: index.NamedState, History: 1, MaxMiB: 4}, // design 47 G §695 + {Name: index.EntryIndexState, History: 1, MaxMiB: 8}, {Name: index.NamedState, History: 1, MaxMiB: 4}, // design 47 G §695 {Name: index.RegistrationsState, History: 1}, {Name: index.ProposalsState, History: 1, MaxMiB: 4}, {Name: index.VersionsState, History: 1, MaxMiB: 64}, {Name: index.BuildEdgesState, History: 1}, {Name: index.InputsState, History: 1}, {Name: index.PinsState, History: 1, MaxMiB: index.PinsCapMiB}, {Name: process.ConsumersState, History: 1}}; !reflect.DeepEqual(m.State, want) { // DLV-I7, DLV-I17 t.Errorf("state = %+v, want %+v", m.State, want) @@ -156,21 +156,21 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build t.Errorf("the indexer names its bucket %s with %d revisions, its manifest's states are %+v", bucket, history, m.State) } } - for bucket, read := range map[string]string{index.PreviewsBucket: "mesh-delivery-checker.previews", + for bucket, read := range map[string]string{ index.ToolchainsBucket("mesh-build-go"): "mesh-build-go.toolchains", index.ToolchainsBucket("mesh-build-node"): "mesh-build-node.toolchains"} { if !slices.Contains(m.Reads, read) || strings.Replace(read, ".", "_", 1) != bucket { t.Errorf("the indexer reads the bucket %s, its manifest's reads are %v (want %s)", bucket, m.Reads, read) } } // DLV-F1r (ADR 0615 §1): the assigner's assignments is read through its first view and views family, never its state. - if want := []string{"mesh-build-go.toolchains", "mesh-build-node.toolchains", "mesh-delivery-builder.build-log", "mesh-delivery-checker.previews"}; !reflect.DeepEqual(sorted(m.Reads), want) { + if want := []string{"mesh-build-go.toolchains", "mesh-build-node.toolchains", "mesh-delivery-builder.build-log"}; !reflect.DeepEqual(sorted(m.Reads), want) { t.Errorf("reads = %v, want %v", m.Reads, want) } // "Reads FORGE" is a consumes of the indexer's stream: the broker grants only buckets by reads (mesh-control // internal/broker/state.go:62-93). // The registry seats' published fill each (§413: no consume of version.numbered) package's series (DLV-I5, design 51 *The versions*). - if want := []string{"go-package-registry.published", "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "mesh-builds.package-published", "mesh-builds.toolchain-agreed", "mesh-catalog.catalog-revision", "mesh-conditions.findings-asked", "mesh-delivery-checker.verdict", "mesh-delivery-forge-bridge.forge.**", - "mesh-delivery.source-moved", "mesh-pipeline-runner.pipeline-ended", "mesh-pipeline-runner.pipeline-started", "mesh-pipeline-runner.run-started", "mesh-wants.version-delivered", "mesh-wants.wait-ended", "mesh-wants.wanted", "node-build-go.step-ended", "node-engine.running", "npm-package-registry.published", "operator-channel.decided", "seat:mesh-module-assigner.views.assignments", "seat:mesh-module-assigner.views.seat-holders"}; !reflect.DeepEqual(sorted(m.Consumes), want) { + if want := []string{"go-package-registry.published", "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "mesh-builds.package-published", "mesh-builds.toolchain-agreed", "mesh-catalog.catalog-revision", "mesh-conditions.findings-asked", + "mesh-delivery.source-moved", "mesh-pipeline-runner.pipeline-ended", "mesh-pipeline-runner.pipeline-started", "mesh-pipeline-runner.run-started", "mesh-wants.wait-ended", "mesh-wants.wanted", "node-build-go.step-ended", "node-engine.running", "npm-package-registry.published", "operator-channel.decided", "seat:mesh-module-assigner.views.assignments", "seat:mesh-module-assigner.views.seat-holders"}; !reflect.DeepEqual(sorted(m.Consumes), want) { t.Errorf("consumes = %v, want %v", m.Consumes, want) } if b, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 || b != "${dir:state}/broker" { @@ -182,7 +182,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build // Three invokes: the file of a registered module's manifest and lock files (registered.go), the forge's trees (the named paths' objects, since the indexer is the one reader of a manifest's paths: // ADR 0499 clauses 2 and 3, ADR 0495 clause 1; and the module directories of the trunk, issue 1900) and whether a // commit is in another (issue 1900). - if want := []string{"seat:git.trees", "seat:git.tags", "seat:go-package-registry.versions", "seat:npm-package-registry.versions", "seat:git.file", "seat:git.contains", "seat:git.open-pulls", "seat:git.branch", "mesh-delivery-forge-bridge.bridge_merged_message", "seat:node-engine.composition", "seat:mesh-delivery.source", "seat:mesh-module-assigner.assignments", "seat:mesh-module-assigner.placement", "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence", "seat:node-build-go.steps", + if want := []string{"seat:git.trees", "seat:git.tags", "seat:go-package-registry.versions", "seat:npm-package-registry.versions", "seat:git.file", "seat:git.contains", "seat:git.branch", "seat:node-engine.composition", "seat:mesh-module-assigner.assignments", "seat:mesh-module-assigner.placement", "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence", "seat:node-build-go.steps", "seat:node-build-node.steps", "seat:node-build-image.steps", "mesh-delivery-builder.builder_versions"}; !reflect.DeepEqual(m.Invokes, want) { t.Errorf("invokes = %v, want %v", m.Invokes, want) } @@ -200,12 +200,12 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build } want = append(want, "indexer_first_views", "indexer_first_view_again", index.ConfigReadersTool, index.BusStateTool, index.RequirementsTool, index.PackageUsersTool, // DLV-F3r, DLV-I10, DLV-I11, DLV-I7, DLV-I8, DLV-I9c index.SeatDefinitionTool, index.SeatDefinitionsAgainTool, index.OutOfStepTool, // DLV-I19 - index.PendingTool, index.TakeInPendingTool, index.DropPendingTool, index.TakeInMergeTool, index.FingerprintsAgainTool, // issue 1966; 1585/T12, T13 (ADR 0633 §5) + index.FingerprintsAgainTool, // issue 1966 // milestone 47/G W4.12 (design 47, the indexer's tools): the catalog graph revisions read and steered. index.CatalogRevisionTool, index.CatalogRevisionsTool, index.CandidateGraphsTool, index.RecomputeTool, index.DropCandidateGraphTool, index.EntryIndexTool, index.NamedDropTool, // design 47 G §675 index.ProposalsTool, index.ProposalDropTool, index.SeedAgainTool, index.InputsTool, index.PinsTool, index.UnpinTool, index.ConsumerTool, index.RunsTool, index.RunDropTool) // design 47 G §667, §8, §579, §64 (A-194) - if !reflect.DeepEqual(m.Tools, want) || len(want) != 45 || !slices.Contains(m.Tools, "modules") { + if !reflect.DeepEqual(m.Tools, want) || len(want) != 41 || !slices.Contains(m.Tools, "modules") { t.Errorf("tools = %v, want %v", m.Tools, want) } // Design 47 G §2, §9: every version of every registration is kept with no age limit, 64 MiB, on the old streams section. @@ -260,6 +260,11 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build if inv == "mesh-delivery-indexer.indexer_read_modules" { // design 47 G §196: the readers' own subtasks move them to the catalog half continue } + // H-A-79: the grants part's and the watchman's invokes of the verdicts go in their own subtasks (the contracts-indexer-bridge + // developer's readers); indexer_pending is served again with the held take-in (design 47 H §744, subtask I5). + if inv == "mesh-delivery-indexer.verdicts" || inv == "mesh-delivery-indexer.indexer_verdicts" || inv == "mesh-delivery-indexer.indexer_pending" { + continue + } if tool, ok := strings.CutPrefix(inv, "mesh-delivery-indexer."); ok && !slices.Contains(m.Tools, tool) { t.Errorf("%s invokes %s, which the indexer does not serve", p, inv) } diff --git a/modules/mesh-delivery-indexer/module.json b/modules/mesh-delivery-indexer/module.json index 43222891..41b23137 100644 --- a/modules/mesh-delivery-indexer/module.json +++ b/modules/mesh-delivery-indexer/module.json @@ -51,10 +51,6 @@ "indexer_seat_definition", "indexer_seat_definitions_again", "indexer_out_of_step", - "indexer_pending", - "indexer_take_in_pending", - "indexer_drop_pending", - "indexer_take_in_merge", "indexer_fingerprints_again", "indexer_catalog_revision", "indexer_catalog_revisions", @@ -82,11 +78,8 @@ "seat:mesh-module-assigner.views.seat-holders", "mesh-builds.package-published", "mesh-builds.toolchain-agreed", - "mesh-wants.version-delivered", "node-build-go.step-ended", "mesh-wants.wait-ended", - "mesh-delivery-checker.verdict", - "mesh-delivery-forge-bridge.forge.**", "mesh-catalog.catalog-revision", "mesh-delivery.source-moved", "mesh-wants.wanted", @@ -113,7 +106,6 @@ { "name": "first-views", "history": 1 }, { "name": "requirements", "history": 5 }, { "name": "seat-definitions", "history": 5 }, - { "name": "pending", "history": 1 }, { "name": "entry-index", "history": 1, "max-mib": 8 }, { "name": "named", "history": 1, "max-mib": 4 }, { "name": "registrations", "history": 1 }, @@ -148,8 +140,7 @@ "reads": [ "mesh-build-go.toolchains", "mesh-build-node.toolchains", - "mesh-delivery-builder.build-log", - "mesh-delivery-checker.previews" + "mesh-delivery-builder.build-log" ], "invokes": [ "seat:git.trees", @@ -158,11 +149,8 @@ "seat:npm-package-registry.versions", "seat:git.file", "seat:git.contains", - "seat:git.open-pulls", "seat:git.branch", - "mesh-delivery-forge-bridge.bridge_merged_message", "seat:node-engine.composition", - "seat:mesh-delivery.source", "seat:mesh-module-assigner.assignments", "seat:mesh-module-assigner.placement", "seat:mesh-hand-acts.hand-act",