I8, H-A-82: the earlier take-in goes; no delivered version is kept
I8: the take-in fed by the forge bridge's messages goes with B1's forgemsg: the FORGE consumer and its handlers, the group end, the pending state with its keys, claims, dry take-in and hand tools (indexer_pending, indexer_take_in_pending, indexer_drop_pending, indexer_take_in_merge, kept again for the held take-in in I5), the catch-up by comparison (kickBehind, TakeInBehind, sayUntaken, takeInHeads, kickCatchUp, CatchUp, merge-untaken), the source numbering in memory (nextSource), the source verb's follow and Src stamping on a source answer, the bridge's merged message call, the pull request candidates written from pull and open messages, the checker's previews and verdict hearing. A repository's trees at a commit are the internal whole (wholeAt), which ApplyMessage takes in; the dry indexer stays for a run on request's candidate. The manifest drops the consumes of forge.**, the checker's verdict and version-delivered, the reads of the checker's previews and the invokes of open-pulls, bridge_merged_message and mesh-delivery.source. RoomAgain says room when a take-in is held on a full store (heldFull, I5 holds it). H-A-82: the fill of delivered versions into the edges goes: HearVersionDelivered, the delivered.<module> key, the first fill's kinds 2, 4 and 6 from the placement and the stream running, and outdated's kinds 2, 4 and 6; their entries read not yet delivered. The tests of the removed paths go (index_test's candidate groups, source_test, branch_take_in_test, previews_test, candidate_manifests_test, the FORGE handle tests); the versions tests now hold kinds 2, 4 and 6 as declarations. The integration tests still drive FORGE and are rewritten in I10. Builds: design 47 H §743, §744, §764 (the indexer); H-A-48, H-A-82 Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy
This commit is contained in:
@@ -266,7 +266,6 @@ func main() {
|
||||
return err
|
||||
})...)
|
||||
tools = append(tools, viewTools(os.Getenv, trunk.State, time.Now)...)
|
||||
tools = append(tools, index.PendingTools(keptRead{stdio.State(index.PendingState)}, keptRead{stdio.State(index.OwnBuildStateName)}, asks, time.Now)...)
|
||||
tools = append(tools, index.FingerprintsTools(asks, time.Now, func(what, why string) error {
|
||||
_, err := acts.Record(meshhandactsclient.Request{What: what, Why: why, Cause: index.FingerprintsAgainTool, Verb: "fingerprints-again"})
|
||||
return err
|
||||
|
||||
@@ -1,46 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"mesh-delivery-indexer/internal/index"
|
||||
)
|
||||
|
||||
// mergedTool is the forge bridge's tool that answers a merge's merged message again (issue 1585): arguments repository,
|
||||
// merge and, when the indexer knows it, number (the bridge then reads a merge made on the forge itself), answer {message}, the message whole as the bridge writes it on FORGE. The indexer's manifest lists it in
|
||||
// invokes, which is the grant to call it.
|
||||
const mergedTool = index.ForgeBridgeModule + ".bridge_merged_message"
|
||||
|
||||
// messageVia asks the forge bridge for a merge's merged message, for a held merge the indexer does not hold in memory (a
|
||||
// restart, a new build), a kept call asked again only while the commit's take-in is pending (A-190). An error is the
|
||||
// bridge's own words.
|
||||
func messageVia(call openCaller, pending func(ctx context.Context, repository, merge string) (bool, error), logf func(string, ...any)) func(ctx context.Context, repository, merge string, number int) ([]byte, error) {
|
||||
return func(ctx context.Context, repository, merge string, number int) ([]byte, error) {
|
||||
ask := map[string]any{"repository": repository, "merge": merge}
|
||||
if number > 0 {
|
||||
ask["number"] = number
|
||||
}
|
||||
raw, err := keptWhileOpen(ctx, call, logf, "message."+repository+"."+merge, mergedTool, ask,
|
||||
func(ctx context.Context) (bool, error) { return pending(ctx, repository, merge) })
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var a struct {
|
||||
Message json.RawMessage `json:"message"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &a); err != nil || len(a.Message) == 0 {
|
||||
return nil, fmt.Errorf("the answer of %s is not {message}: %v", mergedTool, err)
|
||||
}
|
||||
return a.Message, nil
|
||||
}
|
||||
}
|
||||
|
||||
// unreadAgain reads again, after a message heard on FORGE, a pending merge whose message was unread for a transport fault
|
||||
// (index.Indexer.UnreadAgain; issue 1966 note 153): the forge is heard again, so the bridge is asked again. A failure is said.
|
||||
func unreadAgain(ctx context.Context, ix *index.Indexer, logger interface{ Printf(string, ...any) }) {
|
||||
if err := ix.UnreadAgain(ctx); err != nil {
|
||||
logger.Printf("take-in: a merge unread for a transport fault is not read again after this message: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -11,7 +11,6 @@ package main
|
||||
import (
|
||||
"cmp"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
@@ -24,7 +23,6 @@ import (
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscalls"
|
||||
@@ -33,7 +31,6 @@ import (
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
|
||||
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
|
||||
@@ -45,11 +42,6 @@ import (
|
||||
"mesh-delivery-indexer/internal/index"
|
||||
)
|
||||
|
||||
// forgeStream is the bridge's stream as the bus names it: its local name in capitals (mesh-control
|
||||
// internal/catalogue/streams.go StreamBusName). The module's consumer on it is the durable <node>_<module> the controller
|
||||
// makes (internal/broker/modstreams.go HeardIssued); the indexer binds it, pulls and acks.
|
||||
const forgeStream = "FORGE"
|
||||
|
||||
func main() {
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
@@ -177,119 +169,6 @@ type delivered interface {
|
||||
Metadata() (*jetstream.MsgMetadata, error)
|
||||
}
|
||||
|
||||
// groupEndSubject is the whole subject of a delivery group's end on the forge bridge's stream: compared exactly, so a
|
||||
// repository named forge/group, whose merged message ends the same way, is never taken for one.
|
||||
var groupEndSubject = process.ModuleStreamSubject(index.ForgeBridgeModule, forgemsg.ForgeGroupEndEvent)
|
||||
|
||||
// handleGroupEnd takes a group's end in (index.EndGroup): a body that does not read is terminated and said; an end
|
||||
// whose members are not all held yet, or while the build this program runs is not known, is left for redelivery.
|
||||
func handleGroupEnd(ctx context.Context, m delivered, ix *index.Indexer, h *index.Heard, logger *log.Logger) {
|
||||
end, err := forgemsg.DecodeForgeGroupEnd(m.Data())
|
||||
if err != nil {
|
||||
logger.Printf("%s: not a group's end, dropped: %v", m.Subject(), err)
|
||||
h.Outcome, h.Words = index.HeardDropped, err.Error()
|
||||
_ = m.Term()
|
||||
return
|
||||
}
|
||||
h.Kind, h.Branch = forgemsg.ForgeGroupEndEvent, end.Group
|
||||
applied, words, err := ix.EndGroup(ctx, end)
|
||||
h.Written, h.Late = applied.Written, applied.Late
|
||||
if err != nil {
|
||||
logger.Printf("%s: the end of the group %q left for redelivery: %v", m.Subject(), end.Group, err)
|
||||
h.Outcome, h.Words = index.HeardLeft, err.Error()
|
||||
return
|
||||
}
|
||||
h.Outcome, h.Words = index.HeardApplied, words
|
||||
if err := m.DoubleAck(ctx); err != nil {
|
||||
logger.Printf("%s: the end of the group %q applied, its acknowledgement was not confirmed: %v", m.Subject(), end.Group, err)
|
||||
}
|
||||
}
|
||||
|
||||
// handle takes one message of FORGE into the candidates and acknowledges it. A message it can never read is terminated,
|
||||
// never redelivered, and the log names its subject and why. A write that failed stays unacknowledged: the server
|
||||
// delivers it again (the server's ack wait is the only clock). Each message handled is written to the heard state
|
||||
// (index.Heard; nil, or a state not on the bus yet, writes none), so indexer_heard reads what the indexer heard of FORGE.
|
||||
func handle(ctx context.Context, m delivered, ix *index.Indexer, heard index.Store, logger *log.Logger) {
|
||||
h := index.Heard{Subject: m.Subject()}
|
||||
if md, err := m.Metadata(); err == nil && md != nil {
|
||||
h.Sequence, h.Delivered, h.Published = md.Sequence.Stream, md.NumDelivered, md.Timestamp.UTC()
|
||||
}
|
||||
defer func() {
|
||||
if heard == nil {
|
||||
return
|
||||
}
|
||||
h.Handled = time.Now().UTC()
|
||||
if err := index.WriteHeard(ctx, heard, h); err != nil && !errors.Is(err, process.ErrNotYet) {
|
||||
logger.Printf("%s: sequence %d is not written to the heard state: %v", h.Subject, h.Sequence, err)
|
||||
}
|
||||
}()
|
||||
// A delivery group's end (novox/hq ADR 0633 §4) has a body of its own: told apart by its name, before the decode.
|
||||
if m.Subject() == groupEndSubject {
|
||||
handleGroupEnd(ctx, m, ix, &h, logger)
|
||||
return
|
||||
}
|
||||
if kind, known := forgeKind(m.Data()); !known {
|
||||
// A kind the indexer does not take (the bridge's branch-deleted, design 51) is acknowledged and passed over, never
|
||||
// terminated as unreadable: a new kind on FORGE is no fault of the message.
|
||||
h.Kind, h.Outcome, h.Words = kind, index.HeardIgnored, "a "+kind+" message, which the indexer does not take"
|
||||
_ = m.Ack()
|
||||
return
|
||||
}
|
||||
f, err := forgemsg.DecodeForge(m.Data())
|
||||
if err != nil {
|
||||
logger.Printf("%s: not a forge message, dropped: %v", m.Subject(), err)
|
||||
h.Outcome, h.Words = index.HeardDropped, err.Error()
|
||||
_ = m.Term()
|
||||
return
|
||||
}
|
||||
h.Kind, h.Repository, h.Number, h.Branch, h.Commit = f.Kind, f.Repository, f.Number, f.Branch, f.Head
|
||||
if f.Merge != "" {
|
||||
h.Commit = f.Merge
|
||||
}
|
||||
f = index.RevisionForm(f) // a pushed message is taken in as a merged one is, its head where the merge stands (§509, A-269)
|
||||
var applied index.Applied
|
||||
if f.Kind == forgemsg.ForgeMerged && h.Sequence > 0 {
|
||||
// A merge is taken in only by the indexer its revision names (ADR 0633 §1): kept in pending until then.
|
||||
applied, h.Words, err = ix.Arrive(ctx, h.Sequence, f)
|
||||
} else {
|
||||
applied, err = ix.ApplyMessage(ctx, f)
|
||||
}
|
||||
h.Written, h.Late = applied.Written, applied.Late
|
||||
switch {
|
||||
case errors.Is(err, index.ErrUnreadable):
|
||||
logger.Printf("%s: %v; dropped", m.Subject(), err)
|
||||
h.Outcome, h.Words = index.HeardDropped, err.Error()
|
||||
_ = m.Term()
|
||||
case err != nil:
|
||||
logger.Printf("%s: %s#%d left for redelivery: %v", m.Subject(), f.Repository, f.Number, err)
|
||||
h.Outcome, h.Words = index.HeardLeft, err.Error()
|
||||
default:
|
||||
h.Outcome = index.HeardApplied
|
||||
// Waits for the server, so the consumer's info no longer counts the message as unacknowledged when the fill that
|
||||
// follows asks whether the stream is applied (issue 1280); a failed ack is redelivered after the ack wait, which
|
||||
// kicks the fill again.
|
||||
if err := m.DoubleAck(ctx); err != nil {
|
||||
logger.Printf("%s: %s#%d applied, its acknowledgement was not confirmed: %v", m.Subject(), f.Repository, f.Number, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// forgeKind is the kind a FORGE message names, and whether it is one the indexer takes; a body naming no kind is taken,
|
||||
// so DecodeForge says why it is unreadable.
|
||||
func forgeKind(data []byte) (string, bool) {
|
||||
var probe struct {
|
||||
Kind string `json:"kind"`
|
||||
}
|
||||
if json.Unmarshal(data, &probe) != nil || probe.Kind == "" {
|
||||
return "", true
|
||||
}
|
||||
switch probe.Kind {
|
||||
case forgemsg.ForgePull, forgemsg.ForgeMerged, forgemsg.ForgePushed, forgemsg.ForgeRefused, forgemsg.ForgeOpen:
|
||||
return probe.Kind, true
|
||||
}
|
||||
return probe.Kind, false
|
||||
}
|
||||
|
||||
func opOf(e jetstream.KeyValueEntry) bool {
|
||||
return e.Operation() == jetstream.KeyValuePut
|
||||
}
|
||||
@@ -402,7 +281,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
requirements := process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.RequirementsState), "what consumers require is not kept", logger.Printf)
|
||||
// The module graph (ADR 0603 §2) is held in memory, read with the view: each revision's entries carry its nodes.
|
||||
// The state heard (A-80): the forge messages handled (key forge) and what the indexer heard and compares later (the time a call is
|
||||
// unanswered since, the newest toolchain-agreed and version-delivered, the artifacts published so far of a version).
|
||||
// unanswered since, the newest toolchain-agreed, the artifacts published so far of a version).
|
||||
heardStore := awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.HeardState), "what the indexer heard is not kept", logger.Printf)}
|
||||
ownBuildState := process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.OwnBuildStateName), "which build this indexer runs is not kept", logger.Printf)
|
||||
// The runs on request the indexer holds a candidate for (A-174): a state this delivery brings, bound when it appears.
|
||||
@@ -453,8 +332,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
ix := &index.Indexer{Module: cred.Module, NamedSets: named, Findings: writer, OldSections: awaitedStore{b: oldSections}, Requirements: awaitedStore{b: requirements}, Store: kvStore{candidates}, Modules: modules, Manifests: kvStore{manifests}, Taken: awaitedStore{b: taken}, Builds: known,
|
||||
Predefined: awaitedStore{b: predefined}, Logf: logger.Printf,
|
||||
Named: namedVia(calls.Call), Trunk: trunkVia(calls.Call), BranchHead: branchHeadVia(calls.Call), Contains: containsVia(calls.Call), Read: readVia(calls.Call),
|
||||
ReadOpen: openVia(calls.Call), Packages: awaitedStore{b: packages, js: js, name: index.Bucket(cred.Module, index.PackagesState)}, Files: filesVia(calls.Call), GoSum: goSumVia(calls.Call), GoMod: goModVia(calls.Call),
|
||||
OwnBuild: ownBuildVia(calls.Call, cred.Module, cred.Node), OwnBuildState: awaitedStore{b: ownBuildState}, Heard: heardStore, Placement: placementVia(calls.CallKept), RunningWhole: runningWholeVia(js), Runs: awaitedStore{b: runs}, Graph: graph,
|
||||
Packages: awaitedStore{b: packages, js: js, name: index.Bucket(cred.Module, index.PackagesState)}, Files: filesVia(calls.Call), GoSum: goSumVia(calls.Call), GoMod: goModVia(calls.Call),
|
||||
OwnBuild: ownBuildVia(calls.Call, cred.Module, cred.Node), OwnBuildState: awaitedStore{b: ownBuildState}, Heard: heardStore, Runs: awaitedStore{b: runs}, Graph: graph,
|
||||
SeatDefinitions: awaitedStore{b: seatDefs}, ReadFile: readFileVia(calls.Call), SaySeatDefinition: saySeats,
|
||||
Revisions: revisions, EntryIndex: awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.EntryIndexState), "no entry is found by its fingerprint", logger.Printf)}, // Message a kept call: a read, called again through a transport fault (ADR 0668 §1)
|
||||
// Each catalog revision may bring another holder of a build seat: the build modules are found again (buildmodules.go).
|
||||
@@ -468,9 +347,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
Inputs: inputsStore,
|
||||
InputObjects: inObjects,
|
||||
SayVersionReleased: sayVersionReleased(conn, cred), SayVersionsFilled: sayVersionsFilled(conn, cred),
|
||||
// The take-in step (ADR 0633): its state pending and its stream take-in, both this delivery's, bound when they appear;
|
||||
// until then every merge is taken in at once, as before.
|
||||
Pending: awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.PendingState), "no merge is held for the indexer its revision names: each is taken in at once", logger.Printf)},
|
||||
SayTakeIn: sayTakeIn(js, cred)}
|
||||
// The registrations and the changes that wait for their warrant (design 47 G §2, §667): their states are bound when they
|
||||
// appear; a warrant is heard on the router's decided event for this module, below.
|
||||
@@ -478,7 +354,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.RegistrationsState), "no registration is kept or written", logger.Printf)},
|
||||
awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.ProposalsState), "no change of a registration waits for its warrant", logger.Printf)},
|
||||
routerRecordOf(cred.Module, calls.Call, calls.Ask), ix, logger.Printf)
|
||||
ix.Message = messageVia(calls.CallKeptWhileOpen, ix.TakeInPending, logger.Printf) // a kept call, called again through a transport fault while the commit's take-in is pending (ADR 0668 §1, A-190)
|
||||
registry.Pruned = ix.PruneBranchBuilds
|
||||
ix.Registry = registry
|
||||
named.Changed = ix.RemoveUnlisted
|
||||
@@ -535,11 +410,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
started.Store(true)
|
||||
done()
|
||||
logger.Printf("start: the build log held %d entries at the start", len(startEnds))
|
||||
// The source revision of a watched branch (design 47 G §15, §115, §362): asked of the seat mesh-delivery's verb source as a
|
||||
// kept call, once at the start, before the handler of its event source-moved reads anything; a source-moved heard
|
||||
// meanwhile is read after.
|
||||
source := &sourceFollow{read: sourceVia(ctx, calls.CallKeptWhileOpen, ix.BranchWatched, logger.Printf), apply: ix.ApplySource, applyBranch: ix.ApplyBranchSource, logf: logger.Printf}
|
||||
go source.catchUp(ctx)
|
||||
// Only the toolchain reports of nodes the catalogue places the build module on count (issue 1940, ADR 0549 §3): the
|
||||
// assignments is settled first, so the toolchains settle through the gate already knowing it.
|
||||
// The build modules are no names (design 47 *A module's registration*): the holders of the seats whose acting rule is
|
||||
@@ -590,12 +460,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
}
|
||||
buildModules.watching(tools)
|
||||
buildFollow.Store(buildModules)
|
||||
// The checker's merge previews give the candidates their fingerprints: the bucket is watched where it exists, and
|
||||
// otherwise when the checker says it wrote a preview.
|
||||
previews := newPreviewWatch(ctx, js, ix, logger)
|
||||
if err := previews.ensure(); err != nil {
|
||||
return err
|
||||
}
|
||||
// Only now, with the toolchains read, are the fingerprints computed once (before it every Go module would be written as
|
||||
// having no toolchain reported, and back).
|
||||
// The packages the held modules reach and the state does not hold yet are published at their trunk's taken merge, and
|
||||
@@ -660,12 +524,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// indexer_first_views and as view-waits past five minutes, and read again on the next forge message heard.
|
||||
defs := gitWaits(ctx, cred.Module, ix, writer, cred.Node, process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, views.StatusState), "indexer_first_views does not say the wait for the seat git", logger.Printf), logger.Printf)
|
||||
seatDefs.Ready(func(jetstream.KeyValue) { defs.Start(ctx) })
|
||||
// A take-in held while a state was not on the bus is evaluated again at the start (issue 1966 note 154).
|
||||
go func() {
|
||||
if err := ix.PendingAgain(ctx); err != nil {
|
||||
logger.Printf("take-in: pending is not evaluated again at the start: %v", err)
|
||||
}
|
||||
}()
|
||||
// A build module says each time it wrote its toolchains key: its stream, and the indexer's durable consumer on it, are
|
||||
// made by the controller as the forge's are. Where the stream is not there yet the indexer says so and is started again
|
||||
// with the build module, as with the gate's source.
|
||||
@@ -695,9 +553,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// The versions each package's series holds (DLV-I5): the registry seats' published, on the
|
||||
// indexer's durable consumer of the shared events stream. Where it is not there yet, no series moves until the indexer
|
||||
// starts again, said.
|
||||
if err := hub.Handle(meshdelivery.SubjectSourceMovedAll, func(m jetstream.Msg) { source.heard(ctx, m) }); err != nil {
|
||||
return err
|
||||
}
|
||||
// Once the state is bound every kept revision is judged against the pins it holds (design 47 H §749: at its start).
|
||||
pinsBucket.Ready(func(jetstream.KeyValue) { go ix.JudgePins(ctx) })
|
||||
versions := func(m jetstream.Msg) {
|
||||
@@ -711,9 +566,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
if err := hub.Handle(meshbuilds.SubjectToolchainAgreed, func(m jetstream.Msg) { heardToolchainAgreed(ctx, ix, m, logger.Printf) }); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := hub.Handle(meshwants.SubjectVersionDelivered, func(m jetstream.Msg) { heardVersionDelivered(ctx, ix, m, logger.Printf) }); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := hub.Handle(meshwants.SubjectWaitEnded, func(m jetstream.Msg) { heardWaitEnded(ctx, ix, m, logger.Printf) }); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -742,32 +594,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
})
|
||||
hearNamed(ctx, js, cred, notHeard, named, logger.Printf)
|
||||
hearRuns(ctx, js, cred, notHeard, ix, logger.Printf)
|
||||
// The checker's verdicts carry a pull request's out-of-step files (DLV-I20): on the indexer's durable consumer of the
|
||||
// checker's stream asks. Where it is not there yet, no pull request's out-of-step files are kept until the indexer
|
||||
// starts again, said.
|
||||
hear(checkerAsksStream, "no pull request's out-of-step files are kept until the indexer starts again", func(m jetstream.Msg) { heardVerdict(ctx, ix, m, logger.Printf) })
|
||||
if stopped != nil {
|
||||
return stopped
|
||||
}
|
||||
// The pending revisions that name this build are taken in now, oldest first, before the consumer on the forge bridge's
|
||||
// stream starts (ADR 0633 §2). When the build this program runs is not known yet, nothing waits for it: the consumer
|
||||
// starts, and a merged message left unacknowledged asks the build again at its redelivery (clause 1). A hand take-in
|
||||
// (indexer_take_in_pending) is answered at any time: the takeins watch runs already.
|
||||
// A take-in that fails (a bus error, or a merge lost because the forge bridge did not answer) does not end the process (issue
|
||||
// 1585, note 70): what pending holds is evaluated again at the next merged message, toolchain report or hand take-in.
|
||||
if _, words, err := ix.TakeInAtStart(ctx); err != nil {
|
||||
logger.Printf("the take-in at start failed and is asked again at the next merged message, toolchain report or hand take-in: %v", err)
|
||||
} else if words != "" {
|
||||
logger.Printf("take-in at start: %s", words)
|
||||
}
|
||||
cons, err := cred.EventConsumer(ctx, js, forgeStream)
|
||||
if err != nil {
|
||||
// Essential: the forge's merges are what the indexer indexes.
|
||||
return fmt.Errorf("the consumer %s_%s on %s cannot be read (%v): the indexer hears no merge without it, so it stops; where the bus refused it or did not answer, this module's consumes (made by the controller with its grant) have not reached the bus yet", cred.Node, cred.Module, forgeStream, err)
|
||||
}
|
||||
// A merge applied while the taken state was missing gets its taken key (issue 1280): when the state appears (Ready runs
|
||||
// once, now or then), and after each applied message. The gate's source heads never move a key (issue 1126).
|
||||
taken.Ready(func(jetstream.KeyValue) { ix.KickFill(ctx) })
|
||||
// What the indexer heard of FORGE (issue 1126 note 76) is bound when its state appears, as every state a delivery adds
|
||||
// (issue 1790): until then no message is written there, and the rest of the indexer runs.
|
||||
heard := heardStore
|
||||
@@ -779,16 +608,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
index.PackagesState: awaited(packages),
|
||||
index.HeardState: awaited(heard.b), "taken": awaited(taken), "findings": awaited(findings),
|
||||
index.SeatDefinitionsState: awaited(seatDefs)}, errs, logger.Printf)
|
||||
cc, err := process.HearEvents(cons, func(m jetstream.Msg) {
|
||||
handle(ctx, m, ix, heard, logger)
|
||||
ix.KickFill(ctx)
|
||||
unreadAgain(ctx, ix, logger)
|
||||
defs.Heard(ctx)
|
||||
}, logger.Printf)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer cc.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
@@ -798,7 +617,5 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
return err
|
||||
case err := <-tools.errs:
|
||||
return err
|
||||
case err := <-previews.errs:
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,22 +1,13 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"git.novox.be/novox/mesh-sdk/go/busretry"
|
||||
"log"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
|
||||
@@ -83,124 +74,6 @@ const testIndexer = "mesh-delivery-indexer"
|
||||
|
||||
var testAccount = process.Account{Node: "n1", Module: testIndexer}
|
||||
|
||||
func TestAMessageIsIndexedAndAcked(t *testing.T) { // 1238/AC2 (unit part)
|
||||
// Review round 2: the indexer's views emitter writes its finding view-unsaid (Emitter.Findings set).
|
||||
if em := viewsEmitter(nil, process.Account{}, nil, noWire{}, t.Logf); em.Findings == nil {
|
||||
t.Error("the indexer's views emitter writes no view-unsaid finding")
|
||||
}
|
||||
// Milestone 47/G: the object buckets are the module's own by the name its account runs as, never a fixed name.
|
||||
if b := newBusObjects(nil, process.Account{Module: "renamed-indexer"}, index.CatalogRevisionsObjects).bucket(); b != "renamed-indexer_catalog-revisions" {
|
||||
t.Errorf("the catalog graph revisions open as %s, want the account's module's", b)
|
||||
}
|
||||
t.Run("not a forge message: terminated and logged", func(t *testing.T) { // 1238 (never redelivered)
|
||||
var logs bytes.Buffer
|
||||
st := &memStore{vals: map[string][]byte{}}
|
||||
m := &fakeMsg{subject: "mesh.stream.mesh-delivery-forge-bridge.event.forge.x.y.pull", data: []byte(`{"v":1,"kind":"pull"}`)}
|
||||
heard := &memStore{vals: map[string][]byte{}}
|
||||
handle(context.Background(), m, &index.Indexer{Store: st, Builds: index.NewBuilds()}, heard, log.New(&logs, "", 0))
|
||||
if m.termd != 1 || m.acked != 0 || len(st.vals) != 0 || !strings.Contains(logs.String(), m.subject) {
|
||||
t.Errorf("terminated %d acked %d keys %d log %q", m.termd, m.acked, len(st.vals), logs.String())
|
||||
}
|
||||
if h := heardOf(t, heard); h.Outcome != index.HeardDropped || h.Sequence != 41 || h.Words == "" {
|
||||
t.Errorf("heard = %+v", h)
|
||||
}
|
||||
})
|
||||
t.Run("a kind it does not take (branch-deleted): acknowledged and passed over", func(t *testing.T) { // design 51, DLV-FB1
|
||||
st := &memStore{vals: map[string][]byte{}}
|
||||
m := &fakeMsg{subject: "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-sdk.branch-deleted",
|
||||
data: []byte(`{"v":1,"kind":"branch-deleted","repository":"novox/mesh-sdk","branch":"feat/x","head":"` + head + `"}`)}
|
||||
heard := &memStore{vals: map[string][]byte{}}
|
||||
handle(context.Background(), m, &index.Indexer{Store: st, Builds: index.NewBuilds()}, heard, log.New(&bytes.Buffer{}, "", 0))
|
||||
if m.termd != 0 || m.acked != 1 || len(st.vals) != 0 {
|
||||
t.Errorf("terminated %d acked %d keys %d", m.termd, m.acked, len(st.vals))
|
||||
}
|
||||
if h := heardOf(t, heard); h.Outcome != index.HeardIgnored || h.Kind != "branch-deleted" {
|
||||
t.Errorf("heard = %+v", h)
|
||||
}
|
||||
})
|
||||
st := &memStore{vals: map[string][]byte{}}
|
||||
ix := &index.Indexer{Store: st, Builds: index.NewBuilds()}
|
||||
data, _ := forgemsg.EncodeForge(forgemsg.Forge{Kind: forgemsg.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head,
|
||||
Action: forgemsg.PullUpdated, Dirs: []string{"modules/a"}, DirsKnown: true})
|
||||
m := &fakeMsg{subject: "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-delivery.pull", data: data}
|
||||
heard := &memStore{vals: map[string][]byte{}}
|
||||
handle(context.Background(), m, ix, heard, log.New(&bytes.Buffer{}, "", 0))
|
||||
c, err := index.DecodeCandidate(st.vals["feat/x"])
|
||||
if err != nil || len(c.Pulls) != 1 || c.Pulls[0].Modules[0] != "a" || m.acked != 1 || m.doubled != 1 || m.termd != 0 {
|
||||
t.Errorf("candidate %+v err %v acked %d terminated %d", c, err, m.acked, m.termd)
|
||||
}
|
||||
// Issue 1126 note 76: what the indexer heard of FORGE is readable (indexer_heard): subject, sequence, deliveries, time,
|
||||
// repository, commit and what it did.
|
||||
if h := heardOf(t, heard); h.Subject != m.subject || h.Sequence != 41 || h.Delivered != 2 || h.Published.Hour() != 13 || h.Kind != forgemsg.ForgePull ||
|
||||
h.Repository != "novox/mesh-delivery" || h.Number != 12 || h.Commit != head || h.Outcome != index.HeardApplied || h.Handled.IsZero() {
|
||||
t.Errorf("heard = %+v", h)
|
||||
}
|
||||
t.Run("a group's end is told apart by its name before the forge decode", func(t *testing.T) {
|
||||
var logs bytes.Buffer
|
||||
st := &memStore{vals: map[string][]byte{}}
|
||||
heard := &memStore{vals: map[string][]byte{}}
|
||||
// 1585/T9 (ADR 0633 §4): a group's end is told apart by its name before the forge decode: one that reads is acked
|
||||
// (an indexer with no pending state takes the members in one by one), one that does not is dropped as a group's end.
|
||||
for _, c := range []struct {
|
||||
data string
|
||||
termd int
|
||||
logged string
|
||||
}{{`{"v":1,"group":"feat/x","members":[{"repository":"novox/mesh-sdk","number":3,"merge":"` + head + `"}]}`, 0, ""},
|
||||
{`{"v":1,"group":"feat/x"}`, 1, "not a group's end"}} {
|
||||
logs.Reset()
|
||||
m := &fakeMsg{subject: "mesh.stream.mesh-delivery-forge-bridge.event." + forgemsg.ForgeGroupEndEvent, data: []byte(c.data)}
|
||||
handle(context.Background(), m, &index.Indexer{Store: st, Builds: index.NewBuilds()}, heard, log.New(&logs, "", 0))
|
||||
if m.termd != c.termd || m.acked+m.termd != 1 || !strings.Contains(logs.String(), c.logged) {
|
||||
t.Errorf("group end %s: terminated %d acked %d log %q", c.data, m.termd, m.acked, logs.String())
|
||||
}
|
||||
if h := heardOf(t, heard); h.Kind != forgemsg.ForgeGroupEndEvent && c.termd == 0 {
|
||||
t.Errorf("heard of a group's end = %+v", h)
|
||||
}
|
||||
}
|
||||
// The repository forge/group's merged message ends as a group's end does: it is a forge message all the same.
|
||||
logs.Reset()
|
||||
m := &fakeMsg{subject: "mesh.stream.mesh-delivery-forge-bridge.event.forge.forge.group.merged", data: []byte(`{"v":1,"group":"feat/x","members":[{"repository":"novox/mesh-sdk","number":3,"merge":"` + head + `"}]}`)}
|
||||
handle(context.Background(), m, &index.Indexer{Store: st, Builds: index.NewBuilds()}, heard, log.New(&logs, "", 0))
|
||||
if !strings.Contains(logs.String(), "not a forge message") {
|
||||
t.Errorf("forge/group's merged message read as a group's end: %q", logs.String())
|
||||
}
|
||||
})
|
||||
// Issue 1966: a bucket the bus has not made is "not yet"; one it cannot reach (jetstream not enabled, no responders) is the
|
||||
// bus's own error, which stops a pass instead of reading as nothing published.
|
||||
t.Run("a bucket not found is not yet, one the bus cannot reach is its error", func(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
err error
|
||||
notYet bool
|
||||
}{{"not found", jetstream.ErrBucketNotFound, true}, {"not reachable", jetstream.ErrJetStreamNotEnabled, false}} {
|
||||
b := process.AwaitBucket(context.Background(), opener{c.err}, nil, "packages", "none", t.Logf)
|
||||
_, err := awaitedStore{b: b, js: opener{c.err}, name: "packages"}.Keys(context.Background())
|
||||
if errors.Is(err, process.ErrNotYet) != c.notYet || err == nil || (!c.notYet && !errors.Is(err, c.err)) {
|
||||
t.Errorf("%s: Keys answered %v, want not-yet %v", c.name, err, c.notYet)
|
||||
}
|
||||
}
|
||||
})
|
||||
// Design 47 G A-397: the program hears its kept seat events (package-published among them) on their own streams once its
|
||||
// durable on EVENTS is bound, never when that fails.
|
||||
for _, startErr := range []error{nil, errors.New("not composed yet")} {
|
||||
h := &startedHub{startErr: startErr}
|
||||
if err := startHearing(context.Background(), h, nil, process.Account{}); !errors.Is(err, startErr) || h.kept != (startErr == nil) {
|
||||
t.Errorf("start error %v: answered %v, kept streams bound %v", startErr, err, h.kept)
|
||||
}
|
||||
}
|
||||
// Design 47 G A-420: a package-published that cannot be kept is left for the bus, and on its last delivery said given up,
|
||||
// naming its subject, the artifact and version and the last refusal; before it, nothing more is said.
|
||||
for _, n := range []uint64{4, EventDeliveries} {
|
||||
var said []string
|
||||
m := &deliveredMsg{subject: "mesh.seat.mesh-builds.event.package-published", data: []byte(`{"artifact":"m:program","version":"0.1.6"}`), n: n}
|
||||
heardPackagePublished(context.Background(), &index.Indexer{}, m, func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) })
|
||||
givenUp := strings.Contains(strings.Join(said, "\n"), "gives up mesh.seat.mesh-builds.event.package-published: the package-published of m:program 0.1.6, delivered 5 times")
|
||||
if !m.naked || givenUp != (n == EventDeliveries) || (givenUp && !strings.Contains(strings.Join(said, "\n"), "the last refusal: the state versions is not kept here")) {
|
||||
t.Errorf("delivery %d: naked %v, said %q", n, m.naked, said)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// deliveredMsg is a message delivered n times, which records a Nak.
|
||||
type deliveredMsg struct {
|
||||
jetstream.Msg
|
||||
@@ -222,90 +95,11 @@ type opener struct{ err error }
|
||||
|
||||
func (o opener) KeyValue(context.Context, string) (jetstream.KeyValue, error) { return nil, o.err }
|
||||
|
||||
func TestTheSourceIsReadFromTheSeatAndASourceMovedDuringTheCatchUpIsReadAfterIt(t *testing.T) { // design 47 G §15, §115, §362
|
||||
var order []string
|
||||
revision := int64(4)
|
||||
var logs []string
|
||||
var s *sourceFollow
|
||||
s = &sourceFollow{
|
||||
read: func(branch string) (meshdelivery.SourceAnswer, error) {
|
||||
order = append(order, "read "+branch)
|
||||
if branch == "" { // given no branch the seat lists the branches it keeps a source of (A-93)
|
||||
return meshdelivery.SourceAnswer{Branches: []string{"main", "beta", "feat=2e1"}}, nil
|
||||
}
|
||||
if revision == 0 {
|
||||
return meshdelivery.SourceAnswer{}, fmt.Errorf("source.main is not readable")
|
||||
}
|
||||
if len(order) == 1 { // a source-moved is heard while the catch-up reads
|
||||
s.moved(context.Background(), "main")
|
||||
s.moved(context.Background(), "main")
|
||||
s.moved(context.Background(), "beta")
|
||||
}
|
||||
return meshdelivery.SourceAnswer{Branch: branch, Revision: revision}, nil
|
||||
},
|
||||
apply: func(_ context.Context, a meshdelivery.SourceAnswer) error {
|
||||
order = append(order, fmt.Sprintf("apply %s %d", a.Branch, a.Revision))
|
||||
return nil
|
||||
},
|
||||
applyBranch: func(_ context.Context, branch string, a meshdelivery.SourceAnswer) error {
|
||||
order = append(order, fmt.Sprintf("branch %s %d", branch, a.Revision))
|
||||
return nil
|
||||
},
|
||||
logf: func(f string, a ...any) { logs = append(logs, fmt.Sprintf(f, a...)) },
|
||||
}
|
||||
s.catchUp(context.Background())
|
||||
// The trunk, then each other watched branch the seat lists (a name unescaped), then what was heard meanwhile once each,
|
||||
// beta among them: main through the take-in, every other branch into its own revision (A-93).
|
||||
want := []string{"read main", "apply main 4", "read ", "read beta", "branch beta 4", "read feat=2e1", "branch feat.1 4", "read beta", "branch beta 4", "read main", "apply main 4"}
|
||||
if !reflect.DeepEqual(order, want) {
|
||||
t.Errorf("order = %v, want %v", order, want)
|
||||
}
|
||||
s.moved(context.Background(), "main") // after the catch-up: read at once
|
||||
if len(order) != 13 {
|
||||
t.Errorf("a source-moved after the catch-up was not read at once: %v", order)
|
||||
}
|
||||
order, revision = nil, 0
|
||||
s.moved(context.Background(), "main")
|
||||
if len(order) != 1 || len(logs) != 1 || !strings.Contains(logs[0], "not readable") {
|
||||
t.Errorf("a source that cannot be read: order %v, said %v, want it said and nothing applied", order, logs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheSeatsSourceAnswerIsReadAndItsErrorIsAnError(t *testing.T) { // design 47 G §115
|
||||
var asked any
|
||||
read := sourceVia(context.Background(), func(_ busretry.Open, key string, body any) (json.RawMessage, error) {
|
||||
asked = body
|
||||
if key != "seat:mesh-delivery.source" {
|
||||
t.Errorf("asked %q", key)
|
||||
}
|
||||
if body.(meshdelivery.SourceInput).Branch == "bad" {
|
||||
return json.RawMessage(`{"error":"source.bad is not readable"}`), nil
|
||||
}
|
||||
return json.RawMessage(`{"branch":"main","revision":7,"repositories":[{"repository":"novox/mesh-delivery","head":"` + head + `","trunk":"main"}]}`), nil
|
||||
}, func(context.Context, string) (bool, error) { return true, nil }, t.Logf)
|
||||
a, err := read("main")
|
||||
if err != nil || a.Revision != 7 || len(a.Repositories) != 1 || asked.(meshdelivery.SourceInput).Branch != "main" {
|
||||
t.Errorf("source = %+v, %v", a, err)
|
||||
}
|
||||
if _, err := read("bad"); err == nil || !strings.Contains(err.Error(), "not readable") {
|
||||
t.Errorf("an answer with an error was no error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func ptr[T any](v T) *T { return &v }
|
||||
|
||||
// noWire is a finding writer that is none: a non-nil WireWriter for the test of which emitter is made.
|
||||
type noWire struct{ views.WireWriter }
|
||||
|
||||
// Design 47 G §509, §714, A-269: the indexer takes a pushed message alike with a merged one.
|
||||
func TestTheIndexerTakesAPushedMessage(t *testing.T) {
|
||||
for _, kind := range []string{forgemsg.ForgeMerged, forgemsg.ForgePushed} {
|
||||
if got, known := forgeKind([]byte(`{"kind":"` + kind + `"}`)); !known || got != kind {
|
||||
t.Errorf("forgeKind(%s) = %q, %v, want it taken", kind, got, known)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// startedHub records which of an event hub's starts were asked.
|
||||
type startedHub struct {
|
||||
startErr error
|
||||
|
||||
@@ -12,7 +12,6 @@ import (
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk"
|
||||
|
||||
"mesh-delivery-indexer/internal/fingerprint"
|
||||
@@ -240,27 +239,6 @@ func readVia(call toolCaller) func(ctx context.Context, repository, sha, dir str
|
||||
// trunkBranch is the branch pull requests are open into.
|
||||
const trunkBranch = index.TrunkBranch
|
||||
|
||||
// openVia asks the forge which pull requests into the trunk are open now (the seat git's open-pulls), every repository's.
|
||||
// An answer that lacks the list is refused, never read as none.
|
||||
func openVia(call toolCaller) func(ctx context.Context) ([]forgemsg.OpenPull, error) {
|
||||
return func(context.Context) ([]forgemsg.OpenPull, error) {
|
||||
var o git.OpenPullsAnswer
|
||||
if err := askGit(call, git.VerbOpenPulls, git.OpenPullsInput{}, &o); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if o.Pulls == nil {
|
||||
return nil, fmt.Errorf("the answer of %s lists no pulls", gitVerb(git.VerbOpenPulls))
|
||||
}
|
||||
out := []forgemsg.OpenPull{}
|
||||
for _, p := range o.Pulls {
|
||||
if p.Pull.Base == trunkBranch && p.Pull.Number > 0 {
|
||||
out = append(out, forgemsg.OpenPull{Repository: p.Pull.Repository, Number: int(p.Pull.Number)})
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
}
|
||||
|
||||
// branchHeadVia is the head of a repository's branch through the seat git's verb branch.
|
||||
func branchHeadVia(call toolCaller) func(ctx context.Context, repository, branch string) (string, error) {
|
||||
return func(_ context.Context, repository, branch string) (string, error) {
|
||||
|
||||
@@ -47,27 +47,6 @@ func TestNamedViaAsksTheTreesToolWithThePathsAndTakesTheirObjects(t *testing.T)
|
||||
if _, err := namedVia(failed)(context.Background(), "novox/mesh-sdk", "s", nil); err == nil || !strings.Contains(err.Error(), "the forge did not answer") {
|
||||
t.Errorf("an error answer: %v, want its failure", err)
|
||||
}
|
||||
for _, row := range []struct {
|
||||
name, answer string
|
||||
want int
|
||||
refused bool
|
||||
}{
|
||||
{"one open into the trunk, one into another branch", `{"pulls":[{"pull":{"repository":"novox/a","number":3,"base":"main","branch":"b","head":"h"}},{"pull":{"repository":"novox/a","number":4,"base":"dev","branch":"c","head":"h"}}]}`, 1, false},
|
||||
{"none open", `{"pulls":[]}`, 0, false},
|
||||
{"no list", `{}`, 0, true},
|
||||
{"an error", `{"error":"refused","error-kind":"refused"}`, 0, true},
|
||||
} {
|
||||
open := func(key string, _ any) (json.RawMessage, error) {
|
||||
if key != "seat:git.open-pulls" {
|
||||
t.Errorf("called %s", key)
|
||||
}
|
||||
return json.RawMessage(row.answer), nil
|
||||
}
|
||||
got, err := openVia(open)(context.Background())
|
||||
if (err != nil) != row.refused || len(got) != row.want {
|
||||
t.Errorf("open-pulls %s: %v %v, want %d, refused %v", row.name, got, err, row.want, row.refused)
|
||||
}
|
||||
}
|
||||
call = func(string, any) (json.RawMessage, error) {
|
||||
return json.RawMessage(`{"root":"r0","trees":{},"paths":{"":{"src":"1111111111111111111111111111111111111111"}}}`), nil
|
||||
}
|
||||
|
||||
@@ -1,66 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"log"
|
||||
"sync"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"mesh-delivery-indexer/internal/index"
|
||||
)
|
||||
|
||||
// previewWatch reads the merge previews the checker writes (mesh-delivery-checker_previews) and follows them: every change
|
||||
// writes the candidates' fingerprints again (index.Indexer.ApplyPreview). The checker's bucket is made when it is assigned,
|
||||
// after the indexer started, and the indexer holds no timer (ADR 0437 §8) and hears no preview.written (design 47 G A-160): a
|
||||
// bucket absent at the start is not watched until the indexer starts again.
|
||||
type previewWatch struct {
|
||||
ctx context.Context
|
||||
js bucketOpener
|
||||
ix *index.Indexer
|
||||
logger *log.Logger
|
||||
errs chan error
|
||||
mu sync.Mutex
|
||||
watched bool
|
||||
}
|
||||
|
||||
func newPreviewWatch(ctx context.Context, js bucketOpener, ix *index.Indexer, logger *log.Logger) *previewWatch {
|
||||
return &previewWatch{ctx: ctx, js: js, ix: ix, logger: logger, errs: make(chan error, 1)}
|
||||
}
|
||||
|
||||
// ensure watches the previews bucket if it is not watched and exists, settling what it holds, then writes the candidates'
|
||||
// fingerprints from it.
|
||||
func (w *previewWatch) ensure() error {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
if w.watched {
|
||||
return nil
|
||||
}
|
||||
kv, err := w.js.KeyValue(w.ctx, index.PreviewsBucket)
|
||||
switch {
|
||||
case errors.Is(err, jetstream.ErrBucketNotFound):
|
||||
w.logger.Printf("the bucket %s does not exist yet: it is watched from the indexer's next start", index.PreviewsBucket)
|
||||
return nil
|
||||
case err != nil:
|
||||
// Refused or not answered (its grant not on the bus yet, issue 1945): said, and opened again at the checker's next preview.
|
||||
w.logger.Printf("the bucket %s cannot be read (%v): it is watched from the indexer's next start", index.PreviewsBucket, err)
|
||||
return nil
|
||||
}
|
||||
watcher, err := settle(w.ctx, kv, func(e jetstream.KeyValueEntry) { w.ix.PutPreview(e.Key(), e.Value(), !opOf(e)) })
|
||||
if err != nil {
|
||||
process.NotWatched(w.logger.Printf, index.PreviewsBucket, err, "it is watched from the indexer's next start")
|
||||
return nil
|
||||
}
|
||||
w.watched = true
|
||||
go func() {
|
||||
w.errs <- follow(w.ctx, watcher, func(e jetstream.KeyValueEntry) {
|
||||
if err := w.ix.ApplyPreview(w.ctx, e.Key(), e.Value(), !opOf(e)); err != nil {
|
||||
w.logger.Printf("%s.%s could not be taken into the candidates: %v", index.PreviewsBucket, e.Key(), err)
|
||||
}
|
||||
})
|
||||
}()
|
||||
return w.ix.RecomputeCandidates(w.ctx)
|
||||
}
|
||||
@@ -8,18 +8,13 @@ import (
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdeliverychecker"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"mesh-delivery-indexer/internal/index"
|
||||
)
|
||||
|
||||
// The seat definitions' wiring (DLV-I17-I20; design 26): the forge's get-file tool reads them, seat-definition-changed
|
||||
// is said on the seat mesh-catalog's own subject (kept on the shared events stream, seven days), and the checker's verdict event, heard on
|
||||
// the indexer's durable consumer of the checker's stream asks, carries a pull request's out-of-step files.
|
||||
|
||||
// checkerAsksStream is the checker's stream asks as the bus names it (its manifest's streams; StreamBusName).
|
||||
const checkerAsksStream = "ASKS"
|
||||
// is said on the seat mesh-catalog's own subject (kept on the shared events stream, seven days),.
|
||||
|
||||
// readFileVia reads one file of a repository at a commit through the seat git's file; a path the commit does not
|
||||
// hold is nil with no error.
|
||||
@@ -60,23 +55,3 @@ func saySeatDefinition(js jetstream.JetStream, cred process.Account) func(contex
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// heardVerdict takes the checker's verdict event: a pull request's out-of-step files are kept (a clean verdict removes
|
||||
// them). Another event of the stream is acknowledged and passed over; a write that failed is delivered again.
|
||||
func heardVerdict(ctx context.Context, ix *index.Indexer, m jetstream.Msg, logf func(string, ...any)) {
|
||||
if m.Subject() != meshdeliverychecker.HolderSubjectVerdict(index.CheckerModule) {
|
||||
_ = m.Ack()
|
||||
return
|
||||
}
|
||||
var v meshdeliverychecker.VerdictHolderEvent
|
||||
if err := json.Unmarshal(m.Data(), &v); err != nil {
|
||||
logf("%s holds what is no verdict, left: %v", m.Subject(), err)
|
||||
_ = m.Term()
|
||||
return
|
||||
}
|
||||
if err := ix.ApplyVerdict(ctx, v.Repository, int(v.Pull), v.Head, v.OutOfStep); err != nil {
|
||||
logf("%s#%d: its out-of-step files are not kept, delivered again: %v", v.Repository, v.Pull, err)
|
||||
return
|
||||
}
|
||||
_ = m.Ack()
|
||||
}
|
||||
|
||||
@@ -1,135 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
"mesh-delivery-indexer/internal/index"
|
||||
)
|
||||
|
||||
// sourceInvoke is the seat mesh-delivery's verb source as the manifest's invokes names it: a watched branch's source
|
||||
// revision, asked of the seat and never read from the gate's state (design 47 G §15, §115; ADR 0596 §2).
|
||||
const sourceInvoke = "seat:" + meshdelivery.Seat + "." + meshdelivery.VerbSource
|
||||
|
||||
// sourceVia asks the seat for one branch's source revision through call (process.Conn.CallKeptWhileOpen, a read asked again while the branch is watched, A-190) and reads its
|
||||
// declared answer; an answer that carries an error is that error.
|
||||
func sourceVia(ctx context.Context, call openCaller, watched func(ctx context.Context, branch string) (bool, error), logf func(string, ...any)) func(branch string) (meshdelivery.SourceAnswer, error) {
|
||||
return func(branch string) (meshdelivery.SourceAnswer, error) {
|
||||
raw, err := keptWhileOpen(ctx, call, logf, "source."+branch, sourceInvoke, meshdelivery.SourceInput{Branch: branch},
|
||||
func(ctx context.Context) (bool, error) { return watched(ctx, branch) })
|
||||
if err != nil {
|
||||
return meshdelivery.SourceAnswer{}, err
|
||||
}
|
||||
var a meshdelivery.SourceAnswer
|
||||
if err := json.Unmarshal(raw, &a); err != nil {
|
||||
return a, fmt.Errorf("the answer of %s is not a source revision: %w", sourceInvoke, err)
|
||||
}
|
||||
if a.Error != "" {
|
||||
return a, fmt.Errorf("%s: %s", sourceInvoke, a.Error)
|
||||
}
|
||||
return a, nil
|
||||
}
|
||||
}
|
||||
|
||||
// sourceFollow takes the seat's source revisions into the indexer: first the catch-up of the heads the source names, then
|
||||
// each source-moved the seat says (design 47 G §362). A source-moved heard before the catch-up ended is applied after it,
|
||||
// once per branch.
|
||||
type sourceFollow struct {
|
||||
read func(branch string) (meshdelivery.SourceAnswer, error)
|
||||
apply func(context.Context, meshdelivery.SourceAnswer) error
|
||||
// applyBranch takes a watched branch other than the trunk's in (design 47 G §291, A-93); branch is its name, unescaped.
|
||||
applyBranch func(ctx context.Context, branch string, a meshdelivery.SourceAnswer) error
|
||||
logf func(string, ...any)
|
||||
|
||||
mu sync.Mutex
|
||||
caught bool
|
||||
waiting map[string]bool
|
||||
}
|
||||
|
||||
// catchUp reads the trunk branch's source revision and applies it, then every other branch the seat keeps a source of (the
|
||||
// answer given no branch lists them, A-93), then the branches a source-moved named meanwhile. A read that fails is said;
|
||||
// the catch-up ends all the same, and the next source-moved reads again.
|
||||
func (s *sourceFollow) catchUp(ctx context.Context) {
|
||||
s.take(ctx, trunkBranch)
|
||||
var others []string
|
||||
if list, err := s.read(""); err != nil {
|
||||
s.logf("the branches the source names are not read, and each is read at its next source-moved: %v", err)
|
||||
} else {
|
||||
others = list.Branches
|
||||
}
|
||||
for _, branch := range others {
|
||||
if branch != trunkBranch {
|
||||
s.take(ctx, branch)
|
||||
}
|
||||
}
|
||||
s.mu.Lock()
|
||||
s.caught = true
|
||||
branches := s.waiting
|
||||
s.waiting = nil
|
||||
s.mu.Unlock()
|
||||
for _, branch := range slices.Sorted(maps.Keys(branches)) { // a map's order is random: the branches are read in name order
|
||||
s.take(ctx, branch)
|
||||
}
|
||||
}
|
||||
|
||||
// moved is the handler of one source-moved on branch: held until the catch-up ends, read at once after.
|
||||
func (s *sourceFollow) moved(ctx context.Context, branch string) {
|
||||
s.mu.Lock()
|
||||
if !s.caught {
|
||||
if s.waiting == nil {
|
||||
s.waiting = map[string]bool{}
|
||||
}
|
||||
s.waiting[branch] = true
|
||||
s.mu.Unlock()
|
||||
return
|
||||
}
|
||||
s.mu.Unlock()
|
||||
s.take(ctx, branch)
|
||||
}
|
||||
|
||||
// take reads one branch's source revision (branch as the key name source.<branch> has it) and applies it: the trunk's
|
||||
// through the take-in, another watched branch's into its own revision (A-93).
|
||||
func (s *sourceFollow) take(ctx context.Context, branch string) {
|
||||
a, err := s.read(branch)
|
||||
if errors.Is(err, index.ErrAskClosed) { // the branch is no longer watched: not said unanswered (A-190)
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
s.logf("source.%s is not read, and is read again at its next source-moved: %v", branch, err)
|
||||
return
|
||||
}
|
||||
if branch != trunkBranch {
|
||||
name, err := keys.UnescapeName(branch)
|
||||
if err != nil {
|
||||
s.logf("source.%s names no branch: %v", branch, err)
|
||||
return
|
||||
}
|
||||
if s.applyBranch == nil {
|
||||
return
|
||||
}
|
||||
if err := s.applyBranch(ctx, name, a); err != nil {
|
||||
s.logf("source.%s at revision %d could not be taken into the branch's revision: %v", branch, a.Revision, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err := s.apply(ctx, a); err != nil {
|
||||
s.logf("source.%s at revision %d could not be taken into the modules: %v", branch, a.Revision, err)
|
||||
}
|
||||
}
|
||||
|
||||
// heard is the hub's handler of source-moved: the branch is the last token of the subject. The message is acknowledged at once.
|
||||
func (s *sourceFollow) heard(ctx context.Context, m jetstream.Msg) {
|
||||
defer func() { _ = m.Ack() }()
|
||||
if branch := m.Subject()[strings.LastIndex(m.Subject(), ".")+1:]; branch != "" {
|
||||
s.moved(ctx, branch)
|
||||
}
|
||||
}
|
||||
@@ -15,11 +15,7 @@ import (
|
||||
type takerIn interface {
|
||||
TakeIn(ctx context.Context, module string) index.TakenIn
|
||||
TakeInRepository(ctx context.Context, repository string) index.TakenIn
|
||||
CatchUp(ctx context.Context) index.TakenIn
|
||||
SeatDefinitionsAgain(ctx context.Context, module string, pull int, head string) index.TakenIn
|
||||
TakeInPendingByHand(ctx context.Context, key, by, why string) index.TakenIn
|
||||
DropPending(ctx context.Context, key, by, why string, abandon bool) index.TakenIn
|
||||
TakeInMerge(ctx context.Context, repository, merge string, seq uint64, by, why string) index.TakenIn
|
||||
FingerprintsAgain(ctx context.Context, module string) index.TakenIn
|
||||
RecomputeByHand(ctx context.Context, by, why string) index.TakenIn
|
||||
DropCandidateGraph(ctx context.Context, group, heads, by, why string) index.TakenIn
|
||||
@@ -69,18 +65,6 @@ func takenIn(ctx context.Context, ix takerIn, kv jetstream.KeyValue, e jetstream
|
||||
a.Module = r.Module
|
||||
} else if a.Fingerprints {
|
||||
asked, r = "computing of fingerprints again", ix.FingerprintsAgain(ctx, a.Module)
|
||||
} else if a.Merge != "" {
|
||||
asked, r = "hand take-in of a merge", ix.TakeInMerge(ctx, a.MergeRepository, a.Merge, a.Sequence, a.By, a.Why)
|
||||
a.Module = r.Module
|
||||
} else if a.Pending != "" && a.Drop {
|
||||
asked, r = "drop of a pending merge", ix.DropPending(ctx, a.Pending, a.By, a.Why, a.Abandon)
|
||||
a.Module = a.Pending
|
||||
} else if a.Pending != "" {
|
||||
asked, r = "hand take-in of what waits", ix.TakeInPendingByHand(ctx, a.Pending, a.By, a.Why)
|
||||
a.Module = a.Pending
|
||||
} else if a.CatchUp {
|
||||
asked, r = "catch-up of every repository", ix.CatchUp(ctx)
|
||||
a.Module = r.Module
|
||||
} else if a.Repository != "" {
|
||||
asked, r = "take-in of a repository", ix.TakeInRepository(ctx, a.Repository)
|
||||
a.Module = a.Repository
|
||||
|
||||
@@ -11,8 +11,6 @@ import (
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/gopackageregistry"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
||||
@@ -117,23 +115,6 @@ func heardToolchainAgreed(ctx context.Context, ix *index.Indexer, m jetstream.Ms
|
||||
_ = m.Ack()
|
||||
}
|
||||
|
||||
// heardVersionDelivered writes the delivered release's entries of kinds 2, 4 and 6 and keeps the newest delivered version of
|
||||
// its module (design 47 G §374, A-68), as heardToolchainAgreed keeps its event.
|
||||
func heardVersionDelivered(ctx context.Context, ix *index.Indexer, m jetstream.Msg, logf func(string, ...any)) {
|
||||
var d meshwants.VersionDeliveredEvent
|
||||
if err := json.Unmarshal(m.Data(), &d); err != nil {
|
||||
logf("%s holds what is no version-delivered, left: %v", m.Subject(), err)
|
||||
_ = m.Ack()
|
||||
return
|
||||
}
|
||||
if err := ix.HearVersionDelivered(ctx, d); err != nil {
|
||||
logf("versions: the version-delivered of %s %s could not be kept, so the bus gives it again: %v", d.Module, d.Version, err)
|
||||
leftForBus(m, fmt.Sprintf("the version-delivered of %s %s", d.Module, d.Version), err, logf)
|
||||
return
|
||||
}
|
||||
_ = m.Ack()
|
||||
}
|
||||
|
||||
// heardWaitEnded keeps the act of an acting part's ended wait for kind 5 (A-78 a), as heardToolchainAgreed keeps its event.
|
||||
func heardWaitEnded(ctx context.Context, ix *index.Indexer, m jetstream.Msg, logf func(string, ...any)) {
|
||||
var w meshwants.WaitEndedEvent
|
||||
@@ -240,61 +221,6 @@ func afterRevision(say func(context.Context, meshcatalog.CatalogRevisionEvent) e
|
||||
}
|
||||
}
|
||||
|
||||
// placementVia asks the seat mesh-module-assigner's verb placement for one module (A-62), a read.
|
||||
func placementVia(call toolCaller) index.PlacementAsker {
|
||||
return func(_ context.Context, module string) ([]meshmoduleassigner.ModulePlacement, error) {
|
||||
raw, err := call(meshmoduleassigner.Seat+"."+meshmoduleassigner.VerbPlacement, map[string]any{"module": module})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s.%s: %w", meshmoduleassigner.Seat, meshmoduleassigner.VerbPlacement, err)
|
||||
}
|
||||
var a meshmoduleassigner.PlacementAnswer
|
||||
if err := json.Unmarshal(raw, &a); err != nil {
|
||||
return nil, fmt.Errorf("the answer of placement is not a module's placements: %w", err)
|
||||
}
|
||||
return a.Placements, nil
|
||||
}
|
||||
}
|
||||
|
||||
// runningWholeVia reads every subject's last message of the node-engine seat's stream running through an ordered consumer made for
|
||||
// that one read (design 47 G §556), by node and module.
|
||||
func runningWholeVia(js jetstream.JetStream) index.RunningReader {
|
||||
return func(ctx context.Context) (map[string]map[string]nodeengine.Running, error) {
|
||||
stream, err := js.Stream(ctx, nodeengine.StreamRunning)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the stream %s cannot be read whole: %w", nodeengine.StreamRunning, err)
|
||||
}
|
||||
cons, err := stream.OrderedConsumer(ctx, jetstream.OrderedConsumerConfig{DeliverPolicy: jetstream.DeliverLastPerSubjectPolicy})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the ordered consumer for the whole read of %s cannot be made: %w", nodeengine.StreamRunning, err)
|
||||
}
|
||||
out := map[string]map[string]nodeengine.Running{}
|
||||
for {
|
||||
batch, err := cons.FetchNoWait(256)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the whole read of %s failed: %w", nodeengine.StreamRunning, err)
|
||||
}
|
||||
n := 0
|
||||
for m := range batch.Messages() {
|
||||
n++
|
||||
var r nodeengine.Running
|
||||
if json.Unmarshal(m.Data(), &r) != nil || r.Node == "" || r.Module == "" {
|
||||
continue
|
||||
}
|
||||
if out[r.Node] == nil {
|
||||
out[r.Node] = map[string]nodeengine.Running{}
|
||||
}
|
||||
out[r.Node][r.Module] = r
|
||||
}
|
||||
if err := batch.Error(); err != nil {
|
||||
return nil, fmt.Errorf("the whole read of %s failed: %w", nodeengine.StreamRunning, err)
|
||||
}
|
||||
if n == 0 {
|
||||
return out, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// tagsVia reads every tag of a repository that starts with a prefix from the seat git's verb tags as a kept call, page by
|
||||
// page until none is more (design 47 G §412, A-490c: protected or not).
|
||||
func tagsVia(call toolCaller) index.TagsReader {
|
||||
|
||||
@@ -1,190 +0,0 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
|
||||
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk"
|
||||
)
|
||||
|
||||
// A merge the indexer never took in (design 47 G A-463, amended by A-464): at its start and on every source revision the
|
||||
// gate's source names, the indexer compares each repository's `taken` merge with the head that source names. A repository
|
||||
// whose head is newer than its taken merge (the taken merge is that head's ancestor, or no merge is taken) is taken in as
|
||||
// its merged message would have been (the catch-up of indexer_take_in, for that repository), said in the log, with the
|
||||
// finding merge-untaken raised while it stays behind and cleared once it is taken in. A head the taken merge contains (the
|
||||
// source not yet moved to a merge just taken in) is left, and a head the taken merge neither contains nor is in is said.
|
||||
|
||||
// UntakenMergeKind is the finding of a repository whose head the gate's source names but whose merge the indexer never took in.
|
||||
const UntakenMergeKind = "merge-untaken"
|
||||
|
||||
func untakenFindingID(own, key string) string { return own + ".untaken." + key }
|
||||
|
||||
// kickBehind runs the comparison in the background, one at a time: a source revision that arrives while one runs makes it
|
||||
// run once more, on the latest heads. It runs after the first start's catch-up, which takes every head in itself.
|
||||
func (ix *Indexer) kickBehind(ctx context.Context) {
|
||||
if ix.Taken == nil || ix.Message == nil || ix.Modules == nil {
|
||||
return
|
||||
}
|
||||
if ix.catchingUp.Load() {
|
||||
ix.behindWaiting.Store(true) // compared again when that catch-up ends (A-463a)
|
||||
return
|
||||
}
|
||||
if ix.behindRunning.Swap(true) {
|
||||
ix.behindAgain.Store(true)
|
||||
return
|
||||
}
|
||||
ix.catchUpWG.Add(1)
|
||||
go func() {
|
||||
defer ix.catchUpWG.Done()
|
||||
for {
|
||||
ix.behindAgain.Store(false)
|
||||
ix.TakeInBehind(ctx)
|
||||
if !ix.behindAgain.Load() {
|
||||
ix.behindRunning.Store(false)
|
||||
if !ix.behindAgain.Load() || ix.behindRunning.Swap(true) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// TakeInBehind compares each repository's taken merge with the head the latest source names, takes in each repository
|
||||
// that is behind and answers the repositories it took in.
|
||||
func (ix *Indexer) TakeInBehind(ctx context.Context) []string {
|
||||
heads := ix.sourceHeads()
|
||||
if ix.Taken == nil || ix.Message == nil || ix.Modules == nil || len(heads) == 0 {
|
||||
return nil
|
||||
}
|
||||
behind := map[string]string{}
|
||||
taken := map[string]string{}
|
||||
held := ix.pendingHeads(ctx)
|
||||
for _, repository := range slices.Sorted(maps.Keys(heads)) {
|
||||
head := heads[repository]
|
||||
key, err := trunk.RepositoryKey(repository)
|
||||
if head == "" || err != nil {
|
||||
continue
|
||||
}
|
||||
value, _, found, err := ix.Taken.Get(ctx, key)
|
||||
if errors.Is(err, process.ErrNotYet) {
|
||||
return nil // the state is not on the bus yet: the next source revision compares again
|
||||
}
|
||||
if err != nil {
|
||||
ix.logf("merges: taken.%s cannot be read, so %s is not compared with its head: %v", key, repository, err)
|
||||
continue
|
||||
}
|
||||
merge := ""
|
||||
if found {
|
||||
t, err := takein.DecodeTaken(value)
|
||||
if err != nil {
|
||||
ix.logf("merges: taken.%s holds what is not a taken merge, left: %v", key, err)
|
||||
continue
|
||||
}
|
||||
merge = t.Merge
|
||||
}
|
||||
switch {
|
||||
case held[head]:
|
||||
// A head the pending holds (a member waiting for its group's end) is taken in by that path, whole with
|
||||
// its group: it is not behind (A-463a).
|
||||
ix.clearUntaken(ctx, key, "the head of "+repository+" waits in the indexer's pending")
|
||||
case merge == head:
|
||||
ix.clearUntaken(ctx, key, "the indexer took in the head of "+repository)
|
||||
case merge == "":
|
||||
behind[repository], taken[repository] = head, ""
|
||||
case ix.Contains == nil:
|
||||
ix.logf("merges: %s is taken in at %.12s and its head is %.12s, but the indexer reads no ancestry of the forge, so which is newer is not said", repository, merge, head)
|
||||
default:
|
||||
inHead, err := ix.Contains(ctx, repository, merge, head)
|
||||
if err != nil {
|
||||
ix.logf("merges: whether %.12s is in %.12s of %s cannot be read: %v", merge, head, repository, err)
|
||||
continue
|
||||
}
|
||||
if inHead {
|
||||
behind[repository], taken[repository] = head, merge
|
||||
continue
|
||||
}
|
||||
ix.clearUntaken(ctx, key, "the head of "+repository+" is in the merge taken in")
|
||||
if headIn, err := ix.Contains(ctx, repository, head, merge); err == nil && !headIn {
|
||||
ix.logf("merges: %s is taken in at %.12s and its head is %.12s, and neither holds the other: left", repository, merge, head)
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, repository := range slices.Sorted(maps.Keys(behind)) {
|
||||
ix.sayUntaken(ctx, repository, taken[repository], behind[repository])
|
||||
ix.logf("merges: %s is taken in at %.12s but the gate's source names %.12s: taken in now", repository, taken[repository], behind[repository])
|
||||
}
|
||||
if len(behind) == 0 {
|
||||
return nil
|
||||
}
|
||||
unread := ix.takeInHeads(ctx, behind)
|
||||
var done []string
|
||||
for _, repository := range slices.Sorted(maps.Keys(behind)) {
|
||||
if slices.Contains(unread, repository) {
|
||||
continue
|
||||
}
|
||||
done = append(done, repository)
|
||||
key, _ := trunk.RepositoryKey(repository)
|
||||
ix.clearUntaken(ctx, key, fmt.Sprintf("%s taken in at %.12s", repository, behind[repository]))
|
||||
ix.logf("merges: %s taken in at %.12s", repository, behind[repository])
|
||||
}
|
||||
if len(done) > 0 {
|
||||
if err := ix.KeepView(ctx); err != nil {
|
||||
ix.logf("merges: the view could not be kept after %v were taken in: %v", done, err)
|
||||
}
|
||||
}
|
||||
return done
|
||||
}
|
||||
|
||||
func (ix *Indexer) sayUntaken(ctx context.Context, repository, merge, head string) {
|
||||
if ix.Findings == nil {
|
||||
return
|
||||
}
|
||||
key, _ := trunk.RepositoryKey(repository)
|
||||
taken := "no merge"
|
||||
if merge != "" {
|
||||
taken = fmt.Sprintf("%.12s", merge)
|
||||
}
|
||||
f := ix.observed(mcc.ScopeModule, untakenFindingID(ix.Module, key), UntakenMergeKind,
|
||||
fmt.Sprintf("the indexer took in %s at %s, and the gate's source names the head %.12s", repository, taken, head),
|
||||
"A merge the indexer never took in",
|
||||
"A merge into a watched branch was never heard or applied, so no package or module of the repository follows its trunk. The indexer takes the head in itself and clears this; indexer_take_in with catch-up does it by hand.",
|
||||
UntakenMergeKind)
|
||||
if err := ix.Findings.Write(ctx, f); err != nil {
|
||||
ix.logf("merges: the finding %s for %s is not written: %v", UntakenMergeKind, repository, err)
|
||||
}
|
||||
}
|
||||
|
||||
func (ix *Indexer) clearUntaken(ctx context.Context, key, why string) {
|
||||
if ix.Findings == nil {
|
||||
return
|
||||
}
|
||||
if err := ix.clearFinding(ctx, mcc.ScopeModule, untakenFindingID(ix.Module, key), UntakenMergeKind, why); err != nil {
|
||||
ix.logf("merges: the finding %s for %s is not cleared: %v", UntakenMergeKind, key, err)
|
||||
}
|
||||
}
|
||||
|
||||
// pendingHeads are the merge commits the indexer's pending holds (A-463a).
|
||||
func (ix *Indexer) pendingHeads(ctx context.Context) map[string]bool {
|
||||
out := map[string]bool{}
|
||||
if ix.Pending == nil || !ix.pendingOnBus(ctx) {
|
||||
return out
|
||||
}
|
||||
entries, err := ReadPending(ctx, ix.Pending, ix.logf)
|
||||
if err != nil {
|
||||
ix.logf("merges: the pending state cannot be read, so every head is compared: %v", err)
|
||||
return out
|
||||
}
|
||||
for _, e := range entries {
|
||||
if e.Forge.Merge != "" {
|
||||
out[e.Forge.Merge] = true
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -1,63 +0,0 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"maps"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/graph"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
)
|
||||
|
||||
// ApplyBranchSource takes a watched branch other than the first in from the source revision the seat answered for it
|
||||
// (design 47 G §13, §14, §291, ADR 0666 §6, A-85, A-93): a pure computation from the merged messages of the heads the source
|
||||
// names, over a view in memory seeded from the branch's newest revision, written straight into the branch's next revision
|
||||
// (KeepBranchEntries): each module's fingerprint, inputs and graph node at the branch's heads, computed as the trunk's are
|
||||
// (applyMoving). Nothing of the trunk is written: its modules, manifests, taken state, graph and packages stay its own,
|
||||
// and the branch has no bucket of its own. A head whose merged message is not read keeps no revision, said; the next
|
||||
// source-moved of the branch takes it in again.
|
||||
func (ix *Indexer) ApplyBranchSource(ctx context.Context, branch string, a meshdelivery.SourceAnswer) error {
|
||||
if isTrunkBranch(branch) || ix.Revisions == nil || ix.Message == nil || len(a.Repositories) == 0 {
|
||||
return nil
|
||||
}
|
||||
heads := map[string]string{}
|
||||
for _, r := range a.Repositories {
|
||||
heads[r.Repository] = r.Head
|
||||
}
|
||||
view, nodes, _, err := BranchView(ctx, ix.Revisions, branch)
|
||||
if errors.Is(err, process.ErrNotYet) {
|
||||
ix.logf("catalog graph: the branch %s is not taken in yet: its objects %s are not on the bus", branch, CatalogRevisionsObjects)
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
d := ix.dry()
|
||||
// The branch's own view: its modules and graph from its revision, no taken merge, and the packages it publishes in memory
|
||||
// over the trunk's, never written to them.
|
||||
d.ix.Modules, d.ix.Graph, d.ix.Taken = view, nodes, newOverlay(nil)
|
||||
if ix.Packages != nil {
|
||||
d.ix.Packages = newOverlay(ix.Packages)
|
||||
}
|
||||
d.ix.source = sourceNow{rev: uint64(a.Revision), heads: maps.Clone(heads)}
|
||||
d.ix.Logf = func(format string, args ...any) {
|
||||
ix.logf("branch "+branch+", computed and not written: "+format, args...)
|
||||
}
|
||||
if unread := d.ix.takeInHeads(ctx, heads); len(unread) > 0 {
|
||||
ix.logf("catalog graph: the branch %s at source revision %d keeps no revision: %d repositories were not taken in (%v); taken in again at its next source-moved", branch, a.Revision, len(unread), unread)
|
||||
return nil
|
||||
}
|
||||
entries, err := d.ix.viewEntries(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// A cycle with a build edge no running version breaks is refused here as at the trunk's take-in (design 47 G §49, A-326).
|
||||
if cycle := graph.RefusedCycle(WithEdges(ix.withActs(ctx, entries))); cycle != nil {
|
||||
ix.logf("catalog graph: the branch %s at source revision %d keeps no revision: its catalog graph holds a cycle with a build edge, which no running version breaks (design 47 G §49): %s; taken in again at its next source-moved", branch, a.Revision, strings.Join(cycle, ", "))
|
||||
return nil
|
||||
}
|
||||
return ix.KeepBranchEntries(ctx, branch, a.Revision, entries)
|
||||
}
|
||||
@@ -1,228 +0,0 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
)
|
||||
|
||||
// Design 47 G §13, §14, §291, A-93: a watched branch other than the first is taken in as a pure computation from the merged
|
||||
// messages of its heads, straight into branch.<branch>.<n>; the trunk's modules, graph and revision stay the trunk's.
|
||||
func TestABranchOtherThanTheFirstIsTakenInIntoItsOwnRevisionAndTheTrunkIsLeftAlone(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
ix, mods, merged := newFormIndexerUnmerged(t, builderModules(t)...)
|
||||
ix.Revisions = newMemObjects(0)
|
||||
ix.Graph = newMem()
|
||||
ix.Message = func(_ context.Context, repository, head string, _ int) ([]byte, error) {
|
||||
if repository != catalogueRepo || head != merged.Merge {
|
||||
return nil, fmt.Errorf("no merge of %s at %s", repository, head)
|
||||
}
|
||||
return json.Marshal(merged)
|
||||
}
|
||||
var said []meshcatalog.CatalogRevisionEvent
|
||||
ix.SayCatalogRevision = func(_ context.Context, e meshcatalog.CatalogRevisionEvent) error {
|
||||
said = append(said, e)
|
||||
return nil
|
||||
}
|
||||
before := map[string]string{}
|
||||
keys, _ := mods.Keys(ctx)
|
||||
for _, k := range keys {
|
||||
v, _, _, _ := mods.Get(ctx, k)
|
||||
before[k] = string(v)
|
||||
}
|
||||
graphBefore, _ := ix.Graph.Keys(ctx)
|
||||
answer := meshdelivery.SourceAnswer{Branch: "beta", Revision: 9, Repositories: []meshdelivery.SourceHead{{Repository: catalogueRepo, Head: merged.Merge, Trunk: "beta"}}}
|
||||
if err := ix.ApplyBranchSource(ctx, "beta", answer); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(said) != 1 || said[0].Revision != 1 || said[0].Branch != "beta" || said[0].Source != 9 || said[0].Entries != int64(len(builderModules(t))) {
|
||||
t.Fatalf("the branch take-in said %+v, want revision 1 of beta at source 9 with every module", said)
|
||||
}
|
||||
rev, entries, err := ReadRevision(ctx, ix.Revisions, BranchObject("beta", 1))
|
||||
if err != nil || rev.Branch != "beta" || len(entries) != len(builderModules(t)) {
|
||||
t.Fatalf("branch.beta.1 = %+v with %d entries (%v)", rev, len(entries), err)
|
||||
}
|
||||
for _, e := range entries {
|
||||
if e.Commit != merged.Merge || e.Inputs == nil { // computed at the branch's head, with the inputs its fingerprint stands on
|
||||
t.Errorf("the entry of %s is at %q with inputs %v, want the branch's head %s with inputs", e.Module, e.Commit, e.Inputs, merged.Merge)
|
||||
}
|
||||
}
|
||||
if _, any, _ := newestTrunk(ctx, ix.Revisions); any {
|
||||
t.Error("a branch take-in kept a trunk revision")
|
||||
}
|
||||
after := map[string]string{}
|
||||
keys, _ = mods.Keys(ctx)
|
||||
for _, k := range keys {
|
||||
v, _, _, _ := mods.Get(ctx, k)
|
||||
after[k] = string(v)
|
||||
}
|
||||
if !maps.Equal(before, after) {
|
||||
t.Errorf("a branch take-in wrote the trunk's modules")
|
||||
}
|
||||
if graphAfter, _ := ix.Graph.Keys(ctx); !slices.Equal(graphBefore, graphAfter) {
|
||||
t.Errorf("a branch take-in wrote the trunk's graph: %v, was %v", graphAfter, graphBefore)
|
||||
}
|
||||
// Taken in again at the same heads, nothing moved: no second revision.
|
||||
if err := ix.ApplyBranchSource(ctx, "beta", answer); err != nil || len(said) != 1 {
|
||||
t.Errorf("the same heads said %d revisions (%v), want one", len(said), err)
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 G §49, A-326: a branch take-in refuses a cycle with a build edge as the trunk's does: the branch keeps no revision
|
||||
// at those heads and says why, naming every artifact on the cycle.
|
||||
func TestABranchTakeInKeepsNoRevisionWithARefusedCycle(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
mods := builderModules(t)
|
||||
for i, m := range mods {
|
||||
switch m.name {
|
||||
case "avahi": // avahi is built on mesh-tools (kind 3) ...
|
||||
mods[i].manifest = strings.Replace(catalogueManifest(t, "avahi"), `"build": {`, `"build": {
|
||||
"on": [{"arg": "RUNTIME_BASE", "module": "mesh-tools", "artifact": "tools"}],`, 1)
|
||||
case "mesh-tools": // ... and mesh-tools' artifact needs avahi's (kind 2): no build seat breaks the cycle.
|
||||
mods[i].manifest = strings.Replace(m.manifest, `"name": "tools",`, `"name": "tools", "needs": ["avahi:tools"],`, 1)
|
||||
}
|
||||
}
|
||||
ix, _, merged := newFormIndexerUnmerged(t, mods...)
|
||||
ix.Revisions = newMemObjects(0)
|
||||
ix.Graph = newMem()
|
||||
var logs strings.Builder
|
||||
ix.Logf = func(format string, args ...any) { fmt.Fprintf(&logs, format+"\n", args...) }
|
||||
ix.Message = func(context.Context, string, string, int) ([]byte, error) { return json.Marshal(merged) }
|
||||
var said []meshcatalog.CatalogRevisionEvent
|
||||
ix.SayCatalogRevision = func(_ context.Context, e meshcatalog.CatalogRevisionEvent) error {
|
||||
said = append(said, e)
|
||||
return nil
|
||||
}
|
||||
answer := meshdelivery.SourceAnswer{Branch: "beta", Revision: 9, Repositories: []meshdelivery.SourceHead{{Repository: catalogueRepo, Head: merged.Merge, Trunk: "beta"}}}
|
||||
if err := ix.ApplyBranchSource(ctx, "beta", answer); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, err := ReadRevision(ctx, ix.Revisions, BranchObject("beta", 1)); err == nil || len(said) != 0 {
|
||||
t.Errorf("the branch kept a revision with a refused cycle (said %+v)", said)
|
||||
}
|
||||
if want := "mesh-tools:tools, avahi:tools, mesh-tools:tools"; !strings.Contains(logs.String(), want) {
|
||||
t.Errorf("the journal %q does not say the cycle %s", logs.String(), want)
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 G A-401, A-93: a head the source names that has no merged message (a branch newly watched) is read whole at that
|
||||
// commit through the seat git and taken in as a merge of every file of its tree, so the branch keeps its first revision;
|
||||
// a head with a merged message is taken in as before and the seat git is not asked.
|
||||
func TestAHeadWithNoMergedMessageIsReadWholeAtItsCommit(t *testing.T) {
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
hasMessage bool
|
||||
readFails bool
|
||||
wantRevision bool
|
||||
wantRead bool
|
||||
}{
|
||||
{name: "no merged message: read whole", wantRevision: true, wantRead: true},
|
||||
{name: "a merged message: taken in as before", hasMessage: true, wantRevision: true},
|
||||
{name: "no merged message and the seat git cannot read it: no revision", readFails: true, wantRead: true},
|
||||
} {
|
||||
t.Run(row.name, func(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
ix, _, merged := newFormIndexerUnmerged(t, builderModules(t)...)
|
||||
ix.Revisions = newMemObjects(0)
|
||||
ix.Graph = newMem()
|
||||
var logs strings.Builder
|
||||
ix.Logf = func(format string, args ...any) { fmt.Fprintf(&logs, format+"\n", args...) }
|
||||
ix.Message = func(context.Context, string, string, int) ([]byte, error) {
|
||||
if !row.hasMessage {
|
||||
return nil, fmt.Errorf("the forge gives no merged pull request")
|
||||
}
|
||||
return json.Marshal(merged)
|
||||
}
|
||||
reads := 0
|
||||
ix.ReadFile = func(context.Context, string, string, string) ([]byte, error) { return nil, nil } // no predefined list
|
||||
ix.Read = func(_ context.Context, _, sha, dir string) (RepoRead, error) {
|
||||
reads++
|
||||
if row.readFails || sha != merged.Merge {
|
||||
return RepoRead{}, fmt.Errorf("the seat git does not answer")
|
||||
}
|
||||
r := RepoRead{Root: merged.Root, Trees: merged.Trees, Manifests: map[string]string{}, Locks: map[string]map[string]string{}, Packages: map[string][]string{}}
|
||||
if _, ok := merged.Trees[dir]; ok {
|
||||
r.Manifests[dir], r.Locks[dir], r.Packages[dir] = merged.Manifests[dir], merged.Locks[dir], merged.Packages[dir]
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
answer := meshdelivery.SourceAnswer{Branch: "beta", Revision: 9, Repositories: []meshdelivery.SourceHead{{Repository: catalogueRepo, Head: merged.Merge, Trunk: "beta"}}}
|
||||
if err := ix.ApplyBranchSource(ctx, "beta", answer); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if (reads > 0) != row.wantRead {
|
||||
t.Errorf("the seat git was asked %d times, want asked: %v", reads, row.wantRead)
|
||||
}
|
||||
rev, entries, err := ReadRevision(ctx, ix.Revisions, BranchObject("beta", 1))
|
||||
if (err == nil) != row.wantRevision {
|
||||
t.Fatalf("branch.beta.1 read: %v, want a revision: %v", err, row.wantRevision)
|
||||
}
|
||||
if row.wantRevision && (rev.Branch != "beta" || len(entries) != len(builderModules(t))) {
|
||||
t.Errorf("branch.beta.1 = %+v with %d entries, want every module", rev, len(entries))
|
||||
}
|
||||
if said := strings.Contains(logs.String(), "has no merged message"); said != row.wantRead {
|
||||
t.Errorf("the journal %q says the whole read: %v, want %v", logs.String(), said, row.wantRead)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 G A-401, A-402: the message a head read whole gives equals the forge bridge's: a templated module's needs
|
||||
// include its template's text (ADR 0632 §2), and the catalogue's predefined list is carried, or said absent.
|
||||
func TestAHeadReadWholeCarriesTheTemplateNeedsAndThePredefinedList(t *testing.T) {
|
||||
templated := strings.Replace(catalogueManifest(t, "avahi"), `"resources": [`, `"resources": [
|
||||
{"id": "conf", "type": "file", "path": "/etc/avahi-x.conf", "template": "files/x.conf"},`, 1)
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
template []byte // nil: the forge has no such file
|
||||
predefined []byte
|
||||
wantNeeds bool
|
||||
wantList string
|
||||
wantNotFound bool
|
||||
}{
|
||||
{name: "template and predefined list", template: []byte("host=${seat:node-service-manager:reach}"), predefined: []byte(`{"a":1}`), wantNeeds: true, wantList: `{"a":1}`},
|
||||
{name: "no template file and no list", wantNotFound: true},
|
||||
} {
|
||||
t.Run(row.name, func(t *testing.T) {
|
||||
ix, _, merged := newFormIndexerUnmerged(t, mod{name: "avahi", manifest: templated})
|
||||
ix.ReadFile = func(_ context.Context, repository, sha, p string) ([]byte, error) {
|
||||
if repository != catalogueRepo || sha != merged.Merge {
|
||||
return nil, fmt.Errorf("unexpected read of %s at %s", repository, sha)
|
||||
}
|
||||
switch p {
|
||||
case "modules/avahi/files/x.conf":
|
||||
return row.template, nil
|
||||
case "predefined.json":
|
||||
return row.predefined, nil
|
||||
}
|
||||
return nil, fmt.Errorf("unexpected file %s", p)
|
||||
}
|
||||
ix.Read = func(_ context.Context, _, _, dir string) (RepoRead, error) {
|
||||
return RepoRead{Root: merged.Root, Trees: merged.Trees, Manifests: map[string]string{dir: merged.Manifests[dir]}, Locks: map[string]map[string]string{dir: merged.Locks[dir]}}, nil
|
||||
}
|
||||
f, err := ix.wholeAt(context.Background(), catalogueRepo, merged.Merge)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
text := map[string]string{}
|
||||
if row.template != nil {
|
||||
text["files/x.conf"] = string(row.template)
|
||||
}
|
||||
want, _ := catalogue.ModuleNeeds([]byte(templated), text)
|
||||
if got := f.Needs["modules/avahi"]; !slices.Equal(got, want) || (row.wantNeeds && !slices.Contains(got, catalogue.ReadForm+catalogue.ReferenceFormSeatReach)) {
|
||||
t.Errorf("needs %v, want %v", got, want)
|
||||
}
|
||||
if string(f.Predefined) != row.wantList || f.PredefinedRead != row.wantNotFound {
|
||||
t.Errorf("predefined %q (read-absent %v), want %q (%v)", f.Predefined, f.PredefinedRead, row.wantList, row.wantNotFound)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,109 +0,0 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
)
|
||||
|
||||
func manifestPull(t *testing.T, number int, head, manifest string) forgemsg.Forge {
|
||||
f := forgemsg.Forge{V: keys.Version, Kind: forgemsg.ForgePull, Repository: catalogueRepo, Branch: "feat/x", Number: number, Head: head,
|
||||
Action: forgemsg.PullUpdated, Dirs: []string{"modules/avahi"}, DirsKnown: true}
|
||||
if manifest != "" {
|
||||
f.Manifests = map[string]string{"modules/avahi": manifest}
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
func candidateOf(t *testing.T, st *memStore) meshcatalog.CandidatesByGroupValue {
|
||||
t.Helper()
|
||||
key, err := CandidatesKey("feat/x")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
raw, ok := st.vals[key]
|
||||
if !ok {
|
||||
t.Fatalf("candidates holds %d keys, none %s", len(st.vals), key)
|
||||
}
|
||||
c, err := DecodeCandidate(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
func TestACandidateGivesEachTouchedModuleItsManifestAndTheReasonItHasNoPreview(t *testing.T) { // 1451/AC1
|
||||
avahi := catalogueManifest(t, "avahi")
|
||||
ix, _, _ := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z", aInputs))
|
||||
if err := ix.Apply(context.Background(), manifestPull(t, 301, h1, avahi)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c := candidateOf(t, ix.Store.(*memStore)) // DecodeCandidate has run Candidate.check
|
||||
want := meshcatalog.CandidatesByGroupValuePullsItemFpsValue{Manifest: meshcatalogclient.ManifestDigest([]byte(avahi)), None2: "no merge preview is computed"}
|
||||
if len(c.Pulls) != 1 || len(c.Pulls[0].Fps) != 1 || !reflect.DeepEqual(c.Pulls[0].Fps["avahi"], want) {
|
||||
t.Fatalf("pulls = %+v, want Fps[avahi] = %+v", c.Pulls, want)
|
||||
}
|
||||
if c.Fpset == "" || c.Fpset != meshcatalogclient.FingerprintSet(c.Pulls) {
|
||||
t.Errorf("fpset = %q, want %q", c.Fpset, meshcatalogclient.FingerprintSet(c.Pulls))
|
||||
}
|
||||
// The same pull request heard again without its directories (the head unchanged) keeps what it had.
|
||||
again := manifestPull(t, 301, h1, "")
|
||||
again.Dirs, again.DirsKnown = nil, false
|
||||
if err := ix.Apply(context.Background(), again); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if c2 := candidateOf(t, ix.Store.(*memStore)); !reflect.DeepEqual(c2.Pulls[0].Fps["avahi"], want) || c2.Fpset != c.Fpset {
|
||||
t.Errorf("after a message without directories: %+v fpset %q", c2.Pulls[0].Fps, c2.Fpset)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPullWithoutManifestsWritesTheCandidateAsToday(t *testing.T) { // 1451/AC2
|
||||
ix, _, _ := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z", aInputs))
|
||||
if err := ix.Apply(context.Background(), manifestPull(t, 301, h1, "")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c := candidateOf(t, ix.Store.(*memStore))
|
||||
if len(c.Pulls) != 1 || c.Pulls[0].Fps != nil || c.Fpset != "" || len(c.Pulls[0].Modules) != 1 || c.Pulls[0].Modules[0] != "avahi" {
|
||||
t.Errorf("candidate = %+v, want the module avahi and no Fps and no fpset", c)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTwoPullsGivingOneModuleDifferentManifestsAreRefusedNamingBoth(t *testing.T) { // 1451/AC3
|
||||
avahi := catalogueManifest(t, "avahi")
|
||||
// The second pull request carries avahi's manifest changed (a line added at its end) for the same directory.
|
||||
other := avahi + "\n"
|
||||
ix, _, _ := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z", aInputs))
|
||||
if err := ix.Apply(context.Background(), manifestPull(t, 301, h1, avahi)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := candidateOf(t, ix.Store.(*memStore))
|
||||
writes := ix.Store.(*memStore).writes
|
||||
err := ix.Apply(context.Background(), manifestPull(t, 302, h2, other))
|
||||
if !errors.Is(err, ErrUnreadable) {
|
||||
t.Fatalf("err = %v, want a refusal", err)
|
||||
}
|
||||
for _, w := range []string{catalogueRepo + "#301", catalogueRepo + "#302", `"avahi"`} {
|
||||
if !strings.Contains(err.Error(), w) {
|
||||
t.Errorf("the refusal %q does not name %s", err, w)
|
||||
}
|
||||
}
|
||||
// Candidate.check refuses the group too, which is what the refusal stands for.
|
||||
if _, encErr := EncodeCandidate(meshcatalog.CandidatesByGroupValue{V: keys.Version, Group: "feat.x", Heads: before.Heads, Pulls: []meshcatalog.CandidatesByGroupValuePullsItem{before.Pulls[0], {
|
||||
Repository: catalogueRepo, Number: 302, Head: h2, Modules: []string{"avahi"},
|
||||
Fps: map[string]meshcatalog.CandidatesByGroupValuePullsItemFpsValue{"avahi": {Manifest: meshcatalogclient.ManifestDigest([]byte(other)), None2: meshcatalogclient.NoneNoPreview}}}}}); encErr == nil {
|
||||
t.Error("the candidate check accepts two manifests for one module")
|
||||
}
|
||||
if ix.Store.(*memStore).writes != writes {
|
||||
t.Error("a partial write was made")
|
||||
}
|
||||
if after := candidateOf(t, ix.Store.(*memStore)); len(after.Pulls) != 1 || after.Fpset != before.Fpset {
|
||||
t.Errorf("the candidate changed: %+v", after)
|
||||
}
|
||||
}
|
||||
@@ -2,27 +2,18 @@ package index
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"cmp"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"git.novox.be/novox/mesh-sdk/go/stateobjects"
|
||||
"reflect"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdeliverychecker"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
)
|
||||
|
||||
// ErrConflict is what a Store says when a write found the key changed or created since it was read.
|
||||
@@ -152,27 +143,22 @@ type Indexer struct {
|
||||
// Contains says whether a commit is the second or an ancestor of it (the seat git's contains).
|
||||
Contains func(ctx context.Context, repository, commit, in string) (bool, error)
|
||||
Logf func(format string, args ...any)
|
||||
// ReadOpen asks the forge which pull requests into the trunk are open now (the seat git's open-pulls). A catalogue pull message
|
||||
// that finds the candidates of another pull request asks it and deletes those of a pull request no longer open, so a
|
||||
// closed message the indexer missed heals at the next one (issue 1126); nil asks none.
|
||||
ReadOpen func(ctx context.Context) ([]forgemsg.OpenPull, error)
|
||||
|
||||
mu sync.Mutex // guards source, toolchains and previews, which the watches and the forge messages all reach
|
||||
source sourceNow
|
||||
saidSource map[string]uint64 // the newest source revision a take-in.taken named, by watched branch (nextSource)
|
||||
previews map[string]meshdeliverychecker.PreviewsByRepositoryHeadTrunkValue // by previews key (previews.go)
|
||||
toolchains map[string]map[string]nodebuildgo.Toolchain // language, then node (toolchains.go)
|
||||
buildModules map[string]BuildModule // by language: the build modules whose toolchains are taken in (buildmodules.go)
|
||||
floors sync.Map // the modules whose floor FillFloor reads now (tag_floor.go), one fill each
|
||||
versionsFilling atomic.Bool // a FillVersions runs (versions_fill.go)
|
||||
stepsSince map[string]time.Time // by build seat: since when its steps is unanswered (seed_steps.go)
|
||||
tried map[string]bool // repository, commit and directory a registered module's manifest was tried at (registered.go)
|
||||
unread map[string]bool // registered modules whose last read the forge could not answer, read again by RetryUnread
|
||||
moving map[string]forgemsg.Forge // by modules key: a merge naming a module another repository holds, waiting for that trunk to drop it (moves.go)
|
||||
retrying atomic.Bool // a RetryUnread runs in the background (kickRetry)
|
||||
retryAgain atomic.Bool // an event asked for one more RetryUnread after the running one
|
||||
retryWG sync.WaitGroup // the background RetryUnread, for tests to wait on
|
||||
fresh map[string]bool // modules whose import form is computed again by hand (FingerprintsAgain), "*" for every one; under freshMu
|
||||
saidSource map[string]uint64 // the newest source revision a take-in.taken named, by watched branch (nextSource)
|
||||
toolchains map[string]map[string]nodebuildgo.Toolchain // language, then node (toolchains.go)
|
||||
buildModules map[string]BuildModule // by language: the build modules whose toolchains are taken in (buildmodules.go)
|
||||
floors sync.Map // the modules whose floor FillFloor reads now (tag_floor.go), one fill each
|
||||
versionsFilling atomic.Bool // a FillVersions runs (versions_fill.go)
|
||||
stepsSince map[string]time.Time // by build seat: since when its steps is unanswered (seed_steps.go)
|
||||
tried map[string]bool // repository, commit and directory a registered module's manifest was tried at (registered.go)
|
||||
unread map[string]bool // registered modules whose last read the forge could not answer, read again by RetryUnread
|
||||
moving map[string]whole // by modules key: a merge naming a module another repository holds, waiting for that trunk to drop it (moves.go)
|
||||
retrying atomic.Bool // a RetryUnread runs in the background (kickRetry)
|
||||
retryAgain atomic.Bool // an event asked for one more RetryUnread after the running one
|
||||
retryWG sync.WaitGroup // the background RetryUnread, for tests to wait on
|
||||
fresh map[string]bool // modules whose import form is computed again by hand (FingerprintsAgain), "*" for every one; under freshMu
|
||||
freshMu sync.Mutex
|
||||
recompute sync.Mutex // one Recompute at a time
|
||||
untaken map[string]string // repository to the latest merge applied while the taken state was missing (taken.go)
|
||||
@@ -205,28 +191,17 @@ type Indexer struct {
|
||||
// OwnBuildState keeps which build this process knows it runs, or since when and why not (FixOwnBuild); nil keeps none.
|
||||
OwnBuildState Store
|
||||
// Heard keeps what the indexer heard and compares later (A-80): the time a call is unanswered since (steps-since, call-since),
|
||||
// toolchain.<language>, delivered.<module>, pending.<module>.<version>, acted.<acting part>.<dependent>, each key part escaped as a name
|
||||
// toolchain.<language>, pending.<module>.<version>, acted.<acting part>.<dependent>, each key part escaped as a name
|
||||
// (§73), and the mark versions-filled; nil keeps none. It is the state heard, whose key forge keeps the forge messages handled.
|
||||
Heard Store
|
||||
// Placement asks the assigner's verb placement (A-62) and RunningWhole reads the node-engine's stream running whole (§556), both for the
|
||||
// edge entries of the first fill (§377); nil reads none, and the entries of kinds 2, 4 and 6 then read "not yet delivered".
|
||||
Placement PlacementAsker
|
||||
RunningWhole RunningReader
|
||||
Heard Store
|
||||
judgePending atomic.Bool // the start's judge pass failed: judged again at the next message (verdicts.go)
|
||||
// Message asks the forge bridge for the merged message of a merge, repository, merge commit and pull request number in (the number lets the bridge read a merge made on the forge, not through the gate), the message whole out (a
|
||||
// pending value keeps a reference, not a copy: issue 1585); an error is the bridge's words. Nil asks none.
|
||||
Message func(ctx context.Context, repository, merge string, number int) ([]byte, error)
|
||||
heldMu sync.Mutex
|
||||
held map[uint64]forgemsg.Forge // the merged messages Arrive was handed, by stream sequence, until their merge is taken in or dropped
|
||||
// Pending is the state of merges held for the indexer their revision names (pending.go, novox/hq ADR 0633 §1); nil
|
||||
// takes every merge in at once, as before the take-in step.
|
||||
Pending Store
|
||||
// SayTakeIn publishes a take-in event (take-in.pending, take-in.taken; ADR 0633 §5); nil says none.
|
||||
SayTakeIn func(ctx context.Context, t takein.TakeIn) error
|
||||
// Runs is the state runs: the runs on request the indexer holds a candidate for (runcandidates.go, A-174); nil keeps none.
|
||||
Runs Store
|
||||
runMu sync.Mutex // one run event at a time (runcandidates.go)
|
||||
pendingMu sync.Mutex // one take-in step at a time: a merge, a group's end, the start and a hand act (pending.go)
|
||||
runMu sync.Mutex // one run event at a time (runcandidates.go)
|
||||
heldFull atomic.Bool // a take-in is held because the catalog store is full: room is said when it has room again (RoomAgain)
|
||||
pendingMu sync.Mutex // one take-in step at a time: a merge, a group's end, the start and a hand act (pending.go)
|
||||
}
|
||||
|
||||
func (ix *Indexer) logf(format string, args ...any) {
|
||||
@@ -235,33 +210,19 @@ func (ix *Indexer) logf(format string, args ...any) {
|
||||
}
|
||||
}
|
||||
|
||||
// Apply takes one message of the bridge's stream FORGE into the group of its branch. A pull message adds or moves the
|
||||
// pull request, a closed one or a merged message takes it out; a group left with none is deleted. A change is written
|
||||
// by compare-and-set on the revision read, and read again on a conflict. A lone pull request is a group of one. A
|
||||
// merged message also writes the fingerprint of each module of its repository (modules.go).
|
||||
func (ix *Indexer) Apply(ctx context.Context, f forgemsg.Forge) error {
|
||||
_, err := ix.ApplyMessage(ctx, f)
|
||||
return err
|
||||
}
|
||||
|
||||
// ApplyMessage is Apply, and answers which modules a merged message wrote and kept (for the indexer's heard state). Each
|
||||
// applied merged message is said in one line: its repository, commit and pull request, and the modules written.
|
||||
func (ix *Indexer) ApplyMessage(ctx context.Context, f forgemsg.Forge) (Applied, error) {
|
||||
// ApplyMessage takes one repository's trees at a commit in (design 47 H §743): the fingerprint of each module of the repository
|
||||
// (modules.go), the taken key, the catalogue's predefined list, the packages' names and the seat definitions it carries. It
|
||||
// answers which modules it wrote and kept, and says them in one line.
|
||||
func (ix *Indexer) ApplyMessage(ctx context.Context, f whole) (Applied, error) {
|
||||
var applied Applied
|
||||
if f.Kind == forgemsg.ForgeOpen {
|
||||
return applied, ix.applyOpen(ctx, f)
|
||||
}
|
||||
if f.Kind != forgemsg.ForgePull && f.Kind != forgemsg.ForgeMerged {
|
||||
return applied, nil
|
||||
}
|
||||
lateMerge := false
|
||||
if f.Kind == forgemsg.ForgeMerged && ix.Modules != nil {
|
||||
if ix.Modules != nil {
|
||||
w, err := ix.applyMerged(ctx, f)
|
||||
applied = Applied{Written: w.written, Late: w.late}
|
||||
if err != nil {
|
||||
return applied, err
|
||||
}
|
||||
ix.logf("%s", w.said(f.Number))
|
||||
ix.logf("%s", w.said())
|
||||
if w.late == "" { // a late message leaves the taken key, which names a later merge
|
||||
if err := ix.writeTakenKnown(ctx, f, w.asked); err != nil {
|
||||
return applied, err
|
||||
@@ -270,13 +231,12 @@ func (ix *Indexer) ApplyMessage(ctx context.Context, f forgemsg.Forge) (Applied,
|
||||
if err := ix.writePredefined(ctx, f); err != nil {
|
||||
return applied, err
|
||||
}
|
||||
// Each package's name in its own ecosystem, which the builder numbers it under (design 32), kept before the
|
||||
// manifests are judged: a manifest of this merge naming a package this merge brings is judged knowing its name.
|
||||
// Each package's name in its own ecosystem, which the builder numbers it under (design 32).
|
||||
if err := ix.KeepPackageNames(ctx, f); err != nil {
|
||||
return applied, err
|
||||
}
|
||||
}
|
||||
ix.kickRetry(ctx) // a registered module the forge could not read is read again on a merge, in the background (issue 1945)
|
||||
ix.kickRetry(ctx) // a registered module the forge could not read is read again on a take-in, in the background (issue 1945)
|
||||
lateMerge = w.late != ""
|
||||
}
|
||||
// The seat definitions of the contracts modules it carries (DLV-I17, I18): a late merge moves no trunk's.
|
||||
@@ -285,114 +245,7 @@ func (ix *Indexer) ApplyMessage(ctx context.Context, f forgemsg.Forge) (Applied,
|
||||
return applied, err
|
||||
}
|
||||
}
|
||||
var digests map[string]string
|
||||
var held map[string]meshcatalog.ModulesByModuleValue
|
||||
if f.Kind == forgemsg.ForgePull {
|
||||
var err error
|
||||
if digests, err = ix.keepManifests(ctx, f); err != nil {
|
||||
return applied, err
|
||||
}
|
||||
if held, err = ix.heldModules(ctx); err != nil {
|
||||
return applied, err
|
||||
}
|
||||
}
|
||||
return applied, ix.writeCandidate(ctx, f, digests, held)
|
||||
}
|
||||
|
||||
// removeCandidate takes a merged pull request out of its group's candidate and writes nothing else of its message: for a
|
||||
// merge a newer merge of the same repository covers (it needs only the repository, number and branch).
|
||||
func (ix *Indexer) removeCandidate(ctx context.Context, f forgemsg.Forge) error {
|
||||
return ix.writeCandidate(ctx, f, nil, nil)
|
||||
}
|
||||
|
||||
// writeCandidate writes the group's candidate with the pull request f names (taken out for a merged or closed one).
|
||||
func (ix *Indexer) writeCandidate(ctx context.Context, f forgemsg.Forge, digests map[string]string, held map[string]meshcatalog.ModulesByModuleValue) error {
|
||||
if trunkLike[f.Branch] {
|
||||
ix.logf("%s#%d: its branch %q is no delivery group, so no candidate", f.Repository, f.Number, f.Branch)
|
||||
return nil
|
||||
}
|
||||
key, err := CandidatesKey(f.Branch)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: %s#%d: %v", ErrUnreadable, f.Repository, f.Number, err)
|
||||
}
|
||||
return write(ctx, ix.Store, key, func(value []byte, found bool) ([]byte, bool, error) {
|
||||
var cur *meshcatalog.CandidatesByGroupValue
|
||||
if found {
|
||||
c, err := DecodeCandidate(value)
|
||||
if err != nil {
|
||||
return nil, false, fmt.Errorf("candidates.%s holds what is not a candidate: %w", key, err)
|
||||
}
|
||||
cur = &c
|
||||
}
|
||||
next := ix.next(cur, f, digests, held)
|
||||
if next != nil {
|
||||
ix.withPreviews(ctx, next)
|
||||
if err := ix.refuseDisagreement(*next); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case next == nil && cur == nil, next != nil && cur != nil && SameCandidate(*cur, *next):
|
||||
return nil, false, nil
|
||||
case next == nil:
|
||||
return nil, true, nil
|
||||
}
|
||||
data, err := EncodeCandidate(*next)
|
||||
return data, false, err
|
||||
})
|
||||
}
|
||||
|
||||
// applyOpen takes an open message into the candidates (novox/hq ADR 0443 §5): every pull request it does not name is
|
||||
// taken out of its group, and a group left with none is deleted. What the message names is left as it is. A pull request
|
||||
// opened after the bridge read the forge is heard after this message and added again by its own.
|
||||
func (ix *Indexer) applyOpen(ctx context.Context, f forgemsg.Forge) error {
|
||||
named := make(map[forgemsg.OpenPull]bool, len(f.Open))
|
||||
for _, p := range f.Open {
|
||||
named[p] = true
|
||||
}
|
||||
keys, err := ix.Store.Keys(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("listing the candidates: %w", err)
|
||||
}
|
||||
for _, key := range keys {
|
||||
if meshcatalogclient.IsTryGroup(key) {
|
||||
continue // a run's candidate is no pull request's: its run owns it (runcandidates.go)
|
||||
}
|
||||
err := write(ctx, ix.Store, key, func(value []byte, found bool) ([]byte, bool, error) {
|
||||
if !found {
|
||||
return nil, false, nil
|
||||
}
|
||||
c, err := DecodeCandidate(value)
|
||||
if err != nil {
|
||||
return nil, false, fmt.Errorf("candidates.%s holds what is not a candidate: %w", key, err)
|
||||
}
|
||||
kept := slices.DeleteFunc(slices.Clone(c.Pulls), func(p meshcatalog.CandidatesByGroupValuePullsItem) bool {
|
||||
return !named[forgemsg.OpenPull{Repository: p.Repository, Number: int(p.Number)}]
|
||||
})
|
||||
switch {
|
||||
case len(kept) == len(c.Pulls):
|
||||
return nil, false, nil
|
||||
case len(kept) == 0:
|
||||
ix.logf("%s: no pull request of the group is open, so no candidate", key)
|
||||
return nil, true, nil
|
||||
}
|
||||
heads := make([]meshdeliveryclient.Head, len(kept))
|
||||
for i, p := range kept {
|
||||
heads[i] = meshdeliveryclient.Head{Repository: p.Repository, Number: int(p.Number), Head: p.Head}
|
||||
}
|
||||
hash, err := meshdeliveryclient.HeadsHash(heads)
|
||||
if err != nil {
|
||||
return nil, false, fmt.Errorf("%w: candidates.%s: %v", ErrUnreadable, key, err)
|
||||
}
|
||||
c.Pulls, c.Heads = kept, hash
|
||||
data, err := EncodeCandidate(c)
|
||||
return data, false, err
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
return applied, nil
|
||||
}
|
||||
|
||||
// write changes one key of a store by compare-and-set: compute is given the value read (found says there is one) and
|
||||
@@ -440,89 +293,3 @@ func write(ctx context.Context, st Store, key string, compute func(value []byte,
|
||||
func unchanged(next []byte, del bool, held []byte, found bool) bool {
|
||||
return found && !del && next != nil && bytes.Equal(next, held)
|
||||
}
|
||||
|
||||
// next is the group after the message; nil when the group is left with no pull request.
|
||||
//
|
||||
// A pull message that carries manifests (issue 1448) gives each module it touches, in the pull request's Fps, the digest of
|
||||
// its manifest and why it has no fingerprint of the new form (meshcatalogclient.NoneNoPreview), and the candidate its FPSet (issue 1451); a
|
||||
// pull request whose head is held again without its directories keeps the Fps it had. digests are the manifests' digests by
|
||||
// directory; held are the modules the modules bucket holds, which name the module of a changed directory (moduleOfDir).
|
||||
func (ix *Indexer) next(cur *meshcatalog.CandidatesByGroupValue, f forgemsg.Forge, digests map[string]string, held map[string]meshcatalog.ModulesByModuleValue) *meshcatalog.CandidatesByGroupValue {
|
||||
var pulls []meshcatalog.CandidatesByGroupValuePullsItem
|
||||
var base int64
|
||||
if cur != nil {
|
||||
pulls, base = slices.Clone(cur.Pulls), cur.Base
|
||||
}
|
||||
at := slices.IndexFunc(pulls, func(p meshcatalog.CandidatesByGroupValuePullsItem) bool {
|
||||
return p.Repository == f.Repository && p.Number == int64(f.Number)
|
||||
})
|
||||
if f.Kind == forgemsg.ForgeMerged || f.Action == forgemsg.PullClosed {
|
||||
if at < 0 {
|
||||
return cur
|
||||
}
|
||||
pulls = slices.Delete(pulls, at, at+1)
|
||||
} else {
|
||||
entry := meshcatalog.CandidatesByGroupValuePullsItem{Repository: f.Repository, Number: int64(f.Number), Head: f.Head, Modules: []string{}}
|
||||
switch {
|
||||
case f.DirsKnown:
|
||||
for _, dir := range f.Dirs {
|
||||
m := moduleOfDir(held, f, dir)
|
||||
if !slices.Contains(entry.Modules, m) {
|
||||
entry.Modules = append(entry.Modules, m)
|
||||
}
|
||||
if digest, ok := digests[dir]; ok {
|
||||
if entry.Fps == nil {
|
||||
entry.Fps = map[string]meshcatalog.CandidatesByGroupValuePullsItemFpsValue{}
|
||||
}
|
||||
entry.Fps[m] = meshcatalog.CandidatesByGroupValuePullsItemFpsValue{Manifest: digest, None2: meshcatalogclient.NoneNoPreview}
|
||||
}
|
||||
}
|
||||
case at >= 0 && pulls[at].Head == f.Head:
|
||||
// Its directories are not known now, for the head the group already holds: what it knew stands.
|
||||
entry.Modules, entry.Fps = pulls[at].Modules, pulls[at].Fps
|
||||
}
|
||||
if at >= 0 {
|
||||
pulls[at] = entry
|
||||
} else {
|
||||
pulls = append(pulls, entry)
|
||||
}
|
||||
}
|
||||
if len(pulls) == 0 {
|
||||
return nil
|
||||
}
|
||||
slices.SortFunc(pulls, func(a, b meshcatalog.CandidatesByGroupValuePullsItem) int {
|
||||
if c := strings.Compare(a.Repository, b.Repository); c != 0 {
|
||||
return c
|
||||
}
|
||||
return cmp.Compare(a.Number, b.Number)
|
||||
})
|
||||
heads := make([]meshdeliveryclient.Head, len(pulls))
|
||||
for i, p := range pulls {
|
||||
heads[i] = meshdeliveryclient.Head{Repository: p.Repository, Number: int(p.Number), Head: p.Head}
|
||||
}
|
||||
hash, _ := meshdeliveryclient.HeadsHash(heads)
|
||||
return &meshcatalog.CandidatesByGroupValue{V: keys.Version, Group: f.Branch, Base: base, Heads: hash, Pulls: pulls, Fpset: meshcatalogclient.FingerprintSet(pulls)}
|
||||
}
|
||||
|
||||
// refuseDisagreement refuses a candidate in which two pull requests give one module different manifests, as the
|
||||
// candidate's own check does (mesh-sdk Candidate.check), naming both pull requests so the log says which; nothing is
|
||||
// written, and the message is dropped as unreadable and said in the log (main.go handle): the next message of either pull
|
||||
// request is judged again.
|
||||
func (ix *Indexer) refuseDisagreement(c meshcatalog.CandidatesByGroupValue) error {
|
||||
first := map[string]meshcatalog.CandidatesByGroupValuePullsItem{}
|
||||
for _, p := range c.Pulls {
|
||||
for module, m := range p.Fps {
|
||||
before, ok := first[module]
|
||||
if !ok {
|
||||
first[module] = p
|
||||
continue
|
||||
}
|
||||
if !reflect.DeepEqual(before.Fps[module], m) {
|
||||
err := fmt.Errorf("%w: candidates.%s: %s#%d and %s#%d give the module %q different manifests (%s and %s), so the group is not written",
|
||||
ErrUnreadable, c.Group, before.Repository, before.Number, p.Repository, p.Number, module, before.Fps[module].Manifest, m.Manifest)
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -433,22 +433,19 @@ func (ix *Indexer) sayFill(ctx context.Context, event, about string, f meshcatal
|
||||
}
|
||||
}
|
||||
|
||||
// RoomAgain says room and evaluates pending again when a merge held with the reason objects full finds the store
|
||||
// below its filling mark (80 % of its cap): called after anything removes catalog graph revisions (no timer). It answers whether room was said.
|
||||
// RoomAgain says room when a take-in held with the reason objects full (heldFull) finds the store below its filling mark (80 %
|
||||
// of its cap): called after anything removes catalog graph revisions (no timer). It answers whether room was said; the held
|
||||
// take-in hears it (design 47 H §744).
|
||||
func (ix *Indexer) RoomAgain(ctx context.Context) (bool, error) {
|
||||
if ix.Revisions == nil || ix.Pending == nil {
|
||||
if ix.Revisions == nil || !ix.heldFull.Load() {
|
||||
return false, nil
|
||||
}
|
||||
entries, err := ReadPending(ctx, ix.Pending, ix.logf)
|
||||
if err != nil || !slices.ContainsFunc(entries, func(e PendingEntry) bool { return e.Value.Held == HeldObjectsFull }) {
|
||||
return false, err
|
||||
}
|
||||
used, capacity, err := ix.Revisions.Fill(ctx)
|
||||
if err != nil || (capacity > 0 && used*5 >= capacity*4) {
|
||||
return false, err // room is said below the filling mark, 80 %, so one refusal is not followed by another at once
|
||||
}
|
||||
ix.sayFill(ctx, meshcatalog.EventRoom, fmt.Sprintf("%d", used), meshcatalog.Fill{Bytes: used, Cap: capacity})
|
||||
return true, ix.PendingAgain(ctx) // room in catalog-revisions: pending evaluated again
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// TrunkView is the indexer's view of the trunk read from the newest trunk revision the bucket keeps (the lead's ruling
|
||||
|
||||
@@ -8,7 +8,6 @@ import (
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
)
|
||||
@@ -20,7 +19,7 @@ import (
|
||||
|
||||
// writePredefined keeps the predefined list a merged message carries, unless the key names a later merge: a merge older
|
||||
// than the one it names never replaces it (ADR 0348 §7). The same list at the same merge writes nothing (ADR 0289 §1).
|
||||
func (ix *Indexer) writePredefined(ctx context.Context, f forgemsg.Forge) error {
|
||||
func (ix *Indexer) writePredefined(ctx context.Context, f whole) error {
|
||||
if ix.Predefined == nil || len(f.Predefined) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -10,10 +10,8 @@ import (
|
||||
"time"
|
||||
|
||||
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
@@ -174,7 +172,7 @@ func TestTheControlRecords(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
merged := func(merge string) {
|
||||
t.Helper()
|
||||
if err := ix.ApplySeatDefinitions(ctx, forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: repo, Number: 1, Merge: merge,
|
||||
if err := ix.ApplySeatDefinitions(ctx, whole{Repository: repo, Merge: merge,
|
||||
Manifests: map[string]string{meshcatalogclient.RootPath: manifest, "modules/other": `{"module":"other","version":"1.0"}`}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -230,7 +228,7 @@ func TestTheControlRecords(t *testing.T) {
|
||||
otherManifest := strings.Replace(manifest, `"mesh-contracts"`, `"other-contracts"`, 1)
|
||||
mergedOther := func(merge string) {
|
||||
t.Helper()
|
||||
if err := ix.ApplySeatDefinitions(ctx, forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: other, Number: 2, Merge: merge,
|
||||
if err := ix.ApplySeatDefinitions(ctx, whole{Repository: other, Merge: merge,
|
||||
Manifests: map[string]string{meshcatalogclient.RootPath: otherManifest}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -251,90 +249,14 @@ func TestTheControlRecords(t *testing.T) {
|
||||
if g, ok := entry("git.trunk"); !ok || g.Repository != repo {
|
||||
t.Errorf("the second repository's merge without git removed the first's entry: %+v", g)
|
||||
}
|
||||
// A pull request's head: its own entry, the trunk's answered unless it is named.
|
||||
if err := ix.ApplySeatDefinitions(ctx, forgemsg.Forge{Kind: forgemsg.ForgePull, Action: forgemsg.PullUpdated, Repository: repo, Number: 9, Head: head,
|
||||
Dirs: []string{meshcatalogclient.RootPath}, Manifests: map[string]string{meshcatalogclient.RootPath: manifest}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pk := SeatDefinitionKey("git", SourcePull, repo, 9)
|
||||
src := func() memSource {
|
||||
out := memSource{}
|
||||
for k, v := range defs.vals {
|
||||
out[k] = &views.Entry{Value: v}
|
||||
}
|
||||
return out
|
||||
}
|
||||
if a, err := SeatDefinitionOf(src(), nil, "git", "", 0); err != nil || a.Answered != SourceTrunk || a.Entry.Commit != m3 {
|
||||
t.Errorf("the trunk's answer %+v, %v", a, err)
|
||||
}
|
||||
if a, err := SeatDefinitionOf(src(), nil, "git", repo, 9); err != nil || a.Answered != repo+"#9" || a.Entry.Commit != head || a.Trunk.Commit != m3 {
|
||||
t.Errorf("the pull request's answer %+v, %v", a, err)
|
||||
}
|
||||
// Its check's verdict names out-of-step files; a trunk build end names them too, a failed build clears nothing.
|
||||
if err := ix.ApplyVerdict(ctx, repo, 9, head, []string{"go/b.go", "go/a.go"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
end := func(outcome string, files ...string) []byte {
|
||||
raw, _ := json.Marshal(map[string]any{"v": 1, "event": builds.BuildEndedEvent, "module": "mesh-contracts", "ref": "main",
|
||||
"commit": m3, "outcome": outcome, "out-of-step": files})
|
||||
return raw
|
||||
}
|
||||
ix.ApplyOutOfStepBuild(ctx, end("failed", "ts/x.ts"))
|
||||
ix.ApplyOutOfStepBuild(ctx, end("failed"))
|
||||
fsrc := memSource{}
|
||||
for k, v := range findings.vals {
|
||||
fsrc[k] = &views.Entry{Value: v}
|
||||
}
|
||||
rows, err := OutOfStepOf(testSelf, src(), fsrc)
|
||||
if err != nil || len(rows) != 2 || rows[0].Module != "mesh-contracts" || !strings.Contains(rows[0].Summary, "ts/x.ts") ||
|
||||
rows[1].Pull != repo+"#9" || !reflect.DeepEqual(rows[1].Files, []string{"go/a.go", "go/b.go"}) {
|
||||
t.Errorf("out of step %+v, %v; want the trunk's module and the pull request's sorted files", rows, err)
|
||||
}
|
||||
ix.ApplyOutOfStepBuild(ctx, end(builds.BuildDone))
|
||||
if len(findings.vals) != 0 {
|
||||
t.Errorf("findings %v stand after a clean build end", findings.vals)
|
||||
}
|
||||
// Ruling (lead, on A2i's finding 3): indexer_seat_definitions_again for a pull request reads the head's own manifest's
|
||||
// generated-from: a trunk that names none, a head that names it, is read as the contracts module the head is.
|
||||
ix.Modules, ix.Manifests = newMem(), newMem()
|
||||
plain := `{"module":"mesh-contracts","version":"1.0"}`
|
||||
kept, _ := meshcatalogclient.EncodeManifest(meshcatalog.ManifestsByDigestValue{Module: "mesh-contracts", Manifest: plain})
|
||||
mkey, _ := meshcatalogclient.ManifestsKey(meshcatalogclient.ManifestDigest([]byte(plain)))
|
||||
ix.Manifests.(*memStore).vals[mkey] = kept
|
||||
mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: "src2:" + strings.Repeat("a", 64), Manifest: meshcatalogclient.ManifestDigest([]byte(plain)), Repository: repo, Commit: m3, Path: meshcatalogclient.RootPath})
|
||||
ix.Modules.(*memStore).vals["mesh-contracts"] = mod
|
||||
files["h2"] = map[string]string{"module.json": manifest, "seats/git/seat.json": strings.Replace(git, "ADR 0111", "ADR 0400", 1)}
|
||||
if r := ix.SeatDefinitionsAgain(ctx, "mesh-contracts", 9, "h2"); !r.Stored {
|
||||
t.Errorf("again for the pull request's head = %+v, want it read with the head's generated-from", r)
|
||||
}
|
||||
if e, ok := entry(pk); !ok || e.Commit != "h2" {
|
||||
t.Errorf("the pull request's entry after again = %+v (%v)", e, ok)
|
||||
}
|
||||
if r := ix.SeatDefinitionsAgain(ctx, "mesh-contracts", 0, ""); r.Stored || !strings.Contains(r.Answer, "trunk's manifest") {
|
||||
t.Errorf("again for the trunk, whose manifest names no generated-from = %+v", r)
|
||||
}
|
||||
// The pull request closes: its entries and its out-of-step files go, said.
|
||||
if err := ix.ApplySeatDefinitions(ctx, forgemsg.Forge{Kind: forgemsg.ForgePull, Action: forgemsg.PullClosed, Repository: repo, Number: 9, Head: head}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, ok := entry(pk); ok {
|
||||
t.Error("a closed pull request's entry is kept")
|
||||
}
|
||||
if _, ok := defs.vals[OutOfStepKey(repo, 9)]; ok {
|
||||
t.Error("a closed pull request's out-of-step files are kept")
|
||||
}
|
||||
for _, c := range said {
|
||||
if c.Source != SourceTrunk {
|
||||
t.Errorf("said %+v: only a trunk's definition is said (ADR 0636 §4)", c)
|
||||
}
|
||||
}
|
||||
// Design 47 (the indexer reads every definition before it judges): at its start the indexer reads every contracts
|
||||
// module's trunk at its repository's head in the source revision, so a definition merged with no event heard is kept
|
||||
// before any manifest is judged; a repository the source names no head of is read at the module's own commit.
|
||||
kept, _ = meshcatalogclient.EncodeManifest(meshcatalog.ManifestsByDigestValue{Module: "mesh-contracts", Manifest: manifest})
|
||||
mkey, _ = meshcatalogclient.ManifestsKey(meshcatalogclient.ManifestDigest([]byte(manifest)))
|
||||
ix.Modules, ix.Manifests = newMem(), newMem()
|
||||
kept, _ := meshcatalogclient.EncodeManifest(meshcatalog.ManifestsByDigestValue{Module: "mesh-contracts", Manifest: manifest})
|
||||
mkey, _ := meshcatalogclient.ManifestsKey(meshcatalogclient.ManifestDigest([]byte(manifest)))
|
||||
ix.Manifests.(*memStore).vals[mkey] = kept
|
||||
mod, _ = meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: "src2:" + strings.Repeat("a", 64), Manifest: meshcatalogclient.ManifestDigest([]byte(manifest)), Repository: repo, Commit: m3, Path: meshcatalogclient.RootPath})
|
||||
mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: "src2:" + strings.Repeat("a", 64), Manifest: meshcatalogclient.ManifestDigest([]byte(manifest)), Repository: repo, Commit: m3, Path: meshcatalogclient.RootPath})
|
||||
ix.Modules.(*memStore).vals["mesh-contracts"] = mod
|
||||
files["h3"] = map[string]string{"seats/git/seat.json": strings.Replace(git, "ADR 0111", "ADR 0500", 1)}
|
||||
ix.source = sourceNow{rev: 7, heads: map[string]string{repo: "h3"}}
|
||||
@@ -381,7 +303,7 @@ func TestTheControlRecords(t *testing.T) {
|
||||
mergeAt := func(repository, merge string, manifests map[string]string) {
|
||||
t.Helper()
|
||||
ix.ReadFile = func(_ context.Context, _, sha, p string) ([]byte, error) { return []byte(files[sha][p]), nil }
|
||||
if err := ix.ApplySeatDefinitions(ctx, forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: repository, Number: 3, Merge: merge, Manifests: manifests}); err != nil {
|
||||
if err := ix.ApplySeatDefinitions(ctx, whole{Repository: repository, Merge: merge, Manifests: manifests}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
@@ -427,15 +349,12 @@ func TestTheControlRecords(t *testing.T) {
|
||||
t.Errorf("said %+v: a handover removes nothing", c)
|
||||
}
|
||||
}
|
||||
// A pull request of mesh-contracts defining a Provisioning seat fails its check as a definition that does not parse.
|
||||
files["h2"] = map[string]string{"seats/wants/seat.json": seat("wants", "Provisioning")}
|
||||
ix.ReadFile = func(_ context.Context, _, sha, p string) ([]byte, error) { return []byte(files[sha][p]), nil }
|
||||
if err := ix.ApplySeatDefinitions(ctx, forgemsg.Forge{Kind: forgemsg.ForgePull, Action: forgemsg.PullUpdated, Repository: repo, Number: 11, Head: "h2",
|
||||
Dirs: []string{meshcatalogclient.RootPath}, Manifests: map[string]string{meshcatalogclient.RootPath: manifest}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if e, _ := entry(SeatDefinitionKey("wants", SourcePull, repo, 11)); !strings.Contains(e.Unparsed, "mesh-provisioning-contracts") || e.Definition != nil {
|
||||
t.Errorf("a pull request's wants from mesh-contracts = %+v, want it refused naming the domain's module", e)
|
||||
src := func() memSource {
|
||||
out := memSource{}
|
||||
for k, v := range defs.vals {
|
||||
out[k] = &views.Entry{Value: v}
|
||||
}
|
||||
return out
|
||||
}
|
||||
// The verb seat-definitions: no seat lists every trunk seat with its digest (never a pull request's); a seat answers it whole.
|
||||
listed, err := SeatDefinitionsOf(src(), nil, "")
|
||||
@@ -463,7 +382,7 @@ func TestTheControlRecords(t *testing.T) {
|
||||
}}
|
||||
sdk := `{"module":"mesh-sdk","version":"1.0","build":{"artifacts":[{"name":"go","kind":"package","language":"go","paths":["go"]},` +
|
||||
`{"name":"ts","kind":"package","language":"typescript","paths":["src","package.json"],"series":"0.3"}]}}`
|
||||
if err := ix.KeepPackageNames(ctx, forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-sdk", Merge: "m1",
|
||||
if err := ix.KeepPackageNames(ctx, whole{Repository: "novox/mesh-sdk", Merge: "m1",
|
||||
Manifests: map[string]string{meshcatalogclient.RootPath: sdk}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"maps"
|
||||
"reflect"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
|
||||
)
|
||||
|
||||
type dryRun struct {
|
||||
ix *Indexer
|
||||
modules, graph *overlayStore
|
||||
candidates, taken, predefined *overlayStore
|
||||
oldSections *overlayStore
|
||||
}
|
||||
|
||||
// dry is an indexer over overlays of this one's states (overlay.go), for a run on request's candidate (runcandidates.go): what
|
||||
// it computes stays in memory and nothing reaches the bus but the manifests and the packages, which clause 1 writes all the
|
||||
// same. It says no event, kicks nothing in the background, and knows what this one knows of the source and the toolchains. Every field of Indexer is named here (TestDryNamesEveryField holds it so).
|
||||
func (ix *Indexer) dry() *dryRun {
|
||||
over := func(st Store) (Store, *overlayStore) {
|
||||
if st == nil {
|
||||
return nil, nil
|
||||
}
|
||||
o := newOverlay(st)
|
||||
return o, o
|
||||
}
|
||||
d := &dryRun{}
|
||||
x := &Indexer{Module: ix.Module, Manifests: ix.Manifests, Packages: ix.Packages, JudgeID: ix.JudgeID, Files: ix.Files, GoSum: ix.GoSum,
|
||||
GoMod: ix.GoMod, Now: ix.Now, Builds: ix.Builds, Named: ix.Named, Trunk: ix.Trunk, BranchHead: ix.BranchHead, Read: ix.Read, Contains: ix.Contains,
|
||||
OwnBuild: nil, SayTakeIn: nil,
|
||||
// The registry milestone's fields: what a take-in would write on the bus stays out of a dry run, so the states of the
|
||||
// requirements, findings, seat definitions, holds and their bounds are not overlaid, nothing is said, and no hold is
|
||||
// weighed (Holds and Counted nil hold nothing); ReadFile only reads.
|
||||
Requirements: nil, Findings: nil, SeatDefinitions: nil, ReadFile: ix.ReadFile, SaySeatDefinition: nil,
|
||||
OwnBuildState: nil, Heard: nil, underSeats: true,
|
||||
// A dry run keeps no catalog graph revision (catalog_revisions.go): its revision is the take-in's to keep.
|
||||
Revisions: nil, SayCatalogRevision: nil, SayFill: nil, EntryIndex: nil, NamedSets: nil, Versions: nil, SayVersionReleased: nil, SayVersionsFilled: nil, Inputs: nil, InputObjects: nil, Pins: nil, PinsPending: nil,
|
||||
// A dry run writes no registration and asks no one.
|
||||
Registry: nil, SeedModules: nil}
|
||||
x.Logf = func(format string, args ...any) { ix.logf("take-in, computed and not written: "+format, args...) }
|
||||
x.Store, d.candidates = over(ix.Store)
|
||||
x.Modules, d.modules = over(ix.Modules)
|
||||
x.Graph, d.graph = over(ix.Graph)
|
||||
x.Taken, d.taken = over(ix.Taken)
|
||||
x.Predefined, d.predefined = over(ix.Predefined)
|
||||
x.OldSections, d.oldSections = over(ix.OldSections)
|
||||
ix.mu.Lock()
|
||||
x.source = sourceNow{rev: ix.source.rev, heads: maps.Clone(ix.source.heads)}
|
||||
x.toolchains = map[string]map[string]nodebuildgo.Toolchain{}
|
||||
for l, by := range ix.toolchains {
|
||||
x.toolchains[l] = maps.Clone(by)
|
||||
}
|
||||
x.buildModules = maps.Clone(ix.buildModules) // the dry run computes with the build modules the running indexer takes
|
||||
x.moving = maps.Clone(ix.moving)
|
||||
x.untaken = maps.Clone(ix.untaken)
|
||||
ix.mu.Unlock()
|
||||
ix.ownMu.Lock()
|
||||
x.ownFP = ix.ownFP
|
||||
ix.ownMu.Unlock()
|
||||
d.ix = x
|
||||
return d
|
||||
}
|
||||
|
||||
// refuseDisagreement refuses a candidate in which two pull requests give one module different manifests, as the
|
||||
// candidate's own check does (mesh-sdk Candidate.check), naming both pull requests so the log says which; nothing is
|
||||
// written, and the message is dropped as unreadable and said in the log (main.go handle): the next message of either pull
|
||||
// request is judged again.
|
||||
func (ix *Indexer) refuseDisagreement(c meshcatalog.CandidatesByGroupValue) error {
|
||||
first := map[string]meshcatalog.CandidatesByGroupValuePullsItem{}
|
||||
for _, p := range c.Pulls {
|
||||
for module, m := range p.Fps {
|
||||
before, ok := first[module]
|
||||
if !ok {
|
||||
first[module] = p
|
||||
continue
|
||||
}
|
||||
if !reflect.DeepEqual(before.Fps[module], m) {
|
||||
err := fmt.Errorf("%w: candidates.%s: %s#%d and %s#%d give the module %q different manifests (%s and %s), so the group is not written",
|
||||
ErrUnreadable, c.Group, before.Repository, before.Number, p.Repository, p.Number, module, before.Fps[module].Manifest, m.Manifest)
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -58,8 +58,7 @@ func (ix *Indexer) FingerprintsAgain(ctx context.Context, module string) TakenIn
|
||||
ix.freshMu.Lock()
|
||||
ix.fresh = map[string]bool{key: true}
|
||||
ix.freshMu.Unlock()
|
||||
// The path of a merge's recompute: the modules, then the candidates their fingerprints reach (recomputeWith), then the
|
||||
// pending merges a changed fingerprint settles (PendingAgain, as after a toolchain report).
|
||||
// The path of a merge's recompute: the modules, then the candidates their fingerprints reach (recomputeWith).
|
||||
err = ix.recomputeWith(ctx, nil, nil, "")
|
||||
ix.freshMu.Lock()
|
||||
ix.fresh = nil
|
||||
@@ -68,9 +67,6 @@ func (ix *Indexer) FingerprintsAgain(ctx context.Context, module string) TakenIn
|
||||
out.Answer = "nothing was written: " + err.Error()
|
||||
return out
|
||||
}
|
||||
if err := ix.PendingAgain(ctx); err != nil {
|
||||
ix.logf("the pending merges are not evaluated again after the fingerprints were computed again: %v", err)
|
||||
}
|
||||
after, err := ix.fingerprintsHeld(ctx)
|
||||
if err != nil {
|
||||
out.Answer = "computed, but the modules state cannot be read back: " + err.Error()
|
||||
|
||||
@@ -123,3 +123,28 @@ func (m *memFindings) Held() []mcseat.Finding {
|
||||
// testSelf is the module the tests' indexer runs as, its credential's: its findings, its own entry and its graph key are named
|
||||
// from it, never from a constant (design 47 G A-296).
|
||||
const testSelf = "acme-indexer"
|
||||
|
||||
// Apply takes one repository's trees in, as a take-in does (ApplyMessage), for the tests.
|
||||
func (ix *Indexer) Apply(ctx context.Context, f whole) error {
|
||||
_, err := ix.ApplyMessage(ctx, f)
|
||||
return err
|
||||
}
|
||||
|
||||
const otherMerge = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
|
||||
|
||||
// srcT is a source revision's heads as the tests write them; applyS makes it the latest source revision the indexer took in.
|
||||
type srcT struct {
|
||||
Group string
|
||||
Heads map[string]string
|
||||
}
|
||||
|
||||
func sourceOf(repo, head string) srcT { return srcT{Group: "g", Heads: map[string]string{repo: head}} }
|
||||
|
||||
func applyS(ix *Indexer, _ context.Context, rev uint64, s srcT) error {
|
||||
ix.mu.Lock()
|
||||
defer ix.mu.Unlock()
|
||||
if rev >= ix.source.rev {
|
||||
ix.source = sourceNow{rev: rev, heads: maps.Clone(s.Heads)}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -10,7 +10,6 @@ import (
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdeliverychecker"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
)
|
||||
|
||||
@@ -363,11 +362,6 @@ func anUnreadPackagesStateWritesNoFingerprintAndFingerprintsAreComputedAgain(t *
|
||||
if now, _ := moduleOf(t, mods, "avahi"); now.Fp2 != good.Fp2 || ix.fresh != nil {
|
||||
t.Fatalf("avahi holds %q after it, want %q (fresh %v)", now.Fp2, good.Fp2, ix.fresh)
|
||||
}
|
||||
// It runs a merge's recompute, candidates included: a candidate store that cannot be listed fails it, said.
|
||||
ix.Store, ix.previews = &memStore{keysErr: errors.New("no bus")}, map[string]meshdeliverychecker.PreviewsByRepositoryHeadTrunkValue{"k": {}}
|
||||
if r := ix.FingerprintsAgain(ctx, "avahi"); r.Stored || !strings.Contains(r.Answer, "listing the candidates") {
|
||||
t.Fatalf("fingerprints again over candidates that cannot be listed: %+v", r)
|
||||
}
|
||||
if r := ix.FingerprintsAgain(ctx, "nothing"); r.Stored {
|
||||
t.Fatalf("fingerprints again of a module not held answered stored: %+v", r)
|
||||
}
|
||||
|
||||
@@ -5,11 +5,6 @@ import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -121,11 +116,6 @@ func buildRecord(t *testing.T, repo, module, path, at string) []byte {
|
||||
return logged(t, "build-"+module, module, repo, path, "", "", at)
|
||||
}
|
||||
|
||||
func pull(repo string, number int, branch, head string, dirs ...string) forgemsg.Forge {
|
||||
return forgemsg.Forge{V: keys.Version, Kind: forgemsg.ForgePull, Repository: repo, Branch: branch, Number: number, Head: head,
|
||||
Action: forgemsg.PullUpdated, Dirs: dirs, DirsKnown: dirs != nil}
|
||||
}
|
||||
|
||||
func newIndexer(t *testing.T) (*Indexer, *memStore, *strings.Builder) {
|
||||
t.Helper()
|
||||
st, logs := newMem(), &strings.Builder{}
|
||||
@@ -138,282 +128,3 @@ func newIndexer(t *testing.T) (*Indexer, *memStore, *strings.Builder) {
|
||||
_ = a
|
||||
}}, st, logs
|
||||
}
|
||||
|
||||
func read(t *testing.T, st *memStore, key string) (meshcatalog.CandidatesByGroupValue, bool) {
|
||||
t.Helper()
|
||||
v, ok := st.vals[key]
|
||||
if !ok {
|
||||
return meshcatalog.CandidatesByGroupValue{}, false
|
||||
}
|
||||
c, err := DecodeCandidate(v)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return c, true
|
||||
}
|
||||
|
||||
func TestTwoRepositoriesFormOneGroupByBranch(t *testing.T) { // 1238/AC2 (the grouping; the bus is the integration test)
|
||||
ix, st, _ := newIndexer(t)
|
||||
ctx := context.Background()
|
||||
if err := ix.Apply(ctx, pull("novox/mesh-delivery", 12, "feat/x", h1, "modules/a")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ix.Apply(ctx, pull("novox/mesh-catalog", 300, "feat/x", h2, "modules/c")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c, ok := read(t, st, "feat/x")
|
||||
if !ok {
|
||||
t.Fatal("no candidate under feat/x")
|
||||
}
|
||||
hash, _ := meshdeliveryclient.HeadsHash([]meshdeliveryclient.Head{{Repository: "novox/mesh-delivery", Number: 12, Head: h1}, {Repository: "novox/mesh-catalog", Number: 300, Head: h2}})
|
||||
if c.Group != "feat/x" || c.Base != 0 || c.Heads != hash || len(c.Pulls) != 2 {
|
||||
t.Fatalf("candidate = %+v", c)
|
||||
}
|
||||
got := map[string][]string{}
|
||||
for _, p := range c.Pulls {
|
||||
got[p.Repository] = p.Modules
|
||||
}
|
||||
if strings.Join(got["novox/mesh-delivery"], ",") != "a" || strings.Join(got["novox/mesh-catalog"], ",") != "c" {
|
||||
t.Errorf("modules = %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestModuleOfADirectory(t *testing.T) { // 1238/AC3, as R4e has it: the module of a directory is the indexer's own (I2, ADR 0501 clause 6)
|
||||
ix, st, _ := newIndexer(t)
|
||||
mods := newMem()
|
||||
ix.Modules = mods
|
||||
ctx := context.Background()
|
||||
// The trunk holds modules/odd as the module odd-name (its manifest names it so); modules/plain is held by none.
|
||||
merged := mergedMsg("novox/mesh-delivery", map[string]string{"modules/odd": "t"})
|
||||
merged.Manifests["modules/odd"] = `{"module":"odd-name","version":"1.0"}`
|
||||
if err := ix.Apply(ctx, merged); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ix.Apply(ctx, pull("novox/mesh-delivery", 12, "feat/x", h1, "modules/odd", "modules/plain", "modules/odd")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c, _ := read(t, st, "feat/x")
|
||||
if strings.Join(c.Pulls[0].Modules, ",") != "odd-name,plain" {
|
||||
t.Errorf("modules = %v, want the held module for modules/odd, the last element for modules/plain, no repeats", c.Pulls[0].Modules)
|
||||
}
|
||||
held := map[string]meshcatalog.ModulesByModuleValue{"x": {Repository: "novox/other", Path: "modules/a"}}
|
||||
if name, ok := moduleAt(held, "novox/mesh-delivery", "modules/a"); ok {
|
||||
t.Errorf("modules/a of novox/mesh-delivery is %q, held only of novox/other", name)
|
||||
}
|
||||
// A manifest the pull message carries names its module, whatever directory it is in.
|
||||
f := pull("novox/mesh-delivery", 13, "feat/y", h2, "modules/odd")
|
||||
f.Manifests = map[string]string{"modules/odd": `{"module":"renamed","version":"1.0"}`}
|
||||
if name := moduleOfDir(held, f, "modules/odd"); name != "renamed" {
|
||||
t.Errorf("modules/odd = %q, want the module its manifest names", name)
|
||||
}
|
||||
}
|
||||
func TestDirsNotKnownKeepTheModulesOfTheSameHead(t *testing.T) { // 1238/AC4
|
||||
ix, st, _ := newIndexer(t)
|
||||
ctx := context.Background()
|
||||
_ = ix.Apply(ctx, pull("novox/mesh-delivery", 12, "feat/x", h1, "modules/a"))
|
||||
unknown := pull("novox/mesh-delivery", 12, "feat/x", h1)
|
||||
if err := ix.Apply(ctx, unknown); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c, _ := read(t, st, "feat/x")
|
||||
if strings.Join(c.Pulls[0].Modules, ",") != "a" {
|
||||
t.Errorf("modules = %v, want a kept", c.Pulls[0].Modules)
|
||||
}
|
||||
// A new head with directories not known is listed with no modules.
|
||||
unknown.Head = h3
|
||||
if err := ix.Apply(ctx, unknown); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c, _ = read(t, st, "feat/x")
|
||||
if c.Pulls[0].Head != h3 || len(c.Pulls[0].Modules) != 0 {
|
||||
t.Errorf("pull = %+v, want the new head with no modules", c.Pulls[0])
|
||||
}
|
||||
// Known and empty is no modules, not unknown.
|
||||
known := pull("novox/mesh-delivery", 12, "feat/x", h3)
|
||||
known.Dirs, known.DirsKnown = nil, true
|
||||
if err := ix.Apply(ctx, known); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClosedAndMergedTakeAPullOutAndTheLastDeletesTheKey(t *testing.T) { // 1238/AC5
|
||||
for _, how := range []string{"closed", "merged"} {
|
||||
ix, st, _ := newIndexer(t)
|
||||
ctx := context.Background()
|
||||
_ = ix.Apply(ctx, pull("novox/mesh-delivery", 12, "feat/x", h1, "modules/a"))
|
||||
_ = ix.Apply(ctx, pull("novox/mesh-catalog", 300, "feat/x", h2, "modules/c"))
|
||||
gone := func(repo string, n int, head string) forgemsg.Forge {
|
||||
if how == "closed" {
|
||||
f := pull(repo, n, "feat/x", head)
|
||||
f.Action = forgemsg.PullClosed
|
||||
return f
|
||||
}
|
||||
return forgemsg.Forge{V: keys.Version, Kind: forgemsg.ForgeMerged, Repository: repo, Branch: "feat/x", Number: n, Head: head, Merge: h3, Root: "r", Trees: map[string]string{"a": "t"}}
|
||||
}
|
||||
if err := ix.Apply(ctx, gone("novox/mesh-delivery", 12, h1)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c, ok := read(t, st, "feat/x")
|
||||
if !ok || len(c.Pulls) != 1 || c.Pulls[0].Repository != "novox/mesh-catalog" {
|
||||
t.Fatalf("%s: candidate = %+v (kept %v)", how, c, ok)
|
||||
}
|
||||
hash, _ := meshdeliveryclient.HeadsHash([]meshdeliveryclient.Head{headOf(c.Pulls[0])})
|
||||
if c.Heads != hash {
|
||||
t.Errorf("%s: heads hash %q is not the remaining pull request's %q", how, c.Heads, hash)
|
||||
}
|
||||
if err := ix.Apply(ctx, gone("novox/mesh-catalog", 300, h2)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, ok := read(t, st, "feat/x"); ok {
|
||||
t.Errorf("%s: the last pull request left and the key is still there", how)
|
||||
}
|
||||
// A pull request the group does not hold changes nothing and writes nothing.
|
||||
writes := st.writes
|
||||
if err := ix.Apply(ctx, gone("novox/mesh-catalog", 300, h2)); err != nil || st.writes != writes {
|
||||
t.Errorf("%s: closing an unheld pull request wrote (%d writes, err %v)", how, st.writes-writes, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTrunkLikeBranchesMakeNoCandidate(t *testing.T) { // 1238/AC6
|
||||
for _, branch := range []string{"main", "master", "trunk", ""} {
|
||||
ix, st, logs := newIndexer(t)
|
||||
if err := ix.Apply(context.Background(), pull("novox/mesh-delivery", 12, branch, h1, "modules/a")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(st.vals) != 0 || st.writes != 0 {
|
||||
t.Errorf("branch %q: %d keys written", branch, len(st.vals))
|
||||
}
|
||||
if !strings.Contains(logs.String(), "no delivery group") {
|
||||
t.Errorf("branch %q: log %q does not say why", branch, logs.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSameMessageWritesNoNewRevision(t *testing.T) { // 1238/AC7
|
||||
ix, st, _ := newIndexer(t)
|
||||
ctx := context.Background()
|
||||
m := pull("novox/mesh-delivery", 12, "feat/x", h1, "modules/a")
|
||||
_ = ix.Apply(ctx, m)
|
||||
rev, writes := st.revs["feat/x"], st.writes
|
||||
for i := 0; i < 3; i++ {
|
||||
if err := ix.Apply(ctx, m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if st.revs["feat/x"] != rev || st.writes != writes {
|
||||
t.Errorf("the same message wrote again: revision %d to %d", rev, st.revs["feat/x"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestALonePullRequestIsAGroupOfOne(t *testing.T) { // 1238/AC8
|
||||
ix, st, _ := newIndexer(t)
|
||||
if err := ix.Apply(context.Background(), pull("novox/mesh-delivery", 12, "fix/lone", h1, "modules/a")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c, ok := read(t, st, "fix/lone")
|
||||
if !ok || c.Group != "fix/lone" || len(c.Pulls) != 1 {
|
||||
t.Errorf("candidate = %+v (%v)", c, ok)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAConflictReadsAgain(t *testing.T) { // 1238 (compare-and-set: Update with the revision read, read again on a conflict)
|
||||
ix, st, _ := newIndexer(t)
|
||||
ctx := context.Background()
|
||||
_ = ix.Apply(ctx, pull("novox/mesh-delivery", 12, "feat/x", h1, "modules/a"))
|
||||
// Between the indexer's read and its write another writer adds #300 to the group.
|
||||
st.beforeWrite = func() {
|
||||
other := meshcatalog.CandidatesByGroupValue{V: keys.Version, Group: "feat/x", Pulls: []meshcatalog.CandidatesByGroupValuePullsItem{
|
||||
{Repository: "novox/mesh-delivery", Number: 12, Head: h1, Modules: []string{"a"}},
|
||||
{Repository: "novox/mesh-catalog", Number: 300, Head: h2, Modules: []string{"c"}}}}
|
||||
other.Heads, _ = meshdeliveryclient.HeadsHash([]meshdeliveryclient.Head{headOf(other.Pulls[0]), headOf(other.Pulls[1])})
|
||||
data, _ := EncodeCandidate(other)
|
||||
st.put("feat/x", data)
|
||||
}
|
||||
if err := ix.Apply(ctx, pull("novox/mesh-delivery", 12, "feat/x", h3, "modules/a")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c, _ := read(t, st, "feat/x")
|
||||
if len(c.Pulls) != 2 {
|
||||
t.Fatalf("pulls = %+v, want #12 moved and #300 kept (the other writer's)", c.Pulls)
|
||||
}
|
||||
for _, p := range c.Pulls {
|
||||
if p.Number == 12 && p.Head != h3 {
|
||||
t.Errorf("#12 head = %s", p.Head)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The SDK escapes a branch name's odd characters into a key (mesh-sdk a6dd698), so a name such as "feat x?" keys and is
|
||||
// indexed; no branch name is unreadable any more.
|
||||
func TestAnOddBranchNameKeysEscaped(t *testing.T) {
|
||||
ix, st, _ := newIndexer(t)
|
||||
if err := ix.Apply(context.Background(), pull("novox/mesh-delivery", 12, "feat x?", h1, "modules/a")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key, err := CandidatesKey("feat x?")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, ok := st.vals[key]; !ok {
|
||||
t.Errorf("no candidate under %q", key)
|
||||
}
|
||||
}
|
||||
|
||||
func open(pulls ...forgemsg.OpenPull) forgemsg.Forge {
|
||||
return forgemsg.Forge{V: keys.Version, Kind: forgemsg.ForgeOpen, Open: pulls}
|
||||
}
|
||||
|
||||
// 1279/AC7 (ADR 0443 §5): on an open message, candidates lose every pull request it does not name, a group left empty is
|
||||
// deleted, and a pull request added by a later message stays.
|
||||
func TestAnOpenMessageDropsEveryCandidateItDoesNotName(t *testing.T) { // 1279/AC7
|
||||
ix, st, _ := newIndexer(t)
|
||||
ctx := context.Background()
|
||||
for _, f := range []forgemsg.Forge{
|
||||
pull("novox/mesh-delivery", 12, "feat/x", h1, "modules/a"), pull("novox/mesh-catalog", 300, "feat/x", h2, "modules/c"),
|
||||
pull("novox/mesh-delivery", 13, "feat/y", h2, "modules/a"),
|
||||
pull("novox/mesh-catalog", 280, "feat/old", h3, "modules/c"),
|
||||
} {
|
||||
if err := ix.Apply(ctx, f); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
before, _ := read(t, st, "feat/x")
|
||||
|
||||
// Open: mesh-delivery#12 only (and a pull request no group holds, which changes nothing).
|
||||
if err := ix.Apply(ctx, open(forgemsg.OpenPull{Repository: "novox/mesh-delivery", Number: 12}, forgemsg.OpenPull{Repository: "novox/mesh-catalog", Number: 999})); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c, ok := read(t, st, "feat/x")
|
||||
hash, _ := meshdeliveryclient.HeadsHash([]meshdeliveryclient.Head{{Repository: "novox/mesh-delivery", Number: 12, Head: h1}})
|
||||
if !ok || len(c.Pulls) != 1 || c.Pulls[0].Number != 12 || c.Pulls[0].Head != h1 || strings.Join(c.Pulls[0].Modules, ",") != "a" || c.Heads != hash || c.Heads == before.Heads {
|
||||
t.Fatalf("feat/x after the open message = %+v, want only mesh-delivery#12 with its modules and the heads hash of it alone", c)
|
||||
}
|
||||
for _, gone := range []string{"feat/y", "feat/old"} {
|
||||
if _, ok := st.vals[gone]; ok {
|
||||
t.Errorf("the group %s is left with no open pull request and was not deleted", gone)
|
||||
}
|
||||
}
|
||||
|
||||
// A pull request opened after the read is heard as a message after the open message and stays.
|
||||
if err := ix.Apply(ctx, pull("novox/mesh-catalog", 300, "feat/x", h2, "modules/c")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if c, _ := read(t, st, "feat/x"); len(c.Pulls) != 2 {
|
||||
t.Errorf("a pull request added after the open message is not kept: %+v", c)
|
||||
}
|
||||
// The same open message again changes nothing (no write), and an empty one drops every group.
|
||||
writes := st.writes
|
||||
if err := ix.Apply(ctx, open(forgemsg.OpenPull{Repository: "novox/mesh-delivery", Number: 12}, forgemsg.OpenPull{Repository: "novox/mesh-catalog", Number: 300})); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if st.writes != writes {
|
||||
t.Errorf("an open message that names every candidate wrote %d times", st.writes-writes)
|
||||
}
|
||||
if err := ix.Apply(ctx, open()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(st.vals) != 0 {
|
||||
t.Errorf("after an empty open message the candidates hold %v", st.vals)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,7 +7,6 @@ import (
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
)
|
||||
|
||||
@@ -51,7 +50,7 @@ func (ix *Indexer) keepManifest(ctx context.Context, module, raw string) (string
|
||||
|
||||
// keepManifests writes the manifest of each directory a message carries, and answers the digest of each by directory.
|
||||
// A manifest that cannot be kept is left out and said in the log, so the module reads as one with no manifest.
|
||||
func (ix *Indexer) keepManifests(ctx context.Context, f forgemsg.Forge) (map[string]string, error) {
|
||||
func (ix *Indexer) keepManifests(ctx context.Context, f whole) (map[string]string, error) {
|
||||
digests := make(map[string]string, len(f.Manifests))
|
||||
for dir, raw := range f.Manifests {
|
||||
module := manifestModule(raw) // the name the manifest gives itself: one digest always has one module
|
||||
@@ -60,7 +59,7 @@ func (ix *Indexer) keepManifests(ctx context.Context, f forgemsg.Forge) (map[str
|
||||
case err == nil:
|
||||
digests[dir] = digest
|
||||
case errors.Is(err, errCannotBeAValue):
|
||||
ix.logf("%s#%d: the manifest of %q cannot be kept: %v", f.Repository, f.Number, dir, err)
|
||||
ix.logf("%s@%s: the manifest of %q cannot be kept: %v", f.Repository, f.Merge, dir, err)
|
||||
default:
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -6,8 +6,6 @@ import (
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
)
|
||||
|
||||
// The manifests these tests carry are real files, copied unchanged into testdata: the module.json of modules of
|
||||
@@ -39,7 +37,7 @@ func manifestsIndexer(t *testing.T, recs ...[2]any) (*Indexer, *memStore, *memSt
|
||||
return ix, mods, manifests
|
||||
}
|
||||
|
||||
func withManifests(f forgemsg.Forge, manifests map[string]string, locks map[string]map[string]string) forgemsg.Forge {
|
||||
func withManifests(f whole, manifests map[string]string, locks map[string]map[string]string) whole {
|
||||
f.Manifests, f.Locks = manifests, locks
|
||||
return f
|
||||
}
|
||||
@@ -78,16 +76,11 @@ func sameManifestAppliedAgainIsWrittenOnce(t *testing.T) { // 1449/AC2
|
||||
if err := ix.Apply(context.Background(), f); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A second merge, and a pull message carrying it, the digest being the same.
|
||||
// A second merge, the digest being the same.
|
||||
f.Merge = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
|
||||
if err := ix.Apply(context.Background(), f); err != nil {
|
||||
t.Errorf("a second merge: %v", err)
|
||||
}
|
||||
pull := forgemsg.Forge{V: keys.Version, Kind: forgemsg.ForgePull, Repository: catalogueRepo, Branch: "feat/x", Number: 301, Head: h1,
|
||||
Action: forgemsg.PullUpdated, Dirs: []string{"modules/avahi"}, DirsKnown: true, Manifests: map[string]string{"modules/avahi": avahi}}
|
||||
if err := ix.Apply(context.Background(), pull); err != nil {
|
||||
t.Errorf("a pull message: %v", err)
|
||||
}
|
||||
if manifests.writes != 1 || len(manifests.vals) != 1 {
|
||||
t.Errorf("manifests written %d times into %d keys, want once into one", manifests.writes, len(manifests.vals))
|
||||
}
|
||||
|
||||
@@ -9,7 +9,6 @@ import (
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
@@ -35,20 +34,20 @@ import (
|
||||
//
|
||||
// A merge never moves a module backwards (order.go): a merged message older than the repository's taken merge writes and
|
||||
// deletes nothing, and is said; it is asked once, before anything is written.
|
||||
func (ix *Indexer) applyMerged(ctx context.Context, f forgemsg.Forge) (*mergeWrite, error) {
|
||||
func (ix *Indexer) applyMerged(ctx context.Context, f whole) (*mergeWrite, error) {
|
||||
return ix.applyManifests(ctx, f, true)
|
||||
}
|
||||
|
||||
// applyManifests is applyMerged; drop says whether the modules of the repository the message holds no manifest for are
|
||||
// deleted. A message that names only some of a repository's modules (a registered module read on its own, registered.go)
|
||||
// drops none.
|
||||
func (ix *Indexer) applyManifests(ctx context.Context, f forgemsg.Forge, drop bool) (*mergeWrite, error) {
|
||||
func (ix *Indexer) applyManifests(ctx context.Context, f whole, drop bool) (*mergeWrite, error) {
|
||||
return ix.applyMoving(ctx, f, drop, nil)
|
||||
}
|
||||
|
||||
// applyMoving is applyManifests; moves names the modules keys the message may take from the repository that holds them
|
||||
// (a move handed over, moves.go). Every other module held of another repository is left, said and kept waiting.
|
||||
func (ix *Indexer) applyMoving(ctx context.Context, f forgemsg.Forge, drop bool, moves map[string]bool) (*mergeWrite, error) {
|
||||
func (ix *Indexer) applyMoving(ctx context.Context, f whole, drop bool, moves map[string]bool) (*mergeWrite, error) {
|
||||
w := newMergeWrite(f)
|
||||
if drop {
|
||||
if err := ix.lateMerge(ctx, w); err != nil || w.late != "" {
|
||||
@@ -190,7 +189,7 @@ func withV(m meshcatalog.ModulesByModuleValue) meshcatalog.ModulesByModuleValue
|
||||
// namedObjects are, by module directory, the git objects of the paths each manifest of the merge names for every artifact
|
||||
// (catalogue Manifest.NamedPaths): those the message carries, else asked of the forge at the merge commit in one call
|
||||
// (Named). A forge that cannot say gives, for each directory asked, its words, so the module's reason names them.
|
||||
func (ix *Indexer) namedObjects(ctx context.Context, f forgemsg.Forge) (map[string]map[string]string, map[string]string) {
|
||||
func (ix *Indexer) namedObjects(ctx context.Context, f whole) (map[string]map[string]string, map[string]string) {
|
||||
objects := map[string]map[string]string{}
|
||||
need := map[string][]string{}
|
||||
for dir, raw := range f.Manifests {
|
||||
@@ -230,7 +229,7 @@ func (ix *Indexer) namedObjects(ctx context.Context, f forgemsg.Forge) (map[stri
|
||||
// held(key, module) says whether the trunk holds it. A module of another repository, or of none known, is never touched.
|
||||
func (ix *Indexer) dropRemoved(ctx context.Context, repository string, held map[string]meshcatalog.ModulesByModuleValue, trunkHolds func(key string, m meshcatalog.ModulesByModuleValue) bool) error {
|
||||
var handed []string
|
||||
elsewhere := map[string]forgemsg.Forge{}
|
||||
elsewhere := map[string]whole{}
|
||||
for _, key := range slices.Sorted(maps.Keys(held)) {
|
||||
cur := held[key]
|
||||
if cur.Repository == "" || !trunk.SameRepository(cur.Repository, repository) || trunkHolds(key, cur) {
|
||||
@@ -372,7 +371,7 @@ func unreadInputs(tree string, locks, paths map[string]string, packages []string
|
||||
// goSumPackages are the version line's packages of the merge's directory dir (ADR 0573 §2, ADR 0593 §2): the mesh Go
|
||||
// modules of other repositories its go.sum names, and those of its own repository its go.mod replaces by a directory
|
||||
// (siblingPackages), which no go.sum line names. A directory with no go.sum is no Go module and has none.
|
||||
func (ix *Indexer) goSumPackages(ctx context.Context, f forgemsg.Forge, dir string, cur meshcatalog.ModulesByModuleValue, has bool) []string {
|
||||
func (ix *Indexer) goSumPackages(ctx context.Context, f whole, dir string, cur meshcatalog.ModulesByModuleValue, has bool) []string {
|
||||
others := ix.otherPackages(ctx, f, dir, cur, has)
|
||||
if f.Locks[dir]["go.sum"] == "" && len(f.Packages[dir]) == 0 {
|
||||
return others
|
||||
@@ -386,7 +385,7 @@ func (ix *Indexer) goSumPackages(ctx context.Context, f forgemsg.Forge, dir stri
|
||||
|
||||
// siblingPackages are the mesh Go modules of the merge's own repository the go.mod of dir replaces by a directory, read
|
||||
// from the forge (GoMod); where it cannot be read, the held module's own-repository packages stand, said.
|
||||
func (ix *Indexer) siblingPackages(ctx context.Context, f forgemsg.Forge, dir string, cur meshcatalog.ModulesByModuleValue, has bool) []string {
|
||||
func (ix *Indexer) siblingPackages(ctx context.Context, f whole, dir string, cur meshcatalog.ModulesByModuleValue, has bool) []string {
|
||||
held := func() []string {
|
||||
if !has || cur.Inputs2 == nil {
|
||||
return nil
|
||||
@@ -413,7 +412,7 @@ func (ix *Indexer) siblingPackages(ctx context.Context, f forgemsg.Forge, dir st
|
||||
// otherPackages are the mesh Go modules of other repositories the go.sum of the merge's directory dir names: those the
|
||||
// message carries (an older bridge), else the held module's while its go.sum's hash is unchanged, else the go.sum read
|
||||
// from the forge (GoSum).
|
||||
func (ix *Indexer) otherPackages(ctx context.Context, f forgemsg.Forge, dir string, cur meshcatalog.ModulesByModuleValue, has bool) []string {
|
||||
func (ix *Indexer) otherPackages(ctx context.Context, f whole, dir string, cur meshcatalog.ModulesByModuleValue, has bool) []string {
|
||||
if carried := f.Packages[dir]; len(carried) > 0 {
|
||||
return fingerprint.OtherPackages(carried, f.Repository) // its own repository's come with its tree
|
||||
}
|
||||
|
||||
@@ -15,10 +15,8 @@ import (
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
)
|
||||
|
||||
@@ -87,12 +85,12 @@ func rec(t *testing.T, id, module, repo, path, ref, state, at, inputs string) [2
|
||||
// mergedMsg is a merged message of the repository with these module directories' trees, each with a manifest naming its
|
||||
// module, the directory's last element ("r" for the root), as the forge bridge sends it (one manifest per directory that
|
||||
// holds a module.json).
|
||||
func mergedMsg(repo string, trees map[string]string) forgemsg.Forge {
|
||||
func mergedMsg(repo string, trees map[string]string) whole {
|
||||
manifests := map[string]string{}
|
||||
for dir := range trees {
|
||||
manifests[dir] = `{"module":"` + nameOfDir(dir) + `","version":"1.0"}`
|
||||
}
|
||||
return forgemsg.Forge{V: keys.Version, Kind: forgemsg.ForgeMerged, Repository: repo, Branch: "feat/x", Number: 12, Head: h1, Merge: merge, Root: "r0", Trees: trees, Manifests: manifests}
|
||||
return whole{Repository: repo, Branch: "feat/x", Head: h1, Merge: merge, Root: "r0", Trees: trees, Manifests: manifests}
|
||||
}
|
||||
|
||||
func nameOfDir(dir string) string {
|
||||
@@ -138,7 +136,7 @@ func TestAnUnchangedModuleWritesNothing(t *testing.T) { // 1239 (unchanged value
|
||||
{name: "the same merge again", writes: 0, commit: map[string]string{"a": merge, "b": merge, "c": merge}, taken: merge,
|
||||
then: func(ix *Indexer) error { return ix.Apply(context.Background(), mergedMsg(repo, three)) }},
|
||||
{name: "a merge touching one of three modules", writes: 1, commit: map[string]string{"a": next, "b": merge, "c": merge}, taken: next,
|
||||
said: "novox/mesh-delivery at cccccccccccc (#12): merged message applied; modules written: a",
|
||||
said: "novox/mesh-delivery at cccccccccccc: taken in; modules written: a",
|
||||
then: func(ix *Indexer) error {
|
||||
m := mergedMsg(repo, map[string]string{"modules/a": "treeA2", "modules/b": "treeB", "modules/c": "treeC"})
|
||||
m.Merge = next
|
||||
@@ -157,7 +155,7 @@ func TestAnUnchangedModuleWritesNothing(t *testing.T) { // 1239 (unchanged value
|
||||
m.Merge = next
|
||||
return ix.Apply(context.Background(), m)
|
||||
}},
|
||||
{name: "a merge heard while the taken state was missing is filled when the state appears", writes: 2, commit: map[string]string{"a": next, "b": merge, "c": merge}, taken: next,
|
||||
{name: "a merge heard while the taken state was missing is filled when the state appears", writes: 1, commit: map[string]string{"a": next, "b": merge, "c": merge}, taken: next,
|
||||
then: func(ix *Indexer) error { // issue 1280: the key was missing, the merge was taken in without it
|
||||
on := ix.Taken.(*memStore)
|
||||
key, _ := trunk.RepositoryKey(repo)
|
||||
@@ -364,16 +362,6 @@ func TestAnUnchangedModuleWritesNothing(t *testing.T) { // 1239 (unchanged value
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergedMessageRemovesThePullFromItsGroupToo(t *testing.T) { // 1238/AC5 with 1239 (both effects of one message)
|
||||
ix, _ := modulesIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", aInputs))
|
||||
cands := ix.Store.(*memStore)
|
||||
_ = ix.Apply(context.Background(), pull("novox/mesh-delivery", 12, "feat/x", h1, "modules/a"))
|
||||
_ = ix.Apply(context.Background(), mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"}))
|
||||
if _, ok := cands.vals["feat/x"]; ok {
|
||||
t.Error("the merged pull request is still a candidate")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAMergeWritesTheModulesDirectoryAndKeepsItThroughSourceAndRecompute(t *testing.T) { // 1606/AC1, AC2
|
||||
ix, mods := modulesIndexer(t,
|
||||
rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", aInputs),
|
||||
@@ -393,8 +381,8 @@ func TestAMergeWritesTheModulesDirectoryAndKeepsItThroughSourceAndRecompute(t *t
|
||||
if err := ix.Recompute(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m, _ := moduleOf(t, mods, "a"); m.Path != "modules/a" || m.Src != 7 {
|
||||
t.Errorf("after the source and a recompute a = %+v, want path modules/a kept and Src 7", m)
|
||||
if m, _ := moduleOf(t, mods, "a"); m.Path != "modules/a" {
|
||||
t.Errorf("after the source and a recompute a = %+v, want path modules/a kept", m)
|
||||
}
|
||||
raw := string(mods.vals["a"])
|
||||
if !strings.Contains(raw, `"path":"modules/a"`) {
|
||||
@@ -475,7 +463,7 @@ func TestAModuleMovesBetweenRepositoriesOnlyWhenItsTrunkDropsIt(t *testing.T) {
|
||||
gone.Trees, gone.Manifests = nil, nil
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
merges []forgemsg.Forge
|
||||
merges []whole
|
||||
repository string // modules.a's repository at the end, "" for none
|
||||
deletedKeyA bool
|
||||
// Issue 1367: a fresh indexer, no wait in memory, the source naming heads; trunk is the module directories the forge
|
||||
@@ -488,20 +476,20 @@ func TestAModuleMovesBetweenRepositoriesOnlyWhenItsTrunkDropsIt(t *testing.T) {
|
||||
stored bool
|
||||
bGoneAtEnd bool
|
||||
}{
|
||||
{name: "named while its trunk holds it: left", merges: []forgemsg.Forge{held, named}, repository: from},
|
||||
{name: "named, then dropped by its trunk: handed over in place", merges: []forgemsg.Forge{held, named, dropped}, repository: to},
|
||||
{name: "dropped by its trunk, then named: deleted, then written new", merges: []forgemsg.Forge{held, dropped, named}, repository: to, deletedKeyA: true},
|
||||
{name: "dropped by its trunk with no merge naming it: deleted", merges: []forgemsg.Forge{held, dropped}, deletedKeyA: true},
|
||||
{name: "named while its trunk holds it: left", merges: []whole{held, named}, repository: from},
|
||||
{name: "named, then dropped by its trunk: handed over in place", merges: []whole{held, named, dropped}, repository: to},
|
||||
{name: "dropped by its trunk, then named: deleted, then written new", merges: []whole{held, dropped, named}, repository: to, deletedKeyA: true},
|
||||
{name: "dropped by its trunk with no merge naming it: deleted", merges: []whole{held, dropped}, deletedKeyA: true},
|
||||
{name: "no module.json left, another trunk holds it, nothing waits in memory: written from that trunk, never deleted",
|
||||
merges: []forgemsg.Forge{held, gone}, heads: true, trunk: []string{"modules/a", "modules/other"}, repository: to, bGoneAtEnd: true},
|
||||
merges: []whole{held, gone}, heads: true, trunk: []string{"modules/a", "modules/other"}, repository: to, bGoneAtEnd: true},
|
||||
{name: "no module.json left, no other trunk holds it: deleted",
|
||||
merges: []forgemsg.Forge{held, gone}, heads: true, deletedKeyA: true, bGoneAtEnd: true},
|
||||
merges: []whole{held, gone}, heads: true, deletedKeyA: true, bGoneAtEnd: true},
|
||||
{name: "taken in by repository, a head with no module.json: handed over from the other trunk",
|
||||
merges: []forgemsg.Forge{held}, heads: true, trunk: []string{"modules/a"}, takeIn: from, answer: "handed over", stored: true, repository: to, bGoneAtEnd: true},
|
||||
merges: []whole{held}, heads: true, trunk: []string{"modules/a"}, takeIn: from, answer: "handed over", stored: true, repository: to, bGoneAtEnd: true},
|
||||
{name: "taken in by repository, a head with modules: answered by name",
|
||||
merges: []forgemsg.Forge{held}, heads: true, trunk: []string{"modules/a"}, takeIn: to, answer: "by name", repository: from},
|
||||
merges: []whole{held}, heads: true, trunk: []string{"modules/a"}, takeIn: to, answer: "by name", repository: from},
|
||||
{name: "taken in by repository, no head named: answered so",
|
||||
merges: []forgemsg.Forge{held}, heads: true, takeIn: "novox/nobody", answer: "names no head", repository: from},
|
||||
merges: []whole{held}, heads: true, takeIn: "novox/nobody", answer: "names no head", repository: from},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
mods := &deletesSeen{memStore: newMem()}
|
||||
|
||||
@@ -7,18 +7,16 @@ import (
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
)
|
||||
|
||||
// heldByAnotherTrunk reads, for a module the trunk of repository no longer holds, whether the trunk of another repository
|
||||
// holds it at the head the latest source names: the module directories there are asked of the forge (Trunk) and the manifest of
|
||||
// each directory named like the module is read (Read). It answers the merge to hand the module over with, or none: a forge
|
||||
// that cannot say leaves the module to be dropped as it was, said (issue 1367).
|
||||
func (ix *Indexer) heldByAnotherTrunk(ctx context.Context, module, repository string) (forgemsg.Forge, bool) {
|
||||
func (ix *Indexer) heldByAnotherTrunk(ctx context.Context, module, repository string) (whole, bool) {
|
||||
if ix.Trunk == nil || ix.Read == nil {
|
||||
return forgemsg.Forge{}, false
|
||||
return whole{}, false
|
||||
}
|
||||
ix.mu.Lock()
|
||||
heads := maps.Clone(ix.source.heads)
|
||||
@@ -47,7 +45,7 @@ func (ix *Indexer) heldByAnotherTrunk(ctx context.Context, module, repository st
|
||||
if read.Trees[dir] == "" || manifestModule(raw) != module {
|
||||
continue
|
||||
}
|
||||
f := forgemsg.Forge{V: keys.Version, Kind: forgemsg.ForgeMerged, Repository: other, Merge: head, Root: read.Root,
|
||||
f := whole{Repository: other, Merge: head, Root: read.Root,
|
||||
Trees: map[string]string{dir: read.Trees[dir]}, Manifests: map[string]string{dir: raw}}
|
||||
if l := read.Locks[dir]; len(l) > 0 {
|
||||
f.Locks = map[string]map[string]string{dir: l}
|
||||
@@ -59,7 +57,7 @@ func (ix *Indexer) heldByAnotherTrunk(ctx context.Context, module, repository st
|
||||
return f, true
|
||||
}
|
||||
}
|
||||
return forgemsg.Forge{}, false
|
||||
return whole{}, false
|
||||
}
|
||||
|
||||
// nameOfDirectory is the last element of a module directory.
|
||||
@@ -78,8 +76,8 @@ func nameOfDirectory(dir string) string { return dir[strings.LastIndex(dir, "/")
|
||||
|
||||
// waitToMove keeps, by modules key, the part of a merged message naming one module directory the indexer holds of another
|
||||
// repository. A later merge of the same repository replaces it.
|
||||
func (ix *Indexer) waitToMove(key string, f forgemsg.Forge, dir string) {
|
||||
one := forgemsg.Forge{V: f.V, Kind: f.Kind, Repository: f.Repository, Branch: f.Branch, Number: f.Number, Merge: f.Merge, Root: f.Root,
|
||||
func (ix *Indexer) waitToMove(key string, f whole, dir string) {
|
||||
one := whole{Repository: f.Repository, Branch: f.Branch, Merge: f.Merge, Head: f.Head, Root: f.Root,
|
||||
Trees: map[string]string{dir: f.Trees[dir]}, Manifests: map[string]string{dir: f.Manifests[dir]}}
|
||||
if l := f.Locks[dir]; len(l) > 0 {
|
||||
one.Locks = map[string]map[string]string{dir: l}
|
||||
@@ -93,7 +91,7 @@ func (ix *Indexer) waitToMove(key string, f forgemsg.Forge, dir string) {
|
||||
ix.mu.Lock()
|
||||
defer ix.mu.Unlock()
|
||||
if ix.moving == nil {
|
||||
ix.moving = map[string]forgemsg.Forge{}
|
||||
ix.moving = map[string]whole{}
|
||||
}
|
||||
ix.moving[key] = one
|
||||
}
|
||||
@@ -119,7 +117,7 @@ func (ix *Indexer) handOver(ctx context.Context, key string) error {
|
||||
}
|
||||
|
||||
// handOverFrom writes the module at key from the merge f, which names it, in place.
|
||||
func (ix *Indexer) handOverFrom(ctx context.Context, key string, f forgemsg.Forge) error {
|
||||
func (ix *Indexer) handOverFrom(ctx context.Context, key string, f whole) error {
|
||||
if _, err := ix.applyMoving(ctx, f, false, map[string]bool{key: true}); err != nil {
|
||||
return fmt.Errorf("modules.%s, moving to %s: %w", key, f.Repository, err)
|
||||
}
|
||||
|
||||
@@ -8,7 +8,6 @@ import (
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdeliverychecker"
|
||||
|
||||
"mesh-delivery-indexer/internal/fingerprint"
|
||||
)
|
||||
@@ -72,28 +71,6 @@ func TestAModuleNamingNoPathsKeepsItsDirectorysTree(t *testing.T) { // I7, ADR 0
|
||||
}
|
||||
}
|
||||
|
||||
func TestACandidatesModuleTakesTheNamedPathsObjectsFromThePreview(t *testing.T) { // I8, ADR 0499 clause 2 (the merge preview)
|
||||
ix, _, _ := manifestsIndexer(t)
|
||||
raw := fixture(t, "sdk-9cb4d59-mesh-sdk")
|
||||
digest, err := ix.keepManifest(context.Background(), "mesh-sdk", raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
named, _ := fingerprint.NamedTree(digest, sdkObjects)
|
||||
with := meshdeliverychecker.PreviewsByRepositoryHeadTrunkValue{Modules: map[string]meshdeliverychecker.PreviewsByRepositoryHeadTrunkValueModulesValue{".": {Tree: named, Manifest: digest, Paths: sdkObjects}}}
|
||||
without := meshdeliverychecker.PreviewsByRepositoryHeadTrunkValue{Modules: map[string]meshdeliverychecker.PreviewsByRepositoryHeadTrunkValueModulesValue{".": {Tree: "roottree", Manifest: digest}}}
|
||||
if got := ix.candidateModule(context.Background(), "mesh-sdk", ".", without, nil, nil); !strings.Contains(got.None2, "named paths") {
|
||||
t.Errorf("without the objects: %+v, want none naming the named paths", got)
|
||||
}
|
||||
got := ix.candidateModule(context.Background(), "mesh-sdk", ".", with, nil, nil)
|
||||
if strings.Contains(got.None2, "named paths") || got.Manifest != digest {
|
||||
t.Errorf("with the objects: %+v, want them taken (none only for what else is missing)", got)
|
||||
}
|
||||
if got.Fp2 != "" && !reflect.DeepEqual(got.Paths, sdkObjects) {
|
||||
t.Errorf("with a fingerprint: %+v, want the named paths' objects carried for the builder's fetch", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheIndexerAsksTheForgeForTheNamedPathsObjectsOfAMerge(t *testing.T) { // I6 as the lead has it: the indexer is the one reader of paths
|
||||
ix, mods, _ := manifestsIndexer(t)
|
||||
type ask struct {
|
||||
|
||||
@@ -15,11 +15,8 @@ const (
|
||||
PackagesHistory = 5
|
||||
TakenState = "taken"
|
||||
PredefinedListKey = "list"
|
||||
ForgeBridgeModule = "mesh-delivery-forge-bridge"
|
||||
GateModule = "mesh-delivery-gate"
|
||||
CheckerModule = "mesh-delivery-checker"
|
||||
PreviewsBucket = CheckerModule + "_previews"
|
||||
PreviewsHistory = 5
|
||||
toolchainsState = "toolchains"
|
||||
)
|
||||
|
||||
|
||||
@@ -13,37 +13,6 @@ import (
|
||||
// what the ask served no longer stands, so the call stopped and is not said unanswered.
|
||||
var ErrAskClosed = errors.New("the ask is closed: what it served no longer stands")
|
||||
|
||||
// takingIn marks a context as the take-in of one commit's merged message, which is pending while that context lives.
|
||||
type takingIn struct{ repository, merge string }
|
||||
|
||||
type takingInKey struct{}
|
||||
|
||||
// forTakeIn is ctx marked as the take-in of the merged message of repository's commit merge.
|
||||
func forTakeIn(ctx context.Context, repository, merge string) context.Context {
|
||||
return context.WithValue(ctx, takingInKey{}, takingIn{repository, merge})
|
||||
}
|
||||
|
||||
// TakeInPending says the take-in of the commit merge of repository is still pending (A-190, for the forge bridge's
|
||||
// merged_message): its take-in is the one ctx is made for, or a pending entry names it.
|
||||
func (ix *Indexer) TakeInPending(ctx context.Context, repository, merge string) (bool, error) {
|
||||
if ctx.Err() != nil {
|
||||
return false, nil
|
||||
}
|
||||
if t, ok := ctx.Value(takingInKey{}).(takingIn); ok && t.repository == repository && t.merge == merge {
|
||||
return true, nil
|
||||
}
|
||||
if ix.Pending == nil {
|
||||
return false, nil
|
||||
}
|
||||
entries, err := ReadPending(ctx, ix.Pending, ix.logf)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return slices.ContainsFunc(entries, func(e PendingEntry) bool {
|
||||
return e.Forge.Merge == merge && e.Forge.Repository == repository
|
||||
}), nil
|
||||
}
|
||||
|
||||
// SeedEntryHeld says the newest trunk revision still holds the entry of module at the fingerprint fp, the candidate the
|
||||
// seed asks steps for (A-190).
|
||||
func (ix *Indexer) SeedEntryHeld(ctx context.Context, module, fp string) (bool, error) {
|
||||
|
||||
@@ -13,35 +13,9 @@ import (
|
||||
"git.novox.be/novox/mesh-sdk/go/seatdef"
|
||||
)
|
||||
|
||||
// A-190: a kept call is asked again only while what it serves still stands; each of the indexer's four has its predicate.
|
||||
// A-190: a kept call is asked again only while what it serves still stands; each of the indexer's three has its predicate.
|
||||
func TestKeptCallsOpenWhileTheirAskServesSomething(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
t.Run("the take-in of a commit is pending while an entry names it or the take-in is made", func(t *testing.T) {
|
||||
pending := newMem()
|
||||
val, _ := encodePending(Pending{Seq: 1, Ref: &PendingRef{Repository: "novox/r", Merge: "c1"}})
|
||||
if err := pending.Create(ctx, "m.1", val); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ix := &Indexer{Module: testSelf, Pending: pending}
|
||||
cancelled, cancel := context.WithCancel(ctx)
|
||||
cancel()
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
ctx context.Context
|
||||
repo, commit string
|
||||
want bool
|
||||
}{
|
||||
{"held", ctx, "novox/r", "c1", true},
|
||||
{"taken in or dropped", ctx, "novox/r", "c2", false},
|
||||
{"in hand", forTakeIn(ctx, "novox/r", "c2"), "novox/r", "c2", true},
|
||||
{"in hand for another commit", forTakeIn(ctx, "novox/r", "c3"), "novox/r", "c2", false},
|
||||
{"the take-in's context ended", cancelled, "novox/r", "c1", false},
|
||||
} {
|
||||
if got, err := ix.TakeInPending(c.ctx, c.repo, c.commit); err != nil || got != c.want {
|
||||
t.Errorf("%s: pending = %v, %v, want %v", c.name, got, err, c.want)
|
||||
}
|
||||
}
|
||||
})
|
||||
t.Run("the seed's candidate and the versions' module are in the newest trunk revision", func(t *testing.T) {
|
||||
ix := &Indexer{Module: testSelf, Revisions: newMemObjects(1 << 20)}
|
||||
keep := func(n int64, fp string) {
|
||||
|
||||
@@ -9,7 +9,6 @@ import (
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
)
|
||||
@@ -49,7 +48,7 @@ func (w *mergeWrite) taken(digest string) {
|
||||
w.fresh[digest] = true
|
||||
}
|
||||
|
||||
func newMergeWrite(f forgemsg.Forge) *mergeWrite {
|
||||
func newMergeWrite(f whole) *mergeWrite {
|
||||
return &mergeWrite{repository: f.Repository, merge: f.Merge}
|
||||
}
|
||||
|
||||
@@ -121,13 +120,13 @@ func (w *mergeWrite) wrote(key string) {
|
||||
}
|
||||
|
||||
// said is the line of an applied merged message: its repository, commit and pull request, and the modules written.
|
||||
func (w *mergeWrite) said(number int) string {
|
||||
func (w *mergeWrite) said() string {
|
||||
if w.late != "" {
|
||||
return fmt.Sprintf("%s at %.12s (#%d): merged message applied as late: older than the taken %.12s, nothing written", w.repository, w.merge, number, w.late)
|
||||
return fmt.Sprintf("%s at %.12s: taken in as late: older than the taken %.12s, nothing written", w.repository, w.merge, w.late)
|
||||
}
|
||||
written := "none"
|
||||
if len(w.written) > 0 {
|
||||
written = strings.Join(slices.Sorted(slices.Values(w.written)), ", ")
|
||||
}
|
||||
return fmt.Sprintf("%s at %.12s (#%d): merged message applied; modules written: %s", w.repository, w.merge, number, written)
|
||||
return fmt.Sprintf("%s at %.12s: taken in; modules written: %s", w.repository, w.merge, written)
|
||||
}
|
||||
|
||||
@@ -9,7 +9,6 @@ import (
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk"
|
||||
)
|
||||
|
||||
@@ -73,7 +72,7 @@ func moduleAt(held map[string]meshcatalog.ModulesByModuleValue, repository, dir
|
||||
// moduleOfDir is the module of a changed directory of a pull request: the module its manifest in the message names, else
|
||||
// the one the modules bucket holds at that directory of the repository, else the directory's last element (the
|
||||
// catalogue's rule for nearly every manifest).
|
||||
func moduleOfDir(held map[string]meshcatalog.ModulesByModuleValue, f forgemsg.Forge, dir string) string {
|
||||
func moduleOfDir(held map[string]meshcatalog.ModulesByModuleValue, f whole, dir string) string {
|
||||
if name := manifestModule(f.Manifests[dir]); name != "" {
|
||||
return name
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,109 +0,0 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
)
|
||||
|
||||
// PendingTools are the indexer's tools of what waits for a take-in (novox/hq ADR 0633 §5): indexer_pending reads the
|
||||
// pending state, and indexer_take_in_pending and indexer_drop_pending steer it by an ask in takeins, as indexer_take_in
|
||||
// does. pending is the pending state as the tools bundle reads it; ask writes an ask and waits for the indexer's answer.
|
||||
func PendingTools(pending, ownBuild Store, ask func(key string, value []byte) (TakeInAsk, error), now func() time.Time) []stdio.Tool {
|
||||
str := func(args map[string]any, k string) string { s, _ := args[k].(string); return s }
|
||||
hand := func(drop bool) func(map[string]any) (any, error) {
|
||||
return func(args map[string]any) (any, error) {
|
||||
key, value, err := NewPendingAsk(str(args, "key"), str(args, "why"), stdio.CallerOf(args), drop, args["abandon"] == true, now())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
a, err := ask(key, value)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return a.Result, nil
|
||||
}
|
||||
}
|
||||
merge := func(args map[string]any) (any, error) {
|
||||
seq, _ := args["sequence"].(float64)
|
||||
key, value, err := NewMergeAsk(str(args, "repository"), str(args, "merge"), uint64(seq), str(args, "why"), stdio.CallerOf(args), now())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
a, err := ask(key, value)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return a.Result, nil
|
||||
}
|
||||
return []stdio.Tool{
|
||||
{Name: PendingTool,
|
||||
Description: "What waits for a take-in (ADR 0633): merges the indexer holds in its pending state instead of taking them in. " +
|
||||
"A revision (what revision, its key its oldest merge's m.<sequence>) waits only while its message is unread or lost or the " +
|
||||
"objects are full. A group (what group, key g.<group>) is a delivery group whose members are gathered until the " +
|
||||
"forge bridge says its end; a merge in one of its members' repositories waits behind it (behind). Each names its " +
|
||||
"merges, since when it waits, the modules it holds back (holds), the merges whose message the forge bridge's stream no " +
|
||||
"longer holds (lost: they are not taken in; indexer_drop_pending drops them) and the last take-in event said of it. Empty waits: " +
|
||||
"every merge is taken in. The watchman raises take-in-waits past 60 minutes for a revision, 30 for a group; " +
|
||||
"indexer_take_in_pending and indexer_drop_pending steer a wait. own-build says which build the indexer knows it runs and own-build-since since when; it only names the writer, and no merge waits for it.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{}},
|
||||
Run: func(map[string]any) (any, error) {
|
||||
var unread []string
|
||||
entries, err := ReadPending(context.Background(), pending, func(format string, args ...any) {
|
||||
unread = append(unread, fmt.Sprintf(format, args...))
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
answer := takein.PendingAnswer{Waits: PendingWaits(entries), Unread: unread}
|
||||
// The build this indexer knows it runs, and since when.
|
||||
if ownBuild != nil {
|
||||
if v, _, found, err := ownBuild.Get(context.Background(), OwnBuildKey); err == nil && found {
|
||||
var o OwnBuild
|
||||
if json.Unmarshal(v, &o) == nil {
|
||||
answer.OwnBuildSince = o.Since
|
||||
if o.Known {
|
||||
answer.OwnBuild = o.FP
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return answer, nil
|
||||
}},
|
||||
{Name: TakeInPendingTool,
|
||||
Description: "Take in now a pending revision (its m.<sequence> key), for a merge whose message is unread or lost or that is held " +
|
||||
"for room, or a delivery group (its g.<group> key) with the members gathered so far, for when its end never comes (ADR 0633 §5). " +
|
||||
"A hand act: who and why are said in the indexer's journal and on the take-in.taken event, which clears take-in-waits. " +
|
||||
"indexer_pending names the keys.",
|
||||
Input: map[string]any{"type": "object", "required": []string{"key", "why"}, "properties": map[string]any{
|
||||
"key": map[string]any{"type": "string", "description": "the wait's key, m.<sequence> or g.<group>, as indexer_pending names it"},
|
||||
"why": map[string]any{"type": "string", "description": "why it is taken in by hand, in plain words"}}},
|
||||
Run: hand(false)},
|
||||
{Name: DropPendingTool,
|
||||
Description: "Drop one pending merge by hand, abandoned because a later merge replaced it (ADR 0633 §5): it is removed from " +
|
||||
"the pending state and nothing of it is taken in; what waited with it is worked out again. A merge no later pending merge " +
|
||||
"and no taken merge of its repository covers is refused, naming indexer_take_in_pending, which reads its message again; " +
|
||||
"abandon: true drops it all the same. A hand act: who and why are said in the journal and on the take-in.taken event (dropped).",
|
||||
Input: map[string]any{"type": "object", "required": []string{"key", "why"}, "properties": map[string]any{
|
||||
"key": map[string]any{"type": "string", "description": "the pending merge's key, m.<sequence>, as indexer_pending names its merges' wait"},
|
||||
"why": map[string]any{"type": "string", "description": "why it is dropped, in plain words"},
|
||||
"abandon": map[string]any{"type": "boolean", "description": "drop it though no later merge covers it; its changes are then never taken in"}}},
|
||||
Run: hand(true)},
|
||||
{Name: TakeInMergeTool,
|
||||
Description: "Take one merge in by hand (ADR 0633 §5, issue 1585): the indexer asks the forge bridge for the merge's merged message " +
|
||||
"(bridge_merged_message) and runs it through the take-in as a delivered merged message, so it is held for the " +
|
||||
"indexer its revision names or taken in. For a merge the take-in did not take in (indexer_heard names its sequence " +
|
||||
"and why). A hand act: who and why are said in the indexer's journal. The bridge's own words answer when it cannot " +
|
||||
"build the message (the gate holds no such merge, the forge does not answer).",
|
||||
Input: map[string]any{"type": "object", "required": []string{"repository", "merge", "sequence", "why"}, "properties": map[string]any{
|
||||
"repository": map[string]any{"type": "string", "description": "the repository, owner/name"},
|
||||
"merge": map[string]any{"type": "string", "description": "the merge commit, 40 hex digits"},
|
||||
"sequence": map[string]any{"type": "integer", "description": "the merge's sequence on the forge bridge's stream, as indexer_heard names it"},
|
||||
"why": map[string]any{"type": "string", "description": "why it is taken in by hand, in plain words"}}},
|
||||
Run: merge},
|
||||
}
|
||||
}
|
||||
@@ -1,376 +0,0 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"maps"
|
||||
"reflect"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdeliverychecker"
|
||||
sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/grouppackages"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk"
|
||||
|
||||
"mesh-delivery-indexer/internal/fingerprint"
|
||||
)
|
||||
|
||||
// The merge previews the checker writes (mesh-delivery-checker_previews; novox/hq ADR 0484 clause 3, issue 1445): a pull
|
||||
// request's head merged onto its trunk's head, with the tree, the manifest digest and the lock hashes of each module
|
||||
// directory. From one the indexer gives the candidate's touched module its new-form fingerprint, by the rules a module of
|
||||
// the trunk is given its own (Recompute), or says why it has none.
|
||||
|
||||
// The reasons a candidate's touched module has no fingerprint of the new form while a preview exists or is missing.
|
||||
const (
|
||||
// NonePreviewStale is for a module whose preview was made onto a trunk head that is not the trunk's head now.
|
||||
NonePreviewStale = "the merge preview is out of date: the trunk moved since it was made"
|
||||
// NonePreviewConflict opens the reason of a module whose pull request's head does not merge into the trunk; the files
|
||||
// in conflict follow.
|
||||
NonePreviewConflict = "does not merge: the head conflicts with the trunk in "
|
||||
// NonePreviewNoModule is for a module whose directory the preview names no manifest for: mesh-sdk's words, which the
|
||||
// assigner reads as the change removing the module (issue 1297).
|
||||
NonePreviewNoModule = CandidateNoneNoModule
|
||||
// NonePreviewNoManifest is for a module whose manifest at the merge the indexer does not hold.
|
||||
NonePreviewNoManifest = "the manifest of the merge is not kept"
|
||||
// NoneNoDirectory is for a module whose directory in its repository neither the modules bucket nor the preview tells.
|
||||
NoneNoDirectory = "the module's directory in its repository is not known"
|
||||
)
|
||||
|
||||
// PutPreview takes a change of the previews bucket in without writing any candidate: a put of a preview (key
|
||||
// PreviewKey) or its delete. It answers whether the previews held changed. A key or a value that is not a
|
||||
// preview is said in the log and left out.
|
||||
func (ix *Indexer) PutPreview(key string, value []byte, deleted bool) bool {
|
||||
if _, _, _, err := ParsePreviewKey(key); err != nil {
|
||||
ix.logf("previews.%s is not a preview key, left out: %v", key, err)
|
||||
return false
|
||||
}
|
||||
ix.mu.Lock()
|
||||
defer ix.mu.Unlock()
|
||||
if deleted {
|
||||
if _, had := ix.previews[key]; !had {
|
||||
return false
|
||||
}
|
||||
delete(ix.previews, key)
|
||||
return true
|
||||
}
|
||||
pv, err := DecodePreview(value)
|
||||
if err != nil {
|
||||
ix.logf("previews.%s is not a preview, left out: %v", key, err)
|
||||
return false
|
||||
}
|
||||
if ix.previews == nil {
|
||||
ix.previews = map[string]meshdeliverychecker.PreviewsByRepositoryHeadTrunkValue{}
|
||||
}
|
||||
if cur, had := ix.previews[key]; had && SamePreview(cur, pv) {
|
||||
return false
|
||||
}
|
||||
ix.previews[key] = pv
|
||||
return true
|
||||
}
|
||||
|
||||
// ApplyPreview takes one change of the previews in (PutPreview) and, when the previews held changed, writes the
|
||||
// candidates' fingerprints again (RecomputeCandidates).
|
||||
func (ix *Indexer) ApplyPreview(ctx context.Context, key string, value []byte, deleted bool) error {
|
||||
if !ix.PutPreview(key, value, deleted) {
|
||||
return nil
|
||||
}
|
||||
return ix.RecomputeCandidates(ctx)
|
||||
}
|
||||
|
||||
// RecomputeCandidates writes again every candidate whose pull requests' fingerprints, from the previews held, are no longer
|
||||
// what it holds. A candidate is written only when it differs, by compare-and-set on the revision read.
|
||||
func (ix *Indexer) RecomputeCandidates(ctx context.Context) error {
|
||||
if ix.Store == nil {
|
||||
return nil
|
||||
}
|
||||
ix.mu.Lock()
|
||||
none := len(ix.previews) == 0
|
||||
ix.mu.Unlock()
|
||||
if none {
|
||||
return nil // no preview held: every candidate has what the pull messages gave it
|
||||
}
|
||||
keys, err := ix.Store.Keys(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("listing the candidates: %w", err)
|
||||
}
|
||||
for _, key := range keys {
|
||||
err := write(ctx, ix.Store, key, func(value []byte, found bool) ([]byte, bool, error) {
|
||||
if !found {
|
||||
return nil, false, nil
|
||||
}
|
||||
c, err := DecodeCandidate(value)
|
||||
if err != nil {
|
||||
ix.logf("candidates.%s holds what is not a candidate, left: %v", key, err)
|
||||
return nil, false, nil
|
||||
}
|
||||
if !ix.withPreviews(ctx, &c) {
|
||||
return nil, false, nil
|
||||
}
|
||||
data, err := EncodeCandidate(c)
|
||||
return data, false, err
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("candidates.%s: %w", key, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// previewOf is the preview of a head merged onto the trunk head the source heads name: found, or stale when only a preview
|
||||
// onto another trunk head exists, or neither (found false, stale false) when the source is not known or none exists.
|
||||
func (ix *Indexer) previewOf(heads map[string]string, repository, head string) (pv meshdeliverychecker.PreviewsByRepositoryHeadTrunkValue, found, stale bool) {
|
||||
ix.mu.Lock()
|
||||
defer ix.mu.Unlock()
|
||||
trunkHead := heads[repository]
|
||||
if trunkHead == "" {
|
||||
return meshdeliverychecker.PreviewsByRepositoryHeadTrunkValue{}, false, false
|
||||
}
|
||||
if key, err := PreviewKey(repository, head, trunkHead); err == nil {
|
||||
if pv, ok := ix.previews[key]; ok {
|
||||
return pv, true, false
|
||||
}
|
||||
}
|
||||
for _, pv := range ix.previews {
|
||||
if pv.Repository == repository && pv.Head == head {
|
||||
return meshdeliverychecker.PreviewsByRepositoryHeadTrunkValue{}, false, true
|
||||
}
|
||||
}
|
||||
return meshdeliverychecker.PreviewsByRepositoryHeadTrunkValue{}, false, false
|
||||
}
|
||||
|
||||
// withPreviews gives each touched module of the candidate's pull requests the fingerprint its preview gives it, or the
|
||||
// reason it has none, and the candidate its fpset. It answers whether the candidate changed. A module of a pull request with
|
||||
// no preview keeps what it had.
|
||||
//
|
||||
// The candidate's Base is the source revision whose trunk heads the previews were read onto (novox/hq ADR 0437 §5): written
|
||||
// whenever the fingerprints are, so the builder merges a preview onto the trunk head its fingerprint was computed on (the head
|
||||
// of the pull request's repository in that revision), never onto a module value's commit, which stays at the commit its
|
||||
// fingerprint first appeared at.
|
||||
func (ix *Indexer) withPreviews(ctx context.Context, c *meshcatalog.CandidatesByGroupValue) bool {
|
||||
ix.mu.Lock()
|
||||
rev, heads := ix.source.rev, ix.source.heads
|
||||
ix.mu.Unlock()
|
||||
changed, read := false, false
|
||||
// A package one of the candidate's pull requests publishes is built at that pull request's head, every other at its
|
||||
// latest published version (novox/hq ADR 0573 §4): a group's members build against the group's own package.
|
||||
members := map[string]string{}
|
||||
for _, p := range c.Pulls {
|
||||
members[trunk.RepositoryName(p.Repository)] = p.Head
|
||||
}
|
||||
var held map[string]meshcatalog.ModulesByModuleValue
|
||||
listed := false
|
||||
for i := range c.Pulls {
|
||||
p := &c.Pulls[i]
|
||||
pv, found, stale := ix.previewOf(heads, p.Repository, p.Head)
|
||||
if !found && !stale {
|
||||
continue
|
||||
}
|
||||
read = true
|
||||
if !listed {
|
||||
var err error
|
||||
if held, err = ix.heldModules(ctx); err != nil {
|
||||
ix.logf("the modules cannot be listed, so no touched module's directory or package version is known: %v", err)
|
||||
}
|
||||
listed = true
|
||||
}
|
||||
// A touched module's directory: the one the modules bucket holds of the repository, else the preview's directory
|
||||
// whose manifest is the one the pull request gave the module (a module the pull request adds; ADR 0501 clause 6).
|
||||
dirs := map[string]string{}
|
||||
for name, m := range held {
|
||||
if trunk.SameRepository(m.Repository, p.Repository) && m.Path != "" {
|
||||
dirs[name] = m.Path
|
||||
}
|
||||
}
|
||||
for _, m := range p.Modules {
|
||||
if _, known := dirs[m]; known || p.Fps[m].Manifest == "" {
|
||||
continue
|
||||
}
|
||||
for dir, pm := range pv.Modules {
|
||||
if pm.Manifest == p.Fps[m].Manifest {
|
||||
dirs[m] = dir
|
||||
}
|
||||
}
|
||||
}
|
||||
fps := maps.Clone(p.Fps)
|
||||
for _, m := range p.Modules {
|
||||
next := meshcatalog.CandidatesByGroupValuePullsItemFpsValue{Manifest: p.Fps[m].Manifest}
|
||||
switch {
|
||||
case stale:
|
||||
next.None2 = NonePreviewStale
|
||||
case len(pv.Conflicts) > 0:
|
||||
next.None2 = NonePreviewConflict + strings.Join(pv.Conflicts, ", ")
|
||||
default:
|
||||
dir, known := dirs[m]
|
||||
if !known {
|
||||
next.None2 = NoneNoDirectory
|
||||
break
|
||||
}
|
||||
next = ix.candidateModule(ctx, m, dir, pv, held, members)
|
||||
}
|
||||
if fps == nil {
|
||||
fps = map[string]meshcatalog.CandidatesByGroupValuePullsItemFpsValue{}
|
||||
}
|
||||
if !reflect.DeepEqual(fps[m], next) {
|
||||
fps[m] = next
|
||||
changed = true
|
||||
}
|
||||
}
|
||||
if changed || !maps.EqualFunc(fps, p.Fps, func(a, b meshcatalog.CandidatesByGroupValuePullsItemFpsValue) bool { return reflect.DeepEqual(a, b) }) {
|
||||
p.Fps = fps
|
||||
}
|
||||
}
|
||||
if changed {
|
||||
c.Fpset = meshcatalogclient.FingerprintSet(c.Pulls)
|
||||
}
|
||||
if read && rev != 0 && c.Base != int64(rev) && (changed || c.Base == 0) {
|
||||
c.Base, changed = int64(rev), true
|
||||
}
|
||||
return changed
|
||||
}
|
||||
|
||||
// candidateModule is the fingerprint of module m, whose directory is dir, at the preview: the module's inputs from the
|
||||
// merge (tree, manifest, locks), its bases' and the builder's fingerprints as the trunk's modules hold them, the toolchains
|
||||
// the build modules agree on, and the version of each mesh package it uses (candidatePackages), by the rules Recompute
|
||||
// applies to a module of the trunk.
|
||||
func (ix *Indexer) candidateModule(ctx context.Context, m, dir string, pv meshdeliverychecker.PreviewsByRepositoryHeadTrunkValue, held map[string]meshcatalog.ModulesByModuleValue, members map[string]string) meshcatalog.CandidatesByGroupValuePullsItemFpsValue {
|
||||
pm, ok := pv.Modules[dir]
|
||||
if !ok && dir == "" {
|
||||
pm, ok = pv.Modules[meshcatalogclient.RootPath]
|
||||
}
|
||||
if !ok && dir == meshcatalogclient.RootPath {
|
||||
pm, ok = pv.Modules[""]
|
||||
}
|
||||
if !ok {
|
||||
return meshcatalog.CandidatesByGroupValuePullsItemFpsValue{None2: NonePreviewNoModule}
|
||||
}
|
||||
out := meshcatalog.CandidatesByGroupValuePullsItemFpsValue{Manifest: pm.Manifest}
|
||||
if ix.Manifests == nil {
|
||||
out.None2 = NonePreviewNoManifest
|
||||
return out
|
||||
}
|
||||
value, _, found, err := ix.Manifests.Get(ctx, pm.Manifest)
|
||||
if err != nil || !found {
|
||||
out.None2 = NonePreviewNoManifest
|
||||
return out
|
||||
}
|
||||
kept, err := meshcatalogclient.DecodeManifest(value)
|
||||
if err != nil {
|
||||
out.None2 = NonePreviewNoManifest
|
||||
return out
|
||||
}
|
||||
// The preview carries the directory's raw inputs (its tree, the named paths' objects, the lock hashes): the indexer
|
||||
// computes the named tree (ADR 0593 §2). The version line's packages are the trunk module's, unless an older build
|
||||
// module still names them.
|
||||
trunkMod := held[m]
|
||||
packages := fingerprint.OtherPackages(pm.Packages, pv.Repository)
|
||||
if trunkMod.Inputs2 != nil {
|
||||
if len(pm.Packages) == 0 {
|
||||
packages = trunkMod.Inputs2.Packages
|
||||
} else {
|
||||
// The siblings its go.mod replaces by a directory are the trunk module's: no go.sum names them.
|
||||
for _, p := range trunkMod.Inputs2.Packages {
|
||||
if repo, _, ok := trunk.PackageRepository(p); ok && trunk.SameRepository(repo, pv.Repository) {
|
||||
packages = append(packages, p)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
in := inputsOfNamed(m, kept.Manifest, pm.Tree, pm.Locks, pm.Paths, packages, ix.baseFP(ctx))
|
||||
if in.Kept == nil {
|
||||
out.None2 = in.None2
|
||||
return out
|
||||
}
|
||||
if in.None2 != "" && in.None2 != sdkstep.ToolchainNoReport && in.None2 != None2Pending {
|
||||
out.None2 = in.None2 // a context, or a base with no fingerprint
|
||||
return out
|
||||
}
|
||||
tools := ix.agreed()
|
||||
toolchains := map[string]meshcatalogclient.Toolchain2{}
|
||||
for _, artifact := range slices.Sorted(maps.Keys(in.Languages)) {
|
||||
t := tools[in.Languages[artifact]]
|
||||
if t.none != "" {
|
||||
out.None2 = t.none
|
||||
return out
|
||||
}
|
||||
toolchains[artifact] = t.tc
|
||||
}
|
||||
inputs := *in.Kept // no builder input (ADR 0530 §1)
|
||||
var fp, none string
|
||||
var versions sdkstep.PackageVersions
|
||||
if trunkMod.Inputs2 != nil && len(trunkMod.Inputs2.Imports) > 0 {
|
||||
inputs.Imports, inputs.ImportForm = trunkMod.Inputs2.Imports, trunkMod.Inputs2.ImportForm
|
||||
}
|
||||
// The import form against the group's own heads of the packages it publishes (ADR 0573 §4, ADR 0593 §2), as the trunk
|
||||
// computes it once merged, the trunk's held fingerprint standing where nothing the build was built from changed.
|
||||
published := ix.candidatePublished(ctx, members, held)
|
||||
// A package of a repository outside the group by the series the candidate's manifest accepts (DLV-I6); a member's by the
|
||||
// group's head.
|
||||
var pins sdkstep.PackageVersions
|
||||
if _, man, err := ix.manifestRaw(ctx, pm.Manifest); err == nil {
|
||||
series := ix.SeriesAll(ctx)
|
||||
for key, s := range series {
|
||||
if repo, _, _ := trunk.PackageRepository(s.GoModule); members[repo] != "" {
|
||||
delete(series, key)
|
||||
}
|
||||
}
|
||||
published, pins = acceptedSeries(series, man, pv.Repository, published)
|
||||
}
|
||||
if len(inputs.Imports) > 0 && len(fingerprint.Resolve(pv.Repository, inputs.Imports, published).Missing) == 0 {
|
||||
f := formOf(m, pv.Repository, &inputs, toolchains, trunkMod, held, published, pins, ix.sourceHeads(), nil)
|
||||
fp, none, versions = f.fp, f.none, f.versions
|
||||
} else {
|
||||
inputs.Imports, inputs.ImportForm = nil, ""
|
||||
versions = candidatePackages(fingerprint.OtherPackages(inputs.Packages, pv.Repository), held, members, ix.sourceHeads())
|
||||
for p, v := range pins {
|
||||
if _, named := versions[p]; named || slices.Contains(inputs.Packages, p) {
|
||||
if versions == nil {
|
||||
versions = sdkstep.PackageVersions{}
|
||||
}
|
||||
versions[p] = v
|
||||
}
|
||||
}
|
||||
in := inputsFrom(m, inputs, toolchains)
|
||||
in.Packages = fingerprint.VersionLine(pv.Repository, inputs.Packages, versions, published)
|
||||
fp, none = fingerprint.Of(in)
|
||||
}
|
||||
if fp == "" {
|
||||
out.None2 = none
|
||||
return out
|
||||
}
|
||||
// The ask of the candidate's build names every member of the group at its head, also one the module names no package of
|
||||
// (a member that publishes none yet, or one the module reaches through mesh-sdk): the fingerprint stays the trunk's, so
|
||||
// the build is the one the merge keeps (grouppackages.Heads, issue 1921).
|
||||
out.Fp2, out.Packages = fp, grouppackages.Heads(versions, members, pv.Repository)
|
||||
if len(inputs.Paths) > 0 {
|
||||
out.Paths = inputs.Paths // the builder's fetch for the candidate names them (ADR 0499 clause 4)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// candidatePackages is the version each package of a candidate's module is built against (novox/hq ADR 0573 §2, §4): the
|
||||
// head of the candidate's pull request whose repository publishes it (members, by repository), else its latest published
|
||||
// version, as the trunk's modules hold it, else the commit the gate's source names for its repository (heads; design 47 G A-365). A package with none is left out, and the fingerprint is then none.
|
||||
func candidatePackages(packages []string, held map[string]meshcatalog.ModulesByModuleValue, members, heads map[string]string) sdkstep.PackageVersions {
|
||||
versions, _ := fingerprint.HeldVersionsOf(packages, held)
|
||||
for _, p := range packages {
|
||||
repository, _, _ := trunk.PackageRepository(p)
|
||||
if head := members[repository]; head != "" {
|
||||
if versions == nil {
|
||||
versions = sdkstep.PackageVersions{}
|
||||
}
|
||||
versions[p] = head
|
||||
} else if _, named := versions[p]; !named {
|
||||
// A package no module publishes, of a repository outside the group: the commit the gate's source names (A-365).
|
||||
if head := heads[repository]; len(head) == 40 {
|
||||
if versions == nil {
|
||||
versions = sdkstep.PackageVersions{}
|
||||
}
|
||||
versions[p] = head
|
||||
}
|
||||
}
|
||||
}
|
||||
return versions
|
||||
}
|
||||
@@ -1,346 +0,0 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdeliverychecker"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
|
||||
"mesh-delivery-indexer/internal/fingerprint"
|
||||
)
|
||||
|
||||
const (
|
||||
trunkNow = "5555555555555555555555555555555555555555"
|
||||
trunkOld = "4444444444444444444444444444444444444444"
|
||||
mergeSha = "6666666666666666666666666666666666666666"
|
||||
)
|
||||
|
||||
// A trunk of the catalogue with avahi (and the builder, a base and others) merged, the toolchain agreed, and the gate's source
|
||||
// naming the catalogue's trunk head: avahi has a fingerprint of the new form at tree-avahi.
|
||||
func previewBed(t *testing.T, report bool) (*Indexer, *memStore, *memStore) {
|
||||
t.Helper()
|
||||
ix, mods := newFormIndexer(t, builderModules(t)...)
|
||||
if report {
|
||||
reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1")
|
||||
}
|
||||
if err := applyS(ix, context.Background(), 5, srcT{Group: "g", Heads: map[string]string{catalogueRepo: trunkNow}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return ix, mods, ix.Store.(*memStore)
|
||||
}
|
||||
|
||||
// avahiOnTools is avahi's manifest as builderModules has it: standing on mesh-tools.
|
||||
func avahiOnTools(t *testing.T) string {
|
||||
return standingOn(t, catalogueManifest(t, "avahi"), "mesh-tools")
|
||||
}
|
||||
|
||||
// avahiPull is the pull message of the catalogue's pull request 301 touching avahi, with its manifest.
|
||||
func avahiPull(t *testing.T, head string) forgemsg.Forge {
|
||||
t.Helper()
|
||||
f := manifestPull(t, 301, head, avahiOnTools(t))
|
||||
f.Manifests = map[string]string{"modules/avahi": avahiOnTools(t)}
|
||||
return f
|
||||
}
|
||||
|
||||
func avahiPreview(t *testing.T, trunk, head string) meshdeliverychecker.PreviewsByRepositoryHeadTrunkValue {
|
||||
return meshdeliverychecker.PreviewsByRepositoryHeadTrunkValue{Repository: catalogueRepo, Head: head, Trunk: trunk, Merge: mergeSha, Modules: map[string]meshdeliverychecker.PreviewsByRepositoryHeadTrunkValueModulesValue{
|
||||
"modules/avahi": {Tree: "tree-avahi", Manifest: meshcatalogclient.ManifestDigest([]byte(avahiOnTools(t))), Locks: map[string]string{"go.sum": avahiGoSum}}}}
|
||||
}
|
||||
|
||||
func applyPreview(t *testing.T, ix *Indexer, pv meshdeliverychecker.PreviewsByRepositoryHeadTrunkValue) {
|
||||
t.Helper()
|
||||
key, err := PreviewKey(pv.Repository, pv.Head, pv.Trunk)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
data, err := EncodePreview(pv)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ix.ApplyPreview(context.Background(), key, data, false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func fpOf(t *testing.T, st *memStore) meshcatalog.CandidatesByGroupValuePullsItemFpsValue {
|
||||
t.Helper()
|
||||
c := candidateOf(t, st)
|
||||
if len(c.Pulls) != 1 {
|
||||
t.Fatalf("candidate = %+v", c)
|
||||
}
|
||||
return c.Pulls[0].Fps["avahi"]
|
||||
}
|
||||
|
||||
func TestACandidatesModuleGetsTheFingerprintATrunkModuleOfTheSameTreeHas(t *testing.T) { // 1445/AC2
|
||||
ix, mods, cands := previewBed(t, true)
|
||||
trunk, _ := moduleOf(t, mods, "avahi")
|
||||
if trunk.Fp2 == "" {
|
||||
t.Fatalf("the trunk's avahi has no fingerprint of the new form: %+v", trunk)
|
||||
}
|
||||
if err := ix.Apply(context.Background(), avahiPull(t, h1)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := fpOf(t, cands); got.None2 != meshcatalogclient.NoneNoPreview {
|
||||
t.Fatalf("before the preview: %+v", got)
|
||||
}
|
||||
applyPreview(t, ix, avahiPreview(t, trunkNow, h1))
|
||||
got := fpOf(t, cands)
|
||||
if got.Fp2 != trunk.Fp2 || got.None2 != "" || got.Manifest != meshcatalogclient.ManifestDigest([]byte(avahiOnTools(t))) {
|
||||
t.Fatalf("candidate avahi = %+v, want the trunk's fingerprint %s", got, trunk.Fp2)
|
||||
}
|
||||
c := candidateOf(t, cands)
|
||||
if c.Fpset != meshcatalogclient.FingerprintSet(c.Pulls) || c.Fpset == "" {
|
||||
t.Errorf("fpset = %q", c.Fpset)
|
||||
}
|
||||
// The candidate names the source revision the preview was read onto (5, whose catalogue head is the preview's trunk):
|
||||
// the builder merges the preview onto that head, never onto the module value's commit.
|
||||
if c.Base != 5 {
|
||||
t.Errorf("candidate base = %d, want the source revision 5 its preview was read onto", c.Base)
|
||||
}
|
||||
written(t, writtenCandidate{Source: sourceRev{Rev: 5, Heads: map[string]string{catalogueRepo: trunkNow}}, Candidate: c})
|
||||
// Issue 1921: a candidate's build is asked every member of the group at its head, also one that publishes no package yet
|
||||
// and one the module names no package of; the fingerprint stays the trunk's, so the build is the one the merge keeps.
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
members map[string]string // the other pull requests of the group: repository, head
|
||||
want sdkstep.PackageVersions
|
||||
}{
|
||||
{"a group of one names nothing", nil, nil},
|
||||
{"a member that publishes no package yet", map[string]string{"novox/mesh-contracts": h2},
|
||||
sdkstep.PackageVersions{"git.novox.be/novox/mesh-contracts": h2}},
|
||||
{"mesh-sdk by its go module", map[string]string{"novox/mesh-sdk": h2, "novox/mesh-contracts": h3},
|
||||
sdkstep.PackageVersions{"git.novox.be/novox/mesh-sdk/go": h2, "git.novox.be/novox/mesh-contracts": h3}},
|
||||
} {
|
||||
ix, mods, cands := previewBed(t, true)
|
||||
trunk, _ := moduleOf(t, mods, "avahi")
|
||||
if err := ix.Apply(context.Background(), avahiPull(t, h1)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for repository, head := range tc.members {
|
||||
other := forgemsg.Forge{V: forgemsg.Version, Kind: forgemsg.ForgePull, Repository: repository, Branch: "feat/x", Number: 7, Head: head,
|
||||
Action: forgemsg.PullUpdated}
|
||||
if err := ix.Apply(context.Background(), other); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
applyPreview(t, ix, avahiPreview(t, trunkNow, h1))
|
||||
var got meshcatalog.CandidatesByGroupValuePullsItemFpsValue
|
||||
for _, p := range candidateOf(t, cands).Pulls {
|
||||
if p.Repository == catalogueRepo {
|
||||
got = p.Fps["avahi"]
|
||||
}
|
||||
}
|
||||
if got.Fp2 != trunk.Fp2 || !maps.Equal(got.Packages, tc.want) {
|
||||
t.Errorf("%s: candidate avahi = %+v, want the trunk's fingerprint %s and the packages %v", tc.name, got, trunk.Fp2, tc.want)
|
||||
}
|
||||
}
|
||||
// Another tree gives another fingerprint.
|
||||
pv := avahiPreview(t, trunkNow, h1)
|
||||
m := pv.Modules["modules/avahi"]
|
||||
m.Tree = "tree-changed"
|
||||
pv.Modules["modules/avahi"] = m
|
||||
pv.Merge = "7777777777777777777777777777777777777777"
|
||||
applyPreview(t, ix, pv)
|
||||
if other := fpOf(t, cands); other.Fp2 == "" || other.Fp2 == trunk.Fp2 {
|
||||
t.Errorf("a changed tree: %+v", other)
|
||||
}
|
||||
// Issue 1585 (ADR 0573 §2, §4): a candidate's module is built against the head of the candidate's pull request that
|
||||
// publishes a package it uses, else against the package's latest published version; a preview naming a package of
|
||||
// mesh-sdk that nothing published gives no fingerprint, and one of its own repository's comes with its tree.
|
||||
const sdk, own = "git.novox.be/novox/mesh-sdk/go", "git.novox.be/novox/mesh-catalog/contract"
|
||||
held := map[string]meshcatalog.ModulesByModuleValue{"mesh-sdk": {Repository: "novox/mesh-sdk", Path: meshcatalogclient.RootPath, Commit: trunkOld}}
|
||||
if got := candidatePackages([]string{sdk}, held, map[string]string{catalogueRepo: h1}, nil); got[sdk] != trunkOld {
|
||||
t.Errorf("outside a group: %v, want mesh-sdk at its trunk %s", got, trunkOld)
|
||||
}
|
||||
if got := candidatePackages([]string{sdk}, held, map[string]string{catalogueRepo: h1, "novox/mesh-sdk": mergeSha}, nil); got[sdk] != mergeSha {
|
||||
t.Errorf("in a group with mesh-sdk: %v, want its member's head %s", got, mergeSha)
|
||||
}
|
||||
// Design 47 G A-365: a package no module publishes, of a repository outside the group, takes the commit the source names.
|
||||
if got := candidatePackages([]string{sdk}, nil, map[string]string{catalogueRepo: h1}, map[string]string{"novox/mesh-sdk": strings.Repeat("9", 40)}); got[sdk] != strings.Repeat("9", 40) {
|
||||
t.Errorf("an unpublished package outside the group: %v, want the source's head", got)
|
||||
}
|
||||
m.Packages = []string{own, sdk}
|
||||
pv.Modules["modules/avahi"] = m
|
||||
pv.Merge = "8888888888888888888888888888888888888888"
|
||||
applyPreview(t, ix, pv)
|
||||
if got := fpOf(t, cands); got.Fp2 != "" || got.None2 != fingerprint.NoPackageVersion+sdk {
|
||||
t.Errorf("a package nothing published: %+v", got)
|
||||
}
|
||||
// Through the indexer's wiring (A-365): once the gate's source names mesh-sdk's head, the candidate is built against it.
|
||||
if err := applyS(ix, context.Background(), 5, srcT{Group: "g", Heads: map[string]string{catalogueRepo: trunkNow, "novox/mesh-sdk": strings.Repeat("9", 40)}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := fpOf(t, cands); got.Fp2 == "" {
|
||||
t.Errorf("a package nothing published, the source naming its repository: %+v, want a fingerprint", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnUnknownToolchainLeavesTheCandidatesModuleNamingIt(t *testing.T) { // 1445/AC2
|
||||
ix, _, cands := previewBed(t, false)
|
||||
if err := ix.Apply(context.Background(), avahiPull(t, h1)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
applyPreview(t, ix, avahiPreview(t, trunkNow, h1))
|
||||
if got := fpOf(t, cands); got.Fp2 != "" || !strings.Contains(got.None2, sdkstep.ToolchainNoReport) {
|
||||
t.Errorf("candidate avahi = %+v, want none2 naming %q", got, sdkstep.ToolchainNoReport)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPreviewMadeOntoAnOldTrunkIsOutOfDate(t *testing.T) { // 1445/AC3
|
||||
ix, _, cands := previewBed(t, true)
|
||||
if err := ix.Apply(context.Background(), avahiPull(t, h1)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
applyPreview(t, ix, avahiPreview(t, trunkOld, h1))
|
||||
if got := fpOf(t, cands); got.Fp2 != "" || got.None2 != NonePreviewStale {
|
||||
t.Errorf("candidate avahi = %+v, want it out of date", got)
|
||||
}
|
||||
// A preview onto the trunk head now makes it good again, and the trunk moving again makes it stale.
|
||||
applyPreview(t, ix, avahiPreview(t, trunkNow, h1))
|
||||
if got := fpOf(t, cands); got.Fp2 == "" {
|
||||
t.Fatalf("after the right preview: %+v", got)
|
||||
}
|
||||
if err := applyS(ix, context.Background(), 6, srcT{Group: "g", Heads: map[string]string{catalogueRepo: "8888888888888888888888888888888888888888"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := fpOf(t, cands); got.None2 != NonePreviewStale {
|
||||
t.Errorf("after the trunk moved: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPreviewHeardBeforeItsPullRequestGivesTheFingerprintWhenTheCandidateIsWritten(t *testing.T) { // 1445/AC2
|
||||
ix, mods, cands := previewBed(t, true)
|
||||
applyPreview(t, ix, avahiPreview(t, trunkNow, h1))
|
||||
if err := ix.Apply(context.Background(), avahiPull(t, h1)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
trunk, _ := moduleOf(t, mods, "avahi")
|
||||
if got := fpOf(t, cands); got.Fp2 != trunk.Fp2 {
|
||||
t.Errorf("candidate avahi = %+v, want %s", got, trunk.Fp2)
|
||||
}
|
||||
// The same pull request at a new head has no preview of it yet.
|
||||
if err := ix.Apply(context.Background(), avahiPull(t, h2)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := fpOf(t, cands); got.Fp2 != "" || got.None2 != meshcatalogclient.NoneNoPreview {
|
||||
t.Errorf("a new head: %+v, want no preview", got)
|
||||
}
|
||||
// Issue 1921: a head whose tree lacks the directory asked is asked once, however many passes read it, until the head changes;
|
||||
// any other failure (the forge does not answer) is asked again at every pass and said once per head.
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
err error
|
||||
askedAfterTwo int // the same head read twice
|
||||
askedAfterNew int // then another head
|
||||
linesAfterNew int
|
||||
}{
|
||||
{"the directory is not in the tree", fmt.Errorf("listing: %w", ErrNotInTree), 1, 2, 2},
|
||||
{"the forge does not answer", errors.New("the forge does not answer"), 2, 3, 2},
|
||||
} {
|
||||
t.Run("a failing head: "+row.name, func(t *testing.T) {
|
||||
ix, _, _ := previewBed(t, false)
|
||||
var logs strings.Builder
|
||||
ix.Logf = func(format string, args ...any) { logs.WriteString(fmt.Sprintf(format, args...) + "\n") }
|
||||
asked := 0
|
||||
ix.Files = func(context.Context, string, string, []string) (map[string]map[string]string, error) {
|
||||
asked++
|
||||
return nil, row.err
|
||||
}
|
||||
held := map[string]meshcatalog.ModulesByModuleValue{"x": {Inputs2: &meshcatalog.ModulesByModuleValueInputs2{Packages: []string{"git.novox.be/novox/mesh-sdk/go"}}}}
|
||||
for pass, head := range []string{h1, h1, h2} {
|
||||
ix.candidatePublished(context.Background(), map[string]string{"novox/mesh-sdk": head}, held)
|
||||
if pass == 1 && (asked != row.askedAfterTwo || strings.Count(logs.String(), "its packages are not read") != 1) {
|
||||
t.Fatalf("after two passes over %s: asked %d (want %d), journal %q: want one line", head, asked, row.askedAfterTwo, logs.String())
|
||||
}
|
||||
}
|
||||
if asked != row.askedAfterNew || strings.Count(logs.String(), "its packages are not read") != row.linesAfterNew {
|
||||
t.Errorf("asked %d (want %d), journal %q (want %d lines)", asked, row.askedAfterNew, logs.String(), row.linesAfterNew)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPreviewNamingNoManifestOfTheModulesDirectoryGivesAReason(t *testing.T) {
|
||||
t.Run("the same preview again", testTheSamePreviewAppliedAgainWritesNothing)
|
||||
ix, _, cands := previewBed(t, true)
|
||||
if err := ix.Apply(context.Background(), avahiPull(t, h1)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pv := avahiPreview(t, trunkNow, h1)
|
||||
pv.Modules = map[string]meshdeliverychecker.PreviewsByRepositoryHeadTrunkValueModulesValue{"modules/other": pv.Modules["modules/avahi"]}
|
||||
applyPreview(t, ix, pv)
|
||||
if got := fpOf(t, cands); got.None2 != NonePreviewNoModule || !strings.Contains(NonePreviewNoModule, "manifest") {
|
||||
t.Errorf("candidate avahi = %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func testTheSamePreviewAppliedAgainWritesNothing(t *testing.T) {
|
||||
ix, _, cands := previewBed(t, true)
|
||||
if err := ix.Apply(context.Background(), avahiPull(t, h1)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
applyPreview(t, ix, avahiPreview(t, trunkNow, h1))
|
||||
w := cands.writes
|
||||
applyPreview(t, ix, avahiPreview(t, trunkNow, h1))
|
||||
if cands.writes != w {
|
||||
t.Errorf("candidates written %d times for the same preview, want none", cands.writes-w)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPreviewOfConflictsGivesTheCandidatesModuleTheReasonItDoesNotMerge(t *testing.T) { // 1445/AC5
|
||||
ix, _, cands := previewBed(t, true)
|
||||
if err := ix.Apply(context.Background(), avahiPull(t, h1)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
applyPreview(t, ix, meshdeliverychecker.PreviewsByRepositoryHeadTrunkValue{Repository: catalogueRepo, Head: h1, Trunk: trunkNow, Conflicts: []string{"modules/avahi/main.go", "go.mod"}})
|
||||
got := fpOf(t, cands)
|
||||
if got.Fp2 != "" || !strings.HasPrefix(got.None2, "does not merge") || !strings.Contains(got.None2, "go.mod, modules/avahi/main.go") && !strings.Contains(got.None2, "modules/avahi/main.go, go.mod") {
|
||||
t.Errorf("candidate avahi = %+v, want it not merging and naming the files", got)
|
||||
}
|
||||
}
|
||||
|
||||
// writtenCandidate is what the builder's tests read of the indexer (mesh-delivery-builder internal/build): a candidate as the
|
||||
// indexer writes it and the gate's source revision it names, kept in testdata/candidate-written.json so the builder is tested
|
||||
// on what the indexer writes, never on a candidate made by hand. MESH_WRITE_TESTDATA=1 writes it again.
|
||||
type writtenCandidate struct {
|
||||
Source sourceRev `json:"source"`
|
||||
Candidate meshcatalog.CandidatesByGroupValue `json:"candidate"`
|
||||
}
|
||||
|
||||
type sourceRev struct {
|
||||
Rev uint64 `json:"rev"`
|
||||
Heads map[string]string `json:"heads"`
|
||||
}
|
||||
|
||||
func written(t *testing.T, w writtenCandidate) {
|
||||
t.Helper()
|
||||
data, err := json.MarshalIndent(w, "", " ")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
data = append(data, '\n')
|
||||
const file = "testdata/candidate-written.json"
|
||||
if os.Getenv("MESH_WRITE_TESTDATA") == "1" {
|
||||
if err := os.WriteFile(file, data, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
kept, err := os.ReadFile(file)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(kept) != string(data) {
|
||||
t.Errorf("%s is not what the indexer writes now (MESH_WRITE_TESTDATA=1 writes it again):\n%s", file, data)
|
||||
}
|
||||
}
|
||||
@@ -1,75 +0,0 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliverycheckerclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdeliverychecker"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
)
|
||||
|
||||
// A preview as the checker writes it into its state previews, read here (reads: mesh-delivery-checker.previews): the
|
||||
// seat's type and the client's rules; the key is the form the seat names, `<repository>.<head>.<trunk>`, the repository
|
||||
// as a key part (keys.EscapeName).
|
||||
|
||||
// PreviewKey is the key of a head merged onto a trunk commit.
|
||||
func PreviewKey(repository, head, trunk string) (string, error) {
|
||||
if err := meshdeliverycheckerclient.CheckRepository(repository); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := keys.CheckName("head", head); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := keys.CheckName("trunk commit", trunk); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return meshdeliverychecker.PreviewsByRepositoryHeadTrunkKey(keys.EscapeName(repository), head, trunk), nil
|
||||
}
|
||||
|
||||
// ParsePreviewKey is the inverse of PreviewKey.
|
||||
func ParsePreviewKey(key string) (repository, head, trunk string, err error) {
|
||||
parts := strings.Split(key, ".")
|
||||
if len(parts) != 3 {
|
||||
return "", "", "", fmt.Errorf("index: %q is not a preview key, <repository>.<head>.<trunk>", key)
|
||||
}
|
||||
repository, err = keys.UnescapeName(parts[0])
|
||||
if err != nil {
|
||||
return "", "", "", fmt.Errorf("index: %q is not a preview key: %w", key, err)
|
||||
}
|
||||
if _, err := PreviewKey(repository, parts[1], parts[2]); err != nil {
|
||||
return "", "", "", fmt.Errorf("index: %q is not a preview key: %w", key, err)
|
||||
}
|
||||
return repository, parts[1], parts[2], nil
|
||||
}
|
||||
|
||||
// DecodePreview reads a preview.
|
||||
func DecodePreview(data []byte) (meshdeliverychecker.PreviewsByRepositoryHeadTrunkValue, error) {
|
||||
if err := keys.CheckVersion(data); err != nil {
|
||||
return meshdeliverychecker.PreviewsByRepositoryHeadTrunkValue{}, err
|
||||
}
|
||||
p, err := meshdeliverychecker.DecodePreviewsByRepositoryHeadTrunkValue(data)
|
||||
if err != nil {
|
||||
return p, fmt.Errorf("index: a preview does not fit its shape: %w", err)
|
||||
}
|
||||
if err := meshdeliverycheckerclient.CheckPreview(p); err != nil {
|
||||
return meshdeliverychecker.PreviewsByRepositoryHeadTrunkValue{}, err
|
||||
}
|
||||
return p, nil
|
||||
}
|
||||
|
||||
// EncodePreview is p as JSON at the kept values' version (the checker writes previews; the indexer's tests do too).
|
||||
func EncodePreview(p meshdeliverychecker.PreviewsByRepositoryHeadTrunkValue) ([]byte, error) {
|
||||
if err := meshdeliverycheckerclient.CheckPreview(p); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
p.V = keys.Version
|
||||
return json.Marshal(p)
|
||||
}
|
||||
|
||||
// SamePreview says whether two previews are the same value.
|
||||
func SamePreview(a, b meshdeliverychecker.PreviewsByRepositoryHeadTrunkValue) bool {
|
||||
return reflect.DeepEqual(a, b)
|
||||
}
|
||||
@@ -10,9 +10,7 @@ import (
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
)
|
||||
|
||||
// A module registered in the catalogue from its own repository (the operator registers it with the repository and the
|
||||
@@ -130,7 +128,7 @@ func (ix *Indexer) indexAt(ctx context.Context, r Registration, force, reread bo
|
||||
t.why = fmt.Sprintf("the manifest at %q of %s at %s names the module %q", dir, repository, commit, name)
|
||||
return t, nil
|
||||
}
|
||||
f := forgemsg.Forge{V: keys.Version, Kind: forgemsg.ForgeMerged, Repository: repository, Merge: commit, Root: read.Root,
|
||||
f := whole{Repository: repository, Merge: commit, Root: read.Root,
|
||||
Trees: map[string]string{dir: read.Trees[dir]},
|
||||
Manifests: map[string]string{dir: raw}}
|
||||
if l := read.Locks[dir]; len(l) > 0 {
|
||||
@@ -410,7 +408,7 @@ func (ix *Indexer) TakeInRepository(ctx context.Context, repository string) Take
|
||||
out.Answer = "the modules state cannot be read: " + err.Error()
|
||||
return out
|
||||
}
|
||||
f := forgemsg.Forge{V: keys.Version, Kind: forgemsg.ForgeMerged, Repository: out.Repository, Merge: head}
|
||||
f := whole{Repository: out.Repository, Merge: head}
|
||||
if _, err := ix.ApplyMessage(ctx, f); err != nil {
|
||||
out.Answer = fmt.Sprintf("the merge of %s at %.12s could not be taken in: %v", repository, head, err)
|
||||
return out
|
||||
|
||||
@@ -225,8 +225,7 @@ func TestAForgeThatCannotSayLeavesTheModuleUnwrittenAndAsksAgain(t *testing.T) {
|
||||
for _, tc := range []struct{ failure, trigger string }{
|
||||
{"timeout", "the indexer starts again"},
|
||||
{"Gitea API /repos/x/git/trees/y: 404 not found", "the indexer starts again"},
|
||||
{"Gitea API /repos/x/git/trees/y: 404 not found", "a new source revision"},
|
||||
{"Gitea API /repos/x/git/trees/y: 404 not found", "a merged message of another repository"},
|
||||
{"Gitea API /repos/x/git/trees/y: 404 not found", "a take-in of another repository"},
|
||||
{"Gitea API /repos/x/git/trees/y: 404 not found", "a person's take-in"},
|
||||
{"timeout", "a person's take-in"},
|
||||
} {
|
||||
@@ -259,12 +258,7 @@ func TestAForgeThatCannotSayLeavesTheModuleUnwrittenAndAsksAgain(t *testing.T) {
|
||||
if err := ix.IndexRegisteredAll(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case "a new source revision":
|
||||
if err := applyS(ix, ctx, 2, srcT{Heads: map[string]string{"jschoubben/snake": h2, "novox/other": h1}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ix.retryWG.Wait() // read again in the background
|
||||
case "a merged message of another repository":
|
||||
case "a take-in of another repository":
|
||||
if err := ix.Apply(ctx, mergedMsg("novox/other", map[string]string{"": "t9"})); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -24,7 +24,6 @@ import (
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
)
|
||||
@@ -237,7 +236,7 @@ func (ix *Indexer) writeRunCandidate(ctx context.Context, group string, members
|
||||
if name := manifestModule(raw); name != m.Module {
|
||||
return fmt.Errorf("%w: the run %s: the manifest at %q of %s at %.12s names %q, not %s", ErrUnreadable, group, m.dir, m.repository, m.Commit, name, m.Module)
|
||||
}
|
||||
f := forgemsg.Forge{V: keys.Version, Kind: forgemsg.ForgeMerged, Repository: m.repository, Merge: m.Commit, Root: read.Root,
|
||||
f := whole{Repository: m.repository, Merge: m.Commit, Root: read.Root,
|
||||
Trees: map[string]string{m.dir: read.Trees[m.dir]}, Manifests: map[string]string{m.dir: raw}}
|
||||
if l := read.Locks[m.dir]; len(l) > 0 {
|
||||
f.Locks = map[string]map[string]string{m.dir: l}
|
||||
|
||||
@@ -31,7 +31,6 @@ import (
|
||||
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/seatdefs"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
@@ -149,42 +148,15 @@ func digestOf(d seatdef.Definition) (json.RawMessage, string, error) {
|
||||
return raw, hex.EncodeToString(sum[:]), nil
|
||||
}
|
||||
|
||||
// ApplySeatDefinitions keeps the seat definitions a forge message carries (DLV-I17, I18): a merged message's contracts
|
||||
// modules at its merge as the trunk's, a pull message's at its head as that pull request's; a closed pull request, or
|
||||
// a merged one, has its entries removed. Nothing is done without the state or a forge reader.
|
||||
func (ix *Indexer) ApplySeatDefinitions(ctx context.Context, f forgemsg.Forge) error {
|
||||
if ix.SeatDefinitions == nil {
|
||||
// ApplySeatDefinitions keeps the seat definitions of the contracts modules a take-in's trees carry (DLV-I17, I18) as the
|
||||
// trunk's at its commit. Nothing is done without the state or a forge reader.
|
||||
func (ix *Indexer) ApplySeatDefinitions(ctx context.Context, f whole) error {
|
||||
if ix.SeatDefinitions == nil || ix.ReadFile == nil {
|
||||
return nil
|
||||
}
|
||||
if f.Kind == forgemsg.ForgeMerged || (f.Kind == forgemsg.ForgePull && f.Action == forgemsg.PullClosed) {
|
||||
if f.Number > 0 {
|
||||
if err := ix.dropPull(ctx, f.Repository, f.Number); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if f.Kind != forgemsg.ForgeMerged {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
if ix.ReadFile == nil || (f.Kind != forgemsg.ForgeMerged && f.Kind != forgemsg.ForgePull) {
|
||||
return nil
|
||||
}
|
||||
manifests := f.Manifests
|
||||
if f.Kind == forgemsg.ForgePull && len(f.Dirs) > 0 {
|
||||
manifests = map[string]string{}
|
||||
for _, d := range f.Dirs {
|
||||
if raw, ok := f.Manifests[d]; ok {
|
||||
manifests[d] = raw
|
||||
}
|
||||
}
|
||||
}
|
||||
var errs []error
|
||||
for _, c := range contractsModules(manifests) {
|
||||
if f.Kind == forgemsg.ForgeMerged {
|
||||
errs = append(errs, ix.keepTrunk(ctx, f.Repository, f.Merge, c))
|
||||
} else {
|
||||
errs = append(errs, ix.keepPull(ctx, f.Repository, f.Number, f.Head, c))
|
||||
}
|
||||
for _, c := range contractsModules(f.Manifests) {
|
||||
errs = append(errs, ix.keepTrunk(ctx, f.Repository, f.Merge, c))
|
||||
}
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
@@ -8,54 +8,4 @@ package index
|
||||
// of seatdefs.go, the same resolution of the parts and the same rule of which module may define a seat), never by a reader
|
||||
// of their own.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"maps"
|
||||
"slices"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/seatdefs"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/seatdef"
|
||||
)
|
||||
|
||||
// revisionSeats are the seats of the definitions the ready merges carry, read at their merge commits, the newest merge
|
||||
// of a repository governing; none where the indexer has no forge reader. A read the forge does not answer is an error
|
||||
// (the take-in is tried again); a definition that does not parse is said and left out, as the indexer that keeps it
|
||||
// would say it unparsed.
|
||||
func (ix *Indexer) revisionSeats(ctx context.Context, ready []PendingEntry) ([]catalogue.Seat, error) {
|
||||
defs, err := ix.revisionSeatDefinitions(ctx, ready)
|
||||
if err != nil || defs == nil {
|
||||
return nil, err
|
||||
}
|
||||
return catalogue.SeatsFromDefinitions(slices.Collect(maps.Values(defs))), nil
|
||||
}
|
||||
|
||||
// revisionSeatDefinitions are the definitions revisionSeats reads, by seat; nil where the indexer has no forge reader.
|
||||
func (ix *Indexer) revisionSeatDefinitions(ctx context.Context, ready []PendingEntry) (map[string]seatdef.Definition, error) {
|
||||
if ix.ReadFile == nil || ix.Files == nil {
|
||||
return nil, nil
|
||||
}
|
||||
defs := map[string]seatdef.Definition{}
|
||||
for _, e := range ready {
|
||||
if e.Covered || e.Forge.Merge == "" {
|
||||
continue
|
||||
}
|
||||
for _, c := range contractsModules(e.Forge.Manifests) {
|
||||
got, unparsed, err := ix.readDefinitions(ctx, e.Forge.Repository, e.Forge.Merge, c)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for seat, why := range unparsed {
|
||||
ix.logf("the revision's definition of %s (%s) does not parse and is left out: %s", seat, c.Module, why)
|
||||
}
|
||||
for seat, d := range got {
|
||||
if why := seatdefs.Refusal(c.Module, d); why != "" {
|
||||
ix.logf("the revision's definition of %s is left out: %s", seat, why) // as keepTrunk refuses it (ADR 0636 §5)
|
||||
continue
|
||||
}
|
||||
defs[seat] = d
|
||||
}
|
||||
}
|
||||
}
|
||||
return defs, nil
|
||||
}
|
||||
import ()
|
||||
|
||||
@@ -10,7 +10,6 @@ import (
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
@@ -203,8 +202,8 @@ func acceptedSeries(series map[string]SeriesPackage, m catalogue.Manifest, repos
|
||||
// package.json name, each read through the forge. The key keeps what it held and gains the name and the module and
|
||||
// artifact it is built from, so the builder numbers it under that name before its first version. A name the forge does
|
||||
// not answer is said and left for the next merge; a manifest that does not decode is the judge's to refuse.
|
||||
func (ix *Indexer) KeepPackageNames(ctx context.Context, f forgemsg.Forge) error {
|
||||
if ix.Packages == nil || ix.ReadFile == nil || f.Kind != forgemsg.ForgeMerged || f.Merge == "" {
|
||||
func (ix *Indexer) KeepPackageNames(ctx context.Context, f whole) error {
|
||||
if ix.Packages == nil || ix.ReadFile == nil || f.Merge == "" {
|
||||
return nil
|
||||
}
|
||||
var errs []error
|
||||
|
||||
@@ -8,24 +8,15 @@ import (
|
||||
"maps"
|
||||
"path"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk"
|
||||
)
|
||||
|
||||
// The gate's source revision of a watched branch (design 47 G §15, §115), read through the seat mesh-delivery's verb source
|
||||
// and its event source-moved, never the gate's state: it names every repository's merged head, and its revision is the
|
||||
// mesh's source revision of that branch. The indexer keeps the latest it has seen. A module is stamped
|
||||
// with it (Module.Src) when the head the source names for its repository is the merge commit the module's fingerprint
|
||||
// was computed at: from the modules bucket when the merge was heard first, and when the merge is heard after the source.
|
||||
// The gate's source revision of a watched branch (design 47 H §737, §743), heard on its event source-moved: it names the heads
|
||||
// of the repositories it moved, and its number is the mesh's source revision of that branch. The indexer keeps the latest it
|
||||
// took in.
|
||||
|
||||
// sourceNow is the latest source revision seen.
|
||||
type sourceNow struct {
|
||||
@@ -51,236 +42,10 @@ func (ix *Indexer) sourceHeads() map[string]string {
|
||||
return maps.Clone(ix.source.heads)
|
||||
}
|
||||
|
||||
// ApplySource takes the source revision the verb source answered in. An older revision than the latest seen is left. For each
|
||||
// repository the source names, each module built from it whose fingerprint was computed at the head the source names, and
|
||||
// that carries no Src yet, is written again with Src the revision (once: a later revision leaves it); a module computed at another commit is left (a merge heard later is stamped
|
||||
// when it is written, applyMerged), and a repository none of the indexer's modules is built from writes nothing. The
|
||||
// modules of a repository are those the modules bucket holds of it (ADR 0501 clause 6).
|
||||
func (ix *Indexer) ApplySource(ctx context.Context, a meshdelivery.SourceAnswer) error {
|
||||
rev := uint64(a.Revision)
|
||||
s := struct{ Heads map[string]string }{Heads: map[string]string{}}
|
||||
for _, r := range a.Repositories {
|
||||
s.Heads[r.Repository] = r.Head
|
||||
}
|
||||
ix.mu.Lock()
|
||||
if rev < ix.source.rev {
|
||||
ix.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
before := ix.source.heads
|
||||
ix.source = sourceNow{rev: rev, heads: maps.Clone(s.Heads)}
|
||||
ix.mu.Unlock()
|
||||
if ix.Modules == nil {
|
||||
return ix.RecomputeCandidates(ctx)
|
||||
}
|
||||
if err := ix.kickCatchUp(ctx, s.Heads); err != nil {
|
||||
return err
|
||||
}
|
||||
// A merge the indexer never took in is taken in at each source revision, and at the start (A-463).
|
||||
ix.kickBehind(ctx)
|
||||
// A registered module the forge could not read is read again at each new source revision (issue 1945), in the
|
||||
// background: a forge that does not answer must not hold this revision. One taken in later is stamped when it is written
|
||||
// (applyManifests reads srcOf).
|
||||
ix.kickRetry(ctx)
|
||||
held, err := ix.heldModules(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, key := range slices.Sorted(maps.Keys(held)) {
|
||||
repository := ""
|
||||
for r := range s.Heads {
|
||||
if trunk.SameRepository(r, held[key].Repository) {
|
||||
repository = r
|
||||
}
|
||||
}
|
||||
if repository == "" {
|
||||
continue
|
||||
}
|
||||
head := s.Heads[repository]
|
||||
err = write(ctx, ix.Modules, key, func(value []byte, found bool) ([]byte, bool, error) {
|
||||
if !found {
|
||||
return nil, false, nil
|
||||
}
|
||||
cur, err := meshcatalogclient.DecodeModule(value)
|
||||
if err != nil {
|
||||
ix.logf("modules.%s holds what is not a module, left: %v", key, err)
|
||||
return nil, false, nil
|
||||
}
|
||||
// Stamped once, with the first revision naming the commit as the head: a later revision (a merge of another
|
||||
// repository) leaves the value as it is, so no revision of the source rewrites a module that did not change.
|
||||
if cur.Repository != repository || cur.Commit != head || cur.Src != 0 {
|
||||
return nil, false, nil
|
||||
}
|
||||
cur.Src = int64(rev)
|
||||
if cur.FpSrc == 0 {
|
||||
cur.FpSrc = int64(rev) // its fingerprint moved at this merge, whose revision is heard only now (ADR 0601 §1)
|
||||
}
|
||||
data, err := meshcatalogclient.EncodeModule(cur)
|
||||
return data, false, err
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("modules.%s: %w", key, err)
|
||||
}
|
||||
}
|
||||
// A module built against the commit the source named as a repository's head (a package no module publishes, A-365) is
|
||||
// computed again when that head moves: the gate moves the source on the forge bridge's merged message (A-464), before the take-in, and at the take-in for a head no merged moved.
|
||||
if movedPackages(held, before, s.Heads) {
|
||||
if err := ix.Recompute(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
// The stamps are kept with the view (KeepView), so a restart reads them back.
|
||||
if err := ix.KeepView(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
// A trunk that moved makes the previews made onto the old one out of date.
|
||||
return ix.RecomputeCandidates(ctx)
|
||||
}
|
||||
|
||||
// movedPackages is whether a held module's package version is the previous head of a repository whose head the new source moved.
|
||||
func movedPackages(held map[string]meshcatalog.ModulesByModuleValue, before, now map[string]string) bool {
|
||||
for _, m := range held {
|
||||
for p, v := range m.Packages {
|
||||
repo, _, ok := trunk.PackageRepository(p)
|
||||
if ok && before[repo] != "" && before[repo] == v && now[repo] != before[repo] {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// kickCatchUp takes the trunk in at a first start, when the objects keep no trunk revision yet (the lead's ruling A on
|
||||
// milestone 47/G: the catch-up path): the head the source names for every repository, by the merged message the forge
|
||||
// bridge gives for it, each applied as its merge, and revision 1 kept from them all at its end. Until then the revision
|
||||
// the tools answer waits, never an empty catalog. The catch-up runs off the source handler, in its own goroutine, said
|
||||
// when it ends (lead-47's ruling R1 on the rebase): a bridge that does not answer holds the kept call (ADR 0668) to its
|
||||
// bound, and must not hold this source revision's stamps and the pulls' candidates with it. A repository whose message is
|
||||
// not read keeps no revision, said: the next source revision runs the catch-up again (no timer). Objects not on the bus
|
||||
// yet run none, said, until then.
|
||||
func (ix *Indexer) kickCatchUp(ctx context.Context, heads map[string]string) error {
|
||||
if ix.Revisions == nil || ix.Message == nil || len(heads) == 0 {
|
||||
return nil
|
||||
}
|
||||
if _, any, err := newestTrunk(ctx, ix.Revisions); errors.Is(err, process.ErrNotYet) {
|
||||
ix.logf("catalog graph: the trunk is not taken in yet: its objects %s are not on the bus", CatalogRevisionsObjects)
|
||||
return nil
|
||||
} else if err != nil || any {
|
||||
return err
|
||||
}
|
||||
if !ix.catchingUp.CompareAndSwap(false, true) {
|
||||
return nil
|
||||
}
|
||||
heads = maps.Clone(heads)
|
||||
ix.catchUpWG.Add(1)
|
||||
go func() {
|
||||
defer ix.catchUpWG.Done()
|
||||
unread := ix.takeInHeads(ctx, heads)
|
||||
ix.catchingUp.Store(false)
|
||||
if ix.behindWaiting.Swap(false) {
|
||||
defer ix.kickBehind(ctx)
|
||||
}
|
||||
if len(unread) > 0 {
|
||||
ix.logf("catalog graph: the first start's catch-up ended and no first revision is kept: %d repositories were not taken in (%s); taken in again at the next source revision",
|
||||
len(unread), strings.Join(unread, ", "))
|
||||
return
|
||||
}
|
||||
if err := ix.KeepView(ctx); err != nil {
|
||||
ix.logf("catalog graph: the first start's catch-up took in the trunk of %d repositories, but its revision could not be kept: %v; taken in again at the next source revision", len(heads), err)
|
||||
return
|
||||
}
|
||||
ix.logf("catalog graph: the first start's catch-up ended: the trunk of %d repositories taken in", len(heads))
|
||||
}()
|
||||
return nil
|
||||
}
|
||||
|
||||
// takeInHeads applies the forge bridge's merged message of each repository's head, as its merge, and answers the
|
||||
// repositories whose message was not read or not applied.
|
||||
func (ix *Indexer) takeInHeads(ctx context.Context, heads map[string]string) []string {
|
||||
var unread []string
|
||||
for _, repository := range slices.Sorted(maps.Keys(heads)) {
|
||||
head := heads[repository]
|
||||
if head == "" {
|
||||
continue
|
||||
}
|
||||
f, err := ix.mergedAt(ctx, repository, head)
|
||||
if err != nil && ctx.Err() == nil && ix.Read != nil {
|
||||
// A head the source names that has no merged message (a branch newly watched, whose head no merge on it made) is
|
||||
// read whole at that commit, as a clone does (A-401, A-93).
|
||||
ix.logf("catalog graph: %s at %.12s has no merged message (%v): read whole at that commit through the seat git, as a merge of every file of its tree", repository, head, err)
|
||||
f, err = ix.wholeAt(ctx, repository, head)
|
||||
}
|
||||
if err == nil {
|
||||
_, err = ix.ApplyMessage(ctx, f)
|
||||
}
|
||||
if err != nil {
|
||||
unread = append(unread, repository)
|
||||
ix.logf("catalog graph: the trunk of %s at %.12s is not taken in: %v", repository, head, err)
|
||||
}
|
||||
}
|
||||
return unread
|
||||
}
|
||||
|
||||
// CatchUp is the indexer's catch-up tool's work (indexer_take_in with no module and no repository, the catch-up its
|
||||
// manifest names beside the catalog graph revisions): after a restore point of them is restored, the node agent calls it
|
||||
// with no arguments, and the indexer takes in the head the gate's source names for every repository, as the first
|
||||
// start's catch-up does, whatever revision the objects keep, then keeps the view as the next revision when an entry
|
||||
// moved. Its answer names the repositories taken in and those whose merged message could not be read.
|
||||
func (ix *Indexer) CatchUp(ctx context.Context) TakenIn {
|
||||
out := TakenIn{Module: "catch-up"}
|
||||
ix.mu.Lock()
|
||||
heads := maps.Clone(ix.source.heads)
|
||||
ix.mu.Unlock()
|
||||
switch {
|
||||
case len(heads) == 0:
|
||||
out.Answer = "the gate's source names no head yet, so nothing is taken in; the next source revision is taken in as it comes"
|
||||
return out
|
||||
case ix.Modules == nil || ix.Message == nil:
|
||||
out.Answer = "the indexer holds no view of the trunk or reads no merged message, so nothing is taken in"
|
||||
return out
|
||||
case !ix.catchingUp.CompareAndSwap(false, true):
|
||||
out.Answer = "a catch-up runs already; its end keeps the revision"
|
||||
return out
|
||||
}
|
||||
unread := ix.takeInHeads(ctx, heads)
|
||||
ix.catchingUp.Store(false)
|
||||
if ix.behindWaiting.Swap(false) {
|
||||
defer ix.kickBehind(ctx)
|
||||
}
|
||||
if err := ix.KeepView(ctx); err != nil {
|
||||
out.Answer = "the repositories were taken in but the revision could not be kept: " + err.Error()
|
||||
return out
|
||||
}
|
||||
out.Stored = len(unread) == 0
|
||||
out.Unread = unread
|
||||
out.Answer = fmt.Sprintf("the heads the gate's source names for %d repositories taken in", len(heads)-len(unread))
|
||||
if len(unread) > 0 {
|
||||
out.Answer += fmt.Sprintf("; %d not (%s): their merged message was not read, and the call may be made again", len(unread), strings.Join(unread, ", "))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// mergedAt is the forge bridge's merged message of a repository's head.
|
||||
func (ix *Indexer) mergedAt(ctx context.Context, repository, head string) (forgemsg.Forge, error) {
|
||||
data, err := ix.Message(forTakeIn(ctx, repository, head), repository, head, 0)
|
||||
if err != nil {
|
||||
return forgemsg.Forge{}, fmt.Errorf("the forge bridge could not answer: %w", err)
|
||||
}
|
||||
f, err := forgemsg.DecodeForge(data)
|
||||
if err != nil {
|
||||
return f, fmt.Errorf("the forge bridge's answer is not a forge message: %w", err)
|
||||
}
|
||||
if f.Kind != forgemsg.ForgeMerged || f.Merge != head || !trunk.SameRepository(f.Repository, repository) {
|
||||
return f, fmt.Errorf("the forge bridge's answer is not the merge of %s at %s", repository, head)
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
|
||||
// wholeAt is the merged message of a head that has none (A-401): the head's tree and files read through the seat git
|
||||
// (Read), every module directory of it with its manifest, lock files and mesh packages, as a merge whose changed files are
|
||||
// every file of that tree, so it is computed as any other head (applyMoving; A-93).
|
||||
func (ix *Indexer) wholeAt(ctx context.Context, repository, head string) (forgemsg.Forge, error) {
|
||||
f := forgemsg.Forge{V: keys.Version, Kind: forgemsg.ForgeMerged, Repository: repository, Merge: head, Head: head}
|
||||
// wholeAt reads repository's trees at the commit head through the seat git (design 47 H §743): every module directory's manifest,
|
||||
// templates' needs, lock files and packages, and the catalogue's predefined list where the repository is the catalogue's (§769).
|
||||
func (ix *Indexer) wholeAt(ctx context.Context, repository, head string) (whole, error) {
|
||||
f := whole{Repository: repository, Merge: head, Head: head}
|
||||
file := func(p string) ([]byte, error) {
|
||||
if ix.ReadFile == nil {
|
||||
return nil, errors.New("the indexer reads no file of the forge")
|
||||
|
||||
@@ -1,167 +0,0 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
|
||||
)
|
||||
|
||||
const otherMerge = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
|
||||
|
||||
// srcT is a source revision's heads as the tests write them; applyS says it as the verb source answers it.
|
||||
type srcT struct {
|
||||
Group string
|
||||
Heads map[string]string
|
||||
}
|
||||
|
||||
func sourceOf(repo, head string) srcT { return srcT{Group: "g", Heads: map[string]string{repo: head}} }
|
||||
|
||||
func applyS(ix *Indexer, ctx context.Context, rev uint64, s srcT) error {
|
||||
a := meshdelivery.SourceAnswer{Branch: "main", Revision: int64(rev)}
|
||||
for r, h := range s.Heads {
|
||||
a.Repositories = append(a.Repositories, meshdelivery.SourceHead{Repository: r, Head: h, Trunk: "main"})
|
||||
}
|
||||
return ix.ApplySource(ctx, a)
|
||||
}
|
||||
|
||||
func TestASourceRevisionStampsTheModulesOfTheRepositoriesHeadItNames(t *testing.T) { // 1344/AC1
|
||||
ix, mods := modulesIndexer(t)
|
||||
ctx := context.Background()
|
||||
if err := ix.Apply(ctx, mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"})); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before, _ := moduleOf(t, mods, "a")
|
||||
if before.Src != 0 {
|
||||
t.Fatalf("module before the source = %+v, want Src 0", before)
|
||||
}
|
||||
// The source names the head merged for the delivery repository, and another repository's head no module of ours is built from.
|
||||
s := srcT{Group: "g", Heads: map[string]string{"novox/mesh-delivery": merge, "novox/mesh-catalog": otherMerge}}
|
||||
if err := applyS(ix, ctx, 7, s); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, ok := moduleOf(t, mods, "a")
|
||||
want := before
|
||||
want.Src, want.FpSrc = 7, 7 // its fingerprint moved at that merge (ADR 0601 §1)
|
||||
if !ok || !reflect.DeepEqual(m, want) {
|
||||
t.Errorf("module a = %+v (%v), want %+v: the fingerprint and the commit as the merge wrote them, with Src 7", m, ok, want)
|
||||
}
|
||||
if c, ok := moduleOf(t, mods, "c"); ok {
|
||||
t.Errorf("module c = %+v: written for a head no merge message of the indexer carried", c)
|
||||
}
|
||||
// The same revision again changes nothing, and so does a later one: Src is the first revision naming the commit.
|
||||
if err := applyS(ix, ctx, 7, s); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := applyS(ix, ctx, 9, s); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m, _ := moduleOf(t, mods, "a"); m.Src != 7 {
|
||||
t.Errorf("module a Src = %d, want 7", m.Src)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAMergeHeardAfterTheSourceNamesItsHeadIsStampedWhenWritten(t *testing.T) { // 1344/AC1 (the source first)
|
||||
// A source revision older than the one held is left (merged here from a test of its own, for the test budget).
|
||||
t.Run("an older source revision is left", func(t *testing.T) {
|
||||
ix, mods := modulesIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", aInputs))
|
||||
ctx := context.Background()
|
||||
_ = ix.Apply(ctx, mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"}))
|
||||
_ = applyS(ix, ctx, 9, sourceOf("novox/mesh-delivery", merge))
|
||||
_ = applyS(ix, ctx, 7, sourceOf("novox/mesh-delivery", merge))
|
||||
if m, _ := moduleOf(t, mods, "a"); m.Src != 9 {
|
||||
t.Errorf("module a Src = %d, want 9", m.Src)
|
||||
}
|
||||
})
|
||||
ix, mods := modulesIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", aInputs))
|
||||
ctx := context.Background()
|
||||
if err := applyS(ix, ctx, 7, sourceOf("novox/mesh-delivery", merge)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, ok := moduleOf(t, mods, "a"); ok {
|
||||
t.Fatal("a module was written from a source revision alone: its tree is the merge message's")
|
||||
}
|
||||
if err := ix.Apply(ctx, mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"})); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m, ok := moduleOf(t, mods, "a"); !ok || m.Src != 7 || m.Commit != merge || m.Path != "modules/a" {
|
||||
t.Errorf("module a = %+v (%v), want the merge's fingerprint with Src 7", m, ok)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAMergeThatNoSourceRevisionNamesIsWrittenAsToday(t *testing.T) { // 1344/AC2
|
||||
ix, mods := modulesIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", aInputs))
|
||||
ctx := context.Background()
|
||||
// No source revision yet.
|
||||
if err := ix.Apply(ctx, mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"})); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m, ok := moduleOf(t, mods, "a"); !ok || m.Src != 0 || m.Path != "modules/a" {
|
||||
t.Fatalf("module a = %+v (%v), want the merge written with Src 0", m, ok)
|
||||
}
|
||||
// A source revision that names another head of the repository: a merge of the head it does not name keeps Src 0.
|
||||
if err := applyS(ix, ctx, 7, sourceOf("novox/mesh-delivery", otherMerge)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m, _ := moduleOf(t, mods, "a"); m.Src != 0 {
|
||||
t.Errorf("module a Src = %d after a source naming another head, want 0", m.Src)
|
||||
}
|
||||
merged := mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newertree"})
|
||||
merged.Merge = "cccccccccccccccccccccccccccccccccccccccc"
|
||||
if err := ix.Apply(ctx, merged); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m, _ := moduleOf(t, mods, "a"); m.Src != 0 || m.Commit != merged.Merge {
|
||||
t.Errorf("module a = %+v, want the new merge with Src 0", m)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAModuleBuiltAgainstASourceHeadIsComputedAgainWhenThatHeadMoves(t *testing.T) { // design 47 G A-365
|
||||
held := map[string]meshcatalog.ModulesByModuleValue{"a": {Packages: map[string]string{"git.novox.be/novox/mesh-contracts/go/seats/findings": otherMerge}}}
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
before, now map[string]string
|
||||
want bool
|
||||
}{
|
||||
{"the head its version came from moved", map[string]string{"novox/mesh-contracts": otherMerge}, map[string]string{"novox/mesh-contracts": merge}, true},
|
||||
{"the head stands", map[string]string{"novox/mesh-contracts": otherMerge}, map[string]string{"novox/mesh-contracts": otherMerge}, false},
|
||||
{"another repository's head moved", map[string]string{"novox/mesh-contracts": otherMerge}, map[string]string{"novox/mesh-contracts": otherMerge, "novox/mesh-sdk": merge}, false},
|
||||
{"no source seen before", nil, map[string]string{"novox/mesh-contracts": merge}, false},
|
||||
} {
|
||||
if got := movedPackages(held, row.before, row.now); got != row.want {
|
||||
t.Errorf("%s: movedPackages = %v, want %v", row.name, got, row.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 G A-365: through ApplySource, a held module whose package no module publishes was built against the head the source
|
||||
// named; when the source moves that repository's head, the module's fingerprint is computed again against the new head.
|
||||
func TestApplySourceComputesAgainTheModuleBuiltAgainstTheHeadThatMoved(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
const pkg = "git.novox.be/novox/mesh-contracts/go/seats/findings"
|
||||
ix, mods := newFormIndexer(t, mod{"avahi", catalogueManifest(t, "avahi")})
|
||||
reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1")
|
||||
ix.Packages = newMem()
|
||||
setPackages(t, mods, "avahi", []string{pkg})
|
||||
h1, h2 := strings.Repeat("1", 40), strings.Repeat("2", 40)
|
||||
if err := applyS(ix, ctx, 1, sourceOf("novox/mesh-contracts", h1)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ix.Recompute(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
first, _ := moduleOf(t, mods, "avahi")
|
||||
if first.Fp2 == "" || first.Packages[pkg] != h1 {
|
||||
t.Fatalf("before the head moves: fp2 %q none2 %q packages %v, want a fingerprint built against %s", first.Fp2, first.None2, first.Packages, h1)
|
||||
}
|
||||
if err := applyS(ix, ctx, 2, sourceOf("novox/mesh-contracts", h2)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
moved, _ := moduleOf(t, mods, "avahi")
|
||||
if moved.Packages[pkg] != h2 || moved.Fp2 == first.Fp2 {
|
||||
t.Errorf("after the head moves: fp2 %q (was %q) packages %v, want it computed again against %s", moved.Fp2, first.Fp2, moved.Packages, h2)
|
||||
}
|
||||
}
|
||||
@@ -10,7 +10,6 @@ import (
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
)
|
||||
@@ -22,13 +21,13 @@ import (
|
||||
// say keeps the key as it is and says so.
|
||||
|
||||
// writeTaken writes the repository's taken key for the merge f carries, unless it names that merge or a later one.
|
||||
func (ix *Indexer) writeTaken(ctx context.Context, f forgemsg.Forge) error {
|
||||
func (ix *Indexer) writeTaken(ctx context.Context, f whole) error {
|
||||
return ix.writeTakenKnown(ctx, f, "")
|
||||
}
|
||||
|
||||
// writeTakenKnown is writeTaken where the merge is known not to be older than notOlder (lateMerge asked it, issue 1126): a
|
||||
// key naming that merge is written over without asking the forge again.
|
||||
func (ix *Indexer) writeTakenKnown(ctx context.Context, f forgemsg.Forge, notOlder string) error {
|
||||
func (ix *Indexer) writeTakenKnown(ctx context.Context, f whole, notOlder string) error {
|
||||
if ix.Taken == nil || f.Merge == "" {
|
||||
return nil
|
||||
}
|
||||
@@ -104,7 +103,7 @@ func (ix *Indexer) FillTaken(ctx context.Context) error {
|
||||
}
|
||||
}
|
||||
for _, repository := range slices.Sorted(maps.Keys(pending)) {
|
||||
err := ix.writeTaken(ctx, forgemsg.Forge{Repository: repository, Merge: pending[repository]})
|
||||
err := ix.writeTaken(ctx, whole{Repository: repository, Merge: pending[repository]})
|
||||
if err == nil {
|
||||
ix.mu.Lock()
|
||||
if ix.untaken[repository] == pending[repository] {
|
||||
|
||||
@@ -78,13 +78,9 @@ func (ix *Indexer) ApplyToolchain(ctx context.Context, module, key string, value
|
||||
return ix.ToolchainReported(ctx)
|
||||
}
|
||||
|
||||
// ToolchainReported is what a changed toolchain moves: the modules written again (Recompute), then pending evaluated
|
||||
// again (PendingAgain), since the indexer build a pending revision names is a fingerprint the toolchain is part of.
|
||||
// ToolchainReported is what a changed toolchain moves: the modules written again (Recompute).
|
||||
func (ix *Indexer) ToolchainReported(ctx context.Context) error {
|
||||
if err := ix.Recompute(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
return ix.PendingAgain(ctx)
|
||||
return ix.Recompute(ctx)
|
||||
}
|
||||
|
||||
// agreed is the toolchain of each language every reporting node agrees on, or why there is none.
|
||||
@@ -144,7 +140,7 @@ func (ix *Indexer) recomputeWith(ctx context.Context, overlay map[string]meshcat
|
||||
return err
|
||||
}
|
||||
// What the modules moved to reaches the candidates' fingerprints too (a base, the builder, a toolchain).
|
||||
return ix.RecomputeCandidates(ctx)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]meshcatalog.ModulesByModuleValue, w *mergeWrite, repository string) error {
|
||||
|
||||
@@ -10,7 +10,6 @@ import (
|
||||
"reflect"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -19,10 +18,8 @@ import (
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"git.novox.be/novox/mesh-sdk/go/seatdef"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
@@ -105,7 +102,7 @@ func newFormIndexer(t *testing.T, mods ...mod) (*Indexer, *memStore) {
|
||||
}
|
||||
|
||||
// newFormIndexerUnmerged is the indexer and the catalogue merge of these modules, not yet applied.
|
||||
func newFormIndexerUnmerged(t *testing.T, mods ...mod) (*Indexer, *memStore, forgemsg.Forge) {
|
||||
func newFormIndexerUnmerged(t *testing.T, mods ...mod) (*Indexer, *memStore, whole) {
|
||||
t.Helper()
|
||||
var recs [][2]any
|
||||
trees, manifests := map[string]string{}, map[string]string{}
|
||||
@@ -154,15 +151,6 @@ func TestAgreedToolchainChangesAGoModulesReasonWithNoMerge(t *testing.T) { // 14
|
||||
t.Run("a recipe change builds a Go module at once and again once the agreement moves", aRecipeChangeBuildsAGoModuleAtOnceAndAgainOnceTheAgreementMoves)
|
||||
t.Fatalf("before: none2 = %q", m.None2)
|
||||
}
|
||||
// The release that last merged avahi: its source revision names avahi's commit, so Src (and FPSrc) are 7.
|
||||
merged, _ := moduleOf(t, mods, "avahi")
|
||||
src := srcT{Group: "g", Heads: map[string]string{merged.Repository: merged.Commit}}
|
||||
if err := applyS(ix, context.Background(), 7, src); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m, _ := moduleOf(t, mods, "avahi"); m.Src != 7 || m.FpSrc != 7 {
|
||||
t.Fatalf("before: src %d fp-src %d, want 7 and 7", m.Src, m.FpSrc)
|
||||
}
|
||||
if g := graphOf(t, ix, "avahi"); g.None != sdkstep.ToolchainNoReport || g.Toolchains != nil {
|
||||
t.Errorf("before: graph.avahi = %+v, want no toolchain edge and the reason", g)
|
||||
}
|
||||
@@ -185,16 +173,10 @@ func TestAgreedToolchainChangesAGoModulesReasonWithNoMerge(t *testing.T) { // 14
|
||||
t.Errorf("none2 = %q fp2 = %q, want a fingerprint: no builder input is waited for (ADR 0530 §1)", m.None2, m.Fp2)
|
||||
}
|
||||
// ADR 0601 §1 (review round 2): a fingerprint moved with no merge has no release that carries it: fp-src 0, not known,
|
||||
// though the source revision is still the release's at the module's own Src; a source revision heard again stamps nothing.
|
||||
// though the source revision is still the release's at the module's own Src.
|
||||
if m.FpSrc != 0 {
|
||||
t.Errorf("a fingerprint moved by a recompute: fp-src %d, want 0", m.FpSrc)
|
||||
}
|
||||
if err := applyS(ix, context.Background(), 8, src); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if again, _ := moduleOf(t, mods, "avahi"); again.FpSrc != 0 {
|
||||
t.Errorf("a source revision stamped the recomputed fingerprint: fp-src %d", again.FpSrc)
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 *A module's registration* ("No part fixes a name") and *What ADR 0656 leaves to this design* (the indexer reads
|
||||
@@ -517,42 +499,16 @@ func TestBasesAreComputedFirstAndNoFingerprintTakesTheBuilder(t *testing.T) { //
|
||||
firstSaid = append(firstSaid, e)
|
||||
return nil
|
||||
}
|
||||
// Lead-47's ruling R1: the catch-up runs off the source handler, so a bridge that has not answered yet holds no source
|
||||
// revision (the first message is answered only once ApplySource returned; held in the handler it is answered as failed).
|
||||
answers, applied := 0, make(chan struct{})
|
||||
first.Message = func(_ context.Context, repository, head string, _ int) ([]byte, error) {
|
||||
if answers++; answers == 1 {
|
||||
select {
|
||||
case <-applied:
|
||||
return nil, errors.New("the forge bridge does not answer")
|
||||
case <-time.After(10 * time.Second):
|
||||
return nil, errors.New("the catch-up held the source handler")
|
||||
}
|
||||
}
|
||||
if repository != catalogueRepo || head != merged.Merge {
|
||||
return nil, fmt.Errorf("no merge of %s at %s", repository, head)
|
||||
}
|
||||
return json.Marshal(merged)
|
||||
// The take-in of the head the source names keeps revision 1 with every module (design 47 H §743).
|
||||
if err := applyS(first, context.Background(), 6, sourceOf(catalogueRepo, merged.Merge)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
heads := sourceOf(catalogueRepo, merged.Merge)
|
||||
var firstLog []string
|
||||
var logMu sync.Mutex
|
||||
first.Logf = func(f string, a ...any) {
|
||||
logMu.Lock()
|
||||
firstLog = append(firstLog, fmt.Sprintf(f, a...))
|
||||
logMu.Unlock()
|
||||
if err := first.Apply(context.Background(), merged); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err = applyS(first, context.Background(), 5, heads)
|
||||
close(applied)
|
||||
first.catchUpWG.Wait()
|
||||
if err != nil || len(firstSaid) != 0 || !strings.Contains(strings.Join(firstLog, "\n"), "the forge bridge does not answer") {
|
||||
t.Fatalf("a catch-up whose message is not read said %+v (%v), log %q, want no revision and the bridge's silence said", firstSaid, err, firstLog)
|
||||
}
|
||||
err = applyS(first, context.Background(), 6, heads)
|
||||
first.catchUpWG.Wait()
|
||||
if err != nil || len(firstSaid) != 1 || firstSaid[0].Revision != 1 ||
|
||||
firstSaid[0].Entries != int64(len(builderModules(t))) || firstSaid[0].Source != 6 {
|
||||
t.Fatalf("the first start's catch-up said %+v (%v), want trunk revision 1 of source 6 with every module", firstSaid, err)
|
||||
if err := first.KeepView(context.Background()); err != nil || len(firstSaid) != 1 || firstSaid[0].Revision != 1 ||
|
||||
firstSaid[0].Entries != int64(len(builderModules(t))) {
|
||||
t.Fatalf("the take-in said %+v (%v), want trunk revision 1 with every module", firstSaid, err)
|
||||
}
|
||||
// KeepView: a write to the view outside a recompute (here a verdict carried into one module) is the next revision.
|
||||
viewKey, _ := meshcatalogclient.ModulesKey("avahi")
|
||||
@@ -570,24 +526,6 @@ func TestBasesAreComputedFirstAndNoFingerprintTakesTheBuilder(t *testing.T) { //
|
||||
} else if raw, _, _, _ := view.Get(context.Background(), viewKey); !strings.Contains(string(raw), "a field the judge refuses") {
|
||||
t.Errorf("the view at a restart lost the verdict: %s", raw)
|
||||
}
|
||||
// W4.14: the catch-up tool (indexer_take_in with no module and no repository, as the node agent calls it after a
|
||||
// restore) takes in the head the source names for every repository again, whatever revision is kept, and keeps none
|
||||
// when nothing moved; a repository whose message is not read is named in unread.
|
||||
if key, value, err := NewCatchUp("", "node-engine@ace", time.Now()); err != nil || key != CatchUpKey {
|
||||
t.Fatalf("NewCatchUp = %q (%v), want key %q", key, err, CatchUpKey)
|
||||
} else if a, err := DecodeTakeIn(value); err != nil || !a.CatchUp {
|
||||
t.Fatalf("a catch-up ask decodes to %+v (%v), want a catch-up", a, err)
|
||||
}
|
||||
asked := answers
|
||||
if got := first.CatchUp(context.Background()); !got.Stored || answers != asked+1 || len(firstSaid) != 2 || len(got.Unread) != 0 {
|
||||
t.Errorf("the catch-up answered %+v after %d asks, said %d revisions, want the one repository taken in, no new revision", got, answers-asked, len(firstSaid))
|
||||
}
|
||||
first.Message = func(context.Context, string, string, int) ([]byte, error) {
|
||||
return nil, errors.New("the forge bridge does not answer")
|
||||
}
|
||||
if got := first.CatchUp(context.Background()); got.Stored || !reflect.DeepEqual(got.Unread, []string{catalogueRepo}) {
|
||||
t.Errorf("a catch-up whose message is not read answered %+v, want %s unread", got, catalogueRepo)
|
||||
}
|
||||
// The tools bundle answers modules and graph from the newest trunk revision, their marks at its number; with none kept
|
||||
// it says it waits (view-waits), never an empty catalog.
|
||||
trunkStates := TrunkStates{Objects: first.Revisions, Under: func(string) views.Source { return nil }}
|
||||
@@ -824,7 +762,7 @@ func TestAModuleKeepsTheToolchainsItsFingerprintWasComputedWith(t *testing.T) {
|
||||
tools, _ := moduleOf(t, mods, "mesh-tools")
|
||||
sdkMerge := func(commit, tree string) meshcatalog.ModulesByModuleValue {
|
||||
t.Helper()
|
||||
f := forgemsg.Forge{V: keys.Version, Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-sdk", Branch: "feat/x", Number: 3, Head: h1,
|
||||
f := whole{Repository: "novox/mesh-sdk", Branch: "feat/x", Head: h1,
|
||||
Merge: commit, Root: tree, Trees: map[string]string{"": tree}, Manifests: map[string]string{"": `{"module":"mesh-sdk","version":"1.0"}`}}
|
||||
if err := ix.Apply(context.Background(), f); err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -1,67 +0,0 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
|
||||
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
)
|
||||
|
||||
// **A merged message unread past five minutes, the forge heard, is a finding** (issue 1966 note 153; novox/hq ADR 0668 §3).
|
||||
// The indexer's call for a merged message is a kept call: a transport fault leaves the merge unread (Pending.Unread) and
|
||||
// it is read again after every message heard on the forge (UnreadAgain). A message heard on the forge is the forge bridge
|
||||
// heard, the callee of that call: so after each one, a merge still unread since meshconditionsclient.CallUnansweredAfter
|
||||
// or longer is the finding call-unanswered in the indexer's findings, which the condition keeper raises, and a merge no
|
||||
// longer unread (read, taken in, covered or dropped) clears its finding. While nothing is heard on the forge (the bridge's
|
||||
// machine away) no finding is written: the merge stays unread and is read again when the forge is heard. No timer.
|
||||
|
||||
// UnreadCallee is the call the finding names: the forge bridge's tool for one merged message.
|
||||
const UnreadCallee = ForgeBridgeModule + ".bridge_merged_message"
|
||||
|
||||
// unreadCallID is the finding's id for the pending key: the merge it waits on.
|
||||
func unreadCallID(key string) string { return "merged-message." + key }
|
||||
|
||||
// unreadFindings raises the finding of every merge of entries unread past the bound and clears the finding of every other
|
||||
// that the finding writer holds (design 47 G §689, A-29).
|
||||
func (ix *Indexer) unreadFindings(ctx context.Context, entries []PendingEntry) {
|
||||
if ix.Findings == nil {
|
||||
return
|
||||
}
|
||||
unread := map[string]bool{}
|
||||
for _, e := range entries {
|
||||
if e.Value.Unread == "" {
|
||||
continue
|
||||
}
|
||||
id := keyID(unreadCallID(e.Key))
|
||||
unread[id] = true
|
||||
since := e.Value.UnreadSince
|
||||
if since.IsZero() || ix.now().Sub(since) < mcc.CallUnansweredAfter {
|
||||
continue
|
||||
}
|
||||
f := ix.observed(mcc.ScopeCall, unreadCallID(e.Key), mcc.KindCallUnanswered,
|
||||
fmt.Sprintf("the call %s about %s is unanswered since %s", UnreadCallee, unreadCallID(e.Key), since.UTC().Format(time.RFC3339)),
|
||||
"A call whose answer is kept is not answered",
|
||||
fmt.Sprintf("The call %s is made again on every transport fault and on the callee's event; it has not been answered since %s. Its last fault: %s.",
|
||||
UnreadCallee, since.UTC().Format(time.RFC3339), e.Value.Unread), "indexer."+mcc.KindCallUnanswered)
|
||||
f.Needs = TakeInPendingTool + " reads the merge again now; " + PendingTool + " shows it unread."
|
||||
f.Evidence = []mcseat.Evidence{{At: since.UTC(), Said: e.Value.Unread}}
|
||||
if err := ix.Findings.Write(ctx, f); err != nil {
|
||||
ix.logf("take-in: the finding %s is not written: %v", views.FindingKey(f.Scope, f.ID, f.Kind), err)
|
||||
} else {
|
||||
ix.logf("take-in: %s (%s#%d) is unread since %s and the forge is heard: the finding %s", e.Key, e.Forge.Repository, e.Forge.Number, since.Format("2006-01-02T15:04:05Z07:00"), views.FindingKey(f.Scope, f.ID, f.Kind))
|
||||
}
|
||||
}
|
||||
prefix := keyID(strings.TrimSuffix(unreadCallID(""), ".")) + "."
|
||||
for _, f := range ix.Findings.Held() {
|
||||
if f.Scope != mcc.ScopeCall || f.Kind != mcc.KindCallUnanswered || !strings.HasPrefix(f.ID, prefix) || unread[f.ID] {
|
||||
continue
|
||||
}
|
||||
if err := ix.Findings.Clear(ctx, f.Scope, f.ID, f.Kind, "the merge is no longer unread"); err != nil {
|
||||
ix.logf("take-in: the finding %s is not cleared: %v", views.FindingKey(f.Scope, f.ID, f.Kind), err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -17,10 +17,9 @@ import (
|
||||
)
|
||||
|
||||
// What outdated compares against is kept as heard, in the indexer's state heard (design 47 G A-68): toolchain.<language> the newest
|
||||
// toolchain-agreed per language, delivered.<module> the newest version-delivered per module.
|
||||
// toolchain-agreed per language.
|
||||
const (
|
||||
ToolchainPrefix = "toolchain."
|
||||
DeliveredPrefix = "delivered."
|
||||
// ActedPrefix opens acted.<acting part>.<dependent>, the newest wait-ended of an acting part's wait for a dependent, ran or released
|
||||
// (A-78 a), each key part escaped as a name (A-80, §73).
|
||||
ActedPrefix = "acted."
|
||||
@@ -100,66 +99,10 @@ func (ix *Indexer) HearToolchainAgreed(ctx context.Context, t meshbuilds.Toolcha
|
||||
return ix.keepNewest(ctx, ToolchainPrefix+keys.EscapeName(t.Language), t, func([]byte) bool { return true })
|
||||
}
|
||||
|
||||
// HearVersionDelivered takes one version-delivered (design 47 G §374, §376, A-68): the entries of kinds 2, 4 and 6 of the
|
||||
// delivered release, which read "not yet delivered", are written once with each dependency's newest delivered version and
|
||||
// part digest; then the delivered version is kept as the module's newest.
|
||||
func (ix *Indexer) HearVersionDelivered(ctx context.Context, d meshwants.VersionDeliveredEvent) error {
|
||||
if ix.Versions == nil || ix.Heard == nil {
|
||||
return errors.New("the state versions is not kept here")
|
||||
}
|
||||
if _, ok := versionParts(d.Version); !ok {
|
||||
return fmt.Errorf("the version-delivered of %s names the version %q, which is no MAJOR.MINOR.PATCH", d.Module, d.Version)
|
||||
}
|
||||
key := meshcatalog.VersionsByModuleVersionKey(d.Module, d.Version)
|
||||
err := write(ctx, ix.Versions, key, func(raw []byte, found bool) ([]byte, bool, error) {
|
||||
if !found {
|
||||
return nil, false, nil // a version the catalog does not hold (a build of another branch) has no entries to write
|
||||
}
|
||||
v, err := meshcatalog.DecodeVersionsByModuleVersionValue(raw)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
changed := false
|
||||
for i, e := range v.Edges {
|
||||
if e.Version != notYetDelivered {
|
||||
continue
|
||||
}
|
||||
dep, ok := ix.deliveredOf(ctx, strings.SplitN(e.Part, ":", 2)[0])
|
||||
if !ok {
|
||||
continue // the dependency is not delivered yet: the entry reads so until this release's own delivery finds it
|
||||
}
|
||||
v.Edges[i].Version, v.Edges[i].Digest = dep.Version, dep.Parts[e.Part]
|
||||
changed = true
|
||||
}
|
||||
if !changed {
|
||||
return nil, false, nil
|
||||
}
|
||||
data, err := json.Marshal(v)
|
||||
return data, false, err
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return ix.keepNewest(ctx, DeliveredPrefix+keys.EscapeName(d.Module), d, func(kept []byte) bool {
|
||||
var k meshwants.VersionDeliveredEvent
|
||||
return json.Unmarshal(kept, &k) != nil || compareVersions(d.Version, k.Version) >= 0
|
||||
})
|
||||
}
|
||||
|
||||
// deliveredOf is the newest version-delivered kept of a module.
|
||||
func (ix *Indexer) deliveredOf(ctx context.Context, module string) (meshwants.VersionDeliveredEvent, bool) {
|
||||
var d meshwants.VersionDeliveredEvent
|
||||
raw, _, found, err := ix.Heard.Get(ctx, DeliveredPrefix+keys.EscapeName(module))
|
||||
if err != nil || !found || json.Unmarshal(raw, &d) != nil {
|
||||
return d, false
|
||||
}
|
||||
return d, true
|
||||
}
|
||||
|
||||
// Outdated answers the verb outdated (design 47 G §375) for every edge kind: every entry of each module's newest released
|
||||
// version that is behind. Kind 1: the recorded digest differs from the part's digest in the dependency's newest released
|
||||
// version. Kind 3: the recorded toolchain version differs from the version of the newest toolchain-agreed of the dependent
|
||||
// artifact's language (A-80, toolchain.<language>). Kinds 2 and 4: from the dependency's newest version-delivered. Kind 5: the
|
||||
// artifact's language (A-80, toolchain.<language>). Kinds 2, 4 and 6 are declarations, compared with nothing (design 47 H H-A-82). Kind 5: the
|
||||
// acting part has not acted for the dependent's current build (acted.<acting part>.<dependent> against the newest trunk
|
||||
// entry's fingerprint, A-78 a). Kind 6: the delivered version's catalog entry no longer serves the verb or word (A-78 b). A
|
||||
// stale edge (kinds 1, 3) has the act rebuild, an ordering edge is shown and starts nothing. A dependency's newer version that
|
||||
@@ -235,16 +178,6 @@ func (ix *Indexer) Outdated(ctx context.Context, in meshcatalog.OutdatedInput) (
|
||||
continue
|
||||
}
|
||||
o.Recorded, o.Newest = e.Version+" "+e.Digest, tc.Version
|
||||
case 2, 4:
|
||||
d, ok := ix.deliveredOf(ctx, dep)
|
||||
if !ok || e.Version == notYetDelivered {
|
||||
continue
|
||||
}
|
||||
digest := d.Parts[e.Part]
|
||||
if digest == "" || digest == e.Digest {
|
||||
continue
|
||||
}
|
||||
o.Recorded, o.Newest = e.Version+" "+e.Digest, d.Version+" "+digest
|
||||
case 5:
|
||||
cur := trunk[m].Fp
|
||||
if cur == "" {
|
||||
@@ -255,15 +188,6 @@ func (ix *Indexer) Outdated(ctx context.Context, in meshcatalog.OutdatedInput) (
|
||||
continue
|
||||
}
|
||||
o.Recorded, o.Newest = "acted for "+cmp.Or(acted.Build, "no build"), "the current build "+cur
|
||||
case 6:
|
||||
d, ok := ix.deliveredOf(ctx, dep)
|
||||
if !ok || e.Version == notYetDelivered || d.Version == e.Version {
|
||||
continue
|
||||
}
|
||||
if serves, known := ix.deliveredServes(ctx, dep, d.Version, e.Word); !known || serves {
|
||||
continue
|
||||
}
|
||||
o.Recorded, o.Newest = e.Version+" "+e.Word, d.Version+" no longer serves "+e.Word
|
||||
default:
|
||||
continue
|
||||
}
|
||||
@@ -316,53 +240,6 @@ func languageOf(entry meshcatalog.CatalogEntry, e meshcatalog.VersionEdge) strin
|
||||
return one
|
||||
}
|
||||
|
||||
// deliveredServes says whether the catalog entry of a delivered version of module serves word, its verb (seat.verb, as the entry's
|
||||
// serves lists it, the form `seat:` allowed) or a word of its manifest (A-78 b); known is false where that version's entry cannot be read.
|
||||
func (ix *Indexer) deliveredServes(ctx context.Context, module, version, word string) (serves, known bool) {
|
||||
if ix.Revisions == nil {
|
||||
return false, false
|
||||
}
|
||||
vs, err := ModuleVersions(ctx, ix.Versions, module)
|
||||
if err != nil {
|
||||
return false, false
|
||||
}
|
||||
var source int64
|
||||
for _, v := range vs {
|
||||
if v.Version == version {
|
||||
source = v.Source
|
||||
}
|
||||
}
|
||||
if source == 0 {
|
||||
return false, false
|
||||
}
|
||||
names, err := ix.Revisions.Names(ctx)
|
||||
if err != nil {
|
||||
return false, false
|
||||
}
|
||||
var numbers []int64
|
||||
for _, n := range names {
|
||||
if r, ok := TrunkRevisionOf(n); ok {
|
||||
numbers = append(numbers, r)
|
||||
}
|
||||
}
|
||||
slices.SortFunc(numbers, func(a, b int64) int { return cmp.Compare(b, a) })
|
||||
for _, n := range numbers { // the newest revision taken from the source revision the version was built at
|
||||
rev, entries, err := ReadRevision(ctx, ix.Revisions, fmt.Sprintf("trunk.%d", n))
|
||||
if err != nil || rev.Source != source {
|
||||
continue
|
||||
}
|
||||
for _, e := range entries {
|
||||
if e.Module != module {
|
||||
continue
|
||||
}
|
||||
w := strings.TrimPrefix(word, "seat:")
|
||||
_, inManifest := e.Manifest[word]
|
||||
return slices.Contains(e.Serves, w) || slices.Contains(e.Provides, w) || slices.Contains(e.Streams, w) || inManifest, true
|
||||
}
|
||||
}
|
||||
return false, false
|
||||
}
|
||||
|
||||
// firstBranch is the first watched branch of a module's registration, which its released versions are of.
|
||||
func (ix *Indexer) firstBranch(ctx context.Context, module string) string {
|
||||
if ix.Registry != nil {
|
||||
|
||||
@@ -10,7 +10,6 @@ import (
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
@@ -153,19 +152,11 @@ func (ix *Indexer) FillVersions(ctx context.Context, ask VersionsAsker) error {
|
||||
}
|
||||
built[module] = v
|
||||
}
|
||||
// The edge entries (§377), each marked filled: where the stream running or the placement cannot be read the entries of kinds 2, 4
|
||||
// and 6 read "not yet delivered", as at a release, and the first delivery writes them.
|
||||
var running map[string]map[string]nodeengine.Running
|
||||
if ix.RunningWhole != nil {
|
||||
if running, err = ix.RunningWhole(ctx); err != nil {
|
||||
ix.logf("versions fill: the stream running is not read whole: %v", err)
|
||||
running = nil
|
||||
}
|
||||
}
|
||||
// The edge entries (§377), each marked filled; those of kinds 2, 4 and 6 read "not yet delivered", as at a release.
|
||||
answers := map[string]BuilderVersionsAnswer{}
|
||||
for _, module := range slices.Sorted(maps.Keys(built)) {
|
||||
v := built[module]
|
||||
v.Edges = ix.filledEdges(ctx, trunk[module], claimed[module], ask, trunk, built, running, answers)
|
||||
v.Edges = ix.filledEdges(ctx, trunk[module], claimed[module], ask, trunk, built, answers)
|
||||
built[module] = v
|
||||
}
|
||||
for _, module := range slices.Sorted(maps.Keys(built)) {
|
||||
|
||||
@@ -8,28 +8,17 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
)
|
||||
|
||||
// The edge entries of the first fill (design 47 G §377, A-78 c): each module's newest version gets its entries once, each marked
|
||||
// filled. Kinds 1 and 3 come from the dependency version published latest before the version's build was claimed, read with
|
||||
// the times of builder_versions, and its part's digest; kinds 2, 4 and 6 from the version every machine placing the dependency
|
||||
// runs, present or away, the machines from the assigner's placement (A-62) and each node's last running from the node-engine seat's
|
||||
// stream read whole (§556); kind 5 as done for every acting part's running build.
|
||||
|
||||
// PlacementAsker asks the seat mesh-module-assigner's verb placement for a module (a read, A-62).
|
||||
type PlacementAsker func(ctx context.Context, module string) ([]meshmoduleassigner.ModulePlacement, error)
|
||||
|
||||
// RunningReader reads the node-engine seat's stream running whole, every subject's last message (§556): by node, then module.
|
||||
type RunningReader func(ctx context.Context) (map[string]map[string]nodeengine.Running, error)
|
||||
// the times of builder_versions, and its part's digest; kinds 2, 4 and 6 are declarations and read "not yet delivered" (design 47 H
|
||||
// H-A-82: no delivered version is kept); kind 5 as done for every acting part's running build.
|
||||
|
||||
// filledEdges are the edge entries of the entry's newest version, which was filled as built; claimed is when its build was claimed.
|
||||
func (ix *Indexer) filledEdges(ctx context.Context, entry meshcatalog.CatalogEntry, claimed time.Time, ask VersionsAsker, trunk map[string]meshcatalog.CatalogEntry,
|
||||
built map[string]meshcatalog.VersionEntry, running map[string]map[string]nodeengine.Running, answers map[string]BuilderVersionsAnswer) []meshcatalog.VersionEdge {
|
||||
built map[string]meshcatalog.VersionEntry, answers map[string]BuilderVersionsAnswer) []meshcatalog.VersionEdge {
|
||||
var out []meshcatalog.VersionEdge
|
||||
seen := func(e meshcatalog.VersionEdge) bool {
|
||||
return slices.ContainsFunc(out, func(x meshcatalog.VersionEdge) bool { return x.Kind == e.Kind && x.Part == e.Part && x.Word == e.Word })
|
||||
@@ -78,15 +67,7 @@ func (ix *Indexer) filledEdges(ctx context.Context, entry meshcatalog.CatalogEnt
|
||||
}
|
||||
e.Version, e.Digest, e.Stale = best.Version, best.Digest, true
|
||||
case 2, 4, 6:
|
||||
e.Version = notYetDelivered
|
||||
if v, ok := ix.runningVersion(ctx, module, trunk, built, running); ok {
|
||||
e.Version = v.Version
|
||||
for _, a := range v.Artifacts {
|
||||
if a.Artifact == part {
|
||||
e.Digest = a.Digest
|
||||
}
|
||||
}
|
||||
}
|
||||
e.Version = notYetDelivered // a declaration: no delivered version is kept for it (design 47 H H-A-82)
|
||||
case 5:
|
||||
e.Fact = "acted for the running build " + entry.Fp
|
||||
default:
|
||||
@@ -99,51 +80,10 @@ func (ix *Indexer) filledEdges(ctx context.Context, entry meshcatalog.CatalogEnt
|
||||
return out
|
||||
}
|
||||
|
||||
// runningVersion is the version of a module that every machine placing it runs, present or away, when each runs the newest
|
||||
// build; false where no machine places it, a placed machine runs none or another build, or the placement is not answered.
|
||||
func (ix *Indexer) runningVersion(ctx context.Context, module string, trunk map[string]meshcatalog.CatalogEntry, built map[string]meshcatalog.VersionEntry,
|
||||
running map[string]map[string]nodeengine.Running) (meshcatalog.VersionEntry, bool) {
|
||||
v, ok := built[module]
|
||||
e, known := trunk[module]
|
||||
if !ok || !known || e.Fp == "" || ix.Placement == nil {
|
||||
return v, false
|
||||
}
|
||||
placements, err := ix.Placement(ctx, module)
|
||||
if err != nil {
|
||||
ix.logf("versions fill: the placement of %s is not answered: %v", module, err)
|
||||
return v, false
|
||||
}
|
||||
nodes := 0
|
||||
for _, p := range placements {
|
||||
for _, n := range p.Nodes {
|
||||
nodes++
|
||||
r, ok := running[n][module]
|
||||
if !ok || r.Stopped || r.Fingerprint != e.Fp {
|
||||
return v, false
|
||||
}
|
||||
}
|
||||
}
|
||||
return v, nodes > 0
|
||||
}
|
||||
|
||||
// keepFilledFacts keeps what the first fill derived of a version as heard: for each entry of kinds 2, 4 and 6 read as delivered,
|
||||
// the dependency's version as its newest delivered one (A-68), and for each kind-5 edge the act for the running build (A-78 a).
|
||||
// keepFilledFacts keeps what the first fill derived of a version as heard: for each kind-5 edge the act for the running build (A-78 a).
|
||||
func (ix *Indexer) keepFilledFacts(ctx context.Context, v meshcatalog.VersionEntry, entry meshcatalog.CatalogEntry, built map[string]meshcatalog.VersionEntry, at time.Time) {
|
||||
for _, e := range v.Edges {
|
||||
switch e.Kind {
|
||||
case 2, 4, 6:
|
||||
dep := built[strings.SplitN(e.Part, ":", 2)[0]]
|
||||
if e.Version == notYetDelivered || dep.Version != e.Version {
|
||||
continue
|
||||
}
|
||||
parts := map[string]string{}
|
||||
for _, a := range dep.Artifacts {
|
||||
parts[a.Artifact] = a.Digest
|
||||
}
|
||||
d := meshwants.VersionDeliveredEvent{V: 1, Module: dep.Module, Version: dep.Version, Parts: parts, At: &at}
|
||||
if err := ix.keepNewest(ctx, DeliveredPrefix+keys.EscapeName(dep.Module), d, func([]byte) bool { return false }); err != nil {
|
||||
ix.logf("versions fill: the delivered version of %s is not kept: %v", dep.Module, err)
|
||||
}
|
||||
case 5:
|
||||
data, _ := json.Marshal(Acted{V: 1, Part: e.Part, Build: entry.Fp, How: "filled", At: at})
|
||||
if err := write(ctx, ix.Heard, ActedKey(e.Part, v.Module), func(_ []byte, found bool) ([]byte, bool, error) {
|
||||
|
||||
@@ -10,8 +10,6 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
@@ -219,28 +217,13 @@ func TestOutdatedAnswersTheEdgesBehindByTheirPartsDigest(t *testing.T) {
|
||||
if b := behind(); len(b) != 1 || b[0].Module != "app" || b[0].Kind != 1 || b[0].Act != "rebuild" || b[0].Dependency.Artifact != "lib:pkg" || !strings.Contains(b[0].Newest, "d3") || !strings.Contains(b[0].Recorded, "d1") {
|
||||
t.Fatalf("behind after lib's new digest: %+v", b)
|
||||
}
|
||||
// delivery: dep 0.1.1 is delivered, then app 0.1.1 writes its kind 4 entry; dep 0.1.2 delivered later is behind, an ordering edge
|
||||
deliver := func(module, version string, parts map[string]string) {
|
||||
t.Helper()
|
||||
if err := ix.HearVersionDelivered(ctx, meshwants.VersionDeliveredEvent{Module: module, Version: version, Parts: parts}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
deliver("dep", "0.1.1", map[string]string{"dep:program": "x1"})
|
||||
deliver("app", "0.1.1", map[string]string{"app:program": "a1"})
|
||||
// H-A-82: no delivered version is kept, so an entry of kind 4 reads "not yet delivered" and is never behind.
|
||||
v, _ = ModuleVersions(ctx, ix.Versions, "app")
|
||||
if e := v[0].Edges[1]; e.Version != "0.1.1" || e.Digest != "x1" {
|
||||
t.Fatalf("the kind 4 entry after the first delivery: %+v", e)
|
||||
if e := v[0].Edges[1]; e.Kind != 4 || e.Version != notYetDelivered {
|
||||
t.Fatalf("the kind 4 entry: %+v, want it not yet delivered", e)
|
||||
}
|
||||
deliver("dep", "0.1.2", map[string]string{"dep:program": "x2"})
|
||||
deliver("app", "0.1.1", map[string]string{"app:program": "a1"}) // delivered again: written once, not changed
|
||||
v, _ = ModuleVersions(ctx, ix.Versions, "app")
|
||||
if e := v[0].Edges[1]; e.Version != "0.1.1" {
|
||||
t.Errorf("an entry written at the first delivery changed: %+v", e)
|
||||
}
|
||||
b := behind()
|
||||
if len(b) != 2 || b[1].Kind != 4 || b[1].Act != "" || b[1].Dependency.Artifact != "dep:program" {
|
||||
t.Fatalf("behind after dep's second delivery: %+v", b)
|
||||
if b := behind(); len(b) != 1 || b[0].Kind != 1 {
|
||||
t.Fatalf("behind: %+v, want only the kind 1 entry", b)
|
||||
}
|
||||
// the newest toolchain-agreed per language is kept as heard
|
||||
if err := ix.HearToolchainAgreed(ctx, meshbuilds.ToolchainAgreed{Language: "go", Version: "1.25"}); err != nil {
|
||||
@@ -308,18 +291,12 @@ func TestAnInputIsKeptByItsDigestAndSweptAfterADay(t *testing.T) {
|
||||
func TestWhatIsHeardIsKeptEscapedInTheStateHeard(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
ix := &Indexer{Module: testSelf, Versions: newMem(), OwnBuildState: newMem(), Heard: newMem(), Revisions: newMemObjects(1 << 20)}
|
||||
if err := ix.HearVersionDelivered(ctx, meshwants.VersionDeliveredEvent{Module: "a.b", Version: "1.2.3"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ix.HearToolchainAgreed(ctx, meshbuilds.ToolchainAgreed{Language: "go", Version: "1.26"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, ok := ix.deliveredOf(ctx, "a.b"); !ok {
|
||||
t.Error("the delivered version of a dotted module is not read back")
|
||||
}
|
||||
got, _ := ix.Heard.Keys(ctx)
|
||||
slices.Sort(got)
|
||||
if want := []string{"delivered.a=2eb", "toolchain.go"}; !slices.Equal(got, want) {
|
||||
if want := []string{"toolchain.go"}; !slices.Equal(got, want) {
|
||||
t.Errorf("the keys of heard are %v, want %v", got, want)
|
||||
}
|
||||
if own, _ := ix.OwnBuildState.Keys(ctx); len(own) != 0 {
|
||||
@@ -421,29 +398,19 @@ func TestOutdatedAnswersKinds3And5And6(t *testing.T) {
|
||||
if _, ok := behind()[6]; ok {
|
||||
t.Error("kind 6 behind with no newer delivered version")
|
||||
}
|
||||
// H-A-82: a kind-6 entry is a declaration; with no delivered version kept it is never behind.
|
||||
put("prov", meshcatalog.VersionEntry{Version: "0.1.2", Source: 20})
|
||||
if err := ix.HearVersionDelivered(ctx, meshwants.VersionDeliveredEvent{Module: "prov", Version: "0.1.2", Parts: map[string]string{"prov:program": "p2"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if o, ok := behind()[6]; !ok || o.Act != "" || !strings.Contains(o.Newest, "no longer serves") {
|
||||
t.Errorf("kind 6 after a version that serves no more: %+v", o)
|
||||
if _, ok := behind()[6]; ok {
|
||||
t.Error("kind 6 behind with no delivered version kept")
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 G §377, A-78 c: the first fill gives each module's newest version its edge entries, each marked filled: kind 1 from the
|
||||
// dependency version published latest before the build was claimed, kinds 2, 4 and 6 from the version every placing machine runs,
|
||||
// read from the assigner's placement and the stream running, kind 5 as acted for the running build; so the first outdated says only what is truly behind.
|
||||
// dependency version published latest before the build was claimed, kinds 2, 4 and 6 not yet delivered (H-A-82), kind 5 as acted for the running build; so the first outdated says only what is truly behind.
|
||||
func TestTheFirstFillGivesTheNewestVersionsTheirFilledEdges(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
t0 := time.Date(2026, 11, 1, 3, 0, 0, 0, time.UTC)
|
||||
placed := map[string][]string{"prov": {"n1", "n2"}}
|
||||
ix := &Indexer{Module: testSelf, Versions: newMem(), Heard: newMem(), Revisions: newMemObjects(1 << 20), Now: func() time.Time { return t0 },
|
||||
Placement: func(_ context.Context, m string) ([]meshmoduleassigner.ModulePlacement, error) {
|
||||
return []meshmoduleassigner.ModulePlacement{{Branch: "trunk", Nodes: placed[m]}}, nil
|
||||
},
|
||||
RunningWhole: func(context.Context) (map[string]map[string]nodeengine.Running, error) {
|
||||
return map[string]map[string]nodeengine.Running{"n1": {"prov": {Fingerprint: "fp-prov"}}, "n2": {"prov": {Fingerprint: "fp-prov"}}}, nil
|
||||
}}
|
||||
ix := &Indexer{Module: testSelf, Versions: newMem(), Heard: newMem(), Revisions: newMemObjects(1 << 20), Now: func() time.Time { return t0 }}
|
||||
entries := map[string]meshcatalog.CatalogEntry{
|
||||
"lib": {Module: "lib", Manifest: map[string]any{}, Fp: "fp-lib", Artifacts: []meshcatalog.Artifact{{Ref: "lib:pkg", Package: &meshcatalog.ArtifactPackage{Ecosystem: "go", Name: "x/lib"}}}},
|
||||
"actor": {Module: "actor", Manifest: map[string]any{}, Fp: "fp-actor", Artifacts: []meshcatalog.Artifact{{Ref: "actor:program"}}},
|
||||
@@ -490,8 +457,8 @@ func TestTheFirstFillGivesTheNewestVersionsTheirFilledEdges(t *testing.T) {
|
||||
t.Errorf("kind 1 is the version numbered or published before the build was claimed: %+v", e)
|
||||
}
|
||||
case 4:
|
||||
if e.Version != "0.9.1" || e.Digest != "sha256:prov/program" {
|
||||
t.Errorf("kind 4 is the version every machine runs: %+v", e)
|
||||
if e.Version != notYetDelivered || e.Digest != "" { // H-A-82: a declaration, filled with no delivered version
|
||||
t.Errorf("kind 4 is not yet delivered: %+v", e)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
package index
|
||||
|
||||
import "encoding/json"
|
||||
|
||||
// whole is one repository's trees at one commit, as a take-in reads them through the seat git's trees (wholeAt; design 47 H
|
||||
// §743): the module directories, their manifests, lock files' digests, reader capabilities, named paths and mesh packages, and
|
||||
// the catalogue's predefined.json where the repository holds it. It is what a take-in applies (ApplyMessage); nothing else
|
||||
// carries it.
|
||||
type whole struct {
|
||||
Repository string `json:"repository"`
|
||||
// Branch is the watched branch the commit is the head of at the take-in's source revision.
|
||||
Branch string `json:"branch,omitempty"`
|
||||
// Merge is the commit taken in; Head is the same commit.
|
||||
Merge string `json:"merge,omitempty"`
|
||||
Head string `json:"head,omitempty"`
|
||||
Root string `json:"root,omitempty"`
|
||||
Trees map[string]string `json:"trees,omitempty"`
|
||||
Dirs []string `json:"dirs,omitempty"`
|
||||
DirsKnown bool `json:"dirs-known,omitempty"`
|
||||
Manifests map[string]string `json:"manifests,omitempty"`
|
||||
Locks map[string]map[string]string `json:"locks,omitempty"`
|
||||
Needs map[string][]string `json:"needs,omitempty"`
|
||||
Paths map[string]map[string]string `json:"paths,omitempty"`
|
||||
Packages map[string][]string `json:"packages,omitempty"`
|
||||
// Predefined is the catalogue's predefined.json, exact, at the commit; PredefinedRead says the commit was read and holds none.
|
||||
Predefined json.RawMessage `json:"predefined,omitempty"`
|
||||
PredefinedRead bool `json:"predefined-read,omitempty"`
|
||||
}
|
||||
@@ -135,7 +135,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
|
||||
if want := []state{{Name: "candidates", History: 10}, {Name: "heard", History: 64}, {Name: "manifests", History: 1}, {Name: "predefined", History: 5}, {Name: "taken", History: 1}, {Name: "takeins", History: 1}, {Name: "oldsections", History: 1}, {Name: index.RunsState, History: 1}, {Name: "own-build", History: 1}, {Name: index.PackagesState, History: index.PackagesHistory},
|
||||
{Name: views.MarksState, History: 1}, {Name: index.FindingsState, History: 5}, {Name: views.StatusState, History: 1},
|
||||
{Name: index.RequirementsState, History: 5}, {Name: index.SeatDefinitionsState, History: 5},
|
||||
{Name: index.PendingState, History: 1}, {Name: index.EntryIndexState, History: 1, MaxMiB: 8}, {Name: index.NamedState, History: 1, MaxMiB: 4}, // design 47 G §695
|
||||
{Name: index.EntryIndexState, History: 1, MaxMiB: 8}, {Name: index.NamedState, History: 1, MaxMiB: 4}, // design 47 G §695
|
||||
{Name: index.RegistrationsState, History: 1}, {Name: index.ProposalsState, History: 1, MaxMiB: 4},
|
||||
{Name: index.VersionsState, History: 1, MaxMiB: 64}, {Name: index.BuildEdgesState, History: 1}, {Name: index.InputsState, History: 1}, {Name: index.PinsState, History: 1, MaxMiB: index.PinsCapMiB}, {Name: process.ConsumersState, History: 1}}; !reflect.DeepEqual(m.State, want) { // DLV-I7, DLV-I17
|
||||
t.Errorf("state = %+v, want %+v", m.State, want)
|
||||
@@ -156,21 +156,21 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
|
||||
t.Errorf("the indexer names its bucket %s with %d revisions, its manifest's states are %+v", bucket, history, m.State)
|
||||
}
|
||||
}
|
||||
for bucket, read := range map[string]string{index.PreviewsBucket: "mesh-delivery-checker.previews",
|
||||
for bucket, read := range map[string]string{
|
||||
index.ToolchainsBucket("mesh-build-go"): "mesh-build-go.toolchains", index.ToolchainsBucket("mesh-build-node"): "mesh-build-node.toolchains"} {
|
||||
if !slices.Contains(m.Reads, read) || strings.Replace(read, ".", "_", 1) != bucket {
|
||||
t.Errorf("the indexer reads the bucket %s, its manifest's reads are %v (want %s)", bucket, m.Reads, read)
|
||||
}
|
||||
}
|
||||
// DLV-F1r (ADR 0615 §1): the assigner's assignments is read through its first view and views family, never its state.
|
||||
if want := []string{"mesh-build-go.toolchains", "mesh-build-node.toolchains", "mesh-delivery-builder.build-log", "mesh-delivery-checker.previews"}; !reflect.DeepEqual(sorted(m.Reads), want) {
|
||||
if want := []string{"mesh-build-go.toolchains", "mesh-build-node.toolchains", "mesh-delivery-builder.build-log"}; !reflect.DeepEqual(sorted(m.Reads), want) {
|
||||
t.Errorf("reads = %v, want %v", m.Reads, want)
|
||||
}
|
||||
// "Reads FORGE" is a consumes of the indexer's stream: the broker grants only buckets by reads (mesh-control
|
||||
// internal/broker/state.go:62-93).
|
||||
// The registry seats' published fill each (§413: no consume of version.numbered) package's series (DLV-I5, design 51 *The versions*).
|
||||
if want := []string{"go-package-registry.published", "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "mesh-builds.package-published", "mesh-builds.toolchain-agreed", "mesh-catalog.catalog-revision", "mesh-conditions.findings-asked", "mesh-delivery-checker.verdict", "mesh-delivery-forge-bridge.forge.**",
|
||||
"mesh-delivery.source-moved", "mesh-pipeline-runner.pipeline-ended", "mesh-pipeline-runner.pipeline-started", "mesh-pipeline-runner.run-started", "mesh-wants.version-delivered", "mesh-wants.wait-ended", "mesh-wants.wanted", "node-build-go.step-ended", "node-engine.running", "npm-package-registry.published", "operator-channel.decided", "seat:mesh-module-assigner.views.assignments", "seat:mesh-module-assigner.views.seat-holders"}; !reflect.DeepEqual(sorted(m.Consumes), want) {
|
||||
if want := []string{"go-package-registry.published", "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "mesh-builds.package-published", "mesh-builds.toolchain-agreed", "mesh-catalog.catalog-revision", "mesh-conditions.findings-asked",
|
||||
"mesh-delivery.source-moved", "mesh-pipeline-runner.pipeline-ended", "mesh-pipeline-runner.pipeline-started", "mesh-pipeline-runner.run-started", "mesh-wants.wait-ended", "mesh-wants.wanted", "node-build-go.step-ended", "node-engine.running", "npm-package-registry.published", "operator-channel.decided", "seat:mesh-module-assigner.views.assignments", "seat:mesh-module-assigner.views.seat-holders"}; !reflect.DeepEqual(sorted(m.Consumes), want) {
|
||||
t.Errorf("consumes = %v, want %v", m.Consumes, want)
|
||||
}
|
||||
if b, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 || b != "${dir:state}/broker" {
|
||||
@@ -182,7 +182,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
|
||||
// Three invokes: the file of a registered module's manifest and lock files (registered.go), the forge's trees (the named paths' objects, since the indexer is the one reader of a manifest's paths:
|
||||
// ADR 0499 clauses 2 and 3, ADR 0495 clause 1; and the module directories of the trunk, issue 1900) and whether a
|
||||
// commit is in another (issue 1900).
|
||||
if want := []string{"seat:git.trees", "seat:git.tags", "seat:go-package-registry.versions", "seat:npm-package-registry.versions", "seat:git.file", "seat:git.contains", "seat:git.open-pulls", "seat:git.branch", "mesh-delivery-forge-bridge.bridge_merged_message", "seat:node-engine.composition", "seat:mesh-delivery.source", "seat:mesh-module-assigner.assignments", "seat:mesh-module-assigner.placement", "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence", "seat:node-build-go.steps",
|
||||
if want := []string{"seat:git.trees", "seat:git.tags", "seat:go-package-registry.versions", "seat:npm-package-registry.versions", "seat:git.file", "seat:git.contains", "seat:git.branch", "seat:node-engine.composition", "seat:mesh-module-assigner.assignments", "seat:mesh-module-assigner.placement", "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence", "seat:node-build-go.steps",
|
||||
"seat:node-build-node.steps", "seat:node-build-image.steps", "mesh-delivery-builder.builder_versions"}; !reflect.DeepEqual(m.Invokes, want) {
|
||||
t.Errorf("invokes = %v, want %v", m.Invokes, want)
|
||||
}
|
||||
@@ -200,12 +200,12 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
|
||||
}
|
||||
want = append(want, "indexer_first_views", "indexer_first_view_again", index.ConfigReadersTool, index.BusStateTool, index.RequirementsTool, index.PackageUsersTool, // DLV-F3r, DLV-I10, DLV-I11, DLV-I7, DLV-I8, DLV-I9c
|
||||
index.SeatDefinitionTool, index.SeatDefinitionsAgainTool, index.OutOfStepTool, // DLV-I19
|
||||
index.PendingTool, index.TakeInPendingTool, index.DropPendingTool, index.TakeInMergeTool, index.FingerprintsAgainTool, // issue 1966; 1585/T12, T13 (ADR 0633 §5)
|
||||
index.FingerprintsAgainTool, // issue 1966
|
||||
// milestone 47/G W4.12 (design 47, the indexer's tools): the catalog graph revisions read and steered.
|
||||
index.CatalogRevisionTool, index.CatalogRevisionsTool, index.CandidateGraphsTool, index.RecomputeTool, index.DropCandidateGraphTool,
|
||||
index.EntryIndexTool, index.NamedDropTool, // design 47 G §675
|
||||
index.ProposalsTool, index.ProposalDropTool, index.SeedAgainTool, index.InputsTool, index.PinsTool, index.UnpinTool, index.ConsumerTool, index.RunsTool, index.RunDropTool) // design 47 G §667, §8, §579, §64 (A-194)
|
||||
if !reflect.DeepEqual(m.Tools, want) || len(want) != 45 || !slices.Contains(m.Tools, "modules") {
|
||||
if !reflect.DeepEqual(m.Tools, want) || len(want) != 41 || !slices.Contains(m.Tools, "modules") {
|
||||
t.Errorf("tools = %v, want %v", m.Tools, want)
|
||||
}
|
||||
// Design 47 G §2, §9: every version of every registration is kept with no age limit, 64 MiB, on the old streams section.
|
||||
@@ -260,6 +260,11 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
|
||||
if inv == "mesh-delivery-indexer.indexer_read_modules" { // design 47 G §196: the readers' own subtasks move them to the catalog half
|
||||
continue
|
||||
}
|
||||
// H-A-79: the grants part's and the watchman's invokes of the verdicts go in their own subtasks (the contracts-indexer-bridge
|
||||
// developer's readers); indexer_pending is served again with the held take-in (design 47 H §744, subtask I5).
|
||||
if inv == "mesh-delivery-indexer.verdicts" || inv == "mesh-delivery-indexer.indexer_verdicts" || inv == "mesh-delivery-indexer.indexer_pending" {
|
||||
continue
|
||||
}
|
||||
if tool, ok := strings.CutPrefix(inv, "mesh-delivery-indexer."); ok && !slices.Contains(m.Tools, tool) {
|
||||
t.Errorf("%s invokes %s, which the indexer does not serve", p, inv)
|
||||
}
|
||||
|
||||
@@ -51,10 +51,6 @@
|
||||
"indexer_seat_definition",
|
||||
"indexer_seat_definitions_again",
|
||||
"indexer_out_of_step",
|
||||
"indexer_pending",
|
||||
"indexer_take_in_pending",
|
||||
"indexer_drop_pending",
|
||||
"indexer_take_in_merge",
|
||||
"indexer_fingerprints_again",
|
||||
"indexer_catalog_revision",
|
||||
"indexer_catalog_revisions",
|
||||
@@ -82,11 +78,8 @@
|
||||
"seat:mesh-module-assigner.views.seat-holders",
|
||||
"mesh-builds.package-published",
|
||||
"mesh-builds.toolchain-agreed",
|
||||
"mesh-wants.version-delivered",
|
||||
"node-build-go.step-ended",
|
||||
"mesh-wants.wait-ended",
|
||||
"mesh-delivery-checker.verdict",
|
||||
"mesh-delivery-forge-bridge.forge.**",
|
||||
"mesh-catalog.catalog-revision",
|
||||
"mesh-delivery.source-moved",
|
||||
"mesh-wants.wanted",
|
||||
@@ -113,7 +106,6 @@
|
||||
{ "name": "first-views", "history": 1 },
|
||||
{ "name": "requirements", "history": 5 },
|
||||
{ "name": "seat-definitions", "history": 5 },
|
||||
{ "name": "pending", "history": 1 },
|
||||
{ "name": "entry-index", "history": 1, "max-mib": 8 },
|
||||
{ "name": "named", "history": 1, "max-mib": 4 },
|
||||
{ "name": "registrations", "history": 1 },
|
||||
@@ -148,8 +140,7 @@
|
||||
"reads": [
|
||||
"mesh-build-go.toolchains",
|
||||
"mesh-build-node.toolchains",
|
||||
"mesh-delivery-builder.build-log",
|
||||
"mesh-delivery-checker.previews"
|
||||
"mesh-delivery-builder.build-log"
|
||||
],
|
||||
"invokes": [
|
||||
"seat:git.trees",
|
||||
@@ -158,11 +149,8 @@
|
||||
"seat:npm-package-registry.versions",
|
||||
"seat:git.file",
|
||||
"seat:git.contains",
|
||||
"seat:git.open-pulls",
|
||||
"seat:git.branch",
|
||||
"mesh-delivery-forge-bridge.bridge_merged_message",
|
||||
"seat:node-engine.composition",
|
||||
"seat:mesh-delivery.source",
|
||||
"seat:mesh-module-assigner.assignments",
|
||||
"seat:mesh-module-assigner.placement",
|
||||
"seat:mesh-hand-acts.hand-act",
|
||||
|
||||
Reference in New Issue
Block a user