diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go index 3806efd2..011602e1 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go @@ -30,7 +30,7 @@ import ( type asker func(key string, value []byte) (index.TakeInAsk, error) // Tools are the indexer's tools. read is the indexer's state of verdicts, which indexer_verdicts reads; counts the old sections' count. -func Tools(ask asker, now func() time.Time, read stdio.KeptState, counts func() (index.OldSectionsCount, error), acts meshhandactsclient.Recorder) []stdio.Tool { +func Tools(ask asker, now func() time.Time, counts func() (index.OldSectionsCount, error), acts meshhandactsclient.Recorder) []stdio.Tool { str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) } return []stdio.Tool{ {Name: index.OldSectionsTool, @@ -90,55 +90,6 @@ func Tools(ask asker, now func() time.Time, read stdio.KeptState, counts func() } return a.Result, nil }}, - {Name: index.VerdictsTool, - Description: "The indexer's verdicts on manifests that wait or are refused (all: every verdict), each as its verdicts state holds it: " + - "the module, the digest, the state (waits: the manifest carries a field the indexer's judge does not know yet, named in " + - "unknown, and every reader holds the module as a refused one and keeps what runs; refused: the judge's problems), the " + - "judge that made it and when. A module's name narrows it to that module. A waiting manifest is judged again at each " + - "new indexer and refused only once the newest build of the indexer on the trunk still does not know the field; " + - "indexer_judge judges now.", - Input: map[string]any{"type": "object", "properties": map[string]any{ - "module": map[string]any{"type": "string", "description": "only this module's verdicts"}, - "all": map[string]any{"type": "boolean", "description": "list accepted verdicts too"}}}, - Run: func(args map[string]any) (any, error) { - keys, err := read.Keys() - if err != nil { - return nil, fmt.Errorf("the verdicts state cannot be listed: %w", err) - } - all, _ := args["all"].(bool) - rows, unread := index.ListVerdicts(keys, func(key string) ([]byte, error) { - e, err := read.Get(key) - if err != nil || e == nil { - return nil, err - } - return e.Value, nil - }, str(args, "module"), all) - return map[string]any{"verdicts": rows, "unread": unread}, nil - }}, - {Name: index.SayAcceptedTool, - Description: "Say the event manifest-accepted again now for one module, from the accepted verdict on its trunk's manifest the " + - "indexer holds (design 25 § 12.2): for a verdict the bus module is thought to have missed. A hand act, recorded through " + - "mesh-hand-acts before it is asked. Refused for a module the indexer holds no accepted verdict of (refused, set aside " + - "to wait, or not held). Answers what was said: the digest, the verdict's revision and the bus-state entries' count. A " + - "publish that failed twice is also said again by itself at the indexer's next judging pass (its finding accepted-unsaid).", - Input: map[string]any{"type": "object", "required": []string{"module", "why"}, "properties": map[string]any{ - "module": map[string]any{"type": "string", "description": "the module's name, as the catalogue registers it"}, - "why": map[string]any{"type": "string", "description": "why it is said again now, in plain words"}}}, - Run: func(args map[string]any) (any, error) { - key, value, err := index.NewSayAccepted(str(args, "module"), str(args, "why"), stdio.CallerOf(args), now()) - if err != nil { - return nil, err - } - if _, err := acts.Record(meshhandactsclient.Request{What: "said manifest-accepted of " + str(args, "module") + " again by hand", - Why: str(args, "why"), Cause: index.SayAcceptedTool, Verb: "say-accepted"}); err != nil { - return nil, fmt.Errorf("the hand act was not recorded through mesh-hand-acts, so nothing was said again: %w", err) - } - a, err := ask(key, value) - if err != nil { - return nil, err - } - return a.Result, nil - }}, {Name: index.SeatDefinitionsAgainTool, Description: "Read a contracts module's seat definitions again now (design 26): its trunk's, at the commit the indexer " + "holds the module at, or with pull and head a pull request's head (read with the head's own manifest, so its " + @@ -170,27 +121,6 @@ func Tools(ask asker, now func() time.Time, read stdio.KeptState, counts func() } return a.Result, nil }}, - {Name: index.JudgeAgainTool, - Description: "Judge the manifests again now, as the indexer does at its start and after each merge, and answer how the module's " + - "verdict stands (waits, refused or accepted, and why). For a manifest that waits for a judge that knows its fields, or " + - "after a manifest was mended where the indexer has not heard it. A waiting manifest becomes refused here only when the " + - "running indexer is the newest build of itself the trunk holds. With a digest it judges that one manifest of the " + - "module (a candidate's, or one an older judge refused) in the catalogue the trunk holds. indexer_verdicts reads the result.", - Input: map[string]any{"type": "object", "required": []string{"module"}, "properties": map[string]any{ - "module": map[string]any{"type": "string", "description": "the module's name, as the catalogue registers it"}, - "digest": map[string]any{"type": "string", "description": "one manifest digest of the module to judge (indexer_verdicts names it); none judges every manifest the trunk holds"}, - "why": map[string]any{"type": "string", "description": "why it is judged again now, in plain words"}}}, - Run: func(args map[string]any) (any, error) { - key, value, err := index.NewJudgeAgain(str(args, "module"), str(args, "digest"), str(args, "why"), stdio.CallerOf(args), now()) - if err != nil { - return nil, err - } - a, err := ask(key, value) - if err != nil { - return nil, err - } - return a.Result, nil - }}, } } @@ -285,7 +215,7 @@ var errReadOnly = errors.New("the tools bundle only reads this state") func main() { asks := askTakeIn(stdio.State(index.TakeInsState), 25*time.Second) acts := meshhandactsclient.Recorder{Ask: stdio.Ask} - tools := Tools(asks, time.Now, stdio.State(index.VerdictsState), heldCounts(stdio.State(index.OldSectionsState)), acts) + tools := Tools(asks, time.Now, heldCounts(stdio.State(index.OldSectionsState)), acts) // The modules and the module graph are the newest catalog graph revision's (milestone 47/G): revision.go. revisions := bundleObjects{busstate.ObjectsOf(index.CatalogRevisionsObjects)} // The view of the trunk is read from the indexer's own objects once when the runtime starts and again each time the seat's diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/views.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/views.go index e525aeac..94a05dee 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/views.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/views.go @@ -84,7 +84,7 @@ func busStateTool(state func(string) views.Source) stdio.Tool { "answer is at, so every manifest-accepted above it is newer. The bus module's keeper takes its first view from it.", Input: map[string]any{"type": "object", "properties": map[string]any{}}, Run: func(map[string]any) (any, error) { - return index.BusStateOf(state("modules"), state(index.VerdictsState), state("manifests"), state(views.MarksState)) + return index.BusStateOf(state("modules"), state("manifests"), state(views.MarksState)) }} } diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index 301b6dc4..cd7b175b 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -391,8 +391,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return err } defer calls.Close() - // The verdicts on manifests (ADR 0578 §1): a state this delivery brings, bound when it appears. - verdicts := process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.VerdictsState), "no verdict on a manifest is written", logger.Printf) // The published packages (ADR 0593 §2): a state this delivery brings, bound when it appears; until then every // fingerprint takes the version line. packages := process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.PackagesState), "no package is published, so every fingerprint takes the version line", logger.Printf) @@ -407,9 +405,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // unanswered since, the newest toolchain-agreed and version-delivered, the artifacts published so far of a version). heardStore := awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.HeardState), "what the indexer heard is not kept", logger.Printf)} ownBuildState := process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.OwnBuildStateName), "which build this indexer runs is not kept", logger.Printf) - // The manifests a take-in wrote and no judge pass judged yet (issue 1966): a state this delivery brings, bound when it - // appears; until then a take-in's new manifests are kept for their judging in its memory alone. - unjudged := process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.UnjudgedState), "a take-in's new manifests are kept for their judging in its memory alone", logger.Printf) // The runs on request the indexer holds a candidate for (A-174): a state this delivery brings, bound when it appears. runs := process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.RunsState), "no run on request's candidate is written or dropped", logger.Printf) findings := process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.FindingsState), "no finding of the indexer is written", logger.Printf) @@ -445,7 +440,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro named := index.NewNamed(awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.NamedState), "no entry is kept for a want or a running build that names it", logger.Printf)}, logger.Printf) pinsBucket := process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.PinsState), "no record's pin on a catalog graph revision is kept, so no revision is removed", logger.Printf) pinsStore := awaitedStore{b: pinsBucket} - consumers := process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, process.ConsumersState), "indexer_consumer reads no consumer", logger.Printf) // The state inputs and the object store beside it; the seat mesh-catalog's verb input is answered here in chunks of the core // connection, of any size (design 47 G §712, A-237). inputsStore := awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.InputsState), "no input of a run ask is kept, so the verb input answers none", logger.Printf)} @@ -456,8 +450,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return fmt.Errorf("answering the verb input: %w", err) } defer stopInput() - ix := &index.Indexer{Module: cred.Module, NamedSets: named, Findings: writer, OldSections: awaitedStore{b: oldSections}, Unjudged: awaitedStore{b: unjudged}, Requirements: awaitedStore{b: requirements}, Store: kvStore{candidates}, Modules: modules, Manifests: kvStore{manifests}, Taken: awaitedStore{b: taken}, Builds: known, - Verdicts: awaitedStore{b: verdicts}, SayRefused: sayRefused(conn, cred, logger.Printf), SayAccepted: sayAccepted(conn, cred), + ix := &index.Indexer{Module: cred.Module, NamedSets: named, Findings: writer, OldSections: awaitedStore{b: oldSections}, Requirements: awaitedStore{b: requirements}, Store: kvStore{candidates}, Modules: modules, Manifests: kvStore{manifests}, Taken: awaitedStore{b: taken}, Builds: known, Predefined: awaitedStore{b: predefined}, Logf: logger.Printf, Named: namedVia(calls.Call), Trunk: trunkVia(calls.Call), BranchHead: branchHeadVia(calls.Call), Contains: containsVia(calls.Call), Read: readVia(calls.Call), ReadOpen: openVia(calls.Call), Packages: awaitedStore{b: packages, js: js, name: index.Bucket(cred.Module, index.PackagesState)}, Files: filesVia(calls.Call), GoSum: goSumVia(calls.Call), GoMod: goModVia(calls.Call), @@ -667,35 +660,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // indexer_first_views and as view-waits past five minutes, and read again on the next forge message heard. defs := gitWaits(ctx, cred.Module, ix, writer, cred.Node, process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, views.StatusState), "indexer_first_views does not say the wait for the seat git", logger.Printf), logger.Printf) seatDefs.Ready(func(jetstream.KeyValue) { defs.Start(ctx) }) - // Every manifest judged at the start, so a judge that changed with this program judges every one again (ADR 0578 - // §1), once the verdicts state is there; each merge judges them again (index.ApplyMessage). - verdicts.Ready(func(jetstream.KeyValue) { - // A take-in held because the verdicts state was not on the bus yet is evaluated again once the start's pass ended, - // whatever it answered (issue 1966 note 154). - defer func() { - if err := ix.PendingAgain(ctx); err != nil { - logger.Printf("take-in: pending is not evaluated again at the verdicts state's bind: %v", err) - } - }() - // A pass that fails (a read the bus did not answer) is judged again at the next message or source revision the - // indexer hears (index.kickJudge): no timer. - judge := func(fromKept bool) { - j, err := ix.JudgeAtStart(ctx) - if err != nil { - logger.Printf("judging the manifests at start (from kept definitions: %v): %v; judged again at the next message heard", fromKept, err) - return - } - logger.Printf("judged %d manifests at start (from kept definitions: %v): %d verdicts written, %d with problems, of which outgrown and kept accepted: %v, waiting for a judge that knows their fields: %v", - j.Judged, fromKept, j.Written, len(j.Refused), j.Outgrown, j.Waiting) + // A take-in held while a state was not on the bus is evaluated again at the start (issue 1966 note 154). + go func() { + if err := ix.PendingAgain(ctx); err != nil { + logger.Printf("take-in: pending is not evaluated again at the start: %v", err) } - if !defs.Waiting() { - judge(false) - return - } - // The seat git did not answer: the judging waits for it at most index.GitWaitBound, off the start, so nothing else of - // the start is held behind it. - go func() { judge(defs.Ready(ctx)) }() - }) + }() // A build module says each time it wrote its toolchains key: its stream, and the indexer's durable consumer on it, are // made by the controller as the forge's are. Where the stream is not there yet the indexer says so and is started again // with the build module, as with the gate's source. @@ -728,22 +698,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if err := hub.Handle(meshdelivery.SubjectSourceMovedAll, func(m jetstream.Msg) { source.heard(ctx, m) }); err != nil { return err } - pinsLedger := process.NewConsumerLedger(process.EventsStream, cred.EventConsumerName(), consumerSink(consumers), nil, logger.Printf) - for _, subject := range pinsEvents { - if err := hub.Handle(subject, func(m jetstream.Msg) { heardPins(ctx, ix, pinsLedger, m, logger.Printf) }); err != nil { - return err - } - } - // Once the state is bound the pins are made and every kept revision judged (§579: at its start). - pinsBucket.Ready(func(jetstream.KeyValue) { - go func() { - if err := ix.PinsMade(ctx); err != nil { - logger.Printf("pins: the state %s is not made: %v", index.PinsState, err) - return - } - ix.JudgePins(ctx) - }() - }) + // Once the state is bound every kept revision is judged against the pins it holds (design 47 H §749: at its start). + pinsBucket.Ready(func(jetstream.KeyValue) { go ix.JudgePins(ctx) }) versions := func(m jetstream.Msg) { if !heardVersion(ctx, ix, m, logger.Printf) { _ = m.Ack() // nothing else the indexer consumes rides the shared stream @@ -819,9 +775,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // said up to; a first view (the tools bundle) answers at that mark. startViews(ctx, js, cred, marks, writer, map[string]readyOf{"manifests": boundNow(manifests), "candidates": boundNow(candidates), - index.VerdictsState: awaited(verdicts), index.PredefinedState: awaited(predefined), - index.PackagesState: awaited(packages), - index.HeardState: awaited(heard.b), "taken": awaited(taken), "findings": awaited(findings), + index.PredefinedState: awaited(predefined), + index.PackagesState: awaited(packages), + index.HeardState: awaited(heard.b), "taken": awaited(taken), "findings": awaited(findings), index.SeatDefinitionsState: awaited(seatDefs)}, errs, logger.Printf) cc, err := process.HearEvents(cons, func(m jetstream.Msg) { handle(ctx, m, ix, heard, logger) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go index c628ec4a..1c2091c5 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go @@ -225,3 +225,9 @@ func (f *fakeRuns) EndRunPipeline(_ context.Context, e meshpipelinerunner.Pipeli f.ended = e.Run return f.err } + +func (f *fakeRuns) HeardPins(context.Context, *meshpipelinerunner.PipelineStarted, *meshpipelinerunner.PipelineEnded) error { + return nil +} + +func (f *fakeRuns) JudgePins(context.Context) {} diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/pins.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/pins.go index 98e250f0..7849f6ff 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/pins.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/pins.go @@ -2,36 +2,36 @@ package main import ( "context" - "encoding/json" - "time" "github.com/nats-io/nats.go/jetstream" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" "git.novox.be/novox/mesh-sdk/go/process" - - "mesh-delivery-indexer/internal/index" ) -// The indexer's durable consumer of mesh-delivery's pinned, freed, heads-replaced and pins-stated (design 47 G §153, §216, §579): -// each event is applied to the state pins, and only then acknowledged, so one said while the indexer was down is applied when it -// comes back. The consumer's facts are kept in the state consumers (A-176) for indexer_consumer. +// The indexer pins a pipeline's revision on the runner's pipeline-started and frees it on its pipeline-ended, each heard on +// a durable consumer of its own on that event's stream (design 47 H §749, runs.go): an event is applied to the state pins, +// and only then acknowledged, so one said while the indexer was down is applied when it comes back. -// pinsEvents are the subjects the indexer applies to its pins. -var pinsEvents = []string{meshdelivery.SubjectPinned, meshdelivery.SubjectFreed, meshdelivery.SubjectHeadsReplaced, meshdelivery.SubjectPinsStated} +// pinsStreams are the streams of the runner's events the indexer's pins follow. +var pinsStreams = []string{meshpipelinerunner.StreamPipelineStarted, meshpipelinerunner.StreamPipelineEnded} -// pendingOf says how many messages of the module's durable consumer of the events stream have not been given to it yet. +// pendingOf says how many messages of the indexer's durable consumers of the pins' streams have not been given to it yet. func pendingOf(js jetstream.JetStream, cred process.Account) func(context.Context) (uint64, error) { return func(ctx context.Context) (uint64, error) { - cons, err := cred.EventConsumer(ctx, js, process.EventsStream) - if err != nil { - return 0, err + var n uint64 + for _, stream := range pinsStreams { + cons, err := cred.EventConsumer(ctx, js, stream) + if err != nil { + return 0, err + } + info, err := cons.Info(ctx) + if err != nil { + return 0, err + } + n += info.NumPending } - info, err := cons.Info(ctx) - if err != nil { - return 0, err - } - return info.NumPending, nil + return n, nil } } @@ -46,20 +46,3 @@ func consumerSink(b *process.AwaitedBucket) process.ConsumerSink { return err } } - -// heardPins applies one pins event, acknowledges it, and judges the revisions kept: an event not applied is left unacknowledged, so -// the bus gives it again. -func heardPins(ctx context.Context, ix *index.Indexer, ledger *process.ConsumerLedger, m jetstream.Msg, logf func(string, ...any)) { - ledger.Delivered(m) - if err := ix.HeardPins(ctx, m.Subject(), m.Data()); err != nil { - logf("pins: %s is not applied, left for the bus to give again: %v", m.Subject(), err) - return - } - _ = m.Ack() - var body struct { - At time.Time `json:"at"` - } - _ = json.Unmarshal(m.Data(), &body) - ledger.Applied(m, body.At) - ix.JudgePins(ctx) -} diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/runs.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/runs.go index 6c6bdbfb..9a3d42b5 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/runs.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/runs.go @@ -13,10 +13,13 @@ import ( "mesh-delivery-indexer/internal/index" ) -// runTaker is what the indexer does with the runner's events of a run on request (index.Indexer.StartRun, EndRunPipeline). +// runTaker is what the indexer does with the runner's events: a run on request's (index.Indexer.StartRun, EndRunPipeline), +// and a pipeline's pins (HeardPins, design 47 H §749), the kept revisions judged after each pin freed (JudgePins). type runTaker interface { StartRun(ctx context.Context, e meshpipelinerunner.RunStarted) error EndRunPipeline(ctx context.Context, e meshpipelinerunner.PipelineEnded) error + HeardPins(ctx context.Context, started *meshpipelinerunner.PipelineStarted, ended *meshpipelinerunner.PipelineEnded) error + JudgePins(ctx context.Context) } // hearRuns hears the runner's run-started and pipeline-ended (design 47 G §64, §190; A-160, A-174), each on its own stream, on @@ -28,6 +31,7 @@ func hearRuns(ctx context.Context, js jetstream.JetStream, cred process.Account, each func(jetstream.Msg) }{ {meshpipelinerunner.StreamRunStarted, func(m jetstream.Msg) { heardRunStarted(ctx, ix, m, logf) }}, + {meshpipelinerunner.StreamPipelineStarted, func(m jetstream.Msg) { heardPipelineStarted(ctx, ix, m, logf) }}, {meshpipelinerunner.StreamPipelineEnded, func(m jetstream.Msg) { heardPipelineEnded(ctx, ix, m, logf) }}, } { _ = notHeard.Bind(s.stream, "no run on request's candidate is written or dropped until the indexer starts again", func() error { @@ -53,7 +57,19 @@ func heardRunStarted(ctx context.Context, ix runTaker, m delivered, logf func(st settleRun(m, ix.StartRun(ctx, e), "run-started of "+e.Run, logf) } -// heardPipelineEnded takes one pipeline-ended in: of a run's pipeline, its module is kept as ended. +// heardPipelineStarted pins the revision a pipeline-started names (design 47 H §749). +func heardPipelineStarted(ctx context.Context, ix runTaker, m delivered, logf func(string, ...any)) { + var e meshpipelinerunner.PipelineStarted + if err := json.Unmarshal(m.Data(), &e); err != nil { + logf("%s: not a pipeline-started, dropped: %v", m.Subject(), err) + _ = m.Term() + return + } + settleRun(m, ix.HeardPins(ctx, &e, nil), "pipeline-started of "+e.Pipeline, logf) +} + +// heardPipelineEnded takes one pipeline-ended in: its revision's pin is freed and the kept revisions judged (§749); of a +// run's pipeline, its movers are kept as ended. func heardPipelineEnded(ctx context.Context, ix runTaker, m delivered, logf func(string, ...any)) { var e meshpipelinerunner.PipelineEnded if err := json.Unmarshal(m.Data(), &e); err != nil { @@ -61,7 +77,12 @@ func heardPipelineEnded(ctx context.Context, ix runTaker, m delivered, logf func _ = m.Term() return } - settleRun(m, ix.EndRunPipeline(ctx, e), "pipeline-ended of "+e.Module+" in the run "+e.Run, logf) + if err := ix.HeardPins(ctx, nil, &e); err != nil { + settleRun(m, err, "pipeline-ended of "+e.Pipeline, logf) + return + } + settleRun(m, ix.EndRunPipeline(ctx, e), "pipeline-ended of "+e.Pipeline+" in the run "+e.Run, logf) + ix.JudgePins(ctx) } func settleRun(m delivered, err error, what string, logf func(string, ...any)) { diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go index 75d37b44..35c5a45d 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go @@ -16,8 +16,6 @@ type takerIn interface { TakeIn(ctx context.Context, module string) index.TakenIn TakeInRepository(ctx context.Context, repository string) index.TakenIn CatchUp(ctx context.Context) index.TakenIn - JudgeAgain(ctx context.Context, module, digest string) index.TakenIn - SayAcceptedAgain(ctx context.Context, module string) index.TakenIn SeatDefinitionsAgain(ctx context.Context, module string, pull int, head string) index.TakenIn TakeInPendingByHand(ctx context.Context, key, by, why string) index.TakenIn DropPending(ctx context.Context, key, by, why string, abandon bool) index.TakenIn @@ -88,10 +86,6 @@ func takenIn(ctx context.Context, ix takerIn, kv jetstream.KeyValue, e jetstream a.Module = a.Repository } else if a.SeatDefinitions { asked, r = "reading of its seat definitions again", ix.SeatDefinitionsAgain(ctx, a.Module, a.Pull, a.Head) - } else if a.Say { - asked, r = "manifest-accepted said again", ix.SayAcceptedAgain(ctx, a.Module) - } else if a.Judge { - asked, r = "judging again", ix.JudgeAgain(ctx, a.Module, a.Digest) } else { r = ix.TakeIn(ctx, a.Module) } diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/verdicts.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/verdicts.go index 0ee37ae0..c79dd74b 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/verdicts.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/verdicts.go @@ -2,67 +2,14 @@ package main import ( "context" - "encoding/json" - "fmt" "time" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein" "git.novox.be/novox/mesh-sdk/go/process" "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" ) -// sayRefused says the indexer's event manifest-refused (novox/hq ADR 0578 §1): mesh.seat.mesh-catalog.event.manifest-refused, -// with the SDK's envelope headers and the digest and judge as the bus's dedup id, so one refusal is said once. -func sayRefused(conn *nats.Conn, cred process.Account, logf func(string, ...any)) func(context.Context, meshcatalog.ManifestRefusedEvent) { - return func(ctx context.Context, r meshcatalog.ManifestRefusedEvent) { - data, err := json.Marshal(r) - if err != nil { - logf("judge: %s for %s is not said: %v", meshcatalog.EventManifestRefused, r.Module, err) - return - } - msg := nats.NewMsg(seatEventSubject(meshcatalog.EventManifestRefused)) - id := "manifest-refused-" + r.Digest + "-" + r.Judge - msg.Header.Set("x-event-id", id) - msg.Header.Set("x-source", cred.Module) - msg.Header.Set("x-node", cred.Node) - msg.Header.Set("x-time", time.Now().UTC().Format(time.RFC3339Nano)) - msg.Header.Set("content-type", "application/json") - msg.Header.Set(nats.MsgIdHdr, id) - msg.Data = data - // An event is published, never asked (issue 1966): the stream that keeps the subject takes it and drops a - // duplicate by its id; a bus refusing the subject says so on the connection's error handler, not by a wait here. - if err := conn.PublishMsg(msg); err != nil { - logf("judge: %s", fmt.Errorf("saying %s for %s: %w", meshcatalog.EventManifestRefused, r.Module, err)) - } - } -} - -// sayAccepted says the indexer's event manifest-accepted (issue 1966 D3, novox/hq ADR 0595 §4): -// mesh.seat.mesh-catalog.event.manifest-accepted, carrying the module's bus-state entries whole, with the digest and judge as -// the bus's dedup id (meshcatalog.ManifestAcceptedEvent.EventID), so one acceptance is said once. -func sayAccepted(conn *nats.Conn, cred process.Account) func(context.Context, meshcatalog.ManifestAcceptedEvent, string) error { - return func(ctx context.Context, a meshcatalog.ManifestAcceptedEvent, id string) error { - data, err := json.Marshal(a) - if err != nil { - return err - } - msg := nats.NewMsg(seatEventSubject(meshcatalog.EventManifestAccepted)) - msg.Header.Set("x-event-id", id) - msg.Header.Set("x-source", cred.Module) - msg.Header.Set("x-node", cred.Node) - msg.Header.Set("x-time", time.Now().UTC().Format(time.RFC3339Nano)) - msg.Header.Set("content-type", "application/json") - msg.Header.Set(nats.MsgIdHdr, id) - msg.Data = data - // Published, never asked (issue 1966): the event's stream takes it and drops a duplicate by its id; waiting on - // the stream's acknowledgement held each judging pass twenty seconds a manifest on a bus refusing the subject. - // A refusal is said on the connection's error handler; an error here (no connection) is the caller's to retry. - return conn.PublishMsg(msg) - } -} - // sayTakeIn says a take-in event (novox/hq ADR 0633 §5) as the indexer's ordinary event, on the path its other events // take: mesh.mod..event.take-in., the wait's key in the body. A publish that fails is the caller's to say. func sayTakeIn(js jetstream.JetStream, cred process.Account) func(context.Context, takein.TakeIn) error { diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/views.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/views.go index 1f0b0a85..706b203a 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/views.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/views.go @@ -183,6 +183,5 @@ func gitWaits(ctx context.Context, module string, ix *index.Indexer, w index.Fin f := &views.Follower{Owner: git.Seat, Tool: "seat:" + git.Seat + "." + git.VerbTrees, State: index.SeatDefinitionsState, Logf: logf, RaiseWaits: func(s views.Status) { finding(s, true) }, ClearWaits: func(s views.Status) { finding(s, false) }} status.Ready(func(kv jetstream.KeyValue) { f.Said = views.KeepStatus(ctx, kv, logf) }) - return &index.DefinitionsAtStart{Read: ix.ReadTrunkDefinitions, Waits: f, Logf: logf, - Judge: func(ctx context.Context) error { _, err := ix.JudgeAtStart(ctx); return err }} + return &index.DefinitionsAtStart{Read: ix.ReadTrunkDefinitions, Waits: f, Logf: logf} } diff --git a/modules/mesh-delivery-indexer/internal/index/accepted.go b/modules/mesh-delivery-indexer/internal/index/accepted.go deleted file mode 100644 index 6d2122ff..00000000 --- a/modules/mesh-delivery-indexer/internal/index/accepted.go +++ /dev/null @@ -1,190 +0,0 @@ -package index - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "strings" - "time" - - mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" - "git.novox.be/novox/mesh-sdk/go/keys" - - "mesh-delivery-indexer/judge" -) - -// manifest-accepted, never lost silently (design 25 § 12.2; DLV-I9, I9b, I9c). The indexer says it for every newly -// accepted manifest, its bus-state entries empty when it asks for none, and an empty one for a module leaving the trunk, -// so the bus module's orphans see it; a manifest the judge set aside to wait gets none. A publish that fails is published -// once more; one still failing writes the indexer's finding naming the module, said again at each judging pass after -// and cleared once said. indexer_say_accepted says it again for one module by hand, from the verdict it holds. -const ( - SayAcceptedTool = "indexer_say_accepted" - sayAcceptedPrefix = "say." - // UnsaidKind is the finding of a manifest-accepted whose publish failed twice. - UnsaidKind = "accepted-unsaid" -) - -// acceptedOf is the event body for the verdict v (its revision in the verdicts state) on the manifest raw of digest, its -// entries an empty list when it asks for none. -func acceptedOf(digest string, v meshcatalog.VerdictsByDigestValue, revision uint64, raw []byte) meshcatalog.ManifestAcceptedEvent { - m, _ := judge.Manifest(raw, nil) - a := acceptedEvent(digest, v, m) - a.Revision = int64(revision) - return a -} - -// leftOf is the empty manifest-accepted for module, which left the trunk, judged by judgeID; revision is the modules -// state's revision of the value that left (0 when none is held), so its event id (acceptedEventID, the -// module and the revision, since its body names no digest) differs for each leave. -func leftOf(module string, revision uint64, judgeID string) (meshcatalog.ManifestAcceptedEvent, string) { - a := meshcatalog.ManifestAcceptedEvent{V: keys.Version, Module: module, Judge: judgeID, BusState: busStateOf(nil), Revision: int64(revision)} - return a, acceptedEventID(a) -} - -// sayAccepted says a with the event id id, once more when the first publish fails; the module's finding is written when -// both fail and deleted when it is said. Nil SayAccepted says nothing. -func (ix *Indexer) sayAccepted(ctx context.Context, a meshcatalog.ManifestAcceptedEvent, id string) error { - if ix.SayAccepted == nil { - return nil - } - err := ix.SayAccepted(ctx, a, id) - if err != nil { - ix.logf("judge: %s for %s failed, published again: %v", meshcatalog.EventManifestAccepted, a.Module, err) - err = ix.SayAccepted(ctx, a, id) - } - ix.unsaid(ctx, a.Module, err) - return err -} - -// unsaidID is the id of module's unsaid finding. -func unsaidID(own, module string) string { return own + ".accepted." + module } - -// unsaid raises module's finding for err, or clears it with none. -func (ix *Indexer) unsaid(ctx context.Context, module string, err error) { - if ix.Findings == nil { - return - } - var werr error - if err == nil { - werr = ix.clearFinding(ctx, mcc.ScopeModule, unsaidID(ix.Module, module), UnsaidKind, "manifest-accepted of "+module+" was said.") - } else { - werr = ix.Findings.Write(ctx, ix.observed(mcc.ScopeModule, unsaidID(ix.Module, module), UnsaidKind, - fmt.Sprintf("%s of %s was not published: %v", meshcatalog.EventManifestAccepted, module, err), - "The indexer could not tell the bus module that "+module+"'s manifest was accepted", - "The bus module creates and removes a module's bus state from this event. The indexer says it again at its "+ - "next judging pass and clears this once it is said; indexer_say_accepted says it now.", UnsaidKind)) - } - if werr != nil { - ix.logf("judge: the finding of %s for %s is not written: %v", meshcatalog.EventManifestAccepted, module, werr) - } -} - -// sayFor says module's manifest-accepted from what the indexer holds now: from its trunk's accepted verdict, or an empty -// one when the trunk holds the module no more (only when left is allowed: by hand a module with no accepted verdict is -// refused). A held module whose verdict waits or is refused gets none; again gives the event an id of its own, so a hand -// act within the bus's duplicate window is still said. -func (ix *Indexer) sayFor(ctx context.Context, module string, allowLeft, again bool) (string, error) { - held, err := ix.heldModules(ctx) - if err != nil { - return "", fmt.Errorf("the modules state cannot be read: %w", err) - } - m, ok := held[module] - var a meshcatalog.ManifestAcceptedEvent - var id string - switch { - case !ok && !allowLeft: - return "", fmt.Errorf("the indexer holds no module %s, so it has no accepted verdict to say", module) - case !ok: - a, id = leftOf(module, 0, ix.judgeID()) - case m.Manifest == "" || ix.Verdicts == nil || ix.Manifests == nil: - return "", fmt.Errorf("%s has no accepted verdict the indexer holds", module) - default: - key, err := meshcatalogclient.VerdictsKey(m.Manifest) - if err != nil { - return "", err - } - value, revision, found, err := ix.Verdicts.Get(ctx, key) - if err != nil { - return "", fmt.Errorf("the verdict on %s cannot be read: %w", module, err) - } - var v meshcatalog.VerdictsByDigestValue - if found { - v, err = meshcatalogclient.DecodeVerdict(value) - } - if !found || err != nil || meshcatalogclient.VerdictHeld(v) { - if !allowLeft { - return "", fmt.Errorf("%s has no accepted verdict on its manifest %s", module, m.Manifest) - } - ix.unsaid(ctx, module, nil) // a set-aside or refused manifest gets none - return "none: its manifest is not accepted", nil - } - raw, _, found, err := ix.Manifests.Get(ctx, m.Manifest) - kept, derr := meshcatalogclient.DecodeManifest(raw) - if err != nil || !found || derr != nil { - return "", fmt.Errorf("the manifest %s of %s cannot be read", m.Manifest, module) - } - a = acceptedOf(m.Manifest, v, revision, []byte(kept.Manifest)) - id = acceptedEventID(a) - } - if again { - id += "-again-" + ix.now().UTC().Format(time.RFC3339Nano) - } - if err := ix.sayAccepted(ctx, a, id); err != nil { - return "", fmt.Errorf("%s of %s was not published: %w", meshcatalog.EventManifestAccepted, module, err) - } - if a.Digest == "" { - return "said empty: the trunk holds no module " + module, nil - } - return fmt.Sprintf("said for %s, manifest %s at revision %d, %d bus-state entries", module, a.Digest, a.Revision, len(a.BusState)), nil -} - -// SayUnsaid says again each manifest-accepted whose finding stands (the judging pass's end): cleared once said. What stands is -// what the finding writer holds (design 47 G A-38). -func (ix *Indexer) SayUnsaid(ctx context.Context) { - if ix.Findings == nil { - return - } - for _, f := range ix.Findings.Held() { - if f.Kind != UnsaidKind { - continue - } - module := strings.TrimPrefix(f.Subject.ID, ix.Module+".accepted.") - if _, err := ix.sayFor(ctx, module, true, false); err != nil { - ix.logf("judge: %v; its finding stands", err) - } - } -} - -// SayAcceptedAgain answers indexer_say_accepted: module's manifest-accepted said again now from its accepted verdict. -func (ix *Indexer) SayAcceptedAgain(ctx context.Context, module string) TakenIn { - out := TakenIn{Module: module} - answer, err := ix.sayFor(ctx, module, false, true) - if err != nil { - out.Answer = err.Error() - return out - } - out.Stored, out.Answer = true, answer - return out -} - -// NewSayAccepted is the value indexer_say_accepted writes for a module: its key in the takeins state and the value. -func NewSayAccepted(module, why, by string, at time.Time) (string, []byte, error) { - key, data, err := NewTakeIn(module, why, by, at) - if err != nil { - return "", nil, err - } - var a TakeInAsk - if err := json.Unmarshal(data, &a); err != nil { - return "", nil, err - } - if a.Why == "" { - return "", nil, errors.New("say why it is said again (why): it is a hand act") - } - a.Say = true - data, err = json.Marshal(a) - return sayAcceptedPrefix + key, data, err -} diff --git a/modules/mesh-delivery-indexer/internal/index/accepted_problems_test.go b/modules/mesh-delivery-indexer/internal/index/accepted_problems_test.go deleted file mode 100644 index 0336b34a..00000000 --- a/modules/mesh-delivery-indexer/internal/index/accepted_problems_test.go +++ /dev/null @@ -1,25 +0,0 @@ -package index - -import ( - "encoding/json" - "reflect" - "strings" - "testing" - - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" -) - -// Design 47 G §668, §692: manifest-accepted's problems are the validator's strings, the same the verdict writes as -// warnings, in the same order; a mended manifest says an empty list, never null. -func TestManifestAcceptedProblemsAreTheVerdictsWarnings(t *testing.T) { - warnings := []string{`requires "amqp", which no module provides`, "a second problem"} - got := acceptedOf("sha256:aa", meshcatalog.VerdictsByDigestValue{Module: "m", Judge: "j", Warnings: warnings}, 3, []byte(`{"module":"m","version":"1.0"}`)) - if !reflect.DeepEqual(got.Problems, warnings) { - t.Errorf("problems %v, want the verdict's warnings %v", got.Problems, warnings) - } - mended := acceptedOf("sha256:bb", meshcatalog.VerdictsByDigestValue{Module: "m", Judge: "j"}, 4, []byte(`{"module":"m","version":"2.0"}`)) - raw, _ := json.Marshal(mended) - if !strings.Contains(string(raw), `"problems":[]`) { - t.Errorf("a mended manifest's event is %s, want an empty problems list", raw) - } -} diff --git a/modules/mesh-delivery-indexer/internal/index/behind_test.go b/modules/mesh-delivery-indexer/internal/index/behind_test.go deleted file mode 100644 index 770eb47d..00000000 --- a/modules/mesh-delivery-indexer/internal/index/behind_test.go +++ /dev/null @@ -1,112 +0,0 @@ -package index - -import ( - "context" - "encoding/json" - "testing" - "time" - - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk" -) - -// Design 47 G A-463: a merge the indexer never took in is noticed at a source revision and taken in as its merged message -// would have been, with a finding raised while it stays behind and cleared once it is taken in. -func TestAMergeTheIndexerNeverTookInIsTakenInAndSaidWhileItIsBehind(t *testing.T) { - ctx := context.Background() - b := newTakeInBed(t) - b.ix.Taken, b.ix.Findings = newMem(), newFindings() - key, _ := trunk.RepositoryKey(catalogueRepo) - old, head := sha('7'), sha('8') - data, _ := takein.EncodeTaken(takein.Taken{Repository: trunk.RepositoryName(catalogueRepo), Merge: old}) - b.ix.Taken.(*memStore).put(key, data) - ancestry := map[[2]string]bool{{old, head}: true} // old is in head - b.ix.Contains = func(_ context.Context, _, commit, in string) (bool, error) { - return commit == in || ancestry[[2]string{commit, in}], nil - } - b.ix.source = sourceNow{rev: 3, heads: map[string]string{catalogueRepo: head}} - - // The head has no merged message yet: nothing is taken in and the finding stays. - if done := b.ix.TakeInBehind(ctx); len(done) != 0 || len(b.ix.Findings.Held()) != 1 || b.ix.Findings.Held()[0].Kind != UntakenMergeKind { - t.Fatalf("a head whose message is not read: taken in %v, findings %+v: want none taken in and %s raised", done, b.ix.Findings.Held(), UntakenMergeKind) - } - // Its message is there: it is taken in, its taken key moves and the finding is cleared. - b.publish(t, b.merge(30, head, "tree-avahi-3", "")) - if done := b.ix.TakeInBehind(ctx); len(done) != 1 || len(b.ix.Findings.Held()) != 0 || b.commit(t, "avahi") != head { - t.Fatalf("a head with its message: taken in %v, findings %+v, avahi at %s: want it taken in at %s and the finding cleared", done, b.ix.Findings.Held(), b.commit(t, "avahi"), head) - } - if v, _, _, _ := b.ix.Taken.Get(ctx, key); string(v) == string(data) { - t.Errorf("taken.%s still names %s", key, old) - } - // A head the taken merge already holds (the source not yet moved to a merge just taken in) is left. - b.ix.source = sourceNow{rev: 4, heads: map[string]string{catalogueRepo: old}} - if done := b.ix.TakeInBehind(ctx); len(done) != 0 { - t.Errorf("a head older than the taken merge: taken in %v, want none", done) - } - // A source revision starts the comparison by itself (no timer): the taken key is put back to the old merge and the source - // names the head again. - b.ix.Taken.(*memStore).put(key, data) - if err := applyS(b.ix, ctx, 5, sourceOf(catalogueRepo, head)); err != nil { - t.Fatal(err) - } - b.ix.catchUpWG.Wait() - if v, _, _, _ := b.ix.Taken.Get(ctx, key); string(v) == string(data) { - t.Errorf("a source revision naming %s left taken.%s at %s", head, key, old) - } -} - -// Design 47 G A-463a: a head the pending holds (a group member waiting for its group's end) is not behind: it raises no finding -// and is not taken in alone. -func TestAHeadThePendingHoldsIsNotBehind(t *testing.T) { - ctx := context.Background() - b := newTakeInBed(t) - b.ix.Taken, b.ix.Findings = newMem(), newFindings() - key, _ := trunk.RepositoryKey(catalogueRepo) - old, head := sha('7'), sha('8') - data, _ := takein.EncodeTaken(takein.Taken{Repository: trunk.RepositoryName(catalogueRepo), Merge: old}) - b.ix.Taken.(*memStore).put(key, data) - b.ix.Contains = func(_ context.Context, _, commit, in string) (bool, error) { return commit == in || commit == old, nil } - b.publish(t, b.merge(30, head, "tree-avahi-3", "")) - // The head waits in pending as a member of a group not ended. - held := b.merge(30, head, "tree-avahi-3", "") - raw, _ := forgemsg.EncodeForge(held) - pv, _ := json.Marshal(Pending{V: 1, Forge: raw, Group: "feat", Since: time.Now(), LastWriter: "x"}) - b.pending.put(takein.PendingKey(1), pv) - b.ix.source = sourceNow{rev: 3, heads: map[string]string{catalogueRepo: head}} - if done := b.ix.TakeInBehind(ctx); len(done) != 0 || len(b.ix.Findings.Held()) != 0 || b.commit(t, "avahi") == head { - t.Fatalf("a head the pending holds: taken in %v, findings %+v, avahi at %s: want none taken in and none raised", done, b.ix.Findings.Held(), b.commit(t, "avahi")) - } -} - -// Design 47 G A-463a: a source revision heard while a catch-up runs is compared again when that catch-up ends. -func TestASourceRevisionHeardDuringACatchUpIsComparedWhenItEnds(t *testing.T) { - ctx := context.Background() - b := newTakeInBed(t) - b.ix.Taken, b.ix.Findings = newMem(), newFindings() - key, _ := trunk.RepositoryKey(catalogueRepo) - h1, h2 := sha('8'), sha('9') - b.ix.Contains = func(_ context.Context, _, commit, in string) (bool, error) { - return commit == in || (commit == h1 && in == h2), nil - } - b.publish(t, b.merge(30, h1, "tree-avahi-3", "")) - b.publish(t, b.merge(31, h2, "tree-avahi-4", "")) - b.ix.source = sourceNow{rev: 3, heads: map[string]string{catalogueRepo: h1}} - inner := b.ix.Message - heard := false - b.ix.Message = func(c context.Context, r, m string, n int) ([]byte, error) { - if !heard { - heard = true // the catch-up of h1 runs: a new source revision is heard now - if err := applyS(b.ix, ctx, 4, sourceOf(catalogueRepo, h2)); err != nil { - t.Error(err) - } - } - return inner(c, r, m, n) - } - b.ix.CatchUp(ctx) - b.ix.catchUpWG.Wait() - v, _, _, _ := b.ix.Taken.Get(ctx, key) - if got, _ := takein.DecodeTaken(v); got.Merge != h2 { - t.Errorf("taken after the catch-up = %s, want %s: the revision heard during it is compared afterwards\n%s", got.Merge, h2, b.logs) - } -} diff --git a/modules/mesh-delivery-indexer/internal/index/bus_state.go b/modules/mesh-delivery-indexer/internal/index/bus_state.go index c921e53b..98b50afd 100644 --- a/modules/mesh-delivery-indexer/internal/index/bus_state.go +++ b/modules/mesh-delivery-indexer/internal/index/bus_state.go @@ -13,8 +13,8 @@ import ( // every trunk module whose accepted manifest asks for bus state, then each manifest-accepted after its revision. const BusStateTool = "indexer_bus_state" -// BusStateModule is one module of BusStateAnswer: the digest of its trunk's accepted manifest, the verdicts state's -// revision of that verdict (the one its manifest-accepted carries, ADR 0622 §2), and its bus-state entries whole. +// BusStateModule is one module of BusStateAnswer: the digest of its trunk's manifest, the modules state's revision of its +// value, and its bus-state entries whole. No manifest is judged at the take-in (design 47 H §743, §745). type BusStateModule struct { Module string `json:"module"` Digest string `json:"digest"` @@ -22,25 +22,25 @@ type BusStateModule struct { BusState []catalogue.BusStateEntry `json:"bus-state"` } -// BusStateAnswer is indexer_bus_state's answer: Revision is the verdicts state's mark, read before anything else, so -// every manifest-accepted above it is a change after this answer (ADR 0615 §4); Modules in module order. +// BusStateAnswer is indexer_bus_state's answer: Revision is the modules state's mark, read before anything else, so every +// change of modules above it is a change after this answer (ADR 0615 §4); Modules in module order. type BusStateAnswer struct { Revision uint64 `json:"revision"` Modules []BusStateModule `json:"modules"` } -// BusStateOf answers indexer_bus_state from the indexer's own states: its trunk modules, the judge's verdicts and the -// manifests it keeps, and the marks the views family says them up to. A module refused or waiting, or whose manifest -// asks for no bus state, is left out; a value that cannot be read is an error, never a module silently left out. -func BusStateOf(modules, verdicts, manifests, marks views.Source) (BusStateAnswer, error) { +// BusStateOf answers indexer_bus_state from the indexer's own states: its trunk modules and the manifests it keeps, and +// the marks the views family says them up to. A module whose manifest asks for no bus state is left out; a value that +// cannot be read is an error, never a module silently left out. +func BusStateOf(modules, manifests, marks views.Source) (BusStateAnswer, error) { out := BusStateAnswer{Modules: []BusStateModule{}} if marks != nil { - if e, ok, err := marks.Get(VerdictsState); err != nil { - return BusStateAnswer{}, fmt.Errorf("the mark of %s cannot be read: %w", VerdictsState, err) + if e, ok, err := marks.Get(ModulesState); err != nil { + return BusStateAnswer{}, fmt.Errorf("the mark of %s cannot be read: %w", ModulesState, err) } else if ok { m, err := views.DecodeMark(e.Value) if err != nil { - return BusStateAnswer{}, fmt.Errorf("the mark of %s: %w", VerdictsState, err) + return BusStateAnswer{}, fmt.Errorf("the mark of %s: %w", ModulesState, err) } out.Revision = m.Revision } @@ -60,22 +60,7 @@ func BusStateOf(modules, verdicts, manifests, marks views.Source) (BusStateAnswe } mod, err := meshcatalogclient.DecodeModule(e.Value) if err != nil || mod.Manifest == "" { - continue // no manifest held for it, so no verdict - } - key, err := meshcatalogclient.VerdictsKey(mod.Manifest) - if err != nil { - continue - } - ve, ok, err := verdicts.Get(key) - if err != nil { - return BusStateAnswer{}, fmt.Errorf("the verdict on %s cannot be read: %w", name, err) - } - if !ok { - continue - } - v, err := meshcatalogclient.DecodeVerdict(ve.Value) - if err != nil || v.State != meshcatalogclient.VerdictAccepted { - continue + continue // no manifest held for it } me, ok, err := manifests.Get(mod.Manifest) if err != nil { @@ -90,12 +75,12 @@ func BusStateOf(modules, verdicts, manifests, marks views.Source) (BusStateAnswe } m, err := catalogue.Decode([]byte(kept.Manifest)) if err != nil { - return BusStateAnswer{}, fmt.Errorf("the accepted manifest of %s does not read: %w", name, err) + return BusStateAnswer{}, fmt.Errorf("the manifest of %s does not read: %w", name, err) } if len(m.BusState) == 0 { continue } - out.Modules = append(out.Modules, BusStateModule{Module: name, Digest: mod.Manifest, Revision: ve.Revision, BusState: m.BusState}) + out.Modules = append(out.Modules, BusStateModule{Module: name, Digest: mod.Manifest, Revision: e.Revision, BusState: m.BusState}) } return out, nil } diff --git a/modules/mesh-delivery-indexer/internal/index/candidates.go b/modules/mesh-delivery-indexer/internal/index/candidates.go index 5a9f8bd7..f4170329 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidates.go +++ b/modules/mesh-delivery-indexer/internal/index/candidates.go @@ -70,14 +70,9 @@ type Indexer struct { Predefined Store // Taken is the taken bucket: per repository the last merge taken in (taken.go, ADR 0556); nil writes none. Taken Store - // Verdicts is the verdicts state (verdicts.go, novox/hq ADR 0578 §1); nil judges nothing. - Verdicts Store // OldSections is the state of how many accepted modules use each old section (oldsections.go, issue 1966 D4); nil keeps no // count and the judge refuses no use of an old section. OldSections Store - // Unjudged is the state of the manifest digests a take-in wrote and no judge pass judged yet (unjudged.go, issue 1966); - // nil keeps them only in the take-in's memory. - Unjudged Store // Requirements is the requirements state (requirements.go, DLV-I7): what each consumer's go.sum requires of each mesh // Go module, kept at each merge; nil keeps none. Requirements Store @@ -85,12 +80,6 @@ type Indexer struct { reqKept map[string]bool // consumers whose go.sum this process kept (requirementsKept) // JudgeID names the judge a verdict says made it; empty is judge.Identity, the running program's. JudgeID string - // SayRefused says the event manifest-refused for a refusal newly written; nil says nothing. - SayRefused func(ctx context.Context, r meshcatalog.ManifestRefusedEvent) - // SayAccepted publishes the event manifest-accepted a under the bus's duplicate id id (accepted.go; issue 1966 D3, - // DLV-I9): for every accepted verdict newly written, carrying the entries whole, and an empty one for a module leaving - // the trunk; nil says nothing. - SayAccepted func(ctx context.Context, a meshcatalog.ManifestAcceptedEvent, id string) error // Findings is the indexer's finding writer (design 47 G A-29), where a manifest-accepted whose publish failed twice is kept // until said (accepted.go, DLV-I9b); nil writes none. Findings FindingWriter @@ -258,7 +247,6 @@ func (ix *Indexer) Apply(ctx context.Context, f forgemsg.Forge) error { // ApplyMessage is Apply, and answers which modules a merged message wrote and kept (for the indexer's heard state). Each // applied merged message is said in one line: its repository, commit and pull request, and the modules written. func (ix *Indexer) ApplyMessage(ctx context.Context, f forgemsg.Forge) (Applied, error) { - ix.kickJudge(ctx) var applied Applied if f.Kind == forgemsg.ForgeOpen { return applied, ix.applyOpen(ctx, f) @@ -287,13 +275,6 @@ func (ix *Indexer) ApplyMessage(ctx context.Context, f forgemsg.Forge) (Applied, if err := ix.KeepPackageNames(ctx, f); err != nil { return applied, err } - // Every manifest judged again: a rule between manifests can turn on the one this merge moved (ADR 0578 §1). - // What this take-in newly wrote is fresh: judged in full (ADR 0579 §1). - if _, err := ix.JudgeAll(ctx, w.fresh); err != nil { - // Judged again at the next message or source revision heard (kickJudge), as a failed start pass is. - ix.judgePending.Store(true) - ix.logf("judge: the manifests are not judged after %s#%d: %v; judged again at the next message heard", f.Repository, f.Number, err) - } } ix.kickRetry(ctx) // a registered module the forge could not read is read again on a merge, in the background (issue 1945) lateMerge = w.late != "" @@ -314,20 +295,6 @@ func (ix *Indexer) ApplyMessage(ctx context.Context, f forgemsg.Forge) (Applied, if held, err = ix.heldModules(ctx); err != nil { return applied, err } - // Each manifest the pull carries is judged by this judge, unless it holds a verdict already that this judge made or - // that accepts it: a refusal an older judge made never stands for a candidate (or a digest back on the trunk). - over := map[string]string{} - for _, digest := range digests { - if ix.Manifests == nil { - break - } - if value, _, found, err := ix.Manifests.Get(ctx, digest); err == nil && found { - if kept, err := meshcatalogclient.DecodeManifest(value); err == nil { - over[kept.Module] = digest - } - } - } - ix.judgeDigests(ctx, over, fmt.Sprintf("%s#%d", f.Repository, f.Number)) } return applied, ix.writeCandidate(ctx, f, digests, held) } diff --git a/modules/mesh-delivery-indexer/internal/index/control_records_test.go b/modules/mesh-delivery-indexer/internal/index/control_records_test.go index 87cc2cf0..3ad07418 100644 --- a/modules/mesh-delivery-indexer/internal/index/control_records_test.go +++ b/modules/mesh-delivery-indexer/internal/index/control_records_test.go @@ -42,9 +42,9 @@ func TestTheControlRecords(t *testing.T) { t.Errorf("rows = %+v\nwant %+v", got, want) } }) - // DLV-I11 (design 25 § 12.2): indexer_bus_state answers every trunk module whose accepted manifest asks for bus state, - // with the verdict's revision, at the verdicts state's mark; a refused one, one asking none, and a module with no - // manifest held are left out. + // DLV-I11 (design 25 § 12.2): indexer_bus_state answers every trunk module whose manifest asks for bus state, with its + // value's revision, at the modules state's mark; one asking none and a module with no manifest held are left out. No + // verdict is read (I7, design 47 H §745): a manifest the judge once refused is answered as any other. t.Run("indexer_bus_state", func(t *testing.T) { src := memSource{} dAsk, dNone, dRefused := strings.Repeat("a", 64), strings.Repeat("b", 64), strings.Repeat("c", 64) @@ -55,35 +55,25 @@ func TestTheControlRecords(t *testing.T) { } s[key] = &views.Entry{Value: raw, Revision: rev} } - modules, verdicts, manifests, marks := memSource{}, memSource{}, memSource{}, src - putVerdict := func(digest string, rev uint64, v meshcatalog.VerdictsByDigestValue) { - raw, err := meshcatalogclient.EncodeVerdict(v) - if err != nil { - t.Fatal(err) - } - verdicts[digest] = &views.Entry{Value: raw, Revision: rev} - } + modules, manifests, marks := memSource{}, memSource{}, src for name, digest := range map[string]string{"cache": dAsk, "plain": dNone, "bad": dRefused, "bare": ""} { raw, err := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: "src2:" + strings.Repeat("a", 64), Manifest: digest, Repository: "novox/mesh-catalog", Commit: "c"}) if err != nil { t.Fatal(err) } - modules[name] = &views.Entry{Value: raw, Revision: 1} + modules[name] = &views.Entry{Value: raw, Revision: map[string]uint64{"cache": 7, "bad": 9}[name]} } asks := `{"module":"cache","version":"1.0","requires":[{"name":"c","storage":"key-value","template":"state"}]}` for digest, raw := range map[string]string{dAsk: asks, dNone: `{"module":"plain","version":"1.0"}`, dRefused: asks} { put(manifests, digest, 1, meshcatalog.ManifestsByDigestValue{V: keys.Version, Module: "m", Manifest: raw}) } - putVerdict(dAsk, 7, meshcatalog.VerdictsByDigestValue{Module: "cache", State: meshcatalogclient.VerdictAccepted, Judge: "j"}) - putVerdict(dNone, 8, meshcatalog.VerdictsByDigestValue{Module: "plain", State: meshcatalogclient.VerdictAccepted, Judge: "j"}) - putVerdict(dRefused, 9, meshcatalog.VerdictsByDigestValue{Module: "bad", State: meshcatalogclient.VerdictRefused, Judge: "j", Problems: []string{"p"}}) - put(marks, VerdictsState, 1, views.Mark{V: views.Version, State: VerdictsState, Revision: 9}) - got, err := BusStateOf(modules, verdicts, manifests, marks) - if err != nil || got.Revision != 9 || len(got.Modules) != 1 || got.Modules[0].Module != "cache" || - got.Modules[0].Digest != dAsk || got.Modules[0].Revision != 7 || len(got.Modules[0].BusState) != 1 || - got.Modules[0].BusState[0].Name != "c" { - t.Errorf("indexer_bus_state = %+v (%v)", got, err) + put(marks, ModulesState, 1, views.Mark{V: views.Version, State: ModulesState, Revision: 9}) + got, err := BusStateOf(modules, manifests, marks) + if err != nil || got.Revision != 9 || len(got.Modules) != 2 || got.Modules[0].Module != "bad" || got.Modules[1].Module != "cache" || + got.Modules[1].Digest != dAsk || got.Modules[1].Revision != 7 || len(got.Modules[1].BusState) != 1 || + got.Modules[1].BusState[0].Name != "c" { + t.Errorf("indexer_bus_state = %+v (%v); want bad and cache, at the modules mark", got, err) } }) // DLV-I7 (REG S15, the lead's ruling): the requirements state keeps, per mesh Go module, the highest version each @@ -120,34 +110,32 @@ func TestTheControlRecords(t *testing.T) { // and source, says seat-definition-changed only when a digest moved, keeps the trunk's entry over one that does not // parse (its finding), removes a pull request's entries when it closes, keeps a verdict's out-of-step files, and // raises generated-types-out-of-step from a trunk build end, cleared only by a clean one of the same module. - // Milestone 47/G W4.15, design 47 test 54: the seat git not answering at start holds the judging at most its bound, which - // then runs on the kept definitions with the wait said; the read once the forge is heard holds the view and judges again. - t.Run("W4.15 the start waits for the seat git, judges from kept definitions, and judges again once it answers", func(t *testing.T) { + // Milestone 47/G W4.15, design 47 test 54: the seat git not answering at start is waited for at most its bound, the wait + // said; the read once the forge is heard holds the view. No manifest is judged (design 47 H §745). + t.Run("W4.15 the start waits for the seat git and reads again once it answers", func(t *testing.T) { answers := []error{fmt.Errorf("mesh-contracts: %w", errForgeRead), fmt.Errorf("again: %w", errForgeRead), nil} var said []string - judged := 0 d := &DefinitionsAtStart{Bound: time.Millisecond, Waits: waitsFake{&said}, - Read: func(context.Context) (int, error) { err := answers[0]; answers = answers[1:]; return 1, err }, - Judge: func(context.Context) error { judged++; return nil }} + Read: func(context.Context) (int, error) { err := answers[0]; answers = answers[1:]; return 1, err }} ctx := context.Background() d.Start(ctx) if !d.Waiting() || len(said) != 1 || said[0] != "waits "+GitDefinitionsView { t.Fatalf("waiting %v, said %q: want the wait for the seat git said once", d.Waiting(), said) } - if !d.Ready(ctx) || judged != 0 { - t.Fatalf("the start past its bound: want it judged from the kept definitions, judged again %d", judged) + if !d.Ready(ctx) { + t.Fatalf("the start past its bound: want it ready on the kept definitions") } d.Heard(ctx) // the forge heard, the seat still not answering: the wait stands, said once - if !d.Waiting() || len(said) != 1 || judged != 0 { - t.Fatalf("a read still not answered: waiting %v, said %q, judged %d", d.Waiting(), said, judged) + if !d.Waiting() || len(said) != 1 { + t.Fatalf("a read still not answered: waiting %v, said %q", d.Waiting(), said) } d.Heard(ctx) - if d.Waiting() || len(said) != 2 || said[1] != "holds "+GitDefinitionsView || judged != 1 || d.Ready(ctx) { - t.Errorf("the seat answering: waiting %v, said %q, judged again %d: want the view held and every manifest judged again once", d.Waiting(), said, judged) + if d.Waiting() || len(said) != 2 || said[1] != "holds "+GitDefinitionsView || d.Ready(ctx) { + t.Errorf("the seat answering: waiting %v, said %q: want the view held", d.Waiting(), said) } - d.Heard(ctx) // nothing waits: no read, no judging - if judged != 1 || len(answers) != 0 { - t.Errorf("a forge message with nothing waiting read or judged again") + d.Heard(ctx) // nothing waits: no read + if len(answers) != 0 { + t.Errorf("a forge message with nothing waiting read again") } }) t.Run("seat definitions", func(t *testing.T) { @@ -484,23 +472,6 @@ func TestTheControlRecords(t *testing.T) { ts.Name != "@novox/mesh-sdk" || ts.Package != "mesh-sdk/ts" || ts.GoModule != "" || len(named) != 2 { t.Errorf("names at the merge = %+v", named) } - // The judge's known packages (DLV-I14) are those names, never the builder's /. - if known := ix.knownPackages(ctx); len(known) != 2 || !known[sdkGo] || !known["@novox/mesh-sdk"] { - t.Errorf("the judge's known packages = %v", known) - } - // Issue 1966: a Go module the trunk holds a per-module record of is known by its module path, though no series - // names it; a package inside it is not (design 32). - resticMod, at := "git.novox.be/novox/mesh-delivery/modules/restic", time.Unix(1, 0) - rec, err := EncodePublishedPackages(meshcatalog.PackagesByModuleValue{Module: resticMod, Repository: "novox/mesh-delivery", Dir: "modules/restic", At: &at, - Commit: strings.Repeat("a", 40), Requires: "h1:r", Packages: []meshcatalog.PackagesByModuleValuePackagesItem{{Package: resticMod + "/backupgood", Digest: "h1:a"}}}) - if err != nil { - t.Fatal(err) - } - ix.Packages.(*memStore).vals["restic"] = rec - if known := ix.knownPackages(ctx); len(known) != 3 || !known[resticMod] || known[resticMod+"/backupgood"] || !known[sdkGo] || known["restic/go"] || known["unknown/pkg"] { - t.Errorf("the judge's known packages with a module record = %v", known) - } - delete(ix.Packages.(*memStore).vals, "restic") delete(ix.Packages.(*memStore).vals, SeriesKey("@novox/mesh-sdk", "0.3")) for _, h := range []HeardVersion{ {Package: sdkGo, Seat: "go-package-registry", Version: "0.1.15", Commit: "c15"}, @@ -563,7 +534,7 @@ func TestTheControlRecords(t *testing.T) { if err != nil { t.Fatal(err) } - modules[name] = &views.Entry{Value: raw, Revision: 1} + modules[name] = &views.Entry{Value: raw, Revision: map[string]uint64{"cache": 7, "bad": 9}[name]} } got, err := PackageUsersOf(modules, manifests, packages, "") want := []PackageUses{ diff --git a/modules/mesh-delivery-indexer/internal/index/events.go b/modules/mesh-delivery-indexer/internal/index/events.go deleted file mode 100644 index 225cf090..00000000 --- a/modules/mesh-delivery-indexer/internal/index/events.go +++ /dev/null @@ -1,52 +0,0 @@ -package index - -import ( - "encoding/json" - "strconv" - - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" - "git.novox.be/novox/mesh-sdk/go/catalogue" - "git.novox.be/novox/mesh-sdk/go/keys" -) - -// The bodies of the seat's events manifest-accepted and manifest-refused are the contract's types; the indexer, who says -// them, makes them from a verdict (novox/hq ADR 0634 §1: the logic of one holder stays with it). - -// refusedOf is the body of manifest-refused for the verdict v on the manifest of digest. -func refusedOf(digest string, v meshcatalog.VerdictsByDigestValue) meshcatalog.ManifestRefusedEvent { - return meshcatalog.ManifestRefusedEvent{V: keys.Version, Module: v.Module, Digest: digest, Judge: v.Judge, Problems: v.Problems} -} - -// acceptedEvent is the body of manifest-accepted for the verdict v accepting the manifest m of digest, its bus-state -// entries whole (novox/hq ADR 0595 §4); an empty list when m asks for none. -func acceptedEvent(digest string, v meshcatalog.VerdictsByDigestValue, m catalogue.Manifest) meshcatalog.ManifestAcceptedEvent { - // problems are the manifest validator's strings, the verdict's warnings in their order, an empty list when none (design 47 G §668, §692). - problems := append([]string{}, v.Warnings...) - return meshcatalog.ManifestAcceptedEvent{V: keys.Version, Module: v.Module, Digest: digest, Judge: v.Judge, BusState: busStateOf(m.BusState), Problems: problems} -} - -// busStateOf is the manifest's bus-state entries in the event's form (the same JSON). -func busStateOf(entries []catalogue.BusStateEntry) []meshcatalog.ManifestAcceptedEventBusStateItem { - out := []meshcatalog.ManifestAcceptedEventBusStateItem{} - if len(entries) == 0 { - return out - } - raw, err := json.Marshal(entries) - if err == nil { - err = json.Unmarshal(raw, &out) - } - if err != nil { - return []meshcatalog.ManifestAcceptedEventBusStateItem{} - } - return out -} - -// acceptedEventID is the event's id, one per manifest digest and judge, so the bus takes it once: -// `manifest-accepted--`. The empty one of a module leaving the trunk carries no digest, so it is named by -// its module and the indexer's revision: `manifest-accepted-left--`, so two leaves are both kept. -func acceptedEventID(a meshcatalog.ManifestAcceptedEvent) string { - if a.Digest == "" { - return meshcatalog.EventManifestAccepted + "-left-" + a.Module + "-" + strconv.FormatInt(a.Revision, 10) - } - return meshcatalog.EventManifestAccepted + "-" + a.Digest + "-" + a.Judge -} diff --git a/modules/mesh-delivery-indexer/internal/index/helpers_test.go b/modules/mesh-delivery-indexer/internal/index/helpers_test.go index 50bd2b82..8dd5af84 100644 --- a/modules/mesh-delivery-indexer/internal/index/helpers_test.go +++ b/modules/mesh-delivery-indexer/internal/index/helpers_test.go @@ -7,7 +7,6 @@ import ( "slices" "sync" - mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" "git.novox.be/novox/mesh-sdk/go/views" ) @@ -121,11 +120,6 @@ func (m *memFindings) Held() []mcseat.Finding { return out } -// unsaidKey is the key of module's unsaid finding. -func unsaidKey(module string) string { - return views.FindingKey(mcc.ScopeModule, keyID(unsaidID(testSelf, module)), UnsaidKind) -} - // testSelf is the module the tests' indexer runs as, its credential's: its findings, its own entry and its graph key are named // from it, never from a constant (design 47 G A-296). const testSelf = "acme-indexer" diff --git a/modules/mesh-delivery-indexer/internal/index/kept_trunk_test.go b/modules/mesh-delivery-indexer/internal/index/kept_trunk_test.go index 0dd1eb1f..f871def7 100644 --- a/modules/mesh-delivery-indexer/internal/index/kept_trunk_test.go +++ b/modules/mesh-delivery-indexer/internal/index/kept_trunk_test.go @@ -97,7 +97,7 @@ func TestTheKeptTrunkIsReadAtTheStartAndOncePerHeardRevision(t *testing.T) { if raw, err := kept.Manifest(meshcatalog.BareManifestDigest(keptDigest("ca-trust"))); err != nil || raw != `{"module":"ca-trust"}` { t.Errorf("the held manifest = %q (%v), want the entry's manifest", raw, err) } - if a, err := Revision(context.Background(), objects, kept, "avahi", "avahi", meshcatalog.RevisionInput{}); err != nil || a.Revision.Revision != 2 || len(a.Entries) != 2 { + if a, err := Revision(context.Background(), objects, kept, meshcatalog.RevisionInput{}); err != nil || a.Revision.Revision != 2 || len(a.Entries) != 2 { t.Errorf("revision of the newest answered %+v (%v), want revision 2 whole", a, err) } if got := objects.n.Load() - listed; got != 0 { @@ -277,11 +277,11 @@ func TestTheNewestRevisionReadersReadAgainWhenTheRevisionIsRemovedUnderThem(t *t return len(a.Behind) == 1 && a.Behind[0].Kind == 5 && strings.Contains(a.Behind[0].Newest, "fp2"), err }}, "Revision": {"", func(ix *Indexer) (bool, error) { - a, err := Revision(ctx, ix.Revisions, nil, "avahi", "avahi", meshcatalog.RevisionInput{}) + a, err := Revision(ctx, ix.Revisions, nil, meshcatalog.RevisionInput{}) return a.Revision != nil && a.Revision.Revision == 2, err }}, "Revision of a branch": {"dev", func(ix *Indexer) (bool, error) { - a, err := Revision(ctx, ix.Revisions, nil, "avahi", "avahi", meshcatalog.RevisionInput{Branch: "dev"}) + a, err := Revision(ctx, ix.Revisions, nil, meshcatalog.RevisionInput{Branch: "dev"}) return a.Revision != nil && a.Revision.Revision == 2 && len(a.Entries) == 1 && a.Entries[0].Fp == "fp2", err }}, } diff --git a/modules/mesh-delivery-indexer/internal/index/modules.go b/modules/mesh-delivery-indexer/internal/index/modules.go index 8f2ed205..ec1f9762 100644 --- a/modules/mesh-delivery-indexer/internal/index/modules.go +++ b/modules/mesh-delivery-indexer/internal/index/modules.go @@ -260,9 +260,6 @@ func (ix *Indexer) dropRemoved(ctx context.Context, repository string, held map[ return fmt.Errorf("modules.%s: %w", key, err) } ix.logf("modules.%s deleted: %s holds no module of that name at its trunk (issue 1900)", key, repository) - // The bus module hears the module asks nothing now, so its entries reach orphans (design 25 § 12.2, DLV-I9). - a, id := leftOf(key, left, ix.judgeID()) - _ = ix.sayAccepted(ctx, a, id) } for _, key := range handed { if err := ix.handOver(ctx, key); err != nil { diff --git a/modules/mesh-delivery-indexer/internal/index/moves.go b/modules/mesh-delivery-indexer/internal/index/moves.go index 841ebf15..9419dd1e 100644 --- a/modules/mesh-delivery-indexer/internal/index/moves.go +++ b/modules/mesh-delivery-indexer/internal/index/moves.go @@ -120,13 +120,8 @@ func (ix *Indexer) handOver(ctx context.Context, key string) error { // handOverFrom writes the module at key from the merge f, which names it, in place. func (ix *Indexer) handOverFrom(ctx context.Context, key string, f forgemsg.Forge) error { - w, err := ix.applyMoving(ctx, f, false, map[string]bool{key: true}) - if err != nil { + if _, err := ix.applyMoving(ctx, f, false, map[string]bool{key: true}); err != nil { return fmt.Errorf("modules.%s, moving to %s: %w", key, f.Repository, err) } - // A module moved in is judged as a merge's are (ADR 0578 §1, ADR 0579 §1). - if _, err := ix.JudgeAll(ctx, w.fresh); err != nil { - ix.logf("judge: the manifests are not judged after moving %s to %s: %v", key, f.Repository, err) - } return nil } diff --git a/modules/mesh-delivery-indexer/internal/index/names.go b/modules/mesh-delivery-indexer/internal/index/names.go index c096efa2..ddcdf3cd 100644 --- a/modules/mesh-delivery-indexer/internal/index/names.go +++ b/modules/mesh-delivery-indexer/internal/index/names.go @@ -10,8 +10,6 @@ const ( ModulesState = "modules" ManifestsState = "manifests" ManifestsHistory = 1 - // VerdictsState is in verdicts.go. - VerdictsHistory = 5 PredefinedState = "predefined" PackagesState = "packages" PackagesHistory = 5 diff --git a/modules/mesh-delivery-indexer/internal/index/ownbuild.go b/modules/mesh-delivery-indexer/internal/index/ownbuild.go new file mode 100644 index 00000000..4f89e959 --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/ownbuild.go @@ -0,0 +1,84 @@ +package index + +import ( + "context" + "encoding/json" + "errors" + "time" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" +) + +// OwnBuildState is the indexer's state of the build it runs (the bucket mesh-delivery-indexer_own-build, one key, OwnBuildKey): +// written once, at the process's start by FixOwnBuild, so indexer_pending says which build this process knows it runs, and since +// when. The build only names the writer of what is written (A-476 b). +const ( + OwnBuildStateName = "own-build" + OwnBuildKey = "state" +) + +// OwnBuild is the value of the own-build state. +type OwnBuild struct { + V int `json:"v"` + Known bool `json:"known"` + FP string `json:"fp,omitempty"` + Since time.Time `json:"since"` + Why string `json:"why,omitempty"` +} + +// FixOwnBuild fixes, once and at the process's start, which build of the indexer this program is: asked of the node-engine's +// seat (OwnBuild; never the trunk's value of the indexer, which names a build that may be merged and not delivered), before +// this process hears a merge. It is never asked again: while the node-engine replaces the indexer the seat already names the +// successor, and an old process that learnt it later would take revisions in with its old judge. Where the start's ask is not +// answered the build stays unknown for the life of this process, said in the own-build state and the journal; nothing waits for it. +func (ix *Indexer) FixOwnBuild(ctx context.Context) { + ix.ownMu.Lock() + if ix.ownFixed || ix.OwnBuild == nil { + ix.ownMu.Unlock() + return + } + ix.ownFixed = true + ix.ownMu.Unlock() + fp, err := ix.OwnBuild(ctx) + if err == nil && fp == "" { + err = errors.New("the node-engine names no build of the indexer") + } + st := OwnBuild{V: 1, Since: ix.now().UTC()} + if err != nil { + st.Why = err.Error() + ix.logf("judge: the build this program runs is not known (%v); merges are taken in all the same, and it is learnt only at the start", err) + } else { + st.Known, st.FP = true, fp + ix.logf("judge: this program runs the indexer build %s, fixed here at its start", fp) + } + ix.ownMu.Lock() + ix.ownFP, ix.ownSince, ix.ownWhy = fp, st.Since, st.Why + ix.ownMu.Unlock() + if ix.OwnBuildState == nil { + return + } + data, _ := json.Marshal(st) + if _, rev, found, gerr := ix.OwnBuildState.Get(ctx, OwnBuildKey); gerr == nil && found { + err = ix.OwnBuildState.Update(ctx, OwnBuildKey, data, rev) + } else if gerr == nil { + err = ix.OwnBuildState.Create(ctx, OwnBuildKey, data) + } else { + err = gerr + } + if err != nil { + ix.logf("the own-build state cannot be written: %v", err) + } +} + +// ownBuildIsNewest says whether the build of the indexer this program runs (ownFP, from the node-engine) is the newest the +// trunk holds: the indexer's own modules value names it. A newer build merged and not yet delivered ends it at the trunk's +// merge, as one that pins a changed mesh-sdk always is; false when either is unknown. +func (ix *Indexer) ownBuildIsNewest(held map[string]meshcatalog.ModulesByModuleValue) bool { + fp := held[ix.Module].Fp2 + if fp == "" { + return false + } + ix.ownMu.Lock() + defer ix.ownMu.Unlock() + return ix.ownFP != "" && ix.ownFP == fp +} diff --git a/modules/mesh-delivery-indexer/internal/index/pending.go b/modules/mesh-delivery-indexer/internal/index/pending.go index aac05b07..a06bad2f 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending.go +++ b/modules/mesh-delivery-indexer/internal/index/pending.go @@ -366,15 +366,6 @@ func revisionWait(kind string, ready []PendingEntry, waitsFor string) takein.Tak // pendingOnBus says whether the pending state is on the bus: a state this delivery brings is bound when it appears // (issue 1790), and until then every merge is taken in at once, so the build this program runs is not needed. A read // that fails otherwise counts as on the bus: the step that follows answers the error. -// errVerdictsNotYet is a take-in held because the verdicts state is not on the bus yet (takeInWholeBy). -var errVerdictsNotYet = errors.New("the verdicts state is not on the bus yet: nothing is taken in until it is") - -// verdictsOnBus says the verdicts state is on the bus (a take-in judges what it writes there). -func (ix *Indexer) verdictsOnBus(ctx context.Context) bool { - _, _, _, err := ix.Verdicts.Get(ctx, "on-the-bus") - return !errors.Is(err, process.ErrNotYet) -} - func (ix *Indexer) pendingOnBus(ctx context.Context) bool { _, _, _, err := ix.Pending.Get(ctx, takein.PendingKey(0)) return !errors.Is(err, process.ErrNotYet) @@ -760,9 +751,6 @@ func (ix *Indexer) evaluate(ctx context.Context, own string, start bool) (Applie if errors.Is(err, errClaimLost) { return a, "claimed by another take-in", nil } - if errors.Is(err, errVerdictsNotYet) { - return a, "the verdicts state is not on the bus yet: the revision waits for it, taken in when it binds", nil - } if errors.Is(err, ErrObjectsFull) { ix.holdFull(ctx, ready) return a, ix.waitingLine(w) + "; held: " + HeldObjectsFull, nil @@ -918,7 +906,7 @@ func (ix *Indexer) dry() *dryRun { d := &dryRun{} x := &Indexer{Module: ix.Module, Manifests: ix.Manifests, Packages: ix.Packages, JudgeID: ix.JudgeID, Files: ix.Files, GoSum: ix.GoSum, GoMod: ix.GoMod, Now: ix.Now, Builds: ix.Builds, Named: ix.Named, Trunk: ix.Trunk, BranchHead: ix.BranchHead, Read: ix.Read, Contains: ix.Contains, - ReadOpen: nil, OwnBuild: nil, Message: ix.Message, SayRefused: nil, SayAccepted: nil, SayTakeIn: nil, Pending: nil, + ReadOpen: nil, OwnBuild: nil, Message: ix.Message, SayTakeIn: nil, Pending: nil, // The registry milestone's fields: what a take-in would write on the bus stays out of a dry run, so the states of the // requirements, findings, seat definitions, holds and their bounds are not overlaid, nothing is said, and no hold is // weighed (Holds and Counted nil hold nothing); ReadFile only reads. @@ -932,12 +920,9 @@ func (ix *Indexer) dry() *dryRun { x.Store, d.candidates = over(ix.Store) x.Modules, d.modules = over(ix.Modules) x.Graph, d.graph = over(ix.Graph) - x.Verdicts, d.verdicts = over(ix.Verdicts) x.Taken, d.taken = over(ix.Taken) x.Predefined, d.predefined = over(ix.Predefined) x.OldSections, d.oldSections = over(ix.OldSections) - // What the trunk holds unjudged is read through, as fresh; what the dry run would keep stays in its overlay (issue 1966). - x.Unjudged, _ = over(ix.Unjudged) ix.mu.Lock() x.source = sourceNow{rev: ix.source.rev, heads: maps.Clone(ix.source.heads)} x.previews = maps.Clone(ix.previews) @@ -1060,13 +1045,6 @@ func (ix *Indexer) takeInWhole(ctx context.Context, entries []PendingEntry, own, // takeInWholeBy is takeInWhole; who and why, of a hand act, are carried on the take-in.taken events. func (ix *Indexer) takeInWholeBy(ctx context.Context, entries []PendingEntry, own, by, who, why string) (Applied, error) { - if ix.Verdicts != nil && !ix.verdictsOnBus(ctx) && len(entries) > 0 { - // Every manifest a take-in writes is judged in it and its verdict said before take-in.taken (issue 1966 note 154, - // ADR 0668 §4): with the verdicts state not on the bus yet, nothing is taken in; its bind evaluates pending again. - ix.logf("take-in: %d merges wait, oldest %s: the state %s is not on the bus yet, so nothing is taken in until it is, and no manifest is written without its verdict", - len(entries), entries[0].Key, VerdictsState) - return Applied{}, errVerdictsNotYet - } claimed, ok := ix.claim(ctx, entries, own, by) if !ok { return Applied{}, errClaimLost @@ -1162,8 +1140,7 @@ func (ix *Indexer) takenWhat(ctx context.Context, claimed []PendingEntry, applie ix.logf("take-in: %s is written in source revision %d of %s with no fingerprint (%s): no pipeline entry", m, src, branch, e.None) continue } - out = append(out, meshcatalog.TakeInEntry{Module: m, Commit: e.Commit, Fingerprint: e.Fp, Repository: e.Repository, - Pull: pulls[strings.ToLower(e.Repository)]}) + out = append(out, meshcatalog.TakeInEntry{Module: m, Commit: e.Commit, Fingerprint: e.Fp, Repository: e.Repository}) } return branch, src, out } diff --git a/modules/mesh-delivery-indexer/internal/index/pending_test.go b/modules/mesh-delivery-indexer/internal/index/pending_test.go deleted file mode 100644 index de4ecf7c..00000000 --- a/modules/mesh-delivery-indexer/internal/index/pending_test.go +++ /dev/null @@ -1,1143 +0,0 @@ -package index - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "reflect" - "slices" - "strconv" - "strings" - "sync" - "testing" - "time" - - mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" - - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk" - stdio "git.novox.be/novox/mesh-sdk/go" - "git.novox.be/novox/mesh-sdk/go/buscompose" - "git.novox.be/novox/mesh-sdk/go/busretry" - "git.novox.be/novox/mesh-sdk/go/catalogue" - "git.novox.be/novox/mesh-sdk/go/process" - "github.com/nats-io/nats.go" -) - -// takeInBed is an indexer holding the catalogue's avahi and the indexer's own module, both with a fingerprint, its -// pending state and the take-in events it says; base is the catalogue merge it was built from. -type takeInBed struct { - ix *Indexer - mods *memStore - pending *memStore - said []takein.TakeIn - bridge map[string][]byte // what the forge bridge answers (bridge_merged_message), by repository@merge - base forgemsg.Forge - logs *strings.Builder -} - -func newTakeInBed(t *testing.T) *takeInBed { - t.Helper() - ix, mods, f := newFormIndexerUnmerged(t, mod{"avahi", catalogueManifest(t, "avahi")}, - mod{testSelf, renamed(t, "avahi", testSelf)}) - if err := ix.Apply(context.Background(), f); err != nil { - t.Fatal(err) - } - reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1") - reportToolchain(t, ix, "mesh-build-go", "ace", "go", "go1.27.1") - b := &takeInBed{ix: ix, mods: mods, pending: newMem(), bridge: map[string][]byte{}, base: f, logs: &strings.Builder{}} - ix.Verdicts = newMem() - b.pending.maxValue = buscompose.StateMaxValueBytes - ix.Pending = b.pending - ix.Message = func(_ context.Context, repository, merge string, number int) ([]byte, error) { - if data, ok := b.bridge[repository+"@"+merge]; ok { - return data, nil - } - return nil, errors.New("the gate holds no merge of " + repository + " with that commit") - } - ix.SayTakeIn = func(_ context.Context, t takein.TakeIn) error { b.said = append(b.said, t); return nil } - logf := ix.Logf - ix.Logf = func(format string, args ...any) { - b.logs.WriteString(format + "\n") - if logf != nil { - logf(format, args...) - } - } - own := b.fp(t, testSelf) - if own == "" || b.fp(t, "avahi") == "" { - t.Fatalf("the bed's modules have no fingerprint") - } - ix.OwnBuild = func(context.Context) (string, error) { return own, nil } - ix.OwnBuildState = newMem() - ix.FixOwnBuild(context.Background()) // fixed once, at the start - return b -} - -func (b *takeInBed) fp(t *testing.T, module string) string { - t.Helper() - m, _ := moduleOf(t, b.mods, module) - return m.Fp2 -} - -func (b *takeInBed) commit(t *testing.T, module string) string { - t.Helper() - m, _ := moduleOf(t, b.mods, module) - return m.Commit -} - -// merge is a catalogue merge at commit c whose trees of avahi and the indexer are given ("" keeps the base's). -func (b *takeInBed) merge(number int, c, avahi, indexer string) forgemsg.Forge { - f := b.base - f.Number, f.Merge, f.Head = number, c, c - f.Trees = map[string]string{} - for dir, tree := range b.base.Trees { - f.Trees[dir] = tree - } - if avahi != "" { - f.Trees["modules/avahi"] = avahi - } - if indexer != "" { - f.Trees["modules/"+testSelf] = indexer - } - return f -} - -// arriveHeld arrives f while the verdicts state is not on the bus, so the take-in holds it (ADR 0668 §4): the way a merge is -// left pending now that no merge waits for the indexer build it names (A-476 b). -func (b *takeInBed) arriveHeld(t *testing.T, seq uint64, f forgemsg.Forge) { - t.Helper() - bound := b.ix.Verdicts - b.ix.Verdicts = &memStore{unbound: process.ErrNotYet} - defer func() { b.ix.Verdicts = bound }() - b.arrive(t, seq, f) -} - -func (b *takeInBed) arrive(t *testing.T, seq uint64, f forgemsg.Forge) { - t.Helper() - b.publish(t, f) - if _, _, err := b.ix.Arrive(context.Background(), seq, f); err != nil { - t.Fatalf("arrive %d: %v", seq, err) - } -} - -// publish lets the bed's forge bridge answer the merged message f, as bridge_merged_message does. -func (b *takeInBed) publish(t *testing.T, f forgemsg.Forge) { - t.Helper() - data, err := forgemsg.EncodeForge(f) - if err != nil { - t.Fatal(err) - } - b.bridge[f.Repository+"@"+f.Merge] = data -} - -func (b *takeInBed) keys() []string { - k, _ := b.pending.Keys(context.Background()) - slices.Sort(k) - return k -} - -func (b *takeInBed) events() []string { - var out []string - for _, s := range b.said { - out = append(out, s.Kind+" "+s.Key) - } - return out -} - -func sha(c byte) string { return strings.Repeat(string(c), 40) } - -func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) - ctx := context.Background() - t.Run("indexer_fingerprints_again acts before it records the hand act and says a failed record", func(t *testing.T) { // issue 1966 - for _, c := range []struct { - name string - record func(what, why string) error - said string - }{ - {"recorded", func(string, string) error { return nil }, ""}, - {"record refused", func(string, string) error { return errors.New("no ledger") }, "not recorded: no ledger"}, - {"record never answers", func(string, string) error { select {} }, "not recorded within"}, - } { - t.Run(c.name, func(t *testing.T) { - var asked []string - ask := func(key string, _ []byte) (TakeInAsk, error) { - asked = append(asked, key) - return TakeInAsk{Result: &TakenIn{Module: "all", Stored: true}}, nil - } - tool := FingerprintsTools(ask, time.Now, c.record, 10*time.Millisecond)[0] - if _, err := tool.Run(map[string]any{"module": "avahi"}); err == nil { - t.Fatal("a hand act with no why was done") - } - if len(asked) != 0 { - t.Fatalf("asked the indexer without a why: %v", asked) - } - got, err := tool.Run(map[string]any{"module": "avahi", "why": "the packages read came back empty"}) - if err != nil || len(asked) != 1 || !strings.HasPrefix(asked[0], "fingerprints.avahi.") { - t.Fatalf("run: %v, asked %v", err, asked) - } - said, _ := got.(map[string]any)["hand-act"].(string) - if !strings.Contains(said, c.said) || (c.said == "") != (said == "") { - t.Fatalf("hand-act %q, want one saying %q", said, c.said) - } - }) - } - }) - t.Run("T6 a merge while the own build is unknown is taken in all the same (A-476 b)", func(t *testing.T) { - b := newTakeInBed(t) - b.ix.ownFP, b.ix.ownFixed = "", false // this process is started again and asks the engine, which does not answer - b.ix.OwnBuild = func(context.Context) (string, error) { return "", errors.New("the engine does not answer") } - b.ix.FixOwnBuild(ctx) - b.said = nil - if _, _, err := b.ix.Arrive(ctx, 1, b.merge(20, sha('b'), "tree-avahi-2", "tree-indexer-2")); err != nil || len(b.keys()) != 0 || b.commit(t, "avahi") != sha('b') { - t.Errorf("err %v, pending %v, avahi at %s: want the merge taken in at once", err, b.keys(), b.commit(t, "avahi")) - } - if len(b.said) != 1 || b.said[0].Kind != takein.TakeInTaken { - t.Errorf("said %+v, want one take-in.taken and no wait said", b.said) - } - answer, err := PendingTools(b.pending, b.ix.OwnBuildState, nil, nil)[0].Run(map[string]any{}) - if a, _ := answer.(takein.PendingAnswer); err != nil || a.OwnBuild != "" || a.OwnBuildSince.IsZero() || len(a.Waits) != 0 { - t.Errorf("indexer_pending = %+v %v, want no wait and no known build", answer, err) - } - }) - t.Run("T7 and T3 a merge alone, an indexer change, a merge behind it and a revert", func(t *testing.T) { - b := newTakeInBed(t) - own, avahi0 := b.fp(t, testSelf), b.fp(t, "avahi") - // Issue 1966 note 154 (ADR 0668 §4): with the verdicts state not on the bus yet, nothing is taken in and no manifest - // is written without its verdict; its bind evaluates pending again, which takes the merge in with its verdicts. - verdictsBound := b.ix.Verdicts - b.ix.Verdicts = &memStore{unbound: process.ErrNotYet} - b.arrive(t, 1, b.merge(20, sha('b'), "tree-avahi-2", "")) - if b.fp(t, "avahi") != avahi0 || !reflect.DeepEqual(b.keys(), []string{takein.PendingKey(1)}) { - t.Fatalf("verdicts not on the bus: avahi %s (was %s), pending %v: want the merge waiting, nothing written", b.fp(t, "avahi"), avahi0, b.keys()) - } - b.ix.Verdicts = verdictsBound - judgedBefore := len(verdictsBound.(*memStore).vals) - if err := b.ix.PendingAgain(ctx); err != nil { - t.Fatal(err) - } - if n := len(verdictsBound.(*memStore).vals); n <= judgedBefore { - t.Fatalf("taken in at the bind: %d verdicts (were %d), want the new manifest's", n, judgedBefore) - } - // A merge with no change to the indexer and nothing pending is taken in alone, as before. - if avahi1 := b.fp(t, "avahi"); avahi1 == avahi0 || len(b.keys()) != 0 || !slices.Equal(b.events(), []string{"taken " + takein.PendingKey(1)}) { - t.Fatalf("alone: avahi %s (was %s), pending %v, said %v: want taken in, said taken under its key (1966/AC195-1)", avahi1, avahi0, b.keys(), b.events()) - } - b.said = nil - avahi1 := b.fp(t, "avahi") - // A-476 b: a merge naming another indexer build is taken in at once, with the build this program runs: the indexer's entry - // and avahi are written, nothing is held and said pending. - b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) - named := b.fp(t, testSelf) - if named == own || b.fp(t, "avahi") == avahi1 || b.commit(t, "avahi") != sha('c') || len(b.keys()) != 0 { - t.Fatalf("indexer change: indexer %s (own %s), avahi %s at %s, pending %v: want the whole revision taken in at once", named, own, b.fp(t, "avahi"), b.commit(t, "avahi"), b.keys()) - } - if g := graphOf(t, b.ix, testSelf); g.FP != named { - t.Errorf("graph of the indexer names %s, want %s", g.FP, named) - } - if want := []string{"taken " + takein.PendingKey(2)}; !slices.Equal(b.events(), want) { - t.Errorf("said %v, want %v: nothing said pending", b.events(), want) - } - // A later merge is taken in at once too, and a revert to the first indexer. - b.arrive(t, 3, b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2")) - b.arrive(t, 4, b.merge(23, sha('e'), "tree-avahi-5", b.base.Trees["modules/"+testSelf])) - if len(b.keys()) != 0 || b.fp(t, testSelf) != own || b.commit(t, "avahi") != sha('e') { - t.Fatalf("revert: pending %v, indexer %s, avahi at %s: want all taken in", b.keys(), b.fp(t, testSelf), b.commit(t, "avahi")) - } - }) - t.Run("IX22 a merged revision naming a newer indexer is a catalog graph revision with its entry and the merge's others replaced", func(t *testing.T) { // design 47 G §273, §237 - b := newTakeInBed(t) - store := newMemObjects(0) - b.ix.Revisions = store - if err := b.ix.KeepView(ctx); err != nil { - t.Fatal(err) - } - before, _, err := newestTrunk(ctx, store) - if err != nil || before.Revision != 1 { - t.Fatalf("the view's revision %+v (%v), want revision 1", before, err) - } - // The merge changes avahi and the indexer: both are taken in at once (A-476 b), in a revision after the first. - b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) - after, _, err := newestTrunk(ctx, store) - if err != nil || after.Revision <= before.Revision { - t.Fatalf("after the merge naming a newer indexer the newest revision is %+v (%v), want one after revision %d", after, err, before.Revision) - } - if changed := changedEntries(before.Entries, after.Entries); !slices.Equal(changed, []string{"avahi", testSelf}) && !slices.Equal(changed, []string{testSelf, "avahi"}) { - t.Errorf("the revision differs from the previous in %v, want avahi and the indexer", changed) - } - }) - t.Run("P1 a revision that adds a Go module its own indexer uses is computed against that module's packages at its merge", func(t *testing.T) { - // The Go module exists only at the revision's own merge: nothing publishes it before the take-in, so the fingerprint - // the revision names must come from the packages its merge holds (issue 1585, ADR 0593 §2), not wait for a version only - // its own take-in publishes. - const sibling = "git.novox.be/novox/mesh-catalog/modules/mesh-new/pkg" - b := newTakeInBed(t) - f := b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2") - if b.ix.Packages == nil { - b.ix.Packages = newMem() - } - b.ix.GoMod = func(_ context.Context, _, commit, dir string) ([]byte, error) { - if commit == f.Merge && dir == "modules/"+testSelf { - return []byte("module x\n\nreplace " + sibling + " => ../mesh-new/pkg\n"), nil - } - return []byte("module x\n"), nil - } - b.ix.Files = func(_ context.Context, repository, commit string, dirs []string) (map[string]map[string]string, error) { - out := map[string]map[string]string{} - for _, d := range dirs { - if repository != catalogueRepo || commit != f.Merge || d != "modules/mesh-new/pkg" { - return nil, fmt.Errorf("no directory %s: %w", d, ErrNotInTree) - } - out[d] = map[string]string{"go.mod": "m1", "go.sum": "s1", "x.go": "g1"} - } - return out, nil - } - // A Go module the repository published before and its merge no longer holds does not stop the others being published. - const gone = "git.novox.be/novox/mesh-catalog/modules/mesh-gone/pkg" - old, err := EncodePublishedPackages(meshcatalog.PackagesByModuleValue{Module: gone, Repository: catalogueRepo, Dir: "modules/mesh-gone/pkg", - Commit: sha('a'), Requires: "r", At: ptr(time.Now()), Packages: []meshcatalog.PackagesByModuleValuePackagesItem{{Package: gone, Digest: "d"}}}) - if err != nil { - t.Fatal(err) - } - goneKey, _ := PackagesKey(gone) - if err := b.ix.Packages.Create(ctx, goneKey, old); err != nil { - t.Fatal(err) - } - b.arrive(t, 2, f) - if len(b.said) != 1 || b.said[0].Kind != takein.TakeInTaken || b.said[0].WaitsFor != "" || len(b.keys()) != 0 { - t.Errorf("said %+v, pending %v, want m.2 taken in at once", b.said, b.keys()) - } - if _, ok := b.ix.publishedAll(ctx)[sibling]; !ok { - t.Errorf("the packages the revision publishes at its merge lack %s", sibling) - } - }) - t.Run("A-326 a revision whose catalog graph holds a cycle with a build edge no build seat breaks is refused, naming it", func(t *testing.T) { - // avahi's artifact needs the indexer's (kind 2), and the indexer is built on avahi (kind 3): avahi holds no build seat, so - // no running version breaks the cycle, and the revision is refused at take-in naming every artifact on it (design 47 G §49). - b := newTakeInBed(t) - f := b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2") - f.Manifests = map[string]string{} - for dir, raw := range b.base.Manifests { - f.Manifests[dir] = raw - } - f.Manifests["modules/avahi"] = strings.Replace(f.Manifests["modules/avahi"], `"name": "tools",`, `"name": "tools", "needs": ["`+testSelf+`:tools"],`, 1) - f.Manifests["modules/"+testSelf] = strings.Replace(f.Manifests["modules/"+testSelf], `"build": {`, - `"build": { - "on": [{"arg": "RUNTIME_BASE", "module": "avahi", "artifact": "tools"}],`, 1) - b.arrive(t, 2, f) - want := "avahi:tools, " + testSelf + ":tools, avahi:tools" - if len(b.said) == 0 || !strings.Contains(b.said[len(b.said)-1].Refused, want) { - t.Fatalf("said %+v, want the revision refused naming %s", b.said, want) - } - if b.commit(t, "avahi") == sha('c') { - t.Errorf("avahi was taken in at the refused revision") - } - if waits := PendingWaits(readPending(t, b)); len(waits) != 1 || !strings.Contains(waits[0].Refused, want) { - t.Errorf("indexer_pending %+v, want the refusal naming the cycle", waits) - } - }) - t.Run("T10 a start takes what waits in whatever its build; a gathered group stays", func(t *testing.T) { - b := newTakeInBed(t) - b.arriveHeld(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) - g := b.merge(30, sha('f'), "", "") - g.Repository, g.Trees, g.Manifests, g.Locks, g.Group = "novox/mesh-sdk", map[string]string{}, nil, nil, "feat/x" - g.Members = []meshdeliveryclient.Head{{Repository: "novox/mesh-sdk", Number: 30, Head: g.Head}, {Repository: "novox/mesh-host", Number: 4, Head: sha('9')}} - b.arriveHeld(t, 5, g) - own, _ := b.ix.OwnBuild(ctx) - // The indexer starts; its build is not known (the engine does not answer at its start): a hand ask is answered - // meanwhile (R5), and the start waits for nothing. It is never asked again, so it stays unknown until it is started again. - mk := func(ask func(context.Context) (string, error)) *Indexer { - n := &Indexer{Module: testSelf, Store: b.ix.Store, Modules: b.ix.Modules, Manifests: b.ix.Manifests, Graph: b.ix.Graph, Verdicts: b.ix.Verdicts, - Packages: b.ix.Packages, Builds: b.ix.Builds, Pending: b.pending, Message: b.ix.Message, SayTakeIn: b.ix.SayTakeIn, Logf: b.ix.Logf, GoSum: b.ix.GoSum} - n.UseBuildModules(b.ix.BuildModules()) - n.toolchains = b.ix.toolchains - n.OwnBuild = ask - n.FixOwnBuild(ctx) - return n - } - // The build this program runs is not known (the engine does not answer at its start): nothing waits for it, so the start - // takes the revision in all the same (A-476 b), and a hand ask names what waits. - next := mk(func(context.Context) (string, error) { return "", errors.New("not yet") }) - if r := next.TakeInPendingByHand(ctx, "m.nothing", "lead@shanks", "a test"); r.Stored || !strings.Contains(r.Answer, "nothing waits") { - t.Errorf("a hand ask for what does not wait: %+v", r) - } - if _, _, err := next.TakeInAtStart(ctx); err != nil { - t.Fatal(err) - } - if !reflect.DeepEqual(b.keys(), []string{takein.PendingKey(5)}) || b.commit(t, "avahi") != sha('c') { - t.Errorf("start: pending %v, avahi at %s: want the revision taken in, the group left", b.keys(), b.commit(t, "avahi")) - } - if answer, err := PendingTools(b.pending, b.ix.OwnBuildState, nil, nil)[0].Run(map[string]any{}); err != nil || answer.(takein.PendingAnswer).OwnBuild != own { - t.Errorf("indexer_pending = %+v %v, want the build the original program knows", answer, err) - } - }) - t.Run("T8 T9 T11 a group gathered until its end, a merge behind it, one alone, an end again", func(t *testing.T) { - b := newTakeInBed(t) - b.ix.Taken = newMem() - // The commits in the order they were made: one is in another when it comes first. - order := "a756bc" - b.ix.Contains = func(_ context.Context, _, commit, in string) (bool, error) { - return strings.IndexByte(order, commit[0]) <= strings.IndexByte(order, in[0]), nil - } - member := func(repo string, number int, c string) forgemsg.Forge { - f := b.merge(number, c, "", "") - f.Group = "feat/take-in" - f.Members = []meshdeliveryclient.Head{{Repository: catalogueRepo, Number: 40, Head: sha('1')}, {Repository: "novox/mesh-sdk", Number: 41, Head: sha('2')}} - if repo != catalogueRepo { - f.Repository, f.Trees, f.Manifests, f.Locks = repo, map[string]string{}, nil, nil - } - f.Head = map[int]string{40: sha('1'), 41: sha('2')}[number] - return f - } - b.arrive(t, 1, member(catalogueRepo, 40, sha('7'))) - if b.commit(t, "avahi") == sha('7') || !slices.Equal(b.events(), []string{"pending g.feat/take-in"}) { - t.Fatalf("first member: avahi at %s, said %v: want held, the group said pending", b.commit(t, "avahi"), b.events()) - } - // A merge in another repository is taken in alone; one in a member's repository waits behind the group. - alone := b.merge(50, sha('5'), "", "") - alone.Repository, alone.Trees, alone.Manifests, alone.Locks = "novox/hq", map[string]string{}, nil, nil - b.arrive(t, 2, alone) - b.arrive(t, 3, b.merge(51, sha('6'), "tree-avahi-9", "")) - if !reflect.DeepEqual(b.keys(), []string{takein.PendingKey(1), takein.PendingKey(3)}) { - t.Fatalf("pending %v: want the member and the merge behind it", b.keys()) - } - end := forgemsg.ForgeGroupEnd{Group: "feat/take-in", Members: []forgemsg.GroupMerge{{Repository: catalogueRepo, Number: 40, Merge: sha('7')}, {Repository: "novox/mesh-sdk", Number: 41, Merge: sha('b')}}} - if _, _, err := b.ix.EndGroup(ctx, end); err == nil { - t.Fatal("an end before its last member is taken in") - } - b.arrive(t, 4, member("novox/mesh-sdk", 41, sha('b'))) - if _, _, err := b.ix.EndGroup(ctx, end); err != nil { - t.Fatal(err) - } - if len(b.keys()) != 0 || b.commit(t, "avahi") != sha('6') { - t.Fatalf("end: pending %v, avahi at %s: want the group and the merge behind it taken in", b.keys(), b.commit(t, "avahi")) - } - if want := []string{"pending g.feat/take-in", "taken " + takein.PendingKey(2), "pending g.feat/take-in", "taken g.feat/take-in", "taken " + takein.PendingKey(1), "taken " + takein.PendingKey(3), "taken " + takein.PendingKey(4)}; !slices.Equal(b.events(), want) { - t.Errorf("said %v, want %v", b.events(), want) - } - // R1: the end published again after the take-in changes nothing and is not delivered again. - if _, words, err := b.ix.EndGroup(ctx, end); err != nil || !strings.Contains(words, "taken in already") { - t.Errorf("an end again: %q %v", words, err) - } - // T11: a group whose revision changes the indexer: the group's wait is taken and the revision taken in at once (A-476 b). - b.said = nil - g := member(catalogueRepo, 40, sha('c')) - g.Group, g.Trees = "feat/two", b.merge(0, "", "", "tree-indexer-7").Trees - g.Members = g.Members[:1] - b.arrive(t, 9, g) - if _, _, err := b.ix.EndGroup(ctx, forgemsg.ForgeGroupEnd{Group: "feat/two", Members: []forgemsg.GroupMerge{{Repository: catalogueRepo, Number: 40, Merge: sha('c')}}}); err != nil { - t.Fatal(err) - } - if want := []string{"pending g.feat/two", "taken g.feat/two", "taken " + takein.PendingKey(9)}; !slices.Equal(b.events(), want) { - t.Errorf("a group naming another indexer: said %v, want %v", b.events(), want) - } - }) - t.Run("1585 a group wait whose members are all taken in ends itself, behind it a merge is taken in at once, and a start that fails is asked again by the next event", func(t *testing.T) { - b := newTakeInBed(t) - b.ix.Taken = newMem() - order := "a756bc" - b.ix.Contains = func(_ context.Context, _, commit, in string) (bool, error) { - return strings.IndexByte(order, commit[0]) <= strings.IndexByte(order, in[0]), nil - } - taken := func(repo, merge string) { - key, _ := trunk.RepositoryKey(repo) - data, err := takein.EncodeTaken(takein.Taken{Repository: trunk.RepositoryName(repo), Merge: merge}) - if err != nil { - t.Fatal(err) - } - b.ix.Taken.(*memStore).put(key, data) - } - f := b.merge(40, sha('7'), "", "") - f.Group, f.Head = "feat/take-in", sha('1') - f.Members = []meshdeliveryclient.Head{{Repository: catalogueRepo, Number: 40, Head: sha('1')}, {Repository: "novox/mesh-sdk", Number: 41, Head: sha('2')}} - b.arrive(t, 1, f) // the group's other member was taken in by hand: its key is gone and the group's end never came - b.arrive(t, 2, b.merge(51, sha('6'), "tree-avahi-9", "")) - b.said = nil - // Not every member is taken in (the catalogue's is held; the sdk's is not known taken): the wait stays. - taken(catalogueRepo, sha('7')) - if err := b.ix.PendingAgain(ctx); err != nil || len(b.keys()) != 2 || len(b.said) != 0 { - t.Fatalf("a member not taken in: err %v, pending %v, said %v: want the wait kept", err, b.keys(), b.events()) - } - // Every member is taken in: the next evaluation ends the wait (said under the group key) and takes the merge behind it in. - taken("novox/mesh-sdk", sha('b')) - if err := b.ix.PendingAgain(ctx); err != nil { - t.Fatal(err) - } - if len(b.keys()) != 0 || b.commit(t, "avahi") != sha('6') || !slices.Equal(b.events(), []string{"taken g.feat/take-in", "taken " + takein.PendingKey(2)}) { - t.Errorf("all taken: pending %v, avahi at %s, said %v: want the group ended and the merge behind it taken in", b.keys(), b.commit(t, "avahi"), b.events()) - } - held := func() *takeInBed { - b := newTakeInBed(t) - b.arriveHeld(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) - b.arriveHeld(t, 3, b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2")) - return b - } - // A probe that fails keeps the group's wait and stops nothing else: an unrelated merge is still taken in. - b = newTakeInBed(t) - b.ix.Taken = newMem() - b.ix.Contains = func(context.Context, string, string, string) (bool, error) { - return false, errors.New("the forge does not answer") - } - g := b.merge(40, sha('7'), "", "") - g.Group, g.Head = "feat/take-in", sha('1') - g.Members = []meshdeliveryclient.Head{{Repository: catalogueRepo, Number: 40, Head: sha('1')}, {Repository: "novox/mesh-sdk", Number: 41, Head: sha('2')}} - b.arrive(t, 1, g) - key, _ := trunk.RepositoryKey("novox/mesh-sdk") - data, _ := takein.EncodeTaken(takein.Taken{Repository: trunk.RepositoryName("novox/mesh-sdk"), Merge: sha('b')}) - b.ix.Taken.(*memStore).put(key, data) - key, _ = trunk.RepositoryKey(catalogueRepo) - data, _ = takein.EncodeTaken(takein.Taken{Repository: trunk.RepositoryName(catalogueRepo), Merge: sha('7')}) - b.ix.Taken.(*memStore).put(key, data) - alone := b.merge(60, sha('5'), "", "") - alone.Repository, alone.Trees, alone.Manifests, alone.Locks = "novox/hq", map[string]string{}, nil, nil - b.arrive(t, 2, alone) - if len(b.keys()) != 1 || b.keys()[0] != takein.PendingKey(1) || !strings.Contains(b.logs.String(), "cannot be read, so its wait is kept") { - t.Errorf("a failing probe: pending %v: want the group kept, said, and the unrelated merge taken in", b.keys()) - } - // A start that fails leaves its merges pending; the next event the indexer hears evaluates them again (no timer). - b = held() - fails := 1 - b.mods.failWrite = func(string) error { - if fails > 0 { - fails-- - return errors.New("nats: jetstream not enabled") - } - return nil - } - if _, _, err := b.ix.TakeInAtStart(ctx); err == nil || len(b.keys()) != 2 { - t.Fatalf("a failing start: err %v, pending %v: want the error and both merges pending", err, b.keys()) - } - if err := b.ix.PendingAgain(ctx); err != nil || len(b.keys()) != 0 || b.commit(t, "avahi") != sha('d') { - t.Errorf("the next event: err %v, pending %v, avahi at %s: want both taken in", err, b.keys(), b.commit(t, "avahi")) - } - // A start that leaves a merge lost (the forge bridge restarted) is asked again by the next merged message. - b = held() - answers := b.ix.Message - b.ix.held = nil - b.ix.Message = func(context.Context, string, string, int) ([]byte, error) { return nil, errors.New("timed out") } - if _, _, err := b.ix.TakeInAtStart(ctx); err != nil || len(b.keys()) != 2 { - t.Fatalf("a lost start: err %v, pending %v", err, b.keys()) - } - b.ix.Message = answers - b.arrive(t, 4, b.merge(23, sha('e'), "tree-avahi-5", "tree-indexer-2")) - if len(b.keys()) != 0 || b.commit(t, "avahi") != sha('e') { - t.Errorf("after a lost start: pending %v, avahi at %s: want all taken in by the next merged message", b.keys(), b.commit(t, "avahi")) - } - // Issue 1966 note 153, ADR 0668 §1, §4: a transport fault (the bridge restarting past the call's bound) leaves the - // newest merge unread, not lost, and indexer_pending says so; the next message heard on the forge, of any kind, reads - // it again. The bridge's own answer (it holds no such merge) is lost, as before, and a heard message does not ask it. - for _, c := range []struct { - name string - fault error - unread bool - }{ - {"a deadline", fmt.Errorf("the bridge: %w", nats.ErrTimeout), true}, - {"no responders", fmt.Errorf("the bridge: %w", nats.ErrNoResponders), true}, - {"the bridge's answer", &busretry.Answered{Words: "no merge at that commit"}, false}, - } { - b = held() - clock := time.Date(2026, 11, 10, 1, 0, 0, 0, time.UTC) - b.ix.Now, b.ix.Findings = func() time.Time { return clock }, newFindings() - answers := b.ix.Message - b.ix.held = nil - b.ix.Message = func(context.Context, string, string, int) ([]byte, error) { return nil, c.fault } - if _, _, err := b.ix.TakeInAtStart(ctx); err != nil || len(b.keys()) != 2 { - t.Fatalf("%s at the start: err %v, pending %v", c.name, err, b.keys()) - } - waits := PendingWaits(mustReadPending(t, b)) - if len(waits) != 1 || (len(waits[0].Unread) == 1) != c.unread || (len(waits[0].Lost) == 1) == c.unread { - t.Fatalf("%s: indexer_pending %+v: want unread %v, lost %v", c.name, waits, c.unread, !c.unread) - } - // ADR 0668 §3: unread within five minutes is no finding; past them, with nothing heard on the forge (the bridge's - // machine away, only a toolchain report heard), none either; the next message heard on the forge, the read still - // failing, writes call-unanswered naming the bridge's call; the merge read and taken in clears it. - finding := func() (string, bool) { - for k, v := range b.ix.Findings.(*memFindings).vals { - if strings.HasSuffix(k, "."+mcc.KindCallUnanswered) { - return string(v), true - } - } - return "", false - } - if c.unread { - for _, step := range []struct { - after time.Duration - heard bool - finding bool - }{{time.Minute, true, false}, {5 * time.Minute, false, false}, {0, true, true}} { - clock = clock.Add(step.after) - var err error - if step.heard { - err = b.ix.UnreadAgain(ctx) - } else { - err = b.ix.PendingAgain(ctx) - } - if v, found := finding(); err != nil || found != step.finding || found && !strings.Contains(v, UnreadCallee) { - t.Fatalf("%s, %s later, forge heard %v: finding %v %s (%v), want %v", c.name, step.after, step.heard, found, v, err, step.finding) - } - } - } - b.ix.Message = answers - if err := b.ix.UnreadAgain(ctx); err != nil { - t.Fatal(err) - } - if taken := len(b.keys()) == 0 && b.commit(t, "avahi") == sha('d'); taken != c.unread { - t.Errorf("%s, a message heard after: pending %v, avahi at %s: want taken in %v", c.name, b.keys(), b.commit(t, "avahi"), c.unread) - } - if v, found := finding(); found { - t.Errorf("%s: the finding stands after the merge was taken in: %s", c.name, v) - } - } - }) - t.Run("T5 two take-ins of one pending revision: the one that loses the claim writes nothing", func(t *testing.T) { - b := newTakeInBed(t) - b.arriveHeld(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) - named := b.fp(t, testSelf) - b.ix.ownMu.Lock() - b.ix.ownFP = named - b.ix.ownMu.Unlock() - // The other indexer claims the merge between this one's read and its claim. - b.pending.beforeWrite = func() { - v := b.pending.vals[takein.PendingKey(2)] - var p Pending - _ = json.Unmarshal(v, &p) - p.Taking = "the other indexer" - data, _ := json.Marshal(p) - b.pending.put(takein.PendingKey(2), data) - } - if _, _, err := b.ix.TakeInAtStart(ctx); err != nil { - t.Fatal(err) - } - if b.commit(t, "avahi") == sha('c') || len(b.keys()) != 1 || !strings.Contains(b.logs.String(), "is lost") { - t.Errorf("avahi at %s, pending %v: want nothing taken in by the loser, and it said", b.commit(t, "avahi"), b.keys()) - } - }) - t.Run("T5 a take-in whose store fails once leaves no claim behind, and is taken in after", func(t *testing.T) { - claims := func(b *takeInBed) []string { - var out []string - for _, k := range b.keys() { - var p Pending - _ = json.Unmarshal(b.pending.vals[k], &p) - if p.Taking != "" { - out = append(out, k) - } - } - return out - } - once := func(match func(key string) bool) func(string) error { - done := false - return func(key string) error { - if !done && match(key) { - done = true - return errors.New("the store failed once") - } - return nil - } - } - for _, c := range []struct { - name string - fail func(b *takeInBed) - }{ - // The second merge's claim fails: the first's, already made, is released. - {"a claim lost", func(b *takeInBed) { - b.pending.failWrite = once(func(k string) bool { return k == takein.PendingKey(3) }) - }}, - // The take-in itself fails on a module's write after both claims: both are released. - {"a take-in that fails", func(b *takeInBed) { b.mods.failWrite = once(func(string) bool { return true }) }}, - } { - b := newTakeInBed(t) - b.arriveHeld(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) - b.arriveHeld(t, 3, b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2")) - named := b.fp(t, testSelf) - b.ix.ownMu.Lock() - b.ix.ownFP = named - b.ix.ownMu.Unlock() - c.fail(b) - _, _, _ = b.ix.TakeInAtStart(ctx) - if got := claims(b); len(got) != 0 || len(b.keys()) != 2 { - t.Fatalf("%s: pending %v, claimed %v: want both pending and no claim left", c.name, b.keys(), got) - } - // The next take-in, not at a start, is not refused by a claim left behind: it takes both in. - b.arrive(t, 4, b.merge(23, sha('e'), "tree-avahi-5", "tree-indexer-2")) - if len(b.keys()) != 0 || b.commit(t, "avahi") != sha('e') { - t.Errorf("%s: after: pending %v, avahi at %s: want all taken in", c.name, b.keys(), b.commit(t, "avahi")) - } - } - }) - t.Run("W4.6 a full store holds the merge with the reason objects full, says full once, and room takes it in", func(t *testing.T) { - b := newTakeInBed(t) - b.arriveHeld(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) - named := b.fp(t, testSelf) - b.ix.ownMu.Lock() - b.ix.ownFP = named - b.ix.ownMu.Unlock() - store := newMemObjects(100) // 90 bytes held by another revision: full for this one - _ = store.Put(ctx, "trunk.0", make([]byte, 90)) - b.ix.Revisions = store - var said []string - b.ix.SayFill = func(_ context.Context, event, id string, f meshcatalog.Fill) error { - said = append(said, event+" "+f.Refused) - return nil - } - if _, _, err := b.ix.TakeInAtStart(ctx); err != nil { - t.Fatal(err) - } - var p Pending - _ = json.Unmarshal(b.pending.vals[b.keys()[0]], &p) - if len(b.keys()) != 1 || p.Held != HeldObjectsFull || len(said) != 1 || !strings.HasPrefix(said[0], "full ") || !strings.Contains(said[0], HeldObjectsFull) { - t.Fatalf("pending %v held %q, said %q: want the merge held with %q and full said", b.keys(), p.Held, said, HeldObjectsFull) - } - if roomed, err := b.ix.RoomAgain(ctx); err != nil || roomed { - t.Fatalf("room while still full: %v %v", roomed, err) - } - _ = store.Delete(ctx, "trunk.0") // the other revision removed: room - store.capacity = 0 - if roomed, err := b.ix.RoomAgain(ctx); err != nil || !roomed || len(b.keys()) != 0 || said[len(said)-1] != "room " { - t.Errorf("room said %v (%v), said %q, pending %v: want room said and the merge taken in", roomed, err, said, b.keys()) - } - // The same take-in into a store it fills to 85 % of its cap says filling once, and is taken in. - kept := store.used() - b2 := newTakeInBed(t) - b2.arriveHeld(t, 2, b2.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) - b2.ix.ownMu.Lock() - b2.ix.ownFP = b2.fp(t, testSelf) - b2.ix.ownMu.Unlock() - b2.ix.Revisions = newMemObjects(kept * 100 / 85) - var filling []meshcatalog.Fill - b2.ix.SayFill = func(_ context.Context, event, _ string, f meshcatalog.Fill) error { - if event == meshcatalog.EventFilling { - filling = append(filling, f) - } - return nil - } - if _, _, err := b2.ix.TakeInAtStart(ctx); err != nil || len(b2.keys()) != 0 || len(filling) != 1 || filling[0].Store != CatalogRevisionsObjects { - t.Errorf("a take-in to 85 %% said filling %+v (%v), pending %v: want it said once and the merge taken in", filling, err, b2.keys()) - } - }) - t.Run("W4.12 a candidate's half dropped by hand frees its own entries, says room and takes the held merge in; the read tools list it; a recompute that moves nothing is equal", func(t *testing.T) { - bed := func(store *memObjects) *takeInBed { - b := newTakeInBed(t) - b.arriveHeld(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) - b.ix.ownMu.Lock() - b.ix.ownFP = b.fp(t, testSelf) - b.ix.ownMu.Unlock() - b.ix.Revisions = store - return b - } - free := newMemObjects(0) - if _, _, err := bed(free).ix.TakeInAtStart(ctx); err != nil { - t.Fatal(err) - } - size := free.used() // what the merge's revision takes - store := newMemObjects(2 * size) - shared := []byte(`{"module":"b"}`) - if err := KeepRevision(ctx, store, meshcatalog.CatalogRevision{Kind: "candidate", Group: "g1", Heads: "h1", Revision: 1, - Entries: []meshcatalog.EntryRef{{Module: "a", Digest: "da"}, {Module: "b", Digest: "db"}}}, - map[string][]byte{"da": make([]byte, size+10), "db": shared}); err != nil { - t.Fatal(err) - } - if err := KeepRevision(ctx, store, meshcatalog.CatalogRevision{Kind: "candidate", Group: "g2", Heads: "h2", Revision: 1, - Entries: []meshcatalog.EntryRef{{Module: "b", Digest: "db"}}}, map[string][]byte{"db": shared}); err != nil { - t.Fatal(err) - } - b := bed(store) - var said []string - b.ix.SayFill = func(_ context.Context, event, _ string, _ meshcatalog.Fill) error { - said = append(said, event) - return nil - } - if _, _, err := b.ix.TakeInAtStart(ctx); err != nil || len(b.keys()) != 1 { - t.Fatalf("pending %v (%v): want the merge held for objects full", b.keys(), err) - } - var recorded []string - asks := 0 - tools := map[string]stdio.Tool{} - for _, tool := range CatalogRevisionTools(store, func(_ string, value []byte) (TakeInAsk, error) { - asks++ - a, err := DecodeTakeIn(value) - if err != nil { - return a, err - } - var r TakenIn - if a.Recompute { - r = b.ix.RecomputeByHand(ctx, a.By, a.Why) - } else { - r = b.ix.DropCandidateGraph(ctx, a.CandidateGroup, a.CandidateHeads, a.By, a.Why) - } - a.Result = &r - return a, nil - }, time.Now, func(what, why string) error { - recorded = append(recorded, what) - if why == "unrecorded" { - return errors.New("mesh-hand-acts did not answer") - } - return nil - }) { - tools[tool.Name] = tool - } - if _, err := tools[DropCandidateGraphTool].Run(map[string]any{"group": "g1", "heads": "h1", "why": "unrecorded"}); err == nil || asks != 0 { - t.Fatalf("a drop whose hand act is not recorded: %v, %d asks: want refused before it is asked", err, asks) - } - got, err := tools[DropCandidateGraphTool].Run(map[string]any{"group": "g1", "heads": "h1", "why": "its gate event never came"}) - r, _ := got.(*TakenIn) - if err != nil || r == nil || !r.Stored || !slices.Equal(r.Dropped, []string{"entry.da"}) || len(recorded) != 2 { - t.Fatalf("drop answered %+v (%v), recorded %q: want candidate.g1.h1 dropped with entry.da alone, after its hand act", r, err, recorded) - } - if _, err := store.Get(ctx, "entry.db"); err != nil || len(b.keys()) != 0 || !slices.Contains(said, meshcatalog.EventRoom) { - t.Errorf("entry.db %v, pending %v, said %q: want the shared entry kept, room said and the merge taken in", err, b.keys(), said) - } - list, err := tools[CatalogRevisionsTool].Run(map[string]any{}) - l, _ := list.(RevisionsAnswer) - if err != nil || len(l.Revisions) != 3 || l.Revisions[0].Name != "candidate.g2.h2" || l.Revisions[1].Name != "trunk.1" || l.Revisions[2].Name != "trunk.2" || !l.Revisions[2].Current || - l.Fill.Cap != 2*size || l.Fill.Bytes != store.used() || len(l.Unlisted) != 0 { - t.Errorf("indexer_catalog_revisions = %+v (%v): want candidate.g2.h2, trunk.1 of the view and the current trunk.2 of the merge, the fill against its cap", l, err) - } - if c, err := tools[CandidateGraphsTool].Run(map[string]any{}); err != nil || len(c.(RevisionsAnswer).Revisions) != 1 { - t.Errorf("indexer_candidate_graphs = %+v (%v): want g2's alone", c, err) - } - if one, err := tools[CatalogRevisionTool].Run(map[string]any{}); err != nil || one.(map[string]any)["name"] != "trunk.2" { - t.Errorf("indexer_catalog_revision = %v (%v): want the newest trunk revision whole", one, err) - } - if got, err := tools[RecomputeTool].Run(map[string]any{"why": "a check"}); err != nil || !strings.Contains(got.(*TakenIn).Answer, "equal to the kept revision 2") { - t.Errorf("indexer_recompute = %+v (%v): want equal to the kept revision 2", got, err) - } - }) - t.Run("T12 T13 indexer_pending names the wait; a hand take-in and a drop say what they ended", func(t *testing.T) { - b := newTakeInBed(t) - b.arriveHeld(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) - b.arriveHeld(t, 3, b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2")) - var answer takein.PendingAnswer - for _, tool := range PendingTools(b.pending, nil, nil, nil) { - if tool.Name == PendingTool { - out, err := tool.Run(map[string]any{}) - if err != nil { - t.Fatal(err) - } - answer = out.(takein.PendingAnswer) - } - } - if len(answer.Waits) != 1 || answer.Waits[0].Key != takein.PendingKey(2) || !slices.Equal(answer.Waits[0].Holds, []string{testSelf, "avahi"}) || - len(answer.Waits[0].Keys) != 2 { - t.Fatalf("indexer_pending = %+v", answer) - } - // ADR 0668 §6: a later merge covers a pending one only when its commit has the pending merge's as an ancestor. - history := "cd" - b.ix.Contains = func(_ context.Context, _, commit, in string) (bool, error) { - i, j := strings.IndexByte(history, commit[0]), strings.IndexByte(history, in[0]) - return i >= 0 && j >= 0 && i <= j, nil - } - history = "dc" // m.3's commit is not in m.2's later history (a history rewritten) - if r := b.ix.DropPending(ctx, takein.PendingKey(2), "lead@shanks", "replaced by m.3", false); r.Stored || !strings.Contains(r.Answer, "as an ancestor") { - t.Fatalf("a drop a later merge not holding it would cover: %+v", r) - } - history = "cd" - b.said = nil - bound := b.ix.Verdicts - b.ix.Verdicts = &memStore{unbound: process.ErrNotYet} // m.3 stays held while m.2 is dropped - if r := b.ix.DropPending(ctx, takein.PendingKey(2), "lead@shanks", "replaced by m.3", false); !r.Stored { - t.Fatalf("drop: %+v", r) - } - b.ix.Verdicts = bound - if len(b.keys()) != 1 || b.keys()[0] != takein.PendingKey(3) { - t.Fatalf("pending %v: want m.2 dropped and m.3 pending", b.keys()) - } - // Issue 1966 (#650), ADR 0633 §5: the newest pending merge of its repository, which no taken merge covers, is not - // dropped; the answer names abandon, which drops it all the same (below). - if r := b.ix.DropPending(ctx, takein.PendingKey(3), "lead@shanks", "lost message", false); r.Stored || - !strings.Contains(r.Answer, "abandon") || len(b.keys()) != 1 { - t.Fatalf("a drop nothing covers: %+v, pending %v", r, b.keys()) - } - // A pending revision is taken in by the tool, by its key (A-476, principle 4). - if _, _, err := NewPendingAsk(takein.PendingKey(3), "x", "lead@shanks", false, false, time.Unix(0, 0)); err != nil { - t.Fatalf("the ask to take a pending revision in: %v", err) - } - if r := b.ix.TakeInPendingByHand(ctx, takein.PendingKey(3), "lead@shanks", "its message was unread"); !r.Stored || len(b.keys()) != 0 || b.commit(t, "avahi") != sha('d') { - t.Fatalf("hand take-in of a revision: %+v, pending %v, avahi at %s", r, b.keys(), b.commit(t, "avahi")) - } - // A delivery group gathered is taken in by hand (A-476 b: only a group is). - b.ix.Pins = newMem() - b.ix.Revisions = newMemObjects(0) // A-448: the taken event names the catalog graph revision the take-in made - gm := b.merge(24, sha('f'), "tree-avahi-6", "") - gm.Group, gm.Head = "feat/h", sha('1') - gm.Members = []meshdeliveryclient.Head{{Repository: catalogueRepo, Number: 24, Head: sha('1')}, {Repository: "novox/mesh-sdk", Number: 25, Head: sha('2')}} - b.arrive(t, 5, gm) - b.said = nil - if r := b.ix.TakeInPendingByHand(ctx, takein.GroupTakeInKey("feat/h"), "lead@shanks", "the group's end never comes"); !r.Stored { - t.Fatalf("hand take-in: %+v", r) - } - if len(b.keys()) != 0 || b.commit(t, "avahi") != sha('f') || len(b.said) < 1 || b.said[0].Kind != takein.TakeInTaken || b.said[0].By != "lead@shanks" { - t.Errorf("hand take-in: pending %v, avahi at %s, said %+v", b.keys(), b.commit(t, "avahi"), b.said) - } - // §476, §493: a hand take-in's taken event carries the branch, the source revision it makes and its entries. - if len(b.said) >= 1 { - got := b.said[0] - if got.Branch != TrunkBranch || got.Source <= 0 || len(got.Entries) != 2 || got.Entries[1].Module != "avahi" || - got.Entries[1].Commit != sha('f') || got.Entries[1].Fingerprint == "" { - t.Errorf("hand take-in said branch %q source %d entries %+v: want main, a source revision and avahi at its commit", got.Branch, got.Source, got.Entries) - } else if newest, any, _ := newestTrunk(ctx, b.ix.Revisions); !any || got.Revision != newest.Revision { - t.Errorf("hand take-in said revision %d, want the revision it made, %d (A-448)", got.Revision, newest.Revision) - } else if v, _, found, _ := b.ix.Pins.Get(ctx, PinKey(got.Branch, got.Revision)); !found || !strings.Contains(string(v), "pipeline."+got.Branch+"."+strconv.FormatInt(got.Source, 10)+".avahi") { - t.Errorf("the revision the take-in made is pinned %q (found %v), want the record pipeline.%s.%d.avahi (A-449a)", v, found, got.Branch, got.Source) - } else if _, err := takein.EncodeTakeIn(got); err != nil { - t.Errorf("the taken event is refused by its own contract: %v", err) - } - } - b.arriveHeld(t, 4, b.merge(23, sha('e'), "tree-avahi-5", "tree-indexer-2")) - if r := b.ix.DropPending(ctx, takein.PendingKey(4), "lead@shanks", "abandoned on the operator's word", true); !r.Stored || len(b.keys()) != 0 { - t.Errorf("an abandoned drop: %+v, pending %v", r, b.keys()) - } - if _, _, err := NewPendingAsk(takein.PendingKey(4), "x", "lead@shanks", false, true, time.Unix(0, 0)); err == nil { - t.Errorf("abandon on a take-in was asked") - } - }) - t.Run("1585 a merge over the value cap is held by reference, a new indexer with an empty memory takes it in through the forge bridge, and a covered pull request leaves its candidate", func(t *testing.T) { - big := func(f forgemsg.Forge) forgemsg.Forge { - f.Words = strings.Repeat("x", 300*1024) - f.Branch = "feat/g" - return f - } - b := newTakeInBed(t) - avahi0 := b.fp(t, "avahi") - // The forge bridge is not asked at all while the merges are in memory: a 300 KiB merge, over the bucket's cap whole, - // is held by reference and a lone one taken in. - bridgeDown := func(context.Context, string, string, int) ([]byte, error) { - return nil, errors.New("the bridge does not answer") - } - answers := b.ix.Message - b.ix.Message = bridgeDown - b.arrive(t, 1, big(b.merge(20, sha('b'), "tree-avahi-2", ""))) - if b.fp(t, "avahi") == avahi0 || b.commit(t, "avahi") != sha('b') || len(b.keys()) != 0 || len(b.ix.held) != 0 { - t.Fatalf("a big merge alone: avahi at %s, pending %v, in memory %d: want it taken in and forgotten", b.commit(t, "avahi"), b.keys(), len(b.ix.held)) - } - // Two pull requests of one group are open; both then merge (the indexer changed: held for the new build). - for n, head := range map[int]string{21: sha('1'), 22: sha('2')} { - if err := b.ix.Apply(ctx, pull(catalogueRepo, n, "feat/g", head, "modules/avahi")); err != nil { - t.Fatal(err) - } - } - key, _ := CandidatesKey("feat/g") - if _, _, found, _ := b.ix.Store.Get(ctx, key); !found { - t.Fatal("the group has no candidate") - } - b.arriveHeld(t, 2, big(b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))) - b.arriveHeld(t, 3, big(b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2"))) - raw, _, _, _ := b.pending.Get(ctx, takein.PendingKey(2)) - if len(b.keys()) != 2 || len(raw) > 4096 || b.commit(t, "avahi") != sha('b') { - t.Fatalf("a big merge held: pending %v, value %d bytes, avahi at %s: want two small values", b.keys(), len(raw), b.commit(t, "avahi")) - } - named := b.fp(t, testSelf) - // The new indexer starts with nothing in memory and the bridge down: the newest merge is said lost with the bridge's - // words (event, state), the older one is covered and not lost, and nothing is taken in. - next := &Indexer{Module: testSelf, Store: b.ix.Store, Modules: b.ix.Modules, Manifests: b.ix.Manifests, Graph: b.ix.Graph, Verdicts: b.ix.Verdicts, - Packages: b.ix.Packages, Builds: b.ix.Builds, Pending: b.pending, Message: bridgeDown, SayTakeIn: b.ix.SayTakeIn, Logf: b.ix.Logf, - GoSum: b.ix.GoSum, OwnBuild: func(context.Context) (string, error) { return named, nil }} - next.UseBuildModules(b.ix.BuildModules()) - next.toolchains = b.ix.toolchains - next.FixOwnBuild(ctx) - b.said = nil - if _, _, err := next.TakeInAtStart(ctx); err != nil { - t.Fatal(err) - } - waits := PendingWaits(mustReadPending(t, b)) - if len(b.said) != 1 || !slices.Equal(b.said[0].Lost, []string{takein.PendingKey(3)}) || len(waits) != 1 || !slices.Equal(waits[0].Lost, []string{takein.PendingKey(3)}) || b.commit(t, "avahi") != sha('b') { - t.Fatalf("lost: said %+v, waits %+v, avahi at %s: want m.3 lost on the event and in indexer_pending, nothing taken in", b.said, waits, b.commit(t, "avahi")) - } - if _, err := takein.EncodeTakeIn(b.said[0]); err != nil { - t.Errorf("the lost hold's event does not pass its check: %v", err) - } - if v, _, _, _ := b.pending.Get(ctx, takein.PendingKey(3)); !strings.Contains(string(v), "the bridge does not answer") { - t.Errorf("m.3 = %s: want the bridge's words", v) - } - if err := next.PendingAgain(ctx); err != nil || len(b.said) != 1 { - t.Errorf("lost again: err %v, said %v: want it said once", err, b.events()) - } - if r := next.TakeInPendingByHand(ctx, takein.PendingKey(2), "lead@shanks", "try"); r.Stored || b.commit(t, "avahi") != sha('b') { - t.Errorf("hand take-in of a lost merge: %+v, avahi at %s: want nothing taken in", r, b.commit(t, "avahi")) - } - // The bridge answers: the newest merge is read through it, the older one is covered, and the revision is taken in. - b.publish(t, b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2")) // the answer is the message as published, small here - next.Message = answers - b.bridge[catalogueRepo+"@"+sha('d')], _ = forgemsg.EncodeForge(big(b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2"))) - b.logs.Reset() - if err := next.PendingAgain(ctx); err != nil { - t.Fatal(err) - } - if len(b.keys()) != 0 || b.commit(t, "avahi") != sha('d') || !strings.Contains(b.logs.String(), "is covered by") { - t.Fatalf("taken in: pending %v, avahi at %s, journal %q: want both merges taken in, the older covered", b.keys(), b.commit(t, "avahi"), b.logs.String()) - } - if _, _, found, _ := b.ix.Store.Get(ctx, key); found { - t.Errorf("the group's candidate stays after both its pull requests merged (the covered one is not taken out)") - } - // A hand take-in of a merge by its commit asks the bridge and runs the message through Arrive. - b.ix.Message = answers - b.publish(t, big(b.merge(30, sha('e'), "tree-avahi-6", b.base.Trees["modules/"+testSelf]))) - if r := b.ix.TakeInMerge(ctx, catalogueRepo, sha('e'), 9, "lead@shanks", "held by the cap"); !r.Stored || b.commit(t, "avahi") != sha('e') { - t.Fatalf("merge take-in: %+v, avahi at %s: want it taken in", r, b.commit(t, "avahi")) - } - if r := b.ix.TakeInMerge(ctx, catalogueRepo, sha('9'), 10, "lead@shanks", "no such"); r.Stored || !strings.Contains(r.Answer, "gate holds no merge") { - t.Errorf("merge take-in of a merge the bridge does not know: %+v", r) - } - // An old value that holds the message whole reads as before. - old := b.merge(31, sha('f'), "tree-avahi-7", b.base.Trees["modules/"+testSelf]) - rawOld, _ := forgemsg.EncodeForge(old) - data, _ := encodePending(Pending{Forge: rawOld, Since: b.ix.now(), LastWriter: b.fp(t, testSelf)}) - if err := b.pending.Create(ctx, takein.PendingKey(11), data); err != nil { - t.Fatal(err) - } - if err := b.ix.PendingAgain(ctx); err != nil || b.commit(t, "avahi") != sha('f') { - t.Errorf("an old value with the message whole: err %v, avahi at %s: want it taken in", err, b.commit(t, "avahi")) - } - }) - t.Run("1966 a merge at or behind its repository's taken merge is ended without its message being read, one ahead of it is lost as before", func(t *testing.T) { - for _, c := range []struct { - name string - behind bool - }{{"behind taken", true}, {"ahead of taken", false}} { - b := newTakeInBed(t) - f := b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2") - f.Words, f.Branch = strings.Repeat("x", 300*1024), "feat/g" - b.arriveHeld(t, 2, f) // held by reference, with the verdicts state not on the bus - b.ix.Taken = newMem() - b.ix.Contains = func(context.Context, string, string, string) (bool, error) { return c.behind, nil } - data, _ := takein.EncodeTaken(takein.Taken{Repository: trunk.RepositoryName(catalogueRepo), Merge: sha('d')}) - key, _ := trunk.RepositoryKey(catalogueRepo) - b.ix.Taken.(*memStore).put(key, data) - named := b.fp(t, testSelf) - asked := 0 - next := &Indexer{Module: testSelf, Store: b.ix.Store, Modules: b.ix.Modules, Manifests: b.ix.Manifests, Graph: b.ix.Graph, Verdicts: b.ix.Verdicts, - Packages: b.ix.Packages, Builds: b.ix.Builds, Pending: b.pending, Taken: b.ix.Taken, Contains: b.ix.Contains, - Message: func(context.Context, string, string, int) ([]byte, error) { - asked++ - return nil, errors.New("the bridge does not answer") - }, - SayTakeIn: b.ix.SayTakeIn, Logf: b.ix.Logf, GoSum: b.ix.GoSum, OwnBuild: func(context.Context) (string, error) { return named, nil }} - next.toolchains = b.ix.toolchains - next.UseBuildModules(b.ix.BuildModules()) - next.FixOwnBuild(ctx) - if _, _, err := next.TakeInAtStart(ctx); err != nil { - t.Fatal(err) - } - if c.behind && (asked != 0 || len(b.keys()) != 0 || !strings.Contains(b.logs.String(), "covered by the taken merge")) { - t.Errorf("%s: asked %d, pending %v: want no read and the entry ended", c.name, asked, b.keys()) - } - if !c.behind && (asked == 0 || len(b.keys()) != 1) { - t.Errorf("%s: asked %d, pending %v: want the message asked and the entry held", c.name, asked, b.keys()) - } - } - }) - t.Run("a registration is read at the head of its first watched branch", func(t *testing.T) { - // Design 47 G A-85: the trunk is the first watched branch's; photos is watched on nox-mesh alone (2026-11-14). - ctx := context.Background() - ix := &Indexer{Module: testSelf, Registry: &Registry{Registrations: MemoryStore()}, Logf: func(string, ...any) {}} - raw, _ := json.Marshal(meshcatalog.Registration{Module: "photos", Origin: "novox/photos", Path: "module.json", Branches: []string{"nox-mesh", "main"}}) - if err := ix.Registry.Registrations.Create(ctx, RegistrationKey("photos"), raw); err != nil { - t.Fatal(err) - } - var asked string - ix.BranchHead = func(_ context.Context, repository, branch string) (string, error) { - asked = repository + "@" + branch - return strings.Repeat("a", 40), nil - } - r, why, ok := ix.registeredAtBranch(ctx, "photos") - if !ok || why != "" || asked != "novox/photos@nox-mesh" || r.Commit != strings.Repeat("a", 40) || r.Repository != "novox/photos" || r.Path != "." { - t.Errorf("registeredAtBranch(photos) = %+v %q %v, asked %s; want novox/photos at nox-mesh's head, at its root", r, why, ok, asked) - } - if _, why, ok := ix.registeredAtBranch(ctx, "unregistered"); ok || why != "" { - t.Errorf("an unregistered module answered %q %v, want neither", why, ok) - } - }) - t.Run("the dry run names every field of the indexer", func(t *testing.T) { - // dry (pending.go) builds an indexer field by field: a field added to Indexer is named there, or this fails. - named := map[string]bool{} - for _, f := range []string{"Module", "Store", "Modules", "Manifests", "Predefined", "Taken", "Verdicts", "OldSections", "Unjudged", - "JudgeID", "SayRefused", "SayAccepted", "Packages", "Graph", "Files", "GoSum", "GoMod", "Now", "Builds", "Named", "Trunk", "BranchHead", - "Read", "Contains", "Logf", "ReadOpen", "OwnBuild", "Message", "Pending", "SayTakeIn", - "Requirements", "Findings", "SeatDefinitions", "ReadFile", "SaySeatDefinition", "Runs", "OwnBuildState", "Heard", "Placement", "RunningWhole", "Revisions", "SayCatalogRevision", "SayFill", "EntryIndex", "NamedSets", "Registry", "SeedModules", "Versions", "SayVersionReleased", "SayVersionsFilled", "Inputs", "InputObjects", "BuildEdges", "Pins", "PinsPending", "ValidationAsks", "AskVerb"} { - named[f] = true - } - ty := reflect.TypeOf(Indexer{}) - for i := range ty.NumField() { - if f := ty.Field(i); f.IsExported() && !named[f.Name] { - t.Errorf("Indexer.%s is not named by dry", f.Name) - } - } - }) -} - -func mustReadPending(t *testing.T, b *takeInBed) []PendingEntry { - t.Helper() - entries, err := ReadPending(context.Background(), b.pending, nil) - if err != nil { - t.Fatal(err) - } - return entries -} - -// judgeSeatsHave says whether the catalogue's seat list holds the verb on the seat now. -func judgeSeatsHave(seat, verb string) bool { - s, _ := catalogue.SeatNamed(seat) - return slices.ContainsFunc(s.Serves, func(v catalogue.Verb) bool { return v.Name == verb }) -} - -// readPending is the bed's pending state as indexer_pending reads it. -func readPending(t *testing.T, b *takeInBed) []PendingEntry { - t.Helper() - entries, err := ReadPending(context.Background(), b.pending, nil) - if err != nil { - t.Fatal(err) - } - return entries -} - -// pinsHeldUnderKeep is a pins state that counts the writes and reads made while keepMu was not held. -type pinsHeldUnderKeep struct { - *memStore - mu *sync.Mutex - outside int -} - -func (p *pinsHeldUnderKeep) seen() { - if p.mu.TryLock() { // it was free: the caller does not hold it - p.mu.Unlock() - p.outside++ - } -} - -func (p *pinsHeldUnderKeep) Get(ctx context.Context, key string) ([]byte, uint64, bool, error) { - p.seen() - return p.memStore.Get(ctx, key) -} - -func (p *pinsHeldUnderKeep) Create(ctx context.Context, key string, value []byte) error { - p.seen() - return p.memStore.Create(ctx, key, value) -} - -func (p *pinsHeldUnderKeep) Update(ctx context.Context, key string, value []byte, revision uint64) error { - p.seen() - return p.memStore.Update(ctx, key, value, revision) -} - -// A-449a: the revision a take-in made is read and pinned holding keepMu, so no other keep can remove it between the two. -func TestATakeInPinsItsRevisionHoldingKeepMu(t *testing.T) { - ctx := context.Background() - ix := &Indexer{Module: testSelf, Revisions: newMemObjects(0)} - pins := &pinsHeldUnderKeep{memStore: newMem(), mu: &ix.keepMu} - ix.Pins = pins - e := meshcatalog.CatalogEntry{Module: "reader", Repository: "novox/r", Manifest: map[string]any{}, Commit: "c1"} - if err := ix.KeepBranchEntries(ctx, "main", 5, map[string]meshcatalog.CatalogEntry{"reader": e}); err != nil { - t.Fatal(err) - } - if rev := ix.PinPipelines(ctx, "main", 5, []meshcatalog.TakeInEntry{{Module: "reader"}}); rev != 1 { - t.Fatalf("PinPipelines answered revision %d, want 1", rev) - } - if _, _, found, _ := pins.memStore.Get(ctx, PinKey("main", 1)); !found { - t.Error("the revision was not pinned") - } - if pins.outside != 0 { - t.Errorf("%d reads and writes of the pins state were made without keepMu", pins.outside) - } -} diff --git a/modules/mesh-delivery-indexer/internal/index/pins.go b/modules/mesh-delivery-indexer/internal/index/pins.go index a4c0f4ec..2fce27e8 100644 --- a/modules/mesh-delivery-indexer/internal/index/pins.go +++ b/modules/mesh-delivery-indexer/internal/index/pins.go @@ -13,7 +13,6 @@ import ( mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" runner "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/keys" @@ -22,17 +21,13 @@ import ( // The indexer keeps the records pinning each catalog graph revision and removes a revision only on an event (design 47 G §153, // §178, §189, §215, §216, §579, A-169, A-171): its state pins holds one key per revision, `.` (the watched branch -// and the revision's number, the branch escaped as a name is), whose value lists the delivery record ids that pin it, and two more -// keys, `made` (written once, create-only, when the state is first used) and `pins-stated` (the `at` of the newest applied event of -// that name). Pins count only once a pins-stated later than made is applied. +// and the revision's number, the branch escaped as a name is), whose value lists the records that pin it: a take-in's and each +// running pipeline's (design 47 H §743, §749). const ( // PinsState is the state's name; PinsCapMiB is its cap in the manifest (about 1,800 keys of about 0.2 KiB). PinsState = "pins" PinsHistory = 1 PinsCapMiB = 4 - // PinsMadeKey and PinsStatedKey are the two keys of the state that are no revision's. - PinsMadeKey = "made" - PinsStatedKey = "pins-stated" // PinMissingKind is the finding of a pinned revision the indexer no longer holds. PinMissingKind = "pin-missing" // PinsTool reads the pins, UnpinTool removes one record's pin by hand. @@ -46,17 +41,12 @@ type PinValue struct { Missing bool `json:"missing,omitempty"` } -// PinStamp is the value of the keys made and pins-stated. -type PinStamp struct { - At time.Time `json:"at"` -} - // PinKey is a revision's key in the state pins. func PinKey(branch string, revision int64) string { return keys.EscapeName(branch) + "." + strconv.FormatInt(revision, 10) } -// ParsePinKey is the branch and the revision a key of the state pins names; false for made, pins-stated and what is no revision's key. +// ParsePinKey is the branch and the revision a key of the state pins names; false for what is no revision's key. func ParsePinKey(key string) (branch string, revision int64, ok bool) { i := strings.LastIndex(key, ".") if i <= 0 { @@ -82,99 +72,26 @@ func pinObject(key string) (string, bool) { return RevisionObject(meshcatalog.CatalogRevision{Branch: branch, Revision: revision}), true } -func (ix *Indexer) pinsStamp(ctx context.Context, key string) (PinStamp, bool, error) { - var s PinStamp - v, _, found, err := ix.Pins.Get(ctx, key) - if err != nil || !found { - return s, false, err - } - if err := json.Unmarshal(v, &s); err != nil { - return s, false, fmt.Errorf("%s.%s: %w", PinsState, key, err) - } - return s, true, nil -} - -// PinsMade writes the key made once, create-only, when the state is first used; a made already written stays. -func (ix *Indexer) PinsMade(ctx context.Context) error { - if ix.Pins == nil { - return nil - } - data, err := json.Marshal(PinStamp{At: ix.now().UTC()}) - if err != nil { - return err - } - if err := ix.Pins.Create(ctx, PinsMadeKey, data); err != nil && !errors.Is(err, ErrConflict) { - return fmt.Errorf("%s.%s: %w", PinsState, PinsMadeKey, err) - } - return nil -} - -// pinsCounted says the pins count: a pins-stated later than made is applied. -func (ix *Indexer) pinsCounted(ctx context.Context) (bool, error) { - made, ok, err := ix.pinsStamp(ctx, PinsMadeKey) - if err != nil || !ok { - return false, err - } - stated, ok, err := ix.pinsStamp(ctx, PinsStatedKey) - if err != nil || !ok { - return false, err - } - return stated.At.After(made.At), nil -} - -// HeardPins applies one event of the seat mesh-delivery the indexer's durable consumer gave it: pinned, freed, heads-replaced or -// pins-stated, by its subject. An error leaves the event unacknowledged, so the bus gives it again; one applied twice changes nothing. -func (ix *Indexer) HeardPins(ctx context.Context, subject string, body []byte) error { +// HeardPins applies one event of the seat mesh-pipeline-runner the indexer's durable consumers gave it (design 47 H §749): a +// pipeline-started pins the revision it names under the record `pipeline.`, its pipeline-ended frees it. A body +// that is no such event, or names no revision, changes nothing; an error leaves the event unacknowledged, so the bus gives it +// again; one applied twice changes nothing. +func (ix *Indexer) HeardPins(ctx context.Context, started *runner.PipelineStarted, ended *runner.PipelineEnded) error { if ix.Pins == nil { return errors.New("the indexer keeps no state " + PinsState) } - if err := ix.PinsMade(ctx); err != nil { - return err - } - switch subject { - case meshdelivery.SubjectPinned, meshdelivery.SubjectFreed: - var p meshdelivery.Pins - if err := json.Unmarshal(body, &p); err != nil { - return fmt.Errorf("%s is not a pins body: %w", subject, err) - } - for _, c := range p.Catalog { - key := PinKey(c.Branch, c.Revision) - if subject == meshdelivery.SubjectPinned { - if err := ix.pin(ctx, key, p.Record); err != nil { - return err - } - } else if err := ix.unpin(ctx, key, p.Record); err != nil { - return err - } - } - case meshdelivery.SubjectHeadsReplaced: - var h meshdelivery.HeadsReplacedEvent - if err := json.Unmarshal(body, &h); err != nil { - return fmt.Errorf("%s is not a heads-replaced body: %w", subject, err) - } - ix.DropCandidateGraph(ctx, h.Group, h.Heads, "mesh-delivery", "its event heads-replaced: the candidate's heads were replaced or its pull request closed") - case meshdelivery.SubjectPinsStated: - var s meshdelivery.PinsStatedEvent - if err := json.Unmarshal(body, &s); err != nil { - return fmt.Errorf("%s is not a pins-stated body: %w", subject, err) - } - err := write(ctx, ix.Pins, PinsStatedKey, func(old []byte, found bool) ([]byte, bool, error) { - var cur PinStamp - if found && json.Unmarshal(old, &cur) == nil && !s.At.After(cur.At) { - return nil, false, nil - } - data, err := json.Marshal(PinStamp{At: s.At.UTC()}) - return data, false, err - }) - if err != nil { - return err - } - default: - return fmt.Errorf("%s is no event of the pins", subject) + switch { + case started != nil && started.Revision > 0 && started.Pipeline != "": + return ix.pin(ctx, PinKey(started.Branch, started.Revision), PipelineRecord(started.Pipeline)) + case ended != nil && ended.Revision > 0 && ended.Pipeline != "": + return ix.unpin(ctx, PinKey(ended.Branch, ended.Revision), PipelineRecord(ended.Pipeline)) } return nil } +// PipelineRecord is the record a pipeline pins its revision under: `pipeline..` (design 47 H §743, A-449a). +func PipelineRecord(pipeline string) string { return "pipeline." + pipeline } + // pin adds a record's id to a revision's key by compare-and-set; a revision not held is marked missing and said. func (ix *Indexer) pin(ctx context.Context, key, record string) error { object, _ := pinObject(key) @@ -332,10 +249,6 @@ func (ix *Indexer) judgePinsLocked(ctx context.Context) ([]string, error) { if err != nil || pending != 0 { return nil, err } - counted, err := ix.pinsCounted(ctx) - if err != nil || !counted { - return nil, err - } names, err := ix.Revisions.Names(ctx) if err != nil { return nil, err @@ -428,18 +341,13 @@ type PinnedRevision struct { type PinsAnswer struct { Pins []PinnedRevision `json:"pins"` // Unpinned are the kept revisions no record pins, current or not, and Current why each current one is. - Unpinned []string `json:"unpinned,omitempty"` - Made *time.Time `json:"made,omitempty"` - Stated *time.Time `json:"pins-stated,omitempty"` - // Counted says the pins count: a pins-stated later than made is applied. - Counted bool `json:"counted"` + Unpinned []string `json:"unpinned,omitempty"` // Fill is the state's bytes against its cap. Fill meshcatalog.Fill `json:"fill"` } // ReadPins reads the state pins and the revisions kept, for indexer_pins. func ReadPins(ctx context.Context, pins Store, o Objects) (PinsAnswer, error) { - ix := &Indexer{Pins: pins, Revisions: o} var out PinsAnswer names, err := o.Names(ctx) if err != nil { @@ -477,13 +385,6 @@ func ReadPins(ctx context.Context, pins Store, o Objects) (PinsAnswer, error) { out.Unpinned = append(out.Unpinned, n) } } - if s, ok, err := ix.pinsStamp(ctx, PinsMadeKey); err == nil && ok { - out.Made = &s.At - } - if s, ok, err := ix.pinsStamp(ctx, PinsStatedKey); err == nil && ok { - out.Stated = &s.At - } - out.Counted, _ = ix.pinsCounted(ctx) out.Fill = meshcatalog.Fill{Bytes: bytes, Cap: PinsCapMiB << 20, Store: PinsState} return out, nil } @@ -495,8 +396,7 @@ func PinsTools(pins Store, o Objects, ask func(key string, value []byte) (TakeIn {Name: PinsTool, Description: "Every catalog graph revision a delivery record pins in the indexer's state pins (design 47 G §579): the revision's " + "key `.`, the records pinning it, whether it was missing when pinned (finding pin-missing), whether it is " + - "kept and whether it is current and why; the kept revisions no record pins; when the state was made and when the newest " + - "pins-stated was applied, and whether pins count yet (a pins-stated later than made); and the state's fill against its cap. " + + "kept and whether it is current and why; the kept revisions no record pins; and the state's fill against its cap. " + "indexer_consumer reads the consumer the events come through; indexer_unpin removes one record's pin.", Input: map[string]any{"type": "object", "properties": map[string]any{}}, Run: func(map[string]any) (any, error) { @@ -575,11 +475,10 @@ func ConsumerTools(st Store) []stdio.Tool { Run: func(map[string]any) (any, error) { return ReadConsumers(context.Background(), st) }}} } -// PinPipelines pins the catalog graph revision a take-in made for each pipeline the take-in starts, under the record -// `pipeline.` (`..` of each entry, the runner's key), in the indexer's own pins state as a -// pinned of the seat mesh-delivery is applied (design 47 G §579, A-449a): so no later take-in removes the revision before -// the runner hears of it. The runner's pinned at the pipeline's start says the same record (a set), and its freed at the -// pipeline's end removes it. It reads the branch's newest revision and pins it holding keepMu, so no other keep (a verdict, a +// PinPipelines pins the catalog graph revision a take-in made for the one pipeline the take-in starts, under the record +// `pipeline..` (the runner's key, design 47 H §743, A-449a), in the indexer's own pins state: so no later +// take-in removes the revision before the runner starts it. The pipeline's pipeline-started names the same record (a set), +// and its pipeline-ended removes it (§749). It reads the branch's newest revision and pins it holding keepMu, so no other keep (a verdict, a // source, an import) can make it non-current and remove it between the two; it answers that revision, 0 when none is kept. func (ix *Indexer) PinPipelines(ctx context.Context, branch string, source int64, entries []meshcatalog.TakeInEntry) int64 { ix.keepMu.Lock() @@ -588,15 +487,9 @@ func (ix *Indexer) PinPipelines(ctx context.Context, branch string, source int64 if ix.Pins == nil || revision <= 0 || len(entries) == 0 { return revision } - if err := ix.PinsMade(ctx); err != nil { - ix.logf("pins: the revision %d of %s is not pinned for its pipelines: %v", revision, branch, err) - return revision - } - for _, e := range entries { - record := "pipeline." + runner.PipelinesByBranchRevisionModuleKey(keys.EscapeName(branch), strconv.FormatInt(source, 10), e.Module) - if err := ix.pin(ctx, PinKey(branch, revision), record); err != nil { - ix.logf("pins: the revision %d of %s is not pinned for %s: %v", revision, branch, record, err) - } + record := PipelineRecord(runner.PipelinesByBranchSourceKey(keys.EscapeName(branch), strconv.FormatInt(source, 10))) + if err := ix.pin(ctx, PinKey(branch, revision), record); err != nil { + ix.logf("pins: the revision %d of %s is not pinned for %s: %v", revision, branch, record, err) } return revision } diff --git a/modules/mesh-delivery-indexer/internal/index/pins_test.go b/modules/mesh-delivery-indexer/internal/index/pins_test.go index 75ca74fc..fb3172d6 100644 --- a/modules/mesh-delivery-indexer/internal/index/pins_test.go +++ b/modules/mesh-delivery-indexer/internal/index/pins_test.go @@ -8,43 +8,34 @@ import ( "time" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" + runner "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" ) -// Design 47 G §579, A-169, A-171: the indexer keeps the records pinning each catalog graph revision, counts a record once, and -// removes a revision only while its consumer is caught up and after a pins-stated later than made: one two records pin stays -// until both free it, one neither pinned nor current goes, a revision pinned and not held is said, and unpin clears it. +// Design 47 H §749 (G §579, A-169, A-171): the indexer pins a revision on a pipeline-started that names it and frees it on its +// pipeline-ended, counts a record once, and removes a revision only while its consumers are caught up, judged against the pins +// its own state holds, never a count of pins stated: one two pipelines pin stays until both free it, one neither pinned nor +// current goes, a revision pinned and not held is said, and unpin clears it. func TestTheIndexerRemovesARevisionOnlyWhenNoRecordPinsItAndItIsNotCurrent(t *testing.T) { ctx := context.Background() clock := time.Date(2026, 11, 12, 9, 0, 0, 0, time.UTC) pending := uint64(0) ix := &Indexer{Module: testSelf, Revisions: newMemObjects(1 << 20), Pins: newMem(), Findings: newFindings(), Now: func() time.Time { return clock }, PinsPending: func(context.Context) (uint64, error) { return pending, nil }} - reader := meshcatalog.CatalogEntry{Module: "reader", Repository: "novox/r", Manifest: map[string]any{}} - for _, commit := range []string{"c1", "c2", "c3"} { - e := reader - e.Commit = commit - if err := ix.KeepBranchEntries(ctx, "main", 5, map[string]meshcatalog.CatalogEntry{"reader": e}); err != nil { - t.Fatal(err) - } - } kept := func() []string { n, _ := ix.Revisions.Names(ctx) return slices.DeleteFunc(n, func(s string) bool { _, _, ok := RevisionGroup(s); return !ok }) } - say := func(subject string, body any) { + started := func(pipeline string, revision int64) { t.Helper() - raw, _ := json.Marshal(body) - if err := ix.HeardPins(ctx, subject, raw); err != nil { + if err := ix.HeardPins(ctx, &runner.PipelineStarted{Branch: "main", Pipeline: pipeline, Revision: revision}, nil); err != nil { t.Fatal(err) } } - pins := func(record string, revisions ...int64) meshdelivery.Pins { - p := meshdelivery.Pins{Record: record, Milestone: "47/G"} - for _, r := range revisions { - p.Catalog = append(p.Catalog, meshdelivery.PinsCatalogItem{Branch: "main", Revision: r}) + ended := func(pipeline string, revision int64) { + t.Helper() + if err := ix.HeardPins(ctx, nil, &runner.PipelineEnded{Branch: "main", Pipeline: pipeline, Revision: revision}); err != nil { + t.Fatal(err) } - return p } want := func(what string, names ...string) { t.Helper() @@ -52,34 +43,45 @@ func TestTheIndexerRemovesARevisionOnlyWhenNoRecordPinsItAndItIsNotCurrent(t *te t.Errorf("%s: kept %v, want %v", what, got, names) } } - say(meshdelivery.SubjectPinned, pins("r1", 1)) - say(meshdelivery.SubjectPinned, pins("r1", 1)) // heard twice: pins once - say(meshdelivery.SubjectPinned, pins("r2", 1)) - ix.JudgePins(ctx) - want("before a pins-stated later than made is applied", "trunk.1", "trunk.2", "trunk.3") - clock = clock.Add(time.Second) - say(meshdelivery.SubjectPinsStated, meshdelivery.PinsStatedEvent{At: clock}) - pending = 3 + reader := meshcatalog.CatalogEntry{Module: "reader", Repository: "novox/r", Manifest: map[string]any{}} + keep := func(commit string) { + t.Helper() + e := reader + e.Commit = commit + if err := ix.KeepBranchEntries(ctx, "main", 5, map[string]meshcatalog.CatalogEntry{"reader": e}); err != nil { + t.Fatal(err) + } + } + keep("c1") + pending = 3 // pinned before the next keeps, which judge the revisions while the consumers are behind + started("main.5", 1) + started("main.5", 1) // heard twice: pins once + started("main.6", 1) + keep("c2") + keep("c3") ix.JudgePins(ctx) want("while messages of the consumer are pending", "trunk.1", "trunk.2", "trunk.3") pending = 0 ix.JudgePins(ctx) want("pinned by two records, the middle one neither pinned nor current", "trunk.1", "trunk.3") - say(meshdelivery.SubjectFreed, pins("r1", 1)) + if v, _, _, _ := ix.Pins.Get(ctx, "main.1"); !slices.Equal(mustRecords(t, v), []string{"pipeline.main.5", "pipeline.main.6"}) { + t.Errorf("the pin of main.1 holds %s, want the two pipelines' records once each", v) + } + ended("main.5", 1) ix.JudgePins(ctx) want("one of two records freed it", "trunk.1", "trunk.3") - say(meshdelivery.SubjectFreed, pins("r2", 1)) + ended("main.6", 1) ix.JudgePins(ctx) want("both freed it", "trunk.3") if keys, _ := ix.Pins.Keys(ctx); slices.Contains(keys, "main.1") { t.Errorf("the state holds %v, want no key of the removed revision", keys) } // A record pins a revision no longer held: its key says missing and the finding names it; unpin clears both. - say(meshdelivery.SubjectPinned, pins("r3", 1)) + started("main.7", 1) if f := ix.Findings.Held(); len(f) != 1 || f[0].Kind != PinMissingKind { t.Fatalf("findings %v, want one %s", f, PinMissingKind) } - if r := ix.Unpin(ctx, "main.1", "r3", "operator", "freed was lost"); !r.Stored { + if r := ix.Unpin(ctx, "main.1", "pipeline.main.7", "operator", "freed was lost"); !r.Stored { t.Fatalf("unpin answered %q", r.Answer) } if f := ix.Findings.Held(); len(f) != 0 { @@ -87,35 +89,12 @@ func TestTheIndexerRemovesARevisionOnlyWhenNoRecordPinsItAndItIsNotCurrent(t *te } } -func TestAPinsStatedBeforeTheStateWasMadeCountsNothing(t *testing.T) { - ctx := context.Background() - clock := time.Date(2026, 11, 12, 9, 0, 0, 0, time.UTC) - ix := &Indexer{Module: testSelf, Revisions: newMemObjects(1 << 20), Pins: newMem(), Now: func() time.Time { return clock }, - PinsPending: func(context.Context) (uint64, error) { return 0, nil }} - raw, _ := json.Marshal(meshdelivery.PinsStatedEvent{At: clock.Add(-time.Hour)}) - if err := ix.HeardPins(ctx, meshdelivery.SubjectPinsStated, raw); err != nil { +func mustRecords(t *testing.T, v []byte) []string { + t.Helper() + var pv PinValue + if err := json.Unmarshal(v, &pv); err != nil { t.Fatal(err) } - if ok, err := ix.pinsCounted(ctx); err != nil || ok { - t.Errorf("counted %v, %v after a pins-stated older than made, want not counted", ok, err) - } -} - -func TestHeadsReplacedDropsTheCandidatesGraph(t *testing.T) { - ctx := context.Background() - ix := &Indexer{Module: testSelf, Revisions: newMemObjects(1 << 20), Pins: newMem()} - rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Kind: "candidate", Group: "feat-x", Heads: "0123456789ab", Revision: 1}, nil) - if err != nil { - t.Fatal(err) - } - if err := KeepRevision(ctx, ix.Revisions, rev, objects); err != nil { - t.Fatal(err) - } - raw, _ := json.Marshal(meshdelivery.HeadsReplacedEvent{Group: "feat-x", Heads: "0123456789ab", Repository: "novox/r", Number: 1}) - if err := ix.HeardPins(ctx, meshdelivery.SubjectHeadsReplaced, raw); err != nil { - t.Fatal(err) - } - if n, _ := ix.Revisions.Names(ctx); len(n) != 0 { - t.Errorf("the objects hold %v, want the candidate's graph dropped", n) - } + slices.Sort(pv.Records) + return pv.Records } diff --git a/modules/mesh-delivery-indexer/internal/index/pushed_test.go b/modules/mesh-delivery-indexer/internal/index/pushed_test.go deleted file mode 100644 index 0817c6e9..00000000 --- a/modules/mesh-delivery-indexer/internal/index/pushed_test.go +++ /dev/null @@ -1,46 +0,0 @@ -package index - -import ( - "context" - "encoding/json" - "testing" - - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" -) - -// Design 47 G §509, §714, A-269: a pushed message's head stands where a merged message's merge stands: the pending value keeps -// it as its ref, the newest load reads it from the bridge's answer and the pending value read back holds it alike. -func TestAPushedMessageStandsInTheRevisionWhereAMergedMessageDoes(t *testing.T) { - b := newTakeInBed(t) - head := "5555555555555555555555555555555555555555" - pushed := b.base - pushed.Kind, pushed.Number, pushed.Merge, pushed.Base, pushed.Branch, pushed.Head = forgemsg.ForgePushed, 0, "", "", "feat/x", head - data, err := forgemsg.EncodeForge(pushed) - if err != nil { - t.Fatal(err) - } - b.bridge[pushed.Repository+"@"+head] = data // the bridge answers by the head, where a merge's commit would stand - - f := RevisionForm(pushed) - if f.Kind != forgemsg.ForgeMerged || f.Merge != head { - t.Fatalf("RevisionForm = %s %q, want a merged form at the head", f.Kind, f.Merge) - } - value, err := encodePending(Pending{Seq: 9, Ref: refOf(f)}) - if err != nil { - t.Fatal(err) - } - _, got, err := DecodePending(value) - if err != nil || got.Merge != head { - t.Fatalf("DecodePending of the ref = %+v, %v, want the head as its merge", got, err) - } - // A value an earlier indexer wrote with the pushed message whole reads alike. - whole, _ := json.Marshal(Pending{V: 1, Seq: 9, Forge: data}) - if _, got, err = DecodePending(whole); err != nil || got.Merge != head || got.Kind != forgemsg.ForgeMerged { - t.Fatalf("DecodePending of the whole message = %+v, %v", got, err) - } - // The load reads the bridge's pushed answer for the ref and accepts it. - loaded, err := b.ix.load(context.Background(), PendingEntry{Key: "m.9", Value: Pending{Seq: 9}, Forge: RevisionForm(pushed), Light: true}) - if err != nil || loaded.Forge.Merge != head || loaded.Light { - t.Fatalf("load = %+v, %v, want the pushed message read at the head", loaded.Forge, err) - } -} diff --git a/modules/mesh-delivery-indexer/internal/index/registered.go b/modules/mesh-delivery-indexer/internal/index/registered.go index ee59138f..f9c86c41 100644 --- a/modules/mesh-delivery-indexer/internal/index/registered.go +++ b/modules/mesh-delivery-indexer/internal/index/registered.go @@ -139,14 +139,9 @@ func (ix *Indexer) indexAt(ctx context.Context, r Registration, force, reread bo if p := read.Packages[dir]; len(p) > 0 { f.Packages = map[string][]string{dir: p} } - w, err := ix.applyManifests(ctx, f, false) - if err != nil { + if _, err := ix.applyManifests(ctx, f, false); err != nil { return t, err } - // A module registered or taken in is judged as a merge's are (ADR 0578 §1, ADR 0579 §1). - if _, err := ix.JudgeAll(ctx, w.fresh); err != nil { - ix.logf("judge: the manifests are not judged after taking in %s: %v", r.Module, err) - } if _, _, found, err := ix.Modules.Get(ctx, key); err != nil { return t, fmt.Errorf("modules.%s: %w", key, err) } else if !found { diff --git a/modules/mesh-delivery-indexer/internal/index/registrations_test.go b/modules/mesh-delivery-indexer/internal/index/registrations_test.go index 75dd24fd..07d5c47c 100644 --- a/modules/mesh-delivery-indexer/internal/index/registrations_test.go +++ b/modules/mesh-delivery-indexer/internal/index/registrations_test.go @@ -248,6 +248,9 @@ func TestSeedWritesTheModuleGraphWithNoAskAndWaitsForEveryBuildSeat(t *testing.T } // A step the seat answers none for is bypass; the statuses the check judged are the rest. got, _, _ := g.r.Registration(ctx, "a") + if got.Policy.Steps.Validate != "required" { // I7/AC2, design 47 H §745 + t.Errorf("the seed wrote validate %q, want required", got.Policy.Steps.Validate) + } if got.Version != 1 || got.Policy.Steps.Review != "optional" || got.Policy.Steps.Approval != "bypass" || got.Policy.Artifacts["program"].Lint != "bypass" || got.Policy.Artifacts["program"].Integration != "bypass" || got.Policy.Artifacts["program"].Unit != "" { t.Fatalf("the seeded registration: %+v", got) } diff --git a/modules/mesh-delivery-indexer/internal/index/remove_test.go b/modules/mesh-delivery-indexer/internal/index/remove_test.go index 2efc5314..98e105a7 100644 --- a/modules/mesh-delivery-indexer/internal/index/remove_test.go +++ b/modules/mesh-delivery-indexer/internal/index/remove_test.go @@ -26,29 +26,6 @@ func twoModules(t *testing.T) (*Indexer, *memStore) { func TestAMergeThatRemovesAModuleDeletesItsKey(t *testing.T) { // 1900/AC1 ix, mods := twoModules(t) - // DLV-I9 (design 25 § 12.2): the module leaving the trunk is said as an empty manifest-accepted, under an id naming it. - var said []meshcatalog.ManifestAcceptedEvent - var ids []string - ix.SayAccepted = func(_ context.Context, a meshcatalog.ManifestAcceptedEvent, id string) error { - said, ids = append(said, a), append(ids, id) - return nil - } - defer func() { - var left []string - for i, a := range said { - if a.Digest == "" { - left = append(left, a.Module) - // Join item 3: the id is the event's own (acceptedEventID), naming the module and the - // modules state's revision of the value that left, so two leaves of one module are two events. - if a.BusState == nil || len(a.BusState) != 0 || a.Revision == 0 || ids[i] != acceptedEventID(a) || !strings.Contains(ids[i], "-left-b-") { - t.Errorf("the empty manifest-accepted = %+v, id %s", a, ids[i]) - } - } - } - if strings.Join(left, ",") != "b" { - t.Errorf("said empty for %v, want b alone", left) - } - }() if err := ix.Apply(context.Background(), mergedMsg(catalogueRepo, map[string]string{"modules/a": "t2"})); err != nil { t.Fatal(err) } diff --git a/modules/mesh-delivery-indexer/internal/index/runcandidates.go b/modules/mesh-delivery-indexer/internal/index/runcandidates.go index cf11454f..df18d8d2 100644 --- a/modules/mesh-delivery-indexer/internal/index/runcandidates.go +++ b/modules/mesh-delivery-indexer/internal/index/runcandidates.go @@ -102,10 +102,12 @@ func (ix *Indexer) EndRunPipeline(ctx context.Context, e meshpipelinerunner.Pipe ix.runMu.Lock() defer ix.runMu.Unlock() r, err := ix.updateRun(ctx, group, func(r *Run) { - if e.Module != "" && !slices.Contains(r.Ended, e.Module) { - r.Ended = append(r.Ended, e.Module) - slices.Sort(r.Ended) + for _, m := range e.Movers { // a run's pipeline-ended names its movers (H-A-72) + if m.Module != "" && !slices.Contains(r.Ended, m.Module) { + r.Ended = append(r.Ended, m.Module) + } } + slices.Sort(r.Ended) }) if err != nil { return err @@ -208,8 +210,6 @@ func (ix *Indexer) writeRunCandidate(ctx context.Context, group string, members return err } pulls := make([]meshcatalog.CandidatesByGroupValuePullsItem, len(placed)) - var entries []PendingEntry - over := map[string]string{} // One take-in per base branch: the members on one base are taken in together, so one member sees another's. bases := map[string]*dryRun{} for i, m := range placed { @@ -266,14 +266,7 @@ func (ix *Indexer) writeRunCandidate(ctx context.Context, group string, members pulls[i] = meshcatalog.CandidatesByGroupValuePullsItem{Repository: m.repository, Head: m.Commit, Modules: []string{m.Module}, Fps: map[string]meshcatalog.CandidatesByGroupValuePullsItemFpsValue{m.Module: {Manifest: mv.Manifest, Fp2: mv.Fp2, None2: mv.None2, Packages: mv.Packages}}} - over[m.Module] = mv.Manifest - entries = append(entries, PendingEntry{Forge: forgemsg.Forge{Repository: m.repository, Merge: m.Commit, Manifests: map[string]string{m.dir: raw}}}) } - seats, err := ix.revisionSeats(ctx, entries) - if err != nil { - return fmt.Errorf("the run %s: reading the seat definitions of its members: %w", group, err) - } - ix.judgeDigestsWith(ctx, seats, over, "the run "+group) c := meshcatalog.CandidatesByGroupValue{V: keys.Version, Group: group, Heads: hash, Pulls: pulls} c.Fpset = meshcatalogclient.FingerprintSet(c.Pulls) if err := ix.refuseDisagreement(c); err != nil { diff --git a/modules/mesh-delivery-indexer/internal/index/runcandidates_test.go b/modules/mesh-delivery-indexer/internal/index/runcandidates_test.go index 203cfd6e..3d440385 100644 --- a/modules/mesh-delivery-indexer/internal/index/runcandidates_test.go +++ b/modules/mesh-delivery-indexer/internal/index/runcandidates_test.go @@ -46,7 +46,7 @@ func TestARunsCandidateIsBuiltOnItsStartAndDroppedWhenEveryMembersPipelineEnded( started := meshpipelinerunner.RunStarted{V: 1, Run: "7", By: "lead@shanks", Members: []meshpipelinerunner.Member{ {Module: "avahi", Commit: c1, Branch: "feature/1966-x"}, {Module: "netcheck", Commit: c2, Branch: "beta"}}} ended := func(module string) meshpipelinerunner.PipelineEnded { - return meshpipelinerunner.PipelineEnded{V: 1, Pipeline: "p", Module: module, Branch: "b", Commit: "c", Outcome: "passed", Run: "7"} + return meshpipelinerunner.PipelineEnded{V: 1, Pipeline: "p", Movers: []meshpipelinerunner.MoverEnded{{Module: module}}, Branch: "b", Outcome: "passed", Run: "7"} } key, _ := meshcatalogclient.CandidatesKey("run-7") t.Run("built on its start, one pull per member, and dropped with the run after a restart", func(t *testing.T) { @@ -107,7 +107,7 @@ func TestARunsCandidateIsBuiltOnItsStartAndDroppedWhenEveryMembersPipelineEnded( if err := ix.StartRun(ctx, lost); !errors.Is(err, ErrUnreadable) { t.Errorf("a member with no registration: %v, want unreadable", err) } - if err := ix.EndRunPipeline(ctx, meshpipelinerunner.PipelineEnded{Module: "avahi"}); err != nil { + if err := ix.EndRunPipeline(ctx, meshpipelinerunner.PipelineEnded{Movers: []meshpipelinerunner.MoverEnded{{Module: "avahi"}}}); err != nil { t.Error(err) } if k, _ := cands.Keys(ctx); len(k) != 0 { diff --git a/modules/mesh-delivery-indexer/internal/index/seed.go b/modules/mesh-delivery-indexer/internal/index/seed.go index d1669c5d..c4f38aa7 100644 --- a/modules/mesh-delivery-indexer/internal/index/seed.go +++ b/modules/mesh-delivery-indexer/internal/index/seed.go @@ -75,7 +75,8 @@ func (r *Registry) ComposeSeed(modules []SeedModule) (SeedProposal, error) { } reg := meshcatalog.Registration{Module: m.Module, Origin: m.Origin, Path: m.Path, Branches: []string{b.Branch}, Policy: meshcatalog.Policy{Steps: meshcatalog.StepStatuses{Lint: "required", Unit: "required", Integration: "required", - Composition: "required", Review: "optional", Approval: "bypass"}}} + Composition: "required", Review: "optional", Approval: "bypass", + Validate: "required"}}} // design 47 H §745: the step validate is required in every policy by default for _, a := range m.Artifacts { var s meshcatalog.StepStatuses if !slices.Contains(a.Steps, "lint") { diff --git a/modules/mesh-delivery-indexer/internal/index/source.go b/modules/mesh-delivery-indexer/internal/index/source.go index 70e1d9fb..d7c04e8a 100644 --- a/modules/mesh-delivery-indexer/internal/index/source.go +++ b/modules/mesh-delivery-indexer/internal/index/source.go @@ -62,7 +62,6 @@ func (ix *Indexer) ApplySource(ctx context.Context, a meshdelivery.SourceAnswer) for _, r := range a.Repositories { s.Heads[r.Repository] = r.Head } - ix.kickJudge(ctx) ix.mu.Lock() if rev < ix.source.rev { ix.mu.Unlock() diff --git a/modules/mesh-delivery-indexer/internal/index/source_test.go b/modules/mesh-delivery-indexer/internal/index/source_test.go index 995f21ee..764df2d8 100644 --- a/modules/mesh-delivery-indexer/internal/index/source_test.go +++ b/modules/mesh-delivery-indexer/internal/index/source_test.go @@ -21,7 +21,7 @@ type srcT struct { func sourceOf(repo, head string) srcT { return srcT{Group: "g", Heads: map[string]string{repo: head}} } func applyS(ix *Indexer, ctx context.Context, rev uint64, s srcT) error { - a := meshdelivery.SourceAnswer{Branch: "main", Group: s.Group, Revision: int64(rev)} + a := meshdelivery.SourceAnswer{Branch: "main", Revision: int64(rev)} for r, h := range s.Heads { a.Repositories = append(a.Repositories, meshdelivery.SourceHead{Repository: r, Head: h, Trunk: "main"}) } diff --git a/modules/mesh-delivery-indexer/internal/index/startread.go b/modules/mesh-delivery-indexer/internal/index/startread.go index feea611a..526a0cb3 100644 --- a/modules/mesh-delivery-indexer/internal/index/startread.go +++ b/modules/mesh-delivery-indexer/internal/index/startread.go @@ -28,9 +28,7 @@ type ViewWaiter interface { // read again once the seat answers, which judges again every manifest judged from kept definitions. type DefinitionsAtStart struct { // Read reads every contracts module's trunk definitions (Indexer.ReadTrunkDefinitions). - Read func(ctx context.Context) (int, error) - // Judge judges every manifest again, as at the start (Indexer.JudgeAtStart). - Judge func(ctx context.Context) error + Read func(ctx context.Context) (int, error) Waits ViewWaiter Bound time.Duration Logf func(string, ...any) @@ -90,19 +88,11 @@ func (d *DefinitionsAtStart) read(ctx context.Context, when string) bool { default: close(d.answered) } - again := d.fromKept d.fromKept = false d.mu.Unlock() if was && d.Waits != nil { d.Waits.Holds(GitDefinitionsView) } - if again && d.Judge != nil { - if err := d.Judge(ctx); err != nil { - d.logf("judging again the manifests judged from kept definitions: %v; judged again at the next message heard", err) - } else { - d.logf("judged again every manifest judged from kept definitions, now that the seat git answers") - } - } return true } diff --git a/modules/mesh-delivery-indexer/internal/index/takein.go b/modules/mesh-delivery-indexer/internal/index/takein.go index ae21ccc0..3297ad9e 100644 --- a/modules/mesh-delivery-indexer/internal/index/takein.go +++ b/modules/mesh-delivery-indexer/internal/index/takein.go @@ -34,12 +34,6 @@ type TakeInAsk struct { // CatchUp asks the head the gate's source names for every repository to be taken in (CatchUp): the catch-up tool, // indexer_take_in called with no module and no repository, as the node agent calls it after a restore. CatchUp bool `json:"catch-up,omitempty"` - // Judge asks the manifests to be judged again now (indexer_judge_again) instead of the module to be taken in. - Judge bool `json:"judge,omitempty"` - // Digest names, with Judge, the one manifest digest of the module to judge (indexer_judge with a digest). - Digest string `json:"digest,omitempty"` - // Say asks the module's manifest-accepted to be said again now (indexer_say_accepted, DLV-I9c). - Say bool `json:"say,omitempty"` // SeatDefinitions asks a contracts module's seat definitions to be read again now (indexer_seat_definitions_again, // DLV-I19): its trunk's, or with Pull and Head a pull request's head. SeatDefinitions bool `json:"seat-definitions,omitempty"` diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains.go b/modules/mesh-delivery-indexer/internal/index/toolchains.go index c3c1927d..452d3fc7 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains.go @@ -418,11 +418,7 @@ func (ix *Indexer) writeOverlay(ctx context.Context, overlay, finals map[string] return nil, false, nil } if next.Manifest != "" && (!held || cur.Manifest != next.Manifest) { - w.taken(next.Manifest) // a new module, or a changed manifest: fresh, judged in full (ADR 0579 §1) - // Kept on the bus before the module names it, so a judge pass that fails does not lose it (issue 1966). - if err := ix.keepUnjudged(ctx, next.Manifest, key, w); err != nil { - return nil, false, err - } + w.taken(next.Manifest) // a new module, or a changed manifest } data, err := meshcatalogclient.EncodeModule(next) wrote = err == nil && !unchanged(data, false, value, found) diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go index e5bb68e1..d4a8ff0a 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go @@ -645,22 +645,19 @@ func TestBasesAreComputedFirstAndNoFingerprintTakesTheBuilder(t *testing.T) { // t.Fatal(err) } } - if got, err := Revision(context.Background(), verbObjects, nil, testSelf, "n1.node-tools", meshcatalog.RevisionInput{}); err != nil || !got.NoneYet || got.Revision != nil || len(got.Entries) != 0 { + if got, err := Revision(context.Background(), verbObjects, nil, meshcatalog.RevisionInput{}); err != nil || !got.NoneYet || got.Revision != nil || len(got.Entries) != 0 { t.Errorf("revision with none kept answered %+v, %v, want none-yet alone with no error (design 47 G §530)", got, err) } keep(meshcatalog.CatalogRevision{Revision: 3, Kind: "trunk"}, map[string]bool{"node-tools": true, testSelf: true}) keep(meshcatalog.CatalogRevision{Revision: 4, Kind: "candidate", Group: "feat/x", Heads: "0123456789ab"}, map[string]bool{"node-tools": true, "mesh-grants": true}) - for caller, answered := range map[string]bool{"n1.node-tools": true, "n1.avahi": true, "n1.mesh-grants": true, "n1." + testSelf: true, "person.someone": false, "unproven": false} { - got, err := Revision(context.Background(), verbObjects, nil, testSelf, caller, meshcatalog.RevisionInput{}) - if (err == nil) != answered || (answered && (got.Revision.Revision != 3 || len(got.Entries) != 4 || got.Next != "")) { - t.Errorf("revision to %s answered %+v (%v), want answered %v with trunk revision 3 whole", caller, got, err, answered) - } + if got, err := Revision(context.Background(), verbObjects, nil, meshcatalog.RevisionInput{}); err != nil || got.Revision.Revision != 3 || len(got.Entries) != 4 || got.Next != "" { + t.Errorf("revision answered %+v (%v), want trunk revision 3 whole", got, err) } - if got, err := Revision(context.Background(), verbObjects, nil, testSelf, "n1.node-tools", meshcatalog.RevisionInput{Group: "feat/x", Heads: "0123456789ab", After: "mesh-grants"}); err != nil || + if got, err := Revision(context.Background(), verbObjects, nil, meshcatalog.RevisionInput{Group: "feat/x", Heads: "0123456789ab", After: "mesh-grants"}); err != nil || got.Revision.Kind != "candidate" || len(got.Entries) != 1 || got.Entries[0].Module != "node-tools" { t.Errorf("a candidate's revision after mesh-grants answered %+v (%v), want its one entry node-tools", got, err) } - if _, err := Revision(context.Background(), verbObjects, nil, testSelf, "n1.node-tools", meshcatalog.RevisionInput{Revision: 9}); err == nil { + if _, err := Revision(context.Background(), verbObjects, nil, meshcatalog.RevisionInput{Revision: 9}); err == nil { t.Error("a trunk revision not kept was answered") } // ADR 0614 §2, §3: the graph names what each module invokes and serves, and leaves out, over the whole graph, the diff --git a/modules/mesh-delivery-indexer/internal/index/unjudged.go b/modules/mesh-delivery-indexer/internal/index/unjudged.go deleted file mode 100644 index fe5ae072..00000000 --- a/modules/mesh-delivery-indexer/internal/index/unjudged.go +++ /dev/null @@ -1,110 +0,0 @@ -package index - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "maps" - "slices" - - "git.novox.be/novox/mesh-sdk/go/process" -) - -// What a take-in newly wrote is kept until judged (issue 1966, ADR 0668 §4; ADR 0579 §1). A manifest a take-in writes new -// is fresh: judged in full, never taken as accepted before. A judge pass that fails (the verdicts state not on the bus, a -// restart between the write and the pass) used to lose that set, which lived only in the take-in's memory: the next pass -// found no verdict, took the manifest as held before any verdict, and wrote a refusal as a warning. So the take-in writes -// each digest into the indexer's own state unjudged before it writes the module naming it, every judge pass judges those -// digests as fresh, and a pass deletes a digest's key once its verdict is written or no module holds it any more. The -// tool indexer_read_unjudged reads the state; indexer_judge judges again by hand, which ends what it holds. -const ( - // UnjudgedState is the indexer's state of the digests a take-in wrote and no judge pass judged yet: the bucket - // mesh-delivery-indexer_unjudged, one key per manifest digest. - UnjudgedState = "unjudged" -) - -// UnjudgedValue is one key of the unjudged state: the module whose new manifest the digest is, and the merge that wrote it. -type UnjudgedValue struct { - V int `json:"v"` - Module string `json:"module"` - Repository string `json:"repository,omitempty"` - Merge string `json:"merge,omitempty"` -} - -// keepUnjudged writes the digest into the unjudged state before the module naming it is written. A state not on the bus -// yet (an indexer whose delivery brings it) is said and the take-in goes on, judged from its memory alone as before; -// any other failure fails the write, so the module is not written with its digest unkept. -func (ix *Indexer) keepUnjudged(ctx context.Context, digest, module string, w *mergeWrite) error { - if ix.Unjudged == nil { - return nil - } - v := UnjudgedValue{V: 1, Module: module} - if w != nil { - v.Repository, v.Merge = w.repository, w.merge - } - data, err := json.Marshal(v) - if err != nil { - return err - } - err = write(ctx, ix.Unjudged, digest, func(value []byte, found bool) ([]byte, bool, error) { - if found { - return nil, false, nil // kept already: the first take-in that wrote it stands - } - return data, false, nil - }) - if errors.Is(err, process.ErrNotYet) { - ix.logf("judge: the state %s is not on the bus yet, so the new manifest %s of %s is kept for its judging only in this take-in", UnjudgedState, digest, module) - return nil - } - if err != nil { - return fmt.Errorf("%s.%s: %w", UnjudgedState, digest, err) - } - return nil -} - -// withUnjudged is fresh with every digest the unjudged state holds; a state that cannot be listed is said and fresh -// alone is judged. -func (ix *Indexer) withUnjudged(ctx context.Context, fresh map[string]bool) map[string]bool { - if ix.Unjudged == nil { - return fresh - } - keys, err := ix.Unjudged.Keys(ctx) - if err != nil { - if !errors.Is(err, process.ErrNotYet) { - ix.logf("judge: the state %s cannot be listed (%v), so only this take-in's new manifests are judged as new", UnjudgedState, err) - } - return fresh - } - if len(keys) == 0 { - return fresh - } - out := maps.Clone(fresh) - if out == nil { - out = map[string]bool{} - } - for _, k := range keys { - out[k] = true - } - return out -} - -// endUnjudged deletes each key of the unjudged state whose digest the pass judged with its verdict written, or which no -// module holds any more (a later merge replaced it): what is left waits for the next pass. -func (ix *Indexer) endUnjudged(ctx context.Context, judged, held map[string]bool) { - if ix.Unjudged == nil { - return - } - keys, err := ix.Unjudged.Keys(ctx) - if err != nil { - return - } - for _, k := range slices.Sorted(slices.Values(keys)) { - if held[k] && !judged[k] { - continue - } - if err := write(ctx, ix.Unjudged, k, func(_ []byte, found bool) ([]byte, bool, error) { return nil, found, nil }); err != nil { - ix.logf("judge: %s.%s is not ended: %v; judged again as new by the next pass", UnjudgedState, k, err) - } - } -} diff --git a/modules/mesh-delivery-indexer/internal/index/verdicts.go b/modules/mesh-delivery-indexer/internal/index/verdicts.go deleted file mode 100644 index 526bb638..00000000 --- a/modules/mesh-delivery-indexer/internal/index/verdicts.go +++ /dev/null @@ -1,807 +0,0 @@ -package index - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "maps" - "slices" - "strings" - "time" - - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" - "git.novox.be/novox/mesh-sdk/go/catalogue" - "git.novox.be/novox/mesh-sdk/go/process" - - "mesh-delivery-indexer/judge" -) - -// The indexer judges every manifest it holds, once, with its own judge (novox/hq ADR 0578 §1): it writes each verdict in -// its state verdicts (VerdictsState), one key per manifest digest, only when the verdict differs from the one -// held; a module's value in the modules state carries a refusal (Verdict, Problems), so a reader of modules takes it -// without a second read; a new refusal is said as the event manifest-refused; a manifest with a field the judge does not know waits instead (ownBuildIsNewest). It judges at its start, so a judge that -// changed (a new indexer) judges every manifest again, and after every merge it takes in, since a rule between manifests -// can turn on any of them. Nothing polls. - -// VerdictsState is the indexer's state of verdicts, its local name, as the tools bundle reads it. -const VerdictsState = "verdicts" - -// Judged is what one pass of the judge did: the manifests judged, the verdicts written, and the modules refused. -type Judged struct { - Judged, Written int - Refused map[string][]string - // Outgrown are the modules accepted before whose problems are now warnings (ADR 0579 §1). - Outgrown []string - // Waiting are the modules whose manifest carries a field this judge does not know and waits. - Waiting []string -} - -// JudgeAll judges every manifest the modules state names, as one catalogue with the predefined list the indexer keeps, -// and writes what differs. A manifest the manifests state does not hold is left unjudged and said; a write that fails is -// said and the pass goes on, so one key never costs the others their verdict. -// -// **A stricter judge never refuses what it accepted before** (novox/hq ADR 0579 §1): a manifest whose verdict under its -// digest is accepted, or which the indexer held before any verdict (taken as accepted, ADR 0578 §3) and is not in fresh, -// stays accepted, the new problems written as the verdict's Warnings and carried in its module's value, which the -// watchman raises as manifest-outgrown. A manifest in fresh (the digests a merge just took in) with no verdict yet, and -// one already refused, is judged in full. -func (ix *Indexer) JudgeAll(ctx context.Context, fresh map[string]bool) (Judged, error) { - return ix.judgeOver(ctx, fresh, "", "") -} - -// judgeOver is JudgeAll; with a module and a manifest digest it judges that digest alone, in the catalogue the modules -// state holds with the module's manifest replaced by it (a module the state does not hold added), and writes its verdict -// and nothing else: no event, no count of the old sections, and no module's value but one that names the digest judged -// (carryVerdict). A digest with no verdict yet is judged in full, as a fresh one. One digest at a time, so the known -// packages dropped for a digest the trunk does not hold are dropped for that digest alone (review round 1, J7's note). -func (ix *Indexer) judgeOver(ctx context.Context, fresh map[string]bool, overModule, overDigest string) (Judged, error) { - var over map[string]string - if overModule != "" { - over = map[string]string{overModule: overDigest} - } - // A merge, the start pass and indexer_judge may all judge at once: one pass at a time, so none writes a verdict from - // modules another is rewriting. - ix.judgeMu.Lock() - defer ix.judgeMu.Unlock() - if !ix.underSeats { - // The seat list stands still for this pass: a take-in's dry run may put its revision's seats over it (seatsof.go). - defer judge.HoldSeats()() - } - return ix.judgeOverHeld(ctx, fresh, over) -} - -// judgeOverHeld is judgeOver for a caller that holds the judge pass (judgeMu) and keeps the seat list as it wants it -// (judge.HoldSeats, or inside judge.WithSeats): it takes no lock, so a pass that put its seats over the list judges inside -// it without asking the shared lock the exclusive one forbids. -func (ix *Indexer) judgeOverHeld(ctx context.Context, fresh map[string]bool, over map[string]string) (Judged, error) { - var out Judged - if ix.Verdicts == nil || ix.Modules == nil || ix.Manifests == nil { - return out, nil - } - if over == nil { - ix.SayUnsaid(ctx) // a manifest-accepted a pass before failed to publish is said again first (DLV-I9b) - // What a take-in wrote and no pass judged yet is fresh too, whichever pass judges it (unjudged.go, issue 1966). - fresh = ix.withUnjudged(ctx, fresh) - } - held, err := ix.heldModules(ctx) - if err != nil { - return out, err - } - heldDigests, judgedDigests := map[string]bool{}, map[string]bool{} - for _, m := range held { - heldDigests[m.Manifest] = m.Manifest != "" - } - trunkOf := map[string]string{} // a judged change's module to the trunk's manifest digest it changes (DLV-I15) - // The known packages a manifest's `packages` may name (DLV-I14). The digest judged the trunk does not hold (a pull - // request's candidate, one the trunk no longer names) is not judged on them: the checker judges a change's names, - // knowing its delivery group, whose new packages this state does not know yet (the lead's ruling, issue 1966). The - // trunk's own manifests are, and a merge's are again in full (namesAtMerge). - known := ix.knownPackages(ctx) - // A digest judged over the trunk that is the trunk's own held one is said as the full pass says it (manifest-accepted, - // manifest-refused): indexer_judge with it, or the start pass's judgeStale, is that manifest's only judgement (review - // round 1 of issue 1966; ADR 0668 §4). A candidate's verdict is said nowhere. - onTrunk := map[string]bool{} - if over != nil { - fresh = map[string]bool{} - for module, digest := range over { - m := held[module] - onTrunk[module] = m.Manifest == digest - if m.Manifest != "" && m.Manifest != digest { - trunkOf[module] = m.Manifest - } - if m.Manifest != digest { - known = nil - } - m.Manifest = digest - held[module] = m - fresh[digest] = true - } - } - raws := map[string][]byte{} - digests := map[string]string{} - for _, name := range slices.Sorted(maps.Keys(held)) { - digest := held[name].Manifest - if digest == "" { - continue - } - value, _, found, err := ix.Manifests.Get(ctx, digest) - if err != nil { - return out, fmt.Errorf("reading manifests.%s of %s: %w", digest, name, err) - } - if !found { - ix.logf("judge: the manifest of %s (digest %s) is not in the manifests state, so it is not judged", name, digest) - continue - } - kept, err := meshcatalogclient.DecodeManifest(value) - if err != nil { - ix.logf("judge: manifests.%s of %s cannot be read, so it is not judged: %v", digest, name, err) - continue - } - raws[name] = []byte(kept.Manifest) - digests[name] = digest - } - predefined := ix.predefinedList(ctx) - // The old sections end when no module is left in them (issue 1966 D4): the count is the accepted set the last pass held. - var counts judge.OldSectionCounts - if ix.OldSections != nil { - counts = ix.heldOldSections(ctx) - } - refused, waits, unnamed := judge.ShelfWaiting(raws, predefined, 0, counts, known) - // A change's renamed-from is judged against the trunk's manifest of the same module (DLV-I15, design 32). - for module, digest := range trunkOf { - if _, waiting := waits[module]; waiting || raws[module] == nil { - continue - } - value, _, found, err := ix.Manifests.Get(ctx, digest) - kept, derr := meshcatalogclient.DecodeManifest(value) - if err != nil || !found || derr != nil { - ix.logf("judge: the trunk's manifest %s of %s cannot be read, so its renames are not judged against it", digest, module) - continue - } - head, herr := catalogue.Decode(raws[module]) - trunk, terr := catalogue.Decode([]byte(kept.Manifest)) - if herr != nil || terr != nil { - continue // the manifest's own problems say why - } - if p := judge.RenameProblems(module, head, trunk, predefined); len(p) > 0 { - refused[module] = append(refused[module], p...) - } - } - for _, p := range unnamed { - ix.logf("judge: a problem between the manifests that no one module's removal ends, so it refuses none: %s", p) - } - // A package's name belongs to one module (design 47 G §491, §676): the trunk's manifests only, since a candidate's - // revision is validated by the validator built from its own heads (§16). - if over == nil { - if refused == nil { - refused = map[string][]string{} - } - for module, p := range judge.PackageNameProblems(ix.packageEntries(ctx, held, raws)) { - if _, waiting := waits[module]; !waiting { - refused[module] = append(refused[module], p...) - } - } - } - out.Refused = refused - // A manifest with a field this judge does not know waits for a judge that may know it (clause 3 of the record on a - // release's Approve; ADR 0578 §1): it is refused only when this program is the newest build of the indexer the trunk - // holds, so no judge that could know the field is still to come. - newest := ix.ownBuildIsNewest(held) - id := ix.judgeID() - var accepted []catalogue.Manifest - for _, name := range slices.Sorted(maps.Keys(digests)) { - if _, judged := over[name]; over != nil && !judged { - continue - } - out.Judged++ - v := meshcatalog.VerdictsByDigestValue{Module: name, State: meshcatalogclient.VerdictAccepted, Judge: id, At: ptr(ix.now())} - if fields := waits[name]; len(fields) > 0 { - accepted := ix.acceptedBefore(ctx, digests[name], fresh) - if !newest && !accepted { - v.State, v.Unknown = meshcatalogclient.VerdictManifestWaits, fields - out.Waiting = append(out.Waiting, name) - } else { - _, problems := judge.Manifest(raws[name], ix.predefinedList(ctx)) - if accepted { - v.Warnings = problems - out.Outgrown = append(out.Outgrown, name) - } else { - v.State, v.Problems = meshcatalogclient.VerdictRefused, problems - } - } - } else if problems := refused[name]; len(problems) > 0 { - if ix.acceptedBefore(ctx, digests[name], fresh) && !namesAtMerge(name, raws[name], fresh[digests[name]] && over == nil, known) { - v.Warnings = problems - out.Outgrown = append(out.Outgrown, name) - } else { - v.State, v.Problems = meshcatalogclient.VerdictRefused, problems - } - } - if !meshcatalogclient.VerdictRefuses(v) { - if m, err := catalogue.Decode(raws[name]); err == nil { - m.Module = name - accepted = append(accepted, m) - } - } - wrote, revision, err := ix.writeVerdict(ctx, digests[name], v) - if errors.Is(err, process.ErrNotYet) { - return out, fmt.Errorf("the state %s is not on the bus yet: %w", VerdictsState, err) - } - if err != nil { - ix.logf("judge: the verdict on %s (digest %s) is not written: %v", name, digests[name], err) - continue - } - judgedDigests[digests[name]] = true - if wrote && over != nil && !onTrunk[name] { - out.Written++ - ix.logf("judge: %s judged again by %s: %s", digests[name], id, v.State) - } else if wrote { - out.Written++ - if meshcatalogclient.VerdictWaits(v) { - ix.logf("judge: %s", meshcatalogclient.VerdictWhy(v)) - } - if meshcatalogclient.VerdictRefuses(v) { - ix.logf("judge: %s", meshcatalogclient.VerdictWhy(v)) - if ix.SayRefused != nil { - r := refusedOf(digests[name], v) - r.Revision = int64(revision) - ix.SayRefused(ctx, r) - } - } else if !meshcatalogclient.VerdictWaits(v) { - // Every manifest newly accepted is said whole to the bus module, its entries empty when it asks for none - // (issue 1966 D3, ADR 0595 §4, design 25 § 12.2); a manifest set aside to wait gets none (DLV-I9). - a := acceptedOf(digests[name], v, revision, raws[name]) - _ = ix.sayAccepted(ctx, a, acceptedEventID(a)) - } - } else if over == nil && fresh[digests[name]] && !meshcatalogclient.VerdictHeld(v) { - // A manifest a merge just made its module's trunk manifest is newly accepted there, even when a pull request's - // judgement wrote its verdict before the merge and said it nowhere (a candidate's verdict is not said): it is - // said now from the verdict held, with its revision, so the bus module and the grants part hear it (design 25 - // bus-state §1; issue 1966: claude-licence-manager's new builds waited for a grant nothing wrote). - ix.sayHeld(ctx, digests[name], raws[name]) - } - // The reader capabilities the manifest's own words, forms and seat names need, named with the verdict (ADR 0614 §4). - needs, _ := catalogue.ManifestNeeds(raws[name]) - if err := ix.carryVerdict(ctx, name, digests[name], v, needs); err != nil { - ix.logf("judge: modules.%s does not carry its verdict: %v", name, err) - } - } - // The verdicts carried are kept with the view (KeepView), so a restart reads them back. - if err := ix.KeepView(ctx); err != nil { - ix.logf("judge: the verdicts carried are not kept in a catalog graph revision: %v", err) - } - if over != nil { - return out, nil - } - ix.endUnjudged(ctx, judgedDigests, heldDigests) - if err := ix.keepOldSections(ctx, judge.CountOldSections(accepted), judge.CountOwnerReads(accepted)); err != nil { - ix.logf("judge: the count of the old sections is not kept: %v", err) - } - return out, nil -} - -// sayHeld says manifest-accepted for the trunk's manifest of digest from the accepted verdict the verdicts state holds on -// it, with that verdict's revision; a verdict that cannot be read, or that refuses or waits, says nothing. -func (ix *Indexer) sayHeld(ctx context.Context, digest string, raw []byte) { - key, err := meshcatalogclient.VerdictsKey(digest) - if err != nil { - return - } - value, revision, found, err := ix.Verdicts.Get(ctx, key) - if err != nil || !found { - ix.logf("judge: the verdict on %s cannot be read, so its manifest-accepted is not said: %v", digest, err) - return - } - v, err := meshcatalogclient.DecodeVerdict(value) - if err != nil || meshcatalogclient.VerdictHeld(v) { - return - } - a := acceptedOf(digest, v, revision, raw) - _ = ix.sayAccepted(ctx, a, acceptedEventID(a)) -} - -// JudgeAtStart is JudgeAll of every manifest held, none fresh, as the indexer starts (a judge that changed judges every -// one again); one that fails is judged again at the next message or source revision heard (kickJudge). Then every other -// digest of the verdicts state that another judge refused or held waiting (one of a candidate, or one back on the trunk -// later) is judged again by this judge (judgeStale), so a refusal an older judge made never stands for good. -func (ix *Indexer) JudgeAtStart(ctx context.Context) (Judged, error) { - j, err := ix.JudgeAll(ctx, nil) - ix.judgePending.Store(err != nil) - if err == nil { - ix.judgeStale(ctx) - } - return j, err -} - -// judgeID is the judge a verdict of this program names. -func (ix *Indexer) judgeID() string { - if ix.JudgeID != "" { - return ix.JudgeID - } - return judge.Identity() -} - -// staleVerdict says whether the digest's verdict wants this judge's judgement: there is none, or another judge refused -// it or held it waiting. An accepted verdict stays accepted under any later judge (ADR 0579 §1), so judging it again -// changes nothing. -func (ix *Indexer) staleVerdict(ctx context.Context, digest string) bool { - value, _, found, err := ix.Verdicts.Get(ctx, digest) - if err != nil { - return false // unread: left, said by the next pass that reads it - } - if !found { - return true - } - v, err := meshcatalogclient.DecodeVerdict(value) - return err != nil || (meshcatalogclient.VerdictHeld(v) && v.Judge != ix.judgeID()) -} - -// judgeDigests judges each digest of over (module to digest) whose verdict is stale (staleVerdict), one at a time in the -// catalogue the modules state holds; a pass that fails is said and the others go on. -func (ix *Indexer) judgeDigests(ctx context.Context, over map[string]string, why string) { - if ix.Verdicts == nil || ix.Manifests == nil || ix.Modules == nil { - return - } - for _, module := range slices.Sorted(maps.Keys(over)) { - if !ix.staleVerdict(ctx, over[module]) { - continue - } - if _, err := ix.judgeOver(ctx, nil, module, over[module]); err != nil { - ix.logf("judge: the manifest %s of %s (%s) is not judged: %v", over[module], module, why, err) - } - } -} - -// judgeDigestsWith is judgeDigests with the seats put over the catalogue's list for the whole pass (judge.WithSeats): one -// locked pass that takes the seats, and never a second lock under it (the exclusive one of WithSeats and the shared one of -// judgeOver would wait for each other for good). -func (ix *Indexer) judgeDigestsWith(ctx context.Context, seats []catalogue.Seat, over map[string]string, why string) { - if ix.Verdicts == nil || ix.Manifests == nil || ix.Modules == nil { - return - } - ix.judgeMu.Lock() - defer ix.judgeMu.Unlock() - judge.WithSeats(seats, func() { - for _, module := range slices.Sorted(maps.Keys(over)) { - if !ix.staleVerdict(ctx, over[module]) { - continue - } - if _, err := ix.judgeOverHeld(ctx, nil, map[string]string{module: over[module]}); err != nil { - ix.logf("judge: the manifest %s of %s (%s) is not judged: %v", over[module], module, why, err) - } - } - }) -} - -// judgeStale judges again every digest of the verdicts state another judge refused or held waiting, each as a manifest -// of the module its verdict names (judgeDigests). Never on a timer: at the start pass. -func (ix *Indexer) judgeStale(ctx context.Context) { - if ix.Verdicts == nil { - return - } - keys, err := ix.Verdicts.Keys(ctx) - if err != nil { - ix.logf("judge: the verdicts other judges made are not judged again: listing the verdicts: %v", err) - return - } - byModule := map[string][]string{} - for _, digest := range keys { - value, _, found, err := ix.Verdicts.Get(ctx, digest) - if err != nil || !found { - continue - } - if v, err := meshcatalogclient.DecodeVerdict(value); err == nil && meshcatalogclient.VerdictHeld(v) && v.Judge != ix.judgeID() { - byModule[v.Module] = append(byModule[v.Module], digest) - } - } - for _, module := range slices.Sorted(maps.Keys(byModule)) { - for _, digest := range byModule[module] { - ix.judgeDigests(ctx, map[string]string{module: digest}, "made by another judge") - } - } -} - -// namesAtMerge says whether a manifest a merge takes in (atMerge) names a package known does not: it is refused for it even -// when its digest was accepted before, since that verdict was its candidate's, judged without the names (judgeOver), so -// no stricter judge turns an acceptance into a refusal here (ADR 0579 §1). -func namesAtMerge(module string, raw []byte, atMerge bool, known judge.KnownPackages) bool { - if !atMerge || known == nil { - return false - } - m, err := catalogue.Decode(raw) - return err == nil && len(judge.UnknownPackageNames(module, m, known)) > 0 -} - -// knownPackages are the packages a manifest's `packages` may name (DLV-I14, design 32): every name the packages state's -// series keys hold, as KeepPackageNames reads it at each merge, and the Go module path of every per-module record the -// state holds from the trunk: while no build module builds Go package artifacts, a Go module the trunk holds a record of -// has no series and is known by its module path, never by a package inside it (design 32; issue 1966). Nil, which judges -// no name, where the indexer keeps no packages state or it cannot be listed (said): a state the judge cannot read refuses no manifest. -func (ix *Indexer) knownPackages(ctx context.Context) judge.KnownPackages { - if ix.Packages == nil { - return nil - } - keys, err := ix.Packages.Keys(ctx) - if err != nil { - ix.logf("judge: the packages state cannot be listed, so the names of packages are not judged: %v", err) - return nil - } - known := judge.KnownPackages{} - for _, key := range keys { - value, _, found, err := ix.Packages.Get(ctx, key) - if err != nil { - ix.logf("judge: packages.%s cannot be read, so the names of packages are not judged: %v", key, err) - return nil - } - if !IsSeriesKey(key) { - if r, err := DecodePublishedPackages(value); found && err == nil && r.Module != "" { - known[r.Module] = true - } - continue - } - if s, err := DecodeSeriesPackage(value); found && err == nil { - known[s.Name] = true - } - } - return known -} - -// kickJudge judges again a start pass that failed, when the indexer hears something: never on a timer. -func (ix *Indexer) kickJudge(ctx context.Context) { - if !ix.judgePending.Load() { - return - } - if j, err := ix.JudgeAtStart(ctx); err != nil { - ix.logf("judge: the manifests are still not judged: %v", err) - } else { - ix.logf("judge: judged %d manifests again after a failed start: %d verdicts written", j.Judged, j.Written) - } -} - -// acceptedBefore says whether the manifest of digest was accepted before this judgement: its verdict says accepted, or -// it has none and is not fresh (held before any verdict, so taken in under the judges that ran until then). -func (ix *Indexer) acceptedBefore(ctx context.Context, digest string, fresh map[string]bool) bool { - value, _, found, err := ix.Verdicts.Get(ctx, digest) - if err != nil { - return false // unread: judged in full, said by the write that follows - } - if !found { - return !fresh[digest] - } - prev, err := meshcatalogclient.DecodeVerdict(value) - return err == nil && prev.State == meshcatalogclient.VerdictAccepted -} - -// writeVerdict writes v under the digest's key unless the verdict held there says the same (ManifestVerdict.Same); wrote -// says it was written, and revision is the verdicts state's revision of that write, which the event saying it carries -// (novox/hq ADR 0622 §2). It is read back after the write: when another write came between, the revision read holds -// another verdict and revision is 0, so a reader takes the later write's own event and never pairs v with its revision. -func (ix *Indexer) writeVerdict(ctx context.Context, digest string, v meshcatalog.VerdictsByDigestValue) (wrote bool, revision uint64, err error) { - key, err := meshcatalogclient.VerdictsKey(digest) - if err != nil { - return false, 0, err - } - err = write(ctx, ix.Verdicts, key, func(value []byte, found bool) ([]byte, bool, error) { - wrote = false - if found { - if cur, err := meshcatalogclient.DecodeVerdict(value); err == nil && meshcatalogclient.SameVerdict(cur, v) { - return nil, false, nil - } - } - wrote = true - next, err := meshcatalogclient.EncodeVerdict(v) - return next, false, err - }) - if err != nil || !wrote { - return wrote, 0, err - } - if value, rev, found, err := ix.Verdicts.Get(ctx, key); err == nil && found { - if cur, err := meshcatalogclient.DecodeVerdict(value); err == nil && meshcatalogclient.SameVerdict(cur, v) { - revision = rev - } - } - return wrote, revision, nil -} - -// OwnBuildState is the indexer's state of the build it runs (the bucket mesh-delivery-indexer_own-build, one key, OwnBuildKey): -// written once, at the process's start by FixOwnBuild, so indexer_pending says which build this process knows it runs, and since -// when. The build only names the writer of what is written (A-476 b). -const ( - OwnBuildStateName = "own-build" - OwnBuildKey = "state" -) - -// OwnBuild is the value of the own-build state. -type OwnBuild struct { - V int `json:"v"` - Known bool `json:"known"` - FP string `json:"fp,omitempty"` - Since time.Time `json:"since"` - Why string `json:"why,omitempty"` -} - -// FixOwnBuild fixes, once and at the process's start, which build of the indexer this program is: asked of the node-engine's -// seat (OwnBuild; never the trunk's value of the indexer, which names a build that may be merged and not delivered), before -// this process hears a merge. It is never asked again: while the node-engine replaces the indexer the seat already names the -// successor, and an old process that learnt it later would take revisions in with its old judge. Where the start's ask is not -// answered the build stays unknown for the life of this process, said in the own-build state and the journal; nothing waits for it. -func (ix *Indexer) FixOwnBuild(ctx context.Context) { - ix.ownMu.Lock() - if ix.ownFixed || ix.OwnBuild == nil { - ix.ownMu.Unlock() - return - } - ix.ownFixed = true - ix.ownMu.Unlock() - fp, err := ix.OwnBuild(ctx) - if err == nil && fp == "" { - err = errors.New("the node-engine names no build of the indexer") - } - st := OwnBuild{V: 1, Since: ix.now().UTC()} - if err != nil { - st.Why = err.Error() - ix.logf("judge: the build this program runs is not known (%v); merges are taken in all the same, and it is learnt only at the start", err) - } else { - st.Known, st.FP = true, fp - ix.logf("judge: this program runs the indexer build %s, fixed here at its start", fp) - } - ix.ownMu.Lock() - ix.ownFP, ix.ownSince, ix.ownWhy = fp, st.Since, st.Why - ix.ownMu.Unlock() - if ix.OwnBuildState == nil { - return - } - data, _ := json.Marshal(st) - if _, rev, found, gerr := ix.OwnBuildState.Get(ctx, OwnBuildKey); gerr == nil && found { - err = ix.OwnBuildState.Update(ctx, OwnBuildKey, data, rev) - } else if gerr == nil { - err = ix.OwnBuildState.Create(ctx, OwnBuildKey, data) - } else { - err = gerr - } - if err != nil { - ix.logf("the own-build state cannot be written: %v", err) - } -} - -// ownBuildIsNewest says whether the build of the indexer this program runs (ownFP, from the node-engine) is the newest the -// trunk holds: the indexer's own modules value names it. A newer build merged and not yet delivered ends it at the trunk's -// merge, as one that pins a changed mesh-sdk always is; false when either is unknown. -func (ix *Indexer) ownBuildIsNewest(held map[string]meshcatalog.ModulesByModuleValue) bool { - fp := held[ix.Module].Fp2 - if fp == "" { - return false - } - ix.ownMu.Lock() - defer ix.ownMu.Unlock() - return ix.ownFP != "" && ix.ownFP == fp -} - -// carryVerdict puts a refusal into the module's value, and takes one out once the manifest is accepted, while the value -// still names the digest judged, and the reader capabilities the manifest needs beside it (needs, ADR 0614 §4); an -// accepted module's value carries no verdict, so judging writes no value of a module the -// judge accepts (a reader takes a value without a verdict as accepted, meshcatalogclient.TakeVerdict). -func (ix *Indexer) carryVerdict(ctx context.Context, name, digest string, v meshcatalog.VerdictsByDigestValue, needs []string) error { - key, err := meshcatalogclient.ModulesKey(name) - if err != nil { - return err - } - return write(ctx, ix.Modules, key, func(value []byte, found bool) ([]byte, bool, error) { - if !found { - return nil, false, nil - } - cur, err := meshcatalogclient.DecodeModule(value) - if err != nil || cur.Manifest != digest { - return nil, false, nil // gone, unreadable, or moved to another manifest since: the next pass judges it - } - // A wait is written as a refusal naming the fields, so a reader pinned before the verdict waits holds it as refused. - state, problems := "", v.Problems - switch { - case meshcatalogclient.VerdictWaits(v): - state, problems = meshcatalogclient.VerdictRefused, []string{meshcatalogclient.WaitsOn(v)} - case meshcatalogclient.VerdictRefuses(v): - state = meshcatalogclient.VerdictRefused - } - // The judge reads the manifest alone; the forms its templates' text writes came with its merge (ADR 0632 §2) and - // stand, cur being of the same manifest. - needs := withTemplateForms(needs, cur.Needs) - if cur.Verdict == state && slices.Equal(cur.Problems, problems) && slices.Equal(cur.Unknown, v.Unknown) && slices.Equal(cur.Warnings, v.Warnings) && - slices.Equal(cur.Needs, needs) { - return nil, false, nil - } - cur.Verdict, cur.Problems, cur.Unknown, cur.Warnings, cur.Needs = state, problems, v.Unknown, v.Warnings, needs - if state == "" { - cur.Problems = nil - } - next, err := meshcatalogclient.EncodeModule(cur) - return next, false, err - }) -} - -// predefinedList is the catalogue's predefined list the indexer keeps (writePredefined), or nil when it keeps none or -// it cannot be read: the judge then takes each fallback as its value declares it. -func (ix *Indexer) predefinedList(ctx context.Context) catalogue.PredefinedList { - if ix.Predefined == nil { - return nil - } - value, _, found, err := ix.Predefined.Get(ctx, PredefinedListKey) - if err != nil || !found { - return nil - } - var kept struct { - List json.RawMessage `json:"list"` - } - if json.Unmarshal(value, &kept) != nil { - return nil - } - list, err := catalogue.ParsePredefined(kept.List) - if err != nil { - ix.logf("judge: the predefined list kept cannot be read, so fallbacks are judged as their values declare them: %v", err) - return nil - } - return list -} - -// The tools that read and steer the verdicts (the instruction "tools are how a module is spoken to"): indexer_verdicts -// lists the manifests that wait or are refused, and indexer_judge judges one module again now. -const ( - VerdictsTool = "indexer_verdicts" - JudgeAgainTool = "indexer_judge" - judgeAgainPrefix = "judge." -) - -// VerdictRow is one verdict as indexer_verdicts lists it: the manifest's digest, the module, its state, the fields it waits -// on or the problems that refuse it, and the judge that made it. -type VerdictRow struct { - Module string `json:"module"` - Digest string `json:"digest"` - State string `json:"state"` - Words string `json:"words,omitempty"` - Unknown []string `json:"unknown,omitempty"` - Problems []string `json:"problems,omitempty"` - Warnings []string `json:"warnings,omitempty"` - Judge string `json:"judge"` - At time.Time `json:"at"` -} - -// ListVerdicts reads the verdicts under keys with get and lists those of module (every module when empty) that wait or are -// refused, or every one with all, sorted by module and digest. A value that does not read is said in unread by its key. -func ListVerdicts(keys []string, get func(key string) ([]byte, error), module string, all bool) (rows []VerdictRow, unread map[string]string) { - unread = map[string]string{} - for _, key := range keys { - value, err := get(key) - if err != nil || value == nil { - if err != nil { - unread[key] = err.Error() - } - continue - } - v, err := meshcatalogclient.DecodeVerdict(value) - if err != nil { - unread[key] = err.Error() - continue - } - if (module != "" && v.Module != module) || (!all && !meshcatalogclient.VerdictHeld(v)) { - continue - } - rows = append(rows, VerdictRow{Module: v.Module, Digest: key, State: v.State, Words: meshcatalogclient.VerdictWhy(v), Unknown: v.Unknown, - Problems: v.Problems, Warnings: v.Warnings, Judge: v.Judge, At: timeOf(v.At)}) - } - slices.SortFunc(rows, func(a, b VerdictRow) int { - if c := strings.Compare(a.Module, b.Module); c != 0 { - return c - } - return strings.Compare(a.Digest, b.Digest) - }) - return rows, unread -} - -// JudgeAgain judges the manifests again now, as at the start, and answers how the module's stands: the tool -// indexer_judge. A module the modules state does not hold, or whose manifest is not in the manifests state, is -// answered so. With a digest it judges that manifest alone, in the catalogue the trunk holds (judgeOver), whoever made -// its verdict: a candidate's, or one the trunk names again, refused by an older judge. -func (ix *Indexer) JudgeAgain(ctx context.Context, module, digest string) TakenIn { - out := TakenIn{Module: module} - held, err := ix.heldModules(ctx) - if err != nil { - out.Answer = "the modules state cannot be read: " + err.Error() - return out - } - if digest != "" { - // One digest (a candidate's, or one the trunk no longer names), judged by this judge in the catalogue the trunk holds. - value, _, found, err := ix.Manifests.Get(ctx, digest) - kept, derr := meshcatalogclient.DecodeManifest(value) - switch { - case err != nil || !found: - out.Answer = "the indexer keeps no manifest of digest " + digest - return out - case derr != nil: - out.Answer = "the manifest of digest " + digest + " cannot be read: " + derr.Error() - return out - case module != "" && kept.Module != module: - out.Answer = "the manifest of digest " + digest + " is " + kept.Module + "'s, not " + module + "'s" - return out - } - out.Module = kept.Module - if _, err := ix.judgeOver(ctx, nil, kept.Module, digest); err != nil { - out.Answer = "the manifest could not be judged: " + err.Error() - return out - } - return ix.answerVerdict(ctx, out, digest) - } - m, ok := held[module] - if !ok || m.Manifest == "" { - out.Answer = "the indexer holds no manifest of " + module - return out - } - if _, err := ix.JudgeAll(ctx, nil); err != nil { - out.Answer = "the manifests could not be judged: " + err.Error() - return out - } - return ix.answerVerdict(ctx, out, m.Manifest) -} - -// answerVerdict is out answering how the verdict under digest stands. -func (ix *Indexer) answerVerdict(ctx context.Context, out TakenIn, digest string) TakenIn { - module := out.Module - value, _, found, err := ix.Verdicts.Get(ctx, digest) - if err != nil || !found { - out.Answer = "judged, but the verdict on " + module + " cannot be read" - return out - } - v, err := meshcatalogclient.DecodeVerdict(value) - if err != nil { - out.Answer = "judged, but the verdict on " + module + " cannot be read: " + err.Error() - return out - } - out.Stored = true - out.Answer = "judged again by " + v.Judge + ": " + v.State - if meshcatalogclient.VerdictHeld(v) { - out.Answer += " (" + meshcatalogclient.VerdictWhy(v) + ")" - } - return out -} - -// NewJudgeAgain is the value indexer_judge writes for a module, and one digest of its manifest when given: its key in -// the takeins state and the value. -func NewJudgeAgain(module, digest, why, by string, at time.Time) (string, []byte, error) { - if digest != "" { - if _, err := meshcatalogclient.ManifestsKey(digest); err != nil { - return "", nil, err - } - } - key, data, err := NewTakeIn(module, why, by, at) - if err != nil { - return "", nil, err - } - var a TakeInAsk - if err := json.Unmarshal(data, &a); err != nil { - return "", nil, err - } - a.Judge, a.Digest = true, digest - data, err = json.Marshal(a) - return judgeAgainPrefix + key, data, err -} - -// withTemplateForms is needs with the reference forms of had added: a module's templates write forms only (catalogue -// ModuleNeeds), and a merge carried them with the manifest the judge reads again alone. -func withTemplateForms(needs, had []string) []string { - out := slices.Clone(needs) - for _, n := range had { - if strings.HasPrefix(n, catalogue.ReadForm) && !slices.Contains(out, n) { - out = append(out, n) - } - } - slices.Sort(out) - return out -} - -// timeOf is the time a pointer holds, the zero time for none. -func timeOf(t *time.Time) time.Time { - if t == nil { - return time.Time{} - } - return *t -} diff --git a/modules/mesh-delivery-indexer/internal/index/verdicts_test.go b/modules/mesh-delivery-indexer/internal/index/verdicts_test.go deleted file mode 100644 index b574378d..00000000 --- a/modules/mesh-delivery-indexer/internal/index/verdicts_test.go +++ /dev/null @@ -1,612 +0,0 @@ -package index - -import ( - "context" - "errors" - "fmt" - "slices" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" - "git.novox.be/novox/mesh-sdk/go/catalogue" - "git.novox.be/novox/mesh-sdk/go/process" - - "mesh-delivery-indexer/judge" -) - -// The indexer judges every manifest it holds and writes each verdict only when it differs; a module's value carries a -// refusal, which is said once as manifest-refused; a new judge writes again only what it judges otherwise; a manifest -// mended clears the module's refusal (novox/hq ADR 0578 §1). -func TestTheIndexerJudgesEveryManifestOnce(t *testing.T) { - ctx := context.Background() - ix := &Indexer{Module: testSelf, Modules: newMem(), Manifests: newMem(), Verdicts: newMem(), JudgeID: "judge-1", - Now: func() time.Time { return time.Date(2026, 11, 1, 3, 0, 0, 0, time.UTC) }} - var said []meshcatalog.ManifestRefusedEvent - ix.SayRefused = func(_ context.Context, r meshcatalog.ManifestRefusedEvent) { said = append(said, r) } - hold := func(module, raw string) string { - digest, err := ix.keepManifest(ctx, module, raw) - if err != nil { - t.Fatal(err) - } - value, err := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: "src2:" + strings.Repeat("a", 64), Manifest: digest, - Repository: "novox/mesh-catalog", Commit: "c"}) - if err != nil { - t.Fatal(err) - } - ix.Modules.(*memStore).put(module, value) - return digest - } - good := hold("good", `{"module":"good","version":"1.0"}`) - bad := hold("bad", `{"module":"bad","version":"1.0","requires":["amqp"]}`) - // Both taken in by a merge: judged in full. - j, err := ix.JudgeAll(ctx, map[string]bool{good: true, bad: true}) - if err != nil || j.Judged != 2 || j.Written != 2 || len(j.Refused) != 1 { - t.Fatalf("the first pass: %+v %v", j, err) - } - verdict := func(digest string) meshcatalog.VerdictsByDigestValue { - value, _, found, _ := ix.Verdicts.Get(ctx, digest) - v, err := meshcatalogclient.DecodeVerdict(value) - if !found || err != nil { - t.Fatalf("no verdict under %s: %v", digest, err) - } - return v - } - if v := verdict(good); meshcatalogclient.VerdictRefuses(v) || v.Judge != "judge-1" { - t.Fatalf("good: %+v", v) - } - if v := verdict(bad); !meshcatalogclient.VerdictRefuses(v) || !strings.Contains(v.Problems[0], `requires "amqp"`) { - t.Fatalf("bad: %+v", v) - } - module := func(name string) meshcatalog.ModulesByModuleValue { - value, _, _, _ := ix.Modules.Get(ctx, name) - m, err := meshcatalogclient.DecodeModule(value) - if err != nil { - t.Fatal(err) - } - return m - } - if m := module("bad"); m.Verdict != meshcatalogclient.VerdictRefused || len(m.Problems) != 1 { - t.Fatalf("modules.bad carries %+v", m) - } - if m := module("good"); m.Verdict != "" { - t.Fatalf("modules.good carries %q", m.Verdict) - } - // ADR 0622 §2: the event carries the verdicts state's revision of the verdict it says. - if _, rev, _, _ := ix.Verdicts.Get(ctx, bad); len(said) != 1 || said[0].Module != "bad" || said[0].Digest != bad || - said[0].Judge != "judge-1" || said[0].Revision == 0 || said[0].Revision != int64(rev) { - t.Fatalf("said %+v (the verdict's revision %d)", said, rev) - } - // A new judge that judges alike writes nothing and says nothing again. - ix.JudgeID = "judge-2" - if j, err := ix.JudgeAll(ctx, nil); err != nil || j.Written != 0 || len(said) != 1 { - t.Fatalf("a judge that judges alike: %+v %v, said %d", j, err, len(said)) - } - // ADR 0603 §4 (D9): a source revision that changes the indexer builds the new indexer, which at its start judges every - // manifest it holds again, none of them fresh: all of them judged, nothing written for a verdict that stands. - if j, err := ix.JudgeAtStart(ctx); err != nil || j.Judged != 2 || j.Written != 0 || len(said) != 1 { - t.Fatalf("the new indexer at its start: %+v %v, said %d, want both manifests judged and nothing written", j, err, len(said)) - } - // The manifest mended: its new digest is accepted and the module's refusal goes. - mended := hold("bad", `{"module":"bad","version":"2.0"}`) - if j, err := ix.JudgeAll(ctx, map[string]bool{mended: true}); err != nil || j.Written != 1 || len(j.Refused) != 0 { - t.Fatalf("mended: %+v %v", j, err) - } - if m := module("bad"); m.Verdict != "" || m.Problems != nil { - t.Fatalf("modules.bad still carries %+v", m) - } - // ADR 0579 §1: a manifest held before any verdict (taken in under the judges until then), which this judge finds a - // problem in, stays accepted with the problem as a warning, carried in its value and said nowhere as a refusal; the - // same problem in a manifest a merge takes in now, or in one whose digest changed, is refused in full. - old := hold("old", `{"module":"old","version":"1.0","requires":["amqp"]}`) - if j, err := ix.JudgeAll(ctx, nil); err != nil || len(j.Outgrown) != 1 || j.Outgrown[0] != "old" { - t.Fatalf("an outgrown manifest: %+v %v", j, err) - } - if v := verdict(old); meshcatalogclient.VerdictRefuses(v) || len(v.Warnings) != 1 || !strings.Contains(v.Warnings[0], `requires "amqp"`) { - t.Fatalf("old: %+v", v) - } - if m := module("old"); m.Verdict != "" || len(m.Warnings) != 1 || len(said) != 1 { - t.Fatalf("modules.old carries %+v, said %d", m, len(said)) - } - changed := hold("old", `{"module":"old","version":"2.0","requires":["amqp"]}`) - fresh := hold("new", `{"module":"new","version":"1.0","requires":["amqp"]}`) - if j, err := ix.JudgeAll(ctx, map[string]bool{changed: true, fresh: true}); err != nil || len(j.Outgrown) != 0 { - t.Fatalf("a changed and a new manifest: %+v %v", j, err) - } - if !meshcatalogclient.VerdictRefuses(verdict(changed)) || !meshcatalogclient.VerdictRefuses(verdict(fresh)) || module("old").Verdict != meshcatalogclient.VerdictRefused || len(module("old").Warnings) != 0 { - t.Fatalf("the changed and the new manifest are not refused: %+v %+v", verdict(changed), verdict(fresh)) - } - // A merge's write of a module whose digest it leaves as it is carries the verdict over, and nothing is fresh; a changed - // digest is fresh, judged in full (review R1, R2). - w := &mergeWrite{} - oldNeeds := module("old").Needs - same := meshcatalog.ModulesByModuleValue{Fp2: "src2:" + strings.Repeat("c", 64), Manifest: changed, Repository: "novox/mesh-catalog", Commit: "c2"} - if err := ix.writeOverlay(ctx, map[string]meshcatalog.ModulesByModuleValue{"old": same}, nil, w); err != nil { - t.Fatal(err) - } - if m := module("old"); m.Verdict != meshcatalogclient.VerdictRefused || len(m.Problems) == 0 || len(w.fresh) != 0 { - t.Fatalf("a merge rewrote a refused module as %+v, fresh %v", m, w.fresh) - } - if got := module("old").Needs; len(oldNeeds) == 0 || !slices.Equal(got, oldNeeds) { - t.Fatalf("a merge leaving the digest dropped the reader capabilities the judge named: %v, held %v", got, oldNeeds) - } - other := strings.Repeat("d", 64) - same.Manifest = other - if err := ix.writeOverlay(ctx, map[string]meshcatalog.ModulesByModuleValue{"old": same}, nil, w); err != nil || !w.fresh[other] { - t.Fatalf("a changed digest is not fresh: %v %v", w.fresh, err) - } - // 1966 (ADR 0668 §4, ADR 0579 §1): what a take-in newly wrote is kept on the bus until judged. A take-in writes a - // manifest a judge refuses in full, its judge pass fails on the verdicts state, and the next pass, which knows nothing - // of that take-in (a start, a pass kicked by a message), still refuses it in full, never as a warning, and ends the key. - ix.Unjudged = newMem() - keptDigest, err := ix.keepManifest(ctx, "kept", `{"module":"kept","version":"1.0","requires":["amqp"]}`) - if err != nil { - t.Fatal(err) - } - kw := &mergeWrite{repository: "novox/mesh-catalog", merge: "m-kept"} - keptValue := meshcatalog.ModulesByModuleValue{Fp2: "src2:" + strings.Repeat("e", 64), Manifest: keptDigest, Repository: "novox/mesh-catalog", Commit: "c3"} - if err := ix.writeOverlay(ctx, map[string]meshcatalog.ModulesByModuleValue{"kept": keptValue}, nil, kw); err != nil || !kw.fresh[keptDigest] { - t.Fatalf("the take-in's write: %v, fresh %v", err, kw.fresh) - } - if value, _, found, _ := ix.Unjudged.Get(ctx, keptDigest); !found || !strings.Contains(string(value), `"module":"kept"`) || !strings.Contains(string(value), "m-kept") { - t.Fatalf("the new manifest is not kept as unjudged: %s", value) - } - ix.Verdicts.(*memStore).failWrite = func(string) error { return process.ErrNotYet } - if _, err := ix.JudgeAll(ctx, kw.fresh); err == nil { - t.Fatal("a pass whose verdicts state is not on the bus did not fail") - } - ix.Verdicts.(*memStore).failWrite = nil - if j, err := ix.JudgeAll(ctx, nil); err != nil || slices.Contains(j.Outgrown, "kept") { - t.Fatalf("the next pass: %+v %v", j, err) - } - if v := verdict(keptDigest); !meshcatalogclient.VerdictRefuses(v) || len(v.Warnings) != 0 { - t.Fatalf("a refusal of a manifest whose judge pass failed became %+v", v) - } - if keys, err := ix.Unjudged.Keys(ctx); err != nil || len(keys) != 0 { - t.Fatalf("the unjudged state still holds %v (%v) after its verdict is written", keys, err) - } - ix.Unjudged = nil - // 1966/AC10: a newly accepted manifest asking for bus state is said once as manifest-accepted, carrying both entries - // whole; a verdict already held says nothing. - var accepted []meshcatalog.ManifestAcceptedEvent - var ids []string - var failing int // publishes still to fail - ix.SayAccepted = func(_ context.Context, a meshcatalog.ManifestAcceptedEvent, id string) error { - if failing > 0 { - failing-- - return errors.New("no stream answered") - } - accepted, ids = append(accepted, a), append(ids, id) - return nil - } - moved := hold("moved", `{"module":"moved","version":"1.0","requires":[`+ - `{"name":"cache","storage":"key-value","template":"state","options":{"history":8}},`+ - `{"name":"audit","storage":"stream","template":"log","options":{"max-mib":64}}]}`) - // The merge's pass, then a later one that knows nothing of the merge. - for _, fresh := range []map[string]bool{{moved: true}, nil} { - if _, err := ix.JudgeAll(ctx, fresh); err != nil { - t.Fatal(err) - } - } - // ADR 0614 §4: the judge names, with the verdict, the reader capabilities the manifest's own words need: the bus-state - // words, which an engine stating nothing (the baseline) lacks; a manifest of the baseline's words lacks none. - if needs := module("moved").Needs; !slices.Contains(needs, "word:bus-state.storage") || - !slices.Contains(catalogue.MissingManifestReads(needs, nil), "word:bus-state") { - t.Fatalf("modules.moved needs %v", needs) - } - if needs := module("good").Needs; !slices.Contains(needs, "word:module") || len(catalogue.MissingManifestReads(needs, nil)) != 0 { - t.Fatalf("modules.good needs %v", needs) - } - if _, rev, _, _ := ix.Verdicts.Get(ctx, moved); len(accepted) != 1 || accepted[0].Revision == 0 || accepted[0].Revision != int64(rev) { - t.Fatalf("said accepted %+v, want the verdict's revision %d (ADR 0622 §2)", accepted, rev) - } - if len(accepted) != 1 || accepted[0].Module != "moved" || accepted[0].Digest != moved || accepted[0].Judge != "judge-2" || - len(accepted[0].BusState) != 2 || accepted[0].BusState[0].Name != "cache" || accepted[0].BusState[1].Template != "log" || - fmt.Sprint(accepted[0].BusState[0].Options["history"]) != "8" { - t.Fatalf("said accepted %+v", accepted) - } - // DLV-I9 (design 25 § 12.2): a manifest asking for none, newly accepted, is said too, its entries an empty list. - plain := hold("plain", `{"module":"plain","version":"1.0"}`) - if _, err := ix.JudgeAll(ctx, map[string]bool{plain: true}); err != nil { - t.Fatal(err) - } - if last := accepted[len(accepted)-1]; last.Module != "plain" || last.Digest != plain || last.BusState == nil || len(last.BusState) != 0 { - t.Fatalf("a manifest asking for no bus state was said as %+v", last) - } - // Review round 1 of issue 1966: a trunk's held manifest judged by its digest (indexer_judge, judgeStale) is said as the - // full pass says it, accepted or refused; a candidate's digest judged over the trunk is said nowhere. - byDigest := hold("bydigest", `{"module":"bydigest","version":"1.0"}`) - refusedBy := hold("refusedby", `{"module":"refusedby","version":"1.0","requires":["amqp"]}`) - saidBefore := len(said) - for module, digest := range map[string]string{"bydigest": byDigest, "refusedby": refusedBy} { - if _, err := ix.judgeOver(ctx, nil, module, digest); err != nil { - t.Fatal(err) - } - } - if last := accepted[len(accepted)-1]; last.Digest != byDigest || len(said) != saidBefore+1 || said[len(said)-1].Digest != refusedBy { - t.Fatalf("judged by its digest, the trunk's manifest was said as %+v, refusals %d (were %d)", last, len(said), saidBefore) - } - candidate, err := ix.keepManifest(ctx, "bydigest", `{"module":"bydigest","version":"2.0"}`) - if err != nil { - t.Fatal(err) - } - if _, err := ix.judgeOver(ctx, nil, "bydigest", candidate); err != nil || accepted[len(accepted)-1].Digest != byDigest { - t.Fatalf("a candidate's verdict was said: %v %+v", err, accepted[len(accepted)-1]) - } - // Issue 1966 (design 25 bus-state §1): the candidate merged, its verdict unchanged, is newly accepted on the trunk and - // said at the merge's pass, from the verdict held and with its revision, though the pass writes no verdict. - if hold("bydigest", `{"module":"bydigest","version":"2.0"}`) != candidate { - t.Fatal("the merged manifest is not the candidate's") - } - if j, err := ix.JudgeAll(ctx, map[string]bool{candidate: true}); err != nil || j.Written != 0 { - t.Fatalf("the merge's pass: %+v %v", j, err) - } - if _, rev, _, _ := ix.Verdicts.Get(ctx, candidate); accepted[len(accepted)-1].Digest != candidate || accepted[len(accepted)-1].Revision != int64(rev) { - t.Fatalf("the merged candidate's manifest was not said accepted with its verdict's revision %d: last said %+v", rev, accepted[len(accepted)-1]) - } - // DLV-I9b: a failed publish is published again; one failing twice writes the finding naming the module, which the next - // judging pass says again and clears. - ix.Findings = newFindings() - failing = 1 - once := hold("once", `{"module":"once","version":"1.0"}`) - if _, err := ix.JudgeAll(ctx, map[string]bool{once: true}); err != nil || accepted[len(accepted)-1].Module != "once" { - t.Fatalf("a publish failing once was not published again: %v %+v", err, accepted[len(accepted)-1]) - } - failing = 2 - flaky := hold("flaky", `{"module":"flaky","version":"1.0"}`) - if _, err := ix.JudgeAll(ctx, map[string]bool{flaky: true}); err != nil { - t.Fatal(err) - } - if value, found := ix.Findings.(*memFindings).vals[unsaidKey("flaky")]; !found || !strings.Contains(string(value), "flaky") { - t.Fatalf("a publish failing twice wrote no finding naming flaky: %s", value) - } - if _, err := ix.JudgeAll(ctx, nil); err != nil { - t.Fatal(err) - } - if _, found := ix.Findings.(*memFindings).vals[unsaidKey("flaky")]; found || accepted[len(accepted)-1].Module != "flaky" || - accepted[len(accepted)-1].Digest != flaky { - t.Fatalf("the next pass did not say flaky and clear its finding (found %v, last said %+v)", found, accepted[len(accepted)-1]) - } - // DLV-I9c: indexer_say_accepted says it again from the held verdict, under an id of its own; a module with no accepted - // verdict, or none held, is refused. - if r := ix.SayAcceptedAgain(ctx, "flaky"); !r.Stored || accepted[len(accepted)-1].Module != "flaky" || !strings.Contains(ids[len(ids)-1], "-again-") { - t.Fatalf("said again: %+v, id %s", r, ids[len(ids)-1]) - } - for _, m := range []string{"old", "nobody"} { - if r := ix.SayAcceptedAgain(ctx, m); r.Stored { - t.Errorf("%s said again: %+v", m, r) - } - } - if key, value, err := NewSayAccepted("flaky", "the bus module missed it", "jochen@shanks", time.Now()); err != nil || - !strings.HasPrefix(key, sayAcceptedPrefix) || !strings.Contains(string(value), `"say":true`) { - t.Errorf("the ask = %s %s (%v)", key, value, err) - } - if _, _, err := NewSayAccepted("flaky", "", "jochen@shanks", time.Now()); err == nil { - t.Error("an ask saying no why was taken") - } - // DLV-I15: a change's renamed-from is judged against the trunk's manifest of the module: a name the trunk declares is - // accepted, one it does not is refused naming the key. - trunkRenamer := hold("renamer", `{"module":"renamer","version":"1.0","config":{"font-name":{"type":"string","description":"The font.","required":true,"runtime":true}}}`) - if _, err := ix.JudgeAll(ctx, map[string]bool{trunkRenamer: true}); err != nil { - t.Fatal(err) - } - for raw, want := range map[string]string{ - `{"module":"renamer","version":"1.0","config":{"font":{"type":"string","description":"The font.","required":true,"runtime":true,"renamed-from":"font-name"}}}`: "", - `{"module":"renamer","version":"1.0","config":{"font":{"type":"string","description":"The font.","required":true,"runtime":true,"renamed-from":"typeface"}}}`: `renamed from "typeface"`, - } { - head, err := ix.keepManifest(ctx, "renamer", raw) - if err != nil { - t.Fatal(err) - } - ix.JudgeAgain(ctx, "renamer", head) - if v := verdict(head); (want == "") != !meshcatalogclient.VerdictRefuses(v) || (want != "" && !strings.Contains(strings.Join(v.Problems, " "), want)) { - t.Errorf("the change %s: %+v, want refused naming %q", raw, v, want) - } - } - // DLV-I14 and the lead's ruling (issue 1966): a pull request's candidate is not refused for naming a package the - // packages state does not know (the checker judges its names, knowing its delivery group); the same manifest a merge - // takes in is, though its candidate was accepted; a name the state knows is accepted at both. - ix.Packages = newMem() - sdkGo := "git.novox.be/novox/mesh-sdk/go" - if value, err := EncodeSeriesPackage(SeriesPackage{V: 1, Name: sdkGo, Package: "mesh-sdk/go", Series: "0.1"}); err != nil { - t.Fatal(err) - } else { - ix.Packages.(*memStore).put(SeriesKey(sdkGo, "0.1"), value) - } - for raw, refusedAtMerge := range map[string]bool{ - `{"module":"pkguser","version":"1.0","packages":{"@novox/ui":"0.1"}}`: true, - `{"module":"pkguser","version":"2.0","packages":{"` + sdkGo + `":"0.1"}}`: false, - } { - head, err := ix.keepManifest(ctx, "pkguser", raw) - if err != nil { - t.Fatal(err) - } - ix.judgeDigests(ctx, map[string]string{"pkguser": head}, "novox/mesh-catalog#1") - if v := verdict(head); meshcatalogclient.VerdictRefuses(v) { - t.Errorf("the candidate %s is refused: %+v", raw, v) - } - merged := hold("pkguser", raw) - if _, err := ix.JudgeAll(ctx, map[string]bool{merged: true}); err != nil { - t.Fatal(err) - } - if v := verdict(merged); meshcatalogclient.VerdictRefuses(v) != refusedAtMerge || refusedAtMerge && !strings.Contains(strings.Join(v.Problems, " "), `name "@novox/ui", which no package`) { - t.Errorf("the merge of %s: %+v, want refused %v for the name", raw, v, refusedAtMerge) - } - } - ix.Packages = nil - // 1966/AC11: modules using state and none using logs but the foundation: logs counts 0, so a manifest of another - // module using logs is refused and the foundation's is accepted. - ix.OldSections = newMem() - hold("user", `{"module":"user","version":"1.0","state":["x"]}`) - hold("mesh-grants", `{"module":"mesh-grants","version":"1.0","logs":[{"name":"l"}]}`) - if _, err := ix.JudgeAll(ctx, nil); err != nil { - t.Fatal(err) - } - counts := func() map[string]int { - value, _, found, err := ix.OldSections.Get(ctx, OldSectionsKey) - if err != nil || !found { - t.Fatalf("no count of the old sections: %v", err) - } - c, err := DecodeOldSections(value) - if err != nil { - t.Fatal(err) - } - return c.Counts - } - if c := counts(); c["state"] != 1 || c["logs"] != 0 || c["streams"] != 0 || c["objects"] != 0 { - t.Fatalf("the old sections count %v, want state 1 and the rest 0", c) - } - // DLV-J2, DLV-J3 (ADR 0615 §1): one reads entry on the assigner's state counts 1 for the assigner and 0 for the indexer. - hold("reader", `{"module":"reader","version":"1.0","state":["x"],"reads":["mesh-delivery-assigner.wants"]}`) - if _, err := ix.JudgeAll(ctx, nil); err != nil { - t.Fatal(err) - } - if value, _, _, _ := ix.OldSections.Get(ctx, OldSectionsKey); true { - c, _ := DecodeOldSections(value) - if c.Reads["mesh-delivery-assigner"] != 1 || c.Reads["mesh-delivery-assigner.wants"] != 1 || c.Reads["mesh-delivery-indexer"] != 0 { - t.Fatalf("the reads on the owners' state count %v", c.Reads) - } - } - // DLV-J1 (ADR 0619 §1, §3): a foundation part asking bus state in the new form is refused; the nine and the pipeline runner are the foundation. - // Design 47 *A module's registration* (b229f5c4 lines 480-481): a template in the manifest of every part on the list is - // refused, the runner's included. - for _, part := range []string{"mesh-config", "mesh-delivery-pipeline"} { - d := hold(part, `{"module":"`+part+`","version":"1.0","requires":[{"name":"cache","storage":"key-value","template":"state"}]}`) - if _, err := ix.JudgeAll(ctx, map[string]bool{d: true}); err != nil { - t.Fatal(err) - } - if v := verdict(d); !meshcatalogclient.VerdictRefuses(v) || !strings.Contains(strings.Join(v.Problems, " "), "part of the foundation") { - t.Fatalf("%s asking new-form bus state: %+v", part, v) - } - } - if !slices.Equal(judge.Foundation, []string{"nats", "mesh-grants", "mesh-delivery-indexer", "mesh-delivery-assigner", "node-tools", - "mesh-delivery-builder", "mesh-delivery-checker", "mesh-config", "mesh-vault", "mesh-delivery-pipeline", "gitea", "distribution", "mesh-host"}) { - t.Fatalf("the foundation is %v, want exactly the nine of ADR 0619, the pipeline runner (ADR 0666 §4) and the forge and image registry modules (design 47 G §584) and the node-engine's module (§627)", judge.Foundation) - } - late := hold("late", `{"module":"late","version":"1.0","logs":[{"name":"l"}]}`) - grants := hold("mesh-grants", `{"module":"mesh-grants","version":"2.0","logs":[{"name":"l"},{"name":"m"}]}`) - if _, err := ix.JudgeAll(ctx, map[string]bool{late: true, grants: true}); err != nil { - t.Fatal(err) - } - if v := verdict(late); !meshcatalogclient.VerdictRefuses(v) || !strings.Contains(v.Problems[0], "late uses the old section logs, which no other module uses any more") { - t.Fatalf("late: %+v", v) - } - if v := verdict(grants); meshcatalogclient.VerdictRefuses(v) { - t.Fatalf("the foundation's manifest was refused: %+v", v) - } - // 1966/AC11: the sole user of an old section changes its manifest and keeps the section: accepted, since it was accepted - // with it (the count is the accepted set held from the last pass, not recounted from fresh digests). - hold("solo", `{"module":"solo","version":"1.0","objects":[{"name":"o","max-mib":1}]}`) - if _, err := ix.JudgeAll(ctx, nil); err != nil { - t.Fatal(err) - } - solo2 := hold("solo", `{"module":"solo","version":"2.0","objects":[{"name":"o","max-mib":2}]}`) - if _, err := ix.JudgeAll(ctx, map[string]bool{solo2: true}); err != nil { - t.Fatal(err) - } - if v := verdict(solo2); meshcatalogclient.VerdictRefuses(v) { - t.Fatalf("the sole user of objects, changing its manifest and keeping objects, was refused: %+v", v) - } - // A count that cannot be read is never taken as 0: the zero rule is not applied in that pass and the log says so. - var logged []string - ix.Logf = func(f string, a ...any) { logged = append(logged, fmt.Sprintf(f, a...)) } - held := ix.OldSections - ix.OldSections = unreadable{held} - retry := hold("late", `{"module":"late","version":"2.0","logs":[{"name":"l"}]}`) - if _, err := ix.JudgeAll(ctx, map[string]bool{retry: true}); err != nil { - t.Fatal(err) - } - if v := verdict(retry); meshcatalogclient.VerdictRefuses(v) || !strings.Contains(strings.Join(logged, "\n"), "the zero rule is not applied") { - t.Fatalf("a count that could not be read: %+v, logged %q", v, logged) - } - ix.OldSections = held - // Clause 3 of the record on a release's Approve: a manifest with a field this judge does not know waits (no refusal, no - // event) while a newer build of the indexer is on the trunk, and is judged again at the next judge; once this program's - // build is the trunk's newest it is refused; a wait is written into the module's value as a refusal naming the fields. - ownFP := "src2:" + strings.Repeat("e", 64) - hold(testSelf, `{"module":"mesh-delivery-indexer","version":"1.0"}`) - ix.ownFP = ownFP // this program started when the trunk held another build of the indexer than the one it holds now - later := hold("later", `{"module":"later","version":"1.0","a-field-from-later":1,"another":{"x":1}}`) - said = nil - before := len(said) - j, err = ix.JudgeAll(ctx, map[string]bool{later: true}) - if err != nil || len(j.Waiting) != 1 || j.Waiting[0] != "later" || len(said) != before { - t.Fatalf("a manifest with fields from later: %+v %v, said %d", j, err, len(said)) - } - for _, a := range accepted { - if a.Module == "later" { - t.Fatalf("a manifest set aside to wait was said accepted: %+v (DLV-I9)", a) - } - } - if v := verdict(later); !meshcatalogclient.VerdictWaits(v) || meshcatalogclient.VerdictRefuses(v) || strings.Join(v.Unknown, ",") != "a-field-from-later,another" { - t.Fatalf("later: %+v", v) - } - if m := module("later"); m.Verdict != meshcatalogclient.VerdictRefused || len(m.Unknown) != 2 || !meshcatalogclient.VerdictWaits(*meshcatalogclient.ModuleVerdict("later", m)) || - !strings.Contains(m.Problems[0], "does not know yet") { - t.Fatalf("modules.later carries %+v", m) - } - // The build this program runs is asked of the engine, and the trunk moving before the start pass changes nothing: the - // trunk's value of the indexer names a newer build than the one the engine says runs, so the field still waits. - ix.ownFP = "" - ix.OwnBuild = func(context.Context) (string, error) { return ownFP, nil } - if _, err := ix.JudgeAtStart(ctx); err != nil || !meshcatalogclient.VerdictWaits(verdict(later)) { - t.Fatalf("the trunk moved before the start pass: %v %+v", err, verdict(later)) - } - ix.OwnBuild = func(context.Context) (string, error) { return "", errors.New("the engine does not answer") } - ix.ownFP = "" - if _, err := ix.JudgeAtStart(ctx); err != nil || !meshcatalogclient.VerdictWaits(verdict(later)) { - t.Fatalf("an unknown build refused: %v %+v", err, verdict(later)) - } - ix.ownFP = ownFP - // A renamed field is a known mistake: refused at once. - renamed := hold("renamed", `{"module":"renamed","version":"1.0","needs":{"x":"y"}}`) - if j, err := ix.JudgeAll(ctx, map[string]bool{renamed: true}); err != nil || len(j.Waiting) != 1 || !meshcatalogclient.VerdictRefuses(verdict(renamed)) { - t.Fatalf("a renamed field: %+v %v", j, err) - } - // The trunk's newest build of the indexer is this program's: the field is one no judge to come knows. - ix.ownFP = "src2:" + strings.Repeat("a", 64) - if j, err := ix.JudgeAll(ctx, nil); err != nil || len(j.Waiting) != 0 || !meshcatalogclient.VerdictRefuses(verdict(later)) || !strings.Contains(verdict(later).Problems[0], "a-field-from-later") { - t.Fatalf("the newest build refuses: %+v %v %+v", j, err, verdict(later)) - } - if m := module("later"); m.Verdict != meshcatalogclient.VerdictRefused || meshcatalogclient.VerdictWaits(*meshcatalogclient.ModuleVerdict("later", m)) || len(m.Unknown) != 0 { - t.Fatalf("modules.later after the refusal carries %+v", m) - } - // The tools: indexer_verdicts lists the verdicts that wait or are refused (and every one with all), indexer_judge - // judges now and says how the module's verdict stands. - keys, _ := ix.Verdicts.Keys(ctx) - get := func(key string) ([]byte, error) { v, _, _, err := ix.Verdicts.Get(ctx, key); return v, err } - if rows, unread := ListVerdicts(keys, get, "later", false); len(rows) != 1 || rows[0].State != meshcatalogclient.VerdictRefused || len(unread) != 0 { - t.Fatalf("indexer_verdicts of later: %+v %v", rows, unread) - } - if rows, _ := ListVerdicts(keys, get, "", true); len(rows) <= len(keys)-3 || rows[0].Module > rows[len(rows)-1].Module { - t.Fatalf("indexer_verdicts, all: %d rows of %d verdicts", len(rows), len(keys)) - } - // The build is fixed once, at the start, and never learnt later: this program started when the engine could not answer, and - // the engine now names the trunk's newest build (as it does while it replaces the indexer by a successor). indexer_judge - // does not take it for this program's own, so the field still waits and the build stays unknown. - ix.ownFP, ix.ownFixed = "", true - ix.OwnBuild = func(context.Context) (string, error) { return "src2:" + strings.Repeat("a", 64), nil } - if got := ix.JudgeAgain(ctx, "later", ""); !got.Stored || !strings.Contains(got.Answer, "waits") || ix.ownFP != "" { - t.Fatalf("indexer_judge learnt the build the engine names now: %+v (own %q)", got, ix.ownFP) - } - if _, err := ix.JudgeAtStart(ctx); err != nil || ix.ownFP != "" || !meshcatalogclient.VerdictWaits(verdict(later)) { - t.Fatalf("a judge pass learnt the build: %v %q %+v", err, ix.ownFP, verdict(later)) - } - ix.ownFP = ownFP - ix.OwnBuild = nil - if got := ix.JudgeAgain(ctx, "later", ""); !got.Stored || !strings.Contains(got.Answer, "waits") || !strings.Contains(got.Answer, "a-field-from-later") { - t.Fatalf("indexer_judge: %+v", got) - } - if rows, _ := ListVerdicts(keys, get, "later", false); len(rows) != 1 { - t.Fatalf("indexer_verdicts of later after: %+v", rows) - } - if got := ix.JudgeAgain(ctx, "nobody", ""); got.Stored || !strings.Contains(got.Answer, "holds no manifest") { - t.Fatalf("indexer_judge of nobody: %+v", got) - } - if key, value, err := NewJudgeAgain("later", "", "why", "me", time.Now()); err != nil || key != "judge.later" { - t.Fatalf("the ask: %q %v", key, err) - } else if a, err := DecodeTakeIn(value); err != nil || !a.Judge || a.Module != "later" { - t.Fatalf("the ask's value: %+v %v", a, err) - } - // A refusal an older judge made never stands: a digest the trunk no longer names (a candidate's, or one back on the - // trunk later) refused by judge-0 for words it did not know is judged again by this judge at the start, at a pull that - // carries it, and by indexer_judge with its digest; one this judge refused itself is left as it is. - stale := func(module, raw string) string { - digest, err := ix.keepManifest(ctx, module, raw) - if err != nil { - t.Fatal(err) - } - if _, _, err := ix.writeVerdict(ctx, digest, meshcatalog.VerdictsByDigestValue{Module: module, State: meshcatalogclient.VerdictRefused, - Problems: []string{module + " says tool-runner, which this judge does not know"}, Judge: "judge-0", At: ptr(time.Now())}); err != nil { - t.Fatal(err) - } - return digest - } - atStart := stale("tools", `{"module":"tools","version":"7.0"}`) - ownRefusal := stale("client", `{"module":"client","version":"7.0"}`) - if _, _, err := ix.writeVerdict(ctx, ownRefusal, meshcatalog.VerdictsByDigestValue{Module: "client", State: meshcatalogclient.VerdictRefused, - Problems: []string{"x"}, Judge: ix.judgeID(), At: ptr(time.Now())}); err != nil { - t.Fatal(err) - } - if _, err := ix.JudgeAtStart(ctx); err != nil || meshcatalogclient.VerdictRefuses(verdict(atStart)) || verdict(atStart).Judge != ix.judgeID() || !meshcatalogclient.VerdictRefuses(verdict(ownRefusal)) { - t.Fatalf("the start pass: %+v, this judge's own refusal %+v (%v)", verdict(atStart), verdict(ownRefusal), err) - } - byPull := stale("tools", `{"module":"tools","version":"8.0"}`) - pull := forgemsg.Forge{Kind: forgemsg.ForgePull, Repository: "novox/mesh-tools", Number: 3, Branch: "main", - Manifests: map[string]string{"modules/tools": `{"module":"tools","version":"8.0"}`}} - if _, err := ix.ApplyMessage(ctx, pull); err != nil || meshcatalogclient.VerdictRefuses(verdict(byPull)) || verdict(byPull).Judge != ix.judgeID() { - t.Fatalf("a pull carrying a digest an older judge refused: %+v %v", verdict(byPull), err) - } - byTool := stale("tools", `{"module":"tools","version":"9.0"}`) - if got := ix.JudgeAgain(ctx, "client", byTool); got.Stored || !strings.Contains(got.Answer, "tools's, not client's") || !meshcatalogclient.VerdictRefuses(verdict(byTool)) { - t.Fatalf("indexer_judge with another module's digest: %+v", got) - } - if got := ix.JudgeAgain(ctx, "tools", byTool); !got.Stored || !strings.Contains(got.Answer, "accepted") || meshcatalogclient.VerdictRefuses(verdict(byTool)) { - t.Fatalf("indexer_judge with a digest: %+v %+v", got, verdict(byTool)) - } - if key, value, err := NewJudgeAgain("tools", byTool, "why", "me", time.Now()); err != nil || key != "judge.tools" { - t.Fatalf("the ask with a digest: %q %v", key, err) - } else if a, err := DecodeTakeIn(value); err != nil || a.Digest != byTool { - t.Fatalf("the ask's digest: %+v %v", a, err) - } - // Without the verdicts state the indexer judges nothing. - ix.Verdicts = nil - if j, err := ix.JudgeAll(ctx, nil); err != nil || j.Judged != 0 { - t.Fatalf("without the state: %+v %v", j, err) - } -} - -// unreadable is a store whose reads fail. -type unreadable struct{ Store } - -func (unreadable) Get(context.Context, string) ([]byte, uint64, bool, error) { - return nil, 0, false, errors.New("the bus did not answer") -} - -// Design 47 G §491, §676: two modules whose entries carry one Go module path are both refused by the trunk's judge pass, -// each naming both; a module with its own name is accepted. -func TestAPackageNameBelongsToOneModule(t *testing.T) { - ctx := context.Background() - ix := &Indexer{Module: testSelf, Modules: newMem(), Manifests: newMem(), Verdicts: newMem(), JudgeID: "judge-1", - Now: func() time.Time { return time.Date(2026, 11, 1, 3, 0, 0, 0, time.UTC) }} - ix.ReadFile = func(_ context.Context, _, _, path string) ([]byte, error) { - if strings.HasSuffix(path, "go.mod") { - name := "git.example/own/" + strings.Split(path, "/")[0] - if strings.HasPrefix(path, "one/") || strings.HasPrefix(path, "two/") { - name = "git.example/shared" - } - return []byte("module " + name + "\n"), nil - } - return nil, nil - } - hold := func(module string) string { - raw := `{"module":"` + module + `","version":"1.0","build":{"artifacts":[{"name":"go","kind":"package","language":"go","paths":["go"],"series":"0.1","generated-from":"."}]}}` - digest, err := ix.keepManifest(ctx, module, raw) - if err != nil { - t.Fatal(err) - } - value, err := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: "src2:" + strings.Repeat("a", 64), Manifest: digest, - Repository: "novox/r", Path: module, Commit: "c"}) - if err != nil { - t.Fatal(err) - } - ix.Modules.(*memStore).put(module, value) - return digest - } - digests := map[string]bool{hold("one"): true, hold("two"): true, hold("three"): true} - j, err := ix.JudgeAll(ctx, digests) - if err != nil { - t.Fatal(err) - } - if len(j.Refused) != 2 || j.Refused["one"] == nil || j.Refused["two"] == nil || j.Refused["three"] != nil { - t.Fatalf("refused %v, want one and two", j.Refused) - } - for _, m := range []string{"one", "two"} { - if p := strings.Join(j.Refused[m], " "); !strings.Contains(p, "one and two") || !strings.Contains(p, "git.example/shared") { - t.Errorf("%s's problems %q do not name both modules and the package", m, p) - } - } -} diff --git a/modules/mesh-delivery-indexer/internal/index/views.go b/modules/mesh-delivery-indexer/internal/index/views.go index ce197c5e..cddae2d9 100644 --- a/modules/mesh-delivery-indexer/internal/index/views.go +++ b/modules/mesh-delivery-indexer/internal/index/views.go @@ -17,7 +17,6 @@ import ( var FirstViewTools = []ViewedTool{ {Tool: "manifests", State: "manifests", Words: "every manifest the indexer keeps, by its digest."}, {Tool: "modules", State: ModulesState, Words: "every module the indexer holds, its fingerprint, inputs and manifest digest."}, - {Tool: "verdicts", State: VerdictsState, Words: "the judge's verdict on each manifest: accepted, waiting or refused."}, {Tool: "graph", State: graph.State, Words: "the module graph: each module's build, provision and verb edges, whose verb edges a want waits on."}, } @@ -33,7 +32,6 @@ var ReadTools = []ViewedTool{ {Tool: "indexer_read_takeins", State: TakeInsState, Words: "each take-in asked of the indexer: its ask, whether and when the indexer answered it and its answer (design 47 G A-464a)."}, {Tool: "indexer_read_findings", State: FindingsState, Words: "the indexer's findings, which the condition keeper raises and clears."}, {Tool: "indexer_read_seat_definitions", State: SeatDefinitionsState, Words: "the seat definitions the trunk's and each open pull request's contracts modules hold, resolved, or why one does not parse."}, - {Tool: "indexer_read_unjudged", State: UnjudgedState, Words: "the manifests a take-in newly wrote that no judge pass judged yet, by digest, each with its module and merge; indexer_judge judges them."}, } // ViewedTool is one tool over one state of the indexer's. @@ -43,8 +41,8 @@ type ViewedTool struct { // ViewedStates are the states the indexer's views family says: the ones its readers read. var ViewedStates = map[string]bool{ - ManifestsState: true, VerdictsState: true, PredefinedState: true, CandidatesState: true, - PackagesState: true, HeardState: true, TakenState: true, TakeInsState: true, FindingsState: true, SeatDefinitionsState: true, UnjudgedState: true, + ManifestsState: true, PredefinedState: true, CandidatesState: true, + PackagesState: true, HeardState: true, TakenState: true, TakeInsState: true, FindingsState: true, SeatDefinitionsState: true, } // ModulesViewsEvent is what the indexer says on its views family of modules when a new revision of the trunk branch diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index cbaa0b40..b2e567d6 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -115,11 +115,16 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build if m.Module != "mesh-delivery-indexer" { t.Errorf("module = %q", m.Module) } - // 1126: assignments is the seat mesh-module-assigner's (ADR 0567): the indexer holds no assignments state. ADR 0578 §1: it says - // a refused manifest as manifest-refused, and (issue 1966 D3, 1966/AC10) a newly accepted one asking for bus state as - // manifest-accepted, both events of the seat mesh-catalog it holds (ADR 0636 §4), so not among its `emits`. + // 1126: assignments is the seat mesh-module-assigner's (ADR 0567): the indexer holds no assignments state. I7/AC1 (design 47 + // H §745, §764): it judges no manifest at the take-in, so it keeps no verdicts and says no manifest-refused or + // manifest-accepted; its own events are the take-in's two. if own := slices.DeleteFunc(slices.Clone(m.Emits), func(e string) bool { return strings.HasPrefix(e, views.EventPrefix) }); !reflect.DeepEqual(own, []string{takein.TakeInPendingEvent, takein.TakeInTakenEvent}) { - t.Errorf("emits = %v, want the two take-in events (the verdict events are the seat mesh-catalog's)", m.Emits) + t.Errorf("emits = %v, want the two take-in events", m.Emits) + } + for _, gone := range []string{"verdicts", "unjudged", "indexer_verdicts", "indexer_judge", "indexer_say_accepted", "indexer_read_unjudged", "views.verdicts"} { + if slices.Contains(m.Tools, gone) || slices.Contains(m.Emits, gone) || slices.ContainsFunc(m.State, func(s state) bool { return s.Name == gone }) { + t.Errorf("the manifest still declares %s (design 47 H §764: the validation at the take-in goes)", gone) + } } // Design 47 G §700, A-74: the router's record of an ask is read through the seat operator-channel, which the indexer uses, // and its holder is asked of the seat mesh-module-assigner. @@ -127,10 +132,10 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build t.Errorf("uses = %v, invokes = %v, want the seat operator-channel used and seat:mesh-module-assigner.assignments invoked", m.Uses, m.Invokes) } // 1126 note 76: heard keeps the last 64 FORGE messages the indexer handled, for indexer_heard. - if want := []state{{Name: "candidates", History: 10}, {Name: "heard", History: 64}, {Name: "manifests", History: 1}, {Name: "predefined", History: 5}, {Name: "taken", History: 1}, {Name: "takeins", History: 1}, {Name: "verdicts", History: 5}, {Name: "oldsections", History: 1}, {Name: index.RunsState, History: 1}, {Name: "own-build", History: 1}, {Name: index.PackagesState, History: index.PackagesHistory}, + if want := []state{{Name: "candidates", History: 10}, {Name: "heard", History: 64}, {Name: "manifests", History: 1}, {Name: "predefined", History: 5}, {Name: "taken", History: 1}, {Name: "takeins", History: 1}, {Name: "oldsections", History: 1}, {Name: index.RunsState, History: 1}, {Name: "own-build", History: 1}, {Name: index.PackagesState, History: index.PackagesHistory}, {Name: views.MarksState, History: 1}, {Name: index.FindingsState, History: 5}, {Name: views.StatusState, History: 1}, {Name: index.RequirementsState, History: 5}, {Name: index.SeatDefinitionsState, History: 5}, - {Name: index.PendingState, History: 1}, {Name: index.UnjudgedState, History: 1}, {Name: index.EntryIndexState, History: 1, MaxMiB: 8}, {Name: index.NamedState, History: 1, MaxMiB: 4}, // design 47 G §695 + {Name: index.PendingState, History: 1}, {Name: index.EntryIndexState, History: 1, MaxMiB: 8}, {Name: index.NamedState, History: 1, MaxMiB: 4}, // design 47 G §695 {Name: index.RegistrationsState, History: 1}, {Name: index.ProposalsState, History: 1, MaxMiB: 4}, {Name: index.VersionsState, History: 1, MaxMiB: 64}, {Name: index.BuildEdgesState, History: 1}, {Name: index.InputsState, History: 1}, {Name: index.PinsState, History: 1, MaxMiB: index.PinsCapMiB}, {Name: process.ConsumersState, History: 1}}; !reflect.DeepEqual(m.State, want) { // DLV-I7, DLV-I17 t.Errorf("state = %+v, want %+v", m.State, want) @@ -146,8 +151,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build // name and states with their revisions, and the states it reads, each as `.`. // A-296: its own buckets are named from its credential's module (index.Bucket), each a state its manifest keeps. own := func(state string) string { return index.Bucket("a-indexer", state) } - for bucket, history := range map[string]int{own(index.CandidatesState): index.CandidatesHistory, own(index.ManifestsState): index.ManifestsHistory, - own(index.VerdictsState): index.VerdictsHistory, own(index.PackagesState): index.PackagesHistory, own(index.TakenState): 1, own(index.PredefinedState): 5} { + for bucket, history := range map[string]int{own(index.CandidatesState): index.CandidatesHistory, own(index.ManifestsState): index.ManifestsHistory, own(index.PackagesState): index.PackagesHistory, own(index.TakenState): 1, own(index.PredefinedState): 5} { if !slices.ContainsFunc(m.State, func(st state) bool { return "a-indexer_"+st.Name == bucket && st.History == history }) { t.Errorf("the indexer names its bucket %s with %d revisions, its manifest's states are %+v", bucket, history, m.State) } @@ -166,7 +170,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build // internal/broker/state.go:62-93). // The registry seats' published fill each (§413: no consume of version.numbered) package's series (DLV-I5, design 51 *The versions*). if want := []string{"go-package-registry.published", "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "mesh-builds.package-published", "mesh-builds.toolchain-agreed", "mesh-catalog.catalog-revision", "mesh-conditions.findings-asked", "mesh-delivery-checker.verdict", "mesh-delivery-forge-bridge.forge.**", - "mesh-delivery.freed", "mesh-delivery.heads-replaced", "mesh-delivery.pinned", "mesh-delivery.pins-stated", "mesh-delivery.source-moved", "mesh-pipeline-runner.pipeline-ended", "mesh-pipeline-runner.run-started", "mesh-wants.version-delivered", "mesh-wants.wait-ended", "mesh-wants.wanted", "node-build-go.step-ended", "node-engine.running", "npm-package-registry.published", "operator-channel.decided", "seat:mesh-module-assigner.views.assignments", "seat:mesh-module-assigner.views.seat-holders"}; !reflect.DeepEqual(sorted(m.Consumes), want) { + "mesh-delivery.source-moved", "mesh-pipeline-runner.pipeline-ended", "mesh-pipeline-runner.pipeline-started", "mesh-pipeline-runner.run-started", "mesh-wants.version-delivered", "mesh-wants.wait-ended", "mesh-wants.wanted", "node-build-go.step-ended", "node-engine.running", "npm-package-registry.published", "operator-channel.decided", "seat:mesh-module-assigner.views.assignments", "seat:mesh-module-assigner.views.seat-holders"}; !reflect.DeepEqual(sorted(m.Consumes), want) { t.Errorf("consumes = %v, want %v", m.Consumes, want) } if b, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 || b != "${dir:state}/broker" { @@ -182,13 +186,11 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build "seat:node-build-node.steps", "seat:node-build-image.steps", "mesh-delivery-builder.builder_versions"}; !reflect.DeepEqual(m.Invokes, want) { t.Errorf("invokes = %v, want %v", m.Invokes, want) } - // 1945: indexer_take_in, served by its tools bundle through the takeins state; the process serves none. The same bundle - // reads the verdicts state (indexer_verdicts) and asks the indexer to judge again through the takeins state - // (indexer_judge; issue 774, clause 3). 1966/AC11: indexer_old_sections reads the count of the old sections (state oldsections). - // ADR 0621 §1, ADR 0622 §1, ADR 0631 §1, ADR 0663 §1: the first-view tools manifests, modules, verdicts and graph (modules + // 1945: indexer_take_in, served by its tools bundle through the takeins state; the process serves none. 1966/AC11: indexer_old_sections reads the count of the old sections (state oldsections). + // ADR 0621 §1, ADR 0622 §1, ADR 0631 §1, ADR 0663 §1: the first-view tools manifests, modules and graph (modules // kept for the grants part until it reads the catalog half, design 47 G A-354), and design 25 § 12.2's read // tools, all of the tools bundle, which runs in a runtime of its own on the indexer's account (runs-as). - want := []string{"indexer_take_in", "indexer_old_sections", "indexer_verdicts", "indexer_judge"} + want := []string{"indexer_take_in", "indexer_old_sections"} for _, v := range append(append([]index.ViewedTool{}, index.FirstViewTools...), index.ReadTools...) { want = append(want, v.Tool) // Milestone 47/G: modules and graph are the newest catalog graph revision's, which catalog-revision says the changes of. @@ -196,14 +198,14 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build t.Errorf("%s answers the state %s, which the views family does not say", v.Tool, v.State) } } - want = append(want, "indexer_first_views", "indexer_first_view_again", index.ConfigReadersTool, index.BusStateTool, index.RequirementsTool, index.PackageUsersTool, index.SayAcceptedTool, // DLV-F3r, DLV-I10, DLV-I11, DLV-I7, DLV-I8, DLV-I9c + want = append(want, "indexer_first_views", "indexer_first_view_again", index.ConfigReadersTool, index.BusStateTool, index.RequirementsTool, index.PackageUsersTool, // DLV-F3r, DLV-I10, DLV-I11, DLV-I7, DLV-I8, DLV-I9c index.SeatDefinitionTool, index.SeatDefinitionsAgainTool, index.OutOfStepTool, // DLV-I19 index.PendingTool, index.TakeInPendingTool, index.DropPendingTool, index.TakeInMergeTool, index.FingerprintsAgainTool, // issue 1966; 1585/T12, T13 (ADR 0633 §5) // milestone 47/G W4.12 (design 47, the indexer's tools): the catalog graph revisions read and steered. index.CatalogRevisionTool, index.CatalogRevisionsTool, index.CandidateGraphsTool, index.RecomputeTool, index.DropCandidateGraphTool, index.EntryIndexTool, index.NamedDropTool, // design 47 G §675 index.ProposalsTool, index.ProposalDropTool, index.SeedAgainTool, index.InputsTool, index.PinsTool, index.UnpinTool, index.ConsumerTool, index.RunsTool, index.RunDropTool) // design 47 G §667, §8, §579, §64 (A-194) - if !reflect.DeepEqual(m.Tools, want) || len(want) != 50 || !slices.Contains(m.Tools, "modules") { + if !reflect.DeepEqual(m.Tools, want) || len(want) != 45 || !slices.Contains(m.Tools, "modules") { t.Errorf("tools = %v, want %v", m.Tools, want) } // Design 47 G §2, §9: every version of every registration is kept with no age limit, 64 MiB, on the old streams section. @@ -284,7 +286,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build // ADR 0636 §4: seat-definition-changed and -removed are the seat mesh-catalog's events, said on the seat's subject and kept // on the shared events stream, and not among its own emits. // Design 47 G A-360: modules too, at each new trunk revision that changes it. - for _, state := range []string{"candidates", "findings", "graph", "heard", "manifests", "modules", "packages", "predefined", "seat-definitions", "taken", "verdicts"} { + for _, state := range []string{"candidates", "findings", "graph", "heard", "manifests", "modules", "packages", "predefined", "seat-definitions", "taken"} { if !slices.Contains(m.Emits, views.EventPrefix+state) { t.Errorf("emits = %v: want %s%s", m.Emits, views.EventPrefix, state) } diff --git a/modules/mesh-delivery-indexer/module.json b/modules/mesh-delivery-indexer/module.json index f98f5f8f..43222891 100644 --- a/modules/mesh-delivery-indexer/module.json +++ b/modules/mesh-delivery-indexer/module.json @@ -29,11 +29,8 @@ "tools": [ "indexer_take_in", "indexer_old_sections", - "indexer_verdicts", - "indexer_judge", "manifests", "modules", - "verdicts", "graph", "indexer_read_manifests", "indexer_read_predefined", @@ -45,14 +42,12 @@ "indexer_read_takeins", "indexer_read_findings", "indexer_read_seat_definitions", - "indexer_read_unjudged", "indexer_first_views", "indexer_first_view_again", "indexer_config_readers", "indexer_bus_state", "indexer_requirements", "indexer_package_users", - "indexer_say_accepted", "indexer_seat_definition", "indexer_seat_definitions_again", "indexer_out_of_step", @@ -94,12 +89,9 @@ "mesh-delivery-forge-bridge.forge.**", "mesh-catalog.catalog-revision", "mesh-delivery.source-moved", - "mesh-delivery.pinned", - "mesh-delivery.freed", - "mesh-delivery.heads-replaced", - "mesh-delivery.pins-stated", "mesh-wants.wanted", "mesh-pipeline-runner.run-started", + "mesh-pipeline-runner.pipeline-started", "mesh-pipeline-runner.pipeline-ended", "node-engine.running", "operator-channel.decided", @@ -112,7 +104,6 @@ { "name": "predefined", "history": 5 }, { "name": "taken", "history": 1 }, { "name": "takeins", "history": 1 }, - { "name": "verdicts", "history": 5 }, { "name": "oldsections", "history": 1 }, { "name": "runs", "history": 1 }, { "name": "own-build", "history": 1 }, @@ -123,7 +114,6 @@ { "name": "requirements", "history": 5 }, { "name": "seat-definitions", "history": 5 }, { "name": "pending", "history": 1 }, - { "name": "unjudged", "history": 1 }, { "name": "entry-index", "history": 1, "max-mib": 8 }, { "name": "named", "history": 1, "max-mib": 4 }, { "name": "registrations", "history": 1 }, @@ -153,8 +143,7 @@ "views.packages", "views.predefined", "views.seat-definitions", - "views.taken", - "views.verdicts" + "views.taken" ], "reads": [ "mesh-build-go.toolchains",