I7, I6 (the pins' events): no validation at the take-in; a pipeline's events pin its revision
I7: the indexer judges no manifest. The judge pass, its verdicts and unjudged states, the events manifest-accepted and manifest-refused, the tools verdicts, indexer_verdicts, indexer_judge and indexer_say_accepted, and the take-in's wait for the verdicts state go; indexer_bus_state answers from the modules state at its mark. The seed writes validate required in every policy. oldsections stays (H-A-79). I6, the half that follows C-R: the gate's pinned, freed, heads-replaced and pins-stated are no longer heard. A pipeline-started pins the revision it names under pipeline.<branch>.<source>, its pipeline-ended frees it, each on a durable of its own stream; kept revisions are judged against the pins the state holds, never a count (made and pins-stated go). A take-in pins its revision once, for its one pipeline. indexer_pin_drop (§765) is still to do. The entry's pull goes (§763). The tests of the old take-in (pending, behind, pushed) go with it, ahead of I8. The unit suite is green except manifest_test's check that the watchman's and the grants part's invokes of the indexer's verdicts tools are served: their removal is the contracts developer's (H-A-79). Builds: design 47 H §743, §745, §749, §763, §764 (the indexer); H-A-48, H-A-49, H-A-79 Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy
This commit is contained in:
@@ -30,7 +30,7 @@ import (
|
||||
type asker func(key string, value []byte) (index.TakeInAsk, error)
|
||||
|
||||
// Tools are the indexer's tools. read is the indexer's state of verdicts, which indexer_verdicts reads; counts the old sections' count.
|
||||
func Tools(ask asker, now func() time.Time, read stdio.KeptState, counts func() (index.OldSectionsCount, error), acts meshhandactsclient.Recorder) []stdio.Tool {
|
||||
func Tools(ask asker, now func() time.Time, counts func() (index.OldSectionsCount, error), acts meshhandactsclient.Recorder) []stdio.Tool {
|
||||
str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) }
|
||||
return []stdio.Tool{
|
||||
{Name: index.OldSectionsTool,
|
||||
@@ -90,55 +90,6 @@ func Tools(ask asker, now func() time.Time, read stdio.KeptState, counts func()
|
||||
}
|
||||
return a.Result, nil
|
||||
}},
|
||||
{Name: index.VerdictsTool,
|
||||
Description: "The indexer's verdicts on manifests that wait or are refused (all: every verdict), each as its verdicts state holds it: " +
|
||||
"the module, the digest, the state (waits: the manifest carries a field the indexer's judge does not know yet, named in " +
|
||||
"unknown, and every reader holds the module as a refused one and keeps what runs; refused: the judge's problems), the " +
|
||||
"judge that made it and when. A module's name narrows it to that module. A waiting manifest is judged again at each " +
|
||||
"new indexer and refused only once the newest build of the indexer on the trunk still does not know the field; " +
|
||||
"indexer_judge judges now.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{
|
||||
"module": map[string]any{"type": "string", "description": "only this module's verdicts"},
|
||||
"all": map[string]any{"type": "boolean", "description": "list accepted verdicts too"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
keys, err := read.Keys()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the verdicts state cannot be listed: %w", err)
|
||||
}
|
||||
all, _ := args["all"].(bool)
|
||||
rows, unread := index.ListVerdicts(keys, func(key string) ([]byte, error) {
|
||||
e, err := read.Get(key)
|
||||
if err != nil || e == nil {
|
||||
return nil, err
|
||||
}
|
||||
return e.Value, nil
|
||||
}, str(args, "module"), all)
|
||||
return map[string]any{"verdicts": rows, "unread": unread}, nil
|
||||
}},
|
||||
{Name: index.SayAcceptedTool,
|
||||
Description: "Say the event manifest-accepted again now for one module, from the accepted verdict on its trunk's manifest the " +
|
||||
"indexer holds (design 25 § 12.2): for a verdict the bus module is thought to have missed. A hand act, recorded through " +
|
||||
"mesh-hand-acts before it is asked. Refused for a module the indexer holds no accepted verdict of (refused, set aside " +
|
||||
"to wait, or not held). Answers what was said: the digest, the verdict's revision and the bus-state entries' count. A " +
|
||||
"publish that failed twice is also said again by itself at the indexer's next judging pass (its finding accepted-unsaid).",
|
||||
Input: map[string]any{"type": "object", "required": []string{"module", "why"}, "properties": map[string]any{
|
||||
"module": map[string]any{"type": "string", "description": "the module's name, as the catalogue registers it"},
|
||||
"why": map[string]any{"type": "string", "description": "why it is said again now, in plain words"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
key, value, err := index.NewSayAccepted(str(args, "module"), str(args, "why"), stdio.CallerOf(args), now())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err := acts.Record(meshhandactsclient.Request{What: "said manifest-accepted of " + str(args, "module") + " again by hand",
|
||||
Why: str(args, "why"), Cause: index.SayAcceptedTool, Verb: "say-accepted"}); err != nil {
|
||||
return nil, fmt.Errorf("the hand act was not recorded through mesh-hand-acts, so nothing was said again: %w", err)
|
||||
}
|
||||
a, err := ask(key, value)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return a.Result, nil
|
||||
}},
|
||||
{Name: index.SeatDefinitionsAgainTool,
|
||||
Description: "Read a contracts module's seat definitions again now (design 26): its trunk's, at the commit the indexer " +
|
||||
"holds the module at, or with pull and head a pull request's head (read with the head's own manifest, so its " +
|
||||
@@ -170,27 +121,6 @@ func Tools(ask asker, now func() time.Time, read stdio.KeptState, counts func()
|
||||
}
|
||||
return a.Result, nil
|
||||
}},
|
||||
{Name: index.JudgeAgainTool,
|
||||
Description: "Judge the manifests again now, as the indexer does at its start and after each merge, and answer how the module's " +
|
||||
"verdict stands (waits, refused or accepted, and why). For a manifest that waits for a judge that knows its fields, or " +
|
||||
"after a manifest was mended where the indexer has not heard it. A waiting manifest becomes refused here only when the " +
|
||||
"running indexer is the newest build of itself the trunk holds. With a digest it judges that one manifest of the " +
|
||||
"module (a candidate's, or one an older judge refused) in the catalogue the trunk holds. indexer_verdicts reads the result.",
|
||||
Input: map[string]any{"type": "object", "required": []string{"module"}, "properties": map[string]any{
|
||||
"module": map[string]any{"type": "string", "description": "the module's name, as the catalogue registers it"},
|
||||
"digest": map[string]any{"type": "string", "description": "one manifest digest of the module to judge (indexer_verdicts names it); none judges every manifest the trunk holds"},
|
||||
"why": map[string]any{"type": "string", "description": "why it is judged again now, in plain words"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
key, value, err := index.NewJudgeAgain(str(args, "module"), str(args, "digest"), str(args, "why"), stdio.CallerOf(args), now())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
a, err := ask(key, value)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return a.Result, nil
|
||||
}},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -285,7 +215,7 @@ var errReadOnly = errors.New("the tools bundle only reads this state")
|
||||
func main() {
|
||||
asks := askTakeIn(stdio.State(index.TakeInsState), 25*time.Second)
|
||||
acts := meshhandactsclient.Recorder{Ask: stdio.Ask}
|
||||
tools := Tools(asks, time.Now, stdio.State(index.VerdictsState), heldCounts(stdio.State(index.OldSectionsState)), acts)
|
||||
tools := Tools(asks, time.Now, heldCounts(stdio.State(index.OldSectionsState)), acts)
|
||||
// The modules and the module graph are the newest catalog graph revision's (milestone 47/G): revision.go.
|
||||
revisions := bundleObjects{busstate.ObjectsOf(index.CatalogRevisionsObjects)}
|
||||
// The view of the trunk is read from the indexer's own objects once when the runtime starts and again each time the seat's
|
||||
|
||||
@@ -84,7 +84,7 @@ func busStateTool(state func(string) views.Source) stdio.Tool {
|
||||
"answer is at, so every manifest-accepted above it is newer. The bus module's keeper takes its first view from it.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{}},
|
||||
Run: func(map[string]any) (any, error) {
|
||||
return index.BusStateOf(state("modules"), state(index.VerdictsState), state("manifests"), state(views.MarksState))
|
||||
return index.BusStateOf(state("modules"), state("manifests"), state(views.MarksState))
|
||||
}}
|
||||
}
|
||||
|
||||
|
||||
@@ -391,8 +391,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
return err
|
||||
}
|
||||
defer calls.Close()
|
||||
// The verdicts on manifests (ADR 0578 §1): a state this delivery brings, bound when it appears.
|
||||
verdicts := process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.VerdictsState), "no verdict on a manifest is written", logger.Printf)
|
||||
// The published packages (ADR 0593 §2): a state this delivery brings, bound when it appears; until then every
|
||||
// fingerprint takes the version line.
|
||||
packages := process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.PackagesState), "no package is published, so every fingerprint takes the version line", logger.Printf)
|
||||
@@ -407,9 +405,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// unanswered since, the newest toolchain-agreed and version-delivered, the artifacts published so far of a version).
|
||||
heardStore := awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.HeardState), "what the indexer heard is not kept", logger.Printf)}
|
||||
ownBuildState := process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.OwnBuildStateName), "which build this indexer runs is not kept", logger.Printf)
|
||||
// The manifests a take-in wrote and no judge pass judged yet (issue 1966): a state this delivery brings, bound when it
|
||||
// appears; until then a take-in's new manifests are kept for their judging in its memory alone.
|
||||
unjudged := process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.UnjudgedState), "a take-in's new manifests are kept for their judging in its memory alone", logger.Printf)
|
||||
// The runs on request the indexer holds a candidate for (A-174): a state this delivery brings, bound when it appears.
|
||||
runs := process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.RunsState), "no run on request's candidate is written or dropped", logger.Printf)
|
||||
findings := process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.FindingsState), "no finding of the indexer is written", logger.Printf)
|
||||
@@ -445,7 +440,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
named := index.NewNamed(awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.NamedState), "no entry is kept for a want or a running build that names it", logger.Printf)}, logger.Printf)
|
||||
pinsBucket := process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.PinsState), "no record's pin on a catalog graph revision is kept, so no revision is removed", logger.Printf)
|
||||
pinsStore := awaitedStore{b: pinsBucket}
|
||||
consumers := process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, process.ConsumersState), "indexer_consumer reads no consumer", logger.Printf)
|
||||
// The state inputs and the object store beside it; the seat mesh-catalog's verb input is answered here in chunks of the core
|
||||
// connection, of any size (design 47 G §712, A-237).
|
||||
inputsStore := awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.InputsState), "no input of a run ask is kept, so the verb input answers none", logger.Printf)}
|
||||
@@ -456,8 +450,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
return fmt.Errorf("answering the verb input: %w", err)
|
||||
}
|
||||
defer stopInput()
|
||||
ix := &index.Indexer{Module: cred.Module, NamedSets: named, Findings: writer, OldSections: awaitedStore{b: oldSections}, Unjudged: awaitedStore{b: unjudged}, Requirements: awaitedStore{b: requirements}, Store: kvStore{candidates}, Modules: modules, Manifests: kvStore{manifests}, Taken: awaitedStore{b: taken}, Builds: known,
|
||||
Verdicts: awaitedStore{b: verdicts}, SayRefused: sayRefused(conn, cred, logger.Printf), SayAccepted: sayAccepted(conn, cred),
|
||||
ix := &index.Indexer{Module: cred.Module, NamedSets: named, Findings: writer, OldSections: awaitedStore{b: oldSections}, Requirements: awaitedStore{b: requirements}, Store: kvStore{candidates}, Modules: modules, Manifests: kvStore{manifests}, Taken: awaitedStore{b: taken}, Builds: known,
|
||||
Predefined: awaitedStore{b: predefined}, Logf: logger.Printf,
|
||||
Named: namedVia(calls.Call), Trunk: trunkVia(calls.Call), BranchHead: branchHeadVia(calls.Call), Contains: containsVia(calls.Call), Read: readVia(calls.Call),
|
||||
ReadOpen: openVia(calls.Call), Packages: awaitedStore{b: packages, js: js, name: index.Bucket(cred.Module, index.PackagesState)}, Files: filesVia(calls.Call), GoSum: goSumVia(calls.Call), GoMod: goModVia(calls.Call),
|
||||
@@ -667,35 +660,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// indexer_first_views and as view-waits past five minutes, and read again on the next forge message heard.
|
||||
defs := gitWaits(ctx, cred.Module, ix, writer, cred.Node, process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, views.StatusState), "indexer_first_views does not say the wait for the seat git", logger.Printf), logger.Printf)
|
||||
seatDefs.Ready(func(jetstream.KeyValue) { defs.Start(ctx) })
|
||||
// Every manifest judged at the start, so a judge that changed with this program judges every one again (ADR 0578
|
||||
// §1), once the verdicts state is there; each merge judges them again (index.ApplyMessage).
|
||||
verdicts.Ready(func(jetstream.KeyValue) {
|
||||
// A take-in held because the verdicts state was not on the bus yet is evaluated again once the start's pass ended,
|
||||
// whatever it answered (issue 1966 note 154).
|
||||
defer func() {
|
||||
if err := ix.PendingAgain(ctx); err != nil {
|
||||
logger.Printf("take-in: pending is not evaluated again at the verdicts state's bind: %v", err)
|
||||
}
|
||||
}()
|
||||
// A pass that fails (a read the bus did not answer) is judged again at the next message or source revision the
|
||||
// indexer hears (index.kickJudge): no timer.
|
||||
judge := func(fromKept bool) {
|
||||
j, err := ix.JudgeAtStart(ctx)
|
||||
if err != nil {
|
||||
logger.Printf("judging the manifests at start (from kept definitions: %v): %v; judged again at the next message heard", fromKept, err)
|
||||
return
|
||||
}
|
||||
logger.Printf("judged %d manifests at start (from kept definitions: %v): %d verdicts written, %d with problems, of which outgrown and kept accepted: %v, waiting for a judge that knows their fields: %v",
|
||||
j.Judged, fromKept, j.Written, len(j.Refused), j.Outgrown, j.Waiting)
|
||||
// A take-in held while a state was not on the bus is evaluated again at the start (issue 1966 note 154).
|
||||
go func() {
|
||||
if err := ix.PendingAgain(ctx); err != nil {
|
||||
logger.Printf("take-in: pending is not evaluated again at the start: %v", err)
|
||||
}
|
||||
if !defs.Waiting() {
|
||||
judge(false)
|
||||
return
|
||||
}
|
||||
// The seat git did not answer: the judging waits for it at most index.GitWaitBound, off the start, so nothing else of
|
||||
// the start is held behind it.
|
||||
go func() { judge(defs.Ready(ctx)) }()
|
||||
})
|
||||
}()
|
||||
// A build module says each time it wrote its toolchains key: its stream, and the indexer's durable consumer on it, are
|
||||
// made by the controller as the forge's are. Where the stream is not there yet the indexer says so and is started again
|
||||
// with the build module, as with the gate's source.
|
||||
@@ -728,22 +698,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
if err := hub.Handle(meshdelivery.SubjectSourceMovedAll, func(m jetstream.Msg) { source.heard(ctx, m) }); err != nil {
|
||||
return err
|
||||
}
|
||||
pinsLedger := process.NewConsumerLedger(process.EventsStream, cred.EventConsumerName(), consumerSink(consumers), nil, logger.Printf)
|
||||
for _, subject := range pinsEvents {
|
||||
if err := hub.Handle(subject, func(m jetstream.Msg) { heardPins(ctx, ix, pinsLedger, m, logger.Printf) }); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
// Once the state is bound the pins are made and every kept revision judged (§579: at its start).
|
||||
pinsBucket.Ready(func(jetstream.KeyValue) {
|
||||
go func() {
|
||||
if err := ix.PinsMade(ctx); err != nil {
|
||||
logger.Printf("pins: the state %s is not made: %v", index.PinsState, err)
|
||||
return
|
||||
}
|
||||
ix.JudgePins(ctx)
|
||||
}()
|
||||
})
|
||||
// Once the state is bound every kept revision is judged against the pins it holds (design 47 H §749: at its start).
|
||||
pinsBucket.Ready(func(jetstream.KeyValue) { go ix.JudgePins(ctx) })
|
||||
versions := func(m jetstream.Msg) {
|
||||
if !heardVersion(ctx, ix, m, logger.Printf) {
|
||||
_ = m.Ack() // nothing else the indexer consumes rides the shared stream
|
||||
@@ -819,9 +775,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// said up to; a first view (the tools bundle) answers at that mark.
|
||||
startViews(ctx, js, cred, marks, writer,
|
||||
map[string]readyOf{"manifests": boundNow(manifests), "candidates": boundNow(candidates),
|
||||
index.VerdictsState: awaited(verdicts), index.PredefinedState: awaited(predefined),
|
||||
index.PackagesState: awaited(packages),
|
||||
index.HeardState: awaited(heard.b), "taken": awaited(taken), "findings": awaited(findings),
|
||||
index.PredefinedState: awaited(predefined),
|
||||
index.PackagesState: awaited(packages),
|
||||
index.HeardState: awaited(heard.b), "taken": awaited(taken), "findings": awaited(findings),
|
||||
index.SeatDefinitionsState: awaited(seatDefs)}, errs, logger.Printf)
|
||||
cc, err := process.HearEvents(cons, func(m jetstream.Msg) {
|
||||
handle(ctx, m, ix, heard, logger)
|
||||
|
||||
@@ -225,3 +225,9 @@ func (f *fakeRuns) EndRunPipeline(_ context.Context, e meshpipelinerunner.Pipeli
|
||||
f.ended = e.Run
|
||||
return f.err
|
||||
}
|
||||
|
||||
func (f *fakeRuns) HeardPins(context.Context, *meshpipelinerunner.PipelineStarted, *meshpipelinerunner.PipelineEnded) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeRuns) JudgePins(context.Context) {}
|
||||
|
||||
@@ -2,36 +2,36 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"mesh-delivery-indexer/internal/index"
|
||||
)
|
||||
|
||||
// The indexer's durable consumer of mesh-delivery's pinned, freed, heads-replaced and pins-stated (design 47 G §153, §216, §579):
|
||||
// each event is applied to the state pins, and only then acknowledged, so one said while the indexer was down is applied when it
|
||||
// comes back. The consumer's facts are kept in the state consumers (A-176) for indexer_consumer.
|
||||
// The indexer pins a pipeline's revision on the runner's pipeline-started and frees it on its pipeline-ended, each heard on
|
||||
// a durable consumer of its own on that event's stream (design 47 H §749, runs.go): an event is applied to the state pins,
|
||||
// and only then acknowledged, so one said while the indexer was down is applied when it comes back.
|
||||
|
||||
// pinsEvents are the subjects the indexer applies to its pins.
|
||||
var pinsEvents = []string{meshdelivery.SubjectPinned, meshdelivery.SubjectFreed, meshdelivery.SubjectHeadsReplaced, meshdelivery.SubjectPinsStated}
|
||||
// pinsStreams are the streams of the runner's events the indexer's pins follow.
|
||||
var pinsStreams = []string{meshpipelinerunner.StreamPipelineStarted, meshpipelinerunner.StreamPipelineEnded}
|
||||
|
||||
// pendingOf says how many messages of the module's durable consumer of the events stream have not been given to it yet.
|
||||
// pendingOf says how many messages of the indexer's durable consumers of the pins' streams have not been given to it yet.
|
||||
func pendingOf(js jetstream.JetStream, cred process.Account) func(context.Context) (uint64, error) {
|
||||
return func(ctx context.Context) (uint64, error) {
|
||||
cons, err := cred.EventConsumer(ctx, js, process.EventsStream)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
var n uint64
|
||||
for _, stream := range pinsStreams {
|
||||
cons, err := cred.EventConsumer(ctx, js, stream)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
info, err := cons.Info(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
n += info.NumPending
|
||||
}
|
||||
info, err := cons.Info(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return info.NumPending, nil
|
||||
return n, nil
|
||||
}
|
||||
}
|
||||
|
||||
@@ -46,20 +46,3 @@ func consumerSink(b *process.AwaitedBucket) process.ConsumerSink {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// heardPins applies one pins event, acknowledges it, and judges the revisions kept: an event not applied is left unacknowledged, so
|
||||
// the bus gives it again.
|
||||
func heardPins(ctx context.Context, ix *index.Indexer, ledger *process.ConsumerLedger, m jetstream.Msg, logf func(string, ...any)) {
|
||||
ledger.Delivered(m)
|
||||
if err := ix.HeardPins(ctx, m.Subject(), m.Data()); err != nil {
|
||||
logf("pins: %s is not applied, left for the bus to give again: %v", m.Subject(), err)
|
||||
return
|
||||
}
|
||||
_ = m.Ack()
|
||||
var body struct {
|
||||
At time.Time `json:"at"`
|
||||
}
|
||||
_ = json.Unmarshal(m.Data(), &body)
|
||||
ledger.Applied(m, body.At)
|
||||
ix.JudgePins(ctx)
|
||||
}
|
||||
|
||||
@@ -13,10 +13,13 @@ import (
|
||||
"mesh-delivery-indexer/internal/index"
|
||||
)
|
||||
|
||||
// runTaker is what the indexer does with the runner's events of a run on request (index.Indexer.StartRun, EndRunPipeline).
|
||||
// runTaker is what the indexer does with the runner's events: a run on request's (index.Indexer.StartRun, EndRunPipeline),
|
||||
// and a pipeline's pins (HeardPins, design 47 H §749), the kept revisions judged after each pin freed (JudgePins).
|
||||
type runTaker interface {
|
||||
StartRun(ctx context.Context, e meshpipelinerunner.RunStarted) error
|
||||
EndRunPipeline(ctx context.Context, e meshpipelinerunner.PipelineEnded) error
|
||||
HeardPins(ctx context.Context, started *meshpipelinerunner.PipelineStarted, ended *meshpipelinerunner.PipelineEnded) error
|
||||
JudgePins(ctx context.Context)
|
||||
}
|
||||
|
||||
// hearRuns hears the runner's run-started and pipeline-ended (design 47 G §64, §190; A-160, A-174), each on its own stream, on
|
||||
@@ -28,6 +31,7 @@ func hearRuns(ctx context.Context, js jetstream.JetStream, cred process.Account,
|
||||
each func(jetstream.Msg)
|
||||
}{
|
||||
{meshpipelinerunner.StreamRunStarted, func(m jetstream.Msg) { heardRunStarted(ctx, ix, m, logf) }},
|
||||
{meshpipelinerunner.StreamPipelineStarted, func(m jetstream.Msg) { heardPipelineStarted(ctx, ix, m, logf) }},
|
||||
{meshpipelinerunner.StreamPipelineEnded, func(m jetstream.Msg) { heardPipelineEnded(ctx, ix, m, logf) }},
|
||||
} {
|
||||
_ = notHeard.Bind(s.stream, "no run on request's candidate is written or dropped until the indexer starts again", func() error {
|
||||
@@ -53,7 +57,19 @@ func heardRunStarted(ctx context.Context, ix runTaker, m delivered, logf func(st
|
||||
settleRun(m, ix.StartRun(ctx, e), "run-started of "+e.Run, logf)
|
||||
}
|
||||
|
||||
// heardPipelineEnded takes one pipeline-ended in: of a run's pipeline, its module is kept as ended.
|
||||
// heardPipelineStarted pins the revision a pipeline-started names (design 47 H §749).
|
||||
func heardPipelineStarted(ctx context.Context, ix runTaker, m delivered, logf func(string, ...any)) {
|
||||
var e meshpipelinerunner.PipelineStarted
|
||||
if err := json.Unmarshal(m.Data(), &e); err != nil {
|
||||
logf("%s: not a pipeline-started, dropped: %v", m.Subject(), err)
|
||||
_ = m.Term()
|
||||
return
|
||||
}
|
||||
settleRun(m, ix.HeardPins(ctx, &e, nil), "pipeline-started of "+e.Pipeline, logf)
|
||||
}
|
||||
|
||||
// heardPipelineEnded takes one pipeline-ended in: its revision's pin is freed and the kept revisions judged (§749); of a
|
||||
// run's pipeline, its movers are kept as ended.
|
||||
func heardPipelineEnded(ctx context.Context, ix runTaker, m delivered, logf func(string, ...any)) {
|
||||
var e meshpipelinerunner.PipelineEnded
|
||||
if err := json.Unmarshal(m.Data(), &e); err != nil {
|
||||
@@ -61,7 +77,12 @@ func heardPipelineEnded(ctx context.Context, ix runTaker, m delivered, logf func
|
||||
_ = m.Term()
|
||||
return
|
||||
}
|
||||
settleRun(m, ix.EndRunPipeline(ctx, e), "pipeline-ended of "+e.Module+" in the run "+e.Run, logf)
|
||||
if err := ix.HeardPins(ctx, nil, &e); err != nil {
|
||||
settleRun(m, err, "pipeline-ended of "+e.Pipeline, logf)
|
||||
return
|
||||
}
|
||||
settleRun(m, ix.EndRunPipeline(ctx, e), "pipeline-ended of "+e.Pipeline+" in the run "+e.Run, logf)
|
||||
ix.JudgePins(ctx)
|
||||
}
|
||||
|
||||
func settleRun(m delivered, err error, what string, logf func(string, ...any)) {
|
||||
|
||||
@@ -16,8 +16,6 @@ type takerIn interface {
|
||||
TakeIn(ctx context.Context, module string) index.TakenIn
|
||||
TakeInRepository(ctx context.Context, repository string) index.TakenIn
|
||||
CatchUp(ctx context.Context) index.TakenIn
|
||||
JudgeAgain(ctx context.Context, module, digest string) index.TakenIn
|
||||
SayAcceptedAgain(ctx context.Context, module string) index.TakenIn
|
||||
SeatDefinitionsAgain(ctx context.Context, module string, pull int, head string) index.TakenIn
|
||||
TakeInPendingByHand(ctx context.Context, key, by, why string) index.TakenIn
|
||||
DropPending(ctx context.Context, key, by, why string, abandon bool) index.TakenIn
|
||||
@@ -88,10 +86,6 @@ func takenIn(ctx context.Context, ix takerIn, kv jetstream.KeyValue, e jetstream
|
||||
a.Module = a.Repository
|
||||
} else if a.SeatDefinitions {
|
||||
asked, r = "reading of its seat definitions again", ix.SeatDefinitionsAgain(ctx, a.Module, a.Pull, a.Head)
|
||||
} else if a.Say {
|
||||
asked, r = "manifest-accepted said again", ix.SayAcceptedAgain(ctx, a.Module)
|
||||
} else if a.Judge {
|
||||
asked, r = "judging again", ix.JudgeAgain(ctx, a.Module, a.Digest)
|
||||
} else {
|
||||
r = ix.TakeIn(ctx, a.Module)
|
||||
}
|
||||
|
||||
@@ -2,67 +2,14 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
|
||||
// sayRefused says the indexer's event manifest-refused (novox/hq ADR 0578 §1): mesh.seat.mesh-catalog.event.manifest-refused,
|
||||
// with the SDK's envelope headers and the digest and judge as the bus's dedup id, so one refusal is said once.
|
||||
func sayRefused(conn *nats.Conn, cred process.Account, logf func(string, ...any)) func(context.Context, meshcatalog.ManifestRefusedEvent) {
|
||||
return func(ctx context.Context, r meshcatalog.ManifestRefusedEvent) {
|
||||
data, err := json.Marshal(r)
|
||||
if err != nil {
|
||||
logf("judge: %s for %s is not said: %v", meshcatalog.EventManifestRefused, r.Module, err)
|
||||
return
|
||||
}
|
||||
msg := nats.NewMsg(seatEventSubject(meshcatalog.EventManifestRefused))
|
||||
id := "manifest-refused-" + r.Digest + "-" + r.Judge
|
||||
msg.Header.Set("x-event-id", id)
|
||||
msg.Header.Set("x-source", cred.Module)
|
||||
msg.Header.Set("x-node", cred.Node)
|
||||
msg.Header.Set("x-time", time.Now().UTC().Format(time.RFC3339Nano))
|
||||
msg.Header.Set("content-type", "application/json")
|
||||
msg.Header.Set(nats.MsgIdHdr, id)
|
||||
msg.Data = data
|
||||
// An event is published, never asked (issue 1966): the stream that keeps the subject takes it and drops a
|
||||
// duplicate by its id; a bus refusing the subject says so on the connection's error handler, not by a wait here.
|
||||
if err := conn.PublishMsg(msg); err != nil {
|
||||
logf("judge: %s", fmt.Errorf("saying %s for %s: %w", meshcatalog.EventManifestRefused, r.Module, err))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// sayAccepted says the indexer's event manifest-accepted (issue 1966 D3, novox/hq ADR 0595 §4):
|
||||
// mesh.seat.mesh-catalog.event.manifest-accepted, carrying the module's bus-state entries whole, with the digest and judge as
|
||||
// the bus's dedup id (meshcatalog.ManifestAcceptedEvent.EventID), so one acceptance is said once.
|
||||
func sayAccepted(conn *nats.Conn, cred process.Account) func(context.Context, meshcatalog.ManifestAcceptedEvent, string) error {
|
||||
return func(ctx context.Context, a meshcatalog.ManifestAcceptedEvent, id string) error {
|
||||
data, err := json.Marshal(a)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
msg := nats.NewMsg(seatEventSubject(meshcatalog.EventManifestAccepted))
|
||||
msg.Header.Set("x-event-id", id)
|
||||
msg.Header.Set("x-source", cred.Module)
|
||||
msg.Header.Set("x-node", cred.Node)
|
||||
msg.Header.Set("x-time", time.Now().UTC().Format(time.RFC3339Nano))
|
||||
msg.Header.Set("content-type", "application/json")
|
||||
msg.Header.Set(nats.MsgIdHdr, id)
|
||||
msg.Data = data
|
||||
// Published, never asked (issue 1966): the event's stream takes it and drops a duplicate by its id; waiting on
|
||||
// the stream's acknowledgement held each judging pass twenty seconds a manifest on a bus refusing the subject.
|
||||
// A refusal is said on the connection's error handler; an error here (no connection) is the caller's to retry.
|
||||
return conn.PublishMsg(msg)
|
||||
}
|
||||
}
|
||||
|
||||
// sayTakeIn says a take-in event (novox/hq ADR 0633 §5) as the indexer's ordinary event, on the path its other events
|
||||
// take: mesh.mod.<module>.event.take-in.<kind>, the wait's key in the body. A publish that fails is the caller's to say.
|
||||
func sayTakeIn(js jetstream.JetStream, cred process.Account) func(context.Context, takein.TakeIn) error {
|
||||
|
||||
@@ -183,6 +183,5 @@ func gitWaits(ctx context.Context, module string, ix *index.Indexer, w index.Fin
|
||||
f := &views.Follower{Owner: git.Seat, Tool: "seat:" + git.Seat + "." + git.VerbTrees, State: index.SeatDefinitionsState, Logf: logf,
|
||||
RaiseWaits: func(s views.Status) { finding(s, true) }, ClearWaits: func(s views.Status) { finding(s, false) }}
|
||||
status.Ready(func(kv jetstream.KeyValue) { f.Said = views.KeepStatus(ctx, kv, logf) })
|
||||
return &index.DefinitionsAtStart{Read: ix.ReadTrunkDefinitions, Waits: f, Logf: logf,
|
||||
Judge: func(ctx context.Context) error { _, err := ix.JudgeAtStart(ctx); return err }}
|
||||
return &index.DefinitionsAtStart{Read: ix.ReadTrunkDefinitions, Waits: f, Logf: logf}
|
||||
}
|
||||
|
||||
@@ -1,190 +0,0 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
|
||||
"mesh-delivery-indexer/judge"
|
||||
)
|
||||
|
||||
// manifest-accepted, never lost silently (design 25 § 12.2; DLV-I9, I9b, I9c). The indexer says it for every newly
|
||||
// accepted manifest, its bus-state entries empty when it asks for none, and an empty one for a module leaving the trunk,
|
||||
// so the bus module's orphans see it; a manifest the judge set aside to wait gets none. A publish that fails is published
|
||||
// once more; one still failing writes the indexer's finding naming the module, said again at each judging pass after
|
||||
// and cleared once said. indexer_say_accepted says it again for one module by hand, from the verdict it holds.
|
||||
const (
|
||||
SayAcceptedTool = "indexer_say_accepted"
|
||||
sayAcceptedPrefix = "say."
|
||||
// UnsaidKind is the finding of a manifest-accepted whose publish failed twice.
|
||||
UnsaidKind = "accepted-unsaid"
|
||||
)
|
||||
|
||||
// acceptedOf is the event body for the verdict v (its revision in the verdicts state) on the manifest raw of digest, its
|
||||
// entries an empty list when it asks for none.
|
||||
func acceptedOf(digest string, v meshcatalog.VerdictsByDigestValue, revision uint64, raw []byte) meshcatalog.ManifestAcceptedEvent {
|
||||
m, _ := judge.Manifest(raw, nil)
|
||||
a := acceptedEvent(digest, v, m)
|
||||
a.Revision = int64(revision)
|
||||
return a
|
||||
}
|
||||
|
||||
// leftOf is the empty manifest-accepted for module, which left the trunk, judged by judgeID; revision is the modules
|
||||
// state's revision of the value that left (0 when none is held), so its event id (acceptedEventID, the
|
||||
// module and the revision, since its body names no digest) differs for each leave.
|
||||
func leftOf(module string, revision uint64, judgeID string) (meshcatalog.ManifestAcceptedEvent, string) {
|
||||
a := meshcatalog.ManifestAcceptedEvent{V: keys.Version, Module: module, Judge: judgeID, BusState: busStateOf(nil), Revision: int64(revision)}
|
||||
return a, acceptedEventID(a)
|
||||
}
|
||||
|
||||
// sayAccepted says a with the event id id, once more when the first publish fails; the module's finding is written when
|
||||
// both fail and deleted when it is said. Nil SayAccepted says nothing.
|
||||
func (ix *Indexer) sayAccepted(ctx context.Context, a meshcatalog.ManifestAcceptedEvent, id string) error {
|
||||
if ix.SayAccepted == nil {
|
||||
return nil
|
||||
}
|
||||
err := ix.SayAccepted(ctx, a, id)
|
||||
if err != nil {
|
||||
ix.logf("judge: %s for %s failed, published again: %v", meshcatalog.EventManifestAccepted, a.Module, err)
|
||||
err = ix.SayAccepted(ctx, a, id)
|
||||
}
|
||||
ix.unsaid(ctx, a.Module, err)
|
||||
return err
|
||||
}
|
||||
|
||||
// unsaidID is the id of module's unsaid finding.
|
||||
func unsaidID(own, module string) string { return own + ".accepted." + module }
|
||||
|
||||
// unsaid raises module's finding for err, or clears it with none.
|
||||
func (ix *Indexer) unsaid(ctx context.Context, module string, err error) {
|
||||
if ix.Findings == nil {
|
||||
return
|
||||
}
|
||||
var werr error
|
||||
if err == nil {
|
||||
werr = ix.clearFinding(ctx, mcc.ScopeModule, unsaidID(ix.Module, module), UnsaidKind, "manifest-accepted of "+module+" was said.")
|
||||
} else {
|
||||
werr = ix.Findings.Write(ctx, ix.observed(mcc.ScopeModule, unsaidID(ix.Module, module), UnsaidKind,
|
||||
fmt.Sprintf("%s of %s was not published: %v", meshcatalog.EventManifestAccepted, module, err),
|
||||
"The indexer could not tell the bus module that "+module+"'s manifest was accepted",
|
||||
"The bus module creates and removes a module's bus state from this event. The indexer says it again at its "+
|
||||
"next judging pass and clears this once it is said; indexer_say_accepted says it now.", UnsaidKind))
|
||||
}
|
||||
if werr != nil {
|
||||
ix.logf("judge: the finding of %s for %s is not written: %v", meshcatalog.EventManifestAccepted, module, werr)
|
||||
}
|
||||
}
|
||||
|
||||
// sayFor says module's manifest-accepted from what the indexer holds now: from its trunk's accepted verdict, or an empty
|
||||
// one when the trunk holds the module no more (only when left is allowed: by hand a module with no accepted verdict is
|
||||
// refused). A held module whose verdict waits or is refused gets none; again gives the event an id of its own, so a hand
|
||||
// act within the bus's duplicate window is still said.
|
||||
func (ix *Indexer) sayFor(ctx context.Context, module string, allowLeft, again bool) (string, error) {
|
||||
held, err := ix.heldModules(ctx)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("the modules state cannot be read: %w", err)
|
||||
}
|
||||
m, ok := held[module]
|
||||
var a meshcatalog.ManifestAcceptedEvent
|
||||
var id string
|
||||
switch {
|
||||
case !ok && !allowLeft:
|
||||
return "", fmt.Errorf("the indexer holds no module %s, so it has no accepted verdict to say", module)
|
||||
case !ok:
|
||||
a, id = leftOf(module, 0, ix.judgeID())
|
||||
case m.Manifest == "" || ix.Verdicts == nil || ix.Manifests == nil:
|
||||
return "", fmt.Errorf("%s has no accepted verdict the indexer holds", module)
|
||||
default:
|
||||
key, err := meshcatalogclient.VerdictsKey(m.Manifest)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
value, revision, found, err := ix.Verdicts.Get(ctx, key)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("the verdict on %s cannot be read: %w", module, err)
|
||||
}
|
||||
var v meshcatalog.VerdictsByDigestValue
|
||||
if found {
|
||||
v, err = meshcatalogclient.DecodeVerdict(value)
|
||||
}
|
||||
if !found || err != nil || meshcatalogclient.VerdictHeld(v) {
|
||||
if !allowLeft {
|
||||
return "", fmt.Errorf("%s has no accepted verdict on its manifest %s", module, m.Manifest)
|
||||
}
|
||||
ix.unsaid(ctx, module, nil) // a set-aside or refused manifest gets none
|
||||
return "none: its manifest is not accepted", nil
|
||||
}
|
||||
raw, _, found, err := ix.Manifests.Get(ctx, m.Manifest)
|
||||
kept, derr := meshcatalogclient.DecodeManifest(raw)
|
||||
if err != nil || !found || derr != nil {
|
||||
return "", fmt.Errorf("the manifest %s of %s cannot be read", m.Manifest, module)
|
||||
}
|
||||
a = acceptedOf(m.Manifest, v, revision, []byte(kept.Manifest))
|
||||
id = acceptedEventID(a)
|
||||
}
|
||||
if again {
|
||||
id += "-again-" + ix.now().UTC().Format(time.RFC3339Nano)
|
||||
}
|
||||
if err := ix.sayAccepted(ctx, a, id); err != nil {
|
||||
return "", fmt.Errorf("%s of %s was not published: %w", meshcatalog.EventManifestAccepted, module, err)
|
||||
}
|
||||
if a.Digest == "" {
|
||||
return "said empty: the trunk holds no module " + module, nil
|
||||
}
|
||||
return fmt.Sprintf("said for %s, manifest %s at revision %d, %d bus-state entries", module, a.Digest, a.Revision, len(a.BusState)), nil
|
||||
}
|
||||
|
||||
// SayUnsaid says again each manifest-accepted whose finding stands (the judging pass's end): cleared once said. What stands is
|
||||
// what the finding writer holds (design 47 G A-38).
|
||||
func (ix *Indexer) SayUnsaid(ctx context.Context) {
|
||||
if ix.Findings == nil {
|
||||
return
|
||||
}
|
||||
for _, f := range ix.Findings.Held() {
|
||||
if f.Kind != UnsaidKind {
|
||||
continue
|
||||
}
|
||||
module := strings.TrimPrefix(f.Subject.ID, ix.Module+".accepted.")
|
||||
if _, err := ix.sayFor(ctx, module, true, false); err != nil {
|
||||
ix.logf("judge: %v; its finding stands", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// SayAcceptedAgain answers indexer_say_accepted: module's manifest-accepted said again now from its accepted verdict.
|
||||
func (ix *Indexer) SayAcceptedAgain(ctx context.Context, module string) TakenIn {
|
||||
out := TakenIn{Module: module}
|
||||
answer, err := ix.sayFor(ctx, module, false, true)
|
||||
if err != nil {
|
||||
out.Answer = err.Error()
|
||||
return out
|
||||
}
|
||||
out.Stored, out.Answer = true, answer
|
||||
return out
|
||||
}
|
||||
|
||||
// NewSayAccepted is the value indexer_say_accepted writes for a module: its key in the takeins state and the value.
|
||||
func NewSayAccepted(module, why, by string, at time.Time) (string, []byte, error) {
|
||||
key, data, err := NewTakeIn(module, why, by, at)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
var a TakeInAsk
|
||||
if err := json.Unmarshal(data, &a); err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
if a.Why == "" {
|
||||
return "", nil, errors.New("say why it is said again (why): it is a hand act")
|
||||
}
|
||||
a.Say = true
|
||||
data, err = json.Marshal(a)
|
||||
return sayAcceptedPrefix + key, data, err
|
||||
}
|
||||
@@ -1,25 +0,0 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
)
|
||||
|
||||
// Design 47 G §668, §692: manifest-accepted's problems are the validator's strings, the same the verdict writes as
|
||||
// warnings, in the same order; a mended manifest says an empty list, never null.
|
||||
func TestManifestAcceptedProblemsAreTheVerdictsWarnings(t *testing.T) {
|
||||
warnings := []string{`requires "amqp", which no module provides`, "a second problem"}
|
||||
got := acceptedOf("sha256:aa", meshcatalog.VerdictsByDigestValue{Module: "m", Judge: "j", Warnings: warnings}, 3, []byte(`{"module":"m","version":"1.0"}`))
|
||||
if !reflect.DeepEqual(got.Problems, warnings) {
|
||||
t.Errorf("problems %v, want the verdict's warnings %v", got.Problems, warnings)
|
||||
}
|
||||
mended := acceptedOf("sha256:bb", meshcatalog.VerdictsByDigestValue{Module: "m", Judge: "j"}, 4, []byte(`{"module":"m","version":"2.0"}`))
|
||||
raw, _ := json.Marshal(mended)
|
||||
if !strings.Contains(string(raw), `"problems":[]`) {
|
||||
t.Errorf("a mended manifest's event is %s, want an empty problems list", raw)
|
||||
}
|
||||
}
|
||||
@@ -1,112 +0,0 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk"
|
||||
)
|
||||
|
||||
// Design 47 G A-463: a merge the indexer never took in is noticed at a source revision and taken in as its merged message
|
||||
// would have been, with a finding raised while it stays behind and cleared once it is taken in.
|
||||
func TestAMergeTheIndexerNeverTookInIsTakenInAndSaidWhileItIsBehind(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
b := newTakeInBed(t)
|
||||
b.ix.Taken, b.ix.Findings = newMem(), newFindings()
|
||||
key, _ := trunk.RepositoryKey(catalogueRepo)
|
||||
old, head := sha('7'), sha('8')
|
||||
data, _ := takein.EncodeTaken(takein.Taken{Repository: trunk.RepositoryName(catalogueRepo), Merge: old})
|
||||
b.ix.Taken.(*memStore).put(key, data)
|
||||
ancestry := map[[2]string]bool{{old, head}: true} // old is in head
|
||||
b.ix.Contains = func(_ context.Context, _, commit, in string) (bool, error) {
|
||||
return commit == in || ancestry[[2]string{commit, in}], nil
|
||||
}
|
||||
b.ix.source = sourceNow{rev: 3, heads: map[string]string{catalogueRepo: head}}
|
||||
|
||||
// The head has no merged message yet: nothing is taken in and the finding stays.
|
||||
if done := b.ix.TakeInBehind(ctx); len(done) != 0 || len(b.ix.Findings.Held()) != 1 || b.ix.Findings.Held()[0].Kind != UntakenMergeKind {
|
||||
t.Fatalf("a head whose message is not read: taken in %v, findings %+v: want none taken in and %s raised", done, b.ix.Findings.Held(), UntakenMergeKind)
|
||||
}
|
||||
// Its message is there: it is taken in, its taken key moves and the finding is cleared.
|
||||
b.publish(t, b.merge(30, head, "tree-avahi-3", ""))
|
||||
if done := b.ix.TakeInBehind(ctx); len(done) != 1 || len(b.ix.Findings.Held()) != 0 || b.commit(t, "avahi") != head {
|
||||
t.Fatalf("a head with its message: taken in %v, findings %+v, avahi at %s: want it taken in at %s and the finding cleared", done, b.ix.Findings.Held(), b.commit(t, "avahi"), head)
|
||||
}
|
||||
if v, _, _, _ := b.ix.Taken.Get(ctx, key); string(v) == string(data) {
|
||||
t.Errorf("taken.%s still names %s", key, old)
|
||||
}
|
||||
// A head the taken merge already holds (the source not yet moved to a merge just taken in) is left.
|
||||
b.ix.source = sourceNow{rev: 4, heads: map[string]string{catalogueRepo: old}}
|
||||
if done := b.ix.TakeInBehind(ctx); len(done) != 0 {
|
||||
t.Errorf("a head older than the taken merge: taken in %v, want none", done)
|
||||
}
|
||||
// A source revision starts the comparison by itself (no timer): the taken key is put back to the old merge and the source
|
||||
// names the head again.
|
||||
b.ix.Taken.(*memStore).put(key, data)
|
||||
if err := applyS(b.ix, ctx, 5, sourceOf(catalogueRepo, head)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.ix.catchUpWG.Wait()
|
||||
if v, _, _, _ := b.ix.Taken.Get(ctx, key); string(v) == string(data) {
|
||||
t.Errorf("a source revision naming %s left taken.%s at %s", head, key, old)
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 G A-463a: a head the pending holds (a group member waiting for its group's end) is not behind: it raises no finding
|
||||
// and is not taken in alone.
|
||||
func TestAHeadThePendingHoldsIsNotBehind(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
b := newTakeInBed(t)
|
||||
b.ix.Taken, b.ix.Findings = newMem(), newFindings()
|
||||
key, _ := trunk.RepositoryKey(catalogueRepo)
|
||||
old, head := sha('7'), sha('8')
|
||||
data, _ := takein.EncodeTaken(takein.Taken{Repository: trunk.RepositoryName(catalogueRepo), Merge: old})
|
||||
b.ix.Taken.(*memStore).put(key, data)
|
||||
b.ix.Contains = func(_ context.Context, _, commit, in string) (bool, error) { return commit == in || commit == old, nil }
|
||||
b.publish(t, b.merge(30, head, "tree-avahi-3", ""))
|
||||
// The head waits in pending as a member of a group not ended.
|
||||
held := b.merge(30, head, "tree-avahi-3", "")
|
||||
raw, _ := forgemsg.EncodeForge(held)
|
||||
pv, _ := json.Marshal(Pending{V: 1, Forge: raw, Group: "feat", Since: time.Now(), LastWriter: "x"})
|
||||
b.pending.put(takein.PendingKey(1), pv)
|
||||
b.ix.source = sourceNow{rev: 3, heads: map[string]string{catalogueRepo: head}}
|
||||
if done := b.ix.TakeInBehind(ctx); len(done) != 0 || len(b.ix.Findings.Held()) != 0 || b.commit(t, "avahi") == head {
|
||||
t.Fatalf("a head the pending holds: taken in %v, findings %+v, avahi at %s: want none taken in and none raised", done, b.ix.Findings.Held(), b.commit(t, "avahi"))
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 G A-463a: a source revision heard while a catch-up runs is compared again when that catch-up ends.
|
||||
func TestASourceRevisionHeardDuringACatchUpIsComparedWhenItEnds(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
b := newTakeInBed(t)
|
||||
b.ix.Taken, b.ix.Findings = newMem(), newFindings()
|
||||
key, _ := trunk.RepositoryKey(catalogueRepo)
|
||||
h1, h2 := sha('8'), sha('9')
|
||||
b.ix.Contains = func(_ context.Context, _, commit, in string) (bool, error) {
|
||||
return commit == in || (commit == h1 && in == h2), nil
|
||||
}
|
||||
b.publish(t, b.merge(30, h1, "tree-avahi-3", ""))
|
||||
b.publish(t, b.merge(31, h2, "tree-avahi-4", ""))
|
||||
b.ix.source = sourceNow{rev: 3, heads: map[string]string{catalogueRepo: h1}}
|
||||
inner := b.ix.Message
|
||||
heard := false
|
||||
b.ix.Message = func(c context.Context, r, m string, n int) ([]byte, error) {
|
||||
if !heard {
|
||||
heard = true // the catch-up of h1 runs: a new source revision is heard now
|
||||
if err := applyS(b.ix, ctx, 4, sourceOf(catalogueRepo, h2)); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
return inner(c, r, m, n)
|
||||
}
|
||||
b.ix.CatchUp(ctx)
|
||||
b.ix.catchUpWG.Wait()
|
||||
v, _, _, _ := b.ix.Taken.Get(ctx, key)
|
||||
if got, _ := takein.DecodeTaken(v); got.Merge != h2 {
|
||||
t.Errorf("taken after the catch-up = %s, want %s: the revision heard during it is compared afterwards\n%s", got.Merge, h2, b.logs)
|
||||
}
|
||||
}
|
||||
@@ -13,8 +13,8 @@ import (
|
||||
// every trunk module whose accepted manifest asks for bus state, then each manifest-accepted after its revision.
|
||||
const BusStateTool = "indexer_bus_state"
|
||||
|
||||
// BusStateModule is one module of BusStateAnswer: the digest of its trunk's accepted manifest, the verdicts state's
|
||||
// revision of that verdict (the one its manifest-accepted carries, ADR 0622 §2), and its bus-state entries whole.
|
||||
// BusStateModule is one module of BusStateAnswer: the digest of its trunk's manifest, the modules state's revision of its
|
||||
// value, and its bus-state entries whole. No manifest is judged at the take-in (design 47 H §743, §745).
|
||||
type BusStateModule struct {
|
||||
Module string `json:"module"`
|
||||
Digest string `json:"digest"`
|
||||
@@ -22,25 +22,25 @@ type BusStateModule struct {
|
||||
BusState []catalogue.BusStateEntry `json:"bus-state"`
|
||||
}
|
||||
|
||||
// BusStateAnswer is indexer_bus_state's answer: Revision is the verdicts state's mark, read before anything else, so
|
||||
// every manifest-accepted above it is a change after this answer (ADR 0615 §4); Modules in module order.
|
||||
// BusStateAnswer is indexer_bus_state's answer: Revision is the modules state's mark, read before anything else, so every
|
||||
// change of modules above it is a change after this answer (ADR 0615 §4); Modules in module order.
|
||||
type BusStateAnswer struct {
|
||||
Revision uint64 `json:"revision"`
|
||||
Modules []BusStateModule `json:"modules"`
|
||||
}
|
||||
|
||||
// BusStateOf answers indexer_bus_state from the indexer's own states: its trunk modules, the judge's verdicts and the
|
||||
// manifests it keeps, and the marks the views family says them up to. A module refused or waiting, or whose manifest
|
||||
// asks for no bus state, is left out; a value that cannot be read is an error, never a module silently left out.
|
||||
func BusStateOf(modules, verdicts, manifests, marks views.Source) (BusStateAnswer, error) {
|
||||
// BusStateOf answers indexer_bus_state from the indexer's own states: its trunk modules and the manifests it keeps, and
|
||||
// the marks the views family says them up to. A module whose manifest asks for no bus state is left out; a value that
|
||||
// cannot be read is an error, never a module silently left out.
|
||||
func BusStateOf(modules, manifests, marks views.Source) (BusStateAnswer, error) {
|
||||
out := BusStateAnswer{Modules: []BusStateModule{}}
|
||||
if marks != nil {
|
||||
if e, ok, err := marks.Get(VerdictsState); err != nil {
|
||||
return BusStateAnswer{}, fmt.Errorf("the mark of %s cannot be read: %w", VerdictsState, err)
|
||||
if e, ok, err := marks.Get(ModulesState); err != nil {
|
||||
return BusStateAnswer{}, fmt.Errorf("the mark of %s cannot be read: %w", ModulesState, err)
|
||||
} else if ok {
|
||||
m, err := views.DecodeMark(e.Value)
|
||||
if err != nil {
|
||||
return BusStateAnswer{}, fmt.Errorf("the mark of %s: %w", VerdictsState, err)
|
||||
return BusStateAnswer{}, fmt.Errorf("the mark of %s: %w", ModulesState, err)
|
||||
}
|
||||
out.Revision = m.Revision
|
||||
}
|
||||
@@ -60,22 +60,7 @@ func BusStateOf(modules, verdicts, manifests, marks views.Source) (BusStateAnswe
|
||||
}
|
||||
mod, err := meshcatalogclient.DecodeModule(e.Value)
|
||||
if err != nil || mod.Manifest == "" {
|
||||
continue // no manifest held for it, so no verdict
|
||||
}
|
||||
key, err := meshcatalogclient.VerdictsKey(mod.Manifest)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
ve, ok, err := verdicts.Get(key)
|
||||
if err != nil {
|
||||
return BusStateAnswer{}, fmt.Errorf("the verdict on %s cannot be read: %w", name, err)
|
||||
}
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
v, err := meshcatalogclient.DecodeVerdict(ve.Value)
|
||||
if err != nil || v.State != meshcatalogclient.VerdictAccepted {
|
||||
continue
|
||||
continue // no manifest held for it
|
||||
}
|
||||
me, ok, err := manifests.Get(mod.Manifest)
|
||||
if err != nil {
|
||||
@@ -90,12 +75,12 @@ func BusStateOf(modules, verdicts, manifests, marks views.Source) (BusStateAnswe
|
||||
}
|
||||
m, err := catalogue.Decode([]byte(kept.Manifest))
|
||||
if err != nil {
|
||||
return BusStateAnswer{}, fmt.Errorf("the accepted manifest of %s does not read: %w", name, err)
|
||||
return BusStateAnswer{}, fmt.Errorf("the manifest of %s does not read: %w", name, err)
|
||||
}
|
||||
if len(m.BusState) == 0 {
|
||||
continue
|
||||
}
|
||||
out.Modules = append(out.Modules, BusStateModule{Module: name, Digest: mod.Manifest, Revision: ve.Revision, BusState: m.BusState})
|
||||
out.Modules = append(out.Modules, BusStateModule{Module: name, Digest: mod.Manifest, Revision: e.Revision, BusState: m.BusState})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
@@ -70,14 +70,9 @@ type Indexer struct {
|
||||
Predefined Store
|
||||
// Taken is the taken bucket: per repository the last merge taken in (taken.go, ADR 0556); nil writes none.
|
||||
Taken Store
|
||||
// Verdicts is the verdicts state (verdicts.go, novox/hq ADR 0578 §1); nil judges nothing.
|
||||
Verdicts Store
|
||||
// OldSections is the state of how many accepted modules use each old section (oldsections.go, issue 1966 D4); nil keeps no
|
||||
// count and the judge refuses no use of an old section.
|
||||
OldSections Store
|
||||
// Unjudged is the state of the manifest digests a take-in wrote and no judge pass judged yet (unjudged.go, issue 1966);
|
||||
// nil keeps them only in the take-in's memory.
|
||||
Unjudged Store
|
||||
// Requirements is the requirements state (requirements.go, DLV-I7): what each consumer's go.sum requires of each mesh
|
||||
// Go module, kept at each merge; nil keeps none.
|
||||
Requirements Store
|
||||
@@ -85,12 +80,6 @@ type Indexer struct {
|
||||
reqKept map[string]bool // consumers whose go.sum this process kept (requirementsKept)
|
||||
// JudgeID names the judge a verdict says made it; empty is judge.Identity, the running program's.
|
||||
JudgeID string
|
||||
// SayRefused says the event manifest-refused for a refusal newly written; nil says nothing.
|
||||
SayRefused func(ctx context.Context, r meshcatalog.ManifestRefusedEvent)
|
||||
// SayAccepted publishes the event manifest-accepted a under the bus's duplicate id id (accepted.go; issue 1966 D3,
|
||||
// DLV-I9): for every accepted verdict newly written, carrying the entries whole, and an empty one for a module leaving
|
||||
// the trunk; nil says nothing.
|
||||
SayAccepted func(ctx context.Context, a meshcatalog.ManifestAcceptedEvent, id string) error
|
||||
// Findings is the indexer's finding writer (design 47 G A-29), where a manifest-accepted whose publish failed twice is kept
|
||||
// until said (accepted.go, DLV-I9b); nil writes none.
|
||||
Findings FindingWriter
|
||||
@@ -258,7 +247,6 @@ func (ix *Indexer) Apply(ctx context.Context, f forgemsg.Forge) error {
|
||||
// ApplyMessage is Apply, and answers which modules a merged message wrote and kept (for the indexer's heard state). Each
|
||||
// applied merged message is said in one line: its repository, commit and pull request, and the modules written.
|
||||
func (ix *Indexer) ApplyMessage(ctx context.Context, f forgemsg.Forge) (Applied, error) {
|
||||
ix.kickJudge(ctx)
|
||||
var applied Applied
|
||||
if f.Kind == forgemsg.ForgeOpen {
|
||||
return applied, ix.applyOpen(ctx, f)
|
||||
@@ -287,13 +275,6 @@ func (ix *Indexer) ApplyMessage(ctx context.Context, f forgemsg.Forge) (Applied,
|
||||
if err := ix.KeepPackageNames(ctx, f); err != nil {
|
||||
return applied, err
|
||||
}
|
||||
// Every manifest judged again: a rule between manifests can turn on the one this merge moved (ADR 0578 §1).
|
||||
// What this take-in newly wrote is fresh: judged in full (ADR 0579 §1).
|
||||
if _, err := ix.JudgeAll(ctx, w.fresh); err != nil {
|
||||
// Judged again at the next message or source revision heard (kickJudge), as a failed start pass is.
|
||||
ix.judgePending.Store(true)
|
||||
ix.logf("judge: the manifests are not judged after %s#%d: %v; judged again at the next message heard", f.Repository, f.Number, err)
|
||||
}
|
||||
}
|
||||
ix.kickRetry(ctx) // a registered module the forge could not read is read again on a merge, in the background (issue 1945)
|
||||
lateMerge = w.late != ""
|
||||
@@ -314,20 +295,6 @@ func (ix *Indexer) ApplyMessage(ctx context.Context, f forgemsg.Forge) (Applied,
|
||||
if held, err = ix.heldModules(ctx); err != nil {
|
||||
return applied, err
|
||||
}
|
||||
// Each manifest the pull carries is judged by this judge, unless it holds a verdict already that this judge made or
|
||||
// that accepts it: a refusal an older judge made never stands for a candidate (or a digest back on the trunk).
|
||||
over := map[string]string{}
|
||||
for _, digest := range digests {
|
||||
if ix.Manifests == nil {
|
||||
break
|
||||
}
|
||||
if value, _, found, err := ix.Manifests.Get(ctx, digest); err == nil && found {
|
||||
if kept, err := meshcatalogclient.DecodeManifest(value); err == nil {
|
||||
over[kept.Module] = digest
|
||||
}
|
||||
}
|
||||
}
|
||||
ix.judgeDigests(ctx, over, fmt.Sprintf("%s#%d", f.Repository, f.Number))
|
||||
}
|
||||
return applied, ix.writeCandidate(ctx, f, digests, held)
|
||||
}
|
||||
|
||||
@@ -42,9 +42,9 @@ func TestTheControlRecords(t *testing.T) {
|
||||
t.Errorf("rows = %+v\nwant %+v", got, want)
|
||||
}
|
||||
})
|
||||
// DLV-I11 (design 25 § 12.2): indexer_bus_state answers every trunk module whose accepted manifest asks for bus state,
|
||||
// with the verdict's revision, at the verdicts state's mark; a refused one, one asking none, and a module with no
|
||||
// manifest held are left out.
|
||||
// DLV-I11 (design 25 § 12.2): indexer_bus_state answers every trunk module whose manifest asks for bus state, with its
|
||||
// value's revision, at the modules state's mark; one asking none and a module with no manifest held are left out. No
|
||||
// verdict is read (I7, design 47 H §745): a manifest the judge once refused is answered as any other.
|
||||
t.Run("indexer_bus_state", func(t *testing.T) {
|
||||
src := memSource{}
|
||||
dAsk, dNone, dRefused := strings.Repeat("a", 64), strings.Repeat("b", 64), strings.Repeat("c", 64)
|
||||
@@ -55,35 +55,25 @@ func TestTheControlRecords(t *testing.T) {
|
||||
}
|
||||
s[key] = &views.Entry{Value: raw, Revision: rev}
|
||||
}
|
||||
modules, verdicts, manifests, marks := memSource{}, memSource{}, memSource{}, src
|
||||
putVerdict := func(digest string, rev uint64, v meshcatalog.VerdictsByDigestValue) {
|
||||
raw, err := meshcatalogclient.EncodeVerdict(v)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
verdicts[digest] = &views.Entry{Value: raw, Revision: rev}
|
||||
}
|
||||
modules, manifests, marks := memSource{}, memSource{}, src
|
||||
for name, digest := range map[string]string{"cache": dAsk, "plain": dNone, "bad": dRefused, "bare": ""} {
|
||||
raw, err := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: "src2:" + strings.Repeat("a", 64), Manifest: digest,
|
||||
Repository: "novox/mesh-catalog", Commit: "c"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
modules[name] = &views.Entry{Value: raw, Revision: 1}
|
||||
modules[name] = &views.Entry{Value: raw, Revision: map[string]uint64{"cache": 7, "bad": 9}[name]}
|
||||
}
|
||||
asks := `{"module":"cache","version":"1.0","requires":[{"name":"c","storage":"key-value","template":"state"}]}`
|
||||
for digest, raw := range map[string]string{dAsk: asks, dNone: `{"module":"plain","version":"1.0"}`, dRefused: asks} {
|
||||
put(manifests, digest, 1, meshcatalog.ManifestsByDigestValue{V: keys.Version, Module: "m", Manifest: raw})
|
||||
}
|
||||
putVerdict(dAsk, 7, meshcatalog.VerdictsByDigestValue{Module: "cache", State: meshcatalogclient.VerdictAccepted, Judge: "j"})
|
||||
putVerdict(dNone, 8, meshcatalog.VerdictsByDigestValue{Module: "plain", State: meshcatalogclient.VerdictAccepted, Judge: "j"})
|
||||
putVerdict(dRefused, 9, meshcatalog.VerdictsByDigestValue{Module: "bad", State: meshcatalogclient.VerdictRefused, Judge: "j", Problems: []string{"p"}})
|
||||
put(marks, VerdictsState, 1, views.Mark{V: views.Version, State: VerdictsState, Revision: 9})
|
||||
got, err := BusStateOf(modules, verdicts, manifests, marks)
|
||||
if err != nil || got.Revision != 9 || len(got.Modules) != 1 || got.Modules[0].Module != "cache" ||
|
||||
got.Modules[0].Digest != dAsk || got.Modules[0].Revision != 7 || len(got.Modules[0].BusState) != 1 ||
|
||||
got.Modules[0].BusState[0].Name != "c" {
|
||||
t.Errorf("indexer_bus_state = %+v (%v)", got, err)
|
||||
put(marks, ModulesState, 1, views.Mark{V: views.Version, State: ModulesState, Revision: 9})
|
||||
got, err := BusStateOf(modules, manifests, marks)
|
||||
if err != nil || got.Revision != 9 || len(got.Modules) != 2 || got.Modules[0].Module != "bad" || got.Modules[1].Module != "cache" ||
|
||||
got.Modules[1].Digest != dAsk || got.Modules[1].Revision != 7 || len(got.Modules[1].BusState) != 1 ||
|
||||
got.Modules[1].BusState[0].Name != "c" {
|
||||
t.Errorf("indexer_bus_state = %+v (%v); want bad and cache, at the modules mark", got, err)
|
||||
}
|
||||
})
|
||||
// DLV-I7 (REG S15, the lead's ruling): the requirements state keeps, per mesh Go module, the highest version each
|
||||
@@ -120,34 +110,32 @@ func TestTheControlRecords(t *testing.T) {
|
||||
// and source, says seat-definition-changed only when a digest moved, keeps the trunk's entry over one that does not
|
||||
// parse (its finding), removes a pull request's entries when it closes, keeps a verdict's out-of-step files, and
|
||||
// raises generated-types-out-of-step from a trunk build end, cleared only by a clean one of the same module.
|
||||
// Milestone 47/G W4.15, design 47 test 54: the seat git not answering at start holds the judging at most its bound, which
|
||||
// then runs on the kept definitions with the wait said; the read once the forge is heard holds the view and judges again.
|
||||
t.Run("W4.15 the start waits for the seat git, judges from kept definitions, and judges again once it answers", func(t *testing.T) {
|
||||
// Milestone 47/G W4.15, design 47 test 54: the seat git not answering at start is waited for at most its bound, the wait
|
||||
// said; the read once the forge is heard holds the view. No manifest is judged (design 47 H §745).
|
||||
t.Run("W4.15 the start waits for the seat git and reads again once it answers", func(t *testing.T) {
|
||||
answers := []error{fmt.Errorf("mesh-contracts: %w", errForgeRead), fmt.Errorf("again: %w", errForgeRead), nil}
|
||||
var said []string
|
||||
judged := 0
|
||||
d := &DefinitionsAtStart{Bound: time.Millisecond, Waits: waitsFake{&said},
|
||||
Read: func(context.Context) (int, error) { err := answers[0]; answers = answers[1:]; return 1, err },
|
||||
Judge: func(context.Context) error { judged++; return nil }}
|
||||
Read: func(context.Context) (int, error) { err := answers[0]; answers = answers[1:]; return 1, err }}
|
||||
ctx := context.Background()
|
||||
d.Start(ctx)
|
||||
if !d.Waiting() || len(said) != 1 || said[0] != "waits "+GitDefinitionsView {
|
||||
t.Fatalf("waiting %v, said %q: want the wait for the seat git said once", d.Waiting(), said)
|
||||
}
|
||||
if !d.Ready(ctx) || judged != 0 {
|
||||
t.Fatalf("the start past its bound: want it judged from the kept definitions, judged again %d", judged)
|
||||
if !d.Ready(ctx) {
|
||||
t.Fatalf("the start past its bound: want it ready on the kept definitions")
|
||||
}
|
||||
d.Heard(ctx) // the forge heard, the seat still not answering: the wait stands, said once
|
||||
if !d.Waiting() || len(said) != 1 || judged != 0 {
|
||||
t.Fatalf("a read still not answered: waiting %v, said %q, judged %d", d.Waiting(), said, judged)
|
||||
if !d.Waiting() || len(said) != 1 {
|
||||
t.Fatalf("a read still not answered: waiting %v, said %q", d.Waiting(), said)
|
||||
}
|
||||
d.Heard(ctx)
|
||||
if d.Waiting() || len(said) != 2 || said[1] != "holds "+GitDefinitionsView || judged != 1 || d.Ready(ctx) {
|
||||
t.Errorf("the seat answering: waiting %v, said %q, judged again %d: want the view held and every manifest judged again once", d.Waiting(), said, judged)
|
||||
if d.Waiting() || len(said) != 2 || said[1] != "holds "+GitDefinitionsView || d.Ready(ctx) {
|
||||
t.Errorf("the seat answering: waiting %v, said %q: want the view held", d.Waiting(), said)
|
||||
}
|
||||
d.Heard(ctx) // nothing waits: no read, no judging
|
||||
if judged != 1 || len(answers) != 0 {
|
||||
t.Errorf("a forge message with nothing waiting read or judged again")
|
||||
d.Heard(ctx) // nothing waits: no read
|
||||
if len(answers) != 0 {
|
||||
t.Errorf("a forge message with nothing waiting read again")
|
||||
}
|
||||
})
|
||||
t.Run("seat definitions", func(t *testing.T) {
|
||||
@@ -484,23 +472,6 @@ func TestTheControlRecords(t *testing.T) {
|
||||
ts.Name != "@novox/mesh-sdk" || ts.Package != "mesh-sdk/ts" || ts.GoModule != "" || len(named) != 2 {
|
||||
t.Errorf("names at the merge = %+v", named)
|
||||
}
|
||||
// The judge's known packages (DLV-I14) are those names, never the builder's <module>/<artifact>.
|
||||
if known := ix.knownPackages(ctx); len(known) != 2 || !known[sdkGo] || !known["@novox/mesh-sdk"] {
|
||||
t.Errorf("the judge's known packages = %v", known)
|
||||
}
|
||||
// Issue 1966: a Go module the trunk holds a per-module record of is known by its module path, though no series
|
||||
// names it; a package inside it is not (design 32).
|
||||
resticMod, at := "git.novox.be/novox/mesh-delivery/modules/restic", time.Unix(1, 0)
|
||||
rec, err := EncodePublishedPackages(meshcatalog.PackagesByModuleValue{Module: resticMod, Repository: "novox/mesh-delivery", Dir: "modules/restic", At: &at,
|
||||
Commit: strings.Repeat("a", 40), Requires: "h1:r", Packages: []meshcatalog.PackagesByModuleValuePackagesItem{{Package: resticMod + "/backupgood", Digest: "h1:a"}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ix.Packages.(*memStore).vals["restic"] = rec
|
||||
if known := ix.knownPackages(ctx); len(known) != 3 || !known[resticMod] || known[resticMod+"/backupgood"] || !known[sdkGo] || known["restic/go"] || known["unknown/pkg"] {
|
||||
t.Errorf("the judge's known packages with a module record = %v", known)
|
||||
}
|
||||
delete(ix.Packages.(*memStore).vals, "restic")
|
||||
delete(ix.Packages.(*memStore).vals, SeriesKey("@novox/mesh-sdk", "0.3"))
|
||||
for _, h := range []HeardVersion{
|
||||
{Package: sdkGo, Seat: "go-package-registry", Version: "0.1.15", Commit: "c15"},
|
||||
@@ -563,7 +534,7 @@ func TestTheControlRecords(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
modules[name] = &views.Entry{Value: raw, Revision: 1}
|
||||
modules[name] = &views.Entry{Value: raw, Revision: map[string]uint64{"cache": 7, "bad": 9}[name]}
|
||||
}
|
||||
got, err := PackageUsersOf(modules, manifests, packages, "")
|
||||
want := []PackageUses{
|
||||
|
||||
@@ -1,52 +0,0 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strconv"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
)
|
||||
|
||||
// The bodies of the seat's events manifest-accepted and manifest-refused are the contract's types; the indexer, who says
|
||||
// them, makes them from a verdict (novox/hq ADR 0634 §1: the logic of one holder stays with it).
|
||||
|
||||
// refusedOf is the body of manifest-refused for the verdict v on the manifest of digest.
|
||||
func refusedOf(digest string, v meshcatalog.VerdictsByDigestValue) meshcatalog.ManifestRefusedEvent {
|
||||
return meshcatalog.ManifestRefusedEvent{V: keys.Version, Module: v.Module, Digest: digest, Judge: v.Judge, Problems: v.Problems}
|
||||
}
|
||||
|
||||
// acceptedEvent is the body of manifest-accepted for the verdict v accepting the manifest m of digest, its bus-state
|
||||
// entries whole (novox/hq ADR 0595 §4); an empty list when m asks for none.
|
||||
func acceptedEvent(digest string, v meshcatalog.VerdictsByDigestValue, m catalogue.Manifest) meshcatalog.ManifestAcceptedEvent {
|
||||
// problems are the manifest validator's strings, the verdict's warnings in their order, an empty list when none (design 47 G §668, §692).
|
||||
problems := append([]string{}, v.Warnings...)
|
||||
return meshcatalog.ManifestAcceptedEvent{V: keys.Version, Module: v.Module, Digest: digest, Judge: v.Judge, BusState: busStateOf(m.BusState), Problems: problems}
|
||||
}
|
||||
|
||||
// busStateOf is the manifest's bus-state entries in the event's form (the same JSON).
|
||||
func busStateOf(entries []catalogue.BusStateEntry) []meshcatalog.ManifestAcceptedEventBusStateItem {
|
||||
out := []meshcatalog.ManifestAcceptedEventBusStateItem{}
|
||||
if len(entries) == 0 {
|
||||
return out
|
||||
}
|
||||
raw, err := json.Marshal(entries)
|
||||
if err == nil {
|
||||
err = json.Unmarshal(raw, &out)
|
||||
}
|
||||
if err != nil {
|
||||
return []meshcatalog.ManifestAcceptedEventBusStateItem{}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// acceptedEventID is the event's id, one per manifest digest and judge, so the bus takes it once:
|
||||
// `manifest-accepted-<digest>-<judge>`. The empty one of a module leaving the trunk carries no digest, so it is named by
|
||||
// its module and the indexer's revision: `manifest-accepted-left-<module>-<revision>`, so two leaves are both kept.
|
||||
func acceptedEventID(a meshcatalog.ManifestAcceptedEvent) string {
|
||||
if a.Digest == "" {
|
||||
return meshcatalog.EventManifestAccepted + "-left-" + a.Module + "-" + strconv.FormatInt(a.Revision, 10)
|
||||
}
|
||||
return meshcatalog.EventManifestAccepted + "-" + a.Digest + "-" + a.Judge
|
||||
}
|
||||
@@ -7,7 +7,6 @@ import (
|
||||
"slices"
|
||||
"sync"
|
||||
|
||||
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
|
||||
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
)
|
||||
@@ -121,11 +120,6 @@ func (m *memFindings) Held() []mcseat.Finding {
|
||||
return out
|
||||
}
|
||||
|
||||
// unsaidKey is the key of module's unsaid finding.
|
||||
func unsaidKey(module string) string {
|
||||
return views.FindingKey(mcc.ScopeModule, keyID(unsaidID(testSelf, module)), UnsaidKind)
|
||||
}
|
||||
|
||||
// testSelf is the module the tests' indexer runs as, its credential's: its findings, its own entry and its graph key are named
|
||||
// from it, never from a constant (design 47 G A-296).
|
||||
const testSelf = "acme-indexer"
|
||||
|
||||
@@ -97,7 +97,7 @@ func TestTheKeptTrunkIsReadAtTheStartAndOncePerHeardRevision(t *testing.T) {
|
||||
if raw, err := kept.Manifest(meshcatalog.BareManifestDigest(keptDigest("ca-trust"))); err != nil || raw != `{"module":"ca-trust"}` {
|
||||
t.Errorf("the held manifest = %q (%v), want the entry's manifest", raw, err)
|
||||
}
|
||||
if a, err := Revision(context.Background(), objects, kept, "avahi", "avahi", meshcatalog.RevisionInput{}); err != nil || a.Revision.Revision != 2 || len(a.Entries) != 2 {
|
||||
if a, err := Revision(context.Background(), objects, kept, meshcatalog.RevisionInput{}); err != nil || a.Revision.Revision != 2 || len(a.Entries) != 2 {
|
||||
t.Errorf("revision of the newest answered %+v (%v), want revision 2 whole", a, err)
|
||||
}
|
||||
if got := objects.n.Load() - listed; got != 0 {
|
||||
@@ -277,11 +277,11 @@ func TestTheNewestRevisionReadersReadAgainWhenTheRevisionIsRemovedUnderThem(t *t
|
||||
return len(a.Behind) == 1 && a.Behind[0].Kind == 5 && strings.Contains(a.Behind[0].Newest, "fp2"), err
|
||||
}},
|
||||
"Revision": {"", func(ix *Indexer) (bool, error) {
|
||||
a, err := Revision(ctx, ix.Revisions, nil, "avahi", "avahi", meshcatalog.RevisionInput{})
|
||||
a, err := Revision(ctx, ix.Revisions, nil, meshcatalog.RevisionInput{})
|
||||
return a.Revision != nil && a.Revision.Revision == 2, err
|
||||
}},
|
||||
"Revision of a branch": {"dev", func(ix *Indexer) (bool, error) {
|
||||
a, err := Revision(ctx, ix.Revisions, nil, "avahi", "avahi", meshcatalog.RevisionInput{Branch: "dev"})
|
||||
a, err := Revision(ctx, ix.Revisions, nil, meshcatalog.RevisionInput{Branch: "dev"})
|
||||
return a.Revision != nil && a.Revision.Revision == 2 && len(a.Entries) == 1 && a.Entries[0].Fp == "fp2", err
|
||||
}},
|
||||
}
|
||||
|
||||
@@ -260,9 +260,6 @@ func (ix *Indexer) dropRemoved(ctx context.Context, repository string, held map[
|
||||
return fmt.Errorf("modules.%s: %w", key, err)
|
||||
}
|
||||
ix.logf("modules.%s deleted: %s holds no module of that name at its trunk (issue 1900)", key, repository)
|
||||
// The bus module hears the module asks nothing now, so its entries reach orphans (design 25 § 12.2, DLV-I9).
|
||||
a, id := leftOf(key, left, ix.judgeID())
|
||||
_ = ix.sayAccepted(ctx, a, id)
|
||||
}
|
||||
for _, key := range handed {
|
||||
if err := ix.handOver(ctx, key); err != nil {
|
||||
|
||||
@@ -120,13 +120,8 @@ func (ix *Indexer) handOver(ctx context.Context, key string) error {
|
||||
|
||||
// handOverFrom writes the module at key from the merge f, which names it, in place.
|
||||
func (ix *Indexer) handOverFrom(ctx context.Context, key string, f forgemsg.Forge) error {
|
||||
w, err := ix.applyMoving(ctx, f, false, map[string]bool{key: true})
|
||||
if err != nil {
|
||||
if _, err := ix.applyMoving(ctx, f, false, map[string]bool{key: true}); err != nil {
|
||||
return fmt.Errorf("modules.%s, moving to %s: %w", key, f.Repository, err)
|
||||
}
|
||||
// A module moved in is judged as a merge's are (ADR 0578 §1, ADR 0579 §1).
|
||||
if _, err := ix.JudgeAll(ctx, w.fresh); err != nil {
|
||||
ix.logf("judge: the manifests are not judged after moving %s to %s: %v", key, f.Repository, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -10,8 +10,6 @@ const (
|
||||
ModulesState = "modules"
|
||||
ManifestsState = "manifests"
|
||||
ManifestsHistory = 1
|
||||
// VerdictsState is in verdicts.go.
|
||||
VerdictsHistory = 5
|
||||
PredefinedState = "predefined"
|
||||
PackagesState = "packages"
|
||||
PackagesHistory = 5
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
)
|
||||
|
||||
// OwnBuildState is the indexer's state of the build it runs (the bucket mesh-delivery-indexer_own-build, one key, OwnBuildKey):
|
||||
// written once, at the process's start by FixOwnBuild, so indexer_pending says which build this process knows it runs, and since
|
||||
// when. The build only names the writer of what is written (A-476 b).
|
||||
const (
|
||||
OwnBuildStateName = "own-build"
|
||||
OwnBuildKey = "state"
|
||||
)
|
||||
|
||||
// OwnBuild is the value of the own-build state.
|
||||
type OwnBuild struct {
|
||||
V int `json:"v"`
|
||||
Known bool `json:"known"`
|
||||
FP string `json:"fp,omitempty"`
|
||||
Since time.Time `json:"since"`
|
||||
Why string `json:"why,omitempty"`
|
||||
}
|
||||
|
||||
// FixOwnBuild fixes, once and at the process's start, which build of the indexer this program is: asked of the node-engine's
|
||||
// seat (OwnBuild; never the trunk's value of the indexer, which names a build that may be merged and not delivered), before
|
||||
// this process hears a merge. It is never asked again: while the node-engine replaces the indexer the seat already names the
|
||||
// successor, and an old process that learnt it later would take revisions in with its old judge. Where the start's ask is not
|
||||
// answered the build stays unknown for the life of this process, said in the own-build state and the journal; nothing waits for it.
|
||||
func (ix *Indexer) FixOwnBuild(ctx context.Context) {
|
||||
ix.ownMu.Lock()
|
||||
if ix.ownFixed || ix.OwnBuild == nil {
|
||||
ix.ownMu.Unlock()
|
||||
return
|
||||
}
|
||||
ix.ownFixed = true
|
||||
ix.ownMu.Unlock()
|
||||
fp, err := ix.OwnBuild(ctx)
|
||||
if err == nil && fp == "" {
|
||||
err = errors.New("the node-engine names no build of the indexer")
|
||||
}
|
||||
st := OwnBuild{V: 1, Since: ix.now().UTC()}
|
||||
if err != nil {
|
||||
st.Why = err.Error()
|
||||
ix.logf("judge: the build this program runs is not known (%v); merges are taken in all the same, and it is learnt only at the start", err)
|
||||
} else {
|
||||
st.Known, st.FP = true, fp
|
||||
ix.logf("judge: this program runs the indexer build %s, fixed here at its start", fp)
|
||||
}
|
||||
ix.ownMu.Lock()
|
||||
ix.ownFP, ix.ownSince, ix.ownWhy = fp, st.Since, st.Why
|
||||
ix.ownMu.Unlock()
|
||||
if ix.OwnBuildState == nil {
|
||||
return
|
||||
}
|
||||
data, _ := json.Marshal(st)
|
||||
if _, rev, found, gerr := ix.OwnBuildState.Get(ctx, OwnBuildKey); gerr == nil && found {
|
||||
err = ix.OwnBuildState.Update(ctx, OwnBuildKey, data, rev)
|
||||
} else if gerr == nil {
|
||||
err = ix.OwnBuildState.Create(ctx, OwnBuildKey, data)
|
||||
} else {
|
||||
err = gerr
|
||||
}
|
||||
if err != nil {
|
||||
ix.logf("the own-build state cannot be written: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// ownBuildIsNewest says whether the build of the indexer this program runs (ownFP, from the node-engine) is the newest the
|
||||
// trunk holds: the indexer's own modules value names it. A newer build merged and not yet delivered ends it at the trunk's
|
||||
// merge, as one that pins a changed mesh-sdk always is; false when either is unknown.
|
||||
func (ix *Indexer) ownBuildIsNewest(held map[string]meshcatalog.ModulesByModuleValue) bool {
|
||||
fp := held[ix.Module].Fp2
|
||||
if fp == "" {
|
||||
return false
|
||||
}
|
||||
ix.ownMu.Lock()
|
||||
defer ix.ownMu.Unlock()
|
||||
return ix.ownFP != "" && ix.ownFP == fp
|
||||
}
|
||||
@@ -366,15 +366,6 @@ func revisionWait(kind string, ready []PendingEntry, waitsFor string) takein.Tak
|
||||
// pendingOnBus says whether the pending state is on the bus: a state this delivery brings is bound when it appears
|
||||
// (issue 1790), and until then every merge is taken in at once, so the build this program runs is not needed. A read
|
||||
// that fails otherwise counts as on the bus: the step that follows answers the error.
|
||||
// errVerdictsNotYet is a take-in held because the verdicts state is not on the bus yet (takeInWholeBy).
|
||||
var errVerdictsNotYet = errors.New("the verdicts state is not on the bus yet: nothing is taken in until it is")
|
||||
|
||||
// verdictsOnBus says the verdicts state is on the bus (a take-in judges what it writes there).
|
||||
func (ix *Indexer) verdictsOnBus(ctx context.Context) bool {
|
||||
_, _, _, err := ix.Verdicts.Get(ctx, "on-the-bus")
|
||||
return !errors.Is(err, process.ErrNotYet)
|
||||
}
|
||||
|
||||
func (ix *Indexer) pendingOnBus(ctx context.Context) bool {
|
||||
_, _, _, err := ix.Pending.Get(ctx, takein.PendingKey(0))
|
||||
return !errors.Is(err, process.ErrNotYet)
|
||||
@@ -760,9 +751,6 @@ func (ix *Indexer) evaluate(ctx context.Context, own string, start bool) (Applie
|
||||
if errors.Is(err, errClaimLost) {
|
||||
return a, "claimed by another take-in", nil
|
||||
}
|
||||
if errors.Is(err, errVerdictsNotYet) {
|
||||
return a, "the verdicts state is not on the bus yet: the revision waits for it, taken in when it binds", nil
|
||||
}
|
||||
if errors.Is(err, ErrObjectsFull) {
|
||||
ix.holdFull(ctx, ready)
|
||||
return a, ix.waitingLine(w) + "; held: " + HeldObjectsFull, nil
|
||||
@@ -918,7 +906,7 @@ func (ix *Indexer) dry() *dryRun {
|
||||
d := &dryRun{}
|
||||
x := &Indexer{Module: ix.Module, Manifests: ix.Manifests, Packages: ix.Packages, JudgeID: ix.JudgeID, Files: ix.Files, GoSum: ix.GoSum,
|
||||
GoMod: ix.GoMod, Now: ix.Now, Builds: ix.Builds, Named: ix.Named, Trunk: ix.Trunk, BranchHead: ix.BranchHead, Read: ix.Read, Contains: ix.Contains,
|
||||
ReadOpen: nil, OwnBuild: nil, Message: ix.Message, SayRefused: nil, SayAccepted: nil, SayTakeIn: nil, Pending: nil,
|
||||
ReadOpen: nil, OwnBuild: nil, Message: ix.Message, SayTakeIn: nil, Pending: nil,
|
||||
// The registry milestone's fields: what a take-in would write on the bus stays out of a dry run, so the states of the
|
||||
// requirements, findings, seat definitions, holds and their bounds are not overlaid, nothing is said, and no hold is
|
||||
// weighed (Holds and Counted nil hold nothing); ReadFile only reads.
|
||||
@@ -932,12 +920,9 @@ func (ix *Indexer) dry() *dryRun {
|
||||
x.Store, d.candidates = over(ix.Store)
|
||||
x.Modules, d.modules = over(ix.Modules)
|
||||
x.Graph, d.graph = over(ix.Graph)
|
||||
x.Verdicts, d.verdicts = over(ix.Verdicts)
|
||||
x.Taken, d.taken = over(ix.Taken)
|
||||
x.Predefined, d.predefined = over(ix.Predefined)
|
||||
x.OldSections, d.oldSections = over(ix.OldSections)
|
||||
// What the trunk holds unjudged is read through, as fresh; what the dry run would keep stays in its overlay (issue 1966).
|
||||
x.Unjudged, _ = over(ix.Unjudged)
|
||||
ix.mu.Lock()
|
||||
x.source = sourceNow{rev: ix.source.rev, heads: maps.Clone(ix.source.heads)}
|
||||
x.previews = maps.Clone(ix.previews)
|
||||
@@ -1060,13 +1045,6 @@ func (ix *Indexer) takeInWhole(ctx context.Context, entries []PendingEntry, own,
|
||||
|
||||
// takeInWholeBy is takeInWhole; who and why, of a hand act, are carried on the take-in.taken events.
|
||||
func (ix *Indexer) takeInWholeBy(ctx context.Context, entries []PendingEntry, own, by, who, why string) (Applied, error) {
|
||||
if ix.Verdicts != nil && !ix.verdictsOnBus(ctx) && len(entries) > 0 {
|
||||
// Every manifest a take-in writes is judged in it and its verdict said before take-in.taken (issue 1966 note 154,
|
||||
// ADR 0668 §4): with the verdicts state not on the bus yet, nothing is taken in; its bind evaluates pending again.
|
||||
ix.logf("take-in: %d merges wait, oldest %s: the state %s is not on the bus yet, so nothing is taken in until it is, and no manifest is written without its verdict",
|
||||
len(entries), entries[0].Key, VerdictsState)
|
||||
return Applied{}, errVerdictsNotYet
|
||||
}
|
||||
claimed, ok := ix.claim(ctx, entries, own, by)
|
||||
if !ok {
|
||||
return Applied{}, errClaimLost
|
||||
@@ -1162,8 +1140,7 @@ func (ix *Indexer) takenWhat(ctx context.Context, claimed []PendingEntry, applie
|
||||
ix.logf("take-in: %s is written in source revision %d of %s with no fingerprint (%s): no pipeline entry", m, src, branch, e.None)
|
||||
continue
|
||||
}
|
||||
out = append(out, meshcatalog.TakeInEntry{Module: m, Commit: e.Commit, Fingerprint: e.Fp, Repository: e.Repository,
|
||||
Pull: pulls[strings.ToLower(e.Repository)]})
|
||||
out = append(out, meshcatalog.TakeInEntry{Module: m, Commit: e.Commit, Fingerprint: e.Fp, Repository: e.Repository})
|
||||
}
|
||||
return branch, src, out
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -13,7 +13,6 @@ import (
|
||||
|
||||
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
|
||||
runner "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
@@ -22,17 +21,13 @@ import (
|
||||
|
||||
// The indexer keeps the records pinning each catalog graph revision and removes a revision only on an event (design 47 G §153,
|
||||
// §178, §189, §215, §216, §579, A-169, A-171): its state pins holds one key per revision, `<branch>.<revision>` (the watched branch
|
||||
// and the revision's number, the branch escaped as a name is), whose value lists the delivery record ids that pin it, and two more
|
||||
// keys, `made` (written once, create-only, when the state is first used) and `pins-stated` (the `at` of the newest applied event of
|
||||
// that name). Pins count only once a pins-stated later than made is applied.
|
||||
// and the revision's number, the branch escaped as a name is), whose value lists the records that pin it: a take-in's and each
|
||||
// running pipeline's (design 47 H §743, §749).
|
||||
const (
|
||||
// PinsState is the state's name; PinsCapMiB is its cap in the manifest (about 1,800 keys of about 0.2 KiB).
|
||||
PinsState = "pins"
|
||||
PinsHistory = 1
|
||||
PinsCapMiB = 4
|
||||
// PinsMadeKey and PinsStatedKey are the two keys of the state that are no revision's.
|
||||
PinsMadeKey = "made"
|
||||
PinsStatedKey = "pins-stated"
|
||||
// PinMissingKind is the finding of a pinned revision the indexer no longer holds.
|
||||
PinMissingKind = "pin-missing"
|
||||
// PinsTool reads the pins, UnpinTool removes one record's pin by hand.
|
||||
@@ -46,17 +41,12 @@ type PinValue struct {
|
||||
Missing bool `json:"missing,omitempty"`
|
||||
}
|
||||
|
||||
// PinStamp is the value of the keys made and pins-stated.
|
||||
type PinStamp struct {
|
||||
At time.Time `json:"at"`
|
||||
}
|
||||
|
||||
// PinKey is a revision's key in the state pins.
|
||||
func PinKey(branch string, revision int64) string {
|
||||
return keys.EscapeName(branch) + "." + strconv.FormatInt(revision, 10)
|
||||
}
|
||||
|
||||
// ParsePinKey is the branch and the revision a key of the state pins names; false for made, pins-stated and what is no revision's key.
|
||||
// ParsePinKey is the branch and the revision a key of the state pins names; false for what is no revision's key.
|
||||
func ParsePinKey(key string) (branch string, revision int64, ok bool) {
|
||||
i := strings.LastIndex(key, ".")
|
||||
if i <= 0 {
|
||||
@@ -82,99 +72,26 @@ func pinObject(key string) (string, bool) {
|
||||
return RevisionObject(meshcatalog.CatalogRevision{Branch: branch, Revision: revision}), true
|
||||
}
|
||||
|
||||
func (ix *Indexer) pinsStamp(ctx context.Context, key string) (PinStamp, bool, error) {
|
||||
var s PinStamp
|
||||
v, _, found, err := ix.Pins.Get(ctx, key)
|
||||
if err != nil || !found {
|
||||
return s, false, err
|
||||
}
|
||||
if err := json.Unmarshal(v, &s); err != nil {
|
||||
return s, false, fmt.Errorf("%s.%s: %w", PinsState, key, err)
|
||||
}
|
||||
return s, true, nil
|
||||
}
|
||||
|
||||
// PinsMade writes the key made once, create-only, when the state is first used; a made already written stays.
|
||||
func (ix *Indexer) PinsMade(ctx context.Context) error {
|
||||
if ix.Pins == nil {
|
||||
return nil
|
||||
}
|
||||
data, err := json.Marshal(PinStamp{At: ix.now().UTC()})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ix.Pins.Create(ctx, PinsMadeKey, data); err != nil && !errors.Is(err, ErrConflict) {
|
||||
return fmt.Errorf("%s.%s: %w", PinsState, PinsMadeKey, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// pinsCounted says the pins count: a pins-stated later than made is applied.
|
||||
func (ix *Indexer) pinsCounted(ctx context.Context) (bool, error) {
|
||||
made, ok, err := ix.pinsStamp(ctx, PinsMadeKey)
|
||||
if err != nil || !ok {
|
||||
return false, err
|
||||
}
|
||||
stated, ok, err := ix.pinsStamp(ctx, PinsStatedKey)
|
||||
if err != nil || !ok {
|
||||
return false, err
|
||||
}
|
||||
return stated.At.After(made.At), nil
|
||||
}
|
||||
|
||||
// HeardPins applies one event of the seat mesh-delivery the indexer's durable consumer gave it: pinned, freed, heads-replaced or
|
||||
// pins-stated, by its subject. An error leaves the event unacknowledged, so the bus gives it again; one applied twice changes nothing.
|
||||
func (ix *Indexer) HeardPins(ctx context.Context, subject string, body []byte) error {
|
||||
// HeardPins applies one event of the seat mesh-pipeline-runner the indexer's durable consumers gave it (design 47 H §749): a
|
||||
// pipeline-started pins the revision it names under the record `pipeline.<pipeline>`, its pipeline-ended frees it. A body
|
||||
// that is no such event, or names no revision, changes nothing; an error leaves the event unacknowledged, so the bus gives it
|
||||
// again; one applied twice changes nothing.
|
||||
func (ix *Indexer) HeardPins(ctx context.Context, started *runner.PipelineStarted, ended *runner.PipelineEnded) error {
|
||||
if ix.Pins == nil {
|
||||
return errors.New("the indexer keeps no state " + PinsState)
|
||||
}
|
||||
if err := ix.PinsMade(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
switch subject {
|
||||
case meshdelivery.SubjectPinned, meshdelivery.SubjectFreed:
|
||||
var p meshdelivery.Pins
|
||||
if err := json.Unmarshal(body, &p); err != nil {
|
||||
return fmt.Errorf("%s is not a pins body: %w", subject, err)
|
||||
}
|
||||
for _, c := range p.Catalog {
|
||||
key := PinKey(c.Branch, c.Revision)
|
||||
if subject == meshdelivery.SubjectPinned {
|
||||
if err := ix.pin(ctx, key, p.Record); err != nil {
|
||||
return err
|
||||
}
|
||||
} else if err := ix.unpin(ctx, key, p.Record); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
case meshdelivery.SubjectHeadsReplaced:
|
||||
var h meshdelivery.HeadsReplacedEvent
|
||||
if err := json.Unmarshal(body, &h); err != nil {
|
||||
return fmt.Errorf("%s is not a heads-replaced body: %w", subject, err)
|
||||
}
|
||||
ix.DropCandidateGraph(ctx, h.Group, h.Heads, "mesh-delivery", "its event heads-replaced: the candidate's heads were replaced or its pull request closed")
|
||||
case meshdelivery.SubjectPinsStated:
|
||||
var s meshdelivery.PinsStatedEvent
|
||||
if err := json.Unmarshal(body, &s); err != nil {
|
||||
return fmt.Errorf("%s is not a pins-stated body: %w", subject, err)
|
||||
}
|
||||
err := write(ctx, ix.Pins, PinsStatedKey, func(old []byte, found bool) ([]byte, bool, error) {
|
||||
var cur PinStamp
|
||||
if found && json.Unmarshal(old, &cur) == nil && !s.At.After(cur.At) {
|
||||
return nil, false, nil
|
||||
}
|
||||
data, err := json.Marshal(PinStamp{At: s.At.UTC()})
|
||||
return data, false, err
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("%s is no event of the pins", subject)
|
||||
switch {
|
||||
case started != nil && started.Revision > 0 && started.Pipeline != "":
|
||||
return ix.pin(ctx, PinKey(started.Branch, started.Revision), PipelineRecord(started.Pipeline))
|
||||
case ended != nil && ended.Revision > 0 && ended.Pipeline != "":
|
||||
return ix.unpin(ctx, PinKey(ended.Branch, ended.Revision), PipelineRecord(ended.Pipeline))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// PipelineRecord is the record a pipeline pins its revision under: `pipeline.<branch>.<source>` (design 47 H §743, A-449a).
|
||||
func PipelineRecord(pipeline string) string { return "pipeline." + pipeline }
|
||||
|
||||
// pin adds a record's id to a revision's key by compare-and-set; a revision not held is marked missing and said.
|
||||
func (ix *Indexer) pin(ctx context.Context, key, record string) error {
|
||||
object, _ := pinObject(key)
|
||||
@@ -332,10 +249,6 @@ func (ix *Indexer) judgePinsLocked(ctx context.Context) ([]string, error) {
|
||||
if err != nil || pending != 0 {
|
||||
return nil, err
|
||||
}
|
||||
counted, err := ix.pinsCounted(ctx)
|
||||
if err != nil || !counted {
|
||||
return nil, err
|
||||
}
|
||||
names, err := ix.Revisions.Names(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -428,18 +341,13 @@ type PinnedRevision struct {
|
||||
type PinsAnswer struct {
|
||||
Pins []PinnedRevision `json:"pins"`
|
||||
// Unpinned are the kept revisions no record pins, current or not, and Current why each current one is.
|
||||
Unpinned []string `json:"unpinned,omitempty"`
|
||||
Made *time.Time `json:"made,omitempty"`
|
||||
Stated *time.Time `json:"pins-stated,omitempty"`
|
||||
// Counted says the pins count: a pins-stated later than made is applied.
|
||||
Counted bool `json:"counted"`
|
||||
Unpinned []string `json:"unpinned,omitempty"`
|
||||
// Fill is the state's bytes against its cap.
|
||||
Fill meshcatalog.Fill `json:"fill"`
|
||||
}
|
||||
|
||||
// ReadPins reads the state pins and the revisions kept, for indexer_pins.
|
||||
func ReadPins(ctx context.Context, pins Store, o Objects) (PinsAnswer, error) {
|
||||
ix := &Indexer{Pins: pins, Revisions: o}
|
||||
var out PinsAnswer
|
||||
names, err := o.Names(ctx)
|
||||
if err != nil {
|
||||
@@ -477,13 +385,6 @@ func ReadPins(ctx context.Context, pins Store, o Objects) (PinsAnswer, error) {
|
||||
out.Unpinned = append(out.Unpinned, n)
|
||||
}
|
||||
}
|
||||
if s, ok, err := ix.pinsStamp(ctx, PinsMadeKey); err == nil && ok {
|
||||
out.Made = &s.At
|
||||
}
|
||||
if s, ok, err := ix.pinsStamp(ctx, PinsStatedKey); err == nil && ok {
|
||||
out.Stated = &s.At
|
||||
}
|
||||
out.Counted, _ = ix.pinsCounted(ctx)
|
||||
out.Fill = meshcatalog.Fill{Bytes: bytes, Cap: PinsCapMiB << 20, Store: PinsState}
|
||||
return out, nil
|
||||
}
|
||||
@@ -495,8 +396,7 @@ func PinsTools(pins Store, o Objects, ask func(key string, value []byte) (TakeIn
|
||||
{Name: PinsTool,
|
||||
Description: "Every catalog graph revision a delivery record pins in the indexer's state pins (design 47 G §579): the revision's " +
|
||||
"key `<branch>.<revision>`, the records pinning it, whether it was missing when pinned (finding pin-missing), whether it is " +
|
||||
"kept and whether it is current and why; the kept revisions no record pins; when the state was made and when the newest " +
|
||||
"pins-stated was applied, and whether pins count yet (a pins-stated later than made); and the state's fill against its cap. " +
|
||||
"kept and whether it is current and why; the kept revisions no record pins; and the state's fill against its cap. " +
|
||||
"indexer_consumer reads the consumer the events come through; indexer_unpin removes one record's pin.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{}},
|
||||
Run: func(map[string]any) (any, error) {
|
||||
@@ -575,11 +475,10 @@ func ConsumerTools(st Store) []stdio.Tool {
|
||||
Run: func(map[string]any) (any, error) { return ReadConsumers(context.Background(), st) }}}
|
||||
}
|
||||
|
||||
// PinPipelines pins the catalog graph revision a take-in made for each pipeline the take-in starts, under the record
|
||||
// `pipeline.<key>` (`<branch>.<source>.<module>` of each entry, the runner's key), in the indexer's own pins state as a
|
||||
// pinned of the seat mesh-delivery is applied (design 47 G §579, A-449a): so no later take-in removes the revision before
|
||||
// the runner hears of it. The runner's pinned at the pipeline's start says the same record (a set), and its freed at the
|
||||
// pipeline's end removes it. It reads the branch's newest revision and pins it holding keepMu, so no other keep (a verdict, a
|
||||
// PinPipelines pins the catalog graph revision a take-in made for the one pipeline the take-in starts, under the record
|
||||
// `pipeline.<branch>.<source>` (the runner's key, design 47 H §743, A-449a), in the indexer's own pins state: so no later
|
||||
// take-in removes the revision before the runner starts it. The pipeline's pipeline-started names the same record (a set),
|
||||
// and its pipeline-ended removes it (§749). It reads the branch's newest revision and pins it holding keepMu, so no other keep (a verdict, a
|
||||
// source, an import) can make it non-current and remove it between the two; it answers that revision, 0 when none is kept.
|
||||
func (ix *Indexer) PinPipelines(ctx context.Context, branch string, source int64, entries []meshcatalog.TakeInEntry) int64 {
|
||||
ix.keepMu.Lock()
|
||||
@@ -588,15 +487,9 @@ func (ix *Indexer) PinPipelines(ctx context.Context, branch string, source int64
|
||||
if ix.Pins == nil || revision <= 0 || len(entries) == 0 {
|
||||
return revision
|
||||
}
|
||||
if err := ix.PinsMade(ctx); err != nil {
|
||||
ix.logf("pins: the revision %d of %s is not pinned for its pipelines: %v", revision, branch, err)
|
||||
return revision
|
||||
}
|
||||
for _, e := range entries {
|
||||
record := "pipeline." + runner.PipelinesByBranchRevisionModuleKey(keys.EscapeName(branch), strconv.FormatInt(source, 10), e.Module)
|
||||
if err := ix.pin(ctx, PinKey(branch, revision), record); err != nil {
|
||||
ix.logf("pins: the revision %d of %s is not pinned for %s: %v", revision, branch, record, err)
|
||||
}
|
||||
record := PipelineRecord(runner.PipelinesByBranchSourceKey(keys.EscapeName(branch), strconv.FormatInt(source, 10)))
|
||||
if err := ix.pin(ctx, PinKey(branch, revision), record); err != nil {
|
||||
ix.logf("pins: the revision %d of %s is not pinned for %s: %v", revision, branch, record, err)
|
||||
}
|
||||
return revision
|
||||
}
|
||||
|
||||
@@ -8,43 +8,34 @@ import (
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
|
||||
runner "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
)
|
||||
|
||||
// Design 47 G §579, A-169, A-171: the indexer keeps the records pinning each catalog graph revision, counts a record once, and
|
||||
// removes a revision only while its consumer is caught up and after a pins-stated later than made: one two records pin stays
|
||||
// until both free it, one neither pinned nor current goes, a revision pinned and not held is said, and unpin clears it.
|
||||
// Design 47 H §749 (G §579, A-169, A-171): the indexer pins a revision on a pipeline-started that names it and frees it on its
|
||||
// pipeline-ended, counts a record once, and removes a revision only while its consumers are caught up, judged against the pins
|
||||
// its own state holds, never a count of pins stated: one two pipelines pin stays until both free it, one neither pinned nor
|
||||
// current goes, a revision pinned and not held is said, and unpin clears it.
|
||||
func TestTheIndexerRemovesARevisionOnlyWhenNoRecordPinsItAndItIsNotCurrent(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
clock := time.Date(2026, 11, 12, 9, 0, 0, 0, time.UTC)
|
||||
pending := uint64(0)
|
||||
ix := &Indexer{Module: testSelf, Revisions: newMemObjects(1 << 20), Pins: newMem(), Findings: newFindings(), Now: func() time.Time { return clock },
|
||||
PinsPending: func(context.Context) (uint64, error) { return pending, nil }}
|
||||
reader := meshcatalog.CatalogEntry{Module: "reader", Repository: "novox/r", Manifest: map[string]any{}}
|
||||
for _, commit := range []string{"c1", "c2", "c3"} {
|
||||
e := reader
|
||||
e.Commit = commit
|
||||
if err := ix.KeepBranchEntries(ctx, "main", 5, map[string]meshcatalog.CatalogEntry{"reader": e}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
kept := func() []string {
|
||||
n, _ := ix.Revisions.Names(ctx)
|
||||
return slices.DeleteFunc(n, func(s string) bool { _, _, ok := RevisionGroup(s); return !ok })
|
||||
}
|
||||
say := func(subject string, body any) {
|
||||
started := func(pipeline string, revision int64) {
|
||||
t.Helper()
|
||||
raw, _ := json.Marshal(body)
|
||||
if err := ix.HeardPins(ctx, subject, raw); err != nil {
|
||||
if err := ix.HeardPins(ctx, &runner.PipelineStarted{Branch: "main", Pipeline: pipeline, Revision: revision}, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
pins := func(record string, revisions ...int64) meshdelivery.Pins {
|
||||
p := meshdelivery.Pins{Record: record, Milestone: "47/G"}
|
||||
for _, r := range revisions {
|
||||
p.Catalog = append(p.Catalog, meshdelivery.PinsCatalogItem{Branch: "main", Revision: r})
|
||||
ended := func(pipeline string, revision int64) {
|
||||
t.Helper()
|
||||
if err := ix.HeardPins(ctx, nil, &runner.PipelineEnded{Branch: "main", Pipeline: pipeline, Revision: revision}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return p
|
||||
}
|
||||
want := func(what string, names ...string) {
|
||||
t.Helper()
|
||||
@@ -52,34 +43,45 @@ func TestTheIndexerRemovesARevisionOnlyWhenNoRecordPinsItAndItIsNotCurrent(t *te
|
||||
t.Errorf("%s: kept %v, want %v", what, got, names)
|
||||
}
|
||||
}
|
||||
say(meshdelivery.SubjectPinned, pins("r1", 1))
|
||||
say(meshdelivery.SubjectPinned, pins("r1", 1)) // heard twice: pins once
|
||||
say(meshdelivery.SubjectPinned, pins("r2", 1))
|
||||
ix.JudgePins(ctx)
|
||||
want("before a pins-stated later than made is applied", "trunk.1", "trunk.2", "trunk.3")
|
||||
clock = clock.Add(time.Second)
|
||||
say(meshdelivery.SubjectPinsStated, meshdelivery.PinsStatedEvent{At: clock})
|
||||
pending = 3
|
||||
reader := meshcatalog.CatalogEntry{Module: "reader", Repository: "novox/r", Manifest: map[string]any{}}
|
||||
keep := func(commit string) {
|
||||
t.Helper()
|
||||
e := reader
|
||||
e.Commit = commit
|
||||
if err := ix.KeepBranchEntries(ctx, "main", 5, map[string]meshcatalog.CatalogEntry{"reader": e}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
keep("c1")
|
||||
pending = 3 // pinned before the next keeps, which judge the revisions while the consumers are behind
|
||||
started("main.5", 1)
|
||||
started("main.5", 1) // heard twice: pins once
|
||||
started("main.6", 1)
|
||||
keep("c2")
|
||||
keep("c3")
|
||||
ix.JudgePins(ctx)
|
||||
want("while messages of the consumer are pending", "trunk.1", "trunk.2", "trunk.3")
|
||||
pending = 0
|
||||
ix.JudgePins(ctx)
|
||||
want("pinned by two records, the middle one neither pinned nor current", "trunk.1", "trunk.3")
|
||||
say(meshdelivery.SubjectFreed, pins("r1", 1))
|
||||
if v, _, _, _ := ix.Pins.Get(ctx, "main.1"); !slices.Equal(mustRecords(t, v), []string{"pipeline.main.5", "pipeline.main.6"}) {
|
||||
t.Errorf("the pin of main.1 holds %s, want the two pipelines' records once each", v)
|
||||
}
|
||||
ended("main.5", 1)
|
||||
ix.JudgePins(ctx)
|
||||
want("one of two records freed it", "trunk.1", "trunk.3")
|
||||
say(meshdelivery.SubjectFreed, pins("r2", 1))
|
||||
ended("main.6", 1)
|
||||
ix.JudgePins(ctx)
|
||||
want("both freed it", "trunk.3")
|
||||
if keys, _ := ix.Pins.Keys(ctx); slices.Contains(keys, "main.1") {
|
||||
t.Errorf("the state holds %v, want no key of the removed revision", keys)
|
||||
}
|
||||
// A record pins a revision no longer held: its key says missing and the finding names it; unpin clears both.
|
||||
say(meshdelivery.SubjectPinned, pins("r3", 1))
|
||||
started("main.7", 1)
|
||||
if f := ix.Findings.Held(); len(f) != 1 || f[0].Kind != PinMissingKind {
|
||||
t.Fatalf("findings %v, want one %s", f, PinMissingKind)
|
||||
}
|
||||
if r := ix.Unpin(ctx, "main.1", "r3", "operator", "freed was lost"); !r.Stored {
|
||||
if r := ix.Unpin(ctx, "main.1", "pipeline.main.7", "operator", "freed was lost"); !r.Stored {
|
||||
t.Fatalf("unpin answered %q", r.Answer)
|
||||
}
|
||||
if f := ix.Findings.Held(); len(f) != 0 {
|
||||
@@ -87,35 +89,12 @@ func TestTheIndexerRemovesARevisionOnlyWhenNoRecordPinsItAndItIsNotCurrent(t *te
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPinsStatedBeforeTheStateWasMadeCountsNothing(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
clock := time.Date(2026, 11, 12, 9, 0, 0, 0, time.UTC)
|
||||
ix := &Indexer{Module: testSelf, Revisions: newMemObjects(1 << 20), Pins: newMem(), Now: func() time.Time { return clock },
|
||||
PinsPending: func(context.Context) (uint64, error) { return 0, nil }}
|
||||
raw, _ := json.Marshal(meshdelivery.PinsStatedEvent{At: clock.Add(-time.Hour)})
|
||||
if err := ix.HeardPins(ctx, meshdelivery.SubjectPinsStated, raw); err != nil {
|
||||
func mustRecords(t *testing.T, v []byte) []string {
|
||||
t.Helper()
|
||||
var pv PinValue
|
||||
if err := json.Unmarshal(v, &pv); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ok, err := ix.pinsCounted(ctx); err != nil || ok {
|
||||
t.Errorf("counted %v, %v after a pins-stated older than made, want not counted", ok, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHeadsReplacedDropsTheCandidatesGraph(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
ix := &Indexer{Module: testSelf, Revisions: newMemObjects(1 << 20), Pins: newMem()}
|
||||
rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Kind: "candidate", Group: "feat-x", Heads: "0123456789ab", Revision: 1}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := KeepRevision(ctx, ix.Revisions, rev, objects); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
raw, _ := json.Marshal(meshdelivery.HeadsReplacedEvent{Group: "feat-x", Heads: "0123456789ab", Repository: "novox/r", Number: 1})
|
||||
if err := ix.HeardPins(ctx, meshdelivery.SubjectHeadsReplaced, raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n, _ := ix.Revisions.Names(ctx); len(n) != 0 {
|
||||
t.Errorf("the objects hold %v, want the candidate's graph dropped", n)
|
||||
}
|
||||
slices.Sort(pv.Records)
|
||||
return pv.Records
|
||||
}
|
||||
|
||||
@@ -1,46 +0,0 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
)
|
||||
|
||||
// Design 47 G §509, §714, A-269: a pushed message's head stands where a merged message's merge stands: the pending value keeps
|
||||
// it as its ref, the newest load reads it from the bridge's answer and the pending value read back holds it alike.
|
||||
func TestAPushedMessageStandsInTheRevisionWhereAMergedMessageDoes(t *testing.T) {
|
||||
b := newTakeInBed(t)
|
||||
head := "5555555555555555555555555555555555555555"
|
||||
pushed := b.base
|
||||
pushed.Kind, pushed.Number, pushed.Merge, pushed.Base, pushed.Branch, pushed.Head = forgemsg.ForgePushed, 0, "", "", "feat/x", head
|
||||
data, err := forgemsg.EncodeForge(pushed)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.bridge[pushed.Repository+"@"+head] = data // the bridge answers by the head, where a merge's commit would stand
|
||||
|
||||
f := RevisionForm(pushed)
|
||||
if f.Kind != forgemsg.ForgeMerged || f.Merge != head {
|
||||
t.Fatalf("RevisionForm = %s %q, want a merged form at the head", f.Kind, f.Merge)
|
||||
}
|
||||
value, err := encodePending(Pending{Seq: 9, Ref: refOf(f)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, got, err := DecodePending(value)
|
||||
if err != nil || got.Merge != head {
|
||||
t.Fatalf("DecodePending of the ref = %+v, %v, want the head as its merge", got, err)
|
||||
}
|
||||
// A value an earlier indexer wrote with the pushed message whole reads alike.
|
||||
whole, _ := json.Marshal(Pending{V: 1, Seq: 9, Forge: data})
|
||||
if _, got, err = DecodePending(whole); err != nil || got.Merge != head || got.Kind != forgemsg.ForgeMerged {
|
||||
t.Fatalf("DecodePending of the whole message = %+v, %v", got, err)
|
||||
}
|
||||
// The load reads the bridge's pushed answer for the ref and accepts it.
|
||||
loaded, err := b.ix.load(context.Background(), PendingEntry{Key: "m.9", Value: Pending{Seq: 9}, Forge: RevisionForm(pushed), Light: true})
|
||||
if err != nil || loaded.Forge.Merge != head || loaded.Light {
|
||||
t.Fatalf("load = %+v, %v, want the pushed message read at the head", loaded.Forge, err)
|
||||
}
|
||||
}
|
||||
@@ -139,14 +139,9 @@ func (ix *Indexer) indexAt(ctx context.Context, r Registration, force, reread bo
|
||||
if p := read.Packages[dir]; len(p) > 0 {
|
||||
f.Packages = map[string][]string{dir: p}
|
||||
}
|
||||
w, err := ix.applyManifests(ctx, f, false)
|
||||
if err != nil {
|
||||
if _, err := ix.applyManifests(ctx, f, false); err != nil {
|
||||
return t, err
|
||||
}
|
||||
// A module registered or taken in is judged as a merge's are (ADR 0578 §1, ADR 0579 §1).
|
||||
if _, err := ix.JudgeAll(ctx, w.fresh); err != nil {
|
||||
ix.logf("judge: the manifests are not judged after taking in %s: %v", r.Module, err)
|
||||
}
|
||||
if _, _, found, err := ix.Modules.Get(ctx, key); err != nil {
|
||||
return t, fmt.Errorf("modules.%s: %w", key, err)
|
||||
} else if !found {
|
||||
|
||||
@@ -248,6 +248,9 @@ func TestSeedWritesTheModuleGraphWithNoAskAndWaitsForEveryBuildSeat(t *testing.T
|
||||
}
|
||||
// A step the seat answers none for is bypass; the statuses the check judged are the rest.
|
||||
got, _, _ := g.r.Registration(ctx, "a")
|
||||
if got.Policy.Steps.Validate != "required" { // I7/AC2, design 47 H §745
|
||||
t.Errorf("the seed wrote validate %q, want required", got.Policy.Steps.Validate)
|
||||
}
|
||||
if got.Version != 1 || got.Policy.Steps.Review != "optional" || got.Policy.Steps.Approval != "bypass" || got.Policy.Artifacts["program"].Lint != "bypass" || got.Policy.Artifacts["program"].Integration != "bypass" || got.Policy.Artifacts["program"].Unit != "" {
|
||||
t.Fatalf("the seeded registration: %+v", got)
|
||||
}
|
||||
|
||||
@@ -26,29 +26,6 @@ func twoModules(t *testing.T) (*Indexer, *memStore) {
|
||||
|
||||
func TestAMergeThatRemovesAModuleDeletesItsKey(t *testing.T) { // 1900/AC1
|
||||
ix, mods := twoModules(t)
|
||||
// DLV-I9 (design 25 § 12.2): the module leaving the trunk is said as an empty manifest-accepted, under an id naming it.
|
||||
var said []meshcatalog.ManifestAcceptedEvent
|
||||
var ids []string
|
||||
ix.SayAccepted = func(_ context.Context, a meshcatalog.ManifestAcceptedEvent, id string) error {
|
||||
said, ids = append(said, a), append(ids, id)
|
||||
return nil
|
||||
}
|
||||
defer func() {
|
||||
var left []string
|
||||
for i, a := range said {
|
||||
if a.Digest == "" {
|
||||
left = append(left, a.Module)
|
||||
// Join item 3: the id is the event's own (acceptedEventID), naming the module and the
|
||||
// modules state's revision of the value that left, so two leaves of one module are two events.
|
||||
if a.BusState == nil || len(a.BusState) != 0 || a.Revision == 0 || ids[i] != acceptedEventID(a) || !strings.Contains(ids[i], "-left-b-") {
|
||||
t.Errorf("the empty manifest-accepted = %+v, id %s", a, ids[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
if strings.Join(left, ",") != "b" {
|
||||
t.Errorf("said empty for %v, want b alone", left)
|
||||
}
|
||||
}()
|
||||
if err := ix.Apply(context.Background(), mergedMsg(catalogueRepo, map[string]string{"modules/a": "t2"})); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -102,10 +102,12 @@ func (ix *Indexer) EndRunPipeline(ctx context.Context, e meshpipelinerunner.Pipe
|
||||
ix.runMu.Lock()
|
||||
defer ix.runMu.Unlock()
|
||||
r, err := ix.updateRun(ctx, group, func(r *Run) {
|
||||
if e.Module != "" && !slices.Contains(r.Ended, e.Module) {
|
||||
r.Ended = append(r.Ended, e.Module)
|
||||
slices.Sort(r.Ended)
|
||||
for _, m := range e.Movers { // a run's pipeline-ended names its movers (H-A-72)
|
||||
if m.Module != "" && !slices.Contains(r.Ended, m.Module) {
|
||||
r.Ended = append(r.Ended, m.Module)
|
||||
}
|
||||
}
|
||||
slices.Sort(r.Ended)
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -208,8 +210,6 @@ func (ix *Indexer) writeRunCandidate(ctx context.Context, group string, members
|
||||
return err
|
||||
}
|
||||
pulls := make([]meshcatalog.CandidatesByGroupValuePullsItem, len(placed))
|
||||
var entries []PendingEntry
|
||||
over := map[string]string{}
|
||||
// One take-in per base branch: the members on one base are taken in together, so one member sees another's.
|
||||
bases := map[string]*dryRun{}
|
||||
for i, m := range placed {
|
||||
@@ -266,14 +266,7 @@ func (ix *Indexer) writeRunCandidate(ctx context.Context, group string, members
|
||||
pulls[i] = meshcatalog.CandidatesByGroupValuePullsItem{Repository: m.repository, Head: m.Commit, Modules: []string{m.Module},
|
||||
Fps: map[string]meshcatalog.CandidatesByGroupValuePullsItemFpsValue{m.Module: {Manifest: mv.Manifest, Fp2: mv.Fp2, None2: mv.None2,
|
||||
Packages: mv.Packages}}}
|
||||
over[m.Module] = mv.Manifest
|
||||
entries = append(entries, PendingEntry{Forge: forgemsg.Forge{Repository: m.repository, Merge: m.Commit, Manifests: map[string]string{m.dir: raw}}})
|
||||
}
|
||||
seats, err := ix.revisionSeats(ctx, entries)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the run %s: reading the seat definitions of its members: %w", group, err)
|
||||
}
|
||||
ix.judgeDigestsWith(ctx, seats, over, "the run "+group)
|
||||
c := meshcatalog.CandidatesByGroupValue{V: keys.Version, Group: group, Heads: hash, Pulls: pulls}
|
||||
c.Fpset = meshcatalogclient.FingerprintSet(c.Pulls)
|
||||
if err := ix.refuseDisagreement(c); err != nil {
|
||||
|
||||
@@ -46,7 +46,7 @@ func TestARunsCandidateIsBuiltOnItsStartAndDroppedWhenEveryMembersPipelineEnded(
|
||||
started := meshpipelinerunner.RunStarted{V: 1, Run: "7", By: "lead@shanks", Members: []meshpipelinerunner.Member{
|
||||
{Module: "avahi", Commit: c1, Branch: "feature/1966-x"}, {Module: "netcheck", Commit: c2, Branch: "beta"}}}
|
||||
ended := func(module string) meshpipelinerunner.PipelineEnded {
|
||||
return meshpipelinerunner.PipelineEnded{V: 1, Pipeline: "p", Module: module, Branch: "b", Commit: "c", Outcome: "passed", Run: "7"}
|
||||
return meshpipelinerunner.PipelineEnded{V: 1, Pipeline: "p", Movers: []meshpipelinerunner.MoverEnded{{Module: module}}, Branch: "b", Outcome: "passed", Run: "7"}
|
||||
}
|
||||
key, _ := meshcatalogclient.CandidatesKey("run-7")
|
||||
t.Run("built on its start, one pull per member, and dropped with the run after a restart", func(t *testing.T) {
|
||||
@@ -107,7 +107,7 @@ func TestARunsCandidateIsBuiltOnItsStartAndDroppedWhenEveryMembersPipelineEnded(
|
||||
if err := ix.StartRun(ctx, lost); !errors.Is(err, ErrUnreadable) {
|
||||
t.Errorf("a member with no registration: %v, want unreadable", err)
|
||||
}
|
||||
if err := ix.EndRunPipeline(ctx, meshpipelinerunner.PipelineEnded{Module: "avahi"}); err != nil {
|
||||
if err := ix.EndRunPipeline(ctx, meshpipelinerunner.PipelineEnded{Movers: []meshpipelinerunner.MoverEnded{{Module: "avahi"}}}); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
if k, _ := cands.Keys(ctx); len(k) != 0 {
|
||||
|
||||
@@ -75,7 +75,8 @@ func (r *Registry) ComposeSeed(modules []SeedModule) (SeedProposal, error) {
|
||||
}
|
||||
reg := meshcatalog.Registration{Module: m.Module, Origin: m.Origin, Path: m.Path, Branches: []string{b.Branch},
|
||||
Policy: meshcatalog.Policy{Steps: meshcatalog.StepStatuses{Lint: "required", Unit: "required", Integration: "required",
|
||||
Composition: "required", Review: "optional", Approval: "bypass"}}}
|
||||
Composition: "required", Review: "optional", Approval: "bypass",
|
||||
Validate: "required"}}} // design 47 H §745: the step validate is required in every policy by default
|
||||
for _, a := range m.Artifacts {
|
||||
var s meshcatalog.StepStatuses
|
||||
if !slices.Contains(a.Steps, "lint") {
|
||||
|
||||
@@ -62,7 +62,6 @@ func (ix *Indexer) ApplySource(ctx context.Context, a meshdelivery.SourceAnswer)
|
||||
for _, r := range a.Repositories {
|
||||
s.Heads[r.Repository] = r.Head
|
||||
}
|
||||
ix.kickJudge(ctx)
|
||||
ix.mu.Lock()
|
||||
if rev < ix.source.rev {
|
||||
ix.mu.Unlock()
|
||||
|
||||
@@ -21,7 +21,7 @@ type srcT struct {
|
||||
func sourceOf(repo, head string) srcT { return srcT{Group: "g", Heads: map[string]string{repo: head}} }
|
||||
|
||||
func applyS(ix *Indexer, ctx context.Context, rev uint64, s srcT) error {
|
||||
a := meshdelivery.SourceAnswer{Branch: "main", Group: s.Group, Revision: int64(rev)}
|
||||
a := meshdelivery.SourceAnswer{Branch: "main", Revision: int64(rev)}
|
||||
for r, h := range s.Heads {
|
||||
a.Repositories = append(a.Repositories, meshdelivery.SourceHead{Repository: r, Head: h, Trunk: "main"})
|
||||
}
|
||||
|
||||
@@ -28,9 +28,7 @@ type ViewWaiter interface {
|
||||
// read again once the seat answers, which judges again every manifest judged from kept definitions.
|
||||
type DefinitionsAtStart struct {
|
||||
// Read reads every contracts module's trunk definitions (Indexer.ReadTrunkDefinitions).
|
||||
Read func(ctx context.Context) (int, error)
|
||||
// Judge judges every manifest again, as at the start (Indexer.JudgeAtStart).
|
||||
Judge func(ctx context.Context) error
|
||||
Read func(ctx context.Context) (int, error)
|
||||
Waits ViewWaiter
|
||||
Bound time.Duration
|
||||
Logf func(string, ...any)
|
||||
@@ -90,19 +88,11 @@ func (d *DefinitionsAtStart) read(ctx context.Context, when string) bool {
|
||||
default:
|
||||
close(d.answered)
|
||||
}
|
||||
again := d.fromKept
|
||||
d.fromKept = false
|
||||
d.mu.Unlock()
|
||||
if was && d.Waits != nil {
|
||||
d.Waits.Holds(GitDefinitionsView)
|
||||
}
|
||||
if again && d.Judge != nil {
|
||||
if err := d.Judge(ctx); err != nil {
|
||||
d.logf("judging again the manifests judged from kept definitions: %v; judged again at the next message heard", err)
|
||||
} else {
|
||||
d.logf("judged again every manifest judged from kept definitions, now that the seat git answers")
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
|
||||
@@ -34,12 +34,6 @@ type TakeInAsk struct {
|
||||
// CatchUp asks the head the gate's source names for every repository to be taken in (CatchUp): the catch-up tool,
|
||||
// indexer_take_in called with no module and no repository, as the node agent calls it after a restore.
|
||||
CatchUp bool `json:"catch-up,omitempty"`
|
||||
// Judge asks the manifests to be judged again now (indexer_judge_again) instead of the module to be taken in.
|
||||
Judge bool `json:"judge,omitempty"`
|
||||
// Digest names, with Judge, the one manifest digest of the module to judge (indexer_judge with a digest).
|
||||
Digest string `json:"digest,omitempty"`
|
||||
// Say asks the module's manifest-accepted to be said again now (indexer_say_accepted, DLV-I9c).
|
||||
Say bool `json:"say,omitempty"`
|
||||
// SeatDefinitions asks a contracts module's seat definitions to be read again now (indexer_seat_definitions_again,
|
||||
// DLV-I19): its trunk's, or with Pull and Head a pull request's head.
|
||||
SeatDefinitions bool `json:"seat-definitions,omitempty"`
|
||||
|
||||
@@ -418,11 +418,7 @@ func (ix *Indexer) writeOverlay(ctx context.Context, overlay, finals map[string]
|
||||
return nil, false, nil
|
||||
}
|
||||
if next.Manifest != "" && (!held || cur.Manifest != next.Manifest) {
|
||||
w.taken(next.Manifest) // a new module, or a changed manifest: fresh, judged in full (ADR 0579 §1)
|
||||
// Kept on the bus before the module names it, so a judge pass that fails does not lose it (issue 1966).
|
||||
if err := ix.keepUnjudged(ctx, next.Manifest, key, w); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
w.taken(next.Manifest) // a new module, or a changed manifest
|
||||
}
|
||||
data, err := meshcatalogclient.EncodeModule(next)
|
||||
wrote = err == nil && !unchanged(data, false, value, found)
|
||||
|
||||
@@ -645,22 +645,19 @@ func TestBasesAreComputedFirstAndNoFingerprintTakesTheBuilder(t *testing.T) { //
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if got, err := Revision(context.Background(), verbObjects, nil, testSelf, "n1.node-tools", meshcatalog.RevisionInput{}); err != nil || !got.NoneYet || got.Revision != nil || len(got.Entries) != 0 {
|
||||
if got, err := Revision(context.Background(), verbObjects, nil, meshcatalog.RevisionInput{}); err != nil || !got.NoneYet || got.Revision != nil || len(got.Entries) != 0 {
|
||||
t.Errorf("revision with none kept answered %+v, %v, want none-yet alone with no error (design 47 G §530)", got, err)
|
||||
}
|
||||
keep(meshcatalog.CatalogRevision{Revision: 3, Kind: "trunk"}, map[string]bool{"node-tools": true, testSelf: true})
|
||||
keep(meshcatalog.CatalogRevision{Revision: 4, Kind: "candidate", Group: "feat/x", Heads: "0123456789ab"}, map[string]bool{"node-tools": true, "mesh-grants": true})
|
||||
for caller, answered := range map[string]bool{"n1.node-tools": true, "n1.avahi": true, "n1.mesh-grants": true, "n1." + testSelf: true, "person.someone": false, "unproven": false} {
|
||||
got, err := Revision(context.Background(), verbObjects, nil, testSelf, caller, meshcatalog.RevisionInput{})
|
||||
if (err == nil) != answered || (answered && (got.Revision.Revision != 3 || len(got.Entries) != 4 || got.Next != "")) {
|
||||
t.Errorf("revision to %s answered %+v (%v), want answered %v with trunk revision 3 whole", caller, got, err, answered)
|
||||
}
|
||||
if got, err := Revision(context.Background(), verbObjects, nil, meshcatalog.RevisionInput{}); err != nil || got.Revision.Revision != 3 || len(got.Entries) != 4 || got.Next != "" {
|
||||
t.Errorf("revision answered %+v (%v), want trunk revision 3 whole", got, err)
|
||||
}
|
||||
if got, err := Revision(context.Background(), verbObjects, nil, testSelf, "n1.node-tools", meshcatalog.RevisionInput{Group: "feat/x", Heads: "0123456789ab", After: "mesh-grants"}); err != nil ||
|
||||
if got, err := Revision(context.Background(), verbObjects, nil, meshcatalog.RevisionInput{Group: "feat/x", Heads: "0123456789ab", After: "mesh-grants"}); err != nil ||
|
||||
got.Revision.Kind != "candidate" || len(got.Entries) != 1 || got.Entries[0].Module != "node-tools" {
|
||||
t.Errorf("a candidate's revision after mesh-grants answered %+v (%v), want its one entry node-tools", got, err)
|
||||
}
|
||||
if _, err := Revision(context.Background(), verbObjects, nil, testSelf, "n1.node-tools", meshcatalog.RevisionInput{Revision: 9}); err == nil {
|
||||
if _, err := Revision(context.Background(), verbObjects, nil, meshcatalog.RevisionInput{Revision: 9}); err == nil {
|
||||
t.Error("a trunk revision not kept was answered")
|
||||
}
|
||||
// ADR 0614 §2, §3: the graph names what each module invokes and serves, and leaves out, over the whole graph, the
|
||||
|
||||
@@ -1,110 +0,0 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
)
|
||||
|
||||
// What a take-in newly wrote is kept until judged (issue 1966, ADR 0668 §4; ADR 0579 §1). A manifest a take-in writes new
|
||||
// is fresh: judged in full, never taken as accepted before. A judge pass that fails (the verdicts state not on the bus, a
|
||||
// restart between the write and the pass) used to lose that set, which lived only in the take-in's memory: the next pass
|
||||
// found no verdict, took the manifest as held before any verdict, and wrote a refusal as a warning. So the take-in writes
|
||||
// each digest into the indexer's own state unjudged before it writes the module naming it, every judge pass judges those
|
||||
// digests as fresh, and a pass deletes a digest's key once its verdict is written or no module holds it any more. The
|
||||
// tool indexer_read_unjudged reads the state; indexer_judge judges again by hand, which ends what it holds.
|
||||
const (
|
||||
// UnjudgedState is the indexer's state of the digests a take-in wrote and no judge pass judged yet: the bucket
|
||||
// mesh-delivery-indexer_unjudged, one key per manifest digest.
|
||||
UnjudgedState = "unjudged"
|
||||
)
|
||||
|
||||
// UnjudgedValue is one key of the unjudged state: the module whose new manifest the digest is, and the merge that wrote it.
|
||||
type UnjudgedValue struct {
|
||||
V int `json:"v"`
|
||||
Module string `json:"module"`
|
||||
Repository string `json:"repository,omitempty"`
|
||||
Merge string `json:"merge,omitempty"`
|
||||
}
|
||||
|
||||
// keepUnjudged writes the digest into the unjudged state before the module naming it is written. A state not on the bus
|
||||
// yet (an indexer whose delivery brings it) is said and the take-in goes on, judged from its memory alone as before;
|
||||
// any other failure fails the write, so the module is not written with its digest unkept.
|
||||
func (ix *Indexer) keepUnjudged(ctx context.Context, digest, module string, w *mergeWrite) error {
|
||||
if ix.Unjudged == nil {
|
||||
return nil
|
||||
}
|
||||
v := UnjudgedValue{V: 1, Module: module}
|
||||
if w != nil {
|
||||
v.Repository, v.Merge = w.repository, w.merge
|
||||
}
|
||||
data, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = write(ctx, ix.Unjudged, digest, func(value []byte, found bool) ([]byte, bool, error) {
|
||||
if found {
|
||||
return nil, false, nil // kept already: the first take-in that wrote it stands
|
||||
}
|
||||
return data, false, nil
|
||||
})
|
||||
if errors.Is(err, process.ErrNotYet) {
|
||||
ix.logf("judge: the state %s is not on the bus yet, so the new manifest %s of %s is kept for its judging only in this take-in", UnjudgedState, digest, module)
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s.%s: %w", UnjudgedState, digest, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// withUnjudged is fresh with every digest the unjudged state holds; a state that cannot be listed is said and fresh
|
||||
// alone is judged.
|
||||
func (ix *Indexer) withUnjudged(ctx context.Context, fresh map[string]bool) map[string]bool {
|
||||
if ix.Unjudged == nil {
|
||||
return fresh
|
||||
}
|
||||
keys, err := ix.Unjudged.Keys(ctx)
|
||||
if err != nil {
|
||||
if !errors.Is(err, process.ErrNotYet) {
|
||||
ix.logf("judge: the state %s cannot be listed (%v), so only this take-in's new manifests are judged as new", UnjudgedState, err)
|
||||
}
|
||||
return fresh
|
||||
}
|
||||
if len(keys) == 0 {
|
||||
return fresh
|
||||
}
|
||||
out := maps.Clone(fresh)
|
||||
if out == nil {
|
||||
out = map[string]bool{}
|
||||
}
|
||||
for _, k := range keys {
|
||||
out[k] = true
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// endUnjudged deletes each key of the unjudged state whose digest the pass judged with its verdict written, or which no
|
||||
// module holds any more (a later merge replaced it): what is left waits for the next pass.
|
||||
func (ix *Indexer) endUnjudged(ctx context.Context, judged, held map[string]bool) {
|
||||
if ix.Unjudged == nil {
|
||||
return
|
||||
}
|
||||
keys, err := ix.Unjudged.Keys(ctx)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
for _, k := range slices.Sorted(slices.Values(keys)) {
|
||||
if held[k] && !judged[k] {
|
||||
continue
|
||||
}
|
||||
if err := write(ctx, ix.Unjudged, k, func(_ []byte, found bool) ([]byte, bool, error) { return nil, found, nil }); err != nil {
|
||||
ix.logf("judge: %s.%s is not ended: %v; judged again as new by the next pass", UnjudgedState, k, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,807 +0,0 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"mesh-delivery-indexer/judge"
|
||||
)
|
||||
|
||||
// The indexer judges every manifest it holds, once, with its own judge (novox/hq ADR 0578 §1): it writes each verdict in
|
||||
// its state verdicts (VerdictsState), one key per manifest digest, only when the verdict differs from the one
|
||||
// held; a module's value in the modules state carries a refusal (Verdict, Problems), so a reader of modules takes it
|
||||
// without a second read; a new refusal is said as the event manifest-refused; a manifest with a field the judge does not know waits instead (ownBuildIsNewest). It judges at its start, so a judge that
|
||||
// changed (a new indexer) judges every manifest again, and after every merge it takes in, since a rule between manifests
|
||||
// can turn on any of them. Nothing polls.
|
||||
|
||||
// VerdictsState is the indexer's state of verdicts, its local name, as the tools bundle reads it.
|
||||
const VerdictsState = "verdicts"
|
||||
|
||||
// Judged is what one pass of the judge did: the manifests judged, the verdicts written, and the modules refused.
|
||||
type Judged struct {
|
||||
Judged, Written int
|
||||
Refused map[string][]string
|
||||
// Outgrown are the modules accepted before whose problems are now warnings (ADR 0579 §1).
|
||||
Outgrown []string
|
||||
// Waiting are the modules whose manifest carries a field this judge does not know and waits.
|
||||
Waiting []string
|
||||
}
|
||||
|
||||
// JudgeAll judges every manifest the modules state names, as one catalogue with the predefined list the indexer keeps,
|
||||
// and writes what differs. A manifest the manifests state does not hold is left unjudged and said; a write that fails is
|
||||
// said and the pass goes on, so one key never costs the others their verdict.
|
||||
//
|
||||
// **A stricter judge never refuses what it accepted before** (novox/hq ADR 0579 §1): a manifest whose verdict under its
|
||||
// digest is accepted, or which the indexer held before any verdict (taken as accepted, ADR 0578 §3) and is not in fresh,
|
||||
// stays accepted, the new problems written as the verdict's Warnings and carried in its module's value, which the
|
||||
// watchman raises as manifest-outgrown. A manifest in fresh (the digests a merge just took in) with no verdict yet, and
|
||||
// one already refused, is judged in full.
|
||||
func (ix *Indexer) JudgeAll(ctx context.Context, fresh map[string]bool) (Judged, error) {
|
||||
return ix.judgeOver(ctx, fresh, "", "")
|
||||
}
|
||||
|
||||
// judgeOver is JudgeAll; with a module and a manifest digest it judges that digest alone, in the catalogue the modules
|
||||
// state holds with the module's manifest replaced by it (a module the state does not hold added), and writes its verdict
|
||||
// and nothing else: no event, no count of the old sections, and no module's value but one that names the digest judged
|
||||
// (carryVerdict). A digest with no verdict yet is judged in full, as a fresh one. One digest at a time, so the known
|
||||
// packages dropped for a digest the trunk does not hold are dropped for that digest alone (review round 1, J7's note).
|
||||
func (ix *Indexer) judgeOver(ctx context.Context, fresh map[string]bool, overModule, overDigest string) (Judged, error) {
|
||||
var over map[string]string
|
||||
if overModule != "" {
|
||||
over = map[string]string{overModule: overDigest}
|
||||
}
|
||||
// A merge, the start pass and indexer_judge may all judge at once: one pass at a time, so none writes a verdict from
|
||||
// modules another is rewriting.
|
||||
ix.judgeMu.Lock()
|
||||
defer ix.judgeMu.Unlock()
|
||||
if !ix.underSeats {
|
||||
// The seat list stands still for this pass: a take-in's dry run may put its revision's seats over it (seatsof.go).
|
||||
defer judge.HoldSeats()()
|
||||
}
|
||||
return ix.judgeOverHeld(ctx, fresh, over)
|
||||
}
|
||||
|
||||
// judgeOverHeld is judgeOver for a caller that holds the judge pass (judgeMu) and keeps the seat list as it wants it
|
||||
// (judge.HoldSeats, or inside judge.WithSeats): it takes no lock, so a pass that put its seats over the list judges inside
|
||||
// it without asking the shared lock the exclusive one forbids.
|
||||
func (ix *Indexer) judgeOverHeld(ctx context.Context, fresh map[string]bool, over map[string]string) (Judged, error) {
|
||||
var out Judged
|
||||
if ix.Verdicts == nil || ix.Modules == nil || ix.Manifests == nil {
|
||||
return out, nil
|
||||
}
|
||||
if over == nil {
|
||||
ix.SayUnsaid(ctx) // a manifest-accepted a pass before failed to publish is said again first (DLV-I9b)
|
||||
// What a take-in wrote and no pass judged yet is fresh too, whichever pass judges it (unjudged.go, issue 1966).
|
||||
fresh = ix.withUnjudged(ctx, fresh)
|
||||
}
|
||||
held, err := ix.heldModules(ctx)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
heldDigests, judgedDigests := map[string]bool{}, map[string]bool{}
|
||||
for _, m := range held {
|
||||
heldDigests[m.Manifest] = m.Manifest != ""
|
||||
}
|
||||
trunkOf := map[string]string{} // a judged change's module to the trunk's manifest digest it changes (DLV-I15)
|
||||
// The known packages a manifest's `packages` may name (DLV-I14). The digest judged the trunk does not hold (a pull
|
||||
// request's candidate, one the trunk no longer names) is not judged on them: the checker judges a change's names,
|
||||
// knowing its delivery group, whose new packages this state does not know yet (the lead's ruling, issue 1966). The
|
||||
// trunk's own manifests are, and a merge's are again in full (namesAtMerge).
|
||||
known := ix.knownPackages(ctx)
|
||||
// A digest judged over the trunk that is the trunk's own held one is said as the full pass says it (manifest-accepted,
|
||||
// manifest-refused): indexer_judge with it, or the start pass's judgeStale, is that manifest's only judgement (review
|
||||
// round 1 of issue 1966; ADR 0668 §4). A candidate's verdict is said nowhere.
|
||||
onTrunk := map[string]bool{}
|
||||
if over != nil {
|
||||
fresh = map[string]bool{}
|
||||
for module, digest := range over {
|
||||
m := held[module]
|
||||
onTrunk[module] = m.Manifest == digest
|
||||
if m.Manifest != "" && m.Manifest != digest {
|
||||
trunkOf[module] = m.Manifest
|
||||
}
|
||||
if m.Manifest != digest {
|
||||
known = nil
|
||||
}
|
||||
m.Manifest = digest
|
||||
held[module] = m
|
||||
fresh[digest] = true
|
||||
}
|
||||
}
|
||||
raws := map[string][]byte{}
|
||||
digests := map[string]string{}
|
||||
for _, name := range slices.Sorted(maps.Keys(held)) {
|
||||
digest := held[name].Manifest
|
||||
if digest == "" {
|
||||
continue
|
||||
}
|
||||
value, _, found, err := ix.Manifests.Get(ctx, digest)
|
||||
if err != nil {
|
||||
return out, fmt.Errorf("reading manifests.%s of %s: %w", digest, name, err)
|
||||
}
|
||||
if !found {
|
||||
ix.logf("judge: the manifest of %s (digest %s) is not in the manifests state, so it is not judged", name, digest)
|
||||
continue
|
||||
}
|
||||
kept, err := meshcatalogclient.DecodeManifest(value)
|
||||
if err != nil {
|
||||
ix.logf("judge: manifests.%s of %s cannot be read, so it is not judged: %v", digest, name, err)
|
||||
continue
|
||||
}
|
||||
raws[name] = []byte(kept.Manifest)
|
||||
digests[name] = digest
|
||||
}
|
||||
predefined := ix.predefinedList(ctx)
|
||||
// The old sections end when no module is left in them (issue 1966 D4): the count is the accepted set the last pass held.
|
||||
var counts judge.OldSectionCounts
|
||||
if ix.OldSections != nil {
|
||||
counts = ix.heldOldSections(ctx)
|
||||
}
|
||||
refused, waits, unnamed := judge.ShelfWaiting(raws, predefined, 0, counts, known)
|
||||
// A change's renamed-from is judged against the trunk's manifest of the same module (DLV-I15, design 32).
|
||||
for module, digest := range trunkOf {
|
||||
if _, waiting := waits[module]; waiting || raws[module] == nil {
|
||||
continue
|
||||
}
|
||||
value, _, found, err := ix.Manifests.Get(ctx, digest)
|
||||
kept, derr := meshcatalogclient.DecodeManifest(value)
|
||||
if err != nil || !found || derr != nil {
|
||||
ix.logf("judge: the trunk's manifest %s of %s cannot be read, so its renames are not judged against it", digest, module)
|
||||
continue
|
||||
}
|
||||
head, herr := catalogue.Decode(raws[module])
|
||||
trunk, terr := catalogue.Decode([]byte(kept.Manifest))
|
||||
if herr != nil || terr != nil {
|
||||
continue // the manifest's own problems say why
|
||||
}
|
||||
if p := judge.RenameProblems(module, head, trunk, predefined); len(p) > 0 {
|
||||
refused[module] = append(refused[module], p...)
|
||||
}
|
||||
}
|
||||
for _, p := range unnamed {
|
||||
ix.logf("judge: a problem between the manifests that no one module's removal ends, so it refuses none: %s", p)
|
||||
}
|
||||
// A package's name belongs to one module (design 47 G §491, §676): the trunk's manifests only, since a candidate's
|
||||
// revision is validated by the validator built from its own heads (§16).
|
||||
if over == nil {
|
||||
if refused == nil {
|
||||
refused = map[string][]string{}
|
||||
}
|
||||
for module, p := range judge.PackageNameProblems(ix.packageEntries(ctx, held, raws)) {
|
||||
if _, waiting := waits[module]; !waiting {
|
||||
refused[module] = append(refused[module], p...)
|
||||
}
|
||||
}
|
||||
}
|
||||
out.Refused = refused
|
||||
// A manifest with a field this judge does not know waits for a judge that may know it (clause 3 of the record on a
|
||||
// release's Approve; ADR 0578 §1): it is refused only when this program is the newest build of the indexer the trunk
|
||||
// holds, so no judge that could know the field is still to come.
|
||||
newest := ix.ownBuildIsNewest(held)
|
||||
id := ix.judgeID()
|
||||
var accepted []catalogue.Manifest
|
||||
for _, name := range slices.Sorted(maps.Keys(digests)) {
|
||||
if _, judged := over[name]; over != nil && !judged {
|
||||
continue
|
||||
}
|
||||
out.Judged++
|
||||
v := meshcatalog.VerdictsByDigestValue{Module: name, State: meshcatalogclient.VerdictAccepted, Judge: id, At: ptr(ix.now())}
|
||||
if fields := waits[name]; len(fields) > 0 {
|
||||
accepted := ix.acceptedBefore(ctx, digests[name], fresh)
|
||||
if !newest && !accepted {
|
||||
v.State, v.Unknown = meshcatalogclient.VerdictManifestWaits, fields
|
||||
out.Waiting = append(out.Waiting, name)
|
||||
} else {
|
||||
_, problems := judge.Manifest(raws[name], ix.predefinedList(ctx))
|
||||
if accepted {
|
||||
v.Warnings = problems
|
||||
out.Outgrown = append(out.Outgrown, name)
|
||||
} else {
|
||||
v.State, v.Problems = meshcatalogclient.VerdictRefused, problems
|
||||
}
|
||||
}
|
||||
} else if problems := refused[name]; len(problems) > 0 {
|
||||
if ix.acceptedBefore(ctx, digests[name], fresh) && !namesAtMerge(name, raws[name], fresh[digests[name]] && over == nil, known) {
|
||||
v.Warnings = problems
|
||||
out.Outgrown = append(out.Outgrown, name)
|
||||
} else {
|
||||
v.State, v.Problems = meshcatalogclient.VerdictRefused, problems
|
||||
}
|
||||
}
|
||||
if !meshcatalogclient.VerdictRefuses(v) {
|
||||
if m, err := catalogue.Decode(raws[name]); err == nil {
|
||||
m.Module = name
|
||||
accepted = append(accepted, m)
|
||||
}
|
||||
}
|
||||
wrote, revision, err := ix.writeVerdict(ctx, digests[name], v)
|
||||
if errors.Is(err, process.ErrNotYet) {
|
||||
return out, fmt.Errorf("the state %s is not on the bus yet: %w", VerdictsState, err)
|
||||
}
|
||||
if err != nil {
|
||||
ix.logf("judge: the verdict on %s (digest %s) is not written: %v", name, digests[name], err)
|
||||
continue
|
||||
}
|
||||
judgedDigests[digests[name]] = true
|
||||
if wrote && over != nil && !onTrunk[name] {
|
||||
out.Written++
|
||||
ix.logf("judge: %s judged again by %s: %s", digests[name], id, v.State)
|
||||
} else if wrote {
|
||||
out.Written++
|
||||
if meshcatalogclient.VerdictWaits(v) {
|
||||
ix.logf("judge: %s", meshcatalogclient.VerdictWhy(v))
|
||||
}
|
||||
if meshcatalogclient.VerdictRefuses(v) {
|
||||
ix.logf("judge: %s", meshcatalogclient.VerdictWhy(v))
|
||||
if ix.SayRefused != nil {
|
||||
r := refusedOf(digests[name], v)
|
||||
r.Revision = int64(revision)
|
||||
ix.SayRefused(ctx, r)
|
||||
}
|
||||
} else if !meshcatalogclient.VerdictWaits(v) {
|
||||
// Every manifest newly accepted is said whole to the bus module, its entries empty when it asks for none
|
||||
// (issue 1966 D3, ADR 0595 §4, design 25 § 12.2); a manifest set aside to wait gets none (DLV-I9).
|
||||
a := acceptedOf(digests[name], v, revision, raws[name])
|
||||
_ = ix.sayAccepted(ctx, a, acceptedEventID(a))
|
||||
}
|
||||
} else if over == nil && fresh[digests[name]] && !meshcatalogclient.VerdictHeld(v) {
|
||||
// A manifest a merge just made its module's trunk manifest is newly accepted there, even when a pull request's
|
||||
// judgement wrote its verdict before the merge and said it nowhere (a candidate's verdict is not said): it is
|
||||
// said now from the verdict held, with its revision, so the bus module and the grants part hear it (design 25
|
||||
// bus-state §1; issue 1966: claude-licence-manager's new builds waited for a grant nothing wrote).
|
||||
ix.sayHeld(ctx, digests[name], raws[name])
|
||||
}
|
||||
// The reader capabilities the manifest's own words, forms and seat names need, named with the verdict (ADR 0614 §4).
|
||||
needs, _ := catalogue.ManifestNeeds(raws[name])
|
||||
if err := ix.carryVerdict(ctx, name, digests[name], v, needs); err != nil {
|
||||
ix.logf("judge: modules.%s does not carry its verdict: %v", name, err)
|
||||
}
|
||||
}
|
||||
// The verdicts carried are kept with the view (KeepView), so a restart reads them back.
|
||||
if err := ix.KeepView(ctx); err != nil {
|
||||
ix.logf("judge: the verdicts carried are not kept in a catalog graph revision: %v", err)
|
||||
}
|
||||
if over != nil {
|
||||
return out, nil
|
||||
}
|
||||
ix.endUnjudged(ctx, judgedDigests, heldDigests)
|
||||
if err := ix.keepOldSections(ctx, judge.CountOldSections(accepted), judge.CountOwnerReads(accepted)); err != nil {
|
||||
ix.logf("judge: the count of the old sections is not kept: %v", err)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// sayHeld says manifest-accepted for the trunk's manifest of digest from the accepted verdict the verdicts state holds on
|
||||
// it, with that verdict's revision; a verdict that cannot be read, or that refuses or waits, says nothing.
|
||||
func (ix *Indexer) sayHeld(ctx context.Context, digest string, raw []byte) {
|
||||
key, err := meshcatalogclient.VerdictsKey(digest)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
value, revision, found, err := ix.Verdicts.Get(ctx, key)
|
||||
if err != nil || !found {
|
||||
ix.logf("judge: the verdict on %s cannot be read, so its manifest-accepted is not said: %v", digest, err)
|
||||
return
|
||||
}
|
||||
v, err := meshcatalogclient.DecodeVerdict(value)
|
||||
if err != nil || meshcatalogclient.VerdictHeld(v) {
|
||||
return
|
||||
}
|
||||
a := acceptedOf(digest, v, revision, raw)
|
||||
_ = ix.sayAccepted(ctx, a, acceptedEventID(a))
|
||||
}
|
||||
|
||||
// JudgeAtStart is JudgeAll of every manifest held, none fresh, as the indexer starts (a judge that changed judges every
|
||||
// one again); one that fails is judged again at the next message or source revision heard (kickJudge). Then every other
|
||||
// digest of the verdicts state that another judge refused or held waiting (one of a candidate, or one back on the trunk
|
||||
// later) is judged again by this judge (judgeStale), so a refusal an older judge made never stands for good.
|
||||
func (ix *Indexer) JudgeAtStart(ctx context.Context) (Judged, error) {
|
||||
j, err := ix.JudgeAll(ctx, nil)
|
||||
ix.judgePending.Store(err != nil)
|
||||
if err == nil {
|
||||
ix.judgeStale(ctx)
|
||||
}
|
||||
return j, err
|
||||
}
|
||||
|
||||
// judgeID is the judge a verdict of this program names.
|
||||
func (ix *Indexer) judgeID() string {
|
||||
if ix.JudgeID != "" {
|
||||
return ix.JudgeID
|
||||
}
|
||||
return judge.Identity()
|
||||
}
|
||||
|
||||
// staleVerdict says whether the digest's verdict wants this judge's judgement: there is none, or another judge refused
|
||||
// it or held it waiting. An accepted verdict stays accepted under any later judge (ADR 0579 §1), so judging it again
|
||||
// changes nothing.
|
||||
func (ix *Indexer) staleVerdict(ctx context.Context, digest string) bool {
|
||||
value, _, found, err := ix.Verdicts.Get(ctx, digest)
|
||||
if err != nil {
|
||||
return false // unread: left, said by the next pass that reads it
|
||||
}
|
||||
if !found {
|
||||
return true
|
||||
}
|
||||
v, err := meshcatalogclient.DecodeVerdict(value)
|
||||
return err != nil || (meshcatalogclient.VerdictHeld(v) && v.Judge != ix.judgeID())
|
||||
}
|
||||
|
||||
// judgeDigests judges each digest of over (module to digest) whose verdict is stale (staleVerdict), one at a time in the
|
||||
// catalogue the modules state holds; a pass that fails is said and the others go on.
|
||||
func (ix *Indexer) judgeDigests(ctx context.Context, over map[string]string, why string) {
|
||||
if ix.Verdicts == nil || ix.Manifests == nil || ix.Modules == nil {
|
||||
return
|
||||
}
|
||||
for _, module := range slices.Sorted(maps.Keys(over)) {
|
||||
if !ix.staleVerdict(ctx, over[module]) {
|
||||
continue
|
||||
}
|
||||
if _, err := ix.judgeOver(ctx, nil, module, over[module]); err != nil {
|
||||
ix.logf("judge: the manifest %s of %s (%s) is not judged: %v", over[module], module, why, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// judgeDigestsWith is judgeDigests with the seats put over the catalogue's list for the whole pass (judge.WithSeats): one
|
||||
// locked pass that takes the seats, and never a second lock under it (the exclusive one of WithSeats and the shared one of
|
||||
// judgeOver would wait for each other for good).
|
||||
func (ix *Indexer) judgeDigestsWith(ctx context.Context, seats []catalogue.Seat, over map[string]string, why string) {
|
||||
if ix.Verdicts == nil || ix.Manifests == nil || ix.Modules == nil {
|
||||
return
|
||||
}
|
||||
ix.judgeMu.Lock()
|
||||
defer ix.judgeMu.Unlock()
|
||||
judge.WithSeats(seats, func() {
|
||||
for _, module := range slices.Sorted(maps.Keys(over)) {
|
||||
if !ix.staleVerdict(ctx, over[module]) {
|
||||
continue
|
||||
}
|
||||
if _, err := ix.judgeOverHeld(ctx, nil, map[string]string{module: over[module]}); err != nil {
|
||||
ix.logf("judge: the manifest %s of %s (%s) is not judged: %v", over[module], module, why, err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// judgeStale judges again every digest of the verdicts state another judge refused or held waiting, each as a manifest
|
||||
// of the module its verdict names (judgeDigests). Never on a timer: at the start pass.
|
||||
func (ix *Indexer) judgeStale(ctx context.Context) {
|
||||
if ix.Verdicts == nil {
|
||||
return
|
||||
}
|
||||
keys, err := ix.Verdicts.Keys(ctx)
|
||||
if err != nil {
|
||||
ix.logf("judge: the verdicts other judges made are not judged again: listing the verdicts: %v", err)
|
||||
return
|
||||
}
|
||||
byModule := map[string][]string{}
|
||||
for _, digest := range keys {
|
||||
value, _, found, err := ix.Verdicts.Get(ctx, digest)
|
||||
if err != nil || !found {
|
||||
continue
|
||||
}
|
||||
if v, err := meshcatalogclient.DecodeVerdict(value); err == nil && meshcatalogclient.VerdictHeld(v) && v.Judge != ix.judgeID() {
|
||||
byModule[v.Module] = append(byModule[v.Module], digest)
|
||||
}
|
||||
}
|
||||
for _, module := range slices.Sorted(maps.Keys(byModule)) {
|
||||
for _, digest := range byModule[module] {
|
||||
ix.judgeDigests(ctx, map[string]string{module: digest}, "made by another judge")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// namesAtMerge says whether a manifest a merge takes in (atMerge) names a package known does not: it is refused for it even
|
||||
// when its digest was accepted before, since that verdict was its candidate's, judged without the names (judgeOver), so
|
||||
// no stricter judge turns an acceptance into a refusal here (ADR 0579 §1).
|
||||
func namesAtMerge(module string, raw []byte, atMerge bool, known judge.KnownPackages) bool {
|
||||
if !atMerge || known == nil {
|
||||
return false
|
||||
}
|
||||
m, err := catalogue.Decode(raw)
|
||||
return err == nil && len(judge.UnknownPackageNames(module, m, known)) > 0
|
||||
}
|
||||
|
||||
// knownPackages are the packages a manifest's `packages` may name (DLV-I14, design 32): every name the packages state's
|
||||
// series keys hold, as KeepPackageNames reads it at each merge, and the Go module path of every per-module record the
|
||||
// state holds from the trunk: while no build module builds Go package artifacts, a Go module the trunk holds a record of
|
||||
// has no series and is known by its module path, never by a package inside it (design 32; issue 1966). Nil, which judges
|
||||
// no name, where the indexer keeps no packages state or it cannot be listed (said): a state the judge cannot read refuses no manifest.
|
||||
func (ix *Indexer) knownPackages(ctx context.Context) judge.KnownPackages {
|
||||
if ix.Packages == nil {
|
||||
return nil
|
||||
}
|
||||
keys, err := ix.Packages.Keys(ctx)
|
||||
if err != nil {
|
||||
ix.logf("judge: the packages state cannot be listed, so the names of packages are not judged: %v", err)
|
||||
return nil
|
||||
}
|
||||
known := judge.KnownPackages{}
|
||||
for _, key := range keys {
|
||||
value, _, found, err := ix.Packages.Get(ctx, key)
|
||||
if err != nil {
|
||||
ix.logf("judge: packages.%s cannot be read, so the names of packages are not judged: %v", key, err)
|
||||
return nil
|
||||
}
|
||||
if !IsSeriesKey(key) {
|
||||
if r, err := DecodePublishedPackages(value); found && err == nil && r.Module != "" {
|
||||
known[r.Module] = true
|
||||
}
|
||||
continue
|
||||
}
|
||||
if s, err := DecodeSeriesPackage(value); found && err == nil {
|
||||
known[s.Name] = true
|
||||
}
|
||||
}
|
||||
return known
|
||||
}
|
||||
|
||||
// kickJudge judges again a start pass that failed, when the indexer hears something: never on a timer.
|
||||
func (ix *Indexer) kickJudge(ctx context.Context) {
|
||||
if !ix.judgePending.Load() {
|
||||
return
|
||||
}
|
||||
if j, err := ix.JudgeAtStart(ctx); err != nil {
|
||||
ix.logf("judge: the manifests are still not judged: %v", err)
|
||||
} else {
|
||||
ix.logf("judge: judged %d manifests again after a failed start: %d verdicts written", j.Judged, j.Written)
|
||||
}
|
||||
}
|
||||
|
||||
// acceptedBefore says whether the manifest of digest was accepted before this judgement: its verdict says accepted, or
|
||||
// it has none and is not fresh (held before any verdict, so taken in under the judges that ran until then).
|
||||
func (ix *Indexer) acceptedBefore(ctx context.Context, digest string, fresh map[string]bool) bool {
|
||||
value, _, found, err := ix.Verdicts.Get(ctx, digest)
|
||||
if err != nil {
|
||||
return false // unread: judged in full, said by the write that follows
|
||||
}
|
||||
if !found {
|
||||
return !fresh[digest]
|
||||
}
|
||||
prev, err := meshcatalogclient.DecodeVerdict(value)
|
||||
return err == nil && prev.State == meshcatalogclient.VerdictAccepted
|
||||
}
|
||||
|
||||
// writeVerdict writes v under the digest's key unless the verdict held there says the same (ManifestVerdict.Same); wrote
|
||||
// says it was written, and revision is the verdicts state's revision of that write, which the event saying it carries
|
||||
// (novox/hq ADR 0622 §2). It is read back after the write: when another write came between, the revision read holds
|
||||
// another verdict and revision is 0, so a reader takes the later write's own event and never pairs v with its revision.
|
||||
func (ix *Indexer) writeVerdict(ctx context.Context, digest string, v meshcatalog.VerdictsByDigestValue) (wrote bool, revision uint64, err error) {
|
||||
key, err := meshcatalogclient.VerdictsKey(digest)
|
||||
if err != nil {
|
||||
return false, 0, err
|
||||
}
|
||||
err = write(ctx, ix.Verdicts, key, func(value []byte, found bool) ([]byte, bool, error) {
|
||||
wrote = false
|
||||
if found {
|
||||
if cur, err := meshcatalogclient.DecodeVerdict(value); err == nil && meshcatalogclient.SameVerdict(cur, v) {
|
||||
return nil, false, nil
|
||||
}
|
||||
}
|
||||
wrote = true
|
||||
next, err := meshcatalogclient.EncodeVerdict(v)
|
||||
return next, false, err
|
||||
})
|
||||
if err != nil || !wrote {
|
||||
return wrote, 0, err
|
||||
}
|
||||
if value, rev, found, err := ix.Verdicts.Get(ctx, key); err == nil && found {
|
||||
if cur, err := meshcatalogclient.DecodeVerdict(value); err == nil && meshcatalogclient.SameVerdict(cur, v) {
|
||||
revision = rev
|
||||
}
|
||||
}
|
||||
return wrote, revision, nil
|
||||
}
|
||||
|
||||
// OwnBuildState is the indexer's state of the build it runs (the bucket mesh-delivery-indexer_own-build, one key, OwnBuildKey):
|
||||
// written once, at the process's start by FixOwnBuild, so indexer_pending says which build this process knows it runs, and since
|
||||
// when. The build only names the writer of what is written (A-476 b).
|
||||
const (
|
||||
OwnBuildStateName = "own-build"
|
||||
OwnBuildKey = "state"
|
||||
)
|
||||
|
||||
// OwnBuild is the value of the own-build state.
|
||||
type OwnBuild struct {
|
||||
V int `json:"v"`
|
||||
Known bool `json:"known"`
|
||||
FP string `json:"fp,omitempty"`
|
||||
Since time.Time `json:"since"`
|
||||
Why string `json:"why,omitempty"`
|
||||
}
|
||||
|
||||
// FixOwnBuild fixes, once and at the process's start, which build of the indexer this program is: asked of the node-engine's
|
||||
// seat (OwnBuild; never the trunk's value of the indexer, which names a build that may be merged and not delivered), before
|
||||
// this process hears a merge. It is never asked again: while the node-engine replaces the indexer the seat already names the
|
||||
// successor, and an old process that learnt it later would take revisions in with its old judge. Where the start's ask is not
|
||||
// answered the build stays unknown for the life of this process, said in the own-build state and the journal; nothing waits for it.
|
||||
func (ix *Indexer) FixOwnBuild(ctx context.Context) {
|
||||
ix.ownMu.Lock()
|
||||
if ix.ownFixed || ix.OwnBuild == nil {
|
||||
ix.ownMu.Unlock()
|
||||
return
|
||||
}
|
||||
ix.ownFixed = true
|
||||
ix.ownMu.Unlock()
|
||||
fp, err := ix.OwnBuild(ctx)
|
||||
if err == nil && fp == "" {
|
||||
err = errors.New("the node-engine names no build of the indexer")
|
||||
}
|
||||
st := OwnBuild{V: 1, Since: ix.now().UTC()}
|
||||
if err != nil {
|
||||
st.Why = err.Error()
|
||||
ix.logf("judge: the build this program runs is not known (%v); merges are taken in all the same, and it is learnt only at the start", err)
|
||||
} else {
|
||||
st.Known, st.FP = true, fp
|
||||
ix.logf("judge: this program runs the indexer build %s, fixed here at its start", fp)
|
||||
}
|
||||
ix.ownMu.Lock()
|
||||
ix.ownFP, ix.ownSince, ix.ownWhy = fp, st.Since, st.Why
|
||||
ix.ownMu.Unlock()
|
||||
if ix.OwnBuildState == nil {
|
||||
return
|
||||
}
|
||||
data, _ := json.Marshal(st)
|
||||
if _, rev, found, gerr := ix.OwnBuildState.Get(ctx, OwnBuildKey); gerr == nil && found {
|
||||
err = ix.OwnBuildState.Update(ctx, OwnBuildKey, data, rev)
|
||||
} else if gerr == nil {
|
||||
err = ix.OwnBuildState.Create(ctx, OwnBuildKey, data)
|
||||
} else {
|
||||
err = gerr
|
||||
}
|
||||
if err != nil {
|
||||
ix.logf("the own-build state cannot be written: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// ownBuildIsNewest says whether the build of the indexer this program runs (ownFP, from the node-engine) is the newest the
|
||||
// trunk holds: the indexer's own modules value names it. A newer build merged and not yet delivered ends it at the trunk's
|
||||
// merge, as one that pins a changed mesh-sdk always is; false when either is unknown.
|
||||
func (ix *Indexer) ownBuildIsNewest(held map[string]meshcatalog.ModulesByModuleValue) bool {
|
||||
fp := held[ix.Module].Fp2
|
||||
if fp == "" {
|
||||
return false
|
||||
}
|
||||
ix.ownMu.Lock()
|
||||
defer ix.ownMu.Unlock()
|
||||
return ix.ownFP != "" && ix.ownFP == fp
|
||||
}
|
||||
|
||||
// carryVerdict puts a refusal into the module's value, and takes one out once the manifest is accepted, while the value
|
||||
// still names the digest judged, and the reader capabilities the manifest needs beside it (needs, ADR 0614 §4); an
|
||||
// accepted module's value carries no verdict, so judging writes no value of a module the
|
||||
// judge accepts (a reader takes a value without a verdict as accepted, meshcatalogclient.TakeVerdict).
|
||||
func (ix *Indexer) carryVerdict(ctx context.Context, name, digest string, v meshcatalog.VerdictsByDigestValue, needs []string) error {
|
||||
key, err := meshcatalogclient.ModulesKey(name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return write(ctx, ix.Modules, key, func(value []byte, found bool) ([]byte, bool, error) {
|
||||
if !found {
|
||||
return nil, false, nil
|
||||
}
|
||||
cur, err := meshcatalogclient.DecodeModule(value)
|
||||
if err != nil || cur.Manifest != digest {
|
||||
return nil, false, nil // gone, unreadable, or moved to another manifest since: the next pass judges it
|
||||
}
|
||||
// A wait is written as a refusal naming the fields, so a reader pinned before the verdict waits holds it as refused.
|
||||
state, problems := "", v.Problems
|
||||
switch {
|
||||
case meshcatalogclient.VerdictWaits(v):
|
||||
state, problems = meshcatalogclient.VerdictRefused, []string{meshcatalogclient.WaitsOn(v)}
|
||||
case meshcatalogclient.VerdictRefuses(v):
|
||||
state = meshcatalogclient.VerdictRefused
|
||||
}
|
||||
// The judge reads the manifest alone; the forms its templates' text writes came with its merge (ADR 0632 §2) and
|
||||
// stand, cur being of the same manifest.
|
||||
needs := withTemplateForms(needs, cur.Needs)
|
||||
if cur.Verdict == state && slices.Equal(cur.Problems, problems) && slices.Equal(cur.Unknown, v.Unknown) && slices.Equal(cur.Warnings, v.Warnings) &&
|
||||
slices.Equal(cur.Needs, needs) {
|
||||
return nil, false, nil
|
||||
}
|
||||
cur.Verdict, cur.Problems, cur.Unknown, cur.Warnings, cur.Needs = state, problems, v.Unknown, v.Warnings, needs
|
||||
if state == "" {
|
||||
cur.Problems = nil
|
||||
}
|
||||
next, err := meshcatalogclient.EncodeModule(cur)
|
||||
return next, false, err
|
||||
})
|
||||
}
|
||||
|
||||
// predefinedList is the catalogue's predefined list the indexer keeps (writePredefined), or nil when it keeps none or
|
||||
// it cannot be read: the judge then takes each fallback as its value declares it.
|
||||
func (ix *Indexer) predefinedList(ctx context.Context) catalogue.PredefinedList {
|
||||
if ix.Predefined == nil {
|
||||
return nil
|
||||
}
|
||||
value, _, found, err := ix.Predefined.Get(ctx, PredefinedListKey)
|
||||
if err != nil || !found {
|
||||
return nil
|
||||
}
|
||||
var kept struct {
|
||||
List json.RawMessage `json:"list"`
|
||||
}
|
||||
if json.Unmarshal(value, &kept) != nil {
|
||||
return nil
|
||||
}
|
||||
list, err := catalogue.ParsePredefined(kept.List)
|
||||
if err != nil {
|
||||
ix.logf("judge: the predefined list kept cannot be read, so fallbacks are judged as their values declare them: %v", err)
|
||||
return nil
|
||||
}
|
||||
return list
|
||||
}
|
||||
|
||||
// The tools that read and steer the verdicts (the instruction "tools are how a module is spoken to"): indexer_verdicts
|
||||
// lists the manifests that wait or are refused, and indexer_judge judges one module again now.
|
||||
const (
|
||||
VerdictsTool = "indexer_verdicts"
|
||||
JudgeAgainTool = "indexer_judge"
|
||||
judgeAgainPrefix = "judge."
|
||||
)
|
||||
|
||||
// VerdictRow is one verdict as indexer_verdicts lists it: the manifest's digest, the module, its state, the fields it waits
|
||||
// on or the problems that refuse it, and the judge that made it.
|
||||
type VerdictRow struct {
|
||||
Module string `json:"module"`
|
||||
Digest string `json:"digest"`
|
||||
State string `json:"state"`
|
||||
Words string `json:"words,omitempty"`
|
||||
Unknown []string `json:"unknown,omitempty"`
|
||||
Problems []string `json:"problems,omitempty"`
|
||||
Warnings []string `json:"warnings,omitempty"`
|
||||
Judge string `json:"judge"`
|
||||
At time.Time `json:"at"`
|
||||
}
|
||||
|
||||
// ListVerdicts reads the verdicts under keys with get and lists those of module (every module when empty) that wait or are
|
||||
// refused, or every one with all, sorted by module and digest. A value that does not read is said in unread by its key.
|
||||
func ListVerdicts(keys []string, get func(key string) ([]byte, error), module string, all bool) (rows []VerdictRow, unread map[string]string) {
|
||||
unread = map[string]string{}
|
||||
for _, key := range keys {
|
||||
value, err := get(key)
|
||||
if err != nil || value == nil {
|
||||
if err != nil {
|
||||
unread[key] = err.Error()
|
||||
}
|
||||
continue
|
||||
}
|
||||
v, err := meshcatalogclient.DecodeVerdict(value)
|
||||
if err != nil {
|
||||
unread[key] = err.Error()
|
||||
continue
|
||||
}
|
||||
if (module != "" && v.Module != module) || (!all && !meshcatalogclient.VerdictHeld(v)) {
|
||||
continue
|
||||
}
|
||||
rows = append(rows, VerdictRow{Module: v.Module, Digest: key, State: v.State, Words: meshcatalogclient.VerdictWhy(v), Unknown: v.Unknown,
|
||||
Problems: v.Problems, Warnings: v.Warnings, Judge: v.Judge, At: timeOf(v.At)})
|
||||
}
|
||||
slices.SortFunc(rows, func(a, b VerdictRow) int {
|
||||
if c := strings.Compare(a.Module, b.Module); c != 0 {
|
||||
return c
|
||||
}
|
||||
return strings.Compare(a.Digest, b.Digest)
|
||||
})
|
||||
return rows, unread
|
||||
}
|
||||
|
||||
// JudgeAgain judges the manifests again now, as at the start, and answers how the module's stands: the tool
|
||||
// indexer_judge. A module the modules state does not hold, or whose manifest is not in the manifests state, is
|
||||
// answered so. With a digest it judges that manifest alone, in the catalogue the trunk holds (judgeOver), whoever made
|
||||
// its verdict: a candidate's, or one the trunk names again, refused by an older judge.
|
||||
func (ix *Indexer) JudgeAgain(ctx context.Context, module, digest string) TakenIn {
|
||||
out := TakenIn{Module: module}
|
||||
held, err := ix.heldModules(ctx)
|
||||
if err != nil {
|
||||
out.Answer = "the modules state cannot be read: " + err.Error()
|
||||
return out
|
||||
}
|
||||
if digest != "" {
|
||||
// One digest (a candidate's, or one the trunk no longer names), judged by this judge in the catalogue the trunk holds.
|
||||
value, _, found, err := ix.Manifests.Get(ctx, digest)
|
||||
kept, derr := meshcatalogclient.DecodeManifest(value)
|
||||
switch {
|
||||
case err != nil || !found:
|
||||
out.Answer = "the indexer keeps no manifest of digest " + digest
|
||||
return out
|
||||
case derr != nil:
|
||||
out.Answer = "the manifest of digest " + digest + " cannot be read: " + derr.Error()
|
||||
return out
|
||||
case module != "" && kept.Module != module:
|
||||
out.Answer = "the manifest of digest " + digest + " is " + kept.Module + "'s, not " + module + "'s"
|
||||
return out
|
||||
}
|
||||
out.Module = kept.Module
|
||||
if _, err := ix.judgeOver(ctx, nil, kept.Module, digest); err != nil {
|
||||
out.Answer = "the manifest could not be judged: " + err.Error()
|
||||
return out
|
||||
}
|
||||
return ix.answerVerdict(ctx, out, digest)
|
||||
}
|
||||
m, ok := held[module]
|
||||
if !ok || m.Manifest == "" {
|
||||
out.Answer = "the indexer holds no manifest of " + module
|
||||
return out
|
||||
}
|
||||
if _, err := ix.JudgeAll(ctx, nil); err != nil {
|
||||
out.Answer = "the manifests could not be judged: " + err.Error()
|
||||
return out
|
||||
}
|
||||
return ix.answerVerdict(ctx, out, m.Manifest)
|
||||
}
|
||||
|
||||
// answerVerdict is out answering how the verdict under digest stands.
|
||||
func (ix *Indexer) answerVerdict(ctx context.Context, out TakenIn, digest string) TakenIn {
|
||||
module := out.Module
|
||||
value, _, found, err := ix.Verdicts.Get(ctx, digest)
|
||||
if err != nil || !found {
|
||||
out.Answer = "judged, but the verdict on " + module + " cannot be read"
|
||||
return out
|
||||
}
|
||||
v, err := meshcatalogclient.DecodeVerdict(value)
|
||||
if err != nil {
|
||||
out.Answer = "judged, but the verdict on " + module + " cannot be read: " + err.Error()
|
||||
return out
|
||||
}
|
||||
out.Stored = true
|
||||
out.Answer = "judged again by " + v.Judge + ": " + v.State
|
||||
if meshcatalogclient.VerdictHeld(v) {
|
||||
out.Answer += " (" + meshcatalogclient.VerdictWhy(v) + ")"
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// NewJudgeAgain is the value indexer_judge writes for a module, and one digest of its manifest when given: its key in
|
||||
// the takeins state and the value.
|
||||
func NewJudgeAgain(module, digest, why, by string, at time.Time) (string, []byte, error) {
|
||||
if digest != "" {
|
||||
if _, err := meshcatalogclient.ManifestsKey(digest); err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
}
|
||||
key, data, err := NewTakeIn(module, why, by, at)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
var a TakeInAsk
|
||||
if err := json.Unmarshal(data, &a); err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
a.Judge, a.Digest = true, digest
|
||||
data, err = json.Marshal(a)
|
||||
return judgeAgainPrefix + key, data, err
|
||||
}
|
||||
|
||||
// withTemplateForms is needs with the reference forms of had added: a module's templates write forms only (catalogue
|
||||
// ModuleNeeds), and a merge carried them with the manifest the judge reads again alone.
|
||||
func withTemplateForms(needs, had []string) []string {
|
||||
out := slices.Clone(needs)
|
||||
for _, n := range had {
|
||||
if strings.HasPrefix(n, catalogue.ReadForm) && !slices.Contains(out, n) {
|
||||
out = append(out, n)
|
||||
}
|
||||
}
|
||||
slices.Sort(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// timeOf is the time a pointer holds, the zero time for none.
|
||||
func timeOf(t *time.Time) time.Time {
|
||||
if t == nil {
|
||||
return time.Time{}
|
||||
}
|
||||
return *t
|
||||
}
|
||||
@@ -1,612 +0,0 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"mesh-delivery-indexer/judge"
|
||||
)
|
||||
|
||||
// The indexer judges every manifest it holds and writes each verdict only when it differs; a module's value carries a
|
||||
// refusal, which is said once as manifest-refused; a new judge writes again only what it judges otherwise; a manifest
|
||||
// mended clears the module's refusal (novox/hq ADR 0578 §1).
|
||||
func TestTheIndexerJudgesEveryManifestOnce(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
ix := &Indexer{Module: testSelf, Modules: newMem(), Manifests: newMem(), Verdicts: newMem(), JudgeID: "judge-1",
|
||||
Now: func() time.Time { return time.Date(2026, 11, 1, 3, 0, 0, 0, time.UTC) }}
|
||||
var said []meshcatalog.ManifestRefusedEvent
|
||||
ix.SayRefused = func(_ context.Context, r meshcatalog.ManifestRefusedEvent) { said = append(said, r) }
|
||||
hold := func(module, raw string) string {
|
||||
digest, err := ix.keepManifest(ctx, module, raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
value, err := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: "src2:" + strings.Repeat("a", 64), Manifest: digest,
|
||||
Repository: "novox/mesh-catalog", Commit: "c"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ix.Modules.(*memStore).put(module, value)
|
||||
return digest
|
||||
}
|
||||
good := hold("good", `{"module":"good","version":"1.0"}`)
|
||||
bad := hold("bad", `{"module":"bad","version":"1.0","requires":["amqp"]}`)
|
||||
// Both taken in by a merge: judged in full.
|
||||
j, err := ix.JudgeAll(ctx, map[string]bool{good: true, bad: true})
|
||||
if err != nil || j.Judged != 2 || j.Written != 2 || len(j.Refused) != 1 {
|
||||
t.Fatalf("the first pass: %+v %v", j, err)
|
||||
}
|
||||
verdict := func(digest string) meshcatalog.VerdictsByDigestValue {
|
||||
value, _, found, _ := ix.Verdicts.Get(ctx, digest)
|
||||
v, err := meshcatalogclient.DecodeVerdict(value)
|
||||
if !found || err != nil {
|
||||
t.Fatalf("no verdict under %s: %v", digest, err)
|
||||
}
|
||||
return v
|
||||
}
|
||||
if v := verdict(good); meshcatalogclient.VerdictRefuses(v) || v.Judge != "judge-1" {
|
||||
t.Fatalf("good: %+v", v)
|
||||
}
|
||||
if v := verdict(bad); !meshcatalogclient.VerdictRefuses(v) || !strings.Contains(v.Problems[0], `requires "amqp"`) {
|
||||
t.Fatalf("bad: %+v", v)
|
||||
}
|
||||
module := func(name string) meshcatalog.ModulesByModuleValue {
|
||||
value, _, _, _ := ix.Modules.Get(ctx, name)
|
||||
m, err := meshcatalogclient.DecodeModule(value)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return m
|
||||
}
|
||||
if m := module("bad"); m.Verdict != meshcatalogclient.VerdictRefused || len(m.Problems) != 1 {
|
||||
t.Fatalf("modules.bad carries %+v", m)
|
||||
}
|
||||
if m := module("good"); m.Verdict != "" {
|
||||
t.Fatalf("modules.good carries %q", m.Verdict)
|
||||
}
|
||||
// ADR 0622 §2: the event carries the verdicts state's revision of the verdict it says.
|
||||
if _, rev, _, _ := ix.Verdicts.Get(ctx, bad); len(said) != 1 || said[0].Module != "bad" || said[0].Digest != bad ||
|
||||
said[0].Judge != "judge-1" || said[0].Revision == 0 || said[0].Revision != int64(rev) {
|
||||
t.Fatalf("said %+v (the verdict's revision %d)", said, rev)
|
||||
}
|
||||
// A new judge that judges alike writes nothing and says nothing again.
|
||||
ix.JudgeID = "judge-2"
|
||||
if j, err := ix.JudgeAll(ctx, nil); err != nil || j.Written != 0 || len(said) != 1 {
|
||||
t.Fatalf("a judge that judges alike: %+v %v, said %d", j, err, len(said))
|
||||
}
|
||||
// ADR 0603 §4 (D9): a source revision that changes the indexer builds the new indexer, which at its start judges every
|
||||
// manifest it holds again, none of them fresh: all of them judged, nothing written for a verdict that stands.
|
||||
if j, err := ix.JudgeAtStart(ctx); err != nil || j.Judged != 2 || j.Written != 0 || len(said) != 1 {
|
||||
t.Fatalf("the new indexer at its start: %+v %v, said %d, want both manifests judged and nothing written", j, err, len(said))
|
||||
}
|
||||
// The manifest mended: its new digest is accepted and the module's refusal goes.
|
||||
mended := hold("bad", `{"module":"bad","version":"2.0"}`)
|
||||
if j, err := ix.JudgeAll(ctx, map[string]bool{mended: true}); err != nil || j.Written != 1 || len(j.Refused) != 0 {
|
||||
t.Fatalf("mended: %+v %v", j, err)
|
||||
}
|
||||
if m := module("bad"); m.Verdict != "" || m.Problems != nil {
|
||||
t.Fatalf("modules.bad still carries %+v", m)
|
||||
}
|
||||
// ADR 0579 §1: a manifest held before any verdict (taken in under the judges until then), which this judge finds a
|
||||
// problem in, stays accepted with the problem as a warning, carried in its value and said nowhere as a refusal; the
|
||||
// same problem in a manifest a merge takes in now, or in one whose digest changed, is refused in full.
|
||||
old := hold("old", `{"module":"old","version":"1.0","requires":["amqp"]}`)
|
||||
if j, err := ix.JudgeAll(ctx, nil); err != nil || len(j.Outgrown) != 1 || j.Outgrown[0] != "old" {
|
||||
t.Fatalf("an outgrown manifest: %+v %v", j, err)
|
||||
}
|
||||
if v := verdict(old); meshcatalogclient.VerdictRefuses(v) || len(v.Warnings) != 1 || !strings.Contains(v.Warnings[0], `requires "amqp"`) {
|
||||
t.Fatalf("old: %+v", v)
|
||||
}
|
||||
if m := module("old"); m.Verdict != "" || len(m.Warnings) != 1 || len(said) != 1 {
|
||||
t.Fatalf("modules.old carries %+v, said %d", m, len(said))
|
||||
}
|
||||
changed := hold("old", `{"module":"old","version":"2.0","requires":["amqp"]}`)
|
||||
fresh := hold("new", `{"module":"new","version":"1.0","requires":["amqp"]}`)
|
||||
if j, err := ix.JudgeAll(ctx, map[string]bool{changed: true, fresh: true}); err != nil || len(j.Outgrown) != 0 {
|
||||
t.Fatalf("a changed and a new manifest: %+v %v", j, err)
|
||||
}
|
||||
if !meshcatalogclient.VerdictRefuses(verdict(changed)) || !meshcatalogclient.VerdictRefuses(verdict(fresh)) || module("old").Verdict != meshcatalogclient.VerdictRefused || len(module("old").Warnings) != 0 {
|
||||
t.Fatalf("the changed and the new manifest are not refused: %+v %+v", verdict(changed), verdict(fresh))
|
||||
}
|
||||
// A merge's write of a module whose digest it leaves as it is carries the verdict over, and nothing is fresh; a changed
|
||||
// digest is fresh, judged in full (review R1, R2).
|
||||
w := &mergeWrite{}
|
||||
oldNeeds := module("old").Needs
|
||||
same := meshcatalog.ModulesByModuleValue{Fp2: "src2:" + strings.Repeat("c", 64), Manifest: changed, Repository: "novox/mesh-catalog", Commit: "c2"}
|
||||
if err := ix.writeOverlay(ctx, map[string]meshcatalog.ModulesByModuleValue{"old": same}, nil, w); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m := module("old"); m.Verdict != meshcatalogclient.VerdictRefused || len(m.Problems) == 0 || len(w.fresh) != 0 {
|
||||
t.Fatalf("a merge rewrote a refused module as %+v, fresh %v", m, w.fresh)
|
||||
}
|
||||
if got := module("old").Needs; len(oldNeeds) == 0 || !slices.Equal(got, oldNeeds) {
|
||||
t.Fatalf("a merge leaving the digest dropped the reader capabilities the judge named: %v, held %v", got, oldNeeds)
|
||||
}
|
||||
other := strings.Repeat("d", 64)
|
||||
same.Manifest = other
|
||||
if err := ix.writeOverlay(ctx, map[string]meshcatalog.ModulesByModuleValue{"old": same}, nil, w); err != nil || !w.fresh[other] {
|
||||
t.Fatalf("a changed digest is not fresh: %v %v", w.fresh, err)
|
||||
}
|
||||
// 1966 (ADR 0668 §4, ADR 0579 §1): what a take-in newly wrote is kept on the bus until judged. A take-in writes a
|
||||
// manifest a judge refuses in full, its judge pass fails on the verdicts state, and the next pass, which knows nothing
|
||||
// of that take-in (a start, a pass kicked by a message), still refuses it in full, never as a warning, and ends the key.
|
||||
ix.Unjudged = newMem()
|
||||
keptDigest, err := ix.keepManifest(ctx, "kept", `{"module":"kept","version":"1.0","requires":["amqp"]}`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kw := &mergeWrite{repository: "novox/mesh-catalog", merge: "m-kept"}
|
||||
keptValue := meshcatalog.ModulesByModuleValue{Fp2: "src2:" + strings.Repeat("e", 64), Manifest: keptDigest, Repository: "novox/mesh-catalog", Commit: "c3"}
|
||||
if err := ix.writeOverlay(ctx, map[string]meshcatalog.ModulesByModuleValue{"kept": keptValue}, nil, kw); err != nil || !kw.fresh[keptDigest] {
|
||||
t.Fatalf("the take-in's write: %v, fresh %v", err, kw.fresh)
|
||||
}
|
||||
if value, _, found, _ := ix.Unjudged.Get(ctx, keptDigest); !found || !strings.Contains(string(value), `"module":"kept"`) || !strings.Contains(string(value), "m-kept") {
|
||||
t.Fatalf("the new manifest is not kept as unjudged: %s", value)
|
||||
}
|
||||
ix.Verdicts.(*memStore).failWrite = func(string) error { return process.ErrNotYet }
|
||||
if _, err := ix.JudgeAll(ctx, kw.fresh); err == nil {
|
||||
t.Fatal("a pass whose verdicts state is not on the bus did not fail")
|
||||
}
|
||||
ix.Verdicts.(*memStore).failWrite = nil
|
||||
if j, err := ix.JudgeAll(ctx, nil); err != nil || slices.Contains(j.Outgrown, "kept") {
|
||||
t.Fatalf("the next pass: %+v %v", j, err)
|
||||
}
|
||||
if v := verdict(keptDigest); !meshcatalogclient.VerdictRefuses(v) || len(v.Warnings) != 0 {
|
||||
t.Fatalf("a refusal of a manifest whose judge pass failed became %+v", v)
|
||||
}
|
||||
if keys, err := ix.Unjudged.Keys(ctx); err != nil || len(keys) != 0 {
|
||||
t.Fatalf("the unjudged state still holds %v (%v) after its verdict is written", keys, err)
|
||||
}
|
||||
ix.Unjudged = nil
|
||||
// 1966/AC10: a newly accepted manifest asking for bus state is said once as manifest-accepted, carrying both entries
|
||||
// whole; a verdict already held says nothing.
|
||||
var accepted []meshcatalog.ManifestAcceptedEvent
|
||||
var ids []string
|
||||
var failing int // publishes still to fail
|
||||
ix.SayAccepted = func(_ context.Context, a meshcatalog.ManifestAcceptedEvent, id string) error {
|
||||
if failing > 0 {
|
||||
failing--
|
||||
return errors.New("no stream answered")
|
||||
}
|
||||
accepted, ids = append(accepted, a), append(ids, id)
|
||||
return nil
|
||||
}
|
||||
moved := hold("moved", `{"module":"moved","version":"1.0","requires":[`+
|
||||
`{"name":"cache","storage":"key-value","template":"state","options":{"history":8}},`+
|
||||
`{"name":"audit","storage":"stream","template":"log","options":{"max-mib":64}}]}`)
|
||||
// The merge's pass, then a later one that knows nothing of the merge.
|
||||
for _, fresh := range []map[string]bool{{moved: true}, nil} {
|
||||
if _, err := ix.JudgeAll(ctx, fresh); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// ADR 0614 §4: the judge names, with the verdict, the reader capabilities the manifest's own words need: the bus-state
|
||||
// words, which an engine stating nothing (the baseline) lacks; a manifest of the baseline's words lacks none.
|
||||
if needs := module("moved").Needs; !slices.Contains(needs, "word:bus-state.storage") ||
|
||||
!slices.Contains(catalogue.MissingManifestReads(needs, nil), "word:bus-state") {
|
||||
t.Fatalf("modules.moved needs %v", needs)
|
||||
}
|
||||
if needs := module("good").Needs; !slices.Contains(needs, "word:module") || len(catalogue.MissingManifestReads(needs, nil)) != 0 {
|
||||
t.Fatalf("modules.good needs %v", needs)
|
||||
}
|
||||
if _, rev, _, _ := ix.Verdicts.Get(ctx, moved); len(accepted) != 1 || accepted[0].Revision == 0 || accepted[0].Revision != int64(rev) {
|
||||
t.Fatalf("said accepted %+v, want the verdict's revision %d (ADR 0622 §2)", accepted, rev)
|
||||
}
|
||||
if len(accepted) != 1 || accepted[0].Module != "moved" || accepted[0].Digest != moved || accepted[0].Judge != "judge-2" ||
|
||||
len(accepted[0].BusState) != 2 || accepted[0].BusState[0].Name != "cache" || accepted[0].BusState[1].Template != "log" ||
|
||||
fmt.Sprint(accepted[0].BusState[0].Options["history"]) != "8" {
|
||||
t.Fatalf("said accepted %+v", accepted)
|
||||
}
|
||||
// DLV-I9 (design 25 § 12.2): a manifest asking for none, newly accepted, is said too, its entries an empty list.
|
||||
plain := hold("plain", `{"module":"plain","version":"1.0"}`)
|
||||
if _, err := ix.JudgeAll(ctx, map[string]bool{plain: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if last := accepted[len(accepted)-1]; last.Module != "plain" || last.Digest != plain || last.BusState == nil || len(last.BusState) != 0 {
|
||||
t.Fatalf("a manifest asking for no bus state was said as %+v", last)
|
||||
}
|
||||
// Review round 1 of issue 1966: a trunk's held manifest judged by its digest (indexer_judge, judgeStale) is said as the
|
||||
// full pass says it, accepted or refused; a candidate's digest judged over the trunk is said nowhere.
|
||||
byDigest := hold("bydigest", `{"module":"bydigest","version":"1.0"}`)
|
||||
refusedBy := hold("refusedby", `{"module":"refusedby","version":"1.0","requires":["amqp"]}`)
|
||||
saidBefore := len(said)
|
||||
for module, digest := range map[string]string{"bydigest": byDigest, "refusedby": refusedBy} {
|
||||
if _, err := ix.judgeOver(ctx, nil, module, digest); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if last := accepted[len(accepted)-1]; last.Digest != byDigest || len(said) != saidBefore+1 || said[len(said)-1].Digest != refusedBy {
|
||||
t.Fatalf("judged by its digest, the trunk's manifest was said as %+v, refusals %d (were %d)", last, len(said), saidBefore)
|
||||
}
|
||||
candidate, err := ix.keepManifest(ctx, "bydigest", `{"module":"bydigest","version":"2.0"}`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := ix.judgeOver(ctx, nil, "bydigest", candidate); err != nil || accepted[len(accepted)-1].Digest != byDigest {
|
||||
t.Fatalf("a candidate's verdict was said: %v %+v", err, accepted[len(accepted)-1])
|
||||
}
|
||||
// Issue 1966 (design 25 bus-state §1): the candidate merged, its verdict unchanged, is newly accepted on the trunk and
|
||||
// said at the merge's pass, from the verdict held and with its revision, though the pass writes no verdict.
|
||||
if hold("bydigest", `{"module":"bydigest","version":"2.0"}`) != candidate {
|
||||
t.Fatal("the merged manifest is not the candidate's")
|
||||
}
|
||||
if j, err := ix.JudgeAll(ctx, map[string]bool{candidate: true}); err != nil || j.Written != 0 {
|
||||
t.Fatalf("the merge's pass: %+v %v", j, err)
|
||||
}
|
||||
if _, rev, _, _ := ix.Verdicts.Get(ctx, candidate); accepted[len(accepted)-1].Digest != candidate || accepted[len(accepted)-1].Revision != int64(rev) {
|
||||
t.Fatalf("the merged candidate's manifest was not said accepted with its verdict's revision %d: last said %+v", rev, accepted[len(accepted)-1])
|
||||
}
|
||||
// DLV-I9b: a failed publish is published again; one failing twice writes the finding naming the module, which the next
|
||||
// judging pass says again and clears.
|
||||
ix.Findings = newFindings()
|
||||
failing = 1
|
||||
once := hold("once", `{"module":"once","version":"1.0"}`)
|
||||
if _, err := ix.JudgeAll(ctx, map[string]bool{once: true}); err != nil || accepted[len(accepted)-1].Module != "once" {
|
||||
t.Fatalf("a publish failing once was not published again: %v %+v", err, accepted[len(accepted)-1])
|
||||
}
|
||||
failing = 2
|
||||
flaky := hold("flaky", `{"module":"flaky","version":"1.0"}`)
|
||||
if _, err := ix.JudgeAll(ctx, map[string]bool{flaky: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if value, found := ix.Findings.(*memFindings).vals[unsaidKey("flaky")]; !found || !strings.Contains(string(value), "flaky") {
|
||||
t.Fatalf("a publish failing twice wrote no finding naming flaky: %s", value)
|
||||
}
|
||||
if _, err := ix.JudgeAll(ctx, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, found := ix.Findings.(*memFindings).vals[unsaidKey("flaky")]; found || accepted[len(accepted)-1].Module != "flaky" ||
|
||||
accepted[len(accepted)-1].Digest != flaky {
|
||||
t.Fatalf("the next pass did not say flaky and clear its finding (found %v, last said %+v)", found, accepted[len(accepted)-1])
|
||||
}
|
||||
// DLV-I9c: indexer_say_accepted says it again from the held verdict, under an id of its own; a module with no accepted
|
||||
// verdict, or none held, is refused.
|
||||
if r := ix.SayAcceptedAgain(ctx, "flaky"); !r.Stored || accepted[len(accepted)-1].Module != "flaky" || !strings.Contains(ids[len(ids)-1], "-again-") {
|
||||
t.Fatalf("said again: %+v, id %s", r, ids[len(ids)-1])
|
||||
}
|
||||
for _, m := range []string{"old", "nobody"} {
|
||||
if r := ix.SayAcceptedAgain(ctx, m); r.Stored {
|
||||
t.Errorf("%s said again: %+v", m, r)
|
||||
}
|
||||
}
|
||||
if key, value, err := NewSayAccepted("flaky", "the bus module missed it", "jochen@shanks", time.Now()); err != nil ||
|
||||
!strings.HasPrefix(key, sayAcceptedPrefix) || !strings.Contains(string(value), `"say":true`) {
|
||||
t.Errorf("the ask = %s %s (%v)", key, value, err)
|
||||
}
|
||||
if _, _, err := NewSayAccepted("flaky", "", "jochen@shanks", time.Now()); err == nil {
|
||||
t.Error("an ask saying no why was taken")
|
||||
}
|
||||
// DLV-I15: a change's renamed-from is judged against the trunk's manifest of the module: a name the trunk declares is
|
||||
// accepted, one it does not is refused naming the key.
|
||||
trunkRenamer := hold("renamer", `{"module":"renamer","version":"1.0","config":{"font-name":{"type":"string","description":"The font.","required":true,"runtime":true}}}`)
|
||||
if _, err := ix.JudgeAll(ctx, map[string]bool{trunkRenamer: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for raw, want := range map[string]string{
|
||||
`{"module":"renamer","version":"1.0","config":{"font":{"type":"string","description":"The font.","required":true,"runtime":true,"renamed-from":"font-name"}}}`: "",
|
||||
`{"module":"renamer","version":"1.0","config":{"font":{"type":"string","description":"The font.","required":true,"runtime":true,"renamed-from":"typeface"}}}`: `renamed from "typeface"`,
|
||||
} {
|
||||
head, err := ix.keepManifest(ctx, "renamer", raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ix.JudgeAgain(ctx, "renamer", head)
|
||||
if v := verdict(head); (want == "") != !meshcatalogclient.VerdictRefuses(v) || (want != "" && !strings.Contains(strings.Join(v.Problems, " "), want)) {
|
||||
t.Errorf("the change %s: %+v, want refused naming %q", raw, v, want)
|
||||
}
|
||||
}
|
||||
// DLV-I14 and the lead's ruling (issue 1966): a pull request's candidate is not refused for naming a package the
|
||||
// packages state does not know (the checker judges its names, knowing its delivery group); the same manifest a merge
|
||||
// takes in is, though its candidate was accepted; a name the state knows is accepted at both.
|
||||
ix.Packages = newMem()
|
||||
sdkGo := "git.novox.be/novox/mesh-sdk/go"
|
||||
if value, err := EncodeSeriesPackage(SeriesPackage{V: 1, Name: sdkGo, Package: "mesh-sdk/go", Series: "0.1"}); err != nil {
|
||||
t.Fatal(err)
|
||||
} else {
|
||||
ix.Packages.(*memStore).put(SeriesKey(sdkGo, "0.1"), value)
|
||||
}
|
||||
for raw, refusedAtMerge := range map[string]bool{
|
||||
`{"module":"pkguser","version":"1.0","packages":{"@novox/ui":"0.1"}}`: true,
|
||||
`{"module":"pkguser","version":"2.0","packages":{"` + sdkGo + `":"0.1"}}`: false,
|
||||
} {
|
||||
head, err := ix.keepManifest(ctx, "pkguser", raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ix.judgeDigests(ctx, map[string]string{"pkguser": head}, "novox/mesh-catalog#1")
|
||||
if v := verdict(head); meshcatalogclient.VerdictRefuses(v) {
|
||||
t.Errorf("the candidate %s is refused: %+v", raw, v)
|
||||
}
|
||||
merged := hold("pkguser", raw)
|
||||
if _, err := ix.JudgeAll(ctx, map[string]bool{merged: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if v := verdict(merged); meshcatalogclient.VerdictRefuses(v) != refusedAtMerge || refusedAtMerge && !strings.Contains(strings.Join(v.Problems, " "), `name "@novox/ui", which no package`) {
|
||||
t.Errorf("the merge of %s: %+v, want refused %v for the name", raw, v, refusedAtMerge)
|
||||
}
|
||||
}
|
||||
ix.Packages = nil
|
||||
// 1966/AC11: modules using state and none using logs but the foundation: logs counts 0, so a manifest of another
|
||||
// module using logs is refused and the foundation's is accepted.
|
||||
ix.OldSections = newMem()
|
||||
hold("user", `{"module":"user","version":"1.0","state":["x"]}`)
|
||||
hold("mesh-grants", `{"module":"mesh-grants","version":"1.0","logs":[{"name":"l"}]}`)
|
||||
if _, err := ix.JudgeAll(ctx, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
counts := func() map[string]int {
|
||||
value, _, found, err := ix.OldSections.Get(ctx, OldSectionsKey)
|
||||
if err != nil || !found {
|
||||
t.Fatalf("no count of the old sections: %v", err)
|
||||
}
|
||||
c, err := DecodeOldSections(value)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return c.Counts
|
||||
}
|
||||
if c := counts(); c["state"] != 1 || c["logs"] != 0 || c["streams"] != 0 || c["objects"] != 0 {
|
||||
t.Fatalf("the old sections count %v, want state 1 and the rest 0", c)
|
||||
}
|
||||
// DLV-J2, DLV-J3 (ADR 0615 §1): one reads entry on the assigner's state counts 1 for the assigner and 0 for the indexer.
|
||||
hold("reader", `{"module":"reader","version":"1.0","state":["x"],"reads":["mesh-delivery-assigner.wants"]}`)
|
||||
if _, err := ix.JudgeAll(ctx, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if value, _, _, _ := ix.OldSections.Get(ctx, OldSectionsKey); true {
|
||||
c, _ := DecodeOldSections(value)
|
||||
if c.Reads["mesh-delivery-assigner"] != 1 || c.Reads["mesh-delivery-assigner.wants"] != 1 || c.Reads["mesh-delivery-indexer"] != 0 {
|
||||
t.Fatalf("the reads on the owners' state count %v", c.Reads)
|
||||
}
|
||||
}
|
||||
// DLV-J1 (ADR 0619 §1, §3): a foundation part asking bus state in the new form is refused; the nine and the pipeline runner are the foundation.
|
||||
// Design 47 *A module's registration* (b229f5c4 lines 480-481): a template in the manifest of every part on the list is
|
||||
// refused, the runner's included.
|
||||
for _, part := range []string{"mesh-config", "mesh-delivery-pipeline"} {
|
||||
d := hold(part, `{"module":"`+part+`","version":"1.0","requires":[{"name":"cache","storage":"key-value","template":"state"}]}`)
|
||||
if _, err := ix.JudgeAll(ctx, map[string]bool{d: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if v := verdict(d); !meshcatalogclient.VerdictRefuses(v) || !strings.Contains(strings.Join(v.Problems, " "), "part of the foundation") {
|
||||
t.Fatalf("%s asking new-form bus state: %+v", part, v)
|
||||
}
|
||||
}
|
||||
if !slices.Equal(judge.Foundation, []string{"nats", "mesh-grants", "mesh-delivery-indexer", "mesh-delivery-assigner", "node-tools",
|
||||
"mesh-delivery-builder", "mesh-delivery-checker", "mesh-config", "mesh-vault", "mesh-delivery-pipeline", "gitea", "distribution", "mesh-host"}) {
|
||||
t.Fatalf("the foundation is %v, want exactly the nine of ADR 0619, the pipeline runner (ADR 0666 §4) and the forge and image registry modules (design 47 G §584) and the node-engine's module (§627)", judge.Foundation)
|
||||
}
|
||||
late := hold("late", `{"module":"late","version":"1.0","logs":[{"name":"l"}]}`)
|
||||
grants := hold("mesh-grants", `{"module":"mesh-grants","version":"2.0","logs":[{"name":"l"},{"name":"m"}]}`)
|
||||
if _, err := ix.JudgeAll(ctx, map[string]bool{late: true, grants: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if v := verdict(late); !meshcatalogclient.VerdictRefuses(v) || !strings.Contains(v.Problems[0], "late uses the old section logs, which no other module uses any more") {
|
||||
t.Fatalf("late: %+v", v)
|
||||
}
|
||||
if v := verdict(grants); meshcatalogclient.VerdictRefuses(v) {
|
||||
t.Fatalf("the foundation's manifest was refused: %+v", v)
|
||||
}
|
||||
// 1966/AC11: the sole user of an old section changes its manifest and keeps the section: accepted, since it was accepted
|
||||
// with it (the count is the accepted set held from the last pass, not recounted from fresh digests).
|
||||
hold("solo", `{"module":"solo","version":"1.0","objects":[{"name":"o","max-mib":1}]}`)
|
||||
if _, err := ix.JudgeAll(ctx, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
solo2 := hold("solo", `{"module":"solo","version":"2.0","objects":[{"name":"o","max-mib":2}]}`)
|
||||
if _, err := ix.JudgeAll(ctx, map[string]bool{solo2: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if v := verdict(solo2); meshcatalogclient.VerdictRefuses(v) {
|
||||
t.Fatalf("the sole user of objects, changing its manifest and keeping objects, was refused: %+v", v)
|
||||
}
|
||||
// A count that cannot be read is never taken as 0: the zero rule is not applied in that pass and the log says so.
|
||||
var logged []string
|
||||
ix.Logf = func(f string, a ...any) { logged = append(logged, fmt.Sprintf(f, a...)) }
|
||||
held := ix.OldSections
|
||||
ix.OldSections = unreadable{held}
|
||||
retry := hold("late", `{"module":"late","version":"2.0","logs":[{"name":"l"}]}`)
|
||||
if _, err := ix.JudgeAll(ctx, map[string]bool{retry: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if v := verdict(retry); meshcatalogclient.VerdictRefuses(v) || !strings.Contains(strings.Join(logged, "\n"), "the zero rule is not applied") {
|
||||
t.Fatalf("a count that could not be read: %+v, logged %q", v, logged)
|
||||
}
|
||||
ix.OldSections = held
|
||||
// Clause 3 of the record on a release's Approve: a manifest with a field this judge does not know waits (no refusal, no
|
||||
// event) while a newer build of the indexer is on the trunk, and is judged again at the next judge; once this program's
|
||||
// build is the trunk's newest it is refused; a wait is written into the module's value as a refusal naming the fields.
|
||||
ownFP := "src2:" + strings.Repeat("e", 64)
|
||||
hold(testSelf, `{"module":"mesh-delivery-indexer","version":"1.0"}`)
|
||||
ix.ownFP = ownFP // this program started when the trunk held another build of the indexer than the one it holds now
|
||||
later := hold("later", `{"module":"later","version":"1.0","a-field-from-later":1,"another":{"x":1}}`)
|
||||
said = nil
|
||||
before := len(said)
|
||||
j, err = ix.JudgeAll(ctx, map[string]bool{later: true})
|
||||
if err != nil || len(j.Waiting) != 1 || j.Waiting[0] != "later" || len(said) != before {
|
||||
t.Fatalf("a manifest with fields from later: %+v %v, said %d", j, err, len(said))
|
||||
}
|
||||
for _, a := range accepted {
|
||||
if a.Module == "later" {
|
||||
t.Fatalf("a manifest set aside to wait was said accepted: %+v (DLV-I9)", a)
|
||||
}
|
||||
}
|
||||
if v := verdict(later); !meshcatalogclient.VerdictWaits(v) || meshcatalogclient.VerdictRefuses(v) || strings.Join(v.Unknown, ",") != "a-field-from-later,another" {
|
||||
t.Fatalf("later: %+v", v)
|
||||
}
|
||||
if m := module("later"); m.Verdict != meshcatalogclient.VerdictRefused || len(m.Unknown) != 2 || !meshcatalogclient.VerdictWaits(*meshcatalogclient.ModuleVerdict("later", m)) ||
|
||||
!strings.Contains(m.Problems[0], "does not know yet") {
|
||||
t.Fatalf("modules.later carries %+v", m)
|
||||
}
|
||||
// The build this program runs is asked of the engine, and the trunk moving before the start pass changes nothing: the
|
||||
// trunk's value of the indexer names a newer build than the one the engine says runs, so the field still waits.
|
||||
ix.ownFP = ""
|
||||
ix.OwnBuild = func(context.Context) (string, error) { return ownFP, nil }
|
||||
if _, err := ix.JudgeAtStart(ctx); err != nil || !meshcatalogclient.VerdictWaits(verdict(later)) {
|
||||
t.Fatalf("the trunk moved before the start pass: %v %+v", err, verdict(later))
|
||||
}
|
||||
ix.OwnBuild = func(context.Context) (string, error) { return "", errors.New("the engine does not answer") }
|
||||
ix.ownFP = ""
|
||||
if _, err := ix.JudgeAtStart(ctx); err != nil || !meshcatalogclient.VerdictWaits(verdict(later)) {
|
||||
t.Fatalf("an unknown build refused: %v %+v", err, verdict(later))
|
||||
}
|
||||
ix.ownFP = ownFP
|
||||
// A renamed field is a known mistake: refused at once.
|
||||
renamed := hold("renamed", `{"module":"renamed","version":"1.0","needs":{"x":"y"}}`)
|
||||
if j, err := ix.JudgeAll(ctx, map[string]bool{renamed: true}); err != nil || len(j.Waiting) != 1 || !meshcatalogclient.VerdictRefuses(verdict(renamed)) {
|
||||
t.Fatalf("a renamed field: %+v %v", j, err)
|
||||
}
|
||||
// The trunk's newest build of the indexer is this program's: the field is one no judge to come knows.
|
||||
ix.ownFP = "src2:" + strings.Repeat("a", 64)
|
||||
if j, err := ix.JudgeAll(ctx, nil); err != nil || len(j.Waiting) != 0 || !meshcatalogclient.VerdictRefuses(verdict(later)) || !strings.Contains(verdict(later).Problems[0], "a-field-from-later") {
|
||||
t.Fatalf("the newest build refuses: %+v %v %+v", j, err, verdict(later))
|
||||
}
|
||||
if m := module("later"); m.Verdict != meshcatalogclient.VerdictRefused || meshcatalogclient.VerdictWaits(*meshcatalogclient.ModuleVerdict("later", m)) || len(m.Unknown) != 0 {
|
||||
t.Fatalf("modules.later after the refusal carries %+v", m)
|
||||
}
|
||||
// The tools: indexer_verdicts lists the verdicts that wait or are refused (and every one with all), indexer_judge
|
||||
// judges now and says how the module's verdict stands.
|
||||
keys, _ := ix.Verdicts.Keys(ctx)
|
||||
get := func(key string) ([]byte, error) { v, _, _, err := ix.Verdicts.Get(ctx, key); return v, err }
|
||||
if rows, unread := ListVerdicts(keys, get, "later", false); len(rows) != 1 || rows[0].State != meshcatalogclient.VerdictRefused || len(unread) != 0 {
|
||||
t.Fatalf("indexer_verdicts of later: %+v %v", rows, unread)
|
||||
}
|
||||
if rows, _ := ListVerdicts(keys, get, "", true); len(rows) <= len(keys)-3 || rows[0].Module > rows[len(rows)-1].Module {
|
||||
t.Fatalf("indexer_verdicts, all: %d rows of %d verdicts", len(rows), len(keys))
|
||||
}
|
||||
// The build is fixed once, at the start, and never learnt later: this program started when the engine could not answer, and
|
||||
// the engine now names the trunk's newest build (as it does while it replaces the indexer by a successor). indexer_judge
|
||||
// does not take it for this program's own, so the field still waits and the build stays unknown.
|
||||
ix.ownFP, ix.ownFixed = "", true
|
||||
ix.OwnBuild = func(context.Context) (string, error) { return "src2:" + strings.Repeat("a", 64), nil }
|
||||
if got := ix.JudgeAgain(ctx, "later", ""); !got.Stored || !strings.Contains(got.Answer, "waits") || ix.ownFP != "" {
|
||||
t.Fatalf("indexer_judge learnt the build the engine names now: %+v (own %q)", got, ix.ownFP)
|
||||
}
|
||||
if _, err := ix.JudgeAtStart(ctx); err != nil || ix.ownFP != "" || !meshcatalogclient.VerdictWaits(verdict(later)) {
|
||||
t.Fatalf("a judge pass learnt the build: %v %q %+v", err, ix.ownFP, verdict(later))
|
||||
}
|
||||
ix.ownFP = ownFP
|
||||
ix.OwnBuild = nil
|
||||
if got := ix.JudgeAgain(ctx, "later", ""); !got.Stored || !strings.Contains(got.Answer, "waits") || !strings.Contains(got.Answer, "a-field-from-later") {
|
||||
t.Fatalf("indexer_judge: %+v", got)
|
||||
}
|
||||
if rows, _ := ListVerdicts(keys, get, "later", false); len(rows) != 1 {
|
||||
t.Fatalf("indexer_verdicts of later after: %+v", rows)
|
||||
}
|
||||
if got := ix.JudgeAgain(ctx, "nobody", ""); got.Stored || !strings.Contains(got.Answer, "holds no manifest") {
|
||||
t.Fatalf("indexer_judge of nobody: %+v", got)
|
||||
}
|
||||
if key, value, err := NewJudgeAgain("later", "", "why", "me", time.Now()); err != nil || key != "judge.later" {
|
||||
t.Fatalf("the ask: %q %v", key, err)
|
||||
} else if a, err := DecodeTakeIn(value); err != nil || !a.Judge || a.Module != "later" {
|
||||
t.Fatalf("the ask's value: %+v %v", a, err)
|
||||
}
|
||||
// A refusal an older judge made never stands: a digest the trunk no longer names (a candidate's, or one back on the
|
||||
// trunk later) refused by judge-0 for words it did not know is judged again by this judge at the start, at a pull that
|
||||
// carries it, and by indexer_judge with its digest; one this judge refused itself is left as it is.
|
||||
stale := func(module, raw string) string {
|
||||
digest, err := ix.keepManifest(ctx, module, raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, err := ix.writeVerdict(ctx, digest, meshcatalog.VerdictsByDigestValue{Module: module, State: meshcatalogclient.VerdictRefused,
|
||||
Problems: []string{module + " says tool-runner, which this judge does not know"}, Judge: "judge-0", At: ptr(time.Now())}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return digest
|
||||
}
|
||||
atStart := stale("tools", `{"module":"tools","version":"7.0"}`)
|
||||
ownRefusal := stale("client", `{"module":"client","version":"7.0"}`)
|
||||
if _, _, err := ix.writeVerdict(ctx, ownRefusal, meshcatalog.VerdictsByDigestValue{Module: "client", State: meshcatalogclient.VerdictRefused,
|
||||
Problems: []string{"x"}, Judge: ix.judgeID(), At: ptr(time.Now())}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := ix.JudgeAtStart(ctx); err != nil || meshcatalogclient.VerdictRefuses(verdict(atStart)) || verdict(atStart).Judge != ix.judgeID() || !meshcatalogclient.VerdictRefuses(verdict(ownRefusal)) {
|
||||
t.Fatalf("the start pass: %+v, this judge's own refusal %+v (%v)", verdict(atStart), verdict(ownRefusal), err)
|
||||
}
|
||||
byPull := stale("tools", `{"module":"tools","version":"8.0"}`)
|
||||
pull := forgemsg.Forge{Kind: forgemsg.ForgePull, Repository: "novox/mesh-tools", Number: 3, Branch: "main",
|
||||
Manifests: map[string]string{"modules/tools": `{"module":"tools","version":"8.0"}`}}
|
||||
if _, err := ix.ApplyMessage(ctx, pull); err != nil || meshcatalogclient.VerdictRefuses(verdict(byPull)) || verdict(byPull).Judge != ix.judgeID() {
|
||||
t.Fatalf("a pull carrying a digest an older judge refused: %+v %v", verdict(byPull), err)
|
||||
}
|
||||
byTool := stale("tools", `{"module":"tools","version":"9.0"}`)
|
||||
if got := ix.JudgeAgain(ctx, "client", byTool); got.Stored || !strings.Contains(got.Answer, "tools's, not client's") || !meshcatalogclient.VerdictRefuses(verdict(byTool)) {
|
||||
t.Fatalf("indexer_judge with another module's digest: %+v", got)
|
||||
}
|
||||
if got := ix.JudgeAgain(ctx, "tools", byTool); !got.Stored || !strings.Contains(got.Answer, "accepted") || meshcatalogclient.VerdictRefuses(verdict(byTool)) {
|
||||
t.Fatalf("indexer_judge with a digest: %+v %+v", got, verdict(byTool))
|
||||
}
|
||||
if key, value, err := NewJudgeAgain("tools", byTool, "why", "me", time.Now()); err != nil || key != "judge.tools" {
|
||||
t.Fatalf("the ask with a digest: %q %v", key, err)
|
||||
} else if a, err := DecodeTakeIn(value); err != nil || a.Digest != byTool {
|
||||
t.Fatalf("the ask's digest: %+v %v", a, err)
|
||||
}
|
||||
// Without the verdicts state the indexer judges nothing.
|
||||
ix.Verdicts = nil
|
||||
if j, err := ix.JudgeAll(ctx, nil); err != nil || j.Judged != 0 {
|
||||
t.Fatalf("without the state: %+v %v", j, err)
|
||||
}
|
||||
}
|
||||
|
||||
// unreadable is a store whose reads fail.
|
||||
type unreadable struct{ Store }
|
||||
|
||||
func (unreadable) Get(context.Context, string) ([]byte, uint64, bool, error) {
|
||||
return nil, 0, false, errors.New("the bus did not answer")
|
||||
}
|
||||
|
||||
// Design 47 G §491, §676: two modules whose entries carry one Go module path are both refused by the trunk's judge pass,
|
||||
// each naming both; a module with its own name is accepted.
|
||||
func TestAPackageNameBelongsToOneModule(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
ix := &Indexer{Module: testSelf, Modules: newMem(), Manifests: newMem(), Verdicts: newMem(), JudgeID: "judge-1",
|
||||
Now: func() time.Time { return time.Date(2026, 11, 1, 3, 0, 0, 0, time.UTC) }}
|
||||
ix.ReadFile = func(_ context.Context, _, _, path string) ([]byte, error) {
|
||||
if strings.HasSuffix(path, "go.mod") {
|
||||
name := "git.example/own/" + strings.Split(path, "/")[0]
|
||||
if strings.HasPrefix(path, "one/") || strings.HasPrefix(path, "two/") {
|
||||
name = "git.example/shared"
|
||||
}
|
||||
return []byte("module " + name + "\n"), nil
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
hold := func(module string) string {
|
||||
raw := `{"module":"` + module + `","version":"1.0","build":{"artifacts":[{"name":"go","kind":"package","language":"go","paths":["go"],"series":"0.1","generated-from":"."}]}}`
|
||||
digest, err := ix.keepManifest(ctx, module, raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
value, err := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: "src2:" + strings.Repeat("a", 64), Manifest: digest,
|
||||
Repository: "novox/r", Path: module, Commit: "c"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ix.Modules.(*memStore).put(module, value)
|
||||
return digest
|
||||
}
|
||||
digests := map[string]bool{hold("one"): true, hold("two"): true, hold("three"): true}
|
||||
j, err := ix.JudgeAll(ctx, digests)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(j.Refused) != 2 || j.Refused["one"] == nil || j.Refused["two"] == nil || j.Refused["three"] != nil {
|
||||
t.Fatalf("refused %v, want one and two", j.Refused)
|
||||
}
|
||||
for _, m := range []string{"one", "two"} {
|
||||
if p := strings.Join(j.Refused[m], " "); !strings.Contains(p, "one and two") || !strings.Contains(p, "git.example/shared") {
|
||||
t.Errorf("%s's problems %q do not name both modules and the package", m, p)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -17,7 +17,6 @@ import (
|
||||
var FirstViewTools = []ViewedTool{
|
||||
{Tool: "manifests", State: "manifests", Words: "every manifest the indexer keeps, by its digest."},
|
||||
{Tool: "modules", State: ModulesState, Words: "every module the indexer holds, its fingerprint, inputs and manifest digest."},
|
||||
{Tool: "verdicts", State: VerdictsState, Words: "the judge's verdict on each manifest: accepted, waiting or refused."},
|
||||
{Tool: "graph", State: graph.State, Words: "the module graph: each module's build, provision and verb edges, whose verb edges a want waits on."},
|
||||
}
|
||||
|
||||
@@ -33,7 +32,6 @@ var ReadTools = []ViewedTool{
|
||||
{Tool: "indexer_read_takeins", State: TakeInsState, Words: "each take-in asked of the indexer: its ask, whether and when the indexer answered it and its answer (design 47 G A-464a)."},
|
||||
{Tool: "indexer_read_findings", State: FindingsState, Words: "the indexer's findings, which the condition keeper raises and clears."},
|
||||
{Tool: "indexer_read_seat_definitions", State: SeatDefinitionsState, Words: "the seat definitions the trunk's and each open pull request's contracts modules hold, resolved, or why one does not parse."},
|
||||
{Tool: "indexer_read_unjudged", State: UnjudgedState, Words: "the manifests a take-in newly wrote that no judge pass judged yet, by digest, each with its module and merge; indexer_judge judges them."},
|
||||
}
|
||||
|
||||
// ViewedTool is one tool over one state of the indexer's.
|
||||
@@ -43,8 +41,8 @@ type ViewedTool struct {
|
||||
|
||||
// ViewedStates are the states the indexer's views family says: the ones its readers read.
|
||||
var ViewedStates = map[string]bool{
|
||||
ManifestsState: true, VerdictsState: true, PredefinedState: true, CandidatesState: true,
|
||||
PackagesState: true, HeardState: true, TakenState: true, TakeInsState: true, FindingsState: true, SeatDefinitionsState: true, UnjudgedState: true,
|
||||
ManifestsState: true, PredefinedState: true, CandidatesState: true,
|
||||
PackagesState: true, HeardState: true, TakenState: true, TakeInsState: true, FindingsState: true, SeatDefinitionsState: true,
|
||||
}
|
||||
|
||||
// ModulesViewsEvent is what the indexer says on its views family of modules when a new revision of the trunk branch
|
||||
|
||||
@@ -115,11 +115,16 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
|
||||
if m.Module != "mesh-delivery-indexer" {
|
||||
t.Errorf("module = %q", m.Module)
|
||||
}
|
||||
// 1126: assignments is the seat mesh-module-assigner's (ADR 0567): the indexer holds no assignments state. ADR 0578 §1: it says
|
||||
// a refused manifest as manifest-refused, and (issue 1966 D3, 1966/AC10) a newly accepted one asking for bus state as
|
||||
// manifest-accepted, both events of the seat mesh-catalog it holds (ADR 0636 §4), so not among its `emits`.
|
||||
// 1126: assignments is the seat mesh-module-assigner's (ADR 0567): the indexer holds no assignments state. I7/AC1 (design 47
|
||||
// H §745, §764): it judges no manifest at the take-in, so it keeps no verdicts and says no manifest-refused or
|
||||
// manifest-accepted; its own events are the take-in's two.
|
||||
if own := slices.DeleteFunc(slices.Clone(m.Emits), func(e string) bool { return strings.HasPrefix(e, views.EventPrefix) }); !reflect.DeepEqual(own, []string{takein.TakeInPendingEvent, takein.TakeInTakenEvent}) {
|
||||
t.Errorf("emits = %v, want the two take-in events (the verdict events are the seat mesh-catalog's)", m.Emits)
|
||||
t.Errorf("emits = %v, want the two take-in events", m.Emits)
|
||||
}
|
||||
for _, gone := range []string{"verdicts", "unjudged", "indexer_verdicts", "indexer_judge", "indexer_say_accepted", "indexer_read_unjudged", "views.verdicts"} {
|
||||
if slices.Contains(m.Tools, gone) || slices.Contains(m.Emits, gone) || slices.ContainsFunc(m.State, func(s state) bool { return s.Name == gone }) {
|
||||
t.Errorf("the manifest still declares %s (design 47 H §764: the validation at the take-in goes)", gone)
|
||||
}
|
||||
}
|
||||
// Design 47 G §700, A-74: the router's record of an ask is read through the seat operator-channel, which the indexer uses,
|
||||
// and its holder is asked of the seat mesh-module-assigner.
|
||||
@@ -127,10 +132,10 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
|
||||
t.Errorf("uses = %v, invokes = %v, want the seat operator-channel used and seat:mesh-module-assigner.assignments invoked", m.Uses, m.Invokes)
|
||||
}
|
||||
// 1126 note 76: heard keeps the last 64 FORGE messages the indexer handled, for indexer_heard.
|
||||
if want := []state{{Name: "candidates", History: 10}, {Name: "heard", History: 64}, {Name: "manifests", History: 1}, {Name: "predefined", History: 5}, {Name: "taken", History: 1}, {Name: "takeins", History: 1}, {Name: "verdicts", History: 5}, {Name: "oldsections", History: 1}, {Name: index.RunsState, History: 1}, {Name: "own-build", History: 1}, {Name: index.PackagesState, History: index.PackagesHistory},
|
||||
if want := []state{{Name: "candidates", History: 10}, {Name: "heard", History: 64}, {Name: "manifests", History: 1}, {Name: "predefined", History: 5}, {Name: "taken", History: 1}, {Name: "takeins", History: 1}, {Name: "oldsections", History: 1}, {Name: index.RunsState, History: 1}, {Name: "own-build", History: 1}, {Name: index.PackagesState, History: index.PackagesHistory},
|
||||
{Name: views.MarksState, History: 1}, {Name: index.FindingsState, History: 5}, {Name: views.StatusState, History: 1},
|
||||
{Name: index.RequirementsState, History: 5}, {Name: index.SeatDefinitionsState, History: 5},
|
||||
{Name: index.PendingState, History: 1}, {Name: index.UnjudgedState, History: 1}, {Name: index.EntryIndexState, History: 1, MaxMiB: 8}, {Name: index.NamedState, History: 1, MaxMiB: 4}, // design 47 G §695
|
||||
{Name: index.PendingState, History: 1}, {Name: index.EntryIndexState, History: 1, MaxMiB: 8}, {Name: index.NamedState, History: 1, MaxMiB: 4}, // design 47 G §695
|
||||
{Name: index.RegistrationsState, History: 1}, {Name: index.ProposalsState, History: 1, MaxMiB: 4},
|
||||
{Name: index.VersionsState, History: 1, MaxMiB: 64}, {Name: index.BuildEdgesState, History: 1}, {Name: index.InputsState, History: 1}, {Name: index.PinsState, History: 1, MaxMiB: index.PinsCapMiB}, {Name: process.ConsumersState, History: 1}}; !reflect.DeepEqual(m.State, want) { // DLV-I7, DLV-I17
|
||||
t.Errorf("state = %+v, want %+v", m.State, want)
|
||||
@@ -146,8 +151,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
|
||||
// name and states with their revisions, and the states it reads, each as `<holder>.<state>`.
|
||||
// A-296: its own buckets are named from its credential's module (index.Bucket), each a state its manifest keeps.
|
||||
own := func(state string) string { return index.Bucket("a-indexer", state) }
|
||||
for bucket, history := range map[string]int{own(index.CandidatesState): index.CandidatesHistory, own(index.ManifestsState): index.ManifestsHistory,
|
||||
own(index.VerdictsState): index.VerdictsHistory, own(index.PackagesState): index.PackagesHistory, own(index.TakenState): 1, own(index.PredefinedState): 5} {
|
||||
for bucket, history := range map[string]int{own(index.CandidatesState): index.CandidatesHistory, own(index.ManifestsState): index.ManifestsHistory, own(index.PackagesState): index.PackagesHistory, own(index.TakenState): 1, own(index.PredefinedState): 5} {
|
||||
if !slices.ContainsFunc(m.State, func(st state) bool { return "a-indexer_"+st.Name == bucket && st.History == history }) {
|
||||
t.Errorf("the indexer names its bucket %s with %d revisions, its manifest's states are %+v", bucket, history, m.State)
|
||||
}
|
||||
@@ -166,7 +170,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
|
||||
// internal/broker/state.go:62-93).
|
||||
// The registry seats' published fill each (§413: no consume of version.numbered) package's series (DLV-I5, design 51 *The versions*).
|
||||
if want := []string{"go-package-registry.published", "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "mesh-builds.package-published", "mesh-builds.toolchain-agreed", "mesh-catalog.catalog-revision", "mesh-conditions.findings-asked", "mesh-delivery-checker.verdict", "mesh-delivery-forge-bridge.forge.**",
|
||||
"mesh-delivery.freed", "mesh-delivery.heads-replaced", "mesh-delivery.pinned", "mesh-delivery.pins-stated", "mesh-delivery.source-moved", "mesh-pipeline-runner.pipeline-ended", "mesh-pipeline-runner.run-started", "mesh-wants.version-delivered", "mesh-wants.wait-ended", "mesh-wants.wanted", "node-build-go.step-ended", "node-engine.running", "npm-package-registry.published", "operator-channel.decided", "seat:mesh-module-assigner.views.assignments", "seat:mesh-module-assigner.views.seat-holders"}; !reflect.DeepEqual(sorted(m.Consumes), want) {
|
||||
"mesh-delivery.source-moved", "mesh-pipeline-runner.pipeline-ended", "mesh-pipeline-runner.pipeline-started", "mesh-pipeline-runner.run-started", "mesh-wants.version-delivered", "mesh-wants.wait-ended", "mesh-wants.wanted", "node-build-go.step-ended", "node-engine.running", "npm-package-registry.published", "operator-channel.decided", "seat:mesh-module-assigner.views.assignments", "seat:mesh-module-assigner.views.seat-holders"}; !reflect.DeepEqual(sorted(m.Consumes), want) {
|
||||
t.Errorf("consumes = %v, want %v", m.Consumes, want)
|
||||
}
|
||||
if b, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 || b != "${dir:state}/broker" {
|
||||
@@ -182,13 +186,11 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
|
||||
"seat:node-build-node.steps", "seat:node-build-image.steps", "mesh-delivery-builder.builder_versions"}; !reflect.DeepEqual(m.Invokes, want) {
|
||||
t.Errorf("invokes = %v, want %v", m.Invokes, want)
|
||||
}
|
||||
// 1945: indexer_take_in, served by its tools bundle through the takeins state; the process serves none. The same bundle
|
||||
// reads the verdicts state (indexer_verdicts) and asks the indexer to judge again through the takeins state
|
||||
// (indexer_judge; issue 774, clause 3). 1966/AC11: indexer_old_sections reads the count of the old sections (state oldsections).
|
||||
// ADR 0621 §1, ADR 0622 §1, ADR 0631 §1, ADR 0663 §1: the first-view tools manifests, modules, verdicts and graph (modules
|
||||
// 1945: indexer_take_in, served by its tools bundle through the takeins state; the process serves none. 1966/AC11: indexer_old_sections reads the count of the old sections (state oldsections).
|
||||
// ADR 0621 §1, ADR 0622 §1, ADR 0631 §1, ADR 0663 §1: the first-view tools manifests, modules and graph (modules
|
||||
// kept for the grants part until it reads the catalog half, design 47 G A-354), and design 25 § 12.2's read
|
||||
// tools, all of the tools bundle, which runs in a runtime of its own on the indexer's account (runs-as).
|
||||
want := []string{"indexer_take_in", "indexer_old_sections", "indexer_verdicts", "indexer_judge"}
|
||||
want := []string{"indexer_take_in", "indexer_old_sections"}
|
||||
for _, v := range append(append([]index.ViewedTool{}, index.FirstViewTools...), index.ReadTools...) {
|
||||
want = append(want, v.Tool)
|
||||
// Milestone 47/G: modules and graph are the newest catalog graph revision's, which catalog-revision says the changes of.
|
||||
@@ -196,14 +198,14 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
|
||||
t.Errorf("%s answers the state %s, which the views family does not say", v.Tool, v.State)
|
||||
}
|
||||
}
|
||||
want = append(want, "indexer_first_views", "indexer_first_view_again", index.ConfigReadersTool, index.BusStateTool, index.RequirementsTool, index.PackageUsersTool, index.SayAcceptedTool, // DLV-F3r, DLV-I10, DLV-I11, DLV-I7, DLV-I8, DLV-I9c
|
||||
want = append(want, "indexer_first_views", "indexer_first_view_again", index.ConfigReadersTool, index.BusStateTool, index.RequirementsTool, index.PackageUsersTool, // DLV-F3r, DLV-I10, DLV-I11, DLV-I7, DLV-I8, DLV-I9c
|
||||
index.SeatDefinitionTool, index.SeatDefinitionsAgainTool, index.OutOfStepTool, // DLV-I19
|
||||
index.PendingTool, index.TakeInPendingTool, index.DropPendingTool, index.TakeInMergeTool, index.FingerprintsAgainTool, // issue 1966; 1585/T12, T13 (ADR 0633 §5)
|
||||
// milestone 47/G W4.12 (design 47, the indexer's tools): the catalog graph revisions read and steered.
|
||||
index.CatalogRevisionTool, index.CatalogRevisionsTool, index.CandidateGraphsTool, index.RecomputeTool, index.DropCandidateGraphTool,
|
||||
index.EntryIndexTool, index.NamedDropTool, // design 47 G §675
|
||||
index.ProposalsTool, index.ProposalDropTool, index.SeedAgainTool, index.InputsTool, index.PinsTool, index.UnpinTool, index.ConsumerTool, index.RunsTool, index.RunDropTool) // design 47 G §667, §8, §579, §64 (A-194)
|
||||
if !reflect.DeepEqual(m.Tools, want) || len(want) != 50 || !slices.Contains(m.Tools, "modules") {
|
||||
if !reflect.DeepEqual(m.Tools, want) || len(want) != 45 || !slices.Contains(m.Tools, "modules") {
|
||||
t.Errorf("tools = %v, want %v", m.Tools, want)
|
||||
}
|
||||
// Design 47 G §2, §9: every version of every registration is kept with no age limit, 64 MiB, on the old streams section.
|
||||
@@ -284,7 +286,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
|
||||
// ADR 0636 §4: seat-definition-changed and -removed are the seat mesh-catalog's events, said on the seat's subject and kept
|
||||
// on the shared events stream, and not among its own emits.
|
||||
// Design 47 G A-360: modules too, at each new trunk revision that changes it.
|
||||
for _, state := range []string{"candidates", "findings", "graph", "heard", "manifests", "modules", "packages", "predefined", "seat-definitions", "taken", "verdicts"} {
|
||||
for _, state := range []string{"candidates", "findings", "graph", "heard", "manifests", "modules", "packages", "predefined", "seat-definitions", "taken"} {
|
||||
if !slices.Contains(m.Emits, views.EventPrefix+state) {
|
||||
t.Errorf("emits = %v: want %s%s", m.Emits, views.EventPrefix, state)
|
||||
}
|
||||
|
||||
@@ -29,11 +29,8 @@
|
||||
"tools": [
|
||||
"indexer_take_in",
|
||||
"indexer_old_sections",
|
||||
"indexer_verdicts",
|
||||
"indexer_judge",
|
||||
"manifests",
|
||||
"modules",
|
||||
"verdicts",
|
||||
"graph",
|
||||
"indexer_read_manifests",
|
||||
"indexer_read_predefined",
|
||||
@@ -45,14 +42,12 @@
|
||||
"indexer_read_takeins",
|
||||
"indexer_read_findings",
|
||||
"indexer_read_seat_definitions",
|
||||
"indexer_read_unjudged",
|
||||
"indexer_first_views",
|
||||
"indexer_first_view_again",
|
||||
"indexer_config_readers",
|
||||
"indexer_bus_state",
|
||||
"indexer_requirements",
|
||||
"indexer_package_users",
|
||||
"indexer_say_accepted",
|
||||
"indexer_seat_definition",
|
||||
"indexer_seat_definitions_again",
|
||||
"indexer_out_of_step",
|
||||
@@ -94,12 +89,9 @@
|
||||
"mesh-delivery-forge-bridge.forge.**",
|
||||
"mesh-catalog.catalog-revision",
|
||||
"mesh-delivery.source-moved",
|
||||
"mesh-delivery.pinned",
|
||||
"mesh-delivery.freed",
|
||||
"mesh-delivery.heads-replaced",
|
||||
"mesh-delivery.pins-stated",
|
||||
"mesh-wants.wanted",
|
||||
"mesh-pipeline-runner.run-started",
|
||||
"mesh-pipeline-runner.pipeline-started",
|
||||
"mesh-pipeline-runner.pipeline-ended",
|
||||
"node-engine.running",
|
||||
"operator-channel.decided",
|
||||
@@ -112,7 +104,6 @@
|
||||
{ "name": "predefined", "history": 5 },
|
||||
{ "name": "taken", "history": 1 },
|
||||
{ "name": "takeins", "history": 1 },
|
||||
{ "name": "verdicts", "history": 5 },
|
||||
{ "name": "oldsections", "history": 1 },
|
||||
{ "name": "runs", "history": 1 },
|
||||
{ "name": "own-build", "history": 1 },
|
||||
@@ -123,7 +114,6 @@
|
||||
{ "name": "requirements", "history": 5 },
|
||||
{ "name": "seat-definitions", "history": 5 },
|
||||
{ "name": "pending", "history": 1 },
|
||||
{ "name": "unjudged", "history": 1 },
|
||||
{ "name": "entry-index", "history": 1, "max-mib": 8 },
|
||||
{ "name": "named", "history": 1, "max-mib": 4 },
|
||||
{ "name": "registrations", "history": 1 },
|
||||
@@ -153,8 +143,7 @@
|
||||
"views.packages",
|
||||
"views.predefined",
|
||||
"views.seat-definitions",
|
||||
"views.taken",
|
||||
"views.verdicts"
|
||||
"views.taken"
|
||||
],
|
||||
"reads": [
|
||||
"mesh-build-go.toolchains",
|
||||
|
||||
Reference in New Issue
Block a user