I7, I6 (the pins' events): no validation at the take-in; a pipeline's events pin its revision

I7: the indexer judges no manifest. The judge pass, its verdicts and unjudged states,
the events manifest-accepted and manifest-refused, the tools verdicts, indexer_verdicts,
indexer_judge and indexer_say_accepted, and the take-in's wait for the verdicts state
go; indexer_bus_state answers from the modules state at its mark. The seed writes
validate required in every policy. oldsections stays (H-A-79).

I6, the half that follows C-R: the gate's pinned, freed, heads-replaced and
pins-stated are no longer heard. A pipeline-started pins the revision it names under
pipeline.<branch>.<source>, its pipeline-ended frees it, each on a durable of its own
stream; kept revisions are judged against the pins the state holds, never a count
(made and pins-stated go). A take-in pins its revision once, for its one pipeline.
indexer_pin_drop (§765) is still to do.

The entry's pull goes (§763). The tests of the old take-in (pending, behind, pushed)
go with it, ahead of I8. The unit suite is green except manifest_test's check that
the watchman's and the grants part's invokes of the indexer's verdicts tools are
served: their removal is the contracts developer's (H-A-79).

Builds: design 47 H §743, §745, §749, §763, §764 (the indexer); H-A-48, H-A-49, H-A-79

Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy
This commit is contained in:
2026-11-20 04:56:27 +01:00
parent 930d09cadc
commit ea28f39319
45 changed files with 302 additions and 3789 deletions
@@ -30,7 +30,7 @@ import (
type asker func(key string, value []byte) (index.TakeInAsk, error)
// Tools are the indexer's tools. read is the indexer's state of verdicts, which indexer_verdicts reads; counts the old sections' count.
func Tools(ask asker, now func() time.Time, read stdio.KeptState, counts func() (index.OldSectionsCount, error), acts meshhandactsclient.Recorder) []stdio.Tool {
func Tools(ask asker, now func() time.Time, counts func() (index.OldSectionsCount, error), acts meshhandactsclient.Recorder) []stdio.Tool {
str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) }
return []stdio.Tool{
{Name: index.OldSectionsTool,
@@ -90,55 +90,6 @@ func Tools(ask asker, now func() time.Time, read stdio.KeptState, counts func()
}
return a.Result, nil
}},
{Name: index.VerdictsTool,
Description: "The indexer's verdicts on manifests that wait or are refused (all: every verdict), each as its verdicts state holds it: " +
"the module, the digest, the state (waits: the manifest carries a field the indexer's judge does not know yet, named in " +
"unknown, and every reader holds the module as a refused one and keeps what runs; refused: the judge's problems), the " +
"judge that made it and when. A module's name narrows it to that module. A waiting manifest is judged again at each " +
"new indexer and refused only once the newest build of the indexer on the trunk still does not know the field; " +
"indexer_judge judges now.",
Input: map[string]any{"type": "object", "properties": map[string]any{
"module": map[string]any{"type": "string", "description": "only this module's verdicts"},
"all": map[string]any{"type": "boolean", "description": "list accepted verdicts too"}}},
Run: func(args map[string]any) (any, error) {
keys, err := read.Keys()
if err != nil {
return nil, fmt.Errorf("the verdicts state cannot be listed: %w", err)
}
all, _ := args["all"].(bool)
rows, unread := index.ListVerdicts(keys, func(key string) ([]byte, error) {
e, err := read.Get(key)
if err != nil || e == nil {
return nil, err
}
return e.Value, nil
}, str(args, "module"), all)
return map[string]any{"verdicts": rows, "unread": unread}, nil
}},
{Name: index.SayAcceptedTool,
Description: "Say the event manifest-accepted again now for one module, from the accepted verdict on its trunk's manifest the " +
"indexer holds (design 25 § 12.2): for a verdict the bus module is thought to have missed. A hand act, recorded through " +
"mesh-hand-acts before it is asked. Refused for a module the indexer holds no accepted verdict of (refused, set aside " +
"to wait, or not held). Answers what was said: the digest, the verdict's revision and the bus-state entries' count. A " +
"publish that failed twice is also said again by itself at the indexer's next judging pass (its finding accepted-unsaid).",
Input: map[string]any{"type": "object", "required": []string{"module", "why"}, "properties": map[string]any{
"module": map[string]any{"type": "string", "description": "the module's name, as the catalogue registers it"},
"why": map[string]any{"type": "string", "description": "why it is said again now, in plain words"}}},
Run: func(args map[string]any) (any, error) {
key, value, err := index.NewSayAccepted(str(args, "module"), str(args, "why"), stdio.CallerOf(args), now())
if err != nil {
return nil, err
}
if _, err := acts.Record(meshhandactsclient.Request{What: "said manifest-accepted of " + str(args, "module") + " again by hand",
Why: str(args, "why"), Cause: index.SayAcceptedTool, Verb: "say-accepted"}); err != nil {
return nil, fmt.Errorf("the hand act was not recorded through mesh-hand-acts, so nothing was said again: %w", err)
}
a, err := ask(key, value)
if err != nil {
return nil, err
}
return a.Result, nil
}},
{Name: index.SeatDefinitionsAgainTool,
Description: "Read a contracts module's seat definitions again now (design 26): its trunk's, at the commit the indexer " +
"holds the module at, or with pull and head a pull request's head (read with the head's own manifest, so its " +
@@ -170,27 +121,6 @@ func Tools(ask asker, now func() time.Time, read stdio.KeptState, counts func()
}
return a.Result, nil
}},
{Name: index.JudgeAgainTool,
Description: "Judge the manifests again now, as the indexer does at its start and after each merge, and answer how the module's " +
"verdict stands (waits, refused or accepted, and why). For a manifest that waits for a judge that knows its fields, or " +
"after a manifest was mended where the indexer has not heard it. A waiting manifest becomes refused here only when the " +
"running indexer is the newest build of itself the trunk holds. With a digest it judges that one manifest of the " +
"module (a candidate's, or one an older judge refused) in the catalogue the trunk holds. indexer_verdicts reads the result.",
Input: map[string]any{"type": "object", "required": []string{"module"}, "properties": map[string]any{
"module": map[string]any{"type": "string", "description": "the module's name, as the catalogue registers it"},
"digest": map[string]any{"type": "string", "description": "one manifest digest of the module to judge (indexer_verdicts names it); none judges every manifest the trunk holds"},
"why": map[string]any{"type": "string", "description": "why it is judged again now, in plain words"}}},
Run: func(args map[string]any) (any, error) {
key, value, err := index.NewJudgeAgain(str(args, "module"), str(args, "digest"), str(args, "why"), stdio.CallerOf(args), now())
if err != nil {
return nil, err
}
a, err := ask(key, value)
if err != nil {
return nil, err
}
return a.Result, nil
}},
}
}
@@ -285,7 +215,7 @@ var errReadOnly = errors.New("the tools bundle only reads this state")
func main() {
asks := askTakeIn(stdio.State(index.TakeInsState), 25*time.Second)
acts := meshhandactsclient.Recorder{Ask: stdio.Ask}
tools := Tools(asks, time.Now, stdio.State(index.VerdictsState), heldCounts(stdio.State(index.OldSectionsState)), acts)
tools := Tools(asks, time.Now, heldCounts(stdio.State(index.OldSectionsState)), acts)
// The modules and the module graph are the newest catalog graph revision's (milestone 47/G): revision.go.
revisions := bundleObjects{busstate.ObjectsOf(index.CatalogRevisionsObjects)}
// The view of the trunk is read from the indexer's own objects once when the runtime starts and again each time the seat's
@@ -84,7 +84,7 @@ func busStateTool(state func(string) views.Source) stdio.Tool {
"answer is at, so every manifest-accepted above it is newer. The bus module's keeper takes its first view from it.",
Input: map[string]any{"type": "object", "properties": map[string]any{}},
Run: func(map[string]any) (any, error) {
return index.BusStateOf(state("modules"), state(index.VerdictsState), state("manifests"), state(views.MarksState))
return index.BusStateOf(state("modules"), state("manifests"), state(views.MarksState))
}}
}
@@ -391,8 +391,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
return err
}
defer calls.Close()
// The verdicts on manifests (ADR 0578 §1): a state this delivery brings, bound when it appears.
verdicts := process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.VerdictsState), "no verdict on a manifest is written", logger.Printf)
// The published packages (ADR 0593 §2): a state this delivery brings, bound when it appears; until then every
// fingerprint takes the version line.
packages := process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.PackagesState), "no package is published, so every fingerprint takes the version line", logger.Printf)
@@ -407,9 +405,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// unanswered since, the newest toolchain-agreed and version-delivered, the artifacts published so far of a version).
heardStore := awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.HeardState), "what the indexer heard is not kept", logger.Printf)}
ownBuildState := process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.OwnBuildStateName), "which build this indexer runs is not kept", logger.Printf)
// The manifests a take-in wrote and no judge pass judged yet (issue 1966): a state this delivery brings, bound when it
// appears; until then a take-in's new manifests are kept for their judging in its memory alone.
unjudged := process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.UnjudgedState), "a take-in's new manifests are kept for their judging in its memory alone", logger.Printf)
// The runs on request the indexer holds a candidate for (A-174): a state this delivery brings, bound when it appears.
runs := process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.RunsState), "no run on request's candidate is written or dropped", logger.Printf)
findings := process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.FindingsState), "no finding of the indexer is written", logger.Printf)
@@ -445,7 +440,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
named := index.NewNamed(awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.NamedState), "no entry is kept for a want or a running build that names it", logger.Printf)}, logger.Printf)
pinsBucket := process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.PinsState), "no record's pin on a catalog graph revision is kept, so no revision is removed", logger.Printf)
pinsStore := awaitedStore{b: pinsBucket}
consumers := process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, process.ConsumersState), "indexer_consumer reads no consumer", logger.Printf)
// The state inputs and the object store beside it; the seat mesh-catalog's verb input is answered here in chunks of the core
// connection, of any size (design 47 G §712, A-237).
inputsStore := awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.InputsState), "no input of a run ask is kept, so the verb input answers none", logger.Printf)}
@@ -456,8 +450,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
return fmt.Errorf("answering the verb input: %w", err)
}
defer stopInput()
ix := &index.Indexer{Module: cred.Module, NamedSets: named, Findings: writer, OldSections: awaitedStore{b: oldSections}, Unjudged: awaitedStore{b: unjudged}, Requirements: awaitedStore{b: requirements}, Store: kvStore{candidates}, Modules: modules, Manifests: kvStore{manifests}, Taken: awaitedStore{b: taken}, Builds: known,
Verdicts: awaitedStore{b: verdicts}, SayRefused: sayRefused(conn, cred, logger.Printf), SayAccepted: sayAccepted(conn, cred),
ix := &index.Indexer{Module: cred.Module, NamedSets: named, Findings: writer, OldSections: awaitedStore{b: oldSections}, Requirements: awaitedStore{b: requirements}, Store: kvStore{candidates}, Modules: modules, Manifests: kvStore{manifests}, Taken: awaitedStore{b: taken}, Builds: known,
Predefined: awaitedStore{b: predefined}, Logf: logger.Printf,
Named: namedVia(calls.Call), Trunk: trunkVia(calls.Call), BranchHead: branchHeadVia(calls.Call), Contains: containsVia(calls.Call), Read: readVia(calls.Call),
ReadOpen: openVia(calls.Call), Packages: awaitedStore{b: packages, js: js, name: index.Bucket(cred.Module, index.PackagesState)}, Files: filesVia(calls.Call), GoSum: goSumVia(calls.Call), GoMod: goModVia(calls.Call),
@@ -667,35 +660,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// indexer_first_views and as view-waits past five minutes, and read again on the next forge message heard.
defs := gitWaits(ctx, cred.Module, ix, writer, cred.Node, process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, views.StatusState), "indexer_first_views does not say the wait for the seat git", logger.Printf), logger.Printf)
seatDefs.Ready(func(jetstream.KeyValue) { defs.Start(ctx) })
// Every manifest judged at the start, so a judge that changed with this program judges every one again (ADR 0578
// §1), once the verdicts state is there; each merge judges them again (index.ApplyMessage).
verdicts.Ready(func(jetstream.KeyValue) {
// A take-in held because the verdicts state was not on the bus yet is evaluated again once the start's pass ended,
// whatever it answered (issue 1966 note 154).
defer func() {
if err := ix.PendingAgain(ctx); err != nil {
logger.Printf("take-in: pending is not evaluated again at the verdicts state's bind: %v", err)
}
}()
// A pass that fails (a read the bus did not answer) is judged again at the next message or source revision the
// indexer hears (index.kickJudge): no timer.
judge := func(fromKept bool) {
j, err := ix.JudgeAtStart(ctx)
if err != nil {
logger.Printf("judging the manifests at start (from kept definitions: %v): %v; judged again at the next message heard", fromKept, err)
return
}
logger.Printf("judged %d manifests at start (from kept definitions: %v): %d verdicts written, %d with problems, of which outgrown and kept accepted: %v, waiting for a judge that knows their fields: %v",
j.Judged, fromKept, j.Written, len(j.Refused), j.Outgrown, j.Waiting)
// A take-in held while a state was not on the bus is evaluated again at the start (issue 1966 note 154).
go func() {
if err := ix.PendingAgain(ctx); err != nil {
logger.Printf("take-in: pending is not evaluated again at the start: %v", err)
}
if !defs.Waiting() {
judge(false)
return
}
// The seat git did not answer: the judging waits for it at most index.GitWaitBound, off the start, so nothing else of
// the start is held behind it.
go func() { judge(defs.Ready(ctx)) }()
})
}()
// A build module says each time it wrote its toolchains key: its stream, and the indexer's durable consumer on it, are
// made by the controller as the forge's are. Where the stream is not there yet the indexer says so and is started again
// with the build module, as with the gate's source.
@@ -728,22 +698,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
if err := hub.Handle(meshdelivery.SubjectSourceMovedAll, func(m jetstream.Msg) { source.heard(ctx, m) }); err != nil {
return err
}
pinsLedger := process.NewConsumerLedger(process.EventsStream, cred.EventConsumerName(), consumerSink(consumers), nil, logger.Printf)
for _, subject := range pinsEvents {
if err := hub.Handle(subject, func(m jetstream.Msg) { heardPins(ctx, ix, pinsLedger, m, logger.Printf) }); err != nil {
return err
}
}
// Once the state is bound the pins are made and every kept revision judged (§579: at its start).
pinsBucket.Ready(func(jetstream.KeyValue) {
go func() {
if err := ix.PinsMade(ctx); err != nil {
logger.Printf("pins: the state %s is not made: %v", index.PinsState, err)
return
}
ix.JudgePins(ctx)
}()
})
// Once the state is bound every kept revision is judged against the pins it holds (design 47 H §749: at its start).
pinsBucket.Ready(func(jetstream.KeyValue) { go ix.JudgePins(ctx) })
versions := func(m jetstream.Msg) {
if !heardVersion(ctx, ix, m, logger.Printf) {
_ = m.Ack() // nothing else the indexer consumes rides the shared stream
@@ -819,9 +775,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// said up to; a first view (the tools bundle) answers at that mark.
startViews(ctx, js, cred, marks, writer,
map[string]readyOf{"manifests": boundNow(manifests), "candidates": boundNow(candidates),
index.VerdictsState: awaited(verdicts), index.PredefinedState: awaited(predefined),
index.PackagesState: awaited(packages),
index.HeardState: awaited(heard.b), "taken": awaited(taken), "findings": awaited(findings),
index.PredefinedState: awaited(predefined),
index.PackagesState: awaited(packages),
index.HeardState: awaited(heard.b), "taken": awaited(taken), "findings": awaited(findings),
index.SeatDefinitionsState: awaited(seatDefs)}, errs, logger.Printf)
cc, err := process.HearEvents(cons, func(m jetstream.Msg) {
handle(ctx, m, ix, heard, logger)
@@ -225,3 +225,9 @@ func (f *fakeRuns) EndRunPipeline(_ context.Context, e meshpipelinerunner.Pipeli
f.ended = e.Run
return f.err
}
func (f *fakeRuns) HeardPins(context.Context, *meshpipelinerunner.PipelineStarted, *meshpipelinerunner.PipelineEnded) error {
return nil
}
func (f *fakeRuns) JudgePins(context.Context) {}
@@ -2,36 +2,36 @@ package main
import (
"context"
"encoding/json"
"time"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-indexer/internal/index"
)
// The indexer's durable consumer of mesh-delivery's pinned, freed, heads-replaced and pins-stated (design 47 G §153, §216, §579):
// each event is applied to the state pins, and only then acknowledged, so one said while the indexer was down is applied when it
// comes back. The consumer's facts are kept in the state consumers (A-176) for indexer_consumer.
// The indexer pins a pipeline's revision on the runner's pipeline-started and frees it on its pipeline-ended, each heard on
// a durable consumer of its own on that event's stream (design 47 H §749, runs.go): an event is applied to the state pins,
// and only then acknowledged, so one said while the indexer was down is applied when it comes back.
// pinsEvents are the subjects the indexer applies to its pins.
var pinsEvents = []string{meshdelivery.SubjectPinned, meshdelivery.SubjectFreed, meshdelivery.SubjectHeadsReplaced, meshdelivery.SubjectPinsStated}
// pinsStreams are the streams of the runner's events the indexer's pins follow.
var pinsStreams = []string{meshpipelinerunner.StreamPipelineStarted, meshpipelinerunner.StreamPipelineEnded}
// pendingOf says how many messages of the module's durable consumer of the events stream have not been given to it yet.
// pendingOf says how many messages of the indexer's durable consumers of the pins' streams have not been given to it yet.
func pendingOf(js jetstream.JetStream, cred process.Account) func(context.Context) (uint64, error) {
return func(ctx context.Context) (uint64, error) {
cons, err := cred.EventConsumer(ctx, js, process.EventsStream)
if err != nil {
return 0, err
var n uint64
for _, stream := range pinsStreams {
cons, err := cred.EventConsumer(ctx, js, stream)
if err != nil {
return 0, err
}
info, err := cons.Info(ctx)
if err != nil {
return 0, err
}
n += info.NumPending
}
info, err := cons.Info(ctx)
if err != nil {
return 0, err
}
return info.NumPending, nil
return n, nil
}
}
@@ -46,20 +46,3 @@ func consumerSink(b *process.AwaitedBucket) process.ConsumerSink {
return err
}
}
// heardPins applies one pins event, acknowledges it, and judges the revisions kept: an event not applied is left unacknowledged, so
// the bus gives it again.
func heardPins(ctx context.Context, ix *index.Indexer, ledger *process.ConsumerLedger, m jetstream.Msg, logf func(string, ...any)) {
ledger.Delivered(m)
if err := ix.HeardPins(ctx, m.Subject(), m.Data()); err != nil {
logf("pins: %s is not applied, left for the bus to give again: %v", m.Subject(), err)
return
}
_ = m.Ack()
var body struct {
At time.Time `json:"at"`
}
_ = json.Unmarshal(m.Data(), &body)
ledger.Applied(m, body.At)
ix.JudgePins(ctx)
}
@@ -13,10 +13,13 @@ import (
"mesh-delivery-indexer/internal/index"
)
// runTaker is what the indexer does with the runner's events of a run on request (index.Indexer.StartRun, EndRunPipeline).
// runTaker is what the indexer does with the runner's events: a run on request's (index.Indexer.StartRun, EndRunPipeline),
// and a pipeline's pins (HeardPins, design 47 H §749), the kept revisions judged after each pin freed (JudgePins).
type runTaker interface {
StartRun(ctx context.Context, e meshpipelinerunner.RunStarted) error
EndRunPipeline(ctx context.Context, e meshpipelinerunner.PipelineEnded) error
HeardPins(ctx context.Context, started *meshpipelinerunner.PipelineStarted, ended *meshpipelinerunner.PipelineEnded) error
JudgePins(ctx context.Context)
}
// hearRuns hears the runner's run-started and pipeline-ended (design 47 G §64, §190; A-160, A-174), each on its own stream, on
@@ -28,6 +31,7 @@ func hearRuns(ctx context.Context, js jetstream.JetStream, cred process.Account,
each func(jetstream.Msg)
}{
{meshpipelinerunner.StreamRunStarted, func(m jetstream.Msg) { heardRunStarted(ctx, ix, m, logf) }},
{meshpipelinerunner.StreamPipelineStarted, func(m jetstream.Msg) { heardPipelineStarted(ctx, ix, m, logf) }},
{meshpipelinerunner.StreamPipelineEnded, func(m jetstream.Msg) { heardPipelineEnded(ctx, ix, m, logf) }},
} {
_ = notHeard.Bind(s.stream, "no run on request's candidate is written or dropped until the indexer starts again", func() error {
@@ -53,7 +57,19 @@ func heardRunStarted(ctx context.Context, ix runTaker, m delivered, logf func(st
settleRun(m, ix.StartRun(ctx, e), "run-started of "+e.Run, logf)
}
// heardPipelineEnded takes one pipeline-ended in: of a run's pipeline, its module is kept as ended.
// heardPipelineStarted pins the revision a pipeline-started names (design 47 H §749).
func heardPipelineStarted(ctx context.Context, ix runTaker, m delivered, logf func(string, ...any)) {
var e meshpipelinerunner.PipelineStarted
if err := json.Unmarshal(m.Data(), &e); err != nil {
logf("%s: not a pipeline-started, dropped: %v", m.Subject(), err)
_ = m.Term()
return
}
settleRun(m, ix.HeardPins(ctx, &e, nil), "pipeline-started of "+e.Pipeline, logf)
}
// heardPipelineEnded takes one pipeline-ended in: its revision's pin is freed and the kept revisions judged (§749); of a
// run's pipeline, its movers are kept as ended.
func heardPipelineEnded(ctx context.Context, ix runTaker, m delivered, logf func(string, ...any)) {
var e meshpipelinerunner.PipelineEnded
if err := json.Unmarshal(m.Data(), &e); err != nil {
@@ -61,7 +77,12 @@ func heardPipelineEnded(ctx context.Context, ix runTaker, m delivered, logf func
_ = m.Term()
return
}
settleRun(m, ix.EndRunPipeline(ctx, e), "pipeline-ended of "+e.Module+" in the run "+e.Run, logf)
if err := ix.HeardPins(ctx, nil, &e); err != nil {
settleRun(m, err, "pipeline-ended of "+e.Pipeline, logf)
return
}
settleRun(m, ix.EndRunPipeline(ctx, e), "pipeline-ended of "+e.Pipeline+" in the run "+e.Run, logf)
ix.JudgePins(ctx)
}
func settleRun(m delivered, err error, what string, logf func(string, ...any)) {
@@ -16,8 +16,6 @@ type takerIn interface {
TakeIn(ctx context.Context, module string) index.TakenIn
TakeInRepository(ctx context.Context, repository string) index.TakenIn
CatchUp(ctx context.Context) index.TakenIn
JudgeAgain(ctx context.Context, module, digest string) index.TakenIn
SayAcceptedAgain(ctx context.Context, module string) index.TakenIn
SeatDefinitionsAgain(ctx context.Context, module string, pull int, head string) index.TakenIn
TakeInPendingByHand(ctx context.Context, key, by, why string) index.TakenIn
DropPending(ctx context.Context, key, by, why string, abandon bool) index.TakenIn
@@ -88,10 +86,6 @@ func takenIn(ctx context.Context, ix takerIn, kv jetstream.KeyValue, e jetstream
a.Module = a.Repository
} else if a.SeatDefinitions {
asked, r = "reading of its seat definitions again", ix.SeatDefinitionsAgain(ctx, a.Module, a.Pull, a.Head)
} else if a.Say {
asked, r = "manifest-accepted said again", ix.SayAcceptedAgain(ctx, a.Module)
} else if a.Judge {
asked, r = "judging again", ix.JudgeAgain(ctx, a.Module, a.Digest)
} else {
r = ix.TakeIn(ctx, a.Module)
}
@@ -2,67 +2,14 @@ package main
import (
"context"
"encoding/json"
"fmt"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
"git.novox.be/novox/mesh-sdk/go/process"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
)
// sayRefused says the indexer's event manifest-refused (novox/hq ADR 0578 §1): mesh.seat.mesh-catalog.event.manifest-refused,
// with the SDK's envelope headers and the digest and judge as the bus's dedup id, so one refusal is said once.
func sayRefused(conn *nats.Conn, cred process.Account, logf func(string, ...any)) func(context.Context, meshcatalog.ManifestRefusedEvent) {
return func(ctx context.Context, r meshcatalog.ManifestRefusedEvent) {
data, err := json.Marshal(r)
if err != nil {
logf("judge: %s for %s is not said: %v", meshcatalog.EventManifestRefused, r.Module, err)
return
}
msg := nats.NewMsg(seatEventSubject(meshcatalog.EventManifestRefused))
id := "manifest-refused-" + r.Digest + "-" + r.Judge
msg.Header.Set("x-event-id", id)
msg.Header.Set("x-source", cred.Module)
msg.Header.Set("x-node", cred.Node)
msg.Header.Set("x-time", time.Now().UTC().Format(time.RFC3339Nano))
msg.Header.Set("content-type", "application/json")
msg.Header.Set(nats.MsgIdHdr, id)
msg.Data = data
// An event is published, never asked (issue 1966): the stream that keeps the subject takes it and drops a
// duplicate by its id; a bus refusing the subject says so on the connection's error handler, not by a wait here.
if err := conn.PublishMsg(msg); err != nil {
logf("judge: %s", fmt.Errorf("saying %s for %s: %w", meshcatalog.EventManifestRefused, r.Module, err))
}
}
}
// sayAccepted says the indexer's event manifest-accepted (issue 1966 D3, novox/hq ADR 0595 §4):
// mesh.seat.mesh-catalog.event.manifest-accepted, carrying the module's bus-state entries whole, with the digest and judge as
// the bus's dedup id (meshcatalog.ManifestAcceptedEvent.EventID), so one acceptance is said once.
func sayAccepted(conn *nats.Conn, cred process.Account) func(context.Context, meshcatalog.ManifestAcceptedEvent, string) error {
return func(ctx context.Context, a meshcatalog.ManifestAcceptedEvent, id string) error {
data, err := json.Marshal(a)
if err != nil {
return err
}
msg := nats.NewMsg(seatEventSubject(meshcatalog.EventManifestAccepted))
msg.Header.Set("x-event-id", id)
msg.Header.Set("x-source", cred.Module)
msg.Header.Set("x-node", cred.Node)
msg.Header.Set("x-time", time.Now().UTC().Format(time.RFC3339Nano))
msg.Header.Set("content-type", "application/json")
msg.Header.Set(nats.MsgIdHdr, id)
msg.Data = data
// Published, never asked (issue 1966): the event's stream takes it and drops a duplicate by its id; waiting on
// the stream's acknowledgement held each judging pass twenty seconds a manifest on a bus refusing the subject.
// A refusal is said on the connection's error handler; an error here (no connection) is the caller's to retry.
return conn.PublishMsg(msg)
}
}
// sayTakeIn says a take-in event (novox/hq ADR 0633 §5) as the indexer's ordinary event, on the path its other events
// take: mesh.mod.<module>.event.take-in.<kind>, the wait's key in the body. A publish that fails is the caller's to say.
func sayTakeIn(js jetstream.JetStream, cred process.Account) func(context.Context, takein.TakeIn) error {
@@ -183,6 +183,5 @@ func gitWaits(ctx context.Context, module string, ix *index.Indexer, w index.Fin
f := &views.Follower{Owner: git.Seat, Tool: "seat:" + git.Seat + "." + git.VerbTrees, State: index.SeatDefinitionsState, Logf: logf,
RaiseWaits: func(s views.Status) { finding(s, true) }, ClearWaits: func(s views.Status) { finding(s, false) }}
status.Ready(func(kv jetstream.KeyValue) { f.Said = views.KeepStatus(ctx, kv, logf) })
return &index.DefinitionsAtStart{Read: ix.ReadTrunkDefinitions, Waits: f, Logf: logf,
Judge: func(ctx context.Context) error { _, err := ix.JudgeAtStart(ctx); return err }}
return &index.DefinitionsAtStart{Read: ix.ReadTrunkDefinitions, Waits: f, Logf: logf}
}
@@ -1,190 +0,0 @@
package index
import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"time"
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-sdk/go/keys"
"mesh-delivery-indexer/judge"
)
// manifest-accepted, never lost silently (design 25 § 12.2; DLV-I9, I9b, I9c). The indexer says it for every newly
// accepted manifest, its bus-state entries empty when it asks for none, and an empty one for a module leaving the trunk,
// so the bus module's orphans see it; a manifest the judge set aside to wait gets none. A publish that fails is published
// once more; one still failing writes the indexer's finding naming the module, said again at each judging pass after
// and cleared once said. indexer_say_accepted says it again for one module by hand, from the verdict it holds.
const (
SayAcceptedTool = "indexer_say_accepted"
sayAcceptedPrefix = "say."
// UnsaidKind is the finding of a manifest-accepted whose publish failed twice.
UnsaidKind = "accepted-unsaid"
)
// acceptedOf is the event body for the verdict v (its revision in the verdicts state) on the manifest raw of digest, its
// entries an empty list when it asks for none.
func acceptedOf(digest string, v meshcatalog.VerdictsByDigestValue, revision uint64, raw []byte) meshcatalog.ManifestAcceptedEvent {
m, _ := judge.Manifest(raw, nil)
a := acceptedEvent(digest, v, m)
a.Revision = int64(revision)
return a
}
// leftOf is the empty manifest-accepted for module, which left the trunk, judged by judgeID; revision is the modules
// state's revision of the value that left (0 when none is held), so its event id (acceptedEventID, the
// module and the revision, since its body names no digest) differs for each leave.
func leftOf(module string, revision uint64, judgeID string) (meshcatalog.ManifestAcceptedEvent, string) {
a := meshcatalog.ManifestAcceptedEvent{V: keys.Version, Module: module, Judge: judgeID, BusState: busStateOf(nil), Revision: int64(revision)}
return a, acceptedEventID(a)
}
// sayAccepted says a with the event id id, once more when the first publish fails; the module's finding is written when
// both fail and deleted when it is said. Nil SayAccepted says nothing.
func (ix *Indexer) sayAccepted(ctx context.Context, a meshcatalog.ManifestAcceptedEvent, id string) error {
if ix.SayAccepted == nil {
return nil
}
err := ix.SayAccepted(ctx, a, id)
if err != nil {
ix.logf("judge: %s for %s failed, published again: %v", meshcatalog.EventManifestAccepted, a.Module, err)
err = ix.SayAccepted(ctx, a, id)
}
ix.unsaid(ctx, a.Module, err)
return err
}
// unsaidID is the id of module's unsaid finding.
func unsaidID(own, module string) string { return own + ".accepted." + module }
// unsaid raises module's finding for err, or clears it with none.
func (ix *Indexer) unsaid(ctx context.Context, module string, err error) {
if ix.Findings == nil {
return
}
var werr error
if err == nil {
werr = ix.clearFinding(ctx, mcc.ScopeModule, unsaidID(ix.Module, module), UnsaidKind, "manifest-accepted of "+module+" was said.")
} else {
werr = ix.Findings.Write(ctx, ix.observed(mcc.ScopeModule, unsaidID(ix.Module, module), UnsaidKind,
fmt.Sprintf("%s of %s was not published: %v", meshcatalog.EventManifestAccepted, module, err),
"The indexer could not tell the bus module that "+module+"'s manifest was accepted",
"The bus module creates and removes a module's bus state from this event. The indexer says it again at its "+
"next judging pass and clears this once it is said; indexer_say_accepted says it now.", UnsaidKind))
}
if werr != nil {
ix.logf("judge: the finding of %s for %s is not written: %v", meshcatalog.EventManifestAccepted, module, werr)
}
}
// sayFor says module's manifest-accepted from what the indexer holds now: from its trunk's accepted verdict, or an empty
// one when the trunk holds the module no more (only when left is allowed: by hand a module with no accepted verdict is
// refused). A held module whose verdict waits or is refused gets none; again gives the event an id of its own, so a hand
// act within the bus's duplicate window is still said.
func (ix *Indexer) sayFor(ctx context.Context, module string, allowLeft, again bool) (string, error) {
held, err := ix.heldModules(ctx)
if err != nil {
return "", fmt.Errorf("the modules state cannot be read: %w", err)
}
m, ok := held[module]
var a meshcatalog.ManifestAcceptedEvent
var id string
switch {
case !ok && !allowLeft:
return "", fmt.Errorf("the indexer holds no module %s, so it has no accepted verdict to say", module)
case !ok:
a, id = leftOf(module, 0, ix.judgeID())
case m.Manifest == "" || ix.Verdicts == nil || ix.Manifests == nil:
return "", fmt.Errorf("%s has no accepted verdict the indexer holds", module)
default:
key, err := meshcatalogclient.VerdictsKey(m.Manifest)
if err != nil {
return "", err
}
value, revision, found, err := ix.Verdicts.Get(ctx, key)
if err != nil {
return "", fmt.Errorf("the verdict on %s cannot be read: %w", module, err)
}
var v meshcatalog.VerdictsByDigestValue
if found {
v, err = meshcatalogclient.DecodeVerdict(value)
}
if !found || err != nil || meshcatalogclient.VerdictHeld(v) {
if !allowLeft {
return "", fmt.Errorf("%s has no accepted verdict on its manifest %s", module, m.Manifest)
}
ix.unsaid(ctx, module, nil) // a set-aside or refused manifest gets none
return "none: its manifest is not accepted", nil
}
raw, _, found, err := ix.Manifests.Get(ctx, m.Manifest)
kept, derr := meshcatalogclient.DecodeManifest(raw)
if err != nil || !found || derr != nil {
return "", fmt.Errorf("the manifest %s of %s cannot be read", m.Manifest, module)
}
a = acceptedOf(m.Manifest, v, revision, []byte(kept.Manifest))
id = acceptedEventID(a)
}
if again {
id += "-again-" + ix.now().UTC().Format(time.RFC3339Nano)
}
if err := ix.sayAccepted(ctx, a, id); err != nil {
return "", fmt.Errorf("%s of %s was not published: %w", meshcatalog.EventManifestAccepted, module, err)
}
if a.Digest == "" {
return "said empty: the trunk holds no module " + module, nil
}
return fmt.Sprintf("said for %s, manifest %s at revision %d, %d bus-state entries", module, a.Digest, a.Revision, len(a.BusState)), nil
}
// SayUnsaid says again each manifest-accepted whose finding stands (the judging pass's end): cleared once said. What stands is
// what the finding writer holds (design 47 G A-38).
func (ix *Indexer) SayUnsaid(ctx context.Context) {
if ix.Findings == nil {
return
}
for _, f := range ix.Findings.Held() {
if f.Kind != UnsaidKind {
continue
}
module := strings.TrimPrefix(f.Subject.ID, ix.Module+".accepted.")
if _, err := ix.sayFor(ctx, module, true, false); err != nil {
ix.logf("judge: %v; its finding stands", err)
}
}
}
// SayAcceptedAgain answers indexer_say_accepted: module's manifest-accepted said again now from its accepted verdict.
func (ix *Indexer) SayAcceptedAgain(ctx context.Context, module string) TakenIn {
out := TakenIn{Module: module}
answer, err := ix.sayFor(ctx, module, false, true)
if err != nil {
out.Answer = err.Error()
return out
}
out.Stored, out.Answer = true, answer
return out
}
// NewSayAccepted is the value indexer_say_accepted writes for a module: its key in the takeins state and the value.
func NewSayAccepted(module, why, by string, at time.Time) (string, []byte, error) {
key, data, err := NewTakeIn(module, why, by, at)
if err != nil {
return "", nil, err
}
var a TakeInAsk
if err := json.Unmarshal(data, &a); err != nil {
return "", nil, err
}
if a.Why == "" {
return "", nil, errors.New("say why it is said again (why): it is a hand act")
}
a.Say = true
data, err = json.Marshal(a)
return sayAcceptedPrefix + key, data, err
}
@@ -1,25 +0,0 @@
package index
import (
"encoding/json"
"reflect"
"strings"
"testing"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
)
// Design 47 G §668, §692: manifest-accepted's problems are the validator's strings, the same the verdict writes as
// warnings, in the same order; a mended manifest says an empty list, never null.
func TestManifestAcceptedProblemsAreTheVerdictsWarnings(t *testing.T) {
warnings := []string{`requires "amqp", which no module provides`, "a second problem"}
got := acceptedOf("sha256:aa", meshcatalog.VerdictsByDigestValue{Module: "m", Judge: "j", Warnings: warnings}, 3, []byte(`{"module":"m","version":"1.0"}`))
if !reflect.DeepEqual(got.Problems, warnings) {
t.Errorf("problems %v, want the verdict's warnings %v", got.Problems, warnings)
}
mended := acceptedOf("sha256:bb", meshcatalog.VerdictsByDigestValue{Module: "m", Judge: "j"}, 4, []byte(`{"module":"m","version":"2.0"}`))
raw, _ := json.Marshal(mended)
if !strings.Contains(string(raw), `"problems":[]`) {
t.Errorf("a mended manifest's event is %s, want an empty problems list", raw)
}
}
@@ -1,112 +0,0 @@
package index
import (
"context"
"encoding/json"
"testing"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk"
)
// Design 47 G A-463: a merge the indexer never took in is noticed at a source revision and taken in as its merged message
// would have been, with a finding raised while it stays behind and cleared once it is taken in.
func TestAMergeTheIndexerNeverTookInIsTakenInAndSaidWhileItIsBehind(t *testing.T) {
ctx := context.Background()
b := newTakeInBed(t)
b.ix.Taken, b.ix.Findings = newMem(), newFindings()
key, _ := trunk.RepositoryKey(catalogueRepo)
old, head := sha('7'), sha('8')
data, _ := takein.EncodeTaken(takein.Taken{Repository: trunk.RepositoryName(catalogueRepo), Merge: old})
b.ix.Taken.(*memStore).put(key, data)
ancestry := map[[2]string]bool{{old, head}: true} // old is in head
b.ix.Contains = func(_ context.Context, _, commit, in string) (bool, error) {
return commit == in || ancestry[[2]string{commit, in}], nil
}
b.ix.source = sourceNow{rev: 3, heads: map[string]string{catalogueRepo: head}}
// The head has no merged message yet: nothing is taken in and the finding stays.
if done := b.ix.TakeInBehind(ctx); len(done) != 0 || len(b.ix.Findings.Held()) != 1 || b.ix.Findings.Held()[0].Kind != UntakenMergeKind {
t.Fatalf("a head whose message is not read: taken in %v, findings %+v: want none taken in and %s raised", done, b.ix.Findings.Held(), UntakenMergeKind)
}
// Its message is there: it is taken in, its taken key moves and the finding is cleared.
b.publish(t, b.merge(30, head, "tree-avahi-3", ""))
if done := b.ix.TakeInBehind(ctx); len(done) != 1 || len(b.ix.Findings.Held()) != 0 || b.commit(t, "avahi") != head {
t.Fatalf("a head with its message: taken in %v, findings %+v, avahi at %s: want it taken in at %s and the finding cleared", done, b.ix.Findings.Held(), b.commit(t, "avahi"), head)
}
if v, _, _, _ := b.ix.Taken.Get(ctx, key); string(v) == string(data) {
t.Errorf("taken.%s still names %s", key, old)
}
// A head the taken merge already holds (the source not yet moved to a merge just taken in) is left.
b.ix.source = sourceNow{rev: 4, heads: map[string]string{catalogueRepo: old}}
if done := b.ix.TakeInBehind(ctx); len(done) != 0 {
t.Errorf("a head older than the taken merge: taken in %v, want none", done)
}
// A source revision starts the comparison by itself (no timer): the taken key is put back to the old merge and the source
// names the head again.
b.ix.Taken.(*memStore).put(key, data)
if err := applyS(b.ix, ctx, 5, sourceOf(catalogueRepo, head)); err != nil {
t.Fatal(err)
}
b.ix.catchUpWG.Wait()
if v, _, _, _ := b.ix.Taken.Get(ctx, key); string(v) == string(data) {
t.Errorf("a source revision naming %s left taken.%s at %s", head, key, old)
}
}
// Design 47 G A-463a: a head the pending holds (a group member waiting for its group's end) is not behind: it raises no finding
// and is not taken in alone.
func TestAHeadThePendingHoldsIsNotBehind(t *testing.T) {
ctx := context.Background()
b := newTakeInBed(t)
b.ix.Taken, b.ix.Findings = newMem(), newFindings()
key, _ := trunk.RepositoryKey(catalogueRepo)
old, head := sha('7'), sha('8')
data, _ := takein.EncodeTaken(takein.Taken{Repository: trunk.RepositoryName(catalogueRepo), Merge: old})
b.ix.Taken.(*memStore).put(key, data)
b.ix.Contains = func(_ context.Context, _, commit, in string) (bool, error) { return commit == in || commit == old, nil }
b.publish(t, b.merge(30, head, "tree-avahi-3", ""))
// The head waits in pending as a member of a group not ended.
held := b.merge(30, head, "tree-avahi-3", "")
raw, _ := forgemsg.EncodeForge(held)
pv, _ := json.Marshal(Pending{V: 1, Forge: raw, Group: "feat", Since: time.Now(), LastWriter: "x"})
b.pending.put(takein.PendingKey(1), pv)
b.ix.source = sourceNow{rev: 3, heads: map[string]string{catalogueRepo: head}}
if done := b.ix.TakeInBehind(ctx); len(done) != 0 || len(b.ix.Findings.Held()) != 0 || b.commit(t, "avahi") == head {
t.Fatalf("a head the pending holds: taken in %v, findings %+v, avahi at %s: want none taken in and none raised", done, b.ix.Findings.Held(), b.commit(t, "avahi"))
}
}
// Design 47 G A-463a: a source revision heard while a catch-up runs is compared again when that catch-up ends.
func TestASourceRevisionHeardDuringACatchUpIsComparedWhenItEnds(t *testing.T) {
ctx := context.Background()
b := newTakeInBed(t)
b.ix.Taken, b.ix.Findings = newMem(), newFindings()
key, _ := trunk.RepositoryKey(catalogueRepo)
h1, h2 := sha('8'), sha('9')
b.ix.Contains = func(_ context.Context, _, commit, in string) (bool, error) {
return commit == in || (commit == h1 && in == h2), nil
}
b.publish(t, b.merge(30, h1, "tree-avahi-3", ""))
b.publish(t, b.merge(31, h2, "tree-avahi-4", ""))
b.ix.source = sourceNow{rev: 3, heads: map[string]string{catalogueRepo: h1}}
inner := b.ix.Message
heard := false
b.ix.Message = func(c context.Context, r, m string, n int) ([]byte, error) {
if !heard {
heard = true // the catch-up of h1 runs: a new source revision is heard now
if err := applyS(b.ix, ctx, 4, sourceOf(catalogueRepo, h2)); err != nil {
t.Error(err)
}
}
return inner(c, r, m, n)
}
b.ix.CatchUp(ctx)
b.ix.catchUpWG.Wait()
v, _, _, _ := b.ix.Taken.Get(ctx, key)
if got, _ := takein.DecodeTaken(v); got.Merge != h2 {
t.Errorf("taken after the catch-up = %s, want %s: the revision heard during it is compared afterwards\n%s", got.Merge, h2, b.logs)
}
}
@@ -13,8 +13,8 @@ import (
// every trunk module whose accepted manifest asks for bus state, then each manifest-accepted after its revision.
const BusStateTool = "indexer_bus_state"
// BusStateModule is one module of BusStateAnswer: the digest of its trunk's accepted manifest, the verdicts state's
// revision of that verdict (the one its manifest-accepted carries, ADR 0622 §2), and its bus-state entries whole.
// BusStateModule is one module of BusStateAnswer: the digest of its trunk's manifest, the modules state's revision of its
// value, and its bus-state entries whole. No manifest is judged at the take-in (design 47 H §743, §745).
type BusStateModule struct {
Module string `json:"module"`
Digest string `json:"digest"`
@@ -22,25 +22,25 @@ type BusStateModule struct {
BusState []catalogue.BusStateEntry `json:"bus-state"`
}
// BusStateAnswer is indexer_bus_state's answer: Revision is the verdicts state's mark, read before anything else, so
// every manifest-accepted above it is a change after this answer (ADR 0615 §4); Modules in module order.
// BusStateAnswer is indexer_bus_state's answer: Revision is the modules state's mark, read before anything else, so every
// change of modules above it is a change after this answer (ADR 0615 §4); Modules in module order.
type BusStateAnswer struct {
Revision uint64 `json:"revision"`
Modules []BusStateModule `json:"modules"`
}
// BusStateOf answers indexer_bus_state from the indexer's own states: its trunk modules, the judge's verdicts and the
// manifests it keeps, and the marks the views family says them up to. A module refused or waiting, or whose manifest
// asks for no bus state, is left out; a value that cannot be read is an error, never a module silently left out.
func BusStateOf(modules, verdicts, manifests, marks views.Source) (BusStateAnswer, error) {
// BusStateOf answers indexer_bus_state from the indexer's own states: its trunk modules and the manifests it keeps, and
// the marks the views family says them up to. A module whose manifest asks for no bus state is left out; a value that
// cannot be read is an error, never a module silently left out.
func BusStateOf(modules, manifests, marks views.Source) (BusStateAnswer, error) {
out := BusStateAnswer{Modules: []BusStateModule{}}
if marks != nil {
if e, ok, err := marks.Get(VerdictsState); err != nil {
return BusStateAnswer{}, fmt.Errorf("the mark of %s cannot be read: %w", VerdictsState, err)
if e, ok, err := marks.Get(ModulesState); err != nil {
return BusStateAnswer{}, fmt.Errorf("the mark of %s cannot be read: %w", ModulesState, err)
} else if ok {
m, err := views.DecodeMark(e.Value)
if err != nil {
return BusStateAnswer{}, fmt.Errorf("the mark of %s: %w", VerdictsState, err)
return BusStateAnswer{}, fmt.Errorf("the mark of %s: %w", ModulesState, err)
}
out.Revision = m.Revision
}
@@ -60,22 +60,7 @@ func BusStateOf(modules, verdicts, manifests, marks views.Source) (BusStateAnswe
}
mod, err := meshcatalogclient.DecodeModule(e.Value)
if err != nil || mod.Manifest == "" {
continue // no manifest held for it, so no verdict
}
key, err := meshcatalogclient.VerdictsKey(mod.Manifest)
if err != nil {
continue
}
ve, ok, err := verdicts.Get(key)
if err != nil {
return BusStateAnswer{}, fmt.Errorf("the verdict on %s cannot be read: %w", name, err)
}
if !ok {
continue
}
v, err := meshcatalogclient.DecodeVerdict(ve.Value)
if err != nil || v.State != meshcatalogclient.VerdictAccepted {
continue
continue // no manifest held for it
}
me, ok, err := manifests.Get(mod.Manifest)
if err != nil {
@@ -90,12 +75,12 @@ func BusStateOf(modules, verdicts, manifests, marks views.Source) (BusStateAnswe
}
m, err := catalogue.Decode([]byte(kept.Manifest))
if err != nil {
return BusStateAnswer{}, fmt.Errorf("the accepted manifest of %s does not read: %w", name, err)
return BusStateAnswer{}, fmt.Errorf("the manifest of %s does not read: %w", name, err)
}
if len(m.BusState) == 0 {
continue
}
out.Modules = append(out.Modules, BusStateModule{Module: name, Digest: mod.Manifest, Revision: ve.Revision, BusState: m.BusState})
out.Modules = append(out.Modules, BusStateModule{Module: name, Digest: mod.Manifest, Revision: e.Revision, BusState: m.BusState})
}
return out, nil
}
@@ -70,14 +70,9 @@ type Indexer struct {
Predefined Store
// Taken is the taken bucket: per repository the last merge taken in (taken.go, ADR 0556); nil writes none.
Taken Store
// Verdicts is the verdicts state (verdicts.go, novox/hq ADR 0578 §1); nil judges nothing.
Verdicts Store
// OldSections is the state of how many accepted modules use each old section (oldsections.go, issue 1966 D4); nil keeps no
// count and the judge refuses no use of an old section.
OldSections Store
// Unjudged is the state of the manifest digests a take-in wrote and no judge pass judged yet (unjudged.go, issue 1966);
// nil keeps them only in the take-in's memory.
Unjudged Store
// Requirements is the requirements state (requirements.go, DLV-I7): what each consumer's go.sum requires of each mesh
// Go module, kept at each merge; nil keeps none.
Requirements Store
@@ -85,12 +80,6 @@ type Indexer struct {
reqKept map[string]bool // consumers whose go.sum this process kept (requirementsKept)
// JudgeID names the judge a verdict says made it; empty is judge.Identity, the running program's.
JudgeID string
// SayRefused says the event manifest-refused for a refusal newly written; nil says nothing.
SayRefused func(ctx context.Context, r meshcatalog.ManifestRefusedEvent)
// SayAccepted publishes the event manifest-accepted a under the bus's duplicate id id (accepted.go; issue 1966 D3,
// DLV-I9): for every accepted verdict newly written, carrying the entries whole, and an empty one for a module leaving
// the trunk; nil says nothing.
SayAccepted func(ctx context.Context, a meshcatalog.ManifestAcceptedEvent, id string) error
// Findings is the indexer's finding writer (design 47 G A-29), where a manifest-accepted whose publish failed twice is kept
// until said (accepted.go, DLV-I9b); nil writes none.
Findings FindingWriter
@@ -258,7 +247,6 @@ func (ix *Indexer) Apply(ctx context.Context, f forgemsg.Forge) error {
// ApplyMessage is Apply, and answers which modules a merged message wrote and kept (for the indexer's heard state). Each
// applied merged message is said in one line: its repository, commit and pull request, and the modules written.
func (ix *Indexer) ApplyMessage(ctx context.Context, f forgemsg.Forge) (Applied, error) {
ix.kickJudge(ctx)
var applied Applied
if f.Kind == forgemsg.ForgeOpen {
return applied, ix.applyOpen(ctx, f)
@@ -287,13 +275,6 @@ func (ix *Indexer) ApplyMessage(ctx context.Context, f forgemsg.Forge) (Applied,
if err := ix.KeepPackageNames(ctx, f); err != nil {
return applied, err
}
// Every manifest judged again: a rule between manifests can turn on the one this merge moved (ADR 0578 §1).
// What this take-in newly wrote is fresh: judged in full (ADR 0579 §1).
if _, err := ix.JudgeAll(ctx, w.fresh); err != nil {
// Judged again at the next message or source revision heard (kickJudge), as a failed start pass is.
ix.judgePending.Store(true)
ix.logf("judge: the manifests are not judged after %s#%d: %v; judged again at the next message heard", f.Repository, f.Number, err)
}
}
ix.kickRetry(ctx) // a registered module the forge could not read is read again on a merge, in the background (issue 1945)
lateMerge = w.late != ""
@@ -314,20 +295,6 @@ func (ix *Indexer) ApplyMessage(ctx context.Context, f forgemsg.Forge) (Applied,
if held, err = ix.heldModules(ctx); err != nil {
return applied, err
}
// Each manifest the pull carries is judged by this judge, unless it holds a verdict already that this judge made or
// that accepts it: a refusal an older judge made never stands for a candidate (or a digest back on the trunk).
over := map[string]string{}
for _, digest := range digests {
if ix.Manifests == nil {
break
}
if value, _, found, err := ix.Manifests.Get(ctx, digest); err == nil && found {
if kept, err := meshcatalogclient.DecodeManifest(value); err == nil {
over[kept.Module] = digest
}
}
}
ix.judgeDigests(ctx, over, fmt.Sprintf("%s#%d", f.Repository, f.Number))
}
return applied, ix.writeCandidate(ctx, f, digests, held)
}
@@ -42,9 +42,9 @@ func TestTheControlRecords(t *testing.T) {
t.Errorf("rows = %+v\nwant %+v", got, want)
}
})
// DLV-I11 (design 25 § 12.2): indexer_bus_state answers every trunk module whose accepted manifest asks for bus state,
// with the verdict's revision, at the verdicts state's mark; a refused one, one asking none, and a module with no
// manifest held are left out.
// DLV-I11 (design 25 § 12.2): indexer_bus_state answers every trunk module whose manifest asks for bus state, with its
// value's revision, at the modules state's mark; one asking none and a module with no manifest held are left out. No
// verdict is read (I7, design 47 H §745): a manifest the judge once refused is answered as any other.
t.Run("indexer_bus_state", func(t *testing.T) {
src := memSource{}
dAsk, dNone, dRefused := strings.Repeat("a", 64), strings.Repeat("b", 64), strings.Repeat("c", 64)
@@ -55,35 +55,25 @@ func TestTheControlRecords(t *testing.T) {
}
s[key] = &views.Entry{Value: raw, Revision: rev}
}
modules, verdicts, manifests, marks := memSource{}, memSource{}, memSource{}, src
putVerdict := func(digest string, rev uint64, v meshcatalog.VerdictsByDigestValue) {
raw, err := meshcatalogclient.EncodeVerdict(v)
if err != nil {
t.Fatal(err)
}
verdicts[digest] = &views.Entry{Value: raw, Revision: rev}
}
modules, manifests, marks := memSource{}, memSource{}, src
for name, digest := range map[string]string{"cache": dAsk, "plain": dNone, "bad": dRefused, "bare": ""} {
raw, err := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: "src2:" + strings.Repeat("a", 64), Manifest: digest,
Repository: "novox/mesh-catalog", Commit: "c"})
if err != nil {
t.Fatal(err)
}
modules[name] = &views.Entry{Value: raw, Revision: 1}
modules[name] = &views.Entry{Value: raw, Revision: map[string]uint64{"cache": 7, "bad": 9}[name]}
}
asks := `{"module":"cache","version":"1.0","requires":[{"name":"c","storage":"key-value","template":"state"}]}`
for digest, raw := range map[string]string{dAsk: asks, dNone: `{"module":"plain","version":"1.0"}`, dRefused: asks} {
put(manifests, digest, 1, meshcatalog.ManifestsByDigestValue{V: keys.Version, Module: "m", Manifest: raw})
}
putVerdict(dAsk, 7, meshcatalog.VerdictsByDigestValue{Module: "cache", State: meshcatalogclient.VerdictAccepted, Judge: "j"})
putVerdict(dNone, 8, meshcatalog.VerdictsByDigestValue{Module: "plain", State: meshcatalogclient.VerdictAccepted, Judge: "j"})
putVerdict(dRefused, 9, meshcatalog.VerdictsByDigestValue{Module: "bad", State: meshcatalogclient.VerdictRefused, Judge: "j", Problems: []string{"p"}})
put(marks, VerdictsState, 1, views.Mark{V: views.Version, State: VerdictsState, Revision: 9})
got, err := BusStateOf(modules, verdicts, manifests, marks)
if err != nil || got.Revision != 9 || len(got.Modules) != 1 || got.Modules[0].Module != "cache" ||
got.Modules[0].Digest != dAsk || got.Modules[0].Revision != 7 || len(got.Modules[0].BusState) != 1 ||
got.Modules[0].BusState[0].Name != "c" {
t.Errorf("indexer_bus_state = %+v (%v)", got, err)
put(marks, ModulesState, 1, views.Mark{V: views.Version, State: ModulesState, Revision: 9})
got, err := BusStateOf(modules, manifests, marks)
if err != nil || got.Revision != 9 || len(got.Modules) != 2 || got.Modules[0].Module != "bad" || got.Modules[1].Module != "cache" ||
got.Modules[1].Digest != dAsk || got.Modules[1].Revision != 7 || len(got.Modules[1].BusState) != 1 ||
got.Modules[1].BusState[0].Name != "c" {
t.Errorf("indexer_bus_state = %+v (%v); want bad and cache, at the modules mark", got, err)
}
})
// DLV-I7 (REG S15, the lead's ruling): the requirements state keeps, per mesh Go module, the highest version each
@@ -120,34 +110,32 @@ func TestTheControlRecords(t *testing.T) {
// and source, says seat-definition-changed only when a digest moved, keeps the trunk's entry over one that does not
// parse (its finding), removes a pull request's entries when it closes, keeps a verdict's out-of-step files, and
// raises generated-types-out-of-step from a trunk build end, cleared only by a clean one of the same module.
// Milestone 47/G W4.15, design 47 test 54: the seat git not answering at start holds the judging at most its bound, which
// then runs on the kept definitions with the wait said; the read once the forge is heard holds the view and judges again.
t.Run("W4.15 the start waits for the seat git, judges from kept definitions, and judges again once it answers", func(t *testing.T) {
// Milestone 47/G W4.15, design 47 test 54: the seat git not answering at start is waited for at most its bound, the wait
// said; the read once the forge is heard holds the view. No manifest is judged (design 47 H §745).
t.Run("W4.15 the start waits for the seat git and reads again once it answers", func(t *testing.T) {
answers := []error{fmt.Errorf("mesh-contracts: %w", errForgeRead), fmt.Errorf("again: %w", errForgeRead), nil}
var said []string
judged := 0
d := &DefinitionsAtStart{Bound: time.Millisecond, Waits: waitsFake{&said},
Read: func(context.Context) (int, error) { err := answers[0]; answers = answers[1:]; return 1, err },
Judge: func(context.Context) error { judged++; return nil }}
Read: func(context.Context) (int, error) { err := answers[0]; answers = answers[1:]; return 1, err }}
ctx := context.Background()
d.Start(ctx)
if !d.Waiting() || len(said) != 1 || said[0] != "waits "+GitDefinitionsView {
t.Fatalf("waiting %v, said %q: want the wait for the seat git said once", d.Waiting(), said)
}
if !d.Ready(ctx) || judged != 0 {
t.Fatalf("the start past its bound: want it judged from the kept definitions, judged again %d", judged)
if !d.Ready(ctx) {
t.Fatalf("the start past its bound: want it ready on the kept definitions")
}
d.Heard(ctx) // the forge heard, the seat still not answering: the wait stands, said once
if !d.Waiting() || len(said) != 1 || judged != 0 {
t.Fatalf("a read still not answered: waiting %v, said %q, judged %d", d.Waiting(), said, judged)
if !d.Waiting() || len(said) != 1 {
t.Fatalf("a read still not answered: waiting %v, said %q", d.Waiting(), said)
}
d.Heard(ctx)
if d.Waiting() || len(said) != 2 || said[1] != "holds "+GitDefinitionsView || judged != 1 || d.Ready(ctx) {
t.Errorf("the seat answering: waiting %v, said %q, judged again %d: want the view held and every manifest judged again once", d.Waiting(), said, judged)
if d.Waiting() || len(said) != 2 || said[1] != "holds "+GitDefinitionsView || d.Ready(ctx) {
t.Errorf("the seat answering: waiting %v, said %q: want the view held", d.Waiting(), said)
}
d.Heard(ctx) // nothing waits: no read, no judging
if judged != 1 || len(answers) != 0 {
t.Errorf("a forge message with nothing waiting read or judged again")
d.Heard(ctx) // nothing waits: no read
if len(answers) != 0 {
t.Errorf("a forge message with nothing waiting read again")
}
})
t.Run("seat definitions", func(t *testing.T) {
@@ -484,23 +472,6 @@ func TestTheControlRecords(t *testing.T) {
ts.Name != "@novox/mesh-sdk" || ts.Package != "mesh-sdk/ts" || ts.GoModule != "" || len(named) != 2 {
t.Errorf("names at the merge = %+v", named)
}
// The judge's known packages (DLV-I14) are those names, never the builder's <module>/<artifact>.
if known := ix.knownPackages(ctx); len(known) != 2 || !known[sdkGo] || !known["@novox/mesh-sdk"] {
t.Errorf("the judge's known packages = %v", known)
}
// Issue 1966: a Go module the trunk holds a per-module record of is known by its module path, though no series
// names it; a package inside it is not (design 32).
resticMod, at := "git.novox.be/novox/mesh-delivery/modules/restic", time.Unix(1, 0)
rec, err := EncodePublishedPackages(meshcatalog.PackagesByModuleValue{Module: resticMod, Repository: "novox/mesh-delivery", Dir: "modules/restic", At: &at,
Commit: strings.Repeat("a", 40), Requires: "h1:r", Packages: []meshcatalog.PackagesByModuleValuePackagesItem{{Package: resticMod + "/backupgood", Digest: "h1:a"}}})
if err != nil {
t.Fatal(err)
}
ix.Packages.(*memStore).vals["restic"] = rec
if known := ix.knownPackages(ctx); len(known) != 3 || !known[resticMod] || known[resticMod+"/backupgood"] || !known[sdkGo] || known["restic/go"] || known["unknown/pkg"] {
t.Errorf("the judge's known packages with a module record = %v", known)
}
delete(ix.Packages.(*memStore).vals, "restic")
delete(ix.Packages.(*memStore).vals, SeriesKey("@novox/mesh-sdk", "0.3"))
for _, h := range []HeardVersion{
{Package: sdkGo, Seat: "go-package-registry", Version: "0.1.15", Commit: "c15"},
@@ -563,7 +534,7 @@ func TestTheControlRecords(t *testing.T) {
if err != nil {
t.Fatal(err)
}
modules[name] = &views.Entry{Value: raw, Revision: 1}
modules[name] = &views.Entry{Value: raw, Revision: map[string]uint64{"cache": 7, "bad": 9}[name]}
}
got, err := PackageUsersOf(modules, manifests, packages, "")
want := []PackageUses{
@@ -1,52 +0,0 @@
package index
import (
"encoding/json"
"strconv"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/keys"
)
// The bodies of the seat's events manifest-accepted and manifest-refused are the contract's types; the indexer, who says
// them, makes them from a verdict (novox/hq ADR 0634 §1: the logic of one holder stays with it).
// refusedOf is the body of manifest-refused for the verdict v on the manifest of digest.
func refusedOf(digest string, v meshcatalog.VerdictsByDigestValue) meshcatalog.ManifestRefusedEvent {
return meshcatalog.ManifestRefusedEvent{V: keys.Version, Module: v.Module, Digest: digest, Judge: v.Judge, Problems: v.Problems}
}
// acceptedEvent is the body of manifest-accepted for the verdict v accepting the manifest m of digest, its bus-state
// entries whole (novox/hq ADR 0595 §4); an empty list when m asks for none.
func acceptedEvent(digest string, v meshcatalog.VerdictsByDigestValue, m catalogue.Manifest) meshcatalog.ManifestAcceptedEvent {
// problems are the manifest validator's strings, the verdict's warnings in their order, an empty list when none (design 47 G §668, §692).
problems := append([]string{}, v.Warnings...)
return meshcatalog.ManifestAcceptedEvent{V: keys.Version, Module: v.Module, Digest: digest, Judge: v.Judge, BusState: busStateOf(m.BusState), Problems: problems}
}
// busStateOf is the manifest's bus-state entries in the event's form (the same JSON).
func busStateOf(entries []catalogue.BusStateEntry) []meshcatalog.ManifestAcceptedEventBusStateItem {
out := []meshcatalog.ManifestAcceptedEventBusStateItem{}
if len(entries) == 0 {
return out
}
raw, err := json.Marshal(entries)
if err == nil {
err = json.Unmarshal(raw, &out)
}
if err != nil {
return []meshcatalog.ManifestAcceptedEventBusStateItem{}
}
return out
}
// acceptedEventID is the event's id, one per manifest digest and judge, so the bus takes it once:
// `manifest-accepted-<digest>-<judge>`. The empty one of a module leaving the trunk carries no digest, so it is named by
// its module and the indexer's revision: `manifest-accepted-left-<module>-<revision>`, so two leaves are both kept.
func acceptedEventID(a meshcatalog.ManifestAcceptedEvent) string {
if a.Digest == "" {
return meshcatalog.EventManifestAccepted + "-left-" + a.Module + "-" + strconv.FormatInt(a.Revision, 10)
}
return meshcatalog.EventManifestAccepted + "-" + a.Digest + "-" + a.Judge
}
@@ -7,7 +7,6 @@ import (
"slices"
"sync"
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
"git.novox.be/novox/mesh-sdk/go/views"
)
@@ -121,11 +120,6 @@ func (m *memFindings) Held() []mcseat.Finding {
return out
}
// unsaidKey is the key of module's unsaid finding.
func unsaidKey(module string) string {
return views.FindingKey(mcc.ScopeModule, keyID(unsaidID(testSelf, module)), UnsaidKind)
}
// testSelf is the module the tests' indexer runs as, its credential's: its findings, its own entry and its graph key are named
// from it, never from a constant (design 47 G A-296).
const testSelf = "acme-indexer"
@@ -97,7 +97,7 @@ func TestTheKeptTrunkIsReadAtTheStartAndOncePerHeardRevision(t *testing.T) {
if raw, err := kept.Manifest(meshcatalog.BareManifestDigest(keptDigest("ca-trust"))); err != nil || raw != `{"module":"ca-trust"}` {
t.Errorf("the held manifest = %q (%v), want the entry's manifest", raw, err)
}
if a, err := Revision(context.Background(), objects, kept, "avahi", "avahi", meshcatalog.RevisionInput{}); err != nil || a.Revision.Revision != 2 || len(a.Entries) != 2 {
if a, err := Revision(context.Background(), objects, kept, meshcatalog.RevisionInput{}); err != nil || a.Revision.Revision != 2 || len(a.Entries) != 2 {
t.Errorf("revision of the newest answered %+v (%v), want revision 2 whole", a, err)
}
if got := objects.n.Load() - listed; got != 0 {
@@ -277,11 +277,11 @@ func TestTheNewestRevisionReadersReadAgainWhenTheRevisionIsRemovedUnderThem(t *t
return len(a.Behind) == 1 && a.Behind[0].Kind == 5 && strings.Contains(a.Behind[0].Newest, "fp2"), err
}},
"Revision": {"", func(ix *Indexer) (bool, error) {
a, err := Revision(ctx, ix.Revisions, nil, "avahi", "avahi", meshcatalog.RevisionInput{})
a, err := Revision(ctx, ix.Revisions, nil, meshcatalog.RevisionInput{})
return a.Revision != nil && a.Revision.Revision == 2, err
}},
"Revision of a branch": {"dev", func(ix *Indexer) (bool, error) {
a, err := Revision(ctx, ix.Revisions, nil, "avahi", "avahi", meshcatalog.RevisionInput{Branch: "dev"})
a, err := Revision(ctx, ix.Revisions, nil, meshcatalog.RevisionInput{Branch: "dev"})
return a.Revision != nil && a.Revision.Revision == 2 && len(a.Entries) == 1 && a.Entries[0].Fp == "fp2", err
}},
}
@@ -260,9 +260,6 @@ func (ix *Indexer) dropRemoved(ctx context.Context, repository string, held map[
return fmt.Errorf("modules.%s: %w", key, err)
}
ix.logf("modules.%s deleted: %s holds no module of that name at its trunk (issue 1900)", key, repository)
// The bus module hears the module asks nothing now, so its entries reach orphans (design 25 § 12.2, DLV-I9).
a, id := leftOf(key, left, ix.judgeID())
_ = ix.sayAccepted(ctx, a, id)
}
for _, key := range handed {
if err := ix.handOver(ctx, key); err != nil {
@@ -120,13 +120,8 @@ func (ix *Indexer) handOver(ctx context.Context, key string) error {
// handOverFrom writes the module at key from the merge f, which names it, in place.
func (ix *Indexer) handOverFrom(ctx context.Context, key string, f forgemsg.Forge) error {
w, err := ix.applyMoving(ctx, f, false, map[string]bool{key: true})
if err != nil {
if _, err := ix.applyMoving(ctx, f, false, map[string]bool{key: true}); err != nil {
return fmt.Errorf("modules.%s, moving to %s: %w", key, f.Repository, err)
}
// A module moved in is judged as a merge's are (ADR 0578 §1, ADR 0579 §1).
if _, err := ix.JudgeAll(ctx, w.fresh); err != nil {
ix.logf("judge: the manifests are not judged after moving %s to %s: %v", key, f.Repository, err)
}
return nil
}
@@ -10,8 +10,6 @@ const (
ModulesState = "modules"
ManifestsState = "manifests"
ManifestsHistory = 1
// VerdictsState is in verdicts.go.
VerdictsHistory = 5
PredefinedState = "predefined"
PackagesState = "packages"
PackagesHistory = 5
@@ -0,0 +1,84 @@
package index
import (
"context"
"encoding/json"
"errors"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
)
// OwnBuildState is the indexer's state of the build it runs (the bucket mesh-delivery-indexer_own-build, one key, OwnBuildKey):
// written once, at the process's start by FixOwnBuild, so indexer_pending says which build this process knows it runs, and since
// when. The build only names the writer of what is written (A-476 b).
const (
OwnBuildStateName = "own-build"
OwnBuildKey = "state"
)
// OwnBuild is the value of the own-build state.
type OwnBuild struct {
V int `json:"v"`
Known bool `json:"known"`
FP string `json:"fp,omitempty"`
Since time.Time `json:"since"`
Why string `json:"why,omitempty"`
}
// FixOwnBuild fixes, once and at the process's start, which build of the indexer this program is: asked of the node-engine's
// seat (OwnBuild; never the trunk's value of the indexer, which names a build that may be merged and not delivered), before
// this process hears a merge. It is never asked again: while the node-engine replaces the indexer the seat already names the
// successor, and an old process that learnt it later would take revisions in with its old judge. Where the start's ask is not
// answered the build stays unknown for the life of this process, said in the own-build state and the journal; nothing waits for it.
func (ix *Indexer) FixOwnBuild(ctx context.Context) {
ix.ownMu.Lock()
if ix.ownFixed || ix.OwnBuild == nil {
ix.ownMu.Unlock()
return
}
ix.ownFixed = true
ix.ownMu.Unlock()
fp, err := ix.OwnBuild(ctx)
if err == nil && fp == "" {
err = errors.New("the node-engine names no build of the indexer")
}
st := OwnBuild{V: 1, Since: ix.now().UTC()}
if err != nil {
st.Why = err.Error()
ix.logf("judge: the build this program runs is not known (%v); merges are taken in all the same, and it is learnt only at the start", err)
} else {
st.Known, st.FP = true, fp
ix.logf("judge: this program runs the indexer build %s, fixed here at its start", fp)
}
ix.ownMu.Lock()
ix.ownFP, ix.ownSince, ix.ownWhy = fp, st.Since, st.Why
ix.ownMu.Unlock()
if ix.OwnBuildState == nil {
return
}
data, _ := json.Marshal(st)
if _, rev, found, gerr := ix.OwnBuildState.Get(ctx, OwnBuildKey); gerr == nil && found {
err = ix.OwnBuildState.Update(ctx, OwnBuildKey, data, rev)
} else if gerr == nil {
err = ix.OwnBuildState.Create(ctx, OwnBuildKey, data)
} else {
err = gerr
}
if err != nil {
ix.logf("the own-build state cannot be written: %v", err)
}
}
// ownBuildIsNewest says whether the build of the indexer this program runs (ownFP, from the node-engine) is the newest the
// trunk holds: the indexer's own modules value names it. A newer build merged and not yet delivered ends it at the trunk's
// merge, as one that pins a changed mesh-sdk always is; false when either is unknown.
func (ix *Indexer) ownBuildIsNewest(held map[string]meshcatalog.ModulesByModuleValue) bool {
fp := held[ix.Module].Fp2
if fp == "" {
return false
}
ix.ownMu.Lock()
defer ix.ownMu.Unlock()
return ix.ownFP != "" && ix.ownFP == fp
}
@@ -366,15 +366,6 @@ func revisionWait(kind string, ready []PendingEntry, waitsFor string) takein.Tak
// pendingOnBus says whether the pending state is on the bus: a state this delivery brings is bound when it appears
// (issue 1790), and until then every merge is taken in at once, so the build this program runs is not needed. A read
// that fails otherwise counts as on the bus: the step that follows answers the error.
// errVerdictsNotYet is a take-in held because the verdicts state is not on the bus yet (takeInWholeBy).
var errVerdictsNotYet = errors.New("the verdicts state is not on the bus yet: nothing is taken in until it is")
// verdictsOnBus says the verdicts state is on the bus (a take-in judges what it writes there).
func (ix *Indexer) verdictsOnBus(ctx context.Context) bool {
_, _, _, err := ix.Verdicts.Get(ctx, "on-the-bus")
return !errors.Is(err, process.ErrNotYet)
}
func (ix *Indexer) pendingOnBus(ctx context.Context) bool {
_, _, _, err := ix.Pending.Get(ctx, takein.PendingKey(0))
return !errors.Is(err, process.ErrNotYet)
@@ -760,9 +751,6 @@ func (ix *Indexer) evaluate(ctx context.Context, own string, start bool) (Applie
if errors.Is(err, errClaimLost) {
return a, "claimed by another take-in", nil
}
if errors.Is(err, errVerdictsNotYet) {
return a, "the verdicts state is not on the bus yet: the revision waits for it, taken in when it binds", nil
}
if errors.Is(err, ErrObjectsFull) {
ix.holdFull(ctx, ready)
return a, ix.waitingLine(w) + "; held: " + HeldObjectsFull, nil
@@ -918,7 +906,7 @@ func (ix *Indexer) dry() *dryRun {
d := &dryRun{}
x := &Indexer{Module: ix.Module, Manifests: ix.Manifests, Packages: ix.Packages, JudgeID: ix.JudgeID, Files: ix.Files, GoSum: ix.GoSum,
GoMod: ix.GoMod, Now: ix.Now, Builds: ix.Builds, Named: ix.Named, Trunk: ix.Trunk, BranchHead: ix.BranchHead, Read: ix.Read, Contains: ix.Contains,
ReadOpen: nil, OwnBuild: nil, Message: ix.Message, SayRefused: nil, SayAccepted: nil, SayTakeIn: nil, Pending: nil,
ReadOpen: nil, OwnBuild: nil, Message: ix.Message, SayTakeIn: nil, Pending: nil,
// The registry milestone's fields: what a take-in would write on the bus stays out of a dry run, so the states of the
// requirements, findings, seat definitions, holds and their bounds are not overlaid, nothing is said, and no hold is
// weighed (Holds and Counted nil hold nothing); ReadFile only reads.
@@ -932,12 +920,9 @@ func (ix *Indexer) dry() *dryRun {
x.Store, d.candidates = over(ix.Store)
x.Modules, d.modules = over(ix.Modules)
x.Graph, d.graph = over(ix.Graph)
x.Verdicts, d.verdicts = over(ix.Verdicts)
x.Taken, d.taken = over(ix.Taken)
x.Predefined, d.predefined = over(ix.Predefined)
x.OldSections, d.oldSections = over(ix.OldSections)
// What the trunk holds unjudged is read through, as fresh; what the dry run would keep stays in its overlay (issue 1966).
x.Unjudged, _ = over(ix.Unjudged)
ix.mu.Lock()
x.source = sourceNow{rev: ix.source.rev, heads: maps.Clone(ix.source.heads)}
x.previews = maps.Clone(ix.previews)
@@ -1060,13 +1045,6 @@ func (ix *Indexer) takeInWhole(ctx context.Context, entries []PendingEntry, own,
// takeInWholeBy is takeInWhole; who and why, of a hand act, are carried on the take-in.taken events.
func (ix *Indexer) takeInWholeBy(ctx context.Context, entries []PendingEntry, own, by, who, why string) (Applied, error) {
if ix.Verdicts != nil && !ix.verdictsOnBus(ctx) && len(entries) > 0 {
// Every manifest a take-in writes is judged in it and its verdict said before take-in.taken (issue 1966 note 154,
// ADR 0668 §4): with the verdicts state not on the bus yet, nothing is taken in; its bind evaluates pending again.
ix.logf("take-in: %d merges wait, oldest %s: the state %s is not on the bus yet, so nothing is taken in until it is, and no manifest is written without its verdict",
len(entries), entries[0].Key, VerdictsState)
return Applied{}, errVerdictsNotYet
}
claimed, ok := ix.claim(ctx, entries, own, by)
if !ok {
return Applied{}, errClaimLost
@@ -1162,8 +1140,7 @@ func (ix *Indexer) takenWhat(ctx context.Context, claimed []PendingEntry, applie
ix.logf("take-in: %s is written in source revision %d of %s with no fingerprint (%s): no pipeline entry", m, src, branch, e.None)
continue
}
out = append(out, meshcatalog.TakeInEntry{Module: m, Commit: e.Commit, Fingerprint: e.Fp, Repository: e.Repository,
Pull: pulls[strings.ToLower(e.Repository)]})
out = append(out, meshcatalog.TakeInEntry{Module: m, Commit: e.Commit, Fingerprint: e.Fp, Repository: e.Repository})
}
return branch, src, out
}
File diff suppressed because it is too large Load Diff
@@ -13,7 +13,6 @@ import (
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
runner "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/keys"
@@ -22,17 +21,13 @@ import (
// The indexer keeps the records pinning each catalog graph revision and removes a revision only on an event (design 47 G §153,
// §178, §189, §215, §216, §579, A-169, A-171): its state pins holds one key per revision, `<branch>.<revision>` (the watched branch
// and the revision's number, the branch escaped as a name is), whose value lists the delivery record ids that pin it, and two more
// keys, `made` (written once, create-only, when the state is first used) and `pins-stated` (the `at` of the newest applied event of
// that name). Pins count only once a pins-stated later than made is applied.
// and the revision's number, the branch escaped as a name is), whose value lists the records that pin it: a take-in's and each
// running pipeline's (design 47 H §743, §749).
const (
// PinsState is the state's name; PinsCapMiB is its cap in the manifest (about 1,800 keys of about 0.2 KiB).
PinsState = "pins"
PinsHistory = 1
PinsCapMiB = 4
// PinsMadeKey and PinsStatedKey are the two keys of the state that are no revision's.
PinsMadeKey = "made"
PinsStatedKey = "pins-stated"
// PinMissingKind is the finding of a pinned revision the indexer no longer holds.
PinMissingKind = "pin-missing"
// PinsTool reads the pins, UnpinTool removes one record's pin by hand.
@@ -46,17 +41,12 @@ type PinValue struct {
Missing bool `json:"missing,omitempty"`
}
// PinStamp is the value of the keys made and pins-stated.
type PinStamp struct {
At time.Time `json:"at"`
}
// PinKey is a revision's key in the state pins.
func PinKey(branch string, revision int64) string {
return keys.EscapeName(branch) + "." + strconv.FormatInt(revision, 10)
}
// ParsePinKey is the branch and the revision a key of the state pins names; false for made, pins-stated and what is no revision's key.
// ParsePinKey is the branch and the revision a key of the state pins names; false for what is no revision's key.
func ParsePinKey(key string) (branch string, revision int64, ok bool) {
i := strings.LastIndex(key, ".")
if i <= 0 {
@@ -82,99 +72,26 @@ func pinObject(key string) (string, bool) {
return RevisionObject(meshcatalog.CatalogRevision{Branch: branch, Revision: revision}), true
}
func (ix *Indexer) pinsStamp(ctx context.Context, key string) (PinStamp, bool, error) {
var s PinStamp
v, _, found, err := ix.Pins.Get(ctx, key)
if err != nil || !found {
return s, false, err
}
if err := json.Unmarshal(v, &s); err != nil {
return s, false, fmt.Errorf("%s.%s: %w", PinsState, key, err)
}
return s, true, nil
}
// PinsMade writes the key made once, create-only, when the state is first used; a made already written stays.
func (ix *Indexer) PinsMade(ctx context.Context) error {
if ix.Pins == nil {
return nil
}
data, err := json.Marshal(PinStamp{At: ix.now().UTC()})
if err != nil {
return err
}
if err := ix.Pins.Create(ctx, PinsMadeKey, data); err != nil && !errors.Is(err, ErrConflict) {
return fmt.Errorf("%s.%s: %w", PinsState, PinsMadeKey, err)
}
return nil
}
// pinsCounted says the pins count: a pins-stated later than made is applied.
func (ix *Indexer) pinsCounted(ctx context.Context) (bool, error) {
made, ok, err := ix.pinsStamp(ctx, PinsMadeKey)
if err != nil || !ok {
return false, err
}
stated, ok, err := ix.pinsStamp(ctx, PinsStatedKey)
if err != nil || !ok {
return false, err
}
return stated.At.After(made.At), nil
}
// HeardPins applies one event of the seat mesh-delivery the indexer's durable consumer gave it: pinned, freed, heads-replaced or
// pins-stated, by its subject. An error leaves the event unacknowledged, so the bus gives it again; one applied twice changes nothing.
func (ix *Indexer) HeardPins(ctx context.Context, subject string, body []byte) error {
// HeardPins applies one event of the seat mesh-pipeline-runner the indexer's durable consumers gave it (design 47 H §749): a
// pipeline-started pins the revision it names under the record `pipeline.<pipeline>`, its pipeline-ended frees it. A body
// that is no such event, or names no revision, changes nothing; an error leaves the event unacknowledged, so the bus gives it
// again; one applied twice changes nothing.
func (ix *Indexer) HeardPins(ctx context.Context, started *runner.PipelineStarted, ended *runner.PipelineEnded) error {
if ix.Pins == nil {
return errors.New("the indexer keeps no state " + PinsState)
}
if err := ix.PinsMade(ctx); err != nil {
return err
}
switch subject {
case meshdelivery.SubjectPinned, meshdelivery.SubjectFreed:
var p meshdelivery.Pins
if err := json.Unmarshal(body, &p); err != nil {
return fmt.Errorf("%s is not a pins body: %w", subject, err)
}
for _, c := range p.Catalog {
key := PinKey(c.Branch, c.Revision)
if subject == meshdelivery.SubjectPinned {
if err := ix.pin(ctx, key, p.Record); err != nil {
return err
}
} else if err := ix.unpin(ctx, key, p.Record); err != nil {
return err
}
}
case meshdelivery.SubjectHeadsReplaced:
var h meshdelivery.HeadsReplacedEvent
if err := json.Unmarshal(body, &h); err != nil {
return fmt.Errorf("%s is not a heads-replaced body: %w", subject, err)
}
ix.DropCandidateGraph(ctx, h.Group, h.Heads, "mesh-delivery", "its event heads-replaced: the candidate's heads were replaced or its pull request closed")
case meshdelivery.SubjectPinsStated:
var s meshdelivery.PinsStatedEvent
if err := json.Unmarshal(body, &s); err != nil {
return fmt.Errorf("%s is not a pins-stated body: %w", subject, err)
}
err := write(ctx, ix.Pins, PinsStatedKey, func(old []byte, found bool) ([]byte, bool, error) {
var cur PinStamp
if found && json.Unmarshal(old, &cur) == nil && !s.At.After(cur.At) {
return nil, false, nil
}
data, err := json.Marshal(PinStamp{At: s.At.UTC()})
return data, false, err
})
if err != nil {
return err
}
default:
return fmt.Errorf("%s is no event of the pins", subject)
switch {
case started != nil && started.Revision > 0 && started.Pipeline != "":
return ix.pin(ctx, PinKey(started.Branch, started.Revision), PipelineRecord(started.Pipeline))
case ended != nil && ended.Revision > 0 && ended.Pipeline != "":
return ix.unpin(ctx, PinKey(ended.Branch, ended.Revision), PipelineRecord(ended.Pipeline))
}
return nil
}
// PipelineRecord is the record a pipeline pins its revision under: `pipeline.<branch>.<source>` (design 47 H §743, A-449a).
func PipelineRecord(pipeline string) string { return "pipeline." + pipeline }
// pin adds a record's id to a revision's key by compare-and-set; a revision not held is marked missing and said.
func (ix *Indexer) pin(ctx context.Context, key, record string) error {
object, _ := pinObject(key)
@@ -332,10 +249,6 @@ func (ix *Indexer) judgePinsLocked(ctx context.Context) ([]string, error) {
if err != nil || pending != 0 {
return nil, err
}
counted, err := ix.pinsCounted(ctx)
if err != nil || !counted {
return nil, err
}
names, err := ix.Revisions.Names(ctx)
if err != nil {
return nil, err
@@ -428,18 +341,13 @@ type PinnedRevision struct {
type PinsAnswer struct {
Pins []PinnedRevision `json:"pins"`
// Unpinned are the kept revisions no record pins, current or not, and Current why each current one is.
Unpinned []string `json:"unpinned,omitempty"`
Made *time.Time `json:"made,omitempty"`
Stated *time.Time `json:"pins-stated,omitempty"`
// Counted says the pins count: a pins-stated later than made is applied.
Counted bool `json:"counted"`
Unpinned []string `json:"unpinned,omitempty"`
// Fill is the state's bytes against its cap.
Fill meshcatalog.Fill `json:"fill"`
}
// ReadPins reads the state pins and the revisions kept, for indexer_pins.
func ReadPins(ctx context.Context, pins Store, o Objects) (PinsAnswer, error) {
ix := &Indexer{Pins: pins, Revisions: o}
var out PinsAnswer
names, err := o.Names(ctx)
if err != nil {
@@ -477,13 +385,6 @@ func ReadPins(ctx context.Context, pins Store, o Objects) (PinsAnswer, error) {
out.Unpinned = append(out.Unpinned, n)
}
}
if s, ok, err := ix.pinsStamp(ctx, PinsMadeKey); err == nil && ok {
out.Made = &s.At
}
if s, ok, err := ix.pinsStamp(ctx, PinsStatedKey); err == nil && ok {
out.Stated = &s.At
}
out.Counted, _ = ix.pinsCounted(ctx)
out.Fill = meshcatalog.Fill{Bytes: bytes, Cap: PinsCapMiB << 20, Store: PinsState}
return out, nil
}
@@ -495,8 +396,7 @@ func PinsTools(pins Store, o Objects, ask func(key string, value []byte) (TakeIn
{Name: PinsTool,
Description: "Every catalog graph revision a delivery record pins in the indexer's state pins (design 47 G §579): the revision's " +
"key `<branch>.<revision>`, the records pinning it, whether it was missing when pinned (finding pin-missing), whether it is " +
"kept and whether it is current and why; the kept revisions no record pins; when the state was made and when the newest " +
"pins-stated was applied, and whether pins count yet (a pins-stated later than made); and the state's fill against its cap. " +
"kept and whether it is current and why; the kept revisions no record pins; and the state's fill against its cap. " +
"indexer_consumer reads the consumer the events come through; indexer_unpin removes one record's pin.",
Input: map[string]any{"type": "object", "properties": map[string]any{}},
Run: func(map[string]any) (any, error) {
@@ -575,11 +475,10 @@ func ConsumerTools(st Store) []stdio.Tool {
Run: func(map[string]any) (any, error) { return ReadConsumers(context.Background(), st) }}}
}
// PinPipelines pins the catalog graph revision a take-in made for each pipeline the take-in starts, under the record
// `pipeline.<key>` (`<branch>.<source>.<module>` of each entry, the runner's key), in the indexer's own pins state as a
// pinned of the seat mesh-delivery is applied (design 47 G §579, A-449a): so no later take-in removes the revision before
// the runner hears of it. The runner's pinned at the pipeline's start says the same record (a set), and its freed at the
// pipeline's end removes it. It reads the branch's newest revision and pins it holding keepMu, so no other keep (a verdict, a
// PinPipelines pins the catalog graph revision a take-in made for the one pipeline the take-in starts, under the record
// `pipeline.<branch>.<source>` (the runner's key, design 47 H §743, A-449a), in the indexer's own pins state: so no later
// take-in removes the revision before the runner starts it. The pipeline's pipeline-started names the same record (a set),
// and its pipeline-ended removes it (§749). It reads the branch's newest revision and pins it holding keepMu, so no other keep (a verdict, a
// source, an import) can make it non-current and remove it between the two; it answers that revision, 0 when none is kept.
func (ix *Indexer) PinPipelines(ctx context.Context, branch string, source int64, entries []meshcatalog.TakeInEntry) int64 {
ix.keepMu.Lock()
@@ -588,15 +487,9 @@ func (ix *Indexer) PinPipelines(ctx context.Context, branch string, source int64
if ix.Pins == nil || revision <= 0 || len(entries) == 0 {
return revision
}
if err := ix.PinsMade(ctx); err != nil {
ix.logf("pins: the revision %d of %s is not pinned for its pipelines: %v", revision, branch, err)
return revision
}
for _, e := range entries {
record := "pipeline." + runner.PipelinesByBranchRevisionModuleKey(keys.EscapeName(branch), strconv.FormatInt(source, 10), e.Module)
if err := ix.pin(ctx, PinKey(branch, revision), record); err != nil {
ix.logf("pins: the revision %d of %s is not pinned for %s: %v", revision, branch, record, err)
}
record := PipelineRecord(runner.PipelinesByBranchSourceKey(keys.EscapeName(branch), strconv.FormatInt(source, 10)))
if err := ix.pin(ctx, PinKey(branch, revision), record); err != nil {
ix.logf("pins: the revision %d of %s is not pinned for %s: %v", revision, branch, record, err)
}
return revision
}
@@ -8,43 +8,34 @@ import (
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
runner "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
)
// Design 47 G §579, A-169, A-171: the indexer keeps the records pinning each catalog graph revision, counts a record once, and
// removes a revision only while its consumer is caught up and after a pins-stated later than made: one two records pin stays
// until both free it, one neither pinned nor current goes, a revision pinned and not held is said, and unpin clears it.
// Design 47 H §749 (G §579, A-169, A-171): the indexer pins a revision on a pipeline-started that names it and frees it on its
// pipeline-ended, counts a record once, and removes a revision only while its consumers are caught up, judged against the pins
// its own state holds, never a count of pins stated: one two pipelines pin stays until both free it, one neither pinned nor
// current goes, a revision pinned and not held is said, and unpin clears it.
func TestTheIndexerRemovesARevisionOnlyWhenNoRecordPinsItAndItIsNotCurrent(t *testing.T) {
ctx := context.Background()
clock := time.Date(2026, 11, 12, 9, 0, 0, 0, time.UTC)
pending := uint64(0)
ix := &Indexer{Module: testSelf, Revisions: newMemObjects(1 << 20), Pins: newMem(), Findings: newFindings(), Now: func() time.Time { return clock },
PinsPending: func(context.Context) (uint64, error) { return pending, nil }}
reader := meshcatalog.CatalogEntry{Module: "reader", Repository: "novox/r", Manifest: map[string]any{}}
for _, commit := range []string{"c1", "c2", "c3"} {
e := reader
e.Commit = commit
if err := ix.KeepBranchEntries(ctx, "main", 5, map[string]meshcatalog.CatalogEntry{"reader": e}); err != nil {
t.Fatal(err)
}
}
kept := func() []string {
n, _ := ix.Revisions.Names(ctx)
return slices.DeleteFunc(n, func(s string) bool { _, _, ok := RevisionGroup(s); return !ok })
}
say := func(subject string, body any) {
started := func(pipeline string, revision int64) {
t.Helper()
raw, _ := json.Marshal(body)
if err := ix.HeardPins(ctx, subject, raw); err != nil {
if err := ix.HeardPins(ctx, &runner.PipelineStarted{Branch: "main", Pipeline: pipeline, Revision: revision}, nil); err != nil {
t.Fatal(err)
}
}
pins := func(record string, revisions ...int64) meshdelivery.Pins {
p := meshdelivery.Pins{Record: record, Milestone: "47/G"}
for _, r := range revisions {
p.Catalog = append(p.Catalog, meshdelivery.PinsCatalogItem{Branch: "main", Revision: r})
ended := func(pipeline string, revision int64) {
t.Helper()
if err := ix.HeardPins(ctx, nil, &runner.PipelineEnded{Branch: "main", Pipeline: pipeline, Revision: revision}); err != nil {
t.Fatal(err)
}
return p
}
want := func(what string, names ...string) {
t.Helper()
@@ -52,34 +43,45 @@ func TestTheIndexerRemovesARevisionOnlyWhenNoRecordPinsItAndItIsNotCurrent(t *te
t.Errorf("%s: kept %v, want %v", what, got, names)
}
}
say(meshdelivery.SubjectPinned, pins("r1", 1))
say(meshdelivery.SubjectPinned, pins("r1", 1)) // heard twice: pins once
say(meshdelivery.SubjectPinned, pins("r2", 1))
ix.JudgePins(ctx)
want("before a pins-stated later than made is applied", "trunk.1", "trunk.2", "trunk.3")
clock = clock.Add(time.Second)
say(meshdelivery.SubjectPinsStated, meshdelivery.PinsStatedEvent{At: clock})
pending = 3
reader := meshcatalog.CatalogEntry{Module: "reader", Repository: "novox/r", Manifest: map[string]any{}}
keep := func(commit string) {
t.Helper()
e := reader
e.Commit = commit
if err := ix.KeepBranchEntries(ctx, "main", 5, map[string]meshcatalog.CatalogEntry{"reader": e}); err != nil {
t.Fatal(err)
}
}
keep("c1")
pending = 3 // pinned before the next keeps, which judge the revisions while the consumers are behind
started("main.5", 1)
started("main.5", 1) // heard twice: pins once
started("main.6", 1)
keep("c2")
keep("c3")
ix.JudgePins(ctx)
want("while messages of the consumer are pending", "trunk.1", "trunk.2", "trunk.3")
pending = 0
ix.JudgePins(ctx)
want("pinned by two records, the middle one neither pinned nor current", "trunk.1", "trunk.3")
say(meshdelivery.SubjectFreed, pins("r1", 1))
if v, _, _, _ := ix.Pins.Get(ctx, "main.1"); !slices.Equal(mustRecords(t, v), []string{"pipeline.main.5", "pipeline.main.6"}) {
t.Errorf("the pin of main.1 holds %s, want the two pipelines' records once each", v)
}
ended("main.5", 1)
ix.JudgePins(ctx)
want("one of two records freed it", "trunk.1", "trunk.3")
say(meshdelivery.SubjectFreed, pins("r2", 1))
ended("main.6", 1)
ix.JudgePins(ctx)
want("both freed it", "trunk.3")
if keys, _ := ix.Pins.Keys(ctx); slices.Contains(keys, "main.1") {
t.Errorf("the state holds %v, want no key of the removed revision", keys)
}
// A record pins a revision no longer held: its key says missing and the finding names it; unpin clears both.
say(meshdelivery.SubjectPinned, pins("r3", 1))
started("main.7", 1)
if f := ix.Findings.Held(); len(f) != 1 || f[0].Kind != PinMissingKind {
t.Fatalf("findings %v, want one %s", f, PinMissingKind)
}
if r := ix.Unpin(ctx, "main.1", "r3", "operator", "freed was lost"); !r.Stored {
if r := ix.Unpin(ctx, "main.1", "pipeline.main.7", "operator", "freed was lost"); !r.Stored {
t.Fatalf("unpin answered %q", r.Answer)
}
if f := ix.Findings.Held(); len(f) != 0 {
@@ -87,35 +89,12 @@ func TestTheIndexerRemovesARevisionOnlyWhenNoRecordPinsItAndItIsNotCurrent(t *te
}
}
func TestAPinsStatedBeforeTheStateWasMadeCountsNothing(t *testing.T) {
ctx := context.Background()
clock := time.Date(2026, 11, 12, 9, 0, 0, 0, time.UTC)
ix := &Indexer{Module: testSelf, Revisions: newMemObjects(1 << 20), Pins: newMem(), Now: func() time.Time { return clock },
PinsPending: func(context.Context) (uint64, error) { return 0, nil }}
raw, _ := json.Marshal(meshdelivery.PinsStatedEvent{At: clock.Add(-time.Hour)})
if err := ix.HeardPins(ctx, meshdelivery.SubjectPinsStated, raw); err != nil {
func mustRecords(t *testing.T, v []byte) []string {
t.Helper()
var pv PinValue
if err := json.Unmarshal(v, &pv); err != nil {
t.Fatal(err)
}
if ok, err := ix.pinsCounted(ctx); err != nil || ok {
t.Errorf("counted %v, %v after a pins-stated older than made, want not counted", ok, err)
}
}
func TestHeadsReplacedDropsTheCandidatesGraph(t *testing.T) {
ctx := context.Background()
ix := &Indexer{Module: testSelf, Revisions: newMemObjects(1 << 20), Pins: newMem()}
rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Kind: "candidate", Group: "feat-x", Heads: "0123456789ab", Revision: 1}, nil)
if err != nil {
t.Fatal(err)
}
if err := KeepRevision(ctx, ix.Revisions, rev, objects); err != nil {
t.Fatal(err)
}
raw, _ := json.Marshal(meshdelivery.HeadsReplacedEvent{Group: "feat-x", Heads: "0123456789ab", Repository: "novox/r", Number: 1})
if err := ix.HeardPins(ctx, meshdelivery.SubjectHeadsReplaced, raw); err != nil {
t.Fatal(err)
}
if n, _ := ix.Revisions.Names(ctx); len(n) != 0 {
t.Errorf("the objects hold %v, want the candidate's graph dropped", n)
}
slices.Sort(pv.Records)
return pv.Records
}
@@ -1,46 +0,0 @@
package index
import (
"context"
"encoding/json"
"testing"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
)
// Design 47 G §509, §714, A-269: a pushed message's head stands where a merged message's merge stands: the pending value keeps
// it as its ref, the newest load reads it from the bridge's answer and the pending value read back holds it alike.
func TestAPushedMessageStandsInTheRevisionWhereAMergedMessageDoes(t *testing.T) {
b := newTakeInBed(t)
head := "5555555555555555555555555555555555555555"
pushed := b.base
pushed.Kind, pushed.Number, pushed.Merge, pushed.Base, pushed.Branch, pushed.Head = forgemsg.ForgePushed, 0, "", "", "feat/x", head
data, err := forgemsg.EncodeForge(pushed)
if err != nil {
t.Fatal(err)
}
b.bridge[pushed.Repository+"@"+head] = data // the bridge answers by the head, where a merge's commit would stand
f := RevisionForm(pushed)
if f.Kind != forgemsg.ForgeMerged || f.Merge != head {
t.Fatalf("RevisionForm = %s %q, want a merged form at the head", f.Kind, f.Merge)
}
value, err := encodePending(Pending{Seq: 9, Ref: refOf(f)})
if err != nil {
t.Fatal(err)
}
_, got, err := DecodePending(value)
if err != nil || got.Merge != head {
t.Fatalf("DecodePending of the ref = %+v, %v, want the head as its merge", got, err)
}
// A value an earlier indexer wrote with the pushed message whole reads alike.
whole, _ := json.Marshal(Pending{V: 1, Seq: 9, Forge: data})
if _, got, err = DecodePending(whole); err != nil || got.Merge != head || got.Kind != forgemsg.ForgeMerged {
t.Fatalf("DecodePending of the whole message = %+v, %v", got, err)
}
// The load reads the bridge's pushed answer for the ref and accepts it.
loaded, err := b.ix.load(context.Background(), PendingEntry{Key: "m.9", Value: Pending{Seq: 9}, Forge: RevisionForm(pushed), Light: true})
if err != nil || loaded.Forge.Merge != head || loaded.Light {
t.Fatalf("load = %+v, %v, want the pushed message read at the head", loaded.Forge, err)
}
}
@@ -139,14 +139,9 @@ func (ix *Indexer) indexAt(ctx context.Context, r Registration, force, reread bo
if p := read.Packages[dir]; len(p) > 0 {
f.Packages = map[string][]string{dir: p}
}
w, err := ix.applyManifests(ctx, f, false)
if err != nil {
if _, err := ix.applyManifests(ctx, f, false); err != nil {
return t, err
}
// A module registered or taken in is judged as a merge's are (ADR 0578 §1, ADR 0579 §1).
if _, err := ix.JudgeAll(ctx, w.fresh); err != nil {
ix.logf("judge: the manifests are not judged after taking in %s: %v", r.Module, err)
}
if _, _, found, err := ix.Modules.Get(ctx, key); err != nil {
return t, fmt.Errorf("modules.%s: %w", key, err)
} else if !found {
@@ -248,6 +248,9 @@ func TestSeedWritesTheModuleGraphWithNoAskAndWaitsForEveryBuildSeat(t *testing.T
}
// A step the seat answers none for is bypass; the statuses the check judged are the rest.
got, _, _ := g.r.Registration(ctx, "a")
if got.Policy.Steps.Validate != "required" { // I7/AC2, design 47 H §745
t.Errorf("the seed wrote validate %q, want required", got.Policy.Steps.Validate)
}
if got.Version != 1 || got.Policy.Steps.Review != "optional" || got.Policy.Steps.Approval != "bypass" || got.Policy.Artifacts["program"].Lint != "bypass" || got.Policy.Artifacts["program"].Integration != "bypass" || got.Policy.Artifacts["program"].Unit != "" {
t.Fatalf("the seeded registration: %+v", got)
}
@@ -26,29 +26,6 @@ func twoModules(t *testing.T) (*Indexer, *memStore) {
func TestAMergeThatRemovesAModuleDeletesItsKey(t *testing.T) { // 1900/AC1
ix, mods := twoModules(t)
// DLV-I9 (design 25 § 12.2): the module leaving the trunk is said as an empty manifest-accepted, under an id naming it.
var said []meshcatalog.ManifestAcceptedEvent
var ids []string
ix.SayAccepted = func(_ context.Context, a meshcatalog.ManifestAcceptedEvent, id string) error {
said, ids = append(said, a), append(ids, id)
return nil
}
defer func() {
var left []string
for i, a := range said {
if a.Digest == "" {
left = append(left, a.Module)
// Join item 3: the id is the event's own (acceptedEventID), naming the module and the
// modules state's revision of the value that left, so two leaves of one module are two events.
if a.BusState == nil || len(a.BusState) != 0 || a.Revision == 0 || ids[i] != acceptedEventID(a) || !strings.Contains(ids[i], "-left-b-") {
t.Errorf("the empty manifest-accepted = %+v, id %s", a, ids[i])
}
}
}
if strings.Join(left, ",") != "b" {
t.Errorf("said empty for %v, want b alone", left)
}
}()
if err := ix.Apply(context.Background(), mergedMsg(catalogueRepo, map[string]string{"modules/a": "t2"})); err != nil {
t.Fatal(err)
}
@@ -102,10 +102,12 @@ func (ix *Indexer) EndRunPipeline(ctx context.Context, e meshpipelinerunner.Pipe
ix.runMu.Lock()
defer ix.runMu.Unlock()
r, err := ix.updateRun(ctx, group, func(r *Run) {
if e.Module != "" && !slices.Contains(r.Ended, e.Module) {
r.Ended = append(r.Ended, e.Module)
slices.Sort(r.Ended)
for _, m := range e.Movers { // a run's pipeline-ended names its movers (H-A-72)
if m.Module != "" && !slices.Contains(r.Ended, m.Module) {
r.Ended = append(r.Ended, m.Module)
}
}
slices.Sort(r.Ended)
})
if err != nil {
return err
@@ -208,8 +210,6 @@ func (ix *Indexer) writeRunCandidate(ctx context.Context, group string, members
return err
}
pulls := make([]meshcatalog.CandidatesByGroupValuePullsItem, len(placed))
var entries []PendingEntry
over := map[string]string{}
// One take-in per base branch: the members on one base are taken in together, so one member sees another's.
bases := map[string]*dryRun{}
for i, m := range placed {
@@ -266,14 +266,7 @@ func (ix *Indexer) writeRunCandidate(ctx context.Context, group string, members
pulls[i] = meshcatalog.CandidatesByGroupValuePullsItem{Repository: m.repository, Head: m.Commit, Modules: []string{m.Module},
Fps: map[string]meshcatalog.CandidatesByGroupValuePullsItemFpsValue{m.Module: {Manifest: mv.Manifest, Fp2: mv.Fp2, None2: mv.None2,
Packages: mv.Packages}}}
over[m.Module] = mv.Manifest
entries = append(entries, PendingEntry{Forge: forgemsg.Forge{Repository: m.repository, Merge: m.Commit, Manifests: map[string]string{m.dir: raw}}})
}
seats, err := ix.revisionSeats(ctx, entries)
if err != nil {
return fmt.Errorf("the run %s: reading the seat definitions of its members: %w", group, err)
}
ix.judgeDigestsWith(ctx, seats, over, "the run "+group)
c := meshcatalog.CandidatesByGroupValue{V: keys.Version, Group: group, Heads: hash, Pulls: pulls}
c.Fpset = meshcatalogclient.FingerprintSet(c.Pulls)
if err := ix.refuseDisagreement(c); err != nil {
@@ -46,7 +46,7 @@ func TestARunsCandidateIsBuiltOnItsStartAndDroppedWhenEveryMembersPipelineEnded(
started := meshpipelinerunner.RunStarted{V: 1, Run: "7", By: "lead@shanks", Members: []meshpipelinerunner.Member{
{Module: "avahi", Commit: c1, Branch: "feature/1966-x"}, {Module: "netcheck", Commit: c2, Branch: "beta"}}}
ended := func(module string) meshpipelinerunner.PipelineEnded {
return meshpipelinerunner.PipelineEnded{V: 1, Pipeline: "p", Module: module, Branch: "b", Commit: "c", Outcome: "passed", Run: "7"}
return meshpipelinerunner.PipelineEnded{V: 1, Pipeline: "p", Movers: []meshpipelinerunner.MoverEnded{{Module: module}}, Branch: "b", Outcome: "passed", Run: "7"}
}
key, _ := meshcatalogclient.CandidatesKey("run-7")
t.Run("built on its start, one pull per member, and dropped with the run after a restart", func(t *testing.T) {
@@ -107,7 +107,7 @@ func TestARunsCandidateIsBuiltOnItsStartAndDroppedWhenEveryMembersPipelineEnded(
if err := ix.StartRun(ctx, lost); !errors.Is(err, ErrUnreadable) {
t.Errorf("a member with no registration: %v, want unreadable", err)
}
if err := ix.EndRunPipeline(ctx, meshpipelinerunner.PipelineEnded{Module: "avahi"}); err != nil {
if err := ix.EndRunPipeline(ctx, meshpipelinerunner.PipelineEnded{Movers: []meshpipelinerunner.MoverEnded{{Module: "avahi"}}}); err != nil {
t.Error(err)
}
if k, _ := cands.Keys(ctx); len(k) != 0 {
@@ -75,7 +75,8 @@ func (r *Registry) ComposeSeed(modules []SeedModule) (SeedProposal, error) {
}
reg := meshcatalog.Registration{Module: m.Module, Origin: m.Origin, Path: m.Path, Branches: []string{b.Branch},
Policy: meshcatalog.Policy{Steps: meshcatalog.StepStatuses{Lint: "required", Unit: "required", Integration: "required",
Composition: "required", Review: "optional", Approval: "bypass"}}}
Composition: "required", Review: "optional", Approval: "bypass",
Validate: "required"}}} // design 47 H §745: the step validate is required in every policy by default
for _, a := range m.Artifacts {
var s meshcatalog.StepStatuses
if !slices.Contains(a.Steps, "lint") {
@@ -62,7 +62,6 @@ func (ix *Indexer) ApplySource(ctx context.Context, a meshdelivery.SourceAnswer)
for _, r := range a.Repositories {
s.Heads[r.Repository] = r.Head
}
ix.kickJudge(ctx)
ix.mu.Lock()
if rev < ix.source.rev {
ix.mu.Unlock()
@@ -21,7 +21,7 @@ type srcT struct {
func sourceOf(repo, head string) srcT { return srcT{Group: "g", Heads: map[string]string{repo: head}} }
func applyS(ix *Indexer, ctx context.Context, rev uint64, s srcT) error {
a := meshdelivery.SourceAnswer{Branch: "main", Group: s.Group, Revision: int64(rev)}
a := meshdelivery.SourceAnswer{Branch: "main", Revision: int64(rev)}
for r, h := range s.Heads {
a.Repositories = append(a.Repositories, meshdelivery.SourceHead{Repository: r, Head: h, Trunk: "main"})
}
@@ -28,9 +28,7 @@ type ViewWaiter interface {
// read again once the seat answers, which judges again every manifest judged from kept definitions.
type DefinitionsAtStart struct {
// Read reads every contracts module's trunk definitions (Indexer.ReadTrunkDefinitions).
Read func(ctx context.Context) (int, error)
// Judge judges every manifest again, as at the start (Indexer.JudgeAtStart).
Judge func(ctx context.Context) error
Read func(ctx context.Context) (int, error)
Waits ViewWaiter
Bound time.Duration
Logf func(string, ...any)
@@ -90,19 +88,11 @@ func (d *DefinitionsAtStart) read(ctx context.Context, when string) bool {
default:
close(d.answered)
}
again := d.fromKept
d.fromKept = false
d.mu.Unlock()
if was && d.Waits != nil {
d.Waits.Holds(GitDefinitionsView)
}
if again && d.Judge != nil {
if err := d.Judge(ctx); err != nil {
d.logf("judging again the manifests judged from kept definitions: %v; judged again at the next message heard", err)
} else {
d.logf("judged again every manifest judged from kept definitions, now that the seat git answers")
}
}
return true
}
@@ -34,12 +34,6 @@ type TakeInAsk struct {
// CatchUp asks the head the gate's source names for every repository to be taken in (CatchUp): the catch-up tool,
// indexer_take_in called with no module and no repository, as the node agent calls it after a restore.
CatchUp bool `json:"catch-up,omitempty"`
// Judge asks the manifests to be judged again now (indexer_judge_again) instead of the module to be taken in.
Judge bool `json:"judge,omitempty"`
// Digest names, with Judge, the one manifest digest of the module to judge (indexer_judge with a digest).
Digest string `json:"digest,omitempty"`
// Say asks the module's manifest-accepted to be said again now (indexer_say_accepted, DLV-I9c).
Say bool `json:"say,omitempty"`
// SeatDefinitions asks a contracts module's seat definitions to be read again now (indexer_seat_definitions_again,
// DLV-I19): its trunk's, or with Pull and Head a pull request's head.
SeatDefinitions bool `json:"seat-definitions,omitempty"`
@@ -418,11 +418,7 @@ func (ix *Indexer) writeOverlay(ctx context.Context, overlay, finals map[string]
return nil, false, nil
}
if next.Manifest != "" && (!held || cur.Manifest != next.Manifest) {
w.taken(next.Manifest) // a new module, or a changed manifest: fresh, judged in full (ADR 0579 §1)
// Kept on the bus before the module names it, so a judge pass that fails does not lose it (issue 1966).
if err := ix.keepUnjudged(ctx, next.Manifest, key, w); err != nil {
return nil, false, err
}
w.taken(next.Manifest) // a new module, or a changed manifest
}
data, err := meshcatalogclient.EncodeModule(next)
wrote = err == nil && !unchanged(data, false, value, found)
@@ -645,22 +645,19 @@ func TestBasesAreComputedFirstAndNoFingerprintTakesTheBuilder(t *testing.T) { //
t.Fatal(err)
}
}
if got, err := Revision(context.Background(), verbObjects, nil, testSelf, "n1.node-tools", meshcatalog.RevisionInput{}); err != nil || !got.NoneYet || got.Revision != nil || len(got.Entries) != 0 {
if got, err := Revision(context.Background(), verbObjects, nil, meshcatalog.RevisionInput{}); err != nil || !got.NoneYet || got.Revision != nil || len(got.Entries) != 0 {
t.Errorf("revision with none kept answered %+v, %v, want none-yet alone with no error (design 47 G §530)", got, err)
}
keep(meshcatalog.CatalogRevision{Revision: 3, Kind: "trunk"}, map[string]bool{"node-tools": true, testSelf: true})
keep(meshcatalog.CatalogRevision{Revision: 4, Kind: "candidate", Group: "feat/x", Heads: "0123456789ab"}, map[string]bool{"node-tools": true, "mesh-grants": true})
for caller, answered := range map[string]bool{"n1.node-tools": true, "n1.avahi": true, "n1.mesh-grants": true, "n1." + testSelf: true, "person.someone": false, "unproven": false} {
got, err := Revision(context.Background(), verbObjects, nil, testSelf, caller, meshcatalog.RevisionInput{})
if (err == nil) != answered || (answered && (got.Revision.Revision != 3 || len(got.Entries) != 4 || got.Next != "")) {
t.Errorf("revision to %s answered %+v (%v), want answered %v with trunk revision 3 whole", caller, got, err, answered)
}
if got, err := Revision(context.Background(), verbObjects, nil, meshcatalog.RevisionInput{}); err != nil || got.Revision.Revision != 3 || len(got.Entries) != 4 || got.Next != "" {
t.Errorf("revision answered %+v (%v), want trunk revision 3 whole", got, err)
}
if got, err := Revision(context.Background(), verbObjects, nil, testSelf, "n1.node-tools", meshcatalog.RevisionInput{Group: "feat/x", Heads: "0123456789ab", After: "mesh-grants"}); err != nil ||
if got, err := Revision(context.Background(), verbObjects, nil, meshcatalog.RevisionInput{Group: "feat/x", Heads: "0123456789ab", After: "mesh-grants"}); err != nil ||
got.Revision.Kind != "candidate" || len(got.Entries) != 1 || got.Entries[0].Module != "node-tools" {
t.Errorf("a candidate's revision after mesh-grants answered %+v (%v), want its one entry node-tools", got, err)
}
if _, err := Revision(context.Background(), verbObjects, nil, testSelf, "n1.node-tools", meshcatalog.RevisionInput{Revision: 9}); err == nil {
if _, err := Revision(context.Background(), verbObjects, nil, meshcatalog.RevisionInput{Revision: 9}); err == nil {
t.Error("a trunk revision not kept was answered")
}
// ADR 0614 §2, §3: the graph names what each module invokes and serves, and leaves out, over the whole graph, the
@@ -1,110 +0,0 @@
package index
import (
"context"
"encoding/json"
"errors"
"fmt"
"maps"
"slices"
"git.novox.be/novox/mesh-sdk/go/process"
)
// What a take-in newly wrote is kept until judged (issue 1966, ADR 0668 §4; ADR 0579 §1). A manifest a take-in writes new
// is fresh: judged in full, never taken as accepted before. A judge pass that fails (the verdicts state not on the bus, a
// restart between the write and the pass) used to lose that set, which lived only in the take-in's memory: the next pass
// found no verdict, took the manifest as held before any verdict, and wrote a refusal as a warning. So the take-in writes
// each digest into the indexer's own state unjudged before it writes the module naming it, every judge pass judges those
// digests as fresh, and a pass deletes a digest's key once its verdict is written or no module holds it any more. The
// tool indexer_read_unjudged reads the state; indexer_judge judges again by hand, which ends what it holds.
const (
// UnjudgedState is the indexer's state of the digests a take-in wrote and no judge pass judged yet: the bucket
// mesh-delivery-indexer_unjudged, one key per manifest digest.
UnjudgedState = "unjudged"
)
// UnjudgedValue is one key of the unjudged state: the module whose new manifest the digest is, and the merge that wrote it.
type UnjudgedValue struct {
V int `json:"v"`
Module string `json:"module"`
Repository string `json:"repository,omitempty"`
Merge string `json:"merge,omitempty"`
}
// keepUnjudged writes the digest into the unjudged state before the module naming it is written. A state not on the bus
// yet (an indexer whose delivery brings it) is said and the take-in goes on, judged from its memory alone as before;
// any other failure fails the write, so the module is not written with its digest unkept.
func (ix *Indexer) keepUnjudged(ctx context.Context, digest, module string, w *mergeWrite) error {
if ix.Unjudged == nil {
return nil
}
v := UnjudgedValue{V: 1, Module: module}
if w != nil {
v.Repository, v.Merge = w.repository, w.merge
}
data, err := json.Marshal(v)
if err != nil {
return err
}
err = write(ctx, ix.Unjudged, digest, func(value []byte, found bool) ([]byte, bool, error) {
if found {
return nil, false, nil // kept already: the first take-in that wrote it stands
}
return data, false, nil
})
if errors.Is(err, process.ErrNotYet) {
ix.logf("judge: the state %s is not on the bus yet, so the new manifest %s of %s is kept for its judging only in this take-in", UnjudgedState, digest, module)
return nil
}
if err != nil {
return fmt.Errorf("%s.%s: %w", UnjudgedState, digest, err)
}
return nil
}
// withUnjudged is fresh with every digest the unjudged state holds; a state that cannot be listed is said and fresh
// alone is judged.
func (ix *Indexer) withUnjudged(ctx context.Context, fresh map[string]bool) map[string]bool {
if ix.Unjudged == nil {
return fresh
}
keys, err := ix.Unjudged.Keys(ctx)
if err != nil {
if !errors.Is(err, process.ErrNotYet) {
ix.logf("judge: the state %s cannot be listed (%v), so only this take-in's new manifests are judged as new", UnjudgedState, err)
}
return fresh
}
if len(keys) == 0 {
return fresh
}
out := maps.Clone(fresh)
if out == nil {
out = map[string]bool{}
}
for _, k := range keys {
out[k] = true
}
return out
}
// endUnjudged deletes each key of the unjudged state whose digest the pass judged with its verdict written, or which no
// module holds any more (a later merge replaced it): what is left waits for the next pass.
func (ix *Indexer) endUnjudged(ctx context.Context, judged, held map[string]bool) {
if ix.Unjudged == nil {
return
}
keys, err := ix.Unjudged.Keys(ctx)
if err != nil {
return
}
for _, k := range slices.Sorted(slices.Values(keys)) {
if held[k] && !judged[k] {
continue
}
if err := write(ctx, ix.Unjudged, k, func(_ []byte, found bool) ([]byte, bool, error) { return nil, found, nil }); err != nil {
ix.logf("judge: %s.%s is not ended: %v; judged again as new by the next pass", UnjudgedState, k, err)
}
}
}
@@ -1,807 +0,0 @@
package index
import (
"context"
"encoding/json"
"errors"
"fmt"
"maps"
"slices"
"strings"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-indexer/judge"
)
// The indexer judges every manifest it holds, once, with its own judge (novox/hq ADR 0578 §1): it writes each verdict in
// its state verdicts (VerdictsState), one key per manifest digest, only when the verdict differs from the one
// held; a module's value in the modules state carries a refusal (Verdict, Problems), so a reader of modules takes it
// without a second read; a new refusal is said as the event manifest-refused; a manifest with a field the judge does not know waits instead (ownBuildIsNewest). It judges at its start, so a judge that
// changed (a new indexer) judges every manifest again, and after every merge it takes in, since a rule between manifests
// can turn on any of them. Nothing polls.
// VerdictsState is the indexer's state of verdicts, its local name, as the tools bundle reads it.
const VerdictsState = "verdicts"
// Judged is what one pass of the judge did: the manifests judged, the verdicts written, and the modules refused.
type Judged struct {
Judged, Written int
Refused map[string][]string
// Outgrown are the modules accepted before whose problems are now warnings (ADR 0579 §1).
Outgrown []string
// Waiting are the modules whose manifest carries a field this judge does not know and waits.
Waiting []string
}
// JudgeAll judges every manifest the modules state names, as one catalogue with the predefined list the indexer keeps,
// and writes what differs. A manifest the manifests state does not hold is left unjudged and said; a write that fails is
// said and the pass goes on, so one key never costs the others their verdict.
//
// **A stricter judge never refuses what it accepted before** (novox/hq ADR 0579 §1): a manifest whose verdict under its
// digest is accepted, or which the indexer held before any verdict (taken as accepted, ADR 0578 §3) and is not in fresh,
// stays accepted, the new problems written as the verdict's Warnings and carried in its module's value, which the
// watchman raises as manifest-outgrown. A manifest in fresh (the digests a merge just took in) with no verdict yet, and
// one already refused, is judged in full.
func (ix *Indexer) JudgeAll(ctx context.Context, fresh map[string]bool) (Judged, error) {
return ix.judgeOver(ctx, fresh, "", "")
}
// judgeOver is JudgeAll; with a module and a manifest digest it judges that digest alone, in the catalogue the modules
// state holds with the module's manifest replaced by it (a module the state does not hold added), and writes its verdict
// and nothing else: no event, no count of the old sections, and no module's value but one that names the digest judged
// (carryVerdict). A digest with no verdict yet is judged in full, as a fresh one. One digest at a time, so the known
// packages dropped for a digest the trunk does not hold are dropped for that digest alone (review round 1, J7's note).
func (ix *Indexer) judgeOver(ctx context.Context, fresh map[string]bool, overModule, overDigest string) (Judged, error) {
var over map[string]string
if overModule != "" {
over = map[string]string{overModule: overDigest}
}
// A merge, the start pass and indexer_judge may all judge at once: one pass at a time, so none writes a verdict from
// modules another is rewriting.
ix.judgeMu.Lock()
defer ix.judgeMu.Unlock()
if !ix.underSeats {
// The seat list stands still for this pass: a take-in's dry run may put its revision's seats over it (seatsof.go).
defer judge.HoldSeats()()
}
return ix.judgeOverHeld(ctx, fresh, over)
}
// judgeOverHeld is judgeOver for a caller that holds the judge pass (judgeMu) and keeps the seat list as it wants it
// (judge.HoldSeats, or inside judge.WithSeats): it takes no lock, so a pass that put its seats over the list judges inside
// it without asking the shared lock the exclusive one forbids.
func (ix *Indexer) judgeOverHeld(ctx context.Context, fresh map[string]bool, over map[string]string) (Judged, error) {
var out Judged
if ix.Verdicts == nil || ix.Modules == nil || ix.Manifests == nil {
return out, nil
}
if over == nil {
ix.SayUnsaid(ctx) // a manifest-accepted a pass before failed to publish is said again first (DLV-I9b)
// What a take-in wrote and no pass judged yet is fresh too, whichever pass judges it (unjudged.go, issue 1966).
fresh = ix.withUnjudged(ctx, fresh)
}
held, err := ix.heldModules(ctx)
if err != nil {
return out, err
}
heldDigests, judgedDigests := map[string]bool{}, map[string]bool{}
for _, m := range held {
heldDigests[m.Manifest] = m.Manifest != ""
}
trunkOf := map[string]string{} // a judged change's module to the trunk's manifest digest it changes (DLV-I15)
// The known packages a manifest's `packages` may name (DLV-I14). The digest judged the trunk does not hold (a pull
// request's candidate, one the trunk no longer names) is not judged on them: the checker judges a change's names,
// knowing its delivery group, whose new packages this state does not know yet (the lead's ruling, issue 1966). The
// trunk's own manifests are, and a merge's are again in full (namesAtMerge).
known := ix.knownPackages(ctx)
// A digest judged over the trunk that is the trunk's own held one is said as the full pass says it (manifest-accepted,
// manifest-refused): indexer_judge with it, or the start pass's judgeStale, is that manifest's only judgement (review
// round 1 of issue 1966; ADR 0668 §4). A candidate's verdict is said nowhere.
onTrunk := map[string]bool{}
if over != nil {
fresh = map[string]bool{}
for module, digest := range over {
m := held[module]
onTrunk[module] = m.Manifest == digest
if m.Manifest != "" && m.Manifest != digest {
trunkOf[module] = m.Manifest
}
if m.Manifest != digest {
known = nil
}
m.Manifest = digest
held[module] = m
fresh[digest] = true
}
}
raws := map[string][]byte{}
digests := map[string]string{}
for _, name := range slices.Sorted(maps.Keys(held)) {
digest := held[name].Manifest
if digest == "" {
continue
}
value, _, found, err := ix.Manifests.Get(ctx, digest)
if err != nil {
return out, fmt.Errorf("reading manifests.%s of %s: %w", digest, name, err)
}
if !found {
ix.logf("judge: the manifest of %s (digest %s) is not in the manifests state, so it is not judged", name, digest)
continue
}
kept, err := meshcatalogclient.DecodeManifest(value)
if err != nil {
ix.logf("judge: manifests.%s of %s cannot be read, so it is not judged: %v", digest, name, err)
continue
}
raws[name] = []byte(kept.Manifest)
digests[name] = digest
}
predefined := ix.predefinedList(ctx)
// The old sections end when no module is left in them (issue 1966 D4): the count is the accepted set the last pass held.
var counts judge.OldSectionCounts
if ix.OldSections != nil {
counts = ix.heldOldSections(ctx)
}
refused, waits, unnamed := judge.ShelfWaiting(raws, predefined, 0, counts, known)
// A change's renamed-from is judged against the trunk's manifest of the same module (DLV-I15, design 32).
for module, digest := range trunkOf {
if _, waiting := waits[module]; waiting || raws[module] == nil {
continue
}
value, _, found, err := ix.Manifests.Get(ctx, digest)
kept, derr := meshcatalogclient.DecodeManifest(value)
if err != nil || !found || derr != nil {
ix.logf("judge: the trunk's manifest %s of %s cannot be read, so its renames are not judged against it", digest, module)
continue
}
head, herr := catalogue.Decode(raws[module])
trunk, terr := catalogue.Decode([]byte(kept.Manifest))
if herr != nil || terr != nil {
continue // the manifest's own problems say why
}
if p := judge.RenameProblems(module, head, trunk, predefined); len(p) > 0 {
refused[module] = append(refused[module], p...)
}
}
for _, p := range unnamed {
ix.logf("judge: a problem between the manifests that no one module's removal ends, so it refuses none: %s", p)
}
// A package's name belongs to one module (design 47 G §491, §676): the trunk's manifests only, since a candidate's
// revision is validated by the validator built from its own heads (§16).
if over == nil {
if refused == nil {
refused = map[string][]string{}
}
for module, p := range judge.PackageNameProblems(ix.packageEntries(ctx, held, raws)) {
if _, waiting := waits[module]; !waiting {
refused[module] = append(refused[module], p...)
}
}
}
out.Refused = refused
// A manifest with a field this judge does not know waits for a judge that may know it (clause 3 of the record on a
// release's Approve; ADR 0578 §1): it is refused only when this program is the newest build of the indexer the trunk
// holds, so no judge that could know the field is still to come.
newest := ix.ownBuildIsNewest(held)
id := ix.judgeID()
var accepted []catalogue.Manifest
for _, name := range slices.Sorted(maps.Keys(digests)) {
if _, judged := over[name]; over != nil && !judged {
continue
}
out.Judged++
v := meshcatalog.VerdictsByDigestValue{Module: name, State: meshcatalogclient.VerdictAccepted, Judge: id, At: ptr(ix.now())}
if fields := waits[name]; len(fields) > 0 {
accepted := ix.acceptedBefore(ctx, digests[name], fresh)
if !newest && !accepted {
v.State, v.Unknown = meshcatalogclient.VerdictManifestWaits, fields
out.Waiting = append(out.Waiting, name)
} else {
_, problems := judge.Manifest(raws[name], ix.predefinedList(ctx))
if accepted {
v.Warnings = problems
out.Outgrown = append(out.Outgrown, name)
} else {
v.State, v.Problems = meshcatalogclient.VerdictRefused, problems
}
}
} else if problems := refused[name]; len(problems) > 0 {
if ix.acceptedBefore(ctx, digests[name], fresh) && !namesAtMerge(name, raws[name], fresh[digests[name]] && over == nil, known) {
v.Warnings = problems
out.Outgrown = append(out.Outgrown, name)
} else {
v.State, v.Problems = meshcatalogclient.VerdictRefused, problems
}
}
if !meshcatalogclient.VerdictRefuses(v) {
if m, err := catalogue.Decode(raws[name]); err == nil {
m.Module = name
accepted = append(accepted, m)
}
}
wrote, revision, err := ix.writeVerdict(ctx, digests[name], v)
if errors.Is(err, process.ErrNotYet) {
return out, fmt.Errorf("the state %s is not on the bus yet: %w", VerdictsState, err)
}
if err != nil {
ix.logf("judge: the verdict on %s (digest %s) is not written: %v", name, digests[name], err)
continue
}
judgedDigests[digests[name]] = true
if wrote && over != nil && !onTrunk[name] {
out.Written++
ix.logf("judge: %s judged again by %s: %s", digests[name], id, v.State)
} else if wrote {
out.Written++
if meshcatalogclient.VerdictWaits(v) {
ix.logf("judge: %s", meshcatalogclient.VerdictWhy(v))
}
if meshcatalogclient.VerdictRefuses(v) {
ix.logf("judge: %s", meshcatalogclient.VerdictWhy(v))
if ix.SayRefused != nil {
r := refusedOf(digests[name], v)
r.Revision = int64(revision)
ix.SayRefused(ctx, r)
}
} else if !meshcatalogclient.VerdictWaits(v) {
// Every manifest newly accepted is said whole to the bus module, its entries empty when it asks for none
// (issue 1966 D3, ADR 0595 §4, design 25 § 12.2); a manifest set aside to wait gets none (DLV-I9).
a := acceptedOf(digests[name], v, revision, raws[name])
_ = ix.sayAccepted(ctx, a, acceptedEventID(a))
}
} else if over == nil && fresh[digests[name]] && !meshcatalogclient.VerdictHeld(v) {
// A manifest a merge just made its module's trunk manifest is newly accepted there, even when a pull request's
// judgement wrote its verdict before the merge and said it nowhere (a candidate's verdict is not said): it is
// said now from the verdict held, with its revision, so the bus module and the grants part hear it (design 25
// bus-state §1; issue 1966: claude-licence-manager's new builds waited for a grant nothing wrote).
ix.sayHeld(ctx, digests[name], raws[name])
}
// The reader capabilities the manifest's own words, forms and seat names need, named with the verdict (ADR 0614 §4).
needs, _ := catalogue.ManifestNeeds(raws[name])
if err := ix.carryVerdict(ctx, name, digests[name], v, needs); err != nil {
ix.logf("judge: modules.%s does not carry its verdict: %v", name, err)
}
}
// The verdicts carried are kept with the view (KeepView), so a restart reads them back.
if err := ix.KeepView(ctx); err != nil {
ix.logf("judge: the verdicts carried are not kept in a catalog graph revision: %v", err)
}
if over != nil {
return out, nil
}
ix.endUnjudged(ctx, judgedDigests, heldDigests)
if err := ix.keepOldSections(ctx, judge.CountOldSections(accepted), judge.CountOwnerReads(accepted)); err != nil {
ix.logf("judge: the count of the old sections is not kept: %v", err)
}
return out, nil
}
// sayHeld says manifest-accepted for the trunk's manifest of digest from the accepted verdict the verdicts state holds on
// it, with that verdict's revision; a verdict that cannot be read, or that refuses or waits, says nothing.
func (ix *Indexer) sayHeld(ctx context.Context, digest string, raw []byte) {
key, err := meshcatalogclient.VerdictsKey(digest)
if err != nil {
return
}
value, revision, found, err := ix.Verdicts.Get(ctx, key)
if err != nil || !found {
ix.logf("judge: the verdict on %s cannot be read, so its manifest-accepted is not said: %v", digest, err)
return
}
v, err := meshcatalogclient.DecodeVerdict(value)
if err != nil || meshcatalogclient.VerdictHeld(v) {
return
}
a := acceptedOf(digest, v, revision, raw)
_ = ix.sayAccepted(ctx, a, acceptedEventID(a))
}
// JudgeAtStart is JudgeAll of every manifest held, none fresh, as the indexer starts (a judge that changed judges every
// one again); one that fails is judged again at the next message or source revision heard (kickJudge). Then every other
// digest of the verdicts state that another judge refused or held waiting (one of a candidate, or one back on the trunk
// later) is judged again by this judge (judgeStale), so a refusal an older judge made never stands for good.
func (ix *Indexer) JudgeAtStart(ctx context.Context) (Judged, error) {
j, err := ix.JudgeAll(ctx, nil)
ix.judgePending.Store(err != nil)
if err == nil {
ix.judgeStale(ctx)
}
return j, err
}
// judgeID is the judge a verdict of this program names.
func (ix *Indexer) judgeID() string {
if ix.JudgeID != "" {
return ix.JudgeID
}
return judge.Identity()
}
// staleVerdict says whether the digest's verdict wants this judge's judgement: there is none, or another judge refused
// it or held it waiting. An accepted verdict stays accepted under any later judge (ADR 0579 §1), so judging it again
// changes nothing.
func (ix *Indexer) staleVerdict(ctx context.Context, digest string) bool {
value, _, found, err := ix.Verdicts.Get(ctx, digest)
if err != nil {
return false // unread: left, said by the next pass that reads it
}
if !found {
return true
}
v, err := meshcatalogclient.DecodeVerdict(value)
return err != nil || (meshcatalogclient.VerdictHeld(v) && v.Judge != ix.judgeID())
}
// judgeDigests judges each digest of over (module to digest) whose verdict is stale (staleVerdict), one at a time in the
// catalogue the modules state holds; a pass that fails is said and the others go on.
func (ix *Indexer) judgeDigests(ctx context.Context, over map[string]string, why string) {
if ix.Verdicts == nil || ix.Manifests == nil || ix.Modules == nil {
return
}
for _, module := range slices.Sorted(maps.Keys(over)) {
if !ix.staleVerdict(ctx, over[module]) {
continue
}
if _, err := ix.judgeOver(ctx, nil, module, over[module]); err != nil {
ix.logf("judge: the manifest %s of %s (%s) is not judged: %v", over[module], module, why, err)
}
}
}
// judgeDigestsWith is judgeDigests with the seats put over the catalogue's list for the whole pass (judge.WithSeats): one
// locked pass that takes the seats, and never a second lock under it (the exclusive one of WithSeats and the shared one of
// judgeOver would wait for each other for good).
func (ix *Indexer) judgeDigestsWith(ctx context.Context, seats []catalogue.Seat, over map[string]string, why string) {
if ix.Verdicts == nil || ix.Manifests == nil || ix.Modules == nil {
return
}
ix.judgeMu.Lock()
defer ix.judgeMu.Unlock()
judge.WithSeats(seats, func() {
for _, module := range slices.Sorted(maps.Keys(over)) {
if !ix.staleVerdict(ctx, over[module]) {
continue
}
if _, err := ix.judgeOverHeld(ctx, nil, map[string]string{module: over[module]}); err != nil {
ix.logf("judge: the manifest %s of %s (%s) is not judged: %v", over[module], module, why, err)
}
}
})
}
// judgeStale judges again every digest of the verdicts state another judge refused or held waiting, each as a manifest
// of the module its verdict names (judgeDigests). Never on a timer: at the start pass.
func (ix *Indexer) judgeStale(ctx context.Context) {
if ix.Verdicts == nil {
return
}
keys, err := ix.Verdicts.Keys(ctx)
if err != nil {
ix.logf("judge: the verdicts other judges made are not judged again: listing the verdicts: %v", err)
return
}
byModule := map[string][]string{}
for _, digest := range keys {
value, _, found, err := ix.Verdicts.Get(ctx, digest)
if err != nil || !found {
continue
}
if v, err := meshcatalogclient.DecodeVerdict(value); err == nil && meshcatalogclient.VerdictHeld(v) && v.Judge != ix.judgeID() {
byModule[v.Module] = append(byModule[v.Module], digest)
}
}
for _, module := range slices.Sorted(maps.Keys(byModule)) {
for _, digest := range byModule[module] {
ix.judgeDigests(ctx, map[string]string{module: digest}, "made by another judge")
}
}
}
// namesAtMerge says whether a manifest a merge takes in (atMerge) names a package known does not: it is refused for it even
// when its digest was accepted before, since that verdict was its candidate's, judged without the names (judgeOver), so
// no stricter judge turns an acceptance into a refusal here (ADR 0579 §1).
func namesAtMerge(module string, raw []byte, atMerge bool, known judge.KnownPackages) bool {
if !atMerge || known == nil {
return false
}
m, err := catalogue.Decode(raw)
return err == nil && len(judge.UnknownPackageNames(module, m, known)) > 0
}
// knownPackages are the packages a manifest's `packages` may name (DLV-I14, design 32): every name the packages state's
// series keys hold, as KeepPackageNames reads it at each merge, and the Go module path of every per-module record the
// state holds from the trunk: while no build module builds Go package artifacts, a Go module the trunk holds a record of
// has no series and is known by its module path, never by a package inside it (design 32; issue 1966). Nil, which judges
// no name, where the indexer keeps no packages state or it cannot be listed (said): a state the judge cannot read refuses no manifest.
func (ix *Indexer) knownPackages(ctx context.Context) judge.KnownPackages {
if ix.Packages == nil {
return nil
}
keys, err := ix.Packages.Keys(ctx)
if err != nil {
ix.logf("judge: the packages state cannot be listed, so the names of packages are not judged: %v", err)
return nil
}
known := judge.KnownPackages{}
for _, key := range keys {
value, _, found, err := ix.Packages.Get(ctx, key)
if err != nil {
ix.logf("judge: packages.%s cannot be read, so the names of packages are not judged: %v", key, err)
return nil
}
if !IsSeriesKey(key) {
if r, err := DecodePublishedPackages(value); found && err == nil && r.Module != "" {
known[r.Module] = true
}
continue
}
if s, err := DecodeSeriesPackage(value); found && err == nil {
known[s.Name] = true
}
}
return known
}
// kickJudge judges again a start pass that failed, when the indexer hears something: never on a timer.
func (ix *Indexer) kickJudge(ctx context.Context) {
if !ix.judgePending.Load() {
return
}
if j, err := ix.JudgeAtStart(ctx); err != nil {
ix.logf("judge: the manifests are still not judged: %v", err)
} else {
ix.logf("judge: judged %d manifests again after a failed start: %d verdicts written", j.Judged, j.Written)
}
}
// acceptedBefore says whether the manifest of digest was accepted before this judgement: its verdict says accepted, or
// it has none and is not fresh (held before any verdict, so taken in under the judges that ran until then).
func (ix *Indexer) acceptedBefore(ctx context.Context, digest string, fresh map[string]bool) bool {
value, _, found, err := ix.Verdicts.Get(ctx, digest)
if err != nil {
return false // unread: judged in full, said by the write that follows
}
if !found {
return !fresh[digest]
}
prev, err := meshcatalogclient.DecodeVerdict(value)
return err == nil && prev.State == meshcatalogclient.VerdictAccepted
}
// writeVerdict writes v under the digest's key unless the verdict held there says the same (ManifestVerdict.Same); wrote
// says it was written, and revision is the verdicts state's revision of that write, which the event saying it carries
// (novox/hq ADR 0622 §2). It is read back after the write: when another write came between, the revision read holds
// another verdict and revision is 0, so a reader takes the later write's own event and never pairs v with its revision.
func (ix *Indexer) writeVerdict(ctx context.Context, digest string, v meshcatalog.VerdictsByDigestValue) (wrote bool, revision uint64, err error) {
key, err := meshcatalogclient.VerdictsKey(digest)
if err != nil {
return false, 0, err
}
err = write(ctx, ix.Verdicts, key, func(value []byte, found bool) ([]byte, bool, error) {
wrote = false
if found {
if cur, err := meshcatalogclient.DecodeVerdict(value); err == nil && meshcatalogclient.SameVerdict(cur, v) {
return nil, false, nil
}
}
wrote = true
next, err := meshcatalogclient.EncodeVerdict(v)
return next, false, err
})
if err != nil || !wrote {
return wrote, 0, err
}
if value, rev, found, err := ix.Verdicts.Get(ctx, key); err == nil && found {
if cur, err := meshcatalogclient.DecodeVerdict(value); err == nil && meshcatalogclient.SameVerdict(cur, v) {
revision = rev
}
}
return wrote, revision, nil
}
// OwnBuildState is the indexer's state of the build it runs (the bucket mesh-delivery-indexer_own-build, one key, OwnBuildKey):
// written once, at the process's start by FixOwnBuild, so indexer_pending says which build this process knows it runs, and since
// when. The build only names the writer of what is written (A-476 b).
const (
OwnBuildStateName = "own-build"
OwnBuildKey = "state"
)
// OwnBuild is the value of the own-build state.
type OwnBuild struct {
V int `json:"v"`
Known bool `json:"known"`
FP string `json:"fp,omitempty"`
Since time.Time `json:"since"`
Why string `json:"why,omitempty"`
}
// FixOwnBuild fixes, once and at the process's start, which build of the indexer this program is: asked of the node-engine's
// seat (OwnBuild; never the trunk's value of the indexer, which names a build that may be merged and not delivered), before
// this process hears a merge. It is never asked again: while the node-engine replaces the indexer the seat already names the
// successor, and an old process that learnt it later would take revisions in with its old judge. Where the start's ask is not
// answered the build stays unknown for the life of this process, said in the own-build state and the journal; nothing waits for it.
func (ix *Indexer) FixOwnBuild(ctx context.Context) {
ix.ownMu.Lock()
if ix.ownFixed || ix.OwnBuild == nil {
ix.ownMu.Unlock()
return
}
ix.ownFixed = true
ix.ownMu.Unlock()
fp, err := ix.OwnBuild(ctx)
if err == nil && fp == "" {
err = errors.New("the node-engine names no build of the indexer")
}
st := OwnBuild{V: 1, Since: ix.now().UTC()}
if err != nil {
st.Why = err.Error()
ix.logf("judge: the build this program runs is not known (%v); merges are taken in all the same, and it is learnt only at the start", err)
} else {
st.Known, st.FP = true, fp
ix.logf("judge: this program runs the indexer build %s, fixed here at its start", fp)
}
ix.ownMu.Lock()
ix.ownFP, ix.ownSince, ix.ownWhy = fp, st.Since, st.Why
ix.ownMu.Unlock()
if ix.OwnBuildState == nil {
return
}
data, _ := json.Marshal(st)
if _, rev, found, gerr := ix.OwnBuildState.Get(ctx, OwnBuildKey); gerr == nil && found {
err = ix.OwnBuildState.Update(ctx, OwnBuildKey, data, rev)
} else if gerr == nil {
err = ix.OwnBuildState.Create(ctx, OwnBuildKey, data)
} else {
err = gerr
}
if err != nil {
ix.logf("the own-build state cannot be written: %v", err)
}
}
// ownBuildIsNewest says whether the build of the indexer this program runs (ownFP, from the node-engine) is the newest the
// trunk holds: the indexer's own modules value names it. A newer build merged and not yet delivered ends it at the trunk's
// merge, as one that pins a changed mesh-sdk always is; false when either is unknown.
func (ix *Indexer) ownBuildIsNewest(held map[string]meshcatalog.ModulesByModuleValue) bool {
fp := held[ix.Module].Fp2
if fp == "" {
return false
}
ix.ownMu.Lock()
defer ix.ownMu.Unlock()
return ix.ownFP != "" && ix.ownFP == fp
}
// carryVerdict puts a refusal into the module's value, and takes one out once the manifest is accepted, while the value
// still names the digest judged, and the reader capabilities the manifest needs beside it (needs, ADR 0614 §4); an
// accepted module's value carries no verdict, so judging writes no value of a module the
// judge accepts (a reader takes a value without a verdict as accepted, meshcatalogclient.TakeVerdict).
func (ix *Indexer) carryVerdict(ctx context.Context, name, digest string, v meshcatalog.VerdictsByDigestValue, needs []string) error {
key, err := meshcatalogclient.ModulesKey(name)
if err != nil {
return err
}
return write(ctx, ix.Modules, key, func(value []byte, found bool) ([]byte, bool, error) {
if !found {
return nil, false, nil
}
cur, err := meshcatalogclient.DecodeModule(value)
if err != nil || cur.Manifest != digest {
return nil, false, nil // gone, unreadable, or moved to another manifest since: the next pass judges it
}
// A wait is written as a refusal naming the fields, so a reader pinned before the verdict waits holds it as refused.
state, problems := "", v.Problems
switch {
case meshcatalogclient.VerdictWaits(v):
state, problems = meshcatalogclient.VerdictRefused, []string{meshcatalogclient.WaitsOn(v)}
case meshcatalogclient.VerdictRefuses(v):
state = meshcatalogclient.VerdictRefused
}
// The judge reads the manifest alone; the forms its templates' text writes came with its merge (ADR 0632 §2) and
// stand, cur being of the same manifest.
needs := withTemplateForms(needs, cur.Needs)
if cur.Verdict == state && slices.Equal(cur.Problems, problems) && slices.Equal(cur.Unknown, v.Unknown) && slices.Equal(cur.Warnings, v.Warnings) &&
slices.Equal(cur.Needs, needs) {
return nil, false, nil
}
cur.Verdict, cur.Problems, cur.Unknown, cur.Warnings, cur.Needs = state, problems, v.Unknown, v.Warnings, needs
if state == "" {
cur.Problems = nil
}
next, err := meshcatalogclient.EncodeModule(cur)
return next, false, err
})
}
// predefinedList is the catalogue's predefined list the indexer keeps (writePredefined), or nil when it keeps none or
// it cannot be read: the judge then takes each fallback as its value declares it.
func (ix *Indexer) predefinedList(ctx context.Context) catalogue.PredefinedList {
if ix.Predefined == nil {
return nil
}
value, _, found, err := ix.Predefined.Get(ctx, PredefinedListKey)
if err != nil || !found {
return nil
}
var kept struct {
List json.RawMessage `json:"list"`
}
if json.Unmarshal(value, &kept) != nil {
return nil
}
list, err := catalogue.ParsePredefined(kept.List)
if err != nil {
ix.logf("judge: the predefined list kept cannot be read, so fallbacks are judged as their values declare them: %v", err)
return nil
}
return list
}
// The tools that read and steer the verdicts (the instruction "tools are how a module is spoken to"): indexer_verdicts
// lists the manifests that wait or are refused, and indexer_judge judges one module again now.
const (
VerdictsTool = "indexer_verdicts"
JudgeAgainTool = "indexer_judge"
judgeAgainPrefix = "judge."
)
// VerdictRow is one verdict as indexer_verdicts lists it: the manifest's digest, the module, its state, the fields it waits
// on or the problems that refuse it, and the judge that made it.
type VerdictRow struct {
Module string `json:"module"`
Digest string `json:"digest"`
State string `json:"state"`
Words string `json:"words,omitempty"`
Unknown []string `json:"unknown,omitempty"`
Problems []string `json:"problems,omitempty"`
Warnings []string `json:"warnings,omitempty"`
Judge string `json:"judge"`
At time.Time `json:"at"`
}
// ListVerdicts reads the verdicts under keys with get and lists those of module (every module when empty) that wait or are
// refused, or every one with all, sorted by module and digest. A value that does not read is said in unread by its key.
func ListVerdicts(keys []string, get func(key string) ([]byte, error), module string, all bool) (rows []VerdictRow, unread map[string]string) {
unread = map[string]string{}
for _, key := range keys {
value, err := get(key)
if err != nil || value == nil {
if err != nil {
unread[key] = err.Error()
}
continue
}
v, err := meshcatalogclient.DecodeVerdict(value)
if err != nil {
unread[key] = err.Error()
continue
}
if (module != "" && v.Module != module) || (!all && !meshcatalogclient.VerdictHeld(v)) {
continue
}
rows = append(rows, VerdictRow{Module: v.Module, Digest: key, State: v.State, Words: meshcatalogclient.VerdictWhy(v), Unknown: v.Unknown,
Problems: v.Problems, Warnings: v.Warnings, Judge: v.Judge, At: timeOf(v.At)})
}
slices.SortFunc(rows, func(a, b VerdictRow) int {
if c := strings.Compare(a.Module, b.Module); c != 0 {
return c
}
return strings.Compare(a.Digest, b.Digest)
})
return rows, unread
}
// JudgeAgain judges the manifests again now, as at the start, and answers how the module's stands: the tool
// indexer_judge. A module the modules state does not hold, or whose manifest is not in the manifests state, is
// answered so. With a digest it judges that manifest alone, in the catalogue the trunk holds (judgeOver), whoever made
// its verdict: a candidate's, or one the trunk names again, refused by an older judge.
func (ix *Indexer) JudgeAgain(ctx context.Context, module, digest string) TakenIn {
out := TakenIn{Module: module}
held, err := ix.heldModules(ctx)
if err != nil {
out.Answer = "the modules state cannot be read: " + err.Error()
return out
}
if digest != "" {
// One digest (a candidate's, or one the trunk no longer names), judged by this judge in the catalogue the trunk holds.
value, _, found, err := ix.Manifests.Get(ctx, digest)
kept, derr := meshcatalogclient.DecodeManifest(value)
switch {
case err != nil || !found:
out.Answer = "the indexer keeps no manifest of digest " + digest
return out
case derr != nil:
out.Answer = "the manifest of digest " + digest + " cannot be read: " + derr.Error()
return out
case module != "" && kept.Module != module:
out.Answer = "the manifest of digest " + digest + " is " + kept.Module + "'s, not " + module + "'s"
return out
}
out.Module = kept.Module
if _, err := ix.judgeOver(ctx, nil, kept.Module, digest); err != nil {
out.Answer = "the manifest could not be judged: " + err.Error()
return out
}
return ix.answerVerdict(ctx, out, digest)
}
m, ok := held[module]
if !ok || m.Manifest == "" {
out.Answer = "the indexer holds no manifest of " + module
return out
}
if _, err := ix.JudgeAll(ctx, nil); err != nil {
out.Answer = "the manifests could not be judged: " + err.Error()
return out
}
return ix.answerVerdict(ctx, out, m.Manifest)
}
// answerVerdict is out answering how the verdict under digest stands.
func (ix *Indexer) answerVerdict(ctx context.Context, out TakenIn, digest string) TakenIn {
module := out.Module
value, _, found, err := ix.Verdicts.Get(ctx, digest)
if err != nil || !found {
out.Answer = "judged, but the verdict on " + module + " cannot be read"
return out
}
v, err := meshcatalogclient.DecodeVerdict(value)
if err != nil {
out.Answer = "judged, but the verdict on " + module + " cannot be read: " + err.Error()
return out
}
out.Stored = true
out.Answer = "judged again by " + v.Judge + ": " + v.State
if meshcatalogclient.VerdictHeld(v) {
out.Answer += " (" + meshcatalogclient.VerdictWhy(v) + ")"
}
return out
}
// NewJudgeAgain is the value indexer_judge writes for a module, and one digest of its manifest when given: its key in
// the takeins state and the value.
func NewJudgeAgain(module, digest, why, by string, at time.Time) (string, []byte, error) {
if digest != "" {
if _, err := meshcatalogclient.ManifestsKey(digest); err != nil {
return "", nil, err
}
}
key, data, err := NewTakeIn(module, why, by, at)
if err != nil {
return "", nil, err
}
var a TakeInAsk
if err := json.Unmarshal(data, &a); err != nil {
return "", nil, err
}
a.Judge, a.Digest = true, digest
data, err = json.Marshal(a)
return judgeAgainPrefix + key, data, err
}
// withTemplateForms is needs with the reference forms of had added: a module's templates write forms only (catalogue
// ModuleNeeds), and a merge carried them with the manifest the judge reads again alone.
func withTemplateForms(needs, had []string) []string {
out := slices.Clone(needs)
for _, n := range had {
if strings.HasPrefix(n, catalogue.ReadForm) && !slices.Contains(out, n) {
out = append(out, n)
}
}
slices.Sort(out)
return out
}
// timeOf is the time a pointer holds, the zero time for none.
func timeOf(t *time.Time) time.Time {
if t == nil {
return time.Time{}
}
return *t
}
@@ -1,612 +0,0 @@
package index
import (
"context"
"errors"
"fmt"
"slices"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-indexer/judge"
)
// The indexer judges every manifest it holds and writes each verdict only when it differs; a module's value carries a
// refusal, which is said once as manifest-refused; a new judge writes again only what it judges otherwise; a manifest
// mended clears the module's refusal (novox/hq ADR 0578 §1).
func TestTheIndexerJudgesEveryManifestOnce(t *testing.T) {
ctx := context.Background()
ix := &Indexer{Module: testSelf, Modules: newMem(), Manifests: newMem(), Verdicts: newMem(), JudgeID: "judge-1",
Now: func() time.Time { return time.Date(2026, 11, 1, 3, 0, 0, 0, time.UTC) }}
var said []meshcatalog.ManifestRefusedEvent
ix.SayRefused = func(_ context.Context, r meshcatalog.ManifestRefusedEvent) { said = append(said, r) }
hold := func(module, raw string) string {
digest, err := ix.keepManifest(ctx, module, raw)
if err != nil {
t.Fatal(err)
}
value, err := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: "src2:" + strings.Repeat("a", 64), Manifest: digest,
Repository: "novox/mesh-catalog", Commit: "c"})
if err != nil {
t.Fatal(err)
}
ix.Modules.(*memStore).put(module, value)
return digest
}
good := hold("good", `{"module":"good","version":"1.0"}`)
bad := hold("bad", `{"module":"bad","version":"1.0","requires":["amqp"]}`)
// Both taken in by a merge: judged in full.
j, err := ix.JudgeAll(ctx, map[string]bool{good: true, bad: true})
if err != nil || j.Judged != 2 || j.Written != 2 || len(j.Refused) != 1 {
t.Fatalf("the first pass: %+v %v", j, err)
}
verdict := func(digest string) meshcatalog.VerdictsByDigestValue {
value, _, found, _ := ix.Verdicts.Get(ctx, digest)
v, err := meshcatalogclient.DecodeVerdict(value)
if !found || err != nil {
t.Fatalf("no verdict under %s: %v", digest, err)
}
return v
}
if v := verdict(good); meshcatalogclient.VerdictRefuses(v) || v.Judge != "judge-1" {
t.Fatalf("good: %+v", v)
}
if v := verdict(bad); !meshcatalogclient.VerdictRefuses(v) || !strings.Contains(v.Problems[0], `requires "amqp"`) {
t.Fatalf("bad: %+v", v)
}
module := func(name string) meshcatalog.ModulesByModuleValue {
value, _, _, _ := ix.Modules.Get(ctx, name)
m, err := meshcatalogclient.DecodeModule(value)
if err != nil {
t.Fatal(err)
}
return m
}
if m := module("bad"); m.Verdict != meshcatalogclient.VerdictRefused || len(m.Problems) != 1 {
t.Fatalf("modules.bad carries %+v", m)
}
if m := module("good"); m.Verdict != "" {
t.Fatalf("modules.good carries %q", m.Verdict)
}
// ADR 0622 §2: the event carries the verdicts state's revision of the verdict it says.
if _, rev, _, _ := ix.Verdicts.Get(ctx, bad); len(said) != 1 || said[0].Module != "bad" || said[0].Digest != bad ||
said[0].Judge != "judge-1" || said[0].Revision == 0 || said[0].Revision != int64(rev) {
t.Fatalf("said %+v (the verdict's revision %d)", said, rev)
}
// A new judge that judges alike writes nothing and says nothing again.
ix.JudgeID = "judge-2"
if j, err := ix.JudgeAll(ctx, nil); err != nil || j.Written != 0 || len(said) != 1 {
t.Fatalf("a judge that judges alike: %+v %v, said %d", j, err, len(said))
}
// ADR 0603 §4 (D9): a source revision that changes the indexer builds the new indexer, which at its start judges every
// manifest it holds again, none of them fresh: all of them judged, nothing written for a verdict that stands.
if j, err := ix.JudgeAtStart(ctx); err != nil || j.Judged != 2 || j.Written != 0 || len(said) != 1 {
t.Fatalf("the new indexer at its start: %+v %v, said %d, want both manifests judged and nothing written", j, err, len(said))
}
// The manifest mended: its new digest is accepted and the module's refusal goes.
mended := hold("bad", `{"module":"bad","version":"2.0"}`)
if j, err := ix.JudgeAll(ctx, map[string]bool{mended: true}); err != nil || j.Written != 1 || len(j.Refused) != 0 {
t.Fatalf("mended: %+v %v", j, err)
}
if m := module("bad"); m.Verdict != "" || m.Problems != nil {
t.Fatalf("modules.bad still carries %+v", m)
}
// ADR 0579 §1: a manifest held before any verdict (taken in under the judges until then), which this judge finds a
// problem in, stays accepted with the problem as a warning, carried in its value and said nowhere as a refusal; the
// same problem in a manifest a merge takes in now, or in one whose digest changed, is refused in full.
old := hold("old", `{"module":"old","version":"1.0","requires":["amqp"]}`)
if j, err := ix.JudgeAll(ctx, nil); err != nil || len(j.Outgrown) != 1 || j.Outgrown[0] != "old" {
t.Fatalf("an outgrown manifest: %+v %v", j, err)
}
if v := verdict(old); meshcatalogclient.VerdictRefuses(v) || len(v.Warnings) != 1 || !strings.Contains(v.Warnings[0], `requires "amqp"`) {
t.Fatalf("old: %+v", v)
}
if m := module("old"); m.Verdict != "" || len(m.Warnings) != 1 || len(said) != 1 {
t.Fatalf("modules.old carries %+v, said %d", m, len(said))
}
changed := hold("old", `{"module":"old","version":"2.0","requires":["amqp"]}`)
fresh := hold("new", `{"module":"new","version":"1.0","requires":["amqp"]}`)
if j, err := ix.JudgeAll(ctx, map[string]bool{changed: true, fresh: true}); err != nil || len(j.Outgrown) != 0 {
t.Fatalf("a changed and a new manifest: %+v %v", j, err)
}
if !meshcatalogclient.VerdictRefuses(verdict(changed)) || !meshcatalogclient.VerdictRefuses(verdict(fresh)) || module("old").Verdict != meshcatalogclient.VerdictRefused || len(module("old").Warnings) != 0 {
t.Fatalf("the changed and the new manifest are not refused: %+v %+v", verdict(changed), verdict(fresh))
}
// A merge's write of a module whose digest it leaves as it is carries the verdict over, and nothing is fresh; a changed
// digest is fresh, judged in full (review R1, R2).
w := &mergeWrite{}
oldNeeds := module("old").Needs
same := meshcatalog.ModulesByModuleValue{Fp2: "src2:" + strings.Repeat("c", 64), Manifest: changed, Repository: "novox/mesh-catalog", Commit: "c2"}
if err := ix.writeOverlay(ctx, map[string]meshcatalog.ModulesByModuleValue{"old": same}, nil, w); err != nil {
t.Fatal(err)
}
if m := module("old"); m.Verdict != meshcatalogclient.VerdictRefused || len(m.Problems) == 0 || len(w.fresh) != 0 {
t.Fatalf("a merge rewrote a refused module as %+v, fresh %v", m, w.fresh)
}
if got := module("old").Needs; len(oldNeeds) == 0 || !slices.Equal(got, oldNeeds) {
t.Fatalf("a merge leaving the digest dropped the reader capabilities the judge named: %v, held %v", got, oldNeeds)
}
other := strings.Repeat("d", 64)
same.Manifest = other
if err := ix.writeOverlay(ctx, map[string]meshcatalog.ModulesByModuleValue{"old": same}, nil, w); err != nil || !w.fresh[other] {
t.Fatalf("a changed digest is not fresh: %v %v", w.fresh, err)
}
// 1966 (ADR 0668 §4, ADR 0579 §1): what a take-in newly wrote is kept on the bus until judged. A take-in writes a
// manifest a judge refuses in full, its judge pass fails on the verdicts state, and the next pass, which knows nothing
// of that take-in (a start, a pass kicked by a message), still refuses it in full, never as a warning, and ends the key.
ix.Unjudged = newMem()
keptDigest, err := ix.keepManifest(ctx, "kept", `{"module":"kept","version":"1.0","requires":["amqp"]}`)
if err != nil {
t.Fatal(err)
}
kw := &mergeWrite{repository: "novox/mesh-catalog", merge: "m-kept"}
keptValue := meshcatalog.ModulesByModuleValue{Fp2: "src2:" + strings.Repeat("e", 64), Manifest: keptDigest, Repository: "novox/mesh-catalog", Commit: "c3"}
if err := ix.writeOverlay(ctx, map[string]meshcatalog.ModulesByModuleValue{"kept": keptValue}, nil, kw); err != nil || !kw.fresh[keptDigest] {
t.Fatalf("the take-in's write: %v, fresh %v", err, kw.fresh)
}
if value, _, found, _ := ix.Unjudged.Get(ctx, keptDigest); !found || !strings.Contains(string(value), `"module":"kept"`) || !strings.Contains(string(value), "m-kept") {
t.Fatalf("the new manifest is not kept as unjudged: %s", value)
}
ix.Verdicts.(*memStore).failWrite = func(string) error { return process.ErrNotYet }
if _, err := ix.JudgeAll(ctx, kw.fresh); err == nil {
t.Fatal("a pass whose verdicts state is not on the bus did not fail")
}
ix.Verdicts.(*memStore).failWrite = nil
if j, err := ix.JudgeAll(ctx, nil); err != nil || slices.Contains(j.Outgrown, "kept") {
t.Fatalf("the next pass: %+v %v", j, err)
}
if v := verdict(keptDigest); !meshcatalogclient.VerdictRefuses(v) || len(v.Warnings) != 0 {
t.Fatalf("a refusal of a manifest whose judge pass failed became %+v", v)
}
if keys, err := ix.Unjudged.Keys(ctx); err != nil || len(keys) != 0 {
t.Fatalf("the unjudged state still holds %v (%v) after its verdict is written", keys, err)
}
ix.Unjudged = nil
// 1966/AC10: a newly accepted manifest asking for bus state is said once as manifest-accepted, carrying both entries
// whole; a verdict already held says nothing.
var accepted []meshcatalog.ManifestAcceptedEvent
var ids []string
var failing int // publishes still to fail
ix.SayAccepted = func(_ context.Context, a meshcatalog.ManifestAcceptedEvent, id string) error {
if failing > 0 {
failing--
return errors.New("no stream answered")
}
accepted, ids = append(accepted, a), append(ids, id)
return nil
}
moved := hold("moved", `{"module":"moved","version":"1.0","requires":[`+
`{"name":"cache","storage":"key-value","template":"state","options":{"history":8}},`+
`{"name":"audit","storage":"stream","template":"log","options":{"max-mib":64}}]}`)
// The merge's pass, then a later one that knows nothing of the merge.
for _, fresh := range []map[string]bool{{moved: true}, nil} {
if _, err := ix.JudgeAll(ctx, fresh); err != nil {
t.Fatal(err)
}
}
// ADR 0614 §4: the judge names, with the verdict, the reader capabilities the manifest's own words need: the bus-state
// words, which an engine stating nothing (the baseline) lacks; a manifest of the baseline's words lacks none.
if needs := module("moved").Needs; !slices.Contains(needs, "word:bus-state.storage") ||
!slices.Contains(catalogue.MissingManifestReads(needs, nil), "word:bus-state") {
t.Fatalf("modules.moved needs %v", needs)
}
if needs := module("good").Needs; !slices.Contains(needs, "word:module") || len(catalogue.MissingManifestReads(needs, nil)) != 0 {
t.Fatalf("modules.good needs %v", needs)
}
if _, rev, _, _ := ix.Verdicts.Get(ctx, moved); len(accepted) != 1 || accepted[0].Revision == 0 || accepted[0].Revision != int64(rev) {
t.Fatalf("said accepted %+v, want the verdict's revision %d (ADR 0622 §2)", accepted, rev)
}
if len(accepted) != 1 || accepted[0].Module != "moved" || accepted[0].Digest != moved || accepted[0].Judge != "judge-2" ||
len(accepted[0].BusState) != 2 || accepted[0].BusState[0].Name != "cache" || accepted[0].BusState[1].Template != "log" ||
fmt.Sprint(accepted[0].BusState[0].Options["history"]) != "8" {
t.Fatalf("said accepted %+v", accepted)
}
// DLV-I9 (design 25 § 12.2): a manifest asking for none, newly accepted, is said too, its entries an empty list.
plain := hold("plain", `{"module":"plain","version":"1.0"}`)
if _, err := ix.JudgeAll(ctx, map[string]bool{plain: true}); err != nil {
t.Fatal(err)
}
if last := accepted[len(accepted)-1]; last.Module != "plain" || last.Digest != plain || last.BusState == nil || len(last.BusState) != 0 {
t.Fatalf("a manifest asking for no bus state was said as %+v", last)
}
// Review round 1 of issue 1966: a trunk's held manifest judged by its digest (indexer_judge, judgeStale) is said as the
// full pass says it, accepted or refused; a candidate's digest judged over the trunk is said nowhere.
byDigest := hold("bydigest", `{"module":"bydigest","version":"1.0"}`)
refusedBy := hold("refusedby", `{"module":"refusedby","version":"1.0","requires":["amqp"]}`)
saidBefore := len(said)
for module, digest := range map[string]string{"bydigest": byDigest, "refusedby": refusedBy} {
if _, err := ix.judgeOver(ctx, nil, module, digest); err != nil {
t.Fatal(err)
}
}
if last := accepted[len(accepted)-1]; last.Digest != byDigest || len(said) != saidBefore+1 || said[len(said)-1].Digest != refusedBy {
t.Fatalf("judged by its digest, the trunk's manifest was said as %+v, refusals %d (were %d)", last, len(said), saidBefore)
}
candidate, err := ix.keepManifest(ctx, "bydigest", `{"module":"bydigest","version":"2.0"}`)
if err != nil {
t.Fatal(err)
}
if _, err := ix.judgeOver(ctx, nil, "bydigest", candidate); err != nil || accepted[len(accepted)-1].Digest != byDigest {
t.Fatalf("a candidate's verdict was said: %v %+v", err, accepted[len(accepted)-1])
}
// Issue 1966 (design 25 bus-state §1): the candidate merged, its verdict unchanged, is newly accepted on the trunk and
// said at the merge's pass, from the verdict held and with its revision, though the pass writes no verdict.
if hold("bydigest", `{"module":"bydigest","version":"2.0"}`) != candidate {
t.Fatal("the merged manifest is not the candidate's")
}
if j, err := ix.JudgeAll(ctx, map[string]bool{candidate: true}); err != nil || j.Written != 0 {
t.Fatalf("the merge's pass: %+v %v", j, err)
}
if _, rev, _, _ := ix.Verdicts.Get(ctx, candidate); accepted[len(accepted)-1].Digest != candidate || accepted[len(accepted)-1].Revision != int64(rev) {
t.Fatalf("the merged candidate's manifest was not said accepted with its verdict's revision %d: last said %+v", rev, accepted[len(accepted)-1])
}
// DLV-I9b: a failed publish is published again; one failing twice writes the finding naming the module, which the next
// judging pass says again and clears.
ix.Findings = newFindings()
failing = 1
once := hold("once", `{"module":"once","version":"1.0"}`)
if _, err := ix.JudgeAll(ctx, map[string]bool{once: true}); err != nil || accepted[len(accepted)-1].Module != "once" {
t.Fatalf("a publish failing once was not published again: %v %+v", err, accepted[len(accepted)-1])
}
failing = 2
flaky := hold("flaky", `{"module":"flaky","version":"1.0"}`)
if _, err := ix.JudgeAll(ctx, map[string]bool{flaky: true}); err != nil {
t.Fatal(err)
}
if value, found := ix.Findings.(*memFindings).vals[unsaidKey("flaky")]; !found || !strings.Contains(string(value), "flaky") {
t.Fatalf("a publish failing twice wrote no finding naming flaky: %s", value)
}
if _, err := ix.JudgeAll(ctx, nil); err != nil {
t.Fatal(err)
}
if _, found := ix.Findings.(*memFindings).vals[unsaidKey("flaky")]; found || accepted[len(accepted)-1].Module != "flaky" ||
accepted[len(accepted)-1].Digest != flaky {
t.Fatalf("the next pass did not say flaky and clear its finding (found %v, last said %+v)", found, accepted[len(accepted)-1])
}
// DLV-I9c: indexer_say_accepted says it again from the held verdict, under an id of its own; a module with no accepted
// verdict, or none held, is refused.
if r := ix.SayAcceptedAgain(ctx, "flaky"); !r.Stored || accepted[len(accepted)-1].Module != "flaky" || !strings.Contains(ids[len(ids)-1], "-again-") {
t.Fatalf("said again: %+v, id %s", r, ids[len(ids)-1])
}
for _, m := range []string{"old", "nobody"} {
if r := ix.SayAcceptedAgain(ctx, m); r.Stored {
t.Errorf("%s said again: %+v", m, r)
}
}
if key, value, err := NewSayAccepted("flaky", "the bus module missed it", "jochen@shanks", time.Now()); err != nil ||
!strings.HasPrefix(key, sayAcceptedPrefix) || !strings.Contains(string(value), `"say":true`) {
t.Errorf("the ask = %s %s (%v)", key, value, err)
}
if _, _, err := NewSayAccepted("flaky", "", "jochen@shanks", time.Now()); err == nil {
t.Error("an ask saying no why was taken")
}
// DLV-I15: a change's renamed-from is judged against the trunk's manifest of the module: a name the trunk declares is
// accepted, one it does not is refused naming the key.
trunkRenamer := hold("renamer", `{"module":"renamer","version":"1.0","config":{"font-name":{"type":"string","description":"The font.","required":true,"runtime":true}}}`)
if _, err := ix.JudgeAll(ctx, map[string]bool{trunkRenamer: true}); err != nil {
t.Fatal(err)
}
for raw, want := range map[string]string{
`{"module":"renamer","version":"1.0","config":{"font":{"type":"string","description":"The font.","required":true,"runtime":true,"renamed-from":"font-name"}}}`: "",
`{"module":"renamer","version":"1.0","config":{"font":{"type":"string","description":"The font.","required":true,"runtime":true,"renamed-from":"typeface"}}}`: `renamed from "typeface"`,
} {
head, err := ix.keepManifest(ctx, "renamer", raw)
if err != nil {
t.Fatal(err)
}
ix.JudgeAgain(ctx, "renamer", head)
if v := verdict(head); (want == "") != !meshcatalogclient.VerdictRefuses(v) || (want != "" && !strings.Contains(strings.Join(v.Problems, " "), want)) {
t.Errorf("the change %s: %+v, want refused naming %q", raw, v, want)
}
}
// DLV-I14 and the lead's ruling (issue 1966): a pull request's candidate is not refused for naming a package the
// packages state does not know (the checker judges its names, knowing its delivery group); the same manifest a merge
// takes in is, though its candidate was accepted; a name the state knows is accepted at both.
ix.Packages = newMem()
sdkGo := "git.novox.be/novox/mesh-sdk/go"
if value, err := EncodeSeriesPackage(SeriesPackage{V: 1, Name: sdkGo, Package: "mesh-sdk/go", Series: "0.1"}); err != nil {
t.Fatal(err)
} else {
ix.Packages.(*memStore).put(SeriesKey(sdkGo, "0.1"), value)
}
for raw, refusedAtMerge := range map[string]bool{
`{"module":"pkguser","version":"1.0","packages":{"@novox/ui":"0.1"}}`: true,
`{"module":"pkguser","version":"2.0","packages":{"` + sdkGo + `":"0.1"}}`: false,
} {
head, err := ix.keepManifest(ctx, "pkguser", raw)
if err != nil {
t.Fatal(err)
}
ix.judgeDigests(ctx, map[string]string{"pkguser": head}, "novox/mesh-catalog#1")
if v := verdict(head); meshcatalogclient.VerdictRefuses(v) {
t.Errorf("the candidate %s is refused: %+v", raw, v)
}
merged := hold("pkguser", raw)
if _, err := ix.JudgeAll(ctx, map[string]bool{merged: true}); err != nil {
t.Fatal(err)
}
if v := verdict(merged); meshcatalogclient.VerdictRefuses(v) != refusedAtMerge || refusedAtMerge && !strings.Contains(strings.Join(v.Problems, " "), `name "@novox/ui", which no package`) {
t.Errorf("the merge of %s: %+v, want refused %v for the name", raw, v, refusedAtMerge)
}
}
ix.Packages = nil
// 1966/AC11: modules using state and none using logs but the foundation: logs counts 0, so a manifest of another
// module using logs is refused and the foundation's is accepted.
ix.OldSections = newMem()
hold("user", `{"module":"user","version":"1.0","state":["x"]}`)
hold("mesh-grants", `{"module":"mesh-grants","version":"1.0","logs":[{"name":"l"}]}`)
if _, err := ix.JudgeAll(ctx, nil); err != nil {
t.Fatal(err)
}
counts := func() map[string]int {
value, _, found, err := ix.OldSections.Get(ctx, OldSectionsKey)
if err != nil || !found {
t.Fatalf("no count of the old sections: %v", err)
}
c, err := DecodeOldSections(value)
if err != nil {
t.Fatal(err)
}
return c.Counts
}
if c := counts(); c["state"] != 1 || c["logs"] != 0 || c["streams"] != 0 || c["objects"] != 0 {
t.Fatalf("the old sections count %v, want state 1 and the rest 0", c)
}
// DLV-J2, DLV-J3 (ADR 0615 §1): one reads entry on the assigner's state counts 1 for the assigner and 0 for the indexer.
hold("reader", `{"module":"reader","version":"1.0","state":["x"],"reads":["mesh-delivery-assigner.wants"]}`)
if _, err := ix.JudgeAll(ctx, nil); err != nil {
t.Fatal(err)
}
if value, _, _, _ := ix.OldSections.Get(ctx, OldSectionsKey); true {
c, _ := DecodeOldSections(value)
if c.Reads["mesh-delivery-assigner"] != 1 || c.Reads["mesh-delivery-assigner.wants"] != 1 || c.Reads["mesh-delivery-indexer"] != 0 {
t.Fatalf("the reads on the owners' state count %v", c.Reads)
}
}
// DLV-J1 (ADR 0619 §1, §3): a foundation part asking bus state in the new form is refused; the nine and the pipeline runner are the foundation.
// Design 47 *A module's registration* (b229f5c4 lines 480-481): a template in the manifest of every part on the list is
// refused, the runner's included.
for _, part := range []string{"mesh-config", "mesh-delivery-pipeline"} {
d := hold(part, `{"module":"`+part+`","version":"1.0","requires":[{"name":"cache","storage":"key-value","template":"state"}]}`)
if _, err := ix.JudgeAll(ctx, map[string]bool{d: true}); err != nil {
t.Fatal(err)
}
if v := verdict(d); !meshcatalogclient.VerdictRefuses(v) || !strings.Contains(strings.Join(v.Problems, " "), "part of the foundation") {
t.Fatalf("%s asking new-form bus state: %+v", part, v)
}
}
if !slices.Equal(judge.Foundation, []string{"nats", "mesh-grants", "mesh-delivery-indexer", "mesh-delivery-assigner", "node-tools",
"mesh-delivery-builder", "mesh-delivery-checker", "mesh-config", "mesh-vault", "mesh-delivery-pipeline", "gitea", "distribution", "mesh-host"}) {
t.Fatalf("the foundation is %v, want exactly the nine of ADR 0619, the pipeline runner (ADR 0666 §4) and the forge and image registry modules (design 47 G §584) and the node-engine's module (§627)", judge.Foundation)
}
late := hold("late", `{"module":"late","version":"1.0","logs":[{"name":"l"}]}`)
grants := hold("mesh-grants", `{"module":"mesh-grants","version":"2.0","logs":[{"name":"l"},{"name":"m"}]}`)
if _, err := ix.JudgeAll(ctx, map[string]bool{late: true, grants: true}); err != nil {
t.Fatal(err)
}
if v := verdict(late); !meshcatalogclient.VerdictRefuses(v) || !strings.Contains(v.Problems[0], "late uses the old section logs, which no other module uses any more") {
t.Fatalf("late: %+v", v)
}
if v := verdict(grants); meshcatalogclient.VerdictRefuses(v) {
t.Fatalf("the foundation's manifest was refused: %+v", v)
}
// 1966/AC11: the sole user of an old section changes its manifest and keeps the section: accepted, since it was accepted
// with it (the count is the accepted set held from the last pass, not recounted from fresh digests).
hold("solo", `{"module":"solo","version":"1.0","objects":[{"name":"o","max-mib":1}]}`)
if _, err := ix.JudgeAll(ctx, nil); err != nil {
t.Fatal(err)
}
solo2 := hold("solo", `{"module":"solo","version":"2.0","objects":[{"name":"o","max-mib":2}]}`)
if _, err := ix.JudgeAll(ctx, map[string]bool{solo2: true}); err != nil {
t.Fatal(err)
}
if v := verdict(solo2); meshcatalogclient.VerdictRefuses(v) {
t.Fatalf("the sole user of objects, changing its manifest and keeping objects, was refused: %+v", v)
}
// A count that cannot be read is never taken as 0: the zero rule is not applied in that pass and the log says so.
var logged []string
ix.Logf = func(f string, a ...any) { logged = append(logged, fmt.Sprintf(f, a...)) }
held := ix.OldSections
ix.OldSections = unreadable{held}
retry := hold("late", `{"module":"late","version":"2.0","logs":[{"name":"l"}]}`)
if _, err := ix.JudgeAll(ctx, map[string]bool{retry: true}); err != nil {
t.Fatal(err)
}
if v := verdict(retry); meshcatalogclient.VerdictRefuses(v) || !strings.Contains(strings.Join(logged, "\n"), "the zero rule is not applied") {
t.Fatalf("a count that could not be read: %+v, logged %q", v, logged)
}
ix.OldSections = held
// Clause 3 of the record on a release's Approve: a manifest with a field this judge does not know waits (no refusal, no
// event) while a newer build of the indexer is on the trunk, and is judged again at the next judge; once this program's
// build is the trunk's newest it is refused; a wait is written into the module's value as a refusal naming the fields.
ownFP := "src2:" + strings.Repeat("e", 64)
hold(testSelf, `{"module":"mesh-delivery-indexer","version":"1.0"}`)
ix.ownFP = ownFP // this program started when the trunk held another build of the indexer than the one it holds now
later := hold("later", `{"module":"later","version":"1.0","a-field-from-later":1,"another":{"x":1}}`)
said = nil
before := len(said)
j, err = ix.JudgeAll(ctx, map[string]bool{later: true})
if err != nil || len(j.Waiting) != 1 || j.Waiting[0] != "later" || len(said) != before {
t.Fatalf("a manifest with fields from later: %+v %v, said %d", j, err, len(said))
}
for _, a := range accepted {
if a.Module == "later" {
t.Fatalf("a manifest set aside to wait was said accepted: %+v (DLV-I9)", a)
}
}
if v := verdict(later); !meshcatalogclient.VerdictWaits(v) || meshcatalogclient.VerdictRefuses(v) || strings.Join(v.Unknown, ",") != "a-field-from-later,another" {
t.Fatalf("later: %+v", v)
}
if m := module("later"); m.Verdict != meshcatalogclient.VerdictRefused || len(m.Unknown) != 2 || !meshcatalogclient.VerdictWaits(*meshcatalogclient.ModuleVerdict("later", m)) ||
!strings.Contains(m.Problems[0], "does not know yet") {
t.Fatalf("modules.later carries %+v", m)
}
// The build this program runs is asked of the engine, and the trunk moving before the start pass changes nothing: the
// trunk's value of the indexer names a newer build than the one the engine says runs, so the field still waits.
ix.ownFP = ""
ix.OwnBuild = func(context.Context) (string, error) { return ownFP, nil }
if _, err := ix.JudgeAtStart(ctx); err != nil || !meshcatalogclient.VerdictWaits(verdict(later)) {
t.Fatalf("the trunk moved before the start pass: %v %+v", err, verdict(later))
}
ix.OwnBuild = func(context.Context) (string, error) { return "", errors.New("the engine does not answer") }
ix.ownFP = ""
if _, err := ix.JudgeAtStart(ctx); err != nil || !meshcatalogclient.VerdictWaits(verdict(later)) {
t.Fatalf("an unknown build refused: %v %+v", err, verdict(later))
}
ix.ownFP = ownFP
// A renamed field is a known mistake: refused at once.
renamed := hold("renamed", `{"module":"renamed","version":"1.0","needs":{"x":"y"}}`)
if j, err := ix.JudgeAll(ctx, map[string]bool{renamed: true}); err != nil || len(j.Waiting) != 1 || !meshcatalogclient.VerdictRefuses(verdict(renamed)) {
t.Fatalf("a renamed field: %+v %v", j, err)
}
// The trunk's newest build of the indexer is this program's: the field is one no judge to come knows.
ix.ownFP = "src2:" + strings.Repeat("a", 64)
if j, err := ix.JudgeAll(ctx, nil); err != nil || len(j.Waiting) != 0 || !meshcatalogclient.VerdictRefuses(verdict(later)) || !strings.Contains(verdict(later).Problems[0], "a-field-from-later") {
t.Fatalf("the newest build refuses: %+v %v %+v", j, err, verdict(later))
}
if m := module("later"); m.Verdict != meshcatalogclient.VerdictRefused || meshcatalogclient.VerdictWaits(*meshcatalogclient.ModuleVerdict("later", m)) || len(m.Unknown) != 0 {
t.Fatalf("modules.later after the refusal carries %+v", m)
}
// The tools: indexer_verdicts lists the verdicts that wait or are refused (and every one with all), indexer_judge
// judges now and says how the module's verdict stands.
keys, _ := ix.Verdicts.Keys(ctx)
get := func(key string) ([]byte, error) { v, _, _, err := ix.Verdicts.Get(ctx, key); return v, err }
if rows, unread := ListVerdicts(keys, get, "later", false); len(rows) != 1 || rows[0].State != meshcatalogclient.VerdictRefused || len(unread) != 0 {
t.Fatalf("indexer_verdicts of later: %+v %v", rows, unread)
}
if rows, _ := ListVerdicts(keys, get, "", true); len(rows) <= len(keys)-3 || rows[0].Module > rows[len(rows)-1].Module {
t.Fatalf("indexer_verdicts, all: %d rows of %d verdicts", len(rows), len(keys))
}
// The build is fixed once, at the start, and never learnt later: this program started when the engine could not answer, and
// the engine now names the trunk's newest build (as it does while it replaces the indexer by a successor). indexer_judge
// does not take it for this program's own, so the field still waits and the build stays unknown.
ix.ownFP, ix.ownFixed = "", true
ix.OwnBuild = func(context.Context) (string, error) { return "src2:" + strings.Repeat("a", 64), nil }
if got := ix.JudgeAgain(ctx, "later", ""); !got.Stored || !strings.Contains(got.Answer, "waits") || ix.ownFP != "" {
t.Fatalf("indexer_judge learnt the build the engine names now: %+v (own %q)", got, ix.ownFP)
}
if _, err := ix.JudgeAtStart(ctx); err != nil || ix.ownFP != "" || !meshcatalogclient.VerdictWaits(verdict(later)) {
t.Fatalf("a judge pass learnt the build: %v %q %+v", err, ix.ownFP, verdict(later))
}
ix.ownFP = ownFP
ix.OwnBuild = nil
if got := ix.JudgeAgain(ctx, "later", ""); !got.Stored || !strings.Contains(got.Answer, "waits") || !strings.Contains(got.Answer, "a-field-from-later") {
t.Fatalf("indexer_judge: %+v", got)
}
if rows, _ := ListVerdicts(keys, get, "later", false); len(rows) != 1 {
t.Fatalf("indexer_verdicts of later after: %+v", rows)
}
if got := ix.JudgeAgain(ctx, "nobody", ""); got.Stored || !strings.Contains(got.Answer, "holds no manifest") {
t.Fatalf("indexer_judge of nobody: %+v", got)
}
if key, value, err := NewJudgeAgain("later", "", "why", "me", time.Now()); err != nil || key != "judge.later" {
t.Fatalf("the ask: %q %v", key, err)
} else if a, err := DecodeTakeIn(value); err != nil || !a.Judge || a.Module != "later" {
t.Fatalf("the ask's value: %+v %v", a, err)
}
// A refusal an older judge made never stands: a digest the trunk no longer names (a candidate's, or one back on the
// trunk later) refused by judge-0 for words it did not know is judged again by this judge at the start, at a pull that
// carries it, and by indexer_judge with its digest; one this judge refused itself is left as it is.
stale := func(module, raw string) string {
digest, err := ix.keepManifest(ctx, module, raw)
if err != nil {
t.Fatal(err)
}
if _, _, err := ix.writeVerdict(ctx, digest, meshcatalog.VerdictsByDigestValue{Module: module, State: meshcatalogclient.VerdictRefused,
Problems: []string{module + " says tool-runner, which this judge does not know"}, Judge: "judge-0", At: ptr(time.Now())}); err != nil {
t.Fatal(err)
}
return digest
}
atStart := stale("tools", `{"module":"tools","version":"7.0"}`)
ownRefusal := stale("client", `{"module":"client","version":"7.0"}`)
if _, _, err := ix.writeVerdict(ctx, ownRefusal, meshcatalog.VerdictsByDigestValue{Module: "client", State: meshcatalogclient.VerdictRefused,
Problems: []string{"x"}, Judge: ix.judgeID(), At: ptr(time.Now())}); err != nil {
t.Fatal(err)
}
if _, err := ix.JudgeAtStart(ctx); err != nil || meshcatalogclient.VerdictRefuses(verdict(atStart)) || verdict(atStart).Judge != ix.judgeID() || !meshcatalogclient.VerdictRefuses(verdict(ownRefusal)) {
t.Fatalf("the start pass: %+v, this judge's own refusal %+v (%v)", verdict(atStart), verdict(ownRefusal), err)
}
byPull := stale("tools", `{"module":"tools","version":"8.0"}`)
pull := forgemsg.Forge{Kind: forgemsg.ForgePull, Repository: "novox/mesh-tools", Number: 3, Branch: "main",
Manifests: map[string]string{"modules/tools": `{"module":"tools","version":"8.0"}`}}
if _, err := ix.ApplyMessage(ctx, pull); err != nil || meshcatalogclient.VerdictRefuses(verdict(byPull)) || verdict(byPull).Judge != ix.judgeID() {
t.Fatalf("a pull carrying a digest an older judge refused: %+v %v", verdict(byPull), err)
}
byTool := stale("tools", `{"module":"tools","version":"9.0"}`)
if got := ix.JudgeAgain(ctx, "client", byTool); got.Stored || !strings.Contains(got.Answer, "tools's, not client's") || !meshcatalogclient.VerdictRefuses(verdict(byTool)) {
t.Fatalf("indexer_judge with another module's digest: %+v", got)
}
if got := ix.JudgeAgain(ctx, "tools", byTool); !got.Stored || !strings.Contains(got.Answer, "accepted") || meshcatalogclient.VerdictRefuses(verdict(byTool)) {
t.Fatalf("indexer_judge with a digest: %+v %+v", got, verdict(byTool))
}
if key, value, err := NewJudgeAgain("tools", byTool, "why", "me", time.Now()); err != nil || key != "judge.tools" {
t.Fatalf("the ask with a digest: %q %v", key, err)
} else if a, err := DecodeTakeIn(value); err != nil || a.Digest != byTool {
t.Fatalf("the ask's digest: %+v %v", a, err)
}
// Without the verdicts state the indexer judges nothing.
ix.Verdicts = nil
if j, err := ix.JudgeAll(ctx, nil); err != nil || j.Judged != 0 {
t.Fatalf("without the state: %+v %v", j, err)
}
}
// unreadable is a store whose reads fail.
type unreadable struct{ Store }
func (unreadable) Get(context.Context, string) ([]byte, uint64, bool, error) {
return nil, 0, false, errors.New("the bus did not answer")
}
// Design 47 G §491, §676: two modules whose entries carry one Go module path are both refused by the trunk's judge pass,
// each naming both; a module with its own name is accepted.
func TestAPackageNameBelongsToOneModule(t *testing.T) {
ctx := context.Background()
ix := &Indexer{Module: testSelf, Modules: newMem(), Manifests: newMem(), Verdicts: newMem(), JudgeID: "judge-1",
Now: func() time.Time { return time.Date(2026, 11, 1, 3, 0, 0, 0, time.UTC) }}
ix.ReadFile = func(_ context.Context, _, _, path string) ([]byte, error) {
if strings.HasSuffix(path, "go.mod") {
name := "git.example/own/" + strings.Split(path, "/")[0]
if strings.HasPrefix(path, "one/") || strings.HasPrefix(path, "two/") {
name = "git.example/shared"
}
return []byte("module " + name + "\n"), nil
}
return nil, nil
}
hold := func(module string) string {
raw := `{"module":"` + module + `","version":"1.0","build":{"artifacts":[{"name":"go","kind":"package","language":"go","paths":["go"],"series":"0.1","generated-from":"."}]}}`
digest, err := ix.keepManifest(ctx, module, raw)
if err != nil {
t.Fatal(err)
}
value, err := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: "src2:" + strings.Repeat("a", 64), Manifest: digest,
Repository: "novox/r", Path: module, Commit: "c"})
if err != nil {
t.Fatal(err)
}
ix.Modules.(*memStore).put(module, value)
return digest
}
digests := map[string]bool{hold("one"): true, hold("two"): true, hold("three"): true}
j, err := ix.JudgeAll(ctx, digests)
if err != nil {
t.Fatal(err)
}
if len(j.Refused) != 2 || j.Refused["one"] == nil || j.Refused["two"] == nil || j.Refused["three"] != nil {
t.Fatalf("refused %v, want one and two", j.Refused)
}
for _, m := range []string{"one", "two"} {
if p := strings.Join(j.Refused[m], " "); !strings.Contains(p, "one and two") || !strings.Contains(p, "git.example/shared") {
t.Errorf("%s's problems %q do not name both modules and the package", m, p)
}
}
}
@@ -17,7 +17,6 @@ import (
var FirstViewTools = []ViewedTool{
{Tool: "manifests", State: "manifests", Words: "every manifest the indexer keeps, by its digest."},
{Tool: "modules", State: ModulesState, Words: "every module the indexer holds, its fingerprint, inputs and manifest digest."},
{Tool: "verdicts", State: VerdictsState, Words: "the judge's verdict on each manifest: accepted, waiting or refused."},
{Tool: "graph", State: graph.State, Words: "the module graph: each module's build, provision and verb edges, whose verb edges a want waits on."},
}
@@ -33,7 +32,6 @@ var ReadTools = []ViewedTool{
{Tool: "indexer_read_takeins", State: TakeInsState, Words: "each take-in asked of the indexer: its ask, whether and when the indexer answered it and its answer (design 47 G A-464a)."},
{Tool: "indexer_read_findings", State: FindingsState, Words: "the indexer's findings, which the condition keeper raises and clears."},
{Tool: "indexer_read_seat_definitions", State: SeatDefinitionsState, Words: "the seat definitions the trunk's and each open pull request's contracts modules hold, resolved, or why one does not parse."},
{Tool: "indexer_read_unjudged", State: UnjudgedState, Words: "the manifests a take-in newly wrote that no judge pass judged yet, by digest, each with its module and merge; indexer_judge judges them."},
}
// ViewedTool is one tool over one state of the indexer's.
@@ -43,8 +41,8 @@ type ViewedTool struct {
// ViewedStates are the states the indexer's views family says: the ones its readers read.
var ViewedStates = map[string]bool{
ManifestsState: true, VerdictsState: true, PredefinedState: true, CandidatesState: true,
PackagesState: true, HeardState: true, TakenState: true, TakeInsState: true, FindingsState: true, SeatDefinitionsState: true, UnjudgedState: true,
ManifestsState: true, PredefinedState: true, CandidatesState: true,
PackagesState: true, HeardState: true, TakenState: true, TakeInsState: true, FindingsState: true, SeatDefinitionsState: true,
}
// ModulesViewsEvent is what the indexer says on its views family of modules when a new revision of the trunk branch
+19 -17
View File
@@ -115,11 +115,16 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
if m.Module != "mesh-delivery-indexer" {
t.Errorf("module = %q", m.Module)
}
// 1126: assignments is the seat mesh-module-assigner's (ADR 0567): the indexer holds no assignments state. ADR 0578 §1: it says
// a refused manifest as manifest-refused, and (issue 1966 D3, 1966/AC10) a newly accepted one asking for bus state as
// manifest-accepted, both events of the seat mesh-catalog it holds (ADR 0636 §4), so not among its `emits`.
// 1126: assignments is the seat mesh-module-assigner's (ADR 0567): the indexer holds no assignments state. I7/AC1 (design 47
// H §745, §764): it judges no manifest at the take-in, so it keeps no verdicts and says no manifest-refused or
// manifest-accepted; its own events are the take-in's two.
if own := slices.DeleteFunc(slices.Clone(m.Emits), func(e string) bool { return strings.HasPrefix(e, views.EventPrefix) }); !reflect.DeepEqual(own, []string{takein.TakeInPendingEvent, takein.TakeInTakenEvent}) {
t.Errorf("emits = %v, want the two take-in events (the verdict events are the seat mesh-catalog's)", m.Emits)
t.Errorf("emits = %v, want the two take-in events", m.Emits)
}
for _, gone := range []string{"verdicts", "unjudged", "indexer_verdicts", "indexer_judge", "indexer_say_accepted", "indexer_read_unjudged", "views.verdicts"} {
if slices.Contains(m.Tools, gone) || slices.Contains(m.Emits, gone) || slices.ContainsFunc(m.State, func(s state) bool { return s.Name == gone }) {
t.Errorf("the manifest still declares %s (design 47 H §764: the validation at the take-in goes)", gone)
}
}
// Design 47 G §700, A-74: the router's record of an ask is read through the seat operator-channel, which the indexer uses,
// and its holder is asked of the seat mesh-module-assigner.
@@ -127,10 +132,10 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
t.Errorf("uses = %v, invokes = %v, want the seat operator-channel used and seat:mesh-module-assigner.assignments invoked", m.Uses, m.Invokes)
}
// 1126 note 76: heard keeps the last 64 FORGE messages the indexer handled, for indexer_heard.
if want := []state{{Name: "candidates", History: 10}, {Name: "heard", History: 64}, {Name: "manifests", History: 1}, {Name: "predefined", History: 5}, {Name: "taken", History: 1}, {Name: "takeins", History: 1}, {Name: "verdicts", History: 5}, {Name: "oldsections", History: 1}, {Name: index.RunsState, History: 1}, {Name: "own-build", History: 1}, {Name: index.PackagesState, History: index.PackagesHistory},
if want := []state{{Name: "candidates", History: 10}, {Name: "heard", History: 64}, {Name: "manifests", History: 1}, {Name: "predefined", History: 5}, {Name: "taken", History: 1}, {Name: "takeins", History: 1}, {Name: "oldsections", History: 1}, {Name: index.RunsState, History: 1}, {Name: "own-build", History: 1}, {Name: index.PackagesState, History: index.PackagesHistory},
{Name: views.MarksState, History: 1}, {Name: index.FindingsState, History: 5}, {Name: views.StatusState, History: 1},
{Name: index.RequirementsState, History: 5}, {Name: index.SeatDefinitionsState, History: 5},
{Name: index.PendingState, History: 1}, {Name: index.UnjudgedState, History: 1}, {Name: index.EntryIndexState, History: 1, MaxMiB: 8}, {Name: index.NamedState, History: 1, MaxMiB: 4}, // design 47 G §695
{Name: index.PendingState, History: 1}, {Name: index.EntryIndexState, History: 1, MaxMiB: 8}, {Name: index.NamedState, History: 1, MaxMiB: 4}, // design 47 G §695
{Name: index.RegistrationsState, History: 1}, {Name: index.ProposalsState, History: 1, MaxMiB: 4},
{Name: index.VersionsState, History: 1, MaxMiB: 64}, {Name: index.BuildEdgesState, History: 1}, {Name: index.InputsState, History: 1}, {Name: index.PinsState, History: 1, MaxMiB: index.PinsCapMiB}, {Name: process.ConsumersState, History: 1}}; !reflect.DeepEqual(m.State, want) { // DLV-I7, DLV-I17
t.Errorf("state = %+v, want %+v", m.State, want)
@@ -146,8 +151,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
// name and states with their revisions, and the states it reads, each as `<holder>.<state>`.
// A-296: its own buckets are named from its credential's module (index.Bucket), each a state its manifest keeps.
own := func(state string) string { return index.Bucket("a-indexer", state) }
for bucket, history := range map[string]int{own(index.CandidatesState): index.CandidatesHistory, own(index.ManifestsState): index.ManifestsHistory,
own(index.VerdictsState): index.VerdictsHistory, own(index.PackagesState): index.PackagesHistory, own(index.TakenState): 1, own(index.PredefinedState): 5} {
for bucket, history := range map[string]int{own(index.CandidatesState): index.CandidatesHistory, own(index.ManifestsState): index.ManifestsHistory, own(index.PackagesState): index.PackagesHistory, own(index.TakenState): 1, own(index.PredefinedState): 5} {
if !slices.ContainsFunc(m.State, func(st state) bool { return "a-indexer_"+st.Name == bucket && st.History == history }) {
t.Errorf("the indexer names its bucket %s with %d revisions, its manifest's states are %+v", bucket, history, m.State)
}
@@ -166,7 +170,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
// internal/broker/state.go:62-93).
// The registry seats' published fill each (§413: no consume of version.numbered) package's series (DLV-I5, design 51 *The versions*).
if want := []string{"go-package-registry.published", "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "mesh-builds.package-published", "mesh-builds.toolchain-agreed", "mesh-catalog.catalog-revision", "mesh-conditions.findings-asked", "mesh-delivery-checker.verdict", "mesh-delivery-forge-bridge.forge.**",
"mesh-delivery.freed", "mesh-delivery.heads-replaced", "mesh-delivery.pinned", "mesh-delivery.pins-stated", "mesh-delivery.source-moved", "mesh-pipeline-runner.pipeline-ended", "mesh-pipeline-runner.run-started", "mesh-wants.version-delivered", "mesh-wants.wait-ended", "mesh-wants.wanted", "node-build-go.step-ended", "node-engine.running", "npm-package-registry.published", "operator-channel.decided", "seat:mesh-module-assigner.views.assignments", "seat:mesh-module-assigner.views.seat-holders"}; !reflect.DeepEqual(sorted(m.Consumes), want) {
"mesh-delivery.source-moved", "mesh-pipeline-runner.pipeline-ended", "mesh-pipeline-runner.pipeline-started", "mesh-pipeline-runner.run-started", "mesh-wants.version-delivered", "mesh-wants.wait-ended", "mesh-wants.wanted", "node-build-go.step-ended", "node-engine.running", "npm-package-registry.published", "operator-channel.decided", "seat:mesh-module-assigner.views.assignments", "seat:mesh-module-assigner.views.seat-holders"}; !reflect.DeepEqual(sorted(m.Consumes), want) {
t.Errorf("consumes = %v, want %v", m.Consumes, want)
}
if b, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 || b != "${dir:state}/broker" {
@@ -182,13 +186,11 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
"seat:node-build-node.steps", "seat:node-build-image.steps", "mesh-delivery-builder.builder_versions"}; !reflect.DeepEqual(m.Invokes, want) {
t.Errorf("invokes = %v, want %v", m.Invokes, want)
}
// 1945: indexer_take_in, served by its tools bundle through the takeins state; the process serves none. The same bundle
// reads the verdicts state (indexer_verdicts) and asks the indexer to judge again through the takeins state
// (indexer_judge; issue 774, clause 3). 1966/AC11: indexer_old_sections reads the count of the old sections (state oldsections).
// ADR 0621 §1, ADR 0622 §1, ADR 0631 §1, ADR 0663 §1: the first-view tools manifests, modules, verdicts and graph (modules
// 1945: indexer_take_in, served by its tools bundle through the takeins state; the process serves none. 1966/AC11: indexer_old_sections reads the count of the old sections (state oldsections).
// ADR 0621 §1, ADR 0622 §1, ADR 0631 §1, ADR 0663 §1: the first-view tools manifests, modules and graph (modules
// kept for the grants part until it reads the catalog half, design 47 G A-354), and design 25 § 12.2's read
// tools, all of the tools bundle, which runs in a runtime of its own on the indexer's account (runs-as).
want := []string{"indexer_take_in", "indexer_old_sections", "indexer_verdicts", "indexer_judge"}
want := []string{"indexer_take_in", "indexer_old_sections"}
for _, v := range append(append([]index.ViewedTool{}, index.FirstViewTools...), index.ReadTools...) {
want = append(want, v.Tool)
// Milestone 47/G: modules and graph are the newest catalog graph revision's, which catalog-revision says the changes of.
@@ -196,14 +198,14 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
t.Errorf("%s answers the state %s, which the views family does not say", v.Tool, v.State)
}
}
want = append(want, "indexer_first_views", "indexer_first_view_again", index.ConfigReadersTool, index.BusStateTool, index.RequirementsTool, index.PackageUsersTool, index.SayAcceptedTool, // DLV-F3r, DLV-I10, DLV-I11, DLV-I7, DLV-I8, DLV-I9c
want = append(want, "indexer_first_views", "indexer_first_view_again", index.ConfigReadersTool, index.BusStateTool, index.RequirementsTool, index.PackageUsersTool, // DLV-F3r, DLV-I10, DLV-I11, DLV-I7, DLV-I8, DLV-I9c
index.SeatDefinitionTool, index.SeatDefinitionsAgainTool, index.OutOfStepTool, // DLV-I19
index.PendingTool, index.TakeInPendingTool, index.DropPendingTool, index.TakeInMergeTool, index.FingerprintsAgainTool, // issue 1966; 1585/T12, T13 (ADR 0633 §5)
// milestone 47/G W4.12 (design 47, the indexer's tools): the catalog graph revisions read and steered.
index.CatalogRevisionTool, index.CatalogRevisionsTool, index.CandidateGraphsTool, index.RecomputeTool, index.DropCandidateGraphTool,
index.EntryIndexTool, index.NamedDropTool, // design 47 G §675
index.ProposalsTool, index.ProposalDropTool, index.SeedAgainTool, index.InputsTool, index.PinsTool, index.UnpinTool, index.ConsumerTool, index.RunsTool, index.RunDropTool) // design 47 G §667, §8, §579, §64 (A-194)
if !reflect.DeepEqual(m.Tools, want) || len(want) != 50 || !slices.Contains(m.Tools, "modules") {
if !reflect.DeepEqual(m.Tools, want) || len(want) != 45 || !slices.Contains(m.Tools, "modules") {
t.Errorf("tools = %v, want %v", m.Tools, want)
}
// Design 47 G §2, §9: every version of every registration is kept with no age limit, 64 MiB, on the old streams section.
@@ -284,7 +286,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
// ADR 0636 §4: seat-definition-changed and -removed are the seat mesh-catalog's events, said on the seat's subject and kept
// on the shared events stream, and not among its own emits.
// Design 47 G A-360: modules too, at each new trunk revision that changes it.
for _, state := range []string{"candidates", "findings", "graph", "heard", "manifests", "modules", "packages", "predefined", "seat-definitions", "taken", "verdicts"} {
for _, state := range []string{"candidates", "findings", "graph", "heard", "manifests", "modules", "packages", "predefined", "seat-definitions", "taken"} {
if !slices.Contains(m.Emits, views.EventPrefix+state) {
t.Errorf("emits = %v: want %s%s", m.Emits, views.EventPrefix, state)
}
+2 -13
View File
@@ -29,11 +29,8 @@
"tools": [
"indexer_take_in",
"indexer_old_sections",
"indexer_verdicts",
"indexer_judge",
"manifests",
"modules",
"verdicts",
"graph",
"indexer_read_manifests",
"indexer_read_predefined",
@@ -45,14 +42,12 @@
"indexer_read_takeins",
"indexer_read_findings",
"indexer_read_seat_definitions",
"indexer_read_unjudged",
"indexer_first_views",
"indexer_first_view_again",
"indexer_config_readers",
"indexer_bus_state",
"indexer_requirements",
"indexer_package_users",
"indexer_say_accepted",
"indexer_seat_definition",
"indexer_seat_definitions_again",
"indexer_out_of_step",
@@ -94,12 +89,9 @@
"mesh-delivery-forge-bridge.forge.**",
"mesh-catalog.catalog-revision",
"mesh-delivery.source-moved",
"mesh-delivery.pinned",
"mesh-delivery.freed",
"mesh-delivery.heads-replaced",
"mesh-delivery.pins-stated",
"mesh-wants.wanted",
"mesh-pipeline-runner.run-started",
"mesh-pipeline-runner.pipeline-started",
"mesh-pipeline-runner.pipeline-ended",
"node-engine.running",
"operator-channel.decided",
@@ -112,7 +104,6 @@
{ "name": "predefined", "history": 5 },
{ "name": "taken", "history": 1 },
{ "name": "takeins", "history": 1 },
{ "name": "verdicts", "history": 5 },
{ "name": "oldsections", "history": 1 },
{ "name": "runs", "history": 1 },
{ "name": "own-build", "history": 1 },
@@ -123,7 +114,6 @@
{ "name": "requirements", "history": 5 },
{ "name": "seat-definitions", "history": 5 },
{ "name": "pending", "history": 1 },
{ "name": "unjudged", "history": 1 },
{ "name": "entry-index", "history": 1, "max-mib": 8 },
{ "name": "named", "history": 1, "max-mib": 4 },
{ "name": "registrations", "history": 1 },
@@ -153,8 +143,7 @@
"views.packages",
"views.predefined",
"views.seat-definitions",
"views.taken",
"views.verdicts"
"views.taken"
],
"reads": [
"mesh-build-go.toolchains",