From 7372447c6889d52c56c55e22255f8a31596348e9 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 4 Nov 2026 13:58:02 +0100 Subject: [PATCH] DLV-I9, I9b, I9c: manifest-accepted for every accepted manifest and a module leaving the trunk, never lost silently MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The bus module's orphans need to hear a module that asks nothing now (design 25 § 12.2): every newly accepted manifest is said, its entries an empty list when it asks for none, and a module whose directory leaves the trunk gets an empty one; a manifest set aside to wait gets none. A failed publish is published once more; one failing twice writes the finding accepted-unsaid naming the module, said again at the next judging pass and cleared once said. indexer_say_accepted says it again by hand from the held verdict, a hand act through mesh-hand-acts; it refuses a module with no accepted verdict. Claude-Session: https://claude.ai/code/session_01JG73KJNrH5EgFXMKKYSrig --- .../cmd/mesh-delivery-indexer-tools/main.go | 29 ++- .../cmd/mesh-delivery-indexer/main.go | 6 +- .../cmd/mesh-delivery-indexer/takein.go | 3 + .../cmd/mesh-delivery-indexer/verdicts.go | 13 +- .../internal/index/accepted.go | 211 ++++++++++++++++++ .../internal/index/candidates.go | 10 +- .../internal/index/modules.go | 3 + .../internal/index/remove_test.go | 21 ++ .../internal/index/takein.go | 4 +- .../internal/index/verdicts.go | 15 +- .../internal/index/verdicts_test.go | 66 +++++- .../mesh-delivery-indexer/manifest_test.go | 6 +- modules/mesh-delivery-indexer/module.json | 6 +- 13 files changed, 362 insertions(+), 31 deletions(-) create mode 100644 modules/mesh-delivery-indexer/internal/index/accepted.go diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go index 47b8b781..ec0bf378 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go @@ -15,6 +15,7 @@ import ( "time" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/handacts" "mesh-delivery-indexer/internal/index" ) @@ -23,7 +24,7 @@ import ( type asker func(key string, value []byte) (index.TakeInAsk, error) // Tools are the indexer's tools. read is the indexer's state of verdicts, which indexer_verdicts reads; counts the old sections' count. -func Tools(ask asker, now func() time.Time, read stdio.KeptState, counts func() (index.OldSectionsCount, error)) []stdio.Tool { +func Tools(ask asker, now func() time.Time, read stdio.KeptState, counts func() (index.OldSectionsCount, error), acts handacts.Recorder) []stdio.Tool { str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) } return []stdio.Tool{ {Name: index.OldSectionsTool, @@ -102,6 +103,30 @@ func Tools(ask asker, now func() time.Time, read stdio.KeptState, counts func() }, str(args, "module"), all) return map[string]any{"verdicts": rows, "unread": unread}, nil }}, + {Name: index.SayAcceptedTool, + Description: "Say the event manifest-accepted again now for one module, from the accepted verdict on its trunk's manifest the " + + "indexer holds (design 25 § 12.2): for a verdict the bus module is thought to have missed. A hand act, recorded through " + + "mesh-hand-acts before it is asked. Refused for a module the indexer holds no accepted verdict of (refused, set aside " + + "to wait, or not held). Answers what was said: the digest, the verdict's revision and the bus-state entries' count. A " + + "publish that failed twice is also said again by itself at the indexer's next judging pass (its finding accepted-unsaid).", + Input: map[string]any{"type": "object", "required": []string{"module", "why"}, "properties": map[string]any{ + "module": map[string]any{"type": "string", "description": "the module's name, as the catalogue registers it"}, + "why": map[string]any{"type": "string", "description": "why it is said again now, in plain words"}}}, + Run: func(args map[string]any) (any, error) { + key, value, err := index.NewSayAccepted(str(args, "module"), str(args, "why"), stdio.CallerOf(args), now()) + if err != nil { + return nil, err + } + if _, err := acts.Record(handacts.Request{What: "said manifest-accepted of " + str(args, "module") + " again by hand", + Why: str(args, "why"), Cause: index.SayAcceptedTool, Verb: "say-accepted"}); err != nil { + return nil, fmt.Errorf("the hand act was not recorded through mesh-hand-acts, so nothing was said again: %w", err) + } + a, err := ask(key, value) + if err != nil { + return nil, err + } + return a.Result, nil + }}, {Name: index.JudgeAgainTool, Description: "Judge the manifests again now, as the indexer does at its start and after each merge, and answer how the module's " + "verdict stands (waits, refused or accepted, and why). For a manifest that waits for a judge that knows its fields, or " + @@ -194,7 +219,7 @@ func heldCounts(s stdio.KeptState) func() (index.OldSectionsCount, error) { } func main() { - if err := stdio.Serve("", append(Tools(askTakeIn(stdio.State(index.TakeInsState), 25*time.Second), time.Now, stdio.State(index.VerdictsState), heldCounts(stdio.State(index.OldSectionsState))), append(append(catalogTools(), viewTools(os.Getenv, keptSource, time.Now)...), configReadersTool(stdio.NewWatchedStates()), busStateTool(keptSource), requirementsTool(keptSource), packageUsersTool(keptSource))...)); err != nil { + if err := stdio.Serve("", append(Tools(askTakeIn(stdio.State(index.TakeInsState), 25*time.Second), time.Now, stdio.State(index.VerdictsState), heldCounts(stdio.State(index.OldSectionsState)), handacts.Recorder{Ask: stdio.Ask}), append(append(catalogTools(), viewTools(os.Getenv, keptSource, time.Now)...), configReadersTool(stdio.NewWatchedStates()), busStateTool(keptSource), requirementsTool(keptSource), packageUsersTool(keptSource))...)); err != nil { fmt.Fprintf(os.Stderr, "[mesh-delivery-indexer-tools] %v\n", err) os.Exit(1) } diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index 788b6fa6..ea1d8dcd 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -330,8 +330,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro requirements := process.AwaitBucket(ctx, js, conn, index.RequirementsBucket, "what consumers require is not kept", logger.Printf) // The module graph (ADR 0603 §2): a state this delivery brings, bound when it appears; until then no graph is written. graph := process.AwaitBucket(ctx, js, conn, graph.Bucket, "no module graph is written", logger.Printf) - ix := &index.Indexer{OldSections: awaitedStore{oldSections}, Requirements: awaitedStore{requirements}, Store: kvStore{candidates}, Modules: kvStore{modules}, Manifests: kvStore{manifests}, Taken: awaitedStore{taken}, Builds: known, - Verdicts: awaitedStore{verdicts}, SayRefused: sayRefused(js, cred, logger.Printf), SayAccepted: sayAccepted(js, cred, logger.Printf), + findings := process.AwaitBucket(ctx, js, conn, findingsBucket, "no finding of the indexer is written", logger.Printf) + ix := &index.Indexer{Findings: awaitedStore{findings}, OldSections: awaitedStore{oldSections}, Requirements: awaitedStore{requirements}, Store: kvStore{candidates}, Modules: kvStore{modules}, Manifests: kvStore{manifests}, Taken: awaitedStore{taken}, Builds: known, + Verdicts: awaitedStore{verdicts}, SayRefused: sayRefused(js, cred, logger.Printf), SayAccepted: sayAccepted(js, cred), Predefined: awaitedStore{predefined}, ConfigCandidates: awaitedStore{configCandidates}, Logf: logger.Printf, Named: namedVia(calls.Call), Trunk: trunkVia(calls.Call), Contains: containsVia(calls.Call), Read: readVia(calls.Call), ReadOpen: openVia(calls.Call), Packages: awaitedStore{packages}, Files: filesVia(calls.Call), GoSum: goSumVia(calls.Call), GoMod: goModVia(calls.Call), @@ -381,7 +382,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // Only the toolchain reports of nodes the catalogue places the build module on count (issue 1940, ADR 0549 §3): the // placement is settled first, so the toolchains settle through the gate already knowing it. gate := buildrun.NewToolchainGate(sortedModules(), ix.PutToolchain, logger.Printf) - findings := process.AwaitBucket(ctx, js, conn, findingsBucket, "no finding of the indexer is written", logger.Printf) // The placement is the assigner's: read through its first view and its views family, never its bucket (DLV-F1r). placementErrs := make(chan error) followPlacement(ctx, js, cred, calls.Call, gate, func(ctx context.Context) { diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go index 7a98edb7..74bd16a9 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go @@ -16,6 +16,7 @@ type takerIn interface { TakeIn(ctx context.Context, module string) index.TakenIn TakeInRepository(ctx context.Context, repository string) index.TakenIn JudgeAgain(ctx context.Context, module, digest string) index.TakenIn + SayAcceptedAgain(ctx context.Context, module string) index.TakenIn } // takenIn answers one value of the takeins state (the tool indexer_take_in, issue 1945): a value not yet answered takes the @@ -37,6 +38,8 @@ func takenIn(ctx context.Context, ix takerIn, kv jetstream.KeyValue, e jetstream if a.Repository != "" { asked, r = "take-in of a repository", ix.TakeInRepository(ctx, a.Repository) a.Module = a.Repository + } else if a.Say { + asked, r = "manifest-accepted said again", ix.SayAcceptedAgain(ctx, a.Module) } else if a.Judge { asked, r = "judging again", ix.JudgeAgain(ctx, a.Module, a.Digest) } else { diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/verdicts.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/verdicts.go index 79e1b316..9ee2d7e6 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/verdicts.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/verdicts.go @@ -40,15 +40,13 @@ func sayRefused(js jetstream.JetStream, cred process.Account, logf func(string, // sayAccepted says the indexer's event manifest-accepted (issue 1966 D3, novox/hq ADR 0595 §4): // mesh.mod..event.manifest-accepted, carrying the module's bus-state entries whole, with the digest and judge as // the bus's dedup id (delivery.ManifestAccepted.EventID), so one acceptance is said once. -func sayAccepted(js jetstream.JetStream, cred process.Account, logf func(string, ...any)) func(context.Context, delivery.ManifestAccepted) { - return func(ctx context.Context, a delivery.ManifestAccepted) { +func sayAccepted(js jetstream.JetStream, cred process.Account) func(context.Context, delivery.ManifestAccepted, string) error { + return func(ctx context.Context, a delivery.ManifestAccepted, id string) error { data, err := json.Marshal(a) if err != nil { - logf("judge: %s for %s is not said: %v", delivery.ManifestAcceptedEvent, a.Module, err) - return + return err } msg := nats.NewMsg(cred.EventSubject(delivery.ManifestAcceptedEvent)) - id := a.EventID() msg.Header.Set("x-event-id", id) msg.Header.Set("x-source", cred.Module) msg.Header.Set("x-node", cred.Node) @@ -57,8 +55,7 @@ func sayAccepted(js jetstream.JetStream, cred process.Account, logf func(string, msg.Data = data pub, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) defer cancel() - if _, err := js.PublishMsg(pub, msg, jetstream.WithMsgID(id)); err != nil { - logf("judge: %s", fmt.Errorf("saying %s for %s: %w", delivery.ManifestAcceptedEvent, a.Module, err)) - } + _, err = js.PublishMsg(pub, msg, jetstream.WithMsgID(id)) + return err } } diff --git a/modules/mesh-delivery-indexer/internal/index/accepted.go b/modules/mesh-delivery-indexer/internal/index/accepted.go new file mode 100644 index 00000000..096a921e --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/accepted.go @@ -0,0 +1,211 @@ +package index + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "time" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/health" + + "mesh-delivery-indexer/judge" +) + +// manifest-accepted, never lost silently (design 25 § 12.2; DLV-I9, I9b, I9c). The indexer says it for every newly +// accepted manifest, its bus-state entries empty when it asks for none, and an empty one for a module leaving the trunk, +// so the bus module's orphans see it; a manifest the judge set aside to wait gets none. A publish that fails is published +// once more; one still failing writes the indexer's finding naming the module, said again at each judging pass after +// and cleared once said. indexer_say_accepted says it again for one module by hand, from the verdict it holds. +const ( + SayAcceptedTool = "indexer_say_accepted" + sayAcceptedPrefix = "say." + // UnsaidKind is the finding of a manifest-accepted whose publish failed twice. + UnsaidKind = "accepted-unsaid" +) + +// acceptedOf is the event body for the verdict v (its revision in the verdicts state) on the manifest raw of digest, its +// entries an empty list when it asks for none. +func acceptedOf(digest string, v delivery.ManifestVerdict, revision uint64, raw []byte) delivery.ManifestAccepted { + m, _ := judge.Manifest(raw, nil) + a := delivery.AcceptedOf(digest, v, m) + if a.BusState == nil { + a.BusState = []catalogue.BusStateEntry{} + } + a.Revision = revision + return a +} + +// leftOf is the empty manifest-accepted for module, which left the trunk, judged by judgeID; its event id names the module +// and the manifest it last held, since its body names no digest. +func leftOf(module, lastDigest, judgeID string) (delivery.ManifestAccepted, string) { + a := delivery.ManifestAccepted{V: delivery.Version, Module: module, Judge: judgeID, BusState: []catalogue.BusStateEntry{}} + return a, delivery.ManifestAcceptedEvent + "-left-" + module + "-" + lastDigest +} + +// sayAccepted says a with the event id id, once more when the first publish fails; the module's finding is written when +// both fail and deleted when it is said. Nil SayAccepted says nothing. +func (ix *Indexer) sayAccepted(ctx context.Context, a delivery.ManifestAccepted, id string) error { + if ix.SayAccepted == nil { + return nil + } + err := ix.SayAccepted(ctx, a, id) + if err != nil { + ix.logf("judge: %s for %s failed, published again: %v", delivery.ManifestAcceptedEvent, a.Module, err) + err = ix.SayAccepted(ctx, a, id) + } + ix.unsaid(ctx, a.Module, err) + return err +} + +// unsaidKey is the key of module's unsaid finding. +func unsaidKey(module string) string { + return health.FindingKey(health.ScopeModule, delivery.IndexerModule+".accepted."+module, UnsaidKind) +} + +// unsaid writes module's finding for err, or deletes it with none. +func (ix *Indexer) unsaid(ctx context.Context, module string, err error) { + if ix.Findings == nil { + return + } + key := unsaidKey(module) + werr := write(ctx, ix.Findings, key, func(value []byte, found bool) ([]byte, bool, error) { + if err == nil { + return nil, found, nil + } + at := ix.now().UTC() + summary := fmt.Sprintf("%s of %s was not published: %v", delivery.ManifestAcceptedEvent, module, err) + data, ferr := health.EncodeFinding(health.Finding{Key: key, Kind: UnsaidKind, + Subject: health.Subject{Scope: health.ScopeModule, ID: delivery.IndexerModule + ".accepted." + module}, Severity: health.Warning, + Summary: summary, + Headline: "The indexer could not tell the bus module that " + module + "'s manifest was accepted", + Explanation: "The bus module creates and removes a module's bus state from this event. The indexer says it again at its " + + "next judging pass and clears this once it is said; indexer_say_accepted says it now.", + Resolved: "manifest-accepted of " + module + " was said.", + Evidence: []health.Evidence{{At: at, Said: summary}}, Source: UnsaidKind, Observer: delivery.IndexerModule, At: at}) + return data, false, ferr + }) + if werr != nil { + ix.logf("judge: the finding of %s for %s is not written: %v", delivery.ManifestAcceptedEvent, module, werr) + } +} + +// sayFor says module's manifest-accepted from what the indexer holds now: from its trunk's accepted verdict, or an empty +// one when the trunk holds the module no more (only when left is allowed: by hand a module with no accepted verdict is +// refused). A held module whose verdict waits or is refused gets none; again gives the event an id of its own, so a hand +// act within the bus's duplicate window is still said. +func (ix *Indexer) sayFor(ctx context.Context, module string, allowLeft, again bool) (string, error) { + held, err := ix.heldModules(ctx) + if err != nil { + return "", fmt.Errorf("the modules state cannot be read: %w", err) + } + m, ok := held[module] + var a delivery.ManifestAccepted + var id string + switch { + case !ok && !allowLeft: + return "", fmt.Errorf("the indexer holds no module %s, so it has no accepted verdict to say", module) + case !ok: + a, id = leftOf(module, "", ix.judgeID()) + case m.Manifest == "" || ix.Verdicts == nil || ix.Manifests == nil: + return "", fmt.Errorf("%s has no accepted verdict the indexer holds", module) + default: + key, err := delivery.VerdictsKey(m.Manifest) + if err != nil { + return "", err + } + value, revision, found, err := ix.Verdicts.Get(ctx, key) + if err != nil { + return "", fmt.Errorf("the verdict on %s cannot be read: %w", module, err) + } + var v delivery.ManifestVerdict + if found { + v, err = delivery.DecodeManifestVerdict(value) + } + if !found || err != nil || v.Refused() || v.Waits() { + if !allowLeft { + return "", fmt.Errorf("%s has no accepted verdict on its manifest %s", module, m.Manifest) + } + ix.unsaid(ctx, module, nil) // a set-aside or refused manifest gets none + return "none: its manifest is not accepted", nil + } + raw, _, found, err := ix.Manifests.Get(ctx, m.Manifest) + kept, derr := delivery.DecodeManifest(raw) + if err != nil || !found || derr != nil { + return "", fmt.Errorf("the manifest %s of %s cannot be read", m.Manifest, module) + } + a = acceptedOf(m.Manifest, v, revision, []byte(kept.JSON)) + id = a.EventID() + } + if again { + id += "-again-" + ix.now().UTC().Format(time.RFC3339Nano) + } + if err := ix.sayAccepted(ctx, a, id); err != nil { + return "", fmt.Errorf("%s of %s was not published: %w", delivery.ManifestAcceptedEvent, module, err) + } + if a.Digest == "" { + return "said empty: the trunk holds no module " + module, nil + } + return fmt.Sprintf("said for %s, manifest %s at revision %d, %d bus-state entries", module, a.Digest, a.Revision, len(a.BusState)), nil +} + +// SayUnsaid says again each manifest-accepted whose finding stands (the judging pass's end): cleared once said. +func (ix *Indexer) SayUnsaid(ctx context.Context) { + if ix.Findings == nil { + return + } + keys, err := ix.Findings.Keys(ctx) + if err != nil { + return + } + for _, key := range keys { + if !strings.HasSuffix(key, "."+UnsaidKind) { + continue + } + value, _, found, err := ix.Findings.Get(ctx, key) + if err != nil || !found { + continue + } + f, err := health.DecodeFinding(value) + if err != nil || f.Kind != UnsaidKind { + continue + } + module := strings.TrimPrefix(f.Subject.ID, delivery.IndexerModule+".accepted.") + if _, err := ix.sayFor(ctx, module, true, false); err != nil { + ix.logf("judge: %v; its finding stands", err) + } + } +} + +// SayAcceptedAgain answers indexer_say_accepted: module's manifest-accepted said again now from its accepted verdict. +func (ix *Indexer) SayAcceptedAgain(ctx context.Context, module string) TakenIn { + out := TakenIn{Module: module} + answer, err := ix.sayFor(ctx, module, false, true) + if err != nil { + out.Answer = err.Error() + return out + } + out.Stored, out.Answer = true, answer + return out +} + +// NewSayAccepted is the value indexer_say_accepted writes for a module: its key in the takeins state and the value. +func NewSayAccepted(module, why, by string, at time.Time) (string, []byte, error) { + key, data, err := NewTakeIn(module, why, by, at) + if err != nil { + return "", nil, err + } + var a TakeInAsk + if err := json.Unmarshal(data, &a); err != nil { + return "", nil, err + } + if a.Why == "" { + return "", nil, errors.New("say why it is said again (why): it is a hand act") + } + a.Say = true + data, err = json.Marshal(a) + return sayAcceptedPrefix + key, data, err +} diff --git a/modules/mesh-delivery-indexer/internal/index/candidates.go b/modules/mesh-delivery-indexer/internal/index/candidates.go index 9f8562b7..85559fe0 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidates.go +++ b/modules/mesh-delivery-indexer/internal/index/candidates.go @@ -66,9 +66,13 @@ type Indexer struct { JudgeID string // SayRefused says the event manifest-refused for a refusal newly written; nil says nothing. SayRefused func(ctx context.Context, r delivery.ManifestRefused) - // SayAccepted says the event manifest-accepted for an accepted verdict newly written whose manifest asks for bus state, - // carrying the entries whole (verdicts.go; issue 1966 D3); nil says nothing. - SayAccepted func(ctx context.Context, a delivery.ManifestAccepted) + // SayAccepted publishes the event manifest-accepted a under the bus's duplicate id id (accepted.go; issue 1966 D3, + // DLV-I9): for every accepted verdict newly written, carrying the entries whole, and an empty one for a module leaving + // the trunk; nil says nothing. + SayAccepted func(ctx context.Context, a delivery.ManifestAccepted, id string) error + // Findings is the indexer's findings state, where a manifest-accepted whose publish failed twice is kept until said + // (accepted.go, DLV-I9b); nil writes none. + Findings Store // Packages is the packages state (delivery.PackagesBucket, packages.go; novox/hq ADR 0593 §2); nil publishes none, // and every fingerprint takes the version line. Packages Store diff --git a/modules/mesh-delivery-indexer/internal/index/modules.go b/modules/mesh-delivery-indexer/internal/index/modules.go index 6612ff0f..5c9de69a 100644 --- a/modules/mesh-delivery-indexer/internal/index/modules.go +++ b/modules/mesh-delivery-indexer/internal/index/modules.go @@ -249,6 +249,9 @@ func (ix *Indexer) dropRemoved(ctx context.Context, repository string, held map[ return fmt.Errorf("modules.%s: %w", key, err) } ix.logf("modules.%s deleted: %s holds no module of that name at its trunk (issue 1900)", key, repository) + // The bus module hears the module asks nothing now, so its entries reach orphans (design 25 § 12.2, DLV-I9). + a, id := leftOf(key, cur.Manifest, ix.judgeID()) + _ = ix.sayAccepted(ctx, a, id) } for _, key := range handed { if err := ix.handOver(ctx, key); err != nil { diff --git a/modules/mesh-delivery-indexer/internal/index/remove_test.go b/modules/mesh-delivery-indexer/internal/index/remove_test.go index 012fbdc9..1b14c189 100644 --- a/modules/mesh-delivery-indexer/internal/index/remove_test.go +++ b/modules/mesh-delivery-indexer/internal/index/remove_test.go @@ -25,6 +25,27 @@ func twoModules(t *testing.T) (*Indexer, *memStore) { func TestAMergeThatRemovesAModuleDeletesItsKey(t *testing.T) { // 1900/AC1 ix, mods := twoModules(t) + // DLV-I9 (design 25 § 12.2): the module leaving the trunk is said as an empty manifest-accepted, under an id naming it. + var said []delivery.ManifestAccepted + var ids []string + ix.SayAccepted = func(_ context.Context, a delivery.ManifestAccepted, id string) error { + said, ids = append(said, a), append(ids, id) + return nil + } + defer func() { + var left []string + for i, a := range said { + if a.Digest == "" { + left = append(left, a.Module) + if a.BusState == nil || len(a.BusState) != 0 || !strings.Contains(ids[i], "-left-b") { + t.Errorf("the empty manifest-accepted = %+v, id %s", a, ids[i]) + } + } + } + if strings.Join(left, ",") != "b" { + t.Errorf("said empty for %v, want b alone", left) + } + }() if err := ix.Apply(context.Background(), mergedMsg(catalogueRepo, map[string]string{"modules/a": "t2"})); err != nil { t.Fatal(err) } diff --git a/modules/mesh-delivery-indexer/internal/index/takein.go b/modules/mesh-delivery-indexer/internal/index/takein.go index 579ba62b..66f0838c 100644 --- a/modules/mesh-delivery-indexer/internal/index/takein.go +++ b/modules/mesh-delivery-indexer/internal/index/takein.go @@ -33,7 +33,9 @@ type TakeInAsk struct { // Judge asks the manifests to be judged again now (indexer_judge_again) instead of the module to be taken in. Judge bool `json:"judge,omitempty"` // Digest names, with Judge, the one manifest digest of the module to judge (indexer_judge with a digest). - Digest string `json:"digest,omitempty"` + Digest string `json:"digest,omitempty"` + // Say asks the module's manifest-accepted to be said again now (indexer_say_accepted, DLV-I9c). + Say bool `json:"say,omitempty"` By string `json:"by,omitempty"` At time.Time `json:"at"` Result *TakenIn `json:"result,omitempty"` diff --git a/modules/mesh-delivery-indexer/internal/index/verdicts.go b/modules/mesh-delivery-indexer/internal/index/verdicts.go index 7af32aad..4e27b03c 100644 --- a/modules/mesh-delivery-indexer/internal/index/verdicts.go +++ b/modules/mesh-delivery-indexer/internal/index/verdicts.go @@ -65,6 +65,9 @@ func (ix *Indexer) judgeOver(ctx context.Context, fresh map[string]bool, over ma if ix.Verdicts == nil || ix.Modules == nil || ix.Manifests == nil { return out, nil } + if over == nil { + ix.SayUnsaid(ctx) // a manifest-accepted a pass before failed to publish is said again first (DLV-I9b) + } held, err := ix.heldModules(ctx) if err != nil { return out, err @@ -175,13 +178,11 @@ func (ix *Indexer) judgeOver(ctx context.Context, fresh map[string]bool, over ma r.Revision = revision ix.SayRefused(ctx, r) } - } else if ix.SayAccepted != nil { - // A manifest asking for bus state, newly accepted, is said whole to the bus module (issue 1966 D3, ADR 0595 §4). - if m, problems := judge.Manifest(raws[name], predefined); len(problems) == 0 && len(m.BusState) > 0 { - a := delivery.AcceptedOf(digests[name], v, m) - a.Revision = revision - ix.SayAccepted(ctx, a) - } + } else if !v.Waits() { + // Every manifest newly accepted is said whole to the bus module, its entries empty when it asks for none + // (issue 1966 D3, ADR 0595 §4, design 25 § 12.2); a manifest set aside to wait gets none (DLV-I9). + a := acceptedOf(digests[name], v, revision, raws[name]) + _ = ix.sayAccepted(ctx, a, a.EventID()) } } // The reader capabilities the manifest's own words, forms and seat names need, named with the verdict (ADR 0614 §4). diff --git a/modules/mesh-delivery-indexer/internal/index/verdicts_test.go b/modules/mesh-delivery-indexer/internal/index/verdicts_test.go index e7920d5e..665c80ba 100644 --- a/modules/mesh-delivery-indexer/internal/index/verdicts_test.go +++ b/modules/mesh-delivery-indexer/internal/index/verdicts_test.go @@ -136,9 +136,18 @@ func TestTheIndexerJudgesEveryManifestOnce(t *testing.T) { t.Fatalf("a changed digest is not fresh: %v %v", w.fresh, err) } // 1966/AC10: a newly accepted manifest asking for bus state is said once as manifest-accepted, carrying both entries - // whole; a verdict already held, and a manifest asking for none, say nothing. + // whole; a verdict already held says nothing. var accepted []delivery.ManifestAccepted - ix.SayAccepted = func(_ context.Context, a delivery.ManifestAccepted) { accepted = append(accepted, a) } + var ids []string + var failing int // publishes still to fail + ix.SayAccepted = func(_ context.Context, a delivery.ManifestAccepted, id string) error { + if failing > 0 { + failing-- + return errors.New("no stream answered") + } + accepted, ids = append(accepted, a), append(ids, id) + return nil + } moved := hold("moved", `{"module":"moved","version":"1","requires":[`+ `{"name":"cache","storage":"key-value","template":"state","options":{"history":8}},`+ `{"name":"audit","storage":"stream","template":"log","options":{"max-mib":64}}]}`) @@ -164,6 +173,54 @@ func TestTheIndexerJudgesEveryManifestOnce(t *testing.T) { string(accepted[0].BusState[0].Options["history"]) != "8" { t.Fatalf("said accepted %+v", accepted) } + // DLV-I9 (design 25 § 12.2): a manifest asking for none, newly accepted, is said too, its entries an empty list. + plain := hold("plain", `{"module":"plain","version":"1"}`) + if _, err := ix.JudgeAll(ctx, map[string]bool{plain: true}); err != nil { + t.Fatal(err) + } + if last := accepted[len(accepted)-1]; last.Module != "plain" || last.Digest != plain || last.BusState == nil || len(last.BusState) != 0 { + t.Fatalf("a manifest asking for no bus state was said as %+v", last) + } + // DLV-I9b: a failed publish is published again; one failing twice writes the finding naming the module, which the next + // judging pass says again and clears. + ix.Findings = newMem() + failing = 1 + once := hold("once", `{"module":"once","version":"1"}`) + if _, err := ix.JudgeAll(ctx, map[string]bool{once: true}); err != nil || accepted[len(accepted)-1].Module != "once" { + t.Fatalf("a publish failing once was not published again: %v %+v", err, accepted[len(accepted)-1]) + } + failing = 2 + flaky := hold("flaky", `{"module":"flaky","version":"1"}`) + if _, err := ix.JudgeAll(ctx, map[string]bool{flaky: true}); err != nil { + t.Fatal(err) + } + if value, _, found, _ := ix.Findings.Get(ctx, unsaidKey("flaky")); !found || !strings.Contains(string(value), "flaky") { + t.Fatalf("a publish failing twice wrote no finding naming flaky: %s", value) + } + if _, err := ix.JudgeAll(ctx, nil); err != nil { + t.Fatal(err) + } + if _, _, found, _ := ix.Findings.Get(ctx, unsaidKey("flaky")); found || accepted[len(accepted)-1].Module != "flaky" || + accepted[len(accepted)-1].Digest != flaky { + t.Fatalf("the next pass did not say flaky and clear its finding (found %v, last said %+v)", found, accepted[len(accepted)-1]) + } + // DLV-I9c: indexer_say_accepted says it again from the held verdict, under an id of its own; a module with no accepted + // verdict, or none held, is refused. + if r := ix.SayAcceptedAgain(ctx, "flaky"); !r.Stored || accepted[len(accepted)-1].Module != "flaky" || !strings.Contains(ids[len(ids)-1], "-again-") { + t.Fatalf("said again: %+v, id %s", r, ids[len(ids)-1]) + } + for _, m := range []string{"old", "nobody"} { + if r := ix.SayAcceptedAgain(ctx, m); r.Stored { + t.Errorf("%s said again: %+v", m, r) + } + } + if key, value, err := NewSayAccepted("flaky", "the bus module missed it", "jochen@shanks", time.Now()); err != nil || + !strings.HasPrefix(key, sayAcceptedPrefix) || !strings.Contains(string(value), `"say":true`) { + t.Errorf("the ask = %s %s (%v)", key, value, err) + } + if _, _, err := NewSayAccepted("flaky", "", "jochen@shanks", time.Now()); err == nil { + t.Error("an ask saying no why was taken") + } // 1966/AC11: modules using state and none using logs but the foundation: logs counts 0, so a manifest of another // module using logs is refused and the foundation's is accepted. ix.OldSections = newMem() @@ -259,6 +316,11 @@ func TestTheIndexerJudgesEveryManifestOnce(t *testing.T) { if err != nil || len(j.Waiting) != 1 || j.Waiting[0] != "later" || len(said) != before { t.Fatalf("a manifest with fields from later: %+v %v, said %d", j, err, len(said)) } + for _, a := range accepted { + if a.Module == "later" { + t.Fatalf("a manifest set aside to wait was said accepted: %+v (DLV-I9)", a) + } + } if v := verdict(later); !v.Waits() || v.Refused() || strings.Join(v.Unknown, ",") != "a-field-from-later,another" { t.Fatalf("later: %+v", v) } diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index cbc84e63..1d92e787 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -137,7 +137,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build // Three invokes: the file of a registered module's manifest and lock files (registered.go), the forge's trees (the named paths' objects, since the indexer is the one reader of a manifest's paths: // ADR 0499 clauses 2 and 3, ADR 0495 clause 1; and the module directories of the trunk, issue 1900) and whether a // commit is in another (issue 1900). - if want := []string{"mesh-delivery-assigner.placement", "gitea.gitea_commit_trees", "gitea.gitea_contains", "gitea.gitea_get_file", "gitea.gitea_open_pulls", "seat:node-engine.composition"}; !reflect.DeepEqual(m.Invokes, want) { + if want := []string{"mesh-delivery-assigner.placement", "gitea.gitea_commit_trees", "gitea.gitea_contains", "gitea.gitea_get_file", "gitea.gitea_open_pulls", "seat:node-engine.composition", "seat:mesh-hand-acts.hand-act"}; !reflect.DeepEqual(m.Invokes, want) { t.Errorf("invokes = %v, want %v", m.Invokes, want) } // 1945: indexer_take_in, served by its tools bundle through the takeins state; the process serves none. The same bundle @@ -152,8 +152,8 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build t.Errorf("%s answers the state %s, which the views family does not say", v.Tool, v.State) } } - want = append(want, "indexer_first_views", "indexer_first_view_again", index.ConfigReadersTool, index.BusStateTool, index.RequirementsTool, index.PackageUsersTool) // DLV-F3r, DLV-I10, DLV-I11, DLV-I7, DLV-I8 - if !reflect.DeepEqual(m.Tools, want) || len(want) != 23 { + want = append(want, "indexer_first_views", "indexer_first_view_again", index.ConfigReadersTool, index.BusStateTool, index.RequirementsTool, index.PackageUsersTool, index.SayAcceptedTool) // DLV-F3r, DLV-I10, DLV-I11, DLV-I7, DLV-I8, DLV-I9c + if !reflect.DeepEqual(m.Tools, want) || len(want) != 24 { t.Errorf("tools = %v, want %v", m.Tools, want) } if m.RunsAs != m.Module || m.SecretsOwner != m.RunsAs { diff --git a/modules/mesh-delivery-indexer/module.json b/modules/mesh-delivery-indexer/module.json index dd4b3958..569f4f8d 100644 --- a/modules/mesh-delivery-indexer/module.json +++ b/modules/mesh-delivery-indexer/module.json @@ -38,7 +38,8 @@ "indexer_config_readers", "indexer_bus_state", "indexer_requirements", - "indexer_package_users" + "indexer_package_users", + "indexer_say_accepted" ], "consumes": [ "go-package-registry.published", @@ -88,7 +89,8 @@ "gitea.gitea_contains", "gitea.gitea_get_file", "gitea.gitea_open_pulls", - "seat:node-engine.composition" + "seat:node-engine.composition", + "seat:mesh-hand-acts.hand-act" ], "own-secrets": { "broker": "${dir:state}/broker"