From f6163349af3103d7fb8b22cdf03493c7df1d5074 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 20 Nov 2026 04:05:11 +0100 Subject: [PATCH] W1: the watchman reads the H1 take-in, the mover's entry and the want MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The watchman follows the one model in one pass. Its take-in waits are the seat's take-in.pending and take-in.taken, keyed ., timed from the message's own time by the setting take-in-bound (60 minutes by default), and seeded from indexer_pending's held take-ins; the group bound goes. A taken-in mover's bus state is read from its catalog entry by fingerprint through the seat mesh-catalog's verb entry, bus-state-overdue timed from the take-in. The manifest verdicts and their findings, the head timing of source-moved, acting-part-late and delivery-waits go, with their readers, consumes and the indexer_verdicts invoke. A want awaiting its hand placement is not timed, and the want readers follow the H1 want. A record-opened is heard on record-opened.. Builds: design 47 H §741, §744, §764 (the indexer: manifest-refused, manifest-accepted, the watchman's findings; the assigner: acting-part-late, delivery-waits), §772; lead answers H-A-25, H-A-37, H-A-49, H-A-70, H-A-78 (2), H-A-79, H-A-91 (2), H-A-94, H-A-95 (1), H-A-97, H-A-102 Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy --- .../cmd/mesh-delivery-watchman/consumes.go | 79 ++--- .../integration_test.go | 2 +- .../cmd/mesh-delivery-watchman/main.go | 68 ++-- .../cmd/mesh-delivery-watchman/main_test.go | 46 +-- .../internal/watch/busstate.go | 161 +++++++-- .../internal/watch/findings_test.go | 324 ++++++++---------- .../internal/watch/mirror.go | 28 +- .../internal/watch/names.go | 1 - .../internal/watch/pipelines.go | 107 +----- .../internal/watch/pipelines_test.go | 155 ++------- .../internal/watch/records.go | 25 +- .../internal/watch/signals.go | 4 +- .../internal/watch/takein.go | 198 +++++------ .../internal/watch/waits.go | 130 ++----- .../internal/watch/waits_test.go | 138 ++------ .../internal/watch/watch.go | 23 +- .../internal/watch/watch_test.go | 8 +- .../mesh-delivery-watchman/manifest_test.go | 30 +- modules/mesh-delivery-watchman/module.json | 30 +- 19 files changed, 614 insertions(+), 943 deletions(-) diff --git a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/consumes.go b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/consumes.go index 895cfae4..7065d307 100644 --- a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/consumes.go +++ b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/consumes.go @@ -2,7 +2,6 @@ package main import ( "context" - "encoding/json" "git.novox.be/novox/mesh-contracts/go/seats/meshbroker" "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" @@ -26,8 +25,8 @@ import ( "mesh-delivery-watchman/internal/watch" ) -// **What the watchman consumes** (its manifest's `consumes`; novox/hq ADR 0595 §4, design 25 § 12.2; DLV-W1): the seat mesh-catalog's -// manifest-accepted and the seat mesh-broker's created and refused, all on its durable consumer on EVENTS (the seat's +// **What the watchman consumes** (its manifest's `consumes`; novox/hq ADR 0595 §4, design 25 § 12.2; DLV-W1): the seat mesh-broker's +// created and refused (a taken-in mover's bus state is read from its catalog entry, busstate.go), all on its durable consumer on EVENTS (the seat's // events on the seat's subjects like every seat's: the lead's ruling, registry milestone 13:0x); beside it any module // stream its membership names (Hears). Each consumer is the one the grants part makes; one // not there yet is opened when the watchman's membership next changes, and one opened is never opened twice. An event is @@ -35,35 +34,18 @@ import ( // Consumed event names, as the consumes name them (the emitter is read from the subject's emitter token). const ( - eventAccepted = "manifest-accepted" - // eventManifestRefused is the seat mesh-catalog's manifest-refused; meshGrantsSeat the seat of the store whose filling, full, room and waits it hears. - eventManifestRefused = "manifest-refused" - meshGrantsSeat = "mesh-grants" - eventCreated = "created" - eventRefused = "refused" + // meshGrantsSeat is the seat of the store whose filling, full and room it hears. + meshGrantsSeat = "mesh-grants" + eventCreated = "created" + eventRefused = "refused" ) -// consumed hands one consumed event to the watchman, by its subject: manifest-accepted, and the seat mesh-broker's created -// and refused. Anything else is left. +// consumed hands one consumed event to the watchman, by its subject: the seats' events it times, and the seat mesh-broker's +// created and refused. Anything else is left. func consumed(w *watch.Watchman, subject string, body []byte, logf func(string, ...any)) { switch { - case subject == meshcatalog.SubjectManifestAccepted: - var a meshcatalog.ManifestAcceptedEvent - if err := json.Unmarshal(body, &a); err != nil || a.Module == "" { - logf("a manifest-accepted that does not read is left: %v", err) - return - } - w.Accepted(a) - case subject == meshcatalog.SubjectManifestRefused: - var r meshcatalog.ManifestRefusedEvent - if err := json.Unmarshal(body, &r); err != nil || r.Module == "" { - logf("a manifest-refused that does not read is left: %v", err) - return - } - w.ManifestRefused(r) case pipelineSubject(w, subject, body): case engineWaitSubject(w, subject, body): - case moduleWaitSubject(w, subject, body): case recordSubject(w, subject, body): case buildSeatSubject(w, subject, body): case brokerSubject(subject) && strings.HasSuffix(subject, ".event."+eventCreated): @@ -84,7 +66,7 @@ func consumed(w *watch.Watchman, subject string, body []byte, logf func(string, // hearConsumed opens the watchman's durable consumers: on EVENTS now, and on each module stream its membership names as // that membership arrives. A consumer the bus does not hold yet is said and tried again at the membership's next change. func hearConsumed(ctx context.Context, js jetstream.JetStream, cred process.Account, hub *process.EventHub, w *watch.Watchman, binding *process.Binding, logf func(string, ...any)) { - // The seats' events (manifest-accepted, the bus module's created and refused) are one handler of the module's one + // The seats' events (the bus module's created and refused among them) are one handler of the module's one // pull of EVENTS, beside the take-in events and the views reader. The seat mesh-broker's recreated and dropped go to // the binding of the watchman's finding-events (design 47 G A-274, §577), which this handler holds, since a message has // one taker; and the binding asks show again at each bind of the hub's consumer, as its Register does. @@ -99,16 +81,17 @@ func hearConsumed(ctx context.Context, js jetstream.JetStream, cred process.Acco _ = m.Ack() } // One handler per seat it consumes, since a message has one taker (design 47 G A-322): the events the branches follower of the - // catalog half adopts (the seat mesh-catalog's registration-changed, the seat mesh-delivery's source-moved) are the follower's, and - // the watchman takes source-moved from it (AdoptEvent, main.go), never from a second handler. + // catalog half adopts (the seat mesh-catalog's registration-changed, the seat mesh-delivery's source-moved) are the follower's + // alone: the watchman learns no head (design 47 H §741). var patterns []string for _, seat := range []string{meshwants.Seat, meshGrantsSeat, meshpipelinerunner.Seat, nodeengine.Seat, meshbroker.Seat, nodebuildgo.Seat, nodebuildimage.Seat, nodebuildnode.Seat} { patterns = append(patterns, "mesh.seat."+seat+".event.>") } - for _, e := range []string{meshdelivery.EventRecordOpened, meshdelivery.EventRecordEnded, meshdelivery.EventFilling, meshdelivery.EventFull, meshdelivery.EventRoom} { + patterns = append(patterns, meshdelivery.SubjectRecordOpenedAll) // record-opened. (design 47 H-A-25) + for _, e := range []string{meshdelivery.EventRecordEnded, meshdelivery.EventFilling, meshdelivery.EventFull, meshdelivery.EventRoom} { patterns = append(patterns, seatdef.EventSubject(meshdelivery.Seat, e)) } - for _, e := range []string{eventAccepted, eventManifestRefused, meshdelivery.EventFilling, meshdelivery.EventFull, meshdelivery.EventRoom} { + for _, e := range []string{meshdelivery.EventFilling, meshdelivery.EventFull, meshdelivery.EventRoom} { patterns = append(patterns, seatdef.EventSubject(meshcatalog.Seat, e)) } for _, pattern := range patterns { @@ -190,15 +173,10 @@ func brokerSubject(subject string) bool { return strings.HasPrefix(subject, "mesh.seat."+meshbroker.Seat+".event.") } -// pipelineSubject hands the gate's `source-moved`, the runner's `pipeline-started`, `step-started` and `step-ended` and the assigner's -// `step-progress` to the watchman (watch/pipelines.go, watch/steps.go, design 47 G §12, §732): the head of a watched branch, its -// pipeline and each step's clock. It says whether the subject was one of them. +// pipelineSubject hands the runner's `step-started` and `step-ended` and the assigner's `step-progress` to the watchman +// (watch/pipelines.go, watch/steps.go, design 47 G §732, H §741): each step's clock. It says whether the subject was one of them. func pipelineSubject(w *watch.Watchman, subject string, body []byte) bool { switch { - case strings.HasPrefix(subject, "mesh.seat."+meshdelivery.Seat+".event."+meshdelivery.EventSourceMoved+"."): - w.SourceMoved(body) - case strings.HasPrefix(subject, "mesh.seat."+meshpipelinerunner.Seat+".event."+meshpipelinerunner.EventPipelineStarted+"."): - w.PipelineStarted(body) case strings.HasPrefix(subject, "mesh.seat."+meshpipelinerunner.Seat+".event."+meshpipelinerunner.EventStepStarted+"."): w.StepEvent(body) w.StepStarted(body) @@ -217,10 +195,14 @@ func pipelineSubject(w *watch.Watchman, subject string, body []byte) bool { // `record-ended` to the watchman (watch/records.go, design 47 G §148, §181). It says whether the subject was one of them. func recordSubject(w *watch.Watchman, subject string, body []byte) bool { parts := strings.Split(subject, ".") - if len(parts) != 5 || parts[0] != "mesh" || parts[1] != "seat" || parts[3] != "event" { + if len(parts) < 5 || parts[0] != "mesh" || parts[1] != "seat" || parts[3] != "event" { return false } seat, event := parts[2], parts[4] + // record-opened is said on record-opened. (design 47 H-A-25); every other event here has no token after its name. + if want := map[bool]int{true: 6, false: 5}[seat == meshdelivery.Seat && event == meshdelivery.EventRecordOpened]; len(parts) != want { + return false + } switch { case event == meshwants.EventWaitHeld: w.WaitHeld(seat, body) @@ -258,25 +240,6 @@ func engineWaitSubject(w *watch.Watchman, subject string, body []byte) bool { return true } -// moduleWaitSubject hands a module's `wait-held` and `wait-ended` on its own stream, on -// mesh.stream..event..., to the watchman: the form the builder says a key queued behind a package -// in (design 47 G A-192). It says whether the subject was one. -func moduleWaitSubject(w *watch.Watchman, subject string, body []byte) bool { - parts := strings.Split(subject, ".") - if len(parts) != 7 || parts[0] != "mesh" || parts[1] != "stream" || parts[3] != "event" { - return false - } - switch parts[4] { - case nodeengine.EventWaitHeld: - w.ModuleWaitHeld(parts[2], body) - case nodeengine.EventWaitEnded: - w.ModuleWaitEnded(parts[2], body) - default: - return false - } - return true -} - // buildSeatSubject hands the build seats' `ask-taken`, `step-progress` and `step-ended`, on // mesh.seat..event.., to the watchman (design 47 G §483, §496, §602, A-136): it times every ask of a // build seat from them. It says whether the subject was one of them. diff --git a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/integration_test.go b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/integration_test.go index bab80b58..261d6bfc 100644 --- a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/integration_test.go +++ b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/integration_test.go @@ -578,7 +578,7 @@ func aWantIsMirroredAsAPulseAndTheHaveNamingItEndsIt(t *testing.T) { fpOld := meshcatalogclient.SourcePrefix2 + strings.Repeat("a", 64) putWant := func(node, module, fp string) { t.Helper() - body, err := meshwantsclient.EncodeWant(meshwantsclient.Want{Fp: fp, Composed: "sha256:" + strings.Repeat("c", 64)}) + body, err := meshwantsclient.EncodeWant(meshwantsclient.Want{Fp: fp, Version: "0.1.4", Pipeline: "main.42", Rank: 1}) if err != nil { t.Fatal(err) } diff --git a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go index 06174855..29d25829 100644 --- a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go +++ b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go @@ -12,10 +12,10 @@ import ( "fmt" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" "io" + "io/fs" "log" "os" "os/signal" @@ -31,7 +31,6 @@ import ( "git.novox.be/novox/mesh-contracts/go/seats/meshbroker" "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshwatchdogsclient" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein" "mesh-delivery-watchman/internal/socket" "mesh-delivery-watchman/internal/watch" @@ -274,11 +273,17 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro w := watch.New(cred.Module, &kvOut{ctx: ctx, presence: kvs.presence, pulse: kvs.pulse, stuk: kvs.stuck, findings: held, js: js, cred: cred}, time.Now) w.Logf(logger.Printf) - // The seat mesh-delivery's source-moved is the branches follower's event (one taker, design 47 G A-322): the watchman takes it from the - // follower's subscription, beside the follower's handler. - if err := reader.AdoptEvent(meshdelivery.Seat, meshdelivery.EventSourceMoved, func(_ context.Context, data []byte) error { w.SourceMoved(data); return nil }); err != nil { + // A taken-in mover's catalog entry is read by its fingerprint through the seat mesh-catalog's verb entry (design 47 + // H-A-94 (1)), its answer heard on its own goroutine. + entries := meshcatalogclient.NewEntries(calls.Call) + entries.Changed = w.EntryChanged + w.ReadEntriesWith(entries) + // The setting take-in-bound (design 47 H §744, H-A-94 (2)), read at the start from the file the manifest declares. + bound, err := takeInBound(getenv) + if err != nil { return err } + w.SetTakeInBound(bound) // What the wants and the modules hold is ended by their own late read (watch.ReadLate), never by the start's end. w.WantsAt(func() string { return reader.HolderBucket(meshwants.Seat, meshwants.StateWants) }) w.AwaitsWants() @@ -400,7 +405,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } else { atCatalog := atHolder(calls.Call, holders, meshcatalog.Seat) // the indexer holds the seat mesh-catalog seedTakeIn(w, atCatalog, logger.Printf) - seedVerdicts(w, atCatalog, logger.Printf) seedSteps(w, calls.Call, logger.Printf) calls.Close() } @@ -476,6 +480,32 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } } +// takeInBound is the setting take-in-bound, a Go duration such as 60m, read from the file the manifest declares with the +// content ${config:take-in-bound}, which MESH_WATCHMAN_TAKE_IN_BOUND_FILE names; zero, its default, for no path, no file or an +// empty one, and an error naming the file for a content that is no positive duration. +func takeInBound(getenv func(string) string) (time.Duration, error) { + path := getenv("MESH_WATCHMAN_TAKE_IN_BOUND_FILE") + if path == "" { + return 0, nil + } + raw, err := os.ReadFile(path) + if errors.Is(err, fs.ErrNotExist) { + return 0, nil + } + if err != nil { + return 0, err + } + v := strings.TrimSpace(string(raw)) + if v == "" { + return 0, nil + } + d, err := time.ParseDuration(v) + if err != nil || d <= 0 { + return 0, fmt.Errorf("the setting take-in-bound in %s is %q, no positive duration such as 60m", path, v) + } + return d, nil +} + // eventsStream is the shared stream of the modules' ordinary events, as the bus names it. const eventsStream = "EVENTS" @@ -499,10 +529,10 @@ func atHolder(call func(key string, body any) (json.RawMessage, error), holders func seedTakeIn(w *watch.Watchman, call func(key string, body any) (json.RawMessage, error), logf func(string, ...any)) { raw, err := call(pendingTool, map[string]any{}) if err == nil { - var a takein.PendingAnswer - if a, err = takein.DecodePendingAnswer(raw); err == nil { + var a watch.HeldTakeIns + if a, err = watch.DecodeHeldTakeIns(raw); err == nil { w.SeedTakeIn(a) - logf("take-in: %d waits the indexer holds are watched from their own since (%s)", len(a.Waits), pendingTool) + logf("take-in: %d take-ins the indexer holds are watched from their own since (%s)", len(a.Held), pendingTool) return } } @@ -527,20 +557,6 @@ func seedSteps(w *watch.Watchman, call func(key string, body any) (json.RawMessa logf("steps: %s cannot be asked (%v), so a step open before this start is timed only once an event of it is heard", standsVerb, err) } -// seedVerdicts asks the indexer's verdicts and raises the findings they hold (design 47 G §500): a verdict said while the watchman was -// down is then not lost. An indexer that does not answer is said: the findings are then raised from the verdicts heard after. -func seedVerdicts(w *watch.Watchman, call func(key string, body any) (json.RawMessage, error), logf func(string, ...any)) { - raw, err := call(watch.VerdictsTool, map[string]any{"all": true}) - if err == nil { - var rows []watch.VerdictRow - if rows, err = watch.DecodeVerdictRows(raw); err == nil { - w.SeedVerdicts(rows) - return - } - } - logf("verdicts: %s cannot be asked (%v), so a manifest refused before this start is said only once a verdict of it is heard", watch.VerdictsTool, err) -} - // takeInHeard hands one take-in event of the indexer to the watchman, by its name relative to the indexer, and acknowledges it; // one that is not a take-in event is said and acknowledged, since hearing it again would not read it. func takeInHeard(w *watch.Watchman, m jetstream.Msg, logf func(string, ...any)) { @@ -548,7 +564,11 @@ func takeInHeard(w *watch.Watchman, m jetstream.Msg, logf func(string, ...any)) if !ok { logf("the subject %s of the take-in stream is not an event's, left", m.Subject()) } else { - w.TakeIn(event, m.Data()) + at := time.Now() + if meta, err := m.Metadata(); err == nil { + at = meta.Timestamp // the message's own time (design 47 H-A-94 (2)) + } + w.TakeIn(event, m.Data(), at) } if meta, err := m.Metadata(); err == nil && meta.NumPending == 0 { w.TakeInCaughtUp() diff --git a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main_test.go b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main_test.go index 3843658f..df548974 100644 --- a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main_test.go +++ b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main_test.go @@ -5,18 +5,17 @@ import ( "encoding/json" "errors" "fmt" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" "io" "strings" "testing" "time" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" + "git.novox.be/novox/mesh-sdk/go/process" "git.novox.be/novox/mesh-sdk/go/views" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein" - "mesh-delivery-watchman/internal/watch" "github.com/nats-io/nats.go/jetstream" @@ -67,9 +66,10 @@ func TestRunNeedsItsCredential(t *testing.T) { // 1135/AC6 if !out.findings[watch.FindingKey(watch.ScopeModule, "m", watch.KindBusStateRefused+"-l")] { t.Fatalf("refused reached no finding: %v", out.findings) } + // Design 47 H §764, H-A-49: manifest-accepted is retired; it is heard by nothing. consumed(w, "mesh.seat.mesh-catalog.event.manifest-accepted", []byte(`{"module":"m","digest":"d","bus-state":[]}`), say) - if len(out.findings) != 0 { - t.Fatalf("an accepted manifest no longer asking for l left %v", out.findings) + if len(out.findings) != 1 { + t.Fatalf("a manifest-accepted was heard: %v", out.findings) } consumed(w, "mesh.seat.mesh-broker.event.refused", []byte(`{"module":"m","local":"l"}`), say) consumed(w, "mesh.seat.mesh-broker.event.created", []byte(`{"module":"m","local-name":"l"}`), say) @@ -77,39 +77,36 @@ func TestRunNeedsItsCredential(t *testing.T) { // 1135/AC6 t.Fatalf("created left %v", out.findings) } }) - t.Run("design 47 G §500: the indexer's manifest-refused reaches the watchman by its subject", func(t *testing.T) { + t.Run("design 47 H-A-25: a record-opened on record-opened. reaches the watchman by its subject", func(t *testing.T) { out := &findingsOut{findings: map[string]bool{}} w := watch.New(watchmanModule, out, time.Now) - consumed(w, "mesh.seat.mesh-catalog.event.manifest-refused", []byte(`{"module":"m","digest":"d","problems":["x"]}`), func(string, ...any) {}) - if !out.findings[watch.FindingKey(watch.ScopeModule, "m", watch.KindManifestRefused)] { - t.Fatalf("a refused manifest reached %v", out.findings) + at := time.Now().Add(-25 * time.Hour).UTC().Format(time.RFC3339) + consumed(w, "mesh.seat.mesh-delivery.event.record-opened.fedcba9876543210", []byte(`{"record":"r1","milestone":"main.42","branch":"main","source":42,"call":"fedcba9876543210","at":"`+at+`"}`), func(string, ...any) {}) + if !out.findings[watch.FindingKey(watch.ScopeDelivery, "r1", watch.KindRecordOpen)] { + t.Fatalf("a record open 25 hours reached %v", out.findings) } }) t.Run("design 47 G §499, §541: a node-engine's wait-held and wait-ended reach the watchman by their subjects", func(t *testing.T) { out := &findingsOut{findings: map[string]bool{}} w := watch.New(watchmanModule, out, time.Now) say := func(string, ...any) {} - body := fmt.Sprintf(`{"step":"compose","reason":"entry-missing","since":%q}`, time.Now().Add(-2*time.Hour).UTC().Format(time.RFC3339)) + body := fmt.Sprintf(`{"step":"kept-build","reason":"node away","node":"build1","since":%q}`, time.Now().Add(-2*time.Hour).UTC().Format(time.RFC3339)) consumed(w, "mesh.seat.node-engine.event.wait-held.ace.web", []byte(body), say) - if !out.findings[watch.FindingKey(watch.ScopeDelivery, "ace.web", watch.KindDeliveryWaits)] { - t.Fatalf("a hold two hours old reached %v", out.findings) + if !out.findings[watch.FindingKey(watch.ScopeDelivery, "ace.web", watch.KindKeptBuildWaits)] { + t.Fatalf("a wait two hours old reached %v", out.findings) } consumed(w, "mesh.seat.node-engine.event.wait-ended.ace.web", []byte(body), say) if len(out.findings) != 0 { t.Fatalf("wait-ended left %v", out.findings) } }) - t.Run("design 47 G A-192: a module's wait-held on its stream, naming a part, reaches the watchman", func(t *testing.T) { + t.Run("design 47 H §764, H-A-70: a module's wait-held on its stream, naming a part, is an acting part's wait and reaches nothing", func(t *testing.T) { out := &findingsOut{findings: map[string]bool{}} w := watch.New(watchmanModule, out, time.Now) body := fmt.Sprintf(`{"step":"building","part":"mesh-sdk@v0.1.14","module":"web","fingerprint":"src2:f","bound":"1h0m0s","since":%q}`, time.Now().Add(-2*time.Hour).UTC().Format(time.RFC3339)) consumed(w, "mesh.stream.mesh-delivery-builder.event.wait-held.web.src2:f", []byte(body), func(string, ...any) {}) - if len(out.findings) != 1 { - t.Fatalf("a package wait two hours old reached %v", out.findings) - } - consumed(w, "mesh.stream.mesh-delivery-builder.event.wait-ended.web.src2:f", []byte(body), func(string, ...any) {}) if len(out.findings) != 0 { - t.Fatalf("wait-ended left %v", out.findings) + t.Fatalf("a package wait two hours old reached %v", out.findings) } }) err := run(context.Background(), func(string) string { return "" }, io.Discard) @@ -119,11 +116,11 @@ func TestRunNeedsItsCredential(t *testing.T) { // 1135/AC6 t.Run("1966/AC195-5: a seed indexer_pending could not answer, or answered unreadably, ends nothing", func(t *testing.T) { for name, call := range map[string]func(string, any) (json.RawMessage, error){ "cannot be asked": func(string, any) (json.RawMessage, error) { return nil, errors.New("no answer") }, - "unreadable": func(string, any) (json.RawMessage, error) { return json.RawMessage(`{"waits":`), nil }, + "unreadable": func(string, any) (json.RawMessage, error) { return json.RawMessage(`{"held":`), nil }, } { out := &findingsOut{findings: map[string]bool{}} w := watch.New(watchmanModule, out, time.Now) - const key = "m.00000000000000000042" + key := watch.TakeInKey("main", 42) old := watch.Finding{Scope: watch.ScopeDelivery, ID: key, Kind: watch.KindTakeInWaits, Severity: watch.Warning, Source: "take-in", Since: time.Now().Add(-3 * time.Hour), Summary: "held", Said: "held", Headline: "held", Explanation: "held", Needs: "a take-in"} value, err := watch.EncodeFinding(old) @@ -132,12 +129,7 @@ func TestRunNeedsItsCredential(t *testing.T) { // 1135/AC6 } out.findings[old.Key()] = true w.SeedFinding(old.Key(), value) - body, err := takein.EncodeTakeIn(takein.TakeIn{Kind: takein.TakeInPending, Key: "m.00000000000000000043", What: takein.TakeInRevision, Since: time.Now(), Lost: []string{"m.00000000000000000043"}, - Merges: []takein.Merge{{Repository: "novox/a", Number: 1, Merge: "c0ffee"}}}) - if err != nil { - t.Fatal(err) - } - w.TakeIn(takein.TakeInPendingEvent, body) + w.TakeIn(watch.TakeInPendingEvent, []byte(`{"branch":"main","source":43,"why":"the catalog store is full"}`), time.Now()) seedTakeIn(w, call, func(string, ...any) {}) var asked string seedTakeIn(w, atHolder(func(key string, _ any) (json.RawMessage, error) { asked = key; return nil, errors.New("no answer") }, diff --git a/modules/mesh-delivery-watchman/internal/watch/busstate.go b/modules/mesh-delivery-watchman/internal/watch/busstate.go index 2def5880..d03be9e3 100644 --- a/modules/mesh-delivery-watchman/internal/watch/busstate.go +++ b/modules/mesh-delivery-watchman/internal/watch/busstate.go @@ -1,6 +1,7 @@ package watch import ( + "context" "encoding/json" "fmt" "sort" @@ -8,24 +9,27 @@ import ( "time" "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshwatchdogsclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-sdk/go/catalogue" ) // **A refused or overdue creation of bus state is loud** (novox/hq ADR 0595 §4; design 25 § 12.2; DLV-W1). The watchman -// hears, through its `consumes`, the indexer's `manifest-accepted` (which carries the module's bus-state entries whole) -// and the bus module's own `created` and `refused`. An entry an accepted manifest asks for that has no `created` or -// `refused` within OverdueBound is the finding bus-state-overdue, naming the module and the entry; a `refused` is the -// finding bus-state-refused, with why. A later `created` clears both, and so does a later accepted manifest of that -// module that no longer asks for the entry. Never the bus module's to raise: it cannot while it is down. +// learns the bus-state entries a mover's manifest asks for from the mover's catalog entry, read by its fingerprint through +// the seat mesh-catalog's verb `entry` once the indexer's `take-in.taken` names the mover (design 47 H-A-94 (1), H-A-49), +// and hears the bus module's own `created` and `refused`. An entry a taken-in mover asks for that has no `created` or +// `refused` within OverdueBound of its `take-in.taken` is the finding bus-state-overdue, naming the module and the entry; +// a `refused` is the finding bus-state-refused, with why. A later `created` clears both, and so does a later take-in of +// that module whose entry no longer asks for the entry. Never the bus module's to raise: it cannot while it is down. // // The bound is a row of the signals table: a pulse key per entry with OverdueBound to live, whose expiry is the finding, // so no loop looks and a pulse that stood across a restart still acts. Both findings outlive a restart (signalKind), // since what ends them is heard, not read. // -// The bus module says `created` once per entry, so an entry it said created is not waited on again when a later manifest -// of its module is accepted with the same entry (review round 1): its mark, a pulse key with no time to live under the -// row bus-state-created, is read back at a restart (SeedSignal), and goes when a refused is heard or a manifest no -// longer asks for the entry, so a later ask of it waits again. +// The bus module says `created` once per entry, so an entry it said created is not waited on again when a later take-in +// of its module asks for the same entry (review round 1): its mark, a pulse key with no time to live under the +// row bus-state-created, is read back at a restart (SeedSignal), and goes when a refused is heard or an entry no +// longer asks for it, so a later ask of it waits again. // OverdueBound is how long an accepted entry may wait for the bus module's created or refused. const OverdueBound = 5 * time.Minute @@ -59,41 +63,138 @@ func refusedKey(module, local string) string { return FindingKey(ScopeModule, module, KindBusStateRefused+"-"+local) } -// Accepted is the indexer's manifest-accepted heard: each entry it asks for waits OverdueBound for the bus module's word, -// and every entry of the module this watchman waits on or holds a finding of that it no longer asks for is ended. -func (w *Watchman) Accepted(a meshcatalog.ManifestAcceptedEvent) { +// EntryReader reads a mover's catalog entry by its fingerprint through the seat mesh-catalog's verb `entry` +// (meshcatalogclient.Entries): Need answers one kept or asks it once, telling its Changed when answered; Release forgets it. +type EntryReader interface { + Need(ctx context.Context, fingerprint string) (meshcatalog.CatalogEntry, meshcatalogclient.EntryState) + Release(fingerprint string) +} + +// ReadEntriesWith sets the reader of the movers' catalog entries; with none, a take-in's movers wait on no bus state. +func (w *Watchman) ReadEntriesWith(r EntryReader) { w.mu.Lock() defer w.mu.Unlock() - w.manifestAccepted(a.Module, a.Problems) // §500: the problems a stricter validator finds are the module's warning + w.entries = r +} + +// moverTaken is a mover of a take-in whose catalog entry is asked: its module and the take-in's time. +type moverTaken struct { + module string + at time.Time +} + +// moversTaken asks the catalog entry of each mover a take-in.taken names, w.mu held: an entry already answered is read +// now, any other once its answer comes (EntryChanged). +func (w *Watchman) moversTaken(movers []meshcatalog.TakeInEntry, at time.Time) { + if w.entries == nil { + return + } + for _, m := range movers { + if m.Fingerprint == "" || m.Module == "" { + continue + } + w.movers[m.Fingerprint] = moverTaken{module: m.Module, at: at} + if e, state := w.entries.Need(context.Background(), m.Fingerprint); state == meshcatalogclient.EntryAnswered { + w.entryRead(m.Fingerprint, e) + } + } +} + +// EntryChanged hears the answer of a mover's entry asked by its fingerprint (the Entries' Changed): an answered entry's +// bus state is waited on; one the holder does not hold is said, since its bus state cannot be known. +func (w *Watchman) EntryChanged(row meshcatalogclient.EntryRow) { + w.mu.Lock() + defer w.mu.Unlock() + m, ok := w.movers[row.Fingerprint] + if !ok { + return + } + switch { + case row.State == meshcatalogclient.EntryAnswered && row.Entry != nil: + w.entryRead(row.Fingerprint, *row.Entry) + case row.State == meshcatalogclient.EntryNotFound: + w.say("the catalog entry of %s at %s is not held (%s), so its bus state is not waited on", m.module, short(row.Fingerprint), row.Detail) + delete(w.movers, row.Fingerprint) + w.entries.Release(row.Fingerprint) + } +} + +// entryRead is a mover's entry answered, w.mu held: its bus state is waited on from the take-in's time, and the entry +// is forgotten. +func (w *Watchman) entryRead(fingerprint string, e meshcatalog.CatalogEntry) { + m, ok := w.movers[fingerprint] + if !ok { + return + } + delete(w.movers, fingerprint) + defer w.entries.Release(fingerprint) + asked, err := BusStateOfEntry(e) + if err != nil { + w.say("the catalog entry of %s at %s does not read (%v), so its bus state is not waited on", m.module, short(fingerprint), err) + return + } + w.busStateAsked(m.module, fingerprint, asked, m.at) +} + +// BusStateOfEntry is the bus state a catalog entry's manifest asks for: the stored manifest keeps it beside requires, under +// catalogue.BusStateField. +func BusStateOfEntry(e meshcatalog.CatalogEntry) ([]catalogue.BusStateEntry, error) { + raw, err := json.Marshal(e.Manifest) + if err != nil { + return nil, err + } + var m map[string]json.RawMessage + if err := json.Unmarshal(raw, &m); err != nil { + return nil, err + } + var entries []catalogue.BusStateEntry + if b, ok := m[catalogue.BusStateField]; ok { + if err := json.Unmarshal(b, &entries); err != nil { + return nil, err + } + } + return entries, nil +} + +// busStateAsked is a taken-in mover's bus state, w.mu held: each entry it asks for waits OverdueBound from the take-in +// for the bus module's word, and every entry of the module this watchman waits on or holds a finding of that it no +// longer asks for is ended. +func (w *Watchman) busStateAsked(module, fingerprint string, entries []catalogue.BusStateEntry, at time.Time) { asked := map[string]bool{} - for _, e := range a.BusState { + for _, e := range entries { if e.Name == "" { continue } asked[e.Name] = true - if w.busCreated[signalKey(rowBusCreated, busStateID(a.Module, e.Name))] { + if w.busCreated[signalKey(rowBusCreated, busStateID(module, e.Name))] { continue // said created already, and created is said once per entry: nothing to wait for } - // One refused since an earlier verdict waits again: a new verdict asks the bus module again. - w.signal(rowBusState, busStateID(a.Module, e.Name), OverdueBound, signalDue{Do: dueFind, Finding: Finding{ - Scope: ScopeModule, ID: a.Module, Token: KindBusStateOverdue + "-" + e.Name, Kind: KindBusStateOverdue, - Severity: Warning, Source: rowBusState, Since: w.now().Add(OverdueBound).UTC().Truncate(time.Second), - Summary: fmt.Sprintf("the bus module said nothing of %s's %s (%s) within %s of its accepted manifest %s", a.Module, - e.Name, e.Storage, OverdueBound, short(a.Digest)), - Headline: "The bus state " + e.Name + " of " + a.Module + " was not created", - Explanation: "The indexer accepted a manifest of " + a.Module + " asking the bus module for " + e.Name + ", and the bus " + - "module said neither created nor refused within " + OverdueBound.String() + ": it is down, or behind. " + a.Module + + // One refused since an earlier take-in waits again: a new take-in asks the bus module again. + f := Finding{ + Scope: ScopeModule, ID: module, Token: KindBusStateOverdue + "-" + e.Name, Kind: KindBusStateOverdue, + Severity: Warning, Source: rowBusState, Since: at.Add(OverdueBound).UTC().Truncate(time.Second), + Summary: fmt.Sprintf("the bus module said nothing of %s's %s (%s) within %s of its take-in at %s", module, + e.Name, e.Storage, OverdueBound, short(fingerprint)), + Headline: "The bus state " + e.Name + " of " + module + " was not created", + Explanation: "The indexer took in " + module + ", whose manifest asks the bus module for " + e.Name + ", and the bus " + + "module said neither created nor refused within " + OverdueBound.String() + ": it is down, or behind. " + module + " fails at its start while the entry's binding is missing (novox/hq ADR 0595 §1).", - Needs: "a look at the bus module's health and its process's log; nats_first_view_again asks it to read the verdicts again"}}) + Needs: "a look at the bus module's health and its process's log; nats_first_view_again asks it to read the entries again"} + if left := OverdueBound - w.now().Sub(at); left > 0 { + w.signal(rowBusState, busStateID(module, e.Name), left, signalDue{Do: dueFind, Finding: f}) + } else { + w.unsignal(rowBusState, busStateID(module, e.Name)) + w.find(f) + } } - for _, local := range w.busStateEntriesOf(a.Module) { + for _, local := range w.busStateEntriesOf(module) { if asked[local] { continue } - w.unsignal(rowBusState, busStateID(a.Module, local)) - w.unfind(overdueKey(a.Module, local)) - w.unfind(refusedKey(a.Module, local)) - w.uncreated(a.Module, local) + w.unsignal(rowBusState, busStateID(module, local)) + w.unfind(overdueKey(module, local)) + w.unfind(refusedKey(module, local)) + w.uncreated(module, local) } } diff --git a/modules/mesh-delivery-watchman/internal/watch/findings_test.go b/modules/mesh-delivery-watchman/internal/watch/findings_test.go index 896c8807..33d7ce1f 100644 --- a/modules/mesh-delivery-watchman/internal/watch/findings_test.go +++ b/modules/mesh-delivery-watchman/internal/watch/findings_test.go @@ -1,8 +1,11 @@ package watch import ( + "context" "encoding/json" "errors" + "fmt" + "slices" "strings" "sync/atomic" "testing" @@ -12,7 +15,6 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein" "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient/heard" "git.novox.be/novox/mesh-contracts/provisioning/go/clients/meshprovisionerclient" @@ -42,181 +44,116 @@ func TestTheWatchmansFindings(t *testing.T) { // want are the finding keys standing at the end, with their severity. want map[string]string }{ - {"ADR 0633 §5 (1585/T21): a revision waiting for an indexer build is a finding past 60 minutes from its since, not within; a group past 30", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { + {"design 47 H §744, H-A-94 (2): a held take-in is a finding past take-in-bound from its message's time, not within; the setting counts", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { w.TakeInCaughtUp() // a live message, heard after the replay - w.TakeIn(takeInEvent(t, takein.TakeInPending, revKey), takeInBody(t, takein.TakeInPending, revKey, t0)) - if o.ttls[signalKey("take-in", revKey)] != TakeInRevisionBound || len(o.findings) != 0 { - t.Fatalf("a revision's wait: ttls %v, findings %v", o.ttls, o.findings) + w.TakeIn(TakeInPendingEvent, pendingBody(t, "main", 42, "the catalog store is full"), t0) + if o.ttls[signalKey(takeInRow, revKey)] != DefaultTakeInBound || len(o.findings) != 0 { + t.Fatalf("a held take-in: ttls %v, findings %v", o.ttls, o.findings) } - w.TakeIn(takeInEvent(t, takein.TakeInPending, groupKey), takeInBody(t, takein.TakeInPending, groupKey, t0)) - if o.ttls[signalKey("take-in", groupKey)] != TakeInGroupBound { - t.Fatalf("a group's wait: ttls %v", o.ttls) + w.SetTakeInBound(20 * time.Minute) + w.TakeIn(TakeInPendingEvent, pendingBody(t, "feat/x", 7, ""), c.t.Add(-5*time.Minute)) + if o.ttls[signalKey(takeInRow, TakeInKey("feat/x", 7))] != 15*time.Minute { + t.Fatalf("a take-in held five minutes under a 20-minute bound: ttls %v", o.ttls) } - expire(w, c, signalKey("take-in", groupKey), TakeInGroupBound) - if len(o.findings) != 1 { - t.Fatalf("findings at 30 minutes: %v", o.findings) + expire(w, c, signalKey(takeInRow, revKey), DefaultTakeInBound) + if f := o.findings[FindingKey(ScopeDelivery, revKey, KindTakeInWaits)]; !strings.Contains(f.Summary, "the catalog store is full") { + t.Fatalf("the finding does not say why: %v", o.findings) } - expire(w, c, signalKey("take-in", revKey), TakeInRevisionBound-TakeInGroupBound) - }, map[string]string{FindingKey(ScopeDelivery, revKey, KindTakeInWaits): Warning, FindingKey(ScopeDelivery, groupKey, KindTakeInWaits): Warning}}, - {"ADR 0633 §5 (1585/T21): the take-in.taken of a named indexer, a hand take-in or a drop clears the finding", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { + }, map[string]string{FindingKey(ScopeDelivery, revKey, KindTakeInWaits): Warning}}, + {"design 47 H §744: the take-in.taken of the source, the indexer's, a hand take-in's or a skip's, clears the finding", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { w.TakeInCaughtUp() // a live message, heard after the replay - for _, by := range []string{"", "operator@shanks"} { - w.TakeIn(takeInEvent(t, takein.TakeInPending, revKey), takeInBody(t, takein.TakeInPending, revKey, t0)) - expire(w, c, signalKey("take-in", revKey), TakeInRevisionBound) + for _, taken := range []meshcatalog.TakeInTaken{{Branch: "main", Source: 42, Call: "fedcba9876543210", Revision: 3}, + {Branch: "main", Source: 42, By: "operator@shanks", Why: "the store has room", Skipped: true}} { + w.TakeIn(TakeInPendingEvent, pendingBody(t, "main", 42, ""), t0) + expire(w, c, signalKey(takeInRow, revKey), DefaultTakeInBound) if len(o.findings) != 1 { t.Fatalf("findings past the bound: %v", o.findings) } - taken, err := takein.EncodeTakeIn(takein.TakeIn{Kind: takein.TakeInTaken, Key: revKey, What: takein.TakeInRevision, Since: t0, - Merges: []takein.Merge{{Repository: "novox/a", Merge: "c0ffee"}}, By: by, Why: map[bool]string{true: "dropped", false: ""}[by != ""], Dropped: by != ""}) - if err != nil { - t.Fatal(err) - } - w.TakeIn(takeInEvent(t, takein.TakeInTaken, revKey), taken) + body, _ := json.Marshal(taken) + w.TakeIn(TakeInTakenEvent, body, c.t) c.t = t0 } }, map[string]string{}}, - {"ADR 0633 §5 (1585/T21): a group's take-in.taken ends its 30 minutes, and the revision then pending under m. starts its own 60", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { - w.TakeInCaughtUp() // a live message, heard after the replay - w.TakeIn(takeInEvent(t, takein.TakeInPending, groupKey), takeInBody(t, takein.TakeInPending, groupKey, t0)) - expire(w, c, signalKey("take-in", groupKey), 25*time.Minute) - w.TakeIn(takeInEvent(t, takein.TakeInTaken, groupKey), takeInBody(t, takein.TakeInTaken, groupKey, t0)) - w.TakeIn(takeInEvent(t, takein.TakeInPending, revKey), takeInBody(t, takein.TakeInPending, revKey, c.t)) - if len(o.findings) != 0 || o.ttls[signalKey("take-in", revKey)] != TakeInRevisionBound { - t.Fatalf("after the hand-over: findings %v, ttls %v", o.findings, o.ttls) - } - expire(w, c, signalKey("take-in", revKey), TakeInRevisionBound) - }, map[string]string{FindingKey(ScopeDelivery, revKey, KindTakeInWaits): Warning}}, - {"ADR 0633 §5 (1585/T21): a pending event heard again at the start, since past its bound, is said once the replay has caught up, not before", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { - w.TakeIn(takeInEvent(t, takein.TakeInPending, revKey), takeInBody(t, takein.TakeInPending, revKey, t0.Add(-2*time.Hour))) + {"design 47 H §744: a pending heard again at the start, past its bound, is said once the replay has caught up, not before; one followed by its taken never raises", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { + w.TakeIn(TakeInPendingEvent, pendingBody(t, "main", 42, ""), t0.Add(-2*time.Hour)) + w.TakeIn(TakeInPendingEvent, pendingBody(t, "main", 43, ""), t0.Add(-2*time.Hour)) + w.TakeIn(TakeInTakenEvent, takenBody(t, "main", 43), t0.Add(-2*time.Hour)) if len(o.findings) != 0 || len(o.pulses) != 0 { - t.Fatalf("a replayed old wait before the replay caught up: findings %v, pulses %v", o.findings, o.pulses) + t.Fatalf("replayed old waits before the replay caught up: findings %v, pulses %v", o.findings, o.pulses) } w.TakeInCaughtUp() }, map[string]string{FindingKey(ScopeDelivery, revKey, KindTakeInWaits): Warning}}, - {"1966/AC195-5: a seed ends the finding and the held wait of a key indexer_pending no longer names, and keeps one it names with its own since", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { - // A previous run's finding for revKey stands in the bucket (read at the start); a wait of staleKey is held from an event. - old := Finding{Scope: ScopeDelivery, ID: revKey, Kind: KindTakeInWaits, Severity: Warning, Source: "take-in", Since: t0.Add(-3 * time.Hour), Observer: testModule, Summary: "held", Said: "held", Headline: "held", Explanation: "held", Needs: "a take-in"} + {"design 47 H-A-94 (2), H-A-97: a seed from indexer_pending ends the finding and the wait of a key it no longer names, and times one it names from its own since", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { + // A previous run's finding for main.41 stands in the bucket (read at the start); a wait of main.40 is held from an event. + stale := TakeInKey("main", 41) + old := Finding{Scope: ScopeDelivery, ID: stale, Kind: KindTakeInWaits, Severity: Warning, Source: takeInRow, Since: t0.Add(-3 * time.Hour), Observer: testModule, Summary: "held", Said: "held", Headline: "held", Explanation: "held", Needs: "a take-in"} value, err := EncodeFinding(old) if err != nil { t.Fatal(err) } o.findings[old.Key()] = old w.SeedFinding(old.Key(), value) - const staleKey = "m.00000000000000000099" w.TakeInCaughtUp() - w.TakeIn(takeInEvent(t, takein.TakeInPending, staleKey), takeInBody(t, takein.TakeInPending, staleKey, c.t)) - w.SeedTakeIn(takein.PendingAnswer{Waits: []takein.PendingWait{{Key: groupKey, What: takein.TakeInGroup, Keys: []string{groupKey}, - Merges: []takein.Merge{{Repository: "novox/a", Number: 1, Merge: "c0ffee"}}, Since: c.t.Add(-10 * time.Minute)}}}) - if _, ok := o.findings[old.Key()]; ok || w.signals[signalKey("take-in", staleKey)].Finding.ID != "" || w.signals[signalKey("take-in", revKey)].Finding.ID != "" { - t.Fatalf("a wait the answer does not name stands: findings %v, signals %v", o.findings, w.signals) - } - if o.ttls[signalKey("take-in", groupKey)] != TakeInGroupBound-10*time.Minute { - t.Fatalf("the named wait: ttl %v, want the rest of its bound from its own since", o.ttls[signalKey("take-in", groupKey)]) - } - }, map[string]string{}}, - {"1966/AC195-4: a taken event for a key the watchman holds no wait for changes no finding and no signal", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { - w.TakeInCaughtUp() - w.TakeIn(takeInEvent(t, takein.TakeInPending, groupKey), takeInBody(t, takein.TakeInPending, groupKey, t0)) - ttls, pulses := len(o.ttls), len(o.pulses) - w.TakeIn(takeInEvent(t, takein.TakeInTaken, revKey), takeInBody(t, takein.TakeInTaken, revKey, t0)) - if o.findingWrites != 0 || len(o.ttls) != ttls || len(o.pulses) != pulses || o.ttls[signalKey("take-in", groupKey)] != TakeInGroupBound { - t.Fatalf("a taken for an unheld key: findings written %d, ttls %v, pulses %v", o.findingWrites, o.ttls, o.pulses) - } - }, map[string]string{}}, - {"ADR 0633 §5 (1585/T21): a replayed pending followed by its taken never raises", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { - w.TakeIn(takeInEvent(t, takein.TakeInPending, revKey), takeInBody(t, takein.TakeInPending, revKey, t0.Add(-2*time.Hour))) - w.TakeIn(takeInEvent(t, takein.TakeInTaken, revKey), takeInBody(t, takein.TakeInTaken, revKey, t0.Add(-2*time.Hour))) - w.TakeInCaughtUp() - if o.findingWrites != 0 || len(o.pulses) != 0 { - t.Fatalf("a wait that ended in the replay wrote %d findings and left pulses %v", o.findingWrites, o.pulses) - } - }, map[string]string{}}, - {"ADR 0633 §5 (1585/T21): a wait the indexer holds at a restart within its bound is seeded from indexer_pending and raised past it", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { - // The restarted watchman's durable consumer hears nothing of the pending event it acknowledged before; the wait - // began ten minutes ago, and the indexer still holds it. - since := c.t.Add(-10 * time.Minute) - w.SeedTakeIn(takein.PendingAnswer{Waits: []takein.PendingWait{{Key: revKey, What: takein.TakeInRevision, Keys: []string{revKey}, - Merges: []takein.Merge{{Repository: "novox/a", Number: 1, Merge: "c0ffee"}}, Lost: []string{"m.00000000000000000042"}, Since: since}}}) - w.TakeInCaughtUp() - if len(o.findings) != 0 || o.ttls[signalKey("take-in", revKey)] != TakeInRevisionBound-10*time.Minute { - t.Fatalf("within its bound: findings %v, ttl %v; want none yet and the rest of the bound", o.findings, o.ttls[signalKey("take-in", revKey)]) - } - expire(w, c, signalKey("take-in", revKey), TakeInRevisionBound-10*time.Minute) - }, map[string]string{FindingKey(ScopeDelivery, revKey, KindTakeInWaits): Warning}}, - {"ADR 0633 §1 (issue 1921): (A-476 b) a revision held because its message is lost or unread is raised past the revision bound from its since, naming why", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { - w.TakeInCaughtUp() - body, err := takein.EncodeTakeIn(takein.TakeIn{Kind: takein.TakeInPending, Key: revKey, What: takein.TakeInRevision, Since: c.t, - Merges: []takein.Merge{{Repository: "novox/a", Number: 1, Merge: "c0ffee"}}, Lost: []string{revKey}}) + w.TakeIn(TakeInPendingEvent, pendingBody(t, "main", 40, ""), c.t) + seed, err := DecodeHeldTakeIns([]byte(`{"held":[{"branch":"main","source":42,"why":"the catalog store is full","since":"` + c.t.Add(-10*time.Minute).Format(time.RFC3339) + `"}]}`)) if err != nil { t.Fatal(err) } - w.TakeIn(takeInEvent(t, takein.TakeInPending, revKey), body) - if o.ttls[signalKey("take-in", revKey)] != TakeInRevisionBound || len(o.findings) != 0 { - t.Fatalf("a lost message: ttls %v, findings %v", o.ttls, o.findings) + w.SeedTakeIn(seed) + if _, ok := o.findings[old.Key()]; ok || w.signals[signalKey(takeInRow, TakeInKey("main", 40))].Finding.ID != "" { + t.Fatalf("a wait the answer does not name stands: findings %v, signals %v", o.findings, w.signals) } - expire(w, c, signalKey("take-in", revKey), TakeInRevisionBound) - for _, f := range o.findings { - if !strings.Contains(f.Summary, "lost or unread") { - t.Errorf("finding %q does not say why", f.Summary) - } + if o.ttls[signalKey(takeInRow, revKey)] != DefaultTakeInBound-10*time.Minute { + t.Fatalf("the named wait: ttl %v, want the rest of its bound from its own since", o.ttls[signalKey(takeInRow, revKey)]) + } + expire(w, c, signalKey(takeInRow, revKey), DefaultTakeInBound-10*time.Minute) + if _, err := DecodeHeldTakeIns([]byte(`{"held":[{"branch":"main","source":42,"why":"w"}]}`)); err == nil { + t.Error("a held take-in with no since was read") } }, map[string]string{FindingKey(ScopeDelivery, revKey, KindTakeInWaits): Warning}}, - {"ADR 0633 §5 (1585/T21): a drop clears the wait, and the merges left under it said pending again with the same since are raised from it", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { - w.TakeInCaughtUp() // a live message, heard after the replay - w.TakeIn(takeInEvent(t, takein.TakeInPending, revKey), takeInBody(t, takein.TakeInPending, revKey, t0)) - expire(w, c, signalKey("take-in", revKey), TakeInRevisionBound) - if len(o.findings) != 1 { - t.Fatalf("findings past the bound: %v", o.findings) + {"1966/AC195-4: a taken event for a key the watchman holds no wait for changes no finding and no signal", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { + w.TakeInCaughtUp() + w.TakeIn(TakeInPendingEvent, pendingBody(t, "main", 42, ""), t0) + ttls, pulses := len(o.ttls), len(o.pulses) + w.TakeIn(TakeInTakenEvent, takenBody(t, "main", 9), t0) + if o.findingWrites != 0 || len(o.ttls) != ttls || len(o.pulses) != pulses || o.ttls[signalKey(takeInRow, revKey)] != DefaultTakeInBound { + t.Fatalf("a taken for an unheld key: findings written %d, ttls %v, pulses %v", o.findingWrites, o.ttls, o.pulses) } - w.TakeIn(takeInEvent(t, takein.TakeInTaken, revKey), takeInBody(t, takein.TakeInTaken, revKey, t0)) - if len(o.findings) != 0 { - t.Fatalf("the drop left %v", o.findings) - } - next := "m.00000000000000000043" - w.TakeIn(takeInEvent(t, takein.TakeInPending, next), takeInBody(t, takein.TakeInPending, next, t0)) - }, map[string]string{FindingKey(ScopeDelivery, "m.00000000000000000043", KindTakeInWaits): Warning}}, - {"design 47 G §500: manifest-refused raises the refusal, manifest-accepted clears it and carries an outgrown manifest's problems", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { - w.ManifestRefused(meshcatalog.ManifestRefusedEvent{Module: "nats", Problems: []string{"removes a stream"}}) - w.ManifestRefused(meshcatalog.ManifestRefusedEvent{Module: "power", Problems: []string{"shows nothing"}}) - k := FindingKey(ScopeModule, "nats", KindManifestRefused) - if f := o.findings[k]; !strings.Contains(f.Summary, "refused the manifest of nats: removes a stream") { - t.Fatalf("findings = %v", o.findings) - } - w.Accepted(meshcatalog.ManifestAcceptedEvent{Module: "nats", Digest: strings.Repeat("d", 64)}) - w.Accepted(meshcatalog.ManifestAcceptedEvent{Module: "power", Digest: strings.Repeat("d", 64)}) - // ADR 0579 §1: an accepted manifest a stricter judge outgrew is a warning until mended. - w.Accepted(meshcatalog.ManifestAcceptedEvent{Module: "node-tools", Digest: strings.Repeat("d", 64), Problems: []string{"says no credential form"}}) - if f := o.findings[FindingKey(ScopeModule, "node-tools", KindManifestOutgrown)]; f.Severity != Warning || !strings.Contains(f.Summary, "says no credential form") { - t.Fatalf("findings = %v", o.findings) - } - w.Accepted(meshcatalog.ManifestAcceptedEvent{Module: "node-tools", Digest: strings.Repeat("e", 64), Problems: []string{}}) - // The start seeds from the indexer's verdicts: the newest of each module counts. - w.SeedVerdicts([]VerdictRow{ - {Module: "nats", State: meshcatalogclient.VerdictRefused, Problems: []string{"old"}, At: t0}, - {Module: "nats", State: meshcatalogclient.VerdictAccepted, At: t0.Add(time.Hour)}, - {Module: "power", State: meshcatalogclient.VerdictRefused, Problems: []string{"shows nothing"}, At: t0}, - {Module: "node-tools", State: meshcatalogclient.VerdictAccepted, Warnings: []string{"w"}, At: t0}}) - if len(o.findings) != 2 || o.findings[FindingKey(ScopeModule, "power", KindManifestRefused)].Kind == "" || o.findings[FindingKey(ScopeModule, "node-tools", KindManifestOutgrown)].Kind == "" { - t.Fatalf("findings = %v", o.findings) - } - w.Accepted(meshcatalog.ManifestAcceptedEvent{Module: "power"}) - w.Accepted(meshcatalog.ManifestAcceptedEvent{Module: "node-tools"}) }, map[string]string{}}, - {"DLV-W1, 1966/AC37-AC38 (ADR 0595 §4, design 25 § 12.2): an accepted entry the bus module says nothing of in five minutes is overdue until created; a refused one stands until created; a later verdict no longer asking ends both", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { - accepted := func(locals ...string) meshcatalog.ManifestAcceptedEvent { - a := meshcatalog.ManifestAcceptedEvent{Module: "mesh-knowledge", Digest: strings.Repeat("d", 64)} + {"DLV-W1, 1966/AC37-AC38 (ADR 0595 §4), design 47 H-A-94 (1): an entry a taken-in mover's catalog entry asks for that the bus module says nothing of in five minutes is overdue until created; a refused one stands until created; a later take-in no longer asking ends both", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { + entries := &fakeEntries{entries: map[string]meshcatalog.CatalogEntry{}, answer: true} + w.ReadEntriesWith(entries) + source := int64(0) + accepted := func(locals ...string) { + source++ + fp := fmt.Sprintf("src2:%064d", source) + var bus []map[string]string for _, l := range locals { - a.BusState = append(a.BusState, meshcatalog.ManifestAcceptedEventBusStateItem{Name: l, Storage: "ledger"}) + bus = append(bus, map[string]string{"name": l, "storage": "ledger"}) } - return a + entries.entries[fp] = meshcatalog.CatalogEntry{Module: "mesh-knowledge", Fp: fp, Manifest: map[string]any{"module": "mesh-knowledge", "bus-state": bus}} + w.TakeIn(TakeInTakenEvent, takenBody(t, "main", source, meshcatalog.TakeInEntry{Module: "mesh-knowledge", Fingerprint: fp, Commit: "c0ffee", Repository: "novox/mesh-knowledge"}), c.t) + } + // The first entry is asked and answered after: its wait is timed from the take-in, not from the answer. + entries.answer = false + accepted("records", "drafts", "kept") + if len(o.ttls) != 0 || len(entries.released) != 0 { + t.Fatalf("a wait before the entry was answered: %v", o.ttls) + } + fp1 := fmt.Sprintf("src2:%064d", 1) + e1 := entries.entries[fp1] + c.t = c.t.Add(time.Minute) + w.EntryChanged(meshcatalogclient.EntryRow{Fingerprint: fp1, State: meshcatalogclient.EntryAnswered, Entry: &e1}) + entries.answer = true + if !slices.Equal(entries.released, []string{fp1}) { + t.Fatalf("the entry read is not released: %v", entries.released) } - w.Accepted(accepted("records", "drafts", "kept")) pk := func(local string) string { return signalKey(rowBusState, busStateID("mesh-knowledge", local)) } - if o.ttls[pk("records")] != OverdueBound { - t.Fatalf("no five-minute pulse for records: %v", o.ttls) + if o.ttls[pk("records")] != OverdueBound-time.Minute { + t.Fatalf("no pulse for records within five minutes of its take-in: %v", o.ttls) } w.BrokerCreated(BrokerSaid{Module: "mesh-knowledge", Local: "drafts"}) - expire(w, c, pk("drafts"), 5*time.Minute) // created in time: no finding + expire(w, c, pk("drafts"), 4*time.Minute) // created in time: no finding expire(w, c, pk("records"), 0) overdue := o.findings[overdueKey("mesh-knowledge", "records")] if overdue.Kind != KindBusStateOverdue || !strings.Contains(overdue.Summary, "mesh-knowledge's records") { @@ -226,13 +163,13 @@ func TestTheWatchmansFindings(t *testing.T) { if _, open := o.findings[overdueKey("mesh-knowledge", "records")]; open { t.Fatal("created did not clear overdue") } - // Review round 1: created is said once per entry, so a second accepted manifest with the same entries waits on + // Review round 1: created is said once per entry, so a second take-in with the same entries waits on // none of them, and the mark that says so has no time to live. delete(o.pulses, pk("records")) // expired above; the fake keeps an expired pulse - w.Accepted(accepted("records", "drafts")) + accepted("records", "drafts") for _, local := range []string{"records", "drafts"} { if _, waits := o.pulses[pk(local)]; waits { - t.Fatalf("a second accepted manifest waits again on %s, said created already", local) + t.Fatalf("a second take-in waits again on %s, said created already", local) } if mk := signalKey(rowBusCreated, busStateID("mesh-knowledge", local)); o.ttls[mk] != 0 || o.pulses[mk].Owes == "" { t.Fatalf("the mark of %s created is %v with time to live %v", local, o.pulses[mk], o.ttls[mk]) @@ -246,9 +183,9 @@ func TestTheWatchmansFindings(t *testing.T) { if !signalKind(mustEncode(t, o.findings[refusedKey("mesh-knowledge", "kept")])) { t.Error("a refused creation would be dropped at a restart") } - // A later verdict asking for kept alone again waits on it again; one no longer asking for it ends it. - w.Accepted(accepted("kept", "fresh")) - w.Accepted(accepted("fresh")) + // A later take-in asking for kept again waits on it again; one no longer asking for it ends it. + accepted("kept", "fresh") + accepted("fresh") if _, open := o.findings[refusedKey("mesh-knowledge", "kept")]; open { t.Fatal("a verdict no longer asking for kept left its finding") } @@ -257,7 +194,7 @@ func TestTheWatchmansFindings(t *testing.T) { } w.BrokerCreated(BrokerSaid{Module: "mesh-knowledge", Local: "fresh"}) // A refused create stands until created. - w.Accepted(accepted("gone")) + accepted("gone") w.BrokerRefused(BrokerSaid{Module: "mesh-knowledge", Local: "gone"}) }, map[string]string{refusedKey("mesh-knowledge", "gone"): Urgent}}, {"ADR 0615 §5 (review round 1): a consumer the watchman cannot bind is family-not-heard naming its stream, until it binds", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { @@ -493,38 +430,39 @@ func TestTheWatchmansFindings(t *testing.T) { w.DeadLettersHeld(map[string]int{"SEAT_X.y_worker": 1}, who) w.NoticesWaiting(true) }, map[string]string{"bus.SEAT_X.y_worker.max-deliveries": Warning, "bus.DEAD_LETTER_NOTICES.controller.notices-waiting": Warning}}, - {"ADR 0603 §8: a want held by a rule and a provision asked each raise a finding naming the wait past their bound, and clear when it ends", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { + {"design 47 H-A-95 (1): a want awaiting its hand placement is not timed, and is timed once it no longer holds; ADR 0603 §8: a provision asked raises a finding naming the wait past its bound, and clears when it ends", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { w.SetStepBound(30 * time.Minute) - held, _ := meshwantsclient.EncodeWant(meshwantsclient.Want{Fp: fpOld, Self: true, Newest: fpNew, Rule: meshwantsclient.RuleCanary, Words: "the canary ace does not run it yet"}) - released, _ := meshwantsclient.EncodeWant(meshwantsclient.Want{Fp: fpNew, Self: true}) + held, err := meshwantsclient.EncodeWant(meshwantsclient.Want{Fp: fpOld, Hold: true, Words: meshwantsclient.HoldAwaitsHandPlacement}) + if err != nil { + t.Fatal(err) + } + released, err := meshwantsclient.EncodeWant(meshwantsclient.Want{Fp: fpNew, Pipeline: "main.42", Rank: 1}) + if err != nil { + t.Fatal(err) + } asked, _ := meshprovisionerclient.Encode(meshprovisionerclient.Provision{Provision: "oidc", Consumer: meshprovisionerclient.End{Module: "photos", Node: "ace"}, Provider: meshprovisionerclient.End{Module: "keycloak", Node: "novox"}, State: meshprovisionerclient.Asked, At: t0}) provided, _ := meshprovisionerclient.Encode(meshprovisionerclient.Provision{Provision: "oidc", Consumer: meshprovisionerclient.End{Module: "photos", Node: "ace"}, Provider: meshprovisionerclient.End{Module: "keycloak", Node: "novox"}, State: meshprovisionerclient.Provided, At: t0}) - w.EngineRunning("ace", "web", []byte(`{"fingerprint":"`+fpOld+`"}`)) // the node runs what the held want names: the hold is still the wait - w.Want("want.ace.web", held, Put, t0) + w.Want("want.ace.web", held, Put, c.t) w.Provision("ace.photos.oidc", asked, Put) - waits := map[string]string{AssignerWantsBucket: "want.ace.web", ProvisionsBucket: "ace.photos.oidc"} - if len(o.pulses) != 2 || len(o.findings) != 0 { - t.Fatalf("pulses %v findings %v", o.pulses, o.findings) + if _, timed := o.pulses[pulseKeyOf(t, testWants(), "want.ace.web")]; timed || len(o.pulses) != 1 || len(o.stuck) != 0 || len(o.findings) != 0 { + t.Fatalf("a held want is timed: pulses %v stuck %v findings %v", o.pulses, o.stuck, o.findings) } - // The keys are written again as their parts move on, with new words and a new time: the wait still began at t0. + // The provision is written again as it moves on, with new words and a new time: the wait still began at t0. c.t = c.t.Add(20 * time.Minute) - held2, _ := meshwantsclient.EncodeWant(meshwantsclient.Want{Fp: fpOld, Self: true, Newest: fpNew, Rule: meshwantsclient.RuleCanary, Words: "the canary ace runs it, not healthy yet"}) asked2, _ := meshprovisionerclient.Encode(meshprovisionerclient.Provision{Provision: "oidc", Consumer: meshprovisionerclient.End{Module: "photos", Node: "ace"}, Provider: meshprovisionerclient.End{Module: "keycloak", Node: "novox"}, State: meshprovisionerclient.Asked, Why: "the provider is applying", At: c.t}) - w.Want("want.ace.web", held2, Put, c.t) w.Provision("ace.photos.oidc", asked2, Put) - for bucket, key := range waits { - expire(w, c, pulseKeyOf(t, bucket, key), 31*time.Minute) - } - for bucket, key := range waits { - f := o.findings[FindingKey(ScopeDelivery, stuckKeyOf(t, bucket, key), KindStuck)] - if f.Kind != KindStuck || !f.Since.Equal(t0) || !strings.Contains(f.Summary, "bound 30m0s") { - t.Fatalf("%s: findings = %v", bucket, o.findings) - } + expire(w, c, pulseKeyOf(t, ProvisionsBucket, "ace.photos.oidc"), 31*time.Minute) + f := o.findings[FindingKey(ScopeDelivery, stuckKeyOf(t, ProvisionsBucket, "ace.photos.oidc"), KindStuck)] + if f.Kind != KindStuck || !f.Since.Equal(t0) || !strings.Contains(f.Summary, "bound 30m0s") { + t.Fatalf("findings = %v", o.findings) } w.Want("want.ace.web", released, Put, c.t) + if _, timed := o.pulses[pulseKeyOf(t, testWants(), "want.ace.web")]; !timed { + t.Fatalf("a want no longer held is not timed: pulses %v", o.pulses) + } w.Provision("ace.photos.oidc", provided, Put) }, map[string]string{}}, } @@ -551,37 +489,45 @@ func TestTheWatchmansFindings(t *testing.T) { } } -const ( - revKey = "m.00000000000000000042" - groupKey = "g.feat/take-in" -) +// revKey is the take-in wait's key of main's source revision 42 (design 47 H-A-94 (2)). +var revKey = TakeInKey("main", 42) -func takeInEvent(t *testing.T, kind, key string) string { +// pendingBody is a take-in.pending's body, in the contract's form. +func pendingBody(t *testing.T, branch string, source int64, why string) []byte { t.Helper() - e, err := takein.TakeInEvent(kind) - if err != nil { - t.Fatal(err) - } - return e -} - -// takeInBody is a take-in event's body for a key: a group's when it starts with g. -func takeInBody(t *testing.T, kind, key string, since time.Time) []byte { - t.Helper() - body := takein.TakeIn{Kind: kind, Key: key, What: takein.TakeInRevision, Since: since, - Merges: []takein.Merge{{Repository: "novox/a", Number: 1, Merge: "c0ffee"}}} - if key[0] == 'g' { - body.What, body.Group = takein.TakeInGroup, "feat/take-in" - } else if kind == takein.TakeInPending { - body.Lost = []string{key} - } - b, err := takein.EncodeTakeIn(body) + b, err := json.Marshal(meshcatalog.TakeInPending{Branch: branch, Source: source, Why: why}) if err != nil { t.Fatal(err) } return b } +// takenBody is a take-in.taken's body naming its movers, in the contract's form. +func takenBody(t *testing.T, branch string, source int64, movers ...meshcatalog.TakeInEntry) []byte { + t.Helper() + b, err := json.Marshal(meshcatalog.TakeInTaken{Branch: branch, Source: source, Call: "fedcba9876543210", Revision: source, Entries: movers}) + if err != nil { + t.Fatal(err) + } + return b +} + +// fakeEntries answers a mover's catalog entry by fingerprint: at once when answer is set, else asked. +type fakeEntries struct { + entries map[string]meshcatalog.CatalogEntry + answer bool + released []string +} + +func (f *fakeEntries) Need(_ context.Context, fp string) (meshcatalog.CatalogEntry, meshcatalogclient.EntryState) { + if e, ok := f.entries[fp]; ok && f.answer { + return e, meshcatalogclient.EntryAnswered + } + return meshcatalog.CatalogEntry{}, meshcatalogclient.EntryAsked +} + +func (f *fakeEntries) Release(fp string) { f.released = append(f.released, fp) } + // Design 47 G §689, A-14, A-22: a token other than the kind goes into the contract finding's id and reads back as it was // written; the finding's time is its `since`; the resolved words are not on the wire. func TestAFindingIsTheContractsWithItsRefInItsID(t *testing.T) { diff --git a/modules/mesh-delivery-watchman/internal/watch/mirror.go b/modules/mesh-delivery-watchman/internal/watch/mirror.go index af76a4b1..0fba92cb 100644 --- a/modules/mesh-delivery-watchman/internal/watch/mirror.go +++ b/modules/mesh-delivery-watchman/internal/watch/mirror.go @@ -7,7 +7,6 @@ import ( "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" - "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshwatchdogsclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo" @@ -124,7 +123,6 @@ func (w *Watchman) Want(key string, value []byte, op Op, at time.Time) { return case Delete: delete(w.wants, key) - delete(w.held, key) w.moved(w.wantsBucket(), key) return } @@ -136,30 +134,16 @@ func (w *Watchman) Want(key string, value []byte, op Op, at time.Time) { } if want.Removed { delete(w.wants, key) - delete(w.held, key) + w.moved(w.wantsBucket(), key) + return + } + if want.Hold { + // A want awaiting its hand placement (design 47 H §772) is not progress owed, so it is not timed (H-A-95 (1)). + delete(w.wants, key) w.moved(w.wantsBucket(), key) return } w.wants[key] = want.Fp - if want.Newest != "" || want.Rule != 0 { - // A want held by a rule (ADR 0603 §8) waits on the rule, not on the node: its wait is the hold's, from the time the - // assigner wrote it, and the node running what the want names does not end it. - if at.IsZero() { - at = w.now() - } - w.held[key] = true - w.mirrorKept(w.wantsBucket(), key, - describe("the hold on %s on %s at %s (rule %d) to end, bound %s: %s", module, node, shortFP(want.Fp), want.Rule, w.stepBound, want.Words), - "held|"+want.Fp+"|"+want.Newest, at) - return - } - if w.held[key] { - delete(w.held, key) - if pk, err := meshwatchdogsclient.PulseKey(w.wantsBucket(), key); err == nil { - delete(w.starts, pk) - } - w.stuckMoved(w.wantsBucket(), key) // the hold ended: its finding is cleared - } if w.have(node, module) == want.Fp { w.moved(w.wantsBucket(), key) return diff --git a/modules/mesh-delivery-watchman/internal/watch/names.go b/modules/mesh-delivery-watchman/internal/watch/names.go index 3ed7e271..1f85ef71 100644 --- a/modules/mesh-delivery-watchman/internal/watch/names.go +++ b/modules/mesh-delivery-watchman/internal/watch/names.go @@ -30,6 +30,5 @@ func BucketsOf(module string) OwnBuckets { // The findings a value of another part is raised as, named by their words. const ( - FindingManifestOutgrown = "manifest-outgrown" FindingNodeAccountFallback = "node-account-fallback" ) diff --git a/modules/mesh-delivery-watchman/internal/watch/pipelines.go b/modules/mesh-delivery-watchman/internal/watch/pipelines.go index 05cb4a7b..4ae9cd60 100644 --- a/modules/mesh-delivery-watchman/internal/watch/pipelines.go +++ b/modules/mesh-delivery-watchman/internal/watch/pipelines.go @@ -5,115 +5,14 @@ import ( "fmt" "time" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" ) -// What the runner and the gate say of a watched branch's head, heard as events (design 47 G §12, §75, §112): a head the -// gate says moved (`source-moved`) that starts no pipeline within five minutes is the finding `pipeline-stands`, naming the -// branch and its head, raised by the condition keeper and cleared by the runner's `pipeline-started` for that head. - -// KindPipelineStands is a watched branch's head, or a step, standing still (design 47 G §12, §75). +// KindPipelineStands is a pipeline step standing still (design 47 H §741; G §12, §75): the watchman learns no head, so a +// watched branch's head that starts no pipeline is the gate's head-untaken alone, and a pipeline waiting its turn raises +// nothing (steps.go). const KindPipelineStands = "pipeline-stands" -// HeadBound is how long a watched branch's head may stand with no pipeline started (design 47 G §12, §112: the record's five -// minutes). -const HeadBound = 5 * time.Minute - -// headRow is the signals table's row of a head with no pipeline. -const headRow = "head-pipeline" - -// head is a watched branch's head still waiting for its pipeline. -type head struct { - revision int64 - at time.Time -} - -// SourceMoved hears the seat mesh-delivery's `source-moved`: the branch's head is a new source revision, and its pipeline -// is due within HeadBound. A move made by a gathered group is a member governed by the take-in's own bound (ADR 0633 §5), -// so it is not timed here (design 47 G §12). A later move of the same branch replaces the head that waits. -func (w *Watchman) SourceMoved(body []byte) { - var m meshdelivery.SourceMovedEvent - if err := json.Unmarshal(body, &m); err != nil || m.Branch == "" || m.Revision == 0 { - w.say("a source-moved does not read, left: %v", err) - return - } - w.mu.Lock() - defer w.mu.Unlock() - id := headID(m.Branch) - w.unsignal(headRow, id) - w.unfind(FindingKey(ScopeDelivery, id, KindPipelineStands)) - if m.Group != "" { - delete(w.heads, m.Branch) - return - } - at := w.now() - if m.At != nil && !m.At.IsZero() { - at = *m.At - } - w.heads[m.Branch] = head{revision: m.Revision, at: at} - w.timeHead(m.Branch) -} - -// timeHead times the head of a branch that waits for its pipeline, w.mu held: five minutes from its own time, unless the -// branch has a pending revision, whose take-in wait governs it (design 47 G §497) and which times it again once taken in. -func (w *Watchman) timeHead(branch string) { - h, ok := w.heads[branch] - if !ok || w.branchPending(branch) { - return - } - id := headID(branch) - f := Finding{Scope: ScopeDelivery, ID: id, Kind: KindPipelineStands, Severity: Warning, Source: headRow, Since: h.at, - Summary: fmt.Sprintf("the head of %s (source revision %d) has started no pipeline since %s", branch, h.revision, stamp(h.at)), - Said: fmt.Sprintf("no pipeline in %s", HeadBound), - Headline: "A change has started no pipeline", - Explanation: "The head of the watched branch " + branch + " (source revision " + fmt.Sprint(h.revision) + ") moved " + - HeadBound.String() + " ago or more and the pipeline runner has said no pipeline started for it.", - Needs: "the runner to start the pipeline (its verbs mesh-pipeline-runner.stands and mesh-pipeline-runner.list say where it is)"} - left := HeadBound - w.now().Sub(h.at) - if left > 0 { - w.signal(headRow, id, left, signalDue{Do: dueFind, Finding: f}) - return - } - w.find(f) -} - -// branchPending is whether an open take-in wait names the branch (design 47 G §497): its head is behind a pending revision. -func (w *Watchman) branchPending(branch string) bool { - for _, b := range w.pendingBranch { - if b == branch { - return true - } - } - return false -} - -// PipelineStarted hears the runner's `pipeline-started`: it ends the wait of the head it is for (the branch's head at that -// source revision or a later one). A run on request is no head's pipeline. -func (w *Watchman) PipelineStarted(body []byte) { - var p meshpipelinerunner.PipelineStartedEvent - if err := json.Unmarshal(body, &p); err != nil || p.Branch == "" { - w.say("a pipeline-started does not read, left: %v", err) - return - } - if p.Run != "" { - return - } - w.mu.Lock() - defer w.mu.Unlock() - h, ok := w.heads[p.Branch] - if !ok || p.Source < h.revision { - return - } - delete(w.heads, p.Branch) - id := headID(p.Branch) - w.unsignal(headRow, id) - w.unfind(FindingKey(ScopeDelivery, id, KindPipelineStands)) -} - -// headID is the finding's id of a branch's head. -func headID(branch string) string { return "head-" + unsafeKey.ReplaceAllString(branch, "_") } - // KindBuildCapacityWaits is a build seat whose oldest queued step has waited the bound (design 47 G §86). const KindBuildCapacityWaits = "build-capacity-waits" diff --git a/modules/mesh-delivery-watchman/internal/watch/pipelines_test.go b/modules/mesh-delivery-watchman/internal/watch/pipelines_test.go index 230a4739..e45c2b74 100644 --- a/modules/mesh-delivery-watchman/internal/watch/pipelines_test.go +++ b/modules/mesh-delivery-watchman/internal/watch/pipelines_test.go @@ -7,115 +7,8 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein" ) -func moved(branch string, rev int64, group string, at time.Time) []byte { - b, _ := json.Marshal(meshdelivery.SourceMovedEvent{Branch: branch, Revision: rev, Group: group, At: &at}) - return b -} - -// takeInBodyOn is a take-in event's body that names its watched branch (design 47 G §493). -func takeInBodyOn(t *testing.T, kind, key, branch string, since time.Time) []byte { - t.Helper() - var m map[string]any - if err := json.Unmarshal(takeInBody(t, kind, key, since), &m); err != nil { - t.Fatal(err) - } - m["branch"] = branch - b, err := json.Marshal(m) - if err != nil { - t.Fatal(err) - } - return b -} - -func started(branch string, rev int64, run string) []byte { - b, _ := json.Marshal(meshpipelinerunner.PipelineStartedEvent{Branch: branch, Source: rev, Run: run, Module: "m", Pipeline: "p"}) - return b -} - -// design 47 G §12, §79 (test 76): a head with no pipeline within five minutes is `pipeline-stands`, cleared by the -// runner's `pipeline-started` for that head; a gathered group's member and a run on request do not. -func TestAHeadWithNoPipelineStands(t *testing.T) { - key := FindingKey(ScopeDelivery, headID("main"), KindPipelineStands) - cases := []struct { - name string - run func(w *Watchman, o *fakeOut, c *clock) - stand bool - }{ - {"five minutes and no pipeline", func(w *Watchman, o *fakeOut, c *clock) { - w.SourceMoved(moved("main", 7, "", c.t)) - if o.ttls[signalKey(headRow, headID("main"))] != HeadBound || len(o.findings) != 0 { - t.Fatalf("within the bound: %v %v", o.ttls, o.findings) - } - expire(w, c, signalKey(headRow, headID("main")), HeadBound) - }, true}, - {"its pipeline-started clears it", func(w *Watchman, o *fakeOut, c *clock) { - w.SourceMoved(moved("main", 7, "", c.t)) - expire(w, c, signalKey(headRow, headID("main")), HeadBound) - w.PipelineStarted(started("main", 7, "")) - }, false}, - {"a pipeline before the bound raises none", func(w *Watchman, o *fakeOut, c *clock) { - w.SourceMoved(moved("main", 7, "", c.t)) - w.PipelineStarted(started("main", 8, "")) - if _, ok := o.pulses[signalKey(headRow, headID("main"))]; ok { - t.Fatal("the pulse stands") - } - }, false}, - {"a pipeline of an older head does not clear a newer head", func(w *Watchman, o *fakeOut, c *clock) { - w.SourceMoved(moved("main", 7, "", c.t)) - w.PipelineStarted(started("main", 6, "")) - expire(w, c, signalKey(headRow, headID("main")), HeadBound) - }, true}, - {"a gathered group's member is not timed", func(w *Watchman, o *fakeOut, c *clock) { - w.SourceMoved(moved("main", 7, "grp", c.t)) - if len(o.pulses) != 0 { - t.Fatalf("a pulse for a member: %v", o.pulses) - } - }, false}, - {"a head behind a pending revision of its branch is not timed", func(w *Watchman, o *fakeOut, c *clock) { - w.TakeIn(takeInEvent(t, takein.TakeInPending, "m.1"), takeInBodyOn(t, takein.TakeInPending, "m.1", "main", c.t)) - w.SourceMoved(moved("main", 7, "", c.t)) - if _, ok := o.pulses[signalKey(headRow, headID("main"))]; ok { - t.Fatalf("a pulse for a head behind a pending revision: %v", o.pulses) - } - }, false}, - {"a pending revision of another branch does not hold it", func(w *Watchman, o *fakeOut, c *clock) { - w.TakeIn(takeInEvent(t, takein.TakeInPending, "m.1"), takeInBodyOn(t, takein.TakeInPending, "m.1", "other", c.t)) - w.SourceMoved(moved("main", 7, "", c.t)) - expire(w, c, signalKey(headRow, headID("main")), HeadBound) - }, true}, - {"a head whose pending revision is taken in is timed again", func(w *Watchman, o *fakeOut, c *clock) { - w.SourceMoved(moved("main", 7, "", c.t)) - w.TakeIn(takeInEvent(t, takein.TakeInPending, "m.1"), takeInBodyOn(t, takein.TakeInPending, "m.1", "main", c.t)) - if _, ok := o.pulses[signalKey(headRow, headID("main"))]; ok { - t.Fatalf("the head is timed behind a pending revision: %v", o.pulses) - } - c.t = c.t.Add(HeadBound) - w.TakeIn(takeInEvent(t, takein.TakeInTaken, "m.1"), takeInBodyOn(t, takein.TakeInTaken, "m.1", "main", c.t)) - if _, ok := o.findings[key]; !ok { - t.Fatalf("the head, past its five minutes when taken in, stands: %v", o.findings) - } - }, true}, - {"a run on request is no head's pipeline", func(w *Watchman, o *fakeOut, c *clock) { - w.SourceMoved(moved("main", 7, "", c.t)) - w.PipelineStarted(started("main", 7, "run-1")) - expire(w, c, signalKey(headRow, headID("main")), HeadBound) - }, true}, - } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - w, o, c := setup() - tc.run(w, o, c) - if _, ok := o.findings[key]; ok != tc.stand { - t.Fatalf("the finding stands %v, want %v: %v", ok, tc.stand, o.findings) - } - }) - } -} - func queuedStep(pipeline, step, seat, outcome string, started time.Time) []byte { b, _ := json.Marshal(meshpipelinerunner.StepEvent{Pipeline: pipeline, Branch: "main", Module: "m", Commit: "c", Result: meshpipelinerunner.StepResult{Step: step, Seat: seat, Outcome: outcome, Started: &started}}) @@ -249,37 +142,31 @@ func TestARecordOpenADayIsAFinding(t *testing.T) { } } -// design 47 G §502, A-200 (WM16): a record is tied to the entries of the take-in.taken of its milestone `.`; it -// is covered only while one of those entries is held by an `acting-part-late` wait, so another record's late entry covers nothing. -func TestARecordIsCoveredOnlyByTheWaitsOfItsOwnEntries(t *testing.T) { +// design 47 G §502, A-200 (WM16), H-A-25: a record is tied to its source revision by its milestone `.`, the +// key of that revision's take-in wait: while its take-in is held and said, take-in-waits covers it; once taken in it is open +// at 24 hours, and another record's held take-in covers nothing of it. +func TestARecordIsCoveredOnlyWhileItsOwnTakeInIsHeld(t *testing.T) { w, o, c := setup() - taken := func(key string, source int64, module string) { - var m map[string]any - if err := json.Unmarshal(takeInBody(t, takein.TakeInTaken, key, c.t), &m); err != nil { - t.Fatal(err) - } - m["branch"], m["source"] = "main", source - m["entries"] = []map[string]any{{"commit": "c", "fingerprint": "f", "module": module}} - b, _ := json.Marshal(m) - w.TakeIn(takeInEvent(t, takein.TakeInTaken, key), b) - } - taken("m.1", 7, "web") - taken("m.2", 8, "db") + w.TakeInCaughtUp() + w.TakeIn(TakeInTakenEvent, takenBody(t, "main", 8), c.t) + w.TakeIn(TakeInPendingEvent, pendingBody(t, "main", 7, "the catalog store is full"), c.t) rec := func(id, milestone string) []byte { - b, _ := json.Marshal(meshdelivery.RecordOpenedEvent{Record: id, Milestone: milestone, At: &c.t}) + b, _ := json.Marshal(meshdelivery.RecordOpenedEvent{Record: id, Milestone: milestone, Branch: "main", Call: "fedcba9876543210", At: &c.t}) return b } - w.RecordOpen(rec("rWeb", "main.7")) - w.RecordOpen(rec("rDb", "main.8")) - late, _ := json.Marshal(meshwants.Wait{Step: "place", Part: "mesh-delivery-indexer", Node: "ace", Module: "web", Since: c.t, Bound: "30m"}) - w.WaitHeld("mesh-wants", late) - expire(w, c, signalKey(actingRow, "mesh-wants-place-mesh-delivery-indexer-ace-web"), 30*time.Minute) - expire(w, c, signalKey(recordRow, "rWeb"), RecordOpenBound) - expire(w, c, signalKey(recordRow, "rDb"), 0) - if _, ok := o.findings[FindingKey(ScopeDelivery, "rWeb", KindRecordOpen)]; ok { - t.Fatalf("a record whose entry is held by acting-part-late is covered: %v", o.findings) + w.RecordOpen(rec("rHeld", "main.7")) + w.RecordOpen(rec("rTaken", "main.8")) + expire(w, c, signalKey(takeInRow, TakeInKey("main", 7)), DefaultTakeInBound) + expire(w, c, signalKey(recordRow, "rHeld"), RecordOpenBound-DefaultTakeInBound) + expire(w, c, signalKey(recordRow, "rTaken"), 0) + if _, ok := o.findings[FindingKey(ScopeDelivery, "rHeld", KindRecordOpen)]; ok { + t.Fatalf("a record whose take-in is held and said is not covered: %v", o.findings) } - if _, ok := o.findings[FindingKey(ScopeDelivery, "rDb", KindRecordOpen)]; !ok { - t.Fatalf("a record of other entries is open at 24 hours whatever else is late: %v", o.findings) + if _, ok := o.findings[FindingKey(ScopeDelivery, "rTaken", KindRecordOpen)]; !ok { + t.Fatalf("a record taken in is not open at 24 hours whatever else is held: %v", o.findings) + } + w.TakeIn(TakeInTakenEvent, takenBody(t, "main", 7), c.t) + if _, ok := o.findings[FindingKey(ScopeDelivery, "rHeld", KindRecordOpen)]; !ok { + t.Fatalf("a record whose take-in ended is not open: %v", o.findings) } } diff --git a/modules/mesh-delivery-watchman/internal/watch/records.go b/modules/mesh-delivery-watchman/internal/watch/records.go index d08a0e4e..8d600b08 100644 --- a/modules/mesh-delivery-watchman/internal/watch/records.go +++ b/modules/mesh-delivery-watchman/internal/watch/records.go @@ -3,7 +3,6 @@ package watch import ( "encoding/json" "fmt" - "strings" "time" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" @@ -14,15 +13,14 @@ import ( // - `filling`, `full` and `room` of a store against its cap: `store-filling` while it stands past its cap's share, urgent // when it is full, cleared by `room` or by a `filling` under FillingFrom; // - `record-opened` and `record-ended` of a delivery record: `record-open` when it has stood open for 24 hours, unless -// another condition already tells why (`acting-part-late`, or a store `full`). +// another condition already tells why (a store `full`, or the take-in held). // Each is a row of the signals table: its pulse is its bound, written at the event, so a restart loses none. // The kinds of these findings. const ( - KindActingPartLate = "acting-part-late" - KindPutStands = "put-stands" - KindStoreFilling = "store-filling" - KindRecordOpen = "record-open" + KindPutStands = "put-stands" + KindStoreFilling = "store-filling" + KindRecordOpen = "record-open" ) // The rows of the signals table of these findings, which a restart keeps. @@ -126,12 +124,11 @@ func (w *Watchman) RecordEnded(body []byte) { w.unfind(FindingKey(ScopeDelivery, r.Record, KindRecordOpen)) } -// recordCovered says another condition tells the operator why a record is open (design 47 G §502, A-200): the record is tied to the -// entries of its source revision's `take-in.taken` by its milestone, `.`, and is covered while one of those -// entries is held by a wait that raised `acting-part-late`, or a store says `full`; a record whose revision is not known to be taken in -// is covered by a standing `take-in-waits`. +// recordCovered says another condition tells the operator why a record is open (design 47 G §502, A-200): a store says `full`, or +// the record's source revision, its milestone `.` (H-A-25), is not known to be taken in while a `take-in-waits` +// stands. func (w *Watchman) recordCovered(record string) bool { - entries, taken := w.takenEntries[w.recordMilestone[record]] + taken := w.taken[w.recordMilestone[record]] for _, v := range w.findings { f, err := DecodeFinding(v) if err != nil { @@ -140,12 +137,6 @@ func (w *Watchman) recordCovered(record string) bool { switch { case f.Kind == KindStoreFilling && f.Severity == Urgent: return true - case f.Kind == KindActingPartLate && taken: - for _, module := range entries { - if strings.HasSuffix(f.ID, "-"+module) { - return true - } - } case f.Kind == KindTakeInWaits && !taken: return true } diff --git a/modules/mesh-delivery-watchman/internal/watch/signals.go b/modules/mesh-delivery-watchman/internal/watch/signals.go index 9391c456..ac7a4022 100644 --- a/modules/mesh-delivery-watchman/internal/watch/signals.go +++ b/modules/mesh-delivery-watchman/internal/watch/signals.go @@ -777,8 +777,8 @@ var SignalRows = []struct{ Row, Signal, Bound, Kind string }{ {"S10", "the self-check's checked", (3 * SelfCheckEvery).String(), KindSelfCheckSilent}, {"S11", "a machine's tool runner heartbeat", "3 × its interval (" + (3 * HeartbeatEvery).String() + " when it says none)", KindToolsSilent}, {"S21", "a machine's provisioner client's files not placed", ApplyEvery.String(), KindApplyFailing}, - {"take-in", "a source revision or a delivery group the indexer holds back (ADR 0633 §5)", TakeInRevisionBound.String() + " a revision, " + TakeInGroupBound.String() + " a group", KindTakeInWaits}, - {actingRow, "an acting part's wait held, or a put's, a put without hold that stands (the seats' wait-held, ended by wait-ended)", "the wait's own bound", KindActingPartLate + ", " + KindPutStands}, + {takeInRow, "a source revision whose take-in the indexer holds (take-in.pending, ended by take-in.taken; design 47 H §744)", "the setting take-in-bound, " + DefaultTakeInBound.String() + " by default", KindTakeInWaits}, + {actingRow, "a put without hold that stands (the seat mesh-wants' wait-held, ended by wait-ended)", "the wait's own bound", KindPutStands}, {storeRow, "a store of the delivery's parts filling or full (the seats' filling, full and room)", "filling from " + fmt.Sprintf("%.0f", FillingFrom*100) + " %, full urgent", KindStoreFilling}, {recordRow, "a delivery record open (the seat mesh-delivery's record-opened, ended by record-ended)", RecordOpenBound.String(), KindRecordOpen}, {"provisions", "a provision of mesh-provisioner in waits (ADR 0570 §7)", "at once", KindNeedsOperator}, diff --git a/modules/mesh-delivery-watchman/internal/watch/takein.go b/modules/mesh-delivery-watchman/internal/watch/takein.go index e534ac6c..48f85b9e 100644 --- a/modules/mesh-delivery-watchman/internal/watch/takein.go +++ b/modules/mesh-delivery-watchman/internal/watch/takein.go @@ -9,75 +9,107 @@ import ( "time" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein" ) -// KindTakeInWaits is a source revision or a delivery group the indexer holds back and has not taken in within its bound -// (novox/hq ADR 0633 §5, issue 1585): a revision for the indexer build it names, a group for its end. It stands until a -// `take-in.taken` ends the wait, whoever ended it (the named indexer, a revert, a hand take-in or a drop). +// KindTakeInWaits is a source revision the indexer holds back and has not taken in within the setting take-in-bound +// (design 47 H §744, H-A-94 (2); G §497): its take-in cannot be written. It stands until a `take-in.taken` of that source +// ends the wait, whoever ended it (the indexer's take-in, a hand take-in or a skip). const KindTakeInWaits = "take-in-waits" -// The bounds of a take-in wait, from the `take-in.pending` event's own since. +// DefaultTakeInBound is the setting take-in-bound's default (design 47 H §744, H-A-94 (2)). +const DefaultTakeInBound = 60 * time.Minute + +// The take-in events of the indexer, by their names relative to it. const ( - TakeInRevisionBound = 60 * time.Minute - TakeInGroupBound = 30 * time.Minute + TakeInPendingEvent = "take-in.pending" + TakeInTakenEvent = "take-in.taken" ) // takeInRow is the signals table's row of a take-in wait. const takeInRow = "take-in" -// TakeIn hears one take-in event of the indexer, by its name relative to the indexer (`take-in.`), the wait's key -// in its body: a pending one starts the wait for its key, whose bound runs from the event's since, and a taken one ends it. A group's -// taken event ends the group's wait only: the revision it becomes is its own key and starts its own, longer wait. -func (w *Watchman) TakeIn(event string, body []byte) { - kind, err := takein.ParseTakeInEvent(event) - if err != nil { - w.say("a take-in event is left: %v", err) - return +// TakeInKey is a take-in wait's key, `.` (design 47 H-A-94 (2)). +func TakeInKey(branch string, source int64) string { return fmt.Sprintf("%s.%d", branch, source) } + +// HeldTakeIn is one take-in the indexer holds, as indexer_pending answers it (design 47 H-A-97, H-A-102): the contract's +// form with since, when it was held. +type HeldTakeIn struct { + meshcatalog.TakeInPending + Since time.Time `json:"since"` +} + +// HeldTakeIns is the answer of indexer_pending (design 47 H-A-102). +type HeldTakeIns struct { + Held []HeldTakeIn `json:"held"` +} + +// DecodeHeldTakeIns reads the answer of indexer_pending. +func DecodeHeldTakeIns(raw []byte) (HeldTakeIns, error) { + var a HeldTakeIns + if err := json.Unmarshal(raw, &a); err != nil { + return HeldTakeIns{}, fmt.Errorf("not an answer of indexer_pending: %w", err) } - t, err := takein.DecodeTakeIn(body) - if err != nil { - w.say("the body of %s cannot be read, left: %v", event, err) - return + for _, h := range a.Held { + if h.Branch == "" || h.Source == 0 || h.Since.IsZero() { + return HeldTakeIns{}, fmt.Errorf("indexer_pending names a held take-in with no branch, source or since: %+v", h) + } } - if t.Kind != kind { - w.say("the body of %s says the kind %q, left", event, t.Kind) - return - } - // The wait names the watched branch it is of (design 47 G §493), by the contract's body. - var named meshcatalog.TakeInPending - _ = json.Unmarshal(body, &named) - var taken meshcatalog.TakeInTaken - _ = json.Unmarshal(body, &taken) - key := t.Key + return a, nil +} + +// SetTakeInBound sets the setting take-in-bound; zero or less keeps its default. +func (w *Watchman) SetTakeInBound(d time.Duration) { w.mu.Lock() defer w.mu.Unlock() - w.takeIn(kind, key, t) - w.pendingOf(kind, key, named.Branch) - if kind == takein.TakeInTaken && taken.Branch != "" && taken.Source != 0 && len(taken.Entries) > 0 { - var modules []string - for _, e := range taken.Entries { - modules = append(modules, e.Module) - } - w.takenEntries[fmt.Sprintf("%s.%d", taken.Branch, taken.Source)] = modules // a record's milestone (A-200) - w.rejudgeRecords() + if d > 0 { + w.takeInBound = d } } -// SeedTakeIn starts, at the watchman's start and before its consumer on the take-in stream, the wait of every revision -// and group the indexer holds now, as indexer_pending answers them (ADR 0633 §5): the consumer is durable, so the +// TakeIn hears one take-in event of the indexer, by its name relative to the indexer, heard at the message's own time: +// a `take-in.pending` starts the wait of its `.`, whose bound runs from that time, and a `take-in.taken` +// ends it, and asks the catalog entry of each mover it names (busstate.go). +func (w *Watchman) TakeIn(event string, body []byte, at time.Time) { + switch event { + case TakeInPendingEvent: + var p meshcatalog.TakeInPending + if err := json.Unmarshal(body, &p); err != nil || p.Branch == "" || p.Source == 0 { + w.say("the body of %s cannot be read, left: %v", event, err) + return + } + w.mu.Lock() + defer w.mu.Unlock() + w.takeInHeldSince(TakeInKey(p.Branch, p.Source), p.Why, at) + case TakeInTakenEvent: + var t meshcatalog.TakeInTaken + if err := json.Unmarshal(body, &t); err != nil || t.Branch == "" || t.Source == 0 { + w.say("the body of %s cannot be read, left: %v", event, err) + return + } + key := TakeInKey(t.Branch, t.Source) + w.mu.Lock() + defer w.mu.Unlock() + w.takeInEnded(key) + w.taken[key] = true // a record's milestone (A-200, H-A-25) + w.rejudgeRecords() + w.moversTaken(t.Entries, at) + default: + w.say("a take-in event %s is left: not take-in.pending or take-in.taken", event) + } +} + +// SeedTakeIn starts, at the watchman's start and before its consumer on the take-in stream, the wait of every take-in +// the indexer holds now, as indexer_pending answers it (design 47 H-A-94 (2), H-A-97): the consumer is durable, so the // pending events it acknowledged before a restart are not heard again, and a wait within its bound would otherwise be -// lost. Each wait's bound runs from its own since, as a pending event's does; the stream's events that follow replace -// or end it. A wait or finding the answer does not name is ended. -func (w *Watchman) SeedTakeIn(a takein.PendingAnswer) { +// lost. Each wait's bound runs from its own since; the stream's events that follow replace or end it. A wait or finding +// the answer does not name is ended. +func (w *Watchman) SeedTakeIn(a HeldTakeIns) { w.mu.Lock() defer w.mu.Unlock() - // A wait or finding of a key the answer does not name is ended as a take-in.taken would end it: the finding a previous - // run wrote (read at the start, before the seed), a wait held from an event, a signal standing (design 47, Taken in by - // the indexer it names). Only an answered seed gets here: one that could not ask ends nothing. + // Only an answered seed gets here: one that could not ask ends nothing. named := map[string]bool{} - for _, wait := range a.Waits { - named[wait.Key] = true + for _, h := range a.Held { + named[TakeInKey(h.Branch, h.Source)] = true } stale := map[string]bool{} for _, value := range w.findings { @@ -96,64 +128,36 @@ func (w *Watchman) SeedTakeIn(a takein.PendingAnswer) { } } for _, key := range slices.Sorted(maps.Keys(stale)) { - w.takeIn(takein.TakeInTaken, key, takein.TakeIn{Kind: takein.TakeInTaken, Key: key}) + w.takeInEnded(key) } - for _, wait := range a.Waits { - w.takeIn(takein.TakeInPending, wait.Key, takein.TakeIn{Kind: takein.TakeInPending, Key: wait.Key, What: wait.What, - Merges: wait.Merges, Group: wait.Group, Members: wait.Members, WaitsFor: wait.WaitsFor, Since: wait.Since}) + for _, h := range a.Held { + w.takeInHeldSince(TakeInKey(h.Branch, h.Source), h.Why, h.Since) } } -// pendingOf records or ends the branch an open take-in wait names (design 47 G §497), w.mu held: the head of a branch behind a -// pending revision raises no `pipeline-stands`, and is timed again, from the move of the head, once the branch has none. -func (w *Watchman) pendingOf(kind, key, branch string) { - if kind == takein.TakeInTaken { - branch = w.pendingBranch[key] - delete(w.pendingBranch, key) - if branch != "" && !w.branchPending(branch) { - w.timeHead(branch) - } - return - } - if branch == "" { - return - } - w.pendingBranch[key] = branch - if _, ok := w.heads[branch]; ok { - id := headID(branch) - w.unsignal(headRow, id) - w.unfind(FindingKey(ScopeDelivery, id, KindPipelineStands)) - } +// takeInEnded ends the wait of key, w.mu held. +func (w *Watchman) takeInEnded(key string) { + delete(w.takeInHeld, key) + w.unsignal(takeInRow, key) + w.unfind(FindingKey(ScopeDelivery, key, KindTakeInWaits)) } -// takeIn is one take-in event of key, w.mu held. -func (w *Watchman) takeIn(kind, key string, t takein.TakeIn) { - finding := FindingKey(ScopeDelivery, key, KindTakeInWaits) - if kind == takein.TakeInTaken { - delete(w.takeInHeld, key) - w.unsignal(takeInRow, key) - w.unfind(finding) - return +// takeInHeldSince is the take-in of key held since since, for why, w.mu held: its finding is due take-in-bound from since. +func (w *Watchman) takeInHeldSince(key, why string, since time.Time) { + bound := w.takeInBound + why = strings.TrimSpace(why) + if why == "" { + why = "the indexer said no reason" } - bound, what := TakeInRevisionBound, "a source revision is held (a message lost or unread, no room for it, or its refusal)" - if t.What == takein.TakeInGroup { - bound, what = TakeInGroupBound, "a delivery group is gathered and its end has not come" - } - var merges []string - for _, m := range t.Merges { - merges = append(merges, m.Repository) - } - f := Finding{Scope: ScopeDelivery, ID: key, Kind: KindTakeInWaits, Severity: Warning, Source: "take-in", Since: t.Since, - Summary: fmt.Sprintf("%s: %s, since %s (%s)", key, what, stamp(t.Since), strings.Join(merges, ", ")), + f := Finding{Scope: ScopeDelivery, ID: key, Kind: KindTakeInWaits, Severity: Warning, Source: takeInRow, Since: since, + Summary: fmt.Sprintf("%s: the take-in of this source revision is held, since %s (%s)", key, stamp(since), why), Said: fmt.Sprintf("held for %s", bound), Headline: "A merged change is held back by the indexer", - Explanation: "The indexer holds " + what + " and has not taken it in within " + bound.String() + ", so what the merge changes does " + - "not reach the machines.", - Needs: "the take-in by hand (indexer_take_in_pending), or a drop (indexer_drop_pending)"} - left := bound - w.now().Sub(t.Since) - w.unfind(finding) - w.unsignal(takeInRow, key) - delete(w.takeInHeld, key) + Explanation: "The indexer holds the take-in of the source revision " + key + " (" + why + ") and has not taken it in within " + + bound.String() + ", so what the merge changes does not reach the machines.", + Needs: "the take-in by hand (indexer_take_in_pending), or a skip (indexer_take_in_skip)"} + left := bound - w.now().Sub(since) + w.takeInEnded(key) switch { case left > 0: w.signal(takeInRow, key, left, signalDue{Do: dueFind, Finding: f}) diff --git a/modules/mesh-delivery-watchman/internal/watch/waits.go b/modules/mesh-delivery-watchman/internal/watch/waits.go index 034bc357..34fc9f59 100644 --- a/modules/mesh-delivery-watchman/internal/watch/waits.go +++ b/modules/mesh-delivery-watchman/internal/watch/waits.go @@ -10,31 +10,40 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" ) -// What the holders of an acting part's wait say, heard as the seats' events (design 47 G §266, §269, ADR 0596 §2), never read -// from their state: `wait-held` and `wait-ended` of a wait (the seat mesh-wants, and the seat mesh-pipeline-runner for its -// `pipeline` step). A wait held past its bound is the finding `acting-part-late`, timed by that bound alone from the time -// `wait-held` carries as `clock-from` (its `since` where it carries none), and ended by `wait-ended`, whoever ended it (the -// part's new version running, or a release). Its pulse is its bound, written at the event, so a restart loses none. +// What the holder of the seat mesh-wants says of a put's wait (design 47 G §498), heard as the seat's `wait-held` and +// `wait-ended`, never read from its state: a put without hold that stands past its bound is the finding `put-stands`, timed +// by that bound alone from the time `wait-held` carries as `clock-from` (its `since` where it carries none), and ended by +// `wait-ended`. An acting part's wait is no finding (design 47 H §764, H-A-70). Its pulse is its +// bound, written at the event, so a restart loses none. -// actingRow is the row of the signals table of an acting part's wait. +// actingRow is the row of the signals table of a put's wait. const actingRow = "acting-wait" func waitID(seat string, a meshwants.Wait) string { return strings.Join([]string{seat, a.Step, a.Part, a.Node, a.Module}, "-") } -// WaitHeld hears a seat's `wait-held`: the finding is due when the wait's bound has run from its clock. +// WaitHeld hears a seat's `wait-held`: a put's finding is due when its bound has run from its clock; any other wait is left. func (w *Watchman) WaitHeld(seat string, body []byte) { a, ok := decodeWait(w, seat, "wait-held", body) if !ok { return } + bound, err := time.ParseDuration(a.Bound) + if err != nil || bound <= 0 { + w.say("the put of %s says no bound (%q), so it is never late", a.Module, a.Bound) + return + } + from := a.Since + if a.ClockFrom != nil { + from = *a.ClockFrom + } w.mu.Lock() defer w.mu.Unlock() - w.waitHeld(seat, a) + w.putStands(waitID(seat, a), a, bound, from) } -// WaitEnded hears a seat's `wait-ended`: the wait ran or was released, so its finding and its pulse end. +// WaitEnded hears a seat's `wait-ended`: the put ran or was released, so its finding and its pulse end. func (w *Watchman) WaitEnded(seat string, body []byte) { a, ok := decodeWait(w, seat, "wait-ended", body) if !ok { @@ -44,57 +53,21 @@ func (w *Watchman) WaitEnded(seat string, body []byte) { defer w.mu.Unlock() id := waitID(seat, a) w.unsignal(actingRow, id) - w.unfind(FindingKey(ScopeDelivery, id, KindActingPartLate)) w.unfind(FindingKey(ScopeDelivery, id, KindPutStands)) - w.rejudgeRecords() } +// decodeWait reads a put's wait; a wait of another step is left unsaid, since it is no finding. func decodeWait(w *Watchman, seat, event string, body []byte) (meshwants.Wait, bool) { var a meshwants.Wait - if err := json.Unmarshal(body, &a); err != nil || (a.Part == "" && a.Step != meshwants.WaitStepPut) || a.Module == "" || a.Since.IsZero() { + if err := json.Unmarshal(body, &a); err != nil || (a.Step == meshwants.WaitStepPut && (a.Module == "" || a.Since.IsZero())) { w.say("the %s of the seat %s does not read, left: %v", event, seat, err) return a, false } - return a, true -} - -// waitHeld is one wait held, w.mu held. -func (w *Watchman) waitHeld(seat string, a meshwants.Wait) { - bound, err := time.ParseDuration(a.Bound) - if err != nil || bound <= 0 { - w.say("the wait of %s at %s says no bound (%q), so it is never late", a.Module, a.Part, a.Bound) - return - } - from := a.Since - if a.ClockFrom != nil { - from = *a.ClockFrom - } - id := waitID(seat, a) - if a.Step == meshwants.WaitStepPut { - w.putStands(id, a, bound, from) - return - } - f := Finding{Scope: ScopeDelivery, ID: id, Kind: KindActingPartLate, Severity: Warning, Source: actingRow, Since: a.Since, - Summary: fmt.Sprintf("%s on %s waits for %s to run %s at the step %s, since %s (bound %s)", a.Module, orWords(a.Node, "no node"), a.Part, - orWords(a.Version, "its new version"), a.Step, stamp(a.Since), bound), - Said: "held for " + bound.String(), - Headline: "An acting part's new version has not come", - Explanation: "A step waits until " + a.Part + " runs its new version, and its bound of " + bound.String() + - " has run, so what the step does is held back.", - Needs: "the part's new version to run, or the wait released by hand (the holder's release tool)"} - left := bound - w.now().Sub(from) - w.unsignal(actingRow, id) - w.unfind(f.Key()) - if left > 0 { - w.signal(actingRow, id, left, signalDue{Do: dueFind, Finding: f}) - return - } - w.find(f) - w.rejudgeRecords() // a record whose entry this wait holds is covered now (§502) + return a, a.Step == meshwants.WaitStepPut } // putStands is a put's wait held (`step: put`, design 47 G §498): a put without hold that stands past its bound is said as the -// finding `put-stands`, never `acting-part-late`, since no acting part is waited for. w.mu held. +// finding `put-stands`, since no acting part is waited for. w.mu held. func (w *Watchman) putStands(id string, a meshwants.Wait, bound time.Duration, from time.Time) { f := Finding{Scope: ScopeDelivery, ID: id, Kind: KindPutStands, Severity: Warning, Source: actingRow, Since: a.Since, Summary: fmt.Sprintf("a put of %s on %s at %s stands, since %s (bound %s)", a.Module, orWords(a.Node, "no node"), @@ -114,17 +87,15 @@ func (w *Watchman) putStands(id string, a meshwants.Wait, bound time.Duration, f w.find(f) } -// What a node-engine says of a hold it keeps (design 47 G §499, §541), heard on the seat node-engine's `wait-held` and `wait-ended` -// (`mesh.seat.node-engine.event...`, the node the one that says it): a hold of the composition (`step: -// compose`, with its reason) standing past composing's 30 minutes (§266) is the finding `delivery-waits`, naming the node, the -// module and the reason, and a wait for a kept build (`step: kept-build`, naming the build node) standing past building's default -// bound of 60 minutes is `kept-build-waits`. Each clears on `wait-ended`. +// What a node-engine says of a wait for a kept build (design 47 G §499, §541), heard on the seat node-engine's `wait-held` +// and `wait-ended` (`mesh.seat.node-engine.event...`, the node the one that says it): a wait for a kept +// build (`step: kept-build`, naming the build node) standing past building's default bound of 60 minutes is +// `kept-build-waits`, cleared on `wait-ended`. A hold of the composition is no finding (design 47 H +// §764, H-A-91 (2)). -// The findings of a node-engine's waits, and the bounds that stand when its event names none. +// The finding of a node-engine's wait for a kept build, and the bound that stands when its event names none. const ( - KindDeliveryWaits = "delivery-waits" KindKeptBuildWaits = "kept-build-waits" - ComposeBound = 30 * time.Minute KeptBuildBound = 60 * time.Minute ) @@ -133,17 +104,18 @@ const engineRow = "engine-wait" func engineWaitID(node, module string) string { return node + "." + module } -// EngineWaitHeld hears the node-engine of node saying `wait-held` for module. +// EngineWaitHeld hears the node-engine of node saying `wait-held` for module: a wait for a kept build is timed, any other +// left. func (w *Watchman) EngineWaitHeld(node, module string, body []byte) { var a nodeengine.Wait if err := json.Unmarshal(body, &a); err != nil || a.Since.IsZero() { w.say("a wait-held of the node-engine of %s for %s does not read, left: %v", node, module, err) return } - kind, bound := KindDeliveryWaits, ComposeBound - if a.Step == nodeengine.WaitStepKeptBuild { - kind, bound = KindKeptBuildWaits, KeptBuildBound + if a.Step != nodeengine.WaitStepKeptBuild { + return } + bound := KeptBuildBound if d, err := time.ParseDuration(a.Bound); err == nil && d > 0 { bound = d } @@ -152,11 +124,8 @@ func (w *Watchman) EngineWaitHeld(node, module string, body []byte) { from = *a.ClockFrom } id := engineWaitID(node, module) - what := "holds the composition of " + module - if a.Step == nodeengine.WaitStepKeptBuild { - what = "waits for the kept build of " + module + " on " + orWords(a.Node, "no node") - } - f := Finding{Scope: ScopeDelivery, ID: id, Kind: kind, Severity: Warning, Machine: node, Source: engineRow, Since: a.Since, + what := "waits for the kept build of " + module + " on " + orWords(a.Node, "no node") + f := Finding{Scope: ScopeDelivery, ID: id, Kind: KindKeptBuildWaits, Severity: Warning, Machine: node, Source: engineRow, Since: a.Since, Summary: fmt.Sprintf("the node-engine of %s %s: %s, since %s (bound %s)", node, what, orWords(a.Reason, "no reason said"), stamp(a.Since), bound), Said: "held for " + bound.String(), Headline: "A module's delivery is held on a machine", @@ -174,38 +143,11 @@ func (w *Watchman) EngineWaitHeld(node, module string, body []byte) { w.find(f) } -// EngineWaitEnded hears the node-engine of node saying `wait-ended` for module: the hold is over, whichever kind it was. +// EngineWaitEnded hears the node-engine of node saying `wait-ended` for module: the wait is over. func (w *Watchman) EngineWaitEnded(node, module string) { id := engineWaitID(node, module) w.mu.Lock() defer w.mu.Unlock() w.unsignal(engineRow, id) - w.unfind(FindingKey(ScopeDelivery, id, KindDeliveryWaits)) w.unfind(FindingKey(ScopeDelivery, id, KindKeptBuildWaits)) } - -// ModuleWaitHeld hears a module's `wait-held` on its own stream (`mesh.stream..event.wait-held..`), the form -// the builder says a key queued behind a package in, with the package as its `part` (design 47 G A-192, ADR 0603 §8): it is timed as any -// acting part's wait. A wait naming no part is a toolchain wait, which the builder times itself (§672), and is left. -func (w *Watchman) ModuleWaitHeld(holder string, body []byte) { - var a meshwants.Wait - if json.Unmarshal(body, &a) == nil && a.Part == "" { - return - } - a2, ok := decodeWait(w, holder, "wait-held", body) - if !ok { - return - } - w.mu.Lock() - defer w.mu.Unlock() - w.waitHeld(holder, a2) -} - -// ModuleWaitEnded hears the same module's `wait-ended`. -func (w *Watchman) ModuleWaitEnded(holder string, body []byte) { - var a meshwants.Wait - if json.Unmarshal(body, &a) == nil && a.Part == "" { - return - } - w.WaitEnded(holder, body) -} diff --git a/modules/mesh-delivery-watchman/internal/watch/waits_test.go b/modules/mesh-delivery-watchman/internal/watch/waits_test.go index b45b150b..611bb38d 100644 --- a/modules/mesh-delivery-watchman/internal/watch/waits_test.go +++ b/modules/mesh-delivery-watchman/internal/watch/waits_test.go @@ -10,71 +10,9 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" ) -func heldWait(module string, since time.Time, from *time.Time) []byte { - b, _ := json.Marshal(meshwants.Wait{Step: "place", Part: "mesh-delivery-indexer", Version: "src2:new", Node: "ace", Module: module, - Since: since, ClockFrom: from, Bound: "30m"}) - return b -} - -// design 47 G §266, §269 (WM6): a wait held is the finding `acting-part-late` once its bound has run from the time `wait-held` -// carries as `clock-from` (its `since` where it carries none), not within it; `wait-ended` ends it. -func TestAnActingPartsWaitIsLateByItsBoundFromItsClock(t *testing.T) { - id := "mesh-wants-place-mesh-delivery-indexer-ace-web" - key := FindingKey(ScopeDelivery, id, KindActingPartLate) - t.Run("within the bound nothing, past it a finding, wait-ended ends it", func(t *testing.T) { - w, o, c := setup() - w.WaitHeld("mesh-wants", heldWait("web", c.t, nil)) - if o.ttls[signalKey(actingRow, id)] != 30*time.Minute || len(o.findings) != 0 { - t.Fatalf("within the bound: ttl %v, findings %v", o.ttls[signalKey(actingRow, id)], o.findings) - } - expire(w, c, signalKey(actingRow, id), 30*time.Minute) - if _, ok := o.findings[key]; !ok { - t.Fatalf("past its bound: %v", o.findings) - } - w.WaitEnded("mesh-wants", heldWait("web", c.t, nil)) - if len(o.findings) != 0 { - t.Fatalf("wait-ended left %v", o.findings) - } - }) - t.Run("the bound counts from clock-from, not from since", func(t *testing.T) { - w, o, c := setup() - from := c.t.Add(-20 * time.Minute) - w.WaitHeld("mesh-wants", heldWait("web", c.t.Add(-time.Hour), &from)) - if got := o.ttls[signalKey(actingRow, id)]; got != 10*time.Minute { - t.Fatalf("a wait whose clock began 20 minutes ago has %v left, want 10m", got) - } - }) - t.Run("a wait whose bound ran before it was heard is a finding at once", func(t *testing.T) { - w, o, c := setup() - w.WaitHeld("mesh-wants", heldWait("web", c.t.Add(-2*time.Hour), nil)) - if _, ok := o.findings[key]; !ok { - t.Fatalf("a late wait: %v", o.findings) - } - }) - // The grants part's wait is held as a seat's event too (design 47 G §269, G-20 A-12): its body names no node, and its release - // is said `wait-ended` with how `released`, which clears the finding. - t.Run("the seat mesh-grants' granting wait is late by its bound and ended by its release", func(t *testing.T) { - w, o, c := setup() - body := func(how string) []byte { - b, _ := json.Marshal(map[string]any{"step": "grant", "part": "mesh-grants", "version": "fp", "module": "web", "bound": "10m0s", - "since": c.t, "clock-from": c.t, "how": how}) - return b - } - gid := "mesh-grants-grant-mesh-grants--web" - w.WaitHeld("mesh-grants", body("")) - expire(w, c, signalKey(actingRow, gid), 10*time.Minute) - if _, ok := o.findings[FindingKey(ScopeDelivery, gid, KindActingPartLate)]; !ok { - t.Fatalf("past its bound: %v", o.findings) - } - w.WaitEnded("mesh-grants", body("released")) - if len(o.findings) != 0 { - t.Fatalf("the release left %v", o.findings) - } - }) -} - -// design 47 G §498 (WM14): a put's wait (`step: put`, no part) raises `put-stands` once its bound has run, and a pipeline's wait -// raises nothing for it; `wait-ended` ends it. +// design 47 G §498 (WM14): a put's wait (`step: put`, no part) raises `put-stands` once its bound has run, not within it; +// `wait-ended` ends it. Design 47 H §764, H-A-70: an acting part's wait is no finding (acting-part-late retired), so it +// writes no pulse and no finding, past its bound or not. func TestAPutWithoutHoldThatStandsIsPutStands(t *testing.T) { put := func(since time.Time) []byte { b, _ := json.Marshal(meshwants.Wait{Step: meshwants.WaitStepPut, Node: "ace", Module: "web", Fingerprint: "fp-1", Since: since, Bound: "30m"}) @@ -82,6 +20,12 @@ func TestAPutWithoutHoldThatStandsIsPutStands(t *testing.T) { } id := "mesh-wants-put--ace-web" w, o, c := setup() + acting, _ := json.Marshal(meshwants.Wait{Step: "place", Part: "mesh-delivery-indexer", Version: "src2:new", Node: "ace", Module: "web", + Since: c.t.Add(-2 * time.Hour), Bound: "30m"}) + w.WaitHeld("mesh-wants", acting) + if len(o.findings) != 0 || len(o.pulses) != 0 { + t.Fatalf("an acting part's wait is timed: findings %v, pulses %v", o.findings, o.pulses) + } w.WaitHeld("mesh-wants", put(c.t)) if len(o.findings) != 0 { t.Fatalf("within the bound: %v", o.findings) @@ -96,65 +40,31 @@ func TestAPutWithoutHoldThatStandsIsPutStands(t *testing.T) { } } -// design 47 G §499, §541 (WM4, WM5): a node-engine's hold of the composition standing 30 minutes is `delivery-waits` naming the node, -// the module and the reason, a wait for a kept build standing 60 minutes is `kept-build-waits`, and `wait-ended` clears either. +// design 47 G §499, §541 (WM5): a node-engine's wait for a kept build standing 60 minutes is `kept-build-waits` naming the node, +// the module and the reason, and `wait-ended` clears it. Design 47 H §764, H-A-91 (2): a hold of the composition is no finding +// (delivery-waits retired). func TestANodeEnginesWaitIsAFindingPastItsBound(t *testing.T) { held := func(step, reason string, since time.Time) []byte { b, _ := json.Marshal(nodeengine.Wait{Step: step, Reason: reason, Node: "build1", Since: since, Owner: "node-engine"}) return b } - for _, tc := range []struct { - name, step, reason, kind string - bound time.Duration - }{ - {"a hold of the composition", nodeengine.WaitStepCompose, "restore point missing", KindDeliveryWaits, 30 * time.Minute}, - {"a wait for a kept build", nodeengine.WaitStepKeptBuild, "node away", KindKeptBuildWaits, 60 * time.Minute}, - } { - t.Run(tc.name, func(t *testing.T) { - w, o, c := setup() - id := "ace.web" - w.EngineWaitHeld("ace", "web", held(tc.step, tc.reason, c.t)) - if o.ttls[signalKey(engineRow, id)] != tc.bound || len(o.findings) != 0 { - t.Fatalf("within the bound: ttl %v, findings %v", o.ttls[signalKey(engineRow, id)], o.findings) - } - expire(w, c, signalKey(engineRow, id), tc.bound) - f, ok := o.findings[FindingKey(ScopeDelivery, id, tc.kind)] - if !ok || len(o.findings) != 1 || !strings.Contains(f.Summary, tc.reason) { - t.Fatalf("past its bound: %v", o.findings) - } - w.EngineWaitEnded("ace", "web") - if len(o.findings) != 0 { - t.Fatalf("wait-ended left %v", o.findings) - } - }) - } -} - -// design 47 G A-192 (WM13): a key queued behind a package is a wait the builder says on its own stream with the package as its part; -// past its bound it is `acting-part-late`, `wait-ended` clears it, and a toolchain wait (no part) is left to the builder. -func TestAKeyQueuedBehindAPackageIsAnActingPartsWait(t *testing.T) { - body := func(part string, since time.Time) []byte { - b, _ := json.Marshal(meshwants.Wait{Step: "building", Part: part, On: "package:" + part, Module: "web", Fingerprint: "src2:f", Since: since, Bound: "1h0m0s"}) - return b - } - const holder = "builder" w, o, c := setup() - id := waitID(holder, meshwants.Wait{Step: "building", Part: "mesh-sdk@v0.1.14", Module: "web"}) - w.ModuleWaitHeld(holder, body("mesh-sdk@v0.1.14", c.t)) - if o.ttls[signalKey(actingRow, id)] != time.Hour || len(o.findings) != 0 { - t.Fatalf("within the bound: ttl %v, findings %v", o.ttls[signalKey(actingRow, id)], o.findings) + id := "ace.web" + w.EngineWaitHeld("ace", "web", held(nodeengine.WaitStepCompose, "restore point missing", c.t.Add(-time.Hour))) + if len(o.ttls) != 0 || len(o.findings) != 0 { + t.Fatalf("a hold of the composition is timed: ttls %v, findings %v", o.ttls, o.findings) } - expire(w, c, signalKey(actingRow, id), time.Hour) - if _, ok := o.findings[FindingKey(ScopeDelivery, id, KindActingPartLate)]; !ok || len(o.findings) != 1 { + w.EngineWaitHeld("ace", "web", held(nodeengine.WaitStepKeptBuild, "node away", c.t)) + if o.ttls[signalKey(engineRow, id)] != KeptBuildBound || len(o.findings) != 0 { + t.Fatalf("within the bound: ttl %v, findings %v", o.ttls[signalKey(engineRow, id)], o.findings) + } + expire(w, c, signalKey(engineRow, id), KeptBuildBound) + f, ok := o.findings[FindingKey(ScopeDelivery, id, KindKeptBuildWaits)] + if !ok || len(o.findings) != 1 || !strings.Contains(f.Summary, "node away") { t.Fatalf("past its bound: %v", o.findings) } - w.ModuleWaitEnded(holder, body("mesh-sdk@v0.1.14", c.t)) + w.EngineWaitEnded("ace", "web") if len(o.findings) != 0 { t.Fatalf("wait-ended left %v", o.findings) } - w, o, c = setup() - w.ModuleWaitHeld(holder, body("", c.t.Add(-2*time.Hour))) - if len(o.findings) != 0 || len(o.ttls) != 0 { - t.Fatalf("a toolchain wait was timed by the watchman: %v %v", o.ttls, o.findings) - } } diff --git a/modules/mesh-delivery-watchman/internal/watch/watch.go b/modules/mesh-delivery-watchman/internal/watch/watch.go index f65d75b2..dbc256f5 100644 --- a/modules/mesh-delivery-watchman/internal/watch/watch.go +++ b/modules/mesh-delivery-watchman/internal/watch/watch.go @@ -70,7 +70,6 @@ type Watchman struct { haves map[string]string // the fingerprint each node runs of a module, by have key, from the node-engine's running // starts is, by pulse key, the first start of a wait whose mark still stands (mirrorKept). starts map[string]startMark - held map[string]bool // each want key held by a rule (ADR 0603 §8), mirrored as the hold's wait bound time.Duration // findings is every finding written, by key, as written; seededFindings those read at the start and not yet observed // again. @@ -111,13 +110,15 @@ type Watchman struct { // (WantsAt, design 47 G A-321); awaitsWants keeps the seeded entries of any `_wants` bucket until the late read. wantsAt func() string awaitsWants bool - // heads is, by watched branch, the head still waiting for its pipeline (pipelines.go). - heads map[string]head - // pendingBranch is, by key, the watched branch of each open take-in wait that names one (takein.go, design 47 G §497). - pendingBranch map[string]string - // takenEntries are the modules of each taken-in revision, by `.`, and recordMilestone each open record's - // milestone, which joins a record to them (records.go, design 47 G §502). - takenEntries map[string][]string + // takeInBound is the setting take-in-bound (takein.go). + takeInBound time.Duration + // entries reads a taken-in mover's catalog entry by its fingerprint, and movers are the movers whose entry is asked, by + // fingerprint (busstate.go). + entries EntryReader + movers map[string]moverTaken + // taken is each taken-in source revision, by `.`, and recordMilestone each open record's milestone, which + // joins a record to it (records.go, design 47 G §502). + taken map[string]bool recordMilestone map[string]string // queued is, by build seat, each step queued on it for a build node and since when (pipelines.go). queued map[string]map[string]time.Time @@ -132,12 +133,12 @@ type Watchman struct { // New is the watchman of module (its credential's), writing to out, reading the time from now. func New(module string, out Out, now func() time.Time) *Watchman { return &Watchman{module: module, own: BucketsOf(module), out: out, now: now, counts: map[string]int{}, presence: map[string]presence{}, - pulses: map[string]*pulse{}, stuck: map[string]meshwatchdogsclient.Stuck{}, wants: map[string]string{}, haves: map[string]string{}, held: map[string]bool{}, starts: map[string]startMark{}, bound: meshwatchdogsclient.PulseMoveBound, stepBound: StepBound, progressBound: ProgressBound, - findings: map[string][]byte{}, seededFindings: map[string]bool{}, signals: map[string]signalDue{}, busCreated: map[string]bool{}, takeInHeld: map[string]Finding{}, pendingBranch: map[string]string{}, takenEntries: map[string][]string{}, recordMilestone: map[string]string{}, away: map[string]bool{}, + pulses: map[string]*pulse{}, stuck: map[string]meshwatchdogsclient.Stuck{}, wants: map[string]string{}, haves: map[string]string{}, starts: map[string]startMark{}, bound: meshwatchdogsclient.PulseMoveBound, stepBound: StepBound, progressBound: ProgressBound, + findings: map[string][]byte{}, seededFindings: map[string]bool{}, signals: map[string]signalDue{}, busCreated: map[string]bool{}, takeInHeld: map[string]Finding{}, takeInBound: DefaultTakeInBound, movers: map[string]moverTaken{}, taken: map[string]bool{}, recordMilestone: map[string]string{}, away: map[string]bool{}, advised: map[string]int{}, advisedSince: map[string]time.Time{}, provFailed: map[string]map[string]ProvReport{}, closures: map[string]map[string]time.Time{}, waiting: map[string]map[string]string{}, - standing: map[string]map[string]bool{}, readWhole: map[string]bool{}, awaited: map[string]bool{}, heads: map[string]head{}, queued: map[string]map[string]time.Time{}, steps: map[string]*openStep{}, awaySaid: map[string]time.Time{}, opened: map[string]Finding{}} + standing: map[string]map[string]bool{}, readWhole: map[string]bool{}, awaited: map[string]bool{}, queued: map[string]map[string]time.Time{}, steps: map[string]*openStep{}, awaySaid: map[string]time.Time{}, opened: map[string]Finding{}} } // Logf sets where a write that failed is said; it is silent until set. diff --git a/modules/mesh-delivery-watchman/internal/watch/watch_test.go b/modules/mesh-delivery-watchman/internal/watch/watch_test.go index 9550a9bf..b16033b5 100644 --- a/modules/mesh-delivery-watchman/internal/watch/watch_test.go +++ b/modules/mesh-delivery-watchman/internal/watch/watch_test.go @@ -605,9 +605,9 @@ func TestARunningStepPastItsBoundIsStuck(t *testing.T) { // 1908 // its fingerprint. func wantValue(t *testing.T, fp string, removed bool) []byte { t.Helper() - w := meshwantsclient.Want{Fp: fp, Composed: "sha256:" + strings.Repeat("c", 64), Removed: removed} + w := meshwantsclient.Want{Fp: fp, Version: "0.1.4", Pipeline: "main.42", Rank: 1, Removed: removed} // the H1 want (H-A-37) if removed { - w.Composed = "" + w = meshwantsclient.Want{Fp: fp, Removed: true} } b, err := meshwantsclient.EncodeWant(w) if err != nil { @@ -720,11 +720,11 @@ func TestAStuckKeyWhoseKeyIsGoneIsClosedAtTheStart(t *testing.T) { // A pulse of another key of the bucket, and a finding the bucket is the source of, stood at the start. pk := pulseKeyOf(t, row.bucket, "want.n9.gone") w.SeedPulse(pk) - fv, err := EncodeFinding(Finding{Scope: ScopeModule, ID: "gone", Kind: KindManifestRefused, Severity: Warning, Source: row.bucket, Summary: "refused", Since: t0}) + fv, err := EncodeFinding(Finding{Scope: ScopeModule, ID: "gone", Kind: KindStuck, Severity: Warning, Source: row.bucket, Summary: "refused", Since: t0}) if err != nil { t.Fatal(err) } - fk := FindingKey(ScopeModule, "gone", KindManifestRefused) + fk := FindingKey(ScopeModule, "gone", KindStuck) o.findings[fk], _ = DecodeFinding(fv) w.SeedFinding(fk, fv) w.AwaitsWants() diff --git a/modules/mesh-delivery-watchman/manifest_test.go b/modules/mesh-delivery-watchman/manifest_test.go index 0e53eb8a..23f9209d 100644 --- a/modules/mesh-delivery-watchman/manifest_test.go +++ b/modules/mesh-delivery-watchman/manifest_test.go @@ -64,6 +64,12 @@ type manifest struct { MaxAgeDays int `json:"max-age-days"` MaxMiB int `json:"max-mib"` } `json:"streams"` + Config map[string]struct { + Type string `json:"type"` + Description string `json:"description"` + Trusted bool `json:"trusted"` + Default string `json:"default"` + } `json:"config"` OwnSecrets map[string]string `json:"own-secrets"` SecretsOwner string `json:"secrets-owner"` Resources []map[string]any `json:"resources"` @@ -127,12 +133,17 @@ func TestManifestStateAndReads(t *testing.T) { // 1135/AC6 } // The join: its first view asked again by hand is a hand act, recorded through the seat (views.StatusTools). The waits the // indexer holds at the watchman's start are asked of its tool (ADR 0633 §5), a verb edge as the view's. - if want := []string{"mesh-delivery-indexer.indexer_pending", "mesh-delivery-indexer.indexer_verdicts", "seat:mesh-hand-acts.hand-act", "seat:mesh-conditions.conditions", "seat:mesh-broker.show", "seat:mesh-catalog.revision", "seat:mesh-catalog.registrations", "seat:mesh-delivery.source", "seat:mesh-pipeline-runner.stands"}; !reflect.DeepEqual(m.Invokes, want) { + // Design 47 H-A-79, H-A-94 (1): no indexer_verdicts; a mover's entry is read through the seat mesh-catalog's verb entry. + if want := []string{"mesh-delivery-indexer.indexer_pending", "seat:mesh-hand-acts.hand-act", "seat:mesh-conditions.conditions", "seat:mesh-broker.show", "seat:mesh-catalog.revision", "seat:mesh-catalog.registrations", "seat:mesh-catalog.entry", "seat:mesh-delivery.source", "seat:mesh-pipeline-runner.stands"}; !reflect.DeepEqual(m.Invokes, want) { t.Errorf("invokes = %v, want %v", m.Invokes, want) } // DLV-M5 its first views, 1585/T20 the indexer's take-in events, and DLV-W1 (ADR 0595 §4): the accepted verdicts and the bus module's created and refused, durably. - if want := []string{"seat:mesh-wants.views.wants", "seat:mesh-module-assigner.views.seat-holders", "mesh-catalog.registration-changed", "mesh-catalog.catalog-revision", "mesh-delivery-indexer.take-in.pending", "mesh-delivery-indexer.take-in.taken", "mesh-catalog.manifest-accepted", "mesh-catalog.manifest-refused", "mesh-delivery.source-moved", "mesh-delivery.filling", "mesh-delivery.full", "mesh-delivery.room", "mesh-delivery.record-opened", "mesh-delivery.record-ended", "mesh-catalog.filling", "mesh-catalog.full", "mesh-catalog.room", "mesh-wants.filling", "mesh-wants.full", "mesh-wants.room", "mesh-grants.filling", "mesh-grants.full", "mesh-grants.room", "node-build-go.ask-taken", "node-build-go.step-progress", "node-build-go.step-ended", "node-build-image.ask-taken", "node-build-image.step-progress", "node-build-image.step-ended", "node-build-node.ask-taken", "node-build-node.step-progress", "node-build-node.step-ended", "mesh-pipeline-runner.pipeline-started", "mesh-pipeline-runner.step-started", "mesh-pipeline-runner.step-ended", "mesh-wants.step-progress", "mesh-pipeline-runner.wait-held", "mesh-pipeline-runner.wait-ended", "mesh-wants.wait-held", "mesh-wants.wait-ended", "mesh-grants.wait-held", "mesh-grants.wait-ended", "mesh-delivery-builder.wait-held", "mesh-delivery-builder.wait-ended", "node-engine.wait-held", "node-engine.wait-ended", "node-engine.running", "mesh-broker.created", "mesh-broker.refused", "mesh-broker.recreated", "mesh-broker.dropped", "mesh-conditions.findings-asked"}; !reflect.DeepEqual(m.Consumes, want) { - t.Errorf("consumes = %v, want %v", m.Consumes, want) + if want := []string{"seat:mesh-wants.views.wants", "seat:mesh-module-assigner.views.seat-holders", "mesh-catalog.registration-changed", "mesh-catalog.catalog-revision", "mesh-delivery-indexer.take-in.pending", "mesh-delivery-indexer.take-in.taken", "mesh-delivery.source-moved", "mesh-delivery.filling", "mesh-delivery.full", "mesh-delivery.room", "mesh-delivery.record-opened", "mesh-delivery.record-ended", "mesh-catalog.filling", "mesh-catalog.full", "mesh-catalog.room", "mesh-wants.filling", "mesh-wants.full", "mesh-wants.room", "mesh-grants.filling", "mesh-grants.full", "mesh-grants.room", "node-build-go.ask-taken", "node-build-go.step-progress", "node-build-go.step-ended", "node-build-image.ask-taken", "node-build-image.step-progress", "node-build-image.step-ended", "node-build-node.ask-taken", "node-build-node.step-progress", "node-build-node.step-ended", "mesh-pipeline-runner.step-started", "mesh-pipeline-runner.step-ended", "mesh-wants.step-progress", "mesh-wants.wait-held", "mesh-wants.wait-ended", "node-engine.wait-held", "node-engine.wait-ended", "node-engine.running", "mesh-broker.created", "mesh-broker.refused", "mesh-broker.recreated", "mesh-broker.dropped", "mesh-conditions.findings-asked"}; !reflect.DeepEqual(m.Consumes, want) { + t.Errorf("consumes = %v, want %v", m.Consumes, want) // design 47 H §741, §764, H-A-49, H-A-91 (2) + } + // Design 47 H §744, H-A-94 (2): the setting take-in-bound, 60 minutes by default, reaches the process as a file. + if c := m.Config["take-in-bound"]; c.Type != "string" || c.Default != "60m" { + t.Errorf("config take-in-bound = %+v, want a string defaulting to 60m", c) } // 1952/WM1: the watchman holds the seat mesh-watchdogs at mesh scope, its first seat (novox/hq ADR 0564 §1, §4). if !reflect.DeepEqual(m.Claims, []claim{{Name: "mesh-watchdogs", Scope: "mesh", Serves: []string{"signals", "stuck", "presence", "dead-letters", "resync"}}}) { @@ -228,6 +239,19 @@ func TestManifestResources(t *testing.T) { // 1135/AC6 if env["MESH_BROKER_FILE"] != "${dir:state}/broker" { t.Errorf("the process reads %v", env["MESH_BROKER_FILE"]) } + // Design 47 H §744: the setting take-in-bound's file, which the process reads at its start. + if env["MESH_WATCHMAN_TAKE_IN_BOUND_FILE"] != "${dir:state}/take-in-bound" { + t.Errorf("the process reads the take-in bound from %v", env["MESH_WATCHMAN_TAKE_IN_BOUND_FILE"]) + } + var boundFile map[string]any + for _, f := range byType["file"] { + if f["id"] == "take-in-bound" { + boundFile = f + } + } + if boundFile["path"] != "${dir:state}/take-in-bound" || boundFile["content"] != "${config:take-in-bound}" || boundFile["owner"] != user { + t.Errorf("the take-in bound's file = %v", boundFile) + } if h, _ := p["health"].(map[string]any); h["kind"] != "unit" { t.Errorf("health = %v", p["health"]) } diff --git a/modules/mesh-delivery-watchman/module.json b/modules/mesh-delivery-watchman/module.json index b3078253..2c6665b0 100644 --- a/modules/mesh-delivery-watchman/module.json +++ b/modules/mesh-delivery-watchman/module.json @@ -20,12 +20,12 @@ ], "invokes": [ "mesh-delivery-indexer.indexer_pending", - "mesh-delivery-indexer.indexer_verdicts", "seat:mesh-hand-acts.hand-act", "seat:mesh-conditions.conditions", "seat:mesh-broker.show", "seat:mesh-catalog.revision", "seat:mesh-catalog.registrations", + "seat:mesh-catalog.entry", "seat:mesh-delivery.source", "seat:mesh-pipeline-runner.stands" ], @@ -36,8 +36,6 @@ "mesh-catalog.catalog-revision", "mesh-delivery-indexer.take-in.pending", "mesh-delivery-indexer.take-in.taken", - "mesh-catalog.manifest-accepted", - "mesh-catalog.manifest-refused", "mesh-delivery.source-moved", "mesh-delivery.filling", "mesh-delivery.full", @@ -62,18 +60,11 @@ "node-build-node.ask-taken", "node-build-node.step-progress", "node-build-node.step-ended", - "mesh-pipeline-runner.pipeline-started", "mesh-pipeline-runner.step-started", "mesh-pipeline-runner.step-ended", "mesh-wants.step-progress", - "mesh-pipeline-runner.wait-held", - "mesh-pipeline-runner.wait-ended", "mesh-wants.wait-held", "mesh-wants.wait-ended", - "mesh-grants.wait-held", - "mesh-grants.wait-ended", - "mesh-delivery-builder.wait-held", - "mesh-delivery-builder.wait-ended", "node-engine.wait-held", "node-engine.wait-ended", "node-engine.running", @@ -111,6 +102,14 @@ "broker": "${dir:state}/broker" }, "secrets-owner": "mesh-delivery-watchman", + "config": { + "take-in-bound": { + "type": "string", + "description": "how long a source revision's take-in the indexer holds may wait for its take-in.taken before it is the finding take-in-waits, a Go duration such as 60m, counted from its take-in.pending (design 47 H section 744).", + "trusted": false, + "default": "60m" + } + }, "resources": [ { "id": "account", @@ -127,6 +126,14 @@ "place": ".", "unknown": "program" }, + { + "id": "take-in-bound", + "type": "file", + "path": "${dir:state}/take-in-bound", + "owner": "mesh-delivery-watchman", + "mode": "0600", + "content": "${config:take-in-bound}" + }, { "id": "socket-dir", "type": "directory", @@ -144,7 +151,8 @@ "./mesh-delivery-watchman" ], "env": { - "MESH_BROKER_FILE": "${dir:state}/broker" + "MESH_BROKER_FILE": "${dir:state}/broker", + "MESH_WATCHMAN_TAKE_IN_BOUND_FILE": "${dir:state}/take-in-bound" }, "user": "mesh-delivery-watchman", "health": {