W1: the watchman reads the H1 take-in, the mover's entry and the want
The watchman follows the one model in one pass. Its take-in waits are the seat's take-in.pending and take-in.taken, keyed <branch>.<source>, timed from the message's own time by the setting take-in-bound (60 minutes by default), and seeded from indexer_pending's held take-ins; the group bound goes. A taken-in mover's bus state is read from its catalog entry by fingerprint through the seat mesh-catalog's verb entry, bus-state-overdue timed from the take-in. The manifest verdicts and their findings, the head timing of source-moved, acting-part-late and delivery-waits go, with their readers, consumes and the indexer_verdicts invoke. A want awaiting its hand placement is not timed, and the want readers follow the H1 want. A record-opened is heard on record-opened.<call>. Builds: design 47 H §741, §744, §764 (the indexer: manifest-refused, manifest-accepted, the watchman's findings; the assigner: acting-part-late, delivery-waits), §772; lead answers H-A-25, H-A-37, H-A-49, H-A-70, H-A-78 (2), H-A-79, H-A-91 (2), H-A-94, H-A-95 (1), H-A-97, H-A-102 Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy
This commit is contained in:
@@ -2,7 +2,6 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/meshbroker"
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
@@ -26,8 +25,8 @@ import (
|
||||
"mesh-delivery-watchman/internal/watch"
|
||||
)
|
||||
|
||||
// **What the watchman consumes** (its manifest's `consumes`; novox/hq ADR 0595 §4, design 25 § 12.2; DLV-W1): the seat mesh-catalog's
|
||||
// manifest-accepted and the seat mesh-broker's created and refused, all on its durable consumer on EVENTS (the seat's
|
||||
// **What the watchman consumes** (its manifest's `consumes`; novox/hq ADR 0595 §4, design 25 § 12.2; DLV-W1): the seat mesh-broker's
|
||||
// created and refused (a taken-in mover's bus state is read from its catalog entry, busstate.go), all on its durable consumer on EVENTS (the seat's
|
||||
// events on the seat's subjects like every seat's: the lead's ruling, registry milestone 13:0x); beside it any module
|
||||
// stream its membership names (Hears). Each consumer is the one the grants part makes; one
|
||||
// not there yet is opened when the watchman's membership next changes, and one opened is never opened twice. An event is
|
||||
@@ -35,35 +34,18 @@ import (
|
||||
|
||||
// Consumed event names, as the consumes name them (the emitter is read from the subject's emitter token).
|
||||
const (
|
||||
eventAccepted = "manifest-accepted"
|
||||
// eventManifestRefused is the seat mesh-catalog's manifest-refused; meshGrantsSeat the seat of the store whose filling, full, room and waits it hears.
|
||||
eventManifestRefused = "manifest-refused"
|
||||
meshGrantsSeat = "mesh-grants"
|
||||
eventCreated = "created"
|
||||
eventRefused = "refused"
|
||||
// meshGrantsSeat is the seat of the store whose filling, full and room it hears.
|
||||
meshGrantsSeat = "mesh-grants"
|
||||
eventCreated = "created"
|
||||
eventRefused = "refused"
|
||||
)
|
||||
|
||||
// consumed hands one consumed event to the watchman, by its subject: manifest-accepted, and the seat mesh-broker's created
|
||||
// and refused. Anything else is left.
|
||||
// consumed hands one consumed event to the watchman, by its subject: the seats' events it times, and the seat mesh-broker's
|
||||
// created and refused. Anything else is left.
|
||||
func consumed(w *watch.Watchman, subject string, body []byte, logf func(string, ...any)) {
|
||||
switch {
|
||||
case subject == meshcatalog.SubjectManifestAccepted:
|
||||
var a meshcatalog.ManifestAcceptedEvent
|
||||
if err := json.Unmarshal(body, &a); err != nil || a.Module == "" {
|
||||
logf("a manifest-accepted that does not read is left: %v", err)
|
||||
return
|
||||
}
|
||||
w.Accepted(a)
|
||||
case subject == meshcatalog.SubjectManifestRefused:
|
||||
var r meshcatalog.ManifestRefusedEvent
|
||||
if err := json.Unmarshal(body, &r); err != nil || r.Module == "" {
|
||||
logf("a manifest-refused that does not read is left: %v", err)
|
||||
return
|
||||
}
|
||||
w.ManifestRefused(r)
|
||||
case pipelineSubject(w, subject, body):
|
||||
case engineWaitSubject(w, subject, body):
|
||||
case moduleWaitSubject(w, subject, body):
|
||||
case recordSubject(w, subject, body):
|
||||
case buildSeatSubject(w, subject, body):
|
||||
case brokerSubject(subject) && strings.HasSuffix(subject, ".event."+eventCreated):
|
||||
@@ -84,7 +66,7 @@ func consumed(w *watch.Watchman, subject string, body []byte, logf func(string,
|
||||
// hearConsumed opens the watchman's durable consumers: on EVENTS now, and on each module stream its membership names as
|
||||
// that membership arrives. A consumer the bus does not hold yet is said and tried again at the membership's next change.
|
||||
func hearConsumed(ctx context.Context, js jetstream.JetStream, cred process.Account, hub *process.EventHub, w *watch.Watchman, binding *process.Binding, logf func(string, ...any)) {
|
||||
// The seats' events (manifest-accepted, the bus module's created and refused) are one handler of the module's one
|
||||
// The seats' events (the bus module's created and refused among them) are one handler of the module's one
|
||||
// pull of EVENTS, beside the take-in events and the views reader. The seat mesh-broker's recreated and dropped go to
|
||||
// the binding of the watchman's finding-events (design 47 G A-274, §577), which this handler holds, since a message has
|
||||
// one taker; and the binding asks show again at each bind of the hub's consumer, as its Register does.
|
||||
@@ -99,16 +81,17 @@ func hearConsumed(ctx context.Context, js jetstream.JetStream, cred process.Acco
|
||||
_ = m.Ack()
|
||||
}
|
||||
// One handler per seat it consumes, since a message has one taker (design 47 G A-322): the events the branches follower of the
|
||||
// catalog half adopts (the seat mesh-catalog's registration-changed, the seat mesh-delivery's source-moved) are the follower's, and
|
||||
// the watchman takes source-moved from it (AdoptEvent, main.go), never from a second handler.
|
||||
// catalog half adopts (the seat mesh-catalog's registration-changed, the seat mesh-delivery's source-moved) are the follower's
|
||||
// alone: the watchman learns no head (design 47 H §741).
|
||||
var patterns []string
|
||||
for _, seat := range []string{meshwants.Seat, meshGrantsSeat, meshpipelinerunner.Seat, nodeengine.Seat, meshbroker.Seat, nodebuildgo.Seat, nodebuildimage.Seat, nodebuildnode.Seat} {
|
||||
patterns = append(patterns, "mesh.seat."+seat+".event.>")
|
||||
}
|
||||
for _, e := range []string{meshdelivery.EventRecordOpened, meshdelivery.EventRecordEnded, meshdelivery.EventFilling, meshdelivery.EventFull, meshdelivery.EventRoom} {
|
||||
patterns = append(patterns, meshdelivery.SubjectRecordOpenedAll) // record-opened.<call> (design 47 H-A-25)
|
||||
for _, e := range []string{meshdelivery.EventRecordEnded, meshdelivery.EventFilling, meshdelivery.EventFull, meshdelivery.EventRoom} {
|
||||
patterns = append(patterns, seatdef.EventSubject(meshdelivery.Seat, e))
|
||||
}
|
||||
for _, e := range []string{eventAccepted, eventManifestRefused, meshdelivery.EventFilling, meshdelivery.EventFull, meshdelivery.EventRoom} {
|
||||
for _, e := range []string{meshdelivery.EventFilling, meshdelivery.EventFull, meshdelivery.EventRoom} {
|
||||
patterns = append(patterns, seatdef.EventSubject(meshcatalog.Seat, e))
|
||||
}
|
||||
for _, pattern := range patterns {
|
||||
@@ -190,15 +173,10 @@ func brokerSubject(subject string) bool {
|
||||
return strings.HasPrefix(subject, "mesh.seat."+meshbroker.Seat+".event.")
|
||||
}
|
||||
|
||||
// pipelineSubject hands the gate's `source-moved`, the runner's `pipeline-started`, `step-started` and `step-ended` and the assigner's
|
||||
// `step-progress` to the watchman (watch/pipelines.go, watch/steps.go, design 47 G §12, §732): the head of a watched branch, its
|
||||
// pipeline and each step's clock. It says whether the subject was one of them.
|
||||
// pipelineSubject hands the runner's `step-started` and `step-ended` and the assigner's `step-progress` to the watchman
|
||||
// (watch/pipelines.go, watch/steps.go, design 47 G §732, H §741): each step's clock. It says whether the subject was one of them.
|
||||
func pipelineSubject(w *watch.Watchman, subject string, body []byte) bool {
|
||||
switch {
|
||||
case strings.HasPrefix(subject, "mesh.seat."+meshdelivery.Seat+".event."+meshdelivery.EventSourceMoved+"."):
|
||||
w.SourceMoved(body)
|
||||
case strings.HasPrefix(subject, "mesh.seat."+meshpipelinerunner.Seat+".event."+meshpipelinerunner.EventPipelineStarted+"."):
|
||||
w.PipelineStarted(body)
|
||||
case strings.HasPrefix(subject, "mesh.seat."+meshpipelinerunner.Seat+".event."+meshpipelinerunner.EventStepStarted+"."):
|
||||
w.StepEvent(body)
|
||||
w.StepStarted(body)
|
||||
@@ -217,10 +195,14 @@ func pipelineSubject(w *watch.Watchman, subject string, body []byte) bool {
|
||||
// `record-ended` to the watchman (watch/records.go, design 47 G §148, §181). It says whether the subject was one of them.
|
||||
func recordSubject(w *watch.Watchman, subject string, body []byte) bool {
|
||||
parts := strings.Split(subject, ".")
|
||||
if len(parts) != 5 || parts[0] != "mesh" || parts[1] != "seat" || parts[3] != "event" {
|
||||
if len(parts) < 5 || parts[0] != "mesh" || parts[1] != "seat" || parts[3] != "event" {
|
||||
return false
|
||||
}
|
||||
seat, event := parts[2], parts[4]
|
||||
// record-opened is said on record-opened.<call> (design 47 H-A-25); every other event here has no token after its name.
|
||||
if want := map[bool]int{true: 6, false: 5}[seat == meshdelivery.Seat && event == meshdelivery.EventRecordOpened]; len(parts) != want {
|
||||
return false
|
||||
}
|
||||
switch {
|
||||
case event == meshwants.EventWaitHeld:
|
||||
w.WaitHeld(seat, body)
|
||||
@@ -258,25 +240,6 @@ func engineWaitSubject(w *watch.Watchman, subject string, body []byte) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// moduleWaitSubject hands a module's `wait-held` and `wait-ended` on its own stream, on
|
||||
// mesh.stream.<module>.event.<event>.<module>.<fingerprint>, to the watchman: the form the builder says a key queued behind a package
|
||||
// in (design 47 G A-192). It says whether the subject was one.
|
||||
func moduleWaitSubject(w *watch.Watchman, subject string, body []byte) bool {
|
||||
parts := strings.Split(subject, ".")
|
||||
if len(parts) != 7 || parts[0] != "mesh" || parts[1] != "stream" || parts[3] != "event" {
|
||||
return false
|
||||
}
|
||||
switch parts[4] {
|
||||
case nodeengine.EventWaitHeld:
|
||||
w.ModuleWaitHeld(parts[2], body)
|
||||
case nodeengine.EventWaitEnded:
|
||||
w.ModuleWaitEnded(parts[2], body)
|
||||
default:
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// buildSeatSubject hands the build seats' `ask-taken`, `step-progress` and `step-ended`, on
|
||||
// mesh.seat.<seat>.event.<event>.<node>, to the watchman (design 47 G §483, §496, §602, A-136): it times every ask of a
|
||||
// build seat from them. It says whether the subject was one of them.
|
||||
|
||||
@@ -578,7 +578,7 @@ func aWantIsMirroredAsAPulseAndTheHaveNamingItEndsIt(t *testing.T) {
|
||||
fpOld := meshcatalogclient.SourcePrefix2 + strings.Repeat("a", 64)
|
||||
putWant := func(node, module, fp string) {
|
||||
t.Helper()
|
||||
body, err := meshwantsclient.EncodeWant(meshwantsclient.Want{Fp: fp, Composed: "sha256:" + strings.Repeat("c", 64)})
|
||||
body, err := meshwantsclient.EncodeWant(meshwantsclient.Want{Fp: fp, Version: "0.1.4", Pipeline: "main.42", Rank: 1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -12,10 +12,10 @@ import (
|
||||
"fmt"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
"io"
|
||||
"io/fs"
|
||||
"log"
|
||||
"os"
|
||||
"os/signal"
|
||||
@@ -31,7 +31,6 @@ import (
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/meshbroker"
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshwatchdogsclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
|
||||
|
||||
"mesh-delivery-watchman/internal/socket"
|
||||
"mesh-delivery-watchman/internal/watch"
|
||||
@@ -274,11 +273,17 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
|
||||
w := watch.New(cred.Module, &kvOut{ctx: ctx, presence: kvs.presence, pulse: kvs.pulse, stuk: kvs.stuck, findings: held, js: js, cred: cred}, time.Now)
|
||||
w.Logf(logger.Printf)
|
||||
// The seat mesh-delivery's source-moved is the branches follower's event (one taker, design 47 G A-322): the watchman takes it from the
|
||||
// follower's subscription, beside the follower's handler.
|
||||
if err := reader.AdoptEvent(meshdelivery.Seat, meshdelivery.EventSourceMoved, func(_ context.Context, data []byte) error { w.SourceMoved(data); return nil }); err != nil {
|
||||
// A taken-in mover's catalog entry is read by its fingerprint through the seat mesh-catalog's verb entry (design 47
|
||||
// H-A-94 (1)), its answer heard on its own goroutine.
|
||||
entries := meshcatalogclient.NewEntries(calls.Call)
|
||||
entries.Changed = w.EntryChanged
|
||||
w.ReadEntriesWith(entries)
|
||||
// The setting take-in-bound (design 47 H §744, H-A-94 (2)), read at the start from the file the manifest declares.
|
||||
bound, err := takeInBound(getenv)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
w.SetTakeInBound(bound)
|
||||
// What the wants and the modules hold is ended by their own late read (watch.ReadLate), never by the start's end.
|
||||
w.WantsAt(func() string { return reader.HolderBucket(meshwants.Seat, meshwants.StateWants) })
|
||||
w.AwaitsWants()
|
||||
@@ -400,7 +405,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
} else {
|
||||
atCatalog := atHolder(calls.Call, holders, meshcatalog.Seat) // the indexer holds the seat mesh-catalog
|
||||
seedTakeIn(w, atCatalog, logger.Printf)
|
||||
seedVerdicts(w, atCatalog, logger.Printf)
|
||||
seedSteps(w, calls.Call, logger.Printf)
|
||||
calls.Close()
|
||||
}
|
||||
@@ -476,6 +480,32 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
}
|
||||
}
|
||||
|
||||
// takeInBound is the setting take-in-bound, a Go duration such as 60m, read from the file the manifest declares with the
|
||||
// content ${config:take-in-bound}, which MESH_WATCHMAN_TAKE_IN_BOUND_FILE names; zero, its default, for no path, no file or an
|
||||
// empty one, and an error naming the file for a content that is no positive duration.
|
||||
func takeInBound(getenv func(string) string) (time.Duration, error) {
|
||||
path := getenv("MESH_WATCHMAN_TAKE_IN_BOUND_FILE")
|
||||
if path == "" {
|
||||
return 0, nil
|
||||
}
|
||||
raw, err := os.ReadFile(path)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return 0, nil
|
||||
}
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
v := strings.TrimSpace(string(raw))
|
||||
if v == "" {
|
||||
return 0, nil
|
||||
}
|
||||
d, err := time.ParseDuration(v)
|
||||
if err != nil || d <= 0 {
|
||||
return 0, fmt.Errorf("the setting take-in-bound in %s is %q, no positive duration such as 60m", path, v)
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
|
||||
// eventsStream is the shared stream of the modules' ordinary events, as the bus names it.
|
||||
const eventsStream = "EVENTS"
|
||||
|
||||
@@ -499,10 +529,10 @@ func atHolder(call func(key string, body any) (json.RawMessage, error), holders
|
||||
func seedTakeIn(w *watch.Watchman, call func(key string, body any) (json.RawMessage, error), logf func(string, ...any)) {
|
||||
raw, err := call(pendingTool, map[string]any{})
|
||||
if err == nil {
|
||||
var a takein.PendingAnswer
|
||||
if a, err = takein.DecodePendingAnswer(raw); err == nil {
|
||||
var a watch.HeldTakeIns
|
||||
if a, err = watch.DecodeHeldTakeIns(raw); err == nil {
|
||||
w.SeedTakeIn(a)
|
||||
logf("take-in: %d waits the indexer holds are watched from their own since (%s)", len(a.Waits), pendingTool)
|
||||
logf("take-in: %d take-ins the indexer holds are watched from their own since (%s)", len(a.Held), pendingTool)
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -527,20 +557,6 @@ func seedSteps(w *watch.Watchman, call func(key string, body any) (json.RawMessa
|
||||
logf("steps: %s cannot be asked (%v), so a step open before this start is timed only once an event of it is heard", standsVerb, err)
|
||||
}
|
||||
|
||||
// seedVerdicts asks the indexer's verdicts and raises the findings they hold (design 47 G §500): a verdict said while the watchman was
|
||||
// down is then not lost. An indexer that does not answer is said: the findings are then raised from the verdicts heard after.
|
||||
func seedVerdicts(w *watch.Watchman, call func(key string, body any) (json.RawMessage, error), logf func(string, ...any)) {
|
||||
raw, err := call(watch.VerdictsTool, map[string]any{"all": true})
|
||||
if err == nil {
|
||||
var rows []watch.VerdictRow
|
||||
if rows, err = watch.DecodeVerdictRows(raw); err == nil {
|
||||
w.SeedVerdicts(rows)
|
||||
return
|
||||
}
|
||||
}
|
||||
logf("verdicts: %s cannot be asked (%v), so a manifest refused before this start is said only once a verdict of it is heard", watch.VerdictsTool, err)
|
||||
}
|
||||
|
||||
// takeInHeard hands one take-in event of the indexer to the watchman, by its name relative to the indexer, and acknowledges it;
|
||||
// one that is not a take-in event is said and acknowledged, since hearing it again would not read it.
|
||||
func takeInHeard(w *watch.Watchman, m jetstream.Msg, logf func(string, ...any)) {
|
||||
@@ -548,7 +564,11 @@ func takeInHeard(w *watch.Watchman, m jetstream.Msg, logf func(string, ...any))
|
||||
if !ok {
|
||||
logf("the subject %s of the take-in stream is not an event's, left", m.Subject())
|
||||
} else {
|
||||
w.TakeIn(event, m.Data())
|
||||
at := time.Now()
|
||||
if meta, err := m.Metadata(); err == nil {
|
||||
at = meta.Timestamp // the message's own time (design 47 H-A-94 (2))
|
||||
}
|
||||
w.TakeIn(event, m.Data(), at)
|
||||
}
|
||||
if meta, err := m.Metadata(); err == nil && meta.NumPending == 0 {
|
||||
w.TakeInCaughtUp()
|
||||
|
||||
@@ -5,18 +5,17 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
"io"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
|
||||
|
||||
"mesh-delivery-watchman/internal/watch"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
@@ -67,9 +66,10 @@ func TestRunNeedsItsCredential(t *testing.T) { // 1135/AC6
|
||||
if !out.findings[watch.FindingKey(watch.ScopeModule, "m", watch.KindBusStateRefused+"-l")] {
|
||||
t.Fatalf("refused reached no finding: %v", out.findings)
|
||||
}
|
||||
// Design 47 H §764, H-A-49: manifest-accepted is retired; it is heard by nothing.
|
||||
consumed(w, "mesh.seat.mesh-catalog.event.manifest-accepted", []byte(`{"module":"m","digest":"d","bus-state":[]}`), say)
|
||||
if len(out.findings) != 0 {
|
||||
t.Fatalf("an accepted manifest no longer asking for l left %v", out.findings)
|
||||
if len(out.findings) != 1 {
|
||||
t.Fatalf("a manifest-accepted was heard: %v", out.findings)
|
||||
}
|
||||
consumed(w, "mesh.seat.mesh-broker.event.refused", []byte(`{"module":"m","local":"l"}`), say)
|
||||
consumed(w, "mesh.seat.mesh-broker.event.created", []byte(`{"module":"m","local-name":"l"}`), say)
|
||||
@@ -77,39 +77,36 @@ func TestRunNeedsItsCredential(t *testing.T) { // 1135/AC6
|
||||
t.Fatalf("created left %v", out.findings)
|
||||
}
|
||||
})
|
||||
t.Run("design 47 G §500: the indexer's manifest-refused reaches the watchman by its subject", func(t *testing.T) {
|
||||
t.Run("design 47 H-A-25: a record-opened on record-opened.<call> reaches the watchman by its subject", func(t *testing.T) {
|
||||
out := &findingsOut{findings: map[string]bool{}}
|
||||
w := watch.New(watchmanModule, out, time.Now)
|
||||
consumed(w, "mesh.seat.mesh-catalog.event.manifest-refused", []byte(`{"module":"m","digest":"d","problems":["x"]}`), func(string, ...any) {})
|
||||
if !out.findings[watch.FindingKey(watch.ScopeModule, "m", watch.KindManifestRefused)] {
|
||||
t.Fatalf("a refused manifest reached %v", out.findings)
|
||||
at := time.Now().Add(-25 * time.Hour).UTC().Format(time.RFC3339)
|
||||
consumed(w, "mesh.seat.mesh-delivery.event.record-opened.fedcba9876543210", []byte(`{"record":"r1","milestone":"main.42","branch":"main","source":42,"call":"fedcba9876543210","at":"`+at+`"}`), func(string, ...any) {})
|
||||
if !out.findings[watch.FindingKey(watch.ScopeDelivery, "r1", watch.KindRecordOpen)] {
|
||||
t.Fatalf("a record open 25 hours reached %v", out.findings)
|
||||
}
|
||||
})
|
||||
t.Run("design 47 G §499, §541: a node-engine's wait-held and wait-ended reach the watchman by their subjects", func(t *testing.T) {
|
||||
out := &findingsOut{findings: map[string]bool{}}
|
||||
w := watch.New(watchmanModule, out, time.Now)
|
||||
say := func(string, ...any) {}
|
||||
body := fmt.Sprintf(`{"step":"compose","reason":"entry-missing","since":%q}`, time.Now().Add(-2*time.Hour).UTC().Format(time.RFC3339))
|
||||
body := fmt.Sprintf(`{"step":"kept-build","reason":"node away","node":"build1","since":%q}`, time.Now().Add(-2*time.Hour).UTC().Format(time.RFC3339))
|
||||
consumed(w, "mesh.seat.node-engine.event.wait-held.ace.web", []byte(body), say)
|
||||
if !out.findings[watch.FindingKey(watch.ScopeDelivery, "ace.web", watch.KindDeliveryWaits)] {
|
||||
t.Fatalf("a hold two hours old reached %v", out.findings)
|
||||
if !out.findings[watch.FindingKey(watch.ScopeDelivery, "ace.web", watch.KindKeptBuildWaits)] {
|
||||
t.Fatalf("a wait two hours old reached %v", out.findings)
|
||||
}
|
||||
consumed(w, "mesh.seat.node-engine.event.wait-ended.ace.web", []byte(body), say)
|
||||
if len(out.findings) != 0 {
|
||||
t.Fatalf("wait-ended left %v", out.findings)
|
||||
}
|
||||
})
|
||||
t.Run("design 47 G A-192: a module's wait-held on its stream, naming a part, reaches the watchman", func(t *testing.T) {
|
||||
t.Run("design 47 H §764, H-A-70: a module's wait-held on its stream, naming a part, is an acting part's wait and reaches nothing", func(t *testing.T) {
|
||||
out := &findingsOut{findings: map[string]bool{}}
|
||||
w := watch.New(watchmanModule, out, time.Now)
|
||||
body := fmt.Sprintf(`{"step":"building","part":"mesh-sdk@v0.1.14","module":"web","fingerprint":"src2:f","bound":"1h0m0s","since":%q}`, time.Now().Add(-2*time.Hour).UTC().Format(time.RFC3339))
|
||||
consumed(w, "mesh.stream.mesh-delivery-builder.event.wait-held.web.src2:f", []byte(body), func(string, ...any) {})
|
||||
if len(out.findings) != 1 {
|
||||
t.Fatalf("a package wait two hours old reached %v", out.findings)
|
||||
}
|
||||
consumed(w, "mesh.stream.mesh-delivery-builder.event.wait-ended.web.src2:f", []byte(body), func(string, ...any) {})
|
||||
if len(out.findings) != 0 {
|
||||
t.Fatalf("wait-ended left %v", out.findings)
|
||||
t.Fatalf("a package wait two hours old reached %v", out.findings)
|
||||
}
|
||||
})
|
||||
err := run(context.Background(), func(string) string { return "" }, io.Discard)
|
||||
@@ -119,11 +116,11 @@ func TestRunNeedsItsCredential(t *testing.T) { // 1135/AC6
|
||||
t.Run("1966/AC195-5: a seed indexer_pending could not answer, or answered unreadably, ends nothing", func(t *testing.T) {
|
||||
for name, call := range map[string]func(string, any) (json.RawMessage, error){
|
||||
"cannot be asked": func(string, any) (json.RawMessage, error) { return nil, errors.New("no answer") },
|
||||
"unreadable": func(string, any) (json.RawMessage, error) { return json.RawMessage(`{"waits":`), nil },
|
||||
"unreadable": func(string, any) (json.RawMessage, error) { return json.RawMessage(`{"held":`), nil },
|
||||
} {
|
||||
out := &findingsOut{findings: map[string]bool{}}
|
||||
w := watch.New(watchmanModule, out, time.Now)
|
||||
const key = "m.00000000000000000042"
|
||||
key := watch.TakeInKey("main", 42)
|
||||
old := watch.Finding{Scope: watch.ScopeDelivery, ID: key, Kind: watch.KindTakeInWaits, Severity: watch.Warning, Source: "take-in",
|
||||
Since: time.Now().Add(-3 * time.Hour), Summary: "held", Said: "held", Headline: "held", Explanation: "held", Needs: "a take-in"}
|
||||
value, err := watch.EncodeFinding(old)
|
||||
@@ -132,12 +129,7 @@ func TestRunNeedsItsCredential(t *testing.T) { // 1135/AC6
|
||||
}
|
||||
out.findings[old.Key()] = true
|
||||
w.SeedFinding(old.Key(), value)
|
||||
body, err := takein.EncodeTakeIn(takein.TakeIn{Kind: takein.TakeInPending, Key: "m.00000000000000000043", What: takein.TakeInRevision, Since: time.Now(), Lost: []string{"m.00000000000000000043"},
|
||||
Merges: []takein.Merge{{Repository: "novox/a", Number: 1, Merge: "c0ffee"}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
w.TakeIn(takein.TakeInPendingEvent, body)
|
||||
w.TakeIn(watch.TakeInPendingEvent, []byte(`{"branch":"main","source":43,"why":"the catalog store is full"}`), time.Now())
|
||||
seedTakeIn(w, call, func(string, ...any) {})
|
||||
var asked string
|
||||
seedTakeIn(w, atHolder(func(key string, _ any) (json.RawMessage, error) { asked = key; return nil, errors.New("no answer") },
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package watch
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
@@ -8,24 +9,27 @@ import (
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshwatchdogsclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
)
|
||||
|
||||
// **A refused or overdue creation of bus state is loud** (novox/hq ADR 0595 §4; design 25 § 12.2; DLV-W1). The watchman
|
||||
// hears, through its `consumes`, the indexer's `manifest-accepted` (which carries the module's bus-state entries whole)
|
||||
// and the bus module's own `created` and `refused`. An entry an accepted manifest asks for that has no `created` or
|
||||
// `refused` within OverdueBound is the finding bus-state-overdue, naming the module and the entry; a `refused` is the
|
||||
// finding bus-state-refused, with why. A later `created` clears both, and so does a later accepted manifest of that
|
||||
// module that no longer asks for the entry. Never the bus module's to raise: it cannot while it is down.
|
||||
// learns the bus-state entries a mover's manifest asks for from the mover's catalog entry, read by its fingerprint through
|
||||
// the seat mesh-catalog's verb `entry` once the indexer's `take-in.taken` names the mover (design 47 H-A-94 (1), H-A-49),
|
||||
// and hears the bus module's own `created` and `refused`. An entry a taken-in mover asks for that has no `created` or
|
||||
// `refused` within OverdueBound of its `take-in.taken` is the finding bus-state-overdue, naming the module and the entry;
|
||||
// a `refused` is the finding bus-state-refused, with why. A later `created` clears both, and so does a later take-in of
|
||||
// that module whose entry no longer asks for the entry. Never the bus module's to raise: it cannot while it is down.
|
||||
//
|
||||
// The bound is a row of the signals table: a pulse key per entry with OverdueBound to live, whose expiry is the finding,
|
||||
// so no loop looks and a pulse that stood across a restart still acts. Both findings outlive a restart (signalKind),
|
||||
// since what ends them is heard, not read.
|
||||
//
|
||||
// The bus module says `created` once per entry, so an entry it said created is not waited on again when a later manifest
|
||||
// of its module is accepted with the same entry (review round 1): its mark, a pulse key with no time to live under the
|
||||
// row bus-state-created, is read back at a restart (SeedSignal), and goes when a refused is heard or a manifest no
|
||||
// longer asks for the entry, so a later ask of it waits again.
|
||||
// The bus module says `created` once per entry, so an entry it said created is not waited on again when a later take-in
|
||||
// of its module asks for the same entry (review round 1): its mark, a pulse key with no time to live under the
|
||||
// row bus-state-created, is read back at a restart (SeedSignal), and goes when a refused is heard or an entry no
|
||||
// longer asks for it, so a later ask of it waits again.
|
||||
|
||||
// OverdueBound is how long an accepted entry may wait for the bus module's created or refused.
|
||||
const OverdueBound = 5 * time.Minute
|
||||
@@ -59,41 +63,138 @@ func refusedKey(module, local string) string {
|
||||
return FindingKey(ScopeModule, module, KindBusStateRefused+"-"+local)
|
||||
}
|
||||
|
||||
// Accepted is the indexer's manifest-accepted heard: each entry it asks for waits OverdueBound for the bus module's word,
|
||||
// and every entry of the module this watchman waits on or holds a finding of that it no longer asks for is ended.
|
||||
func (w *Watchman) Accepted(a meshcatalog.ManifestAcceptedEvent) {
|
||||
// EntryReader reads a mover's catalog entry by its fingerprint through the seat mesh-catalog's verb `entry`
|
||||
// (meshcatalogclient.Entries): Need answers one kept or asks it once, telling its Changed when answered; Release forgets it.
|
||||
type EntryReader interface {
|
||||
Need(ctx context.Context, fingerprint string) (meshcatalog.CatalogEntry, meshcatalogclient.EntryState)
|
||||
Release(fingerprint string)
|
||||
}
|
||||
|
||||
// ReadEntriesWith sets the reader of the movers' catalog entries; with none, a take-in's movers wait on no bus state.
|
||||
func (w *Watchman) ReadEntriesWith(r EntryReader) {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
w.manifestAccepted(a.Module, a.Problems) // §500: the problems a stricter validator finds are the module's warning
|
||||
w.entries = r
|
||||
}
|
||||
|
||||
// moverTaken is a mover of a take-in whose catalog entry is asked: its module and the take-in's time.
|
||||
type moverTaken struct {
|
||||
module string
|
||||
at time.Time
|
||||
}
|
||||
|
||||
// moversTaken asks the catalog entry of each mover a take-in.taken names, w.mu held: an entry already answered is read
|
||||
// now, any other once its answer comes (EntryChanged).
|
||||
func (w *Watchman) moversTaken(movers []meshcatalog.TakeInEntry, at time.Time) {
|
||||
if w.entries == nil {
|
||||
return
|
||||
}
|
||||
for _, m := range movers {
|
||||
if m.Fingerprint == "" || m.Module == "" {
|
||||
continue
|
||||
}
|
||||
w.movers[m.Fingerprint] = moverTaken{module: m.Module, at: at}
|
||||
if e, state := w.entries.Need(context.Background(), m.Fingerprint); state == meshcatalogclient.EntryAnswered {
|
||||
w.entryRead(m.Fingerprint, e)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// EntryChanged hears the answer of a mover's entry asked by its fingerprint (the Entries' Changed): an answered entry's
|
||||
// bus state is waited on; one the holder does not hold is said, since its bus state cannot be known.
|
||||
func (w *Watchman) EntryChanged(row meshcatalogclient.EntryRow) {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
m, ok := w.movers[row.Fingerprint]
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
switch {
|
||||
case row.State == meshcatalogclient.EntryAnswered && row.Entry != nil:
|
||||
w.entryRead(row.Fingerprint, *row.Entry)
|
||||
case row.State == meshcatalogclient.EntryNotFound:
|
||||
w.say("the catalog entry of %s at %s is not held (%s), so its bus state is not waited on", m.module, short(row.Fingerprint), row.Detail)
|
||||
delete(w.movers, row.Fingerprint)
|
||||
w.entries.Release(row.Fingerprint)
|
||||
}
|
||||
}
|
||||
|
||||
// entryRead is a mover's entry answered, w.mu held: its bus state is waited on from the take-in's time, and the entry
|
||||
// is forgotten.
|
||||
func (w *Watchman) entryRead(fingerprint string, e meshcatalog.CatalogEntry) {
|
||||
m, ok := w.movers[fingerprint]
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
delete(w.movers, fingerprint)
|
||||
defer w.entries.Release(fingerprint)
|
||||
asked, err := BusStateOfEntry(e)
|
||||
if err != nil {
|
||||
w.say("the catalog entry of %s at %s does not read (%v), so its bus state is not waited on", m.module, short(fingerprint), err)
|
||||
return
|
||||
}
|
||||
w.busStateAsked(m.module, fingerprint, asked, m.at)
|
||||
}
|
||||
|
||||
// BusStateOfEntry is the bus state a catalog entry's manifest asks for: the stored manifest keeps it beside requires, under
|
||||
// catalogue.BusStateField.
|
||||
func BusStateOfEntry(e meshcatalog.CatalogEntry) ([]catalogue.BusStateEntry, error) {
|
||||
raw, err := json.Marshal(e.Manifest)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var m map[string]json.RawMessage
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var entries []catalogue.BusStateEntry
|
||||
if b, ok := m[catalogue.BusStateField]; ok {
|
||||
if err := json.Unmarshal(b, &entries); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
// busStateAsked is a taken-in mover's bus state, w.mu held: each entry it asks for waits OverdueBound from the take-in
|
||||
// for the bus module's word, and every entry of the module this watchman waits on or holds a finding of that it no
|
||||
// longer asks for is ended.
|
||||
func (w *Watchman) busStateAsked(module, fingerprint string, entries []catalogue.BusStateEntry, at time.Time) {
|
||||
asked := map[string]bool{}
|
||||
for _, e := range a.BusState {
|
||||
for _, e := range entries {
|
||||
if e.Name == "" {
|
||||
continue
|
||||
}
|
||||
asked[e.Name] = true
|
||||
if w.busCreated[signalKey(rowBusCreated, busStateID(a.Module, e.Name))] {
|
||||
if w.busCreated[signalKey(rowBusCreated, busStateID(module, e.Name))] {
|
||||
continue // said created already, and created is said once per entry: nothing to wait for
|
||||
}
|
||||
// One refused since an earlier verdict waits again: a new verdict asks the bus module again.
|
||||
w.signal(rowBusState, busStateID(a.Module, e.Name), OverdueBound, signalDue{Do: dueFind, Finding: Finding{
|
||||
Scope: ScopeModule, ID: a.Module, Token: KindBusStateOverdue + "-" + e.Name, Kind: KindBusStateOverdue,
|
||||
Severity: Warning, Source: rowBusState, Since: w.now().Add(OverdueBound).UTC().Truncate(time.Second),
|
||||
Summary: fmt.Sprintf("the bus module said nothing of %s's %s (%s) within %s of its accepted manifest %s", a.Module,
|
||||
e.Name, e.Storage, OverdueBound, short(a.Digest)),
|
||||
Headline: "The bus state " + e.Name + " of " + a.Module + " was not created",
|
||||
Explanation: "The indexer accepted a manifest of " + a.Module + " asking the bus module for " + e.Name + ", and the bus " +
|
||||
"module said neither created nor refused within " + OverdueBound.String() + ": it is down, or behind. " + a.Module +
|
||||
// One refused since an earlier take-in waits again: a new take-in asks the bus module again.
|
||||
f := Finding{
|
||||
Scope: ScopeModule, ID: module, Token: KindBusStateOverdue + "-" + e.Name, Kind: KindBusStateOverdue,
|
||||
Severity: Warning, Source: rowBusState, Since: at.Add(OverdueBound).UTC().Truncate(time.Second),
|
||||
Summary: fmt.Sprintf("the bus module said nothing of %s's %s (%s) within %s of its take-in at %s", module,
|
||||
e.Name, e.Storage, OverdueBound, short(fingerprint)),
|
||||
Headline: "The bus state " + e.Name + " of " + module + " was not created",
|
||||
Explanation: "The indexer took in " + module + ", whose manifest asks the bus module for " + e.Name + ", and the bus " +
|
||||
"module said neither created nor refused within " + OverdueBound.String() + ": it is down, or behind. " + module +
|
||||
" fails at its start while the entry's binding is missing (novox/hq ADR 0595 §1).",
|
||||
Needs: "a look at the bus module's health and its process's log; nats_first_view_again asks it to read the verdicts again"}})
|
||||
Needs: "a look at the bus module's health and its process's log; nats_first_view_again asks it to read the entries again"}
|
||||
if left := OverdueBound - w.now().Sub(at); left > 0 {
|
||||
w.signal(rowBusState, busStateID(module, e.Name), left, signalDue{Do: dueFind, Finding: f})
|
||||
} else {
|
||||
w.unsignal(rowBusState, busStateID(module, e.Name))
|
||||
w.find(f)
|
||||
}
|
||||
}
|
||||
for _, local := range w.busStateEntriesOf(a.Module) {
|
||||
for _, local := range w.busStateEntriesOf(module) {
|
||||
if asked[local] {
|
||||
continue
|
||||
}
|
||||
w.unsignal(rowBusState, busStateID(a.Module, local))
|
||||
w.unfind(overdueKey(a.Module, local))
|
||||
w.unfind(refusedKey(a.Module, local))
|
||||
w.uncreated(a.Module, local)
|
||||
w.unsignal(rowBusState, busStateID(module, local))
|
||||
w.unfind(overdueKey(module, local))
|
||||
w.unfind(refusedKey(module, local))
|
||||
w.uncreated(module, local)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,8 +1,11 @@
|
||||
package watch
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
@@ -12,7 +15,6 @@ import (
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient/heard"
|
||||
"git.novox.be/novox/mesh-contracts/provisioning/go/clients/meshprovisionerclient"
|
||||
@@ -42,181 +44,116 @@ func TestTheWatchmansFindings(t *testing.T) {
|
||||
// want are the finding keys standing at the end, with their severity.
|
||||
want map[string]string
|
||||
}{
|
||||
{"ADR 0633 §5 (1585/T21): a revision waiting for an indexer build is a finding past 60 minutes from its since, not within; a group past 30", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) {
|
||||
{"design 47 H §744, H-A-94 (2): a held take-in is a finding past take-in-bound from its message's time, not within; the setting counts", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) {
|
||||
w.TakeInCaughtUp() // a live message, heard after the replay
|
||||
w.TakeIn(takeInEvent(t, takein.TakeInPending, revKey), takeInBody(t, takein.TakeInPending, revKey, t0))
|
||||
if o.ttls[signalKey("take-in", revKey)] != TakeInRevisionBound || len(o.findings) != 0 {
|
||||
t.Fatalf("a revision's wait: ttls %v, findings %v", o.ttls, o.findings)
|
||||
w.TakeIn(TakeInPendingEvent, pendingBody(t, "main", 42, "the catalog store is full"), t0)
|
||||
if o.ttls[signalKey(takeInRow, revKey)] != DefaultTakeInBound || len(o.findings) != 0 {
|
||||
t.Fatalf("a held take-in: ttls %v, findings %v", o.ttls, o.findings)
|
||||
}
|
||||
w.TakeIn(takeInEvent(t, takein.TakeInPending, groupKey), takeInBody(t, takein.TakeInPending, groupKey, t0))
|
||||
if o.ttls[signalKey("take-in", groupKey)] != TakeInGroupBound {
|
||||
t.Fatalf("a group's wait: ttls %v", o.ttls)
|
||||
w.SetTakeInBound(20 * time.Minute)
|
||||
w.TakeIn(TakeInPendingEvent, pendingBody(t, "feat/x", 7, ""), c.t.Add(-5*time.Minute))
|
||||
if o.ttls[signalKey(takeInRow, TakeInKey("feat/x", 7))] != 15*time.Minute {
|
||||
t.Fatalf("a take-in held five minutes under a 20-minute bound: ttls %v", o.ttls)
|
||||
}
|
||||
expire(w, c, signalKey("take-in", groupKey), TakeInGroupBound)
|
||||
if len(o.findings) != 1 {
|
||||
t.Fatalf("findings at 30 minutes: %v", o.findings)
|
||||
expire(w, c, signalKey(takeInRow, revKey), DefaultTakeInBound)
|
||||
if f := o.findings[FindingKey(ScopeDelivery, revKey, KindTakeInWaits)]; !strings.Contains(f.Summary, "the catalog store is full") {
|
||||
t.Fatalf("the finding does not say why: %v", o.findings)
|
||||
}
|
||||
expire(w, c, signalKey("take-in", revKey), TakeInRevisionBound-TakeInGroupBound)
|
||||
}, map[string]string{FindingKey(ScopeDelivery, revKey, KindTakeInWaits): Warning, FindingKey(ScopeDelivery, groupKey, KindTakeInWaits): Warning}},
|
||||
{"ADR 0633 §5 (1585/T21): the take-in.taken of a named indexer, a hand take-in or a drop clears the finding", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) {
|
||||
}, map[string]string{FindingKey(ScopeDelivery, revKey, KindTakeInWaits): Warning}},
|
||||
{"design 47 H §744: the take-in.taken of the source, the indexer's, a hand take-in's or a skip's, clears the finding", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) {
|
||||
w.TakeInCaughtUp() // a live message, heard after the replay
|
||||
for _, by := range []string{"", "operator@shanks"} {
|
||||
w.TakeIn(takeInEvent(t, takein.TakeInPending, revKey), takeInBody(t, takein.TakeInPending, revKey, t0))
|
||||
expire(w, c, signalKey("take-in", revKey), TakeInRevisionBound)
|
||||
for _, taken := range []meshcatalog.TakeInTaken{{Branch: "main", Source: 42, Call: "fedcba9876543210", Revision: 3},
|
||||
{Branch: "main", Source: 42, By: "operator@shanks", Why: "the store has room", Skipped: true}} {
|
||||
w.TakeIn(TakeInPendingEvent, pendingBody(t, "main", 42, ""), t0)
|
||||
expire(w, c, signalKey(takeInRow, revKey), DefaultTakeInBound)
|
||||
if len(o.findings) != 1 {
|
||||
t.Fatalf("findings past the bound: %v", o.findings)
|
||||
}
|
||||
taken, err := takein.EncodeTakeIn(takein.TakeIn{Kind: takein.TakeInTaken, Key: revKey, What: takein.TakeInRevision, Since: t0,
|
||||
Merges: []takein.Merge{{Repository: "novox/a", Merge: "c0ffee"}}, By: by, Why: map[bool]string{true: "dropped", false: ""}[by != ""], Dropped: by != ""})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
w.TakeIn(takeInEvent(t, takein.TakeInTaken, revKey), taken)
|
||||
body, _ := json.Marshal(taken)
|
||||
w.TakeIn(TakeInTakenEvent, body, c.t)
|
||||
c.t = t0
|
||||
}
|
||||
}, map[string]string{}},
|
||||
{"ADR 0633 §5 (1585/T21): a group's take-in.taken ends its 30 minutes, and the revision then pending under m.<seq> starts its own 60", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) {
|
||||
w.TakeInCaughtUp() // a live message, heard after the replay
|
||||
w.TakeIn(takeInEvent(t, takein.TakeInPending, groupKey), takeInBody(t, takein.TakeInPending, groupKey, t0))
|
||||
expire(w, c, signalKey("take-in", groupKey), 25*time.Minute)
|
||||
w.TakeIn(takeInEvent(t, takein.TakeInTaken, groupKey), takeInBody(t, takein.TakeInTaken, groupKey, t0))
|
||||
w.TakeIn(takeInEvent(t, takein.TakeInPending, revKey), takeInBody(t, takein.TakeInPending, revKey, c.t))
|
||||
if len(o.findings) != 0 || o.ttls[signalKey("take-in", revKey)] != TakeInRevisionBound {
|
||||
t.Fatalf("after the hand-over: findings %v, ttls %v", o.findings, o.ttls)
|
||||
}
|
||||
expire(w, c, signalKey("take-in", revKey), TakeInRevisionBound)
|
||||
}, map[string]string{FindingKey(ScopeDelivery, revKey, KindTakeInWaits): Warning}},
|
||||
{"ADR 0633 §5 (1585/T21): a pending event heard again at the start, since past its bound, is said once the replay has caught up, not before", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) {
|
||||
w.TakeIn(takeInEvent(t, takein.TakeInPending, revKey), takeInBody(t, takein.TakeInPending, revKey, t0.Add(-2*time.Hour)))
|
||||
{"design 47 H §744: a pending heard again at the start, past its bound, is said once the replay has caught up, not before; one followed by its taken never raises", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) {
|
||||
w.TakeIn(TakeInPendingEvent, pendingBody(t, "main", 42, ""), t0.Add(-2*time.Hour))
|
||||
w.TakeIn(TakeInPendingEvent, pendingBody(t, "main", 43, ""), t0.Add(-2*time.Hour))
|
||||
w.TakeIn(TakeInTakenEvent, takenBody(t, "main", 43), t0.Add(-2*time.Hour))
|
||||
if len(o.findings) != 0 || len(o.pulses) != 0 {
|
||||
t.Fatalf("a replayed old wait before the replay caught up: findings %v, pulses %v", o.findings, o.pulses)
|
||||
t.Fatalf("replayed old waits before the replay caught up: findings %v, pulses %v", o.findings, o.pulses)
|
||||
}
|
||||
w.TakeInCaughtUp()
|
||||
}, map[string]string{FindingKey(ScopeDelivery, revKey, KindTakeInWaits): Warning}},
|
||||
{"1966/AC195-5: a seed ends the finding and the held wait of a key indexer_pending no longer names, and keeps one it names with its own since", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) {
|
||||
// A previous run's finding for revKey stands in the bucket (read at the start); a wait of staleKey is held from an event.
|
||||
old := Finding{Scope: ScopeDelivery, ID: revKey, Kind: KindTakeInWaits, Severity: Warning, Source: "take-in", Since: t0.Add(-3 * time.Hour), Observer: testModule, Summary: "held", Said: "held", Headline: "held", Explanation: "held", Needs: "a take-in"}
|
||||
{"design 47 H-A-94 (2), H-A-97: a seed from indexer_pending ends the finding and the wait of a key it no longer names, and times one it names from its own since", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) {
|
||||
// A previous run's finding for main.41 stands in the bucket (read at the start); a wait of main.40 is held from an event.
|
||||
stale := TakeInKey("main", 41)
|
||||
old := Finding{Scope: ScopeDelivery, ID: stale, Kind: KindTakeInWaits, Severity: Warning, Source: takeInRow, Since: t0.Add(-3 * time.Hour), Observer: testModule, Summary: "held", Said: "held", Headline: "held", Explanation: "held", Needs: "a take-in"}
|
||||
value, err := EncodeFinding(old)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
o.findings[old.Key()] = old
|
||||
w.SeedFinding(old.Key(), value)
|
||||
const staleKey = "m.00000000000000000099"
|
||||
w.TakeInCaughtUp()
|
||||
w.TakeIn(takeInEvent(t, takein.TakeInPending, staleKey), takeInBody(t, takein.TakeInPending, staleKey, c.t))
|
||||
w.SeedTakeIn(takein.PendingAnswer{Waits: []takein.PendingWait{{Key: groupKey, What: takein.TakeInGroup, Keys: []string{groupKey},
|
||||
Merges: []takein.Merge{{Repository: "novox/a", Number: 1, Merge: "c0ffee"}}, Since: c.t.Add(-10 * time.Minute)}}})
|
||||
if _, ok := o.findings[old.Key()]; ok || w.signals[signalKey("take-in", staleKey)].Finding.ID != "" || w.signals[signalKey("take-in", revKey)].Finding.ID != "" {
|
||||
t.Fatalf("a wait the answer does not name stands: findings %v, signals %v", o.findings, w.signals)
|
||||
}
|
||||
if o.ttls[signalKey("take-in", groupKey)] != TakeInGroupBound-10*time.Minute {
|
||||
t.Fatalf("the named wait: ttl %v, want the rest of its bound from its own since", o.ttls[signalKey("take-in", groupKey)])
|
||||
}
|
||||
}, map[string]string{}},
|
||||
{"1966/AC195-4: a taken event for a key the watchman holds no wait for changes no finding and no signal", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) {
|
||||
w.TakeInCaughtUp()
|
||||
w.TakeIn(takeInEvent(t, takein.TakeInPending, groupKey), takeInBody(t, takein.TakeInPending, groupKey, t0))
|
||||
ttls, pulses := len(o.ttls), len(o.pulses)
|
||||
w.TakeIn(takeInEvent(t, takein.TakeInTaken, revKey), takeInBody(t, takein.TakeInTaken, revKey, t0))
|
||||
if o.findingWrites != 0 || len(o.ttls) != ttls || len(o.pulses) != pulses || o.ttls[signalKey("take-in", groupKey)] != TakeInGroupBound {
|
||||
t.Fatalf("a taken for an unheld key: findings written %d, ttls %v, pulses %v", o.findingWrites, o.ttls, o.pulses)
|
||||
}
|
||||
}, map[string]string{}},
|
||||
{"ADR 0633 §5 (1585/T21): a replayed pending followed by its taken never raises", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) {
|
||||
w.TakeIn(takeInEvent(t, takein.TakeInPending, revKey), takeInBody(t, takein.TakeInPending, revKey, t0.Add(-2*time.Hour)))
|
||||
w.TakeIn(takeInEvent(t, takein.TakeInTaken, revKey), takeInBody(t, takein.TakeInTaken, revKey, t0.Add(-2*time.Hour)))
|
||||
w.TakeInCaughtUp()
|
||||
if o.findingWrites != 0 || len(o.pulses) != 0 {
|
||||
t.Fatalf("a wait that ended in the replay wrote %d findings and left pulses %v", o.findingWrites, o.pulses)
|
||||
}
|
||||
}, map[string]string{}},
|
||||
{"ADR 0633 §5 (1585/T21): a wait the indexer holds at a restart within its bound is seeded from indexer_pending and raised past it", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) {
|
||||
// The restarted watchman's durable consumer hears nothing of the pending event it acknowledged before; the wait
|
||||
// began ten minutes ago, and the indexer still holds it.
|
||||
since := c.t.Add(-10 * time.Minute)
|
||||
w.SeedTakeIn(takein.PendingAnswer{Waits: []takein.PendingWait{{Key: revKey, What: takein.TakeInRevision, Keys: []string{revKey},
|
||||
Merges: []takein.Merge{{Repository: "novox/a", Number: 1, Merge: "c0ffee"}}, Lost: []string{"m.00000000000000000042"}, Since: since}}})
|
||||
w.TakeInCaughtUp()
|
||||
if len(o.findings) != 0 || o.ttls[signalKey("take-in", revKey)] != TakeInRevisionBound-10*time.Minute {
|
||||
t.Fatalf("within its bound: findings %v, ttl %v; want none yet and the rest of the bound", o.findings, o.ttls[signalKey("take-in", revKey)])
|
||||
}
|
||||
expire(w, c, signalKey("take-in", revKey), TakeInRevisionBound-10*time.Minute)
|
||||
}, map[string]string{FindingKey(ScopeDelivery, revKey, KindTakeInWaits): Warning}},
|
||||
{"ADR 0633 §1 (issue 1921): (A-476 b) a revision held because its message is lost or unread is raised past the revision bound from its since, naming why", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) {
|
||||
w.TakeInCaughtUp()
|
||||
body, err := takein.EncodeTakeIn(takein.TakeIn{Kind: takein.TakeInPending, Key: revKey, What: takein.TakeInRevision, Since: c.t,
|
||||
Merges: []takein.Merge{{Repository: "novox/a", Number: 1, Merge: "c0ffee"}}, Lost: []string{revKey}})
|
||||
w.TakeIn(TakeInPendingEvent, pendingBody(t, "main", 40, ""), c.t)
|
||||
seed, err := DecodeHeldTakeIns([]byte(`{"held":[{"branch":"main","source":42,"why":"the catalog store is full","since":"` + c.t.Add(-10*time.Minute).Format(time.RFC3339) + `"}]}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
w.TakeIn(takeInEvent(t, takein.TakeInPending, revKey), body)
|
||||
if o.ttls[signalKey("take-in", revKey)] != TakeInRevisionBound || len(o.findings) != 0 {
|
||||
t.Fatalf("a lost message: ttls %v, findings %v", o.ttls, o.findings)
|
||||
w.SeedTakeIn(seed)
|
||||
if _, ok := o.findings[old.Key()]; ok || w.signals[signalKey(takeInRow, TakeInKey("main", 40))].Finding.ID != "" {
|
||||
t.Fatalf("a wait the answer does not name stands: findings %v, signals %v", o.findings, w.signals)
|
||||
}
|
||||
expire(w, c, signalKey("take-in", revKey), TakeInRevisionBound)
|
||||
for _, f := range o.findings {
|
||||
if !strings.Contains(f.Summary, "lost or unread") {
|
||||
t.Errorf("finding %q does not say why", f.Summary)
|
||||
}
|
||||
if o.ttls[signalKey(takeInRow, revKey)] != DefaultTakeInBound-10*time.Minute {
|
||||
t.Fatalf("the named wait: ttl %v, want the rest of its bound from its own since", o.ttls[signalKey(takeInRow, revKey)])
|
||||
}
|
||||
expire(w, c, signalKey(takeInRow, revKey), DefaultTakeInBound-10*time.Minute)
|
||||
if _, err := DecodeHeldTakeIns([]byte(`{"held":[{"branch":"main","source":42,"why":"w"}]}`)); err == nil {
|
||||
t.Error("a held take-in with no since was read")
|
||||
}
|
||||
}, map[string]string{FindingKey(ScopeDelivery, revKey, KindTakeInWaits): Warning}},
|
||||
{"ADR 0633 §5 (1585/T21): a drop clears the wait, and the merges left under it said pending again with the same since are raised from it", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) {
|
||||
w.TakeInCaughtUp() // a live message, heard after the replay
|
||||
w.TakeIn(takeInEvent(t, takein.TakeInPending, revKey), takeInBody(t, takein.TakeInPending, revKey, t0))
|
||||
expire(w, c, signalKey("take-in", revKey), TakeInRevisionBound)
|
||||
if len(o.findings) != 1 {
|
||||
t.Fatalf("findings past the bound: %v", o.findings)
|
||||
{"1966/AC195-4: a taken event for a key the watchman holds no wait for changes no finding and no signal", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) {
|
||||
w.TakeInCaughtUp()
|
||||
w.TakeIn(TakeInPendingEvent, pendingBody(t, "main", 42, ""), t0)
|
||||
ttls, pulses := len(o.ttls), len(o.pulses)
|
||||
w.TakeIn(TakeInTakenEvent, takenBody(t, "main", 9), t0)
|
||||
if o.findingWrites != 0 || len(o.ttls) != ttls || len(o.pulses) != pulses || o.ttls[signalKey(takeInRow, revKey)] != DefaultTakeInBound {
|
||||
t.Fatalf("a taken for an unheld key: findings written %d, ttls %v, pulses %v", o.findingWrites, o.ttls, o.pulses)
|
||||
}
|
||||
w.TakeIn(takeInEvent(t, takein.TakeInTaken, revKey), takeInBody(t, takein.TakeInTaken, revKey, t0))
|
||||
if len(o.findings) != 0 {
|
||||
t.Fatalf("the drop left %v", o.findings)
|
||||
}
|
||||
next := "m.00000000000000000043"
|
||||
w.TakeIn(takeInEvent(t, takein.TakeInPending, next), takeInBody(t, takein.TakeInPending, next, t0))
|
||||
}, map[string]string{FindingKey(ScopeDelivery, "m.00000000000000000043", KindTakeInWaits): Warning}},
|
||||
{"design 47 G §500: manifest-refused raises the refusal, manifest-accepted clears it and carries an outgrown manifest's problems", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) {
|
||||
w.ManifestRefused(meshcatalog.ManifestRefusedEvent{Module: "nats", Problems: []string{"removes a stream"}})
|
||||
w.ManifestRefused(meshcatalog.ManifestRefusedEvent{Module: "power", Problems: []string{"shows nothing"}})
|
||||
k := FindingKey(ScopeModule, "nats", KindManifestRefused)
|
||||
if f := o.findings[k]; !strings.Contains(f.Summary, "refused the manifest of nats: removes a stream") {
|
||||
t.Fatalf("findings = %v", o.findings)
|
||||
}
|
||||
w.Accepted(meshcatalog.ManifestAcceptedEvent{Module: "nats", Digest: strings.Repeat("d", 64)})
|
||||
w.Accepted(meshcatalog.ManifestAcceptedEvent{Module: "power", Digest: strings.Repeat("d", 64)})
|
||||
// ADR 0579 §1: an accepted manifest a stricter judge outgrew is a warning until mended.
|
||||
w.Accepted(meshcatalog.ManifestAcceptedEvent{Module: "node-tools", Digest: strings.Repeat("d", 64), Problems: []string{"says no credential form"}})
|
||||
if f := o.findings[FindingKey(ScopeModule, "node-tools", KindManifestOutgrown)]; f.Severity != Warning || !strings.Contains(f.Summary, "says no credential form") {
|
||||
t.Fatalf("findings = %v", o.findings)
|
||||
}
|
||||
w.Accepted(meshcatalog.ManifestAcceptedEvent{Module: "node-tools", Digest: strings.Repeat("e", 64), Problems: []string{}})
|
||||
// The start seeds from the indexer's verdicts: the newest of each module counts.
|
||||
w.SeedVerdicts([]VerdictRow{
|
||||
{Module: "nats", State: meshcatalogclient.VerdictRefused, Problems: []string{"old"}, At: t0},
|
||||
{Module: "nats", State: meshcatalogclient.VerdictAccepted, At: t0.Add(time.Hour)},
|
||||
{Module: "power", State: meshcatalogclient.VerdictRefused, Problems: []string{"shows nothing"}, At: t0},
|
||||
{Module: "node-tools", State: meshcatalogclient.VerdictAccepted, Warnings: []string{"w"}, At: t0}})
|
||||
if len(o.findings) != 2 || o.findings[FindingKey(ScopeModule, "power", KindManifestRefused)].Kind == "" || o.findings[FindingKey(ScopeModule, "node-tools", KindManifestOutgrown)].Kind == "" {
|
||||
t.Fatalf("findings = %v", o.findings)
|
||||
}
|
||||
w.Accepted(meshcatalog.ManifestAcceptedEvent{Module: "power"})
|
||||
w.Accepted(meshcatalog.ManifestAcceptedEvent{Module: "node-tools"})
|
||||
}, map[string]string{}},
|
||||
{"DLV-W1, 1966/AC37-AC38 (ADR 0595 §4, design 25 § 12.2): an accepted entry the bus module says nothing of in five minutes is overdue until created; a refused one stands until created; a later verdict no longer asking ends both", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) {
|
||||
accepted := func(locals ...string) meshcatalog.ManifestAcceptedEvent {
|
||||
a := meshcatalog.ManifestAcceptedEvent{Module: "mesh-knowledge", Digest: strings.Repeat("d", 64)}
|
||||
{"DLV-W1, 1966/AC37-AC38 (ADR 0595 §4), design 47 H-A-94 (1): an entry a taken-in mover's catalog entry asks for that the bus module says nothing of in five minutes is overdue until created; a refused one stands until created; a later take-in no longer asking ends both", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) {
|
||||
entries := &fakeEntries{entries: map[string]meshcatalog.CatalogEntry{}, answer: true}
|
||||
w.ReadEntriesWith(entries)
|
||||
source := int64(0)
|
||||
accepted := func(locals ...string) {
|
||||
source++
|
||||
fp := fmt.Sprintf("src2:%064d", source)
|
||||
var bus []map[string]string
|
||||
for _, l := range locals {
|
||||
a.BusState = append(a.BusState, meshcatalog.ManifestAcceptedEventBusStateItem{Name: l, Storage: "ledger"})
|
||||
bus = append(bus, map[string]string{"name": l, "storage": "ledger"})
|
||||
}
|
||||
return a
|
||||
entries.entries[fp] = meshcatalog.CatalogEntry{Module: "mesh-knowledge", Fp: fp, Manifest: map[string]any{"module": "mesh-knowledge", "bus-state": bus}}
|
||||
w.TakeIn(TakeInTakenEvent, takenBody(t, "main", source, meshcatalog.TakeInEntry{Module: "mesh-knowledge", Fingerprint: fp, Commit: "c0ffee", Repository: "novox/mesh-knowledge"}), c.t)
|
||||
}
|
||||
// The first entry is asked and answered after: its wait is timed from the take-in, not from the answer.
|
||||
entries.answer = false
|
||||
accepted("records", "drafts", "kept")
|
||||
if len(o.ttls) != 0 || len(entries.released) != 0 {
|
||||
t.Fatalf("a wait before the entry was answered: %v", o.ttls)
|
||||
}
|
||||
fp1 := fmt.Sprintf("src2:%064d", 1)
|
||||
e1 := entries.entries[fp1]
|
||||
c.t = c.t.Add(time.Minute)
|
||||
w.EntryChanged(meshcatalogclient.EntryRow{Fingerprint: fp1, State: meshcatalogclient.EntryAnswered, Entry: &e1})
|
||||
entries.answer = true
|
||||
if !slices.Equal(entries.released, []string{fp1}) {
|
||||
t.Fatalf("the entry read is not released: %v", entries.released)
|
||||
}
|
||||
w.Accepted(accepted("records", "drafts", "kept"))
|
||||
pk := func(local string) string { return signalKey(rowBusState, busStateID("mesh-knowledge", local)) }
|
||||
if o.ttls[pk("records")] != OverdueBound {
|
||||
t.Fatalf("no five-minute pulse for records: %v", o.ttls)
|
||||
if o.ttls[pk("records")] != OverdueBound-time.Minute {
|
||||
t.Fatalf("no pulse for records within five minutes of its take-in: %v", o.ttls)
|
||||
}
|
||||
w.BrokerCreated(BrokerSaid{Module: "mesh-knowledge", Local: "drafts"})
|
||||
expire(w, c, pk("drafts"), 5*time.Minute) // created in time: no finding
|
||||
expire(w, c, pk("drafts"), 4*time.Minute) // created in time: no finding
|
||||
expire(w, c, pk("records"), 0)
|
||||
overdue := o.findings[overdueKey("mesh-knowledge", "records")]
|
||||
if overdue.Kind != KindBusStateOverdue || !strings.Contains(overdue.Summary, "mesh-knowledge's records") {
|
||||
@@ -226,13 +163,13 @@ func TestTheWatchmansFindings(t *testing.T) {
|
||||
if _, open := o.findings[overdueKey("mesh-knowledge", "records")]; open {
|
||||
t.Fatal("created did not clear overdue")
|
||||
}
|
||||
// Review round 1: created is said once per entry, so a second accepted manifest with the same entries waits on
|
||||
// Review round 1: created is said once per entry, so a second take-in with the same entries waits on
|
||||
// none of them, and the mark that says so has no time to live.
|
||||
delete(o.pulses, pk("records")) // expired above; the fake keeps an expired pulse
|
||||
w.Accepted(accepted("records", "drafts"))
|
||||
accepted("records", "drafts")
|
||||
for _, local := range []string{"records", "drafts"} {
|
||||
if _, waits := o.pulses[pk(local)]; waits {
|
||||
t.Fatalf("a second accepted manifest waits again on %s, said created already", local)
|
||||
t.Fatalf("a second take-in waits again on %s, said created already", local)
|
||||
}
|
||||
if mk := signalKey(rowBusCreated, busStateID("mesh-knowledge", local)); o.ttls[mk] != 0 || o.pulses[mk].Owes == "" {
|
||||
t.Fatalf("the mark of %s created is %v with time to live %v", local, o.pulses[mk], o.ttls[mk])
|
||||
@@ -246,9 +183,9 @@ func TestTheWatchmansFindings(t *testing.T) {
|
||||
if !signalKind(mustEncode(t, o.findings[refusedKey("mesh-knowledge", "kept")])) {
|
||||
t.Error("a refused creation would be dropped at a restart")
|
||||
}
|
||||
// A later verdict asking for kept alone again waits on it again; one no longer asking for it ends it.
|
||||
w.Accepted(accepted("kept", "fresh"))
|
||||
w.Accepted(accepted("fresh"))
|
||||
// A later take-in asking for kept again waits on it again; one no longer asking for it ends it.
|
||||
accepted("kept", "fresh")
|
||||
accepted("fresh")
|
||||
if _, open := o.findings[refusedKey("mesh-knowledge", "kept")]; open {
|
||||
t.Fatal("a verdict no longer asking for kept left its finding")
|
||||
}
|
||||
@@ -257,7 +194,7 @@ func TestTheWatchmansFindings(t *testing.T) {
|
||||
}
|
||||
w.BrokerCreated(BrokerSaid{Module: "mesh-knowledge", Local: "fresh"})
|
||||
// A refused create stands until created.
|
||||
w.Accepted(accepted("gone"))
|
||||
accepted("gone")
|
||||
w.BrokerRefused(BrokerSaid{Module: "mesh-knowledge", Local: "gone"})
|
||||
}, map[string]string{refusedKey("mesh-knowledge", "gone"): Urgent}},
|
||||
{"ADR 0615 §5 (review round 1): a consumer the watchman cannot bind is family-not-heard naming its stream, until it binds", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) {
|
||||
@@ -493,38 +430,39 @@ func TestTheWatchmansFindings(t *testing.T) {
|
||||
w.DeadLettersHeld(map[string]int{"SEAT_X.y_worker": 1}, who)
|
||||
w.NoticesWaiting(true)
|
||||
}, map[string]string{"bus.SEAT_X.y_worker.max-deliveries": Warning, "bus.DEAD_LETTER_NOTICES.controller.notices-waiting": Warning}},
|
||||
{"ADR 0603 §8: a want held by a rule and a provision asked each raise a finding naming the wait past their bound, and clear when it ends", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) {
|
||||
{"design 47 H-A-95 (1): a want awaiting its hand placement is not timed, and is timed once it no longer holds; ADR 0603 §8: a provision asked raises a finding naming the wait past its bound, and clears when it ends", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) {
|
||||
w.SetStepBound(30 * time.Minute)
|
||||
held, _ := meshwantsclient.EncodeWant(meshwantsclient.Want{Fp: fpOld, Self: true, Newest: fpNew, Rule: meshwantsclient.RuleCanary, Words: "the canary ace does not run it yet"})
|
||||
released, _ := meshwantsclient.EncodeWant(meshwantsclient.Want{Fp: fpNew, Self: true})
|
||||
held, err := meshwantsclient.EncodeWant(meshwantsclient.Want{Fp: fpOld, Hold: true, Words: meshwantsclient.HoldAwaitsHandPlacement})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
released, err := meshwantsclient.EncodeWant(meshwantsclient.Want{Fp: fpNew, Pipeline: "main.42", Rank: 1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
asked, _ := meshprovisionerclient.Encode(meshprovisionerclient.Provision{Provision: "oidc", Consumer: meshprovisionerclient.End{Module: "photos", Node: "ace"},
|
||||
Provider: meshprovisionerclient.End{Module: "keycloak", Node: "novox"}, State: meshprovisionerclient.Asked, At: t0})
|
||||
provided, _ := meshprovisionerclient.Encode(meshprovisionerclient.Provision{Provision: "oidc", Consumer: meshprovisionerclient.End{Module: "photos", Node: "ace"},
|
||||
Provider: meshprovisionerclient.End{Module: "keycloak", Node: "novox"}, State: meshprovisionerclient.Provided, At: t0})
|
||||
w.EngineRunning("ace", "web", []byte(`{"fingerprint":"`+fpOld+`"}`)) // the node runs what the held want names: the hold is still the wait
|
||||
w.Want("want.ace.web", held, Put, t0)
|
||||
w.Want("want.ace.web", held, Put, c.t)
|
||||
w.Provision("ace.photos.oidc", asked, Put)
|
||||
waits := map[string]string{AssignerWantsBucket: "want.ace.web", ProvisionsBucket: "ace.photos.oidc"}
|
||||
if len(o.pulses) != 2 || len(o.findings) != 0 {
|
||||
t.Fatalf("pulses %v findings %v", o.pulses, o.findings)
|
||||
if _, timed := o.pulses[pulseKeyOf(t, testWants(), "want.ace.web")]; timed || len(o.pulses) != 1 || len(o.stuck) != 0 || len(o.findings) != 0 {
|
||||
t.Fatalf("a held want is timed: pulses %v stuck %v findings %v", o.pulses, o.stuck, o.findings)
|
||||
}
|
||||
// The keys are written again as their parts move on, with new words and a new time: the wait still began at t0.
|
||||
// The provision is written again as it moves on, with new words and a new time: the wait still began at t0.
|
||||
c.t = c.t.Add(20 * time.Minute)
|
||||
held2, _ := meshwantsclient.EncodeWant(meshwantsclient.Want{Fp: fpOld, Self: true, Newest: fpNew, Rule: meshwantsclient.RuleCanary, Words: "the canary ace runs it, not healthy yet"})
|
||||
asked2, _ := meshprovisionerclient.Encode(meshprovisionerclient.Provision{Provision: "oidc", Consumer: meshprovisionerclient.End{Module: "photos", Node: "ace"},
|
||||
Provider: meshprovisionerclient.End{Module: "keycloak", Node: "novox"}, State: meshprovisionerclient.Asked, Why: "the provider is applying", At: c.t})
|
||||
w.Want("want.ace.web", held2, Put, c.t)
|
||||
w.Provision("ace.photos.oidc", asked2, Put)
|
||||
for bucket, key := range waits {
|
||||
expire(w, c, pulseKeyOf(t, bucket, key), 31*time.Minute)
|
||||
}
|
||||
for bucket, key := range waits {
|
||||
f := o.findings[FindingKey(ScopeDelivery, stuckKeyOf(t, bucket, key), KindStuck)]
|
||||
if f.Kind != KindStuck || !f.Since.Equal(t0) || !strings.Contains(f.Summary, "bound 30m0s") {
|
||||
t.Fatalf("%s: findings = %v", bucket, o.findings)
|
||||
}
|
||||
expire(w, c, pulseKeyOf(t, ProvisionsBucket, "ace.photos.oidc"), 31*time.Minute)
|
||||
f := o.findings[FindingKey(ScopeDelivery, stuckKeyOf(t, ProvisionsBucket, "ace.photos.oidc"), KindStuck)]
|
||||
if f.Kind != KindStuck || !f.Since.Equal(t0) || !strings.Contains(f.Summary, "bound 30m0s") {
|
||||
t.Fatalf("findings = %v", o.findings)
|
||||
}
|
||||
w.Want("want.ace.web", released, Put, c.t)
|
||||
if _, timed := o.pulses[pulseKeyOf(t, testWants(), "want.ace.web")]; !timed {
|
||||
t.Fatalf("a want no longer held is not timed: pulses %v", o.pulses)
|
||||
}
|
||||
w.Provision("ace.photos.oidc", provided, Put)
|
||||
}, map[string]string{}},
|
||||
}
|
||||
@@ -551,37 +489,45 @@ func TestTheWatchmansFindings(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
const (
|
||||
revKey = "m.00000000000000000042"
|
||||
groupKey = "g.feat/take-in"
|
||||
)
|
||||
// revKey is the take-in wait's key of main's source revision 42 (design 47 H-A-94 (2)).
|
||||
var revKey = TakeInKey("main", 42)
|
||||
|
||||
func takeInEvent(t *testing.T, kind, key string) string {
|
||||
// pendingBody is a take-in.pending's body, in the contract's form.
|
||||
func pendingBody(t *testing.T, branch string, source int64, why string) []byte {
|
||||
t.Helper()
|
||||
e, err := takein.TakeInEvent(kind)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return e
|
||||
}
|
||||
|
||||
// takeInBody is a take-in event's body for a key: a group's when it starts with g.
|
||||
func takeInBody(t *testing.T, kind, key string, since time.Time) []byte {
|
||||
t.Helper()
|
||||
body := takein.TakeIn{Kind: kind, Key: key, What: takein.TakeInRevision, Since: since,
|
||||
Merges: []takein.Merge{{Repository: "novox/a", Number: 1, Merge: "c0ffee"}}}
|
||||
if key[0] == 'g' {
|
||||
body.What, body.Group = takein.TakeInGroup, "feat/take-in"
|
||||
} else if kind == takein.TakeInPending {
|
||||
body.Lost = []string{key}
|
||||
}
|
||||
b, err := takein.EncodeTakeIn(body)
|
||||
b, err := json.Marshal(meshcatalog.TakeInPending{Branch: branch, Source: source, Why: why})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// takenBody is a take-in.taken's body naming its movers, in the contract's form.
|
||||
func takenBody(t *testing.T, branch string, source int64, movers ...meshcatalog.TakeInEntry) []byte {
|
||||
t.Helper()
|
||||
b, err := json.Marshal(meshcatalog.TakeInTaken{Branch: branch, Source: source, Call: "fedcba9876543210", Revision: source, Entries: movers})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// fakeEntries answers a mover's catalog entry by fingerprint: at once when answer is set, else asked.
|
||||
type fakeEntries struct {
|
||||
entries map[string]meshcatalog.CatalogEntry
|
||||
answer bool
|
||||
released []string
|
||||
}
|
||||
|
||||
func (f *fakeEntries) Need(_ context.Context, fp string) (meshcatalog.CatalogEntry, meshcatalogclient.EntryState) {
|
||||
if e, ok := f.entries[fp]; ok && f.answer {
|
||||
return e, meshcatalogclient.EntryAnswered
|
||||
}
|
||||
return meshcatalog.CatalogEntry{}, meshcatalogclient.EntryAsked
|
||||
}
|
||||
|
||||
func (f *fakeEntries) Release(fp string) { f.released = append(f.released, fp) }
|
||||
|
||||
// Design 47 G §689, A-14, A-22: a token other than the kind goes into the contract finding's id and reads back as it was
|
||||
// written; the finding's time is its `since`; the resolved words are not on the wire.
|
||||
func TestAFindingIsTheContractsWithItsRefInItsID(t *testing.T) {
|
||||
|
||||
@@ -7,7 +7,6 @@ import (
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshwatchdogsclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
|
||||
@@ -124,7 +123,6 @@ func (w *Watchman) Want(key string, value []byte, op Op, at time.Time) {
|
||||
return
|
||||
case Delete:
|
||||
delete(w.wants, key)
|
||||
delete(w.held, key)
|
||||
w.moved(w.wantsBucket(), key)
|
||||
return
|
||||
}
|
||||
@@ -136,30 +134,16 @@ func (w *Watchman) Want(key string, value []byte, op Op, at time.Time) {
|
||||
}
|
||||
if want.Removed {
|
||||
delete(w.wants, key)
|
||||
delete(w.held, key)
|
||||
w.moved(w.wantsBucket(), key)
|
||||
return
|
||||
}
|
||||
if want.Hold {
|
||||
// A want awaiting its hand placement (design 47 H §772) is not progress owed, so it is not timed (H-A-95 (1)).
|
||||
delete(w.wants, key)
|
||||
w.moved(w.wantsBucket(), key)
|
||||
return
|
||||
}
|
||||
w.wants[key] = want.Fp
|
||||
if want.Newest != "" || want.Rule != 0 {
|
||||
// A want held by a rule (ADR 0603 §8) waits on the rule, not on the node: its wait is the hold's, from the time the
|
||||
// assigner wrote it, and the node running what the want names does not end it.
|
||||
if at.IsZero() {
|
||||
at = w.now()
|
||||
}
|
||||
w.held[key] = true
|
||||
w.mirrorKept(w.wantsBucket(), key,
|
||||
describe("the hold on %s on %s at %s (rule %d) to end, bound %s: %s", module, node, shortFP(want.Fp), want.Rule, w.stepBound, want.Words),
|
||||
"held|"+want.Fp+"|"+want.Newest, at)
|
||||
return
|
||||
}
|
||||
if w.held[key] {
|
||||
delete(w.held, key)
|
||||
if pk, err := meshwatchdogsclient.PulseKey(w.wantsBucket(), key); err == nil {
|
||||
delete(w.starts, pk)
|
||||
}
|
||||
w.stuckMoved(w.wantsBucket(), key) // the hold ended: its finding is cleared
|
||||
}
|
||||
if w.have(node, module) == want.Fp {
|
||||
w.moved(w.wantsBucket(), key)
|
||||
return
|
||||
|
||||
@@ -30,6 +30,5 @@ func BucketsOf(module string) OwnBuckets {
|
||||
|
||||
// The findings a value of another part is raised as, named by their words.
|
||||
const (
|
||||
FindingManifestOutgrown = "manifest-outgrown"
|
||||
FindingNodeAccountFallback = "node-account-fallback"
|
||||
)
|
||||
|
||||
@@ -5,115 +5,14 @@ import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
)
|
||||
|
||||
// What the runner and the gate say of a watched branch's head, heard as events (design 47 G §12, §75, §112): a head the
|
||||
// gate says moved (`source-moved`) that starts no pipeline within five minutes is the finding `pipeline-stands`, naming the
|
||||
// branch and its head, raised by the condition keeper and cleared by the runner's `pipeline-started` for that head.
|
||||
|
||||
// KindPipelineStands is a watched branch's head, or a step, standing still (design 47 G §12, §75).
|
||||
// KindPipelineStands is a pipeline step standing still (design 47 H §741; G §12, §75): the watchman learns no head, so a
|
||||
// watched branch's head that starts no pipeline is the gate's head-untaken alone, and a pipeline waiting its turn raises
|
||||
// nothing (steps.go).
|
||||
const KindPipelineStands = "pipeline-stands"
|
||||
|
||||
// HeadBound is how long a watched branch's head may stand with no pipeline started (design 47 G §12, §112: the record's five
|
||||
// minutes).
|
||||
const HeadBound = 5 * time.Minute
|
||||
|
||||
// headRow is the signals table's row of a head with no pipeline.
|
||||
const headRow = "head-pipeline"
|
||||
|
||||
// head is a watched branch's head still waiting for its pipeline.
|
||||
type head struct {
|
||||
revision int64
|
||||
at time.Time
|
||||
}
|
||||
|
||||
// SourceMoved hears the seat mesh-delivery's `source-moved`: the branch's head is a new source revision, and its pipeline
|
||||
// is due within HeadBound. A move made by a gathered group is a member governed by the take-in's own bound (ADR 0633 §5),
|
||||
// so it is not timed here (design 47 G §12). A later move of the same branch replaces the head that waits.
|
||||
func (w *Watchman) SourceMoved(body []byte) {
|
||||
var m meshdelivery.SourceMovedEvent
|
||||
if err := json.Unmarshal(body, &m); err != nil || m.Branch == "" || m.Revision == 0 {
|
||||
w.say("a source-moved does not read, left: %v", err)
|
||||
return
|
||||
}
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
id := headID(m.Branch)
|
||||
w.unsignal(headRow, id)
|
||||
w.unfind(FindingKey(ScopeDelivery, id, KindPipelineStands))
|
||||
if m.Group != "" {
|
||||
delete(w.heads, m.Branch)
|
||||
return
|
||||
}
|
||||
at := w.now()
|
||||
if m.At != nil && !m.At.IsZero() {
|
||||
at = *m.At
|
||||
}
|
||||
w.heads[m.Branch] = head{revision: m.Revision, at: at}
|
||||
w.timeHead(m.Branch)
|
||||
}
|
||||
|
||||
// timeHead times the head of a branch that waits for its pipeline, w.mu held: five minutes from its own time, unless the
|
||||
// branch has a pending revision, whose take-in wait governs it (design 47 G §497) and which times it again once taken in.
|
||||
func (w *Watchman) timeHead(branch string) {
|
||||
h, ok := w.heads[branch]
|
||||
if !ok || w.branchPending(branch) {
|
||||
return
|
||||
}
|
||||
id := headID(branch)
|
||||
f := Finding{Scope: ScopeDelivery, ID: id, Kind: KindPipelineStands, Severity: Warning, Source: headRow, Since: h.at,
|
||||
Summary: fmt.Sprintf("the head of %s (source revision %d) has started no pipeline since %s", branch, h.revision, stamp(h.at)),
|
||||
Said: fmt.Sprintf("no pipeline in %s", HeadBound),
|
||||
Headline: "A change has started no pipeline",
|
||||
Explanation: "The head of the watched branch " + branch + " (source revision " + fmt.Sprint(h.revision) + ") moved " +
|
||||
HeadBound.String() + " ago or more and the pipeline runner has said no pipeline started for it.",
|
||||
Needs: "the runner to start the pipeline (its verbs mesh-pipeline-runner.stands and mesh-pipeline-runner.list say where it is)"}
|
||||
left := HeadBound - w.now().Sub(h.at)
|
||||
if left > 0 {
|
||||
w.signal(headRow, id, left, signalDue{Do: dueFind, Finding: f})
|
||||
return
|
||||
}
|
||||
w.find(f)
|
||||
}
|
||||
|
||||
// branchPending is whether an open take-in wait names the branch (design 47 G §497): its head is behind a pending revision.
|
||||
func (w *Watchman) branchPending(branch string) bool {
|
||||
for _, b := range w.pendingBranch {
|
||||
if b == branch {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// PipelineStarted hears the runner's `pipeline-started`: it ends the wait of the head it is for (the branch's head at that
|
||||
// source revision or a later one). A run on request is no head's pipeline.
|
||||
func (w *Watchman) PipelineStarted(body []byte) {
|
||||
var p meshpipelinerunner.PipelineStartedEvent
|
||||
if err := json.Unmarshal(body, &p); err != nil || p.Branch == "" {
|
||||
w.say("a pipeline-started does not read, left: %v", err)
|
||||
return
|
||||
}
|
||||
if p.Run != "" {
|
||||
return
|
||||
}
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
h, ok := w.heads[p.Branch]
|
||||
if !ok || p.Source < h.revision {
|
||||
return
|
||||
}
|
||||
delete(w.heads, p.Branch)
|
||||
id := headID(p.Branch)
|
||||
w.unsignal(headRow, id)
|
||||
w.unfind(FindingKey(ScopeDelivery, id, KindPipelineStands))
|
||||
}
|
||||
|
||||
// headID is the finding's id of a branch's head.
|
||||
func headID(branch string) string { return "head-" + unsafeKey.ReplaceAllString(branch, "_") }
|
||||
|
||||
// KindBuildCapacityWaits is a build seat whose oldest queued step has waited the bound (design 47 G §86).
|
||||
const KindBuildCapacityWaits = "build-capacity-waits"
|
||||
|
||||
|
||||
@@ -7,115 +7,8 @@ import (
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
|
||||
)
|
||||
|
||||
func moved(branch string, rev int64, group string, at time.Time) []byte {
|
||||
b, _ := json.Marshal(meshdelivery.SourceMovedEvent{Branch: branch, Revision: rev, Group: group, At: &at})
|
||||
return b
|
||||
}
|
||||
|
||||
// takeInBodyOn is a take-in event's body that names its watched branch (design 47 G §493).
|
||||
func takeInBodyOn(t *testing.T, kind, key, branch string, since time.Time) []byte {
|
||||
t.Helper()
|
||||
var m map[string]any
|
||||
if err := json.Unmarshal(takeInBody(t, kind, key, since), &m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m["branch"] = branch
|
||||
b, err := json.Marshal(m)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func started(branch string, rev int64, run string) []byte {
|
||||
b, _ := json.Marshal(meshpipelinerunner.PipelineStartedEvent{Branch: branch, Source: rev, Run: run, Module: "m", Pipeline: "p"})
|
||||
return b
|
||||
}
|
||||
|
||||
// design 47 G §12, §79 (test 76): a head with no pipeline within five minutes is `pipeline-stands`, cleared by the
|
||||
// runner's `pipeline-started` for that head; a gathered group's member and a run on request do not.
|
||||
func TestAHeadWithNoPipelineStands(t *testing.T) {
|
||||
key := FindingKey(ScopeDelivery, headID("main"), KindPipelineStands)
|
||||
cases := []struct {
|
||||
name string
|
||||
run func(w *Watchman, o *fakeOut, c *clock)
|
||||
stand bool
|
||||
}{
|
||||
{"five minutes and no pipeline", func(w *Watchman, o *fakeOut, c *clock) {
|
||||
w.SourceMoved(moved("main", 7, "", c.t))
|
||||
if o.ttls[signalKey(headRow, headID("main"))] != HeadBound || len(o.findings) != 0 {
|
||||
t.Fatalf("within the bound: %v %v", o.ttls, o.findings)
|
||||
}
|
||||
expire(w, c, signalKey(headRow, headID("main")), HeadBound)
|
||||
}, true},
|
||||
{"its pipeline-started clears it", func(w *Watchman, o *fakeOut, c *clock) {
|
||||
w.SourceMoved(moved("main", 7, "", c.t))
|
||||
expire(w, c, signalKey(headRow, headID("main")), HeadBound)
|
||||
w.PipelineStarted(started("main", 7, ""))
|
||||
}, false},
|
||||
{"a pipeline before the bound raises none", func(w *Watchman, o *fakeOut, c *clock) {
|
||||
w.SourceMoved(moved("main", 7, "", c.t))
|
||||
w.PipelineStarted(started("main", 8, ""))
|
||||
if _, ok := o.pulses[signalKey(headRow, headID("main"))]; ok {
|
||||
t.Fatal("the pulse stands")
|
||||
}
|
||||
}, false},
|
||||
{"a pipeline of an older head does not clear a newer head", func(w *Watchman, o *fakeOut, c *clock) {
|
||||
w.SourceMoved(moved("main", 7, "", c.t))
|
||||
w.PipelineStarted(started("main", 6, ""))
|
||||
expire(w, c, signalKey(headRow, headID("main")), HeadBound)
|
||||
}, true},
|
||||
{"a gathered group's member is not timed", func(w *Watchman, o *fakeOut, c *clock) {
|
||||
w.SourceMoved(moved("main", 7, "grp", c.t))
|
||||
if len(o.pulses) != 0 {
|
||||
t.Fatalf("a pulse for a member: %v", o.pulses)
|
||||
}
|
||||
}, false},
|
||||
{"a head behind a pending revision of its branch is not timed", func(w *Watchman, o *fakeOut, c *clock) {
|
||||
w.TakeIn(takeInEvent(t, takein.TakeInPending, "m.1"), takeInBodyOn(t, takein.TakeInPending, "m.1", "main", c.t))
|
||||
w.SourceMoved(moved("main", 7, "", c.t))
|
||||
if _, ok := o.pulses[signalKey(headRow, headID("main"))]; ok {
|
||||
t.Fatalf("a pulse for a head behind a pending revision: %v", o.pulses)
|
||||
}
|
||||
}, false},
|
||||
{"a pending revision of another branch does not hold it", func(w *Watchman, o *fakeOut, c *clock) {
|
||||
w.TakeIn(takeInEvent(t, takein.TakeInPending, "m.1"), takeInBodyOn(t, takein.TakeInPending, "m.1", "other", c.t))
|
||||
w.SourceMoved(moved("main", 7, "", c.t))
|
||||
expire(w, c, signalKey(headRow, headID("main")), HeadBound)
|
||||
}, true},
|
||||
{"a head whose pending revision is taken in is timed again", func(w *Watchman, o *fakeOut, c *clock) {
|
||||
w.SourceMoved(moved("main", 7, "", c.t))
|
||||
w.TakeIn(takeInEvent(t, takein.TakeInPending, "m.1"), takeInBodyOn(t, takein.TakeInPending, "m.1", "main", c.t))
|
||||
if _, ok := o.pulses[signalKey(headRow, headID("main"))]; ok {
|
||||
t.Fatalf("the head is timed behind a pending revision: %v", o.pulses)
|
||||
}
|
||||
c.t = c.t.Add(HeadBound)
|
||||
w.TakeIn(takeInEvent(t, takein.TakeInTaken, "m.1"), takeInBodyOn(t, takein.TakeInTaken, "m.1", "main", c.t))
|
||||
if _, ok := o.findings[key]; !ok {
|
||||
t.Fatalf("the head, past its five minutes when taken in, stands: %v", o.findings)
|
||||
}
|
||||
}, true},
|
||||
{"a run on request is no head's pipeline", func(w *Watchman, o *fakeOut, c *clock) {
|
||||
w.SourceMoved(moved("main", 7, "", c.t))
|
||||
w.PipelineStarted(started("main", 7, "run-1"))
|
||||
expire(w, c, signalKey(headRow, headID("main")), HeadBound)
|
||||
}, true},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
w, o, c := setup()
|
||||
tc.run(w, o, c)
|
||||
if _, ok := o.findings[key]; ok != tc.stand {
|
||||
t.Fatalf("the finding stands %v, want %v: %v", ok, tc.stand, o.findings)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func queuedStep(pipeline, step, seat, outcome string, started time.Time) []byte {
|
||||
b, _ := json.Marshal(meshpipelinerunner.StepEvent{Pipeline: pipeline, Branch: "main", Module: "m", Commit: "c",
|
||||
Result: meshpipelinerunner.StepResult{Step: step, Seat: seat, Outcome: outcome, Started: &started}})
|
||||
@@ -249,37 +142,31 @@ func TestARecordOpenADayIsAFinding(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// design 47 G §502, A-200 (WM16): a record is tied to the entries of the take-in.taken of its milestone `<branch>.<source revision>`; it
|
||||
// is covered only while one of those entries is held by an `acting-part-late` wait, so another record's late entry covers nothing.
|
||||
func TestARecordIsCoveredOnlyByTheWaitsOfItsOwnEntries(t *testing.T) {
|
||||
// design 47 G §502, A-200 (WM16), H-A-25: a record is tied to its source revision by its milestone `<branch>.<source>`, the
|
||||
// key of that revision's take-in wait: while its take-in is held and said, take-in-waits covers it; once taken in it is open
|
||||
// at 24 hours, and another record's held take-in covers nothing of it.
|
||||
func TestARecordIsCoveredOnlyWhileItsOwnTakeInIsHeld(t *testing.T) {
|
||||
w, o, c := setup()
|
||||
taken := func(key string, source int64, module string) {
|
||||
var m map[string]any
|
||||
if err := json.Unmarshal(takeInBody(t, takein.TakeInTaken, key, c.t), &m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m["branch"], m["source"] = "main", source
|
||||
m["entries"] = []map[string]any{{"commit": "c", "fingerprint": "f", "module": module}}
|
||||
b, _ := json.Marshal(m)
|
||||
w.TakeIn(takeInEvent(t, takein.TakeInTaken, key), b)
|
||||
}
|
||||
taken("m.1", 7, "web")
|
||||
taken("m.2", 8, "db")
|
||||
w.TakeInCaughtUp()
|
||||
w.TakeIn(TakeInTakenEvent, takenBody(t, "main", 8), c.t)
|
||||
w.TakeIn(TakeInPendingEvent, pendingBody(t, "main", 7, "the catalog store is full"), c.t)
|
||||
rec := func(id, milestone string) []byte {
|
||||
b, _ := json.Marshal(meshdelivery.RecordOpenedEvent{Record: id, Milestone: milestone, At: &c.t})
|
||||
b, _ := json.Marshal(meshdelivery.RecordOpenedEvent{Record: id, Milestone: milestone, Branch: "main", Call: "fedcba9876543210", At: &c.t})
|
||||
return b
|
||||
}
|
||||
w.RecordOpen(rec("rWeb", "main.7"))
|
||||
w.RecordOpen(rec("rDb", "main.8"))
|
||||
late, _ := json.Marshal(meshwants.Wait{Step: "place", Part: "mesh-delivery-indexer", Node: "ace", Module: "web", Since: c.t, Bound: "30m"})
|
||||
w.WaitHeld("mesh-wants", late)
|
||||
expire(w, c, signalKey(actingRow, "mesh-wants-place-mesh-delivery-indexer-ace-web"), 30*time.Minute)
|
||||
expire(w, c, signalKey(recordRow, "rWeb"), RecordOpenBound)
|
||||
expire(w, c, signalKey(recordRow, "rDb"), 0)
|
||||
if _, ok := o.findings[FindingKey(ScopeDelivery, "rWeb", KindRecordOpen)]; ok {
|
||||
t.Fatalf("a record whose entry is held by acting-part-late is covered: %v", o.findings)
|
||||
w.RecordOpen(rec("rHeld", "main.7"))
|
||||
w.RecordOpen(rec("rTaken", "main.8"))
|
||||
expire(w, c, signalKey(takeInRow, TakeInKey("main", 7)), DefaultTakeInBound)
|
||||
expire(w, c, signalKey(recordRow, "rHeld"), RecordOpenBound-DefaultTakeInBound)
|
||||
expire(w, c, signalKey(recordRow, "rTaken"), 0)
|
||||
if _, ok := o.findings[FindingKey(ScopeDelivery, "rHeld", KindRecordOpen)]; ok {
|
||||
t.Fatalf("a record whose take-in is held and said is not covered: %v", o.findings)
|
||||
}
|
||||
if _, ok := o.findings[FindingKey(ScopeDelivery, "rDb", KindRecordOpen)]; !ok {
|
||||
t.Fatalf("a record of other entries is open at 24 hours whatever else is late: %v", o.findings)
|
||||
if _, ok := o.findings[FindingKey(ScopeDelivery, "rTaken", KindRecordOpen)]; !ok {
|
||||
t.Fatalf("a record taken in is not open at 24 hours whatever else is held: %v", o.findings)
|
||||
}
|
||||
w.TakeIn(TakeInTakenEvent, takenBody(t, "main", 7), c.t)
|
||||
if _, ok := o.findings[FindingKey(ScopeDelivery, "rHeld", KindRecordOpen)]; !ok {
|
||||
t.Fatalf("a record whose take-in ended is not open: %v", o.findings)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,7 +3,6 @@ package watch
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
|
||||
@@ -14,15 +13,14 @@ import (
|
||||
// - `filling`, `full` and `room` of a store against its cap: `store-filling` while it stands past its cap's share, urgent
|
||||
// when it is full, cleared by `room` or by a `filling` under FillingFrom;
|
||||
// - `record-opened` and `record-ended` of a delivery record: `record-open` when it has stood open for 24 hours, unless
|
||||
// another condition already tells why (`acting-part-late`, or a store `full`).
|
||||
// another condition already tells why (a store `full`, or the take-in held).
|
||||
// Each is a row of the signals table: its pulse is its bound, written at the event, so a restart loses none.
|
||||
|
||||
// The kinds of these findings.
|
||||
const (
|
||||
KindActingPartLate = "acting-part-late"
|
||||
KindPutStands = "put-stands"
|
||||
KindStoreFilling = "store-filling"
|
||||
KindRecordOpen = "record-open"
|
||||
KindPutStands = "put-stands"
|
||||
KindStoreFilling = "store-filling"
|
||||
KindRecordOpen = "record-open"
|
||||
)
|
||||
|
||||
// The rows of the signals table of these findings, which a restart keeps.
|
||||
@@ -126,12 +124,11 @@ func (w *Watchman) RecordEnded(body []byte) {
|
||||
w.unfind(FindingKey(ScopeDelivery, r.Record, KindRecordOpen))
|
||||
}
|
||||
|
||||
// recordCovered says another condition tells the operator why a record is open (design 47 G §502, A-200): the record is tied to the
|
||||
// entries of its source revision's `take-in.taken` by its milestone, `<branch>.<source revision>`, and is covered while one of those
|
||||
// entries is held by a wait that raised `acting-part-late`, or a store says `full`; a record whose revision is not known to be taken in
|
||||
// is covered by a standing `take-in-waits`.
|
||||
// recordCovered says another condition tells the operator why a record is open (design 47 G §502, A-200): a store says `full`, or
|
||||
// the record's source revision, its milestone `<branch>.<source>` (H-A-25), is not known to be taken in while a `take-in-waits`
|
||||
// stands.
|
||||
func (w *Watchman) recordCovered(record string) bool {
|
||||
entries, taken := w.takenEntries[w.recordMilestone[record]]
|
||||
taken := w.taken[w.recordMilestone[record]]
|
||||
for _, v := range w.findings {
|
||||
f, err := DecodeFinding(v)
|
||||
if err != nil {
|
||||
@@ -140,12 +137,6 @@ func (w *Watchman) recordCovered(record string) bool {
|
||||
switch {
|
||||
case f.Kind == KindStoreFilling && f.Severity == Urgent:
|
||||
return true
|
||||
case f.Kind == KindActingPartLate && taken:
|
||||
for _, module := range entries {
|
||||
if strings.HasSuffix(f.ID, "-"+module) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
case f.Kind == KindTakeInWaits && !taken:
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -777,8 +777,8 @@ var SignalRows = []struct{ Row, Signal, Bound, Kind string }{
|
||||
{"S10", "the self-check's checked", (3 * SelfCheckEvery).String(), KindSelfCheckSilent},
|
||||
{"S11", "a machine's tool runner heartbeat", "3 × its interval (" + (3 * HeartbeatEvery).String() + " when it says none)", KindToolsSilent},
|
||||
{"S21", "a machine's provisioner client's files not placed", ApplyEvery.String(), KindApplyFailing},
|
||||
{"take-in", "a source revision or a delivery group the indexer holds back (ADR 0633 §5)", TakeInRevisionBound.String() + " a revision, " + TakeInGroupBound.String() + " a group", KindTakeInWaits},
|
||||
{actingRow, "an acting part's wait held, or a put's, a put without hold that stands (the seats' wait-held, ended by wait-ended)", "the wait's own bound", KindActingPartLate + ", " + KindPutStands},
|
||||
{takeInRow, "a source revision whose take-in the indexer holds (take-in.pending, ended by take-in.taken; design 47 H §744)", "the setting take-in-bound, " + DefaultTakeInBound.String() + " by default", KindTakeInWaits},
|
||||
{actingRow, "a put without hold that stands (the seat mesh-wants' wait-held, ended by wait-ended)", "the wait's own bound", KindPutStands},
|
||||
{storeRow, "a store of the delivery's parts filling or full (the seats' filling, full and room)", "filling from " + fmt.Sprintf("%.0f", FillingFrom*100) + " %, full urgent", KindStoreFilling},
|
||||
{recordRow, "a delivery record open (the seat mesh-delivery's record-opened, ended by record-ended)", RecordOpenBound.String(), KindRecordOpen},
|
||||
{"provisions", "a provision of mesh-provisioner in waits (ADR 0570 §7)", "at once", KindNeedsOperator},
|
||||
|
||||
@@ -9,75 +9,107 @@ import (
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
|
||||
)
|
||||
|
||||
// KindTakeInWaits is a source revision or a delivery group the indexer holds back and has not taken in within its bound
|
||||
// (novox/hq ADR 0633 §5, issue 1585): a revision for the indexer build it names, a group for its end. It stands until a
|
||||
// `take-in.taken` ends the wait, whoever ended it (the named indexer, a revert, a hand take-in or a drop).
|
||||
// KindTakeInWaits is a source revision the indexer holds back and has not taken in within the setting take-in-bound
|
||||
// (design 47 H §744, H-A-94 (2); G §497): its take-in cannot be written. It stands until a `take-in.taken` of that source
|
||||
// ends the wait, whoever ended it (the indexer's take-in, a hand take-in or a skip).
|
||||
const KindTakeInWaits = "take-in-waits"
|
||||
|
||||
// The bounds of a take-in wait, from the `take-in.pending` event's own since.
|
||||
// DefaultTakeInBound is the setting take-in-bound's default (design 47 H §744, H-A-94 (2)).
|
||||
const DefaultTakeInBound = 60 * time.Minute
|
||||
|
||||
// The take-in events of the indexer, by their names relative to it.
|
||||
const (
|
||||
TakeInRevisionBound = 60 * time.Minute
|
||||
TakeInGroupBound = 30 * time.Minute
|
||||
TakeInPendingEvent = "take-in.pending"
|
||||
TakeInTakenEvent = "take-in.taken"
|
||||
)
|
||||
|
||||
// takeInRow is the signals table's row of a take-in wait.
|
||||
const takeInRow = "take-in"
|
||||
|
||||
// TakeIn hears one take-in event of the indexer, by its name relative to the indexer (`take-in.<kind>`), the wait's key
|
||||
// in its body: a pending one starts the wait for its key, whose bound runs from the event's since, and a taken one ends it. A group's
|
||||
// taken event ends the group's wait only: the revision it becomes is its own key and starts its own, longer wait.
|
||||
func (w *Watchman) TakeIn(event string, body []byte) {
|
||||
kind, err := takein.ParseTakeInEvent(event)
|
||||
if err != nil {
|
||||
w.say("a take-in event is left: %v", err)
|
||||
return
|
||||
// TakeInKey is a take-in wait's key, `<branch>.<source>` (design 47 H-A-94 (2)).
|
||||
func TakeInKey(branch string, source int64) string { return fmt.Sprintf("%s.%d", branch, source) }
|
||||
|
||||
// HeldTakeIn is one take-in the indexer holds, as indexer_pending answers it (design 47 H-A-97, H-A-102): the contract's
|
||||
// form with since, when it was held.
|
||||
type HeldTakeIn struct {
|
||||
meshcatalog.TakeInPending
|
||||
Since time.Time `json:"since"`
|
||||
}
|
||||
|
||||
// HeldTakeIns is the answer of indexer_pending (design 47 H-A-102).
|
||||
type HeldTakeIns struct {
|
||||
Held []HeldTakeIn `json:"held"`
|
||||
}
|
||||
|
||||
// DecodeHeldTakeIns reads the answer of indexer_pending.
|
||||
func DecodeHeldTakeIns(raw []byte) (HeldTakeIns, error) {
|
||||
var a HeldTakeIns
|
||||
if err := json.Unmarshal(raw, &a); err != nil {
|
||||
return HeldTakeIns{}, fmt.Errorf("not an answer of indexer_pending: %w", err)
|
||||
}
|
||||
t, err := takein.DecodeTakeIn(body)
|
||||
if err != nil {
|
||||
w.say("the body of %s cannot be read, left: %v", event, err)
|
||||
return
|
||||
for _, h := range a.Held {
|
||||
if h.Branch == "" || h.Source == 0 || h.Since.IsZero() {
|
||||
return HeldTakeIns{}, fmt.Errorf("indexer_pending names a held take-in with no branch, source or since: %+v", h)
|
||||
}
|
||||
}
|
||||
if t.Kind != kind {
|
||||
w.say("the body of %s says the kind %q, left", event, t.Kind)
|
||||
return
|
||||
}
|
||||
// The wait names the watched branch it is of (design 47 G §493), by the contract's body.
|
||||
var named meshcatalog.TakeInPending
|
||||
_ = json.Unmarshal(body, &named)
|
||||
var taken meshcatalog.TakeInTaken
|
||||
_ = json.Unmarshal(body, &taken)
|
||||
key := t.Key
|
||||
return a, nil
|
||||
}
|
||||
|
||||
// SetTakeInBound sets the setting take-in-bound; zero or less keeps its default.
|
||||
func (w *Watchman) SetTakeInBound(d time.Duration) {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
w.takeIn(kind, key, t)
|
||||
w.pendingOf(kind, key, named.Branch)
|
||||
if kind == takein.TakeInTaken && taken.Branch != "" && taken.Source != 0 && len(taken.Entries) > 0 {
|
||||
var modules []string
|
||||
for _, e := range taken.Entries {
|
||||
modules = append(modules, e.Module)
|
||||
}
|
||||
w.takenEntries[fmt.Sprintf("%s.%d", taken.Branch, taken.Source)] = modules // a record's milestone (A-200)
|
||||
w.rejudgeRecords()
|
||||
if d > 0 {
|
||||
w.takeInBound = d
|
||||
}
|
||||
}
|
||||
|
||||
// SeedTakeIn starts, at the watchman's start and before its consumer on the take-in stream, the wait of every revision
|
||||
// and group the indexer holds now, as indexer_pending answers them (ADR 0633 §5): the consumer is durable, so the
|
||||
// TakeIn hears one take-in event of the indexer, by its name relative to the indexer, heard at the message's own time:
|
||||
// a `take-in.pending` starts the wait of its `<branch>.<source>`, whose bound runs from that time, and a `take-in.taken`
|
||||
// ends it, and asks the catalog entry of each mover it names (busstate.go).
|
||||
func (w *Watchman) TakeIn(event string, body []byte, at time.Time) {
|
||||
switch event {
|
||||
case TakeInPendingEvent:
|
||||
var p meshcatalog.TakeInPending
|
||||
if err := json.Unmarshal(body, &p); err != nil || p.Branch == "" || p.Source == 0 {
|
||||
w.say("the body of %s cannot be read, left: %v", event, err)
|
||||
return
|
||||
}
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
w.takeInHeldSince(TakeInKey(p.Branch, p.Source), p.Why, at)
|
||||
case TakeInTakenEvent:
|
||||
var t meshcatalog.TakeInTaken
|
||||
if err := json.Unmarshal(body, &t); err != nil || t.Branch == "" || t.Source == 0 {
|
||||
w.say("the body of %s cannot be read, left: %v", event, err)
|
||||
return
|
||||
}
|
||||
key := TakeInKey(t.Branch, t.Source)
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
w.takeInEnded(key)
|
||||
w.taken[key] = true // a record's milestone (A-200, H-A-25)
|
||||
w.rejudgeRecords()
|
||||
w.moversTaken(t.Entries, at)
|
||||
default:
|
||||
w.say("a take-in event %s is left: not take-in.pending or take-in.taken", event)
|
||||
}
|
||||
}
|
||||
|
||||
// SeedTakeIn starts, at the watchman's start and before its consumer on the take-in stream, the wait of every take-in
|
||||
// the indexer holds now, as indexer_pending answers it (design 47 H-A-94 (2), H-A-97): the consumer is durable, so the
|
||||
// pending events it acknowledged before a restart are not heard again, and a wait within its bound would otherwise be
|
||||
// lost. Each wait's bound runs from its own since, as a pending event's does; the stream's events that follow replace
|
||||
// or end it. A wait or finding the answer does not name is ended.
|
||||
func (w *Watchman) SeedTakeIn(a takein.PendingAnswer) {
|
||||
// lost. Each wait's bound runs from its own since; the stream's events that follow replace or end it. A wait or finding
|
||||
// the answer does not name is ended.
|
||||
func (w *Watchman) SeedTakeIn(a HeldTakeIns) {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
// A wait or finding of a key the answer does not name is ended as a take-in.taken would end it: the finding a previous
|
||||
// run wrote (read at the start, before the seed), a wait held from an event, a signal standing (design 47, Taken in by
|
||||
// the indexer it names). Only an answered seed gets here: one that could not ask ends nothing.
|
||||
// Only an answered seed gets here: one that could not ask ends nothing.
|
||||
named := map[string]bool{}
|
||||
for _, wait := range a.Waits {
|
||||
named[wait.Key] = true
|
||||
for _, h := range a.Held {
|
||||
named[TakeInKey(h.Branch, h.Source)] = true
|
||||
}
|
||||
stale := map[string]bool{}
|
||||
for _, value := range w.findings {
|
||||
@@ -96,64 +128,36 @@ func (w *Watchman) SeedTakeIn(a takein.PendingAnswer) {
|
||||
}
|
||||
}
|
||||
for _, key := range slices.Sorted(maps.Keys(stale)) {
|
||||
w.takeIn(takein.TakeInTaken, key, takein.TakeIn{Kind: takein.TakeInTaken, Key: key})
|
||||
w.takeInEnded(key)
|
||||
}
|
||||
for _, wait := range a.Waits {
|
||||
w.takeIn(takein.TakeInPending, wait.Key, takein.TakeIn{Kind: takein.TakeInPending, Key: wait.Key, What: wait.What,
|
||||
Merges: wait.Merges, Group: wait.Group, Members: wait.Members, WaitsFor: wait.WaitsFor, Since: wait.Since})
|
||||
for _, h := range a.Held {
|
||||
w.takeInHeldSince(TakeInKey(h.Branch, h.Source), h.Why, h.Since)
|
||||
}
|
||||
}
|
||||
|
||||
// pendingOf records or ends the branch an open take-in wait names (design 47 G §497), w.mu held: the head of a branch behind a
|
||||
// pending revision raises no `pipeline-stands`, and is timed again, from the move of the head, once the branch has none.
|
||||
func (w *Watchman) pendingOf(kind, key, branch string) {
|
||||
if kind == takein.TakeInTaken {
|
||||
branch = w.pendingBranch[key]
|
||||
delete(w.pendingBranch, key)
|
||||
if branch != "" && !w.branchPending(branch) {
|
||||
w.timeHead(branch)
|
||||
}
|
||||
return
|
||||
}
|
||||
if branch == "" {
|
||||
return
|
||||
}
|
||||
w.pendingBranch[key] = branch
|
||||
if _, ok := w.heads[branch]; ok {
|
||||
id := headID(branch)
|
||||
w.unsignal(headRow, id)
|
||||
w.unfind(FindingKey(ScopeDelivery, id, KindPipelineStands))
|
||||
}
|
||||
// takeInEnded ends the wait of key, w.mu held.
|
||||
func (w *Watchman) takeInEnded(key string) {
|
||||
delete(w.takeInHeld, key)
|
||||
w.unsignal(takeInRow, key)
|
||||
w.unfind(FindingKey(ScopeDelivery, key, KindTakeInWaits))
|
||||
}
|
||||
|
||||
// takeIn is one take-in event of key, w.mu held.
|
||||
func (w *Watchman) takeIn(kind, key string, t takein.TakeIn) {
|
||||
finding := FindingKey(ScopeDelivery, key, KindTakeInWaits)
|
||||
if kind == takein.TakeInTaken {
|
||||
delete(w.takeInHeld, key)
|
||||
w.unsignal(takeInRow, key)
|
||||
w.unfind(finding)
|
||||
return
|
||||
// takeInHeldSince is the take-in of key held since since, for why, w.mu held: its finding is due take-in-bound from since.
|
||||
func (w *Watchman) takeInHeldSince(key, why string, since time.Time) {
|
||||
bound := w.takeInBound
|
||||
why = strings.TrimSpace(why)
|
||||
if why == "" {
|
||||
why = "the indexer said no reason"
|
||||
}
|
||||
bound, what := TakeInRevisionBound, "a source revision is held (a message lost or unread, no room for it, or its refusal)"
|
||||
if t.What == takein.TakeInGroup {
|
||||
bound, what = TakeInGroupBound, "a delivery group is gathered and its end has not come"
|
||||
}
|
||||
var merges []string
|
||||
for _, m := range t.Merges {
|
||||
merges = append(merges, m.Repository)
|
||||
}
|
||||
f := Finding{Scope: ScopeDelivery, ID: key, Kind: KindTakeInWaits, Severity: Warning, Source: "take-in", Since: t.Since,
|
||||
Summary: fmt.Sprintf("%s: %s, since %s (%s)", key, what, stamp(t.Since), strings.Join(merges, ", ")),
|
||||
f := Finding{Scope: ScopeDelivery, ID: key, Kind: KindTakeInWaits, Severity: Warning, Source: takeInRow, Since: since,
|
||||
Summary: fmt.Sprintf("%s: the take-in of this source revision is held, since %s (%s)", key, stamp(since), why),
|
||||
Said: fmt.Sprintf("held for %s", bound),
|
||||
Headline: "A merged change is held back by the indexer",
|
||||
Explanation: "The indexer holds " + what + " and has not taken it in within " + bound.String() + ", so what the merge changes does " +
|
||||
"not reach the machines.",
|
||||
Needs: "the take-in by hand (indexer_take_in_pending), or a drop (indexer_drop_pending)"}
|
||||
left := bound - w.now().Sub(t.Since)
|
||||
w.unfind(finding)
|
||||
w.unsignal(takeInRow, key)
|
||||
delete(w.takeInHeld, key)
|
||||
Explanation: "The indexer holds the take-in of the source revision " + key + " (" + why + ") and has not taken it in within " +
|
||||
bound.String() + ", so what the merge changes does not reach the machines.",
|
||||
Needs: "the take-in by hand (indexer_take_in_pending), or a skip (indexer_take_in_skip)"}
|
||||
left := bound - w.now().Sub(since)
|
||||
w.takeInEnded(key)
|
||||
switch {
|
||||
case left > 0:
|
||||
w.signal(takeInRow, key, left, signalDue{Do: dueFind, Finding: f})
|
||||
|
||||
@@ -10,31 +10,40 @@ import (
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
)
|
||||
|
||||
// What the holders of an acting part's wait say, heard as the seats' events (design 47 G §266, §269, ADR 0596 §2), never read
|
||||
// from their state: `wait-held` and `wait-ended` of a wait (the seat mesh-wants, and the seat mesh-pipeline-runner for its
|
||||
// `pipeline` step). A wait held past its bound is the finding `acting-part-late`, timed by that bound alone from the time
|
||||
// `wait-held` carries as `clock-from` (its `since` where it carries none), and ended by `wait-ended`, whoever ended it (the
|
||||
// part's new version running, or a release). Its pulse is its bound, written at the event, so a restart loses none.
|
||||
// What the holder of the seat mesh-wants says of a put's wait (design 47 G §498), heard as the seat's `wait-held` and
|
||||
// `wait-ended`, never read from its state: a put without hold that stands past its bound is the finding `put-stands`, timed
|
||||
// by that bound alone from the time `wait-held` carries as `clock-from` (its `since` where it carries none), and ended by
|
||||
// `wait-ended`. An acting part's wait is no finding (design 47 H §764, H-A-70). Its pulse is its
|
||||
// bound, written at the event, so a restart loses none.
|
||||
|
||||
// actingRow is the row of the signals table of an acting part's wait.
|
||||
// actingRow is the row of the signals table of a put's wait.
|
||||
const actingRow = "acting-wait"
|
||||
|
||||
func waitID(seat string, a meshwants.Wait) string {
|
||||
return strings.Join([]string{seat, a.Step, a.Part, a.Node, a.Module}, "-")
|
||||
}
|
||||
|
||||
// WaitHeld hears a seat's `wait-held`: the finding is due when the wait's bound has run from its clock.
|
||||
// WaitHeld hears a seat's `wait-held`: a put's finding is due when its bound has run from its clock; any other wait is left.
|
||||
func (w *Watchman) WaitHeld(seat string, body []byte) {
|
||||
a, ok := decodeWait(w, seat, "wait-held", body)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
bound, err := time.ParseDuration(a.Bound)
|
||||
if err != nil || bound <= 0 {
|
||||
w.say("the put of %s says no bound (%q), so it is never late", a.Module, a.Bound)
|
||||
return
|
||||
}
|
||||
from := a.Since
|
||||
if a.ClockFrom != nil {
|
||||
from = *a.ClockFrom
|
||||
}
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
w.waitHeld(seat, a)
|
||||
w.putStands(waitID(seat, a), a, bound, from)
|
||||
}
|
||||
|
||||
// WaitEnded hears a seat's `wait-ended`: the wait ran or was released, so its finding and its pulse end.
|
||||
// WaitEnded hears a seat's `wait-ended`: the put ran or was released, so its finding and its pulse end.
|
||||
func (w *Watchman) WaitEnded(seat string, body []byte) {
|
||||
a, ok := decodeWait(w, seat, "wait-ended", body)
|
||||
if !ok {
|
||||
@@ -44,57 +53,21 @@ func (w *Watchman) WaitEnded(seat string, body []byte) {
|
||||
defer w.mu.Unlock()
|
||||
id := waitID(seat, a)
|
||||
w.unsignal(actingRow, id)
|
||||
w.unfind(FindingKey(ScopeDelivery, id, KindActingPartLate))
|
||||
w.unfind(FindingKey(ScopeDelivery, id, KindPutStands))
|
||||
w.rejudgeRecords()
|
||||
}
|
||||
|
||||
// decodeWait reads a put's wait; a wait of another step is left unsaid, since it is no finding.
|
||||
func decodeWait(w *Watchman, seat, event string, body []byte) (meshwants.Wait, bool) {
|
||||
var a meshwants.Wait
|
||||
if err := json.Unmarshal(body, &a); err != nil || (a.Part == "" && a.Step != meshwants.WaitStepPut) || a.Module == "" || a.Since.IsZero() {
|
||||
if err := json.Unmarshal(body, &a); err != nil || (a.Step == meshwants.WaitStepPut && (a.Module == "" || a.Since.IsZero())) {
|
||||
w.say("the %s of the seat %s does not read, left: %v", event, seat, err)
|
||||
return a, false
|
||||
}
|
||||
return a, true
|
||||
}
|
||||
|
||||
// waitHeld is one wait held, w.mu held.
|
||||
func (w *Watchman) waitHeld(seat string, a meshwants.Wait) {
|
||||
bound, err := time.ParseDuration(a.Bound)
|
||||
if err != nil || bound <= 0 {
|
||||
w.say("the wait of %s at %s says no bound (%q), so it is never late", a.Module, a.Part, a.Bound)
|
||||
return
|
||||
}
|
||||
from := a.Since
|
||||
if a.ClockFrom != nil {
|
||||
from = *a.ClockFrom
|
||||
}
|
||||
id := waitID(seat, a)
|
||||
if a.Step == meshwants.WaitStepPut {
|
||||
w.putStands(id, a, bound, from)
|
||||
return
|
||||
}
|
||||
f := Finding{Scope: ScopeDelivery, ID: id, Kind: KindActingPartLate, Severity: Warning, Source: actingRow, Since: a.Since,
|
||||
Summary: fmt.Sprintf("%s on %s waits for %s to run %s at the step %s, since %s (bound %s)", a.Module, orWords(a.Node, "no node"), a.Part,
|
||||
orWords(a.Version, "its new version"), a.Step, stamp(a.Since), bound),
|
||||
Said: "held for " + bound.String(),
|
||||
Headline: "An acting part's new version has not come",
|
||||
Explanation: "A step waits until " + a.Part + " runs its new version, and its bound of " + bound.String() +
|
||||
" has run, so what the step does is held back.",
|
||||
Needs: "the part's new version to run, or the wait released by hand (the holder's release tool)"}
|
||||
left := bound - w.now().Sub(from)
|
||||
w.unsignal(actingRow, id)
|
||||
w.unfind(f.Key())
|
||||
if left > 0 {
|
||||
w.signal(actingRow, id, left, signalDue{Do: dueFind, Finding: f})
|
||||
return
|
||||
}
|
||||
w.find(f)
|
||||
w.rejudgeRecords() // a record whose entry this wait holds is covered now (§502)
|
||||
return a, a.Step == meshwants.WaitStepPut
|
||||
}
|
||||
|
||||
// putStands is a put's wait held (`step: put`, design 47 G §498): a put without hold that stands past its bound is said as the
|
||||
// finding `put-stands`, never `acting-part-late`, since no acting part is waited for. w.mu held.
|
||||
// finding `put-stands`, since no acting part is waited for. w.mu held.
|
||||
func (w *Watchman) putStands(id string, a meshwants.Wait, bound time.Duration, from time.Time) {
|
||||
f := Finding{Scope: ScopeDelivery, ID: id, Kind: KindPutStands, Severity: Warning, Source: actingRow, Since: a.Since,
|
||||
Summary: fmt.Sprintf("a put of %s on %s at %s stands, since %s (bound %s)", a.Module, orWords(a.Node, "no node"),
|
||||
@@ -114,17 +87,15 @@ func (w *Watchman) putStands(id string, a meshwants.Wait, bound time.Duration, f
|
||||
w.find(f)
|
||||
}
|
||||
|
||||
// What a node-engine says of a hold it keeps (design 47 G §499, §541), heard on the seat node-engine's `wait-held` and `wait-ended`
|
||||
// (`mesh.seat.node-engine.event.<event>.<node>.<module>`, the node the one that says it): a hold of the composition (`step:
|
||||
// compose`, with its reason) standing past composing's 30 minutes (§266) is the finding `delivery-waits`, naming the node, the
|
||||
// module and the reason, and a wait for a kept build (`step: kept-build`, naming the build node) standing past building's default
|
||||
// bound of 60 minutes is `kept-build-waits`. Each clears on `wait-ended`.
|
||||
// What a node-engine says of a wait for a kept build (design 47 G §499, §541), heard on the seat node-engine's `wait-held`
|
||||
// and `wait-ended` (`mesh.seat.node-engine.event.<event>.<node>.<module>`, the node the one that says it): a wait for a kept
|
||||
// build (`step: kept-build`, naming the build node) standing past building's default bound of 60 minutes is
|
||||
// `kept-build-waits`, cleared on `wait-ended`. A hold of the composition is no finding (design 47 H
|
||||
// §764, H-A-91 (2)).
|
||||
|
||||
// The findings of a node-engine's waits, and the bounds that stand when its event names none.
|
||||
// The finding of a node-engine's wait for a kept build, and the bound that stands when its event names none.
|
||||
const (
|
||||
KindDeliveryWaits = "delivery-waits"
|
||||
KindKeptBuildWaits = "kept-build-waits"
|
||||
ComposeBound = 30 * time.Minute
|
||||
KeptBuildBound = 60 * time.Minute
|
||||
)
|
||||
|
||||
@@ -133,17 +104,18 @@ const engineRow = "engine-wait"
|
||||
|
||||
func engineWaitID(node, module string) string { return node + "." + module }
|
||||
|
||||
// EngineWaitHeld hears the node-engine of node saying `wait-held` for module.
|
||||
// EngineWaitHeld hears the node-engine of node saying `wait-held` for module: a wait for a kept build is timed, any other
|
||||
// left.
|
||||
func (w *Watchman) EngineWaitHeld(node, module string, body []byte) {
|
||||
var a nodeengine.Wait
|
||||
if err := json.Unmarshal(body, &a); err != nil || a.Since.IsZero() {
|
||||
w.say("a wait-held of the node-engine of %s for %s does not read, left: %v", node, module, err)
|
||||
return
|
||||
}
|
||||
kind, bound := KindDeliveryWaits, ComposeBound
|
||||
if a.Step == nodeengine.WaitStepKeptBuild {
|
||||
kind, bound = KindKeptBuildWaits, KeptBuildBound
|
||||
if a.Step != nodeengine.WaitStepKeptBuild {
|
||||
return
|
||||
}
|
||||
bound := KeptBuildBound
|
||||
if d, err := time.ParseDuration(a.Bound); err == nil && d > 0 {
|
||||
bound = d
|
||||
}
|
||||
@@ -152,11 +124,8 @@ func (w *Watchman) EngineWaitHeld(node, module string, body []byte) {
|
||||
from = *a.ClockFrom
|
||||
}
|
||||
id := engineWaitID(node, module)
|
||||
what := "holds the composition of " + module
|
||||
if a.Step == nodeengine.WaitStepKeptBuild {
|
||||
what = "waits for the kept build of " + module + " on " + orWords(a.Node, "no node")
|
||||
}
|
||||
f := Finding{Scope: ScopeDelivery, ID: id, Kind: kind, Severity: Warning, Machine: node, Source: engineRow, Since: a.Since,
|
||||
what := "waits for the kept build of " + module + " on " + orWords(a.Node, "no node")
|
||||
f := Finding{Scope: ScopeDelivery, ID: id, Kind: KindKeptBuildWaits, Severity: Warning, Machine: node, Source: engineRow, Since: a.Since,
|
||||
Summary: fmt.Sprintf("the node-engine of %s %s: %s, since %s (bound %s)", node, what, orWords(a.Reason, "no reason said"), stamp(a.Since), bound),
|
||||
Said: "held for " + bound.String(),
|
||||
Headline: "A module's delivery is held on a machine",
|
||||
@@ -174,38 +143,11 @@ func (w *Watchman) EngineWaitHeld(node, module string, body []byte) {
|
||||
w.find(f)
|
||||
}
|
||||
|
||||
// EngineWaitEnded hears the node-engine of node saying `wait-ended` for module: the hold is over, whichever kind it was.
|
||||
// EngineWaitEnded hears the node-engine of node saying `wait-ended` for module: the wait is over.
|
||||
func (w *Watchman) EngineWaitEnded(node, module string) {
|
||||
id := engineWaitID(node, module)
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
w.unsignal(engineRow, id)
|
||||
w.unfind(FindingKey(ScopeDelivery, id, KindDeliveryWaits))
|
||||
w.unfind(FindingKey(ScopeDelivery, id, KindKeptBuildWaits))
|
||||
}
|
||||
|
||||
// ModuleWaitHeld hears a module's `wait-held` on its own stream (`mesh.stream.<module>.event.wait-held.<module>.<fingerprint>`), the form
|
||||
// the builder says a key queued behind a package in, with the package as its `part` (design 47 G A-192, ADR 0603 §8): it is timed as any
|
||||
// acting part's wait. A wait naming no part is a toolchain wait, which the builder times itself (§672), and is left.
|
||||
func (w *Watchman) ModuleWaitHeld(holder string, body []byte) {
|
||||
var a meshwants.Wait
|
||||
if json.Unmarshal(body, &a) == nil && a.Part == "" {
|
||||
return
|
||||
}
|
||||
a2, ok := decodeWait(w, holder, "wait-held", body)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
w.waitHeld(holder, a2)
|
||||
}
|
||||
|
||||
// ModuleWaitEnded hears the same module's `wait-ended`.
|
||||
func (w *Watchman) ModuleWaitEnded(holder string, body []byte) {
|
||||
var a meshwants.Wait
|
||||
if json.Unmarshal(body, &a) == nil && a.Part == "" {
|
||||
return
|
||||
}
|
||||
w.WaitEnded(holder, body)
|
||||
}
|
||||
|
||||
@@ -10,71 +10,9 @@ import (
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
)
|
||||
|
||||
func heldWait(module string, since time.Time, from *time.Time) []byte {
|
||||
b, _ := json.Marshal(meshwants.Wait{Step: "place", Part: "mesh-delivery-indexer", Version: "src2:new", Node: "ace", Module: module,
|
||||
Since: since, ClockFrom: from, Bound: "30m"})
|
||||
return b
|
||||
}
|
||||
|
||||
// design 47 G §266, §269 (WM6): a wait held is the finding `acting-part-late` once its bound has run from the time `wait-held`
|
||||
// carries as `clock-from` (its `since` where it carries none), not within it; `wait-ended` ends it.
|
||||
func TestAnActingPartsWaitIsLateByItsBoundFromItsClock(t *testing.T) {
|
||||
id := "mesh-wants-place-mesh-delivery-indexer-ace-web"
|
||||
key := FindingKey(ScopeDelivery, id, KindActingPartLate)
|
||||
t.Run("within the bound nothing, past it a finding, wait-ended ends it", func(t *testing.T) {
|
||||
w, o, c := setup()
|
||||
w.WaitHeld("mesh-wants", heldWait("web", c.t, nil))
|
||||
if o.ttls[signalKey(actingRow, id)] != 30*time.Minute || len(o.findings) != 0 {
|
||||
t.Fatalf("within the bound: ttl %v, findings %v", o.ttls[signalKey(actingRow, id)], o.findings)
|
||||
}
|
||||
expire(w, c, signalKey(actingRow, id), 30*time.Minute)
|
||||
if _, ok := o.findings[key]; !ok {
|
||||
t.Fatalf("past its bound: %v", o.findings)
|
||||
}
|
||||
w.WaitEnded("mesh-wants", heldWait("web", c.t, nil))
|
||||
if len(o.findings) != 0 {
|
||||
t.Fatalf("wait-ended left %v", o.findings)
|
||||
}
|
||||
})
|
||||
t.Run("the bound counts from clock-from, not from since", func(t *testing.T) {
|
||||
w, o, c := setup()
|
||||
from := c.t.Add(-20 * time.Minute)
|
||||
w.WaitHeld("mesh-wants", heldWait("web", c.t.Add(-time.Hour), &from))
|
||||
if got := o.ttls[signalKey(actingRow, id)]; got != 10*time.Minute {
|
||||
t.Fatalf("a wait whose clock began 20 minutes ago has %v left, want 10m", got)
|
||||
}
|
||||
})
|
||||
t.Run("a wait whose bound ran before it was heard is a finding at once", func(t *testing.T) {
|
||||
w, o, c := setup()
|
||||
w.WaitHeld("mesh-wants", heldWait("web", c.t.Add(-2*time.Hour), nil))
|
||||
if _, ok := o.findings[key]; !ok {
|
||||
t.Fatalf("a late wait: %v", o.findings)
|
||||
}
|
||||
})
|
||||
// The grants part's wait is held as a seat's event too (design 47 G §269, G-20 A-12): its body names no node, and its release
|
||||
// is said `wait-ended` with how `released`, which clears the finding.
|
||||
t.Run("the seat mesh-grants' granting wait is late by its bound and ended by its release", func(t *testing.T) {
|
||||
w, o, c := setup()
|
||||
body := func(how string) []byte {
|
||||
b, _ := json.Marshal(map[string]any{"step": "grant", "part": "mesh-grants", "version": "fp", "module": "web", "bound": "10m0s",
|
||||
"since": c.t, "clock-from": c.t, "how": how})
|
||||
return b
|
||||
}
|
||||
gid := "mesh-grants-grant-mesh-grants--web"
|
||||
w.WaitHeld("mesh-grants", body(""))
|
||||
expire(w, c, signalKey(actingRow, gid), 10*time.Minute)
|
||||
if _, ok := o.findings[FindingKey(ScopeDelivery, gid, KindActingPartLate)]; !ok {
|
||||
t.Fatalf("past its bound: %v", o.findings)
|
||||
}
|
||||
w.WaitEnded("mesh-grants", body("released"))
|
||||
if len(o.findings) != 0 {
|
||||
t.Fatalf("the release left %v", o.findings)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// design 47 G §498 (WM14): a put's wait (`step: put`, no part) raises `put-stands` once its bound has run, and a pipeline's wait
|
||||
// raises nothing for it; `wait-ended` ends it.
|
||||
// design 47 G §498 (WM14): a put's wait (`step: put`, no part) raises `put-stands` once its bound has run, not within it;
|
||||
// `wait-ended` ends it. Design 47 H §764, H-A-70: an acting part's wait is no finding (acting-part-late retired), so it
|
||||
// writes no pulse and no finding, past its bound or not.
|
||||
func TestAPutWithoutHoldThatStandsIsPutStands(t *testing.T) {
|
||||
put := func(since time.Time) []byte {
|
||||
b, _ := json.Marshal(meshwants.Wait{Step: meshwants.WaitStepPut, Node: "ace", Module: "web", Fingerprint: "fp-1", Since: since, Bound: "30m"})
|
||||
@@ -82,6 +20,12 @@ func TestAPutWithoutHoldThatStandsIsPutStands(t *testing.T) {
|
||||
}
|
||||
id := "mesh-wants-put--ace-web"
|
||||
w, o, c := setup()
|
||||
acting, _ := json.Marshal(meshwants.Wait{Step: "place", Part: "mesh-delivery-indexer", Version: "src2:new", Node: "ace", Module: "web",
|
||||
Since: c.t.Add(-2 * time.Hour), Bound: "30m"})
|
||||
w.WaitHeld("mesh-wants", acting)
|
||||
if len(o.findings) != 0 || len(o.pulses) != 0 {
|
||||
t.Fatalf("an acting part's wait is timed: findings %v, pulses %v", o.findings, o.pulses)
|
||||
}
|
||||
w.WaitHeld("mesh-wants", put(c.t))
|
||||
if len(o.findings) != 0 {
|
||||
t.Fatalf("within the bound: %v", o.findings)
|
||||
@@ -96,65 +40,31 @@ func TestAPutWithoutHoldThatStandsIsPutStands(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// design 47 G §499, §541 (WM4, WM5): a node-engine's hold of the composition standing 30 minutes is `delivery-waits` naming the node,
|
||||
// the module and the reason, a wait for a kept build standing 60 minutes is `kept-build-waits`, and `wait-ended` clears either.
|
||||
// design 47 G §499, §541 (WM5): a node-engine's wait for a kept build standing 60 minutes is `kept-build-waits` naming the node,
|
||||
// the module and the reason, and `wait-ended` clears it. Design 47 H §764, H-A-91 (2): a hold of the composition is no finding
|
||||
// (delivery-waits retired).
|
||||
func TestANodeEnginesWaitIsAFindingPastItsBound(t *testing.T) {
|
||||
held := func(step, reason string, since time.Time) []byte {
|
||||
b, _ := json.Marshal(nodeengine.Wait{Step: step, Reason: reason, Node: "build1", Since: since, Owner: "node-engine"})
|
||||
return b
|
||||
}
|
||||
for _, tc := range []struct {
|
||||
name, step, reason, kind string
|
||||
bound time.Duration
|
||||
}{
|
||||
{"a hold of the composition", nodeengine.WaitStepCompose, "restore point missing", KindDeliveryWaits, 30 * time.Minute},
|
||||
{"a wait for a kept build", nodeengine.WaitStepKeptBuild, "node away", KindKeptBuildWaits, 60 * time.Minute},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
w, o, c := setup()
|
||||
id := "ace.web"
|
||||
w.EngineWaitHeld("ace", "web", held(tc.step, tc.reason, c.t))
|
||||
if o.ttls[signalKey(engineRow, id)] != tc.bound || len(o.findings) != 0 {
|
||||
t.Fatalf("within the bound: ttl %v, findings %v", o.ttls[signalKey(engineRow, id)], o.findings)
|
||||
}
|
||||
expire(w, c, signalKey(engineRow, id), tc.bound)
|
||||
f, ok := o.findings[FindingKey(ScopeDelivery, id, tc.kind)]
|
||||
if !ok || len(o.findings) != 1 || !strings.Contains(f.Summary, tc.reason) {
|
||||
t.Fatalf("past its bound: %v", o.findings)
|
||||
}
|
||||
w.EngineWaitEnded("ace", "web")
|
||||
if len(o.findings) != 0 {
|
||||
t.Fatalf("wait-ended left %v", o.findings)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// design 47 G A-192 (WM13): a key queued behind a package is a wait the builder says on its own stream with the package as its part;
|
||||
// past its bound it is `acting-part-late`, `wait-ended` clears it, and a toolchain wait (no part) is left to the builder.
|
||||
func TestAKeyQueuedBehindAPackageIsAnActingPartsWait(t *testing.T) {
|
||||
body := func(part string, since time.Time) []byte {
|
||||
b, _ := json.Marshal(meshwants.Wait{Step: "building", Part: part, On: "package:" + part, Module: "web", Fingerprint: "src2:f", Since: since, Bound: "1h0m0s"})
|
||||
return b
|
||||
}
|
||||
const holder = "builder"
|
||||
w, o, c := setup()
|
||||
id := waitID(holder, meshwants.Wait{Step: "building", Part: "mesh-sdk@v0.1.14", Module: "web"})
|
||||
w.ModuleWaitHeld(holder, body("mesh-sdk@v0.1.14", c.t))
|
||||
if o.ttls[signalKey(actingRow, id)] != time.Hour || len(o.findings) != 0 {
|
||||
t.Fatalf("within the bound: ttl %v, findings %v", o.ttls[signalKey(actingRow, id)], o.findings)
|
||||
id := "ace.web"
|
||||
w.EngineWaitHeld("ace", "web", held(nodeengine.WaitStepCompose, "restore point missing", c.t.Add(-time.Hour)))
|
||||
if len(o.ttls) != 0 || len(o.findings) != 0 {
|
||||
t.Fatalf("a hold of the composition is timed: ttls %v, findings %v", o.ttls, o.findings)
|
||||
}
|
||||
expire(w, c, signalKey(actingRow, id), time.Hour)
|
||||
if _, ok := o.findings[FindingKey(ScopeDelivery, id, KindActingPartLate)]; !ok || len(o.findings) != 1 {
|
||||
w.EngineWaitHeld("ace", "web", held(nodeengine.WaitStepKeptBuild, "node away", c.t))
|
||||
if o.ttls[signalKey(engineRow, id)] != KeptBuildBound || len(o.findings) != 0 {
|
||||
t.Fatalf("within the bound: ttl %v, findings %v", o.ttls[signalKey(engineRow, id)], o.findings)
|
||||
}
|
||||
expire(w, c, signalKey(engineRow, id), KeptBuildBound)
|
||||
f, ok := o.findings[FindingKey(ScopeDelivery, id, KindKeptBuildWaits)]
|
||||
if !ok || len(o.findings) != 1 || !strings.Contains(f.Summary, "node away") {
|
||||
t.Fatalf("past its bound: %v", o.findings)
|
||||
}
|
||||
w.ModuleWaitEnded(holder, body("mesh-sdk@v0.1.14", c.t))
|
||||
w.EngineWaitEnded("ace", "web")
|
||||
if len(o.findings) != 0 {
|
||||
t.Fatalf("wait-ended left %v", o.findings)
|
||||
}
|
||||
w, o, c = setup()
|
||||
w.ModuleWaitHeld(holder, body("", c.t.Add(-2*time.Hour)))
|
||||
if len(o.findings) != 0 || len(o.ttls) != 0 {
|
||||
t.Fatalf("a toolchain wait was timed by the watchman: %v %v", o.ttls, o.findings)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -70,7 +70,6 @@ type Watchman struct {
|
||||
haves map[string]string // the fingerprint each node runs of a module, by have key, from the node-engine's running
|
||||
// starts is, by pulse key, the first start of a wait whose mark still stands (mirrorKept).
|
||||
starts map[string]startMark
|
||||
held map[string]bool // each want key held by a rule (ADR 0603 §8), mirrored as the hold's wait
|
||||
bound time.Duration
|
||||
// findings is every finding written, by key, as written; seededFindings those read at the start and not yet observed
|
||||
// again.
|
||||
@@ -111,13 +110,15 @@ type Watchman struct {
|
||||
// (WantsAt, design 47 G A-321); awaitsWants keeps the seeded entries of any `<holder>_wants` bucket until the late read.
|
||||
wantsAt func() string
|
||||
awaitsWants bool
|
||||
// heads is, by watched branch, the head still waiting for its pipeline (pipelines.go).
|
||||
heads map[string]head
|
||||
// pendingBranch is, by key, the watched branch of each open take-in wait that names one (takein.go, design 47 G §497).
|
||||
pendingBranch map[string]string
|
||||
// takenEntries are the modules of each taken-in revision, by `<branch>.<source revision>`, and recordMilestone each open record's
|
||||
// milestone, which joins a record to them (records.go, design 47 G §502).
|
||||
takenEntries map[string][]string
|
||||
// takeInBound is the setting take-in-bound (takein.go).
|
||||
takeInBound time.Duration
|
||||
// entries reads a taken-in mover's catalog entry by its fingerprint, and movers are the movers whose entry is asked, by
|
||||
// fingerprint (busstate.go).
|
||||
entries EntryReader
|
||||
movers map[string]moverTaken
|
||||
// taken is each taken-in source revision, by `<branch>.<source>`, and recordMilestone each open record's milestone, which
|
||||
// joins a record to it (records.go, design 47 G §502).
|
||||
taken map[string]bool
|
||||
recordMilestone map[string]string
|
||||
// queued is, by build seat, each step queued on it for a build node and since when (pipelines.go).
|
||||
queued map[string]map[string]time.Time
|
||||
@@ -132,12 +133,12 @@ type Watchman struct {
|
||||
// New is the watchman of module (its credential's), writing to out, reading the time from now.
|
||||
func New(module string, out Out, now func() time.Time) *Watchman {
|
||||
return &Watchman{module: module, own: BucketsOf(module), out: out, now: now, counts: map[string]int{}, presence: map[string]presence{},
|
||||
pulses: map[string]*pulse{}, stuck: map[string]meshwatchdogsclient.Stuck{}, wants: map[string]string{}, haves: map[string]string{}, held: map[string]bool{}, starts: map[string]startMark{}, bound: meshwatchdogsclient.PulseMoveBound, stepBound: StepBound, progressBound: ProgressBound,
|
||||
findings: map[string][]byte{}, seededFindings: map[string]bool{}, signals: map[string]signalDue{}, busCreated: map[string]bool{}, takeInHeld: map[string]Finding{}, pendingBranch: map[string]string{}, takenEntries: map[string][]string{}, recordMilestone: map[string]string{}, away: map[string]bool{},
|
||||
pulses: map[string]*pulse{}, stuck: map[string]meshwatchdogsclient.Stuck{}, wants: map[string]string{}, haves: map[string]string{}, starts: map[string]startMark{}, bound: meshwatchdogsclient.PulseMoveBound, stepBound: StepBound, progressBound: ProgressBound,
|
||||
findings: map[string][]byte{}, seededFindings: map[string]bool{}, signals: map[string]signalDue{}, busCreated: map[string]bool{}, takeInHeld: map[string]Finding{}, takeInBound: DefaultTakeInBound, movers: map[string]moverTaken{}, taken: map[string]bool{}, recordMilestone: map[string]string{}, away: map[string]bool{},
|
||||
advised: map[string]int{}, advisedSince: map[string]time.Time{},
|
||||
provFailed: map[string]map[string]ProvReport{}, closures: map[string]map[string]time.Time{}, waiting: map[string]map[string]string{},
|
||||
|
||||
standing: map[string]map[string]bool{}, readWhole: map[string]bool{}, awaited: map[string]bool{}, heads: map[string]head{}, queued: map[string]map[string]time.Time{}, steps: map[string]*openStep{}, awaySaid: map[string]time.Time{}, opened: map[string]Finding{}}
|
||||
standing: map[string]map[string]bool{}, readWhole: map[string]bool{}, awaited: map[string]bool{}, queued: map[string]map[string]time.Time{}, steps: map[string]*openStep{}, awaySaid: map[string]time.Time{}, opened: map[string]Finding{}}
|
||||
}
|
||||
|
||||
// Logf sets where a write that failed is said; it is silent until set.
|
||||
|
||||
@@ -605,9 +605,9 @@ func TestARunningStepPastItsBoundIsStuck(t *testing.T) { // 1908
|
||||
// its fingerprint.
|
||||
func wantValue(t *testing.T, fp string, removed bool) []byte {
|
||||
t.Helper()
|
||||
w := meshwantsclient.Want{Fp: fp, Composed: "sha256:" + strings.Repeat("c", 64), Removed: removed}
|
||||
w := meshwantsclient.Want{Fp: fp, Version: "0.1.4", Pipeline: "main.42", Rank: 1, Removed: removed} // the H1 want (H-A-37)
|
||||
if removed {
|
||||
w.Composed = ""
|
||||
w = meshwantsclient.Want{Fp: fp, Removed: true}
|
||||
}
|
||||
b, err := meshwantsclient.EncodeWant(w)
|
||||
if err != nil {
|
||||
@@ -720,11 +720,11 @@ func TestAStuckKeyWhoseKeyIsGoneIsClosedAtTheStart(t *testing.T) {
|
||||
// A pulse of another key of the bucket, and a finding the bucket is the source of, stood at the start.
|
||||
pk := pulseKeyOf(t, row.bucket, "want.n9.gone")
|
||||
w.SeedPulse(pk)
|
||||
fv, err := EncodeFinding(Finding{Scope: ScopeModule, ID: "gone", Kind: KindManifestRefused, Severity: Warning, Source: row.bucket, Summary: "refused", Since: t0})
|
||||
fv, err := EncodeFinding(Finding{Scope: ScopeModule, ID: "gone", Kind: KindStuck, Severity: Warning, Source: row.bucket, Summary: "refused", Since: t0})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fk := FindingKey(ScopeModule, "gone", KindManifestRefused)
|
||||
fk := FindingKey(ScopeModule, "gone", KindStuck)
|
||||
o.findings[fk], _ = DecodeFinding(fv)
|
||||
w.SeedFinding(fk, fv)
|
||||
w.AwaitsWants()
|
||||
|
||||
@@ -64,6 +64,12 @@ type manifest struct {
|
||||
MaxAgeDays int `json:"max-age-days"`
|
||||
MaxMiB int `json:"max-mib"`
|
||||
} `json:"streams"`
|
||||
Config map[string]struct {
|
||||
Type string `json:"type"`
|
||||
Description string `json:"description"`
|
||||
Trusted bool `json:"trusted"`
|
||||
Default string `json:"default"`
|
||||
} `json:"config"`
|
||||
OwnSecrets map[string]string `json:"own-secrets"`
|
||||
SecretsOwner string `json:"secrets-owner"`
|
||||
Resources []map[string]any `json:"resources"`
|
||||
@@ -127,12 +133,17 @@ func TestManifestStateAndReads(t *testing.T) { // 1135/AC6
|
||||
}
|
||||
// The join: its first view asked again by hand is a hand act, recorded through the seat (views.StatusTools). The waits the
|
||||
// indexer holds at the watchman's start are asked of its tool (ADR 0633 §5), a verb edge as the view's.
|
||||
if want := []string{"mesh-delivery-indexer.indexer_pending", "mesh-delivery-indexer.indexer_verdicts", "seat:mesh-hand-acts.hand-act", "seat:mesh-conditions.conditions", "seat:mesh-broker.show", "seat:mesh-catalog.revision", "seat:mesh-catalog.registrations", "seat:mesh-delivery.source", "seat:mesh-pipeline-runner.stands"}; !reflect.DeepEqual(m.Invokes, want) {
|
||||
// Design 47 H-A-79, H-A-94 (1): no indexer_verdicts; a mover's entry is read through the seat mesh-catalog's verb entry.
|
||||
if want := []string{"mesh-delivery-indexer.indexer_pending", "seat:mesh-hand-acts.hand-act", "seat:mesh-conditions.conditions", "seat:mesh-broker.show", "seat:mesh-catalog.revision", "seat:mesh-catalog.registrations", "seat:mesh-catalog.entry", "seat:mesh-delivery.source", "seat:mesh-pipeline-runner.stands"}; !reflect.DeepEqual(m.Invokes, want) {
|
||||
t.Errorf("invokes = %v, want %v", m.Invokes, want)
|
||||
}
|
||||
// DLV-M5 its first views, 1585/T20 the indexer's take-in events, and DLV-W1 (ADR 0595 §4): the accepted verdicts and the bus module's created and refused, durably.
|
||||
if want := []string{"seat:mesh-wants.views.wants", "seat:mesh-module-assigner.views.seat-holders", "mesh-catalog.registration-changed", "mesh-catalog.catalog-revision", "mesh-delivery-indexer.take-in.pending", "mesh-delivery-indexer.take-in.taken", "mesh-catalog.manifest-accepted", "mesh-catalog.manifest-refused", "mesh-delivery.source-moved", "mesh-delivery.filling", "mesh-delivery.full", "mesh-delivery.room", "mesh-delivery.record-opened", "mesh-delivery.record-ended", "mesh-catalog.filling", "mesh-catalog.full", "mesh-catalog.room", "mesh-wants.filling", "mesh-wants.full", "mesh-wants.room", "mesh-grants.filling", "mesh-grants.full", "mesh-grants.room", "node-build-go.ask-taken", "node-build-go.step-progress", "node-build-go.step-ended", "node-build-image.ask-taken", "node-build-image.step-progress", "node-build-image.step-ended", "node-build-node.ask-taken", "node-build-node.step-progress", "node-build-node.step-ended", "mesh-pipeline-runner.pipeline-started", "mesh-pipeline-runner.step-started", "mesh-pipeline-runner.step-ended", "mesh-wants.step-progress", "mesh-pipeline-runner.wait-held", "mesh-pipeline-runner.wait-ended", "mesh-wants.wait-held", "mesh-wants.wait-ended", "mesh-grants.wait-held", "mesh-grants.wait-ended", "mesh-delivery-builder.wait-held", "mesh-delivery-builder.wait-ended", "node-engine.wait-held", "node-engine.wait-ended", "node-engine.running", "mesh-broker.created", "mesh-broker.refused", "mesh-broker.recreated", "mesh-broker.dropped", "mesh-conditions.findings-asked"}; !reflect.DeepEqual(m.Consumes, want) {
|
||||
t.Errorf("consumes = %v, want %v", m.Consumes, want)
|
||||
if want := []string{"seat:mesh-wants.views.wants", "seat:mesh-module-assigner.views.seat-holders", "mesh-catalog.registration-changed", "mesh-catalog.catalog-revision", "mesh-delivery-indexer.take-in.pending", "mesh-delivery-indexer.take-in.taken", "mesh-delivery.source-moved", "mesh-delivery.filling", "mesh-delivery.full", "mesh-delivery.room", "mesh-delivery.record-opened", "mesh-delivery.record-ended", "mesh-catalog.filling", "mesh-catalog.full", "mesh-catalog.room", "mesh-wants.filling", "mesh-wants.full", "mesh-wants.room", "mesh-grants.filling", "mesh-grants.full", "mesh-grants.room", "node-build-go.ask-taken", "node-build-go.step-progress", "node-build-go.step-ended", "node-build-image.ask-taken", "node-build-image.step-progress", "node-build-image.step-ended", "node-build-node.ask-taken", "node-build-node.step-progress", "node-build-node.step-ended", "mesh-pipeline-runner.step-started", "mesh-pipeline-runner.step-ended", "mesh-wants.step-progress", "mesh-wants.wait-held", "mesh-wants.wait-ended", "node-engine.wait-held", "node-engine.wait-ended", "node-engine.running", "mesh-broker.created", "mesh-broker.refused", "mesh-broker.recreated", "mesh-broker.dropped", "mesh-conditions.findings-asked"}; !reflect.DeepEqual(m.Consumes, want) {
|
||||
t.Errorf("consumes = %v, want %v", m.Consumes, want) // design 47 H §741, §764, H-A-49, H-A-91 (2)
|
||||
}
|
||||
// Design 47 H §744, H-A-94 (2): the setting take-in-bound, 60 minutes by default, reaches the process as a file.
|
||||
if c := m.Config["take-in-bound"]; c.Type != "string" || c.Default != "60m" {
|
||||
t.Errorf("config take-in-bound = %+v, want a string defaulting to 60m", c)
|
||||
}
|
||||
// 1952/WM1: the watchman holds the seat mesh-watchdogs at mesh scope, its first seat (novox/hq ADR 0564 §1, §4).
|
||||
if !reflect.DeepEqual(m.Claims, []claim{{Name: "mesh-watchdogs", Scope: "mesh", Serves: []string{"signals", "stuck", "presence", "dead-letters", "resync"}}}) {
|
||||
@@ -228,6 +239,19 @@ func TestManifestResources(t *testing.T) { // 1135/AC6
|
||||
if env["MESH_BROKER_FILE"] != "${dir:state}/broker" {
|
||||
t.Errorf("the process reads %v", env["MESH_BROKER_FILE"])
|
||||
}
|
||||
// Design 47 H §744: the setting take-in-bound's file, which the process reads at its start.
|
||||
if env["MESH_WATCHMAN_TAKE_IN_BOUND_FILE"] != "${dir:state}/take-in-bound" {
|
||||
t.Errorf("the process reads the take-in bound from %v", env["MESH_WATCHMAN_TAKE_IN_BOUND_FILE"])
|
||||
}
|
||||
var boundFile map[string]any
|
||||
for _, f := range byType["file"] {
|
||||
if f["id"] == "take-in-bound" {
|
||||
boundFile = f
|
||||
}
|
||||
}
|
||||
if boundFile["path"] != "${dir:state}/take-in-bound" || boundFile["content"] != "${config:take-in-bound}" || boundFile["owner"] != user {
|
||||
t.Errorf("the take-in bound's file = %v", boundFile)
|
||||
}
|
||||
if h, _ := p["health"].(map[string]any); h["kind"] != "unit" {
|
||||
t.Errorf("health = %v", p["health"])
|
||||
}
|
||||
|
||||
@@ -20,12 +20,12 @@
|
||||
],
|
||||
"invokes": [
|
||||
"mesh-delivery-indexer.indexer_pending",
|
||||
"mesh-delivery-indexer.indexer_verdicts",
|
||||
"seat:mesh-hand-acts.hand-act",
|
||||
"seat:mesh-conditions.conditions",
|
||||
"seat:mesh-broker.show",
|
||||
"seat:mesh-catalog.revision",
|
||||
"seat:mesh-catalog.registrations",
|
||||
"seat:mesh-catalog.entry",
|
||||
"seat:mesh-delivery.source",
|
||||
"seat:mesh-pipeline-runner.stands"
|
||||
],
|
||||
@@ -36,8 +36,6 @@
|
||||
"mesh-catalog.catalog-revision",
|
||||
"mesh-delivery-indexer.take-in.pending",
|
||||
"mesh-delivery-indexer.take-in.taken",
|
||||
"mesh-catalog.manifest-accepted",
|
||||
"mesh-catalog.manifest-refused",
|
||||
"mesh-delivery.source-moved",
|
||||
"mesh-delivery.filling",
|
||||
"mesh-delivery.full",
|
||||
@@ -62,18 +60,11 @@
|
||||
"node-build-node.ask-taken",
|
||||
"node-build-node.step-progress",
|
||||
"node-build-node.step-ended",
|
||||
"mesh-pipeline-runner.pipeline-started",
|
||||
"mesh-pipeline-runner.step-started",
|
||||
"mesh-pipeline-runner.step-ended",
|
||||
"mesh-wants.step-progress",
|
||||
"mesh-pipeline-runner.wait-held",
|
||||
"mesh-pipeline-runner.wait-ended",
|
||||
"mesh-wants.wait-held",
|
||||
"mesh-wants.wait-ended",
|
||||
"mesh-grants.wait-held",
|
||||
"mesh-grants.wait-ended",
|
||||
"mesh-delivery-builder.wait-held",
|
||||
"mesh-delivery-builder.wait-ended",
|
||||
"node-engine.wait-held",
|
||||
"node-engine.wait-ended",
|
||||
"node-engine.running",
|
||||
@@ -111,6 +102,14 @@
|
||||
"broker": "${dir:state}/broker"
|
||||
},
|
||||
"secrets-owner": "mesh-delivery-watchman",
|
||||
"config": {
|
||||
"take-in-bound": {
|
||||
"type": "string",
|
||||
"description": "how long a source revision's take-in the indexer holds may wait for its take-in.taken before it is the finding take-in-waits, a Go duration such as 60m, counted from its take-in.pending (design 47 H section 744).",
|
||||
"trusted": false,
|
||||
"default": "60m"
|
||||
}
|
||||
},
|
||||
"resources": [
|
||||
{
|
||||
"id": "account",
|
||||
@@ -127,6 +126,14 @@
|
||||
"place": ".",
|
||||
"unknown": "program"
|
||||
},
|
||||
{
|
||||
"id": "take-in-bound",
|
||||
"type": "file",
|
||||
"path": "${dir:state}/take-in-bound",
|
||||
"owner": "mesh-delivery-watchman",
|
||||
"mode": "0600",
|
||||
"content": "${config:take-in-bound}"
|
||||
},
|
||||
{
|
||||
"id": "socket-dir",
|
||||
"type": "directory",
|
||||
@@ -144,7 +151,8 @@
|
||||
"./mesh-delivery-watchman"
|
||||
],
|
||||
"env": {
|
||||
"MESH_BROKER_FILE": "${dir:state}/broker"
|
||||
"MESH_BROKER_FILE": "${dir:state}/broker",
|
||||
"MESH_WATCHMAN_TAKE_IN_BOUND_FILE": "${dir:state}/take-in-bound"
|
||||
},
|
||||
"user": "mesh-delivery-watchman",
|
||||
"health": {
|
||||
|
||||
Reference in New Issue
Block a user