Files
mesh-delivery/modules/mesh-delivery-watchman/internal/watch/pipelines_test.go
T
jschoubben 161d6e6451 WM16: a record is covered only by what is late for the entries of its own revision
record-opened is joined to the take-in.taken of its milestone, <branch>.<source revision>, whose entries name the modules; the
record-open finding is left unwritten only while one of those modules is held by an acting-part-late wait (or a store says full),
no longer while any wait is late. A record whose revision is not known taken is covered by a standing take-in-waits.

Builds: design 47 G §502, A-200

Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h
2026-11-12 05:56:33 +01:00

286 lines
12 KiB
Go

package watch
import (
"encoding/json"
"testing"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
)
func moved(branch string, rev int64, group string, at time.Time) []byte {
b, _ := json.Marshal(meshdelivery.SourceMovedEvent{Branch: branch, Revision: rev, Group: group, At: &at})
return b
}
// takeInBodyOn is a take-in event's body that names its watched branch (design 47 G §493).
func takeInBodyOn(t *testing.T, kind, key, branch string, since time.Time) []byte {
t.Helper()
var m map[string]any
if err := json.Unmarshal(takeInBody(t, kind, key, since), &m); err != nil {
t.Fatal(err)
}
m["branch"] = branch
b, err := json.Marshal(m)
if err != nil {
t.Fatal(err)
}
return b
}
func started(branch string, rev int64, run string) []byte {
b, _ := json.Marshal(meshpipelinerunner.PipelineStartedEvent{Branch: branch, Source: rev, Run: run, Module: "m", Pipeline: "p"})
return b
}
// design 47 G §12, §79 (test 76): a head with no pipeline within five minutes is `pipeline-stands`, cleared by the
// runner's `pipeline-started` for that head; a gathered group's member and a run on request do not.
func TestAHeadWithNoPipelineStands(t *testing.T) {
key := FindingKey(ScopeDelivery, headID("main"), KindPipelineStands)
cases := []struct {
name string
run func(w *Watchman, o *fakeOut, c *clock)
stand bool
}{
{"five minutes and no pipeline", func(w *Watchman, o *fakeOut, c *clock) {
w.SourceMoved(moved("main", 7, "", c.t))
if o.ttls[signalKey(headRow, headID("main"))] != HeadBound || len(o.findings) != 0 {
t.Fatalf("within the bound: %v %v", o.ttls, o.findings)
}
expire(w, c, signalKey(headRow, headID("main")), HeadBound)
}, true},
{"its pipeline-started clears it", func(w *Watchman, o *fakeOut, c *clock) {
w.SourceMoved(moved("main", 7, "", c.t))
expire(w, c, signalKey(headRow, headID("main")), HeadBound)
w.PipelineStarted(started("main", 7, ""))
}, false},
{"a pipeline before the bound raises none", func(w *Watchman, o *fakeOut, c *clock) {
w.SourceMoved(moved("main", 7, "", c.t))
w.PipelineStarted(started("main", 8, ""))
if _, ok := o.pulses[signalKey(headRow, headID("main"))]; ok {
t.Fatal("the pulse stands")
}
}, false},
{"a pipeline of an older head does not clear a newer head", func(w *Watchman, o *fakeOut, c *clock) {
w.SourceMoved(moved("main", 7, "", c.t))
w.PipelineStarted(started("main", 6, ""))
expire(w, c, signalKey(headRow, headID("main")), HeadBound)
}, true},
{"a gathered group's member is not timed", func(w *Watchman, o *fakeOut, c *clock) {
w.SourceMoved(moved("main", 7, "grp", c.t))
if len(o.pulses) != 0 {
t.Fatalf("a pulse for a member: %v", o.pulses)
}
}, false},
{"a head behind a pending revision of its branch is not timed", func(w *Watchman, o *fakeOut, c *clock) {
w.TakeIn(takeInEvent(t, takein.TakeInPending, "m.1"), takeInBodyOn(t, takein.TakeInPending, "m.1", "main", c.t))
w.SourceMoved(moved("main", 7, "", c.t))
if _, ok := o.pulses[signalKey(headRow, headID("main"))]; ok {
t.Fatalf("a pulse for a head behind a pending revision: %v", o.pulses)
}
}, false},
{"a pending revision of another branch does not hold it", func(w *Watchman, o *fakeOut, c *clock) {
w.TakeIn(takeInEvent(t, takein.TakeInPending, "m.1"), takeInBodyOn(t, takein.TakeInPending, "m.1", "other", c.t))
w.SourceMoved(moved("main", 7, "", c.t))
expire(w, c, signalKey(headRow, headID("main")), HeadBound)
}, true},
{"a head whose pending revision is taken in is timed again", func(w *Watchman, o *fakeOut, c *clock) {
w.SourceMoved(moved("main", 7, "", c.t))
w.TakeIn(takeInEvent(t, takein.TakeInPending, "m.1"), takeInBodyOn(t, takein.TakeInPending, "m.1", "main", c.t))
if _, ok := o.pulses[signalKey(headRow, headID("main"))]; ok {
t.Fatalf("the head is timed behind a pending revision: %v", o.pulses)
}
c.t = c.t.Add(HeadBound)
w.TakeIn(takeInEvent(t, takein.TakeInTaken, "m.1"), takeInBodyOn(t, takein.TakeInTaken, "m.1", "main", c.t))
if _, ok := o.findings[key]; !ok {
t.Fatalf("the head, past its five minutes when taken in, stands: %v", o.findings)
}
}, true},
{"a run on request is no head's pipeline", func(w *Watchman, o *fakeOut, c *clock) {
w.SourceMoved(moved("main", 7, "", c.t))
w.PipelineStarted(started("main", 7, "run-1"))
expire(w, c, signalKey(headRow, headID("main")), HeadBound)
}, true},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
w, o, c := setup()
tc.run(w, o, c)
if _, ok := o.findings[key]; ok != tc.stand {
t.Fatalf("the finding stands %v, want %v: %v", ok, tc.stand, o.findings)
}
})
}
}
func queuedStep(pipeline, step, seat, outcome string, started time.Time) []byte {
b, _ := json.Marshal(meshpipelinerunner.StepEvent{Pipeline: pipeline, Branch: "main", Module: "m", Commit: "c",
Result: meshpipelinerunner.StepResult{Step: step, Seat: seat, Outcome: outcome, Started: &started}})
return b
}
// design 47 G §86: a build seat whose oldest queued step has waited 60 minutes raises one `build-capacity-waits`, cleared
// when that step starts.
func TestABuildSeatsOldestQueuedStepWaitsAnHour(t *testing.T) {
const seat = "node-build-go"
key := FindingKey(ScopeDelivery, seat, KindBuildCapacityWaits)
wait := meshpipelinerunner.StepResultOutcomeWaitingForABuildNode
w, o, c := setup()
w.StepEvent(queuedStep("p1", "unit", seat, wait, c.t))
c.t = c.t.Add(10 * time.Minute)
w.StepEvent(queuedStep("p2", "unit", seat, wait, c.t))
pk := signalKey(capacityRow, seat)
if o.ttls[pk] != 50*time.Minute || len(o.findings) != 0 {
t.Fatalf("within the bound: %v %v", o.ttls, o.findings)
}
expire(w, c, pk, 50*time.Minute)
if _, ok := o.findings[key]; !ok || len(o.findings) != 1 {
t.Fatalf("an hour for the oldest: %v", o.findings)
}
w.StepEvent(queuedStep("p1", "unit", seat, meshpipelinerunner.StepResultOutcomeRunning, c.t))
if len(o.findings) != 0 {
t.Fatalf("the oldest started, the next has waited 50 minutes: %v", o.findings)
}
if o.ttls[pk] != 10*time.Minute {
t.Fatalf("the next one's hour: %v", o.ttls[pk])
}
w.StepEvent(queuedStep("p2", "unit", seat, meshpipelinerunner.StepResultOutcomePassed, c.t))
if _, ok := o.pulses[pk]; ok || len(o.findings) != 0 {
t.Fatalf("the queue is empty: %v %v", o.pulses, o.findings)
}
}
// design 47 G §389, §436 (test 106's cause): a machine whose presence key expires says `machine-away`, once, and one
// heard again says `machine-back`; a machine heard again that was never away says nothing.
func TestAMachinesPresenceIsSaidAsEvents(t *testing.T) {
w, o, c := setup()
w.Connect("node.ace")
w.Connect("node.ace") // heard again while never away
w.Disconnect("node.ace")
w.Disconnect("node.ace")
if len(o.said) != 0 {
t.Fatalf("said before the expiry: %v", o.said)
}
c.t = t0.Add(120 * time.Second)
w.Pulse(pulseKeyOf(t, presenceNS, "ace"), Expired, c.t)
w.Pulse(pulseKeyOf(t, presenceNS, "ace"), Expired, c.t) // the same expiry heard twice
w.Connect("node.ace")
w.Connect("node.ace")
got := []string{}
for _, s := range o.said {
got = append(got, s.subject)
}
want := []string{"mesh.seat.mesh-watchdogs.event.machine-away.ace", "mesh.seat.mesh-watchdogs.event.machine-back.ace"}
if len(got) != 2 || got[0] != want[0] || got[1] != want[1] {
t.Fatalf("said %v, want %v", got, want)
}
}
func fillBody(store string, bytes int64) []byte {
b, _ := json.Marshal(meshdelivery.FillingEvent{Store: store, Bytes: bytes, Cap: 100})
return b
}
// design 47 G §181, §263: a store past its share is `store-filling` naming the part and the store, full is urgent, `room` and a
// fill under 80 % clear it.
func TestAStoreFillingIsAFinding(t *testing.T) {
w, o, _ := setup()
key := FindingKey(ScopeDelivery, "mesh-delivery-records", KindStoreFilling)
w.StoreFill("mesh-delivery", "filling", fillBody("records", 60))
if len(o.findings) != 0 {
t.Fatalf("under 80 %%: %v", o.findings)
}
w.StoreFill("mesh-delivery", "filling", fillBody("records", 75))
if len(o.findings) != 0 {
t.Fatalf("75 %% is under §181's 80 %%: %v", o.findings)
}
w.StoreFill("mesh-delivery", "filling", fillBody("records", 85))
if o.findings[key].Severity != Warning {
t.Fatalf("filling: %v", o.findings)
}
w.StoreFill("mesh-delivery", "full", fillBody("records", 100))
if o.findings[key].Severity != Urgent {
t.Fatalf("full: %v", o.findings)
}
w.StoreFill("mesh-delivery", "room", fillBody("records", 60))
if len(o.findings) != 0 {
t.Fatalf("room: %v", o.findings)
}
}
func recordBody(id string, at time.Time) []byte {
b, _ := json.Marshal(meshdelivery.RecordOpenedEvent{Record: id, Milestone: "47/G", At: &at})
return b
}
// design 47 G §148, §160: a record open 24 hours is `record-open`, cleared by `record-ended`; a record that ends in time
// raises none; while a store is full, which already tells why, it raises none.
func TestARecordOpenADayIsAFinding(t *testing.T) {
w, o, c := setup()
pk := signalKey(recordRow, "r1")
w.RecordOpen(recordBody("r1", c.t))
if o.ttls[pk] != RecordOpenBound || len(o.findings) != 0 {
t.Fatalf("within the bound: %v %v", o.ttls, o.findings)
}
expire(w, c, pk, RecordOpenBound)
key := FindingKey(ScopeDelivery, "r1", KindRecordOpen)
if _, ok := o.findings[key]; !ok {
t.Fatalf("past 24 hours: %v", o.findings)
}
w.StoreFill("mesh-delivery", "full", fillBody("records", 100))
if _, ok := o.findings[key]; ok {
t.Fatalf("a full store covers the record: %v", o.findings)
}
w.StoreFill("mesh-delivery", "room", fillBody("records", 10))
if _, ok := o.findings[key]; !ok {
t.Fatalf("room, and the record stands open again: %v", o.findings)
}
w.RecordEnded(recordBody("r1", c.t))
if len(o.findings) != 0 {
t.Fatalf("record-ended: %v", o.findings)
}
w.RecordOpen(recordBody("r2", c.t))
w.RecordEnded(recordBody("r2", c.t))
if _, ok := o.pulses[signalKey(recordRow, "r2")]; ok {
t.Fatal("the pulse of an ended record stands")
}
}
// design 47 G §502, A-200 (WM16): a record is tied to the entries of the take-in.taken of its milestone `<branch>.<source revision>`; it
// is covered only while one of those entries is held by an `acting-part-late` wait, so another record's late entry covers nothing.
func TestARecordIsCoveredOnlyByTheWaitsOfItsOwnEntries(t *testing.T) {
w, o, c := setup()
taken := func(key string, source int64, module string) {
var m map[string]any
if err := json.Unmarshal(takeInBody(t, takein.TakeInTaken, key, c.t), &m); err != nil {
t.Fatal(err)
}
m["branch"], m["source"] = "main", source
m["entries"] = []map[string]any{{"commit": "c", "fingerprint": "f", "module": module}}
b, _ := json.Marshal(m)
w.TakeIn(takeInEvent(t, takein.TakeInTaken, key), b)
}
taken("m.1", 7, "web")
taken("m.2", 8, "db")
rec := func(id, milestone string) []byte {
b, _ := json.Marshal(meshdelivery.RecordOpenedEvent{Record: id, Milestone: milestone, At: &c.t})
return b
}
w.RecordOpen(rec("rWeb", "main.7"))
w.RecordOpen(rec("rDb", "main.8"))
late, _ := json.Marshal(meshwants.Wait{Step: "place", Part: "mesh-delivery-indexer", Node: "ace", Module: "web", Since: c.t, Bound: "30m"})
w.WaitHeld("mesh-wants", late)
expire(w, c, signalKey(actingRow, "mesh-wants-place-mesh-delivery-indexer-ace-web"), 30*time.Minute)
expire(w, c, signalKey(recordRow, "rWeb"), RecordOpenBound)
expire(w, c, signalKey(recordRow, "rDb"), 0)
if _, ok := o.findings[FindingKey(ScopeDelivery, "rWeb", KindRecordOpen)]; ok {
t.Fatalf("a record whose entry is held by acting-part-late is covered: %v", o.findings)
}
if _, ok := o.findings[FindingKey(ScopeDelivery, "rDb", KindRecordOpen)]; !ok {
t.Fatalf("a record of other entries is open at 24 hours whatever else is late: %v", o.findings)
}
}