record-opened is joined to the take-in.taken of its milestone, <branch>.<source revision>, whose entries name the modules; the record-open finding is left unwritten only while one of those modules is held by an acting-part-late wait (or a store says full), no longer while any wait is late. A record whose revision is not known taken is covered by a standing take-in-waits. Builds: design 47 G §502, A-200 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h
286 lines
12 KiB
Go
286 lines
12 KiB
Go
package watch
|
|
|
|
import (
|
|
"encoding/json"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
|
|
)
|
|
|
|
func moved(branch string, rev int64, group string, at time.Time) []byte {
|
|
b, _ := json.Marshal(meshdelivery.SourceMovedEvent{Branch: branch, Revision: rev, Group: group, At: &at})
|
|
return b
|
|
}
|
|
|
|
// takeInBodyOn is a take-in event's body that names its watched branch (design 47 G §493).
|
|
func takeInBodyOn(t *testing.T, kind, key, branch string, since time.Time) []byte {
|
|
t.Helper()
|
|
var m map[string]any
|
|
if err := json.Unmarshal(takeInBody(t, kind, key, since), &m); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
m["branch"] = branch
|
|
b, err := json.Marshal(m)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return b
|
|
}
|
|
|
|
func started(branch string, rev int64, run string) []byte {
|
|
b, _ := json.Marshal(meshpipelinerunner.PipelineStartedEvent{Branch: branch, Source: rev, Run: run, Module: "m", Pipeline: "p"})
|
|
return b
|
|
}
|
|
|
|
// design 47 G §12, §79 (test 76): a head with no pipeline within five minutes is `pipeline-stands`, cleared by the
|
|
// runner's `pipeline-started` for that head; a gathered group's member and a run on request do not.
|
|
func TestAHeadWithNoPipelineStands(t *testing.T) {
|
|
key := FindingKey(ScopeDelivery, headID("main"), KindPipelineStands)
|
|
cases := []struct {
|
|
name string
|
|
run func(w *Watchman, o *fakeOut, c *clock)
|
|
stand bool
|
|
}{
|
|
{"five minutes and no pipeline", func(w *Watchman, o *fakeOut, c *clock) {
|
|
w.SourceMoved(moved("main", 7, "", c.t))
|
|
if o.ttls[signalKey(headRow, headID("main"))] != HeadBound || len(o.findings) != 0 {
|
|
t.Fatalf("within the bound: %v %v", o.ttls, o.findings)
|
|
}
|
|
expire(w, c, signalKey(headRow, headID("main")), HeadBound)
|
|
}, true},
|
|
{"its pipeline-started clears it", func(w *Watchman, o *fakeOut, c *clock) {
|
|
w.SourceMoved(moved("main", 7, "", c.t))
|
|
expire(w, c, signalKey(headRow, headID("main")), HeadBound)
|
|
w.PipelineStarted(started("main", 7, ""))
|
|
}, false},
|
|
{"a pipeline before the bound raises none", func(w *Watchman, o *fakeOut, c *clock) {
|
|
w.SourceMoved(moved("main", 7, "", c.t))
|
|
w.PipelineStarted(started("main", 8, ""))
|
|
if _, ok := o.pulses[signalKey(headRow, headID("main"))]; ok {
|
|
t.Fatal("the pulse stands")
|
|
}
|
|
}, false},
|
|
{"a pipeline of an older head does not clear a newer head", func(w *Watchman, o *fakeOut, c *clock) {
|
|
w.SourceMoved(moved("main", 7, "", c.t))
|
|
w.PipelineStarted(started("main", 6, ""))
|
|
expire(w, c, signalKey(headRow, headID("main")), HeadBound)
|
|
}, true},
|
|
{"a gathered group's member is not timed", func(w *Watchman, o *fakeOut, c *clock) {
|
|
w.SourceMoved(moved("main", 7, "grp", c.t))
|
|
if len(o.pulses) != 0 {
|
|
t.Fatalf("a pulse for a member: %v", o.pulses)
|
|
}
|
|
}, false},
|
|
{"a head behind a pending revision of its branch is not timed", func(w *Watchman, o *fakeOut, c *clock) {
|
|
w.TakeIn(takeInEvent(t, takein.TakeInPending, "m.1"), takeInBodyOn(t, takein.TakeInPending, "m.1", "main", c.t))
|
|
w.SourceMoved(moved("main", 7, "", c.t))
|
|
if _, ok := o.pulses[signalKey(headRow, headID("main"))]; ok {
|
|
t.Fatalf("a pulse for a head behind a pending revision: %v", o.pulses)
|
|
}
|
|
}, false},
|
|
{"a pending revision of another branch does not hold it", func(w *Watchman, o *fakeOut, c *clock) {
|
|
w.TakeIn(takeInEvent(t, takein.TakeInPending, "m.1"), takeInBodyOn(t, takein.TakeInPending, "m.1", "other", c.t))
|
|
w.SourceMoved(moved("main", 7, "", c.t))
|
|
expire(w, c, signalKey(headRow, headID("main")), HeadBound)
|
|
}, true},
|
|
{"a head whose pending revision is taken in is timed again", func(w *Watchman, o *fakeOut, c *clock) {
|
|
w.SourceMoved(moved("main", 7, "", c.t))
|
|
w.TakeIn(takeInEvent(t, takein.TakeInPending, "m.1"), takeInBodyOn(t, takein.TakeInPending, "m.1", "main", c.t))
|
|
if _, ok := o.pulses[signalKey(headRow, headID("main"))]; ok {
|
|
t.Fatalf("the head is timed behind a pending revision: %v", o.pulses)
|
|
}
|
|
c.t = c.t.Add(HeadBound)
|
|
w.TakeIn(takeInEvent(t, takein.TakeInTaken, "m.1"), takeInBodyOn(t, takein.TakeInTaken, "m.1", "main", c.t))
|
|
if _, ok := o.findings[key]; !ok {
|
|
t.Fatalf("the head, past its five minutes when taken in, stands: %v", o.findings)
|
|
}
|
|
}, true},
|
|
{"a run on request is no head's pipeline", func(w *Watchman, o *fakeOut, c *clock) {
|
|
w.SourceMoved(moved("main", 7, "", c.t))
|
|
w.PipelineStarted(started("main", 7, "run-1"))
|
|
expire(w, c, signalKey(headRow, headID("main")), HeadBound)
|
|
}, true},
|
|
}
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
w, o, c := setup()
|
|
tc.run(w, o, c)
|
|
if _, ok := o.findings[key]; ok != tc.stand {
|
|
t.Fatalf("the finding stands %v, want %v: %v", ok, tc.stand, o.findings)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func queuedStep(pipeline, step, seat, outcome string, started time.Time) []byte {
|
|
b, _ := json.Marshal(meshpipelinerunner.StepEvent{Pipeline: pipeline, Branch: "main", Module: "m", Commit: "c",
|
|
Result: meshpipelinerunner.StepResult{Step: step, Seat: seat, Outcome: outcome, Started: &started}})
|
|
return b
|
|
}
|
|
|
|
// design 47 G §86: a build seat whose oldest queued step has waited 60 minutes raises one `build-capacity-waits`, cleared
|
|
// when that step starts.
|
|
func TestABuildSeatsOldestQueuedStepWaitsAnHour(t *testing.T) {
|
|
const seat = "node-build-go"
|
|
key := FindingKey(ScopeDelivery, seat, KindBuildCapacityWaits)
|
|
wait := meshpipelinerunner.StepResultOutcomeWaitingForABuildNode
|
|
w, o, c := setup()
|
|
w.StepEvent(queuedStep("p1", "unit", seat, wait, c.t))
|
|
c.t = c.t.Add(10 * time.Minute)
|
|
w.StepEvent(queuedStep("p2", "unit", seat, wait, c.t))
|
|
pk := signalKey(capacityRow, seat)
|
|
if o.ttls[pk] != 50*time.Minute || len(o.findings) != 0 {
|
|
t.Fatalf("within the bound: %v %v", o.ttls, o.findings)
|
|
}
|
|
expire(w, c, pk, 50*time.Minute)
|
|
if _, ok := o.findings[key]; !ok || len(o.findings) != 1 {
|
|
t.Fatalf("an hour for the oldest: %v", o.findings)
|
|
}
|
|
w.StepEvent(queuedStep("p1", "unit", seat, meshpipelinerunner.StepResultOutcomeRunning, c.t))
|
|
if len(o.findings) != 0 {
|
|
t.Fatalf("the oldest started, the next has waited 50 minutes: %v", o.findings)
|
|
}
|
|
if o.ttls[pk] != 10*time.Minute {
|
|
t.Fatalf("the next one's hour: %v", o.ttls[pk])
|
|
}
|
|
w.StepEvent(queuedStep("p2", "unit", seat, meshpipelinerunner.StepResultOutcomePassed, c.t))
|
|
if _, ok := o.pulses[pk]; ok || len(o.findings) != 0 {
|
|
t.Fatalf("the queue is empty: %v %v", o.pulses, o.findings)
|
|
}
|
|
}
|
|
|
|
// design 47 G §389, §436 (test 106's cause): a machine whose presence key expires says `machine-away`, once, and one
|
|
// heard again says `machine-back`; a machine heard again that was never away says nothing.
|
|
func TestAMachinesPresenceIsSaidAsEvents(t *testing.T) {
|
|
w, o, c := setup()
|
|
w.Connect("node.ace")
|
|
w.Connect("node.ace") // heard again while never away
|
|
w.Disconnect("node.ace")
|
|
w.Disconnect("node.ace")
|
|
if len(o.said) != 0 {
|
|
t.Fatalf("said before the expiry: %v", o.said)
|
|
}
|
|
c.t = t0.Add(120 * time.Second)
|
|
w.Pulse(pulseKeyOf(t, presenceNS, "ace"), Expired, c.t)
|
|
w.Pulse(pulseKeyOf(t, presenceNS, "ace"), Expired, c.t) // the same expiry heard twice
|
|
w.Connect("node.ace")
|
|
w.Connect("node.ace")
|
|
got := []string{}
|
|
for _, s := range o.said {
|
|
got = append(got, s.subject)
|
|
}
|
|
want := []string{"mesh.seat.mesh-watchdogs.event.machine-away.ace", "mesh.seat.mesh-watchdogs.event.machine-back.ace"}
|
|
if len(got) != 2 || got[0] != want[0] || got[1] != want[1] {
|
|
t.Fatalf("said %v, want %v", got, want)
|
|
}
|
|
}
|
|
|
|
func fillBody(store string, bytes int64) []byte {
|
|
b, _ := json.Marshal(meshdelivery.FillingEvent{Store: store, Bytes: bytes, Cap: 100})
|
|
return b
|
|
}
|
|
|
|
// design 47 G §181, §263: a store past its share is `store-filling` naming the part and the store, full is urgent, `room` and a
|
|
// fill under 80 % clear it.
|
|
func TestAStoreFillingIsAFinding(t *testing.T) {
|
|
w, o, _ := setup()
|
|
key := FindingKey(ScopeDelivery, "mesh-delivery-records", KindStoreFilling)
|
|
w.StoreFill("mesh-delivery", "filling", fillBody("records", 60))
|
|
if len(o.findings) != 0 {
|
|
t.Fatalf("under 80 %%: %v", o.findings)
|
|
}
|
|
w.StoreFill("mesh-delivery", "filling", fillBody("records", 75))
|
|
if len(o.findings) != 0 {
|
|
t.Fatalf("75 %% is under §181's 80 %%: %v", o.findings)
|
|
}
|
|
w.StoreFill("mesh-delivery", "filling", fillBody("records", 85))
|
|
if o.findings[key].Severity != Warning {
|
|
t.Fatalf("filling: %v", o.findings)
|
|
}
|
|
w.StoreFill("mesh-delivery", "full", fillBody("records", 100))
|
|
if o.findings[key].Severity != Urgent {
|
|
t.Fatalf("full: %v", o.findings)
|
|
}
|
|
w.StoreFill("mesh-delivery", "room", fillBody("records", 60))
|
|
if len(o.findings) != 0 {
|
|
t.Fatalf("room: %v", o.findings)
|
|
}
|
|
}
|
|
|
|
func recordBody(id string, at time.Time) []byte {
|
|
b, _ := json.Marshal(meshdelivery.RecordOpenedEvent{Record: id, Milestone: "47/G", At: &at})
|
|
return b
|
|
}
|
|
|
|
// design 47 G §148, §160: a record open 24 hours is `record-open`, cleared by `record-ended`; a record that ends in time
|
|
// raises none; while a store is full, which already tells why, it raises none.
|
|
func TestARecordOpenADayIsAFinding(t *testing.T) {
|
|
w, o, c := setup()
|
|
pk := signalKey(recordRow, "r1")
|
|
w.RecordOpen(recordBody("r1", c.t))
|
|
if o.ttls[pk] != RecordOpenBound || len(o.findings) != 0 {
|
|
t.Fatalf("within the bound: %v %v", o.ttls, o.findings)
|
|
}
|
|
expire(w, c, pk, RecordOpenBound)
|
|
key := FindingKey(ScopeDelivery, "r1", KindRecordOpen)
|
|
if _, ok := o.findings[key]; !ok {
|
|
t.Fatalf("past 24 hours: %v", o.findings)
|
|
}
|
|
w.StoreFill("mesh-delivery", "full", fillBody("records", 100))
|
|
if _, ok := o.findings[key]; ok {
|
|
t.Fatalf("a full store covers the record: %v", o.findings)
|
|
}
|
|
w.StoreFill("mesh-delivery", "room", fillBody("records", 10))
|
|
if _, ok := o.findings[key]; !ok {
|
|
t.Fatalf("room, and the record stands open again: %v", o.findings)
|
|
}
|
|
w.RecordEnded(recordBody("r1", c.t))
|
|
if len(o.findings) != 0 {
|
|
t.Fatalf("record-ended: %v", o.findings)
|
|
}
|
|
w.RecordOpen(recordBody("r2", c.t))
|
|
w.RecordEnded(recordBody("r2", c.t))
|
|
if _, ok := o.pulses[signalKey(recordRow, "r2")]; ok {
|
|
t.Fatal("the pulse of an ended record stands")
|
|
}
|
|
}
|
|
|
|
// design 47 G §502, A-200 (WM16): a record is tied to the entries of the take-in.taken of its milestone `<branch>.<source revision>`; it
|
|
// is covered only while one of those entries is held by an `acting-part-late` wait, so another record's late entry covers nothing.
|
|
func TestARecordIsCoveredOnlyByTheWaitsOfItsOwnEntries(t *testing.T) {
|
|
w, o, c := setup()
|
|
taken := func(key string, source int64, module string) {
|
|
var m map[string]any
|
|
if err := json.Unmarshal(takeInBody(t, takein.TakeInTaken, key, c.t), &m); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
m["branch"], m["source"] = "main", source
|
|
m["entries"] = []map[string]any{{"commit": "c", "fingerprint": "f", "module": module}}
|
|
b, _ := json.Marshal(m)
|
|
w.TakeIn(takeInEvent(t, takein.TakeInTaken, key), b)
|
|
}
|
|
taken("m.1", 7, "web")
|
|
taken("m.2", 8, "db")
|
|
rec := func(id, milestone string) []byte {
|
|
b, _ := json.Marshal(meshdelivery.RecordOpenedEvent{Record: id, Milestone: milestone, At: &c.t})
|
|
return b
|
|
}
|
|
w.RecordOpen(rec("rWeb", "main.7"))
|
|
w.RecordOpen(rec("rDb", "main.8"))
|
|
late, _ := json.Marshal(meshwants.Wait{Step: "place", Part: "mesh-delivery-indexer", Node: "ace", Module: "web", Since: c.t, Bound: "30m"})
|
|
w.WaitHeld("mesh-wants", late)
|
|
expire(w, c, signalKey(actingRow, "mesh-wants-place-mesh-delivery-indexer-ace-web"), 30*time.Minute)
|
|
expire(w, c, signalKey(recordRow, "rWeb"), RecordOpenBound)
|
|
expire(w, c, signalKey(recordRow, "rDb"), 0)
|
|
if _, ok := o.findings[FindingKey(ScopeDelivery, "rWeb", KindRecordOpen)]; ok {
|
|
t.Fatalf("a record whose entry is held by acting-part-late is covered: %v", o.findings)
|
|
}
|
|
if _, ok := o.findings[FindingKey(ScopeDelivery, "rDb", KindRecordOpen)]; !ok {
|
|
t.Fatalf("a record of other entries is open at 24 hours whatever else is late: %v", o.findings)
|
|
}
|
|
}
|