diff --git a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/consumes.go b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/consumes.go index a154a11e..a0c405d4 100644 --- a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/consumes.go +++ b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/consumes.go @@ -5,6 +5,8 @@ import ( "encoding/json" "git.novox.be/novox/mesh-contracts/go/seats/meshbroker" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshpipelinerunner" "strings" "sync" @@ -41,6 +43,8 @@ func consumed(w *watch.Watchman, subject string, body []byte, logf func(string, return } w.Accepted(a) + case pipelineSubject(w, subject, body): + case recordSubject(w, subject, body): case brokerSubject(subject) && strings.HasSuffix(subject, ".event."+eventCreated): if s, err := watch.ReadBrokerSaid(body); err != nil { logf("%v; left", err) @@ -140,3 +144,41 @@ func hearConsumed(ctx context.Context, js jetstream.JetStream, cred process.Acco func brokerSubject(subject string) bool { return strings.HasPrefix(subject, "mesh.seat."+meshbroker.Seat+".event.") } + +// pipelineSubject hands the gate's `source-moved` and the runner's `pipeline-started` to the watchman (watch/pipelines.go, +// design 47 G §12): the head of a watched branch and its pipeline. It says whether the subject was one of them. +func pipelineSubject(w *watch.Watchman, subject string, body []byte) bool { + switch { + case strings.HasPrefix(subject, "mesh.seat."+meshdelivery.Seat+".event."+meshdelivery.EventSourceMoved+"."): + w.SourceMoved(body) + case strings.HasPrefix(subject, "mesh.seat."+meshpipelinerunner.Seat+".event."+meshpipelinerunner.EventPipelineStarted+"."): + w.PipelineStarted(body) + case strings.HasPrefix(subject, "mesh.seat."+meshpipelinerunner.Seat+".event."+meshpipelinerunner.EventStepStarted+"."), + strings.HasPrefix(subject, "mesh.seat."+meshpipelinerunner.Seat+".event."+meshpipelinerunner.EventStepEnded+"."): + w.StepEvent(body) + default: + return false + } + return true +} + +// recordSubject hands the seats' `filling`, `full` and `room` of a store and the seat mesh-delivery's `record-opened` and +// `record-ended` to the watchman (watch/records.go, design 47 G §148, §181). It says whether the subject was one of them. +func recordSubject(w *watch.Watchman, subject string, body []byte) bool { + parts := strings.Split(subject, ".") + if len(parts) != 5 || parts[0] != "mesh" || parts[1] != "seat" || parts[3] != "event" { + return false + } + seat, event := parts[2], parts[4] + switch { + case event == meshdelivery.EventFilling || event == meshdelivery.EventFull || event == meshdelivery.EventRoom: + w.StoreFill(seat, event, body) + case seat == meshdelivery.Seat && event == meshdelivery.EventRecordOpened: + w.RecordOpen(body) + case seat == meshdelivery.Seat && event == meshdelivery.EventRecordEnded: + w.RecordEnded(body) + default: + return false + } + return true +} diff --git a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go index 3fb22fe7..dc99c4ef 100644 --- a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go +++ b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main.go @@ -55,6 +55,23 @@ func main() { type kvOut struct { ctx context.Context presence, pulse, stuk, findings jetstream.KeyValue + js jetstream.JetStream + cred process.Account +} + +// Say publishes an event of the seat mesh-watchdogs, said under the watchman's own account (design 47 G §389, §436). +func (o *kvOut) Say(subject string, body []byte, id string) error { + msg := nats.NewMsg(subject) + msg.Header.Set("x-event-id", id) + msg.Header.Set("x-source", o.cred.Module) + msg.Header.Set("x-node", o.cred.Node) + msg.Header.Set("x-time", time.Now().UTC().Format(time.RFC3339Nano)) + msg.Header.Set("content-type", "application/json") + msg.Data = body + pub, cancel := context.WithTimeout(context.WithoutCancel(o.ctx), 10*time.Second) + defer cancel() + _, err := o.js.PublishMsg(pub, msg, jetstream.WithMsgID(id)) + return err } func (o *kvOut) PutFinding(key string, value []byte) error { @@ -251,7 +268,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return err } - w := watch.New(&kvOut{ctx: ctx, presence: kvs.presence, pulse: kvs.pulse, stuk: kvs.stuck, findings: kvs.findings}, time.Now) + w := watch.New(&kvOut{ctx: ctx, presence: kvs.presence, pulse: kvs.pulse, stuk: kvs.stuck, findings: kvs.findings, js: js, cred: cred}, time.Now) w.Logf(logger.Printf) // What the wants and the modules hold is ended by their own late read (watch.ReadLate), never by the start's end. w.Awaits(watch.AssignerWantsBucket) diff --git a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main_test.go b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main_test.go index a3f306da..a0f47f2d 100644 --- a/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main_test.go +++ b/modules/mesh-delivery-watchman/cmd/mesh-delivery-watchman/main_test.go @@ -152,5 +152,6 @@ func (o *findingsOut) PutPresence(string, bool) error { return nil func (o *findingsOut) PutPulse(string, []byte, time.Duration) error { return nil } func (o *findingsOut) DeletePulse(string) error { o.pulseDeletes++; return nil } func (o *findingsOut) PutStuck(string, []byte) error { return nil } +func (o *findingsOut) Say(string, []byte, string) error { return nil } func (o *findingsOut) PutFinding(key string, _ []byte) error { o.findings[key] = true; return nil } func (o *findingsOut) DeleteFinding(key string) error { delete(o.findings, key); return nil } diff --git a/modules/mesh-delivery-watchman/go.mod b/modules/mesh-delivery-watchman/go.mod index 6a4e9198..55006f65 100644 --- a/modules/mesh-delivery-watchman/go.mod +++ b/modules/mesh-delivery-watchman/go.mod @@ -14,6 +14,8 @@ require ( git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/contracts/go/clients/meshwantsclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-delivery/contracts/go/seats/meshpipelinerunner v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/modules/mesh-grants/busgrants v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000 diff --git a/modules/mesh-delivery-watchman/internal/watch/pipelines.go b/modules/mesh-delivery-watchman/internal/watch/pipelines.go new file mode 100644 index 00000000..ed9e62f2 --- /dev/null +++ b/modules/mesh-delivery-watchman/internal/watch/pipelines.go @@ -0,0 +1,169 @@ +package watch + +import ( + "encoding/json" + "fmt" + "time" + + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshpipelinerunner" +) + +// What the runner and the gate say of a watched branch's head, heard as events (design 47 G §12, §75, §112): a head the +// gate says moved (`source-moved`) that starts no pipeline within five minutes is the finding `pipeline-stands`, naming the +// branch and its head, raised by the condition keeper and cleared by the runner's `pipeline-started` for that head. + +// KindPipelineStands is a watched branch's head, or a step, standing still (design 47 G §12, §75). +const KindPipelineStands = "pipeline-stands" + +// HeadBound is how long a watched branch's head may stand with no pipeline started (design 47 G §12, §112: the record's five +// minutes). +const HeadBound = 5 * time.Minute + +// headRow is the signals table's row of a head with no pipeline. +const headRow = "head-pipeline" + +// head is a watched branch's head still waiting for its pipeline. +type head struct{ revision int64 } + +// SourceMoved hears the seat mesh-delivery's `source-moved`: the branch's head is a new source revision, and its pipeline +// is due within HeadBound. A move made by a gathered group is a member governed by the take-in's own bound (ADR 0633 §5), +// so it is not timed here (design 47 G §12). A later move of the same branch replaces the head that waits. +func (w *Watchman) SourceMoved(body []byte) { + var m meshdelivery.SourceMovedEvent + if err := json.Unmarshal(body, &m); err != nil || m.Branch == "" || m.Revision == 0 { + w.say("a source-moved does not read, left: %v", err) + return + } + w.mu.Lock() + defer w.mu.Unlock() + id := headID(m.Branch) + w.unsignal(headRow, id) + w.unfind(FindingKey(ScopeDelivery, id, KindPipelineStands)) + if m.Group != "" { + delete(w.heads, m.Branch) + return + } + at := w.now() + if m.At != nil && !m.At.IsZero() { + at = *m.At + } + w.heads[m.Branch] = head{revision: m.Revision} + f := Finding{Scope: ScopeDelivery, ID: id, Kind: KindPipelineStands, Severity: Warning, Source: headRow, Since: at, + Summary: fmt.Sprintf("the head of %s (source revision %d) has started no pipeline since %s", m.Branch, m.Revision, stamp(at)), + Said: fmt.Sprintf("no pipeline in %s", HeadBound), + Headline: "A change has started no pipeline", + Explanation: "The head of the watched branch " + m.Branch + " (source revision " + fmt.Sprint(m.Revision) + ") moved " + + HeadBound.String() + " ago or more and the pipeline runner has said no pipeline started for it.", + Needs: "the runner to start the pipeline (its pipeline_stands and list tools say where it is)"} + left := HeadBound - w.now().Sub(at) + if left > 0 { + w.signal(headRow, id, left, signalDue{Do: dueFind, Finding: f}) + return + } + w.find(f) +} + +// PipelineStarted hears the runner's `pipeline-started`: it ends the wait of the head it is for (the branch's head at that +// source revision or a later one). A run on request is no head's pipeline. +func (w *Watchman) PipelineStarted(body []byte) { + var p meshpipelinerunner.PipelineStartedEvent + if err := json.Unmarshal(body, &p); err != nil || p.Branch == "" { + w.say("a pipeline-started does not read, left: %v", err) + return + } + if p.Run != "" { + return + } + w.mu.Lock() + defer w.mu.Unlock() + h, ok := w.heads[p.Branch] + if !ok || p.Source < h.revision { + return + } + delete(w.heads, p.Branch) + id := headID(p.Branch) + w.unsignal(headRow, id) + w.unfind(FindingKey(ScopeDelivery, id, KindPipelineStands)) +} + +// headID is the finding's id of a branch's head. +func headID(branch string) string { return "head-" + unsafeKey.ReplaceAllString(branch, "_") } + +// KindBuildCapacityWaits is a build seat whose oldest queued step has waited the bound (design 47 G §86). +const KindBuildCapacityWaits = "build-capacity-waits" + +// CapacityBound is how long a step may wait queued on a build seat's stream before the seat is a finding (design 47 G §86). +const CapacityBound = 60 * time.Minute + +// capacityRow is the signals table's row of a build seat's queue. +const capacityRow = "build-capacity" + +// StepEvent hears the runner's `step-started` and `step-ended` (design 47 G §86): a step said waiting for a build node is +// queued on its seat, owed no progress; it leaves the queue when it starts running or ends. A seat's finding is due when +// its oldest queued step has waited CapacityBound, and ends when that seat has none that old. +func (w *Watchman) StepEvent(body []byte) { + var e meshpipelinerunner.StepEvent + if err := json.Unmarshal(body, &e); err != nil || e.Pipeline == "" || e.Result.Step == "" { + w.say("a step event does not read, left: %v", err) + return + } + id := e.Pipeline + "/" + e.Result.Step + w.mu.Lock() + defer w.mu.Unlock() + for seat, q := range w.queued { + if _, ok := q[id]; ok && (e.Result.Outcome != meshpipelinerunner.StepResultOutcomeWaitingForABuildNode || e.Result.Seat != seat) { + delete(q, id) + w.capacity(seat) + } + } + if e.Result.Outcome != meshpipelinerunner.StepResultOutcomeWaitingForABuildNode || e.Result.Seat == "" { + return + } + since := w.now() + switch { + case e.Result.Started != nil: + since = *e.Result.Started + case e.At != nil: + since = *e.At + } + seat := e.Result.Seat + if w.queued[seat] == nil { + w.queued[seat] = map[string]time.Time{} + } + if _, ok := w.queued[seat][id]; !ok { + w.queued[seat][id] = since + } + w.capacity(seat) +} + +// capacity judges a build seat's queue again, w.mu held: one signal for its oldest queued step. +func (w *Watchman) capacity(seat string) { + w.unsignal(capacityRow, seat) + key := FindingKey(ScopeDelivery, seat, KindBuildCapacityWaits) + var oldest time.Time + for _, t := range w.queued[seat] { + if oldest.IsZero() || t.Before(oldest) { + oldest = t + } + } + if oldest.IsZero() { + delete(w.queued, seat) + w.unfind(key) + return + } + f := Finding{Scope: ScopeDelivery, ID: seat, Kind: KindBuildCapacityWaits, Severity: Warning, Source: capacityRow, Since: oldest, + Summary: fmt.Sprintf("the oldest step queued on %s has waited since %s", seat, stamp(oldest)), + Said: "queued for " + CapacityBound.String(), + Headline: "Builds wait for a build node", + Explanation: "The oldest step queued on the build seat " + seat + " has waited " + CapacityBound.String() + + " for a build node to take it.", + Needs: "build capacity: a build node present and not paused, or fewer steps asked"} + left := CapacityBound - w.now().Sub(oldest) + if left > 0 { + w.unfind(key) + w.signal(capacityRow, seat, left, signalDue{Do: dueFind, Finding: f}) + return + } + w.find(f) +} diff --git a/modules/mesh-delivery-watchman/internal/watch/pipelines_test.go b/modules/mesh-delivery-watchman/internal/watch/pipelines_test.go new file mode 100644 index 00000000..b384cafd --- /dev/null +++ b/modules/mesh-delivery-watchman/internal/watch/pipelines_test.go @@ -0,0 +1,205 @@ +package watch + +import ( + "encoding/json" + "testing" + "time" + + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshpipelinerunner" +) + +func moved(branch string, rev int64, group string, at time.Time) []byte { + b, _ := json.Marshal(meshdelivery.SourceMovedEvent{Branch: branch, Revision: rev, Group: group, At: &at}) + return b +} + +func started(branch string, rev int64, run string) []byte { + b, _ := json.Marshal(meshpipelinerunner.PipelineStartedEvent{Branch: branch, Source: rev, Run: run, Module: "m", Pipeline: "p"}) + return b +} + +// design 47 G §12, §79 (test 76): a head with no pipeline within five minutes is `pipeline-stands`, cleared by the +// runner's `pipeline-started` for that head; a gathered group's member and a run on request do not. +func TestAHeadWithNoPipelineStands(t *testing.T) { + key := FindingKey(ScopeDelivery, headID("main"), KindPipelineStands) + cases := []struct { + name string + run func(w *Watchman, o *fakeOut, c *clock) + stand bool + }{ + {"five minutes and no pipeline", func(w *Watchman, o *fakeOut, c *clock) { + w.SourceMoved(moved("main", 7, "", c.t)) + if o.ttls[signalKey(headRow, headID("main"))] != HeadBound || len(o.findings) != 0 { + t.Fatalf("within the bound: %v %v", o.ttls, o.findings) + } + expire(w, c, signalKey(headRow, headID("main")), HeadBound) + }, true}, + {"its pipeline-started clears it", func(w *Watchman, o *fakeOut, c *clock) { + w.SourceMoved(moved("main", 7, "", c.t)) + expire(w, c, signalKey(headRow, headID("main")), HeadBound) + w.PipelineStarted(started("main", 7, "")) + }, false}, + {"a pipeline before the bound raises none", func(w *Watchman, o *fakeOut, c *clock) { + w.SourceMoved(moved("main", 7, "", c.t)) + w.PipelineStarted(started("main", 8, "")) + if _, ok := o.pulses[signalKey(headRow, headID("main"))]; ok { + t.Fatal("the pulse stands") + } + }, false}, + {"a pipeline of an older head does not clear a newer head", func(w *Watchman, o *fakeOut, c *clock) { + w.SourceMoved(moved("main", 7, "", c.t)) + w.PipelineStarted(started("main", 6, "")) + expire(w, c, signalKey(headRow, headID("main")), HeadBound) + }, true}, + {"a gathered group's member is not timed", func(w *Watchman, o *fakeOut, c *clock) { + w.SourceMoved(moved("main", 7, "grp", c.t)) + if len(o.pulses) != 0 { + t.Fatalf("a pulse for a member: %v", o.pulses) + } + }, false}, + {"a run on request is no head's pipeline", func(w *Watchman, o *fakeOut, c *clock) { + w.SourceMoved(moved("main", 7, "", c.t)) + w.PipelineStarted(started("main", 7, "run-1")) + expire(w, c, signalKey(headRow, headID("main")), HeadBound) + }, true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + w, o, c := setup() + tc.run(w, o, c) + if _, ok := o.findings[key]; ok != tc.stand { + t.Fatalf("the finding stands %v, want %v: %v", ok, tc.stand, o.findings) + } + }) + } +} + +func queuedStep(pipeline, step, seat, outcome string, started time.Time) []byte { + b, _ := json.Marshal(meshpipelinerunner.StepEvent{Pipeline: pipeline, Branch: "main", Module: "m", Commit: "c", + Result: meshpipelinerunner.StepResult{Step: step, Seat: seat, Outcome: outcome, Started: &started}}) + return b +} + +// design 47 G §86: a build seat whose oldest queued step has waited 60 minutes raises one `build-capacity-waits`, cleared +// when that step starts. +func TestABuildSeatsOldestQueuedStepWaitsAnHour(t *testing.T) { + const seat = "node-build-go" + key := FindingKey(ScopeDelivery, seat, KindBuildCapacityWaits) + wait := meshpipelinerunner.StepResultOutcomeWaitingForABuildNode + w, o, c := setup() + w.StepEvent(queuedStep("p1", "unit", seat, wait, c.t)) + c.t = c.t.Add(10 * time.Minute) + w.StepEvent(queuedStep("p2", "unit", seat, wait, c.t)) + pk := signalKey(capacityRow, seat) + if o.ttls[pk] != 50*time.Minute || len(o.findings) != 0 { + t.Fatalf("within the bound: %v %v", o.ttls, o.findings) + } + expire(w, c, pk, 50*time.Minute) + if _, ok := o.findings[key]; !ok || len(o.findings) != 1 { + t.Fatalf("an hour for the oldest: %v", o.findings) + } + w.StepEvent(queuedStep("p1", "unit", seat, meshpipelinerunner.StepResultOutcomeRunning, c.t)) + if len(o.findings) != 0 { + t.Fatalf("the oldest started, the next has waited 50 minutes: %v", o.findings) + } + if o.ttls[pk] != 10*time.Minute { + t.Fatalf("the next one's hour: %v", o.ttls[pk]) + } + w.StepEvent(queuedStep("p2", "unit", seat, meshpipelinerunner.StepResultOutcomePassed, c.t)) + if _, ok := o.pulses[pk]; ok || len(o.findings) != 0 { + t.Fatalf("the queue is empty: %v %v", o.pulses, o.findings) + } +} + +// design 47 G §389, §436 (test 106's cause): a machine whose presence key expires says `machine-away`, once, and one +// heard again says `machine-back`; a machine heard again that was never away says nothing. +func TestAMachinesPresenceIsSaidAsEvents(t *testing.T) { + w, o, c := setup() + w.Connect("node.ace") + w.Connect("node.ace") // heard again while never away + w.Disconnect("node.ace") + w.Disconnect("node.ace") + if len(o.said) != 0 { + t.Fatalf("said before the expiry: %v", o.said) + } + c.t = t0.Add(120 * time.Second) + w.Pulse(pulseKeyOf(t, presenceNS, "ace"), Expired, c.t) + w.Pulse(pulseKeyOf(t, presenceNS, "ace"), Expired, c.t) // the same expiry heard twice + w.Connect("node.ace") + w.Connect("node.ace") + got := []string{} + for _, s := range o.said { + got = append(got, s.subject) + } + want := []string{"mesh.seat.mesh-watchdogs.event.machine-away.ace", "mesh.seat.mesh-watchdogs.event.machine-back.ace"} + if len(got) != 2 || got[0] != want[0] || got[1] != want[1] { + t.Fatalf("said %v, want %v", got, want) + } +} + +func fillBody(store string, bytes int64) []byte { + b, _ := json.Marshal(meshdelivery.FillingEvent{Store: store, Bytes: bytes, Cap: 100}) + return b +} + +// design 47 G §181, §263: a store past its share is `store-filling` naming the part and the store, full is urgent, `room` and a +// fill under 70 % clear it. +func TestAStoreFillingIsAFinding(t *testing.T) { + w, o, _ := setup() + key := FindingKey(ScopeDelivery, "mesh-delivery-records", KindStoreFilling) + w.StoreFill("mesh-delivery", "filling", fillBody("records", 60)) + if len(o.findings) != 0 { + t.Fatalf("under 70 %%: %v", o.findings) + } + w.StoreFill("mesh-delivery", "filling", fillBody("records", 85)) + if o.findings[key].Severity != Warning { + t.Fatalf("filling: %v", o.findings) + } + w.StoreFill("mesh-delivery", "full", fillBody("records", 100)) + if o.findings[key].Severity != Urgent { + t.Fatalf("full: %v", o.findings) + } + w.StoreFill("mesh-delivery", "room", fillBody("records", 60)) + if len(o.findings) != 0 { + t.Fatalf("room: %v", o.findings) + } +} + +func recordBody(id string, at time.Time) []byte { + b, _ := json.Marshal(meshdelivery.RecordOpenedEvent{Record: id, Milestone: "47/G", At: &at}) + return b +} + +// design 47 G §148, §160: a record open 24 hours is `record-open`, cleared by `record-ended`; a record that ends in time +// raises none; while a store is full, which already tells why, it raises none. +func TestARecordOpenADayIsAFinding(t *testing.T) { + w, o, c := setup() + pk := signalKey(recordRow, "r1") + w.RecordOpen(recordBody("r1", c.t)) + if o.ttls[pk] != RecordOpenBound || len(o.findings) != 0 { + t.Fatalf("within the bound: %v %v", o.ttls, o.findings) + } + expire(w, c, pk, RecordOpenBound) + key := FindingKey(ScopeDelivery, "r1", KindRecordOpen) + if _, ok := o.findings[key]; !ok { + t.Fatalf("past 24 hours: %v", o.findings) + } + w.StoreFill("mesh-delivery", "full", fillBody("records", 100)) + if _, ok := o.findings[key]; ok { + t.Fatalf("a full store covers the record: %v", o.findings) + } + w.StoreFill("mesh-delivery", "room", fillBody("records", 10)) + if _, ok := o.findings[key]; !ok { + t.Fatalf("room, and the record stands open again: %v", o.findings) + } + w.RecordEnded(recordBody("r1", c.t)) + if len(o.findings) != 0 { + t.Fatalf("record-ended: %v", o.findings) + } + w.RecordOpen(recordBody("r2", c.t)) + w.RecordEnded(recordBody("r2", c.t)) + if _, ok := o.pulses[signalKey(recordRow, "r2")]; ok { + t.Fatal("the pulse of an ended record stands") + } +} diff --git a/modules/mesh-delivery-watchman/internal/watch/presence.go b/modules/mesh-delivery-watchman/internal/watch/presence.go index 5fe16e07..b9881a31 100644 --- a/modules/mesh-delivery-watchman/internal/watch/presence.go +++ b/modules/mesh-delivery-watchman/internal/watch/presence.go @@ -8,6 +8,7 @@ import ( "time" "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshwatchdogsclient" + "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshwatchdogs" ) // partPrefix is what the watchman keeps presence of among the module users: the parts of delivery. @@ -186,3 +187,39 @@ func (w *Watchman) Rebuild(ask func(body []byte) ([]byte, error)) error { w.Connections(users) return nil } + +// machineAway says `machine-away` of the seat mesh-watchdogs when a machine's presence key has expired (design 47 G §389, §436): +// the machine, when it was last heard and when this was said, once until the machine is heard again. +func (w *Watchman) machineAway(node string, lastHeard time.Time) { + if _, said := w.awaySaid[node]; said { + return + } + at := w.now().UTC() + body, err := json.Marshal(meshwatchdogs.MachineAwayEvent{V: 1, Machine: node, At: at, LastHeard: lastHeard.UTC()}) + if err != nil { + return + } + if err := w.out.Say(meshwatchdogs.SubjectMachineAway(node), body, "machine-away."+node+"."+at.Format(time.RFC3339Nano)); err != nil { + w.say("cannot say machine-away of %s: %v", node, err) + return + } + w.awaySaid[node] = lastHeard.UTC() +} + +// machineBack says `machine-back` when a machine whose machine-away was said is heard again (design 47 G §389). +func (w *Watchman) machineBack(node string) { + lastHeard, said := w.awaySaid[node] + if !said { + return + } + at := w.now().UTC() + body, err := json.Marshal(meshwatchdogs.MachineBackEvent{V: 1, Machine: node, At: at, LastHeard: lastHeard}) + if err != nil { + return + } + if err := w.out.Say(meshwatchdogs.SubjectMachineBack(node), body, "machine-back."+node+"."+at.Format(time.RFC3339Nano)); err != nil { + w.say("cannot say machine-back of %s: %v", node, err) + return + } + delete(w.awaySaid, node) +} diff --git a/modules/mesh-delivery-watchman/internal/watch/records.go b/modules/mesh-delivery-watchman/internal/watch/records.go new file mode 100644 index 00000000..f51e2bfa --- /dev/null +++ b/modules/mesh-delivery-watchman/internal/watch/records.go @@ -0,0 +1,149 @@ +package watch + +import ( + "encoding/json" + "fmt" + "time" + + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery" +) + +// What the seats say of the delivery's records and stores, heard as the seats' events (design 47 G §148, §181, §218; ADR 0596 +// §2), never read from their state: +// - `filling`, `full` and `room` of a store against its cap: `store-filling` while it stands past its cap's share, urgent +// when it is full, cleared by `room` or by a `filling` under FillingFrom; +// - `record-opened` and `record-ended` of a delivery record: `record-open` when it has stood open for 24 hours, unless +// another condition already tells why (`acting-part-late`, or a store `full`). +// Each is a row of the signals table: its pulse is its bound, written at the event, so a restart loses none. + +// The kinds of these findings. +const ( + KindActingPartLate = "acting-part-late" + KindStoreFilling = "store-filling" + KindRecordOpen = "record-open" +) + +// The rows of the signals table of these findings, which a restart keeps. +const ( + storeRow = "store-fill" + recordRow = "record-open" +) + +// uncovered is the While of a record-open signal: the finding is written at the bound only while no other condition covers +// the record. +const uncovered = "uncovered:record-open" + +// RecordOpenBound is how long a delivery record may stay open before it is a finding (design 47 G §148). +const RecordOpenBound = 24 * time.Hour + +// FillingFrom is the share of a store's cap under which a `filling` clears its finding (design 47 G §181). +const FillingFrom = 0.70 + +// StoreFill hears a seat's `filling`, `full` or `room` of one of its stores. A `filling` is a warning, naming the part and the +// store, unless its share is under FillingFrom, which clears it; `full` is urgent, naming what it refuses; `room` clears either. +func (w *Watchman) StoreFill(seat, event string, body []byte) { + var f meshdelivery.FillingEvent + if err := json.Unmarshal(body, &f); err != nil || f.Store == "" { + w.say("the %s of the seat %s does not read, left: %v", event, seat, err) + return + } + id := seat + "-" + f.Store + key := FindingKey(ScopeDelivery, id, KindStoreFilling) + w.mu.Lock() + defer w.mu.Unlock() + share := 0.0 + if f.Cap > 0 { + share = float64(f.Bytes) / float64(f.Cap) + } + since := w.now() + if f.At != nil && !f.At.IsZero() { + since = *f.At + } + switch { + case event == "room" || (event == "filling" && share < FillingFrom): + w.unfind(key) + case event == "full": + w.find(Finding{Scope: ScopeDelivery, ID: id, Kind: KindStoreFilling, Severity: Urgent, Source: storeRow, Since: since, + Summary: fmt.Sprintf("%s's store %s is full (%d of %d bytes)", seat, f.Store, f.Bytes, f.Cap), + Said: "full", Headline: "A store of the delivery is full", + Explanation: "The store " + f.Store + " of the seat " + seat + " is at its cap, so it refuses " + orWords(f.Refused, "what it is asked to keep") + + " until room is made.", + Needs: "room made in the store (a removal, or its cap raised)"}) + default: + w.find(Finding{Scope: ScopeDelivery, ID: id, Kind: KindStoreFilling, Severity: Warning, Source: storeRow, Since: since, + Summary: fmt.Sprintf("%s's store %s is at %.0f %% of its cap (%d of %d bytes)", seat, f.Store, share*100, f.Bytes, f.Cap), + Said: "filling", Headline: "A store of the delivery is filling", + Explanation: "The store " + f.Store + " of the seat " + seat + " is nearly full; once it is full it refuses new entries.", + Needs: "room made before it is full"}) + } + w.rejudgeRecords() +} + +// RecordOpen hears the seat mesh-delivery's `record-opened`: its finding is due when the record has stood open for 24 hours. +func (w *Watchman) RecordOpen(body []byte) { + var r meshdelivery.RecordOpenedEvent + if err := json.Unmarshal(body, &r); err != nil || r.Record == "" { + w.say("a record-opened does not read, left: %v", err) + return + } + w.mu.Lock() + defer w.mu.Unlock() + since := w.now() + if r.At != nil && !r.At.IsZero() { + since = *r.At + } + f := Finding{Scope: ScopeDelivery, ID: r.Record, Kind: KindRecordOpen, Severity: Warning, Source: recordRow, Since: since, + Summary: fmt.Sprintf("the delivery record of %s has stood open since %s", r.Milestone, stamp(since)), + Said: "open for " + RecordOpenBound.String(), Headline: "A delivery is not on every machine yet", + Explanation: "The delivery record of " + r.Milestone + " has stood open for " + RecordOpenBound.String() + + ": a node does not run what it changed, or a step never answered.", + Needs: "what it waits for (gate_record names it), or the record ended by hand if a node is gone for good (gate_record_end)"} + w.unsignal(recordRow, r.Record) + w.unfind(f.Key()) + if left := RecordOpenBound - w.now().Sub(since); left > 0 { + w.signal(recordRow, r.Record, left, signalDue{Do: dueFind, While: uncovered, Finding: f}) + return + } + w.opened[r.Record] = f + w.rejudgeRecords() +} + +// RecordEnded hears the seat mesh-delivery's `record-ended`: its pulse and its finding end. +func (w *Watchman) RecordEnded(body []byte) { + var r meshdelivery.RecordEndedEvent + if err := json.Unmarshal(body, &r); err != nil || r.Record == "" { + w.say("a record-ended does not read, left: %v", err) + return + } + w.mu.Lock() + defer w.mu.Unlock() + w.unsignal(recordRow, r.Record) + delete(w.opened, r.Record) + w.unfind(FindingKey(ScopeDelivery, r.Record, KindRecordOpen)) +} + +// recordCovered says another condition tells the operator why a record is open: a wait past its bound or a full store. +func (w *Watchman) recordCovered() bool { + for _, v := range w.findings { + f, err := DecodeFinding(v) + if err != nil { + continue + } + if f.Kind == KindActingPartLate || (f.Kind == KindStoreFilling && f.Severity == Urgent) { + return true + } + } + return false +} + +// rejudgeRecords writes the finding of each record past its bound that nothing covers, and ends it when something does. +func (w *Watchman) rejudgeRecords() { + for id, f := range w.opened { + if w.recordCovered() { + w.unfind(f.Key()) + continue + } + w.find(f) + _ = id + } +} diff --git a/modules/mesh-delivery-watchman/internal/watch/signals.go b/modules/mesh-delivery-watchman/internal/watch/signals.go index 92b43f17..4829dc6f 100644 --- a/modules/mesh-delivery-watchman/internal/watch/signals.go +++ b/modules/mesh-delivery-watchman/internal/watch/signals.go @@ -171,6 +171,11 @@ func (w *Watchman) signalExpired(pk string) { case dueEnd: w.unfind(due.Finding.Key()) case dueFind: + if due.While == uncovered { + w.opened[due.Finding.ID] = due.Finding + w.rejudgeRecords() + return + } if due.While != "" && !w.stillWrong(due.While, due.Finding.Kind) { return } @@ -181,6 +186,9 @@ func (w *Watchman) signalExpired(pk string) { // stillWrong says a have's key still says what a row's finding was waited on: failed for apply-failing, unhealthy for // unhealthy. func (w *Watchman) stillWrong(haveKey, kind string) bool { + if haveKey == uncovered { + return !w.recordCovered() + } if node, ok := strings.CutPrefix(haveKey, "silent:"); ok { _, open := w.findings[FindingKey(ScopeMachine, node, KindSilent)] return open @@ -254,6 +262,11 @@ func (w *Watchman) stuckFinding(sk string, s meshwatchdogsclient.Stuck) { if s.Bucket == PresenceBucket { if node, part, _, err := meshwatchdogsclient.ParsePresenceKey(s.Key); err == nil && part == "" && node != "" { key := FindingKey(ScopeMachine, node, KindSilent) + if s.Moved { + w.machineBack(node) + } else { + w.machineAway(node, s.Since) + } defer w.networks() // a silent machine is down on the record: what points at it is held there, said by its silence if s.Moved || w.away[node] { w.unfind(key) @@ -931,6 +944,8 @@ var SignalRows = []struct{ Row, Signal, Bound, Kind string }{ {"S21", "a machine's have failed or its provisioner client's files not placed", ApplyEvery.String(), KindApplyFailing}, {"health", "a have unhealthy in two writes in a row; urgent after " + UnhealthyUrgentAfter.String(), "two writes", KindUnhealthy}, {"take-in", "a source revision or a delivery group the indexer holds back (ADR 0633 §5)", TakeInRevisionBound.String() + " a revision, " + TakeInGroupBound.String() + " a group", KindTakeInWaits}, + {storeRow, "a store of the delivery's parts filling or full (the seats' filling, full and room)", "filling from " + fmt.Sprintf("%.0f", FillingFrom*100) + " %, full urgent", KindStoreFilling}, + {recordRow, "a delivery record open (the seat mesh-delivery's record-opened, ended by record-ended)", RecordOpenBound.String(), KindRecordOpen}, {"provisions", "a provision of mesh-provisioner in waits (ADR 0570 §7)", "at once", KindNeedsOperator}, } diff --git a/modules/mesh-delivery-watchman/internal/watch/watch.go b/modules/mesh-delivery-watchman/internal/watch/watch.go index e73bc987..88fb281e 100644 --- a/modules/mesh-delivery-watchman/internal/watch/watch.go +++ b/modules/mesh-delivery-watchman/internal/watch/watch.go @@ -36,6 +36,8 @@ type Out interface { // PutFinding writes one of the watchman's findings; DeleteFinding deletes one it no longer observes. PutFinding(key string, value []byte) error DeleteFinding(key string) error + // Say publishes an event of the seat mesh-watchdogs on its subject (design 47 G §389), once under its id. + Say(subject string, body []byte, id string) error } type presence struct { @@ -112,6 +114,14 @@ type Watchman struct { // pulses, stuck keys and findings wait for its own end (ReadLate), so a restart while its owner is down neither drops // nor raises again what it holds. awaited map[string]bool + // heads is, by watched branch, the head still waiting for its pipeline (pipelines.go). + heads map[string]head + // queued is, by build seat, each step queued on it for a build node and since when (pipelines.go). + queued map[string]map[string]time.Time + // awaySaid is each machine whose machine-away was said and not yet its machine-back, with when it was last heard (presence.go). + awaySaid map[string]time.Time + // opened is each record past its bound and not ended, with its finding (records.go). + opened map[string]Finding } // New is a watchman writing to out, reading the time from now. @@ -122,7 +132,7 @@ func New(out Out, now func() time.Time) *Watchman { haveState: map[string]nodeengine.HaveByNodeModuleValue{}, unhealthy: map[string]int{}, advised: map[string]int{}, advisedSince: map[string]time.Time{}, control: map[string]bool{}, failedSince: map[string]time.Time{}, provFailed: map[string]map[string]ProvReport{}, closures: map[string]map[string]time.Time{}, waiting: map[string]map[string]string{}, health: map[string]nodeengine.HealthByNodeValue{}, tunnels: map[string]string{}, - standing: map[string]map[string]bool{}, readWhole: map[string]bool{}, awaited: map[string]bool{}} + standing: map[string]map[string]bool{}, readWhole: map[string]bool{}, awaited: map[string]bool{}, heads: map[string]head{}, queued: map[string]map[string]time.Time{}, awaySaid: map[string]time.Time{}, opened: map[string]Finding{}} } // Logf sets where a write that failed is said; it is silent until set. diff --git a/modules/mesh-delivery-watchman/internal/watch/watch_test.go b/modules/mesh-delivery-watchman/internal/watch/watch_test.go index 004b7ad9..740d3834 100644 --- a/modules/mesh-delivery-watchman/internal/watch/watch_test.go +++ b/modules/mesh-delivery-watchman/internal/watch/watch_test.go @@ -26,6 +26,14 @@ type fakeOut struct { stuckWrites int findings map[string]Finding findingWrites int + said []said +} + +type said struct{ subject, id string } + +func (o *fakeOut) Say(subject string, _ []byte, id string) error { + o.said = append(o.said, said{subject, id}) + return nil } func newOut() *fakeOut { diff --git a/modules/mesh-delivery-watchman/manifest_test.go b/modules/mesh-delivery-watchman/manifest_test.go index 79799276..e8d37103 100644 --- a/modules/mesh-delivery-watchman/manifest_test.go +++ b/modules/mesh-delivery-watchman/manifest_test.go @@ -7,6 +7,7 @@ import ( "os" "path/filepath" "reflect" + "regexp" "slices" "strings" "testing" @@ -98,6 +99,10 @@ func TestManifestStateAndReads(t *testing.T) { // 1135/AC6 "mesh-delivery-builder.builds", "mesh-host.health", "mesh-provisioner.provisions", "node-engine.have"}) { t.Errorf("reads = %v", reads) } + // Design 47 G §368: the version is MAJOR.MINOR and nothing else. + if !regexp.MustCompile(`^[0-9]+\.[0-9]+$`).MatchString(m.Version) { + t.Errorf("version = %q, want MAJOR.MINOR", m.Version) + } // DLV-M5 (ADR 0615 §1): the assigner's wants and the indexer's modules are their owners' states, asked by their read // tools and followed by their views families, never read. for _, r := range m.Reads { @@ -111,7 +116,7 @@ func TestManifestStateAndReads(t *testing.T) { // 1135/AC6 t.Errorf("invokes = %v, want %v", m.Invokes, want) } // DLV-M5 its first views, 1585/T20 the indexer's take-in events, and DLV-W1 (ADR 0595 §4): the accepted verdicts and the bus module's created and refused, durably. - if want := []string{"mesh-delivery-assigner.views.wants", "mesh-delivery-indexer.take-in.pending", "mesh-delivery-indexer.take-in.taken", "mesh-delivery-indexer.views.modules", "mesh-catalog.manifest-accepted", "mesh-broker.created", "mesh-broker.refused"}; !reflect.DeepEqual(m.Consumes, want) { + if want := []string{"mesh-delivery-assigner.views.wants", "mesh-delivery-indexer.take-in.pending", "mesh-delivery-indexer.take-in.taken", "mesh-delivery-indexer.views.modules", "mesh-catalog.manifest-accepted", "mesh-delivery.source-moved", "mesh-delivery.filling", "mesh-delivery.full", "mesh-delivery.room", "mesh-delivery.record-opened", "mesh-delivery.record-ended", "mesh-catalog.filling", "mesh-catalog.full", "mesh-catalog.room", "mesh-wants.filling", "mesh-wants.full", "mesh-wants.room", "mesh-grants.filling", "mesh-grants.full", "mesh-grants.room", "mesh-pipeline-runner.pipeline-started", "mesh-pipeline-runner.step-started", "mesh-pipeline-runner.step-ended", "mesh-broker.created", "mesh-broker.refused"}; !reflect.DeepEqual(m.Consumes, want) { t.Errorf("consumes = %v, want %v", m.Consumes, want) } // 1952/WM1: the watchman holds the seat mesh-watchdogs at mesh scope, its first seat (novox/hq ADR 0564 §1, §4). diff --git a/modules/mesh-delivery-watchman/module.json b/modules/mesh-delivery-watchman/module.json index 25acacd9..bd081f2f 100644 --- a/modules/mesh-delivery-watchman/module.json +++ b/modules/mesh-delivery-watchman/module.json @@ -1,6 +1,6 @@ { "module": "mesh-delivery-watchman", - "version": "1", + "version": "0.1", "slug": "watchman", "claims": [ { @@ -36,6 +36,24 @@ "mesh-delivery-indexer.take-in.taken", "mesh-delivery-indexer.views.modules", "mesh-catalog.manifest-accepted", + "mesh-delivery.source-moved", + "mesh-delivery.filling", + "mesh-delivery.full", + "mesh-delivery.room", + "mesh-delivery.record-opened", + "mesh-delivery.record-ended", + "mesh-catalog.filling", + "mesh-catalog.full", + "mesh-catalog.room", + "mesh-wants.filling", + "mesh-wants.full", + "mesh-wants.room", + "mesh-grants.filling", + "mesh-grants.full", + "mesh-grants.room", + "mesh-pipeline-runner.pipeline-started", + "mesh-pipeline-runner.step-started", + "mesh-pipeline-runner.step-ended", "mesh-broker.created", "mesh-broker.refused" ],