Merge branch 'feat/47-G-build' into feat/47-G-build-assigner

This commit is contained in:
2026-11-11 12:05:11 +01:00
13 changed files with 682 additions and 4 deletions
@@ -5,6 +5,8 @@ import (
"encoding/json"
"git.novox.be/novox/mesh-contracts/go/seats/meshbroker"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshpipelinerunner"
"strings"
"sync"
@@ -41,6 +43,8 @@ func consumed(w *watch.Watchman, subject string, body []byte, logf func(string,
return
}
w.Accepted(a)
case pipelineSubject(w, subject, body):
case recordSubject(w, subject, body):
case brokerSubject(subject) && strings.HasSuffix(subject, ".event."+eventCreated):
if s, err := watch.ReadBrokerSaid(body); err != nil {
logf("%v; left", err)
@@ -140,3 +144,41 @@ func hearConsumed(ctx context.Context, js jetstream.JetStream, cred process.Acco
func brokerSubject(subject string) bool {
return strings.HasPrefix(subject, "mesh.seat."+meshbroker.Seat+".event.")
}
// pipelineSubject hands the gate's `source-moved` and the runner's `pipeline-started` to the watchman (watch/pipelines.go,
// design 47 G §12): the head of a watched branch and its pipeline. It says whether the subject was one of them.
func pipelineSubject(w *watch.Watchman, subject string, body []byte) bool {
switch {
case strings.HasPrefix(subject, "mesh.seat."+meshdelivery.Seat+".event."+meshdelivery.EventSourceMoved+"."):
w.SourceMoved(body)
case strings.HasPrefix(subject, "mesh.seat."+meshpipelinerunner.Seat+".event."+meshpipelinerunner.EventPipelineStarted+"."):
w.PipelineStarted(body)
case strings.HasPrefix(subject, "mesh.seat."+meshpipelinerunner.Seat+".event."+meshpipelinerunner.EventStepStarted+"."),
strings.HasPrefix(subject, "mesh.seat."+meshpipelinerunner.Seat+".event."+meshpipelinerunner.EventStepEnded+"."):
w.StepEvent(body)
default:
return false
}
return true
}
// recordSubject hands the seats' `filling`, `full` and `room` of a store and the seat mesh-delivery's `record-opened` and
// `record-ended` to the watchman (watch/records.go, design 47 G §148, §181). It says whether the subject was one of them.
func recordSubject(w *watch.Watchman, subject string, body []byte) bool {
parts := strings.Split(subject, ".")
if len(parts) != 5 || parts[0] != "mesh" || parts[1] != "seat" || parts[3] != "event" {
return false
}
seat, event := parts[2], parts[4]
switch {
case event == meshdelivery.EventFilling || event == meshdelivery.EventFull || event == meshdelivery.EventRoom:
w.StoreFill(seat, event, body)
case seat == meshdelivery.Seat && event == meshdelivery.EventRecordOpened:
w.RecordOpen(body)
case seat == meshdelivery.Seat && event == meshdelivery.EventRecordEnded:
w.RecordEnded(body)
default:
return false
}
return true
}
@@ -55,6 +55,23 @@ func main() {
type kvOut struct {
ctx context.Context
presence, pulse, stuk, findings jetstream.KeyValue
js jetstream.JetStream
cred process.Account
}
// Say publishes an event of the seat mesh-watchdogs, said under the watchman's own account (design 47 G §389, §436).
func (o *kvOut) Say(subject string, body []byte, id string) error {
msg := nats.NewMsg(subject)
msg.Header.Set("x-event-id", id)
msg.Header.Set("x-source", o.cred.Module)
msg.Header.Set("x-node", o.cred.Node)
msg.Header.Set("x-time", time.Now().UTC().Format(time.RFC3339Nano))
msg.Header.Set("content-type", "application/json")
msg.Data = body
pub, cancel := context.WithTimeout(context.WithoutCancel(o.ctx), 10*time.Second)
defer cancel()
_, err := o.js.PublishMsg(pub, msg, jetstream.WithMsgID(id))
return err
}
func (o *kvOut) PutFinding(key string, value []byte) error {
@@ -251,7 +268,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
return err
}
w := watch.New(&kvOut{ctx: ctx, presence: kvs.presence, pulse: kvs.pulse, stuk: kvs.stuck, findings: kvs.findings}, time.Now)
w := watch.New(&kvOut{ctx: ctx, presence: kvs.presence, pulse: kvs.pulse, stuk: kvs.stuck, findings: kvs.findings, js: js, cred: cred}, time.Now)
w.Logf(logger.Printf)
// What the wants and the modules hold is ended by their own late read (watch.ReadLate), never by the start's end.
w.Awaits(watch.AssignerWantsBucket)
@@ -152,5 +152,6 @@ func (o *findingsOut) PutPresence(string, bool) error { return nil
func (o *findingsOut) PutPulse(string, []byte, time.Duration) error { return nil }
func (o *findingsOut) DeletePulse(string) error { o.pulseDeletes++; return nil }
func (o *findingsOut) PutStuck(string, []byte) error { return nil }
func (o *findingsOut) Say(string, []byte, string) error { return nil }
func (o *findingsOut) PutFinding(key string, _ []byte) error { o.findings[key] = true; return nil }
func (o *findingsOut) DeleteFinding(key string) error { delete(o.findings, key); return nil }
+2
View File
@@ -14,6 +14,8 @@ require (
git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/contracts/go/clients/meshwantsclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/contracts/go/seats/meshpipelinerunner v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/modules/mesh-grants/busgrants v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000
@@ -0,0 +1,169 @@
package watch
import (
"encoding/json"
"fmt"
"time"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshpipelinerunner"
)
// What the runner and the gate say of a watched branch's head, heard as events (design 47 G §12, §75, §112): a head the
// gate says moved (`source-moved`) that starts no pipeline within five minutes is the finding `pipeline-stands`, naming the
// branch and its head, raised by the condition keeper and cleared by the runner's `pipeline-started` for that head.
// KindPipelineStands is a watched branch's head, or a step, standing still (design 47 G §12, §75).
const KindPipelineStands = "pipeline-stands"
// HeadBound is how long a watched branch's head may stand with no pipeline started (design 47 G §12, §112: the record's five
// minutes).
const HeadBound = 5 * time.Minute
// headRow is the signals table's row of a head with no pipeline.
const headRow = "head-pipeline"
// head is a watched branch's head still waiting for its pipeline.
type head struct{ revision int64 }
// SourceMoved hears the seat mesh-delivery's `source-moved`: the branch's head is a new source revision, and its pipeline
// is due within HeadBound. A move made by a gathered group is a member governed by the take-in's own bound (ADR 0633 §5),
// so it is not timed here (design 47 G §12). A later move of the same branch replaces the head that waits.
func (w *Watchman) SourceMoved(body []byte) {
var m meshdelivery.SourceMovedEvent
if err := json.Unmarshal(body, &m); err != nil || m.Branch == "" || m.Revision == 0 {
w.say("a source-moved does not read, left: %v", err)
return
}
w.mu.Lock()
defer w.mu.Unlock()
id := headID(m.Branch)
w.unsignal(headRow, id)
w.unfind(FindingKey(ScopeDelivery, id, KindPipelineStands))
if m.Group != "" {
delete(w.heads, m.Branch)
return
}
at := w.now()
if m.At != nil && !m.At.IsZero() {
at = *m.At
}
w.heads[m.Branch] = head{revision: m.Revision}
f := Finding{Scope: ScopeDelivery, ID: id, Kind: KindPipelineStands, Severity: Warning, Source: headRow, Since: at,
Summary: fmt.Sprintf("the head of %s (source revision %d) has started no pipeline since %s", m.Branch, m.Revision, stamp(at)),
Said: fmt.Sprintf("no pipeline in %s", HeadBound),
Headline: "A change has started no pipeline",
Explanation: "The head of the watched branch " + m.Branch + " (source revision " + fmt.Sprint(m.Revision) + ") moved " +
HeadBound.String() + " ago or more and the pipeline runner has said no pipeline started for it.",
Needs: "the runner to start the pipeline (its pipeline_stands and list tools say where it is)"}
left := HeadBound - w.now().Sub(at)
if left > 0 {
w.signal(headRow, id, left, signalDue{Do: dueFind, Finding: f})
return
}
w.find(f)
}
// PipelineStarted hears the runner's `pipeline-started`: it ends the wait of the head it is for (the branch's head at that
// source revision or a later one). A run on request is no head's pipeline.
func (w *Watchman) PipelineStarted(body []byte) {
var p meshpipelinerunner.PipelineStartedEvent
if err := json.Unmarshal(body, &p); err != nil || p.Branch == "" {
w.say("a pipeline-started does not read, left: %v", err)
return
}
if p.Run != "" {
return
}
w.mu.Lock()
defer w.mu.Unlock()
h, ok := w.heads[p.Branch]
if !ok || p.Source < h.revision {
return
}
delete(w.heads, p.Branch)
id := headID(p.Branch)
w.unsignal(headRow, id)
w.unfind(FindingKey(ScopeDelivery, id, KindPipelineStands))
}
// headID is the finding's id of a branch's head.
func headID(branch string) string { return "head-" + unsafeKey.ReplaceAllString(branch, "_") }
// KindBuildCapacityWaits is a build seat whose oldest queued step has waited the bound (design 47 G §86).
const KindBuildCapacityWaits = "build-capacity-waits"
// CapacityBound is how long a step may wait queued on a build seat's stream before the seat is a finding (design 47 G §86).
const CapacityBound = 60 * time.Minute
// capacityRow is the signals table's row of a build seat's queue.
const capacityRow = "build-capacity"
// StepEvent hears the runner's `step-started` and `step-ended` (design 47 G §86): a step said waiting for a build node is
// queued on its seat, owed no progress; it leaves the queue when it starts running or ends. A seat's finding is due when
// its oldest queued step has waited CapacityBound, and ends when that seat has none that old.
func (w *Watchman) StepEvent(body []byte) {
var e meshpipelinerunner.StepEvent
if err := json.Unmarshal(body, &e); err != nil || e.Pipeline == "" || e.Result.Step == "" {
w.say("a step event does not read, left: %v", err)
return
}
id := e.Pipeline + "/" + e.Result.Step
w.mu.Lock()
defer w.mu.Unlock()
for seat, q := range w.queued {
if _, ok := q[id]; ok && (e.Result.Outcome != meshpipelinerunner.StepResultOutcomeWaitingForABuildNode || e.Result.Seat != seat) {
delete(q, id)
w.capacity(seat)
}
}
if e.Result.Outcome != meshpipelinerunner.StepResultOutcomeWaitingForABuildNode || e.Result.Seat == "" {
return
}
since := w.now()
switch {
case e.Result.Started != nil:
since = *e.Result.Started
case e.At != nil:
since = *e.At
}
seat := e.Result.Seat
if w.queued[seat] == nil {
w.queued[seat] = map[string]time.Time{}
}
if _, ok := w.queued[seat][id]; !ok {
w.queued[seat][id] = since
}
w.capacity(seat)
}
// capacity judges a build seat's queue again, w.mu held: one signal for its oldest queued step.
func (w *Watchman) capacity(seat string) {
w.unsignal(capacityRow, seat)
key := FindingKey(ScopeDelivery, seat, KindBuildCapacityWaits)
var oldest time.Time
for _, t := range w.queued[seat] {
if oldest.IsZero() || t.Before(oldest) {
oldest = t
}
}
if oldest.IsZero() {
delete(w.queued, seat)
w.unfind(key)
return
}
f := Finding{Scope: ScopeDelivery, ID: seat, Kind: KindBuildCapacityWaits, Severity: Warning, Source: capacityRow, Since: oldest,
Summary: fmt.Sprintf("the oldest step queued on %s has waited since %s", seat, stamp(oldest)),
Said: "queued for " + CapacityBound.String(),
Headline: "Builds wait for a build node",
Explanation: "The oldest step queued on the build seat " + seat + " has waited " + CapacityBound.String() +
" for a build node to take it.",
Needs: "build capacity: a build node present and not paused, or fewer steps asked"}
left := CapacityBound - w.now().Sub(oldest)
if left > 0 {
w.unfind(key)
w.signal(capacityRow, seat, left, signalDue{Do: dueFind, Finding: f})
return
}
w.find(f)
}
@@ -0,0 +1,205 @@
package watch
import (
"encoding/json"
"testing"
"time"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshpipelinerunner"
)
func moved(branch string, rev int64, group string, at time.Time) []byte {
b, _ := json.Marshal(meshdelivery.SourceMovedEvent{Branch: branch, Revision: rev, Group: group, At: &at})
return b
}
func started(branch string, rev int64, run string) []byte {
b, _ := json.Marshal(meshpipelinerunner.PipelineStartedEvent{Branch: branch, Source: rev, Run: run, Module: "m", Pipeline: "p"})
return b
}
// design 47 G §12, §79 (test 76): a head with no pipeline within five minutes is `pipeline-stands`, cleared by the
// runner's `pipeline-started` for that head; a gathered group's member and a run on request do not.
func TestAHeadWithNoPipelineStands(t *testing.T) {
key := FindingKey(ScopeDelivery, headID("main"), KindPipelineStands)
cases := []struct {
name string
run func(w *Watchman, o *fakeOut, c *clock)
stand bool
}{
{"five minutes and no pipeline", func(w *Watchman, o *fakeOut, c *clock) {
w.SourceMoved(moved("main", 7, "", c.t))
if o.ttls[signalKey(headRow, headID("main"))] != HeadBound || len(o.findings) != 0 {
t.Fatalf("within the bound: %v %v", o.ttls, o.findings)
}
expire(w, c, signalKey(headRow, headID("main")), HeadBound)
}, true},
{"its pipeline-started clears it", func(w *Watchman, o *fakeOut, c *clock) {
w.SourceMoved(moved("main", 7, "", c.t))
expire(w, c, signalKey(headRow, headID("main")), HeadBound)
w.PipelineStarted(started("main", 7, ""))
}, false},
{"a pipeline before the bound raises none", func(w *Watchman, o *fakeOut, c *clock) {
w.SourceMoved(moved("main", 7, "", c.t))
w.PipelineStarted(started("main", 8, ""))
if _, ok := o.pulses[signalKey(headRow, headID("main"))]; ok {
t.Fatal("the pulse stands")
}
}, false},
{"a pipeline of an older head does not clear a newer head", func(w *Watchman, o *fakeOut, c *clock) {
w.SourceMoved(moved("main", 7, "", c.t))
w.PipelineStarted(started("main", 6, ""))
expire(w, c, signalKey(headRow, headID("main")), HeadBound)
}, true},
{"a gathered group's member is not timed", func(w *Watchman, o *fakeOut, c *clock) {
w.SourceMoved(moved("main", 7, "grp", c.t))
if len(o.pulses) != 0 {
t.Fatalf("a pulse for a member: %v", o.pulses)
}
}, false},
{"a run on request is no head's pipeline", func(w *Watchman, o *fakeOut, c *clock) {
w.SourceMoved(moved("main", 7, "", c.t))
w.PipelineStarted(started("main", 7, "run-1"))
expire(w, c, signalKey(headRow, headID("main")), HeadBound)
}, true},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
w, o, c := setup()
tc.run(w, o, c)
if _, ok := o.findings[key]; ok != tc.stand {
t.Fatalf("the finding stands %v, want %v: %v", ok, tc.stand, o.findings)
}
})
}
}
func queuedStep(pipeline, step, seat, outcome string, started time.Time) []byte {
b, _ := json.Marshal(meshpipelinerunner.StepEvent{Pipeline: pipeline, Branch: "main", Module: "m", Commit: "c",
Result: meshpipelinerunner.StepResult{Step: step, Seat: seat, Outcome: outcome, Started: &started}})
return b
}
// design 47 G §86: a build seat whose oldest queued step has waited 60 minutes raises one `build-capacity-waits`, cleared
// when that step starts.
func TestABuildSeatsOldestQueuedStepWaitsAnHour(t *testing.T) {
const seat = "node-build-go"
key := FindingKey(ScopeDelivery, seat, KindBuildCapacityWaits)
wait := meshpipelinerunner.StepResultOutcomeWaitingForABuildNode
w, o, c := setup()
w.StepEvent(queuedStep("p1", "unit", seat, wait, c.t))
c.t = c.t.Add(10 * time.Minute)
w.StepEvent(queuedStep("p2", "unit", seat, wait, c.t))
pk := signalKey(capacityRow, seat)
if o.ttls[pk] != 50*time.Minute || len(o.findings) != 0 {
t.Fatalf("within the bound: %v %v", o.ttls, o.findings)
}
expire(w, c, pk, 50*time.Minute)
if _, ok := o.findings[key]; !ok || len(o.findings) != 1 {
t.Fatalf("an hour for the oldest: %v", o.findings)
}
w.StepEvent(queuedStep("p1", "unit", seat, meshpipelinerunner.StepResultOutcomeRunning, c.t))
if len(o.findings) != 0 {
t.Fatalf("the oldest started, the next has waited 50 minutes: %v", o.findings)
}
if o.ttls[pk] != 10*time.Minute {
t.Fatalf("the next one's hour: %v", o.ttls[pk])
}
w.StepEvent(queuedStep("p2", "unit", seat, meshpipelinerunner.StepResultOutcomePassed, c.t))
if _, ok := o.pulses[pk]; ok || len(o.findings) != 0 {
t.Fatalf("the queue is empty: %v %v", o.pulses, o.findings)
}
}
// design 47 G §389, §436 (test 106's cause): a machine whose presence key expires says `machine-away`, once, and one
// heard again says `machine-back`; a machine heard again that was never away says nothing.
func TestAMachinesPresenceIsSaidAsEvents(t *testing.T) {
w, o, c := setup()
w.Connect("node.ace")
w.Connect("node.ace") // heard again while never away
w.Disconnect("node.ace")
w.Disconnect("node.ace")
if len(o.said) != 0 {
t.Fatalf("said before the expiry: %v", o.said)
}
c.t = t0.Add(120 * time.Second)
w.Pulse(pulseKeyOf(t, presenceNS, "ace"), Expired, c.t)
w.Pulse(pulseKeyOf(t, presenceNS, "ace"), Expired, c.t) // the same expiry heard twice
w.Connect("node.ace")
w.Connect("node.ace")
got := []string{}
for _, s := range o.said {
got = append(got, s.subject)
}
want := []string{"mesh.seat.mesh-watchdogs.event.machine-away.ace", "mesh.seat.mesh-watchdogs.event.machine-back.ace"}
if len(got) != 2 || got[0] != want[0] || got[1] != want[1] {
t.Fatalf("said %v, want %v", got, want)
}
}
func fillBody(store string, bytes int64) []byte {
b, _ := json.Marshal(meshdelivery.FillingEvent{Store: store, Bytes: bytes, Cap: 100})
return b
}
// design 47 G §181, §263: a store past its share is `store-filling` naming the part and the store, full is urgent, `room` and a
// fill under 70 % clear it.
func TestAStoreFillingIsAFinding(t *testing.T) {
w, o, _ := setup()
key := FindingKey(ScopeDelivery, "mesh-delivery-records", KindStoreFilling)
w.StoreFill("mesh-delivery", "filling", fillBody("records", 60))
if len(o.findings) != 0 {
t.Fatalf("under 70 %%: %v", o.findings)
}
w.StoreFill("mesh-delivery", "filling", fillBody("records", 85))
if o.findings[key].Severity != Warning {
t.Fatalf("filling: %v", o.findings)
}
w.StoreFill("mesh-delivery", "full", fillBody("records", 100))
if o.findings[key].Severity != Urgent {
t.Fatalf("full: %v", o.findings)
}
w.StoreFill("mesh-delivery", "room", fillBody("records", 60))
if len(o.findings) != 0 {
t.Fatalf("room: %v", o.findings)
}
}
func recordBody(id string, at time.Time) []byte {
b, _ := json.Marshal(meshdelivery.RecordOpenedEvent{Record: id, Milestone: "47/G", At: &at})
return b
}
// design 47 G §148, §160: a record open 24 hours is `record-open`, cleared by `record-ended`; a record that ends in time
// raises none; while a store is full, which already tells why, it raises none.
func TestARecordOpenADayIsAFinding(t *testing.T) {
w, o, c := setup()
pk := signalKey(recordRow, "r1")
w.RecordOpen(recordBody("r1", c.t))
if o.ttls[pk] != RecordOpenBound || len(o.findings) != 0 {
t.Fatalf("within the bound: %v %v", o.ttls, o.findings)
}
expire(w, c, pk, RecordOpenBound)
key := FindingKey(ScopeDelivery, "r1", KindRecordOpen)
if _, ok := o.findings[key]; !ok {
t.Fatalf("past 24 hours: %v", o.findings)
}
w.StoreFill("mesh-delivery", "full", fillBody("records", 100))
if _, ok := o.findings[key]; ok {
t.Fatalf("a full store covers the record: %v", o.findings)
}
w.StoreFill("mesh-delivery", "room", fillBody("records", 10))
if _, ok := o.findings[key]; !ok {
t.Fatalf("room, and the record stands open again: %v", o.findings)
}
w.RecordEnded(recordBody("r1", c.t))
if len(o.findings) != 0 {
t.Fatalf("record-ended: %v", o.findings)
}
w.RecordOpen(recordBody("r2", c.t))
w.RecordEnded(recordBody("r2", c.t))
if _, ok := o.pulses[signalKey(recordRow, "r2")]; ok {
t.Fatal("the pulse of an ended record stands")
}
}
@@ -8,6 +8,7 @@ import (
"time"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshwatchdogsclient"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshwatchdogs"
)
// partPrefix is what the watchman keeps presence of among the module users: the parts of delivery.
@@ -186,3 +187,39 @@ func (w *Watchman) Rebuild(ask func(body []byte) ([]byte, error)) error {
w.Connections(users)
return nil
}
// machineAway says `machine-away` of the seat mesh-watchdogs when a machine's presence key has expired (design 47 G §389, §436):
// the machine, when it was last heard and when this was said, once until the machine is heard again.
func (w *Watchman) machineAway(node string, lastHeard time.Time) {
if _, said := w.awaySaid[node]; said {
return
}
at := w.now().UTC()
body, err := json.Marshal(meshwatchdogs.MachineAwayEvent{V: 1, Machine: node, At: at, LastHeard: lastHeard.UTC()})
if err != nil {
return
}
if err := w.out.Say(meshwatchdogs.SubjectMachineAway(node), body, "machine-away."+node+"."+at.Format(time.RFC3339Nano)); err != nil {
w.say("cannot say machine-away of %s: %v", node, err)
return
}
w.awaySaid[node] = lastHeard.UTC()
}
// machineBack says `machine-back` when a machine whose machine-away was said is heard again (design 47 G §389).
func (w *Watchman) machineBack(node string) {
lastHeard, said := w.awaySaid[node]
if !said {
return
}
at := w.now().UTC()
body, err := json.Marshal(meshwatchdogs.MachineBackEvent{V: 1, Machine: node, At: at, LastHeard: lastHeard})
if err != nil {
return
}
if err := w.out.Say(meshwatchdogs.SubjectMachineBack(node), body, "machine-back."+node+"."+at.Format(time.RFC3339Nano)); err != nil {
w.say("cannot say machine-back of %s: %v", node, err)
return
}
delete(w.awaySaid, node)
}
@@ -0,0 +1,149 @@
package watch
import (
"encoding/json"
"fmt"
"time"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery"
)
// What the seats say of the delivery's records and stores, heard as the seats' events (design 47 G §148, §181, §218; ADR 0596
// §2), never read from their state:
// - `filling`, `full` and `room` of a store against its cap: `store-filling` while it stands past its cap's share, urgent
// when it is full, cleared by `room` or by a `filling` under FillingFrom;
// - `record-opened` and `record-ended` of a delivery record: `record-open` when it has stood open for 24 hours, unless
// another condition already tells why (`acting-part-late`, or a store `full`).
// Each is a row of the signals table: its pulse is its bound, written at the event, so a restart loses none.
// The kinds of these findings.
const (
KindActingPartLate = "acting-part-late"
KindStoreFilling = "store-filling"
KindRecordOpen = "record-open"
)
// The rows of the signals table of these findings, which a restart keeps.
const (
storeRow = "store-fill"
recordRow = "record-open"
)
// uncovered is the While of a record-open signal: the finding is written at the bound only while no other condition covers
// the record.
const uncovered = "uncovered:record-open"
// RecordOpenBound is how long a delivery record may stay open before it is a finding (design 47 G §148).
const RecordOpenBound = 24 * time.Hour
// FillingFrom is the share of a store's cap under which a `filling` clears its finding (design 47 G §181).
const FillingFrom = 0.70
// StoreFill hears a seat's `filling`, `full` or `room` of one of its stores. A `filling` is a warning, naming the part and the
// store, unless its share is under FillingFrom, which clears it; `full` is urgent, naming what it refuses; `room` clears either.
func (w *Watchman) StoreFill(seat, event string, body []byte) {
var f meshdelivery.FillingEvent
if err := json.Unmarshal(body, &f); err != nil || f.Store == "" {
w.say("the %s of the seat %s does not read, left: %v", event, seat, err)
return
}
id := seat + "-" + f.Store
key := FindingKey(ScopeDelivery, id, KindStoreFilling)
w.mu.Lock()
defer w.mu.Unlock()
share := 0.0
if f.Cap > 0 {
share = float64(f.Bytes) / float64(f.Cap)
}
since := w.now()
if f.At != nil && !f.At.IsZero() {
since = *f.At
}
switch {
case event == "room" || (event == "filling" && share < FillingFrom):
w.unfind(key)
case event == "full":
w.find(Finding{Scope: ScopeDelivery, ID: id, Kind: KindStoreFilling, Severity: Urgent, Source: storeRow, Since: since,
Summary: fmt.Sprintf("%s's store %s is full (%d of %d bytes)", seat, f.Store, f.Bytes, f.Cap),
Said: "full", Headline: "A store of the delivery is full",
Explanation: "The store " + f.Store + " of the seat " + seat + " is at its cap, so it refuses " + orWords(f.Refused, "what it is asked to keep") +
" until room is made.",
Needs: "room made in the store (a removal, or its cap raised)"})
default:
w.find(Finding{Scope: ScopeDelivery, ID: id, Kind: KindStoreFilling, Severity: Warning, Source: storeRow, Since: since,
Summary: fmt.Sprintf("%s's store %s is at %.0f %% of its cap (%d of %d bytes)", seat, f.Store, share*100, f.Bytes, f.Cap),
Said: "filling", Headline: "A store of the delivery is filling",
Explanation: "The store " + f.Store + " of the seat " + seat + " is nearly full; once it is full it refuses new entries.",
Needs: "room made before it is full"})
}
w.rejudgeRecords()
}
// RecordOpen hears the seat mesh-delivery's `record-opened`: its finding is due when the record has stood open for 24 hours.
func (w *Watchman) RecordOpen(body []byte) {
var r meshdelivery.RecordOpenedEvent
if err := json.Unmarshal(body, &r); err != nil || r.Record == "" {
w.say("a record-opened does not read, left: %v", err)
return
}
w.mu.Lock()
defer w.mu.Unlock()
since := w.now()
if r.At != nil && !r.At.IsZero() {
since = *r.At
}
f := Finding{Scope: ScopeDelivery, ID: r.Record, Kind: KindRecordOpen, Severity: Warning, Source: recordRow, Since: since,
Summary: fmt.Sprintf("the delivery record of %s has stood open since %s", r.Milestone, stamp(since)),
Said: "open for " + RecordOpenBound.String(), Headline: "A delivery is not on every machine yet",
Explanation: "The delivery record of " + r.Milestone + " has stood open for " + RecordOpenBound.String() +
": a node does not run what it changed, or a step never answered.",
Needs: "what it waits for (gate_record names it), or the record ended by hand if a node is gone for good (gate_record_end)"}
w.unsignal(recordRow, r.Record)
w.unfind(f.Key())
if left := RecordOpenBound - w.now().Sub(since); left > 0 {
w.signal(recordRow, r.Record, left, signalDue{Do: dueFind, While: uncovered, Finding: f})
return
}
w.opened[r.Record] = f
w.rejudgeRecords()
}
// RecordEnded hears the seat mesh-delivery's `record-ended`: its pulse and its finding end.
func (w *Watchman) RecordEnded(body []byte) {
var r meshdelivery.RecordEndedEvent
if err := json.Unmarshal(body, &r); err != nil || r.Record == "" {
w.say("a record-ended does not read, left: %v", err)
return
}
w.mu.Lock()
defer w.mu.Unlock()
w.unsignal(recordRow, r.Record)
delete(w.opened, r.Record)
w.unfind(FindingKey(ScopeDelivery, r.Record, KindRecordOpen))
}
// recordCovered says another condition tells the operator why a record is open: a wait past its bound or a full store.
func (w *Watchman) recordCovered() bool {
for _, v := range w.findings {
f, err := DecodeFinding(v)
if err != nil {
continue
}
if f.Kind == KindActingPartLate || (f.Kind == KindStoreFilling && f.Severity == Urgent) {
return true
}
}
return false
}
// rejudgeRecords writes the finding of each record past its bound that nothing covers, and ends it when something does.
func (w *Watchman) rejudgeRecords() {
for id, f := range w.opened {
if w.recordCovered() {
w.unfind(f.Key())
continue
}
w.find(f)
_ = id
}
}
@@ -171,6 +171,11 @@ func (w *Watchman) signalExpired(pk string) {
case dueEnd:
w.unfind(due.Finding.Key())
case dueFind:
if due.While == uncovered {
w.opened[due.Finding.ID] = due.Finding
w.rejudgeRecords()
return
}
if due.While != "" && !w.stillWrong(due.While, due.Finding.Kind) {
return
}
@@ -181,6 +186,9 @@ func (w *Watchman) signalExpired(pk string) {
// stillWrong says a have's key still says what a row's finding was waited on: failed for apply-failing, unhealthy for
// unhealthy.
func (w *Watchman) stillWrong(haveKey, kind string) bool {
if haveKey == uncovered {
return !w.recordCovered()
}
if node, ok := strings.CutPrefix(haveKey, "silent:"); ok {
_, open := w.findings[FindingKey(ScopeMachine, node, KindSilent)]
return open
@@ -254,6 +262,11 @@ func (w *Watchman) stuckFinding(sk string, s meshwatchdogsclient.Stuck) {
if s.Bucket == PresenceBucket {
if node, part, _, err := meshwatchdogsclient.ParsePresenceKey(s.Key); err == nil && part == "" && node != "" {
key := FindingKey(ScopeMachine, node, KindSilent)
if s.Moved {
w.machineBack(node)
} else {
w.machineAway(node, s.Since)
}
defer w.networks() // a silent machine is down on the record: what points at it is held there, said by its silence
if s.Moved || w.away[node] {
w.unfind(key)
@@ -931,6 +944,8 @@ var SignalRows = []struct{ Row, Signal, Bound, Kind string }{
{"S21", "a machine's have failed or its provisioner client's files not placed", ApplyEvery.String(), KindApplyFailing},
{"health", "a have unhealthy in two writes in a row; urgent after " + UnhealthyUrgentAfter.String(), "two writes", KindUnhealthy},
{"take-in", "a source revision or a delivery group the indexer holds back (ADR 0633 §5)", TakeInRevisionBound.String() + " a revision, " + TakeInGroupBound.String() + " a group", KindTakeInWaits},
{storeRow, "a store of the delivery's parts filling or full (the seats' filling, full and room)", "filling from " + fmt.Sprintf("%.0f", FillingFrom*100) + " %, full urgent", KindStoreFilling},
{recordRow, "a delivery record open (the seat mesh-delivery's record-opened, ended by record-ended)", RecordOpenBound.String(), KindRecordOpen},
{"provisions", "a provision of mesh-provisioner in waits (ADR 0570 §7)", "at once", KindNeedsOperator},
}
@@ -36,6 +36,8 @@ type Out interface {
// PutFinding writes one of the watchman's findings; DeleteFinding deletes one it no longer observes.
PutFinding(key string, value []byte) error
DeleteFinding(key string) error
// Say publishes an event of the seat mesh-watchdogs on its subject (design 47 G §389), once under its id.
Say(subject string, body []byte, id string) error
}
type presence struct {
@@ -112,6 +114,14 @@ type Watchman struct {
// pulses, stuck keys and findings wait for its own end (ReadLate), so a restart while its owner is down neither drops
// nor raises again what it holds.
awaited map[string]bool
// heads is, by watched branch, the head still waiting for its pipeline (pipelines.go).
heads map[string]head
// queued is, by build seat, each step queued on it for a build node and since when (pipelines.go).
queued map[string]map[string]time.Time
// awaySaid is each machine whose machine-away was said and not yet its machine-back, with when it was last heard (presence.go).
awaySaid map[string]time.Time
// opened is each record past its bound and not ended, with its finding (records.go).
opened map[string]Finding
}
// New is a watchman writing to out, reading the time from now.
@@ -122,7 +132,7 @@ func New(out Out, now func() time.Time) *Watchman {
haveState: map[string]nodeengine.HaveByNodeModuleValue{}, unhealthy: map[string]int{}, advised: map[string]int{}, advisedSince: map[string]time.Time{},
control: map[string]bool{}, failedSince: map[string]time.Time{}, provFailed: map[string]map[string]ProvReport{}, closures: map[string]map[string]time.Time{}, waiting: map[string]map[string]string{},
health: map[string]nodeengine.HealthByNodeValue{}, tunnels: map[string]string{},
standing: map[string]map[string]bool{}, readWhole: map[string]bool{}, awaited: map[string]bool{}}
standing: map[string]map[string]bool{}, readWhole: map[string]bool{}, awaited: map[string]bool{}, heads: map[string]head{}, queued: map[string]map[string]time.Time{}, awaySaid: map[string]time.Time{}, opened: map[string]Finding{}}
}
// Logf sets where a write that failed is said; it is silent until set.
@@ -26,6 +26,14 @@ type fakeOut struct {
stuckWrites int
findings map[string]Finding
findingWrites int
said []said
}
type said struct{ subject, id string }
func (o *fakeOut) Say(subject string, _ []byte, id string) error {
o.said = append(o.said, said{subject, id})
return nil
}
func newOut() *fakeOut {
@@ -7,6 +7,7 @@ import (
"os"
"path/filepath"
"reflect"
"regexp"
"slices"
"strings"
"testing"
@@ -98,6 +99,10 @@ func TestManifestStateAndReads(t *testing.T) { // 1135/AC6
"mesh-delivery-builder.builds", "mesh-host.health", "mesh-provisioner.provisions", "node-engine.have"}) {
t.Errorf("reads = %v", reads)
}
// Design 47 G §368: the version is MAJOR.MINOR and nothing else.
if !regexp.MustCompile(`^[0-9]+\.[0-9]+$`).MatchString(m.Version) {
t.Errorf("version = %q, want MAJOR.MINOR", m.Version)
}
// DLV-M5 (ADR 0615 §1): the assigner's wants and the indexer's modules are their owners' states, asked by their read
// tools and followed by their views families, never read.
for _, r := range m.Reads {
@@ -111,7 +116,7 @@ func TestManifestStateAndReads(t *testing.T) { // 1135/AC6
t.Errorf("invokes = %v, want %v", m.Invokes, want)
}
// DLV-M5 its first views, 1585/T20 the indexer's take-in events, and DLV-W1 (ADR 0595 §4): the accepted verdicts and the bus module's created and refused, durably.
if want := []string{"mesh-delivery-assigner.views.wants", "mesh-delivery-indexer.take-in.pending", "mesh-delivery-indexer.take-in.taken", "mesh-delivery-indexer.views.modules", "mesh-catalog.manifest-accepted", "mesh-broker.created", "mesh-broker.refused"}; !reflect.DeepEqual(m.Consumes, want) {
if want := []string{"mesh-delivery-assigner.views.wants", "mesh-delivery-indexer.take-in.pending", "mesh-delivery-indexer.take-in.taken", "mesh-delivery-indexer.views.modules", "mesh-catalog.manifest-accepted", "mesh-delivery.source-moved", "mesh-delivery.filling", "mesh-delivery.full", "mesh-delivery.room", "mesh-delivery.record-opened", "mesh-delivery.record-ended", "mesh-catalog.filling", "mesh-catalog.full", "mesh-catalog.room", "mesh-wants.filling", "mesh-wants.full", "mesh-wants.room", "mesh-grants.filling", "mesh-grants.full", "mesh-grants.room", "mesh-pipeline-runner.pipeline-started", "mesh-pipeline-runner.step-started", "mesh-pipeline-runner.step-ended", "mesh-broker.created", "mesh-broker.refused"}; !reflect.DeepEqual(m.Consumes, want) {
t.Errorf("consumes = %v, want %v", m.Consumes, want)
}
// 1952/WM1: the watchman holds the seat mesh-watchdogs at mesh scope, its first seat (novox/hq ADR 0564 §1, §4).
+19 -1
View File
@@ -1,6 +1,6 @@
{
"module": "mesh-delivery-watchman",
"version": "1",
"version": "0.1",
"slug": "watchman",
"claims": [
{
@@ -36,6 +36,24 @@
"mesh-delivery-indexer.take-in.taken",
"mesh-delivery-indexer.views.modules",
"mesh-catalog.manifest-accepted",
"mesh-delivery.source-moved",
"mesh-delivery.filling",
"mesh-delivery.full",
"mesh-delivery.room",
"mesh-delivery.record-opened",
"mesh-delivery.record-ended",
"mesh-catalog.filling",
"mesh-catalog.full",
"mesh-catalog.room",
"mesh-wants.filling",
"mesh-wants.full",
"mesh-wants.room",
"mesh-grants.filling",
"mesh-grants.full",
"mesh-grants.room",
"mesh-pipeline-runner.pipeline-started",
"mesh-pipeline-runner.step-started",
"mesh-pipeline-runner.step-ended",
"mesh-broker.created",
"mesh-broker.refused"
],