record-opened is joined to the take-in.taken of its milestone, <branch>.<source revision>, whose entries name the modules; the record-open finding is left unwritten only while one of those modules is held by an acting-part-late wait (or a store says full), no longer while any wait is late. A record whose revision is not known taken is covered by a standing take-in-waits. Builds: design 47 G §502, A-200 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h
166 lines
6.6 KiB
Go
166 lines
6.6 KiB
Go
package watch
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"strings"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
|
|
)
|
|
|
|
// What the seats say of the delivery's records and stores, heard as the seats' events (design 47 G §148, §181, §218; ADR 0596
|
|
// §2), never read from their state:
|
|
// - `filling`, `full` and `room` of a store against its cap: `store-filling` while it stands past its cap's share, urgent
|
|
// when it is full, cleared by `room` or by a `filling` under FillingFrom;
|
|
// - `record-opened` and `record-ended` of a delivery record: `record-open` when it has stood open for 24 hours, unless
|
|
// another condition already tells why (`acting-part-late`, or a store `full`).
|
|
// Each is a row of the signals table: its pulse is its bound, written at the event, so a restart loses none.
|
|
|
|
// The kinds of these findings.
|
|
const (
|
|
KindActingPartLate = "acting-part-late"
|
|
KindPutStands = "put-stands"
|
|
KindStoreFilling = "store-filling"
|
|
KindRecordOpen = "record-open"
|
|
)
|
|
|
|
// The rows of the signals table of these findings, which a restart keeps.
|
|
const (
|
|
storeRow = "store-fill"
|
|
recordRow = "record-open"
|
|
)
|
|
|
|
// uncovered is the While of a record-open signal: the finding is written at the bound only while no other condition covers
|
|
// the record.
|
|
const uncovered = "uncovered:record-open"
|
|
|
|
// RecordOpenBound is how long a delivery record may stay open before it is a finding (design 47 G §148).
|
|
const RecordOpenBound = 24 * time.Hour
|
|
|
|
// FillingFrom is the share of a store's cap under which a `filling` clears its finding (design 47 G §181).
|
|
const FillingFrom = 0.80
|
|
|
|
// StoreFill hears a seat's `filling`, `full` or `room` of one of its stores. A `filling` is a warning, naming the part and the
|
|
// store, unless its share is under FillingFrom, which clears it; `full` is urgent, naming what it refuses; `room` clears either.
|
|
func (w *Watchman) StoreFill(seat, event string, body []byte) {
|
|
var f meshdelivery.FillingEvent
|
|
if err := json.Unmarshal(body, &f); err != nil || f.Store == "" {
|
|
w.say("the %s of the seat %s does not read, left: %v", event, seat, err)
|
|
return
|
|
}
|
|
id := seat + "-" + f.Store
|
|
key := FindingKey(ScopeDelivery, id, KindStoreFilling)
|
|
w.mu.Lock()
|
|
defer w.mu.Unlock()
|
|
share := 0.0
|
|
if f.Cap > 0 {
|
|
share = float64(f.Bytes) / float64(f.Cap)
|
|
}
|
|
since := w.now()
|
|
if f.At != nil && !f.At.IsZero() {
|
|
since = *f.At
|
|
}
|
|
switch {
|
|
case event == "room" || (event == "filling" && share < FillingFrom):
|
|
w.unfind(key)
|
|
case event == "full":
|
|
w.find(Finding{Scope: ScopeDelivery, ID: id, Kind: KindStoreFilling, Severity: Urgent, Source: storeRow, Since: since,
|
|
Summary: fmt.Sprintf("%s's store %s is full (%d of %d bytes)", seat, f.Store, f.Bytes, f.Cap),
|
|
Said: "full", Headline: "A store of the delivery is full",
|
|
Explanation: "The store " + f.Store + " of the seat " + seat + " is at its cap, so it refuses " + orWords(f.Refused, "what it is asked to keep") +
|
|
" until room is made.",
|
|
Needs: "room made in the store (a removal, or its cap raised)"})
|
|
default:
|
|
w.find(Finding{Scope: ScopeDelivery, ID: id, Kind: KindStoreFilling, Severity: Warning, Source: storeRow, Since: since,
|
|
Summary: fmt.Sprintf("%s's store %s is at %.0f %% of its cap (%d of %d bytes)", seat, f.Store, share*100, f.Bytes, f.Cap),
|
|
Said: "filling", Headline: "A store of the delivery is filling",
|
|
Explanation: "The store " + f.Store + " of the seat " + seat + " is nearly full; once it is full it refuses new entries.",
|
|
Needs: "room made before it is full"})
|
|
}
|
|
w.rejudgeRecords()
|
|
}
|
|
|
|
// RecordOpen hears the seat mesh-delivery's `record-opened`: its finding is due when the record has stood open for 24 hours.
|
|
func (w *Watchman) RecordOpen(body []byte) {
|
|
var r meshdelivery.RecordOpenedEvent
|
|
if err := json.Unmarshal(body, &r); err != nil || r.Record == "" {
|
|
w.say("a record-opened does not read, left: %v", err)
|
|
return
|
|
}
|
|
w.mu.Lock()
|
|
defer w.mu.Unlock()
|
|
since := w.now()
|
|
if r.At != nil && !r.At.IsZero() {
|
|
since = *r.At
|
|
}
|
|
w.recordMilestone[r.Record] = r.Milestone
|
|
f := Finding{Scope: ScopeDelivery, ID: r.Record, Kind: KindRecordOpen, Severity: Warning, Source: recordRow, Since: since,
|
|
Summary: fmt.Sprintf("the delivery record of %s has stood open since %s", r.Milestone, stamp(since)),
|
|
Said: "open for " + RecordOpenBound.String(), Headline: "A delivery is not on every machine yet",
|
|
Explanation: "The delivery record of " + r.Milestone + " has stood open for " + RecordOpenBound.String() +
|
|
": a node does not run what it changed, or a step never answered.",
|
|
Needs: "what it waits for (gate_record names it), or the record ended by hand if a node is gone for good (gate_record_end)"}
|
|
w.unsignal(recordRow, r.Record)
|
|
w.unfind(f.Key())
|
|
if left := RecordOpenBound - w.now().Sub(since); left > 0 {
|
|
w.signal(recordRow, r.Record, left, signalDue{Do: dueFind, While: uncovered, Finding: f})
|
|
return
|
|
}
|
|
w.opened[r.Record] = f
|
|
w.rejudgeRecords()
|
|
}
|
|
|
|
// RecordEnded hears the seat mesh-delivery's `record-ended`: its pulse and its finding end.
|
|
func (w *Watchman) RecordEnded(body []byte) {
|
|
var r meshdelivery.RecordEndedEvent
|
|
if err := json.Unmarshal(body, &r); err != nil || r.Record == "" {
|
|
w.say("a record-ended does not read, left: %v", err)
|
|
return
|
|
}
|
|
w.mu.Lock()
|
|
defer w.mu.Unlock()
|
|
w.unsignal(recordRow, r.Record)
|
|
delete(w.opened, r.Record)
|
|
delete(w.recordMilestone, r.Record)
|
|
w.unfind(FindingKey(ScopeDelivery, r.Record, KindRecordOpen))
|
|
}
|
|
|
|
// recordCovered says another condition tells the operator why a record is open (design 47 G §502, A-200): the record is tied to the
|
|
// entries of its source revision's `take-in.taken` by its milestone, `<branch>.<source revision>`, and is covered while one of those
|
|
// entries is held by a wait that raised `acting-part-late`, or a store says `full`; a record whose revision is not known to be taken in
|
|
// is covered by a standing `take-in-waits`.
|
|
func (w *Watchman) recordCovered(record string) bool {
|
|
entries, taken := w.takenEntries[w.recordMilestone[record]]
|
|
for _, v := range w.findings {
|
|
f, err := DecodeFinding(v)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
switch {
|
|
case f.Kind == KindStoreFilling && f.Severity == Urgent:
|
|
return true
|
|
case f.Kind == KindActingPartLate && taken:
|
|
for _, module := range entries {
|
|
if strings.HasSuffix(f.ID, "-"+module) {
|
|
return true
|
|
}
|
|
}
|
|
case f.Kind == KindTakeInWaits && !taken:
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// rejudgeRecords writes the finding of each record past its bound that nothing covers, and ends it when something does.
|
|
func (w *Watchman) rejudgeRecords() {
|
|
for id, f := range w.opened {
|
|
if w.recordCovered(id) {
|
|
w.unfind(f.Key())
|
|
continue
|
|
}
|
|
w.find(f)
|
|
}
|
|
}
|