A check that failed for a part of the mesh not heard was judged again at every change of the nodes. It now says could not run, naming the part, and is kept in the checker's own state could-not-run; only the running of that part asks it again, once per heads and part at one fingerprint (PartRunning). Not asked again within an hour it is the finding check-could-not-run, which clears when the heads are checked again, replaced, or their pull request closes. checker_status reads the state. The node-engine seat's running event is wired once its definition lands. Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ Builds: design 47 G §253, §315
247 lines
9.4 KiB
Go
247 lines
9.4 KiB
Go
package check
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"sort"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliverycheckerclient"
|
|
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdeliverychecker"
|
|
"git.novox.be/novox/mesh-sdk/go/views"
|
|
)
|
|
|
|
// A check that failed because a part of the mesh was not heard says `could not run`, naming the part, and releases its
|
|
// claim on the verdict key (design 47, *A check runs to its end*; issue 1373). It is asked again once per set of heads and
|
|
// per part returning, on the one event that can change its outcome: the node-engine seat's `running.<node>.<module>`
|
|
// naming that part (PartRunning), never on a timer; a `running` of the same part at the same fingerprint after that asks
|
|
// nothing. One not asked again within CouldNotRunBound becomes the checker's finding CouldNotRunKind, which clears when the
|
|
// heads are checked again, replaced, or their pull request closes.
|
|
//
|
|
// What the checker knows of each such check is its own state could-not-run (CouldNotRunState), one key per verdict key,
|
|
// read by checker_status (could_not_run): kept on the bus, so a checker started again asks each one again on the next
|
|
// `running` and raises its finding at the same time.
|
|
|
|
// CouldNotRunState is the checker's own state of its checks that could not run, by verdict key.
|
|
const CouldNotRunState = "could-not-run"
|
|
|
|
// CouldNotRunBucket is the bucket of CouldNotRunState.
|
|
const CouldNotRunBucket = CheckerModule + "_" + CouldNotRunState
|
|
|
|
// CouldNotRunKind is the checker's finding of a check that could not run and was not asked again within CouldNotRunBound.
|
|
const CouldNotRunKind = "check-could-not-run"
|
|
|
|
// CouldNotRunBound is how long a check that could not run waits to be asked again before it is a finding: the check's own
|
|
// bound of an hour.
|
|
const CouldNotRunBound = time.Hour
|
|
|
|
// Part is a part of the mesh a check could not hear: the node it runs on and its module, as the node-engine seat's
|
|
// `running.<node>.<module>` names it. An empty field is not known: an empty Module is any part on Node, and an empty Node is
|
|
// Module wherever it runs; a Part with neither is a part the check could not name, asked again on the first `running`.
|
|
type Part struct {
|
|
Node string `json:"node,omitempty"`
|
|
Module string `json:"module,omitempty"`
|
|
}
|
|
|
|
// String names the part as a verdict says it.
|
|
func (p Part) String() string {
|
|
switch {
|
|
case p.Node != "" && p.Module != "":
|
|
return p.Module + " on " + p.Node
|
|
case p.Module != "":
|
|
return p.Module
|
|
case p.Node != "":
|
|
return "the node " + p.Node
|
|
}
|
|
return "a part of the mesh"
|
|
}
|
|
|
|
// matches says whether a `running` of module on node is the part's return.
|
|
func (p Part) matches(node, module string) bool {
|
|
return (p.Node == "" || p.Node == node) && (p.Module == "" || p.Module == module)
|
|
}
|
|
|
|
// CouldNotRun is one check that could not run, as the state could-not-run keeps it.
|
|
type CouldNotRun struct {
|
|
V int `json:"v"`
|
|
Group string `json:"group"`
|
|
Heads string `json:"heads"`
|
|
Part Part `json:"part"`
|
|
Words string `json:"words"`
|
|
// Since is when its verdict was written; CouldNotRunBound after it, not asked again, it is a finding.
|
|
Since time.Time `json:"since"`
|
|
// AskedAt is the `running` that asked the check of these heads again, `<node>.<module>@<fingerprint>`: a `running` of
|
|
// the same part at the same fingerprint asks nothing more.
|
|
AskedAt string `json:"asked-at,omitempty"`
|
|
// Raised says its finding is written.
|
|
Raised bool `json:"raised,omitempty"`
|
|
}
|
|
|
|
// IsCouldNotRun says whether a verdict failed because a part of the mesh the check reads was not heard (UnheardWords): its
|
|
// claim is released, and it is asked again when that part runs (PartRunning).
|
|
func IsCouldNotRun(v meshdeliverychecker.VerdictsByGroupHeadsValue) bool {
|
|
return v.State == meshdeliverycheckerclient.VerdictFail && containsWords(failedOf(v).Words, UnheardWords)
|
|
}
|
|
|
|
// running is how a `running` event is kept in AskedAt and in the release of a check.
|
|
func runningAt(node, module, fingerprint string) string {
|
|
return node + "." + module + "@" + fingerprint
|
|
}
|
|
|
|
// PartRunning is the node-engine seat's `running.<node>.<module>` at fingerprint: every check that could not run for want
|
|
// of that part, and was not asked again by the same part at the same fingerprint, is released and asked again once.
|
|
func (ch *Checker) PartRunning(ctx context.Context, node, module, fingerprint string) {
|
|
ch.init()
|
|
if ch.CouldNotRun == nil {
|
|
return
|
|
}
|
|
keys, err := ch.CouldNotRun.Keys(ctx)
|
|
if err != nil {
|
|
ch.logf("the checks that could not run cannot be listed, so the running of %s on %s asks none again: %v", module, node, err)
|
|
return
|
|
}
|
|
at := runningAt(node, module, fingerprint)
|
|
released := 0
|
|
for _, key := range keys {
|
|
r, _, ok := ch.couldNotRunOf(ctx, key)
|
|
if !ok || !r.Part.matches(node, module) || r.AskedAt == at {
|
|
continue
|
|
}
|
|
ch.mu.Lock()
|
|
ch.release[key] = at
|
|
ch.mu.Unlock()
|
|
released++
|
|
ch.logf("%s: %s runs (%s), which the check could not hear: asked again", key, module, at)
|
|
}
|
|
if released > 0 {
|
|
ch.Reconsider(ctx)
|
|
}
|
|
}
|
|
|
|
// couldNotRunOf reads one key of the state could-not-run.
|
|
func (ch *Checker) couldNotRunOf(ctx context.Context, key string) (CouldNotRun, uint64, bool) {
|
|
raw, rev, found, err := ch.CouldNotRun.Get(ctx, key)
|
|
if err != nil || !found || len(raw) == 0 {
|
|
return CouldNotRun{}, 0, false
|
|
}
|
|
var r CouldNotRun
|
|
if json.Unmarshal(raw, &r) != nil {
|
|
return CouldNotRun{}, 0, false
|
|
}
|
|
return r, rev, true
|
|
}
|
|
|
|
// takeRelease answers the `running` that released the check of key, once, and forgets it.
|
|
func (ch *Checker) takeRelease(key string) (string, bool) {
|
|
ch.mu.Lock()
|
|
defer ch.mu.Unlock()
|
|
at, ok := ch.release[key]
|
|
delete(ch.release, key)
|
|
return at, ok
|
|
}
|
|
|
|
// keepCouldNotRun writes the check's record after a verdict that could not run, asked again by askedAt ("" for none).
|
|
func (ch *Checker) keepCouldNotRun(ctx context.Context, cs *Case, v meshdeliverychecker.VerdictsByGroupHeadsValue, part Part, askedAt string) {
|
|
if ch.CouldNotRun == nil {
|
|
return
|
|
}
|
|
r := CouldNotRun{V: 1, Group: cs.Group, Heads: cs.Heads, Part: part, Words: failedOf(v).Words, Since: ch.now().UTC(), AskedAt: askedAt}
|
|
data, err := json.Marshal(r)
|
|
if err != nil {
|
|
return
|
|
}
|
|
_, rev, found, err := ch.CouldNotRun.Get(ctx, cs.Key)
|
|
if err == nil && found {
|
|
_, err = ch.CouldNotRun.Update(ctx, cs.Key, data, rev)
|
|
} else if err == nil {
|
|
_, err = ch.CouldNotRun.Create(ctx, cs.Key, data)
|
|
}
|
|
if err != nil {
|
|
ch.logf("%s: could not run, and that cannot be kept, so no running of %s asks it again until it is checked again: %v", cs.Key, part, err)
|
|
return
|
|
}
|
|
if ch.CouldNotRunKept != nil {
|
|
ch.CouldNotRunKept()
|
|
}
|
|
}
|
|
|
|
// clearCouldNotRun forgets the check of key that could not run, and clears its finding: its heads are checked again,
|
|
// replaced, or their pull request closed.
|
|
func (ch *Checker) clearCouldNotRun(ctx context.Context, key string) {
|
|
if ch.CouldNotRun == nil {
|
|
return
|
|
}
|
|
r, _, ok := ch.couldNotRunOf(ctx, key)
|
|
if !ok {
|
|
return
|
|
}
|
|
if d, can := ch.CouldNotRun.(Deleter); can {
|
|
if err := d.Delete(ctx, key); err != nil {
|
|
ch.logf("%s: the record that it could not run cannot be deleted: %v", key, err)
|
|
return
|
|
}
|
|
}
|
|
if r.Raised && ch.Findings != nil {
|
|
if err := ch.Findings.Delete(couldNotRunFinding(key, r, ch.now())); err != nil {
|
|
ch.logf("%s: its finding %s cannot be cleared: %v", key, CouldNotRunKind, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// CouldNotRunDue raises the finding of every check that could not run and was not asked again within CouldNotRunBound by
|
|
// now, and answers when the next one is due (the zero time when none waits). The checker's process waits for that time;
|
|
// a test gives now.
|
|
func (ch *Checker) CouldNotRunDue(ctx context.Context, now time.Time) time.Time {
|
|
if ch.CouldNotRun == nil {
|
|
return time.Time{}
|
|
}
|
|
keys, err := ch.CouldNotRun.Keys(ctx)
|
|
if err != nil {
|
|
ch.logf("the checks that could not run cannot be listed, so none is raised as %s now: %v", CouldNotRunKind, err)
|
|
return time.Time{}
|
|
}
|
|
sort.Strings(keys)
|
|
var next time.Time
|
|
for _, key := range keys {
|
|
r, rev, ok := ch.couldNotRunOf(ctx, key)
|
|
if !ok || r.Raised {
|
|
continue
|
|
}
|
|
due := r.Since.Add(CouldNotRunBound)
|
|
if now.Before(due) {
|
|
if next.IsZero() || due.Before(next) {
|
|
next = due
|
|
}
|
|
continue
|
|
}
|
|
if ch.Findings == nil {
|
|
continue
|
|
}
|
|
if err := ch.Findings.Put(couldNotRunFinding(key, r, now)); err != nil {
|
|
ch.logf("%s: its finding %s cannot be written: %v", key, CouldNotRunKind, err)
|
|
continue
|
|
}
|
|
r.Raised = true
|
|
if data, err := json.Marshal(r); err == nil {
|
|
if _, err := ch.CouldNotRun.Update(ctx, key, data, rev); err != nil && !errors.Is(err, ErrConflict) {
|
|
ch.logf("%s: its finding is written, and that cannot be kept: %v", key, err)
|
|
}
|
|
}
|
|
}
|
|
return next
|
|
}
|
|
|
|
// couldNotRunFinding is the finding of the check of key that could not run.
|
|
func couldNotRunFinding(key string, r CouldNotRun, at time.Time) views.Finding {
|
|
return views.Finding{Observer: CheckerModule, Kind: CouldNotRunKind, ID: key,
|
|
Summary: fmt.Sprintf("the check of %s (heads %s) could not run since %s: %s was not heard, and has not run since",
|
|
r.Group, r.Heads, r.Since.UTC().Format(time.RFC3339), r.Part),
|
|
Headline: fmt.Sprintf("The check of %s could not run", r.Group),
|
|
Explanation: "A check that cannot hear a part of the mesh it reads says so and is asked again when that part runs. " +
|
|
"It has not run for an hour: look at " + r.Part.String() + ". The finding clears when the heads are checked again, " +
|
|
"replaced, or their pull request closes.",
|
|
At: at.UTC()}
|
|
}
|