package check import ( "context" "encoding/json" "errors" "fmt" "sort" "time" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliverycheckerclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdeliverychecker" ) // A check that failed because a part of the mesh was not heard says `could not run`, naming the part, and releases its // claim on the verdict key (design 47, *A check runs to its end*; issue 1373). It is asked again once per set of heads and // per part returning, on the one event that can change its outcome: the node-engine seat's `running..` // naming that part (PartRunning), never on a timer; a `running` of the same part at the same fingerprint after that asks // nothing. One not asked again within CouldNotRunBound becomes the checker's finding CouldNotRunKind, which clears when the // heads are checked again, replaced, or their pull request closes. // // What the checker knows of each such check is its own state could-not-run (CouldNotRunState), one key per verdict key, // read by checker_status (could_not_run): kept on the bus, so a checker started again asks each one again on the next // `running` and raises its finding at the same time. // CouldNotRunState is the checker's own state of its checks that could not run, by verdict key. const CouldNotRunState = "could-not-run" // CouldNotRunBucket is the bucket of CouldNotRunState. const CouldNotRunBucket = CheckerModule + "_" + CouldNotRunState // CouldNotRunKind is the checker's finding of a check that could not run and was not asked again within CouldNotRunBound. const CouldNotRunKind = "check-could-not-run" // CouldNotRunBound is how long a check that could not run waits to be asked again before it is a finding: the check's own // bound of an hour. const CouldNotRunBound = time.Hour // Part is a part of the mesh a check could not hear: the node it runs on and its module, as the node-engine seat's // `running..` names it. An empty field is not known: an empty Module is any part on Node, and an empty Node is // Module wherever it runs; a Part with neither is a part the check could not name, asked again on the first `running`. type Part struct { Node string `json:"node,omitempty"` Module string `json:"module,omitempty"` } // String names the part as a verdict says it. func (p Part) String() string { switch { case p.Node != "" && p.Module != "": return p.Module + " on " + p.Node case p.Module != "": return p.Module case p.Node != "": return "the node " + p.Node } return "a part of the mesh" } // matches says whether a `running` of module on node is the part's return. func (p Part) matches(node, module string) bool { return (p.Node == "" || p.Node == node) && (p.Module == "" || p.Module == module) } // CouldNotRun is one check that could not run, as the state could-not-run keeps it. type CouldNotRun struct { V int `json:"v"` Group string `json:"group"` Heads string `json:"heads"` Part Part `json:"part"` Words string `json:"words"` // Since is when its verdict was written; CouldNotRunBound after it, not asked again, it is a finding. Since time.Time `json:"since"` // AskedAt is the `running` that asked the check of these heads again, `.@`: a `running` of // the same part at the same fingerprint asks nothing more. AskedAt string `json:"asked-at,omitempty"` // Raised says its finding is written. Raised bool `json:"raised,omitempty"` } // IsCouldNotRun says whether a verdict failed because a part of the mesh the check reads was not heard (UnheardWords): its // claim is released, and it is asked again when that part runs (PartRunning). func IsCouldNotRun(v meshdeliverychecker.VerdictsByGroupHeadsValue) bool { return v.State == meshdeliverycheckerclient.VerdictFail && containsWords(failedOf(v).Words, UnheardWords) } // running is how a `running` event is kept in AskedAt and in the release of a check. func runningAt(node, module, fingerprint string) string { return node + "." + module + "@" + fingerprint } // PartRunning is the node-engine seat's `running..` at fingerprint: every check that could not run for want // of that part, and was not asked again by the same part at the same fingerprint, is released and asked again once. func (ch *Checker) PartRunning(ctx context.Context, node, module, fingerprint string) { ch.init() if ch.CouldNotRun == nil { return } keys, err := ch.CouldNotRun.Keys(ctx) if err != nil { ch.logf("the checks that could not run cannot be listed, so the running of %s on %s asks none again: %v", module, node, err) return } at := runningAt(node, module, fingerprint) released := 0 for _, key := range keys { r, _, ok := ch.couldNotRunOf(ctx, key) if !ok || !r.Part.matches(node, module) || r.AskedAt == at { continue } ch.mu.Lock() ch.release[key] = at ch.mu.Unlock() released++ ch.logf("%s: %s runs (%s), which the check could not hear: asked again", key, module, at) } if released > 0 { ch.Reconsider(ctx) } } // couldNotRunOf reads one key of the state could-not-run. func (ch *Checker) couldNotRunOf(ctx context.Context, key string) (CouldNotRun, uint64, bool) { raw, rev, found, err := ch.CouldNotRun.Get(ctx, key) if err != nil || !found || len(raw) == 0 { return CouldNotRun{}, 0, false } var r CouldNotRun if json.Unmarshal(raw, &r) != nil { return CouldNotRun{}, 0, false } return r, rev, true } // takeRelease answers the `running` that released the check of key, once, and forgets it. func (ch *Checker) takeRelease(key string) (string, bool) { ch.mu.Lock() defer ch.mu.Unlock() at, ok := ch.release[key] delete(ch.release, key) return at, ok } // keepCouldNotRun writes the check's record after a verdict that could not run, asked again by askedAt ("" for none). func (ch *Checker) keepCouldNotRun(ctx context.Context, cs *Case, v meshdeliverychecker.VerdictsByGroupHeadsValue, part Part, askedAt string) { if ch.CouldNotRun == nil { return } r := CouldNotRun{V: 1, Group: cs.Group, Heads: cs.Heads, Part: part, Words: failedOf(v).Words, Since: ch.now().UTC(), AskedAt: askedAt} data, err := json.Marshal(r) if err != nil { return } _, rev, found, err := ch.CouldNotRun.Get(ctx, cs.Key) if err == nil && found { _, err = ch.CouldNotRun.Update(ctx, cs.Key, data, rev) } else if err == nil { _, err = ch.CouldNotRun.Create(ctx, cs.Key, data) } if err != nil { ch.logf("%s: could not run, and that cannot be kept, so no running of %s asks it again until it is checked again: %v", cs.Key, part, err) return } if ch.CouldNotRunKept != nil { ch.CouldNotRunKept() } } // clearCouldNotRun forgets the check of key that could not run, and clears its finding: its heads are checked again, // replaced, or their pull request closed. func (ch *Checker) clearCouldNotRun(ctx context.Context, key string) { if ch.CouldNotRun == nil { return } r, _, ok := ch.couldNotRunOf(ctx, key) if !ok { return } if d, can := ch.CouldNotRun.(Deleter); can { if err := d.Delete(ctx, key); err != nil { ch.logf("%s: the record that it could not run cannot be deleted: %v", key, err) return } } if r.Raised && ch.Findings != nil { if err := ch.Findings.Delete(couldNotRunFinding(key, r, ch.now())); err != nil { ch.logf("%s: its finding %s cannot be cleared: %v", key, CouldNotRunKind, err) } } } // CouldNotRunDue raises the finding of every check that could not run and was not asked again within CouldNotRunBound by // now, and answers when the next one is due (the zero time when none waits). The checker's process waits for that time; // a test gives now. func (ch *Checker) CouldNotRunDue(ctx context.Context, now time.Time) time.Time { if ch.CouldNotRun == nil { return time.Time{} } keys, err := ch.CouldNotRun.Keys(ctx) if err != nil { ch.logf("the checks that could not run cannot be listed, so none is raised as %s now: %v", CouldNotRunKind, err) return time.Time{} } sort.Strings(keys) var next time.Time for _, key := range keys { r, rev, ok := ch.couldNotRunOf(ctx, key) if !ok || r.Raised { continue } due := r.Since.Add(CouldNotRunBound) if now.Before(due) { if next.IsZero() || due.Before(next) { next = due } continue } if ch.Findings == nil { continue } if err := ch.Findings.Put(couldNotRunFinding(key, r, now)); err != nil { ch.logf("%s: its finding %s cannot be written: %v", key, CouldNotRunKind, err) continue } r.Raised = true if data, err := json.Marshal(r); err == nil { if _, err := ch.CouldNotRun.Update(ctx, key, data, rev); err != nil && !errors.Is(err, ErrConflict) { ch.logf("%s: its finding is written, and that cannot be kept: %v", key, err) } } } return next } // Finding is the checker's finding of a check that could not run, in plain fields: the checker's writer lays it out as the seat // mesh-conditions' finding (design 47 G §689). type Finding struct { Kind, ID, Summary, Headline, Explanation string At time.Time } // FindingWriter writes and clears the checker's findings. type FindingWriter interface { Put(f Finding) error Delete(f Finding) error } // RaisedCouldNotRun is every finding the checker observes now (design 47 G §384, §461): the check that could not run // and whose finding is written. The finding writer states them at its start and when the keeper asks. func (ch *Checker) RaisedCouldNotRun(ctx context.Context) ([]Finding, error) { if ch.CouldNotRun == nil { return nil, nil } keys, err := ch.CouldNotRun.Keys(ctx) if err != nil { return nil, fmt.Errorf("the checks that could not run cannot be listed: %w", err) } sort.Strings(keys) var out []Finding for _, key := range keys { if r, _, ok := ch.couldNotRunOf(ctx, key); ok && r.Raised { out = append(out, couldNotRunFinding(key, r, r.Since)) } } return out, nil } // couldNotRunFinding is the finding of the check of key that could not run. func couldNotRunFinding(key string, r CouldNotRun, at time.Time) Finding { return Finding{Kind: CouldNotRunKind, ID: key, Summary: fmt.Sprintf("the check of %s (heads %s) could not run since %s: %s was not heard, and has not run since", r.Group, r.Heads, r.Since.UTC().Format(time.RFC3339), r.Part), Headline: fmt.Sprintf("The check of %s could not run", r.Group), Explanation: "A check that cannot hear a part of the mesh it reads says so and is asked again when that part runs. " + "It has not run for an hour: look at " + r.Part.String() + ". The finding clears when the heads are checked again, " + "replaced, or their pull request closes.", At: at.UTC()} }