mesh-delivery/modules/mesh-delivery-checker/internal/check/couldnotrun.go

package check

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"sort"
	"time"

	"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliverycheckerclient"
	"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdeliverychecker"
)

// A check that failed because a part of the mesh was not heard says `could not run`, naming the part, and releases its
// claim on the verdict key (design 47, *A check runs to its end*; issue 1373). It is asked again once per set of heads and
// per part returning, on the one event that can change its outcome: the node-engine seat's `running.<node>.<module>`
// naming that part (PartRunning), never on a timer; a `running` of the same part at the same fingerprint after that asks
// nothing. One not asked again within CouldNotRunBound becomes the checker's finding CouldNotRunKind, which clears when the
// heads are checked again, replaced, or their pull request closes.
//
// What the checker knows of each such check is its own state could-not-run (CouldNotRunState), one key per verdict key,
// read by checker_status (could_not_run): kept on the bus, so a checker started again asks each one again on the next
// `running` and raises its finding at the same time.

// CouldNotRunState is the checker's own state of its checks that could not run, by verdict key.
const CouldNotRunState = "could-not-run"

// CouldNotRunBucket is the bucket of CouldNotRunState.
const CouldNotRunBucket = CheckerModule + "_" + CouldNotRunState

// CouldNotRunKind is the checker's finding of a check that could not run and was not asked again within CouldNotRunBound.
const CouldNotRunKind = "check-could-not-run"

// CouldNotRunBound is how long a check that could not run waits to be asked again before it is a finding: the check's own
// bound of an hour.
const CouldNotRunBound = time.Hour

// Part is a part of the mesh a check could not hear: the node it runs on and its module, as the node-engine seat's
// `running.<node>.<module>` names it. An empty field is not known: an empty Module is any part on Node, and an empty Node is
// Module wherever it runs; a Part with neither is a part the check could not name, asked again on the first `running`.
type Part struct {
	Node   string `json:"node,omitempty"`
	Module string `json:"module,omitempty"`
}

// String names the part as a verdict says it.
func (p Part) String() string {
	switch {
	case p.Node != "" && p.Module != "":
		return p.Module + " on " + p.Node
	case p.Module != "":
		return p.Module
	case p.Node != "":
		return "the node " + p.Node
	}
	return "a part of the mesh"
}

// matches says whether a `running` of module on node is the part's return.
func (p Part) matches(node, module string) bool {
	return (p.Node == "" || p.Node == node) && (p.Module == "" || p.Module == module)
}

// CouldNotRun is one check that could not run, as the state could-not-run keeps it.
type CouldNotRun struct {
	V     int    `json:"v"`
	Group string `json:"group"`
	Heads string `json:"heads"`
	Part  Part   `json:"part"`
	Words string `json:"words"`
	// Since is when its verdict was written; CouldNotRunBound after it, not asked again, it is a finding.
	Since time.Time `json:"since"`
	// AskedAt is the `running` that asked the check of these heads again, `<node>.<module>@<fingerprint>`: a `running` of
	// the same part at the same fingerprint asks nothing more.
	AskedAt string `json:"asked-at,omitempty"`
	// Raised says its finding is written.
	Raised bool `json:"raised,omitempty"`
}

// IsCouldNotRun says whether a verdict failed because a part of the mesh the check reads was not heard (UnheardWords): its
// claim is released, and it is asked again when that part runs (PartRunning).
func IsCouldNotRun(v meshdeliverychecker.VerdictsByGroupHeadsValue) bool {
	return v.State == meshdeliverycheckerclient.VerdictFail && containsWords(failedOf(v).Words, UnheardWords)
}

// running is how a `running` event is kept in AskedAt and in the release of a check.
func runningAt(node, module, fingerprint string) string {
	return node + "." + module + "@" + fingerprint
}

// PartRunning is the node-engine seat's `running.<node>.<module>` at fingerprint: every check that could not run for want
// of that part, and was not asked again by the same part at the same fingerprint, is released and asked again once.
func (ch *Checker) PartRunning(ctx context.Context, node, module, fingerprint string) {
	ch.init()
	if ch.CouldNotRun == nil {
		return
	}
	keys, err := ch.CouldNotRun.Keys(ctx)
	if err != nil {
		ch.logf("the checks that could not run cannot be listed, so the running of %s on %s asks none again: %v", module, node, err)
		return
	}
	at := runningAt(node, module, fingerprint)
	released := 0
	for _, key := range keys {
		r, _, ok := ch.couldNotRunOf(ctx, key)
		if !ok || !r.Part.matches(node, module) || r.AskedAt == at {
			continue
		}
		ch.mu.Lock()
		ch.release[key] = at
		ch.mu.Unlock()
		released++
		ch.logf("%s: %s runs (%s), which the check could not hear: asked again", key, module, at)
	}
	if released > 0 {
		ch.Reconsider(ctx)
	}
}

// couldNotRunOf reads one key of the state could-not-run.
func (ch *Checker) couldNotRunOf(ctx context.Context, key string) (CouldNotRun, uint64, bool) {
	raw, rev, found, err := ch.CouldNotRun.Get(ctx, key)
	if err != nil || !found || len(raw) == 0 {
		return CouldNotRun{}, 0, false
	}
	var r CouldNotRun
	if json.Unmarshal(raw, &r) != nil {
		return CouldNotRun{}, 0, false
	}
	return r, rev, true
}

// takeRelease answers the `running` that released the check of key, once, and forgets it.
func (ch *Checker) takeRelease(key string) (string, bool) {
	ch.mu.Lock()
	defer ch.mu.Unlock()
	at, ok := ch.release[key]
	delete(ch.release, key)
	return at, ok
}

// keepCouldNotRun writes the check's record after a verdict that could not run, asked again by askedAt ("" for none).
func (ch *Checker) keepCouldNotRun(ctx context.Context, cs *Case, v meshdeliverychecker.VerdictsByGroupHeadsValue, part Part, askedAt string) {
	if ch.CouldNotRun == nil {
		return
	}
	r := CouldNotRun{V: 1, Group: cs.Group, Heads: cs.Heads, Part: part, Words: failedOf(v).Words, Since: ch.now().UTC(), AskedAt: askedAt}
	data, err := json.Marshal(r)
	if err != nil {
		return
	}
	_, rev, found, err := ch.CouldNotRun.Get(ctx, cs.Key)
	if err == nil && found {
		_, err = ch.CouldNotRun.Update(ctx, cs.Key, data, rev)
	} else if err == nil {
		_, err = ch.CouldNotRun.Create(ctx, cs.Key, data)
	}
	if err != nil {
		ch.logf("%s: could not run, and that cannot be kept, so no running of %s asks it again until it is checked again: %v", cs.Key, part, err)
		return
	}
	if ch.CouldNotRunKept != nil {
		ch.CouldNotRunKept()
	}
}

// clearCouldNotRun forgets the check of key that could not run, and clears its finding: its heads are checked again,
// replaced, or their pull request closed.
func (ch *Checker) clearCouldNotRun(ctx context.Context, key string) {
	if ch.CouldNotRun == nil {
		return
	}
	r, _, ok := ch.couldNotRunOf(ctx, key)
	if !ok {
		return
	}
	if d, can := ch.CouldNotRun.(Deleter); can {
		if err := d.Delete(ctx, key); err != nil {
			ch.logf("%s: the record that it could not run cannot be deleted: %v", key, err)
			return
		}
	}
	if r.Raised && ch.Findings != nil {
		if err := ch.Findings.Delete(couldNotRunFinding(key, r, ch.now())); err != nil {
			ch.logf("%s: its finding %s cannot be cleared: %v", key, CouldNotRunKind, err)
		}
	}
}

// CouldNotRunDue raises the finding of every check that could not run and was not asked again within CouldNotRunBound by
// now, and answers when the next one is due (the zero time when none waits). The checker's process waits for that time;
// a test gives now.
func (ch *Checker) CouldNotRunDue(ctx context.Context, now time.Time) time.Time {
	if ch.CouldNotRun == nil {
		return time.Time{}
	}
	keys, err := ch.CouldNotRun.Keys(ctx)
	if err != nil {
		ch.logf("the checks that could not run cannot be listed, so none is raised as %s now: %v", CouldNotRunKind, err)
		return time.Time{}
	}
	sort.Strings(keys)
	var next time.Time
	for _, key := range keys {
		r, rev, ok := ch.couldNotRunOf(ctx, key)
		if !ok || r.Raised {
			continue
		}
		due := r.Since.Add(CouldNotRunBound)
		if now.Before(due) {
			if next.IsZero() || due.Before(next) {
				next = due
			}
			continue
		}
		if ch.Findings == nil {
			continue
		}
		if err := ch.Findings.Put(couldNotRunFinding(key, r, now)); err != nil {
			ch.logf("%s: its finding %s cannot be written: %v", key, CouldNotRunKind, err)
			continue
		}
		r.Raised = true
		if data, err := json.Marshal(r); err == nil {
			if _, err := ch.CouldNotRun.Update(ctx, key, data, rev); err != nil && !errors.Is(err, ErrConflict) {
				ch.logf("%s: its finding is written, and that cannot be kept: %v", key, err)
			}
		}
	}
	return next
}

// Finding is the checker's finding of a check that could not run, in plain fields: the checker's writer lays it out as the seat
// mesh-conditions' finding (design 47 G §689).
type Finding struct {
	Kind, ID, Summary, Headline, Explanation string
	At                                       time.Time
}

// FindingWriter writes and clears the checker's findings.
type FindingWriter interface {
	Put(f Finding) error
	Delete(f Finding) error
}

// RaisedCouldNotRun is every finding the checker observes now (design 47 G §384, §461): the check that could not run
// and whose finding is written. The finding writer states them at its start and when the keeper asks.
func (ch *Checker) RaisedCouldNotRun(ctx context.Context) ([]Finding, error) {
	if ch.CouldNotRun == nil {
		return nil, nil
	}
	keys, err := ch.CouldNotRun.Keys(ctx)
	if err != nil {
		return nil, fmt.Errorf("the checks that could not run cannot be listed: %w", err)
	}
	sort.Strings(keys)
	var out []Finding
	for _, key := range keys {
		if r, _, ok := ch.couldNotRunOf(ctx, key); ok && r.Raised {
			out = append(out, couldNotRunFinding(key, r, r.Since))
		}
	}
	return out, nil
}

// couldNotRunFinding is the finding of the check of key that could not run.
func couldNotRunFinding(key string, r CouldNotRun, at time.Time) Finding {
	return Finding{Kind: CouldNotRunKind, ID: key,
		Summary: fmt.Sprintf("the check of %s (heads %s) could not run since %s: %s was not heard, and has not run since",
			r.Group, r.Heads, r.Since.UTC().Format(time.RFC3339), r.Part),
		Headline: fmt.Sprintf("The check of %s could not run", r.Group),
		Explanation: "A check that cannot hear a part of the mesh it reads says so and is asked again when that part runs. " +
			"It has not run for an hour: look at " + r.Part.String() + ". The finding clears when the heads are checked again, " +
			"replaced, or their pull request closes.",
		At: at.UTC()}
}