mesh-delivery/modules/mesh-delivery-checker/internal/check/askagain.go

package check

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"strings"

	"git.novox.be/novox/mesh-sdk/go/delivery"
)

// **A check is asked again by hand** (the lost check of 2026-10-28; tools-are-how-a-module-is-spoken-to: every recovery
// has a tool). checker_ask_again ends a candidate's running verdict, whose wait judged nothing, with AskedAgainWords and
// the why; the checker hears it on its watch of its verdicts (Checker.VerdictWritten), ends the check of those heads
// running in it, and claims and asks it anew, a whole check (IsEnded). It is refused while a step of the candidate's pull
// requests can still run (checker_status's running_steps): that step's end judges it; and while an ask the check put is
// still in its build seat's stream, taken by no node (it has no step key yet): asking again then would run the check twice
// (issue 1791's kind). The checker keeps the asks each running check put in its state asks (AsksState, by verdict key).
// An ask is not put again under its own id instead: the stream de-duplicates an id within its window, so the copy waiting
// stays and the new one is dropped, and past the window both would run; nothing takes a put ask back (buildrun.Asker).

// AsksState is the checker's state of the asks each running check put, by verdict key (CheckAsks); AsksBucket its bucket.
const (
	AsksState  = "asks"
	AsksBucket = delivery.CheckerModule + "_" + AsksState
)

// CheckAsks are the asks a check put, in order, as the state asks keeps them while it runs; its end deletes the key.
type CheckAsks struct {
	Asks []string `json:"asks"`
}

// AsksKV is the state asks: a KV whose key a check's end deletes, so the state holds only the checks that run.
type AsksKV interface {
	KV
	Delete(ctx context.Context, key string) error
}

// forgetAsks deletes the asks a check put at its end; one that cannot be deleted is said, and the next check of the same
// heads writes the key anew.
func (ch *Checker) forgetAsks(ctx context.Context, key string) {
	if err := ch.Asks.Delete(ctx, key); err != nil {
		ch.logf("%s: the asks it put could not be forgotten at its end: %v", key, err)
	}
}

// recordAsks keeps the asks a check put under its verdict key; nil Asks keeps nothing. A record that cannot be written
// is said: checker_ask_again then refuses the check, as it cannot tell whether its ask waits in a stream.
func (ch *Checker) recordAsks(ctx context.Context, key string, asks []string) {
	if ch.Asks == nil {
		return
	}
	data, err := json.Marshal(CheckAsks{Asks: asks})
	if err == nil {
		_, err = putOnce(ctx, ch.Asks, key, data, func(cur []byte) bool { return string(cur) == string(data) })
	}
	if err != nil {
		ch.logf("%s: the asks it put cannot be kept (%v), so %s refuses it", key, err, AskAgainTool)
	}
}

// AskAgainTool asks a candidate's check again by hand.
const AskAgainTool = "checker_ask_again"

// AskedAgainWords open the words of a verdict AskAgainTool wrote; the why follows.
const AskedAgainWords = EndedWords + "asked again by hand (" + AskAgainTool + "): "

// errAskedAgain is the cause a running check is ended with when its verdict was asked again by hand.
var errAskedAgain = errors.New("its verdict was asked again by hand (" + AskAgainTool + ")")

// IsAskedAgain says whether a verdict was written by AskAgainTool.
func IsAskedAgain(v delivery.Verdict) bool {
	return IsEnded(v) && strings.HasPrefix(v.Failed.Words, AskedAgainWords)
}

// VerdictEntry is the verdicts state as the ask-again tool writes it: a key's value and revision (nil and 0 for none),
// and a compare-and-set.
type VerdictEntry interface {
	Entry(key string) (json.RawMessage, uint64, error)
	Update(key string, value []byte, revision uint64) error
}

// AskedAgain is the answer of AskAgainTool.
type AskedAgain struct {
	Key  string `json:"key"`
	Was  string `json:"was"`
	Said string `json:"said"`
	Note string `json:"note"`
}

// AskAgain ends the running verdict of the group's candidate as asked again by hand, saying why. It refuses a group with
// no candidate, a verdict that is not running (a pass or a failure stands until a push or a trunk move), one a step of
// whose pull requests can still run, and one whose asks are not known or one of which no node took yet.
func (r Reader) AskAgain(w VerdictEntry, group, why string) (AskedAgain, error) {
	why = strings.TrimSpace(why)
	if group == "" || why == "" {
		return AskedAgain{}, fmt.Errorf("give the group and why it is asked again")
	}
	if w == nil {
		return AskedAgain{}, fmt.Errorf("the verdicts state cannot be written here")
	}
	c, ok := r.candidate(group)
	if !ok {
		return AskedAgain{}, fmt.Errorf("no candidate is held for the group %q", group)
	}
	key, err := delivery.CheckerVerdictKey(c.Group, c.HeadsHash)
	if err != nil {
		return AskedAgain{}, err
	}
	raw, rev, err := w.Entry(key)
	if err != nil {
		return AskedAgain{}, fmt.Errorf("%s cannot be read: %w", key, err)
	}
	if len(raw) == 0 {
		return AskedAgain{}, fmt.Errorf("%s holds no verdict: the checker claims its check when it looks at the candidate (%s)", key, StatusReadTool)
	}
	v, err := delivery.DecodeVerdict(raw)
	if err != nil {
		return AskedAgain{}, fmt.Errorf("%s holds what is not a verdict: %w", key, err)
	}
	if v.State != delivery.VerdictRunning {
		return AskedAgain{}, fmt.Errorf("%s is %s, not running: a verdict stands until a push or a trunk move asks it again", key, v.State)
	}
	steps := r.readSteps()
	for _, rs := range steps.running {
		if asksOf(c, rs.What) {
			return AskedAgain{}, fmt.Errorf("a step of it runs: %s %s on %s since %s (%s); its end judges it", rs.Module, rs.ID, rs.Node, rs.Since, rs.What)
		}
	}
	if r.AskState.Get == nil {
		return AskedAgain{}, fmt.Errorf("the checker's asks are not readable here, so whether its ask waits in a build seat's stream cannot be told")
	}
	kept, err := r.AskState.Get(key)
	var put CheckAsks
	if err == nil && len(kept) > 0 {
		err = json.Unmarshal(kept, &put)
	}
	if err != nil || len(put.Asks) == 0 {
		return AskedAgain{}, fmt.Errorf("the checker names no ask of %s (%v), so whether one waits in a build seat's stream cannot be told; asking again could run it twice", key, err)
	}
	taken := map[string]bool{}
	for _, s := range steps.all {
		taken[s.step.ID] = true
	}
	for _, id := range put.Asks {
		if !taken[id] {
			return AskedAgain{}, fmt.Errorf("its ask %s waits in its build seat's stream, taken by no node yet: asking again would run it twice; "+
				"the seats' backlog is in mesh-delivery-view.build_usage", id)
		}
	}
	words := AskedAgainWords + why
	ended := delivery.Verdict{Group: v.Group, Heads: v.Heads, Trunk: v.Trunk, State: delivery.VerdictFail,
		Failed: delivery.FailedFirst{Machine: "checker", Words: words}}
	data, err := delivery.EncodeVerdict(ended)
	if err != nil {
		return AskedAgain{}, err
	}
	if err := w.Update(key, data, rev); err != nil {
		return AskedAgain{}, fmt.Errorf("%s changed while it was asked again, so it was not; read it again: %w", key, err)
	}
	return AskedAgain{Key: key, Was: v.State, Said: words,
		Note: "the checker hears it on its watch of its verdicts, ends the wait that judged nothing and asks the check anew, whole; " +
			VerdictsReadTool + " shows it running again"}, nil
}

// VerdictWritten takes a change of the checker's verdicts in (its watch, updates only): a verdict AskAgainTool wrote ends
// the check of those heads running in this process, which then considers them again; with none running, the candidate
// that holds them is considered at once. Any other verdict is the checker's own, and left.
func (ch *Checker) VerdictWritten(ctx context.Context, key string, value []byte) {
	v, err := delivery.DecodeVerdict(value)
	if err != nil || !IsAskedAgain(v) {
		return
	}
	ch.init()
	ch.mu.Lock()
	if l, ok := ch.live[key]; ok {
		l.end(errAskedAgain)
		ch.mu.Unlock()
		ch.logf("%s: %s; the check running for it is ended and asked again", key, v.Failed.Words)
		return
	}
	var group string
	var cand delivery.Candidate
	for k, c := range ch.known {
		if vk, err := delivery.CheckerVerdictKey(c.Group, c.HeadsHash); err == nil && vk == key {
			group, cand = k, c
			break
		}
	}
	ch.mu.Unlock()
	if group == "" {
		ch.logf("%s: %s, but no candidate holds these heads, so nothing is asked", key, v.Failed.Words)
		return
	}
	ch.logf("%s: %s; asked again", key, v.Failed.Words)
	ch.consider(ctx, group, cand)
}