mesh-delivery/modules/mesh-delivery-builder/internal/build/askagain.go

package build

import (
	"context"
	"errors"
	"fmt"
	"regexp"
	"strconv"
	"strings"
	"time"

	"git.novox.be/novox/mesh-sdk/go/buildstep"
	"git.novox.be/novox/mesh-sdk/go/delivery"
)

// **An ask that can no longer be answered is asked again** (novox/hq issue 1945). A build seat's stream gives an ask up
// once it was delivered as often as its worker may (delivery.AskMaxDeliver), and a message given up on, dropped or
// terminated with no step written is answered by no build node: the lane waiting for its step would wait for ever, and on
// restart take over the same id and wait again. The builder notices it at the check bound (CheckAfter, the step bound),
// a context's bound and no timer of its own: it asks Gone, which reads the bus, and asks such an ask again under a new
// id, written into the key, said in its log and as the event build.asked-again, at most Askings times; then the key is
// failed with why. A person asks the same by hand with the tool builder_ask_again (AskAgain).

// DefaultAskings is how many times a build module's part of a build is asked, the first included, when Builder.Askings is 0.
const DefaultAskings = 3

// AskedAgainEvent is the event the builder says when it asked an ask again: mesh.mod.mesh-delivery-builder.event.<this>.
const AskedAgainEvent = "build.asked-again"

// AskedAgain is the body of AskedAgainEvent.
type AskedAgain struct {
	Key         string    `json:"key"`
	Module      string    `json:"module"`
	FP          string    `json:"fp"`
	BuildModule string    `json:"build_module"`
	Language    string    `json:"language"`
	Old         string    `json:"old"`
	New         string    `json:"new"`
	Asking      int       `json:"asking"`
	Askings     int       `json:"askings"`
	Why         string    `json:"why"`
	At          time.Time `json:"at"`
}

// AskSeen is what the bus says of an ask whose step has not ended, as the builder may read it: its build seat's stream
// (information and messages by sequence) and its build module's steps. The builder may not read the seat workers'
// delivery counts (granted to the delivery view alone), so a message given up on is told by what the workers did after it.
type AskSeen struct {
	Stream string
	// Step is whether a build node wrote a step for the ask (running: the step bound of buildrun.Asker judges it).
	Step bool
	// OnStream is whether the stream still holds the ask's message; PutAt is when it was put, Ahead how many messages of
	// its subject wait before it.
	OnStream bool
	PutAt    time.Time
	Ahead    int
	// Later is a step of the same build module that started after the ask was put, when one did.
	Later *delivery.Step
}

// Why is why the ask can no longer be answered, or "" when it can: it has a step, waits behind others of its kind, or no
// build node took any ask since it was put (the nodes are busy or paused: a backlog, never a fault, ADR 0525 §9).
func (s AskSeen) Why() string {
	switch {
	case s.Step:
		return ""
	case !s.OnStream:
		return fmt.Sprintf("no build node wrote a step for it and %s no longer holds it: it was given up on or dropped", s.Stream)
	case s.Ahead > 0 || s.Later == nil:
		return ""
	default:
		return fmt.Sprintf("it waits first in line on %s since %s while %s took a later ask at %s: its stream no longer delivers it (given up after its most deliveries)",
			s.Stream, s.PutAt.UTC().Format(time.RFC3339), s.Later.Node, s.Later.Started.UTC().Format(time.RFC3339))
	}
}

func (b *Builder) askings() int {
	if b.Askings > 0 {
		return b.Askings
	}
	return DefaultAskings
}

var askingWords = regexp.MustCompile(`asked (\S+) again \(asking (\d+)\)`)

// askingOf is the asking a key's step words name for a build module's ask, 1 when none.
func askingOf(step, buildModule string) int {
	for _, m := range askingWords.FindAllStringSubmatch(step, -1) {
		if m[1] == buildModule {
			if n, err := strconv.Atoi(m[2]); err == nil && n > 0 {
				return n
			}
		}
	}
	return 1
}

// AskAgain asks the waiting asks of a build key again now, under new ids (the tool builder_ask_again): it answers the
// languages whose lane waits, none when no lane of this builder waits on the key.
func (b *Builder) AskAgain(key, why string) []string {
	b.init()
	b.mu.Lock()
	defer b.mu.Unlock()
	var asked []string
	for language, c := range b.waiting[key] {
		select {
		case c <- why:
		default: // one is already asked
		}
		asked = append(asked, language)
	}
	return asked
}

// Waiting are the build keys whose lanes wait for a step now, each with its languages.
func (b *Builder) Waiting() map[string][]string {
	b.init()
	b.mu.Lock()
	defer b.mu.Unlock()
	out := map[string][]string{}
	for key, ls := range b.waiting {
		for l := range ls {
			out[key] = append(out[key], l)
		}
	}
	return out
}

func (b *Builder) listen(key, language string) chan string {
	b.init()
	c := make(chan string, 1)
	b.mu.Lock()
	if b.waiting[key] == nil {
		b.waiting[key] = map[string]chan string{}
	}
	b.waiting[key][language] = c
	b.mu.Unlock()
	return c
}

func (b *Builder) unlisten(key, language string, c chan string) {
	b.mu.Lock()
	if b.waiting[key][language] == c {
		delete(b.waiting[key], language)
		if len(b.waiting[key]) == 0 {
			delete(b.waiting, key)
		}
	}
	b.mu.Unlock()
}

var (
	errCheck = errors.New("the check bound passed")
	errAgain = errors.New("asked again")
)

type byHand struct{ why string }

func (h byHand) Error() string { return "asked again by hand: " + h.why }

// answer waits for the step of the lane's ask, and asks it again while it can no longer be answered (Gone) or a person
// asked it again (AskAgain); it answers the step and the ask that has it.
func (b *Builder) answer(ctx context.Context, l *lane, ask delivery.StepAsk, asked *askNotes, module, fp string) (delivery.Step, delivery.StepAsk, error) {
	key, _ := delivery.BuilderBuildKey(module, fp)
	again := b.listen(key, ask.Language)
	defer b.unlisten(key, ask.Language, again)
	asking := askingOf(asked.step, l.module)
	for {
		step, why, err := b.waitOnce(ctx, ask, again)
		switch {
		case errors.Is(err, errCheck):
			continue
		case errors.Is(err, errAgain):
			if asking >= b.askings() {
				return step, ask, fmt.Errorf("the ask %s could not be answered and was asked %d times, the most it is: %s", ask.ID, asking, why)
			}
			asking++
			old := ask.ID
			ask.ID, ask.At = buildstep.NewID(), time.Now()
			if err := asked.note(ctx, ask.Language, ask.ID, fmt.Sprintf("asked %s again (asking %d): %s", l.module, asking, why)); err != nil {
				return step, ask, fmt.Errorf("the ask of %s asked again could not be written to the key: %w", l.module, err)
			}
			if err := b.Asks.Put(ctx, ask); err != nil {
				return step, ask, err
			}
			b.logf("%s: the ask %s of %s is asked again as %s (asking %d of %d): %s", key, old, l.module, ask.ID, asking, b.askings(), why)
			if b.Said != nil {
				b.Said(AskedAgain{Key: key, Module: module, FP: fp, BuildModule: l.module, Language: ask.Language, Old: old, New: ask.ID,
					Asking: asking, Askings: b.askings(), Why: why, At: ask.At})
			}
		default:
			return step, ask, err
		}
	}
}

// waitOnce waits for the ask's step until it ends, a person asks it again, or the check bound passes: then it asks Gone
// and answers errAgain with why when the ask can no longer be answered, errCheck when it waits on.
func (b *Builder) waitOnce(ctx context.Context, ask delivery.StepAsk, again <-chan string) (delivery.Step, string, error) {
	hand, cancel := context.WithCancelCause(ctx)
	defer cancel(nil)
	go func() {
		select {
		case why := <-again:
			cancel(byHand{why})
		case <-hand.Done():
		}
	}()
	wait := hand
	if b.Gone != nil && b.CheckAfter > 0 {
		var stop context.CancelFunc
		wait, stop = context.WithTimeoutCause(hand, b.CheckAfter, errCheck)
		defer stop()
	}
	step, err := b.Asks.Wait(wait, ask)
	if err == nil || ctx.Err() != nil {
		return step, "", err
	}
	var h byHand
	if errors.As(context.Cause(hand), &h) {
		return step, strings.TrimSpace("asked again by hand: " + h.why), errAgain
	}
	if errors.Is(context.Cause(wait), errCheck) {
		why, gerr := b.Gone.Gone(ctx, ask)
		if gerr != nil {
			b.logf("whether the ask %s can still be answered cannot be read, so it is waited for: %v", ask.ID, gerr)
			return step, "", errCheck
		}
		if why == "" {
			return step, "", errCheck
		}
		return step, why, errAgain
	}
	return step, "", err
}