mesh-delivery/modules/mesh-delivery-builder/buildrun/asks.go

package buildrun

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"strings"
	"sync"
	"time"

	"github.com/nats-io/nats.go"
	"github.com/nats-io/nats.go/jetstream"

	"git.novox.be/novox/mesh-delivery/sdk/buildstep"
	"git.novox.be/novox/mesh-sdk/go/busretry"
)

// **An asker puts an ask on its build seat's stream and waits for its step** (novox/hq ADR 0525 §1, §6; issue 1887 A4,
// A5). It chooses no node: whichever build node pulls the ask writes `<node>.<ask id>` in its build module's steps state,
// and the asker reads the first such key that is not running. The builder and the checker share it.

// AskPublisher puts an encoded ask on subject, de-duplicated by its id.
type AskPublisher interface {
	PublishAsk(ctx context.Context, subject, id string, data []byte) error
}

// AskWaiter watches an ask's steps: every value a key `*.<ask id>` of the build module's steps state takes, running or
// ended, on the channel it answers, until ctx ends. A watch, not a poll.
type AskWaiter interface {
	WatchAsk(ctx context.Context, module, askID string) (<-chan buildstep.Step, error)
}

// Asker puts asks and waits for their steps.
type Asker struct {
	Publisher AskPublisher
	Waiter    AskWaiter
	// Seats finds the build module of an ask's language (design 47 G A-82).
	Seats *BuildSeats
	// Bound is how long a running step (a build node took the ask, its key written running) may stand still, with no new phase,
	// before the asker stops waiting; a step that keeps moving is waited for up to 3 Bounds from its start (issue 1930); zero waits as long as ctx. An ask waiting in its stream is not bounded: the stream's backlog is no fault
	// (ADR 0525 §6, §9), and the bound starts again when another node takes the ask.
	Bound time.Duration
	// Dropper, when set, is told to end an ask the asker stops waiting for without its end (ADR 0664 §5): a step given
	// again that the bus did not give within its bound, or a wait ended with a cause its caller names (a superseded
	// fingerprint). Never a wait ended by a shutdown (a plain cancel): the ask is then taken over by the next run.
	Dropper AskDropper
	// Logf says a drop that failed (the log's Printf when unset): the ask then stays on its stream, and it is said.
	Logf func(format string, args ...any)
}

// Withdrawn is the cause a caller gives the context of an ask's wait to withdraw the ask (ADR 0664 §4): a build superseded by a
// newer fingerprint, a candidate taken out, an ask asked again by hand. Asker.Wait drops the ask on this cause alone: a
// plain cancel (a shutdown) and any other cause (a bound passing, a check of whether the ask is gone) leave it on its stream
// for the next run. A cause that wraps a Withdrawn (errors.As) withdraws too.
type Withdrawn struct{ Why string }

func (w Withdrawn) Error() string { return w.Why }

// AskDropper ends an ask: the seat's verb drop, served by any holder of the build seat (no node is named). It writes the
// ask in the build module's shared state of pauses, so every node refuses it when the stream gives it, and the node that
// runs it ends the step (ADR 0664 §5).
type AskDropper interface {
	DropAsk(ctx context.Context, a buildstep.StepAsk, why string) error
}

// SeatCaller calls a seat's verb on whichever holder answers (mesh-sdk process.Conn.Call).
type SeatCaller interface {
	Call(key string, body any) (json.RawMessage, error)
}

// SeatDrops is an AskDropper that calls the verb drop of the ask's build seat over the bus.
type SeatDrops struct{ Caller SeatCaller }

// DropAsk calls <seat>.drop for a, naming its asker and why.
func (d SeatDrops) DropAsk(_ context.Context, a buildstep.StepAsk, why string) error {
	seat, err := buildstep.BuildSeat(a.Language)
	if err != nil {
		return err
	}
	if _, err := d.Caller.Call("seat:"+seat+".drop", map[string]any{"id": a.ID, "by": a.Asker, "why": why}); err != nil {
		return fmt.Errorf("dropping the ask %s: %w", a.ID, err)
	}
	return nil
}

// dropped ends a without its end. A failed drop is not the wait's error, but it is said (Logf).
func (k Asker) dropped(a buildstep.StepAsk, why string) {
	if k.Dropper == nil {
		return
	}
	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
	defer cancel()
	if err := k.Dropper.DropAsk(ctx, a, why); err != nil {
		say := k.Logf
		if say == nil {
			say = log.Printf
		}
		say("the ask %s was not dropped (%s), it stays on its stream: %v", a.ID, why, err)
	}
}

// PutTry is how long one publish of an ask waits for its stream's answer before it is made again (Put).
var PutTry = 10 * time.Second

// Put puts a on its stream: the seat's accept subject of its language and kind, under the builder's name as asker (design 47 G
// §481, §600: `accept.<queue>.builder`). The put is a kept call (novox/hq ADR 0668
// §1, §2; issue 1966, issue 1748 note 27): it carries the ask's id as its message id, so the stream takes it once, and it
// is made again through every transport fault (a deadline, no responders) within busretry.Bound, each try bounded by
// PutTry, until ctx ends. A put that still meets one is an error opening with buildstep.TransportFaultWords, so the key it
// fails is one the builder does not take again by itself (ADR 0664 §3) and builder_build_again does.
func (k Asker) Put(ctx context.Context, a buildstep.StepAsk) error {
	raw, err := buildstep.EncodeStepAsk(a)
	if err != nil {
		return err
	}
	subject, err := buildstep.AskSubjectBy(a.Language, a.Kind, buildstep.AskerBuilder)
	if err != nil {
		return err
	}
	_, err = busretry.DoRepeatable("putting the "+a.Kind+" ask "+a.ID, func() (struct{}, error) {
		if err := ctx.Err(); err != nil {
			return struct{}{}, fmt.Errorf("the put was given up: %v", err) // not a fault to wait out: the build ended
		}
		try, cancel := context.WithTimeout(ctx, PutTry)
		defer cancel()
		err := k.Publisher.PublishAsk(try, subject, a.ID, raw)
		if err != nil && ctx.Err() != nil {
			return struct{}{}, fmt.Errorf("the put was given up: %v", err)
		}
		return struct{}{}, err
	}, nil)
	switch {
	case err == nil:
		return nil
	case busretry.TimedOut(err) || busretry.Undelivered(err):
		return fmt.Errorf("%sthe put of the %s ask %s: %w", buildstep.TransportFaultWords, a.Kind, a.ID, err)
	}
	return fmt.Errorf("putting the %s ask %s: %w", a.Kind, a.ID, err)
}

// Wait waits for the step of an ask already put (a builder that started again takes over its own asks with it, never
// putting them twice) and answers it when it ended, whatever its state. The bound runs from the step's start on the node
// that took the ask, read from its running key, so an ask that waits in the stream for a free slot waits as long as ctx.
func (k Asker) Wait(ctx context.Context, a buildstep.StepAsk) (buildstep.Step, error) {
	bm, err := k.Seats.OfAskLanguage(a.Language)
	if err != nil {
		return buildstep.Step{}, err
	}
	module := bm.Module
	watch, cancel := context.WithCancel(ctx)
	defer cancel()
	steps, err := k.Waiter.WatchAsk(watch, module, a.ID)
	if err != nil {
		return buildstep.Step{}, err
	}
	var bound, outerCap <-chan time.Time
	var timer, capTimer *time.Timer
	defer func() {
		if timer != nil {
			timer.Stop()
		}
		if capTimer != nil {
			capTimer.Stop()
		}
	}()
	running, since, phase, phaseAt := "", time.Time{}, "", time.Time{}
	var agains []string // the node of each run that ended on a transport fault, in order (ADR 0664 §3)
	standStill := func() error {
		if phase == "" {
			return fmt.Errorf("the ask %s's step on %s did not end within %s of its start: it stood still, saying no phase", a.ID, running, k.Bound)
		}
		return fmt.Errorf("the ask %s's step on %s did not end within %s: it stood still in %q since %s",
			a.ID, running, k.Bound, phase, phaseAt.UTC().Format(time.RFC3339))
	}
	for {
		select {
		case <-ctx.Done():
			var w Withdrawn
			if errors.As(context.Cause(ctx), &w) {
				k.dropped(a, context.Cause(ctx).Error())
			}
			return buildstep.Step{}, ctx.Err()
		case <-bound:
			if len(agains) > 0 && phase == "given again after a transport fault" {
				k.dropped(a, "the bus did not give it again within its bound")
			}
			return buildstep.Step{}, standStill()
		case <-outerCap:
			return buildstep.Step{}, fmt.Errorf("the ask %s's step on %s did not end within %s of its start: it kept moving (last phase %q at %s) but never ended",
				a.ID, running, 3*k.Bound, phase, phaseAt.UTC().Format(time.RFC3339))
		case step, ok := <-steps:
			if !ok {
				return buildstep.Step{}, errors.New("the watch of the ask's step ended")
			}
			if seen := stepSeenOf(ctx); seen != nil {
				seen(step)
			}
			if step.State == buildstep.StepAgain {
				// A run ended on a transport fault and the bus gives the ask again: not an end. The bound runs on from the
				// time it is given again, so a bus that never gives it says so (ADR 0664 §3).
				agains = append(agains, step.Node)
				running, since, phase, phaseAt = step.Node, step.Started, "given again after a transport fault", step.AgainAt
				if k.Bound > 0 {
					if timer != nil {
						timer.Stop()
					}
					timer = time.NewTimer(max(time.Until(step.AgainAt.Add(k.Bound)), 0))
					bound = timer.C
				}
				continue
			}
			if buildstep.Ended(step) {
				if step.Passing && step.State == buildstep.StepFailed && (len(agains) > 0 || step.Run > 1) {
					// The step's own count of runs (ADR 0664 §2), not only the ones this wait saw: a builder that started again
					// reads the key's last value and none of the runs before it.
					step.Reason = fmt.Sprintf(buildstep.TransportFaultWords+"%d runs (%s): %s", max(step.Run, len(agains)+1), strings.Join(append(agains, step.Node), ", "), step.Reason)
				}
				return step, nil
			}
			if k.Bound <= 0 {
				continue
			}
			newRun := step.Node != running || !step.Started.Equal(since)
			if !newRun && step.Phase == phase && step.PhaseAt.Equal(phaseAt) {
				continue
			}
			// A node took the ask, or took it again after its run stopped (another node, or the same node's build module
			// replaced mid-run: a new start), or its step entered a phase. The bound is on standing still: it runs from the
			// step's last movement (its phase's start, else its start), so a step that keeps moving is waited for, up to
			// three bounds from its start (issue 1930).
			running, since, phase, phaseAt = step.Node, step.Started, step.Phase, step.PhaseAt
			last := step.Started
			if step.PhaseAt.After(last) {
				last = step.PhaseAt
			}
			left := k.Bound
			if !last.IsZero() {
				left = time.Until(last.Add(k.Bound))
			}
			if timer != nil {
				timer.Stop()
			}
			timer = time.NewTimer(max(left, 0))
			bound = timer.C
			if newRun {
				if capTimer != nil {
					capTimer.Stop()
				}
				outer := 3 * k.Bound
				if !step.Started.IsZero() {
					outer = time.Until(step.Started.Add(3 * k.Bound))
				}
				capTimer = time.NewTimer(max(outer, 0))
				outerCap = capTimer.C
			}
		}
	}
}

type stepSeenKey struct{}

// WithStepSeen is ctx carrying seen, which Asker.Wait hands every value the ask's step key takes while it waits: the node
// that took it, each phase it says while running (buildstep.Step.Phase), and its end. A try names its asks' progress so
// (issue 1890). seen runs in the waiting goroutine and must not block. A seen ctx already carries is kept: both are handed
// every value, the earlier first (a check's begin bound watches the steps of a try that names them).
func WithStepSeen(ctx context.Context, seen func(buildstep.Step)) context.Context {
	if before := stepSeenOf(ctx); before != nil {
		later := seen
		seen = func(s buildstep.Step) { before(s); later(s) }
	}
	return context.WithValue(ctx, stepSeenKey{}, seen)
}

func stepSeenOf(ctx context.Context) func(buildstep.Step) {
	seen, _ := ctx.Value(stepSeenKey{}).(func(buildstep.Step))
	return seen
}

// Ask puts a and waits for its step; a step that did not pass is an error naming its node, its state and its reason.
func (k Asker) Ask(ctx context.Context, a buildstep.StepAsk) (buildstep.Step, error) {
	if err := k.Put(ctx, a); err != nil {
		return buildstep.Step{}, err
	}
	return k.Passed(k.Wait(ctx, a))
}

// Passed is step when it passed, else an error naming where it ended and why.
func (k Asker) Passed(step buildstep.Step, err error) (buildstep.Step, error) {
	if err != nil {
		return step, err
	}
	if step.State != buildstep.StepPassed {
		return step, fmt.Errorf("the ask %s on %s %s: %s", step.ID, step.Node, step.State, step.Reason)
	}
	return step, nil
}

// JetStreamAsks is the bus side of an Asker: a JetStream publish de-duplicated by the ask's id (Nats-Msg-Id), and a watch
// of `*.<ask id>` in the build module's steps bucket.
type JetStreamAsks struct {
	JS jetstream.JetStream
}

// PublishAsk is AskPublisher's.
func (s JetStreamAsks) PublishAsk(ctx context.Context, subject, id string, data []byte) error {
	msg := &nats.Msg{Subject: subject, Data: data, Header: nats.Header{}}
	msg.Header.Set(jetstream.MsgIDHeader, id)
	_, err := s.JS.PublishMsg(ctx, msg)
	return err
}

// WatchAsk is AskWaiter's: the watch hands over what the keys hold now, so a step that ended before it began is read at
// once, and every change after.
func (s JetStreamAsks) WatchAsk(ctx context.Context, module, askID string) (<-chan buildstep.Step, error) {
	pattern, err := buildstep.AskStepWatch(askID)
	if err != nil {
		return nil, err
	}
	kv, err := s.JS.KeyValue(ctx, StepsBucket(module))
	if err != nil {
		return nil, fmt.Errorf("the bucket %s: %w", StepsBucket(module), err)
	}
	w, err := kv.Watch(ctx, pattern)
	if err != nil {
		return nil, err
	}
	out := make(chan buildstep.Step)
	go func() {
		defer close(out)
		defer w.Stop()
		for {
			select {
			case <-ctx.Done():
				return
			case e, ok := <-w.Updates():
				if !ok {
					return
				}
				if e == nil || e.Operation() != jetstream.KeyValuePut {
					continue
				}
				step, err := buildstep.DecodeStep(e.Value())
				if err != nil {
					continue // not a step (a node's pause key never matches an ask id; anything else is not this ask's)
				}
				select {
				case out <- step:
				case <-ctx.Done():
					return
				}
			}
		}
	}()
	return out, nil
}

// FakeAsks is a test double of the bus for an asker's tests: each ask put is kept and, when Answer is set, its step is
// written at once with what Answer says a build node wrote (running or ended). Write writes a step later. Safe for
// concurrent use.
type FakeAsks struct {
	Answer func(a buildstep.StepAsk) buildstep.Step

	mu       sync.Mutex
	asks     []buildstep.StepAsk
	steps    map[string][]buildstep.Step
	watchers map[string][]chan buildstep.Step
}

// PublishAsk is AskPublisher's.
func (f *FakeAsks) PublishAsk(_ context.Context, _, id string, data []byte) error {
	a, err := buildstep.DecodeStepAsk(data)
	if err != nil {
		return err
	}
	f.mu.Lock()
	f.asks = append(f.asks, a)
	f.mu.Unlock()
	if f.Answer != nil {
		f.Write(f.Answer(a))
	}
	return nil
}

// Write writes an ask's step, as a build node does.
func (f *FakeAsks) Write(step buildstep.Step) {
	f.mu.Lock()
	if f.steps == nil {
		f.steps, f.watchers = map[string][]buildstep.Step{}, map[string][]chan buildstep.Step{}
	}
	f.steps[step.ID] = append(f.steps[step.ID], step)
	ws := append([]chan buildstep.Step(nil), f.watchers[step.ID]...)
	f.mu.Unlock()
	for _, w := range ws {
		w <- step
	}
}

// End is Write: kept for the tests that end a step.
func (f *FakeAsks) End(step buildstep.Step) { f.Write(step) }

// WatchAsk is AskWaiter's: what was written so far, then every write.
func (f *FakeAsks) WatchAsk(ctx context.Context, _ string, askID string) (<-chan buildstep.Step, error) {
	out := make(chan buildstep.Step, 64)
	f.mu.Lock()
	if f.steps == nil {
		f.steps, f.watchers = map[string][]buildstep.Step{}, map[string][]chan buildstep.Step{}
	}
	for _, s := range f.steps[askID] {
		out <- s
	}
	f.watchers[askID] = append(f.watchers[askID], out)
	f.mu.Unlock()
	go func() {
		<-ctx.Done()
		f.mu.Lock()
		ws := f.watchers[askID]
		for i, w := range ws {
			if w == out {
				f.watchers[askID] = append(ws[:i], ws[i+1:]...)
				break
			}
		}
		f.mu.Unlock()
	}()
	return out, nil
}

// Asks are the asks put so far.
func (f *FakeAsks) Asks() []buildstep.StepAsk {
	f.mu.Lock()
	defer f.mu.Unlock()
	return append([]buildstep.StepAsk(nil), f.asks...)
}