mesh-delivery/modules/mesh-delivery-builder/internal/build/buildagain.go

package build

import (
	"context"
	"fmt"
	"maps"
	"strings"
	"time"

	"git.novox.be/novox/mesh-sdk/go/delivery"
	"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
)

// **A failed key is built again by hand** (novox/hq issue 1126 note 72; ADR 0429 clause 18: "a fingerprint is built again
// only when its build key failed, as a new revision of the same key"). The builder does it by itself whenever it looks at a
// module again (a modules value, a change of the toolchains, artifacts-gone, a moved trunk): Claimer.Claim writes a failed
// key building as a new revision and the build asks the build seats. Nothing looks at an unchanged module on its own, so a key
// that failed (a node refused the build, a toolchain was missing) stayed failed until the module changed. The tool
// builder_build_again (BuildAgain) makes the builder look at one module now, which does exactly that and no more: it keeps
// the claim's rules, so a key failed for a reason the builder never retries on the same fingerprint stays refused.

// BuildAgainResult is the builder's answer to a request to build a key again.
type BuildAgainResult struct {
	// Refused is why nothing was done, "" when the key was written building as a new revision.
	Refused string
	// Revision is the revision the claim wrote; Asks are the ask ids put by language, as far as they were put within the bound.
	Revision uint64
	Asks     map[string]string
	// Pending says the builder had not claimed the key, or not put every ask, when it answered: not a refusal.
	Pending string
	// Failed is the error of the key when the build ended failed before its asks were put (a plan that cannot be read, a
	// toolchain not agreed): the new revision was written, and nothing was asked.
	Failed string
}

// handWatch follows one request to build a key again until its asks are put.
type handWatch struct {
	rev   uint64
	lanes int
	asks  map[string]string
	done  chan struct{}
	shut  bool
	// ended is set when the look ended before the asks were all put.
	ended bool
}

// settle closes done once the claim is written and every lane has put its ask. Call with b.mu held.
func (w *handWatch) settle() {
	if !w.shut && w.rev > 0 && w.lanes >= 0 && len(w.asks) >= w.lanes {
		w.shut = true
		close(w.done)
	}
}

func (b *Builder) hand(key string, f func(w *handWatch)) {
	b.mu.Lock()
	defer b.mu.Unlock()
	if w := b.hands[key]; w != nil {
		f(w)
	}
}

func (b *Builder) handClaimed(key string, rev uint64) {
	b.hand(key, func(w *handWatch) { w.rev = rev; w.settle() })
}

func (b *Builder) handLanes(key string, n int) {
	b.hand(key, func(w *handWatch) { w.lanes = n; w.settle() })
}

func (b *Builder) handAsk(key, language, id string) {
	b.hand(key, func(w *handWatch) { w.asks[language] = id; w.settle() })
}

// handEnded ends the watch of a request whose look ended: it wrote no claim, or the build ended.
func (b *Builder) handEnded(key string) {
	b.hand(key, func(w *handWatch) {
		if !w.shut {
			w.shut, w.ended = true, true
			close(w.done)
		}
	})
}

// BuildAgain builds the failed key of module at fp again, as the builder does by itself: the key is written building as a
// new revision and the build module asks are put on the build seats. It answers once the asks are put, or at bound. A key
// that is not failed is refused with why: done (nothing to do) or waiting (builder_ask_again asks its waiting asks again).
// ctx is the builder's own context: the build it starts lives on it.
func (b *Builder) BuildAgain(ctx context.Context, module, fp, why, by string, bound time.Duration) BuildAgainResult {
	b.init()
	key, err := delivery.BuilderBuildKey(module, fp)
	if err != nil {
		return BuildAgainResult{Refused: err.Error()}
	}
	value, _, found, err := b.Claimer.Store.Get(ctx, key)
	if err != nil {
		return BuildAgainResult{Refused: fmt.Sprintf("%s cannot be read: %v", key, err)}
	}
	if !found {
		return BuildAgainResult{Refused: fmt.Sprintf("%s has no build key: nothing failed. The builder builds a fingerprint by itself once the indexer writes it to its module", key)}
	}
	cur, err := delivery.DecodeBuilderBuild(value)
	if err != nil {
		return BuildAgainResult{Refused: fmt.Sprintf("%s holds what is not a build: %v", key, err)}
	}
	switch {
	case cur.State == delivery.BuildDone:
		return BuildAgainResult{Refused: fmt.Sprintf("%s is done: nothing to do", key)}
	case cur.State == delivery.BuildBuilding || cur.State == delivery.BuildQueued:
		return BuildAgainResult{Refused: fmt.Sprintf("%s is waiting (%s: %s): %s asks its waiting asks again", key, cur.State, cur.Step, AskAgainTool)}
	case cur.State != delivery.BuildFailed:
		return BuildAgainResult{Refused: fmt.Sprintf("%s is %q, not failed", key, cur.State)}
	case delivery.FailedUntilChanged(cur.Error) || strings.HasPrefix(cur.Error, buildrun.DigestsDifferPrefix):
		return BuildAgainResult{Refused: fmt.Sprintf("%s failed for a reason the builder does not retry on the same fingerprint, it waits for another fingerprint: %s", key, cur.Error)}
	}
	// One critical section: the key is refused when it is looked at or asked already, the module still has the fingerprint,
	// and the watch and the look are registered before anything else can take the key.
	w := &handWatch{lanes: -1, asks: map[string]string{}, done: make(chan struct{})}
	b.mu.Lock()
	m, known := b.known[module]
	switch {
	case !known || m.FP2 != fp:
		b.mu.Unlock()
		return BuildAgainResult{Refused: fmt.Sprintf("module %s does not have the fingerprint %s now (a newer one superseded it, or its modules value is not read yet): the builder builds the module's current fingerprint", module, fp)}
	case m.Manifest == "":
		b.mu.Unlock()
		return BuildAgainResult{Refused: fmt.Sprintf("module %s names no manifest at %s yet, so the builder looks at nothing", module, fp)}
	case b.inflight[key] || b.hands[key] != nil:
		b.mu.Unlock()
		return BuildAgainResult{Refused: fmt.Sprintf("%s is being looked at by this builder now (or asked again by hand): wait for it to end", key)}
	}
	b.hands[key] = w
	b.inflight[key] = true
	b.mu.Unlock()
	defer func() {
		b.mu.Lock()
		if b.hands[key] == w {
			delete(b.hands, key)
		}
		b.mu.Unlock()
	}()
	b.logf("%s: failed (%s); built again by hand, asked by %s: %s", key, cur.Error, by, why)
	b.startOnce(ctx, key, module, m)
	wait, stop := context.WithTimeout(ctx, bound) // a context's bound for a person's answer (ADR 0437 §8)
	defer stop()
	select {
	case <-w.done:
	case <-wait.Done():
	}
	b.mu.Lock()
	res := BuildAgainResult{Revision: w.rev, Asks: maps.Clone(w.asks)}
	ended, shut, lanes := w.ended, w.shut, w.lanes
	b.mu.Unlock()
	switch {
	case res.Revision == 0 && shut:
		res.Refused = fmt.Sprintf("the builder looked at %s and wrote nothing (a toolchain moved, the claim was lost or could not be written): its log says why", key)
	case res.Revision == 0:
		res.Pending = fmt.Sprintf("the builder has not claimed %s within %s: it waits behind other builds; builder_stuck shows where it stands", key, bound)
	case ended && len(res.Asks) < max(lanes, 0)+boolInt(lanes < 0):
		// The look ended before every ask was put: say how the key stands, not that it was asked.
		if v, _, found, err := b.Claimer.Store.Get(ctx, key); err == nil && found {
			if cur, err := delivery.DecodeBuilderBuild(v); err == nil {
				switch cur.State {
				case delivery.BuildFailed:
					res.Failed = cur.Error
				case delivery.BuildDone:
				default:
					res.Pending = fmt.Sprintf("the look ended with the key %s (%s) before every ask was put: its log says why", cur.State, cur.Step)
				}
			}
		}
	case !shut:
		res.Pending = fmt.Sprintf("the key is building at revision %d, and not every ask was put within %s; builder_stuck shows the asks", res.Revision, bound)
	}
	return res
}

func boolInt(b bool) int {
	if b {
		return 1
	}
	return 0
}