mesh-delivery/modules/mesh-delivery-builder/internal/build/stuck.go

package build

import (
	"encoding/json"
	"errors"
	"fmt"
	"sort"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"git.novox.be/novox/mesh-sdk/go/delivery"
	"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
)

// The builder's tools (novox/hq issue 1945; the instruction "tools are how a module is spoken to"): builder_stuck reads
// every key that waits on an ask, builder_ask_again asks a key's build again now. The tools bundle serves them through
// the tool runner; the builder's process serves no tool (ADR 0437 §4), so builder_ask_again writes the ask into the
// builder's state reasks, which the builder watches and answers there.
const (
	StuckTool    = "builder_stuck"
	AskAgainTool = "builder_ask_again"
	// ReAsksState is the builder's state of asks to ask again: the bucket mesh-delivery-builder_reasks, keyed as the build key.
	ReAsksState  = "reasks"
	ReAsksBucket = delivery.BuilderModule + "_" + ReAsksState
)

// ReAsk is a value of the reasks state: why a person asked the key's build again, who and when, and, once the builder
// read it, its answer.
type ReAsk struct {
	V        int       `json:"v"`
	Why      string    `json:"why"`
	By       string    `json:"by,omitempty"`
	At       time.Time `json:"at"`
	Answer   string    `json:"answer,omitempty"`
	Answered time.Time `json:"answered,omitzero"`
}

// DecodeReAsk reads a reasks value.
func DecodeReAsk(data []byte) (ReAsk, error) {
	var r ReAsk
	if err := json.Unmarshal(data, &r); err != nil {
		return r, err
	}
	if r.V != 1 {
		return r, fmt.Errorf("a reasks value of version %d, not 1", r.V)
	}
	return r, nil
}

// NewReAsk is the value builder_ask_again writes for a module's fingerprint: its key and the value.
func NewReAsk(module, fp, why, by string, at time.Time) (string, []byte, error) {
	key, err := delivery.BuilderBuildKey(module, fp)
	if err != nil {
		return "", nil, err
	}
	if strings.TrimSpace(why) == "" {
		return "", nil, errors.New("say why the build is asked again (why)")
	}
	data, err := json.Marshal(ReAsk{V: 1, Why: strings.TrimSpace(why), By: by, At: at.UTC()})
	return key, data, err
}

// StateRead is a state the tools read: its keys, and a key's value with when it was written (zero when not known).
type StateRead struct {
	Keys func() ([]string, error)
	Get  func(key string) (json.RawMessage, time.Time, error)
	// Written is when the key's current value was written, from the state's history: one more round trip, so the tools ask it
	// only for a row they keep, and only when Get did not say.
	Written func(key string) time.Time
}

// StuckAsk is one ask a key waits on.
type StuckAsk struct {
	Language    string `json:"language"`
	BuildModule string `json:"build_module"`
	ID          string `json:"id"`
	Asking      int    `json:"asking"`
	// Asked is when the ask was put, read from its id; Step is what its build module's steps hold of it: none, or its
	// state on its node since its start.
	Asked time.Time `json:"asked,omitzero"`
	Step  string    `json:"step"`
}

// Stuck is one build key that waits on an ask.
type Stuck struct {
	Key      string     `json:"key"`
	Module   string     `json:"module"`
	FP       string     `json:"fp"`
	State    string     `json:"state"`
	Step     string     `json:"step"`
	By       string     `json:"by"`
	Since    time.Time  `json:"since,omitzero"`
	Age      string     `json:"age,omitempty"`
	Asks     []StuckAsk `json:"asks"`
	ReAsk    *ReAsk     `json:"reask,omitempty"`
	Unasked  bool       `json:"unasked,omitempty"`
	NoneStep int        `json:"asks_with_no_step"`
}

// StuckReader reads what builder_stuck answers.
type StuckReader struct {
	Builds StateRead
	ReAsks StateRead
	// Steps are the build modules' steps states, by build module.
	Steps map[string]StateRead
	// Hearing is the builder's state hearing: whether it hears DEAD_LETTERS (issue 734); without it none is said.
	Hearing StateRead
	Now     func() time.Time
	// Logf, when set, gets the one line a call logs: the values it read and the time it took.
	Logf func(format string, args ...any)
}

// ReadWorkers is how many values builder_stuck asks the bus for at once, and MaxReads how many it reads in a call (issue 1827:
// every value is a round trip through the runtime, so one after the other a few thousand keys outlast the call's 30 s).
// A copy of the delivery view's and the checker's: a few lines are not worth a group delivery through mesh-sdk.
const (
	ReadWorkers = 16
	MaxReads    = 1000
)

// parallel runs read(i) for i in [0, n), at most ReadWorkers at a time, and answers the first error.
func parallel(n int, read func(i int) error) error {
	var (
		next   atomic.Int64
		failed atomic.Bool
		once   sync.Once
		first  error
		wg     sync.WaitGroup
	)
	for w := 0; w < min(ReadWorkers, n); w++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			for !failed.Load() {
				i := int(next.Add(1) - 1)
				if i >= n {
					return
				}
				if err := read(i); err != nil {
					once.Do(func() { first = err })
					failed.Store(true)
				}
			}
		}()
	}
	wg.Wait()
	return first
}

// Stuck is every key building or queued with its asks, oldest first; with module, that module's only.
// deadLetters says whether the builder hears the asks the build seats give up on (issue 734), and why not: deaf, a lane
// whose ask is given up waits for its check bound (buildrun.HearingHealth over the kept hearing).
func (r StuckReader) deadLetters() map[string]any {
	if r.Hearing.Get == nil {
		return map[string]any{"heard": false, "why": "the builder's state hearing is not readable here"}
	}
	raw, _, err := r.Hearing.Get(buildrun.HearingKey)
	if err != nil {
		return map[string]any{"heard": false, "why": "the builder's state hearing cannot be read: " + err.Error()}
	}
	a := buildrun.HearingHealth(raw)
	return map[string]any{"heard": a.Healthy, "why": a.Why}
}

func (r StuckReader) Stuck(module string) (map[string]any, error) {
	started := time.Now()
	reads := 0
	defer func() {
		if r.Logf != nil {
			r.Logf("builder_stuck read %d values in %s", reads, time.Since(started).Round(time.Millisecond))
		}
	}()
	keys, err := r.Builds.Keys()
	if err != nil {
		return nil, err
	}
	now := time.Now
	if r.Now != nil {
		now = r.Now
	}
	// Steps are found by the ask's id, which their key ends in: one index per build module, no value read until an ask names it.
	stepKeys := map[string]map[string]string{}
	for bm, s := range r.Steps {
		ks, err := s.Keys()
		if err != nil {
			return nil, fmt.Errorf("the steps of %s: %w", bm, err)
		}
		byID := make(map[string]string, len(ks))
		for _, k := range ks {
			if _, id, err := delivery.ParseStepKey(k); err == nil {
				byID[id] = k
			}
		}
		stepKeys[bm] = byID
	}
	// Narrowed by the key (the module) before any value is read; the state is in the value, so every matching key is read, in
	// parallel, at most MaxReads of them.
	type parsed struct{ key, mod, fp string }
	var matching []parsed
	for _, key := range keys {
		mod, fp, err := delivery.ParseBuilderBuildKey(key)
		if err != nil || (module != "" && mod != module) {
			continue
		}
		matching = append(matching, parsed{key, mod, fp})
	}
	sort.Slice(matching, func(i, j int) bool { return matching[i].key < matching[j].key })
	cut := len(matching) > MaxReads
	total := len(matching)
	if cut {
		matching = matching[:MaxReads]
	}
	values := make([]json.RawMessage, len(matching))
	ats := make([]time.Time, len(matching))
	if err := parallel(len(matching), func(i int) error {
		raw, at, err := r.Builds.Get(matching[i].key)
		if err == nil {
			values[i], ats[i] = raw, at
		}
		return nil
	}); err != nil {
		return nil, err
	}
	reads += len(matching)
	out := []Stuck{}
	for i, m := range matching {
		key, mod, fp := m.key, m.mod, m.fp
		raw, at := values[i], ats[i]
		if raw == nil {
			continue
		}
		v, err := delivery.DecodeBuilderBuild(raw)
		if err != nil || (v.State != delivery.BuildBuilding && v.State != delivery.BuildQueued) {
			continue
		}
		if at.IsZero() && r.Builds.Written != nil {
			at = r.Builds.Written(key)
		}
		s := Stuck{Key: key, Module: mod, FP: fp, State: v.State, Step: v.Step, By: v.By, Since: at, Asks: []StuckAsk{}}
		if !at.IsZero() {
			s.Age = now().Sub(at).Round(time.Second).String()
		}
		for language, id := range v.Asks {
			bm, _ := delivery.BuildModuleOf(language)
			a := StuckAsk{Language: language, BuildModule: bm, ID: id, Asking: askingOf(v.Step, bm), Step: "none"}
			if t, ok := idTime(id); ok {
				a.Asked = t
			}
			if st, ok := r.Steps[bm]; ok {
				if k, ok := stepKeys[bm][id]; ok {
					reads++
					if raw, _, err := st.Get(k); err == nil && raw != nil {
						if step, err := delivery.DecodeStep(raw); err == nil {
							a.Step = fmt.Sprintf("%s on %s since %s", step.State, step.Node, step.Started.UTC().Format(time.RFC3339))
						}
					}
				}
			}
			if a.Step == "none" {
				s.NoneStep++
			}
			s.Asks = append(s.Asks, a)
		}
		sort.Slice(s.Asks, func(i, j int) bool { return s.Asks[i].Language < s.Asks[j].Language })
		s.Unasked = v.State == delivery.BuildBuilding && len(v.Asks) == 0
		if r.ReAsks.Get != nil {
			reads++
			if raw, _, err := r.ReAsks.Get(key); err == nil && raw != nil {
				if ra, err := DecodeReAsk(raw); err == nil {
					s.ReAsk = &ra
				}
			}
		}
		out = append(out, s)
	}
	sort.Slice(out, func(i, j int) bool {
		if !out[i].Since.Equal(out[j].Since) {
			return out[i].Since.Before(out[j].Since)
		}
		return out[i].Key < out[j].Key
	})
	answer := map[string]any{"stuck": out, "count": len(out),
		"delivery_counts": "not readable by the builder: the build seats' workers' information is granted to the delivery view alone (its backlog); an ask given up on is told by the builder at its check bound and asked again"}
	answer["dead_letters"] = r.deadLetters()
	if cut {
		answer["cut"], answer["note"] = true, fmt.Sprintf("read %d of %d build keys in key order and stopped; give a module to read its keys alone", MaxReads, total)
	}
	return answer, nil
}

// StuckBuildModules are the build modules whose steps builder_stuck reads.
var StuckBuildModules = []string{buildrun.GoBuildModule, buildrun.NodeBuildModule, ImageBuildModule}

// idTime is when a step id was made (buildstep.NewID: the time in milliseconds in base 36, then six hex digits).
func idTime(id string) (time.Time, bool) {
	if len(id) <= 6 {
		return time.Time{}, false
	}
	ms, err := strconv.ParseInt(id[:len(id)-6], 36, 64)
	if err != nil || ms <= 0 {
		return time.Time{}, false
	}
	return time.UnixMilli(ms).UTC(), true
}