mesh-delivery/modules/mesh-delivery-builder/internal/build/stuck.go

package build

import (
	"encoding/json"
	"errors"
	"fmt"
	"sort"
	"strconv"
	"strings"
	"time"

	"git.novox.be/novox/mesh-sdk/go/delivery"
	"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
)

// The builder's tools (novox/hq issue 1945; the instruction "tools are how a module is spoken to"): builder_stuck reads
// every key that waits on an ask, builder_ask_again asks a key's build again now. The tools bundle serves them through
// the tool runner; the builder's process serves no tool (ADR 0437 §4), so builder_ask_again writes the ask into the
// builder's state reasks, which the builder watches and answers there.
const (
	StuckTool    = "builder_stuck"
	AskAgainTool = "builder_ask_again"
	// ReAsksState is the builder's state of asks to ask again: the bucket mesh-delivery-builder_reasks, keyed as the build key.
	ReAsksState  = "reasks"
	ReAsksBucket = delivery.BuilderModule + "_" + ReAsksState
)

// ReAsk is a value of the reasks state: why a person asked the key's build again, who and when, and, once the builder
// read it, its answer.
type ReAsk struct {
	V        int       `json:"v"`
	Why      string    `json:"why"`
	By       string    `json:"by,omitempty"`
	At       time.Time `json:"at"`
	Answer   string    `json:"answer,omitempty"`
	Answered time.Time `json:"answered,omitzero"`
}

// DecodeReAsk reads a reasks value.
func DecodeReAsk(data []byte) (ReAsk, error) {
	var r ReAsk
	if err := json.Unmarshal(data, &r); err != nil {
		return r, err
	}
	if r.V != 1 {
		return r, fmt.Errorf("a reasks value of version %d, not 1", r.V)
	}
	return r, nil
}

// NewReAsk is the value builder_ask_again writes for a module's fingerprint: its key and the value.
func NewReAsk(module, fp, why, by string, at time.Time) (string, []byte, error) {
	key, err := delivery.BuilderBuildKey(module, fp)
	if err != nil {
		return "", nil, err
	}
	if strings.TrimSpace(why) == "" {
		return "", nil, errors.New("say why the build is asked again (why)")
	}
	data, err := json.Marshal(ReAsk{V: 1, Why: strings.TrimSpace(why), By: by, At: at.UTC()})
	return key, data, err
}

// StateRead is a state the tools read: its keys, and a key's value with when it was written (zero when not known).
type StateRead struct {
	Keys func() ([]string, error)
	Get  func(key string) (json.RawMessage, time.Time, error)
}

// StuckAsk is one ask a key waits on.
type StuckAsk struct {
	Language    string `json:"language"`
	BuildModule string `json:"build_module"`
	ID          string `json:"id"`
	Asking      int    `json:"asking"`
	// Asked is when the ask was put, read from its id; Step is what its build module's steps hold of it: none, or its
	// state on its node since its start.
	Asked time.Time `json:"asked,omitzero"`
	Step  string    `json:"step"`
}

// Stuck is one build key that waits on an ask.
type Stuck struct {
	Key      string     `json:"key"`
	Module   string     `json:"module"`
	FP       string     `json:"fp"`
	State    string     `json:"state"`
	Step     string     `json:"step"`
	By       string     `json:"by"`
	Since    time.Time  `json:"since,omitzero"`
	Age      string     `json:"age,omitempty"`
	Asks     []StuckAsk `json:"asks"`
	ReAsk    *ReAsk     `json:"reask,omitempty"`
	Unasked  bool       `json:"unasked,omitempty"`
	NoneStep int        `json:"asks_with_no_step"`
}

// StuckReader reads what builder_stuck answers.
type StuckReader struct {
	Builds StateRead
	ReAsks StateRead
	// Steps are the build modules' steps states, by build module.
	Steps map[string]StateRead
	Now   func() time.Time
}

// Stuck is every key building or queued with its asks, oldest first; with module, that module's only.
func (r StuckReader) Stuck(module string) (map[string]any, error) {
	keys, err := r.Builds.Keys()
	if err != nil {
		return nil, err
	}
	now := time.Now
	if r.Now != nil {
		now = r.Now
	}
	stepKeys := map[string][]string{}
	for bm, s := range r.Steps {
		ks, err := s.Keys()
		if err != nil {
			return nil, fmt.Errorf("the steps of %s: %w", bm, err)
		}
		stepKeys[bm] = ks
	}
	out := []Stuck{}
	for _, key := range keys {
		mod, fp, err := delivery.ParseBuilderBuildKey(key)
		if err != nil || (module != "" && mod != module) {
			continue
		}
		raw, at, err := r.Builds.Get(key)
		if err != nil || raw == nil {
			continue
		}
		v, err := delivery.DecodeBuilderBuild(raw)
		if err != nil || (v.State != delivery.BuildBuilding && v.State != delivery.BuildQueued) {
			continue
		}
		s := Stuck{Key: key, Module: mod, FP: fp, State: v.State, Step: v.Step, By: v.By, Since: at, Asks: []StuckAsk{}}
		if !at.IsZero() {
			s.Age = now().Sub(at).Round(time.Second).String()
		}
		for language, id := range v.Asks {
			bm, _ := delivery.BuildModuleOf(language)
			a := StuckAsk{Language: language, BuildModule: bm, ID: id, Asking: askingOf(v.Step, bm), Step: "none"}
			if t, ok := idTime(id); ok {
				a.Asked = t
			}
			if st, ok := r.Steps[bm]; ok {
				for _, k := range stepKeys[bm] {
					if !strings.HasSuffix(k, "."+id) {
						continue
					}
					if raw, _, err := st.Get(k); err == nil && raw != nil {
						if step, err := delivery.DecodeStep(raw); err == nil {
							a.Step = fmt.Sprintf("%s on %s since %s", step.State, step.Node, step.Started.UTC().Format(time.RFC3339))
						}
					}
				}
			}
			if a.Step == "none" {
				s.NoneStep++
			}
			s.Asks = append(s.Asks, a)
		}
		sort.Slice(s.Asks, func(i, j int) bool { return s.Asks[i].Language < s.Asks[j].Language })
		s.Unasked = v.State == delivery.BuildBuilding && len(v.Asks) == 0
		if r.ReAsks.Get != nil {
			if raw, _, err := r.ReAsks.Get(key); err == nil && raw != nil {
				if ra, err := DecodeReAsk(raw); err == nil {
					s.ReAsk = &ra
				}
			}
		}
		out = append(out, s)
	}
	sort.Slice(out, func(i, j int) bool {
		if !out[i].Since.Equal(out[j].Since) {
			return out[i].Since.Before(out[j].Since)
		}
		return out[i].Key < out[j].Key
	})
	return map[string]any{"stuck": out, "count": len(out),
		"delivery_counts": "not readable by the builder: the build seats' workers' information is granted to the delivery view alone (its backlog); an ask given up on is told by the builder at its check bound and asked again"}, nil
}

// StuckBuildModules are the build modules whose steps builder_stuck reads.
var StuckBuildModules = []string{buildrun.GoBuildModule, buildrun.NodeBuildModule, ImageBuildModule}

// idTime is when a step id was made (buildstep.NewID: the time in milliseconds in base 36, then six hex digits).
func idTime(id string) (time.Time, bool) {
	if len(id) <= 6 {
		return time.Time{}, false
	}
	ms, err := strconv.ParseInt(id[:len(id)-6], 36, 64)
	if err != nil || ms <= 0 {
		return time.Time{}, false
	}
	return time.UnixMilli(ms).UTC(), true
}