mesh-delivery/modules/mesh-delivery-builder/internal/build/stuck.go

package build

import (
	"mesh-delivery-builder/internal/versions"

	"encoding/json"
	"errors"
	"fmt"
	"sort"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
	"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
	"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
)

// The builder's tools (novox/hq issue 1945; the instruction "tools are how a module is spoken to"): builder_stuck reads
// every key that waits on an ask, builder_ask_again asks a key's build again now. The tools bundle serves them through
// the tool runner; the builder's process serves no tool (ADR 0437 §4), so builder_ask_again writes the ask into the
// builder's state reasks, which the builder watches and answers there.
const (
	StuckTool    = "builder_stuck"
	AskAgainTool = "builder_ask_again"
	// BuildAgainTool builds a failed key again by hand (issue 1126 note 72): it writes a reasks value of Mode ModeBuild.
	BuildAgainTool = "builder_build_again"
	// AnnounceAgainTool writes a done key again unchanged, so the controller takes it in and composes again: a reasks value
	// of Mode ModeAnnounce.
	AnnounceAgainTool = "builder_announce_again"
	// ReAsksState is the builder's state of asks to ask again: the bucket mesh-delivery-builder_reasks, keyed as the build key.
	ReAsksState = "reasks"
)

// ReAsk is a value of the reasks state: why a person asked the key's build again, who and when, and, once the builder
// read it, its answer.
type ReAsk struct {
	V        int       `json:"v"`
	Why      string    `json:"why"`
	By       string    `json:"by,omitempty"`
	At       time.Time `json:"at"`
	Answer   string    `json:"answer,omitempty"`
	Answered time.Time `json:"answered,omitzero"`
	// Mode is ModeBuild for a request to build a failed key again (BuildAgainTool); empty for an ask asked again.
	Mode   string `json:"mode,omitempty"`
	Module string `json:"module,omitempty"`
	FP     string `json:"fp,omitempty"`
	// Refused, Revision and Asks are the builder's answer to a ModeBuild request: why it did nothing, or the key's new
	// revision (building) and the ask ids it put by language.
	Refused bool `json:"refused,omitempty"`
	// Pending says the builder answered before the key was claimed or all its asks were put: neither a refusal nor done.
	// Failed is the error of a key written building that failed before its asks were put.
	Pending  bool              `json:"pending,omitempty"`
	Failed   string            `json:"failed,omitempty"`
	Revision uint64            `json:"revision,omitempty"`
	Asks     map[string]string `json:"asks,omitempty"`
	// Name is the numbered name (`<module>/<artifact>`) of a ModeFloorAgain value; Version is a ModeRepublish value's
	// request and Queued the builder's answer to it: the versions it queued, `<module> <version>`.
	Name    string   `json:"name,omitempty"`
	Version string   `json:"version,omitempty"`
	Queued  []string `json:"queued,omitempty"`
}

// ModeRepublish is the Mode of a reasks value of builder_publish_again (design 47 G §588): Module and Version, each
// optional, name what to publish again; the builder answers with the versions it queued (Queued).
const ModeRepublish = "republish"

// ModeFloorAgain is the Mode of a reasks value of builder_floor_again (design 51 *The floor*).
const ModeFloorAgain = "floor-again"

// NewRepublish is the value builder_publish_again writes (design 47 G §588): its key, `republish.<unix nanoseconds>`
// (a call is one value, never a build key), and the value. Module and version are each optional.
func NewRepublish(module, version, why, by string, at time.Time) (string, []byte, error) {
	module, version = strings.TrimSpace(module), strings.TrimSpace(version)
	if version != "" && module == "" {
		return "", nil, errors.New("a version is published again under its module: give module with version")
	}
	if strings.TrimSpace(why) == "" {
		return "", nil, errors.New("say why it is published again (why)")
	}
	data, err := json.Marshal(ReAsk{V: 1, Why: strings.TrimSpace(why), By: by, At: at.UTC(), Mode: ModeRepublish, Module: module, Version: version})
	return "republish." + strconv.FormatInt(at.UnixNano(), 10), data, err
}

// NewFloorAgain is the value builder_floor_again writes for a name: the key `floor.<name escaped>`, never a build key,
// and the value.
func NewFloorAgain(name, why, by string, at time.Time) (string, []byte, error) {
	name = strings.TrimSpace(name)
	if module, artifact, ok := strings.Cut(name, "/"); !ok || module == "" || artifact == "" {
		return "", nil, fmt.Errorf("name %q is not a numbered name, <module>/<artifact>, as builder_versions lists it", name)
	}
	if strings.TrimSpace(why) == "" {
		return "", nil, errors.New("say why the floor is asked again (why)")
	}
	data, err := json.Marshal(ReAsk{V: 1, Why: strings.TrimSpace(why), By: by, At: at.UTC(), Mode: ModeFloorAgain, Name: name})
	return "floor." + versions.EscapeToken(name), data, err
}

// ModeOldFormDrop is the Mode of a reasks value of builder_old_form's drop (design 51 *Every build keeps the old
// reference form*): a retired node dropped from the count by hand; Name is the node.
const ModeOldFormDrop = "old-form-drop"

// NewOldFormDrop is the value builder_old_form writes to drop a retired node: the key `old-form.<node>`, never a build
// key, and the value.
func NewOldFormDrop(node, why, by string, at time.Time) (string, []byte, error) {
	node = strings.TrimSpace(node)
	if node == "" || strings.ContainsAny(node, ". *>") {
		return "", nil, fmt.Errorf("node %q is not a node's name", node)
	}
	if strings.TrimSpace(why) == "" {
		return "", nil, errors.New("say why the node leaves the count (why)")
	}
	data, err := json.Marshal(ReAsk{V: 1, Why: strings.TrimSpace(why), By: by, At: at.UTC(), Mode: ModeOldFormDrop, Name: node})
	return "old-form." + node, data, err
}

// ModeReservedDrop is the Mode of a reasks value of builder_reserved_drop (design 47 G section 369): a module's reservation
// of one MAJOR.MINOR dropped by hand; Name is the module and FP the MAJOR.MINOR.
const ModeReservedDrop = "reserved-drop"

// NewReservedDrop is the value builder_reserved_drop writes: the key `reserved.<module>.<MAJOR.MINOR>` (never a build key)
// and the value.
func NewReservedDrop(module, series, why, by string, at time.Time) (string, []byte, error) {
	module, series = strings.TrimSpace(module), strings.TrimSpace(series)
	if module == "" {
		return "", nil, errors.New("module is required")
	}
	if strings.Count(series, ".") != 1 {
		return "", nil, fmt.Errorf("series %q is not MAJOR.MINOR", series)
	}
	if strings.TrimSpace(why) == "" {
		return "", nil, errors.New("say why the reservation is dropped (why)")
	}
	data, err := json.Marshal(ReAsk{V: 1, Why: strings.TrimSpace(why), By: by, At: at.UTC(), Mode: ModeReservedDrop, Name: module, FP: series})
	return "reserved." + versions.ReservedKey(module, series), data, err
}

// ModeBuild is the Mode of a reasks value that asks the key to be built again, not its waiting asks to be asked again.
const ModeBuild = "build"

// ModeAnnounce is the Mode of a reasks value that asks a done key to be announced again.
const ModeAnnounce = "announce"

// DecodeReAsk reads a reasks value.
func DecodeReAsk(data []byte) (ReAsk, error) {
	var r ReAsk
	if err := json.Unmarshal(data, &r); err != nil {
		return r, err
	}
	if r.V != 1 {
		return r, fmt.Errorf("a reasks value of version %d, not 1", r.V)
	}
	return r, nil
}

// NewReAsk is the value builder_ask_again writes for a module's fingerprint: its key and the value.
func NewReAsk(module, fp, why, by string, at time.Time) (string, []byte, error) {
	key, err := builds.BuilderBuildKey(module, fp)
	if err != nil {
		return "", nil, err
	}
	if strings.TrimSpace(why) == "" {
		return "", nil, errors.New("say why the build is asked again (why)")
	}
	data, err := json.Marshal(ReAsk{V: 1, Why: strings.TrimSpace(why), By: by, At: at.UTC()})
	return key, data, err
}

// NewBuildAgain is the value builder_build_again writes for a module's fingerprint: its key and the value.
func NewBuildAgain(module, fp, why, by string, at time.Time) (string, []byte, error) {
	key, err := builds.BuilderBuildKey(module, fp)
	if err != nil {
		return "", nil, err
	}
	if strings.TrimSpace(why) == "" {
		return "", nil, errors.New("say why the key is built again (why)")
	}
	data, err := json.Marshal(ReAsk{V: 1, Why: strings.TrimSpace(why), By: by, At: at.UTC(), Mode: ModeBuild, Module: module, FP: fp})
	return key, data, err
}

// NewAnnounceAgain is the value builder_announce_again writes for a module's fingerprint: its key and the value.
func NewAnnounceAgain(module, fp, why, by string, at time.Time) (string, []byte, error) {
	key, err := builds.BuilderBuildKey(module, fp)
	if err != nil {
		return "", nil, err
	}
	if strings.TrimSpace(why) == "" {
		return "", nil, errors.New("say why the done build is announced again (why)")
	}
	data, err := json.Marshal(ReAsk{V: 1, Why: strings.TrimSpace(why), By: by, At: at.UTC(), Mode: ModeAnnounce, Module: module, FP: fp})
	return key, data, err
}

// StateRead is a state the tools read: its keys, and a key's value with when it was written (zero when not known).
type StateRead struct {
	Keys func() ([]string, error)
	Get  func(key string) (json.RawMessage, time.Time, error)
	// Written is when the key's current value was written, from the state's history: one more round trip, so the tools ask it
	// only for a row they keep, and only when Get did not say.
	Written func(key string) time.Time
}

// StuckAsk is one ask a key waits on.
type StuckAsk struct {
	Language    string `json:"language"`
	BuildModule string `json:"build_module"`
	ID          string `json:"id"`
	Asking      int    `json:"asking"`
	// Asked is when the ask was put, read from its id; Step is what its build module's steps hold of it: none, or its
	// state on its node since its start.
	Asked time.Time `json:"asked,omitzero"`
	Step  string    `json:"step"`
}

// Stuck is one build key that waits on an ask.
type Stuck struct {
	Key      string     `json:"key"`
	Module   string     `json:"module"`
	FP       string     `json:"fp"`
	State    string     `json:"state"`
	Step     string     `json:"step"`
	By       string     `json:"by"`
	Since    time.Time  `json:"since,omitzero"`
	Age      string     `json:"age,omitempty"`
	Asks     []StuckAsk `json:"asks"`
	ReAsk    *ReAsk     `json:"reask,omitempty"`
	Unasked  bool       `json:"unasked,omitempty"`
	NoneStep int        `json:"asks_with_no_step"`
}

// StuckReader reads what builder_stuck answers.
type StuckReader struct {
	Builds StateRead
	ReAsks StateRead
	// Steps are the build seats' steps states (`<seat>.steps`, design 47 G A-82), by build seat.
	Steps map[string]StateRead
	// StepsOf, when set, reads the steps state of a build seat the seat list names (`<seat>.steps`), instead of Steps: the seats
	// are known only once the list is read.
	StepsOf func(seat string) StateRead
	// Seats finds the build module of a build seat, whose name the key's step words give.
	Seats *buildrun.BuildSeats
	// Hearing is the builder's state hearing: whether it hears DEAD_LETTERS (issue 734); without it none is said.
	Hearing StateRead
	Now     func() time.Time
	// Logf, when set, gets the one line a call logs: the values it read and the time it took.
	Logf func(format string, args ...any)
}

// ReadWorkers is how many values builder_stuck asks the bus for at once, and MaxReads how many it reads in a call (issue 1827:
// every value is a round trip through the runtime, so one after the other a few thousand keys outlast the call's 30 s).
// A copy of the delivery view's and the checker's: a few lines are not worth a group delivery through mesh-sdk.
const (
	ReadWorkers = 16
	MaxReads    = 1000
)

// parallel runs read(i) for i in [0, n), at most ReadWorkers at a time, and answers the first error.
func parallel(n int, read func(i int) error) error {
	var (
		next   atomic.Int64
		failed atomic.Bool
		once   sync.Once
		first  error
		wg     sync.WaitGroup
	)
	for w := 0; w < min(ReadWorkers, n); w++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			for !failed.Load() {
				i := int(next.Add(1) - 1)
				if i >= n {
					return
				}
				if err := read(i); err != nil {
					once.Do(func() { first = err })
					failed.Store(true)
				}
			}
		}()
	}
	wg.Wait()
	return first
}

// Stuck is every key building or queued with its asks, oldest first; with module, that module's only.
// deadLetters says whether the builder hears the asks the build seats give up on (issue 734), and why not: deaf, a lane
// whose ask is given up waits for its check bound (buildrun.HearingHealth over the kept hearing).
func (r StuckReader) deadLetters() map[string]any {
	if r.Hearing.Get == nil {
		return map[string]any{"heard": false, "why": "the builder's state hearing is not readable here"}
	}
	raw, _, err := r.Hearing.Get(buildrun.HearingKey)
	if err != nil {
		return map[string]any{"heard": false, "why": "the builder's state hearing cannot be read: " + err.Error()}
	}
	a := buildrun.HearingHealth(raw)
	return map[string]any{"heard": a.Healthy, "why": a.Why}
}

func (r StuckReader) Stuck(module string) (map[string]any, error) {
	started := time.Now()
	reads := 0
	defer func() {
		if r.Logf != nil {
			r.Logf("builder_stuck read %d values in %s", reads, time.Since(started).Round(time.Millisecond))
		}
	}()
	keys, err := r.Builds.Keys()
	if err != nil {
		return nil, err
	}
	now := time.Now
	if r.Now != nil {
		now = r.Now
	}
	stepKeys, steps, err := r.stepIndex()
	if err != nil {
		return nil, err
	}
	// Narrowed by the key (the module) before any value is read; the state is in the value, so every matching key is read, in
	// parallel, at most MaxReads of them.
	type parsed struct{ key, mod, fp string }
	var matching []parsed
	for _, key := range keys {
		mod, fp, err := builds.ParseBuilderBuildKey(key)
		if err != nil || (module != "" && mod != module) {
			continue
		}
		matching = append(matching, parsed{key, mod, fp})
	}
	sort.Slice(matching, func(i, j int) bool { return matching[i].key < matching[j].key })
	cut := len(matching) > MaxReads
	total := len(matching)
	if cut {
		matching = matching[:MaxReads]
	}
	values := make([]json.RawMessage, len(matching))
	ats := make([]time.Time, len(matching))
	if err := parallel(len(matching), func(i int) error {
		raw, at, err := r.Builds.Get(matching[i].key)
		if err == nil {
			values[i], ats[i] = raw, at
		}
		return nil
	}); err != nil {
		return nil, err
	}
	reads += len(matching)
	out := []Stuck{}
	for i, m := range matching {
		key, mod, fp := m.key, m.mod, m.fp
		raw, at := values[i], ats[i]
		if raw == nil {
			continue
		}
		v, err := builds.DecodeBuilderBuild(raw)
		if err != nil || (v.State != builds.BuildBuilding && v.State != builds.BuildQueued) {
			continue
		}
		if at.IsZero() && r.Builds.Written != nil {
			at = r.Builds.Written(key)
		}
		s := Stuck{Key: key, Module: mod, FP: fp, State: v.State, Step: v.Step, By: v.By, Since: at, Asks: []StuckAsk{}}
		if !at.IsZero() {
			s.Age = now().Sub(at).Round(time.Second).String()
		}
		for language, id := range v.Asks {
			seat, _ := buildstep.BuildSeat(language)
			bm, _ := r.Seats.OfSeat(seat)
			a := StuckAsk{Language: language, BuildModule: bm.Module, ID: id, Asking: askingOf(v.Step, bm.Module), Step: "none"}
			if t, ok := idTime(id); ok {
				a.Asked = t
			}
			if st, ok := steps[seat]; ok {
				if k, ok := stepKeys[seat][id]; ok {
					reads++
					if raw, _, err := st.Get(k); err == nil && raw != nil {
						if step, err := buildstep.DecodeStep(raw); err == nil {
							a.Step = fmt.Sprintf("%s on %s since %s", step.State, step.Node, step.Started.UTC().Format(time.RFC3339))
						}
					}
				}
			}
			if a.Step == "none" {
				s.NoneStep++
			}
			s.Asks = append(s.Asks, a)
		}
		sort.Slice(s.Asks, func(i, j int) bool { return s.Asks[i].Language < s.Asks[j].Language })
		s.Unasked = v.State == builds.BuildBuilding && len(v.Asks) == 0
		if r.ReAsks.Get != nil {
			reads++
			if raw, _, err := r.ReAsks.Get(key); err == nil && raw != nil {
				if ra, err := DecodeReAsk(raw); err == nil {
					s.ReAsk = &ra
				}
			}
		}
		out = append(out, s)
	}
	sort.Slice(out, func(i, j int) bool {
		if !out[i].Since.Equal(out[j].Since) {
			return out[i].Since.Before(out[j].Since)
		}
		return out[i].Key < out[j].Key
	})
	answer := map[string]any{"stuck": out, "count": len(out),
		"delivery_counts": "not readable by the builder: the build seats' workers' information is granted to the delivery view alone (its backlog); an ask given up on is told by the builder at its check bound and asked again"}
	answer["dead_letters"] = r.deadLetters()
	if cut {
		answer["cut"], answer["note"] = true, fmt.Sprintf("read %d of %d build keys in key order and stopped; give a module to read its keys alone", MaxReads, total)
	}
	return answer, nil
}

// stepIndex finds the build modules' steps by the ask's id, which their key ends in: one index per build module, no value read
// until an ask names it.
func (r StuckReader) stepIndex() (map[string]map[string]string, map[string]StateRead, error) {
	stepKeys := map[string]map[string]string{}
	steps := r.Steps
	if r.StepsOf != nil {
		steps = map[string]StateRead{}
		for _, m := range r.Seats.All() {
			steps[m.Seat] = r.StepsOf(m.Seat)
		}
	}
	for bm, s := range steps {
		ks, err := s.Keys()
		if err != nil {
			return nil, nil, fmt.Errorf("the steps of %s: %w", bm, err)
		}
		byID := make(map[string]string, len(ks))
		for _, k := range ks {
			_, _, id, err := buildstep.ParseStepKey(k)
			if err != nil {
				// A key neither form parses is named, never passed over in silence (design 47 G §484).
				if r.Logf != nil {
					r.Logf("builder_stuck: the key %q of the steps of %s is a step key of neither form, <asker>.<node>.<id> or <node>.<id>: %v", k, bm, err)
				}
				continue
			}
			byID[id] = k
		}
		stepKeys[bm] = byID
	}
	return stepKeys, steps, nil
}

// idTime is when a step id was made (buildstep.NewID: the time in milliseconds in base 36, then six hex digits).
func idTime(id string) (time.Time, bool) {
	if len(id) <= 6 {
		return time.Time{}, false
	}
	ms, err := strconv.ParseInt(id[:len(id)-6], 36, 64)
	if err != nil || ms <= 0 {
		return time.Time{}, false
	}
	return time.UnixMilli(ms).UTC(), true
}