The builder finds a build module as the indexer's seed does: the build seat whose acting rule `build` has an `on` matching the artifact's language or kind:image, and that seat's holder, read from the seat list's definitions and the claims of the catalog half's manifests. Task.Module, every lane key, the unpublished modules, the toolchain sources, the ends watched, the sayer and builder_stuck (through `<seat>.steps`) derive from it; the floor's sources are the holders of the seats git and mesh-catalog. buildrun.GoBuildModule, NodeBuildModule, LanguageOf, buildstep.BuildModuleOf use, build.ImageBuildModule and StuckBuildModules are gone from the builder (the checker keeps its own names in two small files until its part reads them from the list). Builds: design 47 G §7, §428, §704, §204, A-82 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h
459 lines
18 KiB
Go
459 lines
18 KiB
Go
package build
|
|
|
|
import (
|
|
"mesh-delivery-builder/internal/versions"
|
|
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
|
|
"git.novox.be/novox/mesh-delivery/sdk/buildstep"
|
|
)
|
|
|
|
// The builder's tools (novox/hq issue 1945; the instruction "tools are how a module is spoken to"): builder_stuck reads
|
|
// every key that waits on an ask, builder_ask_again asks a key's build again now. The tools bundle serves them through
|
|
// the tool runner; the builder's process serves no tool (ADR 0437 §4), so builder_ask_again writes the ask into the
|
|
// builder's state reasks, which the builder watches and answers there.
|
|
const (
|
|
StuckTool = "builder_stuck"
|
|
AskAgainTool = "builder_ask_again"
|
|
// BuildAgainTool builds a failed key again by hand (issue 1126 note 72): it writes a reasks value of Mode ModeBuild.
|
|
BuildAgainTool = "builder_build_again"
|
|
// AnnounceAgainTool writes a done key again unchanged, so the controller takes it in and composes again: a reasks value
|
|
// of Mode ModeAnnounce.
|
|
AnnounceAgainTool = "builder_announce_again"
|
|
// ReAsksState is the builder's state of asks to ask again: the bucket mesh-delivery-builder_reasks, keyed as the build key.
|
|
ReAsksState = "reasks"
|
|
ReAsksBucket = builds.BuilderModule + "_" + ReAsksState
|
|
)
|
|
|
|
// ReAsk is a value of the reasks state: why a person asked the key's build again, who and when, and, once the builder
|
|
// read it, its answer.
|
|
type ReAsk struct {
|
|
V int `json:"v"`
|
|
Why string `json:"why"`
|
|
By string `json:"by,omitempty"`
|
|
At time.Time `json:"at"`
|
|
Answer string `json:"answer,omitempty"`
|
|
Answered time.Time `json:"answered,omitzero"`
|
|
// Mode is ModeBuild for a request to build a failed key again (BuildAgainTool); empty for an ask asked again.
|
|
Mode string `json:"mode,omitempty"`
|
|
Module string `json:"module,omitempty"`
|
|
FP string `json:"fp,omitempty"`
|
|
// Refused, Revision and Asks are the builder's answer to a ModeBuild request: why it did nothing, or the key's new
|
|
// revision (building) and the ask ids it put by language.
|
|
Refused bool `json:"refused,omitempty"`
|
|
// Pending says the builder answered before the key was claimed or all its asks were put: neither a refusal nor done.
|
|
// Failed is the error of a key written building that failed before its asks were put.
|
|
Pending bool `json:"pending,omitempty"`
|
|
Failed string `json:"failed,omitempty"`
|
|
Revision uint64 `json:"revision,omitempty"`
|
|
Asks map[string]string `json:"asks,omitempty"`
|
|
// Name is the numbered name (`<module>/<artifact>`) of a ModePublishAgain or ModeFloorAgain value; Do and Version the
|
|
// builder's answer: what it did (versions.Again*) and the version published.
|
|
Name string `json:"name,omitempty"`
|
|
Do string `json:"do,omitempty"`
|
|
Version string `json:"version,omitempty"`
|
|
}
|
|
|
|
// ModePublishAgain is the Mode of a reasks value of builder_publish_again (design 51 *Tools of the builder*).
|
|
const ModePublishAgain = "publish-again"
|
|
|
|
// ModeFloorAgain is the Mode of a reasks value of builder_floor_again (design 51 *The floor*).
|
|
const ModeFloorAgain = "floor-again"
|
|
|
|
// NewPublishAgain is the value builder_publish_again writes for a name's fingerprint: the key of the name's module and
|
|
// fingerprint, and the value.
|
|
func NewPublishAgain(name, fp, why, by string, at time.Time) (string, []byte, error) {
|
|
module, artifact, ok := strings.Cut(strings.TrimSpace(name), "/")
|
|
if !ok || module == "" || artifact == "" {
|
|
return "", nil, fmt.Errorf("name %q is not a numbered name, <module>/<artifact>, as builder_versions lists it", name)
|
|
}
|
|
key, err := builds.BuilderBuildKey(module, fp)
|
|
if err != nil {
|
|
return "", nil, err
|
|
}
|
|
if strings.TrimSpace(why) == "" {
|
|
return "", nil, errors.New("say why it is published again (why)")
|
|
}
|
|
data, err := json.Marshal(ReAsk{V: 1, Why: strings.TrimSpace(why), By: by, At: at.UTC(), Mode: ModePublishAgain, Module: module, FP: fp, Name: strings.TrimSpace(name)})
|
|
return key, data, err
|
|
}
|
|
|
|
// NewFloorAgain is the value builder_floor_again writes for a name: the key `floor.<name escaped>`, never a build key,
|
|
// and the value.
|
|
func NewFloorAgain(name, why, by string, at time.Time) (string, []byte, error) {
|
|
name = strings.TrimSpace(name)
|
|
if module, artifact, ok := strings.Cut(name, "/"); !ok || module == "" || artifact == "" {
|
|
return "", nil, fmt.Errorf("name %q is not a numbered name, <module>/<artifact>, as builder_versions lists it", name)
|
|
}
|
|
if strings.TrimSpace(why) == "" {
|
|
return "", nil, errors.New("say why the floor is asked again (why)")
|
|
}
|
|
data, err := json.Marshal(ReAsk{V: 1, Why: strings.TrimSpace(why), By: by, At: at.UTC(), Mode: ModeFloorAgain, Name: name})
|
|
return "floor." + versions.EscapeToken(name), data, err
|
|
}
|
|
|
|
// ModeOldFormDrop is the Mode of a reasks value of builder_old_form's drop (design 51 *Every build keeps the old
|
|
// reference form*): a retired node dropped from the count by hand; Name is the node.
|
|
const ModeOldFormDrop = "old-form-drop"
|
|
|
|
// NewOldFormDrop is the value builder_old_form writes to drop a retired node: the key `old-form.<node>`, never a build
|
|
// key, and the value.
|
|
func NewOldFormDrop(node, why, by string, at time.Time) (string, []byte, error) {
|
|
node = strings.TrimSpace(node)
|
|
if node == "" || strings.ContainsAny(node, ". *>") {
|
|
return "", nil, fmt.Errorf("node %q is not a node's name", node)
|
|
}
|
|
if strings.TrimSpace(why) == "" {
|
|
return "", nil, errors.New("say why the node leaves the count (why)")
|
|
}
|
|
data, err := json.Marshal(ReAsk{V: 1, Why: strings.TrimSpace(why), By: by, At: at.UTC(), Mode: ModeOldFormDrop, Name: node})
|
|
return "old-form." + node, data, err
|
|
}
|
|
|
|
// ModeReservedDrop is the Mode of a reasks value of builder_reserved_drop (design 47 G section 369): a module's reservation
|
|
// of one MAJOR.MINOR dropped by hand; Name is the module and FP the MAJOR.MINOR.
|
|
const ModeReservedDrop = "reserved-drop"
|
|
|
|
// NewReservedDrop is the value builder_reserved_drop writes: the key `reserved.<module>.<MAJOR.MINOR>` (never a build key)
|
|
// and the value.
|
|
func NewReservedDrop(module, series, why, by string, at time.Time) (string, []byte, error) {
|
|
module, series = strings.TrimSpace(module), strings.TrimSpace(series)
|
|
if module == "" {
|
|
return "", nil, errors.New("module is required")
|
|
}
|
|
if strings.Count(series, ".") != 1 {
|
|
return "", nil, fmt.Errorf("series %q is not MAJOR.MINOR", series)
|
|
}
|
|
if strings.TrimSpace(why) == "" {
|
|
return "", nil, errors.New("say why the reservation is dropped (why)")
|
|
}
|
|
data, err := json.Marshal(ReAsk{V: 1, Why: strings.TrimSpace(why), By: by, At: at.UTC(), Mode: ModeReservedDrop, Name: module, FP: series})
|
|
return "reserved." + versions.ReservedKey(module, series), data, err
|
|
}
|
|
|
|
// ModeBuild is the Mode of a reasks value that asks the key to be built again, not its waiting asks to be asked again.
|
|
const ModeBuild = "build"
|
|
|
|
// ModeAnnounce is the Mode of a reasks value that asks a done key to be announced again.
|
|
const ModeAnnounce = "announce"
|
|
|
|
// DecodeReAsk reads a reasks value.
|
|
func DecodeReAsk(data []byte) (ReAsk, error) {
|
|
var r ReAsk
|
|
if err := json.Unmarshal(data, &r); err != nil {
|
|
return r, err
|
|
}
|
|
if r.V != 1 {
|
|
return r, fmt.Errorf("a reasks value of version %d, not 1", r.V)
|
|
}
|
|
return r, nil
|
|
}
|
|
|
|
// NewReAsk is the value builder_ask_again writes for a module's fingerprint: its key and the value.
|
|
func NewReAsk(module, fp, why, by string, at time.Time) (string, []byte, error) {
|
|
key, err := builds.BuilderBuildKey(module, fp)
|
|
if err != nil {
|
|
return "", nil, err
|
|
}
|
|
if strings.TrimSpace(why) == "" {
|
|
return "", nil, errors.New("say why the build is asked again (why)")
|
|
}
|
|
data, err := json.Marshal(ReAsk{V: 1, Why: strings.TrimSpace(why), By: by, At: at.UTC()})
|
|
return key, data, err
|
|
}
|
|
|
|
// NewBuildAgain is the value builder_build_again writes for a module's fingerprint: its key and the value.
|
|
func NewBuildAgain(module, fp, why, by string, at time.Time) (string, []byte, error) {
|
|
key, err := builds.BuilderBuildKey(module, fp)
|
|
if err != nil {
|
|
return "", nil, err
|
|
}
|
|
if strings.TrimSpace(why) == "" {
|
|
return "", nil, errors.New("say why the key is built again (why)")
|
|
}
|
|
data, err := json.Marshal(ReAsk{V: 1, Why: strings.TrimSpace(why), By: by, At: at.UTC(), Mode: ModeBuild, Module: module, FP: fp})
|
|
return key, data, err
|
|
}
|
|
|
|
// NewAnnounceAgain is the value builder_announce_again writes for a module's fingerprint: its key and the value.
|
|
func NewAnnounceAgain(module, fp, why, by string, at time.Time) (string, []byte, error) {
|
|
key, err := builds.BuilderBuildKey(module, fp)
|
|
if err != nil {
|
|
return "", nil, err
|
|
}
|
|
if strings.TrimSpace(why) == "" {
|
|
return "", nil, errors.New("say why the done build is announced again (why)")
|
|
}
|
|
data, err := json.Marshal(ReAsk{V: 1, Why: strings.TrimSpace(why), By: by, At: at.UTC(), Mode: ModeAnnounce, Module: module, FP: fp})
|
|
return key, data, err
|
|
}
|
|
|
|
// StateRead is a state the tools read: its keys, and a key's value with when it was written (zero when not known).
|
|
type StateRead struct {
|
|
Keys func() ([]string, error)
|
|
Get func(key string) (json.RawMessage, time.Time, error)
|
|
// Written is when the key's current value was written, from the state's history: one more round trip, so the tools ask it
|
|
// only for a row they keep, and only when Get did not say.
|
|
Written func(key string) time.Time
|
|
}
|
|
|
|
// StuckAsk is one ask a key waits on.
|
|
type StuckAsk struct {
|
|
Language string `json:"language"`
|
|
BuildModule string `json:"build_module"`
|
|
ID string `json:"id"`
|
|
Asking int `json:"asking"`
|
|
// Asked is when the ask was put, read from its id; Step is what its build module's steps hold of it: none, or its
|
|
// state on its node since its start.
|
|
Asked time.Time `json:"asked,omitzero"`
|
|
Step string `json:"step"`
|
|
}
|
|
|
|
// Stuck is one build key that waits on an ask.
|
|
type Stuck struct {
|
|
Key string `json:"key"`
|
|
Module string `json:"module"`
|
|
FP string `json:"fp"`
|
|
State string `json:"state"`
|
|
Step string `json:"step"`
|
|
By string `json:"by"`
|
|
Since time.Time `json:"since,omitzero"`
|
|
Age string `json:"age,omitempty"`
|
|
Asks []StuckAsk `json:"asks"`
|
|
ReAsk *ReAsk `json:"reask,omitempty"`
|
|
Unasked bool `json:"unasked,omitempty"`
|
|
NoneStep int `json:"asks_with_no_step"`
|
|
}
|
|
|
|
// StuckReader reads what builder_stuck answers.
|
|
type StuckReader struct {
|
|
Builds StateRead
|
|
ReAsks StateRead
|
|
// Steps are the build seats' steps states (`<seat>.steps`, design 47 G A-82), by build seat.
|
|
Steps map[string]StateRead
|
|
// StepsOf, when set, reads the steps state of a build seat the seat list names (`<seat>.steps`), instead of Steps: the seats
|
|
// are known only once the list is read.
|
|
StepsOf func(seat string) StateRead
|
|
// Seats finds the build module of a build seat, whose name the key's step words give.
|
|
Seats *buildrun.BuildSeats
|
|
// Hearing is the builder's state hearing: whether it hears DEAD_LETTERS (issue 734); without it none is said.
|
|
Hearing StateRead
|
|
Now func() time.Time
|
|
// Logf, when set, gets the one line a call logs: the values it read and the time it took.
|
|
Logf func(format string, args ...any)
|
|
}
|
|
|
|
// ReadWorkers is how many values builder_stuck asks the bus for at once, and MaxReads how many it reads in a call (issue 1827:
|
|
// every value is a round trip through the runtime, so one after the other a few thousand keys outlast the call's 30 s).
|
|
// A copy of the delivery view's and the checker's: a few lines are not worth a group delivery through mesh-sdk.
|
|
const (
|
|
ReadWorkers = 16
|
|
MaxReads = 1000
|
|
)
|
|
|
|
// parallel runs read(i) for i in [0, n), at most ReadWorkers at a time, and answers the first error.
|
|
func parallel(n int, read func(i int) error) error {
|
|
var (
|
|
next atomic.Int64
|
|
failed atomic.Bool
|
|
once sync.Once
|
|
first error
|
|
wg sync.WaitGroup
|
|
)
|
|
for w := 0; w < min(ReadWorkers, n); w++ {
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
for !failed.Load() {
|
|
i := int(next.Add(1) - 1)
|
|
if i >= n {
|
|
return
|
|
}
|
|
if err := read(i); err != nil {
|
|
once.Do(func() { first = err })
|
|
failed.Store(true)
|
|
}
|
|
}
|
|
}()
|
|
}
|
|
wg.Wait()
|
|
return first
|
|
}
|
|
|
|
// Stuck is every key building or queued with its asks, oldest first; with module, that module's only.
|
|
// deadLetters says whether the builder hears the asks the build seats give up on (issue 734), and why not: deaf, a lane
|
|
// whose ask is given up waits for its check bound (buildrun.HearingHealth over the kept hearing).
|
|
func (r StuckReader) deadLetters() map[string]any {
|
|
if r.Hearing.Get == nil {
|
|
return map[string]any{"heard": false, "why": "the builder's state hearing is not readable here"}
|
|
}
|
|
raw, _, err := r.Hearing.Get(buildrun.HearingKey)
|
|
if err != nil {
|
|
return map[string]any{"heard": false, "why": "the builder's state hearing cannot be read: " + err.Error()}
|
|
}
|
|
a := buildrun.HearingHealth(raw)
|
|
return map[string]any{"heard": a.Healthy, "why": a.Why}
|
|
}
|
|
|
|
func (r StuckReader) Stuck(module string) (map[string]any, error) {
|
|
started := time.Now()
|
|
reads := 0
|
|
defer func() {
|
|
if r.Logf != nil {
|
|
r.Logf("builder_stuck read %d values in %s", reads, time.Since(started).Round(time.Millisecond))
|
|
}
|
|
}()
|
|
keys, err := r.Builds.Keys()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
now := time.Now
|
|
if r.Now != nil {
|
|
now = r.Now
|
|
}
|
|
// Steps are found by the ask's id, which their key ends in: one index per build module, no value read until an ask names it.
|
|
stepKeys := map[string]map[string]string{}
|
|
steps := r.Steps
|
|
if r.StepsOf != nil {
|
|
steps = map[string]StateRead{}
|
|
for _, m := range r.Seats.All() {
|
|
steps[m.Seat] = r.StepsOf(m.Seat)
|
|
}
|
|
}
|
|
for bm, s := range steps {
|
|
ks, err := s.Keys()
|
|
if err != nil {
|
|
return nil, fmt.Errorf("the steps of %s: %w", bm, err)
|
|
}
|
|
byID := make(map[string]string, len(ks))
|
|
for _, k := range ks {
|
|
_, _, id, err := buildstep.ParseStepKey(k)
|
|
if err != nil {
|
|
// A key neither form parses is named, never passed over in silence (design 47 G §484).
|
|
if r.Logf != nil {
|
|
r.Logf("builder_stuck: the key %q of the steps of %s is a step key of neither form, <asker>.<node>.<id> or <node>.<id>: %v", k, bm, err)
|
|
}
|
|
continue
|
|
}
|
|
byID[id] = k
|
|
}
|
|
stepKeys[bm] = byID
|
|
}
|
|
// Narrowed by the key (the module) before any value is read; the state is in the value, so every matching key is read, in
|
|
// parallel, at most MaxReads of them.
|
|
type parsed struct{ key, mod, fp string }
|
|
var matching []parsed
|
|
for _, key := range keys {
|
|
mod, fp, err := builds.ParseBuilderBuildKey(key)
|
|
if err != nil || (module != "" && mod != module) {
|
|
continue
|
|
}
|
|
matching = append(matching, parsed{key, mod, fp})
|
|
}
|
|
sort.Slice(matching, func(i, j int) bool { return matching[i].key < matching[j].key })
|
|
cut := len(matching) > MaxReads
|
|
total := len(matching)
|
|
if cut {
|
|
matching = matching[:MaxReads]
|
|
}
|
|
values := make([]json.RawMessage, len(matching))
|
|
ats := make([]time.Time, len(matching))
|
|
if err := parallel(len(matching), func(i int) error {
|
|
raw, at, err := r.Builds.Get(matching[i].key)
|
|
if err == nil {
|
|
values[i], ats[i] = raw, at
|
|
}
|
|
return nil
|
|
}); err != nil {
|
|
return nil, err
|
|
}
|
|
reads += len(matching)
|
|
out := []Stuck{}
|
|
for i, m := range matching {
|
|
key, mod, fp := m.key, m.mod, m.fp
|
|
raw, at := values[i], ats[i]
|
|
if raw == nil {
|
|
continue
|
|
}
|
|
v, err := builds.DecodeBuilderBuild(raw)
|
|
if err != nil || (v.State != builds.BuildBuilding && v.State != builds.BuildQueued) {
|
|
continue
|
|
}
|
|
if at.IsZero() && r.Builds.Written != nil {
|
|
at = r.Builds.Written(key)
|
|
}
|
|
s := Stuck{Key: key, Module: mod, FP: fp, State: v.State, Step: v.Step, By: v.By, Since: at, Asks: []StuckAsk{}}
|
|
if !at.IsZero() {
|
|
s.Age = now().Sub(at).Round(time.Second).String()
|
|
}
|
|
for language, id := range v.Asks {
|
|
seat, _ := buildstep.BuildSeat(language)
|
|
bm, _ := r.Seats.OfSeat(seat)
|
|
a := StuckAsk{Language: language, BuildModule: bm.Module, ID: id, Asking: askingOf(v.Step, bm.Module), Step: "none"}
|
|
if t, ok := idTime(id); ok {
|
|
a.Asked = t
|
|
}
|
|
if st, ok := steps[seat]; ok {
|
|
if k, ok := stepKeys[seat][id]; ok {
|
|
reads++
|
|
if raw, _, err := st.Get(k); err == nil && raw != nil {
|
|
if step, err := buildstep.DecodeStep(raw); err == nil {
|
|
a.Step = fmt.Sprintf("%s on %s since %s", step.State, step.Node, step.Started.UTC().Format(time.RFC3339))
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if a.Step == "none" {
|
|
s.NoneStep++
|
|
}
|
|
s.Asks = append(s.Asks, a)
|
|
}
|
|
sort.Slice(s.Asks, func(i, j int) bool { return s.Asks[i].Language < s.Asks[j].Language })
|
|
s.Unasked = v.State == builds.BuildBuilding && len(v.Asks) == 0
|
|
if r.ReAsks.Get != nil {
|
|
reads++
|
|
if raw, _, err := r.ReAsks.Get(key); err == nil && raw != nil {
|
|
if ra, err := DecodeReAsk(raw); err == nil {
|
|
s.ReAsk = &ra
|
|
}
|
|
}
|
|
}
|
|
out = append(out, s)
|
|
}
|
|
sort.Slice(out, func(i, j int) bool {
|
|
if !out[i].Since.Equal(out[j].Since) {
|
|
return out[i].Since.Before(out[j].Since)
|
|
}
|
|
return out[i].Key < out[j].Key
|
|
})
|
|
answer := map[string]any{"stuck": out, "count": len(out),
|
|
"delivery_counts": "not readable by the builder: the build seats' workers' information is granted to the delivery view alone (its backlog); an ask given up on is told by the builder at its check bound and asked again"}
|
|
answer["dead_letters"] = r.deadLetters()
|
|
if cut {
|
|
answer["cut"], answer["note"] = true, fmt.Sprintf("read %d of %d build keys in key order and stopped; give a module to read its keys alone", MaxReads, total)
|
|
}
|
|
return answer, nil
|
|
}
|
|
|
|
// idTime is when a step id was made (buildstep.NewID: the time in milliseconds in base 36, then six hex digits).
|
|
func idTime(id string) (time.Time, bool) {
|
|
if len(id) <= 6 {
|
|
return time.Time{}, false
|
|
}
|
|
ms, err := strconv.ParseInt(id[:len(id)-6], 36, 64)
|
|
if err != nil || ms <= 0 {
|
|
return time.Time{}, false
|
|
}
|
|
return time.UnixMilli(ms).UTC(), true
|
|
}
|