package build import ( "encoding/json" "errors" "fmt" "sort" "strconv" "strings" "sync" "sync/atomic" "time" "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" ) // The builder's tools (novox/hq issue 1945; the instruction "tools are how a module is spoken to"): builder_stuck reads // every key that waits on an ask, builder_ask_again asks a key's build again now. The tools bundle serves them through // the tool runner; the builder's process serves no tool (ADR 0437 ยง4), so builder_ask_again writes the ask into the // builder's state reasks, which the builder watches and answers there. const ( StuckTool = "builder_stuck" AskAgainTool = "builder_ask_again" // ReAsksState is the builder's state of asks to ask again: the bucket mesh-delivery-builder_reasks, keyed as the build key. ReAsksState = "reasks" ReAsksBucket = delivery.BuilderModule + "_" + ReAsksState ) // ReAsk is a value of the reasks state: why a person asked the key's build again, who and when, and, once the builder // read it, its answer. type ReAsk struct { V int `json:"v"` Why string `json:"why"` By string `json:"by,omitempty"` At time.Time `json:"at"` Answer string `json:"answer,omitempty"` Answered time.Time `json:"answered,omitzero"` } // DecodeReAsk reads a reasks value. func DecodeReAsk(data []byte) (ReAsk, error) { var r ReAsk if err := json.Unmarshal(data, &r); err != nil { return r, err } if r.V != 1 { return r, fmt.Errorf("a reasks value of version %d, not 1", r.V) } return r, nil } // NewReAsk is the value builder_ask_again writes for a module's fingerprint: its key and the value. func NewReAsk(module, fp, why, by string, at time.Time) (string, []byte, error) { key, err := delivery.BuilderBuildKey(module, fp) if err != nil { return "", nil, err } if strings.TrimSpace(why) == "" { return "", nil, errors.New("say why the build is asked again (why)") } data, err := json.Marshal(ReAsk{V: 1, Why: strings.TrimSpace(why), By: by, At: at.UTC()}) return key, data, err } // StateRead is a state the tools read: its keys, and a key's value with when it was written (zero when not known). type StateRead struct { Keys func() ([]string, error) Get func(key string) (json.RawMessage, time.Time, error) // Written is when the key's current value was written, from the state's history: one more round trip, so the tools ask it // only for a row they keep, and only when Get did not say. Written func(key string) time.Time } // StuckAsk is one ask a key waits on. type StuckAsk struct { Language string `json:"language"` BuildModule string `json:"build_module"` ID string `json:"id"` Asking int `json:"asking"` // Asked is when the ask was put, read from its id; Step is what its build module's steps hold of it: none, or its // state on its node since its start. Asked time.Time `json:"asked,omitzero"` Step string `json:"step"` } // Stuck is one build key that waits on an ask. type Stuck struct { Key string `json:"key"` Module string `json:"module"` FP string `json:"fp"` State string `json:"state"` Step string `json:"step"` By string `json:"by"` Since time.Time `json:"since,omitzero"` Age string `json:"age,omitempty"` Asks []StuckAsk `json:"asks"` ReAsk *ReAsk `json:"reask,omitempty"` Unasked bool `json:"unasked,omitempty"` NoneStep int `json:"asks_with_no_step"` } // StuckReader reads what builder_stuck answers. type StuckReader struct { Builds StateRead ReAsks StateRead // Steps are the build modules' steps states, by build module. Steps map[string]StateRead // Hearing is the builder's state hearing: whether it hears DEAD_LETTERS (issue 734); without it none is said. Hearing StateRead Now func() time.Time // Logf, when set, gets the one line a call logs: the values it read and the time it took. Logf func(format string, args ...any) } // ReadWorkers is how many values builder_stuck asks the bus for at once, and MaxReads how many it reads in a call (issue 1827: // every value is a round trip through the runtime, so one after the other a few thousand keys outlast the call's 30 s). // A copy of the delivery view's and the checker's: a few lines are not worth a group delivery through mesh-sdk. const ( ReadWorkers = 16 MaxReads = 1000 ) // parallel runs read(i) for i in [0, n), at most ReadWorkers at a time, and answers the first error. func parallel(n int, read func(i int) error) error { var ( next atomic.Int64 failed atomic.Bool once sync.Once first error wg sync.WaitGroup ) for w := 0; w < min(ReadWorkers, n); w++ { wg.Add(1) go func() { defer wg.Done() for !failed.Load() { i := int(next.Add(1) - 1) if i >= n { return } if err := read(i); err != nil { once.Do(func() { first = err }) failed.Store(true) } } }() } wg.Wait() return first } // Stuck is every key building or queued with its asks, oldest first; with module, that module's only. // deadLetters says whether the builder hears the asks the build seats give up on (issue 734), and why not: deaf, a lane // whose ask is given up waits for its check bound (buildrun.HearingHealth over the kept hearing). func (r StuckReader) deadLetters() map[string]any { if r.Hearing.Get == nil { return map[string]any{"heard": false, "why": "the builder's state hearing is not readable here"} } raw, _, err := r.Hearing.Get(buildrun.HearingKey) if err != nil { return map[string]any{"heard": false, "why": "the builder's state hearing cannot be read: " + err.Error()} } a := buildrun.HearingHealth(raw) return map[string]any{"heard": a.Healthy, "why": a.Why} } func (r StuckReader) Stuck(module string) (map[string]any, error) { started := time.Now() reads := 0 defer func() { if r.Logf != nil { r.Logf("builder_stuck read %d values in %s", reads, time.Since(started).Round(time.Millisecond)) } }() keys, err := r.Builds.Keys() if err != nil { return nil, err } now := time.Now if r.Now != nil { now = r.Now } // Steps are found by the ask's id, which their key ends in: one index per build module, no value read until an ask names it. stepKeys := map[string]map[string]string{} for bm, s := range r.Steps { ks, err := s.Keys() if err != nil { return nil, fmt.Errorf("the steps of %s: %w", bm, err) } byID := make(map[string]string, len(ks)) for _, k := range ks { if _, id, err := delivery.ParseStepKey(k); err == nil { byID[id] = k } } stepKeys[bm] = byID } // Narrowed by the key (the module) before any value is read; the state is in the value, so every matching key is read, in // parallel, at most MaxReads of them. type parsed struct{ key, mod, fp string } var matching []parsed for _, key := range keys { mod, fp, err := delivery.ParseBuilderBuildKey(key) if err != nil || (module != "" && mod != module) { continue } matching = append(matching, parsed{key, mod, fp}) } sort.Slice(matching, func(i, j int) bool { return matching[i].key < matching[j].key }) cut := len(matching) > MaxReads total := len(matching) if cut { matching = matching[:MaxReads] } values := make([]json.RawMessage, len(matching)) ats := make([]time.Time, len(matching)) if err := parallel(len(matching), func(i int) error { raw, at, err := r.Builds.Get(matching[i].key) if err == nil { values[i], ats[i] = raw, at } return nil }); err != nil { return nil, err } reads += len(matching) out := []Stuck{} for i, m := range matching { key, mod, fp := m.key, m.mod, m.fp raw, at := values[i], ats[i] if raw == nil { continue } v, err := delivery.DecodeBuilderBuild(raw) if err != nil || (v.State != delivery.BuildBuilding && v.State != delivery.BuildQueued) { continue } if at.IsZero() && r.Builds.Written != nil { at = r.Builds.Written(key) } s := Stuck{Key: key, Module: mod, FP: fp, State: v.State, Step: v.Step, By: v.By, Since: at, Asks: []StuckAsk{}} if !at.IsZero() { s.Age = now().Sub(at).Round(time.Second).String() } for language, id := range v.Asks { bm, _ := delivery.BuildModuleOf(language) a := StuckAsk{Language: language, BuildModule: bm, ID: id, Asking: askingOf(v.Step, bm), Step: "none"} if t, ok := idTime(id); ok { a.Asked = t } if st, ok := r.Steps[bm]; ok { if k, ok := stepKeys[bm][id]; ok { reads++ if raw, _, err := st.Get(k); err == nil && raw != nil { if step, err := delivery.DecodeStep(raw); err == nil { a.Step = fmt.Sprintf("%s on %s since %s", step.State, step.Node, step.Started.UTC().Format(time.RFC3339)) } } } } if a.Step == "none" { s.NoneStep++ } s.Asks = append(s.Asks, a) } sort.Slice(s.Asks, func(i, j int) bool { return s.Asks[i].Language < s.Asks[j].Language }) s.Unasked = v.State == delivery.BuildBuilding && len(v.Asks) == 0 if r.ReAsks.Get != nil { reads++ if raw, _, err := r.ReAsks.Get(key); err == nil && raw != nil { if ra, err := DecodeReAsk(raw); err == nil { s.ReAsk = &ra } } } out = append(out, s) } sort.Slice(out, func(i, j int) bool { if !out[i].Since.Equal(out[j].Since) { return out[i].Since.Before(out[j].Since) } return out[i].Key < out[j].Key }) answer := map[string]any{"stuck": out, "count": len(out), "delivery_counts": "not readable by the builder: the build seats' workers' information is granted to the delivery view alone (its backlog); an ask given up on is told by the builder at its check bound and asked again"} answer["dead_letters"] = r.deadLetters() if cut { answer["cut"], answer["note"] = true, fmt.Sprintf("read %d of %d build keys in key order and stopped; give a module to read its keys alone", MaxReads, total) } return answer, nil } // StuckBuildModules are the build modules whose steps builder_stuck reads. var StuckBuildModules = []string{buildrun.GoBuildModule, buildrun.NodeBuildModule, ImageBuildModule} // idTime is when a step id was made (buildstep.NewID: the time in milliseconds in base 36, then six hex digits). func idTime(id string) (time.Time, bool) { if len(id) <= 6 { return time.Time{}, false } ms, err := strconv.ParseInt(id[:len(id)-6], 36, 64) if err != nil || ms <= 0 { return time.Time{}, false } return time.UnixMilli(ms).UTC(), true }