package build import ( "encoding/json" "errors" "fmt" "sort" "strconv" "strings" "time" "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" ) // The builder's tools (novox/hq issue 1945; the instruction "tools are how a module is spoken to"): builder_stuck reads // every key that waits on an ask, builder_ask_again asks a key's build again now. The tools bundle serves them through // the tool runner; the builder's process serves no tool (ADR 0437 ยง4), so builder_ask_again writes the ask into the // builder's state reasks, which the builder watches and answers there. const ( StuckTool = "builder_stuck" AskAgainTool = "builder_ask_again" // ReAsksState is the builder's state of asks to ask again: the bucket mesh-delivery-builder_reasks, keyed as the build key. ReAsksState = "reasks" ReAsksBucket = delivery.BuilderModule + "_" + ReAsksState ) // ReAsk is a value of the reasks state: why a person asked the key's build again, who and when, and, once the builder // read it, its answer. type ReAsk struct { V int `json:"v"` Why string `json:"why"` By string `json:"by,omitempty"` At time.Time `json:"at"` Answer string `json:"answer,omitempty"` Answered time.Time `json:"answered,omitzero"` } // DecodeReAsk reads a reasks value. func DecodeReAsk(data []byte) (ReAsk, error) { var r ReAsk if err := json.Unmarshal(data, &r); err != nil { return r, err } if r.V != 1 { return r, fmt.Errorf("a reasks value of version %d, not 1", r.V) } return r, nil } // NewReAsk is the value builder_ask_again writes for a module's fingerprint: its key and the value. func NewReAsk(module, fp, why, by string, at time.Time) (string, []byte, error) { key, err := delivery.BuilderBuildKey(module, fp) if err != nil { return "", nil, err } if strings.TrimSpace(why) == "" { return "", nil, errors.New("say why the build is asked again (why)") } data, err := json.Marshal(ReAsk{V: 1, Why: strings.TrimSpace(why), By: by, At: at.UTC()}) return key, data, err } // StateRead is a state the tools read: its keys, and a key's value with when it was written (zero when not known). type StateRead struct { Keys func() ([]string, error) Get func(key string) (json.RawMessage, time.Time, error) } // StuckAsk is one ask a key waits on. type StuckAsk struct { Language string `json:"language"` BuildModule string `json:"build_module"` ID string `json:"id"` Asking int `json:"asking"` // Asked is when the ask was put, read from its id; Step is what its build module's steps hold of it: none, or its // state on its node since its start. Asked time.Time `json:"asked,omitzero"` Step string `json:"step"` } // Stuck is one build key that waits on an ask. type Stuck struct { Key string `json:"key"` Module string `json:"module"` FP string `json:"fp"` State string `json:"state"` Step string `json:"step"` By string `json:"by"` Since time.Time `json:"since,omitzero"` Age string `json:"age,omitempty"` Asks []StuckAsk `json:"asks"` ReAsk *ReAsk `json:"reask,omitempty"` Unasked bool `json:"unasked,omitempty"` NoneStep int `json:"asks_with_no_step"` } // StuckReader reads what builder_stuck answers. type StuckReader struct { Builds StateRead ReAsks StateRead // Steps are the build modules' steps states, by build module. Steps map[string]StateRead Now func() time.Time } // Stuck is every key building or queued with its asks, oldest first; with module, that module's only. func (r StuckReader) Stuck(module string) (map[string]any, error) { keys, err := r.Builds.Keys() if err != nil { return nil, err } now := time.Now if r.Now != nil { now = r.Now } stepKeys := map[string][]string{} for bm, s := range r.Steps { ks, err := s.Keys() if err != nil { return nil, fmt.Errorf("the steps of %s: %w", bm, err) } stepKeys[bm] = ks } out := []Stuck{} for _, key := range keys { mod, fp, err := delivery.ParseBuilderBuildKey(key) if err != nil || (module != "" && mod != module) { continue } raw, at, err := r.Builds.Get(key) if err != nil || raw == nil { continue } v, err := delivery.DecodeBuilderBuild(raw) if err != nil || (v.State != delivery.BuildBuilding && v.State != delivery.BuildQueued) { continue } s := Stuck{Key: key, Module: mod, FP: fp, State: v.State, Step: v.Step, By: v.By, Since: at, Asks: []StuckAsk{}} if !at.IsZero() { s.Age = now().Sub(at).Round(time.Second).String() } for language, id := range v.Asks { bm, _ := delivery.BuildModuleOf(language) a := StuckAsk{Language: language, BuildModule: bm, ID: id, Asking: askingOf(v.Step, bm), Step: "none"} if t, ok := idTime(id); ok { a.Asked = t } if st, ok := r.Steps[bm]; ok { for _, k := range stepKeys[bm] { if !strings.HasSuffix(k, "."+id) { continue } if raw, _, err := st.Get(k); err == nil && raw != nil { if step, err := delivery.DecodeStep(raw); err == nil { a.Step = fmt.Sprintf("%s on %s since %s", step.State, step.Node, step.Started.UTC().Format(time.RFC3339)) } } } } if a.Step == "none" { s.NoneStep++ } s.Asks = append(s.Asks, a) } sort.Slice(s.Asks, func(i, j int) bool { return s.Asks[i].Language < s.Asks[j].Language }) s.Unasked = v.State == delivery.BuildBuilding && len(v.Asks) == 0 if r.ReAsks.Get != nil { if raw, _, err := r.ReAsks.Get(key); err == nil && raw != nil { if ra, err := DecodeReAsk(raw); err == nil { s.ReAsk = &ra } } } out = append(out, s) } sort.Slice(out, func(i, j int) bool { if !out[i].Since.Equal(out[j].Since) { return out[i].Since.Before(out[j].Since) } return out[i].Key < out[j].Key }) return map[string]any{"stuck": out, "count": len(out), "delivery_counts": "not readable by the builder: the build seats' workers' information is granted to the delivery view alone (its backlog); an ask given up on is told by the builder at its check bound and asked again"}, nil } // StuckBuildModules are the build modules whose steps builder_stuck reads. var StuckBuildModules = []string{buildrun.GoBuildModule, buildrun.NodeBuildModule, ImageBuildModule} // idTime is when a step id was made (buildstep.NewID: the time in milliseconds in base 36, then six hex digits). func idTime(id string) (time.Time, bool) { if len(id) <= 6 { return time.Time{}, false } ms, err := strconv.ParseInt(id[:len(id)-6], 36, 64) if err != nil || ms <= 0 { return time.Time{}, false } return time.UnixMilli(ms).UTC(), true }