package main import ( "context" "errors" "fmt" "regexp" "sort" "strings" "time" "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/inventory" ) // failedModule is one module of a failed plan that failed to build, where it runs and what it said. type failedModule struct { module string machines []string why string } // failedPlanFacts is one failed delivery plan asked of the operator (S23, novox/hq ADR 0449 §1). type failedPlanFacts struct { id, repository, commit string modules []string failed []failedModule // note is where the plan stopped when no module failed to build (a rollout stopped at its first machine). note string // since is the plan's last change: when it failed. since time.Time // refusal is why `plans retry` would not run it, empty when it would. refusal string // passed is set when a newer plan of its repository and branch is done and the retry is refused for that // (novox/hq ADR 0451): the condition asks nothing. passed bool } // machines are the machines the failed modules run on, sorted, each once. func (p failedPlanFacts) machines() []string { seen := map[string]bool{} var out []string for _, m := range p.failed { for _, n := range m.machines { if !seen[n] { seen[n] = true out = append(out, n) } } } sort.Strings(out) return out } // failureSaid is what failed, in the mesh's own words, for the condition's evidence. func (p failedPlanFacts) failureSaid() string { var parts []string for _, m := range p.failed { line := m.module if len(m.machines) > 0 { line += " on " + strings.Join(m.machines, ", ") } if m.why != "" { line += ": " + oneLine(m.why) } parts = append(parts, line) } if len(parts) == 0 { return "stopped: " + oneLine(p.note) } return strings.Join(parts, "; ") } // refusalSaid is why a retry would not run, for the evidence. func (p failedPlanFacts) refusalSaid() string { if p.refusal == "" { return "" } return "; a retry would not run: " + oneLine(p.refusal) } // retryAction is the one answer a failed plan offers: the controller's `plans` verb with `retry` naming the plan, // at the level approve. Only that argument: `plans` refuses a retry that carries others (seatverbs). func retryAction(plan string) conditions.Action { return conditions.Action{Label: "Retry", Verb: "mesh-controller.plans", Level: conditions.LevelApprove, Arguments: map[string]string{"retry": plan}} } var ( planIDWord = regexp.MustCompile(`\bplan-[0-9]+\b`) walkWd = regexp.MustCompile(`\bwalks\b`) walkOne = regexp.MustCompile(`\bwalk\b`) ) // plainRefusal is a retry's refusal in words the operator reads: the plans named as "the plan", a walk said as a // delivery. Whatever still breaks the plain rule (a hash, a dotted name) is left to the condition's evidence, and the // words say only that a retry would not run. func plainRefusal(refusal string) string { return plainRefusalOr(refusal, "a retry would not run now; the details say why") } // plainRefusalOr is plainRefusal with the sentence to say when the refusal's own words cannot be said plainly. func plainRefusalOr(refusal, fallback string) string { r := planIDWord.ReplaceAllString(refusal, "the plan") r = walkWd.ReplaceAllString(r, "deliveries") r = walkOne.ReplaceAllString(r, "delivery") r = strings.TrimSpace(r) if _, ok := conditions.Plain(r); !ok { return fallback } return r } // planFailedWords are the plain words of a failed plan: which modules failed and where, what they said when that // is plain, and Retry when a retry would run, else the refusal's words. func planFailedWords(p failedPlanFacts) (headline, explanation, needs string, actions []conditions.Action, resolved string) { name := deliveryName(p.modules, p.repository) headline = conditions.Capital(name) + " failed" what := deliveryWhat(p.modules, p.repository) var parts []string for i, m := range p.failed { if i == 2 { parts = append(parts, fmt.Sprintf("%d more", len(p.failed)-2)) break } line := m.module if len(m.machines) > 0 { line += " for " + readableList(m.machines) } if w := strings.TrimRight(cutWords(oneLine(m.why), 110), ".;:, "); w != "" { if _, ok := conditions.Plain(w); ok { line += " (" + w + ")" } } parts = append(parts, line) } if len(parts) > 0 { explanation = fmt.Sprintf("The delivery of %s failed: %s did not build.", what, readableList(parts)) } else { explanation = fmt.Sprintf("The delivery of %s stopped before it finished.", what) } switch { case p.passed: explanation += " A newer delivery has passed it, so a retry would send older code. Nothing is needed from you." case p.refusal == "": explanation += " Nothing is tried again until you say so." needs = "try the failed builds again " + FromMeshMCPServer actions = []conditions.Action{retryAction(p.id)} default: explanation += " " + conditions.Capital(plainRefusal(p.refusal)) + "." needs = "look at why it failed; a retry would not run now." } resolved = conditions.Capital(name) + " is no longer failed" return } // cutWords is s cut to at most n bytes at a word. func cutWords(s string, n int) string { if len(s) <= n { return s } t := s[:n] if i := strings.LastIndexByte(t, ' '); i > n/2 { t = t[:i] } return strings.TrimRight(t, " ,;:") + "…" } // gatherFailedPlans is every delivery plan that failed within planFailedWithin and is asked of the operator // (S23, novox/hq ADR 0449 §1): a failed plan that carries a delivery, is no rollout of held updates and was not // stopped by a person. Each carries why `plans retry` would not run it, when it would not. func gatherFailedPlans(ctx context.Context, inv *inventory.Inventory, now time.Time) ([]failedPlanFacts, error) { plans, err := inv.PlansSince(ctx, now.Add(-planFailedWithin)) if err != nil { return nil, fmt.Errorf("the plans cannot be read: %w", err) } var failed []inventory.Plan for _, p := range plans { if failedAndAsked(p) { failed = append(failed, p) } } if len(failed) == 0 { return nil, nil } entries, err := inv.Catalogued(ctx) if err != nil { return nil, fmt.Errorf("the catalogue cannot be read: %w", err) } onPath := ownPathOf(entries) runsOn := map[string][]string{} for _, e := range entries { runsOn[e.Manifest.Module] = e.On } var out []failedPlanFacts for _, p := range failed { out = append(out, failedFactsOf(p, plans, onPath, runsOn)) } return out, nil } // failedAndAsked says a plan is one S23 asks about: failed, a delivery's, not a rollout of held updates, not // stopped by a person. func failedAndAsked(p inventory.Plan) bool { return p.State == inventory.PlanFailed && p.Delivery != nil && p.Release == nil && p.Delivery.Stopped == "" } // failedFactsOf is what S23 says of one failed plan, given every plan, the controller's own path and where each // module runs. func failedFactsOf(p inventory.Plan, plans []inventory.Plan, onPath map[string]string, runsOn map[string][]string) failedPlanFacts { f := failedPlanFacts{id: p.ID, repository: p.Repository, commit: p.Commit, modules: planModules(p), note: p.Note, since: p.Updated} for _, m := range f.modules { if s := p.Modules[m]; s != nil && s.State == "failed" { machines := append([]string(nil), runsOn[m]...) sort.Strings(machines) f.failed = append(f.failed, failedModule{module: m, machines: machines, why: s.Why}) } } if err := retryRefusal(p, plans, onPath); err != nil { f.refusal = err.Error() var passed passedBy f.passed = errors.As(err, &passed) } return f }