package build import ( "context" "fmt" "maps" "strings" "time" "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" ) // **A failed key is built again by hand** (novox/hq issue 1126 note 72; ADR 0429 clause 18: "a fingerprint is built again // only when its build key failed, as a new revision of the same key"). The builder does it by itself whenever it looks at a // module again (a modules value, a change of the toolchains, artifacts-gone, a moved trunk): Claimer.Claim writes a failed // key building as a new revision and the build asks the build seats. Nothing looks at an unchanged module on its own, so a key // that failed (a node refused the build, a toolchain was missing) stayed failed until the module changed. The tool // builder_build_again (BuildAgain) makes the builder look at one module now, which does exactly that and no more: it keeps // the claim's rules, so a key failed for a reason the builder never retries on the same fingerprint stays refused. // BuildAgainResult is the builder's answer to a request to build a key again. type BuildAgainResult struct { // Refused is why nothing was done, "" when the key was written building as a new revision. Refused string // Revision is the revision the claim wrote; Asks are the ask ids put by language, as far as they were put within the bound. Revision uint64 Asks map[string]string // Pending says the builder had not claimed the key, or not put every ask, when it answered: not a refusal. Pending string // Failed is the error of the key when the build ended failed before its asks were put (a plan that cannot be read, a // toolchain not agreed): the new revision was written, and nothing was asked. Failed string } // handWatch follows one request to build a key again until its asks are put. type handWatch struct { rev uint64 lanes int asks map[string]string done chan struct{} shut bool // ended is set when the look ended before the asks were all put. ended bool } // settle closes done once the claim is written and every lane has put its ask. Call with b.mu held. func (w *handWatch) settle() { if !w.shut && w.rev > 0 && w.lanes >= 0 && len(w.asks) >= w.lanes { w.shut = true close(w.done) } } func (b *Builder) hand(key string, f func(w *handWatch)) { b.mu.Lock() defer b.mu.Unlock() if w := b.hands[key]; w != nil { f(w) } } func (b *Builder) handClaimed(key string, rev uint64) { b.hand(key, func(w *handWatch) { w.rev = rev; w.settle() }) } func (b *Builder) handLanes(key string, n int) { b.hand(key, func(w *handWatch) { w.lanes = n; w.settle() }) } func (b *Builder) handAsk(key, language, id string) { b.hand(key, func(w *handWatch) { w.asks[language] = id; w.settle() }) } // handEnded ends the watch of a request whose look ended: it wrote no claim, or the build ended. func (b *Builder) handEnded(key string) { b.hand(key, func(w *handWatch) { if !w.shut { w.shut, w.ended = true, true close(w.done) } }) } // BuildAgain builds the failed key of module at fp again, as the builder does by itself: the key is written building as a // new revision and the build module asks are put on the build seats. It answers once the asks are put, or at bound. A key // that is not failed is refused with why: done (nothing to do) or waiting (builder_ask_again asks its waiting asks again). // ctx is the builder's own context: the build it starts lives on it. func (b *Builder) BuildAgain(ctx context.Context, module, fp, why, by string, bound time.Duration) BuildAgainResult { b.init() key, err := delivery.BuilderBuildKey(module, fp) if err != nil { return BuildAgainResult{Refused: err.Error()} } value, _, found, err := b.Claimer.Store.Get(ctx, key) if err != nil { return BuildAgainResult{Refused: fmt.Sprintf("%s cannot be read: %v", key, err)} } if !found { return BuildAgainResult{Refused: fmt.Sprintf("%s has no build key: nothing failed. The builder builds a fingerprint by itself once the indexer writes it to its module", key)} } cur, err := delivery.DecodeBuilderBuild(value) if err != nil { return BuildAgainResult{Refused: fmt.Sprintf("%s holds what is not a build: %v", key, err)} } switch { case cur.State == delivery.BuildDone: return BuildAgainResult{Refused: fmt.Sprintf("%s is done: nothing to do", key)} case cur.State == delivery.BuildBuilding || cur.State == delivery.BuildQueued: return BuildAgainResult{Refused: fmt.Sprintf("%s is waiting (%s: %s): %s asks its waiting asks again", key, cur.State, cur.Step, AskAgainTool)} case cur.State != delivery.BuildFailed: return BuildAgainResult{Refused: fmt.Sprintf("%s is %q, not failed", key, cur.State)} case delivery.FailedUntilChanged(cur.Error) || strings.HasPrefix(cur.Error, buildrun.DigestsDifferPrefix): return BuildAgainResult{Refused: fmt.Sprintf("%s failed for a reason the builder does not retry on the same fingerprint, it waits for another fingerprint: %s", key, cur.Error)} } // One critical section: the key is refused when it is looked at or asked already, the module still has the fingerprint, // and the watch and the look are registered before anything else can take the key. w := &handWatch{lanes: -1, asks: map[string]string{}, done: make(chan struct{})} b.mu.Lock() m, known := b.known[module] switch { case !known || m.FP2 != fp: b.mu.Unlock() return BuildAgainResult{Refused: fmt.Sprintf("module %s does not have the fingerprint %s now (a newer one superseded it, or its modules value is not read yet): the builder builds the module's current fingerprint", module, fp)} case m.Manifest == "": b.mu.Unlock() return BuildAgainResult{Refused: fmt.Sprintf("module %s names no manifest at %s yet, so the builder looks at nothing", module, fp)} case b.inflight[key] || b.hands[key] != nil: b.mu.Unlock() return BuildAgainResult{Refused: fmt.Sprintf("%s is being looked at by this builder now (or asked again by hand): wait for it to end", key)} } b.hands[key] = w b.inflight[key] = true b.mu.Unlock() defer func() { b.mu.Lock() if b.hands[key] == w { delete(b.hands, key) } b.mu.Unlock() }() b.logf("%s: failed (%s); built again by hand, asked by %s: %s", key, cur.Error, by, why) b.startOnce(ctx, key, module, m) wait, stop := context.WithTimeout(ctx, bound) // a context's bound for a person's answer (ADR 0437 ยง8) defer stop() select { case <-w.done: case <-wait.Done(): } b.mu.Lock() res := BuildAgainResult{Revision: w.rev, Asks: maps.Clone(w.asks)} ended, shut, lanes := w.ended, w.shut, w.lanes b.mu.Unlock() switch { case res.Revision == 0 && shut: res.Refused = fmt.Sprintf("the builder looked at %s and wrote nothing (a toolchain moved, the claim was lost or could not be written): its log says why", key) case res.Revision == 0: res.Pending = fmt.Sprintf("the builder has not claimed %s within %s: it waits behind other builds; builder_stuck shows where it stands", key, bound) case ended && len(res.Asks) < max(lanes, 0)+boolInt(lanes < 0): // The look ended before every ask was put: say how the key stands, not that it was asked. if v, _, found, err := b.Claimer.Store.Get(ctx, key); err == nil && found { if cur, err := delivery.DecodeBuilderBuild(v); err == nil { switch cur.State { case delivery.BuildFailed: res.Failed = cur.Error case delivery.BuildDone: default: res.Pending = fmt.Sprintf("the look ended with the key %s (%s) before every ask was put: its log says why", cur.State, cur.Step) } } } case !shut: res.Pending = fmt.Sprintf("the key is building at revision %d, and not every ask was put within %s; builder_stuck shows the asks", res.Revision, bound) } return res } func boolInt(b bool) int { if b { return 1 } return 0 }