package buildrun import ( "context" "encoding/json" "errors" "fmt" "log" "strings" "sync" "time" "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" "git.novox.be/novox/mesh-delivery/sdk/buildstep" "git.novox.be/novox/mesh-sdk/go/busretry" ) // **An asker puts an ask on its build seat's stream and waits for its step** (novox/hq ADR 0525 §1, §6; issue 1887 A4, // A5). It chooses no node: whichever build node pulls the ask writes `.` in its build module's steps state, // and the asker reads the first such key that is not running. The builder and the checker share it. // AskPublisher puts an encoded ask on subject, de-duplicated by its id. type AskPublisher interface { PublishAsk(ctx context.Context, subject, id string, data []byte) error } // AskWaiter watches an ask's steps: every value a key `*.` of the build module's steps state takes, running or // ended, on the channel it answers, until ctx ends. A watch, not a poll. type AskWaiter interface { WatchAsk(ctx context.Context, module, askID string) (<-chan buildstep.Step, error) } // Asker puts asks and waits for their steps. type Asker struct { Publisher AskPublisher Waiter AskWaiter // Bound is how long a running step (a build node took the ask, its key written running) may stand still, with no new phase, // before the asker stops waiting; a step that keeps moving is waited for up to 3 Bounds from its start (issue 1930); zero waits as long as ctx. An ask waiting in its stream is not bounded: the stream's backlog is no fault // (ADR 0525 §6, §9), and the bound starts again when another node takes the ask. Bound time.Duration // Dropper, when set, is told to end an ask the asker stops waiting for without its end (ADR 0664 §5): a step given // again that the bus did not give within its bound, or a wait ended with a cause its caller names (a superseded // fingerprint). Never a wait ended by a shutdown (a plain cancel): the ask is then taken over by the next run. Dropper AskDropper // Logf says a drop that failed (the log's Printf when unset): the ask then stays on its stream, and it is said. Logf func(format string, args ...any) } // Withdrawn is the cause a caller gives the context of an ask's wait to withdraw the ask (ADR 0664 §4): a build superseded by a // newer fingerprint, a candidate taken out, an ask asked again by hand. Asker.Wait drops the ask on this cause alone: a // plain cancel (a shutdown) and any other cause (a bound passing, a check of whether the ask is gone) leave it on its stream // for the next run. A cause that wraps a Withdrawn (errors.As) withdraws too. type Withdrawn struct{ Why string } func (w Withdrawn) Error() string { return w.Why } // AskDropper ends an ask: the seat's verb drop, served by any holder of the build seat (no node is named). It writes the // ask in the build module's shared state of pauses, so every node refuses it when the stream gives it, and the node that // runs it ends the step (ADR 0664 §5). type AskDropper interface { DropAsk(ctx context.Context, a buildstep.StepAsk, why string) error } // SeatCaller calls a seat's verb on whichever holder answers (mesh-sdk process.Conn.Call). type SeatCaller interface { Call(key string, body any) (json.RawMessage, error) } // SeatDrops is an AskDropper that calls the verb drop of the ask's build seat over the bus. type SeatDrops struct{ Caller SeatCaller } // DropAsk calls .drop for a, naming its asker and why. func (d SeatDrops) DropAsk(_ context.Context, a buildstep.StepAsk, why string) error { seat, err := buildstep.BuildSeat(a.Language) if err != nil { return err } if _, err := d.Caller.Call("seat:"+seat+".drop", map[string]any{"id": a.ID, "by": a.Asker, "why": why}); err != nil { return fmt.Errorf("dropping the ask %s: %w", a.ID, err) } return nil } // dropped ends a without its end. A failed drop is not the wait's error, but it is said (Logf). func (k Asker) dropped(a buildstep.StepAsk, why string) { if k.Dropper == nil { return } ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() if err := k.Dropper.DropAsk(ctx, a, why); err != nil { say := k.Logf if say == nil { say = log.Printf } say("the ask %s was not dropped (%s), it stays on its stream: %v", a.ID, why, err) } } // PutTry is how long one publish of an ask waits for its stream's answer before it is made again (Put). var PutTry = 10 * time.Second // Put puts a on its stream: the seat's accept subject of its language and kind. The put is a kept call (novox/hq ADR 0668 // §1, §2; issue 1966, issue 1748 note 27): it carries the ask's id as its message id, so the stream takes it once, and it // is made again through every transport fault (a deadline, no responders) within busretry.Bound, each try bounded by // PutTry, until ctx ends. A put that still meets one is an error opening with buildstep.TransportFaultWords, so the key it // fails is one the builder does not take again by itself (ADR 0664 §3) and builder_build_again does. func (k Asker) Put(ctx context.Context, a buildstep.StepAsk) error { raw, err := buildstep.EncodeStepAsk(a) if err != nil { return err } subject, err := buildstep.AskSubject(a.Language, a.Kind) if err != nil { return err } _, err = busretry.DoRepeatable("putting the "+a.Kind+" ask "+a.ID, func() (struct{}, error) { if err := ctx.Err(); err != nil { return struct{}{}, fmt.Errorf("the put was given up: %v", err) // not a fault to wait out: the build ended } try, cancel := context.WithTimeout(ctx, PutTry) defer cancel() err := k.Publisher.PublishAsk(try, subject, a.ID, raw) if err != nil && ctx.Err() != nil { return struct{}{}, fmt.Errorf("the put was given up: %v", err) } return struct{}{}, err }, nil) switch { case err == nil: return nil case busretry.TimedOut(err) || busretry.Undelivered(err): return fmt.Errorf("%sthe put of the %s ask %s: %w", buildstep.TransportFaultWords, a.Kind, a.ID, err) } return fmt.Errorf("putting the %s ask %s: %w", a.Kind, a.ID, err) } // Wait waits for the step of an ask already put (a builder that started again takes over its own asks with it, never // putting them twice) and answers it when it ended, whatever its state. The bound runs from the step's start on the node // that took the ask, read from its running key, so an ask that waits in the stream for a free slot waits as long as ctx. func (k Asker) Wait(ctx context.Context, a buildstep.StepAsk) (buildstep.Step, error) { module, err := buildstep.BuildModuleOf(a.Language) if err != nil { return buildstep.Step{}, err } watch, cancel := context.WithCancel(ctx) defer cancel() steps, err := k.Waiter.WatchAsk(watch, module, a.ID) if err != nil { return buildstep.Step{}, err } var bound, outerCap <-chan time.Time var timer, capTimer *time.Timer defer func() { if timer != nil { timer.Stop() } if capTimer != nil { capTimer.Stop() } }() running, since, phase, phaseAt := "", time.Time{}, "", time.Time{} var agains []string // the node of each run that ended on a transport fault, in order (ADR 0664 §3) standStill := func() error { if phase == "" { return fmt.Errorf("the ask %s's step on %s did not end within %s of its start: it stood still, saying no phase", a.ID, running, k.Bound) } return fmt.Errorf("the ask %s's step on %s did not end within %s: it stood still in %q since %s", a.ID, running, k.Bound, phase, phaseAt.UTC().Format(time.RFC3339)) } for { select { case <-ctx.Done(): var w Withdrawn if errors.As(context.Cause(ctx), &w) { k.dropped(a, context.Cause(ctx).Error()) } return buildstep.Step{}, ctx.Err() case <-bound: if len(agains) > 0 && phase == "given again after a transport fault" { k.dropped(a, "the bus did not give it again within its bound") } return buildstep.Step{}, standStill() case <-outerCap: return buildstep.Step{}, fmt.Errorf("the ask %s's step on %s did not end within %s of its start: it kept moving (last phase %q at %s) but never ended", a.ID, running, 3*k.Bound, phase, phaseAt.UTC().Format(time.RFC3339)) case step, ok := <-steps: if !ok { return buildstep.Step{}, errors.New("the watch of the ask's step ended") } if seen := stepSeenOf(ctx); seen != nil { seen(step) } if step.State == buildstep.StepAgain { // A run ended on a transport fault and the bus gives the ask again: not an end. The bound runs on from the // time it is given again, so a bus that never gives it says so (ADR 0664 §3). agains = append(agains, step.Node) running, since, phase, phaseAt = step.Node, step.Started, "given again after a transport fault", step.AgainAt if k.Bound > 0 { if timer != nil { timer.Stop() } timer = time.NewTimer(max(time.Until(step.AgainAt.Add(k.Bound)), 0)) bound = timer.C } continue } if buildstep.Ended(step) { if step.Passing && step.State == buildstep.StepFailed && (len(agains) > 0 || step.Run > 1) { // The step's own count of runs (ADR 0664 §2), not only the ones this wait saw: a builder that started again // reads the key's last value and none of the runs before it. step.Reason = fmt.Sprintf(buildstep.TransportFaultWords+"%d runs (%s): %s", max(step.Run, len(agains)+1), strings.Join(append(agains, step.Node), ", "), step.Reason) } return step, nil } if k.Bound <= 0 { continue } newRun := step.Node != running || !step.Started.Equal(since) if !newRun && step.Phase == phase && step.PhaseAt.Equal(phaseAt) { continue } // A node took the ask, or took it again after its run stopped (another node, or the same node's build module // replaced mid-run: a new start), or its step entered a phase. The bound is on standing still: it runs from the // step's last movement (its phase's start, else its start), so a step that keeps moving is waited for, up to // three bounds from its start (issue 1930). running, since, phase, phaseAt = step.Node, step.Started, step.Phase, step.PhaseAt last := step.Started if step.PhaseAt.After(last) { last = step.PhaseAt } left := k.Bound if !last.IsZero() { left = time.Until(last.Add(k.Bound)) } if timer != nil { timer.Stop() } timer = time.NewTimer(max(left, 0)) bound = timer.C if newRun { if capTimer != nil { capTimer.Stop() } outer := 3 * k.Bound if !step.Started.IsZero() { outer = time.Until(step.Started.Add(3 * k.Bound)) } capTimer = time.NewTimer(max(outer, 0)) outerCap = capTimer.C } } } } type stepSeenKey struct{} // WithStepSeen is ctx carrying seen, which Asker.Wait hands every value the ask's step key takes while it waits: the node // that took it, each phase it says while running (buildstep.Step.Phase), and its end. A try names its asks' progress so // (issue 1890). seen runs in the waiting goroutine and must not block. A seen ctx already carries is kept: both are handed // every value, the earlier first (a check's begin bound watches the steps of a try that names them). func WithStepSeen(ctx context.Context, seen func(buildstep.Step)) context.Context { if before := stepSeenOf(ctx); before != nil { later := seen seen = func(s buildstep.Step) { before(s); later(s) } } return context.WithValue(ctx, stepSeenKey{}, seen) } func stepSeenOf(ctx context.Context) func(buildstep.Step) { seen, _ := ctx.Value(stepSeenKey{}).(func(buildstep.Step)) return seen } // Ask puts a and waits for its step; a step that did not pass is an error naming its node, its state and its reason. func (k Asker) Ask(ctx context.Context, a buildstep.StepAsk) (buildstep.Step, error) { if err := k.Put(ctx, a); err != nil { return buildstep.Step{}, err } return k.Passed(k.Wait(ctx, a)) } // Passed is step when it passed, else an error naming where it ended and why. func (k Asker) Passed(step buildstep.Step, err error) (buildstep.Step, error) { if err != nil { return step, err } if step.State != buildstep.StepPassed { return step, fmt.Errorf("the ask %s on %s %s: %s", step.ID, step.Node, step.State, step.Reason) } return step, nil } // JetStreamAsks is the bus side of an Asker: a JetStream publish de-duplicated by the ask's id (Nats-Msg-Id), and a watch // of `*.` in the build module's steps bucket. type JetStreamAsks struct { JS jetstream.JetStream } // PublishAsk is AskPublisher's. func (s JetStreamAsks) PublishAsk(ctx context.Context, subject, id string, data []byte) error { msg := &nats.Msg{Subject: subject, Data: data, Header: nats.Header{}} msg.Header.Set(jetstream.MsgIDHeader, id) _, err := s.JS.PublishMsg(ctx, msg) return err } // WatchAsk is AskWaiter's: the watch hands over what the keys hold now, so a step that ended before it began is read at // once, and every change after. func (s JetStreamAsks) WatchAsk(ctx context.Context, module, askID string) (<-chan buildstep.Step, error) { pattern, err := buildstep.AskStepWatch(askID) if err != nil { return nil, err } kv, err := s.JS.KeyValue(ctx, StepsBucket(module)) if err != nil { return nil, fmt.Errorf("the bucket %s: %w", StepsBucket(module), err) } w, err := kv.Watch(ctx, pattern) if err != nil { return nil, err } out := make(chan buildstep.Step) go func() { defer close(out) defer w.Stop() for { select { case <-ctx.Done(): return case e, ok := <-w.Updates(): if !ok { return } if e == nil || e.Operation() != jetstream.KeyValuePut { continue } step, err := buildstep.DecodeStep(e.Value()) if err != nil { continue // not a step (a node's pause key never matches an ask id; anything else is not this ask's) } select { case out <- step: case <-ctx.Done(): return } } } }() return out, nil } // FakeAsks is a test double of the bus for an asker's tests: each ask put is kept and, when Answer is set, its step is // written at once with what Answer says a build node wrote (running or ended). Write writes a step later. Safe for // concurrent use. type FakeAsks struct { Answer func(a buildstep.StepAsk) buildstep.Step mu sync.Mutex asks []buildstep.StepAsk steps map[string][]buildstep.Step watchers map[string][]chan buildstep.Step } // PublishAsk is AskPublisher's. func (f *FakeAsks) PublishAsk(_ context.Context, _, id string, data []byte) error { a, err := buildstep.DecodeStepAsk(data) if err != nil { return err } f.mu.Lock() f.asks = append(f.asks, a) f.mu.Unlock() if f.Answer != nil { f.Write(f.Answer(a)) } return nil } // Write writes an ask's step, as a build node does. func (f *FakeAsks) Write(step buildstep.Step) { f.mu.Lock() if f.steps == nil { f.steps, f.watchers = map[string][]buildstep.Step{}, map[string][]chan buildstep.Step{} } f.steps[step.ID] = append(f.steps[step.ID], step) ws := append([]chan buildstep.Step(nil), f.watchers[step.ID]...) f.mu.Unlock() for _, w := range ws { w <- step } } // End is Write: kept for the tests that end a step. func (f *FakeAsks) End(step buildstep.Step) { f.Write(step) } // WatchAsk is AskWaiter's: what was written so far, then every write. func (f *FakeAsks) WatchAsk(ctx context.Context, _ string, askID string) (<-chan buildstep.Step, error) { out := make(chan buildstep.Step, 64) f.mu.Lock() if f.steps == nil { f.steps, f.watchers = map[string][]buildstep.Step{}, map[string][]chan buildstep.Step{} } for _, s := range f.steps[askID] { out <- s } f.watchers[askID] = append(f.watchers[askID], out) f.mu.Unlock() go func() { <-ctx.Done() f.mu.Lock() ws := f.watchers[askID] for i, w := range ws { if w == out { f.watchers[askID] = append(ws[:i], ws[i+1:]...) break } } f.mu.Unlock() }() return out, nil } // Asks are the asks put so far. func (f *FakeAsks) Asks() []buildstep.StepAsk { f.mu.Lock() defer f.mu.Unlock() return append([]buildstep.StepAsk(nil), f.asks...) }