package main import ( "errors" "fmt" "os" "path/filepath" "strings" "time" ) // **The build agent says whether it is busy, and being asked drains it** (novox/hq ADR 0335 §1, §5). Its // container declares `busy` as `mesh-builder busy`: the node-engine runs it inside the container before it // replaces it. It exits 0 while a build is in hand and 1 when none is; and from the first ask the agent takes // no new build request, so its answer ends once the build in hand ends — a replaced builder finishes the check // it runs, instead of the build queue handing it out again from the start (issue 594). // // Two marks in the container's own temporary directory, as the ready mark (ready.go): a new container starts // without them, and a restarted one removes them before it takes work again. // // - in hand: written by the agent when it claims a request, before it reads whether it is drained; removed // when the build ends. // - drained: written by the ask, before it reads whether a build is in hand. // // Each side writes its own mark before it reads the other's, so an ask racing a fetch sees the build in hand, // or the agent sees the drain and hands the request back; never neither. func inHandMark() string { return filepath.Join(os.TempDir(), "mesh-builder.in-hand") } func drainMark() string { return filepath.Join(os.TempDir(), "mesh-builder.drained") } // The drain lapses when no ask has come for drainLapse: the node-engine asks every 20 seconds for as long as it // really waits, so an agent whose replacement was withdrawn — the declaration put back, the walk stopped — // takes work again rather than staying drained for ever (review of ADR 0335). And an agent drained for longer // than the node-engine's bound and the slack says it is not ready: the replacement it was drained for never // came, which is a fault to see. const ( drainLapse = 2 * time.Minute drainTooLong = 15*time.Minute + 5*time.Minute ) // drainedAt is when the drain began and when it was last asked, from its mark; false when there is no mark. func drainedAt() (began, asked time.Time, ok bool) { info, err := os.Stat(drainMark()) if err != nil { return time.Time{}, time.Time{}, false } asked = info.ModTime() began = asked if raw, err := os.ReadFile(drainMark()); err == nil { if t, err := time.Parse(time.RFC3339, strings.TrimSpace(string(raw))); err == nil && !t.After(asked) { began = t } } return began, asked, true } // drained says the agent was asked whether it is busy within drainLapse: it takes no new build request. func drained() bool { return drainedSince(time.Now()) } func drainedSince(now time.Time) bool { _, asked, ok := drainedAt() return ok && now.Sub(asked) < drainLapse } // drainedTooLong is the health check's second half: nil unless the agent has been drained, asked all along, for // longer than its node-engine's bound and the slack. func drainedTooLong(now time.Time) error { began, asked, ok := drainedAt() if !ok || now.Sub(asked) >= drainLapse || now.Sub(began) <= drainTooLong { return nil } return fmt.Errorf("drained since %s and still asked: the replacement its node-engine waits for has not come "+ "within %s (novox/hq ADR 0335)", began.UTC().Format(time.RFC3339), drainTooLong) } // claim marks a request in hand, unless the agent is drained: then nothing is in hand, and the request goes // back to the queue. func claim() bool { // Made, never written over: one build at a time, and a mark already there is a build in hand. f, err := os.OpenFile(inHandMark(), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644) if err != nil { // Unmarked, an ask would say idle while this build runs: the build is not taken. fmt.Fprintf(os.Stderr, "cannot mark a build in hand (%s), so it is handed back: %v\n", inHandMark(), err) return false } _, err = f.WriteString(time.Now().UTC().Format(time.RFC3339) + "\n") if closeErr := f.Close(); err == nil { err = closeErr } if err != nil { _ = os.Remove(inHandMark()) fmt.Fprintf(os.Stderr, "cannot mark a build in hand (%s), so it is handed back: %v\n", inHandMark(), err) return false } if drained() { _ = os.Remove(inHandMark()) fmt.Fprintln(os.Stderr, "asked whether busy, so this agent takes no new build: handed back") return false } return true } // holding says what is in hand, for the ask to print: written once the request is read. func holding(what string) { _ = os.WriteFile(inHandMark(), []byte(time.Now().UTC().Format(time.RFC3339)+" "+what+"\n"), 0o644) } // released ends the build in hand. func released() { if err := os.Remove(inHandMark()); err != nil && !errors.Is(err, os.ErrNotExist) { fmt.Fprintf(os.Stderr, "cannot remove the mark of the build in hand (%s): %v\n", inHandMark(), err) } } // undrained clears both marks at the start of a holder: a container started again is asked afresh. func undrained() error { for _, mark := range []string{drainMark(), inHandMark()} { if err := os.Remove(mark); err != nil && !errors.Is(err, os.ErrNotExist) { return fmt.Errorf("cannot remove %s: %w", mark, err) } } return nil } // busy is the ask: it drains the agent, then says whether a build is in hand. Nil — exit 0 — while one is. func busy() (string, error) { // The mark keeps when the drain began while it is asked within drainLapse, and is written again on every // ask: its time is the last ask. began := time.Now().UTC() if was, _, ok := drainedAt(); ok && drained() { began = was.UTC() } if err := os.WriteFile(drainMark(), []byte(began.Format(time.RFC3339)+"\n"), 0o644); err != nil { // Not drained, it could take the next build: said, and answered busy only for what is in hand now. fmt.Fprintf(os.Stderr, "cannot mark this agent drained (%s): %v\n", drainMark(), err) } held, err := os.ReadFile(inHandMark()) if err != nil { return "", errors.New("idle: no build in hand, and this agent takes no new one") } since, what, _ := strings.Cut(strings.TrimSpace(string(held)), " ") if what == "" { what = "a build" } return fmt.Sprintf("busy: %s, in hand since %s; taking no new build", what, since), nil }