jschoubben
60253c82b7
A check pending redelivery is read before the let-through fetches (none is handed back); a deleted module is done for the plan's key; the queue an ask came from is let go of when it ends. Tries say they run under their own id.
415 lines
15 KiB
Go
415 lines
15 KiB
Go
package link
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/nats-io/nats.go"
|
|
"github.com/nats-io/nats.go/jetstream"
|
|
|
|
"github.com/novox/mesh-controller/internal/broker"
|
|
)
|
|
|
|
// A pull request's check is queued by the pull request (novox/hq ADR 0406 §3, §4, issue 974).
|
|
|
|
// SupersededByKey is the outcome of a check that stopped because its pull request's key moved on: its head or its trunk
|
|
// commit is no longer the one it was asked for. Said as the check's failure, as a kill is; nothing is asked of the forge.
|
|
const SupersededByKey = "superseded: the pull request's key moved on"
|
|
|
|
// unsafeToken is what may not stand in a subject token or a key.
|
|
var unsafeToken = strings.NewReplacer(".", "_", "*", "_", ">", "_", " ", "_", "/", "_", "\t", "_", "\n", "_")
|
|
|
|
// tokenOf is a name as one subject token: any dot, wildcard or space replaced by an underscore.
|
|
func tokenOf(name string) string {
|
|
if name == "" {
|
|
return "_"
|
|
}
|
|
return unsafeToken.Replace(name)
|
|
}
|
|
|
|
// PullKeyOf is a pull request's key in mesh-delivery's bucket, and the last tokens of the subject its check is asked on:
|
|
// `<owner>.<repo>.<number>`, a dot in a name made an underscore.
|
|
func PullKeyOf(owner, repo string, number int) string {
|
|
return tokenOf(owner) + "." + tokenOf(repo) + "." + strconv.Itoa(number)
|
|
}
|
|
|
|
// CheckSubjectOf is the subject a check is asked on, and whether it has one of its own: a check of a pull request on
|
|
// the ready queue or the other, by the pull request; a delivery group's composed check by `group.<name>`; any other
|
|
// check (a branch tried before its pull request) by its own id, so it replaces nothing.
|
|
func CheckSubjectOf(seat string, r BuildRequest) (string, bool) {
|
|
c := r.Check
|
|
if seat != broker.TheBuildSeat || c == nil {
|
|
return "", false
|
|
}
|
|
prefix := broker.CheckSubjectPrefix(seat, c.Ready)
|
|
switch {
|
|
case c.Group != "":
|
|
return prefix + "group." + tokenOf(c.Group), true
|
|
case c.Try || c.Number <= 0:
|
|
return prefix + "try." + tokenOf(r.ID), true
|
|
}
|
|
return prefix + PullKeyOf(c.Owner, c.Repo, c.Number), true
|
|
}
|
|
|
|
// WorkSubjectOf is where an ask is put: a check on the build seat on its own queue, anything else on the seat's.
|
|
func WorkSubjectOf(seat string, r BuildRequest) string {
|
|
if subject, ok := CheckSubjectOf(seat, r); ok {
|
|
return subject
|
|
}
|
|
return BuildWorkOf(seat)
|
|
}
|
|
|
|
// PullKey is what mesh-delivery keeps of a pull request (ADR 0406 §4): its head, the trunk commit its current check is
|
|
// asked against, and its verdict once judged.
|
|
type PullKey struct {
|
|
Head string `json:"head"`
|
|
Trunk string `json:"trunk"`
|
|
Verdict string `json:"verdict,omitempty"`
|
|
}
|
|
|
|
// sameCommit says two spellings name one commit; either may be short, but not under twelve characters.
|
|
func sameCommit(a, b string) bool {
|
|
return a == b || SameHead(a, b)
|
|
}
|
|
|
|
// Moved says a key no longer holds what a check was asked for: its head or its trunk commit differs.
|
|
func (k PullKey) Moved(head, trunk string) bool {
|
|
return !sameCommit(k.Head, head) || (trunk != "" && k.Trunk != "" && !sameCommit(k.Trunk, trunk))
|
|
}
|
|
|
|
// Judged says a key holds the verdict of the check of this head against this trunk commit.
|
|
func (k PullKey) Judged(head, trunk string) bool {
|
|
// A verdict of a check that ran: a pass, a warning or a fail. An error is a check that could not run, and a request
|
|
// after it must run (issue 974, review 2).
|
|
judgedBy := k.Verdict == "pass" || k.Verdict == "warning" || k.Verdict == "fail"
|
|
return judgedBy && sameCommit(k.Head, head) && sameCommit(k.Trunk, trunk) && trunk != ""
|
|
}
|
|
|
|
// PullBucketsOf are the buckets a check's pull request's key is looked for in: the ones its request names, and
|
|
// mesh-delivery's when it names none (a controller from before the request named them).
|
|
func PullBucketsOf(c *CheckRequest) []string {
|
|
if c == nil || len(c.Pulls) == 0 {
|
|
return []string{broker.PullsBucket}
|
|
}
|
|
return c.Pulls
|
|
}
|
|
|
|
// ReadPullKey reads a pull request's key from the named buckets in turn: the first that holds it answers, a bucket that
|
|
// does not exist is passed over, and false when none holds it.
|
|
func ReadPullKey(ctx context.Context, conn *nats.Conn, buckets []string, owner, repo string, number int) (PullKey, bool, error) {
|
|
api, err := jetstream.New(conn)
|
|
if err != nil {
|
|
return PullKey{}, false, err
|
|
}
|
|
return readPullKeyIn(ctx, buckets, func(ctx context.Context, bucket string) (func(context.Context, string) ([]byte, error), error) {
|
|
kv, err := api.KeyValue(ctx, bucket)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return func(ctx context.Context, key string) ([]byte, error) {
|
|
e, err := kv.Get(ctx, key)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return e.Value(), nil
|
|
}, nil
|
|
}, owner, repo, number)
|
|
}
|
|
|
|
// readPullKeyIn is ReadPullKey over buckets opened by open.
|
|
func readPullKeyIn(ctx context.Context, buckets []string,
|
|
open func(ctx context.Context, bucket string) (func(context.Context, string) ([]byte, error), error),
|
|
owner, repo string, number int) (PullKey, bool, error) {
|
|
for _, bucket := range buckets {
|
|
get, err := open(ctx, bucket)
|
|
if errors.Is(err, jetstream.ErrBucketNotFound) {
|
|
continue
|
|
}
|
|
if err != nil {
|
|
return PullKey{}, false, fmt.Errorf("the pull requests' keys in %s: %w", bucket, err)
|
|
}
|
|
value, err := get(ctx, PullKeyOf(owner, repo, number))
|
|
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
|
continue
|
|
}
|
|
if err != nil {
|
|
return PullKey{}, false, err
|
|
}
|
|
var key PullKey
|
|
if err := json.Unmarshal(value, &key); err != nil {
|
|
return PullKey{}, false, fmt.Errorf("the key of %s/%s#%d is not a pull request's: %w", owner, repo, number, err)
|
|
}
|
|
return key, true, nil
|
|
}
|
|
return PullKey{}, false, nil
|
|
}
|
|
|
|
// firstBucketThatExists is the first of the buckets that exists, as exists says; an error other than a missing bucket
|
|
// stands.
|
|
func firstBucketThatExists(ctx context.Context, buckets []string, exists func(ctx context.Context, bucket string) error) (string, error) {
|
|
for _, bucket := range buckets {
|
|
err := exists(ctx, bucket)
|
|
if errors.Is(err, jetstream.ErrBucketNotFound) {
|
|
continue
|
|
}
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return bucket, nil
|
|
}
|
|
return "", fmt.Errorf("none of the pull requests' buckets %v exists", buckets)
|
|
}
|
|
|
|
// WatchPull calls moved once when the pull request's key moves on from what this check was asked for — its head or its
|
|
// trunk commit differs — and not for a key that only gains its verdict. It hears updates, not the key as it stands: the
|
|
// key and the request are written for one announcement by two writers, so a key found older than the request is not
|
|
// a move. The check is then stopped by whoever asked for it, without asking the forge (novox/hq issue 726). The stop
|
|
// ends the watch.
|
|
func WatchPull(ctx context.Context, conn *nats.Conn, r BuildRequest, moved func(PullKey)) (stop func(), err error) {
|
|
c := r.Check
|
|
if c == nil || c.Group != "" || c.Try || c.Number <= 0 {
|
|
return func() {}, nil
|
|
}
|
|
api, err := jetstream.New(conn)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return watchPullIn(ctx, PullBucketsOf(c), func(ctx context.Context, bucket string) (pullWatcher, error) {
|
|
kv, err := api.KeyValue(ctx, bucket)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return kv.Watch(ctx, PullKeyOf(c.Owner, c.Repo, c.Number), jetstream.UpdatesOnly())
|
|
}, r, moved)
|
|
}
|
|
|
|
// pullWatcher is what a watch of one bucket's key gives: the updates, and a stop.
|
|
type pullWatcher interface {
|
|
Updates() <-chan jetstream.KeyValueEntry
|
|
Stop() error
|
|
}
|
|
|
|
// watchPullIn is WatchPull over buckets watched by open: the key is watched in every bucket that exists, since
|
|
// the gate writes its own bucket and mesh-delivery's stops only when the gate takes over (novox/hq ADR 0450 §1,
|
|
// issue 1325); a move in any of them stops the check, once. None existing is an error.
|
|
func watchPullIn(ctx context.Context, buckets []string, open func(ctx context.Context, bucket string) (pullWatcher, error),
|
|
r BuildRequest, moved func(PullKey)) (stop func(), err error) {
|
|
watching, end := context.WithCancel(ctx)
|
|
var watchers []pullWatcher
|
|
for _, bucket := range buckets {
|
|
w, err := open(watching, bucket)
|
|
if errors.Is(err, jetstream.ErrBucketNotFound) {
|
|
continue
|
|
}
|
|
if err != nil {
|
|
end()
|
|
for _, held := range watchers {
|
|
_ = held.Stop()
|
|
}
|
|
return nil, fmt.Errorf("the pull requests' keys in %s: %w", bucket, err)
|
|
}
|
|
watchers = append(watchers, w)
|
|
}
|
|
if len(watchers) == 0 {
|
|
end()
|
|
return nil, fmt.Errorf("the pull requests' keys: none of the buckets %v exists", buckets)
|
|
}
|
|
var once sync.Once
|
|
for _, w := range watchers {
|
|
go func(w pullWatcher) {
|
|
defer w.Stop()
|
|
for {
|
|
select {
|
|
case <-watching.Done():
|
|
return
|
|
case e, ok := <-w.Updates():
|
|
if !ok {
|
|
return
|
|
}
|
|
if e == nil || e.Operation() != jetstream.KeyValuePut {
|
|
continue
|
|
}
|
|
var key PullKey
|
|
if json.Unmarshal(e.Value(), &key) != nil {
|
|
continue
|
|
}
|
|
if key.Moved(r.Ref, r.Check.Trunk) {
|
|
once.Do(func() { moved(key) })
|
|
return
|
|
}
|
|
}
|
|
}
|
|
}(w)
|
|
}
|
|
return end, nil
|
|
}
|
|
|
|
// checkTakeWait is how long a taking loop waits on one queue for an ask that is not there, and takeIdleWait how long it
|
|
// waits on the first when all three are empty.
|
|
//
|
|
// **A fetch's pull request must end before the fetch does** (issue 1649). nats.go gives the server-side request an expiry a tenth
|
|
// shorter than the fetch's wait (fetchExpiryOf), and the server then answers with a status message at the expiry; a wait so
|
|
// short that the answer can land after the fetch gave up left a status message in the pull's buffer, and anything a request
|
|
// could still deliver would be held by a machine that is busy. Each wait leaves at least minFetchMargin between the two.
|
|
const (
|
|
checkTakeWait = 100 * time.Millisecond
|
|
takeIdleWait = time.Second
|
|
)
|
|
|
|
// minFetchMargin is the least time between a fetch's server-side expiry and the end of its wait.
|
|
const minFetchMargin = 10 * time.Millisecond
|
|
|
|
// fetchExpiryOf is the expiry nats.go gives the pull request of a fetch that waits this long: the wait less a tenth of it, or
|
|
// less five seconds.
|
|
func fetchExpiryOf(wait time.Duration) time.Duration {
|
|
return wait - min(wait/10, 5*time.Second)
|
|
}
|
|
|
|
// CheckRunning is what a builder says of the check it is running, in the bucket of the checks running (novox/hq ADR 0406
|
|
// §3): the head and trunk commit it checks, the builder and its build. The key is the pull request's (RunningKeyOf).
|
|
type CheckRunning struct {
|
|
Head string `json:"head"`
|
|
Trunk string `json:"trunk"`
|
|
Builder string `json:"builder"`
|
|
Build string `json:"build"`
|
|
// Started is when the check started, kept on every heartbeat: the entry's own time is the last heartbeat's, and a
|
|
// reader that asks how long a check has run reads this (novox/hq ADR 0433 §6).
|
|
Started time.Time `json:"started"`
|
|
// Seen is when the key was last put, which is the last heartbeat: read from the entry, never written (issue 1649).
|
|
Seen time.Time `json:"-"`
|
|
}
|
|
|
|
// RunningKeyOf is the key a check is said running under: its pull request's tokens, or `group.<name>` for a group's
|
|
// composed check; `try.<id>` for a try, which no other request replaces.
|
|
func RunningKeyOf(r BuildRequest) (string, bool) {
|
|
c := r.Check
|
|
switch {
|
|
case c == nil:
|
|
return "", false
|
|
case c.Try:
|
|
// A try says it runs under its own id, which no other request shares: so a reader of the bucket can tell a try that runs
|
|
// from a check left waiting for a redelivery (ADR 0483 clause 3).
|
|
return "try." + tokenOf(r.ID), true
|
|
case c.Group != "":
|
|
return "group." + tokenOf(c.Group), true
|
|
case c.Number <= 0:
|
|
return "", false
|
|
}
|
|
return PullKeyOf(c.Owner, c.Repo, c.Number), true
|
|
}
|
|
|
|
// runningBucket opens the bucket of the checks running.
|
|
func runningBucket(ctx context.Context, conn *nats.Conn) (jetstream.KeyValue, error) {
|
|
api, err := jetstream.New(conn)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
kv, err := api.KeyValue(ctx, broker.CheckRunningBucket)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("the checks running: %w", err)
|
|
}
|
|
return kv, nil
|
|
}
|
|
|
|
// ReadRunning reads the key of the check running for a request's pull request: false when there is none.
|
|
func ReadRunning(ctx context.Context, conn *nats.Conn, r BuildRequest) (CheckRunning, bool, error) {
|
|
key, ok := RunningKeyOf(r)
|
|
if !ok {
|
|
return CheckRunning{}, false, nil
|
|
}
|
|
kv, err := runningBucket(ctx, conn)
|
|
if err != nil {
|
|
return CheckRunning{}, false, err
|
|
}
|
|
e, err := kv.Get(ctx, key)
|
|
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
|
return CheckRunning{}, false, nil
|
|
}
|
|
if err != nil {
|
|
return CheckRunning{}, false, err
|
|
}
|
|
var running CheckRunning
|
|
if err := json.Unmarshal(e.Value(), &running); err != nil {
|
|
return CheckRunning{}, false, nil // a key of a builder from before this means no match
|
|
}
|
|
running.Seen = e.Created()
|
|
return running, true, nil
|
|
}
|
|
|
|
// Same says a running check is for this request's head and trunk commit: both said, and the same. A key that names neither
|
|
// (an older builder's) is no match.
|
|
func (c CheckRunning) Same(r BuildRequest) bool {
|
|
return r.Check != nil && c.Head != "" && c.Trunk != "" && r.Check.Trunk != "" && sameCommit(c.Head, r.Ref) &&
|
|
sameCommit(c.Trunk, r.Check.Trunk)
|
|
}
|
|
|
|
// keepRunning says a check is running: its key put now, put again at each heartbeat, and deleted by the stop it returns,
|
|
// whatever the outcome. Said to nobody that cannot be told: a bucket that is not there is no reason not to run the check.
|
|
func keepRunning(ctx context.Context, conn *nats.Conn, r BuildRequest, builder string, every time.Duration) (stop func()) {
|
|
key, ok := RunningKeyOf(r)
|
|
if !ok {
|
|
return func() {}
|
|
}
|
|
kv, err := runningBucket(ctx, conn)
|
|
if err != nil {
|
|
fmt.Fprintf(os.Stderr, "%s runs without saying so: %v\n", r.ID, err)
|
|
return func() {}
|
|
}
|
|
return keepRunningIn(ctx, kv, key, r, builder, every, time.Now())
|
|
}
|
|
|
|
// runningStore is what keepRunning needs of the bucket: a put and a delete.
|
|
type runningStore interface {
|
|
Put(ctx context.Context, key string, value []byte) (uint64, error)
|
|
Delete(ctx context.Context, key string, opts ...jetstream.KVDeleteOpt) error
|
|
}
|
|
|
|
// keepRunningIn is keepRunning on a bucket in hand, the check having started at started: the value is made once, so every
|
|
// heartbeat puts the same started.
|
|
func keepRunningIn(ctx context.Context, kv runningStore, key string, r BuildRequest, builder string, every time.Duration, started time.Time) (stop func()) {
|
|
trunk := ""
|
|
if r.Check != nil {
|
|
trunk = r.Check.Trunk
|
|
}
|
|
body, _ := json.Marshal(CheckRunning{Head: r.Ref, Trunk: trunk, Builder: builder, Build: r.ID, Started: started.UTC()})
|
|
put := func() {
|
|
putting, end := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
|
|
defer end()
|
|
if _, err := kv.Put(putting, key, body); err != nil {
|
|
fmt.Fprintf(os.Stderr, "%s could not say it is running: %v\n", r.ID, err)
|
|
}
|
|
}
|
|
put()
|
|
done := make(chan struct{})
|
|
finished := make(chan struct{})
|
|
go func() {
|
|
defer close(finished)
|
|
tick := time.NewTicker(every)
|
|
defer tick.Stop()
|
|
for {
|
|
select {
|
|
case <-done:
|
|
return
|
|
case <-tick.C:
|
|
put()
|
|
}
|
|
}
|
|
}()
|
|
return func() {
|
|
close(done)
|
|
<-finished
|
|
deleting, end := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
|
|
defer end()
|
|
if err := kv.Delete(deleting, key); err != nil {
|
|
fmt.Fprintf(os.Stderr, "%s could not take its running key away; it ages out: %v\n", r.ID, err)
|
|
}
|
|
}
|
|
}
|