jschoubben
9db15c7cf4
mesh/delivery merged: source.main moved to novox/mesh-delivery@b842b9422d2a
mesh/check checking 9db15c7cf4fc on b842b9422d2a
builder_stuck, checker_kept, checker_verdicts, checker_status, checker_previews, checker_pull and checker_try listed a bucket of thousands of keys and read each value (builder_stuck also its history) one round trip at a time before filtering, so they timed out at 30 s. Now keys are narrowed by what the key carries first, history is read only for rows kept, values are read sixteen at a time, at most 1000 a call with the answer saying it was cut, steps newest first, a new try reads only the newest 64 tries, and each tool logs the values read and the time taken. Same kind as the view's fix (PR 52); the helper is copied, not put in mesh-sdk, as a few lines do not earn a group delivery.
80 lines
2.2 KiB
Go
80 lines
2.2 KiB
Go
package check
|
|
|
|
import (
|
|
"fmt"
|
|
"sort"
|
|
"sync"
|
|
"sync/atomic"
|
|
|
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
|
)
|
|
|
|
// ReadWorkers is how many values a read tool asks the bus for at once. Each value is a round trip from the tool through the
|
|
// runtime to the bus, so reading a bucket of thousands of keys one after the other outlasts the 30 s a tool call is given
|
|
// (issue 1827, as the delivery view's issue 1945). It is a copy of the view's: a few lines are not worth a group delivery
|
|
// through mesh-sdk (rules-are-weighed).
|
|
const ReadWorkers = 16
|
|
|
|
// MaxReads bounds the values one call of a read tool reads; past it the answer says it was cut and how to narrow it.
|
|
const MaxReads = 1000
|
|
|
|
// parallel runs read(i) for i in [0, n), at most ReadWorkers at a time, and answers the first error; once one failed no
|
|
// further read starts.
|
|
func parallel(n int, read func(i int) error) error {
|
|
var (
|
|
next atomic.Int64
|
|
failed atomic.Bool
|
|
once sync.Once
|
|
first error
|
|
wg sync.WaitGroup
|
|
)
|
|
for w := 0; w < min(ReadWorkers, n); w++ {
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
for !failed.Load() {
|
|
i := int(next.Add(1) - 1)
|
|
if i >= n {
|
|
return
|
|
}
|
|
if err := read(i); err != nil {
|
|
once.Do(func() { first = err })
|
|
failed.Store(true)
|
|
}
|
|
}
|
|
}()
|
|
}
|
|
wg.Wait()
|
|
return first
|
|
}
|
|
|
|
// capKeys keeps the first MaxReads keys and answers a note when it cut the rest, "" otherwise.
|
|
func capKeys(keys []string, narrow string) ([]string, string) {
|
|
if len(keys) <= MaxReads {
|
|
return keys, ""
|
|
}
|
|
return keys[:MaxReads], fmt.Sprintf("read %d of %d matching keys and stopped; %s", MaxReads, len(keys), narrow)
|
|
}
|
|
|
|
// newestFirst sorts step keys (<node>.<id>) by id, newest first: an id is the time in base 36, then six hex digits, so the
|
|
// longer id is the later and equal lengths compare as text. A key that is no step key goes last.
|
|
func newestFirst(keys []string) {
|
|
id := func(k string) (string, bool) {
|
|
_, id, err := delivery.ParseStepKey(k)
|
|
return id, err == nil
|
|
}
|
|
sort.Slice(keys, func(i, j int) bool {
|
|
a, aok := id(keys[i])
|
|
b, bok := id(keys[j])
|
|
switch {
|
|
case aok != bok:
|
|
return aok
|
|
case len(a) != len(b):
|
|
return len(a) > len(b)
|
|
case a != b:
|
|
return a > b
|
|
}
|
|
return keys[i] > keys[j]
|
|
})
|
|
}
|