jschoubben
c6dd04a7cc
Review round 1 (issue 1126 note 72): the inflight check, the watch and the look were three locks, so a second request or the builder's own look could build the key twice; the answer said asked when the look wrote nothing or the build failed before its asks.
230 lines
11 KiB
Go
230 lines
11 KiB
Go
// Command mesh-delivery-builder-tools serves the builder's tools (novox/hq issue 1945): the runtime launches it for a call.
|
|
// builder_stuck reads the builder's builds and reasks states and the build modules' steps through the builder's state and
|
|
// reads; builder_ask_again writes one value of the reasks state, which the builder watches and answers. The builder is the
|
|
// other program: it serves no tool (ADR 0437 §4).
|
|
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
stdio "git.novox.be/novox/mesh-sdk/go"
|
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
|
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
|
|
|
"mesh-delivery-builder/internal/build"
|
|
)
|
|
|
|
// queueAsker writes a queues value and waits for the builder's answer to it.
|
|
type queueAsker func(key string, value []byte) (json.RawMessage, error)
|
|
|
|
// Tools are the builder's tools: stuck reads, askAgain writes the reasks value, queue writes a queues value and waits for
|
|
// the builder's read, buildAgain writes a reasks value of mode build and waits for the builder's answer.
|
|
func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, queue, buildAgain queueAsker, now func() time.Time) []stdio.Tool {
|
|
str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) }
|
|
return []stdio.Tool{
|
|
{Name: build.StuckTool,
|
|
Description: "Every build key that waits (building or queued), oldest first: its module, fingerprint, step words and since when, and each ask it " +
|
|
"waits on by language with its id, when it was put, its asking (1 for the first; the builder asks an ask again when it can no longer be " +
|
|
"answered, at most 3 times) and what its build module's steps hold of it (none, or its state on its node since its start). The " +
|
|
"seat workers' delivery counts are not readable by the builder. dead_letters says whether the builder hears the asks the build seats give up " +
|
|
"on (issue 734), and why not: heard, a given-up ask is asked again at once; deaf, at its check bound. With module, that module's keys only. Read-only.",
|
|
Input: map[string]any{"type": "object", "properties": map[string]any{
|
|
"module": map[string]any{"type": "string", "description": "only this module's keys"}}},
|
|
Run: func(args map[string]any) (any, error) { return r.Stuck(str(args, "module")) }},
|
|
{Name: build.AskAgainTool,
|
|
Description: "Ask a waiting build key's asks again now, under new ids the key then names, as the builder does by itself for an ask its build " +
|
|
"seat's stream gave up on. The builder answers in the reasks state, which " + build.StuckTool + " shows beside the key; a key no ask of " +
|
|
"which waits is answered so and nothing is asked. An ask whose step already runs on a node is asked again too: it is then " +
|
|
"built twice, the first run's work wasted and its end read by nobody, never a wrong key; builder_stuck shows whether a step runs.",
|
|
Input: map[string]any{"type": "object", "required": []string{"module", "fp", "why"}, "properties": map[string]any{
|
|
"module": map[string]any{"type": "string", "description": "the module's name"},
|
|
"fp": map[string]any{"type": "string", "description": "its new-form fingerprint, src2:<hex>, as builder_stuck shows it"},
|
|
"why": map[string]any{"type": "string", "description": "why it is asked again, in plain words"}}},
|
|
Run: func(args map[string]any) (any, error) {
|
|
key, value, err := build.NewReAsk(str(args, "module"), str(args, "fp"), str(args, "why"), stdio.CallerOf(args), now())
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if err := askAgain(key, value); err != nil {
|
|
return nil, fmt.Errorf("the ask could not be written to %s: %w", build.ReAsksBucket, err)
|
|
}
|
|
return map[string]any{"key": key, "asked": true,
|
|
"next": "the builder answers in " + build.ReAsksState + "." + key + "; " + build.StuckTool + " shows the answer and the new ask"}, nil
|
|
}},
|
|
{Name: build.BuildAgainTool,
|
|
Description: "Build a failed key again (rebuild a module, build a failed key again): the builder writes the key of module and " +
|
|
"fingerprint building as a new revision of the same key and asks the build seats, as it does by itself when it looks at a module " +
|
|
"again (ADR 0429 clause 18). Answers the new revision and the ask id of each build module (go, node, image). Refuses a key that " +
|
|
"is not failed, and says why: done (nothing to do), or waiting (building or queued: " + build.AskAgainTool + " asks its waiting asks " +
|
|
"again); a key failed for a reason the builder never retries on the same fingerprint (a pin missing, a version taken, digests that " +
|
|
"differ) and a fingerprint its module no longer has are refused too. Use it for a key builder_stuck does not show, which failed.",
|
|
Input: map[string]any{"type": "object", "required": []string{"module", "fp", "why"}, "properties": map[string]any{
|
|
"module": map[string]any{"type": "string", "description": "the module's name"},
|
|
"fp": map[string]any{"type": "string", "description": "its new-form fingerprint, src2:<hex>"},
|
|
"why": map[string]any{"type": "string", "description": "why it is built again, in plain words"}}},
|
|
Run: func(args map[string]any) (any, error) {
|
|
key, value, err := build.NewBuildAgain(str(args, "module"), str(args, "fp"), str(args, "why"), stdio.CallerOf(args), now())
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
raw, err := buildAgain(key, value)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
r, err := build.DecodeReAsk(raw)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if r.Refused {
|
|
return nil, errors.New(r.Answer)
|
|
}
|
|
if r.Failed != "" {
|
|
return nil, errors.New(r.Answer)
|
|
}
|
|
if r.Revision == 0 && !r.Pending {
|
|
return nil, fmt.Errorf("the builder answered neither a new revision nor a refusal (an older builder?): %s", r.Answer)
|
|
}
|
|
return map[string]any{"key": key, "revision": r.Revision, "asks": r.Asks, "answer": r.Answer,
|
|
"next": build.StuckTool + " shows the key and its asks while they wait"}, nil
|
|
}},
|
|
{Name: build.SeatQueueTool,
|
|
Description: "What waits on a build seat's stream (SEAT_NODE_BUILD_<LANGUAGE>), as the builder reads it: the stream's message count and " +
|
|
"sequences, and per kind (delivery, check, recheck) its messages oldest first, at most limit each, with sequence, ask id, when asked " +
|
|
"and put, age, the state of the step a build node wrote for it, and whether the builder counts it as still waiting or as given up " +
|
|
"on (its worker delivered it its most times) or taken; and the ask asked last that a node took. The workers' delivery counts are " +
|
|
"not readable by the builder (nats.nats_streams shows them). Read by the builder, which alone may read the stream; read-only.",
|
|
Input: map[string]any{"type": "object", "properties": map[string]any{
|
|
"language": map[string]any{"type": "string", "description": "go, node or image (default go)"},
|
|
"seat": map[string]any{"type": "string", "description": "or the seat: node-build-go, node-build-node, node-build-image"},
|
|
"limit": map[string]any{"type": "integer", "description": fmt.Sprintf("messages read of each kind (default %d, at most %d)", build.DefaultQueueLimit, build.MaxQueueLimit)}}},
|
|
Run: func(args map[string]any) (any, error) {
|
|
language := build.LanguageOf(str(args, "language"))
|
|
if language == "" {
|
|
language = build.LanguageOf(str(args, "seat"))
|
|
}
|
|
if language == "" {
|
|
language = delivery.AskGo
|
|
}
|
|
limit, _ := args["limit"].(float64)
|
|
key, value, err := build.NewQueueAsk(language, int(limit), stdio.CallerOf(args), now())
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return queue(key, value)
|
|
}},
|
|
}
|
|
}
|
|
|
|
// askQueue writes the queues value at key and waits, on a watch of the key, at most bound for the builder's answer.
|
|
func askQueue(s stdio.KeptState, bound time.Duration) queueAsker {
|
|
return askKept(s, build.QueuesState, build.QueuesBucket, "the read", bound, func(v []byte) bool {
|
|
a, err := build.DecodeQueueAsk(v)
|
|
return err == nil && !a.Answered.IsZero()
|
|
})
|
|
}
|
|
|
|
// askKept writes a value at key of a state the builder watches and answers there, and waits, on a watch of the key, at most
|
|
// bound for a value written after it that answered says is the builder's answer.
|
|
func askKept(s stdio.KeptState, state, bucket, what string, bound time.Duration, answered func([]byte) bool) queueAsker {
|
|
return func(key string, value []byte) (json.RawMessage, error) {
|
|
heard := make(chan json.RawMessage, 1)
|
|
var rev uint64
|
|
var mu sync.Mutex
|
|
if err := s.Watch(key, func(c stdio.StateChange) error {
|
|
mu.Lock()
|
|
mine := rev
|
|
mu.Unlock()
|
|
if c.Op != "put" || mine == 0 || c.Revision <= mine {
|
|
return nil
|
|
}
|
|
if answered(c.Value) {
|
|
select {
|
|
case heard <- c.Value:
|
|
default:
|
|
}
|
|
}
|
|
return nil
|
|
}); err != nil {
|
|
return nil, fmt.Errorf("watching %s.%s: %w", state, key, err)
|
|
}
|
|
r, err := s.Put(key, json.RawMessage(value))
|
|
if err != nil {
|
|
return nil, fmt.Errorf("%s could not be written to %s: %w", what, bucket, err)
|
|
}
|
|
mu.Lock()
|
|
rev = r
|
|
mu.Unlock()
|
|
// The watch may have heard the answer before the revision was known: read it once too.
|
|
if e, err := s.Get(key); err == nil && e != nil && e.Revision > r && answered(e.Value) {
|
|
return e.Value, nil
|
|
}
|
|
wait, cancel := context.WithTimeout(context.Background(), bound) // a context's bound, no timer (ADR 0437 §8)
|
|
defer cancel()
|
|
select {
|
|
case v := <-heard:
|
|
return v, nil
|
|
case <-wait.Done():
|
|
return nil, fmt.Errorf("the builder did not answer %s.%s within %s: it is not running, or cannot read %s", state, key, bound, bucket)
|
|
}
|
|
}
|
|
}
|
|
|
|
func read(s stdio.KeptState) build.StateRead {
|
|
return build.StateRead{Keys: s.Keys, Get: func(key string) (json.RawMessage, time.Time, error) {
|
|
e, err := s.Get(key)
|
|
if err != nil || e == nil {
|
|
return nil, time.Time{}, err
|
|
}
|
|
return e.Value, time.Time{}, nil
|
|
}, Written: func(key string) time.Time {
|
|
e, err := s.Get(key)
|
|
if err != nil || e == nil {
|
|
return time.Time{}
|
|
}
|
|
return written(s, key, e.Revision)
|
|
}}
|
|
}
|
|
|
|
// written is when the key's revision was written, from its history, or zero.
|
|
func written(s stdio.KeptState, key string, revision uint64) time.Time {
|
|
h, err := s.History(key)
|
|
if err != nil {
|
|
return time.Time{}
|
|
}
|
|
for _, r := range h {
|
|
if r.Revision == revision {
|
|
return r.At
|
|
}
|
|
}
|
|
return time.Time{}
|
|
}
|
|
|
|
func main() {
|
|
steps := map[string]build.StateRead{}
|
|
for _, bm := range build.StuckBuildModules {
|
|
steps[bm] = read(stdio.State(bm + ".steps"))
|
|
}
|
|
reasks := stdio.State(build.ReAsksState)
|
|
r := build.StuckReader{Logf: func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-delivery-builder-tools] "+f+"\n", a...) }, Builds: read(stdio.State("builds")), ReAsks: read(reasks), Steps: steps,
|
|
Hearing: read(stdio.State(buildrun.HearingState))}
|
|
put := func(key string, value []byte) error {
|
|
_, err := reasks.Put(key, json.RawMessage(value))
|
|
return err
|
|
}
|
|
buildAgain := askKept(reasks, build.ReAsksState, build.ReAsksBucket, "the request", 25*time.Second, func(v []byte) bool {
|
|
a, err := build.DecodeReAsk(v)
|
|
return err == nil && !a.Answered.IsZero()
|
|
})
|
|
if err := stdio.Serve("", Tools(r, put, askQueue(stdio.State(build.QueuesState), 25*time.Second), buildAgain, time.Now)); err != nil {
|
|
fmt.Fprintf(os.Stderr, "[mesh-delivery-builder-tools] %v\n", err)
|
|
os.Exit(1)
|
|
}
|
|
}
|