// Command mesh-delivery-builder-tools serves the builder's tools (novox/hq issue 1945): the runtime launches it for a call. // builder_stuck reads the builder's builds and reasks states and the build modules' steps through the builder's state and // reads; builder_ask_again writes one value of the reasks state, which the builder watches and answers. The builder is the // other program: it serves no tool (ADR 0437 §4). package main import ( "context" "encoding/json" "fmt" "os" "strings" "sync" "time" stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/delivery" "mesh-delivery-builder/internal/build" ) // queueAsker writes a queues value and waits for the builder's answer to it. type queueAsker func(key string, value []byte) (json.RawMessage, error) // Tools are the builder's tools: stuck reads, askAgain writes the reasks value, queue writes a queues value and waits for // the builder's read. func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, queue queueAsker, now func() time.Time) []stdio.Tool { str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) } return []stdio.Tool{ {Name: build.StuckTool, Description: "Every build key that waits (building or queued), oldest first: its module, fingerprint, step words and since when, and each ask it " + "waits on by language with its id, when it was put, its asking (1 for the first; the builder asks an ask again when it can no longer be " + "answered, at most 3 times) and what its build module's steps hold of it (none, or its state on its node since its start). The " + "seat workers' delivery counts are not readable by the builder. With module, that module's keys only. Read-only.", Input: map[string]any{"type": "object", "properties": map[string]any{ "module": map[string]any{"type": "string", "description": "only this module's keys"}}}, Run: func(args map[string]any) (any, error) { return r.Stuck(str(args, "module")) }}, {Name: build.AskAgainTool, Description: "Ask a waiting build key's asks again now, under new ids the key then names, as the builder does by itself for an ask its build " + "seat's stream gave up on. The builder answers in the reasks state, which " + build.StuckTool + " shows beside the key; a key no ask of " + "which waits is answered so and nothing is asked. An ask whose step already runs on a node is asked again too: it is then " + "built twice, the first run's work wasted and its end read by nobody, never a wrong key; builder_stuck shows whether a step runs.", Input: map[string]any{"type": "object", "required": []string{"module", "fp", "why"}, "properties": map[string]any{ "module": map[string]any{"type": "string", "description": "the module's name"}, "fp": map[string]any{"type": "string", "description": "its new-form fingerprint, src2:, as builder_stuck shows it"}, "why": map[string]any{"type": "string", "description": "why it is asked again, in plain words"}}}, Run: func(args map[string]any) (any, error) { key, value, err := build.NewReAsk(str(args, "module"), str(args, "fp"), str(args, "why"), stdio.CallerOf(args), now()) if err != nil { return nil, err } if err := askAgain(key, value); err != nil { return nil, fmt.Errorf("the ask could not be written to %s: %w", build.ReAsksBucket, err) } return map[string]any{"key": key, "asked": true, "next": "the builder answers in " + build.ReAsksState + "." + key + "; " + build.StuckTool + " shows the answer and the new ask"}, nil }}, {Name: build.SeatQueueTool, Description: "What waits on a build seat's stream (SEAT_NODE_BUILD_), as the builder reads it: the stream's message count and " + "sequences, and per kind (delivery, check, recheck) its messages oldest first, at most limit each, with sequence, ask id, when asked " + "and put, age, the state of the step a build node wrote for it, and whether the builder counts it as still waiting or as given up " + "on (its worker delivered it its most times) or taken; and the ask asked last that a node took. The workers' delivery counts are " + "not readable by the builder (nats.nats_streams shows them). Read by the builder, which alone may read the stream; read-only.", Input: map[string]any{"type": "object", "properties": map[string]any{ "language": map[string]any{"type": "string", "description": "go, node or image (default go)"}, "seat": map[string]any{"type": "string", "description": "or the seat: node-build-go, node-build-node, node-build-image"}, "limit": map[string]any{"type": "integer", "description": fmt.Sprintf("messages read of each kind (default %d, at most %d)", build.DefaultQueueLimit, build.MaxQueueLimit)}}}, Run: func(args map[string]any) (any, error) { language := build.LanguageOf(str(args, "language")) if language == "" { language = build.LanguageOf(str(args, "seat")) } if language == "" { language = delivery.AskGo } limit, _ := args["limit"].(float64) key, value, err := build.NewQueueAsk(language, int(limit), stdio.CallerOf(args), now()) if err != nil { return nil, err } return queue(key, value) }}, } } // askQueue writes the queues value at key and waits, on a watch of the key, at most bound for the builder's answer. func askQueue(s stdio.KeptState, bound time.Duration) queueAsker { return func(key string, value []byte) (json.RawMessage, error) { answered := make(chan json.RawMessage, 1) var rev uint64 var mu sync.Mutex if err := s.Watch(key, func(c stdio.StateChange) error { mu.Lock() mine := rev mu.Unlock() if c.Op != "put" || mine == 0 || c.Revision <= mine { return nil } if a, err := build.DecodeQueueAsk(c.Value); err == nil && !a.Answered.IsZero() { select { case answered <- c.Value: default: } } return nil }); err != nil { return nil, fmt.Errorf("watching %s.%s: %w", build.QueuesState, key, err) } r, err := s.Put(key, json.RawMessage(value)) if err != nil { return nil, fmt.Errorf("the read could not be written to %s: %w", build.QueuesBucket, err) } mu.Lock() rev = r mu.Unlock() // The watch may have heard the answer before the revision was known: read it once too. if e, err := s.Get(key); err == nil && e != nil && e.Revision > r { if a, err := build.DecodeQueueAsk(e.Value); err == nil && !a.Answered.IsZero() { return e.Value, nil } } wait, cancel := context.WithTimeout(context.Background(), bound) // a context's bound, no timer (ADR 0437 §8) defer cancel() select { case v := <-answered: return v, nil case <-wait.Done(): return nil, fmt.Errorf("the builder did not answer %s.%s within %s: it is not running, or cannot read %s", build.QueuesState, key, bound, build.QueuesBucket) } } } func read(s stdio.KeptState) build.StateRead { return build.StateRead{Keys: s.Keys, Get: func(key string) (json.RawMessage, time.Time, error) { e, err := s.Get(key) if err != nil || e == nil { return nil, time.Time{}, err } return e.Value, time.Time{}, nil }, Written: func(key string) time.Time { e, err := s.Get(key) if err != nil || e == nil { return time.Time{} } return written(s, key, e.Revision) }} } // written is when the key's revision was written, from its history, or zero. func written(s stdio.KeptState, key string, revision uint64) time.Time { h, err := s.History(key) if err != nil { return time.Time{} } for _, r := range h { if r.Revision == revision { return r.At } } return time.Time{} } func main() { steps := map[string]build.StateRead{} for _, bm := range build.StuckBuildModules { steps[bm] = read(stdio.State(bm + ".steps")) } reasks := stdio.State(build.ReAsksState) r := build.StuckReader{Logf: func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-delivery-builder-tools] "+f+"\n", a...) }, Builds: read(stdio.State("builds")), ReAsks: read(reasks), Steps: steps} put := func(key string, value []byte) error { _, err := reasks.Put(key, json.RawMessage(value)) return err } if err := stdio.Serve("", Tools(r, put, askQueue(stdio.State(build.QueuesState), 25*time.Second), time.Now)); err != nil { fmt.Fprintf(os.Stderr, "[mesh-delivery-builder-tools] %v\n", err) os.Exit(1) } }