mesh-delivery/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go

// Command mesh-delivery-builder-tools serves the builder's tools (novox/hq issue 1945): the runtime launches it for a call.
// builder_stuck reads the builder's builds and reasks states and the build modules' steps through the builder's state and
// reads; builder_ask_again writes one value of the reasks state, which the builder watches and answers. The builder is the
// other program: it serves no tool (ADR 0437 §4).
package main

import (
	"context"
	"encoding/json"
	"fmt"
	"os"
	"strings"
	"sync"
	"time"

	stdio "git.novox.be/novox/mesh-sdk/go"
	"git.novox.be/novox/mesh-sdk/go/delivery"
	"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"

	"mesh-delivery-builder/internal/build"
)

// queueAsker writes a queues value and waits for the builder's answer to it.
type queueAsker func(key string, value []byte) (json.RawMessage, error)

// Tools are the builder's tools: stuck reads, askAgain writes the reasks value, queue writes a queues value and waits for
// the builder's read.
func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, queue queueAsker, now func() time.Time) []stdio.Tool {
	str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) }
	return []stdio.Tool{
		{Name: build.StuckTool,
			Description: "Every build key that waits (building or queued), oldest first: its module, fingerprint, step words and since when, and each ask it " +
				"waits on by language with its id, when it was put, its asking (1 for the first; the builder asks an ask again when it can no longer be " +
				"answered, at most 3 times) and what its build module's steps hold of it (none, or its state on its node since its start). The " +
				"seat workers' delivery counts are not readable by the builder. dead_letters says whether the builder hears the asks the build seats give up " +
				"on (issue 734), and why not: heard, a given-up ask is asked again at once; deaf, at its check bound. With module, that module's keys only. Read-only.",
			Input: map[string]any{"type": "object", "properties": map[string]any{
				"module": map[string]any{"type": "string", "description": "only this module's keys"}}},
			Run: func(args map[string]any) (any, error) { return r.Stuck(str(args, "module")) }},
		{Name: build.AskAgainTool,
			Description: "Ask a waiting build key's asks again now, under new ids the key then names, as the builder does by itself for an ask its build " +
				"seat's stream gave up on. The builder answers in the reasks state, which " + build.StuckTool + " shows beside the key; a key no ask of " +
				"which waits is answered so and nothing is asked. An ask whose step already runs on a node is asked again too: it is then " +
				"built twice, the first run's work wasted and its end read by nobody, never a wrong key; builder_stuck shows whether a step runs.",
			Input: map[string]any{"type": "object", "required": []string{"module", "fp", "why"}, "properties": map[string]any{
				"module": map[string]any{"type": "string", "description": "the module's name"},
				"fp":     map[string]any{"type": "string", "description": "its new-form fingerprint, src2:<hex>, as builder_stuck shows it"},
				"why":    map[string]any{"type": "string", "description": "why it is asked again, in plain words"}}},
			Run: func(args map[string]any) (any, error) {
				key, value, err := build.NewReAsk(str(args, "module"), str(args, "fp"), str(args, "why"), stdio.CallerOf(args), now())
				if err != nil {
					return nil, err
				}
				if err := askAgain(key, value); err != nil {
					return nil, fmt.Errorf("the ask could not be written to %s: %w", build.ReAsksBucket, err)
				}
				return map[string]any{"key": key, "asked": true,
					"next": "the builder answers in " + build.ReAsksState + "." + key + "; " + build.StuckTool + " shows the answer and the new ask"}, nil
			}},
		{Name: build.SeatQueueTool,
			Description: "What waits on a build seat's stream (SEAT_NODE_BUILD_<LANGUAGE>), as the builder reads it: the stream's message count and " +
				"sequences, and per kind (delivery, check, recheck) its messages oldest first, at most limit each, with sequence, ask id, when asked " +
				"and put, age, the state of the step a build node wrote for it, and whether the builder counts it as still waiting or as given up " +
				"on (its worker delivered it its most times) or taken; and the ask asked last that a node took. The workers' delivery counts are " +
				"not readable by the builder (nats.nats_streams shows them). Read by the builder, which alone may read the stream; read-only.",
			Input: map[string]any{"type": "object", "properties": map[string]any{
				"language": map[string]any{"type": "string", "description": "go, node or image (default go)"},
				"seat":     map[string]any{"type": "string", "description": "or the seat: node-build-go, node-build-node, node-build-image"},
				"limit":    map[string]any{"type": "integer", "description": fmt.Sprintf("messages read of each kind (default %d, at most %d)", build.DefaultQueueLimit, build.MaxQueueLimit)}}},
			Run: func(args map[string]any) (any, error) {
				language := build.LanguageOf(str(args, "language"))
				if language == "" {
					language = build.LanguageOf(str(args, "seat"))
				}
				if language == "" {
					language = delivery.AskGo
				}
				limit, _ := args["limit"].(float64)
				key, value, err := build.NewQueueAsk(language, int(limit), stdio.CallerOf(args), now())
				if err != nil {
					return nil, err
				}
				return queue(key, value)
			}},
	}
}

// askQueue writes the queues value at key and waits, on a watch of the key, at most bound for the builder's answer.
func askQueue(s stdio.KeptState, bound time.Duration) queueAsker {
	return func(key string, value []byte) (json.RawMessage, error) {
		answered := make(chan json.RawMessage, 1)
		var rev uint64
		var mu sync.Mutex
		if err := s.Watch(key, func(c stdio.StateChange) error {
			mu.Lock()
			mine := rev
			mu.Unlock()
			if c.Op != "put" || mine == 0 || c.Revision <= mine {
				return nil
			}
			if a, err := build.DecodeQueueAsk(c.Value); err == nil && !a.Answered.IsZero() {
				select {
				case answered <- c.Value:
				default:
				}
			}
			return nil
		}); err != nil {
			return nil, fmt.Errorf("watching %s.%s: %w", build.QueuesState, key, err)
		}
		r, err := s.Put(key, json.RawMessage(value))
		if err != nil {
			return nil, fmt.Errorf("the read could not be written to %s: %w", build.QueuesBucket, err)
		}
		mu.Lock()
		rev = r
		mu.Unlock()
		// The watch may have heard the answer before the revision was known: read it once too.
		if e, err := s.Get(key); err == nil && e != nil && e.Revision > r {
			if a, err := build.DecodeQueueAsk(e.Value); err == nil && !a.Answered.IsZero() {
				return e.Value, nil
			}
		}
		wait, cancel := context.WithTimeout(context.Background(), bound) // a context's bound, no timer (ADR 0437 §8)
		defer cancel()
		select {
		case v := <-answered:
			return v, nil
		case <-wait.Done():
			return nil, fmt.Errorf("the builder did not answer %s.%s within %s: it is not running, or cannot read %s", build.QueuesState, key, bound, build.QueuesBucket)
		}
	}
}

func read(s stdio.KeptState) build.StateRead {
	return build.StateRead{Keys: s.Keys, Get: func(key string) (json.RawMessage, time.Time, error) {
		e, err := s.Get(key)
		if err != nil || e == nil {
			return nil, time.Time{}, err
		}
		return e.Value, time.Time{}, nil
	}, Written: func(key string) time.Time {
		e, err := s.Get(key)
		if err != nil || e == nil {
			return time.Time{}
		}
		return written(s, key, e.Revision)
	}}
}

// written is when the key's revision was written, from its history, or zero.
func written(s stdio.KeptState, key string, revision uint64) time.Time {
	h, err := s.History(key)
	if err != nil {
		return time.Time{}
	}
	for _, r := range h {
		if r.Revision == revision {
			return r.At
		}
	}
	return time.Time{}
}

func main() {
	steps := map[string]build.StateRead{}
	for _, bm := range build.StuckBuildModules {
		steps[bm] = read(stdio.State(bm + ".steps"))
	}
	reasks := stdio.State(build.ReAsksState)
	r := build.StuckReader{Logf: func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-delivery-builder-tools] "+f+"\n", a...) }, Builds: read(stdio.State("builds")), ReAsks: read(reasks), Steps: steps,
		Hearing: read(stdio.State(buildrun.HearingState))}
	put := func(key string, value []byte) error {
		_, err := reasks.Put(key, json.RawMessage(value))
		return err
	}
	if err := stdio.Serve("", Tools(r, put, askQueue(stdio.State(build.QueuesState), 25*time.Second), time.Now)); err != nil {
		fmt.Fprintf(os.Stderr, "[mesh-delivery-builder-tools] %v\n", err)
		os.Exit(1)
	}
}