mesh-delivery/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go

// Command mesh-delivery-builder-tools serves the builder's tools (novox/hq issue 1945): the runtime launches it for a call.
// builder_stuck reads the builder's builds and reasks states and the build modules' steps through the builder's state and
// reads; builder_ask_again writes one value of the reasks state, which the builder watches and answers. The builder is the
// other program: it serves no tool (ADR 0437 §4).
package main

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"strings"
	"sync"
	"time"

	"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
	stdio "git.novox.be/novox/mesh-sdk/go"
	"git.novox.be/novox/mesh-sdk/go/delivery"
	"git.novox.be/novox/mesh-sdk/go/views"

	"mesh-delivery-builder/internal/build"
)

// queueAsker writes a queues value and waits for the builder's answer to it.
type queueAsker func(key string, value []byte) (json.RawMessage, error)

// Tools are the builder's tools: stuck reads, askAgain writes the reasks value, queue writes a queues value and waits for
// the builder's read, buildAgain writes a reasks value of mode build and waits for the builder's answer.
func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, queue, buildAgain queueAsker, since build.LogSince, now func() time.Time) []stdio.Tool {
	str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) }
	return []stdio.Tool{
		{Name: build.StuckTool,
			Description: "Every build key that waits (building or queued), oldest first: its module, fingerprint, step words and since when, and each ask it " +
				"waits on by language with its id, when it was put, its asking (1 for the first; the builder asks an ask again when it can no longer be " +
				"answered, at most 3 times) and what its build module's steps hold of it (none, or its state on its node since its start). The " +
				"seat workers' delivery counts are not readable by the builder. dead_letters says whether the builder hears the asks the build seats give up " +
				"on (issue 734), and why not: heard, a given-up ask is asked again at once; deaf, at its check bound. With module, that module's keys only. Read-only.",
			Input: map[string]any{"type": "object", "properties": map[string]any{
				"module": map[string]any{"type": "string", "description": "only this module's keys"}}},
			Run: func(args map[string]any) (any, error) { return r.Stuck(str(args, "module")) }},
		{Name: build.AskAgainTool,
			Description: "Ask a waiting build key's asks again now, under new ids the key then names, as the builder does by itself for an ask its build " +
				"seat's stream gave up on. The builder answers in the reasks state, which " + build.StuckTool + " shows beside the key; a key no ask of " +
				"which waits is answered so and nothing is asked. An ask whose step already runs on a node is asked again too: it is then " +
				"built twice, the first run's work wasted and its end read by nobody, never a wrong key; builder_stuck shows whether a step runs.",
			Input: map[string]any{"type": "object", "required": []string{"module", "fp", "why"}, "properties": map[string]any{
				"module": map[string]any{"type": "string", "description": "the module's name"},
				"fp":     map[string]any{"type": "string", "description": "its new-form fingerprint, src2:<hex>, as builder_stuck shows it"},
				"why":    map[string]any{"type": "string", "description": "why it is asked again, in plain words"}}},
			Run: func(args map[string]any) (any, error) {
				key, value, err := build.NewReAsk(str(args, "module"), str(args, "fp"), str(args, "why"), stdio.CallerOf(args), now())
				if err != nil {
					return nil, err
				}
				if err := askAgain(key, value); err != nil {
					return nil, fmt.Errorf("the ask could not be written to %s: %w", build.ReAsksBucket, err)
				}
				return map[string]any{"key": key, "asked": true,
					"next": "the builder answers in " + build.ReAsksState + "." + key + "; " + build.StuckTool + " shows the answer and the new ask"}, nil
			}},
		{Name: build.BuildAgainTool,
			Description: "Build a failed key again (rebuild a module, build a failed key again): the builder writes the key of module and " +
				"fingerprint building as a new revision of the same key and asks the build seats, as it does by itself when it looks at a module " +
				"again (ADR 0429 clause 18). Answers the new revision and the ask id of each build module (go, node, image). Refuses a key that " +
				"is not failed, and says why: done (nothing to do), or waiting (building or queued: " + build.AskAgainTool + " asks its waiting asks " +
				"again); a key failed for a reason the builder never retries on the same fingerprint (a pin missing, a version taken, digests that " +
				"differ) and a fingerprint its module no longer has are refused too. Use it for a key builder_stuck does not show, which failed.",
			Input: map[string]any{"type": "object", "required": []string{"module", "fp", "why"}, "properties": map[string]any{
				"module": map[string]any{"type": "string", "description": "the module's name"},
				"fp":     map[string]any{"type": "string", "description": "its new-form fingerprint, src2:<hex>"},
				"why":    map[string]any{"type": "string", "description": "why it is built again, in plain words"}}},
			Run: func(args map[string]any) (any, error) {
				key, value, err := build.NewBuildAgain(str(args, "module"), str(args, "fp"), str(args, "why"), stdio.CallerOf(args), now())
				if err != nil {
					return nil, err
				}
				raw, err := buildAgain(key, value)
				if err != nil {
					return nil, err
				}
				r, err := build.DecodeReAsk(raw)
				if err != nil {
					return nil, err
				}
				if r.Refused {
					return nil, errors.New(r.Answer)
				}
				if r.Failed != "" {
					return nil, errors.New(r.Answer)
				}
				if r.Revision == 0 && !r.Pending {
					return nil, fmt.Errorf("the builder answered neither a new revision nor a refusal (an older builder?): %s", r.Answer)
				}
				return map[string]any{"key": key, "revision": r.Revision, "asks": r.Asks, "answer": r.Answer,
					"next": build.StuckTool + " shows the key and its asks while they wait"}, nil
			}},
		{Name: delivery.BuildsSinceTool,
			Description: "Every build that ended since a time, oldest first, from the builder's log build-log (kept 30 days): per build its module, " +
				"fingerprint, source revision, repository, directory, commit and ref, the build node, when it ended, its outcome (done, failed or " +
				"queued), the manifest built, the bases it was built against and each artifact made with its digest and reference. more says the " +
				"log held more than one answer carries: ask again from the last build's time. What the operator's command nox builds reads. Read-only.",
			Input: map[string]any{"type": "object", "required": []string{"since"}, "properties": map[string]any{
				"since": map[string]any{"type": "string", "description": "RFC 3339 time; the builds that ended at or after it"}}},
			Run: func(args map[string]any) (any, error) {
				at, err := time.Parse(time.RFC3339Nano, str(args, "since"))
				if err != nil {
					return nil, fmt.Errorf("since %q is not an RFC 3339 time: %w", str(args, "since"), err)
				}
				return build.BuildsSince(since, at)
			}},
		{Name: build.AnnounceAgainTool,
			Description: "Announce a done build again (take in again, compose again): the builder writes the done key of module and fingerprint " +
				"again, unchanged, as a new revision of the same key (its content, artifacts and manifest stay identical), so the controller's " +
				"take-in hears it and composes the module and writes its grant again. Use it when a done build was taken in but its composition " +
				"or grant is gone and every want is refused for it. Answers the new revision. Refuses a key that is not done: " + build.BuildAgainTool +
				" builds a failed key again, " + build.AskAgainTool + " asks a waiting key's asks again.",
			Input: map[string]any{"type": "object", "required": []string{"module", "fp", "why"}, "properties": map[string]any{
				"module": map[string]any{"type": "string", "description": "the module's name"},
				"fp":     map[string]any{"type": "string", "description": "its new-form fingerprint, src2:<hex>"},
				"why":    map[string]any{"type": "string", "description": "why it is announced again, in plain words"}}},
			Run: func(args map[string]any) (any, error) {
				key, value, err := build.NewAnnounceAgain(str(args, "module"), str(args, "fp"), str(args, "why"), stdio.CallerOf(args), now())
				if err != nil {
					return nil, err
				}
				raw, err := buildAgain(key, value)
				if err != nil {
					return nil, err
				}
				r, err := build.DecodeReAsk(raw)
				if err != nil {
					return nil, err
				}
				if r.Refused || r.Revision == 0 {
					return nil, errors.New(r.Answer)
				}
				return map[string]any{"key": key, "revision": r.Revision, "answer": r.Answer}, nil
			}},
		{Name: build.SeatQueueTool,
			Description: "What waits on a build seat's stream (SEAT_NODE_BUILD_<LANGUAGE>), as the builder reads it: the stream's message count and " +
				"sequences, and per kind (delivery, check, recheck) its messages oldest first, at most limit each, with sequence, ask id, when asked " +
				"and put, age, the state of the step a build node wrote for it, and whether the builder counts it as still waiting or as given up " +
				"on (its worker delivered it its most times) or taken; and the ask asked last that a node took. The workers' delivery counts are " +
				"not readable by the builder (nats.nats_streams shows them). Read by the builder, which alone may read the stream; read-only.",
			Input: map[string]any{"type": "object", "properties": map[string]any{
				"language": map[string]any{"type": "string", "description": "go, node or image (default go)"},
				"seat":     map[string]any{"type": "string", "description": "or the seat: node-build-go, node-build-node, node-build-image"},
				"limit":    map[string]any{"type": "integer", "description": fmt.Sprintf("messages read of each kind (default %d, at most %d)", build.DefaultQueueLimit, build.MaxQueueLimit)}}},
			Run: func(args map[string]any) (any, error) {
				language := build.LanguageOf(str(args, "language"))
				if language == "" {
					language = build.LanguageOf(str(args, "seat"))
				}
				if language == "" {
					language = delivery.AskGo
				}
				limit, _ := args["limit"].(float64)
				key, value, err := build.NewQueueAsk(language, int(limit), stdio.CallerOf(args), now())
				if err != nil {
					return nil, err
				}
				return queue(key, value)
			}},
	}
}

// askQueue writes the queues value at key and waits, on a watch of the key, at most bound for the builder's answer.
func askQueue(s stdio.KeptState, bound time.Duration) queueAsker {
	return askKept(s, build.QueuesState, build.QueuesBucket, "the read", bound, func(v []byte) bool {
		a, err := build.DecodeQueueAsk(v)
		return err == nil && !a.Answered.IsZero()
	})
}

// askKept writes a value at key of a state the builder watches and answers there, and waits, on a watch of the key, at most
// bound for a value written after it that answered says is the builder's answer.
func askKept(s stdio.KeptState, state, bucket, what string, bound time.Duration, answered func([]byte) bool) queueAsker {
	return func(key string, value []byte) (json.RawMessage, error) {
		heard := make(chan json.RawMessage, 1)
		var rev uint64
		var mu sync.Mutex
		if err := s.Watch(key, func(c stdio.StateChange) error {
			mu.Lock()
			mine := rev
			mu.Unlock()
			if c.Op != "put" || mine == 0 || c.Revision <= mine {
				return nil
			}
			if answered(c.Value) {
				select {
				case heard <- c.Value:
				default:
				}
			}
			return nil
		}); err != nil {
			return nil, fmt.Errorf("watching %s.%s: %w", state, key, err)
		}
		r, err := s.Put(key, json.RawMessage(value))
		if err != nil {
			return nil, fmt.Errorf("%s could not be written to %s: %w", what, bucket, err)
		}
		mu.Lock()
		rev = r
		mu.Unlock()
		// The watch may have heard the answer before the revision was known: read it once too.
		if e, err := s.Get(key); err == nil && e != nil && e.Revision > r && answered(e.Value) {
			return e.Value, nil
		}
		wait, cancel := context.WithTimeout(context.Background(), bound) // a context's bound, no timer (ADR 0437 §8)
		defer cancel()
		select {
		case v := <-heard:
			return v, nil
		case <-wait.Done():
			return nil, fmt.Errorf("the builder did not answer %s.%s within %s: it is not running, or cannot read %s", state, key, bound, bucket)
		}
	}
}

func read(s stdio.KeptState) build.StateRead {
	return build.StateRead{Keys: s.Keys, Get: func(key string) (json.RawMessage, time.Time, error) {
		e, err := s.Get(key)
		if err != nil || e == nil {
			return nil, time.Time{}, err
		}
		return e.Value, time.Time{}, nil
	}, Written: func(key string) time.Time {
		e, err := s.Get(key)
		if err != nil || e == nil {
			return time.Time{}
		}
		return written(s, key, e.Revision)
	}}
}

// written is when the key's revision was written, from its history, or zero.
func written(s stdio.KeptState, key string, revision uint64) time.Time {
	h, err := s.History(key)
	if err != nil {
		return time.Time{}
	}
	for _, r := range h {
		if r.Revision == revision {
			return r.At
		}
	}
	return time.Time{}
}

func main() {
	steps := map[string]build.StateRead{}
	for _, bm := range build.StuckBuildModules {
		steps[bm] = read(stdio.State(bm + ".steps"))
	}
	reasks := stdio.State(build.ReAsksState)
	r := build.StuckReader{Logf: func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-delivery-builder-tools] "+f+"\n", a...) }, Builds: read(stdio.State("builds")), ReAsks: read(reasks), Steps: steps,
		Hearing: read(stdio.State(buildrun.HearingState))}
	put := func(key string, value []byte) error {
		_, err := reasks.Put(key, json.RawMessage(value))
		return err
	}
	buildAgain := askKept(reasks, build.ReAsksState, build.ReAsksBucket, "the request", 25*time.Second, func(v []byte) bool {
		a, err := build.DecodeReAsk(v)
		return err == nil && !a.Answered.IsZero()
	})
	// builder_first_views and builder_first_view_again: where the builder's first views of the indexer's and the assigner's
	// states stand, and asking one again by hand (ADR 0615, DLV-M1).
	tools := append(Tools(r, put, askQueue(stdio.State(build.QueuesState), 25*time.Second), buildAgain, stdio.Log(delivery.BuilderLog).Since, time.Now),
		views.StatusTools(delivery.BuilderModule, "builder_first_views", "builder_first_view_again", stdio.State(views.StatusState), time.Now)...)
	if err := stdio.Serve("", tools); err != nil {
		fmt.Fprintf(os.Stderr, "[mesh-delivery-builder-tools] %v\n", err)
		os.Exit(1)
	}
}