mesh-delivery/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go

// Command mesh-delivery-builder-tools serves the builder's tools (novox/hq issue 1945): the runtime launches it for a call.
// builder_stuck reads the builder's builds and reasks states and the build modules' steps through the builder's state and
// reads; builder_ask_again writes one value of the reasks state, which the builder watches and answers. The builder is the
// other program: it serves no tool (ADR 0437 §4).
package main

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"strings"
	"sync"
	"time"

	"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
	stdio "git.novox.be/novox/mesh-sdk/go"
	"git.novox.be/novox/mesh-sdk/go/delivery"
	"git.novox.be/novox/mesh-sdk/go/handacts"
	"git.novox.be/novox/mesh-sdk/go/views"

	"git.novox.be/novox/mesh-delivery/sdk/enginereads"

	"mesh-delivery-builder/internal/build"
	"mesh-delivery-builder/internal/versions"
)

// EngineReadsTool lists, asks again and drops what each counted node's engine reads, as the builder keeps it.
const EngineReadsTool = "builder_engine_reads"

// queueAsker writes a queues value and waits for the builder's answer to it.
type queueAsker func(key string, value []byte) (json.RawMessage, error)

// Tools are the builder's tools: stuck reads, askAgain writes the reasks value, queue writes a queues value and waits for
// the builder's read, buildAgain writes a reasks value of mode build and waits for the builder's answer.
// tags writes a tag-asks value and waits for the builder's answer; ta reads tag-asks.
func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, queue, buildAgain queueAsker, since build.LogSince, now func() time.Time, vs versions.Reader,
	tags queueAsker, ta, fw, of versions.Reader) []stdio.Tool {
	str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) }
	// handAct writes a reasks value of a versions hand act and answers the builder's answer, or its refusal as an error.
	handAct := func(key string, value []byte, err error) (any, error) {
		if err != nil {
			return nil, err
		}
		raw, err := buildAgain(key, value)
		if err != nil {
			return nil, err
		}
		a, err := build.DecodeReAsk(raw)
		if err != nil {
			return nil, err
		}
		if a.Refused {
			return nil, errors.New(a.Answer)
		}
		return map[string]any{"key": key, "name": a.Name, "do": a.Do, "version": a.Version, "answer": a.Answer}, nil
	}
	tagAct := func(key string, value []byte, err error) (any, error) {
		if err != nil {
			return nil, err
		}
		raw, err := tags(key, value)
		if err != nil {
			return nil, err
		}
		return versions.DecodeTagAsk(raw)
	}
	nameFP := map[string]any{"type": "object", "required": []string{"name", "fp", "why"}, "properties": map[string]any{
		"name": map[string]any{"type": "string", "description": "the numbered name, <module>/<artifact>, as builder_versions lists it"},
		"fp":   map[string]any{"type": "string", "description": "the fingerprint, src2:<hex>"},
		"why":  map[string]any{"type": "string", "description": "why, in plain words"}}}
	branch := func(head bool) map[string]any {
		req := []string{"repository", "branch", "why"}
		props := map[string]any{
			"repository": map[string]any{"type": "string", "description": "<owner>/<name>"},
			"branch":     map[string]any{"type": "string", "description": "the branch; its cleaned name is the tag"},
			"why":        map[string]any{"type": "string", "description": "why, in plain words"}}
		if head {
			req = append(req, "head")
			props["head"] = map[string]any{"type": "string", "description": "the commit the tagged version is built from"}
		}
		return map[string]any{"type": "object", "required": req, "properties": props}
	}
	return []stdio.Tool{
		{Name: versions.PublishAgainTool, Description: versions.PublishAgainDescription, Input: nameFP,
			Run: func(args map[string]any) (any, error) {
				return handAct(build.NewPublishAgain(str(args, "name"), str(args, "fp"), str(args, "why"), stdio.CallerOf(args), now()))
			}},
		{Name: versions.FloorAgainTool, Description: versions.FloorAgainDescription,
			Input: map[string]any{"type": "object", "required": []string{"name", "why"}, "properties": map[string]any{
				"name": map[string]any{"type": "string", "description": "the numbered name, <module>/<artifact>"},
				"why":  map[string]any{"type": "string", "description": "why, in plain words"}}},
			Run: func(args map[string]any) (any, error) {
				return handAct(build.NewFloorAgain(str(args, "name"), str(args, "why"), stdio.CallerOf(args), now()))
			}},
		{Name: versions.TagTool, Description: versions.TagDescription, Input: branch(true),
			Run: func(args map[string]any) (any, error) {
				return tagAct(versions.NewTagAsk(str(args, "repository"), str(args, "branch"), str(args, "head"), str(args, "why"), stdio.CallerOf(args), now()))
			}},
		{Name: versions.UntagTool, Description: versions.UntagDescription, Input: branch(false),
			Run: func(args map[string]any) (any, error) {
				return tagAct(versions.NewUntag(str(args, "repository"), str(args, "branch"), str(args, "why"), stdio.CallerOf(args), now()))
			}},
		{Name: versions.TagAsksTool, Description: versions.TagAsksDescription,
			Input: map[string]any{"type": "object", "properties": map[string]any{
				"repository": map[string]any{"type": "string", "description": "only this repository's asks, <owner>/<name>"}}},
			Run: func(args map[string]any) (any, error) { return versions.AnswerTagAsks(ta, str(args, "repository")) }},
		{Name: versions.OldFormTool, Description: versions.OldFormDescription,
			Input: map[string]any{"type": "object", "properties": map[string]any{
				"drop": map[string]any{"type": "string", "description": "a retired node to drop from the count by hand"},
				"why":  map[string]any{"type": "string", "description": "with drop: why, in plain words"}}},
			Run: func(args map[string]any) (any, error) {
				if node := str(args, "drop"); node != "" {
					return handAct(build.NewOldFormDrop(node, str(args, "why"), stdio.CallerOf(args), now()))
				}
				return versions.AnswerOldForm(of)
			}},
		{Name: versions.FloorWaitsTool, Description: versions.FloorWaitsDescription,
			Input: map[string]any{"type": "object", "properties": map[string]any{}},
			Run:   func(map[string]any) (any, error) { return versions.AnswerFloorWaits(fw) }},
		{Name: versions.VersionsTool, Description: versions.VersionsDescription,
			Input: map[string]any{"type": "object", "required": []string{"name"}, "properties": map[string]any{
				"name":    map[string]any{"type": "string", "description": "the package, image, bundle or archive name"},
				"version": map[string]any{"type": "string", "description": "one version, to ask whether it is recorded"},
				"digest":  map[string]any{"type": "string", "description": "with version: the digest the holder received"}}},
			Run: func(args map[string]any) (any, error) {
				return versions.AnswerVersions(context.Background(), vs, str(args, "name"), str(args, "version"), str(args, "digest"))
			}},
		{Name: build.StuckTool,
			Description: "Every build key that waits (building or queued), oldest first: its module, fingerprint, step words and since when, and each ask it " +
				"waits on by language with its id, when it was put, its asking (1 for the first; the builder asks an ask again when it can no longer be " +
				"answered, at most 3 times) and what its build module's steps hold of it (none, or its state on its node since its start). The " +
				"seat workers' delivery counts are not readable by the builder. dead_letters says whether the builder hears the asks the build seats give up " +
				"on (issue 734), and why not: heard, a given-up ask is asked again at once; deaf, at its check bound. With module, that module's keys only. Read-only.",
			Input: map[string]any{"type": "object", "properties": map[string]any{
				"module": map[string]any{"type": "string", "description": "only this module's keys"}}},
			Run: func(args map[string]any) (any, error) { return r.Stuck(str(args, "module")) }},
		{Name: build.AskAgainTool,
			Description: "Ask a waiting build key's asks again now, under new ids the key then names, as the builder does by itself for an ask its build " +
				"seat's stream gave up on. The builder answers in the reasks state, which " + build.StuckTool + " shows beside the key; a key no ask of " +
				"which waits is answered so and nothing is asked. An ask whose step already runs on a node is asked again too: it is then " +
				"built twice, the first run's work wasted and its end read by nobody, never a wrong key; builder_stuck shows whether a step runs.",
			Input: map[string]any{"type": "object", "required": []string{"module", "fp", "why"}, "properties": map[string]any{
				"module": map[string]any{"type": "string", "description": "the module's name"},
				"fp":     map[string]any{"type": "string", "description": "its new-form fingerprint, src2:<hex>, as builder_stuck shows it"},
				"why":    map[string]any{"type": "string", "description": "why it is asked again, in plain words"}}},
			Run: func(args map[string]any) (any, error) {
				key, value, err := build.NewReAsk(str(args, "module"), str(args, "fp"), str(args, "why"), stdio.CallerOf(args), now())
				if err != nil {
					return nil, err
				}
				if err := askAgain(key, value); err != nil {
					return nil, fmt.Errorf("the ask could not be written to %s: %w", build.ReAsksBucket, err)
				}
				return map[string]any{"key": key, "asked": true,
					"next": "the builder answers in " + build.ReAsksState + "." + key + "; " + build.StuckTool + " shows the answer and the new ask"}, nil
			}},
		{Name: build.BuildAgainTool,
			Description: "Build a failed key again (rebuild a module, build a failed key again): the builder writes the key of module and " +
				"fingerprint building as a new revision of the same key and asks the build seats, as it does by itself when it looks at a module " +
				"again (ADR 0429 clause 18). Answers the new revision and the ask id of each build module (go, node, image). Refuses a key that " +
				"is not failed, and says why: done (nothing to do), or waiting (building or queued: " + build.AskAgainTool + " asks its waiting asks " +
				"again); a key failed for a reason the builder never retries on the same fingerprint (a pin missing, a version taken, digests that " +
				"differ) and a fingerprint its module no longer has are refused too. Use it for a key builder_stuck does not show, which failed.",
			Input: map[string]any{"type": "object", "required": []string{"module", "fp", "why"}, "properties": map[string]any{
				"module": map[string]any{"type": "string", "description": "the module's name"},
				"fp":     map[string]any{"type": "string", "description": "its new-form fingerprint, src2:<hex>"},
				"why":    map[string]any{"type": "string", "description": "why it is built again, in plain words"}}},
			Run: func(args map[string]any) (any, error) {
				key, value, err := build.NewBuildAgain(str(args, "module"), str(args, "fp"), str(args, "why"), stdio.CallerOf(args), now())
				if err != nil {
					return nil, err
				}
				raw, err := buildAgain(key, value)
				if err != nil {
					return nil, err
				}
				r, err := build.DecodeReAsk(raw)
				if err != nil {
					return nil, err
				}
				if r.Refused {
					return nil, errors.New(r.Answer)
				}
				if r.Failed != "" {
					return nil, errors.New(r.Answer)
				}
				if r.Revision == 0 && !r.Pending {
					return nil, fmt.Errorf("the builder answered neither a new revision nor a refusal (an older builder?): %s", r.Answer)
				}
				return map[string]any{"key": key, "revision": r.Revision, "asks": r.Asks, "answer": r.Answer,
					"next": build.StuckTool + " shows the key and its asks while they wait"}, nil
			}},
		{Name: delivery.BuildsSinceTool,
			Description: "Every build that ended since a time, oldest first, from the builder's log build-log (kept 30 days): per build its module, " +
				"fingerprint, source revision, repository, directory, commit and ref, the build node, when it ended, its outcome (done, failed or " +
				"queued), the manifest built, the bases it was built against and each artifact made with its digest and reference. more says the " +
				"log held more than one answer carries: ask again from the last build's time. What the operator's command nox builds reads. Read-only.",
			Input: map[string]any{"type": "object", "required": []string{"since"}, "properties": map[string]any{
				"since": map[string]any{"type": "string", "description": "RFC 3339 time; the builds that ended at or after it"}}},
			Run: func(args map[string]any) (any, error) {
				at, err := time.Parse(time.RFC3339Nano, str(args, "since"))
				if err != nil {
					return nil, fmt.Errorf("since %q is not an RFC 3339 time: %w", str(args, "since"), err)
				}
				return build.BuildsSince(since, at)
			}},
		{Name: build.AnnounceAgainTool,
			Description: "Announce a done build again (take in again, compose again): the builder writes the done key of module and fingerprint " +
				"again, unchanged, as a new revision of the same key (its content, artifacts and manifest stay identical), so the controller's " +
				"take-in hears it and composes the module and writes its grant again. Use it when a done build was taken in but its composition " +
				"or grant is gone and every want is refused for it. Answers the new revision. Refuses a key that is not done: " + build.BuildAgainTool +
				" builds a failed key again, " + build.AskAgainTool + " asks a waiting key's asks again.",
			Input: map[string]any{"type": "object", "required": []string{"module", "fp", "why"}, "properties": map[string]any{
				"module": map[string]any{"type": "string", "description": "the module's name"},
				"fp":     map[string]any{"type": "string", "description": "its new-form fingerprint, src2:<hex>"},
				"why":    map[string]any{"type": "string", "description": "why it is announced again, in plain words"}}},
			Run: func(args map[string]any) (any, error) {
				key, value, err := build.NewAnnounceAgain(str(args, "module"), str(args, "fp"), str(args, "why"), stdio.CallerOf(args), now())
				if err != nil {
					return nil, err
				}
				raw, err := buildAgain(key, value)
				if err != nil {
					return nil, err
				}
				r, err := build.DecodeReAsk(raw)
				if err != nil {
					return nil, err
				}
				if r.Refused || r.Revision == 0 {
					return nil, errors.New(r.Answer)
				}
				return map[string]any{"key": key, "revision": r.Revision, "answer": r.Answer}, nil
			}},
		{Name: build.SeatQueueTool,
			Description: "What waits on a build seat's stream (SEAT_NODE_BUILD_<LANGUAGE>), as the builder reads it: the stream's message count and " +
				"sequences, and per kind (delivery, check, recheck) its messages oldest first, at most limit each, with sequence, ask id, when asked " +
				"and put, age, the state of the step a build node wrote for it, and whether the builder counts it as still waiting or as given up " +
				"on (its worker delivered it its most times) or taken; and the ask asked last that a node took. The workers' delivery counts are " +
				"not readable by the builder (nats.nats_streams shows them). Read by the builder, which alone may read the stream; read-only.",
			Input: map[string]any{"type": "object", "properties": map[string]any{
				"language": map[string]any{"type": "string", "description": "go, node or image (default go)"},
				"seat":     map[string]any{"type": "string", "description": "or the seat: node-build-go, node-build-node, node-build-image"},
				"limit":    map[string]any{"type": "integer", "description": fmt.Sprintf("messages read of each kind (default %d, at most %d)", build.DefaultQueueLimit, build.MaxQueueLimit)}}},
			Run: func(args map[string]any) (any, error) {
				language := build.LanguageOf(str(args, "language"))
				if language == "" {
					language = build.LanguageOf(str(args, "seat"))
				}
				if language == "" {
					language = delivery.AskGo
				}
				limit, _ := args["limit"].(float64)
				key, value, err := build.NewQueueAsk(language, int(limit), stdio.CallerOf(args), now())
				if err != nil {
					return nil, err
				}
				return queue(key, value)
			}},
	}
}

// askQueue writes the queues value at key and waits, on a watch of the key, at most bound for the builder's answer.
func askQueue(s stdio.KeptState, bound time.Duration) queueAsker {
	return askKept(s, build.QueuesState, build.QueuesBucket, "the read", bound, func(v []byte) bool {
		a, err := build.DecodeQueueAsk(v)
		return err == nil && !a.Answered.IsZero()
	})
}

// askKept writes a value at key of a state the builder watches and answers there, and waits, on a watch of the key, at most
// bound for a value written after it that answered says is the builder's answer.
func askKept(s stdio.KeptState, state, bucket, what string, bound time.Duration, answered func([]byte) bool) queueAsker {
	return func(key string, value []byte) (json.RawMessage, error) {
		heard := make(chan json.RawMessage, 1)
		var rev uint64
		var mu sync.Mutex
		if err := s.Watch(key, func(c stdio.StateChange) error {
			mu.Lock()
			mine := rev
			mu.Unlock()
			if c.Op != "put" || mine == 0 || c.Revision <= mine {
				return nil
			}
			if answered(c.Value) {
				select {
				case heard <- c.Value:
				default:
				}
			}
			return nil
		}); err != nil {
			return nil, fmt.Errorf("watching %s.%s: %w", state, key, err)
		}
		r, err := s.Put(key, json.RawMessage(value))
		if err != nil {
			return nil, fmt.Errorf("%s could not be written to %s: %w", what, bucket, err)
		}
		mu.Lock()
		rev = r
		mu.Unlock()
		// The watch may have heard the answer before the revision was known: read it once too.
		if e, err := s.Get(key); err == nil && e != nil && e.Revision > r && answered(e.Value) {
			return e.Value, nil
		}
		wait, cancel := context.WithTimeout(context.Background(), bound) // a context's bound, no timer (ADR 0437 §8)
		defer cancel()
		select {
		case v := <-heard:
			return v, nil
		case <-wait.Done():
			return nil, fmt.Errorf("the builder did not answer %s.%s within %s: it is not running, or cannot read %s", state, key, bound, bucket)
		}
	}
}

func read(s stdio.KeptState) build.StateRead {
	return build.StateRead{Keys: s.Keys, Get: func(key string) (json.RawMessage, time.Time, error) {
		e, err := s.Get(key)
		if err != nil || e == nil {
			return nil, time.Time{}, err
		}
		return e.Value, time.Time{}, nil
	}, Written: func(key string) time.Time {
		e, err := s.Get(key)
		if err != nil || e == nil {
			return time.Time{}
		}
		return written(s, key, e.Revision)
	}}
}

// written is when the key's revision was written, from its history, or zero.
func written(s stdio.KeptState, key string, revision uint64) time.Time {
	h, err := s.History(key)
	if err != nil {
		return time.Time{}
	}
	for _, r := range h {
		if r.Revision == revision {
			return r.At
		}
	}
	return time.Time{}
}

func main() {
	steps := map[string]build.StateRead{}
	for _, bm := range build.StuckBuildModules {
		steps[bm] = read(stdio.State(bm + ".steps"))
	}
	reasks := stdio.State(build.ReAsksState)
	r := build.StuckReader{Logf: func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-delivery-builder-tools] "+f+"\n", a...) }, Builds: read(stdio.State("builds")), ReAsks: read(reasks), Steps: steps,
		Hearing: read(stdio.State(buildrun.HearingState))}
	put := func(key string, value []byte) error {
		_, err := reasks.Put(key, json.RawMessage(value))
		return err
	}
	buildAgain := askKept(reasks, build.ReAsksState, build.ReAsksBucket, "the request", 25*time.Second, func(v []byte) bool {
		a, err := build.DecodeReAsk(v)
		return err == nil && !a.Answered.IsZero()
	})
	vs := stdio.State(versions.State)
	vr := versions.Reader{Keys: vs.Keys, Get: func(key string) (json.RawMessage, bool, error) {
		e, err := vs.Get(key)
		if err != nil || e == nil {
			return nil, false, err
		}
		return e.Value, true, nil
	}}
	ts := stdio.State(versions.TagAsksState)
	tags := askKept(ts, versions.TagAsksState, versions.TagAsksBucket, "the ask", 25*time.Second, func(v []byte) bool {
		a, err := versions.DecodeTagAsk(v)
		return err == nil && !a.Answered.IsZero()
	})
	tr := versions.Reader{Keys: ts.Keys, Get: func(key string) (json.RawMessage, bool, error) {
		e, err := ts.Get(key)
		if err != nil || e == nil {
			return nil, false, err
		}
		return e.Value, true, nil
	}}
	fs := stdio.State(versions.FloorWaitsState)
	fr := versions.Reader{Keys: fs.Keys, Get: func(key string) (json.RawMessage, bool, error) {
		e, err := fs.Get(key)
		if err != nil || e == nil {
			return nil, false, err
		}
		return e.Value, true, nil
	}}
	ofs := stdio.State(versions.OldFormState)
	ofr := versions.Reader{Keys: ofs.Keys, Get: func(key string) (json.RawMessage, bool, error) {
		e, err := ofs.Get(key)
		if err != nil || e == nil {
			return nil, false, err
		}
		return e.Value, true, nil
	}}
	tools := Tools(r, put, askQueue(stdio.State(build.QueuesState), 25*time.Second), buildAgain, stdio.Log(delivery.BuilderLog).Since, time.Now, vr, tags, tr, fr, ofr)
	// What each counted node's engine reads, as the builder keeps it (ADR 0618 §2, ADR 0624 §1): listed, one node asked
	// again, or one node's key dropped.
	tools = append(tools, enginereads.Tool(EngineReadsTool, "the builder", enginereads.KeptState(stdio.State("engine-reads")), true, time.Now))
	// builder_first_views and builder_first_view_again: where the builder's first views of the indexer's and the assigner's
	// states stand, and asking one again by hand (ADR 0615, DLV-M1).
	tools = append(tools, views.StatusTools(delivery.BuilderModule, "builder_first_views", "builder_first_view_again", stdio.State(views.StatusState), handacts.Recorder{Ask: stdio.Ask}, time.Now)...)
	if err := stdio.Serve("", tools); err != nil {
		fmt.Fprintf(os.Stderr, "[mesh-delivery-builder-tools] %v\n", err)
		os.Exit(1)
	}
}