mesh-delivery/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go

// Command mesh-delivery-builder-tools serves the builder's tools (novox/hq issue 1945): the runtime launches it for a call.
// builder_stuck reads the builder's builds and reasks states and the build modules' steps through the builder's state and
// reads; builder_ask_again writes one value of the reasks state, which the builder watches and answers. The builder is the
// other program: it serves no tool (ADR 0437 §4).
package main

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"strings"
	"sync"
	"time"

	"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
	"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
	"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
	"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
	"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
	"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
	stdio "git.novox.be/novox/mesh-sdk/go"
	"git.novox.be/novox/mesh-sdk/go/seatdef"
	"git.novox.be/novox/mesh-sdk/go/seatlist"
	"git.novox.be/novox/mesh-sdk/go/views"

	"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient/enginereads"

	"mesh-delivery-builder/internal/build"
	"mesh-delivery-builder/internal/versions"
)

// EngineReadsTool lists, asks again and drops what each counted node's engine reads, as the builder keeps it.
const EngineReadsTool = "builder_engine_reads"

// queueAsker writes a queues value and waits for the builder's answer to it.
type queueAsker func(key string, value []byte) (json.RawMessage, error)

// Tools are the builder's tools: stuck reads, askAgain writes the reasks value, queue writes a queues value and waits for
// the builder's read, buildAgain writes a reasks value of mode build and waits for the builder's answer.
//
// builder_versions, asked by a name, answers also what the verb reserved answers for it (reserve; design 47 G §572): the
// bundle calls the seat mesh-builds through its invokes, as any caller does. Nil answers none.
func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, queue, buildAgain queueAsker, since build.LogSince, now func() time.Time, vs versions.Reader,
	fw, of versions.Reader, reserve func(name, version string) (meshbuilds.ReservedAnswer, error)) []stdio.Tool {
	str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) }
	// handAct writes a reasks value of a versions hand act and answers the builder's answer, or its refusal as an error.
	handAct := func(key string, value []byte, err error) (any, error) {
		if err != nil {
			return nil, err
		}
		raw, err := buildAgain(key, value)
		if err != nil {
			return nil, err
		}
		a, err := build.DecodeReAsk(raw)
		if err != nil {
			return nil, err
		}
		if a.Refused {
			return nil, errors.New(a.Answer)
		}
		return map[string]any{"key": key, "name": a.Name, "answer": a.Answer}, nil
	}
	return []stdio.Tool{
		{Name: versions.FloorAgainTool, Description: versions.FloorAgainDescription,
			Input: map[string]any{"type": "object", "required": []string{"name", "why"}, "properties": map[string]any{
				"name": map[string]any{"type": "string", "description": "the numbered name, <module>/<artifact>"},
				"why":  map[string]any{"type": "string", "description": "why, in plain words"}}},
			Run: func(args map[string]any) (any, error) {
				return handAct(build.NewFloorAgain(str(args, "name"), str(args, "why"), stdio.CallerOf(args), now()))
			}},
		{Name: versions.OldFormTool, Description: versions.OldFormDescription,
			Input: map[string]any{"type": "object", "properties": map[string]any{
				"drop": map[string]any{"type": "string", "description": "a retired node to drop from the count by hand"},
				"why":  map[string]any{"type": "string", "description": "with drop: why, in plain words"}}},
			Run: func(args map[string]any) (any, error) {
				if node := str(args, "drop"); node != "" {
					return handAct(build.NewOldFormDrop(node, str(args, "why"), stdio.CallerOf(args), now()))
				}
				return versions.AnswerOldForm(of)
			}},
		{Name: versions.FloorWaitsTool, Description: versions.FloorWaitsDescription,
			Input: map[string]any{"type": "object", "properties": map[string]any{}},
			Run:   func(map[string]any) (any, error) { return versions.AnswerFloorWaits(fw) }},
		{Name: versions.VersionsTool, Description: versions.VersionsDescription,
			Input: map[string]any{"type": "object", "required": []string{"name"}, "properties": map[string]any{
				"name":    map[string]any{"type": "string", "description": "the package, image, bundle or archive name"},
				"version": map[string]any{"type": "string", "description": "one version, to ask whether it is recorded"},
				"digest":  map[string]any{"type": "string", "description": "with version: the digest the holder received"}}},
			Run: func(args map[string]any) (any, error) {
				out, err := versions.AnswerVersions(context.Background(), vs, str(args, "name"), str(args, "version"), str(args, "digest"))
				if err != nil || reserve == nil {
					return out, err
				}
				ra, err := reserve(out.Name, str(args, "version"))
				if err != nil {
					return nil, fmt.Errorf("the seat mesh-builds' reserved did not answer for %s: %w", out.Name, err)
				}
				out.Reserved = &ra
				return out, nil
			}},
		{Name: build.StuckTool,
			Description: "Every build key that waits (building or queued), oldest first: its module, fingerprint, step words and since when, and each ask it " +
				"waits on by language with its id, when it was put, its asking (1 for the first; the builder asks an ask again when it can no longer be " +
				"answered, at most 3 times) and what its build module's steps hold of it (none, or its state on its node since its start). The " +
				"seat workers' delivery counts are not readable by the builder. dead_letters says whether the builder hears the asks the build seats give up " +
				"on (issue 734), and why not: heard, a given-up ask is asked again at once; deaf, at its check bound. With module, that module's keys only. Read-only.",
			Input: map[string]any{"type": "object", "properties": map[string]any{
				"module": map[string]any{"type": "string", "description": "only this module's keys"}}},
			Run: func(args map[string]any) (any, error) { return r.Stuck(str(args, "module")) }},
		{Name: build.AskAgainTool,
			Description: "Ask a waiting build key's asks again now, under new ids the key then names, as the builder does by itself for an ask its build " +
				"seat's stream gave up on. The builder answers in the reasks state, which " + build.StuckTool + " shows beside the key; a key no ask of " +
				"which waits is answered so and nothing is asked. An ask whose step already runs on a node is asked again too: it is then " +
				"built twice, the first run's work wasted and its end read by nobody, never a wrong key; builder_stuck shows whether a step runs.",
			Input: map[string]any{"type": "object", "required": []string{"module", "fp", "why"}, "properties": map[string]any{
				"module": map[string]any{"type": "string", "description": "the module's name"},
				"fp":     map[string]any{"type": "string", "description": "its new-form fingerprint, src2:<hex>, as builder_stuck shows it"},
				"why":    map[string]any{"type": "string", "description": "why it is asked again, in plain words"}}},
			Run: func(args map[string]any) (any, error) {
				key, value, err := build.NewReAsk(str(args, "module"), str(args, "fp"), str(args, "why"), stdio.CallerOf(args), now())
				if err != nil {
					return nil, err
				}
				if err := askAgain(key, value); err != nil {
					return nil, fmt.Errorf("the ask could not be written to %s: %w", build.ReAsksBucket, err)
				}
				return map[string]any{"key": key, "asked": true,
					"next": "the builder answers in " + build.ReAsksState + "." + key + "; " + build.StuckTool + " shows the answer and the new ask"}, nil
			}},
		{Name: build.BuildAgainTool,
			Description: "Build a failed key again (rebuild a module, build a failed key again): the builder writes the key of module and " +
				"fingerprint building as a new revision of the same key and asks the build seats, as it does by itself when it looks at a module " +
				"again (ADR 0429 clause 18). Answers the new revision and the ask id of each build module (go, node, image). Also takes a key left " +
				"building that no look of the builder holds (its ask never put or its end never written, ADR 0668 §5): an ask the key names and " +
				"its stream holds is waited for, one never put is put under its id. Refuses a key done (nothing to do) or queued (" +
				build.AskAgainTool + " asks its waiting asks again), and one the builder is looking at now; a key failed for a reason the builder never retries on the same fingerprint (a pin missing, a version taken, digests that " +
				"differ) and a fingerprint its module no longer has are refused too. Use it for a failed key, or one builder_stuck shows building with no ask or no step.",
			Input: map[string]any{"type": "object", "required": []string{"module", "fp", "why"}, "properties": map[string]any{
				"module": map[string]any{"type": "string", "description": "the module's name"},
				"fp":     map[string]any{"type": "string", "description": "its new-form fingerprint, src2:<hex>"},
				"why":    map[string]any{"type": "string", "description": "why it is built again, in plain words"}}},
			Run: func(args map[string]any) (any, error) {
				key, value, err := build.NewBuildAgain(str(args, "module"), str(args, "fp"), str(args, "why"), stdio.CallerOf(args), now())
				if err != nil {
					return nil, err
				}
				raw, err := buildAgain(key, value)
				if err != nil {
					return nil, err
				}
				r, err := build.DecodeReAsk(raw)
				if err != nil {
					return nil, err
				}
				if r.Refused {
					return nil, errors.New(r.Answer)
				}
				if r.Failed != "" {
					return nil, errors.New(r.Answer)
				}
				if r.Revision == 0 && !r.Pending {
					return nil, fmt.Errorf("the builder answered neither a new revision nor a refusal (an older builder?): %s", r.Answer)
				}
				return map[string]any{"key": key, "revision": r.Revision, "asks": r.Asks, "answer": r.Answer,
					"next": build.StuckTool + " shows the key and its asks while they wait"}, nil
			}},
		{Name: builds.BuildsSinceTool,
			Description: "Every build that ended since a time, oldest first, from the builder's log build-log (kept 30 days): per build its module, " +
				"fingerprint, source revision, repository, directory, commit and ref, the build node, when it ended, its outcome (done, failed or " +
				"queued), the manifest built, the bases it was built against and each artifact made with its digest and reference; a failed " +
				"build carries its error and the step it was at. more says the " +
				"log held more than one answer carries: ask again from the last build's time. With module, that module's builds only. What the operator's command nox builds reads. Read-only.",
			Input: map[string]any{"type": "object", "required": []string{"since"}, "properties": map[string]any{
				"since":  map[string]any{"type": "string", "description": "RFC 3339 time; the builds that ended at or after it"},
				"module": map[string]any{"type": "string", "description": "only this module's builds"}}},
			Run: func(args map[string]any) (any, error) {
				at, err := time.Parse(time.RFC3339Nano, str(args, "since"))
				if err != nil {
					return nil, fmt.Errorf("since %q is not an RFC 3339 time: %w", str(args, "since"), err)
				}
				return build.BuildsSince(since, at, str(args, "module"))
			}},
		{Name: build.AnnounceAgainTool,
			Description: "Announce a done build again (take in again, compose again): the builder writes the done key of module and fingerprint " +
				"again, unchanged, as a new revision of the same key (its content, artifacts and manifest stay identical), so the controller's " +
				"take-in hears it and composes the module and writes its grant again. Use it when a done build was taken in but its composition " +
				"or grant is gone and every want is refused for it. Answers the new revision. Refuses a key that is not done: " + build.BuildAgainTool +
				" builds a failed key again, " + build.AskAgainTool + " asks a waiting key's asks again.",
			Input: map[string]any{"type": "object", "required": []string{"module", "fp", "why"}, "properties": map[string]any{
				"module": map[string]any{"type": "string", "description": "the module's name"},
				"fp":     map[string]any{"type": "string", "description": "its new-form fingerprint, src2:<hex>"},
				"why":    map[string]any{"type": "string", "description": "why it is announced again, in plain words"}}},
			Run: func(args map[string]any) (any, error) {
				key, value, err := build.NewAnnounceAgain(str(args, "module"), str(args, "fp"), str(args, "why"), stdio.CallerOf(args), now())
				if err != nil {
					return nil, err
				}
				raw, err := buildAgain(key, value)
				if err != nil {
					return nil, err
				}
				r, err := build.DecodeReAsk(raw)
				if err != nil {
					return nil, err
				}
				if r.Refused || r.Revision == 0 {
					return nil, errors.New(r.Answer)
				}
				return map[string]any{"key": key, "revision": r.Revision, "answer": r.Answer}, nil
			}},
		{Name: build.SeatQueueTool,
			Description: "What waits on a build seat's stream (SEAT_NODE_BUILD_<LANGUAGE>), as the builder reads it: the stream's message count and " +
				"sequences, and per kind (delivery, check, recheck) its messages oldest first, at most limit each, with sequence, ask id, when asked " +
				"and put, age, the state of the step a build node wrote for it, and whether the builder counts it as still waiting or as given up " +
				"on (its worker delivered it its most times) or taken; and the ask asked last that a node took. The workers' delivery counts are " +
				"not readable by the builder (nats.nats_streams shows them). Read by the builder, which alone may read the stream; read-only.",
			Input: map[string]any{"type": "object", "properties": map[string]any{
				"language": map[string]any{"type": "string", "description": "go, node or image (default go)"},
				"seat":     map[string]any{"type": "string", "description": "or the seat: node-build-go, node-build-node, node-build-image"},
				"limit":    map[string]any{"type": "integer", "description": fmt.Sprintf("messages read of each kind (default %d, at most %d)", build.DefaultQueueLimit, build.MaxQueueLimit)}}},
			Run: func(args map[string]any) (any, error) {
				language := build.LanguageOf(str(args, "language"))
				if language == "" {
					language = build.LanguageOf(str(args, "seat"))
				}
				if language == "" {
					language = buildstep.AskGo
				}
				limit, _ := args["limit"].(float64)
				key, value, err := build.NewQueueAsk(language, int(limit), stdio.CallerOf(args), now())
				if err != nil {
					return nil, err
				}
				return queue(key, value)
			}},
	}
}

// askQueue writes the queues value at key and waits, on a watch of the key, at most bound for the builder's answer.
func askQueue(s stdio.KeptState, bound time.Duration) queueAsker {
	return askKept(s, build.QueuesState, build.QueuesBucket, "the read", bound, func(v []byte) bool {
		a, err := build.DecodeQueueAsk(v)
		return err == nil && !a.Answered.IsZero()
	})
}

// askKept writes a value at key of a state the builder watches and answers there, and waits, on a watch of the key, at most
// bound for a value written after it that answered says is the builder's answer.
func askKept(s stdio.KeptState, state, bucket, what string, bound time.Duration, answered func([]byte) bool) queueAsker {
	return func(key string, value []byte) (json.RawMessage, error) {
		heard := make(chan json.RawMessage, 1)
		var rev uint64
		var mu sync.Mutex
		if err := s.Watch(key, func(c stdio.StateChange) error {
			mu.Lock()
			mine := rev
			mu.Unlock()
			if c.Op != "put" || mine == 0 || c.Revision <= mine {
				return nil
			}
			if answered(c.Value) {
				select {
				case heard <- c.Value:
				default:
				}
			}
			return nil
		}); err != nil {
			return nil, fmt.Errorf("watching %s.%s: %w", state, key, err)
		}
		r, err := s.Put(key, json.RawMessage(value))
		if err != nil {
			return nil, fmt.Errorf("%s could not be written to %s: %w", what, bucket, err)
		}
		mu.Lock()
		rev = r
		mu.Unlock()
		// The watch may have heard the answer before the revision was known: read it once too.
		if e, err := s.Get(key); err == nil && e != nil && e.Revision > r && answered(e.Value) {
			return e.Value, nil
		}
		wait, cancel := context.WithTimeout(context.Background(), bound) // a context's bound, no timer (ADR 0437 §8)
		defer cancel()
		select {
		case v := <-heard:
			return v, nil
		case <-wait.Done():
			return nil, fmt.Errorf("the builder did not answer %s.%s within %s: it is not running, or cannot read %s", state, key, bound, bucket)
		}
	}
}

// stepsOfHolder reads a build seat's steps in the state of the seat's holder (`<holder>.steps`), the holder found where it is
// read by mesh-sdk's seatlist helper (design 47 G A-128, A-126), as the manifest's `seat:<seat>.steps` reads name it; a seat
// whose holder cannot be found reads as an error naming it.
func stepsOfHolder(holder func(seat string) (string, error), read func(name string) build.StateRead) func(seat string) build.StateRead {
	return func(seat string) build.StateRead {
		module, err := holder(seat)
		if err != nil {
			fail := fmt.Errorf("the holder of %s is not found: %w", seat, err)
			return build.StateRead{Keys: func() ([]string, error) { return nil, fail },
				Get:     func(string) (json.RawMessage, time.Time, error) { return nil, time.Time{}, fail },
				Written: func(string) time.Time { return time.Time{} }}
		}
		return read(module + ".steps")
	}
}

func read(s stdio.KeptState) build.StateRead {
	return build.StateRead{Keys: s.Keys, Get: func(key string) (json.RawMessage, time.Time, error) {
		e, err := s.Get(key)
		if err != nil || e == nil {
			return nil, time.Time{}, err
		}
		return e.Value, time.Time{}, nil
	}, Written: func(key string) time.Time {
		e, err := s.Get(key)
		if err != nil || e == nil {
			return time.Time{}
		}
		return written(s, key, e.Revision)
	}}
}

// written is when the key's revision was written, from its history, or zero.
func written(s stdio.KeptState, key string, revision uint64) time.Time {
	h, err := s.History(key)
	if err != nil {
		return time.Time{}
	}
	for _, r := range h {
		if r.Revision == revision {
			return r.At
		}
	}
	return time.Time{}
}

// The seat verbs the bundle calls (the manifest's invokes).
const (
	catalogVersionsVerb = "seat:mesh-catalog.versions"
	reservedVerb        = "seat:mesh-builds.reserved"
)

func main() {
	reasks := stdio.State(build.ReAsksState)
	r := build.StuckReader{Logf: func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-delivery-builder-tools] "+f+"\n", a...) }, Builds: read(stdio.State("builds")), ReAsks: read(reasks), StepsOf: stepsOfHolder(func(seat string) (string, error) { return seatlist.HolderOf(stdio.Ask, seat) }, func(name string) build.StateRead { return read(stdio.State(name)) }),
		Hearing: read(stdio.State(buildrun.HearingState))}
	put := func(key string, value []byte) error {
		_, err := reasks.Put(key, json.RawMessage(value))
		return err
	}
	buildAgain := askKept(reasks, build.ReAsksState, build.ReAsksBucket, "the request", 25*time.Second, func(v []byte) bool {
		a, err := build.DecodeReAsk(v)
		return err == nil && !a.Answered.IsZero()
	})
	vs := stdio.State(versions.State)
	vr := versions.Reader{Keys: vs.Keys, Get: func(key string) (json.RawMessage, bool, error) {
		e, err := vs.Get(key)
		if err != nil || e == nil {
			return nil, false, err
		}
		return e.Value, true, nil
	}}
	fs := stdio.State(versions.FloorWaitsState)
	fr := versions.Reader{Keys: fs.Keys, Get: func(key string) (json.RawMessage, bool, error) {
		e, err := fs.Get(key)
		if err != nil || e == nil {
			return nil, false, err
		}
		return e.Value, true, nil
	}}
	ofs := stdio.State(versions.OldFormState)
	ofr := versions.Reader{Keys: ofs.Keys, Get: func(key string) (json.RawMessage, bool, error) {
		e, err := ofs.Get(key)
		if err != nil || e == nil {
			return nil, false, err
		}
		return e.Value, true, nil
	}}
	bs := stdio.State(versions.BoundsState)
	br := versions.Reader{Keys: bs.Keys, Get: func(key string) (json.RawMessage, bool, error) {
		e, err := bs.Get(key)
		if err != nil || e == nil {
			return nil, false, err
		}
		return e.Value, true, nil
	}}
	rs := stdio.State(versions.ReservedState)
	rr := versions.Reader{Keys: rs.Keys, Get: func(key string) (json.RawMessage, bool, error) {
		e, err := rs.Get(key)
		if err != nil || e == nil {
			return nil, false, err
		}
		return e.Value, true, nil
	}}
	// The catalog half's view of the packages the bundle follows itself (design 47 G §572, A-70), as other bundles follow views.
	pk := &views.Reader{Module: builds.BuilderModule, Logf: func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-delivery-builder-tools] "+f+"\n", a...) },
		Acts: meshhandactsclient.Recorder{Ask: stdio.Ask}}
	branches, err := meshcatalogclient.FollowBranches(pk, stdio.Ask)
	if err != nil {
		fmt.Fprintln(os.Stderr, "mesh-delivery-builder-tools:", err)
		os.Exit(1)
	}
	pk.Start(context.Background())
	// The build modules are no names (design 47 G A-82): the build seats on the seat list, each held by the module the catalog
	// half's manifests say; the seat list and its two tools wired once by mesh-sdk (§204).
	gate := seatlist.NewPart("builder", []string{stdio.AnyPerson, "node-tools", "mesh-healers"})
	go func() {
		<-stdio.Served()
		gate.Follow(seatlist.StdioBus(), nil, func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-delivery-builder-tools] "+f+"\n", a...) })
	}()
	r.Seats = buildrun.NewBuildSeats(func() map[string]seatdef.Definition {
		if rd := gate.Reader(); rd != nil {
			return rd.Definitions()
		}
		return nil
	}, versions.HolderOf(branches))
	reserve := func(name, version string) (meshbuilds.ReservedAnswer, error) {
		raw, err := stdio.Ask(reservedVerb, meshbuilds.ReservedInput{Name: name, Version: version})
		if err != nil {
			return meshbuilds.ReservedAnswer{}, err
		}
		var a meshbuilds.ReservedAnswer
		return a, json.Unmarshal(raw, &a)
	}
	tools := Tools(r, put, askQueue(stdio.State(build.QueuesState), 25*time.Second), buildAgain, stdio.Log(builds.BuilderLog).Since, time.Now, vr, fr, ofr, reserve)
	tools = append(tools, buildTool(&build.BuildReader{Builds: read(stdio.State("builds")), Entry: entryOf(stdio.State("builds")), Asked: read(stdio.State("asked")),
		Newest: newestVia(stdio.Ask), Steps: r}))
	tools = append(tools, reservedTools(rr, buildAgain, time.Now, meshhandactsclient.Recorder{Ask: stdio.Ask})...)
	qs, as := stdio.State(versions.PublishQueueState), stdio.State(versions.ActedState)
	tools = append(tools, republishTools(readerOf(qs), readerOf(as), buildAgain, time.Now, meshhandactsclient.Recorder{Ask: stdio.Ask})...)
	ws := stdio.State(build.WaitsState)
	wr := versions.Reader{Keys: ws.Keys, Get: func(key string) (json.RawMessage, bool, error) {
		e, err := ws.Get(key)
		if err != nil || e == nil {
			return nil, false, err
		}
		return e.Value, true, nil
	}}
	tools = append(tools, waitsTools(wr, buildAgain, time.Now, meshhandactsclient.Recorder{Ask: stdio.Ask})...)
	tools = append(tools, toolchainsTools(stdio.State(buildrun.ToolchainsHeardState), meshhandactsclient.Recorder{Ask: stdio.Ask}, time.Now)...)
	// The builder's expiring keys (state bounds): what each bounds and when it ends.
	tools = append(tools, stdio.Tool{Name: versions.BoundsTool, Description: versions.BoundsDescription,
		Input: map[string]any{"type": "object", "properties": map[string]any{}},
		Run: func(map[string]any) (any, error) {
			rows, err := versions.AnswerBounds(br, time.Now())
			return map[string]any{"bounds": rows}, err
		}})
	// What each counted node's engine reads, as the builder keeps it (ADR 0618 §2, ADR 0624 §1): listed, one node asked
	// again, or one node's key dropped.
	tools = append(tools, enginereads.Tool(EngineReadsTool, "the builder", enginereads.KeptState(stdio.State("engine-reads")), true, time.Now))
	// builder_first_views and builder_first_view_again: where the builder's first views of the indexer's and the assigner's
	// states stand, and asking one again by hand (ADR 0615, DLV-M1).
	tools = append(tools, views.StatusTools(builds.BuilderModule, "builder_first_views", "builder_first_view_again", stdio.State(views.StatusState), meshhandactsclient.Recorder{Ask: stdio.Ask}, time.Now)...)
	tools = append(tools, gate.Tools()...)
	if err := stdio.Serve("", tools); err != nil {
		fmt.Fprintf(os.Stderr, "[mesh-delivery-builder-tools] %v\n", err)
		os.Exit(1)
	}
}

// entryOf reads a key with its revision and when it was written.
func entryOf(s stdio.KeptState) func(key string) (json.RawMessage, uint64, time.Time, error) {
	return func(key string) (json.RawMessage, uint64, time.Time, error) {
		e, err := s.Get(key)
		if err != nil || e == nil {
			return nil, 0, time.Time{}, err
		}
		return e.Value, e.Revision, written(s, key, e.Revision), nil
	}
}