Files
mesh-delivery/modules/mesh-delivery-indexer/internal/index/run_inputs.go
T
jschoubben 64def921ea indexer: an input is also keyed by its run ask's id and deleted at the build seat's step-ended of that ask
KeepInput takes the ask's id. It has no caller yet: the indexer publishes no run ask for its validation in this tree.

Builds: design 47 G §354, A-78 d
Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h
2026-11-11 23:19:29 +01:00

234 lines
8.2 KiB
Go

package index
import (
"context"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"slices"
"strings"
"time"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildgo"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/keys"
)
// The indexer's inputs (design 47 G §354): the input of the ask `run` the indexer's validation puts, kept in the old state
// section inputs, one key per input named by its write time and its sha256, read by the seat mesh-catalog's verb input (whose
// callers are the holders of the build seats) and by the tool indexer_inputs. A key older than a day is deleted at each new
// ask, judged over at most the 20 oldest keys.
const (
InputsState = "inputs"
InputsBucket = Module + "_" + InputsState
InputsTool = "indexer_inputs"
// InputsKeptFor is how long an input is kept at the longest.
InputsKeptFor = 24 * time.Hour
// InputsSwept is how many of the oldest keys each new ask judges.
InputsSwept = 20
inputsTime = "20060102T150405Z"
// InputsAskPrefix opens the keys that name an input by the id of the run ask it was kept for (A-78 d): ask.<id>, the id escaped as a
// name (§73), holding the input's own key, so the build seat's step-ended of that ask finds it and deletes it.
InputsAskPrefix = "ask."
)
// InputDigest is the sha256 of an input's exact bytes, as "sha256:" and its hex digits.
func InputDigest(body []byte) string {
sum := sha256.Sum256(body)
return "sha256:" + hex.EncodeToString(sum[:])
}
// KeepInput keeps the exact bytes of the input of the run ask ask under their write time and sha256, after sweeping the keys more
// than a day old among the 20 oldest, and names it by the ask's id (A-78 d); it answers the digest. The same bytes written again in
// the same second are one key.
func (ix *Indexer) KeepInput(ctx context.Context, ask string, body []byte) (string, error) {
if ask == "" {
return "", errors.New("name the run ask the input is kept for")
}
if ix.Inputs == nil {
return "", errors.New("the state inputs is not kept here")
}
if err := ix.sweepInputs(ctx); err != nil {
ix.logf("inputs: the keys older than a day are not all deleted: %v", err)
}
digest := InputDigest(body)
key := meshcatalog.InputsByTimeDigestKey(ix.now().UTC().Format(inputsTime), digest)
data, err := json.Marshal(base64.StdEncoding.EncodeToString(body))
if err != nil {
return "", err
}
if err := ix.Inputs.Create(ctx, key, data); err != nil && !errors.Is(err, ErrConflict) {
return "", err
}
if err := ix.Inputs.Create(ctx, InputsAskPrefix+keys.EscapeName(ask), []byte(key)); err != nil && !errors.Is(err, ErrConflict) {
return "", err
}
return digest, nil
}
// HearInputEnded deletes the input kept for a run ask when the build seat's step-ended of that ask is heard (§354, A-78 d); an ask
// that kept none, or whose input a sweep deleted, changes nothing.
func (ix *Indexer) HearInputEnded(ctx context.Context, e nodebuildgo.StepEnded) error {
if ix.Inputs == nil || e.Ask == "" {
return nil
}
askKey := InputsAskPrefix + keys.EscapeName(e.Ask)
raw, rev, found, err := ix.Inputs.Get(ctx, askKey)
if err != nil || !found {
return err
}
if _, irev, ok, err := ix.Inputs.Get(ctx, string(raw)); err != nil {
return err
} else if ok {
if err := ix.Inputs.Delete(ctx, string(raw), irev); err != nil {
return err
}
}
return ix.Inputs.Delete(ctx, askKey, rev)
}
// sweepInputs deletes, among the 20 oldest keys, those written more than a day ago.
func (ix *Indexer) sweepInputs(ctx context.Context) error {
all, err := ix.Inputs.Keys(ctx)
if err != nil {
return err
}
var errs []error
var inputs, asks []string
for _, k := range all {
if strings.HasPrefix(k, InputsAskPrefix) {
asks = append(asks, k)
} else {
inputs = append(inputs, k)
}
}
slices.Sort(inputs) // the write time leads the key, so the oldest come first
if len(inputs) > InputsSwept {
inputs = inputs[:InputsSwept]
}
for _, k := range inputs {
t, _, ok := meshcatalog.ParseInputsByTimeDigestKey(k)
at, perr := time.Parse(inputsTime, t)
if !ok || perr != nil || ix.now().Sub(at) < InputsKeptFor {
continue
}
if _, rev, found, err := ix.Inputs.Get(ctx, k); err == nil && found {
if err := ix.Inputs.Delete(ctx, k, rev); err != nil {
errs = append(errs, err)
}
}
}
// a name by ask whose input a sweep or its step-ended deleted goes with it
for _, k := range asks {
if raw, rev, found, err := ix.Inputs.Get(ctx, k); err == nil && found {
if _, _, ok, err := ix.Inputs.Get(ctx, string(raw)); err == nil && !ok {
if err := ix.Inputs.Delete(ctx, k, rev); err != nil {
errs = append(errs, err)
}
}
}
}
return errors.Join(errs...)
}
// InputKey is the key of the input of the digest, or none.
func InputKey(ctx context.Context, st Store, digest string) (string, error) {
all, err := st.Keys(ctx)
if err != nil {
return "", err
}
for _, k := range all {
if _, d, ok := meshcatalog.ParseInputsByTimeDigestKey(k); ok && d == digest && !strings.HasPrefix(k, InputsAskPrefix) {
return k, nil
}
}
return "", nil
}
// Input answers the verb input: the exact bytes (base64) of the input of the digest, its digest checked.
func Input(ctx context.Context, st Store, in meshcatalog.InputInput) (meshcatalog.InputAnswer, error) {
var out meshcatalog.InputAnswer
if in.Digest == "" {
return out, errors.New("name the input's sha256 (digest)")
}
key, err := InputKey(ctx, st, in.Digest)
if err != nil {
return out, fmt.Errorf("the state %s cannot be read: %w", InputsState, err)
}
raw, _, found, err := st.Get(ctx, key)
if key == "" || err != nil || !found {
return out, fmt.Errorf("no input of digest %s is kept", in.Digest)
}
text, err := meshcatalog.DecodeInputsByTimeDigestValue(raw)
if err != nil {
return out, fmt.Errorf("the input %s cannot be read: %w", key, err)
}
body, err := base64.StdEncoding.DecodeString(text)
if err != nil || InputDigest(body) != in.Digest {
return out, fmt.Errorf("the input kept under %s does not match its digest", key)
}
return meshcatalog.InputAnswer{Digest: in.Digest, Input: text}, nil
}
// InputTool is the verb input as the tool mesh-catalog.input of the indexer's own runtime.
func InputTool(st Store) stdio.Tool {
return stdio.Tool{Name: meshcatalog.Seat + "." + meshcatalog.VerbInput, Run: func(args map[string]any) (any, error) {
var in meshcatalog.InputInput
if raw, err := json.Marshal(args); err == nil {
_ = json.Unmarshal(raw, &in)
}
return Input(context.Background(), st, in)
}}
}
// InputsAnswer is what indexer_inputs answers: each key kept with its time, digest and size, or one input's bytes.
type InputsAnswer struct {
Inputs []InputKept `json:"inputs"`
}
// InputKept is one input kept.
type InputKept struct {
Key string `json:"key"`
Digest string `json:"digest"`
Bytes int `json:"bytes"`
Input string `json:"input,omitempty"`
}
// InputsReadTool is indexer_inputs (class read): every input the indexer keeps, with its key, digest and size; with a digest,
// that input whole (base64).
func InputsReadTool(st Store) stdio.Tool {
return stdio.Tool{Name: InputsTool, Description: "Read the indexer's inputs of the ask run (design 47 G section 354): each key (write time and sha256) with its size; with a digest, that input's exact bytes, base64.",
Input: map[string]any{"type": "object", "properties": map[string]any{"digest": map[string]any{"type": "string", "description": "one input's sha256; none lists every key"}}},
Run: func(args map[string]any) (any, error) {
ctx := context.Background()
want, _ := args["digest"].(string)
all, err := st.Keys(ctx)
if err != nil {
return nil, err
}
slices.Sort(all)
out := InputsAnswer{Inputs: []InputKept{}}
for _, k := range all {
_, d, ok := meshcatalog.ParseInputsByTimeDigestKey(k)
if !ok || strings.HasPrefix(k, InputsAskPrefix) || (want != "" && d != want) {
continue
}
raw, _, found, err := st.Get(ctx, k)
if err != nil || !found {
continue
}
text, _ := meshcatalog.DecodeInputsByTimeDigestValue(raw)
kept := InputKept{Key: k, Digest: d, Bytes: base64.StdEncoding.DecodedLen(len(text))}
if want != "" {
kept.Input = text
}
out.Inputs = append(out.Inputs, kept)
}
return out, nil
}}
}