mesh-delivery/modules/mesh-delivery-indexer/internal/index/run_inputs.go

package index

import (
	"context"
	"crypto/sha256"
	"encoding/base64"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"regexp"
	"slices"
	"strings"
	"time"

	"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
	"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
	stdio "git.novox.be/novox/mesh-sdk/go"
	"git.novox.be/novox/mesh-sdk/go/buscalls"
	"git.novox.be/novox/mesh-sdk/go/catalogue"
	"git.novox.be/novox/mesh-sdk/go/keys"
	"git.novox.be/novox/mesh-sdk/go/stateobjects"
)

// The indexer's inputs (design 47 G §354): the input of the ask `run` the indexer's validation puts, kept in the old state
// section inputs, one key per input named by its write time and its sha256, read by the seat mesh-catalog's verb input (whose
// callers are the holders of the build seats) and by the tool indexer_inputs. A key older than a day is deleted at each new
// ask, judged over at most the 20 oldest keys.
const (
	InputsState = "inputs"
	InputsTool  = "indexer_inputs"
	// InputsKeptFor is how long an input is kept at the longest.
	InputsKeptFor = 24 * time.Hour
	// InputsSwept is how many of the oldest keys each new ask judges.
	InputsSwept = 20
	inputsTime  = "20060102T150405Z"
	// InputsAskPrefix opens the keys that name an input by the id of the run ask it was kept for (A-78 d): ask.<id>, the id escaped as a
	// name (§73), holding the input's own key, so the build seat's step-ended of that ask finds it and deletes it.
	InputsAskPrefix = "ask."
)

// InputDigest is the sha256 of an input's exact bytes, as "sha256:" and its hex digits.
func InputDigest(body []byte) string {
	sum := sha256.Sum256(body)
	return "sha256:" + hex.EncodeToString(sum[:])
}

// runDigest is the digest a run ask names its input by (buildstep.RunInput.Digest): the 64 hex digits of its sha256, which are also
// the digest of the key's form, a name a state's key may hold (a ':' may not be).
func runDigest(body []byte) string {
	sum := sha256.Sum256(body)
	return hex.EncodeToString(sum[:])
}

// KeepInput keeps the exact bytes of the input of the run ask ask under their write time and sha256, after sweeping the keys more
// than a day old among the 20 oldest, and names it by the ask's id (A-78 d); it answers the digest. The same bytes written again in
// the same second are one key.
func (ix *Indexer) KeepInput(ctx context.Context, ask string, body []byte) (string, error) {
	if ask == "" {
		return "", errors.New("name the run ask the input is kept for")
	}
	if ix.Inputs == nil {
		return "", errors.New("the state inputs is not kept here")
	}
	if err := ix.sweepInputs(ctx); err != nil {
		ix.logf("inputs: the keys older than a day are not all deleted: %v", err)
	}
	digest := runDigest(body)
	key := meshcatalog.InputsByTimeDigestKey(ix.now().UTC().Format(inputsTime), digest)
	if err := inputValues(ix.Inputs, ix.InputObjects).Put(key, base64.StdEncoding.EncodeToString(body)); err != nil && !errors.Is(err, stateobjects.ErrChanged) {
		return "", err
	}
	if err := ix.Inputs.Create(ctx, InputsAskPrefix+keys.EscapeName(ask), []byte(key)); err != nil && !errors.Is(err, ErrConflict) {
		return "", err
	}
	return digest, nil
}

// HearInputEnded deletes the input kept for a run ask when the build seat's step-ended of that ask is heard (§354, A-78 d); an ask
// that kept none, or whose input a sweep deleted, changes nothing.
func (ix *Indexer) HearInputEnded(ctx context.Context, e nodebuildgo.StepEnded) error {
	if ix.Inputs == nil || e.Ask == "" {
		return nil
	}
	askKey := InputsAskPrefix + keys.EscapeName(e.Ask)
	raw, rev, found, err := ix.Inputs.Get(ctx, askKey)
	if err != nil || !found {
		return err
	}
	if _, irev, ok, err := ix.Inputs.Get(ctx, string(raw)); err != nil {
		return err
	} else if ok {
		if err := ix.Inputs.Delete(ctx, string(raw), irev); err != nil {
			return err
		}
	}
	return ix.Inputs.Delete(ctx, askKey, rev)
}

// sweepInputs deletes, among the 20 oldest keys, those written more than a day ago.
func (ix *Indexer) sweepInputs(ctx context.Context) error {
	all, err := ix.Inputs.Keys(ctx)
	if err != nil {
		return err
	}
	var errs []error
	var inputs, asks []string
	for _, k := range all {
		if strings.HasPrefix(k, InputsAskPrefix) {
			asks = append(asks, k)
		} else {
			inputs = append(inputs, k)
		}
	}
	slices.Sort(inputs) // the write time leads the key, so the oldest come first
	if len(inputs) > InputsSwept {
		inputs = inputs[:InputsSwept]
	}
	for _, k := range inputs {
		t, _, ok := meshcatalog.ParseInputsByTimeDigestKey(k)
		at, perr := time.Parse(inputsTime, t)
		if !ok || perr != nil || ix.now().Sub(at) < InputsKeptFor {
			continue
		}
		if _, rev, found, err := ix.Inputs.Get(ctx, k); err == nil && found {
			if err := ix.Inputs.Delete(ctx, k, rev); err != nil {
				errs = append(errs, err)
			}
		}
	}
	// a name by ask whose input a sweep or its step-ended deleted goes with it
	for _, k := range asks {
		if raw, rev, found, err := ix.Inputs.Get(ctx, k); err == nil && found {
			if _, _, ok, err := ix.Inputs.Get(ctx, string(raw)); err == nil && !ok {
				if err := ix.Inputs.Delete(ctx, k, rev); err != nil {
					errs = append(errs, err)
				}
			}
		}
	}
	return errors.Join(errs...)
}

// InputKey is the key of the input of the digest, or none.
func InputKey(ctx context.Context, st Store, digest string) (string, error) {
	all, err := st.Keys(ctx)
	if err != nil {
		return "", err
	}
	for _, k := range all {
		if _, d, ok := meshcatalog.ParseInputsByTimeDigestKey(k); ok && d == digest && !strings.HasPrefix(k, InputsAskPrefix) {
			return k, nil
		}
	}
	return "", nil
}

// inputValues is the state inputs with its object store beside it (mesh-sdk stateobjects): a value that fits a key is in it, a
// larger one is an object with its reference in the key (ADR 0666 §16).
func inputValues(st Store, objs stateobjects.Objects) stateobjects.Values {
	return stateobjects.Values{KV: storeKV{st}, Objects: objs, Store: catalogue.StateObjectsName(InputsState)}
}

// storeKV is a Store as the state's key-value side.
type storeKV struct{ s Store }

func (k storeKV) Get(key string) (*stateobjects.Entry, error) {
	v, rev, found, err := k.s.Get(context.Background(), key)
	if err != nil || !found {
		return nil, err
	}
	return &stateobjects.Entry{Value: v, Revision: rev}, nil
}

func changed(err error) error {
	if errors.Is(err, ErrConflict) {
		return stateobjects.ErrChanged
	}
	return err
}

func (k storeKV) Create(key string, v []byte) error {
	return changed(k.s.Create(context.Background(), key, v))
}

func (k storeKV) Update(key string, v []byte, rev uint64) error {
	return changed(k.s.Update(context.Background(), key, v, rev))
}

func (k storeKV) Delete(key string) error {
	_, rev, found, err := k.s.Get(context.Background(), key)
	if err != nil || !found {
		return err
	}
	return k.s.Delete(context.Background(), key, rev)
}

func (k storeKV) Keys() ([]string, error) { return k.s.Keys(context.Background()) }

// InputBody is the exact bytes of the input of the digest, checked against it, whatever its size.
func InputBody(ctx context.Context, st Store, objs stateobjects.Objects, digest string) ([]byte, error) {
	if digest == "" {
		return nil, errors.New("name the input's sha256 (digest)")
	}
	key, err := InputKey(ctx, st, digest)
	if err != nil {
		return nil, fmt.Errorf("the state %s cannot be read: %w", InputsState, err)
	}
	if key == "" {
		return nil, fmt.Errorf("no input of digest %s is kept", digest)
	}
	var text string
	found, err := inputValues(st, objs).Get(key, &text)
	if err != nil {
		return nil, fmt.Errorf("the input %s cannot be read: %w", key, err)
	}
	if !found {
		return nil, fmt.Errorf("no input of digest %s is kept", digest)
	}
	body, err := base64.StdEncoding.DecodeString(text)
	if err != nil || runDigest(body) != digest {
		return nil, fmt.Errorf("the input kept under %s does not match its digest", key)
	}
	return body, nil
}

var callToken = regexp.MustCompile(`^[A-Za-z0-9_-]{16,64}$`)

// InputVerb answers the verb input (marked chunks): the digest and how many chunks carry the bytes, then sends them in chunks of at
// most buscalls.SeatChunkBytes on the call's chunk subjects, each acknowledged before the next (A-237, A-139). The input is of any
// size. A caller that does not acknowledge within buscalls.SeatAckWithin ends the send, said in logf.
func InputVerb(st Store, objs stateobjects.Objects, line buscalls.ChunkLine, within time.Duration, logf func(string, ...any)) func(caller string, body json.RawMessage) (any, error) {
	return func(_ string, raw json.RawMessage) (any, error) {
		var in meshcatalog.InputInput
		if err := json.Unmarshal(raw, &in); err != nil {
			return nil, fmt.Errorf("the arguments are not JSON: %w", err)
		}
		if !callToken.MatchString(in.Call) {
			return nil, errors.New("call is one token of 16 to 64 letters, digits, - or _: 128 random bits the caller chooses")
		}
		body, err := InputBody(context.Background(), st, objs, in.Digest)
		if err != nil {
			return nil, err
		}
		send := buscalls.SeatChunks{Seat: meshcatalog.Seat, Verb: meshcatalog.VerbInput, Call: in.Call}
		total := int64((len(body) + buscalls.SeatChunkBytes - 1) / buscalls.SeatChunkBytes)
		// the send subscribes to the acknowledgements itself and ends when it has sent every chunk or one is not acknowledged
		go func() {
			if err := send.Send(line, body, within, logf); err != nil {
				logf("the send %s of the input %s: %v", in.Call, in.Digest, err)
			}
		}()
		return meshcatalog.InputAnswer{Digest: in.Digest, Chunks: total}, nil
	}
}

// InputsAnswer is what indexer_inputs answers: each key kept with its time, digest and size, or one input's bytes.
type InputsAnswer struct {
	Inputs []InputKept `json:"inputs"`
}

// InputKept is one input kept.
type InputKept struct {
	Key    string `json:"key"`
	Digest string `json:"digest"`
	Bytes  int    `json:"bytes"`
	Input  string `json:"input,omitempty"`
}

// InputsReadTool is indexer_inputs (class read): every input the indexer keeps, with its key, digest and size; with a digest,
// that input whole (base64).
func InputsReadTool(st Store, objs stateobjects.Objects) stdio.Tool {
	return stdio.Tool{Name: InputsTool, Description: "Read the indexer's inputs of the ask run (design 47 G section 354): each key (write time and sha256) with its size; with a digest, that input's exact bytes, base64.",
		Input: map[string]any{"type": "object", "properties": map[string]any{"digest": map[string]any{"type": "string", "description": "one input's sha256; none lists every key"}}},
		Run: func(args map[string]any) (any, error) {
			ctx := context.Background()
			want, _ := args["digest"].(string)
			all, err := st.Keys(ctx)
			if err != nil {
				return nil, err
			}
			slices.Sort(all)
			out := InputsAnswer{Inputs: []InputKept{}}
			for _, k := range all {
				_, d, ok := meshcatalog.ParseInputsByTimeDigestKey(k)
				if !ok || strings.HasPrefix(k, InputsAskPrefix) || (want != "" && d != want) {
					continue
				}
				raw, _, found, err := st.Get(ctx, k)
				if err != nil || !found {
					continue
				}
				var kept InputKept
				if ref, isRef := stateobjects.ReadRef(raw); isRef {
					kept = InputKept{Key: k, Digest: d, Bytes: base64.StdEncoding.DecodedLen(int(ref.Size) - 2)} // the quotes of the JSON string
				} else {
					text, _ := meshcatalog.DecodeInputsByTimeDigestValue(raw)
					kept = InputKept{Key: k, Digest: d, Bytes: base64.StdEncoding.DecodedLen(len(text))}
				}
				if want != "" {
					var text string
					if ok, err := inputValues(st, objs).Get(k, &text); err != nil || !ok {
						continue
					}
					kept.Input = text
				}
				out.Inputs = append(out.Inputs, kept)
			}
			return out, nil
		}}
}