assigner: a composition input is kept once per content, by its sha256 (A-465a)

The key of the state inputs is the digest alone (seat mesh-wants' key form <digest>);
each ask's name holds the digest, the input and its object end with the last ask that
names it or a day after its last write (at.<digest>); the contract files of the seat are
edited by hand (generator not found).

Builds: design 47 G A-465a
This commit is contained in:
2026-11-17 11:05:12 +01:00
parent bf5f794553
commit 4cdf14680e
6 changed files with 240 additions and 82 deletions
@@ -28,11 +28,12 @@ import (
)
// A step's input by its digest (design 47 G §354, §712, A-237, A-243): the assigner keeps the input of each composition step
// request it asked a build seat to run in its state `inputs`, one key `<time>.<digest>` per input, the value the input's bytes in
// request it asked a build seat to run in its state `inputs`, one key per input, its sha256 alone (A-465a), the value the input's bytes in
// base64 with the state's object store `inputs-objects` holding one over a key's limit (mesh-sdk stateobjects), and the seat
// mesh-wants' verb `input` answers one by its digest in chunks, to the holders of the build seats alone (the verb's callers, §338).
// An input is deleted when the build seat's step-ended of its ask is heard, or a day after its write, whichever is first, the day
// judged at each new ask over at most 20 of the oldest keys. `assigner_inputs` reads them. This mirrors the indexer's
// An input is kept once per content: every ask that names its digest shares it (A-465a). It is deleted, with its object, when the
// step-ended of the last ask that names it is heard, or a day after its last write, whichever is first, the day judged at each new
// ask over at most 20 of the oldest inputs. `assigner_inputs` reads them. This mirrors the indexer's
// (mesh-delivery-indexer internal/index/run_inputs.go): the two are separate modules' code, owed to mesh-sdk.
// inputsTool is the tool that reads the kept inputs.
@@ -45,10 +46,21 @@ const (
inputsSwept = 20
inputsTime = "20060102T150405Z"
// inputsAskPrefix opens the keys that name an input by the id of the ask it was kept for: ask.<id>, the id escaped as a name,
// holding the input's own key, so the build seat's step-ended of that ask finds it.
// holding the digest of the input the ask uses, so the build seat's step-ended of that ask finds it.
inputsAskPrefix = "ask."
// inputsAtPrefix opens the keys that say when an input was last written: at.<digest>, the time in inputsTime.
inputsAtPrefix = "at."
)
// isDigest is whether a key is an input's own key: 64 hex digits.
func isDigest(key string) bool {
if len(key) != 64 {
return false
}
_, err := hex.DecodeString(key)
return err == nil
}
// digestOf is an input's digest as the ask names it: its sha256 in hex.
func digestOf(b []byte) string { sum := sha256.Sum256(b); return hex.EncodeToString(sum[:]) }
@@ -64,8 +76,9 @@ type inputKeeper struct {
used func() (int64, error)
}
// keep keeps the exact bytes of the input of the ask under their write time and sha256, after sweeping the keys more than a day old
// among the 20 oldest, and names it by the ask's id; it answers the digest. The same bytes written again in the same second are one key.
// keep keeps the exact bytes of the input of the ask under their sha256, after sweeping the inputs more than a day old among the 20
// oldest, and names the digest for the ask's id; it answers the digest. The same bytes asked again are the same one input, its
// last write renewed (A-465a).
func (k *inputKeeper) keep(ask string, body []byte) (string, error) {
if ask == "" {
return "", errors.New("name the ask the input is kept for")
@@ -73,29 +86,47 @@ func (k *inputKeeper) keep(ask string, body []byte) (string, error) {
if err := k.sweep(); err != nil {
k.logf("inputs: the keys older than a day are not all deleted: %v", err)
}
if k.used != nil {
used, err := k.used()
if err != nil {
return "", fmt.Errorf("the fill of the state %s is not read, so no input is kept: %w", meshwants.StateInputs, err)
}
if used+int64(len(body)) > inputsCap {
return "", fmt.Errorf("the state %s is at its cap of %d MiB (%d bytes kept), so the ask's input of %d bytes is refused",
meshwants.StateInputs, inputsCap>>20, used, len(body))
}
}
digest := digestOf(body)
key := meshwants.InputsByTimeDigestKey(k.now().UTC().Format(inputsTime), digest)
if err := k.vals.Put(key, base64.StdEncoding.EncodeToString(body)); err != nil && !errors.Is(err, stateobjects.ErrChanged) {
have, err := k.vals.KV.Get(meshwants.InputsByDigestKey(digest))
if err != nil {
return "", err
}
if err := k.vals.KV.Create(inputsAskPrefix+keys.EscapeName(ask), []byte(key)); err != nil && !errors.Is(err, stateobjects.ErrChanged) {
if have == nil {
if k.used != nil {
used, err := k.used()
if err != nil {
return "", fmt.Errorf("the fill of the state %s is not read, so no input is kept: %w", meshwants.StateInputs, err)
}
if used+int64(len(body)) > inputsCap {
return "", fmt.Errorf("the state %s is at its cap of %d MiB (%d bytes kept), so the ask's input of %d bytes is refused",
meshwants.StateInputs, inputsCap>>20, used, len(body))
}
}
if err := k.vals.Put(meshwants.InputsByDigestKey(digest), base64.StdEncoding.EncodeToString(body)); err != nil && !errors.Is(err, stateobjects.ErrChanged) {
return "", err
}
}
at := []byte(k.now().UTC().Format(inputsTime))
e, err := k.vals.KV.Get(inputsAtPrefix + digest)
if err != nil {
return "", err
}
if e == nil {
err = k.vals.KV.Create(inputsAtPrefix+digest, at)
} else {
err = k.vals.KV.Update(inputsAtPrefix+digest, at, e.Revision)
}
if err != nil && !errors.Is(err, stateobjects.ErrChanged) {
return "", err
}
if err := k.vals.KV.Create(inputsAskPrefix+keys.EscapeName(ask), []byte(digest)); err != nil && !errors.Is(err, stateobjects.ErrChanged) {
return "", err
}
return digest, nil
}
// ended deletes the input kept for an ask when the build seat's step-ended of that ask is heard; an ask that kept none, or whose
// input a sweep deleted, changes nothing.
// ended removes the name of the ask from the input it used when the build seat's step-ended of that ask is heard, and deletes the
// input, with its object, when no ask that names it is left (A-465a); an ask that kept none changes nothing.
func (k *inputKeeper) ended(ask string) error {
if ask == "" {
return nil
@@ -105,10 +136,27 @@ func (k *inputKeeper) ended(ask string) error {
if err != nil || e == nil {
return err
}
if err := k.deleteInput(string(e.Value)); err != nil {
digest := string(e.Value)
if err := k.vals.KV.Delete(askKey); err != nil {
return err
}
return k.vals.KV.Delete(askKey)
all, err := k.vals.KV.Keys()
if err != nil {
return err
}
for _, key := range all {
if !strings.HasPrefix(key, inputsAskPrefix) {
continue
}
o, err := k.vals.KV.Get(key)
if err != nil {
return err
}
if o != nil && string(o.Value) == digest {
return nil // another running ask still names it
}
}
return k.deleteInput(digest)
}
// deleteInput deletes an input's key and the object that key names, when it names one (A-465): an owner that ends its own key
@@ -121,6 +169,9 @@ func (k *inputKeeper) deleteInput(key string) error {
if err := k.vals.KV.Delete(key); err != nil {
return err
}
if err := k.vals.KV.Delete(inputsAtPrefix + key); err != nil {
return err
}
if e == nil {
return nil
}
@@ -163,32 +214,47 @@ func (k *inputKeeper) dropAtStart() {
k.logf("inputs: %d objects of deleted keys removed at the start, %d bytes", len(dropped), bytes)
}
// sweep deletes, among the 20 oldest keys, those written more than a day ago, and a name by ask whose input is gone.
// sweep deletes, among the 20 oldest inputs, those last written more than a day ago, every input of the old form (a key that is not
// a digest), and a name by ask whose input is gone.
func (k *inputKeeper) sweep() error {
all, err := k.vals.KV.Keys()
if err != nil {
return err
}
var errs []error
var inputs, asks []string
type kept struct {
key string
at time.Time
}
var inputs []kept
var asks []string
for _, key := range all {
if strings.HasPrefix(key, inputsAskPrefix) {
switch {
case strings.HasPrefix(key, inputsAskPrefix):
asks = append(asks, key)
} else {
inputs = append(inputs, key)
case strings.HasPrefix(key, inputsAtPrefix):
case !isDigest(key):
// the form `<time>.<digest>` an earlier assigner kept, one copy per ask: ended with its object
if err := k.deleteInput(key); err != nil {
errs = append(errs, err)
}
default:
var at time.Time // an input with no write time is the oldest
if e, err := k.vals.KV.Get(inputsAtPrefix + key); err == nil && e != nil {
at, _ = time.Parse(inputsTime, string(e.Value))
}
inputs = append(inputs, kept{key, at})
}
}
slices.Sort(inputs) // the write time leads the key, so the oldest come first
slices.SortFunc(inputs, func(a, b kept) int { return a.at.Compare(b.at) })
if len(inputs) > inputsSwept {
inputs = inputs[:inputsSwept]
}
for _, key := range inputs {
t, _, ok := meshwants.ParseInputsByTimeDigestKey(key)
at, perr := time.Parse(inputsTime, t)
if !ok || perr != nil || k.now().Sub(at) < inputsKeptFor {
for _, in := range inputs {
if k.now().Sub(in.at) < inputsKeptFor {
continue
}
if err := k.deleteInput(key); err != nil {
if err := k.deleteInput(in.key); err != nil {
errs = append(errs, err)
}
}
@@ -204,19 +270,16 @@ func (k *inputKeeper) sweep() error {
return errors.Join(errs...)
}
// keyOf is the key of the input of the digest, or none.
// keyOf is the key of the input of the digest, or none: the digest itself.
func (k *inputKeeper) keyOf(digest string) (string, error) {
all, err := k.vals.KV.Keys()
if err != nil {
if !isDigest(digest) {
return "", nil
}
e, err := k.vals.KV.Get(meshwants.InputsByDigestKey(digest))
if err != nil || e == nil {
return "", err
}
slices.Sort(all)
for _, key := range all {
if _, d, ok := meshwants.ParseInputsByTimeDigestKey(key); ok && d == digest && !strings.HasPrefix(key, inputsAskPrefix) {
return key, nil
}
}
return "", nil
return digest, nil
}
// body is the exact bytes of the input of the digest, checked against it, whatever its size.
@@ -388,10 +451,14 @@ func inputsRead(k *inputKeeper) stdio.Tool {
slices.Sort(all)
out := []inputSaid{}
for _, key := range all {
t, d, ok := meshwants.ParseInputsByTimeDigestKey(key)
if !ok || strings.HasPrefix(key, inputsAskPrefix) || (want != "" && d != want) {
d, ok := meshwants.ParseInputsByDigestKey(key)
if !ok || !isDigest(key) || (want != "" && d != want) {
continue
}
t := ""
if e, err := k.vals.KV.Get(inputsAtPrefix + key); err == nil && e != nil {
t = string(e.Value)
}
var text string
if found, err := k.vals.Get(key, &text); err != nil || !found {
continue
@@ -3,6 +3,7 @@ package main
import (
"bytes"
"crypto/rand"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
@@ -346,3 +347,102 @@ func TestAssignerInputsSaysWhenTheKeyGoneObjectsCannotBeListed(t *testing.T) {
t.Errorf("answer %+v: an unreadable store is not answered as no key-gone objects", m)
}
}
// Design 47 G A-465a: an input is kept once per content under its sha256 alone; every ask names the digest it uses, and the input
// goes, with its object, at the step-ended of the last ask that names it, or a day after its last write; a request the cap still
// refuses says the fill; an input of the old form (a key that is no digest) is ended with its object.
func TestAnInputIsKeptOncePerContentAndEndsWithItsLastAsk(t *testing.T) {
now := time.Date(2026, 11, 17, 10, 0, 0, 0, time.UTC)
objs := &objectsMem{objs: map[string][]byte{}}
k := testKeeper(objs, func() time.Time { return now })
body := bytes.Repeat([]byte{'n'}, 2<<20)
var digests []string
for _, ask := range []string{"ask-1", "ask-2", "ask-3"} {
d, err := k.keep(ask, body)
if err != nil {
t.Fatal(err)
}
digests = append(digests, d)
}
if digests[0] != digestOf(body) || digests[1] != digests[0] || digests[2] != digests[0] {
t.Fatalf("digests = %v, want the sha256 of the bytes each time", digests)
}
if _, err := k.vals.KV.Get(digests[0]); err != nil {
t.Fatal(err)
}
if e, _ := k.vals.KV.Get(digests[0]); e == nil || len(objs.objs) != 1 {
t.Fatalf("one key by the digest and one object wanted, got entry %v and %d objects", e, len(objs.objs))
}
for _, key := range keysOf(t, k) {
if !strings.HasPrefix(key, inputsAskPrefix) && !strings.HasPrefix(key, inputsAtPrefix) && key != digests[0] {
t.Errorf("a key %q beside the one input", key)
}
}
if e, _ := k.vals.KV.Get(inputsAskPrefix + "ask-2"); e == nil || string(e.Value) != digests[0] {
t.Errorf("the ask does not name the digest it uses: %v", e)
}
for _, ask := range []string{"ask-1", "ask-2"} {
if err := k.ended(ask); err != nil {
t.Fatal(err)
}
if _, err := k.body(digests[0]); err != nil {
t.Fatalf("the input went at the end of %s while ask-3 still names it: %v", ask, err)
}
}
if err := k.ended("ask-3"); err != nil {
t.Fatal(err)
}
if _, err := k.body(digests[0]); err == nil || len(objs.objs) != 0 || len(keysOf(t, k)) != 0 {
t.Errorf("the last step-ended left the input (%v), %d objects, keys %v", err, len(objs.objs), keysOf(t, k))
}
// a day after its last write, a renewed write counted
if _, err := k.keep("ask-4", body); err != nil {
t.Fatal(err)
}
now = now.Add(20 * time.Hour)
if _, err := k.keep("ask-5", body); err != nil {
t.Fatal(err)
}
now = now.Add(20 * time.Hour)
if _, err := k.keep("ask-6", []byte("other")); err != nil {
t.Fatal(err)
}
if _, err := k.body(digests[0]); err != nil {
t.Errorf("an input written again 20 hours ago was swept: %v", err)
}
now = now.Add(25 * time.Hour)
if _, err := k.keep("ask-7", []byte("another")); err != nil {
t.Fatal(err)
}
if _, err := k.body(digests[0]); err == nil || len(objs.objs) != 0 {
t.Errorf("an input a day after its last write is kept (%v), %d objects", err, len(objs.objs))
}
// an input of the old form is ended with its object
old := "20261117T090000Z." + digestOf([]byte("old"))
if err := k.vals.Put(old, base64.StdEncoding.EncodeToString(bytes.Repeat([]byte{'o'}, 2<<20))); err != nil {
t.Fatal(err)
}
if err := k.sweep(); err != nil {
t.Fatal(err)
}
if e, _ := k.vals.KV.Get(old); e != nil || len(objs.objs) != 0 {
t.Errorf("the input of the old form is kept (%v) with %d objects", e, len(objs.objs))
}
// a new content the cap refuses says the fill; the same content, already kept, needs no room
full := newTestKeeperWithFill(objs, now, inputsCap-10)
if _, err := full.keep("ask-8", []byte("fits")); err != nil {
t.Fatalf("a content that fits is refused: %v", err)
}
if _, err := full.keep("ask-9", []byte("does not fit at all")); err == nil || !strings.Contains(err.Error(), "bytes kept") {
t.Errorf("the refusal does not name the fill: %v", err)
}
if _, err := full.keep("ask-10", []byte("fits")); err != nil {
t.Errorf("the content already kept is refused at the cap: %v", err)
}
}
func newTestKeeperWithFill(objs *objectsMem, now time.Time, used int64) *inputKeeper {
k := testKeeper(objs, func() time.Time { return now })
k.used = func() (int64, error) { return used, nil }
return k
}
@@ -741,31 +741,26 @@ func DecodeAssignmentRevisionsByEntryDigestValue(raw []byte) (AssignmentRevision
return v, nil
}
// InputsByTimeDigestKey is the key of one value of the state inputs under the form "<time>.<digest>".
func InputsByTimeDigestKey(time, digest string) string { return time + "." + digest }
// InputsByDigestKey is the key of one value of the state inputs under the form "<digest>".
func InputsByDigestKey(digest string) string { return digest }
// ParseInputsByTimeDigestKey reads the parts of a key of the form "<time>.<digest>", in the order of the form; the last result is false for a key of another form.
func ParseInputsByTimeDigestKey(key string) (string, string, bool) {
// ParseInputsByDigestKey reads the parts of a key of the form "<digest>", in the order of the form; the last result is false for a key of another form.
func ParseInputsByDigestKey(key string) (string, bool) {
rest := key
v0, after0, found0 := strings.Cut(rest, ".")
if !found0 || v0 == "" {
return "", "", false
}
rest = after0
if rest == "" {
return "", "", false
return "", false
}
v2 := rest
v0 := rest
rest = ""
if rest != "" {
return "", "", false
return "", false
}
return v0, v2, true
return v0, true
}
// DecodeInputsByTimeDigestValue reads one value of the state inputs under the form "<time>.<digest>", refusing a field InputsByTimeDigestValue does not know and any text after the value.
func DecodeInputsByTimeDigestValue(raw []byte) (InputsByTimeDigestValue, error) {
var v InputsByTimeDigestValue
// DecodeInputsByDigestValue reads one value of the state inputs under the form "<digest>", refusing a field InputsByDigestValue does not know and any text after the value.
func DecodeInputsByDigestValue(raw []byte) (InputsByDigestValue, error) {
var v InputsByDigestValue
dec := json.NewDecoder(bytes.NewReader(raw))
dec.DisallowUnknownFields()
if err := dec.Decode(&v); err != nil {
@@ -1356,5 +1351,5 @@ type AssignmentRevisionsByCandidateGroupHeadsValue = AssignmentRevision
// AssignmentRevisionsByEntryDigestValue is one value of the state assignment-revisions under the key entry.<digest>.
type AssignmentRevisionsByEntryDigestValue = AssignmentEntry
// InputsByTimeDigestValue is one value of the state inputs under the key <time>.<digest>.
type InputsByTimeDigestValue = string
// InputsByDigestValue is one value of the state inputs under the key <digest>.
type InputsByDigestValue = string
@@ -813,10 +813,10 @@
"name": "inputs",
"template": "state",
"history": 1,
"description": "a step's input, named by its write time and its sha256; a value over one chunk is held in the state's objects store (design 47 G sections 354, 712)",
"description": "a step's input, named by its sha256 alone, kept once per content; a value over one chunk is held in the state's objects store (design 47 G sections 354, 712)",
"key-forms": [
{
"key": "<time>.<digest>",
"key": "<digest>",
"value": {
"type": "string",
"description": "the input's bytes, base64"
@@ -171,11 +171,11 @@ export declare function assignmentRevisionsByEntryDigestKey(digest: string): str
/** The parts of a key of the form "entry.<digest>", or undefined for a key of another form. */
export declare function parseAssignmentRevisionsByEntryDigestKey(_key: string): { digest: string } | undefined;
/** The key of one value of the state inputs under the form "<time>.<digest>". */
export declare function inputsByTimeDigestKey(time: string, digest: string): string;
/** The key of one value of the state inputs under the form "<digest>". */
export declare function inputsByDigestKey(digest: string): string;
/** The parts of a key of the form "<time>.<digest>", or undefined for a key of another form. */
export declare function parseInputsByTimeDigestKey(_key: string): { time: string; digest: string } | undefined;
/** The parts of a key of the form "<digest>", or undefined for a key of another form. */
export declare function parseInputsByDigestKey(_key: string): { digest: string } | undefined;
/** an ask the assigner puts to a person: the router's ask whole, as the assigner composes it, and how many times it was written at its key */
export interface Ask {
@@ -692,6 +692,6 @@ export type AssignmentRevisionsByCandidateGroupHeadsValue = AssignmentRevision;
/** is one value of the state assignment-revisions under the key entry.<digest>. */
export type AssignmentRevisionsByEntryDigestValue = AssignmentEntry;
/** is one value of the state inputs under the key <time>.<digest>. */
export type InputsByTimeDigestValue = string;
/** is one value of the state inputs under the key <digest>. */
export type InputsByDigestValue = string;
@@ -403,22 +403,18 @@ export function parseAssignmentRevisionsByEntryDigestKey(_key) {
return { digest };
}
/** The key of one value of the state inputs under the form "<time>.<digest>". */
export function inputsByTimeDigestKey(time, digest) {
return time + "." + digest;
/** The key of one value of the state inputs under the form "<digest>". */
export function inputsByDigestKey(digest) {
return digest;
}
/** The parts of a key of the form "<time>.<digest>", or undefined for a key of another form. */
export function parseInputsByTimeDigestKey(_key) {
/** The parts of a key of the form "<digest>", or undefined for a key of another form. */
export function parseInputsByDigestKey(_key) {
let _rest = _key;
const _at0 = _rest.indexOf(".");
if (_at0 <= 0) return undefined;
const time = _rest.slice(0, _at0);
_rest = _rest.slice(_at0 + 1);
if (_rest === "") return undefined;
const digest = _rest;
_rest = "";
if (_rest !== "") return undefined;
return { time, digest };
return { digest };
}