Indexer: an input leaves nothing in its object store; a full store is said by name and fill

The indexer's inputs-objects bucket filled twice: deleting an input's key (step-ended, the day sweep)
left its object, which only the person's drop tool removed. The input's key and object now go together,
each write first drops the objects no key names, and a write the store refuses names the store and its fill.

Builds: design 47 G §354, §712, A-237
This commit is contained in:
2026-11-16 15:31:26 +01:00
parent 4fabf9d26c
commit ac0befa9b9
2 changed files with 82 additions and 11 deletions
@@ -6,6 +6,7 @@ import (
"crypto/rand"
"encoding/json"
"errors"
"strings"
"sync"
"testing"
"time"
@@ -13,6 +14,7 @@ import (
"github.com/nats-io/nats.go"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
"git.novox.be/novox/mesh-sdk/go/buscalls"
"git.novox.be/novox/mesh-sdk/go/stateobjects"
)
@@ -20,12 +22,22 @@ import (
type objectsMem struct {
mu sync.Mutex
objs map[string][]byte
cap int // the most bytes the bucket holds; zero for no limit (a full bucket refuses: maximum bytes exceeded)
}
func (*objectsMem) MostBytes() int { return 0 }
func (o *objectsMem) Put(name string, data []byte) error {
o.mu.Lock()
defer o.mu.Unlock()
if o.cap > 0 {
used := len(data)
for _, d := range o.objs {
used += len(d)
}
if used > o.cap {
return errors.New("nats: API error: code=503 err_code=10077 description=maximum bytes exceeded")
}
}
o.objs[name] = data
return nil
}
@@ -38,8 +50,21 @@ func (o *objectsMem) Get(name string) ([]byte, error) {
}
return d, nil
}
func (o *objectsMem) Delete(name string) error { return nil }
func (o *objectsMem) List() ([]stateobjects.Object, error) { return nil, nil }
func (o *objectsMem) Delete(name string) error {
o.mu.Lock()
defer o.mu.Unlock()
delete(o.objs, name)
return nil
}
func (o *objectsMem) List() ([]stateobjects.Object, error) {
o.mu.Lock()
defer o.mu.Unlock()
var out []stateobjects.Object
for n, d := range o.objs {
out = append(out, stateobjects.Object{Name: n, Size: int64(len(d))})
}
return out, nil
}
// lineMem is a core bus in memory: a chunk published is kept, and its acknowledgement heard at once.
type lineMem struct {
@@ -131,4 +156,19 @@ func TestAnInputOverOneMiBIsKeptAsAnObjectAndAnsweredInChunks(t *testing.T) {
if _, err := verb("", json.RawMessage(`{"digest":"`+digest+`","call":"x"}`)); err == nil {
t.Error("a call of no token is answered")
}
// the bucket holds only inputs still asked: the ended ask's object goes with its key, and one a past delete left goes at the next write
if err := ix.HearInputEnded(ctx, nodebuildgo.StepEnded{Ask: "ask-big"}); err != nil || len(objs.objs) != 0 {
t.Fatalf("an ended ask's input leaves its object: %d left, %v", len(objs.objs), err)
}
objs.objs["inputs.orphan"] = make([]byte, 100)
objs.cap = len(body)*4/3 + 400 // the base64 text, and the orphan below it
if _, err := ix.KeepInput(ctx, "ask-two", body); err != nil || len(objs.objs) != 1 {
t.Fatalf("a write first removes the objects no key names: %d objects, %v", len(objs.objs), err)
}
// a full bucket is said loudly, naming the bucket and its fill
_, err = ix.KeepInput(ctx, "ask-three", append([]byte{1}, body...))
if err == nil || !strings.Contains(err.Error(), "inputs-objects") || !strings.Contains(err.Error(), "1 objects") {
t.Fatalf("a refused write names the bucket and its fill: %v", err)
}
}
@@ -65,9 +65,18 @@ func (ix *Indexer) KeepInput(ctx context.Context, ask string, body []byte) (stri
if err := ix.sweepInputs(ctx); err != nil {
ix.logf("inputs: the keys older than a day are not all deleted: %v", err)
}
// the object store holds only inputs still asked: an object no key names any longer (one a delete of its key left) goes first
if ix.InputObjects != nil {
if _, err := inputValues(ix.Inputs, ix.InputObjects).DropKeyGone(); err != nil {
ix.logf("inputs: the objects no key names are not all removed: %v", err)
}
}
digest := runDigest(body)
key := meshcatalog.InputsByTimeDigestKey(ix.now().UTC().Format(inputsTime), digest)
if err := inputValues(ix.Inputs, ix.InputObjects).Put(key, base64.StdEncoding.EncodeToString(body)); err != nil && !errors.Is(err, stateobjects.ErrChanged) {
if ix.InputObjects != nil && strings.HasPrefix(err.Error(), "the object for ") {
return "", fmt.Errorf("the object store %s refused the input of %d bytes, holding %s: %w", catalogue.BusName(ix.Module, catalogue.StateObjectsName(InputsState)), len(body), ix.inputsFill(), err)
}
return "", err
}
if err := ix.Inputs.Create(ctx, InputsAskPrefix+keys.EscapeName(ask), []byte(key)); err != nil && !errors.Is(err, ErrConflict) {
@@ -76,6 +85,34 @@ func (ix *Indexer) KeepInput(ctx context.Context, ask string, body []byte) (stri
return digest, nil
}
// inputsFill says what the object store of the inputs holds: its objects and their bytes.
func (ix *Indexer) inputsFill() string {
objs, err := ix.InputObjects.List()
if err != nil {
return fmt.Sprintf("an unreadable fill (%v)", err)
}
var total int64
for _, o := range objs {
total += o.Size
}
return fmt.Sprintf("%d objects of %d bytes", len(objs), total)
}
// deleteInput deletes the input's key and the object its value names, if any: an input leaves nothing in the object store.
func (ix *Indexer) deleteInput(ctx context.Context, key string) error {
raw, rev, found, err := ix.Inputs.Get(ctx, key)
if err != nil || !found {
return err
}
if err := ix.Inputs.Delete(ctx, key, rev); err != nil {
return err
}
if ref, isRef := stateobjects.ReadRef(raw); isRef && ix.InputObjects != nil {
return ix.InputObjects.Delete(ref.Object)
}
return nil
}
// HearInputEnded deletes the input kept for a run ask when the build seat's step-ended of that ask is heard (§354, A-78 d); an ask
// that kept none, or whose input a sweep deleted, changes nothing.
func (ix *Indexer) HearInputEnded(ctx context.Context, e nodebuildgo.StepEnded) error {
@@ -87,12 +124,8 @@ func (ix *Indexer) HearInputEnded(ctx context.Context, e nodebuildgo.StepEnded)
if err != nil || !found {
return err
}
if _, irev, ok, err := ix.Inputs.Get(ctx, string(raw)); err != nil {
if err := ix.deleteInput(ctx, string(raw)); err != nil {
return err
} else if ok {
if err := ix.Inputs.Delete(ctx, string(raw), irev); err != nil {
return err
}
}
return ix.Inputs.Delete(ctx, askKey, rev)
}
@@ -122,10 +155,8 @@ func (ix *Indexer) sweepInputs(ctx context.Context) error {
if !ok || perr != nil || ix.now().Sub(at) < InputsKeptFor {
continue
}
if _, rev, found, err := ix.Inputs.Get(ctx, k); err == nil && found {
if err := ix.Inputs.Delete(ctx, k, rev); err != nil {
errs = append(errs, err)
}
if err := ix.deleteInput(ctx, k); err != nil {
errs = append(errs, err)
}
}
// a name by ask whose input a sweep or its step-ended deleted goes with it