From ac0befa9b92cc265ebf918f462c646176967e8e4 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 16 Nov 2026 15:31:26 +0100 Subject: [PATCH] Indexer: an input leaves nothing in its object store; a full store is said by name and fill MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The indexer's inputs-objects bucket filled twice: deleting an input's key (step-ended, the day sweep) left its object, which only the person's drop tool removed. The input's key and object now go together, each write first drops the objects no key names, and a write the store refuses names the store and its fill. Builds: design 47 G §354, §712, A-237 --- .../internal/index/run_input_chunks_test.go | 44 ++++++++++++++++- .../internal/index/run_inputs.go | 49 +++++++++++++++---- 2 files changed, 82 insertions(+), 11 deletions(-) diff --git a/modules/mesh-delivery-indexer/internal/index/run_input_chunks_test.go b/modules/mesh-delivery-indexer/internal/index/run_input_chunks_test.go index aad96e72..df30fdcf 100644 --- a/modules/mesh-delivery-indexer/internal/index/run_input_chunks_test.go +++ b/modules/mesh-delivery-indexer/internal/index/run_input_chunks_test.go @@ -6,6 +6,7 @@ import ( "crypto/rand" "encoding/json" "errors" + "strings" "sync" "testing" "time" @@ -13,6 +14,7 @@ import ( "github.com/nats-io/nats.go" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo" "git.novox.be/novox/mesh-sdk/go/buscalls" "git.novox.be/novox/mesh-sdk/go/stateobjects" ) @@ -20,12 +22,22 @@ import ( type objectsMem struct { mu sync.Mutex objs map[string][]byte + cap int // the most bytes the bucket holds; zero for no limit (a full bucket refuses: maximum bytes exceeded) } func (*objectsMem) MostBytes() int { return 0 } func (o *objectsMem) Put(name string, data []byte) error { o.mu.Lock() defer o.mu.Unlock() + if o.cap > 0 { + used := len(data) + for _, d := range o.objs { + used += len(d) + } + if used > o.cap { + return errors.New("nats: API error: code=503 err_code=10077 description=maximum bytes exceeded") + } + } o.objs[name] = data return nil } @@ -38,8 +50,21 @@ func (o *objectsMem) Get(name string) ([]byte, error) { } return d, nil } -func (o *objectsMem) Delete(name string) error { return nil } -func (o *objectsMem) List() ([]stateobjects.Object, error) { return nil, nil } +func (o *objectsMem) Delete(name string) error { + o.mu.Lock() + defer o.mu.Unlock() + delete(o.objs, name) + return nil +} +func (o *objectsMem) List() ([]stateobjects.Object, error) { + o.mu.Lock() + defer o.mu.Unlock() + var out []stateobjects.Object + for n, d := range o.objs { + out = append(out, stateobjects.Object{Name: n, Size: int64(len(d))}) + } + return out, nil +} // lineMem is a core bus in memory: a chunk published is kept, and its acknowledgement heard at once. type lineMem struct { @@ -131,4 +156,19 @@ func TestAnInputOverOneMiBIsKeptAsAnObjectAndAnsweredInChunks(t *testing.T) { if _, err := verb("", json.RawMessage(`{"digest":"`+digest+`","call":"x"}`)); err == nil { t.Error("a call of no token is answered") } + + // the bucket holds only inputs still asked: the ended ask's object goes with its key, and one a past delete left goes at the next write + if err := ix.HearInputEnded(ctx, nodebuildgo.StepEnded{Ask: "ask-big"}); err != nil || len(objs.objs) != 0 { + t.Fatalf("an ended ask's input leaves its object: %d left, %v", len(objs.objs), err) + } + objs.objs["inputs.orphan"] = make([]byte, 100) + objs.cap = len(body)*4/3 + 400 // the base64 text, and the orphan below it + if _, err := ix.KeepInput(ctx, "ask-two", body); err != nil || len(objs.objs) != 1 { + t.Fatalf("a write first removes the objects no key names: %d objects, %v", len(objs.objs), err) + } + // a full bucket is said loudly, naming the bucket and its fill + _, err = ix.KeepInput(ctx, "ask-three", append([]byte{1}, body...)) + if err == nil || !strings.Contains(err.Error(), "inputs-objects") || !strings.Contains(err.Error(), "1 objects") { + t.Fatalf("a refused write names the bucket and its fill: %v", err) + } } diff --git a/modules/mesh-delivery-indexer/internal/index/run_inputs.go b/modules/mesh-delivery-indexer/internal/index/run_inputs.go index b0a68c03..8043a8d6 100644 --- a/modules/mesh-delivery-indexer/internal/index/run_inputs.go +++ b/modules/mesh-delivery-indexer/internal/index/run_inputs.go @@ -65,9 +65,18 @@ func (ix *Indexer) KeepInput(ctx context.Context, ask string, body []byte) (stri if err := ix.sweepInputs(ctx); err != nil { ix.logf("inputs: the keys older than a day are not all deleted: %v", err) } + // the object store holds only inputs still asked: an object no key names any longer (one a delete of its key left) goes first + if ix.InputObjects != nil { + if _, err := inputValues(ix.Inputs, ix.InputObjects).DropKeyGone(); err != nil { + ix.logf("inputs: the objects no key names are not all removed: %v", err) + } + } digest := runDigest(body) key := meshcatalog.InputsByTimeDigestKey(ix.now().UTC().Format(inputsTime), digest) if err := inputValues(ix.Inputs, ix.InputObjects).Put(key, base64.StdEncoding.EncodeToString(body)); err != nil && !errors.Is(err, stateobjects.ErrChanged) { + if ix.InputObjects != nil && strings.HasPrefix(err.Error(), "the object for ") { + return "", fmt.Errorf("the object store %s refused the input of %d bytes, holding %s: %w", catalogue.BusName(ix.Module, catalogue.StateObjectsName(InputsState)), len(body), ix.inputsFill(), err) + } return "", err } if err := ix.Inputs.Create(ctx, InputsAskPrefix+keys.EscapeName(ask), []byte(key)); err != nil && !errors.Is(err, ErrConflict) { @@ -76,6 +85,34 @@ func (ix *Indexer) KeepInput(ctx context.Context, ask string, body []byte) (stri return digest, nil } +// inputsFill says what the object store of the inputs holds: its objects and their bytes. +func (ix *Indexer) inputsFill() string { + objs, err := ix.InputObjects.List() + if err != nil { + return fmt.Sprintf("an unreadable fill (%v)", err) + } + var total int64 + for _, o := range objs { + total += o.Size + } + return fmt.Sprintf("%d objects of %d bytes", len(objs), total) +} + +// deleteInput deletes the input's key and the object its value names, if any: an input leaves nothing in the object store. +func (ix *Indexer) deleteInput(ctx context.Context, key string) error { + raw, rev, found, err := ix.Inputs.Get(ctx, key) + if err != nil || !found { + return err + } + if err := ix.Inputs.Delete(ctx, key, rev); err != nil { + return err + } + if ref, isRef := stateobjects.ReadRef(raw); isRef && ix.InputObjects != nil { + return ix.InputObjects.Delete(ref.Object) + } + return nil +} + // HearInputEnded deletes the input kept for a run ask when the build seat's step-ended of that ask is heard (§354, A-78 d); an ask // that kept none, or whose input a sweep deleted, changes nothing. func (ix *Indexer) HearInputEnded(ctx context.Context, e nodebuildgo.StepEnded) error { @@ -87,12 +124,8 @@ func (ix *Indexer) HearInputEnded(ctx context.Context, e nodebuildgo.StepEnded) if err != nil || !found { return err } - if _, irev, ok, err := ix.Inputs.Get(ctx, string(raw)); err != nil { + if err := ix.deleteInput(ctx, string(raw)); err != nil { return err - } else if ok { - if err := ix.Inputs.Delete(ctx, string(raw), irev); err != nil { - return err - } } return ix.Inputs.Delete(ctx, askKey, rev) } @@ -122,10 +155,8 @@ func (ix *Indexer) sweepInputs(ctx context.Context) error { if !ok || perr != nil || ix.now().Sub(at) < InputsKeptFor { continue } - if _, rev, found, err := ix.Inputs.Get(ctx, k); err == nil && found { - if err := ix.Inputs.Delete(ctx, k, rev); err != nil { - errs = append(errs, err) - } + if err := ix.deleteInput(ctx, k); err != nil { + errs = append(errs, err) } } // a name by ask whose input a sweep or its step-ended deleted goes with it