From 9931d4f900be6a46972c8327144b8ebd4fdf67b4 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 19:12:33 +0100 Subject: [PATCH 1/4] A transfer is finished or taken over, never left open (H-A-481) An opener that opened a transfer again with nothing lacked was answered "nothing lacked" and the transfer never finished; another caller with the same digest was refused; and a transfer lacking nothing at its open was finished without its idle bound, so a restart during the finish left it open. Now a reopen with nothing lacked finishes it, by its opener or by another caller; a transfer that lacks a part is taken over by another caller once its last part is older than two minutes; and the idle bound is armed before every finish at an open. Builds: design 47 H2 H-A-481 --- modules/mesh-delivery-sdk/parts/receive.go | 36 ++++++++-- .../mesh-delivery-sdk/parts/receive_test.go | 72 +++++++++++++++++++ 2 files changed, 103 insertions(+), 5 deletions(-) diff --git a/modules/mesh-delivery-sdk/parts/receive.go b/modules/mesh-delivery-sdk/parts/receive.go index 5586f9964..cb125d0d0 100644 --- a/modules/mesh-delivery-sdk/parts/receive.go +++ b/modules/mesh-delivery-sdk/parts/receive.go @@ -27,6 +27,10 @@ const ( IdleAfter = 24 * time.Hour // FullPercent is the fill of the objects past which the receiver says they fill. FullPercent = 80 + // TakeOverAfter is how long a transfer's last part must lie behind it before another caller opening it with the + // same digest takes it over (design 47 H2 H-A-481): a sender restarted mid-transfer goes on, a live one is not + // pushed aside. + TakeOverAfter = 2 * time.Minute ) // Want is one part an opening names: its digest (sha256:) and its size. @@ -180,8 +184,11 @@ func (r *Receiver[M]) fullPct() uint64 { } // Open opens a transfer for opener, or opens it again: a transfer already open answers what it lacks, so a cut transfer -// goes on from there, and only its opener may open it again. A new transfer that would pass the cap is refused; parts -// the holder holds count as held; when nothing is lacked the transfer ends at once. +// goes on from there. A transfer is finished or taken over, never left open (design 47 H2 H-A-481): one opened again +// with nothing lacked (its finish was cut) is finished, by its opener or by another caller with the same digest; one +// that lacks a part is another caller's only once its last part is older than TakeOverAfter, and is refused to it +// before. A new transfer that would pass the cap is refused; parts the holder holds count as held; when nothing is +// lacked the transfer ends at once, its idle bound armed before the finish, so a cut finish never leaves it open. func (r *Receiver[M]) Open(id, opener string, parts []Want, shape M) (Received, error) { switch { case strings.TrimSpace(id) == "" || strings.ContainsAny(id, "/. *>"): @@ -197,8 +204,17 @@ func (r *Receiver[M]) Open(id, opener string, parts []Want, shape M) (Received, if t, found, err := r.Transfers.Get(id); err != nil { return Received{}, err } else if found { - if t.Opener != opener { - return Received{}, fmt.Errorf("refused for %s: transfer %s was opened by another caller", opener, id) + switch { + case len(t.Lacks()) == 0: + return r.finishArmed(t) + case t.Opener != opener && r.now().Sub(t.Last) < TakeOverAfter: + return Received{}, fmt.Errorf("refused for %s: transfer %s was opened by another caller, whose last part came %s ago (taken over after %s)", + opener, id, r.now().Sub(t.Last).Round(time.Second), TakeOverAfter) + case t.Opener != opener: + t.Opener = opener + if err := r.Transfers.Put(t); err != nil { + return Received{}, fmt.Errorf("transfer %s cannot be taken over by %s: %w", id, opener, err) + } } return Received{ID: id, Lacks: t.Lacks()}, r.arm(id) } @@ -227,11 +243,21 @@ func (r *Receiver[M]) Open(id, opener string, parts []Want, shape M) (Received, } r.tell(used) if len(t.Lacks()) == 0 { - return Received{ID: id, Ended: true}, r.finish(t) + return r.finishArmed(t) } return Received{ID: id, Lacks: t.Lacks()}, r.arm(id) } +// finishArmed finishes a transfer that lacks nothing at an open, its idle bound armed first (H-A-481): a finish cut +// by a restart leaves a kept transfer its key still drops, and the next open finishes it. A key not written fails the +// open loudly before the finish, the transfer kept for the next open. +func (r *Receiver[M]) finishArmed(t Transfer[M]) (Received, error) { + if err := r.arm(t.ID); err != nil { + return Received{}, err + } + return Received{ID: t.ID, Ended: true}, r.finish(t) +} + // arm bounds an open transfer by its idle key. A key that cannot be written fails the open loudly (the transfer is // kept, so the opener's next open of it answers what is held and arms it again). func (r *Receiver[M]) arm(id string) error { diff --git a/modules/mesh-delivery-sdk/parts/receive_test.go b/modules/mesh-delivery-sdk/parts/receive_test.go index 254dc48b1..50944b779 100644 --- a/modules/mesh-delivery-sdk/parts/receive_test.go +++ b/modules/mesh-delivery-sdk/parts/receive_test.go @@ -289,6 +289,78 @@ func aHolderTakesAFileInParts(t *testing.T) { } }) + // Design 47 H2 H-A-481: a transfer is finished or taken over, never left open. A finish cut by a restart leaves the + // transfer kept with every part held and its chunks in the objects (cut): the opener's next open, or another caller's + // with the same digest, finishes it; a transfer that lacks a part is another caller's once its last part is older + // than TakeOverAfter, refused to it before; and a transfer that lacks nothing at its open is armed before its finish. + t.Run("a transfer is finished or taken over, never left open", func(t *testing.T) { + var order []string + pkg.Arm = func(id string, _ time.Duration) error { order = append(order, "arm "+id); return nil } + store := pkg.Store + defer func() { pkg.Arm, pkg.Holds, pkg.Store = nil, nil, store }() + cut := func(id string, lacked bool) { + tr := Transfer[packageShape]{ID: id, Opener: "builder", Parts: []Want{part}, Chunks: map[string]int{part.Digest: 1}, + Held: []string{part.Digest}, Opened: now, Last: now, Shape: packageShape{"m", "1"}} + if lacked { + tr.Held, tr.Chunks = nil, map[string]int{part.Digest: 1} + objects[pkg.chunkName(id, part.Digest, 0)] = blob[:1000] + } else { + objects[pkg.chunkName(id, part.Digest, 0)] = blob + } + if err := pkg.Transfers.Put(tr); err != nil { + t.Fatal(err) + } + } + for _, c := range []struct { + name, opener string + lacked bool + after time.Duration + want string // "stored", "lacks" (taken over or gone on) or the refusal's words + }{ + {"the opener opens a cut finish again", "builder", false, 0, "stored"}, + {"another caller opens a cut finish", "other", false, 0, "stored"}, + {"another caller while the opener sends", "other", true, time.Minute, "another caller, whose last part came 1m0s ago"}, + {"another caller once the last part is two minutes old", "other", true, TakeOverAfter, "lacks"}, + } { + pkgLog = nil + cut("c1", c.lacked) + now = now.Add(c.after) + got, err := pkg.Open("c1", c.opener, []Want{part}, packageShape{"m", "1"}) + kept, open, _ := pkg.Transfers.Get("c1") + switch c.want { + case "stored": + if err != nil || !got.Ended || open || len(pkgLog) != 1 || pkgLog[0].outcome != "stored" || string(pkgLog[0].parts[part.Digest]) != string(blob) { + t.Errorf("%s: open = %+v, %v, still open %v, log %+v; want it finished and stored", c.name, got, err, open, pkgLog) + } + case "lacks": + if err != nil || got.Ended || !open || kept.Opener != c.opener || !reflect.DeepEqual(got.Lacks, []string{part.Digest}) { + t.Errorf("%s: open = %+v, %v, opener %q; want it taken over by %s, lacking the part", c.name, got, err, kept.Opener, c.opener) + } + if r, err := pkg.Take("c1", c.opener, part.Digest, 1, true, blob[1000:], nil); err != nil || !r.Ended { + t.Errorf("%s: the new opener's last chunk = %+v, %v", c.name, r, err) + } + default: + if err == nil || !strings.Contains(err.Error(), c.want) || !open || kept.Opener != "builder" { + t.Errorf("%s: open = %v, opener %q; want refused with %q, the transfer kept", c.name, err, kept.Opener, c.want) + } + _, _ = pkg.Drop("c1") + } + if len(objects) != 0 { + t.Errorf("%s: %d objects left", c.name, len(objects)) + } + } + order = nil + pkg.Holds = func(Transfer[packageShape], string) bool { return true } + pkg.Store = func(tr Transfer[packageShape], _ map[string][]byte) error { + order = append(order, "store "+tr.ID) + return nil + } + if got, err := pkg.Open("c2", "builder", []Want{part}, packageShape{"m", "1"}); err != nil || !got.Ended || + !reflect.DeepEqual(order, []string{"arm c2", "store c2"}) { + t.Errorf("a transfer lacking nothing at its open = %+v, %v, %v; want its idle bound armed before its finish", got, err, order) + } + }) + // An open transfer is bounded by its idle key (Arm); the key's end (Expired) drops it idle, arms it again for the // rest when it took a chunk since, and leaves a gone one; a key not written fails the open loudly. The cap the // binding carries (Bound) counts over Cap. From 2b79ee15933731f266b5ee703c7004ffbf5306ef Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 19:14:02 +0100 Subject: [PATCH 2/4] A missing output is said as missing; a failed build keeps its error (H-A-482) The verb output failed "holds no header" on an empty object, since the runtime's object read answers an absent object as empty data without an error: an empty object is now no object, said "the ask kept no output". A failed build kept nothing for the verb output to answer: it now keeps its error text, as the build module bounded it, as the ask's output object with the header a test step's carries. Builds: design 47 H2 H-A-482, H-A-482 addendum (a) --- modules/mesh-delivery-sdk/steprun/askrun.go | 9 ++++++++- .../mesh-delivery-sdk/steprun/askrun_test.go | 17 +++++++++++++++++ modules/mesh-delivery-sdk/steprun/outputs.go | 12 ++++++++---- .../mesh-delivery-sdk/steprun/outputs_test.go | 7 +++++-- modules/mesh-delivery-sdk/steprun/steprun.go | 5 +++-- 5 files changed, 41 insertions(+), 9 deletions(-) diff --git a/modules/mesh-delivery-sdk/steprun/askrun.go b/modules/mesh-delivery-sdk/steprun/askrun.go index c37dae07d..c0ab6fc7e 100644 --- a/modules/mesh-delivery-sdk/steprun/askrun.go +++ b/modules/mesh-delivery-sdk/steprun/askrun.go @@ -84,7 +84,14 @@ func RunAsk(ctx context.Context, steps AskSteps, ask sdkstep.StepAsk) (any, erro if ask.Build != nil { job := *ask.Build job.Fetch = withPackages(job.Fetch, ask.Packages) - return steps.build(ctx, job, ask.Version) + out, err := steps.build(ctx, job, ask.Version) + var dec *Declining + if err != nil && !sdkstep.IsPassing(err) && !errors.As(err, &dec) && ctx.Err() == nil { + // A failed build keeps its error text, as the build module bounded it, as the ask's output object, header and + // text as a test step's, so the verb output answers why after the fact (design 47 H2 H-A-482 and its addendum (a)). + steps.keepRunOutput(ask.ID, err.Error(), nil) + } + return out, err } if ask.Step != nil || ask.Install != nil { return steps.runStep(ctx, ask) diff --git a/modules/mesh-delivery-sdk/steprun/askrun_test.go b/modules/mesh-delivery-sdk/steprun/askrun_test.go index 3ebd439ac..8e35229b7 100644 --- a/modules/mesh-delivery-sdk/steprun/askrun_test.go +++ b/modules/mesh-delivery-sdk/steprun/askrun_test.go @@ -117,6 +117,23 @@ imports {"module":"modules/m","workspace":"/w/1"}` if !slices.Equal(answer.Imports, []string{"git.novox.be/novox/mesh-sdk/go/announce", "git.novox.be/novox/mesh-sdk/go/health"}) { t.Errorf("the answer's imports are %v", answer.Imports) } + + // Design 47 H2 H-A-482 addendum (a): a failed build keeps its error text, as the build module bounded it, as the ask's + // output object, which the verb output answers. + outputs := fakeOutputs{m: map[string][]byte{}} + steps.Outputs, steps.Node = outputs, "ace" + steps.Build = func(context.Context, map[string]any) (any, error) { + return nil, errors.New("esbuild for code: status 1 | ✘ [ERROR] Could not resolve \"./missing\" … 812 bytes left out … main.js:1:1)") + } + failed := ask(sdkstep.BuildJob{Module: "m", Path: "modules/m", Fetch: &sdkstep.Fetch{Repository: "novox/x", Commit: runCommit}, + Artifacts: []sdkstep.AskArtifact{{Artifact: "bundle"}}}) + if _, err := RunAsk(context.Background(), steps, failed); err == nil { + t.Fatal("a failed build answered no error") + } + got2, err := readOutput(outputs, "ace", failed.ID, nil) + if err != nil || !strings.Contains(got2.Output, `bundle: esbuild for code: status 1 | ✘ [ERROR] Could not resolve "./missing"`) || got2.Size != len(got2.Output) { + t.Errorf("the failed build's output = %+v, %v; want its error text kept", got2, err) + } } // A build ask for another toolchain is declined (ADR 0584 clause 2), so the node sits out and the asker puts it again. diff --git a/modules/mesh-delivery-sdk/steprun/outputs.go b/modules/mesh-delivery-sdk/steprun/outputs.go index d7a4323cd..ca1188639 100644 --- a/modules/mesh-delivery-sdk/steprun/outputs.go +++ b/modules/mesh-delivery-sdk/steprun/outputs.go @@ -125,7 +125,8 @@ func EndedHere(node string, keys func() ([]string, error), get func(key string) // readOutput answers the verb output for ask (design 47 H2 §791, H-A-412): from the object `.`, the text's last // OutputTail bytes, its size and failing tests from the header; with no object, no longer kept when this node's steps key // shows the ask ended here more than OutputsMaxAge ago, and refused otherwise, naming why (an ask never run here, or one -// whose output this node did not keep). +// that kept no output). An empty object is no object (H-A-482): the runtime's object read answers an absent object as +// empty data without an error, which is not a header fault. func readOutput(o OutputObjects, node, ask string, ended func(ask string) (time.Time, bool, error)) (sdkstep.OutputAnswer, error) { if o == nil { return sdkstep.OutputAnswer{}, fmt.Errorf("this module keeps no %s store", OutputsStore) @@ -135,11 +136,14 @@ func readOutput(o OutputObjects, node, ask string, ended func(ask string) (time. return sdkstep.OutputAnswer{}, err } got, gerr := o.Get(name) - if gerr == nil { + if gerr == nil && len(got.Data) > 0 { return outputOf(ask, got.Data) } + if gerr == nil { + gerr = fmt.Errorf("the object %s in the %s store is empty", name, OutputsStore) + } if ended == nil { - return sdkstep.OutputAnswer{}, fmt.Errorf("the output %s in the %s store: %w", name, OutputsStore, gerr) + return sdkstep.OutputAnswer{}, fmt.Errorf("the ask %s kept no output: %w", ask, gerr) } at, here, err := ended(ask) switch { @@ -150,7 +154,7 @@ func readOutput(o OutputObjects, node, ask string, ended func(ask string) (time. case !at.IsZero() && now().Sub(at) > OutputsMaxAge: return sdkstep.OutputAnswer{Ask: ask, NoLongerKept: true}, nil default: - return sdkstep.OutputAnswer{}, fmt.Errorf("the ask %s ran on %s, and its output is not in the %s store: %w", ask, node, OutputsStore, gerr) + return sdkstep.OutputAnswer{}, fmt.Errorf("the ask %s ran on %s, and the ask kept no output: %w", ask, node, gerr) } } diff --git a/modules/mesh-delivery-sdk/steprun/outputs_test.go b/modules/mesh-delivery-sdk/steprun/outputs_test.go index 7e26a5e03..d83bc6acb 100644 --- a/modules/mesh-delivery-sdk/steprun/outputs_test.go +++ b/modules/mesh-delivery-sdk/steprun/outputs_test.go @@ -64,7 +64,9 @@ func TestOutputsKeptAreRead(t *testing.T) { return raw } steps := map[string]json.RawMessage{"mesh-delivery-pipeline.ace.a2": step(at.Add(-OutputsMaxAge - time.Hour)), "mesh-delivery-pipeline.ace.a3": step(at.Add(-time.Hour)), - "mesh-delivery-pipeline.g14.a4": step(at.Add(-OutputsMaxAge - time.Hour))} + "mesh-delivery-pipeline.g14.a4": step(at.Add(-OutputsMaxAge - time.Hour)), "mesh-delivery-pipeline.ace.a6": step(at.Add(-time.Hour))} + // H-A-482: the runtime's object read answers an absent object as empty data, no error; that is no output, not a header fault. + out.m["ace.a6"] = []byte{} keys := func() ([]string, error) { return slices.Collect(func(y func(string) bool) { for k := range steps { @@ -82,7 +84,8 @@ func TestOutputsKeptAreRead(t *testing.T) { }{ {ask: "a1", want: sdkstep.OutputAnswer{Ask: "a1", Output: long[len(long)-OutputTail:], Size: len(long), Failing: failing}}, {ask: "a2", want: sdkstep.OutputAnswer{Ask: "a2", NoLongerKept: true}}, - {ask: "a3", fail: "ran on ace, and its output is not in the outputs store"}, + {ask: "a3", fail: "ran on ace, and the ask kept no output"}, + {ask: "a6", fail: "ran on ace, and the ask kept no output: the object ace.a6 in the outputs store is empty"}, {ask: "a4", fail: "never ran on ace"}, {ask: "a5", fail: "never ran on ace"}, {fail: "asked for no ask"}, diff --git a/modules/mesh-delivery-sdk/steprun/steprun.go b/modules/mesh-delivery-sdk/steprun/steprun.go index ac8029b0f..19c615b4d 100644 --- a/modules/mesh-delivery-sdk/steprun/steprun.go +++ b/modules/mesh-delivery-sdk/steprun/steprun.go @@ -267,8 +267,9 @@ func (s AskSteps) testStep(ctx context.Context, ask sdkstep.StepAsk, end *sdkste return nil } -// keepRunOutput keeps a test step's level text as the run's output `.` in the outputs objects, with the -// header of the failing tests and their words from the level's results (design 47 H2 §791, H-A-412 (1)). A holder that +// keepRunOutput keeps a test step's level text, or a failed build's error text (H-A-482), as the run's output `.` in the outputs objects, with the header of the failing tests and their words from the level's results (design 47 +// H2 §791, H-A-412 (1)). A holder that // keeps no outputs, or a write that fails, is said in the journal: the step's end stands on the results. func (s AskSteps) keepRunOutput(ask, text string, failed []sdkstep.Failure) { var failing []sdkstep.FailingTest From 8eee17a368cf381d6d69dd636736773bfc58ad74 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 19:21:47 +0100 Subject: [PATCH 3/4] A reopen verifies its parts; the takeover bound is a field (H-A-481 addendum) A reopen that lacked nothing finished from the held set without reading the chunks back, so a finish cut after its chunks were dropped, or after its store, stored wrong bytes or stored twice. Now each part taken is read back and hashed against its digest as a take does, a failing part is lacked again, and a transfer whose every part the holder already holds is dropped without a second store. The takeover bound is the receiver's field TakeOver, two minutes by default, as its sibling bounds. Builds: design 47 H2 H-A-481 addendum; review round 1 findings (1), (2) --- modules/mesh-delivery-sdk/parts/receive.go | 77 +++++++++++++++++-- .../mesh-delivery-sdk/parts/receive_test.go | 52 +++++++++---- 2 files changed, 108 insertions(+), 21 deletions(-) diff --git a/modules/mesh-delivery-sdk/parts/receive.go b/modules/mesh-delivery-sdk/parts/receive.go index cb125d0d0..d2e9d8c48 100644 --- a/modules/mesh-delivery-sdk/parts/receive.go +++ b/modules/mesh-delivery-sdk/parts/receive.go @@ -121,9 +121,10 @@ type Receiver[M any] struct { // Bound reads the cap the objects' binding carries (BusCap): the bus's own bound, which then counts over Cap. // Optional; an error or a zero falls back to Cap. Bound func() (uint64, error) - // MaxChunk, Idle and Full override MaxReceived, IdleAfter and FullPercent when set. + // MaxChunk, Idle, TakeOver and Full override MaxReceived, IdleAfter, TakeOverAfter and FullPercent when set. MaxChunk int Idle time.Duration + TakeOver time.Duration Full uint64 // Holds says the holder already holds a part, which then counts as held at the open and is not sent. Optional. Holds func(t Transfer[M], digest string) bool @@ -176,6 +177,13 @@ func (r *Receiver[M]) CapNow() uint64 { return r.Cap } +func (r *Receiver[M]) takeOver() time.Duration { + if r.TakeOver > 0 { + return r.TakeOver + } + return TakeOverAfter +} + func (r *Receiver[M]) fullPct() uint64 { if r.Full > 0 { return r.Full @@ -184,10 +192,11 @@ func (r *Receiver[M]) fullPct() uint64 { } // Open opens a transfer for opener, or opens it again: a transfer already open answers what it lacks, so a cut transfer -// goes on from there. A transfer is finished or taken over, never left open (design 47 H2 H-A-481): one opened again -// with nothing lacked (its finish was cut) is finished, by its opener or by another caller with the same digest; one -// that lacks a part is another caller's only once its last part is older than TakeOverAfter, and is refused to it -// before. A new transfer that would pass the cap is refused; parts the holder holds count as held; when nothing is +// goes on from there. A transfer is finished or taken over, never left open (design 47 H2 H-A-481 and its addendum): +// one opened again with nothing lacked (its finish was cut) is finished, by its opener or by another caller with the +// same digest, once each part taken in it is read back and hashes to its digest (a part that does not is lacked again), +// and dropped without a second store when the holder already holds every part; one that lacks a part is another +// caller's only once its last part is older than TakeOver, and is refused to it before. A new transfer that would pass the cap is refused; parts the holder holds count as held; when nothing is // lacked the transfer ends at once, its idle bound armed before the finish, so a cut finish never leaves it open. func (r *Receiver[M]) Open(id, opener string, parts []Want, shape M) (Received, error) { switch { @@ -204,12 +213,28 @@ func (r *Receiver[M]) Open(id, opener string, parts []Want, shape M) (Received, if t, found, err := r.Transfers.Get(id); err != nil { return Received{}, err } else if found { + if len(t.Lacks()) == 0 { + if r.heldWhole(t) { + // The cut came after the store: the holder holds the file, so it is not stored a second time. + if _, err := r.Drop(id); err != nil { + return Received{}, fmt.Errorf("transfer %s is stored already and cannot be dropped: %w", id, err) + } + return Received{ID: id, Ended: true}, nil + } + if !r.verify(&t) { + return r.finishArmed(t) + } + if err := r.Transfers.Put(t); err != nil { + return Received{}, fmt.Errorf("transfer %s cannot be kept: %w", id, err) + } + r.tell(t.FillAtOpen) + } switch { case len(t.Lacks()) == 0: return r.finishArmed(t) - case t.Opener != opener && r.now().Sub(t.Last) < TakeOverAfter: + case t.Opener != opener && r.now().Sub(t.Last) < r.takeOver(): return Received{}, fmt.Errorf("refused for %s: transfer %s was opened by another caller, whose last part came %s ago (taken over after %s)", - opener, id, r.now().Sub(t.Last).Round(time.Second), TakeOverAfter) + opener, id, r.now().Sub(t.Last).Round(time.Second), r.takeOver()) case t.Opener != opener: t.Opener = opener if err := r.Transfers.Put(t); err != nil { @@ -248,6 +273,44 @@ func (r *Receiver[M]) Open(id, opener string, parts []Want, shape M) (Received, return Received{ID: id, Lacks: t.Lacks()}, r.arm(id) } +// heldWhole says the holder already holds every part of t (Holds), so a finish cut after its store stores nothing again. +func (r *Receiver[M]) heldWhole(t Transfer[M]) bool { + if r.Holds == nil { + return false + } + for _, p := range t.Parts { + if !r.Holds(t, p.Digest) { + return false + } + } + return true +} + +// verify reads each part taken in t back from the objects and hashes it against its digest, as Take does at a part's +// last chunk (H-A-481 addendum): a finish cut after its chunks were dropped reads them absent, which the runtime's +// object read answers as empty. A part that fails has its chunks dropped and is lacked again. It answers whether any +// part failed, t changed with it. +func (r *Receiver[M]) verify(t *Transfer[M]) bool { + failed := false + for _, p := range t.Parts { + if !t.Taken(p.Digest) || !slices.Contains(t.Held, p.Digest) { + continue + } + whole, err := r.part(*t, p.Digest) + if err == nil && "sha256:"+sum(whole) == p.Digest { + continue + } + r.dropChunks(*t, p.Digest) + delete(t.Chunks, p.Digest) + t.Held = slices.DeleteFunc(t.Held, func(d string) bool { return d == p.Digest }) + failed = true + } + if failed { + r.reanchor(t) + } + return failed +} + // finishArmed finishes a transfer that lacks nothing at an open, its idle bound armed first (H-A-481): a finish cut // by a restart leaves a kept transfer its key still drops, and the next open finishes it. A key not written fails the // open loudly before the finish, the transfer kept for the next open. diff --git a/modules/mesh-delivery-sdk/parts/receive_test.go b/modules/mesh-delivery-sdk/parts/receive_test.go index 50944b779..a023c63aa 100644 --- a/modules/mesh-delivery-sdk/parts/receive_test.go +++ b/modules/mesh-delivery-sdk/parts/receive_test.go @@ -298,32 +298,46 @@ func aHolderTakesAFileInParts(t *testing.T) { pkg.Arm = func(id string, _ time.Duration) error { order = append(order, "arm "+id); return nil } store := pkg.Store defer func() { pkg.Arm, pkg.Holds, pkg.Store = nil, nil, store }() - cut := func(id string, lacked bool) { + // cut leaves c1 as a cut finish does: "whole" all chunks there, "lacked" a part short, "gone" its chunks dropped + // before the cut, "wrong" a chunk that does not hash. + cut := func(id, how string) { tr := Transfer[packageShape]{ID: id, Opener: "builder", Parts: []Want{part}, Chunks: map[string]int{part.Digest: 1}, Held: []string{part.Digest}, Opened: now, Last: now, Shape: packageShape{"m", "1"}} - if lacked { - tr.Held, tr.Chunks = nil, map[string]int{part.Digest: 1} - objects[pkg.chunkName(id, part.Digest, 0)] = blob[:1000] - } else { + switch how { + case "whole": objects[pkg.chunkName(id, part.Digest, 0)] = blob + case "lacked": + tr.Held = nil + objects[pkg.chunkName(id, part.Digest, 0)] = blob[:1000] + case "wrong": + objects[pkg.chunkName(id, part.Digest, 0)] = []byte("other") } if err := pkg.Transfers.Put(tr); err != nil { t.Fatal(err) } } for _, c := range []struct { - name, opener string - lacked bool - after time.Duration - want string // "stored", "lacks" (taken over or gone on) or the refusal's words + name, opener, how string + after, takeOver time.Duration + storeHolds bool + want string // "stored", "dropped", "lacked again", "lacks" (taken over) or the refusal's words }{ - {"the opener opens a cut finish again", "builder", false, 0, "stored"}, - {"another caller opens a cut finish", "other", false, 0, "stored"}, - {"another caller while the opener sends", "other", true, time.Minute, "another caller, whose last part came 1m0s ago"}, - {"another caller once the last part is two minutes old", "other", true, TakeOverAfter, "lacks"}, + {name: "the opener opens a cut finish again", opener: "builder", how: "whole", want: "stored"}, + {name: "another caller opens a cut finish", opener: "other", how: "whole", want: "stored"}, + {name: "a cut after the chunks were dropped", opener: "builder", how: "gone", want: "lacked again"}, + {name: "a cut finish whose chunk does not hash", opener: "builder", how: "wrong", want: "lacked again"}, + {name: "a cut after the store", opener: "builder", how: "whole", storeHolds: true, want: "dropped"}, + {name: "another caller while the opener sends", opener: "other", how: "lacked", after: time.Minute, want: "another caller, whose last part came 1m0s ago"}, + {name: "another caller once the last part is two minutes old", opener: "other", how: "lacked", after: TakeOverAfter, want: "lacks"}, + {name: "the receiver's own takeover bound", opener: "other", how: "lacked", after: time.Minute, takeOver: 30 * time.Second, want: "lacks"}, } { pkgLog = nil - cut("c1", c.lacked) + pkg.TakeOver = c.takeOver + pkg.Holds = nil + if c.storeHolds { + pkg.Holds = func(Transfer[packageShape], string) bool { return true } + } + cut("c1", c.how) now = now.Add(c.after) got, err := pkg.Open("c1", c.opener, []Want{part}, packageShape{"m", "1"}) kept, open, _ := pkg.Transfers.Get("c1") @@ -332,6 +346,15 @@ func aHolderTakesAFileInParts(t *testing.T) { if err != nil || !got.Ended || open || len(pkgLog) != 1 || pkgLog[0].outcome != "stored" || string(pkgLog[0].parts[part.Digest]) != string(blob) { t.Errorf("%s: open = %+v, %v, still open %v, log %+v; want it finished and stored", c.name, got, err, open, pkgLog) } + case "dropped": + if err != nil || !got.Ended || open || len(pkgLog) != 0 { + t.Errorf("%s: open = %+v, %v, still open %v, log %+v; want it dropped, not stored again", c.name, got, err, open, pkgLog) + } + case "lacked again": + if err != nil || got.Ended || !open || len(pkgLog) != 0 || !reflect.DeepEqual(got.Lacks, []string{part.Digest}) || kept.Taken(part.Digest) { + t.Errorf("%s: open = %+v, %v, log %+v, kept %+v; want the part lacked again, nothing stored", c.name, got, err, pkgLog, kept) + } + _, _ = pkg.Drop("c1") case "lacks": if err != nil || got.Ended || !open || kept.Opener != c.opener || !reflect.DeepEqual(got.Lacks, []string{part.Digest}) { t.Errorf("%s: open = %+v, %v, opener %q; want it taken over by %s, lacking the part", c.name, got, err, kept.Opener, c.opener) @@ -349,6 +372,7 @@ func aHolderTakesAFileInParts(t *testing.T) { t.Errorf("%s: %d objects left", c.name, len(objects)) } } + pkg.TakeOver = 0 order = nil pkg.Holds = func(Transfer[packageShape], string) bool { return true } pkg.Store = func(tr Transfer[packageShape], _ map[string][]byte) error { From dc6dfef6f0adcab9693921e95e9d1b893bd7d62f Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 22 Nov 2026 19:25:40 +0100 Subject: [PATCH 4/4] A reopen drops without a store only when the holder holds the file (H-A-481 second addendum) The drop without a second store judged from the parts the holder holds, and distribution's Holds answers per layer: a publish cut before its store, whose layers were held, was dropped and never said as published. The drop now asks the holder's own HoldsWhole (the file at its name and version); nil never drops, and the reopen finishes as usual. Builds: design 47 H2 H-A-481 second addendum --- modules/mesh-delivery-sdk/parts/receive.go | 20 +++++++++---------- .../mesh-delivery-sdk/parts/receive_test.go | 14 ++++++++----- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/modules/mesh-delivery-sdk/parts/receive.go b/modules/mesh-delivery-sdk/parts/receive.go index d2e9d8c48..fa81503d9 100644 --- a/modules/mesh-delivery-sdk/parts/receive.go +++ b/modules/mesh-delivery-sdk/parts/receive.go @@ -128,6 +128,10 @@ type Receiver[M any] struct { Full uint64 // Holds says the holder already holds a part, which then counts as held at the open and is not sent. Optional. Holds func(t Transfer[M], digest string) bool + // HoldsWhole says the holder holds the file itself the transfer brings (its manifest or reference at the name and + // version its shape names), never from its parts alone (design 47 H2 H-A-481 second addendum): a reopen that lacks + // nothing is then dropped without a second store. Optional: nil never drops, and the reopen finishes as usual. + HoldsWhole func(t Transfer[M]) bool // Check is the caller's check of a whole transfer before anything is stored; its error is the refusal's reason. // Optional. Check func(t Transfer[M]) error @@ -195,7 +199,7 @@ func (r *Receiver[M]) fullPct() uint64 { // goes on from there. A transfer is finished or taken over, never left open (design 47 H2 H-A-481 and its addendum): // one opened again with nothing lacked (its finish was cut) is finished, by its opener or by another caller with the // same digest, once each part taken in it is read back and hashes to its digest (a part that does not is lacked again), -// and dropped without a second store when the holder already holds every part; one that lacks a part is another +// and dropped without a second store when the holder already holds the file itself (HoldsWhole); one that lacks a part is another // caller's only once its last part is older than TakeOver, and is refused to it before. A new transfer that would pass the cap is refused; parts the holder holds count as held; when nothing is // lacked the transfer ends at once, its idle bound armed before the finish, so a cut finish never leaves it open. func (r *Receiver[M]) Open(id, opener string, parts []Want, shape M) (Received, error) { @@ -273,17 +277,11 @@ func (r *Receiver[M]) Open(id, opener string, parts []Want, shape M) (Received, return Received{ID: id, Lacks: t.Lacks()}, r.arm(id) } -// heldWhole says the holder already holds every part of t (Holds), so a finish cut after its store stores nothing again. +// heldWhole says the holder already holds the file t brings (HoldsWhole), so a finish cut after its store stores nothing +// again; parts held alone never say it (H-A-481 second addendum), since a publish cut before its store must still be +// said as published or refused. func (r *Receiver[M]) heldWhole(t Transfer[M]) bool { - if r.Holds == nil { - return false - } - for _, p := range t.Parts { - if !r.Holds(t, p.Digest) { - return false - } - } - return true + return r.HoldsWhole != nil && r.HoldsWhole(t) } // verify reads each part taken in t back from the objects and hashes it against its digest, as Take does at a part's diff --git a/modules/mesh-delivery-sdk/parts/receive_test.go b/modules/mesh-delivery-sdk/parts/receive_test.go index a023c63aa..7ef3c9de5 100644 --- a/modules/mesh-delivery-sdk/parts/receive_test.go +++ b/modules/mesh-delivery-sdk/parts/receive_test.go @@ -297,7 +297,7 @@ func aHolderTakesAFileInParts(t *testing.T) { var order []string pkg.Arm = func(id string, _ time.Duration) error { order = append(order, "arm "+id); return nil } store := pkg.Store - defer func() { pkg.Arm, pkg.Holds, pkg.Store = nil, nil, store }() + defer func() { pkg.Arm, pkg.Holds, pkg.HoldsWhole, pkg.Store = nil, nil, nil, store }() // cut leaves c1 as a cut finish does: "whole" all chunks there, "lacked" a part short, "gone" its chunks dropped // before the cut, "wrong" a chunk that does not hash. cut := func(id, how string) { @@ -319,23 +319,27 @@ func aHolderTakesAFileInParts(t *testing.T) { for _, c := range []struct { name, opener, how string after, takeOver time.Duration - storeHolds bool + storeHolds string // "parts": Holds answers every part; "file": HoldsWhole answers the file want string // "stored", "dropped", "lacked again", "lacks" (taken over) or the refusal's words }{ {name: "the opener opens a cut finish again", opener: "builder", how: "whole", want: "stored"}, {name: "another caller opens a cut finish", opener: "other", how: "whole", want: "stored"}, {name: "a cut after the chunks were dropped", opener: "builder", how: "gone", want: "lacked again"}, {name: "a cut finish whose chunk does not hash", opener: "builder", how: "wrong", want: "lacked again"}, - {name: "a cut after the store", opener: "builder", how: "whole", storeHolds: true, want: "dropped"}, + {name: "a cut after the store: the holder holds the file", opener: "builder", how: "whole", storeHolds: "file", want: "dropped"}, + {name: "a cut before the store: the holder holds the parts, not the file", opener: "builder", how: "whole", storeHolds: "parts", want: "stored"}, {name: "another caller while the opener sends", opener: "other", how: "lacked", after: time.Minute, want: "another caller, whose last part came 1m0s ago"}, {name: "another caller once the last part is two minutes old", opener: "other", how: "lacked", after: TakeOverAfter, want: "lacks"}, {name: "the receiver's own takeover bound", opener: "other", how: "lacked", after: time.Minute, takeOver: 30 * time.Second, want: "lacks"}, } { pkgLog = nil pkg.TakeOver = c.takeOver - pkg.Holds = nil - if c.storeHolds { + pkg.Holds, pkg.HoldsWhole = nil, nil + switch c.storeHolds { + case "parts": pkg.Holds = func(Transfer[packageShape], string) bool { return true } + case "file": + pkg.HoldsWhole = func(tr Transfer[packageShape]) bool { return tr.Shape == packageShape{"m", "1"} } } cut("c1", c.how) now = now.Add(c.after)