diff --git a/modules/mesh-delivery-sdk/parts/receive.go b/modules/mesh-delivery-sdk/parts/receive.go index 5586f9964..fa81503d9 100644 --- a/modules/mesh-delivery-sdk/parts/receive.go +++ b/modules/mesh-delivery-sdk/parts/receive.go @@ -27,6 +27,10 @@ const ( IdleAfter = 24 * time.Hour // FullPercent is the fill of the objects past which the receiver says they fill. FullPercent = 80 + // TakeOverAfter is how long a transfer's last part must lie behind it before another caller opening it with the + // same digest takes it over (design 47 H2 H-A-481): a sender restarted mid-transfer goes on, a live one is not + // pushed aside. + TakeOverAfter = 2 * time.Minute ) // Want is one part an opening names: its digest (sha256:) and its size. @@ -117,12 +121,17 @@ type Receiver[M any] struct { // Bound reads the cap the objects' binding carries (BusCap): the bus's own bound, which then counts over Cap. // Optional; an error or a zero falls back to Cap. Bound func() (uint64, error) - // MaxChunk, Idle and Full override MaxReceived, IdleAfter and FullPercent when set. + // MaxChunk, Idle, TakeOver and Full override MaxReceived, IdleAfter, TakeOverAfter and FullPercent when set. MaxChunk int Idle time.Duration + TakeOver time.Duration Full uint64 // Holds says the holder already holds a part, which then counts as held at the open and is not sent. Optional. Holds func(t Transfer[M], digest string) bool + // HoldsWhole says the holder holds the file itself the transfer brings (its manifest or reference at the name and + // version its shape names), never from its parts alone (design 47 H2 H-A-481 second addendum): a reopen that lacks + // nothing is then dropped without a second store. Optional: nil never drops, and the reopen finishes as usual. + HoldsWhole func(t Transfer[M]) bool // Check is the caller's check of a whole transfer before anything is stored; its error is the refusal's reason. // Optional. Check func(t Transfer[M]) error @@ -172,6 +181,13 @@ func (r *Receiver[M]) CapNow() uint64 { return r.Cap } +func (r *Receiver[M]) takeOver() time.Duration { + if r.TakeOver > 0 { + return r.TakeOver + } + return TakeOverAfter +} + func (r *Receiver[M]) fullPct() uint64 { if r.Full > 0 { return r.Full @@ -180,8 +196,12 @@ func (r *Receiver[M]) fullPct() uint64 { } // Open opens a transfer for opener, or opens it again: a transfer already open answers what it lacks, so a cut transfer -// goes on from there, and only its opener may open it again. A new transfer that would pass the cap is refused; parts -// the holder holds count as held; when nothing is lacked the transfer ends at once. +// goes on from there. A transfer is finished or taken over, never left open (design 47 H2 H-A-481 and its addendum): +// one opened again with nothing lacked (its finish was cut) is finished, by its opener or by another caller with the +// same digest, once each part taken in it is read back and hashes to its digest (a part that does not is lacked again), +// and dropped without a second store when the holder already holds the file itself (HoldsWhole); one that lacks a part is another +// caller's only once its last part is older than TakeOver, and is refused to it before. A new transfer that would pass the cap is refused; parts the holder holds count as held; when nothing is +// lacked the transfer ends at once, its idle bound armed before the finish, so a cut finish never leaves it open. func (r *Receiver[M]) Open(id, opener string, parts []Want, shape M) (Received, error) { switch { case strings.TrimSpace(id) == "" || strings.ContainsAny(id, "/. *>"): @@ -197,8 +217,33 @@ func (r *Receiver[M]) Open(id, opener string, parts []Want, shape M) (Received, if t, found, err := r.Transfers.Get(id); err != nil { return Received{}, err } else if found { - if t.Opener != opener { - return Received{}, fmt.Errorf("refused for %s: transfer %s was opened by another caller", opener, id) + if len(t.Lacks()) == 0 { + if r.heldWhole(t) { + // The cut came after the store: the holder holds the file, so it is not stored a second time. + if _, err := r.Drop(id); err != nil { + return Received{}, fmt.Errorf("transfer %s is stored already and cannot be dropped: %w", id, err) + } + return Received{ID: id, Ended: true}, nil + } + if !r.verify(&t) { + return r.finishArmed(t) + } + if err := r.Transfers.Put(t); err != nil { + return Received{}, fmt.Errorf("transfer %s cannot be kept: %w", id, err) + } + r.tell(t.FillAtOpen) + } + switch { + case len(t.Lacks()) == 0: + return r.finishArmed(t) + case t.Opener != opener && r.now().Sub(t.Last) < r.takeOver(): + return Received{}, fmt.Errorf("refused for %s: transfer %s was opened by another caller, whose last part came %s ago (taken over after %s)", + opener, id, r.now().Sub(t.Last).Round(time.Second), r.takeOver()) + case t.Opener != opener: + t.Opener = opener + if err := r.Transfers.Put(t); err != nil { + return Received{}, fmt.Errorf("transfer %s cannot be taken over by %s: %w", id, opener, err) + } } return Received{ID: id, Lacks: t.Lacks()}, r.arm(id) } @@ -227,11 +272,53 @@ func (r *Receiver[M]) Open(id, opener string, parts []Want, shape M) (Received, } r.tell(used) if len(t.Lacks()) == 0 { - return Received{ID: id, Ended: true}, r.finish(t) + return r.finishArmed(t) } return Received{ID: id, Lacks: t.Lacks()}, r.arm(id) } +// heldWhole says the holder already holds the file t brings (HoldsWhole), so a finish cut after its store stores nothing +// again; parts held alone never say it (H-A-481 second addendum), since a publish cut before its store must still be +// said as published or refused. +func (r *Receiver[M]) heldWhole(t Transfer[M]) bool { + return r.HoldsWhole != nil && r.HoldsWhole(t) +} + +// verify reads each part taken in t back from the objects and hashes it against its digest, as Take does at a part's +// last chunk (H-A-481 addendum): a finish cut after its chunks were dropped reads them absent, which the runtime's +// object read answers as empty. A part that fails has its chunks dropped and is lacked again. It answers whether any +// part failed, t changed with it. +func (r *Receiver[M]) verify(t *Transfer[M]) bool { + failed := false + for _, p := range t.Parts { + if !t.Taken(p.Digest) || !slices.Contains(t.Held, p.Digest) { + continue + } + whole, err := r.part(*t, p.Digest) + if err == nil && "sha256:"+sum(whole) == p.Digest { + continue + } + r.dropChunks(*t, p.Digest) + delete(t.Chunks, p.Digest) + t.Held = slices.DeleteFunc(t.Held, func(d string) bool { return d == p.Digest }) + failed = true + } + if failed { + r.reanchor(t) + } + return failed +} + +// finishArmed finishes a transfer that lacks nothing at an open, its idle bound armed first (H-A-481): a finish cut +// by a restart leaves a kept transfer its key still drops, and the next open finishes it. A key not written fails the +// open loudly before the finish, the transfer kept for the next open. +func (r *Receiver[M]) finishArmed(t Transfer[M]) (Received, error) { + if err := r.arm(t.ID); err != nil { + return Received{}, err + } + return Received{ID: t.ID, Ended: true}, r.finish(t) +} + // arm bounds an open transfer by its idle key. A key that cannot be written fails the open loudly (the transfer is // kept, so the opener's next open of it answers what is held and arms it again). func (r *Receiver[M]) arm(id string) error { diff --git a/modules/mesh-delivery-sdk/parts/receive_test.go b/modules/mesh-delivery-sdk/parts/receive_test.go index 254dc48b1..7ef3c9de5 100644 --- a/modules/mesh-delivery-sdk/parts/receive_test.go +++ b/modules/mesh-delivery-sdk/parts/receive_test.go @@ -289,6 +289,106 @@ func aHolderTakesAFileInParts(t *testing.T) { } }) + // Design 47 H2 H-A-481: a transfer is finished or taken over, never left open. A finish cut by a restart leaves the + // transfer kept with every part held and its chunks in the objects (cut): the opener's next open, or another caller's + // with the same digest, finishes it; a transfer that lacks a part is another caller's once its last part is older + // than TakeOverAfter, refused to it before; and a transfer that lacks nothing at its open is armed before its finish. + t.Run("a transfer is finished or taken over, never left open", func(t *testing.T) { + var order []string + pkg.Arm = func(id string, _ time.Duration) error { order = append(order, "arm "+id); return nil } + store := pkg.Store + defer func() { pkg.Arm, pkg.Holds, pkg.HoldsWhole, pkg.Store = nil, nil, nil, store }() + // cut leaves c1 as a cut finish does: "whole" all chunks there, "lacked" a part short, "gone" its chunks dropped + // before the cut, "wrong" a chunk that does not hash. + cut := func(id, how string) { + tr := Transfer[packageShape]{ID: id, Opener: "builder", Parts: []Want{part}, Chunks: map[string]int{part.Digest: 1}, + Held: []string{part.Digest}, Opened: now, Last: now, Shape: packageShape{"m", "1"}} + switch how { + case "whole": + objects[pkg.chunkName(id, part.Digest, 0)] = blob + case "lacked": + tr.Held = nil + objects[pkg.chunkName(id, part.Digest, 0)] = blob[:1000] + case "wrong": + objects[pkg.chunkName(id, part.Digest, 0)] = []byte("other") + } + if err := pkg.Transfers.Put(tr); err != nil { + t.Fatal(err) + } + } + for _, c := range []struct { + name, opener, how string + after, takeOver time.Duration + storeHolds string // "parts": Holds answers every part; "file": HoldsWhole answers the file + want string // "stored", "dropped", "lacked again", "lacks" (taken over) or the refusal's words + }{ + {name: "the opener opens a cut finish again", opener: "builder", how: "whole", want: "stored"}, + {name: "another caller opens a cut finish", opener: "other", how: "whole", want: "stored"}, + {name: "a cut after the chunks were dropped", opener: "builder", how: "gone", want: "lacked again"}, + {name: "a cut finish whose chunk does not hash", opener: "builder", how: "wrong", want: "lacked again"}, + {name: "a cut after the store: the holder holds the file", opener: "builder", how: "whole", storeHolds: "file", want: "dropped"}, + {name: "a cut before the store: the holder holds the parts, not the file", opener: "builder", how: "whole", storeHolds: "parts", want: "stored"}, + {name: "another caller while the opener sends", opener: "other", how: "lacked", after: time.Minute, want: "another caller, whose last part came 1m0s ago"}, + {name: "another caller once the last part is two minutes old", opener: "other", how: "lacked", after: TakeOverAfter, want: "lacks"}, + {name: "the receiver's own takeover bound", opener: "other", how: "lacked", after: time.Minute, takeOver: 30 * time.Second, want: "lacks"}, + } { + pkgLog = nil + pkg.TakeOver = c.takeOver + pkg.Holds, pkg.HoldsWhole = nil, nil + switch c.storeHolds { + case "parts": + pkg.Holds = func(Transfer[packageShape], string) bool { return true } + case "file": + pkg.HoldsWhole = func(tr Transfer[packageShape]) bool { return tr.Shape == packageShape{"m", "1"} } + } + cut("c1", c.how) + now = now.Add(c.after) + got, err := pkg.Open("c1", c.opener, []Want{part}, packageShape{"m", "1"}) + kept, open, _ := pkg.Transfers.Get("c1") + switch c.want { + case "stored": + if err != nil || !got.Ended || open || len(pkgLog) != 1 || pkgLog[0].outcome != "stored" || string(pkgLog[0].parts[part.Digest]) != string(blob) { + t.Errorf("%s: open = %+v, %v, still open %v, log %+v; want it finished and stored", c.name, got, err, open, pkgLog) + } + case "dropped": + if err != nil || !got.Ended || open || len(pkgLog) != 0 { + t.Errorf("%s: open = %+v, %v, still open %v, log %+v; want it dropped, not stored again", c.name, got, err, open, pkgLog) + } + case "lacked again": + if err != nil || got.Ended || !open || len(pkgLog) != 0 || !reflect.DeepEqual(got.Lacks, []string{part.Digest}) || kept.Taken(part.Digest) { + t.Errorf("%s: open = %+v, %v, log %+v, kept %+v; want the part lacked again, nothing stored", c.name, got, err, pkgLog, kept) + } + _, _ = pkg.Drop("c1") + case "lacks": + if err != nil || got.Ended || !open || kept.Opener != c.opener || !reflect.DeepEqual(got.Lacks, []string{part.Digest}) { + t.Errorf("%s: open = %+v, %v, opener %q; want it taken over by %s, lacking the part", c.name, got, err, kept.Opener, c.opener) + } + if r, err := pkg.Take("c1", c.opener, part.Digest, 1, true, blob[1000:], nil); err != nil || !r.Ended { + t.Errorf("%s: the new opener's last chunk = %+v, %v", c.name, r, err) + } + default: + if err == nil || !strings.Contains(err.Error(), c.want) || !open || kept.Opener != "builder" { + t.Errorf("%s: open = %v, opener %q; want refused with %q, the transfer kept", c.name, err, kept.Opener, c.want) + } + _, _ = pkg.Drop("c1") + } + if len(objects) != 0 { + t.Errorf("%s: %d objects left", c.name, len(objects)) + } + } + pkg.TakeOver = 0 + order = nil + pkg.Holds = func(Transfer[packageShape], string) bool { return true } + pkg.Store = func(tr Transfer[packageShape], _ map[string][]byte) error { + order = append(order, "store "+tr.ID) + return nil + } + if got, err := pkg.Open("c2", "builder", []Want{part}, packageShape{"m", "1"}); err != nil || !got.Ended || + !reflect.DeepEqual(order, []string{"arm c2", "store c2"}) { + t.Errorf("a transfer lacking nothing at its open = %+v, %v, %v; want its idle bound armed before its finish", got, err, order) + } + }) + // An open transfer is bounded by its idle key (Arm); the key's end (Expired) drops it idle, arms it again for the // rest when it took a chunk since, and leaves a gone one; a key not written fails the open loudly. The cap the // binding carries (Bound) counts over Cap. diff --git a/modules/mesh-delivery-sdk/steprun/askrun.go b/modules/mesh-delivery-sdk/steprun/askrun.go index c37dae07d..c0ab6fc7e 100644 --- a/modules/mesh-delivery-sdk/steprun/askrun.go +++ b/modules/mesh-delivery-sdk/steprun/askrun.go @@ -84,7 +84,14 @@ func RunAsk(ctx context.Context, steps AskSteps, ask sdkstep.StepAsk) (any, erro if ask.Build != nil { job := *ask.Build job.Fetch = withPackages(job.Fetch, ask.Packages) - return steps.build(ctx, job, ask.Version) + out, err := steps.build(ctx, job, ask.Version) + var dec *Declining + if err != nil && !sdkstep.IsPassing(err) && !errors.As(err, &dec) && ctx.Err() == nil { + // A failed build keeps its error text, as the build module bounded it, as the ask's output object, header and + // text as a test step's, so the verb output answers why after the fact (design 47 H2 H-A-482 and its addendum (a)). + steps.keepRunOutput(ask.ID, err.Error(), nil) + } + return out, err } if ask.Step != nil || ask.Install != nil { return steps.runStep(ctx, ask) diff --git a/modules/mesh-delivery-sdk/steprun/askrun_test.go b/modules/mesh-delivery-sdk/steprun/askrun_test.go index 3ebd439ac..8e35229b7 100644 --- a/modules/mesh-delivery-sdk/steprun/askrun_test.go +++ b/modules/mesh-delivery-sdk/steprun/askrun_test.go @@ -117,6 +117,23 @@ imports {"module":"modules/m","workspace":"/w/1"}` if !slices.Equal(answer.Imports, []string{"git.novox.be/novox/mesh-sdk/go/announce", "git.novox.be/novox/mesh-sdk/go/health"}) { t.Errorf("the answer's imports are %v", answer.Imports) } + + // Design 47 H2 H-A-482 addendum (a): a failed build keeps its error text, as the build module bounded it, as the ask's + // output object, which the verb output answers. + outputs := fakeOutputs{m: map[string][]byte{}} + steps.Outputs, steps.Node = outputs, "ace" + steps.Build = func(context.Context, map[string]any) (any, error) { + return nil, errors.New("esbuild for code: status 1 | ✘ [ERROR] Could not resolve \"./missing\" … 812 bytes left out … main.js:1:1)") + } + failed := ask(sdkstep.BuildJob{Module: "m", Path: "modules/m", Fetch: &sdkstep.Fetch{Repository: "novox/x", Commit: runCommit}, + Artifacts: []sdkstep.AskArtifact{{Artifact: "bundle"}}}) + if _, err := RunAsk(context.Background(), steps, failed); err == nil { + t.Fatal("a failed build answered no error") + } + got2, err := readOutput(outputs, "ace", failed.ID, nil) + if err != nil || !strings.Contains(got2.Output, `bundle: esbuild for code: status 1 | ✘ [ERROR] Could not resolve "./missing"`) || got2.Size != len(got2.Output) { + t.Errorf("the failed build's output = %+v, %v; want its error text kept", got2, err) + } } // A build ask for another toolchain is declined (ADR 0584 clause 2), so the node sits out and the asker puts it again. diff --git a/modules/mesh-delivery-sdk/steprun/outputs.go b/modules/mesh-delivery-sdk/steprun/outputs.go index d7a4323cd..ca1188639 100644 --- a/modules/mesh-delivery-sdk/steprun/outputs.go +++ b/modules/mesh-delivery-sdk/steprun/outputs.go @@ -125,7 +125,8 @@ func EndedHere(node string, keys func() ([]string, error), get func(key string) // readOutput answers the verb output for ask (design 47 H2 §791, H-A-412): from the object `.`, the text's last // OutputTail bytes, its size and failing tests from the header; with no object, no longer kept when this node's steps key // shows the ask ended here more than OutputsMaxAge ago, and refused otherwise, naming why (an ask never run here, or one -// whose output this node did not keep). +// that kept no output). An empty object is no object (H-A-482): the runtime's object read answers an absent object as +// empty data without an error, which is not a header fault. func readOutput(o OutputObjects, node, ask string, ended func(ask string) (time.Time, bool, error)) (sdkstep.OutputAnswer, error) { if o == nil { return sdkstep.OutputAnswer{}, fmt.Errorf("this module keeps no %s store", OutputsStore) @@ -135,11 +136,14 @@ func readOutput(o OutputObjects, node, ask string, ended func(ask string) (time. return sdkstep.OutputAnswer{}, err } got, gerr := o.Get(name) - if gerr == nil { + if gerr == nil && len(got.Data) > 0 { return outputOf(ask, got.Data) } + if gerr == nil { + gerr = fmt.Errorf("the object %s in the %s store is empty", name, OutputsStore) + } if ended == nil { - return sdkstep.OutputAnswer{}, fmt.Errorf("the output %s in the %s store: %w", name, OutputsStore, gerr) + return sdkstep.OutputAnswer{}, fmt.Errorf("the ask %s kept no output: %w", ask, gerr) } at, here, err := ended(ask) switch { @@ -150,7 +154,7 @@ func readOutput(o OutputObjects, node, ask string, ended func(ask string) (time. case !at.IsZero() && now().Sub(at) > OutputsMaxAge: return sdkstep.OutputAnswer{Ask: ask, NoLongerKept: true}, nil default: - return sdkstep.OutputAnswer{}, fmt.Errorf("the ask %s ran on %s, and its output is not in the %s store: %w", ask, node, OutputsStore, gerr) + return sdkstep.OutputAnswer{}, fmt.Errorf("the ask %s ran on %s, and the ask kept no output: %w", ask, node, gerr) } } diff --git a/modules/mesh-delivery-sdk/steprun/outputs_test.go b/modules/mesh-delivery-sdk/steprun/outputs_test.go index 7e26a5e03..d83bc6acb 100644 --- a/modules/mesh-delivery-sdk/steprun/outputs_test.go +++ b/modules/mesh-delivery-sdk/steprun/outputs_test.go @@ -64,7 +64,9 @@ func TestOutputsKeptAreRead(t *testing.T) { return raw } steps := map[string]json.RawMessage{"mesh-delivery-pipeline.ace.a2": step(at.Add(-OutputsMaxAge - time.Hour)), "mesh-delivery-pipeline.ace.a3": step(at.Add(-time.Hour)), - "mesh-delivery-pipeline.g14.a4": step(at.Add(-OutputsMaxAge - time.Hour))} + "mesh-delivery-pipeline.g14.a4": step(at.Add(-OutputsMaxAge - time.Hour)), "mesh-delivery-pipeline.ace.a6": step(at.Add(-time.Hour))} + // H-A-482: the runtime's object read answers an absent object as empty data, no error; that is no output, not a header fault. + out.m["ace.a6"] = []byte{} keys := func() ([]string, error) { return slices.Collect(func(y func(string) bool) { for k := range steps { @@ -82,7 +84,8 @@ func TestOutputsKeptAreRead(t *testing.T) { }{ {ask: "a1", want: sdkstep.OutputAnswer{Ask: "a1", Output: long[len(long)-OutputTail:], Size: len(long), Failing: failing}}, {ask: "a2", want: sdkstep.OutputAnswer{Ask: "a2", NoLongerKept: true}}, - {ask: "a3", fail: "ran on ace, and its output is not in the outputs store"}, + {ask: "a3", fail: "ran on ace, and the ask kept no output"}, + {ask: "a6", fail: "ran on ace, and the ask kept no output: the object ace.a6 in the outputs store is empty"}, {ask: "a4", fail: "never ran on ace"}, {ask: "a5", fail: "never ran on ace"}, {fail: "asked for no ask"}, diff --git a/modules/mesh-delivery-sdk/steprun/steprun.go b/modules/mesh-delivery-sdk/steprun/steprun.go index ac8029b0f..19c615b4d 100644 --- a/modules/mesh-delivery-sdk/steprun/steprun.go +++ b/modules/mesh-delivery-sdk/steprun/steprun.go @@ -267,8 +267,9 @@ func (s AskSteps) testStep(ctx context.Context, ask sdkstep.StepAsk, end *sdkste return nil } -// keepRunOutput keeps a test step's level text as the run's output `.` in the outputs objects, with the -// header of the failing tests and their words from the level's results (design 47 H2 §791, H-A-412 (1)). A holder that +// keepRunOutput keeps a test step's level text, or a failed build's error text (H-A-482), as the run's output `.` in the outputs objects, with the header of the failing tests and their words from the level's results (design 47 +// H2 §791, H-A-412 (1)). A holder that // keeps no outputs, or a write that fails, is said in the journal: the step's end stands on the results. func (s AskSteps) keepRunOutput(ask, text string, failed []sdkstep.Failure) { var failing []sdkstep.FailingTest