From bfc877209f26f6ea4952418c15cecb70169bee31 Mon Sep 17 00:00:00 2001 From: jochen Date: Sun, 15 Nov 2026 01:30:05 +0100 Subject: [PATCH] Distribution: each bus step of a publish bounded at 10 s and logged by name Builds: design 47 G (publish path of oci-artifact-registry; no design line covers the bounds or log lines) --- .../cmd/distribution-tools/bounded.go | 102 ++++++++++++++++++ .../cmd/distribution-tools/bounded_test.go | 20 ++++ .../cmd/distribution-tools/main.go | 4 +- 3 files changed, 124 insertions(+), 2 deletions(-) create mode 100644 modules/distribution/cmd/distribution-tools/bounded.go create mode 100644 modules/distribution/cmd/distribution-tools/bounded_test.go diff --git a/modules/distribution/cmd/distribution-tools/bounded.go b/modules/distribution/cmd/distribution-tools/bounded.go new file mode 100644 index 000000000..493424b32 --- /dev/null +++ b/modules/distribution/cmd/distribution-tools/bounded.go @@ -0,0 +1,102 @@ +package main + +import ( + "fmt" + "time" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/parts" + + "distribution/internal/holder" +) + +// stepBound bounds each bus operation of a publish well under the tool runner's 30 s call bound, so a hang is an error +// naming its step and the publish answers with it. +const stepBound = 10 * time.Second + +// step runs f, logs its start and its end with its time, and gives up on it after bound: the error names the step. An +// operation given up on keeps running in its goroutine until the bus answers it. +func step[T any](bound time.Duration, what string, f func() (T, error)) (T, error) { + type res struct { + v T + err error + } + start := time.Now() + logf("step %s starts", what) + ch := make(chan res, 1) + go func() { v, err := f(); ch <- res{v, err} }() + select { + case r := <-ch: + logf("step %s ended after %s (%v)", what, time.Since(start).Round(time.Millisecond), r.err) + return r.v, r.err + case <-time.After(bound): + var zero T + err := fmt.Errorf("%s did not answer within %s", what, bound) + logf("step %s: %v", what, err) + return zero, err + } +} + +func stepErr(bound time.Duration, what string, f func() error) error { + _, err := step(bound, what, func() (struct{}, error) { return struct{}{}, f() }) + return err +} + +// boundedObjects is the incoming object store with each operation bounded and logged. +type boundedObjects struct{ parts.Objects } + +func (b boundedObjects) Used() (uint64, error) { + return step(stepBound, "the incoming object store's fill (objects)", b.Objects.Used) +} +func (b boundedObjects) Put(name string, data []byte) error { + return stepErr(stepBound, "the put of a chunk in the incoming object store (objects)", func() error { return b.Objects.Put(name, data) }) +} +func (b boundedObjects) Get(name string) ([]byte, error) { + return step(stepBound, "the get of a chunk from the incoming object store (objects)", func() ([]byte, error) { return b.Objects.Get(name) }) +} +func (b boundedObjects) Delete(name string) error { + return stepErr(stepBound, "the delete of a chunk in the incoming object store (objects)", func() error { return b.Objects.Delete(name) }) +} + +// boundedTransfers is the transfers state with its reads and writes bounded and logged. +type boundedTransfers struct { + parts.Transfers[holder.Opening] +} + +func (b boundedTransfers) Get(id string) (parts.Transfer[holder.Opening], bool, error) { + type r struct { + t parts.Transfer[holder.Opening] + ok bool + } + x, err := step(stepBound, "the read of the transfer in the transfers state", func() (r, error) { + t, ok, err := b.Transfers.Get(id) + return r{t, ok}, err + }) + return x.t, x.ok, err +} +func (b boundedTransfers) Put(t parts.Transfer[holder.Opening]) error { + return stepErr(stepBound, "the write of the transfer in the transfers state", func() error { return b.Transfers.Put(t) }) +} +func (b boundedTransfers) Delete(id string) error { + return stepErr(stepBound, "the delete of the transfer in the transfers state", func() error { return b.Transfers.Delete(id) }) +} + +// boundedCap is the incoming cap read from its binding, bounded and logged. +func boundedCap(c func() (uint64, error)) func() (uint64, error) { + return func() (uint64, error) { + return step(stepBound, "the read of incoming's cap from its binding (objects)", c) + } +} + +// boundedArm is the idle key's write, bounded and logged. +func boundedArm(a func(string, time.Duration) error) func(string, time.Duration) error { + return func(id string, ttl time.Duration) error { + return stepErr(stepBound, "the write of the idle key in the idle state", func() error { return a(id, ttl) }) + } +} + +// boundedFinding is the findings writer's say, bounded and logged: the receiver says the incoming fill through it on an open. +func boundedFinding(f holder.Finding) holder.Finding { + return func(kind, subject, words string, raised bool) { + _ = stepErr(stepBound, "the finding "+kind+" of "+subject, func() error { f(kind, subject, words, raised); return nil }) + } +} diff --git a/modules/distribution/cmd/distribution-tools/bounded_test.go b/modules/distribution/cmd/distribution-tools/bounded_test.go new file mode 100644 index 000000000..9e75bc1f8 --- /dev/null +++ b/modules/distribution/cmd/distribution-tools/bounded_test.go @@ -0,0 +1,20 @@ +package main + +import ( + "strings" + "testing" + "time" +) + +// A bus operation that never answers becomes an error naming its step, within the bound. +func TestAHungStepAnswersWithItsName(t *testing.T) { + block := make(chan struct{}) + defer close(block) + _, err := step(10*time.Millisecond, "the write of the idle key", func() (int, error) { <-block; return 0, nil }) + if err == nil || !strings.Contains(err.Error(), "the write of the idle key did not answer within") { + t.Fatalf("err = %v", err) + } + if v, err := step(time.Second, "x", func() (int, error) { return 7, nil }); v != 7 || err != nil { + t.Fatalf("%d %v", v, err) + } +} diff --git a/modules/distribution/cmd/distribution-tools/main.go b/modules/distribution/cmd/distribution-tools/main.go index b928f84b4..b20db6f5d 100644 --- a/modules/distribution/cmd/distribution-tools/main.go +++ b/modules/distribution/cmd/distribution-tools/main.go @@ -479,8 +479,8 @@ func main() { }() base, _ := url.Parse("http://127.0.0.1:5001") store := &holder.OCI{Base: base, Upstream: &holder.Upstream{}} - h := &holder.Holder{Logf: logf, Objects: parts.BusObjects(holder.IncomingObjects), Builder: holder.Reserved{Ask: stdio.Ask}, Store: store, Say: say, Finding: findings.Say, - Transfers: parts.BusTransfers[holder.Opening](holder.TransfersState), Cap: parts.BusCap(holder.IncomingObjects), Record: recordAct, Idle: parts.BusArm(holder.IdleState)} + h := &holder.Holder{Logf: logf, Objects: boundedObjects{parts.BusObjects(holder.IncomingObjects)}, Builder: holder.Reserved{Ask: stdio.Ask}, Store: store, Say: say, Finding: boundedFinding(findings.Say), + Transfers: boundedTransfers{parts.BusTransfers[holder.Opening](holder.TransfersState)}, Cap: boundedCap(parts.BusCap(holder.IncomingObjects)), Record: recordAct, Idle: boundedArm(parts.BusArm(holder.IdleState))} files := &holder.Files{Container: or(os.Getenv("MESH_REGISTRY_CONTAINER"), "mesh-registry"), Run: holder.ExecRunner, UID: os.Getuid()} v := &holder.Verbs{Holder: h, Catalog: store, Holds: kept[holder.Hold]{state: "holds", key: holder.Hold.Key}, Mirrors: mirrors{}, Files: files, SweepHolder: func() (string, error) { return seatlist.HolderOf(stdio.Ask, holder.SweepSeat) }}