novox/mesh-catalog
9
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

F-catalog-B: npm_incoming and go_incoming read and drop the package seats' transfers; an idle one drops by its key

Browse Source 
The package seats kept incoming, transfers and findings with no read or drop tool (tools review F4).
npm_incoming and go_incoming list a seat's open transfers, incoming's fill and the cap its binding carries,
and drop one by hand. The same fault as distribution's (running review F2, its second instance): the
transfers state's own lifetime dropped a transfer's key and left its chunks, and the drop otherwise waited
for the next publish. transfers keeps for good; each open transfer has a key in the new state idle with its
own time to live, whose end drops it. The holder counts against its binding's cap.

Claude-Session: https://claude.ai/code/session_01JG73KJNrH5EgFXMKKYSrig
This commit is contained in:
jschoubben
2026-11-04 17:14:04 +01:00
parent 8f5611a2d1
commit bccaf75556
9 changed files with 276 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/gitea/internal/packageseats/holder.go
+82 -9
View File
@@ -44,8 +44,8 @@ const (
)
// The bounds of incoming (design 51 *Publishing*, ADR 0611): a chunk at most 1 MiB, a transfer with no chunk for 24
// hours dropped, a finding past 80 % of the cap. A package is one file, far smaller than an image, so the cap is the
// manifest's 2 GiB.
// hours dropped, a finding past 80 % of the cap. The cap is the one incoming's binding carries; IncomingCap is the
// manifest's 2 GiB (a package is one file, far smaller than an image), said only where no binding is read.
const (
MaxChunk = 1 << 20
IncomingCap = 2 << 30
@@ -53,6 +53,9 @@ const (
IncomingFullPct = 80
IncomingObjects = "incoming"
TransfersState = "transfers"
// IdleState is the state of one key per open transfer with its own time to live (a state declared key-ttl): its
// end is the event that drops a transfer left with no chunk for TransferIdle, never a timer.
IdleState = "idle"
// KindPackage is the one kind a package seat holds.
KindPackage = "package"
// Self is the forge module, the holder every outcome names.
@@ -151,7 +154,20 @@ type Holder struct {
// Finding writes (raised) or clears the module's finding of a kind for a subject.
Finding func(kind, subject, words string, raised bool)
Now func() time.Time
Cap uint64
// Cap reads incoming's cap from its binding; IncomingCap when nil or when the binding names none.
Cap func() (uint64, error)
// Idle writes a transfer's idle key with its own time to live, once; one already there is no fault. Nil arms nothing.
Idle func(key string, ttl time.Duration) error
}
// CapNow is incoming's cap as its binding carries it.
func (h *Holder) CapNow() uint64 {
if h.Cap != nil {
if c, err := h.Cap(); err == nil && c > 0 {
return c
}
}
return IncomingCap
}
func (h *Holder) now() time.Time {
@@ -168,10 +184,7 @@ func Reference(name, version string) string { return name + "@" + version }
func key(seat, transfer string) string { return seat + "_" + transfer }
func (h *Holder) receiver() *parts.Receiver[Opening] {
cap := h.Cap
if cap == 0 {
cap = IncomingCap
}
cap := h.CapNow()
return &parts.Receiver[Opening]{Objects: h.Objects, Transfers: h.Transfers, Cap: cap, MaxChunk: MaxChunk,
Idle: TransferIdle, Full: IncomingFullPct, Now: h.now,
// The builder's numbering is asked before anything is uploaded (ADR 0605 §4).
@@ -230,7 +243,6 @@ func (h *Holder) Publish(seat, caller string, p Publish) (Answer, error) {
if err := checkPublish(seat, p); err != nil {
return Answer{}, err
}
_, _ = h.receiver().DropIdle()
// A rebuild of a published version answers it held and uploads nothing; the same version with other content is
// refused, as the registry would refuse it.
if d, ok, err := h.Store.Held(seat, p.Name, p.Version); err != nil {
@@ -253,6 +265,13 @@ func (h *Holder) Publish(seat, caller string, p Publish) (Answer, error) {
if err != nil {
return h.answer(p.Transfer, got), fmt.Errorf("publish: %w", err)
}
// An open transfer is bounded by its idle key; one that cannot be written fails the publish loudly, and the build
// module's next publish of the same transfer answers the part held and arms it again.
if !got.Ended && h.Idle != nil {
if err := h.Idle(key(seat, p.Transfer), TransferIdle); err != nil {
return h.answer(p.Transfer, got), fmt.Errorf("publish: transfer %s is open but its idle bound is not kept, so it would never be dropped: %w", p.Transfer, err)
}
}
return h.answer(p.Transfer, got), nil
}
@@ -324,11 +343,65 @@ func (h *Holder) Retained(seat, name, version, digest string) error {
return h.Say(seat, EventRemoved, Outcome{Name: name, Version: version, Kind: KindPackage, Digest: digest, Reason: ReasonRetention})
}
// Drop drops a transfer by hand; DropIdle every transfer idle for TransferIdle.
// Drop drops a transfer by hand.
func (h *Holder) Drop(seat, transfer string) (Transfer, error) {
return h.receiver().Drop(key(seat, transfer))
}
// Incoming is what npm_incoming and go_incoming answer of a seat: its open transfers, incoming's fill (both seats
// share the store) and the cap its binding carries.
type Incoming struct {
Seat string `json:"seat"`
Transfers []Transfer `json:"transfers"`
Used uint64 `json:"used"`
Cap uint64 `json:"cap"`
}
// IncomingOf reads a seat's open transfers and incoming's fill.
func (h *Holder) IncomingOf(seat string) (Incoming, error) {
if _, ok := Seats[seat]; !ok {
return Incoming{}, fmt.Errorf("the forge holds no seat %s", seat)
}
all, err := h.Transfers.List()
if err != nil {
return Incoming{}, fmt.Errorf("the transfers cannot be read: %w", err)
}
out := Incoming{Seat: seat, Transfers: []Transfer{}, Cap: h.CapNow()}
for _, t := range all {
if t.Shape.Seat == seat {
out.Transfers = append(out.Transfers, t)
}
}
if out.Used, err = h.Objects.Used(); err != nil {
return out, fmt.Errorf("incoming's fill cannot be read: %w", err)
}
return out, nil
}
// Expired is told that a transfer's idle key (its key in the state transfers) ended: a transfer with no chunk for
// TransferIdle is dropped with its chunks; one that took a chunk since is armed again for the rest; one gone is
// nothing to do. It answers whether it dropped it.
func (h *Holder) Expired(id string) (bool, error) {
t, found, err := h.Transfers.Get(id)
if err != nil {
return false, fmt.Errorf("transfer %s's idle bound ended, but it cannot be read: %w", id, err)
}
if !found {
return false, nil
}
idle := h.now().Sub(t.Last)
if idle >= TransferIdle {
if _, err := h.receiver().Drop(id); err != nil {
return false, fmt.Errorf("transfer %s had no chunk for %s and is not dropped: %w", id, TransferIdle, err)
}
return true, nil
}
if h.Idle == nil {
return false, nil
}
return false, h.Idle(id, max(TransferIdle-idle, time.Second))
}
func checkPublish(seat string, p Publish) error {
switch {
case Seats[seat] == "":
modules/gitea/internal/packageseats/holder_test.go
+42
View File
@@ -149,6 +149,48 @@ func TestTheForgeHoldsBothPackageSeats(t *testing.T) {
}
}
})
// Tools review F4 and running review F2's kind: a seat's incoming is read per seat with the cap its binding
// carries; an open transfer is armed with its idle key, re-armed for the rest when it took a chunk since, and
// dropped with its chunks when it had none for 24 hours; a cap from the binding refuses a transfer past it.
t.Run("incoming: read per seat, the binding's cap, an idle transfer dropped when its key ends", func(t *testing.T) {
b := newBed(true)
now := time.Unix(1e9, 0)
b.h.Now = func() time.Time { return now }
armed := map[string]time.Duration{}
b.h.Idle = func(k string, ttl time.Duration) error { armed[k] = ttl; return nil }
b.h.Cap = func() (uint64, error) { return 1000, nil }
big := []byte(strings.Repeat("x", 600))
if _, err := b.h.Publish(NpmSeat, nodeBuild, pub("a", "0.1.0", big)); err != nil {
t.Fatal(err)
}
if _, err := b.h.Publish(GoSeat, goBuild, pub("b", "0.1.0", big)); err != nil || armed["go-package-registry_t-b"] != TransferIdle {
t.Fatalf("publish = %v, armed %v; want the Go transfer's idle key for %s", err, armed, TransferIdle)
}
_, _ = b.h.PublishPart(NpmSeat, nodeBuild, "t-a", digest(big), 0, false, big[:500])
in, err := b.h.IncomingOf(NpmSeat)
if err != nil || len(in.Transfers) != 1 || in.Transfers[0].Shape.Name != "a" || in.Cap != 1000 || in.Used != 500 {
t.Fatalf("IncomingOf(npm) = %+v, %v; want a's transfer alone, 500 used of the binding's 1000", in, err)
}
if _, err := b.h.Publish(NpmSeat, nodeBuild, pub("c", "0.1.0", big)); err == nil || !strings.Contains(err.Error(), "cap") {
t.Errorf("a transfer past the binding's cap = %v; want refused at the cap", err)
}
now = now.Add(time.Hour)
_, _ = b.h.PublishPart(GoSeat, goBuild, "t-b", digest(big), 0, false, big[:10])
now = now.Add(TransferIdle - time.Hour)
if dropped, err := b.h.Expired("go-package-registry_t-b"); err != nil || dropped || armed["go-package-registry_t-b"] != time.Hour {
t.Errorf("b's key ends an hour after a chunk: %v, %v, armed %v; want kept, armed 1h", dropped, err, armed)
}
if dropped, err := b.h.Expired("npm-package-registry_t-a"); err != nil || !dropped {
t.Errorf("a's key ends 24 hours after its last chunk: %v, %v; want dropped", dropped, err)
}
if in, _ := b.h.IncomingOf(NpmSeat); len(in.Transfers) != 0 || in.Used != 10 {
t.Errorf("after a's drop: %+v; want no npm transfer and only b's 10 bytes used", in)
}
b.h.Idle = func(string, time.Duration) error { return errors.New("bus down") }
if _, err := b.h.Publish(NpmSeat, nodeBuild, pub("d", "0.1.0", file)); err == nil || !strings.Contains(err.Error(), "idle bound") {
t.Errorf("a publish whose idle key is not written = %v; want it failed loudly", err)
}
})
t.Run("refusals", func(t *testing.T) {
for _, c := range []struct {
name string