From 2eaa41a420d48f8f6027268aaae23166bab660d7 Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 21 Nov 2026 18:58:03 +0100 Subject: [PATCH] H-A-292, H-A-293: builder_publish_queue reads one page; a build is retired only within its series The review found two faults. The tool paged on through a state of any size, one replay of the bucket per page; it now reads one page of each of its own states and, when the state holds more, answers that page capped, with the keys read and the state's size from its info, never asking a second page and never refusing. And the retiring of superseded done builds keyed the newest reservation and release by module alone, so a standing 0.2 would retire a done 0.1.x; both are now found per module and series (MAJOR.MINOR). Builds: design 47 H H-A-292, H-A-293, H-A-291 (1), H-A-290 (5) --- .../mesh-delivery-builder-tools/republish.go | 41 ++++++----- .../republish_test.go | 44 +++++++++++ .../mesh-delivery-builder/publishitem_test.go | 6 +- .../internal/versions/republish.go | 73 +++++++++++++------ .../internal/versions/republish_test.go | 41 ++++++++--- .../internal/versions/tools.go | 43 ++++++++--- 6 files changed, 184 insertions(+), 64 deletions(-) create mode 100644 modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/republish_test.go diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/republish.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/republish.go index 33be0c73f..e5dd1e7c4 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/republish.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/republish.go @@ -14,8 +14,8 @@ import ( "mesh-delivery-builder/internal/versions" ) -// readerOf is a kept state as the versions package's reader: every value read in its pages, one request for many (design 47 H -// H-A-290 (5)), where a key-by-key read of a queue of two thousand keys outlasts the tool's call. +// readerOf is a kept state as the versions package's reader: its values read in one page, one request for many (design 47 H +// H-A-290 (5), H-A-292), where a key-by-key read of a queue of two thousand keys outlasts the tool's call. func readerOf(s stdio.KeptState) versions.Reader { return versions.Reader{Keys: s.Keys, Get: func(key string) (json.RawMessage, bool, error) { e, err := s.Get(key) @@ -23,31 +23,34 @@ func readerOf(s stdio.KeptState) versions.Reader { return nil, false, err } return e.Value, true, nil - }, All: func() (map[string]json.RawMessage, error) { return allPages(s.Page) }} + }, All: func() (versions.StatePage, error) { return onePage(s.Page, s.Info) }} } -// statePageBytes and statePageBudget bound one page of a whole state read, so its answer fits one bus message and its time. +// statePageBytes and statePageBudget bound the one page of a state read, so its answer fits one bus message and its time. const ( statePageBytes = 768 * 1024 statePageBudget = 10 * time.Second ) -// allPages is every key's value of a state, page after page; a runtime that answers no page says errors.ErrUnsupported. -func allPages(page func(after string, maxBytes int, budget time.Duration) (stdio.StatePage, error)) (map[string]json.RawMessage, error) { - out := map[string]json.RawMessage{} - for after := ""; ; { - pg, err := page(after, statePageBytes, statePageBudget) - if err != nil { - return nil, err - } - for _, e := range pg.Entries { - out[e.Key] = e.Value - } - if pg.Next == "" || pg.Next == after { - return out, nil - } - after = pg.Next +// onePage is the first page of a state's values, and never a second (H-A-292, H-A-293): when the state holds more, the page says +// so with the state's size from its info. A runtime that answers no page says errors.ErrUnsupported. +func onePage(page func(after string, maxBytes int, budget time.Duration) (stdio.StatePage, error), info func() (*stdio.StateInfo, error)) (versions.StatePage, error) { + pg, err := page("", statePageBytes, statePageBudget) + if err != nil { + return versions.StatePage{}, err } + out := versions.StatePage{Values: make(map[string]json.RawMessage, len(pg.Entries)), More: pg.Next != ""} + for _, e := range pg.Entries { + out.Values[e.Key] = e.Value + } + if out.More { + if in, err := info(); err != nil || in == nil { + out.InfoWords = fmt.Sprintf("the state's info cannot be read: %v", err) + } else { + out.Keys, out.Bytes = in.Keys, in.Bytes + } + } + return out, nil } // republishTools are builder_publish_again and builder_publish_queue (design 47 G §588). The publish again is a hand act: diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/republish_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/republish_test.go new file mode 100644 index 000000000..595ebf91f --- /dev/null +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/republish_test.go @@ -0,0 +1,44 @@ +package main + +import ( + "encoding/json" + "errors" + "testing" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// TestAStateIsReadInOnePage: design 47 H H-A-292, H-A-293 — builder_publish_queue reads each of its own states in one page and +// never asks a second: a state past the page answers that page with More and the state's size from its info; one the page holds +// answers whole without reading the info. +func TestAStateIsReadInOnePage(t *testing.T) { + for _, c := range []struct { + name string + next string + }{{"past one page", "k1"}, {"held by one page", ""}} { + t.Run(c.name, func(t *testing.T) { + var asked []string + infos := 0 + page := func(after string, maxBytes int, budget time.Duration) (stdio.StatePage, error) { + asked = append(asked, after) + if after != "" { + return stdio.StatePage{Entries: []stdio.StateEntry{{Key: "k2", Value: json.RawMessage(`2`)}}}, nil + } + return stdio.StatePage{Entries: []stdio.StateEntry{{Key: "k1", Value: json.RawMessage(`1`)}}, Next: c.next}, nil + } + info := func() (*stdio.StateInfo, error) { infos++; return &stdio.StateInfo{Keys: 2, Bytes: 640}, nil } + pg, err := onePage(page, info) + more := c.next != "" + if err != nil || len(asked) != 1 || len(pg.Values) != 1 || pg.More != more || (more && (pg.Keys != 2 || pg.Bytes != 640 || infos != 1)) || (!more && infos != 0) { + t.Errorf("asked the pages after %q, read %v, more %v, size %d keys %d bytes, info read %d times (%v): want one page only, its rest said with the state's size", asked, pg.Values, pg.More, pg.Keys, pg.Bytes, infos, err) + } + }) + } + unsupported := func(string, int, time.Duration) (stdio.StatePage, error) { + return stdio.StatePage{}, errors.ErrUnsupported + } + if _, err := onePage(unsupported, nil); !errors.Is(err, errors.ErrUnsupported) { + t.Errorf("a runtime with no pages answered %v, want ErrUnsupported so the reader reads key by key", err) + } +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishitem_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishitem_test.go index 9777b3b49..4e2c7ea25 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishitem_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishitem_test.go @@ -173,15 +173,15 @@ func TestAPublishItemIsTriedWithAGrowingPauseAndEndsLoudly(t *testing.T) { _ = p.Ask(ctx, "s", "0.1.3") // asked in an earlier run p.allDone = func(context.Context) ([]versions.ReservedVersion, error) { var out []versions.ReservedVersion - for _, ver := range []string{"0.1.3", "0.10.0", "0.2.1"} { + for _, ver := range []string{"0.1.3", "0.1.10", "0.1.2"} { // one series (H-A-293) out = append(out, versions.ReservedVersion{Module: "s", Version: ver, Artifacts: map[string]string{"s:a": "sha256:a"}, BuiltAt: now}) } return out, nil } p.retired = (&versions.Reserver{}).Retired p.AskAllDone(ctx) - if _, ok := kv.vals[versions.PublishItemKey("s", "0.10.0")]; len(kv.vals) != 1 || !ok { - t.Errorf("the start left the items %v, want only the newest, s 0.10.0, asked and the two older retired", slices.Collect(maps.Keys(kv.vals))) + if _, ok := kv.vals[versions.PublishItemKey("s", "0.1.10")]; len(kv.vals) != 1 || !ok { + t.Errorf("the start left the items %v, want only the newest, s 0.1.10, asked and the two older retired", slices.Collect(maps.Keys(kv.vals))) } kv.vals, kv.revs = map[string][]byte{}, map[string]uint64{} // A released version's item is deleted; a version no longer reserved has it deleted at its next try. diff --git a/modules/mesh-delivery-builder/internal/versions/republish.go b/modules/mesh-delivery-builder/internal/versions/republish.go index 922f26d84..6e205cc62 100644 --- a/modules/mesh-delivery-builder/internal/versions/republish.go +++ b/modules/mesh-delivery-builder/internal/versions/republish.go @@ -234,43 +234,58 @@ func newerVersion(a, b string) bool { return false } -// Retired are the done builds without a release among all that a newer reservation or a newer release of the same module -// supersedes, each with why (design 47 H H-A-291 (1); H-A-275: the older digest is dead): keyed ` `. The -// releases are the builder's own record of them (Releases), read once per module. +// Retired are the done builds without a release among all that a newer reservation or a newer release of the same module and +// series (MAJOR.MINOR) supersedes, each with why (design 47 H H-A-291 (1), H-A-293; H-A-275: the older digest is dead): keyed +// ` `. A standing newer series never retires an older one's build. The releases are the builder's own record +// of them (Releases), read once per module. func (r *Reserver) Retired(ctx context.Context, all []ReservedVersion) (map[string]string, error) { - newest := map[string]string{} // module -> its newest reserved version + seriesOf := func(v ReservedVersion) string { + s, _, err := SplitVersion(v.Version) + if err != nil { + return "" + } + return v.Module + " " + s + } + newest := map[string]string{} // module and series -> its newest reserved version for _, v := range all { - if n, ok := newest[v.Module]; !ok || newerVersion(v.Version, n) { - newest[v.Module] = v.Version + k := seriesOf(v) + if n, ok := newest[k]; k != "" && (!ok || newerVersion(v.Version, n)) { + newest[k] = v.Version } } - released := map[string]string{} // module -> its newest release the record keeps + kept := map[string][]string{} // module -> the releases the record keeps + read := map[string]bool{} out := map[string]string{} for _, v := range all { - if v.BuiltAt.IsZero() { + k := seriesOf(v) + if v.BuiltAt.IsZero() || k == "" { continue } - if n := newest[v.Module]; newerVersion(n, v.Version) { + if n := newest[k]; newerVersion(n, v.Version) { out[v.Module+" "+v.Version] = "the newer reservation " + n + " of " + v.Module + " supersedes it" continue } - rel, read := released[v.Module] - if !read && r.Releases != nil { + if !read[v.Module] && r.Releases != nil { + read[v.Module] = true raw, _, found, err := r.Releases.Get(ctx, ReleasedKey(v.Module)) if err != nil { return out, fmt.Errorf("the releases of %s cannot be read: %w", v.Module, err) } var rec ReleasedRecord if found && json.Unmarshal(raw, &rec) == nil { - for _, k := range rec.Kept { - if rel == "" || newerVersion(k.Version, rel) { - rel = k.Version - } + for _, rb := range rec.Kept { + kept[v.Module] = append(kept[v.Module], rb.Version) } } - released[v.Module] = rel } - if rel != "" && newerVersion(rel, v.Version) { + series, _, _ := SplitVersion(v.Version) + rel := "" + for _, kv := range kept[v.Module] { + if s, _, err := SplitVersion(kv); err == nil && s == series && newerVersion(kv, v.Version) && (rel == "" || newerVersion(kv, rel)) { + rel = kv + } + } + if rel != "" { out[v.Module+" "+v.Version] = "the newer release " + rel + " of " + v.Module + " supersedes it" } } @@ -478,6 +493,18 @@ type Queued struct { // tries and its next try; an item that ended says why. ItemCount int `json:"item-count"` Items []PublishItem `json:"items"` + // Capped says a state held more than the one page read of it (H-A-293); Read names each such state, the keys read and its + // size. Every count above is then of the keys read. + Capped bool `json:"capped,omitempty"` + Read []StateCapped `json:"read,omitempty"` +} + +// capped notes a state read only in part. +func (q *Queued) capped(c *StateCapped) { + if c != nil { + q.Capped = true + q.Read = append(q.Read, *c) + } } // QueueCount is how many versions one holder's queue holds of one class, and when the oldest and the newest were queued. @@ -515,7 +542,8 @@ func PublishItemKey(module, version string) string { // listed queue to one module's (one version's), so an item's class can be read. func AnswerQueue(queue, acted, reasks Reader, module, version string) (Queued, error) { out := Queued{Counts: []QueueCount{}, Queue: []QueuedPublish{}, Acted: map[string]uint64{}, Items: []PublishItem{}} - vals, err := queue.values() + vals, c, err := queue.values(PublishQueueState) + out.capped(c) if err != nil { return out, fmt.Errorf("the state %s cannot be read: %w", PublishQueueState, err) } @@ -575,7 +603,8 @@ func AnswerQueue(queue, acted, reasks Reader, module, version string) (Queued, e } } out.Listed = len(out.Queue) - avals, err := acted.values() + avals, c, err := acted.values(ActedState) + out.capped(c) if err != nil { return out, fmt.Errorf("the state %s cannot be read: %w", ActedState, err) } @@ -585,7 +614,8 @@ func AnswerQueue(queue, acted, reasks Reader, module, version string) (Queued, e out.Acted[k] = a.Sequence } } - rvals, err := reasks.values() + rvals, c, err := reasks.values("reasks") + out.capped(c) if err != nil { return out, fmt.Errorf("the re-ask state cannot be read: %w", err) } @@ -612,7 +642,8 @@ const PublishQueueDescription = "The versions the builder will publish again (it "queue time; the first 50 in the order they are sent, each with its class, its artifacts' digests, when it was queued and whether its publish has " + "begun (module, and version, list only that module's, so one item's class can be read); the keys of the state " + ActedState + ", the last stream " + "sequence the builder acted on for each subject; and the publish items (design 47 G A-457a), counted, the first 50 by next try, each version's " + - "tries and next try, and why one ended after its tries. Read-only." + "tries and next try, and why one ended after its tries. Each state is read in one page; a state past it is answered capped, naming the " + + "keys read and the state's size (H-A-293). Read-only." // PublishAgainDescription is builder_publish_again's description. const PublishAgainDescription = "Publish reserved versions again (design 47 G section 588), which builds nothing: with module and version, that reserved " + diff --git a/modules/mesh-delivery-builder/internal/versions/republish_test.go b/modules/mesh-delivery-builder/internal/versions/republish_test.go index 6966e2a74..51d1256a4 100644 --- a/modules/mesh-delivery-builder/internal/versions/republish_test.go +++ b/modules/mesh-delivery-builder/internal/versions/republish_test.go @@ -430,16 +430,27 @@ func TestAHoldersQueueSendsLiveFirstInPasses(t *testing.T) { } }) }) - t.Run("a build a newer reservation or release supersedes is retired at the pass and at the start", func(t *testing.T) { + t.Run("a build a newer patch or release of its series supersedes is retired at the pass and at the start, never by a newer series", func(t *testing.T) { b := newRepublishBed(t) v := b.reserve(t, "m") + b.holders = map[string][]string{SeatArtifacts: {"oci"}} if _, err := b.Reserved.Reserve(ctx, "m", "0.2", []string{"m:prog"}, Source{Fingerprint: "src2:m"}); err != nil { t.Fatal(err) } b.queueAt(t, v, PublishClassStart, b.now) b.Drain(ctx, "oci") + if len(b.sent) != 1 || len(b.ended) != 0 { + t.Errorf("with a standing 0.2: sent %v, ended %v, want m %s sent, its series not superseded (H-A-293)", b.sent, b.ended, v.Version) + } + rel := &reserveMem{mem: newMem()} + raw, _ := json.Marshal(ReleasedRecord{Kept: []ReleasedBuild{{Fingerprint: "f", Version: "0.2.3"}, {Fingerprint: "g", Version: "0.1.9"}}}) + _, _ = rel.Create(ctx, ReleasedKey("m"), raw) + b.Reserved.Releases = rel + b.sent = nil + b.queueAt(t, v, PublishClassStart, b.now) + b.Drain(ctx, "oci") if len(b.sent) != 0 || fmt.Sprint(b.ended) != "[m "+v.Version+"]" { - t.Errorf("at the pass: sent %v, ended %v, want m %s retired", b.sent, b.ended, v.Version) + t.Errorf("at the pass: sent %v, ended %v, want m %s retired by the release 0.1.9", b.sent, b.ended, v.Version) } b.ended = nil b.queueAt(t, v, PublishClassStart, b.now) @@ -450,13 +461,13 @@ func TestAHoldersQueueSendsLiveFirstInPasses(t *testing.T) { if keys, _ := b.queue.KeysOf(ctx, QueueFilter); len(keys) != 0 || fmt.Sprint(b.ended) != "[m "+v.Version+"]" { t.Errorf("at the start: keys %v, ended %v, want m %s retired", keys, b.ended, v.Version) } - rel := &reserveMem{mem: newMem()} - raw, _ := json.Marshal(ReleasedRecord{Kept: []ReleasedBuild{{Fingerprint: "f", Version: "0.1.9"}}}) - _, _ = rel.Create(ctx, ReleasedKey("n"), raw) - r := &Reserver{Releases: rel} - got, err := r.Retired(ctx, []ReservedVersion{{Module: "n", Version: "0.1.4", BuiltAt: b.now}, {Module: "p", Version: "0.1.12", BuiltAt: b.now}}) - if err != nil || len(got) != 1 || !strings.Contains(got["n 0.1.4"], "release 0.1.9") { - t.Errorf("retired %v (%v), want n 0.1.4 by the release 0.1.9 and p 0.1.12 kept", got, err) + only := &reserveMem{mem: newMem()} + raw, _ = json.Marshal(ReleasedRecord{Kept: []ReleasedBuild{{Fingerprint: "f", Version: "0.2.0"}}}) + _, _ = only.Create(ctx, ReleasedKey("n"), raw) + got, err := (&Reserver{Releases: only}).Retired(ctx, []ReservedVersion{{Module: "n", Version: "0.1.4", BuiltAt: b.now}, {Module: "n", Version: "0.1.6"}, + {Module: "n", Version: "0.3.0", BuiltAt: b.now}, {Module: "p", Version: "0.1.2", BuiltAt: b.now}}) + if err != nil || len(got) != 1 || !strings.Contains(got["n 0.1.4"], "reservation 0.1.6") { + t.Errorf("retired %v (%v), want only n 0.1.4, by the newer patch 0.1.6; n 0.3.0 and p 0.1.2 kept, a release of 0.2 retiring neither", got, err) } }) t.Run("a version taken over is not sent, and its item ended", func(t *testing.T) { @@ -491,7 +502,7 @@ func TestThePublishQueueToolAnswersAtAnySize(t *testing.T) { vals[QueueKey(holder, q.Module, q.Version)], _ = json.Marshal(q) } gets := 0 - queue := Reader{All: func() (map[string]json.RawMessage, error) { return vals, nil }, + queue := Reader{All: func() (StatePage, error) { return StatePage{Values: vals}, nil }, Keys: func() ([]string, error) { return slices.Collect(maps.Keys(vals)), nil }, Get: func(k string) (json.RawMessage, bool, error) { gets++; v, ok := vals[k]; return v, ok, nil }} none := Reader{Keys: func() ([]string, error) { return nil, nil }} @@ -512,7 +523,15 @@ func TestThePublishQueueToolAnswersAtAnySize(t *testing.T) { if err != nil || len(one.Queue) != 1 || one.Queue[0].Class != PublishClassLive || one.Total != 2000 { t.Errorf("one module's answer %+v (%v), want its item with its class", one.Queue, err) } - queue.All = func() (map[string]json.RawMessage, error) { return nil, errors.ErrUnsupported } + if a.Capped || len(a.Read) != 0 { + t.Errorf("a state one page holds is answered capped: %+v", a.Read) + } + queue.All = func() (StatePage, error) { return StatePage{Values: vals, More: true, Keys: 5000, Bytes: 3 << 20}, nil } + if c, err := AnswerQueue(queue, none, none, "", ""); err != nil || !c.Capped || len(c.Read) != 1 || c.Read[0].State != PublishQueueState || + c.Read[0].KeysRead != 2000 || c.Read[0].StateKeys != 5000 || c.Read[0].StateBytes != 3<<20 || c.Total != 2000 { + t.Errorf("a state past its one page answered capped %v, read %+v (%v): want capped, 2000 keys read of 5000, its size said (H-A-293)", c.Capped, c.Read, err) + } + queue.All = func() (StatePage, error) { return StatePage{}, errors.ErrUnsupported } if b, err := AnswerQueue(queue, none, none, "", ""); err != nil || b.Total != 2000 || gets != 2000 { t.Errorf("a reader with no bulk read answered %d (%v) with %d single reads, want every key read one by one", b.Total, err, gets) } diff --git a/modules/mesh-delivery-builder/internal/versions/tools.go b/modules/mesh-delivery-builder/internal/versions/tools.go index e5a7740d4..8a562cbf5 100644 --- a/modules/mesh-delivery-builder/internal/versions/tools.go +++ b/modules/mesh-delivery-builder/internal/versions/tools.go @@ -21,26 +21,49 @@ const ( OldFormClosedEvent = "version.old-form-closed" ) -// Reader reads the state as the tools bundle may: every key, and one; and, where All is set, every key's value in one bulk read -// (the state's pages), which a state of thousands of keys needs to be answered at all (design 47 H H-A-290 (5)). +// Reader reads the state as the tools bundle may: every key, and one; and, where All is set, the state's values in one page, +// one request for many (design 47 H H-A-290 (5), H-A-292). type Reader struct { Keys func() ([]string, error) Get func(key string) (json.RawMessage, bool, error) - // All is every key's value read in bulk; nil, or an error satisfying errors.Is(err, errors.ErrUnsupported), reads key by key. - All func() (map[string]json.RawMessage, error) + // All is one page of the state's values; nil, or an error satisfying errors.Is(err, errors.ErrUnsupported), reads key by key. + All func() (StatePage, error) } -// values is every key's value: in bulk where the reader can, else key by key. -func (r Reader) values() (map[string]json.RawMessage, error) { +// StatePage is one page of a state (H-A-292, H-A-293): its values, and, when the state holds more than the page, More with the +// state's size as its info says it (Keys, Bytes; InfoWords when the info cannot be read). No second page is asked. +type StatePage struct { + Values map[string]json.RawMessage + More bool + Keys, Bytes uint64 + InfoWords string +} + +// StateCapped is what an answer says of a state it read only the first page of (H-A-293): how many keys it read, and the +// state's size from its info. +type StateCapped struct { + State string `json:"state"` + KeysRead int `json:"keys-read"` + StateKeys uint64 `json:"state-keys"` + StateBytes uint64 `json:"state-bytes"` + InfoWords string `json:"info,omitempty"` +} + +// values is the state's values: one page where the reader can (capped says what a page past the state's end left out), else +// key by key. +func (r Reader) values(state string) (map[string]json.RawMessage, *StateCapped, error) { if r.All != nil { - out, err := r.All() + pg, err := r.All() if !errors.Is(err, errors.ErrUnsupported) { - return out, err + if err != nil || !pg.More { + return pg.Values, nil, err + } + return pg.Values, &StateCapped{State: state, KeysRead: len(pg.Values), StateKeys: pg.Keys, StateBytes: pg.Bytes, InfoWords: pg.InfoWords}, nil } } keys, err := r.Keys() if err != nil { - return nil, err + return nil, nil, err } out := make(map[string]json.RawMessage, len(keys)) for _, k := range keys { @@ -50,7 +73,7 @@ func (r Reader) values() (map[string]json.RawMessage, error) { } out[k] = raw } - return out, nil + return out, nil, nil } // readStore is a Store over a Reader that refuses every write: the tools bundle only reads versions.