diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/republish.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/republish.go index 33be0c73f..e5dd1e7c4 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/republish.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/republish.go @@ -14,8 +14,8 @@ import ( "mesh-delivery-builder/internal/versions" ) -// readerOf is a kept state as the versions package's reader: every value read in its pages, one request for many (design 47 H -// H-A-290 (5)), where a key-by-key read of a queue of two thousand keys outlasts the tool's call. +// readerOf is a kept state as the versions package's reader: its values read in one page, one request for many (design 47 H +// H-A-290 (5), H-A-292), where a key-by-key read of a queue of two thousand keys outlasts the tool's call. func readerOf(s stdio.KeptState) versions.Reader { return versions.Reader{Keys: s.Keys, Get: func(key string) (json.RawMessage, bool, error) { e, err := s.Get(key) @@ -23,31 +23,34 @@ func readerOf(s stdio.KeptState) versions.Reader { return nil, false, err } return e.Value, true, nil - }, All: func() (map[string]json.RawMessage, error) { return allPages(s.Page) }} + }, All: func() (versions.StatePage, error) { return onePage(s.Page, s.Info) }} } -// statePageBytes and statePageBudget bound one page of a whole state read, so its answer fits one bus message and its time. +// statePageBytes and statePageBudget bound the one page of a state read, so its answer fits one bus message and its time. const ( statePageBytes = 768 * 1024 statePageBudget = 10 * time.Second ) -// allPages is every key's value of a state, page after page; a runtime that answers no page says errors.ErrUnsupported. -func allPages(page func(after string, maxBytes int, budget time.Duration) (stdio.StatePage, error)) (map[string]json.RawMessage, error) { - out := map[string]json.RawMessage{} - for after := ""; ; { - pg, err := page(after, statePageBytes, statePageBudget) - if err != nil { - return nil, err - } - for _, e := range pg.Entries { - out[e.Key] = e.Value - } - if pg.Next == "" || pg.Next == after { - return out, nil - } - after = pg.Next +// onePage is the first page of a state's values, and never a second (H-A-292, H-A-293): when the state holds more, the page says +// so with the state's size from its info. A runtime that answers no page says errors.ErrUnsupported. +func onePage(page func(after string, maxBytes int, budget time.Duration) (stdio.StatePage, error), info func() (*stdio.StateInfo, error)) (versions.StatePage, error) { + pg, err := page("", statePageBytes, statePageBudget) + if err != nil { + return versions.StatePage{}, err } + out := versions.StatePage{Values: make(map[string]json.RawMessage, len(pg.Entries)), More: pg.Next != ""} + for _, e := range pg.Entries { + out.Values[e.Key] = e.Value + } + if out.More { + if in, err := info(); err != nil || in == nil { + out.InfoWords = fmt.Sprintf("the state's info cannot be read: %v", err) + } else { + out.Keys, out.Bytes = in.Keys, in.Bytes + } + } + return out, nil } // republishTools are builder_publish_again and builder_publish_queue (design 47 G §588). The publish again is a hand act: diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/republish_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/republish_test.go new file mode 100644 index 000000000..595ebf91f --- /dev/null +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/republish_test.go @@ -0,0 +1,44 @@ +package main + +import ( + "encoding/json" + "errors" + "testing" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// TestAStateIsReadInOnePage: design 47 H H-A-292, H-A-293 — builder_publish_queue reads each of its own states in one page and +// never asks a second: a state past the page answers that page with More and the state's size from its info; one the page holds +// answers whole without reading the info. +func TestAStateIsReadInOnePage(t *testing.T) { + for _, c := range []struct { + name string + next string + }{{"past one page", "k1"}, {"held by one page", ""}} { + t.Run(c.name, func(t *testing.T) { + var asked []string + infos := 0 + page := func(after string, maxBytes int, budget time.Duration) (stdio.StatePage, error) { + asked = append(asked, after) + if after != "" { + return stdio.StatePage{Entries: []stdio.StateEntry{{Key: "k2", Value: json.RawMessage(`2`)}}}, nil + } + return stdio.StatePage{Entries: []stdio.StateEntry{{Key: "k1", Value: json.RawMessage(`1`)}}, Next: c.next}, nil + } + info := func() (*stdio.StateInfo, error) { infos++; return &stdio.StateInfo{Keys: 2, Bytes: 640}, nil } + pg, err := onePage(page, info) + more := c.next != "" + if err != nil || len(asked) != 1 || len(pg.Values) != 1 || pg.More != more || (more && (pg.Keys != 2 || pg.Bytes != 640 || infos != 1)) || (!more && infos != 0) { + t.Errorf("asked the pages after %q, read %v, more %v, size %d keys %d bytes, info read %d times (%v): want one page only, its rest said with the state's size", asked, pg.Values, pg.More, pg.Keys, pg.Bytes, infos, err) + } + }) + } + unsupported := func(string, int, time.Duration) (stdio.StatePage, error) { + return stdio.StatePage{}, errors.ErrUnsupported + } + if _, err := onePage(unsupported, nil); !errors.Is(err, errors.ErrUnsupported) { + t.Errorf("a runtime with no pages answered %v, want ErrUnsupported so the reader reads key by key", err) + } +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishitem_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishitem_test.go index 9777b3b49..4e2c7ea25 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishitem_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/publishitem_test.go @@ -173,15 +173,15 @@ func TestAPublishItemIsTriedWithAGrowingPauseAndEndsLoudly(t *testing.T) { _ = p.Ask(ctx, "s", "0.1.3") // asked in an earlier run p.allDone = func(context.Context) ([]versions.ReservedVersion, error) { var out []versions.ReservedVersion - for _, ver := range []string{"0.1.3", "0.10.0", "0.2.1"} { + for _, ver := range []string{"0.1.3", "0.1.10", "0.1.2"} { // one series (H-A-293) out = append(out, versions.ReservedVersion{Module: "s", Version: ver, Artifacts: map[string]string{"s:a": "sha256:a"}, BuiltAt: now}) } return out, nil } p.retired = (&versions.Reserver{}).Retired p.AskAllDone(ctx) - if _, ok := kv.vals[versions.PublishItemKey("s", "0.10.0")]; len(kv.vals) != 1 || !ok { - t.Errorf("the start left the items %v, want only the newest, s 0.10.0, asked and the two older retired", slices.Collect(maps.Keys(kv.vals))) + if _, ok := kv.vals[versions.PublishItemKey("s", "0.1.10")]; len(kv.vals) != 1 || !ok { + t.Errorf("the start left the items %v, want only the newest, s 0.1.10, asked and the two older retired", slices.Collect(maps.Keys(kv.vals))) } kv.vals, kv.revs = map[string][]byte{}, map[string]uint64{} // A released version's item is deleted; a version no longer reserved has it deleted at its next try. diff --git a/modules/mesh-delivery-builder/internal/versions/republish.go b/modules/mesh-delivery-builder/internal/versions/republish.go index 922f26d84..6e205cc62 100644 --- a/modules/mesh-delivery-builder/internal/versions/republish.go +++ b/modules/mesh-delivery-builder/internal/versions/republish.go @@ -234,43 +234,58 @@ func newerVersion(a, b string) bool { return false } -// Retired are the done builds without a release among all that a newer reservation or a newer release of the same module -// supersedes, each with why (design 47 H H-A-291 (1); H-A-275: the older digest is dead): keyed ` `. The -// releases are the builder's own record of them (Releases), read once per module. +// Retired are the done builds without a release among all that a newer reservation or a newer release of the same module and +// series (MAJOR.MINOR) supersedes, each with why (design 47 H H-A-291 (1), H-A-293; H-A-275: the older digest is dead): keyed +// ` `. A standing newer series never retires an older one's build. The releases are the builder's own record +// of them (Releases), read once per module. func (r *Reserver) Retired(ctx context.Context, all []ReservedVersion) (map[string]string, error) { - newest := map[string]string{} // module -> its newest reserved version + seriesOf := func(v ReservedVersion) string { + s, _, err := SplitVersion(v.Version) + if err != nil { + return "" + } + return v.Module + " " + s + } + newest := map[string]string{} // module and series -> its newest reserved version for _, v := range all { - if n, ok := newest[v.Module]; !ok || newerVersion(v.Version, n) { - newest[v.Module] = v.Version + k := seriesOf(v) + if n, ok := newest[k]; k != "" && (!ok || newerVersion(v.Version, n)) { + newest[k] = v.Version } } - released := map[string]string{} // module -> its newest release the record keeps + kept := map[string][]string{} // module -> the releases the record keeps + read := map[string]bool{} out := map[string]string{} for _, v := range all { - if v.BuiltAt.IsZero() { + k := seriesOf(v) + if v.BuiltAt.IsZero() || k == "" { continue } - if n := newest[v.Module]; newerVersion(n, v.Version) { + if n := newest[k]; newerVersion(n, v.Version) { out[v.Module+" "+v.Version] = "the newer reservation " + n + " of " + v.Module + " supersedes it" continue } - rel, read := released[v.Module] - if !read && r.Releases != nil { + if !read[v.Module] && r.Releases != nil { + read[v.Module] = true raw, _, found, err := r.Releases.Get(ctx, ReleasedKey(v.Module)) if err != nil { return out, fmt.Errorf("the releases of %s cannot be read: %w", v.Module, err) } var rec ReleasedRecord if found && json.Unmarshal(raw, &rec) == nil { - for _, k := range rec.Kept { - if rel == "" || newerVersion(k.Version, rel) { - rel = k.Version - } + for _, rb := range rec.Kept { + kept[v.Module] = append(kept[v.Module], rb.Version) } } - released[v.Module] = rel } - if rel != "" && newerVersion(rel, v.Version) { + series, _, _ := SplitVersion(v.Version) + rel := "" + for _, kv := range kept[v.Module] { + if s, _, err := SplitVersion(kv); err == nil && s == series && newerVersion(kv, v.Version) && (rel == "" || newerVersion(kv, rel)) { + rel = kv + } + } + if rel != "" { out[v.Module+" "+v.Version] = "the newer release " + rel + " of " + v.Module + " supersedes it" } } @@ -478,6 +493,18 @@ type Queued struct { // tries and its next try; an item that ended says why. ItemCount int `json:"item-count"` Items []PublishItem `json:"items"` + // Capped says a state held more than the one page read of it (H-A-293); Read names each such state, the keys read and its + // size. Every count above is then of the keys read. + Capped bool `json:"capped,omitempty"` + Read []StateCapped `json:"read,omitempty"` +} + +// capped notes a state read only in part. +func (q *Queued) capped(c *StateCapped) { + if c != nil { + q.Capped = true + q.Read = append(q.Read, *c) + } } // QueueCount is how many versions one holder's queue holds of one class, and when the oldest and the newest were queued. @@ -515,7 +542,8 @@ func PublishItemKey(module, version string) string { // listed queue to one module's (one version's), so an item's class can be read. func AnswerQueue(queue, acted, reasks Reader, module, version string) (Queued, error) { out := Queued{Counts: []QueueCount{}, Queue: []QueuedPublish{}, Acted: map[string]uint64{}, Items: []PublishItem{}} - vals, err := queue.values() + vals, c, err := queue.values(PublishQueueState) + out.capped(c) if err != nil { return out, fmt.Errorf("the state %s cannot be read: %w", PublishQueueState, err) } @@ -575,7 +603,8 @@ func AnswerQueue(queue, acted, reasks Reader, module, version string) (Queued, e } } out.Listed = len(out.Queue) - avals, err := acted.values() + avals, c, err := acted.values(ActedState) + out.capped(c) if err != nil { return out, fmt.Errorf("the state %s cannot be read: %w", ActedState, err) } @@ -585,7 +614,8 @@ func AnswerQueue(queue, acted, reasks Reader, module, version string) (Queued, e out.Acted[k] = a.Sequence } } - rvals, err := reasks.values() + rvals, c, err := reasks.values("reasks") + out.capped(c) if err != nil { return out, fmt.Errorf("the re-ask state cannot be read: %w", err) } @@ -612,7 +642,8 @@ const PublishQueueDescription = "The versions the builder will publish again (it "queue time; the first 50 in the order they are sent, each with its class, its artifacts' digests, when it was queued and whether its publish has " + "begun (module, and version, list only that module's, so one item's class can be read); the keys of the state " + ActedState + ", the last stream " + "sequence the builder acted on for each subject; and the publish items (design 47 G A-457a), counted, the first 50 by next try, each version's " + - "tries and next try, and why one ended after its tries. Read-only." + "tries and next try, and why one ended after its tries. Each state is read in one page; a state past it is answered capped, naming the " + + "keys read and the state's size (H-A-293). Read-only." // PublishAgainDescription is builder_publish_again's description. const PublishAgainDescription = "Publish reserved versions again (design 47 G section 588), which builds nothing: with module and version, that reserved " + diff --git a/modules/mesh-delivery-builder/internal/versions/republish_test.go b/modules/mesh-delivery-builder/internal/versions/republish_test.go index 6966e2a74..51d1256a4 100644 --- a/modules/mesh-delivery-builder/internal/versions/republish_test.go +++ b/modules/mesh-delivery-builder/internal/versions/republish_test.go @@ -430,16 +430,27 @@ func TestAHoldersQueueSendsLiveFirstInPasses(t *testing.T) { } }) }) - t.Run("a build a newer reservation or release supersedes is retired at the pass and at the start", func(t *testing.T) { + t.Run("a build a newer patch or release of its series supersedes is retired at the pass and at the start, never by a newer series", func(t *testing.T) { b := newRepublishBed(t) v := b.reserve(t, "m") + b.holders = map[string][]string{SeatArtifacts: {"oci"}} if _, err := b.Reserved.Reserve(ctx, "m", "0.2", []string{"m:prog"}, Source{Fingerprint: "src2:m"}); err != nil { t.Fatal(err) } b.queueAt(t, v, PublishClassStart, b.now) b.Drain(ctx, "oci") + if len(b.sent) != 1 || len(b.ended) != 0 { + t.Errorf("with a standing 0.2: sent %v, ended %v, want m %s sent, its series not superseded (H-A-293)", b.sent, b.ended, v.Version) + } + rel := &reserveMem{mem: newMem()} + raw, _ := json.Marshal(ReleasedRecord{Kept: []ReleasedBuild{{Fingerprint: "f", Version: "0.2.3"}, {Fingerprint: "g", Version: "0.1.9"}}}) + _, _ = rel.Create(ctx, ReleasedKey("m"), raw) + b.Reserved.Releases = rel + b.sent = nil + b.queueAt(t, v, PublishClassStart, b.now) + b.Drain(ctx, "oci") if len(b.sent) != 0 || fmt.Sprint(b.ended) != "[m "+v.Version+"]" { - t.Errorf("at the pass: sent %v, ended %v, want m %s retired", b.sent, b.ended, v.Version) + t.Errorf("at the pass: sent %v, ended %v, want m %s retired by the release 0.1.9", b.sent, b.ended, v.Version) } b.ended = nil b.queueAt(t, v, PublishClassStart, b.now) @@ -450,13 +461,13 @@ func TestAHoldersQueueSendsLiveFirstInPasses(t *testing.T) { if keys, _ := b.queue.KeysOf(ctx, QueueFilter); len(keys) != 0 || fmt.Sprint(b.ended) != "[m "+v.Version+"]" { t.Errorf("at the start: keys %v, ended %v, want m %s retired", keys, b.ended, v.Version) } - rel := &reserveMem{mem: newMem()} - raw, _ := json.Marshal(ReleasedRecord{Kept: []ReleasedBuild{{Fingerprint: "f", Version: "0.1.9"}}}) - _, _ = rel.Create(ctx, ReleasedKey("n"), raw) - r := &Reserver{Releases: rel} - got, err := r.Retired(ctx, []ReservedVersion{{Module: "n", Version: "0.1.4", BuiltAt: b.now}, {Module: "p", Version: "0.1.12", BuiltAt: b.now}}) - if err != nil || len(got) != 1 || !strings.Contains(got["n 0.1.4"], "release 0.1.9") { - t.Errorf("retired %v (%v), want n 0.1.4 by the release 0.1.9 and p 0.1.12 kept", got, err) + only := &reserveMem{mem: newMem()} + raw, _ = json.Marshal(ReleasedRecord{Kept: []ReleasedBuild{{Fingerprint: "f", Version: "0.2.0"}}}) + _, _ = only.Create(ctx, ReleasedKey("n"), raw) + got, err := (&Reserver{Releases: only}).Retired(ctx, []ReservedVersion{{Module: "n", Version: "0.1.4", BuiltAt: b.now}, {Module: "n", Version: "0.1.6"}, + {Module: "n", Version: "0.3.0", BuiltAt: b.now}, {Module: "p", Version: "0.1.2", BuiltAt: b.now}}) + if err != nil || len(got) != 1 || !strings.Contains(got["n 0.1.4"], "reservation 0.1.6") { + t.Errorf("retired %v (%v), want only n 0.1.4, by the newer patch 0.1.6; n 0.3.0 and p 0.1.2 kept, a release of 0.2 retiring neither", got, err) } }) t.Run("a version taken over is not sent, and its item ended", func(t *testing.T) { @@ -491,7 +502,7 @@ func TestThePublishQueueToolAnswersAtAnySize(t *testing.T) { vals[QueueKey(holder, q.Module, q.Version)], _ = json.Marshal(q) } gets := 0 - queue := Reader{All: func() (map[string]json.RawMessage, error) { return vals, nil }, + queue := Reader{All: func() (StatePage, error) { return StatePage{Values: vals}, nil }, Keys: func() ([]string, error) { return slices.Collect(maps.Keys(vals)), nil }, Get: func(k string) (json.RawMessage, bool, error) { gets++; v, ok := vals[k]; return v, ok, nil }} none := Reader{Keys: func() ([]string, error) { return nil, nil }} @@ -512,7 +523,15 @@ func TestThePublishQueueToolAnswersAtAnySize(t *testing.T) { if err != nil || len(one.Queue) != 1 || one.Queue[0].Class != PublishClassLive || one.Total != 2000 { t.Errorf("one module's answer %+v (%v), want its item with its class", one.Queue, err) } - queue.All = func() (map[string]json.RawMessage, error) { return nil, errors.ErrUnsupported } + if a.Capped || len(a.Read) != 0 { + t.Errorf("a state one page holds is answered capped: %+v", a.Read) + } + queue.All = func() (StatePage, error) { return StatePage{Values: vals, More: true, Keys: 5000, Bytes: 3 << 20}, nil } + if c, err := AnswerQueue(queue, none, none, "", ""); err != nil || !c.Capped || len(c.Read) != 1 || c.Read[0].State != PublishQueueState || + c.Read[0].KeysRead != 2000 || c.Read[0].StateKeys != 5000 || c.Read[0].StateBytes != 3<<20 || c.Total != 2000 { + t.Errorf("a state past its one page answered capped %v, read %+v (%v): want capped, 2000 keys read of 5000, its size said (H-A-293)", c.Capped, c.Read, err) + } + queue.All = func() (StatePage, error) { return StatePage{}, errors.ErrUnsupported } if b, err := AnswerQueue(queue, none, none, "", ""); err != nil || b.Total != 2000 || gets != 2000 { t.Errorf("a reader with no bulk read answered %d (%v) with %d single reads, want every key read one by one", b.Total, err, gets) } diff --git a/modules/mesh-delivery-builder/internal/versions/tools.go b/modules/mesh-delivery-builder/internal/versions/tools.go index e5a7740d4..8a562cbf5 100644 --- a/modules/mesh-delivery-builder/internal/versions/tools.go +++ b/modules/mesh-delivery-builder/internal/versions/tools.go @@ -21,26 +21,49 @@ const ( OldFormClosedEvent = "version.old-form-closed" ) -// Reader reads the state as the tools bundle may: every key, and one; and, where All is set, every key's value in one bulk read -// (the state's pages), which a state of thousands of keys needs to be answered at all (design 47 H H-A-290 (5)). +// Reader reads the state as the tools bundle may: every key, and one; and, where All is set, the state's values in one page, +// one request for many (design 47 H H-A-290 (5), H-A-292). type Reader struct { Keys func() ([]string, error) Get func(key string) (json.RawMessage, bool, error) - // All is every key's value read in bulk; nil, or an error satisfying errors.Is(err, errors.ErrUnsupported), reads key by key. - All func() (map[string]json.RawMessage, error) + // All is one page of the state's values; nil, or an error satisfying errors.Is(err, errors.ErrUnsupported), reads key by key. + All func() (StatePage, error) } -// values is every key's value: in bulk where the reader can, else key by key. -func (r Reader) values() (map[string]json.RawMessage, error) { +// StatePage is one page of a state (H-A-292, H-A-293): its values, and, when the state holds more than the page, More with the +// state's size as its info says it (Keys, Bytes; InfoWords when the info cannot be read). No second page is asked. +type StatePage struct { + Values map[string]json.RawMessage + More bool + Keys, Bytes uint64 + InfoWords string +} + +// StateCapped is what an answer says of a state it read only the first page of (H-A-293): how many keys it read, and the +// state's size from its info. +type StateCapped struct { + State string `json:"state"` + KeysRead int `json:"keys-read"` + StateKeys uint64 `json:"state-keys"` + StateBytes uint64 `json:"state-bytes"` + InfoWords string `json:"info,omitempty"` +} + +// values is the state's values: one page where the reader can (capped says what a page past the state's end left out), else +// key by key. +func (r Reader) values(state string) (map[string]json.RawMessage, *StateCapped, error) { if r.All != nil { - out, err := r.All() + pg, err := r.All() if !errors.Is(err, errors.ErrUnsupported) { - return out, err + if err != nil || !pg.More { + return pg.Values, nil, err + } + return pg.Values, &StateCapped{State: state, KeysRead: len(pg.Values), StateKeys: pg.Keys, StateBytes: pg.Bytes, InfoWords: pg.InfoWords}, nil } } keys, err := r.Keys() if err != nil { - return nil, err + return nil, nil, err } out := make(map[string]json.RawMessage, len(keys)) for _, k := range keys { @@ -50,7 +73,7 @@ func (r Reader) values() (map[string]json.RawMessage, error) { } out[k] = raw } - return out, nil + return out, nil, nil } // readStore is a Store over a Reader that refuses every write: the tools bundle only reads versions.