diff --git a/modules/artifact-store-tools/README.md b/modules/artifact-store-tools/README.md index 3a3728e60..c800d6d89 100644 --- a/modules/artifact-store-tools/README.md +++ b/modules/artifact-store-tools/README.md @@ -123,7 +123,9 @@ seat refuses clears it (`sweepseats.go`). ### One pending pass A finished build or a changed want marks the one pending pass (design 47 H, H-A-250); a pass running when one is marked -runs once more when it ends, however many events marked it. A pass reads the builder's builds in pages of the state, one -answer for many keys, never one get per key, and sweeps and observes S25 from that one judgement. A list the registry +runs once more when it ends, however many events marked it. The builder's builds are a followed view: the watch's first view +at start, then its updates, kept in memory (H-A-255), each build's end read once from its key's history as it is taken +in. A pass reads nothing of them on the bus, judges nothing before the first view is in, and sweeps and observes S25 +from that one judgement. A list the registry seats cannot answer ends the pass with the finding `artifact-list-unread`, written once; nothing asks the list again until the next event, and a pass that reads it clears the finding (`holder.go`, `sweepseats.go`). diff --git a/modules/artifact-store-tools/cmd/artifact-store-tools/door.go b/modules/artifact-store-tools/cmd/artifact-store-tools/door.go index 9e5c21578..ee064d47c 100644 --- a/modules/artifact-store-tools/cmd/artifact-store-tools/door.go +++ b/modules/artifact-store-tools/cmd/artifact-store-tools/door.go @@ -91,12 +91,6 @@ func (b runtimeBus) Get(state, key string) ([]byte, bool, error) { } return e.Value, true, nil } -func (b runtimeBus) Page(state, after string) (stdio.StatePage, error) { - if b.followed(state) { - return stdio.StatePage{}, fmt.Errorf("the state %s is followed, not read in pages", state) - } - return stdio.State(state).Page(after, views.PageBytes, views.PageBudget) -} func (runtimeBus) History(state, key string) ([]stdio.StateRevision, error) { return stdio.State(state).History(key) } @@ -122,8 +116,9 @@ func (runtimeBus) Publish(subject string, body any) error { return stdio.SeatPublish(subject, body, "") } -// follow watches the builder's done keys and the wants: a build's end or a want changed marks the one pending pass -// (design 47 H, H-A-250 (1)), which sweeps and observes from one read. Event-driven: nothing reads them on a timer. +// follow watches the builder's done keys and the wants: the builds are a followed view, the watch's first view then its +// updates kept in memory (design 47 H, H-A-255); a build's end or a want changed marks the one pending pass (H-A-250 (1)), +// which sweeps and observes from the views. Event-driven: nothing reads them on a timer. func follow(h *Holder, v *views.Reader) { <-stdio.Served() watch := func(state string, handle func(stdio.StateChange) error) { @@ -137,6 +132,7 @@ func follow(h *Holder, v *views.Reader) { } } watch(BuildsRead, func(c stdio.StateChange) error { + h.Heard(c) if c.Current || c.Op != stdio.OpPut { return nil } @@ -148,6 +144,8 @@ func follow(h *Holder, v *views.Reader) { } return nil }) + // The watch hands every current value over before it returns: the first view is in. + h.TakenIn() // The wants are the assigner's: the first view then its events (ADR 0615). What Watch hands over before it returns // is the first view, not a change. var current atomic.Bool diff --git a/modules/artifact-store-tools/cmd/artifact-store-tools/holder.go b/modules/artifact-store-tools/cmd/artifact-store-tools/holder.go index fa44558ce..7457c9463 100644 --- a/modules/artifact-store-tools/cmd/artifact-store-tools/holder.go +++ b/modules/artifact-store-tools/cmd/artifact-store-tools/holder.go @@ -63,9 +63,6 @@ const ( type Bus interface { Keys(state string) ([]string, error) Get(state, key string) ([]byte, bool, error) - // Page is the values after a key in key order, many in one answer (stdio.KeptState.Page): how a pass reads the - // builder's builds, never one get per key (design 47 H, H-A-250 (2)). - Page(state, after string) (stdio.StatePage, error) History(state, key string) ([]stdio.StateRevision, error) // LogSince is every entry of a read log from a time on (stdio.Log(read).Since). LogSince(read string, at time.Time) ([]stdio.LogEntry, error) @@ -105,6 +102,10 @@ type Holder struct { wantedBy string fmu sync.Mutex // one pass of Observe at a time ended map[string]time.Time // a done key's end, read once from its history + // done is the followed view of the builder's builds (design 47 H, H-A-255): the done builds by key, taken in from the + // watch's first view and its updates (Heard); taken says the first view is in. A pass judges from it, reading nothing. + done map[string]Build + taken bool } // Build is one done build of a module, as its key holds it, with when it ended. @@ -116,42 +117,64 @@ type Build struct { Manifest string } -// Builds is every done build the builder's keys hold, each with its end: the state read in pages, one answer for many -// keys, never one get per key (design 47 H, H-A-250 (2)). A build's end is when the bus wrote its done value, as the -// page says it; only a runtime that does not say it is asked the key's history. +// Builds is every done build of the followed view, by key, each with its end: read from memory, never from the bus +// (design 47 H, H-A-250 (2), H-A-255). Before the view's first view is taken in there is none to judge from, and it says so, +// so no pass sweeps against an empty keep set. func (h *Holder) Builds() ([]Build, error) { - var out []Build - after := "" - for { - page, err := h.Bus.Page(BuildsRead, after) - if err != nil { - return nil, fmt.Errorf("the builder's builds: %w", err) - } - for _, e := range page.Entries { - module, fp, err := builds.ParseBuilderBuildKey(e.Key) - if err != nil { - continue - } - b, err := builds.DecodeBuilderBuild(e.Value) - if err != nil || b.State != builds.BuildDone { - continue - } - at := e.At - if at.IsZero() { - if at, err = h.endOf(e.Key); err != nil { - return nil, err - } - } - out = append(out, Build{Module: module, FP: fp, Key: e.Key, At: at, Artifacts: b.Artifacts, Manifest: b.Manifest}) - } - if page.Next == "" { - return out, nil - } - if page.Next <= after { - return nil, fmt.Errorf("the builder's builds: a page after %q answered %q to ask after, which is no further", after, page.Next) - } - after = page.Next + h.mu.Lock() + defer h.mu.Unlock() + if !h.taken { + return nil, errors.New("the builder's builds are not taken in yet: their first view has not come") } + out := make([]Build, 0, len(h.done)) + for _, b := range h.done { + out = append(out, b) + } + sort.Slice(out, func(a, b int) bool { return out[a].Key < out[b].Key }) + return out, nil +} + +// Heard takes one change of the builder's builds into the followed view: a done value is kept with its end (read once from +// the key's history, again when the key is written done anew), anything else drops the key. +func (h *Holder) Heard(c stdio.StateChange) { + module, fp, err := builds.ParseBuilderBuildKey(c.Key) + if err != nil { + return + } + var b builds.BuilderBuild + if c.Op == stdio.OpPut { + b, err = builds.DecodeBuilderBuild(c.Value) + } + if c.Op != stdio.OpPut || err != nil || b.State != builds.BuildDone { + h.mu.Lock() + delete(h.done, c.Key) + delete(h.ended, c.Key) + h.mu.Unlock() + return + } + if !c.Current { + h.mu.Lock() + delete(h.ended, c.Key) // written done anew: its end is this write's + h.mu.Unlock() + } + at, err := h.endOf(c.Key) + if err != nil { + h.logf("%s is left out of the keep set, its end unread: %v", c.Key, err) + return + } + h.mu.Lock() + if h.done == nil { + h.done = map[string]Build{} + } + h.done[c.Key] = Build{Module: module, FP: fp, Key: c.Key, At: at, Artifacts: b.Artifacts, Manifest: b.Manifest} + h.mu.Unlock() +} + +// TakenIn says the view's first view is in: every current value was Heard. +func (h *Holder) TakenIn() { + h.mu.Lock() + h.taken = true + h.mu.Unlock() } // endOf is when a done key was last written done, read once from its history: a done key is not written again. diff --git a/modules/artifact-store-tools/cmd/artifact-store-tools/holder_integration_test.go b/modules/artifact-store-tools/cmd/artifact-store-tools/holder_integration_test.go index 6cb4f06d3..7c728f0d8 100644 --- a/modules/artifact-store-tools/cmd/artifact-store-tools/holder_integration_test.go +++ b/modules/artifact-store-tools/cmd/artifact-store-tools/holder_integration_test.go @@ -11,7 +11,6 @@ package main import ( "context" "encoding/json" - "sort" "strings" "sync" "testing" @@ -79,31 +78,6 @@ func (b realBus) Get(state, key string) ([]byte, bool, error) { return e.Value(), true, nil } -// Page answers every value after a key in one page, each with when the bus wrote it, as the runtime's page does. -func (b realBus) Page(state, after string) (stdio.StatePage, error) { - keys, err := b.Keys(state) - if err != nil { - return stdio.StatePage{}, err - } - kv, err := b.kv(state) - if err != nil { - return stdio.StatePage{}, err - } - sort.Strings(keys) - p := stdio.StatePage{Entries: []stdio.StateEntry{}} - for _, k := range keys { - if k <= after { - continue - } - e, err := kv.Get(b.ctx, k) - if err != nil { - return stdio.StatePage{}, err - } - p.Entries = append(p.Entries, stdio.StateEntry{Key: k, Value: e.Value(), Revision: e.Revision(), At: e.Created()}) - } - return p, nil -} - func (b realBus) History(state, key string) ([]stdio.StateRevision, error) { kv, err := b.kv(state) if err != nil { @@ -226,6 +200,14 @@ func TestTheSeatSweepsAtABuildsEndOnARealBusWithNoLease(t *testing.T) { } h := &Holder{Bus: findingsAt{realBus: bus}, Findings: findingsThrough{findingsAt{realBus: bus}}, Door: mem.Door, Listed: mem.Listed, Seats: ask, Now: time.Now, Logf: t.Logf} h.Acts = meshhandactsclient.Recorder{Ask: ask} + // The followed view of the builds: the six on the bus handed over as the watch's first view does. + if keys, err := bus.Keys(BuildsRead); err == nil { + for _, k := range keys { + raw, _, _ := bus.Get(BuildsRead, k) + h.Heard(stdio.StateChange{State: BuildsRead, Key: k, Op: stdio.OpPut, Value: raw, Current: true}) + } + } + h.TakenIn() // A build's end heard from the builds bucket, as follow hands it over. kv, _ := bus.kv(BuildsRead) @@ -241,6 +223,7 @@ func TestTheSeatSweepsAtABuildsEndOnARealBusWithNoLease(t *testing.T) { } select { case e := <-w.Updates(): + h.Heard(stdio.StateChange{State: BuildsRead, Key: e.Key(), Op: stdio.OpPut, Value: e.Value()}) h.Pass(ctx, "after the build "+e.Key()) case <-ctx.Done(): t.Fatal("the build's end was not heard") diff --git a/modules/artifact-store-tools/cmd/artifact-store-tools/holder_test.go b/modules/artifact-store-tools/cmd/artifact-store-tools/holder_test.go index 86b56a021..80cc256af 100644 --- a/modules/artifact-store-tools/cmd/artifact-store-tools/holder_test.go +++ b/modules/artifact-store-tools/cmd/artifact-store-tools/holder_test.go @@ -34,7 +34,7 @@ type memBus struct { history map[string][]stdio.StateRevision log []stdio.LogEntry published []string - // calls counts each read by " ": a pass reads the builds in one page, never one get per key. + // calls counts each read by " ": a pass reads nothing of the builds, judged from the followed view. calls map[string]int } @@ -58,6 +58,7 @@ func (b *memBus) LogSince(read string, at time.Time) ([]stdio.LogEntry, error) { func (b *memBus) Keys(state string) ([]string, error) { b.mu.Lock() defer b.mu.Unlock() + b.count("keys", state) var out []string for k := range b.states[state] { out = append(out, k) @@ -74,29 +75,6 @@ func (b *memBus) Get(state, key string) ([]byte, bool, error) { return v, ok, nil } -// Page answers every value after a key in one page, each written when its last history revision says. -func (b *memBus) Page(state, after string) (stdio.StatePage, error) { - b.mu.Lock() - defer b.mu.Unlock() - b.count("page", state) - p := stdio.StatePage{Entries: []stdio.StateEntry{}} - var keys []string - for k := range b.states[state] { - if k > after { - keys = append(keys, k) - } - } - sort.Strings(keys) - for _, k := range keys { - e := stdio.StateEntry{Key: k, Value: b.states[state][k]} - if revs := b.history[state+" "+k]; len(revs) > 0 { - e.At = revs[len(revs)-1].At - } - p.Entries = append(p.Entries, e) - } - return p, nil -} - func (b *memBus) History(state, key string) ([]stdio.StateRevision, error) { b.mu.Lock() defer b.mu.Unlock() @@ -265,7 +243,7 @@ func aMesh(t *testing.T) (*Holder, *memBus, *memDoor, *[]string) { want, _ := json.Marshal(meshwants.WantsByWantNodeModuleValue{V: 1, Fp: fp(1), Self: true}) _ = b.Put(WantsRead, "want.ace.letta", json.RawMessage(want)) var acts []string - h := &Holder{Bus: b, Findings: findingsThrough{b}, Door: func() (Door, error) { return door, nil }, Seats: door.seat, + h := &Holder{Logf: t.Logf, Bus: b, Findings: findingsThrough{b}, Door: func() (Door, error) { return door, nil }, Seats: door.seat, Listed: func(context.Context) (*View, error) { return &View{L: listing}, nil }, Now: func() time.Time { return day.Add(48 * time.Hour) }, Acts: meshhandactsclient.Recorder{Ask: func(key string, body any) (json.RawMessage, error) { @@ -273,9 +251,27 @@ func aMesh(t *testing.T) (*Holder, *memBus, *memDoor, *[]string) { acts = append(acts, key+" "+string(raw)) return json.RawMessage(`{}`), nil }}} + takeIn(h, b) return h, b, door, &acts } +// takeIn hands the builds over as the watch's first view does, then forgets the reads that took: a pass's are counted. +func takeIn(h *Holder, b *memBus) { + keys, _ := b.Keys(BuildsRead) + for _, k := range keys { + raw, _, _ := b.Get(BuildsRead, k) + h.Heard(stdio.StateChange{State: BuildsRead, Key: k, Op: stdio.OpPut, Value: raw, Current: true}) + } + h.TakenIn() + b.calls = nil +} + +// putBuild writes a build's key and hands it over as the watch's update does. +func putBuild(h *Holder, b *memBus, key string, raw []byte) { + _ = b.Put(BuildsRead, key, json.RawMessage(raw)) + h.Heard(stdio.StateChange{State: BuildsRead, Key: key, Op: stdio.OpPut, Value: raw}) +} + // ADR 0571 clause 2, 1126/A4/AC2: the keep set from the store's listing, the wants and the five newest builds; with no // lease key, a build's end and a confirmed collect let go of the eligible within the bound, leave an index for a // confirmed collect, record each deletion as a hand act and say artifacts-gone. @@ -348,7 +344,7 @@ func TestTheSeatKeepsTheFiveNewestAndTheWantedAndSweepsWithNoLease(t *testing.T) raw, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildDone, Manifest: strings.Repeat("d", 64), Artifacts: []builds.BuiltArtifact{ {Name: "image", Kind: "image", Digest: digestOf(imageRef("letta", n)), Reference: artifactref.SeatReference(reg.Seat, "letta", "image", digestOf(imageRef("letta", n)))}, {Name: "files", Kind: "archive", Digest: digestOf(archiveRef("letta", n)), Reference: artifactref.SeatReference(ociseat.Seat, "letta", "files", digestOf(archiveRef("letta", n)))}}}) - _ = b.Put(BuildsRead, key, json.RawMessage(raw)) + putBuild(h, b, key, raw) } } if c.refuse != "" { @@ -456,6 +452,7 @@ func TestAWantedImageTheStoreLacksIsAFindingAndImagesNamesWantsAndHaves(t *testi view.L.Repos[repo].Revisions[digest] = true // A restart: a holder made afresh holds no finding of its own, and clears what the writer holds (design 47 G A-32, A-38). h = &Holder{Bus: h.Bus, Findings: h.Findings, Door: h.Door, Listed: h.Listed, Seats: h.Seats, Now: h.Now, Acts: h.Acts, Logf: h.Logf} + takeIn(h, b) if err := h.Observe(context.Background()); err != nil { t.Fatal(err) } @@ -556,16 +553,24 @@ func TestOnePendingPassReadsTheBuildsOnceAndEndsOnAnUnreadList(t *testing.T) { t.Fatalf("after the passes running is %v and wanted %v", h.running, h.wanted) } }) - t.Run("a pass reads the builds in one page", func(t *testing.T) { + t.Run("a pass reads nothing of the builds", func(t *testing.T) { h, b, door, _ := aMesh(t) h.Pass(context.Background(), "after the build letta.x") - if b.calls["page "+BuildsRead] != 1 || b.calls["get "+BuildsRead] != 0 || b.calls["history "+BuildsRead] != 0 { + if n := b.calls["keys "+BuildsRead] + b.calls["get "+BuildsRead] + b.calls["history "+BuildsRead]; n != 0 { t.Fatalf("a pass read the builds by %v", b.calls) } if !slices.Contains(door.asked, "let go "+imageRef("letta", 2)) { t.Fatalf("the pass swept nothing: %v", door.asked) } }) + t.Run("no pass judges before the first view", func(t *testing.T) { + h, _, door, _ := aMesh(t) + h.taken = false + h.Pass(context.Background(), "after the build letta.x") + if len(door.asked) != 0 { + t.Fatalf("a pass before the builds were taken in asked %v", door.asked) + } + }) t.Run("a list unread ends the pass with its finding once", func(t *testing.T) { h, b, door, _ := aMesh(t) writes, listed, inner := 0, 0, h.Listed