Design 47 H1: the artifact sweep judges from a followed view of the builds, not pages (H-A-255)
A page read costs the whole bucket per page, so the builder's builds are now a followed view: the watch's first view at start, then its updates, kept in memory with each build's end read once from its history. A pass reads nothing of the builds on the bus and judges nothing before the first view is in, so it never sweeps against an empty keep set. Builds: design 47 H §H-A-250 (2), §H-A-255 (1)
This commit is contained in:
@@ -123,7 +123,9 @@ seat refuses clears it (`sweepseats.go`).
|
||||
### One pending pass
|
||||
|
||||
A finished build or a changed want marks the one pending pass (design 47 H, H-A-250); a pass running when one is marked
|
||||
runs once more when it ends, however many events marked it. A pass reads the builder's builds in pages of the state, one
|
||||
answer for many keys, never one get per key, and sweeps and observes S25 from that one judgement. A list the registry
|
||||
runs once more when it ends, however many events marked it. The builder's builds are a followed view: the watch's first view
|
||||
at start, then its updates, kept in memory (H-A-255), each build's end read once from its key's history as it is taken
|
||||
in. A pass reads nothing of them on the bus, judges nothing before the first view is in, and sweeps and observes S25
|
||||
from that one judgement. A list the registry
|
||||
seats cannot answer ends the pass with the finding `artifact-list-unread`, written once; nothing asks the list again
|
||||
until the next event, and a pass that reads it clears the finding (`holder.go`, `sweepseats.go`).
|
||||
|
||||
@@ -91,12 +91,6 @@ func (b runtimeBus) Get(state, key string) ([]byte, bool, error) {
|
||||
}
|
||||
return e.Value, true, nil
|
||||
}
|
||||
func (b runtimeBus) Page(state, after string) (stdio.StatePage, error) {
|
||||
if b.followed(state) {
|
||||
return stdio.StatePage{}, fmt.Errorf("the state %s is followed, not read in pages", state)
|
||||
}
|
||||
return stdio.State(state).Page(after, views.PageBytes, views.PageBudget)
|
||||
}
|
||||
func (runtimeBus) History(state, key string) ([]stdio.StateRevision, error) {
|
||||
return stdio.State(state).History(key)
|
||||
}
|
||||
@@ -122,8 +116,9 @@ func (runtimeBus) Publish(subject string, body any) error {
|
||||
return stdio.SeatPublish(subject, body, "")
|
||||
}
|
||||
|
||||
// follow watches the builder's done keys and the wants: a build's end or a want changed marks the one pending pass
|
||||
// (design 47 H, H-A-250 (1)), which sweeps and observes from one read. Event-driven: nothing reads them on a timer.
|
||||
// follow watches the builder's done keys and the wants: the builds are a followed view, the watch's first view then its
|
||||
// updates kept in memory (design 47 H, H-A-255); a build's end or a want changed marks the one pending pass (H-A-250 (1)),
|
||||
// which sweeps and observes from the views. Event-driven: nothing reads them on a timer.
|
||||
func follow(h *Holder, v *views.Reader) {
|
||||
<-stdio.Served()
|
||||
watch := func(state string, handle func(stdio.StateChange) error) {
|
||||
@@ -137,6 +132,7 @@ func follow(h *Holder, v *views.Reader) {
|
||||
}
|
||||
}
|
||||
watch(BuildsRead, func(c stdio.StateChange) error {
|
||||
h.Heard(c)
|
||||
if c.Current || c.Op != stdio.OpPut {
|
||||
return nil
|
||||
}
|
||||
@@ -148,6 +144,8 @@ func follow(h *Holder, v *views.Reader) {
|
||||
}
|
||||
return nil
|
||||
})
|
||||
// The watch hands every current value over before it returns: the first view is in.
|
||||
h.TakenIn()
|
||||
// The wants are the assigner's: the first view then its events (ADR 0615). What Watch hands over before it returns
|
||||
// is the first view, not a change.
|
||||
var current atomic.Bool
|
||||
|
||||
@@ -63,9 +63,6 @@ const (
|
||||
type Bus interface {
|
||||
Keys(state string) ([]string, error)
|
||||
Get(state, key string) ([]byte, bool, error)
|
||||
// Page is the values after a key in key order, many in one answer (stdio.KeptState.Page): how a pass reads the
|
||||
// builder's builds, never one get per key (design 47 H, H-A-250 (2)).
|
||||
Page(state, after string) (stdio.StatePage, error)
|
||||
History(state, key string) ([]stdio.StateRevision, error)
|
||||
// LogSince is every entry of a read log from a time on (stdio.Log(read).Since).
|
||||
LogSince(read string, at time.Time) ([]stdio.LogEntry, error)
|
||||
@@ -105,6 +102,10 @@ type Holder struct {
|
||||
wantedBy string
|
||||
fmu sync.Mutex // one pass of Observe at a time
|
||||
ended map[string]time.Time // a done key's end, read once from its history
|
||||
// done is the followed view of the builder's builds (design 47 H, H-A-255): the done builds by key, taken in from the
|
||||
// watch's first view and its updates (Heard); taken says the first view is in. A pass judges from it, reading nothing.
|
||||
done map[string]Build
|
||||
taken bool
|
||||
}
|
||||
|
||||
// Build is one done build of a module, as its key holds it, with when it ended.
|
||||
@@ -116,42 +117,64 @@ type Build struct {
|
||||
Manifest string
|
||||
}
|
||||
|
||||
// Builds is every done build the builder's keys hold, each with its end: the state read in pages, one answer for many
|
||||
// keys, never one get per key (design 47 H, H-A-250 (2)). A build's end is when the bus wrote its done value, as the
|
||||
// page says it; only a runtime that does not say it is asked the key's history.
|
||||
// Builds is every done build of the followed view, by key, each with its end: read from memory, never from the bus
|
||||
// (design 47 H, H-A-250 (2), H-A-255). Before the view's first view is taken in there is none to judge from, and it says so,
|
||||
// so no pass sweeps against an empty keep set.
|
||||
func (h *Holder) Builds() ([]Build, error) {
|
||||
var out []Build
|
||||
after := ""
|
||||
for {
|
||||
page, err := h.Bus.Page(BuildsRead, after)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the builder's builds: %w", err)
|
||||
}
|
||||
for _, e := range page.Entries {
|
||||
module, fp, err := builds.ParseBuilderBuildKey(e.Key)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
b, err := builds.DecodeBuilderBuild(e.Value)
|
||||
if err != nil || b.State != builds.BuildDone {
|
||||
continue
|
||||
}
|
||||
at := e.At
|
||||
if at.IsZero() {
|
||||
if at, err = h.endOf(e.Key); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
out = append(out, Build{Module: module, FP: fp, Key: e.Key, At: at, Artifacts: b.Artifacts, Manifest: b.Manifest})
|
||||
}
|
||||
if page.Next == "" {
|
||||
return out, nil
|
||||
}
|
||||
if page.Next <= after {
|
||||
return nil, fmt.Errorf("the builder's builds: a page after %q answered %q to ask after, which is no further", after, page.Next)
|
||||
}
|
||||
after = page.Next
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
if !h.taken {
|
||||
return nil, errors.New("the builder's builds are not taken in yet: their first view has not come")
|
||||
}
|
||||
out := make([]Build, 0, len(h.done))
|
||||
for _, b := range h.done {
|
||||
out = append(out, b)
|
||||
}
|
||||
sort.Slice(out, func(a, b int) bool { return out[a].Key < out[b].Key })
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Heard takes one change of the builder's builds into the followed view: a done value is kept with its end (read once from
|
||||
// the key's history, again when the key is written done anew), anything else drops the key.
|
||||
func (h *Holder) Heard(c stdio.StateChange) {
|
||||
module, fp, err := builds.ParseBuilderBuildKey(c.Key)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
var b builds.BuilderBuild
|
||||
if c.Op == stdio.OpPut {
|
||||
b, err = builds.DecodeBuilderBuild(c.Value)
|
||||
}
|
||||
if c.Op != stdio.OpPut || err != nil || b.State != builds.BuildDone {
|
||||
h.mu.Lock()
|
||||
delete(h.done, c.Key)
|
||||
delete(h.ended, c.Key)
|
||||
h.mu.Unlock()
|
||||
return
|
||||
}
|
||||
if !c.Current {
|
||||
h.mu.Lock()
|
||||
delete(h.ended, c.Key) // written done anew: its end is this write's
|
||||
h.mu.Unlock()
|
||||
}
|
||||
at, err := h.endOf(c.Key)
|
||||
if err != nil {
|
||||
h.logf("%s is left out of the keep set, its end unread: %v", c.Key, err)
|
||||
return
|
||||
}
|
||||
h.mu.Lock()
|
||||
if h.done == nil {
|
||||
h.done = map[string]Build{}
|
||||
}
|
||||
h.done[c.Key] = Build{Module: module, FP: fp, Key: c.Key, At: at, Artifacts: b.Artifacts, Manifest: b.Manifest}
|
||||
h.mu.Unlock()
|
||||
}
|
||||
|
||||
// TakenIn says the view's first view is in: every current value was Heard.
|
||||
func (h *Holder) TakenIn() {
|
||||
h.mu.Lock()
|
||||
h.taken = true
|
||||
h.mu.Unlock()
|
||||
}
|
||||
|
||||
// endOf is when a done key was last written done, read once from its history: a done key is not written again.
|
||||
|
||||
@@ -11,7 +11,6 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
@@ -79,31 +78,6 @@ func (b realBus) Get(state, key string) ([]byte, bool, error) {
|
||||
return e.Value(), true, nil
|
||||
}
|
||||
|
||||
// Page answers every value after a key in one page, each with when the bus wrote it, as the runtime's page does.
|
||||
func (b realBus) Page(state, after string) (stdio.StatePage, error) {
|
||||
keys, err := b.Keys(state)
|
||||
if err != nil {
|
||||
return stdio.StatePage{}, err
|
||||
}
|
||||
kv, err := b.kv(state)
|
||||
if err != nil {
|
||||
return stdio.StatePage{}, err
|
||||
}
|
||||
sort.Strings(keys)
|
||||
p := stdio.StatePage{Entries: []stdio.StateEntry{}}
|
||||
for _, k := range keys {
|
||||
if k <= after {
|
||||
continue
|
||||
}
|
||||
e, err := kv.Get(b.ctx, k)
|
||||
if err != nil {
|
||||
return stdio.StatePage{}, err
|
||||
}
|
||||
p.Entries = append(p.Entries, stdio.StateEntry{Key: k, Value: e.Value(), Revision: e.Revision(), At: e.Created()})
|
||||
}
|
||||
return p, nil
|
||||
}
|
||||
|
||||
func (b realBus) History(state, key string) ([]stdio.StateRevision, error) {
|
||||
kv, err := b.kv(state)
|
||||
if err != nil {
|
||||
@@ -226,6 +200,14 @@ func TestTheSeatSweepsAtABuildsEndOnARealBusWithNoLease(t *testing.T) {
|
||||
}
|
||||
h := &Holder{Bus: findingsAt{realBus: bus}, Findings: findingsThrough{findingsAt{realBus: bus}}, Door: mem.Door, Listed: mem.Listed, Seats: ask, Now: time.Now, Logf: t.Logf}
|
||||
h.Acts = meshhandactsclient.Recorder{Ask: ask}
|
||||
// The followed view of the builds: the six on the bus handed over as the watch's first view does.
|
||||
if keys, err := bus.Keys(BuildsRead); err == nil {
|
||||
for _, k := range keys {
|
||||
raw, _, _ := bus.Get(BuildsRead, k)
|
||||
h.Heard(stdio.StateChange{State: BuildsRead, Key: k, Op: stdio.OpPut, Value: raw, Current: true})
|
||||
}
|
||||
}
|
||||
h.TakenIn()
|
||||
|
||||
// A build's end heard from the builds bucket, as follow hands it over.
|
||||
kv, _ := bus.kv(BuildsRead)
|
||||
@@ -241,6 +223,7 @@ func TestTheSeatSweepsAtABuildsEndOnARealBusWithNoLease(t *testing.T) {
|
||||
}
|
||||
select {
|
||||
case e := <-w.Updates():
|
||||
h.Heard(stdio.StateChange{State: BuildsRead, Key: e.Key(), Op: stdio.OpPut, Value: e.Value()})
|
||||
h.Pass(ctx, "after the build "+e.Key())
|
||||
case <-ctx.Done():
|
||||
t.Fatal("the build's end was not heard")
|
||||
|
||||
@@ -34,7 +34,7 @@ type memBus struct {
|
||||
history map[string][]stdio.StateRevision
|
||||
log []stdio.LogEntry
|
||||
published []string
|
||||
// calls counts each read by "<verb> <state>": a pass reads the builds in one page, never one get per key.
|
||||
// calls counts each read by "<verb> <state>": a pass reads nothing of the builds, judged from the followed view.
|
||||
calls map[string]int
|
||||
}
|
||||
|
||||
@@ -58,6 +58,7 @@ func (b *memBus) LogSince(read string, at time.Time) ([]stdio.LogEntry, error) {
|
||||
func (b *memBus) Keys(state string) ([]string, error) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
b.count("keys", state)
|
||||
var out []string
|
||||
for k := range b.states[state] {
|
||||
out = append(out, k)
|
||||
@@ -74,29 +75,6 @@ func (b *memBus) Get(state, key string) ([]byte, bool, error) {
|
||||
return v, ok, nil
|
||||
}
|
||||
|
||||
// Page answers every value after a key in one page, each written when its last history revision says.
|
||||
func (b *memBus) Page(state, after string) (stdio.StatePage, error) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
b.count("page", state)
|
||||
p := stdio.StatePage{Entries: []stdio.StateEntry{}}
|
||||
var keys []string
|
||||
for k := range b.states[state] {
|
||||
if k > after {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, k := range keys {
|
||||
e := stdio.StateEntry{Key: k, Value: b.states[state][k]}
|
||||
if revs := b.history[state+" "+k]; len(revs) > 0 {
|
||||
e.At = revs[len(revs)-1].At
|
||||
}
|
||||
p.Entries = append(p.Entries, e)
|
||||
}
|
||||
return p, nil
|
||||
}
|
||||
|
||||
func (b *memBus) History(state, key string) ([]stdio.StateRevision, error) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
@@ -265,7 +243,7 @@ func aMesh(t *testing.T) (*Holder, *memBus, *memDoor, *[]string) {
|
||||
want, _ := json.Marshal(meshwants.WantsByWantNodeModuleValue{V: 1, Fp: fp(1), Self: true})
|
||||
_ = b.Put(WantsRead, "want.ace.letta", json.RawMessage(want))
|
||||
var acts []string
|
||||
h := &Holder{Bus: b, Findings: findingsThrough{b}, Door: func() (Door, error) { return door, nil }, Seats: door.seat,
|
||||
h := &Holder{Logf: t.Logf, Bus: b, Findings: findingsThrough{b}, Door: func() (Door, error) { return door, nil }, Seats: door.seat,
|
||||
Listed: func(context.Context) (*View, error) { return &View{L: listing}, nil },
|
||||
Now: func() time.Time { return day.Add(48 * time.Hour) },
|
||||
Acts: meshhandactsclient.Recorder{Ask: func(key string, body any) (json.RawMessage, error) {
|
||||
@@ -273,9 +251,27 @@ func aMesh(t *testing.T) (*Holder, *memBus, *memDoor, *[]string) {
|
||||
acts = append(acts, key+" "+string(raw))
|
||||
return json.RawMessage(`{}`), nil
|
||||
}}}
|
||||
takeIn(h, b)
|
||||
return h, b, door, &acts
|
||||
}
|
||||
|
||||
// takeIn hands the builds over as the watch's first view does, then forgets the reads that took: a pass's are counted.
|
||||
func takeIn(h *Holder, b *memBus) {
|
||||
keys, _ := b.Keys(BuildsRead)
|
||||
for _, k := range keys {
|
||||
raw, _, _ := b.Get(BuildsRead, k)
|
||||
h.Heard(stdio.StateChange{State: BuildsRead, Key: k, Op: stdio.OpPut, Value: raw, Current: true})
|
||||
}
|
||||
h.TakenIn()
|
||||
b.calls = nil
|
||||
}
|
||||
|
||||
// putBuild writes a build's key and hands it over as the watch's update does.
|
||||
func putBuild(h *Holder, b *memBus, key string, raw []byte) {
|
||||
_ = b.Put(BuildsRead, key, json.RawMessage(raw))
|
||||
h.Heard(stdio.StateChange{State: BuildsRead, Key: key, Op: stdio.OpPut, Value: raw})
|
||||
}
|
||||
|
||||
// ADR 0571 clause 2, 1126/A4/AC2: the keep set from the store's listing, the wants and the five newest builds; with no
|
||||
// lease key, a build's end and a confirmed collect let go of the eligible within the bound, leave an index for a
|
||||
// confirmed collect, record each deletion as a hand act and say artifacts-gone.
|
||||
@@ -348,7 +344,7 @@ func TestTheSeatKeepsTheFiveNewestAndTheWantedAndSweepsWithNoLease(t *testing.T)
|
||||
raw, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildDone, Manifest: strings.Repeat("d", 64), Artifacts: []builds.BuiltArtifact{
|
||||
{Name: "image", Kind: "image", Digest: digestOf(imageRef("letta", n)), Reference: artifactref.SeatReference(reg.Seat, "letta", "image", digestOf(imageRef("letta", n)))},
|
||||
{Name: "files", Kind: "archive", Digest: digestOf(archiveRef("letta", n)), Reference: artifactref.SeatReference(ociseat.Seat, "letta", "files", digestOf(archiveRef("letta", n)))}}})
|
||||
_ = b.Put(BuildsRead, key, json.RawMessage(raw))
|
||||
putBuild(h, b, key, raw)
|
||||
}
|
||||
}
|
||||
if c.refuse != "" {
|
||||
@@ -456,6 +452,7 @@ func TestAWantedImageTheStoreLacksIsAFindingAndImagesNamesWantsAndHaves(t *testi
|
||||
view.L.Repos[repo].Revisions[digest] = true
|
||||
// A restart: a holder made afresh holds no finding of its own, and clears what the writer holds (design 47 G A-32, A-38).
|
||||
h = &Holder{Bus: h.Bus, Findings: h.Findings, Door: h.Door, Listed: h.Listed, Seats: h.Seats, Now: h.Now, Acts: h.Acts, Logf: h.Logf}
|
||||
takeIn(h, b)
|
||||
if err := h.Observe(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -556,16 +553,24 @@ func TestOnePendingPassReadsTheBuildsOnceAndEndsOnAnUnreadList(t *testing.T) {
|
||||
t.Fatalf("after the passes running is %v and wanted %v", h.running, h.wanted)
|
||||
}
|
||||
})
|
||||
t.Run("a pass reads the builds in one page", func(t *testing.T) {
|
||||
t.Run("a pass reads nothing of the builds", func(t *testing.T) {
|
||||
h, b, door, _ := aMesh(t)
|
||||
h.Pass(context.Background(), "after the build letta.x")
|
||||
if b.calls["page "+BuildsRead] != 1 || b.calls["get "+BuildsRead] != 0 || b.calls["history "+BuildsRead] != 0 {
|
||||
if n := b.calls["keys "+BuildsRead] + b.calls["get "+BuildsRead] + b.calls["history "+BuildsRead]; n != 0 {
|
||||
t.Fatalf("a pass read the builds by %v", b.calls)
|
||||
}
|
||||
if !slices.Contains(door.asked, "let go "+imageRef("letta", 2)) {
|
||||
t.Fatalf("the pass swept nothing: %v", door.asked)
|
||||
}
|
||||
})
|
||||
t.Run("no pass judges before the first view", func(t *testing.T) {
|
||||
h, _, door, _ := aMesh(t)
|
||||
h.taken = false
|
||||
h.Pass(context.Background(), "after the build letta.x")
|
||||
if len(door.asked) != 0 {
|
||||
t.Fatalf("a pass before the builds were taken in asked %v", door.asked)
|
||||
}
|
||||
})
|
||||
t.Run("a list unread ends the pass with its finding once", func(t *testing.T) {
|
||||
h, b, door, _ := aMesh(t)
|
||||
writes, listed, inner := 0, 0, h.Listed
|
||||
|
||||
Reference in New Issue
Block a user