Design 47 H1: the artifact sweep runs one pending pass from one paged read, and stops on an unread list (H-A-250)

A finished build or a want change started a sweep and an observation each, every one reading the builder's builds key
by key and asking the registry's list again, queued behind one lock: a third of the night's bus flood. Now an event
marks the one pending pass, a pass running when marked runs once more, the builds are read in pages of the state with
each build's end from the page, and a list the registry seats cannot answer ends the pass with artifact-list-unread,
written once and cleared by a pass that reads it.

Builds: design 47 H §H-A-250 (1), (2), (3)
This commit is contained in:
2026-11-21 11:54:46 +01:00
parent eb443cfc24
commit c01a816385
7 changed files with 311 additions and 52 deletions
+8
View File
@@ -119,3 +119,11 @@ removed for `retention` on its seat (`release`, `remove`). It reads a reference
and `<seat>/<module>/<artifact>@<digest>`, as one artifact, and says artifacts-gone in the form the build named it. A
call a seat refuses stops the sweep and writes the finding `sweep-refused` in the module's findings; the next sweep no
seat refuses clears it (`sweepseats.go`).
### One pending pass
A finished build or a changed want marks the one pending pass (design 47 H, H-A-250); a pass running when one is marked
runs once more when it ends, however many events marked it. A pass reads the builder's builds in pages of the state, one
answer for many keys, never one get per key, and sweeps and observes S25 from that one judgement. A list the registry
seats cannot answer ends the pass with the finding `artifact-list-unread`, written once; nothing asks the list again
until the next event, and a pass that reads it clears the finding (`holder.go`, `sweepseats.go`).
@@ -91,6 +91,12 @@ func (b runtimeBus) Get(state, key string) ([]byte, bool, error) {
}
return e.Value, true, nil
}
func (b runtimeBus) Page(state, after string) (stdio.StatePage, error) {
if b.followed(state) {
return stdio.StatePage{}, fmt.Errorf("the state %s is followed, not read in pages", state)
}
return stdio.State(state).Page(after, views.PageBytes, views.PageBudget)
}
func (runtimeBus) History(state, key string) ([]stdio.StateRevision, error) {
return stdio.State(state).History(key)
}
@@ -116,8 +122,8 @@ func (runtimeBus) Publish(subject string, body any) error {
return stdio.SeatPublish(subject, body, "")
}
// follow watches the builder's done keys and the wants: a build's end sweeps and
// observes again; a want changed observes again. Event-driven: nothing reads them on a timer.
// follow watches the builder's done keys and the wants: a build's end or a want changed marks the one pending pass
// (design 47 H, H-A-250 (1)), which sweeps and observes from one read. Event-driven: nothing reads them on a timer.
func follow(h *Holder, v *views.Reader) {
<-stdio.Served()
watch := func(state string, handle func(stdio.StateChange) error) {
@@ -130,11 +136,6 @@ func follow(h *Holder, v *views.Reader) {
time.Sleep(10 * time.Second)
}
}
observe := func() {
if err := h.Observe(context.Background()); err != nil {
h.logf("image-missing could not be observed: %v", err)
}
}
watch(BuildsRead, func(c stdio.StateChange) error {
if c.Current || c.Op != stdio.OpPut {
return nil
@@ -143,7 +144,7 @@ func follow(h *Holder, v *views.Reader) {
State string `json:"state"`
}
if json.Unmarshal(c.Value, &b) == nil && b.State == "done" {
go func() { h.AfterBuild(c.Key); observe() }()
h.Wanted("after the build " + c.Key)
}
return nil
})
@@ -154,7 +155,7 @@ func follow(h *Holder, v *views.Reader) {
for {
err := v.Watch(WantsRead, func(string, []byte, bool) {
if !current.Load() {
go observe()
h.Wanted("a want changed")
}
})
if err == nil {
@@ -164,5 +165,5 @@ func follow(h *Holder, v *views.Reader) {
time.Sleep(10 * time.Second)
}
current.Store(false)
observe()
h.Wanted("the first view of the wants")
}
@@ -63,6 +63,9 @@ const (
type Bus interface {
Keys(state string) ([]string, error)
Get(state, key string) ([]byte, bool, error)
// Page is the values after a key in key order, many in one answer (stdio.KeptState.Page): how a pass reads the
// builder's builds, never one get per key (design 47 H, H-A-250 (2)).
Page(state, after string) (stdio.StatePage, error)
History(state, key string) ([]stdio.StateRevision, error)
// LogSince is every entry of a read log from a time on (stdio.Log(read).Since).
LogSince(read string, at time.Time) ([]stdio.LogEntry, error)
@@ -95,8 +98,13 @@ type Holder struct {
sweep sync.Mutex // one sweep at a time
mu sync.Mutex
fmu sync.Mutex // one pass of Observe at a time
ended map[string]time.Time // a done key's end, read once from its history
// pmu guards the pending pass (H-A-250 (1)): running while a pass runs, wanted when an event marked it since.
pmu sync.Mutex
running bool
wanted bool
wantedBy string
fmu sync.Mutex // one pass of Observe at a time
ended map[string]time.Time // a done key's end, read once from its history
}
// Build is one done build of a module, as its key holds it, with when it ended.
@@ -108,36 +116,42 @@ type Build struct {
Manifest string
}
// Builds is every done build the builder's keys hold, each with its end.
// Builds is every done build the builder's keys hold, each with its end: the state read in pages, one answer for many
// keys, never one get per key (design 47 H, H-A-250 (2)). A build's end is when the bus wrote its done value, as the
// page says it; only a runtime that does not say it is asked the key's history.
func (h *Holder) Builds() ([]Build, error) {
keys, err := h.Bus.Keys(BuildsRead)
if err != nil {
return nil, fmt.Errorf("the builder's builds: %w", err)
}
var out []Build
for _, k := range keys {
module, fp, err := builds.ParseBuilderBuildKey(k)
after := ""
for {
page, err := h.Bus.Page(BuildsRead, after)
if err != nil {
continue
return nil, fmt.Errorf("the builder's builds: %w", err)
}
raw, found, err := h.Bus.Get(BuildsRead, k)
if err != nil {
return nil, fmt.Errorf("the builder's build %s: %w", k, err)
for _, e := range page.Entries {
module, fp, err := builds.ParseBuilderBuildKey(e.Key)
if err != nil {
continue
}
b, err := builds.DecodeBuilderBuild(e.Value)
if err != nil || b.State != builds.BuildDone {
continue
}
at := e.At
if at.IsZero() {
if at, err = h.endOf(e.Key); err != nil {
return nil, err
}
}
out = append(out, Build{Module: module, FP: fp, Key: e.Key, At: at, Artifacts: b.Artifacts, Manifest: b.Manifest})
}
if !found {
continue
if page.Next == "" {
return out, nil
}
b, err := builds.DecodeBuilderBuild(raw)
if err != nil || b.State != builds.BuildDone {
continue
if page.Next <= after {
return nil, fmt.Errorf("the builder's builds: a page after %q answered %q to ask after, which is no further", after, page.Next)
}
at, err := h.endOf(k)
if err != nil {
return nil, err
}
out = append(out, Build{Module: module, FP: fp, Key: k, At: at, Artifacts: b.Artifacts, Manifest: b.Manifest})
after = page.Next
}
return out, nil
}
// endOf is when a done key was last written done, read once from its history: a done key is not written again.
@@ -320,6 +334,8 @@ type judged struct {
mirrors map[string]*Ref
// mirrorsHeld says why no copy is eligible now, when one is not: a kept build whose bases the log does not name.
mirrorsHeld string
// unread is why the registry seats' list could not be read, nil when it was (or none is read).
unread error
}
func (h *Holder) judge(ctx context.Context) (*judged, error) {
@@ -335,7 +351,7 @@ func (h *Holder) judge(ctx context.Context) (*judged, error) {
if h.Listed != nil {
if j.view, err = h.Listed(ctx); err != nil {
h.logf("the store's listing could not be read, so nothing is said gone: %v", err)
j.view = nil
j.view, j.unread = nil, err
}
}
if j.view != nil {
@@ -514,6 +530,11 @@ func (h *Holder) Sweep(ctx context.Context, s Sweeping) (*CollectAnswer, error)
if err != nil {
return nil, err
}
return h.sweepFrom(ctx, j, s)
}
// sweepFrom is a sweep from a judgement already read; the caller holds h.sweep.
func (h *Holder) sweepFrom(ctx context.Context, j *judged, s Sweeping) (*CollectAnswer, error) {
a := &CollectAnswer{DryRun: s.DryRun, WouldLetGo: []string{}, LetGo: []string{}}
var keptArchives, eligible []string
for _, r := range j.sorted() {
@@ -658,15 +679,69 @@ func namedBy(b Build, r string) string {
return r
}
// AfterBuild is the sweep at a build's end.
func (h *Holder) AfterBuild(key string) {
a, err := h.Sweep(context.Background(), Sweeping{Most: MostPerSweep, By: "after the build " + key})
// Wanted marks the pass wanted (design 47 H, H-A-250 (1)): a finished build or a want change asks it, by naming what
// asked. With no pass running one starts; a pass running when it is marked runs once more when it ends, however many
// events marked it, never once per event.
func (h *Holder) Wanted(by string) {
if h.mark(by) {
go h.passes(by)
}
}
// mark sets the pending mark, and says whether no pass ran, so the caller starts one.
func (h *Holder) mark(by string) bool {
h.pmu.Lock()
defer h.pmu.Unlock()
if h.running {
h.wanted, h.wantedBy = true, by
return false
}
h.running = true
return true
}
// passes runs the pass, then once more for as long as an event marked it during the one that ran.
func (h *Holder) passes(by string) {
for {
h.Pass(context.Background(), by)
h.pmu.Lock()
if !h.wanted {
h.running = false
h.pmu.Unlock()
return
}
h.wanted, by = false, h.wantedBy
h.pmu.Unlock()
}
}
// Pass is one pass of the seat: the keep set judged from one read of the builds, the wants and the store's list, then
// the sweep and S25's observation from that one judgement. A list the registry seats cannot answer ends the pass with
// the finding artifact-list-unread, asked again only at the next event (H-A-250 (3)); a list read again clears it.
func (h *Holder) Pass(ctx context.Context, by string) {
h.sweep.Lock()
j, err := h.judge(ctx)
if err != nil {
h.logf("the sweep after %s: %v", key, err)
h.sweep.Unlock()
h.logf("the pass %s: %v", by, err)
return
}
if len(a.LetGo) > 0 || a.Stopped != "" || a.Missing > 0 {
h.logf("the sweep after %s let go of %d (%d left, %d missing): %s", key, len(a.LetGo), a.Left, a.Missing, a.Stopped)
if j.unread != nil {
h.sweep.Unlock()
h.listUnread(j.unread)
return
}
h.listRead()
a, err := h.sweepFrom(ctx, j, Sweeping{Most: MostPerSweep, By: by})
h.sweep.Unlock()
switch {
case err != nil:
h.logf("the sweep %s: %v", by, err)
case len(a.LetGo) > 0 || a.Stopped != "" || a.Missing > 0:
h.logf("the sweep %s let go of %d (%d left, %d missing): %s", by, len(a.LetGo), a.Left, a.Missing, a.Stopped)
}
if err := h.observeFrom(ctx, j); err != nil {
h.logf("image-missing could not be observed: %v", err)
}
}
@@ -857,10 +932,16 @@ func inStoreAt(image, address string) string {
// want's machine, written once and cleared once it no longer holds. Nothing is said without a listing. What stands is what
// the held writer holds (design 47 G A-32, A-38), so a restart neither drops nor rewrites it.
func (h *Holder) Observe(ctx context.Context) error {
now, err := h.observed(ctx)
j, err := h.judge(ctx)
if err != nil {
return err
}
return h.observeFrom(ctx, j)
}
// observeFrom is Observe from a judgement already read.
func (h *Holder) observeFrom(ctx context.Context, j *judged) error {
now := h.observed(j)
if now == nil {
return nil
}
@@ -899,13 +980,9 @@ type Findings interface {
}
// observed is S25's findings now by key, nil without a listing.
func (h *Holder) observed(ctx context.Context) (map[string]meshconditions.Finding, error) {
j, err := h.judge(ctx)
if err != nil {
return nil, err
}
func (h *Holder) observed(j *judged) map[string]meshconditions.Finding {
if j.view == nil {
return nil, nil
return nil
}
address := ""
if d, err := h.Door(); err == nil {
@@ -944,7 +1021,7 @@ func (h *Holder) observed(ctx context.Context) (map[string]meshconditions.Findin
break
}
}
return now, nil
return now
}
// imageMissing is S25's finding, in the controller's words (cmd/mesh-controller/image_missing.go watchMissingImages); its id
@@ -11,6 +11,7 @@ package main
import (
"context"
"encoding/json"
"sort"
"strings"
"sync"
"testing"
@@ -78,6 +79,31 @@ func (b realBus) Get(state, key string) ([]byte, bool, error) {
return e.Value(), true, nil
}
// Page answers every value after a key in one page, each with when the bus wrote it, as the runtime's page does.
func (b realBus) Page(state, after string) (stdio.StatePage, error) {
keys, err := b.Keys(state)
if err != nil {
return stdio.StatePage{}, err
}
kv, err := b.kv(state)
if err != nil {
return stdio.StatePage{}, err
}
sort.Strings(keys)
p := stdio.StatePage{Entries: []stdio.StateEntry{}}
for _, k := range keys {
if k <= after {
continue
}
e, err := kv.Get(b.ctx, k)
if err != nil {
return stdio.StatePage{}, err
}
p.Entries = append(p.Entries, stdio.StateEntry{Key: k, Value: e.Value(), Revision: e.Revision(), At: e.Created()})
}
return p, nil
}
func (b realBus) History(state, key string) ([]stdio.StateRevision, error) {
kv, err := b.kv(state)
if err != nil {
@@ -215,7 +241,7 @@ func TestTheSeatSweepsAtABuildsEndOnARealBusWithNoLease(t *testing.T) {
}
select {
case e := <-w.Updates():
h.AfterBuild(e.Key())
h.Pass(ctx, "after the build "+e.Key())
case <-ctx.Done():
t.Fatal("the build's end was not heard")
}
@@ -34,6 +34,15 @@ type memBus struct {
history map[string][]stdio.StateRevision
log []stdio.LogEntry
published []string
// calls counts each read by "<verb> <state>": a pass reads the builds in one page, never one get per key.
calls map[string]int
}
func (b *memBus) count(verb, state string) {
if b.calls == nil {
b.calls = map[string]int{}
}
b.calls[verb+" "+state]++
}
func (b *memBus) LogSince(read string, at time.Time) ([]stdio.LogEntry, error) {
@@ -60,11 +69,38 @@ func (b *memBus) Keys(state string) ([]string, error) {
func (b *memBus) Get(state, key string) ([]byte, bool, error) {
b.mu.Lock()
defer b.mu.Unlock()
b.count("get", state)
v, ok := b.states[state][key]
return v, ok, nil
}
// Page answers every value after a key in one page, each written when its last history revision says.
func (b *memBus) Page(state, after string) (stdio.StatePage, error) {
b.mu.Lock()
defer b.mu.Unlock()
b.count("page", state)
p := stdio.StatePage{Entries: []stdio.StateEntry{}}
var keys []string
for k := range b.states[state] {
if k > after {
keys = append(keys, k)
}
}
sort.Strings(keys)
for _, k := range keys {
e := stdio.StateEntry{Key: k, Value: b.states[state][k]}
if revs := b.history[state+" "+k]; len(revs) > 0 {
e.At = revs[len(revs)-1].At
}
p.Entries = append(p.Entries, e)
}
return p, nil
}
func (b *memBus) History(state, key string) ([]stdio.StateRevision, error) {
b.mu.Lock()
defer b.mu.Unlock()
b.count("history", state)
return b.history[state+" "+key], nil
}
@@ -323,7 +359,7 @@ func TestTheSeatKeepsTheFiveNewestAndTheWantedAndSweepsWithNoLease(t *testing.T)
}
switch c.how {
case "build":
h.AfterBuild("letta.x")
h.Pass(context.Background(), "after the build letta.x")
var gone []string
for _, x := range door.asked {
if r, ok := strings.CutPrefix(x, "let go "); ok {
@@ -482,3 +518,75 @@ func (f findingsThrough) Held() []meshconditions.Finding {
func (f findingsThrough) Clear(_ context.Context, scope, id, kind, _ string) error {
return f.bus.Delete(FindingsState, views.FindingKey(scope, id, kind))
}
// writesCounted counts the findings written through the test's writer.
type writesCounted struct {
findingsThrough
writes *int
}
func (w writesCounted) Write(ctx context.Context, f meshconditions.Finding) error {
*w.writes++
return w.findingsThrough.Write(ctx, f)
}
// Design 47 H, H-A-250: the starts fold into one pending pass, a pass reads the builds in one page, and a list the
// registry seats cannot answer ends the pass with artifact-list-unread, written once and asked again only at an event.
func TestOnePendingPassReadsTheBuildsOnceAndEndsOnAnUnreadList(t *testing.T) {
t.Run("three events during a pass run exactly one more", func(t *testing.T) {
h, _, _, _ := aMesh(t)
listed, inner := 0, h.Listed
h.Listed = func(ctx context.Context) (*View, error) {
listed++
if listed == 1 {
for _, by := range []string{"after the build a", "after the build b", "a want changed"} {
h.Wanted(by) // a pass runs: each only marks it
}
}
return inner(ctx)
}
if !h.mark("after the build letta.x") {
t.Fatal("no pass ran, and the mark did not start one")
}
h.passes("after the build letta.x")
if listed != 2 {
t.Fatalf("three events during a pass ran %d passes, want 2", listed)
}
if h.running || h.wanted {
t.Fatalf("after the passes running is %v and wanted %v", h.running, h.wanted)
}
})
t.Run("a pass reads the builds in one page", func(t *testing.T) {
h, b, door, _ := aMesh(t)
h.Pass(context.Background(), "after the build letta.x")
if b.calls["page "+BuildsRead] != 1 || b.calls["get "+BuildsRead] != 0 || b.calls["history "+BuildsRead] != 0 {
t.Fatalf("a pass read the builds by %v", b.calls)
}
if !slices.Contains(door.asked, "let go "+imageRef("letta", 2)) {
t.Fatalf("the pass swept nothing: %v", door.asked)
}
})
t.Run("a list unread ends the pass with its finding once", func(t *testing.T) {
h, b, door, _ := aMesh(t)
writes, listed, inner := 0, 0, h.Listed
h.Findings = writesCounted{findingsThrough{b}, &writes}
h.Listed = func(context.Context) (*View, error) {
listed++
return nil, errors.New("the answer exceeds the bus's one megabyte")
}
h.Pass(context.Background(), "after the build a")
h.Pass(context.Background(), "a want changed")
key := views.FindingKey("module", Module, ListUnread)
if raw, found, _ := b.Get(FindingsState, key); !found || writes != 1 || !strings.Contains(string(raw), "one megabyte") {
t.Fatalf("the finding is %s (found %v), written %d times", raw, found, writes)
}
if listed != 2 || len(door.asked) != 0 || len(b.published) != 0 {
t.Fatalf("two passes listed %d times, asked %v and said %v", listed, door.asked, b.published)
}
h.Listed = inner
h.Pass(context.Background(), "after the build b")
if _, found, _ := b.Get(FindingsState, key); found || len(door.asked) == 0 {
t.Fatalf("a pass that read the list left the finding (%v) or swept nothing (%v)", found, door.asked)
}
})
}
@@ -32,7 +32,7 @@ func main() {
h.Acts = meshhandactsclient.Recorder{Ask: stdio.Ask}
// The module's one held writer (design 47 G §689, A-16, A-29, A-32): the holder's findings and the reader's go through it.
w, err := views.NewStdioHeldWriter(views.WriterConfig[meshconditions.Finding, meshconditions.FindingCleared, meshconditions.FindingsStated, meshconditions.FindingsAskedEvent]{
Asked: meshconditions.SubjectFindingsAsked, Module: Module, State: stdio.State("findings"), Kinds: []string{ImageMissing, SweepRefused}})
Asked: meshconditions.SubjectFindingsAsked, Module: Module, State: stdio.State("findings"), Kinds: []string{ImageMissing, SweepRefused, ListUnread}})
if err != nil {
logf("%v", err)
os.Exit(1)
@@ -27,6 +27,9 @@ import (
// SweepRefused is the sweep's finding: a seat refused a hold or a remove the sweep asked.
const SweepRefused = "sweep-refused"
// ListUnread is the pass's finding: the registry seats' list could not be read, so the pass ended (design 47 H, H-A-250 (3)).
const ListUnread = "artifact-list-unread"
// canonical is a kept reference in the older form, `artifact-store://<path>`, whichever form it came in; another is as
// it was.
func canonical(reference string) string {
@@ -140,6 +143,42 @@ func (h *Holder) sweepClean() {
}
}
// listUnread writes the pass's finding artifact-list-unread once: a pass whose list fails again while the writer holds it
// writes nothing more. Nothing asks the list again until the next event marks a pass.
func (h *Holder) listUnread(err error) {
for _, was := range h.Findings.Held() {
if was.Kind == ListUnread {
return
}
}
at := h.now().UTC()
f := meshconditions.Finding{Scope: "module", ID: keys.EscapeName(Module), Kind: ListUnread,
Subject: meshconditions.Subject{Scope: "module", ID: keys.EscapeName(Module)},
Severity: meshconditions.FindingSeverityWarning, Resolver: "operator",
Summary: "the artifact sweep stopped: the registry seats' list could not be read (the evidence names why)",
Headline: "The artifact sweep cannot read the store's list",
Explanation: "The sweep judges what to keep against the store's list, which it asks the registry seats. The list " +
"could not be read, so the sweep removed nothing and observed nothing; what it kept stays kept.",
Needs: "read the registry seat holder's journal for why its list failed; the next finished build or want change asks again.",
Evidence: []meshconditions.Evidence{{At: at, Said: err.Error()}},
Source: "CAT-AS1", Observer: Module, At: at}
if werr := h.Findings.Write(context.Background(), f); werr != nil {
h.logf("the pass's finding (%v) could not be written: %v", err, werr)
}
}
// listRead clears artifact-list-unread, when the held writer holds it, once a pass read the list.
func (h *Holder) listRead() {
for _, was := range h.Findings.Held() {
if was.Kind != ListUnread {
continue
}
if err := h.Findings.Clear(context.Background(), was.Scope, was.ID, was.Kind, "a pass read the registry seats' list"); err != nil {
h.logf("the pass's finding could not be cleared: %v", err)
}
}
}
// keptWhy is why a kept reference is held, as the seat's hold takes it.
func keptWhy(r *Ref) string {
if r == nil || len(r.Why) == 0 {