diff --git a/modules/artifact-store-tools/README.md b/modules/artifact-store-tools/README.md index cc662262c..586f58479 100644 --- a/modules/artifact-store-tools/README.md +++ b/modules/artifact-store-tools/README.md @@ -119,3 +119,16 @@ removed for `retention` on its seat (`release`, `remove`). It reads a reference and `//@`, as one artifact, and says artifacts-gone in the form the build named it. A call a seat refuses stops the sweep and writes the finding `sweep-refused` in the module's findings; the next sweep no seat refuses clears it (`sweepseats.go`). + +### One pending pass + +A finished build or a changed want marks the one pending pass (design 47 H, H-A-250); a pass running when one is marked +runs once more when it ends, however many events marked it. The builder's builds are a followed view: the watch's first view +at start, then its updates, kept in memory (H-A-255), each build's end taken from the builder's log's build-ended entries the +pass reads once, and only for a build the log lacks from its key's history (H-A-258); a watch asked again after an error replaces the view whole with its handover (H-A-257). A pass reads nothing +of them on the bus, judges nothing before the first view is in, and sweeps and observes S25 +from that one judgement. A list the registry +seats cannot answer ends the pass with the finding `artifact-list-unread`, written once; nothing asks the list again +until the next event, and a pass that reads it clears the finding. A pass that cannot judge the builds (the view not yet taken in, or a +done build's end unread) ends the same way with the finding `artifact-builds-unread` (H-A-260) (`holder.go`, +`sweepseats.go`). diff --git a/modules/artifact-store-tools/cmd/artifact-store-tools/door.go b/modules/artifact-store-tools/cmd/artifact-store-tools/door.go index e9292f5fe..cf807bf45 100644 --- a/modules/artifact-store-tools/cmd/artifact-store-tools/door.go +++ b/modules/artifact-store-tools/cmd/artifact-store-tools/door.go @@ -116,12 +116,16 @@ func (runtimeBus) Publish(subject string, body any) error { return stdio.SeatPublish(subject, body, "") } -// follow watches the builder's done keys and the wants: a build's end sweeps and -// observes again; a want changed observes again. Event-driven: nothing reads them on a timer. +// follow watches the builder's done keys and the wants: the builds are a followed view, the watch's first view then its +// updates kept in memory (design 47 H, H-A-255); a build's end or a want changed marks the one pending pass (H-A-250 (1)), +// which sweeps and observes from the views. Event-driven: nothing reads them on a timer. func follow(h *Holder, v *views.Reader) { <-stdio.Served() watch := func(state string, handle func(stdio.StateChange) error) { for { + if state == BuildsRead { + h.Watching() // each handover, the first or one asked again, replaces the view whole (H-A-257) + } if err := stdio.State(state).Watch("", handle); err == nil { return } else { @@ -130,12 +134,8 @@ func follow(h *Holder, v *views.Reader) { time.Sleep(10 * time.Second) } } - observe := func() { - if err := h.Observe(context.Background()); err != nil { - h.logf("image-missing could not be observed: %v", err) - } - } watch(BuildsRead, func(c stdio.StateChange) error { + h.Heard(c) if c.Current || c.Op != stdio.OpPut { return nil } @@ -143,10 +143,12 @@ func follow(h *Holder, v *views.Reader) { State string `json:"state"` } if json.Unmarshal(c.Value, &b) == nil && b.State == "done" { - go func() { h.AfterBuild(c.Key); observe() }() + h.Wanted("after the build " + c.Key) } return nil }) + // The watch hands every current value over before it returns: the first view is in. + h.TakenIn() // The wants are the assigner's: the first view then its events (ADR 0615). What Watch hands over before it returns // is the first view, not a change. var current atomic.Bool @@ -154,7 +156,7 @@ func follow(h *Holder, v *views.Reader) { for { err := v.Watch(WantsRead, func(string, []byte, bool) { if !current.Load() { - go observe() + h.Wanted("a want changed") } }) if err == nil { @@ -164,5 +166,5 @@ func follow(h *Holder, v *views.Reader) { time.Sleep(10 * time.Second) } current.Store(false) - observe() + h.Wanted("the first view of the wants") } diff --git a/modules/artifact-store-tools/cmd/artifact-store-tools/holder.go b/modules/artifact-store-tools/cmd/artifact-store-tools/holder.go index 3490e8f4f..007d88238 100644 --- a/modules/artifact-store-tools/cmd/artifact-store-tools/holder.go +++ b/modules/artifact-store-tools/cmd/artifact-store-tools/holder.go @@ -38,6 +38,10 @@ import ( "git.novox.be/novox/mesh-sdk/go/views" ) +// ErrBuildsUnread is a judgement refused because the builds cannot be judged: the followed view not yet taken in, or a +// done key's end unread (H-A-258, H-A-260). +var ErrBuildsUnread = errors.New("the builder's builds cannot be judged") + // What the holder reads and keeps. const ( WantsRead = assignerModule + ".wants" @@ -95,8 +99,19 @@ type Holder struct { sweep sync.Mutex // one sweep at a time mu sync.Mutex - fmu sync.Mutex // one pass of Observe at a time - ended map[string]time.Time // a done key's end, read once from its history + // pmu guards the pending pass (H-A-250 (1)): running while a pass runs, wanted when an event marked it since. + pmu sync.Mutex + running bool + wanted bool + wantedBy string + fmu sync.Mutex // one pass of Observe at a time + ended map[string]time.Time // a done key's end, read once from its history + // done is the followed view of the builder's builds (design 47 H, H-A-255): the done builds by key, taken in from the + // watch's first view and its updates (Heard); taken says the first view is in. A pass judges from it, reading nothing. + done map[string]Build + taken bool + // next is the handover a watch asked (again) is giving, which replaces done whole once it is in (H-A-257). + next map[string]Build } // Build is one done build of a module, as its key holds it, with when it ended. @@ -108,39 +123,81 @@ type Build struct { Manifest string } -// Builds is every done build the builder's keys hold, each with its end. +// Builds is every done build of the followed view, by key, read from memory, never from the bus (design 47 H, H-A-250 +// (2), H-A-255): each with the end known for it, zero where none is known yet (withEnds gives it). Before the view's first +// view is taken in there is none to judge from, and it says so, so no pass sweeps against an empty keep set. func (h *Holder) Builds() ([]Build, error) { - keys, err := h.Bus.Keys(BuildsRead) - if err != nil { - return nil, fmt.Errorf("the builder's builds: %w", err) + h.mu.Lock() + defer h.mu.Unlock() + if !h.taken { + return nil, fmt.Errorf("%w: they are not taken in yet, their first view has not come", ErrBuildsUnread) } - var out []Build - for _, k := range keys { - module, fp, err := builds.ParseBuilderBuildKey(k) - if err != nil { - continue - } - raw, found, err := h.Bus.Get(BuildsRead, k) - if err != nil { - return nil, fmt.Errorf("the builder's build %s: %w", k, err) - } - if !found { - continue - } - b, err := builds.DecodeBuilderBuild(raw) - if err != nil || b.State != builds.BuildDone { - continue - } - at, err := h.endOf(k) - if err != nil { - return nil, err - } - out = append(out, Build{Module: module, FP: fp, Key: k, At: at, Artifacts: b.Artifacts, Manifest: b.Manifest}) + out := make([]Build, 0, len(h.done)) + for _, b := range h.done { + b.At = h.ended[b.Key] + out = append(out, b) } + sort.Slice(out, func(a, b int) bool { return out[a].Key < out[b].Key }) return out, nil } -// endOf is when a done key was last written done, read once from its history: a done key is not written again. +// Heard takes one change of the builder's builds into the followed view: a done value is kept, anything else drops the +// key. Its end is not read here: withEnds takes it from the builder's log at the next pass (H-A-258); a key written done +// anew forgets the end it had. While a watch's handover is gathered (Watching), every change goes into it, current or +// not, so nothing heard before TakenIn is lost by the swap (H-A-257). +func (h *Holder) Heard(c stdio.StateChange) { + module, fp, err := builds.ParseBuilderBuildKey(c.Key) + if err != nil { + return + } + var b builds.BuilderBuild + if c.Op == stdio.OpPut { + b, err = builds.DecodeBuilderBuild(c.Value) + } + h.mu.Lock() + defer h.mu.Unlock() + if !c.Current { + delete(h.ended, c.Key) // written anew or gone: an end it had is no longer its + } + if h.next == nil && h.done == nil { + h.done = map[string]Build{} + } + into := h.done + if h.next != nil { + into = h.next + } + if c.Op != stdio.OpPut || err != nil || b.State != builds.BuildDone { + delete(into, c.Key) + return + } + into[c.Key] = Build{Module: module, FP: fp, Key: c.Key, Artifacts: b.Artifacts, Manifest: b.Manifest} +} + +// Watching says a watch of the builds is asked, first or again after an error: its handover of current values is +// gathered apart and replaces the view whole at TakenIn, so a key deleted while no watch ran leaves it (H-A-257). +func (h *Holder) Watching() { + h.mu.Lock() + h.next = map[string]Build{} + h.mu.Unlock() +} + +// TakenIn says the watch's handover is in: every current value was Heard. It replaces the view whole, and forgets the +// end of every key it no longer holds. +func (h *Holder) TakenIn() { + h.mu.Lock() + defer h.mu.Unlock() + if h.next != nil { + h.done, h.next = h.next, nil + for k := range h.ended { + if _, kept := h.done[k]; !kept { + delete(h.ended, k) + } + } + } + h.taken = true +} + +// endOf is when a done key was last written done, read from its history and kept: for a build the builder's log lacks. func (h *Holder) endOf(key string) (time.Time, error) { h.mu.Lock() at, known := h.ended[key] @@ -320,6 +377,8 @@ type judged struct { mirrors map[string]*Ref // mirrorsHeld says why no copy is eligible now, when one is not: a kept build whose bases the log does not name. mirrorsHeld string + // unread is why the registry seats' list could not be read, nil when it was (or none is read). + unread error } func (h *Holder) judge(ctx context.Context) (*judged, error) { @@ -327,6 +386,15 @@ func (h *Holder) judge(ctx context.Context) (*judged, error) { if err != nil { return nil, err } + // The builder's log, read once: what each build stood on, and each build's end (H-A-258). + against, ends, logErr := h.logged() + if logErr != nil { + h.logf("the builder's log could not be read, so every copy is kept as unrecorded and each end is read from its history: %v", logErr) + against, ends = nil, nil + } + if builds, err = h.withEnds(builds, ends); err != nil { + return nil, err + } wants, err := h.Wants() if err != nil { return nil, err @@ -335,15 +403,10 @@ func (h *Holder) judge(ctx context.Context) (*judged, error) { if h.Listed != nil { if j.view, err = h.Listed(ctx); err != nil { h.logf("the store's listing could not be read, so nothing is said gone: %v", err) - j.view = nil + j.view, j.unread = nil, err } } if j.view != nil { - against, err := h.against() - if err != nil { - h.logf("the builder's log could not be read, so every copy is kept as unrecorded: %v", err) - against = nil - } j.mirrors, j.mirrorsHeld = Mirrors(j.view, builds, keptBuilds(builds, wants), against) for r, m := range j.mirrors { if len(m.Builds) > 0 { @@ -354,13 +417,47 @@ func (h *Holder) judge(ctx context.Context) (*judged, error) { return j, nil } -// against is what each build the builder's log says ended stood on, by build key: the bases' references. -func (h *Holder) against() (map[string][]string, error) { +// withEnds gives each build its end: the one known, else the builder's log's build-ended entry, else its key's history, +// each kept once read (H-A-258). A done key whose end can be read from neither stays unread, and then nothing is judged: +// never from a view missing a build. +func (h *Holder) withEnds(built []Build, ends map[string]time.Time) ([]Build, error) { + var unread []string + var why error + for i := range built { + if !built[i].At.IsZero() { + continue + } + if at, logged := ends[built[i].Key]; logged { + h.mu.Lock() + if h.ended == nil { + h.ended = map[string]time.Time{} + } + h.ended[built[i].Key] = at + h.mu.Unlock() + built[i].At = at + continue + } + at, err := h.endOf(built[i].Key) + if err != nil { + unread, why = append(unread, built[i].Key), err + continue + } + built[i].At = at + } + if len(unread) > 0 { + return nil, fmt.Errorf("%w: the end of %d done builds could not be read (%s first), so nothing is judged: %v", ErrBuildsUnread, len(unread), unread[0], why) + } + return built, nil +} + +// logged is what each build the builder's log says ended stood on, by build key (the bases' references), and when it +// ended: one read of the log. +func (h *Holder) logged() (map[string][]string, map[string]time.Time, error) { entries, err := h.Bus.LogSince(builds.BuildLogRead, h.now().Add(-builds.BuilderLogDays*24*time.Hour)) if err != nil { - return nil, err + return nil, nil, err } - out := map[string][]string{} + out, ends := map[string][]string{}, map[string]time.Time{} for _, raw := range entries { e, err := builds.DecodeBuildLogEntry(raw.Value) if err != nil || e.Event != builds.BuildEndedEvent { @@ -371,8 +468,11 @@ func (h *Holder) against() (map[string][]string, error) { continue } out[key] = e.Against + if e.At.After(ends[key]) { + ends[key] = e.At + } } - return out, nil + return out, ends, nil } // Mirrors judges each copy in the store's mirror repositories (upstream//, /on-; ADR @@ -514,6 +614,11 @@ func (h *Holder) Sweep(ctx context.Context, s Sweeping) (*CollectAnswer, error) if err != nil { return nil, err } + return h.sweepFrom(ctx, j, s) +} + +// sweepFrom is a sweep from a judgement already read; the caller holds h.sweep. +func (h *Holder) sweepFrom(ctx context.Context, j *judged, s Sweeping) (*CollectAnswer, error) { a := &CollectAnswer{DryRun: s.DryRun, WouldLetGo: []string{}, LetGo: []string{}} var keptArchives, eligible []string for _, r := range j.sorted() { @@ -658,15 +763,73 @@ func namedBy(b Build, r string) string { return r } -// AfterBuild is the sweep at a build's end. -func (h *Holder) AfterBuild(key string) { - a, err := h.Sweep(context.Background(), Sweeping{Most: MostPerSweep, By: "after the build " + key}) +// Wanted marks the pass wanted (design 47 H, H-A-250 (1)): a finished build or a want change asks it, by naming what +// asked. With no pass running one starts; a pass running when it is marked runs once more when it ends, however many +// events marked it, never once per event. +func (h *Holder) Wanted(by string) { + if h.mark(by) { + go h.passes(by) + } +} + +// mark sets the pending mark, and says whether no pass ran, so the caller starts one. +func (h *Holder) mark(by string) bool { + h.pmu.Lock() + defer h.pmu.Unlock() + if h.running { + h.wanted, h.wantedBy = true, by + return false + } + h.running = true + return true +} + +// passes runs the pass, then once more for as long as an event marked it during the one that ran. +func (h *Holder) passes(by string) { + for { + h.Pass(context.Background(), by) + h.pmu.Lock() + if !h.wanted { + h.running = false + h.pmu.Unlock() + return + } + h.wanted, by = false, h.wantedBy + h.pmu.Unlock() + } +} + +// Pass is one pass of the seat: the keep set judged from one read of the builds, the wants and the store's list, then +// the sweep and S25's observation from that one judgement. A list the registry seats cannot answer ends the pass with +// the finding artifact-list-unread, asked again only at the next event (H-A-250 (3)); a list read again clears it. +func (h *Holder) Pass(ctx context.Context, by string) { + h.sweep.Lock() + j, err := h.judge(ctx) if err != nil { - h.logf("the sweep after %s: %v", key, err) + h.sweep.Unlock() + h.logf("the pass %s: %v", by, err) + if errors.Is(err, ErrBuildsUnread) { + h.unreadFinding(BuildsUnread, err) // H-A-260 + } return } - if len(a.LetGo) > 0 || a.Stopped != "" || a.Missing > 0 { - h.logf("the sweep after %s let go of %d (%d left, %d missing): %s", key, len(a.LetGo), a.Left, a.Missing, a.Stopped) + h.readAgain(BuildsUnread) + if j.unread != nil { + h.sweep.Unlock() + h.unreadFinding(ListUnread, j.unread) + return + } + h.readAgain(ListUnread) + a, err := h.sweepFrom(ctx, j, Sweeping{Most: MostPerSweep, By: by}) + h.sweep.Unlock() + switch { + case err != nil: + h.logf("the sweep %s: %v", by, err) + case len(a.LetGo) > 0 || a.Stopped != "" || a.Missing > 0: + h.logf("the sweep %s let go of %d (%d left, %d missing): %s", by, len(a.LetGo), a.Left, a.Missing, a.Stopped) + } + if err := h.observeFrom(ctx, j); err != nil { + h.logf("image-missing could not be observed: %v", err) } } @@ -857,10 +1020,16 @@ func inStoreAt(image, address string) string { // want's machine, written once and cleared once it no longer holds. Nothing is said without a listing. What stands is what // the held writer holds (design 47 G A-32, A-38), so a restart neither drops nor rewrites it. func (h *Holder) Observe(ctx context.Context) error { - now, err := h.observed(ctx) + j, err := h.judge(ctx) if err != nil { return err } + return h.observeFrom(ctx, j) +} + +// observeFrom is Observe from a judgement already read. +func (h *Holder) observeFrom(ctx context.Context, j *judged) error { + now := h.observed(j) if now == nil { return nil } @@ -899,13 +1068,9 @@ type Findings interface { } // observed is S25's findings now by key, nil without a listing. -func (h *Holder) observed(ctx context.Context) (map[string]meshconditions.Finding, error) { - j, err := h.judge(ctx) - if err != nil { - return nil, err - } +func (h *Holder) observed(j *judged) map[string]meshconditions.Finding { if j.view == nil { - return nil, nil + return nil } address := "" if d, err := h.Door(); err == nil { @@ -944,7 +1109,7 @@ func (h *Holder) observed(ctx context.Context) (map[string]meshconditions.Findin break } } - return now, nil + return now } // imageMissing is S25's finding, in the controller's words (cmd/mesh-controller/image_missing.go watchMissingImages); its id diff --git a/modules/artifact-store-tools/cmd/artifact-store-tools/holder_integration_test.go b/modules/artifact-store-tools/cmd/artifact-store-tools/holder_integration_test.go index 3963d3749..4e81af3d0 100644 --- a/modules/artifact-store-tools/cmd/artifact-store-tools/holder_integration_test.go +++ b/modules/artifact-store-tools/cmd/artifact-store-tools/holder_integration_test.go @@ -200,6 +200,15 @@ func TestTheSeatSweepsAtABuildsEndOnARealBusWithNoLease(t *testing.T) { } h := &Holder{Bus: findingsAt{realBus: bus}, Findings: findingsThrough{findingsAt{realBus: bus}}, Door: mem.Door, Listed: mem.Listed, Seats: ask, Now: time.Now, Logf: t.Logf} h.Acts = meshhandactsclient.Recorder{Ask: ask} + // The followed view of the builds: the six on the bus handed over as the watch's first view does. + h.Watching() + if keys, err := bus.Keys(BuildsRead); err == nil { + for _, k := range keys { + raw, _, _ := bus.Get(BuildsRead, k) + h.Heard(stdio.StateChange{State: BuildsRead, Key: k, Op: stdio.OpPut, Value: raw, Current: true}) + } + } + h.TakenIn() // A build's end heard from the builds bucket, as follow hands it over. kv, _ := bus.kv(BuildsRead) @@ -215,7 +224,8 @@ func TestTheSeatSweepsAtABuildsEndOnARealBusWithNoLease(t *testing.T) { } select { case e := <-w.Updates(): - h.AfterBuild(e.Key()) + h.Heard(stdio.StateChange{State: BuildsRead, Key: e.Key(), Op: stdio.OpPut, Value: e.Value()}) + h.Pass(ctx, "after the build "+e.Key()) case <-ctx.Done(): t.Fatal("the build's end was not heard") } diff --git a/modules/artifact-store-tools/cmd/artifact-store-tools/holder_test.go b/modules/artifact-store-tools/cmd/artifact-store-tools/holder_test.go index 72374209f..c92bb5b9a 100644 --- a/modules/artifact-store-tools/cmd/artifact-store-tools/holder_test.go +++ b/modules/artifact-store-tools/cmd/artifact-store-tools/holder_test.go @@ -34,6 +34,17 @@ type memBus struct { history map[string][]stdio.StateRevision log []stdio.LogEntry published []string + // historyErr fails the history read of a key. + historyErr map[string]error + // calls counts each read by " ": a pass reads nothing of the builds, judged from the followed view. + calls map[string]int +} + +func (b *memBus) count(verb, state string) { + if b.calls == nil { + b.calls = map[string]int{} + } + b.calls[verb+" "+state]++ } func (b *memBus) LogSince(read string, at time.Time) ([]stdio.LogEntry, error) { @@ -49,6 +60,7 @@ func (b *memBus) LogSince(read string, at time.Time) ([]stdio.LogEntry, error) { func (b *memBus) Keys(state string) ([]string, error) { b.mu.Lock() defer b.mu.Unlock() + b.count("keys", state) var out []string for k := range b.states[state] { out = append(out, k) @@ -60,11 +72,18 @@ func (b *memBus) Keys(state string) ([]string, error) { func (b *memBus) Get(state, key string) ([]byte, bool, error) { b.mu.Lock() defer b.mu.Unlock() + b.count("get", state) v, ok := b.states[state][key] return v, ok, nil } func (b *memBus) History(state, key string) ([]stdio.StateRevision, error) { + b.mu.Lock() + defer b.mu.Unlock() + b.count("history", state) + if err := b.historyErr[key]; err != nil { + return nil, err + } return b.history[state+" "+key], nil } @@ -229,7 +248,7 @@ func aMesh(t *testing.T) (*Holder, *memBus, *memDoor, *[]string) { want, _ := json.Marshal(meshwants.WantsByWantNodeModuleValue{V: 1, Fp: fp(1), Self: true}) _ = b.Put(WantsRead, "want.ace.letta", json.RawMessage(want)) var acts []string - h := &Holder{Bus: b, Findings: findingsThrough{b}, Door: func() (Door, error) { return door, nil }, Seats: door.seat, + h := &Holder{Logf: t.Logf, Bus: b, Findings: findingsThrough{b}, Door: func() (Door, error) { return door, nil }, Seats: door.seat, Listed: func(context.Context) (*View, error) { return &View{L: listing}, nil }, Now: func() time.Time { return day.Add(48 * time.Hour) }, Acts: meshhandactsclient.Recorder{Ask: func(key string, body any) (json.RawMessage, error) { @@ -237,9 +256,49 @@ func aMesh(t *testing.T) (*Holder, *memBus, *memDoor, *[]string) { acts = append(acts, key+" "+string(raw)) return json.RawMessage(`{}`), nil }}} + takeIn(h, b) return h, b, door, &acts } +// takeIn hands the builds over as the watch's first view does, then forgets the reads that took: a pass's are counted. +func takeIn(h *Holder, b *memBus) { + handOver(h, b) + h.TakenIn() + b.calls = nil +} + +// handOver is a watch's handover of every current value, not yet taken in. +func handOver(h *Holder, b *memBus) { + h.Watching() + keys, _ := b.Keys(BuildsRead) + for _, k := range keys { + raw, _, _ := b.Get(BuildsRead, k) + h.Heard(stdio.StateChange{State: BuildsRead, Key: k, Op: stdio.OpPut, Value: raw, Current: true}) + } +} + +// keyOf is letta's n-th build's key. +func keyOf(n int) string { k, _ := builds.BuilderBuildKey("letta", fp(n)); return k } + +// viewKeys are the keys the followed view holds. +func viewKeys(t *testing.T, h *Holder) []string { + t.Helper() + h.mu.Lock() + defer h.mu.Unlock() + var out []string + for k := range h.done { + out = append(out, k) + } + sort.Strings(out) + return out +} + +// putBuild writes a build's key and hands it over as the watch's update does. +func putBuild(h *Holder, b *memBus, key string, raw []byte) { + _ = b.Put(BuildsRead, key, json.RawMessage(raw)) + h.Heard(stdio.StateChange{State: BuildsRead, Key: key, Op: stdio.OpPut, Value: raw}) +} + // ADR 0571 clause 2, 1126/A4/AC2: the keep set from the store's listing, the wants and the five newest builds; with no // lease key, a build's end and a confirmed collect let go of the eligible within the bound, leave an index for a // confirmed collect, record each deletion as a hand act and say artifacts-gone. @@ -312,7 +371,7 @@ func TestTheSeatKeepsTheFiveNewestAndTheWantedAndSweepsWithNoLease(t *testing.T) raw, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildDone, Manifest: strings.Repeat("d", 64), Artifacts: []builds.BuiltArtifact{ {Name: "image", Kind: "image", Digest: digestOf(imageRef("letta", n)), Reference: artifactref.SeatReference(reg.Seat, "letta", "image", digestOf(imageRef("letta", n)))}, {Name: "files", Kind: "archive", Digest: digestOf(archiveRef("letta", n)), Reference: artifactref.SeatReference(ociseat.Seat, "letta", "files", digestOf(archiveRef("letta", n)))}}}) - _ = b.Put(BuildsRead, key, json.RawMessage(raw)) + putBuild(h, b, key, raw) } } if c.refuse != "" { @@ -323,7 +382,7 @@ func TestTheSeatKeepsTheFiveNewestAndTheWantedAndSweepsWithNoLease(t *testing.T) } switch c.how { case "build": - h.AfterBuild("letta.x") + h.Pass(context.Background(), "after the build letta.x") var gone []string for _, x := range door.asked { if r, ok := strings.CutPrefix(x, "let go "); ok { @@ -420,6 +479,7 @@ func TestAWantedImageTheStoreLacksIsAFindingAndImagesNamesWantsAndHaves(t *testi view.L.Repos[repo].Revisions[digest] = true // A restart: a holder made afresh holds no finding of its own, and clears what the writer holds (design 47 G A-32, A-38). h = &Holder{Bus: h.Bus, Findings: h.Findings, Door: h.Door, Listed: h.Listed, Seats: h.Seats, Now: h.Now, Acts: h.Acts, Logf: h.Logf} + takeIn(h, b) if err := h.Observe(context.Background()); err != nil { t.Fatal(err) } @@ -482,3 +542,154 @@ func (f findingsThrough) Held() []meshconditions.Finding { func (f findingsThrough) Clear(_ context.Context, scope, id, kind, _ string) error { return f.bus.Delete(FindingsState, views.FindingKey(scope, id, kind)) } + +// writesCounted counts the findings written through the test's writer. +type writesCounted struct { + findingsThrough + writes *int +} + +func (w writesCounted) Write(ctx context.Context, f meshconditions.Finding) error { + *w.writes++ + return w.findingsThrough.Write(ctx, f) +} + +// Design 47 H, H-A-250: the starts fold into one pending pass, a pass reads the builds in one page, and a list the +// registry seats cannot answer ends the pass with artifact-list-unread, written once and asked again only at an event. +func TestOnePendingPassReadsTheBuildsOnceAndEndsOnAnUnreadList(t *testing.T) { + t.Run("three events during a pass run exactly one more", func(t *testing.T) { + h, _, _, _ := aMesh(t) + listed, inner := 0, h.Listed + h.Listed = func(ctx context.Context) (*View, error) { + listed++ + if listed == 1 { + for _, by := range []string{"after the build a", "after the build b", "a want changed"} { + h.Wanted(by) // a pass runs: each only marks it + } + } + return inner(ctx) + } + if !h.mark("after the build letta.x") { + t.Fatal("no pass ran, and the mark did not start one") + } + h.passes("after the build letta.x") + if listed != 2 { + t.Fatalf("three events during a pass ran %d passes, want 2", listed) + } + if h.running || h.wanted { + t.Fatalf("after the passes running is %v and wanted %v", h.running, h.wanted) + } + }) + t.Run("a pass reads nothing of the builds and their ends come from the log", func(t *testing.T) { + h, b, door, _ := aMesh(t) + handOver(h, b) // a fresh take-in, counted: no history read for a build the log holds (H-A-258) + h.TakenIn() + h.Pass(context.Background(), "after the build letta.x") + if n := b.calls["keys "+BuildsRead] - 1 + b.calls["get "+BuildsRead] - 7 + b.calls["history "+BuildsRead]; n != 0 { + t.Fatalf("a take-in and a pass read the builds by %v", b.calls) + } + if !slices.Contains(door.asked, "let go "+imageRef("letta", 2)) { + t.Fatalf("the pass swept nothing: %v", door.asked) + } + // Only a build the log lacks is read from its history, once. + h, b, _, _ = aMesh(t) + b.log = append(b.log[:2:2], b.log[3:]...) // letta's 3rd + h.Pass(context.Background(), "a") + h.Pass(context.Background(), "b") + if b.calls["history "+BuildsRead] != 1 { + t.Fatalf("two passes read the builds' history %d times, want once for the build the log lacks", b.calls["history "+BuildsRead]) + } + }) + t.Run("no pass judges before the first view, and says so", func(t *testing.T) { + h, b, door, _ := aMesh(t) + h.taken = false + h.Pass(context.Background(), "after the build letta.x") + _, found, _ := b.Get(FindingsState, views.FindingKey("module", Module, BuildsUnread)) + if len(door.asked) != 0 || !found { + t.Fatalf("a pass before the builds were taken in asked %v; artifact-builds-unread is written: %v", door.asked, found) + } + }) + t.Run("an unread end holds every pass until it is read", func(t *testing.T) { + h, b, door, _ := aMesh(t) + writes := 0 + h.Findings = writesCounted{findingsThrough{b}, &writes} + b.log = append(b.log[:2:2], b.log[3:]...) // letta's 3rd: the log lacks it, and its history fails + b.historyErr = map[string]error{keyOf(3): errors.New("no responders")} + h.Pass(context.Background(), "a") + h.Pass(context.Background(), "b") + key := views.FindingKey("module", Module, BuildsUnread) + if _, found, _ := b.Get(FindingsState, key); !found || writes != 1 || len(door.asked) != 0 { + t.Fatalf("with an end unread the passes asked %v; the finding is written %v, %d times", door.asked, found, writes) + } + b.historyErr = nil + h.Pass(context.Background(), "c") + if _, found, _ := b.Get(FindingsState, key); found || !slices.Contains(door.asked, "let go "+imageRef("letta", 2)) { + t.Fatalf("once the end was read the pass asked %v and left the finding: %v", door.asked, found) + } + }) + t.Run("a change heard during the handover is in the view it gives", func(t *testing.T) { + h, b, _, _ := aMesh(t) + handOver(h, b) + raw, _, _ := b.Get(BuildsRead, keyOf(1)) + h.Heard(stdio.StateChange{State: BuildsRead, Key: keyOf(8), Op: stdio.OpPut, Value: raw}) // after the handover, before TakenIn + h.Heard(stdio.StateChange{State: BuildsRead, Key: keyOf(3), Op: stdio.OpDelete}) + h.TakenIn() + got := viewKeys(t, h) + if !slices.Contains(got, keyOf(8)) || slices.Contains(got, keyOf(3)) { + t.Fatalf("the view after the handover holds %v", got) + } + }) + t.Run("a key deleted or no longer done leaves the view; one written done again takes its new end", func(t *testing.T) { + h, b, _, _ := aMesh(t) + h.Pass(context.Background(), "a") // every end known + building, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: "building"}) + h.Heard(stdio.StateChange{State: BuildsRead, Key: keyOf(3), Op: stdio.OpDelete}) + h.Heard(stdio.StateChange{State: BuildsRead, Key: keyOf(4), Op: stdio.OpPut, Value: building}) + if got := viewKeys(t, h); slices.Contains(got, keyOf(3)) || slices.Contains(got, keyOf(4)) { + t.Fatalf("the view holds %v", got) + } + later := b.log[len(b.log)-1].At.Add(time.Hour) + e := builds.BuildLogEntry{Event: builds.BuildEndedEvent, Module: "letta", FP: fp(5), Repository: "novox/x", Commit: "c", Node: "ace", At: later, Outcome: "done"} + entry, _ := builds.EncodeBuildLogEntry(e) + b.log = append(b.log, stdio.LogEntry{Sequence: 99, Key: "letta", Value: entry, At: later}) + raw, _, _ := b.Get(BuildsRead, keyOf(5)) + h.Heard(stdio.StateChange{State: BuildsRead, Key: keyOf(5), Op: stdio.OpPut, Value: raw}) + h.Pass(context.Background(), "b") + if got, _ := h.Builds(); !slices.ContainsFunc(got, func(x Build) bool { return x.Key == keyOf(5) && x.At.Equal(later) }) { + t.Fatalf("letta's 5th written done again does not end at %v: %+v", later, got) + } + }) + t.Run("a watch asked again replaces the view whole", func(t *testing.T) { + h, b, _, _ := aMesh(t) + gone, _ := builds.BuilderBuildKey("letta", fp(3)) + _ = b.Delete(BuildsRead, gone) // deleted while no watch ran: no change of it is heard + takeIn(h, b) + got, err := h.Builds() + if err != nil || len(got) != 6 || slices.ContainsFunc(got, func(x Build) bool { return x.Key == gone }) { + t.Fatalf("after the watch was asked again the view holds %d builds (%v), %s among them", len(got), err, gone) + } + }) + t.Run("a list unread ends the pass with its finding once", func(t *testing.T) { + h, b, door, _ := aMesh(t) + writes, listed, inner := 0, 0, h.Listed + h.Findings = writesCounted{findingsThrough{b}, &writes} + h.Listed = func(context.Context) (*View, error) { + listed++ + return nil, errors.New("the answer exceeds the bus's one megabyte") + } + h.Pass(context.Background(), "after the build a") + h.Pass(context.Background(), "a want changed") + key := views.FindingKey("module", Module, ListUnread) + if raw, found, _ := b.Get(FindingsState, key); !found || writes != 1 || !strings.Contains(string(raw), "one megabyte") { + t.Fatalf("the finding is %s (found %v), written %d times", raw, found, writes) + } + if listed != 2 || len(door.asked) != 0 || len(b.published) != 0 { + t.Fatalf("two passes listed %d times, asked %v and said %v", listed, door.asked, b.published) + } + h.Listed = inner + h.Pass(context.Background(), "after the build b") + if _, found, _ := b.Get(FindingsState, key); found || len(door.asked) == 0 { + t.Fatalf("a pass that read the list left the finding (%v) or swept nothing (%v)", found, door.asked) + } + }) +} diff --git a/modules/artifact-store-tools/cmd/artifact-store-tools/main.go b/modules/artifact-store-tools/cmd/artifact-store-tools/main.go index acb1d80d5..c4efb1cdb 100644 --- a/modules/artifact-store-tools/cmd/artifact-store-tools/main.go +++ b/modules/artifact-store-tools/cmd/artifact-store-tools/main.go @@ -32,7 +32,7 @@ func main() { h.Acts = meshhandactsclient.Recorder{Ask: stdio.Ask} // The module's one held writer (design 47 G ยง689, A-16, A-29, A-32): the holder's findings and the reader's go through it. w, err := views.NewStdioHeldWriter(views.WriterConfig[meshconditions.Finding, meshconditions.FindingCleared, meshconditions.FindingsStated, meshconditions.FindingsAskedEvent]{ - Asked: meshconditions.SubjectFindingsAsked, Module: Module, State: stdio.State("findings"), Kinds: []string{ImageMissing, SweepRefused}}) + Asked: meshconditions.SubjectFindingsAsked, Module: Module, State: stdio.State("findings"), Kinds: []string{ImageMissing, SweepRefused, ListUnread, BuildsUnread}}) if err != nil { logf("%v", err) os.Exit(1) diff --git a/modules/artifact-store-tools/cmd/artifact-store-tools/sweepseats.go b/modules/artifact-store-tools/cmd/artifact-store-tools/sweepseats.go index 538783e31..ed540cba8 100644 --- a/modules/artifact-store-tools/cmd/artifact-store-tools/sweepseats.go +++ b/modules/artifact-store-tools/cmd/artifact-store-tools/sweepseats.go @@ -27,6 +27,9 @@ import ( // SweepRefused is the sweep's finding: a seat refused a hold or a remove the sweep asked. const SweepRefused = "sweep-refused" +// ListUnread is the pass's finding: the registry seats' list could not be read, so the pass ended (design 47 H, H-A-250 (3)). +const ListUnread = "artifact-list-unread" + // canonical is a kept reference in the older form, `artifact-store://`, whichever form it came in; another is as // it was. func canonical(reference string) string { @@ -140,6 +143,53 @@ func (h *Holder) sweepClean() { } } +// BuildsUnread is the pass's finding: the builder's builds could not be judged, so the pass ended (design 47 H, H-A-260). +const BuildsUnread = "artifact-builds-unread" + +// unreadFinding writes a pass's finding of what it could not read (ListUnread, BuildsUnread) once: a pass that fails the +// same way while the writer holds it writes nothing more. Nothing asks again until the next event marks a pass. +func (h *Holder) unreadFinding(kind string, err error) { + for _, was := range h.Findings.Held() { + if was.Kind == kind { + return + } + } + at := h.now().UTC() + f := meshconditions.Finding{Scope: "module", ID: keys.EscapeName(Module), Kind: kind, + Subject: meshconditions.Subject{Scope: "module", ID: keys.EscapeName(Module)}, + Severity: meshconditions.FindingSeverityWarning, Resolver: "operator", + Summary: "the artifact sweep stopped: the registry seats' list could not be read (the evidence names why)", + Headline: "The artifact sweep cannot read the store's list", + Explanation: "The sweep judges what to keep against the store's list, which it asks the registry seats. The list " + + "could not be read, so the sweep removed nothing and observed nothing; what it kept stays kept.", + Needs: "read the registry seat holder's journal for why its list failed; the next finished build or want change asks again.", + Evidence: []meshconditions.Evidence{{At: at, Said: err.Error()}}, + Source: "CAT-AS1", Observer: Module, At: at} + if kind == BuildsUnread { + f.Summary = "the artifact sweep stopped: the builder's builds could not be judged (the evidence names why)" + f.Headline = "The artifact sweep cannot judge the builds" + f.Explanation = "The sweep keeps each module's newest builds and every wanted one, judged from the builder's builds " + + "and when each ended. Those could not all be read, so the sweep removed nothing and observed nothing; what " + + "it kept stays kept." + f.Needs = "read this module's journal for which build's end is unread; the next finished build or want change asks again." + } + if werr := h.Findings.Write(context.Background(), f); werr != nil { + h.logf("the pass's finding %s (%v) could not be written: %v", kind, err, werr) + } +} + +// readAgain clears a pass's finding of kind, when the held writer holds it, once a pass read what it could not. +func (h *Holder) readAgain(kind string) { + for _, was := range h.Findings.Held() { + if was.Kind != kind { + continue + } + if err := h.Findings.Clear(context.Background(), was.Scope, was.ID, was.Kind, "a pass read it again"); err != nil { + h.logf("the pass's finding %s could not be cleared: %v", kind, err) + } + } +} + // keptWhy is why a kept reference is held, as the seat's hold takes it. func keptWhy(r *Ref) string { if r == nil || len(r.Why) == 0 {