diff --git a/modules/artifact-store-tools/README.md b/modules/artifact-store-tools/README.md index 085a3eb7d..586f58479 100644 --- a/modules/artifact-store-tools/README.md +++ b/modules/artifact-store-tools/README.md @@ -124,9 +124,11 @@ seat refuses clears it (`sweepseats.go`). A finished build or a changed want marks the one pending pass (design 47 H, H-A-250); a pass running when one is marked runs once more when it ends, however many events marked it. The builder's builds are a followed view: the watch's first view -at start, then its updates, kept in memory (H-A-255), each build's end read once from its key's history as it is taken -in; a watch asked again after an error replaces the view whole with its handover (H-A-257). A pass reads nothing +at start, then its updates, kept in memory (H-A-255), each build's end taken from the builder's log's build-ended entries the +pass reads once, and only for a build the log lacks from its key's history (H-A-258); a watch asked again after an error replaces the view whole with its handover (H-A-257). A pass reads nothing of them on the bus, judges nothing before the first view is in, and sweeps and observes S25 from that one judgement. A list the registry seats cannot answer ends the pass with the finding `artifact-list-unread`, written once; nothing asks the list again -until the next event, and a pass that reads it clears the finding (`holder.go`, `sweepseats.go`). +until the next event, and a pass that reads it clears the finding. A pass that cannot judge the builds (the view not yet taken in, or a +done build's end unread) ends the same way with the finding `artifact-builds-unread` (H-A-260) (`holder.go`, +`sweepseats.go`). diff --git a/modules/artifact-store-tools/cmd/artifact-store-tools/holder.go b/modules/artifact-store-tools/cmd/artifact-store-tools/holder.go index 70df322b2..007d88238 100644 --- a/modules/artifact-store-tools/cmd/artifact-store-tools/holder.go +++ b/modules/artifact-store-tools/cmd/artifact-store-tools/holder.go @@ -38,6 +38,10 @@ import ( "git.novox.be/novox/mesh-sdk/go/views" ) +// ErrBuildsUnread is a judgement refused because the builds cannot be judged: the followed view not yet taken in, or a +// done key's end unread (H-A-258, H-A-260). +var ErrBuildsUnread = errors.New("the builder's builds cannot be judged") + // What the holder reads and keeps. const ( WantsRead = assignerModule + ".wants" @@ -119,25 +123,28 @@ type Build struct { Manifest string } -// Builds is every done build of the followed view, by key, each with its end: read from memory, never from the bus -// (design 47 H, H-A-250 (2), H-A-255). Before the view's first view is taken in there is none to judge from, and it says so, -// so no pass sweeps against an empty keep set. +// Builds is every done build of the followed view, by key, read from memory, never from the bus (design 47 H, H-A-250 +// (2), H-A-255): each with the end known for it, zero where none is known yet (withEnds gives it). Before the view's first +// view is taken in there is none to judge from, and it says so, so no pass sweeps against an empty keep set. func (h *Holder) Builds() ([]Build, error) { h.mu.Lock() defer h.mu.Unlock() if !h.taken { - return nil, errors.New("the builder's builds are not taken in yet: their first view has not come") + return nil, fmt.Errorf("%w: they are not taken in yet, their first view has not come", ErrBuildsUnread) } out := make([]Build, 0, len(h.done)) for _, b := range h.done { + b.At = h.ended[b.Key] out = append(out, b) } sort.Slice(out, func(a, b int) bool { return out[a].Key < out[b].Key }) return out, nil } -// Heard takes one change of the builder's builds into the followed view: a done value is kept with its end (read once from -// the key's history, again when the key is written done anew), anything else drops the key. +// Heard takes one change of the builder's builds into the followed view: a done value is kept, anything else drops the +// key. Its end is not read here: withEnds takes it from the builder's log at the next pass (H-A-258); a key written done +// anew forgets the end it had. While a watch's handover is gathered (Watching), every change goes into it, current or +// not, so nothing heard before TakenIn is lost by the swap (H-A-257). func (h *Holder) Heard(c stdio.StateChange) { module, fp, err := builds.ParseBuilderBuildKey(c.Key) if err != nil { @@ -147,34 +154,23 @@ func (h *Holder) Heard(c stdio.StateChange) { if c.Op == stdio.OpPut { b, err = builds.DecodeBuilderBuild(c.Value) } - if c.Op != stdio.OpPut || err != nil || b.State != builds.BuildDone { - h.mu.Lock() - delete(h.done, c.Key) - delete(h.ended, c.Key) - h.mu.Unlock() - return - } - if !c.Current { - h.mu.Lock() - delete(h.ended, c.Key) // written done anew: its end is this write's - h.mu.Unlock() - } - at, err := h.endOf(c.Key) - if err != nil { - h.logf("%s is left out of the keep set, its end unread: %v", c.Key, err) - return - } h.mu.Lock() + defer h.mu.Unlock() + if !c.Current { + delete(h.ended, c.Key) // written anew or gone: an end it had is no longer its + } + if h.next == nil && h.done == nil { + h.done = map[string]Build{} + } into := h.done - if c.Current && h.next != nil { + if h.next != nil { into = h.next } - if into == nil { - h.done = map[string]Build{} - into = h.done + if c.Op != stdio.OpPut || err != nil || b.State != builds.BuildDone { + delete(into, c.Key) + return } - into[c.Key] = Build{Module: module, FP: fp, Key: c.Key, At: at, Artifacts: b.Artifacts, Manifest: b.Manifest} - h.mu.Unlock() + into[c.Key] = Build{Module: module, FP: fp, Key: c.Key, Artifacts: b.Artifacts, Manifest: b.Manifest} } // Watching says a watch of the builds is asked, first or again after an error: its handover of current values is @@ -201,7 +197,7 @@ func (h *Holder) TakenIn() { h.taken = true } -// endOf is when a done key was last written done, read once from its history: a done key is not written again. +// endOf is when a done key was last written done, read from its history and kept: for a build the builder's log lacks. func (h *Holder) endOf(key string) (time.Time, error) { h.mu.Lock() at, known := h.ended[key] @@ -390,6 +386,15 @@ func (h *Holder) judge(ctx context.Context) (*judged, error) { if err != nil { return nil, err } + // The builder's log, read once: what each build stood on, and each build's end (H-A-258). + against, ends, logErr := h.logged() + if logErr != nil { + h.logf("the builder's log could not be read, so every copy is kept as unrecorded and each end is read from its history: %v", logErr) + against, ends = nil, nil + } + if builds, err = h.withEnds(builds, ends); err != nil { + return nil, err + } wants, err := h.Wants() if err != nil { return nil, err @@ -402,11 +407,6 @@ func (h *Holder) judge(ctx context.Context) (*judged, error) { } } if j.view != nil { - against, err := h.against() - if err != nil { - h.logf("the builder's log could not be read, so every copy is kept as unrecorded: %v", err) - against = nil - } j.mirrors, j.mirrorsHeld = Mirrors(j.view, builds, keptBuilds(builds, wants), against) for r, m := range j.mirrors { if len(m.Builds) > 0 { @@ -417,13 +417,47 @@ func (h *Holder) judge(ctx context.Context) (*judged, error) { return j, nil } -// against is what each build the builder's log says ended stood on, by build key: the bases' references. -func (h *Holder) against() (map[string][]string, error) { +// withEnds gives each build its end: the one known, else the builder's log's build-ended entry, else its key's history, +// each kept once read (H-A-258). A done key whose end can be read from neither stays unread, and then nothing is judged: +// never from a view missing a build. +func (h *Holder) withEnds(built []Build, ends map[string]time.Time) ([]Build, error) { + var unread []string + var why error + for i := range built { + if !built[i].At.IsZero() { + continue + } + if at, logged := ends[built[i].Key]; logged { + h.mu.Lock() + if h.ended == nil { + h.ended = map[string]time.Time{} + } + h.ended[built[i].Key] = at + h.mu.Unlock() + built[i].At = at + continue + } + at, err := h.endOf(built[i].Key) + if err != nil { + unread, why = append(unread, built[i].Key), err + continue + } + built[i].At = at + } + if len(unread) > 0 { + return nil, fmt.Errorf("%w: the end of %d done builds could not be read (%s first), so nothing is judged: %v", ErrBuildsUnread, len(unread), unread[0], why) + } + return built, nil +} + +// logged is what each build the builder's log says ended stood on, by build key (the bases' references), and when it +// ended: one read of the log. +func (h *Holder) logged() (map[string][]string, map[string]time.Time, error) { entries, err := h.Bus.LogSince(builds.BuildLogRead, h.now().Add(-builds.BuilderLogDays*24*time.Hour)) if err != nil { - return nil, err + return nil, nil, err } - out := map[string][]string{} + out, ends := map[string][]string{}, map[string]time.Time{} for _, raw := range entries { e, err := builds.DecodeBuildLogEntry(raw.Value) if err != nil || e.Event != builds.BuildEndedEvent { @@ -434,8 +468,11 @@ func (h *Holder) against() (map[string][]string, error) { continue } out[key] = e.Against + if e.At.After(ends[key]) { + ends[key] = e.At + } } - return out, nil + return out, ends, nil } // Mirrors judges each copy in the store's mirror repositories (upstream//, /on-; ADR @@ -771,14 +808,18 @@ func (h *Holder) Pass(ctx context.Context, by string) { if err != nil { h.sweep.Unlock() h.logf("the pass %s: %v", by, err) + if errors.Is(err, ErrBuildsUnread) { + h.unreadFinding(BuildsUnread, err) // H-A-260 + } return } + h.readAgain(BuildsUnread) if j.unread != nil { h.sweep.Unlock() - h.listUnread(j.unread) + h.unreadFinding(ListUnread, j.unread) return } - h.listRead() + h.readAgain(ListUnread) a, err := h.sweepFrom(ctx, j, Sweeping{Most: MostPerSweep, By: by}) h.sweep.Unlock() switch { diff --git a/modules/artifact-store-tools/cmd/artifact-store-tools/holder_test.go b/modules/artifact-store-tools/cmd/artifact-store-tools/holder_test.go index 75fb3747c..c92bb5b9a 100644 --- a/modules/artifact-store-tools/cmd/artifact-store-tools/holder_test.go +++ b/modules/artifact-store-tools/cmd/artifact-store-tools/holder_test.go @@ -34,6 +34,8 @@ type memBus struct { history map[string][]stdio.StateRevision log []stdio.LogEntry published []string + // historyErr fails the history read of a key. + historyErr map[string]error // calls counts each read by " ": a pass reads nothing of the builds, judged from the followed view. calls map[string]int } @@ -79,6 +81,9 @@ func (b *memBus) History(state, key string) ([]stdio.StateRevision, error) { b.mu.Lock() defer b.mu.Unlock() b.count("history", state) + if err := b.historyErr[key]; err != nil { + return nil, err + } return b.history[state+" "+key], nil } @@ -257,14 +262,35 @@ func aMesh(t *testing.T) (*Holder, *memBus, *memDoor, *[]string) { // takeIn hands the builds over as the watch's first view does, then forgets the reads that took: a pass's are counted. func takeIn(h *Holder, b *memBus) { + handOver(h, b) + h.TakenIn() + b.calls = nil +} + +// handOver is a watch's handover of every current value, not yet taken in. +func handOver(h *Holder, b *memBus) { h.Watching() keys, _ := b.Keys(BuildsRead) for _, k := range keys { raw, _, _ := b.Get(BuildsRead, k) h.Heard(stdio.StateChange{State: BuildsRead, Key: k, Op: stdio.OpPut, Value: raw, Current: true}) } - h.TakenIn() - b.calls = nil +} + +// keyOf is letta's n-th build's key. +func keyOf(n int) string { k, _ := builds.BuilderBuildKey("letta", fp(n)); return k } + +// viewKeys are the keys the followed view holds. +func viewKeys(t *testing.T, h *Holder) []string { + t.Helper() + h.mu.Lock() + defer h.mu.Unlock() + var out []string + for k := range h.done { + out = append(out, k) + } + sort.Strings(out) + return out } // putBuild writes a build's key and hands it over as the watch's update does. @@ -554,22 +580,83 @@ func TestOnePendingPassReadsTheBuildsOnceAndEndsOnAnUnreadList(t *testing.T) { t.Fatalf("after the passes running is %v and wanted %v", h.running, h.wanted) } }) - t.Run("a pass reads nothing of the builds", func(t *testing.T) { + t.Run("a pass reads nothing of the builds and their ends come from the log", func(t *testing.T) { h, b, door, _ := aMesh(t) + handOver(h, b) // a fresh take-in, counted: no history read for a build the log holds (H-A-258) + h.TakenIn() h.Pass(context.Background(), "after the build letta.x") - if n := b.calls["keys "+BuildsRead] + b.calls["get "+BuildsRead] + b.calls["history "+BuildsRead]; n != 0 { - t.Fatalf("a pass read the builds by %v", b.calls) + if n := b.calls["keys "+BuildsRead] - 1 + b.calls["get "+BuildsRead] - 7 + b.calls["history "+BuildsRead]; n != 0 { + t.Fatalf("a take-in and a pass read the builds by %v", b.calls) } if !slices.Contains(door.asked, "let go "+imageRef("letta", 2)) { t.Fatalf("the pass swept nothing: %v", door.asked) } + // Only a build the log lacks is read from its history, once. + h, b, _, _ = aMesh(t) + b.log = append(b.log[:2:2], b.log[3:]...) // letta's 3rd + h.Pass(context.Background(), "a") + h.Pass(context.Background(), "b") + if b.calls["history "+BuildsRead] != 1 { + t.Fatalf("two passes read the builds' history %d times, want once for the build the log lacks", b.calls["history "+BuildsRead]) + } }) - t.Run("no pass judges before the first view", func(t *testing.T) { - h, _, door, _ := aMesh(t) + t.Run("no pass judges before the first view, and says so", func(t *testing.T) { + h, b, door, _ := aMesh(t) h.taken = false h.Pass(context.Background(), "after the build letta.x") - if len(door.asked) != 0 { - t.Fatalf("a pass before the builds were taken in asked %v", door.asked) + _, found, _ := b.Get(FindingsState, views.FindingKey("module", Module, BuildsUnread)) + if len(door.asked) != 0 || !found { + t.Fatalf("a pass before the builds were taken in asked %v; artifact-builds-unread is written: %v", door.asked, found) + } + }) + t.Run("an unread end holds every pass until it is read", func(t *testing.T) { + h, b, door, _ := aMesh(t) + writes := 0 + h.Findings = writesCounted{findingsThrough{b}, &writes} + b.log = append(b.log[:2:2], b.log[3:]...) // letta's 3rd: the log lacks it, and its history fails + b.historyErr = map[string]error{keyOf(3): errors.New("no responders")} + h.Pass(context.Background(), "a") + h.Pass(context.Background(), "b") + key := views.FindingKey("module", Module, BuildsUnread) + if _, found, _ := b.Get(FindingsState, key); !found || writes != 1 || len(door.asked) != 0 { + t.Fatalf("with an end unread the passes asked %v; the finding is written %v, %d times", door.asked, found, writes) + } + b.historyErr = nil + h.Pass(context.Background(), "c") + if _, found, _ := b.Get(FindingsState, key); found || !slices.Contains(door.asked, "let go "+imageRef("letta", 2)) { + t.Fatalf("once the end was read the pass asked %v and left the finding: %v", door.asked, found) + } + }) + t.Run("a change heard during the handover is in the view it gives", func(t *testing.T) { + h, b, _, _ := aMesh(t) + handOver(h, b) + raw, _, _ := b.Get(BuildsRead, keyOf(1)) + h.Heard(stdio.StateChange{State: BuildsRead, Key: keyOf(8), Op: stdio.OpPut, Value: raw}) // after the handover, before TakenIn + h.Heard(stdio.StateChange{State: BuildsRead, Key: keyOf(3), Op: stdio.OpDelete}) + h.TakenIn() + got := viewKeys(t, h) + if !slices.Contains(got, keyOf(8)) || slices.Contains(got, keyOf(3)) { + t.Fatalf("the view after the handover holds %v", got) + } + }) + t.Run("a key deleted or no longer done leaves the view; one written done again takes its new end", func(t *testing.T) { + h, b, _, _ := aMesh(t) + h.Pass(context.Background(), "a") // every end known + building, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: "building"}) + h.Heard(stdio.StateChange{State: BuildsRead, Key: keyOf(3), Op: stdio.OpDelete}) + h.Heard(stdio.StateChange{State: BuildsRead, Key: keyOf(4), Op: stdio.OpPut, Value: building}) + if got := viewKeys(t, h); slices.Contains(got, keyOf(3)) || slices.Contains(got, keyOf(4)) { + t.Fatalf("the view holds %v", got) + } + later := b.log[len(b.log)-1].At.Add(time.Hour) + e := builds.BuildLogEntry{Event: builds.BuildEndedEvent, Module: "letta", FP: fp(5), Repository: "novox/x", Commit: "c", Node: "ace", At: later, Outcome: "done"} + entry, _ := builds.EncodeBuildLogEntry(e) + b.log = append(b.log, stdio.LogEntry{Sequence: 99, Key: "letta", Value: entry, At: later}) + raw, _, _ := b.Get(BuildsRead, keyOf(5)) + h.Heard(stdio.StateChange{State: BuildsRead, Key: keyOf(5), Op: stdio.OpPut, Value: raw}) + h.Pass(context.Background(), "b") + if got, _ := h.Builds(); !slices.ContainsFunc(got, func(x Build) bool { return x.Key == keyOf(5) && x.At.Equal(later) }) { + t.Fatalf("letta's 5th written done again does not end at %v: %+v", later, got) } }) t.Run("a watch asked again replaces the view whole", func(t *testing.T) { diff --git a/modules/artifact-store-tools/cmd/artifact-store-tools/main.go b/modules/artifact-store-tools/cmd/artifact-store-tools/main.go index 00736089f..c4efb1cdb 100644 --- a/modules/artifact-store-tools/cmd/artifact-store-tools/main.go +++ b/modules/artifact-store-tools/cmd/artifact-store-tools/main.go @@ -32,7 +32,7 @@ func main() { h.Acts = meshhandactsclient.Recorder{Ask: stdio.Ask} // The module's one held writer (design 47 G ยง689, A-16, A-29, A-32): the holder's findings and the reader's go through it. w, err := views.NewStdioHeldWriter(views.WriterConfig[meshconditions.Finding, meshconditions.FindingCleared, meshconditions.FindingsStated, meshconditions.FindingsAskedEvent]{ - Asked: meshconditions.SubjectFindingsAsked, Module: Module, State: stdio.State("findings"), Kinds: []string{ImageMissing, SweepRefused, ListUnread}}) + Asked: meshconditions.SubjectFindingsAsked, Module: Module, State: stdio.State("findings"), Kinds: []string{ImageMissing, SweepRefused, ListUnread, BuildsUnread}}) if err != nil { logf("%v", err) os.Exit(1) diff --git a/modules/artifact-store-tools/cmd/artifact-store-tools/sweepseats.go b/modules/artifact-store-tools/cmd/artifact-store-tools/sweepseats.go index e64e97be8..ed540cba8 100644 --- a/modules/artifact-store-tools/cmd/artifact-store-tools/sweepseats.go +++ b/modules/artifact-store-tools/cmd/artifact-store-tools/sweepseats.go @@ -143,16 +143,19 @@ func (h *Holder) sweepClean() { } } -// listUnread writes the pass's finding artifact-list-unread once: a pass whose list fails again while the writer holds it -// writes nothing more. Nothing asks the list again until the next event marks a pass. -func (h *Holder) listUnread(err error) { +// BuildsUnread is the pass's finding: the builder's builds could not be judged, so the pass ended (design 47 H, H-A-260). +const BuildsUnread = "artifact-builds-unread" + +// unreadFinding writes a pass's finding of what it could not read (ListUnread, BuildsUnread) once: a pass that fails the +// same way while the writer holds it writes nothing more. Nothing asks again until the next event marks a pass. +func (h *Holder) unreadFinding(kind string, err error) { for _, was := range h.Findings.Held() { - if was.Kind == ListUnread { + if was.Kind == kind { return } } at := h.now().UTC() - f := meshconditions.Finding{Scope: "module", ID: keys.EscapeName(Module), Kind: ListUnread, + f := meshconditions.Finding{Scope: "module", ID: keys.EscapeName(Module), Kind: kind, Subject: meshconditions.Subject{Scope: "module", ID: keys.EscapeName(Module)}, Severity: meshconditions.FindingSeverityWarning, Resolver: "operator", Summary: "the artifact sweep stopped: the registry seats' list could not be read (the evidence names why)", @@ -162,19 +165,27 @@ func (h *Holder) listUnread(err error) { Needs: "read the registry seat holder's journal for why its list failed; the next finished build or want change asks again.", Evidence: []meshconditions.Evidence{{At: at, Said: err.Error()}}, Source: "CAT-AS1", Observer: Module, At: at} + if kind == BuildsUnread { + f.Summary = "the artifact sweep stopped: the builder's builds could not be judged (the evidence names why)" + f.Headline = "The artifact sweep cannot judge the builds" + f.Explanation = "The sweep keeps each module's newest builds and every wanted one, judged from the builder's builds " + + "and when each ended. Those could not all be read, so the sweep removed nothing and observed nothing; what " + + "it kept stays kept." + f.Needs = "read this module's journal for which build's end is unread; the next finished build or want change asks again." + } if werr := h.Findings.Write(context.Background(), f); werr != nil { - h.logf("the pass's finding (%v) could not be written: %v", err, werr) + h.logf("the pass's finding %s (%v) could not be written: %v", kind, err, werr) } } -// listRead clears artifact-list-unread, when the held writer holds it, once a pass read the list. -func (h *Holder) listRead() { +// readAgain clears a pass's finding of kind, when the held writer holds it, once a pass read what it could not. +func (h *Holder) readAgain(kind string) { for _, was := range h.Findings.Held() { - if was.Kind != ListUnread { + if was.Kind != kind { continue } - if err := h.Findings.Clear(context.Background(), was.Scope, was.ID, was.Kind, "a pass read the registry seats' list"); err != nil { - h.logf("the pass's finding could not be cleared: %v", err) + if err := h.Findings.Clear(context.Background(), was.Scope, was.ID, was.Kind, "a pass read it again"); err != nil { + h.logf("the pass's finding %s could not be cleared: %v", kind, err) } } }