Merge pull request 'Design 47 H1: the artifact sweep runs one pending pass from a followed view of the builds (H-A-250)' (#727) from fix/47-H1-artifact-sweep-one-pass into main
This commit was merged in pull request #727.
This commit is contained in:
@@ -119,3 +119,16 @@ removed for `retention` on its seat (`release`, `remove`). It reads a reference
|
||||
and `<seat>/<module>/<artifact>@<digest>`, as one artifact, and says artifacts-gone in the form the build named it. A
|
||||
call a seat refuses stops the sweep and writes the finding `sweep-refused` in the module's findings; the next sweep no
|
||||
seat refuses clears it (`sweepseats.go`).
|
||||
|
||||
### One pending pass
|
||||
|
||||
A finished build or a changed want marks the one pending pass (design 47 H, H-A-250); a pass running when one is marked
|
||||
runs once more when it ends, however many events marked it. The builder's builds are a followed view: the watch's first view
|
||||
at start, then its updates, kept in memory (H-A-255), each build's end taken from the builder's log's build-ended entries the
|
||||
pass reads once, and only for a build the log lacks from its key's history (H-A-258); a watch asked again after an error replaces the view whole with its handover (H-A-257). A pass reads nothing
|
||||
of them on the bus, judges nothing before the first view is in, and sweeps and observes S25
|
||||
from that one judgement. A list the registry
|
||||
seats cannot answer ends the pass with the finding `artifact-list-unread`, written once; nothing asks the list again
|
||||
until the next event, and a pass that reads it clears the finding. A pass that cannot judge the builds (the view not yet taken in, or a
|
||||
done build's end unread) ends the same way with the finding `artifact-builds-unread` (H-A-260) (`holder.go`,
|
||||
`sweepseats.go`).
|
||||
|
||||
@@ -116,12 +116,16 @@ func (runtimeBus) Publish(subject string, body any) error {
|
||||
return stdio.SeatPublish(subject, body, "")
|
||||
}
|
||||
|
||||
// follow watches the builder's done keys and the wants: a build's end sweeps and
|
||||
// observes again; a want changed observes again. Event-driven: nothing reads them on a timer.
|
||||
// follow watches the builder's done keys and the wants: the builds are a followed view, the watch's first view then its
|
||||
// updates kept in memory (design 47 H, H-A-255); a build's end or a want changed marks the one pending pass (H-A-250 (1)),
|
||||
// which sweeps and observes from the views. Event-driven: nothing reads them on a timer.
|
||||
func follow(h *Holder, v *views.Reader) {
|
||||
<-stdio.Served()
|
||||
watch := func(state string, handle func(stdio.StateChange) error) {
|
||||
for {
|
||||
if state == BuildsRead {
|
||||
h.Watching() // each handover, the first or one asked again, replaces the view whole (H-A-257)
|
||||
}
|
||||
if err := stdio.State(state).Watch("", handle); err == nil {
|
||||
return
|
||||
} else {
|
||||
@@ -130,12 +134,8 @@ func follow(h *Holder, v *views.Reader) {
|
||||
time.Sleep(10 * time.Second)
|
||||
}
|
||||
}
|
||||
observe := func() {
|
||||
if err := h.Observe(context.Background()); err != nil {
|
||||
h.logf("image-missing could not be observed: %v", err)
|
||||
}
|
||||
}
|
||||
watch(BuildsRead, func(c stdio.StateChange) error {
|
||||
h.Heard(c)
|
||||
if c.Current || c.Op != stdio.OpPut {
|
||||
return nil
|
||||
}
|
||||
@@ -143,10 +143,12 @@ func follow(h *Holder, v *views.Reader) {
|
||||
State string `json:"state"`
|
||||
}
|
||||
if json.Unmarshal(c.Value, &b) == nil && b.State == "done" {
|
||||
go func() { h.AfterBuild(c.Key); observe() }()
|
||||
h.Wanted("after the build " + c.Key)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
// The watch hands every current value over before it returns: the first view is in.
|
||||
h.TakenIn()
|
||||
// The wants are the assigner's: the first view then its events (ADR 0615). What Watch hands over before it returns
|
||||
// is the first view, not a change.
|
||||
var current atomic.Bool
|
||||
@@ -154,7 +156,7 @@ func follow(h *Holder, v *views.Reader) {
|
||||
for {
|
||||
err := v.Watch(WantsRead, func(string, []byte, bool) {
|
||||
if !current.Load() {
|
||||
go observe()
|
||||
h.Wanted("a want changed")
|
||||
}
|
||||
})
|
||||
if err == nil {
|
||||
@@ -164,5 +166,5 @@ func follow(h *Holder, v *views.Reader) {
|
||||
time.Sleep(10 * time.Second)
|
||||
}
|
||||
current.Store(false)
|
||||
observe()
|
||||
h.Wanted("the first view of the wants")
|
||||
}
|
||||
|
||||
@@ -38,6 +38,10 @@ import (
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
)
|
||||
|
||||
// ErrBuildsUnread is a judgement refused because the builds cannot be judged: the followed view not yet taken in, or a
|
||||
// done key's end unread (H-A-258, H-A-260).
|
||||
var ErrBuildsUnread = errors.New("the builder's builds cannot be judged")
|
||||
|
||||
// What the holder reads and keeps.
|
||||
const (
|
||||
WantsRead = assignerModule + ".wants"
|
||||
@@ -95,8 +99,19 @@ type Holder struct {
|
||||
|
||||
sweep sync.Mutex // one sweep at a time
|
||||
mu sync.Mutex
|
||||
fmu sync.Mutex // one pass of Observe at a time
|
||||
ended map[string]time.Time // a done key's end, read once from its history
|
||||
// pmu guards the pending pass (H-A-250 (1)): running while a pass runs, wanted when an event marked it since.
|
||||
pmu sync.Mutex
|
||||
running bool
|
||||
wanted bool
|
||||
wantedBy string
|
||||
fmu sync.Mutex // one pass of Observe at a time
|
||||
ended map[string]time.Time // a done key's end, read once from its history
|
||||
// done is the followed view of the builder's builds (design 47 H, H-A-255): the done builds by key, taken in from the
|
||||
// watch's first view and its updates (Heard); taken says the first view is in. A pass judges from it, reading nothing.
|
||||
done map[string]Build
|
||||
taken bool
|
||||
// next is the handover a watch asked (again) is giving, which replaces done whole once it is in (H-A-257).
|
||||
next map[string]Build
|
||||
}
|
||||
|
||||
// Build is one done build of a module, as its key holds it, with when it ended.
|
||||
@@ -108,39 +123,81 @@ type Build struct {
|
||||
Manifest string
|
||||
}
|
||||
|
||||
// Builds is every done build the builder's keys hold, each with its end.
|
||||
// Builds is every done build of the followed view, by key, read from memory, never from the bus (design 47 H, H-A-250
|
||||
// (2), H-A-255): each with the end known for it, zero where none is known yet (withEnds gives it). Before the view's first
|
||||
// view is taken in there is none to judge from, and it says so, so no pass sweeps against an empty keep set.
|
||||
func (h *Holder) Builds() ([]Build, error) {
|
||||
keys, err := h.Bus.Keys(BuildsRead)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the builder's builds: %w", err)
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
if !h.taken {
|
||||
return nil, fmt.Errorf("%w: they are not taken in yet, their first view has not come", ErrBuildsUnread)
|
||||
}
|
||||
var out []Build
|
||||
for _, k := range keys {
|
||||
module, fp, err := builds.ParseBuilderBuildKey(k)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
raw, found, err := h.Bus.Get(BuildsRead, k)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the builder's build %s: %w", k, err)
|
||||
}
|
||||
if !found {
|
||||
continue
|
||||
}
|
||||
b, err := builds.DecodeBuilderBuild(raw)
|
||||
if err != nil || b.State != builds.BuildDone {
|
||||
continue
|
||||
}
|
||||
at, err := h.endOf(k)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, Build{Module: module, FP: fp, Key: k, At: at, Artifacts: b.Artifacts, Manifest: b.Manifest})
|
||||
out := make([]Build, 0, len(h.done))
|
||||
for _, b := range h.done {
|
||||
b.At = h.ended[b.Key]
|
||||
out = append(out, b)
|
||||
}
|
||||
sort.Slice(out, func(a, b int) bool { return out[a].Key < out[b].Key })
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// endOf is when a done key was last written done, read once from its history: a done key is not written again.
|
||||
// Heard takes one change of the builder's builds into the followed view: a done value is kept, anything else drops the
|
||||
// key. Its end is not read here: withEnds takes it from the builder's log at the next pass (H-A-258); a key written done
|
||||
// anew forgets the end it had. While a watch's handover is gathered (Watching), every change goes into it, current or
|
||||
// not, so nothing heard before TakenIn is lost by the swap (H-A-257).
|
||||
func (h *Holder) Heard(c stdio.StateChange) {
|
||||
module, fp, err := builds.ParseBuilderBuildKey(c.Key)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
var b builds.BuilderBuild
|
||||
if c.Op == stdio.OpPut {
|
||||
b, err = builds.DecodeBuilderBuild(c.Value)
|
||||
}
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
if !c.Current {
|
||||
delete(h.ended, c.Key) // written anew or gone: an end it had is no longer its
|
||||
}
|
||||
if h.next == nil && h.done == nil {
|
||||
h.done = map[string]Build{}
|
||||
}
|
||||
into := h.done
|
||||
if h.next != nil {
|
||||
into = h.next
|
||||
}
|
||||
if c.Op != stdio.OpPut || err != nil || b.State != builds.BuildDone {
|
||||
delete(into, c.Key)
|
||||
return
|
||||
}
|
||||
into[c.Key] = Build{Module: module, FP: fp, Key: c.Key, Artifacts: b.Artifacts, Manifest: b.Manifest}
|
||||
}
|
||||
|
||||
// Watching says a watch of the builds is asked, first or again after an error: its handover of current values is
|
||||
// gathered apart and replaces the view whole at TakenIn, so a key deleted while no watch ran leaves it (H-A-257).
|
||||
func (h *Holder) Watching() {
|
||||
h.mu.Lock()
|
||||
h.next = map[string]Build{}
|
||||
h.mu.Unlock()
|
||||
}
|
||||
|
||||
// TakenIn says the watch's handover is in: every current value was Heard. It replaces the view whole, and forgets the
|
||||
// end of every key it no longer holds.
|
||||
func (h *Holder) TakenIn() {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
if h.next != nil {
|
||||
h.done, h.next = h.next, nil
|
||||
for k := range h.ended {
|
||||
if _, kept := h.done[k]; !kept {
|
||||
delete(h.ended, k)
|
||||
}
|
||||
}
|
||||
}
|
||||
h.taken = true
|
||||
}
|
||||
|
||||
// endOf is when a done key was last written done, read from its history and kept: for a build the builder's log lacks.
|
||||
func (h *Holder) endOf(key string) (time.Time, error) {
|
||||
h.mu.Lock()
|
||||
at, known := h.ended[key]
|
||||
@@ -320,6 +377,8 @@ type judged struct {
|
||||
mirrors map[string]*Ref
|
||||
// mirrorsHeld says why no copy is eligible now, when one is not: a kept build whose bases the log does not name.
|
||||
mirrorsHeld string
|
||||
// unread is why the registry seats' list could not be read, nil when it was (or none is read).
|
||||
unread error
|
||||
}
|
||||
|
||||
func (h *Holder) judge(ctx context.Context) (*judged, error) {
|
||||
@@ -327,6 +386,15 @@ func (h *Holder) judge(ctx context.Context) (*judged, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// The builder's log, read once: what each build stood on, and each build's end (H-A-258).
|
||||
against, ends, logErr := h.logged()
|
||||
if logErr != nil {
|
||||
h.logf("the builder's log could not be read, so every copy is kept as unrecorded and each end is read from its history: %v", logErr)
|
||||
against, ends = nil, nil
|
||||
}
|
||||
if builds, err = h.withEnds(builds, ends); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
wants, err := h.Wants()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -335,15 +403,10 @@ func (h *Holder) judge(ctx context.Context) (*judged, error) {
|
||||
if h.Listed != nil {
|
||||
if j.view, err = h.Listed(ctx); err != nil {
|
||||
h.logf("the store's listing could not be read, so nothing is said gone: %v", err)
|
||||
j.view = nil
|
||||
j.view, j.unread = nil, err
|
||||
}
|
||||
}
|
||||
if j.view != nil {
|
||||
against, err := h.against()
|
||||
if err != nil {
|
||||
h.logf("the builder's log could not be read, so every copy is kept as unrecorded: %v", err)
|
||||
against = nil
|
||||
}
|
||||
j.mirrors, j.mirrorsHeld = Mirrors(j.view, builds, keptBuilds(builds, wants), against)
|
||||
for r, m := range j.mirrors {
|
||||
if len(m.Builds) > 0 {
|
||||
@@ -354,13 +417,47 @@ func (h *Holder) judge(ctx context.Context) (*judged, error) {
|
||||
return j, nil
|
||||
}
|
||||
|
||||
// against is what each build the builder's log says ended stood on, by build key: the bases' references.
|
||||
func (h *Holder) against() (map[string][]string, error) {
|
||||
// withEnds gives each build its end: the one known, else the builder's log's build-ended entry, else its key's history,
|
||||
// each kept once read (H-A-258). A done key whose end can be read from neither stays unread, and then nothing is judged:
|
||||
// never from a view missing a build.
|
||||
func (h *Holder) withEnds(built []Build, ends map[string]time.Time) ([]Build, error) {
|
||||
var unread []string
|
||||
var why error
|
||||
for i := range built {
|
||||
if !built[i].At.IsZero() {
|
||||
continue
|
||||
}
|
||||
if at, logged := ends[built[i].Key]; logged {
|
||||
h.mu.Lock()
|
||||
if h.ended == nil {
|
||||
h.ended = map[string]time.Time{}
|
||||
}
|
||||
h.ended[built[i].Key] = at
|
||||
h.mu.Unlock()
|
||||
built[i].At = at
|
||||
continue
|
||||
}
|
||||
at, err := h.endOf(built[i].Key)
|
||||
if err != nil {
|
||||
unread, why = append(unread, built[i].Key), err
|
||||
continue
|
||||
}
|
||||
built[i].At = at
|
||||
}
|
||||
if len(unread) > 0 {
|
||||
return nil, fmt.Errorf("%w: the end of %d done builds could not be read (%s first), so nothing is judged: %v", ErrBuildsUnread, len(unread), unread[0], why)
|
||||
}
|
||||
return built, nil
|
||||
}
|
||||
|
||||
// logged is what each build the builder's log says ended stood on, by build key (the bases' references), and when it
|
||||
// ended: one read of the log.
|
||||
func (h *Holder) logged() (map[string][]string, map[string]time.Time, error) {
|
||||
entries, err := h.Bus.LogSince(builds.BuildLogRead, h.now().Add(-builds.BuilderLogDays*24*time.Hour))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, nil, err
|
||||
}
|
||||
out := map[string][]string{}
|
||||
out, ends := map[string][]string{}, map[string]time.Time{}
|
||||
for _, raw := range entries {
|
||||
e, err := builds.DecodeBuildLogEntry(raw.Value)
|
||||
if err != nil || e.Event != builds.BuildEndedEvent {
|
||||
@@ -371,8 +468,11 @@ func (h *Holder) against() (map[string][]string, error) {
|
||||
continue
|
||||
}
|
||||
out[key] = e.Against
|
||||
if e.At.After(ends[key]) {
|
||||
ends[key] = e.At
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
return out, ends, nil
|
||||
}
|
||||
|
||||
// Mirrors judges each copy in the store's mirror repositories (upstream/<host>/<path>, <module>/on-<argument>; ADR
|
||||
@@ -514,6 +614,11 @@ func (h *Holder) Sweep(ctx context.Context, s Sweeping) (*CollectAnswer, error)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return h.sweepFrom(ctx, j, s)
|
||||
}
|
||||
|
||||
// sweepFrom is a sweep from a judgement already read; the caller holds h.sweep.
|
||||
func (h *Holder) sweepFrom(ctx context.Context, j *judged, s Sweeping) (*CollectAnswer, error) {
|
||||
a := &CollectAnswer{DryRun: s.DryRun, WouldLetGo: []string{}, LetGo: []string{}}
|
||||
var keptArchives, eligible []string
|
||||
for _, r := range j.sorted() {
|
||||
@@ -658,15 +763,73 @@ func namedBy(b Build, r string) string {
|
||||
return r
|
||||
}
|
||||
|
||||
// AfterBuild is the sweep at a build's end.
|
||||
func (h *Holder) AfterBuild(key string) {
|
||||
a, err := h.Sweep(context.Background(), Sweeping{Most: MostPerSweep, By: "after the build " + key})
|
||||
// Wanted marks the pass wanted (design 47 H, H-A-250 (1)): a finished build or a want change asks it, by naming what
|
||||
// asked. With no pass running one starts; a pass running when it is marked runs once more when it ends, however many
|
||||
// events marked it, never once per event.
|
||||
func (h *Holder) Wanted(by string) {
|
||||
if h.mark(by) {
|
||||
go h.passes(by)
|
||||
}
|
||||
}
|
||||
|
||||
// mark sets the pending mark, and says whether no pass ran, so the caller starts one.
|
||||
func (h *Holder) mark(by string) bool {
|
||||
h.pmu.Lock()
|
||||
defer h.pmu.Unlock()
|
||||
if h.running {
|
||||
h.wanted, h.wantedBy = true, by
|
||||
return false
|
||||
}
|
||||
h.running = true
|
||||
return true
|
||||
}
|
||||
|
||||
// passes runs the pass, then once more for as long as an event marked it during the one that ran.
|
||||
func (h *Holder) passes(by string) {
|
||||
for {
|
||||
h.Pass(context.Background(), by)
|
||||
h.pmu.Lock()
|
||||
if !h.wanted {
|
||||
h.running = false
|
||||
h.pmu.Unlock()
|
||||
return
|
||||
}
|
||||
h.wanted, by = false, h.wantedBy
|
||||
h.pmu.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
// Pass is one pass of the seat: the keep set judged from one read of the builds, the wants and the store's list, then
|
||||
// the sweep and S25's observation from that one judgement. A list the registry seats cannot answer ends the pass with
|
||||
// the finding artifact-list-unread, asked again only at the next event (H-A-250 (3)); a list read again clears it.
|
||||
func (h *Holder) Pass(ctx context.Context, by string) {
|
||||
h.sweep.Lock()
|
||||
j, err := h.judge(ctx)
|
||||
if err != nil {
|
||||
h.logf("the sweep after %s: %v", key, err)
|
||||
h.sweep.Unlock()
|
||||
h.logf("the pass %s: %v", by, err)
|
||||
if errors.Is(err, ErrBuildsUnread) {
|
||||
h.unreadFinding(BuildsUnread, err) // H-A-260
|
||||
}
|
||||
return
|
||||
}
|
||||
if len(a.LetGo) > 0 || a.Stopped != "" || a.Missing > 0 {
|
||||
h.logf("the sweep after %s let go of %d (%d left, %d missing): %s", key, len(a.LetGo), a.Left, a.Missing, a.Stopped)
|
||||
h.readAgain(BuildsUnread)
|
||||
if j.unread != nil {
|
||||
h.sweep.Unlock()
|
||||
h.unreadFinding(ListUnread, j.unread)
|
||||
return
|
||||
}
|
||||
h.readAgain(ListUnread)
|
||||
a, err := h.sweepFrom(ctx, j, Sweeping{Most: MostPerSweep, By: by})
|
||||
h.sweep.Unlock()
|
||||
switch {
|
||||
case err != nil:
|
||||
h.logf("the sweep %s: %v", by, err)
|
||||
case len(a.LetGo) > 0 || a.Stopped != "" || a.Missing > 0:
|
||||
h.logf("the sweep %s let go of %d (%d left, %d missing): %s", by, len(a.LetGo), a.Left, a.Missing, a.Stopped)
|
||||
}
|
||||
if err := h.observeFrom(ctx, j); err != nil {
|
||||
h.logf("image-missing could not be observed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -857,10 +1020,16 @@ func inStoreAt(image, address string) string {
|
||||
// want's machine, written once and cleared once it no longer holds. Nothing is said without a listing. What stands is what
|
||||
// the held writer holds (design 47 G A-32, A-38), so a restart neither drops nor rewrites it.
|
||||
func (h *Holder) Observe(ctx context.Context) error {
|
||||
now, err := h.observed(ctx)
|
||||
j, err := h.judge(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return h.observeFrom(ctx, j)
|
||||
}
|
||||
|
||||
// observeFrom is Observe from a judgement already read.
|
||||
func (h *Holder) observeFrom(ctx context.Context, j *judged) error {
|
||||
now := h.observed(j)
|
||||
if now == nil {
|
||||
return nil
|
||||
}
|
||||
@@ -899,13 +1068,9 @@ type Findings interface {
|
||||
}
|
||||
|
||||
// observed is S25's findings now by key, nil without a listing.
|
||||
func (h *Holder) observed(ctx context.Context) (map[string]meshconditions.Finding, error) {
|
||||
j, err := h.judge(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
func (h *Holder) observed(j *judged) map[string]meshconditions.Finding {
|
||||
if j.view == nil {
|
||||
return nil, nil
|
||||
return nil
|
||||
}
|
||||
address := ""
|
||||
if d, err := h.Door(); err == nil {
|
||||
@@ -944,7 +1109,7 @@ func (h *Holder) observed(ctx context.Context) (map[string]meshconditions.Findin
|
||||
break
|
||||
}
|
||||
}
|
||||
return now, nil
|
||||
return now
|
||||
}
|
||||
|
||||
// imageMissing is S25's finding, in the controller's words (cmd/mesh-controller/image_missing.go watchMissingImages); its id
|
||||
|
||||
@@ -200,6 +200,15 @@ func TestTheSeatSweepsAtABuildsEndOnARealBusWithNoLease(t *testing.T) {
|
||||
}
|
||||
h := &Holder{Bus: findingsAt{realBus: bus}, Findings: findingsThrough{findingsAt{realBus: bus}}, Door: mem.Door, Listed: mem.Listed, Seats: ask, Now: time.Now, Logf: t.Logf}
|
||||
h.Acts = meshhandactsclient.Recorder{Ask: ask}
|
||||
// The followed view of the builds: the six on the bus handed over as the watch's first view does.
|
||||
h.Watching()
|
||||
if keys, err := bus.Keys(BuildsRead); err == nil {
|
||||
for _, k := range keys {
|
||||
raw, _, _ := bus.Get(BuildsRead, k)
|
||||
h.Heard(stdio.StateChange{State: BuildsRead, Key: k, Op: stdio.OpPut, Value: raw, Current: true})
|
||||
}
|
||||
}
|
||||
h.TakenIn()
|
||||
|
||||
// A build's end heard from the builds bucket, as follow hands it over.
|
||||
kv, _ := bus.kv(BuildsRead)
|
||||
@@ -215,7 +224,8 @@ func TestTheSeatSweepsAtABuildsEndOnARealBusWithNoLease(t *testing.T) {
|
||||
}
|
||||
select {
|
||||
case e := <-w.Updates():
|
||||
h.AfterBuild(e.Key())
|
||||
h.Heard(stdio.StateChange{State: BuildsRead, Key: e.Key(), Op: stdio.OpPut, Value: e.Value()})
|
||||
h.Pass(ctx, "after the build "+e.Key())
|
||||
case <-ctx.Done():
|
||||
t.Fatal("the build's end was not heard")
|
||||
}
|
||||
|
||||
@@ -34,6 +34,17 @@ type memBus struct {
|
||||
history map[string][]stdio.StateRevision
|
||||
log []stdio.LogEntry
|
||||
published []string
|
||||
// historyErr fails the history read of a key.
|
||||
historyErr map[string]error
|
||||
// calls counts each read by "<verb> <state>": a pass reads nothing of the builds, judged from the followed view.
|
||||
calls map[string]int
|
||||
}
|
||||
|
||||
func (b *memBus) count(verb, state string) {
|
||||
if b.calls == nil {
|
||||
b.calls = map[string]int{}
|
||||
}
|
||||
b.calls[verb+" "+state]++
|
||||
}
|
||||
|
||||
func (b *memBus) LogSince(read string, at time.Time) ([]stdio.LogEntry, error) {
|
||||
@@ -49,6 +60,7 @@ func (b *memBus) LogSince(read string, at time.Time) ([]stdio.LogEntry, error) {
|
||||
func (b *memBus) Keys(state string) ([]string, error) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
b.count("keys", state)
|
||||
var out []string
|
||||
for k := range b.states[state] {
|
||||
out = append(out, k)
|
||||
@@ -60,11 +72,18 @@ func (b *memBus) Keys(state string) ([]string, error) {
|
||||
func (b *memBus) Get(state, key string) ([]byte, bool, error) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
b.count("get", state)
|
||||
v, ok := b.states[state][key]
|
||||
return v, ok, nil
|
||||
}
|
||||
|
||||
func (b *memBus) History(state, key string) ([]stdio.StateRevision, error) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
b.count("history", state)
|
||||
if err := b.historyErr[key]; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return b.history[state+" "+key], nil
|
||||
}
|
||||
|
||||
@@ -229,7 +248,7 @@ func aMesh(t *testing.T) (*Holder, *memBus, *memDoor, *[]string) {
|
||||
want, _ := json.Marshal(meshwants.WantsByWantNodeModuleValue{V: 1, Fp: fp(1), Self: true})
|
||||
_ = b.Put(WantsRead, "want.ace.letta", json.RawMessage(want))
|
||||
var acts []string
|
||||
h := &Holder{Bus: b, Findings: findingsThrough{b}, Door: func() (Door, error) { return door, nil }, Seats: door.seat,
|
||||
h := &Holder{Logf: t.Logf, Bus: b, Findings: findingsThrough{b}, Door: func() (Door, error) { return door, nil }, Seats: door.seat,
|
||||
Listed: func(context.Context) (*View, error) { return &View{L: listing}, nil },
|
||||
Now: func() time.Time { return day.Add(48 * time.Hour) },
|
||||
Acts: meshhandactsclient.Recorder{Ask: func(key string, body any) (json.RawMessage, error) {
|
||||
@@ -237,9 +256,49 @@ func aMesh(t *testing.T) (*Holder, *memBus, *memDoor, *[]string) {
|
||||
acts = append(acts, key+" "+string(raw))
|
||||
return json.RawMessage(`{}`), nil
|
||||
}}}
|
||||
takeIn(h, b)
|
||||
return h, b, door, &acts
|
||||
}
|
||||
|
||||
// takeIn hands the builds over as the watch's first view does, then forgets the reads that took: a pass's are counted.
|
||||
func takeIn(h *Holder, b *memBus) {
|
||||
handOver(h, b)
|
||||
h.TakenIn()
|
||||
b.calls = nil
|
||||
}
|
||||
|
||||
// handOver is a watch's handover of every current value, not yet taken in.
|
||||
func handOver(h *Holder, b *memBus) {
|
||||
h.Watching()
|
||||
keys, _ := b.Keys(BuildsRead)
|
||||
for _, k := range keys {
|
||||
raw, _, _ := b.Get(BuildsRead, k)
|
||||
h.Heard(stdio.StateChange{State: BuildsRead, Key: k, Op: stdio.OpPut, Value: raw, Current: true})
|
||||
}
|
||||
}
|
||||
|
||||
// keyOf is letta's n-th build's key.
|
||||
func keyOf(n int) string { k, _ := builds.BuilderBuildKey("letta", fp(n)); return k }
|
||||
|
||||
// viewKeys are the keys the followed view holds.
|
||||
func viewKeys(t *testing.T, h *Holder) []string {
|
||||
t.Helper()
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
var out []string
|
||||
for k := range h.done {
|
||||
out = append(out, k)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// putBuild writes a build's key and hands it over as the watch's update does.
|
||||
func putBuild(h *Holder, b *memBus, key string, raw []byte) {
|
||||
_ = b.Put(BuildsRead, key, json.RawMessage(raw))
|
||||
h.Heard(stdio.StateChange{State: BuildsRead, Key: key, Op: stdio.OpPut, Value: raw})
|
||||
}
|
||||
|
||||
// ADR 0571 clause 2, 1126/A4/AC2: the keep set from the store's listing, the wants and the five newest builds; with no
|
||||
// lease key, a build's end and a confirmed collect let go of the eligible within the bound, leave an index for a
|
||||
// confirmed collect, record each deletion as a hand act and say artifacts-gone.
|
||||
@@ -312,7 +371,7 @@ func TestTheSeatKeepsTheFiveNewestAndTheWantedAndSweepsWithNoLease(t *testing.T)
|
||||
raw, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildDone, Manifest: strings.Repeat("d", 64), Artifacts: []builds.BuiltArtifact{
|
||||
{Name: "image", Kind: "image", Digest: digestOf(imageRef("letta", n)), Reference: artifactref.SeatReference(reg.Seat, "letta", "image", digestOf(imageRef("letta", n)))},
|
||||
{Name: "files", Kind: "archive", Digest: digestOf(archiveRef("letta", n)), Reference: artifactref.SeatReference(ociseat.Seat, "letta", "files", digestOf(archiveRef("letta", n)))}}})
|
||||
_ = b.Put(BuildsRead, key, json.RawMessage(raw))
|
||||
putBuild(h, b, key, raw)
|
||||
}
|
||||
}
|
||||
if c.refuse != "" {
|
||||
@@ -323,7 +382,7 @@ func TestTheSeatKeepsTheFiveNewestAndTheWantedAndSweepsWithNoLease(t *testing.T)
|
||||
}
|
||||
switch c.how {
|
||||
case "build":
|
||||
h.AfterBuild("letta.x")
|
||||
h.Pass(context.Background(), "after the build letta.x")
|
||||
var gone []string
|
||||
for _, x := range door.asked {
|
||||
if r, ok := strings.CutPrefix(x, "let go "); ok {
|
||||
@@ -420,6 +479,7 @@ func TestAWantedImageTheStoreLacksIsAFindingAndImagesNamesWantsAndHaves(t *testi
|
||||
view.L.Repos[repo].Revisions[digest] = true
|
||||
// A restart: a holder made afresh holds no finding of its own, and clears what the writer holds (design 47 G A-32, A-38).
|
||||
h = &Holder{Bus: h.Bus, Findings: h.Findings, Door: h.Door, Listed: h.Listed, Seats: h.Seats, Now: h.Now, Acts: h.Acts, Logf: h.Logf}
|
||||
takeIn(h, b)
|
||||
if err := h.Observe(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -482,3 +542,154 @@ func (f findingsThrough) Held() []meshconditions.Finding {
|
||||
func (f findingsThrough) Clear(_ context.Context, scope, id, kind, _ string) error {
|
||||
return f.bus.Delete(FindingsState, views.FindingKey(scope, id, kind))
|
||||
}
|
||||
|
||||
// writesCounted counts the findings written through the test's writer.
|
||||
type writesCounted struct {
|
||||
findingsThrough
|
||||
writes *int
|
||||
}
|
||||
|
||||
func (w writesCounted) Write(ctx context.Context, f meshconditions.Finding) error {
|
||||
*w.writes++
|
||||
return w.findingsThrough.Write(ctx, f)
|
||||
}
|
||||
|
||||
// Design 47 H, H-A-250: the starts fold into one pending pass, a pass reads the builds in one page, and a list the
|
||||
// registry seats cannot answer ends the pass with artifact-list-unread, written once and asked again only at an event.
|
||||
func TestOnePendingPassReadsTheBuildsOnceAndEndsOnAnUnreadList(t *testing.T) {
|
||||
t.Run("three events during a pass run exactly one more", func(t *testing.T) {
|
||||
h, _, _, _ := aMesh(t)
|
||||
listed, inner := 0, h.Listed
|
||||
h.Listed = func(ctx context.Context) (*View, error) {
|
||||
listed++
|
||||
if listed == 1 {
|
||||
for _, by := range []string{"after the build a", "after the build b", "a want changed"} {
|
||||
h.Wanted(by) // a pass runs: each only marks it
|
||||
}
|
||||
}
|
||||
return inner(ctx)
|
||||
}
|
||||
if !h.mark("after the build letta.x") {
|
||||
t.Fatal("no pass ran, and the mark did not start one")
|
||||
}
|
||||
h.passes("after the build letta.x")
|
||||
if listed != 2 {
|
||||
t.Fatalf("three events during a pass ran %d passes, want 2", listed)
|
||||
}
|
||||
if h.running || h.wanted {
|
||||
t.Fatalf("after the passes running is %v and wanted %v", h.running, h.wanted)
|
||||
}
|
||||
})
|
||||
t.Run("a pass reads nothing of the builds and their ends come from the log", func(t *testing.T) {
|
||||
h, b, door, _ := aMesh(t)
|
||||
handOver(h, b) // a fresh take-in, counted: no history read for a build the log holds (H-A-258)
|
||||
h.TakenIn()
|
||||
h.Pass(context.Background(), "after the build letta.x")
|
||||
if n := b.calls["keys "+BuildsRead] - 1 + b.calls["get "+BuildsRead] - 7 + b.calls["history "+BuildsRead]; n != 0 {
|
||||
t.Fatalf("a take-in and a pass read the builds by %v", b.calls)
|
||||
}
|
||||
if !slices.Contains(door.asked, "let go "+imageRef("letta", 2)) {
|
||||
t.Fatalf("the pass swept nothing: %v", door.asked)
|
||||
}
|
||||
// Only a build the log lacks is read from its history, once.
|
||||
h, b, _, _ = aMesh(t)
|
||||
b.log = append(b.log[:2:2], b.log[3:]...) // letta's 3rd
|
||||
h.Pass(context.Background(), "a")
|
||||
h.Pass(context.Background(), "b")
|
||||
if b.calls["history "+BuildsRead] != 1 {
|
||||
t.Fatalf("two passes read the builds' history %d times, want once for the build the log lacks", b.calls["history "+BuildsRead])
|
||||
}
|
||||
})
|
||||
t.Run("no pass judges before the first view, and says so", func(t *testing.T) {
|
||||
h, b, door, _ := aMesh(t)
|
||||
h.taken = false
|
||||
h.Pass(context.Background(), "after the build letta.x")
|
||||
_, found, _ := b.Get(FindingsState, views.FindingKey("module", Module, BuildsUnread))
|
||||
if len(door.asked) != 0 || !found {
|
||||
t.Fatalf("a pass before the builds were taken in asked %v; artifact-builds-unread is written: %v", door.asked, found)
|
||||
}
|
||||
})
|
||||
t.Run("an unread end holds every pass until it is read", func(t *testing.T) {
|
||||
h, b, door, _ := aMesh(t)
|
||||
writes := 0
|
||||
h.Findings = writesCounted{findingsThrough{b}, &writes}
|
||||
b.log = append(b.log[:2:2], b.log[3:]...) // letta's 3rd: the log lacks it, and its history fails
|
||||
b.historyErr = map[string]error{keyOf(3): errors.New("no responders")}
|
||||
h.Pass(context.Background(), "a")
|
||||
h.Pass(context.Background(), "b")
|
||||
key := views.FindingKey("module", Module, BuildsUnread)
|
||||
if _, found, _ := b.Get(FindingsState, key); !found || writes != 1 || len(door.asked) != 0 {
|
||||
t.Fatalf("with an end unread the passes asked %v; the finding is written %v, %d times", door.asked, found, writes)
|
||||
}
|
||||
b.historyErr = nil
|
||||
h.Pass(context.Background(), "c")
|
||||
if _, found, _ := b.Get(FindingsState, key); found || !slices.Contains(door.asked, "let go "+imageRef("letta", 2)) {
|
||||
t.Fatalf("once the end was read the pass asked %v and left the finding: %v", door.asked, found)
|
||||
}
|
||||
})
|
||||
t.Run("a change heard during the handover is in the view it gives", func(t *testing.T) {
|
||||
h, b, _, _ := aMesh(t)
|
||||
handOver(h, b)
|
||||
raw, _, _ := b.Get(BuildsRead, keyOf(1))
|
||||
h.Heard(stdio.StateChange{State: BuildsRead, Key: keyOf(8), Op: stdio.OpPut, Value: raw}) // after the handover, before TakenIn
|
||||
h.Heard(stdio.StateChange{State: BuildsRead, Key: keyOf(3), Op: stdio.OpDelete})
|
||||
h.TakenIn()
|
||||
got := viewKeys(t, h)
|
||||
if !slices.Contains(got, keyOf(8)) || slices.Contains(got, keyOf(3)) {
|
||||
t.Fatalf("the view after the handover holds %v", got)
|
||||
}
|
||||
})
|
||||
t.Run("a key deleted or no longer done leaves the view; one written done again takes its new end", func(t *testing.T) {
|
||||
h, b, _, _ := aMesh(t)
|
||||
h.Pass(context.Background(), "a") // every end known
|
||||
building, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: "building"})
|
||||
h.Heard(stdio.StateChange{State: BuildsRead, Key: keyOf(3), Op: stdio.OpDelete})
|
||||
h.Heard(stdio.StateChange{State: BuildsRead, Key: keyOf(4), Op: stdio.OpPut, Value: building})
|
||||
if got := viewKeys(t, h); slices.Contains(got, keyOf(3)) || slices.Contains(got, keyOf(4)) {
|
||||
t.Fatalf("the view holds %v", got)
|
||||
}
|
||||
later := b.log[len(b.log)-1].At.Add(time.Hour)
|
||||
e := builds.BuildLogEntry{Event: builds.BuildEndedEvent, Module: "letta", FP: fp(5), Repository: "novox/x", Commit: "c", Node: "ace", At: later, Outcome: "done"}
|
||||
entry, _ := builds.EncodeBuildLogEntry(e)
|
||||
b.log = append(b.log, stdio.LogEntry{Sequence: 99, Key: "letta", Value: entry, At: later})
|
||||
raw, _, _ := b.Get(BuildsRead, keyOf(5))
|
||||
h.Heard(stdio.StateChange{State: BuildsRead, Key: keyOf(5), Op: stdio.OpPut, Value: raw})
|
||||
h.Pass(context.Background(), "b")
|
||||
if got, _ := h.Builds(); !slices.ContainsFunc(got, func(x Build) bool { return x.Key == keyOf(5) && x.At.Equal(later) }) {
|
||||
t.Fatalf("letta's 5th written done again does not end at %v: %+v", later, got)
|
||||
}
|
||||
})
|
||||
t.Run("a watch asked again replaces the view whole", func(t *testing.T) {
|
||||
h, b, _, _ := aMesh(t)
|
||||
gone, _ := builds.BuilderBuildKey("letta", fp(3))
|
||||
_ = b.Delete(BuildsRead, gone) // deleted while no watch ran: no change of it is heard
|
||||
takeIn(h, b)
|
||||
got, err := h.Builds()
|
||||
if err != nil || len(got) != 6 || slices.ContainsFunc(got, func(x Build) bool { return x.Key == gone }) {
|
||||
t.Fatalf("after the watch was asked again the view holds %d builds (%v), %s among them", len(got), err, gone)
|
||||
}
|
||||
})
|
||||
t.Run("a list unread ends the pass with its finding once", func(t *testing.T) {
|
||||
h, b, door, _ := aMesh(t)
|
||||
writes, listed, inner := 0, 0, h.Listed
|
||||
h.Findings = writesCounted{findingsThrough{b}, &writes}
|
||||
h.Listed = func(context.Context) (*View, error) {
|
||||
listed++
|
||||
return nil, errors.New("the answer exceeds the bus's one megabyte")
|
||||
}
|
||||
h.Pass(context.Background(), "after the build a")
|
||||
h.Pass(context.Background(), "a want changed")
|
||||
key := views.FindingKey("module", Module, ListUnread)
|
||||
if raw, found, _ := b.Get(FindingsState, key); !found || writes != 1 || !strings.Contains(string(raw), "one megabyte") {
|
||||
t.Fatalf("the finding is %s (found %v), written %d times", raw, found, writes)
|
||||
}
|
||||
if listed != 2 || len(door.asked) != 0 || len(b.published) != 0 {
|
||||
t.Fatalf("two passes listed %d times, asked %v and said %v", listed, door.asked, b.published)
|
||||
}
|
||||
h.Listed = inner
|
||||
h.Pass(context.Background(), "after the build b")
|
||||
if _, found, _ := b.Get(FindingsState, key); found || len(door.asked) == 0 {
|
||||
t.Fatalf("a pass that read the list left the finding (%v) or swept nothing (%v)", found, door.asked)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -32,7 +32,7 @@ func main() {
|
||||
h.Acts = meshhandactsclient.Recorder{Ask: stdio.Ask}
|
||||
// The module's one held writer (design 47 G §689, A-16, A-29, A-32): the holder's findings and the reader's go through it.
|
||||
w, err := views.NewStdioHeldWriter(views.WriterConfig[meshconditions.Finding, meshconditions.FindingCleared, meshconditions.FindingsStated, meshconditions.FindingsAskedEvent]{
|
||||
Asked: meshconditions.SubjectFindingsAsked, Module: Module, State: stdio.State("findings"), Kinds: []string{ImageMissing, SweepRefused}})
|
||||
Asked: meshconditions.SubjectFindingsAsked, Module: Module, State: stdio.State("findings"), Kinds: []string{ImageMissing, SweepRefused, ListUnread, BuildsUnread}})
|
||||
if err != nil {
|
||||
logf("%v", err)
|
||||
os.Exit(1)
|
||||
|
||||
@@ -27,6 +27,9 @@ import (
|
||||
// SweepRefused is the sweep's finding: a seat refused a hold or a remove the sweep asked.
|
||||
const SweepRefused = "sweep-refused"
|
||||
|
||||
// ListUnread is the pass's finding: the registry seats' list could not be read, so the pass ended (design 47 H, H-A-250 (3)).
|
||||
const ListUnread = "artifact-list-unread"
|
||||
|
||||
// canonical is a kept reference in the older form, `artifact-store://<path>`, whichever form it came in; another is as
|
||||
// it was.
|
||||
func canonical(reference string) string {
|
||||
@@ -140,6 +143,53 @@ func (h *Holder) sweepClean() {
|
||||
}
|
||||
}
|
||||
|
||||
// BuildsUnread is the pass's finding: the builder's builds could not be judged, so the pass ended (design 47 H, H-A-260).
|
||||
const BuildsUnread = "artifact-builds-unread"
|
||||
|
||||
// unreadFinding writes a pass's finding of what it could not read (ListUnread, BuildsUnread) once: a pass that fails the
|
||||
// same way while the writer holds it writes nothing more. Nothing asks again until the next event marks a pass.
|
||||
func (h *Holder) unreadFinding(kind string, err error) {
|
||||
for _, was := range h.Findings.Held() {
|
||||
if was.Kind == kind {
|
||||
return
|
||||
}
|
||||
}
|
||||
at := h.now().UTC()
|
||||
f := meshconditions.Finding{Scope: "module", ID: keys.EscapeName(Module), Kind: kind,
|
||||
Subject: meshconditions.Subject{Scope: "module", ID: keys.EscapeName(Module)},
|
||||
Severity: meshconditions.FindingSeverityWarning, Resolver: "operator",
|
||||
Summary: "the artifact sweep stopped: the registry seats' list could not be read (the evidence names why)",
|
||||
Headline: "The artifact sweep cannot read the store's list",
|
||||
Explanation: "The sweep judges what to keep against the store's list, which it asks the registry seats. The list " +
|
||||
"could not be read, so the sweep removed nothing and observed nothing; what it kept stays kept.",
|
||||
Needs: "read the registry seat holder's journal for why its list failed; the next finished build or want change asks again.",
|
||||
Evidence: []meshconditions.Evidence{{At: at, Said: err.Error()}},
|
||||
Source: "CAT-AS1", Observer: Module, At: at}
|
||||
if kind == BuildsUnread {
|
||||
f.Summary = "the artifact sweep stopped: the builder's builds could not be judged (the evidence names why)"
|
||||
f.Headline = "The artifact sweep cannot judge the builds"
|
||||
f.Explanation = "The sweep keeps each module's newest builds and every wanted one, judged from the builder's builds " +
|
||||
"and when each ended. Those could not all be read, so the sweep removed nothing and observed nothing; what " +
|
||||
"it kept stays kept."
|
||||
f.Needs = "read this module's journal for which build's end is unread; the next finished build or want change asks again."
|
||||
}
|
||||
if werr := h.Findings.Write(context.Background(), f); werr != nil {
|
||||
h.logf("the pass's finding %s (%v) could not be written: %v", kind, err, werr)
|
||||
}
|
||||
}
|
||||
|
||||
// readAgain clears a pass's finding of kind, when the held writer holds it, once a pass read what it could not.
|
||||
func (h *Holder) readAgain(kind string) {
|
||||
for _, was := range h.Findings.Held() {
|
||||
if was.Kind != kind {
|
||||
continue
|
||||
}
|
||||
if err := h.Findings.Clear(context.Background(), was.Scope, was.ID, was.Kind, "a pass read it again"); err != nil {
|
||||
h.logf("the pass's finding %s could not be cleared: %v", kind, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// keptWhy is why a kept reference is held, as the seat's hold takes it.
|
||||
func keptWhy(r *Ref) string {
|
||||
if r == nil || len(r.Why) == 0 {
|
||||
|
||||
Reference in New Issue
Block a user