Compare commits

..
Author SHA1 Message Date
jschoubben a12c194f6c indexer: the indexer reads each catalog graph entry from the bus once per process
An entry object entry.<digest> is written once and never changes, yet every
revision read (the verb revision on each page, the composition steps, about 20
callers of ReadRevision) took one object Get per entry, each bounded by the bus
client's deadline: composition steps ended "could not run: the step's catalog
revision is not read ... context deadline exceeded". The indexer now keeps the
bytes of entry objects it has read or written in memory (bound 64 MiB, dropped
whole and logged once when passed); a Delete drops the entry, so a revision that
still lists it fails naming it. Revision lists are read from the bus as before.

Builds: design 47 G §354 (composition reads the revision), A-252
2026-11-16 22:00:11 +01:00
mesh-tools 38f1c2f10a Merge pull request 'assigner: a composition step reads its catalog revision once and again on a timeout' (#232) from fix/composition-reads-revision-once into main 2026-11-16 20:32:07 +00:00
jschoubben bece2421f1 assigner: a composition step reads its catalog revision once and again on a timeout
Builds: design 47 G §354, A-252, A-261
2026-11-16 21:31:34 +01:00
21 changed files with 500 additions and 1012 deletions
@@ -0,0 +1,181 @@
package main
import (
"context"
"encoding/json"
"fmt"
"strings"
"sync"
"time"
mcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-sdk/go/buscalls"
)
// The reads of a composition step (design 47 G §354, A-252, A-261): the entries of one catalog revision are read once and shared by
// every composition step of that revision, since an entry is content-addressed and a revision never changes; a read that times out
// or fails in transport is made again with a growing wait for a bounded time, said in the journal, and only then ends the step as
// could not run.
const (
// readsBound is how long a read is made again before the step could not run.
readsBound = 3 * time.Minute
// readsFirstWait and readsMaxWait bound the wait between two tries.
readsFirstWait = time.Second
readsMaxWait = 15 * time.Second
// readsKept is how many catalog revisions' halves are kept; the oldest read goes first.
readsKept = 8
)
// readKey names one revision of one watched branch.
type readKey struct {
branch string
revision int
}
// readSlot is one revision's read: done closes when half or err is set.
type readSlot struct {
done chan struct{}
half mcc.Half
err error
}
// compositionReads shares the reads of composition steps.
type compositionReads struct {
mu sync.Mutex
revs map[readKey]*readSlot
order []readKey
seats map[string]json.RawMessage // by seat and digest: the definition, raw
}
// transient says whether a read's fault is of the transport or a timeout, so that it is worth reading again.
func transient(err error, said string) bool {
text := said
if err != nil {
text = err.Error() + " " + said
}
text = strings.ToLower(text)
for _, w := range []string{"deadline exceeded", "timeout", "timed out", "no responders", "connection", "i/o", "eof", "broken pipe", "unavailable"} {
if strings.Contains(text, w) {
return true
}
}
return false
}
// callRetry calls the seat verb key, again with a growing wait while its fault is transient, for readsBound; it answers the body of
// a good answer, or the last fault.
func (r *compositionRun) callRetry(ctx context.Context, key string, body any) (json.RawMessage, string, error) {
start := r.now()
wait := readsFirstWait
for try := 1; ; try++ {
raw, err := r.call(key, body)
var inner json.RawMessage
var said string
if err == nil {
var ok bool
if inner, said, ok = buscalls.Unwrap(raw); ok {
return inner, "", nil
}
}
if !transient(err, said) {
return nil, said, err
}
if r.now().Sub(start)+wait > readsBound {
return nil, said, err
}
r.logf("composition step: %s is not read (try %d, %v %s), read again in %s", key, try, err, said, wait)
if serr := r.waitFor(ctx, wait); serr != nil {
return nil, said, serr
}
if wait *= 2; wait > readsMaxWait {
wait = readsMaxWait
}
}
}
// waitFor waits d, or until ctx ends.
func (r *compositionRun) waitFor(ctx context.Context, d time.Duration) error {
if r.sleep != nil {
return r.sleep(ctx, d)
}
t := time.NewTimer(d)
defer t.Stop()
select {
case <-ctx.Done():
return ctx.Err()
case <-t.C:
return nil
}
}
// half is the revision's half, read once for every step of it: a step that finds it being read waits for that read, and a read that
// failed is not kept, so the next step reads again.
func (r *compositionRun) half(ctx context.Context, key readKey, read func() (mcc.Half, error)) (mcc.Half, error) {
c := r.reads
c.mu.Lock()
if c.revs == nil {
c.revs = map[readKey]*readSlot{}
}
if s, ok := c.revs[key]; ok {
c.mu.Unlock()
select {
case <-s.done:
if s.err == nil {
return s.half, nil
}
case <-ctx.Done():
return mcc.Half{}, ctx.Err()
}
// The read this step waited for failed: read it here (the failed slot is already dropped).
return r.half(ctx, key, read)
}
s := &readSlot{done: make(chan struct{})}
c.revs[key] = s
c.order = append(c.order, key)
for len(c.order) > readsKept {
delete(c.revs, c.order[0])
c.order = c.order[1:]
}
c.mu.Unlock()
s.half, s.err = read()
if s.err != nil {
c.mu.Lock()
if c.revs[key] == s {
delete(c.revs, key)
}
for i, k := range c.order {
if k == key {
c.order = append(c.order[:i], c.order[i+1:]...)
break
}
}
c.mu.Unlock()
}
close(s.done)
return s.half, s.err
}
// seatKept is the definition kept of a seat at a digest.
func (c *compositionReads) seatKept(seat, digest string) (json.RawMessage, bool) {
c.mu.Lock()
defer c.mu.Unlock()
d, ok := c.seats[seat+"@"+digest]
return d, ok
}
func (c *compositionReads) keepSeat(seat, digest string, def json.RawMessage) {
c.mu.Lock()
defer c.mu.Unlock()
if c.seats == nil {
c.seats = map[string]json.RawMessage{}
}
c.seats[seat+"@"+digest] = def
}
func unreadErr(what string, err error, said string) error {
if err != nil {
return fmt.Errorf("%s: %w", what, err)
}
return fmt.Errorf("%s: %s", what, said)
}
@@ -18,7 +18,6 @@ import (
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer"
"git.novox.be/novox/mesh-sdk/go/buscalls"
"git.novox.be/novox/mesh-sdk/go/keys"
)
@@ -122,6 +121,9 @@ type compositionRun struct {
// put puts a run ask on the build seat's check queue under the assigner's name.
put func(ctx context.Context, a buildstep.StepAsk) error
ends *runEnds
// reads shares the catalog reads among steps (compositionreads.go); sleep waits between two tries of a read (nil: a timer).
reads *compositionReads
sleep func(ctx context.Context, d time.Duration) error
// ask asks a seat verb on one node, for the run's answer kept as the build seat's object (buildstep.StepEnded.ReadOutput).
ask buildstep.AskVerb
now func() time.Time
@@ -136,11 +138,11 @@ type machineEnd struct {
// run runs one composition step and answers what it came to: failed when a machine does not compose, could not run when a machine
// could not be composed, else passed with a warning per machine away or not judged.
func (r *compositionRun) run(ctx context.Context, _ string, run meshwants.StepRun) stepOutcome {
half, err := r.revision(run)
half, err := r.revision(ctx, run)
if err != nil {
return couldNotRun("the step's catalog revision is not read: " + err.Error())
}
seats, err := r.seats(half.Revision.Definitions)
seats, err := r.seats(ctx, half.Revision.Definitions)
if err != nil {
return couldNotRun(err.Error())
}
@@ -148,7 +150,7 @@ func (r *compositionRun) run(ctx context.Context, _ string, run meshwants.StepRu
if err != nil {
return couldNotRun(err.Error())
}
programOf, err := r.programs(half)
programOf, err := r.programs(ctx, half)
if err != nil {
return couldNotRun(err.Error())
}
@@ -275,20 +277,24 @@ func verdictOf(node string, answer []byte) machineEnd {
return machineEnd{couldNot: node + ": the composer's verdict has no case " + changeCase}
}
// revision reads the step's catalog revision whole, page by page, through mesh-catalog's `revision` (A-252).
func (r *compositionRun) revision(run meshwants.StepRun) (mcc.Half, error) {
// revision reads the step's catalog revision whole, page by page, through mesh-catalog's `revision` (A-252), once for every step of
// it (compositionreads.go).
func (r *compositionRun) revision(ctx context.Context, run meshwants.StepRun) (mcc.Half, error) {
if run.Revision <= 0 {
return mcc.Half{}, errors.New("the step names no catalog revision")
}
return r.half(ctx, readKey{run.Branch, int(run.Revision)}, func() (mcc.Half, error) { return r.readRevision(ctx, run) })
}
func (r *compositionRun) readRevision(ctx context.Context, run meshwants.StepRun) (mcc.Half, error) {
half := mcc.Half{Entries: map[string]meshcatalog.CatalogEntry{}}
in := meshcatalog.RevisionInput{Branch: keys.EscapeName(run.Branch), Revision: run.Revision}
for {
raw, err := r.call(mcc.RevisionVerb, in)
body, said, err := r.callRetry(ctx, mcc.RevisionVerb, in)
if err != nil {
return mcc.Half{}, err
}
body, said, ok := buscalls.Unwrap(raw)
if !ok {
if body == nil {
return mcc.Half{}, fmt.Errorf("%s answered: %s", mcc.RevisionVerb, said)
}
var a meshcatalog.RevisionAnswer
@@ -314,20 +320,21 @@ func (r *compositionRun) revision(run meshwants.StepRun) (mcc.Half, error) {
// seats are the definitions of the revision's (seat, digest) pairs, raw (A-261): the revision's definitions carry no definition
// yet (A-234), so each is mesh-catalog's seat-definitions answer for the seat where the trunk's digest is the revision's; a digest
// neither holds is an error naming the seat and the digest.
func (r *compositionRun) seats(defs []meshcatalog.CatalogRevisionDefinitionsItem) ([]json.RawMessage, error) {
func (r *compositionRun) seats(ctx context.Context, defs []meshcatalog.CatalogRevisionDefinitionsItem) ([]json.RawMessage, error) {
out := make([]json.RawMessage, 0, len(defs))
for _, d := range defs {
raw, err := r.call(seatDefinitionsKey, meshcatalog.SeatDefinitionsInput{Seat: d.Seat})
if err != nil {
return nil, fmt.Errorf("the definition of the seat %s is not read: %w", d.Seat, err)
if def, ok := r.reads.seatKept(d.Seat, d.Digest); ok { // a definition at a digest never changes
out = append(out, def)
continue
}
body, said, ok := buscalls.Unwrap(raw)
if !ok {
return nil, fmt.Errorf("the definition of the seat %s is not read: %s", d.Seat, said)
what := "the definition of the seat " + d.Seat + " is not read"
body, said, err := r.callRetry(ctx, seatDefinitionsKey, meshcatalog.SeatDefinitionsInput{Seat: d.Seat})
if err != nil || body == nil {
return nil, unreadErr(what, err, said)
}
var a meshcatalog.SeatDefinitionsAnswer
if err := json.Unmarshal(body, &a); err != nil {
return nil, fmt.Errorf("the definition of the seat %s is not read: %w", d.Seat, err)
return nil, fmt.Errorf("%s: %w", what, err)
}
if a.Entry == nil || a.Entry.Digest != d.Digest || a.Entry.Definition == nil {
return nil, fmt.Errorf("no definition of the seat %s with the digest %s is held: neither the revision nor the trunk holds it", d.Seat, d.Digest)
@@ -336,6 +343,7 @@ func (r *compositionRun) seats(defs []meshcatalog.CatalogRevisionDefinitionsItem
if err != nil {
return nil, err
}
r.reads.keepSeat(d.Seat, d.Digest, def)
out = append(out, def)
}
return out, nil
@@ -412,7 +420,7 @@ func programOfEntry(e meshcatalog.CatalogEntry) (buildstep.RunProgram, error) {
// programs answers, per machine, the composer its run builds (A-226, A-243, A-252): where the revision's node-engine entry differs
// from the trunk's, that build for every machine; else the build the machine's node-engine runs, as its `running` says, its entry
// read through mesh-catalog's `entry`; a machine whose node-engine's `running` is not heard gets the revision's entry, said (A-451).
func (r *compositionRun) programs(half mcc.Half) (func(node string) (buildstep.RunProgram, error), error) {
func (r *compositionRun) programs(ctx context.Context, half mcc.Half) (func(node string) (buildstep.RunProgram, error), error) {
engine, ok := nodeEngineEntry(half.Entries)
if !ok {
return nil, fmt.Errorf("the catalog revision %d holds no module claiming the seat %s", half.Revision.Revision, ne.Seat)
@@ -436,17 +444,14 @@ func (r *compositionRun) programs(half mcc.Half) (func(node string) (buildstep.R
if ok {
return p, nil
}
raw, err := r.call(entryVerbKey, meshcatalog.EntryInput{Fingerprint: running.Fingerprint})
if err != nil {
return buildstep.RunProgram{}, fmt.Errorf("the entry of the node-engine's build %s is not read: %w", running.Fingerprint, err)
}
body, said, ok := buscalls.Unwrap(raw)
if !ok {
return buildstep.RunProgram{}, fmt.Errorf("the entry of the node-engine's build %s is not read: %s", running.Fingerprint, said)
what := "the entry of the node-engine's build " + running.Fingerprint + " is not read"
body, said, err := r.callRetry(ctx, entryVerbKey, meshcatalog.EntryInput{Fingerprint: running.Fingerprint})
if err != nil || body == nil {
return buildstep.RunProgram{}, unreadErr(what, err, said)
}
var a meshcatalog.EntryAnswer
if err := json.Unmarshal(body, &a); err != nil {
return buildstep.RunProgram{}, fmt.Errorf("the entry of the node-engine's build %s is not read: %w", running.Fingerprint, err)
return buildstep.RunProgram{}, fmt.Errorf("%s: %w", what, err)
}
p, err = programOfEntry(a.Entry)
if err != nil {
@@ -82,7 +82,7 @@ func newRunStand(t *testing.T, revisionEngine, trunkEngine, seatDigest string) *
ends := &runEnds{}
s.r = &compositionRun{
inputs: &compositionInputs{machines: func(string, string) []string { return []string{"ace", "g14", "shanks"} }, callFile: engines, bound: time.Second},
keeper: keeper, call: call, ends: ends, away: away, now: time.Now, logf: t.Logf,
keeper: keeper, reads: &compositionReads{}, call: call, ends: ends, away: away, now: time.Now, logf: t.Logf,
trunk: func() (mcc.Half, bool) {
return mcc.Half{Entries: map[string]meshcatalog.CatalogEntry{"engine": engine(trunkEngine, commitE1)}}, true
},
@@ -187,3 +187,84 @@ func TestTheCompositionStepRunsOneAskPerMachine(t *testing.T) {
t.Errorf("an ask at the cap came to %+v, %d asks", out, len(full.asks))
}
}
// Design 47 G §354, A-252: the entries of one catalog revision are read once for every step of it, and a read that times out is
// made again with a growing wait for a bounded time, said in the journal, before the step could not run.
func TestTheCompositionStepReadsTheRevisionOnceAndAgainOnATimeout(t *testing.T) {
step := meshwants.StepRun{Module: "m", Branch: "main", Revision: 7, Step: meshwants.StepRunStepComposition}
for _, tc := range []struct {
name string
faults int // consecutive faults of the `revision` read before it answers
fault error // the fault
steps int
want string // the outcome of the last step
revisionCalls int
waits int
}{
{"two steps of one revision read it once", 0, nil, 2, meshwants.StepEndedOutcomeFailed, 2, 0},
{"a timeout is read again", 3, errors.New("context deadline exceeded"), 1, meshwants.StepEndedOutcomeFailed, 5, 3},
{"a fault that stays ends could not run", 1000, errors.New("nats: timeout"), 1, meshwants.StepEndedOutcomeCouldNotRun, -1, -1},
{"a fault of the answer is not read again", 1, errors.New("no such revision"), 1, meshwants.StepEndedOutcomeCouldNotRun, 1, 0},
} {
t.Run(tc.name, func(t *testing.T) {
s := newRunStand(t, "src2:e1", "src2:e1", "d1")
inner := s.r.call
var mu sync.Mutex
calls, faults := 0, tc.faults
s.r.call = func(key string, body any) (json.RawMessage, error) {
if key == mcc.RevisionVerb {
mu.Lock()
calls++
f := faults > 0
faults--
mu.Unlock()
if f {
return nil, tc.fault
}
}
return inner(key, body)
}
clock := time.Unix(1000, 0)
var waits []time.Duration
s.r.now = func() time.Time { mu.Lock(); defer mu.Unlock(); return clock }
s.r.sleep = func(_ context.Context, d time.Duration) error {
mu.Lock()
defer mu.Unlock()
waits = append(waits, d)
clock = clock.Add(d)
return nil
}
var wg sync.WaitGroup
outs := make([]stepOutcome, tc.steps)
for i := range outs {
wg.Add(1)
go func() { defer wg.Done(); outs[i] = s.r.run(context.Background(), "ask-step", step) }()
}
wg.Wait()
for _, o := range outs {
if o.Outcome != tc.want {
t.Fatalf("a step came to %+v, want %s", o, tc.want)
}
}
if tc.waits == -1 { // never answers: bounded, growing waits
if len(waits) < 3 || waits[0] != readsFirstWait || waits[1] != 2*readsFirstWait || waits[len(waits)-1] != readsMaxWait {
t.Errorf("the waits %v do not grow to %s", waits, readsMaxWait)
}
var total time.Duration
for _, w := range waits {
total += w
}
if total > readsBound {
t.Errorf("read again for %s, bound %s", total, readsBound)
}
if !strings.Contains(outs[0].Said, "catalog revision is not read") || !strings.Contains(outs[0].Said, "timeout") {
t.Errorf("the step said %q", outs[0].Said)
}
return
}
if calls != tc.revisionCalls || len(waits) != tc.waits {
t.Errorf("`revision` asked %d times with %d waits, want %d and %d", calls, len(waits), tc.revisionCalls, tc.waits)
}
})
}
}
@@ -675,7 +675,7 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
}
// The composition step (design 47 G §22, §354): the seat mesh-wants' accepted kind step, run by one run ask per machine.
if bus != nil && js != nil && inputs != nil && p.publishWants != nil {
run := &compositionRun{inputs: &compositionInputs{machines: w.machinesOf, callFile: bus.CallFile}, keeper: inputs, call: bus.CallKept,
run := &compositionRun{inputs: &compositionInputs{machines: w.machinesOf, callFile: bus.CallFile}, keeper: inputs, reads: &compositionReads{}, call: bus.CallKept,
trunk: trunkOf(iv, w.trunkName), running: w.runningOf, away: away, put: runAskPublisher(js), ends: ends, ask: bus.CallFile, now: time.Now, logf: logger.Printf}
steps := &compositionSteps{node: thisNode, say: p.publishWants, run: run.run, now: time.Now, logf: logger.Printf,
asker: stepAskerOf(seatlist.Definition(gate.Reader, meshwants.Seat))}
@@ -285,7 +285,9 @@ func main() {
acts := meshhandactsclient.Recorder{Ask: stdio.Ask}
tools := Tools(asks, time.Now, stdio.State(index.VerdictsState), heldCounts(stdio.State(index.OldSectionsState)), acts)
// The modules and the module graph are the newest catalog graph revision's (milestone 47/G): revision.go.
revisions := bundleObjects{busstate.ObjectsOf(index.CatalogRevisionsObjects)}
revisions := index.KeepEntries(bundleObjects{busstate.ObjectsOf(index.CatalogRevisionsObjects)}, index.EntryKeptBytes, func(format string, a ...any) {
fmt.Fprintf(os.Stderr, "[mesh-delivery-indexer-tools] "+format+"\n", a...)
})
trunk := index.TrunkStates{Objects: revisions, Under: keptSource, Kept: &index.KeptTrunk{Say: func(format string, a ...any) {
fmt.Fprintf(os.Stderr, "[mesh-delivery-indexer-tools] "+format+"\n", a...)
}}}
@@ -356,7 +356,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// The view of the trunk is the newest trunk revision of the catalog graph the objects keep (milestone 47/G, ADR 0656 §6:
// one form of the fact), read once here and kept as a revision after every write; objects not on the bus yet, or none
// kept, start it empty, and the first start's catch-up takes the trunk in once the gate's source is read (catchUp).
revisions := newBusObjects(js, cred, index.CatalogRevisionsObjects)
revisions := index.KeepEntries(newBusObjects(js, cred, index.CatalogRevisionsObjects), index.EntryKeptBytes, logger.Printf)
modules, graph, at, err := index.TrunkView(ctx, revisions)
switch {
case errors.Is(err, process.ErrNotYet):
@@ -0,0 +1,113 @@
package index
import (
"context"
"strings"
"sync"
)
// EntryKeptBytes bounds the bytes of entry objects the indexer holds in memory.
const EntryKeptBytes = 64 << 20
// entryKeeper is the catalog graph's objects with every entry object (entry.<digest>) kept in memory once read or written:
// an entry is written once and never changes, and every revision that lists it shares it, so the indexer reads each entry
// from the bus once per process (design 47 G §354, A-252). Revision lists and every other name go to the bus as before. A
// Delete drops the entry, so a revision that still lists it fails naming it, as before.
type entryKeeper struct {
Objects
limit int64
logf func(string, ...any)
mu sync.Mutex
kept map[string][]byte
size int64
// Hits and Reads count the entry Gets answered from memory and from the bus.
hits, reads int64
}
func newEntryKeeper(inner Objects, limit int64, logf func(string, ...any)) *entryKeeper {
return &entryKeeper{Objects: inner, limit: limit, logf: logf, kept: map[string][]byte{}}
}
func isEntry(name string) bool { return strings.HasPrefix(name, "entry.") }
// keep holds data under name; over the bound it drops everything kept first and says so once per drop.
func (k *entryKeeper) keep(name string, data []byte) {
k.mu.Lock()
defer k.mu.Unlock()
if old, ok := k.kept[name]; ok {
k.size -= int64(len(old))
delete(k.kept, name)
}
if int64(len(data)) > k.limit {
return
}
if k.size+int64(len(data)) > k.limit {
k.logf("the %d entry objects kept in memory (%d bytes) pass the bound of %d bytes: dropped, they are read from the bus again", len(k.kept), k.size, k.limit)
k.kept, k.size = map[string][]byte{}, 0
}
k.kept[name] = data
k.size += int64(len(data))
}
func (k *entryKeeper) drop(name string) {
k.mu.Lock()
defer k.mu.Unlock()
if old, ok := k.kept[name]; ok {
k.size -= int64(len(old))
delete(k.kept, name)
}
}
func (k *entryKeeper) Get(ctx context.Context, name string) ([]byte, error) {
if !isEntry(name) {
return k.Objects.Get(ctx, name)
}
k.mu.Lock()
data, ok := k.kept[name]
if ok {
k.hits++
}
k.mu.Unlock()
if ok {
return data, nil
}
data, err := k.Objects.Get(ctx, name)
if err != nil {
return nil, err
}
k.mu.Lock()
k.reads++
k.mu.Unlock()
k.keep(name, data)
return data, nil
}
func (k *entryKeeper) Put(ctx context.Context, name string, data []byte) error {
if err := k.Objects.Put(ctx, name, data); err != nil {
if isEntry(name) {
k.drop(name)
}
return err
}
if isEntry(name) {
k.keep(name, append([]byte(nil), data...))
}
return nil
}
func (k *entryKeeper) Delete(ctx context.Context, name string) error {
if isEntry(name) {
k.drop(name)
}
err := k.Objects.Delete(ctx, name)
if isEntry(name) {
k.drop(name)
}
return err
}
// KeepEntries is inner with its entry objects kept in memory (see entryKeeper), up to limit bytes; logf says a drop.
func KeepEntries(inner Objects, limit int64, logf func(string, ...any)) Objects {
return newEntryKeeper(inner, limit, logf)
}
@@ -0,0 +1,80 @@
package index
import (
"context"
"errors"
"testing"
)
type countingObjects struct {
Objects
data map[string][]byte
gets int
}
func (c *countingObjects) Put(_ context.Context, n string, d []byte) error { c.data[n] = d; return nil }
func (c *countingObjects) Get(_ context.Context, n string) ([]byte, error) {
c.gets++
d, ok := c.data[n]
if !ok {
return nil, ErrNoObject
}
return d, nil
}
func (c *countingObjects) Delete(_ context.Context, n string) error { delete(c.data, n); return nil }
func TestEntryKeeper(t *testing.T) {
ctx := context.Background()
for _, tc := range []struct {
name string
limit int64
run func(t *testing.T, k *entryKeeper, in *countingObjects)
wantGets int
wantDrops int
}{
{"second Get of an entry is from memory", 1000, func(t *testing.T, k *entryKeeper, in *countingObjects) {
in.data["entry.a"] = []byte("A")
for i := 0; i < 3; i++ {
if d, err := k.Get(ctx, "entry.a"); err != nil || string(d) != "A" {
t.Fatalf("get: %q %v", d, err)
}
}
}, 1, 0},
{"a Put entry is kept, never read", 1000, func(t *testing.T, k *entryKeeper, in *countingObjects) {
if err := k.Put(ctx, "entry.a", []byte("A")); err != nil {
t.Fatal(err)
}
if d, _ := k.Get(ctx, "entry.a"); string(d) != "A" {
t.Fatalf("got %q", d)
}
}, 0, 0},
{"Delete drops it and the next Get is an error naming no object", 1000, func(t *testing.T, k *entryKeeper, in *countingObjects) {
_ = k.Put(ctx, "entry.a", []byte("A"))
_ = k.Delete(ctx, "entry.a")
if _, err := k.Get(ctx, "entry.a"); !errors.Is(err, ErrNoObject) {
t.Fatalf("err %v", err)
}
}, 1, 0},
{"a name that is no entry always reaches the bus", 1000, func(t *testing.T, k *entryKeeper, in *countingObjects) {
in.data["trunk.1"] = []byte("L")
_, _ = k.Get(ctx, "trunk.1")
_, _ = k.Get(ctx, "trunk.1")
}, 2, 0},
{"over the bound everything kept is dropped and said", 3, func(t *testing.T, k *entryKeeper, in *countingObjects) {
in.data["entry.a"], in.data["entry.b"] = []byte("AA"), []byte("BB")
_, _ = k.Get(ctx, "entry.a")
_, _ = k.Get(ctx, "entry.b") // drops a
_, _ = k.Get(ctx, "entry.a") // from the bus again
}, 3, 2},
} {
t.Run(tc.name, func(t *testing.T) {
in := &countingObjects{data: map[string][]byte{}}
drops := 0
k := newEntryKeeper(in, tc.limit, func(string, ...any) { drops++ })
tc.run(t, k, in)
if in.gets != tc.wantGets || drops != tc.wantDrops {
t.Fatalf("bus gets %d (want %d), drops said %d (want %d)", in.gets, tc.wantGets, drops, tc.wantDrops)
}
})
}
}
@@ -130,27 +130,6 @@ func Tools(r view.Reader, u view.UsageReader) []stdio.Tool {
module, _ := args["module"].(string)
return r.Reach(repository, module)
}},
{Name: view.PipelinesReadTool,
Description: "The pipeline runner's pipelines as the view keeps them from its events (the newest " + fmt.Sprint(view.MaxPipelines) + "), newest first, " +
"each whole: its module, branch, source and catalog revision, commit, group or run, fingerprint, outcome (running, passed, failed, refused, superseded, " +
"cancelled) and why, when it started and ended, its test counts, and each step with its status in the policy, its outcome, why, what it waits on, " +
"its ask and the node and times; a running step carries the latest progress its build seat said of its ask (phase fetch, build, run or report, " +
"and when it was said). Narrow by branch, module, outcome and since (an RFC 3339 time: only pipelines the view kept an event of at or after it); " +
"at most limit (default " + fmt.Sprint(view.DefaultPipelinesLimit) + ", at most " + fmt.Sprint(view.MaxPipelinesLimit) + "); matching says how many fit and cut that the limit cut them. " +
"The runner's step log lines are not here. Read-only.",
Input: map[string]any{"type": "object", "properties": map[string]any{
"branch": map[string]any{"type": "string", "description": "only this branch's pipelines"},
"module": map[string]any{"type": "string", "description": "only this module's pipelines"},
"outcome": map[string]any{"type": "string", "description": "only pipelines with this outcome"},
"since": map[string]any{"type": "string", "description": "RFC 3339 time: only pipelines kept an event of at or after it"},
"limit": map[string]any{"type": "integer", "description": "at most this many pipelines, newest first"}}},
Run: func(args map[string]any) (any, error) {
q, err := view.ParsePipelinesQuery(args)
if err != nil {
return nil, err
}
return r.Pipelines(q)
}},
{Name: view.TiersTool,
Description: "A source revision's tiers, worked out now and never stored (novox/hq ADR 0603 §6): the modules the revision moves " +
"(each whose fingerprint last changed at it), ordered by the module graph (a module after the bases, the providers " +
@@ -509,7 +488,6 @@ func main() {
r.Builds, r.HavesState = read(stdio.State("mesh-delivery-builder.builds")), read(stdio.State("node-engine.have"))
r.ReachState = read(stdio.State("reach"))
r.WaitsState = read(stdio.State(viewstate.WaitsState))
r.PipelinesState, r.ProgressState = read(stdio.State(viewstate.PipelinesState)), read(stdio.State(viewstate.PipelineProgressState))
r.SignersState = seatStateRead(holderOf(stdio.Ask), func(name string) view.StateRead { return read(stdio.State(name)) }, view.VaultSeat, view.SignersState)
r.ConditionsState, r.AttemptsState = read(stdio.State(view.StatusConditionsState)), read(stdio.State(view.StatusAttemptsState))
r.PresenceState, r.StuckState = read(stdio.State(view.StatusPresenceState)), read(stdio.State(view.StatusStuckState))
@@ -66,7 +66,6 @@ const (
func TestModuleToolAnswersTheStoredValue(t *testing.T) {
t.Run("indexer_packages answers the published modules", packagesToolAnswersThePublishedModules)
t.Run("view_tiers names each module's wait", tiersToolSaysEachModulesWait)
t.Run("view_pipelines reads the kept pipelines", pipelinesToolReadsThePipelines)
t.Run("each tool answers through the owners' read tools as before", ownersAnswerTheToolsAsBefore)
t.Run("node_manifest_reads asks every node's engine in one call", manifestReadsAsksEveryNode)
r := fake{"a": withFP2, "b": withNone}.reader()
@@ -697,27 +696,3 @@ func TestBuildSeatStatesAreReadInTheHoldersState(t *testing.T) {
t.Errorf("the provisions read %v, %v; want acme-mesh-provisioner.provisions", named, err)
}
}
// view_pipelines answers the view's kept pipelines with their steps and the progress of a running step, narrowed by its arguments, and
// refuses a since that is no time.
func pipelinesToolReadsThePipelines(t *testing.T) {
r := view.Reader{
PipelinesState: fake{
"main.9.web": `{"pipeline":"main.9.web","module":"web","branch":"main","commit":"c","outcome":"running","heard":"2026-11-16T03:00:00Z","started":"2026-11-16T02:00:00Z",` +
`"steps":[{"step":"build","artifact":"web:program","status":"required","outcome":"running","ask":"a1"}]}`,
"feat.8.db": `{"pipeline":"feat.8.db","module":"db","branch":"feat","commit":"d","outcome":"passed","heard":"2026-11-16T01:00:00Z","steps":[]}`,
}.reader().State,
ProgressState: fake{"a1": `{"ask":"a1","phase":"fetch","at":"2026-11-16T03:00:00Z","seat":"node-build-go","node":"ace"}`}.reader().State,
}
out, err := run(t, r, view.PipelinesReadTool, map[string]any{"branch": "main", "since": "2026-11-16T02:30:00Z", "limit": float64(5)})
a, ok := out.(view.PipelinesAnswer)
if err != nil || !ok || len(a.Pipelines) != 1 || a.Pipelines[0].Module != "web" || a.Pipelines[0].Steps[0].Progress == nil || a.Pipelines[0].Steps[0].Progress.Phase != "fetch" {
t.Fatalf("%+v %v", out, err)
}
if out, _ := run(t, r, view.PipelinesReadTool, map[string]any{}); len(out.(view.PipelinesAnswer).Pipelines) != 2 {
t.Errorf("no arguments gives every pipeline: %+v", out)
}
if _, err := run(t, r, view.PipelinesReadTool, map[string]any{"since": "soon"}); err == nil {
t.Error("a since that is no time was answered")
}
}
@@ -191,9 +191,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
if err := hearWaits(ctx, hub, js, own.Waits, logger.Printf); err != nil {
return fmt.Errorf("the waits' events are not handled: %w", err)
}
if err := hearPipelines(ctx, hub, js, own.Pipelines, own.Progress, time.Now, logger.Printf); err != nil {
return fmt.Errorf("the pipelines' events are not handled: %w", err)
}
if err := hub.StartUntilBound(ctx, js, cred, nil, nil); err != nil {
logger.Printf("the consumer is not heard yet (tried again until it is): %v", err)
}
@@ -1,108 +0,0 @@
package main
import (
"context"
"errors"
"time"
view "mesh-delivery-view/internal/view"
"git.novox.be/novox/mesh-sdk/go/process"
"github.com/nats-io/nats.go/jetstream"
)
// bucketStore is a key-value bucket of the view's own as view.PipelineStore: a put over a revision is a compare-and-set.
type bucketStore struct {
ctx context.Context
js jetstream.JetStream
bucket string
}
func (s bucketStore) kv() (jetstream.KeyValue, error) { return s.js.KeyValue(s.ctx, s.bucket) }
func (s bucketStore) Get(key string) ([]byte, uint64, error) {
kv, err := s.kv()
if err != nil {
return nil, 0, err
}
e, err := kv.Get(s.ctx, key)
if errors.Is(err, jetstream.ErrKeyNotFound) {
return nil, 0, view.ErrNotKept
}
if err != nil {
return nil, 0, err
}
if e.Operation() != jetstream.KeyValuePut {
return nil, 0, view.ErrNotKept
}
return e.Value(), e.Revision(), nil
}
func (s bucketStore) Put(key string, value []byte, revision uint64) error {
kv, err := s.kv()
if err != nil {
return err
}
if revision == 0 {
_, err = kv.Create(s.ctx, key, value)
return err
}
_, err = kv.Update(s.ctx, key, value, revision)
return err
}
func (s bucketStore) Delete(key string) error {
kv, err := s.kv()
if err != nil {
return err
}
if err := kv.Delete(s.ctx, key); errors.Is(err, jetstream.ErrKeyNotFound) {
return view.ErrNotKept
} else {
return err
}
}
func (s bucketStore) Keys() ([]string, error) {
kv, err := s.kv()
if err != nil {
return nil, err
}
keys, err := kv.Keys(s.ctx)
if errors.Is(err, jetstream.ErrNoKeysFound) {
return nil, nil
}
return keys, err
}
// hearPipelines registers the handlers for the pipeline runner's pipeline and step events and the build seats' step-progress on the
// process's hub, each kept in the view's states pipelines and pipeline-progress; one that could not be kept is not acknowledged, so
// the bus gives it again (design 47 G *Pipelines per module*).
func hearPipelines(ctx context.Context, hub *process.EventHub, js jetstream.JetStream, pipelines, progress string, now func() time.Time, logf func(string, ...any)) error {
ps, gs := bucketStore{ctx, js, pipelines}, bucketStore{ctx, js, progress}
for _, subject := range view.PipelineSubjects() {
if err := hub.Handle(subject, func(m jetstream.Msg) {
if err := view.ApplyPipelineEvent(m.Subject(), m.Data(), ps, gs, now()); err != nil {
logf("the pipeline event %s is not kept (the bus gives it again): %v", m.Subject(), err)
_ = m.Nak()
return
}
_ = m.Ack()
}); err != nil {
return err
}
}
for _, subject := range view.ProgressSubjects() {
if err := hub.Handle(subject, func(m jetstream.Msg) {
if err := view.ApplyProgress(m.Subject(), m.Data(), gs); err != nil {
logf("the step progress %s is not kept (the bus gives it again): %v", m.Subject(), err)
_ = m.Nak()
return
}
_ = m.Ack()
}); err != nil {
return err
}
}
return nil
}
@@ -1,80 +0,0 @@
//go:build integration
package main
import (
"context"
"encoding/json"
"errors"
"mesh-delivery-view/internal/view"
"testing"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-view/viewstate"
"github.com/nats-io/nats-server/v2/server"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
)
// The pipelines are kept in the view's own buckets of a real bus: a compare-and-set that loses is refused, a key not there reads as not
// kept, and the events fill them whole.
func TestPipelinesAreKeptInTheViewsBucketsOnTheBus(t *testing.T) {
srv, err := server.NewServer(&server.Options{Host: "127.0.0.1", Port: -1, JetStream: true, StoreDir: t.TempDir(), NoLog: true, NoSigs: true})
if err != nil {
t.Fatal(err)
}
go srv.Start()
if !srv.ReadyForConnections(10 * time.Second) {
t.Fatal("the bus did not come up")
}
t.Cleanup(srv.Shutdown)
nc, err := nats.Connect(srv.ClientURL())
if err != nil {
t.Fatal(err)
}
t.Cleanup(nc.Close)
js, err := jetstream.New(nc)
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
b := viewstate.BucketsOf(testView)
for _, name := range []string{b.Pipelines, b.Progress} {
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: name, History: 1}); err != nil {
t.Fatal(err)
}
}
ps, gs := bucketStore{ctx, js, b.Pipelines}, bucketStore{ctx, js, b.Progress}
if _, _, err := ps.Get("nothing"); !errors.Is(err, view.ErrNotKept) {
t.Fatalf("a key not there: %v", err)
}
now := time.Now()
const sub = "mesh.seat.mesh-pipeline-runner.event."
if err := view.ApplyPipelineEvent(sub+"pipeline-started.m.main.c1", []byte(`{"v":1,"pipeline":"main.1.m","module":"m","branch":"main","commit":"c1","steps":[{"step":"build","artifact":"m:program","status":"required"}]}`), ps, gs, now); err != nil {
t.Fatal(err)
}
if err := view.ApplyPipelineEvent(sub+"step-started.m.main.c1", []byte(`{"v":1,"pipeline":"main.1.m","module":"m","branch":"main","commit":"c1","result":{"step":"build","artifact":"m:program","status":"required","outcome":"running","ask":"a1"}}`), ps, gs, now); err != nil {
t.Fatal(err)
}
if err := view.ApplyProgress("mesh.seat.node-build-go.event.step-progress.ace", []byte(`{"asker":"r","ask":"a1","phase":"run","at":"2026-11-16T03:01:00Z"}`), gs); err != nil {
t.Fatal(err)
}
_, rev, err := ps.Get("main.1.m")
if err != nil || rev == 0 {
t.Fatalf("the pipeline is kept: %d %v", rev, err)
}
if err := ps.Put("main.1.m", []byte(`{}`), rev-1); err == nil {
t.Error("a write over a stale revision was kept")
}
if err := ps.Put("main.1.m", []byte(`{}`), 0); err == nil {
t.Error("a create over a kept key was kept")
}
read := func(s bucketStore) view.StateRead {
return view.StateRead{Keys: s.Keys, Get: func(k string) (json.RawMessage, error) { v, _, err := s.Get(k); return v, err }}
}
a, err := (view.Reader{PipelinesState: read(ps), ProgressState: read(gs)}).Pipelines(view.PipelinesQuery{})
if err != nil || len(a.Pipelines) != 1 || a.Pipelines[0].Steps[0].Progress == nil || a.Pipelines[0].Steps[0].Progress.Phase != "run" {
t.Fatalf("%+v %v", a, err)
}
}
@@ -1,24 +0,0 @@
package main
import (
"testing"
view "mesh-delivery-view/internal/view"
"git.novox.be/novox/mesh-sdk/go/process"
"github.com/nats-io/nats.go/jetstream"
)
// The pipelines' and the progress's subjects overlap none of the waits' nor each other: the hub gives a message one taker.
func TestPipelineSubjectsOverlapNoOtherHandler(t *testing.T) {
hub := process.NewEventHub(func(string, ...any) {})
patterns := append(append(waitPatterns(), view.PipelineSubjects()...), view.ProgressSubjects()...)
if len(patterns) != len(waitPatterns())+4+3 {
t.Fatalf("patterns %v", patterns)
}
for _, p := range patterns {
if err := hub.Handle(p, func(jetstream.Msg) {}); err != nil {
t.Fatalf("the pattern %s overlaps another: %v", p, err)
}
}
}
-4
View File
@@ -16,10 +16,6 @@ require (
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildnode v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildimage v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds v0.0.0-00010101000000-000000000000
@@ -1,474 +0,0 @@
package view
import (
"encoding/json"
"errors"
"fmt"
"sort"
"strings"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildimage"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildnode"
"git.novox.be/novox/mesh-sdk/go/keys"
)
// The view keeps the pipeline runner's pipelines in its own state `pipelines` and the latest progress of each running ask in its
// state `pipeline-progress`, from the seat mesh-pipeline-runner's events pipeline-started, step-started, step-ended and
// pipeline-ended and the build seats' step-progress; view_pipelines reads them (design 47 G *Pipelines per module*).
// PipelinesReadTool reads the view's pipelines, each whole with its steps and the progress of the steps that run.
const PipelinesReadTool = "view_pipelines"
// MaxPipelines is how many pipelines the state `pipelines` keeps: when more are kept, the longest unheard ones go. MaxProgress is the
// same for the state `pipeline-progress`, whose entry also goes when its step ends. DefaultPipelinesLimit is how many a read gives
// when no limit is asked, MaxPipelinesLimit the most.
const (
MaxPipelines = 500
MaxProgress = 200
DefaultPipelinesLimit = 50
MaxPipelinesLimit = 500
pruneSlack = 50 // a state over its cap by less than this is not cut yet, so a replay does not read it at every new key
pipelineWrites = 5 // how often a write over a changed key is tried again before the event is left for the bus to give again
)
// The seat events the view hears for its pipelines, and the build seats whose step-progress it hears.
const (
EventStepProgress = "step-progress"
)
// PipelineSubjects are the subjects of the pipeline runner's events the view hears, each the seat contract's.
func PipelineSubjects() []string {
return []string{meshpipelinerunner.SubjectPipelineStartedAll, meshpipelinerunner.SubjectStepStartedAll,
meshpipelinerunner.SubjectStepEndedAll, meshpipelinerunner.SubjectPipelineEndedAll}
}
// ProgressSubjects are the build seats' step-progress subjects the view hears, each the seat contract's.
func ProgressSubjects() []string {
return []string{nodebuildgo.SubjectStepProgressAll, nodebuildimage.SubjectStepProgressAll, nodebuildnode.SubjectStepProgressAll}
}
// PipelineRecord is the value of a pipelines key: the pipeline as the runner said it (the steps with their outcomes, why, times)
// and Heard, when the view last kept an event of it.
type PipelineRecord struct {
meshpipelinerunner.Pipeline
Heard time.Time `json:"heard"`
}
// ProgressRecord is the value of a pipeline-progress key: the latest progress said of one ask, with the seat and the node that said it.
type ProgressRecord struct {
Ask string `json:"ask"`
Asker string `json:"asker,omitempty"`
Phase string `json:"phase"`
At time.Time `json:"at"`
Seat string `json:"seat"`
Node string `json:"node,omitempty"`
}
// ErrNotKept is what a store's Get says of a key it does not hold.
var ErrNotKept = errors.New("not kept")
// PipelineStore is the view's own bucket as the pipelines are kept in it: a write over a revision is refused when the key moved
// (revision 0 creates it).
type PipelineStore interface {
Get(key string) (value []byte, revision uint64, err error)
Put(key string, value []byte, revision uint64) error
Delete(key string) error
Keys() ([]string, error)
}
// ProgressKey is the key of an ask's progress.
func ProgressKey(ask string) string { return keys.EscapeName(ask) }
// ApplyPipelineEvent keeps one of the pipeline runner's events (its subject names which) in pipelines, and the progress of an ask a
// step's end or the pipeline's end closes in progress. An event that cannot be kept is an error and the bus gives it again.
func ApplyPipelineEvent(subject string, body []byte, pipelines, progress PipelineStore, now time.Time) error {
parts := strings.Split(subject, ".")
if len(parts) < 5 || parts[0] != "mesh" || parts[1] != "seat" || parts[3] != "event" {
return fmt.Errorf("%s is no seat event's subject", subject)
}
event := parts[4]
switch event {
case meshpipelinerunner.EventPipelineStarted:
var e meshpipelinerunner.PipelineStarted
if err := json.Unmarshal(body, &e); err != nil {
return fmt.Errorf("%s: %w", subject, err)
}
return changePipeline(pipelines, e.Pipeline, now, func(p *PipelineRecord) {
fillPipeline(p, e.Pipeline, e.Module, e.Branch, e.Commit, e.Group, e.Run, e.Fingerprint, e.Revision, e.Source)
if e.Published != nil {
p.Published = e.Published
}
if p.Started == nil {
p.Started = e.At
}
for _, s := range e.Steps {
if stepIndex(p.Steps, s.Step, s.Artifact) < 0 {
p.Steps = append(p.Steps, meshpipelinerunner.StepResult{Step: s.Step, Artifact: s.Artifact, Status: s.Status})
}
}
})
case meshpipelinerunner.EventStepStarted, meshpipelinerunner.EventStepEnded:
var e meshpipelinerunner.StepEvent
if err := json.Unmarshal(body, &e); err != nil {
return fmt.Errorf("%s: %w", subject, err)
}
err := changePipeline(pipelines, e.Pipeline, now, func(p *PipelineRecord) {
fillPipeline(p, e.Pipeline, e.Module, e.Branch, e.Commit, e.Group, e.Run, e.Fingerprint, e.Revision, e.Source)
i := stepIndex(p.Steps, e.Result.Step, e.Result.Artifact)
switch {
case i < 0:
p.Steps = append(p.Steps, e.Result)
case p.Outcome != meshpipelinerunner.PipelineEndedOutcomeRunning && p.Outcome != "" && p.Steps[i].Outcome != "":
// The pipeline's end said every step whole: a step's event heard after it changes nothing.
default:
p.Steps[i] = e.Result
}
})
if err != nil {
return err
}
if event == meshpipelinerunner.EventStepEnded && e.Result.Ask != "" {
return deleteKey(progress, ProgressKey(e.Result.Ask))
}
return nil
case meshpipelinerunner.EventPipelineEnded:
var e meshpipelinerunner.PipelineEnded
if err := json.Unmarshal(body, &e); err != nil {
return fmt.Errorf("%s: %w", subject, err)
}
err := changePipeline(pipelines, e.Pipeline, now, func(p *PipelineRecord) {
fillPipeline(p, e.Pipeline, e.Module, e.Branch, e.Commit, e.Group, e.Run, e.Fingerprint, e.Revision, e.Source)
p.Outcome, p.Why, p.Steps, p.Tests, p.OverBudget, p.Widens = e.Outcome, e.Why, e.Steps, e.Tests, e.OverBudget, e.Widens
p.EndCount, p.Ended = e.EndCount, e.Ended
if e.Started != nil {
p.Started = e.Started
}
})
if err != nil {
return err
}
for _, s := range e.Steps {
if s.Ask != "" {
if err := deleteKey(progress, ProgressKey(s.Ask)); err != nil {
return err
}
}
}
return nil
}
return fmt.Errorf("%s is no pipeline event the view hears", subject)
}
// fillPipeline puts what an event names of its pipeline into p; a pipeline first heard of by a step's event has an outcome of
// running until its end says another.
func fillPipeline(p *PipelineRecord, pipeline, module, branch, commit, group, run, fingerprint string, revision, source int64) {
q := &p.Pipeline
q.Pipeline, q.Module, q.Branch, q.Commit = pipeline, module, branch, commit
q.Group, q.Run, q.Fingerprint, q.Revision, q.Source = group, run, fingerprint, revision, source
if q.Outcome == "" {
q.Outcome = meshpipelinerunner.PipelineEndedOutcomeRunning
}
}
func stepIndex(steps []meshpipelinerunner.StepResult, step, artifact string) int {
for i, s := range steps {
if s.Step == step && s.Artifact == artifact {
return i
}
}
return -1
}
// changePipeline reads a pipeline's record (a new one when none), lets change alter it and writes it over the revision it read,
// again when the key moved in between; a new key over the cap cuts the oldest heard (prune).
func changePipeline(store PipelineStore, key string, now time.Time, change func(*PipelineRecord)) error {
if strings.TrimSpace(key) == "" {
return errors.New("the event names no pipeline")
}
var err error
for try := 0; try < pipelineWrites; try++ {
var rec PipelineRecord
raw, rev, gerr := store.Get(key)
if gerr != nil && !errors.Is(gerr, ErrNotKept) {
return gerr
}
if rev != 0 && len(raw) > 0 {
if jerr := json.Unmarshal(raw, &rec); jerr != nil {
return fmt.Errorf("the pipeline %s is kept in a form that does not read: %w", key, jerr)
}
}
change(&rec)
rec.Heard = now.UTC()
out, jerr := json.Marshal(rec)
if jerr != nil {
return jerr
}
if err = store.Put(key, out, rev); err == nil {
if rev == 0 {
return prunePipelines(store, MaxPipelines)
}
return nil
}
}
return fmt.Errorf("the pipeline %s moved while it was kept: %w", key, err)
}
func deleteKey(store PipelineStore, key string) error {
if err := store.Delete(key); err != nil && !errors.Is(err, ErrNotKept) {
return err
}
return nil
}
// prunePipelines cuts the pipelines to max when more than max plus the slack are kept: the ones heard longest ago go.
func prunePipelines(store PipelineStore, max int) error {
all, err := store.Keys()
if err != nil || len(all) <= max+pruneSlack {
return err
}
type kept struct {
key string
heard time.Time
}
var ks []kept
for _, k := range all {
raw, _, err := store.Get(k)
if err != nil {
if errors.Is(err, ErrNotKept) {
continue
}
return err
}
var h struct {
Heard time.Time `json:"heard"`
}
_ = json.Unmarshal(raw, &h)
ks = append(ks, kept{k, h.Heard})
}
sort.Slice(ks, func(i, j int) bool { return ks[i].heard.Before(ks[j].heard) })
for i := 0; i < len(ks)-max; i++ {
if err := deleteKey(store, ks[i].key); err != nil {
return err
}
}
return nil
}
// ApplyProgress keeps one build seat's step-progress (a subject mesh.seat.<seat>.event.step-progress.<node>): the latest per ask, so a
// progress older than the one kept changes nothing.
func ApplyProgress(subject string, body []byte, progress PipelineStore) error {
parts := strings.Split(subject, ".")
if len(parts) != 6 || parts[0] != "mesh" || parts[1] != "seat" || parts[3] != "event" || parts[4] != EventStepProgress {
return fmt.Errorf("%s is no step-progress subject", subject)
}
var e struct {
Ask string `json:"ask"`
Asker string `json:"asker"`
Phase string `json:"phase"`
At time.Time `json:"at"`
}
if err := json.Unmarshal(body, &e); err != nil {
return fmt.Errorf("%s: %w", subject, err)
}
if e.Ask == "" || e.Phase == "" {
return fmt.Errorf("%s names no ask or no phase", subject)
}
key := ProgressKey(e.Ask)
rec := ProgressRecord{Ask: e.Ask, Asker: e.Asker, Phase: e.Phase, At: e.At.UTC(), Seat: parts[2], Node: parts[5]}
out, err := json.Marshal(rec)
if err != nil {
return err
}
for try := 0; try < pipelineWrites; try++ {
raw, rev, gerr := progress.Get(key)
if gerr != nil && !errors.Is(gerr, ErrNotKept) {
return gerr
}
if rev != 0 {
var have ProgressRecord
if json.Unmarshal(raw, &have) == nil && !have.At.Before(rec.At) {
return nil
}
}
if err = progress.Put(key, out, rev); err == nil {
if rev == 0 {
return pruneProgress(progress, MaxProgress)
}
return nil
}
}
return fmt.Errorf("the progress of %s moved while it was kept: %w", e.Ask, err)
}
// pruneProgress cuts the progress to max when more than max plus the slack are kept: the oldest said go (an ask whose end was never
// heard).
func pruneProgress(store PipelineStore, max int) error {
all, err := store.Keys()
if err != nil || len(all) <= max+pruneSlack {
return err
}
type kept struct {
key string
at time.Time
}
var ks []kept
for _, k := range all {
raw, _, err := store.Get(k)
if err != nil {
if errors.Is(err, ErrNotKept) {
continue
}
return err
}
var p ProgressRecord
_ = json.Unmarshal(raw, &p)
ks = append(ks, kept{k, p.At})
}
sort.Slice(ks, func(i, j int) bool { return ks[i].at.Before(ks[j].at) })
for i := 0; i < len(ks)-max; i++ {
if err := deleteKey(store, ks[i].key); err != nil {
return err
}
}
return nil
}
// PipelinesQuery is what view_pipelines asks: each field narrows, none means all.
type PipelinesQuery struct {
Branch, Module, Outcome string
Since time.Time // pipelines the view kept an event of at or after it
Limit int
}
// StepProgressView is a running step's latest progress: its phase and when it was said.
type StepProgressView struct {
Phase string `json:"phase"`
At time.Time `json:"at"`
Seat string `json:"seat"`
Node string `json:"node,omitempty"`
}
// PipelineStepView is a step as the runner said it, with the progress of the step's ask while it runs.
type PipelineStepView struct {
meshpipelinerunner.StepResult
Progress *StepProgressView `json:"progress,omitempty"`
}
// PipelineView is a pipeline whole with its steps, Heard when the view last kept an event of it.
type PipelineView struct {
meshpipelinerunner.Pipeline
Steps []PipelineStepView `json:"steps"`
Heard time.Time `json:"heard"`
}
// PipelinesAnswer is view_pipelines' answer: the pipelines newest first, Matching how many fit before the limit cut them.
type PipelinesAnswer struct {
Pipelines []PipelineView `json:"pipelines"`
Matching int `json:"matching"`
Cut bool `json:"cut,omitempty"`
// Unreadable names the keys of a state that do not read.
Unreadable []string `json:"unreadable,omitempty"`
}
// ParsePipelinesQuery reads the tool's arguments; since is an RFC 3339 time, limit a whole number.
func ParsePipelinesQuery(args map[string]any) (PipelinesQuery, error) {
var q PipelinesQuery
q.Branch, _ = args["branch"].(string)
q.Module, _ = args["module"].(string)
q.Outcome, _ = args["outcome"].(string)
if raw, _ := args["since"].(string); strings.TrimSpace(raw) != "" {
at, err := time.Parse(time.RFC3339Nano, strings.TrimSpace(raw))
if err != nil {
return q, fmt.Errorf("since %q is not an RFC 3339 time: %w", raw, err)
}
q.Since = at
}
switch l := args["limit"].(type) {
case nil:
case float64:
if l < 0 || l != float64(int(l)) {
return q, fmt.Errorf("limit %v is no count", l)
}
q.Limit = int(l)
default:
return q, fmt.Errorf("limit %v is no count", l)
}
return q, nil
}
// Pipelines is view_pipelines: the kept pipelines that fit q, newest first (by their start, else when heard), each with the latest
// progress of its running steps from the progress state.
func (r Reader) Pipelines(q PipelinesQuery) (PipelinesAnswer, error) {
out := PipelinesAnswer{Pipelines: []PipelineView{}}
if r.PipelinesState.Keys == nil && r.PipelinesState.All == nil {
return out, errors.New("the pipelines state cannot be read here")
}
all, err := values(r.PipelinesState)
if err != nil {
return out, err
}
var progress map[string]json.RawMessage
if r.ProgressState.Keys != nil || r.ProgressState.All != nil {
if progress, err = values(r.ProgressState); err != nil {
return out, fmt.Errorf("the pipelines' progress cannot be read: %w", err)
}
}
var found []PipelineView
for _, key := range sortedKeys(all) {
var rec PipelineRecord
if err := json.Unmarshal(all[key], &rec); err != nil {
out.Unreadable = append(out.Unreadable, key)
continue
}
if q.Branch != "" && rec.Branch != q.Branch || q.Module != "" && rec.Module != q.Module || q.Outcome != "" && rec.Outcome != q.Outcome ||
!q.Since.IsZero() && rec.Heard.Before(q.Since) {
continue
}
pv := PipelineView{Pipeline: rec.Pipeline, Heard: rec.Heard, Steps: make([]PipelineStepView, 0, len(rec.Steps))}
for _, s := range rec.Steps {
sv := PipelineStepView{StepResult: s}
if s.Ask != "" && s.Outcome == "running" {
var p ProgressRecord
if raw := progress[ProgressKey(s.Ask)]; len(raw) > 0 && json.Unmarshal(raw, &p) == nil {
sv.Progress = &StepProgressView{Phase: p.Phase, At: p.At, Seat: p.Seat, Node: p.Node}
}
}
pv.Steps = append(pv.Steps, sv)
}
found = append(found, pv)
}
when := func(p PipelineView) time.Time {
if p.Started != nil {
return *p.Started
}
return p.Heard
}
sort.SliceStable(found, func(i, j int) bool { return when(found[i]).After(when(found[j])) })
out.Matching = len(found)
limit := q.Limit
if limit <= 0 {
limit = DefaultPipelinesLimit
}
if limit > MaxPipelinesLimit {
limit = MaxPipelinesLimit
}
if len(found) > limit {
found, out.Cut = found[:limit], true
}
if found != nil {
out.Pipelines = found
}
return out, nil
}
func sortedKeys(m map[string]json.RawMessage) []string {
ks := make([]string, 0, len(m))
for k := range m {
ks = append(ks, k)
}
sort.Strings(ks)
return ks
}
@@ -1,197 +0,0 @@
package view
import (
"encoding/json"
"errors"
"fmt"
"sort"
"testing"
"time"
)
// memStore is a bucket in memory with a revision per key; failPuts makes that many puts fail as a moved key does.
type memStore struct {
vals map[string][]byte
revs map[string]uint64
next uint64
failPuts int
}
func newMem() *memStore { return &memStore{vals: map[string][]byte{}, revs: map[string]uint64{}} }
func (m *memStore) Get(k string) ([]byte, uint64, error) {
v, ok := m.vals[k]
if !ok {
return nil, 0, ErrNotKept
}
return v, m.revs[k], nil
}
func (m *memStore) Put(k string, v []byte, rev uint64) error {
if m.failPuts > 0 {
m.failPuts--
return errors.New("wrong last sequence")
}
if m.revs[k] != rev {
return errors.New("wrong last sequence")
}
m.next++
m.vals[k], m.revs[k] = v, m.next
return nil
}
func (m *memStore) Delete(k string) error { delete(m.vals, k); delete(m.revs, k); return nil }
func (m *memStore) Keys() ([]string, error) {
var ks []string
for k := range m.vals {
ks = append(ks, k)
}
sort.Strings(ks)
return ks, nil
}
func (m *memStore) read() StateRead {
return StateRead{Keys: m.Keys, Get: func(k string) (json.RawMessage, error) { return m.vals[k], nil }}
}
const runner = "mesh.seat.mesh-pipeline-runner.event."
func pat(h int) time.Time { return time.Date(2026, 11, 16, h, 0, 0, 0, time.UTC) }
func started(p, module string, h int) string {
return fmt.Sprintf(`{"v":1,"pipeline":%q,"module":%q,"branch":"main","commit":"c1","at":%q,"steps":[{"step":"build","artifact":%q,"status":"required"},{"step":"unit","status":"required"}]}`,
p, module, pat(h).Format(time.RFC3339), module+":program")
}
func TestPipelinesAreKeptFromTheRunnersEventsAndReadWithTheirProgress(t *testing.T) {
ps, gs := newMem(), newMem()
apply := func(subject, body string, h int) {
t.Helper()
if err := ApplyPipelineEvent(subject, []byte(body), ps, gs, pat(h)); err != nil {
t.Fatalf("%s: %v", subject, err)
}
}
apply(runner+"pipeline-started.adwaita.main.c1", started("main.969.adwaita", "adwaita", 1), 1)
apply(runner+"pipeline-started.avahi.main.c1", started("main.970.avahi", "avahi", 2), 2)
running := `{"v":1,"pipeline":"main.969.adwaita","module":"adwaita","branch":"main","commit":"c1","result":{"step":"build","artifact":"adwaita:program","status":"required","outcome":"running","ask":"ask/1","seat":"node-build-go"}}`
apply(runner+"step-started.adwaita.main.c1", running, 3)
if err := ApplyProgress("mesh.seat.node-build-go.event.step-progress.ace", []byte(`{"asker":"mesh-pipeline-runner","ask":"ask/1","phase":"build","at":"2026-11-16T03:01:00Z"}`), gs); err != nil {
t.Fatal(err)
}
// An older progress of the same ask changes nothing: the latest is kept.
if err := ApplyProgress("mesh.seat.node-build-go.event.step-progress.g14", []byte(`{"asker":"x","ask":"ask/1","phase":"fetch","at":"2026-11-16T03:00:00Z"}`), gs); err != nil {
t.Fatal(err)
}
r := Reader{PipelinesState: ps.read(), ProgressState: gs.read()}
got, err := r.Pipelines(PipelinesQuery{})
if err != nil || len(got.Pipelines) != 2 || got.Pipelines[0].Module != "avahi" || got.Pipelines[1].Module != "adwaita" {
t.Fatalf("newest first: %+v, %v", got, err)
}
ad := got.Pipelines[1]
if ad.Outcome != "running" || len(ad.Steps) != 2 || ad.Steps[0].Progress == nil || ad.Steps[0].Progress.Phase != "build" || ad.Steps[0].Progress.Node != "ace" ||
!ad.Steps[0].Progress.At.Equal(time.Date(2026, 11, 16, 3, 1, 0, 0, time.UTC)) || ad.Steps[1].Progress != nil || ad.Steps[1].Status != "required" {
t.Fatalf("the running step carries the latest progress: %+v", ad.Steps)
}
// The step's end closes its progress; the pipeline's end says the outcome and every step whole.
apply(runner+"step-ended.adwaita.main.c1", `{"v":1,"pipeline":"main.969.adwaita","module":"adwaita","branch":"main","commit":"c1","result":{"step":"build","artifact":"adwaita:program","status":"required","outcome":"passed","ask":"ask/1"}}`, 4)
if len(gs.vals) != 0 {
t.Errorf("the progress of an ended step stays: %v", gs.vals)
}
apply(runner+"pipeline-ended.adwaita.main.c1", `{"v":1,"pipeline":"main.969.adwaita","module":"adwaita","branch":"main","commit":"c1","outcome":"failed","why":"unit failed","ended":"2026-11-16T05:00:00Z",
"steps":[{"step":"build","artifact":"adwaita:program","status":"required","outcome":"passed"},{"step":"unit","status":"required","outcome":"failed","why":"TestX"}]}`, 5)
// A step event heard after the end changes nothing.
apply(runner+"step-started.adwaita.main.c1", running, 6)
for _, tc := range []struct {
name string
q PipelinesQuery
want []string
}{
{"all", PipelinesQuery{}, []string{"avahi", "adwaita"}},
{"outcome", PipelinesQuery{Outcome: "failed"}, []string{"adwaita"}},
{"module", PipelinesQuery{Module: "avahi"}, []string{"avahi"}},
{"branch", PipelinesQuery{Branch: "other"}, nil},
{"since heard", PipelinesQuery{Since: pat(6)}, []string{"adwaita"}},
{"limit", PipelinesQuery{Limit: 1}, []string{"avahi"}},
} {
a, err := r.Pipelines(tc.q)
var mods []string
for _, p := range a.Pipelines {
mods = append(mods, p.Module)
}
if err != nil || fmt.Sprint(mods) != fmt.Sprint(tc.want) {
t.Errorf("%s: %v, %v (want %v)", tc.name, mods, err, tc.want)
}
}
a, _ := r.Pipelines(PipelinesQuery{Limit: 1})
if !a.Cut || a.Matching != 2 {
t.Errorf("a cut says so: %+v", a)
}
a, _ = r.Pipelines(PipelinesQuery{Outcome: "failed"})
if w := a.Pipelines[0]; w.Why != "unit failed" || w.Ended == nil || w.Steps[1].Outcome != "failed" || w.Steps[0].Progress != nil || w.Steps[0].Outcome != "passed" {
t.Errorf("the ended pipeline is whole: %+v", w)
}
}
func TestAStepEventBeforeItsPipelinesStartCreatesThePipelineAndTheStartFillsIt(t *testing.T) {
ps, gs := newMem(), newMem()
if err := ApplyPipelineEvent(runner+"step-ended.m.main.c1", []byte(`{"v":1,"pipeline":"main.1.m","module":"m","branch":"main","commit":"c1","result":{"step":"build","artifact":"m:program","status":"required","outcome":"passed"}}`), ps, gs, pat(1)); err != nil {
t.Fatal(err)
}
if err := ApplyPipelineEvent(runner+"pipeline-started.m.main.c1", []byte(started("main.1.m", "m", 0)), ps, gs, pat(2)); err != nil {
t.Fatal(err)
}
a, _ := Reader{PipelinesState: ps.read()}.Pipelines(PipelinesQuery{})
if p := a.Pipelines[0]; len(p.Steps) != 2 || p.Steps[0].Outcome != "passed" || p.Steps[1].Step != "unit" || p.Started == nil || p.Outcome != "running" {
t.Fatalf("%+v", p)
}
}
func TestPipelineWritesRetryAMovedKeyAndRefuseWhatDoesNotReadAndPruneToTheCap(t *testing.T) {
ps, gs := newMem(), newMem()
ps.failPuts = pipelineWrites - 1
if err := ApplyPipelineEvent(runner+"pipeline-started.m.main.c1", []byte(started("main.1.m", "m", 0)), ps, gs, pat(1)); err != nil || len(ps.vals) != 1 {
t.Fatalf("a moved key is tried again: %v", err)
}
ps.failPuts = pipelineWrites
if err := ApplyPipelineEvent(runner+"pipeline-started.m.main.c1", []byte(started("main.1.m", "m", 0)), ps, gs, pat(1)); err == nil {
t.Error("a key that keeps moving is left for the bus to give again")
}
for _, bad := range [][2]string{{runner + "pipeline-started.m.main.c1", `{`}, {runner + "pipeline-started.m.main.c1", `{"module":"m"}`}, {runner + "wait-held.m.main.c1", `{}`}, {"mesh.mod.x.y", `{}`}} {
if err := ApplyPipelineEvent(bad[0], []byte(bad[1]), ps, gs, pat(1)); err == nil {
t.Errorf("%s %s was kept", bad[0], bad[1])
}
}
if err := ApplyProgress("mesh.seat.node-build-go.event.step-progress.ace", []byte(`{"ask":"","phase":"build"}`), gs); err == nil {
t.Error("a progress of no ask was kept")
}
// Over its cap by more than the slack, the longest unheard go, the newest stay.
ps = newMem()
for i := 0; i < MaxPipelines+pruneSlack+1; i++ {
p := fmt.Sprintf("main.%d.m", i)
if err := ApplyPipelineEvent(runner+"pipeline-started.m.main.c1", []byte(started(p, "m", 0)), ps, gs, pat(0).Add(time.Duration(i)*time.Second)); err != nil {
t.Fatal(err)
}
}
if len(ps.vals) != MaxPipelines {
t.Fatalf("%d pipelines kept, the cap is %d", len(ps.vals), MaxPipelines)
}
if _, _, err := ps.Get("main.0.m"); err == nil {
t.Error("the oldest heard stays")
}
if _, _, err := ps.Get(fmt.Sprintf("main.%d.m", MaxPipelines+pruneSlack)); err != nil {
t.Error("the newest heard went")
}
}
func TestPipelinesQueryReadsItsArguments(t *testing.T) {
q, err := ParsePipelinesQuery(map[string]any{"branch": "main", "since": "2026-11-16T01:00:00Z", "limit": float64(3)})
if err != nil || q.Branch != "main" || q.Limit != 3 || !q.Since.Equal(pat(1)) {
t.Fatalf("%+v %v", q, err)
}
for _, bad := range []map[string]any{{"since": "yesterday"}, {"limit": "3"}, {"limit": float64(-1)}, {"limit": 1.5}} {
if _, err := ParsePipelinesQuery(bad); err == nil {
t.Errorf("%v was read", bad)
}
}
if _, err := (Reader{}).Pipelines(PipelinesQuery{}); err == nil {
t.Error("a state that cannot be read here reads as none")
}
}
@@ -57,8 +57,6 @@ type Reader struct {
DifferencesState StateRead // the assigner's differences
HavesState StateRead // the node-engine's haves
ReachState StateRead // the view's own reach (ADR 0556)
PipelinesState StateRead // the view's own pipelines, from the pipeline runner's events
ProgressState StateRead // the view's own pipeline-progress: the latest progress of each running ask
WaitsState StateRead // the view's own waits: the parts' wait-held (design 47 G §512, A-242)
// HeardState is the FORGE messages the indexer handled (issue 1126 note 76); unset reads none.
HeardState StateRead
+4 -13
View File
@@ -198,9 +198,7 @@ func TestManifestIsTakenAndReadsOnly(t *testing.T) { // 1134/AC7
"mesh-catalog.catalog-revision", "mesh-catalog.registration-changed", "mesh-delivery.source-moved", "mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", "mesh-delivery-indexer.views.candidates", "mesh-delivery-indexer.views.taken", "mesh-delivery-teller.views.answers",
"mesh-wants.wait-held", "mesh-wants.wait-ended", "node-engine.wait-held", "node-engine.wait-ended", "mesh-pipeline-runner.wait-held", "mesh-pipeline-runner.wait-ended",
"mesh-grants.wait-held", "mesh-grants.wait-ended", "mesh-delivery-builder.wait-held", "mesh-delivery-builder.wait-ended",
"mesh-broker.recreated", "mesh-broker.dropped",
"mesh-pipeline-runner.pipeline-started", "mesh-pipeline-runner.step-started", "mesh-pipeline-runner.step-ended", "mesh-pipeline-runner.pipeline-ended",
"node-build-go.step-progress", "node-build-image.step-progress", "node-build-node.step-progress"}; !reflect.DeepEqual(m.Consumes, want) { // design 47 G §512, A-242: every part's wait-held and wait-ended
"mesh-broker.recreated", "mesh-broker.dropped"}; !reflect.DeepEqual(m.Consumes, want) { // design 47 G §512, A-242: every part's wait-held and wait-ended
t.Errorf("consumes = %v, want %v", m.Consumes, want)
}
// The one event the view says: a source revision is delivered (1906/D9, ADR 0539 §17), derived by it from the wants and the haves.
@@ -214,7 +212,7 @@ func TestManifestIsTakenAndReadsOnly(t *testing.T) { // 1134/AC7
// The states the view writes: the operator's answers to the asks that need no proof (ADR 0528 clause 6, 1297/B9), which the
// teller reads, and how far each repository reached (ADR 0556 clause 2, issue 1280), which the gate reads.
// Its first views of the owners' states and the finding of one unanswered past its bound (ADR 0615 §6).
if want := []state{{Name: "answers", History: 5}, {Name: "reach", History: 1}, {Name: "waits", History: 1}, {Name: "pipelines", History: 1}, {Name: "pipeline-progress", History: 1}, {Name: "first-views", History: 1}, {Name: "view-marks", History: 1}}; !reflect.DeepEqual(m.State, want) {
if want := []state{{Name: "answers", History: 5}, {Name: "reach", History: 1}, {Name: "waits", History: 1}, {Name: "first-views", History: 1}, {Name: "view-marks", History: 1}}; !reflect.DeepEqual(m.State, want) {
t.Errorf("state = %+v, want %+v", m.State, want)
}
if _, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 {
@@ -225,7 +223,7 @@ func TestManifestIsTakenAndReadsOnly(t *testing.T) { // 1134/AC7
}
// The two read tools of the indexer's modules (issue 1562), carried by a bundle the runtime launches; the indexer
// itself serves no tool (ADR 0437 §4). Carrying them gives the tool runner read access to every bucket the view reads.
if want := []string{"view_status", "indexer_module", "indexer_modules", "indexer_candidate", "indexer_candidates", "indexer_predefined", "indexer_config_candidates", "indexer_packages", "builder_build", "builder_builds", "builder_history", "assigner_wants", "assigner_want", "assigner_differences", "node_haves", "node_trusts", "build_usage", "view_reach", "view_pipelines", "indexer_module_history", "indexer_heard", "view_tiers", "view_first_views", "view_first_view_again", "node_manifest_reads", "view_seat_list", "view_seat_list_again", "answers"}; !reflect.DeepEqual(m.Tools, want) {
if want := []string{"view_status", "indexer_module", "indexer_modules", "indexer_candidate", "indexer_candidates", "indexer_predefined", "indexer_config_candidates", "indexer_packages", "builder_build", "builder_builds", "builder_history", "assigner_wants", "assigner_want", "assigner_differences", "node_haves", "node_trusts", "build_usage", "view_reach", "indexer_module_history", "indexer_heard", "view_tiers", "view_first_views", "view_first_view_again", "node_manifest_reads", "view_seat_list", "view_seat_list_again", "answers"}; !reflect.DeepEqual(m.Tools, want) {
t.Errorf("tools = %v, want %v", m.Tools, want)
}
// D-11: the holders and buckets the view names are its manifest's (no contract names a holder, novox/hq ADR 0634 §1): a
@@ -287,18 +285,11 @@ func TestManifestIsTakenAndReadsOnly(t *testing.T) { // 1134/AC7
own = append(own, "a-view_"+s.Name)
}
b := viewstate.BucketsOf("a-view")
for _, want := range []string{b.Answers, b.Reach, b.Waits, b.Pipelines, b.Progress} {
for _, want := range []string{b.Answers, b.Reach, b.Waits} {
if !slices.Contains(own, want) {
t.Errorf("the view keeps %s, its manifest's states are %v", want, own)
}
}
// The pipelines are the runner's events and the progress the build seats': each subject the view hears is of an event its manifest consumes.
for _, subject := range append(view.PipelineSubjects(), view.ProgressSubjects()...) {
p := strings.Split(subject, ".") // mesh.seat.<seat>.event.<event>...
if want := p[2] + "." + p[4]; !slices.Contains(m.Consumes, want) {
t.Errorf("the view hears %s, its manifest consumes %v", subject, m.Consumes)
}
}
// The packages key is the seat's key of its state packages for a module (<module>), as the indexer writes it: the
// escaped path, with no dot, so a reader never takes it for a <package>.<series> key.
const path = "git.novox.be/novox/mesh-sdk/go"
+1 -17
View File
@@ -22,7 +22,6 @@
"node_trusts",
"build_usage",
"view_reach",
"view_pipelines",
"indexer_module_history",
"indexer_heard",
"view_tiers",
@@ -61,14 +60,6 @@
"name": "waits",
"history": 1
},
{
"name": "pipelines",
"history": 1
},
{
"name": "pipeline-progress",
"history": 1
},
{
"name": "first-views",
"history": 1
@@ -103,14 +94,7 @@
"mesh-delivery-builder.wait-held",
"mesh-delivery-builder.wait-ended",
"mesh-broker.recreated",
"mesh-broker.dropped",
"mesh-pipeline-runner.pipeline-started",
"mesh-pipeline-runner.step-started",
"mesh-pipeline-runner.step-ended",
"mesh-pipeline-runner.pipeline-ended",
"node-build-go.step-progress",
"node-build-image.step-progress",
"node-build-node.step-progress"
"mesh-broker.dropped"
],
"invokes": [
"mesh-delivery-assigner.assigner_read_differences",
+3 -13
View File
@@ -26,22 +26,12 @@ const modulesHistory = 20
// ReachState is the delivery view's state `reach`: one key per repository (RepositoryKey), how far its modules reached.
const ReachState = "reach"
// PipelinesState is the delivery view's state `pipelines`: one key per pipeline (its name, <branch>.<revision>.<module>), the
// pipeline whole with its steps as the pipeline runner's events said them, and PipelineProgressState its state
// `pipeline-progress`: one key per running ask, the latest progress a build seat said of it.
const (
PipelinesState = "pipelines"
PipelineProgressState = "pipeline-progress"
)
// Buckets are the delivery view's own state buckets, `<module>_<state>`: waits, reach, answers, pipelines and the pipelines'
// progress.
type Buckets struct{ Waits, Reach, Answers, Pipelines, Progress string }
// Buckets are the delivery view's own state buckets, `<module>_<state>`: waits, reach and answers.
type Buckets struct{ Waits, Reach, Answers string }
// BucketsOf names the view's own buckets from module, its credential's, never a constant (design 47 G A-296).
func BucketsOf(module string) Buckets {
return Buckets{Waits: module + "_" + WaitsState, Reach: module + "_" + ReachState, Answers: module + "_" + ViewAnswersState,
Pipelines: module + "_" + PipelinesState, Progress: module + "_" + PipelineProgressState}
return Buckets{Waits: module + "_" + WaitsState, Reach: module + "_" + ReachState, Answers: module + "_" + ViewAnswersState}
}
// Reach is the value of a reach key: how far the modules of one repository reached on the nodes. Indexed is the last