Compare commits
54
Commits
go/v0.1.60
...
go/v0.1.46
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
93495203b5 | ||
|
|
5628ca2919 | ||
|
|
05c59ae000 | ||
|
|
e1630b57c6 | ||
|
|
5c4c4bc821 | ||
|
|
599bf62d23 | ||
|
|
0de195c5aa | ||
|
|
3160bb1fb8 | ||
|
|
93efda5db3 | ||
|
|
f01731fd4e | ||
|
|
251745f859 | ||
|
|
d1ec9d9072 | ||
|
|
14254dd0ba | ||
|
|
1e50642e3c | ||
|
|
b0082ec04f | ||
|
|
6b6e4c81f0 | ||
|
|
374d060cc0 | ||
|
|
1f878eec58 | ||
|
|
c9515da86f | ||
|
|
5f7137e676 | ||
|
|
902947a044 | ||
|
|
b95b3ef989 | ||
|
|
0f167db73d | ||
|
|
e5f54c5cdb | ||
|
|
bdb4edd820 | ||
|
|
31c2c5c7fe | ||
|
|
ad0052c6f5 | ||
|
|
aef8632d95 | ||
|
|
dedcc3ead7 | ||
|
|
a40bbec44a | ||
|
|
fe6c5961ae | ||
|
|
588a6ff8d5 | ||
|
|
ced27928d9 | ||
|
|
2cb6971194 | ||
|
|
6a730c33e6 | ||
|
|
d1aab1d08d | ||
|
|
c6b4b6054b | ||
|
|
4df45a3793 | ||
|
|
e2e13ddd29 | ||
|
|
dcf5b7fa8f | ||
|
|
1fe6ade359 | ||
|
|
9e09767a4b | ||
|
|
59443ab3f7 | ||
|
|
92d0ab8602 | ||
|
|
3a31331ad3 | ||
|
|
d1581666d2 | ||
|
|
d4d7895392 | ||
|
|
d6c209a10c | ||
|
|
51ac838732 | ||
|
|
06a52a10e0 | ||
|
|
773e2e33a1 | ||
|
|
d817d85fca | ||
|
|
6d5000871d | ||
|
|
5c7648744d |
@@ -216,7 +216,7 @@ type runPrep struct {
|
||||
seats []json.RawMessage
|
||||
manifests map[string]composer.Manifest
|
||||
none map[string]string
|
||||
programOf func(node string) (buildstep.RunProgram, string, error)
|
||||
programOf func(node string) (buildstep.RunProgram, string, map[string]string, error)
|
||||
err error
|
||||
}
|
||||
|
||||
@@ -274,7 +274,7 @@ func (r *compositionRun) start(ctx context.Context, key string, run meshwants.Co
|
||||
end(node + " gave no composition input: " + in.Why)
|
||||
return
|
||||
}
|
||||
program, engineFp, err := prep.programOf(node)
|
||||
program, engineFp, pins, err := prep.programOf(node)
|
||||
if err != nil {
|
||||
end(node + ": " + err.Error())
|
||||
return
|
||||
@@ -317,7 +317,8 @@ func (r *compositionRun) start(ctx context.Context, key string, run meshwants.Co
|
||||
return
|
||||
}
|
||||
a := buildstep.StepAsk{ID: ask, Kind: buildstep.AskCheck, Language: buildstep.AskGo, Asker: runAsker, At: r.now().UTC(),
|
||||
Run: &buildstep.RunJob{Program: program, Heads: []buildstep.RunHead{}, Input: ref, Reader: "the composition of " + node + " by the node-engine at " + program.Commit}}
|
||||
Run: &buildstep.RunJob{Program: program, Heads: []buildstep.RunHead{}, Input: ref, Reader: "the composition of " + node + " by the node-engine at " + program.Commit},
|
||||
Packages: pins} // the composer is built against its entry's pins, as every build (H-A-203, H-A-192)
|
||||
if err := r.put(ctx, a); err != nil {
|
||||
end(node + ": its run ask " + ask + " is not put: " + err.Error())
|
||||
}
|
||||
@@ -732,53 +733,76 @@ func programOfEntry(e meshcatalog.CatalogEntry) (buildstep.RunProgram, error) {
|
||||
return buildstep.RunProgram{Repository: e.Repository, Commit: e.Commit, Dir: dir, Package: composer.Package}, nil
|
||||
}
|
||||
|
||||
// pinsOfEntry are the mesh packages' versions an entry pins, the ones it names with a version (H-A-203, H-A-183): the pins
|
||||
// a run's program is built against, in the form the entry keeps and the builder sends. nil when it pins none.
|
||||
func pinsOfEntry(e meshcatalog.CatalogEntry) map[string]string {
|
||||
var out map[string]string
|
||||
for path, version := range e.Packages {
|
||||
if version == "" {
|
||||
continue
|
||||
}
|
||||
if out == nil {
|
||||
out = map[string]string{}
|
||||
}
|
||||
out[path] = version
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// programs answers, per machine, the composer its run builds and the node-engine fingerprint it was chosen from (A-226, A-243,
|
||||
// A-252, §719): where the revision's node-engine entry differs from the trunk's, that build for every machine; else the build the
|
||||
// machine's node-engine runs, as its `running` says, its entry read through mesh-catalog's `entry`; a machine whose node-engine's
|
||||
// `running` is not heard gets the revision's entry, said (A-451).
|
||||
func (r *compositionRun) programs(ctx context.Context, half mcc.Half) (func(node string) (buildstep.RunProgram, string, error), error) {
|
||||
func (r *compositionRun) programs(ctx context.Context, half mcc.Half) (func(node string) (buildstep.RunProgram, string, map[string]string, error), error) {
|
||||
engine, ok := nodeEngineEntry(half.Entries)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("the catalog revision %d holds no module claiming the seat %s", half.Revision.Revision, ne.Seat)
|
||||
}
|
||||
if trunk, ok := r.trunk(); !ok || func() bool { t, ok := nodeEngineEntry(trunk.Entries); return !ok || t.Fp != engine.Fp }() {
|
||||
p, err := programOfEntry(engine)
|
||||
return func(string) (buildstep.RunProgram, string, error) { return p, engine.Fp, err }, nil
|
||||
return func(string) (buildstep.RunProgram, string, map[string]string, error) {
|
||||
return p, engine.Fp, pinsOfEntry(engine), err
|
||||
}, nil
|
||||
}
|
||||
var mu sync.Mutex
|
||||
byFp := map[string]buildstep.RunProgram{}
|
||||
return func(node string) (buildstep.RunProgram, string, error) {
|
||||
type built struct {
|
||||
program buildstep.RunProgram
|
||||
pins map[string]string
|
||||
}
|
||||
byFp := map[string]built{}
|
||||
return func(node string) (buildstep.RunProgram, string, map[string]string, error) {
|
||||
running, ok := r.running(node, engine.Module)
|
||||
if !ok || running.Fingerprint == "" {
|
||||
r.logf("%s: no build of %s is heard running there, so its composer is the catalog revision's entry %s (A-451)", node,
|
||||
engine.Module, engine.Fp)
|
||||
p, err := programOfEntry(engine)
|
||||
return p, engine.Fp, err
|
||||
return p, engine.Fp, pinsOfEntry(engine), err
|
||||
}
|
||||
fp := running.Fingerprint
|
||||
mu.Lock()
|
||||
p, ok := byFp[fp]
|
||||
b, ok := byFp[fp]
|
||||
mu.Unlock()
|
||||
if ok {
|
||||
return p, fp, nil
|
||||
return b.program, fp, b.pins, nil
|
||||
}
|
||||
what := "the entry of the node-engine's build " + fp + " is not read"
|
||||
body, said, err := r.callRetry(ctx, entryVerbKey, meshcatalog.EntryInput{Fingerprint: fp})
|
||||
if err != nil || body == nil {
|
||||
return buildstep.RunProgram{}, fp, unreadErr(what, err, said)
|
||||
return buildstep.RunProgram{}, fp, nil, unreadErr(what, err, said)
|
||||
}
|
||||
var a meshcatalog.EntryAnswer
|
||||
if err := json.Unmarshal(body, &a); err != nil {
|
||||
return buildstep.RunProgram{}, fp, fmt.Errorf("%s: %w", what, err)
|
||||
return buildstep.RunProgram{}, fp, nil, fmt.Errorf("%s: %w", what, err)
|
||||
}
|
||||
p, err = programOfEntry(a.Entry)
|
||||
p, err := programOfEntry(a.Entry)
|
||||
if err != nil {
|
||||
return p, fp, err
|
||||
return p, fp, nil, err
|
||||
}
|
||||
pins := pinsOfEntry(a.Entry)
|
||||
mu.Lock()
|
||||
byFp[fp] = p
|
||||
byFp[fp] = built{program: p, pins: pins}
|
||||
mu.Unlock()
|
||||
return p, fp, nil
|
||||
return p, fp, pins, nil
|
||||
}, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -53,7 +53,7 @@ func newRunStand(t *testing.T, revisionEngine, trunkEngine, seatDigest string) *
|
||||
s := &runStand{asks: map[string]buildstep.StepAsk{}, requests: map[string]composer.Request{}}
|
||||
engine := func(fp, commit string) meshcatalog.CatalogEntry {
|
||||
return meshcatalog.CatalogEntry{Module: "engine", Fp: fp, Repository: "novox/mesh-host", Commit: commit, Path: ".", Serves: []string{ne.Seat + ".composition"},
|
||||
Manifest: map[string]any{"module": "engine"}}
|
||||
Manifest: map[string]any{"module": "engine"}, Packages: map[string]string{"git.novox.be/novox/mesh-sdk/go": "v0.1.25-" + fp, "git.novox.be/novox/x/go": ""}}
|
||||
}
|
||||
revisionCommit := commitE1
|
||||
if revisionEngine != "src2:e1" {
|
||||
@@ -186,6 +186,13 @@ func TestTheCompositionStepRunsOneAskPerMachine(t *testing.T) {
|
||||
if s.asks["g14"].Run.Program.Commit != commitE2 {
|
||||
t.Errorf("g14's composer is built at %s, want the build its node-engine runs", s.asks["g14"].Run.Program.Commit)
|
||||
}
|
||||
// Each composer is built against the pins of the node-engine entry it is built from, a pin with no version left out
|
||||
// (H-A-203, H-A-192).
|
||||
for node, fp := range map[string]string{"ace": "src2:e1", "g14": "src2:e2"} {
|
||||
if got := s.asks[node].Packages; len(got) != 1 || got["git.novox.be/novox/mesh-sdk/go"] != "v0.1.25-"+fp {
|
||||
t.Errorf("%s's composer is built against the pins %v; want its entry's", node, got)
|
||||
}
|
||||
}
|
||||
if req.Depth != composer.DepthCompose || req.BlameTries != 20 || len(req.Machines) != 1 || req.Machines[0] != "ace" || len(req.Cases) != 1 || req.Cases[0].Name != changeCase ||
|
||||
string(req.Input) != `{"node":"ace"}` || len(req.Seats) != 1 || string(req.Seats[0]) != `{"seat":"s1"}` {
|
||||
t.Errorf("the request of ace %+v", req)
|
||||
|
||||
@@ -15,7 +15,21 @@ import (
|
||||
|
||||
// fillOf is the store's fill now: its bytes and cap, when the bucket says them and has a cap.
|
||||
func (g *graphKeeper) fillOf(ctx context.Context) (used, capacity int64, ok bool) {
|
||||
f, is := g.objects.(assigngraph.Filler)
|
||||
return fillIn(ctx, g.objects)
|
||||
}
|
||||
|
||||
// passObjects is the store as the pass reads its fill: a handle of its own where one is given, so the pass's read never shares one
|
||||
// with a keeping that runs beside it (design 47 H-A-261).
|
||||
func (g *graphKeeper) passObjects() assigngraph.Objects {
|
||||
if g.reads != nil {
|
||||
return g.reads
|
||||
}
|
||||
return g.objects
|
||||
}
|
||||
|
||||
// fillIn is the fill of the bucket o, when it says one and has a cap.
|
||||
func fillIn(ctx context.Context, o assigngraph.Objects) (used, capacity int64, ok bool) {
|
||||
f, is := o.(assigngraph.Filler)
|
||||
if !is {
|
||||
return 0, 0, false
|
||||
}
|
||||
@@ -35,13 +49,13 @@ func (p *passer) sayFill(event, about string, used, capacity int64, refused stri
|
||||
// its cap), so a fill reached while the process was down is said too.
|
||||
func (p *passer) fillAtStart(ctx context.Context) {
|
||||
g := p.graph
|
||||
used, capacity, ok := g.fillOf(ctx)
|
||||
used, capacity, ok := fillIn(ctx, g.passObjects())
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
switch {
|
||||
case used >= capacity:
|
||||
g.full = true
|
||||
g.setFull(true)
|
||||
p.sayFill(meshwants.EventFull, "start", used, capacity, "the next revision")
|
||||
case used*5 >= capacity*4:
|
||||
p.sayFill(meshwants.EventFilling, "start", used, capacity, "")
|
||||
@@ -52,20 +66,20 @@ func (p *passer) fillAtStart(ctx context.Context) {
|
||||
// 80 % again. It answers true while the store is full.
|
||||
func (p *passer) fullBefore(ctx context.Context, c assigngraph.Computed) bool {
|
||||
g := p.graph
|
||||
used, capacity, ok := g.fillOf(ctx)
|
||||
used, capacity, ok := fillIn(ctx, g.passObjects())
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
if g.full && used*5 < capacity*4 {
|
||||
g.full = false
|
||||
if g.isFull() && used*5 < capacity*4 {
|
||||
g.setFull(false)
|
||||
p.sayFill(meshwants.EventRoom, fmt.Sprint(used), used, capacity, "")
|
||||
}
|
||||
if used+assigngraph.NewBytes(ctx, g.objects, c) <= capacity {
|
||||
g.full = false
|
||||
if used+assigngraph.NewBytes(ctx, g.passObjects(), c) <= capacity {
|
||||
g.setFull(false)
|
||||
return false
|
||||
}
|
||||
if !g.full {
|
||||
g.full = true
|
||||
if !g.isFull() {
|
||||
g.setFull(true)
|
||||
p.sayFill(meshwants.EventFull, fmt.Sprint(used), used, capacity, "the next revision")
|
||||
}
|
||||
p.logf("the assignment graph revision is not kept: %s is full (%d of %d bytes), the last revision stays and no want is written until there is room", assigngraph.RevisionsObjects, used, capacity)
|
||||
|
||||
@@ -61,14 +61,14 @@ func TestTheAssignmentGraphStoreSaysFillingFullAndRoom(t *testing.T) {
|
||||
if len(events) != 0 {
|
||||
t.Fatalf("events %v", events)
|
||||
}
|
||||
o.base = 700
|
||||
o.base, o.cap = 8500, 10000 // past 80 %, with room for the next revision, which the pass judges itself (H-A-261)
|
||||
p.graph.started = false
|
||||
p.keepGraph(ctx)
|
||||
p.keepGraph(ctx) // a start again: its first pass judges the fill
|
||||
if len(events) != 1 || events[0] != meshwants.EventFilling {
|
||||
t.Fatalf("at the start past 80 %%: %v", events)
|
||||
}
|
||||
c := assigngraph.Computed{Revision: meshwants.AssignmentRevision{Entries: []meshwants.NodeEntryRef{}}}
|
||||
o.base = 990
|
||||
o.base, o.cap = 990, 1000
|
||||
if !p.fullBefore(ctx, c) || !p.graph.full || events[len(events)-1] != meshwants.EventFull {
|
||||
t.Fatalf("a revision that does not fit: events %v", events)
|
||||
}
|
||||
@@ -76,8 +76,15 @@ func TestTheAssignmentGraphStoreSaysFillingFullAndRoom(t *testing.T) {
|
||||
if n := len(events); n != 2 {
|
||||
t.Errorf("full is said once: %v", events)
|
||||
}
|
||||
// H-A-261: the verdict is the pass's own: with the store full the pass writes no want, though a revision is kept.
|
||||
if held := p.keepGraph(ctx); held != heldFull {
|
||||
t.Errorf("a pass with the store full answers %q, want %q", held, heldFull)
|
||||
}
|
||||
o.base = 100
|
||||
if p.fullBefore(ctx, c) || p.graph.full || events[len(events)-1] != meshwants.EventRoom {
|
||||
t.Errorf("room: events %v", events)
|
||||
}
|
||||
if held := p.keepGraph(ctx); held != "" {
|
||||
t.Errorf("a pass with room and a revision kept answers %q, want the wants written", held)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
mcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
@@ -24,11 +25,18 @@ import (
|
||||
// that follows are written from the revision it holds.
|
||||
type graphKeeper struct {
|
||||
objects assigngraph.Objects
|
||||
// reads is the same store through a handle of the pass's own, for the pass's read of its fill (graphfill.go); nil reads objects.
|
||||
reads assigngraph.Objects
|
||||
// halves is every followed branch's catalog graph revision held now.
|
||||
halves func() map[string]mcc.Half
|
||||
// last is the revision last computed, so a pass whose inputs did not change keeps and says nothing.
|
||||
last meshwants.AssignmentRevision
|
||||
has bool
|
||||
// mu guards what the keeping writes and the pass reads: the revision kept, full, and the need handed over (design 47 H-A-259).
|
||||
mu sync.Mutex
|
||||
// handedRev is the revision last handed to the keeping, so a pass whose inputs did not change hands nothing; handed is false
|
||||
// until one is, and again when its keeping failed or found the store full, so the next pass hands it again.
|
||||
handedRev meshwants.AssignmentRevision
|
||||
handed bool
|
||||
// handedOnce says a revision was handed: the first carries the start's removal (H-A-261).
|
||||
handedOnce bool
|
||||
// kept is the revision the wants follow, and kept's source the trunk's source revision it carries; ok is false until one is kept.
|
||||
kept meshwants.AssignmentRevision
|
||||
source int64
|
||||
@@ -40,13 +48,55 @@ type graphKeeper struct {
|
||||
revisions *revisionRemover
|
||||
// computed is the revision computed by the last pass, with its entries: its edges.
|
||||
computed assigngraph.Computed
|
||||
// The keeping off the pass loop (H-A-259): need is what waits to be kept, keeping says one runs; background runs it (nil: at
|
||||
// once, in the caller), and wake makes a pass once a keeping changed what the pass reads.
|
||||
need *keepNeed
|
||||
keeping bool
|
||||
background func(func())
|
||||
wake func()
|
||||
}
|
||||
|
||||
// keepGraph computes the revision of this pass and keeps it when the pass's inputs differ from the last computed.
|
||||
func (p *passer) keepGraph(ctx context.Context) {
|
||||
// keepNeed is one need of the keeping: a computed revision to keep (nil: none), and whether the revisions not current are judged.
|
||||
type keepNeed struct {
|
||||
c *assigngraph.Computed
|
||||
judge bool
|
||||
}
|
||||
|
||||
// keptNow is the revision the wants follow, with its trunk source, and whether one is kept.
|
||||
func (g *graphKeeper) keptNow() (meshwants.AssignmentRevision, int64, bool) {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
return g.kept, g.source, g.ok
|
||||
}
|
||||
|
||||
// isFull says the store did not hold the last revision handed over, so no want is written (design 47 G §181).
|
||||
func (g *graphKeeper) isFull() bool {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
return g.full
|
||||
}
|
||||
|
||||
func (g *graphKeeper) setFull(full bool) {
|
||||
g.mu.Lock()
|
||||
g.full = full
|
||||
g.mu.Unlock()
|
||||
}
|
||||
|
||||
// heldNoRevision is the pass's words while no assignment graph revision is kept: it writes no want (design 47 H-A-261).
|
||||
const heldNoRevision = "held: no assignment graph revision is kept yet, so no want is written until the first keeping ends (H-A-261)"
|
||||
|
||||
// heldFull is the pass's words while the next revision does not fit its store (design 47 G §181).
|
||||
const heldFull = "the assignment graph store is full: no want is written (design 47 G §181)"
|
||||
|
||||
// keepGraph computes the revision of this pass, judges the store's fill itself, and hands the revision to the keeping when the
|
||||
// pass's inputs differ from the last handed. The keeping and the removing of revisions run off the pass loop, one at a time, and the
|
||||
// pass never waits on them: it judges and writes the wants from the graph it holds (design 47 H-A-259). It answers why the pass
|
||||
// writes no want, or "" when it writes them: none while the next revision does not fit the store (G §181), and none while no
|
||||
// revision is kept (H-A-261), whose keeping wakes a pass when it ends.
|
||||
func (p *passer) keepGraph(ctx context.Context) string {
|
||||
g := p.graph
|
||||
if g == nil || g.objects == nil || g.halves == nil {
|
||||
return
|
||||
return ""
|
||||
}
|
||||
c := assigngraph.Compute(p.graphInputs())
|
||||
g.computed = c
|
||||
@@ -56,40 +106,130 @@ func (p *passer) keepGraph(ctx context.Context) {
|
||||
p.logf("the assignment graph leaves out %s", u)
|
||||
}
|
||||
}
|
||||
if !g.started {
|
||||
if !g.started { // the first pass judges the fill once (graphfill.go)
|
||||
g.started = true
|
||||
p.fillAtStart(ctx)
|
||||
}
|
||||
if g.has && g.ok && assigngraph.Same(g.last, c.Revision) {
|
||||
if p.fullBefore(ctx, c) {
|
||||
g.mu.Lock()
|
||||
g.handed = false // handed once it fits
|
||||
g.mu.Unlock()
|
||||
return heldFull
|
||||
}
|
||||
g.mu.Lock()
|
||||
same := g.handed && assigngraph.Same(g.handedRev, c.Revision)
|
||||
first := !g.handedOnce
|
||||
if !same {
|
||||
g.handedRev, g.handed, g.handedOnce = c.Revision, true, true
|
||||
}
|
||||
g.mu.Unlock()
|
||||
if !same {
|
||||
// The start's removal of the revisions not current folds into the first keeping: it keeps the first pass's revision, then
|
||||
// removes (H-A-261).
|
||||
p.needKeeping(ctx, keepNeed{c: &c, judge: first})
|
||||
}
|
||||
if _, _, ok := g.keptNow(); !ok {
|
||||
return heldNoRevision
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// needKeeping hands a need to the keeping: one keeping runs at a time, and a need that comes while one runs is folded into the one
|
||||
// that waits (the newest revision, and a judging asked by either), so it runs once more (design 47 H-A-259).
|
||||
func (p *passer) needKeeping(ctx context.Context, n keepNeed) {
|
||||
g := p.graph
|
||||
g.mu.Lock()
|
||||
if w := g.need; w != nil {
|
||||
n.judge = n.judge || w.judge
|
||||
if n.c == nil {
|
||||
n.c = w.c
|
||||
}
|
||||
}
|
||||
g.need = &n
|
||||
if g.keeping {
|
||||
g.mu.Unlock()
|
||||
return
|
||||
}
|
||||
g.keeping = true
|
||||
g.mu.Unlock()
|
||||
run := func() {
|
||||
for {
|
||||
g.mu.Lock()
|
||||
next := g.need
|
||||
g.need = nil
|
||||
if next == nil {
|
||||
g.keeping = false
|
||||
g.mu.Unlock()
|
||||
return
|
||||
}
|
||||
g.mu.Unlock()
|
||||
p.keepOne(ctx, *next)
|
||||
}
|
||||
}
|
||||
if g.background != nil {
|
||||
g.background(run)
|
||||
return
|
||||
}
|
||||
run()
|
||||
}
|
||||
|
||||
// keepOne keeps one computed revision, says assignment-revision for a revision kept now and removes the
|
||||
// revisions that ceased to be current; it says its duration in the journal, and wakes a pass when it changed what the pass reads.
|
||||
func (p *passer) keepOne(ctx context.Context, n keepNeed) {
|
||||
g := p.graph
|
||||
start := time.Now()
|
||||
changed := false
|
||||
defer func() {
|
||||
p.logf("the keeping of the assignment graph took %s", time.Since(start).Round(time.Millisecond))
|
||||
if changed && g.background != nil && g.wake != nil {
|
||||
g.wake()
|
||||
}
|
||||
}()
|
||||
if n.c != nil {
|
||||
_, _, had := g.keptNow()
|
||||
fresh := p.keepRevision(ctx, *n.c)
|
||||
_, _, has := g.keptNow()
|
||||
n.judge = n.judge || fresh
|
||||
changed = fresh || had != has // the first revision kept: the pass writes the wants it held (H-A-261)
|
||||
}
|
||||
if n.judge && g.revisions != nil {
|
||||
g.revisions.judge() // the revision before ceased to be current
|
||||
}
|
||||
}
|
||||
|
||||
// keepRevision keeps c when the store holds it, and says whether a revision was kept now.
|
||||
func (p *passer) keepRevision(ctx context.Context, c assigngraph.Computed) bool {
|
||||
g := p.graph
|
||||
again := func() { // the next pass hands this revision again
|
||||
g.mu.Lock()
|
||||
g.handed = false
|
||||
g.mu.Unlock()
|
||||
}
|
||||
used, _, _ := g.fillOf(ctx)
|
||||
if p.fullBefore(ctx, c) {
|
||||
return
|
||||
}
|
||||
rev, fresh, err := assigngraph.Keep(ctx, g.objects, c, meshwants.EventAssignmentRevision, p.clock())
|
||||
if err != nil {
|
||||
p.logf("the assignment graph revision is not kept, so no want is said as wanted until it is: %v", err)
|
||||
return
|
||||
again()
|
||||
return false
|
||||
}
|
||||
g.last, g.has, g.kept, g.ok = c.Revision, true, rev, true
|
||||
g.source = 0
|
||||
var source int64
|
||||
for _, cat := range rev.Catalogs {
|
||||
if cat.Branch == p.w.trunkName() {
|
||||
g.source = cat.Source
|
||||
source = cat.Source
|
||||
}
|
||||
}
|
||||
g.mu.Lock()
|
||||
g.kept, g.ok, g.source = rev, true, source
|
||||
g.mu.Unlock()
|
||||
if !fresh {
|
||||
return
|
||||
return false
|
||||
}
|
||||
p.fillingAfter(ctx, used)
|
||||
if g.revisions != nil {
|
||||
g.revisions.judge() // the revision before ceased to be current
|
||||
}
|
||||
id := fmt.Sprintf("%s-%s-%d", meshwants.EventAssignmentRevision, rev.Kind, rev.Revision)
|
||||
if err := p.wantsSay(meshwants.EventAssignmentRevision, assigngraph.Event(rev), id); err != nil {
|
||||
p.logf("the assignment revision %d is kept but not said: %v", rev.Revision, err)
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// graphInputs are what the pass's assignment graph is computed from: every followed branch's catalog half, the placements, each
|
||||
@@ -147,14 +287,20 @@ func (p *passer) sayWanted(node, module, fp, pipeline string, removed bool) {
|
||||
return
|
||||
}
|
||||
g := p.graph
|
||||
if g == nil || !g.ok {
|
||||
var kept meshwants.AssignmentRevision
|
||||
var source int64
|
||||
ok := false
|
||||
if g != nil {
|
||||
kept, source, ok = g.keptNow()
|
||||
}
|
||||
if !ok {
|
||||
p.logf("the want of %s on %s is written, but no assignment graph revision is kept, so wanted is not said", module, node)
|
||||
return
|
||||
}
|
||||
at := p.clock().UTC()
|
||||
body := meshwants.WantedEvent{V: keys.Version, Node: node, Module: module, Fp: fp, Pipeline: pipeline, Removed: removed, Revision: g.kept.Revision, Source: g.source, At: &at}
|
||||
body := meshwants.WantedEvent{V: keys.Version, Node: node, Module: module, Fp: fp, Pipeline: pipeline, Removed: removed, Revision: kept.Revision, Source: source, At: &at}
|
||||
event := strings.TrimPrefix(meshwants.SubjectWanted(keys.EscapeName(node), keys.EscapeName(module)), "mesh.seat."+meshwants.Seat+".event.")
|
||||
id := fmt.Sprintf("wanted.%s.%s.%s.%t.%d.%s", keys.EscapeName(node), keys.EscapeName(module), fp, removed, g.kept.Revision, at.Format(time.RFC3339Nano))
|
||||
id := fmt.Sprintf("wanted.%s.%s.%s.%t.%d.%s", keys.EscapeName(node), keys.EscapeName(module), fp, removed, kept.Revision, at.Format(time.RFC3339Nano))
|
||||
if err := p.wantsSay(event, body, id); err != nil {
|
||||
p.logf("wanted of %s on %s is not said: %v", module, node, err)
|
||||
}
|
||||
@@ -185,7 +331,7 @@ func (p *passer) namedBuild(module, trunkFP string, unreleased ...bool) func(fp
|
||||
// the mesh-sdk merge moved its fingerprint).
|
||||
return p.w.released.withParts(module, p.w.ends.madeOf(module, fp))
|
||||
}
|
||||
if p.graph == nil || !p.graph.ok {
|
||||
if _, _, kept := p.graphKept(); !kept {
|
||||
p.unheardRelease(module, fp, "no assignment graph revision is kept yet, so no release is looked up")
|
||||
return assign.KeptBuild{}, false
|
||||
}
|
||||
@@ -212,3 +358,11 @@ func (p *passer) unheardRelease(module, fp, why string) {
|
||||
p.logf("the release of %s at fingerprint %s is not heard: %s; its wants and candidates carry no version and no parts", module, fp, why)
|
||||
}
|
||||
}
|
||||
|
||||
// graphKept is the revision the wants follow, with its trunk source, and whether one is kept; with no graph kept, none is.
|
||||
func (p *passer) graphKept() (meshwants.AssignmentRevision, int64, bool) {
|
||||
if p.graph == nil {
|
||||
return meshwants.AssignmentRevision{}, 0, false
|
||||
}
|
||||
return p.graph.keptNow()
|
||||
}
|
||||
|
||||
@@ -3,8 +3,10 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -12,7 +14,9 @@ import (
|
||||
mcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
mpr "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
|
||||
"mesh-delivery-assigner/internal/assigngraph"
|
||||
"mesh-delivery-assigner/internal/buckets"
|
||||
@@ -132,3 +136,141 @@ func TestTheWantsAreSaidWantedFromTheKeptAssignmentGraphRevision(t *testing.T) {
|
||||
t.Errorf("dropping the run's candidate dropped trunk: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// blockedObjects is an object bucket whose listing waits while a gate stands: a keeping that does not end until it is opened.
|
||||
type blockedObjects struct {
|
||||
memObjects
|
||||
mu sync.Mutex
|
||||
gate chan struct{}
|
||||
}
|
||||
|
||||
func (b *blockedObjects) Names(ctx context.Context) ([]string, error) {
|
||||
b.mu.Lock()
|
||||
gate := b.gate
|
||||
b.mu.Unlock()
|
||||
if gate != nil {
|
||||
<-gate
|
||||
}
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
return b.memObjects.Names(ctx)
|
||||
}
|
||||
|
||||
func (b *blockedObjects) Put(ctx context.Context, name string, data []byte) error {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
return b.memObjects.Put(ctx, name, data)
|
||||
}
|
||||
|
||||
func (b *blockedObjects) Get(ctx context.Context, name string) ([]byte, error) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
return b.memObjects.Get(ctx, name)
|
||||
}
|
||||
|
||||
// block stands a gate, opened by the function it answers.
|
||||
func (b *blockedObjects) block() func() {
|
||||
gate := make(chan struct{})
|
||||
b.mu.Lock()
|
||||
b.gate = gate
|
||||
b.mu.Unlock()
|
||||
return func() {
|
||||
b.mu.Lock()
|
||||
b.gate = nil
|
||||
b.mu.Unlock()
|
||||
close(gate)
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 H-A-259, H-A-261: the keeping of the assignment graph runs off the pass loop. At the start, while the first keeping is
|
||||
// blocked, a pass returns and writes no want; the keeping's end wakes a pass, which writes the passed pipeline's want. Later, while a
|
||||
// keeping of new inputs is blocked, a pass writes the wants from the revision it holds, and a pass of the same inputs hands nothing
|
||||
// more. Each pass and each keeping says its duration.
|
||||
func TestAPassWritesWantsWhileAKeepingIsBlocked(t *testing.T) {
|
||||
fp := mcc.SourcePrefix2 + strings.Repeat("a", 64)
|
||||
wantsKV := &memKV{values: map[string][]byte{}}
|
||||
p := newPasser(kvSet{buckets.Wants(): wantsKV, buckets.Asks(): &memKV{values: map[string][]byte{}}}, time.Now())
|
||||
var mu sync.Mutex
|
||||
var journal strings.Builder
|
||||
p.logf = func(f string, a ...any) { mu.Lock(); fmt.Fprintf(&journal, f+"\n", a...); mu.Unlock() }
|
||||
said := func(words string) bool {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
return strings.Contains(journal.String(), words)
|
||||
}
|
||||
mdigest := strings.Repeat("e", 64)
|
||||
mod, _ := mcc.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fp, Manifest: mdigest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: mcc.RootPath})
|
||||
putModuleRaw(p.w, trunkBranch, "web", mod)
|
||||
p.w.manifests[mdigest] = parsed{module: "web", manifest: catalogue.Manifest{Module: "web"}}
|
||||
placeOn(t, p.w, "ace", "web")
|
||||
p.ends = newPipelineEnds(newMemEnds(), func() {}, t.Logf)
|
||||
p.ends.trunk = p.w.trunkName
|
||||
_, _ = p.ends.take(ended(trunkBranch, 5, mpr.PipelineEndedOutcomePassed, mover("web", fp, "0.1.2", 1, mpr.StepResultOutcomePassed)))
|
||||
objects := &blockedObjects{memObjects: memObjects{}}
|
||||
var kept sync.WaitGroup
|
||||
woke := make(chan struct{}, 8)
|
||||
half := mcc.Half{Revision: meshcatalog.CatalogRevision{Branch: trunkBranch, Kind: "trunk", Revision: 4, Source: 9,
|
||||
Entries: []meshcatalog.EntryRef{{Module: "web", Digest: "sha256:" + strings.Repeat("a", 64)}}},
|
||||
Entries: map[string]meshcatalog.CatalogEntry{"web": {Module: "web", Fp: fp}}}
|
||||
p.graph = &graphKeeper{objects: objects, halves: func() map[string]mcc.Half { return map[string]mcc.Half{trunkBranch: half} },
|
||||
background: func(f func()) { kept.Go(f) }, wake: func() { woke <- struct{}{} }}
|
||||
pass := func() {
|
||||
t.Helper()
|
||||
done := make(chan struct{})
|
||||
go func() { p.pass(context.Background()); close(done) }()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("the pass waits on the keeping of the assignment graph")
|
||||
}
|
||||
}
|
||||
wantOf := func(node string) (mwc.Want, error) {
|
||||
key, _ := mwc.WantKey(node, "web")
|
||||
return mwc.DecodeWant(wantsKV.values[key])
|
||||
}
|
||||
open := objects.block()
|
||||
t.Cleanup(func() { // the keeping ends before the test does, and says its duration
|
||||
objects.mu.Lock()
|
||||
standing := objects.gate != nil
|
||||
objects.mu.Unlock()
|
||||
if standing {
|
||||
open()
|
||||
}
|
||||
kept.Wait()
|
||||
if !said("the keeping of the assignment graph took ") {
|
||||
t.Error("the keeping does not say its duration")
|
||||
}
|
||||
})
|
||||
|
||||
pass() // the start: the first keeping is blocked
|
||||
if _, err := wantOf("ace"); err == nil {
|
||||
t.Error("a pass at the start wrote a want before an assignment graph revision is kept")
|
||||
}
|
||||
open()
|
||||
select {
|
||||
case <-woke:
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("the first keeping's end wakes no pass")
|
||||
}
|
||||
pass()
|
||||
if w, err := wantOf("ace"); err != nil || w.Fp != fp || w.Version != "0.1.2" {
|
||||
t.Errorf("the want of web on ace = %+v (%v), want the passed version once a revision is kept", w, err)
|
||||
}
|
||||
|
||||
open = objects.block() // a keeping of new inputs is blocked
|
||||
placeOn(t, p.w, "g14", "web")
|
||||
pass()
|
||||
pass() // the same inputs again
|
||||
if w, err := wantOf("g14"); err != nil || w.Fp != fp || w.Version != "0.1.2" {
|
||||
t.Errorf("the want of web on g14 = %+v (%v), want the passed version while a keeping is blocked", w, err)
|
||||
}
|
||||
p.graph.mu.Lock()
|
||||
keeping, waiting := p.graph.keeping, p.graph.need
|
||||
p.graph.mu.Unlock()
|
||||
if !keeping || waiting != nil {
|
||||
t.Errorf("keeping %v, need waiting %+v: the pass of the same inputs handed its revision again", keeping, waiting)
|
||||
}
|
||||
if !said("the pass took ") {
|
||||
t.Error("the pass does not say its duration")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
"io"
|
||||
"maps"
|
||||
"mesh-delivery-assigner/internal/assigngraph"
|
||||
"mesh-delivery-assigner/internal/assignment"
|
||||
"mesh-delivery-assigner/internal/buckets"
|
||||
"mesh-delivery-assigner/internal/puts"
|
||||
@@ -190,6 +191,11 @@ func startBed(t *testing.T, leaveOut string) *bed {
|
||||
if _, err := js.CreateObjectStore(ctx, jetstream.ObjectStoreConfig{Bucket: catalogue.BusName("mesh-delivery-assigner", catalogue.StateObjectsName(buckets.InputsState))}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The assignment graph revisions' object store, which the grants part makes from the manifest's objects: no want is written
|
||||
// before a revision is kept in it (design 47 H-A-261).
|
||||
if _, err := js.CreateObjectStore(ctx, jetstream.ObjectStoreConfig{Bucket: catalogue.ObjectsBucket("mesh-delivery-assigner", assigngraph.RevisionsObjects)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The builder's module log, as the runtime raises it (ADR 0297): the running commits are read from it (ADR 0571 clause 1).
|
||||
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: buscompose.LogStreamName(builds.BuilderModule, builds.BuilderLog),
|
||||
Subjects: []string{buscompose.LogSubject(builds.BuilderModule, builds.BuilderLog) + ".>"}, AllowDirect: true}); err != nil {
|
||||
|
||||
@@ -613,7 +613,10 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
|
||||
p.publishWants = wantsPublisher(ctx, js, thisNode)
|
||||
if iv != nil && iv.h != nil {
|
||||
objects := newBusObjects(js, process.Account{Node: thisNode, Module: assignment.Module()}, assigngraph.RevisionsObjects)
|
||||
p.graph = &graphKeeper{objects: objects, halves: iv.h.views}
|
||||
// The keeping and removing of revisions runs off the pass loop, and a keeping that changed what the pass reads wakes
|
||||
// one (design 47 H-A-259).
|
||||
p.graph = &graphKeeper{objects: objects, halves: iv.h.views, background: func(f func()) { go f() }, wake: wake,
|
||||
reads: newBusObjects(js, process.Account{Node: thisNode, Module: assignment.Module()}, assigngraph.RevisionsObjects)}
|
||||
keep := func(ctx context.Context, ev meshcatalog.CatalogRevisionEvent) error {
|
||||
return p.keepCandidate(ctx, iv.call, ev)
|
||||
}
|
||||
@@ -628,7 +631,7 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
|
||||
// The revisions that are not current are removed when one ceases to be current and once now (revisions.go,
|
||||
// design 47 H §764, H-A-86).
|
||||
p.graph.revisions = &revisionRemover{ctx: ctx, objects: objects, inUse: inUseOf(w), logf: logger.Printf}
|
||||
p.graph.revisions.judge()
|
||||
// The start's removal of the revisions not current folds into the first pass's keeping (H-A-261).
|
||||
}
|
||||
}
|
||||
// The node-engine seat's manifest-reads-changed, for one node (design 47 H-A-104): its words are written into the state
|
||||
|
||||
@@ -88,6 +88,43 @@ func (b *busObjects) Names(ctx context.Context) ([]string, error) {
|
||||
return names, nil
|
||||
}
|
||||
|
||||
// PutDescribed stores data under name with the entry's module and fingerprint in its metadata, which the listing says (assigngraph.Describer,
|
||||
// design 47 H-A-259).
|
||||
func (b *busObjects) PutDescribed(ctx context.Context, name string, data []byte, module, fp string) error {
|
||||
o, err := b.open(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = o.Put(ctx, jetstream.ObjectMeta{Name: name, Metadata: map[string]string{entryModuleMeta: module, entryFpMeta: fp}}, bytes.NewReader(data))
|
||||
return err
|
||||
}
|
||||
|
||||
// Described is every object with the module and fingerprint its metadata says, in one listing.
|
||||
func (b *busObjects) Described(ctx context.Context) ([]assigngraph.Described, error) {
|
||||
o, err := b.open(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
infos, err := o.List(ctx)
|
||||
if errors.Is(err, jetstream.ErrNoObjectsFound) {
|
||||
return []assigngraph.Described{}, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]assigngraph.Described, 0, len(infos))
|
||||
for _, i := range infos {
|
||||
out = append(out, assigngraph.Described{Name: i.Name, Module: i.Metadata[entryModuleMeta], Fp: i.Metadata[entryFpMeta]})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// The metadata keys of an entry object's module and fingerprint.
|
||||
const (
|
||||
entryModuleMeta = "module"
|
||||
entryFpMeta = "fp"
|
||||
)
|
||||
|
||||
// Delete removes the object of that name; one not held is no error (design 47 G §579: a revision nothing pins is removed).
|
||||
func (b *busObjects) Delete(ctx context.Context, name string) error {
|
||||
o, err := b.open(ctx)
|
||||
|
||||
@@ -141,13 +141,14 @@ func (p *passer) clock() time.Time {
|
||||
}
|
||||
|
||||
func (p *passer) pass(ctx context.Context) {
|
||||
start := time.Now()
|
||||
defer func() { p.logf("the pass took %s", time.Since(start).Round(time.Millisecond)) }() // design 47 H-A-259
|
||||
p.assignmentsSeat(ctx)
|
||||
p.keepGraph(ctx)
|
||||
if p.graph == nil || !p.graph.full { // a full assignment graph store keeps the last revision and writes no new want (design 47 G §181)
|
||||
if held := p.keepGraph(ctx); held == "" {
|
||||
p.wants(ctx)
|
||||
p.sayPass(ctx, "judged")
|
||||
} else {
|
||||
p.sayPass(ctx, "the assignment graph store is full: no want is written (design 47 G §181)")
|
||||
p.sayPass(ctx, held) // a full store keeps the last revision (design 47 G §181); none kept yet writes no want (H-A-261)
|
||||
}
|
||||
p.puts(ctx)
|
||||
p.nodeWants(ctx)
|
||||
|
||||
@@ -2,6 +2,7 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strconv"
|
||||
@@ -53,7 +54,7 @@ func trunkRevision(name string) (int64, bool) {
|
||||
// remove removes each kept revision that is not current, and then the entries no kept revision or candidate lists and no want or
|
||||
// running build names (design 47 G §179).
|
||||
func (k *revisionRemover) remove() error {
|
||||
names, err := k.objects.Names(k.ctx)
|
||||
names, described, err := k.listing()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -107,7 +108,7 @@ func (k *revisionRemover) remove() error {
|
||||
if !ok || listed[digest] {
|
||||
continue
|
||||
}
|
||||
e, err := assigngraph.ReadEntry(k.ctx, k.objects, digest)
|
||||
e, err := k.entryOf(n, digest, described)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -138,3 +139,46 @@ func inUseOf(w *world) func(module, fp string) bool {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// listing is every object's name in one read and, where the bucket says them, each entry object's module and fingerprint by name
|
||||
// (design 47 H-A-259); nil described where the bucket says none.
|
||||
func (k *revisionRemover) listing() ([]string, map[string]assigngraph.Described, error) {
|
||||
d, ok := k.objects.(assigngraph.Describer)
|
||||
if !ok {
|
||||
names, err := k.objects.Names(k.ctx)
|
||||
return names, nil, err
|
||||
}
|
||||
listed, err := d.Described(k.ctx)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
names := make([]string, 0, len(listed))
|
||||
described := make(map[string]assigngraph.Described, len(listed))
|
||||
for _, l := range listed {
|
||||
names = append(names, l.Name)
|
||||
described[l.Name] = l
|
||||
}
|
||||
return names, described, nil
|
||||
}
|
||||
|
||||
// entryOf is the module and fingerprint of the entry object name: as the listing says them, with no read; an entry put before the
|
||||
// listing said them is read once and put again with them, so no later judging reads it (design 47 H-A-259).
|
||||
func (k *revisionRemover) entryOf(name, digest string, described map[string]assigngraph.Described) (meshwants.AssignmentEntry, error) {
|
||||
if l, ok := described[name]; ok && l.Module != "" && l.Fp != "" {
|
||||
return meshwants.AssignmentEntry{Module: l.Module, Fp: l.Fp}, nil
|
||||
}
|
||||
data, err := k.objects.Get(k.ctx, name)
|
||||
if err != nil {
|
||||
return meshwants.AssignmentEntry{}, fmt.Errorf("the entry %s: %w", digest, err)
|
||||
}
|
||||
var e meshwants.AssignmentEntry
|
||||
if err := json.Unmarshal(data, &e); err != nil {
|
||||
return e, fmt.Errorf("the entry %s: %w", digest, err)
|
||||
}
|
||||
if described != nil {
|
||||
if err := assigngraph.PutEntry(k.ctx, k.objects, name, data, e); err != nil {
|
||||
k.logf("the entry %s is not put again with its module and fingerprint, so the next judging reads it again: %v", digest, err)
|
||||
}
|
||||
}
|
||||
return e, nil
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
@@ -12,42 +13,98 @@ import (
|
||||
|
||||
func (m memObjects) Delete(_ context.Context, name string) error { delete(m, name); return nil }
|
||||
|
||||
// describedObjects is memObjects whose listing says each entry's module and fingerprint (assigngraph.Describer), counting the
|
||||
// entry objects read.
|
||||
type describedObjects struct {
|
||||
memObjects
|
||||
meta map[string][2]string
|
||||
reads int
|
||||
}
|
||||
|
||||
func (d *describedObjects) Get(ctx context.Context, name string) ([]byte, error) {
|
||||
if strings.HasPrefix(name, "entry.") {
|
||||
d.reads++
|
||||
}
|
||||
return d.memObjects.Get(ctx, name)
|
||||
}
|
||||
|
||||
func (d *describedObjects) PutDescribed(_ context.Context, name string, data []byte, module, fp string) error {
|
||||
d.memObjects[name], d.meta[name] = data, [2]string{module, fp}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *describedObjects) Described(context.Context) ([]assigngraph.Described, error) {
|
||||
var out []assigngraph.Described
|
||||
for n := range d.memObjects {
|
||||
out = append(out, assigngraph.Described{Name: n, Module: d.meta[n][0], Fp: d.meta[n][1]})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Design 47 H §764, H-A-86: with the gate's pins gone, an assignment graph revision that is not current is removed, with the
|
||||
// entries no kept revision or candidate lists and no want or running build names; the current revision, the entries it lists,
|
||||
// a running build's entry and a run's kept candidate with its entries stay.
|
||||
// H-A-259: where the listing says each entry's module and fingerprint, the judging reads no entry object; an entry put before it
|
||||
// said them is read once and put again with them.
|
||||
func TestARevisionNotCurrentIsRemoved(t *testing.T) {
|
||||
objects := memObjects{}
|
||||
entry := func(module, fp string) string {
|
||||
e := meshwants.AssignmentEntry{Node: "n1", Module: module, Fp: fp}
|
||||
d := assigngraph.EntryDigest(e)
|
||||
raw, _ := json.Marshal(e)
|
||||
objects[assigngraph.EntryObject(d)] = raw
|
||||
return d
|
||||
}
|
||||
shared, old, running, cand := entry("a", "fp-a"), entry("b", "fp-b1"), entry("c", "fp-c1"), entry("d", "fp-d2")
|
||||
revision := func(n int64, digests ...string) {
|
||||
rev := meshwants.AssignmentRevision{Revision: n}
|
||||
for _, d := range digests {
|
||||
rev.Entries = append(rev.Entries, meshwants.NodeEntryRef{Node: "n1", Digest: d})
|
||||
for _, listed := range []string{"plain", "described", "described-later"} {
|
||||
objects := memObjects{}
|
||||
described := &describedObjects{memObjects: objects, meta: map[string][2]string{}}
|
||||
entry := func(module, fp string) string {
|
||||
e := meshwants.AssignmentEntry{Node: "n1", Module: module, Fp: fp}
|
||||
d := assigngraph.EntryDigest(e)
|
||||
raw, _ := json.Marshal(e)
|
||||
objects[assigngraph.EntryObject(d)] = raw
|
||||
if listed == "described" {
|
||||
described.meta[assigngraph.EntryObject(d)] = [2]string{module, fp}
|
||||
}
|
||||
return d
|
||||
}
|
||||
raw, _ := json.Marshal(rev)
|
||||
objects[assigngraph.TrunkObject(n)] = raw
|
||||
}
|
||||
candRaw, _ := json.Marshal(meshwants.AssignmentRevision{Revision: 1, Kind: "candidate", Group: "run-7", Heads: "h1",
|
||||
Entries: []meshwants.NodeEntryRef{{Node: "n1", Module: "d", Digest: cand}}})
|
||||
objects[assigngraph.CandidateObject("run-7", "h1")] = candRaw
|
||||
revision(1, shared, old, running)
|
||||
revision(2, shared, old)
|
||||
revision(3, shared)
|
||||
k := &revisionRemover{ctx: context.Background(), objects: objects, logf: t.Logf,
|
||||
inUse: func(module, fp string) bool { return module == "c" && fp == "fp-c1" }}
|
||||
k.judge()
|
||||
has := func(name string) bool { _, ok := objects[name]; return ok }
|
||||
for name, want := range map[string]bool{assigngraph.TrunkObject(1): false, assigngraph.TrunkObject(2): false, assigngraph.TrunkObject(3): true,
|
||||
assigngraph.EntryObject(old): false, assigngraph.EntryObject(running): true, assigngraph.EntryObject(shared): true,
|
||||
assigngraph.EntryObject(cand): true, assigngraph.CandidateObject("run-7", "h1"): true} {
|
||||
if has(name) != want {
|
||||
t.Errorf("%s kept = %v, want %v", name, has(name), want)
|
||||
shared, old, running, cand := entry("a", "fp-a"), entry("b", "fp-b1"), entry("c", "fp-c1"), entry("d", "fp-d2")
|
||||
revision := func(n int64, digests ...string) {
|
||||
rev := meshwants.AssignmentRevision{Revision: n}
|
||||
for _, d := range digests {
|
||||
rev.Entries = append(rev.Entries, meshwants.NodeEntryRef{Node: "n1", Digest: d})
|
||||
}
|
||||
raw, _ := json.Marshal(rev)
|
||||
objects[assigngraph.TrunkObject(n)] = raw
|
||||
}
|
||||
candRaw, _ := json.Marshal(meshwants.AssignmentRevision{Revision: 1, Kind: "candidate", Group: "run-7", Heads: "h1",
|
||||
Entries: []meshwants.NodeEntryRef{{Node: "n1", Module: "d", Digest: cand}}})
|
||||
objects[assigngraph.CandidateObject("run-7", "h1")] = candRaw
|
||||
revision(1, shared, old, running)
|
||||
revision(2, shared, old)
|
||||
revision(3, shared)
|
||||
var o revisionObjects = objects
|
||||
if listed != "plain" {
|
||||
o = described
|
||||
}
|
||||
k := &revisionRemover{ctx: context.Background(), objects: o, logf: t.Logf,
|
||||
inUse: func(module, fp string) bool { return module == "c" && fp == "fp-c1" }}
|
||||
k.judge()
|
||||
has := func(name string) bool { _, ok := objects[name]; return ok }
|
||||
for name, want := range map[string]bool{assigngraph.TrunkObject(1): false, assigngraph.TrunkObject(2): false, assigngraph.TrunkObject(3): true,
|
||||
assigngraph.EntryObject(old): false, assigngraph.EntryObject(running): true, assigngraph.EntryObject(shared): true,
|
||||
assigngraph.EntryObject(cand): true, assigngraph.CandidateObject("run-7", "h1"): true} {
|
||||
if has(name) != want {
|
||||
t.Errorf("%s: %s kept = %v, want %v", listed, name, has(name), want)
|
||||
}
|
||||
}
|
||||
switch listed {
|
||||
case "described":
|
||||
if described.reads != 0 {
|
||||
t.Errorf("described: the judging read %d entry objects, want none", described.reads)
|
||||
}
|
||||
case "described-later":
|
||||
if described.meta[assigngraph.EntryObject(running)] != [2]string{"c", "fp-c1"} {
|
||||
t.Errorf("described-later: the entry in use is not put again with its module and fingerprint: %v", described.meta)
|
||||
}
|
||||
reads := described.reads
|
||||
revision(4, shared)
|
||||
k.judge()
|
||||
if described.reads != reads {
|
||||
t.Errorf("described-later: a second judging read %d entry objects again", described.reads-reads)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -345,7 +345,8 @@ func (r *compositionRun) startValidate(ctx context.Context, key string, run mesh
|
||||
return
|
||||
}
|
||||
a := buildstep.StepAsk{ID: ask, Kind: buildstep.AskCheck, Language: buildstep.AskGo, Asker: runAsker, At: r.now().UTC(),
|
||||
Run: &buildstep.RunJob{Program: program, Heads: []buildstep.RunHead{}, Input: ref, Reader: "the manifest validator of " + step.Pipeline + " at " + program.Commit}}
|
||||
Run: &buildstep.RunJob{Program: program, Heads: []buildstep.RunHead{}, Input: ref, Reader: "the manifest validator of " + step.Pipeline + " at " + program.Commit},
|
||||
Packages: pinsOfEntry(entry)} // the validator is built against its entry's pins at the step's revision (H-A-203, H-A-192, H-A-205)
|
||||
if err := r.put(ctx, a); err != nil {
|
||||
couldNot("its run ask " + ask + " is not put: " + err.Error())
|
||||
}
|
||||
|
||||
@@ -25,7 +25,8 @@ import (
|
||||
// is judged against an empty case before, said in its end, and no revision below it is asked.
|
||||
func TestTheValidateStepRefusesAManifestProblemAndACycle(t *testing.T) {
|
||||
indexer := meshcatalog.CatalogEntry{Module: "indexer", Repository: "novox/mesh-delivery", Commit: commitE3, Path: "modules/mesh-delivery-indexer",
|
||||
Fp: "src2:ix", Serves: []string{"seat:" + meshcatalog.Seat + ".revision"}, Manifest: map[string]any{"module": "indexer"}, ManifestDigest: "d-ix"}
|
||||
Fp: "src2:ix", Serves: []string{"seat:" + meshcatalog.Seat + ".revision"}, Manifest: map[string]any{"module": "indexer"}, ManifestDigest: "d-ix",
|
||||
Packages: map[string]string{"git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient": "v0.1.4"}}
|
||||
entry := func(module, digest string, edges ...meshcatalog.Edge) meshcatalog.CatalogEntry {
|
||||
return meshcatalog.CatalogEntry{Module: module, Manifest: map[string]any{"module": module, "d": digest}, ManifestDigest: digest, Edges: edges}
|
||||
}
|
||||
@@ -124,6 +125,10 @@ func TestTheValidateStepRefusesAManifestProblemAndACycle(t *testing.T) {
|
||||
if p := asks[0].Run.Program; p.Commit != commitE3 || p.Dir != "modules/mesh-delivery-indexer" || p.Package != validatorPackage || len(asks[0].Run.Heads) != 0 {
|
||||
t.Errorf("the validator is built from %+v", asks[0].Run)
|
||||
}
|
||||
// The validator is built against the indexer entry's pins at the step's revision (H-A-203, H-A-192, H-A-205).
|
||||
if got := asks[0].Packages; len(got) != 1 || got["git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"] != "v0.1.4" {
|
||||
t.Errorf("the validator is built against the pins %v; want its entry's", got)
|
||||
}
|
||||
if c := requests[0].Cases; len(c) != 2 || c[0].Name != changeCase || c[1].Name != beforeCase || len(c[1].Manifests) != 3 || requests[0].Predefined != nil || requests[0].Packages != nil {
|
||||
t.Errorf("the validator's request is %+v", requests[0])
|
||||
}
|
||||
|
||||
@@ -30,6 +30,29 @@ type Objects interface {
|
||||
Names(ctx context.Context) ([]string, error)
|
||||
}
|
||||
|
||||
// Described is one object as the bucket's listing says it: its name, and for an entry object the module and fingerprint put beside
|
||||
// its bytes (empty for one put before them).
|
||||
type Described struct {
|
||||
Name, Module, Fp string
|
||||
}
|
||||
|
||||
// Describer is an object bucket whose listing says, in one read, each entry object's module and fingerprint, so the removal of the
|
||||
// revisions not current reads no object per entry (design 47 H-A-259).
|
||||
type Describer interface {
|
||||
// PutDescribed stores data whole under name, with module and fp beside it in the listing.
|
||||
PutDescribed(ctx context.Context, name string, data []byte, module, fp string) error
|
||||
// Described is every object, as the listing says it.
|
||||
Described(ctx context.Context) ([]Described, error)
|
||||
}
|
||||
|
||||
// PutEntry stores the entry object e under name, with its module and fingerprint beside it where the bucket says them.
|
||||
func PutEntry(ctx context.Context, o Objects, name string, data []byte, e meshwants.AssignmentEntry) error {
|
||||
if d, ok := o.(Describer); ok {
|
||||
return d.PutDescribed(ctx, name, data, e.Module, e.Fp)
|
||||
}
|
||||
return o.Put(ctx, name, data)
|
||||
}
|
||||
|
||||
// EntryObject is the object name of the entry of that digest (the seat's key form entry.<digest>).
|
||||
func EntryObject(digest string) string { return "entry." + digest }
|
||||
|
||||
@@ -144,7 +167,7 @@ func writeEntries(ctx context.Context, o Objects, c Computed, rev meshwants.Assi
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := o.Put(ctx, name, data); err != nil {
|
||||
if err := PutEntry(ctx, o, name, data, e); err != nil {
|
||||
return fmt.Errorf("the entry of %s/%s in %s: %w", ref.Node, ref.Module, RevisionsObjects, err)
|
||||
}
|
||||
held[name] = true
|
||||
|
||||
@@ -239,11 +239,15 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// and version-released heard; nothing is reserved before the fill.
|
||||
reservedKV := process.AwaitBucket(ctx, js, conn, own.reserved, "no version is reserved: builds are made unreserved", logger.Printf)
|
||||
reserved := newReservations(awaitedReserved{awaitedVersions{b: reservedKV, bucket: own.reserved}}, calls.Call, pubBoundsKV.KV, found, b.Reconsider, logger.Printf)
|
||||
// The index by digest and by version beside the reservations is written again once the state is bound (H-A-269 (1), H-A-271).
|
||||
reservedKV.Ready(func(jetstream.KeyValue) { reserved.reindex(ctx) })
|
||||
// Each artifact the holder says published at a reserved version is said as the seat mesh-builds' package-published (design 47 G §52).
|
||||
reserved.say = func(ctx context.Context, p meshbuilds.PackagePublished) error {
|
||||
return emitSeatEvent(context.WithoutCancel(ctx), js, cred, meshbuilds.SubjectPackagePublished, "package-published-"+p.Module+"-"+p.Version+"-"+p.Artifact, p)
|
||||
}
|
||||
b.Reserved = reserved
|
||||
// A take-over of a series' reservation by another fingerprint withdraws the older build (H-A-225).
|
||||
reserved.withdraw = b.Withdraw
|
||||
// The last versions-filled the stream keeps is read at the start (design 47 G §369, §372): a restart does not lose it.
|
||||
reserved.readFilled(ctx, lastKeptOf(js, cred, logger.Printf), sleeps)
|
||||
// A state bound later numbers the builds looked at without it.
|
||||
@@ -258,6 +262,26 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
Entry: entryVia(calls.Call), Send: sendVia(calls.Call),
|
||||
Unreleased: reserved.unreleasedWords, Logf: logger.Printf,
|
||||
Holders: holders} // a registry seat's holder from the seat-holders view, from the start (design 47 G A-416, A-416a)
|
||||
// A number a registry seat already holds is never reserved (H-A-275): read before reserving, through the seats' verb versions.
|
||||
reserved.r.HeldNumbers = func(ctx context.Context, module, fp string, refs []string) ([]string, error) {
|
||||
return republish.HeldNumbers(ctx, module, fp, refs, registryVersionsVia(calls.Call))
|
||||
}
|
||||
// A done build whose number a registry holds elsewhere is built again at the next free number (H-A-279), on the builder's context.
|
||||
reserved.rebuild = func(_ context.Context, module, fp, why string) error {
|
||||
res := b.BuildAgain(ctx, module, fp, why, "the builder", buildAgainBound)
|
||||
switch {
|
||||
case res.Refused != "":
|
||||
return errors.New(res.Refused)
|
||||
case res.Failed != "":
|
||||
return errors.New(res.Failed)
|
||||
case res.Pending != "":
|
||||
logger.Printf("%s at %s is built again: %s", module, fp, res.Pending)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
reserved.r.HeldElsewhere = func(ctx context.Context, module string, skipped []string) {
|
||||
reserved.heldElsewhere(ctx, module, "the registries hold "+strings.Join(skipped, ", ")+", so the reserve skipped them")
|
||||
}
|
||||
publishSet, err := publishSettingsOf(getenv)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -273,6 +297,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
return reserved.r.Of(ctx, module, version)
|
||||
},
|
||||
republish: republish.Republish, endedWith: reserved.unreleased,
|
||||
// A publish is asked again only while no published of it may be unheard and the registry's record lacks it (H-A-269 (2), H-A-272).
|
||||
unheard: unheardVia(js, process.EventsStream, cred.EventConsumerName()),
|
||||
judge: func(ctx context.Context, v versions.ReservedVersion) (versions.Judged, error) {
|
||||
return republish.Held(ctx, v, registryVersionsVia(calls.Call))
|
||||
},
|
||||
markHeld: reserved.markHeld, moveElsewhere: reserved.movedElsewhere,
|
||||
allDone: func(ctx context.Context) ([]versions.ReservedVersion, error) {
|
||||
all, err := reserved.r.All(ctx)
|
||||
return builtOf(all), err
|
||||
|
||||
@@ -47,10 +47,20 @@ type publishItems struct {
|
||||
republish func(ctx context.Context, module, version string) ([]string, error)
|
||||
endedWith func(ctx context.Context, v versions.ReservedVersion, why string) // raises the finding version-unreleased
|
||||
allDone func(ctx context.Context) ([]versions.ReservedVersion, error)
|
||||
set publishSettings
|
||||
now func() time.Time
|
||||
after func(time.Duration) <-chan time.Time
|
||||
logf func(string, ...any)
|
||||
// unheard is how many messages the builder's durable on the events stream holds pending and unacknowledged, read before each
|
||||
// try (design 47 H H-A-269 (2), H-A-272); nil reads none.
|
||||
unheard func(ctx context.Context) (int, error)
|
||||
// judge is what the registries' own record says of a version's missing artifacts (H-A-272, H-A-274, H-A-275); nil judges none.
|
||||
judge func(ctx context.Context, v versions.ReservedVersion) (versions.Judged, error)
|
||||
// markHeld marks the artifacts the registries hold with the built digest published from their record, as their published heard
|
||||
// would (H-A-274); moveElsewhere moves a version a registry holds at another digest to the next free number and asks its
|
||||
// publish there (H-A-275). Nil does neither.
|
||||
markHeld func(ctx context.Context, v versions.ReservedVersion, held []versions.HeldArtifact)
|
||||
moveElsewhere func(ctx context.Context, v versions.ReservedVersion, j versions.Judged) error
|
||||
set publishSettings
|
||||
now func() time.Time
|
||||
after func(time.Duration) <-chan time.Time
|
||||
logf func(string, ...any)
|
||||
}
|
||||
|
||||
func (p *publishItems) wait(ctx context.Context, d time.Duration) bool {
|
||||
@@ -138,6 +148,12 @@ func (p *publishItems) Try(ctx context.Context, kv jetstream.KeyValue, e jetstre
|
||||
if p.endedWith != nil {
|
||||
p.endedWith(ctx, v, "the publish ended after "+fmt.Sprint(r.Tries)+" tries")
|
||||
}
|
||||
} else if why, gone := p.notAgain(ctx, v); gone {
|
||||
return // moved to another number, whose own item is asked (H-A-275)
|
||||
} else if why != "" {
|
||||
// A published of the version may be unheard, or the registry's record ended it: no ask, and no try spent (H-A-272, H-A-274).
|
||||
p.logf("%s %s: the publish is not asked again: %s; tried again after %s", r.Module, r.Version, why, p.set.pauseAfter(r.Tries))
|
||||
r.NextTry = p.now().UTC().Add(p.set.pauseAfter(r.Tries))
|
||||
} else {
|
||||
r.Tries++
|
||||
if _, err := p.republish(ctx, r.Module, r.Version); err != nil {
|
||||
@@ -154,6 +170,85 @@ func (p *publishItems) Try(ctx context.Context, kv jetstream.KeyValue, e jetstre
|
||||
}
|
||||
}
|
||||
|
||||
// notAgain says why a version's publish is not asked again now, empty when it is (design 47 H H-A-269 (2), H-A-272, H-A-274,
|
||||
// H-A-275): the builder's durable holds UnheardBound messages or more not yet heard, so a published of it may be among them; or the
|
||||
// registries' own record holds every missing artifact at its version with its built digest, which marks them published from that
|
||||
// record; or a registry holds the version at another digest, which moves the build to the next free number (gone: this item is
|
||||
// ended with the old number). A count or a record that cannot be read is said, and does not hold the ask.
|
||||
func (p *publishItems) notAgain(ctx context.Context, v versions.ReservedVersion) (why string, gone bool) {
|
||||
if p.unheard != nil {
|
||||
n, err := p.unheard(ctx)
|
||||
switch {
|
||||
case err != nil:
|
||||
p.logf("%s %s: the builder's events consumer cannot be read, so the publish is asked again as it stands: %v", v.Module, v.Version, err)
|
||||
case n >= versions.UnheardBound:
|
||||
return fmt.Sprintf("the builder's events consumer holds %d messages not yet heard (the bound is %d), a published of it may be among them", n, versions.UnheardBound), false
|
||||
}
|
||||
}
|
||||
if p.judge == nil {
|
||||
return "", false
|
||||
}
|
||||
j, err := p.judge(ctx, v)
|
||||
switch {
|
||||
case err != nil:
|
||||
p.logf("%s %s: the registries' record cannot be read, so the publish is asked again: %v", v.Module, v.Version, err)
|
||||
case len(j.Elsewhere) > 0 && p.moveElsewhere != nil:
|
||||
if err := p.moveElsewhere(ctx, v, j); err != nil {
|
||||
p.logf("%s %s: a registry holds the number at another digest, and the build is not moved: %v", v.Module, v.Version, err)
|
||||
return "a registry holds " + v.Version + " at another digest", false
|
||||
}
|
||||
return "", true
|
||||
case j.All:
|
||||
if p.markHeld != nil {
|
||||
p.markHeld(ctx, v, j.Held)
|
||||
}
|
||||
return "the registries hold every missing artifact at " + v.Version + " with its built digest: marked published from their record", false
|
||||
default:
|
||||
p.logf("%s %s: asked again: %s", v.Module, v.Version, j.Lacks)
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
// unheardVia reads the builder's durable on the events stream: its messages pending and those given and not yet acknowledged
|
||||
// (design 47 H H-A-272), what it has not yet heard.
|
||||
func unheardVia(js jetstream.JetStream, stream, durable string) func(context.Context) (int, error) {
|
||||
return func(ctx context.Context) (int, error) {
|
||||
c, err := js.Consumer(ctx, stream, durable)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
info, err := c.Info(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return int(info.NumPending) + info.NumAckPending, nil
|
||||
}
|
||||
}
|
||||
|
||||
// registryVersionsVia asks a registry seat's verb versions for a name (H-A-272): each version it holds with its digest.
|
||||
func registryVersionsVia(call toolCaller) func(context.Context, string, string) ([]versions.HeldVersion, error) {
|
||||
return func(_ context.Context, seat, name string) ([]versions.HeldVersion, error) {
|
||||
raw, err := call("seat:"+seat+"."+versions.VerbVersions, map[string]any{"name": name})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var a struct { // the four registry seats answer versions alike: each version with its digest
|
||||
Versions []struct {
|
||||
Version string `json:"version"`
|
||||
Digest string `json:"digest"`
|
||||
} `json:"versions"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &a); err != nil {
|
||||
return nil, fmt.Errorf("%s's versions answered what is no list of versions: %w", seat, err)
|
||||
}
|
||||
out := make([]versions.HeldVersion, 0, len(a.Versions))
|
||||
for _, v := range a.Versions {
|
||||
out = append(out, versions.HeldVersion{Version: v.Version, Digest: v.Digest})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
}
|
||||
|
||||
// numberFile reads a setting's number from the file the manifest declares: no file or an empty one is def; a content that is no
|
||||
// number of at least one is an error naming the file.
|
||||
func numberFile(path string, def int) (int, error) {
|
||||
|
||||
@@ -274,3 +274,76 @@ func TestThePublishItemsAreWiredToTheReservationsAndTheReAskWatch(t *testing.T)
|
||||
t.Errorf("a dropped reservation kept its item")
|
||||
}
|
||||
}
|
||||
|
||||
// TestAPublishIsNotAskedAgainWhileItsPublishedMayBeUnheard: design 47 H H-A-269 (2), H-A-272, H-A-274, H-A-275 — a try asks the
|
||||
// publish again only while the builder's durable holds fewer than a hundred messages not yet heard and the registries' record lacks
|
||||
// an artifact; a record holding every missing artifact with the built digest marks them published from it; a record holding the
|
||||
// version at another digest moves the build to the next free number and ends this item; a skipped try spends no try and is written
|
||||
// again with its pause; a count that cannot be read holds no ask.
|
||||
func TestAPublishIsNotAskedAgainWhileItsPublishedMayBeUnheard(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
now := time.Date(2026, 11, 21, 15, 21, 0, 0, time.UTC)
|
||||
v := versions.ReservedVersion{Module: "node-tools", Version: "0.1.37", Artifacts: map[string]string{"node-tools:program": "sha256:7c9b10"}, BuiltAt: now}
|
||||
at := versions.HeldArtifact{Ref: "node-tools:program", Seat: versions.SeatArtifacts, Name: "node-tools/program", Version: "0.1.37", Digest: "sha256:7c9b10"}
|
||||
other := at
|
||||
other.Digest = "sha256:83bec1"
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
unheard int
|
||||
countErr error
|
||||
judged versions.Judged
|
||||
wantAsked, wantMark, wantMov bool
|
||||
}{
|
||||
{name: "thousands behind: no ask, the registry not read", unheard: 2944},
|
||||
{name: "at the bound: no ask", unheard: versions.UnheardBound},
|
||||
{name: "heard up, the registry holds it: marked from its record, no ask", unheard: versions.UnheardBound - 1, judged: versions.Judged{All: true, Held: []versions.HeldArtifact{at}}, wantMark: true},
|
||||
{name: "the registry holds the number at another digest: moved, no ask", unheard: 3, judged: versions.Judged{Elsewhere: []versions.HeldArtifact{other}}, wantMov: true},
|
||||
{name: "heard up, the registry lacks it: asked", unheard: 3, judged: versions.Judged{Lacks: "none"}, wantAsked: true},
|
||||
{name: "the count unreadable, the registry lacks it: asked", countErr: errors.New("no consumer"), judged: versions.Judged{Lacks: "none"}, wantAsked: true},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
kv := &itemKV{vals: map[string][]byte{}, revs: map[string]uint64{}}
|
||||
asked, judged, marked, moved := 0, 0, 0, 0
|
||||
p := &publishItems{reasks: func() jetstream.KeyValue { return kv }, set: defaultPublishSettings, now: func() time.Time { return now }, logf: t.Logf,
|
||||
reserved: func(context.Context, string, string) (versions.ReservedVersion, bool, error) { return v, true, nil },
|
||||
republish: func(context.Context, string, string) ([]string, error) { asked++; return nil, nil },
|
||||
unheard: func(context.Context) (int, error) { return c.unheard, c.countErr },
|
||||
judge: func(context.Context, versions.ReservedVersion) (versions.Judged, error) {
|
||||
judged++
|
||||
return c.judged, nil
|
||||
},
|
||||
markHeld: func(_ context.Context, _ versions.ReservedVersion, h []versions.HeldArtifact) {
|
||||
if len(h) == 1 && h[0].Digest == "sha256:7c9b10" {
|
||||
marked++
|
||||
}
|
||||
},
|
||||
moveElsewhere: func(context.Context, versions.ReservedVersion, versions.Judged) error {
|
||||
moved++
|
||||
delete(kv.vals, versions.PublishItemKey(v.Module, v.Version)) // the takeover ends the old item
|
||||
return nil
|
||||
}}
|
||||
if err := p.Ask(ctx, v.Module, v.Version); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key := versions.PublishItemKey(v.Module, v.Version)
|
||||
e, _ := kv.Get(ctx, key)
|
||||
p.Try(ctx, kv, e, decodeItem(t, e.Value()))
|
||||
if (asked == 1) != c.wantAsked || (marked == 1) != c.wantMark || (moved == 1) != c.wantMov {
|
||||
t.Fatalf("asked %d, marked %d, moved %d; want asked %v, marked %v, moved %v", asked, marked, moved, c.wantAsked, c.wantMark, c.wantMov)
|
||||
}
|
||||
if c.wantMov {
|
||||
if _, ok := kv.vals[key]; ok {
|
||||
t.Error("the moved version's old item is written again")
|
||||
}
|
||||
return
|
||||
}
|
||||
got := decodeItem(t, kv.vals[key])
|
||||
if got.Tries != map[bool]int{true: 1, false: 0}[c.wantAsked] || !got.NextTry.After(now) {
|
||||
t.Errorf("tries %d, next try %s: want a try spent only when asked, and the item written again with its pause", got.Tries, got.NextTry)
|
||||
}
|
||||
if c.unheard >= versions.UnheardBound && judged != 0 {
|
||||
t.Error("the registry is read while a published may be unheard")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -549,8 +549,14 @@ func (m memVersions) Update(_ context.Context, key string, v []byte, _ uint64) (
|
||||
m[key] = v
|
||||
return 2, nil
|
||||
}
|
||||
func (m memVersions) KeysOf(context.Context, string) ([]string, error) {
|
||||
return slices.Collect(maps.Keys(m)), nil
|
||||
func (m memVersions) KeysOf(_ context.Context, filter string) ([]string, error) {
|
||||
var out []string
|
||||
for k := range m {
|
||||
if versions.FilterMatches(filter, k) { // as the bus reads a filter
|
||||
out = append(out, k)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// memBuilds is the builder's builds in memory.
|
||||
|
||||
@@ -50,8 +50,12 @@ type reservations struct {
|
||||
bound time.Duration
|
||||
wake func(context.Context)
|
||||
logf func(string, ...any)
|
||||
// withdraw ends the running build of a module's fingerprint, superseded, with why (H-A-225); nil ends none.
|
||||
withdraw func(module, fingerprint, why string)
|
||||
// call asks a seat's verb: the git seat's tag (design 47 G §370).
|
||||
call toolCaller
|
||||
// rebuild builds a done key of module at fp again (the build-again path, A-490; H-A-279); nil builds none.
|
||||
rebuild func(ctx context.Context, module, fp, why string) error
|
||||
// say says package-published on the seat mesh-builds; nil says none.
|
||||
say func(context.Context, meshbuilds.PackagePublished) error
|
||||
}
|
||||
@@ -75,10 +79,21 @@ func newReservations(store versions.ReserveStore, call toolCaller, bounds func()
|
||||
}
|
||||
return out, nil
|
||||
}}
|
||||
rv.r.Took = rv.took
|
||||
rv.r.Took, rv.r.Logf = rv.took, logf
|
||||
return rv
|
||||
}
|
||||
|
||||
// reindex writes the reservations' index again from the reservations, once at the builder's start (H-A-269 (1), H-A-271), and
|
||||
// says so.
|
||||
func (rv *reservations) reindex(ctx context.Context) {
|
||||
n, err := rv.r.Reindex(ctx)
|
||||
if err != nil {
|
||||
rv.logf("the reservations' index is written again from %d reservations, not whole: %v", n, err)
|
||||
return
|
||||
}
|
||||
rv.logf("the reservations' index is written again from %d reservations", n)
|
||||
}
|
||||
|
||||
// took is a build's takeover of the reservation of an older build never released (design 47 G A-455): the older version is
|
||||
// dropped, its bound ended and its finding cleared (the newer build's reservation has its own), and the builder's log says so.
|
||||
func (rv *reservations) took(ctx context.Context, older, newer versions.ReservedVersion) {
|
||||
@@ -92,6 +107,11 @@ func (rv *reservations) took(ctx context.Context, older, newer versions.Reserved
|
||||
}
|
||||
rv.dropBound(ctx, versions.ReservedKey(older.Module, older.Series))
|
||||
rv.found.clear(ctx, versions.VersionUnreleased, meshconditionsclient.ScopeModule, older.Module)
|
||||
if older.Fingerprint != newer.Fingerprint && rv.withdraw != nil {
|
||||
// The older build is ended superseded, as the picture's supersession ends one (H-A-225, ADR 0664 §4): nothing publishes
|
||||
// under the dropped number.
|
||||
rv.withdraw(older.Module, older.Fingerprint, "its reservation "+older.Version+" was taken over by the build of "+newer.Fingerprint+", which reserves "+newer.Version)
|
||||
}
|
||||
}
|
||||
|
||||
var _ build.Reservations = (*reservations)(nil)
|
||||
@@ -110,6 +130,11 @@ func (rv *reservations) ReservedFor(ctx context.Context, module, fingerprint str
|
||||
return versions.ReservedVersion{}, false, nil
|
||||
}
|
||||
|
||||
// TakenOver is the reservation that took module's series over from the build of fingerprint (H-A-244).
|
||||
func (rv *reservations) TakenOver(ctx context.Context, module, fingerprint string) (versions.ReservedVersion, bool, error) {
|
||||
return rv.r.TakenOver(ctx, module, fingerprint)
|
||||
}
|
||||
|
||||
func (rv *reservations) Filled() bool { return rv.filled.Load() }
|
||||
|
||||
func (rv *reservations) Reserve(ctx context.Context, module, series string, artifacts []string, src versions.Source) (versions.ReservedVersion, error) {
|
||||
@@ -253,6 +278,7 @@ func (rv *reservations) heard(ctx context.Context, m jetstream.Msg) bool {
|
||||
rv.released(ctx, ev.Module, ev.Version)
|
||||
}
|
||||
rv.found.clear(ctx, versions.VersionUnreleased, meshconditionsclient.ScopeModule, ev.Module)
|
||||
rv.found.clear(ctx, versions.VersionHeldElsewhere, meshconditionsclient.ScopeModule, ev.Module)
|
||||
default:
|
||||
return false
|
||||
}
|
||||
@@ -337,21 +363,19 @@ func (rv *reservations) tagged(ctx context.Context, ev meshcatalog.VersionReleas
|
||||
return nil
|
||||
}
|
||||
|
||||
// noMark says why a published marked nothing: which reservation holds that version, and what it holds of digests.
|
||||
// noMark says why a published marked nothing: which reservation holds that version, and what it holds of digests, read through the
|
||||
// index by version (H-A-269 (1), H-A-271).
|
||||
func (rv *reservations) noMark(ctx context.Context, o versions.Outcome) string {
|
||||
version := strings.TrimPrefix(o.Version, "v")
|
||||
if version == "" || o.Digest == "" {
|
||||
return "it names no version or no digest (a by-digest publish)"
|
||||
}
|
||||
all, err := rv.r.All(ctx)
|
||||
at, err := rv.r.AtVersion(ctx, version) // the index by version, never every reservation (H-A-271)
|
||||
if err != nil {
|
||||
return "the reservations cannot be read: " + err.Error()
|
||||
return "the reservations at " + version + " cannot be read: " + err.Error()
|
||||
}
|
||||
var out []string
|
||||
for _, v := range all {
|
||||
if v.Version != version {
|
||||
continue
|
||||
}
|
||||
for _, v := range at {
|
||||
out = append(out, fmt.Sprintf("%s holds %d built digests %v, published %v (a published heard before the build recorded its digests is kept until it does)", v.Module, len(v.Artifacts), v.Artifacts, v.Published))
|
||||
}
|
||||
if len(out) == 0 {
|
||||
@@ -383,3 +407,49 @@ func reserveDoneAct(reserve func(ctx context.Context, module, fp string) (versio
|
||||
func (rv *reservations) ReleasedFor(ctx context.Context, module, fingerprint string) (bool, error) {
|
||||
return rv.r.ReleasedFor(ctx, module, fingerprint)
|
||||
}
|
||||
|
||||
// markHeld marks the artifacts the registries' record holds at their version with the built digest published, as their published
|
||||
// heard would, and says package-published of each (design 47 H H-A-274): a published heard and lost never leaves the publish
|
||||
// asked for ever. The version's release follows from the catalog as for any published.
|
||||
func (rv *reservations) markHeld(ctx context.Context, v versions.ReservedVersion, held []versions.HeldArtifact) {
|
||||
for _, h := range held {
|
||||
rv.logf("%s %s: %s holds %s %s with the built digest %s: marked published from its record", v.Module, v.Version, h.Seat, h.Name, h.Version, h.Digest)
|
||||
rv.published(ctx, versions.Outcome{Name: h.Name, Version: h.Version, Digest: h.Digest, Holder: h.Seat})
|
||||
}
|
||||
}
|
||||
|
||||
// heldElsewhere raises version-held-elsewhere about module (design 47 H H-A-275): numbers a registry holds were skipped or moved
|
||||
// from, said with why.
|
||||
func (rv *reservations) heldElsewhere(ctx context.Context, module, why string) {
|
||||
rv.logf("%s: %s", module, why)
|
||||
rv.found.raise(ctx, finding{kind: versions.VersionHeldElsewhere, scope: meshconditionsclient.ScopeModule, id: module,
|
||||
summary: module + ": " + why,
|
||||
headline: "A number a registry already holds is not used",
|
||||
explanation: "A registry seat holds a version of " + module + " the builder did not publish there: " + why + ".",
|
||||
resolved: "The catalog says version-released for " + module + ".",
|
||||
needs: "builder_reserved shows the reservation and its withdrawn numbers; the registry seat's versions shows what it holds."})
|
||||
}
|
||||
|
||||
// movedElsewhere takes a done build whose number a registry holds at another digest off that number (design 47 H H-A-275,
|
||||
// H-A-279): every artifact carries its version from its build, so no digest survives a move. The number is withdrawn as a gap
|
||||
// (H-A-226), the finding raised naming both digests, and the build of the same fingerprint asked again (the build-again path of a
|
||||
// done key whose version is not released, A-490): its reserve takes the next number no registry holds, it records fresh digests
|
||||
// and its own publish runs; the takeover of the withdrawn reservation ends its publish item and bound.
|
||||
func (rv *reservations) movedElsewhere(ctx context.Context, v versions.ReservedVersion, j versions.Judged) error {
|
||||
if rv.rebuild == nil {
|
||||
return errors.New("no build again is wired")
|
||||
}
|
||||
if _, ok, err := rv.r.Withdraw(ctx, v.Module, v.Version); err != nil || !ok {
|
||||
if err == nil {
|
||||
err = fmt.Errorf("no reservation of %s stands at %s", v.Module, v.Version)
|
||||
}
|
||||
return err
|
||||
}
|
||||
var at []string
|
||||
for _, h := range j.Elsewhere {
|
||||
at = append(at, fmt.Sprintf("%s holds %s %s at %s, built %s", h.Seat, h.Name, h.Version, h.Digest, v.Artifacts[h.Ref]))
|
||||
}
|
||||
why := fmt.Sprintf("%s is withdrawn and the build of %s is built again at the next free number: %s", v.Version, v.Fingerprint, strings.Join(at, "; "))
|
||||
rv.heldElsewhere(ctx, v.Module, why)
|
||||
return rv.rebuild(ctx, v.Module, v.Fingerprint, why)
|
||||
}
|
||||
|
||||
@@ -48,7 +48,7 @@ func (m *memReserve) Update(_ context.Context, key string, v []byte, rev uint64)
|
||||
func (m *memReserve) KeysOf(_ context.Context, filter string) ([]string, error) {
|
||||
var out []string
|
||||
for k := range m.vals {
|
||||
if filter == versions.AllFilter || strings.HasPrefix(k, strings.TrimSuffix(filter, "*")) {
|
||||
if versions.FilterMatches(filter, k) { // as the bus reads a filter
|
||||
out = append(out, k)
|
||||
}
|
||||
}
|
||||
@@ -266,7 +266,8 @@ func TestAPublishedHeardBeforeTheDigestsIsSaidWhenTheBuildRecordsThem(t *testing
|
||||
}
|
||||
|
||||
// TestABuildTakingOverAnOlderReservationDropsItsBoundAndSaysSo: design 47 G A-455 — a build that reserves while the series is
|
||||
// reserved by an older build never released drops the older one's bound and finding and says so in the builder's journal.
|
||||
// reserved by an older build never released drops the older one's bound and finding and says so in the builder's journal; and
|
||||
// H-A-225 — the older build is withdrawn, so nothing publishes under the dropped number, while a re-run of the newer keeps it.
|
||||
func TestABuildTakingOverAnOlderReservationDropsItsBoundAndSaysSo(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
st := &memFindings{vals: map[string][]byte{}}
|
||||
@@ -279,6 +280,8 @@ func TestABuildTakingOverAnOlderReservationDropsItsBoundAndSaysSo(t *testing.T)
|
||||
rv := newReservations(&memReserve{vals: map[string][]byte{}, revs: map[string]uint64{}}, call, func() jetstream.KeyValue { return bounds }, found,
|
||||
nil, func(f string, a ...any) { journal = append(journal, fmt.Sprintf(f, a...)) })
|
||||
rv.filled.Store(true)
|
||||
var withdrawn []string
|
||||
rv.withdraw = func(module, fp, why string) { withdrawn = append(withdrawn, module+" "+fp+": "+why) }
|
||||
older, err := rv.Reserve(ctx, "m", "0.1", []string{"m:a"}, versions.Source{Fingerprint: "fp-old"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -305,6 +308,12 @@ func TestABuildTakingOverAnOlderReservationDropsItsBoundAndSaysSo(t *testing.T)
|
||||
if got, ok, _ := rv.ReservedFor(ctx, "m", "fp-new"); !ok || got.Version != "0.1.1" {
|
||||
t.Errorf("reserved for fp-new: %+v %v", got, ok)
|
||||
}
|
||||
if len(withdrawn) != 1 || withdrawn[0] != "m fp-old: its reservation 0.1.0 was taken over by the build of fp-new, which reserves 0.1.1" {
|
||||
t.Errorf("withdrawn %q; want the build of fp-old withdrawn once", withdrawn)
|
||||
}
|
||||
if again, err := rv.Reserve(ctx, "m", "0.1", []string{"m:a"}, versions.Source{Fingerprint: "fp-new"}); err != nil || again.Version != "0.1.1" || len(withdrawn) != 1 {
|
||||
t.Errorf("a re-run of fp-new reserved %q (%v), withdrew %q; want 0.1.1 kept, nothing withdrawn", again.Version, err, withdrawn)
|
||||
}
|
||||
}
|
||||
|
||||
// ansKV records the value versionActed writes back to a reasks key.
|
||||
@@ -384,3 +393,87 @@ func TestAReserveDoneIsAnsweredWithTheVersionAndItsQueuedPublish(t *testing.T) {
|
||||
t.Errorf("the reserving by hand is not said in the log: %q", logged)
|
||||
}
|
||||
}
|
||||
|
||||
// TestARegistrysRecordEndsOrMovesAPublish: design 47 H H-A-274 — the artifacts a registry's record holds with the built digest are
|
||||
// marked published from it and package-published is said, as a heard published would; H-A-275 — a done build whose number a
|
||||
// registry holds at another digest (H-A-279) has its number withdrawn, raises version-held-elsewhere naming both digests, and is
|
||||
// built again: at the next number no registry holds, with a fresh digest, published there; the old number's item ends.
|
||||
func TestARegistrysRecordEndsOrMovesAPublish(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
fs := &memFindings{vals: map[string][]byte{}}
|
||||
none := func(string, any) (json.RawMessage, error) {
|
||||
return json.Marshal(meshcatalog.VersionsAnswer{Module: "node-tools"})
|
||||
}
|
||||
bounds := &boundsKV{keys: map[string]bool{}}
|
||||
rv := newReservations(&memReserve{vals: map[string][]byte{}, revs: map[string]uint64{}}, none, func() jetstream.KeyValue { return bounds },
|
||||
&findings{module: builds.BuilderModule, w: fs, logf: t.Logf, now: time.Now}, nil, t.Logf)
|
||||
var said []meshbuilds.PackagePublished
|
||||
rv.say = func(_ context.Context, p meshbuilds.PackagePublished) error { said = append(said, p); return nil }
|
||||
var asked, ended []string
|
||||
rv.built = func(_ context.Context, m, v string) error { asked = append(asked, m+" "+v); return nil }
|
||||
rv.released = func(_ context.Context, m, v string) { ended = append(ended, m+" "+v) }
|
||||
registry := []string{"0.1.37"}
|
||||
rv.r.HeldNumbers = func(context.Context, string, string, []string) ([]string, error) { return registry, nil }
|
||||
rv.filled.Store(true)
|
||||
res, err := rv.Reserve(ctx, "node-tools", "0.1", []string{"node-tools:program"}, versions.Source{Fingerprint: "fp1"})
|
||||
if err != nil || res.Version != "0.1.0" {
|
||||
t.Fatalf("reserved %q (%v)", res.Version, err)
|
||||
}
|
||||
if err := rv.Built(ctx, "node-tools", res.Version, map[string]string{"node-tools:program": "sha256:7c9b10"}, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cur, _, _ := rv.r.Of(ctx, "node-tools", res.Version)
|
||||
held := versions.HeldArtifact{Ref: "node-tools:program", Seat: versions.SeatArtifacts, Name: "node-tools/program", Version: res.Version, Digest: "sha256:7c9b10"}
|
||||
rv.markHeld(ctx, cur, []versions.HeldArtifact{held})
|
||||
if len(said) != 1 || said[0].Artifact != "node-tools:program" || said[0].Version != res.Version {
|
||||
t.Fatalf("package-published said %+v, want node-tools:program at %s from the registry's record", said, res.Version)
|
||||
}
|
||||
if cur, _, _ = rv.r.Of(ctx, "node-tools", res.Version); len(cur.Missing()) != 0 {
|
||||
t.Errorf("the record marked nothing: missing %v", cur.Missing())
|
||||
}
|
||||
// Another reservation of the module, done, whose number the registry holds at another digest: withdrawn, built again at the
|
||||
// next free number with a fresh digest, and published there (H-A-279).
|
||||
if _, err := rv.r.Released(ctx, "node-tools", res.Version); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
asked, ended, said, registry = nil, nil, nil, []string{"0.1.0", "0.1.1", "0.1.2"}
|
||||
raw, _ := json.Marshal(versions.ReservedVersion{V: 1, Module: "node-tools", Series: "0.1", Version: "0.1.1", Fingerprint: "fp2",
|
||||
Artifacts: map[string]string{"node-tools:program": "sha256:7c9b10"}, BuiltAt: time.Now()})
|
||||
if _, err := rv.r.Store.Create(ctx, versions.ReservedKey("node-tools", "0.1"), raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rv.rebuild = func(ctx context.Context, module, fp, _ string) error {
|
||||
again, err := rv.Reserve(ctx, module, "0.1", []string{"node-tools:program"}, versions.Source{Fingerprint: fp, Rebuild: true})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := rv.Built(ctx, module, again.Version, map[string]string{"node-tools:program": "sha256:fresh"}, nil); err != nil {
|
||||
return err
|
||||
}
|
||||
rv.published(ctx, versions.Outcome{Name: "node-tools/program", Version: again.Version, Digest: "sha256:fresh"})
|
||||
return nil
|
||||
}
|
||||
cur, _, _ = rv.r.Of(ctx, "node-tools", "0.1.1")
|
||||
elsewhere := held
|
||||
elsewhere.Version, elsewhere.Digest = "0.1.1", "sha256:83bec1"
|
||||
if err := rv.movedElsewhere(ctx, cur, versions.Judged{Elsewhere: []versions.HeldArtifact{elsewhere}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
again, ok, _ := rv.r.Of(ctx, "node-tools", "")
|
||||
if !ok || again.Version != "0.1.3" || again.Withdrawn["0.1.1"] != "fp2" || again.Artifacts["node-tools:program"] != "sha256:fresh" {
|
||||
t.Fatalf("built again as %+v, want 0.1.3 with a fresh digest and 0.1.1 withdrawn", again)
|
||||
}
|
||||
if len(said) != 1 || said[0].Version != "0.1.3" || said[0].Digest != "sha256:fresh" {
|
||||
t.Errorf("package-published said %+v, want the fresh build at 0.1.3", said)
|
||||
}
|
||||
if len(asked) != 1 || asked[0] != "node-tools 0.1.3" || len(ended) != 1 || ended[0] != "node-tools 0.1.1" {
|
||||
t.Errorf("asked %v, ended %v: want the new number's publish asked and the old one's item ended", asked, ended)
|
||||
}
|
||||
raised := ""
|
||||
for _, v := range fs.vals {
|
||||
raised += string(v)
|
||||
}
|
||||
if !strings.Contains(raised, versions.VersionHeldElsewhere) || !strings.Contains(raised, "sha256:83bec1") || !strings.Contains(raised, "sha256:7c9b10") {
|
||||
t.Errorf("no version-held-elsewhere naming both digests: %s", raised)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -149,6 +149,9 @@ type Reservations interface {
|
||||
ReservedFor(ctx context.Context, module, fingerprint string) (versions.ReservedVersion, bool, error)
|
||||
// ReleasedFor says whether a version was released for the build of fingerprint, read from the builder's own record (A-456a).
|
||||
ReleasedFor(ctx context.Context, module, fingerprint string) (bool, error)
|
||||
// TakenOver is the reservation of module's series that took the series over from the build of fingerprint, whose number
|
||||
// it keeps withdrawn (design 47 H H-A-226), and whether one does (H-A-244).
|
||||
TakenOver(ctx context.Context, module, fingerprint string) (versions.ReservedVersion, bool, error)
|
||||
}
|
||||
|
||||
// reserves says whether a look at a trunk build is one that reserves: the build has a series of the form MAJOR.MINOR
|
||||
|
||||
@@ -29,8 +29,14 @@ type fakeReservations struct {
|
||||
built map[string]string
|
||||
sources []versions.Source
|
||||
deps map[string][]meshbuilds.Dependency
|
||||
next string // the version Reserve answers when set, else the series' .5
|
||||
fail error // the reservation cannot be made (the catalog's versions cannot be read)
|
||||
next string // the version Reserve answers when set, else the series' .5
|
||||
fail error // the reservation cannot be made (the catalog's versions cannot be read)
|
||||
tookBy map[string]versions.ReservedVersion // by fingerprint, the reservation that took its series over (H-A-244)
|
||||
}
|
||||
|
||||
func (f *fakeReservations) TakenOver(_ context.Context, _, fp string) (versions.ReservedVersion, bool, error) {
|
||||
v, ok := f.tookBy[fp]
|
||||
return v, ok, nil
|
||||
}
|
||||
|
||||
func (f *fakeReservations) ReleasedFor(_ context.Context, _, fp string) (bool, error) {
|
||||
@@ -354,7 +360,10 @@ func (r *realReserve) Built(ctx context.Context, module, version string, digests
|
||||
|
||||
type reserveMap struct{ *memStore }
|
||||
|
||||
func (m reserveMap) KeysOf(ctx context.Context, _ string) ([]string, error) { return m.Keys(ctx) }
|
||||
func (m reserveMap) KeysOf(ctx context.Context, filter string) ([]string, error) {
|
||||
all, err := m.Keys(ctx)
|
||||
return slices.DeleteFunc(all, func(k string) bool { return !versions.FilterMatches(filter, k) }), err // as the bus reads a filter
|
||||
}
|
||||
func (m reserveMap) Delete(_ context.Context, key string) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
@@ -362,10 +371,10 @@ func (m reserveMap) Delete(_ context.Context, key string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// TestABuildAgainReservesAboveTheCatalogsHighestNotItsOwnStandingNumber: design 47 G A-490b, at the build, through the real
|
||||
// reserver — a done key whose own fingerprint has an unreleased 0.1.30 standing, the catalog's highest 0.1.16, is built again
|
||||
// under 0.1.17 (never 0.1.31), the standing reservation taken over.
|
||||
func TestABuildAgainReservesAboveTheCatalogsHighestNotItsOwnStandingNumber(t *testing.T) {
|
||||
// TestABuildAgainKeepsItsOwnStandingNumber: design 47 G A-490b as H-A-225 amends it, at the build, through the real reserver —
|
||||
// a done key whose own fingerprint has an unreleased 0.1.30 standing, the catalog's highest 0.1.16, is built again under 0.1.30
|
||||
// (never 0.1.31, never a new number): the run building carries the standing number until it is released.
|
||||
func TestABuildAgainKeepsItsOwnStandingNumber(t *testing.T) {
|
||||
r := newRig(t, goBundle)
|
||||
r.mf.byDigest[mdigest('a')] = catalogue.Manifest{Module: "m", Version: "0.1", Build: &catalogue.Build{Artifacts: []catalogue.Artifact{goBundle}}}
|
||||
if v := r.consider(t, "modules/m"); v.State != builds.BuildDone {
|
||||
@@ -381,8 +390,8 @@ func TestABuildAgainReservesAboveTheCatalogsHighestNotItsOwnStandingNumber(t *te
|
||||
r.b.Reserved = real
|
||||
got := r.b.BuildAgain(context.Background(), "m", fp1, "its publish was refused", "shanks.lead", 2*time.Second)
|
||||
r.b.Wait()
|
||||
cur, ok, err := real.Of(context.Background(), "m", "0.1.17")
|
||||
if got.Refused != "" || err != nil || !ok || cur.Fingerprint != fp1 {
|
||||
t.Fatalf("answer %+v; reservation of 0.1.17 = %+v (%v, %v); want built again under 0.1.17, the catalog's highest 0.1.16 plus one", got, cur, ok, err)
|
||||
cur, ok, err := real.Of(context.Background(), "m", "")
|
||||
if got.Refused != "" || err != nil || !ok || cur.Fingerprint != fp1 || cur.Version != "0.1.30" {
|
||||
t.Fatalf("answer %+v; reservation = %+v (%v, %v); want built again under its standing 0.1.30", got, cur, ok, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -25,6 +25,12 @@ import (
|
||||
// below the newest the picture holds, whose entries the picture no longer names, asks the catalog seat's verb revision for
|
||||
// its catalog revision by number, once, and is built when that kept revision is of its source and names every entry it
|
||||
// asks; it never waits, since the picture only moves on. Only a revision above the newest is waited for.
|
||||
//
|
||||
// **An ask naming a revision is held by its number** (design 47 H H-A-222): when the picture does not name the ask's
|
||||
// entries, the ask's catalog revision is read by number, and the ask is built when the catalog keeps it and it names every
|
||||
// fingerprint asked, whatever the source the picture holds says (a re-pin keeps a later revision of the same source, H-A-211,
|
||||
// H-A-213, so the source alone cannot say it). The source is compared only for an ask naming no revision; an ask whose
|
||||
// revision the catalog does not answer and whose source is not below the picture's waits as before.
|
||||
|
||||
// DefaultRevisionWait is the bound of a step ask's wait for its revision, its setting revision-wait-minutes' default (H-A-156).
|
||||
const DefaultRevisionWait = 2 * time.Minute
|
||||
@@ -73,10 +79,13 @@ func (b *Builder) holdsRevision(branch string, source int64, entries []meshbuild
|
||||
return true
|
||||
}
|
||||
|
||||
// keptRevisionHolds answers no outcome when the catalog keeps the ask's revision (its catalog revision by number, of its
|
||||
// source revision) and that revision names every entry of the ask; else failed or could not run, with its words.
|
||||
// keptRevisionHolds answers no outcome when the catalog keeps the ask's revision (its catalog revision by number) and that
|
||||
// revision names every entry of the ask (H-A-205, H-A-222); else failed or could not run, with its words.
|
||||
func (b *Builder) keptRevisionHolds(ctx context.Context, run meshbuilds.StepRun, held int64) (outcome, said string) {
|
||||
below := fmt.Sprintf("revision %d of %s is below the %d the builder's picture holds", run.Source, run.Branch, held)
|
||||
below := fmt.Sprintf("revision %d of %s is not named by the builder's picture, which holds source revision %d", run.Source, run.Branch, held)
|
||||
if run.Source < held {
|
||||
below = fmt.Sprintf("revision %d of %s is below the %d the builder's picture holds", run.Source, run.Branch, held)
|
||||
}
|
||||
if b.KeptRevision == nil || run.Revision <= 0 {
|
||||
return meshbuilds.StepEndedOutcomeFailed, below + ", and the ask names no catalog revision to read it by"
|
||||
}
|
||||
@@ -85,11 +94,15 @@ func (b *Builder) keptRevisionHolds(ctx context.Context, run meshbuilds.StepRun,
|
||||
b.logf("%s: %s, and the catalog's revision %d cannot be read: %v", run.Pipeline, below, run.Revision, err)
|
||||
return meshbuilds.StepEndedOutcomeCouldNotRun, fmt.Sprintf("%s, and the catalog's revision %d cannot be read: %v", below, run.Revision, err)
|
||||
}
|
||||
// An ask names the revision and the source of one revision (H-A-218, H-A-219): a kept revision of another source is
|
||||
// not the ask's.
|
||||
if source != run.Source {
|
||||
return meshbuilds.StepEndedOutcomeFailed, fmt.Sprintf("%s, and the catalog's revision %d is of source revision %d", below, run.Revision, source)
|
||||
b.logf("%s: %s, and the catalog's revision %d is of source revision %d, the ask names %d: the step fails, nothing is built", run.Pipeline, below, run.Revision, source, run.Source)
|
||||
return meshbuilds.StepEndedOutcomeFailed, fmt.Sprintf("%s, and the catalog's revision %d is of source revision %d, the ask names %d", below, run.Revision, source, run.Source)
|
||||
}
|
||||
for _, e := range run.Entries {
|
||||
if fps[e.Module] != e.Fingerprint {
|
||||
b.logf("%s: %s, and the catalog's revision %d names %s at %q, not %s: the step fails, nothing is built", run.Pipeline, below, run.Revision, e.Module, fps[e.Module], e.Fingerprint)
|
||||
return meshbuilds.StepEndedOutcomeFailed, fmt.Sprintf("%s, and the catalog's revision %d names %s at %q, not %s", below, run.Revision, e.Module, fps[e.Module], e.Fingerprint)
|
||||
}
|
||||
}
|
||||
@@ -116,8 +129,17 @@ func (b *Builder) awaitRevision(ctx context.Context, run meshbuilds.StepRun) (ou
|
||||
if b.Picture == nil || run.Source <= 0 || run.Branch == "" {
|
||||
return "", ""
|
||||
}
|
||||
if held, ok := b.Picture(run.Branch); ok && run.Source < held && !b.holdsRevision(run.Branch, run.Source, run.Entries) {
|
||||
return b.keptRevisionHolds(ctx, run, held)
|
||||
if held, ok := b.Picture(run.Branch); !b.holdsRevision(run.Branch, run.Source, run.Entries) {
|
||||
below := ok && run.Source < held
|
||||
if (run.Revision > 0 && b.KeptRevision != nil) || below {
|
||||
// By its number (H-A-222), whatever the source the picture holds; for an ask naming none, only below the
|
||||
// picture (H-A-205). A kept revision of another source or naming other fingerprints is a mismatch the wait
|
||||
// cannot mend: it fails at once; only a revision the catalog does not answer is waited for.
|
||||
outcome, said := b.keptRevisionHolds(ctx, run, held)
|
||||
if outcome == "" || below || outcome == meshbuilds.StepEndedOutcomeFailed {
|
||||
return outcome, said
|
||||
}
|
||||
}
|
||||
}
|
||||
bound := b.revisionWait()
|
||||
notHeld := fmt.Sprintf("revision %d of %s not in the builder's picture", run.Source, run.Branch)
|
||||
|
||||
@@ -266,6 +266,23 @@ func (b *Builder) EntryGone(branch string, first bool, module string) {
|
||||
}
|
||||
}
|
||||
|
||||
// Withdraw ends the running build of module's fingerprint, superseded, with why (design 47 H H-A-225, ADR 0664 §4): its key
|
||||
// fails "withdrawn" and its asks are withdrawn from the build seats. A fingerprint with no build running is left.
|
||||
func (b *Builder) Withdraw(module, fingerprint, why string) {
|
||||
key, err := builds.BuilderBuildKey(module, fingerprint)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
b.init()
|
||||
b.mu.Lock()
|
||||
withdraw := b.builds[key]
|
||||
b.mu.Unlock()
|
||||
if withdraw != nil {
|
||||
b.logf("%s at %s: the build is withdrawn: %s", module, fingerprint, why)
|
||||
withdraw(buildrun.Withdrawn{Why: why})
|
||||
}
|
||||
}
|
||||
|
||||
func (b *Builder) take(ctx context.Context, o Origin, key string, m meshcatalog.ModulesByModuleValue, e []meshcatalog.Edge) {
|
||||
b.mu.Lock()
|
||||
hk := o.Branch + "\x00" + key
|
||||
@@ -707,8 +724,9 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB
|
||||
var reserved versions.ReservedVersion
|
||||
hasReserved := false
|
||||
if rebuild && cl.Build.State == builds.BuildBuilding {
|
||||
// Reserved afresh, above the highest the catalog's versions answers, taking over the older unreleased reservation of this
|
||||
// fingerprint (A-455), never its standing number kept (A-490b); the build's own publish then runs.
|
||||
// Reserved as A-490b amended by H-A-225 has it: this fingerprint's standing reservation is kept while it is unreleased
|
||||
// and above the highest the catalog's versions answers, else the next above that highest is taken (A-455); the build's
|
||||
// own publish then runs.
|
||||
if b.Reserved.Filled() {
|
||||
reserved, hasReserved = b.reserve(ctx, module, m, plan, b.originOf(key), true)
|
||||
}
|
||||
|
||||
@@ -327,6 +327,37 @@ func TestABundleAndAnArchiveAreBuiltOnOneNodeAndDone(t *testing.T) { // 1607/AC7
|
||||
}
|
||||
}
|
||||
|
||||
// H-A-225 (ADR 0664 §4): a build withdrawn while it runs (its reservation taken over by another fingerprint) ends failed
|
||||
// "withdrawn" with why, so it never publishes under the dropped number; a fingerprint with no build running is left.
|
||||
func TestAWithdrawnBuildEndsWithdrawn(t *testing.T) {
|
||||
r := newRig(t, goBundle)
|
||||
asked, release := make(chan struct{}), make(chan struct{})
|
||||
answer := r.answer
|
||||
r.asks.Answer = func(a buildstep.StepAsk) buildstep.Step {
|
||||
close(asked)
|
||||
<-release
|
||||
return answer(a)
|
||||
}
|
||||
r.b.Withdraw("m", fp1, "nothing runs yet") // left: no build of it runs
|
||||
data, err := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-catalog", Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20",
|
||||
Fp2: fp1, Manifest: mdigest('a'), Path: "modules/m", Toolchains: CatalogToolchains(agreedToolchains)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r.b.Module(context.Background(), "m", data)
|
||||
select {
|
||||
case <-asked:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("no ask was put")
|
||||
}
|
||||
r.b.Withdraw("m", fp1, "its reservation 0.1.3 was taken over by the build of src2:b, which reserves 0.1.4")
|
||||
close(release)
|
||||
r.b.Wait()
|
||||
if v := valueOf(t, r.st, r.key); v.State != builds.BuildFailed || !strings.Contains(v.Error, "withdrawn: its reservation 0.1.3 was taken over") {
|
||||
t.Fatalf("the key = %+v, want failed withdrawn naming the take-over", v)
|
||||
}
|
||||
}
|
||||
|
||||
// 1887/AC1, AC2: a trunk fingerprint with no key is one ask on the Go seat's delivery stream, named in the building key
|
||||
// before it is put, and the builder calls no node; the step a node wrote for it makes the key done naming its reference.
|
||||
func TestATrunkFingerprintIsOneAskOnTheDeliveryStream(t *testing.T) {
|
||||
|
||||
@@ -136,7 +136,8 @@ func (b *Builder) buildEntry(ctx context.Context, e meshbuilds.BuildEntry, pins
|
||||
// passedWithItsVersion is the verdict of a build that ended done (design 47 G A-456): a build that reserves a version (a build
|
||||
// of the first watched branch's trunk, with a MAJOR.MINOR series and artifacts) passes only when a version is reserved for its
|
||||
// fingerprint; one that ended done with none fails loudly, naming builder_reserve_done, so no pipeline says passed for a build
|
||||
// that can never be released. A build that reserves none (a candidate's, a series of before G) passes as it did.
|
||||
// that can never be released, and one whose series another fingerprint took over fails saying it is stale, naming that
|
||||
// fingerprint and its version (design 47 H H-A-244). A build that reserves none (a candidate's, a series of before G) passes as it did.
|
||||
func (b *Builder) passedWithItsVersion(ctx context.Context, entry meshcatalog.CatalogEntry) (outcome, said string) {
|
||||
if b.Reserved == nil {
|
||||
return meshbuilds.StepEndedOutcomePassed, ""
|
||||
@@ -168,6 +169,12 @@ func (b *Builder) passedWithItsVersion(ctx context.Context, entry meshcatalog.Ca
|
||||
} else if released {
|
||||
return meshbuilds.StepEndedOutcomePassed, "" // a released build counts as reserved (A-456a)
|
||||
}
|
||||
if by, took, err := b.Reserved.TakenOver(ctx, entry.Module, entry.Fp); err != nil {
|
||||
return meshbuilds.StepEndedOutcomeCouldNotRun, fmt.Sprintf("%s at %s: the reservations cannot be read: %v", entry.Module, entry.Fp, err)
|
||||
} else if took {
|
||||
// H-A-244 (2): the series was taken over by a newer fingerprint, whose number it carries now: the build is stale.
|
||||
return meshbuilds.StepEndedOutcomeFailed, fmt.Sprintf("%s: built at %s but its series moved to %s %s: stale", entry.Module, entry.Fp, by.Fingerprint, by.Version)
|
||||
}
|
||||
return meshbuilds.StepEndedOutcomeFailed, fmt.Sprintf("%s: built at %s, but no version is reserved for it; builder_reserve_done reserves one for the done build", entry.Module, entry.Fp)
|
||||
}
|
||||
return meshbuilds.StepEndedOutcomePassed, ""
|
||||
|
||||
@@ -89,18 +89,23 @@ func TestABuildStepPassesAReservingBuildOnlyWithItsReservedVersion(t *testing.T)
|
||||
for _, c := range []struct {
|
||||
name, version string
|
||||
reserve, released, unreadable bool
|
||||
takenOver bool
|
||||
want, said string
|
||||
}{
|
||||
{"no version reserved", "0.1", false, false, false, meshbuilds.StepEndedOutcomeFailed, "built at " + fp1 + ", but no version is reserved for it"},
|
||||
{"a version reserved", "0.1", true, false, false, meshbuilds.StepEndedOutcomePassed, ""},
|
||||
{"a released build whose reservation is gone", "0.1", false, true, false, meshbuilds.StepEndedOutcomePassed, ""},
|
||||
{"the plan cannot be read", "0.1", false, false, true, meshbuilds.StepEndedOutcomeCouldNotRun, "plan cannot be read"},
|
||||
{"a series of before G", "1", false, false, false, meshbuilds.StepEndedOutcomePassed, ""},
|
||||
{"no version reserved", "0.1", false, false, false, false, meshbuilds.StepEndedOutcomeFailed, "built at " + fp1 + ", but no version is reserved for it"},
|
||||
{"H-A-244: its series was taken over by a newer fingerprint", "0.1", false, false, false, true, meshbuilds.StepEndedOutcomeFailed, "m: built at " + fp1 + " but its series moved to " + fp2 + " 0.1.7: stale"},
|
||||
{"a version reserved", "0.1", true, false, false, false, meshbuilds.StepEndedOutcomePassed, ""},
|
||||
{"a released build whose reservation is gone", "0.1", false, true, false, false, meshbuilds.StepEndedOutcomePassed, ""},
|
||||
{"the plan cannot be read", "0.1", false, false, true, false, meshbuilds.StepEndedOutcomeCouldNotRun, "plan cannot be read"},
|
||||
{"a series of before G", "1", false, false, false, false, meshbuilds.StepEndedOutcomePassed, ""},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
r := newRig(t, goBundle)
|
||||
r.mf.byDigest[mdigest('a')] = catalogue.Manifest{Module: "m", Version: c.version, Build: &catalogue.Build{Artifacts: []catalogue.Artifact{goBundle}}}
|
||||
res := &fakeReservations{filled: true, released: map[string]bool{fp1: c.released}}
|
||||
if c.takenOver {
|
||||
res.tookBy = map[string]versions.ReservedVersion{fp1: {Module: "m", Series: "0.1", Version: "0.1.7", Fingerprint: fp2}}
|
||||
}
|
||||
if v := r.consider(t, "modules/m"); v.State != builds.BuildDone { // built while the builder reserved nothing, as the live key was
|
||||
t.Fatalf("build = %+v, want done", v)
|
||||
}
|
||||
@@ -112,7 +117,7 @@ func TestABuildStepPassesAReservingBuildOnlyWithItsReservedVersion(t *testing.T)
|
||||
return meshcatalog.CatalogEntry{Module: "m", Repository: "novox/m", Commit: "c1", Fp: fp1, ManifestDigest: digestOf(c.unreadable), Path: "modules/m"}, nil
|
||||
}
|
||||
end := r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.1.m", Step: meshbuilds.StepRunStepBuild, Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}})
|
||||
if end.Outcome != c.want || !strings.Contains(end.Said, c.said) || (c.want == meshbuilds.StepEndedOutcomeFailed && !strings.Contains(end.Said, "builder_reserve_done")) {
|
||||
if end.Outcome != c.want || !strings.Contains(end.Said, c.said) || (c.want == meshbuilds.StepEndedOutcomeFailed && strings.Contains(end.Said, "builder_reserve_done") == c.takenOver) {
|
||||
t.Errorf("end = %+v, want %s saying %q", end, c.want, c.said)
|
||||
}
|
||||
})
|
||||
@@ -356,7 +361,7 @@ func aStepAskWaitsForItsRevision(t *testing.T) {
|
||||
held.Store(40)
|
||||
asked := make(chan struct{}, 1)
|
||||
r, res, entry := revisionRig(t, &held, asked)
|
||||
var keptReads atomic.Int64 // H-A-205: a revision above the newest is waited for, never read by number
|
||||
var keptReads atomic.Int64 // H-A-205, H-A-222: a revision above the newest is read by number once; not answered, it is waited for
|
||||
r.b.KeptRevision = func(context.Context, string, int64) (int64, map[string]string, error) {
|
||||
keptReads.Add(1)
|
||||
return 0, nil, errors.New("not asked")
|
||||
@@ -385,8 +390,8 @@ func aStepAskWaitsForItsRevision(t *testing.T) {
|
||||
t.Fatal("the ask was not woken by the entry the picture took in")
|
||||
}
|
||||
r.b.Wait()
|
||||
if end.Outcome != meshbuilds.StepEndedOutcomePassed || len(res.sources) != 1 || res.sources[0].Revision != 41 || keptReads.Load() != 0 {
|
||||
t.Errorf("end = %+v, reserved from %+v, kept revisions read %d; want passed with one version reserved from revision 41, none read", end, res.sources, keptReads.Load())
|
||||
if end.Outcome != meshbuilds.StepEndedOutcomePassed || len(res.sources) != 1 || res.sources[0].Revision != 41 || keptReads.Load() != 1 {
|
||||
t.Errorf("end = %+v, reserved from %+v, kept revisions read %d; want passed with one version reserved from revision 41, read by number once", end, res.sources, keptReads.Load())
|
||||
}
|
||||
|
||||
}
|
||||
@@ -425,6 +430,45 @@ func anOlderKeptRevisionIsReadByNumber(t *testing.T) {
|
||||
if end := r2.b.BuildStep(t.Context(), run); end.Outcome != meshbuilds.StepEndedOutcomeFailed || !strings.Contains(end.Said, "names m at \"src2:other\"") {
|
||||
t.Errorf("end = %+v; want failed naming the fingerprint the kept revision names", end)
|
||||
}
|
||||
// H-A-222: an ask of the very source the picture holds, whose entries the picture no longer names (a re-pin kept revision 48
|
||||
// of the same source 46 and the picture took its entries in), is held by its revision read by number, never waited for.
|
||||
r3, res3, _ := revisionRig(t, &held, make(chan struct{}, 1))
|
||||
r3.b.RevisionWait = time.Nanosecond
|
||||
r3.b.FirstBranch = func(branch, module string) bool { return branch == "main" && module == "m" }
|
||||
var read3 []string
|
||||
r3.b.KeptRevision = func(_ context.Context, branch string, revision int64) (int64, map[string]string, error) {
|
||||
read3 = append(read3, fmt.Sprintf("%s %d", branch, revision))
|
||||
return 46, map[string]string{"m": fp1}, nil
|
||||
}
|
||||
same := meshbuilds.StepRun{Pipeline: "main.48.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Revision: 48, Source: 46,
|
||||
Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}}
|
||||
end3 := r3.b.BuildStep(t.Context(), same)
|
||||
r3.b.Wait()
|
||||
if end3.Outcome != meshbuilds.StepEndedOutcomePassed || !slices.Equal(read3, []string{"main 48"}) || len(res3.sources) != 1 || res3.sources[0].CatalogRevision != 48 {
|
||||
t.Errorf("end = %+v, kept revisions read %v, reserved %+v; want passed, revision 48 of main read once and reserved at it", end3, read3, res3.sources)
|
||||
}
|
||||
// H-A-218, H-A-222: a kept revision of another source than the ask names, or naming another fingerprint, is a mismatch:
|
||||
// the step fails at once naming it, never waiting out its bound (the bound here is long, the ask is not below the picture).
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
source int64
|
||||
fp string
|
||||
said string
|
||||
}{
|
||||
{"another source", 45, fp1, "the catalog's revision 48 is of source revision 45, the ask names 46"},
|
||||
{"another fingerprint", 46, "src2:other", "names m at \"src2:other\""},
|
||||
} {
|
||||
rc, resc, _ := revisionRig(t, &held, make(chan struct{}, 1))
|
||||
rc.b.RevisionWait = time.Hour
|
||||
rc.b.KeptRevision = func(context.Context, string, int64) (int64, map[string]string, error) {
|
||||
return c.source, map[string]string{"m": c.fp}, nil
|
||||
}
|
||||
end := rc.b.BuildStep(t.Context(), same)
|
||||
rc.b.Wait()
|
||||
if end.Outcome != meshbuilds.StepEndedOutcomeFailed || !strings.Contains(end.Said, c.said) || len(resc.sources) != 0 {
|
||||
t.Errorf("%s: end = %+v, reserved %+v; want failed at once saying %q, nothing reserved", c.name, end, resc.sources, c.said)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func aStepAskPastItsRevisionBoundFails(t *testing.T) {
|
||||
|
||||
@@ -0,0 +1,190 @@
|
||||
package versions
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
)
|
||||
|
||||
// **The reservations' index by digest and by version** (design 47 H H-A-269 (1), H-A-271): beside the reservations, in the same
|
||||
// state, the builder keeps which reservation keys hold a built digest and which stand at a version, so a registry seat's published
|
||||
// finds its reservation in one read and a published whose digest no reservation holds is judged from its version's keys in one
|
||||
// more, never a scan of every reservation. The index keys have three tokens, so the reservations' filter (two tokens) never lists
|
||||
// them. It is maintained at reserve, at a digest recorded, at release and at drop; an entry left behind by a write that ended
|
||||
// halfway is harmless (every read checks the reservation it names), and a missing one is written again by Reindex at the start.
|
||||
|
||||
const (
|
||||
indexDigestPrefix = "index.digest."
|
||||
indexVersionPrefix = "index.version."
|
||||
// IndexDigestFilter and IndexVersionFilter are the filters of the index's two halves.
|
||||
IndexDigestFilter = indexDigestPrefix + "*"
|
||||
IndexVersionFilter = indexVersionPrefix + "*"
|
||||
)
|
||||
|
||||
// IndexDigestKey is the index key of the reservations holding digest as a built artifact's.
|
||||
func IndexDigestKey(digest string) string { return indexDigestPrefix + EscapeToken(digest) }
|
||||
|
||||
// IndexVersionKey is the index key of the reservations standing at version (MAJOR.MINOR.PATCH).
|
||||
func IndexVersionKey(version string) string { return indexVersionPrefix + EscapeToken(version) }
|
||||
|
||||
// IndexEntry is an index key's value: the reservation keys, sorted.
|
||||
type IndexEntry struct {
|
||||
Keys []string `json:"keys"`
|
||||
}
|
||||
|
||||
// indexed reads an index key's reservation keys.
|
||||
func (r *Reserver) indexed(ctx context.Context, key string) ([]string, error) {
|
||||
raw, _, found, err := r.Store.Get(ctx, key)
|
||||
if err != nil || !found {
|
||||
return nil, err
|
||||
}
|
||||
var e IndexEntry
|
||||
if err := json.Unmarshal(raw, &e); err != nil {
|
||||
return nil, fmt.Errorf("%s holds what is no index entry: %w", key, err)
|
||||
}
|
||||
return e.Keys, nil
|
||||
}
|
||||
|
||||
// index adds (or, with add false, removes) reserved under the index key key, by compare-and-set; an entry left empty is deleted.
|
||||
func (r *Reserver) index(ctx context.Context, key, reserved string, add bool) error {
|
||||
for {
|
||||
raw, rev, found, err := r.Store.Get(ctx, key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var e IndexEntry
|
||||
if found {
|
||||
_ = json.Unmarshal(raw, &e)
|
||||
}
|
||||
has := slices.Contains(e.Keys, reserved)
|
||||
if has == add {
|
||||
return nil
|
||||
}
|
||||
if add {
|
||||
e.Keys = append(e.Keys, reserved)
|
||||
sort.Strings(e.Keys)
|
||||
} else {
|
||||
e.Keys = slices.DeleteFunc(e.Keys, func(k string) bool { return k == reserved })
|
||||
}
|
||||
if len(e.Keys) == 0 {
|
||||
return r.Store.Delete(ctx, key)
|
||||
}
|
||||
out, _ := json.Marshal(e)
|
||||
if found {
|
||||
_, err = r.Store.Update(ctx, key, out, rev)
|
||||
} else {
|
||||
_, err = r.Store.Create(ctx, key, out)
|
||||
}
|
||||
if errors.Is(err, ErrConflict) {
|
||||
continue
|
||||
}
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// indexKeys are the index keys a reservation stands under: its version's and each built digest's.
|
||||
func indexKeys(v ReservedVersion) []string {
|
||||
out := []string{}
|
||||
if v.Version != "" {
|
||||
out = append(out, IndexVersionKey(v.Version))
|
||||
}
|
||||
for _, d := range v.Artifacts {
|
||||
if d != "" && !slices.Contains(out, IndexDigestKey(d)) {
|
||||
out = append(out, IndexDigestKey(d))
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// reindex moves reservation key from the index keys of was (its value before, zero when there was none) to those of now (zero when
|
||||
// it is gone).
|
||||
func (r *Reserver) reindex(ctx context.Context, key string, was, now ReservedVersion) error {
|
||||
add, drop := indexKeys(now), indexKeys(was)
|
||||
var errs []error
|
||||
for _, k := range add {
|
||||
if err := r.index(ctx, k, key, true); err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
}
|
||||
for _, k := range drop {
|
||||
if !slices.Contains(add, k) {
|
||||
if err := r.index(ctx, k, key, false); err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := errors.Join(errs...); err != nil {
|
||||
return fmt.Errorf("the index of %s is not written (Reindex writes it at the start): %w", key, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// AtVersion are the reservations standing at version, read through the index by version (H-A-271): one read of the index and
|
||||
// one of each reservation it names.
|
||||
func (r *Reserver) AtVersion(ctx context.Context, version string) ([]ReservedVersion, error) {
|
||||
keys, err := r.indexed(ctx, IndexVersionKey(version))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []ReservedVersion
|
||||
for _, k := range keys {
|
||||
cur, _, found, err := r.get(ctx, k)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
if found && cur.Version == version {
|
||||
out = append(out, cur)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Reindex writes the index again from the reservations, once at the builder's start (H-A-269 (1), H-A-271): each reservation is
|
||||
// added under its keys, and a reservation an index key names that does not stand under it (read again) is removed from it. Both
|
||||
// are the same compare-and-set as every other index write, so a reservation written meanwhile keeps its entries. It answers how
|
||||
// many reservations it read.
|
||||
func (r *Reserver) Reindex(ctx context.Context) (int, error) {
|
||||
all, err := r.All(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
var errs []error
|
||||
for _, v := range all {
|
||||
key := ReservedKey(v.Module, v.Series)
|
||||
for _, k := range indexKeys(v) {
|
||||
if err := r.index(ctx, k, key, true); err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, filter := range []string{IndexDigestFilter, IndexVersionFilter} {
|
||||
keys, err := r.Store.KeysOf(ctx, filter)
|
||||
if err != nil {
|
||||
return len(all), fmt.Errorf("the index cannot be listed: %w", err)
|
||||
}
|
||||
for _, k := range keys {
|
||||
named, err := r.indexed(ctx, k)
|
||||
if err != nil {
|
||||
errs = append(errs, err)
|
||||
continue
|
||||
}
|
||||
for _, rk := range named {
|
||||
cur, _, found, err := r.get(ctx, rk)
|
||||
if err != nil {
|
||||
errs = append(errs, err)
|
||||
continue
|
||||
}
|
||||
if found && slices.Contains(indexKeys(cur), k) {
|
||||
continue
|
||||
}
|
||||
if err := r.index(ctx, k, rk, false); err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return len(all), errors.Join(errs...)
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
package versions
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// countingReserve counts the listings of the reservations and the reads of a reservation key (two tokens), so a test sees that
|
||||
// a published reads its reservation and no other.
|
||||
type countingReserve struct {
|
||||
*reserveMem
|
||||
lists, reads atomic.Int64
|
||||
}
|
||||
|
||||
func (c *countingReserve) KeysOf(ctx context.Context, filter string) ([]string, error) {
|
||||
c.lists.Add(1)
|
||||
return c.reserveMem.KeysOf(ctx, filter)
|
||||
}
|
||||
|
||||
func (c *countingReserve) Get(ctx context.Context, key string) ([]byte, uint64, bool, error) {
|
||||
if !strings.HasPrefix(key, "index.") {
|
||||
c.reads.Add(1)
|
||||
}
|
||||
return c.reserveMem.Get(ctx, key)
|
||||
}
|
||||
|
||||
// TestTheIndexFindsAPublishedReservationInOneRead: design 47 H H-A-269 (1), H-A-271 — a registry seat's published finds its
|
||||
// reservation through the index by digest, reading that reservation alone and listing none; a published whose digest no
|
||||
// reservation holds is judged from its version's keys (kept on a build that recorded no digest yet, A-415), never a scan; the
|
||||
// index goes with a release and a drop, and is written again at the start from reservations written before it existed.
|
||||
func TestTheIndexFindsAPublishedReservationInOneRead(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
fill := func(t *testing.T) (*Reserver, *countingReserve) {
|
||||
store := &countingReserve{reserveMem: &reserveMem{mem: newMem()}}
|
||||
r := &Reserver{Store: store, ReleasedOf: func(context.Context, string) ([]string, error) { return nil, nil }}
|
||||
for i := range 40 {
|
||||
m := fmt.Sprintf("m%02d", i)
|
||||
if _, err := r.Reserve(ctx, m, "0.1", []string{m + ":a"}, Source{Fingerprint: "fp" + m}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if i%2 == 0 { // half are built, each with its own digest
|
||||
if _, err := r.Built(ctx, m, "0.1.0", m+":a", "sha256:"+m, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
store.lists.Store(0)
|
||||
store.reads.Store(0)
|
||||
return r, store
|
||||
}
|
||||
t.Run("a published by digest reads its one reservation", func(t *testing.T) {
|
||||
r, store := fill(t)
|
||||
marked, _, err := r.Published(ctx, "m10-a", "v0.1.0", "sha256:m10")
|
||||
if err != nil || len(marked) != 1 || marked[0].Reserved.Module != "m10" || marked[0].Artifact != "m10:a" {
|
||||
t.Fatalf("the published marked %+v (%v), want m10:a", marked, err)
|
||||
}
|
||||
if l, n := store.lists.Load(), store.reads.Load(); l != 0 || n != 1 {
|
||||
t.Errorf("the published listed %d times and read %d reservations, want none listed and one read", l, n)
|
||||
}
|
||||
})
|
||||
t.Run("a digest no reservation holds is judged from its version's keys", func(t *testing.T) {
|
||||
r, store := fill(t)
|
||||
marked, kept, err := r.Published(ctx, "m11-a", "0.1.0", "sha256:early")
|
||||
if err != nil || len(marked) != 0 {
|
||||
t.Fatalf("an unbuilt digest marked %v (%v)", marked, err)
|
||||
}
|
||||
// The 20 unbuilt reservations at 0.1.0 keep it as heard (A-415); the built ones are read and left.
|
||||
if len(kept) != 20 || store.lists.Load() != 0 || store.reads.Load() != 40 {
|
||||
t.Errorf("kept on %d (want 20), listed %d (want 0), read %d (want the 40 at 0.1.0)", len(kept), store.lists.Load(), store.reads.Load())
|
||||
}
|
||||
at, err := r.AtVersion(ctx, "0.9.9")
|
||||
if err != nil || len(at) != 0 || store.lists.Load() != 0 {
|
||||
t.Errorf("a version no reservation stands at answered %v (%v), listed %d", at, err, store.lists.Load())
|
||||
}
|
||||
})
|
||||
t.Run("a release and a drop take the reservation out of the index", func(t *testing.T) {
|
||||
r, store := fill(t)
|
||||
if _, err := r.Released(ctx, "m10", "0.1.0"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := r.Drop(ctx, "m12", "0.1"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, k := range []string{IndexDigestKey("sha256:m10"), IndexDigestKey("sha256:m12")} {
|
||||
if _, _, found, _ := store.Get(ctx, k); found {
|
||||
t.Errorf("%s stands after its reservation went", k)
|
||||
}
|
||||
}
|
||||
keys, _ := r.indexed(ctx, IndexVersionKey("0.1.0"))
|
||||
for _, k := range keys {
|
||||
if k == ReservedKey("m10", "0.1") || k == ReservedKey("m12", "0.1") {
|
||||
t.Errorf("the version's index still names %s", k)
|
||||
}
|
||||
}
|
||||
})
|
||||
t.Run("the index is written again at the start from the reservations", func(t *testing.T) {
|
||||
store := &countingReserve{reserveMem: &reserveMem{mem: newMem()}}
|
||||
r := &Reserver{Store: store}
|
||||
// A reservation written before the index existed, and an index entry naming a reservation that is gone.
|
||||
raw, _ := json.Marshal(ReservedVersion{V: 1, Module: "old", Series: "0.2", Version: "0.2.4", Artifacts: map[string]string{"old:a": "sha256:old"}})
|
||||
if _, err := store.Create(ctx, ReservedKey("old", "0.2"), raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stale, _ := json.Marshal(IndexEntry{Keys: []string{ReservedKey("gone", "0.1")}})
|
||||
if _, err := store.Create(ctx, IndexDigestKey("sha256:gone"), stale); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n, err := r.Reindex(ctx); err != nil || n != 1 {
|
||||
t.Fatalf("the reindex read %d reservations (%v), want 1", n, err)
|
||||
}
|
||||
if _, _, found, _ := store.Get(ctx, IndexDigestKey("sha256:gone")); found {
|
||||
t.Error("the index entry of a reservation that is gone stands after the reindex")
|
||||
}
|
||||
if keys, _ := store.KeysOf(ctx, AllFilter); len(keys) != 1 {
|
||||
t.Errorf("the reservations list %v: the index keys are listed among them", keys)
|
||||
}
|
||||
store.lists.Store(0)
|
||||
store.reads.Store(0)
|
||||
marked, _, err := r.Published(ctx, "old-a", "0.2.4", "sha256:old")
|
||||
if err != nil || len(marked) != 1 || store.lists.Load() != 0 || store.reads.Load() != 1 {
|
||||
t.Errorf("after the reindex the published marked %v (%v), listed %d, read %d", marked, err, store.lists.Load(), store.reads.Load())
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -33,8 +34,10 @@ const (
|
||||
QueueFull = "queue full"
|
||||
// NotStoredWords opens what a refused artifact is said as.
|
||||
NotStoredWords = "not stored: "
|
||||
// VerbPublish is the registry seats' verb a publish again sends.
|
||||
VerbPublish = "publish"
|
||||
// VerbPublish is the registry seats' verb a publish again sends, VerbVersions the one whose record says what a seat holds
|
||||
// (H-A-272).
|
||||
VerbPublish = "publish"
|
||||
VerbVersions = "versions"
|
||||
)
|
||||
|
||||
// RepublishTransfer is a publish again's transfer name, the builder's own (the registry part's publish: the caller names its
|
||||
@@ -77,6 +80,9 @@ type QueuedPublish struct {
|
||||
Begun bool `json:"begun,omitempty"`
|
||||
}
|
||||
|
||||
// QueueFilter is the filter of every queued version: a queue key has three tokens, which AllFilter (two) never lists on the bus.
|
||||
const QueueFilter = "*.*.*"
|
||||
|
||||
// QueueKey is the key of holder's queued version of module.
|
||||
func QueueKey(holder, module, version string) string {
|
||||
return EscapeToken(holder) + "." + EscapeToken(module) + "." + EscapeToken(version)
|
||||
@@ -402,7 +408,7 @@ const PublishAgainDescription = "Publish reserved versions again (design 47 G se
|
||||
|
||||
// Resume takes up, at the builder's start, every queued version again, a begun one sent anew (§588).
|
||||
func (r *Republisher) Resume(ctx context.Context) error {
|
||||
keys, err := r.Queue.KeysOf(ctx, AllFilter)
|
||||
keys, err := r.Queue.KeysOf(ctx, QueueFilter)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the state %s cannot be listed: %w", PublishQueueState, err)
|
||||
}
|
||||
@@ -501,25 +507,11 @@ func (r *Republisher) send(ctx context.Context, q QueuedPublish) {
|
||||
}
|
||||
sort.Strings(refs)
|
||||
for _, ref := range refs {
|
||||
var art *meshcatalog.Artifact
|
||||
for i := range e.Artifacts {
|
||||
if e.Artifacts[i].Ref == ref {
|
||||
art = &e.Artifacts[i]
|
||||
}
|
||||
}
|
||||
seat, ok := "", false
|
||||
if art != nil {
|
||||
seat, ok = SeatOfArtifact(*art)
|
||||
}
|
||||
seat, name, kind, ok := publishTarget(e, q.Module, ref)
|
||||
if !ok {
|
||||
r.say(ctx, v, ref+": the catalog entry names no registry seat for it")
|
||||
return
|
||||
}
|
||||
name := NameOf(q.Module, strings.TrimPrefix(ref, q.Module+":"))
|
||||
if art.Package != nil && art.Package.Name != "" {
|
||||
name = art.Package.Name
|
||||
}
|
||||
kind := art.Kind
|
||||
bound, cancel := context.WithTimeout(ctx, UnreleasedBound) // a context's bound, no timer (ADR 0437 §8)
|
||||
err := r.Send(bound, Publish{Seat: seat, Kind: kind, Name: name, Version: v.Version, Digest: q.Artifacts[ref], Fingerprint: v.Fingerprint,
|
||||
Commit: v.Commit, Transfer: RepublishTransfer(q.Module, v.Version, strings.TrimPrefix(ref, q.Module+":"))})
|
||||
@@ -543,6 +535,28 @@ func (r *Republisher) send(ctx context.Context, q QueuedPublish) {
|
||||
}
|
||||
}
|
||||
|
||||
// publishTarget is the registry seat, the name and the kind module's artifact ref is published as, from its catalog entry; ok is
|
||||
// false when the entry names no registry seat for it.
|
||||
func publishTarget(e meshcatalog.CatalogEntry, module, ref string) (seat, name, kind string, ok bool) {
|
||||
var art *meshcatalog.Artifact
|
||||
for i := range e.Artifacts {
|
||||
if e.Artifacts[i].Ref == ref {
|
||||
art = &e.Artifacts[i]
|
||||
}
|
||||
}
|
||||
if art == nil {
|
||||
return "", "", "", false
|
||||
}
|
||||
if seat, ok = SeatOfArtifact(*art); !ok {
|
||||
return "", "", "", false
|
||||
}
|
||||
name = NameOf(module, strings.TrimPrefix(ref, module+":"))
|
||||
if art.Package != nil && art.Package.Name != "" {
|
||||
name = art.Package.Name
|
||||
}
|
||||
return seat, name, art.Kind, true
|
||||
}
|
||||
|
||||
func (r *Republisher) say(ctx context.Context, v ReservedVersion, why string) {
|
||||
r.logf("%s %s is not published again: %s", v.Module, v.Version, why)
|
||||
if r.Unreleased != nil {
|
||||
@@ -579,3 +593,105 @@ func (r *Reserver) MarkUnstored(ctx context.Context, module, version, artifact,
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
|
||||
// UnheardBound is how many messages the builder's durable on the events stream may hold pending and unacknowledged while a
|
||||
// publish is asked again (design 47 H H-A-269 (2), H-A-272): at or above it a published of the version may still be unheard, so
|
||||
// the try is skipped. A constant of the builder, no setting.
|
||||
const UnheardBound = 100
|
||||
|
||||
// HeldVersion is one version a registry seat's verb versions answers of a name, with its digest.
|
||||
type HeldVersion struct {
|
||||
Version, Digest string
|
||||
}
|
||||
|
||||
// HeldArtifact is an artifact of a reservation as a registry seat's record holds it at the reservation's version: its reference,
|
||||
// the seat and the name it is published as, the version and the digest the seat holds.
|
||||
type HeldArtifact struct {
|
||||
Ref, Seat, Name, Version, Digest string
|
||||
}
|
||||
|
||||
// Judged is what the registries' record says of a reserved version's missing artifacts (design 47 H H-A-272, H-A-274, H-A-275):
|
||||
// those held at the version with the built digest (Held), those held at the version with another digest (Elsewhere), and, when an
|
||||
// artifact is held at neither, the first one lacking (Lacks). All says every missing artifact is held with its built digest.
|
||||
type Judged struct {
|
||||
Held, Elsewhere []HeldArtifact
|
||||
Lacks string
|
||||
All bool
|
||||
}
|
||||
|
||||
// Held judges v's missing artifacts against the registries' record (design 47 H H-A-269 (2), H-A-272, H-A-274, H-A-275): each
|
||||
// artifact's registry seat and name as a publish again sends them, its record read through the seat's verb versions (versionsOf).
|
||||
// A version with no missing artifact is All.
|
||||
func (r *Republisher) Held(ctx context.Context, v ReservedVersion, versionsOf func(ctx context.Context, seat, name string) ([]HeldVersion, error)) (Judged, error) {
|
||||
missing := v.Missing()
|
||||
if len(missing) == 0 {
|
||||
return Judged{All: true}, nil
|
||||
}
|
||||
e, err := r.Entry(ctx, v.Fingerprint)
|
||||
if err != nil {
|
||||
return Judged{}, fmt.Errorf("the catalog entry %s of %s %s cannot be read: %w", v.Fingerprint, v.Module, v.Version, err)
|
||||
}
|
||||
var out Judged
|
||||
for _, ref := range missing {
|
||||
seat, name, _, ok := publishTarget(e, v.Module, ref)
|
||||
if !ok {
|
||||
if out.Lacks == "" {
|
||||
out.Lacks = ref + ": the catalog entry names no registry seat for it"
|
||||
}
|
||||
continue
|
||||
}
|
||||
held, err := versionsOf(ctx, seat, name)
|
||||
if err != nil {
|
||||
return Judged{}, fmt.Errorf("%s's versions of %s cannot be read: %w", seat, name, err)
|
||||
}
|
||||
at, found := HeldArtifact{}, false
|
||||
for _, h := range held {
|
||||
if strings.TrimPrefix(h.Version, "v") != v.Version {
|
||||
continue
|
||||
}
|
||||
at, found = HeldArtifact{Ref: ref, Seat: seat, Name: name, Version: h.Version, Digest: h.Digest}, true
|
||||
if h.Digest == v.Artifacts[ref] {
|
||||
break
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case !found:
|
||||
if out.Lacks == "" {
|
||||
out.Lacks = fmt.Sprintf("%s holds no %s %s", seat, name, v.Version)
|
||||
}
|
||||
case at.Digest == v.Artifacts[ref]:
|
||||
out.Held = append(out.Held, at)
|
||||
default:
|
||||
out.Elsewhere = append(out.Elsewhere, at)
|
||||
}
|
||||
}
|
||||
out.All = len(out.Held) == len(missing)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// HeldNumbers are the versions (MAJOR.MINOR.PATCH) a registry seat's record holds of any of module's artifacts refs, as the catalog
|
||||
// entry of fingerprint names their seats and names (design 47 H H-A-275): a number one holds is never reserved. An artifact the
|
||||
// entry names no registry seat for is read nowhere.
|
||||
func (r *Republisher) HeldNumbers(ctx context.Context, module, fingerprint string, refs []string, versionsOf func(ctx context.Context, seat, name string) ([]HeldVersion, error)) ([]string, error) {
|
||||
e, err := r.Entry(ctx, fingerprint)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the catalog entry %s of %s cannot be read: %w", fingerprint, module, err)
|
||||
}
|
||||
var out []string
|
||||
for _, ref := range refs {
|
||||
seat, name, _, ok := publishTarget(e, module, ref)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
held, err := versionsOf(ctx, seat, name)
|
||||
if err != nil {
|
||||
return out, fmt.Errorf("%s's versions of %s cannot be read: %w", seat, name, err)
|
||||
}
|
||||
for _, h := range held {
|
||||
if v := strings.TrimPrefix(h.Version, "v"); v != "" && !slices.Contains(out, v) {
|
||||
out = append(out, v)
|
||||
}
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
@@ -196,3 +196,42 @@ func TestRepublish(t *testing.T) {
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestTheRegistriesRecordSaysWhetherAVersionIsHeld: design 47 H H-A-272, H-A-274, H-A-275 — a missing artifact its registry seat
|
||||
// holds at the version (the Go form too) with the built digest is held; at the version with another digest it is held elsewhere;
|
||||
// a version the seat lacks is said by name. The numbers a registry holds of the module's artifacts are what reserving skips.
|
||||
func TestTheRegistriesRecordSaysWhetherAVersionIsHeld(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
entry := meshcatalog.CatalogEntry{Artifacts: []meshcatalog.Artifact{{Ref: "m:lib", Kind: "package", Language: "go", Package: &meshcatalog.ArtifactPackage{Ecosystem: "go", Name: "git.example/m"}}}}
|
||||
r := &Republisher{Entry: func(context.Context, string) (meshcatalog.CatalogEntry, error) { return entry, nil }}
|
||||
v := ReservedVersion{Module: "m", Version: "0.1.37", Fingerprint: "fp", Artifacts: map[string]string{"m:lib": "sha256:a"}}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
holds []HeldVersion
|
||||
all, elsewhere, lacks bool
|
||||
}{
|
||||
{name: "held in the Go form with the built digest", holds: []HeldVersion{{Version: "v0.1.36", Digest: "sha256:z"}, {Version: "v0.1.37", Digest: "sha256:a"}}, all: true},
|
||||
{name: "another digest at the version", holds: []HeldVersion{{Version: "0.1.37", Digest: "sha256:b"}}, elsewhere: true},
|
||||
{name: "the version lacked", holds: []HeldVersion{{Version: "0.1.36", Digest: "sha256:a"}}, lacks: true},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
var seat, name string
|
||||
versionsOf := func(_ context.Context, s, n string) ([]HeldVersion, error) { seat, name = s, n; return c.holds, nil }
|
||||
j, err := r.Held(ctx, v, versionsOf)
|
||||
if err != nil || j.All != c.all || (len(j.Elsewhere) == 1) != c.elsewhere || (j.Lacks != "") != c.lacks || seat != SeatGoPackages || name != "git.example/m" {
|
||||
t.Fatalf("judged %+v (%v) asking %s for %s, want all %v, elsewhere %v, lacks %v asking %s for git.example/m", j, err, seat, name, c.all, c.elsewhere, c.lacks, SeatGoPackages)
|
||||
}
|
||||
if c.lacks && !strings.Contains(j.Lacks, "0.1.37") {
|
||||
t.Errorf("what the registry lacks is said as %q", j.Lacks)
|
||||
}
|
||||
nums, err := r.HeldNumbers(ctx, "m", "fp", []string{"m:lib"}, versionsOf)
|
||||
if err != nil || len(nums) != len(c.holds) {
|
||||
t.Errorf("the held numbers are %v (%v), want one per version the seat holds", nums, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
v.Published = map[string]bool{"m:lib": true}
|
||||
if j, err := r.Held(ctx, v, nil); err != nil || !j.All {
|
||||
t.Errorf("a version with no missing artifact is judged %+v (%v), want all held without a read", j, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,6 +5,8 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -35,6 +37,9 @@ const (
|
||||
UnreleasedBound = 10 * time.Minute
|
||||
// VersionUnreleased is that finding.
|
||||
VersionUnreleased = "version-unreleased"
|
||||
// VersionHeldElsewhere is the finding of a number a registry seat already holds, skipped at reserving or moved from (design 47
|
||||
// H H-A-275), scope module.
|
||||
VersionHeldElsewhere = "version-held-elsewhere"
|
||||
)
|
||||
|
||||
// ReservedVersion is the value of a reservation key.
|
||||
@@ -74,6 +79,9 @@ type ReservedVersion struct {
|
||||
// Kept are the registry seats' published heard while the build had not yet recorded every digest and whose digest marked
|
||||
// no artifact then: kept as heard, they mark their artifact when the build records its digests (design 47 G A-415).
|
||||
Kept []KeptPublished `json:"kept-published,omitempty"`
|
||||
// Withdrawn are the versions of the series a take-over by another fingerprint withdrew, each with the fingerprint whose
|
||||
// build it was (design 47 H H-A-225): a late publish of one is refused as withdrawn. Kept while the series' key stands.
|
||||
Withdrawn map[string]string `json:"withdrawn,omitempty"`
|
||||
}
|
||||
|
||||
// KeptPublished is a registry seat's published kept on a reservation, as heard (design 47 G A-415).
|
||||
@@ -130,6 +138,50 @@ type Reserver struct {
|
||||
// DefaultReleasedKept.
|
||||
ReleasedKept int
|
||||
Now func() time.Time
|
||||
// Logf says an index write that failed (H-A-269 (1)): the reservation stands, and Reindex writes its index at the next
|
||||
// start. Nil says nothing.
|
||||
Logf func(string, ...any)
|
||||
// HeldNumbers are the versions a registry seat's record holds of any of module's artifacts (design 47 H H-A-275), read before a
|
||||
// number is reserved: a number one holds is skipped. Nil reads none; a read that fails is said and the reserve proceeds.
|
||||
HeldNumbers func(ctx context.Context, module, fingerprint string, refs []string) ([]string, error)
|
||||
// HeldElsewhere is told the numbers of module a reserve skipped because a registry holds them (H-A-275); nil is told none.
|
||||
HeldElsewhere func(ctx context.Context, module string, skipped []string)
|
||||
}
|
||||
|
||||
// heldNumbers are the patches of series a registry holds of module's artifacts (H-A-275); a read that fails is said, and none.
|
||||
func (r *Reserver) heldNumbers(ctx context.Context, module, series, fingerprint string, refs []string) map[int]bool {
|
||||
if r.HeldNumbers == nil {
|
||||
return nil
|
||||
}
|
||||
vs, err := r.HeldNumbers(ctx, module, fingerprint, refs)
|
||||
if err != nil {
|
||||
r.said(fmt.Errorf("%s: the registries' versions are not read before reserving, so a number they hold is not skipped: %w", module, err))
|
||||
}
|
||||
out := map[int]bool{}
|
||||
for _, v := range vs {
|
||||
if s, p, err := SplitVersion(v); err == nil && s == series {
|
||||
out[p] = true
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// nextFree is the first patch after after that no registry holds, and the held numbers it skipped (H-A-275).
|
||||
func nextFree(series string, after int, held map[int]bool) (int, []string) {
|
||||
var skipped []string
|
||||
p := after + 1
|
||||
for held[p] {
|
||||
skipped = append(skipped, fmt.Sprintf("%s.%d", series, p))
|
||||
p++
|
||||
}
|
||||
return p, skipped
|
||||
}
|
||||
|
||||
// said is told the outcome of a reindex: a failure is said, the reservation's own write stands.
|
||||
func (r *Reserver) said(err error) {
|
||||
if err != nil && r.Logf != nil {
|
||||
r.Logf("%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Reserver) now() time.Time {
|
||||
@@ -154,7 +206,10 @@ func SplitVersion(version string) (series string, patch int, err error) {
|
||||
|
||||
// Reserve reserves the next version of module's series for a build that starts, for artifacts (their names), and answers
|
||||
// it. The next patch is the one after the higher of the highest released and the one reserved, 0 when there is neither. A
|
||||
// conflicting write (another build reserved first) is read again and the next taken.
|
||||
// conflicting write (another build reserved first) is read again and the next taken. A reservation standing for the same
|
||||
// fingerprint whose version is not released and is above the catalog's highest is kept and answered as it stands, a build again or a re-run after a restart
|
||||
// alike (design 47 H H-A-225, which amends G A-490b): the run already building carries that number. A reservation another
|
||||
// fingerprint takes over is withdrawn: its version is kept in Withdrawn, and Took is told so the older build is ended.
|
||||
func (r *Reserver) Reserve(ctx context.Context, module, series string, artifacts []string, src Source) (ReservedVersion, error) {
|
||||
if r.Filled != nil && !r.Filled() {
|
||||
return ReservedVersion{}, ErrNotFilled
|
||||
@@ -170,18 +225,26 @@ func (r *Reserver) Reserve(ctx context.Context, module, series string, artifacts
|
||||
}
|
||||
}
|
||||
key := ReservedKey(module, series)
|
||||
held := r.heldNumbers(ctx, module, series, src.Fingerprint, artifacts)
|
||||
for {
|
||||
cur, rev, found, err := r.get(ctx, key)
|
||||
if err != nil {
|
||||
return ReservedVersion{}, err
|
||||
}
|
||||
withdrawn := found && hasKey(cur.Withdrawn, cur.Version) // a number a registry holds elsewhere (H-A-279)
|
||||
if found && !withdrawn && cur.Fingerprint == src.Fingerprint && cur.Fingerprint != "" && !slices.Contains(released, cur.Version) {
|
||||
if _, p, err := SplitVersion(cur.Version); err == nil && p > high {
|
||||
return cur, nil // kept until released (H-A-225); one at or below the catalog's highest can never be released
|
||||
}
|
||||
}
|
||||
next := high
|
||||
if found && !(src.Rebuild && cur.Fingerprint == src.Fingerprint) {
|
||||
if found && (withdrawn || !(src.Rebuild && cur.Fingerprint == src.Fingerprint)) { // a withdrawn number is never taken again (H-A-226)
|
||||
if _, p, err := SplitVersion(cur.Version); err == nil && p > next {
|
||||
next = p
|
||||
}
|
||||
}
|
||||
out := ReservedVersion{V: 1, Module: module, Series: series, Version: fmt.Sprintf("%s.%d", series, next+1), Artifacts: map[string]string{}, At: r.now(),
|
||||
patch, skipped := nextFree(series, next, held)
|
||||
out := ReservedVersion{V: 1, Module: module, Series: series, Version: fmt.Sprintf("%s.%d", series, patch), Artifacts: map[string]string{}, At: r.now(),
|
||||
Repository: src.Repository, Commit: src.Commit, Fingerprint: src.Fingerprint, Branch: src.Branch, Source: src.Revision, Revision: src.CatalogRevision}
|
||||
if src.GoPackage {
|
||||
out.Tag = GoTagOf(out.Version)
|
||||
@@ -189,6 +252,23 @@ func (r *Reserver) Reserve(ctx context.Context, module, series string, artifacts
|
||||
for _, a := range artifacts {
|
||||
out.Artifacts[a] = ""
|
||||
}
|
||||
if found {
|
||||
out.Withdrawn = maps.Clone(cur.Withdrawn)
|
||||
if cur.Fingerprint != src.Fingerprint {
|
||||
if out.Withdrawn == nil {
|
||||
out.Withdrawn = map[string]string{}
|
||||
}
|
||||
out.Withdrawn[cur.Version] = cur.Fingerprint // a dropped number stays a gap (H-A-226)
|
||||
}
|
||||
// One at or below the catalog's highest released is refused by the plain rule anyway: the map never grows without end.
|
||||
maps.DeleteFunc(out.Withdrawn, func(v, _ string) bool {
|
||||
_, p, err := SplitVersion(v)
|
||||
return err != nil || p <= high
|
||||
})
|
||||
if len(out.Withdrawn) == 0 {
|
||||
out.Withdrawn = nil
|
||||
}
|
||||
}
|
||||
raw, _ := json.Marshal(out)
|
||||
if found {
|
||||
_, err = r.Store.Update(ctx, key, raw, rev)
|
||||
@@ -201,6 +281,10 @@ func (r *Reserver) Reserve(ctx context.Context, module, series string, artifacts
|
||||
if err != nil {
|
||||
return ReservedVersion{}, fmt.Errorf("%s cannot be reserved: %w", key, err)
|
||||
}
|
||||
r.said(r.reindex(ctx, key, cur, out)) // cur is zero when there was none
|
||||
if len(skipped) > 0 && r.HeldElsewhere != nil {
|
||||
r.HeldElsewhere(ctx, module, skipped)
|
||||
}
|
||||
if found && r.Took != nil {
|
||||
r.Took(ctx, cur, out)
|
||||
}
|
||||
@@ -220,6 +304,38 @@ func (r *Reserver) get(ctx context.Context, key string) (ReservedVersion, uint64
|
||||
return v, rev, true, nil
|
||||
}
|
||||
|
||||
// Withdraw withdraws module's reservation of version for the build of its fingerprint, whose number a registry holds at another
|
||||
// digest (design 47 H H-A-275, H-A-279): the number is kept in Withdrawn and stays a gap (H-A-226), so the build again of that
|
||||
// fingerprint reserves the next number no registry holds, with fresh digests. It answers the reservation as withdrawn, and false
|
||||
// when none stands at version.
|
||||
func (r *Reserver) Withdraw(ctx context.Context, module, version string) (ReservedVersion, bool, error) {
|
||||
series, _, err := SplitVersion(version)
|
||||
if err != nil {
|
||||
return ReservedVersion{}, false, err
|
||||
}
|
||||
key := ReservedKey(module, series)
|
||||
for {
|
||||
cur, rev, found, err := r.get(ctx, key)
|
||||
if err != nil || !found || cur.Version != version {
|
||||
return cur, false, err
|
||||
}
|
||||
if hasKey(cur.Withdrawn, version) {
|
||||
return cur, true, nil
|
||||
}
|
||||
if cur.Withdrawn == nil {
|
||||
cur.Withdrawn = map[string]string{}
|
||||
}
|
||||
cur.Withdrawn[version] = cur.Fingerprint
|
||||
raw, _ := json.Marshal(cur)
|
||||
if _, err := r.Store.Update(ctx, key, raw, rev); errors.Is(err, ErrConflict) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
return cur, false, fmt.Errorf("%s cannot be withdrawn: %w", key, err)
|
||||
}
|
||||
return cur, true, nil
|
||||
}
|
||||
}
|
||||
|
||||
// Built records artifact's digest under the reserved version of module's series; once every artifact has one the build is
|
||||
// ended (BuiltAt). A reservation moved on to another version (a newer build took the next) is left alone and answers false.
|
||||
func (r *Reserver) Built(ctx context.Context, module, version, artifact, digest string, deps []meshbuilds.Dependency) (bool, error) {
|
||||
@@ -233,6 +349,8 @@ func (r *Reserver) Built(ctx context.Context, module, version, artifact, digest
|
||||
if err != nil || !found || cur.Version != version {
|
||||
return false, err
|
||||
}
|
||||
was := cur
|
||||
was.Artifacts = maps.Clone(cur.Artifacts)
|
||||
cur.Artifacts[artifact] = digest
|
||||
if len(deps) > 0 {
|
||||
if cur.Dependencies == nil {
|
||||
@@ -249,6 +367,7 @@ func (r *Reserver) Built(ctx context.Context, module, version, artifact, digest
|
||||
} else if err != nil {
|
||||
return false, fmt.Errorf("%s cannot be written: %w", key, err)
|
||||
}
|
||||
r.said(r.reindex(ctx, key, was, cur))
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
@@ -367,6 +486,7 @@ func (r *Reserver) Released(ctx context.Context, module, version string) ([]stri
|
||||
if err := r.Store.Delete(ctx, k); err != nil {
|
||||
return gone, fmt.Errorf("%s cannot be deleted: %w", k, err)
|
||||
}
|
||||
r.said(r.reindex(ctx, k, cur, ReservedVersion{}))
|
||||
gone = append(gone, k)
|
||||
}
|
||||
sort.Strings(gone)
|
||||
@@ -430,15 +550,39 @@ func (r *Reserver) All(ctx context.Context) ([]ReservedVersion, error) {
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// TakenOver is the reservation of module's series whose withdrawn numbers hold one of the build of fingerprint (design 47 H
|
||||
// H-A-226), and whether there is one: the series moved to that reservation's fingerprint and version (H-A-244).
|
||||
func (r *Reserver) TakenOver(ctx context.Context, module, fingerprint string) (ReservedVersion, bool, error) {
|
||||
all, err := r.All(ctx)
|
||||
if err != nil {
|
||||
return ReservedVersion{}, false, err
|
||||
}
|
||||
for _, v := range all {
|
||||
if v.Module != module || v.Fingerprint == fingerprint {
|
||||
continue
|
||||
}
|
||||
for _, fp := range v.Withdrawn {
|
||||
if fp == fingerprint {
|
||||
return v, true, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
return ReservedVersion{}, false, nil
|
||||
}
|
||||
|
||||
// Drop deletes module's reservation of series on a person's word (builder_reserved_drop); the caller writes the hand-act
|
||||
// log. It answers whether there was one.
|
||||
func (r *Reserver) Drop(ctx context.Context, module, series string) (bool, error) {
|
||||
key := ReservedKey(module, series)
|
||||
_, _, found, err := r.get(ctx, key)
|
||||
cur, _, found, err := r.get(ctx, key)
|
||||
if err != nil || !found {
|
||||
return false, err
|
||||
}
|
||||
return true, r.Store.Delete(ctx, key)
|
||||
if err := r.Store.Delete(ctx, key); err != nil {
|
||||
return true, err
|
||||
}
|
||||
r.said(r.reindex(ctx, key, cur, ReservedVersion{}))
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// Marked is an artifact of a reservation a registry seat's published newly marked: the reservation as it stands then.
|
||||
@@ -461,29 +605,47 @@ func (m Marked) PackagePublished(digest string, at time.Time) meshbuilds.Package
|
||||
// whose built digest it names are marked published. It answers each artifact it newly marked, with the reservation as marked.
|
||||
// A published that marks no artifact of a reservation at version whose build has not recorded every digest is kept on that
|
||||
// reservation as heard (name, version, digest), and the keys it is kept on are answered too (design 47 G A-415): Kept marks
|
||||
// it when the build records its digests.
|
||||
// it when the build records its digests. The reservation is found through the index (H-A-269 (1), H-A-271): the reservations
|
||||
// holding the digest in one read, and only when none holds it at version, those standing at the version in one more; never a
|
||||
// scan of every reservation.
|
||||
func (r *Reserver) Published(ctx context.Context, name, version, digest string) ([]Marked, []string, error) {
|
||||
version = strings.TrimPrefix(version, "v")
|
||||
if version == "" || digest == "" {
|
||||
return nil, nil, nil
|
||||
}
|
||||
keys, err := r.Store.KeysOf(ctx, AllFilter)
|
||||
keys, err := r.indexed(ctx, IndexDigestKey(digest))
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("the reservations cannot be listed: %w", err)
|
||||
return nil, nil, fmt.Errorf("the index of digest %s cannot be read: %w", digest, err)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
marked, kept, matched, err := r.publishedOn(ctx, keys, name, version, digest, false)
|
||||
if err != nil || matched {
|
||||
return marked, kept, err
|
||||
}
|
||||
keys, err = r.indexed(ctx, IndexVersionKey(version))
|
||||
if err != nil {
|
||||
return marked, kept, fmt.Errorf("the index of version %s cannot be read: %w", version, err)
|
||||
}
|
||||
m, k, _, err := r.publishedOn(ctx, keys, name, version, digest, true)
|
||||
return append(marked, m...), append(kept, k...), err
|
||||
}
|
||||
|
||||
// publishedOn takes a published in on the reservations under keys: an artifact whose built digest it names is marked; with keep,
|
||||
// a reservation at version whose build has not recorded every digest and that it matches nothing of keeps it (A-415). It also
|
||||
// answers whether a reservation at version holds the digest.
|
||||
func (r *Reserver) publishedOn(ctx context.Context, keys []string, name, version, digest string, keep bool) ([]Marked, []string, bool, error) {
|
||||
var marked []Marked
|
||||
var kept []string
|
||||
anyMatched := false
|
||||
for _, k := range keys {
|
||||
again:
|
||||
cur, rev, found, err := r.get(ctx, k)
|
||||
if err != nil {
|
||||
return marked, kept, err
|
||||
return marked, kept, anyMatched, err
|
||||
}
|
||||
if !found || cur.Version != version {
|
||||
continue
|
||||
}
|
||||
changed, matched, keep := false, false, false
|
||||
changed, matched, keepIt := false, false, false
|
||||
var arts []string
|
||||
for a, d := range cur.Artifacts {
|
||||
if d != digest {
|
||||
@@ -498,10 +660,11 @@ func (r *Reserver) Published(ctx context.Context, name, version, digest string)
|
||||
arts = append(arts, a)
|
||||
}
|
||||
}
|
||||
anyMatched = anyMatched || matched
|
||||
sort.Strings(arts)
|
||||
if !matched && cur.BuiltAt.IsZero() && !cur.keeps(name, digest) {
|
||||
if keep && !matched && cur.BuiltAt.IsZero() && !cur.keeps(name, digest) {
|
||||
cur.Kept = append(cur.Kept, KeptPublished{Name: name, Version: version, Digest: digest})
|
||||
changed, keep = true, true
|
||||
changed, keepIt = true, true
|
||||
}
|
||||
if !changed {
|
||||
continue
|
||||
@@ -510,16 +673,16 @@ func (r *Reserver) Published(ctx context.Context, name, version, digest string)
|
||||
if _, err := r.Store.Update(ctx, k, raw, rev); errors.Is(err, ErrConflict) {
|
||||
goto again
|
||||
} else if err != nil {
|
||||
return marked, kept, fmt.Errorf("%s cannot be written: %w", k, err)
|
||||
return marked, kept, anyMatched, fmt.Errorf("%s cannot be written: %w", k, err)
|
||||
}
|
||||
for _, a := range arts {
|
||||
marked = append(marked, Marked{Reserved: cur, Artifact: a})
|
||||
}
|
||||
if keep {
|
||||
if keepIt {
|
||||
kept = append(kept, k)
|
||||
}
|
||||
}
|
||||
return marked, kept, nil
|
||||
return marked, kept, anyMatched, nil
|
||||
}
|
||||
|
||||
func (v ReservedVersion) keeps(name, digest string) bool {
|
||||
@@ -711,11 +874,24 @@ func AnswerReservedByName(ctx context.Context, r Reader, carriers func(context.C
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
if series, _, err := SplitVersion(version); err == nil {
|
||||
if cur, _, found, err := rs.get(ctx, ReservedKey(module, series)); err == nil && found && hasKey(cur.Withdrawn, version) {
|
||||
// A late publish of a build a take-over withdrew (H-A-225): refused as what it is.
|
||||
return meshbuilds.ReservedAnswer{Error: fmt.Sprintf("%s (%s) version %s was withdrawn: the build of %s it was reserved for was taken over by %s, which reserves %s",
|
||||
name, module, version, cur.Withdrawn[version], cur.Fingerprint, cur.Version)}, nil
|
||||
}
|
||||
}
|
||||
return meshbuilds.ReservedAnswer{Error: fmt.Sprintf("%s (%s) has no version %s reserved or released", name, module, version)}, nil
|
||||
}
|
||||
return meshbuilds.ReservedAnswer{Error: fmt.Sprintf("%s (%s) has no reserved version", name, module)}, nil
|
||||
}
|
||||
|
||||
// hasKey says whether m holds k.
|
||||
func hasKey(m map[string]string, k string) bool {
|
||||
_, ok := m[k]
|
||||
return ok
|
||||
}
|
||||
|
||||
// NoEntryCarries opens the answer of reserved given a name no entry carries.
|
||||
const NoEntryCarries = "no entry carries the name "
|
||||
|
||||
|
||||
@@ -221,18 +221,53 @@ func TestAReleaseIsRecordedByFingerprintBeforeItsReservationGoes(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestARebuildReservesAboveTheCatalogsHighestWhateverItsStandingNumberWas: design 47 G A-490b — a build again of a done key replaces
|
||||
// its fingerprint's standing reservation (and tells Took) and numbers above the catalog's highest alone, whether the standing number
|
||||
// is below it or above it; another fingerprint's standing reservation stays a floor.
|
||||
// TestTheWithdrawnVersionsAreDroppedOnceReleasedPast: design 47 H H-A-225, H-A-226, H-A-244 — a dropped number stays a gap, kept in
|
||||
// the series key's Withdrawn while it is above the catalog's highest released; at each write those at or below it go, so the
|
||||
// map never grows without end.
|
||||
func TestTheWithdrawnVersionsAreDroppedOnceReleasedPast(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := &reserveMem{mem: newMem()}
|
||||
highest := "0.1.3"
|
||||
r := &Reserver{Store: store, Filled: func() bool { return true },
|
||||
ReleasedOf: func(context.Context, string) ([]string, error) { return []string{highest}, nil }}
|
||||
for _, fp := range []string{"src2:a", "src2:b", "src2:c"} {
|
||||
if _, err := r.Reserve(ctx, "m", "0.1", []string{"m:a"}, Source{Fingerprint: fp}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
cur, _, _, _ := r.get(ctx, ReservedKey("m", "0.1"))
|
||||
if cur.Version != "0.1.6" || len(cur.Withdrawn) != 2 || cur.Withdrawn["0.1.4"] != "src2:a" || cur.Withdrawn["0.1.5"] != "src2:b" {
|
||||
t.Fatalf("reservation %+v; want 0.1.6 with 0.1.4 and 0.1.5 withdrawn", cur)
|
||||
}
|
||||
// H-A-244: the build whose number was withdrawn finds the reservation its series moved to; the one standing finds none.
|
||||
if by, took, err := r.TakenOver(ctx, "m", "src2:a"); err != nil || !took || by.Fingerprint != "src2:c" || by.Version != "0.1.6" {
|
||||
t.Errorf("taken over from src2:a by %+v (%v, %v); want src2:c 0.1.6", by, took, err)
|
||||
}
|
||||
if _, took, err := r.TakenOver(ctx, "m", "src2:c"); err != nil || took {
|
||||
t.Errorf("the standing build src2:c is said taken over (%v, %v)", took, err)
|
||||
}
|
||||
highest = "0.1.5" // the catalog released past both withdrawn numbers but the newest
|
||||
got, err := r.Reserve(ctx, "m", "0.1", []string{"m:a"}, Source{Fingerprint: "src2:d"})
|
||||
if err != nil || got.Version != "0.1.7" || len(got.Withdrawn) != 1 || got.Withdrawn["0.1.6"] != "src2:c" {
|
||||
t.Errorf("reserved %+v (%v); want 0.1.7 keeping only 0.1.6 withdrawn", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestARebuildReservesAboveTheCatalogsHighestWhateverItsStandingNumberWas: design 47 G A-490b as H-A-225 amends it — a build again
|
||||
// of a done key, or a re-run after a restart, keeps its fingerprint's standing reservation while it is unreleased and above the
|
||||
// catalog's highest; one at or below it is replaced (Took told) by the next above the catalog's highest alone; another
|
||||
// fingerprint's standing reservation stays a floor and is withdrawn.
|
||||
func TestARebuildReservesAboveTheCatalogsHighestWhateverItsStandingNumberWas(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
for _, c := range []struct {
|
||||
name, standing, standingFp string
|
||||
want string
|
||||
rebuild, kept bool
|
||||
}{
|
||||
{name: "standing below the catalog's highest", standing: "0.1.0", standingFp: "fp", want: "0.1.17"},
|
||||
{name: "standing above the catalog's highest", standing: "0.1.30", standingFp: "fp", want: "0.1.17"},
|
||||
{name: "another fingerprint's standing number stays a floor", standing: "0.1.30", standingFp: "other", want: "0.1.31"},
|
||||
{name: "standing below the catalog's highest", standing: "0.1.0", standingFp: "fp", want: "0.1.17", rebuild: true},
|
||||
{name: "H-A-225: standing above the catalog's highest is kept by a build again", standing: "0.1.30", standingFp: "fp", want: "0.1.30", rebuild: true, kept: true},
|
||||
{name: "H-A-225: a re-run after a restart keeps its standing number", standing: "0.1.111", standingFp: "fp", want: "0.1.111", kept: true},
|
||||
{name: "another fingerprint's standing number stays a floor", standing: "0.1.30", standingFp: "other", want: "0.1.31", rebuild: true},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
store := &reserveMem{mem: newMem()}
|
||||
@@ -246,10 +281,68 @@ func TestARebuildReservesAboveTheCatalogsHighestWhateverItsStandingNumberWas(t *
|
||||
r.Took = func(_ context.Context, older, newer ReservedVersion) {
|
||||
took = append(took, older.Version+">"+newer.Version)
|
||||
}
|
||||
got, err := r.Reserve(ctx, "m", "0.1", []string{"m:a"}, Source{Fingerprint: "fp", Rebuild: true})
|
||||
got, err := r.Reserve(ctx, "m", "0.1", []string{"m:a"}, Source{Fingerprint: "fp", Rebuild: c.rebuild})
|
||||
if c.kept {
|
||||
if err != nil || got.Version != c.want || len(took) != 0 || got.Artifacts["m:a"] != "d" {
|
||||
t.Fatalf("reserved %+v (%v), took %v; want the standing %s kept as it stands", got, err, took, c.want)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil || got.Version != c.want || len(took) != 1 || took[0] != c.standing+">"+c.want {
|
||||
t.Fatalf("reserved %q (%v), took %v; want %s replacing %s", got.Version, err, took, c.want, c.standing)
|
||||
}
|
||||
if w := got.Withdrawn[c.standing]; (c.standingFp != "fp") != (w == c.standingFp) {
|
||||
t.Errorf("withdrawn %v; want %s withdrawn only when another fingerprint stood", got.Withdrawn, c.standing)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestANumberARegistryHoldsIsNeverReserved: design 47 H H-A-275 (the rule of H-A-245 for a registry's record) — a reserve skips
|
||||
// every number a registry seat holds of the module's artifacts and says the skipped ones; a read that fails lets it proceed; a
|
||||
// done build whose number a registry holds at another digest has it withdrawn as a gap (H-A-226), and its build again reserves the
|
||||
// next free number with fresh digests (H-A-279), the index following it.
|
||||
func TestANumberARegistryHoldsIsNeverReserved(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store := &reserveMem{mem: newMem()}
|
||||
held := []string{"0.1.37", "0.1.38", "0.2.0"}
|
||||
var readErr error
|
||||
var skipped []string
|
||||
r := &Reserver{Store: store, ReleasedOf: func(context.Context, string) ([]string, error) { return []string{"0.1.36"}, nil },
|
||||
HeldNumbers: func(_ context.Context, module, fp string, refs []string) ([]string, error) {
|
||||
if module != "node-tools" || fp != "fp1" || len(refs) != 1 {
|
||||
t.Errorf("the registries are read for %s %s %v", module, fp, refs)
|
||||
}
|
||||
return held, readErr
|
||||
},
|
||||
HeldElsewhere: func(_ context.Context, _ string, s []string) { skipped = append(skipped, s...) }}
|
||||
v, err := r.Reserve(ctx, "node-tools", "0.1", []string{"node-tools:program"}, Source{Fingerprint: "fp1", GoPackage: true})
|
||||
if err != nil || v.Version != "0.1.39" || len(skipped) != 2 {
|
||||
t.Fatalf("reserved %q (%v), skipped %v: want 0.1.39 past the two numbers the registry holds", v.Version, err, skipped)
|
||||
}
|
||||
// The build ends; the registry holds 0.1.39 at another digest meanwhile: the number is withdrawn, and the build again of the same
|
||||
// fingerprint reserves the next number no registry holds, with no digest yet (H-A-279).
|
||||
if _, err := r.Built(ctx, "node-tools", "0.1.39", "node-tools:program", "sha256:7c9b10", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if w, ok, err := r.Withdraw(ctx, "node-tools", "0.1.39"); err != nil || !ok || w.Withdrawn["0.1.39"] != "fp1" {
|
||||
t.Fatalf("withdrew %+v (%v %v), want 0.1.39 withdrawn for fp1", w, ok, err)
|
||||
}
|
||||
held = []string{"0.1.37", "0.1.38", "0.1.39", "0.1.40", "0.2.0"}
|
||||
// The build again of a done key reserves as a rebuild (A-490b): a withdrawn number is never its floor's to reuse.
|
||||
again, err := r.Reserve(ctx, "node-tools", "0.1", []string{"node-tools:program"}, Source{Fingerprint: "fp1", GoPackage: true, Rebuild: true})
|
||||
if err != nil || again.Version != "0.1.41" || again.Artifacts["node-tools:program"] != "" || again.Withdrawn["0.1.39"] != "fp1" || again.Tag != GoTagOf("0.1.41") {
|
||||
t.Fatalf("the build again reserved %+v (%v), want 0.1.41 with no digest and 0.1.39 kept withdrawn", again, err)
|
||||
}
|
||||
if at, _ := r.AtVersion(ctx, "0.1.39"); len(at) != 0 {
|
||||
t.Errorf("the index by version still names the withdrawn number: %v", at)
|
||||
}
|
||||
if _, ok, _ := r.Withdraw(ctx, "node-tools", "0.1.39"); ok {
|
||||
t.Error("a number no reservation stands at any more was withdrawn")
|
||||
}
|
||||
// A registry that cannot be read lets the reserve proceed.
|
||||
readErr, skipped = errors.New("registry down"), nil
|
||||
if v, err := r.Reserve(ctx, "node-tools", "0.2", []string{"node-tools:program"}, Source{Fingerprint: "fp1"}); err != nil || v.Version != "0.2.1" {
|
||||
t.Errorf("with the registry down reserved %q (%v), want 0.2.1 (what it answered still skipped)", v.Version, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -67,6 +67,20 @@ func TestReservedByNameAnswersFromTheViewAndTheCatalog(t *testing.T) {
|
||||
t.Errorf("%s = %+v, want error naming %q alone", name, got, c.want)
|
||||
}
|
||||
}
|
||||
// H-A-225: another fingerprint taking the series over withdraws the older's version; a late publish of it is refused as
|
||||
// withdrawn, naming the fingerprint that took it over, never as a version that was never reserved.
|
||||
if _, err := r.Reserve(ctx, "mesh-sdk", "0.1", []string{"mesh-sdk:ts"}, Source{Fingerprint: "src2:a"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := r.Reserve(ctx, "mesh-sdk", "0.1", []string{"mesh-sdk:ts"}, Source{Fingerprint: "src2:b"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := ask("@novox/mesh-sdk", "0.1.5"); !strings.Contains(got.Error, "version 0.1.5 was withdrawn: the build of src2:a it was reserved for was taken over by src2:b, which reserves 0.1.6") {
|
||||
t.Errorf("a late publish of a withdrawn version = %+v, want refused as withdrawn", got)
|
||||
}
|
||||
if got := ask("@novox/mesh-sdk", "9.9.9"); !strings.Contains(got.Error, "reserved or released") {
|
||||
t.Errorf("a version never reserved = %+v, want refused as neither reserved nor released", got)
|
||||
}
|
||||
if _, err := AnswerReservedByName(ctx, rd, Carriers(packagesMem(nil)), released, "x", ""); err == nil {
|
||||
t.Error("an unreadable view was answered as if no entry carried the name")
|
||||
}
|
||||
|
||||
@@ -4,12 +4,28 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// AllFilter is the key filter of every version of every name: two tokens.
|
||||
const AllFilter = "*.*"
|
||||
|
||||
// FilterMatches says whether key is under filter as the bus reads a key filter: token by token, a `*` any one token, so AllFilter
|
||||
// lists the two-token keys alone and never the index's three-token keys (H-A-269 (1)).
|
||||
func FilterMatches(filter, key string) bool {
|
||||
ft, kt := strings.Split(filter, "."), strings.Split(key, ".")
|
||||
if len(ft) != len(kt) {
|
||||
return false
|
||||
}
|
||||
for i := range ft {
|
||||
if ft[i] != "*" && ft[i] != kt[i] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// ErrConflict is what a Store answers when a Create finds the key or an Update finds another revision.
|
||||
var ErrConflict = errors.New("the key exists or changed")
|
||||
|
||||
|
||||
@@ -45,13 +45,9 @@ func (s readStore) KeysOf(_ context.Context, filter string) ([]string, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if filter == AllFilter {
|
||||
return all, nil
|
||||
}
|
||||
prefix := strings.TrimSuffix(filter, "*")
|
||||
var out []string
|
||||
for _, k := range all {
|
||||
if strings.HasPrefix(k, prefix) && !strings.Contains(k[len(prefix):], ".") {
|
||||
if FilterMatches(filter, k) {
|
||||
out = append(out, k)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -66,21 +66,9 @@ func (m *mem) KeysOf(_ context.Context, filter string) ([]string, error) {
|
||||
if m.down != nil {
|
||||
return nil, m.down
|
||||
}
|
||||
prefix := strings.TrimSuffix(filter, "*")
|
||||
var out []string
|
||||
for k := range m.vals {
|
||||
if filter == AllFilter {
|
||||
out = append(out, k)
|
||||
continue
|
||||
}
|
||||
if strings.Count(filter, ".") == 2 { // a three-token filter, `a.*.*` as the bus reads it
|
||||
ft, kt := strings.Split(filter, "."), strings.Split(k, ".")
|
||||
if len(kt) == 3 && kt[0] == ft[0] {
|
||||
out = append(out, k)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(k, prefix) && !strings.Contains(k[len(prefix):], ".") {
|
||||
if FilterMatches(filter, k) { // token by token, as the bus reads a filter
|
||||
out = append(out, k)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,6 +17,7 @@ import (
|
||||
"log"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"syscall"
|
||||
"time"
|
||||
@@ -289,7 +290,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// The indexer's findings are the seat mesh-conditions' generated finding, written through mesh-sdk's held finding writer
|
||||
// (design 47 G §689, A-29): it holds what is written before the state is bound, and starts once the state is.
|
||||
writer, err := views.NewHeldWriter(views.WriterConfig[mcseat.Finding, mcseat.FindingCleared, mcseat.FindingsStated, mcseat.FindingsAskedEvent]{
|
||||
Module: cred.Module, State: calls.State(index.FindingsState), Node: cred.Node, Asked: mcseat.SubjectFindingsAsked, JS: js, Conn: conn, Logf: logger.Printf})
|
||||
Module: cred.Module, State: calls.State(index.FindingsState), Node: cred.Node, Asked: mcseat.SubjectFindingsAsked, JS: js, Conn: conn, Logf: logger.Printf,
|
||||
// its own kinds only (G §580, H-A-242, H-A-247): the tool runner's writer states its kinds over the same state
|
||||
Kinds: index.FindingKinds, Retired: index.RetiredFindingKinds})
|
||||
if err != nil {
|
||||
return fmt.Errorf("the indexer's finding writer cannot be made: %w", err)
|
||||
}
|
||||
@@ -378,7 +381,16 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
logger.Printf("start: %s", name)
|
||||
return func() { logger.Printf("start: %s done in %s", name, time.Since(began).Round(time.Millisecond)) }
|
||||
}
|
||||
done := step("reading the build log")
|
||||
// A module of the current trunk graph with no registration is registered once at the start, as a kept trunk revision
|
||||
// registers one entering the graph (design 47 H H-A-262, H-A-264 (1)); said loudly, whatever it answers.
|
||||
done := step("registering the trunk graph's modules that have no registration")
|
||||
if written, err := ix.RegisterTrunkGraph(ctx); err != nil {
|
||||
logger.Printf("start: the trunk graph's modules with no registration are not all registered (%d were): %v", len(written), err)
|
||||
} else {
|
||||
logger.Printf("start: %d modules of the trunk graph had no registration and are registered now: %s", len(written), strings.Join(written, ", "))
|
||||
}
|
||||
done()
|
||||
done = step("reading the build log")
|
||||
// The log's entries at the start are read before any fingerprint is computed (below).
|
||||
var started atomic.Bool
|
||||
var startEnds []stdio.LogEntry // the log's entries read at the start, whose imports are kept once the packages are published
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"path"
|
||||
"slices"
|
||||
@@ -115,10 +116,15 @@ func trunkVia(call toolCaller) func(ctx context.Context, repository, sha string)
|
||||
}
|
||||
|
||||
// filesVia asks the forge for every blob under each directory at a commit (the seat git's trees with files, ADR 0593 §2),
|
||||
// by directory, then path relative to it. An answer that lists no directory asked is an error naming it.
|
||||
// by directory, then path relative to it. An answer that lists no directory asked is an error naming it. The seat answers
|
||||
// a directory the commit does not hold with the kind not-found naming it (the contract's files field; design 47 H H-A-232):
|
||||
// that is a failure of the tree too (index.ErrNotInTree), so the indexer reads each directory alone and leaves the gone one out.
|
||||
func filesVia(call toolCaller) func(ctx context.Context, repository, sha string, dirs []string) (map[string]map[string]string, error) {
|
||||
return func(_ context.Context, repository, sha string, dirs []string) (map[string]map[string]string, error) {
|
||||
a, err := treesOf(call, repository, git.TreesInput{Commit: sha, Files: dirs})
|
||||
if errors.Is(err, index.ErrNotFound) {
|
||||
return nil, fmt.Errorf("%w: %w", err, index.ErrNotInTree)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -75,6 +75,20 @@ func TestTheTrunkIsTheDirectoriesTheForgeListsTrees(t *testing.T) { // 1900/AC3
|
||||
if _, err := filesVia(noDir)(context.Background(), "novox/mesh-sdk", strings.Repeat("a", 40), []string{"go"}); !errors.Is(err, index.ErrNotInTree) {
|
||||
t.Errorf("a missing directory: %v, want ErrNotInTree", err)
|
||||
}
|
||||
// H-A-232 (2): the seat's not-found naming a directory the commit lacks is a failure of the tree too, so the
|
||||
// one-directory fallback runs; a failed listing of another kind is not.
|
||||
gone := func(string, any) (json.RawMessage, error) {
|
||||
return json.RawMessage(`{"error":"the directory modules/mesh-build-go/buildstep is not in novox/mesh-catalog at x","error-kind":"not-found"}`), nil
|
||||
}
|
||||
if _, err := filesVia(gone)(context.Background(), "novox/mesh-catalog", strings.Repeat("a", 40), []string{"modules/mesh-build-go/buildstep"}); !errors.Is(err, index.ErrNotInTree) {
|
||||
t.Errorf("a directory the seat says not-found: %v, want ErrNotInTree", err)
|
||||
}
|
||||
failed := func(string, any) (json.RawMessage, error) {
|
||||
return json.RawMessage(`{"error":"the forge did not answer","error-kind":"failed"}`), nil
|
||||
}
|
||||
if _, err := filesVia(failed)(context.Background(), "novox/mesh-catalog", strings.Repeat("a", 40), []string{"modules/a"}); err == nil || errors.Is(err, index.ErrNotInTree) {
|
||||
t.Errorf("a listing that failed: %v, want an error that is not ErrNotInTree", err)
|
||||
}
|
||||
down := func(string, any) (json.RawMessage, error) { return nil, errors.New("down") }
|
||||
if _, err := filesVia(down)(context.Background(), "novox/mesh-sdk", strings.Repeat("a", 40), []string{"go"}); err == nil || errors.Is(err, index.ErrNotInTree) {
|
||||
t.Errorf("a failed call: %v, want an error that is not ErrNotInTree", err)
|
||||
|
||||
@@ -7,6 +7,8 @@ import (
|
||||
"reflect"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -172,25 +174,43 @@ func (f *fakeBranchJS) Consumer(_ context.Context, stream, name string) (jetstre
|
||||
|
||||
func (f *fakeBranchJS) CreateConsumer(_ context.Context, stream string, cfg jetstream.ConsumerConfig) (jetstream.Consumer, error) {
|
||||
f.made = append(f.made, cfg)
|
||||
c := &fakeBranchConsumer{}
|
||||
c := newFakeBranchConsumer(stream, cfg.Durable)
|
||||
f.held[stream+"/"+cfg.Durable] = c
|
||||
return c, nil
|
||||
}
|
||||
|
||||
// fakeBranchConsumer hands one source-moved to the hearing as soon as it is consumed.
|
||||
// fakeBranchConsumer hands one source-moved to the hearing as soon as it is consumed. It answers its CachedInfo as the bus
|
||||
// does, with its stream, name and created time, which the SDK's guard of one pull per durable keys by (design 47 H-A-196).
|
||||
type fakeBranchConsumer struct {
|
||||
jetstream.Consumer
|
||||
info jetstream.ConsumerInfo
|
||||
heard int
|
||||
}
|
||||
|
||||
type fakeConsumeContext struct{ jetstream.ConsumeContext }
|
||||
// fakeConsumersMade numbers the fakes, so each is created at a time of its own, as each durable the bus makes is.
|
||||
var fakeConsumersMade atomic.Int64
|
||||
|
||||
func (fakeConsumeContext) Stop() {}
|
||||
func newFakeBranchConsumer(stream, durable string) *fakeBranchConsumer {
|
||||
return &fakeBranchConsumer{info: jetstream.ConsumerInfo{Stream: stream, Name: durable, Config: jetstream.ConsumerConfig{Durable: durable},
|
||||
Created: time.Unix(0, fakeConsumersMade.Add(1)).UTC()}}
|
||||
}
|
||||
|
||||
func (c *fakeBranchConsumer) CachedInfo() *jetstream.ConsumerInfo { return &c.info }
|
||||
|
||||
// fakeConsumeContext is a consume that ends when stopped, as the guard's release waits for (H-A-196).
|
||||
type fakeConsumeContext struct {
|
||||
jetstream.ConsumeContext
|
||||
stop sync.Once
|
||||
closed chan struct{}
|
||||
}
|
||||
|
||||
func (c *fakeConsumeContext) Stop() { c.stop.Do(func() { close(c.closed) }) }
|
||||
func (c *fakeConsumeContext) Closed() <-chan struct{} { return c.closed }
|
||||
|
||||
func (c *fakeBranchConsumer) Consume(handle jetstream.MessageHandler, _ ...jetstream.PullConsumeOpt) (jetstream.ConsumeContext, error) {
|
||||
c.heard++
|
||||
handle(msgOf{f: &fakeMsg{subject: "mesh.seat.mesh-delivery.event.source-moved.main.2"}})
|
||||
return fakeConsumeContext{}, nil
|
||||
return &fakeConsumeContext{closed: make(chan struct{})}, nil
|
||||
}
|
||||
|
||||
// Design 47 H §743, H-A-134: a branch's durable the bus does not hold the indexer makes, as a module's seat-event consumer is
|
||||
@@ -207,7 +227,7 @@ func TestABranchDurableNotThereIsMadeAndHeard(t *testing.T) {
|
||||
name := "n1_mesh-delivery-indexer_" + keys.EscapeName("main")
|
||||
var existing *fakeBranchConsumer
|
||||
if tc.held {
|
||||
existing = &fakeBranchConsumer{}
|
||||
existing = newFakeBranchConsumer(meshdelivery.StreamSourceMoved, name)
|
||||
js.held[meshdelivery.StreamSourceMoved+"/"+name] = existing
|
||||
}
|
||||
var heard []string
|
||||
|
||||
@@ -0,0 +1,147 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"mesh-delivery-indexer/internal/index"
|
||||
)
|
||||
|
||||
// askedBy is a takerIn that names the hand act it was asked.
|
||||
type askedBy struct{ asked []string }
|
||||
|
||||
func (s *askedBy) did(act string) index.TakenIn {
|
||||
s.asked = append(s.asked, act)
|
||||
return index.TakenIn{Module: "m", Stored: true, Answer: act}
|
||||
}
|
||||
func (s *askedBy) SeatDefinitionsAgain(context.Context, string) index.TakenIn {
|
||||
return s.did("seat-definitions")
|
||||
}
|
||||
func (s *askedBy) FingerprintsAgain(context.Context, string) index.TakenIn {
|
||||
return s.did("fingerprints")
|
||||
}
|
||||
func (s *askedBy) RecomputeByHand(context.Context, string, string) index.TakenIn {
|
||||
return s.did("recompute")
|
||||
}
|
||||
func (s *askedBy) DropCandidateGraph(context.Context, string, string, string, string) index.TakenIn {
|
||||
return s.did("candidate-group")
|
||||
}
|
||||
func (s *askedBy) NamedDrop(context.Context, string, string, string, string) index.TakenIn {
|
||||
return s.did("named-fingerprint")
|
||||
}
|
||||
func (s *askedBy) PinDrop(context.Context, string, string, string) index.TakenIn {
|
||||
return s.did("pin-drop")
|
||||
}
|
||||
func (s *askedBy) DropRunByHand(context.Context, string, string, string) index.TakenIn {
|
||||
return s.did("run-drop")
|
||||
}
|
||||
func (s *askedBy) RegistryAct(context.Context, index.TakeInAsk) index.TakenIn {
|
||||
return s.did("registry")
|
||||
}
|
||||
func (s *askedBy) HeldAgain(context.Context, string) index.TakenIn { return s.did("held") }
|
||||
func (s *askedBy) HeldSkip(context.Context, string, int64, string, string) index.TakenIn {
|
||||
return s.did("held-skip")
|
||||
}
|
||||
func (s *askedBy) TakeInAgain(context.Context, string) index.TakenIn { return s.did("again") }
|
||||
|
||||
// answeredKV keeps the answer takenIn writes back.
|
||||
type answeredKV struct {
|
||||
jetstream.KeyValue
|
||||
written []byte
|
||||
}
|
||||
|
||||
func (k *answeredKV) Update(_ context.Context, _ string, value []byte, _ uint64) (uint64, error) {
|
||||
k.written = value
|
||||
return 2, nil
|
||||
}
|
||||
|
||||
type askEntry struct {
|
||||
jetstream.KeyValueEntry
|
||||
key string
|
||||
value []byte
|
||||
}
|
||||
|
||||
func (e askEntry) Key() string { return e.key }
|
||||
func (e askEntry) Value() []byte { return e.value }
|
||||
func (e askEntry) Revision() uint64 { return 1 }
|
||||
func (e askEntry) Operation() jetstream.KeyValueOp { return jetstream.KeyValuePut }
|
||||
|
||||
// Design 47 H-A-208, H-A-210, H §774: every hand tool's takeins value, built by the tool's own Run path (or the constructor it
|
||||
// calls), is read by DecodeTakeIn and acted on by takenIn as the act it names; indexer_take_in_again's was refused, its Again
|
||||
// missing from a hand-kept list. Every field of TakeInAsk says by its tag whether it is an act, so an act added is read.
|
||||
func TestEveryHandToolsAskIsReadAndActedOn(t *testing.T) {
|
||||
t.Run("every field is classed", func(t *testing.T) {
|
||||
ty := reflect.TypeFor[index.TakeInAsk]()
|
||||
for i := range ty.NumField() {
|
||||
if c := ty.Field(i).Tag.Get("takein"); !slices.Contains([]string{"act", "argument", "ask"}, c) {
|
||||
t.Errorf("TakeInAsk.%s has takein %q, want act, argument or ask", ty.Field(i).Name, c)
|
||||
}
|
||||
}
|
||||
})
|
||||
var values [][]byte
|
||||
ask := func(_ string, value []byte) (index.TakeInAsk, error) {
|
||||
values = append(values, value)
|
||||
return index.TakeInAsk{V: 1, Result: &index.TakenIn{Stored: true}, Answered: time.Now()}, nil
|
||||
}
|
||||
record := func(string, string) error { return nil }
|
||||
tools := map[string]func(map[string]any) (any, error){}
|
||||
for _, tl := range slices.Concat(index.HeldTools(nil, ask, time.Now, record), index.PinsTools(nil, nil, ask, time.Now, record),
|
||||
index.RunsTools(nil, nil, ask, time.Now, record), index.NamedTools(ask, time.Now, record),
|
||||
index.FingerprintsTools(ask, time.Now, record, time.Minute), index.CatalogRevisionTools(nil, ask, time.Now, record),
|
||||
index.RegistrationTools(nil, nil, ask, time.Now, func(string, string, string, string) error { return nil })) {
|
||||
tools[tl.Name] = tl.Run
|
||||
}
|
||||
tools[index.SeatDefinitionsAgainTool] = func(args map[string]any) (any, error) { // its tool lives in the tools bundle
|
||||
_, value, err := index.NewSeatDefinitionsAgain(args["module"].(string), "why", "by", time.Now())
|
||||
values = append(values, value)
|
||||
return nil, err
|
||||
}
|
||||
for _, c := range []struct {
|
||||
tool string
|
||||
args map[string]any
|
||||
act string
|
||||
}{
|
||||
{index.TakeInAgainTool, map[string]any{"repository": "novox/mesh-host", "why": "rule changed"}, "again"},
|
||||
{index.TakeInPendingTool, map[string]any{"branch": "main"}, "held"},
|
||||
{index.TakeInSkipTool, map[string]any{"branch": "main", "source": float64(4), "why": "stale"}, "held-skip"},
|
||||
{index.PinDropTool, map[string]any{"key": "main.7", "why": "no pipeline"}, "pin-drop"},
|
||||
{index.RunDropTool, map[string]any{"run": "run-12", "why": "never ends"}, "run-drop"},
|
||||
{index.NamedDropTool, map[string]any{"fingerprint": "abc", "member": "ace.feh", "why": "gone"}, "named-fingerprint"},
|
||||
{index.FingerprintsAgainTool, map[string]any{"module": "feh", "why": "unread packages"}, "fingerprints"},
|
||||
{index.RecomputeTool, map[string]any{"why": "check"}, "recompute"},
|
||||
{index.DropCandidateGraphTool, map[string]any{"group": "g", "heads": "h", "why": "no gate event"}, "candidate-group"},
|
||||
{index.SeedAgainTool, map[string]any{"why": "empty"}, "registry"},
|
||||
{index.SeatDefinitionsAgainTool, map[string]any{"module": "mesh-contracts"}, "seat-definitions"},
|
||||
} {
|
||||
t.Run(c.tool, func(t *testing.T) {
|
||||
run, ok := tools[c.tool]
|
||||
if !ok {
|
||||
t.Fatalf("no tool %s", c.tool)
|
||||
}
|
||||
values = nil
|
||||
if _, err := run(c.args); err != nil || len(values) != 1 {
|
||||
t.Fatalf("%s wrote %d values (%v)", c.tool, len(values), err)
|
||||
}
|
||||
if _, err := index.DecodeTakeIn(values[0]); err != nil {
|
||||
t.Fatalf("DecodeTakeIn refuses %s: %v", values[0], err)
|
||||
}
|
||||
ix, kv := &askedBy{}, &answeredKV{}
|
||||
var said []string
|
||||
takenIn(t.Context(), ix, kv, askEntry{key: "k", value: values[0]}, func(f string, a ...any) { said = append(said, f) })
|
||||
if !slices.Equal(ix.asked, []string{c.act}) {
|
||||
t.Errorf("takenIn asked %v of %s, want %s (said %s)", ix.asked, values[0], c.act, strings.Join(said, "; "))
|
||||
}
|
||||
var answered index.TakeInAsk
|
||||
if err := json.Unmarshal(kv.written, &answered); err != nil || !answered.Done() {
|
||||
t.Errorf("takenIn wrote back %s (%v), want the answered ask", kv.written, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -156,6 +156,7 @@ type Indexer struct {
|
||||
filling atomic.Bool // a FillTaken runs in the background (KickFill)
|
||||
fillAgain atomic.Bool // a kick arrived while it ran
|
||||
underSeats bool // a dry run inside judge.WithSeats: its judge passes do not hold the seat list (verdicts.go)
|
||||
isDry bool // a dry run (dryrun.go): it shares the live packages state, so it drops no kept package key (packages.go)
|
||||
judgeMu sync.Mutex // one judge pass at a time (verdicts.go)
|
||||
keepMu sync.Mutex // one catalog graph revision kept at a time (keepEntries)
|
||||
// Pins is the state pins (pins.go); nil keeps none. PinsPending says how many messages the indexer's durable consumer of the
|
||||
|
||||
@@ -446,6 +446,11 @@ func (ix *Indexer) keepBranchEntries(ctx context.Context, branch string, src int
|
||||
ix.sayFill(ctx, meshcatalog.EventFilling, fmt.Sprintf("%d", rev.Revision), meshcatalog.Fill{Bytes: after, Cap: capacity})
|
||||
}
|
||||
ix.logf("catalog graph: revision %d of source revision %d kept (%s), %d entries, %d changed", rev.Revision, src, call.Call, len(rev.Entries), len(changed))
|
||||
// A module entering the trunk's graph is registered before the revision is said, at every kind of kept trunk revision
|
||||
// (design 47 H H-A-262, H-A-264).
|
||||
if isTrunkBranch(branch) {
|
||||
ix.registerEntering(ctx, slices.Collect(maps.Values(entries)))
|
||||
}
|
||||
if ix.SayCatalogRevision != nil {
|
||||
if err := ix.SayCatalogRevision(ctx, meshcatalog.CatalogRevisionEvent{V: CatalogEntryLayout, Revision: rev.Revision, Kind: rev.Kind,
|
||||
Branch: keys.EscapeName(branch), Source: src, Call: call.Call, CallKind: call.Kind, Module: call.Module, Version: call.Version,
|
||||
|
||||
@@ -35,7 +35,7 @@ func (ix *Indexer) dry() *dryRun {
|
||||
// requirements, findings, seat definitions, holds and their bounds are not overlaid, nothing is said, and no hold is
|
||||
// weighed (Holds and Counted nil hold nothing); ReadFile only reads.
|
||||
Requirements: nil, Findings: nil, SeatDefinitions: nil, ReadFile: ix.ReadFile, SaySeatDefinition: nil,
|
||||
Heard: nil, underSeats: true,
|
||||
Heard: nil, underSeats: true, isDry: true,
|
||||
// A dry run keeps no catalog graph revision (catalog_revisions.go): its revision is the take-in's to keep.
|
||||
Revisions: nil, SayCatalogRevision: nil, SayFill: nil, EntryIndex: nil, NamedSets: nil, Versions: nil, SayVersionReleased: nil, SayVersionsFilled: nil, Inputs: nil, InputObjects: nil, Pins: nil, PinsPending: nil,
|
||||
// A dry run writes no registration and asks no one.
|
||||
|
||||
@@ -0,0 +1,206 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"go/ast"
|
||||
"go/parser"
|
||||
"go/token"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient/heard"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
)
|
||||
|
||||
// Design 47 H H-A-240, H-A-242, H-A-247, G §580: every finding kind the daemon's code raises or clears is one its writer owns
|
||||
// (FindingKinds, and a views reader's views.ViewKinds), so the writer never refuses a kind of the indexer's own. The test
|
||||
// reads the daemon's code (this package and cmd/mesh-delivery-indexer) for every place a kind is given to the writer: the
|
||||
// kind argument of observed, finding, clearFinding, clearHeld and a writer's Clear, the Kind of a Finding literal, and the
|
||||
// kinds of an SDK part handed the writer (partKinds). mcc.KindCallUnanswered stays readable, so a return to the tool runner's
|
||||
// kind fails the test. A
|
||||
// kind there is a string, a constant of the package or a constant named in imported below; any other form fails the test,
|
||||
// naming its place, but for the pass-through parameter `kind`, a held finding's `.Kind` and the kind cut from a key a views
|
||||
// reader names, which carry a kind given elsewhere.
|
||||
func TestFindingKindsHoldEveryKindTheCodeRaises(t *testing.T) {
|
||||
imported := map[string]string{"mcc.KindCallUnanswered": mcc.KindCallUnanswered, "views.ViewWaitsKind": views.ViewWaitsKind}
|
||||
// kindArg is the place of the kind among a raising call's arguments.
|
||||
kindArg := map[string]int{"observed": 2, "finding": 2, "clearFinding": 3, "clearHeld": 4, "Clear": 3}
|
||||
// partKinds are the kinds each SDK part raises through the writer it is handed (H-A-247 (1)); handed says a literal's field
|
||||
// gives it the writer.
|
||||
partKinds := map[string][]string{"views.ProcessReader": {views.EventUnhandledKind}, "views.Emitter": {views.UnsaidKind, views.TooLargeKind},
|
||||
"heard.Writer": {heard.Kind}}
|
||||
handed := func(e ast.Expr) bool {
|
||||
kv, ok := e.(*ast.KeyValueExpr)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
id, ok := kv.Key.(*ast.Ident)
|
||||
return ok && (id.Name == "Findings" || id.Name == "Put")
|
||||
}
|
||||
fset := token.NewFileSet()
|
||||
var files []*ast.File
|
||||
consts := map[string]string{}
|
||||
for _, dir := range []string{".", filepath.Join("..", "..", "cmd", "mesh-delivery-indexer")} {
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, e := range entries {
|
||||
if !strings.HasSuffix(e.Name(), ".go") || strings.HasSuffix(e.Name(), "_test.go") {
|
||||
continue
|
||||
}
|
||||
f, err := parser.ParseFile(fset, filepath.Join(dir, e.Name()), nil, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
files = append(files, f)
|
||||
if dir != "." {
|
||||
continue
|
||||
}
|
||||
for _, d := range f.Decls {
|
||||
g, ok := d.(*ast.GenDecl)
|
||||
if !ok || g.Tok != token.CONST {
|
||||
continue
|
||||
}
|
||||
for _, s := range g.Specs {
|
||||
v := s.(*ast.ValueSpec)
|
||||
for i, n := range v.Names {
|
||||
if i < len(v.Values) {
|
||||
if l, ok := v.Values[i].(*ast.BasicLit); ok && l.Kind == token.STRING {
|
||||
consts[n.Name], _ = strconv.Unquote(l.Value)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
declared := append(slices.Clone(FindingKinds), views.ViewKinds...)
|
||||
raised := map[string]string{} // kind -> a place it is raised
|
||||
kindOf := func(e ast.Expr) (string, bool) {
|
||||
switch x := e.(type) {
|
||||
case *ast.BasicLit:
|
||||
s, err := strconv.Unquote(x.Value)
|
||||
return s, err == nil
|
||||
case *ast.Ident:
|
||||
if x.Name == "kind" {
|
||||
return "", true // a pass-through parameter: its callers give the kind
|
||||
}
|
||||
s, ok := consts[x.Name]
|
||||
if !ok {
|
||||
s, ok = imported["index."+x.Name]
|
||||
}
|
||||
return s, ok
|
||||
case *ast.SliceExpr:
|
||||
return "", true // the kind of a key a views reader names: its own kinds (views.ViewKinds)
|
||||
case *ast.SelectorExpr:
|
||||
if x.Sel.Name == "Kind" {
|
||||
return "", true // a held finding's kind, given when it was raised
|
||||
}
|
||||
if p, ok := x.X.(*ast.Ident); ok {
|
||||
if p.Name == "index" {
|
||||
s, ok := consts[x.Sel.Name]
|
||||
return s, ok
|
||||
}
|
||||
s, ok := imported[p.Name+"."+x.Sel.Name]
|
||||
return s, ok
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
note := func(e ast.Expr) {
|
||||
k, ok := kindOf(e)
|
||||
if !ok {
|
||||
t.Errorf("%s: a finding kind the test cannot read; give it as a constant (or name it in imported)", fset.Position(e.Pos()))
|
||||
return
|
||||
}
|
||||
if k != "" {
|
||||
raised[k] = fset.Position(e.Pos()).String()
|
||||
}
|
||||
}
|
||||
for _, f := range files {
|
||||
ast.Inspect(f, func(n ast.Node) bool {
|
||||
switch x := n.(type) {
|
||||
case *ast.CallExpr:
|
||||
var name string
|
||||
switch fn := x.Fun.(type) {
|
||||
case *ast.Ident:
|
||||
name = fn.Name
|
||||
case *ast.SelectorExpr:
|
||||
name = fn.Sel.Name
|
||||
}
|
||||
if i, ok := kindArg[name]; ok && len(x.Args) > i {
|
||||
if name == "Clear" && len(x.Args) != 5 {
|
||||
return true // not a finding writer's Clear(ctx, scope, id, kind, why)
|
||||
}
|
||||
note(x.Args[i])
|
||||
}
|
||||
case *ast.CompositeLit:
|
||||
// an SDK part handed the writer raises its own kinds through it (H-A-247 (1))
|
||||
if s, ok := x.Type.(*ast.SelectorExpr); ok && slices.ContainsFunc(x.Elts, handed) {
|
||||
if p, ok := s.X.(*ast.Ident); ok && p.Name != "index" { // the indexer's own code is read above
|
||||
part := p.Name + "." + s.Sel.Name
|
||||
if kinds, ok := partKinds[part]; ok {
|
||||
for _, k := range kinds {
|
||||
raised[k] = fset.Position(x.Pos()).String() + " (" + part + ")"
|
||||
}
|
||||
} else {
|
||||
t.Errorf("%s: %s is handed the finding writer, and the test does not know its kinds: name them in partKinds",
|
||||
fset.Position(x.Pos()), part)
|
||||
}
|
||||
}
|
||||
}
|
||||
if s, ok := x.Type.(*ast.SelectorExpr); ok && s.Sel.Name == "Finding" {
|
||||
for _, el := range x.Elts {
|
||||
if kv, ok := el.(*ast.KeyValueExpr); ok {
|
||||
if id, ok := kv.Key.(*ast.Ident); ok && id.Name == "Kind" {
|
||||
note(kv.Value)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return true
|
||||
})
|
||||
}
|
||||
for _, k := range []string{PinMissingKind, OutOfStepKind, PackagesNotPublishedKind, CallUnansweredKind, views.EventUnhandledKind, heard.Kind} {
|
||||
if _, ok := raised[k]; !ok {
|
||||
t.Errorf("the test no longer finds the kind %s the code raises: it reads the code wrongly", k)
|
||||
}
|
||||
}
|
||||
for k, at := range raised {
|
||||
if !slices.Contains(declared, k) {
|
||||
t.Errorf("%s: the code raises the finding kind %q, which the writer's list FindingKinds does not declare", at, k)
|
||||
}
|
||||
}
|
||||
// a retired kind is one no code raises and the writer does not own (H-A-247 (3))
|
||||
for _, r := range RetiredFindingKinds {
|
||||
if at, ok := raised[r.Kind]; ok || slices.Contains(declared, r.Kind) {
|
||||
t.Errorf("the retired kind %q is still declared or raised (%s)", r.Kind, at)
|
||||
}
|
||||
}
|
||||
// H-A-249: the retired call-unanswered names the ids the indexer's unanswered calls are written under, as in a key
|
||||
covers := func(scope, id string) bool {
|
||||
for _, r := range RetiredFindingKinds {
|
||||
if r.Kind == mcc.KindCallUnanswered && (r.Scope == "" || r.Scope == scope) && (slices.Contains(r.IDs, id) ||
|
||||
slices.ContainsFunc(r.Prefixes, func(p string) bool { return strings.HasPrefix(id, p) })) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
for _, id := range []string{keyID("node-build-go." + StepsVerb), keyID("node-build-image." + StepsVerb), keyID(floorKey("mesh-tools")),
|
||||
keyID(BuilderVersionsCall)} {
|
||||
if !covers(mcc.ScopeCall, id) {
|
||||
t.Errorf("the retired call-unanswered does not cover the indexer's id %s", id)
|
||||
}
|
||||
}
|
||||
// the approval gate's findings: scope machine, id <node>=2e<ask>
|
||||
if covers("machine", "novox=2eask-1") || covers("machine", "node-build-go.steps") || covers(mcc.ScopeCall, "novox=2eask-1") {
|
||||
t.Error("the retired call-unanswered covers an id of the tool runner's")
|
||||
}
|
||||
}
|
||||
@@ -4,7 +4,9 @@ import (
|
||||
"context"
|
||||
|
||||
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient/heard"
|
||||
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
)
|
||||
|
||||
// FindingsState is the indexer's state of its findings (design 47 G §383, §687).
|
||||
@@ -12,6 +14,26 @@ const (
|
||||
FindingsState = "findings"
|
||||
)
|
||||
|
||||
// CallUnansweredKind is the indexer's finding of a call nobody answered within its bound (scope call, its id the call): a kind
|
||||
// of its own, since the tool runner's writer over the same state owns call-unanswered (design 47 H H-A-247 (2)).
|
||||
const CallUnansweredKind = "indexer-call-unanswered"
|
||||
|
||||
// FindingKinds are the finding kinds the indexer's daemon raises, the kinds its writer owns (design 47 G §580, H H-A-240,
|
||||
// H-A-242, H-A-247): its own, the process reader's event-unhandled and the family-not-heard of the heard writer it is handed
|
||||
// (H-A-247 (1)); a views reader's and the emitter's (views.ViewKinds) are always the writer's besides. The tool runner's
|
||||
// writer over the same state `findings` owns its own. TestFindingKindsHoldEveryKindTheCodeRaises holds the code to this list.
|
||||
var FindingKinds = []string{PinMissingKind, OutOfStepKind, PackagesNotPublishedKind, HistoryFillingKind, SeedWaitsKind,
|
||||
SeedLeftOutKind, SeatDefinitionUnparsedKind, SeatDefinedTwiceKind, SeatOutsideItsDomainKind, CallUnansweredKind,
|
||||
views.EventUnhandledKind, heard.Kind}
|
||||
|
||||
// RetiredFindingKinds are kinds no code of the indexer raises any more, which its writer clears once at its start (H-A-247
|
||||
// (3)): merge-untaken went with H-A-82; call-unanswered became indexer-call-unanswered (H-A-247 (2)), and since the tool
|
||||
// runner's writer owns call-unanswered in the same state, only the ids the indexer wrote under it, in scope call, are cleared (H-A-249):
|
||||
// the seed's <build seat>.steps (every build seat is named node-build-<kind>), the floor's mesh-catalog.versions-floor.<module>
|
||||
// and the versions fill's mesh-delivery-builder.builder_versions.
|
||||
var RetiredFindingKinds = []views.RetiredKind{{Kind: "merge-untaken"}, {Kind: mcc.KindCallUnanswered, Scope: mcc.ScopeCall,
|
||||
IDs: []string{keyID(BuilderVersionsCall)}, Prefixes: []string{"node-build-", keyID(FloorCall) + "."}}}
|
||||
|
||||
// FindingWriter is the indexer's finding writer, mesh-sdk's held writer of the seat mesh-conditions' generated finding (design
|
||||
// 47 G A-29, A-38): Write raises a finding, Clear clears one, Held answers the findings it holds, so the indexer reads what
|
||||
// stands from the writer.
|
||||
|
||||
@@ -0,0 +1,246 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"path"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
|
||||
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
)
|
||||
|
||||
// seedSteps are the seed's module-wide statuses (§8, design 47 H §745): the statuses the check judged until then, lint, unit
|
||||
// tests, integration tests, composition and validate required, code review optional, approval bypass.
|
||||
func seedSteps() meshcatalog.StepStatuses {
|
||||
return meshcatalog.StepStatuses{Lint: "required", Unit: "required", Integration: "required",
|
||||
Composition: "required", Review: "optional", Approval: "bypass", Validate: "required"}
|
||||
}
|
||||
|
||||
// GraphModule is a module of a trunk revision's module graph as its registration is composed when it enters the graph
|
||||
// (design 47 H H-A-262): its repository, the path of its manifest and the names of its artifacts.
|
||||
type GraphModule struct {
|
||||
Module string
|
||||
Origin string
|
||||
Path string
|
||||
Artifacts []string
|
||||
// LeftOut says why the seed left the module out (its finding seed-left-out, A-324): such a module is not registered by
|
||||
// the graph (H-A-265 (2)). Empty: it is registered.
|
||||
LeftOut string
|
||||
}
|
||||
|
||||
// GraphModulesOf are the modules of a trunk revision's entries, sorted by module.
|
||||
func GraphModulesOf(entries []meshcatalog.CatalogEntry) []GraphModule {
|
||||
out := make([]GraphModule, 0, len(entries))
|
||||
for _, e := range entries {
|
||||
m := GraphModule{Module: e.Module, Origin: e.Repository, Path: path.Join(e.Path, ModuleManifest)}
|
||||
for _, a := range e.Artifacts {
|
||||
m.Artifacts = append(m.Artifacts, artifactName(a.Ref))
|
||||
}
|
||||
out = append(out, m)
|
||||
}
|
||||
slices.SortFunc(out, func(a, b GraphModule) int { return strings.Compare(a.Module, b.Module) })
|
||||
return out
|
||||
}
|
||||
|
||||
// enteringPolicy is the policy of a module entering the graph (H-A-262, H-A-263): the module-wide steps every other
|
||||
// registered module of its repository holds alike, else the seed's; a per-artifact status only for an artifact name the module
|
||||
// has that every sibling naming it holds identically; nothing else.
|
||||
func enteringPolicy(m GraphModule, siblings []meshcatalog.Registration) meshcatalog.Policy {
|
||||
p := meshcatalog.Policy{Steps: seedSteps()}
|
||||
if len(siblings) > 0 {
|
||||
alike := true
|
||||
for _, s := range siblings[1:] {
|
||||
alike = alike && s.Policy.Steps == siblings[0].Policy.Steps
|
||||
}
|
||||
if alike {
|
||||
p.Steps = siblings[0].Policy.Steps
|
||||
}
|
||||
}
|
||||
for _, a := range m.Artifacts {
|
||||
var held []meshcatalog.StepStatuses
|
||||
for _, s := range siblings {
|
||||
if st, ok := s.Policy.Artifacts[a]; ok {
|
||||
held = append(held, st)
|
||||
}
|
||||
}
|
||||
if len(held) == 0 || slices.ContainsFunc(held[1:], func(st meshcatalog.StepStatuses) bool { return st != held[0] }) {
|
||||
continue
|
||||
}
|
||||
if p.Artifacts == nil {
|
||||
p.Artifacts = map[string]meshcatalog.StepStatuses{}
|
||||
}
|
||||
p.Artifacts[a] = held[0]
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// RegisterEntering registers every module of mods that has no registration (design 47 H H-A-262 to H-A-266; the operator,
|
||||
// 2026-11-14: a module in the module graph is a registration): origin its repository, path its manifest's, the branch its
|
||||
// repository's registered modules watch first alike, else the repository's default branch read through the seat git
|
||||
// (enteringBranch), the policy of its repository's other registered modules (enteringPolicy), version 1 with no ask, said
|
||||
// registration-changed. A module already registered is left as it is, and none is ever removed here. Skipped, each said in the
|
||||
// journal: a module with a register proposal pending (its ask stands, H-A-265 (1)), one the seed left out (LeftOut, H-A-265
|
||||
// (2)), and one whose default branch is not read (registered at the next kept revision, H-A-266). While no registration is
|
||||
// held it registers none: the seed writes the first (H-A-264 (2)). It answers the modules it registered, sorted.
|
||||
func (r *Registry) RegisterEntering(ctx context.Context, mods []GraphModule, by string) ([]string, error) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
// One listing finds the modules with no registration; the registrations themselves are read only when one has none.
|
||||
keys, err := r.Registrations.Keys(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the registrations are not listed: %w", err)
|
||||
}
|
||||
if len(keys) == 0 {
|
||||
// With none held, the seed is the first and only writer: its policy reads the build seats' steps (H-A-264 (2)).
|
||||
r.logf("registrations: none is held, so no module entering the module graph is registered here: the seed writes the first (H-A-264)")
|
||||
return nil, nil
|
||||
}
|
||||
listed := map[string]bool{}
|
||||
for _, k := range keys {
|
||||
listed[k] = true
|
||||
}
|
||||
var entering []GraphModule
|
||||
for _, m := range mods {
|
||||
if m.Module != "" && !listed[RegistrationKey(m.Module)] {
|
||||
entering = append(entering, m)
|
||||
}
|
||||
}
|
||||
if len(entering) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
held, err := r.RegistrationsAnswer(ctx, meshcatalog.RegistrationsInput{})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the registrations are not read: %w", err)
|
||||
}
|
||||
have := map[string]bool{}
|
||||
byOrigin := map[string][]meshcatalog.Registration{}
|
||||
for _, reg := range held.Registrations {
|
||||
have[reg.Module] = true
|
||||
byOrigin[reg.Origin] = append(byOrigin[reg.Origin], reg)
|
||||
}
|
||||
var written []string
|
||||
var errs []string
|
||||
for _, m := range entering {
|
||||
if have[m.Module] {
|
||||
continue
|
||||
}
|
||||
if m.LeftOut != "" {
|
||||
r.logf("registrations: %s entered the module graph and is not registered: the seed left it out (%s), so it keeps seed-left-out (H-A-265)", m.Module, m.LeftOut)
|
||||
continue
|
||||
}
|
||||
if p, _, found, err := r.load(ctx, ProposalKey(verbRegister, m.Module, "")); err != nil {
|
||||
errs = append(errs, fmt.Sprintf("%s: its register proposal is not read: %v", m.Module, err))
|
||||
continue
|
||||
} else if found && p.State == ProposalOpen {
|
||||
r.logf("registrations: %s entered the module graph with a register proposal pending (ask %s): it is not registered here, the operator's Approve registers it (H-A-265)", m.Module, p.Ask)
|
||||
continue
|
||||
}
|
||||
branch, err := r.enteringBranch(m.Origin, byOrigin[m.Origin])
|
||||
if err != nil {
|
||||
r.logf("registrations: %s entered the module graph and is not registered now: %v; the next kept trunk revision registers it (H-A-266)", m.Module, err)
|
||||
continue
|
||||
}
|
||||
reg := meshcatalog.Registration{Module: m.Module, Origin: m.Origin, Path: m.Path, Branches: []string{branch},
|
||||
Policy: enteringPolicy(m, byOrigin[m.Origin])}
|
||||
p := Proposal{V: 1, Verb: verbRegister, Module: m.Module, Caller: by, At: r.now(), State: ProposalOpen,
|
||||
Lines: []string{fmt.Sprintf("%s: registered as it entered the module graph (%s, %s)", m.Module, m.Origin, m.Path)}}
|
||||
if err := r.write(ctx, "", p, reg, oc.Warrant{Ask: "module-graph"}); err != nil {
|
||||
errs = append(errs, fmt.Sprintf("%s: %v", m.Module, err))
|
||||
continue
|
||||
}
|
||||
have[m.Module] = true
|
||||
written = append(written, m.Module)
|
||||
r.logf("registrations: %s entered the module graph of %s with no registration: registered as version 1 from %s (%s), watching %s (H-A-262)",
|
||||
m.Module, TrunkBranch, m.Origin, m.Path, branch)
|
||||
}
|
||||
if len(errs) > 0 {
|
||||
return written, fmt.Errorf("the registrations of modules entering the module graph are not written: %s", strings.Join(errs, "; "))
|
||||
}
|
||||
return written, nil
|
||||
}
|
||||
|
||||
// enteringBranch is the branch a module entering the graph watches (H-A-265 (3), H-A-266): the first watched branch every
|
||||
// registered module of its repository holds alike, else the repository's default branch as the seed reads it from the seat
|
||||
// git; an error when that is not read.
|
||||
func (r *Registry) enteringBranch(origin string, siblings []meshcatalog.Registration) (string, error) {
|
||||
first := ""
|
||||
alike := len(siblings) > 0
|
||||
for _, s := range siblings {
|
||||
if len(s.Branches) == 0 || (first != "" && s.Branches[0] != first) {
|
||||
alike = false
|
||||
break
|
||||
}
|
||||
first = s.Branches[0]
|
||||
}
|
||||
if alike && first != "" {
|
||||
return first, nil
|
||||
}
|
||||
if r.Branch == nil {
|
||||
return "", fmt.Errorf("the default branch of %s is not asked of the seat git", origin)
|
||||
}
|
||||
b, err := r.Branch(origin, "")
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("the seat git does not answer the default branch of %s: %w", origin, err)
|
||||
}
|
||||
if b.Error != "" || b.Branch == "" {
|
||||
return "", fmt.Errorf("the default branch of %s is not read: %s", origin, b.Error)
|
||||
}
|
||||
return b.Branch, nil
|
||||
}
|
||||
|
||||
// registerEntering registers each module of a kept trunk revision's entries that has no registration (H-A-262, H-A-264 (1),
|
||||
// (3)): a module entering the trunk's graph, at any kind of kept trunk revision. A dry run, or an indexer with no
|
||||
// registrations, registers none. A failure is said in the journal: the revision is kept whatever it answers.
|
||||
func (ix *Indexer) registerEntering(ctx context.Context, entries []meshcatalog.CatalogEntry) {
|
||||
if ix.isDry || ix.Registry == nil || ix.Registry.Registrations == nil {
|
||||
return
|
||||
}
|
||||
written, err := ix.Registry.RegisterEntering(ctx, ix.withLeftOut(GraphModulesOf(entries)), ix.Registry.Module)
|
||||
if len(written) > 0 {
|
||||
ix.logf("registrations: %d modules entering the module graph of %s registered: %s (H-A-262)", len(written), TrunkBranch, strings.Join(written, ", "))
|
||||
}
|
||||
if err != nil {
|
||||
ix.logf("registrations: %v (H-A-262)", err)
|
||||
}
|
||||
}
|
||||
|
||||
// RegisterTrunkGraph registers, once at the indexer's start, each module of the current trunk graph that has no registration
|
||||
// (H-A-264 (1)), as a kept trunk revision does; it answers the modules it registered.
|
||||
func (ix *Indexer) RegisterTrunkGraph(ctx context.Context) ([]string, error) {
|
||||
if ix.isDry || ix.Registry == nil || ix.Registry.Registrations == nil || ix.Revisions == nil {
|
||||
return nil, nil
|
||||
}
|
||||
_, entries, found, err := readNewestOf(ctx, ix.Revisions, TrunkBranch)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the newest trunk revision is not read: %w", err)
|
||||
}
|
||||
if !found {
|
||||
return nil, nil
|
||||
}
|
||||
return ix.Registry.RegisterEntering(ctx, ix.withLeftOut(GraphModulesOf(entries)), ix.Registry.Module)
|
||||
}
|
||||
|
||||
// withLeftOut marks each module the indexer holds the finding seed-left-out of (A-324, H-A-265 (2)).
|
||||
func (ix *Indexer) withLeftOut(mods []GraphModule) []GraphModule {
|
||||
if ix.Findings == nil {
|
||||
return mods
|
||||
}
|
||||
left := map[string]string{}
|
||||
for _, f := range ix.Findings.Held() {
|
||||
if f.Kind == SeedLeftOutKind && f.Scope == mcc.ScopeModule {
|
||||
left[f.ID] = f.Summary
|
||||
}
|
||||
}
|
||||
for i := range mods {
|
||||
if why, ok := left[keyID(mods[i].Module)]; ok {
|
||||
if why == "" {
|
||||
why = "seed-left-out"
|
||||
}
|
||||
mods[i].LeftOut = why
|
||||
}
|
||||
}
|
||||
return mods
|
||||
}
|
||||
@@ -0,0 +1,164 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"slices"
|
||||
"testing"
|
||||
|
||||
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
|
||||
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
|
||||
gitseat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git"
|
||||
|
||||
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
)
|
||||
|
||||
// Design 47 H H-A-262, H-A-263, H-A-264 (ADR 0666 §2 read with the operator's rule of 2026-11-14): a module that enters a
|
||||
// watched trunk's module graph is registered as the revision is kept, before it is said, with no ask: origin its repository,
|
||||
// path its manifest's, the branch its repository's modules watch first alike (else its default branch), the module-wide steps its repository's other modules hold alike (else the
|
||||
// seed's) and a per-artifact status only where every sibling naming that artifact holds it identically, said
|
||||
// registration-changed. A module registered already is untouched; one with a register proposal pending, one the seed left out
|
||||
// and one whose default branch is not read are skipped (H-A-265, H-A-266); a branch that is not the trunk registers none, an indexer
|
||||
// holding no registration leaves the first to the seed, and the start registers the current trunk graph's unregistered ones.
|
||||
func TestAModuleEnteringTheTrunkGraphIsRegistered(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
g := newRegRig()
|
||||
var order []string
|
||||
say := g.r.Say
|
||||
g.r.Say = func(ctx context.Context, e meshcatalog.RegistrationChangedEvent) error {
|
||||
order = append(order, "registration-changed "+e.Module)
|
||||
return say(ctx, e)
|
||||
}
|
||||
ix := &Indexer{Module: testSelf, Revisions: newMemObjects(1 << 20), Findings: newFindings(), Now: g.r.Now}
|
||||
// The seat git's default branches: novox/new's is develop, novox/mixed's trunk, novox/broken's not read (H-A-266).
|
||||
g.r.Branch = func(repo, branch string) (gitseat.BranchAnswer, error) {
|
||||
switch repo {
|
||||
case "novox/new":
|
||||
return gitseat.BranchAnswer{Branch: "develop", Repository: repo}, nil
|
||||
case "novox/mixed":
|
||||
return gitseat.BranchAnswer{Branch: "trunk", Repository: repo}, nil
|
||||
case "novox/broken":
|
||||
return gitseat.BranchAnswer{}, errors.New("the forge does not answer")
|
||||
}
|
||||
return gitseat.BranchAnswer{Branch: "main", Repository: repo}, nil
|
||||
}
|
||||
ix.SayCatalogRevision = func(_ context.Context, e meshcatalog.CatalogRevisionEvent) error {
|
||||
order = append(order, "catalog-revision "+e.Branch)
|
||||
return nil
|
||||
}
|
||||
entry := func(module, repo, dir string, artifacts ...string) meshcatalog.CatalogEntry {
|
||||
e := meshcatalog.CatalogEntry{Module: module, Repository: repo, Path: dir, Commit: "c1", Manifest: map[string]any{}}
|
||||
for _, a := range artifacts {
|
||||
e.Artifacts = append(e.Artifacts, meshcatalog.Artifact{Ref: module + ":" + a})
|
||||
}
|
||||
return e
|
||||
}
|
||||
keep := func(branch string, src int64, entries ...meshcatalog.CatalogEntry) {
|
||||
t.Helper()
|
||||
m := map[string]meshcatalog.CatalogEntry{}
|
||||
for _, e := range entries {
|
||||
m[e.Module] = e
|
||||
}
|
||||
if err := ix.KeepBranchEntries(ctx, branch, src, m, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// No registration held: the seed is the first writer, so a kept trunk revision registers nothing (H-A-264 (2)).
|
||||
ix.Registry = g.r
|
||||
keep("main", 1, entry("core", "novox/c", "go/core", "lib"))
|
||||
if regs, _ := g.r.RegistrationsAnswer(ctx, meshcatalog.RegistrationsInput{}); len(regs.Registrations) != 0 {
|
||||
t.Fatalf("with no registration held a trunk revision registered %v, want none: the seed writes the first", regs.Registrations)
|
||||
}
|
||||
|
||||
steps := meshcatalog.StepStatuses{Lint: "required", Unit: "required", Integration: "required", Composition: "required", Review: "optional", Approval: "bypass"}
|
||||
other := steps
|
||||
other.Lint = "optional"
|
||||
held := []meshcatalog.Registration{
|
||||
{Module: "core", Origin: "novox/c", Path: "go/core/module.json", Branches: []string{"stable"}, Policy: meshcatalog.Policy{Steps: steps,
|
||||
Artifacts: map[string]meshcatalog.StepStatuses{"lib": {Integration: "bypass"}, "files": {Lint: "bypass", Unit: "bypass"}}}},
|
||||
{Module: "tools", Origin: "novox/c", Path: "go/tools/module.json", Branches: []string{"stable"}, Policy: meshcatalog.Policy{Steps: steps,
|
||||
Artifacts: map[string]meshcatalog.StepStatuses{"lib": {Integration: "bypass"}, "files": {Unit: "bypass"}}}},
|
||||
{Module: "x1", Origin: "novox/mixed", Path: "x1/module.json", Branches: []string{"main"}, Policy: meshcatalog.Policy{Steps: steps}},
|
||||
{Module: "x2", Origin: "novox/mixed", Path: "x2/module.json", Branches: []string{"beta"}, Policy: meshcatalog.Policy{Steps: other}},
|
||||
}
|
||||
for _, reg := range held {
|
||||
if err := g.r.write(ctx, "", Proposal{V: 1, Verb: verbRegister, Module: reg.Module, Caller: "test", State: ProposalOpen}, reg, oc.Warrant{Ask: "test"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
g.events, order = nil, nil
|
||||
// pend has a register proposal pending (H-A-265 (1)); left was left out by the seed (A-324, H-A-265 (2)).
|
||||
pending, _ := json.Marshal(Proposal{V: 1, Verb: verbRegister, Module: "pend", Ask: "reg-1", State: ProposalOpen})
|
||||
if err := g.r.Proposals.Create(ctx, ProposalKey(verbRegister, "pend", ""), pending); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ix.Findings.Write(ctx, mcseat.Finding{Scope: mcc.ScopeModule, ID: keyID("left"), Kind: SeedLeftOutKind, Summary: "no build seat builds left:app"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// A take-in that lists new modules beside a registered one.
|
||||
keep("main", 2, entry("core", "novox/c", "go/core", "lib"), entry("lib2", "novox/c", "go/lib2", "lib", "files", "bin"),
|
||||
entry("solo", "novox/new", "solo", "app"), entry("x3", "novox/mixed", "x3"), entry("pend", "novox/c", "go/pend"),
|
||||
entry("left", "novox/c", "go/left", "app"), entry("lost", "novox/broken", "lost"))
|
||||
got := func(module string) meshcatalog.Registration {
|
||||
t.Helper()
|
||||
r, found, err := g.r.Registration(ctx, module)
|
||||
if err != nil || !found {
|
||||
t.Fatalf("%s is not registered (%v)", module, err)
|
||||
}
|
||||
return r
|
||||
}
|
||||
lib2 := got("lib2")
|
||||
if lib2.Version != 1 || lib2.Origin != "novox/c" || lib2.Path != "go/lib2/module.json" || !slices.Equal(lib2.Branches, []string{"stable"}) {
|
||||
t.Errorf("lib2 registered as %+v, want version 1 from novox/c, go/lib2/module.json, watching stable, the branch its siblings watch first alike (H-A-266), not its default branch main", lib2)
|
||||
}
|
||||
if lib2.Policy.Steps != steps {
|
||||
t.Errorf("lib2's steps %+v, want its repository's modules' alike steps %+v (H-A-263)", lib2.Policy.Steps, steps)
|
||||
}
|
||||
if a := lib2.Policy.Artifacts; len(a) != 1 || a["lib"] != (meshcatalog.StepStatuses{Integration: "bypass"}) {
|
||||
t.Errorf("lib2's artifacts %+v, want lib alone, the one status its siblings hold identically (files differs, bin none)", a)
|
||||
}
|
||||
if solo := got("solo"); solo.Policy.Steps != seedSteps() || len(solo.Policy.Artifacts) != 0 || !slices.Equal(solo.Branches, []string{"develop"}) {
|
||||
t.Errorf("solo, alone in its repository, has %+v watching %v, want the seed's default, watching its default branch develop", solo.Policy, solo.Branches)
|
||||
}
|
||||
if x3 := got("x3"); x3.Policy.Steps != seedSteps() || !slices.Equal(x3.Branches, []string{"trunk"}) {
|
||||
t.Errorf("x3, whose siblings' steps and branches differ, has %+v watching %v, want the seed's default, watching the default branch trunk", x3.Policy.Steps, x3.Branches)
|
||||
}
|
||||
for _, m := range []string{"pend", "left", "lost"} {
|
||||
if _, found, _ := g.r.Registration(ctx, m); found {
|
||||
t.Errorf("%s was registered, want it skipped: a pending proposal, the seed's left-out, a default branch not read", m)
|
||||
}
|
||||
}
|
||||
if core := got("core"); core.Version != 1 || core.By != "test" {
|
||||
t.Errorf("core, registered already, became %+v, want it untouched", core)
|
||||
}
|
||||
if len(g.asks) != 0 {
|
||||
t.Errorf("registering modules entering the graph asked %v, want no ask", g.asks)
|
||||
}
|
||||
wantOrder := []string{"registration-changed lib2", "registration-changed solo", "registration-changed x3", "catalog-revision main"}
|
||||
if !slices.Equal(order, wantOrder) {
|
||||
t.Errorf("said %v, want %v: each registration said before the revision", order, wantOrder)
|
||||
}
|
||||
|
||||
// A branch that is not the trunk registers none (H-A-264 (3)); a module that leaves the graph keeps its registration.
|
||||
keep("feature", 3, entry("feat", "novox/c", "go/feat", "lib"))
|
||||
keep("main", 4, entry("core", "novox/c", "go/core", "lib"))
|
||||
if _, found, _ := g.r.Registration(ctx, "feat"); found {
|
||||
t.Error("a revision of the branch feature registered feat, want only the trunk's")
|
||||
}
|
||||
got("lib2")
|
||||
|
||||
// At the start, a module of the current trunk graph with no registration is registered (H-A-264 (1)); again, none.
|
||||
ix.Registry = nil
|
||||
keep("main", 5, entry("core", "novox/c", "go/core", "lib"), entry("late", "novox/c", "go/late"))
|
||||
ix.Registry = g.r
|
||||
if written, err := ix.RegisterTrunkGraph(ctx); err != nil || !slices.Equal(written, []string{"late"}) {
|
||||
t.Fatalf("the start registered %v (%v), want late", written, err)
|
||||
}
|
||||
if written, err := ix.RegisterTrunkGraph(ctx); err != nil || len(written) != 0 {
|
||||
t.Errorf("a second start registered %v (%v), want none", written, err)
|
||||
}
|
||||
}
|
||||
@@ -68,7 +68,7 @@ func TestImportsReportedMoveTheFingerprintOnlyWithAnImportedPackage(t *testing.T
|
||||
}
|
||||
mods.put("mesh-sdk", sdkModule)
|
||||
held, _ := ix.heldModules(ctx)
|
||||
if err := ix.publish(ctx, "novox/mesh-sdk", c1, held, false); err != nil {
|
||||
if err := ix.publish(ctx, "novox/mesh-sdk", c1, held, false, TrunkBranch); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pub := ix.publishedAll(ctx)[meshSDK]
|
||||
@@ -99,7 +99,7 @@ func TestImportsReportedMoveTheFingerprintOnlyWithAnImportedPackage(t *testing.T
|
||||
|
||||
// mesh-sdk merges a change of buildstep only: avahi does not import it.
|
||||
held, _ = ix.heldModules(ctx)
|
||||
if err := ix.publish(ctx, "novox/mesh-sdk", c2, held, false); err != nil {
|
||||
if err := ix.publish(ctx, "novox/mesh-sdk", c2, held, false, TrunkBranch); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ix.Recompute(ctx); err != nil {
|
||||
@@ -111,7 +111,7 @@ func TestImportsReportedMoveTheFingerprintOnlyWithAnImportedPackage(t *testing.T
|
||||
|
||||
// mesh-sdk merges a change of announce: avahi moves, to the import form, pinned to the new version.
|
||||
held, _ = ix.heldModules(ctx)
|
||||
if err := ix.publish(ctx, "novox/mesh-sdk", c3, held, false); err != nil {
|
||||
if err := ix.publish(ctx, "novox/mesh-sdk", c3, held, false, TrunkBranch); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ix.Recompute(ctx); err != nil {
|
||||
@@ -195,7 +195,7 @@ func aSiblingDirectoryAloneMovesItsImporter(t *testing.T) {
|
||||
fpAt := func(commit string) meshcatalog.ModulesByModuleValue {
|
||||
t.Helper()
|
||||
held, _ := ix.heldModules(ctx)
|
||||
if err := ix.publish(ctx, catalogueRepo, commit, held, false); err != nil {
|
||||
if err := ix.publish(ctx, catalogueRepo, commit, held, false, TrunkBranch); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ix.Recompute(ctx); err != nil {
|
||||
@@ -274,7 +274,7 @@ func anAcceptedSeriesPinsItsNewest(t *testing.T) {
|
||||
trunk := func(commit string) func() error {
|
||||
return func() error {
|
||||
held, _ := ix.heldModules(ctx)
|
||||
return ix.publish(ctx, "novox/mesh-sdk", commit, held, false)
|
||||
return ix.publish(ctx, "novox/mesh-sdk", commit, held, false, TrunkBranch)
|
||||
}
|
||||
}
|
||||
hear := func(version, commit string) func() error {
|
||||
@@ -328,7 +328,7 @@ func anUnreadPackagesStateWritesNoFingerprintAndFingerprintsAreComputedAgain(t *
|
||||
}
|
||||
mods.put("mesh-sdk", raw)
|
||||
held, _ := ix.heldModules(ctx)
|
||||
if err := ix.publish(ctx, "novox/mesh-sdk", c1, held, false); err != nil {
|
||||
if err := ix.publish(ctx, "novox/mesh-sdk", c1, held, false, TrunkBranch); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ix.Recompute(ctx); err != nil {
|
||||
@@ -339,7 +339,7 @@ func anUnreadPackagesStateWritesNoFingerprintAndFingerprintsAreComputedAgain(t *
|
||||
t.Fatal(err)
|
||||
}
|
||||
held, _ = ix.heldModules(ctx)
|
||||
if err := ix.publish(ctx, "novox/mesh-sdk", c3, held, false); err != nil {
|
||||
if err := ix.publish(ctx, "novox/mesh-sdk", c3, held, false, TrunkBranch); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ix.Recompute(ctx); err != nil {
|
||||
|
||||
@@ -93,16 +93,20 @@ func TestTheKeptTrunkIsReadAtTheStartAndOncePerHeardRevision(t *testing.T) {
|
||||
if e, ok, err := modules.Get("ca-trust"); err != nil || !ok || e.Revision != 2 {
|
||||
t.Errorf("after the event ca-trust = %+v %v (%v), want it at revision 2", e, ok, err)
|
||||
}
|
||||
// A-487c: the manifests by digest and the verb revision asked of the trunk's newest answer from the view held.
|
||||
// A-487c: the manifests by digest answer from the view held; the verb revision asked of the trunk's newest lists the bucket
|
||||
// once for which revision is newest (H-A-222) and takes the entries of the view held at that very revision.
|
||||
listed = objects.n.Load()
|
||||
if raw, err := kept.Manifest(meshcatalogclient.BareManifestDigest(keptDigest("ca-trust"))); err != nil || raw != `{"module":"ca-trust"}` {
|
||||
t.Errorf("the held manifest = %q (%v), want the entry's manifest", raw, err)
|
||||
}
|
||||
if got := objects.n.Load() - listed; got != 0 {
|
||||
t.Errorf("the manifest listed the bucket %d times, want none", got)
|
||||
}
|
||||
if a, err := Revision(context.Background(), objects, kept, meshcatalog.RevisionInput{}); err != nil || a.Revision.Revision != 2 || len(a.Entries) != 2 {
|
||||
t.Errorf("revision of the newest answered %+v (%v), want revision 2 whole", a, err)
|
||||
}
|
||||
if got := objects.n.Load() - listed; got != 0 {
|
||||
t.Errorf("the manifest and the verb revision listed the bucket %d times, want none", got)
|
||||
if got := objects.n.Load() - listed; got != 1 {
|
||||
t.Errorf("the verb revision listed the bucket %d times, want once, for the newest", got)
|
||||
}
|
||||
if len(said) != 2 || !strings.Contains(said[1], "read revision 2 (was 1), 2 modules, in ") {
|
||||
t.Errorf("said %q, want each read with its revision, module count and time", said)
|
||||
@@ -126,6 +130,16 @@ func TestTheKeptTrunkIsReadAtTheStartAndOncePerHeardRevision(t *testing.T) {
|
||||
if keys, _ := modules.Keys(); len(keys) != 3 {
|
||||
t.Errorf("the next heard event left the view at %v, want revision 3's three modules", keys)
|
||||
}
|
||||
// H-A-222: two revisions kept that the view has not heard are answered by the verb revision from the store, whole; the
|
||||
// view held stays a cache at its own revision.
|
||||
keep(4, "avahi", "ca-trust", "dbus")
|
||||
keep(5, "avahi", "ca-trust", "dbus", "feh")
|
||||
if a, err := Revision(context.Background(), objects, kept, meshcatalog.RevisionInput{}); err != nil || a.Revision == nil || a.Revision.Revision != 5 || len(a.Entries) != 4 {
|
||||
t.Errorf("revision of the newest answered %+v (%v) while the view held 3, want revision 5 whole from the store", a, err)
|
||||
}
|
||||
if keys, _ := modules.Keys(); len(keys) != 3 {
|
||||
t.Errorf("the verb moved the view held to %v, want it at revision 3 until it hears the event", keys)
|
||||
}
|
||||
}
|
||||
|
||||
// A-487: the verb module reads the one module it is asked.
|
||||
|
||||
@@ -156,7 +156,7 @@ func (ix *Indexer) applyMoving(ctx context.Context, f whole, drop bool, moves ma
|
||||
// 0593 §3). A forge that cannot list them leaves the state as it was, said: the importers move at the next merge.
|
||||
reach := maps.Clone(held)
|
||||
maps.Copy(reach, overlay)
|
||||
if err := ix.publish(ctx, f.Repository, f.Merge, reach, false); err != nil {
|
||||
if err := ix.publish(ctx, f.Repository, f.Merge, reach, false, f.Branch); err != nil {
|
||||
ix.logf("%s at %s: its packages are not published: %v", f.Repository, f.Merge, err)
|
||||
}
|
||||
// What this merge moved reaches the modules standing on it (a base's tree, build-agent's fingerprint), in the graph's order.
|
||||
|
||||
@@ -60,8 +60,9 @@ func (t *goModTree) sameRepositoryDir(required string) (string, bool) {
|
||||
}
|
||||
|
||||
// sameRepositoryRequired are the directories (from the root, sorted, each once) of every same-repository Go module the
|
||||
// go.mod of each start directory requires, transitively through those modules' own go.mod files (H-A-202, H-A-157). The
|
||||
// start directories themselves are not among them.
|
||||
// go.mod of each start directory requires, transitively through those modules' own go.mod files (H-A-202, H-A-157). A
|
||||
// same-repository module a go.mod names in a replace directive counts as required, whether or not a require line names it
|
||||
// (H-A-224). The start directories themselves are not among them.
|
||||
func (t *goModTree) sameRepositoryRequired(start []string) []string {
|
||||
seen := map[string]bool{}
|
||||
for _, s := range start {
|
||||
@@ -72,7 +73,8 @@ func (t *goModTree) sameRepositoryRequired(start []string) []string {
|
||||
for len(queue) > 0 {
|
||||
dir := queue[0]
|
||||
queue = queue[1:]
|
||||
for _, req := range fingerprint.MeshRequiresOf(t.goMod(dir)) {
|
||||
raw := t.goMod(dir)
|
||||
for _, req := range append(fingerprint.MeshRequiresOf(raw), fingerprint.ReplacedSiblings(raw, t.f.Repository)...) {
|
||||
d, ok := t.sameRepositoryDir(req)
|
||||
if !ok || seen[d] {
|
||||
continue
|
||||
|
||||
@@ -2,6 +2,7 @@ package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -141,4 +142,23 @@ func TestTheEntryPinsEveryRequirementOfTheModuleGraph(t *testing.T) {
|
||||
if _, pinned := pins[builds]; pinned || len(pins) != 1 {
|
||||
t.Errorf("pins = %v, want %s only, never a module of the repository", pins, conditions)
|
||||
}
|
||||
// H-A-224: a same-repository module the go.mod names only in a replace directive (the watchman's buildrun) counts as
|
||||
// required: its own requirement of another repository's client is pinned too.
|
||||
const assigner = "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
|
||||
objects["modules/d/buildrun"] = strings.Repeat("d", 40)
|
||||
ix, mods, _ = ptpIndexer(t, objects)
|
||||
mod["modules/g"] += "\nreplace git.novox.be/novox/mesh-delivery/modules/d/buildrun => ../d/buildrun\n"
|
||||
mod["modules/d/buildrun"] = "module git.novox.be/novox/mesh-delivery/modules/d/buildrun\n\nrequire " + assigner + ptpZero + "\n"
|
||||
ix.GoMod = func(_ context.Context, _, _, dir string) ([]byte, error) {
|
||||
if c, ok := mod[dir]; ok {
|
||||
return []byte(c), nil
|
||||
}
|
||||
return []byte(ptpGoMods[dir]), nil
|
||||
}
|
||||
if err := ix.Apply(context.Background(), f); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m, _ := moduleOf(t, mods, "g"); m.Inputs2 == nil || !slices.Contains(m.Inputs2.Packages, assigner) || !slices.Contains(m.Inputs2.Packages, conditions) {
|
||||
t.Errorf("packages = %+v, want the replaced sibling's requirement %s pinned beside the client's", m.Inputs2, assigner)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,6 +14,8 @@ import (
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
|
||||
|
||||
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk"
|
||||
@@ -138,8 +140,14 @@ func reached(held map[string]meshcatalog.ModulesByModuleValue) (modules, imports
|
||||
// directory, else by the Go module whose go.mod is at its directory or the nearest above it (asked of the forge
|
||||
// directory by directory). A module the listing has no go.mod for is said and left out.
|
||||
func (ix *Indexer) packagesAt(ctx context.Context, repository, commit string, modules, imports []string) (map[string]meshcatalog.PackagesByModuleValue, error) {
|
||||
out, _, err := ix.packagesIn(ctx, repository, commit, modules, imports)
|
||||
return out, err
|
||||
}
|
||||
|
||||
// packagesIn is packagesAt, with the modules whose directory the tree at the commit does not hold (gone), by module path.
|
||||
func (ix *Indexer) packagesIn(ctx context.Context, repository, commit string, modules, imports []string) (map[string]meshcatalog.PackagesByModuleValue, []string, error) {
|
||||
if ix.Files == nil || commit == "" {
|
||||
return nil, nil
|
||||
return nil, nil, nil
|
||||
}
|
||||
dirOf := map[string]string{} // module path to its directory in the repository
|
||||
for _, m := range modules {
|
||||
@@ -182,7 +190,7 @@ func (ix *Indexer) packagesAt(ctx context.Context, repository, commit string, mo
|
||||
_, dir, _ := trunk.PackageRepository(imp)
|
||||
for d := dir; ; d = parent(d) {
|
||||
if err := list([]string{d}); err != nil {
|
||||
return nil, err
|
||||
return nil, nil, err
|
||||
}
|
||||
if _, ok := listings[d]["go.mod"]; ok {
|
||||
m := strings.TrimSuffix(trunk.ForgeModulePrefix+strings.TrimPrefix(repository, "novox/")+"/"+d, "/")
|
||||
@@ -201,27 +209,40 @@ func (ix *Indexer) packagesAt(ctx context.Context, repository, commit string, mo
|
||||
absent := map[string]bool{}
|
||||
dirs := slices.Sorted(maps.Values(dirOf))
|
||||
if err := list(dirs); err != nil {
|
||||
if !errors.Is(err, ErrNotInTree) || len(dirs) < 2 {
|
||||
return nil, err
|
||||
if !errors.Is(err, ErrNotInTree) {
|
||||
return nil, nil, err
|
||||
}
|
||||
whole := err
|
||||
for _, d := range dirs {
|
||||
if err := list([]string{d}); err != nil {
|
||||
if len(dirs) > 1 {
|
||||
err = list([]string{d})
|
||||
}
|
||||
if err != nil {
|
||||
if !errors.Is(err, ErrNotInTree) {
|
||||
return nil, err
|
||||
return nil, nil, err
|
||||
}
|
||||
ix.logf("%s at %s: the directory %q is not in the tree, so the module there is not read: %v", repository, commit, d, err)
|
||||
absent[d] = true
|
||||
}
|
||||
}
|
||||
// Every directory asked is missing: gone only when the commit holds a tree at all (the seat's trees of the commit, no
|
||||
// files), else the tree itself fails (H-A-232, H-A-233).
|
||||
if len(absent) == len(dirs) {
|
||||
return nil, whole // the tree itself fails
|
||||
if ix.Trunk == nil {
|
||||
return nil, nil, whole
|
||||
}
|
||||
if _, err := ix.Trunk(ctx, repository, commit); err != nil {
|
||||
return nil, nil, fmt.Errorf("%w (and the commit's trees: %v)", whole, err)
|
||||
}
|
||||
ix.logf("%s at %s: the commit holds a tree, so every directory asked (%s) is gone", repository, commit, strings.Join(dirs, ", "))
|
||||
}
|
||||
}
|
||||
out := map[string]meshcatalog.PackagesByModuleValue{}
|
||||
var gone []string
|
||||
for _, m := range slices.Sorted(maps.Keys(dirOf)) {
|
||||
dir := dirOf[m]
|
||||
if absent[dir] {
|
||||
gone = append(gone, m)
|
||||
continue
|
||||
}
|
||||
requires, pkgs, err := fingerprint.Packages(m, listings[dir])
|
||||
@@ -236,7 +257,7 @@ func (ix *Indexer) packagesAt(ctx context.Context, repository, commit string, mo
|
||||
out[m] = meshcatalog.PackagesByModuleValue{Module: m, Repository: trunk.RepositoryName(repository), Dir: d, Commit: commit,
|
||||
Requires: requires, Packages: pkgs}
|
||||
}
|
||||
return out, nil
|
||||
return out, gone, nil
|
||||
}
|
||||
|
||||
// parent is the directory above d, "" for the repository's root.
|
||||
@@ -252,8 +273,10 @@ func parent(d string) string {
|
||||
}
|
||||
|
||||
// publish writes the packages state for the Go modules of repository the held modules (and the merge's overlay) reach,
|
||||
// as the merge commit holds them: a key whose commit, requirements and digests are what it holds already is left.
|
||||
func (ix *Indexer) publish(ctx context.Context, repository, commit string, held map[string]meshcatalog.ModulesByModuleValue, onlyMissing bool) error {
|
||||
// as the merge commit holds them: a key whose commit, requirements and digests are what it holds already is left. The kept
|
||||
// key of a package whose directory the commit does not hold is dropped at the trunk's merge on the real indexer only, never
|
||||
// at a branch's take-in (branch) or in a dry run, which share the live packages state (H-A-232 (1)).
|
||||
func (ix *Indexer) publish(ctx context.Context, repository, commit string, held map[string]meshcatalog.ModulesByModuleValue, onlyMissing bool, branch string) error {
|
||||
if ix.Packages == nil || ix.Files == nil {
|
||||
return nil
|
||||
}
|
||||
@@ -263,6 +286,7 @@ func (ix *Indexer) publish(ctx context.Context, repository, commit string, held
|
||||
return nil
|
||||
}
|
||||
have := ix.publishedAll(ctx)
|
||||
drop := !ix.isDry && isTrunkBranch(branch)
|
||||
if onlyMissing {
|
||||
// Imports a held module already resolves are not asked again.
|
||||
var ms, is []string
|
||||
@@ -287,10 +311,28 @@ func (ix *Indexer) publish(ctx context.Context, repository, commit string, held
|
||||
modules[repo] = append(modules[repo], m)
|
||||
}
|
||||
}
|
||||
got, err := ix.packagesAt(ctx, repo, commit, modules[repo], imports[repo])
|
||||
got, gone, err := ix.packagesIn(ctx, repo, commit, modules[repo], imports[repo])
|
||||
if err != nil {
|
||||
ix.notPublished(ctx, repo, commit, err)
|
||||
return err
|
||||
}
|
||||
ix.published(ctx, repo, commit)
|
||||
// A repository's packages are those in the merge's tree (design 47 H H-A-232): the kept key of a package whose directory
|
||||
// the trunk's merge no longer holds is dropped, said, so the forge is never asked for it again.
|
||||
for _, m := range gone {
|
||||
if _, kept := have[m]; !kept || !drop {
|
||||
continue
|
||||
}
|
||||
key, err := PackagesKey(m)
|
||||
if err != nil {
|
||||
ix.logf("%s: %v", m, err)
|
||||
continue
|
||||
}
|
||||
if err := write(ctx, ix.Packages, key, func(_ []byte, found bool) ([]byte, bool, error) { return nil, found, nil }); err != nil {
|
||||
return fmt.Errorf("packages.%s: %w", key, err)
|
||||
}
|
||||
ix.logf("package %s of %s: its directory is gone at %s: its kept key dropped", m, repo, commit)
|
||||
}
|
||||
for _, m := range slices.Sorted(maps.Keys(got)) {
|
||||
next := got[m]
|
||||
key, err := PackagesKey(m)
|
||||
@@ -316,6 +358,36 @@ func (ix *Indexer) publish(ctx context.Context, repository, commit string, held
|
||||
return nil
|
||||
}
|
||||
|
||||
// PackagesNotPublishedKind is the finding of a repository whose packages a take-in's publish could not list, for a reason
|
||||
// that is not a gone directory (design 47 H H-A-233, H-A-234): scope merge, id the repository, with the forge's words;
|
||||
// raised at once and cleared at the repository's next publish that lists.
|
||||
const PackagesNotPublishedKind = "packages-not-published"
|
||||
|
||||
// notPublished raises the finding packages-not-published of repository.
|
||||
func (ix *Indexer) notPublished(ctx context.Context, repository, commit string, cause error) {
|
||||
if ix.Findings == nil {
|
||||
return
|
||||
}
|
||||
f := ix.observed(mcc.ScopeMerge, repository, PackagesNotPublishedKind,
|
||||
fmt.Sprintf("the packages of %s are not published at %.12s", repository, commit),
|
||||
"A repository's packages are not published",
|
||||
fmt.Sprintf("The take-in of %s at %s could not list its packages (%v), so their digests stay at their last published commit and a change to one does not move its importers.", repository, commit, cause),
|
||||
"indexer."+PackagesNotPublishedKind)
|
||||
if err := ix.Findings.Write(ctx, f); err != nil {
|
||||
ix.logf("%s: the finding %s is not written: %v", repository, PackagesNotPublishedKind, err)
|
||||
}
|
||||
}
|
||||
|
||||
// published clears the finding packages-not-published of repository: its publish listed.
|
||||
func (ix *Indexer) published(ctx context.Context, repository, commit string) {
|
||||
if ix.Findings == nil {
|
||||
return
|
||||
}
|
||||
if err := ix.clearFinding(ctx, mcc.ScopeMerge, repository, PackagesNotPublishedKind, "its packages are listed at "+commit); err != nil {
|
||||
ix.logf("%s: the finding %s is not cleared: %v", repository, PackagesNotPublishedKind, err)
|
||||
}
|
||||
}
|
||||
|
||||
// FillPackages publishes, at the repository's taken merge (the trunk the modules say), each reached Go module the
|
||||
// packages state does not hold yet: the state of a new indexer, or of a module a build newly reported an import of.
|
||||
// A repository with no taken merge is left for its next merge.
|
||||
@@ -349,7 +421,7 @@ func (ix *Indexer) FillPackages(ctx context.Context) error {
|
||||
continue
|
||||
}
|
||||
began := time.Now()
|
||||
if err := ix.publish(ctx, repo, t.Merge, held, true); err != nil {
|
||||
if err := ix.publish(ctx, repo, t.Merge, held, true, TrunkBranch); err != nil {
|
||||
ix.logf("%s: its packages are not published at %s: %v", repo, t.Merge, err)
|
||||
}
|
||||
ix.logf("publishing the packages at start: %s at %.12s took %s", repo, t.Merge, time.Since(began).Round(time.Millisecond))
|
||||
|
||||
@@ -0,0 +1,166 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
)
|
||||
|
||||
// Design 47 H H-A-232 (1), H-A-233, H-A-234: a repository's packages are those in the merge's tree. The kept key of a
|
||||
// package whose directory the trunk's merge no longer holds is dropped, said, and the repository's other packages are
|
||||
// published all the same (issue 1585's one-directory fallback); a branch's take-in and a dry run, which share the live
|
||||
// packages state, drop nothing. A repository whose every asked directory is gone, at a commit that holds a tree, drops them
|
||||
// and raises nothing; a listing that fails for another reason, or a commit with no tree, raises the finding
|
||||
// packages-not-published (scope merge, id the repository), cleared at the next publish that lists.
|
||||
func TestARepositorysPackagesAreThoseInTheMergesTree(t *testing.T) {
|
||||
const (
|
||||
live = "git.novox.be/novox/mesh-catalog/modules/disk-load/diskreadings"
|
||||
gone = "git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep"
|
||||
liveDir = "modules/disk-load/diskreadings"
|
||||
goneDir = "modules/mesh-build-go/buildstep"
|
||||
)
|
||||
ctx := context.Background()
|
||||
old, merge := strings.Repeat("d", 40), strings.Repeat("e", 40)
|
||||
reaching := func(m string) map[string]meshcatalog.ModulesByModuleValue {
|
||||
return map[string]meshcatalog.ModulesByModuleValue{
|
||||
"user": {Repository: catalogueRepo, Inputs2: &meshcatalog.ModulesByModuleValueInputs2{Packages: []string{m}}},
|
||||
}
|
||||
}
|
||||
type setup struct {
|
||||
ix *Indexer
|
||||
findings *memFindings
|
||||
said *[]string
|
||||
down *error // the forge's listing fails for a reason that is not a gone directory
|
||||
noTree *error // the seat's trees of the commit, without files, fail
|
||||
}
|
||||
newSetup := func(t *testing.T, kept ...string) setup {
|
||||
t.Helper()
|
||||
var down, noTree error
|
||||
said := []string{}
|
||||
files := func(_ context.Context, repository, sha string, dirs []string) (map[string]map[string]string, error) {
|
||||
if down != nil {
|
||||
return nil, down
|
||||
}
|
||||
out := map[string]map[string]string{}
|
||||
for _, d := range dirs {
|
||||
if d != liveDir {
|
||||
// as filesVia says the seat's not-found naming the directory
|
||||
return nil, fmt.Errorf("the directory %s is not in %s at %s: %w", d, repository, sha, ErrNotInTree)
|
||||
}
|
||||
out[d] = map[string]string{"go.mod": "m1", "readings.go": "r-" + sha[:1]}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
trees := func(context.Context, string, string) ([]string, error) { return []string{"modules/disk-load"}, noTree }
|
||||
findings := newFindings()
|
||||
ix := &Indexer{Module: testSelf, Packages: newMem(), Files: files, Trunk: trees, Findings: findings,
|
||||
Logf: func(format string, args ...any) { said = append(said, fmt.Sprintf(format, args...)) }}
|
||||
dirOf := map[string]string{live: liveDir, gone: goneDir}
|
||||
for _, m := range kept {
|
||||
key, err := PackagesKey(m)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
raw, err := EncodePublishedPackages(meshcatalog.PackagesByModuleValue{Module: m, Repository: catalogueRepo, Dir: dirOf[m], Commit: old,
|
||||
Requires: "req", At: ptr(time.Unix(0, 0).UTC()), Packages: []meshcatalog.PackagesByModuleValuePackagesItem{{Package: m, Digest: "x"}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ix.Packages.(*memStore).put(key, raw)
|
||||
}
|
||||
return setup{ix: ix, findings: findings, said: &said, down: &down, noTree: &noTree}
|
||||
}
|
||||
notPublished := func(s setup) bool {
|
||||
for _, f := range s.findings.Held() {
|
||||
if f.Kind == PackagesNotPublishedKind && f.Scope == mcc.ScopeMerge && f.ID == keyID(catalogueRepo) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
dropped := "package " + gone + " of " + catalogueRepo + ": its directory is gone at " + merge + ": its kept key dropped"
|
||||
|
||||
t.Run("the trunk's merge drops a gone package's key and publishes the others; a failed listing is a finding until one lists", func(t *testing.T) {
|
||||
s := newSetup(t, live, gone)
|
||||
*s.down = errors.New("seat:git.trees (failed): the forge did not answer")
|
||||
if err := s.ix.publish(ctx, catalogueRepo, merge, reaching(live), false, TrunkBranch); err == nil {
|
||||
t.Fatal("a listing that failed published")
|
||||
}
|
||||
if !notPublished(s) {
|
||||
t.Fatalf("a listing that failed raised no finding %s: %v", PackagesNotPublishedKind, s.findings.Held())
|
||||
}
|
||||
*s.down = nil
|
||||
if err := s.ix.publish(ctx, catalogueRepo, merge, reaching(live), false, TrunkBranch); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
have := s.ix.publishedAll(ctx)
|
||||
if have[live].Commit != merge {
|
||||
t.Errorf("packages of %s at %q, want the merge %s: a gone sibling kept it from being published", live, have[live].Commit, merge)
|
||||
}
|
||||
if _, kept := have[gone]; kept {
|
||||
t.Errorf("the kept key of %s, whose directory the merge does not hold, stays", gone)
|
||||
}
|
||||
if !strings.Contains(strings.Join(*s.said, "\n"), dropped) {
|
||||
t.Errorf("the journal does not say %q:\n%s", dropped, strings.Join(*s.said, "\n"))
|
||||
}
|
||||
if notPublished(s) {
|
||||
t.Error("the finding packages-not-published stays after a publish that listed")
|
||||
}
|
||||
})
|
||||
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
dry bool
|
||||
branch string
|
||||
}{
|
||||
{"a branch's take-in keeps the key: only the trunk's merge says a directory is gone", false, "feat/x"},
|
||||
{"a dry run keeps the key: it shares the live packages state", true, TrunkBranch},
|
||||
} {
|
||||
t.Run(row.name, func(t *testing.T) {
|
||||
s := newSetup(t, live, gone)
|
||||
ix := s.ix
|
||||
if row.dry {
|
||||
ix = s.ix.dry().ix
|
||||
}
|
||||
if err := ix.publish(ctx, catalogueRepo, merge, reaching(live), false, row.branch); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, kept := s.ix.publishedAll(ctx)[gone]; !kept {
|
||||
t.Errorf("the kept key of %s was dropped", gone)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
t.Run("a one-key repository whose directory is gone drops it and raises nothing, when the commit holds a tree", func(t *testing.T) {
|
||||
s := newSetup(t, gone)
|
||||
if err := s.ix.publish(ctx, catalogueRepo, merge, reaching(gone), false, TrunkBranch); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, kept := s.ix.publishedAll(ctx)[gone]; kept {
|
||||
t.Errorf("the kept key of %s, the repository's one package, whose directory is gone, stays", gone)
|
||||
}
|
||||
if notPublished(s) {
|
||||
t.Error("a gone directory raised packages-not-published")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("every asked directory missing at a commit with no tree is the tree's failure: the keys stay and it is a finding", func(t *testing.T) {
|
||||
s := newSetup(t, gone)
|
||||
*s.noTree = errors.New("seat:git.trees (not-found): Gitea API GET: 404 Not Found")
|
||||
if err := s.ix.publish(ctx, catalogueRepo, merge, reaching(gone), false, TrunkBranch); err == nil {
|
||||
t.Fatal("a commit with no tree published")
|
||||
}
|
||||
if _, kept := s.ix.publishedAll(ctx)[gone]; !kept {
|
||||
t.Errorf("the kept key of %s was dropped at a commit with no tree", gone)
|
||||
}
|
||||
if !notPublished(s) {
|
||||
t.Error("a commit with no tree raised no finding")
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -23,17 +23,22 @@ const RevisionInvoke = "seat:" + meshcatalog.Seat + "." + meshcatalog.VerbRevisi
|
||||
// H-A-66: every account may call every verb). With no trunk revision kept yet the answer is `none-yet` alone (design 47 G §530), never an empty
|
||||
// catalog and never an error.
|
||||
//
|
||||
// kept, when not nil, is the trunk's newest revision held in memory: the trunk's newest answers from it with no read of
|
||||
// the bucket (A-487c); a candidate, a branch and an older number read the bucket.
|
||||
// The trunk's newest is the newest the bucket keeps at the time of the ask, read from the store (a listing of the names and
|
||||
// one object, design 47 H-A-222), never the view held in memory, which runs behind until it hears the revision. kept, when
|
||||
// not nil, is that view: a cache of the entries only, used when it holds the very revision read; else the entries are read
|
||||
// from the bucket. A candidate, a branch and an older number read the bucket.
|
||||
func Revision(ctx context.Context, o Objects, kept *KeptTrunk, in meshcatalog.RevisionInput) (meshcatalog.RevisionAnswer, error) {
|
||||
var out meshcatalog.RevisionAnswer
|
||||
var newest meshcatalog.CatalogRevision
|
||||
var ok bool
|
||||
var trunkEntries, branchEntries []meshcatalog.CatalogEntry
|
||||
var err error
|
||||
if kept != nil {
|
||||
newest, trunkEntries, ok, err = kept.Newest(ctx, o)
|
||||
} else {
|
||||
newest, ok, err := newestOf(ctx, o, TrunkBranch)
|
||||
cached := false
|
||||
if err == nil && ok && kept != nil {
|
||||
if held, entries, any, kerr := kept.Newest(ctx, o); kerr == nil && any && held.Revision == newest.Revision {
|
||||
trunkEntries, cached = entries, true
|
||||
}
|
||||
}
|
||||
if !cached && (err != nil || ok) {
|
||||
// readNewestOf reads the entries whole, and the newest again when the pin judge removed one under the read.
|
||||
newest, trunkEntries, ok, err = readNewestOf(ctx, o, TrunkBranch)
|
||||
}
|
||||
if err != nil {
|
||||
|
||||
@@ -74,9 +74,7 @@ func (r *Registry) ComposeSeed(modules []SeedModule) (SeedProposal, error) {
|
||||
return out, fmt.Errorf("the default branch of %s is not read: %s", m.Origin, b.Error)
|
||||
}
|
||||
reg := meshcatalog.Registration{Module: m.Module, Origin: m.Origin, Path: m.Path, Branches: []string{b.Branch},
|
||||
Policy: meshcatalog.Policy{Steps: meshcatalog.StepStatuses{Lint: "required", Unit: "required", Integration: "required",
|
||||
Composition: "required", Review: "optional", Approval: "bypass",
|
||||
Validate: "required"}}} // design 47 H §745: the step validate is required in every policy by default
|
||||
Policy: meshcatalog.Policy{Steps: seedSteps()}} // design 47 H §745: the step validate is required in every policy by default
|
||||
for _, a := range m.Artifacts {
|
||||
var s meshcatalog.StepStatuses
|
||||
if !slices.Contains(a.Steps, "lint") {
|
||||
@@ -124,7 +122,7 @@ func (r *Registry) ProposeSeed(ctx context.Context, modules []SeedModule, again
|
||||
return "", err
|
||||
}
|
||||
if len(seed.Unanswered) > 0 {
|
||||
return "", fmt.Errorf("the build seats %v have not answered steps: the seed waits for them (call-unanswered after five minutes)", seed.Unanswered)
|
||||
return "", fmt.Errorf("the build seats %v have not answered steps: the seed waits for them (indexer-call-unanswered after five minutes)", seed.Unanswered)
|
||||
}
|
||||
regs, left = seed.Registrations, seed.Left
|
||||
}
|
||||
|
||||
@@ -57,7 +57,7 @@ func buildSeatOf(defs map[string]seatdef.Definition, a meshcatalog.Artifact) (st
|
||||
// SeedModulesFrom reads the modules the newest trunk revision holds for the seed (design 47 G §704, §8): for each artifact of
|
||||
// each module the seat list's build seat answers steps with {module, artifact, fingerprint, fetch} from the module's trunk
|
||||
// entry, the fetch its repository, commit, directory and the artifact's named paths. A seat that does not answer leaves
|
||||
// its artifact unanswered, which holds the seed; its call-unanswered finding (scope call, id <seat>.steps) is written once
|
||||
// its artifact unanswered, which holds the seed; its indexer-call-unanswered finding (scope call, id <seat>.steps) is written once
|
||||
// it is unanswered five minutes and cleared when it answers.
|
||||
func (ix *Indexer) SeedModulesFrom(ctx context.Context, defs map[string]seatdef.Definition, steps StepsCaller) ([]SeedModule, error) {
|
||||
if ix.Revisions == nil {
|
||||
@@ -202,7 +202,7 @@ func artifactName(ref string) string {
|
||||
return ref
|
||||
}
|
||||
|
||||
// stepsFindings writes call-unanswered for each seat unanswered for five minutes and clears it for each that answered.
|
||||
// stepsFindings writes indexer-call-unanswered for each seat unanswered for five minutes and clears it for each that answered.
|
||||
func (ix *Indexer) stepsFindings(ctx context.Context, answered map[string]bool, unanswered map[string]error) {
|
||||
if ix.Findings == nil {
|
||||
return
|
||||
@@ -223,11 +223,11 @@ func (ix *Indexer) stepsFindings(ctx context.Context, answered map[string]bool,
|
||||
}
|
||||
for _, seat := range slices.Sorted(slices.Values(raise)) {
|
||||
id := seat + "." + StepsVerb
|
||||
f := ix.observed(mcc.ScopeCall, id, mcc.KindCallUnanswered,
|
||||
f := ix.observed(mcc.ScopeCall, id, CallUnansweredKind,
|
||||
fmt.Sprintf("the call %s is unanswered since %s", id, since[seat].UTC().Format(time.RFC3339)),
|
||||
"A call whose answer is kept is not answered",
|
||||
fmt.Sprintf("The seed asks %s of every artifact its build seat builds; it is not answered since %s (%v), so the first registrations wait for it.", id, since[seat].UTC().Format(time.RFC3339), unanswered[seat]),
|
||||
"indexer."+mcc.KindCallUnanswered)
|
||||
"indexer."+CallUnansweredKind)
|
||||
if err := ix.Findings.Write(ctx, f); err != nil {
|
||||
ix.logf("seed: the finding %s is not written: %v", id, err)
|
||||
}
|
||||
@@ -236,7 +236,7 @@ func (ix *Indexer) stepsFindings(ctx context.Context, answered map[string]bool,
|
||||
if _, bad := unanswered[seat]; bad {
|
||||
continue
|
||||
}
|
||||
if err := ix.clearFinding(ctx, mcc.ScopeCall, seat+"."+StepsVerb, mcc.KindCallUnanswered, "the seat answers steps"); err != nil {
|
||||
if err := ix.clearFinding(ctx, mcc.ScopeCall, seat+"."+StepsVerb, CallUnansweredKind, "the seat answers steps"); err != nil {
|
||||
ix.logf("seed: the finding %s.%s is not cleared: %v", seat, StepsVerb, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,7 +15,7 @@ import (
|
||||
|
||||
// Design 47 G §704: the seed asks `steps` of the build seat whose `build` rule matches each artifact, with {module,
|
||||
// artifact, fingerprint, fetch} from the newest trunk entry; a seat that does not answer holds the seed and, after five
|
||||
// minutes, is the finding call-unanswered (scope call, id <seat>.steps), cleared when it answers.
|
||||
// minutes, is the finding indexer-call-unanswered (scope call, id <seat>.steps), cleared when it answers.
|
||||
func TestTheSeedAsksStepsOfTheBuildSeatsOfTheTrunkEntries(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
now := time.Date(2026, 11, 1, 3, 0, 0, 0, time.UTC)
|
||||
@@ -85,7 +85,7 @@ func TestTheSeedAsksStepsOfTheBuildSeatsOfTheTrunkEntries(t *testing.T) {
|
||||
if _, err := ix.SeedModulesFrom(ctx, defs, call); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if h := w.Held(); len(h) != 1 || h[0].Scope != mcc.ScopeCall || h[0].ID != "node-build-go.steps" || h[0].Kind != mcc.KindCallUnanswered {
|
||||
if h := w.Held(); len(h) != 1 || h[0].Scope != mcc.ScopeCall || h[0].ID != "node-build-go.steps" || h[0].Kind != CallUnansweredKind {
|
||||
t.Fatalf("the findings after five minutes: %+v", h)
|
||||
}
|
||||
goUp = true
|
||||
|
||||
@@ -22,7 +22,7 @@ import (
|
||||
// filled per module whenever the indexer starts and at each catalog revision kept, not once (A-490d, A-490f): the key
|
||||
// versions-filled closes only the builder's half of the fill.
|
||||
|
||||
// FloorCall is the id of the finding call-unanswered when a floor read (the forge's tags or a package holder's versions) is
|
||||
// FloorCall is the id of the finding indexer-call-unanswered when a floor read (the forge's tags or a package holder's versions) is
|
||||
// unanswered five minutes after the first (A-490h).
|
||||
const FloorCall = "mesh-catalog.versions-floor"
|
||||
|
||||
@@ -91,7 +91,7 @@ func (ix *Indexer) FloorModules(ctx context.Context) ([]string, error) {
|
||||
// FillFloor fills one module's floor, the highest version among its packages' tags and holders' versions (A-490g). Its reads are
|
||||
// kept calls made through mesh-sdk's form, which asks again through every transport fault; a read it gives up on (its own
|
||||
// pause spent) is asked again at once, never given up, so the read has no bound (A-490i). The first unanswered try starts the
|
||||
// module's clock: five minutes later the finding call-unanswered is written, and cleared when a read answers. A read the
|
||||
// module's clock: five minutes later the finding indexer-call-unanswered is written, and cleared when a read answers. A read the
|
||||
// callee answered with a failure ends the fill, left for the next start or revision. One fill of a module runs at a time.
|
||||
func (ix *Indexer) FillFloor(ctx context.Context, module string, src FloorSources) error {
|
||||
if _, running := ix.floors.LoadOrStore(module, true); running {
|
||||
@@ -198,7 +198,7 @@ func FloorSinceKey(module string) string { return floorSinceKey(module) }
|
||||
|
||||
func floorSinceKey(module string) string { return "call-since." + keys.EscapeName(floorKey(module)) }
|
||||
|
||||
// floorUnanswered writes call-unanswered once the module's floor read is unanswered five minutes (the time kept in the own state).
|
||||
// floorUnanswered writes indexer-call-unanswered once the module's floor read is unanswered five minutes (the time kept in the own state).
|
||||
func (ix *Indexer) floorUnanswered(ctx context.Context, module string, first time.Time, cause error) {
|
||||
now := ix.now()
|
||||
since := ix.unansweredSince(ctx, floorSinceKey(module), first)
|
||||
@@ -206,11 +206,11 @@ func (ix *Indexer) floorUnanswered(ctx context.Context, module string, first tim
|
||||
return
|
||||
}
|
||||
at := since.UTC().Format(time.RFC3339)
|
||||
f := ix.observed(mcc.ScopeCall, floorKey(module), mcc.KindCallUnanswered,
|
||||
f := ix.observed(mcc.ScopeCall, floorKey(module), CallUnansweredKind,
|
||||
fmt.Sprintf("the floor of %s's versions is unanswered since %s", module, at),
|
||||
"A call whose answer is kept is not answered",
|
||||
fmt.Sprintf("The floor of %s's versions reads the forge's tags and the package holders' versions; a read is not answered since %s (%v), so it has no version.", module, at, cause),
|
||||
"indexer."+mcc.KindCallUnanswered)
|
||||
"indexer."+CallUnansweredKind)
|
||||
if err := ix.Findings.Write(ctx, f); err != nil {
|
||||
ix.logf("floor: the finding %s is not written: %v", floorKey(module), err)
|
||||
}
|
||||
@@ -219,7 +219,7 @@ func (ix *Indexer) floorUnanswered(ctx context.Context, module string, first tim
|
||||
func (ix *Indexer) floorAnswered(ctx context.Context, module string) {
|
||||
ix.answered(ctx, floorSinceKey(module))
|
||||
if ix.Findings != nil {
|
||||
if err := ix.clearFinding(ctx, mcc.ScopeCall, floorKey(module), mcc.KindCallUnanswered, "the floor's reads answer"); err != nil {
|
||||
if err := ix.clearFinding(ctx, mcc.ScopeCall, floorKey(module), CallUnansweredKind, "the floor's reads answer"); err != nil {
|
||||
ix.logf("floor: the finding %s is not cleared: %v", floorKey(module), err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,6 +4,8 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
@@ -18,45 +20,48 @@ const (
|
||||
)
|
||||
|
||||
// TakeInAsk is a value of the takeins state: which hand act a person asked, why, who and when, and, once the indexer read
|
||||
// it, its answer.
|
||||
// it, its answer. Every field says by its tag takein what it is: an act (a field that alone names a hand act the indexer
|
||||
// serves), an argument of an act, or of the ask itself (version, who, why, when, the answer); DecodeTakeIn reads the acts
|
||||
// from these tags, so a hand act added here is read the moment its field is (H-A-208, H-A-210: indexer_take_in_again's
|
||||
// Again was refused by a hand-kept list).
|
||||
type TakeInAsk struct {
|
||||
V int `json:"v"`
|
||||
Module string `json:"module,omitempty"`
|
||||
Why string `json:"why,omitempty"`
|
||||
V int `json:"v" takein:"ask"`
|
||||
Module string `json:"module,omitempty" takein:"argument"`
|
||||
Why string `json:"why,omitempty" takein:"ask"`
|
||||
// SeatDefinitions asks a contracts module's trunk seat definitions to be read again now (indexer_seat_definitions_again,
|
||||
// DLV-I19).
|
||||
SeatDefinitions bool `json:"seat-definitions,omitempty"`
|
||||
SeatDefinitions bool `json:"seat-definitions,omitempty" takein:"act"`
|
||||
// Fingerprints asks the fingerprints of Module ("all": every module) to be computed again now, from the published
|
||||
// packages as they are read (indexer_fingerprints_again, fingerprints.go).
|
||||
Fingerprints bool `json:"fingerprints,omitempty"`
|
||||
Fingerprints bool `json:"fingerprints,omitempty" takein:"act"`
|
||||
// Recompute asks the catalog graph to be computed again now and compared with the newest kept revision (indexer_recompute).
|
||||
Recompute bool `json:"recompute,omitempty"`
|
||||
Recompute bool `json:"recompute,omitempty" takein:"act"`
|
||||
// CandidateGroup and CandidateHeads, with Drop, ask that candidate's catalog half to be dropped (indexer_drop_candidate_graph).
|
||||
CandidateGroup string `json:"candidate-group,omitempty"`
|
||||
CandidateHeads string `json:"candidate-heads,omitempty"`
|
||||
Drop bool `json:"drop,omitempty"`
|
||||
CandidateGroup string `json:"candidate-group,omitempty" takein:"act"`
|
||||
CandidateHeads string `json:"candidate-heads,omitempty" takein:"argument"`
|
||||
Drop bool `json:"drop,omitempty" takein:"argument"`
|
||||
// NamedFingerprint and NamedMember, with Drop, ask one member to be removed from that fingerprint's set of the state named
|
||||
// (indexer_named_drop, named.go, design 47 G §675).
|
||||
NamedFingerprint string `json:"named-fingerprint,omitempty"`
|
||||
NamedFingerprint string `json:"named-fingerprint,omitempty" takein:"act"`
|
||||
// PinDrop asks one pin of the state pins, by its key, to be dropped (indexer_pin_drop, design 47 H §765).
|
||||
PinDrop string `json:"pin-drop,omitempty"`
|
||||
PinDrop string `json:"pin-drop,omitempty" takein:"act"`
|
||||
// Held names the watched branch whose held source-moved is taken in again now (indexer_take_in_pending) or, with Skip and
|
||||
// HeldSource, ended unwritten (indexer_take_in_skip; design 47 H §744).
|
||||
Held string `json:"held,omitempty"`
|
||||
HeldSource int64 `json:"held-source,omitempty"`
|
||||
Skip bool `json:"skip,omitempty"`
|
||||
NamedMember string `json:"named-member,omitempty"`
|
||||
Held string `json:"held,omitempty" takein:"act"`
|
||||
HeldSource int64 `json:"held-source,omitempty" takein:"argument"`
|
||||
Skip bool `json:"skip,omitempty" takein:"argument"`
|
||||
NamedMember string `json:"named-member,omitempty" takein:"argument"`
|
||||
// Again names the repository taken in again at its kept merge commit (indexer_take_in_again, take_in_again.go; H-A-208, H-A-210).
|
||||
Again string `json:"again,omitempty"`
|
||||
Again string `json:"again,omitempty" takein:"act"`
|
||||
// RunDrop asks one run on request, by its group run-<id>, to be dropped with its candidate (indexer_run_drop, runs_tools.go, A-194).
|
||||
RunDrop string `json:"run-drop,omitempty"`
|
||||
RunDrop string `json:"run-drop,omitempty" takein:"act"`
|
||||
// Registry asks one of the registrations' acts (registrations.go): a trusted change of one, the verb registrations,
|
||||
// indexer_proposal_drop or indexer_seed_again (design 47 G §3, §4, §8, §667).
|
||||
Registry *RegistryAct `json:"registry,omitempty"`
|
||||
By string `json:"by,omitempty"`
|
||||
At time.Time `json:"at"`
|
||||
Result *TakenIn `json:"result,omitempty"`
|
||||
Answered time.Time `json:"answered,omitzero"`
|
||||
Registry *RegistryAct `json:"registry,omitempty" takein:"act"`
|
||||
By string `json:"by,omitempty" takein:"ask"`
|
||||
At time.Time `json:"at" takein:"ask"`
|
||||
Result *TakenIn `json:"result,omitempty" takein:"ask"`
|
||||
Answered time.Time `json:"answered,omitzero" takein:"ask"`
|
||||
}
|
||||
|
||||
// Done says the ask is answered.
|
||||
@@ -71,8 +76,33 @@ func DecodeTakeIn(data []byte) (TakeInAsk, error) {
|
||||
if a.V != 1 {
|
||||
return a, fmt.Errorf("a takeins value of version %d, not 1", a.V)
|
||||
}
|
||||
if a.Module == "" && !a.Fingerprints && !a.Recompute && a.CandidateGroup == "" && a.NamedFingerprint == "" && a.PinDrop == "" && a.Held == "" && a.RunDrop == "" && a.Registry == nil {
|
||||
if len(a.Acts()) == 0 {
|
||||
return a, errors.New("a takeins value naming no hand act the indexer serves")
|
||||
}
|
||||
return a, nil
|
||||
}
|
||||
|
||||
// takeInActs are the indices of TakeInAsk's fields tagged takein:"act".
|
||||
var takeInActs = func() []int {
|
||||
var out []int
|
||||
t := reflect.TypeFor[TakeInAsk]()
|
||||
for i := range t.NumField() {
|
||||
if t.Field(i).Tag.Get("takein") == "act" {
|
||||
out = append(out, i)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}()
|
||||
|
||||
// Acts names the hand acts the ask names, by their json names: its fields tagged takein:"act" that are set.
|
||||
func (a TakeInAsk) Acts() []string {
|
||||
var out []string
|
||||
v := reflect.ValueOf(a)
|
||||
for _, i := range takeInActs {
|
||||
if !v.Field(i).IsZero() {
|
||||
name, _, _ := strings.Cut(v.Type().Field(i).Tag.Get("json"), ",")
|
||||
out = append(out, name)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -63,7 +63,7 @@ func versionNameOf(module string, a meshcatalog.Artifact) string {
|
||||
|
||||
// FillVersions fills the state versions once from builder_versions and says versions-filled (§372). A fill already made
|
||||
// is not made again; a builder_versions that does not answer holds the fill and, after five minutes, is the finding
|
||||
// call-unanswered (scope call, id mesh-delivery-builder.builder_versions), cleared when it answers.
|
||||
// indexer-call-unanswered (scope call, id mesh-delivery-builder.builder_versions), cleared when it answers.
|
||||
func (ix *Indexer) FillVersions(ctx context.Context, ask VersionsAsker) error {
|
||||
if ix.Versions == nil || ix.Heard == nil {
|
||||
return errors.New("the state versions is not kept here")
|
||||
@@ -127,7 +127,7 @@ func (ix *Indexer) FillVersions(ctx context.Context, ask VersionsAsker) error {
|
||||
}
|
||||
ix.answered(ctx, versionsFillSince)
|
||||
if ix.Findings != nil {
|
||||
if err := ix.clearFinding(ctx, mcc.ScopeCall, BuilderVersionsCall, mcc.KindCallUnanswered, "builder_versions answers"); err != nil {
|
||||
if err := ix.clearFinding(ctx, mcc.ScopeCall, BuilderVersionsCall, CallUnansweredKind, "builder_versions answers"); err != nil {
|
||||
ix.logf("versions fill: the finding %s is not cleared: %v", BuilderVersionsCall, err)
|
||||
}
|
||||
}
|
||||
@@ -184,18 +184,18 @@ func (ix *Indexer) FillVersions(ctx context.Context, ask VersionsAsker) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// fillUnanswered writes call-unanswered once builder_versions is unanswered five minutes (the time kept in the own state).
|
||||
// fillUnanswered writes indexer-call-unanswered once builder_versions is unanswered five minutes (the time kept in the own state).
|
||||
func (ix *Indexer) fillUnanswered(ctx context.Context, cause error) {
|
||||
now := ix.now()
|
||||
since := ix.unansweredSince(ctx, versionsFillSince, now)
|
||||
if ix.Findings == nil || now.Sub(since) < mcc.CallUnansweredAfter {
|
||||
return
|
||||
}
|
||||
f := ix.observed(mcc.ScopeCall, BuilderVersionsCall, mcc.KindCallUnanswered,
|
||||
f := ix.observed(mcc.ScopeCall, BuilderVersionsCall, CallUnansweredKind,
|
||||
fmt.Sprintf("the call %s is unanswered since %s", BuilderVersionsCall, since.UTC().Format(time.RFC3339)),
|
||||
"A call whose answer is kept is not answered",
|
||||
fmt.Sprintf("The first fill of the catalog's versions asks %s of every artifact; it is not answered since %s (%v), so no version is released and the builder reserves nothing.", BuilderVersionsCall, since.UTC().Format(time.RFC3339), cause),
|
||||
"indexer."+mcc.KindCallUnanswered)
|
||||
"indexer."+CallUnansweredKind)
|
||||
if err := ix.Findings.Write(ctx, f); err != nil {
|
||||
ix.logf("versions fill: the finding %s is not written: %v", BuilderVersionsCall, err)
|
||||
}
|
||||
|
||||
@@ -87,7 +87,7 @@ func TestAVersionIsReleasedOnceEveryArtifactIsPublished(t *testing.T) {
|
||||
|
||||
// Design 47 G §372, A-350: the first fill records, once, each module's highest version the builder gave a number, numbered or
|
||||
// published, over its artifacts as released and says versions-filled once; a builder_versions that does not answer holds it and, after five minutes, is the finding
|
||||
// call-unanswered; a fill made is not made again.
|
||||
// indexer-call-unanswered; a fill made is not made again.
|
||||
func TestTheFirstFillOfTheVersionsComesFromTheBuildersNumbers(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
now := time.Date(2026, 11, 1, 3, 0, 0, 0, time.UTC)
|
||||
@@ -501,7 +501,7 @@ func TestAnInputIsDeletedAtTheStepEndedOfItsAsk(t *testing.T) {
|
||||
|
||||
// Design 47 G §412 as A-490c, A-490d and A-490e amend it: a module the versions hold none of is filled from the highest tag of its
|
||||
// Go package in the repository it was taken in from, per module and not once; a forge that does not answer fills nothing and
|
||||
// raises call-unanswered after five minutes.
|
||||
// raises indexer-call-unanswered after five minutes.
|
||||
func TestTheTagFloorFillsAModuleWithNoVersionFromItsGoTags(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
now := time.Date(2026, 11, 19, 17, 0, 0, 0, time.UTC)
|
||||
@@ -623,7 +623,7 @@ func TestAFloorEntryIsReplacedByThePublishHeardAtItsVersion(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// ADR 0668 §3, design 47 G A-490i: the finding call-unanswered of a module's floor clears, with its call-since key, on every way the
|
||||
// ADR 0668 §3, design 47 G A-490i: the finding indexer-call-unanswered of a module's floor clears, with its call-since key, on every way the
|
||||
// fill ends but the end of its context: a release gave the module a version, the module left the trunk, or the callee answered with a
|
||||
// failure (FloorModules never picks the module again to clear it).
|
||||
func TestTheFloorFindingClearsOnEveryExitOfTheFill(t *testing.T) {
|
||||
|
||||
@@ -0,0 +1,155 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
nodeengine "git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
|
||||
condseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
|
||||
wants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
|
||||
"mesh-delivery-pipeline/internal/pipeline"
|
||||
)
|
||||
|
||||
// envAskUnansweredBound names the file the manifest writes the setting ask-unanswered-bound into, `${config:<key>}`
|
||||
// (design 47 H H-A-238 (1)).
|
||||
const envAskUnansweredBound = "MESH_RUNNER_ASK_UNANSWERED_BOUND_FILE"
|
||||
|
||||
// envAskSilentBound names the file of the setting ask-silent-bound (H-A-243 (2)).
|
||||
const envAskSilentBound = "MESH_RUNNER_ASK_SILENT_BOUND_FILE"
|
||||
|
||||
// askUnansweredBound reads the setting ask-unanswered-bound at the start, a Go duration (H-A-235, H-A-238 (1)): its default
|
||||
// ten minutes when the file is missing or empty, an error naming the file for a content that is no positive duration.
|
||||
func askUnansweredBound(getenv func(string) string) (time.Duration, error) {
|
||||
return durationSetting(getenv, envAskUnansweredBound, pipeline.DefaultAskUnansweredBound)
|
||||
}
|
||||
|
||||
// askSilentBound reads the setting ask-silent-bound at the start, a Go duration, forty-five minutes by default (H-A-243 (2)).
|
||||
func askSilentBound(getenv func(string) string) (time.Duration, error) {
|
||||
return durationSetting(getenv, envAskSilentBound, pipeline.DefaultAskSilentBound)
|
||||
}
|
||||
|
||||
func durationSetting(getenv func(string) string, env string, def time.Duration) (time.Duration, error) {
|
||||
v, err := settingText(getenv(env))
|
||||
if err != nil || v == "" {
|
||||
return def, err
|
||||
}
|
||||
d, err := time.ParseDuration(v)
|
||||
if err != nil || d <= 0 {
|
||||
return 0, fmt.Errorf("%s holds %q, which is not a positive duration", getenv(env), v)
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
|
||||
// askClock is the bound of every ask but a test run's over the state bounds (H-A-238 (5)): a key armed with its own time to
|
||||
// live before the ask is put, deleted at its holder's first word, its expiry heard by watchBounds; the finding call-unanswered,
|
||||
// scope call, id the ask, raised at the second bound and cleared when the ask is answered after all (H-A-238 (2)).
|
||||
func askClock(ctx context.Context, bounds jetstream.KeyValue, bound, silent time.Duration, w callFindings, module string, now func() time.Time) *pipeline.AskClock {
|
||||
return &pipeline.AskClock{Bound: bound, Silent: silent,
|
||||
Arm: func(key string, ttl time.Duration) error {
|
||||
_, err := bounds.Create(ctx, key, []byte(`{"ask":"`+key+`"}`), jetstream.KeyTTL(ttl))
|
||||
if errors.Is(err, jetstream.ErrKeyExists) {
|
||||
return nil // armed already: the same ask put again at a start keeps its first bound
|
||||
}
|
||||
return err
|
||||
},
|
||||
// A first word is heard for every ask, most of them unbounded or answered already: a key not there is deleted of
|
||||
// nothing and writes no marker.
|
||||
Unbound: func(key string) error {
|
||||
if _, err := bounds.Get(ctx, key); errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return nil
|
||||
} else if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := bounds.Delete(ctx, key); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
},
|
||||
Raise: func(onSeat, ask string, within time.Duration) error {
|
||||
if w == nil {
|
||||
return errors.New("the runner has no finding writer")
|
||||
}
|
||||
return w.Write(ctx, condseat.Finding{Scope: mcc.ScopeCall, ID: ask, Kind: mcc.KindCallUnanswered, Severity: condseat.FindingSeverityWarning,
|
||||
Summary: fmt.Sprintf("the ask %s of the seat %s is unanswered", ask, onSeat),
|
||||
Headline: "A step's ask is not answered",
|
||||
Explanation: fmt.Sprintf("The runner put the ask %s on the seat %s and heard no word of its holder about it within %s, so its step ended could not run and its pipeline went on (H-A-235, H-A-243).", ask, onSeat, within),
|
||||
Source: module + "." + mcc.KindCallUnanswered, Observer: module, At: now().UTC(), Subject: condseat.Subject{Scope: mcc.ScopeCall, ID: ask}})
|
||||
},
|
||||
Clear: func(_, ask string) error {
|
||||
if w == nil {
|
||||
return nil
|
||||
}
|
||||
return w.Clear(ctx, mcc.ScopeCall, ask, mcc.KindCallUnanswered, "the ask is answered")
|
||||
}}
|
||||
}
|
||||
|
||||
// answered deletes the bounds of the ask on onSeat at its holder's first word, said when it fails: the bound then passes and
|
||||
// its expiry finds the step answered or asks it once more.
|
||||
func answered(c *pipeline.AskClock, onSeat, ask string, logf func(string, ...any)) {
|
||||
if err := c.Answered(onSeat, ask); err != nil {
|
||||
logf("the bounds of ask %s on %s are not deleted at its holder's word: %v", ask, onSeat, err)
|
||||
}
|
||||
}
|
||||
|
||||
// hearWantsProgress takes a step progress of the seat mesh-wants as its holder's first word on the ask it names (H-A-238 (3)).
|
||||
func hearWantsProgress(hub handlers, c *pipeline.AskClock, logf func(string, ...any)) error {
|
||||
return hub.Handle(wants.SubjectStepProgressAll, func(m jetstream.Msg) {
|
||||
var ev wants.StepProgressEvent
|
||||
if err := json.Unmarshal(m.Data(), &ev); err != nil || ev.Ask == "" {
|
||||
logf("%s is no step-progress: %v", m.Subject(), err)
|
||||
_ = m.Term()
|
||||
return
|
||||
}
|
||||
answered(c, wants.Seat, ev.Ask, logf)
|
||||
_ = m.Ack()
|
||||
})
|
||||
}
|
||||
|
||||
// holdersNotReadDelay is how long a running heard before the seat-holders view is read waits before it is heard again.
|
||||
const holdersNotReadDelay = 2 * time.Second
|
||||
|
||||
// hearHolderStarted puts again the open asks of each silent seat whose holder the node-engine says running (H-A-243 (1)).
|
||||
func hearHolderStarted(hub handlers, started func(module string) (int, error), logf func(string, ...any)) error {
|
||||
return hub.Handle(nodeengine.SubjectRunningAll, func(m jetstream.Msg) {
|
||||
var ev nodeengine.Running
|
||||
if err := json.Unmarshal(m.Data(), &ev); err != nil || ev.Module == "" {
|
||||
logf("%s is no running: %v", m.Subject(), err)
|
||||
_ = m.Term()
|
||||
return
|
||||
}
|
||||
if ev.Stopped {
|
||||
_ = m.Ack()
|
||||
return
|
||||
}
|
||||
n, err := started(ev.Module)
|
||||
switch {
|
||||
case errors.Is(err, pipeline.ErrHoldersNotRead):
|
||||
_ = m.NakWithDelay(holdersNotReadDelay) // heard again once the view is read, without spinning until then
|
||||
return
|
||||
case err != nil:
|
||||
logf("the asks of the seats %s holds, put again as it runs on %s: %d, and one is not: %v", ev.Module, ev.Node, n, err)
|
||||
}
|
||||
_ = m.Ack()
|
||||
})
|
||||
}
|
||||
|
||||
// askBoundExpired takes the expiry of an ask's bound (H-A-235): the first asks once more, the second ends its step.
|
||||
func askBoundExpired(unanswered func(onSeat, ask string, again bool) error, logf func(string, ...any)) func(key string) bool {
|
||||
return func(key string) bool {
|
||||
s, ask, again, ok := pipeline.ParseAskBound(key)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
if err := unanswered(s, ask, again); err != nil {
|
||||
logf("the bound of ask %s on %s passed and is not taken: %v", ask, s, err)
|
||||
}
|
||||
return true
|
||||
}
|
||||
}
|
||||
@@ -72,8 +72,8 @@ func dropUnanswered(ctx context.Context, w callFindings, module string, now func
|
||||
}
|
||||
}
|
||||
|
||||
// boundExpired takes the expiry of a key of the state bounds: a drop's says the drop unanswered (A-222); another key is none
|
||||
// of the runner's, an ask having no bound of its own (§535, §647 as A-466y amends them).
|
||||
// boundExpired takes the expiry of a drop's key of the state bounds: it says the drop unanswered (A-222); an ask's key is
|
||||
// taken by askBoundExpired (H-A-235).
|
||||
func boundExpired(unanswered func(buildSeat, ask string) error, logf func(string, ...any)) func(key string) {
|
||||
return func(key string) {
|
||||
if s, ask, ok := pipeline.ParseDropBound(key); ok {
|
||||
|
||||
@@ -2,9 +2,12 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"slices"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
|
||||
condseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
|
||||
|
||||
@@ -50,3 +53,25 @@ func TestADropBoundsExpirySaysCallUnanswered(t *testing.T) {
|
||||
t.Errorf("an answered drop: %v, cleared %v, unbound %v", err, w.cleared, unbound)
|
||||
}
|
||||
}
|
||||
|
||||
// H-A-238 (5), review: at a start, a first bound's expiry is taken only when the first read holds no entry for its second
|
||||
// bound (put again and taken: a delete marker; put again: a live put; both passed: an expiry, the second taken alone); a
|
||||
// second bound's expiry is taken; a delete marker or a drop's expiry is none.
|
||||
func TestMissedBoundsAtTheStart(t *testing.T) {
|
||||
const s = "node-build-go"
|
||||
key := func(a string) string { return pipeline.AskBoundKey(s, a) }
|
||||
again := func(a string) string { return pipeline.AskAgainBoundKey(s, pipeline.AgainID(a)) }
|
||||
last := map[string]jetstream.KeyValueOp{
|
||||
key("lostaaaa000001"): jetstream.KeyValuePurge, // passed while down, never asked again: taken
|
||||
key("takenaaa000002"): jetstream.KeyValuePurge, again("takenaaa000002"): jetstream.KeyValueDelete, // asked again and taken
|
||||
key("askedaaa000003"): jetstream.KeyValuePurge, again("askedaaa000003"): jetstream.KeyValuePut, // asked again, bound live
|
||||
key("bothaaaa000004"): jetstream.KeyValuePurge, again("bothaaaa000004"): jetstream.KeyValuePurge, // both passed
|
||||
key("wordaaaa000005"): jetstream.KeyValueDelete, // a first word
|
||||
pipeline.DropBoundKey(s, "dropaaaa000006"): jetstream.KeyValuePurge,
|
||||
}
|
||||
want := []string{key("lostaaaa000001"), again("bothaaaa000004")}
|
||||
slices.Sort(want)
|
||||
if got := missedBounds(last); !slices.Equal(got, want) {
|
||||
t.Errorf("missedBounds = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -67,8 +67,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// The bound of each superseded step's drop: a key with its own time to live, its expiry heard (A-222, A-77).
|
||||
bounds, err := process.EssentialBucket(ctx, js, "pipeline runner", ownBucket(cred.Module, "bounds"), "bound a drop no holder answers")
|
||||
// The bound of each superseded step's drop and of each ask: a key with its own time to live, its expiry heard (A-222, A-77;
|
||||
// H-A-238 (5)).
|
||||
bounds, err := process.EssentialBucket(ctx, js, "pipeline runner", ownBucket(cred.Module, "bounds"), "bound a drop or an ask no holder answers")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -158,8 +159,24 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
starter := &pipeline.Starter{Pipelines: store, Logf: logger.Printf, Say: say,
|
||||
Registration: rv.Registrations.Registration, Steps: busStepAsker{ctx: ctx, js: js, cred: cred}, Name: cred.Module,
|
||||
Kept: keptReader(ctx, kept), Toolchain: toolchainReader(ctx, toolchains),
|
||||
Fills: fills, Recovered: kvRecoveries{ctx, recovered}.Has, Holders: rv.Holders.Holders, Ended: hearer.EndedAtPlacing}
|
||||
Fills: fills, Recovered: kvRecoveries{ctx, recovered}.Has, Holders: rv.Holders.Holders, Ended: hearer.EndedAtPlacing,
|
||||
Findings: rv.Findings, Observer: cred.Module} // a mover waiting on its registration is registration-missing (H-A-262)
|
||||
hearer.AskSteps = starter.AskSteps
|
||||
hearer.RegistrationWaits = func() { starter.JudgeRegistrationMissing(store) } // at every end (H-A-265 (4))
|
||||
// Every ask but a test run's is bounded until its holder's first word (H-A-235, H-A-238), its bounds keys of the state
|
||||
// bounds.
|
||||
askBound, err := askUnansweredBound(getenv)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
silentBound, err := askSilentBound(getenv)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
logger.Printf("asks: an ask unanswered bound %s, asked once more, then its step ends could not run; an ask of a seat that says nothing before its end bound %s", askBound, silentBound)
|
||||
clock := askClock(ctx, bounds, askBound, silentBound, rv.Findings, cred.Module, time.Now)
|
||||
starter.Asks, hearer.Asks = clock, clock
|
||||
starter.Supersede = func(newer seat.Pipeline) { hearer.SupersedeRunning(newer) }
|
||||
hearer.Unplaced = func(p seat.Pipeline, module string) string { return starter.Unplaced(p, module, rv.Catalog.View) }
|
||||
hearer.AskPinHeld = func(key, module string, pins map[string]string) {
|
||||
if _, err := starter.AskPinHeld(hearer.Store, key, module, pins, widener.Revision); err != nil {
|
||||
@@ -296,10 +313,26 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
}
|
||||
return hearer.RunTaken(ask, node)
|
||||
}
|
||||
if err := hearAskTaken(heard, taken, logger.Printf); err != nil {
|
||||
if err := hearAskTaken(heard, taken, clock, logger.Printf); err != nil {
|
||||
return err
|
||||
}
|
||||
expired := boundExpired(dropUnanswered(ctx, rv.Findings, cred.Module, time.Now), logger.Printf)
|
||||
if err := hearWantsProgress(heard, clock, logger.Printf); err != nil {
|
||||
return err
|
||||
}
|
||||
// The seat's write verbs are answered by the process that holds the pipelines (A-107); made before the bounds are heard,
|
||||
// since an ask's expired bound asks it once more through them (H-A-235).
|
||||
verbs := writeVerbs(hearer, starter, func(branch string) (pipeline.Half, bool) { return rv.Catalog.View(branch) }, calls)
|
||||
verbs.Runs = pipeline.RunsRead{Runs: runsKept{ctx: ctx, kv: runsKV}, Object: runObjs.Get, Fill: starter.TestRunsFill}
|
||||
if err := hearHolderStarted(heard, verbs.HolderStarted, logger.Printf); err != nil {
|
||||
return err
|
||||
}
|
||||
dropExpired := boundExpired(dropUnanswered(ctx, rv.Findings, cred.Module, time.Now), logger.Printf)
|
||||
askExpired := askBoundExpired(verbs.AskUnanswered, logger.Printf)
|
||||
expired := func(key string) {
|
||||
if !askExpired(key) {
|
||||
dropExpired(key)
|
||||
}
|
||||
}
|
||||
if err := watchBounds(ctx, bounds, expired, logger.Printf); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -335,9 +368,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// The steps' ends are kept seat events, each on its own stream (step-ended of the build seats, mesh-builds and
|
||||
// mesh-wants), never on EVENTS: bound here, or no step of any pipeline is ever heard ending (G-1 §24).
|
||||
logger.Printf("the kept seat events heard: %d streams bound", hub.StartKept(ctx, js, cred))
|
||||
// The seat's write verbs are answered here, by the process that holds the pipelines (A-107).
|
||||
verbs := writeVerbs(hearer, starter, func(branch string) (pipeline.Half, bool) { return rv.Catalog.View(branch) }, calls)
|
||||
verbs.Runs = pipeline.RunsRead{Runs: runsKept{ctx: ctx, kv: runsKV}, Object: runObjs.Get, Fill: starter.TestRunsFill}
|
||||
stopVerbs, err := answerVerbs(calls, seats, verbs)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
+69
-1
@@ -107,6 +107,17 @@ func TestTheQueueRunsPipelinesInTurn(t *testing.T) {
|
||||
const router = "operator-router"
|
||||
answer("mesh.seat."+seatlist.HolderSeat+".call."+seatlist.HolderVerb+".*", map[string]any{"seats": []any{
|
||||
map[string]any{"seat": oc.Seat, "holders": []any{map[string]any{"module": router}}}}})
|
||||
// Each mesh-builds step ask as its holder hears it, on its plain subscription.
|
||||
var buildAsksMu sync.Mutex
|
||||
buildAsks := map[string]int{}
|
||||
if _, err := nc.Subscribe(builds.AcceptSubjectStep, func(m *nats.Msg) {
|
||||
buildAsksMu.Lock()
|
||||
buildAsks[m.Header.Get(nats.MsgIdHdr)]++
|
||||
buildAsksMu.Unlock()
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
buildAsked := func(id string) int { buildAsksMu.Lock(); defer buildAsksMu.Unlock(); return buildAsks[id] }
|
||||
records, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: router + "_" + oc.StateAsks, History: 1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -153,6 +164,8 @@ func TestTheQueueRunsPipelinesInTurn(t *testing.T) {
|
||||
if err := os.WriteFile(waitingCap, []byte("1\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
askBound := filepath.Join(dir, "ask-unanswered-bound")
|
||||
silentBound := filepath.Join(dir, "ask-silent-bound")
|
||||
logs := &syncBuffer{}
|
||||
start := func() func() {
|
||||
runCtx, stop := context.WithCancel(ctx)
|
||||
@@ -164,6 +177,10 @@ func TestTheQueueRunsPipelinesInTurn(t *testing.T) {
|
||||
return broker
|
||||
case envPipelinesWaitingCap:
|
||||
return waitingCap
|
||||
case envAskUnansweredBound:
|
||||
return askBound
|
||||
case envAskSilentBound:
|
||||
return silentBound
|
||||
}
|
||||
return ""
|
||||
}, logs)
|
||||
@@ -359,9 +376,16 @@ func TestTheQueueRunsPipelinesInTurn(t *testing.T) {
|
||||
}
|
||||
|
||||
// At the runner's next start the waiting main.10 is placed from its own state, saying wait-ended, which clears
|
||||
// pipelines-stalled; the start begins no pipeline of its own.
|
||||
// pipelines-stalled; the start begins no pipeline of its own. That start's ask-unanswered-bound is one second, its
|
||||
// ask-silent-bound four.
|
||||
stopRunner()
|
||||
setMaxMsgSize(t, ctx, js, "KV_"+module+"_"+seat.StateQueue, -1)
|
||||
if err := os.WriteFile(askBound, []byte("1s\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(silentBound, []byte("4s\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := pipelineKeys(t, ctx, pipelines)
|
||||
stopRunner = start()
|
||||
outcome("main.10", seat.PipelineOutcomeRunning)
|
||||
@@ -370,6 +394,50 @@ func TestTheQueueRunsPipelinesInTurn(t *testing.T) {
|
||||
if after := pipelineKeys(t, ctx, pipelines); !slices.Equal(before, after) {
|
||||
t.Fatalf("the start wrote pipelines %v, want %v", after, before)
|
||||
}
|
||||
|
||||
// H-A-243: main.10's build ask on mesh-builds, whose holder says nothing before its end, is never asked again for
|
||||
// ask-unanswered-bound (a build running four of those bounds is not touched); its one bound,
|
||||
// ask-silent-bound, ends its step could not run "its ask <id> had no end", the pipeline ending failed, its turn freed and
|
||||
// the finding call-unanswered naming the ask.
|
||||
build := stepFor(get("main.10"), "app", seat.StepResultStepBuild)
|
||||
if build.Ask == "" || build.Started == nil {
|
||||
t.Fatalf("main.10's build = %+v, want it asked", build)
|
||||
}
|
||||
// Review of H-A-238 (5): the bound expires while the runner is down; at its next start the expiry marker it left is
|
||||
// read and taken at once, never a fresh bound's full length later.
|
||||
boundsKV, err := js.KeyValue(ctx, module+"_bounds")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitFor(t, ctx, logs, func() bool {
|
||||
_, err := boundsKV.Get(ctx, pipeline.AskBoundKey(builds.Seat, build.Ask))
|
||||
return err == nil
|
||||
})
|
||||
stopRunner()
|
||||
waitFor(t, ctx, logs, func() bool {
|
||||
_, err := boundsKV.Get(ctx, pipeline.AskBoundKey(builds.Seat, build.Ask))
|
||||
return errors.Is(err, jetstream.ErrKeyNotFound)
|
||||
})
|
||||
restarted := time.Now()
|
||||
stopRunner = start()
|
||||
outcome("main.10", seat.PipelineOutcomeFailed)
|
||||
if e := stepFor(get("main.10"), "app", seat.StepResultStepBuild).Ended; e == nil || e.Sub(restarted) >= 3*time.Second {
|
||||
t.Fatalf("main.10's build ended %v, the runner restarted at %v; want its missed bound taken at the start", e, restarted)
|
||||
}
|
||||
if !strings.Contains(logs.String(), "expired while the runner was away: taken now") {
|
||||
t.Fatal("the start did not say the bound it took")
|
||||
}
|
||||
s := stepFor(get("main.10"), "app", seat.StepResultStepBuild)
|
||||
if s.Outcome != seat.StepResultOutcomeCouldNotRun || s.Why != "its ask "+build.Ask+" had no end" || s.Ended == nil || s.Ended.Sub(*build.Started) < 4*time.Second {
|
||||
t.Fatalf("main.10's build = %+v, want could not run, its ask with no end, at its silent bound of 4s", s)
|
||||
}
|
||||
if strings.Contains(logs.String(), ": asked once more as ") || buildAsked(pipeline.AgainID(build.Ask)) != 0 {
|
||||
t.Fatalf("the silent build's ask was asked again for ask-unanswered-bound (put %d times, again %d)", buildAsked(build.Ask), buildAsked(pipeline.AgainID(build.Ask)))
|
||||
}
|
||||
waitFor(t, ctx, logs, func() bool {
|
||||
e, err := findings.Get(ctx, "call."+build.Ask+".call-unanswered")
|
||||
return err == nil && strings.Contains(string(e.Value()), build.Ask) && len(running()) == 0
|
||||
})
|
||||
}
|
||||
|
||||
func stepFor(p seat.Pipeline, module, step string) seat.StepResult {
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
@@ -141,6 +142,8 @@ func hearStepEnded(hub handlers, h *pipeline.Hearer, logf func(string, ...any))
|
||||
logf("the step-ended of ask %s is not taken yet, heard again: %v", ev.ID, err)
|
||||
_ = m.Nak()
|
||||
default:
|
||||
// The step's end is its holder's word on the ask, the first for mesh-builds (H-A-238 (3)).
|
||||
answered(h.Asks, bs.Seat, ev.ID, logf)
|
||||
_ = m.Ack()
|
||||
}
|
||||
})
|
||||
@@ -153,8 +156,9 @@ func hearStepEnded(hub handlers, h *pipeline.Hearer, logf func(string, ...any))
|
||||
|
||||
// hearAskTaken keeps on its step the node whose build seat holder took an ask (§602, §647; A-182), so a drop of it is asked of
|
||||
// that node's holder, and begins the clock of a test run's ask (§725).
|
||||
func hearAskTaken(hub handlers, taken func(ask, node string) error, logf func(string, ...any)) error {
|
||||
func hearAskTaken(hub handlers, taken func(ask, node string) error, c *pipeline.AskClock, logf func(string, ...any)) error {
|
||||
for _, ts := range pipeline.TakenSeats {
|
||||
ts := ts
|
||||
err := hub.Handle(ts.Subject, func(m jetstream.Msg) {
|
||||
var ev nodebuildgo.AskTaken
|
||||
if err := json.Unmarshal(m.Data(), &ev); err != nil || ev.Ask == "" {
|
||||
@@ -167,6 +171,8 @@ func hearAskTaken(hub handlers, taken func(ask, node string) error, logf func(st
|
||||
_ = m.Nak()
|
||||
return
|
||||
}
|
||||
// The seat's ask-taken is its holder's first word on the ask (H-A-238 (3)).
|
||||
answered(c, ts.Seat, ev.Ask, logf)
|
||||
_ = m.Ack()
|
||||
})
|
||||
if err != nil {
|
||||
@@ -176,14 +182,17 @@ func hearAskTaken(hub handlers, taken func(ask, node string) error, logf func(st
|
||||
return nil
|
||||
}
|
||||
|
||||
// watchBounds hears the end of each drop's bound, the bus's marker of the key's expiry, and says a drop no holder answered
|
||||
// unanswered (A-222, A-77): no timer in the process.
|
||||
// watchBounds hears the end of each drop's and each ask's bound, the bus's marker of the key's expiry (A-222, A-77; H-A-235,
|
||||
// H-A-238 (5)): no timer in the process. An ask's bound that expired while the runner was down left its marker before the
|
||||
// watch began: the watch's first entries, each key's last, are kept to their end, and the ask bounds missedBounds names are
|
||||
// taken as expired then. Then every expiry is heard as it comes.
|
||||
func watchBounds(ctx context.Context, bounds jetstream.KeyValue, expired func(key string), logf func(string, ...any)) error {
|
||||
w, err := bounds.WatchAll(ctx, jetstream.UpdatesOnly())
|
||||
w, err := bounds.WatchAll(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
go func() {
|
||||
last := map[string]jetstream.KeyValueOp{} // each key's last entry, until the first read ends
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
@@ -193,16 +202,49 @@ func watchBounds(ctx context.Context, bounds jetstream.KeyValue, expired func(ke
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if e == nil || e.Operation() != jetstream.KeyValuePurge {
|
||||
if e == nil {
|
||||
for _, key := range missedBounds(last) {
|
||||
logf("the bound %s expired while the runner was away: taken now", key)
|
||||
expired(key)
|
||||
}
|
||||
last = nil // the first read is over: what follows is new
|
||||
continue
|
||||
}
|
||||
expired(e.Key())
|
||||
if last != nil {
|
||||
last[e.Key()] = e.Operation()
|
||||
continue
|
||||
}
|
||||
if e.Operation() == jetstream.KeyValuePurge {
|
||||
expired(e.Key())
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
}
|
||||
|
||||
// missedBounds are the ask bounds the first read found expired that are still to be taken (H-A-238 (5)): a second bound's
|
||||
// expiry always; a first bound's only when the read holds no entry at all for its second bound's key, since a live second
|
||||
// bound, its expiry or its delete (the holder's ask-taken) says the ask was asked again already. A delete marker is a first
|
||||
// word, never an expiry; a drop's bound is not taken here.
|
||||
func missedBounds(last map[string]jetstream.KeyValueOp) []string {
|
||||
var out []string
|
||||
for key, op := range last {
|
||||
if op != jetstream.KeyValuePurge {
|
||||
continue
|
||||
}
|
||||
onSeat, ask, again, ok := pipeline.ParseAskBound(key)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if _, asked := last[pipeline.AskAgainBoundKey(onSeat, pipeline.AgainID(ask))]; again || !asked {
|
||||
out = append(out, key)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// keptReader reads the kept results of the packages named (§25, §416): the entries of the state kept whose package is a
|
||||
// name or lies beneath it, in one pass over the bucket.
|
||||
func keptReader(ctx context.Context, kept jetstream.KeyValue) func([]string) (map[string][]nodebuildgo.KeptResult, error) {
|
||||
|
||||
+30
-3
@@ -16,6 +16,7 @@ import (
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
nodeengine "git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
||||
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
@@ -99,6 +100,19 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) {
|
||||
if _, err := holders.Put(ctx, "node-build-go", []byte(`{"holders":[{"module":"mesh-build-go"}]}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// mesh-builds is held by the builder, which takes its asks on a plain subscription and says nothing before an end.
|
||||
if _, err := holders.Put(ctx, builds.Seat, []byte(`{"holders":[{"module":"mesh-delivery-builder"}]}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var buildAsksMu sync.Mutex
|
||||
buildAsks := map[string]int{}
|
||||
if _, err := nc.Subscribe(builds.AcceptSubjectStep, func(m *nats.Msg) {
|
||||
buildAsksMu.Lock()
|
||||
buildAsks[m.Header.Get(nats.MsgIdHdr)]++
|
||||
buildAsksMu.Unlock()
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// The registrations: app tests its program and tools bundle from one directory, its unit step required and its
|
||||
// integration step optional; web is built by node-build-node.
|
||||
@@ -331,6 +345,17 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) {
|
||||
const p3 = "main.1"
|
||||
takeIn("app", "c1", "src2:aa", 3, 1)
|
||||
placed(p3) // started by the take-in alone: the runner starts no pipeline by sweep (design 47 H §747)
|
||||
// H-A-243 (1): the node-engine's running of the builder puts its open build ask again, under its own id: an ask put while
|
||||
// it was down is lost; a running of a module holding no silent seat puts nothing again.
|
||||
firstBuild := step(read(p3), "", seat.StepResultStepBuild).Ask
|
||||
publish(nodeengine.SubjectRunning("n1", "mesh-build-go"), nodeengine.Running{Module: "mesh-build-go", Node: "n1", At: time.Now().UTC()})
|
||||
publish(nodeengine.SubjectRunning("n1", "mesh-delivery-builder"), nodeengine.Running{Module: "mesh-delivery-builder", Node: "n1", At: time.Now().UTC()})
|
||||
waitFor(t, ctx, logs, func() bool {
|
||||
buildAsksMu.Lock()
|
||||
n := buildAsks[firstBuild]
|
||||
buildAsksMu.Unlock()
|
||||
return n == 2 && strings.Count(logs.String(), "the ask "+firstBuild+" of the build step of "+p3+" is put again on mesh-builds") == 1
|
||||
})
|
||||
passBuild(p3, "app", "c1", "src2:aa")
|
||||
unitRun, unit := runOfStep(p3, "app:program", "unit")
|
||||
integRun, integ := runOfStep(p3, "app:program", "integration")
|
||||
@@ -350,7 +375,8 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) {
|
||||
if s := step(read(p3), "app:tools", "unit"); s.Outcome != seat.StepResultOutcomeWaitingForABuildNode || s.Ask != "" || s.Shared[0].Ask != unit.Ask {
|
||||
t.Fatalf("the tools' unit step = %+v; want it waiting for a build node, reading the program's run", s)
|
||||
}
|
||||
// Its wait has no bound (§647 as A-466y amends it): the state bounds holds no key for a queued ask, only drops' keys.
|
||||
// A test run's ask keeps §725's bounds alone (H-A-238 (4)): the state bounds holds no key naming it, only drops' and other
|
||||
// asks' keys.
|
||||
boundsKV, err := js.KeyValue(ctx, module+"_bounds")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -359,8 +385,9 @@ func TestTestRunsAreAskedOncePerLevelAndMoveEveryReader(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
} else {
|
||||
for _, k := range ks {
|
||||
if !strings.HasPrefix(k, pipeline.DropBoundPrefix) {
|
||||
t.Errorf("the state bounds holds %s after the asks were put; want no bound of a queued ask", k)
|
||||
_, ask, _, isAsk := pipeline.ParseAskBound(k)
|
||||
if (!isAsk && !strings.HasPrefix(k, pipeline.DropBoundPrefix)) || ask == unit.Ask || ask == integ.Ask {
|
||||
t.Errorf("the state bounds holds %s after the asks were put; want no bound of a test run's ask", k)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -10,7 +10,7 @@ require (
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-contracts/go/seats/nodeengine v0.0.0-00010101000000-000000000000 // indirect
|
||||
git.novox.be/novox/mesh-contracts/go/seats/nodeengine v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions v0.0.0-00010101000000-000000000000
|
||||
|
||||
@@ -0,0 +1,349 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
wants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
nodebuildgo "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
|
||||
)
|
||||
|
||||
// DefaultAskUnansweredBound is the setting ask-unanswered-bound's default (design 47 H H-A-235, H-A-238 (1)): ten minutes
|
||||
// from an ask until its holder's first word.
|
||||
const DefaultAskUnansweredBound = 10 * time.Minute
|
||||
|
||||
// DefaultAskSilentBound is the setting ask-silent-bound's default (H-A-243 (2)): forty-five minutes from an ask on a seat whose
|
||||
// holder says no word before the step's end (mesh-builds, mesh-wants) until that end.
|
||||
const DefaultAskSilentBound = 45 * time.Minute
|
||||
|
||||
// Silent says whether the holder of a seat the runner asks says nothing of an ask before its step's end: every seat but the
|
||||
// build seats, which say ask-taken (H-A-241, H-A-243).
|
||||
func Silent(onSeat string) bool { return !BoundSeat(onSeat) }
|
||||
|
||||
// AgainID is the id an ask is put again under (H-A-243 (3)): the first id's time part and six hex digits of the sha256 of the
|
||||
// first id and ".2", the form the holders accept, so no duplicate window is relied on.
|
||||
func AgainID(id string) string {
|
||||
if len(id) <= 6 {
|
||||
return id
|
||||
}
|
||||
sum := sha256.Sum256([]byte(id + ".2"))
|
||||
return id[:len(id)-6] + hex.EncodeToString(sum[:3])
|
||||
}
|
||||
|
||||
// The keys of the state bounds that time an ask the runner put (H-A-238 (5)): `ask.<seat>.<ask>` armed before the ask is put,
|
||||
// `ask2.<seat>.<ask>` armed when it is asked once more, each with the setting as its own time to live.
|
||||
const (
|
||||
AskBoundPrefix = "ask."
|
||||
AskAgainBoundPrefix = "ask2."
|
||||
)
|
||||
|
||||
// AskBoundKey is the key of the first bound of the ask on onSeat; AskAgainBoundKey that of its second.
|
||||
func AskBoundKey(onSeat, ask string) string { return AskBoundPrefix + onSeat + "." + ask }
|
||||
func AskAgainBoundKey(onSeat, ask string) string { return AskAgainBoundPrefix + onSeat + "." + ask }
|
||||
|
||||
// ParseAskBound reads a key of the state bounds as an ask's bound: the seat, the ask and whether it is the second bound.
|
||||
func ParseAskBound(key string) (onSeat, ask string, again, ok bool) {
|
||||
rest, found := strings.CutPrefix(key, AskAgainBoundPrefix)
|
||||
again = found
|
||||
if !found {
|
||||
if rest, found = strings.CutPrefix(key, AskBoundPrefix); !found {
|
||||
return "", "", false, false
|
||||
}
|
||||
}
|
||||
onSeat, ask, found = strings.Cut(rest, ".")
|
||||
return onSeat, ask, again, found && onSeat != "" && ask != "" && !strings.Contains(ask, ".")
|
||||
}
|
||||
|
||||
// UnansweredWhy is the why of a step whose ask was not answered within its two bounds (H-A-235).
|
||||
func UnansweredWhy(ask string) string { return "its ask " + ask + " was not answered" }
|
||||
|
||||
// SilentWhy is the why of a step on a silent seat whose ask had no end within ask-silent-bound (H-A-243 (2)).
|
||||
func SilentWhy(ask string) string { return "its ask " + ask + " had no end" }
|
||||
|
||||
// AskClock bounds every ask the runner puts on a seat but a test run's, which keeps §725's bounds (H-A-235, H-A-238 (4)):
|
||||
// Arm writes a key of the state bounds with Bound as its own time to live, Unbound deletes one; Raise raises the finding
|
||||
// call-unanswered of an ask that ended its step, Clear clears it once the ask is answered after all (H-A-238 (2)).
|
||||
type AskClock struct {
|
||||
// Bound is ask-unanswered-bound, for a seat that says ask-taken; Silent is ask-silent-bound, for a silent seat.
|
||||
Bound time.Duration
|
||||
Silent time.Duration
|
||||
Arm func(key string, ttl time.Duration) error
|
||||
Unbound func(key string) error
|
||||
Raise func(onSeat, ask string, bound time.Duration) error
|
||||
Clear func(onSeat, ask string) error
|
||||
}
|
||||
|
||||
// arm arms the first bound of a, before it is put (H-A-238 (5)); a test run's ask, and a nil clock, arm none.
|
||||
func (c *AskClock) arm(a Asked) error {
|
||||
if c == nil || c.Arm == nil || a.Run != "" {
|
||||
return nil
|
||||
}
|
||||
if err := c.Arm(AskBoundKey(a.Seat, a.ID), c.boundOf(a.Seat)); err != nil {
|
||||
return fmt.Errorf("the bound of ask %s is not armed: %w", a.ID, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// boundOf is the bound of an ask on onSeat: ask-silent-bound for a silent seat (H-A-243 (2)), else ask-unanswered-bound.
|
||||
func (c *AskClock) boundOf(onSeat string) time.Duration {
|
||||
if !Silent(onSeat) {
|
||||
return c.bound()
|
||||
}
|
||||
if c.Silent > 0 {
|
||||
return c.Silent
|
||||
}
|
||||
return DefaultAskSilentBound
|
||||
}
|
||||
|
||||
func (c *AskClock) bound() time.Duration {
|
||||
if c.Bound > 0 {
|
||||
return c.Bound
|
||||
}
|
||||
return DefaultAskUnansweredBound
|
||||
}
|
||||
|
||||
// Answered deletes the bounds of the ask heard on onSeat, its holder's first word: an ask-taken, a step progress or, for a
|
||||
// silent seat, the step's end (H-A-238 (3), H-A-241). Heard under a first id, the second bound of the ask put again under
|
||||
// AgainID goes too.
|
||||
func (c *AskClock) Answered(onSeat, ask string) error {
|
||||
if c == nil || c.Unbound == nil || ask == "" {
|
||||
return nil
|
||||
}
|
||||
return errors.Join(c.Unbound(AskBoundKey(onSeat, ask)), c.Unbound(AskAgainBoundKey(onSeat, ask)),
|
||||
c.Unbound(AskAgainBoundKey(onSeat, AgainID(ask))))
|
||||
}
|
||||
|
||||
// answeredLate clears the finding of the ask of step s, which ended unanswered or with no end, its holder heard after all
|
||||
// (H-A-238 (2)).
|
||||
func (c *AskClock) answeredLate(s seat.StepResult, logf func(string, ...any)) {
|
||||
if c == nil || c.Clear == nil || s.Ask == "" || (s.Why != UnansweredWhy(s.Ask) && s.Why != SilentWhy(s.Ask)) {
|
||||
return
|
||||
}
|
||||
if err := c.Clear(s.Seat, s.Ask); err != nil {
|
||||
logf("the finding of ask %s, answered after its step ended, is not cleared: %v", s.Ask, err)
|
||||
}
|
||||
}
|
||||
|
||||
// AskUnanswered takes the expiry of a bound of the ask on onSeat (H-A-235, H-A-243). On a silent seat its one bound,
|
||||
// ask-silent-bound, ends the step could not run "its ask <id> had no end", never asking again (that would double a running
|
||||
// build). On a seat that says ask-taken the first bound puts the ask once more under AgainID, said in the journal, its
|
||||
// second bound armed under that id; the second ends the step could not run "its ask <id> was not answered". An ended step
|
||||
// raises the finding call-unanswered, so the pipeline ends and its turn frees. A bound whose ask no running step names any
|
||||
// longer asks nothing.
|
||||
func (v Verbs) AskUnanswered(onSeat, ask string, again bool) error {
|
||||
h, c := v.Hearer, v.Starter.Asks
|
||||
if c == nil {
|
||||
return nil
|
||||
}
|
||||
key, p, i, err := v.runningAsk(onSeat, ask)
|
||||
if err != nil || key == "" {
|
||||
return err
|
||||
}
|
||||
s := p.Steps[i]
|
||||
if Silent(onSeat) || again {
|
||||
why, within := UnansweredWhy(s.Ask), 2*c.bound()
|
||||
if Silent(onSeat) {
|
||||
why, within = SilentWhy(s.Ask), c.boundOf(onSeat)
|
||||
}
|
||||
h.logf("the ask %s of the %s step of %s on %s had no word from its holder within %s: the step ends could not run (H-A-235, H-A-243)",
|
||||
s.Ask, s.Step, key, onSeat, within)
|
||||
if err := h.endUnanswered(key, s.Ask, why); err != nil {
|
||||
return err
|
||||
}
|
||||
if c.Raise != nil {
|
||||
return c.Raise(onSeat, s.Ask, within)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
a, err := v.askAgain(p, s, AgainID(s.Ask), h.now())
|
||||
if err != nil {
|
||||
h.logf("the ask %s of the %s step of %s on %s had no word from its holder within %s and cannot be put again: %v", s.Ask, s.Step, key, onSeat, c.bound(), err)
|
||||
} else {
|
||||
h.logf("the ask %s of the %s step of %s on %s had no word from its holder within %s: asked once more as %s (H-A-235, H-A-243)", s.Ask, s.Step, key, onSeat, c.bound(), a.ID)
|
||||
}
|
||||
if c.Arm != nil {
|
||||
if aerr := c.Arm(AskAgainBoundKey(onSeat, AgainID(s.Ask)), c.bound()); aerr != nil {
|
||||
return fmt.Errorf("the second bound of ask %s is not armed, so it is not asked again: %w", s.Ask, aerr)
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return nil // its second bound ends the step
|
||||
}
|
||||
v.Starter.carry(key, []Asked{*a})
|
||||
return askAll(v.Starter.Steps, key, []Asked{*a}, func(commit string) bool { return v.Starter.recovered(key, commit) }, nil)
|
||||
}
|
||||
|
||||
// HolderStarted puts again every open ask of each silent seat module holds (H-A-243 (1)): its holder takes its asks on a plain
|
||||
// subscription, so an ask put while it was down is lost; heard on the node-engine's running of module. Each ask is put under
|
||||
// its own id, a rerun's marks kept, said in the journal; its bound stands as armed.
|
||||
func (v Verbs) HolderStarted(module string) (int, error) {
|
||||
st := v.Starter
|
||||
if st == nil || st.Holders == nil {
|
||||
return 0, nil
|
||||
}
|
||||
var silent []string
|
||||
for onSeat := range acceptStep {
|
||||
if !Silent(onSeat) {
|
||||
continue
|
||||
}
|
||||
holders, read := st.Holders(onSeat)
|
||||
if !read {
|
||||
return 0, ErrHoldersNotRead
|
||||
}
|
||||
if containsString(holders, module) {
|
||||
silent = append(silent, onSeat)
|
||||
}
|
||||
}
|
||||
if len(silent) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
pkeys, err := allKeys(v.Hearer.Store)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("the pipelines are not listed: %w", err)
|
||||
}
|
||||
now, asked := v.Hearer.now(), 0
|
||||
var first error
|
||||
for _, key := range pkeys {
|
||||
p, _, err := v.Hearer.Store.Get(key)
|
||||
if err != nil || p.Outcome != seat.PipelineOutcomeRunning {
|
||||
continue
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
var due []Asked
|
||||
for _, s := range p.Steps {
|
||||
if !open(s) || s.Ask == "" || seen[s.Ask] || !containsString(silent, s.Seat) {
|
||||
continue
|
||||
}
|
||||
seen[s.Ask] = true
|
||||
a, err := v.askAgain(p, s, s.Ask, now)
|
||||
if err != nil {
|
||||
v.Hearer.logf("the ask %s of the %s step of %s on %s is not put again as %s starts: %v", s.Ask, s.Step, key, s.Seat, module, err)
|
||||
continue
|
||||
}
|
||||
v.Hearer.logf("the ask %s of the %s step of %s is put again on %s: its holder %s started, and takes no ask put while it was down (H-A-243)",
|
||||
s.Ask, s.Step, key, s.Seat, module)
|
||||
due = append(due, *a)
|
||||
}
|
||||
if len(due) == 0 {
|
||||
continue
|
||||
}
|
||||
st.carry(key, due)
|
||||
if err := askAll(st.Steps, key, due, func(commit string) bool { return st.recovered(key, commit) }, nil); err != nil && first == nil {
|
||||
first = err
|
||||
}
|
||||
asked += len(due)
|
||||
}
|
||||
return asked, first
|
||||
}
|
||||
|
||||
// ErrHoldersNotRead is HolderStarted's answer before the seat-holders view's first view is read: the running is heard again.
|
||||
var ErrHoldersNotRead = errors.New("the seat-holders view is not read yet")
|
||||
|
||||
func containsString(list []string, s string) bool {
|
||||
for _, x := range list {
|
||||
if x == s {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// runningAsk is the pipeline and the index of its running step asked under ask, or put again under it, of onSeat; an empty
|
||||
// key when none is.
|
||||
func (v Verbs) runningAsk(onSeat, ask string) (string, seat.Pipeline, int, error) {
|
||||
pkeys, err := allKeys(v.Hearer.Store)
|
||||
if err != nil {
|
||||
return "", seat.Pipeline{}, -1, fmt.Errorf("the pipelines are not listed: %w", err)
|
||||
}
|
||||
for _, key := range pkeys {
|
||||
p, _, err := v.Hearer.Store.Get(key)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if i := stepOfAsk(p, ask); i >= 0 && open(p.Steps[i]) && p.Steps[i].Seat == onSeat {
|
||||
return key, p, i, nil
|
||||
}
|
||||
}
|
||||
return "", seat.Pipeline{}, -1, nil
|
||||
}
|
||||
|
||||
// askAgain is the ask of running step s as it was put, under id: a rerun's marks kept.
|
||||
func (v Verbs) askAgain(p seat.Pipeline, s seat.StepResult, id string, now time.Time) (*Asked, error) {
|
||||
a, err := v.placedAsk(p, s, id, now)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if s.Rerun {
|
||||
markRerun(a, s.By)
|
||||
}
|
||||
return a, nil
|
||||
}
|
||||
|
||||
// markRerun marks a's body a rerun by caller.
|
||||
func markRerun(a *Asked, caller string) {
|
||||
switch b := a.Ask.(type) {
|
||||
case nodebuildgo.StepAsk:
|
||||
if b.Step != nil {
|
||||
step := *b.Step
|
||||
step.Rerun, step.By = true, caller
|
||||
b.Step = &step
|
||||
}
|
||||
a.Ask = b
|
||||
case builds.StepRun:
|
||||
b.Rerun, b.By = true, caller
|
||||
a.Ask = b
|
||||
case wants.StepRun:
|
||||
b.Rerun, b.By = true, caller
|
||||
a.Ask = b
|
||||
}
|
||||
}
|
||||
|
||||
// endUnanswered ends the running step of pipeline key asked under ask could not run, saying why (H-A-235, H-A-243): every
|
||||
// other mover's validate reading the ask with it, a composition its validate held ended not asked, the pipeline ended when
|
||||
// nothing else is open, said and logged as any step's end.
|
||||
func (h *Hearer) endUnanswered(key, ask, why string) error {
|
||||
for {
|
||||
p, rev, err := h.Store.Get(key)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
i := stepOfAsk(p, ask)
|
||||
if i < 0 || !open(p.Steps[i]) {
|
||||
return nil
|
||||
}
|
||||
now := h.now()
|
||||
s := &p.Steps[i]
|
||||
s.Outcome, s.Why, s.Ended = seat.StepResultOutcomeCouldNotRun, why, &now
|
||||
p.Unlogged = append(p.Unlogged, endEntry(key, *s, now))
|
||||
var readers []int
|
||||
if s.Step == seat.StepResultStepValidate {
|
||||
readers = sameValidate(&p, i, now)
|
||||
_, notAsked := releaseCompositions(&p, now)
|
||||
readers = append(readers, notAsked...)
|
||||
}
|
||||
asked, err := askApproval(key, &p, now, h.Asker)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ended := h.endPipeline(&p, now)
|
||||
if _, err := h.Store.Update(key, p, rev); errors.Is(err, ErrChanged) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
return fmt.Errorf("pipeline %s cannot be written: %w", key, err)
|
||||
}
|
||||
h.Failures.StepEnded(h.Store, p, *s)
|
||||
for _, j := range append([]int{i}, readers...) {
|
||||
if err := h.say(seat.EventStepEnded, p, &p.Steps[j]); err != nil {
|
||||
return fmt.Errorf("pipeline %s's %s is written but not said: %w", key, seat.EventStepEnded, err)
|
||||
}
|
||||
}
|
||||
h.appendUnlogged(key)
|
||||
return h.said(p, asked, ended)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,197 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
wants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
nodebuildgo "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
|
||||
sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
|
||||
)
|
||||
|
||||
// H-A-235, H-A-238: every ask but a test run's is bounded from its put until its holder's first word. A bound that passes asks
|
||||
// once more under the ask's own id, its second bound armed; past the second the step ends could not run "its ask <id> was not
|
||||
// answered", the finding raised, and the pipeline ends; a first word deletes both bounds, and a late one clears the finding.
|
||||
func TestAnAskNobodyAnswersIsAskedOnceMoreThenEndsItsStep(t *testing.T) {
|
||||
v, store, said, asks, _, _ := verbsFixture()
|
||||
var armed, unbound, raised, cleared []string
|
||||
logs := []string{}
|
||||
v.Hearer.Logf = func(f string, a ...any) { logs = append(logs, f) }
|
||||
c := &AskClock{Bound: 10 * time.Minute,
|
||||
Arm: func(key string, ttl time.Duration) error { armed = append(armed, key+" "+ttl.String()); return nil },
|
||||
Unbound: func(key string) error { unbound = append(unbound, key); return nil },
|
||||
Raise: func(s, ask string, _ time.Duration) error { raised = append(raised, s+" "+ask); return nil },
|
||||
Clear: func(s, ask string) error { cleared = append(cleared, s+" "+ask); return nil }}
|
||||
v.Starter.Asks, v.Hearer.Asks = c, c
|
||||
|
||||
// A test run's ask keeps §725's bounds; a nil clock arms none.
|
||||
if err := c.arm(Asked{Seat: nodebuildgo.Seat, ID: "r1", Run: "run"}); err != nil || len(armed) != 0 {
|
||||
t.Fatalf("a test run's ask armed %v (%v)", armed, err)
|
||||
}
|
||||
if err := (*AskClock)(nil).arm(Asked{Seat: nodebuildgo.Seat, ID: "x"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Every step but app:program's lint has ended; that lint failed, and its rerun's ask is armed before it is put.
|
||||
p, rev, _ := store.Get(pkey)
|
||||
for i := range p.Steps {
|
||||
if open(p.Steps[i]) {
|
||||
p.Steps[i].Outcome = seat.StepResultOutcomePassed
|
||||
}
|
||||
}
|
||||
p.Steps[stepIndex(p, "lint", "app:program")].Outcome = seat.StepResultOutcomeFailed
|
||||
store.Update(pkey, p, rev)
|
||||
r, err := v.Rerun("operator", seat.RerunInput{Pipeline: pkey, Step: "lint", Artifact: "app:program", Why: "retry"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ask := r.Result.Ask
|
||||
if want := []string{AskBoundKey(nodebuildgo.Seat, ask) + " 10m0s"}; !slices.Equal(armed, want) || len(asks.ids) != 1 {
|
||||
t.Fatalf("armed %v asking %v, want %v before one ask", armed, asks.ids, want)
|
||||
}
|
||||
// The first bound passes: asked once more under the id derived from the first, in the form the holders accept, a rerun
|
||||
// still, the second bound armed under it and said (H-A-243 (3)).
|
||||
again := AgainID(ask)
|
||||
if err := sdkstep.CheckAskID(again); err != nil || again == ask || again[:len(again)-6] != ask[:len(ask)-6] {
|
||||
t.Fatalf("AgainID(%s) = %s (%v), want the first's time part and other hex digits in the ask id form", ask, again, err)
|
||||
}
|
||||
if err := v.AskUnanswered(nodebuildgo.Seat, ask, false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(asks.ids) != 2 || asks.ids[1] != again || armed[1] != AskAgainBoundKey(nodebuildgo.Seat, again)+" 10m0s" {
|
||||
t.Fatalf("asked %v, armed %v, want %s once more and its second bound", asks.ids, armed, again)
|
||||
}
|
||||
if b, ok := asks.asks[1].(nodebuildgo.StepAsk); !ok || b.ID != again || b.Step == nil || !b.Step.Rerun || b.Step.By != "operator" {
|
||||
t.Errorf("the ask put again = %+v", asks.asks[1])
|
||||
}
|
||||
if !slices.ContainsFunc(logs, func(l string) bool { return strings.Contains(l, "asked once more") }) {
|
||||
t.Errorf("no journal line names the ask asked again: %v", logs)
|
||||
}
|
||||
if got, _, _ := store.Get(pkey); got.Steps[stepIndex(got, "lint", "app:program")].Outcome != seat.StepResultOutcomeRunning {
|
||||
t.Fatal("the step ended at the first bound")
|
||||
}
|
||||
// The second bound passes: the step ends could not run, the finding is raised, the pipeline ends failed.
|
||||
if err := v.AskUnanswered(nodebuildgo.Seat, again, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, _, _ := store.Get(pkey)
|
||||
s := got.Steps[stepIndex(got, "lint", "app:program")]
|
||||
if s.Outcome != seat.StepResultOutcomeCouldNotRun || s.Why != "its ask "+ask+" was not answered" || s.Ended == nil {
|
||||
t.Errorf("the step = %+v", s)
|
||||
}
|
||||
if got.Outcome != seat.PipelineOutcomeFailed || len(asks.ids) != 2 {
|
||||
t.Errorf("the pipeline is %s (%s), asked %d, want failed and nothing more asked", got.Outcome, got.Why, len(asks.ids))
|
||||
}
|
||||
if !slices.Equal(raised, []string{nodebuildgo.Seat + " " + ask}) || !slices.Contains(*said, "pipeline-ended.main.5") {
|
||||
t.Errorf("raised %v, said %v", raised, *said)
|
||||
}
|
||||
// A bound whose step ended asks nothing; a first word deletes both bounds; a late end clears the finding.
|
||||
if err := v.AskUnanswered(nodebuildgo.Seat, ask, false); err != nil || len(asks.ids) != 2 {
|
||||
t.Errorf("a bound of an ended step asked %v (%v)", asks.ids, err)
|
||||
}
|
||||
if err := c.Answered(nodebuildgo.Seat, ask); err != nil || !slices.Equal(unbound, []string{AskBoundKey(nodebuildgo.Seat, ask), AskAgainBoundKey(nodebuildgo.Seat, ask), AskAgainBoundKey(nodebuildgo.Seat, again)}) {
|
||||
t.Errorf("a first word deleted %v (%v)", unbound, err)
|
||||
}
|
||||
if err := v.Hearer.Taken(again, "n1"); err != nil || !slices.Equal(cleared, []string{nodebuildgo.Seat + " " + ask}) {
|
||||
t.Errorf("a late ask-taken cleared %v (%v)", cleared, err)
|
||||
}
|
||||
for _, k := range []string{AskBoundKey("mesh-wants", "a1"), AskAgainBoundKey("mesh-wants", "a1")} {
|
||||
if s, a, again, ok := ParseAskBound(k); !ok || s != "mesh-wants" || a != "a1" || again != strings.HasPrefix(k, AskAgainBoundPrefix) {
|
||||
t.Errorf("ParseAskBound(%s) = %s %s %v %v", k, s, a, again, ok)
|
||||
}
|
||||
}
|
||||
if _, _, _, ok := ParseAskBound(DropBoundKey("node-build-go", "a1")); ok {
|
||||
t.Error("a drop's bound read as an ask's")
|
||||
}
|
||||
}
|
||||
|
||||
// H-A-243: an ask on a seat whose holder says nothing before the step's end (mesh-builds) is never asked again for a bound:
|
||||
// its one bound, ask-silent-bound, ends the step could not run "its ask <id> had no end", the finding raised. The node-engine's
|
||||
// running of the seat's holder puts every open ask of the seat again under its own id, a module holding no such seat none.
|
||||
func TestASilentSeatsAskIsPutAgainWhenItsHolderStartsAndEndsAtItsSilentBound(t *testing.T) {
|
||||
v, store, _, asks, _, _ := verbsFixture()
|
||||
var armed, raised []string
|
||||
c := &AskClock{Bound: time.Minute, Silent: 45 * time.Minute,
|
||||
Arm: func(key string, ttl time.Duration) error { armed = append(armed, key+" "+ttl.String()); return nil },
|
||||
Raise: func(s, ask string, within time.Duration) error {
|
||||
raised = append(raised, s+" "+ask+" "+within.String())
|
||||
return nil
|
||||
}}
|
||||
v.Starter.Asks, v.Hearer.Asks = c, c
|
||||
v.Starter.Holders = func(s string) ([]string, bool) {
|
||||
return map[string][]string{builds.Seat: {"mesh-delivery-builder"}}[s], true
|
||||
}
|
||||
p, rev, _ := store.Get(pkey)
|
||||
// Every step has ended but app's build, asked of mesh-builds.
|
||||
b := stepIndex(p, seat.StepResultStepBuild, "")
|
||||
for i := range p.Steps {
|
||||
if open(p.Steps[i]) && i != b {
|
||||
p.Steps[i].Outcome = seat.StepResultOutcomePassed
|
||||
}
|
||||
}
|
||||
if p.Steps[b].Seat != builds.Seat || !open(p.Steps[b]) {
|
||||
t.Fatalf("the build step = %+v, want running on mesh-builds", p.Steps[b])
|
||||
}
|
||||
store.Update(pkey, p, rev)
|
||||
build := p.Steps[b].Ask
|
||||
if err := c.arm(Asked{Seat: builds.Seat, ID: build}); err != nil || !slices.Equal(armed, []string{AskBoundKey(builds.Seat, build) + " 45m0s"}) {
|
||||
t.Fatalf("armed %v (%v), want the silent bound", armed, err)
|
||||
}
|
||||
// A module holding no silent seat puts nothing again; the builder's running puts the build again under its own id.
|
||||
if n, err := v.HolderStarted("mesh-build-go"); err != nil || n != 0 || len(asks.ids) != 0 {
|
||||
t.Fatalf("another holder's start put %d (%v)", n, err)
|
||||
}
|
||||
if n, err := v.HolderStarted("mesh-delivery-builder"); err != nil || n != 1 || len(asks.ids) != 1 || asks.ids[0] != build {
|
||||
t.Fatalf("the builder's start put %d asks %v (%v), want %s again", n, asks.ids, err, build)
|
||||
}
|
||||
if _, ok := asks.asks[0].(builds.StepRun); !ok || asks.at[0] != builds.AcceptSubjectStep {
|
||||
t.Errorf("the ask put again = %+v on %s", asks.asks[0], asks.at[0])
|
||||
}
|
||||
// The silent bound passes: no ask again, the step ends could not run, the finding is raised, the pipeline ends.
|
||||
if err := v.AskUnanswered(builds.Seat, build, false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, _, _ := store.Get(pkey)
|
||||
s := got.Steps[b]
|
||||
if len(asks.ids) != 1 || s.Outcome != seat.StepResultOutcomeCouldNotRun || s.Why != "its ask "+build+" had no end" || got.Outcome != seat.PipelineOutcomeFailed {
|
||||
t.Errorf("asked %v; the build %+v; the pipeline %s", asks.ids, s, got.Outcome)
|
||||
}
|
||||
if !slices.Equal(raised, []string{builds.Seat + " " + build + " 45m0s"}) {
|
||||
t.Errorf("raised %v", raised)
|
||||
}
|
||||
}
|
||||
|
||||
// H-A-252: the pipeline's one validate ask names no mover (H-A-51), and its holder's end names the mover whose body it took:
|
||||
// it is matched by its ask alone and ends the validate step of every mover under it, whichever mover it names.
|
||||
func TestAValidateEndNamingOneMoverEndsEveryMoversValidate(t *testing.T) {
|
||||
h, store, _, _, _, _ := fixture()
|
||||
p, rev, _ := store.Get(pkey)
|
||||
p.Movers = []seat.Mover{{Module: "app", Commit: "c1", Fingerprint: "src2:aa"}, {Module: "lib", Commit: "c2", Fingerprint: "src2:bb"}}
|
||||
p.Steps = []seat.StepResult{
|
||||
{Module: "app", Step: seat.StepResultStepValidate, Status: seat.StepResultStatusRequired, Outcome: seat.StepResultOutcomeRunning, Ask: "v1", Seat: wants.Seat},
|
||||
{Module: "lib", Step: seat.StepResultStepValidate, Status: seat.StepResultStatusRequired, Outcome: seat.StepResultOutcomeRunning, Ask: "v1", Seat: wants.Seat}}
|
||||
store.Update(pkey, p, rev)
|
||||
var bs StepSeat
|
||||
for _, x := range StepSeats {
|
||||
if x.Seat == wants.Seat {
|
||||
bs = x
|
||||
}
|
||||
}
|
||||
ev := StepEnded{ID: "v1", Node: "n1", Pipeline: pkey, Module: "lib", Commit: "c2", Fingerprint: "src2:bb", Step: seat.StepResultStepValidate, Outcome: "passed"}
|
||||
if _, err := h.OnStepEnded(bs, bs.Prefix+"n1", ev); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, _, _ := store.Get(pkey)
|
||||
for _, s := range got.Steps {
|
||||
if s.Outcome != seat.StepResultOutcomePassed {
|
||||
t.Errorf("%s's validate = %s, want passed by the one end", s.Module, s.Outcome)
|
||||
}
|
||||
}
|
||||
if got.Outcome != seat.PipelineOutcomePassed {
|
||||
t.Errorf("the pipeline = %s (%s), want passed", got.Outcome, got.Why)
|
||||
}
|
||||
}
|
||||
@@ -11,8 +11,8 @@ import (
|
||||
)
|
||||
|
||||
// DropBound is how long a superseded step's drop may go unanswered before it is said (design 47 G A-222, A-77): five minutes,
|
||||
// a key of the state `bounds` with its own time to live. An ask waiting for its holder has no such bound (§535, §647 as
|
||||
// A-466y amends them): it is timed by §86's build-capacity finding and, for a test run, by run-untaken-bound (§725).
|
||||
// a key of the state `bounds` with its own time to live. An ask waiting for its holder is bounded by ask-unanswered-bound
|
||||
// (H-A-235, H-A-238; AskClock), a test run's by run-untaken-bound (§725).
|
||||
const DropBound = 5 * time.Minute
|
||||
|
||||
// TakenSeat is a build seat whose holder says `ask-taken` when it takes an ask (design 47 G §602): where the runner hears it.
|
||||
|
||||
@@ -387,17 +387,23 @@ func stepAsk(p seat.Pipeline, m seat.Mover, e catalog.CatalogEntry, a catalog.Ar
|
||||
Module: m.Module, Pipeline: p.Pipeline, Revision: p.Revision, Source: p.Source, Step: step}}
|
||||
}
|
||||
|
||||
// askAll puts each ask on its seat's work queue; the first refusal is named, the rest are still put. An ask waits for its
|
||||
// holder with no bound of the runner's own (§535, §647 as A-466y amends them).
|
||||
func askAll(asker StepAsker, key string, asks []Asked, recovery func(commit string) bool) error {
|
||||
// askAll puts each ask on its seat's work queue; the first refusal is named, the rest are still put. arm, when not nil, arms
|
||||
// each ask's bound before it is put (H-A-235, H-A-238 (5)); an ask whose bound is not armed is not put.
|
||||
func askAll(asker StepAsker, key string, asks []Asked, recovery func(commit string) bool, arm func(Asked) error) error {
|
||||
var first error
|
||||
for _, a := range asks {
|
||||
accept, ok := acceptStep[a.Seat]
|
||||
if to, named := acceptRecovery[a.Seat]; named && recovery != nil && recovery(a.Commit) {
|
||||
accept = to
|
||||
}
|
||||
err := fmt.Errorf("the seat %s is no build seat the runner asks", a.Seat)
|
||||
if ok {
|
||||
var err error
|
||||
switch {
|
||||
case !ok:
|
||||
err = fmt.Errorf("the seat %s is no build seat the runner asks", a.Seat)
|
||||
case arm != nil:
|
||||
err = arm(a)
|
||||
}
|
||||
if err == nil {
|
||||
err = asker.AskStep(accept, a.ID, a.Ask)
|
||||
}
|
||||
if err != nil && first == nil {
|
||||
|
||||
@@ -176,6 +176,52 @@ func TestThePipelinesWaitTheirTurn(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// H-A-244 (1), H-A-223: a mover a newer revision moves leaves an older waiting pipeline also when the newer one is placed at
|
||||
// once, not only when it waits: the waiting pipeline keeps the movers still its own, and with none left ends superseded,
|
||||
// never placed.
|
||||
func TestANewerPlacedPipelineSupersedesAnOlderWaitingOne(t *testing.T) {
|
||||
s, mem, q, said, _, _, view, _ := queueFixture()
|
||||
take := takeIn(t, s, view)
|
||||
take(4, "app")
|
||||
take(5, "lib", "tool")
|
||||
if p, _, _ := mem.Get("main.5"); p.Outcome != seat.PipelineOutcomeWaiting {
|
||||
t.Fatalf("main.5 = %s, want waiting", p.Outcome)
|
||||
}
|
||||
s.Queue.AtOnce = 3
|
||||
take(6, "lib")
|
||||
if p, _, _ := mem.Get("main.6"); p.Outcome != seat.PipelineOutcomeRunning {
|
||||
t.Fatalf("main.6 = %s, want placed at once", p.Outcome)
|
||||
}
|
||||
p5, _, _ := mem.Get("main.5")
|
||||
if p5.Outcome != seat.PipelineOutcomeWaiting || len(p5.Movers) != 1 || p5.Movers[0].Module != "tool" || !slices.Equal(p5.Superseded, []string{"lib"}) {
|
||||
t.Fatalf("main.5 = %s moving %+v, superseded %v; want waiting with tool alone, lib superseded", p5.Outcome, p5.Movers, p5.Superseded)
|
||||
}
|
||||
// At its turn it is placed with tool alone.
|
||||
p4, rev, _ := mem.Get("main.4")
|
||||
p4.Outcome = seat.PipelineOutcomePassed
|
||||
mem.Update("main.4", p4, rev)
|
||||
if _, err := s.Turn("main.4", view); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if p, _, _ := mem.Get("main.5"); p.Outcome != seat.PipelineOutcomeRunning || len(stepsOf(p, "lib")) != 0 || len(stepsOf(p, "tool")) == 0 {
|
||||
t.Fatalf("main.5 at its turn = %s with steps %+v, want running with tool's steps alone", p.Outcome, p.Steps)
|
||||
}
|
||||
|
||||
// A waiting pipeline whose one mover a newer placed pipeline moves ends superseded, never placed.
|
||||
s, mem, q, said, _, _, view, _ = queueFixture()
|
||||
take = takeIn(t, s, view)
|
||||
take(4, "app")
|
||||
take(5, "tool")
|
||||
s.Queue.AtOnce = 3
|
||||
take(7, "tool")
|
||||
if p, _, _ := mem.Get("main.5"); p.Outcome != seat.PipelineOutcomeSuperseded || len(p.Steps) != 0 || p.Ended == nil {
|
||||
t.Errorf("main.5 = %s with %d steps, want superseded and never placed", p.Outcome, len(p.Steps))
|
||||
}
|
||||
if _, ok := q["waiting.00000000000000000001"]; ok || !slices.Contains(*said, "pipeline-ended.main.5") {
|
||||
t.Errorf("the queue %v, said %v; want main.5's waiting key gone and its pipeline-ended said", q, *said)
|
||||
}
|
||||
}
|
||||
|
||||
// R13/AC1 (design 47 H §748, ADR 0682 §7): a newer waiting pipeline of the same branch takes the movers it also moves out of
|
||||
// an older waiting one, which ends superseded, saying wait-ended and pipeline-ended, once no mover is left; a running
|
||||
// pipeline moving the same module runs to its end.
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
|
||||
condseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
)
|
||||
|
||||
// RegistrationMissingKind is the runner's finding of a mover that waits on its module's registration (design 47 H H-A-262,
|
||||
// G A-327): scope module, id the module, raised while a mover of a running pipeline has no registration to place its steps
|
||||
// from, cleared when the registration arrives or no running pipeline still waits on it (H-A-265 (4)).
|
||||
const RegistrationMissingKind = "registration-missing"
|
||||
|
||||
// registrationMissingID is the finding's id: the module, escaped as a key's name is.
|
||||
func registrationMissingID(module string) string { return keys.EscapeName(module) }
|
||||
|
||||
// registrationMissing raises registration-missing of module, waited on by the pipeline key (H-A-262).
|
||||
func (s *Starter) registrationMissing(module, key string) {
|
||||
if s.Findings == nil {
|
||||
return
|
||||
}
|
||||
id := registrationMissingID(module)
|
||||
err := s.Findings.Write(context.Background(), condseat.Finding{Scope: mcc.ScopeModule, ID: id, Kind: RegistrationMissingKind,
|
||||
Severity: condseat.FindingSeverityWarning,
|
||||
Summary: fmt.Sprintf("%s has no registration: the pipeline %s waits on it for its steps", module, key),
|
||||
Headline: "A pipeline waits on a module's registration",
|
||||
Explanation: fmt.Sprintf("The pipeline %s moves %s, which has no registration, so none of its steps is placed (A-327). The indexer registers a module the take-in first lists (H-A-262); this is cleared when the registration of %s arrives.",
|
||||
key, module, module),
|
||||
Needs: "the registration of " + module,
|
||||
Source: "pipeline-runner." + RegistrationMissingKind, Observer: s.Observer, At: s.now(),
|
||||
Subject: condseat.Subject{Scope: mcc.ScopeModule, ID: id}})
|
||||
if err != nil {
|
||||
s.logf("the finding %s of %s is not written: %v", RegistrationMissingKind, module, err)
|
||||
}
|
||||
}
|
||||
|
||||
// registrationArrived clears registration-missing of module, saying why, when the findings hold it (H-A-262, H-A-265 (4)).
|
||||
func (s *Starter) registrationArrived(module, why string) {
|
||||
if s.Findings == nil {
|
||||
return
|
||||
}
|
||||
id := registrationMissingID(module)
|
||||
for _, f := range s.Findings.Held() {
|
||||
if f.Scope != mcc.ScopeModule || f.ID != id || f.Kind != RegistrationMissingKind {
|
||||
continue
|
||||
}
|
||||
if err := s.Findings.Clear(context.Background(), mcc.ScopeModule, id, RegistrationMissingKind, why); err != nil {
|
||||
s.logf("the finding %s of %s is not cleared: %v", RegistrationMissingKind, module, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// JudgeRegistrationMissing clears registration-missing of every module the registrations held now name, and of every module
|
||||
// no running pipeline still has an unplaced mover of (H-A-265 (4)): a pipeline that ended, failed (H-A-163) or cancelled,
|
||||
// with the mover still waiting leaves no finding behind. It is judged at a pipeline's end and in Waiting.
|
||||
func (s *Starter) JudgeRegistrationMissing(store Updater) {
|
||||
if s.Findings == nil {
|
||||
return
|
||||
}
|
||||
var modules []string
|
||||
for _, f := range s.Findings.Held() {
|
||||
if f.Scope != mcc.ScopeModule || f.Kind != RegistrationMissingKind {
|
||||
continue
|
||||
}
|
||||
module, err := keys.UnescapeName(f.ID)
|
||||
if err != nil {
|
||||
module = f.ID
|
||||
}
|
||||
if s.Registration != nil {
|
||||
if _, ok := s.Registration(module); ok {
|
||||
s.registrationArrived(module, "the registration of "+module+" arrived")
|
||||
continue
|
||||
}
|
||||
}
|
||||
modules = append(modules, module)
|
||||
}
|
||||
if len(modules) == 0 || store == nil {
|
||||
return
|
||||
}
|
||||
pkeys, err := allKeys(store)
|
||||
if err != nil {
|
||||
s.logf("the finding %s is not judged: the pipelines are not listed: %v", RegistrationMissingKind, err)
|
||||
return
|
||||
}
|
||||
waiting := map[string]bool{}
|
||||
for _, key := range pkeys {
|
||||
p, _, err := store.Get(key)
|
||||
if err != nil {
|
||||
s.logf("the finding %s is not judged: the pipeline %s is not read: %v", RegistrationMissingKind, key, err)
|
||||
return
|
||||
}
|
||||
if p.Outcome != seat.PipelineOutcomeRunning {
|
||||
continue
|
||||
}
|
||||
for _, m := range modules {
|
||||
if Moves(p, m) && !placed(p, m) {
|
||||
waiting[m] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, m := range modules {
|
||||
if !waiting[m] {
|
||||
s.registrationArrived(m, "no running pipeline still waits on the registration of "+m)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -162,6 +162,9 @@ type Starter struct {
|
||||
// Ended says a pipeline that ended at its placing (holderless): the step-ended of each step of steps, then its end
|
||||
// (Hearer.EndedAtPlacing); nil says it in the journal alone.
|
||||
Ended func(p seat.Pipeline, steps []int) error
|
||||
// Supersede takes out of every older running pipeline of newer's branch each mover whose fingerprint newer moved
|
||||
// (H-A-223; Hearer.SupersedeRunning); nil leaves running pipelines as they are.
|
||||
Supersede func(newer seat.Pipeline)
|
||||
// Fills refuses a start while a store of the runner is full (§366); nil refuses none.
|
||||
Fills *Fills
|
||||
// Recovered says whether a commit is named a recovery (§81, §86): its steps are asked on the build seats' stream delivery.
|
||||
@@ -180,9 +183,15 @@ type Starter struct {
|
||||
// Queue bounds the pipelines asked at once to pipelines-at-once, the others waiting in order (design 47 H §748); nil
|
||||
// places every start at once.
|
||||
Queue *Queue
|
||||
// Asks bounds every ask but a test run's until its holder's first word (H-A-235, H-A-238); nil bounds none.
|
||||
Asks *AskClock
|
||||
// RunTurn places a run on request whose turn in the queue came, from its candidate's newest revision (A-222); nil places
|
||||
// it at the runner's next start.
|
||||
RunTurn func(UnplacedRun)
|
||||
// Findings is the runner's finding writer, through which a mover waiting on its registration raises
|
||||
// registration-missing (H-A-262); Observer is the runner's own module name. nil raises none.
|
||||
Findings Findings
|
||||
Observer string
|
||||
}
|
||||
|
||||
// holderless ends at once, as it is placed, a pipeline with a build step whose seat the seat mesh-module-assigner's view
|
||||
@@ -249,7 +258,7 @@ func (s *Starter) askSteps(key string, asks []Asked) error {
|
||||
var err error
|
||||
if len(asks) > 0 {
|
||||
s.carry(key, asks)
|
||||
err = askAll(s.Steps, key, asks, func(commit string) bool { return s.recovered(key, commit) })
|
||||
err = askAll(s.Steps, key, asks, func(commit string) bool { return s.recovered(key, commit) }, s.Asks.arm)
|
||||
}
|
||||
s.sayStarted(key, placed)
|
||||
return err
|
||||
@@ -523,6 +532,7 @@ func (s *Starter) Start(b Begin) (bool, error) {
|
||||
asks, unregistered, ended, notAsked = s.placeMovers(&p, s.Registration, b.Entries, now)
|
||||
for _, m := range unregistered {
|
||||
s.logf("%s: %s has no registration yet: its steps wait on its registration, which places them (A-327)", key, m)
|
||||
s.registrationMissing(m, key)
|
||||
}
|
||||
}
|
||||
if err := s.create(key, p); err != nil {
|
||||
@@ -538,11 +548,16 @@ func (s *Starter) Start(b Begin) (bool, error) {
|
||||
// A waiting pipeline says pipeline-started at its write, so its revision is pinned while it waits (H-A-52), and its
|
||||
// wait on its turn.
|
||||
s.sayStartedPipeline(p, now)
|
||||
// A mover whose fingerprint this revision moved leaves every older running pipeline of the branch (H-A-223).
|
||||
if s.Supersede != nil {
|
||||
s.Supersede(p)
|
||||
}
|
||||
// ... and every older waiting one, whether this one waits or is placed at once (H-A-244 (1)).
|
||||
if _, err := s.supersedeWaiting(p); err != nil {
|
||||
s.logf("the older waiting pipelines of %s are not superseded by %s: %v", p.Branch, key, err)
|
||||
}
|
||||
if !place {
|
||||
s.sayQueueWait(seat.EventWaitHeld, p, now, "")
|
||||
if _, err := s.supersedeWaiting(p); err != nil {
|
||||
s.logf("the older waiting pipelines of %s are not superseded by %s: %v", p.Branch, key, err)
|
||||
}
|
||||
s.Queue.overCap(key, s.logf)
|
||||
return true, nil
|
||||
}
|
||||
@@ -657,6 +672,7 @@ func (s *Starter) Registered(store Updater, reg catalog.Registration, view func(
|
||||
if reg.Module == "" {
|
||||
return 0, nil
|
||||
}
|
||||
defer s.registrationArrived(reg.Module, "the registration of "+reg.Module+" arrived") // the finding of a mover that waited on it goes (H-A-262)
|
||||
return s.placeWaiting(store, func(p seat.Pipeline) bool { return Moves(p, reg.Module) && !placed(p, reg.Module) },
|
||||
func(module string) (catalog.Registration, bool) { return reg, module == reg.Module },
|
||||
view, "the registration of "+reg.Module+", which arrived after the pipeline started")
|
||||
@@ -670,6 +686,7 @@ func (s *Starter) Waiting(store Updater, branch string, view func(branch string)
|
||||
if s.Registration == nil {
|
||||
return 0, nil
|
||||
}
|
||||
defer s.JudgeRegistrationMissing(store) // the finding of a mover registered now, or no longer waited on, goes (H-A-262, H-A-265 (4))
|
||||
return s.placeWaiting(store, func(p seat.Pipeline) bool {
|
||||
if branch != "" && p.Branch != branch {
|
||||
return false
|
||||
@@ -722,6 +739,7 @@ func (s *Starter) placeWaiting(store Updater, due func(seat.Pipeline) bool, reg
|
||||
// and it holds the pipeline's end only while every placed mover's required steps passed (H-A-163, H-A-171).
|
||||
for _, m := range unregistered {
|
||||
s.logf("%s: %s has no registration yet: its steps wait on its registration, which places them (A-327)", key, m)
|
||||
s.registrationMissing(m, key)
|
||||
}
|
||||
if len(p.Steps) == before {
|
||||
continue
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
condseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -203,6 +204,8 @@ func TestAPipelineStartedBeforeItsRegistrationIsPlacedWhenItArrives(t *testing.T
|
||||
var said []string
|
||||
s := &Starter{Pipelines: store, Steps: asks, Name: "the-runner", Say: func(subject string, _ any) error { said = append(said, subject); return nil },
|
||||
Registration: func(m string) (catalog.Registration, bool) { return reg, registered && m == "app" }}
|
||||
findings := newHeld()
|
||||
s.Findings, s.Observer = findings, "the-runner"
|
||||
view := func(string) (Half, bool) { return Half{Entries: entries}, true }
|
||||
taken := catalog.TakeInTaken{Branch: "main", Source: 4, Entries: []catalog.TakeInEntry{{Module: "app", Commit: "c1", Fingerprint: "src2:aa"}}}
|
||||
if n, err := s.TakenIn(taken, view); n != 1 || err != nil {
|
||||
@@ -211,8 +214,15 @@ func TestAPipelineStartedBeforeItsRegistrationIsPlacedWhenItArrives(t *testing.T
|
||||
if p, _, _ := store.Get("main.4"); len(p.Steps) != 0 || len(asks.asks) != 0 {
|
||||
t.Fatalf("before its registration the pipeline has %d steps and %d asks, want none", len(p.Steps), len(asks.asks))
|
||||
}
|
||||
// H-A-262: the mover waiting on its registration is the finding registration-missing, scope module, id the module.
|
||||
if f, ok := findings.held["module.app."+RegistrationMissingKind]; !ok || f.Observer != "the-runner" || !strings.Contains(f.Summary, "main.4") {
|
||||
t.Fatalf("the mover waiting on its registration raised %v, want registration-missing of app naming main.4", findings.held)
|
||||
}
|
||||
registered = true
|
||||
n, err := s.Registered(store.memStore, reg, view)
|
||||
if _, ok := findings.held["module.app."+RegistrationMissingKind]; ok {
|
||||
t.Errorf("the registration of app arrived and its registration-missing is still held")
|
||||
}
|
||||
p, _, _ := store.Get("main.4")
|
||||
if n != 1 || err != nil || len(p.Steps) == 0 || len(asks.asks) == 0 || p.Outcome != seat.PipelineOutcomeRunning {
|
||||
t.Fatalf("the registration placed %d (%v): %d steps, %d asks, want the pipeline's steps placed and asked", n, err, len(p.Steps), len(asks.asks))
|
||||
@@ -303,6 +313,41 @@ func TestAPipelineStartedBeforeItsRegistrationIsPlacedWhenItArrives(t *testing.T
|
||||
if !slices.ContainsFunc(logged, func(l string) bool { return strings.Contains(l, "next.8: contracts has no registration yet") }) {
|
||||
t.Errorf("placing next.8 said %q, want that contracts waits on its registration (A-327)", logged)
|
||||
}
|
||||
// H-A-262: the mover still waiting is raised from the placing too, and cleared once the registrations held name it.
|
||||
if _, ok := findings.held["module.contracts."+RegistrationMissingKind]; !ok {
|
||||
t.Errorf("placing next.8 raised %v, want registration-missing of contracts", findings.held)
|
||||
}
|
||||
// H-A-265 (4): a pipeline that ends with the mover still waiting (failed, H-A-163) leaves no finding: judged at its end
|
||||
// (the hearer's said) and in Waiting, it clears once no running pipeline has an unplaced mover of the module.
|
||||
next8, rev8, _ := store.Get("next.8")
|
||||
next8.Outcome = seat.PipelineOutcomeFailed
|
||||
if _, err := store.Update("next.8", next8, rev8); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
h := &Hearer{Store: store.memStore, Say: func(string, any) error { return nil }, RegistrationWaits: func() { s.JudgeRegistrationMissing(store.memStore) }}
|
||||
if err := h.said(next8, nil, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, ok := findings.held["module.contracts."+RegistrationMissingKind]; ok {
|
||||
t.Errorf("next.8 ended with contracts unplaced and no other pipeline waits on it, yet its registration-missing is held")
|
||||
}
|
||||
// Still waited on by a running pipeline, it stays at an end; the registrations held naming it clear it in Waiting.
|
||||
findings.held["module.contracts."+RegistrationMissingKind] = condseat.Finding{Scope: "module", ID: "contracts", Kind: RegistrationMissingKind}
|
||||
taken = catalog.TakeInTaken{Branch: "next", Source: 9, Entries: []catalog.TakeInEntry{{Module: "contracts", Commit: "c5", Fingerprint: "src2:ff"}}}
|
||||
if n, err := s.TakenIn(taken, late); n != 1 || err != nil {
|
||||
t.Fatalf("the take-in of contracts started %d (%v)", n, err)
|
||||
}
|
||||
s.JudgeRegistrationMissing(store.memStore)
|
||||
if _, ok := findings.held["module.contracts."+RegistrationMissingKind]; !ok {
|
||||
t.Errorf("next.9 still waits on contracts' registration, yet its registration-missing was cleared")
|
||||
}
|
||||
s.Registration = func(string) (catalog.Registration, bool) { return reg, true }
|
||||
if _, err := s.Waiting(store.memStore, "", late); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, ok := findings.held["module.contracts."+RegistrationMissingKind]; ok {
|
||||
t.Errorf("the registrations held name contracts and its registration-missing is still held")
|
||||
}
|
||||
}
|
||||
|
||||
// R1/AC1 (design 47 H §747): a start names an event, never a sweep. A take-in starts the pipelines of its own entries
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -146,6 +147,9 @@ type Hearer struct {
|
||||
Now func() time.Time
|
||||
// Failures says a failed head as the finding pipeline-failed (§33); nil says none.
|
||||
Failures *Failures
|
||||
// RegistrationWaits judges, at every pipeline's end, the finding registration-missing of the modules no running pipeline
|
||||
// still waits on (Starter.JudgeRegistrationMissing, H-A-265 (4)); nil judges none.
|
||||
RegistrationWaits func()
|
||||
// Ager ages the runner's own kept data at each step's end (§330); nil ages none.
|
||||
Ager *Ager
|
||||
// Fills says the stores' fill at each step's end (§366); nil says none.
|
||||
@@ -175,6 +179,9 @@ type Hearer struct {
|
||||
// Unplaced says why a mover of a pipeline has no step placed, for the why of its end (H-A-171; Starter.Unplaced): it
|
||||
// waits on its registration or on a publish; nil says only that it has no step placed.
|
||||
Unplaced func(p seat.Pipeline, module string) string
|
||||
// Asks clears the finding of an ask that ended its step unanswered once it is answered after all (H-A-238 (2)); nil
|
||||
// clears none.
|
||||
Asks *AskClock
|
||||
// Rejoin keys a pipeline running again once its approval is answered and it still runs (H-A-106 (b); Starter.Rejoin);
|
||||
// nil keys none.
|
||||
Rejoin func(key string)
|
||||
@@ -235,6 +242,12 @@ func (h *Hearer) OnStepEndedPinned(bs StepSeat, subject string, ev StepEnded, sh
|
||||
return seat.Pipeline{}, fmt.Errorf("%w: ask %s names pipeline %q, which the runner cannot read: %v", ErrRefused, ev.ID, ev.Pipeline, err)
|
||||
}
|
||||
i := stepOfAsk(p, ev.ID)
|
||||
// The validate ask is the pipeline's one, naming no mover (H-A-51): its end, naming the mover whose body the holder
|
||||
// took, is matched by its ask alone and ends every mover's validate step under it (H-A-252).
|
||||
oneValidate := i >= 0 && p.Steps[i].Step == seat.StepResultStepValidate && p.Steps[i].Seat == wants.Seat
|
||||
if oneValidate {
|
||||
i = validateStepOfAsk(p, ev.ID, ev.Module)
|
||||
}
|
||||
var m seat.Mover
|
||||
if i >= 0 {
|
||||
m, _ = MoverOf(p, p.Steps[i].Module)
|
||||
@@ -244,10 +257,11 @@ func (h *Hearer) OnStepEndedPinned(bs StepSeat, subject string, ev StepEnded, sh
|
||||
return p, fmt.Errorf("%w: pipeline %s asked no step under ask %s", ErrRefused, ev.Pipeline, ev.ID)
|
||||
case p.Steps[i].Seat != bs.Seat:
|
||||
return p, fmt.Errorf("%w: ask %s was asked of %s, not %s", ErrRefused, ev.ID, p.Steps[i].Seat, bs.Seat)
|
||||
case ev.Commit != m.Commit || ev.Fingerprint != m.Fingerprint || ev.Module != m.Module || ev.Step != p.Steps[i].Step:
|
||||
case ev.Step != p.Steps[i].Step || (!oneValidate && (ev.Commit != m.Commit || ev.Fingerprint != m.Fingerprint || ev.Module != m.Module)):
|
||||
return p, fmt.Errorf("%w: ask %s ended %s of %s at %s (%s), asked as %s of %s at %s (%s)", ErrRefused, ev.ID, ev.Step,
|
||||
ev.Module, ev.Commit, ev.Fingerprint, p.Steps[i].Step, p.Steps[i].Module, m.Commit, m.Fingerprint)
|
||||
case !open(p.Steps[i]):
|
||||
h.Asks.answeredLate(p.Steps[i], h.logf)
|
||||
if run := sharedRun(p.Steps[i]); run != "" && h.TestRuns != nil {
|
||||
if r, _, err := h.TestRuns.Get(run); err == nil && r.Ended == nil && r.Ask == ev.ID {
|
||||
// The asking step's end was written and its run's was not (a write failed): the run ends now, heard again.
|
||||
@@ -455,6 +469,9 @@ func (h *Hearer) said(p seat.Pipeline, asked []int, ended bool) error {
|
||||
}
|
||||
if ended {
|
||||
h.Failures.Ended(p)
|
||||
if h.RegistrationWaits != nil {
|
||||
h.RegistrationWaits()
|
||||
}
|
||||
err := h.say(seat.EventPipelineEnded, p, nil)
|
||||
// Every end, a cancel's alike, places the next waiting (§748, H-A-53), said or not.
|
||||
if h.Turn != nil {
|
||||
@@ -582,9 +599,36 @@ func endOutcome(o string) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// validateStepOfAsk is the validate step the end of the shared validate ask id ends first (H-A-252): the open one of the mover
|
||||
// the end names, else any open one under the ask (sameValidate ends the rest), else the first under it.
|
||||
func validateStepOfAsk(p seat.Pipeline, id, module string) int {
|
||||
first, anyOpen := -1, -1
|
||||
for i, s := range p.Steps {
|
||||
if s.Step != seat.StepResultStepValidate || s.Ask == "" || (s.Ask != id && AgainID(s.Ask) != id) {
|
||||
continue
|
||||
}
|
||||
if first < 0 {
|
||||
first = i
|
||||
}
|
||||
if open(s) {
|
||||
if s.Module == module {
|
||||
return i
|
||||
}
|
||||
if anyOpen < 0 {
|
||||
anyOpen = i
|
||||
}
|
||||
}
|
||||
}
|
||||
if anyOpen >= 0 {
|
||||
return anyOpen
|
||||
}
|
||||
return first
|
||||
}
|
||||
|
||||
func stepOfAsk(p seat.Pipeline, id string) int {
|
||||
for i, s := range p.Steps {
|
||||
if id != "" && s.Ask == id {
|
||||
// An ask put again is put under AgainID of the step's ask (H-A-243 (3)): its holder's answer under either is the step's.
|
||||
if id != "" && s.Ask != "" && (s.Ask == id || AgainID(s.Ask) == id) {
|
||||
return i
|
||||
}
|
||||
}
|
||||
@@ -611,9 +655,16 @@ func endPipeline(p *seat.Pipeline, now time.Time, unplaced func(seat.Pipeline, s
|
||||
}
|
||||
// A mover with no step placed waits on its registration or a publish: it holds the end only while every placed mover's
|
||||
// required steps passed, since once one failed or could not run the pipeline can no longer pass (H-A-163, H-A-171).
|
||||
// A mover that left for a newer pipeline (H-A-223) holds nothing and fails nothing: its steps are not judged, and a
|
||||
// pipeline every mover left ends superseded.
|
||||
if allLeft(*p) {
|
||||
p.Outcome, p.Why, p.Ended = seat.PipelineOutcomeSuperseded, "every mover left it for a newer pipeline (H-A-223)", &now
|
||||
p.EndCount++
|
||||
return true
|
||||
}
|
||||
var waiting []string
|
||||
for _, m := range p.Movers {
|
||||
if !placed(*p, m.Module) {
|
||||
if !placed(*p, m.Module) && !slices.Contains(p.Superseded, m.Module) {
|
||||
waiting = append(waiting, m.Module)
|
||||
}
|
||||
}
|
||||
@@ -621,6 +672,8 @@ func endPipeline(p *seat.Pipeline, now time.Time, unplaced func(seat.Pipeline, s
|
||||
var notAsked []int
|
||||
for i, s := range p.Steps {
|
||||
switch {
|
||||
case slices.Contains(p.Superseded, s.Module):
|
||||
continue
|
||||
case s.Step == seat.StepResultStepApproval && s.Outcome == seat.StepResultOutcomeWaiting:
|
||||
notAsked = append(notAsked, i)
|
||||
continue
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -361,7 +363,7 @@ func TestARecoverysStepsAreAskedOnTheStreamDelivery(t *testing.T) {
|
||||
}{{true, "mesh.seat.node-build-go.accept.delivery.mesh-pipeline-runner," + wants.AcceptSubjectStep}, {false, "mesh.seat.node-build-go.accept.check.mesh-pipeline-runner," + wants.AcceptSubjectStep}} {
|
||||
var to []string
|
||||
asker := askerFunc(func(accept, _ string, _ any) error { to = append(to, accept); return nil })
|
||||
if err := askAll(asker, "k", asks, func(string) bool { return c.recovery }); err != nil || strings.Join(to, ",") != c.want {
|
||||
if err := askAll(asker, "k", asks, func(string) bool { return c.recovery }, nil); err != nil || strings.Join(to, ",") != c.want {
|
||||
t.Errorf("recovery %v: asked on %v (%v), want %s", c.recovery, to, err, c.want)
|
||||
}
|
||||
}
|
||||
@@ -377,3 +379,105 @@ func TestARecoverysStepsAreAskedOnTheStreamDelivery(t *testing.T) {
|
||||
t.Errorf("recovered: said %v", said)
|
||||
}
|
||||
}
|
||||
|
||||
// A mover whose fingerprint a newer revision moved leaves a running pipeline (H-A-223, H-A-163): its open steps end
|
||||
// superseded, never failed, the other mover's go on and the pipeline ends by them alone; one every mover left ends
|
||||
// superseded.
|
||||
func TestAMovedMoverLeavesARunningPipeline(t *testing.T) {
|
||||
t0 := time.Date(2026, 11, 21, 1, 0, 0, 0, time.UTC)
|
||||
running := func() seat.Pipeline {
|
||||
return seat.Pipeline{V: 1, Pipeline: "main.54", Branch: "main", Source: 54, Revision: 54, Outcome: seat.PipelineOutcomeRunning, Started: &t0,
|
||||
Movers: []seat.Mover{{Module: "a", Commit: "c1", Fingerprint: "src2:a1"}, {Module: "b", Commit: "c1", Fingerprint: "src2:b1"}},
|
||||
Steps: []seat.StepResult{
|
||||
{Module: "a", Step: seat.StepResultStepBuild, Status: seat.StepResultStatusRequired, Outcome: seat.StepResultOutcomeRunning, Ask: "x1", Started: &t0},
|
||||
{Module: "b", Step: seat.StepResultStepBuild, Status: seat.StepResultStatusRequired, Outcome: seat.StepResultOutcomeRunning, Ask: "x2", Started: &t0},
|
||||
{Module: "b", Step: seat.StepResultStepLint, Artifact: "b:program", Status: seat.StepResultStatusRequired, Outcome: seat.StepResultOutcomeRunning, Ask: "x3", Started: &t0,
|
||||
Seat: nodebuildgo.Seat, Node: "n2"}}}
|
||||
}
|
||||
rows := []struct {
|
||||
name string
|
||||
newer []seat.Mover
|
||||
want string
|
||||
}{
|
||||
{"a newer revision moving b: b's steps end superseded, a's go on and end the pipeline passed alone",
|
||||
[]seat.Mover{{Module: "b", Commit: "c2", Fingerprint: "src2:b2"}}, "running; a build running; b build superseded; b lint superseded | passed"},
|
||||
{"a newer revision moving every mover ends the pipeline superseded",
|
||||
[]seat.Mover{{Module: "a", Commit: "c2", Fingerprint: "src2:a2"}, {Module: "b", Commit: "c2", Fingerprint: "src2:b2"}},
|
||||
"superseded; a build superseded; b build superseded; b lint superseded | superseded"},
|
||||
{"a newer revision at the same fingerprint takes nothing", []seat.Mover{{Module: "b", Commit: "c1", Fingerprint: "src2:b1"}},
|
||||
"running; a build running; b build running; b lint running | running"},
|
||||
}
|
||||
for _, r := range rows {
|
||||
t.Run(r.name, func(t *testing.T) {
|
||||
store := memStore{}
|
||||
store["main.54"] = struct {
|
||||
p seat.Pipeline
|
||||
rev uint64
|
||||
}{running(), 1}
|
||||
var said, dropped []string
|
||||
h := &Hearer{Store: store, Say: func(subject string, _ any) error { said = append(said, subject); return nil }, Now: func() time.Time { return t0 },
|
||||
DropRun: func(buildSeat, node, ask, why string) error {
|
||||
dropped = append(dropped, buildSeat+" "+node+" "+ask+" "+fmt.Sprint(strings.Contains(why, "main.57")))
|
||||
return nil
|
||||
}}
|
||||
h.SupersedeRunning(seat.Pipeline{Pipeline: "main.57", Branch: "main", Revision: 57, Movers: r.newer})
|
||||
// b's lint ask on its build node is dropped naming the newer pipeline; the build ask on mesh-builds, which
|
||||
// serves no drop, is not (H-A-223).
|
||||
wantDropped := ""
|
||||
if r.newer[0].Fingerprint != "src2:b1" {
|
||||
wantDropped = "node-build-go n2 x3 true"
|
||||
}
|
||||
if got := strings.Join(dropped, ","); got != wantDropped {
|
||||
t.Errorf("dropped %q; want %q", got, wantDropped)
|
||||
}
|
||||
p, _, _ := store.Get("main.54")
|
||||
got := []string{p.Outcome}
|
||||
for _, s := range p.Steps {
|
||||
got = append(got, s.Module+" "+s.Step+" "+s.Outcome)
|
||||
if s.Outcome == seat.StepResultOutcomeSuperseded && !strings.Contains(s.Why, "main.57") {
|
||||
t.Errorf("the superseded step's why %q names no newer pipeline", s.Why)
|
||||
}
|
||||
}
|
||||
// The mover left holds nothing: a's build passing ends the pipeline by a alone.
|
||||
if p.Outcome == seat.PipelineOutcomeRunning {
|
||||
p.Steps[0].Outcome, p.Steps[0].Ended = seat.StepResultOutcomePassed, &t0
|
||||
endPipeline(&p, t0, nil)
|
||||
}
|
||||
if s := strings.Join(got, "; ") + " | " + p.Outcome; s != r.want {
|
||||
t.Errorf("got %s\nwant %s", s, r.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
// The live path: a newer pipeline started through Start, a take-in's or a release revision's alike, takes the moved
|
||||
// mover out of the older running pipeline over the same store (H-A-223, H-A-213).
|
||||
starts := []struct {
|
||||
name string
|
||||
begin Begin
|
||||
}{
|
||||
{"a take-in", Begin{Branch: "main", Source: 55, Revision: 55, Call: "0123456789abcdef", Movers: []seat.Mover{{Module: "b", Commit: "c2", Fingerprint: "src2:b2"}}}},
|
||||
{"a release revision", Begin{Branch: "main", Source: 54, Revision: 57, Call: "release of mesh-sdk 0.1.26", Movers: []seat.Mover{{Module: "b", Commit: "c1", Fingerprint: "src2:b2"}}}},
|
||||
}
|
||||
for _, r := range starts {
|
||||
t.Run("started by "+r.name, func(t *testing.T) {
|
||||
store := &MoverStore{Raw: newMemRaw()}
|
||||
raw, _ := json.Marshal(running())
|
||||
if err := store.Create("main.54", raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
h := &Hearer{Store: store, Say: func(string, any) error { return nil }, Now: func() time.Time { return t0 }}
|
||||
s := &Starter{Pipelines: store, Store: store, Say: func(string, any) error { return nil }, Now: func() time.Time { return t0 },
|
||||
Supersede: func(newer seat.Pipeline) { h.SupersedeRunning(newer) }}
|
||||
if ok, err := s.Start(r.begin); !ok || err != nil {
|
||||
t.Fatalf("start = %v, %v", ok, err)
|
||||
}
|
||||
p, _, _ := store.Get("main.54")
|
||||
var got []string
|
||||
for _, st := range p.Steps {
|
||||
got = append(got, st.Module+" "+st.Step+" "+st.Outcome)
|
||||
}
|
||||
if s := p.Outcome + "; " + strings.Join(got, "; "); s != "running; a build running; b build superseded; b lint superseded" {
|
||||
t.Errorf("the older pipeline after %s = %s; want b left it", r.name, s)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
|
||||
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
wants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
)
|
||||
|
||||
// Drop asks a build seat's `drop` of a superseded step's ask as a kept call (design 47 G §31, §350; A-182): of node's holder
|
||||
@@ -33,6 +34,9 @@ func (h *Hearer) Taken(ask, node string) error {
|
||||
break
|
||||
}
|
||||
i := stepOfAsk(p, ask)
|
||||
if i >= 0 && !open(p.Steps[i]) {
|
||||
h.Asks.answeredLate(p.Steps[i], h.logf)
|
||||
}
|
||||
if i < 0 || !open(p.Steps[i]) || p.Steps[i].Node == node {
|
||||
break
|
||||
}
|
||||
@@ -50,12 +54,123 @@ func (h *Hearer) Taken(ask, node string) error {
|
||||
|
||||
func ptrTime(t time.Time) *time.Time { return &t }
|
||||
|
||||
// allLeft says whether every mover of p left it for a newer pipeline (H-A-223).
|
||||
func allLeft(p seat.Pipeline) bool {
|
||||
if len(p.Movers) == 0 {
|
||||
return false
|
||||
}
|
||||
for _, m := range p.Movers {
|
||||
if !slices.Contains(p.Superseded, m.Module) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// SupersedeRunning takes out of every older running pipeline of newer's branch, read by that branch's keys alone, each mover whose fingerprint newer moved
|
||||
// (H-A-223): its open steps end superseded, naming newer, never failed; a step reading a test run leaves it, as a cancel's
|
||||
// does; the mover is kept in the pipeline's superseded, so the end judgement (H-A-163) counts it as left and newer carries
|
||||
// it. A pipeline then judged ends as at a step's end, by its other movers alone; one every mover left ends superseded.
|
||||
// A run on request's pipeline is never touched. It answers the pipelines it touched.
|
||||
func (h *Hearer) SupersedeRunning(newer seat.Pipeline) []string {
|
||||
pkeys, err := branchKeys(h.Store, newer.Branch)
|
||||
if err != nil {
|
||||
h.logf("the running pipelines of %s are not listed, so none is superseded by %s: %v", newer.Branch, newer.Pipeline, err)
|
||||
return nil
|
||||
}
|
||||
var touched []string
|
||||
for _, key := range pkeys {
|
||||
if key == newer.Pipeline {
|
||||
continue
|
||||
}
|
||||
if h.supersedeRunningOne(key, newer) {
|
||||
touched = append(touched, key)
|
||||
}
|
||||
}
|
||||
return touched
|
||||
}
|
||||
|
||||
func (h *Hearer) supersedeRunningOne(key string, newer seat.Pipeline) bool {
|
||||
for {
|
||||
p, rev, err := h.Store.Get(key)
|
||||
if err != nil || p.Outcome != seat.PipelineOutcomeRunning || p.Branch != newer.Branch || p.Run != "" || newer.Run != "" ||
|
||||
p.Revision >= newer.Revision {
|
||||
return false
|
||||
}
|
||||
var moved []string
|
||||
for _, m := range p.Movers {
|
||||
if slices.Contains(p.Superseded, m.Module) {
|
||||
continue
|
||||
}
|
||||
if nm, ok := MoverOf(newer, m.Module); ok && nm.Fingerprint != m.Fingerprint {
|
||||
moved = append(moved, m.Module)
|
||||
}
|
||||
}
|
||||
if len(moved) == 0 {
|
||||
return false
|
||||
}
|
||||
now := h.now()
|
||||
why := "superseded: the newer " + newer.Pipeline + " moved its fingerprint, and carries it (H-A-223)"
|
||||
var ended []int
|
||||
var leave, drop []seat.StepResult
|
||||
for i := range p.Steps {
|
||||
st := &p.Steps[i]
|
||||
if !slices.Contains(moved, st.Module) || (!open(*st) && st.Outcome != seat.StepResultOutcomeWaiting) {
|
||||
continue
|
||||
}
|
||||
switch {
|
||||
case sharedRun(*st) != "" && st.Seat != wants.Seat:
|
||||
leave = append(leave, *st)
|
||||
case st.Ask != "" && BoundSeat(st.Seat):
|
||||
drop = append(drop, *st) // its own ask on a build node: the seat's drop cancels it
|
||||
}
|
||||
st.Outcome, st.Why, st.Ended = seat.StepResultOutcomeSuperseded, why, &now
|
||||
p.Unlogged = append(p.Unlogged, endEntry(key, *st, now))
|
||||
ended = append(ended, i)
|
||||
}
|
||||
p.Superseded = append(p.Superseded, moved...)
|
||||
done := h.endPipeline(&p, now)
|
||||
if _, err := h.Store.Update(key, p, rev); errors.Is(err, ErrChanged) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
h.logf("the running pipeline %s is not written, so its movers moved by %s stay: %v", key, newer.Pipeline, err)
|
||||
return false
|
||||
}
|
||||
h.logf("%s: %s moved by the newer %s leave it, %d of its open steps ended superseded (H-A-223)", key, strings.Join(moved, ", "), newer.Pipeline, len(ended))
|
||||
for _, i := range ended {
|
||||
if err := h.say(seat.EventStepEnded, p, &p.Steps[i]); err != nil {
|
||||
h.logf("step-ended of %s's %s %s, superseded, is not said: %v", key, p.Steps[i].Module, p.Steps[i].Step, err)
|
||||
}
|
||||
}
|
||||
for _, st := range leave {
|
||||
h.leaveRun(key, st, "the mover "+st.Module+" left "+key+" for the newer "+newer.Pipeline)
|
||||
}
|
||||
// Its running asks on the build nodes are cancelled through the seat's drop (H-A-223). The seat mesh-builds serves no
|
||||
// drop, so a build ask there runs to its end, which the runner ignores: the step is no longer open.
|
||||
for _, st := range drop {
|
||||
if h.DropRun == nil {
|
||||
h.logf("%s: the ask %s of %s's %s is not dropped: the runner has no drop", key, st.Ask, st.Module, st.Step)
|
||||
continue
|
||||
}
|
||||
if err := h.DropRun(st.Seat, st.Node, st.Ask, why); err != nil {
|
||||
h.logf("%s: the drop of the ask %s of %s's %s is not asked of %s: %v", key, st.Ask, st.Module, st.Step, st.Seat, err)
|
||||
}
|
||||
}
|
||||
if err := h.said(p, nil, done); err != nil {
|
||||
h.logf("%v", err)
|
||||
}
|
||||
h.appendUnlogged(key)
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
// supersedeWaiting takes, in each older waiting pipeline of newer's branch, the movers newer also moves out of it (design 47 H
|
||||
// §748, ADR 0682 §7): a waiting pipeline left with no mover ends superseded, its waiting key deleted, saying wait-ended and its
|
||||
// pipeline-ended. newer is a pipeline just written waiting; a running pipeline is never touched, it runs to its end. It
|
||||
// answers the pipelines it ended.
|
||||
// pipeline-ended. newer is a pipeline just written, waiting or placed: a mover a newer revision moves leaves every pipeline not
|
||||
// ended, a waiting one as a running one (H-A-223, H-A-244 (1)), so a waiting pipeline is placed with only the movers still its
|
||||
// own; a running pipeline's movers leave it by SupersedeRunning. It answers the pipelines it ended.
|
||||
func (s *Starter) supersedeWaiting(newer seat.Pipeline) ([]string, error) {
|
||||
if s.Queue == nil || newer.Outcome != seat.PipelineOutcomeWaiting {
|
||||
if s.Queue == nil || (newer.Outcome != seat.PipelineOutcomeWaiting && newer.Outcome != seat.PipelineOutcomeRunning) {
|
||||
return nil, nil
|
||||
}
|
||||
wkeys, pipelines, err := s.Queue.Waiting()
|
||||
@@ -82,7 +197,7 @@ func (s *Starter) supersedeWaiting(newer seat.Pipeline) ([]string, error) {
|
||||
func (s *Starter) supersedeOne(wkey, key string, newer seat.Pipeline) (bool, error) {
|
||||
for {
|
||||
p, rev, err := s.Store.Get(key)
|
||||
if err != nil || p.Outcome != seat.PipelineOutcomeWaiting || p.Branch != newer.Branch || p.Seq >= newer.Seq {
|
||||
if err != nil || p.Outcome != seat.PipelineOutcomeWaiting || p.Branch != newer.Branch || !olderWaiting(p, newer) {
|
||||
return false, nil
|
||||
}
|
||||
left := slices.DeleteFunc(slices.Clone(p.Movers), func(m seat.Mover) bool { return Moves(newer, m.Module) })
|
||||
@@ -121,6 +236,15 @@ func (s *Starter) supersedeOne(wkey, key string, newer seat.Pipeline) (bool, err
|
||||
}
|
||||
}
|
||||
|
||||
// olderWaiting says whether the waiting pipeline p is older than newer: by its sequence in the queue when newer waits too,
|
||||
// by its catalog revision when newer was placed at once (H-A-244 (1)); a run on request's pipelines are never compared.
|
||||
func olderWaiting(p, newer seat.Pipeline) bool {
|
||||
if newer.Outcome == seat.PipelineOutcomeWaiting {
|
||||
return p.Seq < newer.Seq
|
||||
}
|
||||
return p.Run == "" && newer.Run == "" && p.Revision < newer.Revision
|
||||
}
|
||||
|
||||
// supersededWhy writes, into mover module's build steps, that a dependency it builds on was taken out of p by a newer
|
||||
// pipeline, so it builds against the newest published package (design 47 H §748; H-A-53, H-A-107).
|
||||
func supersededWhy(p seat.Pipeline, module string, entries map[string]catalog.CatalogEntry, steps []seat.StepResult) {
|
||||
|
||||
@@ -3,10 +3,10 @@ package pipeline
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
wants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
nodebuildgo "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
|
||||
@@ -297,11 +297,15 @@ func (v Verbs) Rerun(caller string, in seat.RerunInput) (seat.RerunAnswer, error
|
||||
return -1, err
|
||||
}
|
||||
}
|
||||
oldAsk := s.Ask
|
||||
*s = seat.StepResult{Step: s.Step, Module: s.Module, Artifact: s.Artifact, Status: s.Status, Fingerprint: s.Fingerprint, Seat: s.Seat,
|
||||
Outcome: seat.StepResultOutcomeWaiting, By: caller, Rerun: true}
|
||||
if ask != nil {
|
||||
s.Outcome, s.Started, s.Ask = seat.StepResultOutcomeRunning, &now, ask.ID
|
||||
}
|
||||
if ask != nil && s.Step == seat.StepResultStepValidate && s.Seat == wants.Seat {
|
||||
rerunValidate(p, i, oldAsk, caller, now)
|
||||
}
|
||||
// The pipeline runs again; an approval never asked waits again, to be asked when the step passes.
|
||||
if p.Outcome != seat.PipelineOutcomeRunning {
|
||||
p.Outcome, p.Why, p.Ended = seat.PipelineOutcomeRunning, "", nil
|
||||
@@ -321,7 +325,7 @@ func (v Verbs) Rerun(caller string, in seat.RerunInput) (seat.RerunAnswer, error
|
||||
}
|
||||
if ask != nil {
|
||||
v.Starter.carry(in.Pipeline, []Asked{*ask})
|
||||
if err := askAll(v.Starter.Steps, in.Pipeline, []Asked{*ask}, func(commit string) bool { return v.Starter.recovered(in.Pipeline, commit) }); err != nil {
|
||||
if err := askAll(v.Starter.Steps, in.Pipeline, []Asked{*ask}, func(commit string) bool { return v.Starter.recovered(in.Pipeline, commit) }, v.Starter.Asks.arm); err != nil {
|
||||
return seat.RerunAnswer{}, fmt.Errorf("the step is written running but not asked, it is asked again at the next start: %w", err)
|
||||
}
|
||||
}
|
||||
@@ -468,7 +472,7 @@ func (v Verbs) rerunTest(caller string, in seat.RerunInput, st seat.StepResult)
|
||||
break
|
||||
}
|
||||
v.Starter.carry(in.Pipeline, []Asked{*ask})
|
||||
if err := askAll(v.Starter.Steps, in.Pipeline, []Asked{*ask}, func(commit string) bool { return v.Starter.recovered(in.Pipeline, commit) }); err != nil {
|
||||
if err := askAll(v.Starter.Steps, in.Pipeline, []Asked{*ask}, func(commit string) bool { return v.Starter.recovered(in.Pipeline, commit) }, v.Starter.Asks.arm); err != nil {
|
||||
return seat.RerunAnswer{}, fmt.Errorf("the step is written running but not asked, it is asked again at the next start: %w", err)
|
||||
}
|
||||
}
|
||||
@@ -480,6 +484,29 @@ func (v Verbs) rerunTest(caller string, in seat.RerunInput, st seat.StepResult)
|
||||
return seat.RerunAnswer{Pipeline: in.Pipeline, Result: &res}, nil
|
||||
}
|
||||
|
||||
// rerunValidate puts, with the rerun of validate step i, every other mover's validate step under the same ask back to running
|
||||
// under its one new id, a mover that left the pipeline skipped, so the pipeline keeps one validate ask (H-A-51); and each
|
||||
// composition its validate held ended not asked waits again, asked once validate passed as at the placing (§745).
|
||||
func rerunValidate(p *seat.Pipeline, i int, oldAsk, caller string, now time.Time) {
|
||||
id := p.Steps[i].Ask
|
||||
for j := range p.Steps {
|
||||
st := &p.Steps[j]
|
||||
if slices.Contains(p.Superseded, st.Module) {
|
||||
continue
|
||||
}
|
||||
switch {
|
||||
case j != i && st.Step == seat.StepResultStepValidate && st.Seat == wants.Seat && oldAsk != "" &&
|
||||
(st.Ask == oldAsk || st.Ask == AgainID(oldAsk)) && st.Outcome != seat.StepResultOutcomeSuperseded && st.Outcome != seat.StepResultOutcomeCancelled:
|
||||
*st = seat.StepResult{Step: st.Step, Module: st.Module, Artifact: st.Artifact, Status: st.Status, Fingerprint: st.Fingerprint, Seat: st.Seat,
|
||||
Outcome: seat.StepResultOutcomeRunning, Started: &now, Ask: id, By: caller, Rerun: true}
|
||||
case st.Step == seat.StepResultStepComposition && st.Seat == wants.Seat && st.Outcome == seat.StepResultOutcomeNotAsked &&
|
||||
strings.HasPrefix(st.Why, "validate ended "):
|
||||
*st = seat.StepResult{Step: st.Step, Module: st.Module, Artifact: st.Artifact, Status: st.Status, Fingerprint: st.Fingerprint, Seat: st.Seat,
|
||||
Outcome: seat.StepResultOutcomeWaiting, Why: "asked once validate passed"}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// passedReader says whether reader's step of run passed, or a person accepted it (§30): a step that moves with no new run.
|
||||
func (v Verbs) passedReader(reader RunReader, run string) bool {
|
||||
p, _, err := v.Hearer.Store.Get(reader.Pipeline)
|
||||
@@ -500,8 +527,24 @@ func (v Verbs) passedReader(reader RunReader, run string) bool {
|
||||
}
|
||||
|
||||
// askOf is the ask to put on the serving seat for step s again: the one Place makes for it, under a new id (an ask put twice
|
||||
// under one id is kept once), saying it is a rerun and by whom.
|
||||
// under one id is kept once), saying it is a rerun and by whom. The validate ask is the pipeline's one, naming no mover
|
||||
// (H-A-51): its rerun asks that one ask again; a composition's rerun builds the mover's composition ask (H-A-236).
|
||||
func (v Verbs) askOf(p seat.Pipeline, s seat.StepResult, caller string, now time.Time) (*Asked, error) {
|
||||
id := AskID(now, p.Pipeline, s.Module, s.Artifact, s.Step)
|
||||
if s.Seat == wants.Seat && s.Step == seat.StepResultStepValidate {
|
||||
id = AskID(now, p.Pipeline, "", "", seat.StepResultStepValidate)
|
||||
}
|
||||
a, err := v.placedAsk(p, s, id, now)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
markRerun(a, caller)
|
||||
return a, nil
|
||||
}
|
||||
|
||||
// placedAsk is the ask of step s under id: a validate or composition step of the seat mesh-wants built directly for its
|
||||
// mover (H-A-236), any other the one Place makes for it.
|
||||
func (v Verbs) placedAsk(p seat.Pipeline, s seat.StepResult, id string, now time.Time) (*Asked, error) {
|
||||
if v.Starter == nil || v.Starter.Registration == nil || v.Starter.Steps == nil || (v.View == nil && p.Run == "") {
|
||||
return nil, errors.New("the runner cannot ask a step: it holds no registration or no ask")
|
||||
}
|
||||
@@ -509,6 +552,16 @@ func (v Verbs) askOf(p seat.Pipeline, s seat.StepResult, caller string, now time
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("%s moves no module %q", p.Pipeline, s.Module)
|
||||
}
|
||||
if s.Seat == wants.Seat {
|
||||
switch s.Step {
|
||||
case seat.StepResultStepValidate:
|
||||
a := wantsAsk(p, m, id, wants.StepRunStepValidate)
|
||||
return &a, nil
|
||||
case seat.StepResultStepComposition:
|
||||
a := wantsAsk(p, m, id, wants.StepRunStepComposition)
|
||||
return &a, nil
|
||||
}
|
||||
}
|
||||
reg, ok := v.Starter.Registration(m.Module)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("%s has no registration", m.Module)
|
||||
@@ -533,21 +586,9 @@ func (v Verbs) askOf(p seat.Pipeline, s seat.StepResult, caller string, now time
|
||||
if a.ID != base {
|
||||
continue
|
||||
}
|
||||
a.ID = AskID(now, p.Pipeline, m.Module, s.Artifact, s.Step)
|
||||
switch b := a.Ask.(type) {
|
||||
case nodebuildgo.StepAsk:
|
||||
b.ID = a.ID
|
||||
if b.Step != nil {
|
||||
step := *b.Step
|
||||
step.Rerun, step.By = true, caller
|
||||
b.Step = &step
|
||||
}
|
||||
a.Ask = b
|
||||
case builds.StepRun:
|
||||
b.Rerun, b.By = true, caller
|
||||
a.Ask = b
|
||||
case wants.StepRun:
|
||||
b.Rerun, b.By = true, caller
|
||||
a.ID = id
|
||||
if b, ok := a.Ask.(nodebuildgo.StepAsk); ok {
|
||||
b.ID = id
|
||||
a.Ask = b
|
||||
}
|
||||
return &a, nil
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
|
||||
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
wants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
nodebuildgo "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
)
|
||||
@@ -252,6 +253,47 @@ func TestRerunOfARunAsksWithItsCandidatesHalf(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// H-A-236: a rerun of a failed pipeline's validate step asks the pipeline's one validate ask again (H-A-51), naming no
|
||||
// mover in its id; a rerun of a composition step builds the mover's composition ask directly. Neither is refused "placed no
|
||||
// ask now".
|
||||
func TestRerunOfValidateOrCompositionAsksTheSharedAsk(t *testing.T) {
|
||||
v, store, _, asks, _, _ := verbsFixture()
|
||||
reg, entries := placeFixture()
|
||||
reg.Policy.Steps.Validate, reg.Policy.Steps.Composition = "required", "required"
|
||||
v.Starter.Registration = func(string) (catalog.Registration, bool) { return reg, true }
|
||||
p, rev, _ := store.Get(pkey)
|
||||
p.Steps, _ = Place(p, p.Movers[0], reg, entries, time.Unix(1, 0), "the-runner")
|
||||
for i := range p.Steps {
|
||||
switch p.Steps[i].Step {
|
||||
case seat.StepResultStepValidate:
|
||||
p.Steps[i].Outcome = seat.StepResultOutcomeFailed
|
||||
case seat.StepResultStepComposition:
|
||||
p.Steps[i].Outcome = seat.StepResultOutcomeNotAsked
|
||||
}
|
||||
}
|
||||
p.Outcome = seat.PipelineOutcomeFailed
|
||||
store.Update(pkey, p, rev)
|
||||
now := v.now()
|
||||
for _, c := range []struct{ step, module string }{{seat.StepResultStepValidate, ""}, {seat.StepResultStepComposition, "app"}} {
|
||||
n := len(asks.asks)
|
||||
r, err := v.Rerun("operator", seat.RerunInput{Pipeline: pkey, Module: "app", Step: c.step, Why: "the holder restarted"})
|
||||
want := AskID(now, pkey, c.module, "", c.step)
|
||||
if err != nil || r.Result == nil || r.Result.Ask != want || r.Result.Outcome != seat.StepResultOutcomeRunning {
|
||||
t.Fatalf("a rerun of %s = %+v, %v, want running under %s", c.step, r.Result, err, want)
|
||||
}
|
||||
if len(asks.asks) != n+1 || asks.ids[n] != want {
|
||||
t.Fatalf("a rerun of %s asked %v, want one ask under %s", c.step, asks.ids[n:], want)
|
||||
}
|
||||
b, ok := asks.asks[n].(wants.StepRun)
|
||||
if !ok || b.Step != c.step || b.Module != "app" || b.Pipeline != pkey || !b.Rerun || b.By != "operator" || b.Asker != seat.Seat {
|
||||
t.Errorf("the %s ask = %+v", c.step, asks.asks[n])
|
||||
}
|
||||
p, rev, _ = store.Get(pkey)
|
||||
p.Steps[stepIndex(p, c.step, "")].Outcome = seat.StepResultOutcomeFailed
|
||||
store.Update(pkey, p, rev)
|
||||
}
|
||||
}
|
||||
|
||||
func stepIndex(p seat.Pipeline, step, artifact string) int {
|
||||
for i, s := range p.Steps {
|
||||
if s.Step == step && s.Artifact == artifact {
|
||||
@@ -340,3 +382,44 @@ func TestStartAndAskAgain(t *testing.T) {
|
||||
t.Error("an approval running was asked again")
|
||||
}
|
||||
}
|
||||
|
||||
// Review of H-A-236 (H-A-51): a rerun of validate puts every mover's validate step under the old ask back to running under
|
||||
// the one new id, a mover that left skipped, and the compositions its failed validate ended not asked wait again; the new
|
||||
// ask's end ends both validates and asks both compositions.
|
||||
func TestAValidateRerunMovesEveryMoversValidateAndReopensItsCompositions(t *testing.T) {
|
||||
v, store, _, asks, _, _ := verbsFixture()
|
||||
p, rev, _ := store.Get(pkey)
|
||||
p.Outcome = seat.PipelineOutcomeFailed
|
||||
p.Movers = append(p.Movers, seat.Mover{Module: "lib", Commit: "c2", Fingerprint: "src2:bb"}, seat.Mover{Module: "gone", Commit: "c3", Fingerprint: "src2:cc"})
|
||||
p.Superseded = []string{"gone"}
|
||||
notAsked := "validate ended failed: nothing is composed"
|
||||
p.Steps = nil
|
||||
for _, m := range []string{"app", "lib", "gone"} {
|
||||
p.Steps = append(p.Steps,
|
||||
seat.StepResult{Module: m, Step: seat.StepResultStepValidate, Status: seat.StepResultStatusRequired, Seat: wants.Seat, Ask: "old", Outcome: seat.StepResultOutcomeFailed},
|
||||
seat.StepResult{Module: m, Step: seat.StepResultStepComposition, Status: seat.StepResultStatusRequired, Seat: wants.Seat, Outcome: seat.StepResultOutcomeNotAsked, Why: notAsked})
|
||||
}
|
||||
p.Steps[4].Outcome, p.Steps[5].Outcome = seat.StepResultOutcomeSuperseded, seat.StepResultOutcomeSuperseded
|
||||
store.Update(pkey, p, rev)
|
||||
r, err := v.Rerun("operator", seat.RerunInput{Pipeline: pkey, Module: "app", Step: seat.StepResultStepValidate, Why: "again"})
|
||||
if err != nil || len(asks.ids) != 1 || r.Result.Ask != asks.ids[0] {
|
||||
t.Fatalf("the rerun = %+v (%v), asked %v; want one validate ask", r.Result, err, asks.ids)
|
||||
}
|
||||
got, _, _ := store.Get(pkey)
|
||||
for i, want := range []string{"running " + r.Result.Ask, "waiting ", "running " + r.Result.Ask, "waiting ", "superseded old", "superseded "} {
|
||||
if s := got.Steps[i]; s.Outcome+" "+s.Ask != want {
|
||||
t.Errorf("%s's %s = %s %q, want %s", s.Module, s.Step, s.Outcome, s.Ask, want)
|
||||
}
|
||||
}
|
||||
bs := StepSeats[len(StepSeats)-1]
|
||||
ev := StepEnded{ID: r.Result.Ask, Node: "n1", Pipeline: pkey, Module: "lib", Commit: "c2", Fingerprint: "src2:bb", Step: seat.StepResultStepValidate, Outcome: "passed"}
|
||||
if _, err := v.Hearer.OnStepEnded(bs, bs.Prefix+"n1", ev); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, _, _ = store.Get(pkey)
|
||||
for _, i := range []int{1, 3} {
|
||||
if s := got.Steps[i]; s.Outcome != seat.StepResultOutcomeRunning || s.Ask == "" {
|
||||
t.Errorf("%s's composition = %s %q, want asked once validate passed", s.Module, s.Outcome, s.Ask)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -84,7 +84,7 @@ func TestManifest(t *testing.T) {
|
||||
"mesh-delivery-indexer.take-in.taken", "mesh-builds.package-published", "mesh-builds.toolchain-agreed", "mesh-builds.step-ended",
|
||||
"operator-channel.decided", "node-build-go.step-ended", "node-build-node.step-ended",
|
||||
"node-build-image.step-ended", "node-build-go.ask-taken", "node-build-node.ask-taken", "node-build-image.ask-taken",
|
||||
"node-build-go.step-progress", "node-build-node.step-progress", "node-build-image.step-progress", "mesh-wants.step-ended", "mesh-catalog.registration-changed",
|
||||
"node-build-go.step-progress", "node-build-node.step-progress", "node-build-image.step-progress", "mesh-wants.step-ended", "mesh-wants.step-progress", "node-engine.running", "mesh-catalog.registration-changed",
|
||||
"mesh-catalog.catalog-revision", "mesh-catalog.version-released", "mesh-conditions.findings-asked", "mesh-delivery.recovery-merged",
|
||||
// A-186: the seat list its write verbs' callers are checked against, heard as it changes.
|
||||
"mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed",
|
||||
@@ -199,7 +199,7 @@ func runSettingsOf(raw []byte) bool {
|
||||
if json.Unmarshal(raw, &m) != nil {
|
||||
return false
|
||||
}
|
||||
want := map[string]string{"test-runs-cap-mib": "number", "test-runs-kept": "string", "test-step-bound": "string", "run-untaken-bound": "string", "widens-read-bound": "string", "revisions-kept": "number",
|
||||
want := map[string]string{"test-runs-cap-mib": "number", "test-runs-kept": "string", "test-step-bound": "string", "run-untaken-bound": "string", "ask-unanswered-bound": "string", "ask-silent-bound": "string", "widens-read-bound": "string", "revisions-kept": "number",
|
||||
"pipelines-at-once": "number", "pipelines-waiting-cap": "number"}
|
||||
if len(m.Config) != len(want) || m.Config["test-runs-cap-mib"]["maximum"] != float64(64) || m.Config["test-runs-cap-mib"]["default"] != float64(16) ||
|
||||
m.Config["pipelines-at-once"]["minimum"] != float64(1) || m.Config["pipelines-at-once"]["maximum"] != float64(8) ||
|
||||
|
||||
@@ -58,6 +58,8 @@
|
||||
"node-build-node.step-progress",
|
||||
"node-build-image.step-progress",
|
||||
"mesh-wants.step-ended",
|
||||
"mesh-wants.step-progress",
|
||||
"node-engine.running",
|
||||
"mesh-catalog.registration-changed",
|
||||
"mesh-catalog.catalog-revision",
|
||||
"mesh-catalog.version-released",
|
||||
@@ -151,6 +153,18 @@
|
||||
"description": "how long a test run's ask may wait untaken after the run's start before the run ends could not run, a Go duration such as 24h (design 47 G section 725, A-466s, A-469x).",
|
||||
"trusted": false,
|
||||
"default": "24h"
|
||||
},
|
||||
"ask-unanswered-bound": {
|
||||
"type": "string",
|
||||
"description": "how long an ask the runner puts on a seat, but a test run's, may go without its holder's first word (an ask-taken, a step progress or the step's end) before it is asked once more, and as long again before its step ends could not run, a Go duration such as 10m (design 47 H H-A-235, H-A-238).",
|
||||
"trusted": false,
|
||||
"default": "10m"
|
||||
},
|
||||
"ask-silent-bound": {
|
||||
"type": "string",
|
||||
"description": "how long an ask the runner puts on a seat whose holder says nothing before the step's end (mesh-builds, mesh-wants) may go without that end before its step ends could not run, a Go duration such as 45m (design 47 H H-A-243).",
|
||||
"trusted": false,
|
||||
"default": "45m"
|
||||
}
|
||||
},
|
||||
"state": [
|
||||
@@ -278,6 +292,22 @@
|
||||
"mode": "0600",
|
||||
"content": "${config:run-untaken-bound}"
|
||||
},
|
||||
{
|
||||
"id": "ask-unanswered-bound",
|
||||
"type": "file",
|
||||
"path": "${dir:state}/ask-unanswered-bound",
|
||||
"owner": "mesh-delivery-pipeline",
|
||||
"mode": "0600",
|
||||
"content": "${config:ask-unanswered-bound}"
|
||||
},
|
||||
{
|
||||
"id": "ask-silent-bound",
|
||||
"type": "file",
|
||||
"path": "${dir:state}/ask-silent-bound",
|
||||
"owner": "mesh-delivery-pipeline",
|
||||
"mode": "0600",
|
||||
"content": "${config:ask-silent-bound}"
|
||||
},
|
||||
{
|
||||
"id": "widens-read-bound",
|
||||
"type": "file",
|
||||
@@ -326,6 +356,8 @@
|
||||
"MESH_RUNNER_WIDENS_READ_BOUND_FILE": "${dir:state}/widens-read-bound",
|
||||
"MESH_RUNNER_REVISIONS_KEPT_FILE": "${dir:state}/revisions-kept",
|
||||
"MESH_RUNNER_RUN_UNTAKEN_BOUND_FILE": "${dir:state}/run-untaken-bound",
|
||||
"MESH_RUNNER_ASK_UNANSWERED_BOUND_FILE": "${dir:state}/ask-unanswered-bound",
|
||||
"MESH_RUNNER_ASK_SILENT_BOUND_FILE": "${dir:state}/ask-silent-bound",
|
||||
"MESH_RUNNER_PIPELINES_AT_ONCE_FILE": "${dir:state}/pipelines-at-once",
|
||||
"MESH_RUNNER_PIPELINES_WAITING_CAP_FILE": "${dir:state}/pipelines-waiting-cap"
|
||||
},
|
||||
|
||||
@@ -45,9 +45,14 @@ type Transfer[M any] struct {
|
||||
// chunks taken per part.
|
||||
Held []string `json:"held,omitempty"`
|
||||
Chunks map[string]int `json:"chunks,omitempty"`
|
||||
Opened time.Time `json:"opened"`
|
||||
Last time.Time `json:"last"`
|
||||
Shape M `json:"shape"`
|
||||
// FillAtOpen is the objects' fill read from the store's figures when the transfer opened (or when chunks were dropped
|
||||
// since), and TakenSince the bytes of the chunks taken after that read: the fill told per chunk is their sum, kept
|
||||
// with the transfer, never read again per chunk (design 47 H lead answers H-A-227, H-A-228).
|
||||
FillAtOpen uint64 `json:"fill_at_open,omitempty"`
|
||||
TakenSince uint64 `json:"taken_since,omitempty"`
|
||||
Opened time.Time `json:"opened"`
|
||||
Last time.Time `json:"last"`
|
||||
Shape M `json:"shape"`
|
||||
}
|
||||
|
||||
// Lacks is the digests of the parts the transfer does not hold yet, sorted.
|
||||
@@ -197,11 +202,12 @@ func (r *Receiver[M]) Open(id, opener string, parts []Want, shape M) (Received,
|
||||
}
|
||||
return Received{ID: id, Lacks: t.Lacks()}, r.arm(id)
|
||||
}
|
||||
t := Transfer[M]{ID: id, Opener: opener, Parts: parts, Chunks: map[string]int{}, Opened: r.now(), Last: r.now(), Shape: shape}
|
||||
used, err := r.Objects.Used()
|
||||
if err != nil {
|
||||
return Received{}, fmt.Errorf("the objects' fill cannot be read: %w", err)
|
||||
}
|
||||
t := Transfer[M]{ID: id, Opener: opener, Parts: parts, Chunks: map[string]int{}, FillAtOpen: used, Opened: r.now(),
|
||||
Last: r.now(), Shape: shape}
|
||||
var size uint64
|
||||
for _, p := range parts {
|
||||
size += uint64(max(p.Size, 0))
|
||||
@@ -219,7 +225,7 @@ func (r *Receiver[M]) Open(id, opener string, parts []Want, shape M) (Received,
|
||||
if err := r.Transfers.Put(t); err != nil {
|
||||
return Received{}, fmt.Errorf("transfer %s cannot be kept: %w", id, err)
|
||||
}
|
||||
r.fill()
|
||||
r.tell(used)
|
||||
if len(t.Lacks()) == 0 {
|
||||
return Received{ID: id, Ended: true}, r.finish(t)
|
||||
}
|
||||
@@ -271,6 +277,7 @@ func (r *Receiver[M]) Take(id, opener, digest string, sequence int, last bool, d
|
||||
// A part the holder lacks travels again whole from chunk 0 (design 51 Publishing): its earlier chunks are dropped.
|
||||
r.dropChunks(t, digest)
|
||||
t.Chunks[digest] = 0
|
||||
r.reanchor(&t)
|
||||
}
|
||||
if err := r.Objects.Put(r.chunkName(id, digest, sequence), data); err != nil {
|
||||
return Received{}, fmt.Errorf("the objects did not take the chunk: %w", err)
|
||||
@@ -280,6 +287,7 @@ func (r *Receiver[M]) Take(id, opener, digest string, sequence int, last bool, d
|
||||
t.Chunks = map[string]int{}
|
||||
}
|
||||
t.Chunks[digest] = sequence + 1
|
||||
t.TakenSince += uint64(len(data))
|
||||
t.Last = r.now()
|
||||
if last {
|
||||
whole, err := r.part(t, digest)
|
||||
@@ -289,8 +297,9 @@ func (r *Receiver[M]) Take(id, opener, digest string, sequence int, last bool, d
|
||||
if got := "sha256:" + sum(whole); got != digest {
|
||||
r.dropChunks(t, digest)
|
||||
delete(t.Chunks, digest)
|
||||
r.reanchor(&t)
|
||||
_ = r.Transfers.Put(t)
|
||||
r.fill()
|
||||
r.tell(t.FillAtOpen)
|
||||
return Received{}, fmt.Errorf("refused: the bytes of part %s hash to %s; send the part again", digest, got)
|
||||
}
|
||||
t.Held = append(t.Held, digest)
|
||||
@@ -298,7 +307,7 @@ func (r *Receiver[M]) Take(id, opener, digest string, sequence int, last bool, d
|
||||
if err := r.Transfers.Put(t); err != nil {
|
||||
return Received{}, err
|
||||
}
|
||||
r.fill()
|
||||
r.tell(t.FillAtOpen + t.TakenSince)
|
||||
if len(t.Lacks()) == 0 {
|
||||
return Received{ID: id, Ended: true}, r.finish(t)
|
||||
}
|
||||
@@ -365,19 +374,36 @@ func (r *Receiver[M]) chunkName(id, digest string, seq int) string {
|
||||
return fmt.Sprintf("%s%s/%s/%d", r.Prefix, id, strings.TrimPrefix(digest, "sha256:"), seq)
|
||||
}
|
||||
|
||||
// fill tells Filled the objects' fill.
|
||||
// fill reads the objects' fill from the store's figures and tells Filled: at a transfer's finish and drop, never per
|
||||
// chunk (H-A-227).
|
||||
func (r *Receiver[M]) fill() {
|
||||
cap := r.CapNow()
|
||||
if r.Filled == nil || cap == 0 {
|
||||
if r.Filled == nil {
|
||||
return
|
||||
}
|
||||
used, err := r.Objects.Used()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
r.tell(used)
|
||||
}
|
||||
|
||||
// tell tells Filled a fill already known: read at the open, or that read plus the bytes taken since.
|
||||
func (r *Receiver[M]) tell(used uint64) {
|
||||
cap := r.CapNow()
|
||||
if r.Filled == nil || cap == 0 {
|
||||
return
|
||||
}
|
||||
r.Filled(used, cap, used*100 >= cap*r.fullPct())
|
||||
}
|
||||
|
||||
// reanchor reads the fill again after a transfer's chunks were dropped, whose sizes it does not keep: the read becomes
|
||||
// the transfer's FillAtOpen and nothing is taken since. A read that fails keeps the figures it had.
|
||||
func (r *Receiver[M]) reanchor(t *Transfer[M]) {
|
||||
if used, err := r.Objects.Used(); err == nil {
|
||||
t.FillAtOpen, t.TakenSince = used, 0
|
||||
}
|
||||
}
|
||||
|
||||
// Drop drops a transfer, by hand or because it was idle: its chunks and its key.
|
||||
func (r *Receiver[M]) Drop(id string) (Transfer[M], error) {
|
||||
t, found, err := r.Transfers.Get(id)
|
||||
|
||||
@@ -29,6 +29,14 @@ func (m memObjects) Used() (uint64, error) {
|
||||
return u, nil
|
||||
}
|
||||
|
||||
// countedObjects counts the reads of the objects' fill.
|
||||
type countedObjects struct {
|
||||
memObjects
|
||||
reads int
|
||||
}
|
||||
|
||||
func (c *countedObjects) Used() (uint64, error) { c.reads++; return c.memObjects.Used() }
|
||||
|
||||
// memTransfers keeps each transfer as the bus keeps it, its JSON, so every case reads back what a key-value state
|
||||
// gives (BusTransfers): an empty map kept with omitempty comes back nil.
|
||||
type memTransfers[M any] map[string][]byte
|
||||
@@ -244,6 +252,43 @@ func aHolderTakesAFileInParts(t *testing.T) {
|
||||
}
|
||||
})
|
||||
|
||||
// Design 47 H lead answers H-A-227, H-A-228: the fill is read from the store once when a transfer opens and kept in the
|
||||
// transfer with the bytes taken since; each chunk tells that sum and reads nothing, also through a receiver built
|
||||
// anew per call, as the holders build it; the finish reads it again.
|
||||
t.Run("the fill is read once at the open and summed per chunk", func(t *testing.T) {
|
||||
store := memObjects{"other": make([]byte, 100)}
|
||||
counted := &countedObjects{memObjects: store}
|
||||
transfers := memTransfers[packageShape]{}
|
||||
var told []uint64
|
||||
fresh := func() *Receiver[packageShape] {
|
||||
return &Receiver[packageShape]{Objects: counted, Transfers: transfers, Cap: 1 << 20, MaxChunk: 100, Now: func() time.Time { return now },
|
||||
Filled: func(used, _ uint64, _ bool) { told = append(told, used) },
|
||||
Store: func(Transfer[packageShape], map[string][]byte) error { return nil }}
|
||||
}
|
||||
file := []byte(strings.Repeat("b", 1000))
|
||||
w := Want{Digest: digestOf(file), Size: int64(len(file))}
|
||||
if _, err := fresh().Open("t9", "builder", []Want{w}, packageShape{"m", "1"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for seq := 0; seq < 9; seq++ {
|
||||
if _, err := fresh().Take("t9", "builder", w.Digest, seq, false, file[seq*100:(seq+1)*100], nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if counted.reads != 1 {
|
||||
t.Errorf("the fill was read %d times across an open and 9 chunks, want once", counted.reads)
|
||||
}
|
||||
if want := []uint64{100, 200, 300, 400, 500, 600, 700, 800, 900, 1000}; !reflect.DeepEqual(told, want) {
|
||||
t.Errorf("fills told %v, want %v (the read at the open plus the bytes taken since)", told, want)
|
||||
}
|
||||
if got, err := fresh().Take("t9", "builder", w.Digest, 9, true, file[900:], nil); err != nil || !got.Ended {
|
||||
t.Fatalf("the last chunk = %+v, %v", got, err)
|
||||
}
|
||||
if counted.reads != 2 || told[len(told)-1] != 100 {
|
||||
t.Errorf("after the finish: %d reads, told %d; want the store read again, 100", counted.reads, told[len(told)-1])
|
||||
}
|
||||
})
|
||||
|
||||
// An open transfer is bounded by its idle key (Arm); the key's end (Expired) drops it idle, arms it again for the
|
||||
// rest when it took a chunk since, and leaves a gone one; a key not written fails the open loudly. The cap the
|
||||
// binding carries (Bound) counts over Cap.
|
||||
|
||||
Reference in New Issue
Block a user