Compare commits
18
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
18136ce063 | ||
|
|
a5495bb15e | ||
|
|
5d3164b5b7 | ||
|
|
96d8136e2b | ||
|
|
c45f1621fb | ||
|
|
fd2cb13961 | ||
|
|
915a54f136 | ||
|
|
1c0309df23 | ||
|
|
7dd2891f32 | ||
|
|
b7e364e47b | ||
|
|
dca2545eb7 | ||
|
|
bd5852b817 | ||
|
|
1bfd54425b | ||
|
|
c6c86801ca | ||
|
|
33feb148ce | ||
|
|
f44442097c | ||
|
|
1f765ef17b | ||
|
|
203454b1a9 |
@@ -19,6 +19,7 @@ import (
|
||||
mpr "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/testbus"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
"mesh-delivery-assigner/internal/assigngraph"
|
||||
@@ -170,15 +171,21 @@ func TestTheWantsAreSaidWantedFromTheKeptAssignmentGraphRevision(t *testing.T) {
|
||||
}
|
||||
|
||||
// blockedObjects is an object bucket whose listing waits while a gate stands: a keeping that does not end until it is opened.
|
||||
// A listing that meets the gate closes entered: the keeping took its need and runs it (H-A-476).
|
||||
type blockedObjects struct {
|
||||
memObjects
|
||||
mu sync.Mutex
|
||||
gate chan struct{}
|
||||
mu sync.Mutex
|
||||
gate chan struct{}
|
||||
entered chan struct{}
|
||||
}
|
||||
|
||||
func (b *blockedObjects) Names(ctx context.Context) ([]string, error) {
|
||||
b.mu.Lock()
|
||||
gate := b.gate
|
||||
gate, entered := b.gate, b.entered
|
||||
if gate != nil && entered != nil {
|
||||
close(entered)
|
||||
b.entered = nil
|
||||
}
|
||||
b.mu.Unlock()
|
||||
if gate != nil {
|
||||
<-gate
|
||||
@@ -204,16 +211,33 @@ func (b *blockedObjects) Get(ctx context.Context, name string) ([]byte, error) {
|
||||
func (b *blockedObjects) block() func() {
|
||||
gate := make(chan struct{})
|
||||
b.mu.Lock()
|
||||
b.gate = gate
|
||||
b.gate, b.entered = gate, make(chan struct{})
|
||||
b.mu.Unlock()
|
||||
return func() {
|
||||
b.mu.Lock()
|
||||
b.gate = nil
|
||||
b.gate, b.entered = nil, nil
|
||||
b.mu.Unlock()
|
||||
close(gate)
|
||||
}
|
||||
}
|
||||
|
||||
// awaitBlocked returns once a keeping meets the standing gate, so what the keeping took is read after it took it, and fails the
|
||||
// test when none did within bound (H-A-476: a test waits on the signal of the work it reads after, never on an order it assumes).
|
||||
func (b *blockedObjects) awaitBlocked(t *testing.T, bound time.Duration, what string) {
|
||||
t.Helper()
|
||||
b.mu.Lock()
|
||||
entered := b.entered
|
||||
b.mu.Unlock()
|
||||
if entered == nil {
|
||||
return // met already
|
||||
}
|
||||
select {
|
||||
case <-entered:
|
||||
case <-time.After(bound):
|
||||
t.Fatalf("no keeping met the gate within %s: %s", bound, what)
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 H-A-259, H-A-261: the keeping of the assignment graph runs off the pass loop. At the start, while the first keeping is
|
||||
// blocked, a pass returns and writes no want; the keeping's end wakes a pass, which writes the passed pipeline's want. Later, while a
|
||||
// keeping of new inputs is blocked, a pass writes the wants from the revision it holds, and a pass of the same inputs hands nothing
|
||||
@@ -222,6 +246,7 @@ func TestAPassWritesWantsWhileAKeepingIsBlocked(t *testing.T) {
|
||||
fp := mcc.SourcePrefix2 + strings.Repeat("a", 64)
|
||||
wantsKV := &memKV{values: map[string][]byte{}}
|
||||
p := newPasser(kvSet{buckets.Wants(): wantsKV, buckets.Asks(): &memKV{values: map[string][]byte{}}}, time.Now())
|
||||
bound := testbus.SetupBound(t) // every wait on a goroutine's work, never a value in the test (H-A-476)
|
||||
var mu sync.Mutex
|
||||
var journal strings.Builder
|
||||
p.logf = func(f string, a ...any) { mu.Lock(); fmt.Fprintf(&journal, f+"\n", a...); mu.Unlock() }
|
||||
@@ -252,7 +277,7 @@ func TestAPassWritesWantsWhileAKeepingIsBlocked(t *testing.T) {
|
||||
go func() { p.pass(context.Background()); close(done) }()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(10 * time.Second):
|
||||
case <-time.After(bound):
|
||||
t.Fatal("the pass waits on the keeping of the assignment graph")
|
||||
}
|
||||
}
|
||||
@@ -281,7 +306,7 @@ func TestAPassWritesWantsWhileAKeepingIsBlocked(t *testing.T) {
|
||||
open()
|
||||
select {
|
||||
case <-woke:
|
||||
case <-time.After(10 * time.Second):
|
||||
case <-time.After(bound):
|
||||
t.Fatal("the first keeping's end wakes no pass")
|
||||
}
|
||||
pass()
|
||||
@@ -292,7 +317,8 @@ func TestAPassWritesWantsWhileAKeepingIsBlocked(t *testing.T) {
|
||||
open = objects.block() // a keeping of new inputs is blocked
|
||||
placeOn(t, p.w, "g14", "web")
|
||||
pass()
|
||||
pass() // the same inputs again
|
||||
pass() // the same inputs again
|
||||
objects.awaitBlocked(t, bound, "the keeping of new inputs never took its need") // H-A-476: read the need once it was taken
|
||||
if w, err := wantOf("g14"); err != nil || w.Fp != fp || w.Version != "0.1.2" {
|
||||
t.Errorf("the want of web on g14 = %+v (%v), want the passed version while a keeping is blocked", w, err)
|
||||
}
|
||||
|
||||
@@ -17,6 +17,7 @@ import (
|
||||
mprc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshpipelinerunnerclient"
|
||||
mpcat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
mpr "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/pipelinekey"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
@@ -166,6 +167,12 @@ func (e *pipelineEnds) take(p mpr.PipelineEnded) (bool, error) {
|
||||
if p.Outcome != mpr.PipelineEndedOutcomePassed || p.Run != "" {
|
||||
return false, nil
|
||||
}
|
||||
// The pipeline's key is read by the one parse of the runner's seat's key forms (design 47 H2 H-A-485): a key it does not
|
||||
// know is refused loudly, and nothing of the pipeline is kept, never an empty tier progress.
|
||||
if _, err := pipelinekey.Parse(p.Pipeline); err != nil {
|
||||
e.logf("the pipeline-ended of %s is refused, and none of its movers' versions is kept: %v", p.Pipeline, err)
|
||||
return false, nil
|
||||
}
|
||||
changed := false
|
||||
var kept []tierMover
|
||||
versioned := map[string]bool{}
|
||||
|
||||
@@ -96,6 +96,17 @@ func TestThePassedVersionIsKept(t *testing.T) {
|
||||
ended("main", 5, pass, mover("web", "fp1", "0.1.2", 1, ok))}, &End{Pipeline: "main.6", Source: 6, Fp: "fp2", Version: "0.1.3", Rank: 1}},
|
||||
{"a newer version replaces", []mpr.PipelineEnded{ended("main", 5, pass, mover("web", "fp1", "0.1.2", 1, ok)),
|
||||
ended("main", 6, pass, mover("web", "fp2", "0.1.10", 1, ok))}, &End{Pipeline: "main.6", Source: 6, Fp: "fp2", Version: "0.1.10", Rank: 1}},
|
||||
// H-A-485: an again pipeline's end is kept as a take-in's; a key of no form the runner's seat defines is refused, nothing kept.
|
||||
{"an again pipeline keeps its version", []mpr.PipelineEnded{func() mpr.PipelineEnded {
|
||||
p := ended("main", 5, pass, mover("web", "fp1", "0.1.2", 2, ok))
|
||||
p.Pipeline = "main.5.again.2"
|
||||
return p
|
||||
}()}, &End{Pipeline: "main.5.again.2", Source: 5, Fp: "fp1", Version: "0.1.2", Rank: 2}},
|
||||
{"a key of no form is refused", []mpr.PipelineEnded{func() mpr.PipelineEnded {
|
||||
p := ended("main", 5, pass, mover("web", "fp1", "0.1.2", 2, ok))
|
||||
p.Pipeline = "main.5.later.2"
|
||||
return p
|
||||
}()}, nil},
|
||||
{"a run on request keeps nothing", []mpr.PipelineEnded{func() mpr.PipelineEnded {
|
||||
p := ended("main", 5, pass, mover("web", "fp1", "0.1.2", 1, ok))
|
||||
p.Run = "r1"
|
||||
@@ -109,6 +120,9 @@ func TestThePassedVersionIsKept(t *testing.T) {
|
||||
}
|
||||
}
|
||||
got, kept := e.endOf("web", "main")
|
||||
if ks, _ := e.st.keys(); c.name == "a key of no form is refused" && len(ks) != 0 {
|
||||
t.Errorf("%s: the state written holds %v, want nothing (no empty tier progress)", c.name, ks)
|
||||
}
|
||||
switch {
|
||||
case c.want == nil && kept:
|
||||
t.Errorf("%s: kept %+v", c.name, got)
|
||||
|
||||
@@ -6,12 +6,14 @@ import (
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
|
||||
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/pipelinekey"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
|
||||
@@ -58,10 +60,16 @@ type tierProgress struct {
|
||||
// tierKey is the key of pipeline's tier progress; pipeline is the runner's key `<branch>.<source>`, its branch escaped.
|
||||
func tierKey(pipeline string) string { return tierPrefix + pipeline }
|
||||
|
||||
// isTierKey says whether a key of the state written is a pipeline's tier progress: three parts, the first `pipeline` (a module's
|
||||
// end has two).
|
||||
// isTierKey says whether a key of the state written is a pipeline's tier progress: `pipeline.` and a pipeline's key of any form
|
||||
// the runner's seat defines, read by the one parse (design 47 H2 H-A-485), so an again pipeline's progress is one as a take-in's
|
||||
// is (a module's end, `<module>.<branch>`, is none).
|
||||
func isTierKey(key string) bool {
|
||||
return strings.HasPrefix(key, tierPrefix) && strings.Count(key, ".") == 2
|
||||
rest, ok := strings.CutPrefix(key, tierPrefix)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
_, err := pipelinekey.Parse(rest)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// startTiers writes the tier progress of a passed pipeline, holding every passed mover, versioned or not, its first tier the
|
||||
@@ -166,11 +174,11 @@ func (p *passer) tiers(n assign.Nodes, placedOn func(module, branch string) []st
|
||||
continue
|
||||
}
|
||||
pipeline := strings.TrimPrefix(key, tierPrefix)
|
||||
branchEsc, number, _ := strings.Cut(pipeline, ".")
|
||||
branch, err := keys.UnescapeName(branchEsc)
|
||||
pk, err := pipelinekey.Parse(pipeline)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
branch, number := pk.Branch, strconv.FormatInt(pk.Revision, 10)
|
||||
for {
|
||||
t, rev, ok := p.ends.tiersOf(pipeline)
|
||||
if !ok {
|
||||
|
||||
@@ -28,7 +28,12 @@ import (
|
||||
// tier-bound: the finding tier-late names it and its machine, pipeline-wanted says tier-late, and no consumer want is written.
|
||||
func TestAPipelinesWantsAreWrittenTierByTier(t *testing.T) {
|
||||
db, web := meshcatalogclient.SourcePrefix2+strings.Repeat("d", 64), meshcatalogclient.SourcePrefix2+strings.Repeat("c", 64)
|
||||
for _, runs := range []bool{true, false} {
|
||||
// H-A-485: a pipeline started again at the head, keyed `<branch>.<revision>.again.<n>`, moves tier by tier as a take-in's does.
|
||||
for _, c := range []struct {
|
||||
runs bool
|
||||
pipeline string
|
||||
}{{true, trunkBranch + ".5"}, {false, trunkBranch + ".5"}, {true, trunkBranch + ".5.again.2"}, {false, trunkBranch + ".5.again.2"}} {
|
||||
runs, pipeline := c.runs, c.pipeline
|
||||
start := time.Date(2026, 11, 20, 4, 0, 0, 0, time.UTC)
|
||||
now := start
|
||||
wantsKV := &memKV{values: map[string][]byte{}}
|
||||
@@ -41,7 +46,7 @@ func TestAPipelinesWantsAreWrittenTierByTier(t *testing.T) {
|
||||
var said []meshwants.PipelineWanted
|
||||
p.publishWants = func(event string, body any, _ string) error {
|
||||
if b, ok := body.(meshwants.PipelineWantedEvent); ok {
|
||||
if event != "pipeline-wanted."+trunkBranch+".5" {
|
||||
if event != "pipeline-wanted."+pipeline {
|
||||
t.Errorf("pipeline-wanted said as %s", event)
|
||||
}
|
||||
said = append(said, b)
|
||||
@@ -65,7 +70,7 @@ func TestAPipelinesWantsAreWrittenTierByTier(t *testing.T) {
|
||||
// The pipeline of revision 5 (its key's number) of source 4: a re-pin's revision (H-A-213, H-A-216).
|
||||
e := ended(trunkBranch, 5, mpr.PipelineEndedOutcomePassed, mover("db", db, "0.2.0", 1, mpr.StepResultOutcomePassed),
|
||||
mover("web", web, "0.1.2", 2, mpr.StepResultOutcomePassed))
|
||||
e.Source = 4
|
||||
e.Source, e.Pipeline = 4, pipeline
|
||||
if _, err := p.ends.take(e); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -76,7 +81,7 @@ func TestAPipelinesWantsAreWrittenTierByTier(t *testing.T) {
|
||||
}
|
||||
p.wants(context.Background())
|
||||
if !wanted("g14", "db") || !wanted("ace", "db") || wanted("g14", "web") || wanted("ace", "web") {
|
||||
t.Fatalf("runs %v: tier 1 alone is not what is wanted first: %v", runs, wantsKV.values)
|
||||
t.Fatalf("%s runs %v: tier 1 alone is not what is wanted first: %v", pipeline, runs, wantsKV.values)
|
||||
}
|
||||
if runs {
|
||||
raw, _ := json.Marshal(nodeengine.Running{Node: "g14", Module: "db", Fingerprint: db})
|
||||
@@ -86,15 +91,15 @@ func TestAPipelinesWantsAreWrittenTierByTier(t *testing.T) {
|
||||
now = start.Add(time.Minute)
|
||||
p.wants(context.Background())
|
||||
if !wanted("g14", "web") || !wanted("ace", "web") {
|
||||
t.Fatalf("the consumer's want is not written once its provider runs: %v", wantsKV.values)
|
||||
t.Fatalf("%s: the consumer's want is not written once its provider runs: %v", pipeline, wantsKV.values)
|
||||
}
|
||||
raw, _ = json.Marshal(nodeengine.Running{Node: "g14", Module: "web", Fingerprint: web})
|
||||
_ = p.w.hearRunning("running.g14.web", raw)
|
||||
p.wants(context.Background())
|
||||
if len(said) != 1 || said[0].Outcome != meshwants.PipelineWantedOutcomeRunning || said[0].Revision != 5 || said[0].Source != 4 || len(findings) != 0 {
|
||||
t.Errorf("every tier runs: said %+v, findings %v", said, findings)
|
||||
t.Errorf("%s: every tier runs: said %+v, findings %v", pipeline, said, findings)
|
||||
}
|
||||
if _, _, ok := p.ends.tiersOf(trunkBranch + ".5"); ok {
|
||||
if _, _, ok := p.ends.tiersOf(pipeline); ok {
|
||||
t.Error("the tier progress stands after every tier ran")
|
||||
}
|
||||
continue
|
||||
@@ -107,7 +112,7 @@ func TestAPipelinesWantsAreWrittenTierByTier(t *testing.T) {
|
||||
now = start.Add(30 * time.Minute)
|
||||
p.wants(context.Background())
|
||||
p.wants(context.Background())
|
||||
f, ok := findings["delivery."+assign.KeyID(trunkBranch+".5")+".tier-late"]
|
||||
f, ok := findings["delivery."+assign.KeyID(pipeline)+".tier-late"]
|
||||
if !ok || !strings.Contains(f.Summary, "db on g14") || strings.Contains(f.Summary, "ace") {
|
||||
t.Errorf("tier-late is not raised naming db on g14 alone: %v", findings)
|
||||
}
|
||||
@@ -116,7 +121,7 @@ func TestAPipelinesWantsAreWrittenTierByTier(t *testing.T) {
|
||||
t.Errorf("pipeline-wanted tier-late is not said once, naming revision 5 of source 4: %+v", said)
|
||||
}
|
||||
// A tier progress kept before it held its source says the key's number as both (H-A-216).
|
||||
p.sayPipelineWanted(trunkBranch+".5", trunkBranch, "5", 0, meshwants.PipelineWantedOutcomeRunning, nil)
|
||||
p.sayPipelineWanted(pipeline, trunkBranch, "5", 0, meshwants.PipelineWantedOutcomeRunning, nil)
|
||||
if len(said) != 2 || said[1].Revision != 5 || said[1].Source != 5 {
|
||||
t.Errorf("a progress with no source: said %+v, want revision and source 5", said)
|
||||
}
|
||||
|
||||
@@ -73,7 +73,7 @@ func publishQueueVia(ask func(key string, body any) (json.RawMessage, error)) fu
|
||||
}
|
||||
}
|
||||
|
||||
// publishActTools are builder_publish_drop and builder_publish_first (design 47 H2 §793): hand acts, recorded in the hand-act log
|
||||
// publishActTools are builder_publish_drop (design 47 H2 §793; builder_publish_first is retired, H-A-483 third addendum): hand acts, recorded in the hand-act log
|
||||
// first, then written to reasks for the builder, which owns the work queue, to perform and answer.
|
||||
func publishActTools(ask queueAsker, now func() time.Time, acts meshhandactsclient.Recorder) []stdio.Tool {
|
||||
str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) }
|
||||
@@ -108,7 +108,6 @@ func publishActTools(ask queueAsker, now func() time.Time, acts meshhandactsclie
|
||||
}
|
||||
return []stdio.Tool{
|
||||
tool(versions.PublishDropTool, build.ModePublishDrop, versions.PublishDropDescription, "dropped the publish of"),
|
||||
tool(versions.PublishFirstTool, build.ModePublishFirst, versions.PublishFirstDescription, "put first the publish of"),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -78,8 +78,8 @@ func TestAStateIsReadInOnePage(t *testing.T) {
|
||||
}
|
||||
})
|
||||
// Design 47 H2 §793, H-A-454: builder_publish_queue asks the seat mesh-builds' verb publish-queue, which the builder serves from
|
||||
// its own stream, and answers its counts, list and cap with the states beside them; builder_publish_drop and
|
||||
// builder_publish_first record the hand act first, then write their reasks value.
|
||||
// its own stream, and answers its counts, list and cap with the states beside them; builder_publish_drop records the hand act
|
||||
// first, then writes its reasks value; builder_publish_first is retired (H-A-483, third addendum).
|
||||
t.Run("the queue is the builder's verb publish-queue", func(t *testing.T) {
|
||||
var asked []string
|
||||
ask := func(key string, body any) (json.RawMessage, error) {
|
||||
@@ -123,7 +123,7 @@ func TestAStateIsReadInOnePage(t *testing.T) {
|
||||
t.Errorf("asked %v, answered %+v (%v): want each holder asked in turn at its node, answered together, the silent one with why", asked, hs, err)
|
||||
}
|
||||
})
|
||||
t.Run("drop and first record the hand act, then ask the builder", func(t *testing.T) {
|
||||
t.Run("drop records the hand act, then asks the builder; first is retired", func(t *testing.T) {
|
||||
var order []string
|
||||
ask := func(key string, value []byte) (json.RawMessage, error) {
|
||||
order = append(order, "ask "+key)
|
||||
@@ -141,8 +141,8 @@ func TestAStateIsReadInOnePage(t *testing.T) {
|
||||
t.Fatalf("%s: %v", tool.Name, err)
|
||||
}
|
||||
}
|
||||
if len(tools) != 2 || fmt.Sprint(order) != "[record ask publish-drop.5000000000 record ask publish-first.5000000000]" {
|
||||
t.Errorf("tools %d, order %v: want each hand act recorded before its reasks value", len(tools), order)
|
||||
if len(tools) != 1 || tools[0].Name != versions.PublishDropTool || fmt.Sprint(order) != "[record ask publish-drop.5000000000]" {
|
||||
t.Errorf("tools %d, order %v: want builder_publish_drop alone, its hand act recorded before its reasks value", len(tools), order)
|
||||
}
|
||||
if _, err := tools[0].Run(map[string]any{"module": "m", "why": "no version"}); err == nil {
|
||||
t.Error("a drop naming no version is not refused")
|
||||
|
||||
@@ -273,7 +273,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
Entry: entryVia(calls.Call), Send: sendVia(calls.CallKeptAsk),
|
||||
Unreleased: reserved.unreleasedWords, Logf: logger.Printf,
|
||||
Holders: holders, // a registry seat's holder from the seat-holders view, from the start (design 47 G A-416, A-416a)
|
||||
Started: time.Now()} // a build ended from now on is of the live class, sent first (H-A-290 (1))
|
||||
Started: time.Now()} // a build ended from now on is of the live class (H-A-290 (1))
|
||||
// A number a registry seat already holds is never reserved (H-A-275): read before reserving, through the seats' verb versions.
|
||||
reserved.r.HeldNumbers = func(ctx context.Context, module, fp string, refs []string) ([]string, error) {
|
||||
return republish.HeldNumbers(ctx, module, fp, refs, registryVersionsVia(calls.Call))
|
||||
@@ -298,7 +298,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
pq.ackWait = publishSet.PauseMax // a sent item unheard is delivered again after the longest pause (H-A-440 (1))
|
||||
pq.ackWait = publishSet.PauseMax // a sent item unheard is delivered again after the longest pause (H-A-440 (1))
|
||||
reserved.pause = publishSet.Pause // a version-released whose tag the forge did not write waits the pause (H-A-484 addendum)
|
||||
releasedKept, err := numberFile(getenv("MESH_BUILDER_RELEASED_KEPT_FILE"), versions.DefaultReleasedKept)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -322,15 +323,17 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// A version that never publishes (withdrawn or taken over) has its publish item ended by the queue too (H-A-290 (2)).
|
||||
republish.Ended = items.Ended
|
||||
pq.take = items.take
|
||||
reasksKV.Ready(func(jetstream.KeyValue) { items.AskAllDone(ctx) })
|
||||
// At the start, a built reservation the catalog holds released is taken in as its version-released would (H-A-484 addendum),
|
||||
// before the publish items of the rest are asked.
|
||||
reasksKV.Ready(func(jetstream.KeyValue) {
|
||||
reserved.releasedAtStart(ctx)
|
||||
items.AskAllDone(ctx)
|
||||
})
|
||||
wirePublishItems(reserved, items, awaitedReserved{awaitedVersions{b: actedKV, bucket: own.acted}}, releasedKept)
|
||||
acts.Republish = republish.Republish
|
||||
acts.PublishDrop = func(ctx context.Context, module, version, by, why string) (string, error) {
|
||||
return pq.drop(ctx, items, module, version, by, why)
|
||||
}
|
||||
acts.PublishFirst = func(ctx context.Context, module, version, _, _ string) (string, error) {
|
||||
return pq.first(ctx, items, module, version)
|
||||
}
|
||||
// builder_reserve_done: a done trunk build that reserved none is given a version, and its publish queued as publish again does.
|
||||
acts.ReserveDone = reserveDoneAct(b.ReserveDone, republish.Republish, logger.Printf)
|
||||
// The operator is asked once a key's event builds are spent (design 47 G §568): the builder's asker on the seat operator-channel.
|
||||
|
||||
+153
-74
@@ -28,15 +28,15 @@ import (
|
||||
"mesh-delivery-builder/internal/versions"
|
||||
)
|
||||
|
||||
// Design 47 H2 §793 (whole, against the bus): the builder's work queue publishes, its pull consumers made as it learns a holder.
|
||||
// A live publish enqueued behind 100 again items while the builder was away is sent first; a dropped item is never sent, and
|
||||
// put again within the duplicates window is not stored; the move of the old state run twice puts each item once, and the
|
||||
// builder removes the old state's bucket itself (H-A-461 (1)). The seat mesh-builds' verb publish-queue reads the stream: counts
|
||||
// per holder and class, the oldest and newest, the live item listed first (H-A-454). An idle dispatch sends a live item put
|
||||
// before an again one first, 50 rounds (H-A-473, third addendum). Then §796's tag at another commit, its
|
||||
// reservation's version-reserved waited on by name (§821), and
|
||||
// §794's backlog held six minutes, as steps (H-A-453 (3)).
|
||||
func TestThePublishQueueSendsLiveFirstAndNeverADroppedItem(t *testing.T) {
|
||||
// Design 47 H2 §793 (whole, against the bus): the builder's work queue publishes, one pull consumer made as it learns a holder
|
||||
// (H-A-483 and its addenda), the class durables of an earlier release retired. No item is lost, a failed send comes back after
|
||||
// its pause behind an item put meanwhile, and a dropped item never comes back; 100 again items and a live one put while the
|
||||
// builder was away are all sent, the dropped one never, and put again within the duplicates window it is not stored; the move
|
||||
// of the old state run twice puts each item once, and the builder removes the old state's bucket itself (H-A-461 (1)). The seat
|
||||
// mesh-builds' verb publish-queue reads the stream: counts per holder and class, the oldest and newest, the items listed in
|
||||
// stream order (H-A-454; H-A-483, third addendum). Then §796's tag at another commit, its reservation's version-reserved waited on by name (§821), and §794's
|
||||
// backlog held six minutes, as steps (H-A-453 (3)).
|
||||
func TestThePublishQueueLosesNoItemAndNeverSendsADroppedItem(t *testing.T) {
|
||||
b := startBed(t)
|
||||
ctx := b.ctx
|
||||
// The stream as the manifest's old streams section declares it: the module's own, retention work-queue, duplicates 24 hours.
|
||||
@@ -45,66 +45,153 @@ func TestThePublishQueueSendsLiveFirstAndNeverADroppedItem(t *testing.T) {
|
||||
Retention: jetstream.WorkQueuePolicy, Discard: jetstream.DiscardNew, Duplicates: 24 * time.Hour, MaxAge: 7 * 24 * time.Hour}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// An idle dispatch (H-A-473, its third addendum): its two pulls wait; a live item put, then an again one, is sent live first;
|
||||
// and an again item put after a live one, while the dispatch runs an again item put before both, is sent after the live one.
|
||||
// Every time, 50 rounds, each a holder of its own.
|
||||
sentOf := make(chan string, 8)
|
||||
idle := &publishQueue{js: b.js, ackWait: time.Minute, logf: func(string, ...any) {}, binding: publishesOf("mesh-delivery-builder"),
|
||||
take: func(_ context.Context, m itemMsg) {
|
||||
var q versions.QueuedPublish
|
||||
_ = json.Unmarshal(m.Data(), &q)
|
||||
_ = m.Ack()
|
||||
sentOf <- q.Module
|
||||
}}
|
||||
t.Run("idle dispatch", func(t *testing.T) {
|
||||
roundBound := testbus.SetupBound(t) // the level's set-up share (MESH_TEST_LEVEL_BOUND), as the other whole tests
|
||||
for r := range 50 {
|
||||
holder := fmt.Sprintf("idle%d", r)
|
||||
idle.learn(ctx, holder)
|
||||
for _, class := range []string{versions.PublishClassLive, versions.PublishClassStart} { // both pulls stand waiting
|
||||
c, err := b.js.Consumer(ctx, bind.Stream, publishesConsumer(holder, class))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
// One consumer per holder (H-A-483 and its addenda): the holder's class durables of an earlier release are retired, the item
|
||||
// one of them held unacknowledged reaches the one consumer; no item is lost; a failed send comes back after its pause, and a
|
||||
// live item put while it waits is sent before it returns; a dropped item never comes back.
|
||||
t.Run("one consumer", func(t *testing.T) {
|
||||
const holder, pause = "paused", 3 * time.Second // the pause is far longer than a send of an item put now
|
||||
bound := testbus.SetupBound(t)
|
||||
kv := &itemKV{vals: map[string][]byte{}, revs: map[string]uint64{}}
|
||||
type send struct {
|
||||
module string
|
||||
at time.Time
|
||||
}
|
||||
var mu sync.Mutex
|
||||
published, failed := map[string]bool{}, map[string]bool{}
|
||||
sentCh := make(chan send, 16)
|
||||
items := &publishItems{reasks: func() jetstream.KeyValue { return kv }, set: publishSettings{Pause: pause, PauseMax: time.Minute, Tries: 5},
|
||||
now: time.Now, logf: t.Logf,
|
||||
reserved: func(_ context.Context, m, v string) (versions.ReservedVersion, bool, error) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
return versions.ReservedVersion{Module: m, Version: v, Artifacts: map[string]string{m + ":a": "sha256:a"}, Published: map[string]bool{m + ":a": published[m]}, BuiltAt: time.Now()}, true, nil
|
||||
},
|
||||
send: func(_ context.Context, q versions.QueuedPublish) versions.SendFate {
|
||||
sentCh <- send{q.Module, time.Now()}
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if (q.Module == "fail" || q.Module == "dropme") && !failed[q.Module] { // the first send of each fails
|
||||
failed[q.Module] = true
|
||||
return versions.SendFate{Failed: "the registry is down"}
|
||||
}
|
||||
for {
|
||||
info, err := c.Info(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if info.NumWaiting > 0 {
|
||||
break
|
||||
}
|
||||
published[q.Module] = true
|
||||
return versions.SendFate{Sent: true}
|
||||
}}
|
||||
pq := &publishQueue{js: b.js, ackWait: time.Minute, logf: t.Logf, take: items.take, binding: publishesOf("mesh-delivery-builder")}
|
||||
put := func(module, class string) {
|
||||
q := versions.QueuedPublish{V: 1, Holder: holder, Module: module, Version: "0.1.0", Artifacts: map[string]string{module + ":a": "sha256:a"}, Class: class, Queued: time.Now()}
|
||||
data, _ := json.Marshal(q)
|
||||
if _, err := b.js.Publish(ctx, pq.subject(bind, q), data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
hear := func() { // each publish sent is heard: its item acknowledged once it is held
|
||||
mu.Lock()
|
||||
var done []string
|
||||
for m, ok := range published {
|
||||
if ok {
|
||||
done = append(done, m)
|
||||
}
|
||||
}
|
||||
put := func(module, class string) {
|
||||
q := versions.QueuedPublish{V: 1, Holder: holder, Module: module, Version: "0.1.0", Class: class, Queued: time.Now()}
|
||||
data, _ := json.Marshal(q)
|
||||
if _, err := b.js.Publish(ctx, idle.subject(bind, q), data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mu.Unlock()
|
||||
for _, m := range done {
|
||||
items.heard(ctx, m, "0.1.0")
|
||||
}
|
||||
sent := func(n int) []string {
|
||||
var order []string
|
||||
for range n {
|
||||
select {
|
||||
case m := <-sentOf:
|
||||
order = append(order, m)
|
||||
case <-time.After(roundBound):
|
||||
t.Fatalf("round %d: sent %v of %d items", r, order, n)
|
||||
}
|
||||
}
|
||||
return order
|
||||
}
|
||||
next := func() send {
|
||||
t.Helper()
|
||||
select {
|
||||
case s := <-sentCh:
|
||||
hear()
|
||||
return s
|
||||
case <-time.After(bound):
|
||||
t.Fatal("no item was sent within the level's set-up share")
|
||||
}
|
||||
put("live", versions.PublishClassLive)
|
||||
put("again", versions.PublishClassStart)
|
||||
if order := sent(2); order[0] != "live" {
|
||||
t.Errorf("round %d: an idle dispatch sent %v, want the live item put first sent first", r, order)
|
||||
return send{}
|
||||
}
|
||||
// The two class durables an earlier release made, the live one holding an item given and not acknowledged.
|
||||
for _, class := range []string{versions.PublishClassLive, versions.PublishClassStart} {
|
||||
if _, err := b.js.CreateOrUpdateConsumer(ctx, bind.Stream, jetstream.ConsumerConfig{Durable: publishesConsumer(holder) + "-" + class,
|
||||
FilterSubject: bind.Subject + ".publish." + holder + "." + class + ".>", AckPolicy: jetstream.AckExplicitPolicy, AckWait: time.Hour}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
put("before", versions.PublishClassStart)
|
||||
put("live2", versions.PublishClassLive)
|
||||
put("after", versions.PublishClassStart)
|
||||
if order := sent(3); slices.Index(order, "after") < slices.Index(order, "live2") {
|
||||
t.Errorf("round %d: the dispatch sent %v, want the again item put after the live one sent after it", r, order)
|
||||
}
|
||||
put("given", versions.PublishClassLive)
|
||||
old, err := b.js.Consumer(ctx, bind.Stream, publishesConsumer(holder)+"-"+versions.PublishClassLive)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m, err := old.Next(); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if md, _ := m.Metadata(); md.NumDelivered != 1 {
|
||||
t.Fatalf("the old live durable gave %+v", md)
|
||||
}
|
||||
pq.learn(ctx, holder)
|
||||
stream, err := b.js.Stream(ctx, bind.Stream)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var names []string
|
||||
for n := range stream.ConsumerNames(ctx).Name() {
|
||||
if strings.HasPrefix(n, publishesConsumer(holder)) {
|
||||
names = append(names, n)
|
||||
}
|
||||
}
|
||||
if !slices.Equal(names, []string{publishesConsumer(holder)}) {
|
||||
t.Fatalf("the consumers of %s are %v, want only %s: the class durables retired", holder, names, publishesConsumer(holder))
|
||||
}
|
||||
if s := next(); s.module != "given" {
|
||||
t.Fatalf("sent %s first, want the item the old durable held unacknowledged", s.module)
|
||||
}
|
||||
put("fail", versions.PublishClassStart)
|
||||
put("dropme", versions.PublishClassStart)
|
||||
var firstFail time.Time
|
||||
for range 2 {
|
||||
if s := next(); s.module == "fail" {
|
||||
firstFail = s.at
|
||||
}
|
||||
}
|
||||
put("live", versions.PublishClassLive) // put while both failed items wait out their pause
|
||||
if s := next(); s.module != "live" {
|
||||
t.Fatalf("sent %s, want the live item put while the failed ones wait their pause", s.module)
|
||||
}
|
||||
if answer, err := pq.drop(ctx, items, "dropme", "0.1.0", "op@node", "a test drops it"); err != nil {
|
||||
t.Fatalf("drop: %v", err)
|
||||
} else {
|
||||
t.Log(answer)
|
||||
}
|
||||
if s := next(); s.module != "fail" || s.at.Sub(firstFail) < pause {
|
||||
t.Fatalf("sent %s %s after the failed send, want fail again after its pause %s", s.module, s.at.Sub(firstFail), pause)
|
||||
}
|
||||
// Every item is acknowledged and the dropped one is gone: nothing is left to come back.
|
||||
c, err := b.js.Consumer(ctx, bind.Stream, publishesConsumer(holder))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
st, err := b.js.Stream(ctx, bind.Stream)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for deadline := time.Now().Add(bound); ; {
|
||||
hear()
|
||||
info, err := c.Info(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sinfo, err := st.Info(ctx, jetstream.WithSubjectFilter(bind.Subject+".publish."+holder+".>"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if info.NumAckPending == 0 && info.NumPending == 0 && len(sinfo.State.Subjects) == 0 {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("the queue of %s holds %v, its consumer %d pending and %d unacknowledged: want every item acknowledged and the dropped one gone", holder, sinfo.State.Subjects, info.NumPending, info.NumAckPending)
|
||||
}
|
||||
select {
|
||||
case s := <-sentCh:
|
||||
t.Fatalf("sent %s again, want nothing more", s.module)
|
||||
case <-time.After(10 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
})
|
||||
@@ -155,21 +242,13 @@ func TestThePublishQueueSendsLiveFirstAndNeverADroppedItem(t *testing.T) {
|
||||
raw, err := publishQueueVerb(ctx, pq)("op@node", json.RawMessage(`{}`))
|
||||
if a, ok := raw.(meshbuilds.PublishQueueAnswer); err != nil || !ok || a.Total != 101 || len(a.Counts) != 2 || a.Counts[0].Class != versions.PublishClassStart ||
|
||||
a.Counts[0].Count != 100 || a.Counts[1].Count != 1 || a.Counts[0].Oldest.After(a.Counts[0].Newest) || len(a.Queue) != versions.PublishQueueListCap ||
|
||||
a.Queue[0].Module != "live" || a.Capped {
|
||||
t.Errorf("publish-queue answered %+v (%v), want 101 items, oci's 100 again and 1 live, fifty listed, the live first", raw, err)
|
||||
a.Queue[0].Module != "a000" || a.Capped {
|
||||
t.Errorf("publish-queue answered %+v (%v), want 101 items, oci's 100 again and 1 live, fifty listed in stream order, a000 first", raw, err)
|
||||
}
|
||||
if err := pq.learnAll(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case first := <-sentCh:
|
||||
if first != "live" {
|
||||
t.Errorf("the first item sent is %s, want the live one, enqueued behind 100 again items", first)
|
||||
}
|
||||
case <-time.After(30 * time.Second):
|
||||
t.Fatal("nothing was sent")
|
||||
}
|
||||
for range 4 { // the again items in flight: at most four held unacknowledged, their publish unheard
|
||||
for range 4 { // the items in flight: at most four held unacknowledged, their publish unheard
|
||||
select {
|
||||
case <-sentCh:
|
||||
case <-time.After(30 * time.Second):
|
||||
|
||||
@@ -7,7 +7,6 @@ import (
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -531,9 +530,9 @@ func builtOf(all []versions.ReservedVersion) []versions.ReservedVersion {
|
||||
// the manifest's old streams section with the work-queue retention, the module's own, consumed by the module's one process on the
|
||||
// module's account (H-A-464), which declares no bus and so is the stream's one reader by the module's shape; each item on
|
||||
// `mesh.stream.<module>.event.publish.<holder>.<class>.<module>.<version>` under the message id
|
||||
// `<holder>.<class>.<module>.<version>`. The builder makes the pull consumers at run time as it learns a holder,
|
||||
// `publishes-<holder>-<class>`, with ack wait publish-pause-max, max ack pending four and max deliveries ten, and pulls the live
|
||||
// class first, at most four sends at once per holder.
|
||||
// `<holder>.<class>.<module>.<version>`. The builder makes one pull consumer at run time as it learns a holder,
|
||||
// `publishes-<holder>`, with ack wait publish-pause-max, max ack pending four and max deliveries ten, and sends its items in the
|
||||
// order the consumer gives them, at most four at once per holder (H-A-483 and its addenda).
|
||||
type publishQueue struct {
|
||||
js jetstream.JetStream
|
||||
binding func() (publishesStream, error)
|
||||
@@ -555,16 +554,16 @@ func publishesOf(module string) func() (publishesStream, error) {
|
||||
return func() (publishesStream, error) { return b, nil }
|
||||
}
|
||||
|
||||
// The consumers' numbers (H-A-446 (2)): at most four sends in flight per holder and class, and ten deliveries, the work queue's
|
||||
// largest; the builder ends an item at publish-tries anyway.
|
||||
// The consumer's numbers (H-A-446 (2)): at most four sends in flight per holder, and ten deliveries, the work queue's largest;
|
||||
// the builder ends an item at publish-tries anyway.
|
||||
const (
|
||||
publishesMaxAckPending = versions.PublishSendsAtOnce
|
||||
publishesMaxDeliver = 10
|
||||
)
|
||||
|
||||
// publishesConsumer is the durable consumer of holder's items of class.
|
||||
func publishesConsumer(holder, class string) string {
|
||||
return versions.PublishesLocal + "-" + holder + "-" + class
|
||||
// publishesConsumer is the one durable consumer of holder's items (H-A-483 addendum).
|
||||
func publishesConsumer(holder string) string {
|
||||
return versions.PublishesLocal + "-" + holder
|
||||
}
|
||||
|
||||
// subject is the item's subject on the bus.
|
||||
@@ -616,8 +615,15 @@ func (pq *publishQueue) learnAll(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// learn starts holder's pull once.
|
||||
// learn starts holder's pull once. A holder is a module name, so one ending in -live or -again, the class durables an earlier
|
||||
// release made, is refused, said (H-A-483, third addendum): its consumer would be another holder's old durable.
|
||||
func (pq *publishQueue) learn(ctx context.Context, holder string) {
|
||||
for _, class := range []string{versions.PublishClassLive, versions.PublishClassStart} {
|
||||
if strings.HasSuffix(holder, "-"+class) {
|
||||
pq.logf("the publishes of %s are refused: a holder is a module name, and one ending in -%s names the class durable of an earlier release (H-A-483)", holder, class)
|
||||
return
|
||||
}
|
||||
}
|
||||
pq.mu.Lock()
|
||||
if pq.pulling == nil {
|
||||
pq.pulling = map[string]bool{}
|
||||
@@ -636,223 +642,71 @@ func (pq *publishQueue) learn(ctx context.Context, holder string) {
|
||||
}
|
||||
}
|
||||
|
||||
// pull makes holder's two consumers and dispatches their items, the live class first, at most four sends at once: an again item
|
||||
// is run only when no live one waits on the queue or is in flight to the dispatch (dispatch, H-A-473).
|
||||
// pull makes holder's one consumer and dispatches its items (design 47 H2 §793, H-A-483 and its addenda): first it retires the
|
||||
// holder's class durables an earlier release made (`-live`, `-again`), whose filters a work queue refuses beside the one; their
|
||||
// unacknowledged items stay on the queue and reach the one consumer.
|
||||
func (pq *publishQueue) pull(ctx context.Context, holder string) error {
|
||||
b, err := pq.binding()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
cons := map[string]jetstream.Consumer{}
|
||||
for _, class := range []string{versions.PublishClassStart, versions.PublishClassLive} {
|
||||
c, err := pq.js.CreateOrUpdateConsumer(ctx, b.Stream, jetstream.ConsumerConfig{Durable: publishesConsumer(holder, class),
|
||||
FilterSubject: b.Subject + ".publish." + versions.EscapeToken(holder) + "." + class + ".>", AckPolicy: jetstream.AckExplicitPolicy,
|
||||
AckWait: pq.ackWait, MaxAckPending: publishesMaxAckPending, MaxDeliver: publishesMaxDeliver})
|
||||
if err != nil {
|
||||
return fmt.Errorf("the consumer %s: %w", publishesConsumer(holder, class), err)
|
||||
for _, class := range []string{versions.PublishClassLive, versions.PublishClassStart} {
|
||||
old := publishesConsumer(holder) + "-" + class
|
||||
err := pq.js.DeleteConsumer(ctx, b.Stream, old)
|
||||
switch {
|
||||
case err == nil:
|
||||
pq.logf("the consumer %s of an earlier release is removed: the publishes of %s have one consumer (H-A-483)", old, holder)
|
||||
case !errors.Is(err, jetstream.ErrConsumerNotFound):
|
||||
return fmt.Errorf("the consumer %s of an earlier release is not removed: %w", old, err)
|
||||
}
|
||||
cons[class] = c
|
||||
}
|
||||
pq.logf("pulling the publishes of %s, the live first", holder)
|
||||
go pq.dispatch(ctx, holder, cons[versions.PublishClassLive], cons[versions.PublishClassStart])
|
||||
c, err := pq.js.CreateOrUpdateConsumer(ctx, b.Stream, jetstream.ConsumerConfig{Durable: publishesConsumer(holder),
|
||||
FilterSubject: b.Subject + ".publish." + versions.EscapeToken(holder) + ".>", AckPolicy: jetstream.AckExplicitPolicy,
|
||||
AckWait: pq.ackWait, MaxAckPending: publishesMaxAckPending, MaxDeliver: publishesMaxDeliver})
|
||||
if err != nil {
|
||||
return fmt.Errorf("the consumer %s: %w", publishesConsumer(holder), err)
|
||||
}
|
||||
pq.logf("pulling the publishes of %s", holder)
|
||||
go pq.dispatch(ctx, holder, c)
|
||||
return nil
|
||||
}
|
||||
|
||||
// dispatch runs the items of one holder, at most four at once, every live item before an again one (design 47 H2 §793,
|
||||
// H-A-473 and its third addendum): on every turn no live item is held it fetches live without waiting, up to its free slots,
|
||||
// and takes again items only when that fetch returned nothing. With again items held it fetches live once more; only when
|
||||
// that returns nothing are they run, and while the live pull waits only once the live consumer's delivered sequence is no
|
||||
// further than the live items received: a gap is a live item in flight to the waiting pull, and the again items wait until it
|
||||
// arrives and runs. When neither holds an item, a pull of one item waits on each, the bus's only wake for an item put or
|
||||
// delivered again. A pull the bus refuses ends the dispatch, said; the holder is pulled again when an item of it is put.
|
||||
func (pq *publishQueue) dispatch(ctx context.Context, holder string, live, again jetstream.Consumer) {
|
||||
// dispatch runs the items of one holder in the order its one consumer gives them, at most four at once (design 47 H2 §793,
|
||||
// H-A-483): each free slot takes the next item through a pull that waits, so an item put or handed back after its pause is
|
||||
// received the moment the bus gives it; an item handed back reappears only after its pause, so an item put meanwhile goes
|
||||
// before it. A pull the bus refuses ends the dispatch, said; the holder is pulled again when an item of it is put.
|
||||
func (pq *publishQueue) dispatch(ctx context.Context, holder string, c jetstream.Consumer) {
|
||||
sem := make(chan struct{}, versions.PublishSendsAtOnce)
|
||||
run := func(m jetstream.Msg) {
|
||||
go func() {
|
||||
defer func() { <-sem }()
|
||||
pq.take(ctx, m)
|
||||
}()
|
||||
}
|
||||
failed := func(err error) {
|
||||
pq.logf("the publishes of %s are not pulled: %v; they are pulled when an item of it is put", holder, err)
|
||||
pq.mu.Lock()
|
||||
delete(pq.pulling, holder)
|
||||
pq.mu.Unlock()
|
||||
}
|
||||
refused := func(err error) bool {
|
||||
return err != nil && !errors.Is(err, jetstream.ErrNoMessages) && !errors.Is(err, nats.ErrTimeout) && ctx.Err() == nil
|
||||
}
|
||||
// received is the live consumer's highest delivery sequence the dispatch holds; it starts at what the consumer delivered
|
||||
// before this dispatch, to pulls no longer there.
|
||||
var received uint64
|
||||
if info, err := live.Info(ctx); err != nil {
|
||||
if refused(err) {
|
||||
failed(err)
|
||||
return
|
||||
}
|
||||
} else {
|
||||
received = info.Delivered.Consumer
|
||||
}
|
||||
gotLive := func(ms ...jetstream.Msg) {
|
||||
for _, m := range ms {
|
||||
if md, err := m.Metadata(); err == nil && md.Sequence.Consumer > received {
|
||||
received = md.Sequence.Consumer
|
||||
}
|
||||
}
|
||||
}
|
||||
inFlight := func() bool { // a live item delivered to the waiting pull and not yet received
|
||||
info, err := live.Info(ctx)
|
||||
if err != nil {
|
||||
pq.logf("the publishes of %s: the live consumer cannot be read (%v); the again items wait for the live pull's answer", holder, err)
|
||||
return true
|
||||
}
|
||||
return info.Delivered.Consumer > received
|
||||
}
|
||||
now := func(c jetstream.Consumer, n int) ([]jetstream.Msg, error) {
|
||||
b, err := c.FetchNoWait(n)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var got []jetstream.Msg
|
||||
for m := range b.Messages() {
|
||||
got = append(got, m)
|
||||
}
|
||||
return got, b.Error()
|
||||
}
|
||||
type waited struct {
|
||||
m jetstream.Msg
|
||||
err error
|
||||
}
|
||||
liveDone, againDone := make(chan waited, 1), make(chan waited, 1)
|
||||
wait := func(c jetstream.Consumer, done chan<- waited) {
|
||||
go func() {
|
||||
var w waited
|
||||
b, err := c.Fetch(1)
|
||||
if err == nil {
|
||||
for m := range b.Messages() {
|
||||
w.m = m
|
||||
}
|
||||
err = b.Error()
|
||||
}
|
||||
w.err = err
|
||||
done <- w
|
||||
}()
|
||||
}
|
||||
var heldLive jetstream.Msg // a live item the waiting pull received
|
||||
var heldAgain []jetstream.Msg // again items received, not yet run
|
||||
var liveWaiting, againWaiting bool
|
||||
heard := func(w waited, waiting *bool, keep func(jetstream.Msg)) bool {
|
||||
*waiting = false
|
||||
if refused(w.err) {
|
||||
failed(w.err)
|
||||
return false
|
||||
}
|
||||
if w.m != nil {
|
||||
keep(w.m)
|
||||
}
|
||||
return true
|
||||
}
|
||||
keepLive := func(m jetstream.Msg) { gotLive(m); heldLive = m }
|
||||
keepAgain := func(m jetstream.Msg) { heldAgain = append(heldAgain, m) }
|
||||
for {
|
||||
select {
|
||||
case sem <- struct{}{}:
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
n := 1
|
||||
slots:
|
||||
for n < cap(sem) {
|
||||
select {
|
||||
case sem <- struct{}{}:
|
||||
n++
|
||||
default:
|
||||
break slots
|
||||
var got jetstream.Msg
|
||||
b, err := c.Fetch(1)
|
||||
if err == nil {
|
||||
for m := range b.Messages() {
|
||||
got = m
|
||||
}
|
||||
err = b.Error()
|
||||
}
|
||||
select {
|
||||
case w := <-liveDone:
|
||||
if !heard(w, &liveWaiting, keepLive) {
|
||||
return
|
||||
}
|
||||
default:
|
||||
}
|
||||
select {
|
||||
case w := <-againDone:
|
||||
if !heard(w, &againWaiting, keepAgain) {
|
||||
return
|
||||
}
|
||||
default:
|
||||
}
|
||||
var got []jetstream.Msg
|
||||
held := false // again items held for a live item in flight
|
||||
if heldLive != nil {
|
||||
got, heldLive = []jetstream.Msg{heldLive}, nil
|
||||
} else {
|
||||
ms, err := now(live, n)
|
||||
if refused(err) {
|
||||
failed(err)
|
||||
return
|
||||
}
|
||||
gotLive(ms...)
|
||||
got = ms
|
||||
}
|
||||
if len(got) == 0 {
|
||||
if len(heldAgain) == 0 && !againWaiting {
|
||||
ms, err := now(again, n)
|
||||
if refused(err) {
|
||||
failed(err)
|
||||
return
|
||||
}
|
||||
heldAgain = ms
|
||||
}
|
||||
if len(heldAgain) > 0 { // with the again items held, live once more: one put since the first fetch goes first
|
||||
ms, err := now(live, n)
|
||||
if refused(err) {
|
||||
failed(err)
|
||||
return
|
||||
}
|
||||
gotLive(ms...)
|
||||
got = ms
|
||||
}
|
||||
switch {
|
||||
case len(got) > 0 || len(heldAgain) == 0:
|
||||
case liveWaiting && inFlight():
|
||||
held = true
|
||||
default:
|
||||
k := min(n, len(heldAgain))
|
||||
got, heldAgain = heldAgain[:k], heldAgain[k:]
|
||||
}
|
||||
}
|
||||
for _, m := range got {
|
||||
run(m)
|
||||
}
|
||||
for range n - len(got) {
|
||||
if err != nil && !errors.Is(err, jetstream.ErrNoMessages) && !errors.Is(err, nats.ErrTimeout) && ctx.Err() == nil {
|
||||
<-sem
|
||||
}
|
||||
if len(got) > 0 {
|
||||
continue
|
||||
}
|
||||
if !liveWaiting {
|
||||
liveWaiting = true
|
||||
wait(live, liveDone)
|
||||
}
|
||||
if !againWaiting && !held {
|
||||
againWaiting = true
|
||||
wait(again, againDone)
|
||||
}
|
||||
select {
|
||||
case w := <-liveDone:
|
||||
if !heard(w, &liveWaiting, keepLive) {
|
||||
return
|
||||
}
|
||||
case w := <-againDone:
|
||||
if !heard(w, &againWaiting, keepAgain) {
|
||||
return
|
||||
}
|
||||
case <-ctx.Done():
|
||||
pq.logf("the publishes of %s are not pulled: %v; they are pulled when an item of it is put", holder, err)
|
||||
pq.mu.Lock()
|
||||
delete(pq.pulling, holder)
|
||||
pq.mu.Unlock()
|
||||
return
|
||||
}
|
||||
if got == nil {
|
||||
<-sem
|
||||
continue
|
||||
}
|
||||
go func(m jetstream.Msg) {
|
||||
defer func() { <-sem }()
|
||||
pq.take(ctx, m)
|
||||
}(got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -956,45 +810,6 @@ func (pq *publishQueue) drop(ctx context.Context, items *publishItems, module, v
|
||||
return fmt.Sprintf("the publish of %s %s is dropped: %d items terminated, and none is sent or put again", module, version, len(qs)), nil
|
||||
}
|
||||
|
||||
// first is builder_publish_first (§793): each again item of the version is put again as live, under the message id
|
||||
// `<holder>.live.<module>.<version>`, and the old terminated; a live id the queue holds within its window takes nothing, and the
|
||||
// old stays, said.
|
||||
func (pq *publishQueue) first(ctx context.Context, items *publishItems, module, version string) (string, error) {
|
||||
qs, st, err := pq.itemsOf(ctx, module, version, versions.PublishClassStart)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("the work queue %s cannot be read: %w", versions.PublishesLocal, err)
|
||||
}
|
||||
if len(qs) == 0 {
|
||||
return "", fmt.Errorf("the work queue %s holds no again item of %s %s", versions.PublishesLocal, module, version)
|
||||
}
|
||||
var moved, kept []string
|
||||
for subj, q := range qs {
|
||||
q.Class = versions.PublishClassLive
|
||||
dup, err := pq.put(ctx, q)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("%s %s is not put again as live at %s: %w", module, version, q.Holder, err)
|
||||
}
|
||||
if dup {
|
||||
kept = append(kept, q.Holder)
|
||||
continue
|
||||
}
|
||||
if err := st.Purge(ctx, jetstream.WithPurgeSubject(subj)); err != nil {
|
||||
return "", fmt.Errorf("the again item of %s %s at %s is not terminated: %w", module, version, q.Holder, err)
|
||||
}
|
||||
moved = append(moved, q.Holder)
|
||||
}
|
||||
for _, h := range items.settle(module, version, func(h heldItem) bool { return h.q.Class == versions.PublishClassStart }) {
|
||||
_ = h.m.Term()
|
||||
}
|
||||
sort.Strings(moved)
|
||||
answer := fmt.Sprintf("%s %s is put first as live at %s, its again item terminated", module, version, strings.Join(moved, ", "))
|
||||
if len(kept) > 0 {
|
||||
sort.Strings(kept)
|
||||
answer += fmt.Sprintf("; at %s the work queue holds its live id within the duplicates window, so its again item stays", strings.Join(kept, ", "))
|
||||
}
|
||||
return answer, nil
|
||||
}
|
||||
|
||||
// endItem writes the version's publish item ended with answer, made when none stands: Try and take then leave it, and a start
|
||||
// asks it no more.
|
||||
func (p *publishItems) endItem(ctx context.Context, module, version, answer string) error {
|
||||
|
||||
@@ -440,3 +440,17 @@ func TestAPublishIsNotAskedAgainWhileItsPublishedMayBeUnheard(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 H2 H-A-483 (third addendum): a holder is a module name, so one ending in -live or -again, the class durables of an
|
||||
// earlier release, is refused at the learn, said, and nothing of it is pulled.
|
||||
func TestAHolderNamedLikeAClassDurableIsRefused(t *testing.T) {
|
||||
for _, holder := range []string{"oci-live", "oci-again"} {
|
||||
var said []string
|
||||
pq := &publishQueue{logf: func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) },
|
||||
binding: func() (publishesStream, error) { return publishesStream{}, errors.New("no bus in a unit test") }}
|
||||
pq.learn(context.Background(), holder)
|
||||
if len(said) != 1 || !strings.Contains(said[0], "the publishes of "+holder+" are refused") || pq.pulling[holder] {
|
||||
t.Errorf("%s: said %q, pulling %v, want it refused at the learn, said once, nothing pulled", holder, said, pq.pulling[holder])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -65,7 +65,7 @@ func reAsked(ctx context.Context, b askAgainer, acts versions.Acts, items *publi
|
||||
released(ctx, b, kv, e, r, logf)
|
||||
return
|
||||
}
|
||||
if r.Mode == build.ModeRepublish || r.Mode == build.ModePublishDrop || r.Mode == build.ModePublishFirst || r.Mode == build.ModeFloorAgain || r.Mode == build.ModeOldFormDrop || r.Mode == build.ModeReservedDrop || r.Mode == build.ModeReserveDone {
|
||||
if r.Mode == build.ModeRepublish || r.Mode == build.ModePublishDrop || r.Mode == build.ModeFloorAgain || r.Mode == build.ModeOldFormDrop || r.Mode == build.ModeReservedDrop || r.Mode == build.ModeReserveDone {
|
||||
// A version's hand act (design 51 *Tools of the builder*) may build again, so it is answered beside the watch.
|
||||
go versionActed(ctx, acts, kv, e, r, logf)
|
||||
return
|
||||
@@ -208,11 +208,8 @@ func versionActed(ctx context.Context, acts versions.Acts, kv jetstream.KeyValue
|
||||
r.Answer = "queued " + fmt.Sprint(len(r.Queued)) + " versions to publish again: " + strings.Join(r.Queued, ", ") + "; builder_publish_queue shows them"
|
||||
}
|
||||
}
|
||||
} else if r.Mode == build.ModePublishDrop || r.Mode == build.ModePublishFirst {
|
||||
} else if r.Mode == build.ModePublishDrop {
|
||||
act := acts.PublishDrop
|
||||
if r.Mode == build.ModePublishFirst {
|
||||
act = acts.PublishFirst
|
||||
}
|
||||
if act == nil {
|
||||
err = errors.New("this builder keeps no work queue of publishes")
|
||||
} else {
|
||||
|
||||
@@ -59,6 +59,9 @@ type reservations struct {
|
||||
withdraw func(module, fingerprint, why string)
|
||||
// call asks a seat's verb: the git seat's tag (design 47 G §370).
|
||||
call toolCaller
|
||||
// pause is the publish pause a version-released whose tag was not written is handed back with (H-A-484 addendum); zero hands
|
||||
// it back at once.
|
||||
pause time.Duration
|
||||
// rebuild builds a done key of module at fp again (the build-again path, A-490; H-A-279); nil builds none.
|
||||
rebuild func(ctx context.Context, module, fp, why string) error
|
||||
// say says package-published on the seat mesh-builds; nil says none.
|
||||
@@ -310,32 +313,20 @@ func (rv *reservations) heard(ctx context.Context, m jetstream.Msg) bool {
|
||||
rv.logf("%s holds what is no version-released, left: %v", m.Subject(), err)
|
||||
break
|
||||
}
|
||||
if err := rv.tagged(ctx, ev); err != nil {
|
||||
if refused, err := rv.release(ctx, ev); err != nil {
|
||||
rv.logf("%v; heard again", err)
|
||||
if nakErr := m.Nak(); nakErr != nil {
|
||||
// A tag the seat git's holder did not write is handed back with the publish pause, so the deliveries the consumer
|
||||
// allows span the holder's outage (design 47 H2 H-A-484 addendum).
|
||||
nak := m.Nak
|
||||
if refused && rv.pause > 0 {
|
||||
nak = func() error { return m.NakWithDelay(rv.pause) }
|
||||
}
|
||||
if nakErr := nak(); nakErr != nil {
|
||||
rv.logf("leaving %s for the bus to give again: %v", m.Subject(), nakErr)
|
||||
}
|
||||
givenUp(m, "the version-released of "+ev.Module+" "+ev.Version, err, rv.logf)
|
||||
return true
|
||||
}
|
||||
gone, err := rv.r.Released(ctx, ev.Module, ev.Version)
|
||||
if err != nil {
|
||||
rv.logf("%v; heard again", err)
|
||||
if nakErr := m.Nak(); nakErr != nil {
|
||||
rv.logf("leaving %s for the bus to give again: %v", m.Subject(), nakErr)
|
||||
}
|
||||
givenUp(m, "the version-released of "+ev.Module+" "+ev.Version, err, rv.logf)
|
||||
return true
|
||||
}
|
||||
for _, k := range gone {
|
||||
rv.dropBound(ctx, k)
|
||||
}
|
||||
if rv.released != nil {
|
||||
rv.released(ctx, ev.Module, ev.Version)
|
||||
}
|
||||
rv.found.clear(ctx, versions.VersionUnreleased, meshconditionsclient.ScopeModule, ev.Module)
|
||||
rv.found.clear(ctx, versions.VersionHeldElsewhere, meshconditionsclient.ScopeModule, ev.Module)
|
||||
rv.found.clear(ctx, versions.VersionTaggedElsewhere, meshconditionsclient.ScopeModule, ev.Module)
|
||||
default:
|
||||
return false
|
||||
}
|
||||
@@ -345,6 +336,62 @@ func (rv *reservations) heard(ctx context.Context, m jetstream.Msg) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// release takes a released version in as the catalog's version-released says it (design 47 G §369, §370): its tag written, its
|
||||
// reservation released, its bound and publish item ended and its findings cleared. refused says the tag was not written (the
|
||||
// seat git did not write it, or the reservation could not be read to tag it); the reservation then stands.
|
||||
func (rv *reservations) release(ctx context.Context, ev meshcatalog.VersionReleasedEvent) (refused bool, err error) {
|
||||
if err := rv.tagged(ctx, ev); err != nil {
|
||||
return true, err
|
||||
}
|
||||
gone, err := rv.r.Released(ctx, ev.Module, ev.Version)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
for _, k := range gone {
|
||||
rv.dropBound(ctx, k)
|
||||
}
|
||||
if rv.released != nil {
|
||||
rv.released(ctx, ev.Module, ev.Version)
|
||||
}
|
||||
rv.found.clear(ctx, versions.VersionUnreleased, meshconditionsclient.ScopeModule, ev.Module)
|
||||
rv.found.clear(ctx, versions.VersionHeldElsewhere, meshconditionsclient.ScopeModule, ev.Module)
|
||||
rv.found.clear(ctx, versions.VersionTaggedElsewhere, meshconditionsclient.ScopeModule, ev.Module)
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// releasedAtStart takes in, at the builder's start, every built, unreleased reservation whose version the catalog's verb versions
|
||||
// holds released, as its version-released would (design 47 H2 H-A-484 addendum): a version-released the bus gave up on, or one
|
||||
// said while the builder was away, is taken all the same, its tag written, its reservation released and its publish item ended.
|
||||
// A module whose versions cannot be read, and a release that is not taken, are said and left to its next version-released.
|
||||
func (rv *reservations) releasedAtStart(ctx context.Context) {
|
||||
all, err := rv.r.All(ctx)
|
||||
if err != nil {
|
||||
rv.logf("the reservations are not held against the catalog's released versions at the start: %v", err)
|
||||
return
|
||||
}
|
||||
read := map[string][]string{}
|
||||
for _, v := range builtOf(all) {
|
||||
if versions.NotStanding(v, true) != "" {
|
||||
continue
|
||||
}
|
||||
released, ok := read[v.Module]
|
||||
if !ok {
|
||||
if released, err = rv.r.ReleasedOf(ctx, v.Module); err != nil {
|
||||
rv.logf("%s: the catalog's released versions cannot be read at the start, so its built reservations wait for its next version-released: %v", v.Module, err)
|
||||
}
|
||||
read[v.Module] = released
|
||||
}
|
||||
if !slices.Contains(released, v.Version) {
|
||||
continue
|
||||
}
|
||||
if _, err := rv.release(ctx, meshcatalog.VersionReleasedEvent{Module: v.Module, Version: v.Version}); err != nil {
|
||||
rv.logf("%s %s is released in the catalog and not taken in at the start: %v", v.Module, v.Version, err)
|
||||
continue
|
||||
}
|
||||
rv.logf("%s %s is released in the catalog: taken in at the start, as its version-released", v.Module, v.Version)
|
||||
}
|
||||
}
|
||||
|
||||
// published marks the artifacts of the reservation a registry seat's published names (its version and digest). inProgress is told
|
||||
// before a pass that may read many reservations, so the handler says in progress to the bus (design 47 H H-A-280); nil is told
|
||||
// nothing.
|
||||
|
||||
@@ -164,14 +164,15 @@ func TestAReleasedVersionIsTaggedAtTheCommitItWasBuiltFrom(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
var tags []git.TagInput
|
||||
refuse := ""
|
||||
var standing []git.Tag // the tags the seat git's tags answers (design 47 H2 §796)
|
||||
var standing []git.Tag // the tags the seat git's tags answers (design 47 H2 §796)
|
||||
var catalog []meshcatalog.VersionEntry // the released versions the catalog's verb versions answers
|
||||
call := func(addr string, body any) (json.RawMessage, error) {
|
||||
switch addr {
|
||||
case "seat:git.tags":
|
||||
return json.Marshal(map[string]any{"form": 2, "tags": standing, "a-field-of-a-later-form": true})
|
||||
case "seat:git.tag":
|
||||
default:
|
||||
return json.Marshal(meshcatalog.VersionsAnswer{})
|
||||
return json.Marshal(meshcatalog.VersionsAnswer{Versions: catalog})
|
||||
}
|
||||
tags = append(tags, body.(git.TagInput))
|
||||
return json.Marshal(git.TagAnswer{Performed: refuse == "", Error: refuse})
|
||||
@@ -203,11 +204,44 @@ func TestAReleasedVersionIsTaggedAtTheCommitItWasBuiltFrom(t *testing.T) {
|
||||
if m := release("pkg", "0.1.0"); !m.naked || m.acked {
|
||||
t.Fatalf("a tag the forge did not write: naked %v acked %v; want the release heard again", m.naked, m.acked)
|
||||
}
|
||||
// Design 47 H2 H-A-484 addendum: with a publish pause set, a refused tag hands the version-released back after the pause, so
|
||||
// the deliveries its consumer allows span the holder's outage.
|
||||
rv.pause = 30 * time.Second
|
||||
if m := release("pkg", "0.1.0"); m.delayed != 30*time.Second || m.naked || m.acked {
|
||||
t.Fatalf("a tag the forge did not write, with the pause: delayed %v naked %v acked %v; want handed back after 30s", m.delayed, m.naked, m.acked)
|
||||
}
|
||||
rv.pause = 0
|
||||
if all, _ := rv.r.All(ctx); len(all) != 1 {
|
||||
t.Fatalf("the reservation went with an unwritten tag: %v", all)
|
||||
}
|
||||
// Design 47 H2 H-A-484 addendum: at the builder's start, a built reservation the catalog holds released is tagged and released
|
||||
// as its version-released would (a version-released the bus gave up on), and one the catalog does not hold, or not built,
|
||||
// stands.
|
||||
refuse = ""
|
||||
tags = nil
|
||||
var ended []string
|
||||
rv.released = func(_ context.Context, module, version string) { ended = append(ended, module+" "+version) }
|
||||
rv.releasedAtStart(ctx) // built nothing yet: nothing taken
|
||||
if all, _ := rv.r.All(ctx); len(all) != 1 || len(tags) != 0 {
|
||||
t.Fatalf("at the start with no build done: reservations %v, tags %+v; want the reservation standing, no tag", all, tags)
|
||||
}
|
||||
if _, err := rv.r.Built(ctx, "pkg", "0.1.0", "pkg:lib", "sha256:aa", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rv.releasedAtStart(ctx) // the catalog holds none released: it stands
|
||||
if all, _ := rv.r.All(ctx); len(all) != 1 || len(tags) != 0 {
|
||||
t.Fatalf("at the start with nothing released: reservations %v, tags %+v; want the reservation standing", all, tags)
|
||||
}
|
||||
catalog = []meshcatalog.VersionEntry{{Module: "pkg", Version: "0.1.0"}}
|
||||
rv.releasedAtStart(ctx)
|
||||
if all, _ := rv.r.All(ctx); len(all) != 0 || len(tags) != 1 || tags[0] != (git.TagInput{Repository: "o/r", Commit: "c0ffee", Tag: "go/v0.1.0"}) || !slices.Equal(ended, []string{"pkg 0.1.0"}) {
|
||||
t.Fatalf("at the start with 0.1.0 released: reservations %v, tags %+v, items ended %v; want it tagged, released and its item ended", all, tags, ended)
|
||||
}
|
||||
catalog, rv.released = nil, nil
|
||||
if _, err := rv.Reserve(ctx, "pkg", "0.1", []string{"pkg:lib"}, goPkg); err != nil { // the version is reserved again for the rows below
|
||||
t.Fatal(err)
|
||||
}
|
||||
tags = nil
|
||||
if m := release("pkg", "0.1.0"); !m.acked || len(tags) != 1 || tags[0] != (git.TagInput{Repository: "o/r", Commit: "c0ffee", Tag: "go/v0.1.0"}) {
|
||||
t.Fatalf("the release: acked %v, tags %+v; want go/v0.1.0 at c0ffee of o/r", m.acked, tags)
|
||||
}
|
||||
|
||||
@@ -74,14 +74,11 @@ const ModePublish = "publish"
|
||||
// optional, name what to publish again; the builder answers with the versions it queued (Queued).
|
||||
const ModeRepublish = "republish"
|
||||
|
||||
// ModePublishDrop and ModePublishFirst are the Modes of a reasks value of builder_publish_drop and builder_publish_first (design
|
||||
// 47 H2 §793): Module and Version name the item; the builder terminates it, or puts it again as live and terminates the old.
|
||||
const (
|
||||
ModePublishDrop = "publish-drop"
|
||||
ModePublishFirst = "publish-first"
|
||||
)
|
||||
// ModePublishDrop is the Mode of a reasks value of builder_publish_drop (design 47 H2 §793): Module and Version name the item;
|
||||
// the builder terminates it.
|
||||
const ModePublishDrop = "publish-drop"
|
||||
|
||||
// NewPublishAct is the value builder_publish_drop (mode ModePublishDrop) or builder_publish_first (ModePublishFirst) writes
|
||||
// NewPublishAct is the value builder_publish_drop (mode ModePublishDrop) writes
|
||||
// (§793): its key, `<mode>.<unix nanoseconds>` (a call is one value), and the value. Module, version and why are required.
|
||||
func NewPublishAct(mode, module, version, why, by string, at time.Time) (string, []byte, error) {
|
||||
module, version = strings.TrimSpace(module), strings.TrimSpace(version)
|
||||
|
||||
@@ -27,10 +27,8 @@ type Acts struct {
|
||||
ReserveDone func(ctx context.Context, module, fp, by, why string, at time.Time) (string, error)
|
||||
// DropReserved drops a module's reservation of one MAJOR.MINOR by hand (builder_reserved_drop): its answer, or why not.
|
||||
DropReserved func(ctx context.Context, module, series, by, why string, at time.Time) (string, error)
|
||||
// PublishDrop terminates one publish item, and PublishFirst puts it again as live and terminates the old (builder_publish_drop,
|
||||
// builder_publish_first, design 47 H2 §793): the answer, or why not.
|
||||
PublishDrop func(ctx context.Context, module, version, by, why string) (string, error)
|
||||
PublishFirst func(ctx context.Context, module, version, by, why string) (string, error)
|
||||
// PublishDrop terminates one publish item (builder_publish_drop, design 47 H2 §793): the answer, or why not.
|
||||
PublishDrop func(ctx context.Context, module, version, by, why string) (string, error)
|
||||
}
|
||||
|
||||
// FloorAgain is builder_floor_again of name: refused, since the builder numbers nothing of its own and no claim asks a floor
|
||||
|
||||
@@ -37,9 +37,8 @@ const (
|
||||
PublishQueueTool = "builder_publish_queue"
|
||||
// QueueFull is what version-unreleased says of a version the work queue's cap refused.
|
||||
QueueFull = "queue full"
|
||||
// PublishDropTool and PublishFirstTool are builder_publish_drop and builder_publish_first (§793).
|
||||
PublishDropTool = "builder_publish_drop"
|
||||
PublishFirstTool = "builder_publish_first"
|
||||
// PublishDropTool is builder_publish_drop (§793); builder_publish_first is retired (H-A-483, third addendum).
|
||||
PublishDropTool = "builder_publish_drop"
|
||||
// NotStoredWords opens what a refused artifact is said as.
|
||||
NotStoredWords = "not stored: "
|
||||
// VerbPublish is the registry seats' verb a publish again sends, VerbVersions the one whose record says what a seat holds
|
||||
@@ -48,7 +47,7 @@ const (
|
||||
VerbVersions = "versions"
|
||||
// PublishClassLive is the class of a publish whose build ended in this run, which a pipeline's build step waits on;
|
||||
// PublishClassStart that of a publish asked again at a start, of a build that ended before it. A holder's items are pulled
|
||||
// live first, each class oldest first (design 47 H H-A-290 (1), H2 §793).
|
||||
// by its one consumer in the order they are put (design 47 H H-A-290 (1), H2 §793, H-A-483).
|
||||
PublishClassLive = "live"
|
||||
PublishClassStart = "again"
|
||||
// PublishSendsAtOnce bounds the sends of one holder that run at once, its consumers' max ack pending (H-A-290 (3), §793).
|
||||
@@ -97,21 +96,19 @@ type QueuedPublish struct {
|
||||
Class string `json:"class,omitempty"`
|
||||
}
|
||||
|
||||
// rank is the place of q's class in a holder's queue: the live first.
|
||||
func (q QueuedPublish) rank() int {
|
||||
if q.Class == PublishClassLive {
|
||||
return 0
|
||||
}
|
||||
return 1
|
||||
}
|
||||
|
||||
// sortQueued sorts a holder's queued versions in the order they are sent: the live class first, each class oldest first.
|
||||
func sortQueued(qs []QueuedPublish) {
|
||||
sort.SliceStable(qs, func(i, j int) bool {
|
||||
if a, b := qs[i].rank(), qs[j].rank(); a != b {
|
||||
// streamOrder sorts the keys of the work queue's items in stream order: each key is its item's stream sequence (the builder's
|
||||
// reader, publishitem.go), compared as a number; a key that is no number sorts after them by its words.
|
||||
func streamOrder(keys []string) {
|
||||
sort.SliceStable(keys, func(i, j int) bool {
|
||||
a, aerr := strconv.ParseUint(keys[i], 10, 64)
|
||||
b, berr := strconv.ParseUint(keys[j], 10, 64)
|
||||
switch {
|
||||
case aerr == nil && berr == nil:
|
||||
return a < b
|
||||
case aerr == nil || berr == nil:
|
||||
return aerr == nil
|
||||
}
|
||||
return qs[i].Queued.Before(qs[j].Queued)
|
||||
return keys[i] < keys[j]
|
||||
})
|
||||
}
|
||||
|
||||
@@ -456,7 +453,7 @@ func (r *Republisher) Republish(ctx context.Context, module, version string) ([]
|
||||
type Queued struct {
|
||||
Total int `json:"total"`
|
||||
Counts []QueueCount `json:"counts"`
|
||||
// Queue are the first queued versions, per holder in the order they are sent (the live class first, each class oldest first);
|
||||
// Queue are the first queued versions, in stream order, the order each holder's one consumer gives them (H-A-483);
|
||||
// with a module named, only that module's.
|
||||
Queue []QueuedPublish `json:"queue"`
|
||||
// Listed says how many of Total the answer lists.
|
||||
@@ -512,7 +509,8 @@ func PublishItemKey(module, version string) string {
|
||||
|
||||
// PublishQueueOf is the seat mesh-builds' verb publish-queue (design 47 H2 §793; H-A-454): the builder's read of its own work queue
|
||||
// publishes (queue, its items read in bulk, capped past its bound), small at any size of the queue (H-A-290 (5)): the items' count
|
||||
// per holder and class with the oldest and newest queue time, and the first PublishQueueListCap in the order they are sent; module
|
||||
// per holder and class with the oldest and newest queue time, and the first PublishQueueListCap in stream order, the order the
|
||||
// holder's one consumer gives them (H-A-483, third addendum); module
|
||||
// (and version) narrow the list to one module's (one version's), so an item's class can be read.
|
||||
func PublishQueueOf(queue Reader, module, version string) (meshbuilds.PublishQueueAnswer, error) {
|
||||
out := meshbuilds.PublishQueueAnswer{Counts: []meshbuilds.PublishQueueAnswerCountsItem{}, Queue: []meshbuilds.PublishQueueAnswerQueueItem{}}
|
||||
@@ -525,7 +523,13 @@ func PublishQueueOf(queue Reader, module, version string) (meshbuilds.PublishQue
|
||||
}
|
||||
counts := map[[2]string]*meshbuilds.PublishQueueAnswerCountsItem{}
|
||||
var listed []QueuedPublish
|
||||
for _, raw := range vals {
|
||||
keys := make([]string, 0, len(vals))
|
||||
for k := range vals {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
streamOrder(keys)
|
||||
for _, k := range keys {
|
||||
raw := vals[k]
|
||||
var q QueuedPublish
|
||||
if json.Unmarshal(raw, &q) != nil {
|
||||
continue
|
||||
@@ -559,25 +563,12 @@ func PublishQueueOf(queue Reader, module, version string) (meshbuilds.PublishQue
|
||||
}
|
||||
return out.Counts[i].Class < out.Counts[j].Class
|
||||
})
|
||||
sort.SliceStable(listed, func(i, j int) bool { return listed[i].Holder < listed[j].Holder })
|
||||
byHolder := map[string][]QueuedPublish{}
|
||||
var holders []string
|
||||
for _, q := range listed {
|
||||
if byHolder[q.Holder] == nil {
|
||||
holders = append(holders, q.Holder)
|
||||
}
|
||||
byHolder[q.Holder] = append(byHolder[q.Holder], q)
|
||||
}
|
||||
for _, h := range holders {
|
||||
qs := byHolder[h]
|
||||
sortQueued(qs)
|
||||
for _, q := range qs {
|
||||
if len(out.Queue) == PublishQueueListCap {
|
||||
break
|
||||
}
|
||||
out.Queue = append(out.Queue, meshbuilds.PublishQueueAnswerQueueItem{Holder: q.Holder, Module: q.Module, Version: q.Version,
|
||||
Class: q.Class, Artifacts: q.Artifacts, Queued: q.Queued})
|
||||
if len(out.Queue) == PublishQueueListCap {
|
||||
break
|
||||
}
|
||||
out.Queue = append(out.Queue, meshbuilds.PublishQueueAnswerQueueItem{Holder: q.Holder, Module: q.Module, Version: q.Version,
|
||||
Class: q.Class, Artifacts: q.Artifacts, Queued: q.Queued})
|
||||
}
|
||||
out.Listed = int64(len(out.Queue))
|
||||
return out, nil
|
||||
@@ -631,20 +622,17 @@ func AnswerQueue(q meshbuilds.PublishQueueAnswer, acted, reasks Reader) (Queued,
|
||||
|
||||
// PublishQueueDescription is builder_publish_queue's description.
|
||||
const PublishQueueDescription = "The versions the builder will publish again (its work queue " + PublishesLocal + ", design 47 G section 588, H2 section 793): " +
|
||||
"how many each registry seat holder's items hold per class (live, a build a pipeline's step waits on, pulled first, or again, asked again at a start) " +
|
||||
"with the oldest and newest queue time; the first 50 in the order they are pulled, each with its class, its artifacts' digests and when it was " +
|
||||
"how many each registry seat holder's items hold per class (live, a build a pipeline's step waits on, or again, asked again at a start) " +
|
||||
"with the oldest and newest queue time; the first 50 in stream order, the order each holder's one consumer pulls them, each with its class, its artifacts' digests and when it was " +
|
||||
"queued (module, and version, list only that module's, so one item's class can be read); the keys of the state " + ActedState + ", the last stream " +
|
||||
"sequence the builder acted on for each subject; and the publish items (design 47 G A-457a), counted, the first 50 by next try, each version's " +
|
||||
"tries and next try, and why one ended (its tries spent, or dropped). A read past its pages is answered capped, naming the items read and the " +
|
||||
"queue's size (H-A-293). Read-only."
|
||||
|
||||
// PublishDropDescription and PublishFirstDescription are builder_publish_drop's and builder_publish_first's (§793).
|
||||
// PublishDropDescription is builder_publish_drop's (§793).
|
||||
const PublishDropDescription = "Drop one version's publish (design 47 H2 section 793): its items on the work queue " + PublishesLocal + " are " +
|
||||
"terminated at every holder, and its publish item ended with who dropped it and why, so it is never sent or put again; builder_publish_again " +
|
||||
"asks it anew. A hand act, recorded in the hand-act log first."
|
||||
const PublishFirstDescription = "Put one version's publish first (design 47 H2 section 793): each of its again items on the work queue " +
|
||||
PublishesLocal + " is put again as live, under the message id <holder>.live.<module>.<version>, pulled before every again item, and the old " +
|
||||
"terminated. A hand act, recorded in the hand-act log first."
|
||||
|
||||
// PublishAgainDescription is builder_publish_again's description.
|
||||
const PublishAgainDescription = "Publish reserved versions again (design 47 G section 588), which builds nothing: with module and version, that reserved " +
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
@@ -404,8 +405,9 @@ func answerOf(queue, acted, reasks Reader, module, version string) (Queued, erro
|
||||
}
|
||||
|
||||
// TestThePublishQueueToolAnswersAtAnySize: design 47 H H-A-290 (5) — builder_publish_queue over two thousand queued versions answers
|
||||
// within a second and small: counts per holder and class with the oldest and newest queue time, a list capped at fifty in the
|
||||
// order they are sent, and one module's item with its class; each state read in bulk where the reader can.
|
||||
// within a second and small: counts per holder and class with the oldest and newest queue time, a list capped at fifty in stream
|
||||
// order (H-A-483, third addendum: each key is its item's stream sequence), and one module's item with its class; each state read
|
||||
// in bulk where the reader can.
|
||||
func TestThePublishQueueToolAnswersAtAnySize(t *testing.T) {
|
||||
at := time.Date(2026, 11, 20, 17, 0, 0, 0, time.UTC)
|
||||
vals := map[string]json.RawMessage{}
|
||||
@@ -418,7 +420,7 @@ func TestThePublishQueueToolAnswersAtAnySize(t *testing.T) {
|
||||
class = PublishClassLive
|
||||
}
|
||||
q := QueuedPublish{V: 1, Holder: holder, Module: fmt.Sprintf("m%d", i), Version: "0.1.0", Artifacts: map[string]string{"a": "sha256:aa"}, Queued: at.Add(time.Duration(i) * time.Second), Class: class}
|
||||
vals[QueueKey(holder, q.Module, q.Version)], _ = json.Marshal(q)
|
||||
vals[strconv.Itoa(i+1)], _ = json.Marshal(q) // the reader keys each item by its stream sequence
|
||||
}
|
||||
gets := 0
|
||||
queue := Reader{All: func() (StatePage, error) { return StatePage{Values: vals}, nil },
|
||||
@@ -435,8 +437,10 @@ func TestThePublishQueueToolAnswersAtAnySize(t *testing.T) {
|
||||
if fmt.Sprintln(a.Counts[1].Holder, a.Counts[1].Class, a.Counts[1].Count, a.Counts[1].Oldest.Equal(at.Add(1991*time.Second)), a.Counts[1].Newest.Equal(at.Add(1999*time.Second))) != "go "+PublishClassLive+" 5 true true\n" || len(a.Counts) != 4 {
|
||||
t.Errorf("counts %+v, want per holder and class with their oldest and newest", a.Counts)
|
||||
}
|
||||
if a.Queue[0].Class != PublishClassLive || a.Queue[0].Holder != "go" {
|
||||
t.Errorf("the list opens with %+v, want the go holder's live class, sent first", a.Queue[0])
|
||||
for i, q := range a.Queue {
|
||||
if q.Module != fmt.Sprintf("m%d", i) {
|
||||
t.Fatalf("the list holds %s at %d, want m%d: the items in stream order, whatever their holder or class", q.Module, i, i)
|
||||
}
|
||||
}
|
||||
one, err := answerOf(queue, none, none, "m1995", "")
|
||||
if err != nil || len(one.Queue) != 1 || one.Queue[0].Class != PublishClassLive || one.Total != 2000 {
|
||||
|
||||
@@ -57,7 +57,7 @@ func VerbDescription(verb string) string {
|
||||
case VerbDrop:
|
||||
return "End an ask on the seat's work queue for a caller that no longer waits for it (its step ended by its bound): one waiting is ended dropped when it is pulled, one taken is cancelled; both end the step could-not-run, its why naming the drop and who asked it, so no holder works on it after (design 47 H2 §778; H-A-401; ADR 0664 §4)."
|
||||
case VerbPublishQueue:
|
||||
return "What the builder's work queue publishes holds, read by the builder from its own stream (design 47 H2 §793; H-A-454): the items' count per registry seat holder and class with the oldest and newest queue time, and the first 50 in the order they are pulled (the live class first, each class oldest first); module, and version, list only that module's items. A queue read past 4,000 items is answered capped, naming the items read and the queue's size."
|
||||
return "What the builder's work queue publishes holds, read by the builder from its own stream (design 47 H2 §793; H-A-454): the items' count per registry seat holder and class with the oldest and newest queue time, and the first 50 in stream order, the order each holder's one consumer pulls them (H-A-483); module, and version, list only that module's items. A queue read past 4,000 items is answered capped, naming the items read and the queue's size."
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -338,7 +338,7 @@ const (
|
||||
type PublishQueueAnswerQueueItem struct {
|
||||
// each artifact reference, <module>:<artifact>, with the digest to publish
|
||||
Artifacts map[string]string `json:"artifacts,omitempty"`
|
||||
// live, a build a pipeline's step waits on, pulled first; again, asked again at a start
|
||||
// live, a build a pipeline's step waits on; again, asked again at a start
|
||||
Class string `json:"class"`
|
||||
// the registry seat's holder the item is sent to, a module
|
||||
Holder string `json:"holder"`
|
||||
@@ -368,7 +368,7 @@ type PublishQueueAnswer struct {
|
||||
ItemsRead int64 `json:"items-read,omitempty"`
|
||||
// how many of total the answer lists
|
||||
Listed int64 `json:"listed"`
|
||||
// the first 50 items listed, per holder in the order they are pulled
|
||||
// the first 50 items listed, in stream order
|
||||
Queue []PublishQueueAnswerQueueItem `json:"queue"`
|
||||
// with capped: the queue's size in bytes
|
||||
QueueBytes int64 `json:"queue-bytes,omitempty"`
|
||||
|
||||
@@ -14,13 +14,13 @@ const Seat = "mesh-pipeline-runner"
|
||||
|
||||
// Form is the form of the seat's values, events and answers this contract was generated at (design 47 H2 §780):
|
||||
// a reader of a value of another form says "form N, this reader knows M".
|
||||
const Form = 7
|
||||
const Form = 8
|
||||
|
||||
// The forms and the digest of the fields this contract was generated from: the generator's compare fails a definition
|
||||
// whose fields changed while its forms did not (design 47 H2 §780).
|
||||
const (
|
||||
generatedForms = "7"
|
||||
generatedFieldsDigest = "AcceptAnswer=224df0394653 Accepted=d2b011e5090f AskAgainAnswer=224df0394653 CancelAnswer=a0dcc32b3f57 EndAgainAnswer=a0dcc32b3f57 FailedStep=cd7184342136 Fill=7a76c9ceb157 FillingEvent=5bd4df78ed84 FullEvent=5bd4df78ed84 KeptAnswer=dfe0d48bc169 KeptByLevelDigestValue=f06c90f7da7f KeptResult=af5f81f189b7 ListAnswer=284210b3c054 LogEntry=b059967b2ae4 Member=8a103baec01f Mover=5db376275bcf MoverEnded=4b35095f6829 MoverRow=b61f41d131a3 Pipeline=2663792125bf PipelineEnded=523f8fc0fe41 PipelineEndedEvent=cb44007f7173 PipelineEndedPhases=9d1c33ce4cc4 PipelineEndedPhasesBuild=a08d49692a09 PipelineEndedPhasesComposition=a08d49692a09 PipelineEndedPhasesIntegration=a08d49692a09 PipelineEndedPhasesLint=a08d49692a09 PipelineEndedPhasesUnit=a08d49692a09 PipelineEndedPhasesValidate=a08d49692a09 PipelineRow=f58dbdcaf2d4 PipelineStarted=a3d000f095db PipelineStartedEvent=eaa401af4117 PipelinesByBranchRevisionUnfinishedModuleValue=693580eed498 PipelinesByBranchRevisionValue=693580eed498 PipelinesByBranchRunRunValue=693580eed498 PipelinesByBranchSourcePublishedPackageVersionValue=693580eed498 PipelinesByStepsPipelineMoverValue=94de99db7816 PlaceAnswer=a0dcc32b3f57 PlaceUnfinishedAnswer=ed94ced4513a Published=d9af3024dacc QueueByQueueNextValue=765a2918a728 QueueByRunningPipelineValue=ed94ced4513a QueueByWaitingSequenceValue=99605298ae13 RePinnedEvent=92fd04bfa6d3 RerunAnswer=9b7efd0df344 ReviewAnswer=f397b8965179 RoomEvent=5bd4df78ed84 RunAnswer=96bd45aa2e62 RunStarted=b280617410c3 RunStartedEvent=0767a9358fa3 Shared=c981cf6aa860 SharedAskedBy=c9b30cc4d0d0 ShowAnswer=48b8cecc529a StandsAnswer=0afa3693bf3b StartAnswer=ed94ced4513a StepEndedEvent=799020642c19 StepEvent=64f18c40ed5d StepLogByPipelineValue=ffdde090a101 StepResult=0dac0e15d017 StepStartedEvent=799020642c19 TestCounts=9c1332427159 TestRun=ca5a87f3b6b8 TestRunPackage=a7d30558470d TestRunReader=c9b30cc4d0d0 TestRunsAnswer=7ab81a45ca33 TestRunsDropAnswer=8ee70b35469d ToolchainUsed=43b6d42376fb Wait=21e4ee7a3ef1 WaitEndedEvent=2bf6d38041c0 WaitHeldEvent=2bf6d38041c0 Waiting=ed94ced4513a WhyCarriedAnswer=8e2fcea6479f Widening=e2790a307466"
|
||||
generatedForms = "8"
|
||||
generatedFieldsDigest = "AcceptAnswer=224df0394653 Accepted=d2b011e5090f AskAgainAnswer=224df0394653 CancelAnswer=a0dcc32b3f57 EndAgainAnswer=a0dcc32b3f57 FailedStep=cd7184342136 Fill=7a76c9ceb157 FillingEvent=5bd4df78ed84 FullEvent=5bd4df78ed84 KeptAnswer=dfe0d48bc169 KeptByLevelDigestValue=f06c90f7da7f KeptResult=af5f81f189b7 ListAnswer=284210b3c054 LogEntry=b059967b2ae4 Member=8a103baec01f Mover=5db376275bcf MoverEnded=4b35095f6829 MoverRow=b61f41d131a3 Pipeline=2663792125bf PipelineEnded=523f8fc0fe41 PipelineEndedEvent=cb44007f7173 PipelineEndedPhases=9d1c33ce4cc4 PipelineEndedPhasesBuild=a08d49692a09 PipelineEndedPhasesComposition=a08d49692a09 PipelineEndedPhasesIntegration=a08d49692a09 PipelineEndedPhasesLint=a08d49692a09 PipelineEndedPhasesUnit=a08d49692a09 PipelineEndedPhasesValidate=a08d49692a09 PipelineRow=f58dbdcaf2d4 PipelineStarted=a3d000f095db PipelineStartedEvent=eaa401af4117 PipelinesByBranchRevisionAgainNValue=693580eed498 PipelinesByBranchRevisionUnfinishedModuleValue=693580eed498 PipelinesByBranchRevisionValue=693580eed498 PipelinesByBranchRunRunValue=693580eed498 PipelinesByBranchSourcePublishedPackageVersionValue=693580eed498 PipelinesByStepsPipelineMoverValue=94de99db7816 PlaceAnswer=a0dcc32b3f57 PlaceUnfinishedAnswer=ed94ced4513a Published=d9af3024dacc QueueByQueueNextValue=765a2918a728 QueueByRunningPipelineValue=ed94ced4513a QueueByWaitingSequenceValue=99605298ae13 RePinnedEvent=92fd04bfa6d3 RerunAnswer=9b7efd0df344 ReviewAnswer=f397b8965179 RoomEvent=5bd4df78ed84 RunAnswer=96bd45aa2e62 RunStarted=b280617410c3 RunStartedEvent=0767a9358fa3 Shared=c981cf6aa860 SharedAskedBy=c9b30cc4d0d0 ShowAnswer=48b8cecc529a StandsAnswer=0afa3693bf3b StartAnswer=ed94ced4513a StepEndedEvent=799020642c19 StepEvent=64f18c40ed5d StepLogByPipelineValue=ffdde090a101 StepResult=0dac0e15d017 StepStartedEvent=799020642c19 TestCounts=9c1332427159 TestRun=ca5a87f3b6b8 TestRunPackage=a7d30558470d TestRunReader=c9b30cc4d0d0 TestRunsAnswer=7ab81a45ca33 TestRunsDropAnswer=8ee70b35469d ToolchainUsed=43b6d42376fb Wait=21e4ee7a3ef1 WaitEndedEvent=2bf6d38041c0 WaitHeldEvent=2bf6d38041c0 Waiting=ed94ced4513a WhyCarriedAnswer=8e2fcea6479f Widening=e2790a307466"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -69,7 +69,7 @@ func VerbDescription(verb string) string {
|
||||
case VerbRun:
|
||||
return "The run on request of one or more members, each a module at a named commit, through their pipelines; delivers nothing, replacing the checker's try (ADR 0666 section 18). Says run-started."
|
||||
case VerbStart:
|
||||
return "A delivering pipeline for a watched branch's head, by hand; written in the hand-act log."
|
||||
return "A delivering pipeline for a watched branch's head, by hand; written in the hand-act log. With modules, a pipeline at the head keyed <branch>.<revision>.again.<n> carrying the named modules whose newest entry has no passed pipeline, at the take-in's tiers; a module whose newest entry passed is refused by name. Exactly one of module or modules (design 47 H2 H-A-479)."
|
||||
case VerbShow:
|
||||
return "One pipeline: naming a mover, its steps alone; with no mover, each mover's row (its module, outcome and failed step), never every step of every mover, or the steps of any mover the step and state filters hold; and its step log, with the stores' fill (design 47 H2 §789, H-A-376). Reads only."
|
||||
case VerbList:
|
||||
@@ -354,6 +354,44 @@ func DecodePipelinesByBranchRevisionUnfinishedModuleValue(raw []byte) (Pipelines
|
||||
return v, formRead(raw), nil
|
||||
}
|
||||
|
||||
// PipelinesByBranchRevisionAgainNKey is the key of one value of the state pipelines under the form "<branch>.<revision>.again.<n>".
|
||||
func PipelinesByBranchRevisionAgainNKey(branch, revision, n string) string {
|
||||
return branch + "." + revision + ".again." + n
|
||||
}
|
||||
|
||||
// ParsePipelinesByBranchRevisionAgainNKey reads the parts of a key of the form "<branch>.<revision>.again.<n>", in the order of the form; the last result is false for a key of another form.
|
||||
func ParsePipelinesByBranchRevisionAgainNKey(key string) (string, string, string, bool) {
|
||||
rest := key
|
||||
v0, after0, found0 := strings.Cut(rest, ".")
|
||||
if !found0 || v0 == "" {
|
||||
return "", "", "", false
|
||||
}
|
||||
rest = after0
|
||||
v2, after2, found2 := strings.Cut(rest, ".again.")
|
||||
if !found2 || v2 == "" {
|
||||
return "", "", "", false
|
||||
}
|
||||
rest = after2
|
||||
if rest == "" {
|
||||
return "", "", "", false
|
||||
}
|
||||
v4 := rest
|
||||
rest = ""
|
||||
if rest != "" {
|
||||
return "", "", "", false
|
||||
}
|
||||
return v0, v2, v4, true
|
||||
}
|
||||
|
||||
// DecodePipelinesByBranchRevisionAgainNValue reads one value of the state pipelines under the form "<branch>.<revision>.again.<n>" as another part wrote it: a field PipelinesByBranchRevisionAgainNValue does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
|
||||
func DecodePipelinesByBranchRevisionAgainNValue(raw []byte) (PipelinesByBranchRevisionAgainNValue, int, error) {
|
||||
var v PipelinesByBranchRevisionAgainNValue
|
||||
if err := json.Unmarshal(raw, &v); err != nil {
|
||||
return v, 0, err
|
||||
}
|
||||
return v, formRead(raw), nil
|
||||
}
|
||||
|
||||
// PipelinesByStepsPipelineMoverKey is the key of one value of the state pipelines under the form "steps.<pipeline>.<mover>".
|
||||
func PipelinesByStepsPipelineMoverKey(pipeline, mover string) string {
|
||||
return "steps." + pipeline + "." + mover
|
||||
@@ -1244,7 +1282,10 @@ type RunAnswer struct {
|
||||
// StartInput is what start takes.
|
||||
type StartInput struct {
|
||||
Branch string `json:"branch"`
|
||||
Module string `json:"module"`
|
||||
// the one mover of the head's pipeline <branch>.<revision>; absent when modules is given
|
||||
Module string `json:"module,omitempty"`
|
||||
// the modules started again at the head, each whose newest entry has no passed pipeline (a failed, cut or superseded one); absent when module is given (design 47 H2 H-A-479)
|
||||
Modules []string `json:"modules,omitempty"`
|
||||
// why, written with the caller the runtime proves in the hand-act log
|
||||
Why string `json:"why"`
|
||||
}
|
||||
@@ -1637,6 +1678,9 @@ type PipelinesByBranchRunRunValue = Pipeline
|
||||
// PipelinesByBranchRevisionUnfinishedModuleValue is one value of the state pipelines under the key <branch>.<revision>.unfinished.<module>.
|
||||
type PipelinesByBranchRevisionUnfinishedModuleValue = Pipeline
|
||||
|
||||
// PipelinesByBranchRevisionAgainNValue is one value of the state pipelines under the key <branch>.<revision>.again.<n>.
|
||||
type PipelinesByBranchRevisionAgainNValue = Pipeline
|
||||
|
||||
// PipelinesByStepsPipelineMoverValue is one value of the state pipelines under the key steps.<pipeline>.<mover>.
|
||||
type PipelinesByStepsPipelineMoverValue struct {
|
||||
// is the form of what this value carries, left out while it is 1 (design 47 H2 §780).
|
||||
|
||||
@@ -156,7 +156,7 @@
|
||||
{
|
||||
"name": "publish-queue",
|
||||
"class": "read",
|
||||
"description": "What the builder's work queue publishes holds, read by the builder from its own stream (design 47 H2 §793; H-A-454): the items' count per registry seat holder and class with the oldest and newest queue time, and the first 50 in the order they are pulled (the live class first, each class oldest first); module, and version, list only that module's items. A queue read past 4,000 items is answered capped, naming the items read and the queue's size.",
|
||||
"description": "What the builder's work queue publishes holds, read by the builder from its own stream (design 47 H2 §793; H-A-454): the items' count per registry seat holder and class with the oldest and newest queue time, and the first 50 in stream order, the order each holder's one consumer pulls them (H-A-483); module, and version, list only that module's items. A queue read past 4,000 items is answered capped, naming the items read and the queue's size.",
|
||||
"input": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
@@ -243,7 +243,7 @@
|
||||
"live",
|
||||
"again"
|
||||
],
|
||||
"description": "live, a build a pipeline's step waits on, pulled first; again, asked again at a start"
|
||||
"description": "live, a build a pipeline's step waits on; again, asked again at a start"
|
||||
},
|
||||
"artifacts": {
|
||||
"type": "object",
|
||||
@@ -266,7 +266,7 @@
|
||||
"queued"
|
||||
]
|
||||
},
|
||||
"description": "the first 50 items listed, per holder in the order they are pulled"
|
||||
"description": "the first 50 items listed, in stream order"
|
||||
},
|
||||
"listed": {
|
||||
"type": "integer",
|
||||
|
||||
@@ -50,12 +50,20 @@
|
||||
},
|
||||
{
|
||||
"name": "start",
|
||||
"description": "A delivering pipeline for a watched branch's head, by hand; written in the hand-act log.",
|
||||
"description": "A delivering pipeline for a watched branch's head, by hand; written in the hand-act log. With modules, a pipeline at the head keyed <branch>.<revision>.again.<n> carrying the named modules whose newest entry has no passed pipeline, at the take-in's tiers; a module whose newest entry passed is refused by name. Exactly one of module or modules (design 47 H2 H-A-479).",
|
||||
"input": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"module": {
|
||||
"type": "string"
|
||||
"type": "string",
|
||||
"description": "the one mover of the head's pipeline <branch>.<revision>; absent when modules is given"
|
||||
},
|
||||
"modules": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
},
|
||||
"description": "the modules started again at the head, each whose newest entry has no passed pipeline (a failed, cut or superseded one); absent when module is given (design 47 H2 H-A-479)"
|
||||
},
|
||||
"branch": {
|
||||
"type": "string"
|
||||
@@ -66,7 +74,6 @@
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"module",
|
||||
"branch",
|
||||
"why"
|
||||
]
|
||||
@@ -910,6 +917,13 @@
|
||||
},
|
||||
"description": "a pipeline of its own of one mover kept in the runner's state unfinished, put in it by place-unfinished; the revision the unfinished entry names (design 47 H2 §86, H-A-398)"
|
||||
},
|
||||
{
|
||||
"key": "<branch>.<revision>.again.<n>",
|
||||
"value": {
|
||||
"$ref": "#/defs/Pipeline"
|
||||
},
|
||||
"description": "a pipeline started again at the head by start with modules, its movers the named modules whose newest entry has no passed pipeline; n from 1, taken by writing on absence (design 47 H2 H-A-479)"
|
||||
},
|
||||
{
|
||||
"key": "steps.<pipeline>.<mover>",
|
||||
"value": {
|
||||
@@ -2353,5 +2367,5 @@
|
||||
]
|
||||
}
|
||||
},
|
||||
"form": 7
|
||||
"form": 8
|
||||
}
|
||||
|
||||
@@ -251,7 +251,7 @@ export interface PublishQueueAnswerCountsItem {
|
||||
export interface PublishQueueAnswerQueueItem {
|
||||
/** each artifact reference, <module>:<artifact>, with the digest to publish */
|
||||
"artifacts"?: Record<string, string>;
|
||||
/** live, a build a pipeline's step waits on, pulled first; again, asked again at a start */
|
||||
/** live, a build a pipeline's step waits on; again, asked again at a start */
|
||||
"class": "live" | "again";
|
||||
/** the registry seat's holder the item is sent to, a module */
|
||||
"holder": string;
|
||||
@@ -275,7 +275,7 @@ export interface PublishQueueAnswer {
|
||||
"items-read"?: number;
|
||||
/** how many of total the answer lists */
|
||||
"listed": number;
|
||||
/** the first 50 items listed, per holder in the order they are pulled */
|
||||
/** the first 50 items listed, in stream order */
|
||||
"queue": PublishQueueAnswerQueueItem[];
|
||||
/** with capped: the queue's size in bytes */
|
||||
"queue-bytes"?: number;
|
||||
|
||||
@@ -121,6 +121,12 @@ export declare function pipelinesByBranchRevisionUnfinishedModuleKey(branch: str
|
||||
/** The parts of a key of the form "<branch>.<revision>.unfinished.<module>", or undefined for a key of another form. */
|
||||
export declare function parsePipelinesByBranchRevisionUnfinishedModuleKey(_key: string): { branch: string; revision: string; module: string } | undefined;
|
||||
|
||||
/** The key of one value of the state pipelines under the form "<branch>.<revision>.again.<n>". */
|
||||
export declare function pipelinesByBranchRevisionAgainNKey(branch: string, revision: string, n: string): string;
|
||||
|
||||
/** The parts of a key of the form "<branch>.<revision>.again.<n>", or undefined for a key of another form. */
|
||||
export declare function parsePipelinesByBranchRevisionAgainNKey(_key: string): { branch: string; revision: string; n: string } | undefined;
|
||||
|
||||
/** The key of one value of the state pipelines under the form "steps.<pipeline>.<mover>". */
|
||||
export declare function pipelinesByStepsPipelineMoverKey(pipeline: string, mover: string): string;
|
||||
|
||||
@@ -699,7 +705,10 @@ export interface RunAnswer {
|
||||
/** is what start takes. */
|
||||
export interface StartInput {
|
||||
"branch": string;
|
||||
"module": string;
|
||||
/** the one mover of the head's pipeline <branch>.<revision>; absent when modules is given */
|
||||
"module"?: string;
|
||||
/** the modules started again at the head, each whose newest entry has no passed pipeline (a failed, cut or superseded one); absent when module is given (design 47 H2 H-A-479) */
|
||||
"modules"?: string[];
|
||||
/** why, written with the caller the runtime proves in the hand-act log */
|
||||
"why": string;
|
||||
}
|
||||
@@ -1031,6 +1040,9 @@ export type PipelinesByBranchRunRunValue = Pipeline;
|
||||
/** is one value of the state pipelines under the key <branch>.<revision>.unfinished.<module>. */
|
||||
export type PipelinesByBranchRevisionUnfinishedModuleValue = Pipeline;
|
||||
|
||||
/** is one value of the state pipelines under the key <branch>.<revision>.again.<n>. */
|
||||
export type PipelinesByBranchRevisionAgainNValue = Pipeline;
|
||||
|
||||
/** is one value of the state pipelines under the key steps.<pipeline>.<mover>. */
|
||||
export interface PipelinesByStepsPipelineMoverValue {
|
||||
/** is the form of what this value carries, left out while it is 1 (design 47 H2 §780). */
|
||||
|
||||
@@ -199,6 +199,29 @@ export function parsePipelinesByBranchRevisionUnfinishedModuleKey(_key) {
|
||||
return { branch, revision, module };
|
||||
}
|
||||
|
||||
/** The key of one value of the state pipelines under the form "<branch>.<revision>.again.<n>". */
|
||||
export function pipelinesByBranchRevisionAgainNKey(branch, revision, n) {
|
||||
return branch + "." + revision + ".again." + n;
|
||||
}
|
||||
|
||||
/** The parts of a key of the form "<branch>.<revision>.again.<n>", or undefined for a key of another form. */
|
||||
export function parsePipelinesByBranchRevisionAgainNKey(_key) {
|
||||
let _rest = _key;
|
||||
const _at0 = _rest.indexOf(".");
|
||||
if (_at0 <= 0) return undefined;
|
||||
const branch = _rest.slice(0, _at0);
|
||||
_rest = _rest.slice(_at0 + 1);
|
||||
const _at2 = _rest.indexOf(".again.");
|
||||
if (_at2 <= 0) return undefined;
|
||||
const revision = _rest.slice(0, _at2);
|
||||
_rest = _rest.slice(_at2 + 7);
|
||||
if (_rest === "") return undefined;
|
||||
const n = _rest;
|
||||
_rest = "";
|
||||
if (_rest !== "") return undefined;
|
||||
return { branch, revision, n };
|
||||
}
|
||||
|
||||
/** The key of one value of the state pipelines under the form "steps.<pipeline>.<mover>". */
|
||||
export function pipelinesByStepsPipelineMoverKey(pipeline, mover) {
|
||||
return "steps." + pipeline + "." + mover;
|
||||
|
||||
@@ -105,9 +105,14 @@ func (ix *Indexer) HeardPins(ctx context.Context, started *runner.PipelineStarte
|
||||
}
|
||||
}
|
||||
}
|
||||
// A pipeline that did not pass withdraws the reservations its revision holds (design 47 H2 §805, H-A-418 (1)).
|
||||
// A pipeline that did not pass withdraws the reservations its revision holds of its own movers, never another open
|
||||
// pipeline's at that revision (design 47 H2 §805, H-A-418 (1), H-A-479 second addendum).
|
||||
if ended.Outcome != runner.PipelineEndedOutcomePassed {
|
||||
if err := ix.Withdraw(ctx, ended.Branch, ended.Revision); err != nil {
|
||||
movers := make([]string, 0, len(ended.Movers))
|
||||
for _, m := range ended.Movers {
|
||||
movers = append(movers, m.Module)
|
||||
}
|
||||
if err := ix.Withdraw(ctx, ended.Branch, ended.Revision, movers); err != nil {
|
||||
return fmt.Errorf("the reservations of %s are not withdrawn: %w", BranchObject(ended.Branch, ended.Revision), err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
@@ -151,7 +152,14 @@ func TestAReleaseRepinsAnEntryNamingItsPackageWithNoPin(t *testing.T) {
|
||||
if err := ix.Settle(ctx, next); err != nil || newest() != 4 {
|
||||
t.Fatalf("the reservation of 0.1.29 settled (%v) keeping main's newest %d, want 4", err, newest())
|
||||
}
|
||||
if err := ix.Withdraw(ctx, "main", 3); err != nil || newest() != 5 {
|
||||
// H-A-479 second addendum: a pipeline at the same revision whose movers are others withdraws none of mesh-sdk's.
|
||||
if err := ix.Withdraw(ctx, "main", 3, []string{"app"}); err != nil || newest() != 4 {
|
||||
t.Fatalf("another pipeline's withdrawal (%v) kept main's newest %d, want 4: nothing of mesh-sdk withdrawn", err, newest())
|
||||
}
|
||||
if _, _, kept, _ := heard.Get(ctx, ReservedKey(next, "mesh-sdk")); !kept {
|
||||
t.Fatalf("another pipeline's withdrawal took mesh-sdk's reservation")
|
||||
}
|
||||
if err := ix.Withdraw(ctx, "main", 3, []string{"mesh-sdk"}); err != nil || newest() != 5 {
|
||||
t.Fatalf("the withdrawal (%v) kept main's newest %d, want 5", err, newest())
|
||||
}
|
||||
if now, _ := moduleOf(t, mods, "g"); now.Packages["git.novox.be/novox/mesh-sdk/go"] != "v0.1.28" {
|
||||
@@ -160,4 +168,31 @@ func TestAReleaseRepinsAnEntryNamingItsPackageWithNoPin(t *testing.T) {
|
||||
if _, _, kept, _ := heard.Get(ctx, ReservedKey(next, "mesh-sdk")); kept {
|
||||
t.Errorf("the withdrawn reservation is still kept")
|
||||
}
|
||||
// H-A-479 second addendum: two pipelines' movers reserve at one revision while its gathering is open; withdrawing one
|
||||
// pipeline's mover keeps the gathering and the other's reservation, withdrawing the last removes the gathering too.
|
||||
six := BranchObject("main", 6)
|
||||
for _, m := range []string{"one", "two"} {
|
||||
raw, _ := json.Marshal(Reserved{V: 1, Revision: six, Module: m})
|
||||
if err := heard.Create(ctx, ReservedKey(six, m), raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := heard.Create(ctx, SettlingKey(six), []byte(`{}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ix.Withdraw(ctx, "main", 6, []string{"one"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, _, gathering, _ := heard.Get(ctx, SettlingKey(six))
|
||||
_, _, one, _ := heard.Get(ctx, ReservedKey(six, "one"))
|
||||
_, _, two, _ := heard.Get(ctx, ReservedKey(six, "two"))
|
||||
if !gathering || one || !two {
|
||||
t.Fatalf("withdrawing one: gathering %v, one kept %v, two kept %v; want the gathering and two kept, one gone", gathering, one, two)
|
||||
}
|
||||
if err := ix.Withdraw(ctx, "main", 6, []string{"two"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, gathering, _ := heard.Get(ctx, SettlingKey(six)); gathering {
|
||||
t.Error("withdrawing the last reservation kept the revision's gathering")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -350,20 +350,35 @@ func (ix *Indexer) settleNow(ctx context.Context, revision string) error {
|
||||
return err
|
||||
}
|
||||
|
||||
// Withdraw takes a pipeline-ended not passed of branch's revision in: each reservation the revision holds is withdrawn, its
|
||||
// series' numbered set back to the newest released version, its importers re-pinned to that, and its key deleted.
|
||||
func (ix *Indexer) Withdraw(ctx context.Context, branch string, revision int64) error {
|
||||
// Withdraw takes a pipeline-ended not passed of branch's revision in: each reservation the revision holds of one of the
|
||||
// ended pipeline's own movers is withdrawn, its series' numbered set back to the newest released version, its importers
|
||||
// re-pinned to that, and its key deleted. Another open pipeline's reservations at the same revision are kept, and the
|
||||
// revision's gathering is withdrawn only when none of them is left (design 47 H2 H-A-479 second addendum).
|
||||
func (ix *Indexer) Withdraw(ctx context.Context, branch string, revision int64, movers []string) error {
|
||||
if ix.Heard == nil {
|
||||
return nil
|
||||
}
|
||||
name := BranchObject(branch, revision)
|
||||
if _, rev, found, err := ix.Heard.Get(ctx, SettlingKey(name)); err == nil && found {
|
||||
_ = ix.Heard.Delete(ctx, SettlingKey(name), rev) // the gathering is withdrawn with it
|
||||
}
|
||||
held, revs, err := ix.reservedOf(ctx, name)
|
||||
if err != nil || len(held) == 0 {
|
||||
all, revs, err := ix.reservedOf(ctx, name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
held := map[string]Reserved{}
|
||||
for key, r := range all {
|
||||
if slices.Contains(movers, r.Module) {
|
||||
held[key] = r
|
||||
} else {
|
||||
delete(revs, key)
|
||||
}
|
||||
}
|
||||
if len(held) == len(all) {
|
||||
if _, rev, found, err := ix.Heard.Get(ctx, SettlingKey(name)); err == nil && found {
|
||||
_ = ix.Heard.Delete(ctx, SettlingKey(name), rev) // the gathering is withdrawn with it
|
||||
}
|
||||
}
|
||||
if len(held) == 0 {
|
||||
return nil
|
||||
}
|
||||
for _, r := range held {
|
||||
for _, one := range r.Reservations {
|
||||
if err := ix.withdrawNumbered(ctx, one); err != nil {
|
||||
|
||||
@@ -19,11 +19,11 @@ func VerbTools(ask func(key string, body any) (json.RawMessage, error)) []stdio.
|
||||
}
|
||||
return []stdio.Tool{
|
||||
verbTool[seat.StartInput](ask, "start", seat.VerbStart,
|
||||
"Start by hand the delivering pipeline of the head of a watched branch for a module, by the seat verb start: the branch, the module and why. A head whose pipeline is written already is refused by the verb, naming it.",
|
||||
obj([]string{"branch", "module", "why"}, map[string]any{"branch": str, "module": str, "why": str}),
|
||||
"Start by hand the delivering pipeline of the head of a watched branch for a module, by the seat verb start: the branch, the module and why. A head whose pipeline is written already is refused by the verb, naming it. With modules instead of module, the named modules whose newest entry has no passed pipeline (a failed, cut or superseded one) are started again at the head, keyed <branch>.<revision>.again.<n>; a module whose newest entry passed is refused by name (design 47 H2 H-A-479).",
|
||||
obj([]string{"branch", "why"}, map[string]any{"branch": str, "module": str, "modules": map[string]any{"type": "array", "items": str}, "why": str}),
|
||||
func(in seat.StartInput) string {
|
||||
if in.Branch == "" || in.Module == "" {
|
||||
return "start names the watched branch and the module"
|
||||
if in.Branch == "" || (in.Module == "") == (len(in.Modules) == 0) {
|
||||
return "start names the watched branch and exactly one of the module or the modules"
|
||||
}
|
||||
return whyOf(in.Why)
|
||||
}),
|
||||
|
||||
@@ -18,7 +18,11 @@ func TestVerbTools(t *testing.T) {
|
||||
{"start", "runner_start", "seat:mesh-pipeline-runner.start", "", map[string]any{"branch": "main", "module": "i3", "why": "cancelled"},
|
||||
seat.StartInput{Branch: "main", Module: "i3", Why: "cancelled"}},
|
||||
{"start no why", "runner_start", "", "names why", map[string]any{"branch": "main", "module": "i3"}, nil},
|
||||
{"start no module", "runner_start", "", "branch and the module", map[string]any{"branch": "main", "why": "x"}, nil},
|
||||
{"start no module", "runner_start", "", "exactly one of the module or the modules", map[string]any{"branch": "main", "why": "x"}, nil},
|
||||
// H-A-479: modules start the named modules again at the head; both together are refused.
|
||||
{"start modules", "runner_start", "seat:mesh-pipeline-runner.start", "", map[string]any{"branch": "main", "modules": []any{"i3", "lib"}, "why": "failed"},
|
||||
seat.StartInput{Branch: "main", Modules: []string{"i3", "lib"}, Why: "failed"}},
|
||||
{"start module and modules", "runner_start", "", "exactly one of the module or the modules", map[string]any{"branch": "main", "module": "i3", "modules": []any{"lib"}, "why": "x"}, nil},
|
||||
{"run", "runner_run", "seat:mesh-pipeline-runner.run", "", map[string]any{"members": []any{map[string]any{}}, "why": "x"},
|
||||
seat.RunInput{Members: []seat.Member{{}}, Why: "x"}},
|
||||
{"run no members", "runner_run", "", "members", map[string]any{}, nil},
|
||||
|
||||
@@ -96,3 +96,34 @@ func TestARevisionHeardIsHandedToItsStart(t *testing.T) {
|
||||
t.Errorf("a body that is no event = %v, want it ended", err)
|
||||
}
|
||||
}
|
||||
|
||||
// H-A-479 addendum (2): a revision is held when the catalog seat's revision verb answers that revision, read fresh; none-yet,
|
||||
// another revision or a refusal saying "no such object" is gone; another failure is said as it is.
|
||||
func TestRevisionHeldReadsTheCatalogsRevisionFresh(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
answer catalog.RevisionAnswer
|
||||
err error
|
||||
gone bool
|
||||
fails bool
|
||||
}{
|
||||
{name: "held", answer: catalog.RevisionAnswer{Revision: &catalog.CatalogRevision{Revision: 258}}},
|
||||
{name: "another revision", answer: catalog.RevisionAnswer{Revision: &catalog.CatalogRevision{Revision: 259}}, gone: true, fails: true},
|
||||
{name: "none yet", answer: catalog.RevisionAnswer{NoneYet: true}, gone: true, fails: true},
|
||||
{name: "removed", err: errors.New("revision main.258: no such object"), gone: true, fails: true},
|
||||
{name: "unreachable", err: errors.New("no responders"), fails: true},
|
||||
} {
|
||||
var asked catalog.RevisionInput
|
||||
call := func(_ string, body any) (json.RawMessage, error) {
|
||||
asked = body.(catalog.RevisionInput)
|
||||
if c.err != nil {
|
||||
return nil, c.err
|
||||
}
|
||||
return json.Marshal(c.answer)
|
||||
}
|
||||
err := revisionHeld(call)("main", 258)
|
||||
if (err != nil) != c.fails || errors.Is(err, pipeline.ErrRevisionGone) != c.gone || asked.Revision != 258 || asked.Branch != "main" {
|
||||
t.Errorf("%s: %v (asked %+v)", c.name, err, asked)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,10 +4,14 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscalls"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"git.novox.be/novox/mesh-sdk/go/seatlist"
|
||||
|
||||
@@ -65,5 +69,37 @@ func writeVerbs(h *pipeline.Hearer, s *pipeline.Starter, view func(string) (pipe
|
||||
candidate := func(group, branch string, revision int64) (pipeline.Half, error) {
|
||||
return readCandidate(calls.CallKept, catalog.CatalogRevisionEvent{Kind: "candidate", Group: group, Branch: branch, Revision: revision})
|
||||
}
|
||||
return pipeline.Verbs{Hearer: h, Starter: s, View: view, Candidate: candidate, Acts: meshhandactsclient.Recorder{Ask: calls.Call}}
|
||||
return pipeline.Verbs{Hearer: h, Starter: s, View: view, Candidate: candidate, Acts: meshhandactsclient.Recorder{Ask: calls.Call},
|
||||
RevisionHeld: revisionHeld(calls.CallKept)}
|
||||
}
|
||||
|
||||
// revisionHeld reads the first page of a branch's catalog revision through the catalog seat's verb revision, never a kept
|
||||
// copy, so a rerun reopening an ended pipeline knows whether the catalog still keeps its revision (design 47 H2 H-A-479
|
||||
// addendum (2)): an answer of that revision is kept; none-yet, another revision, or a refusal saying the catalog's
|
||||
// "no such object" is gone (pipeline.ErrRevisionGone); any other failure is said as it is.
|
||||
func revisionHeld(call func(key string, body any) (json.RawMessage, error)) func(string, int64) error {
|
||||
return func(branch string, revision int64) error {
|
||||
raw, err := call(meshcatalogclient.RevisionVerb, catalog.RevisionInput{Branch: keys.EscapeName(branch), Revision: revision})
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "no such object") {
|
||||
return fmt.Errorf("%w: %v", pipeline.ErrRevisionGone, err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
answer, said, ok := buscalls.Unwrap(raw)
|
||||
if !ok {
|
||||
if strings.Contains(said, "no such object") {
|
||||
return fmt.Errorf("%w: %s", pipeline.ErrRevisionGone, said)
|
||||
}
|
||||
return fmt.Errorf("%s answered: %s", meshcatalogclient.RevisionVerb, said)
|
||||
}
|
||||
var a catalog.RevisionAnswer
|
||||
if err := json.Unmarshal(answer, &a); err != nil {
|
||||
return fmt.Errorf("%s answered no revision: %w", meshcatalogclient.RevisionVerb, err)
|
||||
}
|
||||
if a.NoneYet || a.Revision == nil || a.Revision.Revision != revision {
|
||||
return fmt.Errorf("%w: %s answered no revision %d of %s", pipeline.ErrRevisionGone, meshcatalogclient.RevisionVerb, revision, branch)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
@@ -38,8 +38,9 @@ func widenerOf(call func(key string, body any) (json.RawMessage, error), bound t
|
||||
const (
|
||||
envWidensReadBound = "MESH_RUNNER_WIDENS_READ_BOUND_FILE"
|
||||
envRevisionsKept = "MESH_RUNNER_REVISIONS_KEPT_FILE"
|
||||
// defaultReadBound is how long one read of widens waits for its answer: a few seconds, never the call's 30.
|
||||
defaultReadBound = 5 * time.Second
|
||||
// defaultReadBound is how long one read of widens waits for its answer: 30 s, so a cold grants-for (one that composes a
|
||||
// revision's grants the grants part did not keep yet) is not cut (design 47 H2 H-A-478).
|
||||
defaultReadBound = 30 * time.Second
|
||||
// defaultRevisionsKept is how many catalog revisions the runner keeps once read; a revision of the catalog graph is some 1.4 MB.
|
||||
defaultRevisionsKept = 6
|
||||
)
|
||||
|
||||
@@ -115,7 +115,7 @@ func TestWidensSettings(t *testing.T) {
|
||||
kept int
|
||||
ok bool
|
||||
}{
|
||||
{"defaults", nil, 5 * time.Second, 6, true},
|
||||
{"defaults", nil, 30 * time.Second, 6, true},
|
||||
{"set", map[string]string{envWidensReadBound: "2s\n", envRevisionsKept: "3\n"}, 2 * time.Second, 3, true},
|
||||
{"no duration", map[string]string{envWidensReadBound: "5"}, 0, 0, false},
|
||||
{"no number", map[string]string{envRevisionsKept: "0"}, 0, 0, false},
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"sync"
|
||||
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/pipelinekey"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
)
|
||||
|
||||
@@ -28,23 +29,20 @@ func StepsKey(pipeline, mover string) string {
|
||||
// stepsPrefix is the start of every mover key of one pipeline.
|
||||
func stepsPrefix(pipeline string) string { return StepsKey(pipeline, "") }
|
||||
|
||||
// IsStepsKey says whether a key of the state pipelines is a mover's steps rather than a pipeline's record: `steps.` and a
|
||||
// pipeline key of a take-in's (4 parts), a run's (5 parts, its third `run`), an unfinished mover's own (6 parts, its fourth
|
||||
// `unfinished`, H-A-398) or a publish's (7 parts) and one mover.
|
||||
// IsStepsKey says whether a key of the state pipelines is a mover's steps rather than a pipeline's record: `steps.`, a pipeline's
|
||||
// key of any form the seat defines (a take-in's, a run's, a publish's, an unfinished mover's own or one started again at the head,
|
||||
// H-A-398, H-A-479), read by the one parse of the seat's key forms (design 47 H2 H-A-485), and one mover.
|
||||
func IsStepsKey(key string) bool {
|
||||
parts := strings.Split(key, ".")
|
||||
if parts[0] != "steps" {
|
||||
rest, ok := strings.CutPrefix(key, "steps.")
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
switch len(parts) {
|
||||
case 4, 7:
|
||||
return true
|
||||
case 5:
|
||||
return parts[2] == runToken
|
||||
case 6:
|
||||
return parts[3] == unfinishedToken
|
||||
i := strings.LastIndex(rest, ".")
|
||||
if i <= 0 || i == len(rest)-1 {
|
||||
return false
|
||||
}
|
||||
return false
|
||||
_, err := pipelinekey.Parse(rest[:i])
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// RawState is the state pipelines as bytes: a key's value and revision (nil, 0 and no error when it has none), a create by
|
||||
|
||||
@@ -25,7 +25,7 @@ func (h *Hearer) Released(ev catalog.VersionReleasedEvent, branch string) (int,
|
||||
// A pipeline is keyed by its revision (H-A-213 (4)), so the pipelines of a source are those naming it, of either key
|
||||
// form a source's pipelines were kept under.
|
||||
var pkeys []string
|
||||
for _, f := range []string{b + ".*", b + ".*.published.*.*", b + ".*." + unfinishedToken + ".*"} {
|
||||
for _, f := range []string{b + ".*", b + ".*.published.*.*", b + ".*." + unfinishedToken + ".*", b + ".*." + againToken + ".*"} {
|
||||
k, err := h.Store.Keys(f)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("the pipelines of source %d are not listed: %w", ev.Source, err)
|
||||
|
||||
@@ -69,8 +69,19 @@ func keyRevision(b Begin) int64 {
|
||||
}
|
||||
|
||||
// keyForms are the filters of every key form of the state pipelines: a revision's, a run's, a publish's kept from before
|
||||
// a release revision started one pipeline (H-A-213 (2)), and an unfinished mover's pipeline of its own (H-A-398).
|
||||
var keyForms = []string{"*.*", "*." + runToken + ".*", "*.*.published.*.*", "*.*." + unfinishedToken + ".*"}
|
||||
// a release revision started one pipeline (H-A-213 (2)), an unfinished mover's pipeline of its own (H-A-398), and a pipeline
|
||||
// started again at the head (H-A-479).
|
||||
var keyForms = []string{"*.*", "*." + runToken + ".*", "*.*.published.*.*", "*.*." + unfinishedToken + ".*", "*.*." + againToken + ".*"}
|
||||
|
||||
// againToken is the third part of the key of a pipeline started again at the head by start with modules,
|
||||
// `<branch>.<revision>.again.<n>` (design 47 H2 H-A-479 addendum (1)).
|
||||
const againToken = "again"
|
||||
|
||||
// AgainPipelineKey is the key of the n-th pipeline started again at revision of branch, `<branch>.<revision>.again.<n>`
|
||||
// (H-A-479 addendum (1)).
|
||||
func AgainPipelineKey(branch string, revision int64, n int) string {
|
||||
return seat.PipelinesByBranchRevisionAgainNKey(keys.EscapeName(branch), strconv.FormatInt(revision, 10), strconv.Itoa(n))
|
||||
}
|
||||
|
||||
// AllKeys are every pipeline's key, of each key form (§746, §747; H-A-83).
|
||||
func AllKeys(store interface {
|
||||
|
||||
@@ -655,6 +655,50 @@ func TestNoWriteOfATestRunActsOnAReadBeforeItsLatestChange(t *testing.T) {
|
||||
t.Error("the step's finding was written before its run's end")
|
||||
}
|
||||
})
|
||||
// H-A-479 addendum (2): a test step's rerun reopening an ended pipeline says its pipeline-started before its run is
|
||||
// started again, so its revision is pinned before the step is asked.
|
||||
t.Run("a test step's rerun reopening an ended pipeline pins its revision first", func(t *testing.T) {
|
||||
f := newRunFixture()
|
||||
p := f.start(t, 4, 9)
|
||||
unit := stepNamed(p, "app:program", "unit")
|
||||
run := unit.Shared[0].Run
|
||||
ended := time.Now()
|
||||
if _, err := f.h.OnStepEnded(stepSeatOf(t, "node-build-go"), "mesh.seat.node-build-go.event.step-ended.ace", StepEnded{ID: unit.Ask,
|
||||
Pipeline: p.Pipeline, Module: "app", Commit: "c1", Fingerprint: "src2:aa", Step: "unit", Node: "ace", Outcome: "failed", Ended: &ended}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
q, rev, _ := f.store.Get(p.Pipeline)
|
||||
q.Outcome, q.Ended = seat.PipelineOutcomeFailed, &ended
|
||||
if _, err := f.store.Update(p.Pipeline, q, rev); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before, _, _ := f.runs.Get(run)
|
||||
var started []string
|
||||
f.s.Say = func(_ string, body any) error {
|
||||
if ps, ok := body.(seat.PipelineStarted); ok {
|
||||
if r, _, _ := f.runs.Get(run); r.Ask != before.Ask || r.Ended == nil {
|
||||
t.Errorf("pipeline-started said after the run was started again (ask %s, was %s)", r.Ask, before.Ask)
|
||||
}
|
||||
started = append(started, ps.Pipeline)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
v := Verbs{Hearer: f.h, Starter: f.s, View: func(string) (Half, bool) {
|
||||
return Half{Entries: map[string]catalog.CatalogEntry{
|
||||
"app": {Module: "app", Path: "modules/app", Artifacts: []catalog.Artifact{{Ref: "app:program", Language: "go"}, {Ref: "app:tools", Language: "go"}}},
|
||||
"mesh-build-go": {Module: "mesh-build-go", Acts: []catalog.ActingRule{{Seat: "node-build-go", Step: "build", On: "go"}}}}}, true
|
||||
}, Acts: &actLog{}, RevisionHeld: func(string, int64) error { return nil }}
|
||||
a, err := v.Rerun("op@shanks", seat.RerunInput{Pipeline: p.Pipeline, Step: "unit", Artifact: "app:program", Why: "fixed"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(started) != 1 || started[0] != p.Pipeline {
|
||||
t.Errorf("pipeline-started at the reopening = %v, want %s once", started, p.Pipeline)
|
||||
}
|
||||
if r, _, _ := f.runs.Get(run); r.Ask != a.Result.Ask || r.Ended != nil {
|
||||
t.Errorf("the rerun's run = %+v; want the new ask running", r)
|
||||
}
|
||||
})
|
||||
// A rerun whose read of the key is changed before its write (a step joining the ended run) writes from the latest read,
|
||||
// carrying that reader, never answering the change as an error nor putting back the earlier readers.
|
||||
t.Run("a rerun writes its run from the latest read", func(t *testing.T) {
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -29,8 +30,15 @@ type Verbs struct {
|
||||
Acts views.HandActs
|
||||
// Runs answers the read verb test-runs, answered by the process that holds test-runs-cap-mib (A-470i).
|
||||
Runs RunsRead
|
||||
// RevisionHeld says whether a branch's catalog revision is still kept, read through the catalog seat's verb revision
|
||||
// at the time of the ask: nil when it is, an error wrapping ErrRevisionGone when it is removed (design 47 H2 H-A-479
|
||||
// addendum (2)).
|
||||
RevisionHeld func(branch string, revision int64) error
|
||||
}
|
||||
|
||||
// ErrRevisionGone is a catalog revision the catalog no longer keeps (H-A-479 addendum (2)).
|
||||
var ErrRevisionGone = errors.New("the catalog revision is gone")
|
||||
|
||||
func (v Verbs) now() time.Time { return v.Hearer.now() }
|
||||
|
||||
func (v Verbs) record(verb, what, why string) error {
|
||||
@@ -275,13 +283,89 @@ func (v Verbs) EndAgain(_ string, in seat.EndAgainInput) (seat.EndAgainAnswer, e
|
||||
// the ask saying it is a rerun and the caller the bus proved; a failed pipeline runs again and ends once more with its end
|
||||
// count one higher (§325), an approval never asked waiting again. A review waits again; an approval is asked again by
|
||||
// `ask-again`.
|
||||
//
|
||||
// A rerun of a step in an ended delivering pipeline reopens it: its revision is read first and the rerun refused by name,
|
||||
// nothing asked, when the catalog removed it; once reopened its pipeline-started is said again, so the indexer pins its
|
||||
// revision again until its next end (design 47 H2 H-A-479 addendum (2)).
|
||||
func (v Verbs) Rerun(caller string, in seat.RerunInput) (seat.RerunAnswer, error) {
|
||||
if in.Step == seat.StepResultStepApproval {
|
||||
return seat.RerunAnswer{}, errors.New("an approval is asked again by ask-again, not rerun")
|
||||
}
|
||||
reopens, err := v.reopening(in.Pipeline)
|
||||
if err != nil {
|
||||
return seat.RerunAnswer{}, err
|
||||
}
|
||||
return v.rerun(caller, in, reopens)
|
||||
}
|
||||
|
||||
// sayReopened says pipeline-started again for p, written reopened by a rerun, before its step is asked again, so the
|
||||
// indexer pins its revision again before anything reads it (H-A-479 addendum (2)).
|
||||
func (v Verbs) sayReopened(p seat.Pipeline, reopens bool) {
|
||||
if !reopens || v.Starter == nil || p.Ended != nil {
|
||||
return
|
||||
}
|
||||
v.Starter.sayStartedPipeline(p, v.now())
|
||||
v.Starter.logf("%s reopened by a rerun: its pipeline-started is said again, so its revision %d is pinned again (H-A-479)", p.Pipeline, p.Revision)
|
||||
}
|
||||
|
||||
// reopening says whether a rerun of pipeline key reopens an ended delivering pipeline, refusing it when its revision is
|
||||
// gone (H-A-479 addendum (2)). A pipeline not read, a run's, or one the rerun refuses anyway is left to the rerun.
|
||||
func (v Verbs) reopening(key string) (bool, error) {
|
||||
p, _, err := v.Hearer.Store.Get(key)
|
||||
if err != nil || p.Ended == nil || p.Run != "" || p.Revision <= 0 ||
|
||||
p.Outcome == seat.PipelineOutcomeCancelled || p.Outcome == seat.PipelineOutcomeSuperseded {
|
||||
return false, nil
|
||||
}
|
||||
// A reopening never builds beside an open pipeline carrying one of its movers at the same revision (H-A-479 third
|
||||
// addendum).
|
||||
if open, err := v.openBeside(p); err != nil {
|
||||
return false, err
|
||||
} else if open != "" {
|
||||
return false, fmt.Errorf("the pipeline %s ended and is not reopened: its pipeline %s is running: nothing was asked", key, open)
|
||||
}
|
||||
if v.RevisionHeld != nil {
|
||||
if err := v.RevisionHeld(p.Branch, p.Revision); errors.Is(err, ErrRevisionGone) {
|
||||
return false, fmt.Errorf("the pipeline %s ended and its revision %d is gone; start the mover again at the head: nothing was asked (%v)", key, p.Revision, err)
|
||||
} else if err != nil {
|
||||
return false, fmt.Errorf("the pipeline %s ended and its revision %d is not read, so it is not reopened: nothing was asked: %w", key, p.Revision, err)
|
||||
}
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// openBeside is the key of another delivering pipeline of p's branch and revision, not ended, carrying one of p's movers it
|
||||
// did not leave; empty when none (H-A-479 third addendum).
|
||||
func (v Verbs) openBeside(p seat.Pipeline) (string, error) {
|
||||
pkeys, err := branchKeys(v.Hearer.Store, p.Branch)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("the pipelines of %s are not listed, so %s is not reopened: nothing was asked: %w", p.Branch, p.Pipeline, err)
|
||||
}
|
||||
for _, key := range pkeys {
|
||||
if key == p.Pipeline {
|
||||
continue
|
||||
}
|
||||
o, _, err := v.Hearer.Store.Get(key)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("the pipeline %s is not read, so %s is not reopened: nothing was asked: %w", key, p.Pipeline, err)
|
||||
}
|
||||
if o.Run != "" || o.Ended != nil || o.Revision != p.Revision ||
|
||||
(o.Outcome != seat.PipelineOutcomeRunning && o.Outcome != seat.PipelineOutcomeWaiting) {
|
||||
continue
|
||||
}
|
||||
for _, m := range p.Movers {
|
||||
if Moves(o, m.Module) && !slices.Contains(o.Superseded, m.Module) {
|
||||
return key, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
|
||||
// rerun asks one step again (Rerun).
|
||||
func (v Verbs) rerun(caller string, in seat.RerunInput, reopens bool) (seat.RerunAnswer, error) {
|
||||
if p, _, err := v.Hearer.Store.Get(in.Pipeline); err == nil {
|
||||
if i, err := stepOf(p, in.Module, in.Step, in.Artifact); err == nil && v.Hearer.TestRuns != nil && sharedRun(p.Steps[i]) != "" && BoundSeat(p.Steps[i].Seat) {
|
||||
return v.rerunTest(caller, in, p.Steps[i])
|
||||
return v.rerunTest(caller, in, p.Steps[i], reopens)
|
||||
}
|
||||
}
|
||||
var ask *Asked
|
||||
@@ -332,6 +416,7 @@ func (v Verbs) Rerun(caller string, in seat.RerunInput) (seat.RerunAnswer, error
|
||||
if err != nil {
|
||||
return seat.RerunAnswer{}, err
|
||||
}
|
||||
v.sayReopened(p, reopens)
|
||||
if ask != nil {
|
||||
v.Starter.carry(in.Pipeline, []Asked{*ask})
|
||||
if err := askAll(v.Starter.Steps, in.Pipeline, []Asked{*ask}, func(commit string) bool { return v.Starter.recovered(in.Pipeline, commit) }, v.Starter.Asks.arm, v.Starter.logf); err != nil {
|
||||
@@ -349,7 +434,7 @@ func (v Verbs) Rerun(caller string, in seat.RerunInput) (seat.RerunAnswer, error
|
||||
// rerunTest asks a test step of a test run again (design 47 G §727, A-469l): it joins its key's run while that runs, reads one
|
||||
// that started after the step's last end, or else starts a new run under a new ask, replacing the ended key by compare-and-set
|
||||
// and carrying its readers into the new value, so the new run's end moves every former reader that did not pass.
|
||||
func (v Verbs) rerunTest(caller string, in seat.RerunInput, st seat.StepResult) (seat.RerunAnswer, error) {
|
||||
func (v Verbs) rerunTest(caller string, in seat.RerunInput, st seat.StepResult, reopens bool) (seat.RerunAnswer, error) {
|
||||
h := v.Hearer
|
||||
run := sharedRun(st)
|
||||
if open(st) {
|
||||
@@ -430,6 +515,7 @@ func (v Verbs) rerunTest(caller string, in seat.RerunInput, st seat.StepResult)
|
||||
if err != nil {
|
||||
return seat.RerunAnswer{}, err
|
||||
}
|
||||
v.sayReopened(p, reopens)
|
||||
switch {
|
||||
case join || read:
|
||||
v.Starter.joinRun(run, reader)
|
||||
@@ -649,10 +735,17 @@ func (v Verbs) Run(caller string, in seat.RunInput) (seat.RunAnswer, error) {
|
||||
|
||||
// Start starts by hand a delivering pipeline for the head of a watched branch the runner's catalog half holds, recorded in
|
||||
// the hand-act log first (§306, §394): the pipeline of that branch's source revision and the module, as a take-in would
|
||||
// start it. A head whose pipeline is written already starts nothing and is refused, naming the pipeline.
|
||||
// start it. A head whose pipeline is written already starts nothing and is refused, naming the pipeline. With modules it
|
||||
// starts the named modules again at the head instead (startAgain, H-A-479); exactly one of module or modules is given.
|
||||
func (v Verbs) Start(_ string, in seat.StartInput) (seat.StartAnswer, error) {
|
||||
if in.Module != "" && len(in.Modules) > 0 {
|
||||
return seat.StartAnswer{}, errors.New("start names the module or the modules, not both: nothing was started")
|
||||
}
|
||||
if in.Branch != "" && len(in.Modules) > 0 {
|
||||
return v.startAgain(in)
|
||||
}
|
||||
if in.Branch == "" || in.Module == "" {
|
||||
return seat.StartAnswer{}, errors.New("start names the watched branch and the module")
|
||||
return seat.StartAnswer{}, errors.New("start names the watched branch and the module or the modules")
|
||||
}
|
||||
if v.Starter == nil || v.View == nil {
|
||||
return seat.StartAnswer{}, errors.New("the runner cannot start a pipeline: it holds no starter or no catalog half")
|
||||
@@ -681,6 +774,117 @@ func (v Verbs) Start(_ string, in seat.StartInput) (seat.StartAnswer, error) {
|
||||
return seat.StartAnswer{Pipeline: key}, nil
|
||||
}
|
||||
|
||||
// startAgain starts by hand, recorded in the hand-act log first, a pipeline at the head of a watched branch carrying the
|
||||
// named modules whose newest entry, the head's, has no passed pipeline (a failed, cut or superseded one), with the head's
|
||||
// revision as its candidate and the take-in's tiers (design 47 H2 H-A-479): keyed `<branch>.<revision>.again.<n>`, n from 1,
|
||||
// taken by writing on absence (addendum (1)). Its pipeline-started pins the head's revision as a take-in's does, and its
|
||||
// versions are reserved at its build steps as a take-in's are. A module the head holds no entry of, or whose newest entry
|
||||
// passed in a pipeline, is refused by name, nothing started.
|
||||
func (v Verbs) startAgain(in seat.StartInput) (seat.StartAnswer, error) {
|
||||
if v.Starter == nil || v.View == nil || v.Hearer == nil || v.Hearer.Store == nil {
|
||||
return seat.StartAnswer{}, errors.New("the runner cannot start a pipeline again: it holds no starter, no catalog half or no state pipelines")
|
||||
}
|
||||
half, ok := v.View(in.Branch)
|
||||
if !ok || half.Source <= 0 {
|
||||
return seat.StartAnswer{}, fmt.Errorf("the catalog half of %s is not held at a source revision yet: nothing was started", in.Branch)
|
||||
}
|
||||
movers := make([]seat.Mover, 0, len(in.Modules))
|
||||
var absent []string
|
||||
for _, m := range in.Modules {
|
||||
e, ok := half.Entries[m]
|
||||
if !ok {
|
||||
absent = append(absent, m)
|
||||
continue
|
||||
}
|
||||
if slices.ContainsFunc(movers, func(x seat.Mover) bool { return x.Module == m }) {
|
||||
continue
|
||||
}
|
||||
movers = append(movers, seat.Mover{Module: m, Commit: e.Commit, Fingerprint: e.Fp, Repository: e.Repository})
|
||||
}
|
||||
if len(absent) > 0 {
|
||||
return seat.StartAnswer{}, fmt.Errorf("the head of %s (revision %d) holds no entry of %s: nothing was started", in.Branch, half.Revision, strings.Join(absent, ", "))
|
||||
}
|
||||
passed, running, err := v.heldEntries(in.Branch, movers)
|
||||
if err != nil {
|
||||
return seat.StartAnswer{}, err
|
||||
}
|
||||
if len(passed) > 0 {
|
||||
return seat.StartAnswer{}, fmt.Errorf("the newest entry of %s passed already: nothing was started", strings.Join(passed, ", "))
|
||||
}
|
||||
if len(running) > 0 {
|
||||
return seat.StartAnswer{}, fmt.Errorf("the newest entry of %s: nothing was started", strings.Join(running, ", "))
|
||||
}
|
||||
movers = Tiers(movers, half.Entries, v.Starter.logf)
|
||||
names := make([]string, 0, len(movers))
|
||||
for _, m := range movers {
|
||||
names = append(names, m.Module)
|
||||
}
|
||||
prefix := AgainPipelineKey(in.Branch, half.Revision, 0)
|
||||
prefix = prefix[:len(prefix)-1]
|
||||
held, err := v.Hearer.Store.Keys(prefix + "*")
|
||||
if err != nil {
|
||||
return seat.StartAnswer{}, fmt.Errorf("the pipelines started again at revision %d of %s are not listed: nothing was started: %w", half.Revision, in.Branch, err)
|
||||
}
|
||||
n := 1
|
||||
for _, k := range held {
|
||||
if i, err := strconv.Atoi(strings.TrimPrefix(k, prefix)); err == nil && i >= n {
|
||||
n = i + 1
|
||||
}
|
||||
}
|
||||
if err := v.record(seat.VerbStart, fmt.Sprintf("started %s again at the head of %s by hand", strings.Join(names, ", "), in.Branch), in.Why); err != nil {
|
||||
return seat.StartAnswer{}, err
|
||||
}
|
||||
// The next n is taken without a bound: each refusal means the key exists (H-A-479 second addendum).
|
||||
for ; ; n++ {
|
||||
key := AgainPipelineKey(in.Branch, half.Revision, n)
|
||||
started, err := v.Starter.Start(Begin{Branch: in.Branch, Source: half.Source, Revision: half.Revision, Entries: half.Entries,
|
||||
Key: key, Movers: movers})
|
||||
if err != nil {
|
||||
return seat.StartAnswer{}, err
|
||||
}
|
||||
if started {
|
||||
v.Starter.logf("%s starts %s again at the head by hand (H-A-479)", key, strings.Join(names, ", "))
|
||||
return seat.StartAnswer{Pipeline: key}, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// heldEntries are the movers whose module, at the mover's fingerprint, a delivering pipeline of branch carries and did not
|
||||
// leave: passed, those a passed pipeline holds (a newest entry that passed is not started again, H-A-479); running, those a
|
||||
// pipeline not ended holds, each said "<module>: its pipeline <key> is running", so two pipelines never build one entry side
|
||||
// by side (H-A-479 second addendum).
|
||||
func (v Verbs) heldEntries(branch string, movers []seat.Mover) (passed, running []string, err error) {
|
||||
pkeys, err := branchKeys(v.Hearer.Store, branch)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("the pipelines of %s are not listed, so nothing was started: %w", branch, err)
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, key := range pkeys {
|
||||
p, _, err := v.Hearer.Store.Get(key)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("the pipeline %s is not read, so nothing was started: %w", key, err)
|
||||
}
|
||||
open := p.Ended == nil && (p.Outcome == seat.PipelineOutcomeRunning || p.Outcome == seat.PipelineOutcomeWaiting)
|
||||
if p.Run != "" || (p.Outcome != seat.PipelineOutcomePassed && !open) {
|
||||
continue
|
||||
}
|
||||
for _, m := range movers {
|
||||
if seen[m.Module] || slices.Contains(p.Superseded, m.Module) {
|
||||
continue
|
||||
}
|
||||
if pm, ok := MoverOf(p, m.Module); ok && pm.Fingerprint == m.Fingerprint {
|
||||
seen[m.Module] = true
|
||||
if open {
|
||||
running = append(running, m.Module+": its pipeline "+key+" is running")
|
||||
} else {
|
||||
passed = append(passed, m.Module+" (in "+key+")")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return passed, running, nil
|
||||
}
|
||||
|
||||
// AskAgain asks a declined approval step once more (§306, §36), recorded in the hand-act log first: the step waits again,
|
||||
// marked asked again by the caller the bus proved, and is asked at once when the pipeline's every other required step
|
||||
// passed, under an ask of its own (the earlier ask's decision ended it).
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -423,3 +425,180 @@ func TestAValidateRerunMovesEveryMoversValidateAndReopensItsCompositions(t *test
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// H-A-479: start with modules starts a pipeline at the head keyed <branch>.<revision>.again.<n> carrying the named modules
|
||||
// whose newest entry has no passed pipeline, at the take-in's tiers; a module whose newest entry passed is refused by name,
|
||||
// nothing started; exactly one of module or modules is given; a second start takes the next n.
|
||||
func TestStartWithModulesStartsAFailedMoverAgainAtTheHead(t *testing.T) {
|
||||
v, store, _, _, acts, _ := verbsFixture()
|
||||
made := created{}
|
||||
var started []seat.PipelineStarted
|
||||
v.Starter.Pipelines = made
|
||||
v.Starter.Say = func(_ string, body any) error {
|
||||
if ps, ok := body.(seat.PipelineStarted); ok {
|
||||
started = append(started, ps)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
_, entries := placeFixture()
|
||||
app := entries["app"]
|
||||
app.Fp, app.Commit, app.Repository = "src2:aa", "c1", "o/app"
|
||||
entries["app"] = app
|
||||
entries["lib"] = catalog.CatalogEntry{Module: "lib", Fp: "src2:bb", Commit: "c2", Repository: "o/lib"}
|
||||
v.View = func(string) (Half, bool) { return Half{Source: 7, Revision: 9, Entries: entries}, true }
|
||||
// The fixture's main.5 moves app at src2:aa and failed; lib's newest entry passed in main.8.
|
||||
e := store[pkey]
|
||||
e.p.Outcome = seat.PipelineOutcomeFailed
|
||||
store[pkey] = e
|
||||
store["main.8"] = struct {
|
||||
p seat.Pipeline
|
||||
rev uint64
|
||||
}{seat.Pipeline{Pipeline: "main.8", Branch: "main", Source: 6, Revision: 8, Outcome: seat.PipelineOutcomePassed,
|
||||
Movers: []seat.Mover{{Module: "lib", Fingerprint: "src2:bb"}, {Module: "app", Fingerprint: "src2:aa"}}, Superseded: []string{"app"}}, 1}
|
||||
|
||||
if _, err := v.Start("operator", seat.StartInput{Branch: "main", Module: "app", Modules: []string{"app"}, Why: "x"}); err == nil {
|
||||
t.Error("a start naming both module and modules was taken")
|
||||
}
|
||||
// A pipeline still running at app's head entry refuses the start: two pipelines never build one entry side by side.
|
||||
store["main.7"] = struct {
|
||||
p seat.Pipeline
|
||||
rev uint64
|
||||
}{seat.Pipeline{Pipeline: "main.7", Branch: "main", Source: 6, Revision: 7, Outcome: seat.PipelineOutcomeRunning,
|
||||
Movers: []seat.Mover{{Module: "app", Fingerprint: "src2:aa"}}}, 1}
|
||||
if _, err := v.Start("operator", seat.StartInput{Branch: "main", Modules: []string{"app"}, Why: "x"}); err == nil || !strings.Contains(err.Error(), "its pipeline main.7 is running") {
|
||||
t.Errorf("a start of app while main.7 runs it = %v; want refused naming main.7", err)
|
||||
}
|
||||
delete(store, "main.7")
|
||||
if _, err := v.Start("operator", seat.StartInput{Branch: "main", Modules: []string{"app", "lib"}, Why: "x"}); err == nil || !strings.Contains(err.Error(), "lib") {
|
||||
t.Errorf("a start naming lib, whose newest entry passed, = %v; want refused naming lib", err)
|
||||
}
|
||||
if len(made) != 0 {
|
||||
t.Fatalf("a refused start wrote %d pipelines", len(made))
|
||||
}
|
||||
a, err := v.Start("operator", seat.StartInput{Branch: "main", Modules: []string{"app"}, Why: "its pipeline failed at an unchanged entry"})
|
||||
if err != nil || a.Pipeline != "main.9.again.1" || made["main.9.again.1"] == nil {
|
||||
t.Fatalf("start with modules = %+v, %v; written %v", a, err, len(made))
|
||||
}
|
||||
var p seat.Pipeline
|
||||
if err := json.Unmarshal(made["main.9.again.1"], &p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if p.Revision != 9 || p.Source != 7 || len(p.Movers) != 1 || p.Movers[0].Module != "app" || p.Movers[0].Fingerprint != "src2:aa" || p.Movers[0].Tier != 1 {
|
||||
t.Errorf("the pipeline = %+v, want app at src2:aa, tier 1, at revision 9 of source 7", p)
|
||||
}
|
||||
if len(started) != 1 || started[0].Pipeline != "main.9.again.1" || started[0].Revision != 9 {
|
||||
t.Errorf("pipeline-started = %+v, want main.9.again.1 at revision 9 (its revision pinned)", started)
|
||||
}
|
||||
// The next start at the same head, once main.9.again.1 failed, takes the next n.
|
||||
at := time.Unix(3, 0)
|
||||
p.Outcome, p.Ended = seat.PipelineOutcomeFailed, &at
|
||||
store["main.9.again.1"] = struct {
|
||||
p seat.Pipeline
|
||||
rev uint64
|
||||
}{p, 1}
|
||||
if a, err := v.Start("operator", seat.StartInput{Branch: "main", Modules: []string{"app"}, Why: "again"}); err != nil || a.Pipeline != "main.9.again.2" {
|
||||
t.Errorf("a second start with modules = %+v, %v; want main.9.again.2", a, err)
|
||||
}
|
||||
if len(*acts) != 2 {
|
||||
t.Errorf("hand acts %v, want the two starts recorded", *acts)
|
||||
}
|
||||
// main.9.again.1 passed: its version is written into it as into any pipeline of its source, and a further start of app is
|
||||
// refused naming it, read through the branch's key forms.
|
||||
p.Outcome = seat.PipelineOutcomePassed
|
||||
store["main.9.again.1"] = struct {
|
||||
p seat.Pipeline
|
||||
rev uint64
|
||||
}{p, 1}
|
||||
if wrote, err := v.Hearer.Released(catalog.VersionReleasedEvent{Module: "app", Version: "0.1.2", Source: 7}, "main"); err != nil || wrote != 1 ||
|
||||
store["main.9.again.1"].p.Movers[0].Version != "0.1.2" {
|
||||
t.Errorf("the release of app 0.1.2 wrote %d (%v), the again pipeline's mover %+v; want its version written", wrote, err, store["main.9.again.1"].p.Movers)
|
||||
}
|
||||
if _, err := v.Start("operator", seat.StartInput{Branch: "main", Modules: []string{"app"}, Why: "x"}); err == nil || !strings.Contains(err.Error(), "main.9.again.1") {
|
||||
t.Errorf("a start of app after main.9.again.1 passed = %v; want refused naming it", err)
|
||||
}
|
||||
if !IsStepsKey(StepsKey("main.9.again.1", "app")) {
|
||||
t.Error("a mover key of an again pipeline is not read as a mover's steps")
|
||||
}
|
||||
// H-A-485: the mover keys are read by the one parse of the seat's key forms; a pipeline key of no form (its n or its
|
||||
// revision no number) is no mover's steps.
|
||||
for _, k := range []string{"steps.main.9.again.x.app", "steps.main.r9.app", "steps.main.9.again.0.app"} {
|
||||
if IsStepsKey(k) {
|
||||
t.Errorf("%s, of no pipeline key form, is read as a mover's steps", k)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// H-A-479 addendum (2): a rerun of a step in an ended pipeline pins its revision again at the reopening (its pipeline-started
|
||||
// said again), and is refused by name, nothing asked, when the revision is already removed.
|
||||
func TestARerunOfAnEndedPipelinePinsItsRevisionAgainOrIsRefusedWhenItIsGone(t *testing.T) {
|
||||
v, store, _, asks, _, _ := verbsFixture()
|
||||
var started []seat.PipelineStarted
|
||||
v.Starter.Say = func(_ string, body any) error {
|
||||
if ps, ok := body.(seat.PipelineStarted); ok {
|
||||
if len(asks.asks) != 0 {
|
||||
t.Errorf("pipeline-started said after the step was asked again (%d asks): its revision is pinned too late", len(asks.asks))
|
||||
}
|
||||
started = append(started, ps)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
e := store[pkey]
|
||||
at := time.Unix(2, 0)
|
||||
e.p.Revision, e.p.Outcome, e.p.Ended = 5, seat.PipelineOutcomeFailed, &at
|
||||
for i := range e.p.Steps {
|
||||
if e.p.Steps[i].Artifact == "app:program" && e.p.Steps[i].Step == "lint" {
|
||||
e.p.Steps[i].Outcome = seat.StepResultOutcomeFailed
|
||||
}
|
||||
}
|
||||
store[pkey] = e
|
||||
gone := true
|
||||
v.RevisionHeld = func(branch string, revision int64) error {
|
||||
if gone {
|
||||
return fmt.Errorf("revision %s.%d: no such object: %w", branch, revision, ErrRevisionGone)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
in := seat.RerunInput{Pipeline: pkey, Step: "lint", Artifact: "app:program", Why: "retry"}
|
||||
// H-A-479 third addendum: another open pipeline carrying app at the same revision refuses the reopening.
|
||||
store["main.5.again.1"] = struct {
|
||||
p seat.Pipeline
|
||||
rev uint64
|
||||
}{seat.Pipeline{Pipeline: "main.5.again.1", Branch: "main", Source: 5, Revision: 5, Outcome: seat.PipelineOutcomeRunning,
|
||||
Movers: []seat.Mover{{Module: "app", Fingerprint: "src2:aa"}}}, 1}
|
||||
gone = false
|
||||
if _, err := v.Rerun("operator", in); err == nil || !strings.Contains(err.Error(), "its pipeline main.5.again.1 is running") {
|
||||
t.Fatalf("a reopening beside main.5.again.1 = %v; want refused naming it", err)
|
||||
}
|
||||
if len(asks.asks) != 0 || store[pkey].p.Outcome != seat.PipelineOutcomeFailed {
|
||||
t.Fatalf("a refused reopening asked %d, left the pipeline %s", len(asks.asks), store[pkey].p.Outcome)
|
||||
}
|
||||
delete(store, "main.5.again.1")
|
||||
gone = true
|
||||
if _, err := v.Rerun("operator", in); err == nil || !strings.Contains(err.Error(), "its revision 5 is gone; start the mover again at the head") {
|
||||
t.Fatalf("a rerun of an ended pipeline whose revision is gone = %v; want refused by name", err)
|
||||
}
|
||||
if len(asks.asks) != 0 || store[pkey].p.Outcome != seat.PipelineOutcomeFailed || len(started) != 0 {
|
||||
t.Fatalf("a refused rerun asked %d, left the pipeline %s, said %d starts", len(asks.asks), store[pkey].p.Outcome, len(started))
|
||||
}
|
||||
gone = false
|
||||
if _, err := v.Rerun("operator", in); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(started) != 1 || started[0].Pipeline != pkey || started[0].Revision != 5 {
|
||||
t.Errorf("pipeline-started at the reopening = %+v, want %s at revision 5 (its revision pinned again)", started, pkey)
|
||||
}
|
||||
// A rerun of a running pipeline reopens nothing and says no start.
|
||||
e = store[pkey]
|
||||
for i := range e.p.Steps {
|
||||
if e.p.Steps[i].Artifact == "app:program" && e.p.Steps[i].Step == "unit" {
|
||||
e.p.Steps[i].Outcome = seat.StepResultOutcomeFailed
|
||||
}
|
||||
}
|
||||
store[pkey] = e
|
||||
if _, err := v.Rerun("operator", seat.RerunInput{Pipeline: pkey, Step: "unit", Artifact: "app:program", Why: "x"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(started) != 1 {
|
||||
t.Errorf("a rerun of a running pipeline said pipeline-started again: %+v", started)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -237,7 +237,8 @@ func runSettingsOf(raw []byte) bool {
|
||||
}
|
||||
if len(m.Config) != len(want)+len(words) || m.Config["test-runs-cap-mib"]["maximum"] != float64(64) || m.Config["test-runs-cap-mib"]["default"] != float64(16) ||
|
||||
m.Config["pipelines-at-once"]["minimum"] != float64(1) || m.Config["pipelines-at-once"]["maximum"] != float64(8) ||
|
||||
m.Config["pipelines-at-once"]["default"] != float64(1) || m.Config["pipelines-waiting-cap"]["default"] != float64(64) {
|
||||
m.Config["pipelines-at-once"]["default"] != float64(1) || m.Config["pipelines-waiting-cap"]["default"] != float64(64) ||
|
||||
m.Config["widens-read-bound"]["default"] != "30s" {
|
||||
return false
|
||||
}
|
||||
env := map[string]bool{}
|
||||
|
||||
@@ -138,9 +138,9 @@
|
||||
},
|
||||
"widens-read-bound": {
|
||||
"type": "string",
|
||||
"description": "how long one read of a pipeline end's widens (the catalog's revision, the seat holders, the grants part) waits for its answer before it is cut and said could not run, a Go duration such as 5s (design 47 G A-472, A-472a).",
|
||||
"description": "how long one read of a pipeline end's widens (the catalog's revision, the seat holders, the grants part) waits for its answer before it is cut and said could not run, a Go duration such as 30s (design 47 G A-472, A-472a, H2 H-A-478: long enough that a cold grants-for is not cut).",
|
||||
"trusted": false,
|
||||
"default": "5s"
|
||||
"default": "30s"
|
||||
},
|
||||
"revisions-kept": {
|
||||
"type": "number",
|
||||
|
||||
@@ -13,6 +13,7 @@ require (
|
||||
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/containerimageregistry v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/gopackageregistry v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildnode v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/npmpackageregistry v0.0.0-00010101000000-000000000000
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
// Package pipelinekey reads the key of a pipeline in the seat mesh-pipeline-runner's state pipelines, in every key form the
|
||||
// seat defines, through the seat's contract (design 47 H2 H-A-485): one parse, used by the runner, the assigner and the view, so
|
||||
// a pipeline started again at the head (`<branch>.<revision>.again.<n>`, H-A-479) or an unfinished mover's own pipeline
|
||||
// (`<branch>.<revision>.unfinished.<module>`, H-A-398) is read as a take-in's is. A key of no form the seat defines is refused
|
||||
// with an error, never read as an empty one.
|
||||
package pipelinekey
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
)
|
||||
|
||||
// The key forms of a pipeline in the state pipelines (the seat's key forms, design 47 H §746, H-A-213 (2), H-A-398, H-A-479).
|
||||
const (
|
||||
FormRevision = "<branch>.<revision>"
|
||||
FormRun = "<branch>.run.<run>"
|
||||
FormPublished = "<branch>.<source>.published.<package>.<version>"
|
||||
FormUnfinished = "<branch>.<revision>.unfinished.<module>"
|
||||
FormAgain = "<branch>.<revision>.again.<n>"
|
||||
)
|
||||
|
||||
// Key is a pipeline's key read: its form, its watched branch (unescaped), and the parts its form names: the revision (a
|
||||
// published form's source), the run, the package and version (escaped, as the key holds them), the unfinished mover's
|
||||
// module (unescaped) and the again form's n.
|
||||
type Key struct {
|
||||
Form string
|
||||
Branch string
|
||||
Revision int64
|
||||
Run string
|
||||
Package string
|
||||
Version string
|
||||
Module string
|
||||
N int
|
||||
}
|
||||
|
||||
// Parse reads key as a pipeline's key of one of the seat's forms; any other key is an error naming the forms.
|
||||
func Parse(key string) (Key, error) {
|
||||
parts := strings.Split(key, ".")
|
||||
var k Key
|
||||
var branch, revision, n, module string
|
||||
ok := false
|
||||
switch {
|
||||
case len(parts) == 2:
|
||||
branch, revision, ok = seat.ParsePipelinesByBranchRevisionKey(key)
|
||||
k.Form = FormRevision
|
||||
case len(parts) == 3 && parts[1] == "run":
|
||||
branch, k.Run, ok = seat.ParsePipelinesByBranchRunRunKey(key)
|
||||
k.Form = FormRun
|
||||
case len(parts) == 4 && parts[2] == "unfinished":
|
||||
branch, revision, module, ok = seat.ParsePipelinesByBranchRevisionUnfinishedModuleKey(key)
|
||||
k.Form = FormUnfinished
|
||||
case len(parts) == 4 && parts[2] == "again":
|
||||
branch, revision, n, ok = seat.ParsePipelinesByBranchRevisionAgainNKey(key)
|
||||
k.Form = FormAgain
|
||||
case len(parts) == 5 && parts[2] == "published":
|
||||
branch, revision, k.Package, k.Version, ok = seat.ParsePipelinesByBranchSourcePublishedPackageVersionKey(key)
|
||||
k.Form = FormPublished
|
||||
}
|
||||
if !ok {
|
||||
return Key{}, refused(key, "it is of no form the seat defines")
|
||||
}
|
||||
var err error
|
||||
if k.Branch, err = keys.UnescapeName(branch); err != nil || k.Branch == "" {
|
||||
return Key{}, refused(key, "its branch does not read")
|
||||
}
|
||||
if k.Form != FormRun {
|
||||
if k.Revision, err = strconv.ParseInt(revision, 10, 64); err != nil || k.Revision <= 0 {
|
||||
return Key{}, refused(key, "its revision is no number above 0")
|
||||
}
|
||||
}
|
||||
switch k.Form {
|
||||
case FormRun:
|
||||
if k.Run == "" {
|
||||
return Key{}, refused(key, "it names no run")
|
||||
}
|
||||
case FormUnfinished:
|
||||
if k.Module, err = keys.UnescapeName(module); err != nil || k.Module == "" {
|
||||
return Key{}, refused(key, "its module does not read")
|
||||
}
|
||||
case FormAgain:
|
||||
if k.N, err = strconv.Atoi(n); err != nil || k.N < 1 {
|
||||
return Key{}, refused(key, "its n is no number from 1")
|
||||
}
|
||||
case FormPublished:
|
||||
if k.Package == "" || k.Version == "" {
|
||||
return Key{}, refused(key, "it names no package or no version")
|
||||
}
|
||||
}
|
||||
return k, nil
|
||||
}
|
||||
|
||||
// refused is the error of a key Parse does not read.
|
||||
func refused(key, why string) error {
|
||||
return fmt.Errorf("%q is no key of a pipeline of the seat %s (%s): %s", key, seat.Seat,
|
||||
strings.Join([]string{FormRevision, FormRun, FormPublished, FormUnfinished, FormAgain}, ", "), why)
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
package pipelinekey
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// H-A-485: every key form of a pipeline the seat mesh-pipeline-runner defines is read, the again form and the unfinished form as
|
||||
// a take-in's, its branch and module unescaped; a key of no form, or whose parts do not read, is refused naming why.
|
||||
func TestEveryPipelineKeyFormIsReadAndAnyOtherRefused(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
key string
|
||||
want Key
|
||||
why string
|
||||
}{
|
||||
{key: "main.265", want: Key{Form: FormRevision, Branch: "main", Revision: 265}},
|
||||
{key: "main.265.again.2", want: Key{Form: FormAgain, Branch: "main", Revision: 265, N: 2}},
|
||||
{key: "feat=2fx.12.unfinished.mesh-tools", want: Key{Form: FormUnfinished, Branch: "feat/x", Revision: 12, Module: "mesh-tools"}},
|
||||
{key: "main.run.r7", want: Key{Form: FormRun, Branch: "main", Run: "r7"}},
|
||||
{key: "main.40.published.lib.0=2e1=2e2", want: Key{Form: FormPublished, Branch: "main", Revision: 40, Package: "lib", Version: "0=2e1=2e2"}},
|
||||
{key: "main", why: "no form"},
|
||||
{key: "main.265.again", why: "no form"},
|
||||
{key: "main.265.later.2", why: "no form"},
|
||||
{key: "main.265.again.0", why: "its n"},
|
||||
{key: "main.265.again.x", why: "its n"},
|
||||
{key: "main.r1", why: "its revision"},
|
||||
{key: "main.0", why: "its revision"},
|
||||
{key: "ma=zz.5", why: "its branch"},
|
||||
{key: "main.5.unfinished.=zz", why: "its module"},
|
||||
{key: "main.5.6.7.8.9", why: "no form"},
|
||||
} {
|
||||
got, err := Parse(c.key)
|
||||
if c.why != "" {
|
||||
if err == nil || !strings.Contains(err.Error(), c.why) || !strings.Contains(err.Error(), FormAgain) {
|
||||
t.Errorf("%s: read %+v (%v); want it refused, %s, naming the forms", c.key, got, err, c.why)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err != nil || got != c.want {
|
||||
t.Errorf("%s: read %+v (%v); want %+v", c.key, got, err, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -314,11 +314,14 @@ func recordHandActAs(cause string) func(what, why string) error {
|
||||
func serveTools() error {
|
||||
tv := &toolViews{}
|
||||
gate := newSeatGate()
|
||||
p := &grants.Part{Read: runtimeReader{views: tv}, Call: runtimeBus{}, Store: grants.BigValues(runtimeStore{}, nil), Emit: runtimeBus{}, SeatList: gate}
|
||||
// Its journal is its standard error, which the tool runner keeps: grants-for says each stage's time there (H-A-478).
|
||||
logf := func(f string, a ...any) { fmt.Fprintf(os.Stderr, "mesh-grants: "+f+"\n", a...) }
|
||||
p := &grants.Part{Read: runtimeReader{views: tv}, Call: runtimeBus{}, Store: grants.BigValues(runtimeStore{}, nil), Emit: runtimeBus{}, SeatList: gate,
|
||||
Logf: logf}
|
||||
// The reader asks through the runtime, which serves once Serve has its channel.
|
||||
go func() {
|
||||
<-stdio.Served()
|
||||
gate.Follow(seatlist.StdioBus(), nil, func(f string, a ...any) { fmt.Fprintf(os.Stderr, "mesh-grants: "+f+"\n", a...) })
|
||||
gate.Follow(seatlist.StdioBus(), nil, logf)
|
||||
}()
|
||||
tools := append(toolsOf(p), gate.Tools()...)
|
||||
for i := range tools {
|
||||
|
||||
@@ -5342,6 +5342,52 @@ func TestGrantsFor(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// grants-for composes each revision's grants once and keeps them with the revision (design 47 H2 H-A-478 and its addendum):
|
||||
// a second ask for the same revisions composes nothing again; an ask of another candidate against the same newest composes
|
||||
// the candidate alone; each stage's time is in the journal.
|
||||
func TestGrantsForComposesARevisionOnce(t *testing.T) {
|
||||
candidateX := strings.Replace(xManifest, `"state"`, `"emits":["hello"],"state"`, 1)
|
||||
b := fixture()
|
||||
b.revision = map[string]map[string]string{"newest": {"mod-x": xManifest, "mod-y": xManifest}, "candidate": {"mod-x": candidateX, "mod-y": xManifest}}
|
||||
b.revisionNumber = map[string]int64{"newest": 9, "candidate": 8}
|
||||
p := partOn(b)
|
||||
var journal []string
|
||||
p.Logf = func(f string, a ...any) { journal = append(journal, fmt.Sprintf(f, a...)) }
|
||||
ask := func(in meshgrants.GrantsForInput) (meshgrants.GrantsFor, string) {
|
||||
t.Helper()
|
||||
journal = nil
|
||||
got, err := p.GrantsFor(in)
|
||||
if err != nil || got.Error != "" {
|
||||
t.Fatalf("grants-for: %+v, %v", got, err)
|
||||
}
|
||||
if len(journal) != 1 || !strings.HasPrefix(journal[0], "grants-for main ") || !strings.Contains(journal[0], "the inputs ") {
|
||||
t.Fatalf("grants-for said %q, want one line of its stages' times", journal)
|
||||
}
|
||||
return got, journal[0]
|
||||
}
|
||||
first, said := ask(meshgrants.GrantsForInput{Branch: "main", Source: "7"})
|
||||
if len(first.Modules) != 1 || first.Modules[0].Module != "mod-x" || strings.Count(said, "(composed)") != 2 {
|
||||
t.Errorf("the first ask answered %+v and said %q, want mod-x and both revisions' grants composed", first, said)
|
||||
}
|
||||
again, said := ask(meshgrants.GrantsForInput{Branch: "main", Source: "7"})
|
||||
if strings.Contains(said, "composed") || strings.Count(said, "(kept)") != 2 || !reflect.DeepEqual(first, again) {
|
||||
t.Errorf("a second ask for the same revisions said %q and answered %+v, want both kept and %+v", said, again, first)
|
||||
}
|
||||
// Inputs that differ compose both again (H-A-478 second addendum); the same inputs then keep them again.
|
||||
b.put(grants.ReadResolved, "a.mod-x.greeting", map[string]any{"node": "a", "module": "mod-x", "key": "greeting", "value": "hello"})
|
||||
if _, said = ask(meshgrants.GrantsForInput{Branch: "main", Source: "7"}); strings.Count(said, "(composed)") != 2 {
|
||||
t.Errorf("an ask whose inputs differ said %q, want both revisions' grants composed again", said)
|
||||
}
|
||||
if _, said = ask(meshgrants.GrantsForInput{Branch: "main", Source: "7"}); strings.Count(said, "(kept)") != 2 {
|
||||
t.Errorf("an ask with the same inputs again said %q, want both kept", said)
|
||||
}
|
||||
// A run's candidate is another revision: composed once, the newest's grants kept.
|
||||
_, said = ask(meshgrants.GrantsForInput{Branch: "main", Run: "r1"})
|
||||
if !strings.Contains(said, "the newest's grants (kept)") || !strings.Contains(said, "the candidate's grants (composed)") {
|
||||
t.Errorf("an ask of another candidate said %q, want the newest's grants kept and the candidate's composed", said)
|
||||
}
|
||||
}
|
||||
|
||||
// grants-for keeps what H-A-195 says of the catalog: so many revisions to a branch as the setting
|
||||
// grants-for-revisions-per-branch says, a run's candidate counted within its branch's; only the followed branches', freed when
|
||||
// one leaves them; and an event below the newest held is the numbering started again, which drops what is kept.
|
||||
@@ -5372,8 +5418,11 @@ func TestGrantsForKeepsWhatItMay(t *testing.T) {
|
||||
perBranch int
|
||||
then func(t *testing.T, b *fakeBus, p *grants.Part, followed *[]string)
|
||||
pages int
|
||||
// composed is how many of the two revisions' grants the grants-for after it composes again: a revision no longer
|
||||
// kept has no grants kept either (H-A-478).
|
||||
composed int
|
||||
}{
|
||||
{name: "the setting bounds a branch's revisions: with 2, the candidate is dropped once two newer are kept", pages: 2,
|
||||
{name: "the setting bounds a branch's revisions: with 2, the candidate is dropped once two newer are kept", pages: 2, composed: 1,
|
||||
then: func(t *testing.T, b *fakeBus, p *grants.Part, _ *[]string) {
|
||||
p.RevisionsPerBranch = setting(t, "2\n")
|
||||
hear(t, p, 10, 0)
|
||||
@@ -5381,14 +5430,14 @@ func TestGrantsForKeepsWhatItMay(t *testing.T) {
|
||||
_, _ = p.GrantsFor(meshgrants.GrantsForInput{Branch: "main", Source: "7"})
|
||||
}},
|
||||
{name: "a run's candidate counts within its branch's bound: with 1, the candidate and the newest each drop the other", run: true,
|
||||
perBranch: 1, pages: 4, then: func(*testing.T, *fakeBus, *grants.Part, *[]string) {}},
|
||||
{name: "a branch that leaves the watched branches frees its revisions", pages: 4,
|
||||
perBranch: 1, pages: 4, composed: 2, then: func(*testing.T, *fakeBus, *grants.Part, *[]string) {}},
|
||||
{name: "a branch that leaves the watched branches frees its revisions", pages: 4, composed: 2,
|
||||
then: func(t *testing.T, _ *fakeBus, p *grants.Part, followed *[]string) {
|
||||
*followed = nil
|
||||
_, _ = p.GrantsFor(meshgrants.GrantsForInput{Branch: "main", Source: "7"})
|
||||
*followed = []string{"main"}
|
||||
}},
|
||||
{name: "an event below the newest held drops what is kept of the branch", pages: 4,
|
||||
{name: "an event below the newest held drops what is kept of the branch", pages: 4, composed: 2,
|
||||
then: func(t *testing.T, _ *fakeBus, p *grants.Part, _ *[]string) { hear(t, p, 3, 0) }},
|
||||
} {
|
||||
t.Run(row.name, func(t *testing.T) {
|
||||
@@ -5411,10 +5460,15 @@ func TestGrantsForKeepsWhatItMay(t *testing.T) {
|
||||
}
|
||||
row.then(t, b, p, &followed)
|
||||
before := len(slices.DeleteFunc(slices.Clone(b.calls), func(c string) bool { return c != grants.CallRevision }))
|
||||
var said []string
|
||||
p.Logf = func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }
|
||||
_, _ = p.GrantsFor(in)
|
||||
if got := len(slices.DeleteFunc(slices.Clone(b.calls), func(c string) bool { return c != grants.CallRevision })) - before; got != row.pages {
|
||||
t.Errorf("grants-for read %d revision pages, want %d", got, row.pages)
|
||||
}
|
||||
if len(said) != 1 || strings.Count(said[0], "(composed)") != row.composed {
|
||||
t.Errorf("grants-for said %q, want %d of the revisions' grants composed again", said, row.composed)
|
||||
}
|
||||
})
|
||||
}
|
||||
if n, err := grants.RevisionsPerBranchSetting(filepath.Join(t.TempDir(), "none")); n != grants.DefaultRevisionsPerBranch || err != nil {
|
||||
|
||||
@@ -2,6 +2,8 @@ package grants
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
@@ -13,6 +15,7 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/identity-and-access/go/seats/meshgrants"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
@@ -94,9 +97,13 @@ func (p *Part) following(branch string) bool {
|
||||
// revision was taken in as, so grants-for asks the seat mesh-catalog's revision only for a revision it does not hold, never
|
||||
// to learn the newest.
|
||||
type revisionsKept struct {
|
||||
mu sync.Mutex
|
||||
order map[string][]string
|
||||
kept map[string]map[string]catalogue.Manifest
|
||||
mu sync.Mutex
|
||||
order map[string][]string
|
||||
kept map[string]map[string]catalogue.Manifest
|
||||
// grants are the grants each kept revision composes to, by module (grantsOfEntries), with the digest of the inputs they were
|
||||
// composed with: composed once per revision and inputs, kept as long as the revision is (design 47 H2 H-A-478 and its
|
||||
// addenda).
|
||||
grants map[string]keptGrants
|
||||
newest map[string]int64
|
||||
sources map[string]map[int64]int64
|
||||
sourced map[string][]int64
|
||||
@@ -113,6 +120,7 @@ func revisionKey(branch string, r meshcatalog.CatalogRevision) string {
|
||||
func (k *revisionsKept) init() {
|
||||
if k.kept == nil {
|
||||
k.kept, k.order, k.newest = map[string]map[string]catalogue.Manifest{}, map[string][]string{}, map[string]int64{}
|
||||
k.grants = map[string]keptGrants{}
|
||||
k.sources, k.sourced = map[string]map[int64]int64{}, map[string][]int64{}
|
||||
}
|
||||
}
|
||||
@@ -124,6 +132,44 @@ func (k *revisionsKept) get(branch string, r meshcatalog.CatalogRevision) (map[s
|
||||
return m, ok
|
||||
}
|
||||
|
||||
// keptGrants are a revision's composed grants, by module, and the digest of the inputs they were composed with.
|
||||
type keptGrants struct {
|
||||
inputs string
|
||||
grants map[string]map[string]bool
|
||||
}
|
||||
|
||||
// grantsOf is the grants kept of the revision key names, composed with the inputs of the digest given; false when none is.
|
||||
func (k *revisionsKept) grantsOf(key, inputs string) (map[string]map[string]bool, bool) {
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
g, ok := k.grants[key]
|
||||
if !ok || g.inputs != inputs {
|
||||
return nil, false
|
||||
}
|
||||
return g.grants, true
|
||||
}
|
||||
|
||||
// keepGrants keeps the grants the revision key names composes to with the inputs of the digest given, in place of any kept
|
||||
// before, while that revision is kept: so the grants are bounded as the revisions are (H-A-195's setting).
|
||||
func (k *revisionsKept) keepGrants(key, inputs string, g map[string]map[string]bool) {
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
if _, ok := k.kept[key]; ok {
|
||||
k.grants[key] = keptGrants{inputs: inputs, grants: g}
|
||||
}
|
||||
}
|
||||
|
||||
// inputsDigest is the sha256 of the inputs as JSON: what a revision's kept grants are composed with (H-A-478 second
|
||||
// addendum). Empty when they do not marshal: nothing is kept then.
|
||||
func inputsDigest(in Inputs) string {
|
||||
raw, err := json.Marshal(in)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
sum := sha256.Sum256(raw)
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
func (k *revisionsKept) put(branch string, r meshcatalog.CatalogRevision, manifests map[string]catalogue.Manifest, max int) {
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
@@ -135,6 +181,7 @@ func (k *revisionsKept) put(branch string, r meshcatalog.CatalogRevision, manife
|
||||
k.kept[key] = manifests
|
||||
for len(k.order[branch]) > max {
|
||||
delete(k.kept, k.order[branch][0])
|
||||
delete(k.grants, k.order[branch][0])
|
||||
k.order[branch] = k.order[branch][1:]
|
||||
}
|
||||
if r.Kind != "candidate" && r.Source != 0 {
|
||||
@@ -167,6 +214,7 @@ func boolsOf[V any](m map[string]V) map[string]bool {
|
||||
func (k *revisionsKept) dropLocked(branch string) {
|
||||
for _, key := range k.order[branch] {
|
||||
delete(k.kept, key)
|
||||
delete(k.grants, key)
|
||||
}
|
||||
delete(k.order, branch)
|
||||
delete(k.newest, branch)
|
||||
@@ -250,14 +298,15 @@ func (p *Part) HearCatalogRevision(_ context.Context, data []byte) error {
|
||||
// is answered from what it holds with no read; one it knows but does not hold is read by that number. Otherwise a revision
|
||||
// read whole is kept: once the first page of an answer names a revision that is kept, it is answered from what is kept,
|
||||
// without the pages after it. The branch's newest is read on by the number its first page names, so its pages are of one
|
||||
// revision though a newer one is written between them; a revision whose pages name another is not kept.
|
||||
func (p *Part) revisionEntries(in meshcatalog.RevisionInput) (manifests map[string]catalogue.Manifest, none bool, err error) {
|
||||
// revision though a newer one is written between them; a revision whose pages name another is not kept. kept names the revision
|
||||
// as it is kept (revisionKey), empty when it is not: what its grants are kept by (H-A-478).
|
||||
func (p *Part) revisionEntries(in meshcatalog.RevisionInput) (manifests map[string]catalogue.Manifest, kept string, none bool, err error) {
|
||||
branch := in.Branch
|
||||
followed := p.following(branch)
|
||||
newestAsked := in.Group == "" && in.Heads == "" && in.Source == 0 && in.Revision == 0
|
||||
if n, ok := p.revisions.known(in); ok {
|
||||
if m, held := p.revisions.get(branch, meshcatalog.CatalogRevision{Revision: n}); held {
|
||||
return m, false, nil
|
||||
return m, revisionKey(branch, meshcatalog.CatalogRevision{Revision: n}), false, nil
|
||||
}
|
||||
if newestAsked {
|
||||
in.Revision = n
|
||||
@@ -269,14 +318,14 @@ func (p *Part) revisionEntries(in meshcatalog.RevisionInput) (manifests map[stri
|
||||
for {
|
||||
raw, err := p.Call.Call(CallRevision, in)
|
||||
if err != nil {
|
||||
return nil, false, fmt.Errorf("%s does not answer: %w", CallRevision, err)
|
||||
return nil, "", false, fmt.Errorf("%s does not answer: %w", CallRevision, err)
|
||||
}
|
||||
var a meshcatalog.RevisionAnswer
|
||||
if err := json.Unmarshal(raw, &a); err != nil {
|
||||
return nil, false, fmt.Errorf("%s answered what is not a revision: %w", CallRevision, err)
|
||||
return nil, "", false, fmt.Errorf("%s answered what is not a revision: %w", CallRevision, err)
|
||||
}
|
||||
if a.NoneYet {
|
||||
return nil, true, nil
|
||||
return nil, "", true, nil
|
||||
}
|
||||
switch {
|
||||
case a.Revision == nil:
|
||||
@@ -286,7 +335,7 @@ func (p *Part) revisionEntries(in meshcatalog.RevisionInput) (manifests map[stri
|
||||
p.revisions.asked(branch, a.Revision.Revision)
|
||||
}
|
||||
if m, ok := p.revisions.get(branch, *a.Revision); ok {
|
||||
return m, false, nil
|
||||
return m, revisionKey(branch, *a.Revision), false, nil
|
||||
}
|
||||
first = a.Revision
|
||||
if newestAsked {
|
||||
@@ -302,15 +351,16 @@ func (p *Part) revisionEntries(in meshcatalog.RevisionInput) (manifests map[stri
|
||||
}
|
||||
m, err := catalogue.Decode(body)
|
||||
if err != nil {
|
||||
return nil, false, fmt.Errorf("the manifest of %s in the revision does not decode: %w", e.Module, err)
|
||||
return nil, "", false, fmt.Errorf("the manifest of %s in the revision does not decode: %w", e.Module, err)
|
||||
}
|
||||
manifests[e.Module] = m
|
||||
}
|
||||
if a.Next == "" || a.Next == in.After {
|
||||
if keep && first != nil && followed {
|
||||
p.revisions.put(branch, *first, manifests, p.revisionsPerBranch())
|
||||
kept = revisionKey(branch, *first)
|
||||
}
|
||||
return manifests, false, nil
|
||||
return manifests, kept, false, nil
|
||||
}
|
||||
in.After = a.Next
|
||||
}
|
||||
@@ -389,26 +439,41 @@ func (p *Part) GrantsFor(in meshgrants.GrantsForInput) (meshgrants.GrantsFor, er
|
||||
default:
|
||||
return meshgrants.GrantsFor{}, errors.New("grants-for names a source revision or a run")
|
||||
}
|
||||
cand, none, err := p.revisionEntries(candidate)
|
||||
// Each stage's time in the journal (H-A-478): the inputs, each revision's read and each one's grants, composed or kept.
|
||||
var stages []string
|
||||
stage := func(what string, start time.Time) {
|
||||
stages = append(stages, fmt.Sprintf("%s %s", what, time.Since(start).Round(time.Millisecond)))
|
||||
}
|
||||
defer func() {
|
||||
if p.Logf != nil {
|
||||
p.Logf("grants-for %s: %s", candidateName(candidate), strings.Join(stages, ", "))
|
||||
}
|
||||
}()
|
||||
start := time.Now()
|
||||
cand, candKept, none, err := p.revisionEntries(candidate)
|
||||
stage("the candidate read", start)
|
||||
if err != nil || none {
|
||||
return meshgrants.GrantsFor{Error: orWhy(err, "the candidate's catalog half is not held: the branch has no revision of it")}, nil
|
||||
}
|
||||
newest, none, err := p.revisionEntries(meshcatalog.RevisionInput{Branch: in.Branch})
|
||||
start = time.Now()
|
||||
newest, newestKept, none, err := p.revisionEntries(meshcatalog.RevisionInput{Branch: in.Branch})
|
||||
stage("the newest read", start)
|
||||
if err != nil || none {
|
||||
return meshgrants.GrantsFor{Error: orWhy(err, "the branch's newest catalog half is not held")}, nil
|
||||
}
|
||||
base := map[string]catalogue.Manifest{}
|
||||
for module := range cand {
|
||||
if m, ok := newest[module]; ok {
|
||||
base[module] = m
|
||||
}
|
||||
}
|
||||
start = time.Now()
|
||||
input := p.inputs()
|
||||
was, err := grantsOfEntries(input, base)
|
||||
digest := inputsDigest(input)
|
||||
stage("the inputs", start)
|
||||
start = time.Now()
|
||||
was, composed, err := p.grantsOfRevision(input, digest, newestKept, newest)
|
||||
stage("the newest's grants ("+composed+")", start)
|
||||
if err != nil {
|
||||
return meshgrants.GrantsFor{Error: "the branch's newest does not compose: " + err.Error()}, nil
|
||||
}
|
||||
will, err := grantsOfEntries(input, cand)
|
||||
start = time.Now()
|
||||
will, composed, err := p.grantsOfRevision(input, digest, candKept, cand)
|
||||
stage("the candidate's grants ("+composed+")", start)
|
||||
if err != nil {
|
||||
return meshgrants.GrantsFor{Error: "the candidate does not compose: " + err.Error()}, nil
|
||||
}
|
||||
@@ -435,6 +500,33 @@ func (p *Part) GrantsFor(in meshgrants.GrantsForInput) (meshgrants.GrantsFor, er
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// candidateName names the candidate an ask is of, for the journal.
|
||||
func candidateName(c meshcatalog.RevisionInput) string {
|
||||
if c.Group != "" {
|
||||
return c.Branch + " " + c.Group
|
||||
}
|
||||
return c.Branch + " source " + strconv.FormatInt(c.Source, 10)
|
||||
}
|
||||
|
||||
// grantsOfRevision is the grants a revision's manifests compose to with in (of the digest inputs), by module: those kept of the
|
||||
// revision kept names when they were composed with the same inputs, else composed and kept with it (H-A-478: once per revision
|
||||
// and inputs). how says which.
|
||||
func (p *Part) grantsOfRevision(in Inputs, inputs, kept string, manifests map[string]catalogue.Manifest) (grants map[string]map[string]bool, how string, err error) {
|
||||
if kept != "" && inputs != "" {
|
||||
if g, ok := p.revisions.grantsOf(kept, inputs); ok {
|
||||
return g, "kept", nil
|
||||
}
|
||||
}
|
||||
g, err := grantsOfEntries(in, manifests)
|
||||
if err != nil {
|
||||
return nil, "composed", err
|
||||
}
|
||||
if kept != "" && inputs != "" {
|
||||
p.revisions.keepGrants(kept, inputs, g)
|
||||
}
|
||||
return g, "composed", nil
|
||||
}
|
||||
|
||||
func orWhy(err error, why string) string {
|
||||
if err != nil {
|
||||
return err.Error()
|
||||
|
||||
@@ -170,7 +170,8 @@ type Part struct {
|
||||
Emit Emitter
|
||||
// Create creates a declared stream the bus lacks: the process's connection; nil in the tools, which ask the process.
|
||||
Create Creator
|
||||
// Logf logs what no state or condition says (an error of acting on an answer): the process's; nil in the tools.
|
||||
// Logf logs what no state or condition says (an error of acting on an answer): the process's journal, and in the tools
|
||||
// program its standard error (grants-for's stage times, H-A-478).
|
||||
Logf func(string, ...any)
|
||||
// Objects is the part's object bucket of sealed lists: the process's connection; nil in the tools.
|
||||
Objects Objects
|
||||
|
||||
Reference in New Issue
Block a user