Join: merge feat/47-H2-b into feat/47-H2
No conflicts. The mesh-builds contract is regenerated with seatgen: -b's publish-queue verb and feat/47-H2's descriptions meet in one VerbDescription (seatgen -check clean). Builds: design 47 H2 (join of the develop step, playbook 07)
This commit is contained in:
@@ -57,8 +57,8 @@ type Asker struct {
|
||||
Expiry *Expiry
|
||||
// Logf says a drop that failed (the log's Printf when unset): the ask then stays on its stream, and it is said.
|
||||
Logf func(format string, args ...any)
|
||||
// Outputs reads the answer a step's key names as its output-object, being over what the steps state holds (design 47 G
|
||||
// A-290), through the build seat's verb output on the step's node (buildstep.Step.Whole); Wait answers the step whole.
|
||||
// Outputs reads the answer a step's key names as its answer-object, being over what the steps state holds (design 47 G
|
||||
// A-290), through the build seat's verb answer on the step's node (buildstep.Step.Whole; H-A-412); Wait answers the step whole.
|
||||
// Nil reads none: such a step's wait is an error saying so.
|
||||
Outputs buildstep.AskVerb
|
||||
}
|
||||
@@ -337,7 +337,7 @@ func (k Asker) Wait(ctx context.Context, a buildstep.StepAsk) (buildstep.Step, e
|
||||
}
|
||||
whole, err := step.Whole(a.Language, a.Run != nil, k.Outputs)
|
||||
if err != nil {
|
||||
return buildstep.Step{}, fmt.Errorf("the ask %s's step on %s ended %s, and its answer %s cannot be read: %w", a.ID, step.Node, step.State, step.OutputObject, err)
|
||||
return buildstep.Step{}, fmt.Errorf("the ask %s's step on %s ended %s, and its answer %s cannot be read: %w", a.ID, step.Node, step.State, step.AnswerObject, err)
|
||||
}
|
||||
return whole, nil
|
||||
}
|
||||
|
||||
@@ -55,17 +55,17 @@ func TestAnAskerPutsOnceAndReadsTheStep(t *testing.T) {
|
||||
if _, err := k.Ask(context.Background(), goAsk("mvl0frv18d1958")); err == nil || !strings.Contains(err.Error(), "shanks refused: go 1.27 installed") {
|
||||
t.Fatalf("a refused step: %v", err)
|
||||
}
|
||||
// Design 47 G A-290: a step whose key names its answer as the object output.<ask> is answered whole, read through the
|
||||
// build seat's verb output on the step's node; with no verb to read it the wait says so, never the step without its answer.
|
||||
// Design 47 G A-290, H-A-412: a step whose key names its answer as the object answer.<ask> is answered whole, read through the
|
||||
// build seat's verb answer on the step's node; with no verb to read it the wait says so, never the step without its answer.
|
||||
f.Answer = func(a buildstep.StepAsk) buildstep.Step {
|
||||
return buildstep.Step{ID: a.ID, Node: "ace", State: buildstep.StepPassed, OutputObject: buildstep.OutputObjectName(a.ID)}
|
||||
return buildstep.Step{ID: a.ID, Node: "ace", State: buildstep.StepPassed, AnswerObject: buildstep.AnswerObjectName(a.ID)}
|
||||
}
|
||||
var asked string
|
||||
k.Outputs = func(key string, body any) (json.RawMessage, error) {
|
||||
asked = key
|
||||
return json.RawMessage(`{"ask":"` + body.(map[string]any)["ask"].(string) + `","file":"eyJhcnRpZmFjdHMiOltdfQ=="}`), nil
|
||||
}
|
||||
if step, err := k.Ask(context.Background(), goAsk("mvl0frv18d1961")); err != nil || string(step.Result) != `{"artifacts":[]}` || step.OutputObject != "" || asked != "seat:node-build-go.output@ace" {
|
||||
if step, err := k.Ask(context.Background(), goAsk("mvl0frv18d1961")); err != nil || string(step.Result) != `{"artifacts":[]}` || step.AnswerObject != "" || asked != "seat:node-build-go.answer@ace" {
|
||||
t.Fatalf("a step whose answer is an object: %+v, %v, asked %q", step, err, asked)
|
||||
}
|
||||
k.Outputs = nil
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"reflect"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds/forms"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/containerimageregistryclient/reference"
|
||||
@@ -260,6 +261,11 @@ type BuilderBuild struct {
|
||||
// time at most, then asks the operator. A person's build (builder_build_again, the operator's answer) starts the count
|
||||
// again. A reader that does not know the field reads none.
|
||||
EventBuilds int `json:"event-builds,omitempty"`
|
||||
// Ended and Outcome are, on a done or failed key, when the build ended and how (done or failed), written with its end, so a
|
||||
// reader of the builds (the artifact store's sweep) reads no key history (design 47 H2 §797). A reader that does not know the
|
||||
// fields reads the key as before; a key written before them names neither.
|
||||
Ended time.Time `json:"ended,omitzero"`
|
||||
Outcome string `json:"outcome,omitempty"`
|
||||
}
|
||||
|
||||
func (b BuilderBuild) check() error {
|
||||
|
||||
@@ -0,0 +1,131 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
|
||||
"mesh-delivery-builder/internal/build"
|
||||
)
|
||||
|
||||
// dropVerbKey is the seat verb builder_ask_drop asks (the manifest's invokes).
|
||||
const dropVerbKey = "seat:" + meshbuilds.Seat + "." + meshbuilds.VerbDrop
|
||||
|
||||
// takenReader is the builder's state taken, as the bundle reads it (stdio.State).
|
||||
type takenReader interface {
|
||||
Keys() ([]string, error)
|
||||
Get(key string) (*stdio.StateEntry, error)
|
||||
}
|
||||
|
||||
// TakenAsk is an ask of the seat mesh-builds the builder took (its id in the state taken, kept for ask-taken-bound), or
|
||||
// whose drop it keeps before its pull, as builder_asks shows it.
|
||||
type TakenAsk struct {
|
||||
Ask string `json:"ask"`
|
||||
At time.Time `json:"at,omitzero"`
|
||||
By string `json:"by,omitempty"`
|
||||
Why string `json:"why,omitempty"`
|
||||
}
|
||||
|
||||
// WaitingRow is an ask the runtime reads as waiting on the seat's work queue, as builder_asks shows it: State is
|
||||
// waiting (taken by no holder yet), waiting again (handed out before and given back, its id not taken) or held (handed
|
||||
// out again while the builder holds its id in the state taken: its run answers it).
|
||||
type WaitingRow struct {
|
||||
stdio.WaitingAsk
|
||||
State string `json:"state"`
|
||||
}
|
||||
|
||||
// The states of a waiting ask.
|
||||
const (
|
||||
StateWaiting = "waiting"
|
||||
StateWaitingAgain = "waiting again"
|
||||
StateHeld = "held"
|
||||
)
|
||||
|
||||
// asksTools are builder_asks, what waits on and is taken from the seat mesh-builds' work queue: the waiting asks read
|
||||
// through the runtime (stdio.QueueWaiting, H-A-410), the taken ones and the drops kept before a pull from the builder's
|
||||
// state taken; and builder_ask_drop, which ends one ask through the seat's verb drop once the hand act is recorded
|
||||
// (design 47 H2 §777, §778; H-A-401, H-A-404, H-A-410).
|
||||
func asksTools(waiting func(seat string) (stdio.Waiting, error), taken takenReader, drop func(meshbuilds.DropInput) (json.RawMessage, error), now func() time.Time, acts meshhandactsclient.Recorder) []stdio.Tool {
|
||||
str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) }
|
||||
return []stdio.Tool{
|
||||
{Name: build.AsksTool,
|
||||
Description: "What waits on and is taken from the seat mesh-builds' work queue: waiting, each build step ask the queue holds unacknowledged, oldest " +
|
||||
"first, with its id, the module that asked it, its age and its state: " + StateWaiting + " (taken by no holder yet), " + StateWaitingAgain +
|
||||
" (handed out before and given back) or " + StateHeld + " (its id taken by the builder, its run answers it); capped when the runtime cut the read; taken, each ask the builder took " +
|
||||
"within ask-taken-bound, with when; dropped, each drop kept for an ask not pulled yet, with who dropped it and why. Read-only.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{}},
|
||||
Run: func(map[string]any) (any, error) {
|
||||
w, err := waiting(meshbuilds.Seat)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the waiting asks of %s are not read: %w", meshbuilds.Seat, err)
|
||||
}
|
||||
keys, err := taken.Keys()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the state taken is not read: %w", err)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
tk, dr := []TakenAsk{}, []TakenAsk{}
|
||||
for _, k := range keys {
|
||||
e, err := taken.Get(k)
|
||||
if err != nil || e == nil {
|
||||
continue
|
||||
}
|
||||
if id, ok := strings.CutPrefix(k, build.DroppedPrefix); ok {
|
||||
var d build.Dropped
|
||||
_ = json.Unmarshal(e.Value, &d)
|
||||
dr = append(dr, TakenAsk{Ask: id, At: e.At, By: d.By, Why: d.Why})
|
||||
} else if id, ok := strings.CutPrefix(k, meshbuilds.Seat+"."); ok {
|
||||
tk = append(tk, TakenAsk{Ask: id, At: e.At})
|
||||
}
|
||||
}
|
||||
held := map[string]bool{}
|
||||
for _, a := range tk {
|
||||
held[a.Ask] = true
|
||||
}
|
||||
rows := make([]WaitingRow, 0, len(w.Asks))
|
||||
for _, a := range w.Asks {
|
||||
r := WaitingRow{WaitingAsk: a, State: StateWaiting}
|
||||
if held[a.ID] {
|
||||
r.State = StateHeld
|
||||
} else if a.Again {
|
||||
r.State = StateWaitingAgain
|
||||
}
|
||||
rows = append(rows, r)
|
||||
}
|
||||
return map[string]any{"seat": meshbuilds.Seat, "waiting": rows, "capped": w.Capped, "taken": tk, "dropped": dr}, nil
|
||||
}},
|
||||
{Name: build.AskDropTool,
|
||||
Description: "End one ask on the seat mesh-builds' work queue through the seat's verb drop: an ask the builder runs is cancelled, one not " +
|
||||
"pulled yet is kept dropped for ask-taken-bound; either way its step ends could-not-run naming the drop and who asked it. Written in " +
|
||||
"the hand-act log (mesh-hand-acts).",
|
||||
Input: map[string]any{"type": "object", "required": []string{"ask", "why"}, "properties": map[string]any{
|
||||
"ask": map[string]any{"type": "string", "description": "the ask's id"},
|
||||
"why": map[string]any{"type": "string", "description": "why it is dropped, in plain words"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
ask, why := str(args, "ask"), str(args, "why")
|
||||
if ask == "" || why == "" {
|
||||
return nil, errors.New("give the ask's id and why it is dropped")
|
||||
}
|
||||
if _, err := acts.Record(meshhandactsclient.Request{What: fmt.Sprintf("dropped the ask %s of the seat %s by hand", ask, meshbuilds.Seat),
|
||||
Why: why, Cause: build.AskDropTool, Verb: meshbuilds.VerbDrop}); err != nil {
|
||||
return nil, fmt.Errorf("the hand act was not recorded through mesh-hand-acts, so nothing was dropped: %w", err)
|
||||
}
|
||||
raw, err := drop(meshbuilds.DropInput{ID: ask, Why: why, By: stdio.CallerOf(args)})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the seat %s's drop did not answer for %s: %w", meshbuilds.Seat, ask, err)
|
||||
}
|
||||
var a meshbuilds.DropAnswer
|
||||
if err := json.Unmarshal(raw, &a); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]any{"ask": ask, "dropped": true, "held": a["held"]}, nil
|
||||
}},
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,141 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/seatlist"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
)
|
||||
|
||||
// BuildCurrentTool is build_current (design 47 H2 §801).
|
||||
const BuildCurrentTool = "build_current"
|
||||
|
||||
// buildCurrentDescription is build_current's description.
|
||||
const buildCurrentDescription = "What every holder of a build seat takes and runs now (design 47 H2 section 801): the holders named by the " +
|
||||
"seat-holders view, each build seat on each node, and each holder's own answer to its seat's verb current, asked in turn and " +
|
||||
"answered together: whether it takes asks and why not (paused, capacity 0, unhealthy) since when, and every ask it runs, whoever " +
|
||||
"asked it. A holder that does not answer is listed with why. Read-only."
|
||||
|
||||
// seatHoldersState is the assigner's view of who holds which seat on which node (ADR 0571 §6).
|
||||
const seatHoldersState = "seat-holders"
|
||||
|
||||
// heldBuildSeat is one holder of a build seat as build_current answers it: the seat, its node and module, and the holder's
|
||||
// answer to current, or why there is none.
|
||||
type heldBuildSeat struct {
|
||||
Seat string `json:"seat"`
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module,omitempty"`
|
||||
Current json.RawMessage `json:"current,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// buildCurrentTool is build_current (§801; H-A-446 (1)): the holders of each build seat (seats) named from the seat-holders view
|
||||
// (holders, keyed `<seat>.<node>`), each holder's current asked in turn at its node through ask (stdio.Ask), answered together;
|
||||
// it reads no presence and chooses no node.
|
||||
func buildCurrentTool(seats func() []string, holders func() (map[string]json.RawMessage, error), ask func(key string, body any) (json.RawMessage, error)) stdio.Tool {
|
||||
return stdio.Tool{Name: BuildCurrentTool, Description: buildCurrentDescription,
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{}},
|
||||
Run: func(map[string]any) (any, error) {
|
||||
view, err := holders()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []heldBuildSeat
|
||||
for _, seat := range seats() {
|
||||
for key, raw := range view {
|
||||
node, ok := strings.CutPrefix(key, seat+".")
|
||||
if !ok || node == "" {
|
||||
continue
|
||||
}
|
||||
var held struct {
|
||||
Holders []struct {
|
||||
Module string `json:"module"`
|
||||
} `json:"holders"`
|
||||
}
|
||||
if json.Unmarshal(raw, &held) != nil || len(held.Holders) == 0 {
|
||||
continue
|
||||
}
|
||||
out = append(out, heldBuildSeat{Seat: seat, Node: node, Module: held.Holders[0].Module})
|
||||
}
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
if out[i].Seat != out[j].Seat {
|
||||
return out[i].Seat < out[j].Seat
|
||||
}
|
||||
return out[i].Node < out[j].Node
|
||||
})
|
||||
for i := range out { // in turn, one holder at a time
|
||||
raw, err := ask("seat:"+out[i].Seat+".current@"+out[i].Node, map[string]any{})
|
||||
if err != nil {
|
||||
out[i].Error = err.Error()
|
||||
continue
|
||||
}
|
||||
out[i].Current = raw
|
||||
}
|
||||
if out == nil {
|
||||
out = []heldBuildSeat{}
|
||||
}
|
||||
return map[string]any{"holders": out, "count": len(out)}, nil
|
||||
}}
|
||||
}
|
||||
|
||||
// seatHolders follows the seat-holders view of the claimant of the seat mesh-module-assigner once the catalog half names it.
|
||||
type seatHolders struct {
|
||||
mu sync.Mutex
|
||||
values func() map[string]json.RawMessage
|
||||
why string
|
||||
}
|
||||
|
||||
// follow starts the follow when the bundle's first views stand (the claimant is read from the catalog half).
|
||||
func (s *seatHolders) follow(ctx context.Context, branches *meshcatalogclient.Branches, r *views.Reader) {
|
||||
s.mu.Lock()
|
||||
s.why = "the catalog half's first views are not read yet"
|
||||
s.mu.Unlock()
|
||||
go func() {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-branches.FirstViews(ctx):
|
||||
}
|
||||
claimant, ok := branches.Claimant(seatlist.HolderSeat)
|
||||
if !ok {
|
||||
s.mu.Lock()
|
||||
s.why = "no module claims the seat " + seatlist.HolderSeat + " on the catalog half"
|
||||
s.mu.Unlock()
|
||||
return
|
||||
}
|
||||
w, ok := meshcatalogclient.ReaderWiring(r).(interface {
|
||||
Follow(owner, state, tool string) (func() map[string]json.RawMessage, error)
|
||||
})
|
||||
var values func() map[string]json.RawMessage
|
||||
err := fmt.Errorf("the bundle's reader follows no plain view")
|
||||
if ok {
|
||||
values, err = w.Follow(claimant, seatHoldersState, claimant+"."+seatHoldersState)
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if err != nil {
|
||||
s.why = fmt.Sprintf("the seat-holders view of %s is not followed: %v", claimant, err)
|
||||
return
|
||||
}
|
||||
s.values, s.why = values, ""
|
||||
}()
|
||||
}
|
||||
|
||||
// read is the view's values, or why it is not read.
|
||||
func (s *seatHolders) read() (map[string]json.RawMessage, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.values == nil {
|
||||
return nil, errors.New("the seat-holders view is not read: " + s.why)
|
||||
}
|
||||
return s.values(), nil
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"mesh-delivery-builder/internal/versions"
|
||||
)
|
||||
|
||||
const (
|
||||
// HealthTool is builder_health, the builder's health as its manifest's health asks it (design 47 H2 §794, H-A-435).
|
||||
HealthTool = "builder_health"
|
||||
// BehindAfter is how long a durable consumer's oldest unacknowledged message may wait before the health names it: an
|
||||
// events backlog, or a handler standing still on one message (§794).
|
||||
BehindAfter = 5 * time.Minute
|
||||
)
|
||||
|
||||
// healthTool is builder_health: healthy when it answers, and each of the builder's durable consumers' oldest unacknowledged
|
||||
// message waits less than BehindAfter, as the consumers' facts the builder writes at every delivery and every acknowledgement
|
||||
// say them (its state consumers, the runner's runner_health form; H-A-435 and its addendum); otherwise unhealthy, naming each
|
||||
// consumer behind and how long.
|
||||
func healthTool(consumers versions.Reader, now func() time.Time) stdio.Tool {
|
||||
return stdio.Tool{Name: HealthTool,
|
||||
Description: "The builder's health: healthy when it answers and each of its durable consumers' oldest unacknowledged message waits less " +
|
||||
"than five minutes (no events backlog, no handler standing still on a message); otherwise it names each consumer behind and how long. " +
|
||||
"Read from the builder's state consumers, which it writes at every message delivered and acknowledged. Read-only.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{}},
|
||||
Run: func(map[string]any) (any, error) { return builderHealth(consumers, now()), nil }}
|
||||
}
|
||||
|
||||
// builderHealth is builder_health's answer at now.
|
||||
func builderHealth(consumers versions.Reader, now time.Time) health.Answer {
|
||||
keys, err := consumers.Keys()
|
||||
if err != nil {
|
||||
return health.Unhealthy(fmt.Sprintf("The ages of the builder's consumers cannot be read from its state %s (%v).", process.ConsumersState, err))
|
||||
}
|
||||
if len(keys) == 0 {
|
||||
return health.Healthy(fmt.Sprintf("The builder answers and has heard no event yet: its state %s holds no consumer.", process.ConsumersState))
|
||||
}
|
||||
sort.Strings(keys)
|
||||
var behind []string
|
||||
for _, k := range keys {
|
||||
raw, ok, err := consumers.Get(k)
|
||||
if err != nil {
|
||||
return health.Unhealthy(fmt.Sprintf("The age of the consumer %s cannot be read (%v).", k, err))
|
||||
}
|
||||
var f process.ConsumerFacts
|
||||
if ok {
|
||||
if err := json.Unmarshal(raw, &f); err != nil {
|
||||
return health.Unhealthy(fmt.Sprintf("The age of the consumer %s does not read (%v).", k, err))
|
||||
}
|
||||
}
|
||||
if !f.OldestUnacknowledged.IsZero() {
|
||||
if age := now.Sub(f.OldestUnacknowledged); age >= BehindAfter {
|
||||
behind = append(behind, fmt.Sprintf("%s (its oldest unacknowledged message waits %s, %d more not yet given)", k, age.Round(time.Second), f.Pending))
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(behind) > 0 {
|
||||
return health.Unhealthy(fmt.Sprintf("The builder's consumers behind by %s or more: %s.", BehindAfter, strings.Join(behind, ", ")))
|
||||
}
|
||||
return health.Healthy(fmt.Sprintf("The builder answers, and each of its %d consumers is less than %s behind.", len(keys), BehindAfter))
|
||||
}
|
||||
@@ -21,6 +21,7 @@ import (
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"git.novox.be/novox/mesh-sdk/go/seatdef"
|
||||
"git.novox.be/novox/mesh-sdk/go/seatlist"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
@@ -432,8 +433,9 @@ func main() {
|
||||
tools = append(tools, buildTool(&build.BuildReader{Builds: read(stdio.State("builds")), Entry: entryOf(stdio.State("builds")), Asked: read(stdio.State("asked")),
|
||||
Newest: newestVia(stdio.Ask), Steps: r}))
|
||||
tools = append(tools, reservedTools(rr, buildAgain, time.Now, meshhandactsclient.Recorder{Ask: stdio.Ask})...)
|
||||
qs, as := stdio.State(versions.PublishQueueState), stdio.State(versions.ActedState)
|
||||
tools = append(tools, republishTools(readerOf(qs), readerOf(as), readerOf(reasks), buildAgain, time.Now, meshhandactsclient.Recorder{Ask: stdio.Ask})...)
|
||||
as := stdio.State(versions.ActedState)
|
||||
tools = append(tools, publishActTools(buildAgain, time.Now, meshhandactsclient.Recorder{Ask: stdio.Ask})...)
|
||||
tools = append(tools, republishTools(publishQueueVia(stdio.Ask), readerOf(as), readerOf(reasks), buildAgain, time.Now, meshhandactsclient.Recorder{Ask: stdio.Ask})...)
|
||||
ws := stdio.State(build.WaitsState)
|
||||
wr := versions.Reader{Keys: ws.Keys, Get: func(key string) (json.RawMessage, bool, error) {
|
||||
e, err := ws.Get(key)
|
||||
@@ -442,7 +444,12 @@ func main() {
|
||||
}
|
||||
return e.Value, true, nil
|
||||
}}
|
||||
tools = append(tools, asksTools(stdio.QueueWaiting, stdio.State(stdio.TakenState), func(in meshbuilds.DropInput) (json.RawMessage, error) {
|
||||
return stdio.Ask(dropVerbKey, in)
|
||||
}, time.Now, meshhandactsclient.Recorder{Ask: stdio.Ask})...)
|
||||
tools = append(tools, waitsTools(wr, buildAgain, time.Now, meshhandactsclient.Recorder{Ask: stdio.Ask})...)
|
||||
tools = append(tools, unbuildableTool(unbuildablePage(stdio.State(build.UnbuildableState)))) // the plans no seat builds (§795)
|
||||
tools = append(tools, healthTool(readerOf(stdio.State(process.ConsumersState)), time.Now)) // the builder's health (§794, H-A-435)
|
||||
tools = append(tools, toolchainsTools(stdio.State(buildrun.ToolchainsHeardState), meshhandactsclient.Recorder{Ask: stdio.Ask}, time.Now)...)
|
||||
// The builder's expiring keys (state bounds): what each bounds and when it ends.
|
||||
tools = append(tools, stdio.Tool{Name: versions.BoundsTool, Description: versions.BoundsDescription,
|
||||
@@ -458,6 +465,16 @@ func main() {
|
||||
// states stand, and asking one again by hand (ADR 0615, DLV-M1).
|
||||
tools = append(tools, views.StatusTools(os.Getenv(stdio.ServedAs), "builder_first_views", "builder_first_view_again", stdio.State(views.StatusState), meshhandactsclient.Recorder{Ask: stdio.Ask}, time.Now)...)
|
||||
tools = append(tools, gate.Tools()...)
|
||||
// build_current (design 47 H2 §801; H-A-446 (1)): each build seat's holders from the seat-holders view, each asked current.
|
||||
holders := &seatHolders{}
|
||||
holders.follow(context.Background(), branches, pk)
|
||||
tools = append(tools, buildCurrentTool(func() []string {
|
||||
var out []string
|
||||
for _, m := range r.Seats.All() {
|
||||
out = append(out, m.Seat)
|
||||
}
|
||||
return out
|
||||
}, holders.read, stdio.Ask))
|
||||
if err := stdio.Serve("", tools); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "[mesh-delivery-builder-tools] %v\n", err)
|
||||
os.Exit(1)
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
|
||||
"mesh-delivery-builder/internal/build"
|
||||
@@ -53,10 +54,68 @@ func onePage(page func(after string, maxBytes int, budget time.Duration) (stdio.
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// publishQueueVerbKey is the seat verb builder_publish_queue asks (the manifest's invokes): the builder reads its own work queue
|
||||
// publishes (design 47 H2 §793, H-A-454).
|
||||
const publishQueueVerbKey = "seat:" + meshbuilds.Seat + "." + meshbuilds.VerbPublishQueue
|
||||
|
||||
// publishQueueVia asks the seat mesh-builds' verb publish-queue through ask (stdio.Ask).
|
||||
func publishQueueVia(ask func(key string, body any) (json.RawMessage, error)) func(module, version string) (meshbuilds.PublishQueueAnswer, error) {
|
||||
return func(module, version string) (meshbuilds.PublishQueueAnswer, error) {
|
||||
var a meshbuilds.PublishQueueAnswer
|
||||
raw, err := ask(publishQueueVerbKey, meshbuilds.PublishQueueInput{Module: module, Version: version})
|
||||
if err != nil {
|
||||
return a, fmt.Errorf("the builder's verb %s did not answer: %w", meshbuilds.VerbPublishQueue, err)
|
||||
}
|
||||
if err := json.Unmarshal(raw, &a); err != nil {
|
||||
return a, fmt.Errorf("the verb %s answered what is no queue: %w", meshbuilds.VerbPublishQueue, err)
|
||||
}
|
||||
return a, nil
|
||||
}
|
||||
}
|
||||
|
||||
// publishActTools are builder_publish_drop and builder_publish_first (design 47 H2 §793): hand acts, recorded in the hand-act log
|
||||
// first, then written to reasks for the builder, which owns the work queue, to perform and answer.
|
||||
func publishActTools(ask queueAsker, now func() time.Time, acts meshhandactsclient.Recorder) []stdio.Tool {
|
||||
str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) }
|
||||
tool := func(name, mode, desc, what string) stdio.Tool {
|
||||
return stdio.Tool{Name: name, Description: desc,
|
||||
Input: map[string]any{"type": "object", "required": []string{"module", "version", "why"}, "properties": map[string]any{
|
||||
"module": map[string]any{"type": "string", "description": "the item's module"},
|
||||
"version": map[string]any{"type": "string", "description": "the item's version, MAJOR.MINOR.PATCH"},
|
||||
"why": map[string]any{"type": "string", "description": "why, in plain words"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
key, value, err := build.NewPublishAct(mode, str(args, "module"), str(args, "version"), str(args, "why"), stdio.CallerOf(args), now())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err := acts.Record(meshhandactsclient.Request{What: fmt.Sprintf("%s %s %s by hand", what, str(args, "module"), str(args, "version")),
|
||||
Why: str(args, "why"), Cause: name, Verb: mode}); err != nil {
|
||||
return nil, fmt.Errorf("the hand act was not recorded through mesh-hand-acts, so nothing was done: %w", err)
|
||||
}
|
||||
raw, err := ask(key, value)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
a, err := build.DecodeReAsk(raw)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if a.Refused {
|
||||
return nil, errors.New(a.Answer)
|
||||
}
|
||||
return map[string]any{"key": key, "answer": a.Answer}, nil
|
||||
}}
|
||||
}
|
||||
return []stdio.Tool{
|
||||
tool(versions.PublishDropTool, build.ModePublishDrop, versions.PublishDropDescription, "dropped the publish of"),
|
||||
tool(versions.PublishFirstTool, build.ModePublishFirst, versions.PublishFirstDescription, "put first the publish of"),
|
||||
}
|
||||
}
|
||||
|
||||
// republishTools are builder_publish_again and builder_publish_queue (design 47 G §588). The publish again is a hand act:
|
||||
// it is recorded in the hand-act log first, then written to reasks for the builder, which owns the queue, to answer
|
||||
// with the versions it queued.
|
||||
func republishTools(queue, acted, reasks versions.Reader, ask queueAsker, now func() time.Time, acts meshhandactsclient.Recorder) []stdio.Tool {
|
||||
func republishTools(queue func(module, version string) (meshbuilds.PublishQueueAnswer, error), acted, reasks versions.Reader, ask queueAsker, now func() time.Time, acts meshhandactsclient.Recorder) []stdio.Tool {
|
||||
str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) }
|
||||
return []stdio.Tool{
|
||||
{Name: versions.PublishAgainTool, Description: versions.PublishAgainDescription,
|
||||
@@ -91,7 +150,11 @@ func republishTools(queue, acted, reasks versions.Reader, ask queueAsker, now fu
|
||||
"module": map[string]any{"type": "string", "description": "list only this module's queued versions, each with its class"},
|
||||
"version": map[string]any{"type": "string", "description": "with module: only this version, MAJOR.MINOR.PATCH"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
return versions.AnswerQueue(queue, acted, reasks, str(args, "module"), str(args, "version"))
|
||||
q, err := queue(str(args, "module"), str(args, "version"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return versions.AnswerQueue(q, acted, reasks)
|
||||
}},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,10 +3,18 @@ package main
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
|
||||
"mesh-delivery-builder/internal/build"
|
||||
|
||||
"mesh-delivery-builder/internal/versions"
|
||||
)
|
||||
|
||||
// TestAStateIsReadInOnePage: design 47 H H-A-292, H-A-293 — builder_publish_queue reads each of its own states in one page and
|
||||
@@ -41,4 +49,92 @@ func TestAStateIsReadInOnePage(t *testing.T) {
|
||||
if _, err := onePage(unsupported, nil); !errors.Is(err, errors.ErrUnsupported) {
|
||||
t.Errorf("a runtime with no pages answered %v, want ErrUnsupported so the reader reads key by key", err)
|
||||
}
|
||||
// Design 47 H2 §794, H-A-435: builder_health reads the state consumers; a consumer whose oldest unacknowledged message waits
|
||||
// six minutes is named, one within five minutes is not.
|
||||
t.Run("builder_health names a consumer behind by five minutes", func(t *testing.T) {
|
||||
now := time.Unix(10000, 0)
|
||||
facts := map[string]json.RawMessage{
|
||||
"builder-events": json.RawMessage(`{"stream":"EVENTS","name":"builder-events","pending":40,"oldest-unacknowledged":"` + now.Add(-6*time.Minute).UTC().Format(time.RFC3339) + `"}`),
|
||||
"builder-kept": json.RawMessage(`{"stream":"K","name":"builder-kept","oldest-unacknowledged":"` + now.Add(-time.Minute).UTC().Format(time.RFC3339) + `"}`)}
|
||||
r := versions.Reader{Keys: func() ([]string, error) { return []string{"builder-kept", "builder-events"}, nil },
|
||||
Get: func(k string) (json.RawMessage, bool, error) { v, ok := facts[k]; return v, ok, nil }}
|
||||
a := builderHealth(r, now)
|
||||
if a.Healthy || a.Why != "The builder's consumers behind by 5m0s or more: builder-events (its oldest unacknowledged message waits 6m0s, 40 more not yet given)." {
|
||||
t.Errorf("health = %+v, want unhealthy naming builder-events alone", a)
|
||||
}
|
||||
if a := builderHealth(r, now.Add(-2*time.Minute)); !a.Healthy {
|
||||
t.Errorf("health four minutes in = %+v, want healthy", a)
|
||||
}
|
||||
})
|
||||
// Design 47 H2 §793, H-A-454: builder_publish_queue asks the seat mesh-builds' verb publish-queue, which the builder serves from
|
||||
// its own stream, and answers its counts, list and cap with the states beside them; builder_publish_drop and
|
||||
// builder_publish_first record the hand act first, then write their reasks value.
|
||||
t.Run("the queue is the builder's verb publish-queue", func(t *testing.T) {
|
||||
var asked []string
|
||||
ask := func(key string, body any) (json.RawMessage, error) {
|
||||
in := body.(meshbuilds.PublishQueueInput)
|
||||
asked = append(asked, key+" "+in.Module)
|
||||
return json.Marshal(meshbuilds.PublishQueueAnswer{Total: 9000, Listed: 1, Capped: true, ItemsRead: 4000, QueueItems: 9000, QueueBytes: 4096,
|
||||
Counts: []meshbuilds.PublishQueueAnswerCountsItem{{Holder: "oci", Class: versions.PublishClassStart, Count: 4000}},
|
||||
Queue: []meshbuilds.PublishQueueAnswerQueueItem{{Holder: "oci", Module: "m", Version: "0.1.0", Class: versions.PublishClassStart}}})
|
||||
}
|
||||
none := versions.Reader{Keys: func() ([]string, error) { return nil, nil }}
|
||||
tools := republishTools(publishQueueVia(ask), none, none, nil, time.Now, meshhandactsclient.Recorder{})
|
||||
raw, err := tools[1].Run(map[string]any{"module": "m"})
|
||||
a, _ := raw.(versions.Queued)
|
||||
if err != nil || fmt.Sprint(asked) != "[seat:mesh-builds.publish-queue m]" || a.Total != 9000 || len(a.Queue) != 1 || a.Queue[0].Module != "m" ||
|
||||
len(a.Counts) != 1 || !a.Capped || len(a.Read) != 1 || a.Read[0].KeysRead != 4000 || a.Read[0].StateKeys != 9000 {
|
||||
t.Errorf("%s asked %v, answered %+v (%v): want the verb asked with the module, its answer carried with its cap", tools[1].Name, asked, a, err)
|
||||
}
|
||||
})
|
||||
// Design 47 H2 §801, H-A-446 (1): build_current names the holders from the seat-holders view and asks each holder's current in
|
||||
// turn at its node, answering them together; a holder that does not answer is listed with why.
|
||||
t.Run("build_current asks every build seat's holder in turn", func(t *testing.T) {
|
||||
view := map[string]json.RawMessage{
|
||||
"node-build-go.g14": json.RawMessage(`{"holders":[{"module":"mesh-build-go"}]}`),
|
||||
"node-build-go.ace": json.RawMessage(`{"holders":[{"module":"mesh-build-go"}]}`),
|
||||
"node-build-node.ace": json.RawMessage(`{"holders":[{"module":"mesh-build-node"}]}`),
|
||||
"mesh-builds": json.RawMessage(`{"holders":[{"module":"mesh-delivery-builder"}]}`)}
|
||||
var asked []string
|
||||
ask := func(key string, body any) (json.RawMessage, error) {
|
||||
asked = append(asked, key)
|
||||
if strings.HasSuffix(key, "@g14") {
|
||||
return nil, errors.New("no answer")
|
||||
}
|
||||
return json.RawMessage(`{"takes":true,"asks":[{"id":"a1","asker":"mesh-delivery-builder"}]}`), nil
|
||||
}
|
||||
tool := buildCurrentTool(func() []string { return []string{"node-build-go", "node-build-node"} }, func() (map[string]json.RawMessage, error) { return view, nil }, ask)
|
||||
raw, err := tool.Run(map[string]any{})
|
||||
a, _ := raw.(map[string]any)
|
||||
hs, _ := a["holders"].([]heldBuildSeat)
|
||||
if err != nil || fmt.Sprint(asked) != "[seat:node-build-go.current@ace seat:node-build-go.current@g14 seat:node-build-node.current@ace]" ||
|
||||
len(hs) != 3 || hs[1].Error != "no answer" || hs[0].Module != "mesh-build-go" || !strings.Contains(string(hs[2].Current), `"a1"`) {
|
||||
t.Errorf("asked %v, answered %+v (%v): want each holder asked in turn at its node, answered together, the silent one with why", asked, hs, err)
|
||||
}
|
||||
})
|
||||
t.Run("drop and first record the hand act, then ask the builder", func(t *testing.T) {
|
||||
var order []string
|
||||
ask := func(key string, value []byte) (json.RawMessage, error) {
|
||||
order = append(order, "ask "+key)
|
||||
r, _ := build.DecodeReAsk(value)
|
||||
r.Answer = "done"
|
||||
return json.Marshal(r)
|
||||
}
|
||||
acts := meshhandactsclient.Recorder{Ask: func(key string, body any) (json.RawMessage, error) {
|
||||
order = append(order, "record")
|
||||
return json.RawMessage(`{"id":"h1"}`), nil
|
||||
}}
|
||||
tools := publishActTools(ask, func() time.Time { return time.Unix(5, 0) }, acts)
|
||||
for _, tool := range tools {
|
||||
if _, err := tool.Run(map[string]any{"module": "m", "version": "0.1.2", "why": "a test"}); err != nil {
|
||||
t.Fatalf("%s: %v", tool.Name, err)
|
||||
}
|
||||
}
|
||||
if len(tools) != 2 || fmt.Sprint(order) != "[record ask publish-drop.5000000000 record ask publish-first.5000000000]" {
|
||||
t.Errorf("tools %d, order %v: want each hand act recorded before its reasks value", len(tools), order)
|
||||
}
|
||||
if _, err := tools[0].Run(map[string]any{"module": "m", "why": "no version"}); err == nil {
|
||||
t.Error("a drop naming no version is not refused")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -2,14 +2,19 @@ package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
|
||||
"mesh-delivery-builder/internal/build"
|
||||
)
|
||||
|
||||
type fakeHeard struct{ m map[string]json.RawMessage }
|
||||
@@ -70,4 +75,38 @@ func TestToolchainsToolsListAndDropAKeyAfterTheHandAct(t *testing.T) {
|
||||
if _, err := run(buildrun.ToolchainsDropTool, map[string]any{"key": "ace.go", "why": "ace left"}); err != nil || len(st.m) != 0 || asked < 2 {
|
||||
t.Fatalf("the drop: %v, keys left %d", err, len(st.m))
|
||||
}
|
||||
|
||||
// Design 47 H2 §777, H-A-410: builder_asks answers the waiting asks the runtime reads, held told from waiting again by
|
||||
// the state taken, and the taken asks and the drops kept; builder_ask_drop asks the seat's drop after the hand act.
|
||||
taken := &fakeHeard{m: map[string]json.RawMessage{"mesh-builds.a2": json.RawMessage(`{"seat":"mesh-builds","ask":"a2"}`),
|
||||
"dropped.a4": json.RawMessage(`{"by":"op@shanks","why":"by hand"}`)}}
|
||||
var dropped []meshbuilds.DropInput
|
||||
asks := asksTools(func(seat string) (stdio.Waiting, error) {
|
||||
if seat != meshbuilds.Seat {
|
||||
t.Errorf("the waiting asks were read of %s", seat)
|
||||
}
|
||||
return stdio.Waiting{Asks: []stdio.WaitingAsk{{ID: "a1", Module: "mesh-delivery-pipeline"}, {ID: "a2", Again: true}, {ID: "a3", Again: true}}}, nil
|
||||
}, taken, func(in meshbuilds.DropInput) (json.RawMessage, error) {
|
||||
dropped = append(dropped, in)
|
||||
return json.RawMessage(`{"held":true}`), nil
|
||||
}, time.Now, acts)
|
||||
tools = asks
|
||||
got, err = run(build.AsksTool, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a := got.(map[string]any)
|
||||
w := a["waiting"].([]WaitingRow)
|
||||
if len(w) != 3 || w[0].State != StateWaiting || w[1].State != StateHeld || w[2].State != StateWaitingAgain ||
|
||||
!reflect.DeepEqual(a["taken"], []TakenAsk{{Ask: "a2"}}) || !reflect.DeepEqual(a["dropped"], []TakenAsk{{Ask: "a4", By: "op@shanks", Why: "by hand"}}) {
|
||||
t.Errorf("builder_asks = %+v", a)
|
||||
}
|
||||
failAct = errors.New("no log")
|
||||
if _, err := run(build.AskDropTool, map[string]any{"ask": "a1", "why": "stuck"}); err == nil || len(dropped) != 0 {
|
||||
t.Fatalf("a hand act not recorded still dropped: %v", err)
|
||||
}
|
||||
failAct = nil
|
||||
if got, err := run(build.AskDropTool, map[string]any{"ask": "a1", "why": "stuck"}); err != nil || len(dropped) != 1 || dropped[0].ID != "a1" || dropped[0].Why != "stuck" || got.(map[string]any)["held"] != true {
|
||||
t.Fatalf("the drop: %v, %v, asked %+v", got, err, dropped)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"sort"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
|
||||
"mesh-delivery-builder/internal/build"
|
||||
"mesh-delivery-builder/internal/versions"
|
||||
)
|
||||
|
||||
// unbuildableTool is builder_unbuildable (design 47 H2 §795, H-A-425): the builder's state unbuildable, the plans no seat builds,
|
||||
// read in one page of at most 768 KB (capped says what it left out), with the summary line of the rows read.
|
||||
func unbuildableTool(page func() (versions.StatePage, error)) stdio.Tool {
|
||||
return stdio.Tool{Name: build.UnbuildableTool,
|
||||
Description: "The modules whose artifact no build seat builds, or whose build seat no module claims, as the builder judged their plans " +
|
||||
"(its state " + build.UnbuildableState + ", worked out again at every start and every change of the seat list): each module, the fingerprint " +
|
||||
"judged, the artifact's language and kind, why and when; with the one summary line the builder says of them. At most 768 KB, capped " +
|
||||
"saying what was left out. Read-only.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{}},
|
||||
Run: func(map[string]any) (any, error) {
|
||||
pg, err := page()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rows := []build.Unbuildable{}
|
||||
var unread []string
|
||||
for k, raw := range pg.Values {
|
||||
var u build.Unbuildable
|
||||
if err := json.Unmarshal(raw, &u); err != nil || u.Module == "" {
|
||||
unread = append(unread, k)
|
||||
continue
|
||||
}
|
||||
rows = append(rows, u)
|
||||
}
|
||||
sort.Slice(rows, func(i, j int) bool { return rows[i].Module < rows[j].Module })
|
||||
sort.Strings(unread)
|
||||
out := map[string]any{"line": build.UnbuildableLine(rows), "unbuildable": rows}
|
||||
if len(unread) > 0 {
|
||||
out["unreadable"] = unread
|
||||
}
|
||||
if pg.More {
|
||||
out["capped"] = versions.StateCapped{State: build.UnbuildableState, KeysRead: len(pg.Values), StateKeys: pg.Keys, StateBytes: pg.Bytes, InfoWords: pg.InfoWords}
|
||||
}
|
||||
return out, nil
|
||||
}}
|
||||
}
|
||||
|
||||
// unbuildablePage is the first page of the state unbuildable; a runtime that pages nothing is read key by key.
|
||||
func unbuildablePage(s stdio.KeptState) func() (versions.StatePage, error) {
|
||||
return func() (versions.StatePage, error) {
|
||||
pg, err := onePage(s.Page, s.Info)
|
||||
if !errors.Is(err, errors.ErrUnsupported) {
|
||||
return pg, err
|
||||
}
|
||||
keys, err := s.Keys()
|
||||
if err != nil {
|
||||
return versions.StatePage{}, err
|
||||
}
|
||||
out := versions.StatePage{Values: map[string]json.RawMessage{}}
|
||||
for _, k := range keys {
|
||||
if e, err := s.Get(k); err == nil && e != nil {
|
||||
out.Values[k] = e.Value
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
)
|
||||
|
||||
// handlers is where a part of the builder registers its handler of the events it hears: the event hub, or agedHub over it.
|
||||
type handlers interface {
|
||||
Handle(pattern string, handle func(jetstream.Msg)) error
|
||||
}
|
||||
|
||||
// ledgers are the builder's durable consumers' ledgers, one per consumer as a message's metadata names it (the events consumer
|
||||
// and each kept seat event's), writing each consumer's facts into the state consumers, which builder_health reads (design 47 H2
|
||||
// §794, H-A-435 and its addendum: the runner's runner_health form).
|
||||
type ledgers struct {
|
||||
sink process.ConsumerSink
|
||||
logf func(string, ...any)
|
||||
now func() time.Time
|
||||
|
||||
mu sync.Mutex
|
||||
byName map[string]*process.ConsumerLedger
|
||||
}
|
||||
|
||||
// of is the ledger of the consumer m came on, nil for a message that names none.
|
||||
func (l *ledgers) of(m jetstream.Msg) *process.ConsumerLedger {
|
||||
md, err := m.Metadata()
|
||||
if err != nil || md == nil || md.Consumer == "" {
|
||||
return nil
|
||||
}
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
if l.byName == nil {
|
||||
l.byName = map[string]*process.ConsumerLedger{}
|
||||
}
|
||||
led, ok := l.byName[md.Consumer]
|
||||
if !ok {
|
||||
led = process.NewConsumerLedger(md.Stream, md.Consumer, l.sink, l.now, l.logf)
|
||||
l.byName[md.Consumer] = led
|
||||
}
|
||||
return led
|
||||
}
|
||||
|
||||
// agedHub registers each handler on the hub so that every message it is handed is noted delivered on its consumer's ledger,
|
||||
// and applied once the handler acknowledges or ends it (the runner's agedHub); a message the handler asks again stays
|
||||
// unacknowledged, and grows old.
|
||||
type agedHub struct {
|
||||
hub handlers
|
||||
ledgers *ledgers
|
||||
}
|
||||
|
||||
func (a agedHub) Handle(pattern string, handle func(jetstream.Msg)) error {
|
||||
return a.hub.Handle(pattern, func(m jetstream.Msg) {
|
||||
led := a.ledgers.of(m)
|
||||
if led == nil {
|
||||
handle(m)
|
||||
return
|
||||
}
|
||||
led.Delivered(m)
|
||||
handle(&agedMsg{Msg: m, ledger: led})
|
||||
})
|
||||
}
|
||||
|
||||
// agedMsg is a message whose acknowledgement or end is noted applied on its consumer's ledger, once.
|
||||
type agedMsg struct {
|
||||
jetstream.Msg
|
||||
ledger *process.ConsumerLedger
|
||||
once sync.Once
|
||||
}
|
||||
|
||||
func (m *agedMsg) applied() { m.once.Do(func() { m.ledger.Applied(m.Msg, time.Time{}) }) }
|
||||
|
||||
func (m *agedMsg) Ack() error { defer m.applied(); return m.Msg.Ack() }
|
||||
|
||||
func (m *agedMsg) DoubleAck(ctx context.Context) error {
|
||||
defer m.applied()
|
||||
return m.Msg.DoubleAck(ctx)
|
||||
}
|
||||
|
||||
func (m *agedMsg) Term() error { defer m.applied(); return m.Msg.Term() }
|
||||
|
||||
func (m *agedMsg) TermWithReason(r string) error { defer m.applied(); return m.Msg.TermWithReason(r) }
|
||||
|
||||
// consumersSink writes a consumer's facts in the state consumers; while the state is not on the bus the bucket keeps the latest.
|
||||
func consumersSink(b *process.AwaitedBucket) process.ConsumerSink {
|
||||
return func(ctx context.Context, k string, v []byte) error {
|
||||
_, err := b.Put(ctx, k, v)
|
||||
if errors.Is(err, process.ErrNotYet) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
|
||||
// SlowEvent is how long the handling of one heard event may take before the builder says it in its journal (design 47 H2
|
||||
// §794): the line names the event's subject, the part that took it and how long it took.
|
||||
const SlowEvent = 100 * time.Millisecond
|
||||
|
||||
// eventTaker is one part of the builder that takes events off its durable consumer: take answers whether the message was its.
|
||||
type eventTaker struct {
|
||||
part string
|
||||
take func(jetstream.Msg) bool
|
||||
}
|
||||
|
||||
// takeEvent hands m to each taker in turn until one takes it, else to rest, and says the handling when it passed SlowEvent
|
||||
// (design 47 H2 §794). now is the clock, a test's when set.
|
||||
func takeEvent(m jetstream.Msg, takers []eventTaker, rest eventTaker, now func() time.Time, logf func(string, ...any)) {
|
||||
if now == nil {
|
||||
now = time.Now
|
||||
}
|
||||
start := now()
|
||||
part := rest.part
|
||||
for _, t := range takers {
|
||||
if t.take(m) {
|
||||
part = t.part
|
||||
break
|
||||
}
|
||||
}
|
||||
if part == rest.part {
|
||||
rest.take(m)
|
||||
}
|
||||
if took := now().Sub(start); took > SlowEvent {
|
||||
logf("the event %s took %s to handle, over %s: %s took it", m.Subject(), took.Round(time.Millisecond), SlowEvent, part)
|
||||
}
|
||||
}
|
||||
@@ -103,9 +103,16 @@ func revisionOf(t *testing.T, b *bed, module string) uint64 {
|
||||
return e.Revision()
|
||||
}
|
||||
|
||||
// TestAGoneArtifactIsBuiltAgain: 1937/AC3 and AC4, one whole test of the two (design 47 H2 §819: the builder's integration count does
|
||||
// not rise with the work queue publishes' whole test).
|
||||
func TestAGoneArtifactIsBuiltAgain(t *testing.T) {
|
||||
t.Run("a done key whose blob is gone is built again at start", aDoneKeyWhoseBlobIsGoneIsBuiltAgainAtStart)
|
||||
t.Run("artifacts-gone leads the builder to build the key again", artifactsGoneLeadsTheBuilderToBuildTheKeyAgain)
|
||||
}
|
||||
|
||||
// 1937/AC3 (integration): the builder, started whole, finds the done key of the trunk's fingerprint naming a blob held
|
||||
// answers gone, builds it again as a new revision whose step says so, and leaves the rebuilt key done.
|
||||
func TestADoneKeyWhoseBlobIsGoneIsBuiltAgainAtStart(t *testing.T) {
|
||||
func aDoneKeyWhoseBlobIsGoneIsBuiltAgainAtStart(t *testing.T) {
|
||||
b := startBed(t)
|
||||
eventsConsumer(t, b, "n1")
|
||||
manifest := b.archive(t, "ca-trust", "modules/ca-trust", commit)
|
||||
@@ -135,7 +142,7 @@ func TestADoneKeyWhoseBlobIsGoneIsBuiltAgainAtStart(t *testing.T) {
|
||||
|
||||
// 1937/AC4 (integration): a done key whose blob held answers held at start stays done; when the controller says
|
||||
// artifacts-gone for it, the builder asks again and builds it again.
|
||||
func TestArtifactsGoneLeadsTheBuilderToBuildTheKeyAgain(t *testing.T) {
|
||||
func artifactsGoneLeadsTheBuilderToBuildTheKeyAgain(t *testing.T) {
|
||||
b := startBed(t)
|
||||
eventsConsumer(t, b, "n1")
|
||||
manifest := b.archive(t, "ca-trust", "modules/ca-trust", commit)
|
||||
|
||||
@@ -9,7 +9,6 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
|
||||
"maps"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -20,6 +19,8 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
|
||||
|
||||
"github.com/nats-io/nats-server/v2/server"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
@@ -35,8 +36,10 @@ import (
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/ociartifactregistry"
|
||||
sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
|
||||
runner "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/steprun"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscalls"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"git.novox.be/novox/mesh-sdk/go/seatdef"
|
||||
"git.novox.be/novox/mesh-sdk/go/views/viewstest"
|
||||
|
||||
"mesh-delivery-builder/internal/build"
|
||||
@@ -87,6 +90,34 @@ type fakeBuildModules struct {
|
||||
// hang, while it holds a channel, keeps every build step running until the channel is closed: a step that never ends,
|
||||
// so the builder's step bound runs out (design 51's round-2 case).
|
||||
hang atomic.Pointer[chan struct{}]
|
||||
// big, while set, keeps every passed step's answer as its answer object, as one over what the steps state holds per value
|
||||
// is kept, handed through the build seat's verb answer in chunks (design 47 H2 H-A-412); answers holds those objects by name.
|
||||
big atomic.Bool
|
||||
answers sync.Map
|
||||
}
|
||||
|
||||
// serveAnswers answers the verb answer of the seat node-build-go on every node as the build module's holder does: the
|
||||
// answer object the ask's step key names, in chunks (design 47 H2 H-A-412).
|
||||
func (f *fakeBuildModules) serveAnswers(t *testing.T, nc *nats.Conn) {
|
||||
sub, err := nc.Subscribe("mesh.seat.node-build-go.call.answer.>", func(m *nats.Msg) {
|
||||
var in struct{ Ask, Call string }
|
||||
_ = json.Unmarshal(m.Data, &in)
|
||||
body, ok := f.answers.Load(sdkstep.AnswerObjectName(in.Ask))
|
||||
if !ok {
|
||||
_ = m.Respond([]byte(`{"error":{"kind":"not-found","message":"no answer of ` + in.Ask + `"}}`))
|
||||
return
|
||||
}
|
||||
of := (len(body.([]byte)) + buscalls.SeatChunkBytes - 1) / buscalls.SeatChunkBytes
|
||||
answer, _ := json.Marshal(map[string]any{"result": map[string]any{"ask": in.Ask, "chunks": of}})
|
||||
_ = m.Respond(answer)
|
||||
go func() {
|
||||
_ = buscalls.SeatChunks{Seat: "node-build-go", Verb: "answer", Call: in.Call}.Send(nc, body.([]byte), buscalls.SeatAckWithin, t.Logf)
|
||||
}()
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
||||
}
|
||||
|
||||
// open lets the nodes pull again and wakes their pullers.
|
||||
@@ -171,6 +202,16 @@ func (f *fakeBuildModules) serve(t *testing.T, module string, nodes ...string) {
|
||||
// both forms overlaps the named worker, which a work queue refuses.
|
||||
put := func(key string, value any) (uint64, error) {
|
||||
raw, _ := value.(json.RawMessage)
|
||||
// big: an ended step's answer is kept as its answer object and the key names it, as the build module's step run keeps
|
||||
// an answer over what the steps state holds per value (steprun's keepOver; design 47 G A-290, H2 H-A-412).
|
||||
if step, err := sdkstep.DecodeStep(raw); f.big.Load() && err == nil && step.State == sdkstep.StepPassed && len(step.Result) > 0 {
|
||||
object := sdkstep.AnswerObjectName(step.ID)
|
||||
f.answers.Store(object, []byte(step.Result))
|
||||
step.Result, step.AnswerObject = nil, object
|
||||
if raw, err = sdkstep.EncodeStep(step); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
return kv.Put(f.ctx, key, raw)
|
||||
}
|
||||
call := func(name string) runner.StepFunc {
|
||||
@@ -299,6 +340,12 @@ func startBedWith(t *testing.T, closed bool) *bed {
|
||||
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: builds.StateBucket(builds.BuilderModule, versions.BoundsState), History: 1, LimitMarkerTTL: time.Second}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The seat mesh-builds' work queue and its one worker, as the broker derives them from the seat's definition (design 47
|
||||
// H2 §777), and the state taken the builder takes each ask once in (§775, §820).
|
||||
b.seatQueue(t)
|
||||
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: builds.StateBucket(builds.BuilderModule, process.TakenState), History: 1, LimitMarkerTTL: time.Second}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.out = &saidLog{}
|
||||
// The builder's module log, as the runtime raises it (ADR 0297): its stream on the log's subjects.
|
||||
logStream := buscompose.LogStreamName(builds.BuilderModule, builds.BuilderLog)
|
||||
@@ -320,6 +367,7 @@ func startBedWith(t *testing.T, closed bool) *bed {
|
||||
b.fake.serve(t, "mesh-build-go", "n1", "n2")
|
||||
b.fake.serve(t, "mesh-build-node", "n1", "n2")
|
||||
b.fake.serve(t, "mesh-build-image", "n1") // the image build module runs on n1 only
|
||||
b.fake.serveAnswers(t, ctl)
|
||||
b.report(t, "mesh-build-go", "n1", "go", "go1.27.1", "1")
|
||||
b.report(t, "mesh-build-go", "n2", "go", "go1.27.1", "1")
|
||||
b.report(t, "mesh-build-node", "n1", "typescript", "tsc5", "1")
|
||||
@@ -692,10 +740,13 @@ func TestBuilderMakesADoneKeyWithTheDigests(t *testing.T) { // 1607/AC7
|
||||
func TestAnArchiveOnlyModuleAndARootModuleAreBuilt(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
name, path string
|
||||
two bool
|
||||
}{{"no toolchain", "modules/ca-trust", false}, {"root", meshcatalogclient.RootPath, false}, {"two instances together", "modules/ca-trust", true}} {
|
||||
two, big bool
|
||||
}{{"no toolchain", "modules/ca-trust", false, false}, {"root", meshcatalogclient.RootPath, false, false}, {"two instances together", "modules/ca-trust", true, false},
|
||||
// Design 47 H2 H-A-412: a build step's end over what the steps state holds is read through the seat's verb answer.
|
||||
{"an answer over the steps state", "modules/ca-trust", false, true}} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
b := startBed(t)
|
||||
b.fake.big.Store(c.big)
|
||||
b.archive(t, "ca-trust", c.path, commit)
|
||||
b.start(t, "n1", nil)
|
||||
if c.two {
|
||||
@@ -705,7 +756,7 @@ func TestAnArchiveOnlyModuleAndARootModuleAreBuilt(t *testing.T) {
|
||||
if c.name == "no toolchain" && (len(v.Toolchains) != 0 || len(v.Artifacts) != 1 || strings.HasPrefix(v.Step, "one node (")) {
|
||||
t.Errorf("build = %+v, want done naming no toolchain", v)
|
||||
}
|
||||
if c.two && (len(v.Artifacts) != 1 || v.Artifacts[0].Kind != "archive" || v.Artifacts[0].Digest != digestOf("files")) {
|
||||
if (c.two || c.big) && (len(v.Artifacts) != 1 || v.Artifacts[0].Kind != "archive" || v.Artifacts[0].Digest != digestOf("files")) {
|
||||
t.Errorf("build = %+v, want the archive's digest", v)
|
||||
}
|
||||
if n := b.fake.count("mesh-build-go.build_go_build"); c.two && n < 1 { // each instance that built, on one node
|
||||
@@ -1252,3 +1303,29 @@ func (r *registry) numberedOf(name string) []string {
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// seatQueue makes the seat mesh-builds' work-queue stream and its worker from the seat's definition in the contracts
|
||||
// module, as the broker derives them (buscompose.SeatQueue; design 47 H2 §777).
|
||||
func (b *bed) seatQueue(t *testing.T) {
|
||||
t.Helper()
|
||||
raw, err := os.ReadFile(filepath.Join("..", "..", "..", "mesh-delivery-contracts", "seats", meshbuilds.Seat, "seat.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
def, err := seatdef.Parse(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stream, worker, ok, err := buscompose.SeatQueue(meshbuilds.Seat, def.Accepts)
|
||||
if err != nil || !ok {
|
||||
t.Fatalf("the seat's work queue: %v %v", ok, err)
|
||||
}
|
||||
if _, err := b.js.CreateStream(b.ctx, jetstream.StreamConfig{Name: stream.Name, Subjects: stream.Subjects, Retention: jetstream.WorkQueuePolicy,
|
||||
MaxAge: time.Duration(stream.MaxAge) * time.Second, MaxMsgs: int64(stream.MaxMsgs), Duplicates: time.Duration(stream.DuplicatesSeconds) * time.Second}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := b.js.CreateOrUpdateConsumer(b.ctx, stream.Name, jetstream.ConsumerConfig{Durable: worker.Name, FilterSubjects: worker.Filters, AckPolicy: jetstream.AckExplicitPolicy,
|
||||
AckWait: time.Duration(worker.AckWaitSeconds) * time.Second, MaxDeliver: worker.MaxDeliver, MaxAckPending: worker.MaxAckPending}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
"os/signal"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
@@ -37,6 +38,7 @@ import (
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/operatorask"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"git.novox.be/novox/mesh-sdk/go/seatlist"
|
||||
@@ -190,7 +192,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
Claimer: build.Claimer{Store: kvStore{buildsKV}, By: cred.Node + "." + cred.Module},
|
||||
Seats: seats,
|
||||
Asks: buildrun.Asker{Name: meshbuilds.Seat, Seats: seats, Publisher: asks, Waiter: buildrun.LostAskWaiter{Inner: heard, Dead: dead, Hearing: hearing}, Bound: bound, Dropper: buildrun.SeatDrops{Caller: calls},
|
||||
Bounds: &stepBounds{bounds: pubBoundsKV.KV, js: js, now: time.Now, logf: logger.Printf}, Expiry: stepExpiry},
|
||||
Bounds: &stepBounds{bounds: pubBoundsKV.KV, js: js, now: time.Now, logf: logger.Printf}, Expiry: stepExpiry,
|
||||
// Design 47 H2 H-A-412: a step's end over what the steps state holds is read through its build seat's verb answer, in
|
||||
// chunks (process.Conn.CallFile, the chunked-call form), on the node that ran it.
|
||||
Outputs: calls.CallFile},
|
||||
Held: heldOnTheBus{js: js, seats: seats},
|
||||
// An ask that can no longer be answered is asked again (issue 1945), read at the step bound: a context's bound.
|
||||
Gone: heldOnTheBus{js: js, seats: seats},
|
||||
@@ -235,6 +240,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
return emitEvent(context.WithoutCancel(ctx), js, cred, event, id, body)
|
||||
}, waitBound, logger.Printf)
|
||||
waitsBucket.Ready(func(kv jetstream.KeyValue) { b.Waits.(*waitsKept).start(ctx, kv) })
|
||||
// The plans no seat builds (design 47 H2 §795, H-A-425): kept in the state unbuildable, read by builder_unbuildable.
|
||||
b.Unbuildables = unbuildableKept{b: process.AwaitBucket(ctx, js, conn, builds.StateBucket(cred.Module, build.UnbuildableState), "the plans no seat builds are said, and builder_unbuildable lists none", logger.Printf)}
|
||||
// The versions a build reserves at its start (design 47 G §369): its state reserved, and the catalog's versions-filled
|
||||
// and version-released heard; nothing is reserved before the fill.
|
||||
reservedKV := process.AwaitBucket(ctx, js, conn, own.reserved, "no version is reserved: builds are made unreserved", logger.Printf)
|
||||
@@ -245,6 +252,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
reserved.say = func(ctx context.Context, p meshbuilds.PackagePublished) error {
|
||||
return emitSeatEvent(context.WithoutCancel(ctx), js, cred, meshbuilds.SubjectPackagePublished, "package-published-"+p.Module+"-"+p.Version+"-"+p.Artifact, p)
|
||||
}
|
||||
// A package's number reserved is said as the seat mesh-builds' version-reserved, so the indexer re-pins at the reservation (§805).
|
||||
reserved.names = versions.NamesFrom(versions.EntriesOf(branches))
|
||||
reserved.r.Names = reserved.names // a Go package's module path names its tag's directory (§796)
|
||||
reserved.sayReserved = func(ctx context.Context, ev meshbuilds.VersionReservedEvent) error {
|
||||
return emitSeatEvent(context.WithoutCancel(ctx), js, cred, meshbuilds.SubjectVersionReserved, "version-reserved-"+ev.Module+"-"+ev.Version+"-"+ev.Artifact, ev)
|
||||
}
|
||||
b.Reserved = reserved
|
||||
// A take-over of a series' reservation by another fingerprint withdraws the older build (H-A-225).
|
||||
reserved.withdraw = b.Withdraw
|
||||
@@ -255,11 +268,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// The versions' hand acts (builder_publish_again, builder_floor_again) come through reasks, as builder_build_again.
|
||||
acts := versions.Acts{Numberer: numberer, DropReserved: reserved.dropped, Reconsider: b.Reconsider}
|
||||
// A reserved version left unpublished is published again, by the builder on an event or by a person's tool (design 47 G
|
||||
// §588): its states publish-queue and acted, bound once the bus holds them.
|
||||
queueKV := process.AwaitBucket(ctx, js, conn, own.queue, "no version is published again on an event or by builder_publish_again", logger.Printf)
|
||||
// §588): its items on the work queue publishes, the module's own stream of its old streams section (design 47 H2 §793, H-A-454).
|
||||
actedKV := process.AwaitBucket(ctx, js, conn, own.acted, "no version is published again on an event or by builder_publish_again", logger.Printf)
|
||||
republish := &versions.Republisher{Reserved: reserved.r, Queue: awaitedQueue{awaitedReserved{awaitedVersions{b: queueKV, bucket: own.queue}}},
|
||||
Entry: entryVia(calls.Call), Send: sendVia(calls.Call),
|
||||
pq := &publishQueue{js: js, logf: logger.Printf, binding: publishesOf(cred.Module)}
|
||||
republish := &versions.Republisher{Reserved: reserved.r, Put: pq.put,
|
||||
Entry: entryVia(calls.Call), Send: sendVia(calls.CallKeptAsk),
|
||||
Unreleased: reserved.unreleasedWords, Logf: logger.Printf,
|
||||
Holders: holders, // a registry seat's holder from the seat-holders view, from the start (design 47 G A-416, A-416a)
|
||||
Started: time.Now()} // a build ended from now on is of the live class, sent first (H-A-290 (1))
|
||||
@@ -287,6 +300,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
pq.ackWait = publishSet.PauseMax // a sent item unheard is delivered again after the longest pause (H-A-440 (1))
|
||||
releasedKept, err := numberFile(getenv("MESH_BUILDER_RELEASED_KEPT_FILE"), versions.DefaultReleasedKept)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -305,15 +319,22 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
},
|
||||
markHeld: reserved.markHeld, moveElsewhere: reserved.movedElsewhere,
|
||||
allDone: func(ctx context.Context) ([]versions.ReservedVersion, error) { return reserved.r.All(ctx) },
|
||||
send: republish.SendQueued,
|
||||
retired: reserved.r.Retired} // only each module's newest done build is asked at the start (H-A-291 (1))
|
||||
// A version that never publishes (withdrawn or taken over) has its publish item ended by the queue too (H-A-290 (2)).
|
||||
republish.Ended = items.Done
|
||||
republish.Ended = items.Ended
|
||||
pq.take = items.take
|
||||
reasksKV.Ready(func(jetstream.KeyValue) { items.AskAllDone(ctx) })
|
||||
wirePublishItems(reserved, items, awaitedReserved{awaitedVersions{b: actedKV, bucket: own.acted}}, releasedKept)
|
||||
acts.Republish = republish.Republish
|
||||
acts.PublishDrop = func(ctx context.Context, module, version, by, why string) (string, error) {
|
||||
return pq.drop(ctx, items, module, version, by, why)
|
||||
}
|
||||
acts.PublishFirst = func(ctx context.Context, module, version, _, _ string) (string, error) {
|
||||
return pq.first(ctx, items, module, version)
|
||||
}
|
||||
// builder_reserve_done: a done trunk build that reserved none is given a version, and its publish queued as publish again does.
|
||||
acts.ReserveDone = reserveDoneAct(b.ReserveDone, republish.Republish, logger.Printf)
|
||||
again := &republishing{rp: republish}
|
||||
// The operator is asked once a key's event builds are spent (design 47 G §568): the builder's asker on the seat operator-channel.
|
||||
decided := &decisions{subject: oc.SubjectDecided(cred.Module)}
|
||||
retakeActed := awaitedReserved{awaitedVersions{b: actedKV, bucket: own.acted}}
|
||||
@@ -327,8 +348,13 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
return fmt.Errorf("hearing the operator's answers: %w", err)
|
||||
}
|
||||
retake := &retaking{rt: &build.Retaker{Builder: b, Acted: retakeActed, Entry: entryVia(calls.Call), Ask: operator.Ask, Logf: logger.Printf}}
|
||||
queueKV.Ready(again.bound(ctx, js, logger.Printf))
|
||||
actedKV.Ready(again.bound(ctx, js, logger.Printf))
|
||||
// What waits on the work queue is pulled, and the old state's items are put on it once (§793).
|
||||
go func() {
|
||||
if err := pq.learnAll(ctx); err != nil {
|
||||
logger.Printf("the publishes waiting on the work queue %s are not pulled yet: %v; each holder's are pulled when an item of it is put", versions.PublishesLocal, err)
|
||||
}
|
||||
moveOld(ctx, js, own.queue, cred.Module, republish, calls.Call, logger.Printf)
|
||||
}()
|
||||
errs := make(chan error, 10)
|
||||
b.Members = versions.ModuleFrom(versions.EntriesOf(branches)) // the part a resolved package names (design 47 G §466)
|
||||
// The builder's log (ADR 0571 §1): each build's start and end in its module log `build-log`, read by the view, the
|
||||
@@ -453,18 +479,19 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
floor := floorWake{b: b, waits: floorWaits, logf: logger.Printf}
|
||||
// And the four registry seats' published, refused and removed (DLV-B4), which move a numbered version.
|
||||
seatHeard := seatHearer{n: numberer, b: b, logf: logger.Printf, pub: pubs, res: reserved}
|
||||
events := func(m jetstream.Msg) {
|
||||
if decided.heard(m) { // the operator's answer to an ask about a key (§568)
|
||||
return
|
||||
}
|
||||
if retake.heard(ctx, m, logger.Printf) { // a key failed on a transport fault, built again when its cause passed (§568)
|
||||
return
|
||||
}
|
||||
if reads.heard(m, logger.Printf) || heardTools.Reported(ctx, m) || heardTools.Running(ctx, m) || floor.heard(ctx, m) || reserved.heard(ctx, m) || seatHeard.heard(ctx, m) { // reads: the node-engine seat's manifest-reads-changed (ADR 0618 §4)
|
||||
return
|
||||
}
|
||||
heardGone(ctx, b, m, logger.Printf)
|
||||
// Each part takes the events it hears in turn; a handling over SlowEvent is said with the part that took it (design 47 H2 §794).
|
||||
takers := []eventTaker{
|
||||
{"the operator's answer to an ask about a key", decided.heard}, // §568
|
||||
{"the retake of a key failed on a transport fault", func(m jetstream.Msg) bool { return retake.heard(ctx, m, logger.Printf) }}, // §568
|
||||
{"what a node's engine reads", func(m jetstream.Msg) bool { return reads.heard(m, logger.Printf) }}, // manifest-reads-changed (ADR 0618 §4)
|
||||
{"the toolchains reported", func(m jetstream.Msg) bool { return heardTools.Reported(ctx, m) }},
|
||||
{"the toolchains running", func(m jetstream.Msg) bool { return heardTools.Running(ctx, m) }},
|
||||
{"the floor", func(m jetstream.Msg) bool { return floor.heard(ctx, m) }},
|
||||
{"the reservations", func(m jetstream.Msg) bool { return reserved.heard(ctx, m) }},
|
||||
{"the registry seats' outcomes", func(m jetstream.Msg) bool { return seatHeard.heard(ctx, m) }},
|
||||
}
|
||||
gone := eventTaker{"the artifacts gone", func(m jetstream.Msg) bool { heardGone(ctx, b, m, logger.Printf); return true }}
|
||||
events := func(m jetstream.Msg) { takeEvent(m, takers, gone, nil, logger.Printf) }
|
||||
// The seats' events (artifacts-gone, the registry seats' words, the build seats' toolchain-reported, the indexer's take-in.taken and
|
||||
// manifest-reads-changed and the seat mesh-conditions' condition-cleared: seatEventPatterns); the owners' views events and the
|
||||
// catalog half's catalog-revision and registration-changed and the gate's source-moved are the views reader's (a message has
|
||||
@@ -473,6 +500,18 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// modules again (design 47 G A-82, §204).
|
||||
// The runner's build step ask, the builder's one entry point for a candidate (design 47 G A-97).
|
||||
b.StepEntry = newEntryWaits(branches, half.manifests).entry
|
||||
// The seat's work queue (design 47 H2 §777): pulled through its one worker, each ask taken once in the state taken.
|
||||
takenBound, err := askTakenBound()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var ackWait atomic.Int64 // the worker's ack wait, read when it is bound
|
||||
ackWait.Store(int64(buscompose.SeatQueueDefaultAckWait))
|
||||
steps := &stepQueue{node: cred.Node, builder: b, taken: calls.State(process.TakenState), takenBound: takenBound,
|
||||
inProgress: inProgressOf(seatlist.Definition(seatGate.Reader, meshbuilds.Seat), func() time.Duration { return time.Duration(ackWait.Load()) }),
|
||||
say: func(ctx context.Context, subject, id string, body any) error {
|
||||
return emitSeatEvent(ctx, js, cred, subject, id, body)
|
||||
}, logf: logger.Printf, now: time.Now, running: map[string]*runningAsk{}}
|
||||
// The seat mesh-builds' verbs reserved, answered here (A-115, A-127): the reservations, the catalog half's carriers and the seat
|
||||
// mesh-catalog's versions; and published, from the reservations and releases (H-A-328).
|
||||
stopReserved, err := calls.AnswerSeat(meshbuilds.Seat, seatlist.Definition(seatGate.Reader, meshbuilds.Seat), map[string]process.SeatHandler{
|
||||
@@ -481,18 +520,27 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
logger.Printf("reserved: no entry carries the name %s; the entries looked at: %s", name, versions.LookedAt(versions.EntriesOf(branches)))
|
||||
}),
|
||||
// published (H-A-328): whether a package built at a fingerprint is published, asked by the runner writing a wait on it.
|
||||
meshbuilds.VerbPublished: publishedVerb(ctx, reserved.r.PublishedOf)})
|
||||
meshbuilds.VerbPublished: publishedVerb(ctx, reserved.r.PublishedOf),
|
||||
// drop (§778; H-A-401, H-A-404): an ask its caller no longer waits for ends could-not-run naming the drop.
|
||||
meshbuilds.VerbDrop: dropVerb(steps),
|
||||
// publish-queue (§793; H-A-454): the work queue publishes read from its own stream, for builder_publish_queue.
|
||||
meshbuilds.VerbPublishQueue: publishQueueVerb(ctx, pq)})
|
||||
if err != nil {
|
||||
return fmt.Errorf("answering the verb reserved: %w", err)
|
||||
}
|
||||
defer stopReserved()
|
||||
if err := serveSteps(ctx, conn, js, cred, b, logger.Printf); err != nil {
|
||||
if err := serveSteps(ctx, conn, js, steps, &ackWait); err != nil {
|
||||
return err
|
||||
}
|
||||
seatGate.Follow(seatlist.ProcessBus(calls.Call, hub), func() { seatsChanged(ctx) }, logger.Printf)
|
||||
// A change of the seat list judges every plan no seat builds again and says them in one line (design 47 H2 §795).
|
||||
// Every message the builder's own handlers take is noted on its consumer's ledger, in the state consumers that
|
||||
// builder_health reads (design 47 H2 §794, H-A-435).
|
||||
aged := agedHub{hub: hub, ledgers: &ledgers{logf: logger.Printf, sink: consumersSink(process.AwaitBucket(ctx, js, conn,
|
||||
builds.StateBucket(cred.Module, process.ConsumersState), "builder_health reads no consumer's age", logger.Printf))}}
|
||||
seatGate.Follow(seatlist.ProcessBus(calls.Call, aged), func() { seatsChanged(ctx); b.SayUnbuildable(ctx) }, logger.Printf)
|
||||
half.holders = func() { seatsChanged(ctx) }
|
||||
for _, pattern := range seatEventPatterns {
|
||||
if err := hub.Handle(pattern, events); err != nil {
|
||||
if err := aged.Handle(pattern, events); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
@@ -510,6 +558,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
if err := b.Sweep(ctx, half.pictured()...); err != nil {
|
||||
errs <- err
|
||||
}
|
||||
b.SayUnbuildable(ctx) // at the start, once the first views are in: the plans no seat builds in one line (§795)
|
||||
}
|
||||
branches.Changed = half.Changed(ctx)
|
||||
// A catalog revision the builder hears wakes a floor whose tags or requirements were silent (design 47 G A-119).
|
||||
|
||||
+180
@@ -0,0 +1,180 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
|
||||
"mesh-delivery-builder/internal/versions"
|
||||
)
|
||||
|
||||
// Design 47 H2 §793 (whole, against the bus): the builder's work queue publishes, its pull consumers made as it learns a holder.
|
||||
// A live publish enqueued behind 100 again items while the builder was away is sent first; a dropped item is never sent, and
|
||||
// put again within the duplicates window is not stored; the move of the old state run twice puts each item once, and asks the
|
||||
// bus module to drop the old state. The seat mesh-builds' verb publish-queue reads the stream: counts per holder and class, the
|
||||
// oldest and newest, the live item listed first (H-A-454).
|
||||
func TestThePublishQueueSendsLiveFirstAndNeverADroppedItem(t *testing.T) {
|
||||
b := startBed(t)
|
||||
ctx := b.ctx
|
||||
// The stream as the manifest's old streams section declares it: the module's own, retention work-queue, duplicates 24 hours.
|
||||
bind, _ := publishesOf("mesh-delivery-builder")()
|
||||
if _, err := b.js.CreateOrUpdateStream(ctx, jetstream.StreamConfig{Name: bind.Stream, Subjects: []string{bind.Subject + ".publish.>"},
|
||||
Retention: jetstream.WorkQueuePolicy, Discard: jetstream.DiscardNew, Duplicates: 24 * time.Hour, MaxAge: 7 * 24 * time.Hour}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kv := &itemKV{vals: map[string][]byte{}, revs: map[string]uint64{}}
|
||||
var mu sync.Mutex
|
||||
var sent []string
|
||||
published := map[string]bool{} // the modules whose publish is heard
|
||||
sentCh := make(chan string, 200)
|
||||
items := &publishItems{reasks: func() jetstream.KeyValue { return kv }, set: publishSettings{Pause: time.Second, PauseMax: time.Minute, Tries: 5},
|
||||
now: time.Now, logf: t.Logf,
|
||||
reserved: func(_ context.Context, m, v string) (versions.ReservedVersion, bool, error) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
return versions.ReservedVersion{Module: m, Version: v, Artifacts: map[string]string{m + ":a": "sha256:a"}, Published: map[string]bool{m + ":a": published[m]}, BuiltAt: time.Now()}, true, nil
|
||||
},
|
||||
send: func(_ context.Context, q versions.QueuedPublish) versions.SendFate {
|
||||
mu.Lock()
|
||||
sent = append(sent, q.Module)
|
||||
mu.Unlock()
|
||||
sentCh <- q.Module
|
||||
return versions.SendFate{Sent: true}
|
||||
}}
|
||||
pq := &publishQueue{js: b.js, ackWait: time.Minute, logf: t.Logf, take: items.take, binding: publishesOf("mesh-delivery-builder")}
|
||||
item := func(module, class string) versions.QueuedPublish {
|
||||
return versions.QueuedPublish{V: 1, Holder: "oci", Module: module, Version: "0.1.0", Artifacts: map[string]string{module + ":a": "sha256:a"}, Class: class, Queued: time.Now()}
|
||||
}
|
||||
away := func(q versions.QueuedPublish) { // put while the builder was away: no holder learnt, nothing pulled
|
||||
data, _ := json.Marshal(q)
|
||||
if _, err := b.js.Publish(ctx, pq.subject(bind, q), data, jetstream.WithMsgID(versions.PublishMsgID(q.Holder, q.Class, q.Module, q.Version))); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for i := range 100 {
|
||||
away(item(fmt.Sprintf("a%03d", i), versions.PublishClassStart))
|
||||
}
|
||||
away(item("dropme", versions.PublishClassStart))
|
||||
away(item("live", versions.PublishClassLive))
|
||||
if answer, err := pq.drop(ctx, items, "dropme", "0.1.0", "op@node", "a test drops it"); err != nil {
|
||||
t.Fatalf("drop: %v", err)
|
||||
} else {
|
||||
t.Log(answer)
|
||||
}
|
||||
if dup, err := pq.put(ctx, item("dropme", versions.PublishClassStart)); err != nil || !dup {
|
||||
t.Errorf("a dropped item put again within the window: duplicate %v (%v), want it not stored", dup, err)
|
||||
}
|
||||
raw, err := publishQueueVerb(ctx, pq)("op@node", json.RawMessage(`{}`))
|
||||
if a, ok := raw.(meshbuilds.PublishQueueAnswer); err != nil || !ok || a.Total != 101 || len(a.Counts) != 2 || a.Counts[0].Class != versions.PublishClassStart ||
|
||||
a.Counts[0].Count != 100 || a.Counts[1].Count != 1 || a.Counts[0].Oldest.After(a.Counts[0].Newest) || len(a.Queue) != versions.PublishQueueListCap ||
|
||||
a.Queue[0].Module != "live" || a.Capped {
|
||||
t.Errorf("publish-queue answered %+v (%v), want 101 items, oci's 100 again and 1 live, fifty listed, the live first", raw, err)
|
||||
}
|
||||
if err := pq.learnAll(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case first := <-sentCh:
|
||||
if first != "live" {
|
||||
t.Errorf("the first item sent is %s, want the live one, enqueued behind 100 again items", first)
|
||||
}
|
||||
case <-time.After(30 * time.Second):
|
||||
t.Fatal("nothing was sent")
|
||||
}
|
||||
for range 4 { // the again items in flight: at most four held unacknowledged, their publish unheard
|
||||
select {
|
||||
case <-sentCh:
|
||||
case <-time.After(30 * time.Second):
|
||||
t.Fatal("the again items were not sent")
|
||||
}
|
||||
}
|
||||
deadline := time.After(60 * time.Second)
|
||||
for { // each publish sent is heard: its item acknowledged, the next sent
|
||||
mu.Lock()
|
||||
n, done := len(sent), slices.Clone(sent)
|
||||
for _, m := range done {
|
||||
published[m] = true
|
||||
}
|
||||
mu.Unlock()
|
||||
if n >= 101 {
|
||||
break
|
||||
}
|
||||
for _, m := range done {
|
||||
items.heard(ctx, m, "0.1.0")
|
||||
}
|
||||
select {
|
||||
case <-sentCh:
|
||||
case <-deadline:
|
||||
t.Fatalf("sent %d of 101", n)
|
||||
}
|
||||
}
|
||||
mu.Lock()
|
||||
if slices.Contains(sent, "dropme") || len(sent) != 101 {
|
||||
t.Errorf("sent %d items (dropme among them: %v), want the 101 and never the dropped one", len(sent), slices.Contains(sent, "dropme"))
|
||||
}
|
||||
mu.Unlock()
|
||||
|
||||
// The move of the old state, run twice, puts each item once (at another holder, so the pull of oci is not asked).
|
||||
old, err := b.js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: builds.StateBucket("mesh-delivery-builder", versions.PublishQueueState)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
call := func(string, any) (json.RawMessage, error) {
|
||||
return json.Marshal(meshcatalog.VersionsAnswer{Module: "m"})
|
||||
}
|
||||
rv := newReservations(&memReserve{vals: map[string][]byte{}, revs: map[string]uint64{}}, call, func() jetstream.KeyValue { return &boundsKV{keys: map[string]bool{}} },
|
||||
&findings{module: builds.BuilderModule, w: &memFindings{vals: map[string][]byte{}}, logf: t.Logf, now: time.Now}, nil, t.Logf)
|
||||
rv.filled.Store(true)
|
||||
for _, m := range []string{"m1", "m2", "m3"} {
|
||||
v, err := rv.Reserve(ctx, m, "0.1", []string{m + ":a"}, versions.Source{Fingerprint: "src2:" + m})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := rv.Built(ctx, m, v.Version, map[string]string{m + ":a": "sha256:a"}, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
raw, _ := json.Marshal(versions.QueuedPublish{V: 1, Holder: "go", Module: m, Version: v.Version, Artifacts: map[string]string{m + ":a": "sha256:a"}, Class: "asked-again-at-start"})
|
||||
if _, err := old.Put(ctx, versions.QueueKey("go", m, v.Version), raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
rp := &versions.Republisher{Reserved: rv.r, Put: func(ctx context.Context, q versions.QueuedPublish) (bool, error) {
|
||||
data, _ := json.Marshal(q)
|
||||
ack, err := b.js.Publish(ctx, pq.subject(bind, q), data, jetstream.WithMsgID(versions.PublishMsgID(q.Holder, q.Class, q.Module, q.Version)))
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return ack.Duplicate, nil
|
||||
}, Started: time.Now(), Logf: t.Logf}
|
||||
var drops []string
|
||||
drop := func(tool string, body any) (json.RawMessage, error) {
|
||||
drops = append(drops, fmt.Sprint(tool, body))
|
||||
return json.RawMessage(`{}`), nil
|
||||
}
|
||||
for range 2 {
|
||||
moveOld(ctx, b.js, builds.StateBucket("mesh-delivery-builder", versions.PublishQueueState), "mesh-delivery-builder", rp, drop, t.Logf)
|
||||
}
|
||||
st, err := b.js.Stream(ctx, bind.Stream)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
info, err := st.Info(ctx, jetstream.WithSubjectFilter(bind.Subject+".publish.go.>"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(info.State.Subjects) != 3 || info.State.Subjects[bind.Subject+".publish.go.again.m1.0=2E1=2E0"] != 1 || len(drops) != 2 {
|
||||
t.Errorf("the move run twice left %v on the work queue, dropped %v: want each of the three once, as again, and the old state dropped through the bus module", info.State.Subjects, drops)
|
||||
}
|
||||
}
|
||||
@@ -7,20 +7,28 @@ import (
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"mesh-delivery-builder/internal/build"
|
||||
"mesh-delivery-builder/internal/versions"
|
||||
)
|
||||
|
||||
// **A version's publish is the builder's own work until its release is written** (design 47 G A-457a, §588): each reservation of a
|
||||
// done build is a publish item in the builder's re-ask state, the mechanism builder_reserve_done and publish again use. It is
|
||||
// asked when the build ends and at the builder's start; each try queues the publish and writes the item again with one more try
|
||||
// and a longer pause; the release deletes it; after its tries it ends loudly and stays readable in builder_publish_queue.
|
||||
// **A version's publish is the builder's own work until its release is written** (design 47 G A-457a, §588; H2 §793): each
|
||||
// reservation of a done build is a publish item in the builder's re-ask state, asked when the build ends and at the builder's
|
||||
// start; its try puts the publish on the work queue publishes, and each send counts one try in the item and hands a failed one
|
||||
// back with a longer pause; the release deletes it; after its tries it ends loudly and stays readable in builder_publish_queue.
|
||||
|
||||
// publishSettings are the settings publish-pause, publish-pause-max and publish-tries.
|
||||
type publishSettings struct {
|
||||
@@ -28,8 +36,8 @@ type publishSettings struct {
|
||||
Tries int
|
||||
}
|
||||
|
||||
// defaultPublishSettings are the settings' defaults: 30 s, 30 min and 8 tries.
|
||||
var defaultPublishSettings = publishSettings{Pause: 30 * time.Second, PauseMax: 30 * time.Minute, Tries: 8}
|
||||
// defaultPublishSettings are the settings' defaults: 30 s, 30 min and 5 tries (H-A-446 (2)).
|
||||
var defaultPublishSettings = publishSettings{Pause: 30 * time.Second, PauseMax: 30 * time.Minute, Tries: 5}
|
||||
|
||||
// pauseAfter is the pause after the try-th try: the setting doubled per try, up to the maximum.
|
||||
func (s publishSettings) pauseAfter(try int) time.Duration {
|
||||
@@ -61,10 +69,15 @@ type publishItems struct {
|
||||
// publish there (H-A-275). Nil does neither.
|
||||
markHeld func(ctx context.Context, v versions.ReservedVersion, held []versions.HeldArtifact)
|
||||
moveElsewhere func(ctx context.Context, v versions.ReservedVersion, j versions.Judged) error
|
||||
set publishSettings
|
||||
now func() time.Time
|
||||
after func(time.Duration) <-chan time.Time
|
||||
logf func(string, ...any)
|
||||
// send sends one item pulled from the work queue (Republisher.SendQueued); nil sends none.
|
||||
send func(ctx context.Context, q versions.QueuedPublish) versions.SendFate
|
||||
set publishSettings
|
||||
now func() time.Time
|
||||
after func(time.Duration) <-chan time.Time
|
||||
logf func(string, ...any)
|
||||
|
||||
mu sync.Mutex
|
||||
held map[string][]heldItem // "<module> <version>": the items sent, their publish not yet heard
|
||||
}
|
||||
|
||||
func (p *publishItems) wait(ctx context.Context, d time.Duration) bool {
|
||||
@@ -130,8 +143,11 @@ func (p *publishItems) AskAllDone(ctx context.Context) {
|
||||
}
|
||||
}
|
||||
|
||||
// Done deletes a version's publish item: its release is written.
|
||||
// Done deletes a version's publish item and acknowledges its held items: its release is written.
|
||||
func (p *publishItems) Done(ctx context.Context, module, version string) {
|
||||
for _, h := range p.settle(module, version, func(heldItem) bool { return true }) {
|
||||
_ = h.m.Ack()
|
||||
}
|
||||
kv := p.reasks()
|
||||
if kv == nil {
|
||||
return
|
||||
@@ -141,9 +157,10 @@ func (p *publishItems) Done(ctx context.Context, module, version string) {
|
||||
}
|
||||
}
|
||||
|
||||
// Try waits for the item's next try, then, while the version is still reserved, queues its publish and writes the item again with
|
||||
// one more try and the pause after it; once the tries are spent it ends the item loudly. A version no longer reserved (released,
|
||||
// dropped or taken over) has its item deleted; one whose artifacts are published keeps it until its release is written.
|
||||
// Try waits for the item's next try, then, while the version is still reserved, puts its publish on the work queue publishes
|
||||
// under its message id (design 47 H2 §793; H-A-448): an item the queue holds already is put once. It counts no try: each send
|
||||
// does, and hands its item back with the pause (take). A put that queued nothing is put again after the pause. A version no
|
||||
// longer reserved (released, dropped or taken over) has its item deleted.
|
||||
func (p *publishItems) Try(ctx context.Context, kv jetstream.KeyValue, e jetstream.KeyValueEntry, r build.ReAsk) {
|
||||
if !p.wait(ctx, r.NextTry.Sub(p.now())) {
|
||||
return
|
||||
@@ -157,37 +174,215 @@ func (p *publishItems) Try(ctx context.Context, kv jetstream.KeyValue, e jetstre
|
||||
p.logf("%s %s: its reservation cannot be read, the publish item waits: %v", r.Module, r.Version, err)
|
||||
return
|
||||
}
|
||||
if why := versions.NotStanding(v, held); why != "" { // dropped, taken over, released or withdrawn: never because its artifacts are published (A-456b, H-A-290 (2))
|
||||
if why := versions.NotStanding(v, held); why != "" { // dropped, taken over, released or withdrawn (A-456b, H-A-290 (2))
|
||||
p.logf("%s %s: the publish item is ended: %s", r.Module, r.Version, why)
|
||||
_ = kv.Delete(ctx, e.Key())
|
||||
return
|
||||
}
|
||||
queued, err := p.republish(ctx, r.Module, r.Version)
|
||||
if err == nil && len(queued) > 0 {
|
||||
return
|
||||
}
|
||||
r.NextTry = p.now().UTC().Add(p.set.pauseAfter(r.Tries))
|
||||
p.logf("%s %s: the publish is not on the work queue %s (%v); it is put again after %s", r.Module, r.Version, versions.PublishesLocal, err, p.set.pauseAfter(r.Tries))
|
||||
if data, err := json.Marshal(r); err == nil {
|
||||
if _, err := kv.Update(ctx, e.Key(), data, e.Revision()); err != nil && !errors.Is(err, jetstream.ErrKeyExists) && !errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
|
||||
p.logf("%s: the publish item could not be written: %v", e.Key(), err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// itemMsg is what take reads of an item pulled from the work queue (jetstream.Msg).
|
||||
type itemMsg interface {
|
||||
Data() []byte
|
||||
Ack() error
|
||||
NakWithDelay(time.Duration) error
|
||||
Term() error
|
||||
}
|
||||
|
||||
// take handles one item pulled from the work queue (§793; H-A-440 (2), H-A-448): an item whose publish item ended (dropped, or
|
||||
// its tries spent) or whose version never publishes is terminated; one whose artifacts are all published is acknowledged; once
|
||||
// publish-tries sends are spent it is terminated and version-unreleased said; while a published of it may be unheard it is handed
|
||||
// back with the pause, and one the registries' record confirms is acknowledged (H-A-274); else it is sent, the try counted in its
|
||||
// publish item, and a send that fails handed back with the pause as its delay, so the queue delivers it again and nothing is put
|
||||
// again. A sent item is acknowledged when its publish is heard (heard, Done) and delivered again after the ack wait, unheard.
|
||||
func (p *publishItems) take(ctx context.Context, m itemMsg) {
|
||||
var q versions.QueuedPublish
|
||||
if err := json.Unmarshal(m.Data(), &q); err != nil || q.Module == "" || q.Version == "" {
|
||||
p.logf("an item of the work queue %s that is no publish is terminated: %v", versions.PublishesLocal, err)
|
||||
_ = m.Term()
|
||||
return
|
||||
}
|
||||
kv := p.reasks()
|
||||
if kv == nil {
|
||||
p.logf("%s %s: the re-ask state is not on the bus yet; the item is handed back", q.Module, q.Version)
|
||||
_ = m.NakWithDelay(p.set.Pause)
|
||||
return
|
||||
}
|
||||
key := versions.PublishItemKey(q.Module, q.Version)
|
||||
e, err := kv.Get(ctx, key)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) { // put by builder_publish_again or the move: its tries are counted from now
|
||||
if err = p.Ask(ctx, q.Module, q.Version); err == nil {
|
||||
e, err = kv.Get(ctx, key)
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
p.logf("%s %s: its publish item cannot be read (%v); the item is handed back", q.Module, q.Version, err)
|
||||
_ = m.NakWithDelay(p.set.Pause)
|
||||
return
|
||||
}
|
||||
r, err := build.DecodeReAsk(e.Value())
|
||||
if err != nil {
|
||||
_ = m.Term()
|
||||
return
|
||||
}
|
||||
if r.Answer != "" {
|
||||
p.logf("%s %s at %s is terminated: its publish item ended: %s", q.Module, q.Version, q.Holder, r.Answer)
|
||||
_ = m.Term()
|
||||
return
|
||||
}
|
||||
v, found, err := p.reserved(ctx, q.Module, q.Version)
|
||||
if err != nil {
|
||||
p.logf("%s %s: its reservation cannot be read (%v); the item is handed back", q.Module, q.Version, err)
|
||||
_ = m.NakWithDelay(p.set.pauseAfter(r.Tries))
|
||||
return
|
||||
}
|
||||
if why := versions.NotStanding(v, found); why != "" {
|
||||
p.logf("%s %s at %s is terminated: %s", q.Module, q.Version, q.Holder, why)
|
||||
_ = m.Term()
|
||||
_ = kv.Delete(ctx, key)
|
||||
return
|
||||
}
|
||||
if allPublished(v, q) {
|
||||
p.logf("%s %s at %s is acknowledged: its artifacts are published", q.Module, q.Version, q.Holder)
|
||||
_ = m.Ack()
|
||||
return
|
||||
}
|
||||
if r.Tries >= p.set.Tries {
|
||||
_ = m.Term()
|
||||
r.Answer = fmt.Sprintf("the publish of %s %s was tried %d times and the version is not released; builder_publish_again tries it again", r.Module, r.Version, r.Tries)
|
||||
r.Answered = p.now().UTC()
|
||||
p.logf("%s: %s", e.Key(), r.Answer)
|
||||
p.logf("%s: %s", key, r.Answer)
|
||||
if p.endedWith != nil {
|
||||
p.endedWith(ctx, v, "the publish ended after "+fmt.Sprint(r.Tries)+" tries")
|
||||
}
|
||||
} else if why, gone := p.notAgain(ctx, v); gone {
|
||||
return // moved to another number, whose own item is asked (H-A-275)
|
||||
} else if why != "" {
|
||||
// A published of the version may be unheard, or the registry's record ended it: no ask, and no try spent (H-A-272, H-A-274).
|
||||
p.logf("%s %s: the publish is not asked again: %s; tried again after %s", r.Module, r.Version, why, p.set.pauseAfter(r.Tries))
|
||||
r.NextTry = p.now().UTC().Add(p.set.pauseAfter(r.Tries))
|
||||
} else {
|
||||
r.Tries++
|
||||
if _, err := p.republish(ctx, r.Module, r.Version); err != nil {
|
||||
p.logf("%s %s: try %d queued no publish: %v", r.Module, r.Version, r.Tries, err)
|
||||
}
|
||||
r.NextTry = p.now().UTC().Add(p.set.pauseAfter(r.Tries))
|
||||
p.write(ctx, kv, key, r, e.Revision())
|
||||
return
|
||||
}
|
||||
if why, gone := p.notAgain(ctx, v); gone {
|
||||
_ = m.Term() // moved to another number, whose own item is asked (H-A-275)
|
||||
return
|
||||
} else if why != "" {
|
||||
if v2, ok, err := p.reserved(ctx, q.Module, q.Version); err == nil && ok && allPublished(v2, q) {
|
||||
p.logf("%s %s at %s is acknowledged: %s", q.Module, q.Version, q.Holder, why)
|
||||
_ = m.Ack()
|
||||
return
|
||||
}
|
||||
p.logf("%s %s: the publish is not sent again: %s; handed back for %s", q.Module, q.Version, why, p.set.pauseAfter(r.Tries))
|
||||
_ = m.NakWithDelay(p.set.pauseAfter(r.Tries))
|
||||
return
|
||||
}
|
||||
fate := versions.SendFate{Failed: "this builder sends no publish"}
|
||||
if p.send != nil {
|
||||
fate = p.send(ctx, q)
|
||||
}
|
||||
if fate.Ended != "" {
|
||||
p.logf("%s %s at %s is terminated: %s", q.Module, q.Version, q.Holder, fate.Ended)
|
||||
_ = m.Term()
|
||||
return
|
||||
}
|
||||
r.Tries++
|
||||
r.NextTry = p.now().UTC().Add(p.set.pauseAfter(r.Tries))
|
||||
p.write(ctx, kv, key, r, e.Revision())
|
||||
if fate.Failed != "" {
|
||||
p.logf("%s %s at %s: try %d sent no publish: %s; handed back for %s", q.Module, q.Version, q.Holder, r.Tries, fate.Failed, p.set.pauseAfter(r.Tries))
|
||||
_ = m.NakWithDelay(p.set.pauseAfter(r.Tries))
|
||||
return
|
||||
}
|
||||
p.hold(q, m)
|
||||
}
|
||||
|
||||
// write writes a publish item at its revision; a newer write wins.
|
||||
func (p *publishItems) write(ctx context.Context, kv jetstream.KeyValue, key string, r build.ReAsk, rev uint64) {
|
||||
data, err := json.Marshal(r)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if _, err := kv.Update(ctx, e.Key(), data, e.Revision()); err != nil && !errors.Is(err, jetstream.ErrKeyExists) && !errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
|
||||
p.logf("%s: the publish item could not be written: %v", e.Key(), err)
|
||||
if _, err := kv.Update(ctx, key, data, rev); err != nil && !errors.Is(err, jetstream.ErrKeyExists) && !errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
|
||||
p.logf("%s: the publish item could not be written: %v", key, err)
|
||||
}
|
||||
}
|
||||
|
||||
// allPublished says none of q's artifacts is missing from v: its publish is heard.
|
||||
func allPublished(v versions.ReservedVersion, q versions.QueuedPublish) bool {
|
||||
missing := map[string]bool{}
|
||||
for _, ref := range v.Missing() {
|
||||
missing[ref] = true
|
||||
}
|
||||
for ref := range q.Artifacts {
|
||||
if missing[ref] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// heldItem is an item sent and not yet acknowledged: its publish is not heard yet.
|
||||
type heldItem struct {
|
||||
q versions.QueuedPublish
|
||||
m itemMsg
|
||||
}
|
||||
|
||||
// hold keeps a sent item until its publish is heard.
|
||||
func (p *publishItems) hold(q versions.QueuedPublish, m itemMsg) {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
if p.held == nil {
|
||||
p.held = map[string][]heldItem{}
|
||||
}
|
||||
k := q.Module + " " + q.Version
|
||||
p.held[k] = append(p.held[k], heldItem{q: q, m: m})
|
||||
}
|
||||
|
||||
// settle takes every held item of module's version that pick picks out of the held ones.
|
||||
func (p *publishItems) settle(module, version string, pick func(heldItem) bool) []heldItem {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
k := module + " " + version
|
||||
var out, keep []heldItem
|
||||
for _, h := range p.held[k] {
|
||||
if pick(h) {
|
||||
out = append(out, h)
|
||||
} else {
|
||||
keep = append(keep, h)
|
||||
}
|
||||
}
|
||||
if len(keep) == 0 {
|
||||
delete(p.held, k)
|
||||
} else {
|
||||
p.held[k] = keep
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// heard acknowledges each held item of module's version whose artifacts are all published now: its publish is heard (§793).
|
||||
func (p *publishItems) heard(ctx context.Context, module, version string) {
|
||||
v, found, err := p.reserved(ctx, module, version)
|
||||
if err != nil || !found {
|
||||
return
|
||||
}
|
||||
for _, h := range p.settle(module, version, func(h heldItem) bool { return allPublished(v, h.q) }) {
|
||||
p.logf("%s %s at %s is acknowledged: its publish is heard", module, version, h.q.Holder)
|
||||
_ = h.m.Ack()
|
||||
}
|
||||
}
|
||||
|
||||
// Ended deletes a version's publish item and terminates its held items: its build is withdrawn or superseded (H-A-290 (2),
|
||||
// H-A-291 (1)); an item still on the work queue is terminated when it is pulled.
|
||||
func (p *publishItems) Ended(ctx context.Context, module, version string) {
|
||||
p.Done(ctx, module, version)
|
||||
for _, h := range p.settle(module, version, func(heldItem) bool { return true }) {
|
||||
_ = h.m.Term()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -312,10 +507,11 @@ func publishSettingsOf(getenv func(string) string) (publishSettings, error) {
|
||||
return publishSettings{Pause: time.Duration(pause) * time.Second, PauseMax: time.Duration(pmax) * time.Minute, Tries: tries}, nil
|
||||
}
|
||||
|
||||
// wirePublishItems joins the reservations to the publish items (design 47 G A-457a, A-456a, A-456b): a done build asks its item, a
|
||||
// release, a drop and a takeover end it, and a release is recorded in releases, the last kept of each module.
|
||||
// wirePublishItems joins the reservations to the publish items (design 47 G A-457a, A-456a, A-456b, H2 §793): a done build asks its
|
||||
// item, a release ends it and acknowledges its held items, a drop and a takeover end it and terminate them, a published heard
|
||||
// acknowledges those it publishes, and a release is recorded in releases, the last kept of each module.
|
||||
func wirePublishItems(rv *reservations, items *publishItems, releases versions.ReserveStore, kept int) {
|
||||
rv.built, rv.released = items.Ask, items.Done
|
||||
rv.built, rv.released, rv.itemEnded, rv.itemHeard = items.Ask, items.Done, items.Ended, items.heard
|
||||
rv.r.ReleasedKept, rv.r.Releases = kept, releases
|
||||
}
|
||||
|
||||
@@ -330,3 +526,438 @@ func builtOf(all []versions.ReservedVersion) []versions.ReservedVersion {
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// publishQueue is the builder's work queue publishes on the bus (design 47 H2 §793; H-A-440, H-A-446 (2), H-A-454): a stream of
|
||||
// the manifest's old streams section with the work-queue retention, the module's own, consumed by the process on the module's
|
||||
// account; each item on `mesh.stream.<module>.event.publish.<holder>.<class>.<module>.<version>` under the message id
|
||||
// `<holder>.<class>.<module>.<version>`. The builder makes the pull consumers at run time as it learns a holder,
|
||||
// `publishes-<holder>-<class>`, with ack wait publish-pause-max, max ack pending four and max deliveries ten, and pulls the live
|
||||
// class first, at most four sends at once per holder.
|
||||
type publishQueue struct {
|
||||
js jetstream.JetStream
|
||||
binding func() (publishesStream, error)
|
||||
ackWait time.Duration
|
||||
take func(ctx context.Context, m itemMsg)
|
||||
logf func(string, ...any)
|
||||
|
||||
mu sync.Mutex
|
||||
pulling map[string]bool
|
||||
}
|
||||
|
||||
// publishesStream is where the work queue publishes is on the bus: its stream's name and the subject its items open with.
|
||||
type publishesStream struct{ Stream, Subject string }
|
||||
|
||||
// publishesOf is module's work queue publishes, its own stream (H-A-454): the stream's bus name and its events' subject.
|
||||
func publishesOf(module string) func() (publishesStream, error) {
|
||||
b := publishesStream{Stream: catalogue.StreamBusName(versions.PublishesLocal),
|
||||
Subject: strings.TrimSuffix(buscompose.ModuleStreamSubject(module, ""), ".")}
|
||||
return func() (publishesStream, error) { return b, nil }
|
||||
}
|
||||
|
||||
// The consumers' numbers (H-A-446 (2)): at most four sends in flight per holder and class, and ten deliveries, the work queue's
|
||||
// largest; the builder ends an item at publish-tries anyway.
|
||||
const (
|
||||
publishesMaxAckPending = versions.PublishSendsAtOnce
|
||||
publishesMaxDeliver = 10
|
||||
)
|
||||
|
||||
// publishesConsumer is the durable consumer of holder's items of class.
|
||||
func publishesConsumer(holder, class string) string {
|
||||
return versions.PublishesLocal + "-" + holder + "-" + class
|
||||
}
|
||||
|
||||
// subject is the item's subject on the bus.
|
||||
func (pq *publishQueue) subject(b publishesStream, q versions.QueuedPublish) string {
|
||||
return b.Subject + "." + versions.PublishEvent(q.Holder, q.Class, q.Module, q.Version)
|
||||
}
|
||||
|
||||
// put puts q on the work queue under its message id; duplicate says the queue holds that id within its window. A put the full
|
||||
// queue refuses is versions.ErrQueueFull. The item's holder is learnt: its pull starts.
|
||||
func (pq *publishQueue) put(ctx context.Context, q versions.QueuedPublish) (bool, error) {
|
||||
b, err := pq.binding()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
data, err := json.Marshal(q)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
ack, err := pq.js.PublishMsg(ctx, &nats.Msg{Subject: pq.subject(b, q), Data: data}, jetstream.WithMsgID(versions.PublishMsgID(q.Holder, q.Class, q.Module, q.Version)))
|
||||
if err != nil {
|
||||
if w := err.Error(); strings.Contains(w, "maximum bytes") || strings.Contains(w, "maximum messages") || errors.Is(err, jetstream.ErrMaxBytesExceeded) {
|
||||
return false, fmt.Errorf("%w: %v", versions.ErrQueueFull, err)
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
pq.learn(ctx, q.Holder)
|
||||
return ack.Duplicate, nil
|
||||
}
|
||||
|
||||
// learnAll learns every holder whose items the work queue holds: at the start, so what waits there is pulled.
|
||||
func (pq *publishQueue) learnAll(ctx context.Context) error {
|
||||
b, err := pq.binding()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
st, err := pq.js.Stream(ctx, b.Stream)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
info, err := st.Info(ctx, jetstream.WithSubjectFilter(b.Subject+".publish.>"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for subj := range info.State.Subjects {
|
||||
if holder, _, ok := strings.Cut(strings.TrimPrefix(subj, b.Subject+".publish."), "."); ok {
|
||||
pq.learn(ctx, holder) // a holder is a module, its name one token
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// learn starts holder's pull once.
|
||||
func (pq *publishQueue) learn(ctx context.Context, holder string) {
|
||||
pq.mu.Lock()
|
||||
if pq.pulling == nil {
|
||||
pq.pulling = map[string]bool{}
|
||||
}
|
||||
if pq.pulling[holder] {
|
||||
pq.mu.Unlock()
|
||||
return
|
||||
}
|
||||
pq.pulling[holder] = true
|
||||
pq.mu.Unlock()
|
||||
if err := pq.pull(ctx, holder); err != nil {
|
||||
pq.logf("the publishes of %s are not pulled: %v; they are pulled when an item of it is put", holder, err)
|
||||
pq.mu.Lock()
|
||||
delete(pq.pulling, holder)
|
||||
pq.mu.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
// pull makes holder's two consumers and dispatches their items, the live class first, at most four sends at once: an again item
|
||||
// waits while a live one waits on the queue.
|
||||
func (pq *publishQueue) pull(ctx context.Context, holder string) error {
|
||||
b, err := pq.binding()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
chans := map[string]chan jetstream.Msg{}
|
||||
var live jetstream.Consumer
|
||||
for _, class := range []string{versions.PublishClassStart, versions.PublishClassLive} { // the again first: the live is first by the dispatch, not by luck
|
||||
c, err := pq.js.CreateOrUpdateConsumer(ctx, b.Stream, jetstream.ConsumerConfig{Durable: publishesConsumer(holder, class),
|
||||
FilterSubject: b.Subject + ".publish." + versions.EscapeToken(holder) + "." + class + ".>", AckPolicy: jetstream.AckExplicitPolicy,
|
||||
AckWait: pq.ackWait, MaxAckPending: publishesMaxAckPending, MaxDeliver: publishesMaxDeliver})
|
||||
if err != nil {
|
||||
return fmt.Errorf("the consumer %s: %w", publishesConsumer(holder, class), err)
|
||||
}
|
||||
ch := make(chan jetstream.Msg, publishesMaxAckPending)
|
||||
cc, err := c.Consume(func(m jetstream.Msg) { ch <- m }, jetstream.PullMaxMessages(publishesMaxAckPending))
|
||||
if err != nil {
|
||||
return fmt.Errorf("the consumer %s is not pulled: %w", publishesConsumer(holder, class), err)
|
||||
}
|
||||
go func() { <-ctx.Done(); cc.Stop() }()
|
||||
chans[class] = ch
|
||||
if class == versions.PublishClassLive {
|
||||
live = c
|
||||
}
|
||||
}
|
||||
liveWaits := func() bool {
|
||||
info, err := live.Info(ctx)
|
||||
return err == nil && info.NumPending > 0
|
||||
}
|
||||
pq.logf("pulling the publishes of %s, the live first", holder)
|
||||
go pq.dispatch(ctx, chans[versions.PublishClassLive], chans[versions.PublishClassStart], liveWaits)
|
||||
return nil
|
||||
}
|
||||
|
||||
// dispatch runs the items of one holder, at most four at once, every live item before an again one: an again item pulled while
|
||||
// a live one waits on the queue (liveWaits) runs after it.
|
||||
func (pq *publishQueue) dispatch(ctx context.Context, live, again <-chan jetstream.Msg, liveWaits func() bool) {
|
||||
sem := make(chan struct{}, versions.PublishSendsAtOnce)
|
||||
run := func(m jetstream.Msg) {
|
||||
go func() {
|
||||
defer func() { <-sem }()
|
||||
pq.take(ctx, m)
|
||||
}()
|
||||
}
|
||||
var spare jetstream.Msg // an again item pulled while a live one waited
|
||||
for {
|
||||
select {
|
||||
case sem <- struct{}{}:
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
select {
|
||||
case m := <-live:
|
||||
run(m)
|
||||
continue
|
||||
default:
|
||||
}
|
||||
m := spare
|
||||
spare = nil
|
||||
if m == nil {
|
||||
select {
|
||||
case lm := <-live:
|
||||
run(lm)
|
||||
continue
|
||||
case m = <-again:
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
if liveWaits() {
|
||||
spare = m
|
||||
select {
|
||||
case lm := <-live:
|
||||
run(lm)
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
continue
|
||||
}
|
||||
select {
|
||||
case lm := <-live: // came while the queue was asked
|
||||
spare = m
|
||||
run(lm)
|
||||
default:
|
||||
run(m)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// itemsOf are the items the work queue holds of module's version in class (empty for any class), each with its subject.
|
||||
func (pq *publishQueue) itemsOf(ctx context.Context, module, version, class string) (map[string]versions.QueuedPublish, jetstream.Stream, error) {
|
||||
b, err := pq.binding()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
st, err := pq.js.Stream(ctx, b.Stream)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
cls := "*"
|
||||
if class != "" {
|
||||
cls = class
|
||||
}
|
||||
info, err := st.Info(ctx, jetstream.WithSubjectFilter(b.Subject+".publish.*."+cls+"."+versions.EscapeToken(module)+"."+versions.EscapeToken(version)))
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
out := map[string]versions.QueuedPublish{}
|
||||
for subj := range info.State.Subjects {
|
||||
raw, err := st.GetLastMsgForSubject(ctx, subj)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
var q versions.QueuedPublish
|
||||
if json.Unmarshal(raw.Data, &q) == nil {
|
||||
out[subj] = q
|
||||
}
|
||||
}
|
||||
return out, st, nil
|
||||
}
|
||||
|
||||
// publishQueueItemsRead bounds the verb publish-queue's read of the work queue: at most 4,000 items, past them the answer is capped,
|
||||
// naming the items read and the queue's size (H-A-293).
|
||||
const publishQueueItemsRead = 4000
|
||||
|
||||
// reader is the work queue as the versions package's reader, for the seat mesh-builds' verb publish-queue (§793, H-A-454): its
|
||||
// items read from the stream by sequence, at most publishQueueItemsRead, each keyed by its sequence; a queue holding more is read
|
||||
// capped, with the stream's message count and bytes.
|
||||
func (pq *publishQueue) reader(ctx context.Context) versions.Reader {
|
||||
none := func() ([]string, error) { return nil, errors.ErrUnsupported }
|
||||
return versions.Reader{Keys: none, All: func() (versions.StatePage, error) {
|
||||
out := versions.StatePage{Values: map[string]json.RawMessage{}}
|
||||
b, err := pq.binding()
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
st, err := pq.js.Stream(ctx, b.Stream)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
info, err := st.Info(ctx)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
held := int(info.State.Msgs)
|
||||
if held == 0 {
|
||||
return out, nil
|
||||
}
|
||||
// A work queue takes no second consumer over its subjects, so its items are read by sequence, the deleted passed over.
|
||||
for seq := info.State.FirstSeq; seq <= info.State.LastSeq && len(out.Values) < publishQueueItemsRead; seq++ {
|
||||
m, err := st.GetMsg(ctx, seq)
|
||||
if errors.Is(err, jetstream.ErrMsgNotFound) {
|
||||
continue // acknowledged or terminated since
|
||||
}
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
if strings.HasPrefix(m.Subject, b.Subject+".publish.") {
|
||||
out.Values[strconv.FormatUint(seq, 10)] = json.RawMessage(m.Data)
|
||||
}
|
||||
}
|
||||
if held > len(out.Values) && len(out.Values) >= publishQueueItemsRead {
|
||||
out.More, out.Keys, out.Bytes = true, info.State.Msgs, info.State.Bytes
|
||||
}
|
||||
return out, nil
|
||||
}}
|
||||
}
|
||||
|
||||
// drop is builder_publish_drop (§793): the version's publish item is ended with who dropped it and why, so no item of it is ever
|
||||
// sent or put again; its items waiting on the work queue are removed, and one held is terminated.
|
||||
func (pq *publishQueue) drop(ctx context.Context, items *publishItems, module, version, by, why string) (string, error) {
|
||||
qs, st, err := pq.itemsOf(ctx, module, version, "")
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("the work queue %s cannot be read: %w", versions.PublishesLocal, err)
|
||||
}
|
||||
if err := items.endItem(ctx, module, version, fmt.Sprintf("dropped by %s: %s", by, why)); err != nil {
|
||||
return "", err
|
||||
}
|
||||
for subj := range qs {
|
||||
if err := st.Purge(ctx, jetstream.WithPurgeSubject(subj)); err != nil {
|
||||
return "", fmt.Errorf("the item %s is not terminated: %w", subj, err)
|
||||
}
|
||||
}
|
||||
for _, h := range items.settle(module, version, func(heldItem) bool { return true }) {
|
||||
_ = h.m.Term()
|
||||
}
|
||||
return fmt.Sprintf("the publish of %s %s is dropped: %d items terminated, and none is sent or put again", module, version, len(qs)), nil
|
||||
}
|
||||
|
||||
// first is builder_publish_first (§793): each again item of the version is put again as live, under the message id
|
||||
// `<holder>.live.<module>.<version>`, and the old terminated; a live id the queue holds within its window takes nothing, and the
|
||||
// old stays, said.
|
||||
func (pq *publishQueue) first(ctx context.Context, items *publishItems, module, version string) (string, error) {
|
||||
qs, st, err := pq.itemsOf(ctx, module, version, versions.PublishClassStart)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("the work queue %s cannot be read: %w", versions.PublishesLocal, err)
|
||||
}
|
||||
if len(qs) == 0 {
|
||||
return "", fmt.Errorf("the work queue %s holds no again item of %s %s", versions.PublishesLocal, module, version)
|
||||
}
|
||||
var moved, kept []string
|
||||
for subj, q := range qs {
|
||||
q.Class = versions.PublishClassLive
|
||||
dup, err := pq.put(ctx, q)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("%s %s is not put again as live at %s: %w", module, version, q.Holder, err)
|
||||
}
|
||||
if dup {
|
||||
kept = append(kept, q.Holder)
|
||||
continue
|
||||
}
|
||||
if err := st.Purge(ctx, jetstream.WithPurgeSubject(subj)); err != nil {
|
||||
return "", fmt.Errorf("the again item of %s %s at %s is not terminated: %w", module, version, q.Holder, err)
|
||||
}
|
||||
moved = append(moved, q.Holder)
|
||||
}
|
||||
for _, h := range items.settle(module, version, func(h heldItem) bool { return h.q.Class == versions.PublishClassStart }) {
|
||||
_ = h.m.Term()
|
||||
}
|
||||
sort.Strings(moved)
|
||||
answer := fmt.Sprintf("%s %s is put first as live at %s, its again item terminated", module, version, strings.Join(moved, ", "))
|
||||
if len(kept) > 0 {
|
||||
sort.Strings(kept)
|
||||
answer += fmt.Sprintf("; at %s the work queue holds its live id within the duplicates window, so its again item stays", strings.Join(kept, ", "))
|
||||
}
|
||||
return answer, nil
|
||||
}
|
||||
|
||||
// endItem writes the version's publish item ended with answer, made when none stands: Try and take then leave it, and a start
|
||||
// asks it no more.
|
||||
func (p *publishItems) endItem(ctx context.Context, module, version, answer string) error {
|
||||
kv := p.reasks()
|
||||
if kv == nil {
|
||||
return errors.New("the re-ask state is not on the bus yet")
|
||||
}
|
||||
key := versions.PublishItemKey(module, version)
|
||||
for range 5 {
|
||||
r := build.ReAsk{V: 1, Why: "the build ended with its version unreleased", By: "the builder", At: p.now().UTC(), Mode: build.ModePublish, Module: module, Version: version}
|
||||
e, err := kv.Get(ctx, key)
|
||||
var rev uint64
|
||||
switch {
|
||||
case errors.Is(err, jetstream.ErrKeyNotFound):
|
||||
case err != nil:
|
||||
return err
|
||||
default:
|
||||
if r, err = build.DecodeReAsk(e.Value()); err != nil {
|
||||
return err
|
||||
}
|
||||
rev = e.Revision()
|
||||
}
|
||||
r.Answer, r.Answered = answer, p.now().UTC()
|
||||
data, err := json.Marshal(r)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if rev == 0 {
|
||||
_, err = kv.Create(ctx, key, data)
|
||||
} else {
|
||||
_, err = kv.Update(ctx, key, data, rev)
|
||||
}
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
if !errors.Is(err, jetstream.ErrKeyExists) && !errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return errors.New("the publish item of " + module + " " + version + " changed five times while it was ended")
|
||||
}
|
||||
|
||||
// moveOld puts the items of the old state publish-queue on the work queue once at the start (§793): each under its message id, so
|
||||
// a move run twice puts each once within the duplicates window; then the bucket is dropped through the bus module (its tool
|
||||
// nats_drop, on the operator's warrant). A builder whose bus holds no such bucket moves nothing.
|
||||
func moveOld(ctx context.Context, js jetstream.JetStream, bucket, module string, rp *versions.Republisher, call toolCaller, logf func(string, ...any)) {
|
||||
kv, err := js.KeyValue(ctx, bucket)
|
||||
if errors.Is(err, jetstream.ErrBucketNotFound) {
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
logf("the old state %s cannot be read, so its items are not moved to the work queue %s: %v", versions.PublishQueueState, versions.PublishesLocal, err)
|
||||
return
|
||||
}
|
||||
keys, err := kv.ListKeys(ctx)
|
||||
if err != nil {
|
||||
logf("the old state %s cannot be listed, so its items are not moved: %v", versions.PublishQueueState, err)
|
||||
return
|
||||
}
|
||||
var vals []json.RawMessage
|
||||
for k := range keys.Keys() {
|
||||
if e, err := kv.Get(ctx, k); err == nil {
|
||||
vals = append(vals, json.RawMessage(e.Value()))
|
||||
}
|
||||
}
|
||||
n, err := rp.MoveQueued(ctx, vals)
|
||||
if err != nil {
|
||||
logf("the old state %s is not dropped: %d of its %d items moved to the work queue %s before: %v", versions.PublishQueueState, n, len(vals), versions.PublishesLocal, err)
|
||||
return
|
||||
}
|
||||
logf("the old state %s: %d of its %d items put on the work queue %s; it is dropped through the bus module", versions.PublishQueueState, n, len(vals), versions.PublishesLocal)
|
||||
if call == nil {
|
||||
return
|
||||
}
|
||||
if _, err := call(natsDropTool, map[string]any{"module": module, "local": versions.PublishQueueState,
|
||||
"why": "its items are on the builder's work queue " + versions.PublishesLocal + " (design 47 H2 section 793)"}); err != nil {
|
||||
logf("the old state %s is not dropped through the bus module: %v", versions.PublishQueueState, err)
|
||||
}
|
||||
}
|
||||
|
||||
// natsDropTool is the bus module's drop of an entry no accepted manifest asks for (H-A-126), which the old state goes by.
|
||||
const natsDropTool = "nats.nats_drop"
|
||||
|
||||
// publishQueueVerb is the handler of the seat mesh-builds' verb publish-queue (§793; H-A-454): the builder reads its own work queue
|
||||
// and answers its counts per holder and class, the oldest and newest, and a capped list; builder_publish_queue asks it.
|
||||
func publishQueueVerb(ctx context.Context, pq *publishQueue) process.SeatHandler {
|
||||
return func(_ string, body json.RawMessage) (any, error) {
|
||||
var in meshbuilds.PublishQueueInput
|
||||
if len(body) > 0 {
|
||||
if err := json.Unmarshal(body, &in); err != nil {
|
||||
return nil, fmt.Errorf("the arguments do not fit the verb publish-queue: %w", err)
|
||||
}
|
||||
}
|
||||
return versions.PublishQueueOf(pq.reader(ctx), in.Module, in.Version)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -89,6 +90,29 @@ func (k *itemKV) Delete(_ context.Context, key string, _ ...jetstream.KVDeleteOp
|
||||
return nil
|
||||
}
|
||||
|
||||
// itemMsgFake is an item pulled from the work queue: what take settled it with.
|
||||
type itemMsgFake struct {
|
||||
data []byte
|
||||
settled []string // ack, term, nak <delay>
|
||||
}
|
||||
|
||||
func (m *itemMsgFake) Data() []byte { return m.data }
|
||||
func (m *itemMsgFake) Ack() error { m.settled = append(m.settled, "ack"); return nil }
|
||||
func (m *itemMsgFake) Term() error { m.settled = append(m.settled, "term"); return nil }
|
||||
func (m *itemMsgFake) NakWithDelay(d time.Duration) error {
|
||||
m.settled = append(m.settled, "nak "+d.String())
|
||||
return nil
|
||||
}
|
||||
|
||||
func queuedMsg(t *testing.T, module, version string) *itemMsgFake {
|
||||
t.Helper()
|
||||
raw, err := json.Marshal(versions.QueuedPublish{V: 1, Holder: "oci", Module: module, Version: version, Artifacts: map[string]string{module + ":a": "sha256:a"}, Class: versions.PublishClassLive})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return &itemMsgFake{data: raw}
|
||||
}
|
||||
|
||||
func decodeItem(t *testing.T, raw []byte) build.ReAsk {
|
||||
t.Helper()
|
||||
r, err := build.DecodeReAsk(raw)
|
||||
@@ -98,21 +122,21 @@ func decodeItem(t *testing.T, raw []byte) build.ReAsk {
|
||||
return r
|
||||
}
|
||||
|
||||
// TestAPublishItemIsTriedWithAGrowingPauseAndEndsLoudly: design 47 G A-457a — a version's publish item is asked once, each try queues
|
||||
// the publish and writes the item again with one more try and the pause doubled up to its maximum, the release deletes it, and
|
||||
// after the setting's tries it ends with its words, raises the condition and stays; a version no longer reserved has it deleted.
|
||||
// TestAPublishItemIsTriedWithAGrowingPauseAndEndsLoudly: design 47 G A-457a, H2 §793 (H-A-440 (2), H-A-448) — a version's publish
|
||||
// item is asked once; its try puts the publish on the work queue and counts no try; each send counts one and a failed send hands
|
||||
// the item back with the pause doubled up to its maximum; after the setting's tries the item is terminated and ends with its words,
|
||||
// raises the condition and stays; an item whose artifacts are published is acknowledged; the release deletes it; a version no
|
||||
// longer reserved has it deleted.
|
||||
func TestAPublishItemIsTriedWithAGrowingPauseAndEndsLoudly(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
kv := &itemKV{vals: map[string][]byte{}, revs: map[string]uint64{}}
|
||||
now := time.Date(2026, 11, 17, 1, 0, 0, 0, time.UTC)
|
||||
var waited []time.Duration
|
||||
var queued, ended []string
|
||||
var queued, ended, sent []string
|
||||
reserved := true
|
||||
v := versions.ReservedVersion{Module: "m", Version: "0.1.2", Artifacts: map[string]string{"m:a": "sha256:a"}, BuiltAt: now}
|
||||
p := &publishItems{reasks: func() jetstream.KeyValue { return kv }, set: publishSettings{Pause: 30 * time.Second, PauseMax: 2 * time.Minute, Tries: 4},
|
||||
now: func() time.Time { return now }, logf: t.Logf,
|
||||
after: func(d time.Duration) <-chan time.Time {
|
||||
waited = append(waited, d)
|
||||
now = now.Add(d)
|
||||
c := make(chan time.Time, 1)
|
||||
c <- now
|
||||
@@ -126,7 +150,11 @@ func TestAPublishItemIsTriedWithAGrowingPauseAndEndsLoudly(t *testing.T) {
|
||||
},
|
||||
republish: func(_ context.Context, m, ver string) ([]string, error) {
|
||||
queued = append(queued, m+" "+ver)
|
||||
return nil, errors.New("no holder")
|
||||
return []string{m + " " + ver}, nil
|
||||
},
|
||||
send: func(_ context.Context, q versions.QueuedPublish) versions.SendFate {
|
||||
sent = append(sent, q.Module+" "+q.Version)
|
||||
return versions.SendFate{Failed: "no holder answers"}
|
||||
},
|
||||
endedWith: func(_ context.Context, v versions.ReservedVersion, why string) {
|
||||
ended = append(ended, v.Version+": "+why)
|
||||
@@ -135,25 +163,34 @@ func TestAPublishItemIsTriedWithAGrowingPauseAndEndsLoudly(t *testing.T) {
|
||||
if err := p.Ask(ctx, "m", "0.1.2"); err != nil || p.Ask(ctx, "m", "0.1.2") != nil || len(kv.vals) != 1 {
|
||||
t.Fatalf("asked: %v, %d items, want one item however often it is asked", err, len(kv.vals))
|
||||
}
|
||||
for i := 0; i < 5; i++ {
|
||||
e, _ := kv.Get(ctx, key)
|
||||
p.Try(ctx, kv, e, decodeItem(t, e.Value()))
|
||||
e, _ := kv.Get(ctx, key)
|
||||
p.Try(ctx, kv, e, decodeItem(t, e.Value()))
|
||||
if len(queued) != 1 || decodeItem(t, kv.vals[key]).Tries != 0 {
|
||||
t.Fatalf("the try queued %v, tries %d: want the publish put once, no try counted", queued, decodeItem(t, kv.vals[key]).Tries)
|
||||
}
|
||||
var settled []string
|
||||
for range 5 {
|
||||
m := queuedMsg(t, "m", "0.1.2")
|
||||
p.take(ctx, m)
|
||||
settled = append(settled, m.settled...)
|
||||
}
|
||||
got := decodeItem(t, kv.vals[key])
|
||||
if len(queued) != 4 || got.Tries != 4 || got.Answer == "" || got.Answered.IsZero() || len(ended) != 1 || !strings.Contains(ended[0], "4 tries") {
|
||||
t.Fatalf("queued %v, item %+v, ended %v: want four tries, then the end said with its words and the condition", queued, got, ended)
|
||||
if len(sent) != 4 || got.Tries != 4 || got.Answer == "" || got.Answered.IsZero() || len(ended) != 1 || !strings.Contains(ended[0], "4 tries") {
|
||||
t.Fatalf("sent %v, item %+v, ended %v: want four sends, then the end said with its words and the condition", sent, got, ended)
|
||||
}
|
||||
if want := []time.Duration{30 * time.Second, time.Minute, 2 * time.Minute, 2 * time.Minute}; len(waited) != len(want) || waited[0] != want[0] || waited[1] != want[1] || waited[2] != want[2] || waited[3] != want[3] {
|
||||
t.Errorf("pauses before the tries and the end = %v, want %v (doubled, capped)", waited, want)
|
||||
if want := "[nak 30s nak 1m0s nak 2m0s nak 2m0s term]"; fmt.Sprint(settled) != want {
|
||||
t.Errorf("the items were settled %v, want %s (handed back with the pause doubled and capped, then terminated)", settled, want)
|
||||
}
|
||||
// An item whose artifacts are all published stays until its release is written, and is tried (A-456b).
|
||||
queued = nil
|
||||
m := queuedMsg(t, "m", "0.1.2")
|
||||
if p.take(ctx, m); fmt.Sprint(m.settled) != "[term]" || len(sent) != 4 {
|
||||
t.Errorf("an item of an ended publish item was settled %v, sent %d: want it terminated unsent", m.settled, len(sent))
|
||||
}
|
||||
// An item whose artifacts are all published is acknowledged unsent; its publish item stays until its release is written (A-456b).
|
||||
_ = p.Ask(ctx, "m", "0.3.0")
|
||||
pk := versions.PublishItemKey("m", "0.3.0")
|
||||
pe, _ := kv.Get(ctx, pk)
|
||||
p.Try(ctx, kv, pe, decodeItem(t, pe.Value()))
|
||||
if _, ok := kv.vals[pk]; !ok || len(queued) != 1 || decodeItem(t, kv.vals[pk]).Tries != 1 {
|
||||
t.Errorf("an item whose artifacts are published was deleted or not tried: %v, queued %v", kv.vals, queued)
|
||||
m = queuedMsg(t, "m", "0.3.0")
|
||||
if p.take(ctx, m); fmt.Sprint(m.settled) != "[ack]" || len(sent) != 4 || !kv.has(pk) {
|
||||
t.Errorf("an item whose artifacts are published was settled %v, sent %d, kept %v: want it acknowledged unsent, its publish item kept", m.settled, len(sent), kv.has(pk))
|
||||
}
|
||||
kv.vals, kv.revs = map[string][]byte{}, map[string]uint64{}
|
||||
// At the start every reservation of a done build with no release is asked, one item each.
|
||||
@@ -194,7 +231,7 @@ func TestAPublishItemIsTriedWithAGrowingPauseAndEndsLoudly(t *testing.T) {
|
||||
queued = nil
|
||||
_ = p.Ask(ctx, "m", "0.1.2")
|
||||
v.Withdrawn = map[string]string{"0.1.2": "fp"}
|
||||
e, _ := kv.Get(ctx, key)
|
||||
e, _ = kv.Get(ctx, key)
|
||||
p.Try(ctx, kv, e, decodeItem(t, e.Value()))
|
||||
if len(kv.vals) != 0 || len(queued) != 0 {
|
||||
t.Errorf("a withdrawn reservation left its item %v, queued %v", kv.vals, queued)
|
||||
@@ -209,7 +246,8 @@ func TestAPublishItemIsTriedWithAGrowingPauseAndEndsLoudly(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestThePublishSettingsAreReadFromTheirFiles: design 47 G A-457a — the three settings, 30 s, 30 min and 8 tries unless set, and a
|
||||
// TestThePublishSettingsAreReadFromTheirFiles: design 47 G A-457a, H2 H-A-446 (2) — the three settings, 30 s, 30 min and 5 tries
|
||||
// unless set, and a
|
||||
// content that is no number is refused naming its file.
|
||||
func TestThePublishSettingsAreReadFromTheirFiles(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
@@ -222,7 +260,7 @@ func TestThePublishSettingsAreReadFromTheirFiles(t *testing.T) {
|
||||
}
|
||||
env := map[string]string{}
|
||||
got, err := publishSettingsOf(func(k string) string { return env[k] })
|
||||
if err != nil || got != defaultPublishSettings || got.Pause != 30*time.Second || got.PauseMax != 30*time.Minute || got.Tries != 8 {
|
||||
if err != nil || got != defaultPublishSettings || got.Pause != 30*time.Second || got.PauseMax != 30*time.Minute || got.Tries != 5 {
|
||||
t.Fatalf("defaults = %+v (%v)", got, err)
|
||||
}
|
||||
env["MESH_BUILDER_PUBLISH_PAUSE_FILE"], env["MESH_BUILDER_PUBLISH_PAUSE_MAX_FILE"], env["MESH_BUILDER_PUBLISH_TRIES_FILE"] = write("p", "5\n"), write("m", "2"), write("t", "3")
|
||||
@@ -353,9 +391,12 @@ func TestAPublishIsNotAskedAgainWhileItsPublishedMayBeUnheard(t *testing.T) {
|
||||
kv := &itemKV{vals: map[string][]byte{}, revs: map[string]uint64{}}
|
||||
asked, judged, marked, moved := 0, 0, 0, 0
|
||||
p := &publishItems{reasks: func() jetstream.KeyValue { return kv }, set: defaultPublishSettings, now: func() time.Time { return now }, logf: t.Logf,
|
||||
reserved: func(context.Context, string, string) (versions.ReservedVersion, bool, error) { return v, true, nil },
|
||||
republish: func(context.Context, string, string) ([]string, error) { asked++; return nil, nil },
|
||||
unheard: func(context.Context) (int, error) { return c.unheard, c.countErr },
|
||||
reserved: func(context.Context, string, string) (versions.ReservedVersion, bool, error) { return v, true, nil },
|
||||
send: func(context.Context, versions.QueuedPublish) versions.SendFate {
|
||||
asked++
|
||||
return versions.SendFate{Sent: true}
|
||||
},
|
||||
unheard: func(context.Context) (int, error) { return c.unheard, c.countErr },
|
||||
judge: func(context.Context, versions.ReservedVersion) (versions.Judged, error) {
|
||||
judged++
|
||||
return c.judged, nil
|
||||
@@ -374,20 +415,24 @@ func TestAPublishIsNotAskedAgainWhileItsPublishedMayBeUnheard(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key := versions.PublishItemKey(v.Module, v.Version)
|
||||
e, _ := kv.Get(ctx, key)
|
||||
p.Try(ctx, kv, e, decodeItem(t, e.Value()))
|
||||
m := queuedMsg(t, v.Module, v.Version)
|
||||
m.data = []byte(strings.ReplaceAll(string(m.data), `"node-tools:a"`, `"node-tools:program"`))
|
||||
p.take(ctx, m)
|
||||
if (asked == 1) != c.wantAsked || (marked == 1) != c.wantMark || (moved == 1) != c.wantMov {
|
||||
t.Fatalf("asked %d, marked %d, moved %d; want asked %v, marked %v, moved %v", asked, marked, moved, c.wantAsked, c.wantMark, c.wantMov)
|
||||
}
|
||||
if c.wantMov {
|
||||
if _, ok := kv.vals[key]; ok {
|
||||
t.Error("the moved version's old item is written again")
|
||||
if _, ok := kv.vals[key]; ok || fmt.Sprint(m.settled) != "[term]" {
|
||||
t.Errorf("the moved version's old item is written again, or its item settled %v, not terminated", m.settled)
|
||||
}
|
||||
return
|
||||
}
|
||||
got := decodeItem(t, kv.vals[key])
|
||||
if got.Tries != map[bool]int{true: 1, false: 0}[c.wantAsked] || !got.NextTry.After(now) {
|
||||
t.Errorf("tries %d, next try %s: want a try spent only when asked, and the item written again with its pause", got.Tries, got.NextTry)
|
||||
if got.Tries != map[bool]int{true: 1, false: 0}[c.wantAsked] || (c.wantAsked && !got.NextTry.After(now)) {
|
||||
t.Errorf("tries %d, next try %s: want a try spent only when sent, the item written with its pause", got.Tries, got.NextTry)
|
||||
}
|
||||
if !c.wantAsked && fmt.Sprint(m.settled) != "[nak 30s]" {
|
||||
t.Errorf("an item not sent was settled %v, want it handed back with the pause and no try spent", m.settled)
|
||||
}
|
||||
if c.unheard >= versions.UnheardBound && judged != 0 {
|
||||
t.Error("the registry is read while a published may be unheard")
|
||||
|
||||
@@ -65,7 +65,7 @@ func reAsked(ctx context.Context, b askAgainer, acts versions.Acts, items *publi
|
||||
released(ctx, b, kv, e, r, logf)
|
||||
return
|
||||
}
|
||||
if r.Mode == build.ModeRepublish || r.Mode == build.ModeFloorAgain || r.Mode == build.ModeOldFormDrop || r.Mode == build.ModeReservedDrop || r.Mode == build.ModeReserveDone {
|
||||
if r.Mode == build.ModeRepublish || r.Mode == build.ModePublishDrop || r.Mode == build.ModePublishFirst || r.Mode == build.ModeFloorAgain || r.Mode == build.ModeOldFormDrop || r.Mode == build.ModeReservedDrop || r.Mode == build.ModeReserveDone {
|
||||
// A version's hand act (design 51 *Tools of the builder*) may build again, so it is answered beside the watch.
|
||||
go versionActed(ctx, acts, kv, e, r, logf)
|
||||
return
|
||||
@@ -208,6 +208,16 @@ func versionActed(ctx context.Context, acts versions.Acts, kv jetstream.KeyValue
|
||||
r.Answer = "queued " + fmt.Sprint(len(r.Queued)) + " versions to publish again: " + strings.Join(r.Queued, ", ") + "; builder_publish_queue shows them"
|
||||
}
|
||||
}
|
||||
} else if r.Mode == build.ModePublishDrop || r.Mode == build.ModePublishFirst {
|
||||
act := acts.PublishDrop
|
||||
if r.Mode == build.ModePublishFirst {
|
||||
act = acts.PublishFirst
|
||||
}
|
||||
if act == nil {
|
||||
err = errors.New("this builder keeps no work queue of publishes")
|
||||
} else {
|
||||
r.Answer, err = act(ctx, r.Module, r.Version, r.By, r.Why)
|
||||
}
|
||||
} else if r.Mode == build.ModeReserveDone {
|
||||
if acts.ReserveDone == nil {
|
||||
err = errors.New("this builder keeps no reservations")
|
||||
|
||||
@@ -2,35 +2,22 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/gopackageregistry"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"mesh-delivery-builder/internal/versions"
|
||||
)
|
||||
|
||||
// The builder's publish again (design 47 G §588, A-73, A-76): the wiring of versions.Republisher to the bus.
|
||||
|
||||
// awaitedQueue is the state publish-queue as versions.QueueStore: bound once the bus holds it; its bytes are the cap's.
|
||||
type awaitedQueue struct{ awaitedReserved }
|
||||
|
||||
func (s awaitedQueue) Bytes(ctx context.Context) (uint64, error) {
|
||||
kv, err := s.kv()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
st, err := kv.Status(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return st.Bytes(), nil
|
||||
}
|
||||
|
||||
// entryVia reads a catalog graph entry by its fingerprint through the seat mesh-catalog's verb entry.
|
||||
func entryVia(call toolCaller) func(ctx context.Context, fingerprint string) (meshcatalog.CatalogEntry, error) {
|
||||
return func(ctx context.Context, fingerprint string) (meshcatalog.CatalogEntry, error) {
|
||||
@@ -46,12 +33,28 @@ func entryVia(call toolCaller) func(ctx context.Context, fingerprint string) (me
|
||||
}
|
||||
}
|
||||
|
||||
// keptCaller is a kept call carrying an ask id (process.Conn.CallKeptAsk).
|
||||
type keptCaller func(ask process.KeptAsk, key string, body any) (json.RawMessage, error)
|
||||
|
||||
// publishAskID is the ask id of p's publish (design 47 H2 §775): the same seat, name, version and digest make the same id,
|
||||
// so a publish made again is one the holder takes once (ADR 0668 §2); an id holds no dot, as every ask id.
|
||||
func publishAskID(p versions.Publish) string {
|
||||
sum := sha256.Sum256([]byte(p.Seat + "\n" + p.Name + "\n" + p.Version + "\n" + p.Digest))
|
||||
return "publish" + hex.EncodeToString(sum[:8])
|
||||
}
|
||||
|
||||
// sendVia sends a publish to the registry seat with the digest and no parts (A-73): the holder takes the bytes from the
|
||||
// artifact store, addressed by digest. The four seats' publish take one wire form; an answer that lacks parts, or a fault of
|
||||
// the call, is the holder's refusal, said in its words (not stored when the holder lacks the bytes).
|
||||
func sendVia(call toolCaller) func(ctx context.Context, p versions.Publish) error {
|
||||
// the call, is the holder's refusal, said in its words (not stored when the holder lacks the bytes). The publish is a kept
|
||||
// call carrying its ask id (design 47 H2 §775), made again through transport faults at the kept-call pace within ctx's bound.
|
||||
func sendVia(call keptCaller) func(ctx context.Context, p versions.Publish) error {
|
||||
return func(ctx context.Context, p versions.Publish) error {
|
||||
raw, err := callWithin(ctx, call, "seat:"+p.Seat+"."+versions.VerbPublish, publishBody(p))
|
||||
ask := process.KeptAsk{ID: publishAskID(p)}
|
||||
if d, ok := ctx.Deadline(); ok {
|
||||
ask.Bound = time.Until(d)
|
||||
}
|
||||
raw, err := callWithin(ctx, func(key string, body any) (json.RawMessage, error) { return call(ask, key, body) },
|
||||
"seat:"+p.Seat+"."+versions.VerbPublish, publishBody(p))
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s's publish of %s %s: %w", p.Seat, p.Name, p.Version, err)
|
||||
}
|
||||
@@ -93,21 +96,3 @@ func callWithin(ctx context.Context, call toolCaller, key string, body any) (jso
|
||||
func publishBody(p versions.Publish) gopackageregistry.PublishInput {
|
||||
return gopackageregistry.PublishInput{Commit: p.Commit, Digest: p.Digest, Fingerprint: p.Fingerprint, Kind: p.Kind, Name: p.Name, Transfer: p.Transfer, Version: p.Version}
|
||||
}
|
||||
|
||||
// republishing starts the publish again once both its states are bound: the queue is taken up again (design 47 G §588). A version's
|
||||
// publish is asked when its build ends and at the start (A-457a), never waited on a running or manifest-reads-changed.
|
||||
type republishing struct {
|
||||
rp *versions.Republisher
|
||||
ready atomic.Int32
|
||||
}
|
||||
|
||||
func (g *republishing) bound(ctx context.Context, js jetstream.JetStream, logf func(string, ...any)) func(jetstream.KeyValue) {
|
||||
return func(jetstream.KeyValue) {
|
||||
if g.ready.Add(1) != 2 {
|
||||
return
|
||||
}
|
||||
if err := g.rp.Resume(ctx); err != nil {
|
||||
logf("the queued publishes are not taken up again: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,6 +13,8 @@ import (
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/npmpackageregistry"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/ociartifactregistry"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"mesh-delivery-builder/internal/build"
|
||||
"mesh-delivery-builder/internal/versions"
|
||||
)
|
||||
@@ -48,12 +50,14 @@ func TestRepublishSend(t *testing.T) {
|
||||
"a fault": {"", errors.New("no responders"), "no responders"},
|
||||
"no sense": {`[`, nil, "no publish answer"},
|
||||
} {
|
||||
send := sendVia(func(key string, body any) (json.RawMessage, error) {
|
||||
asked = key
|
||||
var id string
|
||||
send := sendVia(func(ask process.KeptAsk, key string, body any) (json.RawMessage, error) {
|
||||
asked, id = key, ask.ID
|
||||
return json.RawMessage(c.answer), c.err
|
||||
})
|
||||
err := send(context.Background(), p)
|
||||
if asked != "seat:oci-artifact-registry.publish" || (c.want == "") != (err == nil) || (err != nil && !strings.Contains(err.Error(), c.want)) {
|
||||
// design 47 H2 §775: the publish carries its ask id, the same for the same publish
|
||||
if asked != "seat:oci-artifact-registry.publish" || id != publishAskID(p) || strings.Contains(id, ".") || (c.want == "") != (err == nil) || (err != nil && !strings.Contains(err.Error(), c.want)) {
|
||||
t.Errorf("%s: asked %q, error %v, want %q", name, asked, err, c.want)
|
||||
}
|
||||
}
|
||||
@@ -68,8 +72,10 @@ func TestRepublishSend(t *testing.T) {
|
||||
return json.RawMessage(`{}`), nil
|
||||
}
|
||||
for name, call := range map[string]func(context.Context) error{
|
||||
"the publish": func(ctx context.Context) error { return sendVia(slow)(ctx, p) },
|
||||
"the entry": func(ctx context.Context) error { _, err := entryVia(slow)(ctx, "fp"); return err },
|
||||
"the publish": func(ctx context.Context) error {
|
||||
return sendVia(func(_ process.KeptAsk, k string, b any) (json.RawMessage, error) { return slow(k, b) })(ctx, p)
|
||||
},
|
||||
"the entry": func(ctx context.Context) error { _, err := entryVia(slow)(ctx, "fp"); return err },
|
||||
} {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
go func() { <-called; cancel() }()
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
@@ -44,12 +45,15 @@ type reservations struct {
|
||||
// built asks a done build's publish (design 47 G A-457a), released ends its item; nil are none.
|
||||
built func(ctx context.Context, module, version string) error
|
||||
released func(ctx context.Context, module, version string)
|
||||
filled atomic.Bool
|
||||
bounds func() jetstream.KeyValue
|
||||
found *findings
|
||||
bound time.Duration
|
||||
wake func(context.Context)
|
||||
logf func(string, ...any)
|
||||
// ended ends the publish item of a build withdrawn, superseded or dropped, its held items terminated; heard acknowledges the
|
||||
// held items of a version a published newly marked (design 47 H2 §793). Nil are none.
|
||||
itemEnded, itemHeard func(ctx context.Context, module, version string)
|
||||
filled atomic.Bool
|
||||
bounds func() jetstream.KeyValue
|
||||
found *findings
|
||||
bound time.Duration
|
||||
wake func(context.Context)
|
||||
logf func(string, ...any)
|
||||
// withdraw ends the running build of a module's fingerprint, superseded, with why (H-A-225); nil ends none.
|
||||
withdraw func(module, fingerprint, why string)
|
||||
// call asks a seat's verb: the git seat's tag (design 47 G §370).
|
||||
@@ -58,6 +62,47 @@ type reservations struct {
|
||||
rebuild func(ctx context.Context, module, fp, why string) error
|
||||
// say says package-published on the seat mesh-builds; nil says none.
|
||||
say func(context.Context, meshbuilds.PackagePublished) error
|
||||
// names is each package's name as its series key names it (for Go, its module path), and sayReserved says version-reserved on
|
||||
// the seat mesh-builds (design 47 H2 §805); nil says none.
|
||||
names versions.PackageNames
|
||||
sayReserved func(context.Context, meshbuilds.VersionReservedEvent) error
|
||||
}
|
||||
|
||||
// reservedSaid says version-reserved of each package a reservation newly numbers (design 47 H2 §805, H-A-420 (2)): its module,
|
||||
// artifact, the package's path, the version, the fingerprint and the catalog revision it was reserved at, as the indexer names
|
||||
// a revision, `<branch>.<n>`. A reservation that names no catalog revision is said in the journal, not on the seat: the indexer
|
||||
// re-pins at a revision, and there is none to name.
|
||||
func (rv *reservations) reservedSaid(ctx context.Context, v versions.ReservedVersion) {
|
||||
if rv.sayReserved == nil || rv.names == nil {
|
||||
return
|
||||
}
|
||||
refs := make([]string, 0, len(v.Artifacts))
|
||||
for ref := range v.Artifacts {
|
||||
refs = append(refs, ref)
|
||||
}
|
||||
slices.Sort(refs)
|
||||
for _, ref := range refs {
|
||||
_, artifact, _ := strings.Cut(ref, ":")
|
||||
path, known, err := rv.names(ctx, v.Module, artifact)
|
||||
if err != nil {
|
||||
rv.logf("version-reserved of %s at %s is not said: its package's name cannot be read: %v", ref, v.Version, err)
|
||||
continue
|
||||
}
|
||||
if !known {
|
||||
continue // no package: nothing imports it by a version
|
||||
}
|
||||
if v.Branch == "" || v.Revision == 0 {
|
||||
rv.logf("version-reserved of %s at %s is not said: the reservation names no catalog revision to re-pin at", ref, v.Version)
|
||||
continue
|
||||
}
|
||||
ev := meshbuilds.VersionReservedEvent{Module: v.Module, Artifact: ref, Path: path, Version: v.Version, Fingerprint: v.Fingerprint,
|
||||
Revision: fmt.Sprintf("%s.%d", v.Branch, v.Revision)}
|
||||
if err := rv.sayReserved(ctx, ev); err != nil {
|
||||
rv.logf("the version-reserved of %s at %s is not said: %v", ref, v.Version, err)
|
||||
} else {
|
||||
rv.logf("version-reserved said: %s (%s) at %s, revision %s", ref, path, v.Version, ev.Revision)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// newReservations makes the reserving over store; the released versions of a module are asked of the seat mesh-catalog's
|
||||
@@ -79,7 +124,9 @@ func newReservations(store versions.ReserveStore, call toolCaller, bounds func()
|
||||
}
|
||||
return out, nil
|
||||
}}
|
||||
rv.r.Took, rv.r.Logf = rv.took, logf
|
||||
rv.r.Took, rv.r.Logf, rv.r.Reserved = rv.took, logf, rv.reservedSaid
|
||||
// A module's tags are read before it reserves, and a number tagged at another commit skipped (design 47 H2 §796).
|
||||
rv.r.Tags, rv.r.TaggedElsewhere = tagsVia(call), rv.taggedElsewhere
|
||||
return rv
|
||||
}
|
||||
|
||||
@@ -102,8 +149,8 @@ func (rv *reservations) took(ctx context.Context, older, newer versions.Reserved
|
||||
} else {
|
||||
rv.logf("%s: the reservation of %s (build of %s, never released) is dropped: the build of %s takes the series over and reserves %s", newer.Module, older.Version, older.Fingerprint, newer.Fingerprint, newer.Version)
|
||||
}
|
||||
if rv.released != nil {
|
||||
rv.released(ctx, older.Module, older.Version) // its publish item goes with its reservation (A-456b)
|
||||
if rv.itemEnded != nil {
|
||||
rv.itemEnded(ctx, older.Module, older.Version) // its publish item goes with its reservation (A-456b), its items terminated (§793)
|
||||
}
|
||||
rv.dropBound(ctx, versions.ReservedKey(older.Module, older.Series))
|
||||
rv.found.clear(ctx, versions.VersionUnreleased, meshconditionsclient.ScopeModule, older.Module)
|
||||
@@ -279,6 +326,7 @@ func (rv *reservations) heard(ctx context.Context, m jetstream.Msg) bool {
|
||||
}
|
||||
rv.found.clear(ctx, versions.VersionUnreleased, meshconditionsclient.ScopeModule, ev.Module)
|
||||
rv.found.clear(ctx, versions.VersionHeldElsewhere, meshconditionsclient.ScopeModule, ev.Module)
|
||||
rv.found.clear(ctx, versions.VersionTaggedElsewhere, meshconditionsclient.ScopeModule, ev.Module)
|
||||
default:
|
||||
return false
|
||||
}
|
||||
@@ -305,6 +353,15 @@ func (rv *reservations) published(ctx context.Context, o versions.Outcome, inPro
|
||||
for _, m := range out.Marked {
|
||||
rv.sayPublished(ctx, m, o.Digest)
|
||||
}
|
||||
if rv.itemHeard != nil { // the publish items a published newly marked are acknowledged (§793)
|
||||
seen := map[string]bool{}
|
||||
for _, m := range out.Marked {
|
||||
if k := m.Reserved.Module + " " + m.Reserved.Version; !seen[k] {
|
||||
seen[k] = true
|
||||
rv.itemHeard(ctx, m.Reserved.Module, m.Reserved.Version)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// sayPublished says the seat mesh-builds' package-published of an artifact a published marked.
|
||||
@@ -327,8 +384,8 @@ func (rv *reservations) dropped(ctx context.Context, module, series, by, why str
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if ok && held && rv.released != nil {
|
||||
rv.released(ctx, module, cur.Version) // its publish item goes with its reservation (A-456b)
|
||||
if ok && held && rv.itemEnded != nil {
|
||||
rv.itemEnded(ctx, module, cur.Version) // its publish item goes with its reservation (A-456b), its items terminated (§793)
|
||||
}
|
||||
if !ok {
|
||||
return "", fmt.Errorf("%s holds no reservation of %s", module, series)
|
||||
@@ -436,6 +493,19 @@ func (rv *reservations) heldElsewhere(ctx context.Context, module, why string) {
|
||||
needs: "builder_reserved shows the reservation and its withdrawn numbers; the registry seat's versions shows what it holds."})
|
||||
}
|
||||
|
||||
// taggedElsewhere raises version-tagged-elsewhere about module (design 47 H2 §796): numbers whose git tag stands at another
|
||||
// commit were skipped at reserving; a tag is never moved. Cleared at the module's next version-released.
|
||||
func (rv *reservations) taggedElsewhere(ctx context.Context, module string, skipped []string) {
|
||||
why := "the tags of " + strings.Join(skipped, ", ") + " stand at another commit, so the reserve skipped them"
|
||||
rv.logf("%s: %s", module, why)
|
||||
rv.found.raise(ctx, finding{kind: versions.VersionTaggedElsewhere, scope: meshconditionsclient.ScopeModule, id: module,
|
||||
summary: module + ": " + why,
|
||||
headline: "A number whose tag stands at another commit is not used",
|
||||
explanation: "The repository holds a tag of " + module + " at a commit other than the build's: " + why + ". A tag is never moved.",
|
||||
resolved: "The catalog says version-released for " + module + ".",
|
||||
needs: "builder_reserved shows the reservation; the seat git's tags shows each tag with its commit."})
|
||||
}
|
||||
|
||||
// movedElsewhere takes a done build whose number a registry holds at another digest off that number (design 47 H H-A-275,
|
||||
// H-A-279): every artifact carries its version from its build, so no digest survives a move. The number is withdrawn as a gap
|
||||
// (H-A-226), the finding raised naming both digests, and the build of the same fingerprint asked again (the build-again path of a
|
||||
|
||||
@@ -163,17 +163,27 @@ func TestAReleasedVersionIsTaggedAtTheCommitItWasBuiltFrom(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
var tags []git.TagInput
|
||||
refuse := ""
|
||||
var standing []git.Tag // the tags the seat git's tags answers (design 47 H2 §796)
|
||||
call := func(addr string, body any) (json.RawMessage, error) {
|
||||
if addr != "seat:git.tag" {
|
||||
switch addr {
|
||||
case "seat:git.tags":
|
||||
return json.Marshal(map[string]any{"form": 2, "tags": standing, "a-field-of-a-later-form": true})
|
||||
case "seat:git.tag":
|
||||
default:
|
||||
return json.Marshal(meshcatalog.VersionsAnswer{})
|
||||
}
|
||||
tags = append(tags, body.(git.TagInput))
|
||||
return json.Marshal(git.TagAnswer{Performed: refuse == "", Error: refuse})
|
||||
}
|
||||
found := &findings{module: builds.BuilderModule, w: &memFindings{vals: map[string][]byte{}}, logf: t.Logf, now: func() time.Time { return time.Unix(0, 0) }}
|
||||
fw := &memFindings{vals: map[string][]byte{}}
|
||||
found := &findings{module: builds.BuilderModule, w: fw, logf: t.Logf, now: func() time.Time { return time.Unix(0, 0) }}
|
||||
rv := newReservations(&memReserve{vals: map[string][]byte{}, revs: map[string]uint64{}}, call, func() jetstream.KeyValue { return &boundsKV{keys: map[string]bool{}} }, found, nil, t.Logf)
|
||||
rv.filled.Store(true)
|
||||
goPkg := versions.Source{Repository: "o/r", Commit: "c0ffee", GoPackage: true}
|
||||
// Design 47 H2 §796: the tag names the Go module's directory, read from its module path: mesh-sdk's `go` here.
|
||||
rv.r.Names = func(_ context.Context, module, artifact string) (string, bool, error) {
|
||||
return "git.example/o/r/go", artifact == "lib" || artifact == "pkg", nil
|
||||
}
|
||||
goPkg := versions.Source{Repository: "o/r", Commit: "c0ffee", GoPackage: true, GoArtifact: "lib"}
|
||||
if _, err := rv.Reserve(ctx, "pkg", "0.1", []string{"pkg:lib"}, goPkg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -204,7 +214,7 @@ func TestAReleasedVersionIsTaggedAtTheCommitItWasBuiltFrom(t *testing.T) {
|
||||
if _, err := rv.Reserve(ctx, "pkg", "0.1", []string{"pkg:lib"}, goPkg); err != nil { // build A: 0.1.0 (the stub catalog holds none released)
|
||||
t.Fatal(err)
|
||||
}
|
||||
if v, err := rv.Reserve(ctx, "pkg", "0.1", []string{"pkg:lib"}, versions.Source{Repository: "o/r", Commit: "b0b", GoPackage: true}); err != nil || v.Version != "0.1.1" { // build B supersedes A
|
||||
if v, err := rv.Reserve(ctx, "pkg", "0.1", []string{"pkg:lib"}, versions.Source{Repository: "o/r", Commit: "b0b", GoPackage: true, GoArtifact: "lib"}); err != nil || v.Version != "0.1.1" { // build B supersedes A
|
||||
t.Fatalf("second build reserved %q (%v)", v.Version, err)
|
||||
}
|
||||
release("pkg", "0.1.0") // A's version is never released in fact; were it said, it is no reservation of the key
|
||||
@@ -214,6 +224,41 @@ func TestAReleasedVersionIsTaggedAtTheCommitItWasBuiltFrom(t *testing.T) {
|
||||
if release("pkg", "0.1.1"); len(tags) != 1 || tags[0].Commit != "b0b" {
|
||||
t.Fatalf("the newer build's version: tags %+v; want go/v0.1.1 at b0b", tags)
|
||||
}
|
||||
// Design 47 H2 §796: a tag standing at another commit makes the next number and the finding version-tagged-elsewhere, which
|
||||
// clears at the module's next version-released.
|
||||
standing = []git.Tag{{Tag: "go/v0.2.0", Commit: "elsewhere"}, {Tag: "go/v0.2.1", Commit: "elsewhere"}}
|
||||
if v, err := rv.Reserve(ctx, "pkg", "0.2", []string{"pkg:lib"}, goPkg); err != nil || v.Version != "0.2.2" || v.Tag != "go/v0.2.2" {
|
||||
t.Fatalf("reserved %+v (%v) with 0.2.0 and 0.2.1 tagged elsewhere, want 0.2.2 tagged go/v0.2.2", v, err)
|
||||
}
|
||||
key := meshconditionsclient.Key(meshconditionsclient.ScopeModule, "pkg", versions.VersionTaggedElsewhere)
|
||||
if got, err := decodeFinding(fw.vals[key]); err != nil || !strings.Contains(got.Summary, "0.2.0") || !strings.Contains(got.Summary, "0.2.1") {
|
||||
t.Fatalf("the finding %s = %+v (%v), want it naming 0.2.0 and 0.2.1", key, got, err)
|
||||
}
|
||||
if release("pkg", "0.2.2"); fw.vals[key] != nil {
|
||||
t.Errorf("version-released of pkg left %s standing", key)
|
||||
}
|
||||
standing = nil
|
||||
// Design 47 H2 §805, H-A-420 (2): a package's number newly reserved is said as version-reserved, its path the package's name
|
||||
// and its revision the indexer's `<branch>.<n>`; a reservation kept as it stood, an artifact that is no package and a
|
||||
// reservation naming no catalog revision say none.
|
||||
var said []meshbuilds.VersionReservedEvent
|
||||
rv.names = func(_ context.Context, module, artifact string) (string, bool, error) {
|
||||
return "git.example/o/r/" + artifact, module == "lib2" && artifact == "pkg", nil
|
||||
}
|
||||
rv.sayReserved = func(_ context.Context, ev meshbuilds.VersionReservedEvent) error { said = append(said, ev); return nil }
|
||||
at := versions.Source{Repository: "o/r", Commit: "c1", Fingerprint: "src2:l2", GoPackage: true, GoArtifact: "pkg", Branch: "main", CatalogRevision: 158}
|
||||
for range 2 {
|
||||
if _, err := rv.Reserve(ctx, "lib2", "0.3", []string{"lib2:pkg", "lib2:bin"}, at); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err := rv.Reserve(ctx, "lib2", "0.4", []string{"lib2:pkg"}, versions.Source{Fingerprint: "src2:none"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := meshbuilds.VersionReservedEvent{Module: "lib2", Artifact: "lib2:pkg", Path: "git.example/o/r/pkg", Version: "0.3.0", Fingerprint: "src2:l2", Revision: "main.158"}
|
||||
if len(said) != 1 || said[0] != want {
|
||||
t.Errorf("version-reserved said %+v, want once %+v", said, want)
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 G A-415: a published heard before the build recorded its digests is kept on its reservation, and when the build
|
||||
@@ -412,6 +457,7 @@ func TestARegistrysRecordEndsOrMovesAPublish(t *testing.T) {
|
||||
var asked, ended []string
|
||||
rv.built = func(_ context.Context, m, v string) error { asked = append(asked, m+" "+v); return nil }
|
||||
rv.released = func(_ context.Context, m, v string) { ended = append(ended, m+" "+v) }
|
||||
rv.itemEnded = rv.released // a build withdrawn or superseded ends its publish item too (§793)
|
||||
registry := []string{"0.1.37"}
|
||||
rv.r.HeldNumbers = func(context.Context, string, string, []string) ([]string, error) { return registry, nil }
|
||||
rv.filled.Store(true)
|
||||
@@ -531,4 +577,55 @@ func TestAPublishedThatMarksNothingIsSaidFromWhatItRead(t *testing.T) {
|
||||
if store.reads != 2 || !strings.Contains(strings.Join(logs, "\n"), "m05 holds 1 built digests") || !strings.Contains(strings.Join(logs, "\n"), "14 reservations stand at it") {
|
||||
t.Errorf("%d reads for two publisheds, journal %q: want one read each and the line made from it", store.reads, logs)
|
||||
}
|
||||
// Design 47 H2 §794: an event whose handling passed 100 ms is said with how long and the part that took it; one within it
|
||||
// is not. And H-A-435: every message the builder's handlers take is noted on its consumer's ledger, so the state consumers
|
||||
// that builder_health reads holds a message left unacknowledged until it is acknowledged.
|
||||
t.Run("a slow handling is said and a message left unacknowledged is kept", func(t *testing.T) {
|
||||
var said []string
|
||||
clock := time.Unix(1000, 0)
|
||||
now := func() time.Time { return clock }
|
||||
slow := []eventTaker{{"the reservations", func(jetstream.Msg) bool { clock = clock.Add(150 * time.Millisecond); return true }}}
|
||||
fast := []eventTaker{{"the floor", func(jetstream.Msg) bool { clock = clock.Add(100 * time.Millisecond); return true }}}
|
||||
rest := eventTaker{"the artifacts gone", func(jetstream.Msg) bool { return true }}
|
||||
takeEvent(&agedTestMsg{subject: "a.published", seq: 1}, slow, rest, now, func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) })
|
||||
takeEvent(&agedTestMsg{subject: "a.floor", seq: 2}, fast, rest, now, func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) })
|
||||
if len(said) != 1 || said[0] != "the event a.published took 150ms to handle, over 100ms: the reservations took it" {
|
||||
t.Errorf("said %q, want one line for the handling over 100 ms", said)
|
||||
}
|
||||
facts := map[string]string{}
|
||||
l := &ledgers{now: now, sink: func(_ context.Context, k string, v []byte) error { facts[k] = string(v); return nil }}
|
||||
hub := &oneHandler{}
|
||||
aged := agedHub{hub: hub, ledgers: l}
|
||||
_ = aged.Handle("x.>", func(m jetstream.Msg) {
|
||||
if m.Subject() == "x.ack" {
|
||||
_ = m.Ack()
|
||||
}
|
||||
})
|
||||
hub.handle(&agedTestMsg{subject: "x.left", seq: 7})
|
||||
if !strings.Contains(facts["builder-events"], `"oldest-unacknowledged":"1970-01-01T00:16:40.25Z"`) {
|
||||
t.Errorf("facts %q, want the message left unacknowledged kept with its delivery", facts)
|
||||
}
|
||||
hub.handle(&agedTestMsg{subject: "x.ack", seq: 8})
|
||||
if !strings.Contains(facts["builder-events"], `"acknowledged":8`) {
|
||||
t.Errorf("facts %q, want the acknowledged message applied", facts)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// oneHandler is an event hub with one handler.
|
||||
type oneHandler struct{ handle func(jetstream.Msg) }
|
||||
|
||||
func (h *oneHandler) Handle(_ string, f func(jetstream.Msg)) error { h.handle = f; return nil }
|
||||
|
||||
// agedTestMsg is a message of the builder's events consumer, with the metadata a ledger reads.
|
||||
type agedTestMsg struct {
|
||||
jetstream.Msg
|
||||
subject string
|
||||
seq uint64
|
||||
}
|
||||
|
||||
func (m *agedTestMsg) Subject() string { return m.subject }
|
||||
func (m *agedTestMsg) Ack() error { return nil }
|
||||
func (m *agedTestMsg) Metadata() (*jetstream.MsgMetadata, error) {
|
||||
return &jetstream.MsgMetadata{Sequence: jetstream.SequencePair{Stream: m.seq}, NumDelivered: 1, Consumer: "builder-events", Stream: "EVENTS"}, nil
|
||||
}
|
||||
|
||||
@@ -3,14 +3,94 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
|
||||
"mesh-delivery-builder/internal/build"
|
||||
"mesh-delivery-builder/internal/versions"
|
||||
)
|
||||
|
||||
// takenFake is the state taken: keys created once, with their time to live.
|
||||
type takenFake struct {
|
||||
mu sync.Mutex
|
||||
vals map[string][]byte
|
||||
ttls map[string]time.Duration
|
||||
}
|
||||
|
||||
func (f *takenFake) TakeOnce(seat, ask string, bound time.Duration) (bool, error) {
|
||||
_, err := f.CreateTTL(seat+"."+ask, map[string]string{"seat": seat, "ask": ask}, bound)
|
||||
return err == nil, nil
|
||||
}
|
||||
func (f *takenFake) Get(key string) (*stdio.StateEntry, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if v, ok := f.vals[key]; ok {
|
||||
return &stdio.StateEntry{Key: key, Value: v}, nil
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
func (f *takenFake) CreateTTL(key string, value any, ttl time.Duration) (uint64, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if _, ok := f.vals[key]; ok {
|
||||
return 0, stdio.ErrChanged
|
||||
}
|
||||
raw, _ := json.Marshal(value)
|
||||
f.vals[key], f.ttls[key] = raw, ttl
|
||||
return 1, nil
|
||||
}
|
||||
|
||||
// blockingBuilder runs a build step until its context ends, and answers the drop that ended it.
|
||||
type blockingBuilder struct{ started chan string }
|
||||
|
||||
func (b blockingBuilder) BuildStep(ctx context.Context, ask string, run meshbuilds.StepRun) meshbuilds.StepEnded {
|
||||
b.started <- ask
|
||||
<-ctx.Done()
|
||||
end := meshbuilds.StepEnded{ID: ask, Pipeline: run.Pipeline, Outcome: meshbuilds.StepEndedOutcomeCouldNotRun}
|
||||
if d := build.DroppedOf(ctx); d != nil {
|
||||
end.Said = d.Error()
|
||||
}
|
||||
return end
|
||||
}
|
||||
|
||||
// msgFake is a message pulled from the seat's work queue.
|
||||
type msgFake struct {
|
||||
h nats.Header
|
||||
data []byte
|
||||
acked chan struct{}
|
||||
}
|
||||
|
||||
func (m *msgFake) Data() []byte { return m.data }
|
||||
func (m *msgFake) Headers() nats.Header { return m.h }
|
||||
func (m *msgFake) Ack() error { close(m.acked); return nil }
|
||||
func (m *msgFake) Nak() error { return nil }
|
||||
func (m *msgFake) Term() error { return nil }
|
||||
func (m *msgFake) InProgress() error { return nil }
|
||||
|
||||
// await fails the test unless c yields within 5 s.
|
||||
func await[T any](t *testing.T, c <-chan T, what string) {
|
||||
t.Helper()
|
||||
select {
|
||||
case <-c:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatalf("%s did not happen within 5 s", what)
|
||||
}
|
||||
}
|
||||
|
||||
func stepMsg(id string) *msgFake {
|
||||
data, _ := json.Marshal(meshbuilds.StepRun{Pipeline: "main.1.m", Step: meshbuilds.StepRunStepBuild, Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: "src2:x"}}})
|
||||
return &msgFake{h: nats.Header{"x-event-id": []string{id}}, data: data, acked: make(chan struct{})}
|
||||
}
|
||||
|
||||
// Design 47 G A-115, A-127: the builder's process answers the verb reserved, given a name, from the catalog half's carriers and the
|
||||
// seat mesh-catalog's versions, and given a module from its reservations.
|
||||
func TestTheProcessAnswersReserved(t *testing.T) {
|
||||
@@ -47,4 +127,55 @@ func TestTheProcessAnswersReserved(t *testing.T) {
|
||||
if _, err := handle("", json.RawMessage(`{"module":`)); err == nil {
|
||||
t.Error("arguments that are no JSON were answered")
|
||||
}
|
||||
|
||||
// Design 47 H2 §778, H-A-401, H-A-404: the verb drop cancels an ask that runs here, and keeps the drop of one not
|
||||
// pulled yet in the state taken for ask-taken-bound; both end could-not-run naming the drop and who asked it.
|
||||
taken := &takenFake{vals: map[string][]byte{}, ttls: map[string]time.Duration{}}
|
||||
started := make(chan string, 2)
|
||||
var mu sync.Mutex
|
||||
said := map[string]meshbuilds.StepEnded{}
|
||||
var subjects []string
|
||||
q := &stepQueue{node: "n1", builder: blockingBuilder{started: started}, taken: taken, takenBound: 10 * time.Minute,
|
||||
inProgress: func() time.Duration { return 0 }, logf: t.Logf, now: time.Now, running: map[string]*runningAsk{},
|
||||
say: func(_ context.Context, subject, id string, body any) error {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
subjects = append(subjects, subject)
|
||||
if end, ok := body.(meshbuilds.StepEnded); ok {
|
||||
said[id] = end
|
||||
}
|
||||
return nil
|
||||
}}
|
||||
drop := dropVerb(q)
|
||||
running := stepMsg("ask-1")
|
||||
q.take(t.Context(), running)
|
||||
await(t, started, "the build of ask-1")
|
||||
if got, err := drop("mesh-delivery-pipeline", json.RawMessage(`{"id":"ask-1","why":"its step ended by its bound"}`)); err != nil || got.(meshbuilds.DropAnswer)["held"] != true {
|
||||
t.Fatalf("drop of a running ask = %v, %v; want held", got, err)
|
||||
}
|
||||
await(t, running.acked, "the end of ask-1")
|
||||
if got, err := drop("mesh-delivery-pipeline", json.RawMessage(`{"id":"ask-2","by":"op@shanks","why":"by hand"}`)); err != nil || got.(meshbuilds.DropAnswer)["held"] != false {
|
||||
t.Fatalf("drop of an ask not pulled = %v, %v; want not held", got, err)
|
||||
}
|
||||
if taken.ttls[build.DroppedKey("ask-2")] != 10*time.Minute {
|
||||
t.Errorf("the drop of ask-2 is kept for %v, want ask-taken-bound", taken.ttls[build.DroppedKey("ask-2")])
|
||||
}
|
||||
later := stepMsg("ask-2")
|
||||
q.take(t.Context(), later)
|
||||
await(t, started, "the build step of ask-2")
|
||||
await(t, later.acked, "the end of ask-2")
|
||||
mu.Lock()
|
||||
if e := said["ask-1"]; e.Outcome != meshbuilds.StepEndedOutcomeCouldNotRun || e.Said != "the ask was dropped by mesh-delivery-pipeline through the seat mesh-builds' drop: its step ended by its bound" {
|
||||
t.Errorf("the running ask ended %+v", e)
|
||||
}
|
||||
if e := said["ask-2"]; e.Outcome != meshbuilds.StepEndedOutcomeCouldNotRun || !strings.Contains(e.Said, "dropped by op@shanks") {
|
||||
t.Errorf("the ask dropped before its pull ended %+v", e)
|
||||
}
|
||||
if _, ok := taken.vals["mesh-builds.ask-2"]; ok || strings.Count(strings.Join(subjects, " "), "ask-taken") != 1 {
|
||||
t.Errorf("the ask dropped before its pull was taken: subjects said %v", subjects)
|
||||
}
|
||||
mu.Unlock()
|
||||
if _, err := drop("", json.RawMessage(`{"id":"ask-3"}`)); err == nil {
|
||||
t.Error("a drop with no why was taken")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,19 +2,11 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sync"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"mesh-delivery-builder/internal/build"
|
||||
)
|
||||
|
||||
// **The runner's build step ask is the builder's one entry point for a candidate** (design 47 G A-97, A-84, A-99): the seat
|
||||
@@ -72,30 +64,3 @@ func (w *entryWaits) entry(ctx context.Context, fingerprint string) (meshcatalog
|
||||
}
|
||||
return e, w.manifest.keep(e) // the manifest the build is planned from
|
||||
}
|
||||
|
||||
// serveSteps hears the step asks on the seat's accept subject and answers each one's end, in its own goroutine.
|
||||
func serveSteps(ctx context.Context, conn *nats.Conn, js jetstream.JetStream, cred process.Account, b *build.Builder, logf func(string, ...any)) error {
|
||||
sub, err := conn.Subscribe(meshbuilds.AcceptSubjectStep, func(m *nats.Msg) {
|
||||
id := m.Header.Get("x-event-id")
|
||||
if id == "" {
|
||||
id = m.Header.Get(nats.MsgIdHdr)
|
||||
}
|
||||
var run meshbuilds.StepRun
|
||||
if err := json.Unmarshal(m.Data, &run); err != nil || id == "" || run.Step != meshbuilds.StepRunStepBuild {
|
||||
logf("a step ask on %s that is no build step with an id is left: %v", m.Subject, err)
|
||||
return
|
||||
}
|
||||
go func() {
|
||||
end := b.BuildStep(ctx, run)
|
||||
end.ID = id
|
||||
if err := emitSeatEvent(context.WithoutCancel(ctx), js, cred, meshbuilds.SubjectStepEnded(cred.Node), id, end); err != nil {
|
||||
logf("saying the end of the build step %s: %v", id, err)
|
||||
}
|
||||
}()
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
go func() { <-ctx.Done(); _ = sub.Drain() }()
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -4,14 +4,15 @@ package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
)
|
||||
|
||||
// The runner's build step ask on the seat mesh-builds' accept subject, naming an entry by its fingerprint, is built through the
|
||||
@@ -30,16 +31,33 @@ func TestTheRunnersBuildStepAskIsAnsweredAsStepEnded(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer s.Unsubscribe()
|
||||
b.start(t, "n1", nil)
|
||||
b.await(t, "netcheck", isState(builds.BuildDone))
|
||||
taken := make(chan *nats.Msg, 4)
|
||||
st, err := sub.ChanSubscribe(meshbuilds.SubjectAskTakenAll, taken)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer st.Unsubscribe()
|
||||
// Design 47 H2 §777: an ask put on the seat's work queue while the builder is stopped waits there, and is taken (ask-taken)
|
||||
// and ended once the builder starts.
|
||||
run := meshbuilds.StepRun{Pipeline: "main.1.netcheck", Step: meshbuilds.StepRunStepBuild, Branch: "main", Revision: 1,
|
||||
Entries: []meshbuilds.BuildEntry{{Module: "netcheck", Fingerprint: fp}}}
|
||||
data, _ := json.Marshal(run)
|
||||
msg := nats.NewMsg(meshbuilds.AcceptSubjectStep)
|
||||
msg.Data = data
|
||||
msg.Header.Set("x-event-id", "ask-1")
|
||||
if err := sub.PublishMsg(msg); err != nil {
|
||||
t.Fatal(err)
|
||||
msg.Header.Set(nats.MsgIdHdr, "ask-1")
|
||||
if _, err := b.js.PublishMsg(b.ctx, msg); err != nil {
|
||||
t.Fatalf("the ask is not put on the seat's work queue: %v", err)
|
||||
}
|
||||
b.start(t, "n1", nil)
|
||||
select {
|
||||
case m := <-taken:
|
||||
var at meshbuilds.AskTakenEvent
|
||||
if err := json.Unmarshal(m.Data, &at); err != nil || at.Ask != "ask-1" || at.Node != "n1" || at.At.IsZero() || m.Subject != meshbuilds.SubjectAskTaken("ask-1") {
|
||||
t.Fatalf("ask-taken on %s = %+v, %v; want ask-1 taken on n1", m.Subject, at, err)
|
||||
}
|
||||
case <-time.After(20 * time.Second):
|
||||
t.Fatal("no ask-taken within 20 s of the start")
|
||||
}
|
||||
select {
|
||||
case m := <-ended:
|
||||
@@ -75,4 +93,24 @@ func TestTheRunnersBuildStepAskIsAnsweredAsStepEnded(t *testing.T) {
|
||||
case <-time.After(20 * time.Second):
|
||||
t.Fatal("no step-ended for ask-2 within 20 s")
|
||||
}
|
||||
// Each ask is acknowledged once its end was said: the seat's worker holds none pending.
|
||||
stream, worker := buscompose.SeatQueueStreamName(meshbuilds.Seat), buscompose.SeatQueueWorkerName(meshbuilds.Seat)
|
||||
deadline := time.Now().Add(10 * time.Second)
|
||||
for {
|
||||
c, err := b.js.Consumer(b.ctx, stream, worker)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
info, err := c.Info(b.ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if info.NumPending == 0 && info.NumAckPending == 0 {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("the seat's worker holds %d waiting and %d taken after both asks ended, want none", info.NumPending, info.NumAckPending)
|
||||
}
|
||||
runtime.Gosched()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,309 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"git.novox.be/novox/mesh-sdk/go/seatdef"
|
||||
|
||||
"mesh-delivery-builder/internal/build"
|
||||
)
|
||||
|
||||
// **The seat mesh-builds takes its asks from a work queue and says ask-taken** (design 47 H2 §777, §775; ADR 0525 §1, §2,
|
||||
// §6; ADR 0668 §2): the broker derives the seat's stream and its one shared worker from the seat's definition; the builder
|
||||
// pulls every ask waiting at once, takes each once by its id in its state taken (TakenKey <seat>.<ask>, for
|
||||
// ask-taken-bound; H-A-381 (2)), says ask-taken at the pull, sends in-progress while it builds, and acknowledges an ask
|
||||
// when it said the ask's end. An ask put while the builder is down waits on the stream.
|
||||
|
||||
// askTakenBoundSetting is the setting the taken key's time to live is (H-A-381 (2)): the holder never receives the
|
||||
// caller's bound, so the largest caller's, the runner's ask-unanswered-bound, is its default.
|
||||
const (
|
||||
askTakenBoundSetting = "ask-taken-bound"
|
||||
defaultAskTakenBound = 10 * time.Minute
|
||||
)
|
||||
|
||||
// askTaker is the state taken (process.State): an ask taken once by its id, and the drop of an ask not pulled yet kept
|
||||
// under build.DroppedKey (H-A-404).
|
||||
type askTaker interface {
|
||||
TakeOnce(seat, ask string, bound time.Duration) (bool, error)
|
||||
Get(key string) (*stdio.StateEntry, error)
|
||||
CreateTTL(key string, value any, ttl time.Duration) (uint64, error)
|
||||
}
|
||||
|
||||
// stepBuilder is build.Builder's BuildStep.
|
||||
type stepBuilder interface {
|
||||
BuildStep(ctx context.Context, ask string, run meshbuilds.StepRun) meshbuilds.StepEnded
|
||||
}
|
||||
|
||||
// queueMsg is what the queue reads of a message pulled from the seat's work queue (jetstream.Msg).
|
||||
type queueMsg interface {
|
||||
Data() []byte
|
||||
Headers() nats.Header
|
||||
Ack() error
|
||||
Nak() error
|
||||
Term() error
|
||||
InProgress() error
|
||||
}
|
||||
|
||||
// stepQueue is the builder's pull of the seat mesh-builds' work queue.
|
||||
type stepQueue struct {
|
||||
node string
|
||||
builder stepBuilder
|
||||
taken askTaker
|
||||
takenBound time.Duration
|
||||
// inProgress is how often a running ask is said in progress: the seat definition's in-progress.
|
||||
inProgress func() time.Duration
|
||||
// say publishes a seat event of the builder (emitSeatEvent).
|
||||
say func(ctx context.Context, subject, id string, body any) error
|
||||
logf func(string, ...any)
|
||||
now func() time.Time
|
||||
|
||||
mu sync.Mutex
|
||||
running map[string]*runningAsk
|
||||
}
|
||||
|
||||
// runningAsk is an ask the builder took and builds now.
|
||||
type runningAsk struct {
|
||||
run meshbuilds.StepRun
|
||||
taken time.Time
|
||||
cancel context.CancelCauseFunc
|
||||
}
|
||||
|
||||
// askIDOf is the ask's id a put carries: its event id, else its message id.
|
||||
func askIDOf(h nats.Header) string {
|
||||
if id := h.Get("x-event-id"); id != "" {
|
||||
return id
|
||||
}
|
||||
return h.Get(nats.MsgIdHdr)
|
||||
}
|
||||
|
||||
// take handles one ask pulled from the queue: an ask that is no build step with an id is terminated, said; one this
|
||||
// builder runs already (handed again while it runs) is left to the run that holds it; any other is taken (ask-taken said
|
||||
// the first time its id is taken) and built on its own goroutine, in progress while it runs, acknowledged once its end is
|
||||
// said. A repeat of an id taken before that runs here no longer (the builder started again) is answered as its builds
|
||||
// stand now, read afresh by BuildStep (H-A-381 (1)): an entry built is answered built, never built twice.
|
||||
func (q *stepQueue) take(ctx context.Context, m queueMsg) {
|
||||
id := askIDOf(m.Headers())
|
||||
var run meshbuilds.StepRun
|
||||
err := json.Unmarshal(m.Data(), &run)
|
||||
if err != nil || id == "" || run.Step != meshbuilds.StepRunStepBuild {
|
||||
q.logf("a step ask on the seat %s's work queue that is no build step with an id is ended: %v", meshbuilds.Seat, err)
|
||||
_ = m.Term()
|
||||
return
|
||||
}
|
||||
q.mu.Lock()
|
||||
if _, held := q.running[id]; held {
|
||||
q.mu.Unlock()
|
||||
q.logf("the ask %s (%s) was handed again while it runs here: its run answers it", id, run.Pipeline)
|
||||
return
|
||||
}
|
||||
// An ask dropped before it was pulled ends could-not-run naming the drop, with no build (H-A-404).
|
||||
if d := q.droppedAsk(id); d != nil {
|
||||
work, cancel := context.WithCancelCause(ctx)
|
||||
cancel(d)
|
||||
ra := &runningAsk{run: run, taken: q.now(), cancel: cancel}
|
||||
q.running[id] = ra
|
||||
q.mu.Unlock()
|
||||
q.logf("the ask %s (%s) was dropped before it was pulled: %s", id, run.Pipeline, d.Error())
|
||||
go q.build(work, m, id, ra)
|
||||
return
|
||||
}
|
||||
first, err := q.taken.TakeOnce(meshbuilds.Seat, id, q.takenBound)
|
||||
if err != nil {
|
||||
q.mu.Unlock()
|
||||
q.logf("the ask %s (%s) is not taken, the state %s refused it: %v; it is handed again", id, run.Pipeline, process.TakenState, err)
|
||||
_ = m.Nak()
|
||||
return
|
||||
}
|
||||
work, cancel := context.WithCancelCause(ctx)
|
||||
ra := &runningAsk{run: run, taken: q.now(), cancel: cancel}
|
||||
q.running[id] = ra
|
||||
q.mu.Unlock()
|
||||
if first {
|
||||
if err := q.say(ctx, meshbuilds.SubjectAskTaken(id), id+"-taken", meshbuilds.AskTakenEvent{Ask: id, Node: q.node, At: ra.taken.UTC()}); err != nil {
|
||||
q.logf("saying ask-taken of %s: %v", id, err)
|
||||
}
|
||||
} else {
|
||||
q.logf("the ask %s (%s) was taken before: it is answered as its builds stand now", id, run.Pipeline)
|
||||
}
|
||||
go q.build(work, m, id, ra)
|
||||
}
|
||||
|
||||
// build runs one taken ask to its end and says it.
|
||||
func (q *stepQueue) build(ctx context.Context, m queueMsg, id string, ra *runningAsk) {
|
||||
defer func() {
|
||||
ra.cancel(nil)
|
||||
q.mu.Lock()
|
||||
delete(q.running, id)
|
||||
q.mu.Unlock()
|
||||
}()
|
||||
every := q.inProgress()
|
||||
var tick <-chan time.Time
|
||||
if every > 0 {
|
||||
t := time.NewTicker(every)
|
||||
defer t.Stop()
|
||||
tick = t.C
|
||||
}
|
||||
done := make(chan meshbuilds.StepEnded, 1)
|
||||
go func() { done <- q.builder.BuildStep(ctx, id, ra.run) }()
|
||||
var end meshbuilds.StepEnded
|
||||
for waiting := true; waiting; {
|
||||
select {
|
||||
case end = <-done:
|
||||
waiting = false
|
||||
case <-tick:
|
||||
_ = m.InProgress()
|
||||
}
|
||||
}
|
||||
if err := q.say(context.WithoutCancel(ctx), meshbuilds.SubjectStepEnded(q.node), id, end); err != nil {
|
||||
q.logf("saying the end of the build step %s: %v; the ask is handed again", id, err)
|
||||
_ = m.Nak()
|
||||
return
|
||||
}
|
||||
_ = m.Ack()
|
||||
}
|
||||
|
||||
// droppedAsk is the drop kept for the ask id in the state taken, nil for none.
|
||||
func (q *stepQueue) droppedAsk(id string) *build.Dropped {
|
||||
e, err := q.taken.Get(build.DroppedKey(id))
|
||||
if err != nil || e == nil {
|
||||
return nil
|
||||
}
|
||||
var d build.Dropped
|
||||
if json.Unmarshal(e.Value, &d) != nil {
|
||||
return nil
|
||||
}
|
||||
return &d
|
||||
}
|
||||
|
||||
// drop is the seat mesh-builds' verb drop (§778; H-A-401, H-A-404): the drop is kept in the state taken under
|
||||
// build.DroppedKey for ask-taken-bound, so a pull of the ask within the bound ends it, and an ask that runs here is
|
||||
// cancelled; both end the step could-not-run naming the drop and who asked it. held says whether it ran here.
|
||||
func (q *stepQueue) drop(id, by, why string) (held bool, err error) {
|
||||
if strings.TrimSpace(id) == "" || strings.TrimSpace(why) == "" {
|
||||
return false, errors.New("give the ask's id and why it is dropped")
|
||||
}
|
||||
d := &build.Dropped{By: by, Why: why}
|
||||
if _, err := q.taken.CreateTTL(build.DroppedKey(id), d, q.takenBound); err != nil && !errors.Is(err, stdio.ErrChanged) {
|
||||
return false, fmt.Errorf("the drop of %s is not kept in the state %s: %w", id, process.TakenState, err)
|
||||
}
|
||||
q.mu.Lock()
|
||||
ra := q.running[id]
|
||||
q.mu.Unlock()
|
||||
if ra != nil {
|
||||
ra.cancel(d)
|
||||
}
|
||||
q.logf("the ask %s is dropped by %s: %s", id, by, why)
|
||||
return ra != nil, nil
|
||||
}
|
||||
|
||||
// dropVerb is the handler of the seat mesh-builds' verb drop: who drops it is the input's by, else the caller.
|
||||
func dropVerb(q *stepQueue) process.SeatHandler {
|
||||
return func(caller string, body json.RawMessage) (any, error) {
|
||||
var in meshbuilds.DropInput
|
||||
if err := json.Unmarshal(body, &in); err != nil {
|
||||
return nil, fmt.Errorf("the arguments do not fit the verb drop: %w", err)
|
||||
}
|
||||
by := in.By
|
||||
if by == "" {
|
||||
by = caller
|
||||
}
|
||||
held, err := q.drop(in.ID, by, in.Why)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return meshbuilds.DropAnswer{"held": held}, nil
|
||||
}
|
||||
}
|
||||
|
||||
// running answers the asks this builder runs now, by id.
|
||||
func (q *stepQueue) runningAsks() map[string]runningAsk {
|
||||
q.mu.Lock()
|
||||
defer q.mu.Unlock()
|
||||
out := make(map[string]runningAsk, len(q.running))
|
||||
for id, ra := range q.running {
|
||||
out[id] = *ra
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// inProgressOf is the seat's in-progress as its definition names it, once the seat list is read; before, a third of the
|
||||
// worker's ack wait, which the broker derived from the same definition (60 s and 20 s, §777).
|
||||
func inProgressOf(def func() (seatdef.Definition, bool), ackWait func() time.Duration) func() time.Duration {
|
||||
return func() time.Duration {
|
||||
if d, ok := def(); ok {
|
||||
if _, worker, ok, err := buscompose.SeatQueue(meshbuilds.Seat, d.Accepts); err == nil && ok && worker.InProgressSeconds > 0 {
|
||||
return time.Duration(worker.InProgressSeconds) * time.Second
|
||||
}
|
||||
}
|
||||
return ackWait() / 3
|
||||
}
|
||||
}
|
||||
|
||||
// serveSteps pulls the seat's work queue through its one shared worker, every ask waiting at once (at most the worker's
|
||||
// max ack pending), once the broker made it: a worker not there yet is said, and bound when the bus says it was created.
|
||||
func serveSteps(ctx context.Context, conn *nats.Conn, js jetstream.JetStream, q *stepQueue, ackWait *atomic.Int64) error {
|
||||
stream, worker := buscompose.SeatQueueStreamName(meshbuilds.Seat), buscompose.SeatQueueWorkerName(meshbuilds.Seat)
|
||||
var once sync.Once
|
||||
bind := func() bool {
|
||||
c, err := js.Consumer(ctx, stream, worker)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
bound := false
|
||||
once.Do(func() {
|
||||
bound = true
|
||||
info := c.CachedInfo()
|
||||
if info != nil && info.Config.AckWait > 0 {
|
||||
ackWait.Store(int64(info.Config.AckWait))
|
||||
}
|
||||
pending := 1000
|
||||
if info != nil && info.Config.MaxAckPending > 0 {
|
||||
pending = info.Config.MaxAckPending
|
||||
}
|
||||
cc, err := c.Consume(func(m jetstream.Msg) { q.take(ctx, m) }, jetstream.PullMaxMessages(pending))
|
||||
if err != nil {
|
||||
q.logf("the seat %s's work queue %s is not pulled: %v", meshbuilds.Seat, worker, err)
|
||||
return
|
||||
}
|
||||
q.logf("pulling the seat %s's asks from %s", meshbuilds.Seat, worker)
|
||||
go func() { <-ctx.Done(); cc.Stop() }()
|
||||
})
|
||||
return bound
|
||||
}
|
||||
if bind() {
|
||||
return nil
|
||||
}
|
||||
q.logf("the seat %s's work queue %s on %s is not there yet: its asks wait on it, and it is pulled when the bus says it was created", meshbuilds.Seat, worker, stream)
|
||||
sub, err := conn.Subscribe("$JS.EVENT.ADVISORY.CONSUMER.CREATED."+stream+"."+worker, func(*nats.Msg) { go bind() })
|
||||
if err != nil {
|
||||
return fmt.Errorf("hearing the creation of the seat %s's work queue: %w", meshbuilds.Seat, err)
|
||||
}
|
||||
go func() { <-ctx.Done(); _ = sub.Unsubscribe() }()
|
||||
if bind() { // made between the first look and the subscription
|
||||
_ = sub.Unsubscribe()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// askTakenBound reads the setting ask-taken-bound (§781).
|
||||
func askTakenBound() (time.Duration, error) {
|
||||
d, err := stdio.SettingDuration(askTakenBoundSetting, defaultAskTakenBound)
|
||||
if err != nil {
|
||||
return 0, errors.New("the builder's " + err.Error())
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"mesh-delivery-builder/internal/build"
|
||||
)
|
||||
|
||||
// unbuildableKept is the builder's state unbuildable on the bus as build.Unbuildables (design 47 H2 §795, H-A-425).
|
||||
type unbuildableKept struct{ b *process.AwaitedBucket }
|
||||
|
||||
func (u unbuildableKept) Put(ctx context.Context, module string, value []byte) error {
|
||||
_, err := u.b.Put(ctx, module, value)
|
||||
if errors.Is(err, process.ErrNotYet) {
|
||||
return nil // kept, and written when the state appears
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// Delete removes module's key when the state holds it: a key not there is no error, and writes no marker.
|
||||
func (u unbuildableKept) Delete(ctx context.Context, module string) error {
|
||||
kv := u.b.KV()
|
||||
if kv == nil {
|
||||
return nil
|
||||
}
|
||||
if _, err := kv.Get(ctx, module); errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return nil
|
||||
} else if err != nil {
|
||||
return err
|
||||
}
|
||||
return kv.Delete(ctx, module)
|
||||
}
|
||||
|
||||
func (u unbuildableKept) Keys(ctx context.Context) ([]string, error) {
|
||||
kv := u.b.KV()
|
||||
if kv == nil {
|
||||
return nil, nil
|
||||
}
|
||||
l, err := kv.ListKeys(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = l.Stop() }()
|
||||
var out []string
|
||||
for k := range l.Keys() {
|
||||
out = append(out, k)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
var _ build.Unbuildables = unbuildableKept{}
|
||||
@@ -145,33 +145,44 @@ func floorVia(call toolCaller, requirements func(context.Context, string) ([]str
|
||||
}
|
||||
return out, nil
|
||||
},
|
||||
Tags: func(_ context.Context, repository, prefix string) ([]string, error) {
|
||||
// The seat git's verb tags, whichever module holds the seat (design 47 G §7, A-82), page by page.
|
||||
var out []string
|
||||
for page := int64(1); ; page++ {
|
||||
raw, err := call("seat:"+gitseat.Seat+"."+gitseat.VerbTags, gitseat.TagsInput{Repository: repository, Prefix: prefix, Page: page})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var a gitseat.TagsAnswer
|
||||
if err := json.Unmarshal(raw, &a); err != nil {
|
||||
return nil, fmt.Errorf("the seat git's tags answered what is no list of tags: %w", err)
|
||||
}
|
||||
if a.Error != "" {
|
||||
return nil, fmt.Errorf("the seat git's tags: %s", a.Error)
|
||||
}
|
||||
for _, t := range a.Tags {
|
||||
out = append(out, t.Tag)
|
||||
}
|
||||
if !a.More {
|
||||
return out, nil
|
||||
}
|
||||
Tags: func(ctx context.Context, repository, prefix string) ([]string, error) {
|
||||
tags, _, err := tagsVia(call)(ctx, repository, prefix)
|
||||
out := make([]string, 0, len(tags))
|
||||
for _, t := range tags {
|
||||
out = append(out, t.Tag)
|
||||
}
|
||||
return out, err
|
||||
},
|
||||
Requirements: requirements,
|
||||
}
|
||||
}
|
||||
|
||||
// tagsVia reads a repository's tags starting with prefix, each with the commit it names, through the seat git's verb tags,
|
||||
// whichever module holds the seat (design 47 G §7, A-82), page by page; its answer is read tolerantly, form 2 naming each tag's
|
||||
// commit (design 47 H2 §796, §780). away says the read met a transport fault or the error kind away (§776).
|
||||
func tagsVia(call toolCaller) func(ctx context.Context, repository, prefix string) ([]gitseat.Tag, bool, error) {
|
||||
return func(_ context.Context, repository, prefix string) ([]gitseat.Tag, bool, error) {
|
||||
var out []gitseat.Tag
|
||||
for page := int64(1); ; page++ {
|
||||
raw, err := call("seat:"+gitseat.Seat+"."+gitseat.VerbTags, gitseat.TagsInput{Repository: repository, Prefix: prefix, Page: page})
|
||||
if err != nil {
|
||||
return nil, true, err
|
||||
}
|
||||
a, _, err := gitseat.DecodeTagsAnswer(raw)
|
||||
if err != nil {
|
||||
return nil, false, fmt.Errorf("the seat git's tags answered what is no list of tags: %w", err)
|
||||
}
|
||||
if a.Error != "" {
|
||||
return nil, a.ErrorKind == gitseat.ErrorKindAway, fmt.Errorf("the seat git's tags: %s (%s)", a.Error, a.ErrorKind)
|
||||
}
|
||||
out = append(out, a.Tags...)
|
||||
if !a.More {
|
||||
return out, false, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// emitEvent says one of the builder's events, mesh.mod.<module>.event.<event>, with the SDK's envelope headers and id as
|
||||
// the bus's dedup id.
|
||||
func emitEvent(ctx context.Context, js jetstream.JetStream, cred process.Account, event, id string, body any) error {
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
|
||||
@@ -252,6 +253,23 @@ func (c Claimer) Claim(ctx context.Context, module, fingerprint, manifest string
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// endOf gives a done or failed build its end and outcome (design 47 H2 §797): the end the key holds already for that outcome, else
|
||||
// now. A build that has not ended names neither.
|
||||
func endOf(was, b builds.BuilderBuild, now func() time.Time) builds.BuilderBuild {
|
||||
if b.State != builds.BuildDone && b.State != builds.BuildFailed {
|
||||
b.Ended, b.Outcome = time.Time{}, ""
|
||||
return b
|
||||
}
|
||||
b.Outcome = b.State
|
||||
if b.Ended.IsZero() {
|
||||
b.Ended = now().UTC()
|
||||
if was.Outcome == b.State && !was.Ended.IsZero() {
|
||||
b.Ended = was.Ended
|
||||
}
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func (c Claimer) write(_ context.Context, key string, b builds.BuilderBuild, put func([]byte) (uint64, error)) (*Claim, error) {
|
||||
data, err := builds.EncodeBuilderBuild(b)
|
||||
if err != nil {
|
||||
@@ -272,6 +290,7 @@ func (c Claimer) write(_ context.Context, key string, b builds.BuilderBuild, put
|
||||
// the key holds. A key that changed meanwhile is ErrConflict: the claim is no longer this builder's.
|
||||
func (c Claimer) Finish(ctx context.Context, cl *Claim, b builds.BuilderBuild) error {
|
||||
b.V = keys.Version
|
||||
b = endOf(cl.Build, b, time.Now)
|
||||
if builds.SameBuilderBuild(cl.Build, b) {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"slices"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
|
||||
)
|
||||
@@ -209,9 +210,14 @@ func TestFinishMovesTheKeyByCompareAndSetAndWritesOnlyADifferentValue(t *testing
|
||||
if err := c.Finish(context.Background(), got, done); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if v := valueOf(t, st, got.Key); v.State != builds.BuildDone || v.Artifacts[0].Digest != digest {
|
||||
v := valueOf(t, st, got.Key)
|
||||
if v.State != builds.BuildDone || v.Artifacts[0].Digest != digest {
|
||||
t.Errorf("the key holds %+v", v)
|
||||
}
|
||||
// Design 47 H2 §797: the done key carries its end and outcome, so a reader of the builds reads no key history.
|
||||
if v.Outcome != builds.BuildDone || v.Ended.IsZero() || time.Since(v.Ended) > time.Minute {
|
||||
t.Errorf("the done key ended %v with outcome %q, want now and done", v.Ended, v.Outcome)
|
||||
}
|
||||
writes := st.writes
|
||||
if err := c.Finish(context.Background(), got, done); err != nil && !errors.Is(err, ErrConflict) {
|
||||
t.Fatal(err)
|
||||
@@ -219,6 +225,12 @@ func TestFinishMovesTheKeyByCompareAndSetAndWritesOnlyADifferentValue(t *testing
|
||||
if st.writes != writes {
|
||||
t.Errorf("the same value was written again (%d writes, was %d)", st.writes, writes)
|
||||
}
|
||||
if again := endOf(v, done, time.Now); again.Ended != v.Ended || again.Outcome != builds.BuildDone {
|
||||
t.Errorf("the same end given again ended %v, want the end the key holds, %v", again.Ended, v.Ended)
|
||||
}
|
||||
if building := endOf(v, builds.BuilderBuild{State: builds.BuildBuilding}, time.Now); !building.Ended.IsZero() || building.Outcome != "" {
|
||||
t.Errorf("a building key carries an end: %+v", building)
|
||||
}
|
||||
|
||||
// Finish after the key changed is a conflict, not an overwrite (merged into this test for the test budget).
|
||||
st = newMem()
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
package build
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// **A dropped ask ends its step could-not-run, naming the drop** (design 47 H2 §777, §778; H-A-401, H-A-404; ADR 0664 §4):
|
||||
// the seat mesh-builds' drop ends an ask its caller no longer waits for. A taken ask that runs is cancelled with a Dropped
|
||||
// cause; one not pulled yet is kept in the state taken under DroppedKey, and its pull ends it the same way. Either way the
|
||||
// build step's finish answers it could-not-run with Dropped's words, so no build runs for it after.
|
||||
|
||||
// The tools of the seat's queue (§777's tools table): AsksTool reads what waits and what is taken, AskDropTool ends one
|
||||
// ask through the seat's verb drop, the hand act recorded first.
|
||||
const (
|
||||
AsksTool = "builder_asks"
|
||||
AskDropTool = "builder_ask_drop"
|
||||
)
|
||||
|
||||
// DroppedPrefix is the key prefix in the state taken of a drop of an ask not pulled yet (H-A-404).
|
||||
const DroppedPrefix = "dropped."
|
||||
|
||||
// DroppedKey is the key in the state taken of the drop of the ask id.
|
||||
func DroppedKey(id string) string { return DroppedPrefix + id }
|
||||
|
||||
// Dropped is why a step's context was cancelled by the seat's verb drop, and the value its key keeps: who asked it and why.
|
||||
type Dropped struct {
|
||||
By string `json:"by,omitempty"`
|
||||
Why string `json:"why"`
|
||||
}
|
||||
|
||||
func (d *Dropped) Error() string {
|
||||
by := d.By
|
||||
if by == "" {
|
||||
by = "its asker"
|
||||
}
|
||||
return fmt.Sprintf("the ask was dropped by %s through the seat mesh-builds' drop: %s", by, d.Why)
|
||||
}
|
||||
|
||||
// DroppedOf is the drop that cancelled ctx, nil when none did.
|
||||
func DroppedOf(ctx context.Context) *Dropped {
|
||||
var d *Dropped
|
||||
if errors.As(context.Cause(ctx), &d) {
|
||||
return d
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -199,7 +199,9 @@ func (b *Builder) reserve(ctx context.Context, module string, m meshcatalog.Modu
|
||||
}
|
||||
src := versions.Source{Repository: m.Repository, Commit: m.Commit, Fingerprint: m.Fp2, Branch: o.Branch, Revision: o.Source, CatalogRevision: o.Revision, Rebuild: rebuild}
|
||||
for _, t := range plan.Tasks {
|
||||
src.GoPackage = src.GoPackage || (t.Kind == catalogue.ArtifactPackage && t.Language == "go")
|
||||
if t.Kind == catalogue.ArtifactPackage && t.Language == "go" && !src.GoPackage {
|
||||
src.GoPackage, src.GoArtifact = true, t.Artifact // its module path names its tag's directory (design 47 H2 §796)
|
||||
}
|
||||
}
|
||||
res, err := b.Reserved.Reserve(ctx, module, plan.Series, refs, src)
|
||||
if err != nil {
|
||||
|
||||
@@ -82,7 +82,10 @@ func (n *NotPinned) Error() string {
|
||||
// the plan is not kept, so it is made again when they are read (design 47 G A-82).
|
||||
type NoBuildModule struct {
|
||||
Artifact string
|
||||
Err error
|
||||
// Language and Kind are the artifact's, as the seat list is asked for them: what the summary of the plans no seat builds counts
|
||||
// (design 47 H2 §795).
|
||||
Language, Kind string
|
||||
Err error
|
||||
}
|
||||
|
||||
func (n *NoBuildModule) Error() string { return fmt.Sprintf("artifact %s: %v", n.Artifact, n.Err) }
|
||||
@@ -103,7 +106,7 @@ func PlanOf(m catalogue.Manifest, seats *buildrun.BuildSeats) (Plan, error) {
|
||||
task := func(a catalogue.Artifact, language, kind string) error {
|
||||
bm, err := seats.For(language, kind)
|
||||
if err != nil {
|
||||
return &NoBuildModule{Artifact: a.Name, Err: err}
|
||||
return &NoBuildModule{Artifact: a.Name, Language: language, Kind: kind, Err: err}
|
||||
}
|
||||
t := Task{Artifact: a.Name, Kind: a.Kind, Module: bm.Module, Language: bm.Toolchain, From: a.From}
|
||||
if a.Kind != catalogue.ArtifactUpstream {
|
||||
|
||||
@@ -1,13 +1,16 @@
|
||||
package build
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/seatdef"
|
||||
@@ -114,9 +117,58 @@ func TestAPlanNamesTheHoldersOfTheBuildSeats(t *testing.T) {
|
||||
if !errors.As(err, &none) {
|
||||
t.Fatalf("with no holder: %v, want NoBuildModule", err)
|
||||
}
|
||||
if none.Language != "" || none.Kind != "image" {
|
||||
t.Errorf("the plan no seat builds names %q %q, want the image's kind (design 47 H2 §795)", none.Language, none.Kind)
|
||||
}
|
||||
// Design 47 H2 §795, H-A-425: 6,500 plans no seat builds are kept in the state unbuildable and said in one summary line, with
|
||||
// each language and kind and its count, never a line per module; a module whose look judges its plan so writes its key and
|
||||
// no line of its own, and the module is removed once it builds.
|
||||
t.Run("6,500 plans no seat builds are one line", func(t *testing.T) {
|
||||
var lines []string
|
||||
kept := &memUnbuildables{vals: map[string][]byte{}}
|
||||
mf := &fakeManifests{byDigest: map[string]catalogue.Manifest{mdigest('a'): m, mdigest('b'): manifestOf(catalogue.Artifact{Name: "web", Kind: "bundle", Language: "typescript"})}}
|
||||
b := &Builder{OwnModule: builds.BuilderModule, Seats: seats, Claimer: Claimer{Store: newMem(), By: "novox.builder"}, Manifests: mf, Unbuildables: kept,
|
||||
Logf: func(f string, a ...any) { lines = append(lines, fmt.Sprintf(f, a...)) }}
|
||||
holders["node-build-go"], holders["node-build-image"] = "", ""
|
||||
b.init()
|
||||
for i := range 6500 {
|
||||
digest := mdigest('a')
|
||||
if i%5 == 0 {
|
||||
digest = mdigest('b')
|
||||
}
|
||||
b.known[fmt.Sprintf("m%04d", i)] = meshcatalog.ModulesByModuleValue{Fp2: fp1, Manifest: digest}
|
||||
}
|
||||
kept.vals["gone"] = []byte(`{"module":"gone"}`) // a module that builds now
|
||||
judged := b.SayUnbuildable(context.Background())
|
||||
if len(judged) != 6500 || len(kept.vals) != 6500 || kept.vals["gone"] != nil {
|
||||
t.Fatalf("judged %d, kept %d (gone kept: %v), want 6,500 each and gone removed", len(judged), len(kept.vals), kept.vals["gone"] != nil)
|
||||
}
|
||||
if len(lines) != 1 || lines[0] != "6500 "+UnbuildableSaid+": image 5200, typescript bundle 1300" {
|
||||
t.Errorf("said %d lines %q, want one naming each language and kind with its count", len(lines), lines)
|
||||
}
|
||||
lines = nil
|
||||
delete(kept.vals, "m0001")
|
||||
b.try(context.Background(), "m0001", b.known["m0001"])
|
||||
if kept.vals["m0001"] == nil || len(lines) != 0 {
|
||||
t.Errorf("a look at a plan no seat builds kept %s and said %q, want its key kept and no line", kept.vals["m0001"], lines)
|
||||
}
|
||||
})
|
||||
holders["node-build-go"], holders["node-build-image"] = "acme-go", "acme-img"
|
||||
p, err := PlanOf(m, seats)
|
||||
if err != nil || len(p.Tasks) != 2 || p.Tasks[0].Module != "acme-img" || p.Tasks[1].Module != "acme-go" || p.Tasks[1].Language != "go" || p.Tasks[0].Language != "" {
|
||||
t.Fatalf("plan = %+v, %v", p, err)
|
||||
}
|
||||
}
|
||||
|
||||
// memUnbuildables is the state unbuildable in memory.
|
||||
type memUnbuildables struct{ vals map[string][]byte }
|
||||
|
||||
func (m *memUnbuildables) Put(_ context.Context, k string, v []byte) error { m.vals[k] = v; return nil }
|
||||
func (m *memUnbuildables) Delete(_ context.Context, k string) error { delete(m.vals, k); return nil }
|
||||
func (m *memUnbuildables) Keys(context.Context) ([]string, error) {
|
||||
out := make([]string, 0, len(m.vals))
|
||||
for k := range m.vals {
|
||||
out = append(out, k)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
@@ -249,7 +249,8 @@ func TestAModuleWhoseOnlyArtifactIsAGoPackageIsBuiltUnderItsReservedVersion(t *t
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
r := newRig(t, goPackage)
|
||||
r.mf.byDigest[mdigest('a')] = catalogue.Manifest{Module: "m", Version: c.series, Build: &catalogue.Build{Artifacts: []catalogue.Artifact{goPackage}}}
|
||||
r.b.Reserved = &fakeReservations{filled: true}
|
||||
res := &fakeReservations{filled: true}
|
||||
r.b.Reserved = res
|
||||
r.bm.packages["go-overlay"] = "git.novox.be/novox/mesh-networking/go@v0.1.5" // the Go build module's published reference
|
||||
var asked []buildstep.StepAsk
|
||||
r.asks.Answer = func(a buildstep.StepAsk) buildstep.Step {
|
||||
@@ -264,6 +265,10 @@ func TestAModuleWhoseOnlyArtifactIsAGoPackageIsBuiltUnderItsReservedVersion(t *t
|
||||
len(asked[0].Build.Artifacts) != 1 || asked[0].Build.Artifacts[0] != (buildstep.AskArtifact{Artifact: "go-overlay", Seat: c.seat}) {
|
||||
t.Fatalf("asks %+v, want one Go ask with version %q naming go-overlay on seat %q", asked, c.version, c.seat)
|
||||
}
|
||||
// Design 47 H2 §796: the reservation names the Go package's artifact, whose module path names its tag's directory.
|
||||
if c.version != "" && (len(res.sources) != 1 || !res.sources[0].GoPackage || res.sources[0].GoArtifact != "go-overlay") {
|
||||
t.Errorf("reserved from %+v, want one source naming the Go package go-overlay", res.sources)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -61,8 +61,10 @@ type Builder struct {
|
||||
Toolchains ToolchainEntries
|
||||
// Waits keeps the toolchain waits of design 47 G §672. Nil holds none: an entry whose fingerprint names another toolchain is
|
||||
// left unbuilt, as before.
|
||||
Waits Waits
|
||||
Manifests Manifests
|
||||
Waits Waits
|
||||
// Unbuildables keeps the plans no seat builds in the builder's state unbuildable (design 47 H2 §795, H-A-425). Nil keeps none.
|
||||
Unbuildables Unbuildables
|
||||
Manifests Manifests
|
||||
// Left, when set, is told a key a look claimed and left building because its end could not be written (issue 1966,
|
||||
// novox/hq ADR 0668 §3; 1748 note 27): no look holds it then, so nothing ends it but builder_build_again or the
|
||||
// builder's next start. Ended is told each key whose end was written.
|
||||
@@ -418,7 +420,7 @@ func (b *Builder) Sweep(ctx context.Context, pictured ...string) error {
|
||||
}
|
||||
closed := builds.BuilderBuild{State: builds.BuildFailed, By: b.Claimer.By, Manifest: cur.Manifest, Step: cur.Step,
|
||||
Error: fmt.Sprintf("closed at start: left building by %s, and the module no longer has this fingerprint, so nothing will build it", cur.By)}
|
||||
data, err := builds.EncodeBuilderBuild(closed)
|
||||
data, err := builds.EncodeBuilderBuild(endOf(cur, closed, time.Now))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -639,8 +641,10 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB
|
||||
var noModule *NoBuildModule
|
||||
if errors.As(planErr, &noModule) {
|
||||
// The seat list or the catalog half does not name the build module yet: nothing is claimed, and the look is made again
|
||||
// when they do (Reconsider, at each change of either; design 47 G A-82).
|
||||
end("is not built yet: %v", planErr)
|
||||
// when they do (Reconsider, at each change of either; design 47 G A-82). The plan is kept in the state unbuildable and
|
||||
// said in the one summary line (SayUnbuildable), never in a line per module (design 47 H2 §795).
|
||||
whyEnded = "is not built yet: " + planErr.Error()
|
||||
b.judgedUnbuildable(ctx, module, m.Fp2, noModule)
|
||||
return
|
||||
}
|
||||
nothing := read && planErr == nil && len(plan.Tasks) == 0
|
||||
@@ -696,6 +700,7 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB
|
||||
return
|
||||
}
|
||||
b.handClaimed(key, cl.Rev)
|
||||
b.buildable(ctx, module) // the module builds: no longer a plan no seat builds (design 47 H2 §795, H-A-425)
|
||||
if b.Waits != nil {
|
||||
b.Waits.End(ctx, module, m.Fp2, WaitPublished) // a key queued behind a package is claimed once it is published
|
||||
}
|
||||
|
||||
@@ -17,16 +17,20 @@ import (
|
||||
// fingerprint (the existing claim, plan and lanes) and answers the step's end. A build is per fingerprint, so no pull request
|
||||
// number is needed, and the builder follows no candidate.
|
||||
|
||||
// BuildStep builds the entries of one step ask and answers its end: passed when every entry's build is done, failed when one
|
||||
// failed (said), could not run when an entry could not be read, nothing to run for an ask naming none. The ask's id and the node
|
||||
// are the caller's to set.
|
||||
func (b *Builder) BuildStep(ctx context.Context, run meshbuilds.StepRun) meshbuilds.StepEnded {
|
||||
// BuildStep builds the entries of the step ask ask and answers its end: passed when every entry's build is done, failed when
|
||||
// one failed (said), could not run when an entry could not be read, nothing to run for an ask naming none. The node is the
|
||||
// caller's to set.
|
||||
func (b *Builder) BuildStep(ctx context.Context, ask string, run meshbuilds.StepRun) meshbuilds.StepEnded {
|
||||
b.init()
|
||||
started := b.now()
|
||||
end := meshbuilds.StepEnded{V: 1, Pipeline: run.Pipeline, Step: meshbuilds.StepEndedStepBuild, Rerun: run.Rerun, By: run.By, Entries: run.Entries, Node: b.Node, Started: &started}
|
||||
// finish is the one place the ask is answered: its line names the pipeline, the movers, the step and the outcome, with
|
||||
// the words of an end that did not pass (H-A-320 (1), H-A-333).
|
||||
end := meshbuilds.StepEnded{V: 1, ID: ask, Pipeline: run.Pipeline, Step: meshbuilds.StepEndedStepBuild, Rerun: run.Rerun, By: run.By, Entries: run.Entries, Node: b.Node, Started: &started}
|
||||
// finish is the one place the ask is answered: one journal line, "answered <ask> <pipeline> <movers> <step> <outcome>"
|
||||
// (design 47 H2 §792; H-A-320 (1), H-A-333).
|
||||
finish := func(outcome, said string) meshbuilds.StepEnded {
|
||||
// An ask dropped through the seat's drop ends could-not-run naming the drop, whatever its run had reached (H-A-401).
|
||||
if d := DroppedOf(ctx); d != nil {
|
||||
outcome, said = meshbuilds.StepEndedOutcomeCouldNotRun, d.Error()
|
||||
}
|
||||
at := b.now()
|
||||
end.Ended, end.Outcome, end.Said = &at, outcome, said
|
||||
movers := make([]string, 0, len(run.Entries))
|
||||
@@ -37,13 +41,12 @@ func (b *Builder) BuildStep(ctx context.Context, run meshbuilds.StepRun) meshbui
|
||||
if mv == "" {
|
||||
mv = "-"
|
||||
}
|
||||
if said != "" {
|
||||
b.logf("%s %s %s step answered %s: %s", run.Pipeline, mv, end.Step, outcome, said)
|
||||
} else {
|
||||
b.logf("%s %s %s step answered %s", run.Pipeline, mv, end.Step, outcome)
|
||||
}
|
||||
b.logf("answered %s %s %s %s %s", ask, run.Pipeline, mv, end.Step, outcome)
|
||||
return end
|
||||
}
|
||||
if DroppedOf(ctx) != nil { // dropped before it was pulled (H-A-404)
|
||||
return finish(meshbuilds.StepEndedOutcomeCouldNotRun, "")
|
||||
}
|
||||
if len(run.Entries) == 0 {
|
||||
return finish(meshbuilds.StepEndedOutcomeNothingToRun, "the ask names no entry")
|
||||
}
|
||||
|
||||
@@ -34,13 +34,15 @@ func TestABuildStepAnswersItsEnd(t *testing.T) {
|
||||
}
|
||||
return meshcatalog.CatalogEntry{}, errors.New("not found")
|
||||
}
|
||||
asked := 0
|
||||
ask := func(entries ...meshbuilds.BuildEntry) meshbuilds.StepEnded {
|
||||
return r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.1.m", Step: meshbuilds.StepRunStepBuild, Entries: entries})
|
||||
asked++
|
||||
return r.b.BuildStep(t.Context(), fmt.Sprintf("ask%d", asked), meshbuilds.StepRun{Pipeline: "main.1.m", Step: meshbuilds.StepRunStepBuild, Entries: entries})
|
||||
}
|
||||
// Every answer is one journal line naming the pipeline, the movers, the step and the outcome (H-A-320 (1), H-A-333).
|
||||
// Every answer is one journal line, "answered <ask> <pipeline> <movers> <step> <outcome>" (§792; H-A-320 (1), H-A-333).
|
||||
var answers []string
|
||||
r.b.Logf = func(f string, a ...any) {
|
||||
if l := fmt.Sprintf(f, a...); strings.Contains(l, " step answered ") {
|
||||
if l := fmt.Sprintf(f, a...); strings.HasPrefix(l, "answered ") {
|
||||
answers = append(answers, l)
|
||||
}
|
||||
}
|
||||
@@ -50,13 +52,19 @@ func TestABuildStepAnswersItsEnd(t *testing.T) {
|
||||
if end := ask(meshbuilds.BuildEntry{Module: "m", Fingerprint: "src2:" + strings.Repeat("2", 64)}); end.Outcome != meshbuilds.StepEndedOutcomeCouldNotRun || !strings.Contains(end.Said, "cannot be read") {
|
||||
t.Errorf("an entry the seat does not hold: %+v", end)
|
||||
}
|
||||
if len(answers) != 2 || answers[0] != "main.1.m - build step answered nothing to run: the ask names no entry" ||
|
||||
!strings.HasPrefix(answers[1], "main.1.m m build step answered could not run: the entry m at ") {
|
||||
if want := []string{"answered ask1 main.1.m - build nothing to run", "answered ask2 main.1.m m build could not run"}; !reflect.DeepEqual(answers, want) {
|
||||
t.Errorf("the answers journaled %q, want one line per answer", answers)
|
||||
}
|
||||
if end := ask(meshbuilds.BuildEntry{Module: "m", Fingerprint: fp1, Commit: "other"}); end.Outcome != meshbuilds.StepEndedOutcomeCouldNotRun {
|
||||
t.Errorf("an entry at another commit: %+v", end)
|
||||
}
|
||||
// H-A-401, H-A-404: an ask dropped through the seat's drop ends could-not-run naming the drop and who asked it.
|
||||
dropped, drop := context.WithCancelCause(t.Context())
|
||||
drop(&Dropped{By: "mesh-delivery-pipeline", Why: "its step ended by its bound"})
|
||||
if end := r.b.BuildStep(dropped, "ask9", meshbuilds.StepRun{Pipeline: "main.1.m", Step: meshbuilds.StepRunStepBuild, Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}}); end.Outcome != meshbuilds.StepEndedOutcomeCouldNotRun ||
|
||||
end.Said != "the ask was dropped by mesh-delivery-pipeline through the seat mesh-builds' drop: its step ended by its bound" || end.ID != "ask9" {
|
||||
t.Errorf("a dropped ask: %+v", end)
|
||||
}
|
||||
// A build already done is passed at once, naming each entry as it was asked.
|
||||
done, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildDone, Manifest: mdigest('a')})
|
||||
if _, err := r.st.Create(t.Context(), r.key, done); err != nil {
|
||||
@@ -128,7 +136,7 @@ func TestABuildStepPassesAReservingBuildOnlyWithItsReservedVersion(t *testing.T)
|
||||
r.b.StepEntry = func(context.Context, string) (meshcatalog.CatalogEntry, error) {
|
||||
return meshcatalog.CatalogEntry{Module: "m", Repository: "novox/m", Commit: "c1", Fp: fp1, ManifestDigest: digestOf(c.unreadable), Path: "modules/m"}, nil
|
||||
}
|
||||
end := r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.1.m", Step: meshbuilds.StepRunStepBuild, Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}})
|
||||
end := r.b.BuildStep(t.Context(), "a1", meshbuilds.StepRun{Pipeline: "main.1.m", Step: meshbuilds.StepRunStepBuild, Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}})
|
||||
if end.Outcome != c.want || !strings.Contains(end.Said, c.said) || (c.want == meshbuilds.StepEndedOutcomeFailed && strings.Contains(end.Said, "builder_reserve_done") == c.takenOver) {
|
||||
t.Errorf("end = %+v, want %s saying %q", end, c.want, c.said)
|
||||
}
|
||||
@@ -199,7 +207,7 @@ func aPackageWithNoPinWaitsForItsRelease(t *testing.T) {
|
||||
r.b.PackagePublished(unpinnedX, "0.1.4-feat-x.2") // a group's tagged version: no pin
|
||||
ended := make(chan meshbuilds.StepEnded, 1)
|
||||
go func() {
|
||||
ended <- r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.9.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Source: 41,
|
||||
ended <- r.b.BuildStep(t.Context(), "a1", meshbuilds.StepRun{Pipeline: "main.9.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Source: 41,
|
||||
Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}})
|
||||
}()
|
||||
awaitWaiting(t, waiting, ended)
|
||||
@@ -225,7 +233,7 @@ func aPackageWithNoPinWaitsForItsRelease(t *testing.T) {
|
||||
}
|
||||
// A done key reports the pins it keeps it was built with, never the newer one a publish gives now (H-A-207).
|
||||
r.b.PackagePublished(unpinnedX, "v0.1.5")
|
||||
again := r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.9.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Source: 41,
|
||||
again := r.b.BuildStep(t.Context(), "a1", meshbuilds.StepRun{Pipeline: "main.9.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Source: 41,
|
||||
Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}})
|
||||
if again.Outcome != meshbuilds.StepEndedOutcomePassed || again.Packages[unpinnedX] != "v0.1.3" {
|
||||
t.Errorf("the done key's end = %+v; want passed reporting x at v0.1.3, as built", again)
|
||||
@@ -247,7 +255,7 @@ func aPackageWithNoPinWaitsForItsRelease(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r.st.vals[r.key] = raw
|
||||
if bare := r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.9.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Source: 41,
|
||||
if bare := r.b.BuildStep(t.Context(), "a1", meshbuilds.StepRun{Pipeline: "main.9.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Source: 41,
|
||||
Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}}); len(bare.Packages) != 0 {
|
||||
t.Errorf("a done key keeping no pins reported %v; want none", bare.Packages)
|
||||
}
|
||||
@@ -256,7 +264,7 @@ func aPackageWithNoPinWaitsForItsRelease(t *testing.T) {
|
||||
func aPackageWithNoReleaseFailsPastItsBound(t *testing.T) {
|
||||
r, _, _ := unpinnedRig(t)
|
||||
r.b.PackageWait = time.Nanosecond
|
||||
end := r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.9.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Source: 41,
|
||||
end := r.b.BuildStep(t.Context(), "a1", meshbuilds.StepRun{Pipeline: "main.9.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Source: 41,
|
||||
Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}})
|
||||
r.b.Wait()
|
||||
if _, built := r.st.vals[r.key]; end.Outcome != meshbuilds.StepEndedOutcomeFailed || built ||
|
||||
@@ -265,7 +273,7 @@ func aPackageWithNoReleaseFailsPastItsBound(t *testing.T) {
|
||||
}
|
||||
// A publish heard before gives the pin at once.
|
||||
r.b.PackagePublished(unpinnedX, "v0.1.2")
|
||||
if end := r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.9.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Source: 41,
|
||||
if end := r.b.BuildStep(t.Context(), "a1", meshbuilds.StepRun{Pipeline: "main.9.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Source: 41,
|
||||
Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}}); strings.Contains(end.Said, "no released version") {
|
||||
t.Errorf("a publish heard did not pin the package: %+v", end)
|
||||
}
|
||||
@@ -281,7 +289,7 @@ func theSeatsVersionsPinBeforeTheWait(t *testing.T) {
|
||||
read = append(read, path)
|
||||
return []string{"0.2.1", "0.1.9", "0.1.8"}, nil // newest first
|
||||
}
|
||||
end := r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.9.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Source: 41,
|
||||
end := r.b.BuildStep(t.Context(), "a1", meshbuilds.StepRun{Pipeline: "main.9.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Source: 41,
|
||||
Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}})
|
||||
r.b.Wait()
|
||||
asks := r.asks.Asks()
|
||||
@@ -292,7 +300,7 @@ func theSeatsVersionsPinBeforeTheWait(t *testing.T) {
|
||||
r2.b.PublishedVersions = func(context.Context, string) ([]string, error) { return []string{"0.2.1"}, nil }
|
||||
ended := make(chan meshbuilds.StepEnded, 1)
|
||||
go func() {
|
||||
ended <- r2.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.9.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Source: 41,
|
||||
ended <- r2.b.BuildStep(t.Context(), "a1", meshbuilds.StepRun{Pipeline: "main.9.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Source: 41,
|
||||
Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}})
|
||||
}()
|
||||
awaitWaiting(t, waiting, ended)
|
||||
@@ -345,7 +353,7 @@ func aRevisionHeldWakesTheAsk(t *testing.T) {
|
||||
r.b.Wait()
|
||||
ended := make(chan meshbuilds.StepEnded, 1)
|
||||
go func() {
|
||||
ended <- r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.8.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Revision: 8, Source: 41,
|
||||
ended <- r.b.BuildStep(t.Context(), "a1", meshbuilds.StepRun{Pipeline: "main.8.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Revision: 8, Source: 41,
|
||||
Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}})
|
||||
}()
|
||||
select {
|
||||
@@ -398,7 +406,7 @@ func aStepAskWaitsForItsRevision(t *testing.T) {
|
||||
}
|
||||
ended := make(chan meshbuilds.StepEnded, 1)
|
||||
go func() {
|
||||
ended <- r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.7.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Revision: 7, Source: 41,
|
||||
ended <- r.b.BuildStep(t.Context(), "a1", meshbuilds.StepRun{Pipeline: "main.7.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Revision: 7, Source: 41,
|
||||
Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}})
|
||||
}()
|
||||
select {
|
||||
@@ -442,7 +450,7 @@ func anOlderKeptRevisionIsReadByNumber(t *testing.T) {
|
||||
}
|
||||
run := meshbuilds.StepRun{Pipeline: "main.47.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Revision: 47, Source: 45,
|
||||
Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}}
|
||||
end := r.b.BuildStep(t.Context(), run)
|
||||
end := r.b.BuildStep(t.Context(), "a1", run)
|
||||
r.b.Wait()
|
||||
if end.Outcome != meshbuilds.StepEndedOutcomePassed || !slices.Equal(read, []string{"main 47"}) {
|
||||
t.Errorf("end = %+v, kept revisions read %v; want passed, revision 47 of main read once", end, read)
|
||||
@@ -457,7 +465,7 @@ func anOlderKeptRevisionIsReadByNumber(t *testing.T) {
|
||||
r2.b.KeptRevision = func(context.Context, string, int64) (int64, map[string]string, error) {
|
||||
return 45, map[string]string{"m": "src2:other"}, nil
|
||||
}
|
||||
if end := r2.b.BuildStep(t.Context(), run); end.Outcome != meshbuilds.StepEndedOutcomeFailed || !strings.Contains(end.Said, "names m at \"src2:other\"") {
|
||||
if end := r2.b.BuildStep(t.Context(), "a1", run); end.Outcome != meshbuilds.StepEndedOutcomeFailed || !strings.Contains(end.Said, "names m at \"src2:other\"") {
|
||||
t.Errorf("end = %+v; want failed naming the fingerprint the kept revision names", end)
|
||||
}
|
||||
// H-A-222: an ask of the very source the picture holds, whose entries the picture no longer names (a re-pin kept revision 48
|
||||
@@ -472,7 +480,7 @@ func anOlderKeptRevisionIsReadByNumber(t *testing.T) {
|
||||
}
|
||||
same := meshbuilds.StepRun{Pipeline: "main.48.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Revision: 48, Source: 46,
|
||||
Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}}
|
||||
end3 := r3.b.BuildStep(t.Context(), same)
|
||||
end3 := r3.b.BuildStep(t.Context(), "a1", same)
|
||||
r3.b.Wait()
|
||||
if end3.Outcome != meshbuilds.StepEndedOutcomePassed || !slices.Equal(read3, []string{"main 48"}) || len(res3.sources) != 1 || res3.sources[0].CatalogRevision != 48 {
|
||||
t.Errorf("end = %+v, kept revisions read %v, reserved %+v; want passed, revision 48 of main read once and reserved at it", end3, read3, res3.sources)
|
||||
@@ -493,7 +501,7 @@ func anOlderKeptRevisionIsReadByNumber(t *testing.T) {
|
||||
rc.b.KeptRevision = func(context.Context, string, int64) (int64, map[string]string, error) {
|
||||
return c.source, map[string]string{"m": c.fp}, nil
|
||||
}
|
||||
end := rc.b.BuildStep(t.Context(), same)
|
||||
end := rc.b.BuildStep(t.Context(), "a1", same)
|
||||
rc.b.Wait()
|
||||
if end.Outcome != meshbuilds.StepEndedOutcomeFailed || !strings.Contains(end.Said, c.said) || len(resc.sources) != 0 {
|
||||
t.Errorf("%s: end = %+v, reserved %+v; want failed at once saying %q, nothing reserved", c.name, end, resc.sources, c.said)
|
||||
@@ -506,14 +514,14 @@ func aStepAskPastItsRevisionBoundFails(t *testing.T) {
|
||||
held.Store(40)
|
||||
r, res, _ := revisionRig(t, &held, make(chan struct{}, 1))
|
||||
r.b.RevisionWait = time.Nanosecond
|
||||
end := r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.7.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Revision: 7, Source: 41,
|
||||
end := r.b.BuildStep(t.Context(), "a1", meshbuilds.StepRun{Pipeline: "main.7.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Revision: 7, Source: 41,
|
||||
Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}})
|
||||
r.b.Wait()
|
||||
if _, built := r.st.vals[r.key]; end.Outcome != meshbuilds.StepEndedOutcomeFailed || !strings.Contains(end.Said, "revision 41 of main not in the builder's picture") || built || len(res.sources) != 0 {
|
||||
t.Errorf("end = %+v, built %v, reserved %+v; want failed naming revision 41 of main, nothing built", end, built, res.sources)
|
||||
}
|
||||
// A step ask with no source (a run on request) does not wait (H-A-156).
|
||||
end = r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "run.1.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}})
|
||||
end = r.b.BuildStep(t.Context(), "a1", meshbuilds.StepRun{Pipeline: "run.1.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}})
|
||||
if strings.Contains(end.Said, "not in the builder's picture") {
|
||||
t.Errorf("an ask with no source waited: %+v", end)
|
||||
}
|
||||
|
||||
@@ -74,6 +74,27 @@ const ModePublish = "publish"
|
||||
// optional, name what to publish again; the builder answers with the versions it queued (Queued).
|
||||
const ModeRepublish = "republish"
|
||||
|
||||
// ModePublishDrop and ModePublishFirst are the Modes of a reasks value of builder_publish_drop and builder_publish_first (design
|
||||
// 47 H2 §793): Module and Version name the item; the builder terminates it, or puts it again as live and terminates the old.
|
||||
const (
|
||||
ModePublishDrop = "publish-drop"
|
||||
ModePublishFirst = "publish-first"
|
||||
)
|
||||
|
||||
// NewPublishAct is the value builder_publish_drop (mode ModePublishDrop) or builder_publish_first (ModePublishFirst) writes
|
||||
// (§793): its key, `<mode>.<unix nanoseconds>` (a call is one value), and the value. Module, version and why are required.
|
||||
func NewPublishAct(mode, module, version, why, by string, at time.Time) (string, []byte, error) {
|
||||
module, version = strings.TrimSpace(module), strings.TrimSpace(version)
|
||||
if module == "" || version == "" {
|
||||
return "", nil, errors.New("give the item's module and version")
|
||||
}
|
||||
if strings.TrimSpace(why) == "" {
|
||||
return "", nil, errors.New("say why, in plain words (why)")
|
||||
}
|
||||
data, err := json.Marshal(ReAsk{V: 1, Why: strings.TrimSpace(why), By: by, At: at.UTC(), Mode: mode, Module: module, Version: version})
|
||||
return mode + "." + strconv.FormatInt(at.UnixNano(), 10), data, err
|
||||
}
|
||||
|
||||
// ModeFloorAgain is the Mode of a reasks value of builder_floor_again (design 51 *The floor*).
|
||||
const ModeFloorAgain = "floor-again"
|
||||
|
||||
|
||||
@@ -0,0 +1,156 @@
|
||||
package build
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// **The plans no seat builds** (design 47 H2 §795, H-A-425): a module whose artifact no build seat builds, or whose build seat
|
||||
// no module claims, is kept in the builder's state unbuildable, one key per module, written when its plan is judged and removed
|
||||
// when the module builds; builder_unbuildable reads that state. The judgement is made again at every start and every change
|
||||
// of the seat list (SayUnbuildable), and said in one summary line each time, never a line per module.
|
||||
const (
|
||||
// UnbuildableState is the builder's state of the plans no seat builds, one key <module>, 8 MiB.
|
||||
UnbuildableState = "unbuildable"
|
||||
// UnbuildableTool is builder_unbuildable, which lists that state.
|
||||
UnbuildableTool = "builder_unbuildable"
|
||||
// UnbuildableSaid opens the summary line.
|
||||
UnbuildableSaid = "modules have an artifact no seat builds"
|
||||
)
|
||||
|
||||
// Unbuildable is one key of the state unbuildable: the module, the fingerprint whose plan was judged, the first artifact's
|
||||
// language and kind no seat builds, and why.
|
||||
type Unbuildable struct {
|
||||
Module string `json:"module"`
|
||||
Fingerprint string `json:"fingerprint"`
|
||||
Language string `json:"language,omitempty"`
|
||||
Kind string `json:"kind"`
|
||||
Why string `json:"why"`
|
||||
At time.Time `json:"at"`
|
||||
}
|
||||
|
||||
// Unbuildables keeps the state unbuildable: Put writes a module's key, Delete removes it (a key not there is no error) and Keys
|
||||
// lists the modules it holds.
|
||||
type Unbuildables interface {
|
||||
Put(ctx context.Context, module string, value []byte) error
|
||||
Delete(ctx context.Context, module string) error
|
||||
Keys(ctx context.Context) ([]string, error)
|
||||
}
|
||||
|
||||
// unbuildableOf is the key value of module's plan judged at fp with none.
|
||||
func unbuildableOf(module, fp string, none *NoBuildModule, at time.Time) Unbuildable {
|
||||
return Unbuildable{Module: module, Fingerprint: fp, Language: none.Language, Kind: none.Kind, Why: none.Error(), At: at.UTC()}
|
||||
}
|
||||
|
||||
// judgedUnbuildable writes module's key: its plan, judged at fp, names an artifact no seat builds.
|
||||
func (b *Builder) judgedUnbuildable(ctx context.Context, module, fp string, none *NoBuildModule) {
|
||||
if b.Unbuildables == nil {
|
||||
return
|
||||
}
|
||||
raw, _ := json.Marshal(unbuildableOf(module, fp, none, b.now()))
|
||||
if err := b.Unbuildables.Put(ctx, module, raw); err != nil {
|
||||
b.logf("%s: the plan no seat builds is not kept in the state %s: %v", module, UnbuildableState, err)
|
||||
}
|
||||
}
|
||||
|
||||
// buildable removes module's key: the module builds.
|
||||
func (b *Builder) buildable(ctx context.Context, module string) {
|
||||
if b.Unbuildables == nil {
|
||||
return
|
||||
}
|
||||
if err := b.Unbuildables.Delete(ctx, module); err != nil {
|
||||
b.logf("%s: its key of the state %s is not removed: %v", module, UnbuildableState, err)
|
||||
}
|
||||
}
|
||||
|
||||
// SayUnbuildable judges the plan of every module the builder knows again, keeps the ones no seat builds in the state unbuildable
|
||||
// (writing the modules it does not hold yet and removing the ones that build now), and says them in one line: how many, and
|
||||
// each language and kind with its count (design 47 H2 §795). It answers the modules judged so.
|
||||
func (b *Builder) SayUnbuildable(ctx context.Context) []Unbuildable {
|
||||
b.init()
|
||||
b.mu.Lock()
|
||||
known := sortedKeys(b.known)
|
||||
modules := make(map[string]string, len(known)) // the fingerprint each is known at
|
||||
digests := make(map[string]string, len(known))
|
||||
for _, k := range known {
|
||||
modules[k], digests[k] = b.known[k].Fp2, b.known[k].Manifest
|
||||
}
|
||||
b.mu.Unlock()
|
||||
var out []Unbuildable
|
||||
now := b.now()
|
||||
for _, k := range known {
|
||||
if modules[k] == "" || digests[k] == "" {
|
||||
continue
|
||||
}
|
||||
_, err, _ := b.planOf(ctx, digests[k])
|
||||
var none *NoBuildModule
|
||||
if errors.As(err, &none) {
|
||||
out = append(out, unbuildableOf(k, modules[k], none, now))
|
||||
}
|
||||
}
|
||||
if b.Unbuildables != nil {
|
||||
held := map[string]bool{}
|
||||
keys, err := b.Unbuildables.Keys(ctx)
|
||||
if err != nil {
|
||||
b.logf("the state %s cannot be listed, so the plans no seat builds are written whole: %v", UnbuildableState, err)
|
||||
}
|
||||
for _, k := range keys {
|
||||
held[k] = true
|
||||
}
|
||||
judged := map[string]bool{}
|
||||
for _, u := range out {
|
||||
judged[u.Module] = true
|
||||
if held[u.Module] {
|
||||
continue // written when its plan was judged at its look
|
||||
}
|
||||
raw, _ := json.Marshal(u)
|
||||
if err := b.Unbuildables.Put(ctx, u.Module, raw); err != nil {
|
||||
b.logf("%s: the plan no seat builds is not kept in the state %s: %v", u.Module, UnbuildableState, err)
|
||||
}
|
||||
}
|
||||
for _, k := range keys {
|
||||
if !judged[k] {
|
||||
b.buildable(ctx, k)
|
||||
}
|
||||
}
|
||||
}
|
||||
b.logf("%s", UnbuildableLine(out))
|
||||
return out
|
||||
}
|
||||
|
||||
// UnbuildableLine is the one summary line of the plans no seat builds: "N modules have an artifact no seat builds:" with each
|
||||
// language and kind and its count, most first (design 47 H2 §795).
|
||||
func UnbuildableLine(us []Unbuildable) string {
|
||||
counts := map[string]int{}
|
||||
for _, u := range us {
|
||||
what := u.Kind
|
||||
if u.Language != "" {
|
||||
what = u.Language + " " + u.Kind
|
||||
}
|
||||
counts[what]++
|
||||
}
|
||||
whats := make([]string, 0, len(counts))
|
||||
for w := range counts {
|
||||
whats = append(whats, w)
|
||||
}
|
||||
sort.Slice(whats, func(i, j int) bool {
|
||||
if counts[whats[i]] != counts[whats[j]] {
|
||||
return counts[whats[i]] > counts[whats[j]]
|
||||
}
|
||||
return whats[i] < whats[j]
|
||||
})
|
||||
parts := make([]string, len(whats))
|
||||
for i, w := range whats {
|
||||
parts[i] = fmt.Sprintf("%s %d", w, counts[w])
|
||||
}
|
||||
line := fmt.Sprintf("%d %s", len(us), UnbuildableSaid)
|
||||
if len(parts) > 0 {
|
||||
line += ": " + strings.Join(parts, ", ")
|
||||
}
|
||||
return line
|
||||
}
|
||||
@@ -27,6 +27,10 @@ type Acts struct {
|
||||
ReserveDone func(ctx context.Context, module, fp, by, why string, at time.Time) (string, error)
|
||||
// DropReserved drops a module's reservation of one MAJOR.MINOR by hand (builder_reserved_drop): its answer, or why not.
|
||||
DropReserved func(ctx context.Context, module, series, by, why string, at time.Time) (string, error)
|
||||
// PublishDrop terminates one publish item, and PublishFirst puts it again as live and terminates the old (builder_publish_drop,
|
||||
// builder_publish_first, design 47 H2 §793): the answer, or why not.
|
||||
PublishDrop func(ctx context.Context, module, version, by, why string) (string, error)
|
||||
PublishFirst func(ctx context.Context, module, version, by, why string) (string, error)
|
||||
}
|
||||
|
||||
// FloorAgain is builder_floor_again of name: refused, since the builder numbers nothing of its own and no claim asks a floor
|
||||
|
||||
@@ -14,6 +14,13 @@ import (
|
||||
type countingReserve struct {
|
||||
*reserveMem
|
||||
lists, reads atomic.Int64
|
||||
// indexReads and writes count the reads of an index key and the writes of any key (design 47 H2 §794).
|
||||
indexReads, writes atomic.Int64
|
||||
}
|
||||
|
||||
func (c *countingReserve) Update(ctx context.Context, key string, value []byte, rev uint64) (uint64, error) {
|
||||
c.writes.Add(1)
|
||||
return c.reserveMem.Update(ctx, key, value, rev)
|
||||
}
|
||||
|
||||
func (c *countingReserve) KeysOf(ctx context.Context, filter string) ([]string, error) {
|
||||
@@ -24,6 +31,8 @@ func (c *countingReserve) KeysOf(ctx context.Context, filter string) ([]string,
|
||||
func (c *countingReserve) Get(ctx context.Context, key string) ([]byte, uint64, bool, error) {
|
||||
if !strings.HasPrefix(key, "index.") {
|
||||
c.reads.Add(1)
|
||||
} else {
|
||||
c.indexReads.Add(1)
|
||||
}
|
||||
return c.reserveMem.Get(ctx, key)
|
||||
}
|
||||
@@ -52,6 +61,31 @@ func TestTheIndexFindsAPublishedReservationInOneRead(t *testing.T) {
|
||||
store.reads.Store(0)
|
||||
return r, store
|
||||
}
|
||||
// Design 47 H2 §794: 1,000 published events over 2,000 reservations cost one read of the index and one write each.
|
||||
t.Run("1,000 publisheds over 2,000 reservations read the index once and write once each", func(t *testing.T) {
|
||||
store := &countingReserve{reserveMem: &reserveMem{mem: newMem()}}
|
||||
r := &Reserver{Store: store, ReleasedOf: func(context.Context, string) ([]string, error) { return nil, nil }}
|
||||
for i := range 2000 {
|
||||
m := fmt.Sprintf("r%04d", i)
|
||||
if _, err := r.Reserve(ctx, m, "0.1", []string{m + ":a"}, Source{Fingerprint: "fp" + m}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := r.Built(ctx, m, "0.1.0", m+":a", "sha256:"+m, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
store.indexReads.Store(0)
|
||||
store.writes.Store(0)
|
||||
for i := range 1000 {
|
||||
m := fmt.Sprintf("r%04d", 2*i)
|
||||
if marked, _, err := r.Published(ctx, m+"-a", "0.1.0", "sha256:"+m); err != nil || len(marked) != 1 {
|
||||
t.Fatalf("the published of %s marked %v (%v)", m, marked, err)
|
||||
}
|
||||
}
|
||||
if ir, w, l := store.indexReads.Load(), store.writes.Load(), store.lists.Load(); ir != 1000 || w != 1000 || l != 0 {
|
||||
t.Errorf("1,000 publisheds read the index %d times, wrote %d and listed %d, want 1,000 reads, 1,000 writes and no listing", ir, w, l)
|
||||
}
|
||||
})
|
||||
t.Run("a published by digest reads its one reservation", func(t *testing.T) {
|
||||
r, store := fill(t)
|
||||
marked, _, err := r.Published(ctx, "m10-a", "v0.1.0", "sha256:m10")
|
||||
|
||||
@@ -12,27 +12,34 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
)
|
||||
|
||||
// **A reserved version of a done build is published by the builder until its release is written** (design 47 G §588, A-457a): the
|
||||
// publish is an item of the builder's re-ask state, asked when the build ends and at its start, tried with a growing pause and
|
||||
// ended loudly after its tries; each registry seat's holder, as the seat-holders view names it (A-416, A-416a), has one queue of
|
||||
// versions, kept in the state `publish-queue`, sent one at a time. A send is the registry seat's `publish` with the digest and no
|
||||
// ended loudly after its tries; each registry seat's holder, as the seat-holders view names it (A-416, A-416a), has its items on
|
||||
// the builder's work queue `publishes` (design 47 H2 §793), subject `publish.<holder>.<class>.<module>.<version>`, pulled by one
|
||||
// consumer per holder and class, the live class first. A send is the registry seat's `publish` with the digest and no
|
||||
// parts (A-73): the holder takes the bytes from the artifact store by digest, and refuses what no store holds. Nothing waits on a
|
||||
// node-engine `running` or `manifest-reads-changed` any more.
|
||||
|
||||
const (
|
||||
// PublishQueueState is the builder's old state section of the queues, capped at PublishQueueMiB.
|
||||
// PublishQueueState is the builder's old state of the queues, whose items are put on the work queue once at the first start
|
||||
// of this release, then dropped through the bus module (design 47 H2 §793).
|
||||
PublishQueueState = "publish-queue"
|
||||
PublishQueueMiB = 4
|
||||
// PublishesLocal is the builder's work queue of publish items, asked for in its requires (§793, §820; H-A-448).
|
||||
PublishesLocal = "publishes"
|
||||
// ActedState holds, one key per subject, the last stream sequence the builder acted on; capped at ActedMiB.
|
||||
ActedState = "acted"
|
||||
ActedMiB = 1
|
||||
// PublishQueueTool is builder_publish_queue.
|
||||
PublishQueueTool = "builder_publish_queue"
|
||||
// QueueFull is what version-unreleased says of a version the queue's cap refused.
|
||||
// QueueFull is what version-unreleased says of a version the work queue's cap refused.
|
||||
QueueFull = "queue full"
|
||||
// PublishDropTool and PublishFirstTool are builder_publish_drop and builder_publish_first (§793).
|
||||
PublishDropTool = "builder_publish_drop"
|
||||
PublishFirstTool = "builder_publish_first"
|
||||
// NotStoredWords opens what a refused artifact is said as.
|
||||
NotStoredWords = "not stored: "
|
||||
// VerbPublish is the registry seats' verb a publish again sends, VerbVersions the one whose record says what a seat holds
|
||||
@@ -40,11 +47,11 @@ const (
|
||||
VerbPublish = "publish"
|
||||
VerbVersions = "versions"
|
||||
// PublishClassLive is the class of a publish whose build ended in this run, which a pipeline's build step waits on;
|
||||
// PublishClassStart that of a publish asked again at a start, of a build that ended before it. A holder's queue sends every
|
||||
// live one before any of the start, each class oldest first (design 47 H H-A-290 (1)).
|
||||
PublishClassLive = "build-ended-this-run"
|
||||
PublishClassStart = "asked-again-at-start"
|
||||
// PublishSendsAtOnce bounds the sends of one holder's queue that run at once (H-A-290 (3)).
|
||||
// PublishClassStart that of a publish asked again at a start, of a build that ended before it. A holder's items are pulled
|
||||
// live first, each class oldest first (design 47 H H-A-290 (1), H2 §793).
|
||||
PublishClassLive = "live"
|
||||
PublishClassStart = "again"
|
||||
// PublishSendsAtOnce bounds the sends of one holder that run at once, its consumers' max ack pending (H-A-290 (3), §793).
|
||||
PublishSendsAtOnce = 4
|
||||
// PublishQueueListCap caps the queued versions and the publish items builder_publish_queue lists (H-A-290 (5)).
|
||||
PublishQueueListCap = 50
|
||||
@@ -77,9 +84,8 @@ func SeatOfArtifact(a meshcatalog.Artifact) (string, bool) {
|
||||
return seat, ok
|
||||
}
|
||||
|
||||
// QueuedPublish is the value of a publish-queue key `<holder module>.<module>.<version>` (each token escaped, §73): the
|
||||
// artifacts of the version this holder publishes, by reference with their digests, when it was queued and whether its
|
||||
// publish has begun.
|
||||
// QueuedPublish is one item of the work queue publishes (§793), and the value of a key of the old state publish-queue: the
|
||||
// artifacts of the version this holder publishes, by reference with their digests, and when it was queued.
|
||||
type QueuedPublish struct {
|
||||
V int `json:"v"`
|
||||
Holder string `json:"holder"`
|
||||
@@ -87,7 +93,6 @@ type QueuedPublish struct {
|
||||
Version string `json:"version"`
|
||||
Artifacts map[string]string `json:"artifacts"`
|
||||
Queued time.Time `json:"queued"`
|
||||
Begun bool `json:"begun,omitempty"`
|
||||
// Class is PublishClassLive or PublishClassStart (H-A-290 (1)); a value written before it had one is of the start.
|
||||
Class string `json:"class,omitempty"`
|
||||
}
|
||||
@@ -110,25 +115,31 @@ func sortQueued(qs []QueuedPublish) {
|
||||
})
|
||||
}
|
||||
|
||||
// QueueFilter is the filter of every queued version: a queue key has three tokens, which AllFilter (two) never lists on the bus.
|
||||
const QueueFilter = "*.*.*"
|
||||
|
||||
// QueueKey is the key of holder's queued version of module.
|
||||
// QueueKey is the key of holder's queued version of module in the old state publish-queue.
|
||||
func QueueKey(holder, module, version string) string {
|
||||
return EscapeToken(holder) + "." + EscapeToken(module) + "." + EscapeToken(version)
|
||||
}
|
||||
|
||||
// PublishMsgID is the message id of holder's item of module's version in class (§793): the work queue takes one item of each
|
||||
// id within its duplicates window, so an item put twice is put once.
|
||||
func PublishMsgID(holder, class, module, version string) string {
|
||||
return EscapeToken(holder) + "." + class + "." + EscapeToken(module) + "." + EscapeToken(version)
|
||||
}
|
||||
|
||||
// PublishEvent is the subject of holder's item of module's version in class within the work queue publishes (§793):
|
||||
// `publish.<holder>.<class>.<module>.<version>`, each token escaped.
|
||||
func PublishEvent(holder, class, module, version string) string {
|
||||
return "publish." + PublishMsgID(holder, class, module, version)
|
||||
}
|
||||
|
||||
// ErrQueueFull is a put the work queue refused as full: said as QueueFull, and asked again at the item's next try (§820).
|
||||
var ErrQueueFull = errors.New("the work queue " + PublishesLocal + " is full")
|
||||
|
||||
// Acted is the value of an acted key: the last stream sequence acted on for its subject.
|
||||
type Acted struct {
|
||||
Sequence uint64 `json:"sequence"`
|
||||
}
|
||||
|
||||
// QueueStore is the queue's bucket: compare-and-set, deletion and how many bytes it holds, for its cap.
|
||||
type QueueStore interface {
|
||||
ReserveStore
|
||||
Bytes(ctx context.Context) (uint64, error)
|
||||
}
|
||||
|
||||
// Publish is one artifact's publish again: what the registry seat's publish takes, without parts.
|
||||
type Publish struct {
|
||||
Seat, Kind, Name, Version, Digest, Fingerprint, Commit, Transfer string
|
||||
@@ -137,7 +148,9 @@ type Publish struct {
|
||||
// Republisher publishes reserved versions again (design 47 G §588).
|
||||
type Republisher struct {
|
||||
Reserved *Reserver
|
||||
Queue QueueStore
|
||||
// Put puts one item on the work queue publishes under its message id (PublishMsgID): duplicate says the queue holds that
|
||||
// id within its duplicates window and took nothing; a full queue is ErrQueueFull.
|
||||
Put func(ctx context.Context, q QueuedPublish) (duplicate bool, err error)
|
||||
// Entry reads the catalog graph entry of a fingerprint (the seat mesh-catalog's verb entry).
|
||||
Entry func(ctx context.Context, fingerprint string) (meshcatalog.CatalogEntry, error)
|
||||
// Send sends one publish to a registry seat; an error is the holder's refusal or the call's fault, in its words.
|
||||
@@ -146,8 +159,6 @@ type Republisher struct {
|
||||
Unreleased func(ctx context.Context, v ReservedVersion, why string)
|
||||
Now func() time.Time
|
||||
Logf func(string, ...any)
|
||||
// Go starts the sending of a holder's queue; nil starts a goroutine.
|
||||
Go func(func())
|
||||
// Holders answers a registry seat's holders: the catalog's SeatHolders, the part's one holder function that its FollowSeat
|
||||
// of the assignments takes too, read from the builder's start (design 47 G A-416, A-416a); nil answers none.
|
||||
Holders func(seat string) []string
|
||||
@@ -158,11 +169,7 @@ type Republisher struct {
|
||||
// nil ends none.
|
||||
Ended func(ctx context.Context, module, version string)
|
||||
|
||||
mu sync.Mutex
|
||||
busy map[string]bool
|
||||
// relist marks a holder whose pass lists again before its next start-class send: a live publish was queued meanwhile
|
||||
// (H-A-291 (2)).
|
||||
relist map[string]bool
|
||||
mu sync.Mutex
|
||||
// lastQueued is the newest time a version was queued at: a clock that has not moved between two versions (a coarse clock,
|
||||
// or a fixed one) must not make the queue's order the order of its keys, so each is queued after the last.
|
||||
lastQueued time.Time
|
||||
@@ -292,18 +299,9 @@ func (r *Reserver) Retired(ctx context.Context, all []ReservedVersion) (map[stri
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// drop deletes every queued key of module's version and ends its publish item: it never publishes (H-A-290 (2)), said with why.
|
||||
// drop ends module's version's publish item: it never publishes (H-A-290 (2)), said with why; its items on the work queue are
|
||||
// terminated when they are pulled (§793).
|
||||
func (r *Republisher) drop(ctx context.Context, module, version, why string) {
|
||||
ctx = context.WithoutCancel(ctx)
|
||||
keys, err := r.Queue.KeysOf(ctx, "*."+EscapeToken(module)+"."+EscapeToken(version))
|
||||
if err != nil {
|
||||
r.logf("publish again: %s %s never publishes (%s), and its queued keys cannot be listed: %v", module, version, why, err)
|
||||
}
|
||||
for _, k := range keys {
|
||||
if err := r.Queue.Delete(ctx, k); err != nil {
|
||||
r.logf("%s cannot be deleted from %s: %v", k, PublishQueueState, err)
|
||||
}
|
||||
}
|
||||
r.end(ctx, module, version, why)
|
||||
}
|
||||
|
||||
@@ -338,13 +336,6 @@ func (r *Republisher) queue(ctx context.Context, vs []ReservedVersion, holdersOf
|
||||
sort.SliceStable(vs, func(i, j int) bool { return vs[i].At.After(vs[j].At) })
|
||||
var queued []string
|
||||
var errs []error
|
||||
started := map[string]bool{}
|
||||
// Every holder given a version is drained, whatever a later version met: a version queued and never drained waits forever.
|
||||
defer func() {
|
||||
for holder := range started {
|
||||
r.drainLater(ctx, holder)
|
||||
}
|
||||
}()
|
||||
for _, v := range vs {
|
||||
if max > 0 && len(queued) >= max {
|
||||
break
|
||||
@@ -396,7 +387,6 @@ func (r *Republisher) queue(ctx context.Context, vs []ReservedVersion, holdersOf
|
||||
return queued, err
|
||||
}
|
||||
did = did || ok
|
||||
started[holder] = true
|
||||
}
|
||||
if did {
|
||||
queued = append(queued, v.Module+" "+v.Version)
|
||||
@@ -405,42 +395,25 @@ func (r *Republisher) queue(ctx context.Context, vs []ReservedVersion, holdersOf
|
||||
return queued, errors.Join(errs...)
|
||||
}
|
||||
|
||||
// enqueue puts one holder's artifacts of a version in its queue: not again when it is queued already, and not past the
|
||||
// section's cap (said as queue full).
|
||||
// enqueue puts one holder's artifacts of a version on the work queue under its message id: not again while the queue holds that
|
||||
// id (its duplicates window), which stands queued, and not past the queue's cap (said as queue full).
|
||||
func (r *Republisher) enqueue(ctx context.Context, v ReservedVersion, holder string, arts map[string]string) (bool, error) {
|
||||
key := QueueKey(holder, v.Module, v.Version)
|
||||
if _, _, found, err := r.Queue.Get(ctx, key); err != nil {
|
||||
return false, err
|
||||
} else if found {
|
||||
r.logf("publish again: %s %s is queued at %s already", v.Module, v.Version, holder)
|
||||
return false, nil
|
||||
}
|
||||
raw, _ := json.Marshal(QueuedPublish{V: 1, Holder: holder, Module: v.Module, Version: v.Version, Artifacts: arts, Queued: r.queuedAt(), Class: r.classOf(v)})
|
||||
used, err := r.Queue.Bytes(ctx)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("the size of the state %s cannot be read: %w", PublishQueueState, err)
|
||||
}
|
||||
if used+uint64(len(raw)) > PublishQueueMiB<<20 {
|
||||
q := QueuedPublish{V: 1, Holder: holder, Module: v.Module, Version: v.Version, Artifacts: arts, Queued: r.queuedAt(), Class: r.classOf(v)}
|
||||
dup, err := r.Put(ctx, q)
|
||||
switch {
|
||||
case errors.Is(err, ErrQueueFull):
|
||||
r.logf("publish again: %s %s is not queued at %s: %v; its publish item asks it again at its next try", v.Module, v.Version, holder, err)
|
||||
if r.Unreleased != nil {
|
||||
r.Unreleased(ctx, v, QueueFull)
|
||||
}
|
||||
return false, nil
|
||||
case err != nil:
|
||||
return false, fmt.Errorf("%s %s cannot be queued at %s: %w", v.Module, v.Version, holder, err)
|
||||
case dup:
|
||||
r.logf("publish again: %s %s is queued at %s already (class %s): the work queue holds its message id", v.Module, v.Version, holder, q.Class)
|
||||
return true, nil
|
||||
}
|
||||
if _, err := r.Queue.Create(ctx, key, raw); err != nil {
|
||||
if errors.Is(err, ErrConflict) {
|
||||
return false, nil
|
||||
}
|
||||
return false, fmt.Errorf("%s cannot be queued: %w", key, err)
|
||||
}
|
||||
r.logf("publish again: %s %s is queued to publish at %s: %d artifacts, class %s", v.Module, v.Version, holder, len(arts), r.classOf(v))
|
||||
if r.classOf(v) == PublishClassLive { // a pass sending a start list lists again, so this one goes next (H-A-291 (2))
|
||||
r.mu.Lock()
|
||||
if r.relist == nil {
|
||||
r.relist = map[string]bool{}
|
||||
}
|
||||
r.relist[holder] = true
|
||||
r.mu.Unlock()
|
||||
}
|
||||
r.logf("publish again: %s %s is queued to publish at %s: %d artifacts, class %s", v.Module, v.Version, holder, len(arts), q.Class)
|
||||
return true, nil
|
||||
}
|
||||
|
||||
@@ -537,17 +510,20 @@ func PublishItemKey(module, version string) string {
|
||||
return PublishItemKeyPrefix + EscapeToken(module) + "." + EscapeToken(version)
|
||||
}
|
||||
|
||||
// AnswerQueue is builder_publish_queue over the three states as the tools bundle reads them (H-A-290 (5)): each state read in
|
||||
// bulk (Reader.All), the answer counted and capped at PublishQueueListCap whatever the queue holds; module (and version) narrow the
|
||||
// listed queue to one module's (one version's), so an item's class can be read.
|
||||
func AnswerQueue(queue, acted, reasks Reader, module, version string) (Queued, error) {
|
||||
out := Queued{Counts: []QueueCount{}, Queue: []QueuedPublish{}, Acted: map[string]uint64{}, Items: []PublishItem{}}
|
||||
vals, c, err := queue.values(PublishQueueState)
|
||||
out.capped(c)
|
||||
if err != nil {
|
||||
return out, fmt.Errorf("the state %s cannot be read: %w", PublishQueueState, err)
|
||||
// PublishQueueOf is the seat mesh-builds' verb publish-queue (design 47 H2 §793; H-A-454): the builder's read of its own work queue
|
||||
// publishes (queue, its items read in bulk, capped past its bound), small at any size of the queue (H-A-290 (5)): the items' count
|
||||
// per holder and class with the oldest and newest queue time, and the first PublishQueueListCap in the order they are sent; module
|
||||
// (and version) narrow the list to one module's (one version's), so an item's class can be read.
|
||||
func PublishQueueOf(queue Reader, module, version string) (meshbuilds.PublishQueueAnswer, error) {
|
||||
out := meshbuilds.PublishQueueAnswer{Counts: []meshbuilds.PublishQueueAnswerCountsItem{}, Queue: []meshbuilds.PublishQueueAnswerQueueItem{}}
|
||||
vals, c, err := queue.values(PublishesLocal)
|
||||
if c != nil {
|
||||
out.Capped, out.ItemsRead, out.QueueItems, out.QueueBytes = true, int64(c.KeysRead), int64(c.StateKeys), int64(c.StateBytes)
|
||||
}
|
||||
counts := map[[2]string]*QueueCount{}
|
||||
if err != nil {
|
||||
return out, fmt.Errorf("the work queue %s cannot be read: %w", PublishesLocal, err)
|
||||
}
|
||||
counts := map[[2]string]*meshbuilds.PublishQueueAnswerCountsItem{}
|
||||
var listed []QueuedPublish
|
||||
for _, raw := range vals {
|
||||
var q QueuedPublish
|
||||
@@ -560,7 +536,7 @@ func AnswerQueue(queue, acted, reasks Reader, module, version string) (Queued, e
|
||||
out.Total++
|
||||
c := counts[[2]string{q.Holder, q.Class}]
|
||||
if c == nil {
|
||||
c = &QueueCount{Holder: q.Holder, Class: q.Class, Oldest: q.Queued, Newest: q.Queued}
|
||||
c = &meshbuilds.PublishQueueAnswerCountsItem{Holder: q.Holder, Class: q.Class, Oldest: q.Queued, Newest: q.Queued}
|
||||
counts[[2]string{q.Holder, q.Class}] = c
|
||||
}
|
||||
c.Count++
|
||||
@@ -599,10 +575,27 @@ func AnswerQueue(queue, acted, reasks Reader, module, version string) (Queued, e
|
||||
if len(out.Queue) == PublishQueueListCap {
|
||||
break
|
||||
}
|
||||
out.Queue = append(out.Queue, q)
|
||||
out.Queue = append(out.Queue, meshbuilds.PublishQueueAnswerQueueItem{Holder: q.Holder, Module: q.Module, Version: q.Version,
|
||||
Class: q.Class, Artifacts: q.Artifacts, Queued: q.Queued})
|
||||
}
|
||||
}
|
||||
out.Listed = len(out.Queue)
|
||||
out.Listed = int64(len(out.Queue))
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// AnswerQueue is builder_publish_queue: the seat mesh-builds' publish-queue answer the builder gave (q, §793, H-A-454) and the two
|
||||
// states as the tools bundle reads them (H-A-290 (5)), each read in bulk (Reader.All), capped at PublishQueueListCap.
|
||||
func AnswerQueue(q meshbuilds.PublishQueueAnswer, acted, reasks Reader) (Queued, error) {
|
||||
out := Queued{Total: int(q.Total), Counts: []QueueCount{}, Queue: []QueuedPublish{}, Listed: int(q.Listed), Acted: map[string]uint64{}, Items: []PublishItem{}}
|
||||
if q.Capped {
|
||||
out.capped(&StateCapped{State: PublishesLocal, KeysRead: int(q.ItemsRead), StateKeys: uint64(q.QueueItems), StateBytes: uint64(q.QueueBytes)})
|
||||
}
|
||||
for _, c := range q.Counts {
|
||||
out.Counts = append(out.Counts, QueueCount{Holder: c.Holder, Class: c.Class, Count: int(c.Count), Oldest: c.Oldest, Newest: c.Newest})
|
||||
}
|
||||
for _, it := range q.Queue {
|
||||
out.Queue = append(out.Queue, QueuedPublish{V: 1, Holder: it.Holder, Module: it.Module, Version: it.Version, Artifacts: it.Artifacts, Queued: it.Queued, Class: it.Class})
|
||||
}
|
||||
avals, c, err := acted.values(ActedState)
|
||||
out.capped(c)
|
||||
if err != nil {
|
||||
@@ -637,13 +630,21 @@ func AnswerQueue(queue, acted, reasks Reader, module, version string) (Queued, e
|
||||
}
|
||||
|
||||
// PublishQueueDescription is builder_publish_queue's description.
|
||||
const PublishQueueDescription = "The versions the builder will publish again (its state " + PublishQueueState + ", design 47 G section 588, H H-A-290): " +
|
||||
"how many each registry seat holder's queue holds per class (build-ended-this-run, sent first, or asked-again-at-start) with the oldest and newest " +
|
||||
"queue time; the first 50 in the order they are sent, each with its class, its artifacts' digests, when it was queued and whether its publish has " +
|
||||
"begun (module, and version, list only that module's, so one item's class can be read); the keys of the state " + ActedState + ", the last stream " +
|
||||
const PublishQueueDescription = "The versions the builder will publish again (its work queue " + PublishesLocal + ", design 47 G section 588, H2 section 793): " +
|
||||
"how many each registry seat holder's items hold per class (live, a build a pipeline's step waits on, pulled first, or again, asked again at a start) " +
|
||||
"with the oldest and newest queue time; the first 50 in the order they are pulled, each with its class, its artifacts' digests and when it was " +
|
||||
"queued (module, and version, list only that module's, so one item's class can be read); the keys of the state " + ActedState + ", the last stream " +
|
||||
"sequence the builder acted on for each subject; and the publish items (design 47 G A-457a), counted, the first 50 by next try, each version's " +
|
||||
"tries and next try, and why one ended after its tries. Each state is read in one page; a state past it is answered capped, naming the " +
|
||||
"keys read and the state's size (H-A-293). Read-only."
|
||||
"tries and next try, and why one ended (its tries spent, or dropped). A read past its pages is answered capped, naming the items read and the " +
|
||||
"queue's size (H-A-293). Read-only."
|
||||
|
||||
// PublishDropDescription and PublishFirstDescription are builder_publish_drop's and builder_publish_first's (§793).
|
||||
const PublishDropDescription = "Drop one version's publish (design 47 H2 section 793): its items on the work queue " + PublishesLocal + " are " +
|
||||
"terminated at every holder, and its publish item ended with who dropped it and why, so it is never sent or put again; builder_publish_again " +
|
||||
"asks it anew. A hand act, recorded in the hand-act log first."
|
||||
const PublishFirstDescription = "Put one version's publish first (design 47 H2 section 793): each of its again items on the work queue " +
|
||||
PublishesLocal + " is put again as live, under the message id <holder>.live.<module>.<version>, pulled before every again item, and the old " +
|
||||
"terminated. A hand act, recorded in the hand-act log first."
|
||||
|
||||
// PublishAgainDescription is builder_publish_again's description.
|
||||
const PublishAgainDescription = "Publish reserved versions again (design 47 G section 588), which builds nothing: with module and version, that reserved " +
|
||||
@@ -652,17 +653,14 @@ const PublishAgainDescription = "Publish reserved versions again (design 47 G se
|
||||
"with the versions it queued; each outcome is said as package-published, or left in version-unreleased with the holder's words. A hand act, " +
|
||||
"recorded in the hand-act log."
|
||||
|
||||
// Resume takes up, at the builder's start, every queued version again, a begun one sent anew (§588): a version whose reservation
|
||||
// was withdrawn or taken over is dropped with its publish item ended (H-A-290 (2)), and one of the live class whose build ended
|
||||
// before this run is of the start's now (H-A-290 (1)). It reads the reservations once.
|
||||
func (r *Republisher) Resume(ctx context.Context) error {
|
||||
keys, err := r.Queue.KeysOf(ctx, QueueFilter)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the state %s cannot be listed: %w", PublishQueueState, err)
|
||||
}
|
||||
// MoveQueued puts the values of the old state publish-queue on the work queue, once (§793): each under its message id, so a
|
||||
// move run twice puts each once within the duplicates window; its class as of this run (a build ended before it is of the
|
||||
// start's, H-A-290 (1)); a version whose reservation was withdrawn, taken over or superseded is not put, its publish item
|
||||
// ended (H-A-290 (2), H-A-291 (1)). It reads the reservations once, and answers how many it put.
|
||||
func (r *Republisher) MoveQueued(ctx context.Context, vals []json.RawMessage) (int, error) {
|
||||
all, err := r.Reserved.All(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the queued publishes are not taken up: %w", err)
|
||||
return 0, fmt.Errorf("the queued publishes are not moved: %w", err)
|
||||
}
|
||||
reserved := make(map[string]ReservedVersion, len(all))
|
||||
for _, v := range all {
|
||||
@@ -670,16 +668,12 @@ func (r *Republisher) Resume(ctx context.Context) error {
|
||||
}
|
||||
retired, err := r.Reserved.Retired(ctx, all)
|
||||
if err != nil {
|
||||
r.logf("publish again: the superseded builds are not retired at the start: %v", err)
|
||||
r.logf("publish again: the superseded builds are not retired at the move: %v", err)
|
||||
}
|
||||
holders := map[string]bool{}
|
||||
for _, k := range keys {
|
||||
raw, rev, found, err := r.Queue.Get(ctx, k)
|
||||
if err != nil || !found {
|
||||
continue
|
||||
}
|
||||
put := 0
|
||||
for _, raw := range vals {
|
||||
var q QueuedPublish
|
||||
if json.Unmarshal(raw, &q) != nil {
|
||||
if json.Unmarshal(raw, &q) != nil || q.Holder == "" {
|
||||
continue
|
||||
}
|
||||
v, ok := reserved[q.Module+" "+q.Version]
|
||||
@@ -688,126 +682,38 @@ func (r *Republisher) Resume(ctx context.Context) error {
|
||||
why = retired[q.Module+" "+q.Version]
|
||||
}
|
||||
if why != "" {
|
||||
if err := r.Queue.Delete(ctx, k); err != nil {
|
||||
r.logf("%s cannot be deleted from %s: %v", k, PublishQueueState, err)
|
||||
}
|
||||
r.end(ctx, q.Module, q.Version, why)
|
||||
continue
|
||||
}
|
||||
if class := r.classOf(v); q.Class != class {
|
||||
q.Class = class
|
||||
if raw, err := json.Marshal(q); err == nil {
|
||||
if _, err := r.Queue.Update(ctx, k, raw, rev); err != nil {
|
||||
r.logf("%s keeps its class %s at the start: %v", k, q.Class, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
holders[q.Holder] = true
|
||||
}
|
||||
for h := range holders {
|
||||
r.drainLater(ctx, h)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Republisher) drainLater(ctx context.Context, holder string) {
|
||||
r.mu.Lock()
|
||||
if r.busy == nil {
|
||||
r.busy = map[string]bool{}
|
||||
}
|
||||
if r.busy[holder] {
|
||||
r.mu.Unlock()
|
||||
return
|
||||
}
|
||||
r.busy[holder] = true
|
||||
r.mu.Unlock()
|
||||
run := func() {
|
||||
defer func() { r.mu.Lock(); delete(r.busy, holder); r.mu.Unlock() }()
|
||||
r.Drain(ctx, holder)
|
||||
}
|
||||
if r.Go != nil {
|
||||
r.Go(run)
|
||||
return
|
||||
}
|
||||
go run()
|
||||
}
|
||||
|
||||
// Drain sends holder's queued versions in passes (H-A-290 (3)): a pass lists the holder's keys once, reads each once, and sends
|
||||
// them in their order (the live class first, each class oldest first), PublishSendsAtOnce at a time; the keys are listed again
|
||||
// only when the pass is spent, or when a live publish was queued while it sends a start list: then the sends in flight end and
|
||||
// the keys are listed again, so the live one goes next (H-A-291 (2)); a start-class one queued meanwhile waits for the pass's end.
|
||||
// Each send is bounded by UnreleasedBound; a key is deleted when its outcome is said.
|
||||
func (r *Republisher) Drain(ctx context.Context, holder string) {
|
||||
for ctx.Err() == nil {
|
||||
r.mu.Lock()
|
||||
delete(r.relist, holder)
|
||||
r.mu.Unlock()
|
||||
keys, err := r.Queue.KeysOf(ctx, EscapeToken(holder)+".*.*")
|
||||
q.Class = r.classOf(v)
|
||||
dup, err := r.Put(ctx, q)
|
||||
if err != nil {
|
||||
r.logf("the queue of %s cannot be listed: %v", holder, err)
|
||||
return
|
||||
return put, fmt.Errorf("%s %s is not moved to the work queue %s: %w", q.Module, q.Version, PublishesLocal, err)
|
||||
}
|
||||
items := make([]QueuedPublish, 0, len(keys))
|
||||
for _, k := range keys {
|
||||
if raw, _, found, err := r.Queue.Get(ctx, k); err == nil && found {
|
||||
var q QueuedPublish
|
||||
if json.Unmarshal(raw, &q) == nil {
|
||||
items = append(items, q)
|
||||
}
|
||||
}
|
||||
if !dup {
|
||||
put++
|
||||
}
|
||||
if len(items) == 0 {
|
||||
return
|
||||
}
|
||||
sortQueued(items)
|
||||
sem := make(chan struct{}, PublishSendsAtOnce)
|
||||
var wg sync.WaitGroup
|
||||
for _, q := range items {
|
||||
if ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
sem <- struct{}{}
|
||||
r.mu.Lock()
|
||||
again := q.Class != PublishClassLive && r.relist[holder]
|
||||
r.mu.Unlock()
|
||||
if again {
|
||||
<-sem
|
||||
r.logf("publish again: %s's pass lists again after its sends in flight: a live publish was queued", holder)
|
||||
break
|
||||
}
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer func() { <-sem; wg.Done() }()
|
||||
r.send(ctx, q)
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
return put, nil
|
||||
}
|
||||
|
||||
// send publishes one queued version's artifacts and says the outcome, then deletes its key. Every call it makes honours the one
|
||||
// bound it is given, UnreleasedBound (a context's bound, no timer, ADR 0437 §8), and a send that returns without sending says why
|
||||
// (H-A-290 (4)).
|
||||
func (r *Republisher) send(ctx context.Context, q QueuedPublish) {
|
||||
key := QueueKey(q.Holder, q.Module, q.Version)
|
||||
defer func() {
|
||||
if err := r.Queue.Delete(context.WithoutCancel(ctx), key); err != nil {
|
||||
r.logf("%s cannot be deleted from %s: %v", key, PublishQueueState, err)
|
||||
}
|
||||
}()
|
||||
// SendFate is what SendQueued did with one item (§793): Sent, every artifact sent (its item acknowledged when its publish is
|
||||
// heard or confirmed); Ended, why the version never publishes (its item terminated); Failed, the holder's or the call's words
|
||||
// (its item handed back with the pause).
|
||||
type SendFate struct {
|
||||
Sent bool
|
||||
Ended, Failed string
|
||||
}
|
||||
|
||||
// SendQueued publishes one item's artifacts and says the outcome. Every call it makes honours the one bound it is given,
|
||||
// UnreleasedBound (a context's bound, no timer, ADR 0437 §8), and a send that returns without sending says why (H-A-290 (4)).
|
||||
func (r *Republisher) SendQueued(ctx context.Context, q QueuedPublish) SendFate {
|
||||
bound, cancel := context.WithTimeout(ctx, UnreleasedBound)
|
||||
defer cancel()
|
||||
if !q.Begun {
|
||||
q.Begun = true
|
||||
raw, _ := json.Marshal(q)
|
||||
if _, rev, found, err := r.Queue.Get(bound, key); err == nil && found {
|
||||
_, _ = r.Queue.Update(bound, key, raw, rev)
|
||||
}
|
||||
}
|
||||
all, err := r.Reserved.All(bound)
|
||||
if err != nil {
|
||||
r.logf("publish again: %s %s is not sent to %s: its reservation cannot be read: %v; its publish item asks it again", q.Module, q.Version, q.Holder, err)
|
||||
return
|
||||
r.logf("publish again: %s %s is not sent to %s: its reservation cannot be read: %v; it is handed back", q.Module, q.Version, q.Holder, err)
|
||||
return SendFate{Failed: "its reservation cannot be read: " + err.Error()}
|
||||
}
|
||||
var v ReservedVersion
|
||||
found := false
|
||||
@@ -830,43 +736,64 @@ func (r *Republisher) send(ctx context.Context, q QueuedPublish) {
|
||||
}
|
||||
if why != "" {
|
||||
r.end(ctx, q.Module, q.Version, why)
|
||||
return
|
||||
return SendFate{Ended: why}
|
||||
}
|
||||
e, err := r.Entry(bound, v.Fingerprint)
|
||||
if err != nil {
|
||||
r.say(ctx, v, fmt.Sprintf("the catalog entry %s cannot be read: %v", v.Fingerprint, err))
|
||||
return
|
||||
words := fmt.Sprintf("the catalog entry %s cannot be read: %v", v.Fingerprint, err)
|
||||
r.say(ctx, v, words)
|
||||
return SendFate{Failed: words}
|
||||
}
|
||||
missing := map[string]bool{}
|
||||
for _, ref := range v.Missing() {
|
||||
missing[ref] = true
|
||||
}
|
||||
refs := make([]string, 0, len(q.Artifacts))
|
||||
for ref := range q.Artifacts {
|
||||
if _, no := v.Unstored[ref]; no || !missing[ref] {
|
||||
continue // published already, or refused as not stored: said once, never sent again
|
||||
}
|
||||
refs = append(refs, ref)
|
||||
}
|
||||
sort.Strings(refs)
|
||||
if len(refs) == 0 {
|
||||
return SendFate{Ended: "none of its artifacts is left to send: each is published or refused as not stored"}
|
||||
}
|
||||
var failed []string
|
||||
sent := 0
|
||||
for _, ref := range refs {
|
||||
seat, name, kind, ok := publishTarget(e, q.Module, ref)
|
||||
if !ok {
|
||||
r.say(ctx, v, ref+": the catalog entry names no registry seat for it")
|
||||
return
|
||||
return SendFate{Failed: ref + ": the catalog entry names no registry seat for it"}
|
||||
}
|
||||
err := r.Send(bound, Publish{Seat: seat, Kind: kind, Name: name, Version: v.Version, Digest: q.Artifacts[ref], Fingerprint: v.Fingerprint,
|
||||
Commit: v.Commit, Transfer: RepublishTransfer(q.Module, v.Version, strings.TrimPrefix(ref, q.Module+":"))})
|
||||
if err == nil {
|
||||
r.logf("publish again: %s %s: %s sent to %s as %s %s, digest %s; the package-published is said when the seat says published", q.Module, q.Version, ref, seat, name, v.Version, q.Artifacts[ref])
|
||||
sent++
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
r.logf("publish again: %s %s: %s refused by %s: %v", q.Module, q.Version, ref, seat, err)
|
||||
words := err.Error()
|
||||
if strings.Contains(words, strings.TrimSuffix(NotStoredWords, ": ")) {
|
||||
if ok, merr := r.Reserved.MarkUnstored(bound, q.Module, q.Version, ref, words); merr != nil {
|
||||
r.logf("%s %s: the unstored %s is not kept: %v", q.Module, q.Version, ref, merr)
|
||||
} else if ok {
|
||||
r.say(ctx, v, ref+" "+NotStoredWords+words)
|
||||
}
|
||||
continue
|
||||
r.logf("publish again: %s %s: %s refused by %s: %v", q.Module, q.Version, ref, seat, err)
|
||||
words := err.Error()
|
||||
if strings.Contains(words, strings.TrimSuffix(NotStoredWords, ": ")) {
|
||||
if ok, merr := r.Reserved.MarkUnstored(bound, q.Module, q.Version, ref, words); merr != nil {
|
||||
r.logf("%s %s: the unstored %s is not kept: %v", q.Module, q.Version, ref, merr)
|
||||
} else if ok {
|
||||
r.say(ctx, v, ref+" "+NotStoredWords+words)
|
||||
}
|
||||
r.say(ctx, v, ref+": "+words)
|
||||
continue
|
||||
}
|
||||
r.say(ctx, v, ref+": "+words)
|
||||
failed = append(failed, ref+": "+words)
|
||||
}
|
||||
switch {
|
||||
case len(failed) > 0:
|
||||
return SendFate{Failed: strings.Join(failed, "; ")}
|
||||
case sent == 0:
|
||||
return SendFate{Ended: "each artifact left was refused as not stored"}
|
||||
}
|
||||
return SendFate{Sent: true}
|
||||
}
|
||||
|
||||
// publishTarget is the registry seat, the name and the kind module's artifact ref is published as, from its catalog entry; ok is
|
||||
|
||||
@@ -9,51 +9,41 @@ import (
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"testing/synctest"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
)
|
||||
|
||||
type queueMem struct {
|
||||
*reserveMem
|
||||
bytes uint64
|
||||
// listings counts the listings of one holder's keys (a filter `<holder>.*.*`), what a drain's pass lists.
|
||||
listings atomic.Int32
|
||||
}
|
||||
|
||||
func (q *queueMem) Bytes(context.Context) (uint64, error) { return q.bytes, nil }
|
||||
|
||||
func (q *queueMem) KeysOf(ctx context.Context, filter string) ([]string, error) {
|
||||
if strings.HasSuffix(filter, ".*.*") && filter != QueueFilter {
|
||||
q.listings.Add(1)
|
||||
}
|
||||
return q.reserveMem.KeysOf(ctx, filter)
|
||||
}
|
||||
|
||||
// republishBed is a Republisher over memory states: reservations of module m (a bundle by the oci seat and a Go package by
|
||||
// the Go seat), one holder module `oci` that claims the oci seat, and a Send and a Unreleased that record.
|
||||
// the Go seat), one holder module `oci` that claims the oci seat, a work queue that records its puts (deduplicating by message
|
||||
// id when window is set, refusing when full) and sends each put at once as its pull would, and a Send and a Unreleased that
|
||||
// record.
|
||||
type republishBed struct {
|
||||
*Republisher
|
||||
reserved *reserveMem
|
||||
queue *queueMem
|
||||
now time.Time
|
||||
mu sync.Mutex // sent and said, which a holder's sends at once write
|
||||
mu sync.Mutex // sent and said, which sends at once write
|
||||
sent []Publish
|
||||
said []string
|
||||
ended []string
|
||||
refuse string // the words Send fails with, empty for none
|
||||
// holders are the registry seats' holders the seat-holders view names (design 47 G A-416).
|
||||
holders map[string][]string
|
||||
// puts are the items the work queue took; window deduplicates by message id; full refuses every put; held keeps the puts
|
||||
// unsent.
|
||||
puts []QueuedPublish
|
||||
ids map[string]bool
|
||||
window, full bool
|
||||
held bool
|
||||
}
|
||||
|
||||
func newRepublishBed(t *testing.T) *republishBed {
|
||||
t.Helper()
|
||||
b := &republishBed{reserved: &reserveMem{mem: newMem()}, queue: &queueMem{reserveMem: &reserveMem{mem: newMem()}}, now: time.Date(2026, 11, 11, 12, 0, 0, 0, time.UTC)}
|
||||
b := &republishBed{reserved: &reserveMem{mem: newMem()}, now: time.Date(2026, 11, 11, 12, 0, 0, 0, time.UTC), ids: map[string]bool{}}
|
||||
res := &Reserver{Store: b.reserved, Now: func() time.Time { return b.now }, ReleasedOf: func(context.Context, string) ([]string, error) { return nil, nil }}
|
||||
b.Republisher = &Republisher{Reserved: res, Queue: b.queue, Now: func() time.Time { return b.now }, Go: func(f func()) { f() },
|
||||
b.Republisher = &Republisher{Reserved: res, Now: func() time.Time { return b.now },
|
||||
Entry: func(_ context.Context, fp string) (meshcatalog.CatalogEntry, error) {
|
||||
if fp == "src2:holder" {
|
||||
return meshcatalog.CatalogEntry{Serves: []string{SeatArtifacts + ".publish", SeatArtifacts + ".versions"}}, nil
|
||||
@@ -83,6 +73,21 @@ func newRepublishBed(t *testing.T) *republishBed {
|
||||
b.ended = append(b.ended, module+" "+version)
|
||||
},
|
||||
Logf: t.Logf}
|
||||
b.Put = func(ctx context.Context, q QueuedPublish) (bool, error) {
|
||||
if b.full {
|
||||
return false, fmt.Errorf("%w: maximum bytes exceeded", ErrQueueFull)
|
||||
}
|
||||
id := PublishMsgID(q.Holder, q.Class, q.Module, q.Version)
|
||||
if b.window && b.ids[id] {
|
||||
return true, nil
|
||||
}
|
||||
b.ids[id] = true
|
||||
b.puts = append(b.puts, q)
|
||||
if !b.held {
|
||||
b.SendQueued(ctx, q)
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
b.Holders = func(seat string) []string { return b.holders[seat] }
|
||||
return b
|
||||
}
|
||||
@@ -125,8 +130,8 @@ func TestRepublish(t *testing.T) {
|
||||
p.Transfer != "republish-m-"+strings.ReplaceAll(v.Version, ".", "-")+"-prog" || strings.ContainsAny(p.Transfer, "/. *>") {
|
||||
t.Errorf("sent %+v, want the bundle by digest to the oci seat, the Go package left to its own seat's holder", p)
|
||||
}
|
||||
if keys, _ := b.queue.KeysOf(ctx, AllFilter); len(keys) != 0 {
|
||||
t.Errorf("a key stays in the queue after its outcome: %v", keys)
|
||||
if len(b.puts) != 1 || b.puts[0].Holder != "oci" || b.puts[0].Class != PublishClassLive {
|
||||
t.Errorf("the work queue took %+v, want one live item at oci", b.puts)
|
||||
}
|
||||
})
|
||||
t.Run("an entry that cannot be read stops neither the other versions nor the drain", func(t *testing.T) {
|
||||
@@ -173,22 +178,42 @@ func TestRepublish(t *testing.T) {
|
||||
t.Run("a queue past its cap says queue full", func(t *testing.T) {
|
||||
b := newRepublishBed(t)
|
||||
b.reserve(t, "m")
|
||||
b.queue.bytes = PublishQueueMiB << 20
|
||||
b.full = true
|
||||
b.holders = map[string][]string{SeatArtifacts: {"oci"}}
|
||||
_, _ = b.Republish(ctx, "", "")
|
||||
if len(b.sent) != 0 || len(b.said) != 1 || !strings.HasSuffix(b.said[0], QueueFull) {
|
||||
t.Errorf("sent %d, said %v, want nothing sent and queue full said", len(b.sent), b.said)
|
||||
}
|
||||
})
|
||||
t.Run("a restart sends the queued again, a begun one anew", func(t *testing.T) {
|
||||
t.Run("an item put twice within the duplicates window is put once, and stands queued", func(t *testing.T) {
|
||||
b := newRepublishBed(t)
|
||||
v := b.reserve(t, "m")
|
||||
raw, _ := json.Marshal(QueuedPublish{V: 1, Holder: "oci", Module: "m", Version: v.Version, Artifacts: map[string]string{"m:prog": "sha256:aa"}, Queued: b.now, Begun: true})
|
||||
if _, err := b.queue.Create(ctx, QueueKey("oci", "m", v.Version), raw); err != nil {
|
||||
b.window, b.held = true, true
|
||||
b.holders = map[string][]string{SeatArtifacts: {"oci"}}
|
||||
q1, _ := b.Republish(ctx, "m", v.Version)
|
||||
q2, _ := b.Republish(ctx, "m", v.Version)
|
||||
if len(b.puts) != 1 || len(q1) != 1 || len(q2) != 1 {
|
||||
t.Errorf("puts %d, queued %v then %v: want one item, queued both times", len(b.puts), q1, q2)
|
||||
}
|
||||
})
|
||||
t.Run("the move puts each item of the old state once, its class as of this run, a superseded one not", func(t *testing.T) {
|
||||
b := newRepublishBed(t)
|
||||
v := b.reserve(t, "m")
|
||||
w := b.reserve(t, "n")
|
||||
b.Started = b.now
|
||||
b.window, b.held = true, true
|
||||
raw, _ := json.Marshal(QueuedPublish{V: 1, Holder: "oci", Module: "m", Version: v.Version, Artifacts: map[string]string{"m:prog": "sha256:aa"}, Queued: b.now, Class: "build-ended-this-run"})
|
||||
gone, _ := json.Marshal(QueuedPublish{V: 1, Holder: "oci", Module: "n", Version: w.Version, Artifacts: map[string]string{"n:prog": "sha256:aa"}, Queued: b.now})
|
||||
if _, _, err := b.Reserved.Withdraw(ctx, "n", w.Version); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := b.Resume(ctx); err != nil || len(b.sent) != 1 {
|
||||
t.Errorf("resume sent %d (%v), want the begun version sent anew", len(b.sent), err)
|
||||
for range 2 {
|
||||
if _, err := b.MoveQueued(ctx, []json.RawMessage{raw, gone}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if len(b.puts) != 1 || b.puts[0].Class != PublishClassStart || fmt.Sprint(b.ended) != "[n "+w.Version+" n "+w.Version+"]" {
|
||||
t.Errorf("puts %+v, ended %v: want m once as again (its build ended before this run), n's item ended", b.puts, b.ended)
|
||||
}
|
||||
})
|
||||
t.Run("after a fresh start the tool queues at the seat's holder, no running heard", func(t *testing.T) {
|
||||
@@ -215,7 +240,7 @@ func TestRepublish(t *testing.T) {
|
||||
return json.RawMessage(`{"mode":"publish","module":"m","version":"0.1.0","tries":3,"next-try":"2026-11-17T01:00:00Z","why":"w"}`), true, nil
|
||||
}}
|
||||
none := Reader{Keys: func() ([]string, error) { return nil, nil }}
|
||||
answer, err := AnswerQueue(none, none, items, "", "")
|
||||
answer, err := answerOf(none, none, items, "", "")
|
||||
if err != nil || len(answer.Items) != 1 || answer.Items[0].Tries != 3 || answer.Items[0].Version != "0.1.0" {
|
||||
t.Errorf("builder_publish_queue items = %+v (%v), want the one publish item with its tries", answer.Items, err)
|
||||
}
|
||||
@@ -262,98 +287,31 @@ func TestTheRegistriesRecordSaysWhetherAVersionIsHeld(t *testing.T) {
|
||||
}
|
||||
|
||||
// queueAt writes a queued version of module m at holder oci directly, as a start finds it.
|
||||
func (b *republishBed) queueAt(t *testing.T, v ReservedVersion, class string, at time.Time) {
|
||||
t.Helper()
|
||||
raw, _ := json.Marshal(QueuedPublish{V: 1, Holder: "oci", Module: v.Module, Version: v.Version, Artifacts: map[string]string{v.Module + ":prog": "sha256:aa"}, Queued: at, Class: class})
|
||||
if _, err := b.queue.Create(context.Background(), QueueKey("oci", v.Module, v.Version), raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
func (b *republishBed) queueAt(v ReservedVersion, class string, at time.Time) QueuedPublish {
|
||||
return QueuedPublish{V: 1, Holder: "oci", Module: v.Module, Version: v.Version, Artifacts: map[string]string{v.Module + ":prog": "sha256:aa"}, Queued: at, Class: class}
|
||||
}
|
||||
|
||||
// TestAHoldersQueueSendsLiveFirstInPasses: design 47 H H-A-290 (1)-(3) — a publish whose build ended in this run is sent before
|
||||
// any of a build that ended before the start, each class oldest first; a pass lists the holder's keys once and sends them, at most
|
||||
// four at a time; a build whose reservation was withdrawn or taken over never publishes, its key dropped and its publish item ended,
|
||||
// at enqueue, at the pass and at the start.
|
||||
func TestAHoldersQueueSendsLiveFirstInPasses(t *testing.T) {
|
||||
// TestAQueuedItemIsSentWhileItsBuildStands: design 47 H H-A-290 (2), H-A-291 (1), H2 §793 — an item is put at the holder in the
|
||||
// class of its build (live when it ended in this run); a build whose reservation was withdrawn, taken over or superseded by a newer
|
||||
// patch or release of its series never publishes, its publish item ended, at the put and at the send; an item whose artifacts are
|
||||
// all published or refused as not stored ends unsent.
|
||||
func TestAQueuedItemIsSentWhileItsBuildStands(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
t.Run("the live class first, at most four sends at once", func(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
b := newRepublishBed(t)
|
||||
var old []ReservedVersion
|
||||
for _, m := range []string{"o1", "o2", "o3", "o4", "o5"} {
|
||||
old = append(old, b.reserve(t, m))
|
||||
}
|
||||
b.Started = b.now
|
||||
b.reserve(t, "live")
|
||||
for i, v := range old {
|
||||
b.queueAt(t, v, PublishClassStart, b.now.Add(time.Duration(i)*time.Second))
|
||||
}
|
||||
b.now = b.now.Add(time.Hour) // the live one is queued last, after every old one
|
||||
b.holders = map[string][]string{SeatArtifacts: {"oci"}}
|
||||
release := make(chan struct{})
|
||||
var arrived []string
|
||||
var inflight, most int
|
||||
send := b.Send
|
||||
b.Send = func(ctx context.Context, p Publish) error {
|
||||
b.mu.Lock()
|
||||
arrived = append(arrived, p.Name)
|
||||
inflight++
|
||||
most = max(most, inflight)
|
||||
b.mu.Unlock()
|
||||
<-release
|
||||
b.mu.Lock()
|
||||
inflight--
|
||||
b.mu.Unlock()
|
||||
return send(ctx, p)
|
||||
}
|
||||
b.Go = func(f func()) { go f() }
|
||||
if q, err := b.Republish(ctx, "live", ""); err != nil || len(q) != 1 {
|
||||
t.Fatalf("queued %v (%v), want the live version", q, err)
|
||||
}
|
||||
synctest.Wait() // every send the bound lets run waits on release
|
||||
b.mu.Lock()
|
||||
first := slices.Clone(arrived)
|
||||
b.mu.Unlock()
|
||||
if len(first) != PublishSendsAtOnce || !slices.Contains(first, "live/prog") {
|
||||
t.Errorf("the sends running at once are %v: want %d, the live one among them", first, PublishSendsAtOnce)
|
||||
}
|
||||
close(release)
|
||||
synctest.Wait()
|
||||
if len(b.sent) != 6 || most != PublishSendsAtOnce {
|
||||
t.Errorf("sent %d, at most %d at once: want all six, %d at once", len(b.sent), most, PublishSendsAtOnce)
|
||||
}
|
||||
})
|
||||
})
|
||||
t.Run("a pass lists the keys once for all its sends", func(t *testing.T) {
|
||||
t.Run("a build that ended before this run is put as again, one after it as live", func(t *testing.T) {
|
||||
b := newRepublishBed(t)
|
||||
var vs []ReservedVersion
|
||||
for i := range 6 {
|
||||
vs = append(vs, b.reserve(t, fmt.Sprintf("m%d", i)))
|
||||
}
|
||||
for i, v := range vs {
|
||||
b.queueAt(t, v, PublishClassStart, b.now.Add(time.Duration(i)*time.Second))
|
||||
}
|
||||
b.Drain(ctx, "oci")
|
||||
if n := b.queue.listings.Load(); len(b.sent) != 6 || n != 2 {
|
||||
t.Errorf("sent %d with %d listings, want six sends from one pass and one listing that finds it spent", len(b.sent), n)
|
||||
}
|
||||
})
|
||||
t.Run("a start makes a live key of a build before it a start key", func(t *testing.T) {
|
||||
b := newRepublishBed(t)
|
||||
v := b.reserve(t, "m")
|
||||
old := b.reserve(t, "o")
|
||||
b.Started = b.now
|
||||
b.queueAt(t, v, PublishClassLive, b.now)
|
||||
b.Go = func(func()) {} // the start's drain is not run here
|
||||
if err := b.Resume(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
raw, _, _, _ := b.queue.Get(ctx, QueueKey("oci", "m", v.Version))
|
||||
var q QueuedPublish
|
||||
if json.Unmarshal(raw, &q) != nil || q.Class != PublishClassStart {
|
||||
t.Errorf("the key after the start is %s, want the class %s: its build did not end in this run", raw, PublishClassStart)
|
||||
live := b.reserve(t, "l")
|
||||
b.held = true
|
||||
b.holders = map[string][]string{SeatArtifacts: {"oci"}}
|
||||
_, _ = b.Republish(ctx, "o", old.Version)
|
||||
_, _ = b.Republish(ctx, "l", live.Version)
|
||||
if len(b.puts) != 2 || b.puts[0].Class != PublishClassStart || b.puts[1].Class != PublishClassLive ||
|
||||
PublishEvent("oci", PublishClassLive, "l", live.Version) != "publish.oci.live.l.0=2E1=2E0" {
|
||||
t.Errorf("puts %+v, want o again and l live, on publish.<holder>.<class>.<module>.<version>", b.puts)
|
||||
}
|
||||
})
|
||||
for _, at := range []string{"enqueue", "pass", "start"} {
|
||||
for _, at := range []string{"put", "send"} {
|
||||
t.Run("a withdrawn build never publishes, at the "+at, func(t *testing.T) {
|
||||
b := newRepublishBed(t)
|
||||
v := b.reserve(t, "m")
|
||||
@@ -361,105 +319,34 @@ func TestAHoldersQueueSendsLiveFirstInPasses(t *testing.T) {
|
||||
if _, ok, err := b.Reserved.Withdraw(ctx, "m", v.Version); err != nil || !ok {
|
||||
t.Fatalf("withdrawn %v (%v)", ok, err)
|
||||
}
|
||||
switch at {
|
||||
case "enqueue":
|
||||
b.queueAt(t, v, PublishClassLive, b.now)
|
||||
b.Go = func(func()) {} // a key queued before the withdrawal is dropped by the enqueue itself
|
||||
if at == "put" {
|
||||
if q, _ := b.Republish(ctx, "m", v.Version); len(q) != 0 {
|
||||
t.Errorf("queued %v", q)
|
||||
}
|
||||
case "pass":
|
||||
b.queueAt(t, v, PublishClassLive, b.now)
|
||||
b.Drain(ctx, "oci")
|
||||
case "start":
|
||||
b.queueAt(t, v, PublishClassLive, b.now)
|
||||
b.Go = func(func()) {}
|
||||
if err := b.Resume(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
} else if f := b.SendQueued(ctx, b.queueAt(v, PublishClassLive, b.now)); f.Ended == "" {
|
||||
t.Errorf("the send of a withdrawn build answered %+v, want it ended", f)
|
||||
}
|
||||
keys, _ := b.queue.KeysOf(ctx, QueueFilter)
|
||||
if len(b.sent) != 0 || len(keys) != 0 || fmt.Sprint(b.ended) != "[m "+v.Version+"]" {
|
||||
t.Errorf("sent %v, keys %v, items ended %v: want nothing sent, the key dropped and m %s's item ended", b.sent, keys, b.ended, v.Version)
|
||||
if len(b.sent) != 0 || len(b.puts) != 0 || fmt.Sprint(b.ended) != "[m "+v.Version+"]" {
|
||||
t.Errorf("sent %v, puts %v, items ended %v: want nothing sent or put and m %s's item ended", b.sent, b.puts, b.ended, v.Version)
|
||||
}
|
||||
})
|
||||
}
|
||||
t.Run("a live publish queued mid-pass goes after the sends in flight, before the rest of the start list", func(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
b := newRepublishBed(t)
|
||||
var old []ReservedVersion
|
||||
for _, m := range []string{"o1", "o2", "o3", "o4", "o5", "o6"} {
|
||||
old = append(old, b.reserve(t, m))
|
||||
}
|
||||
b.Started = b.now
|
||||
b.reserve(t, "live")
|
||||
for i, v := range old {
|
||||
b.queueAt(t, v, PublishClassStart, b.now.Add(time.Duration(i)*time.Second))
|
||||
}
|
||||
b.now = b.now.Add(time.Hour)
|
||||
b.holders = map[string][]string{SeatArtifacts: {"oci"}}
|
||||
first, then := make(chan struct{}), make(chan struct{}) // the four in flight wait on first, every later send on then
|
||||
var arrived []string
|
||||
send := b.Send
|
||||
b.Send = func(ctx context.Context, p Publish) error {
|
||||
b.mu.Lock()
|
||||
arrived = append(arrived, p.Name)
|
||||
gate := map[bool]chan struct{}{true: first, false: then}[len(arrived) <= PublishSendsAtOnce]
|
||||
b.mu.Unlock()
|
||||
<-gate
|
||||
return send(ctx, p)
|
||||
}
|
||||
b.Go = func(f func()) { go f() }
|
||||
b.drainLater(ctx, "oci")
|
||||
synctest.Wait() // four start sends in flight, the pass holding o5 and o6
|
||||
if q, err := b.Republish(ctx, "live", ""); err != nil || len(q) != 1 {
|
||||
t.Fatalf("queued %v (%v)", q, err)
|
||||
}
|
||||
close(first)
|
||||
synctest.Wait() // the sends after the four: with the live one among them, or o5 and o6 alone when the pass goes on
|
||||
b.mu.Lock()
|
||||
next := slices.Clone(arrived[PublishSendsAtOnce:])
|
||||
b.mu.Unlock()
|
||||
if !slices.Contains(next, "live/prog") {
|
||||
t.Errorf("after the four in flight the sends are %v, want the live one among them, the pass listed again", next)
|
||||
}
|
||||
close(then)
|
||||
synctest.Wait()
|
||||
if len(arrived) != 7 {
|
||||
t.Errorf("sent %v, want all seven", arrived)
|
||||
}
|
||||
})
|
||||
})
|
||||
t.Run("a build a newer patch or release of its series supersedes is retired at the pass and at the start, never by a newer series", func(t *testing.T) {
|
||||
t.Run("a build a newer patch or release of its series supersedes is ended at the send, never by a newer series", func(t *testing.T) {
|
||||
b := newRepublishBed(t)
|
||||
v := b.reserve(t, "m")
|
||||
b.holders = map[string][]string{SeatArtifacts: {"oci"}}
|
||||
if _, err := b.Reserved.Reserve(ctx, "m", "0.2", []string{"m:prog"}, Source{Fingerprint: "src2:m"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.queueAt(t, v, PublishClassStart, b.now)
|
||||
b.Drain(ctx, "oci")
|
||||
if len(b.sent) != 1 || len(b.ended) != 0 {
|
||||
t.Errorf("with a standing 0.2: sent %v, ended %v, want m %s sent, its series not superseded (H-A-293)", b.sent, b.ended, v.Version)
|
||||
if f := b.SendQueued(ctx, b.queueAt(v, PublishClassStart, b.now)); !f.Sent || len(b.sent) != 1 || len(b.ended) != 0 {
|
||||
t.Errorf("with a standing 0.2: %+v, sent %v, ended %v, want m %s sent, its series not superseded (H-A-293)", f, b.sent, b.ended, v.Version)
|
||||
}
|
||||
rel := &reserveMem{mem: newMem()}
|
||||
raw, _ := json.Marshal(ReleasedRecord{Kept: []ReleasedBuild{{Fingerprint: "f", Version: "0.2.3"}, {Fingerprint: "g", Version: "0.1.9"}}})
|
||||
_, _ = rel.Create(ctx, ReleasedKey("m"), raw)
|
||||
b.Reserved.Releases = rel
|
||||
b.sent = nil
|
||||
b.queueAt(t, v, PublishClassStart, b.now)
|
||||
b.Drain(ctx, "oci")
|
||||
if len(b.sent) != 0 || fmt.Sprint(b.ended) != "[m "+v.Version+"]" {
|
||||
t.Errorf("at the pass: sent %v, ended %v, want m %s retired by the release 0.1.9", b.sent, b.ended, v.Version)
|
||||
}
|
||||
b.ended = nil
|
||||
b.queueAt(t, v, PublishClassStart, b.now)
|
||||
b.Go = func(func()) {}
|
||||
if err := b.Resume(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if keys, _ := b.queue.KeysOf(ctx, QueueFilter); len(keys) != 0 || fmt.Sprint(b.ended) != "[m "+v.Version+"]" {
|
||||
t.Errorf("at the start: keys %v, ended %v, want m %s retired", keys, b.ended, v.Version)
|
||||
if f := b.SendQueued(ctx, b.queueAt(v, PublishClassStart, b.now)); f.Ended == "" || len(b.sent) != 0 || fmt.Sprint(b.ended) != "[m "+v.Version+"]" {
|
||||
t.Errorf("at the send: %+v, sent %v, ended %v, want m %s retired by the release 0.1.9", f, b.sent, b.ended, v.Version)
|
||||
}
|
||||
only := &reserveMem{mem: newMem()}
|
||||
raw, _ = json.Marshal(ReleasedRecord{Kept: []ReleasedBuild{{Fingerprint: "f", Version: "0.2.0"}}})
|
||||
@@ -473,15 +360,47 @@ func TestAHoldersQueueSendsLiveFirstInPasses(t *testing.T) {
|
||||
t.Run("a version taken over is not sent, and its item ended", func(t *testing.T) {
|
||||
b := newRepublishBed(t)
|
||||
v := b.reserve(t, "m")
|
||||
b.queueAt(t, v, PublishClassStart, b.now)
|
||||
if _, err := b.Reserved.Reserve(ctx, "m", "0.1", []string{"m:prog"}, Source{Fingerprint: "src2:other"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.Drain(ctx, "oci")
|
||||
if len(b.sent) != 0 || fmt.Sprint(b.ended) != "[m "+v.Version+"]" {
|
||||
t.Errorf("sent %v, ended %v: want nothing sent and the item ended", b.sent, b.ended)
|
||||
if f := b.SendQueued(ctx, b.queueAt(v, PublishClassStart, b.now)); f.Ended == "" || len(b.sent) != 0 || fmt.Sprint(b.ended) != "[m "+v.Version+"]" {
|
||||
t.Errorf("%+v, sent %v, ended %v: want nothing sent and the item ended", f, b.sent, b.ended)
|
||||
}
|
||||
})
|
||||
t.Run("an item whose artifacts are published or refused as not stored ends unsent", func(t *testing.T) {
|
||||
b := newRepublishBed(t)
|
||||
v := b.reserve(t, "m")
|
||||
b.refuse = "not stored: no blob"
|
||||
if f := b.SendQueued(ctx, b.queueAt(v, PublishClassLive, b.now)); f.Ended == "" || len(b.sent) != 1 {
|
||||
t.Errorf("a not stored refusal answered %+v, sent %d: want it ended after one send", f, len(b.sent))
|
||||
}
|
||||
if f := b.SendQueued(ctx, b.queueAt(v, PublishClassLive, b.now)); f.Ended == "" || len(b.sent) != 1 {
|
||||
t.Errorf("again: %+v, sent %d: want it ended unsent", f, len(b.sent))
|
||||
}
|
||||
b.refuse = "the holder is full"
|
||||
w := b.reserve(t, "n")
|
||||
if f := b.SendQueued(ctx, b.queueAt(w, PublishClassLive, b.now)); f.Failed == "" || f.Sent {
|
||||
t.Errorf("a refusal in other words answered %+v, want it failed, handed back", f)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// answerOf is builder_publish_queue whole: the builder's verb publish-queue over its work queue (queue), then the tool over the
|
||||
// verb's answer and the two states (design 47 H2 §793, H-A-454).
|
||||
func answerOf(queue, acted, reasks Reader, module, version string) (Queued, error) {
|
||||
q, err := PublishQueueOf(queue, module, version)
|
||||
if err != nil {
|
||||
return Queued{}, err
|
||||
}
|
||||
raw, err := json.Marshal(q) // the verb's answer travels the bus
|
||||
if err != nil {
|
||||
return Queued{}, err
|
||||
}
|
||||
var heard meshbuilds.PublishQueueAnswer
|
||||
if err := json.Unmarshal(raw, &heard); err != nil {
|
||||
return Queued{}, err
|
||||
}
|
||||
return AnswerQueue(heard, acted, reasks)
|
||||
}
|
||||
|
||||
// TestThePublishQueueToolAnswersAtAnySize: design 47 H H-A-290 (5) — builder_publish_queue over two thousand queued versions answers
|
||||
@@ -507,7 +426,7 @@ func TestThePublishQueueToolAnswersAtAnySize(t *testing.T) {
|
||||
Get: func(k string) (json.RawMessage, bool, error) { gets++; v, ok := vals[k]; return v, ok, nil }}
|
||||
none := Reader{Keys: func() ([]string, error) { return nil, nil }}
|
||||
start := time.Now()
|
||||
a, err := AnswerQueue(queue, none, none, "", "")
|
||||
a, err := answerOf(queue, none, none, "", "")
|
||||
took := time.Since(start)
|
||||
raw, _ := json.Marshal(a)
|
||||
if err != nil || took > time.Second || a.Total != 2000 || len(a.Queue) != PublishQueueListCap || a.Listed != PublishQueueListCap || len(raw) > 64<<10 || gets != 0 {
|
||||
@@ -519,7 +438,7 @@ func TestThePublishQueueToolAnswersAtAnySize(t *testing.T) {
|
||||
if a.Queue[0].Class != PublishClassLive || a.Queue[0].Holder != "go" {
|
||||
t.Errorf("the list opens with %+v, want the go holder's live class, sent first", a.Queue[0])
|
||||
}
|
||||
one, err := AnswerQueue(queue, none, none, "m1995", "")
|
||||
one, err := answerOf(queue, none, none, "m1995", "")
|
||||
if err != nil || len(one.Queue) != 1 || one.Queue[0].Class != PublishClassLive || one.Total != 2000 {
|
||||
t.Errorf("one module's answer %+v (%v), want its item with its class", one.Queue, err)
|
||||
}
|
||||
@@ -527,12 +446,12 @@ func TestThePublishQueueToolAnswersAtAnySize(t *testing.T) {
|
||||
t.Errorf("a state one page holds is answered capped: %+v", a.Read)
|
||||
}
|
||||
queue.All = func() (StatePage, error) { return StatePage{Values: vals, More: true, Keys: 5000, Bytes: 3 << 20}, nil }
|
||||
if c, err := AnswerQueue(queue, none, none, "", ""); err != nil || !c.Capped || len(c.Read) != 1 || c.Read[0].State != PublishQueueState ||
|
||||
if c, err := answerOf(queue, none, none, "", ""); err != nil || !c.Capped || len(c.Read) != 1 || c.Read[0].State != PublishesLocal ||
|
||||
c.Read[0].KeysRead != 2000 || c.Read[0].StateKeys != 5000 || c.Read[0].StateBytes != 3<<20 || c.Total != 2000 {
|
||||
t.Errorf("a state past its one page answered capped %v, read %+v (%v): want capped, 2000 keys read of 5000, its size said (H-A-293)", c.Capped, c.Read, err)
|
||||
}
|
||||
queue.All = func() (StatePage, error) { return StatePage{}, errors.ErrUnsupported }
|
||||
if b, err := AnswerQueue(queue, none, none, "", ""); err != nil || b.Total != 2000 || gets != 2000 {
|
||||
if b, err := answerOf(queue, none, none, "", ""); err != nil || b.Total != 2000 || gets != 2000 {
|
||||
t.Errorf("a reader with no bulk read answered %d (%v) with %d single reads, want every key read one by one", b.Total, err, gets)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -10,8 +10,10 @@ import (
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
)
|
||||
@@ -40,6 +42,9 @@ const (
|
||||
// VersionHeldElsewhere is the finding of a number a registry seat already holds, skipped at reserving or moved from (design 47
|
||||
// H H-A-275), scope module.
|
||||
VersionHeldElsewhere = "version-held-elsewhere"
|
||||
// VersionTaggedElsewhere is the finding of a number whose git tag stands at another commit, skipped at reserving (design 47
|
||||
// H2 §796), scope module; cleared at the module's next version-released.
|
||||
VersionTaggedElsewhere = "version-tagged-elsewhere"
|
||||
)
|
||||
|
||||
// ReservedVersion is the value of a reservation key.
|
||||
@@ -59,7 +64,7 @@ type ReservedVersion struct {
|
||||
PublishedAt map[string]time.Time `json:"published-at,omitempty"`
|
||||
At time.Time `json:"at"`
|
||||
// Repository and Commit are what the build was made from, and Tag the git tag its release is written as (a Go
|
||||
// package's, GoTagOf): empty for a module with no Go package (design 47 G §370).
|
||||
// package's, GoTagOf): empty for a module with no Go package (design 47 G §370), or whose Go package's directory is not known.
|
||||
Repository string `json:"repository,omitempty"`
|
||||
Commit string `json:"commit,omitempty"`
|
||||
Tag string `json:"tag,omitempty"`
|
||||
@@ -112,6 +117,9 @@ type Source struct {
|
||||
// Fingerprint is the fingerprint of the catalog entry the build is made from (design 47 G A-76).
|
||||
Fingerprint string
|
||||
GoPackage bool
|
||||
// GoArtifact is the artifact that is the module's Go package, whose module path names the directory its tag is written in
|
||||
// (design 47 H2 §796).
|
||||
GoArtifact string
|
||||
// Branch is the watched branch of the catalog revision whose entry the build came from, and Revision the source revision of
|
||||
// that catalog revision (design 47 G §371, A-57), kept on the reservation for the publication.
|
||||
Branch string
|
||||
@@ -145,6 +153,9 @@ type Reserver struct {
|
||||
// Took is told when a build took over the series' reservation of an older build never released (design 47 G A-455): the
|
||||
// older reservation, which no want follows any more, is gone from the key, replaced by the newer's. Nil is told none.
|
||||
Took func(ctx context.Context, older, newer ReservedVersion)
|
||||
// Reserved is told each reservation written, a number newly reserved (never one kept as it stood), so the holder says
|
||||
// version-reserved of its packages (design 47 H2 §805). Nil is told none.
|
||||
Reserved func(ctx context.Context, v ReservedVersion)
|
||||
// Releases is the builder's own record of the versions it saw released, by module and fingerprint (design 47 G A-456a): the
|
||||
// reservation is deleted at release, so a rerun of a released build's step reads it here, never from the catalog. Nil keeps none.
|
||||
Releases ReserveStore
|
||||
@@ -160,6 +171,92 @@ type Reserver struct {
|
||||
HeldNumbers func(ctx context.Context, module, fingerprint string, refs []string) ([]string, error)
|
||||
// HeldElsewhere is told the numbers of module a reserve skipped because a registry holds them (H-A-275); nil is told none.
|
||||
HeldElsewhere func(ctx context.Context, module string, skipped []string)
|
||||
// Names answers a Go package's module path, which names the directory its tag is written in (design 47 H2 §796); nil, or a
|
||||
// name not known, tags the version nowhere, said.
|
||||
Names PackageNames
|
||||
// Tags reads repository's tags starting with prefix, each with the commit it names, through the seat git's tags (design 47
|
||||
// H2 §796); away says the read met a transport fault or the error kind away, which lets the reservation go on, said. Nil
|
||||
// reads none.
|
||||
Tags func(ctx context.Context, repository, prefix string) (tags []git.Tag, away bool, err error)
|
||||
// TaggedElsewhere is told the numbers of module a reserve skipped because their tag stands at another commit (§796); nil
|
||||
// is told none.
|
||||
TaggedElsewhere func(ctx context.Context, module string, skipped []string)
|
||||
|
||||
tagsMu sync.Mutex
|
||||
// tagsRead are the tags last read of each module, by series patch to the commit, with the catalog revision they were read
|
||||
// under: a module's tags are read once per catalog revision it reserves under (§796).
|
||||
tagsRead map[string]tagsAt
|
||||
}
|
||||
|
||||
// tagsAt are a module's tags of one series read under one catalog revision: patch to the commit its tag names.
|
||||
type tagsAt struct {
|
||||
revision, prefix string
|
||||
commits map[int]string
|
||||
}
|
||||
|
||||
// goDirectory is the directory of module's Go package in its repository (§796), from its module path; ok is false, said, when
|
||||
// no name or no directory of the repository is known.
|
||||
func (r *Reserver) goDirectory(ctx context.Context, module string, src Source) (string, bool) {
|
||||
if r.Names == nil {
|
||||
return "", false
|
||||
}
|
||||
path, known, err := r.Names(ctx, module, src.GoArtifact)
|
||||
switch {
|
||||
case err != nil:
|
||||
r.said(fmt.Errorf("%s: its Go package's name cannot be read, so its version is tagged nowhere: %w", module, err))
|
||||
return "", false
|
||||
case !known:
|
||||
r.said(fmt.Errorf("%s: no name of its Go package %s is known yet, so its version is tagged nowhere", module, src.GoArtifact))
|
||||
return "", false
|
||||
}
|
||||
dir, ok := GoDirectory(path, src.Repository)
|
||||
if !ok {
|
||||
r.said(fmt.Errorf("%s: its Go package %s is not a module path of the repository %s, so its version is tagged nowhere", module, path, src.Repository))
|
||||
}
|
||||
return dir, ok
|
||||
}
|
||||
|
||||
// taggedAt are the patches of series whose tag `<directory>/v<series>.<patch>` stands, each with its commit, read once per
|
||||
// catalog revision the module reserves under (§796). A read that meets a transport fault or away answers none, said, and the
|
||||
// reservation goes on; any other failure refuses the reservation, naming it.
|
||||
func (r *Reserver) taggedAt(ctx context.Context, module, series, dir string, src Source) (map[int]string, error) {
|
||||
if r.Tags == nil {
|
||||
return nil, nil
|
||||
}
|
||||
prefix := TagPrefix(dir) + series + "."
|
||||
revision := ""
|
||||
if src.CatalogRevision > 0 {
|
||||
revision = fmt.Sprintf("%s.%d", src.Branch, src.CatalogRevision)
|
||||
r.tagsMu.Lock()
|
||||
got, ok := r.tagsRead[module]
|
||||
r.tagsMu.Unlock()
|
||||
if ok && got.revision == revision && got.prefix == prefix {
|
||||
return got.commits, nil
|
||||
}
|
||||
}
|
||||
tags, away, err := r.Tags(ctx, src.Repository, prefix)
|
||||
if err != nil && away {
|
||||
r.said(fmt.Errorf("%s: its tags %s* cannot be read (%w), so the reservation goes on without them", module, prefix, err))
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the tags %s* of %s cannot be read: %w", prefix, module, err)
|
||||
}
|
||||
commits := map[int]string{}
|
||||
for _, t := range tags {
|
||||
if s, p, err := SplitVersion(strings.TrimPrefix(t.Tag, TagPrefix(dir))); err == nil && s == series && strings.HasPrefix(t.Tag, prefix) {
|
||||
commits[p] = t.Commit
|
||||
}
|
||||
}
|
||||
if revision != "" {
|
||||
r.tagsMu.Lock()
|
||||
if r.tagsRead == nil {
|
||||
r.tagsRead = map[string]tagsAt{}
|
||||
}
|
||||
r.tagsRead[module] = tagsAt{revision: revision, prefix: prefix, commits: commits}
|
||||
r.tagsMu.Unlock()
|
||||
}
|
||||
return commits, nil
|
||||
}
|
||||
|
||||
// heldNumbers are the patches of series a registry holds of module's artifacts (H-A-275); a read that fails is said, and none.
|
||||
@@ -180,15 +277,20 @@ func (r *Reserver) heldNumbers(ctx context.Context, module, series, fingerprint
|
||||
return out
|
||||
}
|
||||
|
||||
// nextFree is the first patch after after that no registry holds, and the held numbers it skipped (H-A-275).
|
||||
func nextFree(series string, after int, held map[int]bool) (int, []string) {
|
||||
var skipped []string
|
||||
p := after + 1
|
||||
for held[p] {
|
||||
skipped = append(skipped, fmt.Sprintf("%s.%d", series, p))
|
||||
p++
|
||||
// nextFree is the first patch after after that no registry holds and whose tag stands at no other commit than commit, with the
|
||||
// held numbers it skipped (H-A-275) and those tagged elsewhere (design 47 H2 §796).
|
||||
func nextFree(series string, after int, held map[int]bool, tagged map[int]string, commit string) (int, []string, []string) {
|
||||
var skipped, elsewhere []string
|
||||
for p := after + 1; ; p++ {
|
||||
switch c, ok := tagged[p]; {
|
||||
case held[p]:
|
||||
skipped = append(skipped, fmt.Sprintf("%s.%d", series, p))
|
||||
case ok && c != commit:
|
||||
elsewhere = append(elsewhere, fmt.Sprintf("%s.%d (tagged at %s)", series, p, c))
|
||||
default:
|
||||
return p, skipped, elsewhere
|
||||
}
|
||||
}
|
||||
return p, skipped
|
||||
}
|
||||
|
||||
// said is told the outcome of a reindex: a failure is said, the reservation's own write stands.
|
||||
@@ -240,6 +342,15 @@ func (r *Reserver) Reserve(ctx context.Context, module, series string, artifacts
|
||||
}
|
||||
key := ReservedKey(module, series)
|
||||
held := r.heldNumbers(ctx, module, series, src.Fingerprint, artifacts)
|
||||
dir, tagged := "", false
|
||||
var tags map[int]string
|
||||
if src.GoPackage {
|
||||
if dir, tagged = r.goDirectory(ctx, module, src); tagged {
|
||||
if tags, err = r.taggedAt(ctx, module, series, dir, src); err != nil {
|
||||
return ReservedVersion{}, err
|
||||
}
|
||||
}
|
||||
}
|
||||
for {
|
||||
cur, rev, found, err := r.get(ctx, key)
|
||||
if err != nil {
|
||||
@@ -257,11 +368,11 @@ func (r *Reserver) Reserve(ctx context.Context, module, series string, artifacts
|
||||
next = p
|
||||
}
|
||||
}
|
||||
patch, skipped := nextFree(series, next, held)
|
||||
patch, skipped, elsewhere := nextFree(series, next, held, tags, src.Commit)
|
||||
out := ReservedVersion{V: 1, Module: module, Series: series, Version: fmt.Sprintf("%s.%d", series, patch), Artifacts: map[string]string{}, At: r.now(),
|
||||
Repository: src.Repository, Commit: src.Commit, Fingerprint: src.Fingerprint, Branch: src.Branch, Source: src.Revision, Revision: src.CatalogRevision}
|
||||
if src.GoPackage {
|
||||
out.Tag = GoTagOf(out.Version)
|
||||
if tagged {
|
||||
out.Tag = GoTagOf(dir, out.Version)
|
||||
}
|
||||
for _, a := range artifacts {
|
||||
out.Artifacts[a] = ""
|
||||
@@ -299,9 +410,15 @@ func (r *Reserver) Reserve(ctx context.Context, module, series string, artifacts
|
||||
if len(skipped) > 0 && r.HeldElsewhere != nil {
|
||||
r.HeldElsewhere(ctx, module, skipped)
|
||||
}
|
||||
if len(elsewhere) > 0 && r.TaggedElsewhere != nil {
|
||||
r.TaggedElsewhere(ctx, module, elsewhere)
|
||||
}
|
||||
if found && r.Took != nil {
|
||||
r.Took(ctx, cur, out)
|
||||
}
|
||||
if r.Reserved != nil {
|
||||
r.Reserved(ctx, out)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,14 +1,16 @@
|
||||
package versions
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
||||
)
|
||||
|
||||
type reserveMem struct {
|
||||
@@ -412,8 +414,8 @@ func TestANumberARegistryHoldsIsNeverReserved(t *testing.T) {
|
||||
held = []string{"0.1.37", "0.1.38", "0.1.39", "0.1.40", "0.2.0"}
|
||||
// The build again of a done key reserves as a rebuild (A-490b): a withdrawn number is never its floor's to reuse.
|
||||
again, err := r.Reserve(ctx, "node-tools", "0.1", []string{"node-tools:program"}, Source{Fingerprint: "fp1", GoPackage: true, Rebuild: true})
|
||||
if err != nil || again.Version != "0.1.41" || again.Artifacts["node-tools:program"] != "" || again.Withdrawn["0.1.39"] != "fp1" || again.Tag != GoTagOf("0.1.41") {
|
||||
t.Fatalf("the build again reserved %+v (%v), want 0.1.41 with no digest and 0.1.39 kept withdrawn", again, err)
|
||||
if err != nil || again.Version != "0.1.41" || again.Artifacts["node-tools:program"] != "" || again.Withdrawn["0.1.39"] != "fp1" || again.Tag != "" {
|
||||
t.Fatalf("the build again reserved %+v (%v), want 0.1.41 with no digest, 0.1.39 kept withdrawn, and no tag (no Go package name known)", again, err)
|
||||
}
|
||||
if at, _ := r.AtVersion(ctx, "0.1.39"); len(at) != 0 {
|
||||
t.Errorf("the index by version still names the withdrawn number: %v", at)
|
||||
@@ -426,4 +428,50 @@ func TestANumberARegistryHoldsIsNeverReserved(t *testing.T) {
|
||||
if v, err := r.Reserve(ctx, "node-tools", "0.2", []string{"node-tools:program"}, Source{Fingerprint: "fp1"}); err != nil || v.Version != "0.2.1" {
|
||||
t.Errorf("with the registry down reserved %q (%v), want 0.2.1 (what it answered still skipped)", v.Version, err)
|
||||
}
|
||||
// Design 47 H2 §796: a Go package's tags are read before it reserves, once per catalog revision; a number whose tag
|
||||
// `<directory>/v<version>` stands at another commit is skipped and said, one at the build's own commit is not; the version's
|
||||
// tag names the Go module's directory; a read met away lets the reservation go on, any other failure refuses it.
|
||||
r.HeldNumbers = nil
|
||||
var tagReads int
|
||||
var tagsErr error
|
||||
var tagsAway bool
|
||||
var elsewhere []string
|
||||
tags := []git.Tag{{Tag: "modules/nt/v0.3.0", Commit: "c-other"}, {Tag: "modules/nt/v0.3.1", Commit: "c-mine"}, {Tag: "modules/nt/v0.3.2", Commit: "c-other2"}}
|
||||
r.Names = func(_ context.Context, module, artifact string) (string, bool, error) {
|
||||
return "git.example/o/r/modules/nt", module == "node-tools" && artifact == "lib", nil
|
||||
}
|
||||
r.Tags = func(_ context.Context, repository, prefix string) ([]git.Tag, bool, error) {
|
||||
tagReads++
|
||||
if repository != "o/r" || prefix != "modules/nt/v0.3." {
|
||||
t.Errorf("tags read of %s %q, want o/r modules/nt/v0.3.", repository, prefix)
|
||||
}
|
||||
return tags, tagsAway, tagsErr
|
||||
}
|
||||
r.TaggedElsewhere = func(_ context.Context, _ string, s []string) { elsewhere = append(elsewhere, s...) }
|
||||
at := Source{Fingerprint: "fp3", GoPackage: true, GoArtifact: "lib", Repository: "o/r", Commit: "c-mine", Branch: "main", CatalogRevision: 7}
|
||||
v3, err := r.Reserve(ctx, "node-tools", "0.3", []string{"node-tools:lib"}, at)
|
||||
if err != nil || v3.Version != "0.3.1" || v3.Tag != "modules/nt/v0.3.1" || len(elsewhere) != 1 || !strings.Contains(elsewhere[0], "0.3.0") {
|
||||
t.Fatalf("reserved %+v (%v), tagged elsewhere %v: want 0.3.1 (0.3.0 tagged at another commit, 0.3.1 at the build's own), tag modules/nt/v0.3.1", v3, err, elsewhere)
|
||||
}
|
||||
at.Fingerprint, at.Commit, elsewhere = "fp4", "c-new", nil
|
||||
if v4, err := r.Reserve(ctx, "node-tools", "0.3", []string{"node-tools:lib"}, at); err != nil || v4.Version != "0.3.3" || tagReads != 1 || len(elsewhere) != 1 || !strings.Contains(elsewhere[0], "0.3.2") {
|
||||
t.Fatalf("reserved %+v (%v) after %d reads, tagged elsewhere %v: want 0.3.3 past 0.3.2's tag, the tags read once for revision main.7", v4, err, tagReads, elsewhere)
|
||||
}
|
||||
at.CatalogRevision, at.Fingerprint, tagsErr, tagsAway = 8, "fp5", errors.New("the forge is away"), true
|
||||
if v5, err := r.Reserve(ctx, "node-tools", "0.3", []string{"node-tools:lib"}, at); err != nil || v5.Version != "0.3.4" || tagReads != 2 {
|
||||
t.Fatalf("with the tags away reserved %+v (%v) after %d reads, want 0.3.4 and the reservation going on", v5, err, tagReads)
|
||||
}
|
||||
at.CatalogRevision, at.Fingerprint, tagsAway = 9, "fp6", false
|
||||
if _, err := r.Reserve(ctx, "node-tools", "0.3", []string{"node-tools:lib"}, at); err == nil || !strings.Contains(err.Error(), "tags") {
|
||||
t.Errorf("a tags read refused not away reserved anyway (%v), want the reservation refused naming the tags", err)
|
||||
}
|
||||
if d, ok := GoDirectory("git.example/novox/mesh-sdk/go", "https://git.example/novox/mesh-sdk.git"); !ok || d != "go" || GoTagOf(d, "0.1.9") != "go/v0.1.9" {
|
||||
t.Errorf("mesh-sdk's Go package: directory %q (%v), want go and the tag go/v0.1.9", d, ok)
|
||||
}
|
||||
if d, ok := GoDirectory("git.example/o/r", "o/r"); !ok || d != "" || GoTagOf(d, "1.0.0") != "v1.0.0" {
|
||||
t.Errorf("a root Go module: directory %q (%v), want the root and the tag v1.0.0", d, ok)
|
||||
}
|
||||
if _, ok := GoDirectory("git.example/o/rr/x", "o/r"); ok {
|
||||
t.Error("a module path of another repository was given a directory")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,4 +1,31 @@
|
||||
package versions
|
||||
|
||||
// GoTagOf is the git tag a Go package's released version is written as (design 47 G §370, design 51 *The versions*).
|
||||
func GoTagOf(version string) string { return "go/v" + version }
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk"
|
||||
)
|
||||
|
||||
// GoTagOf is the git tag a Go package's released version is written as (design 47 H2 §796): `<directory>/v<version>`, the
|
||||
// directory of its Go module in the repository, which the Go toolchain's direct fallback reads; `v<version>` at the root. The
|
||||
// `go/v<version>` tags go from new versions.
|
||||
func GoTagOf(directory, version string) string { return TagPrefix(directory) + version }
|
||||
|
||||
// GoDirectory is the directory of the Go module path modulePath in repository (`<owner>/<name>`, or a clone address naming
|
||||
// it): the part of the path after the repository's own, as the Go toolchain reads a module kept in a subdirectory; empty at
|
||||
// the root. ok is false when the path is not one of that repository.
|
||||
func GoDirectory(modulePath, repository string) (string, bool) {
|
||||
repo := "/" + strings.ToLower(trunk.RepositoryName(repository))
|
||||
lower := strings.ToLower(modulePath)
|
||||
for from := 0; ; {
|
||||
i := strings.Index(lower[from:], repo)
|
||||
if i < 0 || repo == "/" {
|
||||
return "", false
|
||||
}
|
||||
end := from + i + len(repo)
|
||||
if end == len(lower) || lower[end] == '/' {
|
||||
return strings.Trim(modulePath[end:], "/"), true
|
||||
}
|
||||
from = end
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,6 +13,8 @@ import (
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
|
||||
|
||||
"mesh-delivery-builder/internal/versions"
|
||||
)
|
||||
|
||||
// The controller's manifest reader is mesh-controller internal/catalogue (Manifest, manifest.go: Consumes :521,
|
||||
@@ -43,7 +45,9 @@ type manifest struct {
|
||||
Module string `json:"module"`
|
||||
Version string `json:"version"`
|
||||
Slug string `json:"slug"`
|
||||
Claims []struct {
|
||||
// Class is core: the builder claims a seat (design 47 H2 §810, §814; H-A-394 (1)).
|
||||
Class string `json:"class"`
|
||||
Claims []struct {
|
||||
Name string `json:"name"`
|
||||
Scope string `json:"scope"`
|
||||
Serves []string `json:"serves"`
|
||||
@@ -58,6 +62,9 @@ type manifest struct {
|
||||
MaxAgeDays int `json:"max-age-days"`
|
||||
MaxMiB int `json:"max-mib"`
|
||||
LastPerSubject bool `json:"last-per-subject"`
|
||||
Retention string `json:"retention"`
|
||||
Duplicates int `json:"duplicates"`
|
||||
MaxDeliveries int `json:"max-deliveries"`
|
||||
} `json:"streams"`
|
||||
Logs []struct {
|
||||
Name string `json:"name"`
|
||||
@@ -75,10 +82,11 @@ type manifest struct {
|
||||
Maximum float64 `json:"maximum"`
|
||||
Trusted bool `json:"trusted"`
|
||||
Default any `json:"default"`
|
||||
Runtime bool `json:"runtime"`
|
||||
} `json:"config"`
|
||||
OwnSecrets map[string]string `json:"own-secrets"`
|
||||
SecretsOwner string `json:"secrets-owner"`
|
||||
Resources []map[string]any `json:"resources"`
|
||||
OwnSecrets map[string]any `json:"own-secrets"`
|
||||
SecretsOwner string `json:"secrets-owner"`
|
||||
Resources []map[string]any `json:"resources"`
|
||||
Build struct {
|
||||
Artifacts []artifact `json:"artifacts"`
|
||||
} `json:"build"`
|
||||
@@ -114,7 +122,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
|
||||
if m.Module != "mesh-delivery-builder" {
|
||||
t.Errorf("module = %q", m.Module)
|
||||
}
|
||||
if want := []state{{Name: "builds", History: 5}, {Name: "engine-reads", History: 1}, {Name: "findings", History: 5}, {Name: "floor-waits", History: 1}, {Name: "hearing", History: 1}, {Name: "instances", History: 1}, {Name: "old-form", History: 1}, {Name: "bounds", History: 1, KeyTTL: true}, {Name: "queues", History: 1}, {Name: "toolchains-heard", History: 1}, {Name: "waits", History: 1, MaxMiB: 4}, {Name: "reserved", History: 1}, {Name: "publish-queue", History: 1, MaxMiB: 4}, {Name: "acted", History: 1, MaxMiB: 1}, {Name: "asked", History: 1}, {Name: "reasks", History: 1}, {Name: "versions", History: 1}, {Name: "first-views", History: 1}}; !reflect.DeepEqual(m.State, want) { // builds.BuilderBuildsHistory
|
||||
if want := []state{{Name: "builds", History: 5, MaxMiB: 128}, {Name: "engine-reads", History: 1, MaxMiB: 8}, {Name: "findings", History: 5, MaxMiB: 32}, {Name: "floor-waits", History: 1, MaxMiB: 8}, {Name: "hearing", History: 1, MaxMiB: 8}, {Name: "instances", History: 1, MaxMiB: 8}, {Name: "old-form", History: 1, MaxMiB: 8}, {Name: "bounds", History: 1, KeyTTL: true, MaxMiB: 16}, {Name: "taken", History: 1, KeyTTL: true, MaxMiB: 8}, {Name: "queues", History: 1, MaxMiB: 8}, {Name: "toolchains-heard", History: 1, MaxMiB: 8}, {Name: "waits", History: 1, MaxMiB: 4}, {Name: "reserved", History: 1, MaxMiB: 8}, {Name: "publish-queue", History: 1, MaxMiB: 4}, {Name: "acted", History: 1, MaxMiB: 1}, {Name: "asked", History: 1, MaxMiB: 8}, {Name: "reasks", History: 1, MaxMiB: 8}, {Name: "versions", History: 1, MaxMiB: 32}, {Name: "first-views", History: 1, MaxMiB: 8}, {Name: "unbuildable", History: 1, MaxMiB: 8}, {Name: "consumers", History: 1, MaxMiB: 8}}; !reflect.DeepEqual(m.State, want) { // builds.BuilderBuildsHistory // caps per H-A-437 over the fills read on novox 2026-11-22 (design 47 H2 §812, §814)
|
||||
t.Errorf("state = %+v, want %+v", m.State, want)
|
||||
}
|
||||
// 1887/A4c: the builder chooses no node, so it reads neither presence nor have, invokes no build module's tool and
|
||||
@@ -145,10 +153,10 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
|
||||
// The watchdogs seat's presence says a node's machine is heard (issue 1966, ADR 0596 §2, ADR 0668 §3).
|
||||
if want := []string{"seat:mesh-catalog.entry", "seat:mesh-catalog.registrations", "seat:mesh-catalog.seat-definitions", "seat:mesh-catalog.revision", "seat:mesh-delivery.source", "seat:container-image-registry.publish", "seat:container-image-registry.remove", "seat:container-image-registry.versions",
|
||||
"seat:git.tag", "seat:git.tags",
|
||||
"seat:go-package-registry.publish", "seat:go-package-registry.remove", "seat:go-package-registry.versions", "seat:mesh-artifact-sweep.held", "seat:mesh-builds.reserved", "seat:mesh-catalog.versions", "seat:mesh-hand-acts.hand-act", "seat:mesh-module-assigner.assignments", "seat:mesh-watchdogs.presence", "seat:node-build-go.drop", "seat:node-build-image.drop", "seat:node-build-node.drop", "seat:node-engine.manifest-reads", "seat:npm-package-registry.publish", "seat:npm-package-registry.remove",
|
||||
"seat:go-package-registry.publish", "seat:go-package-registry.remove", "seat:go-package-registry.versions", "seat:mesh-artifact-sweep.held", "seat:mesh-builds.reserved", "seat:mesh-builds.drop", "seat:mesh-builds.publish-queue", "seat:mesh-catalog.versions", "seat:mesh-hand-acts.hand-act", "seat:mesh-module-assigner.assignments", "seat:mesh-watchdogs.presence", "seat:node-build-go.drop", "seat:node-build-image.drop", "seat:node-build-node.drop", "seat:node-build-go.answer", "seat:node-build-image.answer", "seat:node-build-node.answer", "seat:node-engine.manifest-reads", "seat:npm-package-registry.publish", "seat:npm-package-registry.remove",
|
||||
"seat:npm-package-registry.versions", "seat:oci-artifact-registry.publish", "seat:oci-artifact-registry.remove", "seat:oci-artifact-registry.versions",
|
||||
// A-231: the build asks on each build seat's queue delivery (accept.delivery.<asker>).
|
||||
"seat:node-build-go.accept.delivery", "seat:node-build-node.accept.delivery", "seat:node-build-image.accept.delivery"}; !reflect.DeepEqual(sorted(m.Invokes), sorted(want)) {
|
||||
"seat:node-build-go.accept.delivery", "seat:node-build-node.accept.delivery", "seat:node-build-image.accept.delivery", "nats.nats_drop", "seat:node-build-go.current", "seat:node-build-image.current", "seat:node-build-node.current"}; !reflect.DeepEqual(sorted(m.Invokes), sorted(want)) {
|
||||
t.Errorf("invokes = %v, want %v", m.Invokes, want)
|
||||
}
|
||||
// The builder hears the build seats' toolchain-reported and the node-engine seat's running (design 47 G §492, §546) and reads no
|
||||
@@ -168,27 +176,36 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
|
||||
"oci-artifact-registry.kept", "oci-artifact-registry.published", "oci-artifact-registry.refused", "oci-artifact-registry.removed", "operator-channel.decided"}; !reflect.DeepEqual(sorted(m.Consumes), sorted(want)) {
|
||||
t.Errorf("consumes = %v, want %v", m.Consumes, want)
|
||||
}
|
||||
if b, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 || b != "${dir:state}/broker" {
|
||||
t.Errorf("own-secrets = %v, want exactly broker", m.OwnSecrets)
|
||||
// H-A-454 (second addendum): beside the module's broker, the process's own account's credential broker-builder.
|
||||
if b, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 2 || b != "${dir:state}/broker" ||
|
||||
fmt.Sprint(m.OwnSecrets["broker-builder"]) != "map[owner:mesh-delivery-builder path:${dir:state}/broker-builder.secret]" {
|
||||
t.Errorf("own-secrets = %v, want broker and broker-builder", m.OwnSecrets)
|
||||
}
|
||||
if m.SecretsOwner != "mesh-delivery-builder" {
|
||||
t.Errorf("secrets-owner = %q", m.SecretsOwner)
|
||||
}
|
||||
// 1945: the builder's tools (builder_seat_queue: note 62), served by its tools bundle, and the event it says when it asks an ask again.
|
||||
if want := []string{"builder_announce_again", "builder_ask_again", "builder_bounds", "builder_build", "builder_build_again", "builder_builds_since", "builder_engine_reads", "builder_first_view_again", "builder_first_views",
|
||||
"builder_floor_again", "builder_floor_waits", "builder_old_form", "builder_publish_again", "builder_publish_queue", "builder_release", "builder_reserve_done", "builder_reserved", "builder_reserved_drop", "builder_seat_list", "builder_seat_list_again", "builder_seat_queue", "builder_stuck", "builder_toolchains", "builder_toolchains_drop", "builder_versions", "builder_waits"}; !reflect.DeepEqual(sorted(m.Tools), want) {
|
||||
if want := []string{"build_current", "builder_announce_again", "builder_ask_again", "builder_ask_drop", "builder_asks", "builder_bounds", "builder_build", "builder_build_again", "builder_builds_since", "builder_engine_reads", "builder_first_view_again", "builder_first_views",
|
||||
"builder_floor_again", "builder_floor_waits", "builder_health", "builder_old_form", "builder_publish_again", "builder_publish_queue", "builder_release", "builder_reserve_done", "builder_reserved", "builder_reserved_drop", "builder_seat_list", "builder_seat_list_again", "builder_seat_queue", "builder_stuck", "builder_toolchains", "builder_toolchains_drop", "builder_unbuildable", "builder_versions", "builder_waits"}; !reflect.DeepEqual(sorted(m.Tools), want) {
|
||||
t.Errorf("tools = %v, want %v", m.Tools, want)
|
||||
}
|
||||
// Design 47 G §672: the toolchain waits' stream, kept 7 days within 16 MiB, and the bound's setting, 60 minutes.
|
||||
// Design 47 G A-30, §383: and its own findings' events, which its finding writer says.
|
||||
if len(m.Streams) != 3 || m.Streams[0].Name != "waits" || !reflect.DeepEqual(m.Streams[0].Events, []string{"wait-held.**", "wait-ended.**"}) || m.Streams[0].MaxAgeDays != 7 || m.Streams[0].MaxMiB != 16 ||
|
||||
if len(m.Streams) != 4 || m.Streams[0].Name != "waits" || !reflect.DeepEqual(m.Streams[0].Events, []string{"wait-held.**", "wait-ended.**"}) || m.Streams[0].MaxAgeDays != 7 || m.Streams[0].MaxMiB != 16 ||
|
||||
m.Streams[1].Name != builds.BuildsStream || !reflect.DeepEqual(m.Streams[1].Events, []string{builds.BuildsStreamEvents}) || m.Streams[1].MaxAgeDays != 7 || !m.Streams[1].LastPerSubject ||
|
||||
m.Streams[2].Name != "mesh-delivery-builder-findings" || !reflect.DeepEqual(m.Streams[2].Events, []string{"finding-written.*", "finding-cleared.*", "findings-stated.*"}) {
|
||||
m.Streams[2].Name != "mesh-delivery-builder-findings" || !reflect.DeepEqual(m.Streams[2].Events, []string{"finding-written.*", "finding-cleared.*", "findings-stated.*"}) ||
|
||||
m.Streams[2].MaxMiB != 8 || m.Streams[2].MaxAgeDays != 7 || // design 47 H2 §783: the findings history 8 MiB, 7 days
|
||||
// Design 47 H2 §793, §820, H-A-446 (2), H-A-454: the work queue publishes in the old section, 64 MiB, 7 days, duplicates 24 h.
|
||||
m.Streams[3].Name != versions.PublishesLocal || !reflect.DeepEqual(m.Streams[3].Events, []string{"publish.**"}) || m.Streams[3].Retention != "work-queue" ||
|
||||
m.Streams[3].Duplicates != 24 || m.Streams[3].MaxDeliveries != 10 || m.Streams[3].MaxAgeDays != 7 || m.Streams[3].MaxMiB != 64 {
|
||||
t.Errorf("streams = %+v, want waits with wait-held.** and wait-ended.**, 7 days, 16 MiB, the builds' stream (build.ended.**, 7 days, kept last per subject: design 47 G A-181) and the findings' stream", m.Streams)
|
||||
}
|
||||
if c, ok := m.Config["toolchain-wait-minutes"]; !ok || c.Default != float64(60) || len(m.Config) != 7 || m.Config["revision-wait-minutes"].Default != float64(2) || m.Config["package-wait-minutes"].Default != float64(10) ||
|
||||
m.Config["publish-pause"].Default != float64(30) || m.Config["publish-pause-max"].Default != float64(30) || m.Config["publish-tries"].Default != float64(8) {
|
||||
t.Errorf("config = %+v, want toolchain-wait-minutes of 60, revision-wait-minutes of 2 (H-A-156), package-wait-minutes of 10 (H-A-165), the publish settings 30 s, 30 min and 8 tries (design 47 G §588, §633, A-457a)", m.Config)
|
||||
if c, ok := m.Config["toolchain-wait-minutes"]; !ok || c.Default != float64(60) || len(m.Config) != 10 ||
|
||||
// design 47 H2 §775, H-A-381 (2): the kept-call pace and the taken key's bound, read by the process
|
||||
m.Config["kept-call-pace-first"].Default != "1s" || m.Config["kept-call-pace-max"].Default != "30s" || m.Config["ask-taken-bound"].Default != "10m" ||
|
||||
!m.Config["kept-call-pace-first"].Runtime || !m.Config["kept-call-pace-max"].Runtime || !m.Config["ask-taken-bound"].Runtime || m.Config["revision-wait-minutes"].Default != float64(2) || m.Config["package-wait-minutes"].Default != float64(10) ||
|
||||
m.Config["publish-pause"].Default != float64(30) || m.Config["publish-pause-max"].Default != float64(30) || m.Config["publish-tries"].Default != float64(5) || m.Config["publish-tries"].Maximum != 10 {
|
||||
t.Errorf("config = %+v, want toolchain-wait-minutes of 60, revision-wait-minutes of 2 (H-A-156), package-wait-minutes of 10 (H-A-165), the publish settings 30 s, 30 min and 5 tries, at most 10 (design 47 G §588, §633, A-457a; H2 H-A-446 (2))", m.Config)
|
||||
}
|
||||
// Design 47 G §349: version.numbered is no longer said; old-form-closed on the family version, apart from the builds' ends.
|
||||
if want := []string{"build.asked-again", "version.old-form-closed"}; !reflect.DeepEqual(m.Emits, want) {
|
||||
@@ -198,9 +215,10 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
|
||||
if len(m.Logs) != 1 || m.Logs[0].Name != builds.BuilderLog || m.Logs[0].MaxAgeDays != builds.BuilderLogDays || m.Logs[0].MaxMiB != builds.BuilderLogMiB {
|
||||
t.Errorf("logs = %+v, want %s kept %d days in %d MiB", m.Logs, builds.BuilderLog, builds.BuilderLogDays, builds.BuilderLogMiB)
|
||||
}
|
||||
// Design 47 G §52, §369, H-A-328: the seat mesh-builds, held once for the mesh, serving reserved and published.
|
||||
if len(m.Claims) != 1 || m.Claims[0].Name != "mesh-builds" || m.Claims[0].Scope != "mesh" || !reflect.DeepEqual(m.Claims[0].Serves, []string{"reserved", "published"}) {
|
||||
t.Errorf("claims = %+v, want mesh-builds, scope mesh, serving reserved and published", m.Claims)
|
||||
// Design 47 G §52, §369, H-A-328, H2 §778, §793: the seat mesh-builds, held once for the mesh, serving reserved, published, drop
|
||||
// and publish-queue (H-A-454).
|
||||
if len(m.Claims) != 1 || m.Claims[0].Name != "mesh-builds" || m.Claims[0].Scope != "mesh" || !reflect.DeepEqual(m.Claims[0].Serves, []string{"reserved", "published", "drop", "publish-queue"}) {
|
||||
t.Errorf("claims = %+v, want mesh-builds, scope mesh, serving reserved, published, drop and publish-queue", m.Claims)
|
||||
}
|
||||
// A-115: the process answers the verb itself and says so in `serves` beside its process, with no `bus`; the bundle serves no tool named so.
|
||||
var resources []map[string]any
|
||||
@@ -208,8 +226,8 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, r := range resources {
|
||||
if r["type"] == "process" && !reflect.DeepEqual(r["serves"], []any{"seat:mesh-builds.reserved", "seat:mesh-builds.published"}) {
|
||||
t.Errorf("the process serves %v, want seat:mesh-builds.reserved and published (A-115, H-A-328)", r["serves"])
|
||||
if r["type"] == "process" && !reflect.DeepEqual(r["serves"], []any{"seat:mesh-builds.reserved", "seat:mesh-builds.published", "seat:mesh-builds.drop", "seat:mesh-builds.publish-queue"}) {
|
||||
t.Errorf("the process serves %v, want seat:mesh-builds.reserved, published, drop and publish-queue (A-115, H-A-328, H2 §778, §793)", r["serves"])
|
||||
}
|
||||
}
|
||||
if slices.Contains(m.Tools, "reserved") {
|
||||
@@ -240,8 +258,13 @@ func TestManifestProcess(t *testing.T) { // 1607/AC1
|
||||
if p["artifact"] != "program" || !reflect.DeepEqual(p["run"], []any{"./mesh-delivery-builder"}) {
|
||||
t.Errorf("process = %v", p)
|
||||
}
|
||||
// Design 47 H2 §794, H-A-435 (second addendum): the process's health stays kind unit, builder_health a tool beside it (a
|
||||
// tool-only health is refused, ADR 0227 rule 8, ADR 0240 rule 2).
|
||||
if h, _ := p["health"].(map[string]any); h["kind"] != "unit" || len(h) != 1 {
|
||||
t.Errorf("health = %v, want kind unit alone", p["health"])
|
||||
}
|
||||
env, _ := p["env"].(map[string]any)
|
||||
if len(env) != 8 || env["MESH_BUILDER_REVISION_WAIT_FILE"] != "${dir:state}/revision-wait-minutes" || env["MESH_BUILDER_PACKAGE_WAIT_FILE"] != "${dir:state}/package-wait-minutes" || env["MESH_BUILDER_PUBLISH_TRIES_FILE"] != "${dir:state}/publish-tries" || env["MESH_BROKER_FILE"] != "${dir:state}/broker" || env["MESH_BUILDER_TOOLCHAIN_WAIT_FILE"] != "${dir:state}/toolchain-wait-minutes" {
|
||||
if len(env) != 8 || env["MESH_BUILDER_REVISION_WAIT_FILE"] != "${dir:state}/revision-wait-minutes" || env["MESH_BUILDER_PACKAGE_WAIT_FILE"] != "${dir:state}/package-wait-minutes" || env["MESH_BUILDER_PUBLISH_TRIES_FILE"] != "${dir:state}/publish-tries" || env["MESH_BROKER_FILE"] != "${dir:state}/broker-builder.secret" || env["MESH_BUILDER_TOOLCHAIN_WAIT_FILE"] != "${dir:state}/toolchain-wait-minutes" {
|
||||
t.Errorf("env = %v, want the broker file and the path of the toolchain wait's file, never the setting itself (design 47 G §672, A-71)", p["env"])
|
||||
}
|
||||
for k, v := range env {
|
||||
@@ -263,8 +286,23 @@ func TestManifestProcess(t *testing.T) { // 1607/AC1
|
||||
t.Errorf("a settings resource %v: the builder holds no setting since the capacity is the build modules' (ADR 0525 §5)", r)
|
||||
}
|
||||
}
|
||||
if _, bus := p["bus"]; bus {
|
||||
t.Errorf("the process declares a bus: it runs on its own broker credential")
|
||||
// Design 47 H2 §793, H-A-454 (second addendum): the process runs on an account of its own, its bus naming the work queue
|
||||
// publishes in queues and what it does on the bus: the module's emits, consumes and state, and its invokes but the verb
|
||||
// publish-queue it serves and the build seats' current build_current asks (§801); its credential is broker-builder.
|
||||
bus, _ := p["bus"].(map[string]any)
|
||||
var states []string
|
||||
for _, st := range m.State {
|
||||
states = append(states, st.Name)
|
||||
}
|
||||
var invokes []string
|
||||
for _, i := range m.Invokes {
|
||||
if !slices.Contains([]string{"seat:mesh-builds.publish-queue", "seat:node-build-go.current", "seat:node-build-image.current", "seat:node-build-node.current"}, i) { // the tools bundle's
|
||||
invokes = append(invokes, i)
|
||||
}
|
||||
}
|
||||
if fmt.Sprint(bus["queues"]) != "[publishes]" || fmt.Sprint(bus["emits"]) != fmt.Sprint(m.Emits) || fmt.Sprint(bus["consumes"]) != fmt.Sprint(m.Consumes) ||
|
||||
fmt.Sprint(bus["state"]) != fmt.Sprint(states) || fmt.Sprint(bus["invokes"]) != fmt.Sprint(invokes) || len(bus) != 5 || env["MESH_BROKER_FILE"] != "${dir:state}/broker-builder.secret" {
|
||||
t.Errorf("the process's bus = %v, broker file %v: want publishes in queues, the module's emits, consumes, state and invokes, on broker-builder", bus, env["MESH_BROKER_FILE"])
|
||||
}
|
||||
// H-A-186, design 47 G §370: a Go package another module imports is a declared package artifact of its module, so every
|
||||
// release publishes it: builds (ADR 0634 §1, ADR 0606 §4), imported by the build modules and the node-engine, and
|
||||
@@ -306,7 +344,9 @@ func TestServesNoRequestSubjectAndHoldsNoTicker(t *testing.T) { // 1607/AC6 (ADR
|
||||
}
|
||||
for _, f := range forbidden {
|
||||
// The one subscription is the seat mesh-builds' accepted kind `step`, the runner's build ask (design 47 G A-97, A-107).
|
||||
if f == "Subscribe(" && filepath.ToSlash(path) == "cmd/mesh-delivery-builder/step.go" {
|
||||
// Design 47 H2 §777: the seat's work queue is pulled, its worker bound when the bus says it was created (the
|
||||
// one subscription), and a taken ask is said in progress while it is built (ADR 0525 §6), the queue's own cadence.
|
||||
if (f == "Subscribe(" || f == "time.NewTicker") && filepath.ToSlash(path) == "cmd/mesh-delivery-builder/stepqueue.go" {
|
||||
continue
|
||||
}
|
||||
if strings.Contains(string(src), f) {
|
||||
|
||||
@@ -2,13 +2,16 @@
|
||||
"module": "mesh-delivery-builder",
|
||||
"version": "0.1",
|
||||
"slug": "builder",
|
||||
"class": "core",
|
||||
"claims": [
|
||||
{
|
||||
"name": "mesh-builds",
|
||||
"scope": "mesh",
|
||||
"serves": [
|
||||
"reserved",
|
||||
"published"
|
||||
"published",
|
||||
"drop",
|
||||
"publish-queue"
|
||||
]
|
||||
}
|
||||
],
|
||||
@@ -36,9 +39,14 @@
|
||||
"builder_toolchains",
|
||||
"builder_toolchains_drop",
|
||||
"builder_waits",
|
||||
"builder_unbuildable",
|
||||
"builder_health",
|
||||
"builder_ask_drop",
|
||||
"builder_asks",
|
||||
"builder_versions",
|
||||
"builder_first_views",
|
||||
"builder_first_view_again"
|
||||
"builder_first_view_again",
|
||||
"build_current"
|
||||
],
|
||||
"emits": [
|
||||
"build.asked-again",
|
||||
@@ -89,6 +97,8 @@
|
||||
"seat:go-package-registry.versions",
|
||||
"seat:mesh-artifact-sweep.held",
|
||||
"seat:mesh-builds.reserved",
|
||||
"seat:mesh-builds.drop",
|
||||
"seat:mesh-builds.publish-queue",
|
||||
"seat:mesh-catalog.versions",
|
||||
"seat:mesh-hand-acts.hand-act",
|
||||
"seat:mesh-module-assigner.assignments",
|
||||
@@ -96,6 +106,9 @@
|
||||
"seat:node-build-go.drop",
|
||||
"seat:node-build-image.drop",
|
||||
"seat:node-build-node.drop",
|
||||
"seat:node-build-go.answer",
|
||||
"seat:node-build-image.answer",
|
||||
"seat:node-build-node.answer",
|
||||
"seat:node-engine.manifest-reads",
|
||||
"seat:npm-package-registry.remove",
|
||||
"seat:npm-package-registry.versions",
|
||||
@@ -112,32 +125,40 @@
|
||||
"seat:mesh-delivery.source",
|
||||
"seat:node-build-go.accept.delivery",
|
||||
"seat:node-build-node.accept.delivery",
|
||||
"seat:node-build-image.accept.delivery"
|
||||
"seat:node-build-image.accept.delivery",
|
||||
"nats.nats_drop",
|
||||
"seat:node-build-go.current",
|
||||
"seat:node-build-image.current",
|
||||
"seat:node-build-node.current"
|
||||
],
|
||||
"state": [
|
||||
{ "name": "builds", "history": 5 },
|
||||
{ "name": "engine-reads", "history": 1 },
|
||||
{ "name": "findings", "history": 5 },
|
||||
{ "name": "floor-waits", "history": 1 },
|
||||
{ "name": "hearing", "history": 1 },
|
||||
{ "name": "instances", "history": 1 },
|
||||
{ "name": "old-form", "history": 1 },
|
||||
{ "name": "bounds", "history": 1, "key-ttl": true },
|
||||
{ "name": "queues", "history": 1 },
|
||||
{ "name": "toolchains-heard", "history": 1 },
|
||||
{ "name": "builds", "history": 5, "max-mib": 128 },
|
||||
{ "name": "engine-reads", "history": 1, "max-mib": 8 },
|
||||
{ "name": "findings", "history": 5, "max-mib": 32 },
|
||||
{ "name": "floor-waits", "history": 1, "max-mib": 8 },
|
||||
{ "name": "hearing", "history": 1, "max-mib": 8 },
|
||||
{ "name": "instances", "history": 1, "max-mib": 8 },
|
||||
{ "name": "old-form", "history": 1, "max-mib": 8 },
|
||||
{ "name": "bounds", "history": 1, "key-ttl": true, "max-mib": 16 },
|
||||
{ "name": "taken", "history": 1, "key-ttl": true, "max-mib": 8 },
|
||||
{ "name": "queues", "history": 1, "max-mib": 8 },
|
||||
{ "name": "toolchains-heard", "history": 1, "max-mib": 8 },
|
||||
{ "name": "waits", "history": 1, "max-mib": 4 },
|
||||
{ "name": "reserved", "history": 1 },
|
||||
{ "name": "reserved", "history": 1, "max-mib": 8 },
|
||||
{ "name": "publish-queue", "history": 1, "max-mib": 4 },
|
||||
{ "name": "acted", "history": 1, "max-mib": 1 },
|
||||
{ "name": "asked", "history": 1 },
|
||||
{ "name": "reasks", "history": 1 },
|
||||
{ "name": "versions", "history": 1 },
|
||||
{ "name": "first-views", "history": 1 }
|
||||
{ "name": "asked", "history": 1, "max-mib": 8 },
|
||||
{ "name": "reasks", "history": 1, "max-mib": 8 },
|
||||
{ "name": "versions", "history": 1, "max-mib": 32 },
|
||||
{ "name": "first-views", "history": 1, "max-mib": 8 },
|
||||
{ "name": "unbuildable", "history": 1, "max-mib": 8 },
|
||||
{ "name": "consumers", "history": 1, "max-mib": 8 }
|
||||
],
|
||||
"streams": [
|
||||
{ "name": "waits", "events": ["wait-held.**", "wait-ended.**"], "max-age-days": 7, "max-mib": 16 },
|
||||
{ "name": "mesh-delivery-builder-builds", "events": ["build.ended.**"], "max-age-days": 7, "max-mib": 16, "last-per-subject": true },
|
||||
{ "name": "mesh-delivery-builder-findings", "events": ["finding-written.*", "finding-cleared.*", "findings-stated.*"], "max-age-days": 7, "max-mib": 16 }
|
||||
{ "name": "mesh-delivery-builder-findings", "events": ["finding-written.*", "finding-cleared.*", "findings-stated.*"], "max-age-days": 7, "max-mib": 8 },
|
||||
{ "name": "publishes", "events": ["publish.**"], "retention": "work-queue", "duplicates": 24, "max-deliveries": 10, "max-age-days": 7, "max-mib": 64 }
|
||||
],
|
||||
"config": {
|
||||
"toolchain-wait-minutes": {
|
||||
@@ -164,6 +185,27 @@
|
||||
"trusted": false,
|
||||
"default": 10
|
||||
},
|
||||
"ask-taken-bound": {
|
||||
"type": "string",
|
||||
"description": "how long the builder keeps the id of an ask it took from the seat mesh-builds' work queue, so a repeat of it is taken once, a Go duration such as 10m (design 47 H2 §775; H-A-381 (2)).",
|
||||
"trusted": false,
|
||||
"default": "10m",
|
||||
"runtime": true
|
||||
},
|
||||
"kept-call-pace-first": {
|
||||
"type": "string",
|
||||
"description": "the first wait before a kept call met by a transport fault is made again, doubled per call up to kept-call-pace-max, a Go duration (design 47 H2 §775).",
|
||||
"trusted": false,
|
||||
"default": "1s",
|
||||
"runtime": true
|
||||
},
|
||||
"kept-call-pace-max": {
|
||||
"type": "string",
|
||||
"description": "the most a kept call's wait between calls grows to, a Go duration (design 47 H2 §775).",
|
||||
"trusted": false,
|
||||
"default": "30s",
|
||||
"runtime": true
|
||||
},
|
||||
"publish-pause": {
|
||||
"type": "number",
|
||||
"description": "seconds before a version's second publish try, doubled per try up to publish-pause-max (design 47 G A-457a).",
|
||||
@@ -190,11 +232,11 @@
|
||||
},
|
||||
"publish-tries": {
|
||||
"type": "number",
|
||||
"description": "how many times the builder tries a done build's publish before it ends loudly (log and condition) and leaves it readable in builder_publish_queue (design 47 G A-457a).",
|
||||
"description": "how many times the builder tries a done build's publish before it ends loudly (log and condition) and leaves it readable in builder_publish_queue, at most 10, the work queue publishes' max-deliveries (design 47 G A-457a; H2 H-A-446 (2)).",
|
||||
"minimum": 1,
|
||||
"maximum": 100,
|
||||
"maximum": 10,
|
||||
"trusted": false,
|
||||
"default": 8
|
||||
"default": 5
|
||||
}
|
||||
},
|
||||
"logs": [
|
||||
@@ -213,7 +255,11 @@
|
||||
"operator-channel"
|
||||
],
|
||||
"own-secrets": {
|
||||
"broker": "${dir:state}/broker"
|
||||
"broker": "${dir:state}/broker",
|
||||
"broker-builder": {
|
||||
"path": "${dir:state}/broker-builder.secret",
|
||||
"owner": "mesh-delivery-builder"
|
||||
}
|
||||
},
|
||||
"secrets-owner": "mesh-delivery-builder",
|
||||
"resources": [
|
||||
@@ -295,13 +341,15 @@
|
||||
"artifact": "program",
|
||||
"serves": [
|
||||
"seat:mesh-builds.reserved",
|
||||
"seat:mesh-builds.published"
|
||||
"seat:mesh-builds.published",
|
||||
"seat:mesh-builds.drop",
|
||||
"seat:mesh-builds.publish-queue"
|
||||
],
|
||||
"run": [
|
||||
"./mesh-delivery-builder"
|
||||
],
|
||||
"env": {
|
||||
"MESH_BROKER_FILE": "${dir:state}/broker",
|
||||
"MESH_BROKER_FILE": "${dir:state}/broker-builder.secret",
|
||||
"MESH_BUILDER_TOOLCHAIN_WAIT_FILE": "${dir:state}/toolchain-wait-minutes",
|
||||
"MESH_BUILDER_REVISION_WAIT_FILE": "${dir:state}/revision-wait-minutes",
|
||||
"MESH_BUILDER_PACKAGE_WAIT_FILE": "${dir:state}/package-wait-minutes",
|
||||
@@ -311,6 +359,113 @@
|
||||
"MESH_BUILDER_RELEASED_KEPT_FILE": "${dir:state}/released-kept"
|
||||
},
|
||||
"user": "mesh-delivery-builder",
|
||||
"bus": {
|
||||
"emits": [
|
||||
"build.asked-again",
|
||||
"version.old-form-closed"
|
||||
],
|
||||
"consumes": [
|
||||
"mesh-conditions.findings-asked",
|
||||
"container-image-registry.kept",
|
||||
"container-image-registry.published",
|
||||
"container-image-registry.refused",
|
||||
"container-image-registry.removed",
|
||||
"go-package-registry.kept",
|
||||
"go-package-registry.published",
|
||||
"go-package-registry.refused",
|
||||
"go-package-registry.removed",
|
||||
"mesh-artifact-sweep.artifacts-gone",
|
||||
"mesh-catalog.seat-definition-changed",
|
||||
"mesh-catalog.seat-definition-removed",
|
||||
"mesh-catalog.version-released",
|
||||
"mesh-catalog.versions-filled",
|
||||
"mesh-conditions.condition-cleared",
|
||||
"mesh-delivery-indexer.take-in.taken",
|
||||
"node-build-go.toolchain-reported",
|
||||
"node-build-node.toolchain-reported",
|
||||
"node-engine.manifest-reads-changed",
|
||||
"node-engine.running",
|
||||
"operator-channel.decided",
|
||||
"npm-package-registry.kept",
|
||||
"npm-package-registry.published",
|
||||
"npm-package-registry.refused",
|
||||
"npm-package-registry.removed",
|
||||
"oci-artifact-registry.kept",
|
||||
"oci-artifact-registry.published",
|
||||
"oci-artifact-registry.refused",
|
||||
"oci-artifact-registry.removed",
|
||||
"seat:mesh-module-assigner.views.assignments",
|
||||
"seat:mesh-module-assigner.views.seat-holders",
|
||||
"mesh-catalog.catalog-revision",
|
||||
"mesh-catalog.registration-changed",
|
||||
"mesh-delivery.source-moved"
|
||||
],
|
||||
"queues": [
|
||||
"publishes"
|
||||
],
|
||||
"state": [
|
||||
"builds",
|
||||
"engine-reads",
|
||||
"findings",
|
||||
"floor-waits",
|
||||
"hearing",
|
||||
"instances",
|
||||
"old-form",
|
||||
"bounds",
|
||||
"taken",
|
||||
"queues",
|
||||
"toolchains-heard",
|
||||
"waits",
|
||||
"reserved",
|
||||
"publish-queue",
|
||||
"acted",
|
||||
"asked",
|
||||
"reasks",
|
||||
"versions",
|
||||
"first-views",
|
||||
"unbuildable",
|
||||
"consumers"
|
||||
],
|
||||
"invokes": [
|
||||
"seat:container-image-registry.remove",
|
||||
"seat:container-image-registry.versions",
|
||||
"seat:git.tag",
|
||||
"seat:git.tags",
|
||||
"seat:go-package-registry.remove",
|
||||
"seat:go-package-registry.versions",
|
||||
"seat:mesh-artifact-sweep.held",
|
||||
"seat:mesh-builds.reserved",
|
||||
"seat:mesh-builds.drop",
|
||||
"seat:mesh-catalog.versions",
|
||||
"seat:mesh-hand-acts.hand-act",
|
||||
"seat:mesh-module-assigner.assignments",
|
||||
"seat:mesh-watchdogs.presence",
|
||||
"seat:node-build-go.drop",
|
||||
"seat:node-build-image.drop",
|
||||
"seat:node-build-node.drop",
|
||||
"seat:node-build-go.answer",
|
||||
"seat:node-build-image.answer",
|
||||
"seat:node-build-node.answer",
|
||||
"seat:node-engine.manifest-reads",
|
||||
"seat:npm-package-registry.remove",
|
||||
"seat:npm-package-registry.versions",
|
||||
"seat:oci-artifact-registry.remove",
|
||||
"seat:oci-artifact-registry.versions",
|
||||
"seat:container-image-registry.publish",
|
||||
"seat:go-package-registry.publish",
|
||||
"seat:npm-package-registry.publish",
|
||||
"seat:oci-artifact-registry.publish",
|
||||
"seat:mesh-catalog.entry",
|
||||
"seat:mesh-catalog.registrations",
|
||||
"seat:mesh-catalog.seat-definitions",
|
||||
"seat:mesh-catalog.revision",
|
||||
"seat:mesh-delivery.source",
|
||||
"seat:node-build-go.accept.delivery",
|
||||
"seat:node-build-node.accept.delivery",
|
||||
"seat:node-build-image.accept.delivery",
|
||||
"nats.nats_drop"
|
||||
]
|
||||
},
|
||||
"health": {
|
||||
"kind": "unit"
|
||||
}
|
||||
|
||||
@@ -19,13 +19,14 @@ const Form = 1
|
||||
// whose fields changed while its forms did not (design 47 H2 §780).
|
||||
const (
|
||||
generatedForms = "1"
|
||||
generatedFieldsDigest = "AskTakenEvent=6d5d3da7ac39 BuildEntry=3c829a0a3d58 Dependency=90a32b07c563 DropAnswer=7538f2f5e68e PackagePublished=34dd069a6aab PackagePublishedEvent=8e12888c1559 PublishedAnswer=69a801982819 ReservedAnswer=2272a4d6f0f4 ReservedAnswerArtifactsItem=d74eb6e00abb StepAccept=18efeefb6c34 StepEnded=09a07bfbcb15 StepEndedEvent=207787118dac StepRun=dc5893aef7e5 ToolchainAgreed=f3d9ce993734 ToolchainAgreedDigestsItem=1d42893a96b9 ToolchainAgreedEvent=e3502d6f194d VersionReservedEvent=984547d6ad07"
|
||||
generatedFieldsDigest = "AskTakenEvent=6d5d3da7ac39 BuildEntry=3c829a0a3d58 Dependency=90a32b07c563 DropAnswer=7538f2f5e68e PackagePublished=34dd069a6aab PackagePublishedEvent=8e12888c1559 PublishQueueAnswer=0f1326e35e17 PublishQueueAnswerCountsItem=b770db2f0920 PublishQueueAnswerQueueItem=b1eebc09f41e PublishedAnswer=69a801982819 ReservedAnswer=2272a4d6f0f4 ReservedAnswerArtifactsItem=d74eb6e00abb StepAccept=18efeefb6c34 StepEnded=09a07bfbcb15 StepEndedEvent=207787118dac StepRun=dc5893aef7e5 ToolchainAgreed=f3d9ce993734 ToolchainAgreedDigestsItem=1d42893a96b9 ToolchainAgreedEvent=e3502d6f194d VersionReservedEvent=984547d6ad07"
|
||||
)
|
||||
|
||||
const (
|
||||
VerbReserved = "reserved"
|
||||
VerbPublished = "published"
|
||||
VerbDrop = "drop"
|
||||
VerbReserved = "reserved"
|
||||
VerbPublished = "published"
|
||||
VerbDrop = "drop"
|
||||
VerbPublishQueue = "publish-queue"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -42,7 +43,7 @@ const (
|
||||
|
||||
// Verbs is every verb the seat serves, in its definition's order: a new copy each call.
|
||||
func Verbs() []string {
|
||||
return []string{VerbReserved, VerbPublished, VerbDrop}
|
||||
return []string{VerbReserved, VerbPublished, VerbDrop, VerbPublishQueue}
|
||||
}
|
||||
|
||||
// VerbDescription is a verb's description as the seat's definition says it, "" for a verb the seat does not serve or
|
||||
@@ -54,7 +55,9 @@ func VerbDescription(verb string) string {
|
||||
case VerbPublished:
|
||||
return "Whether an artifact built at a build fingerprint is published, from the builder's reservations and the registries' record, for the pipeline runner writing a mover's wait on a package: a package published before the wait was written is read, not waited for (design 47 H-A-326, H-A-328, H-A-274)."
|
||||
case VerbDrop:
|
||||
return "End an ask on the seat's work queue for a caller that no longer waits for it (its step ended by its bound): one waiting is ended dropped when it is pulled, one taken is cancelled; both end the step dropped, naming who dropped it and why, so no holder works on it after (design 47 H2 §778; ADR 0664 §4)."
|
||||
return "End an ask on the seat's work queue for a caller that no longer waits for it (its step ended by its bound): one waiting is ended dropped when it is pulled, one taken is cancelled; both end the step could-not-run, its why naming the drop and who asked it, so no holder works on it after (design 47 H2 §778; H-A-401; ADR 0664 §4)."
|
||||
case VerbPublishQueue:
|
||||
return "What the builder's work queue publishes holds, read by the builder from its own stream (design 47 H2 §793; H-A-454): the items' count per registry seat holder and class with the oldest and newest queue time, and the first 50 in the order they are pulled (the live class first, each class oldest first); module, and version, list only that module's items. A queue read past 4,000 items is answered capped, naming the items read and the queue's size."
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -303,6 +306,78 @@ type DropInput struct {
|
||||
// DropAnswer is what drop answers.
|
||||
type DropAnswer = map[string]any
|
||||
|
||||
// PublishQueueInput is what publish-queue takes.
|
||||
type PublishQueueInput struct {
|
||||
// list only this module's items
|
||||
Module string `json:"module,omitempty"`
|
||||
// with module: only this version's, MAJOR.MINOR.PATCH
|
||||
Version string `json:"version,omitempty"`
|
||||
}
|
||||
|
||||
// PublishQueueAnswerCountsItem is a type of the contract.
|
||||
type PublishQueueAnswerCountsItem struct {
|
||||
// the items' class
|
||||
Class string `json:"class"`
|
||||
// how many items of the class the holder's queue holds
|
||||
Count int64 `json:"count"`
|
||||
// the registry seat's holder
|
||||
Holder string `json:"holder"`
|
||||
// when the newest of them was queued
|
||||
Newest time.Time `json:"newest"`
|
||||
// when the oldest of them was queued
|
||||
Oldest time.Time `json:"oldest"`
|
||||
}
|
||||
|
||||
// The values of PublishQueueAnswerCountsItem's class.
|
||||
const (
|
||||
PublishQueueAnswerCountsItemClassLive = "live"
|
||||
PublishQueueAnswerCountsItemClassAgain = "again"
|
||||
)
|
||||
|
||||
// PublishQueueAnswerQueueItem is a type of the contract.
|
||||
type PublishQueueAnswerQueueItem struct {
|
||||
// each artifact reference, <module>:<artifact>, with the digest to publish
|
||||
Artifacts map[string]string `json:"artifacts,omitempty"`
|
||||
// live, a build a pipeline's step waits on, pulled first; again, asked again at a start
|
||||
Class string `json:"class"`
|
||||
// the registry seat's holder the item is sent to, a module
|
||||
Holder string `json:"holder"`
|
||||
// the module whose version is published
|
||||
Module string `json:"module"`
|
||||
// when the item was put on the work queue
|
||||
Queued time.Time `json:"queued"`
|
||||
// the reserved version, MAJOR.MINOR.PATCH
|
||||
Version string `json:"version"`
|
||||
}
|
||||
|
||||
// The values of PublishQueueAnswerQueueItem's class.
|
||||
const (
|
||||
PublishQueueAnswerQueueItemClassLive = "live"
|
||||
PublishQueueAnswerQueueItemClassAgain = "again"
|
||||
)
|
||||
|
||||
// PublishQueueAnswer is what publish-queue answers.
|
||||
type PublishQueueAnswer struct {
|
||||
// true when the queue held more items than were read; every count is then of the items read
|
||||
Capped bool `json:"capped,omitempty"`
|
||||
// per holder and class, sorted by holder, then class
|
||||
Counts []PublishQueueAnswerCountsItem `json:"counts"`
|
||||
// is the form of what this value carries, left out while it is 1 (design 47 H2 §780).
|
||||
Form int64 `json:"form,omitempty"`
|
||||
// with capped: how many items were read
|
||||
ItemsRead int64 `json:"items-read,omitempty"`
|
||||
// how many of total the answer lists
|
||||
Listed int64 `json:"listed"`
|
||||
// the first 50 items listed, per holder in the order they are pulled
|
||||
Queue []PublishQueueAnswerQueueItem `json:"queue"`
|
||||
// with capped: the queue's size in bytes
|
||||
QueueBytes int64 `json:"queue-bytes,omitempty"`
|
||||
// with capped: how many items the queue holds, from its stream's state
|
||||
QueueItems int64 `json:"queue-items,omitempty"`
|
||||
// how many items were read
|
||||
Total int64 `json:"total"`
|
||||
}
|
||||
|
||||
// StepAccept is the body of step.
|
||||
type StepAccept = StepRun
|
||||
|
||||
@@ -368,6 +443,15 @@ func DecodeDropAnswer(raw []byte) (DropAnswer, int, error) {
|
||||
return v, formRead(raw), nil
|
||||
}
|
||||
|
||||
// DecodePublishQueueAnswer reads a PublishQueueAnswer as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
|
||||
func DecodePublishQueueAnswer(raw []byte) (PublishQueueAnswer, int, error) {
|
||||
var v PublishQueueAnswer
|
||||
if err := json.Unmarshal(raw, &v); err != nil {
|
||||
return v, 0, err
|
||||
}
|
||||
return v, formRead(raw), nil
|
||||
}
|
||||
|
||||
// DecodePackagePublishedEvent reads a PackagePublishedEvent as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
|
||||
func DecodePackagePublishedEvent(raw []byte) (PackagePublishedEvent, int, error) {
|
||||
var v PackagePublishedEvent
|
||||
|
||||
@@ -125,7 +125,7 @@
|
||||
},
|
||||
{
|
||||
"name": "drop",
|
||||
"description": "End an ask on the seat's work queue for a caller that no longer waits for it (its step ended by its bound): one waiting is ended dropped when it is pulled, one taken is cancelled; both end the step dropped, naming who dropped it and why, so no holder works on it after (design 47 H2 §778; ADR 0664 §4).",
|
||||
"description": "End an ask on the seat's work queue for a caller that no longer waits for it (its step ended by its bound): one waiting is ended dropped when it is pulled, one taken is cancelled; both end the step could-not-run, its why naming the drop and who asked it, so no holder works on it after (design 47 H2 §778; H-A-401; ADR 0664 §4).",
|
||||
"input": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
@@ -152,6 +152,150 @@
|
||||
"description": "whether the ask was held here"
|
||||
},
|
||||
"class": "act"
|
||||
},
|
||||
{
|
||||
"name": "publish-queue",
|
||||
"class": "read",
|
||||
"description": "What the builder's work queue publishes holds, read by the builder from its own stream (design 47 H2 §793; H-A-454): the items' count per registry seat holder and class with the oldest and newest queue time, and the first 50 in the order they are pulled (the live class first, each class oldest first); module, and version, list only that module's items. A queue read past 4,000 items is answered capped, naming the items read and the queue's size.",
|
||||
"input": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"module": {
|
||||
"type": "string",
|
||||
"description": "list only this module's items"
|
||||
},
|
||||
"version": {
|
||||
"type": "string",
|
||||
"description": "with module: only this version's, MAJOR.MINOR.PATCH"
|
||||
}
|
||||
},
|
||||
"required": []
|
||||
},
|
||||
"answer": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"total": {
|
||||
"type": "integer",
|
||||
"description": "how many items were read"
|
||||
},
|
||||
"counts": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"holder": {
|
||||
"type": "string",
|
||||
"description": "the registry seat's holder"
|
||||
},
|
||||
"class": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"live",
|
||||
"again"
|
||||
],
|
||||
"description": "the items' class"
|
||||
},
|
||||
"count": {
|
||||
"type": "integer",
|
||||
"description": "how many items of the class the holder's queue holds"
|
||||
},
|
||||
"oldest": {
|
||||
"type": "string",
|
||||
"format": "date-time",
|
||||
"description": "when the oldest of them was queued"
|
||||
},
|
||||
"newest": {
|
||||
"type": "string",
|
||||
"format": "date-time",
|
||||
"description": "when the newest of them was queued"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"holder",
|
||||
"class",
|
||||
"count",
|
||||
"oldest",
|
||||
"newest"
|
||||
]
|
||||
},
|
||||
"description": "per holder and class, sorted by holder, then class"
|
||||
},
|
||||
"queue": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"holder": {
|
||||
"type": "string",
|
||||
"description": "the registry seat's holder the item is sent to, a module"
|
||||
},
|
||||
"module": {
|
||||
"type": "string",
|
||||
"description": "the module whose version is published"
|
||||
},
|
||||
"version": {
|
||||
"type": "string",
|
||||
"description": "the reserved version, MAJOR.MINOR.PATCH"
|
||||
},
|
||||
"class": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"live",
|
||||
"again"
|
||||
],
|
||||
"description": "live, a build a pipeline's step waits on, pulled first; again, asked again at a start"
|
||||
},
|
||||
"artifacts": {
|
||||
"type": "object",
|
||||
"additionalProperties": {
|
||||
"type": "string"
|
||||
},
|
||||
"description": "each artifact reference, <module>:<artifact>, with the digest to publish"
|
||||
},
|
||||
"queued": {
|
||||
"type": "string",
|
||||
"format": "date-time",
|
||||
"description": "when the item was put on the work queue"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"holder",
|
||||
"module",
|
||||
"version",
|
||||
"class",
|
||||
"queued"
|
||||
]
|
||||
},
|
||||
"description": "the first 50 items listed, per holder in the order they are pulled"
|
||||
},
|
||||
"listed": {
|
||||
"type": "integer",
|
||||
"description": "how many of total the answer lists"
|
||||
},
|
||||
"capped": {
|
||||
"type": "boolean",
|
||||
"description": "true when the queue held more items than were read; every count is then of the items read"
|
||||
},
|
||||
"items-read": {
|
||||
"type": "integer",
|
||||
"description": "with capped: how many items were read"
|
||||
},
|
||||
"queue-items": {
|
||||
"type": "integer",
|
||||
"description": "with capped: how many items the queue holds, from its stream's state"
|
||||
},
|
||||
"queue-bytes": {
|
||||
"type": "integer",
|
||||
"description": "with capped: the queue's size in bytes"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"total",
|
||||
"counts",
|
||||
"queue",
|
||||
"listed"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"accepts": [
|
||||
|
||||
@@ -8,6 +8,7 @@ export declare const verbs: {
|
||||
readonly reserved: "reserved";
|
||||
readonly published: "published";
|
||||
readonly drop: "drop";
|
||||
readonly publishQueue: "publish-queue";
|
||||
};
|
||||
|
||||
export declare const accepts: {
|
||||
@@ -226,6 +227,64 @@ export interface DropInput {
|
||||
/** is what drop answers. */
|
||||
export type DropAnswer = Record<string, unknown>;
|
||||
|
||||
/** is what publish-queue takes. */
|
||||
export interface PublishQueueInput {
|
||||
/** list only this module's items */
|
||||
"module"?: string;
|
||||
/** with module: only this version's, MAJOR.MINOR.PATCH */
|
||||
"version"?: string;
|
||||
}
|
||||
|
||||
export interface PublishQueueAnswerCountsItem {
|
||||
/** the items' class */
|
||||
"class": "live" | "again";
|
||||
/** how many items of the class the holder's queue holds */
|
||||
"count": number;
|
||||
/** the registry seat's holder */
|
||||
"holder": string;
|
||||
/** when the newest of them was queued */
|
||||
"newest": string;
|
||||
/** when the oldest of them was queued */
|
||||
"oldest": string;
|
||||
}
|
||||
|
||||
export interface PublishQueueAnswerQueueItem {
|
||||
/** each artifact reference, <module>:<artifact>, with the digest to publish */
|
||||
"artifacts"?: Record<string, string>;
|
||||
/** live, a build a pipeline's step waits on, pulled first; again, asked again at a start */
|
||||
"class": "live" | "again";
|
||||
/** the registry seat's holder the item is sent to, a module */
|
||||
"holder": string;
|
||||
/** the module whose version is published */
|
||||
"module": string;
|
||||
/** when the item was put on the work queue */
|
||||
"queued": string;
|
||||
/** the reserved version, MAJOR.MINOR.PATCH */
|
||||
"version": string;
|
||||
}
|
||||
|
||||
/** is what publish-queue answers. */
|
||||
export interface PublishQueueAnswer {
|
||||
/** true when the queue held more items than were read; every count is then of the items read */
|
||||
"capped"?: boolean;
|
||||
/** per holder and class, sorted by holder, then class */
|
||||
"counts": PublishQueueAnswerCountsItem[];
|
||||
/** is the form of what this value carries, left out while it is 1 (design 47 H2 §780). */
|
||||
"form"?: number;
|
||||
/** with capped: how many items were read */
|
||||
"items-read"?: number;
|
||||
/** how many of total the answer lists */
|
||||
"listed": number;
|
||||
/** the first 50 items listed, per holder in the order they are pulled */
|
||||
"queue": PublishQueueAnswerQueueItem[];
|
||||
/** with capped: the queue's size in bytes */
|
||||
"queue-bytes"?: number;
|
||||
/** with capped: how many items the queue holds, from its stream's state */
|
||||
"queue-items"?: number;
|
||||
/** how many items were read */
|
||||
"total": number;
|
||||
}
|
||||
|
||||
/** is the body of step. */
|
||||
export type StepAccept = StepRun;
|
||||
|
||||
|
||||
@@ -8,6 +8,7 @@ export const verbs = {
|
||||
reserved: "reserved",
|
||||
published: "published",
|
||||
drop: "drop",
|
||||
publishQueue: "publish-queue",
|
||||
};
|
||||
|
||||
export const accepts = {
|
||||
|
||||
Reference in New Issue
Block a user