Merge pull request 'Design 47 H2: the delivery parts' classes and caps, the builder's publish queue, the indexer's seat list, the runner and the forge bridge' (#322) from feat/47-H2 into main

This commit was merged in pull request #322.
This commit is contained in:
2026-11-22 04:56:12 +00:00
758 changed files with 30158 additions and 21690 deletions
@@ -0,0 +1,162 @@
package main
import (
"encoding/json"
"errors"
"fmt"
"slices"
"strings"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/process"
)
// The seat mesh-wants' work queue read and steered by hand (design 47 H2 §777, the tools table; H-A-410): assigner_asks
// answers what waits on the seat's queue (read by the runtime without taking it) and what the assigner took and keeps
// dropped in its state taken; assigner_ask_drop ends one ask through the seat's act drop, written in the hand-act log first.
const (
asksTool = "assigner_asks"
askDropTool = "assigner_ask_drop"
)
// asksAnswerMost is the most an asks answer carries of the taken keys (§811: at most 768 KB with capped and total).
const asksAnswerMost = 768 << 10
// takenReader is what assigner_asks reads of the state taken (stdio.KeptState).
type takenReader interface {
Keys() ([]string, error)
Get(key string) (*stdio.StateEntry, error)
}
// asksOf is assigner_asks' answer: the asks on the seat's queue, each waiting, waiting again or held, the ids taken (key mesh-wants.<ask>) and the drops of
// asks not pulled yet (dropped.<ask>), each list cut at the answer's bound with capped and total.
func asksOf(waiting func(seat string) (stdio.Waiting, error), taken takenReader) (map[string]any, error) {
w, err := waiting(meshwants.Seat)
if err != nil {
return nil, fmt.Errorf("the asks waiting on the seat %s's work queue are not read: %w", meshwants.Seat, err)
}
keys, err := taken.Keys()
if err != nil {
return nil, fmt.Errorf("the state %s is not listed: %w", process.TakenState, err)
}
slices.Sort(keys)
// An ask the queue handed out and not acknowledged (again) is held when this holder took it, else it waits again after a
// nak (H-A-410's addenda): the bus says what it knows, the state taken the rest.
size := 0
waits := make([]map[string]any, 0, len(w.Asks))
for _, a := range w.Asks {
size += len(a.ID) + len(a.Module) + 64
state := "waiting"
if a.Again {
state = "waiting again"
if slices.Contains(keys, process.TakenKey(meshwants.Seat, a.ID)) {
state = "held"
}
}
waits = append(waits, map[string]any{"id": a.ID, "module": a.Module, "age_seconds": a.AgeSeconds, "state": state})
}
takenIDs, dropped := []string{}, []map[string]any{}
total, capped := 0, w.Capped
for _, k := range keys {
if id, ok := strings.CutPrefix(k, process.TakenKey(meshwants.Seat, "")); ok {
total++
if size += len(id) + 4; size > asksAnswerMost {
capped = true
continue
}
takenIDs = append(takenIDs, id)
continue
}
id, ok := strings.CutPrefix(k, droppedPrefix)
if !ok {
continue
}
total++
e, err := taken.Get(k)
if err != nil || e == nil {
continue
}
var d askDrop
if json.Unmarshal(e.Value, &d) != nil {
continue
}
if size += len(id) + len(d.By) + len(d.Why) + 64; size > asksAnswerMost {
capped = true
continue
}
dropped = append(dropped, map[string]any{"id": id, "by": d.By, "why": d.Why, "at": d.At})
}
return map[string]any{"waiting": waits, "taken": takenIDs, "dropped": dropped, "capped": capped, "total": total + len(w.Asks)}, nil
}
// asksTools are the module's tools of the seat's work queue.
func asksTools(waiting func(string) (stdio.Waiting, error), taken takenReader, ask func(string, any) (json.RawMessage, error)) []stdio.Tool {
return []stdio.Tool{
{Name: asksTool, Description: "The asks of the seat mesh-wants' work queue (design 47 H2 §777): each one on it (its id, the module " +
"that asked it, its age and whether it waits, waits again after a hand-back or is held here), the ids the assigner took (kept for ask-taken-bound) and the drops of asks not pulled yet, " +
"cut at 768 KB with capped and total. Read-only.",
Input: schema(map[string]string{}, nil),
Run: func(map[string]any) (any, error) {
return asksOf(waiting, taken)
}},
{Name: askDropTool, Description: "End one ask of the seat mesh-wants by hand (design 47 H2 §777, §778): written in the hand-act log " +
"first, then the seat's act drop ends it: one running is cancelled, one waiting is ended when it is pulled; either ends " +
"its step could not run naming the drop and who asked it.",
Input: schema(map[string]string{"ask": "the ask's id", "why": "why the ask is ended"}, []string{"ask", "why"}),
Run: func(args map[string]any) (any, error) {
id, why := strings.TrimSpace(arg(args, "ask")), strings.TrimSpace(arg(args, "why"))
if id == "" || why == "" {
return nil, errors.New("name the ask and why it is ended")
}
by := stdio.CallerOf(args)
taken, err := meshhandactsclient.Recorder{Ask: ask}.Record(meshhandactsclient.Request{What: askDropTool + " " + id, Why: why + " (" + by + ")", Cause: "ask-drop"})
if err != nil {
return nil, fmt.Errorf("the hand act was not recorded, so nothing was dropped: %w", err)
}
raw, err := ask("seat:"+meshwants.Seat+"."+meshwants.VerbDrop, meshwants.DropInput{ID: id, By: askDropTool + " (" + by + ")", Why: why})
if err != nil {
return nil, err
}
var out map[string]any
if err := json.Unmarshal(raw, &out); err != nil {
return nil, fmt.Errorf("%s answered what is not JSON: %w", meshwants.VerbDrop, err)
}
out["hand-act"] = taken.Log
return out, nil
}},
}
}
// keepNowTool is the tool assigner_keep_now (design 47 H2 §813 (2); H-A-417): written in the hand-act log first, then the seat
// mesh-wants' act keep-now runs the keeping of the assignment graph's revisions at once in the assigner's process.
const keepNowTool = "assigner_keep_now"
func keepNowTools(ask func(string, any) (json.RawMessage, error)) []stdio.Tool {
return []stdio.Tool{{Name: keepNowTool, Description: "Keep the assignment graph's revisions now (design 47 H2 §813 (2)): written in the " +
"hand-act log first, then the assigner keeps the pass's newest computed revision at once and answers the revision kept, or " +
"the fault the keeping met (a failed keeping runs again after keeping-retry; keeping-failed is raised at the third in a row).",
Input: schema(map[string]string{"why": "why the keeping runs now"}, []string{"why"}),
Run: func(args map[string]any) (any, error) {
why := strings.TrimSpace(arg(args, "why"))
if why == "" {
return nil, errors.New("say why the keeping runs now")
}
taken, err := meshhandactsclient.Recorder{Ask: ask}.Record(meshhandactsclient.Request{What: keepNowTool, Why: why + " (" + stdio.CallerOf(args) + ")", Cause: "keep-now"})
if err != nil {
return nil, fmt.Errorf("the hand act was not recorded, so nothing was kept: %w", err)
}
raw, err := ask("seat:"+meshwants.Seat+"."+meshwants.VerbKeepNow, meshwants.KeepNowInput{Why: why})
if err != nil {
return nil, err
}
var out map[string]any
if err := json.Unmarshal(raw, &out); err != nil {
return nil, fmt.Errorf("%s answered what is not JSON: %w", meshwants.VerbKeepNow, err)
}
out["hand-act"] = taken.Log
return out, nil
}}}
}
@@ -68,7 +68,7 @@ func TestTheAssignersAsksAreTheSeatMeshWantsAsks(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if v, err := meshwants.DecodeAsksByHoldModuleFingerprintValue(raw); err != nil || v.Asking != 2 {
if v, _, err := meshwants.DecodeAsksByHoldModuleFingerprintValue(raw); err != nil || v.Asking != 2 {
t.Errorf("the ask %s is not the seat's Ask: %+v, %v", raw, v, err)
}
}
@@ -15,6 +15,8 @@ import (
nec "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
ne "git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/process"
)
// The inputs of a composition run (design 47 G §354, §718, A-243, A-252): one request per candidate and machine, carrying that
@@ -72,28 +74,44 @@ type nodeInput struct {
// compositionInputs asks the machines of a step for their inputs.
type compositionInputs struct {
machines func(module, branch string) []string
// callFile asks a seat verb marked chunks, addressed to one node as `seat:<seat>.<verb>@<node>`, and answers the verb's answer with
// the whole file received added as its member `file`, base64 (process.Conn.CallFile; design 47 G A-237b, A-237c, A-249). The call
// token is the body's `call`.
callFile func(key string, body any) (json.RawMessage, error)
bound time.Duration
// callFile asks a seat verb marked chunks as a kept call (process.Conn.CallFileKept; design 47 H2 §775, H-A-405), addressed to
// one node as `seat:<seat>.<verb>@<node>`, carrying the ask's id and stopping at its bound, and answers the verb's answer with
// the whole file received added as its member `file`, base64 (design 47 G A-237b, A-237c, A-249). The call token is the
// body's `call`.
callFile func(ask process.KeptAsk, key string, body any) (json.RawMessage, error)
// bound is compose-ask-bound: how long a machine's compose-candidate is asked again through a transport fault (§775).
bound time.Duration
}
// of asks the machine node for its input with the candidate's key (§718): asked only by the composition step that starts the
// machine's composition run, once per candidate and machine. A node that is silent, unready or answers no input has Why set.
func (c *compositionInputs) of(ctx context.Context, node, candidate string) nodeInput {
// composeAskBoundSetting is the holder's bound on its compose-candidate asks (design 47 H2 §775): past it the machine gave no
// input, naming every call.
const (
composeAskBoundSetting = "compose-ask-bound"
defaultComposeAskBound = 10 * time.Second
)
// composeAskBound reads the setting compose-ask-bound (§781).
func composeAskBound() (time.Duration, error) {
d, err := stdio.SettingDuration(composeAskBoundSetting, defaultComposeAskBound)
if err != nil {
return 0, errors.New("the assigner's " + err.Error())
}
return d, nil
}
// composeAskID is the id of the compose-candidate ask of the step asked under stepAsk to node (H-A-405 (2)): one per node per
// step, so the engines' mesh-wide state taken keeps each apart, and a repeat to the same node carries the same id.
func composeAskID(stepAsk, node string) string { return stepAsk + "." + node }
// of asks the machine node for its input with the candidate's key (§718), as a kept call under the id of stepAsk, the step
// that starts the machine's composition run: asked once per candidate and machine, again only through a transport fault until
// compose-ask-bound (§775). A node that is silent past the bound, unready or answers no input has Why set.
func (c *compositionInputs) of(ctx context.Context, node, candidate, stepAsk string) nodeInput {
bound := c.bound
if bound <= 0 {
bound = nec.AnswerBound
bound = defaultComposeAskBound
}
return c.one(ctx, bound, node, candidate)
}
// one asks one node for its input.
func (c *compositionInputs) one(ctx context.Context, bound time.Duration, node, key string) nodeInput {
in := nodeInput{Node: node}
at, cancel := context.WithTimeout(ctx, bound)
defer cancel()
type reply struct {
raw json.RawMessage
err error
@@ -106,13 +124,14 @@ func (c *compositionInputs) one(ctx context.Context, bound time.Duration, node,
got <- reply{nil, err}
return
}
raw, err := c.callFile(composeCandidateKey+"@"+node, ne.ComposeCandidateInput{Candidate: key, Input: true, Call: hex.EncodeToString(bits[:])})
raw, err := c.callFile(process.KeptAsk{ID: composeAskID(stepAsk, node), Bound: bound}, composeCandidateKey+"@"+node,
ne.ComposeCandidateInput{Candidate: candidate, Input: true, Call: hex.EncodeToString(bits[:])})
got <- reply{raw, err}
}()
var raw json.RawMessage
select {
case <-at.Done():
in.Why = "no answer within its bound: " + at.Err().Error()
case <-ctx.Done():
in.Why = "its step's wait ended: " + context.Cause(ctx).Error()
return in
case r := <-got:
if r.err != nil {
@@ -5,6 +5,7 @@ import (
"encoding/base64"
"encoding/json"
"errors"
"git.novox.be/novox/mesh-sdk/go/process"
"regexp"
"strings"
"sync"
@@ -71,11 +72,12 @@ func TestAStepsInputsAreAskedOfEachMachine(t *testing.T) {
// the machines are asked side by side, so the record of what each was asked is guarded.
var mu sync.Mutex
asked := map[string]string{}
call := func(key string, body any) (json.RawMessage, error) {
asks := map[string]process.KeptAsk{}
call := func(ask process.KeptAsk, key string, body any) (json.RawMessage, error) {
raw, _ := json.Marshal(body)
key, node, _ := strings.Cut(key, "@") // addressed to one node (A-237b)
mu.Lock()
asked[node] = key + " " + string(raw)
asked[node], asks[node] = key+" "+string(raw), ask
mu.Unlock()
switch node {
case "g14":
@@ -98,6 +100,10 @@ func TestAStepsInputsAreAskedOfEachMachine(t *testing.T) {
if got[0].Node != "ace" || got[0].Why != "" || !strings.Contains(string(got[0].Body), "12345678901234567890") {
t.Errorf("ace: %+v", got[0])
}
// Design 47 H2 §775, H-A-405: each machine is asked as a kept call under the step's ask with the node, until compose-ask-bound.
if asks["ace"] != (process.KeptAsk{ID: "step-1.ace", Bound: time.Second}) || asks["g14"].ID != "step-1.g14" {
t.Errorf("the kept asks %+v", asks)
}
if !strings.Contains(got[1].Why, "no responders") || !strings.Contains(got[2].Why, "unready") || got[1].Body != nil || got[2].Body != nil {
t.Errorf("g14 and novox: %+v %+v", got[1], got[2])
}
@@ -115,7 +121,7 @@ func TestAStepsInputsAreAskedOfEachMachine(t *testing.T) {
func TestAStepsInputIsTheFileOfTheNodesChunks(t *testing.T) {
input := `{"n":12345678901234567890,"big":"` + strings.Repeat("x", 1<<20) + `"}`
file := base64.StdEncoding.EncodeToString([]byte(input))
call := func(key string, body any) (json.RawMessage, error) {
call := func(_ process.KeptAsk, key string, body any) (json.RawMessage, error) {
_, node, _ := strings.Cut(key, "@")
answer := `{"candidate":"run-r1","node":"` + node + `","verdict":"composed","modules":[],"own":{},"took":1,"v":1,"chunks":3`
if node == "g14" {
@@ -155,7 +161,7 @@ func gatherAll(c *compositionInputs, run meshwants.StepRun, away func(string) bo
continue
}
wg.Add(1)
go func() { defer wg.Done(); out[i] = c.of(context.Background(), node, key) }()
go func() { defer wg.Done(); out[i] = c.of(context.Background(), node, key, "step-1") }()
}
wg.Wait()
return out, nil
@@ -22,6 +22,7 @@ import (
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer"
"git.novox.be/novox/mesh-sdk/go/keys"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-assigner/internal/assign"
)
@@ -55,6 +56,9 @@ type compositionRun struct {
keeper *inputKeeper
// call asks a seat verb held once for the mesh (process.Conn.CallKept).
call func(key string, body any) (json.RawMessage, error)
// callFile asks a seat verb held once for the mesh whose answer comes in chunks, as a kept call (process.Conn.CallFileKept),
// answering the verb's answer with the bytes received as its member `file`, base64: the validate step's seat-list (H-A-443).
callFile func(ask process.KeptAsk, key string, body any) (json.RawMessage, error)
// trunk is the trunk's catalog half as the assigner holds it.
trunk func() (mcc.Half, bool)
// newest is the newest catalog half the assigner's views hold of each watched branch (halves.views), and watched a module's
@@ -126,7 +130,7 @@ func (r *compositionRun) run(ctx context.Context, id string, step meshwants.Step
continue
}
if j.started {
go r.start(ctx, j.key, j.run, prep)
go r.start(ctx, j.key, j.run, prep, id)
}
keys = append(keys, j.key)
}
@@ -241,9 +245,9 @@ func (p *runPrep) get(ctx context.Context) error {
return p.err
}
// start composes the machine of the run it started (§718): the node's input of this moment, its request kept once in `inputs`,
// start composes the machine of the run the step asked under stepAsk started (§718): the node's input of this moment, its request kept once in `inputs`,
// the composer program chosen and recorded with the request's digest, and one run ask put. A fault ends the run could not run.
func (r *compositionRun) start(ctx context.Context, key string, run meshwants.CompositionRun, prep *runPrep) {
func (r *compositionRun) start(ctx context.Context, key string, run meshwants.CompositionRun, prep *runPrep, stepAsk string) {
node, ask := run.Machine, run.Ask
end := func(said string) {
if _, err := r.runs.endRun(key, ask, meshwants.CompositionRunOutcomeCouldNotRun, said, nil); err != nil {
@@ -269,7 +273,7 @@ func (r *compositionRun) start(ctx context.Context, key string, run meshwants.Co
end(node + ": " + err.Error())
return
}
in := r.inputs.of(ctx, node, run.Candidate)
in := r.inputs.of(ctx, node, run.Candidate, stepAsk) // a kept call under the starting step's ask with the node (H-A-405)
if in.Why != "" {
end(node + " gave no composition input: " + in.Why)
return
@@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"errors"
"git.novox.be/novox/mesh-sdk/go/process"
"maps"
"slices"
"strings"
@@ -103,7 +104,7 @@ func newRunStand(t *testing.T, revisionEngine, trunkEngine, seatDigest string) *
}
return nil, errors.New("no such verb " + key)
}
engines := func(key string, _ any) (json.RawMessage, error) {
engines := func(_ process.KeptAsk, key string, _ any) (json.RawMessage, error) {
_, node, _ := strings.Cut(key, "@") // addressed to one node (A-237b)
return json.RawMessage(`{"candidate":"main.7","node":"` + node + `","verdict":"composed","input":{"node":"` + node + `"},"modules":[],"own":{},"took":1,"v":1}`), nil
}
@@ -638,7 +639,7 @@ func TestARerunMovesEveryReaderThatDidNotPass(t *testing.T) {
}
got := map[string]meshwants.StepEnded{}
for _, m := range []string{"a", "b", "d"} {
steps.take(context.Background(), "ask-"+m, step(m, false))
pullOne(steps, "ask-"+m, step(m, false))
e := <-ends
got[e.Module] = e
}
@@ -649,7 +650,7 @@ func TestARerunMovesEveryReaderThatDidNotPass(t *testing.T) {
mu.Lock()
fixed = true
mu.Unlock()
steps.take(context.Background(), "ask-a2", step("a", true))
pullOne(steps, "ask-a2", step("a", true))
moved := map[string]meshwants.StepEnded{}
for range 2 {
select {
@@ -126,7 +126,7 @@ func TestOneCompositionRunPerCandidateAndMachine(t *testing.T) {
from := b.journal.mark()
_, stop := b.startStoppable(t)
b.journal.await(t, from, "the composition step is heard on", "the composition step served", 30*time.Second)
b.journal.await(t, from, "the composition step is pulled from", "the composition step served", 30*time.Second)
select {
case <-b.seatListRead:
case <-time.After(30 * time.Second):
@@ -2,14 +2,11 @@ package main
import (
"context"
"encoding/json"
"errors"
"sync"
"time"
"github.com/nats-io/nats.go"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-sdk/go/process"
)
// The composition step (design 47 G §22, §354, §63): the assigner takes the seat mesh-wants' accepted kind `step`, the
@@ -44,9 +41,18 @@ type compositionSteps struct {
logf func(string, ...any)
// ticks is a ticker of every d, its channel and its stop (time.NewTicker); nil is the clock's. A test hands its own.
ticks func(d time.Duration) (<-chan time.Time, func())
mu sync.Mutex
// stops ends the wait of a running step by its ask's id, when its pipeline is superseded or cancelled (leave).
stops map[string]context.CancelFunc
// taken is the state taken (stepqueue.go): each ask pulled is taken once by its id, for takenBound; the drops of asks not
// pulled yet are kept there too (H-A-404).
taken takenStore
takenBound time.Duration
// inProgress is how often a running step's pulled ask is said in progress: the seat definition's in-progress (§777).
inProgress func() time.Duration
mu sync.Mutex
// stops ends the wait of a running step by its ask's id, when its pipeline is superseded or cancelled (leave), or the ask is
// dropped (the cause an *askDrop).
stops map[string]context.CancelCauseFunc
// held are the messages pulled from the seat's work queue for each running step's ask, acknowledged once its end is said.
held map[string][]queueMsg
}
// progressEvery is how often a running composition step says its progress (design 47 G §496, A-262).
@@ -92,20 +98,6 @@ func (p *stepPhase) get() string {
return p.phase
}
// take takes one ask of the accepted kind `step` by its id and body: a step that is no composition or validate step, or an
// ask with no id, is left with a log line; one taken is run in its own goroutine and its end said.
func (c *compositionSteps) take(ctx context.Context, id string, data []byte) bool {
var run meshwants.StepRun
err := json.Unmarshal(data, &run)
how := c.howOf(run.Step, false)
if err != nil || id == "" || how == nil {
c.logf("a step ask that is no composition or validate step with an id is left: %v", err)
return false
}
go c.say1(ctx, id, run, how, "")
return true
}
// howOf is what runs a step, or judges it once taken up at the start: by its step, composition or validate; nil for another.
func (c *compositionSteps) howOf(step string, resumed bool) func(context.Context, string, meshwants.StepRun) stepOutcome {
switch {
@@ -124,16 +116,16 @@ func (c *compositionSteps) howOf(step string, resumed bool) func(context.Context
// say1 runs one step with how and says its end, naming in `shared` the runs it read (§723); again is, for an end said again, the
// ask of the run whose end makes it say again (sayEnd). A step whose wait was ended by its pipeline's supersede or cancel (leave)
// says nothing: the runner ended it already.
func (c *compositionSteps) say1(ctx context.Context, id string, run meshwants.StepRun, how func(context.Context, string, meshwants.StepRun) stepOutcome, again string) {
func (c *compositionSteps) say1(parent context.Context, id string, run meshwants.StepRun, how func(context.Context, string, meshwants.StepRun) stepOutcome, again string) {
started := c.now().UTC()
ctx, stopWait := context.WithCancel(ctx)
ctx, stopWait := context.WithCancelCause(parent)
c.mu.Lock()
if c.stops == nil {
c.stops = map[string]context.CancelFunc{}
c.stops = map[string]context.CancelCauseFunc{}
}
if c.stops[id] != nil { // the step asked under id is running already: it says its own end
c.mu.Unlock()
stopWait()
stopWait(nil)
return
}
c.stops[id] = stopWait
@@ -142,14 +134,25 @@ func (c *compositionSteps) say1(ctx context.Context, id string, run meshwants.St
c.mu.Lock()
delete(c.stops, id)
c.mu.Unlock()
stopWait()
stopWait(nil)
}()
phase := &stepPhase{phase: meshwants.StepProgressEventPhaseFetch}
stop := c.sayProgress(id, run.Asker, phase)
stopInProgress := c.inProgressEvery(id)
o := how(context.WithValue(ctx, phaseKey{}, phase), id, run)
stopInProgress()
stop()
if ctx.Err() != nil {
if parent.Err() != nil { // the assigner stops: its pulled asks are handed again to its next start
return
}
var d *askDrop
if errors.As(context.Cause(ctx), &d) { // dropped through the seat's drop (§778, H-A-401)
c.sayEnd(id, run, &started, stepOutcome{Outcome: meshwants.StepEndedOutcomeCouldNotRun, Said: d.Error(), Shared: o.Shared}, again)
return
}
c.logf("the composition step %s of %s says no end: its wait ended (%v)", id, run.Pipeline, context.Cause(ctx))
c.settle(id, true) // the runner ended it already: its ask is over
return
}
c.sayEnd(id, run, &started, o, again)
@@ -175,9 +178,13 @@ func (c *compositionSteps) sayEnd(id string, run meshwants.StepRun, started *tim
msgID = id + "." + again
}
if err := c.say(meshwants.EventStepEnded+"."+c.node, end, msgID); err != nil {
c.logf("saying the end of the composition step %s: %v", id, err)
c.logf("saying the end of the composition step %s: %v; its ask is handed again", id, err)
c.settle(id, false)
return
}
c.settle(id, true) // the ask's end is said: its pulled messages leave the seat's work queue (§777)
// One journal line per ask answered, in the one place it is answered (design 47 H2 §792).
c.logf("answered %s %s %s %s %s", id, run.Pipeline, run.Module, step, o.Outcome)
if c.ended != nil {
c.ended(id, run, ended)
}
@@ -210,7 +217,7 @@ func (c *compositionSteps) leave(ids []string) {
defer c.mu.Unlock()
for _, id := range ids {
if stop := c.stops[id]; stop != nil {
stop()
stop(nil)
}
}
}
@@ -243,21 +250,3 @@ func (c *compositionSteps) sayProgress(id, asker string, phase *stepPhase) (stop
}()
return func() { close(done); <-ended }
}
// stepAskID is an ask's id as the asker names it: its event id, else its message id.
func stepAskID(m *nats.Msg) string {
if id := m.Header.Get("x-event-id"); id != "" {
return id
}
return m.Header.Get(nats.MsgIdHdr)
}
// serveCompositionSteps hears the composition step asks on the seat mesh-wants' accept subject.
func serveCompositionSteps(ctx context.Context, conn process.Subscriber, c *compositionSteps) error {
sub, err := conn.Subscribe(meshwants.AcceptSubjectStep, func(m *nats.Msg) { c.take(ctx, stepAskID(m), m.Data) })
if err != nil {
return err
}
go func() { <-ctx.Done(); _ = sub.Drain() }()
return nil
}
@@ -106,7 +106,7 @@ func TestTheCompositionStepComposesEachMachineThroughTheBuildSeat(t *testing.T)
from := b.journal.mark()
b.start(t)
b.journal.await(t, from, "the composition step is heard on", "the composition step served", 30*time.Second)
b.journal.await(t, from, "the composition step is pulled from", "the composition step served", 30*time.Second)
b.journal.await(t, from, "n3 is away (machine-away)", "the machine-away of n3 heard", 30*time.Second)
select {
@@ -114,14 +114,8 @@ func TestTheCompositionStepComposesEachMachineThroughTheBuildSeat(t *testing.T)
case <-time.After(30 * time.Second):
t.Fatal("the assigner did not read the seat list within 30 s")
}
step, _ := json.Marshal(meshwants.StepRun{Pipeline: "main.3.m", Step: meshwants.StepRunStepComposition, Module: "m", Branch: trunkBranch,
putStep(b.ctx, t, b.js, "step-ask-1", meshwants.StepRun{Pipeline: "main.3.m", Step: meshwants.StepRunStepComposition, Module: "m", Branch: trunkBranch,
Revision: 3, Commit: engineCommit, Fingerprint: "src2:m1"})
msg := nats.NewMsg(meshwants.AcceptSubjectStep)
msg.Header.Set("x-event-id", "step-ask-1")
msg.Data = step
if err := nc.PublishMsg(msg); err != nil {
t.Fatal(err)
}
// The stand-in Go build seat takes the run asks put under the assigner's name, reads each request by the verb input and says
// its end with the composer's verdict: n1 composes, n2 does not.
@@ -201,6 +195,7 @@ func TestTheCompositionStepComposesEachMachineThroughTheBuildSeat(t *testing.T)
t.Fatalf("the inputs stay kept past their step-ended: %v", keys)
}
}
queueEmpty(b.ctx, t, b.js) // the step's ask acknowledged at its end (design 47 H2 §777)
}
// readInput reads a run's input as a build seat does: the seat mesh-wants' verb input answers its digest and chunk count, then its
@@ -288,9 +283,37 @@ func TestTheValidateStepServedWholeRefusesAManifestProblemAndACycle(t *testing.T
}
t.Cleanup(func() { _ = ends.Unsubscribe() })
subscribed(t, nc)
// Design 47 H2 §777: a validate ask put while the assigner is stopped waits on the seat's work queue, and is taken (ask-taken)
// and ended after its start.
taken := make(chan *nats.Msg, 8)
takens, err := nc.ChanSubscribe(meshwants.SubjectAskTakenAll, taken)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = takens.Unsubscribe() })
subscribed(t, nc)
putStep(b.ctx, t, b.js, "validate-9", meshwants.StepRun{Pipeline: trunkBranch + ".9", Step: meshwants.StepRunStepValidate, Module: "a",
Branch: trunkBranch, Revision: 9, Commit: commit})
from := b.journal.mark()
b.start(t)
b.journal.await(t, from, "the composition step is heard on", "the steps served", 30*time.Second)
b.journal.await(t, from, "the composition step is pulled from", "the steps served", 30*time.Second)
select {
case m := <-taken:
var at meshwants.AskTakenEvent
if err := json.Unmarshal(m.Data, &at); err != nil || at.Ask != "validate-9" || at.Node != "n1" || at.At.IsZero() || m.Subject != meshwants.SubjectAskTaken("validate-9") {
t.Fatalf("ask-taken on %s = %+v, %v; want validate-9 taken on n1", m.Subject, at, err)
}
case <-time.After(30 * time.Second):
t.Fatal("no ask-taken within 30 s of the start for the ask put while the assigner was stopped")
}
select {
case e := <-ended:
if e.ID != "validate-9" || e.Outcome != meshwants.StepEndedOutcomeFailed || e.Said != reasonCycle+": a:x → b:y → a:x" {
t.Errorf("the validate step of a take-in making a cycle ended %+v", e)
}
case <-time.After(30 * time.Second):
t.Fatal("the validate ask put while the assigner was stopped said no end")
}
select {
case <-b.seatListRead:
case <-time.After(30 * time.Second):
@@ -306,12 +329,9 @@ func TestTheValidateStepServedWholeRefusesAManifestProblemAndACycle(t *testing.T
id := fmt.Sprintf("validate-%d", revision)
step, _ := json.Marshal(meshwants.StepRun{Pipeline: fmt.Sprintf("%s.%d", trunkBranch, revision), Step: meshwants.StepRunStepValidate, Module: module,
Branch: trunkBranch, Revision: revision, Commit: commit})
msg := nats.NewMsg(meshwants.AcceptSubjectStep)
msg.Header.Set("x-event-id", id)
msg.Data = step
if err := nc.PublishMsg(msg); err != nil {
t.Fatal(err)
}
var asked meshwants.StepRun
_ = json.Unmarshal(step, &asked)
putStep(b.ctx, t, b.js, id, asked)
if run != nil {
run()
}
@@ -342,6 +362,12 @@ func TestTheValidateStepServedWholeRefusesAManifestProblemAndACycle(t *testing.T
if err != nil || len(req.Cases) != 2 || req.Cases[0].Name != changeCase || req.Cases[1].Name != beforeCase {
t.Fatalf("the validator's request: %+v %v", req, err)
}
// The revision names no definitions: the request holds the trunk's seat list as seat-list handed it, named by its digest (H-A-443).
if list, digest := seatListBytes(); req.SeatList != digest {
t.Errorf("the validator's request names the seat list %q, want %q", req.SeatList, digest)
} else if got, _ := json.Marshal(req.Seats); string(got) != string(list) {
t.Errorf("the validator's request holds seats %d bytes long, not the seat list's %d", len(got), len(list))
}
v := composer.Verdict{V: composer.Version}
for _, c := range req.Cases {
reads := map[string]composer.ManifestRead{}
@@ -365,10 +391,8 @@ func TestTheValidateStepServedWholeRefusesAManifestProblemAndACycle(t *testing.T
if e := ask(8, "web", validator); e.Outcome != meshwants.StepEndedOutcomePassed {
t.Errorf("the validate step of a manifest with no problem ended %+v", e)
}
if e := ask(9, "a", nil); e.Outcome != meshwants.StepEndedOutcomeFailed || e.Said != reasonCycle+": a:x → b:y → a:x" {
t.Errorf("the validate step of a take-in making a cycle ended %+v", e)
}
if info, err := worker.Info(b.ctx); err != nil || info.NumPending != 0 {
t.Errorf("a validator is asked for the cycle: %+v %v", info, err)
}
queueEmpty(b.ctx, t, b.js) // every ask acknowledged at its end (§777)
}
@@ -3,15 +3,115 @@ package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"slices"
"strings"
"sync"
"testing"
"time"
"github.com/nats-io/nats.go"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
stdio "git.novox.be/novox/mesh-sdk/go"
)
// fakeMsg is one ask pulled from the seat's work queue, recording how it was settled.
type fakeMsg struct {
data []byte
h nats.Header
mu sync.Mutex
acks, naks, terms, inProgresss int
}
func (m *fakeMsg) Data() []byte { return m.data }
func (m *fakeMsg) Headers() nats.Header { return m.h }
func (m *fakeMsg) count(n *int) error { m.mu.Lock(); *n++; m.mu.Unlock(); return nil }
func (m *fakeMsg) Ack() error { return m.count(&m.acks) }
func (m *fakeMsg) Nak() error { return m.count(&m.naks) }
func (m *fakeMsg) Term() error { return m.count(&m.terms) }
func (m *fakeMsg) InProgress() error { return m.count(&m.inProgresss) }
func (m *fakeMsg) settled() (acks, naks, terms, progress int) {
m.mu.Lock()
defer m.mu.Unlock()
return m.acks, m.naks, m.terms, m.inProgresss
}
// fakeTaken is the state taken: its keys, each with its time to live, and a refusal of TakeOnce when refuse is set.
type fakeTaken struct {
mu sync.Mutex
values map[string][]byte
ttls map[string]time.Duration
refuse error
}
func newFakeTaken() *fakeTaken {
return &fakeTaken{values: map[string][]byte{}, ttls: map[string]time.Duration{}}
}
func (f *fakeTaken) TakeOnce(seat, ask string, bound time.Duration) (bool, error) {
f.mu.Lock()
defer f.mu.Unlock()
if f.refuse != nil {
return false, f.refuse
}
key := seat + "." + ask
if _, ok := f.values[key]; ok {
return false, nil
}
f.values[key], f.ttls[key] = []byte(`{}`), bound
return true, nil
}
func (f *fakeTaken) Get(key string) (*stdio.StateEntry, error) {
f.mu.Lock()
defer f.mu.Unlock()
v, ok := f.values[key]
if !ok {
return nil, nil
}
return &stdio.StateEntry{Key: key, Value: v}, nil
}
func (f *fakeTaken) RenewTTL(key string, value any, ttl time.Duration) (uint64, error) {
raw, err := json.Marshal(value)
f.mu.Lock()
defer f.mu.Unlock()
f.values[key], f.ttls[key] = raw, ttl
return 1, err
}
func (f *fakeTaken) Delete(key string) error {
f.mu.Lock()
defer f.mu.Unlock()
delete(f.values, key)
return nil
}
func (f *fakeTaken) Keys() ([]string, error) {
f.mu.Lock()
defer f.mu.Unlock()
var out []string
for k := range f.values {
out = append(out, k)
}
return out, nil
}
// pullOne hands c one ask pulled from the seat's work queue under id, with a state taken of its own when c has none.
func pullOne(c *compositionSteps, id string, body []byte) *fakeMsg {
if c.taken == nil {
c.taken = newFakeTaken()
}
m := &fakeMsg{data: body, h: nats.Header{}}
if id != "" {
m.h.Set("x-event-id", id)
}
c.pulled(context.Background(), m)
return m
}
// saidEvents keeps what a publisher says.
type saidEvents struct {
mu sync.Mutex
@@ -38,24 +138,34 @@ func TestTheCompositionStepIsTakenAndItsEndSaid(t *testing.T) {
said := newSaidEvents()
at := time.Date(2026, 11, 12, 9, 0, 0, 0, time.UTC)
var asked meshwants.StepRun
c := &compositionSteps{node: "n1", say: said.say, now: func() time.Time { return at }, logf: t.Logf,
var lines []string
var linesMu sync.Mutex
logf := func(f string, a ...any) {
linesMu.Lock()
lines = append(lines, fmt.Sprintf(f, a...))
linesMu.Unlock()
}
c := &compositionSteps{node: "n1", say: said.say, now: func() time.Time { return at }, logf: logf,
run: func(_ context.Context, id string, run meshwants.StepRun) stepOutcome {
asked = run
return stepOutcome{Outcome: meshwants.StepEndedOutcomeFailed, Said: "a and b own /etc/x", Warnings: []string{"n3 asleep"},
Shared: []meshwants.Shared{{Machine: "n1", Run: "main=2e4.n1", Ask: "run-1"}}}
}}
other, _ := json.Marshal(map[string]any{"pipeline": "main.4.m", "step": "build", "module": "m"})
if c.take(context.Background(), "ask-0", other) {
if m := pullOne(c, "ask-0", other); m.terms != 1 {
t.Fatal("a step ask of another step is taken")
}
body, _ := json.Marshal(meshwants.StepRun{Pipeline: "main.4.m", Step: meshwants.StepRunStepComposition, Module: "m", Branch: "main",
Commit: "c0ffee", Revision: 4, Fingerprint: "fp1", Rerun: true, By: "lead@n1"})
if c.take(context.Background(), "", body) {
if m := pullOne(c, "", body); m.terms != 1 {
t.Fatal("a step ask with no id is taken")
}
if !c.take(context.Background(), "ask-1", body) {
if m := pullOne(c, "ask-1", body); m.terms != 0 {
t.Fatal("the composition step is not taken")
}
if e := <-said.heard; e != meshwants.EventAskTaken+".ask-1" { // said at the pull (H2 §777)
t.Fatalf("said %s first, want ask-taken", e)
}
select {
case <-said.heard:
case <-time.After(5 * time.Second):
@@ -73,6 +183,211 @@ func TestTheCompositionStepIsTakenAndItsEndSaid(t *testing.T) {
len(end.Shared) != 1 || end.Shared[0] != (meshwants.Shared{Machine: "n1", Run: "main=2e4.n1", Ask: "run-1"}) { // §723, §729
t.Fatalf("step-ended %+v", end)
}
// Design 47 H2 §792: the ask answered is one journal line, said where its end is said.
linesMu.Lock()
answered := slices.DeleteFunc(slices.Clone(lines), func(l string) bool { return !strings.HasPrefix(l, "answered ") })
linesMu.Unlock()
if !slices.Equal(answered, []string{"answered ask-1 main.4.m m composition failed"}) {
t.Errorf("the journal says %q, want one answered line", answered)
}
// Design 47 H2 §777, §775 (H-A-381, H-A-401, H-A-404): the asks of the seat's work queue, taken once by id, said ask-taken at
// the pull, in progress while they run, acknowledged once their end is said, and ended by the seat's drop.
t.Run("work queue", queueCases)
}
func queueCases(t *testing.T) {
at := time.Date(2026, 11, 22, 1, 0, 0, 0, time.UTC)
body, _ := json.Marshal(meshwants.StepRun{Pipeline: "main.4.m", Step: meshwants.StepRunStepValidate, Module: "m", Branch: "main", Revision: 4})
type bed struct {
c *compositionSteps
said *saidEvents
taken *fakeTaken
runs chan string
release chan struct{}
tick chan time.Time
}
newBed := func() *bed {
b := &bed{said: newSaidEvents(), taken: newFakeTaken(), runs: make(chan string, 8), release: make(chan struct{}), tick: make(chan time.Time)}
b.said.heard = make(chan string, 16)
run := func(ctx context.Context, id string, _ meshwants.StepRun) stepOutcome {
b.runs <- id
select {
case <-b.release:
case <-ctx.Done():
}
return stepOutcome{Outcome: meshwants.StepEndedOutcomePassed}
}
b.c = &compositionSteps{node: "n1", say: b.said.say, now: func() time.Time { return at }, logf: t.Logf, taken: b.taken, takenBound: 10 * time.Minute,
validate: run, inProgress: func() time.Duration { return 20 * time.Second },
ticks: func(d time.Duration) (<-chan time.Time, func()) {
if d == 20*time.Second {
return b.tick, func() {}
}
return make(chan time.Time), func() {}
}}
return b
}
ended := func(t *testing.T, b *bed) meshwants.StepEnded {
t.Helper()
for {
select {
case e := <-b.said.heard:
if e == meshwants.EventStepEnded+".n1" {
b.said.mu.Lock()
defer b.said.mu.Unlock()
return b.said.said[e].(meshwants.StepEnded)
}
case <-time.After(5 * time.Second):
t.Fatal("no step-ended is said")
}
}
}
settledAt := func(t *testing.T, m *fakeMsg, acks int) {
t.Helper()
deadline := time.After(5 * time.Second)
for {
if a, _, _, _ := m.settled(); a == acks {
return
}
select {
case <-deadline:
t.Fatalf("the ask is acknowledged %d times, want %d", m.acks, acks)
case <-time.After(time.Millisecond):
}
}
}
t.Run("taken once, said, in progress, acknowledged at its end", func(t *testing.T) {
b := newBed()
m := pullOne(b.c, "ask-1", body)
if id := <-b.runs; id != "ask-1" {
t.Fatalf("ran %s", id)
}
b.said.mu.Lock()
taken, ok := b.said.said[meshwants.EventAskTaken+".ask-1"].(meshwants.AskTakenEvent)
takenID := b.said.ids[meshwants.EventAskTaken+".ask-1"]
b.said.mu.Unlock()
if !ok || taken.Ask != "ask-1" || taken.Node != "n1" || !taken.At.Equal(at) || takenID != "ask-1.taken" || b.taken.ttls["mesh-wants.ask-1"] != 10*time.Minute {
t.Fatalf("ask-taken %+v under %q, taken %v", taken, takenID, b.taken.ttls)
}
b.tick <- at
b.tick <- at
// handed again while it runs: held for the run, which answers both at its one end
again := pullOne(b.c, "ask-1", body)
if a, n, _, p := m.settled(); a != 0 || n != 0 || p < 1 {
t.Fatalf("before its end the ask is acked %d, nakked %d, in progress %d times", a, n, p)
}
close(b.release)
if e := ended(t, b); e.ID != "ask-1" || e.Outcome != meshwants.StepEndedOutcomePassed {
t.Fatalf("step-ended %+v", e)
}
settledAt(t, m, 1)
settledAt(t, again, 1)
select {
case id := <-b.runs:
t.Fatalf("the ask handed again ran again (%s)", id)
default:
}
})
t.Run("taken before is run again, said no ask-taken", func(t *testing.T) {
b := newBed()
_, _ = b.taken.TakeOnce(meshwants.Seat, "ask-2", time.Minute)
close(b.release)
m := pullOne(b.c, "ask-2", body)
<-b.runs
ended(t, b)
settledAt(t, m, 1)
b.said.mu.Lock()
_, said := b.said.said[meshwants.EventAskTaken+".ask-2"]
b.said.mu.Unlock()
if said {
t.Fatal("ask-taken is said again for an ask taken before")
}
})
t.Run("a refused take is handed again", func(t *testing.T) {
b := newBed()
b.taken.refuse = errors.New("no bus")
m := pullOne(b.c, "ask-3", body)
if _, n, _, _ := m.settled(); n != 1 || len(b.runs) != 0 {
t.Fatalf("nakked %d, ran %d", n, len(b.runs))
}
})
t.Run("dropped while it runs", func(t *testing.T) {
b := newBed()
m := pullOne(b.c, "ask-4", body)
<-b.runs
if _, err := b.c.dropAsk("n1.mesh-pipeline-runner", meshwants.DropInput{ID: "ask-4"}); err == nil {
t.Fatal("a drop with no why is taken")
}
a, err := b.c.dropAsk("n1.mesh-pipeline-runner", meshwants.DropInput{ID: "ask-4", Why: "its bound passed"})
if err != nil || a["held"] != true {
t.Fatalf("drop answered %v, %v", a, err)
}
e := ended(t, b)
if e.Outcome != meshwants.StepEndedOutcomeCouldNotRun || e.Said != "the ask was dropped by n1.mesh-pipeline-runner: its bound passed" {
t.Fatalf("step-ended %+v", e)
}
settledAt(t, m, 1)
})
t.Run("dropped before it was pulled", func(t *testing.T) {
b := newBed()
a, err := b.c.dropAsk("n1.mesh-pipeline-runner", meshwants.DropInput{ID: "ask-5", By: "the runner", Why: "its bound passed"})
if err != nil || a["held"] != false || b.taken.ttls["dropped.ask-5"] != 10*time.Minute {
t.Fatalf("drop answered %v, %v; taken %v", a, err, b.taken.ttls)
}
m := pullOne(b.c, "ask-5", body)
e := ended(t, b)
if e.ID != "ask-5" || e.Outcome != meshwants.StepEndedOutcomeCouldNotRun || e.Said != "the ask was dropped by the runner: its bound passed" {
t.Fatalf("step-ended %+v", e)
}
settledAt(t, m, 1)
if len(b.runs) != 0 {
t.Fatal("a dropped ask ran")
}
if v, _ := b.taken.Get("dropped.ask-5"); v != nil {
t.Fatal("the drop's key stays once its ask ended")
}
})
t.Run("the asks tools", func(t *testing.T) {
taken := newFakeTaken()
_, _ = taken.TakeOnce(meshwants.Seat, "ask-6", time.Minute)
_, _ = taken.RenewTTL("dropped.ask-7", askDrop{By: "the runner", Why: "its bound passed", At: at}, time.Minute)
waiting := func(seat string) (stdio.Waiting, error) {
if seat != meshwants.Seat {
t.Errorf("the waiting asks of %s are read", seat)
}
return stdio.Waiting{Asks: []stdio.WaitingAsk{{ID: "ask-8", Module: "mesh-delivery-pipeline", AgeSeconds: 3},
{ID: "ask-6", Module: "mesh-delivery-pipeline", Again: true}, {ID: "ask-9", Module: "mesh-delivery-pipeline", Again: true}}}, nil
}
var asked []string
ask := func(key string, body any) (json.RawMessage, error) {
raw, _ := json.Marshal(body)
asked = append(asked, key+" "+string(raw))
if strings.HasSuffix(key, "."+meshwants.VerbDrop) {
return json.RawMessage(`{"held":false}`), nil
}
return json.RawMessage(`{}`), nil
}
tools := map[string]stdio.Tool{}
for _, tool := range asksTools(waiting, taken, ask) {
tools[tool.Name] = tool
}
out, err := tools[asksTool].Run(map[string]any{})
a, _ := out.(map[string]any)
w, _ := a["waiting"].([]map[string]any)
d, _ := a["dropped"].([]map[string]any)
if err != nil || len(w) != 3 || w[0]["id"] != "ask-8" || w[0]["state"] != "waiting" || w[1]["state"] != "held" || w[2]["state"] != "waiting again" || !slices.Equal(a["taken"].([]string), []string{"ask-6"}) || len(d) != 1 ||
d[0]["id"] != "ask-7" || d[0]["by"] != "the runner" || a["total"] != 5 || a["capped"] != false {
t.Fatalf("assigner_asks answered %+v, %v", out, err)
}
if _, err := tools[askDropTool].Run(map[string]any{"ask": "ask-8"}); err == nil || len(asked) != 0 {
t.Fatalf("a drop with no why: %v, asked %v", err, asked)
}
out, err = tools[askDropTool].Run(map[string]any{"ask": "ask-8", "why": "stuck"})
if err != nil || len(asked) != 2 || !strings.HasPrefix(asked[0], "seat:mesh-hand-acts.hand-act ") || !strings.Contains(asked[0], askDropTool+" ask-8") ||
!strings.HasPrefix(asked[1], "seat:mesh-wants.drop ") || !strings.Contains(asked[1], `"id":"ask-8"`) || !strings.Contains(asked[1], `"why":"stuck"`) {
t.Fatalf("assigner_ask_drop answered %v, %v; asked %v", out, err, asked)
}
})
}
// Design 47 G §496, A-262: a running composition step says step-progress.<node> on every tick of its ticker, with the ask, its
@@ -98,9 +413,10 @@ func TestARunningCompositionStepSaysItsProgress(t *testing.T) {
}}
// A11/AC1 (design 47 H §752, H-A-88): with no callers in mesh-wants, the progress names the asker the step run carries.
body, _ := json.Marshal(meshwants.StepRun{Pipeline: "main.4", Step: meshwants.StepRunStepComposition, Module: "m", Branch: "main", Asker: "mesh-pipeline-runner"})
if !c.take(context.Background(), "ask-1", body) {
if m := pullOne(c, "ask-1", body); m.terms != 0 {
t.Fatal("the composition step is not taken")
}
<-said.heard // ask-taken, at the pull (H2 §777)
progress := meshwants.EventStepProgress + ".n1"
for range 2 {
tick <- at
@@ -149,7 +465,7 @@ func TestASupersededStepLeavesItsRuns(t *testing.T) {
}}
body := func(s meshwants.StepRun) []byte { raw, _ := json.Marshal(s); return raw }
for _, p := range []string{"main.3.a", "main.3.b"} {
if !c.take(context.Background(), "ask-"+p, body(step(p))) {
if m := pullOne(c, "ask-"+p, body(step(p))); m.terms != 0 {
t.Fatal("not taken")
}
}
@@ -5,18 +5,25 @@ import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"slices"
"strings"
"sync"
"time"
mcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
mcatc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/keys"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/views"
"mesh-delivery-assigner/internal/assign"
"mesh-delivery-assigner/internal/assigngraph"
"mesh-delivery-assigner/internal/assignment"
)
// graphKeeper is the pass's assignment graph (design 47 G §292, A-67): per catalog revision taken in it computes the revision
@@ -28,7 +35,7 @@ type graphKeeper struct {
// reads is the same store through a handle of the pass's own, for the pass's read of its fill (graphfill.go); nil reads objects.
reads assigngraph.Objects
// halves is every followed branch's catalog graph revision held now.
halves func() map[string]mcc.Half
halves func() map[string]mcatc.Half
// mu guards what the keeping writes and the pass reads: the revision kept, full, and the need handed over (design 47 H-A-259).
mu sync.Mutex
// handedRev is the revision last handed to the keeping, so a pass whose inputs did not change hands nothing; handed is false
@@ -46,14 +53,40 @@ type graphKeeper struct {
full, started bool
// revisions removes a revision that ceased to be current (revisions.go, H-A-86); nil keeps every revision.
revisions *revisionRemover
// computed is the revision computed by the last pass, with its entries: its edges.
computed assigngraph.Computed
// computed is the revision computed by the last pass, with its entries: its edges; hasComputed says a pass computed one.
computed assigngraph.Computed
hasComputed bool
// keepMu runs one keeping at a time, the pass's (keepOne) or one by hand (keepNow).
keepMu sync.Mutex
// The keeping off the pass loop (H-A-259): need is what waits to be kept, keeping says one runs; background runs it (nil: at
// once, in the caller), and wake makes a pass once a keeping changed what the pass reads.
need *keepNeed
keeping bool
background func(func())
wake func()
// A keeping that fails runs again after retry, keeping-retry, by after (time.AfterFunc; a test hands its own); failures
// counts the failures in a row, and keeping-failed is raised at the third (design 47 H2 §813 (2)).
retry time.Duration
after func(d time.Duration, f func())
failures int
}
// The keeping's setting and finding (design 47 H2 §813 (2); ADR 0682 §11: the holder's own deadline).
const (
keepingRetrySetting = "keeping-retry"
defaultKeepingRetry = 30 * time.Second
kindKeepingFailed = "keeping-failed"
// keepingFailedAt is the failures in a row that raise keeping-failed.
keepingFailedAt = 3
)
// keepingRetry reads the setting keeping-retry (§781).
func keepingRetry() (time.Duration, error) {
d, err := stdio.SettingDuration(keepingRetrySetting, defaultKeepingRetry)
if err != nil {
return 0, errors.New("the assigner's " + err.Error())
}
return d, nil
}
// keepNeed is one need of the keeping: a computed revision to keep (nil: none), and whether the revisions not current are judged.
@@ -99,7 +132,9 @@ func (p *passer) keepGraph(ctx context.Context) string {
return ""
}
c := assigngraph.Compute(p.graphInputs())
g.computed = c
g.mu.Lock()
g.computed, g.hasComputed = c, true
g.mu.Unlock()
for _, u := range c.Uncomputed {
if g.said != u {
g.said = u
@@ -177,6 +212,8 @@ func (p *passer) needKeeping(ctx context.Context, n keepNeed) {
// revisions that ceased to be current; it says its duration in the journal, and wakes a pass when it changed what the pass reads.
func (p *passer) keepOne(ctx context.Context, n keepNeed) {
g := p.graph
g.keepMu.Lock()
defer g.keepMu.Unlock()
start := time.Now()
changed := false
defer func() {
@@ -199,6 +236,12 @@ func (p *passer) keepOne(ctx context.Context, n keepNeed) {
// keepRevision keeps c when the store holds it, and says whether a revision was kept now.
func (p *passer) keepRevision(ctx context.Context, c assigngraph.Computed) bool {
fresh, _ := p.keepRevisionOf(ctx, c)
return fresh
}
// keepRevisionOf is keepRevision answering the fault the keeping met too.
func (p *passer) keepRevisionOf(ctx context.Context, c assigngraph.Computed) (bool, error) {
g := p.graph
again := func() { // the next pass hands this revision again
g.mu.Lock()
@@ -208,10 +251,11 @@ func (p *passer) keepRevision(ctx context.Context, c assigngraph.Computed) bool
used, _, _ := g.fillOf(ctx)
rev, fresh, err := assigngraph.Keep(ctx, g.objects, c, meshwants.EventAssignmentRevision, p.clock())
if err != nil {
p.logf("the assignment graph revision is not kept, so no want is said as wanted until it is: %v", err)
again()
return false
p.keepingFailed(ctx, c, err)
return false, err
}
p.keepingSucceeded()
var source int64
for _, cat := range rev.Catalogs {
if cat.Branch == p.w.trunkName() {
@@ -222,14 +266,54 @@ func (p *passer) keepRevision(ctx context.Context, c assigngraph.Computed) bool
g.kept, g.ok, g.source = rev, true, source
g.mu.Unlock()
if !fresh {
return false
return false, nil
}
p.fillingAfter(ctx, used)
id := fmt.Sprintf("%s-%s-%d", meshwants.EventAssignmentRevision, rev.Kind, rev.Revision)
if err := p.wantsSay(meshwants.EventAssignmentRevision, assigngraph.Event(rev), id); err != nil {
p.logf("the assignment revision %d is kept but not said: %v", rev.Revision, err)
}
return true
return true, nil
}
// keepNow is the seat mesh-wants' act keep-now (design 47 H2 §813 (2); H-A-417): the keeping of the pass's newest computed
// revision, run at once, one at a time with the pass's; it answers the revision kept, or the fault the keeping met (a failed
// one is run again after keeping-retry, as every keeping is). The hand act is the tool's to write, before it calls.
func (p *passer) keepNow(ctx context.Context, in meshwants.KeepNowInput) (meshwants.KeepNowAnswer, error) {
g := p.graph
if strings.TrimSpace(in.Why) == "" {
return meshwants.KeepNowAnswer{}, errors.New("say why the keeping runs now")
}
if g == nil || g.objects == nil {
return meshwants.KeepNowAnswer{}, errors.New("the assigner keeps no assignment graph here")
}
g.mu.Lock()
c, ok := g.computed, g.hasComputed
g.mu.Unlock()
if !ok {
return meshwants.KeepNowAnswer{}, errors.New("no pass has computed an assignment graph revision yet")
}
g.keepMu.Lock()
defer g.keepMu.Unlock()
p.logf("the keeping of the assignment graph runs now by hand: %s", in.Why)
if _, err := p.keepRevisionOf(ctx, c); err != nil {
return meshwants.KeepNowAnswer{Fault: err.Error()}, nil
}
rev, _, _ := g.keptNow()
return meshwants.KeepNowAnswer{Revision: &rev}, nil
}
// keepNowVerb is the process's handler of the seat mesh-wants' act keep-now.
func keepNowVerb(ctx context.Context, p *passer) map[string]process.SeatHandler {
return map[string]process.SeatHandler{
meshwants.VerbKeepNow: func(_ string, raw json.RawMessage) (any, error) {
var in meshwants.KeepNowInput
if err := json.Unmarshal(raw, &in); err != nil {
return nil, fmt.Errorf("the arguments are not JSON: %w", err)
}
return p.keepNow(ctx, in)
},
}
}
// graphInputs are what the pass's assignment graph is computed from: every followed branch's catalog half, the placements, each
@@ -366,3 +450,63 @@ func (p *passer) graphKept() (meshwants.AssignmentRevision, int64, bool) {
}
return p.graph.keptNow()
}
// keepingFailed marks the keeping of c wanted again after keeping-retry (design 47 H2 §813 (2)): run then unless a newer
// revision was handed meanwhile, whose own keeping runs. The third failure in a row raises keeping-failed (scope module).
func (p *passer) keepingFailed(ctx context.Context, c assigngraph.Computed, err error) {
g := p.graph
g.mu.Lock()
g.failures++
n, retry, after := g.failures, g.retry, g.after
g.mu.Unlock()
if retry <= 0 {
retry = defaultKeepingRetry
}
if after == nil {
after = func(d time.Duration, f func()) { time.AfterFunc(d, f) }
}
p.logf("the assignment graph revision is not kept (%d in a row), so no want is said as wanted until it is; kept again in %s: %v", n, retry, err)
if n == keepingFailedAt && p.findings.Put != nil {
if err := p.findings.Put(keepingFailedFinding(n, err).Finding(p.clock())); err != nil {
p.logf("the finding %s is not written: %v", kindKeepingFailed, err)
}
}
after(retry, func() {
g.mu.Lock()
newer := g.handed && !assigngraph.Same(g.handedRev, c.Revision)
if !newer {
g.handedRev, g.handed = c.Revision, true
}
g.mu.Unlock()
if !newer {
p.needKeeping(ctx, keepNeed{c: &c})
}
})
}
// keepingSucceeded ends a run of failures: keeping-failed, raised at the third, is cleared.
func (p *passer) keepingSucceeded() {
g := p.graph
g.mu.Lock()
n := g.failures
g.failures = 0
g.mu.Unlock()
if n >= keepingFailedAt && p.findings.Delete != nil {
o := keepingFailedFinding(n, nil)
if err := p.findings.Delete(views.FindingKey(o.Scope, assign.KeyID(o.ID), o.Kind)); err != nil {
p.logf("the finding %s is not deleted: %v", kindKeepingFailed, err)
}
}
}
// keepingFailedFinding is keeping-failed of the assigner on this node, after n failures in a row (design 47 H2 §813 (2)).
func keepingFailedFinding(n int, err error) assign.Observed {
summary := fmt.Sprintf("the keeping of the assignment graph revisions failed %d times in a row", n)
if err != nil {
summary += ": " + err.Error()
}
return assign.Observed{Scope: mcc.ScopeModule, ID: assignment.Module() + "." + thisNode, Kind: kindKeepingFailed,
Severity: mcseat.FindingSeverityWarning, Summary: summary, Headline: "The assignment graph is not kept",
Explanation: summary + ". No want is said as wanted until a keeping succeeds; it is run again after keeping-retry.",
Source: "assigner." + kindKeepingFailed}
}
@@ -3,6 +3,7 @@ package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"slices"
"strings"
@@ -11,6 +12,7 @@ import (
"time"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
mcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
@@ -18,6 +20,7 @@ import (
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"mesh-delivery-assigner/internal/assign"
"mesh-delivery-assigner/internal/assigngraph"
"mesh-delivery-assigner/internal/buckets"
)
@@ -92,6 +95,35 @@ func TestTheWantsAreSaidWantedFromTheKeptAssignmentGraphRevision(t *testing.T) {
if err := json.Unmarshal(events[len(events)-1].body, &w); err != nil || len(events) != 3 || !w.Removed || w.Fp != fpTest {
t.Errorf("a removing want said %+v (%v), %d events", w, err, len(events))
}
// Design 47 H2 §813 (1), H-A-416: a want carries its run, its pipeline's source in decimal, or the newest source held for a
// want of no pipeline; it is written by compare-and-set refusing a lower run, and one that differs only by its run stands.
t.Run("a want's run", func(t *testing.T) {
wkey, _ := mwc.WantKey("g14", "db")
stored := func() mwc.Want { w, _ := mwc.DecodeWant(kv.values[wkey]); return w }
p.putWant(ctx, "g14", "db", mwc.Want{Fp: fpTest, Version: "0.1.2", Pipeline: "main.12", Rank: 1})
if w := stored(); w.Run != "12" || w.Pipeline != "main.12" {
t.Fatalf("the want of pipeline main.12 is %+v, want run 12", w)
}
puts := kv.puts
p.putWant(ctx, "g14", "db", mwc.Want{Fp: fpTest, Version: "0.1.1", Pipeline: "main.11", Rank: 1})
if w := stored(); w.Run != "12" || w.Version != "0.1.2" || kv.puts != puts {
t.Errorf("a want of a lower run replaced %+v (%d writes)", w, kv.puts-puts)
}
p.putWant(ctx, "g14", "db", mwc.Want{Fp: fpTest, Version: "0.1.2", Pipeline: "main.12", Rank: 1})
if kv.puts != puts {
t.Error("a want that stands was written again")
}
skey, _ := mwc.WantKey("shanks", "db")
p.putWant(ctx, "shanks", "db", mwc.Want{Fp: fpTest, Put: true, Words: "put: by hand"})
if w, _ := mwc.DecodeWant(kv.values[skey]); w.Run != "9" {
t.Errorf("a put's want is %+v, want the newest source held, 9", w)
}
// a placement's build after the delivery of run 12: the newest source held for it is the want's own, never refused
p.putWant(ctx, "g14", "db", mwc.Want{Fp: fpTest, Version: "0.1.0"})
if w := stored(); w.Run != "12" || w.Version != "0.1.0" {
t.Errorf("a placement's want after run 12 is %+v, want it written at run 12", w)
}
})
// A-201, A-235: a run's candidate catalog revision, read whole through the seat's verb revision, keeps the run's assignment
// graph under candidate.<group>.<heads> in place of its branch's half; it says nothing and writes no want, a try's candidate
@@ -273,4 +305,65 @@ func TestAPassWritesWantsWhileAKeepingIsBlocked(t *testing.T) {
if !said("the pass took ") {
t.Error("the pass does not say its duration")
}
open()
kept.Wait()
// Design 47 H2 §813 (2): a keeping that fails runs again after keeping-retry with no pass, raising keeping-failed at the third
// failure in a row, cleared at the next that succeeds.
t.Run("a failed keeping runs again", func(t *testing.T) {
failing := &failingObjects{blockedObjects: objects, fail: 3}
var retries []time.Duration
var put []mcseat.Finding
var deleted []string
p.findings = assign.Findings{Put: func(f mcseat.Finding) error { put = append(put, f); return nil },
Delete: func(key string) error { deleted = append(deleted, key); return nil }}
p.graph.objects, p.graph.background, p.graph.retry = failing, nil, 30*time.Second
p.graph.after = func(d time.Duration, f func()) { retries = append(retries, d); f() }
placeOn(t, p.w, "shanks", "web") // new inputs: a keeping is handed
pass()
if len(retries) != 3 || retries[0] != 30*time.Second || len(put) != 1 || put[0].Kind != kindKeepingFailed || put[0].Scope != "module" ||
len(deleted) != 1 || !strings.Contains(deleted[0], kindKeepingFailed) || p.graph.failures != 0 {
t.Fatalf("retries %v, raised %+v, cleared %v, failures %d", retries, put, deleted, p.graph.failures)
}
if _, _, ok := p.graph.keptNow(); !ok || failing.fail != 0 {
t.Fatal("the revision is not kept once the store takes it")
}
// keep-now (H-A-417): the keeping by hand answers the revision kept, or the fault met; it needs a why.
if _, err := p.keepNow(context.Background(), meshwants.KeepNowInput{}); err == nil {
t.Error("a keeping by hand with no why runs")
}
if a, err := p.keepNow(context.Background(), meshwants.KeepNowInput{Why: "the store was full"}); err != nil || a.Revision == nil || a.Fault != "" {
t.Errorf("keep-now answered %+v, %v; want the revision kept", a, err)
}
failing.fail, retries = 2, nil
p.graph.after = func(d time.Duration, _ func()) { retries = append(retries, d) }
placeOn(t, p.w, "novox", "web") // a new revision, whose keeping the store refuses at the pass and again by hand
pass()
if a, err := p.keepNow(context.Background(), meshwants.KeepNowInput{Why: "again"}); err != nil || a.Revision != nil || !strings.Contains(a.Fault, "the store answers nothing") || len(retries) != 2 {
t.Errorf("keep-now on a refusing store answered %+v, %v, retries %v; want the fault and a retry", a, err, retries)
}
var asked []string
tool := keepNowTools(func(key string, body any) (json.RawMessage, error) {
raw, _ := json.Marshal(body)
asked = append(asked, key+" "+string(raw))
return json.RawMessage(`{}`), nil
})[0]
if _, err := tool.Run(map[string]any{"why": "by hand"}); err != nil || len(asked) != 2 || !strings.HasPrefix(asked[0], "seat:mesh-hand-acts.hand-act ") ||
asked[1] != `seat:mesh-wants.keep-now {"why":"by hand"}` {
t.Errorf("assigner_keep_now: %v, asked %v", err, asked)
}
})
}
// failingObjects refuses the next fail puts, then is objects.
type failingObjects struct {
*blockedObjects
fail int
}
func (f *failingObjects) Put(ctx context.Context, name string, data []byte) error {
if f.fail > 0 {
f.fail--
return errors.New("the store answers nothing")
}
return f.blockedObjects.Put(ctx, name, data)
}
@@ -50,6 +50,7 @@ import (
mdseat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/announce"
"git.novox.be/novox/mesh-sdk/go/buscalls"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/process"
@@ -177,7 +178,7 @@ func startBed(t *testing.T, leaveOut string) *bed {
continue
}
cfg := jetstream.KeyValueConfig{Bucket: name, History: 5}
if name == plc.PulsesBucket() || name == buckets.CompositionRuns() {
if name == plc.PulsesBucket() || name == buckets.CompositionRuns() || name == buckets.Taken() {
cfg.History, cfg.LimitMarkerTTL = 1, time.Minute // a time to live per key, as the controller makes it (key-ttl)
}
kv, err := js.CreateKeyValue(ctx, cfg)
@@ -206,6 +207,9 @@ func startBed(t *testing.T, leaveOut string) *bed {
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "EVENTS", Subjects: []string{"mesh.mod.*.event.>", "mesh.seat.*.event.>"}}); err != nil {
t.Fatal(err)
}
// The seat mesh-wants' work queue and its one worker, as the broker derives them from the seat's definition (design 47 H2
// §777); the assigner pulls its step asks from it.
seatQueue(ctx, t, js)
// The node-engine seat's stream of `running`, kept last per subject, which the assigner hears the health of what runs from (§520).
if _, err := js.CreateStream(ctx, keptStream(nodeengine.StreamRunning, nodeengine.SubjectRunningAll, jetstream.StreamConfig{MaxMsgsPerSubject: 1})); err != nil {
t.Fatal(err)
@@ -1938,7 +1942,9 @@ func awaitKept(t *testing.T, b *bed, stream string, n uint64) {
// serveSeatList answers the seat mesh-catalog's verb seat-definitions as the indexer does (ADR 0636 §4): with no seat, every
// trunk seat with its digest; with one, its definition resolved, from the mesh-sdk test copy of every definition. The channel
// it answers is closed once every listed seat's definition was asked, the end of a reader's first read.
// it answers is closed once every listed seat's definition was asked, the end of a reader's first read. It also answers the
// chunked read verb seat-list as the indexer does (design 47 H2 H-A-443): the definitions as one JSON array sorted by seat,
// its sha256 and size, then the bytes on the call's chunk subjects (seatListBytes).
func serveSeatList(t *testing.T, nc *nats.Conn) <-chan struct{} {
t.Helper()
var mu sync.Mutex
@@ -1981,5 +1987,94 @@ func serveSeatList(t *testing.T, nc *nats.Conn) <-chan struct{} {
}
t.Cleanup(func() { _ = sub.Unsubscribe() })
}
list, digest := seatListBytes()
hand := func(m *nats.Msg) {
var in meshcatalog.SeatListInput
_ = json.Unmarshal(m.Data, &in)
send := buscalls.SeatChunks{Seat: meshcatalog.Seat, Verb: meshcatalog.VerbSeatList, Call: in.Call}
chunks := int64((len(list) + buscalls.SeatChunkBytes - 1) / buscalls.SeatChunkBytes)
raw, _ := json.Marshal(map[string]any{"result": meshcatalog.SeatListAnswer{Digest: digest, Size: int64(len(list)), Chunks: chunks}, "node": "n0"})
_ = m.Respond(raw)
go func() { _ = send.Send(nc, list, buscalls.SeatAckWithin, t.Logf) }()
}
for _, subject := range []string{"mesh.seat." + meshcatalog.Seat + ".tool." + meshcatalog.VerbSeatList, "mesh.seat." + meshcatalog.Seat + ".call." + meshcatalog.VerbSeatList + ".*"} {
sub, err := nc.Subscribe(subject, hand)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = sub.Unsubscribe() })
}
return read
}
// seatListBytes are the bed's seat list as the verb seat-list hands it: every definition of the mesh-sdk test copy as one JSON
// array sorted by seat, and its sha256 (H-A-441 (a)).
func seatListBytes() ([]byte, string) {
defs := slices.Clone(seatlisttest.Definitions())
slices.SortFunc(defs, func(a, b seatdef.Definition) int { return strings.Compare(a.Seat, b.Seat) })
list := make([]json.RawMessage, 0, len(defs))
for _, d := range defs {
raw, _ := json.Marshal(d)
list = append(list, raw)
}
raw, _ := json.Marshal(list)
return raw, seatListDigest(raw)
}
// seatQueue makes the seat mesh-wants' work queue and its one shared worker from the seat's definition, as the broker derives
// them (design 47 H2 §777).
func seatQueue(ctx context.Context, t *testing.T, js jetstream.JetStream) {
t.Helper()
raw, err := os.ReadFile(filepath.Join("..", "..", "..", "mesh-delivery-contracts", "seats", meshwants.Seat, "seat.json"))
if err != nil {
t.Fatal(err)
}
def, err := seatdef.Parse(raw)
if err != nil {
t.Fatal(err)
}
stream, worker, ok, err := buscompose.SeatQueue(meshwants.Seat, def.Accepts)
if err != nil || !ok {
t.Fatalf("the seat's work queue: %v %v", ok, err)
}
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: stream.Name, Subjects: stream.Subjects, Retention: jetstream.WorkQueuePolicy,
MaxAge: time.Duration(stream.MaxAge) * time.Second, MaxMsgs: int64(stream.MaxMsgs), Duplicates: time.Duration(stream.DuplicatesSeconds) * time.Second}); err != nil {
t.Fatal(err)
}
if _, err := js.CreateOrUpdateConsumer(ctx, stream.Name, jetstream.ConsumerConfig{Durable: worker.Name, FilterSubjects: worker.Filters, AckPolicy: jetstream.AckExplicitPolicy,
AckWait: time.Duration(worker.AckWaitSeconds) * time.Second, MaxDeliver: worker.MaxDeliver, MaxAckPending: worker.MaxAckPending}); err != nil {
t.Fatal(err)
}
}
// putStep puts one step ask on the seat mesh-wants' work queue under id, as the runner puts it (design 47 H2 §777).
func putStep(ctx context.Context, t *testing.T, js jetstream.JetStream, id string, step meshwants.StepRun) {
t.Helper()
msg := nats.NewMsg(meshwants.AcceptSubjectStep)
msg.Header.Set("x-event-id", id)
msg.Header.Set(nats.MsgIdHdr, id)
msg.Data, _ = json.Marshal(step)
if _, err := js.PublishMsg(ctx, msg); err != nil {
t.Fatalf("the ask %s is not put on the seat's work queue: %v", id, err)
}
}
// queueEmpty fails t unless the seat mesh-wants' worker holds no ask, waiting or unacknowledged (design 47 H2 §777).
func queueEmpty(ctx context.Context, t *testing.T, js jetstream.JetStream) {
t.Helper()
deadline := time.Now().Add(10 * time.Second)
for {
c, err := js.Consumer(ctx, buscompose.SeatQueueStreamName(meshwants.Seat), buscompose.SeatQueueWorkerName(meshwants.Seat))
if err != nil {
t.Fatal(err)
}
info, err := c.Info(ctx)
if err == nil && info.NumPending == 0 && info.NumAckPending == 0 {
return
}
if time.Now().After(deadline) {
t.Fatalf("the seat's worker holds asks after their ends: %+v %v", info, err)
}
time.Sleep(20 * time.Millisecond)
}
}
@@ -14,8 +14,10 @@ import (
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-sdk/go/buscalls"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"io"
"log"
"maps"
"mesh-delivery-assigner/internal/assigngraph"
"mesh-delivery-assigner/internal/assignment"
"mesh-delivery-assigner/internal/buckets"
@@ -23,6 +25,7 @@ import (
"os/signal"
"slices"
"strings"
"sync/atomic"
"syscall"
"time"
@@ -89,7 +92,7 @@ func ownBuckets() []string {
return []string{buckets.Asks(), buckets.Differences(),
buckets.Puts(), buckets.Wants(), plc.AssignmentsBucket(), plc.ProposalsBucket(),
plc.CandidatesBucket(), plc.PulsesBucket(), assign.FindingsBucket(), assign.PoliciesBucket(), plc.SeatHoldersBucket(),
engineReadsBucket(), retiredBucket(), copiesBucket(), writtenBucket(), buckets.Waits(), buckets.PutWaits(), buckets.Inputs(), buckets.CompositionRuns()}
engineReadsBucket(), retiredBucket(), copiesBucket(), writtenBucket(), buckets.Waits(), buckets.PutWaits(), buckets.Inputs(), buckets.CompositionRuns(), buckets.Taken()}
}
// engineReadsState is the assigner's state of what each node's engine says it reads (design 47 H-A-104), one key `<node>`
@@ -136,6 +139,7 @@ var keptStates = []announce.State{
{Name: buckets.PutWaitsState, Readable: true}, // the put waits said (putwaits.go, design 47 G A-211)
{Name: buckets.InputsState, Readable: true}, // read by the tool process alone: the verb input and assigner_inputs (design 47 G §354)
{Name: buckets.CompositionRunsState, Readable: true}, // the composition runs, read by the process and its tools (design 47 G §719, A-468c)
{Name: process.TakenState, Readable: true}, // the asks of the seat mesh-wants taken once, and their drops (design 47 H2 §775, §777)
}
// kvSet is every bucket the assigner opened, by name.
@@ -615,7 +619,11 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
objects := newBusObjects(js, process.Account{Node: thisNode, Module: assignment.Module()}, assigngraph.RevisionsObjects)
// The keeping and removing of revisions runs off the pass loop, and a keeping that changed what the pass reads wakes
// one (design 47 H-A-259).
p.graph = &graphKeeper{objects: objects, halves: iv.h.views, background: func(f func()) { go f() }, wake: wake,
retry, err := keepingRetry()
if err != nil {
return err
}
p.graph = &graphKeeper{objects: objects, halves: iv.h.views, background: func(f func()) { go f() }, wake: wake, retry: retry,
reads: newBusObjects(js, process.Account{Node: thisNode, Module: assignment.Module()}, assigngraph.RevisionsObjects)}
keep := func(ctx context.Context, ev meshcatalog.CatalogRevisionEvent) error {
return p.keepCandidate(ctx, iv.call, ev)
@@ -667,10 +675,25 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
}
// The composition step (design 47 G §22, §354): the seat mesh-wants' accepted kind step, run by one run ask per machine.
if bus != nil && js != nil && inputs != nil && p.publishWants != nil && runs != nil {
run := &compositionRun{runs: runs, inputs: &compositionInputs{machines: w.machinesOf, callFile: bus.CallFile}, keeper: inputs, reads: &compositionReads{}, call: bus.CallKept,
askBound, err := composeAskBound()
if err != nil {
return err
}
run := &compositionRun{runs: runs, inputs: &compositionInputs{machines: w.machinesOf, callFile: bus.CallFileKept, bound: askBound}, keeper: inputs, reads: &compositionReads{}, call: bus.CallKept, callFile: bus.CallFileKept,
trunk: trunkOf(iv, w.trunkName), newest: iv.newestOf(), watched: iv.watchedOf, assigned: w.assignedOn, assignment: w.assignmentOf, findings: findings, running: w.runningOf, away: func(machine string) bool { return w.awayNow()[machine] }, put: runAskPublisher(js), ask: bus.CallFile, now: time.Now, logf: logger.Printf}
// The seat's work queue (design 47 H2 §777): pulled through its one worker, each ask taken once in the state taken.
takenBound, err := askTakenBound()
if err != nil {
return err
}
var ackWait atomic.Int64 // the worker's ack wait, read when it is bound
ackWait.Store(int64(buscompose.SeatQueueDefaultAckWait))
steps := &compositionSteps{node: thisNode, say: p.publishWants, run: run.run, resume: run.resume,
validate: run.validate, resumeValidate: run.resumeValidate, ended: runs.stepEnded, now: time.Now, logf: logger.Printf}
validate: run.validate, resumeValidate: run.resumeValidate, ended: runs.stepEnded, now: time.Now, logf: logger.Printf,
takenBound: takenBound, inProgress: inProgressOf(seatlist.Definition(gate.Reader, meshwants.Seat), func() time.Duration { return time.Duration(ackWait.Load()) })}
if e, ok := bus.(emitting); ok {
steps.taken = e.State(process.TakenState)
}
move := steps.move(ctx) // the readers a rerun carried (§727, A-470d)
runs.moved.Store(&move)
// The state's watch wakes the steps waiting on a run and hears a bound's expiry; the runs standing at the start are taken up
@@ -685,9 +708,14 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
_, err := meshhandactsclient.Recorder{Ask: bus.Call}.Record(q)
return err
}
stopVerbs, err := e.AnswerSeat(meshwants.Seat, seatlist.Definition(gate.Reader, meshwants.Seat), compositionVerbs(runs, steps, record))
verbs := compositionVerbs(runs, steps, record)
maps.Copy(verbs, dropVerb(steps)) // the seat's act drop (§778, H-A-404)
if p.graph != nil {
maps.Copy(verbs, keepNowVerb(ctx, p)) // the keeping by hand (§813 (2), H-A-417)
}
stopVerbs, err := e.AnswerSeat(meshwants.Seat, seatlist.Definition(gate.Reader, meshwants.Seat), verbs)
if err != nil {
logger.Printf("the verbs %s and %s are not answered until the assigner starts again: %v", meshwants.VerbCompositions, meshwants.VerbCompositionsDrop, err)
logger.Printf("the verbs %s, %s and %s are not answered until the assigner starts again: %v", meshwants.VerbCompositions, meshwants.VerbCompositionsDrop, meshwants.VerbDrop, err)
} else {
go func() { <-ctx.Done(); stopVerbs() }()
}
@@ -700,10 +728,10 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
}
p.ends.left.Store(&left)
}
if err := serveCompositionSteps(ctx, nc, steps); err != nil {
logger.Printf("the composition step is not heard, so no composition step runs until the assigner starts again: %v", err)
} else {
logger.Printf("the composition step is heard on %s", meshwants.AcceptSubjectStep)
if steps.taken == nil {
logger.Printf("the state %s is not bound, so no composition step is pulled until the assigner starts again", process.TakenState)
} else if err := serveStepQueue(ctx, nc, js, steps, func(d time.Duration) { ackWait.Store(int64(d)) }); err != nil {
logger.Printf("the composition step is not pulled, so no composition step runs until the assigner starts again: %v", err)
}
}
if bus != nil {
@@ -7,8 +7,10 @@ import (
"errors"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"mesh-delivery-assigner/internal/buckets"
"reflect"
"slices"
"sort"
"strconv"
"strings"
"time"
@@ -21,6 +23,8 @@ import (
"git.novox.be/novox/mesh-sdk/go/seatlist"
"sync"
"github.com/nats-io/nats.go/jetstream"
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/graph"
"git.novox.be/novox/mesh-sdk/go/catalogue"
@@ -303,11 +307,93 @@ func (p *passer) putWant(ctx context.Context, node, module string, want mwc.Want
fp = before.Fp
}
}
if p.written(ctx, buckets.Wants(), key, func() ([]byte, error) { return mwc.EncodeWant(want) }) {
if p.wantWritten(ctx, node, module, key, want) {
p.sayWanted(node, module, fp, want.Pipeline, want.Removed)
}
}
// runOf is the run a want carries (design 47 H2 §813 (1); H-A-355 (1), H-A-416): the source revision of the pipeline that
// released it, `<branch>.<source>`, in decimal; for a want of no pipeline (a put, a placement's build, a removal) the newest
// source the assigner holds for it: the trunk's source of its kept assignment graph revision, or the run of the want written
// (held) when that is newer, so a person's put or placement after a delivery is never refused as older; "" when it holds none.
func (p *passer) runOf(want mwc.Want, held *mwc.Want) string {
if i := strings.LastIndexByte(want.Pipeline, '.'); i >= 0 {
if n, err := strconv.ParseInt(want.Pipeline[i+1:], 10, 64); err == nil && n > 0 {
return strconv.FormatInt(n, 10)
}
}
var newest int64
if p.graph != nil {
if _, source, ok := p.graph.keptNow(); ok {
newest = source
}
}
if held != nil {
newest = max(newest, runNumber(held.Run))
}
if newest <= 0 {
return ""
}
return strconv.FormatInt(newest, 10)
}
// runNumber is a want's run as a number, 0 for none.
func runNumber(run string) int64 {
n, _ := strconv.ParseInt(run, 10, 64)
return n
}
// wantWritten writes the want of module on node under key by compare-and-set, carrying its run (design 47 H2 §813 (1)): a want
// that differs from the one written only by its run is not written again (its run stays, so a node never composes again for
// it), and one whose run is lower than the run written is refused, said in the journal. It answers whether it wrote.
func (p *passer) wantWritten(ctx context.Context, node, module, key string, want mwc.Want) bool {
var held *mwc.Want
if raw := p.w.value(buckets.Wants(), key); raw != nil {
if h, err := mwc.DecodeWant(raw); err == nil {
held = &h
}
}
want.Run = p.runOf(want, held)
if held != nil {
same := *held
same.Run = want.Run
if reflect.DeepEqual(same, want) {
want.Run = held.Run // it stands: nothing is written
} else if runNumber(held.Run) > runNumber(want.Run) {
p.logf("the want of %s on %s is not written: its run %q is lower than the run %q written (§813 (1))", module, node, want.Run, held.Run)
return false
}
}
value, err := mwc.EncodeWant(want)
if err != nil {
p.logf("%s %s is not written: %v", buckets.Wants(), key, err)
return false
}
if held := p.w.value(buckets.Wants(), key); held != nil && bytes.Equal(held, value) {
return false
}
kv := p.kvs[buckets.Wants()]
entry, err := kv.Get(ctx, key)
switch {
case errors.Is(err, jetstream.ErrKeyNotFound):
_, err = kv.Create(ctx, key, value)
case err == nil:
if h, derr := mwc.DecodeWant(entry.Value()); derr == nil && runNumber(h.Run) > runNumber(want.Run) {
p.logf("the want of %s on %s is not written: its run %q is lower than the run %q written (§813 (1))", module, node, want.Run, h.Run)
return false
}
_, err = kv.Update(ctx, key, value, entry.Revision())
}
if err != nil {
if !errors.Is(err, context.Canceled) {
p.logf("%s %s could not be written: %v; the next pass writes it again", buckets.Wants(), key, err)
}
return false
}
p.w.put(buckets.Wants(), key, value)
return true
}
// staleAsk says why the hold, restore or record ask at key (read as the SDK reads it) is about something the trunk has left
// behind, or "" when it is not: a module no longer on the trunk, a fingerprint that is no longer the module's trunk fingerprint,
// or, for a record ask, a composition the module's node no longer waits on. A backup ask, and a revert ask (design 47 G §710), is stale when the module's want on its
@@ -7,6 +7,7 @@ import (
"errors"
"fmt"
mpr "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-assigner/internal/buckets"
"mesh-delivery-assigner/internal/puts"
"os"
@@ -56,6 +57,7 @@ func (m *memKV) Put(_ context.Context, key string, value []byte) (uint64, error)
func (m *memKV) Create(_ context.Context, key string, value []byte, _ ...jetstream.KVCreateOpt) (uint64, error) {
m.values[key] = value
m.puts++ // a write that reaches the bucket (the wants' compare-and-set, design 47 H2 §813 (1))
return uint64(len(m.values)), nil
}
@@ -252,6 +254,7 @@ func (m *memKV) Update(_ context.Context, key string, value []byte, revision uin
return 0, errors.New("wrong last sequence")
}
m.values[key] = value
m.puts++
return uint64(len(value)), nil
}
@@ -295,6 +298,9 @@ func (b *fakeBus) CallOn(node, key string, body any) (json.RawMessage, error) {
}
func (b *fakeBus) CallKept(key string, body any) (json.RawMessage, error) { return b.Call(key, body) }
func (b *fakeBus) CallFile(key string, body any) (json.RawMessage, error) { return b.Call(key, body) }
func (b *fakeBus) CallFileKept(_ process.KeptAsk, key string, body any) (json.RawMessage, error) {
return b.Call(key, body)
}
func (b *fakeBus) Emit(eventType string, body any) error {
b.events = append(b.events, fmt.Sprintf("%s %+v", eventType, body))
return nil
@@ -21,6 +21,7 @@ import (
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-assigner/internal/assign"
)
@@ -44,6 +45,8 @@ type caller interface {
CallKept(key string, body any) (json.RawMessage, error)
// CallFile is Call of a verb marked chunks, its answer read whole (process.Conn.CallFile; design 47 G A-291).
CallFile(key string, body any) (json.RawMessage, error)
// CallFileKept is CallFile as a kept call carrying its ask's id, until its bound (process.Conn.CallFileKept; H2 §775).
CallFileKept(ask process.KeptAsk, key string, body any) (json.RawMessage, error)
Emit(eventType string, body any) error
}
@@ -0,0 +1,318 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"sync"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/seatdef"
)
// **The seat mesh-wants takes its asks from a work queue and says ask-taken** (design 47 H2 §777, §775, §778; ADR 0525 §2,
// §6; ADR 0668 §2; H-A-381, H-A-401, H-A-404): the broker derives the seat's stream and its one shared worker from the seat's
// definition; the assigner pulls every ask waiting at once, takes each once by its id in its state taken (key
// <seat>.<ask>, for ask-taken-bound), says ask-taken at the pull, sends in-progress while the step runs and acknowledges an
// ask once it said the ask's end. An ask put while the assigner is down waits on the stream. The seat's act drop ends an ask
// its caller no longer waits for: one running here is cancelled, one not pulled yet is kept in taken as dropped.<ask> for
// ask-taken-bound, and either ends its step could not run naming the drop and who asked it.
// askTakenBoundSetting is the setting the taken key's time to live is (H-A-381 (2)): the holder never receives the
// caller's bound, so the largest caller's, the runner's ask-unanswered-bound, is its default.
const (
askTakenBoundSetting = "ask-taken-bound"
defaultAskTakenBound = 10 * time.Minute
)
// droppedPrefix keys a drop of an ask not pulled yet in the state taken (H-A-404).
const droppedPrefix = "dropped."
// takenStore is the state taken (process.State): an ask taken once by its id, and the drops of asks not pulled yet.
type takenStore interface {
TakeOnce(seat, ask string, bound time.Duration) (bool, error)
Get(key string) (*stdio.StateEntry, error)
RenewTTL(key string, value any, ttl time.Duration) (uint64, error)
Delete(key string) error
}
// queueMsg is what the steps read of a message pulled from the seat's work queue (jetstream.Msg).
type queueMsg interface {
Data() []byte
Headers() nats.Header
Ack() error
Nak() error
Term() error
InProgress() error
}
// askDrop is one drop of an ask: who asked it, why and when; the cause of a running step's end.
type askDrop struct {
By string `json:"by,omitempty"`
Why string `json:"why"`
At time.Time `json:"at"`
}
func (d *askDrop) Error() string {
by := d.By
if by == "" {
by = "its asker"
}
return fmt.Sprintf("the ask was dropped by %s: %s", by, d.Why)
}
// askIDOf is the ask's id a put carries: its event id, else its message id.
func askIDOf(h nats.Header) string {
if id := h.Get("x-event-id"); id != "" {
return id
}
return h.Get(nats.MsgIdHdr)
}
// pulled handles one ask pulled from the seat's work queue: an ask that is no composition or validate step with an id is
// terminated, said; one dropped before it was pulled ends could not run naming the drop; one running here already (handed
// again while it runs) is held for that run, which acknowledges it at its end; any other is taken once by its id (ask-taken
// said the first time) and run, in progress while it runs. A repeat of an id taken before that runs here no longer is run
// again against the runs as they stand (H-A-381 (1), H-A-401): its composition runs are joined, never run twice.
func (c *compositionSteps) pulled(ctx context.Context, m queueMsg) {
id := askIDOf(m.Headers())
var run meshwants.StepRun
err := json.Unmarshal(m.Data(), &run)
how := c.howOf(run.Step, false)
if err != nil || id == "" || how == nil {
c.logf("a step ask on the seat %s's work queue that is no composition or validate step with an id is ended: %v", meshwants.Seat, err)
_ = m.Term()
return
}
if d := c.droppedBefore(id); d != nil {
c.hold(id, m)
c.logf("the ask %s (%s) was dropped before it was pulled: %v", id, run.Pipeline, d)
c.sayEnd(id, run, nil, stepOutcome{Outcome: meshwants.StepEndedOutcomeCouldNotRun, Said: d.Error()}, "")
if err := c.taken.Delete(droppedPrefix + id); err != nil {
c.logf("the drop of the ask %s cannot be deleted from the state %s: %v", id, process.TakenState, err)
}
return
}
if c.running(id) {
c.hold(id, m)
c.logf("the ask %s (%s) was handed again while it runs here: its run answers it", id, run.Pipeline)
return
}
first, err := c.taken.TakeOnce(meshwants.Seat, id, c.takenBound)
if err != nil {
c.logf("the ask %s (%s) is not taken, the state %s refused it: %v; it is handed again", id, run.Pipeline, process.TakenState, err)
_ = m.Nak()
return
}
c.hold(id, m)
if first {
at := c.now().UTC()
if err := c.say(meshwants.EventAskTaken+"."+id, meshwants.AskTakenEvent{Ask: id, Node: c.node, At: at}, id+".taken"); err != nil {
c.logf("saying ask-taken of %s: %v", id, err)
}
} else {
c.logf("the ask %s (%s) was taken before: it is run again against its composition runs as they stand", id, run.Pipeline)
}
go c.say1(ctx, id, run, how, "")
}
// hold keeps m for the step asked under id: acknowledged once its end is said, in progress while it runs.
func (c *compositionSteps) hold(id string, m queueMsg) {
c.mu.Lock()
defer c.mu.Unlock()
if c.held == nil {
c.held = map[string][]queueMsg{}
}
c.held[id] = append(c.held[id], m)
}
// settle acknowledges (ack) or hands again (nak) every message held for id.
func (c *compositionSteps) settle(id string, ack bool) {
c.mu.Lock()
ms := c.held[id]
delete(c.held, id)
c.mu.Unlock()
for _, m := range ms {
if ack {
_ = m.Ack()
} else {
_ = m.Nak()
}
}
}
// inProgressEvery says every message held for id in progress every d, until the returned stop is called.
func (c *compositionSteps) inProgressEvery(id string) (stop func()) {
every := time.Duration(0)
if c.inProgress != nil {
every = c.inProgress()
}
if every <= 0 {
return func() {}
}
ticks := c.ticks
if ticks == nil {
ticks = func(d time.Duration) (<-chan time.Time, func()) { t := time.NewTicker(d); return t.C, t.Stop }
}
tick, stopTicks := ticks(every)
done, ended := make(chan struct{}), make(chan struct{})
go func() {
defer close(ended)
defer stopTicks()
for {
select {
case <-done:
return
case <-tick:
c.mu.Lock()
ms := append([]queueMsg(nil), c.held[id]...)
c.mu.Unlock()
for _, m := range ms {
_ = m.InProgress()
}
}
}
}()
return func() { close(done); <-ended }
}
// droppedBefore is the drop kept for an ask not pulled yet, nil for none.
func (c *compositionSteps) droppedBefore(id string) *askDrop {
if c.taken == nil {
return nil
}
e, err := c.taken.Get(droppedPrefix + id)
if err != nil {
c.logf("whether the ask %s was dropped cannot be read from the state %s: %v; it is taken", id, process.TakenState, err)
return nil
}
if e == nil {
return nil
}
var d askDrop
if json.Unmarshal(e.Value, &d) != nil {
return nil
}
return &d
}
// dropAsk is the seat's act drop (§778, H-A-404): a step running here for the ask is cancelled with the drop as its cause and
// ends could not run naming it; one not running here is kept in taken as dropped.<ask> for ask-taken-bound, so its pull ends
// it the same way. It answers whether the ask was held here.
func (c *compositionSteps) dropAsk(caller string, in meshwants.DropInput) (meshwants.DropAnswer, error) {
id, why := strings.TrimSpace(in.ID), strings.TrimSpace(in.Why)
if id == "" || why == "" {
return nil, errors.New("give the ask's id and why it is dropped")
}
by := strings.TrimSpace(in.By)
if by == "" {
by = caller
}
d := &askDrop{By: by, Why: why, At: c.now().UTC()}
c.mu.Lock()
stop := c.stops[id]
c.mu.Unlock()
if stop != nil {
stop(d)
c.logf("the ask %s running here is dropped: %v", id, d)
return meshwants.DropAnswer{"held": true}, nil
}
if c.taken == nil {
return nil, fmt.Errorf("the state %s is not bound, so the drop of %s cannot be kept", process.TakenState, id)
}
if _, err := c.taken.RenewTTL(droppedPrefix+id, d, c.takenBound); err != nil {
return nil, fmt.Errorf("writing the drop of %s in the state %s: %w", id, process.TakenState, err)
}
c.logf("the ask %s, not running here, is kept dropped until it is pulled: %v", id, d)
return meshwants.DropAnswer{"held": false}, nil
}
// dropVerb is the process's handler of the seat mesh-wants' act drop.
func dropVerb(c *compositionSteps) map[string]process.SeatHandler {
return map[string]process.SeatHandler{
meshwants.VerbDrop: func(caller string, raw json.RawMessage) (any, error) {
var in meshwants.DropInput
if err := json.Unmarshal(raw, &in); err != nil {
return nil, fmt.Errorf("the arguments are not JSON: %w", err)
}
return c.dropAsk(caller, in)
},
}
}
// inProgressOf is the seat's in-progress as its definition names it, once the seat list is read; before, a third of the
// worker's ack wait, which the broker derived from the same definition (60 s and 20 s, §777; H-A-401).
func inProgressOf(def func() (seatdef.Definition, bool), ackWait func() time.Duration) func() time.Duration {
return func() time.Duration {
if d, ok := def(); ok {
if _, worker, ok, err := buscompose.SeatQueue(meshwants.Seat, d.Accepts); err == nil && ok && worker.InProgressSeconds > 0 {
return time.Duration(worker.InProgressSeconds) * time.Second
}
}
return ackWait() / 3
}
}
// serveStepQueue pulls the seat's work queue through its one shared worker, every ask waiting at once (at most the worker's
// max ack pending), once the broker made it: a worker not there yet is said, and bound when the bus says it was created.
func serveStepQueue(ctx context.Context, conn process.Subscriber, js jetstream.JetStream, c *compositionSteps, ackWait func(time.Duration)) error {
stream, worker := buscompose.SeatQueueStreamName(meshwants.Seat), buscompose.SeatQueueWorkerName(meshwants.Seat)
var once sync.Once
bind := func() bool {
cons, err := js.Consumer(ctx, stream, worker)
if err != nil {
return false
}
bound := false
once.Do(func() {
bound = true
info := cons.CachedInfo()
if info != nil && info.Config.AckWait > 0 {
ackWait(info.Config.AckWait)
}
pending := 1000
if info != nil && info.Config.MaxAckPending > 0 {
pending = info.Config.MaxAckPending
}
cc, err := cons.Consume(func(m jetstream.Msg) { c.pulled(ctx, m) }, jetstream.PullMaxMessages(pending))
if err != nil {
c.logf("the seat %s's work queue %s is not pulled: %v", meshwants.Seat, worker, err)
return
}
c.logf("the composition step is pulled from %s on %s", worker, stream)
go func() { <-ctx.Done(); cc.Stop() }()
})
return bound
}
if bind() {
return nil
}
c.logf("the seat %s's work queue %s on %s is not there yet: its asks wait on it, and it is pulled when the bus says it was created", meshwants.Seat, worker, stream)
sub, err := conn.Subscribe("$JS.EVENT.ADVISORY.CONSUMER.CREATED."+stream+"."+worker, func(*nats.Msg) { go bind() })
if err != nil {
return fmt.Errorf("hearing the creation of the seat %s's work queue: %w", meshwants.Seat, err)
}
go func() { <-ctx.Done(); _ = sub.Unsubscribe() }()
if bind() { // made between the first look and the subscription
_ = sub.Unsubscribe()
}
return nil
}
// askTakenBound reads the setting ask-taken-bound (§781).
func askTakenBound() (time.Duration, error) {
d, err := stdio.SettingDuration(askTakenBoundSetting, defaultAskTakenBound)
if err != nil {
return 0, errors.New("the assigner's " + err.Error())
}
return d, nil
}
@@ -22,6 +22,7 @@ import (
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/catalogkey"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/seatlist"
"git.novox.be/novox/mesh-sdk/go/stateobjects"
"mesh-delivery-assigner/internal/buckets"
@@ -416,6 +417,9 @@ func serveTools() error {
tools = append(tools, whyTools(runtimeStore{})...)
tools = append(tools, askAgainTools(runtimeStore{}, stdio.Ask, time.Now)...)
tools = append(tools, wantDropTools(runtimeStore{}, stdio.Ask)...)
tools = append(tools, asksTools(stdio.QueueWaiting, stdio.State(process.TakenState), stdio.Ask)...) // the seat's work queue (§777)
tools = append(tools, keepNowTools(stdio.Ask)...)
tools = append(tools, validateBeforeTools(runtimeStore{})...) // what a validate step did not count (§814) // the keeping by hand (§813 (2))
return stdio.Serve("", tools)
}
@@ -2,6 +2,10 @@ package main
import (
"context"
"crypto/rand"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
@@ -15,6 +19,10 @@ import (
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/graph"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-assigner/internal/buckets"
)
// The step validate (design 47 H §745; H-A-34, H-A-51, H-A-108): the assigner serves it as an accepted kind `step` of the seat
@@ -23,8 +31,8 @@ import (
// reads: the cycle the revision's entries make (graph.RefusedCycle) ends it at once; else the manifest validator built from
// the revision's own entry of the module holding the seat mesh-catalog, on the Go build seat as the composer is, runs in the
// sandbox over the revision's manifests and its base's (H-A-154), its input kept and served on mesh-wants' input.
// Every mover's step ends the same: failed `cycle` naming the cycle, failed `manifest-refused` naming every problem of a
// manifest the revision alters or between manifests one it alters takes part in, or passed.
// Every mover's step ends the same: failed `cycle` naming the cycle, failed `manifest-refused` naming every problem the case
// before does not hold (design 47 H2 §814; assigner_validate_before answers the ones it holds), or passed.
const (
// validatorPackage is the manifest validator's import path in the module holding the seat mesh-catalog (the indexer's
@@ -112,16 +120,12 @@ func (r *compositionRun) judgeValidate(ctx context.Context, step meshwants.StepR
if err != nil {
return couldNotRun("the step's catalog revision is not read: " + err.Error())
}
before, err := r.before(ctx, step, half)
if err != nil {
return couldNotRun("the base of the case before is not read: " + err.Error())
}
// A revision that names no base (the branch's first take-in) is judged against an empty case before, and says so (H-A-151).
empty := ""
if half.Revision.Base <= 0 {
empty = emptyBeforeWords
}
if problems := causedProblems(v.Change, v.Before, altered(half, before)); len(problems) > 0 {
if problems := causedProblems(v.Change, v.Before); len(problems) > 0 {
said := reasonManifestRefused + ": " + strings.Join(problems, "; ")
if empty != "" {
said += " (" + empty + ")"
@@ -173,60 +177,44 @@ func (r *compositionRun) before(ctx context.Context, step meshwants.StepRun, hal
return r.revision(ctx, base)
}
// altered are the modules whose manifest the revision adds or changes against the one before it.
func altered(half, before mcc.Half) []string {
var out []string
for name, e := range half.Entries {
if len(e.Manifest) == 0 {
continue
}
b, ok := before.Entries[name]
switch {
case !ok || len(b.Manifest) == 0:
out = append(out, name)
case e.ManifestDigest != "" && b.ManifestDigest != "":
if e.ManifestDigest != b.ManifestDigest {
out = append(out, name)
}
default:
x, _ := json.Marshal(e.Manifest)
y, _ := json.Marshal(b.Manifest)
if string(x) != string(y) {
out = append(out, name)
}
}
}
slices.Sort(out)
return out
// causedProblems are the problems the revision causes (ADR 0578 §2; design 47 H2 §814): of every manifest, and between
// manifests, each problem the case before did not have; a problem the case before holds is never counted, whether the
// revision alters that manifest or not (assigner_validate_before answers those).
func causedProblems(change, before composer.CaseVerdict) []string {
return problemsBy(change, before, false)
}
// causedProblems are the problems the revision causes (ADR 0578 §2): every problem of a manifest it alters, and of any other
// manifest, or between manifests, each problem the case before did not have.
func causedProblems(change, before composer.CaseVerdict, touched []string) []string {
// uncountedProblems are the change case's problems the case before holds too, which causedProblems does not count (§814).
func uncountedProblems(change, before composer.CaseVerdict) []string {
return problemsBy(change, before, true)
}
// problemsBy are the change case's problems, per manifest, that the case before had (had) or did not have (!had).
func problemsBy(change, before composer.CaseVerdict, had bool) []string {
var out []string
for _, module := range slices.Sorted(maps.Keys(change.Manifests)) {
read := change.Manifests[module]
if read.Accepted || read.Refused == "" {
continue
}
had := map[string]bool{}
held := map[string]bool{}
for _, p := range strings.Split(before.Manifests[module].Refused, "\n") {
had[p] = true
held[p] = true
}
var caused []string
var named []string
for _, p := range strings.Split(read.Refused, "\n") {
if p != "" && (slices.Contains(touched, module) || !had[p]) {
caused = append(caused, p)
if p != "" && held[p] == had {
named = append(named, p)
}
}
if len(caused) == 0 {
if len(named) == 0 {
continue
}
what := module
if module == betweenManifests {
what = "between manifests"
}
out = append(out, what+": "+strings.Join(caused, "; "))
out = append(out, what+": "+strings.Join(named, "; "))
}
return out
}
@@ -300,7 +288,7 @@ func (r *compositionRun) startValidate(ctx context.Context, key string, run mesh
return
}
program.Package = validatorPackage
seats, err := r.seats(ctx, half.Revision.Definitions)
seats, list, err := r.validateSeats(ctx, ask, half.Revision.Definitions)
if err != nil {
couldNot(err.Error())
return
@@ -317,7 +305,7 @@ func (r *compositionRun) startValidate(ctx context.Context, key string, run mesh
}
// No predefined key list and no known package names until a design line names the verb that gives them (H-A-108 (2)).
req := composer.Request{V: composer.Version, Cases: []composer.Case{{Name: changeCase, Manifests: change}, {Name: beforeCase, Manifests: prior}},
Seats: seats}
Seats: seats, SeatList: list}
raw, err := json.Marshal(req)
if err != nil {
couldNot("its request cannot be encoded: " + err.Error())
@@ -380,3 +368,126 @@ func validateRunVerdict(answer []byte) (map[string]any, string) {
}
return verdict, ""
}
// validateBeforeTool is the tool that answers what a validate step did not count (design 47 H2 §814).
const validateBeforeTool = "assigner_validate_before"
// validateBefore answers, for the validate run of pipeline, the problems of the change case that its case before holds too,
// which the step did not count, for module alone when one is named; read from the run the state composition-runs keeps.
func validateBefore(st store, pipeline, module string) (map[string]any, error) {
key, err := validateKeyOf(meshwants.StepRun{Pipeline: strings.TrimSpace(pipeline)})
if err != nil {
return nil, err
}
raw, _, ok, err := st.get(buckets.CompositionRunsState, key)
if err != nil {
return nil, fmt.Errorf("the state %s is not read: %w", buckets.CompositionRunsState, err)
}
if !ok {
return nil, fmt.Errorf("no validate run %s is kept", key)
}
var run meshwants.CompositionRun
if err := json.Unmarshal(raw, &run); err != nil {
return nil, fmt.Errorf("the validate run %s does not read: %w", key, err)
}
var v validateVerdict
if b, err := json.Marshal(run.Verdict); err != nil || json.Unmarshal(b, &v) != nil {
return nil, fmt.Errorf("the validate run %s holds no verdict that reads", key)
}
change, before := v.Change, v.Before
if module = strings.TrimSpace(module); module != "" {
change = composer.CaseVerdict{Name: change.Name, Manifests: map[string]composer.ManifestRead{module: change.Manifests[module]}}
}
uncounted := uncountedProblems(change, before)
if uncounted == nil {
uncounted = []string{}
}
return map[string]any{"run": key, "outcome": run.Outcome, "uncounted": uncounted}, nil
}
// validateBeforeTools are the module's tool of the validate step's before case.
func validateBeforeTools(st store) []stdio.Tool {
return []stdio.Tool{{Name: validateBeforeTool, Description: "The problems a validate step did not count (design 47 H2 §814): of the " +
"validate run of a pipeline's take-in revision, every problem of the change case its case before holds too, per manifest, " +
"for one module when named. Read-only.",
Input: schema(map[string]string{"pipeline": "the pipeline's key, <branch>.<revision>", "module": "one module; absent: every one"}, []string{"pipeline"}),
Run: func(args map[string]any) (any, error) {
return validateBefore(st, arg(args, "pipeline"), arg(args, "module"))
}}}
}
// seatListKey is the seat mesh-catalog's chunked read verb seat-list (design 47 H2 §814, H-A-441, H-A-443).
const seatListKey = "seat:" + meshcatalog.Seat + "." + meshcatalog.VerbSeatList
// validateSeats are the seat definitions a validate run judges by, as one JSON array in composer.Request.Seats' form sorted by
// seat, and their digest, the sha256 of those bytes (H-A-441, H-A-443): a revision that names its own definitions keeps them,
// each read through seat-definitions; one that names none takes the trunk's seat list, the bytes the seat mesh-catalog's verb
// seat-list hands, asked as a kept call under the run's ask.
func (r *compositionRun) validateSeats(ctx context.Context, ask string, defs []meshcatalog.CatalogRevisionDefinitionsItem) ([]json.RawMessage, string, error) {
if len(defs) == 0 {
return r.seatList(ask)
}
sorted := slices.Clone(defs)
slices.SortStableFunc(sorted, func(a, b meshcatalog.CatalogRevisionDefinitionsItem) int { return strings.Compare(a.Seat, b.Seat) })
seats, err := r.seats(ctx, sorted)
if err != nil {
return nil, "", err
}
raw, err := json.Marshal(seats)
if err != nil {
return nil, "", fmt.Errorf("the revision's seat definitions cannot be encoded: %w", err)
}
var list []json.RawMessage
if err := json.Unmarshal(raw, &list); err != nil {
return nil, "", fmt.Errorf("the revision's seat definitions do not read again: %w", err)
}
return list, seatListDigest(raw), nil
}
// seatListDigest is the digest of a seat list's bytes, as the indexer names its object in seat-lists: the sha256, hex.
func seatListDigest(raw []byte) string {
sum := sha256.Sum256(raw)
return hex.EncodeToString(sum[:])
}
// seatList is the trunk's seat list as the verb seat-list hands it (H-A-443): its digest and size, then the object's bytes in
// chunks; bytes whose size or sha256 is not the answer's, or that are no JSON array, are refused, so every run of one digest
// judges the same bytes.
func (r *compositionRun) seatList(ask string) ([]json.RawMessage, string, error) {
what := "the trunk's seat list is not read through " + seatListKey
if r.callFile == nil {
return nil, "", errors.New(what + ": the assigner holds no chunked call")
}
var bits [16]byte
if _, err := rand.Read(bits[:]); err != nil {
return nil, "", fmt.Errorf("%s: %w", what, err)
}
raw, err := r.callFile(process.KeptAsk{ID: ask + "." + meshcatalog.VerbSeatList, Bound: readsBound}, seatListKey,
meshcatalog.SeatListInput{Call: hex.EncodeToString(bits[:])})
if err != nil {
return nil, "", fmt.Errorf("%s: %w", what, err)
}
var a struct {
meshcatalog.SeatListAnswer
File string `json:"file"`
}
if err := json.Unmarshal(raw, &a); err != nil {
return nil, "", fmt.Errorf("%s: its answer does not read: %w", what, err)
}
if a.Digest == "" {
return nil, "", errors.New(what + ": its answer names no digest")
}
file, err := base64.StdEncoding.DecodeString(a.File)
if err != nil {
return nil, "", fmt.Errorf("%s: the bytes received are not base64: %w", what, err)
}
if int64(len(file)) != a.Size || seatListDigest(file) != a.Digest {
return nil, "", fmt.Errorf("%s: %d bytes of digest %s were received, the answer names %d bytes of digest %s", what, len(file),
seatListDigest(file), a.Size, a.Digest)
}
var list []json.RawMessage
if err := json.Unmarshal(file, &list); err != nil || len(list) == 0 {
return nil, "", fmt.Errorf("%s: the seat list %s is no JSON array of seat definitions", what, a.Digest)
}
return list, a.Digest, nil
}
@@ -2,8 +2,10 @@ package main
import (
"context"
"encoding/base64"
"encoding/json"
"errors"
"slices"
"strings"
"sync"
"testing"
@@ -14,6 +16,9 @@ import (
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-assigner/internal/buckets"
)
// A10/AC1, A10/AC2 (design 47 H §745; H-A-34, H-A-51, H-A-108): every mover's validate step of one take-in reads one validate
@@ -60,19 +65,40 @@ func TestTheValidateStepRefusesAManifestProblemAndACycle(t *testing.T) {
if !ok || in.Branch != "main" {
return nil, errors.New("asked another revision")
}
// Revision 3 names no definitions of its own: it is judged by the trunk's seat list, the bytes seat-list hands (H-A-443 (2)).
var defs []meshcatalog.CatalogRevisionDefinitionsItem
if in.Revision != 3 {
defs = []meshcatalog.CatalogRevisionDefinitionsItem{{Seat: "s1", Digest: "sd"}}
}
return json.Marshal(meshcatalog.RevisionAnswer{Revision: &meshcatalog.CatalogRevision{Revision: in.Revision, Base: base(in.Revision),
Definitions: []meshcatalog.CatalogRevisionDefinitionsItem{{Seat: "s1", Digest: "sd"}}}, Entries: entries})
Definitions: defs}, Entries: entries})
case seatDefinitionsKey:
return json.Marshal(meshcatalog.SeatDefinitionsAnswer{Found: true, Entry: &meshcatalog.SeatDefinitionsAnswerEntry{Seat: "s1", Digest: "sd", Definition: map[string]any{"seat": "s1"}}})
}
return nil, errors.New("no such verb " + key)
}
// The trunk's seat list as the verb seat-list hands it (H-A-443 (1)): digest and size, then the bytes, which CallFileKept adds as file.
trunkList := []byte(`[{"seat":"a-seat"},{"seat":"s1","v":2}]`)
listAnswer := func(file []byte, digest string, size int) json.RawMessage {
raw, _ := json.Marshal(map[string]any{"digest": digest, "size": size, "chunks": 1, "file": base64.StdEncoding.EncodeToString(file)})
return raw
}
var listAsks []process.KeptAsk
listed := listAnswer(trunkList, seatListDigest(trunkList), len(trunkList))
callFile := func(ask process.KeptAsk, key string, body any) (json.RawMessage, error) {
in, _ := body.(meshcatalog.SeatListInput)
if key != seatListKey || len(in.Call) != 32 {
return nil, errors.New("asked " + key + " with the call " + in.Call)
}
listAsks = append(listAsks, ask)
return listed, nil
}
var mu sync.Mutex
var asks []buildstep.StepAsk
var requests []composer.Request
keeper := testKeeper(&objectsMem{objs: map[string][]byte{}}, time.Now)
runs := newCompositionRuns(newRunsMem(), defaultRunSettings, time.Now, t.Logf)
r := &compositionRun{runs: runs, keeper: keeper, reads: &compositionReads{}, call: call, now: time.Now, logf: t.Logf,
r := &compositionRun{runs: runs, keeper: keeper, reads: &compositionReads{}, call: call, callFile: callFile, now: time.Now, logf: t.Logf,
put: func(_ context.Context, a buildstep.StepAsk) error {
body, err := keeper.body(a.Run.Input.Digest)
if err != nil {
@@ -86,8 +112,8 @@ func TestTheValidateStepRefusesAManifestProblemAndACycle(t *testing.T) {
asks, requests = append(asks, a), append(requests, req)
mu.Unlock()
answer, _ := json.Marshal(composer.Verdict{V: composer.Version, Cases: []composer.CaseVerdict{
{Name: changeCase, Manifests: map[string]composer.ManifestRead{"web": {Refused: `unknown field "colour"`}, "db": {Refused: "an old problem"}, "indexer": {Accepted: true}}},
{Name: beforeCase, Manifests: map[string]composer.ManifestRead{"db": {Refused: "an old problem"}}}}})
{Name: changeCase, Manifests: map[string]composer.ManifestRead{"web": {Refused: "unknown field \"colour\"\nan inherited problem"}, "db": {Refused: "an old problem"}, "indexer": {Accepted: true}}},
{Name: beforeCase, Manifests: map[string]composer.ManifestRead{"db": {Refused: "an old problem"}, "web": {Refused: "an inherited problem"}}}}})
go func() {
if err := runs.heardEnd(buildstep.StepEnded{Asker: runAsker, Ask: a.ID, Outcome: buildstep.OutcomePassed, Node: "novox"},
func(buildstep.StepEnded) ([]byte, error) { return answer, nil }); err != nil {
@@ -115,10 +141,21 @@ func TestTheValidateStepRefusesAManifestProblemAndACycle(t *testing.T) {
wg.Wait()
for i, o := range outs {
if o.Outcome != meshwants.StepEndedOutcomeFailed || !strings.HasPrefix(o.Said, reasonManifestRefused+": ") || !strings.Contains(o.Said, `web: unknown field "colour"`) ||
strings.Contains(o.Said, "an old problem") {
strings.Contains(o.Said, "an old problem") || strings.Contains(o.Said, "an inherited problem") { // H2 §814: only new problems
t.Errorf("mover %d's validate step ended %+v", i, o)
}
}
// assigner_validate_before (H2 §814): the problems the before case holds, which the step did not count.
runsKV := memStore{buckets.CompositionRunsState: &memKV{values: map[string][]byte{}}}
kept, _, _, _ := runs.read("main.7" + validateKeySuffix)
runsKV[buckets.CompositionRunsState].values["main.7"+validateKeySuffix], _ = json.Marshal(kept)
before, err := validateBeforeTools(runsKV)[0].Run(map[string]any{"pipeline": "main.7", "module": "web"})
if b, _ := before.(map[string]any); err != nil || !slices.Equal(b["uncounted"].([]string), []string{"web: an inherited problem"}) {
t.Errorf("assigner_validate_before answered %+v, %v", before, err)
}
if _, err := validateBeforeTools(runsKV)[0].Run(map[string]any{"pipeline": "main.6"}); err == nil {
t.Error("assigner_validate_before answers a run not kept")
}
if len(asks) != 1 {
t.Fatalf("%d validator asks were put for one take-in, want 1", len(asks))
}
@@ -132,6 +169,11 @@ func TestTheValidateStepRefusesAManifestProblemAndACycle(t *testing.T) {
if c := requests[0].Cases; len(c) != 2 || c[0].Name != changeCase || c[1].Name != beforeCase || len(c[1].Manifests) != 3 || requests[0].Predefined != nil || requests[0].Packages != nil {
t.Errorf("the validator's request is %+v", requests[0])
}
// A revision naming its own definitions keeps them, and the request names the sha256 of their bytes (H-A-443 (2)); no seat
// list is asked.
if own := []byte(`[{"seat":"s1"}]`); requests[0].SeatList != seatListDigest(own) || len(requests[0].Seats) != 1 || string(requests[0].Seats[0]) != `{"seat":"s1"}` || len(listAsks) != 0 {
t.Errorf("a revision naming its definitions was judged by %s %s (%d seat-list asks), want %s", requests[0].SeatList, requests[0].Seats, len(listAsks), seatListDigest(own))
}
if _, _, ok, _ := runs.read("main.7" + validateKeySuffix); !ok {
t.Error("the validate run is not the key <pipeline>.validate of composition-runs")
}
@@ -144,6 +186,29 @@ func TestTheValidateStepRefusesAManifestProblemAndACycle(t *testing.T) {
o.Said != reasonManifestRefused+`: web: unknown field "colour" (`+emptyBeforeWords+")" || len(asks) != 2 || len(requests[1].Cases[1].Manifests) != 0 {
t.Errorf("a revision naming no base ended %+v (%d validator asks), want it judged against an empty case before, said", o, len(asks))
}
// Revision 3 names no definitions: Seats are the trunk's seat list's bytes as seat-list handed them, named by its digest,
// asked once as a kept call under the run's ask (H-A-441 (b), H-A-443).
if got, _ := json.Marshal(requests[1].Seats); requests[1].SeatList != seatListDigest(trunkList) || string(got) != string(trunkList) ||
len(listAsks) != 1 || listAsks[0].ID != asks[1].ID+"."+meshcatalog.VerbSeatList || listAsks[0].Bound <= 0 {
t.Errorf("a revision naming no definitions was judged by %s %s (asks %+v), want the trunk's seat list %s", requests[1].SeatList, got, listAsks, seatListDigest(trunkList))
}
// Bytes the answer does not name, by size or by digest, or that are no seat list, are refused: the run judges no other bytes.
for _, row := range []struct {
name string
answer json.RawMessage
}{
{"a digest of other bytes", listAnswer(trunkList, seatListDigest([]byte("[]")), len(trunkList))},
{"another size", listAnswer(trunkList, seatListDigest(trunkList), len(trunkList)+1)},
{"no digest", listAnswer(trunkList, "", len(trunkList))},
{"no JSON array", listAnswer([]byte(`{"seat":"s1"}`), seatListDigest([]byte(`{"seat":"s1"}`)), len(`{"seat":"s1"}`))},
{"an empty list", listAnswer([]byte(`[]`), seatListDigest([]byte(`[]`)), 2)},
} {
listed = row.answer
if seats, digest, err := r.seatList("ask-row"); err == nil || !strings.Contains(err.Error(), seatListKey) {
t.Errorf("%s: the seat list read %s %s, %v; want it refused, naming the verb", row.name, digest, seats, err)
}
}
listed = listAnswer(trunkList, seatListDigest(trunkList), len(trunkList))
// Revision 12 holds revision 7's manifests and names base 7: judged against it, web's problem is the revision's, db's old one
// is not, and the case before holds base 7's manifests.
if o := r.validate(ctx, "ask-base", step("web", 12)); o.Outcome != meshwants.StepEndedOutcomeFailed ||
@@ -66,7 +66,7 @@ func DecodeCandidate(data []byte) (seat.CandidatesByProposalNodeValue, error) {
if err := keys.CheckVersion(data); err != nil {
return seat.CandidatesByProposalNodeValue{}, err
}
c, err := seat.DecodeCandidatesByProposalNodeValue(data)
c, _, err := seat.DecodeCandidatesByProposalNodeValue(data)
if err != nil {
return seat.CandidatesByProposalNodeValue{}, fmt.Errorf("assignments: a candidate does not fit its shape: %w", err)
}
@@ -428,7 +428,7 @@ func DecodeProposal(data []byte) (seat.Proposal, error) {
if err := keys.CheckVersion(data); err != nil {
return seat.Proposal{}, err
}
p, err := seat.DecodeProposalsByProposalValue(data)
p, _, err := seat.DecodeProposalsByProposalValue(data)
if err != nil {
return seat.Proposal{}, fmt.Errorf("assignments: a proposal does not fit its shape: %w", err)
}
@@ -44,6 +44,10 @@ func Inputs() string { return own(InputsState) }
func CompositionRuns() string { return own(CompositionRunsState) }
func Policies() string { return own(PoliciesState) }
// Taken is the bucket of the state taken (design 47 H2 §775, §777): the asks of the seat mesh-wants taken once, and their
// drops; named as mesh-sdk's process.TakenState.
func Taken() string { return own("taken") }
// AsksHistory is the revisions the bucket of asks keeps per key.
const AsksHistory = 5
@@ -70,9 +70,20 @@ func TestManifest(t *testing.T) {
}
if want := map[string]int{"asks": 5, "differences": 64, "puts": 20, "wants": 20,
"assignments": 64, "placement": 64, "copies": 1, "proposals": 5, "candidates": 5, "pulses": 0, "findings": 5, "policies": 20, "seat-holders": 5, views.MarksState: 1, views.StatusState: 1,
"engine-reads": 1, "written": 5, "waits": 5, "put-waits": 1, "inputs": 1, "composition-runs": 1}; !reflect.DeepEqual(states, want) {
"engine-reads": 1, "written": 5, "waits": 5, "put-waits": 1, "inputs": 1, "composition-runs": 1, "taken": 1}; !reflect.DeepEqual(states, want) {
t.Errorf("state = %v, want %v", states, want)
}
// design 47 H2 §810, §812, §814; H-A-394 (1), H-A-437: it claims a seat, so its class is core, and every state names its
// cap: 8 MiB, 16 where the live fill passed 2 MiB (findings 3.35, proposals 3.48 MiB on 2026-11-22), as set before for inputs and composition-runs.
caps := map[string]int{}
for _, s := range m.State {
caps[s.Name] = s.MaxMiB
}
if want := map[string]int{"asks": 8, "differences": 8, "puts": 8, "wants": 8,
"assignments": 8, "placement": 8, "copies": 8, "proposals": 16, "candidates": 8, "pulses": 8, "findings": 16, "policies": 8, "seat-holders": 8, views.MarksState: 8, views.StatusState: 8,
"engine-reads": 8, "written": 8, "waits": 8, "put-waits": 8, "inputs": 64, "composition-runs": 64, "taken": 8}; !reflect.DeepEqual(caps, want) || m.Class != catalogue.ClassCore {
t.Errorf("class %q, caps = %v, want core and %v", m.Class, caps, want)
}
if _, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 {
t.Errorf("own-secrets = %v, want exactly broker (or no credential is issued)", m.OwnSecrets)
}
@@ -93,7 +104,7 @@ func TestManifest(t *testing.T) {
// revision, entry and difference from its revisions kept as objects in the old section, form 1, capped at 512 MiB.
if len(m.Claims) != 2 || m.Claims[0].Name != meshmoduleassigner.Seat || m.Claims[0].At() != catalogue.ScopeMesh ||
!slices.Equal(m.Claims[0].Serves, []string{"assignments", "proposals", "propose", "withdraw", "again", "placement"}) ||
m.Claims[1].Name != meshwants.Seat || m.Claims[1].At() != catalogue.ScopeMesh || !slices.Equal(m.Claims[1].Serves, []string{"upgrade", "bus", "revision", "entry", "difference", "input", "compositions", "compositions-drop"}) {
m.Claims[1].Name != meshwants.Seat || m.Claims[1].At() != catalogue.ScopeMesh || !slices.Equal(m.Claims[1].Serves, []string{"upgrade", "bus", "revision", "entry", "difference", "input", "compositions", "compositions-drop", "keep-now", "drop"}) {
t.Errorf("the assigner claims %+v, want mesh-module-assigner with its five verbs and mesh-wants with its five", m.Claims)
}
if len(m.Objects) != 1 || m.Objects[0].Name != "assignment-revisions" || m.Objects[0].MaxMiB != 512 || m.Objects[0].Form != 1 {
@@ -105,7 +116,7 @@ func TestManifest(t *testing.T) {
// which lists and asks again what each node's engine reads.
if !slices.Equal(m.Tools, []string{"assigner_put", "assigner_release", "assigner_puts", "assigner_waits", "assigner_inputs", "assigner_inputs_drop", "assigner_compositions", "assigner_compositions_drop", "assignments", "seat-holders", "wants", "asks",
"assigner_read_assignments", "assigner_read_seat_holders", "assigner_read_wants", "assigner_read_findings", "assigner_read_differences", "assigner_read_copies",
"assigner_first_views", "assigner_first_view_again", "assigner_engine_reads", "assigner_seat_list", "assigner_seat_list_again", "assigner_ask_again", "assigner_written", "assigner_why", "assigner_want_drop"}) {
"assigner_first_views", "assigner_first_view_again", "assigner_engine_reads", "assigner_seat_list", "assigner_seat_list_again", "assigner_ask_again", "assigner_written", "assigner_why", "assigner_want_drop", "assigner_asks", "assigner_ask_drop", "assigner_keep_now", "assigner_validate_before"}) {
t.Errorf("the assigner serves tools %v, want its three put tools, three first views, six read tools, assigner_engine_reads, the seat list's two (ADR 0636 §7) assigner_ask_again (ADR 0668 §5) and assigner_why (design 47 G A-439)", m.Tools)
}
if m.RunsAs != m.Module || m.SecretsOwner != m.RunsAs {
@@ -148,13 +159,14 @@ func TestManifest(t *testing.T) {
if want := []string{"mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules",
buckets.TellerModule + "." + buckets.AnswersReadTool, // A-175, A-178: the teller's answers' first view
"seat:mesh-catalog.entry", "seat:mesh-catalog.registrations", "seat:mesh-catalog.revision", "seat:mesh-catalog.seat-definitions", // ADR 0636 §4: the seat list
"seat:mesh-catalog.seat-list", // design 47 H2 H-A-443: the validate step's trunk seat list, in chunks
"seat:mesh-conditions.conditions", // design 47 G §706: the conditions read at each judging of an outside wait
"seat:mesh-delivery.source", // §507: the branches the gate keeps a source revision of, before the first registrations
"seat:mesh-hand-acts.hand-act", "seat:mesh-hand-acts.hand-acts", "seat:mesh-hand-acts.warranted", // ADR 0565 §9, §11: the seat alone
"seat:node-backup.backed-up", "seat:node-backup.now",
"seat:node-build-go.accept.check", // A-231
"seat:node-build-go.answer", // A-254, H-A-412: a composition run's answer over the inline bound, read through the build seat's verb answer
"seat:node-build-go.drop", // §720, A-469s: a composition run past its bound drops its ask: the composition run's ask on the Go build seat's queue check (accept.check.<asker>)
"seat:node-build-go.output", // A-254: a composition run's answer kept as the build seat's object
"seat:node-engine.compose-candidate", // §354, A-226: each machine's composition input
"seat:operator-channel.asks"}; !slices.Equal(slices.Sorted(slices.Values(m.Invokes)), want) { // ADR 0668 §4: the ended asks at bind
t.Errorf("invokes = %v, want %v", m.Invokes, want)
@@ -172,9 +184,11 @@ func TestManifest(t *testing.T) {
}
file, _ := os.ReadFile("module.json")
if err := json.Unmarshal(file, &raw); err != nil || raw.Config["outside-wait-minutes"].Default != float64(30) || raw.Config["outside-settle-seconds"].Default != float64(90) ||
raw.Config["ask-operator-for"].Default != "take-off,remove-node,stops" || len(raw.Config) != 10 || raw.Config["tier-bound"].Default != float64(30) || raw.Config["entry-asks"].Default != float64(5) ||
raw.Config["ask-operator-for"].Default != "take-off,remove-node,stops" || len(raw.Config) != 15 || raw.Config["tier-bound"].Default != float64(30) || raw.Config["entry-asks"].Default != float64(5) ||
raw.Config["composition-runs-cap-mib"].Default != float64(16) || raw.Config["composition-runs-kept"].Default != "24h" ||
raw.Config["composition-step-bound"].Default != "30m" || raw.Config["run-untaken-bound"].Default != "24h" || raw.Config["blame-tries"].Default != float64(20) {
raw.Config["composition-step-bound"].Default != "30m" || raw.Config["run-untaken-bound"].Default != "24h" || raw.Config["blame-tries"].Default != float64(20) ||
// design 47 H2 §775, H-A-381 (2): the taken key's bound and the kept-call pace, runtime settings
raw.Config["ask-taken-bound"].Default != "10m" || raw.Config["kept-call-pace-first"].Default != "1s" || raw.Config["kept-call-pace-max"].Default != "30s" || raw.Config["compose-ask-bound"].Default != "10s" || raw.Config["keeping-retry"].Default != "30s" {
t.Errorf("config = %+v (%v), want outside-wait-minutes of 30, outside-settle-seconds of 90 and ask-operator-for of take-off,remove-node,stops (A-425a)", raw.Config, err)
}
for _, f := range []struct{ env, path, key string }{
+67 -23
View File
@@ -1,6 +1,7 @@
{
"module": "mesh-delivery-assigner",
"version": "0.3",
"class": "core",
"slug": "assigner",
"runs-as": "mesh-delivery-assigner",
"claims": [
@@ -27,7 +28,9 @@
"difference",
"input",
"compositions",
"compositions-drop"
"compositions-drop",
"keep-now",
"drop"
]
}
],
@@ -70,6 +73,7 @@
"seat:mesh-catalog.registrations",
{"verb": "seat:mesh-catalog.revision", "fields": ["none-yet"]},
"seat:mesh-catalog.seat-definitions",
"seat:mesh-catalog.seat-list",
"seat:mesh-conditions.conditions",
"seat:mesh-delivery.source",
"seat:mesh-hand-acts.hand-act",
@@ -77,7 +81,7 @@
"seat:mesh-hand-acts.warranted",
"seat:node-backup.backed-up",
"seat:node-backup.now",
"seat:node-build-go.output",
"seat:node-build-go.answer",
"seat:node-build-go.drop",
"seat:node-engine.compose-candidate",
"seat:operator-channel.asks",
@@ -129,30 +133,35 @@
"assigner_ask_again",
"assigner_written",
"assigner_why",
"assigner_want_drop"
"assigner_want_drop",
"assigner_asks",
"assigner_ask_drop",
"assigner_keep_now",
"assigner_validate_before"
],
"state": [
{ "name": "assignments", "history": 64 },
{ "name": "placement", "history": 64 },
{ "name": "copies", "history": 1 },
{ "name": "proposals", "history": 5 },
{ "name": "candidates", "history": 5 },
{ "name": "pulses", "key-ttl": true },
{ "name": "asks", "history": 5 },
{ "name": "differences", "history": 64 },
{ "name": "puts", "history": 20 },
{ "name": "wants", "history": 20 },
{ "name": "findings", "history": 5 },
{ "name": "policies", "history": 20 },
{ "name": "seat-holders", "history": 5 },
{ "name": "view-marks", "history": 1 },
{ "name": "first-views", "history": 1 },
{ "name": "engine-reads", "history": 1 },
{ "name": "written", "history": 5 },
{ "name": "waits", "history": 5 },
{ "name": "put-waits", "history": 1 },
{ "name": "assignments", "history": 64, "max-mib": 8 },
{ "name": "placement", "history": 64, "max-mib": 8 },
{ "name": "copies", "history": 1, "max-mib": 8 },
{ "name": "proposals", "history": 5, "max-mib": 16 },
{ "name": "candidates", "history": 5, "max-mib": 8 },
{ "name": "pulses", "key-ttl": true, "max-mib": 8 },
{ "name": "asks", "history": 5, "max-mib": 8 },
{ "name": "differences", "history": 64, "max-mib": 8 },
{ "name": "puts", "history": 20, "max-mib": 8 },
{ "name": "wants", "history": 20, "max-mib": 8 },
{ "name": "findings", "history": 5, "max-mib": 16 },
{ "name": "policies", "history": 20, "max-mib": 8 },
{ "name": "seat-holders", "history": 5, "max-mib": 8 },
{ "name": "view-marks", "history": 1, "max-mib": 8 },
{ "name": "first-views", "history": 1, "max-mib": 8 },
{ "name": "engine-reads", "history": 1, "max-mib": 8 },
{ "name": "written", "history": 5, "max-mib": 8 },
{ "name": "waits", "history": 5, "max-mib": 8 },
{ "name": "put-waits", "history": 1, "max-mib": 8 },
{ "name": "inputs", "history": 1, "max-mib": 64 },
{ "name": "composition-runs", "history": 1, "key-ttl": true, "max-mib": 64 }
{ "name": "composition-runs", "history": 1, "key-ttl": true, "max-mib": 64 },
{ "name": "taken", "history": 1, "key-ttl": true, "max-mib": 8 }
],
"config": {
"outside-wait-minutes": {
@@ -226,6 +235,41 @@
"maximum": 1000,
"trusted": false,
"default": 20
},
"ask-taken-bound": {
"type": "string",
"description": "how long the assigner keeps the id of an ask it took from the seat mesh-wants' work queue, so a repeat of it is taken once, and the drop of an ask not pulled yet, a Go duration such as 10m (design 47 H2 §775, §778; H-A-381 (2), H-A-404).",
"trusted": false,
"default": "10m",
"runtime": true
},
"compose-ask-bound": {
"type": "string",
"description": "how long the assigner asks a machine's node-engine compose-candidate again through a transport fault before the machine gave no input, a Go duration such as 10s (design 47 H2 §775).",
"trusted": false,
"default": "10s",
"runtime": true
},
"keeping-retry": {
"type": "string",
"description": "how long after a failed keeping of the assignment graph revisions the assigner keeps them again; keeping-failed is raised at the third failure in a row, a Go duration such as 30s (design 47 H2 §813 (2)).",
"trusted": false,
"default": "30s",
"runtime": true
},
"kept-call-pace-first": {
"type": "string",
"description": "the first wait before a kept call met by a transport fault is made again, doubled per call up to kept-call-pace-max, a Go duration (design 47 H2 §775).",
"trusted": false,
"default": "1s",
"runtime": true
},
"kept-call-pace-max": {
"type": "string",
"description": "the most a kept call's wait between calls grows to, a Go duration (design 47 H2 §775).",
"trusted": false,
"default": "30s",
"runtime": true
}
},
"objects": [
@@ -57,8 +57,8 @@ type Asker struct {
Expiry *Expiry
// Logf says a drop that failed (the log's Printf when unset): the ask then stays on its stream, and it is said.
Logf func(format string, args ...any)
// Outputs reads the answer a step's key names as its output-object, being over what the steps state holds (design 47 G
// A-290), through the build seat's verb output on the step's node (buildstep.Step.Whole); Wait answers the step whole.
// Outputs reads the answer a step's key names as its answer-object, being over what the steps state holds (design 47 G
// A-290), through the build seat's verb answer on the step's node (buildstep.Step.Whole; H-A-412); Wait answers the step whole.
// Nil reads none: such a step's wait is an error saying so.
Outputs buildstep.AskVerb
}
@@ -337,7 +337,7 @@ func (k Asker) Wait(ctx context.Context, a buildstep.StepAsk) (buildstep.Step, e
}
whole, err := step.Whole(a.Language, a.Run != nil, k.Outputs)
if err != nil {
return buildstep.Step{}, fmt.Errorf("the ask %s's step on %s ended %s, and its answer %s cannot be read: %w", a.ID, step.Node, step.State, step.OutputObject, err)
return buildstep.Step{}, fmt.Errorf("the ask %s's step on %s ended %s, and its answer %s cannot be read: %w", a.ID, step.Node, step.State, step.AnswerObject, err)
}
return whole, nil
}
@@ -55,17 +55,17 @@ func TestAnAskerPutsOnceAndReadsTheStep(t *testing.T) {
if _, err := k.Ask(context.Background(), goAsk("mvl0frv18d1958")); err == nil || !strings.Contains(err.Error(), "shanks refused: go 1.27 installed") {
t.Fatalf("a refused step: %v", err)
}
// Design 47 G A-290: a step whose key names its answer as the object output.<ask> is answered whole, read through the
// build seat's verb output on the step's node; with no verb to read it the wait says so, never the step without its answer.
// Design 47 G A-290, H-A-412: a step whose key names its answer as the object answer.<ask> is answered whole, read through the
// build seat's verb answer on the step's node; with no verb to read it the wait says so, never the step without its answer.
f.Answer = func(a buildstep.StepAsk) buildstep.Step {
return buildstep.Step{ID: a.ID, Node: "ace", State: buildstep.StepPassed, OutputObject: buildstep.OutputObjectName(a.ID)}
return buildstep.Step{ID: a.ID, Node: "ace", State: buildstep.StepPassed, AnswerObject: buildstep.AnswerObjectName(a.ID)}
}
var asked string
k.Outputs = func(key string, body any) (json.RawMessage, error) {
asked = key
return json.RawMessage(`{"ask":"` + body.(map[string]any)["ask"].(string) + `","file":"eyJhcnRpZmFjdHMiOltdfQ=="}`), nil
}
if step, err := k.Ask(context.Background(), goAsk("mvl0frv18d1961")); err != nil || string(step.Result) != `{"artifacts":[]}` || step.OutputObject != "" || asked != "seat:node-build-go.output@ace" {
if step, err := k.Ask(context.Background(), goAsk("mvl0frv18d1961")); err != nil || string(step.Result) != `{"artifacts":[]}` || step.AnswerObject != "" || asked != "seat:node-build-go.answer@ace" {
t.Fatalf("a step whose answer is an object: %+v, %v, asked %q", step, err, asked)
}
k.Outputs = nil
@@ -4,7 +4,6 @@ import (
"strings"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
)
// The words of a key that failed on a transport fault (design 47 G §568, A-76): `transport fault: <class>, <seat> <node>,
@@ -23,8 +22,9 @@ const (
)
// FaultClass is the class of a call's error or a step's reason: the words the builder's retake reads a transport fault by
// (buildstep.PassingCause), read as no responders, a name lookup, a deadline or, for every other refused, reset or cut
// connection, a connection.
// (buildstep.PassingCause), read as no responders (a registry's bundle that exited included, design 47 H2 §799), a name
// lookup, a deadline or, for every other refused, reset or cut connection (a forge's 401 or 5xx included), a connection.
// A build module's why keeps its part, `fetch: ` or `publish: ` (§799), in the cause TransportFault says.
func FaultClass(cause string) string {
c := strings.ToLower(cause)
has := func(words ...string) bool {
@@ -36,7 +36,7 @@ func FaultClass(cause string) string {
return false
}
switch {
case has("no responders", strings.ToLower(buildstep.HolderSilentWords), "jetstream not enabled"):
case has("no responders", "bundle exited", "jetstream not enabled"):
return ClassNoResponders
case has("no such host", "name resolution", "lookup ", "could not resolve", "enotfound", "eai_again"):
return ClassNameLookup
@@ -1,17 +1,23 @@
package buildrun
import "testing"
import (
"strings"
"testing"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
)
// Design 47 G §568, A-76: the class of a transport fault is read from the call's error, and its words are one form.
func TestFaultClassAndWords(t *testing.T) {
for cause, class := range map[string]string{
"nats: no responders available for request": ClassNoResponders,
"the holder is silent: seat x answered": ClassNoResponders,
"lookup proxy: no such host": ClassNameLookup,
"Get https://r: context deadline exceeded": ClassDeadline,
"read tcp: i/o timeout": ClassDeadline,
"dial tcp: connection refused": ClassConnection,
"ssh: kex_exchange_identification: read: reset": ClassConnection,
"nats: no responders available for request": ClassNoResponders,
"publish: the bundle exited": ClassNoResponders,
"fetch: git fetch: The requested URL returned error: 401": ClassConnection,
"lookup proxy: no such host": ClassNameLookup,
"Get https://r: context deadline exceeded": ClassDeadline,
"read tcp: i/o timeout": ClassDeadline,
"dial tcp: connection refused": ClassConnection,
"ssh: kex_exchange_identification: read: reset": ClassConnection,
} {
if got := FaultClass(cause); got != class {
t.Errorf("FaultClass(%q) = %q, want %q", cause, got, class)
@@ -20,4 +26,10 @@ func TestFaultClassAndWords(t *testing.T) {
if got, want := TransportFault(ClassDeadline, "node-build-go", "ace", "i/o timeout"), "transport fault: deadline, node-build-go ace, i/o timeout"; got != want {
t.Errorf("TransportFault = %q, want %q", got, want)
}
// §799: a fetch's and a publish's why are said apart, each in the cause of its key's transport fault.
for _, why := range []string{"fetch: git fetch: HTTP 503", "publish: nats: no responders available for request"} {
if got := TransportFault(FaultClass(why), "node-build-go", "ace", why); !builds.FailedOnTransportFault(got) || !strings.HasSuffix(got, ", "+why) {
t.Errorf("TransportFault of %q = %q", why, got)
}
}
}
@@ -26,7 +26,7 @@ func (s *PresenceSet) Put(key string, value []byte, deleted bool) bool {
}
on := false
if !deleted {
p, err := meshwatchdogs.DecodePresenceByNodeValue(value)
p, _, err := meshwatchdogs.DecodePresenceByNodeValue(value)
if err != nil {
return false
}
@@ -10,6 +10,7 @@ import (
"reflect"
"slices"
"strings"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds/forms"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/containerimageregistryclient/reference"
@@ -260,6 +261,11 @@ type BuilderBuild struct {
// time at most, then asks the operator. A person's build (builder_build_again, the operator's answer) starts the count
// again. A reader that does not know the field reads none.
EventBuilds int `json:"event-builds,omitempty"`
// Ended and Outcome are, on a done or failed key, when the build ended and how (done or failed), written with its end, so a
// reader of the builds (the artifact store's sweep) reads no key history (design 47 H2 §797). A reader that does not know the
// fields reads the key as before; a key written before them names neither.
Ended time.Time `json:"ended,omitzero"`
Outcome string `json:"outcome,omitempty"`
}
func (b BuilderBuild) check() error {
@@ -0,0 +1,131 @@
package main
import (
"encoding/json"
"errors"
"fmt"
"sort"
"strings"
"time"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
stdio "git.novox.be/novox/mesh-sdk/go"
"mesh-delivery-builder/internal/build"
)
// dropVerbKey is the seat verb builder_ask_drop asks (the manifest's invokes).
const dropVerbKey = "seat:" + meshbuilds.Seat + "." + meshbuilds.VerbDrop
// takenReader is the builder's state taken, as the bundle reads it (stdio.State).
type takenReader interface {
Keys() ([]string, error)
Get(key string) (*stdio.StateEntry, error)
}
// TakenAsk is an ask of the seat mesh-builds the builder took (its id in the state taken, kept for ask-taken-bound), or
// whose drop it keeps before its pull, as builder_asks shows it.
type TakenAsk struct {
Ask string `json:"ask"`
At time.Time `json:"at,omitzero"`
By string `json:"by,omitempty"`
Why string `json:"why,omitempty"`
}
// WaitingRow is an ask the runtime reads as waiting on the seat's work queue, as builder_asks shows it: State is
// waiting (taken by no holder yet), waiting again (handed out before and given back, its id not taken) or held (handed
// out again while the builder holds its id in the state taken: its run answers it).
type WaitingRow struct {
stdio.WaitingAsk
State string `json:"state"`
}
// The states of a waiting ask.
const (
StateWaiting = "waiting"
StateWaitingAgain = "waiting again"
StateHeld = "held"
)
// asksTools are builder_asks, what waits on and is taken from the seat mesh-builds' work queue: the waiting asks read
// through the runtime (stdio.QueueWaiting, H-A-410), the taken ones and the drops kept before a pull from the builder's
// state taken; and builder_ask_drop, which ends one ask through the seat's verb drop once the hand act is recorded
// (design 47 H2 §777, §778; H-A-401, H-A-404, H-A-410).
func asksTools(waiting func(seat string) (stdio.Waiting, error), taken takenReader, drop func(meshbuilds.DropInput) (json.RawMessage, error), now func() time.Time, acts meshhandactsclient.Recorder) []stdio.Tool {
str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) }
return []stdio.Tool{
{Name: build.AsksTool,
Description: "What waits on and is taken from the seat mesh-builds' work queue: waiting, each build step ask the queue holds unacknowledged, oldest " +
"first, with its id, the module that asked it, its age and its state: " + StateWaiting + " (taken by no holder yet), " + StateWaitingAgain +
" (handed out before and given back) or " + StateHeld + " (its id taken by the builder, its run answers it); capped when the runtime cut the read; taken, each ask the builder took " +
"within ask-taken-bound, with when; dropped, each drop kept for an ask not pulled yet, with who dropped it and why. Read-only.",
Input: map[string]any{"type": "object", "properties": map[string]any{}},
Run: func(map[string]any) (any, error) {
w, err := waiting(meshbuilds.Seat)
if err != nil {
return nil, fmt.Errorf("the waiting asks of %s are not read: %w", meshbuilds.Seat, err)
}
keys, err := taken.Keys()
if err != nil {
return nil, fmt.Errorf("the state taken is not read: %w", err)
}
sort.Strings(keys)
tk, dr := []TakenAsk{}, []TakenAsk{}
for _, k := range keys {
e, err := taken.Get(k)
if err != nil || e == nil {
continue
}
if id, ok := strings.CutPrefix(k, build.DroppedPrefix); ok {
var d build.Dropped
_ = json.Unmarshal(e.Value, &d)
dr = append(dr, TakenAsk{Ask: id, At: e.At, By: d.By, Why: d.Why})
} else if id, ok := strings.CutPrefix(k, meshbuilds.Seat+"."); ok {
tk = append(tk, TakenAsk{Ask: id, At: e.At})
}
}
held := map[string]bool{}
for _, a := range tk {
held[a.Ask] = true
}
rows := make([]WaitingRow, 0, len(w.Asks))
for _, a := range w.Asks {
r := WaitingRow{WaitingAsk: a, State: StateWaiting}
if held[a.ID] {
r.State = StateHeld
} else if a.Again {
r.State = StateWaitingAgain
}
rows = append(rows, r)
}
return map[string]any{"seat": meshbuilds.Seat, "waiting": rows, "capped": w.Capped, "taken": tk, "dropped": dr}, nil
}},
{Name: build.AskDropTool,
Description: "End one ask on the seat mesh-builds' work queue through the seat's verb drop: an ask the builder runs is cancelled, one not " +
"pulled yet is kept dropped for ask-taken-bound; either way its step ends could-not-run naming the drop and who asked it. Written in " +
"the hand-act log (mesh-hand-acts).",
Input: map[string]any{"type": "object", "required": []string{"ask", "why"}, "properties": map[string]any{
"ask": map[string]any{"type": "string", "description": "the ask's id"},
"why": map[string]any{"type": "string", "description": "why it is dropped, in plain words"}}},
Run: func(args map[string]any) (any, error) {
ask, why := str(args, "ask"), str(args, "why")
if ask == "" || why == "" {
return nil, errors.New("give the ask's id and why it is dropped")
}
if _, err := acts.Record(meshhandactsclient.Request{What: fmt.Sprintf("dropped the ask %s of the seat %s by hand", ask, meshbuilds.Seat),
Why: why, Cause: build.AskDropTool, Verb: meshbuilds.VerbDrop}); err != nil {
return nil, fmt.Errorf("the hand act was not recorded through mesh-hand-acts, so nothing was dropped: %w", err)
}
raw, err := drop(meshbuilds.DropInput{ID: ask, Why: why, By: stdio.CallerOf(args)})
if err != nil {
return nil, fmt.Errorf("the seat %s's drop did not answer for %s: %w", meshbuilds.Seat, ask, err)
}
var a meshbuilds.DropAnswer
if err := json.Unmarshal(raw, &a); err != nil {
return nil, err
}
return map[string]any{"ask": ask, "dropped": true, "held": a["held"]}, nil
}},
}
}
@@ -0,0 +1,141 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"sort"
"strings"
"sync"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/seatlist"
"git.novox.be/novox/mesh-sdk/go/views"
)
// BuildCurrentTool is build_current (design 47 H2 §801).
const BuildCurrentTool = "build_current"
// buildCurrentDescription is build_current's description.
const buildCurrentDescription = "What every holder of a build seat takes and runs now (design 47 H2 section 801): the holders named by the " +
"seat-holders view, each build seat on each node, and each holder's own answer to its seat's verb current, asked in turn and " +
"answered together: whether it takes asks and why not (paused, capacity 0, unhealthy) since when, and every ask it runs, whoever " +
"asked it. A holder that does not answer is listed with why. Read-only."
// seatHoldersState is the assigner's view of who holds which seat on which node (ADR 0571 §6).
const seatHoldersState = "seat-holders"
// heldBuildSeat is one holder of a build seat as build_current answers it: the seat, its node and module, and the holder's
// answer to current, or why there is none.
type heldBuildSeat struct {
Seat string `json:"seat"`
Node string `json:"node"`
Module string `json:"module,omitempty"`
Current json.RawMessage `json:"current,omitempty"`
Error string `json:"error,omitempty"`
}
// buildCurrentTool is build_current (§801; H-A-446 (1)): the holders of each build seat (seats) named from the seat-holders view
// (holders, keyed `<seat>.<node>`), each holder's current asked in turn at its node through ask (stdio.Ask), answered together;
// it reads no presence and chooses no node.
func buildCurrentTool(seats func() []string, holders func() (map[string]json.RawMessage, error), ask func(key string, body any) (json.RawMessage, error)) stdio.Tool {
return stdio.Tool{Name: BuildCurrentTool, Description: buildCurrentDescription,
Input: map[string]any{"type": "object", "properties": map[string]any{}},
Run: func(map[string]any) (any, error) {
view, err := holders()
if err != nil {
return nil, err
}
var out []heldBuildSeat
for _, seat := range seats() {
for key, raw := range view {
node, ok := strings.CutPrefix(key, seat+".")
if !ok || node == "" {
continue
}
var held struct {
Holders []struct {
Module string `json:"module"`
} `json:"holders"`
}
if json.Unmarshal(raw, &held) != nil || len(held.Holders) == 0 {
continue
}
out = append(out, heldBuildSeat{Seat: seat, Node: node, Module: held.Holders[0].Module})
}
}
sort.Slice(out, func(i, j int) bool {
if out[i].Seat != out[j].Seat {
return out[i].Seat < out[j].Seat
}
return out[i].Node < out[j].Node
})
for i := range out { // in turn, one holder at a time
raw, err := ask("seat:"+out[i].Seat+".current@"+out[i].Node, map[string]any{})
if err != nil {
out[i].Error = err.Error()
continue
}
out[i].Current = raw
}
if out == nil {
out = []heldBuildSeat{}
}
return map[string]any{"holders": out, "count": len(out)}, nil
}}
}
// seatHolders follows the seat-holders view of the claimant of the seat mesh-module-assigner once the catalog half names it.
type seatHolders struct {
mu sync.Mutex
values func() map[string]json.RawMessage
why string
}
// follow starts the follow when the bundle's first views stand (the claimant is read from the catalog half).
func (s *seatHolders) follow(ctx context.Context, branches *meshcatalogclient.Branches, r *views.Reader) {
s.mu.Lock()
s.why = "the catalog half's first views are not read yet"
s.mu.Unlock()
go func() {
select {
case <-ctx.Done():
return
case <-branches.FirstViews(ctx):
}
claimant, ok := branches.Claimant(seatlist.HolderSeat)
if !ok {
s.mu.Lock()
s.why = "no module claims the seat " + seatlist.HolderSeat + " on the catalog half"
s.mu.Unlock()
return
}
w, ok := meshcatalogclient.ReaderWiring(r).(interface {
Follow(owner, state, tool string) (func() map[string]json.RawMessage, error)
})
var values func() map[string]json.RawMessage
err := fmt.Errorf("the bundle's reader follows no plain view")
if ok {
values, err = w.Follow(claimant, seatHoldersState, claimant+"."+seatHoldersState)
}
s.mu.Lock()
defer s.mu.Unlock()
if err != nil {
s.why = fmt.Sprintf("the seat-holders view of %s is not followed: %v", claimant, err)
return
}
s.values, s.why = values, ""
}()
}
// read is the view's values, or why it is not read.
func (s *seatHolders) read() (map[string]json.RawMessage, error) {
s.mu.Lock()
defer s.mu.Unlock()
if s.values == nil {
return nil, errors.New("the seat-holders view is not read: " + s.why)
}
return s.values(), nil
}
@@ -0,0 +1,40 @@
package main
import (
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/health"
"mesh-delivery-builder/internal/versions"
)
const (
// HealthTool is builder_health, the builder's health as its manifest's health asks it (design 47 H2 §794, H-A-435).
HealthTool = "builder_health"
// eventsBehindBoundSetting is the setting events-behind-bound (§794; H-A-461 (2), P11): how long a durable consumer's
// oldest unacknowledged message may wait before the health names it, an events backlog or a handler standing still on one
// message; defaultEventsBehindBound when none is set.
eventsBehindBoundSetting = "events-behind-bound"
defaultEventsBehindBound = 5 * time.Minute
)
// healthTool is builder_health: healthy when it answers, and each of the builder's durable consumers' oldest unacknowledged
// message waits less than the setting events-behind-bound, as the consumers' facts the builder writes at every delivery and
// every acknowledgement say them (its state consumers, the runner's runner_health form; H-A-435 and its addendum); otherwise
// unhealthy, naming each consumer behind and how long. A setting that holds no duration is said as the health's answer.
func healthTool(consumers versions.Reader, now func() time.Time) stdio.Tool {
return stdio.Tool{Name: HealthTool,
Description: "The builder's health: healthy when it answers and each of its durable consumers' oldest unacknowledged message waits less " +
"than the setting events-behind-bound (five minutes by default; no events backlog, no handler standing still on a message); otherwise it " +
"names each consumer behind and how long. Read from the builder's state consumers, which it writes at every message delivered and " +
"acknowledged. Read-only.",
Input: map[string]any{"type": "object", "properties": map[string]any{}},
Run: func(map[string]any) (any, error) {
bound, err := stdio.SettingDuration(eventsBehindBoundSetting, defaultEventsBehindBound)
if err != nil {
return health.Unhealthy("The builder's " + err.Error() + "."), nil
}
return versions.BuilderHealth(consumers, now(), bound), nil
}}
}
@@ -21,6 +21,7 @@ import (
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/seatdef"
"git.novox.be/novox/mesh-sdk/go/seatlist"
"git.novox.be/novox/mesh-sdk/go/views"
@@ -432,8 +433,9 @@ func main() {
tools = append(tools, buildTool(&build.BuildReader{Builds: read(stdio.State("builds")), Entry: entryOf(stdio.State("builds")), Asked: read(stdio.State("asked")),
Newest: newestVia(stdio.Ask), Steps: r}))
tools = append(tools, reservedTools(rr, buildAgain, time.Now, meshhandactsclient.Recorder{Ask: stdio.Ask})...)
qs, as := stdio.State(versions.PublishQueueState), stdio.State(versions.ActedState)
tools = append(tools, republishTools(readerOf(qs), readerOf(as), readerOf(reasks), buildAgain, time.Now, meshhandactsclient.Recorder{Ask: stdio.Ask})...)
as := stdio.State(versions.ActedState)
tools = append(tools, publishActTools(buildAgain, time.Now, meshhandactsclient.Recorder{Ask: stdio.Ask})...)
tools = append(tools, republishTools(publishQueueVia(stdio.Ask), readerOf(as), readerOf(reasks), buildAgain, time.Now, meshhandactsclient.Recorder{Ask: stdio.Ask})...)
ws := stdio.State(build.WaitsState)
wr := versions.Reader{Keys: ws.Keys, Get: func(key string) (json.RawMessage, bool, error) {
e, err := ws.Get(key)
@@ -442,7 +444,12 @@ func main() {
}
return e.Value, true, nil
}}
tools = append(tools, asksTools(stdio.QueueWaiting, stdio.State(stdio.TakenState), func(in meshbuilds.DropInput) (json.RawMessage, error) {
return stdio.Ask(dropVerbKey, in)
}, time.Now, meshhandactsclient.Recorder{Ask: stdio.Ask})...)
tools = append(tools, waitsTools(wr, buildAgain, time.Now, meshhandactsclient.Recorder{Ask: stdio.Ask})...)
tools = append(tools, unbuildableTool(unbuildablePage(stdio.State(build.UnbuildableState)))) // the plans no seat builds (§795)
tools = append(tools, healthTool(readerOf(stdio.State(process.ConsumersState)), time.Now)) // the builder's health (§794, H-A-435)
tools = append(tools, toolchainsTools(stdio.State(buildrun.ToolchainsHeardState), meshhandactsclient.Recorder{Ask: stdio.Ask}, time.Now)...)
// The builder's expiring keys (state bounds): what each bounds and when it ends.
tools = append(tools, stdio.Tool{Name: versions.BoundsTool, Description: versions.BoundsDescription,
@@ -458,6 +465,16 @@ func main() {
// states stand, and asking one again by hand (ADR 0615, DLV-M1).
tools = append(tools, views.StatusTools(os.Getenv(stdio.ServedAs), "builder_first_views", "builder_first_view_again", stdio.State(views.StatusState), meshhandactsclient.Recorder{Ask: stdio.Ask}, time.Now)...)
tools = append(tools, gate.Tools()...)
// build_current (design 47 H2 §801; H-A-446 (1)): each build seat's holders from the seat-holders view, each asked current.
holders := &seatHolders{}
holders.follow(context.Background(), branches, pk)
tools = append(tools, buildCurrentTool(func() []string {
var out []string
for _, m := range r.Seats.All() {
out = append(out, m.Seat)
}
return out
}, holders.read, stdio.Ask))
if err := stdio.Serve("", tools); err != nil {
fmt.Fprintf(os.Stderr, "[mesh-delivery-builder-tools] %v\n", err)
os.Exit(1)
@@ -8,6 +8,7 @@ import (
"time"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
stdio "git.novox.be/novox/mesh-sdk/go"
"mesh-delivery-builder/internal/build"
@@ -53,10 +54,68 @@ func onePage(page func(after string, maxBytes int, budget time.Duration) (stdio.
return out, nil
}
// publishQueueVerbKey is the seat verb builder_publish_queue asks (the manifest's invokes): the builder reads its own work queue
// publishes (design 47 H2 §793, H-A-454).
const publishQueueVerbKey = "seat:" + meshbuilds.Seat + "." + meshbuilds.VerbPublishQueue
// publishQueueVia asks the seat mesh-builds' verb publish-queue through ask (stdio.Ask).
func publishQueueVia(ask func(key string, body any) (json.RawMessage, error)) func(module, version string) (meshbuilds.PublishQueueAnswer, error) {
return func(module, version string) (meshbuilds.PublishQueueAnswer, error) {
var a meshbuilds.PublishQueueAnswer
raw, err := ask(publishQueueVerbKey, meshbuilds.PublishQueueInput{Module: module, Version: version})
if err != nil {
return a, fmt.Errorf("the builder's verb %s did not answer: %w", meshbuilds.VerbPublishQueue, err)
}
if err := json.Unmarshal(raw, &a); err != nil {
return a, fmt.Errorf("the verb %s answered what is no queue: %w", meshbuilds.VerbPublishQueue, err)
}
return a, nil
}
}
// publishActTools are builder_publish_drop and builder_publish_first (design 47 H2 §793): hand acts, recorded in the hand-act log
// first, then written to reasks for the builder, which owns the work queue, to perform and answer.
func publishActTools(ask queueAsker, now func() time.Time, acts meshhandactsclient.Recorder) []stdio.Tool {
str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) }
tool := func(name, mode, desc, what string) stdio.Tool {
return stdio.Tool{Name: name, Description: desc,
Input: map[string]any{"type": "object", "required": []string{"module", "version", "why"}, "properties": map[string]any{
"module": map[string]any{"type": "string", "description": "the item's module"},
"version": map[string]any{"type": "string", "description": "the item's version, MAJOR.MINOR.PATCH"},
"why": map[string]any{"type": "string", "description": "why, in plain words"}}},
Run: func(args map[string]any) (any, error) {
key, value, err := build.NewPublishAct(mode, str(args, "module"), str(args, "version"), str(args, "why"), stdio.CallerOf(args), now())
if err != nil {
return nil, err
}
if _, err := acts.Record(meshhandactsclient.Request{What: fmt.Sprintf("%s %s %s by hand", what, str(args, "module"), str(args, "version")),
Why: str(args, "why"), Cause: name, Verb: mode}); err != nil {
return nil, fmt.Errorf("the hand act was not recorded through mesh-hand-acts, so nothing was done: %w", err)
}
raw, err := ask(key, value)
if err != nil {
return nil, err
}
a, err := build.DecodeReAsk(raw)
if err != nil {
return nil, err
}
if a.Refused {
return nil, errors.New(a.Answer)
}
return map[string]any{"key": key, "answer": a.Answer}, nil
}}
}
return []stdio.Tool{
tool(versions.PublishDropTool, build.ModePublishDrop, versions.PublishDropDescription, "dropped the publish of"),
tool(versions.PublishFirstTool, build.ModePublishFirst, versions.PublishFirstDescription, "put first the publish of"),
}
}
// republishTools are builder_publish_again and builder_publish_queue (design 47 G §588). The publish again is a hand act:
// it is recorded in the hand-act log first, then written to reasks for the builder, which owns the queue, to answer
// with the versions it queued.
func republishTools(queue, acted, reasks versions.Reader, ask queueAsker, now func() time.Time, acts meshhandactsclient.Recorder) []stdio.Tool {
func republishTools(queue func(module, version string) (meshbuilds.PublishQueueAnswer, error), acted, reasks versions.Reader, ask queueAsker, now func() time.Time, acts meshhandactsclient.Recorder) []stdio.Tool {
str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) }
return []stdio.Tool{
{Name: versions.PublishAgainTool, Description: versions.PublishAgainDescription,
@@ -91,7 +150,11 @@ func republishTools(queue, acted, reasks versions.Reader, ask queueAsker, now fu
"module": map[string]any{"type": "string", "description": "list only this module's queued versions, each with its class"},
"version": map[string]any{"type": "string", "description": "with module: only this version, MAJOR.MINOR.PATCH"}}},
Run: func(args map[string]any) (any, error) {
return versions.AnswerQueue(queue, acted, reasks, str(args, "module"), str(args, "version"))
q, err := queue(str(args, "module"), str(args, "version"))
if err != nil {
return nil, err
}
return versions.AnswerQueue(q, acted, reasks)
}},
}
}
@@ -3,10 +3,19 @@ package main
import (
"encoding/json"
"errors"
"fmt"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/health"
"mesh-delivery-builder/internal/build"
"mesh-delivery-builder/internal/versions"
)
// TestAStateIsReadInOnePage: design 47 H H-A-292, H-A-293 — builder_publish_queue reads each of its own states in one page and
@@ -41,4 +50,102 @@ func TestAStateIsReadInOnePage(t *testing.T) {
if _, err := onePage(unsupported, nil); !errors.Is(err, errors.ErrUnsupported) {
t.Errorf("a runtime with no pages answered %v, want ErrUnsupported so the reader reads key by key", err)
}
// Design 47 H2 §794, H-A-435: builder_health reads the state consumers; a consumer whose oldest unacknowledged message waits
// six minutes is named, one within five minutes is not.
t.Run("builder_health names a consumer behind by five minutes", func(t *testing.T) {
now := time.Unix(10000, 0)
facts := map[string]json.RawMessage{
"builder-events": json.RawMessage(`{"stream":"EVENTS","name":"builder-events","pending":40,"oldest-unacknowledged":"` + now.Add(-6*time.Minute).UTC().Format(time.RFC3339) + `"}`),
"builder-kept": json.RawMessage(`{"stream":"K","name":"builder-kept","oldest-unacknowledged":"` + now.Add(-time.Minute).UTC().Format(time.RFC3339) + `"}`)}
r := versions.Reader{Keys: func() ([]string, error) { return []string{"builder-kept", "builder-events"}, nil },
Get: func(k string) (json.RawMessage, bool, error) { v, ok := facts[k]; return v, ok, nil }}
a := versions.BuilderHealth(r, now, defaultEventsBehindBound)
if a.Healthy || a.Why != "The builder's consumers behind by 5m0s or more: builder-events (its oldest unacknowledged message waits 6m0s, 40 more not yet given)." {
t.Errorf("health = %+v, want unhealthy naming builder-events alone", a)
}
if a := versions.BuilderHealth(r, now.Add(-2*time.Minute), defaultEventsBehindBound); !a.Healthy {
t.Errorf("health four minutes in = %+v, want healthy", a)
}
// H-A-461 (2): the bound is the setting events-behind-bound, five minutes when none is set.
tool := healthTool(r, func() time.Time { return now })
for _, c := range []struct{ set, want string }{{"", "unhealthy"}, {"10m", "healthy"}, {"soon", "events-behind-bound"}} {
t.Setenv(stdio.SettingWord(eventsBehindBoundSetting), c.set)
got, _ := tool.Run(nil)
a := got.(health.Answer)
if (c.want == "healthy") != a.Healthy || (c.want == "events-behind-bound" && !strings.Contains(a.Why, c.want)) {
t.Errorf("with events-behind-bound %q the health = %+v, want %s", c.set, a, c.want)
}
}
})
// Design 47 H2 §793, H-A-454: builder_publish_queue asks the seat mesh-builds' verb publish-queue, which the builder serves from
// its own stream, and answers its counts, list and cap with the states beside them; builder_publish_drop and
// builder_publish_first record the hand act first, then write their reasks value.
t.Run("the queue is the builder's verb publish-queue", func(t *testing.T) {
var asked []string
ask := func(key string, body any) (json.RawMessage, error) {
in := body.(meshbuilds.PublishQueueInput)
asked = append(asked, key+" "+in.Module)
return json.Marshal(meshbuilds.PublishQueueAnswer{Total: 9000, Listed: 1, Capped: true, ItemsRead: 4000, QueueItems: 9000, QueueBytes: 4096,
Counts: []meshbuilds.PublishQueueAnswerCountsItem{{Holder: "oci", Class: versions.PublishClassStart, Count: 4000}},
Queue: []meshbuilds.PublishQueueAnswerQueueItem{{Holder: "oci", Module: "m", Version: "0.1.0", Class: versions.PublishClassStart}}})
}
none := versions.Reader{Keys: func() ([]string, error) { return nil, nil }}
tools := republishTools(publishQueueVia(ask), none, none, nil, time.Now, meshhandactsclient.Recorder{})
raw, err := tools[1].Run(map[string]any{"module": "m"})
a, _ := raw.(versions.Queued)
if err != nil || fmt.Sprint(asked) != "[seat:mesh-builds.publish-queue m]" || a.Total != 9000 || len(a.Queue) != 1 || a.Queue[0].Module != "m" ||
len(a.Counts) != 1 || !a.Capped || len(a.Read) != 1 || a.Read[0].KeysRead != 4000 || a.Read[0].StateKeys != 9000 {
t.Errorf("%s asked %v, answered %+v (%v): want the verb asked with the module, its answer carried with its cap", tools[1].Name, asked, a, err)
}
})
// Design 47 H2 §801, H-A-446 (1): build_current names the holders from the seat-holders view and asks each holder's current in
// turn at its node, answering them together; a holder that does not answer is listed with why.
t.Run("build_current asks every build seat's holder in turn", func(t *testing.T) {
view := map[string]json.RawMessage{
"node-build-go.g14": json.RawMessage(`{"holders":[{"module":"mesh-build-go"}]}`),
"node-build-go.ace": json.RawMessage(`{"holders":[{"module":"mesh-build-go"}]}`),
"node-build-node.ace": json.RawMessage(`{"holders":[{"module":"mesh-build-node"}]}`),
"mesh-builds": json.RawMessage(`{"holders":[{"module":"mesh-delivery-builder"}]}`)}
var asked []string
ask := func(key string, body any) (json.RawMessage, error) {
asked = append(asked, key)
if strings.HasSuffix(key, "@g14") {
return nil, errors.New("no answer")
}
return json.RawMessage(`{"takes":true,"asks":[{"id":"a1","asker":"mesh-delivery-builder"}]}`), nil
}
tool := buildCurrentTool(func() []string { return []string{"node-build-go", "node-build-node"} }, func() (map[string]json.RawMessage, error) { return view, nil }, ask)
raw, err := tool.Run(map[string]any{})
a, _ := raw.(map[string]any)
hs, _ := a["holders"].([]heldBuildSeat)
if err != nil || fmt.Sprint(asked) != "[seat:node-build-go.current@ace seat:node-build-go.current@g14 seat:node-build-node.current@ace]" ||
len(hs) != 3 || hs[1].Error != "no answer" || hs[0].Module != "mesh-build-go" || !strings.Contains(string(hs[2].Current), `"a1"`) {
t.Errorf("asked %v, answered %+v (%v): want each holder asked in turn at its node, answered together, the silent one with why", asked, hs, err)
}
})
t.Run("drop and first record the hand act, then ask the builder", func(t *testing.T) {
var order []string
ask := func(key string, value []byte) (json.RawMessage, error) {
order = append(order, "ask "+key)
r, _ := build.DecodeReAsk(value)
r.Answer = "done"
return json.Marshal(r)
}
acts := meshhandactsclient.Recorder{Ask: func(key string, body any) (json.RawMessage, error) {
order = append(order, "record")
return json.RawMessage(`{"id":"h1"}`), nil
}}
tools := publishActTools(ask, func() time.Time { return time.Unix(5, 0) }, acts)
for _, tool := range tools {
if _, err := tool.Run(map[string]any{"module": "m", "version": "0.1.2", "why": "a test"}); err != nil {
t.Fatalf("%s: %v", tool.Name, err)
}
}
if len(tools) != 2 || fmt.Sprint(order) != "[record ask publish-drop.5000000000 record ask publish-first.5000000000]" {
t.Errorf("tools %d, order %v: want each hand act recorded before its reasks value", len(tools), order)
}
if _, err := tools[0].Run(map[string]any{"module": "m", "why": "no version"}); err == nil {
t.Error("a drop naming no version is not refused")
}
})
}
@@ -2,14 +2,19 @@ package main
import (
"encoding/json"
"errors"
"reflect"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
stdio "git.novox.be/novox/mesh-sdk/go"
"mesh-delivery-builder/internal/build"
)
type fakeHeard struct{ m map[string]json.RawMessage }
@@ -70,4 +75,38 @@ func TestToolchainsToolsListAndDropAKeyAfterTheHandAct(t *testing.T) {
if _, err := run(buildrun.ToolchainsDropTool, map[string]any{"key": "ace.go", "why": "ace left"}); err != nil || len(st.m) != 0 || asked < 2 {
t.Fatalf("the drop: %v, keys left %d", err, len(st.m))
}
// Design 47 H2 §777, H-A-410: builder_asks answers the waiting asks the runtime reads, held told from waiting again by
// the state taken, and the taken asks and the drops kept; builder_ask_drop asks the seat's drop after the hand act.
taken := &fakeHeard{m: map[string]json.RawMessage{"mesh-builds.a2": json.RawMessage(`{"seat":"mesh-builds","ask":"a2"}`),
"dropped.a4": json.RawMessage(`{"by":"op@shanks","why":"by hand"}`)}}
var dropped []meshbuilds.DropInput
asks := asksTools(func(seat string) (stdio.Waiting, error) {
if seat != meshbuilds.Seat {
t.Errorf("the waiting asks were read of %s", seat)
}
return stdio.Waiting{Asks: []stdio.WaitingAsk{{ID: "a1", Module: "mesh-delivery-pipeline"}, {ID: "a2", Again: true}, {ID: "a3", Again: true}}}, nil
}, taken, func(in meshbuilds.DropInput) (json.RawMessage, error) {
dropped = append(dropped, in)
return json.RawMessage(`{"held":true}`), nil
}, time.Now, acts)
tools = asks
got, err = run(build.AsksTool, nil)
if err != nil {
t.Fatal(err)
}
a := got.(map[string]any)
w := a["waiting"].([]WaitingRow)
if len(w) != 3 || w[0].State != StateWaiting || w[1].State != StateHeld || w[2].State != StateWaitingAgain ||
!reflect.DeepEqual(a["taken"], []TakenAsk{{Ask: "a2"}}) || !reflect.DeepEqual(a["dropped"], []TakenAsk{{Ask: "a4", By: "op@shanks", Why: "by hand"}}) {
t.Errorf("builder_asks = %+v", a)
}
failAct = errors.New("no log")
if _, err := run(build.AskDropTool, map[string]any{"ask": "a1", "why": "stuck"}); err == nil || len(dropped) != 0 {
t.Fatalf("a hand act not recorded still dropped: %v", err)
}
failAct = nil
if got, err := run(build.AskDropTool, map[string]any{"ask": "a1", "why": "stuck"}); err != nil || len(dropped) != 1 || dropped[0].ID != "a1" || dropped[0].Why != "stuck" || got.(map[string]any)["held"] != true {
t.Fatalf("the drop: %v, %v, asked %+v", got, err, dropped)
}
}
@@ -0,0 +1,70 @@
package main
import (
"encoding/json"
"errors"
"sort"
stdio "git.novox.be/novox/mesh-sdk/go"
"mesh-delivery-builder/internal/build"
"mesh-delivery-builder/internal/versions"
)
// unbuildableTool is builder_unbuildable (design 47 H2 §795, H-A-425): the builder's state unbuildable, the plans no seat builds,
// read in one page of at most 768 KB (capped says what it left out), with the summary line of the rows read.
func unbuildableTool(page func() (versions.StatePage, error)) stdio.Tool {
return stdio.Tool{Name: build.UnbuildableTool,
Description: "The modules whose artifact no build seat builds, or whose build seat no module claims, as the builder judged their plans " +
"(its state " + build.UnbuildableState + ", worked out again at every start and every change of the seat list): each module, the fingerprint " +
"judged, the artifact's language and kind, why and when; with the one summary line the builder says of them. At most 768 KB, capped " +
"saying what was left out. Read-only.",
Input: map[string]any{"type": "object", "properties": map[string]any{}},
Run: func(map[string]any) (any, error) {
pg, err := page()
if err != nil {
return nil, err
}
rows := []build.Unbuildable{}
var unread []string
for k, raw := range pg.Values {
var u build.Unbuildable
if err := json.Unmarshal(raw, &u); err != nil || u.Module == "" {
unread = append(unread, k)
continue
}
rows = append(rows, u)
}
sort.Slice(rows, func(i, j int) bool { return rows[i].Module < rows[j].Module })
sort.Strings(unread)
out := map[string]any{"line": build.UnbuildableLine(rows), "unbuildable": rows}
if len(unread) > 0 {
out["unreadable"] = unread
}
if pg.More {
out["capped"] = versions.StateCapped{State: build.UnbuildableState, KeysRead: len(pg.Values), StateKeys: pg.Keys, StateBytes: pg.Bytes, InfoWords: pg.InfoWords}
}
return out, nil
}}
}
// unbuildablePage is the first page of the state unbuildable; a runtime that pages nothing is read key by key.
func unbuildablePage(s stdio.KeptState) func() (versions.StatePage, error) {
return func() (versions.StatePage, error) {
pg, err := onePage(s.Page, s.Info)
if !errors.Is(err, errors.ErrUnsupported) {
return pg, err
}
keys, err := s.Keys()
if err != nil {
return versions.StatePage{}, err
}
out := versions.StatePage{Values: map[string]json.RawMessage{}}
for _, k := range keys {
if e, err := s.Get(k); err == nil && e != nil {
out.Values[k] = e.Value
}
}
return out, nil
}
}
@@ -0,0 +1,99 @@
package main
import (
"context"
"errors"
"sync"
"time"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/process"
)
// handlers is where a part of the builder registers its handler of the events it hears: the event hub, or agedHub over it.
type handlers interface {
Handle(pattern string, handle func(jetstream.Msg)) error
}
// ledgers are the builder's durable consumers' ledgers, one per consumer as a message's metadata names it (the events consumer
// and each kept seat event's), writing each consumer's facts into the state consumers, which builder_health reads (design 47 H2
// §794, H-A-435 and its addendum: the runner's runner_health form).
type ledgers struct {
sink process.ConsumerSink
logf func(string, ...any)
now func() time.Time
mu sync.Mutex
byName map[string]*process.ConsumerLedger
}
// of is the ledger of the consumer m came on, nil for a message that names none.
func (l *ledgers) of(m jetstream.Msg) *process.ConsumerLedger {
md, err := m.Metadata()
if err != nil || md == nil || md.Consumer == "" {
return nil
}
l.mu.Lock()
defer l.mu.Unlock()
if l.byName == nil {
l.byName = map[string]*process.ConsumerLedger{}
}
led, ok := l.byName[md.Consumer]
if !ok {
led = process.NewConsumerLedger(md.Stream, md.Consumer, l.sink, l.now, l.logf)
l.byName[md.Consumer] = led
}
return led
}
// agedHub registers each handler on the hub so that every message it is handed is noted delivered on its consumer's ledger,
// and applied once the handler acknowledges or ends it (the runner's agedHub); a message the handler asks again stays
// unacknowledged, and grows old.
type agedHub struct {
hub handlers
ledgers *ledgers
}
func (a agedHub) Handle(pattern string, handle func(jetstream.Msg)) error {
return a.hub.Handle(pattern, func(m jetstream.Msg) {
led := a.ledgers.of(m)
if led == nil {
handle(m)
return
}
led.Delivered(m)
handle(&agedMsg{Msg: m, ledger: led})
})
}
// agedMsg is a message whose acknowledgement or end is noted applied on its consumer's ledger, once.
type agedMsg struct {
jetstream.Msg
ledger *process.ConsumerLedger
once sync.Once
}
func (m *agedMsg) applied() { m.once.Do(func() { m.ledger.Applied(m.Msg, time.Time{}) }) }
func (m *agedMsg) Ack() error { defer m.applied(); return m.Msg.Ack() }
func (m *agedMsg) DoubleAck(ctx context.Context) error {
defer m.applied()
return m.Msg.DoubleAck(ctx)
}
func (m *agedMsg) Term() error { defer m.applied(); return m.Msg.Term() }
func (m *agedMsg) TermWithReason(r string) error { defer m.applied(); return m.Msg.TermWithReason(r) }
// consumersSink writes a consumer's facts in the state consumers; while the state is not on the bus the bucket keeps the latest.
func consumersSink(b *process.AwaitedBucket) process.ConsumerSink {
return func(ctx context.Context, k string, v []byte) error {
_, err := b.Put(ctx, k, v)
if errors.Is(err, process.ErrNotYet) {
return nil
}
return err
}
}
@@ -0,0 +1,55 @@
package main
import (
"errors"
"time"
"github.com/nats-io/nats.go/jetstream"
stdio "git.novox.be/novox/mesh-sdk/go"
)
// slowEventSetting is the setting slow-event (design 47 H2 §794; H-A-461 (2), P11): how long the handling of one heard event
// may take before the builder says it in its journal, the line naming the event's subject, the part that took it and how long
// it took; defaultSlowEvent when none is set.
const (
slowEventSetting = "slow-event"
defaultSlowEvent = 100 * time.Millisecond
)
// slowEvent reads the setting slow-event.
func slowEvent() (time.Duration, error) {
d, err := stdio.SettingDuration(slowEventSetting, defaultSlowEvent)
if err != nil {
return 0, errors.New("the builder's " + err.Error())
}
return d, nil
}
// eventTaker is one part of the builder that takes events off its durable consumer: take answers whether the message was its.
type eventTaker struct {
part string
take func(jetstream.Msg) bool
}
// takeEvent hands m to each taker in turn until one takes it, else to rest, and says the handling when it passed slow, the
// setting slow-event (design 47 H2 §794). now is the clock, a test's when set.
func takeEvent(m jetstream.Msg, takers []eventTaker, rest eventTaker, slow time.Duration, now func() time.Time, logf func(string, ...any)) {
if now == nil {
now = time.Now
}
start := now()
part := rest.part
for _, t := range takers {
if t.take(m) {
part = t.part
break
}
}
if part == rest.part {
rest.take(m)
}
if took := now().Sub(start); took > slow {
logf("the event %s took %s to handle, over %s: %s took it", m.Subject(), took.Round(time.Millisecond), slow, part)
}
}
@@ -103,9 +103,16 @@ func revisionOf(t *testing.T, b *bed, module string) uint64 {
return e.Revision()
}
// TestAGoneArtifactIsBuiltAgain: 1937/AC3 and AC4, one whole test of the two (design 47 H2 §819: the builder's integration count does
// not rise with the work queue publishes' whole test).
func TestAGoneArtifactIsBuiltAgain(t *testing.T) {
t.Run("a done key whose blob is gone is built again at start", aDoneKeyWhoseBlobIsGoneIsBuiltAgainAtStart)
t.Run("artifacts-gone leads the builder to build the key again", artifactsGoneLeadsTheBuilderToBuildTheKeyAgain)
}
// 1937/AC3 (integration): the builder, started whole, finds the done key of the trunk's fingerprint naming a blob held
// answers gone, builds it again as a new revision whose step says so, and leaves the rebuilt key done.
func TestADoneKeyWhoseBlobIsGoneIsBuiltAgainAtStart(t *testing.T) {
func aDoneKeyWhoseBlobIsGoneIsBuiltAgainAtStart(t *testing.T) {
b := startBed(t)
eventsConsumer(t, b, "n1")
manifest := b.archive(t, "ca-trust", "modules/ca-trust", commit)
@@ -135,7 +142,7 @@ func TestADoneKeyWhoseBlobIsGoneIsBuiltAgainAtStart(t *testing.T) {
// 1937/AC4 (integration): a done key whose blob held answers held at start stays done; when the controller says
// artifacts-gone for it, the builder asks again and builds it again.
func TestArtifactsGoneLeadsTheBuilderToBuildTheKeyAgain(t *testing.T) {
func artifactsGoneLeadsTheBuilderToBuildTheKeyAgain(t *testing.T) {
b := startBed(t)
eventsConsumer(t, b, "n1")
manifest := b.archive(t, "ca-trust", "modules/ca-trust", commit)
@@ -9,7 +9,6 @@ import (
"encoding/json"
"errors"
"fmt"
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
"maps"
"os"
"path/filepath"
@@ -20,6 +19,8 @@ import (
"testing"
"time"
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
"github.com/nats-io/nats-server/v2/server"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
@@ -35,8 +36,10 @@ import (
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/ociartifactregistry"
sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
runner "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/steprun"
"git.novox.be/novox/mesh-sdk/go/buscalls"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/seatdef"
"git.novox.be/novox/mesh-sdk/go/views/viewstest"
"mesh-delivery-builder/internal/build"
@@ -87,6 +90,34 @@ type fakeBuildModules struct {
// hang, while it holds a channel, keeps every build step running until the channel is closed: a step that never ends,
// so the builder's step bound runs out (design 51's round-2 case).
hang atomic.Pointer[chan struct{}]
// big, while set, keeps every passed step's answer as its answer object, as one over what the steps state holds per value
// is kept, handed through the build seat's verb answer in chunks (design 47 H2 H-A-412); answers holds those objects by name.
big atomic.Bool
answers sync.Map
}
// serveAnswers answers the verb answer of the seat node-build-go on every node as the build module's holder does: the
// answer object the ask's step key names, in chunks (design 47 H2 H-A-412).
func (f *fakeBuildModules) serveAnswers(t *testing.T, nc *nats.Conn) {
sub, err := nc.Subscribe("mesh.seat.node-build-go.call.answer.>", func(m *nats.Msg) {
var in struct{ Ask, Call string }
_ = json.Unmarshal(m.Data, &in)
body, ok := f.answers.Load(sdkstep.AnswerObjectName(in.Ask))
if !ok {
_ = m.Respond([]byte(`{"error":{"kind":"not-found","message":"no answer of ` + in.Ask + `"}}`))
return
}
of := (len(body.([]byte)) + buscalls.SeatChunkBytes - 1) / buscalls.SeatChunkBytes
answer, _ := json.Marshal(map[string]any{"result": map[string]any{"ask": in.Ask, "chunks": of}})
_ = m.Respond(answer)
go func() {
_ = buscalls.SeatChunks{Seat: "node-build-go", Verb: "answer", Call: in.Call}.Send(nc, body.([]byte), buscalls.SeatAckWithin, t.Logf)
}()
})
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = sub.Unsubscribe() })
}
// open lets the nodes pull again and wakes their pullers.
@@ -171,6 +202,16 @@ func (f *fakeBuildModules) serve(t *testing.T, module string, nodes ...string) {
// both forms overlaps the named worker, which a work queue refuses.
put := func(key string, value any) (uint64, error) {
raw, _ := value.(json.RawMessage)
// big: an ended step's answer is kept as its answer object and the key names it, as the build module's step run keeps
// an answer over what the steps state holds per value (steprun's keepOver; design 47 G A-290, H2 H-A-412).
if step, err := sdkstep.DecodeStep(raw); f.big.Load() && err == nil && step.State == sdkstep.StepPassed && len(step.Result) > 0 {
object := sdkstep.AnswerObjectName(step.ID)
f.answers.Store(object, []byte(step.Result))
step.Result, step.AnswerObject = nil, object
if raw, err = sdkstep.EncodeStep(step); err != nil {
return 0, err
}
}
return kv.Put(f.ctx, key, raw)
}
call := func(name string) runner.StepFunc {
@@ -299,6 +340,12 @@ func startBedWith(t *testing.T, closed bool) *bed {
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: builds.StateBucket(builds.BuilderModule, versions.BoundsState), History: 1, LimitMarkerTTL: time.Second}); err != nil {
t.Fatal(err)
}
// The seat mesh-builds' work queue and its one worker, as the broker derives them from the seat's definition (design 47
// H2 §777), and the state taken the builder takes each ask once in (§775, §820).
b.seatQueue(t)
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: builds.StateBucket(builds.BuilderModule, process.TakenState), History: 1, LimitMarkerTTL: time.Second}); err != nil {
t.Fatal(err)
}
b.out = &saidLog{}
// The builder's module log, as the runtime raises it (ADR 0297): its stream on the log's subjects.
logStream := buscompose.LogStreamName(builds.BuilderModule, builds.BuilderLog)
@@ -320,6 +367,7 @@ func startBedWith(t *testing.T, closed bool) *bed {
b.fake.serve(t, "mesh-build-go", "n1", "n2")
b.fake.serve(t, "mesh-build-node", "n1", "n2")
b.fake.serve(t, "mesh-build-image", "n1") // the image build module runs on n1 only
b.fake.serveAnswers(t, ctl)
b.report(t, "mesh-build-go", "n1", "go", "go1.27.1", "1")
b.report(t, "mesh-build-go", "n2", "go", "go1.27.1", "1")
b.report(t, "mesh-build-node", "n1", "typescript", "tsc5", "1")
@@ -692,10 +740,13 @@ func TestBuilderMakesADoneKeyWithTheDigests(t *testing.T) { // 1607/AC7
func TestAnArchiveOnlyModuleAndARootModuleAreBuilt(t *testing.T) {
for _, c := range []struct {
name, path string
two bool
}{{"no toolchain", "modules/ca-trust", false}, {"root", meshcatalogclient.RootPath, false}, {"two instances together", "modules/ca-trust", true}} {
two, big bool
}{{"no toolchain", "modules/ca-trust", false, false}, {"root", meshcatalogclient.RootPath, false, false}, {"two instances together", "modules/ca-trust", true, false},
// Design 47 H2 H-A-412: a build step's end over what the steps state holds is read through the seat's verb answer.
{"an answer over the steps state", "modules/ca-trust", false, true}} {
t.Run(c.name, func(t *testing.T) {
b := startBed(t)
b.fake.big.Store(c.big)
b.archive(t, "ca-trust", c.path, commit)
b.start(t, "n1", nil)
if c.two {
@@ -705,7 +756,7 @@ func TestAnArchiveOnlyModuleAndARootModuleAreBuilt(t *testing.T) {
if c.name == "no toolchain" && (len(v.Toolchains) != 0 || len(v.Artifacts) != 1 || strings.HasPrefix(v.Step, "one node (")) {
t.Errorf("build = %+v, want done naming no toolchain", v)
}
if c.two && (len(v.Artifacts) != 1 || v.Artifacts[0].Kind != "archive" || v.Artifacts[0].Digest != digestOf("files")) {
if (c.two || c.big) && (len(v.Artifacts) != 1 || v.Artifacts[0].Kind != "archive" || v.Artifacts[0].Digest != digestOf("files")) {
t.Errorf("build = %+v, want the archive's digest", v)
}
if n := b.fake.count("mesh-build-go.build_go_build"); c.two && n < 1 { // each instance that built, on one node
@@ -1252,3 +1303,29 @@ func (r *registry) numberedOf(name string) []string {
}
return out
}
// seatQueue makes the seat mesh-builds' work-queue stream and its worker from the seat's definition in the contracts
// module, as the broker derives them (buscompose.SeatQueue; design 47 H2 §777).
func (b *bed) seatQueue(t *testing.T) {
t.Helper()
raw, err := os.ReadFile(filepath.Join("..", "..", "..", "mesh-delivery-contracts", "seats", meshbuilds.Seat, "seat.json"))
if err != nil {
t.Fatal(err)
}
def, err := seatdef.Parse(raw)
if err != nil {
t.Fatal(err)
}
stream, worker, ok, err := buscompose.SeatQueue(meshbuilds.Seat, def.Accepts)
if err != nil || !ok {
t.Fatalf("the seat's work queue: %v %v", ok, err)
}
if _, err := b.js.CreateStream(b.ctx, jetstream.StreamConfig{Name: stream.Name, Subjects: stream.Subjects, Retention: jetstream.WorkQueuePolicy,
MaxAge: time.Duration(stream.MaxAge) * time.Second, MaxMsgs: int64(stream.MaxMsgs), Duplicates: time.Duration(stream.DuplicatesSeconds) * time.Second}); err != nil {
t.Fatal(err)
}
if _, err := b.js.CreateOrUpdateConsumer(b.ctx, stream.Name, jetstream.ConsumerConfig{Durable: worker.Name, FilterSubjects: worker.Filters, AckPolicy: jetstream.AckExplicitPolicy,
AckWait: time.Duration(worker.AckWaitSeconds) * time.Second, MaxDeliver: worker.MaxDeliver, MaxAckPending: worker.MaxAckPending}); err != nil {
t.Fatal(err)
}
}
@@ -15,6 +15,7 @@ import (
"os/signal"
"strconv"
"strings"
"sync/atomic"
"syscall"
"time"
@@ -37,6 +38,7 @@ import (
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/operatorask"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/seatlist"
@@ -190,7 +192,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
Claimer: build.Claimer{Store: kvStore{buildsKV}, By: cred.Node + "." + cred.Module},
Seats: seats,
Asks: buildrun.Asker{Name: meshbuilds.Seat, Seats: seats, Publisher: asks, Waiter: buildrun.LostAskWaiter{Inner: heard, Dead: dead, Hearing: hearing}, Bound: bound, Dropper: buildrun.SeatDrops{Caller: calls},
Bounds: &stepBounds{bounds: pubBoundsKV.KV, js: js, now: time.Now, logf: logger.Printf}, Expiry: stepExpiry},
Bounds: &stepBounds{bounds: pubBoundsKV.KV, js: js, now: time.Now, logf: logger.Printf}, Expiry: stepExpiry,
// Design 47 H2 H-A-412: a step's end over what the steps state holds is read through its build seat's verb answer, in
// chunks (process.Conn.CallFile, the chunked-call form), on the node that ran it.
Outputs: calls.CallFile},
Held: heldOnTheBus{js: js, seats: seats},
// An ask that can no longer be answered is asked again (issue 1945), read at the step bound: a context's bound.
Gone: heldOnTheBus{js: js, seats: seats},
@@ -235,6 +240,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
return emitEvent(context.WithoutCancel(ctx), js, cred, event, id, body)
}, waitBound, logger.Printf)
waitsBucket.Ready(func(kv jetstream.KeyValue) { b.Waits.(*waitsKept).start(ctx, kv) })
// The plans no seat builds (design 47 H2 §795, H-A-425): kept in the state unbuildable, read by builder_unbuildable.
b.Unbuildables = unbuildableKept{b: process.AwaitBucket(ctx, js, conn, builds.StateBucket(cred.Module, build.UnbuildableState), "the plans no seat builds are said, and builder_unbuildable lists none", logger.Printf)}
// The versions a build reserves at its start (design 47 G §369): its state reserved, and the catalog's versions-filled
// and version-released heard; nothing is reserved before the fill.
reservedKV := process.AwaitBucket(ctx, js, conn, own.reserved, "no version is reserved: builds are made unreserved", logger.Printf)
@@ -245,6 +252,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
reserved.say = func(ctx context.Context, p meshbuilds.PackagePublished) error {
return emitSeatEvent(context.WithoutCancel(ctx), js, cred, meshbuilds.SubjectPackagePublished, "package-published-"+p.Module+"-"+p.Version+"-"+p.Artifact, p)
}
// A package's number reserved is said as the seat mesh-builds' version-reserved, so the indexer re-pins at the reservation (§805).
reserved.names = versions.NamesFrom(versions.EntriesOf(branches))
reserved.r.Names = reserved.names // a Go package's module path names its tag's directory (§796)
reserved.sayReserved = sayReservedOn(js, cred)
b.Reserved = reserved
// A take-over of a series' reservation by another fingerprint withdraws the older build (H-A-225).
reserved.withdraw = b.Withdraw
@@ -255,11 +266,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// The versions' hand acts (builder_publish_again, builder_floor_again) come through reasks, as builder_build_again.
acts := versions.Acts{Numberer: numberer, DropReserved: reserved.dropped, Reconsider: b.Reconsider}
// A reserved version left unpublished is published again, by the builder on an event or by a person's tool (design 47 G
// §588): its states publish-queue and acted, bound once the bus holds them.
queueKV := process.AwaitBucket(ctx, js, conn, own.queue, "no version is published again on an event or by builder_publish_again", logger.Printf)
// §588): its items on the work queue publishes, the module's own stream of its old streams section (design 47 H2 §793, H-A-454).
actedKV := process.AwaitBucket(ctx, js, conn, own.acted, "no version is published again on an event or by builder_publish_again", logger.Printf)
republish := &versions.Republisher{Reserved: reserved.r, Queue: awaitedQueue{awaitedReserved{awaitedVersions{b: queueKV, bucket: own.queue}}},
Entry: entryVia(calls.Call), Send: sendVia(calls.Call),
pq := &publishQueue{js: js, logf: logger.Printf, binding: publishesOf(cred.Module)}
republish := &versions.Republisher{Reserved: reserved.r, Put: pq.put,
Entry: entryVia(calls.Call), Send: sendVia(calls.CallKeptAsk),
Unreleased: reserved.unreleasedWords, Logf: logger.Printf,
Holders: holders, // a registry seat's holder from the seat-holders view, from the start (design 47 G A-416, A-416a)
Started: time.Now()} // a build ended from now on is of the live class, sent first (H-A-290 (1))
@@ -287,6 +298,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
if err != nil {
return err
}
pq.ackWait = publishSet.PauseMax // a sent item unheard is delivered again after the longest pause (H-A-440 (1))
releasedKept, err := numberFile(getenv("MESH_BUILDER_RELEASED_KEPT_FILE"), versions.DefaultReleasedKept)
if err != nil {
return err
@@ -305,15 +317,22 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
},
markHeld: reserved.markHeld, moveElsewhere: reserved.movedElsewhere,
allDone: func(ctx context.Context) ([]versions.ReservedVersion, error) { return reserved.r.All(ctx) },
send: republish.SendQueued,
retired: reserved.r.Retired} // only each module's newest done build is asked at the start (H-A-291 (1))
// A version that never publishes (withdrawn or taken over) has its publish item ended by the queue too (H-A-290 (2)).
republish.Ended = items.Done
republish.Ended = items.Ended
pq.take = items.take
reasksKV.Ready(func(jetstream.KeyValue) { items.AskAllDone(ctx) })
wirePublishItems(reserved, items, awaitedReserved{awaitedVersions{b: actedKV, bucket: own.acted}}, releasedKept)
acts.Republish = republish.Republish
acts.PublishDrop = func(ctx context.Context, module, version, by, why string) (string, error) {
return pq.drop(ctx, items, module, version, by, why)
}
acts.PublishFirst = func(ctx context.Context, module, version, _, _ string) (string, error) {
return pq.first(ctx, items, module, version)
}
// builder_reserve_done: a done trunk build that reserved none is given a version, and its publish queued as publish again does.
acts.ReserveDone = reserveDoneAct(b.ReserveDone, republish.Republish, logger.Printf)
again := &republishing{rp: republish}
// The operator is asked once a key's event builds are spent (design 47 G §568): the builder's asker on the seat operator-channel.
decided := &decisions{subject: oc.SubjectDecided(cred.Module)}
retakeActed := awaitedReserved{awaitedVersions{b: actedKV, bucket: own.acted}}
@@ -327,8 +346,13 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
return fmt.Errorf("hearing the operator's answers: %w", err)
}
retake := &retaking{rt: &build.Retaker{Builder: b, Acted: retakeActed, Entry: entryVia(calls.Call), Ask: operator.Ask, Logf: logger.Printf}}
queueKV.Ready(again.bound(ctx, js, logger.Printf))
actedKV.Ready(again.bound(ctx, js, logger.Printf))
// What waits on the work queue is pulled, and the old state's items are put on it once (§793).
go func() {
if err := pq.learnAll(ctx); err != nil {
logger.Printf("the publishes waiting on the work queue %s are not pulled yet: %v; each holder's are pulled when an item of it is put", versions.PublishesLocal, err)
}
moveOld(ctx, js, own.queue, republish, logger.Printf)
}()
errs := make(chan error, 10)
b.Members = versions.ModuleFrom(versions.EntriesOf(branches)) // the part a resolved package names (design 47 G §466)
// The builder's log (ADR 0571 §1): each build's start and end in its module log `build-log`, read by the view, the
@@ -453,18 +477,24 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
floor := floorWake{b: b, waits: floorWaits, logf: logger.Printf}
// And the four registry seats' published, refused and removed (DLV-B4), which move a numbered version.
seatHeard := seatHearer{n: numberer, b: b, logf: logger.Printf, pub: pubs, res: reserved}
events := func(m jetstream.Msg) {
if decided.heard(m) { // the operator's answer to an ask about a key (§568)
return
}
if retake.heard(ctx, m, logger.Printf) { // a key failed on a transport fault, built again when its cause passed (§568)
return
}
if reads.heard(m, logger.Printf) || heardTools.Reported(ctx, m) || heardTools.Running(ctx, m) || floor.heard(ctx, m) || reserved.heard(ctx, m) || seatHeard.heard(ctx, m) { // reads: the node-engine seat's manifest-reads-changed (ADR 0618 §4)
return
}
heardGone(ctx, b, m, logger.Printf)
// Each part takes the events it hears in turn; a handling over the setting slow-event is said with the part that took it
// (design 47 H2 §794; H-A-461 (2)).
slow, err := slowEvent()
if err != nil {
return err
}
takers := []eventTaker{
{"the operator's answer to an ask about a key", decided.heard}, // §568
{"the retake of a key failed on a transport fault", func(m jetstream.Msg) bool { return retake.heard(ctx, m, logger.Printf) }}, // §568
{"what a node's engine reads", func(m jetstream.Msg) bool { return reads.heard(m, logger.Printf) }}, // manifest-reads-changed (ADR 0618 §4)
{"the toolchains reported", func(m jetstream.Msg) bool { return heardTools.Reported(ctx, m) }},
{"the toolchains running", func(m jetstream.Msg) bool { return heardTools.Running(ctx, m) }},
{"the floor", func(m jetstream.Msg) bool { return floor.heard(ctx, m) }},
{"the reservations", func(m jetstream.Msg) bool { return reserved.heard(ctx, m) }},
{"the registry seats' outcomes", func(m jetstream.Msg) bool { return seatHeard.heard(ctx, m) }},
}
gone := eventTaker{"the artifacts gone", func(m jetstream.Msg) bool { heardGone(ctx, b, m, logger.Printf); return true }}
events := func(m jetstream.Msg) { takeEvent(m, takers, gone, slow, nil, logger.Printf) }
// The seats' events (artifacts-gone, the registry seats' words, the build seats' toolchain-reported, the indexer's take-in.taken and
// manifest-reads-changed and the seat mesh-conditions' condition-cleared: seatEventPatterns); the owners' views events and the
// catalog half's catalog-revision and registration-changed and the gate's source-moved are the views reader's (a message has
@@ -473,6 +503,18 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// modules again (design 47 G A-82, §204).
// The runner's build step ask, the builder's one entry point for a candidate (design 47 G A-97).
b.StepEntry = newEntryWaits(branches, half.manifests).entry
// The seat's work queue (design 47 H2 §777): pulled through its one worker, each ask taken once in the state taken.
takenBound, err := askTakenBound()
if err != nil {
return err
}
var ackWait atomic.Int64 // the worker's ack wait, read when it is bound
ackWait.Store(int64(buscompose.SeatQueueDefaultAckWait))
steps := &stepQueue{node: cred.Node, builder: b, taken: calls.State(process.TakenState), takenBound: takenBound,
inProgress: inProgressOf(seatlist.Definition(seatGate.Reader, meshbuilds.Seat), func() time.Duration { return time.Duration(ackWait.Load()) }),
say: func(ctx context.Context, subject, id string, body any) error {
return emitSeatEvent(ctx, js, cred, subject, id, body)
}, logf: logger.Printf, now: time.Now, running: map[string]*runningAsk{}}
// The seat mesh-builds' verbs reserved, answered here (A-115, A-127): the reservations, the catalog half's carriers and the seat
// mesh-catalog's versions; and published, from the reservations and releases (H-A-328).
stopReserved, err := calls.AnswerSeat(meshbuilds.Seat, seatlist.Definition(seatGate.Reader, meshbuilds.Seat), map[string]process.SeatHandler{
@@ -481,18 +523,27 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
logger.Printf("reserved: no entry carries the name %s; the entries looked at: %s", name, versions.LookedAt(versions.EntriesOf(branches)))
}),
// published (H-A-328): whether a package built at a fingerprint is published, asked by the runner writing a wait on it.
meshbuilds.VerbPublished: publishedVerb(ctx, reserved.r.PublishedOf)})
meshbuilds.VerbPublished: publishedVerb(ctx, reserved.r.PublishedOf),
// drop (§778; H-A-401, H-A-404): an ask its caller no longer waits for ends could-not-run naming the drop.
meshbuilds.VerbDrop: dropVerb(steps),
// publish-queue (§793; H-A-454): the work queue publishes read from its own stream, for builder_publish_queue.
meshbuilds.VerbPublishQueue: publishQueueVerb(ctx, pq)})
if err != nil {
return fmt.Errorf("answering the verb reserved: %w", err)
}
defer stopReserved()
if err := serveSteps(ctx, conn, js, cred, b, logger.Printf); err != nil {
if err := serveSteps(ctx, conn, js, steps, &ackWait); err != nil {
return err
}
seatGate.Follow(seatlist.ProcessBus(calls.Call, hub), func() { seatsChanged(ctx) }, logger.Printf)
// A change of the seat list judges every plan no seat builds again and says them in one line (design 47 H2 §795).
// Every message the builder's own handlers take is noted on its consumer's ledger, in the state consumers that
// builder_health reads (design 47 H2 §794, H-A-435).
aged := agedHub{hub: hub, ledgers: &ledgers{logf: logger.Printf, sink: consumersSink(process.AwaitBucket(ctx, js, conn,
builds.StateBucket(cred.Module, process.ConsumersState), "builder_health reads no consumer's age", logger.Printf))}}
seatGate.Follow(seatlist.ProcessBus(calls.Call, aged), func() { seatsChanged(ctx); b.SayUnbuildable(ctx) }, logger.Printf)
half.holders = func() { seatsChanged(ctx) }
for _, pattern := range seatEventPatterns {
if err := hub.Handle(pattern, events); err != nil {
if err := aged.Handle(pattern, events); err != nil {
return err
}
}
@@ -510,6 +561,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
if err := b.Sweep(ctx, half.pictured()...); err != nil {
errs <- err
}
b.SayUnbuildable(ctx) // at the start, once the first views are in: the plans no seat builds in one line (§795)
}
branches.Changed = half.Changed(ctx)
// A catalog revision the builder hears wakes a floor whose tags or requirements were silent (design 47 G A-119).
@@ -0,0 +1,323 @@
//go:build integration
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"slices"
"strings"
"sync"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
gitseat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/views"
"mesh-delivery-builder/internal/versions"
)
// Design 47 H2 §793 (whole, against the bus): the builder's work queue publishes, its pull consumers made as it learns a holder.
// A live publish enqueued behind 100 again items while the builder was away is sent first; a dropped item is never sent, and
// put again within the duplicates window is not stored; the move of the old state run twice puts each item once, and the
// builder removes the old state's bucket itself (H-A-461 (1)). The seat mesh-builds' verb publish-queue reads the stream: counts
// per holder and class, the oldest and newest, the live item listed first (H-A-454). Then §796's tag at another commit, its
// reservation's version-reserved waited on by name (§821), and
// §794's backlog held six minutes, as steps (H-A-453 (3)).
func TestThePublishQueueSendsLiveFirstAndNeverADroppedItem(t *testing.T) {
b := startBed(t)
ctx := b.ctx
// The stream as the manifest's old streams section declares it: the module's own, retention work-queue, duplicates 24 hours.
bind, _ := publishesOf("mesh-delivery-builder")()
if _, err := b.js.CreateOrUpdateStream(ctx, jetstream.StreamConfig{Name: bind.Stream, Subjects: []string{bind.Subject + ".publish.>"},
Retention: jetstream.WorkQueuePolicy, Discard: jetstream.DiscardNew, Duplicates: 24 * time.Hour, MaxAge: 7 * 24 * time.Hour}); err != nil {
t.Fatal(err)
}
kv := &itemKV{vals: map[string][]byte{}, revs: map[string]uint64{}}
var mu sync.Mutex
var sent []string
published := map[string]bool{} // the modules whose publish is heard
sentCh := make(chan string, 200)
items := &publishItems{reasks: func() jetstream.KeyValue { return kv }, set: publishSettings{Pause: time.Second, PauseMax: time.Minute, Tries: 5},
now: time.Now, logf: t.Logf,
reserved: func(_ context.Context, m, v string) (versions.ReservedVersion, bool, error) {
mu.Lock()
defer mu.Unlock()
return versions.ReservedVersion{Module: m, Version: v, Artifacts: map[string]string{m + ":a": "sha256:a"}, Published: map[string]bool{m + ":a": published[m]}, BuiltAt: time.Now()}, true, nil
},
send: func(_ context.Context, q versions.QueuedPublish) versions.SendFate {
mu.Lock()
sent = append(sent, q.Module)
mu.Unlock()
sentCh <- q.Module
return versions.SendFate{Sent: true}
}}
pq := &publishQueue{js: b.js, ackWait: time.Minute, logf: t.Logf, take: items.take, binding: publishesOf("mesh-delivery-builder")}
item := func(module, class string) versions.QueuedPublish {
return versions.QueuedPublish{V: 1, Holder: "oci", Module: module, Version: "0.1.0", Artifacts: map[string]string{module + ":a": "sha256:a"}, Class: class, Queued: time.Now()}
}
away := func(q versions.QueuedPublish) { // put while the builder was away: no holder learnt, nothing pulled
data, _ := json.Marshal(q)
if _, err := b.js.Publish(ctx, pq.subject(bind, q), data, jetstream.WithMsgID(versions.PublishMsgID(q.Holder, q.Class, q.Module, q.Version))); err != nil {
t.Fatal(err)
}
}
for i := range 100 {
away(item(fmt.Sprintf("a%03d", i), versions.PublishClassStart))
}
away(item("dropme", versions.PublishClassStart))
away(item("live", versions.PublishClassLive))
if answer, err := pq.drop(ctx, items, "dropme", "0.1.0", "op@node", "a test drops it"); err != nil {
t.Fatalf("drop: %v", err)
} else {
t.Log(answer)
}
if dup, err := pq.put(ctx, item("dropme", versions.PublishClassStart)); err != nil || !dup {
t.Errorf("a dropped item put again within the window: duplicate %v (%v), want it not stored", dup, err)
}
raw, err := publishQueueVerb(ctx, pq)("op@node", json.RawMessage(`{}`))
if a, ok := raw.(meshbuilds.PublishQueueAnswer); err != nil || !ok || a.Total != 101 || len(a.Counts) != 2 || a.Counts[0].Class != versions.PublishClassStart ||
a.Counts[0].Count != 100 || a.Counts[1].Count != 1 || a.Counts[0].Oldest.After(a.Counts[0].Newest) || len(a.Queue) != versions.PublishQueueListCap ||
a.Queue[0].Module != "live" || a.Capped {
t.Errorf("publish-queue answered %+v (%v), want 101 items, oci's 100 again and 1 live, fifty listed, the live first", raw, err)
}
if err := pq.learnAll(ctx); err != nil {
t.Fatal(err)
}
select {
case first := <-sentCh:
if first != "live" {
t.Errorf("the first item sent is %s, want the live one, enqueued behind 100 again items", first)
}
case <-time.After(30 * time.Second):
t.Fatal("nothing was sent")
}
for range 4 { // the again items in flight: at most four held unacknowledged, their publish unheard
select {
case <-sentCh:
case <-time.After(30 * time.Second):
t.Fatal("the again items were not sent")
}
}
deadline := time.After(60 * time.Second)
for { // each publish sent is heard: its item acknowledged, the next sent
mu.Lock()
n, done := len(sent), slices.Clone(sent)
for _, m := range done {
published[m] = true
}
mu.Unlock()
if n >= 101 {
break
}
for _, m := range done {
items.heard(ctx, m, "0.1.0")
}
select {
case <-sentCh:
case <-deadline:
t.Fatalf("sent %d of 101", n)
}
}
mu.Lock()
if slices.Contains(sent, "dropme") || len(sent) != 101 {
t.Errorf("sent %d items (dropme among them: %v), want the 101 and never the dropped one", len(sent), slices.Contains(sent, "dropme"))
}
mu.Unlock()
// The move of the old state, run twice, puts each item once (at another holder, so the pull of oci is not asked).
old, err := b.js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: builds.StateBucket("mesh-delivery-builder", versions.PublishQueueState)})
if err != nil {
t.Fatal(err)
}
var tagsWritten []string
call := func(tool string, body any) (json.RawMessage, error) {
switch tool {
case "seat:" + gitseat.Seat + "." + gitseat.VerbTags: // m4's 0.2.0 is tagged at another commit (§796)
return json.Marshal(gitseat.TagsAnswer{Form: 2, Tags: []gitseat.Tag{{Tag: "modules/m4/v0.2.0", Commit: "c-other"}}})
case "seat:" + gitseat.Seat + ".tag":
tagsWritten = append(tagsWritten, body.(gitseat.TagInput).Tag+"@"+body.(gitseat.TagInput).Commit)
return json.Marshal(gitseat.TagAnswer{Performed: true})
}
return json.Marshal(meshcatalog.VersionsAnswer{Module: "m"})
}
found := &memFindings{vals: map[string][]byte{}}
rv := newReservations(&memReserve{vals: map[string][]byte{}, revs: map[string]uint64{}}, call, func() jetstream.KeyValue { return &boundsKV{keys: map[string]bool{}} },
&findings{module: builds.BuilderModule, w: found, logf: t.Logf, now: time.Now}, nil, t.Logf)
rv.filled.Store(true)
type oldItem struct {
key string
raw []byte
}
var olds []oldItem
for _, m := range []string{"m1", "m2", "m3"} {
v, err := rv.Reserve(ctx, m, "0.1", []string{m + ":a"}, versions.Source{Fingerprint: "src2:" + m})
if err != nil {
t.Fatal(err)
}
if err := rv.Built(ctx, m, v.Version, map[string]string{m + ":a": "sha256:a"}, nil); err != nil {
t.Fatal(err)
}
raw, _ := json.Marshal(versions.QueuedPublish{V: 1, Holder: "go", Module: m, Version: v.Version, Artifacts: map[string]string{m + ":a": "sha256:a"}, Class: "asked-again-at-start"})
if _, err := old.Put(ctx, versions.QueueKey("go", m, v.Version), raw); err != nil {
t.Fatal(err)
}
olds = append(olds, oldItem{versions.QueueKey("go", m, v.Version), raw})
}
rp := &versions.Republisher{Reserved: rv.r, Put: func(ctx context.Context, q versions.QueuedPublish) (bool, error) {
data, _ := json.Marshal(q)
ack, err := b.js.Publish(ctx, pq.subject(bind, q), data, jetstream.WithMsgID(versions.PublishMsgID(q.Holder, q.Class, q.Module, q.Version)))
if err != nil {
return false, err
}
return ack.Duplicate, nil
}, Started: time.Now(), Logf: t.Logf}
oldBucket := builds.StateBucket("mesh-delivery-builder", versions.PublishQueueState)
moveOld(ctx, b.js, oldBucket, rp, t.Logf)
if _, err := b.js.KeyValue(ctx, oldBucket); !errors.Is(err, jetstream.ErrBucketNotFound) {
t.Errorf("after the move the old state's bucket answers %v, want it removed by the builder (H-A-461 (1))", err)
}
// A start that stopped before the removal moves the same items again: the duplicates window keeps each once.
again, err := b.js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: oldBucket})
if err != nil {
t.Fatal(err)
}
for _, e := range olds {
if _, err := again.Put(ctx, e.key, e.raw); err != nil {
t.Fatal(err)
}
}
moveOld(ctx, b.js, oldBucket, rp, t.Logf)
moveOld(ctx, b.js, oldBucket, rp, t.Logf) // no bucket left: nothing moves
if _, err := b.js.KeyValue(ctx, oldBucket); !errors.Is(err, jetstream.ErrBucketNotFound) {
t.Errorf("after the second move the old state's bucket answers %v, want it removed", err)
}
st, err := b.js.Stream(ctx, bind.Stream)
if err != nil {
t.Fatal(err)
}
info, err := st.Info(ctx, jetstream.WithSubjectFilter(bind.Subject+".publish.go.>"))
if err != nil {
t.Fatal(err)
}
if len(info.State.Subjects) != 3 || info.State.Subjects[bind.Subject+".publish.go.again.m1.0=2E1=2E0"] != 1 {
t.Errorf("the move run twice left %v on the work queue: want each of the three once, as again", info.State.Subjects)
}
// Design 47 H2 §796 (whole, H-A-453 (3)): a module's tags are read before it reserves; a number whose tag stands at another
// commit is skipped, the next number taken and version-tagged-elsewhere raised; the release tags <directory>/v<version> at
// the build's commit, and the module's version-released clears the finding.
rv.r.Names = func(context.Context, string, string) (string, bool, error) {
return "git.example/o/r/modules/m4", true, nil
}
// Design 47 H2 §821, §805: the reservation is said as the seat mesh-builds' version-reserved, said as the process says it
// (sayReservedOn), and waited on by the seat and the event's name.
rv.names, rv.sayReserved = rv.r.Names, sayReservedOn(b.js, process.Account{Module: builds.BuilderModule, Node: "n1"})
said, err := nats.Connect(b.srvURL, nats.UserInfo("ctl", "pw"))
if err != nil {
t.Fatal(err)
}
defer said.Close()
reservedCh := make(chan *nats.Msg, 4)
rs, err := said.ChanSubscribe(meshbuilds.SubjectVersionReserved, reservedCh)
if err != nil {
t.Fatal(err)
}
defer rs.Unsubscribe()
if err := said.Flush(); err != nil {
t.Fatal(err)
}
v4, err := rv.Reserve(ctx, "m4", "0.2", []string{"m4:lib"}, versions.Source{Fingerprint: "src2:m4", GoPackage: true, GoArtifact: "lib",
Repository: "o/r", Commit: "c-mine", Branch: "main", CatalogRevision: 1})
elsewhere := views.FindingKey(meshconditionsclient.ScopeModule, "m4", versions.VersionTaggedElsewhere)
if _, raised := found.vals[elsewhere]; err != nil || v4.Version != "0.2.1" || v4.Tag != "modules/m4/v0.2.1" || !raised {
t.Fatalf("m4 reserved %+v (%v), %s raised %v: want 0.2.1 past 0.2.0's tag at another commit, its tag modules/m4/v0.2.1, and the finding", v4, err, elsewhere, raised)
}
select {
case m := <-reservedCh:
ev, _, err := meshbuilds.DecodeVersionReservedEvent(m.Data)
if err != nil || ev.Module != "m4" || ev.Artifact != "m4:lib" || ev.Path != "git.example/o/r/modules/m4" || ev.Version != "0.2.1" ||
ev.Fingerprint != "src2:m4" || ev.Revision != "main.1" || m.Header.Get("x-source") != builds.BuilderModule {
t.Errorf("version-reserved = %+v (%v) from %q, want m4:lib at 0.2.1, its path, fingerprint src2:m4 and revision main.1, said by the builder", ev, err, m.Header.Get("x-source"))
}
case <-time.After(10 * time.Second):
t.Fatal("no version-reserved of m4's reservation within 10 s on " + meshbuilds.SubjectVersionReserved)
}
catalogEvents, err := b.js.StreamNameBySubject(ctx, meshcatalog.SubjectVersionReleased) // the bed's stream of the catalog's events
if err != nil {
t.Fatal(err)
}
evs, err := b.js.CreateOrUpdateConsumer(ctx, catalogEvents, jetstream.ConsumerConfig{Durable: "builder-events", AckPolicy: jetstream.AckExplicitPolicy,
FilterSubject: meshcatalog.SubjectVersionReleased, DeliverPolicy: jetstream.DeliverNewPolicy})
if err != nil {
t.Fatal(err)
}
next := func(module, version string) jetstream.Msg {
t.Helper()
data, _ := json.Marshal(meshcatalog.VersionReleasedEvent{Module: module, Version: version})
if _, err := b.js.Publish(ctx, meshcatalog.SubjectVersionReleased, data); err != nil {
t.Fatal(err)
}
m, err := evs.Next(jetstream.FetchMaxWait(10 * time.Second))
if err != nil {
t.Fatal(err)
}
return m
}
if !rv.heard(ctx, next("m4", "0.2.1")) {
t.Fatal("the reservations did not take version-released")
}
if _, raised := found.vals[elsewhere]; raised || !slices.Equal(tagsWritten, []string{"modules/m4/v0.2.1@c-mine"}) {
t.Errorf("after version-released %s raised %v, tags written %v: want it cleared and modules/m4/v0.2.1 written at c-mine, no tag moved", elsewhere, raised, tagsWritten)
}
// Design 47 H2 §794 (whole, H-A-453 (3), H-A-462): a message the builder's handler leaves unacknowledged is kept on its
// consumer's ledger in the state consumers on the bus; held six minutes, the health names that backlog, and within five
// minutes it does not.
consumers, err := b.js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: builds.StateBucket(builds.BuilderModule, process.ConsumersState)})
if err != nil {
t.Fatal(err)
}
held := time.Now()
hub := &oneHandler{}
aged := agedHub{hub: hub, ledgers: &ledgers{logf: t.Logf, now: func() time.Time { return held },
sink: consumersSink(process.AwaitBucket(ctx, b.js, nil, builds.StateBucket(builds.BuilderModule, process.ConsumersState), "", t.Logf))}}
_ = aged.Handle(meshcatalog.SubjectVersionReleased, func(m jetstream.Msg) { _ = m.Nak() }) // a handler standing still: asked again
hub.handle(next("m5", "0.1.0"))
state := versions.Reader{Keys: func() ([]string, error) {
l, err := consumers.ListKeys(ctx)
if err != nil {
return nil, err
}
var keys []string
for k := range l.Keys() {
keys = append(keys, k)
}
return keys, nil
}, Get: func(k string) (json.RawMessage, bool, error) {
e, err := consumers.Get(ctx, k)
if errors.Is(err, jetstream.ErrKeyNotFound) {
return nil, false, nil
}
if err != nil {
return nil, false, err
}
return e.Value(), true, nil
}}
if a := versions.BuilderHealth(state, held.Add(6*time.Minute), 5*time.Minute); a.Healthy || !strings.Contains(a.Why, "builder-events (its oldest unacknowledged message waits 6m0s") {
t.Errorf("the health with a backlog held six minutes = %+v, want unhealthy naming builder-events", a)
}
if a := versions.BuilderHealth(state, held.Add(4*time.Minute), 5*time.Minute); !a.Healthy {
t.Errorf("the health four minutes in = %+v, want healthy", a)
}
}
@@ -7,20 +7,28 @@ import (
"fmt"
"io/fs"
"os"
"sort"
"strconv"
"strings"
"sync"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-builder/internal/build"
"mesh-delivery-builder/internal/versions"
)
// **A version's publish is the builder's own work until its release is written** (design 47 G A-457a, §588): each reservation of a
// done build is a publish item in the builder's re-ask state, the mechanism builder_reserve_done and publish again use. It is
// asked when the build ends and at the builder's start; each try queues the publish and writes the item again with one more try
// and a longer pause; the release deletes it; after its tries it ends loudly and stays readable in builder_publish_queue.
// **A version's publish is the builder's own work until its release is written** (design 47 G A-457a, §588; H2 §793): each
// reservation of a done build is a publish item in the builder's re-ask state, asked when the build ends and at the builder's
// start; its try puts the publish on the work queue publishes, and each send counts one try in the item and hands a failed one
// back with a longer pause; the release deletes it; after its tries it ends loudly and stays readable in builder_publish_queue.
// publishSettings are the settings publish-pause, publish-pause-max and publish-tries.
type publishSettings struct {
@@ -28,8 +36,8 @@ type publishSettings struct {
Tries int
}
// defaultPublishSettings are the settings' defaults: 30 s, 30 min and 8 tries.
var defaultPublishSettings = publishSettings{Pause: 30 * time.Second, PauseMax: 30 * time.Minute, Tries: 8}
// defaultPublishSettings are the settings' defaults: 30 s, 30 min and 5 tries (H-A-446 (2)).
var defaultPublishSettings = publishSettings{Pause: 30 * time.Second, PauseMax: 30 * time.Minute, Tries: 5}
// pauseAfter is the pause after the try-th try: the setting doubled per try, up to the maximum.
func (s publishSettings) pauseAfter(try int) time.Duration {
@@ -61,10 +69,15 @@ type publishItems struct {
// publish there (H-A-275). Nil does neither.
markHeld func(ctx context.Context, v versions.ReservedVersion, held []versions.HeldArtifact)
moveElsewhere func(ctx context.Context, v versions.ReservedVersion, j versions.Judged) error
set publishSettings
now func() time.Time
after func(time.Duration) <-chan time.Time
logf func(string, ...any)
// send sends one item pulled from the work queue (Republisher.SendQueued); nil sends none.
send func(ctx context.Context, q versions.QueuedPublish) versions.SendFate
set publishSettings
now func() time.Time
after func(time.Duration) <-chan time.Time
logf func(string, ...any)
mu sync.Mutex
held map[string][]heldItem // "<module> <version>": the items sent, their publish not yet heard
}
func (p *publishItems) wait(ctx context.Context, d time.Duration) bool {
@@ -130,8 +143,11 @@ func (p *publishItems) AskAllDone(ctx context.Context) {
}
}
// Done deletes a version's publish item: its release is written.
// Done deletes a version's publish item and acknowledges its held items: its release is written.
func (p *publishItems) Done(ctx context.Context, module, version string) {
for _, h := range p.settle(module, version, func(heldItem) bool { return true }) {
_ = h.m.Ack()
}
kv := p.reasks()
if kv == nil {
return
@@ -141,9 +157,10 @@ func (p *publishItems) Done(ctx context.Context, module, version string) {
}
}
// Try waits for the item's next try, then, while the version is still reserved, queues its publish and writes the item again with
// one more try and the pause after it; once the tries are spent it ends the item loudly. A version no longer reserved (released,
// dropped or taken over) has its item deleted; one whose artifacts are published keeps it until its release is written.
// Try waits for the item's next try, then, while the version is still reserved, puts its publish on the work queue publishes
// under its message id (design 47 H2 §793; H-A-448): an item the queue holds already is put once. It counts no try: each send
// does, and hands its item back with the pause (take). A put that queued nothing is put again after the pause. A version no
// longer reserved (released, dropped or taken over) has its item deleted.
func (p *publishItems) Try(ctx context.Context, kv jetstream.KeyValue, e jetstream.KeyValueEntry, r build.ReAsk) {
if !p.wait(ctx, r.NextTry.Sub(p.now())) {
return
@@ -157,37 +174,215 @@ func (p *publishItems) Try(ctx context.Context, kv jetstream.KeyValue, e jetstre
p.logf("%s %s: its reservation cannot be read, the publish item waits: %v", r.Module, r.Version, err)
return
}
if why := versions.NotStanding(v, held); why != "" { // dropped, taken over, released or withdrawn: never because its artifacts are published (A-456b, H-A-290 (2))
if why := versions.NotStanding(v, held); why != "" { // dropped, taken over, released or withdrawn (A-456b, H-A-290 (2))
p.logf("%s %s: the publish item is ended: %s", r.Module, r.Version, why)
_ = kv.Delete(ctx, e.Key())
return
}
queued, err := p.republish(ctx, r.Module, r.Version)
if err == nil && len(queued) > 0 {
return
}
r.NextTry = p.now().UTC().Add(p.set.pauseAfter(r.Tries))
p.logf("%s %s: the publish is not on the work queue %s (%v); it is put again after %s", r.Module, r.Version, versions.PublishesLocal, err, p.set.pauseAfter(r.Tries))
if data, err := json.Marshal(r); err == nil {
if _, err := kv.Update(ctx, e.Key(), data, e.Revision()); err != nil && !errors.Is(err, jetstream.ErrKeyExists) && !errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
p.logf("%s: the publish item could not be written: %v", e.Key(), err)
}
}
}
// itemMsg is what take reads of an item pulled from the work queue (jetstream.Msg).
type itemMsg interface {
Data() []byte
Ack() error
NakWithDelay(time.Duration) error
Term() error
}
// take handles one item pulled from the work queue (§793; H-A-440 (2), H-A-448): an item whose publish item ended (dropped, or
// its tries spent) or whose version never publishes is terminated; one whose artifacts are all published is acknowledged; once
// publish-tries sends are spent it is terminated and version-unreleased said; while a published of it may be unheard it is handed
// back with the pause, and one the registries' record confirms is acknowledged (H-A-274); else it is sent, the try counted in its
// publish item, and a send that fails handed back with the pause as its delay, so the queue delivers it again and nothing is put
// again. A sent item is acknowledged when its publish is heard (heard, Done) and delivered again after the ack wait, unheard.
func (p *publishItems) take(ctx context.Context, m itemMsg) {
var q versions.QueuedPublish
if err := json.Unmarshal(m.Data(), &q); err != nil || q.Module == "" || q.Version == "" {
p.logf("an item of the work queue %s that is no publish is terminated: %v", versions.PublishesLocal, err)
_ = m.Term()
return
}
kv := p.reasks()
if kv == nil {
p.logf("%s %s: the re-ask state is not on the bus yet; the item is handed back", q.Module, q.Version)
_ = m.NakWithDelay(p.set.Pause)
return
}
key := versions.PublishItemKey(q.Module, q.Version)
e, err := kv.Get(ctx, key)
if errors.Is(err, jetstream.ErrKeyNotFound) { // put by builder_publish_again or the move: its tries are counted from now
if err = p.Ask(ctx, q.Module, q.Version); err == nil {
e, err = kv.Get(ctx, key)
}
}
if err != nil {
p.logf("%s %s: its publish item cannot be read (%v); the item is handed back", q.Module, q.Version, err)
_ = m.NakWithDelay(p.set.Pause)
return
}
r, err := build.DecodeReAsk(e.Value())
if err != nil {
_ = m.Term()
return
}
if r.Answer != "" {
p.logf("%s %s at %s is terminated: its publish item ended: %s", q.Module, q.Version, q.Holder, r.Answer)
_ = m.Term()
return
}
v, found, err := p.reserved(ctx, q.Module, q.Version)
if err != nil {
p.logf("%s %s: its reservation cannot be read (%v); the item is handed back", q.Module, q.Version, err)
_ = m.NakWithDelay(p.set.pauseAfter(r.Tries))
return
}
if why := versions.NotStanding(v, found); why != "" {
p.logf("%s %s at %s is terminated: %s", q.Module, q.Version, q.Holder, why)
_ = m.Term()
_ = kv.Delete(ctx, key)
return
}
if allPublished(v, q) {
p.logf("%s %s at %s is acknowledged: its artifacts are published", q.Module, q.Version, q.Holder)
_ = m.Ack()
return
}
if r.Tries >= p.set.Tries {
_ = m.Term()
r.Answer = fmt.Sprintf("the publish of %s %s was tried %d times and the version is not released; builder_publish_again tries it again", r.Module, r.Version, r.Tries)
r.Answered = p.now().UTC()
p.logf("%s: %s", e.Key(), r.Answer)
p.logf("%s: %s", key, r.Answer)
if p.endedWith != nil {
p.endedWith(ctx, v, "the publish ended after "+fmt.Sprint(r.Tries)+" tries")
}
} else if why, gone := p.notAgain(ctx, v); gone {
return // moved to another number, whose own item is asked (H-A-275)
} else if why != "" {
// A published of the version may be unheard, or the registry's record ended it: no ask, and no try spent (H-A-272, H-A-274).
p.logf("%s %s: the publish is not asked again: %s; tried again after %s", r.Module, r.Version, why, p.set.pauseAfter(r.Tries))
r.NextTry = p.now().UTC().Add(p.set.pauseAfter(r.Tries))
} else {
r.Tries++
if _, err := p.republish(ctx, r.Module, r.Version); err != nil {
p.logf("%s %s: try %d queued no publish: %v", r.Module, r.Version, r.Tries, err)
}
r.NextTry = p.now().UTC().Add(p.set.pauseAfter(r.Tries))
p.write(ctx, kv, key, r, e.Revision())
return
}
if why, gone := p.notAgain(ctx, v); gone {
_ = m.Term() // moved to another number, whose own item is asked (H-A-275)
return
} else if why != "" {
if v2, ok, err := p.reserved(ctx, q.Module, q.Version); err == nil && ok && allPublished(v2, q) {
p.logf("%s %s at %s is acknowledged: %s", q.Module, q.Version, q.Holder, why)
_ = m.Ack()
return
}
p.logf("%s %s: the publish is not sent again: %s; handed back for %s", q.Module, q.Version, why, p.set.pauseAfter(r.Tries))
_ = m.NakWithDelay(p.set.pauseAfter(r.Tries))
return
}
fate := versions.SendFate{Failed: "this builder sends no publish"}
if p.send != nil {
fate = p.send(ctx, q)
}
if fate.Ended != "" {
p.logf("%s %s at %s is terminated: %s", q.Module, q.Version, q.Holder, fate.Ended)
_ = m.Term()
return
}
r.Tries++
r.NextTry = p.now().UTC().Add(p.set.pauseAfter(r.Tries))
p.write(ctx, kv, key, r, e.Revision())
if fate.Failed != "" {
p.logf("%s %s at %s: try %d sent no publish: %s; handed back for %s", q.Module, q.Version, q.Holder, r.Tries, fate.Failed, p.set.pauseAfter(r.Tries))
_ = m.NakWithDelay(p.set.pauseAfter(r.Tries))
return
}
p.hold(q, m)
}
// write writes a publish item at its revision; a newer write wins.
func (p *publishItems) write(ctx context.Context, kv jetstream.KeyValue, key string, r build.ReAsk, rev uint64) {
data, err := json.Marshal(r)
if err != nil {
return
}
if _, err := kv.Update(ctx, e.Key(), data, e.Revision()); err != nil && !errors.Is(err, jetstream.ErrKeyExists) && !errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
p.logf("%s: the publish item could not be written: %v", e.Key(), err)
if _, err := kv.Update(ctx, key, data, rev); err != nil && !errors.Is(err, jetstream.ErrKeyExists) && !errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
p.logf("%s: the publish item could not be written: %v", key, err)
}
}
// allPublished says none of q's artifacts is missing from v: its publish is heard.
func allPublished(v versions.ReservedVersion, q versions.QueuedPublish) bool {
missing := map[string]bool{}
for _, ref := range v.Missing() {
missing[ref] = true
}
for ref := range q.Artifacts {
if missing[ref] {
return false
}
}
return true
}
// heldItem is an item sent and not yet acknowledged: its publish is not heard yet.
type heldItem struct {
q versions.QueuedPublish
m itemMsg
}
// hold keeps a sent item until its publish is heard.
func (p *publishItems) hold(q versions.QueuedPublish, m itemMsg) {
p.mu.Lock()
defer p.mu.Unlock()
if p.held == nil {
p.held = map[string][]heldItem{}
}
k := q.Module + " " + q.Version
p.held[k] = append(p.held[k], heldItem{q: q, m: m})
}
// settle takes every held item of module's version that pick picks out of the held ones.
func (p *publishItems) settle(module, version string, pick func(heldItem) bool) []heldItem {
p.mu.Lock()
defer p.mu.Unlock()
k := module + " " + version
var out, keep []heldItem
for _, h := range p.held[k] {
if pick(h) {
out = append(out, h)
} else {
keep = append(keep, h)
}
}
if len(keep) == 0 {
delete(p.held, k)
} else {
p.held[k] = keep
}
return out
}
// heard acknowledges each held item of module's version whose artifacts are all published now: its publish is heard (§793).
func (p *publishItems) heard(ctx context.Context, module, version string) {
v, found, err := p.reserved(ctx, module, version)
if err != nil || !found {
return
}
for _, h := range p.settle(module, version, func(h heldItem) bool { return allPublished(v, h.q) }) {
p.logf("%s %s at %s is acknowledged: its publish is heard", module, version, h.q.Holder)
_ = h.m.Ack()
}
}
// Ended deletes a version's publish item and terminates its held items: its build is withdrawn or superseded (H-A-290 (2),
// H-A-291 (1)); an item still on the work queue is terminated when it is pulled.
func (p *publishItems) Ended(ctx context.Context, module, version string) {
p.Done(ctx, module, version)
for _, h := range p.settle(module, version, func(heldItem) bool { return true }) {
_ = h.m.Term()
}
}
@@ -312,10 +507,11 @@ func publishSettingsOf(getenv func(string) string) (publishSettings, error) {
return publishSettings{Pause: time.Duration(pause) * time.Second, PauseMax: time.Duration(pmax) * time.Minute, Tries: tries}, nil
}
// wirePublishItems joins the reservations to the publish items (design 47 G A-457a, A-456a, A-456b): a done build asks its item, a
// release, a drop and a takeover end it, and a release is recorded in releases, the last kept of each module.
// wirePublishItems joins the reservations to the publish items (design 47 G A-457a, A-456a, A-456b, H2 §793): a done build asks its
// item, a release ends it and acknowledges its held items, a drop and a takeover end it and terminate them, a published heard
// acknowledges those it publishes, and a release is recorded in releases, the last kept of each module.
func wirePublishItems(rv *reservations, items *publishItems, releases versions.ReserveStore, kept int) {
rv.built, rv.released = items.Ask, items.Done
rv.built, rv.released, rv.itemEnded, rv.itemHeard = items.Ask, items.Done, items.Ended, items.heard
rv.r.ReleasedKept, rv.r.Releases = kept, releases
}
@@ -330,3 +526,434 @@ func builtOf(all []versions.ReservedVersion) []versions.ReservedVersion {
}
return out
}
// publishQueue is the builder's work queue publishes on the bus (design 47 H2 §793; H-A-440, H-A-446 (2), H-A-454): a stream of
// the manifest's old streams section with the work-queue retention, the module's own, consumed by the module's one process on the
// module's account (H-A-464), which declares no bus and so is the stream's one reader by the module's shape; each item on
// `mesh.stream.<module>.event.publish.<holder>.<class>.<module>.<version>` under the message id
// `<holder>.<class>.<module>.<version>`. The builder makes the pull consumers at run time as it learns a holder,
// `publishes-<holder>-<class>`, with ack wait publish-pause-max, max ack pending four and max deliveries ten, and pulls the live
// class first, at most four sends at once per holder.
type publishQueue struct {
js jetstream.JetStream
binding func() (publishesStream, error)
ackWait time.Duration
take func(ctx context.Context, m itemMsg)
logf func(string, ...any)
mu sync.Mutex
pulling map[string]bool
}
// publishesStream is where the work queue publishes is on the bus: its stream's name and the subject its items open with.
type publishesStream struct{ Stream, Subject string }
// publishesOf is module's work queue publishes, its own stream (H-A-454): the stream's bus name and its events' subject.
func publishesOf(module string) func() (publishesStream, error) {
b := publishesStream{Stream: catalogue.StreamBusName(versions.PublishesLocal),
Subject: strings.TrimSuffix(buscompose.ModuleStreamSubject(module, ""), ".")}
return func() (publishesStream, error) { return b, nil }
}
// The consumers' numbers (H-A-446 (2)): at most four sends in flight per holder and class, and ten deliveries, the work queue's
// largest; the builder ends an item at publish-tries anyway.
const (
publishesMaxAckPending = versions.PublishSendsAtOnce
publishesMaxDeliver = 10
)
// publishesConsumer is the durable consumer of holder's items of class.
func publishesConsumer(holder, class string) string {
return versions.PublishesLocal + "-" + holder + "-" + class
}
// subject is the item's subject on the bus.
func (pq *publishQueue) subject(b publishesStream, q versions.QueuedPublish) string {
return b.Subject + "." + versions.PublishEvent(q.Holder, q.Class, q.Module, q.Version)
}
// put puts q on the work queue under its message id; duplicate says the queue holds that id within its window. A put the full
// queue refuses is versions.ErrQueueFull. The item's holder is learnt: its pull starts.
func (pq *publishQueue) put(ctx context.Context, q versions.QueuedPublish) (bool, error) {
b, err := pq.binding()
if err != nil {
return false, err
}
data, err := json.Marshal(q)
if err != nil {
return false, err
}
ack, err := pq.js.PublishMsg(ctx, &nats.Msg{Subject: pq.subject(b, q), Data: data}, jetstream.WithMsgID(versions.PublishMsgID(q.Holder, q.Class, q.Module, q.Version)))
if err != nil {
if w := err.Error(); strings.Contains(w, "maximum bytes") || strings.Contains(w, "maximum messages") || errors.Is(err, jetstream.ErrMaxBytesExceeded) {
return false, fmt.Errorf("%w: %v", versions.ErrQueueFull, err)
}
return false, err
}
pq.learn(ctx, q.Holder)
return ack.Duplicate, nil
}
// learnAll learns every holder whose items the work queue holds: at the start, so what waits there is pulled.
func (pq *publishQueue) learnAll(ctx context.Context) error {
b, err := pq.binding()
if err != nil {
return err
}
st, err := pq.js.Stream(ctx, b.Stream)
if err != nil {
return err
}
info, err := st.Info(ctx, jetstream.WithSubjectFilter(b.Subject+".publish.>"))
if err != nil {
return err
}
for subj := range info.State.Subjects {
if holder, _, ok := strings.Cut(strings.TrimPrefix(subj, b.Subject+".publish."), "."); ok {
pq.learn(ctx, holder) // a holder is a module, its name one token
}
}
return nil
}
// learn starts holder's pull once.
func (pq *publishQueue) learn(ctx context.Context, holder string) {
pq.mu.Lock()
if pq.pulling == nil {
pq.pulling = map[string]bool{}
}
if pq.pulling[holder] {
pq.mu.Unlock()
return
}
pq.pulling[holder] = true
pq.mu.Unlock()
if err := pq.pull(ctx, holder); err != nil {
pq.logf("the publishes of %s are not pulled: %v; they are pulled when an item of it is put", holder, err)
pq.mu.Lock()
delete(pq.pulling, holder)
pq.mu.Unlock()
}
}
// pull makes holder's two consumers and dispatches their items, the live class first, at most four sends at once: an again item
// waits while a live one waits on the queue.
func (pq *publishQueue) pull(ctx context.Context, holder string) error {
b, err := pq.binding()
if err != nil {
return err
}
chans := map[string]chan jetstream.Msg{}
var live jetstream.Consumer
for _, class := range []string{versions.PublishClassStart, versions.PublishClassLive} { // the again first: the live is first by the dispatch, not by luck
c, err := pq.js.CreateOrUpdateConsumer(ctx, b.Stream, jetstream.ConsumerConfig{Durable: publishesConsumer(holder, class),
FilterSubject: b.Subject + ".publish." + versions.EscapeToken(holder) + "." + class + ".>", AckPolicy: jetstream.AckExplicitPolicy,
AckWait: pq.ackWait, MaxAckPending: publishesMaxAckPending, MaxDeliver: publishesMaxDeliver})
if err != nil {
return fmt.Errorf("the consumer %s: %w", publishesConsumer(holder, class), err)
}
ch := make(chan jetstream.Msg, publishesMaxAckPending)
cc, err := c.Consume(func(m jetstream.Msg) { ch <- m }, jetstream.PullMaxMessages(publishesMaxAckPending))
if err != nil {
return fmt.Errorf("the consumer %s is not pulled: %w", publishesConsumer(holder, class), err)
}
go func() { <-ctx.Done(); cc.Stop() }()
chans[class] = ch
if class == versions.PublishClassLive {
live = c
}
}
liveWaits := func() bool {
info, err := live.Info(ctx)
return err == nil && info.NumPending > 0
}
pq.logf("pulling the publishes of %s, the live first", holder)
go pq.dispatch(ctx, chans[versions.PublishClassLive], chans[versions.PublishClassStart], liveWaits)
return nil
}
// dispatch runs the items of one holder, at most four at once, every live item before an again one: an again item pulled while
// a live one waits on the queue (liveWaits) runs after it.
func (pq *publishQueue) dispatch(ctx context.Context, live, again <-chan jetstream.Msg, liveWaits func() bool) {
sem := make(chan struct{}, versions.PublishSendsAtOnce)
run := func(m jetstream.Msg) {
go func() {
defer func() { <-sem }()
pq.take(ctx, m)
}()
}
var spare jetstream.Msg // an again item pulled while a live one waited
for {
select {
case sem <- struct{}{}:
case <-ctx.Done():
return
}
select {
case m := <-live:
run(m)
continue
default:
}
m := spare
spare = nil
if m == nil {
select {
case lm := <-live:
run(lm)
continue
case m = <-again:
case <-ctx.Done():
return
}
}
if liveWaits() {
spare = m
select {
case lm := <-live:
run(lm)
case <-ctx.Done():
return
}
continue
}
select {
case lm := <-live: // came while the queue was asked
spare = m
run(lm)
default:
run(m)
}
}
}
// itemsOf are the items the work queue holds of module's version in class (empty for any class), each with its subject.
func (pq *publishQueue) itemsOf(ctx context.Context, module, version, class string) (map[string]versions.QueuedPublish, jetstream.Stream, error) {
b, err := pq.binding()
if err != nil {
return nil, nil, err
}
st, err := pq.js.Stream(ctx, b.Stream)
if err != nil {
return nil, nil, err
}
cls := "*"
if class != "" {
cls = class
}
info, err := st.Info(ctx, jetstream.WithSubjectFilter(b.Subject+".publish.*."+cls+"."+versions.EscapeToken(module)+"."+versions.EscapeToken(version)))
if err != nil {
return nil, nil, err
}
out := map[string]versions.QueuedPublish{}
for subj := range info.State.Subjects {
raw, err := st.GetLastMsgForSubject(ctx, subj)
if err != nil {
continue
}
var q versions.QueuedPublish
if json.Unmarshal(raw.Data, &q) == nil {
out[subj] = q
}
}
return out, st, nil
}
// publishQueueItemsRead bounds the verb publish-queue's read of the work queue: at most 4,000 items, past them the answer is capped,
// naming the items read and the queue's size (H-A-293).
const publishQueueItemsRead = 4000
// reader is the work queue as the versions package's reader, for the seat mesh-builds' verb publish-queue (§793, H-A-454): its
// items read from the stream by sequence, at most publishQueueItemsRead, each keyed by its sequence; a queue holding more is read
// capped, with the stream's message count and bytes.
func (pq *publishQueue) reader(ctx context.Context) versions.Reader {
none := func() ([]string, error) { return nil, errors.ErrUnsupported }
return versions.Reader{Keys: none, All: func() (versions.StatePage, error) {
out := versions.StatePage{Values: map[string]json.RawMessage{}}
b, err := pq.binding()
if err != nil {
return out, err
}
st, err := pq.js.Stream(ctx, b.Stream)
if err != nil {
return out, err
}
info, err := st.Info(ctx)
if err != nil {
return out, err
}
held := int(info.State.Msgs)
if held == 0 {
return out, nil
}
// A work queue takes no second consumer over its subjects, so its items are read by sequence, the deleted passed over.
for seq := info.State.FirstSeq; seq <= info.State.LastSeq && len(out.Values) < publishQueueItemsRead; seq++ {
m, err := st.GetMsg(ctx, seq)
if errors.Is(err, jetstream.ErrMsgNotFound) {
continue // acknowledged or terminated since
}
if err != nil {
return out, err
}
if strings.HasPrefix(m.Subject, b.Subject+".publish.") {
out.Values[strconv.FormatUint(seq, 10)] = json.RawMessage(m.Data)
}
}
if held > len(out.Values) && len(out.Values) >= publishQueueItemsRead {
out.More, out.Keys, out.Bytes = true, info.State.Msgs, info.State.Bytes
}
return out, nil
}}
}
// drop is builder_publish_drop (§793): the version's publish item is ended with who dropped it and why, so no item of it is ever
// sent or put again; its items waiting on the work queue are removed, and one held is terminated.
func (pq *publishQueue) drop(ctx context.Context, items *publishItems, module, version, by, why string) (string, error) {
qs, st, err := pq.itemsOf(ctx, module, version, "")
if err != nil {
return "", fmt.Errorf("the work queue %s cannot be read: %w", versions.PublishesLocal, err)
}
if err := items.endItem(ctx, module, version, fmt.Sprintf("dropped by %s: %s", by, why)); err != nil {
return "", err
}
for subj := range qs {
if err := st.Purge(ctx, jetstream.WithPurgeSubject(subj)); err != nil {
return "", fmt.Errorf("the item %s is not terminated: %w", subj, err)
}
}
for _, h := range items.settle(module, version, func(heldItem) bool { return true }) {
_ = h.m.Term()
}
return fmt.Sprintf("the publish of %s %s is dropped: %d items terminated, and none is sent or put again", module, version, len(qs)), nil
}
// first is builder_publish_first (§793): each again item of the version is put again as live, under the message id
// `<holder>.live.<module>.<version>`, and the old terminated; a live id the queue holds within its window takes nothing, and the
// old stays, said.
func (pq *publishQueue) first(ctx context.Context, items *publishItems, module, version string) (string, error) {
qs, st, err := pq.itemsOf(ctx, module, version, versions.PublishClassStart)
if err != nil {
return "", fmt.Errorf("the work queue %s cannot be read: %w", versions.PublishesLocal, err)
}
if len(qs) == 0 {
return "", fmt.Errorf("the work queue %s holds no again item of %s %s", versions.PublishesLocal, module, version)
}
var moved, kept []string
for subj, q := range qs {
q.Class = versions.PublishClassLive
dup, err := pq.put(ctx, q)
if err != nil {
return "", fmt.Errorf("%s %s is not put again as live at %s: %w", module, version, q.Holder, err)
}
if dup {
kept = append(kept, q.Holder)
continue
}
if err := st.Purge(ctx, jetstream.WithPurgeSubject(subj)); err != nil {
return "", fmt.Errorf("the again item of %s %s at %s is not terminated: %w", module, version, q.Holder, err)
}
moved = append(moved, q.Holder)
}
for _, h := range items.settle(module, version, func(h heldItem) bool { return h.q.Class == versions.PublishClassStart }) {
_ = h.m.Term()
}
sort.Strings(moved)
answer := fmt.Sprintf("%s %s is put first as live at %s, its again item terminated", module, version, strings.Join(moved, ", "))
if len(kept) > 0 {
sort.Strings(kept)
answer += fmt.Sprintf("; at %s the work queue holds its live id within the duplicates window, so its again item stays", strings.Join(kept, ", "))
}
return answer, nil
}
// endItem writes the version's publish item ended with answer, made when none stands: Try and take then leave it, and a start
// asks it no more.
func (p *publishItems) endItem(ctx context.Context, module, version, answer string) error {
kv := p.reasks()
if kv == nil {
return errors.New("the re-ask state is not on the bus yet")
}
key := versions.PublishItemKey(module, version)
for range 5 {
r := build.ReAsk{V: 1, Why: "the build ended with its version unreleased", By: "the builder", At: p.now().UTC(), Mode: build.ModePublish, Module: module, Version: version}
e, err := kv.Get(ctx, key)
var rev uint64
switch {
case errors.Is(err, jetstream.ErrKeyNotFound):
case err != nil:
return err
default:
if r, err = build.DecodeReAsk(e.Value()); err != nil {
return err
}
rev = e.Revision()
}
r.Answer, r.Answered = answer, p.now().UTC()
data, err := json.Marshal(r)
if err != nil {
return err
}
if rev == 0 {
_, err = kv.Create(ctx, key, data)
} else {
_, err = kv.Update(ctx, key, data, rev)
}
if err == nil {
return nil
}
if !errors.Is(err, jetstream.ErrKeyExists) && !errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
return err
}
}
return errors.New("the publish item of " + module + " " + version + " changed five times while it was ended")
}
// moveOld puts the items of the old state publish-queue on the work queue once at the start (§793): each under its message id, so
// a move run twice puts each once within the duplicates window; then the builder, which made that bucket, removes it itself
// (H-A-461 (1): the bus module's keeper holds no entry for it, so its drop refuses it). A builder whose bus holds no such
// bucket moves nothing; a move that did not put every item keeps the bucket for the next start.
func moveOld(ctx context.Context, js jetstream.JetStream, bucket string, rp *versions.Republisher, logf func(string, ...any)) {
kv, err := js.KeyValue(ctx, bucket)
if errors.Is(err, jetstream.ErrBucketNotFound) {
return
}
if err != nil {
logf("the old state %s cannot be read, so its items are not moved to the work queue %s: %v", versions.PublishQueueState, versions.PublishesLocal, err)
return
}
keys, err := kv.ListKeys(ctx)
if err != nil {
logf("the old state %s cannot be listed, so its items are not moved: %v", versions.PublishQueueState, err)
return
}
var vals []json.RawMessage
for k := range keys.Keys() {
if e, err := kv.Get(ctx, k); err == nil {
vals = append(vals, json.RawMessage(e.Value()))
}
}
n, err := rp.MoveQueued(ctx, vals)
if err != nil {
logf("the old state %s is not removed: %d of its %d items moved to the work queue %s before: %v", versions.PublishQueueState, n, len(vals), versions.PublishesLocal, err)
return
}
if err := js.DeleteKeyValue(ctx, bucket); err != nil && !errors.Is(err, jetstream.ErrBucketNotFound) {
logf("the old state %s: %d of its %d items put on the work queue %s, but its bucket is not removed: %v; the next start moves it again", versions.PublishQueueState, n, len(vals), versions.PublishesLocal, err)
return
}
logf("the old state %s: %d of its %d items put on the work queue %s; its bucket is removed", versions.PublishQueueState, n, len(vals), versions.PublishesLocal)
}
// publishQueueVerb is the handler of the seat mesh-builds' verb publish-queue (§793; H-A-454): the builder reads its own work queue
// and answers its counts per holder and class, the oldest and newest, and a capped list; builder_publish_queue asks it.
func publishQueueVerb(ctx context.Context, pq *publishQueue) process.SeatHandler {
return func(_ string, body json.RawMessage) (any, error) {
var in meshbuilds.PublishQueueInput
if len(body) > 0 {
if err := json.Unmarshal(body, &in); err != nil {
return nil, fmt.Errorf("the arguments do not fit the verb publish-queue: %w", err)
}
}
return versions.PublishQueueOf(pq.reader(ctx), in.Module, in.Version)
}
}
@@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"errors"
"fmt"
"maps"
"os"
"path/filepath"
@@ -89,6 +90,29 @@ func (k *itemKV) Delete(_ context.Context, key string, _ ...jetstream.KVDeleteOp
return nil
}
// itemMsgFake is an item pulled from the work queue: what take settled it with.
type itemMsgFake struct {
data []byte
settled []string // ack, term, nak <delay>
}
func (m *itemMsgFake) Data() []byte { return m.data }
func (m *itemMsgFake) Ack() error { m.settled = append(m.settled, "ack"); return nil }
func (m *itemMsgFake) Term() error { m.settled = append(m.settled, "term"); return nil }
func (m *itemMsgFake) NakWithDelay(d time.Duration) error {
m.settled = append(m.settled, "nak "+d.String())
return nil
}
func queuedMsg(t *testing.T, module, version string) *itemMsgFake {
t.Helper()
raw, err := json.Marshal(versions.QueuedPublish{V: 1, Holder: "oci", Module: module, Version: version, Artifacts: map[string]string{module + ":a": "sha256:a"}, Class: versions.PublishClassLive})
if err != nil {
t.Fatal(err)
}
return &itemMsgFake{data: raw}
}
func decodeItem(t *testing.T, raw []byte) build.ReAsk {
t.Helper()
r, err := build.DecodeReAsk(raw)
@@ -98,21 +122,21 @@ func decodeItem(t *testing.T, raw []byte) build.ReAsk {
return r
}
// TestAPublishItemIsTriedWithAGrowingPauseAndEndsLoudly: design 47 G A-457a — a version's publish item is asked once, each try queues
// the publish and writes the item again with one more try and the pause doubled up to its maximum, the release deletes it, and
// after the setting's tries it ends with its words, raises the condition and stays; a version no longer reserved has it deleted.
// TestAPublishItemIsTriedWithAGrowingPauseAndEndsLoudly: design 47 G A-457a, H2 §793 (H-A-440 (2), H-A-448) — a version's publish
// item is asked once; its try puts the publish on the work queue and counts no try; each send counts one and a failed send hands
// the item back with the pause doubled up to its maximum; after the setting's tries the item is terminated and ends with its words,
// raises the condition and stays; an item whose artifacts are published is acknowledged; the release deletes it; a version no
// longer reserved has it deleted.
func TestAPublishItemIsTriedWithAGrowingPauseAndEndsLoudly(t *testing.T) {
ctx := context.Background()
kv := &itemKV{vals: map[string][]byte{}, revs: map[string]uint64{}}
now := time.Date(2026, 11, 17, 1, 0, 0, 0, time.UTC)
var waited []time.Duration
var queued, ended []string
var queued, ended, sent []string
reserved := true
v := versions.ReservedVersion{Module: "m", Version: "0.1.2", Artifacts: map[string]string{"m:a": "sha256:a"}, BuiltAt: now}
p := &publishItems{reasks: func() jetstream.KeyValue { return kv }, set: publishSettings{Pause: 30 * time.Second, PauseMax: 2 * time.Minute, Tries: 4},
now: func() time.Time { return now }, logf: t.Logf,
after: func(d time.Duration) <-chan time.Time {
waited = append(waited, d)
now = now.Add(d)
c := make(chan time.Time, 1)
c <- now
@@ -126,7 +150,11 @@ func TestAPublishItemIsTriedWithAGrowingPauseAndEndsLoudly(t *testing.T) {
},
republish: func(_ context.Context, m, ver string) ([]string, error) {
queued = append(queued, m+" "+ver)
return nil, errors.New("no holder")
return []string{m + " " + ver}, nil
},
send: func(_ context.Context, q versions.QueuedPublish) versions.SendFate {
sent = append(sent, q.Module+" "+q.Version)
return versions.SendFate{Failed: "no holder answers"}
},
endedWith: func(_ context.Context, v versions.ReservedVersion, why string) {
ended = append(ended, v.Version+": "+why)
@@ -135,25 +163,34 @@ func TestAPublishItemIsTriedWithAGrowingPauseAndEndsLoudly(t *testing.T) {
if err := p.Ask(ctx, "m", "0.1.2"); err != nil || p.Ask(ctx, "m", "0.1.2") != nil || len(kv.vals) != 1 {
t.Fatalf("asked: %v, %d items, want one item however often it is asked", err, len(kv.vals))
}
for i := 0; i < 5; i++ {
e, _ := kv.Get(ctx, key)
p.Try(ctx, kv, e, decodeItem(t, e.Value()))
e, _ := kv.Get(ctx, key)
p.Try(ctx, kv, e, decodeItem(t, e.Value()))
if len(queued) != 1 || decodeItem(t, kv.vals[key]).Tries != 0 {
t.Fatalf("the try queued %v, tries %d: want the publish put once, no try counted", queued, decodeItem(t, kv.vals[key]).Tries)
}
var settled []string
for range 5 {
m := queuedMsg(t, "m", "0.1.2")
p.take(ctx, m)
settled = append(settled, m.settled...)
}
got := decodeItem(t, kv.vals[key])
if len(queued) != 4 || got.Tries != 4 || got.Answer == "" || got.Answered.IsZero() || len(ended) != 1 || !strings.Contains(ended[0], "4 tries") {
t.Fatalf("queued %v, item %+v, ended %v: want four tries, then the end said with its words and the condition", queued, got, ended)
if len(sent) != 4 || got.Tries != 4 || got.Answer == "" || got.Answered.IsZero() || len(ended) != 1 || !strings.Contains(ended[0], "4 tries") {
t.Fatalf("sent %v, item %+v, ended %v: want four sends, then the end said with its words and the condition", sent, got, ended)
}
if want := []time.Duration{30 * time.Second, time.Minute, 2 * time.Minute, 2 * time.Minute}; len(waited) != len(want) || waited[0] != want[0] || waited[1] != want[1] || waited[2] != want[2] || waited[3] != want[3] {
t.Errorf("pauses before the tries and the end = %v, want %v (doubled, capped)", waited, want)
if want := "[nak 30s nak 1m0s nak 2m0s nak 2m0s term]"; fmt.Sprint(settled) != want {
t.Errorf("the items were settled %v, want %s (handed back with the pause doubled and capped, then terminated)", settled, want)
}
// An item whose artifacts are all published stays until its release is written, and is tried (A-456b).
queued = nil
m := queuedMsg(t, "m", "0.1.2")
if p.take(ctx, m); fmt.Sprint(m.settled) != "[term]" || len(sent) != 4 {
t.Errorf("an item of an ended publish item was settled %v, sent %d: want it terminated unsent", m.settled, len(sent))
}
// An item whose artifacts are all published is acknowledged unsent; its publish item stays until its release is written (A-456b).
_ = p.Ask(ctx, "m", "0.3.0")
pk := versions.PublishItemKey("m", "0.3.0")
pe, _ := kv.Get(ctx, pk)
p.Try(ctx, kv, pe, decodeItem(t, pe.Value()))
if _, ok := kv.vals[pk]; !ok || len(queued) != 1 || decodeItem(t, kv.vals[pk]).Tries != 1 {
t.Errorf("an item whose artifacts are published was deleted or not tried: %v, queued %v", kv.vals, queued)
m = queuedMsg(t, "m", "0.3.0")
if p.take(ctx, m); fmt.Sprint(m.settled) != "[ack]" || len(sent) != 4 || !kv.has(pk) {
t.Errorf("an item whose artifacts are published was settled %v, sent %d, kept %v: want it acknowledged unsent, its publish item kept", m.settled, len(sent), kv.has(pk))
}
kv.vals, kv.revs = map[string][]byte{}, map[string]uint64{}
// At the start every reservation of a done build with no release is asked, one item each.
@@ -194,7 +231,7 @@ func TestAPublishItemIsTriedWithAGrowingPauseAndEndsLoudly(t *testing.T) {
queued = nil
_ = p.Ask(ctx, "m", "0.1.2")
v.Withdrawn = map[string]string{"0.1.2": "fp"}
e, _ := kv.Get(ctx, key)
e, _ = kv.Get(ctx, key)
p.Try(ctx, kv, e, decodeItem(t, e.Value()))
if len(kv.vals) != 0 || len(queued) != 0 {
t.Errorf("a withdrawn reservation left its item %v, queued %v", kv.vals, queued)
@@ -209,7 +246,8 @@ func TestAPublishItemIsTriedWithAGrowingPauseAndEndsLoudly(t *testing.T) {
}
}
// TestThePublishSettingsAreReadFromTheirFiles: design 47 G A-457a — the three settings, 30 s, 30 min and 8 tries unless set, and a
// TestThePublishSettingsAreReadFromTheirFiles: design 47 G A-457a, H2 H-A-446 (2) — the three settings, 30 s, 30 min and 5 tries
// unless set, and a
// content that is no number is refused naming its file.
func TestThePublishSettingsAreReadFromTheirFiles(t *testing.T) {
dir := t.TempDir()
@@ -222,7 +260,7 @@ func TestThePublishSettingsAreReadFromTheirFiles(t *testing.T) {
}
env := map[string]string{}
got, err := publishSettingsOf(func(k string) string { return env[k] })
if err != nil || got != defaultPublishSettings || got.Pause != 30*time.Second || got.PauseMax != 30*time.Minute || got.Tries != 8 {
if err != nil || got != defaultPublishSettings || got.Pause != 30*time.Second || got.PauseMax != 30*time.Minute || got.Tries != 5 {
t.Fatalf("defaults = %+v (%v)", got, err)
}
env["MESH_BUILDER_PUBLISH_PAUSE_FILE"], env["MESH_BUILDER_PUBLISH_PAUSE_MAX_FILE"], env["MESH_BUILDER_PUBLISH_TRIES_FILE"] = write("p", "5\n"), write("m", "2"), write("t", "3")
@@ -353,9 +391,12 @@ func TestAPublishIsNotAskedAgainWhileItsPublishedMayBeUnheard(t *testing.T) {
kv := &itemKV{vals: map[string][]byte{}, revs: map[string]uint64{}}
asked, judged, marked, moved := 0, 0, 0, 0
p := &publishItems{reasks: func() jetstream.KeyValue { return kv }, set: defaultPublishSettings, now: func() time.Time { return now }, logf: t.Logf,
reserved: func(context.Context, string, string) (versions.ReservedVersion, bool, error) { return v, true, nil },
republish: func(context.Context, string, string) ([]string, error) { asked++; return nil, nil },
unheard: func(context.Context) (int, error) { return c.unheard, c.countErr },
reserved: func(context.Context, string, string) (versions.ReservedVersion, bool, error) { return v, true, nil },
send: func(context.Context, versions.QueuedPublish) versions.SendFate {
asked++
return versions.SendFate{Sent: true}
},
unheard: func(context.Context) (int, error) { return c.unheard, c.countErr },
judge: func(context.Context, versions.ReservedVersion) (versions.Judged, error) {
judged++
return c.judged, nil
@@ -374,20 +415,24 @@ func TestAPublishIsNotAskedAgainWhileItsPublishedMayBeUnheard(t *testing.T) {
t.Fatal(err)
}
key := versions.PublishItemKey(v.Module, v.Version)
e, _ := kv.Get(ctx, key)
p.Try(ctx, kv, e, decodeItem(t, e.Value()))
m := queuedMsg(t, v.Module, v.Version)
m.data = []byte(strings.ReplaceAll(string(m.data), `"node-tools:a"`, `"node-tools:program"`))
p.take(ctx, m)
if (asked == 1) != c.wantAsked || (marked == 1) != c.wantMark || (moved == 1) != c.wantMov {
t.Fatalf("asked %d, marked %d, moved %d; want asked %v, marked %v, moved %v", asked, marked, moved, c.wantAsked, c.wantMark, c.wantMov)
}
if c.wantMov {
if _, ok := kv.vals[key]; ok {
t.Error("the moved version's old item is written again")
if _, ok := kv.vals[key]; ok || fmt.Sprint(m.settled) != "[term]" {
t.Errorf("the moved version's old item is written again, or its item settled %v, not terminated", m.settled)
}
return
}
got := decodeItem(t, kv.vals[key])
if got.Tries != map[bool]int{true: 1, false: 0}[c.wantAsked] || !got.NextTry.After(now) {
t.Errorf("tries %d, next try %s: want a try spent only when asked, and the item written again with its pause", got.Tries, got.NextTry)
if got.Tries != map[bool]int{true: 1, false: 0}[c.wantAsked] || (c.wantAsked && !got.NextTry.After(now)) {
t.Errorf("tries %d, next try %s: want a try spent only when sent, the item written with its pause", got.Tries, got.NextTry)
}
if !c.wantAsked && fmt.Sprint(m.settled) != "[nak 30s]" {
t.Errorf("an item not sent was settled %v, want it handed back with the pause and no try spent", m.settled)
}
if c.unheard >= versions.UnheardBound && judged != 0 {
t.Error("the registry is read while a published may be unheard")
@@ -65,7 +65,7 @@ func reAsked(ctx context.Context, b askAgainer, acts versions.Acts, items *publi
released(ctx, b, kv, e, r, logf)
return
}
if r.Mode == build.ModeRepublish || r.Mode == build.ModeFloorAgain || r.Mode == build.ModeOldFormDrop || r.Mode == build.ModeReservedDrop || r.Mode == build.ModeReserveDone {
if r.Mode == build.ModeRepublish || r.Mode == build.ModePublishDrop || r.Mode == build.ModePublishFirst || r.Mode == build.ModeFloorAgain || r.Mode == build.ModeOldFormDrop || r.Mode == build.ModeReservedDrop || r.Mode == build.ModeReserveDone {
// A version's hand act (design 51 *Tools of the builder*) may build again, so it is answered beside the watch.
go versionActed(ctx, acts, kv, e, r, logf)
return
@@ -208,6 +208,16 @@ func versionActed(ctx context.Context, acts versions.Acts, kv jetstream.KeyValue
r.Answer = "queued " + fmt.Sprint(len(r.Queued)) + " versions to publish again: " + strings.Join(r.Queued, ", ") + "; builder_publish_queue shows them"
}
}
} else if r.Mode == build.ModePublishDrop || r.Mode == build.ModePublishFirst {
act := acts.PublishDrop
if r.Mode == build.ModePublishFirst {
act = acts.PublishFirst
}
if act == nil {
err = errors.New("this builder keeps no work queue of publishes")
} else {
r.Answer, err = act(ctx, r.Module, r.Version, r.By, r.Why)
}
} else if r.Mode == build.ModeReserveDone {
if acts.ReserveDone == nil {
err = errors.New("this builder keeps no reservations")
@@ -2,35 +2,22 @@ package main
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"sync/atomic"
"github.com/nats-io/nats.go/jetstream"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/gopackageregistry"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-builder/internal/versions"
)
// The builder's publish again (design 47 G §588, A-73, A-76): the wiring of versions.Republisher to the bus.
// awaitedQueue is the state publish-queue as versions.QueueStore: bound once the bus holds it; its bytes are the cap's.
type awaitedQueue struct{ awaitedReserved }
func (s awaitedQueue) Bytes(ctx context.Context) (uint64, error) {
kv, err := s.kv()
if err != nil {
return 0, err
}
st, err := kv.Status(ctx)
if err != nil {
return 0, err
}
return st.Bytes(), nil
}
// entryVia reads a catalog graph entry by its fingerprint through the seat mesh-catalog's verb entry.
func entryVia(call toolCaller) func(ctx context.Context, fingerprint string) (meshcatalog.CatalogEntry, error) {
return func(ctx context.Context, fingerprint string) (meshcatalog.CatalogEntry, error) {
@@ -46,12 +33,28 @@ func entryVia(call toolCaller) func(ctx context.Context, fingerprint string) (me
}
}
// keptCaller is a kept call carrying an ask id (process.Conn.CallKeptAsk).
type keptCaller func(ask process.KeptAsk, key string, body any) (json.RawMessage, error)
// publishAskID is the ask id of p's publish (design 47 H2 §775): the same seat, name, version and digest make the same id,
// so a publish made again is one the holder takes once (ADR 0668 §2); an id holds no dot, as every ask id.
func publishAskID(p versions.Publish) string {
sum := sha256.Sum256([]byte(p.Seat + "\n" + p.Name + "\n" + p.Version + "\n" + p.Digest))
return "publish" + hex.EncodeToString(sum[:8])
}
// sendVia sends a publish to the registry seat with the digest and no parts (A-73): the holder takes the bytes from the
// artifact store, addressed by digest. The four seats' publish take one wire form; an answer that lacks parts, or a fault of
// the call, is the holder's refusal, said in its words (not stored when the holder lacks the bytes).
func sendVia(call toolCaller) func(ctx context.Context, p versions.Publish) error {
// the call, is the holder's refusal, said in its words (not stored when the holder lacks the bytes). The publish is a kept
// call carrying its ask id (design 47 H2 §775), made again through transport faults at the kept-call pace within ctx's bound.
func sendVia(call keptCaller) func(ctx context.Context, p versions.Publish) error {
return func(ctx context.Context, p versions.Publish) error {
raw, err := callWithin(ctx, call, "seat:"+p.Seat+"."+versions.VerbPublish, publishBody(p))
ask := process.KeptAsk{ID: publishAskID(p)}
if d, ok := ctx.Deadline(); ok {
ask.Bound = time.Until(d)
}
raw, err := callWithin(ctx, func(key string, body any) (json.RawMessage, error) { return call(ask, key, body) },
"seat:"+p.Seat+"."+versions.VerbPublish, publishBody(p))
if err != nil {
return fmt.Errorf("%s's publish of %s %s: %w", p.Seat, p.Name, p.Version, err)
}
@@ -93,21 +96,3 @@ func callWithin(ctx context.Context, call toolCaller, key string, body any) (jso
func publishBody(p versions.Publish) gopackageregistry.PublishInput {
return gopackageregistry.PublishInput{Commit: p.Commit, Digest: p.Digest, Fingerprint: p.Fingerprint, Kind: p.Kind, Name: p.Name, Transfer: p.Transfer, Version: p.Version}
}
// republishing starts the publish again once both its states are bound: the queue is taken up again (design 47 G §588). A version's
// publish is asked when its build ends and at the start (A-457a), never waited on a running or manifest-reads-changed.
type republishing struct {
rp *versions.Republisher
ready atomic.Int32
}
func (g *republishing) bound(ctx context.Context, js jetstream.JetStream, logf func(string, ...any)) func(jetstream.KeyValue) {
return func(jetstream.KeyValue) {
if g.ready.Add(1) != 2 {
return
}
if err := g.rp.Resume(ctx); err != nil {
logf("the queued publishes are not taken up again: %v", err)
}
}
}
@@ -13,6 +13,8 @@ import (
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/npmpackageregistry"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/ociartifactregistry"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-builder/internal/build"
"mesh-delivery-builder/internal/versions"
)
@@ -48,12 +50,14 @@ func TestRepublishSend(t *testing.T) {
"a fault": {"", errors.New("no responders"), "no responders"},
"no sense": {`[`, nil, "no publish answer"},
} {
send := sendVia(func(key string, body any) (json.RawMessage, error) {
asked = key
var id string
send := sendVia(func(ask process.KeptAsk, key string, body any) (json.RawMessage, error) {
asked, id = key, ask.ID
return json.RawMessage(c.answer), c.err
})
err := send(context.Background(), p)
if asked != "seat:oci-artifact-registry.publish" || (c.want == "") != (err == nil) || (err != nil && !strings.Contains(err.Error(), c.want)) {
// design 47 H2 §775: the publish carries its ask id, the same for the same publish
if asked != "seat:oci-artifact-registry.publish" || id != publishAskID(p) || strings.Contains(id, ".") || (c.want == "") != (err == nil) || (err != nil && !strings.Contains(err.Error(), c.want)) {
t.Errorf("%s: asked %q, error %v, want %q", name, asked, err, c.want)
}
}
@@ -68,8 +72,10 @@ func TestRepublishSend(t *testing.T) {
return json.RawMessage(`{}`), nil
}
for name, call := range map[string]func(context.Context) error{
"the publish": func(ctx context.Context) error { return sendVia(slow)(ctx, p) },
"the entry": func(ctx context.Context) error { _, err := entryVia(slow)(ctx, "fp"); return err },
"the publish": func(ctx context.Context) error {
return sendVia(func(_ process.KeptAsk, k string, b any) (json.RawMessage, error) { return slow(k, b) })(ctx, p)
},
"the entry": func(ctx context.Context) error { _, err := entryVia(slow)(ctx, "fp"); return err },
} {
ctx, cancel := context.WithCancel(context.Background())
go func() { <-called; cancel() }()
@@ -6,6 +6,7 @@ import (
"errors"
"fmt"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
"slices"
"strings"
"sync/atomic"
"time"
@@ -15,6 +16,7 @@ import (
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-builder/internal/build"
"mesh-delivery-builder/internal/versions"
@@ -44,12 +46,15 @@ type reservations struct {
// built asks a done build's publish (design 47 G A-457a), released ends its item; nil are none.
built func(ctx context.Context, module, version string) error
released func(ctx context.Context, module, version string)
filled atomic.Bool
bounds func() jetstream.KeyValue
found *findings
bound time.Duration
wake func(context.Context)
logf func(string, ...any)
// ended ends the publish item of a build withdrawn, superseded or dropped, its held items terminated; heard acknowledges the
// held items of a version a published newly marked (design 47 H2 §793). Nil are none.
itemEnded, itemHeard func(ctx context.Context, module, version string)
filled atomic.Bool
bounds func() jetstream.KeyValue
found *findings
bound time.Duration
wake func(context.Context)
logf func(string, ...any)
// withdraw ends the running build of a module's fingerprint, superseded, with why (H-A-225); nil ends none.
withdraw func(module, fingerprint, why string)
// call asks a seat's verb: the git seat's tag (design 47 G §370).
@@ -58,6 +63,55 @@ type reservations struct {
rebuild func(ctx context.Context, module, fp, why string) error
// say says package-published on the seat mesh-builds; nil says none.
say func(context.Context, meshbuilds.PackagePublished) error
// names is each package's name as its series key names it (for Go, its module path), and sayReserved says version-reserved on
// the seat mesh-builds (design 47 H2 §805); nil says none.
names versions.PackageNames
sayReserved func(context.Context, meshbuilds.VersionReservedEvent) error
}
// reservedSaid says version-reserved of each package a reservation newly numbers (design 47 H2 §805, H-A-420 (2)): its module,
// artifact, the package's path, the version, the fingerprint and the catalog revision it was reserved at, as the indexer names
// a revision, `<branch>.<n>`. A reservation that names no catalog revision is said in the journal, not on the seat: the indexer
// re-pins at a revision, and there is none to name.
func (rv *reservations) reservedSaid(ctx context.Context, v versions.ReservedVersion) {
if rv.sayReserved == nil || rv.names == nil {
return
}
refs := make([]string, 0, len(v.Artifacts))
for ref := range v.Artifacts {
refs = append(refs, ref)
}
slices.Sort(refs)
for _, ref := range refs {
_, artifact, _ := strings.Cut(ref, ":")
path, known, err := rv.names(ctx, v.Module, artifact)
if err != nil {
rv.logf("version-reserved of %s at %s is not said: its package's name cannot be read: %v", ref, v.Version, err)
continue
}
if !known {
continue // no package: nothing imports it by a version
}
if v.Branch == "" || v.Revision == 0 {
rv.logf("version-reserved of %s at %s is not said: the reservation names no catalog revision to re-pin at", ref, v.Version)
continue
}
ev := meshbuilds.VersionReservedEvent{Module: v.Module, Artifact: ref, Path: path, Version: v.Version, Fingerprint: v.Fingerprint,
Revision: fmt.Sprintf("%s.%d", v.Branch, v.Revision)}
if err := rv.sayReserved(ctx, ev); err != nil {
rv.logf("the version-reserved of %s at %s is not said: %v", ref, v.Version, err)
} else {
rv.logf("version-reserved said: %s (%s) at %s, revision %s", ref, path, v.Version, ev.Revision)
}
}
}
// sayReservedOn says version-reserved on the seat mesh-builds' events as cred, under an event id of the module, the version and
// the artifact, so a reservation said again is stored once (design 47 H2 §805, §821).
func sayReservedOn(js jetstream.JetStream, cred process.Account) func(context.Context, meshbuilds.VersionReservedEvent) error {
return func(ctx context.Context, ev meshbuilds.VersionReservedEvent) error {
return emitSeatEvent(context.WithoutCancel(ctx), js, cred, meshbuilds.SubjectVersionReserved, "version-reserved-"+ev.Module+"-"+ev.Version+"-"+ev.Artifact, ev)
}
}
// newReservations makes the reserving over store; the released versions of a module are asked of the seat mesh-catalog's
@@ -79,7 +133,9 @@ func newReservations(store versions.ReserveStore, call toolCaller, bounds func()
}
return out, nil
}}
rv.r.Took, rv.r.Logf = rv.took, logf
rv.r.Took, rv.r.Logf, rv.r.Reserved = rv.took, logf, rv.reservedSaid
// A module's tags are read before it reserves, and a number tagged at another commit skipped (design 47 H2 §796).
rv.r.Tags, rv.r.TaggedElsewhere = tagsVia(call), rv.taggedElsewhere
return rv
}
@@ -102,8 +158,8 @@ func (rv *reservations) took(ctx context.Context, older, newer versions.Reserved
} else {
rv.logf("%s: the reservation of %s (build of %s, never released) is dropped: the build of %s takes the series over and reserves %s", newer.Module, older.Version, older.Fingerprint, newer.Fingerprint, newer.Version)
}
if rv.released != nil {
rv.released(ctx, older.Module, older.Version) // its publish item goes with its reservation (A-456b)
if rv.itemEnded != nil {
rv.itemEnded(ctx, older.Module, older.Version) // its publish item goes with its reservation (A-456b), its items terminated (§793)
}
rv.dropBound(ctx, versions.ReservedKey(older.Module, older.Series))
rv.found.clear(ctx, versions.VersionUnreleased, meshconditionsclient.ScopeModule, older.Module)
@@ -279,6 +335,7 @@ func (rv *reservations) heard(ctx context.Context, m jetstream.Msg) bool {
}
rv.found.clear(ctx, versions.VersionUnreleased, meshconditionsclient.ScopeModule, ev.Module)
rv.found.clear(ctx, versions.VersionHeldElsewhere, meshconditionsclient.ScopeModule, ev.Module)
rv.found.clear(ctx, versions.VersionTaggedElsewhere, meshconditionsclient.ScopeModule, ev.Module)
default:
return false
}
@@ -305,6 +362,15 @@ func (rv *reservations) published(ctx context.Context, o versions.Outcome, inPro
for _, m := range out.Marked {
rv.sayPublished(ctx, m, o.Digest)
}
if rv.itemHeard != nil { // the publish items a published newly marked are acknowledged (§793)
seen := map[string]bool{}
for _, m := range out.Marked {
if k := m.Reserved.Module + " " + m.Reserved.Version; !seen[k] {
seen[k] = true
rv.itemHeard(ctx, m.Reserved.Module, m.Reserved.Version)
}
}
}
}
// sayPublished says the seat mesh-builds' package-published of an artifact a published marked.
@@ -327,8 +393,8 @@ func (rv *reservations) dropped(ctx context.Context, module, series, by, why str
if err != nil {
return "", err
}
if ok && held && rv.released != nil {
rv.released(ctx, module, cur.Version) // its publish item goes with its reservation (A-456b)
if ok && held && rv.itemEnded != nil {
rv.itemEnded(ctx, module, cur.Version) // its publish item goes with its reservation (A-456b), its items terminated (§793)
}
if !ok {
return "", fmt.Errorf("%s holds no reservation of %s", module, series)
@@ -436,6 +502,19 @@ func (rv *reservations) heldElsewhere(ctx context.Context, module, why string) {
needs: "builder_reserved shows the reservation and its withdrawn numbers; the registry seat's versions shows what it holds."})
}
// taggedElsewhere raises version-tagged-elsewhere about module (design 47 H2 §796): numbers whose git tag stands at another
// commit were skipped at reserving; a tag is never moved. Cleared at the module's next version-released.
func (rv *reservations) taggedElsewhere(ctx context.Context, module string, skipped []string) {
why := "the tags of " + strings.Join(skipped, ", ") + " stand at another commit, so the reserve skipped them"
rv.logf("%s: %s", module, why)
rv.found.raise(ctx, finding{kind: versions.VersionTaggedElsewhere, scope: meshconditionsclient.ScopeModule, id: module,
summary: module + ": " + why,
headline: "A number whose tag stands at another commit is not used",
explanation: "The repository holds a tag of " + module + " at a commit other than the build's: " + why + ". A tag is never moved.",
resolved: "The catalog says version-released for " + module + ".",
needs: "builder_reserved shows the reservation; the seat git's tags shows each tag with its commit."})
}
// movedElsewhere takes a done build whose number a registry holds at another digest off that number (design 47 H H-A-275,
// H-A-279): every artifact carries its version from its build, so no digest survives a move. The number is withdrawn as a gap
// (H-A-226), the finding raised naming both digests, and the build of the same fingerprint asked again (the build-again path of a
@@ -16,6 +16,7 @@ import (
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
stdio "git.novox.be/novox/mesh-sdk/go"
"mesh-delivery-builder/internal/build"
"mesh-delivery-builder/internal/versions"
@@ -163,17 +164,27 @@ func TestAReleasedVersionIsTaggedAtTheCommitItWasBuiltFrom(t *testing.T) {
ctx := context.Background()
var tags []git.TagInput
refuse := ""
var standing []git.Tag // the tags the seat git's tags answers (design 47 H2 §796)
call := func(addr string, body any) (json.RawMessage, error) {
if addr != "seat:git.tag" {
switch addr {
case "seat:git.tags":
return json.Marshal(map[string]any{"form": 2, "tags": standing, "a-field-of-a-later-form": true})
case "seat:git.tag":
default:
return json.Marshal(meshcatalog.VersionsAnswer{})
}
tags = append(tags, body.(git.TagInput))
return json.Marshal(git.TagAnswer{Performed: refuse == "", Error: refuse})
}
found := &findings{module: builds.BuilderModule, w: &memFindings{vals: map[string][]byte{}}, logf: t.Logf, now: func() time.Time { return time.Unix(0, 0) }}
fw := &memFindings{vals: map[string][]byte{}}
found := &findings{module: builds.BuilderModule, w: fw, logf: t.Logf, now: func() time.Time { return time.Unix(0, 0) }}
rv := newReservations(&memReserve{vals: map[string][]byte{}, revs: map[string]uint64{}}, call, func() jetstream.KeyValue { return &boundsKV{keys: map[string]bool{}} }, found, nil, t.Logf)
rv.filled.Store(true)
goPkg := versions.Source{Repository: "o/r", Commit: "c0ffee", GoPackage: true}
// Design 47 H2 §796: the tag names the Go module's directory, read from its module path: mesh-sdk's `go` here.
rv.r.Names = func(_ context.Context, module, artifact string) (string, bool, error) {
return "git.example/o/r/go", artifact == "lib" || artifact == "pkg", nil
}
goPkg := versions.Source{Repository: "o/r", Commit: "c0ffee", GoPackage: true, GoArtifact: "lib"}
if _, err := rv.Reserve(ctx, "pkg", "0.1", []string{"pkg:lib"}, goPkg); err != nil {
t.Fatal(err)
}
@@ -204,7 +215,7 @@ func TestAReleasedVersionIsTaggedAtTheCommitItWasBuiltFrom(t *testing.T) {
if _, err := rv.Reserve(ctx, "pkg", "0.1", []string{"pkg:lib"}, goPkg); err != nil { // build A: 0.1.0 (the stub catalog holds none released)
t.Fatal(err)
}
if v, err := rv.Reserve(ctx, "pkg", "0.1", []string{"pkg:lib"}, versions.Source{Repository: "o/r", Commit: "b0b", GoPackage: true}); err != nil || v.Version != "0.1.1" { // build B supersedes A
if v, err := rv.Reserve(ctx, "pkg", "0.1", []string{"pkg:lib"}, versions.Source{Repository: "o/r", Commit: "b0b", GoPackage: true, GoArtifact: "lib"}); err != nil || v.Version != "0.1.1" { // build B supersedes A
t.Fatalf("second build reserved %q (%v)", v.Version, err)
}
release("pkg", "0.1.0") // A's version is never released in fact; were it said, it is no reservation of the key
@@ -214,6 +225,41 @@ func TestAReleasedVersionIsTaggedAtTheCommitItWasBuiltFrom(t *testing.T) {
if release("pkg", "0.1.1"); len(tags) != 1 || tags[0].Commit != "b0b" {
t.Fatalf("the newer build's version: tags %+v; want go/v0.1.1 at b0b", tags)
}
// Design 47 H2 §796: a tag standing at another commit makes the next number and the finding version-tagged-elsewhere, which
// clears at the module's next version-released.
standing = []git.Tag{{Tag: "go/v0.2.0", Commit: "elsewhere"}, {Tag: "go/v0.2.1", Commit: "elsewhere"}}
if v, err := rv.Reserve(ctx, "pkg", "0.2", []string{"pkg:lib"}, goPkg); err != nil || v.Version != "0.2.2" || v.Tag != "go/v0.2.2" {
t.Fatalf("reserved %+v (%v) with 0.2.0 and 0.2.1 tagged elsewhere, want 0.2.2 tagged go/v0.2.2", v, err)
}
key := meshconditionsclient.Key(meshconditionsclient.ScopeModule, "pkg", versions.VersionTaggedElsewhere)
if got, err := decodeFinding(fw.vals[key]); err != nil || !strings.Contains(got.Summary, "0.2.0") || !strings.Contains(got.Summary, "0.2.1") {
t.Fatalf("the finding %s = %+v (%v), want it naming 0.2.0 and 0.2.1", key, got, err)
}
if release("pkg", "0.2.2"); fw.vals[key] != nil {
t.Errorf("version-released of pkg left %s standing", key)
}
standing = nil
// Design 47 H2 §805, H-A-420 (2): a package's number newly reserved is said as version-reserved, its path the package's name
// and its revision the indexer's `<branch>.<n>`; a reservation kept as it stood, an artifact that is no package and a
// reservation naming no catalog revision say none.
var said []meshbuilds.VersionReservedEvent
rv.names = func(_ context.Context, module, artifact string) (string, bool, error) {
return "git.example/o/r/" + artifact, module == "lib2" && artifact == "pkg", nil
}
rv.sayReserved = func(_ context.Context, ev meshbuilds.VersionReservedEvent) error { said = append(said, ev); return nil }
at := versions.Source{Repository: "o/r", Commit: "c1", Fingerprint: "src2:l2", GoPackage: true, GoArtifact: "pkg", Branch: "main", CatalogRevision: 158}
for range 2 {
if _, err := rv.Reserve(ctx, "lib2", "0.3", []string{"lib2:pkg", "lib2:bin"}, at); err != nil {
t.Fatal(err)
}
}
if _, err := rv.Reserve(ctx, "lib2", "0.4", []string{"lib2:pkg"}, versions.Source{Fingerprint: "src2:none"}); err != nil {
t.Fatal(err)
}
want := meshbuilds.VersionReservedEvent{Module: "lib2", Artifact: "lib2:pkg", Path: "git.example/o/r/pkg", Version: "0.3.0", Fingerprint: "src2:l2", Revision: "main.158"}
if len(said) != 1 || said[0] != want {
t.Errorf("version-reserved said %+v, want once %+v", said, want)
}
}
// Design 47 G A-415: a published heard before the build recorded its digests is kept on its reservation, and when the build
@@ -412,6 +458,7 @@ func TestARegistrysRecordEndsOrMovesAPublish(t *testing.T) {
var asked, ended []string
rv.built = func(_ context.Context, m, v string) error { asked = append(asked, m+" "+v); return nil }
rv.released = func(_ context.Context, m, v string) { ended = append(ended, m+" "+v) }
rv.itemEnded = rv.released // a build withdrawn or superseded ends its publish item too (§793)
registry := []string{"0.1.37"}
rv.r.HeldNumbers = func(context.Context, string, string, []string) ([]string, error) { return registry, nil }
rv.filled.Store(true)
@@ -531,4 +578,71 @@ func TestAPublishedThatMarksNothingIsSaidFromWhatItRead(t *testing.T) {
if store.reads != 2 || !strings.Contains(strings.Join(logs, "\n"), "m05 holds 1 built digests") || !strings.Contains(strings.Join(logs, "\n"), "14 reservations stand at it") {
t.Errorf("%d reads for two publisheds, journal %q: want one read each and the line made from it", store.reads, logs)
}
// Design 47 H2 §794: an event whose handling passed 100 ms is said with how long and the part that took it; one within it
// is not. And H-A-435: every message the builder's handlers take is noted on its consumer's ledger, so the state consumers
// that builder_health reads holds a message left unacknowledged until it is acknowledged.
t.Run("a slow handling is said and a message left unacknowledged is kept", func(t *testing.T) {
var said []string
clock := time.Unix(1000, 0)
now := func() time.Time { return clock }
slow := []eventTaker{{"the reservations", func(jetstream.Msg) bool { clock = clock.Add(150 * time.Millisecond); return true }}}
fast := []eventTaker{{"the floor", func(jetstream.Msg) bool { clock = clock.Add(100 * time.Millisecond); return true }}}
rest := eventTaker{"the artifacts gone", func(jetstream.Msg) bool { return true }}
takeEvent(&agedTestMsg{subject: "a.published", seq: 1}, slow, rest, defaultSlowEvent, now, func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) })
takeEvent(&agedTestMsg{subject: "a.floor", seq: 2}, fast, rest, defaultSlowEvent, now, func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) })
if len(said) != 1 || said[0] != "the event a.published took 150ms to handle, over 100ms: the reservations took it" {
t.Errorf("said %q, want one line for the handling over 100 ms", said)
}
// H-A-461 (2): the bound is the setting slow-event, 100 ms when none is set; at 200ms the 150 ms handling is not said.
for _, c := range []struct {
set string
want time.Duration
}{{"", defaultSlowEvent}, {"200ms", 200 * time.Millisecond}, {"soon", 0}} {
t.Setenv(stdio.SettingWord(slowEventSetting), c.set)
if d, err := slowEvent(); d != c.want || (c.want == 0) != (err != nil) {
t.Errorf("with slow-event %q the bound is %v (%v), want %v", c.set, d, err, c.want)
}
}
said = nil
takeEvent(&agedTestMsg{subject: "a.published", seq: 3}, slow, rest, 200*time.Millisecond, now, func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) })
if len(said) != 0 {
t.Errorf("said %q with slow-event 200ms, want nothing for a 150 ms handling", said)
}
clock = clock.Add(-150 * time.Millisecond) // the ledger's times below read as before this row
facts := map[string]string{}
l := &ledgers{now: now, sink: func(_ context.Context, k string, v []byte) error { facts[k] = string(v); return nil }}
hub := &oneHandler{}
aged := agedHub{hub: hub, ledgers: l}
_ = aged.Handle("x.>", func(m jetstream.Msg) {
if m.Subject() == "x.ack" {
_ = m.Ack()
}
})
hub.handle(&agedTestMsg{subject: "x.left", seq: 7})
if !strings.Contains(facts["builder-events"], `"oldest-unacknowledged":"1970-01-01T00:16:40.25Z"`) {
t.Errorf("facts %q, want the message left unacknowledged kept with its delivery", facts)
}
hub.handle(&agedTestMsg{subject: "x.ack", seq: 8})
if !strings.Contains(facts["builder-events"], `"acknowledged":8`) {
t.Errorf("facts %q, want the acknowledged message applied", facts)
}
})
}
// oneHandler is an event hub with one handler.
type oneHandler struct{ handle func(jetstream.Msg) }
func (h *oneHandler) Handle(_ string, f func(jetstream.Msg)) error { h.handle = f; return nil }
// agedTestMsg is a message of the builder's events consumer, with the metadata a ledger reads.
type agedTestMsg struct {
jetstream.Msg
subject string
seq uint64
}
func (m *agedTestMsg) Subject() string { return m.subject }
func (m *agedTestMsg) Ack() error { return nil }
func (m *agedTestMsg) Metadata() (*jetstream.MsgMetadata, error) {
return &jetstream.MsgMetadata{Sequence: jetstream.SequencePair{Stream: m.seq}, NumDelivered: 1, Consumer: "builder-events", Stream: "EVENTS"}, nil
}
@@ -3,14 +3,94 @@ package main
import (
"context"
"encoding/json"
"strings"
"sync"
"testing"
"time"
"github.com/nats-io/nats.go"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
stdio "git.novox.be/novox/mesh-sdk/go"
"mesh-delivery-builder/internal/build"
"mesh-delivery-builder/internal/versions"
)
// takenFake is the state taken: keys created once, with their time to live.
type takenFake struct {
mu sync.Mutex
vals map[string][]byte
ttls map[string]time.Duration
}
func (f *takenFake) TakeOnce(seat, ask string, bound time.Duration) (bool, error) {
_, err := f.CreateTTL(seat+"."+ask, map[string]string{"seat": seat, "ask": ask}, bound)
return err == nil, nil
}
func (f *takenFake) Get(key string) (*stdio.StateEntry, error) {
f.mu.Lock()
defer f.mu.Unlock()
if v, ok := f.vals[key]; ok {
return &stdio.StateEntry{Key: key, Value: v}, nil
}
return nil, nil
}
func (f *takenFake) CreateTTL(key string, value any, ttl time.Duration) (uint64, error) {
f.mu.Lock()
defer f.mu.Unlock()
if _, ok := f.vals[key]; ok {
return 0, stdio.ErrChanged
}
raw, _ := json.Marshal(value)
f.vals[key], f.ttls[key] = raw, ttl
return 1, nil
}
// blockingBuilder runs a build step until its context ends, and answers the drop that ended it.
type blockingBuilder struct{ started chan string }
func (b blockingBuilder) BuildStep(ctx context.Context, ask string, run meshbuilds.StepRun) meshbuilds.StepEnded {
b.started <- ask
<-ctx.Done()
end := meshbuilds.StepEnded{ID: ask, Pipeline: run.Pipeline, Outcome: meshbuilds.StepEndedOutcomeCouldNotRun}
if d := build.DroppedOf(ctx); d != nil {
end.Said = d.Error()
}
return end
}
// msgFake is a message pulled from the seat's work queue.
type msgFake struct {
h nats.Header
data []byte
acked chan struct{}
}
func (m *msgFake) Data() []byte { return m.data }
func (m *msgFake) Headers() nats.Header { return m.h }
func (m *msgFake) Ack() error { close(m.acked); return nil }
func (m *msgFake) Nak() error { return nil }
func (m *msgFake) Term() error { return nil }
func (m *msgFake) InProgress() error { return nil }
// await fails the test unless c yields within 5 s.
func await[T any](t *testing.T, c <-chan T, what string) {
t.Helper()
select {
case <-c:
case <-time.After(5 * time.Second):
t.Fatalf("%s did not happen within 5 s", what)
}
}
func stepMsg(id string) *msgFake {
data, _ := json.Marshal(meshbuilds.StepRun{Pipeline: "main.1.m", Step: meshbuilds.StepRunStepBuild, Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: "src2:x"}}})
return &msgFake{h: nats.Header{"x-event-id": []string{id}}, data: data, acked: make(chan struct{})}
}
// Design 47 G A-115, A-127: the builder's process answers the verb reserved, given a name, from the catalog half's carriers and the
// seat mesh-catalog's versions, and given a module from its reservations.
func TestTheProcessAnswersReserved(t *testing.T) {
@@ -47,4 +127,55 @@ func TestTheProcessAnswersReserved(t *testing.T) {
if _, err := handle("", json.RawMessage(`{"module":`)); err == nil {
t.Error("arguments that are no JSON were answered")
}
// Design 47 H2 §778, H-A-401, H-A-404: the verb drop cancels an ask that runs here, and keeps the drop of one not
// pulled yet in the state taken for ask-taken-bound; both end could-not-run naming the drop and who asked it.
taken := &takenFake{vals: map[string][]byte{}, ttls: map[string]time.Duration{}}
started := make(chan string, 2)
var mu sync.Mutex
said := map[string]meshbuilds.StepEnded{}
var subjects []string
q := &stepQueue{node: "n1", builder: blockingBuilder{started: started}, taken: taken, takenBound: 10 * time.Minute,
inProgress: func() time.Duration { return 0 }, logf: t.Logf, now: time.Now, running: map[string]*runningAsk{},
say: func(_ context.Context, subject, id string, body any) error {
mu.Lock()
defer mu.Unlock()
subjects = append(subjects, subject)
if end, ok := body.(meshbuilds.StepEnded); ok {
said[id] = end
}
return nil
}}
drop := dropVerb(q)
running := stepMsg("ask-1")
q.take(t.Context(), running)
await(t, started, "the build of ask-1")
if got, err := drop("mesh-delivery-pipeline", json.RawMessage(`{"id":"ask-1","why":"its step ended by its bound"}`)); err != nil || got.(meshbuilds.DropAnswer)["held"] != true {
t.Fatalf("drop of a running ask = %v, %v; want held", got, err)
}
await(t, running.acked, "the end of ask-1")
if got, err := drop("mesh-delivery-pipeline", json.RawMessage(`{"id":"ask-2","by":"op@shanks","why":"by hand"}`)); err != nil || got.(meshbuilds.DropAnswer)["held"] != false {
t.Fatalf("drop of an ask not pulled = %v, %v; want not held", got, err)
}
if taken.ttls[build.DroppedKey("ask-2")] != 10*time.Minute {
t.Errorf("the drop of ask-2 is kept for %v, want ask-taken-bound", taken.ttls[build.DroppedKey("ask-2")])
}
later := stepMsg("ask-2")
q.take(t.Context(), later)
await(t, started, "the build step of ask-2")
await(t, later.acked, "the end of ask-2")
mu.Lock()
if e := said["ask-1"]; e.Outcome != meshbuilds.StepEndedOutcomeCouldNotRun || e.Said != "the ask was dropped by mesh-delivery-pipeline through the seat mesh-builds' drop: its step ended by its bound" {
t.Errorf("the running ask ended %+v", e)
}
if e := said["ask-2"]; e.Outcome != meshbuilds.StepEndedOutcomeCouldNotRun || !strings.Contains(e.Said, "dropped by op@shanks") {
t.Errorf("the ask dropped before its pull ended %+v", e)
}
if _, ok := taken.vals["mesh-builds.ask-2"]; ok || strings.Count(strings.Join(subjects, " "), "ask-taken") != 1 {
t.Errorf("the ask dropped before its pull was taken: subjects said %v", subjects)
}
mu.Unlock()
if _, err := drop("", json.RawMessage(`{"id":"ask-3"}`)); err == nil {
t.Error("a drop with no why was taken")
}
}
@@ -2,19 +2,11 @@ package main
import (
"context"
"encoding/json"
"fmt"
"sync"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-builder/internal/build"
)
// **The runner's build step ask is the builder's one entry point for a candidate** (design 47 G A-97, A-84, A-99): the seat
@@ -72,30 +64,3 @@ func (w *entryWaits) entry(ctx context.Context, fingerprint string) (meshcatalog
}
return e, w.manifest.keep(e) // the manifest the build is planned from
}
// serveSteps hears the step asks on the seat's accept subject and answers each one's end, in its own goroutine.
func serveSteps(ctx context.Context, conn *nats.Conn, js jetstream.JetStream, cred process.Account, b *build.Builder, logf func(string, ...any)) error {
sub, err := conn.Subscribe(meshbuilds.AcceptSubjectStep, func(m *nats.Msg) {
id := m.Header.Get("x-event-id")
if id == "" {
id = m.Header.Get(nats.MsgIdHdr)
}
var run meshbuilds.StepRun
if err := json.Unmarshal(m.Data, &run); err != nil || id == "" || run.Step != meshbuilds.StepRunStepBuild {
logf("a step ask on %s that is no build step with an id is left: %v", m.Subject, err)
return
}
go func() {
end := b.BuildStep(ctx, run)
end.ID = id
if err := emitSeatEvent(context.WithoutCancel(ctx), js, cred, meshbuilds.SubjectStepEnded(cred.Node), id, end); err != nil {
logf("saying the end of the build step %s: %v", id, err)
}
}()
})
if err != nil {
return err
}
go func() { <-ctx.Done(); _ = sub.Drain() }()
return nil
}
@@ -4,14 +4,15 @@ package main
import (
"encoding/json"
"runtime"
"strings"
"testing"
"time"
"github.com/nats-io/nats.go"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
"git.novox.be/novox/mesh-sdk/go/buscompose"
)
// The runner's build step ask on the seat mesh-builds' accept subject, naming an entry by its fingerprint, is built through the
@@ -30,16 +31,33 @@ func TestTheRunnersBuildStepAskIsAnsweredAsStepEnded(t *testing.T) {
t.Fatal(err)
}
defer s.Unsubscribe()
b.start(t, "n1", nil)
b.await(t, "netcheck", isState(builds.BuildDone))
taken := make(chan *nats.Msg, 4)
st, err := sub.ChanSubscribe(meshbuilds.SubjectAskTakenAll, taken)
if err != nil {
t.Fatal(err)
}
defer st.Unsubscribe()
// Design 47 H2 §777: an ask put on the seat's work queue while the builder is stopped waits there, and is taken (ask-taken)
// and ended once the builder starts.
run := meshbuilds.StepRun{Pipeline: "main.1.netcheck", Step: meshbuilds.StepRunStepBuild, Branch: "main", Revision: 1,
Entries: []meshbuilds.BuildEntry{{Module: "netcheck", Fingerprint: fp}}}
data, _ := json.Marshal(run)
msg := nats.NewMsg(meshbuilds.AcceptSubjectStep)
msg.Data = data
msg.Header.Set("x-event-id", "ask-1")
if err := sub.PublishMsg(msg); err != nil {
t.Fatal(err)
msg.Header.Set(nats.MsgIdHdr, "ask-1")
if _, err := b.js.PublishMsg(b.ctx, msg); err != nil {
t.Fatalf("the ask is not put on the seat's work queue: %v", err)
}
b.start(t, "n1", nil)
select {
case m := <-taken:
var at meshbuilds.AskTakenEvent
if err := json.Unmarshal(m.Data, &at); err != nil || at.Ask != "ask-1" || at.Node != "n1" || at.At.IsZero() || m.Subject != meshbuilds.SubjectAskTaken("ask-1") {
t.Fatalf("ask-taken on %s = %+v, %v; want ask-1 taken on n1", m.Subject, at, err)
}
case <-time.After(20 * time.Second):
t.Fatal("no ask-taken within 20 s of the start")
}
select {
case m := <-ended:
@@ -75,4 +93,24 @@ func TestTheRunnersBuildStepAskIsAnsweredAsStepEnded(t *testing.T) {
case <-time.After(20 * time.Second):
t.Fatal("no step-ended for ask-2 within 20 s")
}
// Each ask is acknowledged once its end was said: the seat's worker holds none pending.
stream, worker := buscompose.SeatQueueStreamName(meshbuilds.Seat), buscompose.SeatQueueWorkerName(meshbuilds.Seat)
deadline := time.Now().Add(10 * time.Second)
for {
c, err := b.js.Consumer(b.ctx, stream, worker)
if err != nil {
t.Fatal(err)
}
info, err := c.Info(b.ctx)
if err != nil {
t.Fatal(err)
}
if info.NumPending == 0 && info.NumAckPending == 0 {
break
}
if time.Now().After(deadline) {
t.Fatalf("the seat's worker holds %d waiting and %d taken after both asks ended, want none", info.NumPending, info.NumAckPending)
}
runtime.Gosched()
}
}
@@ -0,0 +1,309 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/seatdef"
"mesh-delivery-builder/internal/build"
)
// **The seat mesh-builds takes its asks from a work queue and says ask-taken** (design 47 H2 §777, §775; ADR 0525 §1, §2,
// §6; ADR 0668 §2): the broker derives the seat's stream and its one shared worker from the seat's definition; the builder
// pulls every ask waiting at once, takes each once by its id in its state taken (TakenKey <seat>.<ask>, for
// ask-taken-bound; H-A-381 (2)), says ask-taken at the pull, sends in-progress while it builds, and acknowledges an ask
// when it said the ask's end. An ask put while the builder is down waits on the stream.
// askTakenBoundSetting is the setting the taken key's time to live is (H-A-381 (2)): the holder never receives the
// caller's bound, so the largest caller's, the runner's ask-unanswered-bound, is its default.
const (
askTakenBoundSetting = "ask-taken-bound"
defaultAskTakenBound = 10 * time.Minute
)
// askTaker is the state taken (process.State): an ask taken once by its id, and the drop of an ask not pulled yet kept
// under build.DroppedKey (H-A-404).
type askTaker interface {
TakeOnce(seat, ask string, bound time.Duration) (bool, error)
Get(key string) (*stdio.StateEntry, error)
CreateTTL(key string, value any, ttl time.Duration) (uint64, error)
}
// stepBuilder is build.Builder's BuildStep.
type stepBuilder interface {
BuildStep(ctx context.Context, ask string, run meshbuilds.StepRun) meshbuilds.StepEnded
}
// queueMsg is what the queue reads of a message pulled from the seat's work queue (jetstream.Msg).
type queueMsg interface {
Data() []byte
Headers() nats.Header
Ack() error
Nak() error
Term() error
InProgress() error
}
// stepQueue is the builder's pull of the seat mesh-builds' work queue.
type stepQueue struct {
node string
builder stepBuilder
taken askTaker
takenBound time.Duration
// inProgress is how often a running ask is said in progress: the seat definition's in-progress.
inProgress func() time.Duration
// say publishes a seat event of the builder (emitSeatEvent).
say func(ctx context.Context, subject, id string, body any) error
logf func(string, ...any)
now func() time.Time
mu sync.Mutex
running map[string]*runningAsk
}
// runningAsk is an ask the builder took and builds now.
type runningAsk struct {
run meshbuilds.StepRun
taken time.Time
cancel context.CancelCauseFunc
}
// askIDOf is the ask's id a put carries: its event id, else its message id.
func askIDOf(h nats.Header) string {
if id := h.Get("x-event-id"); id != "" {
return id
}
return h.Get(nats.MsgIdHdr)
}
// take handles one ask pulled from the queue: an ask that is no build step with an id is terminated, said; one this
// builder runs already (handed again while it runs) is left to the run that holds it; any other is taken (ask-taken said
// the first time its id is taken) and built on its own goroutine, in progress while it runs, acknowledged once its end is
// said. A repeat of an id taken before that runs here no longer (the builder started again) is answered as its builds
// stand now, read afresh by BuildStep (H-A-381 (1)): an entry built is answered built, never built twice.
func (q *stepQueue) take(ctx context.Context, m queueMsg) {
id := askIDOf(m.Headers())
var run meshbuilds.StepRun
err := json.Unmarshal(m.Data(), &run)
if err != nil || id == "" || run.Step != meshbuilds.StepRunStepBuild {
q.logf("a step ask on the seat %s's work queue that is no build step with an id is ended: %v", meshbuilds.Seat, err)
_ = m.Term()
return
}
q.mu.Lock()
if _, held := q.running[id]; held {
q.mu.Unlock()
q.logf("the ask %s (%s) was handed again while it runs here: its run answers it", id, run.Pipeline)
return
}
// An ask dropped before it was pulled ends could-not-run naming the drop, with no build (H-A-404).
if d := q.droppedAsk(id); d != nil {
work, cancel := context.WithCancelCause(ctx)
cancel(d)
ra := &runningAsk{run: run, taken: q.now(), cancel: cancel}
q.running[id] = ra
q.mu.Unlock()
q.logf("the ask %s (%s) was dropped before it was pulled: %s", id, run.Pipeline, d.Error())
go q.build(work, m, id, ra)
return
}
first, err := q.taken.TakeOnce(meshbuilds.Seat, id, q.takenBound)
if err != nil {
q.mu.Unlock()
q.logf("the ask %s (%s) is not taken, the state %s refused it: %v; it is handed again", id, run.Pipeline, process.TakenState, err)
_ = m.Nak()
return
}
work, cancel := context.WithCancelCause(ctx)
ra := &runningAsk{run: run, taken: q.now(), cancel: cancel}
q.running[id] = ra
q.mu.Unlock()
if first {
if err := q.say(ctx, meshbuilds.SubjectAskTaken(id), id+"-taken", meshbuilds.AskTakenEvent{Ask: id, Node: q.node, At: ra.taken.UTC()}); err != nil {
q.logf("saying ask-taken of %s: %v", id, err)
}
} else {
q.logf("the ask %s (%s) was taken before: it is answered as its builds stand now", id, run.Pipeline)
}
go q.build(work, m, id, ra)
}
// build runs one taken ask to its end and says it.
func (q *stepQueue) build(ctx context.Context, m queueMsg, id string, ra *runningAsk) {
defer func() {
ra.cancel(nil)
q.mu.Lock()
delete(q.running, id)
q.mu.Unlock()
}()
every := q.inProgress()
var tick <-chan time.Time
if every > 0 {
t := time.NewTicker(every)
defer t.Stop()
tick = t.C
}
done := make(chan meshbuilds.StepEnded, 1)
go func() { done <- q.builder.BuildStep(ctx, id, ra.run) }()
var end meshbuilds.StepEnded
for waiting := true; waiting; {
select {
case end = <-done:
waiting = false
case <-tick:
_ = m.InProgress()
}
}
if err := q.say(context.WithoutCancel(ctx), meshbuilds.SubjectStepEnded(q.node), id, end); err != nil {
q.logf("saying the end of the build step %s: %v; the ask is handed again", id, err)
_ = m.Nak()
return
}
_ = m.Ack()
}
// droppedAsk is the drop kept for the ask id in the state taken, nil for none.
func (q *stepQueue) droppedAsk(id string) *build.Dropped {
e, err := q.taken.Get(build.DroppedKey(id))
if err != nil || e == nil {
return nil
}
var d build.Dropped
if json.Unmarshal(e.Value, &d) != nil {
return nil
}
return &d
}
// drop is the seat mesh-builds' verb drop (§778; H-A-401, H-A-404): the drop is kept in the state taken under
// build.DroppedKey for ask-taken-bound, so a pull of the ask within the bound ends it, and an ask that runs here is
// cancelled; both end the step could-not-run naming the drop and who asked it. held says whether it ran here.
func (q *stepQueue) drop(id, by, why string) (held bool, err error) {
if strings.TrimSpace(id) == "" || strings.TrimSpace(why) == "" {
return false, errors.New("give the ask's id and why it is dropped")
}
d := &build.Dropped{By: by, Why: why}
if _, err := q.taken.CreateTTL(build.DroppedKey(id), d, q.takenBound); err != nil && !errors.Is(err, stdio.ErrChanged) {
return false, fmt.Errorf("the drop of %s is not kept in the state %s: %w", id, process.TakenState, err)
}
q.mu.Lock()
ra := q.running[id]
q.mu.Unlock()
if ra != nil {
ra.cancel(d)
}
q.logf("the ask %s is dropped by %s: %s", id, by, why)
return ra != nil, nil
}
// dropVerb is the handler of the seat mesh-builds' verb drop: who drops it is the input's by, else the caller.
func dropVerb(q *stepQueue) process.SeatHandler {
return func(caller string, body json.RawMessage) (any, error) {
var in meshbuilds.DropInput
if err := json.Unmarshal(body, &in); err != nil {
return nil, fmt.Errorf("the arguments do not fit the verb drop: %w", err)
}
by := in.By
if by == "" {
by = caller
}
held, err := q.drop(in.ID, by, in.Why)
if err != nil {
return nil, err
}
return meshbuilds.DropAnswer{"held": held}, nil
}
}
// running answers the asks this builder runs now, by id.
func (q *stepQueue) runningAsks() map[string]runningAsk {
q.mu.Lock()
defer q.mu.Unlock()
out := make(map[string]runningAsk, len(q.running))
for id, ra := range q.running {
out[id] = *ra
}
return out
}
// inProgressOf is the seat's in-progress as its definition names it, once the seat list is read; before, a third of the
// worker's ack wait, which the broker derived from the same definition (60 s and 20 s, §777).
func inProgressOf(def func() (seatdef.Definition, bool), ackWait func() time.Duration) func() time.Duration {
return func() time.Duration {
if d, ok := def(); ok {
if _, worker, ok, err := buscompose.SeatQueue(meshbuilds.Seat, d.Accepts); err == nil && ok && worker.InProgressSeconds > 0 {
return time.Duration(worker.InProgressSeconds) * time.Second
}
}
return ackWait() / 3
}
}
// serveSteps pulls the seat's work queue through its one shared worker, every ask waiting at once (at most the worker's
// max ack pending), once the broker made it: a worker not there yet is said, and bound when the bus says it was created.
func serveSteps(ctx context.Context, conn *nats.Conn, js jetstream.JetStream, q *stepQueue, ackWait *atomic.Int64) error {
stream, worker := buscompose.SeatQueueStreamName(meshbuilds.Seat), buscompose.SeatQueueWorkerName(meshbuilds.Seat)
var once sync.Once
bind := func() bool {
c, err := js.Consumer(ctx, stream, worker)
if err != nil {
return false
}
bound := false
once.Do(func() {
bound = true
info := c.CachedInfo()
if info != nil && info.Config.AckWait > 0 {
ackWait.Store(int64(info.Config.AckWait))
}
pending := 1000
if info != nil && info.Config.MaxAckPending > 0 {
pending = info.Config.MaxAckPending
}
cc, err := c.Consume(func(m jetstream.Msg) { q.take(ctx, m) }, jetstream.PullMaxMessages(pending))
if err != nil {
q.logf("the seat %s's work queue %s is not pulled: %v", meshbuilds.Seat, worker, err)
return
}
q.logf("pulling the seat %s's asks from %s", meshbuilds.Seat, worker)
go func() { <-ctx.Done(); cc.Stop() }()
})
return bound
}
if bind() {
return nil
}
q.logf("the seat %s's work queue %s on %s is not there yet: its asks wait on it, and it is pulled when the bus says it was created", meshbuilds.Seat, worker, stream)
sub, err := conn.Subscribe("$JS.EVENT.ADVISORY.CONSUMER.CREATED."+stream+"."+worker, func(*nats.Msg) { go bind() })
if err != nil {
return fmt.Errorf("hearing the creation of the seat %s's work queue: %w", meshbuilds.Seat, err)
}
go func() { <-ctx.Done(); _ = sub.Unsubscribe() }()
if bind() { // made between the first look and the subscription
_ = sub.Unsubscribe()
}
return nil
}
// askTakenBound reads the setting ask-taken-bound (§781).
func askTakenBound() (time.Duration, error) {
d, err := stdio.SettingDuration(askTakenBoundSetting, defaultAskTakenBound)
if err != nil {
return 0, errors.New("the builder's " + err.Error())
}
return d, nil
}
@@ -0,0 +1,56 @@
package main
import (
"context"
"errors"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-builder/internal/build"
)
// unbuildableKept is the builder's state unbuildable on the bus as build.Unbuildables (design 47 H2 §795, H-A-425).
type unbuildableKept struct{ b *process.AwaitedBucket }
func (u unbuildableKept) Put(ctx context.Context, module string, value []byte) error {
_, err := u.b.Put(ctx, module, value)
if errors.Is(err, process.ErrNotYet) {
return nil // kept, and written when the state appears
}
return err
}
// Delete removes module's key when the state holds it: a key not there is no error, and writes no marker.
func (u unbuildableKept) Delete(ctx context.Context, module string) error {
kv := u.b.KV()
if kv == nil {
return nil
}
if _, err := kv.Get(ctx, module); errors.Is(err, jetstream.ErrKeyNotFound) {
return nil
} else if err != nil {
return err
}
return kv.Delete(ctx, module)
}
func (u unbuildableKept) Keys(ctx context.Context) ([]string, error) {
kv := u.b.KV()
if kv == nil {
return nil, nil
}
l, err := kv.ListKeys(ctx)
if err != nil {
return nil, err
}
defer func() { _ = l.Stop() }()
var out []string
for k := range l.Keys() {
out = append(out, k)
}
return out, nil
}
var _ build.Unbuildables = unbuildableKept{}
@@ -145,31 +145,44 @@ func floorVia(call toolCaller, requirements func(context.Context, string) ([]str
}
return out, nil
},
Tags: func(_ context.Context, repository, prefix string) ([]string, error) {
// The seat git's verb tags, whichever module holds the seat (design 47 G §7, A-82), page by page.
var out []string
for page := int64(1); ; page++ {
raw, err := call("seat:"+gitseat.Seat+"."+gitseat.VerbTags, gitseat.TagsInput{Repository: repository, Prefix: prefix, Page: page})
if err != nil {
return nil, err
}
var a gitseat.TagsAnswer
if err := json.Unmarshal(raw, &a); err != nil {
return nil, fmt.Errorf("the seat git's tags answered what is no list of tags: %w", err)
}
if a.Error != "" {
return nil, fmt.Errorf("the seat git's tags: %s", a.Error)
}
out = append(out, a.Tags...)
if !a.More {
return out, nil
}
Tags: func(ctx context.Context, repository, prefix string) ([]string, error) {
tags, _, err := tagsVia(call)(ctx, repository, prefix)
out := make([]string, 0, len(tags))
for _, t := range tags {
out = append(out, t.Tag)
}
return out, err
},
Requirements: requirements,
}
}
// tagsVia reads a repository's tags starting with prefix, each with the commit it names, through the seat git's verb tags,
// whichever module holds the seat (design 47 G §7, A-82), page by page; its answer is read tolerantly, form 2 naming each tag's
// commit (design 47 H2 §796, §780). away says the read met a transport fault or the error kind away (§776).
func tagsVia(call toolCaller) func(ctx context.Context, repository, prefix string) ([]gitseat.Tag, bool, error) {
return func(_ context.Context, repository, prefix string) ([]gitseat.Tag, bool, error) {
var out []gitseat.Tag
for page := int64(1); ; page++ {
raw, err := call("seat:"+gitseat.Seat+"."+gitseat.VerbTags, gitseat.TagsInput{Repository: repository, Prefix: prefix, Page: page})
if err != nil {
return nil, true, err
}
a, _, err := gitseat.DecodeTagsAnswer(raw)
if err != nil {
return nil, false, fmt.Errorf("the seat git's tags answered what is no list of tags: %w", err)
}
if a.Error != "" {
return nil, a.ErrorKind == gitseat.ErrorKindAway, fmt.Errorf("the seat git's tags: %s (%s)", a.Error, a.ErrorKind)
}
out = append(out, a.Tags...)
if !a.More {
return out, false, nil
}
}
}
}
// emitEvent says one of the builder's events, mesh.mod.<module>.event.<event>, with the SDK's envelope headers and id as
// the bus's dedup id.
func emitEvent(ctx context.Context, js jetstream.JetStream, cred process.Account, event, id string, body any) error {
@@ -7,6 +7,7 @@ import (
"slices"
"sort"
"strings"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
@@ -252,6 +253,23 @@ func (c Claimer) Claim(ctx context.Context, module, fingerprint, manifest string
return nil, nil
}
// endOf gives a done or failed build its end and outcome (design 47 H2 §797): the end the key holds already for that outcome, else
// now. A build that has not ended names neither.
func endOf(was, b builds.BuilderBuild, now func() time.Time) builds.BuilderBuild {
if b.State != builds.BuildDone && b.State != builds.BuildFailed {
b.Ended, b.Outcome = time.Time{}, ""
return b
}
b.Outcome = b.State
if b.Ended.IsZero() {
b.Ended = now().UTC()
if was.Outcome == b.State && !was.Ended.IsZero() {
b.Ended = was.Ended
}
}
return b
}
func (c Claimer) write(_ context.Context, key string, b builds.BuilderBuild, put func([]byte) (uint64, error)) (*Claim, error) {
data, err := builds.EncodeBuilderBuild(b)
if err != nil {
@@ -272,6 +290,7 @@ func (c Claimer) write(_ context.Context, key string, b builds.BuilderBuild, put
// the key holds. A key that changed meanwhile is ErrConflict: the claim is no longer this builder's.
func (c Claimer) Finish(ctx context.Context, cl *Claim, b builds.BuilderBuild) error {
b.V = keys.Version
b = endOf(cl.Build, b, time.Now)
if builds.SameBuilderBuild(cl.Build, b) {
return nil
}
@@ -6,6 +6,7 @@ import (
"slices"
"sync"
"testing"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
)
@@ -209,9 +210,14 @@ func TestFinishMovesTheKeyByCompareAndSetAndWritesOnlyADifferentValue(t *testing
if err := c.Finish(context.Background(), got, done); err != nil {
t.Fatal(err)
}
if v := valueOf(t, st, got.Key); v.State != builds.BuildDone || v.Artifacts[0].Digest != digest {
v := valueOf(t, st, got.Key)
if v.State != builds.BuildDone || v.Artifacts[0].Digest != digest {
t.Errorf("the key holds %+v", v)
}
// Design 47 H2 §797: the done key carries its end and outcome, so a reader of the builds reads no key history.
if v.Outcome != builds.BuildDone || v.Ended.IsZero() || time.Since(v.Ended) > time.Minute {
t.Errorf("the done key ended %v with outcome %q, want now and done", v.Ended, v.Outcome)
}
writes := st.writes
if err := c.Finish(context.Background(), got, done); err != nil && !errors.Is(err, ErrConflict) {
t.Fatal(err)
@@ -219,6 +225,12 @@ func TestFinishMovesTheKeyByCompareAndSetAndWritesOnlyADifferentValue(t *testing
if st.writes != writes {
t.Errorf("the same value was written again (%d writes, was %d)", st.writes, writes)
}
if again := endOf(v, done, time.Now); again.Ended != v.Ended || again.Outcome != builds.BuildDone {
t.Errorf("the same end given again ended %v, want the end the key holds, %v", again.Ended, v.Ended)
}
if building := endOf(v, builds.BuilderBuild{State: builds.BuildBuilding}, time.Now); !building.Ended.IsZero() || building.Outcome != "" {
t.Errorf("a building key carries an end: %+v", building)
}
// Finish after the key changed is a conflict, not an overwrite (merged into this test for the test budget).
st = newMem()
@@ -0,0 +1,48 @@
package build
import (
"context"
"errors"
"fmt"
)
// **A dropped ask ends its step could-not-run, naming the drop** (design 47 H2 §777, §778; H-A-401, H-A-404; ADR 0664 §4):
// the seat mesh-builds' drop ends an ask its caller no longer waits for. A taken ask that runs is cancelled with a Dropped
// cause; one not pulled yet is kept in the state taken under DroppedKey, and its pull ends it the same way. Either way the
// build step's finish answers it could-not-run with Dropped's words, so no build runs for it after.
// The tools of the seat's queue (§777's tools table): AsksTool reads what waits and what is taken, AskDropTool ends one
// ask through the seat's verb drop, the hand act recorded first.
const (
AsksTool = "builder_asks"
AskDropTool = "builder_ask_drop"
)
// DroppedPrefix is the key prefix in the state taken of a drop of an ask not pulled yet (H-A-404).
const DroppedPrefix = "dropped."
// DroppedKey is the key in the state taken of the drop of the ask id.
func DroppedKey(id string) string { return DroppedPrefix + id }
// Dropped is why a step's context was cancelled by the seat's verb drop, and the value its key keeps: who asked it and why.
type Dropped struct {
By string `json:"by,omitempty"`
Why string `json:"why"`
}
func (d *Dropped) Error() string {
by := d.By
if by == "" {
by = "its asker"
}
return fmt.Sprintf("the ask was dropped by %s through the seat mesh-builds' drop: %s", by, d.Why)
}
// DroppedOf is the drop that cancelled ctx, nil when none did.
func DroppedOf(ctx context.Context) *Dropped {
var d *Dropped
if errors.As(context.Cause(ctx), &d) {
return d
}
return nil
}
@@ -199,7 +199,9 @@ func (b *Builder) reserve(ctx context.Context, module string, m meshcatalog.Modu
}
src := versions.Source{Repository: m.Repository, Commit: m.Commit, Fingerprint: m.Fp2, Branch: o.Branch, Revision: o.Source, CatalogRevision: o.Revision, Rebuild: rebuild}
for _, t := range plan.Tasks {
src.GoPackage = src.GoPackage || (t.Kind == catalogue.ArtifactPackage && t.Language == "go")
if t.Kind == catalogue.ArtifactPackage && t.Language == "go" && !src.GoPackage {
src.GoPackage, src.GoArtifact = true, t.Artifact // its module path names its tag's directory (design 47 H2 §796)
}
}
res, err := b.Reserved.Reserve(ctx, module, plan.Series, refs, src)
if err != nil {
@@ -82,7 +82,10 @@ func (n *NotPinned) Error() string {
// the plan is not kept, so it is made again when they are read (design 47 G A-82).
type NoBuildModule struct {
Artifact string
Err error
// Language and Kind are the artifact's, as the seat list is asked for them: what the summary of the plans no seat builds counts
// (design 47 H2 §795).
Language, Kind string
Err error
}
func (n *NoBuildModule) Error() string { return fmt.Sprintf("artifact %s: %v", n.Artifact, n.Err) }
@@ -103,7 +106,7 @@ func PlanOf(m catalogue.Manifest, seats *buildrun.BuildSeats) (Plan, error) {
task := func(a catalogue.Artifact, language, kind string) error {
bm, err := seats.For(language, kind)
if err != nil {
return &NoBuildModule{Artifact: a.Name, Err: err}
return &NoBuildModule{Artifact: a.Name, Language: language, Kind: kind, Err: err}
}
t := Task{Artifact: a.Name, Kind: a.Kind, Module: bm.Module, Language: bm.Toolchain, From: a.From}
if a.Kind != catalogue.ArtifactUpstream {
@@ -1,13 +1,16 @@
package build
import (
"context"
"errors"
"fmt"
"strings"
"testing"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/seatdef"
@@ -114,9 +117,58 @@ func TestAPlanNamesTheHoldersOfTheBuildSeats(t *testing.T) {
if !errors.As(err, &none) {
t.Fatalf("with no holder: %v, want NoBuildModule", err)
}
if none.Language != "" || none.Kind != "image" {
t.Errorf("the plan no seat builds names %q %q, want the image's kind (design 47 H2 §795)", none.Language, none.Kind)
}
// Design 47 H2 §795, H-A-425: 6,500 plans no seat builds are kept in the state unbuildable and said in one summary line, with
// each language and kind and its count, never a line per module; a module whose look judges its plan so writes its key and
// no line of its own, and the module is removed once it builds.
t.Run("6,500 plans no seat builds are one line", func(t *testing.T) {
var lines []string
kept := &memUnbuildables{vals: map[string][]byte{}}
mf := &fakeManifests{byDigest: map[string]catalogue.Manifest{mdigest('a'): m, mdigest('b'): manifestOf(catalogue.Artifact{Name: "web", Kind: "bundle", Language: "typescript"})}}
b := &Builder{OwnModule: builds.BuilderModule, Seats: seats, Claimer: Claimer{Store: newMem(), By: "novox.builder"}, Manifests: mf, Unbuildables: kept,
Logf: func(f string, a ...any) { lines = append(lines, fmt.Sprintf(f, a...)) }}
holders["node-build-go"], holders["node-build-image"] = "", ""
b.init()
for i := range 6500 {
digest := mdigest('a')
if i%5 == 0 {
digest = mdigest('b')
}
b.known[fmt.Sprintf("m%04d", i)] = meshcatalog.ModulesByModuleValue{Fp2: fp1, Manifest: digest}
}
kept.vals["gone"] = []byte(`{"module":"gone"}`) // a module that builds now
judged := b.SayUnbuildable(context.Background())
if len(judged) != 6500 || len(kept.vals) != 6500 || kept.vals["gone"] != nil {
t.Fatalf("judged %d, kept %d (gone kept: %v), want 6,500 each and gone removed", len(judged), len(kept.vals), kept.vals["gone"] != nil)
}
if len(lines) != 1 || lines[0] != "6500 "+UnbuildableSaid+": image 5200, typescript bundle 1300" {
t.Errorf("said %d lines %q, want one naming each language and kind with its count", len(lines), lines)
}
lines = nil
delete(kept.vals, "m0001")
b.try(context.Background(), "m0001", b.known["m0001"])
if kept.vals["m0001"] == nil || len(lines) != 0 {
t.Errorf("a look at a plan no seat builds kept %s and said %q, want its key kept and no line", kept.vals["m0001"], lines)
}
})
holders["node-build-go"], holders["node-build-image"] = "acme-go", "acme-img"
p, err := PlanOf(m, seats)
if err != nil || len(p.Tasks) != 2 || p.Tasks[0].Module != "acme-img" || p.Tasks[1].Module != "acme-go" || p.Tasks[1].Language != "go" || p.Tasks[0].Language != "" {
t.Fatalf("plan = %+v, %v", p, err)
}
}
// memUnbuildables is the state unbuildable in memory.
type memUnbuildables struct{ vals map[string][]byte }
func (m *memUnbuildables) Put(_ context.Context, k string, v []byte) error { m.vals[k] = v; return nil }
func (m *memUnbuildables) Delete(_ context.Context, k string) error { delete(m.vals, k); return nil }
func (m *memUnbuildables) Keys(context.Context) ([]string, error) {
out := make([]string, 0, len(m.vals))
for k := range m.vals {
out = append(out, k)
}
return out, nil
}
@@ -249,7 +249,8 @@ func TestAModuleWhoseOnlyArtifactIsAGoPackageIsBuiltUnderItsReservedVersion(t *t
t.Run(c.name, func(t *testing.T) {
r := newRig(t, goPackage)
r.mf.byDigest[mdigest('a')] = catalogue.Manifest{Module: "m", Version: c.series, Build: &catalogue.Build{Artifacts: []catalogue.Artifact{goPackage}}}
r.b.Reserved = &fakeReservations{filled: true}
res := &fakeReservations{filled: true}
r.b.Reserved = res
r.bm.packages["go-overlay"] = "git.novox.be/novox/mesh-networking/go@v0.1.5" // the Go build module's published reference
var asked []buildstep.StepAsk
r.asks.Answer = func(a buildstep.StepAsk) buildstep.Step {
@@ -264,6 +265,10 @@ func TestAModuleWhoseOnlyArtifactIsAGoPackageIsBuiltUnderItsReservedVersion(t *t
len(asked[0].Build.Artifacts) != 1 || asked[0].Build.Artifacts[0] != (buildstep.AskArtifact{Artifact: "go-overlay", Seat: c.seat}) {
t.Fatalf("asks %+v, want one Go ask with version %q naming go-overlay on seat %q", asked, c.version, c.seat)
}
// Design 47 H2 §796: the reservation names the Go package's artifact, whose module path names its tag's directory.
if c.version != "" && (len(res.sources) != 1 || !res.sources[0].GoPackage || res.sources[0].GoArtifact != "go-overlay") {
t.Errorf("reserved from %+v, want one source naming the Go package go-overlay", res.sources)
}
})
}
}
@@ -61,8 +61,10 @@ type Builder struct {
Toolchains ToolchainEntries
// Waits keeps the toolchain waits of design 47 G §672. Nil holds none: an entry whose fingerprint names another toolchain is
// left unbuilt, as before.
Waits Waits
Manifests Manifests
Waits Waits
// Unbuildables keeps the plans no seat builds in the builder's state unbuildable (design 47 H2 §795, H-A-425). Nil keeps none.
Unbuildables Unbuildables
Manifests Manifests
// Left, when set, is told a key a look claimed and left building because its end could not be written (issue 1966,
// novox/hq ADR 0668 §3; 1748 note 27): no look holds it then, so nothing ends it but builder_build_again or the
// builder's next start. Ended is told each key whose end was written.
@@ -418,7 +420,7 @@ func (b *Builder) Sweep(ctx context.Context, pictured ...string) error {
}
closed := builds.BuilderBuild{State: builds.BuildFailed, By: b.Claimer.By, Manifest: cur.Manifest, Step: cur.Step,
Error: fmt.Sprintf("closed at start: left building by %s, and the module no longer has this fingerprint, so nothing will build it", cur.By)}
data, err := builds.EncodeBuilderBuild(closed)
data, err := builds.EncodeBuilderBuild(endOf(cur, closed, time.Now))
if err != nil {
return err
}
@@ -639,8 +641,10 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB
var noModule *NoBuildModule
if errors.As(planErr, &noModule) {
// The seat list or the catalog half does not name the build module yet: nothing is claimed, and the look is made again
// when they do (Reconsider, at each change of either; design 47 G A-82).
end("is not built yet: %v", planErr)
// when they do (Reconsider, at each change of either; design 47 G A-82). The plan is kept in the state unbuildable and
// said in the one summary line (SayUnbuildable), never in a line per module (design 47 H2 §795).
whyEnded = "is not built yet: " + planErr.Error()
b.judgedUnbuildable(ctx, module, m.Fp2, noModule)
return
}
nothing := read && planErr == nil && len(plan.Tasks) == 0
@@ -696,6 +700,7 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB
return
}
b.handClaimed(key, cl.Rev)
b.buildable(ctx, module) // the module builds: no longer a plan no seat builds (design 47 H2 §795, H-A-425)
if b.Waits != nil {
b.Waits.End(ctx, module, m.Fp2, WaitPublished) // a key queued behind a package is claimed once it is published
}
@@ -17,16 +17,20 @@ import (
// fingerprint (the existing claim, plan and lanes) and answers the step's end. A build is per fingerprint, so no pull request
// number is needed, and the builder follows no candidate.
// BuildStep builds the entries of one step ask and answers its end: passed when every entry's build is done, failed when one
// failed (said), could not run when an entry could not be read, nothing to run for an ask naming none. The ask's id and the node
// are the caller's to set.
func (b *Builder) BuildStep(ctx context.Context, run meshbuilds.StepRun) meshbuilds.StepEnded {
// BuildStep builds the entries of the step ask ask and answers its end: passed when every entry's build is done, failed when
// one failed (said), could not run when an entry could not be read, nothing to run for an ask naming none. The node is the
// caller's to set.
func (b *Builder) BuildStep(ctx context.Context, ask string, run meshbuilds.StepRun) meshbuilds.StepEnded {
b.init()
started := b.now()
end := meshbuilds.StepEnded{V: 1, Pipeline: run.Pipeline, Step: meshbuilds.StepEndedStepBuild, Rerun: run.Rerun, By: run.By, Entries: run.Entries, Node: b.Node, Started: &started}
// finish is the one place the ask is answered: its line names the pipeline, the movers, the step and the outcome, with
// the words of an end that did not pass (H-A-320 (1), H-A-333).
end := meshbuilds.StepEnded{V: 1, ID: ask, Pipeline: run.Pipeline, Step: meshbuilds.StepEndedStepBuild, Rerun: run.Rerun, By: run.By, Entries: run.Entries, Node: b.Node, Started: &started}
// finish is the one place the ask is answered: one journal line, "answered <ask> <pipeline> <movers> <step> <outcome>"
// (design 47 H2 §792; H-A-320 (1), H-A-333).
finish := func(outcome, said string) meshbuilds.StepEnded {
// An ask dropped through the seat's drop ends could-not-run naming the drop, whatever its run had reached (H-A-401).
if d := DroppedOf(ctx); d != nil {
outcome, said = meshbuilds.StepEndedOutcomeCouldNotRun, d.Error()
}
at := b.now()
end.Ended, end.Outcome, end.Said = &at, outcome, said
movers := make([]string, 0, len(run.Entries))
@@ -37,13 +41,12 @@ func (b *Builder) BuildStep(ctx context.Context, run meshbuilds.StepRun) meshbui
if mv == "" {
mv = "-"
}
if said != "" {
b.logf("%s %s %s step answered %s: %s", run.Pipeline, mv, end.Step, outcome, said)
} else {
b.logf("%s %s %s step answered %s", run.Pipeline, mv, end.Step, outcome)
}
b.logf("answered %s %s %s %s %s", ask, run.Pipeline, mv, end.Step, outcome)
return end
}
if DroppedOf(ctx) != nil { // dropped before it was pulled (H-A-404)
return finish(meshbuilds.StepEndedOutcomeCouldNotRun, "")
}
if len(run.Entries) == 0 {
return finish(meshbuilds.StepEndedOutcomeNothingToRun, "the ask names no entry")
}
@@ -34,13 +34,15 @@ func TestABuildStepAnswersItsEnd(t *testing.T) {
}
return meshcatalog.CatalogEntry{}, errors.New("not found")
}
asked := 0
ask := func(entries ...meshbuilds.BuildEntry) meshbuilds.StepEnded {
return r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.1.m", Step: meshbuilds.StepRunStepBuild, Entries: entries})
asked++
return r.b.BuildStep(t.Context(), fmt.Sprintf("ask%d", asked), meshbuilds.StepRun{Pipeline: "main.1.m", Step: meshbuilds.StepRunStepBuild, Entries: entries})
}
// Every answer is one journal line naming the pipeline, the movers, the step and the outcome (H-A-320 (1), H-A-333).
// Every answer is one journal line, "answered <ask> <pipeline> <movers> <step> <outcome>" (§792; H-A-320 (1), H-A-333).
var answers []string
r.b.Logf = func(f string, a ...any) {
if l := fmt.Sprintf(f, a...); strings.Contains(l, " step answered ") {
if l := fmt.Sprintf(f, a...); strings.HasPrefix(l, "answered ") {
answers = append(answers, l)
}
}
@@ -50,13 +52,19 @@ func TestABuildStepAnswersItsEnd(t *testing.T) {
if end := ask(meshbuilds.BuildEntry{Module: "m", Fingerprint: "src2:" + strings.Repeat("2", 64)}); end.Outcome != meshbuilds.StepEndedOutcomeCouldNotRun || !strings.Contains(end.Said, "cannot be read") {
t.Errorf("an entry the seat does not hold: %+v", end)
}
if len(answers) != 2 || answers[0] != "main.1.m - build step answered nothing to run: the ask names no entry" ||
!strings.HasPrefix(answers[1], "main.1.m m build step answered could not run: the entry m at ") {
if want := []string{"answered ask1 main.1.m - build nothing to run", "answered ask2 main.1.m m build could not run"}; !reflect.DeepEqual(answers, want) {
t.Errorf("the answers journaled %q, want one line per answer", answers)
}
if end := ask(meshbuilds.BuildEntry{Module: "m", Fingerprint: fp1, Commit: "other"}); end.Outcome != meshbuilds.StepEndedOutcomeCouldNotRun {
t.Errorf("an entry at another commit: %+v", end)
}
// H-A-401, H-A-404: an ask dropped through the seat's drop ends could-not-run naming the drop and who asked it.
dropped, drop := context.WithCancelCause(t.Context())
drop(&Dropped{By: "mesh-delivery-pipeline", Why: "its step ended by its bound"})
if end := r.b.BuildStep(dropped, "ask9", meshbuilds.StepRun{Pipeline: "main.1.m", Step: meshbuilds.StepRunStepBuild, Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}}); end.Outcome != meshbuilds.StepEndedOutcomeCouldNotRun ||
end.Said != "the ask was dropped by mesh-delivery-pipeline through the seat mesh-builds' drop: its step ended by its bound" || end.ID != "ask9" {
t.Errorf("a dropped ask: %+v", end)
}
// A build already done is passed at once, naming each entry as it was asked.
done, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildDone, Manifest: mdigest('a')})
if _, err := r.st.Create(t.Context(), r.key, done); err != nil {
@@ -128,7 +136,7 @@ func TestABuildStepPassesAReservingBuildOnlyWithItsReservedVersion(t *testing.T)
r.b.StepEntry = func(context.Context, string) (meshcatalog.CatalogEntry, error) {
return meshcatalog.CatalogEntry{Module: "m", Repository: "novox/m", Commit: "c1", Fp: fp1, ManifestDigest: digestOf(c.unreadable), Path: "modules/m"}, nil
}
end := r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.1.m", Step: meshbuilds.StepRunStepBuild, Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}})
end := r.b.BuildStep(t.Context(), "a1", meshbuilds.StepRun{Pipeline: "main.1.m", Step: meshbuilds.StepRunStepBuild, Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}})
if end.Outcome != c.want || !strings.Contains(end.Said, c.said) || (c.want == meshbuilds.StepEndedOutcomeFailed && strings.Contains(end.Said, "builder_reserve_done") == c.takenOver) {
t.Errorf("end = %+v, want %s saying %q", end, c.want, c.said)
}
@@ -199,7 +207,7 @@ func aPackageWithNoPinWaitsForItsRelease(t *testing.T) {
r.b.PackagePublished(unpinnedX, "0.1.4-feat-x.2") // a group's tagged version: no pin
ended := make(chan meshbuilds.StepEnded, 1)
go func() {
ended <- r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.9.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Source: 41,
ended <- r.b.BuildStep(t.Context(), "a1", meshbuilds.StepRun{Pipeline: "main.9.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Source: 41,
Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}})
}()
awaitWaiting(t, waiting, ended)
@@ -225,7 +233,7 @@ func aPackageWithNoPinWaitsForItsRelease(t *testing.T) {
}
// A done key reports the pins it keeps it was built with, never the newer one a publish gives now (H-A-207).
r.b.PackagePublished(unpinnedX, "v0.1.5")
again := r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.9.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Source: 41,
again := r.b.BuildStep(t.Context(), "a1", meshbuilds.StepRun{Pipeline: "main.9.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Source: 41,
Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}})
if again.Outcome != meshbuilds.StepEndedOutcomePassed || again.Packages[unpinnedX] != "v0.1.3" {
t.Errorf("the done key's end = %+v; want passed reporting x at v0.1.3, as built", again)
@@ -247,7 +255,7 @@ func aPackageWithNoPinWaitsForItsRelease(t *testing.T) {
t.Fatal(err)
}
r.st.vals[r.key] = raw
if bare := r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.9.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Source: 41,
if bare := r.b.BuildStep(t.Context(), "a1", meshbuilds.StepRun{Pipeline: "main.9.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Source: 41,
Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}}); len(bare.Packages) != 0 {
t.Errorf("a done key keeping no pins reported %v; want none", bare.Packages)
}
@@ -256,7 +264,7 @@ func aPackageWithNoPinWaitsForItsRelease(t *testing.T) {
func aPackageWithNoReleaseFailsPastItsBound(t *testing.T) {
r, _, _ := unpinnedRig(t)
r.b.PackageWait = time.Nanosecond
end := r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.9.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Source: 41,
end := r.b.BuildStep(t.Context(), "a1", meshbuilds.StepRun{Pipeline: "main.9.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Source: 41,
Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}})
r.b.Wait()
if _, built := r.st.vals[r.key]; end.Outcome != meshbuilds.StepEndedOutcomeFailed || built ||
@@ -265,7 +273,7 @@ func aPackageWithNoReleaseFailsPastItsBound(t *testing.T) {
}
// A publish heard before gives the pin at once.
r.b.PackagePublished(unpinnedX, "v0.1.2")
if end := r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.9.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Source: 41,
if end := r.b.BuildStep(t.Context(), "a1", meshbuilds.StepRun{Pipeline: "main.9.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Source: 41,
Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}}); strings.Contains(end.Said, "no released version") {
t.Errorf("a publish heard did not pin the package: %+v", end)
}
@@ -281,7 +289,7 @@ func theSeatsVersionsPinBeforeTheWait(t *testing.T) {
read = append(read, path)
return []string{"0.2.1", "0.1.9", "0.1.8"}, nil // newest first
}
end := r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.9.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Source: 41,
end := r.b.BuildStep(t.Context(), "a1", meshbuilds.StepRun{Pipeline: "main.9.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Source: 41,
Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}})
r.b.Wait()
asks := r.asks.Asks()
@@ -292,7 +300,7 @@ func theSeatsVersionsPinBeforeTheWait(t *testing.T) {
r2.b.PublishedVersions = func(context.Context, string) ([]string, error) { return []string{"0.2.1"}, nil }
ended := make(chan meshbuilds.StepEnded, 1)
go func() {
ended <- r2.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.9.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Source: 41,
ended <- r2.b.BuildStep(t.Context(), "a1", meshbuilds.StepRun{Pipeline: "main.9.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Source: 41,
Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}})
}()
awaitWaiting(t, waiting, ended)
@@ -345,7 +353,7 @@ func aRevisionHeldWakesTheAsk(t *testing.T) {
r.b.Wait()
ended := make(chan meshbuilds.StepEnded, 1)
go func() {
ended <- r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.8.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Revision: 8, Source: 41,
ended <- r.b.BuildStep(t.Context(), "a1", meshbuilds.StepRun{Pipeline: "main.8.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Revision: 8, Source: 41,
Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}})
}()
select {
@@ -398,7 +406,7 @@ func aStepAskWaitsForItsRevision(t *testing.T) {
}
ended := make(chan meshbuilds.StepEnded, 1)
go func() {
ended <- r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.7.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Revision: 7, Source: 41,
ended <- r.b.BuildStep(t.Context(), "a1", meshbuilds.StepRun{Pipeline: "main.7.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Revision: 7, Source: 41,
Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}})
}()
select {
@@ -442,7 +450,7 @@ func anOlderKeptRevisionIsReadByNumber(t *testing.T) {
}
run := meshbuilds.StepRun{Pipeline: "main.47.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Revision: 47, Source: 45,
Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}}
end := r.b.BuildStep(t.Context(), run)
end := r.b.BuildStep(t.Context(), "a1", run)
r.b.Wait()
if end.Outcome != meshbuilds.StepEndedOutcomePassed || !slices.Equal(read, []string{"main 47"}) {
t.Errorf("end = %+v, kept revisions read %v; want passed, revision 47 of main read once", end, read)
@@ -457,7 +465,7 @@ func anOlderKeptRevisionIsReadByNumber(t *testing.T) {
r2.b.KeptRevision = func(context.Context, string, int64) (int64, map[string]string, error) {
return 45, map[string]string{"m": "src2:other"}, nil
}
if end := r2.b.BuildStep(t.Context(), run); end.Outcome != meshbuilds.StepEndedOutcomeFailed || !strings.Contains(end.Said, "names m at \"src2:other\"") {
if end := r2.b.BuildStep(t.Context(), "a1", run); end.Outcome != meshbuilds.StepEndedOutcomeFailed || !strings.Contains(end.Said, "names m at \"src2:other\"") {
t.Errorf("end = %+v; want failed naming the fingerprint the kept revision names", end)
}
// H-A-222: an ask of the very source the picture holds, whose entries the picture no longer names (a re-pin kept revision 48
@@ -472,7 +480,7 @@ func anOlderKeptRevisionIsReadByNumber(t *testing.T) {
}
same := meshbuilds.StepRun{Pipeline: "main.48.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Revision: 48, Source: 46,
Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}}
end3 := r3.b.BuildStep(t.Context(), same)
end3 := r3.b.BuildStep(t.Context(), "a1", same)
r3.b.Wait()
if end3.Outcome != meshbuilds.StepEndedOutcomePassed || !slices.Equal(read3, []string{"main 48"}) || len(res3.sources) != 1 || res3.sources[0].CatalogRevision != 48 {
t.Errorf("end = %+v, kept revisions read %v, reserved %+v; want passed, revision 48 of main read once and reserved at it", end3, read3, res3.sources)
@@ -493,7 +501,7 @@ func anOlderKeptRevisionIsReadByNumber(t *testing.T) {
rc.b.KeptRevision = func(context.Context, string, int64) (int64, map[string]string, error) {
return c.source, map[string]string{"m": c.fp}, nil
}
end := rc.b.BuildStep(t.Context(), same)
end := rc.b.BuildStep(t.Context(), "a1", same)
rc.b.Wait()
if end.Outcome != meshbuilds.StepEndedOutcomeFailed || !strings.Contains(end.Said, c.said) || len(resc.sources) != 0 {
t.Errorf("%s: end = %+v, reserved %+v; want failed at once saying %q, nothing reserved", c.name, end, resc.sources, c.said)
@@ -506,14 +514,14 @@ func aStepAskPastItsRevisionBoundFails(t *testing.T) {
held.Store(40)
r, res, _ := revisionRig(t, &held, make(chan struct{}, 1))
r.b.RevisionWait = time.Nanosecond
end := r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.7.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Revision: 7, Source: 41,
end := r.b.BuildStep(t.Context(), "a1", meshbuilds.StepRun{Pipeline: "main.7.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Revision: 7, Source: 41,
Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}})
r.b.Wait()
if _, built := r.st.vals[r.key]; end.Outcome != meshbuilds.StepEndedOutcomeFailed || !strings.Contains(end.Said, "revision 41 of main not in the builder's picture") || built || len(res.sources) != 0 {
t.Errorf("end = %+v, built %v, reserved %+v; want failed naming revision 41 of main, nothing built", end, built, res.sources)
}
// A step ask with no source (a run on request) does not wait (H-A-156).
end = r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "run.1.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}})
end = r.b.BuildStep(t.Context(), "a1", meshbuilds.StepRun{Pipeline: "run.1.m", Step: meshbuilds.StepRunStepBuild, Branch: "main", Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}})
if strings.Contains(end.Said, "not in the builder's picture") {
t.Errorf("an ask with no source waited: %+v", end)
}
@@ -74,6 +74,27 @@ const ModePublish = "publish"
// optional, name what to publish again; the builder answers with the versions it queued (Queued).
const ModeRepublish = "republish"
// ModePublishDrop and ModePublishFirst are the Modes of a reasks value of builder_publish_drop and builder_publish_first (design
// 47 H2 §793): Module and Version name the item; the builder terminates it, or puts it again as live and terminates the old.
const (
ModePublishDrop = "publish-drop"
ModePublishFirst = "publish-first"
)
// NewPublishAct is the value builder_publish_drop (mode ModePublishDrop) or builder_publish_first (ModePublishFirst) writes
// (§793): its key, `<mode>.<unix nanoseconds>` (a call is one value), and the value. Module, version and why are required.
func NewPublishAct(mode, module, version, why, by string, at time.Time) (string, []byte, error) {
module, version = strings.TrimSpace(module), strings.TrimSpace(version)
if module == "" || version == "" {
return "", nil, errors.New("give the item's module and version")
}
if strings.TrimSpace(why) == "" {
return "", nil, errors.New("say why, in plain words (why)")
}
data, err := json.Marshal(ReAsk{V: 1, Why: strings.TrimSpace(why), By: by, At: at.UTC(), Mode: mode, Module: module, Version: version})
return mode + "." + strconv.FormatInt(at.UnixNano(), 10), data, err
}
// ModeFloorAgain is the Mode of a reasks value of builder_floor_again (design 51 *The floor*).
const ModeFloorAgain = "floor-again"
@@ -0,0 +1,156 @@
package build
import (
"context"
"encoding/json"
"errors"
"fmt"
"sort"
"strings"
"time"
)
// **The plans no seat builds** (design 47 H2 §795, H-A-425): a module whose artifact no build seat builds, or whose build seat
// no module claims, is kept in the builder's state unbuildable, one key per module, written when its plan is judged and removed
// when the module builds; builder_unbuildable reads that state. The judgement is made again at every start and every change
// of the seat list (SayUnbuildable), and said in one summary line each time, never a line per module.
const (
// UnbuildableState is the builder's state of the plans no seat builds, one key <module>, 8 MiB.
UnbuildableState = "unbuildable"
// UnbuildableTool is builder_unbuildable, which lists that state.
UnbuildableTool = "builder_unbuildable"
// UnbuildableSaid opens the summary line.
UnbuildableSaid = "modules have an artifact no seat builds"
)
// Unbuildable is one key of the state unbuildable: the module, the fingerprint whose plan was judged, the first artifact's
// language and kind no seat builds, and why.
type Unbuildable struct {
Module string `json:"module"`
Fingerprint string `json:"fingerprint"`
Language string `json:"language,omitempty"`
Kind string `json:"kind"`
Why string `json:"why"`
At time.Time `json:"at"`
}
// Unbuildables keeps the state unbuildable: Put writes a module's key, Delete removes it (a key not there is no error) and Keys
// lists the modules it holds.
type Unbuildables interface {
Put(ctx context.Context, module string, value []byte) error
Delete(ctx context.Context, module string) error
Keys(ctx context.Context) ([]string, error)
}
// unbuildableOf is the key value of module's plan judged at fp with none.
func unbuildableOf(module, fp string, none *NoBuildModule, at time.Time) Unbuildable {
return Unbuildable{Module: module, Fingerprint: fp, Language: none.Language, Kind: none.Kind, Why: none.Error(), At: at.UTC()}
}
// judgedUnbuildable writes module's key: its plan, judged at fp, names an artifact no seat builds.
func (b *Builder) judgedUnbuildable(ctx context.Context, module, fp string, none *NoBuildModule) {
if b.Unbuildables == nil {
return
}
raw, _ := json.Marshal(unbuildableOf(module, fp, none, b.now()))
if err := b.Unbuildables.Put(ctx, module, raw); err != nil {
b.logf("%s: the plan no seat builds is not kept in the state %s: %v", module, UnbuildableState, err)
}
}
// buildable removes module's key: the module builds.
func (b *Builder) buildable(ctx context.Context, module string) {
if b.Unbuildables == nil {
return
}
if err := b.Unbuildables.Delete(ctx, module); err != nil {
b.logf("%s: its key of the state %s is not removed: %v", module, UnbuildableState, err)
}
}
// SayUnbuildable judges the plan of every module the builder knows again, keeps the ones no seat builds in the state unbuildable
// (writing the modules it does not hold yet and removing the ones that build now), and says them in one line: how many, and
// each language and kind with its count (design 47 H2 §795). It answers the modules judged so.
func (b *Builder) SayUnbuildable(ctx context.Context) []Unbuildable {
b.init()
b.mu.Lock()
known := sortedKeys(b.known)
modules := make(map[string]string, len(known)) // the fingerprint each is known at
digests := make(map[string]string, len(known))
for _, k := range known {
modules[k], digests[k] = b.known[k].Fp2, b.known[k].Manifest
}
b.mu.Unlock()
var out []Unbuildable
now := b.now()
for _, k := range known {
if modules[k] == "" || digests[k] == "" {
continue
}
_, err, _ := b.planOf(ctx, digests[k])
var none *NoBuildModule
if errors.As(err, &none) {
out = append(out, unbuildableOf(k, modules[k], none, now))
}
}
if b.Unbuildables != nil {
held := map[string]bool{}
keys, err := b.Unbuildables.Keys(ctx)
if err != nil {
b.logf("the state %s cannot be listed, so the plans no seat builds are written whole: %v", UnbuildableState, err)
}
for _, k := range keys {
held[k] = true
}
judged := map[string]bool{}
for _, u := range out {
judged[u.Module] = true
if held[u.Module] {
continue // written when its plan was judged at its look
}
raw, _ := json.Marshal(u)
if err := b.Unbuildables.Put(ctx, u.Module, raw); err != nil {
b.logf("%s: the plan no seat builds is not kept in the state %s: %v", u.Module, UnbuildableState, err)
}
}
for _, k := range keys {
if !judged[k] {
b.buildable(ctx, k)
}
}
}
b.logf("%s", UnbuildableLine(out))
return out
}
// UnbuildableLine is the one summary line of the plans no seat builds: "N modules have an artifact no seat builds:" with each
// language and kind and its count, most first (design 47 H2 §795).
func UnbuildableLine(us []Unbuildable) string {
counts := map[string]int{}
for _, u := range us {
what := u.Kind
if u.Language != "" {
what = u.Language + " " + u.Kind
}
counts[what]++
}
whats := make([]string, 0, len(counts))
for w := range counts {
whats = append(whats, w)
}
sort.Slice(whats, func(i, j int) bool {
if counts[whats[i]] != counts[whats[j]] {
return counts[whats[i]] > counts[whats[j]]
}
return whats[i] < whats[j]
})
parts := make([]string, len(whats))
for i, w := range whats {
parts[i] = fmt.Sprintf("%s %d", w, counts[w])
}
line := fmt.Sprintf("%d %s", len(us), UnbuildableSaid)
if len(parts) > 0 {
line += ": " + strings.Join(parts, ", ")
}
return line
}
@@ -27,6 +27,10 @@ type Acts struct {
ReserveDone func(ctx context.Context, module, fp, by, why string, at time.Time) (string, error)
// DropReserved drops a module's reservation of one MAJOR.MINOR by hand (builder_reserved_drop): its answer, or why not.
DropReserved func(ctx context.Context, module, series, by, why string, at time.Time) (string, error)
// PublishDrop terminates one publish item, and PublishFirst puts it again as live and terminates the old (builder_publish_drop,
// builder_publish_first, design 47 H2 §793): the answer, or why not.
PublishDrop func(ctx context.Context, module, version, by, why string) (string, error)
PublishFirst func(ctx context.Context, module, version, by, why string) (string, error)
}
// FloorAgain is builder_floor_again of name: refused, since the builder numbers nothing of its own and no claim asks a floor
@@ -0,0 +1,49 @@
package versions
import (
"encoding/json"
"fmt"
"sort"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/health"
"git.novox.be/novox/mesh-sdk/go/process"
)
// BuilderHealth is builder_health's answer at now (design 47 H2 §794, H-A-435, H-A-462): healthy when each of the builder's
// durable consumers' oldest unacknowledged message waits less than behindAfter, as the consumers' facts the builder writes at
// every delivery and every acknowledgement say them (its state consumers, read through consumers); otherwise unhealthy, naming
// each consumer behind and how long. The judgement is the module's shared code, so the builder's whole test proves it.
func BuilderHealth(consumers Reader, now time.Time, behindAfter time.Duration) health.Answer {
keys, err := consumers.Keys()
if err != nil {
return health.Unhealthy(fmt.Sprintf("The ages of the builder's consumers cannot be read from its state %s (%v).", process.ConsumersState, err))
}
if len(keys) == 0 {
return health.Healthy(fmt.Sprintf("The builder answers and has heard no event yet: its state %s holds no consumer.", process.ConsumersState))
}
sort.Strings(keys)
var behind []string
for _, k := range keys {
raw, ok, err := consumers.Get(k)
if err != nil {
return health.Unhealthy(fmt.Sprintf("The age of the consumer %s cannot be read (%v).", k, err))
}
var f process.ConsumerFacts
if ok {
if err := json.Unmarshal(raw, &f); err != nil {
return health.Unhealthy(fmt.Sprintf("The age of the consumer %s does not read (%v).", k, err))
}
}
if !f.OldestUnacknowledged.IsZero() {
if age := now.Sub(f.OldestUnacknowledged); age >= behindAfter {
behind = append(behind, fmt.Sprintf("%s (its oldest unacknowledged message waits %s, %d more not yet given)", k, age.Round(time.Second), f.Pending))
}
}
}
if len(behind) > 0 {
return health.Unhealthy(fmt.Sprintf("The builder's consumers behind by %s or more: %s.", behindAfter, strings.Join(behind, ", ")))
}
return health.Healthy(fmt.Sprintf("The builder answers, and each of its %d consumers is less than %s behind.", len(keys), behindAfter))
}
@@ -14,6 +14,13 @@ import (
type countingReserve struct {
*reserveMem
lists, reads atomic.Int64
// indexReads and writes count the reads of an index key and the writes of any key (design 47 H2 §794).
indexReads, writes atomic.Int64
}
func (c *countingReserve) Update(ctx context.Context, key string, value []byte, rev uint64) (uint64, error) {
c.writes.Add(1)
return c.reserveMem.Update(ctx, key, value, rev)
}
func (c *countingReserve) KeysOf(ctx context.Context, filter string) ([]string, error) {
@@ -24,6 +31,8 @@ func (c *countingReserve) KeysOf(ctx context.Context, filter string) ([]string,
func (c *countingReserve) Get(ctx context.Context, key string) ([]byte, uint64, bool, error) {
if !strings.HasPrefix(key, "index.") {
c.reads.Add(1)
} else {
c.indexReads.Add(1)
}
return c.reserveMem.Get(ctx, key)
}
@@ -52,6 +61,31 @@ func TestTheIndexFindsAPublishedReservationInOneRead(t *testing.T) {
store.reads.Store(0)
return r, store
}
// Design 47 H2 §794: 1,000 published events over 2,000 reservations cost one read of the index and one write each.
t.Run("1,000 publisheds over 2,000 reservations read the index once and write once each", func(t *testing.T) {
store := &countingReserve{reserveMem: &reserveMem{mem: newMem()}}
r := &Reserver{Store: store, ReleasedOf: func(context.Context, string) ([]string, error) { return nil, nil }}
for i := range 2000 {
m := fmt.Sprintf("r%04d", i)
if _, err := r.Reserve(ctx, m, "0.1", []string{m + ":a"}, Source{Fingerprint: "fp" + m}); err != nil {
t.Fatal(err)
}
if _, err := r.Built(ctx, m, "0.1.0", m+":a", "sha256:"+m, nil); err != nil {
t.Fatal(err)
}
}
store.indexReads.Store(0)
store.writes.Store(0)
for i := range 1000 {
m := fmt.Sprintf("r%04d", 2*i)
if marked, _, err := r.Published(ctx, m+"-a", "0.1.0", "sha256:"+m); err != nil || len(marked) != 1 {
t.Fatalf("the published of %s marked %v (%v)", m, marked, err)
}
}
if ir, w, l := store.indexReads.Load(), store.writes.Load(), store.lists.Load(); ir != 1000 || w != 1000 || l != 0 {
t.Errorf("1,000 publisheds read the index %d times, wrote %d and listed %d, want 1,000 reads, 1,000 writes and no listing", ir, w, l)
}
})
t.Run("a published by digest reads its one reservation", func(t *testing.T) {
r, store := fill(t)
marked, _, err := r.Published(ctx, "m10-a", "v0.1.0", "sha256:m10")
@@ -39,7 +39,7 @@ func NamesIn(s PackagesState) PackageNames {
if !found {
continue
}
v, err := meshcatalog.DecodePackagesByPackageSeriesValue(value) // the indexer's value of a series key (mesh-sdk's contract, J4)
v, _, err := meshcatalog.DecodePackagesByPackageSeriesValue(value) // the indexer's value of a series key (mesh-sdk's contract, J4)
if err != nil || v.Package != built {
continue
}
@@ -56,7 +56,7 @@ func NamedModule(key string, value []byte) (module, name string, ok bool) {
if !isSeriesKey(key) {
return "", "", false
}
v, err := meshcatalog.DecodePackagesByPackageSeriesValue(value)
v, _, err := meshcatalog.DecodePackagesByPackageSeriesValue(value)
if err != nil || v.Name == "" {
return "", "", false
}
@@ -99,7 +99,7 @@ func Carriers(s PackagesState) func(ctx context.Context, name string) ([]Built,
if !found {
continue
}
v, err := meshcatalog.DecodePackagesByPackageSeriesValue(value)
v, _, err := meshcatalog.DecodePackagesByPackageSeriesValue(value)
if err != nil || v.Name != name {
continue
}
@@ -12,27 +12,34 @@ import (
"sync"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
)
// **A reserved version of a done build is published by the builder until its release is written** (design 47 G §588, A-457a): the
// publish is an item of the builder's re-ask state, asked when the build ends and at its start, tried with a growing pause and
// ended loudly after its tries; each registry seat's holder, as the seat-holders view names it (A-416, A-416a), has one queue of
// versions, kept in the state `publish-queue`, sent one at a time. A send is the registry seat's `publish` with the digest and no
// ended loudly after its tries; each registry seat's holder, as the seat-holders view names it (A-416, A-416a), has its items on
// the builder's work queue `publishes` (design 47 H2 §793), subject `publish.<holder>.<class>.<module>.<version>`, pulled by one
// consumer per holder and class, the live class first. A send is the registry seat's `publish` with the digest and no
// parts (A-73): the holder takes the bytes from the artifact store by digest, and refuses what no store holds. Nothing waits on a
// node-engine `running` or `manifest-reads-changed` any more.
const (
// PublishQueueState is the builder's old state section of the queues, capped at PublishQueueMiB.
// PublishQueueState is the builder's old state of the queues, whose items are put on the work queue once at the first start
// of this release, then removed by the builder itself, which made it (design 47 H2 §793; H-A-461 (1)).
PublishQueueState = "publish-queue"
PublishQueueMiB = 4
// PublishesLocal is the builder's work queue of publish items, asked for in its requires (§793, §820; H-A-448).
PublishesLocal = "publishes"
// ActedState holds, one key per subject, the last stream sequence the builder acted on; capped at ActedMiB.
ActedState = "acted"
ActedMiB = 1
// PublishQueueTool is builder_publish_queue.
PublishQueueTool = "builder_publish_queue"
// QueueFull is what version-unreleased says of a version the queue's cap refused.
// QueueFull is what version-unreleased says of a version the work queue's cap refused.
QueueFull = "queue full"
// PublishDropTool and PublishFirstTool are builder_publish_drop and builder_publish_first (§793).
PublishDropTool = "builder_publish_drop"
PublishFirstTool = "builder_publish_first"
// NotStoredWords opens what a refused artifact is said as.
NotStoredWords = "not stored: "
// VerbPublish is the registry seats' verb a publish again sends, VerbVersions the one whose record says what a seat holds
@@ -40,11 +47,11 @@ const (
VerbPublish = "publish"
VerbVersions = "versions"
// PublishClassLive is the class of a publish whose build ended in this run, which a pipeline's build step waits on;
// PublishClassStart that of a publish asked again at a start, of a build that ended before it. A holder's queue sends every
// live one before any of the start, each class oldest first (design 47 H H-A-290 (1)).
PublishClassLive = "build-ended-this-run"
PublishClassStart = "asked-again-at-start"
// PublishSendsAtOnce bounds the sends of one holder's queue that run at once (H-A-290 (3)).
// PublishClassStart that of a publish asked again at a start, of a build that ended before it. A holder's items are pulled
// live first, each class oldest first (design 47 H H-A-290 (1), H2 §793).
PublishClassLive = "live"
PublishClassStart = "again"
// PublishSendsAtOnce bounds the sends of one holder that run at once, its consumers' max ack pending (H-A-290 (3), §793).
PublishSendsAtOnce = 4
// PublishQueueListCap caps the queued versions and the publish items builder_publish_queue lists (H-A-290 (5)).
PublishQueueListCap = 50
@@ -77,9 +84,8 @@ func SeatOfArtifact(a meshcatalog.Artifact) (string, bool) {
return seat, ok
}
// QueuedPublish is the value of a publish-queue key `<holder module>.<module>.<version>` (each token escaped, §73): the
// artifacts of the version this holder publishes, by reference with their digests, when it was queued and whether its
// publish has begun.
// QueuedPublish is one item of the work queue publishes (§793), and the value of a key of the old state publish-queue: the
// artifacts of the version this holder publishes, by reference with their digests, and when it was queued.
type QueuedPublish struct {
V int `json:"v"`
Holder string `json:"holder"`
@@ -87,7 +93,6 @@ type QueuedPublish struct {
Version string `json:"version"`
Artifacts map[string]string `json:"artifacts"`
Queued time.Time `json:"queued"`
Begun bool `json:"begun,omitempty"`
// Class is PublishClassLive or PublishClassStart (H-A-290 (1)); a value written before it had one is of the start.
Class string `json:"class,omitempty"`
}
@@ -110,25 +115,31 @@ func sortQueued(qs []QueuedPublish) {
})
}
// QueueFilter is the filter of every queued version: a queue key has three tokens, which AllFilter (two) never lists on the bus.
const QueueFilter = "*.*.*"
// QueueKey is the key of holder's queued version of module.
// QueueKey is the key of holder's queued version of module in the old state publish-queue.
func QueueKey(holder, module, version string) string {
return EscapeToken(holder) + "." + EscapeToken(module) + "." + EscapeToken(version)
}
// PublishMsgID is the message id of holder's item of module's version in class (§793): the work queue takes one item of each
// id within its duplicates window, so an item put twice is put once.
func PublishMsgID(holder, class, module, version string) string {
return EscapeToken(holder) + "." + class + "." + EscapeToken(module) + "." + EscapeToken(version)
}
// PublishEvent is the subject of holder's item of module's version in class within the work queue publishes (§793):
// `publish.<holder>.<class>.<module>.<version>`, each token escaped.
func PublishEvent(holder, class, module, version string) string {
return "publish." + PublishMsgID(holder, class, module, version)
}
// ErrQueueFull is a put the work queue refused as full: said as QueueFull, and asked again at the item's next try (§820).
var ErrQueueFull = errors.New("the work queue " + PublishesLocal + " is full")
// Acted is the value of an acted key: the last stream sequence acted on for its subject.
type Acted struct {
Sequence uint64 `json:"sequence"`
}
// QueueStore is the queue's bucket: compare-and-set, deletion and how many bytes it holds, for its cap.
type QueueStore interface {
ReserveStore
Bytes(ctx context.Context) (uint64, error)
}
// Publish is one artifact's publish again: what the registry seat's publish takes, without parts.
type Publish struct {
Seat, Kind, Name, Version, Digest, Fingerprint, Commit, Transfer string
@@ -137,7 +148,9 @@ type Publish struct {
// Republisher publishes reserved versions again (design 47 G §588).
type Republisher struct {
Reserved *Reserver
Queue QueueStore
// Put puts one item on the work queue publishes under its message id (PublishMsgID): duplicate says the queue holds that
// id within its duplicates window and took nothing; a full queue is ErrQueueFull.
Put func(ctx context.Context, q QueuedPublish) (duplicate bool, err error)
// Entry reads the catalog graph entry of a fingerprint (the seat mesh-catalog's verb entry).
Entry func(ctx context.Context, fingerprint string) (meshcatalog.CatalogEntry, error)
// Send sends one publish to a registry seat; an error is the holder's refusal or the call's fault, in its words.
@@ -146,8 +159,6 @@ type Republisher struct {
Unreleased func(ctx context.Context, v ReservedVersion, why string)
Now func() time.Time
Logf func(string, ...any)
// Go starts the sending of a holder's queue; nil starts a goroutine.
Go func(func())
// Holders answers a registry seat's holders: the catalog's SeatHolders, the part's one holder function that its FollowSeat
// of the assignments takes too, read from the builder's start (design 47 G A-416, A-416a); nil answers none.
Holders func(seat string) []string
@@ -158,11 +169,7 @@ type Republisher struct {
// nil ends none.
Ended func(ctx context.Context, module, version string)
mu sync.Mutex
busy map[string]bool
// relist marks a holder whose pass lists again before its next start-class send: a live publish was queued meanwhile
// (H-A-291 (2)).
relist map[string]bool
mu sync.Mutex
// lastQueued is the newest time a version was queued at: a clock that has not moved between two versions (a coarse clock,
// or a fixed one) must not make the queue's order the order of its keys, so each is queued after the last.
lastQueued time.Time
@@ -292,18 +299,9 @@ func (r *Reserver) Retired(ctx context.Context, all []ReservedVersion) (map[stri
return out, nil
}
// drop deletes every queued key of module's version and ends its publish item: it never publishes (H-A-290 (2)), said with why.
// drop ends module's version's publish item: it never publishes (H-A-290 (2)), said with why; its items on the work queue are
// terminated when they are pulled (§793).
func (r *Republisher) drop(ctx context.Context, module, version, why string) {
ctx = context.WithoutCancel(ctx)
keys, err := r.Queue.KeysOf(ctx, "*."+EscapeToken(module)+"."+EscapeToken(version))
if err != nil {
r.logf("publish again: %s %s never publishes (%s), and its queued keys cannot be listed: %v", module, version, why, err)
}
for _, k := range keys {
if err := r.Queue.Delete(ctx, k); err != nil {
r.logf("%s cannot be deleted from %s: %v", k, PublishQueueState, err)
}
}
r.end(ctx, module, version, why)
}
@@ -338,13 +336,6 @@ func (r *Republisher) queue(ctx context.Context, vs []ReservedVersion, holdersOf
sort.SliceStable(vs, func(i, j int) bool { return vs[i].At.After(vs[j].At) })
var queued []string
var errs []error
started := map[string]bool{}
// Every holder given a version is drained, whatever a later version met: a version queued and never drained waits forever.
defer func() {
for holder := range started {
r.drainLater(ctx, holder)
}
}()
for _, v := range vs {
if max > 0 && len(queued) >= max {
break
@@ -396,7 +387,6 @@ func (r *Republisher) queue(ctx context.Context, vs []ReservedVersion, holdersOf
return queued, err
}
did = did || ok
started[holder] = true
}
if did {
queued = append(queued, v.Module+" "+v.Version)
@@ -405,42 +395,25 @@ func (r *Republisher) queue(ctx context.Context, vs []ReservedVersion, holdersOf
return queued, errors.Join(errs...)
}
// enqueue puts one holder's artifacts of a version in its queue: not again when it is queued already, and not past the
// section's cap (said as queue full).
// enqueue puts one holder's artifacts of a version on the work queue under its message id: not again while the queue holds that
// id (its duplicates window), which stands queued, and not past the queue's cap (said as queue full).
func (r *Republisher) enqueue(ctx context.Context, v ReservedVersion, holder string, arts map[string]string) (bool, error) {
key := QueueKey(holder, v.Module, v.Version)
if _, _, found, err := r.Queue.Get(ctx, key); err != nil {
return false, err
} else if found {
r.logf("publish again: %s %s is queued at %s already", v.Module, v.Version, holder)
return false, nil
}
raw, _ := json.Marshal(QueuedPublish{V: 1, Holder: holder, Module: v.Module, Version: v.Version, Artifacts: arts, Queued: r.queuedAt(), Class: r.classOf(v)})
used, err := r.Queue.Bytes(ctx)
if err != nil {
return false, fmt.Errorf("the size of the state %s cannot be read: %w", PublishQueueState, err)
}
if used+uint64(len(raw)) > PublishQueueMiB<<20 {
q := QueuedPublish{V: 1, Holder: holder, Module: v.Module, Version: v.Version, Artifacts: arts, Queued: r.queuedAt(), Class: r.classOf(v)}
dup, err := r.Put(ctx, q)
switch {
case errors.Is(err, ErrQueueFull):
r.logf("publish again: %s %s is not queued at %s: %v; its publish item asks it again at its next try", v.Module, v.Version, holder, err)
if r.Unreleased != nil {
r.Unreleased(ctx, v, QueueFull)
}
return false, nil
case err != nil:
return false, fmt.Errorf("%s %s cannot be queued at %s: %w", v.Module, v.Version, holder, err)
case dup:
r.logf("publish again: %s %s is queued at %s already (class %s): the work queue holds its message id", v.Module, v.Version, holder, q.Class)
return true, nil
}
if _, err := r.Queue.Create(ctx, key, raw); err != nil {
if errors.Is(err, ErrConflict) {
return false, nil
}
return false, fmt.Errorf("%s cannot be queued: %w", key, err)
}
r.logf("publish again: %s %s is queued to publish at %s: %d artifacts, class %s", v.Module, v.Version, holder, len(arts), r.classOf(v))
if r.classOf(v) == PublishClassLive { // a pass sending a start list lists again, so this one goes next (H-A-291 (2))
r.mu.Lock()
if r.relist == nil {
r.relist = map[string]bool{}
}
r.relist[holder] = true
r.mu.Unlock()
}
r.logf("publish again: %s %s is queued to publish at %s: %d artifacts, class %s", v.Module, v.Version, holder, len(arts), q.Class)
return true, nil
}
@@ -537,17 +510,20 @@ func PublishItemKey(module, version string) string {
return PublishItemKeyPrefix + EscapeToken(module) + "." + EscapeToken(version)
}
// AnswerQueue is builder_publish_queue over the three states as the tools bundle reads them (H-A-290 (5)): each state read in
// bulk (Reader.All), the answer counted and capped at PublishQueueListCap whatever the queue holds; module (and version) narrow the
// listed queue to one module's (one version's), so an item's class can be read.
func AnswerQueue(queue, acted, reasks Reader, module, version string) (Queued, error) {
out := Queued{Counts: []QueueCount{}, Queue: []QueuedPublish{}, Acted: map[string]uint64{}, Items: []PublishItem{}}
vals, c, err := queue.values(PublishQueueState)
out.capped(c)
if err != nil {
return out, fmt.Errorf("the state %s cannot be read: %w", PublishQueueState, err)
// PublishQueueOf is the seat mesh-builds' verb publish-queue (design 47 H2 §793; H-A-454): the builder's read of its own work queue
// publishes (queue, its items read in bulk, capped past its bound), small at any size of the queue (H-A-290 (5)): the items' count
// per holder and class with the oldest and newest queue time, and the first PublishQueueListCap in the order they are sent; module
// (and version) narrow the list to one module's (one version's), so an item's class can be read.
func PublishQueueOf(queue Reader, module, version string) (meshbuilds.PublishQueueAnswer, error) {
out := meshbuilds.PublishQueueAnswer{Counts: []meshbuilds.PublishQueueAnswerCountsItem{}, Queue: []meshbuilds.PublishQueueAnswerQueueItem{}}
vals, c, err := queue.values(PublishesLocal)
if c != nil {
out.Capped, out.ItemsRead, out.QueueItems, out.QueueBytes = true, int64(c.KeysRead), int64(c.StateKeys), int64(c.StateBytes)
}
counts := map[[2]string]*QueueCount{}
if err != nil {
return out, fmt.Errorf("the work queue %s cannot be read: %w", PublishesLocal, err)
}
counts := map[[2]string]*meshbuilds.PublishQueueAnswerCountsItem{}
var listed []QueuedPublish
for _, raw := range vals {
var q QueuedPublish
@@ -560,7 +536,7 @@ func AnswerQueue(queue, acted, reasks Reader, module, version string) (Queued, e
out.Total++
c := counts[[2]string{q.Holder, q.Class}]
if c == nil {
c = &QueueCount{Holder: q.Holder, Class: q.Class, Oldest: q.Queued, Newest: q.Queued}
c = &meshbuilds.PublishQueueAnswerCountsItem{Holder: q.Holder, Class: q.Class, Oldest: q.Queued, Newest: q.Queued}
counts[[2]string{q.Holder, q.Class}] = c
}
c.Count++
@@ -599,10 +575,27 @@ func AnswerQueue(queue, acted, reasks Reader, module, version string) (Queued, e
if len(out.Queue) == PublishQueueListCap {
break
}
out.Queue = append(out.Queue, q)
out.Queue = append(out.Queue, meshbuilds.PublishQueueAnswerQueueItem{Holder: q.Holder, Module: q.Module, Version: q.Version,
Class: q.Class, Artifacts: q.Artifacts, Queued: q.Queued})
}
}
out.Listed = len(out.Queue)
out.Listed = int64(len(out.Queue))
return out, nil
}
// AnswerQueue is builder_publish_queue: the seat mesh-builds' publish-queue answer the builder gave (q, §793, H-A-454) and the two
// states as the tools bundle reads them (H-A-290 (5)), each read in bulk (Reader.All), capped at PublishQueueListCap.
func AnswerQueue(q meshbuilds.PublishQueueAnswer, acted, reasks Reader) (Queued, error) {
out := Queued{Total: int(q.Total), Counts: []QueueCount{}, Queue: []QueuedPublish{}, Listed: int(q.Listed), Acted: map[string]uint64{}, Items: []PublishItem{}}
if q.Capped {
out.capped(&StateCapped{State: PublishesLocal, KeysRead: int(q.ItemsRead), StateKeys: uint64(q.QueueItems), StateBytes: uint64(q.QueueBytes)})
}
for _, c := range q.Counts {
out.Counts = append(out.Counts, QueueCount{Holder: c.Holder, Class: c.Class, Count: int(c.Count), Oldest: c.Oldest, Newest: c.Newest})
}
for _, it := range q.Queue {
out.Queue = append(out.Queue, QueuedPublish{V: 1, Holder: it.Holder, Module: it.Module, Version: it.Version, Artifacts: it.Artifacts, Queued: it.Queued, Class: it.Class})
}
avals, c, err := acted.values(ActedState)
out.capped(c)
if err != nil {
@@ -637,13 +630,21 @@ func AnswerQueue(queue, acted, reasks Reader, module, version string) (Queued, e
}
// PublishQueueDescription is builder_publish_queue's description.
const PublishQueueDescription = "The versions the builder will publish again (its state " + PublishQueueState + ", design 47 G section 588, H H-A-290): " +
"how many each registry seat holder's queue holds per class (build-ended-this-run, sent first, or asked-again-at-start) with the oldest and newest " +
"queue time; the first 50 in the order they are sent, each with its class, its artifacts' digests, when it was queued and whether its publish has " +
"begun (module, and version, list only that module's, so one item's class can be read); the keys of the state " + ActedState + ", the last stream " +
const PublishQueueDescription = "The versions the builder will publish again (its work queue " + PublishesLocal + ", design 47 G section 588, H2 section 793): " +
"how many each registry seat holder's items hold per class (live, a build a pipeline's step waits on, pulled first, or again, asked again at a start) " +
"with the oldest and newest queue time; the first 50 in the order they are pulled, each with its class, its artifacts' digests and when it was " +
"queued (module, and version, list only that module's, so one item's class can be read); the keys of the state " + ActedState + ", the last stream " +
"sequence the builder acted on for each subject; and the publish items (design 47 G A-457a), counted, the first 50 by next try, each version's " +
"tries and next try, and why one ended after its tries. Each state is read in one page; a state past it is answered capped, naming the " +
"keys read and the state's size (H-A-293). Read-only."
"tries and next try, and why one ended (its tries spent, or dropped). A read past its pages is answered capped, naming the items read and the " +
"queue's size (H-A-293). Read-only."
// PublishDropDescription and PublishFirstDescription are builder_publish_drop's and builder_publish_first's (§793).
const PublishDropDescription = "Drop one version's publish (design 47 H2 section 793): its items on the work queue " + PublishesLocal + " are " +
"terminated at every holder, and its publish item ended with who dropped it and why, so it is never sent or put again; builder_publish_again " +
"asks it anew. A hand act, recorded in the hand-act log first."
const PublishFirstDescription = "Put one version's publish first (design 47 H2 section 793): each of its again items on the work queue " +
PublishesLocal + " is put again as live, under the message id <holder>.live.<module>.<version>, pulled before every again item, and the old " +
"terminated. A hand act, recorded in the hand-act log first."
// PublishAgainDescription is builder_publish_again's description.
const PublishAgainDescription = "Publish reserved versions again (design 47 G section 588), which builds nothing: with module and version, that reserved " +
@@ -652,17 +653,14 @@ const PublishAgainDescription = "Publish reserved versions again (design 47 G se
"with the versions it queued; each outcome is said as package-published, or left in version-unreleased with the holder's words. A hand act, " +
"recorded in the hand-act log."
// Resume takes up, at the builder's start, every queued version again, a begun one sent anew (§588): a version whose reservation
// was withdrawn or taken over is dropped with its publish item ended (H-A-290 (2)), and one of the live class whose build ended
// before this run is of the start's now (H-A-290 (1)). It reads the reservations once.
func (r *Republisher) Resume(ctx context.Context) error {
keys, err := r.Queue.KeysOf(ctx, QueueFilter)
if err != nil {
return fmt.Errorf("the state %s cannot be listed: %w", PublishQueueState, err)
}
// MoveQueued puts the values of the old state publish-queue on the work queue, once (§793): each under its message id, so a
// move run twice puts each once within the duplicates window; its class as of this run (a build ended before it is of the
// start's, H-A-290 (1)); a version whose reservation was withdrawn, taken over or superseded is not put, its publish item
// ended (H-A-290 (2), H-A-291 (1)). It reads the reservations once, and answers how many it put.
func (r *Republisher) MoveQueued(ctx context.Context, vals []json.RawMessage) (int, error) {
all, err := r.Reserved.All(ctx)
if err != nil {
return fmt.Errorf("the queued publishes are not taken up: %w", err)
return 0, fmt.Errorf("the queued publishes are not moved: %w", err)
}
reserved := make(map[string]ReservedVersion, len(all))
for _, v := range all {
@@ -670,16 +668,12 @@ func (r *Republisher) Resume(ctx context.Context) error {
}
retired, err := r.Reserved.Retired(ctx, all)
if err != nil {
r.logf("publish again: the superseded builds are not retired at the start: %v", err)
r.logf("publish again: the superseded builds are not retired at the move: %v", err)
}
holders := map[string]bool{}
for _, k := range keys {
raw, rev, found, err := r.Queue.Get(ctx, k)
if err != nil || !found {
continue
}
put := 0
for _, raw := range vals {
var q QueuedPublish
if json.Unmarshal(raw, &q) != nil {
if json.Unmarshal(raw, &q) != nil || q.Holder == "" {
continue
}
v, ok := reserved[q.Module+" "+q.Version]
@@ -688,126 +682,38 @@ func (r *Republisher) Resume(ctx context.Context) error {
why = retired[q.Module+" "+q.Version]
}
if why != "" {
if err := r.Queue.Delete(ctx, k); err != nil {
r.logf("%s cannot be deleted from %s: %v", k, PublishQueueState, err)
}
r.end(ctx, q.Module, q.Version, why)
continue
}
if class := r.classOf(v); q.Class != class {
q.Class = class
if raw, err := json.Marshal(q); err == nil {
if _, err := r.Queue.Update(ctx, k, raw, rev); err != nil {
r.logf("%s keeps its class %s at the start: %v", k, q.Class, err)
}
}
}
holders[q.Holder] = true
}
for h := range holders {
r.drainLater(ctx, h)
}
return nil
}
func (r *Republisher) drainLater(ctx context.Context, holder string) {
r.mu.Lock()
if r.busy == nil {
r.busy = map[string]bool{}
}
if r.busy[holder] {
r.mu.Unlock()
return
}
r.busy[holder] = true
r.mu.Unlock()
run := func() {
defer func() { r.mu.Lock(); delete(r.busy, holder); r.mu.Unlock() }()
r.Drain(ctx, holder)
}
if r.Go != nil {
r.Go(run)
return
}
go run()
}
// Drain sends holder's queued versions in passes (H-A-290 (3)): a pass lists the holder's keys once, reads each once, and sends
// them in their order (the live class first, each class oldest first), PublishSendsAtOnce at a time; the keys are listed again
// only when the pass is spent, or when a live publish was queued while it sends a start list: then the sends in flight end and
// the keys are listed again, so the live one goes next (H-A-291 (2)); a start-class one queued meanwhile waits for the pass's end.
// Each send is bounded by UnreleasedBound; a key is deleted when its outcome is said.
func (r *Republisher) Drain(ctx context.Context, holder string) {
for ctx.Err() == nil {
r.mu.Lock()
delete(r.relist, holder)
r.mu.Unlock()
keys, err := r.Queue.KeysOf(ctx, EscapeToken(holder)+".*.*")
q.Class = r.classOf(v)
dup, err := r.Put(ctx, q)
if err != nil {
r.logf("the queue of %s cannot be listed: %v", holder, err)
return
return put, fmt.Errorf("%s %s is not moved to the work queue %s: %w", q.Module, q.Version, PublishesLocal, err)
}
items := make([]QueuedPublish, 0, len(keys))
for _, k := range keys {
if raw, _, found, err := r.Queue.Get(ctx, k); err == nil && found {
var q QueuedPublish
if json.Unmarshal(raw, &q) == nil {
items = append(items, q)
}
}
if !dup {
put++
}
if len(items) == 0 {
return
}
sortQueued(items)
sem := make(chan struct{}, PublishSendsAtOnce)
var wg sync.WaitGroup
for _, q := range items {
if ctx.Err() != nil {
break
}
sem <- struct{}{}
r.mu.Lock()
again := q.Class != PublishClassLive && r.relist[holder]
r.mu.Unlock()
if again {
<-sem
r.logf("publish again: %s's pass lists again after its sends in flight: a live publish was queued", holder)
break
}
wg.Add(1)
go func() {
defer func() { <-sem; wg.Done() }()
r.send(ctx, q)
}()
}
wg.Wait()
}
return put, nil
}
// send publishes one queued version's artifacts and says the outcome, then deletes its key. Every call it makes honours the one
// bound it is given, UnreleasedBound (a context's bound, no timer, ADR 0437 §8), and a send that returns without sending says why
// (H-A-290 (4)).
func (r *Republisher) send(ctx context.Context, q QueuedPublish) {
key := QueueKey(q.Holder, q.Module, q.Version)
defer func() {
if err := r.Queue.Delete(context.WithoutCancel(ctx), key); err != nil {
r.logf("%s cannot be deleted from %s: %v", key, PublishQueueState, err)
}
}()
// SendFate is what SendQueued did with one item (§793): Sent, every artifact sent (its item acknowledged when its publish is
// heard or confirmed); Ended, why the version never publishes (its item terminated); Failed, the holder's or the call's words
// (its item handed back with the pause).
type SendFate struct {
Sent bool
Ended, Failed string
}
// SendQueued publishes one item's artifacts and says the outcome. Every call it makes honours the one bound it is given,
// UnreleasedBound (a context's bound, no timer, ADR 0437 §8), and a send that returns without sending says why (H-A-290 (4)).
func (r *Republisher) SendQueued(ctx context.Context, q QueuedPublish) SendFate {
bound, cancel := context.WithTimeout(ctx, UnreleasedBound)
defer cancel()
if !q.Begun {
q.Begun = true
raw, _ := json.Marshal(q)
if _, rev, found, err := r.Queue.Get(bound, key); err == nil && found {
_, _ = r.Queue.Update(bound, key, raw, rev)
}
}
all, err := r.Reserved.All(bound)
if err != nil {
r.logf("publish again: %s %s is not sent to %s: its reservation cannot be read: %v; its publish item asks it again", q.Module, q.Version, q.Holder, err)
return
r.logf("publish again: %s %s is not sent to %s: its reservation cannot be read: %v; it is handed back", q.Module, q.Version, q.Holder, err)
return SendFate{Failed: "its reservation cannot be read: " + err.Error()}
}
var v ReservedVersion
found := false
@@ -830,43 +736,64 @@ func (r *Republisher) send(ctx context.Context, q QueuedPublish) {
}
if why != "" {
r.end(ctx, q.Module, q.Version, why)
return
return SendFate{Ended: why}
}
e, err := r.Entry(bound, v.Fingerprint)
if err != nil {
r.say(ctx, v, fmt.Sprintf("the catalog entry %s cannot be read: %v", v.Fingerprint, err))
return
words := fmt.Sprintf("the catalog entry %s cannot be read: %v", v.Fingerprint, err)
r.say(ctx, v, words)
return SendFate{Failed: words}
}
missing := map[string]bool{}
for _, ref := range v.Missing() {
missing[ref] = true
}
refs := make([]string, 0, len(q.Artifacts))
for ref := range q.Artifacts {
if _, no := v.Unstored[ref]; no || !missing[ref] {
continue // published already, or refused as not stored: said once, never sent again
}
refs = append(refs, ref)
}
sort.Strings(refs)
if len(refs) == 0 {
return SendFate{Ended: "none of its artifacts is left to send: each is published or refused as not stored"}
}
var failed []string
sent := 0
for _, ref := range refs {
seat, name, kind, ok := publishTarget(e, q.Module, ref)
if !ok {
r.say(ctx, v, ref+": the catalog entry names no registry seat for it")
return
return SendFate{Failed: ref + ": the catalog entry names no registry seat for it"}
}
err := r.Send(bound, Publish{Seat: seat, Kind: kind, Name: name, Version: v.Version, Digest: q.Artifacts[ref], Fingerprint: v.Fingerprint,
Commit: v.Commit, Transfer: RepublishTransfer(q.Module, v.Version, strings.TrimPrefix(ref, q.Module+":"))})
if err == nil {
r.logf("publish again: %s %s: %s sent to %s as %s %s, digest %s; the package-published is said when the seat says published", q.Module, q.Version, ref, seat, name, v.Version, q.Artifacts[ref])
sent++
continue
}
if err != nil {
r.logf("publish again: %s %s: %s refused by %s: %v", q.Module, q.Version, ref, seat, err)
words := err.Error()
if strings.Contains(words, strings.TrimSuffix(NotStoredWords, ": ")) {
if ok, merr := r.Reserved.MarkUnstored(bound, q.Module, q.Version, ref, words); merr != nil {
r.logf("%s %s: the unstored %s is not kept: %v", q.Module, q.Version, ref, merr)
} else if ok {
r.say(ctx, v, ref+" "+NotStoredWords+words)
}
continue
r.logf("publish again: %s %s: %s refused by %s: %v", q.Module, q.Version, ref, seat, err)
words := err.Error()
if strings.Contains(words, strings.TrimSuffix(NotStoredWords, ": ")) {
if ok, merr := r.Reserved.MarkUnstored(bound, q.Module, q.Version, ref, words); merr != nil {
r.logf("%s %s: the unstored %s is not kept: %v", q.Module, q.Version, ref, merr)
} else if ok {
r.say(ctx, v, ref+" "+NotStoredWords+words)
}
r.say(ctx, v, ref+": "+words)
continue
}
r.say(ctx, v, ref+": "+words)
failed = append(failed, ref+": "+words)
}
switch {
case len(failed) > 0:
return SendFate{Failed: strings.Join(failed, "; ")}
case sent == 0:
return SendFate{Ended: "each artifact left was refused as not stored"}
}
return SendFate{Sent: true}
}
// publishTarget is the registry seat, the name and the kind module's artifact ref is published as, from its catalog entry; ok is
@@ -9,51 +9,41 @@ import (
"slices"
"strings"
"sync"
"sync/atomic"
"testing"
"testing/synctest"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
)
type queueMem struct {
*reserveMem
bytes uint64
// listings counts the listings of one holder's keys (a filter `<holder>.*.*`), what a drain's pass lists.
listings atomic.Int32
}
func (q *queueMem) Bytes(context.Context) (uint64, error) { return q.bytes, nil }
func (q *queueMem) KeysOf(ctx context.Context, filter string) ([]string, error) {
if strings.HasSuffix(filter, ".*.*") && filter != QueueFilter {
q.listings.Add(1)
}
return q.reserveMem.KeysOf(ctx, filter)
}
// republishBed is a Republisher over memory states: reservations of module m (a bundle by the oci seat and a Go package by
// the Go seat), one holder module `oci` that claims the oci seat, and a Send and a Unreleased that record.
// the Go seat), one holder module `oci` that claims the oci seat, a work queue that records its puts (deduplicating by message
// id when window is set, refusing when full) and sends each put at once as its pull would, and a Send and a Unreleased that
// record.
type republishBed struct {
*Republisher
reserved *reserveMem
queue *queueMem
now time.Time
mu sync.Mutex // sent and said, which a holder's sends at once write
mu sync.Mutex // sent and said, which sends at once write
sent []Publish
said []string
ended []string
refuse string // the words Send fails with, empty for none
// holders are the registry seats' holders the seat-holders view names (design 47 G A-416).
holders map[string][]string
// puts are the items the work queue took; window deduplicates by message id; full refuses every put; held keeps the puts
// unsent.
puts []QueuedPublish
ids map[string]bool
window, full bool
held bool
}
func newRepublishBed(t *testing.T) *republishBed {
t.Helper()
b := &republishBed{reserved: &reserveMem{mem: newMem()}, queue: &queueMem{reserveMem: &reserveMem{mem: newMem()}}, now: time.Date(2026, 11, 11, 12, 0, 0, 0, time.UTC)}
b := &republishBed{reserved: &reserveMem{mem: newMem()}, now: time.Date(2026, 11, 11, 12, 0, 0, 0, time.UTC), ids: map[string]bool{}}
res := &Reserver{Store: b.reserved, Now: func() time.Time { return b.now }, ReleasedOf: func(context.Context, string) ([]string, error) { return nil, nil }}
b.Republisher = &Republisher{Reserved: res, Queue: b.queue, Now: func() time.Time { return b.now }, Go: func(f func()) { f() },
b.Republisher = &Republisher{Reserved: res, Now: func() time.Time { return b.now },
Entry: func(_ context.Context, fp string) (meshcatalog.CatalogEntry, error) {
if fp == "src2:holder" {
return meshcatalog.CatalogEntry{Serves: []string{SeatArtifacts + ".publish", SeatArtifacts + ".versions"}}, nil
@@ -83,6 +73,21 @@ func newRepublishBed(t *testing.T) *republishBed {
b.ended = append(b.ended, module+" "+version)
},
Logf: t.Logf}
b.Put = func(ctx context.Context, q QueuedPublish) (bool, error) {
if b.full {
return false, fmt.Errorf("%w: maximum bytes exceeded", ErrQueueFull)
}
id := PublishMsgID(q.Holder, q.Class, q.Module, q.Version)
if b.window && b.ids[id] {
return true, nil
}
b.ids[id] = true
b.puts = append(b.puts, q)
if !b.held {
b.SendQueued(ctx, q)
}
return false, nil
}
b.Holders = func(seat string) []string { return b.holders[seat] }
return b
}
@@ -125,8 +130,8 @@ func TestRepublish(t *testing.T) {
p.Transfer != "republish-m-"+strings.ReplaceAll(v.Version, ".", "-")+"-prog" || strings.ContainsAny(p.Transfer, "/. *>") {
t.Errorf("sent %+v, want the bundle by digest to the oci seat, the Go package left to its own seat's holder", p)
}
if keys, _ := b.queue.KeysOf(ctx, AllFilter); len(keys) != 0 {
t.Errorf("a key stays in the queue after its outcome: %v", keys)
if len(b.puts) != 1 || b.puts[0].Holder != "oci" || b.puts[0].Class != PublishClassLive {
t.Errorf("the work queue took %+v, want one live item at oci", b.puts)
}
})
t.Run("an entry that cannot be read stops neither the other versions nor the drain", func(t *testing.T) {
@@ -173,22 +178,42 @@ func TestRepublish(t *testing.T) {
t.Run("a queue past its cap says queue full", func(t *testing.T) {
b := newRepublishBed(t)
b.reserve(t, "m")
b.queue.bytes = PublishQueueMiB << 20
b.full = true
b.holders = map[string][]string{SeatArtifacts: {"oci"}}
_, _ = b.Republish(ctx, "", "")
if len(b.sent) != 0 || len(b.said) != 1 || !strings.HasSuffix(b.said[0], QueueFull) {
t.Errorf("sent %d, said %v, want nothing sent and queue full said", len(b.sent), b.said)
}
})
t.Run("a restart sends the queued again, a begun one anew", func(t *testing.T) {
t.Run("an item put twice within the duplicates window is put once, and stands queued", func(t *testing.T) {
b := newRepublishBed(t)
v := b.reserve(t, "m")
raw, _ := json.Marshal(QueuedPublish{V: 1, Holder: "oci", Module: "m", Version: v.Version, Artifacts: map[string]string{"m:prog": "sha256:aa"}, Queued: b.now, Begun: true})
if _, err := b.queue.Create(ctx, QueueKey("oci", "m", v.Version), raw); err != nil {
b.window, b.held = true, true
b.holders = map[string][]string{SeatArtifacts: {"oci"}}
q1, _ := b.Republish(ctx, "m", v.Version)
q2, _ := b.Republish(ctx, "m", v.Version)
if len(b.puts) != 1 || len(q1) != 1 || len(q2) != 1 {
t.Errorf("puts %d, queued %v then %v: want one item, queued both times", len(b.puts), q1, q2)
}
})
t.Run("the move puts each item of the old state once, its class as of this run, a superseded one not", func(t *testing.T) {
b := newRepublishBed(t)
v := b.reserve(t, "m")
w := b.reserve(t, "n")
b.Started = b.now
b.window, b.held = true, true
raw, _ := json.Marshal(QueuedPublish{V: 1, Holder: "oci", Module: "m", Version: v.Version, Artifacts: map[string]string{"m:prog": "sha256:aa"}, Queued: b.now, Class: "build-ended-this-run"})
gone, _ := json.Marshal(QueuedPublish{V: 1, Holder: "oci", Module: "n", Version: w.Version, Artifacts: map[string]string{"n:prog": "sha256:aa"}, Queued: b.now})
if _, _, err := b.Reserved.Withdraw(ctx, "n", w.Version); err != nil {
t.Fatal(err)
}
if err := b.Resume(ctx); err != nil || len(b.sent) != 1 {
t.Errorf("resume sent %d (%v), want the begun version sent anew", len(b.sent), err)
for range 2 {
if _, err := b.MoveQueued(ctx, []json.RawMessage{raw, gone}); err != nil {
t.Fatal(err)
}
}
if len(b.puts) != 1 || b.puts[0].Class != PublishClassStart || fmt.Sprint(b.ended) != "[n "+w.Version+" n "+w.Version+"]" {
t.Errorf("puts %+v, ended %v: want m once as again (its build ended before this run), n's item ended", b.puts, b.ended)
}
})
t.Run("after a fresh start the tool queues at the seat's holder, no running heard", func(t *testing.T) {
@@ -215,7 +240,7 @@ func TestRepublish(t *testing.T) {
return json.RawMessage(`{"mode":"publish","module":"m","version":"0.1.0","tries":3,"next-try":"2026-11-17T01:00:00Z","why":"w"}`), true, nil
}}
none := Reader{Keys: func() ([]string, error) { return nil, nil }}
answer, err := AnswerQueue(none, none, items, "", "")
answer, err := answerOf(none, none, items, "", "")
if err != nil || len(answer.Items) != 1 || answer.Items[0].Tries != 3 || answer.Items[0].Version != "0.1.0" {
t.Errorf("builder_publish_queue items = %+v (%v), want the one publish item with its tries", answer.Items, err)
}
@@ -262,98 +287,31 @@ func TestTheRegistriesRecordSaysWhetherAVersionIsHeld(t *testing.T) {
}
// queueAt writes a queued version of module m at holder oci directly, as a start finds it.
func (b *republishBed) queueAt(t *testing.T, v ReservedVersion, class string, at time.Time) {
t.Helper()
raw, _ := json.Marshal(QueuedPublish{V: 1, Holder: "oci", Module: v.Module, Version: v.Version, Artifacts: map[string]string{v.Module + ":prog": "sha256:aa"}, Queued: at, Class: class})
if _, err := b.queue.Create(context.Background(), QueueKey("oci", v.Module, v.Version), raw); err != nil {
t.Fatal(err)
}
func (b *republishBed) queueAt(v ReservedVersion, class string, at time.Time) QueuedPublish {
return QueuedPublish{V: 1, Holder: "oci", Module: v.Module, Version: v.Version, Artifacts: map[string]string{v.Module + ":prog": "sha256:aa"}, Queued: at, Class: class}
}
// TestAHoldersQueueSendsLiveFirstInPasses: design 47 H H-A-290 (1)-(3) — a publish whose build ended in this run is sent before
// any of a build that ended before the start, each class oldest first; a pass lists the holder's keys once and sends them, at most
// four at a time; a build whose reservation was withdrawn or taken over never publishes, its key dropped and its publish item ended,
// at enqueue, at the pass and at the start.
func TestAHoldersQueueSendsLiveFirstInPasses(t *testing.T) {
// TestAQueuedItemIsSentWhileItsBuildStands: design 47 H H-A-290 (2), H-A-291 (1), H2 §793 — an item is put at the holder in the
// class of its build (live when it ended in this run); a build whose reservation was withdrawn, taken over or superseded by a newer
// patch or release of its series never publishes, its publish item ended, at the put and at the send; an item whose artifacts are
// all published or refused as not stored ends unsent.
func TestAQueuedItemIsSentWhileItsBuildStands(t *testing.T) {
ctx := context.Background()
t.Run("the live class first, at most four sends at once", func(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
b := newRepublishBed(t)
var old []ReservedVersion
for _, m := range []string{"o1", "o2", "o3", "o4", "o5"} {
old = append(old, b.reserve(t, m))
}
b.Started = b.now
b.reserve(t, "live")
for i, v := range old {
b.queueAt(t, v, PublishClassStart, b.now.Add(time.Duration(i)*time.Second))
}
b.now = b.now.Add(time.Hour) // the live one is queued last, after every old one
b.holders = map[string][]string{SeatArtifacts: {"oci"}}
release := make(chan struct{})
var arrived []string
var inflight, most int
send := b.Send
b.Send = func(ctx context.Context, p Publish) error {
b.mu.Lock()
arrived = append(arrived, p.Name)
inflight++
most = max(most, inflight)
b.mu.Unlock()
<-release
b.mu.Lock()
inflight--
b.mu.Unlock()
return send(ctx, p)
}
b.Go = func(f func()) { go f() }
if q, err := b.Republish(ctx, "live", ""); err != nil || len(q) != 1 {
t.Fatalf("queued %v (%v), want the live version", q, err)
}
synctest.Wait() // every send the bound lets run waits on release
b.mu.Lock()
first := slices.Clone(arrived)
b.mu.Unlock()
if len(first) != PublishSendsAtOnce || !slices.Contains(first, "live/prog") {
t.Errorf("the sends running at once are %v: want %d, the live one among them", first, PublishSendsAtOnce)
}
close(release)
synctest.Wait()
if len(b.sent) != 6 || most != PublishSendsAtOnce {
t.Errorf("sent %d, at most %d at once: want all six, %d at once", len(b.sent), most, PublishSendsAtOnce)
}
})
})
t.Run("a pass lists the keys once for all its sends", func(t *testing.T) {
t.Run("a build that ended before this run is put as again, one after it as live", func(t *testing.T) {
b := newRepublishBed(t)
var vs []ReservedVersion
for i := range 6 {
vs = append(vs, b.reserve(t, fmt.Sprintf("m%d", i)))
}
for i, v := range vs {
b.queueAt(t, v, PublishClassStart, b.now.Add(time.Duration(i)*time.Second))
}
b.Drain(ctx, "oci")
if n := b.queue.listings.Load(); len(b.sent) != 6 || n != 2 {
t.Errorf("sent %d with %d listings, want six sends from one pass and one listing that finds it spent", len(b.sent), n)
}
})
t.Run("a start makes a live key of a build before it a start key", func(t *testing.T) {
b := newRepublishBed(t)
v := b.reserve(t, "m")
old := b.reserve(t, "o")
b.Started = b.now
b.queueAt(t, v, PublishClassLive, b.now)
b.Go = func(func()) {} // the start's drain is not run here
if err := b.Resume(ctx); err != nil {
t.Fatal(err)
}
raw, _, _, _ := b.queue.Get(ctx, QueueKey("oci", "m", v.Version))
var q QueuedPublish
if json.Unmarshal(raw, &q) != nil || q.Class != PublishClassStart {
t.Errorf("the key after the start is %s, want the class %s: its build did not end in this run", raw, PublishClassStart)
live := b.reserve(t, "l")
b.held = true
b.holders = map[string][]string{SeatArtifacts: {"oci"}}
_, _ = b.Republish(ctx, "o", old.Version)
_, _ = b.Republish(ctx, "l", live.Version)
if len(b.puts) != 2 || b.puts[0].Class != PublishClassStart || b.puts[1].Class != PublishClassLive ||
PublishEvent("oci", PublishClassLive, "l", live.Version) != "publish.oci.live.l.0=2E1=2E0" {
t.Errorf("puts %+v, want o again and l live, on publish.<holder>.<class>.<module>.<version>", b.puts)
}
})
for _, at := range []string{"enqueue", "pass", "start"} {
for _, at := range []string{"put", "send"} {
t.Run("a withdrawn build never publishes, at the "+at, func(t *testing.T) {
b := newRepublishBed(t)
v := b.reserve(t, "m")
@@ -361,105 +319,34 @@ func TestAHoldersQueueSendsLiveFirstInPasses(t *testing.T) {
if _, ok, err := b.Reserved.Withdraw(ctx, "m", v.Version); err != nil || !ok {
t.Fatalf("withdrawn %v (%v)", ok, err)
}
switch at {
case "enqueue":
b.queueAt(t, v, PublishClassLive, b.now)
b.Go = func(func()) {} // a key queued before the withdrawal is dropped by the enqueue itself
if at == "put" {
if q, _ := b.Republish(ctx, "m", v.Version); len(q) != 0 {
t.Errorf("queued %v", q)
}
case "pass":
b.queueAt(t, v, PublishClassLive, b.now)
b.Drain(ctx, "oci")
case "start":
b.queueAt(t, v, PublishClassLive, b.now)
b.Go = func(func()) {}
if err := b.Resume(ctx); err != nil {
t.Fatal(err)
}
} else if f := b.SendQueued(ctx, b.queueAt(v, PublishClassLive, b.now)); f.Ended == "" {
t.Errorf("the send of a withdrawn build answered %+v, want it ended", f)
}
keys, _ := b.queue.KeysOf(ctx, QueueFilter)
if len(b.sent) != 0 || len(keys) != 0 || fmt.Sprint(b.ended) != "[m "+v.Version+"]" {
t.Errorf("sent %v, keys %v, items ended %v: want nothing sent, the key dropped and m %s's item ended", b.sent, keys, b.ended, v.Version)
if len(b.sent) != 0 || len(b.puts) != 0 || fmt.Sprint(b.ended) != "[m "+v.Version+"]" {
t.Errorf("sent %v, puts %v, items ended %v: want nothing sent or put and m %s's item ended", b.sent, b.puts, b.ended, v.Version)
}
})
}
t.Run("a live publish queued mid-pass goes after the sends in flight, before the rest of the start list", func(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
b := newRepublishBed(t)
var old []ReservedVersion
for _, m := range []string{"o1", "o2", "o3", "o4", "o5", "o6"} {
old = append(old, b.reserve(t, m))
}
b.Started = b.now
b.reserve(t, "live")
for i, v := range old {
b.queueAt(t, v, PublishClassStart, b.now.Add(time.Duration(i)*time.Second))
}
b.now = b.now.Add(time.Hour)
b.holders = map[string][]string{SeatArtifacts: {"oci"}}
first, then := make(chan struct{}), make(chan struct{}) // the four in flight wait on first, every later send on then
var arrived []string
send := b.Send
b.Send = func(ctx context.Context, p Publish) error {
b.mu.Lock()
arrived = append(arrived, p.Name)
gate := map[bool]chan struct{}{true: first, false: then}[len(arrived) <= PublishSendsAtOnce]
b.mu.Unlock()
<-gate
return send(ctx, p)
}
b.Go = func(f func()) { go f() }
b.drainLater(ctx, "oci")
synctest.Wait() // four start sends in flight, the pass holding o5 and o6
if q, err := b.Republish(ctx, "live", ""); err != nil || len(q) != 1 {
t.Fatalf("queued %v (%v)", q, err)
}
close(first)
synctest.Wait() // the sends after the four: with the live one among them, or o5 and o6 alone when the pass goes on
b.mu.Lock()
next := slices.Clone(arrived[PublishSendsAtOnce:])
b.mu.Unlock()
if !slices.Contains(next, "live/prog") {
t.Errorf("after the four in flight the sends are %v, want the live one among them, the pass listed again", next)
}
close(then)
synctest.Wait()
if len(arrived) != 7 {
t.Errorf("sent %v, want all seven", arrived)
}
})
})
t.Run("a build a newer patch or release of its series supersedes is retired at the pass and at the start, never by a newer series", func(t *testing.T) {
t.Run("a build a newer patch or release of its series supersedes is ended at the send, never by a newer series", func(t *testing.T) {
b := newRepublishBed(t)
v := b.reserve(t, "m")
b.holders = map[string][]string{SeatArtifacts: {"oci"}}
if _, err := b.Reserved.Reserve(ctx, "m", "0.2", []string{"m:prog"}, Source{Fingerprint: "src2:m"}); err != nil {
t.Fatal(err)
}
b.queueAt(t, v, PublishClassStart, b.now)
b.Drain(ctx, "oci")
if len(b.sent) != 1 || len(b.ended) != 0 {
t.Errorf("with a standing 0.2: sent %v, ended %v, want m %s sent, its series not superseded (H-A-293)", b.sent, b.ended, v.Version)
if f := b.SendQueued(ctx, b.queueAt(v, PublishClassStart, b.now)); !f.Sent || len(b.sent) != 1 || len(b.ended) != 0 {
t.Errorf("with a standing 0.2: %+v, sent %v, ended %v, want m %s sent, its series not superseded (H-A-293)", f, b.sent, b.ended, v.Version)
}
rel := &reserveMem{mem: newMem()}
raw, _ := json.Marshal(ReleasedRecord{Kept: []ReleasedBuild{{Fingerprint: "f", Version: "0.2.3"}, {Fingerprint: "g", Version: "0.1.9"}}})
_, _ = rel.Create(ctx, ReleasedKey("m"), raw)
b.Reserved.Releases = rel
b.sent = nil
b.queueAt(t, v, PublishClassStart, b.now)
b.Drain(ctx, "oci")
if len(b.sent) != 0 || fmt.Sprint(b.ended) != "[m "+v.Version+"]" {
t.Errorf("at the pass: sent %v, ended %v, want m %s retired by the release 0.1.9", b.sent, b.ended, v.Version)
}
b.ended = nil
b.queueAt(t, v, PublishClassStart, b.now)
b.Go = func(func()) {}
if err := b.Resume(ctx); err != nil {
t.Fatal(err)
}
if keys, _ := b.queue.KeysOf(ctx, QueueFilter); len(keys) != 0 || fmt.Sprint(b.ended) != "[m "+v.Version+"]" {
t.Errorf("at the start: keys %v, ended %v, want m %s retired", keys, b.ended, v.Version)
if f := b.SendQueued(ctx, b.queueAt(v, PublishClassStart, b.now)); f.Ended == "" || len(b.sent) != 0 || fmt.Sprint(b.ended) != "[m "+v.Version+"]" {
t.Errorf("at the send: %+v, sent %v, ended %v, want m %s retired by the release 0.1.9", f, b.sent, b.ended, v.Version)
}
only := &reserveMem{mem: newMem()}
raw, _ = json.Marshal(ReleasedRecord{Kept: []ReleasedBuild{{Fingerprint: "f", Version: "0.2.0"}}})
@@ -473,15 +360,47 @@ func TestAHoldersQueueSendsLiveFirstInPasses(t *testing.T) {
t.Run("a version taken over is not sent, and its item ended", func(t *testing.T) {
b := newRepublishBed(t)
v := b.reserve(t, "m")
b.queueAt(t, v, PublishClassStart, b.now)
if _, err := b.Reserved.Reserve(ctx, "m", "0.1", []string{"m:prog"}, Source{Fingerprint: "src2:other"}); err != nil {
t.Fatal(err)
}
b.Drain(ctx, "oci")
if len(b.sent) != 0 || fmt.Sprint(b.ended) != "[m "+v.Version+"]" {
t.Errorf("sent %v, ended %v: want nothing sent and the item ended", b.sent, b.ended)
if f := b.SendQueued(ctx, b.queueAt(v, PublishClassStart, b.now)); f.Ended == "" || len(b.sent) != 0 || fmt.Sprint(b.ended) != "[m "+v.Version+"]" {
t.Errorf("%+v, sent %v, ended %v: want nothing sent and the item ended", f, b.sent, b.ended)
}
})
t.Run("an item whose artifacts are published or refused as not stored ends unsent", func(t *testing.T) {
b := newRepublishBed(t)
v := b.reserve(t, "m")
b.refuse = "not stored: no blob"
if f := b.SendQueued(ctx, b.queueAt(v, PublishClassLive, b.now)); f.Ended == "" || len(b.sent) != 1 {
t.Errorf("a not stored refusal answered %+v, sent %d: want it ended after one send", f, len(b.sent))
}
if f := b.SendQueued(ctx, b.queueAt(v, PublishClassLive, b.now)); f.Ended == "" || len(b.sent) != 1 {
t.Errorf("again: %+v, sent %d: want it ended unsent", f, len(b.sent))
}
b.refuse = "the holder is full"
w := b.reserve(t, "n")
if f := b.SendQueued(ctx, b.queueAt(w, PublishClassLive, b.now)); f.Failed == "" || f.Sent {
t.Errorf("a refusal in other words answered %+v, want it failed, handed back", f)
}
})
}
// answerOf is builder_publish_queue whole: the builder's verb publish-queue over its work queue (queue), then the tool over the
// verb's answer and the two states (design 47 H2 §793, H-A-454).
func answerOf(queue, acted, reasks Reader, module, version string) (Queued, error) {
q, err := PublishQueueOf(queue, module, version)
if err != nil {
return Queued{}, err
}
raw, err := json.Marshal(q) // the verb's answer travels the bus
if err != nil {
return Queued{}, err
}
var heard meshbuilds.PublishQueueAnswer
if err := json.Unmarshal(raw, &heard); err != nil {
return Queued{}, err
}
return AnswerQueue(heard, acted, reasks)
}
// TestThePublishQueueToolAnswersAtAnySize: design 47 H H-A-290 (5) — builder_publish_queue over two thousand queued versions answers
@@ -507,7 +426,7 @@ func TestThePublishQueueToolAnswersAtAnySize(t *testing.T) {
Get: func(k string) (json.RawMessage, bool, error) { gets++; v, ok := vals[k]; return v, ok, nil }}
none := Reader{Keys: func() ([]string, error) { return nil, nil }}
start := time.Now()
a, err := AnswerQueue(queue, none, none, "", "")
a, err := answerOf(queue, none, none, "", "")
took := time.Since(start)
raw, _ := json.Marshal(a)
if err != nil || took > time.Second || a.Total != 2000 || len(a.Queue) != PublishQueueListCap || a.Listed != PublishQueueListCap || len(raw) > 64<<10 || gets != 0 {
@@ -519,7 +438,7 @@ func TestThePublishQueueToolAnswersAtAnySize(t *testing.T) {
if a.Queue[0].Class != PublishClassLive || a.Queue[0].Holder != "go" {
t.Errorf("the list opens with %+v, want the go holder's live class, sent first", a.Queue[0])
}
one, err := AnswerQueue(queue, none, none, "m1995", "")
one, err := answerOf(queue, none, none, "m1995", "")
if err != nil || len(one.Queue) != 1 || one.Queue[0].Class != PublishClassLive || one.Total != 2000 {
t.Errorf("one module's answer %+v (%v), want its item with its class", one.Queue, err)
}
@@ -527,12 +446,12 @@ func TestThePublishQueueToolAnswersAtAnySize(t *testing.T) {
t.Errorf("a state one page holds is answered capped: %+v", a.Read)
}
queue.All = func() (StatePage, error) { return StatePage{Values: vals, More: true, Keys: 5000, Bytes: 3 << 20}, nil }
if c, err := AnswerQueue(queue, none, none, "", ""); err != nil || !c.Capped || len(c.Read) != 1 || c.Read[0].State != PublishQueueState ||
if c, err := answerOf(queue, none, none, "", ""); err != nil || !c.Capped || len(c.Read) != 1 || c.Read[0].State != PublishesLocal ||
c.Read[0].KeysRead != 2000 || c.Read[0].StateKeys != 5000 || c.Read[0].StateBytes != 3<<20 || c.Total != 2000 {
t.Errorf("a state past its one page answered capped %v, read %+v (%v): want capped, 2000 keys read of 5000, its size said (H-A-293)", c.Capped, c.Read, err)
}
queue.All = func() (StatePage, error) { return StatePage{}, errors.ErrUnsupported }
if b, err := AnswerQueue(queue, none, none, "", ""); err != nil || b.Total != 2000 || gets != 2000 {
if b, err := answerOf(queue, none, none, "", ""); err != nil || b.Total != 2000 || gets != 2000 {
t.Errorf("a reader with no bulk read answered %d (%v) with %d single reads, want every key read one by one", b.Total, err, gets)
}
}
@@ -10,8 +10,10 @@ import (
"sort"
"strconv"
"strings"
"sync"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
)
@@ -40,6 +42,9 @@ const (
// VersionHeldElsewhere is the finding of a number a registry seat already holds, skipped at reserving or moved from (design 47
// H H-A-275), scope module.
VersionHeldElsewhere = "version-held-elsewhere"
// VersionTaggedElsewhere is the finding of a number whose git tag stands at another commit, skipped at reserving (design 47
// H2 §796), scope module; cleared at the module's next version-released.
VersionTaggedElsewhere = "version-tagged-elsewhere"
)
// ReservedVersion is the value of a reservation key.
@@ -59,7 +64,7 @@ type ReservedVersion struct {
PublishedAt map[string]time.Time `json:"published-at,omitempty"`
At time.Time `json:"at"`
// Repository and Commit are what the build was made from, and Tag the git tag its release is written as (a Go
// package's, GoTagOf): empty for a module with no Go package (design 47 G §370).
// package's, GoTagOf): empty for a module with no Go package (design 47 G §370), or whose Go package's directory is not known.
Repository string `json:"repository,omitempty"`
Commit string `json:"commit,omitempty"`
Tag string `json:"tag,omitempty"`
@@ -112,6 +117,9 @@ type Source struct {
// Fingerprint is the fingerprint of the catalog entry the build is made from (design 47 G A-76).
Fingerprint string
GoPackage bool
// GoArtifact is the artifact that is the module's Go package, whose module path names the directory its tag is written in
// (design 47 H2 §796).
GoArtifact string
// Branch is the watched branch of the catalog revision whose entry the build came from, and Revision the source revision of
// that catalog revision (design 47 G §371, A-57), kept on the reservation for the publication.
Branch string
@@ -145,6 +153,9 @@ type Reserver struct {
// Took is told when a build took over the series' reservation of an older build never released (design 47 G A-455): the
// older reservation, which no want follows any more, is gone from the key, replaced by the newer's. Nil is told none.
Took func(ctx context.Context, older, newer ReservedVersion)
// Reserved is told each reservation written, a number newly reserved (never one kept as it stood), so the holder says
// version-reserved of its packages (design 47 H2 §805). Nil is told none.
Reserved func(ctx context.Context, v ReservedVersion)
// Releases is the builder's own record of the versions it saw released, by module and fingerprint (design 47 G A-456a): the
// reservation is deleted at release, so a rerun of a released build's step reads it here, never from the catalog. Nil keeps none.
Releases ReserveStore
@@ -160,6 +171,92 @@ type Reserver struct {
HeldNumbers func(ctx context.Context, module, fingerprint string, refs []string) ([]string, error)
// HeldElsewhere is told the numbers of module a reserve skipped because a registry holds them (H-A-275); nil is told none.
HeldElsewhere func(ctx context.Context, module string, skipped []string)
// Names answers a Go package's module path, which names the directory its tag is written in (design 47 H2 §796); nil, or a
// name not known, tags the version nowhere, said.
Names PackageNames
// Tags reads repository's tags starting with prefix, each with the commit it names, through the seat git's tags (design 47
// H2 §796); away says the read met a transport fault or the error kind away, which lets the reservation go on, said. Nil
// reads none.
Tags func(ctx context.Context, repository, prefix string) (tags []git.Tag, away bool, err error)
// TaggedElsewhere is told the numbers of module a reserve skipped because their tag stands at another commit (§796); nil
// is told none.
TaggedElsewhere func(ctx context.Context, module string, skipped []string)
tagsMu sync.Mutex
// tagsRead are the tags last read of each module, by series patch to the commit, with the catalog revision they were read
// under: a module's tags are read once per catalog revision it reserves under (§796).
tagsRead map[string]tagsAt
}
// tagsAt are a module's tags of one series read under one catalog revision: patch to the commit its tag names.
type tagsAt struct {
revision, prefix string
commits map[int]string
}
// goDirectory is the directory of module's Go package in its repository (§796), from its module path; ok is false, said, when
// no name or no directory of the repository is known.
func (r *Reserver) goDirectory(ctx context.Context, module string, src Source) (string, bool) {
if r.Names == nil {
return "", false
}
path, known, err := r.Names(ctx, module, src.GoArtifact)
switch {
case err != nil:
r.said(fmt.Errorf("%s: its Go package's name cannot be read, so its version is tagged nowhere: %w", module, err))
return "", false
case !known:
r.said(fmt.Errorf("%s: no name of its Go package %s is known yet, so its version is tagged nowhere", module, src.GoArtifact))
return "", false
}
dir, ok := GoDirectory(path, src.Repository)
if !ok {
r.said(fmt.Errorf("%s: its Go package %s is not a module path of the repository %s, so its version is tagged nowhere", module, path, src.Repository))
}
return dir, ok
}
// taggedAt are the patches of series whose tag `<directory>/v<series>.<patch>` stands, each with its commit, read once per
// catalog revision the module reserves under (§796). A read that meets a transport fault or away answers none, said, and the
// reservation goes on; any other failure refuses the reservation, naming it.
func (r *Reserver) taggedAt(ctx context.Context, module, series, dir string, src Source) (map[int]string, error) {
if r.Tags == nil {
return nil, nil
}
prefix := TagPrefix(dir) + series + "."
revision := ""
if src.CatalogRevision > 0 {
revision = fmt.Sprintf("%s.%d", src.Branch, src.CatalogRevision)
r.tagsMu.Lock()
got, ok := r.tagsRead[module]
r.tagsMu.Unlock()
if ok && got.revision == revision && got.prefix == prefix {
return got.commits, nil
}
}
tags, away, err := r.Tags(ctx, src.Repository, prefix)
if err != nil && away {
r.said(fmt.Errorf("%s: its tags %s* cannot be read (%w), so the reservation goes on without them", module, prefix, err))
return nil, nil
}
if err != nil {
return nil, fmt.Errorf("the tags %s* of %s cannot be read: %w", prefix, module, err)
}
commits := map[int]string{}
for _, t := range tags {
if s, p, err := SplitVersion(strings.TrimPrefix(t.Tag, TagPrefix(dir))); err == nil && s == series && strings.HasPrefix(t.Tag, prefix) {
commits[p] = t.Commit
}
}
if revision != "" {
r.tagsMu.Lock()
if r.tagsRead == nil {
r.tagsRead = map[string]tagsAt{}
}
r.tagsRead[module] = tagsAt{revision: revision, prefix: prefix, commits: commits}
r.tagsMu.Unlock()
}
return commits, nil
}
// heldNumbers are the patches of series a registry holds of module's artifacts (H-A-275); a read that fails is said, and none.
@@ -180,15 +277,20 @@ func (r *Reserver) heldNumbers(ctx context.Context, module, series, fingerprint
return out
}
// nextFree is the first patch after after that no registry holds, and the held numbers it skipped (H-A-275).
func nextFree(series string, after int, held map[int]bool) (int, []string) {
var skipped []string
p := after + 1
for held[p] {
skipped = append(skipped, fmt.Sprintf("%s.%d", series, p))
p++
// nextFree is the first patch after after that no registry holds and whose tag stands at no other commit than commit, with the
// held numbers it skipped (H-A-275) and those tagged elsewhere (design 47 H2 §796).
func nextFree(series string, after int, held map[int]bool, tagged map[int]string, commit string) (int, []string, []string) {
var skipped, elsewhere []string
for p := after + 1; ; p++ {
switch c, ok := tagged[p]; {
case held[p]:
skipped = append(skipped, fmt.Sprintf("%s.%d", series, p))
case ok && c != commit:
elsewhere = append(elsewhere, fmt.Sprintf("%s.%d (tagged at %s)", series, p, c))
default:
return p, skipped, elsewhere
}
}
return p, skipped
}
// said is told the outcome of a reindex: a failure is said, the reservation's own write stands.
@@ -240,6 +342,15 @@ func (r *Reserver) Reserve(ctx context.Context, module, series string, artifacts
}
key := ReservedKey(module, series)
held := r.heldNumbers(ctx, module, series, src.Fingerprint, artifacts)
dir, tagged := "", false
var tags map[int]string
if src.GoPackage {
if dir, tagged = r.goDirectory(ctx, module, src); tagged {
if tags, err = r.taggedAt(ctx, module, series, dir, src); err != nil {
return ReservedVersion{}, err
}
}
}
for {
cur, rev, found, err := r.get(ctx, key)
if err != nil {
@@ -257,11 +368,11 @@ func (r *Reserver) Reserve(ctx context.Context, module, series string, artifacts
next = p
}
}
patch, skipped := nextFree(series, next, held)
patch, skipped, elsewhere := nextFree(series, next, held, tags, src.Commit)
out := ReservedVersion{V: 1, Module: module, Series: series, Version: fmt.Sprintf("%s.%d", series, patch), Artifacts: map[string]string{}, At: r.now(),
Repository: src.Repository, Commit: src.Commit, Fingerprint: src.Fingerprint, Branch: src.Branch, Source: src.Revision, Revision: src.CatalogRevision}
if src.GoPackage {
out.Tag = GoTagOf(out.Version)
if tagged {
out.Tag = GoTagOf(dir, out.Version)
}
for _, a := range artifacts {
out.Artifacts[a] = ""
@@ -299,9 +410,15 @@ func (r *Reserver) Reserve(ctx context.Context, module, series string, artifacts
if len(skipped) > 0 && r.HeldElsewhere != nil {
r.HeldElsewhere(ctx, module, skipped)
}
if len(elsewhere) > 0 && r.TaggedElsewhere != nil {
r.TaggedElsewhere(ctx, module, elsewhere)
}
if found && r.Took != nil {
r.Took(ctx, cur, out)
}
if r.Reserved != nil {
r.Reserved(ctx, out)
}
return out, nil
}
}
@@ -1,14 +1,16 @@
package versions
import (
"encoding/json"
"context"
"encoding/json"
"errors"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
"reflect"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
)
type reserveMem struct {
@@ -412,8 +414,8 @@ func TestANumberARegistryHoldsIsNeverReserved(t *testing.T) {
held = []string{"0.1.37", "0.1.38", "0.1.39", "0.1.40", "0.2.0"}
// The build again of a done key reserves as a rebuild (A-490b): a withdrawn number is never its floor's to reuse.
again, err := r.Reserve(ctx, "node-tools", "0.1", []string{"node-tools:program"}, Source{Fingerprint: "fp1", GoPackage: true, Rebuild: true})
if err != nil || again.Version != "0.1.41" || again.Artifacts["node-tools:program"] != "" || again.Withdrawn["0.1.39"] != "fp1" || again.Tag != GoTagOf("0.1.41") {
t.Fatalf("the build again reserved %+v (%v), want 0.1.41 with no digest and 0.1.39 kept withdrawn", again, err)
if err != nil || again.Version != "0.1.41" || again.Artifacts["node-tools:program"] != "" || again.Withdrawn["0.1.39"] != "fp1" || again.Tag != "" {
t.Fatalf("the build again reserved %+v (%v), want 0.1.41 with no digest, 0.1.39 kept withdrawn, and no tag (no Go package name known)", again, err)
}
if at, _ := r.AtVersion(ctx, "0.1.39"); len(at) != 0 {
t.Errorf("the index by version still names the withdrawn number: %v", at)
@@ -426,4 +428,50 @@ func TestANumberARegistryHoldsIsNeverReserved(t *testing.T) {
if v, err := r.Reserve(ctx, "node-tools", "0.2", []string{"node-tools:program"}, Source{Fingerprint: "fp1"}); err != nil || v.Version != "0.2.1" {
t.Errorf("with the registry down reserved %q (%v), want 0.2.1 (what it answered still skipped)", v.Version, err)
}
// Design 47 H2 §796: a Go package's tags are read before it reserves, once per catalog revision; a number whose tag
// `<directory>/v<version>` stands at another commit is skipped and said, one at the build's own commit is not; the version's
// tag names the Go module's directory; a read met away lets the reservation go on, any other failure refuses it.
r.HeldNumbers = nil
var tagReads int
var tagsErr error
var tagsAway bool
var elsewhere []string
tags := []git.Tag{{Tag: "modules/nt/v0.3.0", Commit: "c-other"}, {Tag: "modules/nt/v0.3.1", Commit: "c-mine"}, {Tag: "modules/nt/v0.3.2", Commit: "c-other2"}}
r.Names = func(_ context.Context, module, artifact string) (string, bool, error) {
return "git.example/o/r/modules/nt", module == "node-tools" && artifact == "lib", nil
}
r.Tags = func(_ context.Context, repository, prefix string) ([]git.Tag, bool, error) {
tagReads++
if repository != "o/r" || prefix != "modules/nt/v0.3." {
t.Errorf("tags read of %s %q, want o/r modules/nt/v0.3.", repository, prefix)
}
return tags, tagsAway, tagsErr
}
r.TaggedElsewhere = func(_ context.Context, _ string, s []string) { elsewhere = append(elsewhere, s...) }
at := Source{Fingerprint: "fp3", GoPackage: true, GoArtifact: "lib", Repository: "o/r", Commit: "c-mine", Branch: "main", CatalogRevision: 7}
v3, err := r.Reserve(ctx, "node-tools", "0.3", []string{"node-tools:lib"}, at)
if err != nil || v3.Version != "0.3.1" || v3.Tag != "modules/nt/v0.3.1" || len(elsewhere) != 1 || !strings.Contains(elsewhere[0], "0.3.0") {
t.Fatalf("reserved %+v (%v), tagged elsewhere %v: want 0.3.1 (0.3.0 tagged at another commit, 0.3.1 at the build's own), tag modules/nt/v0.3.1", v3, err, elsewhere)
}
at.Fingerprint, at.Commit, elsewhere = "fp4", "c-new", nil
if v4, err := r.Reserve(ctx, "node-tools", "0.3", []string{"node-tools:lib"}, at); err != nil || v4.Version != "0.3.3" || tagReads != 1 || len(elsewhere) != 1 || !strings.Contains(elsewhere[0], "0.3.2") {
t.Fatalf("reserved %+v (%v) after %d reads, tagged elsewhere %v: want 0.3.3 past 0.3.2's tag, the tags read once for revision main.7", v4, err, tagReads, elsewhere)
}
at.CatalogRevision, at.Fingerprint, tagsErr, tagsAway = 8, "fp5", errors.New("the forge is away"), true
if v5, err := r.Reserve(ctx, "node-tools", "0.3", []string{"node-tools:lib"}, at); err != nil || v5.Version != "0.3.4" || tagReads != 2 {
t.Fatalf("with the tags away reserved %+v (%v) after %d reads, want 0.3.4 and the reservation going on", v5, err, tagReads)
}
at.CatalogRevision, at.Fingerprint, tagsAway = 9, "fp6", false
if _, err := r.Reserve(ctx, "node-tools", "0.3", []string{"node-tools:lib"}, at); err == nil || !strings.Contains(err.Error(), "tags") {
t.Errorf("a tags read refused not away reserved anyway (%v), want the reservation refused naming the tags", err)
}
if d, ok := GoDirectory("git.example/novox/mesh-sdk/go", "https://git.example/novox/mesh-sdk.git"); !ok || d != "go" || GoTagOf(d, "0.1.9") != "go/v0.1.9" {
t.Errorf("mesh-sdk's Go package: directory %q (%v), want go and the tag go/v0.1.9", d, ok)
}
if d, ok := GoDirectory("git.example/o/r", "o/r"); !ok || d != "" || GoTagOf(d, "1.0.0") != "v1.0.0" {
t.Errorf("a root Go module: directory %q (%v), want the root and the tag v1.0.0", d, ok)
}
if _, ok := GoDirectory("git.example/o/rr/x", "o/r"); ok {
t.Error("a module path of another repository was given a directory")
}
}
@@ -1,4 +1,31 @@
package versions
// GoTagOf is the git tag a Go package's released version is written as (design 47 G §370, design 51 *The versions*).
func GoTagOf(version string) string { return "go/v" + version }
import (
"strings"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk"
)
// GoTagOf is the git tag a Go package's released version is written as (design 47 H2 §796): `<directory>/v<version>`, the
// directory of its Go module in the repository, which the Go toolchain's direct fallback reads; `v<version>` at the root. The
// `go/v<version>` tags go from new versions.
func GoTagOf(directory, version string) string { return TagPrefix(directory) + version }
// GoDirectory is the directory of the Go module path modulePath in repository (`<owner>/<name>`, or a clone address naming
// it): the part of the path after the repository's own, as the Go toolchain reads a module kept in a subdirectory; empty at
// the root. ok is false when the path is not one of that repository.
func GoDirectory(modulePath, repository string) (string, bool) {
repo := "/" + strings.ToLower(trunk.RepositoryName(repository))
lower := strings.ToLower(modulePath)
for from := 0; ; {
i := strings.Index(lower[from:], repo)
if i < 0 || repo == "/" {
return "", false
}
end := from + i + len(repo)
if end == len(lower) || lower[end] == '/' {
return strings.Trim(modulePath[end:], "/"), true
}
from = end
}
}
+61 -23
View File
@@ -13,6 +13,8 @@ import (
"testing"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"mesh-delivery-builder/internal/versions"
)
// The controller's manifest reader is mesh-controller internal/catalogue (Manifest, manifest.go: Consumes :521,
@@ -43,7 +45,9 @@ type manifest struct {
Module string `json:"module"`
Version string `json:"version"`
Slug string `json:"slug"`
Claims []struct {
// Class is core: the builder claims a seat (design 47 H2 §810, §814; H-A-394 (1)).
Class string `json:"class"`
Claims []struct {
Name string `json:"name"`
Scope string `json:"scope"`
Serves []string `json:"serves"`
@@ -58,6 +62,9 @@ type manifest struct {
MaxAgeDays int `json:"max-age-days"`
MaxMiB int `json:"max-mib"`
LastPerSubject bool `json:"last-per-subject"`
Retention string `json:"retention"`
Duplicates int `json:"duplicates"`
MaxDeliveries int `json:"max-deliveries"`
} `json:"streams"`
Logs []struct {
Name string `json:"name"`
@@ -75,10 +82,11 @@ type manifest struct {
Maximum float64 `json:"maximum"`
Trusted bool `json:"trusted"`
Default any `json:"default"`
Runtime bool `json:"runtime"`
} `json:"config"`
OwnSecrets map[string]string `json:"own-secrets"`
SecretsOwner string `json:"secrets-owner"`
Resources []map[string]any `json:"resources"`
OwnSecrets map[string]any `json:"own-secrets"`
SecretsOwner string `json:"secrets-owner"`
Resources []map[string]any `json:"resources"`
Build struct {
Artifacts []artifact `json:"artifacts"`
} `json:"build"`
@@ -114,7 +122,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
if m.Module != "mesh-delivery-builder" {
t.Errorf("module = %q", m.Module)
}
if want := []state{{Name: "builds", History: 5}, {Name: "engine-reads", History: 1}, {Name: "findings", History: 5}, {Name: "floor-waits", History: 1}, {Name: "hearing", History: 1}, {Name: "instances", History: 1}, {Name: "old-form", History: 1}, {Name: "bounds", History: 1, KeyTTL: true}, {Name: "queues", History: 1}, {Name: "toolchains-heard", History: 1}, {Name: "waits", History: 1, MaxMiB: 4}, {Name: "reserved", History: 1}, {Name: "publish-queue", History: 1, MaxMiB: 4}, {Name: "acted", History: 1, MaxMiB: 1}, {Name: "asked", History: 1}, {Name: "reasks", History: 1}, {Name: "versions", History: 1}, {Name: "first-views", History: 1}}; !reflect.DeepEqual(m.State, want) { // builds.BuilderBuildsHistory
if want := []state{{Name: "builds", History: 5, MaxMiB: 128}, {Name: "engine-reads", History: 1, MaxMiB: 8}, {Name: "findings", History: 5, MaxMiB: 32}, {Name: "floor-waits", History: 1, MaxMiB: 8}, {Name: "hearing", History: 1, MaxMiB: 8}, {Name: "instances", History: 1, MaxMiB: 8}, {Name: "old-form", History: 1, MaxMiB: 8}, {Name: "bounds", History: 1, KeyTTL: true, MaxMiB: 16}, {Name: "taken", History: 1, KeyTTL: true, MaxMiB: 8}, {Name: "queues", History: 1, MaxMiB: 8}, {Name: "toolchains-heard", History: 1, MaxMiB: 8}, {Name: "waits", History: 1, MaxMiB: 4}, {Name: "reserved", History: 1, MaxMiB: 8}, {Name: "acted", History: 1, MaxMiB: 1}, {Name: "asked", History: 1, MaxMiB: 8}, {Name: "reasks", History: 1, MaxMiB: 8}, {Name: "versions", History: 1, MaxMiB: 32}, {Name: "first-views", History: 1, MaxMiB: 8}, {Name: "unbuildable", History: 1, MaxMiB: 8}, {Name: "consumers", History: 1, MaxMiB: 8}}; !reflect.DeepEqual(m.State, want) { // builds.BuilderBuildsHistory // caps per H-A-437 over the fills read on novox 2026-11-22 (design 47 H2 §812, §814)
t.Errorf("state = %+v, want %+v", m.State, want)
}
// 1887/A4c: the builder chooses no node, so it reads neither presence nor have, invokes no build module's tool and
@@ -145,10 +153,10 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
// The watchdogs seat's presence says a node's machine is heard (issue 1966, ADR 0596 §2, ADR 0668 §3).
if want := []string{"seat:mesh-catalog.entry", "seat:mesh-catalog.registrations", "seat:mesh-catalog.seat-definitions", "seat:mesh-catalog.revision", "seat:mesh-delivery.source", "seat:container-image-registry.publish", "seat:container-image-registry.remove", "seat:container-image-registry.versions",
"seat:git.tag", "seat:git.tags",
"seat:go-package-registry.publish", "seat:go-package-registry.remove", "seat:go-package-registry.versions", "seat:mesh-artifact-sweep.held", "seat:mesh-builds.reserved", "seat:mesh-catalog.versions", "seat:mesh-hand-acts.hand-act", "seat:mesh-module-assigner.assignments", "seat:mesh-watchdogs.presence", "seat:node-build-go.drop", "seat:node-build-image.drop", "seat:node-build-node.drop", "seat:node-engine.manifest-reads", "seat:npm-package-registry.publish", "seat:npm-package-registry.remove",
"seat:go-package-registry.publish", "seat:go-package-registry.remove", "seat:go-package-registry.versions", "seat:mesh-artifact-sweep.held", "seat:mesh-builds.reserved", "seat:mesh-builds.drop", "seat:mesh-builds.publish-queue", "seat:mesh-catalog.versions", "seat:mesh-hand-acts.hand-act", "seat:mesh-module-assigner.assignments", "seat:mesh-watchdogs.presence", "seat:node-build-go.drop", "seat:node-build-image.drop", "seat:node-build-node.drop", "seat:node-build-go.answer", "seat:node-build-image.answer", "seat:node-build-node.answer", "seat:node-engine.manifest-reads", "seat:npm-package-registry.publish", "seat:npm-package-registry.remove",
"seat:npm-package-registry.versions", "seat:oci-artifact-registry.publish", "seat:oci-artifact-registry.remove", "seat:oci-artifact-registry.versions",
// A-231: the build asks on each build seat's queue delivery (accept.delivery.<asker>).
"seat:node-build-go.accept.delivery", "seat:node-build-node.accept.delivery", "seat:node-build-image.accept.delivery"}; !reflect.DeepEqual(sorted(m.Invokes), sorted(want)) {
"seat:node-build-go.accept.delivery", "seat:node-build-node.accept.delivery", "seat:node-build-image.accept.delivery", "seat:node-build-go.current", "seat:node-build-image.current", "seat:node-build-node.current"}; !reflect.DeepEqual(sorted(m.Invokes), sorted(want)) {
t.Errorf("invokes = %v, want %v", m.Invokes, want)
}
// The builder hears the build seats' toolchain-reported and the node-engine seat's running (design 47 G §492, §546) and reads no
@@ -168,27 +176,38 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
"oci-artifact-registry.kept", "oci-artifact-registry.published", "oci-artifact-registry.refused", "oci-artifact-registry.removed", "operator-channel.decided"}; !reflect.DeepEqual(sorted(m.Consumes), sorted(want)) {
t.Errorf("consumes = %v, want %v", m.Consumes, want)
}
// H-A-464: the process runs on the module's account, so the module's broker is the one credential.
if b, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 || b != "${dir:state}/broker" {
t.Errorf("own-secrets = %v, want exactly broker", m.OwnSecrets)
t.Errorf("own-secrets = %v, want the module's broker alone", m.OwnSecrets)
}
if m.SecretsOwner != "mesh-delivery-builder" {
t.Errorf("secrets-owner = %q", m.SecretsOwner)
}
// 1945: the builder's tools (builder_seat_queue: note 62), served by its tools bundle, and the event it says when it asks an ask again.
if want := []string{"builder_announce_again", "builder_ask_again", "builder_bounds", "builder_build", "builder_build_again", "builder_builds_since", "builder_engine_reads", "builder_first_view_again", "builder_first_views",
"builder_floor_again", "builder_floor_waits", "builder_old_form", "builder_publish_again", "builder_publish_queue", "builder_release", "builder_reserve_done", "builder_reserved", "builder_reserved_drop", "builder_seat_list", "builder_seat_list_again", "builder_seat_queue", "builder_stuck", "builder_toolchains", "builder_toolchains_drop", "builder_versions", "builder_waits"}; !reflect.DeepEqual(sorted(m.Tools), want) {
if want := []string{"build_current", "builder_announce_again", "builder_ask_again", "builder_ask_drop", "builder_asks", "builder_bounds", "builder_build", "builder_build_again", "builder_builds_since", "builder_engine_reads", "builder_first_view_again", "builder_first_views",
"builder_floor_again", "builder_floor_waits", "builder_health", "builder_old_form", "builder_publish_again", "builder_publish_queue", "builder_release", "builder_reserve_done", "builder_reserved", "builder_reserved_drop", "builder_seat_list", "builder_seat_list_again", "builder_seat_queue", "builder_stuck", "builder_toolchains", "builder_toolchains_drop", "builder_unbuildable", "builder_versions", "builder_waits"}; !reflect.DeepEqual(sorted(m.Tools), want) {
t.Errorf("tools = %v, want %v", m.Tools, want)
}
// Design 47 G §672: the toolchain waits' stream, kept 7 days within 16 MiB, and the bound's setting, 60 minutes.
// Design 47 G A-30, §383: and its own findings' events, which its finding writer says.
if len(m.Streams) != 3 || m.Streams[0].Name != "waits" || !reflect.DeepEqual(m.Streams[0].Events, []string{"wait-held.**", "wait-ended.**"}) || m.Streams[0].MaxAgeDays != 7 || m.Streams[0].MaxMiB != 16 ||
if len(m.Streams) != 4 || m.Streams[0].Name != "waits" || !reflect.DeepEqual(m.Streams[0].Events, []string{"wait-held.**", "wait-ended.**"}) || m.Streams[0].MaxAgeDays != 7 || m.Streams[0].MaxMiB != 16 ||
m.Streams[1].Name != builds.BuildsStream || !reflect.DeepEqual(m.Streams[1].Events, []string{builds.BuildsStreamEvents}) || m.Streams[1].MaxAgeDays != 7 || !m.Streams[1].LastPerSubject ||
m.Streams[2].Name != "mesh-delivery-builder-findings" || !reflect.DeepEqual(m.Streams[2].Events, []string{"finding-written.*", "finding-cleared.*", "findings-stated.*"}) {
m.Streams[2].Name != "mesh-delivery-builder-findings" || !reflect.DeepEqual(m.Streams[2].Events, []string{"finding-written.*", "finding-cleared.*", "findings-stated.*"}) ||
m.Streams[2].MaxMiB != 8 || m.Streams[2].MaxAgeDays != 7 || // design 47 H2 §783: the findings history 8 MiB, 7 days
// Design 47 H2 §793, §820, H-A-446 (2), H-A-454: the work queue publishes in the old section, 64 MiB, 7 days, duplicates 24 h.
m.Streams[3].Name != versions.PublishesLocal || !reflect.DeepEqual(m.Streams[3].Events, []string{"publish.**"}) || m.Streams[3].Retention != "work-queue" ||
m.Streams[3].Duplicates != 24 || m.Streams[3].MaxDeliveries != 10 || m.Streams[3].MaxAgeDays != 7 || m.Streams[3].MaxMiB != 64 {
t.Errorf("streams = %+v, want waits with wait-held.** and wait-ended.**, 7 days, 16 MiB, the builds' stream (build.ended.**, 7 days, kept last per subject: design 47 G A-181) and the findings' stream", m.Streams)
}
if c, ok := m.Config["toolchain-wait-minutes"]; !ok || c.Default != float64(60) || len(m.Config) != 7 || m.Config["revision-wait-minutes"].Default != float64(2) || m.Config["package-wait-minutes"].Default != float64(10) ||
m.Config["publish-pause"].Default != float64(30) || m.Config["publish-pause-max"].Default != float64(30) || m.Config["publish-tries"].Default != float64(8) {
t.Errorf("config = %+v, want toolchain-wait-minutes of 60, revision-wait-minutes of 2 (H-A-156), package-wait-minutes of 10 (H-A-165), the publish settings 30 s, 30 min and 8 tries (design 47 G §588, §633, A-457a)", m.Config)
if c, ok := m.Config["toolchain-wait-minutes"]; !ok || c.Default != float64(60) || len(m.Config) != 12 ||
// design 47 H2 §775, H-A-381 (2): the kept-call pace and the taken key's bound, read by the process
m.Config["kept-call-pace-first"].Default != "1s" || m.Config["kept-call-pace-max"].Default != "30s" || m.Config["ask-taken-bound"].Default != "10m" ||
!m.Config["kept-call-pace-first"].Runtime || !m.Config["kept-call-pace-max"].Runtime || !m.Config["ask-taken-bound"].Runtime ||
// H-A-461 (2): the bounds of §794 are settings read at run time, 100 ms and 5 minutes by default
m.Config["slow-event"].Default != "100ms" || m.Config["events-behind-bound"].Default != "5m" || !m.Config["slow-event"].Runtime || !m.Config["events-behind-bound"].Runtime ||
m.Config["revision-wait-minutes"].Default != float64(2) || m.Config["package-wait-minutes"].Default != float64(10) ||
m.Config["publish-pause"].Default != float64(30) || m.Config["publish-pause-max"].Default != float64(30) || m.Config["publish-tries"].Default != float64(5) || m.Config["publish-tries"].Maximum != 10 {
t.Errorf("config = %+v, want toolchain-wait-minutes of 60, revision-wait-minutes of 2 (H-A-156), package-wait-minutes of 10 (H-A-165), the publish settings 30 s, 30 min and 5 tries, at most 10 (design 47 G §588, §633, A-457a; H2 H-A-446 (2))", m.Config)
}
// Design 47 G §349: version.numbered is no longer said; old-form-closed on the family version, apart from the builds' ends.
if want := []string{"build.asked-again", "version.old-form-closed"}; !reflect.DeepEqual(m.Emits, want) {
@@ -198,9 +217,10 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
if len(m.Logs) != 1 || m.Logs[0].Name != builds.BuilderLog || m.Logs[0].MaxAgeDays != builds.BuilderLogDays || m.Logs[0].MaxMiB != builds.BuilderLogMiB {
t.Errorf("logs = %+v, want %s kept %d days in %d MiB", m.Logs, builds.BuilderLog, builds.BuilderLogDays, builds.BuilderLogMiB)
}
// Design 47 G §52, §369, H-A-328: the seat mesh-builds, held once for the mesh, serving reserved and published.
if len(m.Claims) != 1 || m.Claims[0].Name != "mesh-builds" || m.Claims[0].Scope != "mesh" || !reflect.DeepEqual(m.Claims[0].Serves, []string{"reserved", "published"}) {
t.Errorf("claims = %+v, want mesh-builds, scope mesh, serving reserved and published", m.Claims)
// Design 47 G §52, §369, H-A-328, H2 §778, §793: the seat mesh-builds, held once for the mesh, serving reserved, published, drop
// and publish-queue (H-A-454).
if len(m.Claims) != 1 || m.Claims[0].Name != "mesh-builds" || m.Claims[0].Scope != "mesh" || !reflect.DeepEqual(m.Claims[0].Serves, []string{"reserved", "published", "drop", "publish-queue"}) {
t.Errorf("claims = %+v, want mesh-builds, scope mesh, serving reserved, published, drop and publish-queue", m.Claims)
}
// A-115: the process answers the verb itself and says so in `serves` beside its process, with no `bus`; the bundle serves no tool named so.
var resources []map[string]any
@@ -208,8 +228,8 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
t.Fatal(err)
}
for _, r := range resources {
if r["type"] == "process" && !reflect.DeepEqual(r["serves"], []any{"seat:mesh-builds.reserved", "seat:mesh-builds.published"}) {
t.Errorf("the process serves %v, want seat:mesh-builds.reserved and published (A-115, H-A-328)", r["serves"])
if r["type"] == "process" && !reflect.DeepEqual(r["serves"], []any{"seat:mesh-builds.reserved", "seat:mesh-builds.published", "seat:mesh-builds.drop", "seat:mesh-builds.publish-queue"}) {
t.Errorf("the process serves %v, want seat:mesh-builds.reserved, published, drop and publish-queue (A-115, H-A-328, H2 §778, §793)", r["serves"])
}
}
if slices.Contains(m.Tools, "reserved") {
@@ -240,6 +260,11 @@ func TestManifestProcess(t *testing.T) { // 1607/AC1
if p["artifact"] != "program" || !reflect.DeepEqual(p["run"], []any{"./mesh-delivery-builder"}) {
t.Errorf("process = %v", p)
}
// Design 47 H2 §794, H-A-435 (second addendum): the process's health stays kind unit, builder_health a tool beside it (a
// tool-only health is refused, ADR 0227 rule 8, ADR 0240 rule 2).
if h, _ := p["health"].(map[string]any); h["kind"] != "unit" || len(h) != 1 {
t.Errorf("health = %v, want kind unit alone", p["health"])
}
env, _ := p["env"].(map[string]any)
if len(env) != 8 || env["MESH_BUILDER_REVISION_WAIT_FILE"] != "${dir:state}/revision-wait-minutes" || env["MESH_BUILDER_PACKAGE_WAIT_FILE"] != "${dir:state}/package-wait-minutes" || env["MESH_BUILDER_PUBLISH_TRIES_FILE"] != "${dir:state}/publish-tries" || env["MESH_BROKER_FILE"] != "${dir:state}/broker" || env["MESH_BUILDER_TOOLCHAIN_WAIT_FILE"] != "${dir:state}/toolchain-wait-minutes" {
t.Errorf("env = %v, want the broker file and the path of the toolchain wait's file, never the setting itself (design 47 G §672, A-71)", p["env"])
@@ -263,8 +288,19 @@ func TestManifestProcess(t *testing.T) { // 1607/AC1
t.Errorf("a settings resource %v: the builder holds no setting since the capacity is the build modules' (ADR 0525 §5)", r)
}
}
if _, bus := p["bus"]; bus {
t.Errorf("the process declares a bus: it runs on its own broker credential")
// Design 47 H2 §793, H-A-464: the process declares no bus and runs on the module's account, its credential the module's
// broker; as the module's one process it is the one reader of the work queue publishes (ADR 0436 §6).
if _, has := p["bus"]; has || env["MESH_BROKER_FILE"] != "${dir:state}/broker" {
t.Errorf("the process's bus = %v, broker file %v: want no bus, on the module's broker", p["bus"], env["MESH_BROKER_FILE"])
}
processes := 0
for _, r := range m.Resources {
if r["type"] == "process" {
processes++
}
}
if processes != 1 {
t.Errorf("%d processes: the work queue publishes needs the module's one process as its reader (H-A-464)", processes)
}
// H-A-186, design 47 G §370: a Go package another module imports is a declared package artifact of its module, so every
// release publishes it: builds (ADR 0634 §1, ADR 0606 §4), imported by the build modules and the node-engine, and
@@ -306,7 +342,9 @@ func TestServesNoRequestSubjectAndHoldsNoTicker(t *testing.T) { // 1607/AC6 (ADR
}
for _, f := range forbidden {
// The one subscription is the seat mesh-builds' accepted kind `step`, the runner's build ask (design 47 G A-97, A-107).
if f == "Subscribe(" && filepath.ToSlash(path) == "cmd/mesh-delivery-builder/step.go" {
// Design 47 H2 §777: the seat's work queue is pulled, its worker bound when the bus says it was created (the
// one subscription), and a taken ask is said in progress while it is built (ADR 0525 §6), the queue's own cadence.
if (f == "Subscribe(" || f == "time.NewTicker") && filepath.ToSlash(path) == "cmd/mesh-delivery-builder/stepqueue.go" {
continue
}
if strings.Contains(string(src), f) {
+80 -24
View File
@@ -2,13 +2,16 @@
"module": "mesh-delivery-builder",
"version": "0.1",
"slug": "builder",
"class": "core",
"claims": [
{
"name": "mesh-builds",
"scope": "mesh",
"serves": [
"reserved",
"published"
"published",
"drop",
"publish-queue"
]
}
],
@@ -36,9 +39,14 @@
"builder_toolchains",
"builder_toolchains_drop",
"builder_waits",
"builder_unbuildable",
"builder_health",
"builder_ask_drop",
"builder_asks",
"builder_versions",
"builder_first_views",
"builder_first_view_again"
"builder_first_view_again",
"build_current"
],
"emits": [
"build.asked-again",
@@ -89,6 +97,8 @@
"seat:go-package-registry.versions",
"seat:mesh-artifact-sweep.held",
"seat:mesh-builds.reserved",
"seat:mesh-builds.drop",
"seat:mesh-builds.publish-queue",
"seat:mesh-catalog.versions",
"seat:mesh-hand-acts.hand-act",
"seat:mesh-module-assigner.assignments",
@@ -96,6 +106,9 @@
"seat:node-build-go.drop",
"seat:node-build-image.drop",
"seat:node-build-node.drop",
"seat:node-build-go.answer",
"seat:node-build-image.answer",
"seat:node-build-node.answer",
"seat:node-engine.manifest-reads",
"seat:npm-package-registry.remove",
"seat:npm-package-registry.versions",
@@ -112,32 +125,38 @@
"seat:mesh-delivery.source",
"seat:node-build-go.accept.delivery",
"seat:node-build-node.accept.delivery",
"seat:node-build-image.accept.delivery"
"seat:node-build-image.accept.delivery",
"seat:node-build-go.current",
"seat:node-build-image.current",
"seat:node-build-node.current"
],
"state": [
{ "name": "builds", "history": 5 },
{ "name": "engine-reads", "history": 1 },
{ "name": "findings", "history": 5 },
{ "name": "floor-waits", "history": 1 },
{ "name": "hearing", "history": 1 },
{ "name": "instances", "history": 1 },
{ "name": "old-form", "history": 1 },
{ "name": "bounds", "history": 1, "key-ttl": true },
{ "name": "queues", "history": 1 },
{ "name": "toolchains-heard", "history": 1 },
{ "name": "builds", "history": 5, "max-mib": 128 },
{ "name": "engine-reads", "history": 1, "max-mib": 8 },
{ "name": "findings", "history": 5, "max-mib": 32 },
{ "name": "floor-waits", "history": 1, "max-mib": 8 },
{ "name": "hearing", "history": 1, "max-mib": 8 },
{ "name": "instances", "history": 1, "max-mib": 8 },
{ "name": "old-form", "history": 1, "max-mib": 8 },
{ "name": "bounds", "history": 1, "key-ttl": true, "max-mib": 16 },
{ "name": "taken", "history": 1, "key-ttl": true, "max-mib": 8 },
{ "name": "queues", "history": 1, "max-mib": 8 },
{ "name": "toolchains-heard", "history": 1, "max-mib": 8 },
{ "name": "waits", "history": 1, "max-mib": 4 },
{ "name": "reserved", "history": 1 },
{ "name": "publish-queue", "history": 1, "max-mib": 4 },
{ "name": "reserved", "history": 1, "max-mib": 8 },
{ "name": "acted", "history": 1, "max-mib": 1 },
{ "name": "asked", "history": 1 },
{ "name": "reasks", "history": 1 },
{ "name": "versions", "history": 1 },
{ "name": "first-views", "history": 1 }
{ "name": "asked", "history": 1, "max-mib": 8 },
{ "name": "reasks", "history": 1, "max-mib": 8 },
{ "name": "versions", "history": 1, "max-mib": 32 },
{ "name": "first-views", "history": 1, "max-mib": 8 },
{ "name": "unbuildable", "history": 1, "max-mib": 8 },
{ "name": "consumers", "history": 1, "max-mib": 8 }
],
"streams": [
{ "name": "waits", "events": ["wait-held.**", "wait-ended.**"], "max-age-days": 7, "max-mib": 16 },
{ "name": "mesh-delivery-builder-builds", "events": ["build.ended.**"], "max-age-days": 7, "max-mib": 16, "last-per-subject": true },
{ "name": "mesh-delivery-builder-findings", "events": ["finding-written.*", "finding-cleared.*", "findings-stated.*"], "max-age-days": 7, "max-mib": 16 }
{ "name": "mesh-delivery-builder-findings", "events": ["finding-written.*", "finding-cleared.*", "findings-stated.*"], "max-age-days": 7, "max-mib": 8 },
{ "name": "publishes", "events": ["publish.**"], "retention": "work-queue", "duplicates": 24, "max-deliveries": 10, "max-age-days": 7, "max-mib": 64 }
],
"config": {
"toolchain-wait-minutes": {
@@ -164,6 +183,41 @@
"trusted": false,
"default": 10
},
"ask-taken-bound": {
"type": "string",
"description": "how long the builder keeps the id of an ask it took from the seat mesh-builds' work queue, so a repeat of it is taken once, a Go duration such as 10m (design 47 H2 §775; H-A-381 (2)).",
"trusted": false,
"default": "10m",
"runtime": true
},
"kept-call-pace-first": {
"type": "string",
"description": "the first wait before a kept call met by a transport fault is made again, doubled per call up to kept-call-pace-max, a Go duration (design 47 H2 §775).",
"trusted": false,
"default": "1s",
"runtime": true
},
"kept-call-pace-max": {
"type": "string",
"description": "the most a kept call's wait between calls grows to, a Go duration (design 47 H2 §775).",
"trusted": false,
"default": "30s",
"runtime": true
},
"slow-event": {
"type": "string",
"description": "how long the handling of one heard event may take before the builder says it in its journal, with its duration and the part that took it, a Go duration (design 47 H2 §794; H-A-461 (2)).",
"trusted": false,
"default": "100ms",
"runtime": true
},
"events-behind-bound": {
"type": "string",
"description": "how long a durable consumer's oldest unacknowledged message may wait before builder_health names it, an events backlog or a handler standing still, a Go duration (design 47 H2 §794; H-A-461 (2)).",
"trusted": false,
"default": "5m",
"runtime": true
},
"publish-pause": {
"type": "number",
"description": "seconds before a version's second publish try, doubled per try up to publish-pause-max (design 47 G A-457a).",
@@ -190,11 +244,11 @@
},
"publish-tries": {
"type": "number",
"description": "how many times the builder tries a done build's publish before it ends loudly (log and condition) and leaves it readable in builder_publish_queue (design 47 G A-457a).",
"description": "how many times the builder tries a done build's publish before it ends loudly (log and condition) and leaves it readable in builder_publish_queue, at most 10, the work queue publishes' max-deliveries (design 47 G A-457a; H2 H-A-446 (2)).",
"minimum": 1,
"maximum": 100,
"maximum": 10,
"trusted": false,
"default": 8
"default": 5
}
},
"logs": [
@@ -295,7 +349,9 @@
"artifact": "program",
"serves": [
"seat:mesh-builds.reserved",
"seat:mesh-builds.published"
"seat:mesh-builds.published",
"seat:mesh-builds.drop",
"seat:mesh-builds.publish-queue"
],
"run": [
"./mesh-delivery-builder"
@@ -213,10 +213,10 @@ func TestTakeInBodiesAndNoneYetAreTheSeatsTypes(t *testing.T) {
}
// design 47 H-A-48: the indexer stamps no source revision on a module; the take-in's commit names it, so a module value and an
// entry's inputs carrying src or fp-src are not the seat's.
// entry's inputs carrying src or fp-src are not the seat's: a reader leaves them unread (design 47 H2 §780).
func TestNoSourceStamps(t *testing.T) {
if _, err := seat.DecodeModulesByModuleValue([]byte(`{"v":1,"repository":"mesh-sdk","commit":"` + hex64("a")[:40] + `","manifest":"` + hex64("b") + `","path":"go","src":41,"fp-src":41}`)); err == nil {
t.Error("a module value carrying src and fp-src is read")
if v, form, err := seat.DecodeModulesByModuleValue([]byte(`{"v":1,"repository":"mesh-sdk","commit":"` + hex64("a")[:40] + `","manifest":"` + hex64("b") + `","path":"go","src":41,"fp-src":41}`)); err != nil || v.Repository != "mesh-sdk" || form != 1 {
t.Errorf("a module value carrying src and fp-src is not read past them: %+v %d %v", v, form, err)
}
var in seat.EntryInputs
dec := json.NewDecoder(strings.NewReader(`{"src":41,"fp-src":41}`))
@@ -6,6 +6,7 @@ import (
"fmt"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/sourcebranches"
"slices"
"strings"
"time"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
@@ -166,9 +167,19 @@ func (r *Registrations) ask(_ context.Context, _ string) (views.Answer, error) {
values := make(map[string]json.RawMessage, len(a.Registrations))
for _, reg := range a.Registrations {
v, _ := json.Marshal(reg)
values[reg.Module] = v
values[KeyOf(reg)] = v
}
return views.Answer{V: 1, Owner: seat.Seat, State: seat.StateRegistrations, Created: registrationsCreated, Values: values, At: time.Now()}, nil
}
// KeyOf is a registration's key in the seat's state registrations: its module, or source.<owner>.<name> for a
// source-only registration, which names none (design 47 H2 §815); a part reads either by Registration with that key.
func KeyOf(reg seat.Registration) string {
if reg.Module != "" {
return reg.Module
}
owner, name, _ := strings.Cut(reg.Origin, "/")
return seat.RegistrationsBySourceOwnerNameKey(owner, name)
}
func clipBytes(b []byte) string { return sourcebranches.ClipBytes(b) }
@@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"fmt"
"strings"
"sync"
"testing"
"time"
@@ -36,6 +37,12 @@ func (f *fakeRegistrations) call(key string, body any) (json.RawMessage, error)
}
a := seat.RegistrationsAnswer{Registrations: []seat.Registration{}}
for m, v := range versions {
if name, ok := strings.CutPrefix(m, "source.novox."); ok {
// A source-only registration names no module and no path (design 47 H2 §815).
a.Registrations = append(a.Registrations, seat.Registration{Version: v, Origin: "novox/" + name, Branches: []string{"main"},
Check: &seat.SourceCheck{Step: "unit"}})
continue
}
a.Registrations = append(a.Registrations, seat.Registration{Module: m, Version: v, Origin: "novox/" + m, Path: "module.json", Branches: []string{"main"}})
}
return json.Marshal(a)
@@ -75,7 +82,7 @@ func TestRegistrationsAreAFirstViewThenTheirEvents(t *testing.T) {
return seat.Registration{}
}
}
f := &fakeRegistrations{versions: map[string]int64{"nats": 1, "zsh": 3}}
f := &fakeRegistrations{versions: map[string]int64{"nats": 1, "zsh": 3, "source.novox.hq": 1}}
r := NewRegistrations(f.call)
r.Follower.Wait = time.Nanosecond
r.Follower.Changed = typedChanged(func(reg seat.Registration, gone bool) {
@@ -95,10 +102,14 @@ func TestRegistrationsAreAFirstViewThenTheirEvents(t *testing.T) {
if reg, ok := r.Registration("nats"); !ok || reg.Version != 1 || reg.Origin != "novox/nats" {
t.Fatalf("first view holds nats as %+v, %v", reg, ok)
}
if all := r.All(); len(all) != 2 || all[0].Module != "nats" || all[1].Module != "zsh" {
// design 47 H2 §815: a source-only registration is held under its key source.<owner>.<name>.
if reg, ok := r.Registration("source.novox.hq"); !ok || reg.Check == nil || reg.Origin != "novox/hq" {
t.Fatalf("first view holds the source-only registration as %+v, %v", reg, ok)
}
if all := r.All(); len(all) != 3 || all[1].Module != "nats" || all[2].Module != "zsh" {
t.Fatalf("All = %+v", all)
}
for range 2 {
for range 3 {
next()
}
@@ -119,7 +130,7 @@ func TestRegistrationsAreAFirstViewThenTheirEvents(t *testing.T) {
// The answer to the ask the event starts was read before the change was written: it is asked once more.
f.mu.Lock()
f.stale = map[string]int64{"nats": 2, "zsh": 3}
f.stale = map[string]int64{"nats": 2, "zsh": 3, "source.novox.hq": 1}
f.mu.Unlock()
f.set("zsh", 4)
if err := r.Hear(ctx, changedEvent("zsh", 4)); err != nil {
@@ -58,7 +58,7 @@ func DecodeSource(data []byte) (seat.SourceByBranchValue, error) {
if err := keys.CheckVersion(data); err != nil {
return seat.SourceByBranchValue{}, err
}
s, err := seat.DecodeSourceByBranchValue(data)
s, _, err := seat.DecodeSourceByBranchValue(data)
if err != nil {
return s, fmt.Errorf("meshdeliveryclient: a source does not fit its shape: %w", err)
}
@@ -80,7 +80,7 @@ func DecodeOwed(data []byte) (seat.OwedByRepositoryNumberHeadActValue, error) {
if err := keys.CheckVersion(data); err != nil {
return seat.OwedByRepositoryNumberHeadActValue{}, err
}
o, err := seat.DecodeOwedByRepositoryNumberHeadActValue(data)
o, _, err := seat.DecodeOwedByRepositoryNumberHeadActValue(data)
if err != nil {
return o, fmt.Errorf("meshdeliveryclient: an owed value does not fit its shape: %w", err)
}
@@ -140,7 +140,8 @@ func TestGateValuesRefuseWhatTheirRulesRefuse(t *testing.T) {
if _, err := DecodeSource([]byte(`{"v":2,"source":1,"call":"c","heads":{"a":"b"}}`)); !errors.Is(err, keys.ErrUnknownVersion) {
t.Errorf("another version: %v", err)
}
if _, err := DecodeSource([]byte(`{"v":1,"source":1,"call":"c","heads":{"a":"b"},"extra":1}`)); err == nil {
t.Error("an unknown field decodes")
// design 47 H2 §780: a reader leaves a field it does not know unread.
if s, err := DecodeSource([]byte(`{"v":1,"source":1,"call":"0123456789abcdef","heads":{"a":"b"},"extra":1}`)); err != nil || s.Source != 1 {
t.Errorf("an unknown field is not read past: %+v %v", s, err)
}
}
@@ -233,7 +233,7 @@ func DecodeWant(data []byte) (Want, error) {
if err := keys.CheckVersion(data); err != nil {
return Want{}, err
}
w, err := seat.DecodeWantsByWantNodeModuleValue(data)
w, _, err := seat.DecodeWantsByWantNodeModuleValue(data)
if err != nil {
return Want{}, fmt.Errorf("wants: value does not fit its shape: %w", err)
}
@@ -272,7 +272,7 @@ func DecodeNodeWant(data []byte) (NodeWant, error) {
if err := keys.CheckVersion(data); err != nil {
return NodeWant{}, err
}
w, err := seat.DecodeWantsByWantNodeNodeValue(data)
w, _, err := seat.DecodeWantsByWantNodeNodeValue(data)
if err != nil {
return NodeWant{}, fmt.Errorf("wants: a node's own want does not fit its shape: %w", err)
}
@@ -298,7 +298,7 @@ func DecodeHeld(data []byte) (Held, error) {
if err := keys.CheckVersion(data); err != nil {
return Held{}, err
}
h, err := seat.DecodeWantsByHeldModuleFingerprintValue(data)
h, _, err := seat.DecodeWantsByHeldModuleFingerprintValue(data)
if err != nil {
return Held{}, fmt.Errorf("wants: value does not fit its shape: %w", err)
}
@@ -76,17 +76,18 @@ func TestWantsAreWhatTheEarlierCodecWrote(t *testing.T) {
t.Errorf("%s: written", name)
}
}
// design 47 H §750, H-A-37: a want names no digest, rule or newest; the earlier fields are refused.
// design 47 H §750, H-A-37: a want names no digest, rule or newest; a reader leaves the earlier fields unread
// (design 47 H2 §780).
for _, old := range []string{`"composed":"` + c + `"`, `"parts":{"x:image":"` + d + `"}`, `"rule":3`, `"newest":"` + b + `"`, `"kept-on":"n1"`} {
if _, err := DecodeWant([]byte(`{"v":1,"fp":"` + a + `",` + old + `}`)); err == nil {
t.Errorf("a want naming %s is read", old)
if w, err := DecodeWant([]byte(`{"v":1,"fp":"` + a + `",` + old + `}`)); err != nil || w.Fp != a {
t.Errorf("a want naming %s is not read past it: %+v %v", old, w, err)
}
}
if _, err := DecodeWant([]byte(`{"fp":"` + a + `","self":true}`)); !errors.Is(err, keys.ErrUnknownVersion) {
t.Errorf("no version: %v", err)
}
if _, err := DecodeWant([]byte(`{"v":1,"fp":"` + a + `","self":true,"extra":1}`)); err == nil {
t.Error("an unknown field is read")
if w, err := DecodeWant([]byte(`{"v":1,"fp":"` + a + `","self":true,"extra":1}`)); err != nil || !w.Self {
t.Errorf("an unknown field is not read past (design 47 H2 §780): %+v %v", w, err)
}
})
t.Run("node want and hold", func(t *testing.T) {
@@ -108,8 +109,9 @@ func TestWantsAreWhatTheEarlierCodecWrote(t *testing.T) {
t.Errorf("%s: written", name)
}
}
if _, err := DecodeNodeWant([]byte(`{"v":1,"composed":"` + c + `","fp":"` + a + `"}`)); err == nil {
t.Error("a fingerprint in a node's own want is read")
// design 47 H2 §780: a fingerprint is no field of a node's own want, and a reader leaves it unread.
if back, err := DecodeNodeWant([]byte(`{"v":1,"composed":"` + c + `","fp":"` + a + `"}`)); err != nil || back.Composed != c {
t.Errorf("a node's own want naming a fingerprint is not read past it: %+v %v", back, err)
}
raw, err = EncodeHeld(Held{Node: "anchor", Words: "unhealthy"})
if err != nil {
@@ -4,15 +4,27 @@
package containerimageregistry
import (
"bytes"
"encoding/json"
"errors"
"strings"
)
// Seat is the seat's name.
const Seat = "container-image-registry"
// Form is the form of the seat's values, events and answers this contract was generated at (design 47 H2 §780):
// a reader of a value of another form says "form N, this reader knows M".
const Form = 1
// FormRegistry is the form of what the part registry adds.
const FormRegistry = 1
// The forms and the digest of the fields this contract was generated from: the generator's compare fails a definition
// whose fields changed while its forms did not (design 47 H2 §780).
const (
generatedForms = "1 registry:1"
generatedFieldsDigest = "Artifact=79361e4ab1cd HeldEvent=b6d4c12500df Hold=c5a97a2f2d5a HoldAnswer=7d536ccb729f HoldsBySeatNameDigestValue=db093998a869 KeptEvent=ec3ef89ef8a7 ListAnswer=ca49dd7f20ee Listed=b9f1d1232e64 MirrorAnswer=0d59df97b53b Outcome=1e66f7bf6725 PublishAnswer=a0df0b922e1e PublishPartAnswer=aebe3c6c89e9 PublishedEvent=ec3ef89ef8a7 RefusedEvent=ec3ef89ef8a7 ReleaseAnswer=d30bd6000a16 ReleasedEvent=b6d4c12500df RemoveAnswer=bccb5a322d69 RemovedEvent=ec3ef89ef8a7 Repository=740e54e0b3b0 TagsAnswer=fe178191458a TagsAnswerTagsItem=eee02e024d64 VersionsAnswer=47e11ca2aea9 WindowsByCallerValue=23d9294dde92 WindowsByCallerValueDroppedItem=d748f0e58825"
)
const (
VerbPublish = "publish"
VerbPublishPart = "publish-part"
@@ -39,6 +51,32 @@ func Verbs() []string {
return []string{VerbPublish, VerbPublishPart, VerbVersions, VerbRemove, VerbMirror, VerbTags, VerbHold, VerbRelease, VerbList}
}
// VerbDescription is a verb's description as the seat's definition says it, "" for a verb the seat does not serve or
// does not describe (design 47 H2 H-A-449).
func VerbDescription(verb string) string {
switch verb {
case VerbPublish:
return "Open a transfer, naming the artifact and the digest of every part; answers the digests the holder lacks, which the caller then sends by publish-part. A transfer already open answers the parts held, so a cut transfer goes on from there. With no parts and a version, the held digest is published under that version and nothing is stored (the trunk's claim)."
case VerbPublishPart:
return "Send one chunk of one part of an open transfer; the last chunk of the last part ends the transfer, and the holder hashes what it received, asks builder_versions, stores it and says published or refused. A chunk's message stays under the bus's largest message (1 MiB), so at most 760 KiB of data, base64."
case VerbVersions:
return "The versions the seat holds of one name, newest first: each one's version, tag, digest, kind and when it was published."
case VerbRemove:
return "Remove one tagged version; the holder refuses an untagged one but for the retention removal of a bundle, archive or copied base, which the artifact sweep's holder asks; the caller is not judged (design 47 H2 §813 (7)). Every removal emits removed with its reason. A tagged version the store cannot remove alone, because another tag names its digest, stays whole, its tag and bytes kept: the answer says kept with the reason shared-digest and the tags still naming the digest, removed false, and kept is emitted, never removed; the caller ends there without a retry."
case VerbMirror:
return "Copy an outside image into the store by its pinned digest; emits published or refused."
case VerbTags:
return "The tags of one image and the digest each names."
case VerbHold:
return "Keep one untagged bundle, archive or copied base from removal by retention, as the artifact sweep's rules keep it (ADR 0617 §2); never an image of the holder's own. Emits held."
case VerbRelease:
return "Let a hold go; emits released. A release of what is not held answers released false and emits nothing, never an error: the outcome asked holds already. By hand only through the holder's own tool, the operator's session as the proved caller, recorded as a hand act."
case VerbList:
return "What the store holds of one repository, or of every repository the seat holds when none is named: every image manifest and index, each with its digest, whether it is an index (and the manifests an index names), the tags naming it, its size and whether it is held. Read from the store itself, so a manifest no tag names is listed too; it reads only (ADR 0632 §3)."
}
return ""
}
// Events is every event published on the seat's subject: a new copy each call.
func Events() []string {
return []string{EventPublished, EventRefused, EventRemoved, EventKept, EventHeld, EventReleased}
@@ -102,18 +140,13 @@ func ParseHoldsBySeatNameDigestKey(key string) (string, string, string, bool) {
return v0, v2, v4, true
}
// DecodeHoldsBySeatNameDigestValue reads one value of the state holds under the form "<seat>.<name>.<digest>", refusing a field HoldsBySeatNameDigestValue does not know and any text after the value.
func DecodeHoldsBySeatNameDigestValue(raw []byte) (HoldsBySeatNameDigestValue, error) {
// DecodeHoldsBySeatNameDigestValue reads one value of the state holds under the form "<seat>.<name>.<digest>" as another part wrote it: a field HoldsBySeatNameDigestValue does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodeHoldsBySeatNameDigestValue(raw []byte) (HoldsBySeatNameDigestValue, int, error) {
var v HoldsBySeatNameDigestValue
dec := json.NewDecoder(bytes.NewReader(raw))
dec.DisallowUnknownFields()
if err := dec.Decode(&v); err != nil {
return v, err
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
if dec.More() {
return v, errors.New("text after the value")
}
return v, nil
return v, formRead(raw), nil
}
// WindowsByCallerKey is the key of one value of the state windows under the form "<caller>".
@@ -133,18 +166,13 @@ func ParseWindowsByCallerKey(key string) (string, bool) {
return v0, true
}
// DecodeWindowsByCallerValue reads one value of the state windows under the form "<caller>", refusing a field WindowsByCallerValue does not know and any text after the value.
func DecodeWindowsByCallerValue(raw []byte) (WindowsByCallerValue, error) {
// DecodeWindowsByCallerValue reads one value of the state windows under the form "<caller>" as another part wrote it: a field WindowsByCallerValue does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodeWindowsByCallerValue(raw []byte) (WindowsByCallerValue, int, error) {
var v WindowsByCallerValue
dec := json.NewDecoder(bytes.NewReader(raw))
dec.DisallowUnknownFields()
if err := dec.Decode(&v); err != nil {
return v, err
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
if dec.More() {
return v, errors.New("text after the value")
}
return v, nil
return v, formRead(raw), nil
}
// Artifact is one published thing, as every registry event and answer names it.
@@ -182,6 +210,8 @@ type Hold struct {
By string `json:"by,omitempty"`
// the digest
Digest string `json:"digest"`
// is the form of what this value carries, left out while it is 1 (design 47 H2 §780).
Form int64 `json:"form,omitempty"`
// the module holding the seat
Holder string `json:"holder"`
// the name
@@ -216,6 +246,8 @@ type Outcome struct {
Digest string `json:"digest"`
// the build's fingerprint
Fingerprint string `json:"fingerprint,omitempty"`
// is the form of what this value carries, left out while it is 1 (design 47 H2 §780).
Form int64 `json:"form,omitempty"`
// the module holding the seat that said it
Holder string `json:"holder"`
// what it is
@@ -296,6 +328,8 @@ const (
// PublishAnswer is what publish answers.
type PublishAnswer struct {
// is the form of what this value carries, left out while it is 1 (design 47 H2 §780).
Form int64 `json:"form,omitempty"`
// the digest is held already: nothing to send; the outcome follows as an event
Held bool `json:"held,omitempty"`
// the digests of the parts the holder lacks, to send by publish-part
@@ -322,6 +356,8 @@ type PublishPartInput struct {
type PublishPartAnswer struct {
// the part is whole and its bytes match its digest
Complete bool `json:"complete,omitempty"`
// is the form of what this value carries, left out while it is 1 (design 47 H2 §780).
Form int64 `json:"form,omitempty"`
// the part
Part string `json:"part"`
// the chunks of the part held so far
@@ -340,6 +376,8 @@ type VersionsInput struct {
// VersionsAnswer is what versions answers.
type VersionsAnswer struct {
// is the form of what this value carries, left out while it is 1 (design 47 H2 §780).
Form int64 `json:"form,omitempty"`
// the name
Name string `json:"name"`
// newest first
@@ -369,6 +407,8 @@ const (
// RemoveAnswer is what remove answers.
type RemoveAnswer struct {
Artifact *Artifact `json:"artifact,omitempty"`
// is the form of what this value carries, left out while it is 1 (design 47 H2 §780).
Form int64 `json:"form,omitempty"`
// it stays whole, its tag and its bytes: the store cannot remove the tag alone while another tag names its digest (reason, tags)
Kept bool `json:"kept,omitempty"`
// why it was kept: shared-digest, another tag names its digest
@@ -398,6 +438,8 @@ type MirrorInput struct {
type MirrorAnswer struct {
// the digest stored
Digest string `json:"digest"`
// is the form of what this value carries, left out while it is 1 (design 47 H2 §780).
Form int64 `json:"form,omitempty"`
// the reference the seat answers
Reference string `json:"reference"`
}
@@ -418,6 +460,8 @@ type TagsAnswerTagsItem struct {
// TagsAnswer is what tags answers.
type TagsAnswer struct {
// is the form of what this value carries, left out while it is 1 (design 47 H2 §780).
Form int64 `json:"form,omitempty"`
// the image
Name string `json:"name"`
// every tag
@@ -436,6 +480,8 @@ type HoldInput struct {
// HoldAnswer is what hold answers.
type HoldAnswer struct {
// is the form of what this value carries, left out while it is 1 (design 47 H2 §780).
Form int64 `json:"form,omitempty"`
// it is held
Held bool `json:"held"`
// when the hold began, RFC 3339
@@ -454,6 +500,8 @@ type ReleaseInput struct {
// ReleaseAnswer is what release answers.
type ReleaseAnswer struct {
// is the form of what this value carries, left out while it is 1 (design 47 H2 §780).
Form int64 `json:"form,omitempty"`
// a hold was let go; false when there was none
Released bool `json:"released"`
}
@@ -466,6 +514,8 @@ type ListInput struct {
// ListAnswer is what list answers.
type ListAnswer struct {
// is the form of what this value carries, left out while it is 1 (design 47 H2 §780).
Form int64 `json:"form,omitempty"`
// each repository named, in order of its name
Repositories []Repository `json:"repositories"`
}
@@ -492,9 +542,11 @@ type ReleasedEvent = Hold
type HoldsBySeatNameDigestValue struct {
By string `json:"by"`
Digest string `json:"digest"`
Kind string `json:"kind"`
Name string `json:"name"`
Seat string `json:"seat"`
// is the form of what this value carries, left out while it is 1 (design 47 H2 §780).
Form int64 `json:"form,omitempty"`
Kind string `json:"kind"`
Name string `json:"name"`
Seat string `json:"seat"`
// when (RFC 3339 time)
Since string `json:"since"`
Why string `json:"why,omitempty"`
@@ -515,6 +567,8 @@ type WindowsByCallerValue struct {
Closed string `json:"closed,omitempty"`
// retired nodes dropped by hand
Dropped []WindowsByCallerValueDroppedItem `json:"dropped,omitempty"`
// is the form of what this value carries, left out while it is 1 (design 47 H2 §780).
Form int64 `json:"form,omitempty"`
// the nodes holding it open
Holding []string `json:"holding,omitempty"`
LastHolding []string `json:"last_holding,omitempty"`
@@ -526,3 +580,149 @@ type WindowsByCallerValue struct {
// of each node holding the window, why it holds it
Why map[string]string `json:"why,omitempty"`
}
// DecodePublishAnswer reads a PublishAnswer as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodePublishAnswer(raw []byte) (PublishAnswer, int, error) {
var v PublishAnswer
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
return v, formRead(raw), nil
}
// DecodePublishPartAnswer reads a PublishPartAnswer as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodePublishPartAnswer(raw []byte) (PublishPartAnswer, int, error) {
var v PublishPartAnswer
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
return v, formRead(raw), nil
}
// DecodeVersionsAnswer reads a VersionsAnswer as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodeVersionsAnswer(raw []byte) (VersionsAnswer, int, error) {
var v VersionsAnswer
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
return v, formRead(raw), nil
}
// DecodeRemoveAnswer reads a RemoveAnswer as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodeRemoveAnswer(raw []byte) (RemoveAnswer, int, error) {
var v RemoveAnswer
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
return v, formRead(raw), nil
}
// DecodeMirrorAnswer reads a MirrorAnswer as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodeMirrorAnswer(raw []byte) (MirrorAnswer, int, error) {
var v MirrorAnswer
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
return v, formRead(raw), nil
}
// DecodeTagsAnswer reads a TagsAnswer as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodeTagsAnswer(raw []byte) (TagsAnswer, int, error) {
var v TagsAnswer
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
return v, formRead(raw), nil
}
// DecodeHoldAnswer reads a HoldAnswer as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodeHoldAnswer(raw []byte) (HoldAnswer, int, error) {
var v HoldAnswer
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
return v, formRead(raw), nil
}
// DecodeReleaseAnswer reads a ReleaseAnswer as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodeReleaseAnswer(raw []byte) (ReleaseAnswer, int, error) {
var v ReleaseAnswer
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
return v, formRead(raw), nil
}
// DecodeListAnswer reads a ListAnswer as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodeListAnswer(raw []byte) (ListAnswer, int, error) {
var v ListAnswer
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
return v, formRead(raw), nil
}
// DecodePublishedEvent reads a PublishedEvent as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodePublishedEvent(raw []byte) (PublishedEvent, int, error) {
var v PublishedEvent
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
return v, formRead(raw), nil
}
// DecodeRefusedEvent reads a RefusedEvent as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodeRefusedEvent(raw []byte) (RefusedEvent, int, error) {
var v RefusedEvent
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
return v, formRead(raw), nil
}
// DecodeRemovedEvent reads a RemovedEvent as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodeRemovedEvent(raw []byte) (RemovedEvent, int, error) {
var v RemovedEvent
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
return v, formRead(raw), nil
}
// DecodeKeptEvent reads a KeptEvent as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodeKeptEvent(raw []byte) (KeptEvent, int, error) {
var v KeptEvent
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
return v, formRead(raw), nil
}
// DecodeHeldEvent reads a HeldEvent as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodeHeldEvent(raw []byte) (HeldEvent, int, error) {
var v HeldEvent
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
return v, formRead(raw), nil
}
// DecodeReleasedEvent reads a ReleasedEvent as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodeReleasedEvent(raw []byte) (ReleasedEvent, int, error) {
var v ReleasedEvent
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
return v, formRead(raw), nil
}
// formRead is the form a value says on the wire: 1 when it says none, or is no object.
func formRead(raw []byte) int {
var f struct {
Form int `json:"form"`
}
if json.Unmarshal(raw, &f) != nil || f.Form < 1 {
return 1
}
return f.Form
}
@@ -4,9 +4,7 @@
package git
import (
"bytes"
"encoding/json"
"errors"
"strings"
"time"
)
@@ -14,6 +12,17 @@ import (
// Seat is the seat's name.
const Seat = "git"
// Form is the form of the seat's values, events and answers this contract was generated at (design 47 H2 §780):
// a reader of a value of another form says "form N, this reader knows M".
const Form = 2
// The forms and the digest of the fields this contract was generated from: the generator's compare fails a definition
// whose fields changed while its forms did not (design 47 H2 §780).
const (
generatedForms = "2"
generatedFieldsDigest = "BranchAnswer=1cc6612a710e BranchMovedEvent=baaa6fdf37f2 Call=ebef3639e061 CallMember=8d7bf9b9c258 CallsByCallCallValue=05142ac90d52 Change=5a13d1572e59 CommentAnswer=74ec3ece9f05 ContainsAnswer=071d0e9742bd ErrorKind=3d26b2c1442a FailedStep=d68341819717 FailedStepsByModuleFingerprintStepValue=0fce6f5d3273 FileAnswer=8fcc52522004 ForgeDeleteEvent=35a0828a1238 ForgePullEvent=303345c390d7 ForgeStatusEvent=689488c43094 MemberOutcome=c35056683738 MergeCalled=cb5a36d22c92 MergeCalledEvent=8597a63a790c MergeCalledMembersItem=211bdc9bab48 MergeCalledRefusedItem=bbf9ce4382c8 MergePullAnswer=4ada4a7040bb Merging=482fbecedd58 NoteAnswer=74ec3ece9f05 OpenPull=56217c4c348a OpenPullsAnswer=a15ee399a171 Pull=6e3fdd146b1c PullAnswer=a8def6c28bf9 PullClosedEvent=c2b7ac1de473 PullMergedEvent=619d9c863307 PullUpdatedEvent=f86ee614998e RepoWritableEvent=1a8c6e6344ba Status=b2980533a044 StatusAnswer=74ec3ece9f05 StatusesAnswer=1c524343c18d Tag=b419017f7885 TagAnswer=8c1fa0ac9f1b TagsAnswer=77e4374f7d62 TreesAnswer=bc1ffccd2716 ViewAnswer=74ec3ece9f05"
)
const (
VerbPull = "pull"
VerbBranch = "branch"
@@ -48,6 +57,42 @@ func Verbs() []string {
return []string{VerbPull, VerbBranch, VerbOpenPulls, VerbTags, VerbTrees, VerbFile, VerbContains, VerbStatuses, VerbStatus, VerbComment, VerbView, VerbNote, VerbMergePull, VerbTag}
}
// VerbDescription is a verb's description as the seat's definition says it, "" for a verb the seat does not serve or
// does not describe (design 47 H2 H-A-449).
func VerbDescription(verb string) string {
switch verb {
case VerbPull:
return "One pull request, by its repository and number or by its merge commit: its number, base, head branch, head, merge commit, author and state; for an open one, also what the forge would merge now (merging). A pull request the forge does not hold answers an error naming it."
case VerbBranch:
return "A branch's head (design 47 *The seat git*, *Watched branches and the catalog graph per branch*). A branch the forge does not hold answers the error kind not-found."
case VerbOpenPulls:
return "Every open pull request of every repository, its head and base, the statuses its base's protection requires and those its head has."
case VerbTags:
return "A repository's tags, a page at a time, newest first; the page's bound is said, and more when another page exists."
case VerbTrees:
return "The trees of one commit: the root tree and the tree of every directory that holds a module.json, by their git hashes; with files, every blob under each named directory; with paths, the object of each named path. A listing the forge cut is refused, never answered in part. A directory named in files or paths that the commit does not hold answers the error kind not-found (design 47 H2 §796)."
case VerbFile:
return "One file at one commit, branch or tag, decoded; a file over 512 KiB is refused. An absent file is answered absent only when the forge said so."
case VerbContains:
return "Whether one commit contains another: an ancestor of it, or the same."
case VerbStatuses:
return "The statuses a commit carries, the newest of each context."
case VerbStatus:
return "Set a commit status. An act: its id is the caller's, and the same id said again sets it again to the same."
case VerbComment:
return "Add a comment to a pull request. The comment carries the act's id, and an id the pull request already holds is skipped (held)."
case VerbView:
return "Keep the delivery's one view comment on a pull request: made the first time, edited in place after; it carries the act's id, and an id already held is skipped (held)."
case VerbNote:
return "Append one line to a commit's note under refs/notes/mesh-plan. The line carries the act's id, and an id the note already holds is skipped (held)."
case VerbMergePull:
return "Merge one or more pull requests with the forge tools' account, each at the head the caller names: one merge call, a delivery's start and, with two or more members, its group (design 47 H §734, §735, §766; ADR 0682 §1, §2). Every member is judged before any is merged: still the pull request's head, its base a watched branch, no run at the head's build-input fingerprint failed a required step (judged from its own state failed-steps), registrations heard; no branch protection is read. When one member is refused, none is merged. Then it merges them in the order named, writing the call in its state calls before the first merge, and says merge-called once. It answers once, at the call's end, naming the call and each member's outcome; a member already merged is refused already-merged, naming its commit."
case VerbTag:
return "Write the git tag of a released version, <directory>/vMAJOR.MINOR.PATCH for a Go package (design 47 H2 §796), at the commit its build was made from; the builder writes it only on the catalog's version-released (design 47 G section 370). A tag already at that commit is answered performed. An error answer carries error and error-kind."
}
return ""
}
// Events is every event published on the seat's subject: a new copy each call.
func Events() []string {
return []string{EventPullUpdated, EventPullMerged, EventPullClosed, EventForgePull, EventForgeStatus, EventForgeDelete, EventRepoWritable, EventBranchMoved, EventMergeCalled}
@@ -129,18 +174,13 @@ func ParseFailedStepsByModuleFingerprintStepKey(key string) (string, string, str
return v0, v2, v4, true
}
// DecodeFailedStepsByModuleFingerprintStepValue reads one value of the state failed-steps under the form "<module>.<fingerprint>.<step>", refusing a field FailedStepsByModuleFingerprintStepValue does not know and any text after the value.
func DecodeFailedStepsByModuleFingerprintStepValue(raw []byte) (FailedStepsByModuleFingerprintStepValue, error) {
// DecodeFailedStepsByModuleFingerprintStepValue reads one value of the state failed-steps under the form "<module>.<fingerprint>.<step>" as another part wrote it: a field FailedStepsByModuleFingerprintStepValue does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodeFailedStepsByModuleFingerprintStepValue(raw []byte) (FailedStepsByModuleFingerprintStepValue, int, error) {
var v FailedStepsByModuleFingerprintStepValue
dec := json.NewDecoder(bytes.NewReader(raw))
dec.DisallowUnknownFields()
if err := dec.Decode(&v); err != nil {
return v, err
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
if dec.More() {
return v, errors.New("text after the value")
}
return v, nil
return v, formRead(raw), nil
}
// CallsByCallCallKey is the key of one value of the state calls under the form "call.<call>".
@@ -164,18 +204,13 @@ func ParseCallsByCallCallKey(key string) (string, bool) {
return v1, true
}
// DecodeCallsByCallCallValue reads one value of the state calls under the form "call.<call>", refusing a field CallsByCallCallValue does not know and any text after the value.
func DecodeCallsByCallCallValue(raw []byte) (CallsByCallCallValue, error) {
// DecodeCallsByCallCallValue reads one value of the state calls under the form "call.<call>" as another part wrote it: a field CallsByCallCallValue does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodeCallsByCallCallValue(raw []byte) (CallsByCallCallValue, int, error) {
var v CallsByCallCallValue
dec := json.NewDecoder(bytes.NewReader(raw))
dec.DisallowUnknownFields()
if err := dec.Decode(&v); err != nil {
return v, err
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
if dec.More() {
return v, errors.New("text after the value")
}
return v, nil
return v, formRead(raw), nil
}
// Call is one merge call, written before its first merge and updated as each member merges (design 47 H §766).
@@ -184,7 +219,9 @@ type Call struct {
Account string `json:"account"`
At time.Time `json:"at"`
// the caller as who@node
By string `json:"by"`
By string `json:"by"`
// is the form of what this value carries, left out while it is 1 (design 47 H2 §780).
Form int64 `json:"form,omitempty"`
Members []CallMember `json:"members"`
// merge-called was said for it
Said bool `json:"said"`
@@ -223,7 +260,7 @@ type Change struct {
Removed []string `json:"removed,omitempty"`
}
// ErrorKind is which kind of error, set with error alone: refused, the forge or the holder refused and error says why in the forge's own words; not-found, the forge said that what was asked about (a repository, a commit, a pull request, a branch or a tag) does not exist, so a caller never reads it out of error's words; failed, the forge failed or could not be reached, so the act or read could not be performed.
// ErrorKind is which kind of error, set with error alone: refused, the forge or the holder refused and error says why in the forge's own words; not-found, the forge said that what was asked about (a repository, a commit, a pull request, a branch or a tag) does not exist, so a caller never reads it out of error's words; failed, the forge failed or could not be reached, so the act or read could not be performed; away, a part the holder depends on (the forge, its store) is away, with part naming what did not answer and words: a caller reads it as a transport fault and calls again (design 47 H2 §776).
type ErrorKind = string
// The values of ErrorKind.
@@ -231,6 +268,7 @@ const (
ErrorKindRefused = "refused"
ErrorKindNotFound = "not-found"
ErrorKindFailed = "failed"
ErrorKindAway = "away"
)
// FailedStep is a failed required step at a build-input fingerprint, with the commits it was seen at and the run that failed it (design 47 *Judged from its own state, never by a call*); cleared by a later passing rerun of that step whose proved caller is the operator or a healer.
@@ -242,7 +280,9 @@ type FailedStep struct {
Commits []string `json:"commits"`
// the build-input fingerprint the step failed at
Fingerprint string `json:"fingerprint"`
Module string `json:"module"`
// is the form of what this value carries, left out while it is 1 (design 47 H2 §780).
Form int64 `json:"form,omitempty"`
Module string `json:"module"`
// the run that failed it: the runner's pipeline key
Run string `json:"run"`
// why it failed, as the runner's step-ended said
@@ -341,6 +381,13 @@ type Status struct {
Target string `json:"target,omitempty"`
}
// Tag is a tag and the commit it names (design 47 H2 §796).
type Tag struct {
// the commit it names, its full 40-hex sha
Commit string `json:"commit"`
Tag string `json:"tag"`
}
// MergeCalledMembersItem is a type of the contract.
type MergeCalledMembersItem struct {
// the pull request's base, a watched branch
@@ -369,6 +416,8 @@ type MergeCalled struct {
By string `json:"by"`
// the merge call, 16 lowercase hex characters made once per call
Call string `json:"call"`
// is the form of what this value carries, left out while it is 1 (design 47 H2 §780).
Form int64 `json:"form,omitempty"`
// one per landed member
Members []MergeCalledMembersItem `json:"members"`
// one per member not merged after every member was judged (design 47 H §767)
@@ -390,8 +439,14 @@ type PullAnswer struct {
// why the forge could not answer, or refused: set alone with error-kind, and a caller reads it as a failure, never as none
Error string `json:"error,omitempty"`
ErrorKind ErrorKind `json:"error-kind,omitempty"`
Merging *Merging `json:"merging,omitempty"`
Pull *Pull `json:"pull,omitempty"`
// is the form of what this value carries, left out while it is 1 (design 47 H2 §780).
Form int64 `json:"form,omitempty"`
Merging *Merging `json:"merging,omitempty"`
// with the error kind away, what did not answer (design 47 H2 §776)
Part string `json:"part,omitempty"`
Pull *Pull `json:"pull,omitempty"`
// with the error kind away, the holder's words (design 47 H2 §776)
Words string `json:"words,omitempty"`
}
// BranchInput is what branch takes.
@@ -407,9 +462,15 @@ type BranchAnswer struct {
// why the forge could not answer, or refused: set alone with error-kind, and a caller reads it as a failure, never as none
Error string `json:"error,omitempty"`
ErrorKind ErrorKind `json:"error-kind,omitempty"`
// is the form of what this value carries, left out while it is 1 (design 47 H2 §780).
Form int64 `json:"form,omitempty"`
// a commit, its full 40-hex sha
Head string `json:"head,omitempty"`
Head string `json:"head,omitempty"`
// with the error kind away, what did not answer (design 47 H2 §776)
Part string `json:"part,omitempty"`
Repository string `json:"repository,omitempty"`
// with the error kind away, the holder's words (design 47 H2 §776)
Words string `json:"words,omitempty"`
}
// OpenPullsInput is what open-pulls takes.
@@ -421,8 +482,14 @@ type OpenPullsAnswer struct {
// why the forge could not answer, or refused: set alone with error-kind, and a caller reads it as a failure, never as none
Error string `json:"error,omitempty"`
ErrorKind ErrorKind `json:"error-kind,omitempty"`
// is the form of what this value carries, left out while it is 1 (design 47 H2 §780).
Form int64 `json:"form,omitempty"`
// with the error kind away, what did not answer (design 47 H2 §776)
Part string `json:"part,omitempty"`
// every open pull request; said empty when none is open, so an answer that lacks it is never read as none
Pulls []OpenPull `json:"pulls"`
// with the error kind away, the holder's words (design 47 H2 §776)
Words string `json:"words,omitempty"`
}
// TagsInput is what tags takes.
@@ -442,14 +509,20 @@ type TagsAnswer struct {
// why the forge could not answer, or refused: set alone with error-kind, and a caller reads it as a failure, never as none
Error string `json:"error,omitempty"`
ErrorKind ErrorKind `json:"error-kind,omitempty"`
// is the form of what this value carries, left out while it is 1 (design 47 H2 §780).
Form int64 `json:"form,omitempty"`
// another page exists
More bool `json:"more,omitempty"`
// the page
Page int64 `json:"page,omitempty"`
// with the error kind away, what did not answer (design 47 H2 §776)
Part string `json:"part,omitempty"`
// the repository, <owner>/<name>
Repository string `json:"repository,omitempty"`
// the tags of this page
Tags []string `json:"tags,omitempty"`
// the tags of this page, each with the commit it names (design 47 H2 §796)
Tags []Tag `json:"tags,omitempty"`
// with the error kind away, the holder's words (design 47 H2 §776)
Words string `json:"words,omitempty"`
}
// TreesInput is what trees takes.
@@ -473,12 +546,18 @@ type TreesAnswer struct {
ErrorKind ErrorKind `json:"error-kind,omitempty"`
// directory to {path: blob id}
Files map[string]map[string]string `json:"files,omitempty"`
// is the form of what this value carries, left out while it is 1 (design 47 H2 §780).
Form int64 `json:"form,omitempty"`
// with the error kind away, what did not answer (design 47 H2 §776)
Part string `json:"part,omitempty"`
// directory to {path: object id}
Paths map[string]map[string]string `json:"paths,omitempty"`
// the root tree's hash
Root string `json:"root,omitempty"`
// directory to its tree's hash, the root as the empty name
Trees map[string]string `json:"trees,omitempty"`
// with the error kind away, the holder's words (design 47 H2 §776)
Words string `json:"words,omitempty"`
}
// FileInput is what file takes.
@@ -502,12 +581,18 @@ type FileAnswer struct {
// why the forge could not answer, or refused: set alone with error-kind, and a caller reads it as a failure, never as none
Error string `json:"error,omitempty"`
ErrorKind ErrorKind `json:"error-kind,omitempty"`
// is the form of what this value carries, left out while it is 1 (design 47 H2 §780).
Form int64 `json:"form,omitempty"`
// with the error kind away, what did not answer (design 47 H2 §776)
Part string `json:"part,omitempty"`
// the path
Path string `json:"path,omitempty"`
// what was read
Ref string `json:"ref,omitempty"`
// its size in bytes
Size int64 `json:"size,omitempty"`
// with the error kind away, the holder's words (design 47 H2 §776)
Words string `json:"words,omitempty"`
}
// ContainsInput is what contains takes.
@@ -531,8 +616,14 @@ type ContainsAnswer struct {
// why the forge could not answer, or refused: set alone with error-kind, and a caller reads it as a failure, never as none
Error string `json:"error,omitempty"`
ErrorKind ErrorKind `json:"error-kind,omitempty"`
// is the form of what this value carries, left out while it is 1 (design 47 H2 §780).
Form int64 `json:"form,omitempty"`
// the other
In string `json:"in,omitempty"`
// with the error kind away, what did not answer (design 47 H2 §776)
Part string `json:"part,omitempty"`
// with the error kind away, the holder's words (design 47 H2 §776)
Words string `json:"words,omitempty"`
}
// StatusesInput is what statuses takes.
@@ -552,8 +643,14 @@ type StatusesAnswer struct {
// why the forge could not answer, or refused: set alone with error-kind, and a caller reads it as a failure, never as none
Error string `json:"error,omitempty"`
ErrorKind ErrorKind `json:"error-kind,omitempty"`
// is the form of what this value carries, left out while it is 1 (design 47 H2 §780).
Form int64 `json:"form,omitempty"`
// with the error kind away, what did not answer (design 47 H2 §776)
Part string `json:"part,omitempty"`
// the newest of each context
Statuses []Status `json:"statuses,omitempty"`
// with the error kind away, the holder's words (design 47 H2 §776)
Words string `json:"words,omitempty"`
}
// StatusInput is what status takes.
@@ -588,12 +685,18 @@ type StatusAnswer struct {
// why the forge could not answer, or refused: set alone with error-kind, and a caller reads it as a failure, never as none
Error string `json:"error,omitempty"`
ErrorKind ErrorKind `json:"error-kind,omitempty"`
// is the form of what this value carries, left out while it is 1 (design 47 H2 §780).
Form int64 `json:"form,omitempty"`
// the id was already held on the pull request or the note, so nothing was done again
Held bool `json:"held,omitempty"`
// the act's id, as the caller gave it
ID string `json:"id,omitempty"`
// with the error kind away, what did not answer (design 47 H2 §776)
Part string `json:"part,omitempty"`
// the forge holds the act now
Performed bool `json:"performed,omitempty"`
// with the error kind away, the holder's words (design 47 H2 §776)
Words string `json:"words,omitempty"`
}
// CommentInput is what comment takes.
@@ -613,12 +716,18 @@ type CommentAnswer struct {
// why the forge could not answer, or refused: set alone with error-kind, and a caller reads it as a failure, never as none
Error string `json:"error,omitempty"`
ErrorKind ErrorKind `json:"error-kind,omitempty"`
// is the form of what this value carries, left out while it is 1 (design 47 H2 §780).
Form int64 `json:"form,omitempty"`
// the id was already held on the pull request or the note, so nothing was done again
Held bool `json:"held,omitempty"`
// the act's id, as the caller gave it
ID string `json:"id,omitempty"`
// with the error kind away, what did not answer (design 47 H2 §776)
Part string `json:"part,omitempty"`
// the forge holds the act now
Performed bool `json:"performed,omitempty"`
// with the error kind away, the holder's words (design 47 H2 §776)
Words string `json:"words,omitempty"`
}
// ViewInput is what view takes.
@@ -638,12 +747,18 @@ type ViewAnswer struct {
// why the forge could not answer, or refused: set alone with error-kind, and a caller reads it as a failure, never as none
Error string `json:"error,omitempty"`
ErrorKind ErrorKind `json:"error-kind,omitempty"`
// is the form of what this value carries, left out while it is 1 (design 47 H2 §780).
Form int64 `json:"form,omitempty"`
// the id was already held on the pull request or the note, so nothing was done again
Held bool `json:"held,omitempty"`
// the act's id, as the caller gave it
ID string `json:"id,omitempty"`
// with the error kind away, what did not answer (design 47 H2 §776)
Part string `json:"part,omitempty"`
// the forge holds the act now
Performed bool `json:"performed,omitempty"`
// with the error kind away, the holder's words (design 47 H2 §776)
Words string `json:"words,omitempty"`
}
// NoteInput is what note takes.
@@ -663,12 +778,18 @@ type NoteAnswer struct {
// why the forge could not answer, or refused: set alone with error-kind, and a caller reads it as a failure, never as none
Error string `json:"error,omitempty"`
ErrorKind ErrorKind `json:"error-kind,omitempty"`
// is the form of what this value carries, left out while it is 1 (design 47 H2 §780).
Form int64 `json:"form,omitempty"`
// the id was already held on the pull request or the note, so nothing was done again
Held bool `json:"held,omitempty"`
// the act's id, as the caller gave it
ID string `json:"id,omitempty"`
// with the error kind away, what did not answer (design 47 H2 §776)
Part string `json:"part,omitempty"`
// the forge holds the act now
Performed bool `json:"performed,omitempty"`
// with the error kind away, the holder's words (design 47 H2 §776)
Words string `json:"words,omitempty"`
}
// MergePullInputPullsItem is a type of the contract.
@@ -696,8 +817,14 @@ type MergePullAnswer struct {
// why the forge could not answer, or refused: set alone with error-kind, and a caller reads it as a failure, never as none
Error string `json:"error,omitempty"`
ErrorKind ErrorKind `json:"error-kind,omitempty"`
// is the form of what this value carries, left out while it is 1 (design 47 H2 §780).
Form int64 `json:"form,omitempty"`
// each member's outcome, in the order named
Members []MemberOutcome `json:"members,omitempty"`
// with the error kind away, what did not answer (design 47 H2 §776)
Part string `json:"part,omitempty"`
// with the error kind away, the holder's words (design 47 H2 §776)
Words string `json:"words,omitempty"`
}
// TagInput is what tag takes.
@@ -715,10 +842,16 @@ type TagAnswer struct {
// why the forge could not write it: set alone with error-kind
Error string `json:"error,omitempty"`
ErrorKind ErrorKind `json:"error-kind,omitempty"`
// is the form of what this value carries, left out while it is 1 (design 47 H2 §780).
Form int64 `json:"form,omitempty"`
// with the error kind away, what did not answer (design 47 H2 §776)
Part string `json:"part,omitempty"`
// the forge holds the tag at that commit now
Performed bool `json:"performed,omitempty"`
// the tag, as asked
Tag string `json:"tag,omitempty"`
// with the error kind away, the holder's words (design 47 H2 §776)
Words string `json:"words,omitempty"`
}
// PullUpdatedEvent is the body of the event pull-updated.
@@ -732,6 +865,8 @@ type PullUpdatedEvent struct {
// its head branch
Branch string `json:"branch"`
Change *Change `json:"change,omitempty"`
// is the form of what this value carries, left out while it is 1 (design 47 H2 §780).
Form int64 `json:"form,omitempty"`
// a commit, its full 40-hex sha
Head string `json:"head"`
// the pull request's number
@@ -753,6 +888,8 @@ type PullMergedEvent struct {
// its head branch
Branch string `json:"branch"`
Change *Change `json:"change,omitempty"`
// is the form of what this value carries, left out while it is 1 (design 47 H2 §780).
Form int64 `json:"form,omitempty"`
// a commit, its full 40-hex sha
Head string `json:"head"`
// the head's statuses by context, their states; absent when they could not be read
@@ -779,6 +916,8 @@ type PullClosedEvent struct {
Base string `json:"base"`
// its head branch
Branch string `json:"branch"`
// is the form of what this value carries, left out while it is 1 (design 47 H2 §780).
Form int64 `json:"form,omitempty"`
// a commit, its full 40-hex sha
Head string `json:"head"`
// the pull request's number
@@ -801,6 +940,8 @@ type ForgePullEvent struct {
Base string `json:"base,omitempty"`
// its head branch
Branch string `json:"branch,omitempty"`
// is the form of what this value carries, left out while it is 1 (design 47 H2 §780).
Form int64 `json:"form,omitempty"`
// its head commit
Head string `json:"head,omitempty"`
// the merge commit
@@ -823,6 +964,8 @@ type ForgeStatusEvent struct {
Context string `json:"context"`
// what it says
Description string `json:"description,omitempty"`
// is the form of what this value carries, left out while it is 1 (design 47 H2 §780).
Form int64 `json:"form,omitempty"`
// the repository, <owner>/<name>
Repository string `json:"repository"`
// who set it
@@ -833,6 +976,8 @@ type ForgeStatusEvent struct {
// ForgeDeleteEvent is the body of the event forge-delete.
type ForgeDeleteEvent struct {
// is the form of what this value carries, left out while it is 1 (design 47 H2 §780).
Form int64 `json:"form,omitempty"`
// the deleted branch or tag
Ref string `json:"ref"`
// branch or tag
@@ -847,6 +992,8 @@ type ForgeDeleteEvent struct {
type RepoWritableEvent struct {
// the account that writes it now
Account string `json:"account"`
// is the form of what this value carries, left out while it is 1 (design 47 H2 §780).
Form int64 `json:"form,omitempty"`
// the repository, <owner>/<name>
Repository string `json:"repository"`
}
@@ -855,6 +1002,8 @@ type RepoWritableEvent struct {
type BranchMovedEvent struct {
At *time.Time `json:"at,omitempty"`
Branch string `json:"branch"`
// is the form of what this value carries, left out while it is 1 (design 47 H2 §780).
Form int64 `json:"form,omitempty"`
// the head the webhook said, a commit's full 40-hex sha; read again by branch
Head string `json:"head,omitempty"`
// the repository, <owner>/<name>
@@ -869,3 +1018,221 @@ type FailedStepsByModuleFingerprintStepValue = FailedStep
// CallsByCallCallValue is one value of the state calls under the key call.<call>.
type CallsByCallCallValue = Call
// DecodePullAnswer reads a PullAnswer as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodePullAnswer(raw []byte) (PullAnswer, int, error) {
var v PullAnswer
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
return v, formRead(raw), nil
}
// DecodeBranchAnswer reads a BranchAnswer as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodeBranchAnswer(raw []byte) (BranchAnswer, int, error) {
var v BranchAnswer
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
return v, formRead(raw), nil
}
// DecodeOpenPullsAnswer reads a OpenPullsAnswer as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodeOpenPullsAnswer(raw []byte) (OpenPullsAnswer, int, error) {
var v OpenPullsAnswer
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
return v, formRead(raw), nil
}
// DecodeTagsAnswer reads a TagsAnswer as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodeTagsAnswer(raw []byte) (TagsAnswer, int, error) {
var v TagsAnswer
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
return v, formRead(raw), nil
}
// DecodeTreesAnswer reads a TreesAnswer as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodeTreesAnswer(raw []byte) (TreesAnswer, int, error) {
var v TreesAnswer
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
return v, formRead(raw), nil
}
// DecodeFileAnswer reads a FileAnswer as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodeFileAnswer(raw []byte) (FileAnswer, int, error) {
var v FileAnswer
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
return v, formRead(raw), nil
}
// DecodeContainsAnswer reads a ContainsAnswer as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodeContainsAnswer(raw []byte) (ContainsAnswer, int, error) {
var v ContainsAnswer
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
return v, formRead(raw), nil
}
// DecodeStatusesAnswer reads a StatusesAnswer as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodeStatusesAnswer(raw []byte) (StatusesAnswer, int, error) {
var v StatusesAnswer
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
return v, formRead(raw), nil
}
// DecodeStatusAnswer reads a StatusAnswer as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodeStatusAnswer(raw []byte) (StatusAnswer, int, error) {
var v StatusAnswer
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
return v, formRead(raw), nil
}
// DecodeCommentAnswer reads a CommentAnswer as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodeCommentAnswer(raw []byte) (CommentAnswer, int, error) {
var v CommentAnswer
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
return v, formRead(raw), nil
}
// DecodeViewAnswer reads a ViewAnswer as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodeViewAnswer(raw []byte) (ViewAnswer, int, error) {
var v ViewAnswer
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
return v, formRead(raw), nil
}
// DecodeNoteAnswer reads a NoteAnswer as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodeNoteAnswer(raw []byte) (NoteAnswer, int, error) {
var v NoteAnswer
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
return v, formRead(raw), nil
}
// DecodeMergePullAnswer reads a MergePullAnswer as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodeMergePullAnswer(raw []byte) (MergePullAnswer, int, error) {
var v MergePullAnswer
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
return v, formRead(raw), nil
}
// DecodeTagAnswer reads a TagAnswer as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodeTagAnswer(raw []byte) (TagAnswer, int, error) {
var v TagAnswer
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
return v, formRead(raw), nil
}
// DecodePullUpdatedEvent reads a PullUpdatedEvent as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodePullUpdatedEvent(raw []byte) (PullUpdatedEvent, int, error) {
var v PullUpdatedEvent
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
return v, formRead(raw), nil
}
// DecodePullMergedEvent reads a PullMergedEvent as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodePullMergedEvent(raw []byte) (PullMergedEvent, int, error) {
var v PullMergedEvent
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
return v, formRead(raw), nil
}
// DecodePullClosedEvent reads a PullClosedEvent as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodePullClosedEvent(raw []byte) (PullClosedEvent, int, error) {
var v PullClosedEvent
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
return v, formRead(raw), nil
}
// DecodeForgePullEvent reads a ForgePullEvent as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodeForgePullEvent(raw []byte) (ForgePullEvent, int, error) {
var v ForgePullEvent
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
return v, formRead(raw), nil
}
// DecodeForgeStatusEvent reads a ForgeStatusEvent as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodeForgeStatusEvent(raw []byte) (ForgeStatusEvent, int, error) {
var v ForgeStatusEvent
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
return v, formRead(raw), nil
}
// DecodeForgeDeleteEvent reads a ForgeDeleteEvent as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodeForgeDeleteEvent(raw []byte) (ForgeDeleteEvent, int, error) {
var v ForgeDeleteEvent
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
return v, formRead(raw), nil
}
// DecodeRepoWritableEvent reads a RepoWritableEvent as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodeRepoWritableEvent(raw []byte) (RepoWritableEvent, int, error) {
var v RepoWritableEvent
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
return v, formRead(raw), nil
}
// DecodeBranchMovedEvent reads a BranchMovedEvent as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodeBranchMovedEvent(raw []byte) (BranchMovedEvent, int, error) {
var v BranchMovedEvent
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
return v, formRead(raw), nil
}
// DecodeMergeCalledEvent reads a MergeCalledEvent as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodeMergeCalledEvent(raw []byte) (MergeCalledEvent, int, error) {
var v MergeCalledEvent
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
return v, formRead(raw), nil
}
// formRead is the form a value says on the wire: 1 when it says none, or is no object.
func formRead(raw []byte) int {
var f struct {
Form int `json:"form"`
}
if json.Unmarshal(raw, &f) != nil || f.Form < 1 {
return 1
}
return f.Form
}
@@ -61,7 +61,7 @@ func TestFixtures(t *testing.T) {
decode: func(b []byte) (any, error) { var v OpenPullsAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil},
{name: "tags asks", subject: "", given: "{\"page\":1,\"prefix\":\"go/v\",\"repository\":\"novox/mesh-sdk\"}", wire: "{\"page\":1,\"prefix\":\"go/v\",\"repository\":\"novox/mesh-sdk\"}",
decode: func(b []byte) (any, error) { var v TagsInput; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil},
{name: "tags answers", subject: "", given: "{\"bound\":50,\"more\":true,\"page\":1,\"repository\":\"novox/mesh-sdk\",\"tags\":[\"go/v0.1.15\",\"go/v0.1.14\"]}", wire: "{\"bound\":50,\"more\":true,\"page\":1,\"repository\":\"novox/mesh-sdk\",\"tags\":[\"go/v0.1.15\",\"go/v0.1.14\"]}",
{name: "tags answers", subject: "", given: "{\"bound\":50,\"more\":true,\"page\":1,\"repository\":\"novox/mesh-sdk\",\"tags\":[{\"commit\":\"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\",\"tag\":\"go/v0.1.15\"},{\"commit\":\"cccccccccccccccccccccccccccccccccccccccc\",\"tag\":\"go/v0.1.14\"}]}", wire: "{\"bound\":50,\"more\":true,\"page\":1,\"repository\":\"novox/mesh-sdk\",\"tags\":[{\"commit\":\"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\",\"tag\":\"go/v0.1.15\"},{\"commit\":\"cccccccccccccccccccccccccccccccccccccccc\",\"tag\":\"go/v0.1.14\"}]}",
decode: func(b []byte) (any, error) { var v TagsAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil},
{name: "tags answers an error, never none", subject: "", given: "{\"error\":\"the forge answered 404 for novox/nothing\",\"error-kind\":\"refused\"}", wire: "{\"error\":\"the forge answered 404 for novox/nothing\",\"error-kind\":\"refused\"}",
decode: func(b []byte) (any, error) { var v TagsAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil},
@@ -139,6 +139,10 @@ func TestFixtures(t *testing.T) {
decode: func(b []byte) (any, error) { var v MergePullAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil},
{name: "merge-pull answers an error, never none", subject: "", given: "{\"error\":\"the forge answered 502\",\"error-kind\":\"failed\"}", wire: "{\"error\":\"the forge answered 502\",\"error-kind\":\"failed\"}",
decode: func(b []byte) (any, error) { var v MergePullAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil},
{name: "trees answers away while the forge's store is away", subject: "", given: "{\"error\":\"away: the forge: its store is unavailable\",\"error-kind\":\"away\",\"part\":\"the forge\",\"words\":\"its store is unavailable\"}", wire: "{\"error\":\"away: the forge: its store is unavailable\",\"error-kind\":\"away\",\"part\":\"the forge\",\"words\":\"its store is unavailable\"}",
decode: func(b []byte) (any, error) { var v TreesAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil},
{name: "trees answers a missing directory as not-found", subject: "", given: "{\"error\":\"modules/gone is not in aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"error-kind\":\"not-found\"}", wire: "{\"error\":\"modules/gone is not in aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"error-kind\":\"not-found\"}",
decode: func(b []byte) (any, error) { var v TreesAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil},
}
for _, c := range cases {
v, err := c.decode([]byte(c.given))
@@ -3,9 +3,27 @@
// Package gopackageregistry is the contract of the seat "go-package-registry", defined whole (seats/go-package-registry/seat.json): The mesh's Go modules: Go's module proxy protocol's read side, and publishing over the bus.
package gopackageregistry
import (
"encoding/json"
)
// Seat is the seat's name.
const Seat = "go-package-registry"
// Form is the form of the seat's values, events and answers this contract was generated at (design 47 H2 §780):
// a reader of a value of another form says "form N, this reader knows M".
const Form = 1
// FormRegistry is the form of what the part registry adds.
const FormRegistry = 1
// The forms and the digest of the fields this contract was generated from: the generator's compare fails a definition
// whose fields changed while its forms did not (design 47 H2 §780).
const (
generatedForms = "1 registry:1"
generatedFieldsDigest = "Artifact=79361e4ab1cd KeptEvent=ec3ef89ef8a7 Outcome=1e66f7bf6725 PublishAnswer=a0df0b922e1e PublishPartAnswer=aebe3c6c89e9 PublishedEvent=ec3ef89ef8a7 RefusedEvent=ec3ef89ef8a7 RemoveAnswer=bccb5a322d69 RemovedEvent=ec3ef89ef8a7 VersionsAnswer=47e11ca2aea9"
)
const (
VerbPublish = "publish"
VerbPublishPart = "publish-part"
@@ -25,6 +43,22 @@ func Verbs() []string {
return []string{VerbPublish, VerbPublishPart, VerbVersions, VerbRemove}
}
// VerbDescription is a verb's description as the seat's definition says it, "" for a verb the seat does not serve or
// does not describe (design 47 H2 H-A-449).
func VerbDescription(verb string) string {
switch verb {
case VerbPublish:
return "Open a transfer, naming the artifact and the digest of every part; answers the digests the holder lacks, which the caller then sends by publish-part. A transfer already open answers the parts held, so a cut transfer goes on from there. With no parts and a version, the held digest is published under that version and nothing is stored (the trunk's claim)."
case VerbPublishPart:
return "Send one chunk of one part of an open transfer; the last chunk of the last part ends the transfer, and the holder hashes what it received, asks builder_versions, stores it and says published or refused. A chunk's message stays under the bus's largest message (1 MiB), so at most 760 KiB of data, base64."
case VerbVersions:
return "The versions the seat holds of one name, newest first: each one's version, tag, digest, kind and when it was published."
case VerbRemove:
return "Remove one tagged version; the holder refuses an untagged one but for the retention removal of a bundle, archive or copied base, which the artifact sweep's holder asks; the caller is not judged (design 47 H2 §813 (7)). Every removal emits removed with its reason. A tagged version the store cannot remove alone, because another tag names its digest, stays whole, its tag and bytes kept: the answer says kept with the reason shared-digest and the tags still naming the digest, removed false, and kept is emitted, never removed; the caller ends there without a retry."
}
return ""
}
// Events is every event published on the seat's subject: a new copy each call.
func Events() []string {
return []string{EventPublished, EventRefused, EventRemoved, EventKept}
@@ -83,6 +117,8 @@ type Outcome struct {
Digest string `json:"digest"`
// the build's fingerprint
Fingerprint string `json:"fingerprint,omitempty"`
// is the form of what this value carries, left out while it is 1 (design 47 H2 §780).
Form int64 `json:"form,omitempty"`
// the module holding the seat that said it
Holder string `json:"holder"`
// what it is
@@ -155,6 +191,8 @@ const (
// PublishAnswer is what publish answers.
type PublishAnswer struct {
// is the form of what this value carries, left out while it is 1 (design 47 H2 §780).
Form int64 `json:"form,omitempty"`
// the digest is held already: nothing to send; the outcome follows as an event
Held bool `json:"held,omitempty"`
// the digests of the parts the holder lacks, to send by publish-part
@@ -181,6 +219,8 @@ type PublishPartInput struct {
type PublishPartAnswer struct {
// the part is whole and its bytes match its digest
Complete bool `json:"complete,omitempty"`
// is the form of what this value carries, left out while it is 1 (design 47 H2 §780).
Form int64 `json:"form,omitempty"`
// the part
Part string `json:"part"`
// the chunks of the part held so far
@@ -199,6 +239,8 @@ type VersionsInput struct {
// VersionsAnswer is what versions answers.
type VersionsAnswer struct {
// is the form of what this value carries, left out while it is 1 (design 47 H2 §780).
Form int64 `json:"form,omitempty"`
// the name
Name string `json:"name"`
// newest first
@@ -228,6 +270,8 @@ const (
// RemoveAnswer is what remove answers.
type RemoveAnswer struct {
Artifact *Artifact `json:"artifact,omitempty"`
// is the form of what this value carries, left out while it is 1 (design 47 H2 §780).
Form int64 `json:"form,omitempty"`
// it stays whole, its tag and its bytes: the store cannot remove the tag alone while another tag names its digest (reason, tags)
Kept bool `json:"kept,omitempty"`
// why it was kept: shared-digest, another tag names its digest
@@ -260,3 +304,86 @@ type GoPackageRegistryForm struct {
// the read side's address
URL string `json:"url"`
}
// DecodePublishAnswer reads a PublishAnswer as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodePublishAnswer(raw []byte) (PublishAnswer, int, error) {
var v PublishAnswer
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
return v, formRead(raw), nil
}
// DecodePublishPartAnswer reads a PublishPartAnswer as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodePublishPartAnswer(raw []byte) (PublishPartAnswer, int, error) {
var v PublishPartAnswer
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
return v, formRead(raw), nil
}
// DecodeVersionsAnswer reads a VersionsAnswer as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodeVersionsAnswer(raw []byte) (VersionsAnswer, int, error) {
var v VersionsAnswer
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
return v, formRead(raw), nil
}
// DecodeRemoveAnswer reads a RemoveAnswer as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodeRemoveAnswer(raw []byte) (RemoveAnswer, int, error) {
var v RemoveAnswer
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
return v, formRead(raw), nil
}
// DecodePublishedEvent reads a PublishedEvent as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodePublishedEvent(raw []byte) (PublishedEvent, int, error) {
var v PublishedEvent
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
return v, formRead(raw), nil
}
// DecodeRefusedEvent reads a RefusedEvent as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodeRefusedEvent(raw []byte) (RefusedEvent, int, error) {
var v RefusedEvent
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
return v, formRead(raw), nil
}
// DecodeRemovedEvent reads a RemovedEvent as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodeRemovedEvent(raw []byte) (RemovedEvent, int, error) {
var v RemovedEvent
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
return v, formRead(raw), nil
}
// DecodeKeptEvent reads a KeptEvent as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodeKeptEvent(raw []byte) (KeptEvent, int, error) {
var v KeptEvent
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
return v, formRead(raw), nil
}
// formRead is the form a value says on the wire: 1 when it says none, or is no object.
func formRead(raw []byte) int {
var f struct {
Form int `json:"form"`
}
if json.Unmarshal(raw, &f) != nil || f.Form < 1 {
return 1
}
return f.Form
}
@@ -3,11 +3,25 @@
// Package meshartifactsweep is the contract of the seat "mesh-artifact-sweep", defined whole (seats/mesh-artifact-sweep/seat.json): no description.
package meshartifactsweep
import "time"
import (
"encoding/json"
"time"
)
// Seat is the seat's name.
const Seat = "mesh-artifact-sweep"
// Form is the form of the seat's values, events and answers this contract was generated at (design 47 H2 §780):
// a reader of a value of another form says "form N, this reader knows M".
const Form = 1
// The forms and the digest of the fields this contract was generated from: the generator's compare fails a definition
// whose fields changed while its forms did not (design 47 H2 §780).
const (
generatedForms = "1"
generatedFieldsDigest = "ArtifactsAnswer=7538f2f5e68e ArtifactsGone=191972da3c7b ArtifactsGoneEvent=28bdc3f8c44d CollectAnswer=7538f2f5e68e HeldAnswer=a5c68f6effa8 ImagesAnswer=7538f2f5e68e MirrorsAnswer=7538f2f5e68e"
)
const (
VerbArtifacts = "artifacts"
VerbHeld = "held"
@@ -25,6 +39,24 @@ func Verbs() []string {
return []string{VerbArtifacts, VerbHeld, VerbCollect, VerbMirrors, VerbImages}
}
// VerbDescription is a verb's description as the seat's definition says it, "" for a verb the seat does not serve or
// does not describe (design 47 H2 H-A-449).
func VerbDescription(verb string) string {
switch verb {
case VerbArtifacts:
return "Every artifact the mesh made that lives in its artifact store, each with its repository, digest, kind (image or archive) and state: kept, and why (a want names it, or one of the five newest builds of its module in the builder's log), eligible (made, kept for no reason, not yet let go), or unrecorded (in the store and named by nothing the mesh made: never removed). Reads only."
case VerbHeld:
return "Whether the artifact store holds each reference given, asked with one HEAD each: held, gone (the store answered 404) or unknown with why. The builder asks it before it reuses a done key. Reads only."
case VerbCollect:
return "The artifact store's sweep, now: what is eligible is let go of. Without confirm a dry run: what would be let go, nothing held or deleted. With confirm and why: every kept archive held first, then each eligible artifact let go of, oldest first, an index with the platform manifests no kept index names, each deletion recorded in the hand-act log. Bounded by most (500 by default, at most 5000) and 45 seconds; what is left is said. Never touches a digest the mesh did not make."
case VerbMirrors:
return "The bases the mesh copied into its artifact store, each kept (a kept build stood on it), eligible (only builds no longer kept stood on it) or unrecorded (no build the builder's log holds stood on it: never removed), by what the builder's log says each build stood on. Records nothing by hand."
case VerbImages:
return "Every container image one machine's want and have name, each with the resources naming it: what the machine keeps when it prunes its images. Reads only."
}
return ""
}
// Events is every event published on the seat's subject: a new copy each call.
func Events() []string {
return []string{EventArtifactsGone}
@@ -47,6 +79,8 @@ type ArtifactsGone struct {
Event string `json:"event"`
// the key's fingerprint
Fingerprint string `json:"fingerprint"`
// is the form of what this value carries, left out while it is 1 (design 47 H2 §780).
Form int64 `json:"form,omitempty"`
// every reference of the key the store does not hold, sorted
Gone []string `json:"gone"`
// the builder's done key
@@ -80,6 +114,8 @@ type HeldInput struct {
// HeldAnswer is what held answers.
type HeldAnswer struct {
// is the form of what this value carries, left out while it is 1 (design 47 H2 §780).
Form int64 `json:"form,omitempty"`
// the references the store answered 404 for
Gone []string `json:"gone"`
// the references the store holds
@@ -127,3 +163,68 @@ type ImagesAnswer = map[string]any
// ArtifactsGoneEvent is the body of the event artifacts-gone.
type ArtifactsGoneEvent = ArtifactsGone
// DecodeArtifactsAnswer reads a ArtifactsAnswer as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodeArtifactsAnswer(raw []byte) (ArtifactsAnswer, int, error) {
var v ArtifactsAnswer
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
return v, formRead(raw), nil
}
// DecodeHeldAnswer reads a HeldAnswer as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodeHeldAnswer(raw []byte) (HeldAnswer, int, error) {
var v HeldAnswer
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
return v, formRead(raw), nil
}
// DecodeCollectAnswer reads a CollectAnswer as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodeCollectAnswer(raw []byte) (CollectAnswer, int, error) {
var v CollectAnswer
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
return v, formRead(raw), nil
}
// DecodeMirrorsAnswer reads a MirrorsAnswer as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodeMirrorsAnswer(raw []byte) (MirrorsAnswer, int, error) {
var v MirrorsAnswer
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
return v, formRead(raw), nil
}
// DecodeImagesAnswer reads a ImagesAnswer as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodeImagesAnswer(raw []byte) (ImagesAnswer, int, error) {
var v ImagesAnswer
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
return v, formRead(raw), nil
}
// DecodeArtifactsGoneEvent reads a ArtifactsGoneEvent as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodeArtifactsGoneEvent(raw []byte) (ArtifactsGoneEvent, int, error) {
var v ArtifactsGoneEvent
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
return v, formRead(raw), nil
}
// formRead is the form a value says on the wire: 1 when it says none, or is no object.
func formRead(raw []byte) int {
var f struct {
Form int `json:"form"`
}
if json.Unmarshal(raw, &f) != nil || f.Form < 1 {
return 1
}
return f.Form
}
@@ -3,14 +3,30 @@
// Package meshbuilds is the contract of the seat "mesh-builds", defined whole (seats/mesh-builds/seat.json): The builder's seat: it says each artifact it published and each agreed toolchain, and answers a module's reserved version. Held once for the mesh (design 47 G section 52, section 307).
package meshbuilds
import "time"
import (
"encoding/json"
"time"
)
// Seat is the seat's name.
const Seat = "mesh-builds"
// Form is the form of the seat's values, events and answers this contract was generated at (design 47 H2 §780):
// a reader of a value of another form says "form N, this reader knows M".
const Form = 1
// The forms and the digest of the fields this contract was generated from: the generator's compare fails a definition
// whose fields changed while its forms did not (design 47 H2 §780).
const (
VerbReserved = "reserved"
VerbPublished = "published"
generatedForms = "1"
generatedFieldsDigest = "AskTakenEvent=6d5d3da7ac39 BuildEntry=3c829a0a3d58 Dependency=90a32b07c563 DropAnswer=7538f2f5e68e PackagePublished=34dd069a6aab PackagePublishedEvent=8e12888c1559 PublishQueueAnswer=0f1326e35e17 PublishQueueAnswerCountsItem=b770db2f0920 PublishQueueAnswerQueueItem=b1eebc09f41e PublishedAnswer=69a801982819 ReservedAnswer=2272a4d6f0f4 ReservedAnswerArtifactsItem=d74eb6e00abb StepAccept=18efeefb6c34 StepEnded=09a07bfbcb15 StepEndedEvent=207787118dac StepRun=dc5893aef7e5 ToolchainAgreed=f3d9ce993734 ToolchainAgreedDigestsItem=1d42893a96b9 ToolchainAgreedEvent=e3502d6f194d VersionReservedEvent=984547d6ad07"
)
const (
VerbReserved = "reserved"
VerbPublished = "published"
VerbDrop = "drop"
VerbPublishQueue = "publish-queue"
)
const (
@@ -21,16 +37,34 @@ const (
EventPackagePublished = "package-published"
EventToolchainAgreed = "toolchain-agreed"
EventStepEnded = "step-ended"
EventAskTaken = "ask-taken"
EventVersionReserved = "version-reserved"
)
// Verbs is every verb the seat serves, in its definition's order: a new copy each call.
func Verbs() []string {
return []string{VerbReserved, VerbPublished}
return []string{VerbReserved, VerbPublished, VerbDrop, VerbPublishQueue}
}
// VerbDescription is a verb's description as the seat's definition says it, "" for a verb the seat does not serve or
// does not describe (design 47 H2 H-A-449).
func VerbDescription(verb string) string {
switch verb {
case VerbReserved:
return "A module's reserved version with each artifact's digest, for a registry holder deciding whether to publish; asked by the module or by a package's name in its ecosystem (design 47 G section 369, section 478; design 51, Publishing)."
case VerbPublished:
return "Whether an artifact built at a build fingerprint is published, from the builder's reservations and the registries' record, for the pipeline runner writing a mover's wait on a package: a package published before the wait was written is read, not waited for (design 47 H-A-326, H-A-328, H-A-274)."
case VerbDrop:
return "End an ask on the seat's work queue for a caller that no longer waits for it (its step ended by its bound): one waiting is ended dropped when it is pulled, one taken is cancelled; both end the step could-not-run, its why naming the drop and who asked it, so no holder works on it after (design 47 H2 §778; H-A-401; ADR 0664 §4)."
case VerbPublishQueue:
return "What the builder's work queue publishes holds, read by the builder from its own stream (design 47 H2 §793; H-A-454): the items' count per registry seat holder and class with the oldest and newest queue time, and the first 50 in the order they are pulled (the live class first, each class oldest first); module, and version, list only that module's items. A queue read past 4,000 items is answered capped, naming the items read and the queue's size."
}
return ""
}
// Events is every event published on the seat's subject: a new copy each call.
func Events() []string {
return []string{EventPackagePublished, EventToolchainAgreed, EventStepEnded}
return []string{EventPackagePublished, EventToolchainAgreed, EventStepEnded, EventAskTaken, EventVersionReserved}
}
// Family is where the seat's events are kept: the shared events stream, filtered to the seat's subjects.
@@ -60,6 +94,15 @@ func SubjectStepEnded(node string) string { return "mesh.seat.mesh-builds.event.
// SubjectStepEndedAll hears every step-ended.
const SubjectStepEndedAll = "mesh.seat.mesh-builds.event.step-ended.*"
// SubjectAskTaken is where ask-taken is published, on mesh.seat.mesh-builds.event.ask-taken.<ask>.
func SubjectAskTaken(ask string) string { return "mesh.seat.mesh-builds.event.ask-taken." + ask }
// SubjectAskTakenAll hears every ask-taken.
const SubjectAskTakenAll = "mesh.seat.mesh-builds.event.ask-taken.*"
// SubjectVersionReserved is where version-reserved is published.
const SubjectVersionReserved = "mesh.seat.mesh-builds.event.version-reserved"
// AcceptSubjectStep is where step is sent.
const AcceptSubjectStep = "mesh.seat.mesh-builds.accept.step"
@@ -94,6 +137,8 @@ type PackagePublished struct {
Dependencies []Dependency `json:"dependencies,omitempty"`
// the published artifact's digest
Digest string `json:"digest"`
// is the form of what this value carries, left out while it is 1 (design 47 H2 §780).
Form int64 `json:"form,omitempty"`
// the module
Module string `json:"module"`
// the catalog revision number of that branch the build was made from, the indexer's count (design 47 H-A-218, H-A-220); absent from a reservation made before the field
@@ -112,6 +157,8 @@ type StepEnded struct {
Ended *time.Time `json:"ended,omitempty"`
// each entry the step built, by module and fingerprint
Entries []BuildEntry `json:"entries"`
// is the form of what this value carries, left out while it is 1 (design 47 H2 §780).
Form int64 `json:"form,omitempty"`
// the ask's id
ID string `json:"id"`
// the node the builder ran it on, as in the subject
@@ -180,6 +227,8 @@ type ToolchainAgreed struct {
At *time.Time `json:"at,omitempty"`
// each digest the nodes reporting the agreed version report, as it stood when the version was agreed, with those nodes (design 47 G section 472, section 495)
Digests []ToolchainAgreedDigestsItem `json:"digests"`
// is the form of what this value carries, left out while it is 1 (design 47 H2 §780).
Form int64 `json:"form,omitempty"`
// the language whose agreed toolchain changed
Language string `json:"language"`
V int64 `json:"v"`
@@ -210,6 +259,8 @@ type ReservedAnswer struct {
Artifacts []ReservedAnswerArtifactsItem `json:"artifacts,omitempty"`
// a name no entry carries, or a version neither reserved nor released, naming which; set alone (design 47 G 582, 679)
Error string `json:"error,omitempty"`
// is the form of what this value carries, left out while it is 1 (design 47 H2 §780).
Form int64 `json:"form,omitempty"`
// the module
Module string `json:"module,omitempty"`
// true when the version is released: the catalog holds it, and artifacts are the ones its release recorded with their digests; false when it is reserved (design 47 G 582, 679)
@@ -234,12 +285,99 @@ type PublishedAnswer struct {
At *time.Time `json:"at,omitempty"`
// the published artifact's digest, when published; absent for a release kept before the digest was kept (design 47 H-A-331)
Digest string `json:"digest,omitempty"`
// is the form of what this value carries, left out while it is 1 (design 47 H2 §780).
Form int64 `json:"form,omitempty"`
// true when the artifact built at the fingerprint is published; a number a take-over withdrew is never published (design 47 H-A-331)
Published bool `json:"published"`
// the version it is published at, MAJOR.MINOR.PATCH, when published
Version string `json:"version,omitempty"`
}
// DropInput is what drop takes.
type DropInput struct {
// who drops it
By string `json:"by,omitempty"`
// the ask's id
ID string `json:"id"`
// why the caller no longer waits
Why string `json:"why"`
}
// DropAnswer is what drop answers.
type DropAnswer = map[string]any
// PublishQueueInput is what publish-queue takes.
type PublishQueueInput struct {
// list only this module's items
Module string `json:"module,omitempty"`
// with module: only this version's, MAJOR.MINOR.PATCH
Version string `json:"version,omitempty"`
}
// PublishQueueAnswerCountsItem is a type of the contract.
type PublishQueueAnswerCountsItem struct {
// the items' class
Class string `json:"class"`
// how many items of the class the holder's queue holds
Count int64 `json:"count"`
// the registry seat's holder
Holder string `json:"holder"`
// when the newest of them was queued
Newest time.Time `json:"newest"`
// when the oldest of them was queued
Oldest time.Time `json:"oldest"`
}
// The values of PublishQueueAnswerCountsItem's class.
const (
PublishQueueAnswerCountsItemClassLive = "live"
PublishQueueAnswerCountsItemClassAgain = "again"
)
// PublishQueueAnswerQueueItem is a type of the contract.
type PublishQueueAnswerQueueItem struct {
// each artifact reference, <module>:<artifact>, with the digest to publish
Artifacts map[string]string `json:"artifacts,omitempty"`
// live, a build a pipeline's step waits on, pulled first; again, asked again at a start
Class string `json:"class"`
// the registry seat's holder the item is sent to, a module
Holder string `json:"holder"`
// the module whose version is published
Module string `json:"module"`
// when the item was put on the work queue
Queued time.Time `json:"queued"`
// the reserved version, MAJOR.MINOR.PATCH
Version string `json:"version"`
}
// The values of PublishQueueAnswerQueueItem's class.
const (
PublishQueueAnswerQueueItemClassLive = "live"
PublishQueueAnswerQueueItemClassAgain = "again"
)
// PublishQueueAnswer is what publish-queue answers.
type PublishQueueAnswer struct {
// true when the queue held more items than were read; every count is then of the items read
Capped bool `json:"capped,omitempty"`
// per holder and class, sorted by holder, then class
Counts []PublishQueueAnswerCountsItem `json:"counts"`
// is the form of what this value carries, left out while it is 1 (design 47 H2 §780).
Form int64 `json:"form,omitempty"`
// with capped: how many items were read
ItemsRead int64 `json:"items-read,omitempty"`
// how many of total the answer lists
Listed int64 `json:"listed"`
// the first 50 items listed, per holder in the order they are pulled
Queue []PublishQueueAnswerQueueItem `json:"queue"`
// with capped: the queue's size in bytes
QueueBytes int64 `json:"queue-bytes,omitempty"`
// with capped: how many items the queue holds, from its stream's state
QueueItems int64 `json:"queue-items,omitempty"`
// how many items were read
Total int64 `json:"total"`
}
// StepAccept is the body of step.
type StepAccept = StepRun
@@ -251,3 +389,121 @@ type ToolchainAgreedEvent = ToolchainAgreed
// StepEndedEvent is the body of the event step-ended.
type StepEndedEvent = StepEnded
// AskTakenEvent is the body of the event ask-taken.
type AskTakenEvent struct {
// the ask's id
Ask string `json:"ask"`
// when it was taken
At time.Time `json:"at"`
// is the form of what this value carries, left out while it is 1 (design 47 H2 §780).
Form int64 `json:"form,omitempty"`
// the node whose holder took it
Node string `json:"node"`
}
// VersionReservedEvent is the body of the event version-reserved.
type VersionReservedEvent struct {
Artifact string `json:"artifact"`
Fingerprint string `json:"fingerprint"`
// is the form of what this value carries, left out while it is 1 (design 47 H2 §780).
Form int64 `json:"form,omitempty"`
Module string `json:"module"`
// the package's path
Path string `json:"path"`
// the catalog revision the reservation was made at
Revision string `json:"revision"`
Version string `json:"version"`
}
// DecodeReservedAnswer reads a ReservedAnswer as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodeReservedAnswer(raw []byte) (ReservedAnswer, int, error) {
var v ReservedAnswer
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
return v, formRead(raw), nil
}
// DecodePublishedAnswer reads a PublishedAnswer as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodePublishedAnswer(raw []byte) (PublishedAnswer, int, error) {
var v PublishedAnswer
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
return v, formRead(raw), nil
}
// DecodeDropAnswer reads a DropAnswer as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodeDropAnswer(raw []byte) (DropAnswer, int, error) {
var v DropAnswer
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
return v, formRead(raw), nil
}
// DecodePublishQueueAnswer reads a PublishQueueAnswer as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodePublishQueueAnswer(raw []byte) (PublishQueueAnswer, int, error) {
var v PublishQueueAnswer
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
return v, formRead(raw), nil
}
// DecodePackagePublishedEvent reads a PackagePublishedEvent as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodePackagePublishedEvent(raw []byte) (PackagePublishedEvent, int, error) {
var v PackagePublishedEvent
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
return v, formRead(raw), nil
}
// DecodeToolchainAgreedEvent reads a ToolchainAgreedEvent as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodeToolchainAgreedEvent(raw []byte) (ToolchainAgreedEvent, int, error) {
var v ToolchainAgreedEvent
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
return v, formRead(raw), nil
}
// DecodeStepEndedEvent reads a StepEndedEvent as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodeStepEndedEvent(raw []byte) (StepEndedEvent, int, error) {
var v StepEndedEvent
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
return v, formRead(raw), nil
}
// DecodeAskTakenEvent reads a AskTakenEvent as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodeAskTakenEvent(raw []byte) (AskTakenEvent, int, error) {
var v AskTakenEvent
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
return v, formRead(raw), nil
}
// DecodeVersionReservedEvent reads a VersionReservedEvent as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodeVersionReservedEvent(raw []byte) (VersionReservedEvent, int, error) {
var v VersionReservedEvent
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
return v, formRead(raw), nil
}
// formRead is the form a value says on the wire: 1 when it says none, or is no object.
func formRead(raw []byte) int {
var f struct {
Form int `json:"form"`
}
if json.Unmarshal(raw, &f) != nil || f.Form < 1 {
return 1
}
return f.Form
}
@@ -25,6 +25,10 @@ func TestFixtures(t *testing.T) {
decode: func(b []byte) (any, error) { var v StepEndedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-builds\\.event\\.step-ended\\.[^.*> ]+$")},
{name: "the build step failed on its entry", subject: "mesh.seat.mesh-builds.event.step-ended.laptop", given: "{\"entries\":[{\"fingerprint\":\"fp-0123\",\"module\":\"nats\"}],\"id\":\"m2kx9a1b3c4d6b\",\"node\":\"laptop\",\"outcome\":\"failed\",\"pipeline\":\"main.12.nats\",\"said\":\"nats:program did not build\",\"step\":\"build\",\"v\":1}", wire: "{\"entries\":[{\"fingerprint\":\"fp-0123\",\"module\":\"nats\"}],\"id\":\"m2kx9a1b3c4d6b\",\"node\":\"laptop\",\"outcome\":\"failed\",\"pipeline\":\"main.12.nats\",\"said\":\"nats:program did not build\",\"step\":\"build\",\"v\":1}",
decode: func(b []byte) (any, error) { var v StepEndedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-builds\\.event\\.step-ended\\.[^.*> ]+$")},
{name: "an ask taken at the pull", subject: "mesh.seat.mesh-builds.event.ask-taken.main-3589-mesh-sdk-unit", given: "{\"ask\":\"main-3589-mesh-sdk-unit\",\"at\":\"2026-11-21T22:00:00Z\",\"node\":\"anchor\"}", wire: "{\"ask\":\"main-3589-mesh-sdk-unit\",\"at\":\"2026-11-21T22:00:00Z\",\"node\":\"anchor\"}",
decode: func(b []byte) (any, error) { var v AskTakenEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-builds\\.event\\.ask-taken\\.[^.*> ]+$")},
{name: "a version reserved at a fingerprint", subject: "mesh.seat.mesh-builds.event.version-reserved", given: "{\"artifact\":\"mesh-sdk:go\",\"fingerprint\":\"sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\",\"form\":1,\"module\":\"mesh-sdk\",\"path\":\"git.novox.be/novox/mesh-sdk/go\",\"revision\":\"412\",\"version\":\"0.1.21\"}", wire: "{\"artifact\":\"mesh-sdk:go\",\"fingerprint\":\"sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\",\"form\":1,\"module\":\"mesh-sdk\",\"path\":\"git.novox.be/novox/mesh-sdk/go\",\"revision\":\"412\",\"version\":\"0.1.21\"}",
decode: func(b []byte) (any, error) { var v VersionReservedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-builds\\.event\\.version-reserved$")},
{name: "reserved asks a module", subject: "", given: "{\"module\":\"mesh-sdk\"}", wire: "{\"module\":\"mesh-sdk\"}",
decode: func(b []byte) (any, error) { var v ReservedInput; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil},
{name: "reserved asks a package by its name", subject: "", given: "{\"name\":\"git.novox.be/novox/mesh-sdk/go\",\"version\":\"0.1.15\"}", wire: "{\"name\":\"git.novox.be/novox/mesh-sdk/go\",\"version\":\"0.1.15\"}",
File diff suppressed because it is too large Load Diff
@@ -83,6 +83,12 @@ func TestFixtures(t *testing.T) {
err := json.Unmarshal(b, &v)
return v, err
}, subjectIs: nil},
{name: "a seat-claiming module's entry carries its class core (H-A-394, H-A-431)", subject: "", given: "{\"acts\":[{\"bound\":\"60m\",\"on\":\"go\",\"seat\":\"node-build-go\",\"step\":\"build\"}],\"awaits\":[\"database\"],\"class\":\"core\",\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"fp\":\"src2:b427bdde1f100e2f52490cfa9df0c5112dd3700226b4892349f0c14a15a4af56\",\"left-out\":[\"mesh-catalog.revision\"],\"main\":\"cmd/mesh-build-go\",\"manifest\":{\"module\":\"mesh-build-go\",\"version\":\"1\"},\"module\":\"mesh-build-go\",\"needs\":[\"form:setting\",\"word:claims\"],\"path\":\"modules/mesh-build-go\",\"repository\":\"novox/mesh-catalog\",\"serves\":[\"node-build-go.run\"],\"v\":1,\"verdict\":\"accepted\"}", wire: "{\"acts\":[{\"bound\":\"60m\",\"on\":\"go\",\"seat\":\"node-build-go\",\"step\":\"build\"}],\"awaits\":[\"database\"],\"class\":\"core\",\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"fp\":\"src2:b427bdde1f100e2f52490cfa9df0c5112dd3700226b4892349f0c14a15a4af56\",\"left-out\":[\"mesh-catalog.revision\"],\"main\":\"cmd/mesh-build-go\",\"manifest\":{\"module\":\"mesh-build-go\",\"version\":\"1\"},\"module\":\"mesh-build-go\",\"needs\":[\"form:setting\",\"word:claims\"],\"path\":\"modules/mesh-build-go\",\"repository\":\"novox/mesh-catalog\",\"serves\":[\"node-build-go.run\"],\"v\":1,\"verdict\":\"accepted\"}",
decode: func(b []byte) (any, error) {
var v CatalogRevisionsByEntryDigestValue
err := json.Unmarshal(b, &v)
return v, err
}, subjectIs: nil},
{name: "a module's registration", subject: "", given: "{\"at\":\"2026-11-10T10:00:00Z\",\"branches\":[\"main\"],\"by\":\"operator@workstation\",\"module\":\"nats\",\"origin\":\"novox/mesh-catalog\",\"path\":\"modules/nats/module.json\",\"policy\":{\"artifacts\":{\"program\":{\"unit\":\"bypass\"}},\"steps\":{\"approval\":\"bypass\",\"composition\":\"required\",\"integration\":\"required\",\"lint\":\"required\",\"review\":\"optional\",\"unit\":\"required\"}},\"v\":1,\"version\":2,\"warrant\":\"ask-m2kx9a1b3c4d5e\"}", wire: "{\"at\":\"2026-11-10T10:00:00Z\",\"branches\":[\"main\"],\"by\":\"operator@workstation\",\"module\":\"nats\",\"origin\":\"novox/mesh-catalog\",\"path\":\"modules/nats/module.json\",\"policy\":{\"artifacts\":{\"program\":{\"unit\":\"bypass\"}},\"steps\":{\"approval\":\"bypass\",\"composition\":\"required\",\"integration\":\"required\",\"lint\":\"required\",\"review\":\"optional\",\"unit\":\"required\"}},\"v\":1,\"version\":2,\"warrant\":\"ask-m2kx9a1b3c4d5e\"}",
decode: func(b []byte) (any, error) {
var v RegistrationsByModuleValue
@@ -119,6 +125,18 @@ func TestFixtures(t *testing.T) {
err := json.Unmarshal(b, &v)
return v, err
}, subjectIs: nil},
{name: "a source-only registration of a repository with no module", subject: "", given: "{\"at\":\"2026-11-22T00:10:00Z\",\"branches\":[\"main\"],\"by\":\"operator@workstation\",\"check\":{\"seat\":\"node-build-go\",\"step\":\"unit\"},\"origin\":\"novox/hq\",\"policy\":{\"steps\":{\"approval\":\"bypass\",\"review\":\"optional\",\"unit\":\"required\"}},\"v\":1,\"version\":1,\"warrant\":\"ask-m2kx9a1b3c4d6a\"}", wire: "{\"at\":\"2026-11-22T00:10:00Z\",\"branches\":[\"main\"],\"by\":\"operator@workstation\",\"check\":{\"seat\":\"node-build-go\",\"step\":\"unit\"},\"origin\":\"novox/hq\",\"policy\":{\"steps\":{\"approval\":\"bypass\",\"review\":\"optional\",\"unit\":\"required\"}},\"v\":1,\"version\":1,\"warrant\":\"ask-m2kx9a1b3c4d6a\"}",
decode: func(b []byte) (any, error) {
var v RegistrationsBySourceOwnerNameValue
err := json.Unmarshal(b, &v)
return v, err
}, subjectIs: nil},
{name: "one version of a source-only registration in the history", subject: "", given: "{\"at\":\"2026-11-22T00:10:00Z\",\"branches\":[\"main\"],\"by\":\"operator@workstation\",\"check\":{\"seat\":\"node-build-go\",\"step\":\"unit\"},\"origin\":\"novox/hq\",\"policy\":{\"steps\":{\"approval\":\"bypass\",\"review\":\"optional\",\"unit\":\"required\"}},\"v\":1,\"version\":1,\"warrant\":\"ask-m2kx9a1b3c4d6a\"}", wire: "{\"at\":\"2026-11-22T00:10:00Z\",\"branches\":[\"main\"],\"by\":\"operator@workstation\",\"check\":{\"seat\":\"node-build-go\",\"step\":\"unit\"},\"origin\":\"novox/hq\",\"policy\":{\"steps\":{\"approval\":\"bypass\",\"review\":\"optional\",\"unit\":\"required\"}},\"v\":1,\"version\":1,\"warrant\":\"ask-m2kx9a1b3c4d6a\"}",
decode: func(b []byte) (any, error) {
var v RegistrationHistoryBySourceOwnerNameVersionValue
err := json.Unmarshal(b, &v)
return v, err
}, subjectIs: nil},
{name: "revision asks the trunk's newest", subject: "", given: "{}", wire: "{}",
decode: func(b []byte) (any, error) { var v RevisionInput; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil},
{name: "revision asks a candidate's, from a page on", subject: "", given: "{\"after\":\"mesh-build-go\",\"group\":\"feat/47-G\",\"heads\":\"0123456789ab\"}", wire: "{\"after\":\"mesh-build-go\",\"group\":\"feat/47-G\",\"heads\":\"0123456789ab\"}",
@@ -163,6 +181,14 @@ func TestFixtures(t *testing.T) {
decode: func(b []byte) (any, error) { var v InputAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil},
{name: "entry asks by fingerprint", subject: "", given: "{\"fingerprint\":\"src2:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\"}", wire: "{\"fingerprint\":\"src2:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\"}",
decode: func(b []byte) (any, error) { var v EntryInput; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil},
{name: "registrations of one repository from a cursor", subject: "", given: "{\"next\":\"c-120\",\"repository\":\"novox/mesh-catalog\"}", wire: "{\"next\":\"c-120\",\"repository\":\"novox/mesh-catalog\"}",
decode: func(b []byte) (any, error) { var v RegistrationsInput; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil},
{name: "registrations answers a capped page with a source-only registration and the next cursor", subject: "", given: "{\"capped\":true,\"next\":\"c-121\",\"registrations\":[{\"at\":\"2026-11-22T00:10:00Z\",\"branches\":[\"main\"],\"by\":\"operator@workstation\",\"check\":{\"seat\":\"node-build-go\",\"step\":\"unit\"},\"origin\":\"novox/hq\",\"policy\":{\"steps\":{\"approval\":\"bypass\",\"review\":\"optional\",\"unit\":\"required\"}},\"v\":1,\"version\":1,\"warrant\":\"ask-m2kx9a1b3c4d6a\"}],\"total\":118}", wire: "{\"capped\":true,\"next\":\"c-121\",\"registrations\":[{\"at\":\"2026-11-22T00:10:00Z\",\"branches\":[\"main\"],\"by\":\"operator@workstation\",\"check\":{\"seat\":\"node-build-go\",\"step\":\"unit\"},\"origin\":\"novox/hq\",\"policy\":{\"steps\":{\"approval\":\"bypass\",\"review\":\"optional\",\"unit\":\"required\"}},\"v\":1,\"version\":1,\"warrant\":\"ask-m2kx9a1b3c4d6a\"}],\"total\":118}",
decode: func(b []byte) (any, error) { var v RegistrationsAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil},
{name: "seat-list asks the seat list's bytes on a call's chunks", subject: "", given: "{\"call\":\"0123456789abcdef0123456789abcdef\"}", wire: "{\"call\":\"0123456789abcdef0123456789abcdef\"}",
decode: func(b []byte) (any, error) { var v SeatListInput; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil},
{name: "seat-list answers the seat list's digest, size and how many chunks carry its bytes", subject: "", given: "{\"chunks\":1,\"digest\":\"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\",\"size\":412345}", wire: "{\"chunks\":1,\"digest\":\"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\",\"size\":412345}",
decode: func(b []byte) (any, error) { var v SeatListAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil},
}
for _, c := range cases {
v, err := c.decode([]byte(c.given))
@@ -4,9 +4,7 @@
package meshdelivery
import (
"bytes"
"encoding/json"
"errors"
"strings"
"time"
)
@@ -14,6 +12,17 @@ import (
// Seat is the seat's name.
const Seat = "mesh-delivery"
// Form is the form of the seat's values, events and answers this contract was generated at (design 47 H2 §780):
// a reader of a value of another form says "form N, this reader knows M".
const Form = 2
// The forms and the digest of the fields this contract was generated from: the generator's compare fails a definition
// whose fields changed while its forms did not (design 47 H2 §780).
const (
generatedForms = "2"
generatedFieldsDigest = "DeliveriesAnswer=d6ae3ca4e5da Fill=80f442916429 FillingEvent=5bd4df78ed84 FullEvent=5bd4df78ed84 HeadsByRepositoryBranchValue=b32fd594ee25 Owed=a30ee01924cc OwedByRepositoryNumberHeadActValue=f16ff5733d62 OwedEvent=86af3b56de28 RecordEndedEvent=566edb8684e5 RecordEndedEventAwayItem=44ad7d26bd16 RecordOpenedEvent=9cf9afaa25b4 RecoveryAnswer=8c705e17cdf8 RecoveryMergedEvent=014d4d00d2e5 RoomEvent=5bd4df78ed84 ShowAnswer=7538f2f5e68e SourceAnswer=6ae84f05f10d SourceByBranchValue=31aa822718c0 SourceHead=e722e2d9492f SourceMovedEvent=5231909928e9 SourceMovedEventHeadsItem=774b4bf88ba4 StalledAnswer=7538f2f5e68e"
)
const (
VerbDeliveries = "deliveries"
VerbShow = "show"
@@ -38,6 +47,24 @@ func Verbs() []string {
return []string{VerbDeliveries, VerbShow, VerbSource, VerbStalled, VerbRecovery}
}
// VerbDescription is a verb's description as the seat's definition says it, "" for a verb the seat does not serve or
// does not describe (design 47 H2 H-A-449).
func VerbDescription(verb string) string {
switch verb {
case VerbDeliveries:
return "Every delivery not final, and those that ended in the last day, one line each: its id (owner/repository@commit), its state, its group, what it waits for and since when. Narrowed by state, repository or group. Filtered by repository, module, outcome and since while it reads, at most 768 KB a page, capped saying the rest is read from the cursor next (design 47 H2 §811)."
case VerbShow:
return "One delivery or group whole: its delivery plan (what it builds, what each machine receives, what is not an ordinary send), every transition with when and why, the machine steps of its walk, its group and its order."
case VerbSource:
return "One watched branch's source revision (design 47 *Watched branches and the catalog graph per branch*: source.<branch> in place of source.main): The source revision the gate holds: every repository it names, its trunk branch and that trunk's head as the gate merged or seeded it, and the merge call that moved it last. Read, never written: the node-engines read it to call a reverted build's catch-up tool once per repository, never the gate's state (design 47). Given no branch it answers branches alone, the branch of each source key the gate keeps; given a branch, that branch's source revision (design 47 G section 690)."
case VerbStalled:
return "Every delivery held past its state's bound, with the transition the table lets healer H2 take for it, or none."
case VerbRecovery:
return "Mark a merge its author names a recovery: the commit and why; the delivery record is marked recovery with the caller the runtime proves, and recovery-merged is said (design 47 G section 305)."
}
return ""
}
// Events is every event published on the seat's subject: a new copy each call.
func Events() []string {
return []string{EventOwed, EventRecordOpened, EventRecordEnded, EventFilling, EventFull, EventRoom, EventSourceMoved, EventRecoveryMerged}
@@ -117,18 +144,13 @@ func ParseSourceByBranchKey(key string) (string, bool) {
return v0, true
}
// DecodeSourceByBranchValue reads one value of the state source under the form "<branch>", refusing a field SourceByBranchValue does not know and any text after the value.
func DecodeSourceByBranchValue(raw []byte) (SourceByBranchValue, error) {
// DecodeSourceByBranchValue reads one value of the state source under the form "<branch>" as another part wrote it: a field SourceByBranchValue does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodeSourceByBranchValue(raw []byte) (SourceByBranchValue, int, error) {
var v SourceByBranchValue
dec := json.NewDecoder(bytes.NewReader(raw))
dec.DisallowUnknownFields()
if err := dec.Decode(&v); err != nil {
return v, err
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
if dec.More() {
return v, errors.New("text after the value")
}
return v, nil
return v, formRead(raw), nil
}
// OwedByRepositoryNumberHeadActKey is the key of one value of the state owed under the form "<repository>.<number>.<head>.<act>".
@@ -165,18 +187,13 @@ func ParseOwedByRepositoryNumberHeadActKey(key string) (string, string, string,
return v0, v2, v4, v6, true
}
// DecodeOwedByRepositoryNumberHeadActValue reads one value of the state owed under the form "<repository>.<number>.<head>.<act>", refusing a field OwedByRepositoryNumberHeadActValue does not know and any text after the value.
func DecodeOwedByRepositoryNumberHeadActValue(raw []byte) (OwedByRepositoryNumberHeadActValue, error) {
// DecodeOwedByRepositoryNumberHeadActValue reads one value of the state owed under the form "<repository>.<number>.<head>.<act>" as another part wrote it: a field OwedByRepositoryNumberHeadActValue does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodeOwedByRepositoryNumberHeadActValue(raw []byte) (OwedByRepositoryNumberHeadActValue, int, error) {
var v OwedByRepositoryNumberHeadActValue
dec := json.NewDecoder(bytes.NewReader(raw))
dec.DisallowUnknownFields()
if err := dec.Decode(&v); err != nil {
return v, err
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
if dec.More() {
return v, errors.New("text after the value")
}
return v, nil
return v, formRead(raw), nil
}
// HeadsByRepositoryBranchKey is the key of one value of the state heads under the form "<repository>.<branch>".
@@ -201,18 +218,13 @@ func ParseHeadsByRepositoryBranchKey(key string) (string, string, bool) {
return v0, v2, true
}
// DecodeHeadsByRepositoryBranchValue reads one value of the state heads under the form "<repository>.<branch>", refusing a field HeadsByRepositoryBranchValue does not know and any text after the value.
func DecodeHeadsByRepositoryBranchValue(raw []byte) (HeadsByRepositoryBranchValue, error) {
// DecodeHeadsByRepositoryBranchValue reads one value of the state heads under the form "<repository>.<branch>" as another part wrote it: a field HeadsByRepositoryBranchValue does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodeHeadsByRepositoryBranchValue(raw []byte) (HeadsByRepositoryBranchValue, int, error) {
var v HeadsByRepositoryBranchValue
dec := json.NewDecoder(bytes.NewReader(raw))
dec.DisallowUnknownFields()
if err := dec.Decode(&v); err != nil {
return v, err
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
if dec.More() {
return v, errors.New("text after the value")
}
return v, nil
return v, formRead(raw), nil
}
// Fill is the delivery records' ledger's fill against its cap.
@@ -222,6 +234,8 @@ type Fill struct {
Bytes int64 `json:"bytes"`
// bytes
Cap int64 `json:"cap"`
// is the form of what this value carries, left out while it is 1 (design 47 H2 §780).
Form int64 `json:"form,omitempty"`
// what full refuses, such as the next record
Refused string `json:"refused,omitempty"`
// the kept data, by its name in the holder's manifest
@@ -238,6 +252,8 @@ type Owed struct {
Context string `json:"context,omitempty"`
// a status: what it says
Description string `json:"description,omitempty"`
// is the form of what this value carries, left out while it is 1 (design 47 H2 §780).
Form int64 `json:"form,omitempty"`
// the pull request's head the act is for
Head string `json:"head"`
// the act's id, the same each time it is said again; a comment and a note line carry it, so the forge holds it once
@@ -246,6 +262,8 @@ type Owed struct {
Line string `json:"line,omitempty"`
// the pull request's number
Number int64 `json:"number"`
// the delivery record the act is for, by its source revision, <branch>.<source>; absent on an act said before it was named, which bridge_owed_again by record never finds (design 47 H2 §807, H-A-432)
Record string `json:"record,omitempty"`
// the repository, <owner>/<name>
Repository string `json:"repository"`
// when the gate first said it
@@ -282,8 +300,16 @@ type DeliveriesInput struct {
All string `json:"all,omitempty"`
// a group's id (its branch name)
Group string `json:"group,omitempty"`
// only the deliveries that move this module, applied while reading (design 47 H2 §811)
Module string `json:"module,omitempty"`
// the cursor the previous page answered, opaque; absent for the first page (design 47 H2 §779)
Next string `json:"next,omitempty"`
// only the deliveries that ended with this outcome, applied while reading (design 47 H2 §811)
Outcome string `json:"outcome,omitempty"`
// owner/repository
Repository string `json:"repository,omitempty"`
// only the deliveries opened at or after this time, applied while reading (design 47 H2 §811)
Since *time.Time `json:"since,omitempty"`
// one state, e.g. delivering or held
State string `json:"state,omitempty"`
}
@@ -295,7 +321,18 @@ const (
)
// DeliveriesAnswer is what deliveries answers.
type DeliveriesAnswer = map[string]any
type DeliveriesAnswer struct {
// yes when the answer stopped at 768 KB; the rest is read from next (design 47 H2 §811)
Capped bool `json:"capped,omitempty"`
// this page's deliveries, the open ones first, then the newest
Deliveries []map[string]any `json:"deliveries"`
// is the form of what this value carries, left out while it is 1 (design 47 H2 §780).
Form int64 `json:"form,omitempty"`
// the cursor of the next page, opaque; empty at the end (design 47 H2 §779)
Next string `json:"next,omitempty"`
// how many deliveries the filters hold in all (design 47 H2 §811)
Total int64 `json:"total,omitempty"`
}
// ShowInput is what show takes.
type ShowInput struct {
@@ -322,6 +359,8 @@ type SourceAnswer struct {
Call string `json:"call,omitempty"`
// why it could not be read
Error string `json:"error,omitempty"`
// is the form of what this value carries, left out while it is 1 (design 47 H2 §780).
Form int64 `json:"form,omitempty"`
// every repository the source revision names, by name
Repositories []SourceHead `json:"repositories,omitempty"`
// the source revision's number on the bus; 0 before the first
@@ -350,8 +389,10 @@ type RecoveryAnswer struct {
// the caller the runtime proved
By string `json:"by,omitempty"`
// the merge commit
Commit string `json:"commit"`
Recovery bool `json:"recovery"`
Commit string `json:"commit"`
// is the form of what this value carries, left out while it is 1 (design 47 H2 §780).
Form int64 `json:"form,omitempty"`
Recovery bool `json:"recovery"`
// the repository
Repository string `json:"repository,omitempty"`
}
@@ -367,6 +408,8 @@ type RecordOpenedEvent struct {
Branch string `json:"branch"`
// the merge call
Call string `json:"call"`
// is the form of what this value carries, left out while it is 1 (design 47 H2 §780).
Form int64 `json:"form,omitempty"`
// its milestone, <branch>.<source>
Milestone string `json:"milestone"`
// the record's id
@@ -393,6 +436,8 @@ type RecordEndedEvent struct {
Branch string `json:"branch"`
// the merge call
Call string `json:"call"`
// is the form of what this value carries, left out while it is 1 (design 47 H2 §780).
Form int64 `json:"form,omitempty"`
// its milestone, <branch>.<source>
Milestone string `json:"milestone"`
// the record's id
@@ -421,6 +466,8 @@ type SourceMovedEvent struct {
Branch string `json:"branch"`
// the merge call that moved it
Call string `json:"call"`
// is the form of what this value carries, left out while it is 1 (design 47 H2 §780).
Form int64 `json:"form,omitempty"`
// the repositories it moved
Heads []SourceMovedEventHeadsItem `json:"heads"`
// the source revision, numbered by the gate alone
@@ -439,6 +486,8 @@ type RecoveryMergedEvent struct {
By string `json:"by,omitempty"`
// the merge commit
Commit string `json:"commit"`
// is the form of what this value carries, left out while it is 1 (design 47 H2 §780).
Form int64 `json:"form,omitempty"`
// the module the merge changed, from the delivery record
Module string `json:"module,omitempty"`
// the repository
@@ -452,6 +501,8 @@ type RecoveryMergedEvent struct {
type SourceByBranchValue struct {
// the merge call that moved it; a call the newest revision already carries writes nothing (design 47 H §737, H-A-33)
Call string `json:"call"`
// is the form of what this value carries, left out while it is 1 (design 47 H2 §780).
Form int64 `json:"form,omitempty"`
// repository to the merged head, a full commit
Heads map[string]string `json:"heads"`
// the branch's source revision, one above its newest
@@ -465,18 +516,150 @@ type OwedByRepositoryNumberHeadActValue struct {
// status=<context>, view, note or comment.<seq>
Act string `json:"act"`
Context string `json:"context,omitempty"`
Ref string `json:"ref,omitempty"`
State string `json:"state,omitempty"`
Target string `json:"target,omitempty"`
Text string `json:"text,omitempty"`
// is the form of what this value carries, left out while it is 1 (design 47 H2 §780).
Form int64 `json:"form,omitempty"`
Ref string `json:"ref,omitempty"`
State string `json:"state,omitempty"`
Target string `json:"target,omitempty"`
Text string `json:"text,omitempty"`
// the layout version
V int64 `json:"v,omitempty"`
}
// HeadsByRepositoryBranchValue is one value of the state heads under the key <repository>.<branch>.
type HeadsByRepositoryBranchValue struct {
// is the form of what this value carries, left out while it is 1 (design 47 H2 §780).
Form int64 `json:"form,omitempty"`
// the head head-moved said
Head string `json:"head"`
// when it was first heard
Since time.Time `json:"since"`
}
// DecodeDeliveriesAnswer reads a DeliveriesAnswer as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodeDeliveriesAnswer(raw []byte) (DeliveriesAnswer, int, error) {
var v DeliveriesAnswer
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
return v, formRead(raw), nil
}
// DecodeShowAnswer reads a ShowAnswer as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodeShowAnswer(raw []byte) (ShowAnswer, int, error) {
var v ShowAnswer
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
return v, formRead(raw), nil
}
// DecodeSourceAnswer reads a SourceAnswer as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodeSourceAnswer(raw []byte) (SourceAnswer, int, error) {
var v SourceAnswer
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
return v, formRead(raw), nil
}
// DecodeStalledAnswer reads a StalledAnswer as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodeStalledAnswer(raw []byte) (StalledAnswer, int, error) {
var v StalledAnswer
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
return v, formRead(raw), nil
}
// DecodeRecoveryAnswer reads a RecoveryAnswer as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodeRecoveryAnswer(raw []byte) (RecoveryAnswer, int, error) {
var v RecoveryAnswer
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
return v, formRead(raw), nil
}
// DecodeOwedEvent reads a OwedEvent as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodeOwedEvent(raw []byte) (OwedEvent, int, error) {
var v OwedEvent
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
return v, formRead(raw), nil
}
// DecodeRecordOpenedEvent reads a RecordOpenedEvent as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodeRecordOpenedEvent(raw []byte) (RecordOpenedEvent, int, error) {
var v RecordOpenedEvent
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
return v, formRead(raw), nil
}
// DecodeRecordEndedEvent reads a RecordEndedEvent as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodeRecordEndedEvent(raw []byte) (RecordEndedEvent, int, error) {
var v RecordEndedEvent
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
return v, formRead(raw), nil
}
// DecodeFillingEvent reads a FillingEvent as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodeFillingEvent(raw []byte) (FillingEvent, int, error) {
var v FillingEvent
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
return v, formRead(raw), nil
}
// DecodeFullEvent reads a FullEvent as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodeFullEvent(raw []byte) (FullEvent, int, error) {
var v FullEvent
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
return v, formRead(raw), nil
}
// DecodeRoomEvent reads a RoomEvent as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodeRoomEvent(raw []byte) (RoomEvent, int, error) {
var v RoomEvent
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
return v, formRead(raw), nil
}
// DecodeSourceMovedEvent reads a SourceMovedEvent as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodeSourceMovedEvent(raw []byte) (SourceMovedEvent, int, error) {
var v SourceMovedEvent
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
return v, formRead(raw), nil
}
// DecodeRecoveryMergedEvent reads a RecoveryMergedEvent as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780).
func DecodeRecoveryMergedEvent(raw []byte) (RecoveryMergedEvent, int, error) {
var v RecoveryMergedEvent
if err := json.Unmarshal(raw, &v); err != nil {
return v, 0, err
}
return v, formRead(raw), nil
}
// formRead is the form a value says on the wire: 1 when it says none, or is no object.
func formRead(raw []byte) int {
var f struct {
Form int `json:"form"`
}
if json.Unmarshal(raw, &f) != nil || f.Form < 1 {
return 1
}
return f.Form
}
@@ -63,6 +63,10 @@ func TestFixtures(t *testing.T) {
decode: func(b []byte) (any, error) { var v RecoveryInput; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil},
{name: "recovery answers the caller the runtime proved", subject: "", given: "{\"by\":\"alice\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"recovery\":true,\"repository\":\"novox/mesh-sdk\"}", wire: "{\"by\":\"alice\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"recovery\":true,\"repository\":\"novox/mesh-sdk\"}",
decode: func(b []byte) (any, error) { var v RecoveryAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil},
{name: "deliveries asks one repository's of one module since a time, from a cursor", subject: "", given: "{\"module\":\"nats\",\"next\":\"c-77\",\"outcome\":\"delivered\",\"repository\":\"novox/mesh-catalog\",\"since\":\"2026-11-20T00:00:00Z\"}", wire: "{\"module\":\"nats\",\"next\":\"c-77\",\"outcome\":\"delivered\",\"repository\":\"novox/mesh-catalog\",\"since\":\"2026-11-20T00:00:00Z\"}",
decode: func(b []byte) (any, error) { var v DeliveriesInput; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil},
{name: "deliveries answers a capped page with the next cursor", subject: "", given: "{\"capped\":true,\"deliveries\":[{\"id\":\"novox/mesh-catalog@0123456789abcdef0123456789abcdef01234567\",\"state\":\"delivered\"}],\"next\":\"c-78\",\"total\":412}", wire: "{\"capped\":true,\"deliveries\":[{\"id\":\"novox/mesh-catalog@0123456789abcdef0123456789abcdef01234567\",\"state\":\"delivered\"}],\"next\":\"c-78\",\"total\":412}",
decode: func(b []byte) (any, error) { var v DeliveriesAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil},
}
for _, c := range cases {
v, err := c.decode([]byte(c.given))

Some files were not shown because too many files have changed in this diff Show More