Compare commits

..
Author SHA1 Message Date
jschoubben 7a32a8b783 view: keep the pipeline runner's pipelines and serve view_pipelines
The view hears the seat mesh-pipeline-runner's pipeline-started, step-started, step-ended and pipeline-ended and the build seats' step-progress, keeps each pipeline whole in its own state pipelines (newest 500) and the latest progress per running ask in pipeline-progress (200), and view_pipelines reads them.

Builds: the lead's handover of 2026-11-16 (part 1 of the delivery view through its own pipeline); design 47 G, Pipelines per module (no milestone line cited by the handover)
2026-11-16 21:37:07 +01:00
232 changed files with 4113 additions and 16360 deletions
@@ -1,375 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"slices"
"strings"
"time"
"github.com/nats-io/nats.go/jetstream"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
"git.novox.be/novox/mesh-sdk/go/buscalls"
)
// A composition run's bound (design 47 G §720; A-466n, A-466p, A-466q, A-466s, A-468b, A-469b, A-469s, A-469y, A-470): a run
// has one clock, the key `bound.<run key>` of composition-runs with its own time to live composition-step-bound, written at the
// build seat's ask-taken of the run's ask and written again at each step-progress of that ask; an ask still waiting in the queue
// has no clock but `untaken.<run key>`, written at the run's start with the time to live run-untaken-bound. Either's expiry,
// heard as the bus's event, ends the run: its ask dropped through the build seat's drop, the run ended could not run naming the
// ask and the bound, so every reader ends could not run with it. While the ask waits for a build node each reader is said waiting
// by a wait-held on mesh-wants (`on: build node`), ended `ran` at ask-taken or `superseded` when the run ends untaken.
// waitOnBuildNode is what a composition run's waiting reader waits on (A-469b, A-469y).
const waitOnBuildNode = "build node"
// dropVerbKey is the Go build seat's drop, asked as a kept call (§119, A-182).
const dropVerbKey = "seat:" + nodebuildgo.Seat + "." + nodebuildgo.VerbDrop
// dropOver asks the Go build seat's drop of a run ask as a kept call, of any holder, which records it (A-469s).
func dropOver(call func(key string, body any) (json.RawMessage, error)) func(ask, why string) error {
return func(ask, why string) error {
raw, err := call(dropVerbKey, nodebuildgo.DropInput{ID: ask, Why: why, By: runAsker})
if err != nil {
return err
}
if _, said, ok := buscalls.Unwrap(raw); !ok {
return fmt.Errorf("%s answered: %s", dropVerbKey, said)
}
return nil
}
}
// waitOf is a reader's wait for a build node to take its run's ask (A-469y).
func waitOf(run meshwants.CompositionRun, reader meshwants.CompositionReader) meshwants.Wait {
w := meshwants.Wait{Step: "pipeline", Owner: meshwants.Seat, Module: reader.Module, Node: run.Machine, On: waitOnBuildNode, Fp: reader.Fingerprint,
Words: fmt.Sprintf("the composition step of %s waits for a build node to take the composition run %s of %s (ask %s)", reader.Pipeline,
runKeyOf(run.Candidate, run.Machine), run.Machine, run.Ask)}
if run.Started != nil {
w.Since = *run.Started
}
return w
}
// sayWaits says, for each reader, its wait held or ended (how: ran, superseded); a wait that cannot be said is journaled.
func (r *compositionRuns) sayWaits(run meshwants.CompositionRun, readers []meshwants.CompositionReader, how string) {
if r.say == nil {
return
}
for _, c := range readers {
w := waitOf(run, c)
event, id := meshwants.EventWaitHeld, "composition."+runKeyOf(run.Candidate, run.Machine)+"."+run.Ask+"."+c.ID+".held"
if how != "" {
w.How, event, id = how, meshwants.EventWaitEnded, strings.TrimSuffix(id, ".held")+".ended"
}
if err := r.say(event, w, id); err != nil {
r.logf("the %s of %s on the composition run of %s is not said: %v", event, c.Pipeline, run.Machine, err)
}
}
}
// untaken is whether the run of ask has not been taken yet: its untaken key stands holding that ask (A-470n).
func (r *compositionRuns) untaken(key, ask string) bool {
held, _, standing, err := r.boundOf(runUntakenPrefix + key)
return err == nil && standing && held == ask
}
// boundOf reads a bound key: the ask it holds and its revision; standing false when it is not written.
func (r *compositionRuns) boundOf(boundKey string) (string, uint64, bool, error) {
raw, rev, err := r.kv.get(boundKey)
if err != nil || raw == nil {
return "", 0, false, err
}
var v boundValue
_ = json.Unmarshal(raw, &v)
return v.Ask, rev, true, nil
}
// armBound writes the bound key prefix+key holding ask, judged from a read of the run, then reads the run again (A-470n): when
// it no longer names ask, or has ended, the bound just written is deleted at its revision, so no bound written from the earlier
// read ends the run after it, and a bound another run wrote since is kept. It answers whether the bound stands for the run.
func (r *compositionRuns) armBound(prefix, key, ask string, ttl time.Duration) (bool, error) {
rev, err := r.kv.arm(prefix+key, ask, ttl)
if err != nil {
return false, err
}
run, _, ok, err := r.read(key)
if err != nil {
return false, err
}
if ok && run.Ended == nil && run.Ask == ask {
return true, nil
}
if err := r.kv.delAt(prefix+key, rev); err != nil && !errors.Is(err, errRunChanged) {
return false, err
}
return false, nil
}
// heardAskTaken begins a run's clock at its ask's ask-taken: the bound key armed holding the ask, the untaken key of that ask
// deleted, each reader's wait ended ran. An ask no running run names is another asker's, or a late signal of a run replaced or
// ended, which moves nothing and is journaled (A-470n).
func (r *compositionRuns) heardAskTaken(e nodebuildgo.AskTaken) error {
key, run, err := r.runOfAsk(e.Asker, e.Ask)
if err != nil {
return err
}
if key == "" {
r.late("ask-taken", e.Asker, e.Ask)
return nil
}
armed, err := r.armBound(runBoundPrefix, key, e.Ask, r.set.stepBound)
if err != nil {
return fmt.Errorf("the bound of the composition run %s is not armed: %w", key, err)
}
if !armed {
r.late("ask-taken", e.Asker, e.Ask)
return nil
}
if r.untaken(key, e.Ask) {
if err := r.clearBounds(key, e.Ask, runUntakenPrefix); err != nil {
return err
}
r.sayWaits(run, run.Readers, "ran")
}
return nil
}
// heardProgress begins a running run's clock again at each step-progress of its ask (§720: owed progress, not run length); a
// late one moves nothing and is journaled (A-470n).
func (r *compositionRuns) heardProgress(e nodebuildgo.StepProgress) error {
key, _, err := r.runOfAsk(e.Asker, e.Ask)
if err != nil {
return err
}
armed := false
if key != "" {
if armed, err = r.armBound(runBoundPrefix, key, e.Ask, r.set.stepBound); err != nil {
return err
}
}
if !armed {
r.late("step-progress", e.Asker, e.Ask)
}
return nil
}
// late journals a signal of an ask the assigner put whose run no key names running any more (A-470n).
func (r *compositionRuns) late(signal, asker, ask string) {
if asker == runAsker && ask != "" {
r.logf("the %s of the run ask %s is late: no composition run names it running, so it moves nothing (A-470n)", signal, ask)
}
}
// runOfAsk is the running run of an ask the assigner put, or none.
func (r *compositionRuns) runOfAsk(asker, ask string) (string, meshwants.CompositionRun, error) {
if asker != runAsker || ask == "" {
return "", meshwants.CompositionRun{}, nil
}
key, run, err := r.keyOfAsk(ask)
if err != nil || key == "" || run.Ended != nil {
return "", meshwants.CompositionRun{}, err
}
return key, run, nil
}
// expired ends the run whose bound key's time to live ended (§720): its ask dropped, the run ended could not run naming the
// ask and the bound. A run that ended already, or a key of none, takes nothing.
func (r *compositionRuns) expired(boundKey string) {
key, bound, setting := "", time.Duration(0), ""
switch {
case strings.HasPrefix(boundKey, runBoundPrefix):
key, bound, setting = strings.TrimPrefix(boundKey, runBoundPrefix), r.set.stepBound, "composition-step-bound"
case strings.HasPrefix(boundKey, runUntakenPrefix):
key, bound, setting = strings.TrimPrefix(boundKey, runUntakenPrefix), r.set.untaken, "run-untaken-bound"
default:
return
}
run, _, ok, err := r.read(key)
if err != nil || !ok || run.Ended != nil {
return
}
// The marker of an expiry holds no value (A-470o): the run ends only while its bound is still absent when read again, and
// the run is taken for bound., untaken for untaken., so an expiry heard after a bound was armed again, or of a run replaced
// since, ends nothing.
other := runUntakenPrefix
if setting == "run-untaken-bound" {
other = runBoundPrefix
}
_, _, armed, err := r.boundOf(boundKey)
if err != nil {
r.logf("the bound %s is not read again, so its expiry ends nothing: %v", boundKey, err)
return
}
held, _, standing, err := r.boundOf(other + key)
if err != nil {
r.logf("the bound %s is not read, so the expiry of %s ends nothing: %v", other+key, boundKey, err)
return
}
if armed || (standing && held == run.Ask) {
r.logf("the expiry of %s ends nothing: the bound is armed again, or the run of ask %s is not where it bounds (A-470o)", boundKey, run.Ask)
return
}
said := fmt.Sprintf("%s: its run ask %s said no progress within %s (%s)", run.Machine, run.Ask, bound, setting)
if setting == "run-untaken-bound" {
said = fmt.Sprintf("%s: its run ask %s was taken by no build node within %s (%s)", run.Machine, run.Ask, bound, setting)
}
if r.drop != nil {
if err := r.drop(run.Ask, said); err != nil {
r.logf("the drop of the run ask %s of the composition run %s is not answered: %v", run.Ask, key, err)
}
}
// The untaken key is gone with its expiry, so the run ended by it is said untaken here: its readers' waits end superseded.
if _, err := r.endRunOf(key, run.Ask, meshwants.CompositionRunOutcomeCouldNotRun, said, nil, setting == "run-untaken-bound"); err != nil {
r.logf("the composition run %s past its bound is not ended: %v", key, err)
}
}
// endRun ends a run (end) and clears its bounds: the untaken key's readers said superseded, both bound keys deleted.
func (r *compositionRuns) endRun(key, ask, outcome, said string, verdict map[string]any) (bool, error) {
return r.endRunOf(key, ask, outcome, said, verdict, r.untaken(key, ask))
}
// endRunOf is endRun of a run whose ask is untaken, or not, as its caller judged.
func (r *compositionRuns) endRunOf(key, ask, outcome, said string, verdict map[string]any, untaken bool) (bool, error) {
return r.endRunMoving(key, ask, outcome, said, verdict, untaken, true)
}
// endRunMoving is endRunOf, moving the readers a rerun carried or not: a drop says its readers' ends itself (A-470r).
func (r *compositionRuns) endRunMoving(key, ask, outcome, said string, verdict map[string]any, untaken, move bool) (bool, error) {
// The readers a rerun carried move with the run's end (§727, A-470d): chosen before the end is written, while the rerun's own
// step still waits on it, so it alone says its own end; each mover waits for the end in the key.
if moved := r.moved.Load(); moved != nil && move {
if run, _, ok, err := r.read(key); err == nil && ok && run.Ended == nil && run.Ask == ask {
(*moved)(run)
}
}
ended, err := r.end(key, ask, outcome, said, verdict)
if err != nil || !ended {
return ended, err
}
if untaken {
if run, _, ok, err := r.read(key); err == nil && ok {
r.sayWaits(run, run.Readers, "superseded")
}
}
if err := r.clearBounds(key, ask, runBoundPrefix, runUntakenPrefix); err != nil {
r.logf("the bound keys of the ended composition run %s are not deleted: %v", key, err)
}
return true, nil
}
// readersOfOpen are the readers of every run with no `ended`, each with its ask's id once: what the assigner takes up again at
// its start (§719), each reader's end said from the keys alone (A-468). A reader whose step said its end already, carried into a
// rerun's run, is taken up only when it is marked `rerun` (A-470l, A-470m); each with the ask of the open run it is taken up from.
func (r *compositionRuns) readersOfOpen() ([]openReader, error) {
all, err := r.keys()
if err != nil {
return nil, err
}
seen := map[string]bool{}
var out []openReader
for _, key := range all {
if !isRunKey(key) {
continue
}
run, _, ok, err := r.read(key)
if err != nil || !ok || run.Ended != nil {
continue
}
for _, c := range run.Readers {
if c.Ended != nil && !c.Rerun {
continue // it ended passed already: only a reader marked rerun says its end again
}
if !seen[c.ID] {
seen[c.ID] = true
out = append(out, openReader{CompositionReader: c, ask: run.Ask})
}
}
}
return out, nil
}
// openReader is a reader taken up at the start and the ask of the open run it is taken up from.
type openReader struct {
meshwants.CompositionReader
ask string
}
// keysOf are the keys of the candidate's runs that the reader reads.
func (r *compositionRuns) keysOf(candidate, id string) ([]string, error) {
all, err := r.keys()
if err != nil {
return nil, err
}
var out []string
prefix := keysPart(candidate) + "."
for _, key := range all {
if !isRunKey(key) || !strings.HasPrefix(key, prefix) {
continue
}
if run, _, ok, err := r.read(key); err == nil && ok && hasReader(run.Readers, id) {
out = append(out, key)
}
}
slices.Sort(out) // by machine, as a step names its machines
return out, nil
}
// hearRunBounds hears the build seats' ask-taken and step-progress on the assigner's durable consumer (§720, A-466q): each
// handed to the runs; one that cannot be written is given again by the bus.
func hearRunBounds(hub interface {
Handle(string, func(jetstream.Msg)) error
}, r *compositionRuns, logf func(string, ...any)) error {
if err := hub.Handle(nodebuildgo.SubjectAskTakenAll, func(m jetstream.Msg) {
var e nodebuildgo.AskTaken
if err := json.Unmarshal(m.Data(), &e); err != nil {
logf("%s holds what is no ask-taken, left: %v", m.Subject(), err)
_ = m.Ack()
return
}
ackOrNak(m, r.heardAskTaken(e), logf)
}); err != nil {
return err
}
return hub.Handle(nodebuildgo.SubjectStepProgressAll, func(m jetstream.Msg) {
var e nodebuildgo.StepProgress
if err := json.Unmarshal(m.Data(), &e); err != nil {
logf("%s holds what is no step-progress, left: %v", m.Subject(), err)
_ = m.Ack()
return
}
ackOrNak(m, r.heardProgress(e), logf)
})
}
func ackOrNak(m jetstream.Msg, err error, logf func(string, ...any)) {
if err != nil {
logf("%s is not taken, so the bus gives it again: %v", m.Subject(), err)
_ = m.Nak()
return
}
_ = m.Ack()
}
// takeUp says, at the assigner's start, the end of each reader of a run with no `ended` once its runs end (§719, A-468): the
// step's goroutine is gone, so its end is judged from the candidate's keys naming it.
func (c *compositionSteps) takeUp(ctx context.Context, r *compositionRuns) {
readers, err := r.readersOfOpen()
if err != nil {
c.logf("composition-runs: the runs standing at the start are not read, so their readers are not taken up: %v", err)
return
}
for _, reader := range readers {
c.logf("the composition step %s of %s is taken up again by its runs (ask %s)", reader.Pipeline, reader.Module, reader.ID)
again := "" // a reader whose step said its end already says it again under the open run's ask (A-470q)
if reader.Ended != nil {
again = reader.ask
}
go c.say1(ctx, reader.ID, stepRunOf(reader.CompositionReader), c.resume, again)
}
}
@@ -10,6 +10,7 @@ import (
"errors"
"fmt"
"slices"
"sync"
"time"
nec "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
@@ -17,8 +18,8 @@ import (
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
)
// The inputs of a composition run (design 47 G §354, §718, A-243, A-252): one request per candidate and machine, carrying that
// machine's input, asked of the machine's node-engine by the seat's verb compose-candidate when the run starts.
// The inputs of a composition step (design 47 G §354, A-243, A-252): one request per machine, each carrying that machine's input,
// asked of the machine's node-engine by the seat's verb compose-candidate.
// composeCandidateKey is the seat verb's call key, asked of one node.
const composeCandidateKey = "seat:" + ne.Seat + "." + ne.VerbComposeCandidate
@@ -79,14 +80,37 @@ type compositionInputs struct {
bound time.Duration
}
// of asks the machine node for its input with the candidate's key (§718): asked only by the composition step that starts the
// machine's composition run, once per candidate and machine. A node that is silent, unready or answers no input has Why set.
func (c *compositionInputs) of(ctx context.Context, node, candidate string) nodeInput {
// gather asks each machine of the step's module and branch for its input with the step's candidate key, and answers every machine in
// name order: one away (away) is not asked; a node that is silent, unready or answers no input has Why set; no machine at all is an
// error. The run keeps each request built from an input under its run ask (compositionRun.machine).
func (c *compositionInputs) gather(ctx context.Context, run meshwants.StepRun, away func(node string) bool) ([]nodeInput, error) {
key, err := candidateKeyOf(run)
if err != nil {
return nil, err
}
nodes := c.machines(run.Module, run.Branch)
if len(nodes) == 0 {
return nil, fmt.Errorf("no node places %s on the branch %q, so the composition step has no machine to compose", run.Module, run.Branch)
}
bound := c.bound
if bound <= 0 {
bound = nec.AnswerBound
}
return c.one(ctx, bound, node, candidate)
out := make([]nodeInput, len(nodes))
var wg sync.WaitGroup
for i, node := range nodes {
if away != nil && away(node) {
out[i] = nodeInput{Node: node, Away: true}
continue
}
wg.Add(1)
go func() {
defer wg.Done()
out[i] = c.one(ctx, bound, node, key)
}()
}
wg.Wait()
return out, nil
}
// one asks one node for its input.
@@ -68,7 +68,7 @@ func TestTheMachinesOfAStep(t *testing.T) {
// §354, A-243, A-252, §64: each machine is asked compose-candidate with the step's key and input true, its numbers as written; a
// silent or unready node says why, and a machine away is not asked.
func TestAStepsInputsAreAskedOfEachMachine(t *testing.T) {
// the machines are asked side by side, so the record of what each was asked is guarded.
// gather asks the machines side by side, so the record of what each was asked is guarded.
var mu sync.Mutex
asked := map[string]string{}
call := func(key string, body any) (json.RawMessage, error) {
@@ -86,7 +86,7 @@ func TestAStepsInputsAreAskedOfEachMachine(t *testing.T) {
return json.RawMessage(`{"candidate":"run-r1","node":"` + node + `","verdict":"composed","input":{"n":12345678901234567890,"node":"` + node + `"},"modules":[],"own":{},"took":1,"v":1}`), nil
}
c := &compositionInputs{machines: func(module, branch string) []string { return []string{"ace", "g14", "novox", "shanks"} }, callFile: call, bound: time.Second}
got, err := gatherAll(c, meshwants.StepRun{Module: "m", Branch: "main", Run: "r1"}, func(node string) bool { return node == "shanks" })
got, err := c.gather(context.Background(), meshwants.StepRun{Module: "m", Branch: "main", Run: "r1"}, func(node string) bool { return node == "shanks" })
if err != nil || len(got) != 4 {
t.Fatalf("%+v %v", got, err)
}
@@ -105,7 +105,7 @@ func TestAStepsInputsAreAskedOfEachMachine(t *testing.T) {
t.Errorf("the machine away is asked or not said away: %+v", got[3])
}
none := &compositionInputs{machines: func(string, string) []string { return nil }, callFile: call}
if _, err := gatherAll(none, meshwants.StepRun{Module: "m", Run: "r1"}, nil); err == nil {
if _, err := none.gather(context.Background(), meshwants.StepRun{Module: "m", Run: "r1"}, nil); err == nil {
t.Error("a step with no machine is gathered")
}
}
@@ -124,7 +124,7 @@ func TestAStepsInputIsTheFileOfTheNodesChunks(t *testing.T) {
return json.RawMessage(answer + `,"file":"` + file + `"}`), nil
}
c := &compositionInputs{machines: func(module, branch string) []string { return []string{"ace", "g14"} }, callFile: call, bound: time.Second}
got, err := gatherAll(c, meshwants.StepRun{Module: "m", Branch: "main", Run: "r1"}, func(string) bool { return false })
got, err := c.gather(context.Background(), meshwants.StepRun{Module: "m", Branch: "main", Run: "r1"}, func(string) bool { return false })
if err != nil || len(got) != 2 {
t.Fatalf("%+v %v", got, err)
}
@@ -135,28 +135,3 @@ func TestAStepsInputIsTheFileOfTheNodesChunks(t *testing.T) {
t.Errorf("g14: %+v", got[1])
}
}
// gatherAll asks each machine of a step for its input as each run's start does (compositionInputs.of), side by side; a machine away
// is not asked; no machine is an error.
func gatherAll(c *compositionInputs, run meshwants.StepRun, away func(string) bool) ([]nodeInput, error) {
key, err := candidateKeyOf(run)
if err != nil {
return nil, err
}
nodes := c.machines(run.Module, run.Branch)
if len(nodes) == 0 {
return nil, errors.New("no machine")
}
out := make([]nodeInput, len(nodes))
var wg sync.WaitGroup
for i, node := range nodes {
if away != nil && away(node) {
out[i] = nodeInput{Node: node, Away: true}
continue
}
wg.Add(1)
go func() { defer wg.Done(); out[i] = c.of(context.Background(), node, key) }()
}
wg.Wait()
return out, nil
}
@@ -1,181 +0,0 @@
package main
import (
"context"
"encoding/json"
"fmt"
"strings"
"sync"
"time"
mcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-sdk/go/buscalls"
)
// The reads of a composition step (design 47 G §354, A-252, A-261): the entries of one catalog revision are read once and shared by
// every composition step of that revision, since an entry is content-addressed and a revision never changes; a read that times out
// or fails in transport is made again with a growing wait for a bounded time, said in the journal, and only then ends the step as
// could not run.
const (
// readsBound is how long a read is made again before the step could not run.
readsBound = 3 * time.Minute
// readsFirstWait and readsMaxWait bound the wait between two tries.
readsFirstWait = time.Second
readsMaxWait = 15 * time.Second
// readsKept is how many catalog revisions' halves are kept; the oldest read goes first.
readsKept = 8
)
// readKey names one revision of one watched branch.
type readKey struct {
branch string
revision int
}
// readSlot is one revision's read: done closes when half or err is set.
type readSlot struct {
done chan struct{}
half mcc.Half
err error
}
// compositionReads shares the reads of composition steps.
type compositionReads struct {
mu sync.Mutex
revs map[readKey]*readSlot
order []readKey
seats map[string]json.RawMessage // by seat and digest: the definition, raw
}
// transient says whether a read's fault is of the transport or a timeout, so that it is worth reading again.
func transient(err error, said string) bool {
text := said
if err != nil {
text = err.Error() + " " + said
}
text = strings.ToLower(text)
for _, w := range []string{"deadline exceeded", "timeout", "timed out", "no responders", "connection", "i/o", "eof", "broken pipe", "unavailable"} {
if strings.Contains(text, w) {
return true
}
}
return false
}
// callRetry calls the seat verb key, again with a growing wait while its fault is transient, for readsBound; it answers the body of
// a good answer, or the last fault.
func (r *compositionRun) callRetry(ctx context.Context, key string, body any) (json.RawMessage, string, error) {
start := r.now()
wait := readsFirstWait
for try := 1; ; try++ {
raw, err := r.call(key, body)
var inner json.RawMessage
var said string
if err == nil {
var ok bool
if inner, said, ok = buscalls.Unwrap(raw); ok {
return inner, "", nil
}
}
if !transient(err, said) {
return nil, said, err
}
if r.now().Sub(start)+wait > readsBound {
return nil, said, err
}
r.logf("composition step: %s is not read (try %d, %v %s), read again in %s", key, try, err, said, wait)
if serr := r.waitFor(ctx, wait); serr != nil {
return nil, said, serr
}
if wait *= 2; wait > readsMaxWait {
wait = readsMaxWait
}
}
}
// waitFor waits d, or until ctx ends.
func (r *compositionRun) waitFor(ctx context.Context, d time.Duration) error {
if r.sleep != nil {
return r.sleep(ctx, d)
}
t := time.NewTimer(d)
defer t.Stop()
select {
case <-ctx.Done():
return ctx.Err()
case <-t.C:
return nil
}
}
// half is the revision's half, read once for every step of it: a step that finds it being read waits for that read, and a read that
// failed is not kept, so the next step reads again.
func (r *compositionRun) half(ctx context.Context, key readKey, read func() (mcc.Half, error)) (mcc.Half, error) {
c := r.reads
c.mu.Lock()
if c.revs == nil {
c.revs = map[readKey]*readSlot{}
}
if s, ok := c.revs[key]; ok {
c.mu.Unlock()
select {
case <-s.done:
if s.err == nil {
return s.half, nil
}
case <-ctx.Done():
return mcc.Half{}, ctx.Err()
}
// The read this step waited for failed: read it here (the failed slot is already dropped).
return r.half(ctx, key, read)
}
s := &readSlot{done: make(chan struct{})}
c.revs[key] = s
c.order = append(c.order, key)
for len(c.order) > readsKept {
delete(c.revs, c.order[0])
c.order = c.order[1:]
}
c.mu.Unlock()
s.half, s.err = read()
if s.err != nil {
c.mu.Lock()
if c.revs[key] == s {
delete(c.revs, key)
}
for i, k := range c.order {
if k == key {
c.order = append(c.order[:i], c.order[i+1:]...)
break
}
}
c.mu.Unlock()
}
close(s.done)
return s.half, s.err
}
// seatKept is the definition kept of a seat at a digest.
func (c *compositionReads) seatKept(seat, digest string) (json.RawMessage, bool) {
c.mu.Lock()
defer c.mu.Unlock()
d, ok := c.seats[seat+"@"+digest]
return d, ok
}
func (c *compositionReads) keepSeat(seat, digest string, def json.RawMessage) {
c.mu.Lock()
defer c.mu.Unlock()
if c.seats == nil {
c.seats = map[string]json.RawMessage{}
}
c.seats[seat+"@"+digest] = def
}
func unreadErr(what string, err error, said string) error {
if err != nil {
return fmt.Errorf("%s: %w", what, err)
}
return fmt.Errorf("%s: %s", what, said)
}
@@ -18,17 +18,16 @@ import (
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer"
"git.novox.be/novox/mesh-sdk/go/buscalls"
"git.novox.be/novox/mesh-sdk/go/keys"
)
// The run of a composition step (design 47 G §354, §63, §64, §718; A-226, A-238, A-240, A-252, A-254, A-261): the step reads one
// composition run per machine of its candidate (compositionruns.go), and the step that starts a machine's run composes it: one
// compose-candidate ask of the node, one composer.Request, depth compose, the one machine and one case `change`, its manifests,
// verdicts and seats from the step's catalog revision read through mesh-catalog's `revision`, kept once in `inputs`; one run ask
// of the seat node-build-go, naming no head, its program the node-engine build's catalog entry with the composer's package,
// chosen once and recorded in the run. The run's end is heard on the build seat's step-ended of that ask, its answer (a
// composer.Verdict) read from `output`, and written in the run's key, whose change every reading step hears. A machine the
// watchman said away (machine-away) is not composed and passes with a warning (§64).
// The run of a composition step (design 47 G §354, §63, §64; A-226, A-238, A-240, A-243, A-252, A-254, A-261): one
// composer.Request per machine, depth compose, the one machine and one case `change`, its manifests, verdicts and seats from the
// step's catalog revision read through mesh-catalog's `revision`; one run ask per machine of the seat node-build-go, naming no
// head, its program the node-engine build's catalog entry with the composer's package; the run's end heard on the build seat's
// step-ended of that ask, its answer (a composer.Verdict) read inline from `output`. A machine the watchman said away
// (machine-away) is not composed and passes with a warning (§64).
const (
// runAsker is the assigner's name as an asker of a build seat, its seat's own name: the last token of the subject its run
@@ -43,10 +42,74 @@ const (
seatDefinitionsKey = "seat:" + meshcatalog.Seat + "." + meshcatalog.VerbSeatDefinitions
)
// runEnds hands each build seat's step-ended of an ask to the composition run that waits for it (A-240), by the ask's id.
type runEnds struct {
mu sync.Mutex
waits map[string]chan buildstep.StepEnded
}
// expect is the end of the ask id, once heard; called before the ask is put, so no end is missed.
func (r *runEnds) expect(id string) <-chan buildstep.StepEnded {
r.mu.Lock()
defer r.mu.Unlock()
if r.waits == nil {
r.waits = map[string]chan buildstep.StepEnded{}
}
ch := make(chan buildstep.StepEnded, 1)
r.waits[id] = ch
return ch
}
// forget stops waiting for the ask id.
func (r *runEnds) forget(id string) {
r.mu.Lock()
delete(r.waits, id)
r.mu.Unlock()
}
// heard hands an end to the run that waits for its ask; one nobody waits for is another asker's, left.
func (r *runEnds) heard(e buildstep.StepEnded) {
r.mu.Lock()
ch := r.waits[e.Ask]
delete(r.waits, e.Ask)
r.mu.Unlock()
if ch != nil {
ch <- e
}
}
// awayNodes are the machines the watchman said away (machine-away of the seat mesh-watchdogs, §389) and not back since.
type awayNodes struct {
mu sync.Mutex
away map[string]bool
}
// set keeps machine away (true) or back (false).
func (a *awayNodes) set(machine string, away bool) {
a.mu.Lock()
defer a.mu.Unlock()
if a.away == nil {
a.away = map[string]bool{}
}
if away {
a.away[machine] = true
} else {
delete(a.away, machine)
}
}
// is says machine is away.
func (a *awayNodes) is(machine string) bool {
if a == nil {
return false
}
a.mu.Lock()
defer a.mu.Unlock()
return a.away[machine]
}
// compositionRun runs a composition step.
type compositionRun struct {
// runs are the composition runs of the state composition-runs (compositionruns.go).
runs *compositionRuns
inputs *compositionInputs
keeper *inputKeeper
// call asks a seat verb held once for the mesh (process.Conn.CallKept).
@@ -55,14 +118,10 @@ type compositionRun struct {
trunk func() (mcc.Half, bool)
// running is the last `running` heard of a module on a node (world.runningOf), never the node-engine's bucket.
running func(node, module string) (ne.Running, bool)
// away says the watchman holds machine away (machine-away, not back since): the one presence the assigner keeps (world.awayNow),
// so a durable on the watchman's streams has one hearer.
away func(machine string) bool
away *awayNodes
// put puts a run ask on the build seat's check queue under the assigner's name.
put func(ctx context.Context, a buildstep.StepAsk) error
// reads shares the catalog reads among steps (compositionreads.go); sleep waits between two tries of a read (nil: a timer).
reads *compositionReads
sleep func(ctx context.Context, d time.Duration) error
put func(ctx context.Context, a buildstep.StepAsk) error
ends *runEnds
// ask asks a seat verb on one node, for the run's answer kept as the build seat's object (buildstep.StepEnded.ReadOutput).
ask buildstep.AskVerb
now func() time.Time
@@ -71,105 +130,56 @@ type compositionRun struct {
// machineEnd is what one machine came to.
type machineEnd struct {
failed, couldNot string
warnings []string
// judged is set where the composer judged the machine, composing or not (A-467a).
judged bool
failed, couldNot, warning string
}
// run runs one composition step and answers what it came to: per machine not away it joins or starts the candidate's
// composition run (§718), waits for each run's end in its key, and judges each machine from it; failed when a machine does not
// compose, could not run when a machine could not be composed or was not judged, else passed with a warning per machine away.
func (r *compositionRun) run(ctx context.Context, id string, step meshwants.StepRun) stepOutcome {
candidate, err := candidateKeyOf(step)
// run runs one composition step and answers what it came to: failed when a machine does not compose, could not run when a machine
// could not be composed, else passed with a warning per machine away or not judged.
func (r *compositionRun) run(ctx context.Context, _ string, run meshwants.StepRun) stepOutcome {
half, err := r.revision(run)
if err != nil {
return couldNotRun("the step's catalog revision is not read: " + err.Error())
}
seats, err := r.seats(half.Revision.Definitions)
if err != nil {
return couldNotRun(err.Error())
}
nodes := r.inputs.machines(step.Module, step.Branch)
if len(nodes) == 0 {
return couldNotRun(fmt.Sprintf("no node places %s on the branch %q, so the composition step has no machine to compose", step.Module, step.Branch))
manifests, verdicts, err := caseOf(half)
if err != nil {
return couldNotRun(err.Error())
}
reader := readerOf(id, step)
prep := &runPrep{r: r, step: step}
var ends []machineEnd
var keys []string
for _, node := range nodes {
if r.away != nil && r.away(node) {
ends = append(ends, machineEnd{warnings: []string{node + " is away (the watchman's machine-away), so it is not composed"}})
continue
}
join := r.runs.join
if step.Rerun { // a step asked again (§727)
join = r.runs.rejoin
}
j, err := join(reader, candidate, node, buildstep.NewID())
if err != nil {
ends = append(ends, machineEnd{couldNot: node + ": " + err.Error()})
continue
}
if j.started {
go r.start(ctx, j.key, j.run, prep)
}
keys = append(keys, j.key)
programOf, err := r.programs(half)
if err != nil {
return couldNotRun(err.Error())
}
inputs, err := r.inputs.gather(ctx, run, r.away.is)
if err != nil {
return couldNotRun(err.Error())
}
return r.read(ctx, step.Module, keys, ends)
}
// read waits for the end of each run of keys and judges the step of module from them, naming in its progress and its end the
// runs it reads and their asks (§723, §729).
func (r *compositionRun) read(ctx context.Context, module string, keys []string, ends []machineEnd) stepOutcome {
setShared(ctx, r.runs.sharedOf(keys, nil))
setPhase(ctx, meshwants.StepProgressEventPhaseRun)
runs := r.runs.await(ctx, keys)
ends := make([]machineEnd, len(inputs))
var wg sync.WaitGroup
for i, in := range inputs {
wg.Add(1)
go func() {
defer wg.Done()
ends[i] = r.machine(ctx, in, buildstep.NewID(), programOf, manifests, verdicts, seats)
}()
}
wg.Wait()
setPhase(ctx, meshwants.StepProgressEventPhaseReport)
for _, key := range keys {
ends = append(ends, judgeRun(module, runs[key]))
}
out := outcomeOf(ends)
out.Shared = r.runs.sharedOf(keys, runs)
return out
}
// resume judges a step taken up at the assigner's start (§719, A-468): the runs of its candidate naming it, once each has ended.
func (r *compositionRun) resume(ctx context.Context, id string, step meshwants.StepRun) stepOutcome {
candidate, err := candidateKeyOf(step)
if err != nil {
return couldNotRun(err.Error())
}
keys, err := r.runs.keysOf(candidate, id)
if err != nil {
return couldNotRun("the step's composition runs are not read: " + err.Error())
}
return r.read(ctx, step.Module, keys, nil)
}
// outcomeOf is a step's outcome from its machines: failed before could not run before passed (§354); a step that judged no
// machine never passes (A-467a), each machine not judged named with why.
func outcomeOf(ends []machineEnd) stepOutcome {
var out stepOutcome
var failed, couldNot []string
judged := 0
for _, e := range ends {
if e.judged {
judged++
}
failed = appendSome(failed, e.failed)
couldNot = appendSome(couldNot, e.couldNot)
out.Warnings = append(out.Warnings, e.warnings...)
out.Warnings = appendSome(out.Warnings, e.warning)
}
switch {
case len(failed) > 0:
out.Outcome, out.Said = meshwants.StepEndedOutcomeFailed, strings.Join(failed, "; ")
case len(couldNot) > 0:
out.Outcome, out.Said = meshwants.StepEndedOutcomeCouldNotRun, strings.Join(couldNot, "; ")
case judged == 0:
said := "no machine was judged"
if len(out.Warnings) == 0 {
said += ": the step has no machine"
} else {
said += ": " + strings.Join(out.Warnings, "; ")
}
out.Outcome, out.Said = meshwants.StepEndedOutcomeCouldNotRun, said
default:
out.Outcome = meshwants.StepEndedOutcomePassed
}
@@ -187,207 +197,98 @@ func couldNotRun(said string) stepOutcome {
return stepOutcome{Outcome: meshwants.StepEndedOutcomeCouldNotRun, Said: said}
}
// runPrep is what the step that starts a run reads of its catalog revision, once for every machine it starts: the seats, the
// case's manifests and verdicts, and the composer program per machine.
type runPrep struct {
r *compositionRun
step meshwants.StepRun
once sync.Once
// read once
seats []json.RawMessage
manifests map[string]composer.Manifest
verdicts map[string]json.RawMessage
none map[string]string
programOf func(node string) (buildstep.RunProgram, string, error)
err error
}
func (p *runPrep) get(ctx context.Context) error {
p.once.Do(func() {
half, err := p.r.revision(ctx, p.step)
if err != nil {
p.err = errors.New("the step's catalog revision is not read: " + err.Error())
return
}
if p.seats, p.err = p.r.seats(ctx, half.Revision.Definitions); p.err != nil {
return
}
if p.manifests, p.verdicts, p.none, p.err = caseOf(half); p.err != nil {
return
}
p.programOf, p.err = p.r.programs(ctx, half)
})
return p.err
}
// start composes the machine of the run it started (§718): the node's input of this moment, its request kept once in `inputs`,
// the composer program chosen and recorded with the request's digest, and one run ask put. A fault ends the run could not run.
func (r *compositionRun) start(ctx context.Context, key string, run meshwants.CompositionRun, prep *runPrep) {
node, ask := run.Machine, run.Ask
end := func(said string) {
if _, err := r.runs.endRun(key, ask, meshwants.CompositionRunOutcomeCouldNotRun, said, nil); err != nil {
r.logf("the composition run %s is not ended: %v", key, err)
}
// machine composes one machine: its request kept under its ask, the ask put, its end heard and its verdict read.
func (r *compositionRun) machine(ctx context.Context, in nodeInput, ask string, programOf func(node string) (buildstep.RunProgram, error),
manifests map[string]composer.Manifest, verdicts map[string]json.RawMessage, seats []json.RawMessage) machineEnd {
node := in.Node
switch {
case in.Away:
return machineEnd{warning: node + " is away (the watchman's machine-away), so it is not composed"}
case in.Why != "":
return machineEnd{couldNot: node + " gave no composition input: " + in.Why}
}
if err := prep.get(ctx); err != nil {
end(node + ": " + err.Error())
return
}
in := r.inputs.of(ctx, node, run.Candidate)
if in.Why != "" {
end(node + " gave no composition input: " + in.Why)
return
}
program, engineFp, err := prep.programOf(node)
program, err := programOf(node)
if err != nil {
end(node + ": " + err.Error())
return
return machineEnd{couldNot: node + ": " + err.Error()}
}
req := composer.Request{V: composer.Version, Input: in.Body, Depth: composer.DepthCompose, Machines: []string{node},
Cases: []composer.Case{{Name: changeCase, Manifests: prep.manifests, Verdicts: prep.verdicts, None: prep.none}}, Seats: prep.seats,
BlameTries: r.runs.set.blameTries} // the composer's bound of its search for culprits (§721, A-466r)
Cases: []composer.Case{{Name: changeCase, Manifests: manifests, Verdicts: verdicts}}, Seats: seats}
raw, err := json.Marshal(req)
if err != nil {
end(node + ": its request cannot be encoded: " + err.Error())
return
return machineEnd{couldNot: node + ": its request cannot be encoded: " + err.Error()}
}
digest, err := r.keeper.keep(ask, raw)
if err != nil {
if _, err := r.keeper.keep(ask, raw); err != nil {
r.logf("the composition request of %s is refused: %v", node, err)
end(node + ": its request is not kept: " + err.Error())
return
return machineEnd{couldNot: node + ": its request is not kept: " + err.Error()}
}
ref, err := buildstep.RunInputOf(inputVerbKey, raw)
if err != nil {
end(node + ": " + err.Error())
return
}
if _, _, err := r.runs.change(key, func(c *meshwants.CompositionRun) bool {
if c.Ask != ask || c.Ended != nil {
return false
}
c.Program = &meshwants.CompositionRunProgram{Repository: program.Repository, Commit: program.Commit, Directory: program.Dir, Fingerprint: engineFp}
c.Input = digest
return true
}); err != nil {
end(node + ": the run's program and input are not written: " + err.Error())
return
return machineEnd{couldNot: node + ": " + err.Error()}
}
a := buildstep.StepAsk{ID: ask, Kind: buildstep.AskCheck, Language: buildstep.AskGo, Asker: runAsker, At: r.now().UTC(),
Run: &buildstep.RunJob{Program: program, Heads: []buildstep.RunHead{}, Input: ref, Reader: "the composition of " + node + " by the node-engine at " + program.Commit}}
ended := r.ends.expect(ask)
defer r.ends.forget(ask)
if err := r.put(ctx, a); err != nil {
end(node + ": its run ask " + ask + " is not put: " + err.Error())
return machineEnd{couldNot: node + ": its run ask " + ask + " is not put: " + err.Error()}
}
}
// notTried is the problem the composer names a module with that its search for culprits did not try, past the request's
// blame_tries (design 47 G §721; the composer's composer.NotTried in mesh-host, compared by its words, A-470b).
const notTried = "not tried: the search reached its bound"
// judgeRun judges one machine of the step of module from the run it read (§721, §722): could not run with the run's words; else
// from the machine's composer verdict: not judged is could not run (A-467); a verdict blaming the step's module fails it, or is
// could not run where the module was not tried (A-470b); a machine that does not compose and blames no module fails every step on
// it, on the node's own account; a verdict with no `blamed` (a composer before §721) is judged as before, with the warning "the
// composer names no culprits"; otherwise the machine passes, with a warning for each other module blamed there.
func judgeRun(module string, run meshwants.CompositionRun) machineEnd {
node := run.Machine
if run.Ended == nil {
return machineEnd{couldNot: node + ": the step stopped before its composition run " + runKeyOf(run.Candidate, node) + " ended"}
var e buildstep.StepEnded
select {
case <-ctx.Done():
return machineEnd{couldNot: node + ": the step stopped before its run ask " + ask + " ended"}
case e = <-ended:
}
if run.Outcome != meshwants.CompositionRunOutcomePassed {
return machineEnd{couldNot: run.Said}
if e.Outcome != buildstep.OutcomePassed {
return machineEnd{couldNot: fmt.Sprintf("%s: its run ask %s ended %s on %s: %s", node, ask, e.Outcome, e.Node, e.Said)}
}
m, err := machineOf(run.Verdict)
answer, err := e.ReadOutput(buildstep.AskGo, r.ask) // the program's last line of standard output (A-254)
if err != nil {
return machineEnd{couldNot: node + ": the run's verdict does not read: " + err.Error()}
return machineEnd{couldNot: node + ": " + err.Error()}
}
if m.NotJudged != "" { // a step never passes on a machine it did not judge (A-467)
return machineEnd{couldNot: node + " was not judged: " + m.NotJudged}
}
why := strings.Join(appendSome(append(slices.Clone(m.Problems), m.Refusals...), m.Reason), "; ")
switch {
case m.Blamed == nil && !m.Composes: // an old composer (§721)
return machineEnd{judged: true, failed: node + " does not compose: " + why, warnings: []string{node + ": the composer names no culprits"}}
case len(m.Blamed) == 0 && !m.Composes: // the node's own account (A-452)
return machineEnd{judged: true, failed: node + " does not compose and blames no module: " + why}
}
end := machineEnd{judged: true}
var own, others []string
for _, b := range m.Blamed {
switch {
case b.Module == module && b.Problem == notTried:
end.couldNot = fmt.Sprintf("%s holds %s untried: the composer's search for culprits reached its bound of blame_tries (A-470b)", node, module)
case b.Module == module:
own = append(own, b.Problem)
default:
others = append(others, node+" holds "+b.Module+": "+b.Problem)
}
}
if len(own) > 0 {
end.failed, end.couldNot = node+" holds "+module+": "+strings.Join(own, "; "), ""
}
end.warnings = others
return end
return verdictOf(node, answer)
}
// machineOf is the composer's verdict of one machine as a run keeps it.
func machineOf(v map[string]any) (composer.Machine, error) {
var m composer.Machine
raw, err := json.Marshal(v)
if err == nil {
err = json.Unmarshal(raw, &m)
}
return m, err
}
// runVerdict reads the composer's answer of one machine: its verdict as the run keeps it, or why it is none.
func runVerdict(node string, answer []byte) (map[string]any, string) {
// verdictOf reads the composer's verdict of one machine: not composing is failed, naming why; not judged passes with a warning.
func verdictOf(node string, answer []byte) machineEnd {
var v composer.Verdict
if err := json.Unmarshal(answer, &v); err != nil {
return nil, node + ": the composer answered no verdict: " + err.Error()
return machineEnd{couldNot: node + ": the composer answered no verdict: " + err.Error()}
}
if v.V != composer.Version {
return nil, fmt.Sprintf("%s: the composer answered a verdict of version %d, the assigner reads %d", node, v.V, composer.Version)
return machineEnd{couldNot: fmt.Sprintf("%s: the composer answered a verdict of version %d, the assigner reads %d", node, v.V, composer.Version)}
}
for _, c := range v.Cases {
if c.Name != changeCase {
continue
}
m, ok := c.Machines[node]
if !ok {
return nil, node + ": the composer's verdict names no such machine"
switch {
case !ok:
return machineEnd{couldNot: node + ": the composer's verdict names no such machine"}
case m.NotJudged != "":
return machineEnd{warning: node + " was not judged: " + m.NotJudged}
case !m.Composes:
return machineEnd{failed: node + " does not compose: " + strings.Join(appendSome(append(slices.Clone(m.Problems), m.Refusals...), m.Reason), "; ")}
}
raw, err := json.Marshal(m)
var out map[string]any
if err == nil {
err = json.Unmarshal(raw, &out)
}
if err != nil {
return nil, node + ": the composer's verdict cannot be kept: " + err.Error()
}
return out, ""
return machineEnd{}
}
return nil, node + ": the composer's verdict has no case " + changeCase
return machineEnd{couldNot: node + ": the composer's verdict has no case " + changeCase}
}
// revision reads the step's catalog revision whole, page by page, through mesh-catalog's `revision` (A-252), once for every step of
// it (compositionreads.go).
func (r *compositionRun) revision(ctx context.Context, run meshwants.StepRun) (mcc.Half, error) {
// revision reads the step's catalog revision whole, page by page, through mesh-catalog's `revision` (A-252).
func (r *compositionRun) revision(run meshwants.StepRun) (mcc.Half, error) {
if run.Revision <= 0 {
return mcc.Half{}, errors.New("the step names no catalog revision")
}
return r.half(ctx, readKey{run.Branch, int(run.Revision)}, func() (mcc.Half, error) { return r.readRevision(ctx, run) })
}
func (r *compositionRun) readRevision(ctx context.Context, run meshwants.StepRun) (mcc.Half, error) {
half := mcc.Half{Entries: map[string]meshcatalog.CatalogEntry{}}
in := meshcatalog.RevisionInput{Branch: keys.EscapeName(run.Branch), Revision: run.Revision}
for {
body, said, err := r.callRetry(ctx, mcc.RevisionVerb, in)
raw, err := r.call(mcc.RevisionVerb, in)
if err != nil {
return mcc.Half{}, err
}
if body == nil {
body, said, ok := buscalls.Unwrap(raw)
if !ok {
return mcc.Half{}, fmt.Errorf("%s answered: %s", mcc.RevisionVerb, said)
}
var a meshcatalog.RevisionAnswer
@@ -410,17 +311,28 @@ func (r *compositionRun) readRevision(ctx context.Context, run meshwants.StepRun
}
}
// seats are the definitions the composer is handed, raw. A revision that names definitions (A-261): the definition of each (seat, digest)
// pair, from mesh-catalog's seat-definitions where the trunk's digest is the revision's; a digest neither holds is an error naming the seat
// and the digest. A revision that names none (A-234's stand-in, A-467): every definition of the trunk seat list, read through
// seat-definitions; a list or a definition that cannot be read is an error naming it, never an empty list.
func (r *compositionRun) seats(ctx context.Context, defs []meshcatalog.CatalogRevisionDefinitionsItem) ([]json.RawMessage, error) {
if len(defs) == 0 {
return r.trunkSeats(ctx)
}
// seats are the definitions of the revision's (seat, digest) pairs, raw (A-261): the revision's definitions carry no definition
// yet (A-234), so each is mesh-catalog's seat-definitions answer for the seat where the trunk's digest is the revision's; a digest
// neither holds is an error naming the seat and the digest.
func (r *compositionRun) seats(defs []meshcatalog.CatalogRevisionDefinitionsItem) ([]json.RawMessage, error) {
out := make([]json.RawMessage, 0, len(defs))
for _, d := range defs {
def, err := r.seatAt(ctx, d.Seat, d.Digest)
raw, err := r.call(seatDefinitionsKey, meshcatalog.SeatDefinitionsInput{Seat: d.Seat})
if err != nil {
return nil, fmt.Errorf("the definition of the seat %s is not read: %w", d.Seat, err)
}
body, said, ok := buscalls.Unwrap(raw)
if !ok {
return nil, fmt.Errorf("the definition of the seat %s is not read: %s", d.Seat, said)
}
var a meshcatalog.SeatDefinitionsAnswer
if err := json.Unmarshal(body, &a); err != nil {
return nil, fmt.Errorf("the definition of the seat %s is not read: %w", d.Seat, err)
}
if a.Entry == nil || a.Entry.Digest != d.Digest || a.Entry.Definition == nil {
return nil, fmt.Errorf("no definition of the seat %s with the digest %s is held: neither the revision nor the trunk holds it", d.Seat, d.Digest)
}
def, err := json.Marshal(a.Entry.Definition)
if err != nil {
return nil, err
}
@@ -429,78 +341,20 @@ func (r *compositionRun) seats(ctx context.Context, defs []meshcatalog.CatalogRe
return out, nil
}
// trunkSeats are the definitions of every seat of the trunk's seat list (seat-definitions with no seat names them with their digests).
func (r *compositionRun) trunkSeats(ctx context.Context) ([]json.RawMessage, error) {
what := "the trunk's seat list is not read"
body, said, err := r.callRetry(ctx, seatDefinitionsKey, meshcatalog.SeatDefinitionsInput{})
if err != nil || body == nil {
return nil, unreadErr(what, err, said)
}
var list meshcatalog.SeatDefinitionsAnswer
if err := json.Unmarshal(body, &list); err != nil {
return nil, fmt.Errorf("%s: %w", what, err)
}
if len(list.Seats) == 0 {
return nil, errors.New(what + ": mesh-catalog's seat-definitions names no seat, so the composer has no seat list to judge by")
}
out := make([]json.RawMessage, 0, len(list.Seats))
for _, s := range list.Seats {
def, err := r.seatAt(ctx, s.Seat, s.Digest)
if err != nil {
return nil, err
}
out = append(out, def)
}
return out, nil
}
// seatAt is the definition of a seat at a digest, kept once read (a definition at a digest never changes).
func (r *compositionRun) seatAt(ctx context.Context, seat, digest string) (json.RawMessage, error) {
if def, ok := r.reads.seatKept(seat, digest); ok {
return def, nil
}
what := "the definition of the seat " + seat + " is not read"
body, said, err := r.callRetry(ctx, seatDefinitionsKey, meshcatalog.SeatDefinitionsInput{Seat: seat})
if err != nil || body == nil {
return nil, unreadErr(what, err, said)
}
var a meshcatalog.SeatDefinitionsAnswer
if err := json.Unmarshal(body, &a); err != nil {
return nil, fmt.Errorf("%s: %w", what, err)
}
if a.Entry == nil || a.Entry.Digest != digest || a.Entry.Definition == nil {
return nil, fmt.Errorf("no definition of the seat %s with the digest %s is held: neither the revision nor the trunk holds it", seat, digest)
}
def, err := json.Marshal(a.Entry.Definition)
if err != nil {
return nil, err
}
r.reads.keepSeat(seat, digest, def)
return def, nil
}
// caseOf is the case's manifests, verdicts and none from the revision's entries: each manifest whole, the verdict of each one not
// accepted (a module without one is accepted), and the revision's none words of each module it gives no fingerprint, whose
// build can never be made (design 47 G A-470j, A-470k).
func caseOf(half mcc.Half) (map[string]composer.Manifest, map[string]json.RawMessage, map[string]string, error) {
// caseOf is the case's manifests and verdicts from the revision's entries: each manifest whole, and the verdict of each one not
// accepted (a module without one is accepted).
func caseOf(half mcc.Half) (map[string]composer.Manifest, map[string]json.RawMessage, error) {
manifests := map[string]composer.Manifest{}
verdicts := map[string]json.RawMessage{}
var none map[string]string
for name, e := range half.Entries {
if len(e.Manifest) == 0 {
continue
}
raw, err := json.Marshal(e.Manifest)
if err != nil {
return nil, nil, nil, fmt.Errorf("the manifest of %s cannot be encoded: %w", name, err)
return nil, nil, fmt.Errorf("the manifest of %s cannot be encoded: %w", name, err)
}
manifests[name] = composer.Manifest{Raw: raw}
if e.Fp == "" && e.None != "" {
if none == nil {
none = map[string]string{}
}
none[name] = e.None
}
if e.Verdict == "" || e.Verdict == mcc.VerdictAccepted {
continue
}
@@ -513,11 +367,11 @@ func caseOf(half mcc.Half) (map[string]composer.Manifest, map[string]json.RawMes
}
raw, err = json.Marshal(v)
if err != nil {
return nil, nil, nil, err
return nil, nil, err
}
verdicts[name] = raw
}
return manifests, verdicts, none, nil
return manifests, verdicts, nil
}
// nodeEngineEntry is the entry of the module that claims the seat node-engine, or false.
@@ -555,56 +409,76 @@ func programOfEntry(e meshcatalog.CatalogEntry) (buildstep.RunProgram, error) {
return buildstep.RunProgram{Repository: e.Repository, Commit: e.Commit, Dir: dir, Package: composer.Package}, nil
}
// programs answers, per machine, the composer its run builds and the node-engine fingerprint it was chosen from (A-226, A-243,
// A-252, §719): where the revision's node-engine entry differs from the trunk's, that build for every machine; else the build the
// machine's node-engine runs, as its `running` says, its entry read through mesh-catalog's `entry`; a machine whose node-engine's
// `running` is not heard gets the revision's entry, said (A-451).
func (r *compositionRun) programs(ctx context.Context, half mcc.Half) (func(node string) (buildstep.RunProgram, string, error), error) {
// programs answers, per machine, the composer its run builds (A-226, A-243, A-252): where the revision's node-engine entry differs
// from the trunk's, that build for every machine; else the build the machine's node-engine runs, as its `running` says, its entry
// read through mesh-catalog's `entry`; a machine whose node-engine's `running` is not heard gets the revision's entry, said (A-451).
func (r *compositionRun) programs(half mcc.Half) (func(node string) (buildstep.RunProgram, error), error) {
engine, ok := nodeEngineEntry(half.Entries)
if !ok {
return nil, fmt.Errorf("the catalog revision %d holds no module claiming the seat %s", half.Revision.Revision, ne.Seat)
}
if trunk, ok := r.trunk(); !ok || func() bool { t, ok := nodeEngineEntry(trunk.Entries); return !ok || t.Fp != engine.Fp }() {
p, err := programOfEntry(engine)
return func(string) (buildstep.RunProgram, string, error) { return p, engine.Fp, err }, nil
return func(string) (buildstep.RunProgram, error) { return p, err }, nil
}
var mu sync.Mutex
byFp := map[string]buildstep.RunProgram{}
return func(node string) (buildstep.RunProgram, string, error) {
return func(node string) (buildstep.RunProgram, error) {
running, ok := r.running(node, engine.Module)
if !ok || running.Fingerprint == "" {
r.logf("%s: no build of %s is heard running there, so its composer is the catalog revision's entry %s (A-451)", node,
engine.Module, engine.Fp)
p, err := programOfEntry(engine)
return p, engine.Fp, err
return programOfEntry(engine)
}
fp := running.Fingerprint
mu.Lock()
p, ok := byFp[fp]
p, ok := byFp[running.Fingerprint]
mu.Unlock()
if ok {
return p, fp, nil
return p, nil
}
what := "the entry of the node-engine's build " + fp + " is not read"
body, said, err := r.callRetry(ctx, entryVerbKey, meshcatalog.EntryInput{Fingerprint: fp})
if err != nil || body == nil {
return buildstep.RunProgram{}, fp, unreadErr(what, err, said)
raw, err := r.call(entryVerbKey, meshcatalog.EntryInput{Fingerprint: running.Fingerprint})
if err != nil {
return buildstep.RunProgram{}, fmt.Errorf("the entry of the node-engine's build %s is not read: %w", running.Fingerprint, err)
}
body, said, ok := buscalls.Unwrap(raw)
if !ok {
return buildstep.RunProgram{}, fmt.Errorf("the entry of the node-engine's build %s is not read: %s", running.Fingerprint, said)
}
var a meshcatalog.EntryAnswer
if err := json.Unmarshal(body, &a); err != nil {
return buildstep.RunProgram{}, fp, fmt.Errorf("%s: %w", what, err)
return buildstep.RunProgram{}, fmt.Errorf("the entry of the node-engine's build %s is not read: %w", running.Fingerprint, err)
}
p, err = programOfEntry(a.Entry)
if err != nil {
return p, fp, err
return p, err
}
mu.Lock()
byFp[fp] = p
byFp[running.Fingerprint] = p
mu.Unlock()
return p, fp, nil
return p, nil
}, nil
}
// heardPresence keeps a machine-away (away true) or machine-back of the seat mesh-watchdogs (§389, §436): the machine it names.
func heardPresence(a *awayNodes, away bool, m jetstream.Msg, logf func(string, ...any)) {
var e struct {
Machine string `json:"machine"`
}
if err := json.Unmarshal(m.Data(), &e); err != nil || e.Machine == "" {
logf("%s holds no machine, left: %v", m.Subject(), err)
} else {
a.set(e.Machine, away)
if away {
logf("%s is away (machine-away): a composition step does not compose it until its machine-back", e.Machine)
} else {
logf("%s is back (machine-back): a composition step composes it again", e.Machine)
}
}
if err := m.Ack(); err != nil {
logf("acknowledging %s: %v", m.Subject(), err)
}
}
// trunkOf is the trunk's catalog half the assigner's views hold, trunk naming the first watched branch, or false while none is
// held.
func trunkOf(iv *catalogViews, trunk func() string) func() (mcc.Half, bool) {
@@ -4,7 +4,6 @@ import (
"context"
"encoding/json"
"errors"
"slices"
"strings"
"sync"
"testing"
@@ -31,16 +30,6 @@ type runStand struct {
requests map[string]composer.Request // by node
entries []string // the fingerprints `entry` was asked
r *compositionRun
runs *compositionRuns
puts int // the run asks put
// noDefinitions: the revision names no seat definition (A-234's stand-in); trunkSeats: the trunk's seat list as (seat, digest)
// pairs; unjudged: the machine the composer does not judge; listFault: the trunk's seat list cannot be read.
noDefinitions bool
trunkSeats []meshcatalog.SeatDefinitionsAnswerSeatsItem
unjudged string
listFault error
// verdict, when set, is the composer's verdict of a machine (by node), else the stand's own.
verdict func(node string) (composer.Machine, bool)
}
func newRunStand(t *testing.T, revisionEngine, trunkEngine, seatDigest string) *runStand {
@@ -57,8 +46,7 @@ func newRunStand(t *testing.T, revisionEngine, trunkEngine, seatDigest string) *
"": {Revision: &meshcatalog.CatalogRevision{Revision: 7, Branch: "main", Definitions: []meshcatalog.CatalogRevisionDefinitionsItem{{Seat: "s1", Digest: "d1"}}},
Entries: []meshcatalog.CatalogEntry{engine(revisionEngine, revisionCommit), {Module: "bad", Verdict: mcc.VerdictRefused, Inputs: &meshcatalog.EntryInputs{Problems: []string{"no seat x"}},
Manifest: map[string]any{"module": "bad"}}}, Next: "bad"},
"bad": {Revision: &meshcatalog.CatalogRevision{Revision: 7}, Entries: []meshcatalog.CatalogEntry{{Module: "m", Verdict: mcc.VerdictAccepted, Manifest: map[string]any{"module": "m"}},
{Module: "lidarr", None: "builders disagree: node", Manifest: map[string]any{"module": "lidarr"}}}}, // no fingerprint (A-470j)
"bad": {Revision: &meshcatalog.CatalogRevision{Revision: 7}, Entries: []meshcatalog.CatalogEntry{{Module: "m", Verdict: mcc.VerdictAccepted, Manifest: map[string]any{"module": "m"}}}},
}
call := func(key string, body any) (json.RawMessage, error) {
raw, _ := json.Marshal(body)
@@ -69,22 +57,8 @@ func newRunStand(t *testing.T, revisionEngine, trunkEngine, seatDigest string) *
if in.Revision != 7 || in.Branch != "main" {
return nil, errors.New("asked another revision")
}
page := pages[in.After]
if s.noDefinitions && page.Revision != nil {
rev := *page.Revision
rev.Definitions = nil
page.Revision = &rev
}
return json.Marshal(page)
return json.Marshal(pages[in.After])
case seatDefinitionsKey:
var sin meshcatalog.SeatDefinitionsInput
_ = json.Unmarshal(raw, &sin)
if sin.Seat == "" {
if s.listFault != nil {
return nil, s.listFault
}
return json.Marshal(meshcatalog.SeatDefinitionsAnswer{Seats: s.trunkSeats})
}
return json.Marshal(meshcatalog.SeatDefinitionsAnswer{Found: true, Entry: &meshcatalog.SeatDefinitionsAnswerEntry{Seat: "s1", Digest: seatDigest, Definition: map[string]any{"seat": "s1"}}})
case entryVerbKey:
var in meshcatalog.EntryInput
@@ -102,12 +76,13 @@ func newRunStand(t *testing.T, revisionEngine, trunkEngine, seatDigest string) *
return json.RawMessage(`{"candidate":"main.7","node":"` + node + `","verdict":"composed","input":{"node":"` + node + `"},"modules":[],"own":{},"took":1,"v":1}`), nil
}
running := map[string]string{"ace": "src2:e1", "g14": "src2:e2"}
away := func(machine string) bool { return machine == "shanks" }
away := &awayNodes{}
away.set("shanks", true)
keeper := testKeeper(&objectsMem{objs: map[string][]byte{}}, time.Now)
s.runs = newCompositionRuns(newRunsMem(), defaultRunSettings, time.Now, t.Logf)
s.r = &compositionRun{runs: s.runs,
ends := &runEnds{}
s.r = &compositionRun{
inputs: &compositionInputs{machines: func(string, string) []string { return []string{"ace", "g14", "shanks"} }, callFile: engines, bound: time.Second},
keeper: keeper, reads: &compositionReads{}, call: call, away: away, now: time.Now, logf: t.Logf,
keeper: keeper, call: call, ends: ends, away: away, now: time.Now, logf: t.Logf,
trunk: func() (mcc.Half, bool) {
return mcc.Half{Entries: map[string]meshcatalog.CatalogEntry{"engine": engine(trunkEngine, commitE1)}}, true
},
@@ -129,27 +104,11 @@ func newRunStand(t *testing.T, revisionEngine, trunkEngine, seatDigest string) *
s.asks[node], s.requests[node] = a, req
s.mu.Unlock()
m := composer.Machine{Composes: true}
override, set := composer.Machine{}, false
if s.verdict != nil {
override, set = s.verdict(node)
}
if set {
m = override
} else if node == s.unjudged {
m = composer.Machine{NotJudged: "the composer was handed no seat list"}
} else if node == "g14" {
if node == "g14" {
m = composer.Machine{Reason: "two modules own one path", Problems: []string{"/etc/x is m's and bad's"}}
}
v, _ := json.Marshal(composer.Verdict{V: composer.Version, Cases: []composer.CaseVerdict{{Name: changeCase, Machines: map[string]composer.Machine{node: m}}}})
s.mu.Lock()
s.puts++
s.mu.Unlock()
go func() {
if err := s.runs.heardEnd(buildstep.StepEnded{Ask: a.ID, Asker: runAsker, Node: "builder1", Outcome: buildstep.OutcomePassed, Output: string(v)},
func(e buildstep.StepEnded) ([]byte, error) { return []byte(e.Output), nil }); err != nil {
t.Error(err)
}
}()
go ends.heard(buildstep.StepEnded{Ask: a.ID, Asker: runAsker, Node: "builder1", Outcome: buildstep.OutcomePassed, Output: string(v)})
return nil
},
}
@@ -165,8 +124,7 @@ func TestTheCompositionStepRunsOneAskPerMachine(t *testing.T) {
s := newRunStand(t, "src2:e1", "src2:e1", "d1")
out := s.r.run(context.Background(), "ask-step", step)
if out.Outcome != meshwants.StepEndedOutcomeFailed || !strings.Contains(out.Said, "g14 does not compose") || !strings.Contains(out.Said, "/etc/x") ||
strings.Contains(out.Said, "ace") || len(out.Warnings) != 2 || !strings.Contains(out.Warnings[0], "shanks is away") ||
out.Warnings[1] != "g14: the composer names no culprits" {
strings.Contains(out.Said, "ace") || len(out.Warnings) != 1 || !strings.Contains(out.Warnings[0], "shanks is away") {
t.Fatalf("the step came to %+v", out)
}
if len(s.asks) != 2 {
@@ -180,12 +138,12 @@ func TestTheCompositionStepRunsOneAskPerMachine(t *testing.T) {
if s.asks["g14"].Run.Program.Commit != commitE2 {
t.Errorf("g14's composer is built at %s, want the build its node-engine runs", s.asks["g14"].Run.Program.Commit)
}
if req.Depth != composer.DepthCompose || req.BlameTries != 20 || len(req.Machines) != 1 || req.Machines[0] != "ace" || len(req.Cases) != 1 || req.Cases[0].Name != changeCase ||
if req.Depth != composer.DepthCompose || len(req.Machines) != 1 || req.Machines[0] != "ace" || len(req.Cases) != 1 || req.Cases[0].Name != changeCase ||
string(req.Input) != `{"node":"ace"}` || len(req.Seats) != 1 || string(req.Seats[0]) != `{"seat":"s1"}` {
t.Errorf("the request of ace %+v", req)
}
c := req.Cases[0]
if string(c.Manifests["m"].Raw) != `{"module":"m"}` || len(c.Manifests) != 4 || len(c.None) != 1 || c.None["lidarr"] != "builders disagree: node" || len(c.Verdicts) != 1 || !strings.Contains(string(c.Verdicts["bad"]), `"refused"`) ||
if string(c.Manifests["m"].Raw) != `{"module":"m"}` || len(c.Manifests) != 3 || len(c.Verdicts) != 1 || !strings.Contains(string(c.Verdicts["bad"]), `"refused"`) ||
!strings.Contains(string(c.Verdicts["bad"]), "no seat x") {
t.Errorf("the case %+v %s", c.Manifests, c.Verdicts)
}
@@ -221,45 +179,6 @@ func TestTheCompositionStepRunsOneAskPerMachine(t *testing.T) {
t.Errorf("a seat digest not held came to %+v, %d asks", out, len(other.asks))
}
// A revision naming no definition (A-234's stand-in): the composer is handed the trunk seat list's definitions, read through
// seat-definitions (A-467); a list that cannot be read, or names no seat, ends the step could not run, naming it, with no ask put.
bare := newRunStand(t, "src2:e1", "src2:e1", "d1")
bare.noDefinitions, bare.trunkSeats = true, []meshcatalog.SeatDefinitionsAnswerSeatsItem{{Seat: "s1", Digest: "d1"}}
if out := bare.r.run(context.Background(), "ask-step", step); out.Outcome != meshwants.StepEndedOutcomeFailed ||
len(bare.requests["ace"].Seats) != 1 || string(bare.requests["ace"].Seats[0]) != `{"seat":"s1"}` {
t.Errorf("a revision with no definitions came to %+v, seats %v", out, bare.requests["ace"].Seats)
}
for name, fault := range map[string]error{"unreadable": errors.New("no responder"), "empty": nil} {
s := newRunStand(t, "src2:e1", "src2:e1", "d1")
s.noDefinitions, s.listFault = true, fault
if out := s.r.run(context.Background(), "ask-step", step); out.Outcome != meshwants.StepEndedOutcomeCouldNotRun ||
!strings.Contains(out.Said, "seat list is not read") || len(s.asks) != 0 {
t.Errorf("the %s seat list came to %+v, %d asks", name, out, len(s.asks))
}
}
// A machine the composer did not judge is could not run, naming why: a step that judged nothing never passes (A-467).
nj := newRunStand(t, "src2:e1", "src2:e1", "d1")
nj.unjudged = "g14"
if out := nj.r.run(context.Background(), "ask-step", step); out.Outcome != meshwants.StepEndedOutcomeCouldNotRun ||
!strings.Contains(out.Said, "g14 was not judged: the composer was handed no seat list") {
t.Errorf("a machine not judged came to %+v", out)
}
// No machine judged: every machine away, or no machine, never passes; each is named with why (A-467a).
for name, machines := range map[string][]string{"every machine away": {"shanks"}, "no machine": {}} { // none placed: the inputs say so already
want := "no machine was judged"
if len(machines) == 0 {
want = "has no machine to compose"
}
none := newRunStand(t, "src2:e1", "src2:e1", "d1")
none.r.inputs.machines = func(string, string) []string { return machines }
if out := none.r.run(context.Background(), "ask-step", step); out.Outcome != meshwants.StepEndedOutcomeCouldNotRun ||
!strings.Contains(out.Said, want) || (len(machines) > 0 && !strings.Contains(out.Said, "shanks is away")) || len(none.asks) != 0 {
t.Errorf("%s came to %+v, %d asks", name, out, len(none.asks))
}
}
// At the inputs state's cap the ask is refused, naming the store (§354).
full := newRunStand(t, "src2:e1", "src2:e1", "d1")
full.r.keeper.used = func() (int64, error) { return inputsCap, nil }
@@ -268,308 +187,3 @@ func TestTheCompositionStepRunsOneAskPerMachine(t *testing.T) {
t.Errorf("an ask at the cap came to %+v, %d asks", out, len(full.asks))
}
}
// Design 47 G §354, A-252: the entries of one catalog revision are read once for every step of it, and a read that times out is
// made again with a growing wait for a bounded time, said in the journal, before the step could not run.
func TestTheCompositionStepReadsTheRevisionOnceAndAgainOnATimeout(t *testing.T) {
step := meshwants.StepRun{Module: "m", Branch: "main", Revision: 7, Step: meshwants.StepRunStepComposition}
for _, tc := range []struct {
name string
faults int // consecutive faults of the `revision` read before it answers
fault error // the fault
steps int
want string // the outcome of the last step
revisionCalls int
waits int
}{
{"two steps of one revision read it once", 0, nil, 2, meshwants.StepEndedOutcomeFailed, 2, 0},
{"a timeout is read again", 3, errors.New("context deadline exceeded"), 1, meshwants.StepEndedOutcomeFailed, 5, 3},
{"a fault that stays ends could not run", 1000, errors.New("nats: timeout"), 1, meshwants.StepEndedOutcomeCouldNotRun, -1, -1},
{"a fault of the answer is not read again", 1, errors.New("no such revision"), 1, meshwants.StepEndedOutcomeCouldNotRun, 1, 0},
} {
t.Run(tc.name, func(t *testing.T) {
s := newRunStand(t, "src2:e1", "src2:e1", "d1")
inner := s.r.call
var mu sync.Mutex
calls, faults := 0, tc.faults
s.r.call = func(key string, body any) (json.RawMessage, error) {
if key == mcc.RevisionVerb {
mu.Lock()
calls++
f := faults > 0
faults--
mu.Unlock()
if f {
return nil, tc.fault
}
}
return inner(key, body)
}
clock := time.Unix(1000, 0)
var waits []time.Duration
s.r.now = func() time.Time { mu.Lock(); defer mu.Unlock(); return clock }
s.r.sleep = func(_ context.Context, d time.Duration) error {
mu.Lock()
defer mu.Unlock()
waits = append(waits, d)
clock = clock.Add(d)
return nil
}
var wg sync.WaitGroup
outs := make([]stepOutcome, tc.steps)
for i := range outs {
wg.Add(1)
go func() { defer wg.Done(); outs[i] = s.r.run(context.Background(), "ask-step", step) }()
}
wg.Wait()
for _, o := range outs {
if o.Outcome != tc.want {
t.Fatalf("a step came to %+v, want %s", o, tc.want)
}
}
if tc.waits == -1 { // never answers: bounded, growing waits
if len(waits) < 3 || waits[0] != readsFirstWait || waits[1] != 2*readsFirstWait || waits[len(waits)-1] != readsMaxWait {
t.Errorf("the waits %v do not grow to %s", waits, readsMaxWait)
}
var total time.Duration
for _, w := range waits {
total += w
}
if total > readsBound {
t.Errorf("read again for %s, bound %s", total, readsBound)
}
if !strings.Contains(outs[0].Said, "catalog revision is not read") || !strings.Contains(outs[0].Said, "timeout") {
t.Errorf("the step said %q", outs[0].Said)
}
return
}
if calls != tc.revisionCalls || len(waits) != tc.waits {
t.Errorf("`revision` asked %d times with %d waits, want %d and %d", calls, len(waits), tc.revisionCalls, tc.waits)
}
})
}
}
// Design 47 G §718, §719, §720: three composition steps of one revision on two machines start two runs, one request and one run
// ask per machine, and each step reads both verdicts; the run records its program with the node-engine fingerprint it was chosen
// from and the digest of its kept request, which names no module of the asking pipeline; a step of another candidate starts its
// own runs, and a run that could not run is never read again.
func TestTheStepsOfOneCandidateReadOneRunPerMachine(t *testing.T) {
s := newRunStand(t, "src2:e1", "src2:e1", "d1")
s.r.inputs.machines = func(string, string) []string { return []string{"ace", "g14"} }
var wg sync.WaitGroup
outs := make([]stepOutcome, 3)
for i, m := range []string{"m", "m2", "m3"} {
wg.Add(1)
go func() {
defer wg.Done()
outs[i] = s.r.run(context.Background(), "ask-"+m, meshwants.StepRun{Pipeline: "main.7." + m, Module: m, Branch: "main", Revision: 7, Step: meshwants.StepRunStepComposition})
}()
}
wg.Wait()
for i, o := range outs {
if len(o.Shared) != 2 || o.Shared[0].Machine != "ace" || o.Shared[0].Run != runKeyOf("main.7", "ace") || o.Shared[0].Ask == "" ||
o.Shared[1].Machine != "g14" || o.Shared[1].Ask == o.Shared[0].Ask { // §723, §729: per machine the run read and its ask
t.Errorf("step %d names the runs %+v", i, o.Shared)
}
if o.Outcome != meshwants.StepEndedOutcomeFailed || !strings.Contains(o.Said, "g14 does not compose") {
t.Errorf("step %d came to %+v, want both machines read", i, o)
}
}
if s.puts != 2 {
t.Fatalf("%d run asks put, want one per machine", s.puts)
}
ace, _, _, _ := s.runs.read(runKeyOf("main.7", "ace"))
g14, _, _, _ := s.runs.read(runKeyOf("main.7", "g14"))
if len(ace.Readers) != 3 || len(g14.Readers) != 3 || ace.Program == nil || ace.Program.Fingerprint != "src2:e1" || ace.Program.Commit != commitE1 ||
g14.Program == nil || g14.Program.Fingerprint != "src2:e2" || ace.Input == "" || ace.Input != digestOf(mustJSON(t, s.requests["ace"])) ||
ace.Outcome != meshwants.CompositionRunOutcomePassed || g14.Outcome != meshwants.CompositionRunOutcomePassed || g14.Verdict["composes"] != false {
t.Errorf("the runs: ace %+v, g14 %+v", ace, g14)
}
if req := s.requests["ace"]; strings.Contains(string(mustJSON(t, req)), "main.7.m") || string(req.Input) != `{"node":"ace"}` {
t.Errorf("the request names the asking pipeline or not the node's input: %s", mustJSON(t, req))
}
// Another candidate (a run on request) starts its own runs.
if o := s.r.run(context.Background(), "ask-r", meshwants.StepRun{Pipeline: "run-r1.m", Module: "m", Branch: "main", Revision: 7, Run: "r1", Step: meshwants.StepRunStepComposition}); o.Outcome != meshwants.StepEndedOutcomeFailed || s.puts != 4 {
t.Errorf("a step of another candidate came to %+v with %d asks, want its own two runs", o, s.puts)
}
// A run whose ask could not be put could not run; the next step of the candidate starts it again.
f := newRunStand(t, "src2:e1", "src2:e1", "d1")
f.r.inputs.machines = func(string, string) []string { return []string{"ace"} }
put, refused := f.r.put, 0
f.r.put = func(ctx context.Context, a buildstep.StepAsk) error {
if refused == 0 {
refused++
return errors.New("no stream")
}
return put(ctx, a)
}
first := f.r.run(context.Background(), "ask-1", meshwants.StepRun{Pipeline: "main.7.m", Module: "m", Branch: "main", Revision: 7, Step: meshwants.StepRunStepComposition})
second := f.r.run(context.Background(), "ask-2", meshwants.StepRun{Pipeline: "main.7.m2", Module: "m2", Branch: "main", Revision: 7, Step: meshwants.StepRunStepComposition})
if first.Outcome != meshwants.StepEndedOutcomeCouldNotRun || !strings.Contains(first.Said, "is not put") || second.Outcome != meshwants.StepEndedOutcomePassed || f.puts != 1 {
t.Errorf("after a run that could not run: %+v then %+v, %d asks", first, second, f.puts)
}
}
func mustJSON(t *testing.T, v any) []byte {
t.Helper()
raw, err := json.Marshal(v)
if err != nil {
t.Fatal(err)
}
return raw
}
// Design 47 G §721, §722, A-470b: a step judges each machine from its verdict's `blamed`: a verdict blaming the step's module fails
// it and passes another module's step with a warning naming the module; a machine that does not compose and blames no module fails
// every step; a verdict with no `blamed` is judged as before and says "the composer names no culprits"; a module the search did not
// try is could not run for its own step; a run that could not run is could not run.
func TestAMachineIsJudgedByTheModulesItBlames(t *testing.T) {
at := time.Now()
runOf := func(verdict string) meshwants.CompositionRun {
var v map[string]any
if err := json.Unmarshal([]byte(verdict), &v); err != nil {
t.Fatal(err)
}
return meshwants.CompositionRun{Candidate: "main.7", Machine: "ace", Ended: &at, Outcome: meshwants.CompositionRunOutcomePassed, Verdict: v}
}
blamesA := `{"composes":false,"reason":"two modules own one path","blamed":[{"module":"a","problem":"/etc/x is a's and c's"}]}`
for _, tc := range []struct {
name, module string
run meshwants.CompositionRun
failed, couldNo string
warnings []string
}{
{"blamed module fails", "a", runOf(blamesA), "ace holds a: /etc/x is a's and c's", "", nil},
{"another module passes with a warning", "b", runOf(blamesA), "", "", []string{"ace holds a: /etc/x is a's and c's"}},
{"blames no module fails both", "b", runOf(`{"composes":false,"reason":"the node's own composition","blamed":[]}`), "ace does not compose and blames no module: the node's own composition", "", nil},
{"an old composer", "b", runOf(`{"composes":false,"reason":"x"}`), "ace does not compose: x", "", []string{"ace: the composer names no culprits"}},
{"composes", "b", runOf(`{"composes":true,"blamed":[]}`), "", "", nil},
{"own module not tried", "a", runOf(`{"composes":false,"blamed":[{"module":"a","problem":"not tried: the search reached its bound"}]}`), "",
"ace holds a untried: the composer's search for culprits reached its bound of blame_tries (A-470b)", nil},
{"another not tried", "b", runOf(`{"composes":false,"blamed":[{"module":"a","problem":"not tried: the search reached its bound"}]}`), "", "",
[]string{"ace holds a: not tried: the search reached its bound"}},
{"not judged", "a", runOf(`{"composes":false,"not_judged":"no seat list","blamed":[]}`), "", "ace was not judged: no seat list", nil},
{"could not run", "a", meshwants.CompositionRun{Machine: "ace", Ended: &at, Outcome: meshwants.CompositionRunOutcomeCouldNotRun, Said: "ace: gone"}, "", "ace: gone", nil},
} {
t.Run(tc.name, func(t *testing.T) {
e := judgeRun(tc.module, tc.run)
if e.failed != tc.failed || e.couldNot != tc.couldNo || !slices.Equal(e.warnings, tc.warnings) {
t.Errorf("judged %+v, want failed %q, could not run %q, warnings %q", e, tc.failed, tc.couldNo, tc.warnings)
}
})
}
}
// Design 47 G §727, A-468g, A-470d: a rerun of a composition step starts a new run only on the machine it did not pass, replacing
// that key and carrying every former reader, the one that did not pass marked rerun with the rerun's caller; at the new run's end
// that reader says its end again, marked rerun, naming the new run's ask; a reader that passed does not move; the machine the
// step passed keeps its verdict.
func TestARerunMovesEveryReaderThatDidNotPass(t *testing.T) {
s := newRunStand(t, "src2:e1", "src2:e1", "d1")
s.r.inputs.machines = func(string, string) []string { return []string{"ace", "g14"} }
var mu sync.Mutex
fixed := false
s.verdict = func(node string) (composer.Machine, bool) {
mu.Lock()
defer mu.Unlock()
m := composer.Machine{Composes: true}
m.Blamed = []struct {
Module string `json:"module"`
Problem string `json:"problem"`
}{}
if node == "g14" && !fixed {
m.Composes, m.Reason = false, "two modules own one path"
m.Blamed = append(m.Blamed, struct {
Module string `json:"module"`
Problem string `json:"problem"`
}{"a", "/etc/x"}, struct {
Module string `json:"module"`
Problem string `json:"problem"`
}{"d", "/etc/x"})
}
return m, true
}
ends := make(chan meshwants.StepEnded, 16)
msgIDs := map[string]string{} // the message id of each step's latest end
steps := &compositionSteps{node: "n1", now: time.Now, logf: t.Logf, run: s.r.run, resume: s.r.resume, ended: s.runs.stepEnded,
say: func(event string, body any, msgID string) error {
if e, ok := body.(meshwants.StepEnded); ok {
mu.Lock()
msgIDs[e.ID] = msgID
mu.Unlock()
ends <- e
}
return nil
}}
move := steps.move(context.Background())
s.runs.moved.Store(&move)
step := func(m string, rerun bool) []byte {
raw, _ := json.Marshal(meshwants.StepRun{Pipeline: "main.7." + m, Module: m, Branch: "main", Revision: 7, Step: meshwants.StepRunStepComposition, Rerun: rerun, By: map[bool]string{true: "op@n1"}[rerun]})
return raw
}
got := map[string]meshwants.StepEnded{}
for _, m := range []string{"a", "b", "d"} {
steps.take(context.Background(), "ask-"+m, step(m, false))
e := <-ends
got[e.Module] = e
}
if got["a"].Outcome != meshwants.StepEndedOutcomeFailed || got["d"].Outcome != meshwants.StepEndedOutcomeFailed || got["b"].Outcome != meshwants.StepEndedOutcomePassed {
t.Fatalf("the first ends: %+v", got)
}
former := got["d"].Shared[1].Ask
mu.Lock()
fixed = true
mu.Unlock()
steps.take(context.Background(), "ask-a2", step("a", true))
moved := map[string]meshwants.StepEnded{}
for range 2 {
select {
case e := <-ends:
moved[e.ID] = e
case <-time.After(5 * time.Second):
t.Fatalf("the ends after the rerun: %+v", moved)
}
}
a, d := moved["ask-a2"], moved["ask-d"]
if a.Outcome != meshwants.StepEndedOutcomePassed || d.Outcome != meshwants.StepEndedOutcomePassed || !d.Rerun || d.By != "op@n1" ||
len(d.Shared) != 2 || d.Shared[1].Machine != "g14" || d.Shared[1].Ask == former || d.Shared[1].Ask != a.Shared[1].Ask {
t.Fatalf("after the rerun: a %+v, d %+v", a, d)
}
mu.Lock()
// A-470q: the first end keeps the ask's id; the end said again names the moved run's ask, so the bus does not drop it.
if msgIDs["ask-a2"] != "ask-a2" || msgIDs["ask-d"] != "ask-d."+a.Shared[1].Ask || msgIDs["ask-b"] != "ask-b" {
t.Errorf("the message ids of the ends: %v", msgIDs)
}
mu.Unlock()
if s.puts != 3 {
t.Errorf("%d run asks, want one more for the machine a did not pass", s.puts)
}
select {
case e := <-ends:
t.Errorf("a reader that passed moved: %+v", e)
case <-time.After(50 * time.Millisecond):
}
run, _, _, _ := s.runs.read(runKeyOf("main.7", "g14"))
marks := map[string]bool{}
for _, c := range run.Readers {
marks[c.ID] = c.Rerun
}
if len(run.Readers) != 3 || !marks["ask-d"] || marks["ask-b"] || !marks["ask-a2"] {
t.Errorf("the new run's readers %+v", run.Readers)
}
// A-470m: each reader's `ended` is written once its step said its end, and ask-b's carried into the new run.
deadline := time.After(5 * time.Second)
for {
wake := s.runs.wake()
run, _, _, _ = s.runs.read(runKeyOf("main.7", "g14"))
if !slices.ContainsFunc(run.Readers, func(c meshwants.CompositionReader) bool { return c.Ended == nil }) {
break
}
select {
case <-wake:
case <-deadline:
t.Fatalf("a reader of the new run names no end: %+v", run.Readers)
}
}
}
@@ -1,896 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"io/fs"
"os"
"slices"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/nats-io/nats.go/jetstream"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-sdk/go/keys"
"mesh-delivery-assigner/internal/buckets"
)
// The composition runs (design 47 G §718, §719; A-468, A-469d, A-469w, A-469x): the assigner composes each machine once per
// candidate, whatever number of composition steps name it. Each run is one key of its old state `composition-runs`,
// `<candidate key>.<machine>`, each part escaped as a key's name; the step that starts a run writes the key create-only,
// so two steps of one candidate starting at once start one run and the second reads it. A reader is the composition step's
// StepRun with its ask's id (A-468), added and removed by compare-and-set. A key is deleted once composition-runs-kept has
// passed since its `ended`, judged at each new run over at most 20 of the oldest ended keys, as the state inputs is
// (A-465b, A-468b); at the cap (composition-runs-cap-mib, the manifest's max-mib its ceiling, A-469d) no run is started.
const (
// runsSwept is how many of the oldest ended keys each new run judges.
runsSwept = 20
// runBoundPrefix and runUntakenPrefix open a run's two bound keys, each with its own time to live (§720, A-468b,
// A-470): `bound.<run key>` from the ask's ask-taken, `untaken.<run key>` from the run's start.
runBoundPrefix = "bound."
runUntakenPrefix = "untaken."
)
// runKeyOf is the key of a candidate's composition run of a machine (§719).
func runKeyOf(candidate, machine string) string {
return keysPart(candidate) + "." + keys.EscapeName(machine)
}
// keysPart is a name as a key's part.
func keysPart(name string) string { return keys.EscapeName(name) }
// isRunKey is whether a key of the state is a run's own, not one of its bound keys.
func isRunKey(key string) bool {
return !strings.HasPrefix(key, runBoundPrefix) && !strings.HasPrefix(key, runUntakenPrefix)
}
// runSettings are the settings of the composition runs, each read at the start from the file the manifest declares.
type runSettings struct {
// capBytes is composition-runs-cap-mib in bytes.
capBytes int64
// kept is composition-runs-kept, stepBound composition-step-bound, untaken run-untaken-bound (A-469x).
kept, stepBound, untaken time.Duration
// blameTries is blame-tries, carried in each run's request (§721).
blameTries int
}
// defaultRunSettings are the settings' defaults, as the manifest gives them.
var defaultRunSettings = runSettings{capBytes: 16 << 20, kept: 24 * time.Hour, stepBound: 30 * time.Minute, untaken: 24 * time.Hour, blameTries: 20}
// runSettingsOf reads the settings composition-runs-cap-mib, composition-runs-kept, composition-step-bound,
// run-untaken-bound and blame-tries from the files the environment names (A-469x, as outsideSettings): a missing or empty
// file is its default; a content that is no positive whole number or Go duration is an error naming the file.
func runSettingsOf(getenv func(string) string) (runSettings, error) {
s := defaultRunSettings
mib, err := settingFile(getenv("MESH_ASSIGNER_COMPOSITION_RUNS_CAP_MIB_FILE"))
if err != nil {
return s, err
}
if mib > 0 {
s.capBytes = int64(mib) << 20
}
tries, err := settingFile(getenv("MESH_ASSIGNER_BLAME_TRIES_FILE"))
if err != nil {
return s, err
}
if tries > 0 {
s.blameTries = tries
}
for env, to := range map[string]*time.Duration{"MESH_ASSIGNER_COMPOSITION_RUNS_KEPT_FILE": &s.kept,
"MESH_ASSIGNER_COMPOSITION_STEP_BOUND_FILE": &s.stepBound, "MESH_ASSIGNER_RUN_UNTAKEN_BOUND_FILE": &s.untaken} {
d, err := durationFile(getenv(env))
if err != nil {
return s, err
}
if d > 0 {
*to = d
}
}
return s, nil
}
// durationFile reads one setting's file holding a Go duration: 0 for no path, no file or an empty one.
func durationFile(path string) (time.Duration, error) {
if path == "" {
return 0, nil
}
raw, err := os.ReadFile(path)
if errors.Is(err, fs.ErrNotExist) {
return 0, nil
}
if err != nil {
return 0, err
}
v := strings.TrimSpace(string(raw))
if v == "" {
return 0, nil
}
d, err := time.ParseDuration(v)
if err != nil || d <= 0 {
return 0, fmt.Errorf("%s holds %q, which is not a positive Go duration such as 30m", path, v)
}
return d, nil
}
// errRunChanged is a compare-and-set of a run's key that met another write.
var errRunChanged = errors.New("the composition run's key changed")
// runsKV is the state composition-runs: the bus bucket, or a test's.
type runsKV interface {
// get is a key's value and revision; nil for a key not there.
get(key string) ([]byte, uint64, error)
// create writes a key that does not stand, update one at its revision: errRunChanged when it does not stand so.
create(key string, value []byte) (uint64, error)
update(key string, value []byte, revision uint64) (uint64, error)
// delAt deletes a key at its revision: errRunChanged when it no longer stands so, nothing for a key not there. No write
// of the state is made from a value read before its latest change.
delAt(key string, revision uint64) error
keys() ([]string, error)
// arm writes a bound key afresh holding the ask of the run it bounds, with its own time to live (§720, A-470n): at the key's
// latest revision, a delete marker counted, read again when another write came between; its expiry is the bus's event. It
// answers the revision written.
arm(key, ask string, ttl time.Duration) (uint64, error)
// used is how many bytes the state holds.
used() (int64, error)
}
// busRuns is runsKV over the bus bucket.
type busRuns struct {
ctx context.Context
js jetstream.JetStream
kv jetstream.KeyValue
}
func (b busRuns) get(key string) ([]byte, uint64, error) {
e, err := b.kv.Get(b.ctx, key)
if errors.Is(err, jetstream.ErrKeyNotFound) {
return nil, 0, nil
}
if err != nil {
return nil, 0, err
}
return e.Value(), e.Revision(), nil
}
func (b busRuns) create(key string, value []byte) (uint64, error) {
rev, err := b.kv.Create(b.ctx, key, value)
if errors.Is(err, jetstream.ErrKeyExists) {
return 0, errRunChanged
}
return rev, err
}
func (b busRuns) update(key string, value []byte, revision uint64) (uint64, error) {
rev, err := b.kv.Update(b.ctx, key, value, revision)
if errors.Is(err, jetstream.ErrKeyExists) || errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
return 0, errRunChanged
}
return rev, err
}
func (b busRuns) delAt(key string, revision uint64) error {
err := b.kv.Delete(b.ctx, key, jetstream.LastRevision(revision))
if errors.Is(err, jetstream.ErrKeyExists) || errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
return errRunChanged
}
return err
}
func (b busRuns) keys() ([]string, error) {
l, err := b.kv.ListKeys(b.ctx)
if err != nil {
return nil, err
}
var out []string
for k := range l.Keys() {
out = append(out, k)
}
return out, nil
}
// boundValue is a bound key's value: the key and the ask of the run it bounds (A-470n).
type boundValue struct {
Bound string `json:"bound"`
Ask string `json:"ask"`
}
// arm publishes the bound key holding ask with ttl as its own time to live, expecting the key's latest message (a delete
// marker counted), so a write between is read again and never overwritten unseen (A-470n); the earlier message, replaced,
// takes its time to live with it, so the expiry is heard only ttl after the last write.
func (b busRuns) arm(key, ask string, ttl time.Duration) (uint64, error) {
if b.js == nil {
return 0, errors.New("no bus to arm a bound on")
}
raw, err := json.Marshal(boundValue{Bound: key, Ask: ask})
if err != nil {
return 0, err
}
subject := "$KV." + b.kv.Bucket() + "." + key
for {
st, err := b.js.Stream(b.ctx, "KV_"+b.kv.Bucket())
if err != nil {
return 0, err
}
var last uint64
m, err := st.GetLastMsgForSubject(b.ctx, subject)
switch {
case err == nil:
last = m.Sequence
case !errors.Is(err, jetstream.ErrMsgNotFound):
return 0, err
}
ack, err := b.js.Publish(b.ctx, subject, raw, jetstream.WithMsgTTL(ttl), jetstream.WithExpectLastSequencePerSubject(last))
if err != nil && strings.Contains(err.Error(), "wrong last sequence") {
continue
}
if err != nil {
return 0, err
}
return ack.Sequence, nil
}
}
func (b busRuns) used() (int64, error) {
st, err := b.kv.Status(b.ctx)
if err != nil {
return 0, err
}
return int64(st.Bytes()), nil
}
// compositionRuns keeps the composition runs in the state composition-runs.
type compositionRuns struct {
kv runsKV
set runSettings
now func() time.Time
logf func(string, ...any)
mu sync.Mutex
// changed is closed and replaced at each change of the state heard (heard): a step waiting on a run reads its key again.
changed chan struct{}
// seen are the keys the state's watch handed and has not seen deleted: a listing of the bucket misses a key that is
// being written at that moment, so every lookup takes these beside it (keys).
seen map[string]bool
// say says an event of the seat mesh-wants (a reader's wait, A-469y); drop asks the build seat's drop of a run ask (§119);
// nil says or drops nothing.
say publisher
drop func(ask, why string) error
// moved is told each run that ended, for the readers a rerun carried to say their end again (compositionSteps.move).
// Set once the steps are served, after the hub starts.
moved atomic.Pointer[func(run meshwants.CompositionRun)]
}
func newCompositionRuns(kv runsKV, set runSettings, now func() time.Time, logf func(string, ...any)) *compositionRuns {
return &compositionRuns{kv: kv, set: set, now: now, logf: logf, changed: make(chan struct{})}
}
// heard takes a change of the state from its watch (a module watches its own state, principle 1).
func (r *compositionRuns) heard() {
r.mu.Lock()
close(r.changed)
r.changed = make(chan struct{})
r.mu.Unlock()
}
// saw keeps a key the state's watch handed: standing, or deleted or expired.
func (r *compositionRuns) saw(key string, standing bool) {
r.mu.Lock()
defer r.mu.Unlock()
if r.seen == nil {
r.seen = map[string]bool{}
}
if standing {
r.seen[key] = true
} else {
delete(r.seen, key)
}
}
// keys are the state's keys: the bucket's listing with the keys its watch handed beside it, sorted. A key read gone later
// is skipped by its reader.
func (r *compositionRuns) keys() ([]string, error) {
all, err := r.kv.keys()
if err != nil {
return nil, err
}
r.mu.Lock()
for key := range r.seen {
all = append(all, key)
}
r.mu.Unlock()
slices.Sort(all)
return slices.Compact(all), nil
}
// wake is closed at the next change heard.
func (r *compositionRuns) wake() <-chan struct{} {
r.mu.Lock()
defer r.mu.Unlock()
return r.changed
}
// read is a run's value and revision, or false.
func (r *compositionRuns) read(key string) (meshwants.CompositionRun, uint64, bool, error) {
raw, rev, err := r.kv.get(key)
if err != nil || raw == nil {
return meshwants.CompositionRun{}, 0, false, err
}
var run meshwants.CompositionRun
if err := json.Unmarshal(raw, &run); err != nil {
return meshwants.CompositionRun{}, 0, false, fmt.Errorf("the composition run %s does not read: %w", key, err)
}
return run, rev, true, nil
}
// change writes a run's key by compare-and-set: f changes the value read and says whether to write it; false leaves it.
// It answers the value written or left, and false for a key not there.
func (r *compositionRuns) change(key string, f func(*meshwants.CompositionRun) bool) (meshwants.CompositionRun, bool, error) {
for {
run, rev, ok, err := r.read(key)
if err != nil || !ok {
return run, ok, err
}
if !f(&run) {
return run, true, nil
}
raw, err := json.Marshal(run)
if err != nil {
return run, true, err
}
if _, err := r.kv.update(key, raw, rev); errors.Is(err, errRunChanged) {
continue
} else if err != nil {
return run, true, err
}
r.heard()
return run, true, nil
}
}
// fill is the state's fill against its cap.
func (r *compositionRuns) fill() (meshwants.Fill, error) {
at := r.now().UTC()
used, err := r.kv.used()
return meshwants.Fill{Store: buckets.CompositionRunsState, Bytes: used, Cap: r.set.capBytes, At: &at}, err
}
// joined is a reader's place in a run: the run's key, whether this reader started it (so it composes the machine), and the
// value as it stood.
type joined struct {
key string
started bool
run meshwants.CompositionRun
}
// join finds or starts the candidate's run of the machine for reader (§718, §719, §720): a run that stands and has not
// ended, or that ended other than could not run, is read, the reader added; none, or one that could not run, is started:
// written create-only (or replacing the ended key by compare-and-set), the reader its first, ask its run ask's id. At the
// cap no run is started, an error naming the fill.
func (r *compositionRuns) join(reader meshwants.CompositionReader, candidate, machine, ask string) (joined, error) {
key := runKeyOf(candidate, machine)
for {
run, rev, ok, err := r.read(key)
if err != nil {
return joined{}, err
}
if ok && (run.Ended == nil || run.Outcome != meshwants.CompositionRunOutcomeCouldNotRun) {
if hasReader(run.Readers, reader.ID) {
return joined{key: key, run: run}, nil
}
run.Readers = append(run.Readers, reader)
raw, err := json.Marshal(run)
if err != nil {
return joined{}, err
}
if _, err := r.kv.update(key, raw, rev); errors.Is(err, errRunChanged) {
continue
} else if err != nil {
return joined{}, err
}
r.heard()
if run.Ended == nil && r.untaken(key, run.Ask) {
r.sayWaits(run, []meshwants.CompositionReader{reader}, "")
}
return joined{key: key, run: run}, nil
}
if err := r.room(); err != nil {
return joined{}, err
}
started := r.now().UTC()
fresh := meshwants.CompositionRun{Candidate: candidate, Machine: machine, Ask: ask, Started: &started,
Readers: []meshwants.CompositionReader{reader}}
raw, err := json.Marshal(fresh)
if err != nil {
return joined{}, err
}
if ok {
_, err = r.kv.update(key, raw, rev) // could not run is never read: a new run replaces it (§720)
} else {
_, err = r.kv.create(key, raw)
}
if errors.Is(err, errRunChanged) {
continue
} else if err != nil {
return joined{}, err
}
r.heard()
// An ask waiting in the queue has no clock but run-untaken-bound from its start (A-466s, A-468b, A-470, A-470n).
if _, err := r.armBound(runUntakenPrefix, key, ask, r.set.untaken); err != nil {
r.logf("the untaken bound of the composition run %s is not armed: %v", key, err)
}
r.sayWaits(fresh, fresh.Readers, "")
return joined{key: key, started: true, run: fresh}, nil
}
}
// room sweeps the state and says whether a new run may start: an error naming the fill at the cap (as A-465a).
func (r *compositionRuns) room() error {
if err := r.sweep(); err != nil {
r.logf("composition-runs: the keys past composition-runs-kept are not all deleted: %v", err)
}
f, err := r.fill()
if err != nil {
return fmt.Errorf("the fill of the state %s is not read, so no composition run is started: %w", buckets.CompositionRunsState, err)
}
if f.Bytes >= f.Cap {
return fmt.Errorf("the state %s is at its cap of %d MiB (%d bytes kept), so no composition run is started", buckets.CompositionRunsState,
f.Cap>>20, f.Bytes)
}
return nil
}
// sweep deletes, among the 20 oldest ended runs, those ended more than composition-runs-kept ago, with their bound keys.
func (r *compositionRuns) sweep() error {
all, err := r.keys()
if err != nil {
return err
}
type endedAt struct {
key string
at time.Time
}
var ended []endedAt
for _, key := range all {
if !isRunKey(key) {
continue
}
run, _, ok, err := r.read(key)
if err != nil || !ok || run.Ended == nil {
continue
}
ended = append(ended, endedAt{key, *run.Ended})
}
slices.SortFunc(ended, func(a, b endedAt) int { return a.at.Compare(b.at) })
if len(ended) > runsSwept {
ended = ended[:runsSwept]
}
var errs []error
for _, e := range ended {
if r.now().Sub(e.at) >= r.set.kept {
errs = append(errs, r.deleteRun(e.key, func(run meshwants.CompositionRun) bool {
return run.Ended != nil && r.now().Sub(*run.Ended) >= r.set.kept
}))
}
}
return errors.Join(errs...)
}
// deleteRun deletes a run's key with its bound keys while the value it holds is still one gone judges gone, each delete at the
// revision just read and judged again on a conflict: a run that replaced the key meanwhile, and its bounds, stand.
func (r *compositionRuns) deleteRun(key string, gone func(meshwants.CompositionRun) bool) error {
for {
run, rev, ok, err := r.read(key)
if err != nil || !ok || !gone(run) {
return err
}
if err := r.clearBounds(key, run.Ask, runBoundPrefix, runUntakenPrefix); err != nil {
return err
}
if err := r.kv.delAt(key, rev); errors.Is(err, errRunChanged) {
continue
} else if err != nil {
return err
}
r.heard()
return nil
}
}
// clearBounds deletes the bound keys of the run of ask while key holds that run (or none) and the bound holds that ask: each at
// the revision it was read at, read again on a conflict, so a bound a newer run of the key armed is never deleted.
func (r *compositionRuns) clearBounds(key, ask string, prefixes ...string) error {
var errs []error
for _, prefix := range prefixes {
for {
run, _, ok, err := r.read(key)
if err != nil {
errs = append(errs, err)
break
}
if ok && run.Ask != ask {
break // a newer run holds the key: its bounds are its own
}
held, rev, standing, err := r.boundOf(prefix + key)
if err != nil || !standing {
errs = append(errs, err)
break
}
if held != ask {
break // the bound of another ask (A-470n): not this run's to delete
}
if err := r.kv.delAt(prefix+key, rev); errors.Is(err, errRunChanged) {
continue
} else if err != nil {
errs = append(errs, err)
}
break
}
}
return errors.Join(errs...)
}
// stepEnded writes at as the `ended` of the reader asked under id in each run of its step's candidate naming it, by
// compare-and-set (A-470m): the time its step said its end, carried with it into a rerun's run.
func (r *compositionRuns) stepEnded(id string, step meshwants.StepRun, at time.Time) {
candidate, err := candidateKeyOf(step)
if err != nil {
r.logf("the end of the composition step %s is not written on its runs: %v", id, err)
return
}
keys, err := r.keysOf(candidate, id)
if err != nil {
r.logf("the end of the composition step %s is not written on its runs: %v", id, err)
return
}
at = at.UTC()
for _, key := range keys {
if _, _, err := r.change(key, func(c *meshwants.CompositionRun) bool {
i := slices.IndexFunc(c.Readers, func(c meshwants.CompositionReader) bool { return c.ID == id })
if i < 0 {
return false
}
c.Readers[i].Ended = &at
return true
}); err != nil {
r.logf("the end of the composition step %s is not written on the composition run %s: %v", id, key, err)
}
}
}
// hasReader is whether readers holds the step asked under id.
func hasReader(readers []meshwants.CompositionReader, id string) bool {
return slices.ContainsFunc(readers, func(c meshwants.CompositionReader) bool { return c.ID == id })
}
// readerOf is a composition step as a run's reader: its StepRun with its ask's id (A-468).
func readerOf(id string, s meshwants.StepRun) meshwants.CompositionReader {
return meshwants.CompositionReader{ID: id, Pipeline: s.Pipeline, Step: s.Step, Module: s.Module, Branch: s.Branch, Commit: s.Commit,
Revision: s.Revision, Source: s.Source, Fingerprint: s.Fingerprint, Run: s.Run, Rerun: s.Rerun, By: s.By}
}
// stepRunOf is a reader's StepRun again, for its step-ended.
func stepRunOf(c meshwants.CompositionReader) meshwants.StepRun {
return meshwants.StepRun{Pipeline: c.Pipeline, Step: c.Step, Module: c.Module, Branch: c.Branch, Commit: c.Commit, Revision: c.Revision,
Source: c.Source, Fingerprint: c.Fingerprint, Run: c.Run, Rerun: c.Rerun, By: c.By}
}
// watchRuns hears the changes of the state composition-runs (principle 1): each change wakes the steps waiting on a run, and
// the expiry of a bound key, the bus's marker, is handed to expired (§720). After the values that stand, ready is called.
func watchRuns(ctx context.Context, kv jetstream.KeyValue, r *compositionRuns, expired func(key string), ready func()) error {
w, err := kv.WatchAll(ctx)
if err != nil {
return err
}
go func() {
defer func() { _ = w.Stop() }()
for {
select {
case <-ctx.Done():
return
case e, ok := <-w.Updates():
if !ok {
return
}
if e == nil {
if ready != nil {
ready()
ready = nil
}
continue
}
r.saw(e.Key(), e.Operation() == jetstream.KeyValuePut)
if e.Operation() == jetstream.KeyValuePurge && !isRunKey(e.Key()) && expired != nil {
expired(e.Key())
}
r.heard()
}
}
}()
return nil
}
// end ends the run of ask in its key with an outcome, its words and the machine's verdict, by compare-and-set; it answers
// false when the key holds another run or the run has ended already.
func (r *compositionRuns) end(key, ask, outcome, said string, verdict map[string]any) (bool, error) {
ended := false
_, _, err := r.change(key, func(c *meshwants.CompositionRun) bool {
if c.Ask != ask || c.Ended != nil {
return false
}
at := r.now().UTC()
c.Ended, c.Outcome, c.Said, c.Verdict = &at, outcome, said, verdict
ended = true
return true
})
return ended, err
}
// keyOfAsk is the key of the run whose ask is ask, or none.
func (r *compositionRuns) keyOfAsk(ask string) (string, meshwants.CompositionRun, error) {
all, err := r.keys()
if err != nil {
return "", meshwants.CompositionRun{}, err
}
for _, key := range all {
if !isRunKey(key) {
continue
}
run, _, ok, err := r.read(key)
if err == nil && ok && run.Ask == ask {
return key, run, nil
}
}
return "", meshwants.CompositionRun{}, nil
}
// heardEnd ends the run of the ask a build seat's step-ended names (§719, A-240): a run ask that ended other than passed
// could not run; a passed one's answer, read with read, is the machine's verdict, or why it is none. An end of an ask no run
// names is another asker's, or of a run gone, and changes nothing.
func (r *compositionRuns) heardEnd(e buildstep.StepEnded, read func(buildstep.StepEnded) ([]byte, error)) error {
if e.Asker != runAsker || e.Ask == "" {
return nil
}
key, run, err := r.keyOfAsk(e.Ask)
if err != nil || key == "" {
return err
}
if run.Ended != nil { // ended by its bound or a drop: the end heard later is ignored (§720)
r.logf("the step-ended of the run ask %s is ignored: its composition run %s ended before it (%s)", e.Ask, key, run.Said)
return nil
}
node := run.Machine
outcome, said := meshwants.CompositionRunOutcomeCouldNotRun, ""
var verdict map[string]any
if e.Outcome != buildstep.OutcomePassed {
said = fmt.Sprintf("%s: its run ask %s ended %s on %s: %s", node, e.Ask, e.Outcome, e.Node, e.Said)
} else if answer, err := read(e); err != nil { // the program's last line of standard output (A-254)
said = node + ": " + err.Error()
} else if verdict, said = runVerdict(node, answer); said == "" {
outcome = meshwants.CompositionRunOutcomePassed
}
_, err = r.endRun(key, e.Ask, outcome, said, verdict)
return err
}
// await waits until each run of keys has ended, through the state's changes heard, or ctx ends: it answers each run as it
// stands then; a key gone could not run.
func (r *compositionRuns) await(ctx context.Context, keys []string) map[string]meshwants.CompositionRun {
out := map[string]meshwants.CompositionRun{}
for {
wake := r.wake()
open := false
for _, key := range keys {
run, _, ok, err := r.read(key)
switch {
case err != nil:
open = true
continue
case !ok:
at := r.now().UTC()
candidate, machine, _ := strings.Cut(key, ".")
run = meshwants.CompositionRun{Candidate: unescape(candidate), Machine: unescape(machine), Ended: &at,
Outcome: meshwants.CompositionRunOutcomeCouldNotRun, Said: unescape(machine) + ": its composition run " + key + " is gone"}
case run.Ended == nil:
open = true
}
out[key] = run
}
if !open {
return out
}
select {
case <-ctx.Done():
return out
case <-wake:
}
}
}
// unescape is a key's part as a name again; a part that does not read stays as it is.
func unescape(part string) string {
if name, err := keys.UnescapeName(part); err == nil {
return name
}
return part
}
// sharedOf names per machine the runs of keys and their asks (§729): from runs where given, else as each key stands.
func (r *compositionRuns) sharedOf(keys []string, runs map[string]meshwants.CompositionRun) []meshwants.Shared {
out := []meshwants.Shared{}
for _, key := range keys {
run, ok := runs[key]
if !ok {
if run, _, ok, _ = r.read(key); !ok {
continue
}
}
out = append(out, meshwants.Shared{Machine: run.Machine, Run: key, Ask: run.Ask})
}
return out
}
// leave removes the composition steps of a superseded or cancelled pipeline from every run's readers (§723, A-469n), and answers
// their asks' ids, each told to stop before a run is ended; a running run left with no reader has its ask dropped and ends could not run, "no reader is left" (A-469z),
// so a later step of the candidate starts a new run. A run other readers read goes on for them.
func (r *compositionRuns) leave(pipeline string, stop func(ids []string)) ([]string, error) {
all, err := r.keys()
if err != nil {
return nil, err
}
var ids []string
var errs []error
for _, key := range all {
if !isRunKey(key) {
continue
}
var left []string
var leaving []meshwants.CompositionReader
run, ok, err := r.change(key, func(c *meshwants.CompositionRun) bool {
left, leaving = nil, nil
keep := c.Readers[:0:0]
for _, reader := range c.Readers {
if reader.Pipeline == pipeline && reader.Step == meshwants.CompositionReaderStepComposition {
left = append(left, reader.ID)
leaving = append(leaving, reader)
continue
}
keep = append(keep, reader)
}
c.Readers = keep
return len(left) > 0
})
if err != nil {
errs = append(errs, err)
continue
}
if !ok || len(left) == 0 {
continue
}
ids = append(ids, left...)
if run.Ended == nil && r.untaken(key, run.Ask) { // a reader leaving a run not yet taken: its wait ends at once (A-470u)
r.sayWaits(run, leaving, "superseded")
}
if stop != nil {
stop(left) // the steps that left say no end, the run's included
}
if run.Ended != nil || len(run.Readers) > 0 {
continue
}
why := "no reader is left: the pipeline " + pipeline + " is superseded or cancelled"
if r.drop != nil {
if err := r.drop(run.Ask, why); err != nil {
r.logf("the drop of the run ask %s of the composition run %s is not answered: %v", run.Ask, key, err)
}
}
if _, err := r.endRun(key, run.Ask, meshwants.CompositionRunOutcomeCouldNotRun, run.Machine+": "+why, nil); err != nil {
errs = append(errs, err)
}
}
return ids, errors.Join(errs...)
}
// passedOn is whether the step of module passed on the machine of an ended run (§722): neither failed nor could not run there.
func passedOn(module string, run meshwants.CompositionRun) bool {
e := judgeRun(module, run)
return e.failed == "" && e.couldNot == ""
}
// formerOf is the place in readers of a step's former ask: the reader of the same pipeline's composition step, or -1.
func formerOf(readers []meshwants.CompositionReader, pipeline string) int {
return slices.IndexFunc(readers, func(c meshwants.CompositionReader) bool {
return c.Pipeline == pipeline && c.Step == meshwants.CompositionReaderStepComposition
})
}
// rejoin is join for a step asked again (§727, A-468g, A-470d): on a machine whose ended run it passed it keeps that verdict,
// reading the run under its new ask; a running run it joins; an ended run it did not read and that could run it reads, as one
// started after its last end; otherwise it starts a new run, replacing the ended key by compare-and-set and carrying every former
// reader into the new value: each that did not pass on the former run marked `rerun` with by the rerun's caller, each that passed
// not (A-470d).
func (r *compositionRuns) rejoin(reader meshwants.CompositionReader, candidate, machine, ask string) (joined, error) {
key := runKeyOf(candidate, machine)
for {
run, rev, ok, err := r.read(key)
if err != nil {
return joined{}, err
}
if !ok {
return r.join(reader, candidate, machine, ask)
}
at := formerOf(run.Readers, reader.Pipeline)
ended := run.Ended != nil
keep := ended && at >= 0 && passedOn(reader.Module, run)
read := !ended || (at < 0 && run.Outcome != meshwants.CompositionRunOutcomeCouldNotRun)
if keep || read {
if at >= 0 && run.Readers[at].ID == reader.ID {
return joined{key: key, run: run}, nil
}
var former []meshwants.CompositionReader
if at >= 0 {
former = []meshwants.CompositionReader{run.Readers[at]}
run.Readers[at] = reader
} else {
run.Readers = append(run.Readers, reader)
}
if err := r.write(key, run, rev); errors.Is(err, errRunChanged) {
continue
} else if err != nil {
return joined{}, err
}
if run.Ended == nil && r.untaken(key, run.Ask) { // as join: a reader of a run not yet taken waits for a build node (A-469y)
r.sayWaits(run, former, "superseded") // the place it takes: that wait ends at once (A-470u)
r.sayWaits(run, []meshwants.CompositionReader{reader}, "")
}
return joined{key: key, run: run}, nil
}
if err := r.room(); err != nil {
return joined{}, err
}
started := r.now().UTC()
fresh := meshwants.CompositionRun{Candidate: candidate, Machine: machine, Ask: ask, Started: &started}
for i, c := range run.Readers {
if i == at {
continue
}
c.Rerun, c.By = !passedOn(c.Module, run), ""
if c.Rerun {
c.By = reader.By
}
fresh.Readers = append(fresh.Readers, c)
}
fresh.Readers = append(fresh.Readers, reader)
if err := r.clearBounds(key, run.Ask, runBoundPrefix, runUntakenPrefix); err != nil { // the former run's bounds go with it
r.logf("the bound keys of the former composition run %s are not deleted: %v", key, err)
}
if err := r.write(key, fresh, rev); errors.Is(err, errRunChanged) {
continue
} else if err != nil {
return joined{}, err
}
if _, err := r.armBound(runUntakenPrefix, key, ask, r.set.untaken); err != nil {
r.logf("the untaken bound of the composition run %s is not armed: %v", key, err)
}
r.sayWaits(fresh, fresh.Readers, "")
return joined{key: key, started: true, run: fresh}, nil
}
}
// write writes a run's value at its revision.
func (r *compositionRuns) write(key string, run meshwants.CompositionRun, rev uint64) error {
raw, err := json.Marshal(run)
if err != nil {
return err
}
if _, err := r.kv.update(key, raw, rev); err != nil {
return err
}
r.heard()
return nil
}
@@ -1,404 +0,0 @@
//go:build integration
package main
import (
"encoding/json"
"fmt"
"strings"
"sync"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient/enginereads"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
mpr "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer"
"mesh-delivery-assigner/internal/buckets"
)
// Test 108 (design 47 G §733; §718 to §723, §727, §728; A-468 to A-470f): the assigner started whole against the bus composes
// each machine once per candidate. Two or three composition steps of one candidate on two machines make one key, one request and
// one run ask per machine, and each step reads both verdicts; a machine blaming a and b fails their steps and passes c's with a
// warning; a verdict with no `blamed` is judged as before, "the composer names no culprits", and a machine blaming none fails
// every step; a step of another candidate starts its own runs. A taken ask that keeps saying progress is not ended by the bound,
// one silent past it ends could not run with its ask dropped, a step-ended after that is ignored and journaled, and an ask never
// taken ends at run-untaken-bound; a bound holds its run's ask (A-470n). A rerun of a starts one run on the machine it did not
// pass and moves b; a superseded step leaves
// a run running for its other reader; an assigner restarted during a run takes it up by its ask; compositions-drop ends the
// readers and deletes the key; at the cap a step ends could not run, naming the fill.
func TestOneCompositionRunPerCandidateAndMachine(t *testing.T) {
b := startBed(t, "")
b.settings = map[string]string{"MESH_ASSIGNER_COMPOSITION_STEP_BOUND_FILE": "2s", "MESH_ASSIGNER_RUN_UNTAKEN_BOUND_FILE": "15s", // past a restart
"MESH_ASSIGNER_COMPOSITION_RUNS_CAP_MIB_FILE": "1"}
nc, err := nats.Connect(b.srv.ClientURL())
if err != nil {
t.Fatal(err)
}
t.Cleanup(nc.Close)
const commit = "1111111111111111111111111111111111111111"
if _, err := b.js.CreateOrUpdateConsumer(b.ctx, "EVENTS", jetstream.ConsumerConfig{Durable: "n1_mesh-delivery-assigner",
FilterSubjects: []string{enginereads.SubjectPrefix + "*", "mesh.mod." + buckets.IndexerModule + ".event.views.>", meshcatalog.SubjectCatalogRevisionAll,
nodebuildgo.SubjectStepEndedAll, nodebuildgo.SubjectAskTakenAll, nodebuildgo.SubjectStepProgressAll, pipelineEndedSubject},
AckPolicy: jetstream.AckExplicitPolicy, AckWait: 2 * time.Second}); err != nil { // an event handed to the stopped assigner's pull comes again soon
t.Fatal(err)
}
for _, node := range []string{"n1", "n2"} {
raw, _ := json.Marshal(nodeengine.Running{V: 1, Node: node, Module: "engine", Fingerprint: "src2:e1", At: time.Now().UTC()})
if _, err := b.js.Publish(b.ctx, nodeengine.SubjectRunning(node, "engine"), raw); err != nil {
t.Fatal(err)
}
}
awaitKept(t, b, nodeengine.StreamRunning, 2)
engine := meshcatalog.CatalogEntry{Module: "engine", Fp: "src2:e1", Repository: "novox/mesh-host", Commit: commit, Path: ".",
Serves: []string{nodeengine.Seat + ".composition"}, Manifest: map[string]any{"module": "engine", "version": "0.1"}}
b.cat.mu.Lock()
b.cat.entries[trunkBranch] = map[string]meshcatalog.CatalogEntry{"engine": engine}
for _, m := range []string{"a", "b", "c"} {
b.cat.entries[trunkBranch][m] = meshcatalog.CatalogEntry{Module: m, Fp: "src2:" + m, Repository: "novox/" + m, Commit: commit, Path: ".", Manifest: map[string]any{"module": m}}
}
b.cat.rev[trunkBranch] = 3
b.cat.mu.Unlock()
for _, node := range []string{"n1", "n2"} {
for _, m := range []string{"a", "b", "c"} {
b.place(t, node, m)
}
}
var mu sync.Mutex
inputsAsked := map[string]int{} // by candidate and node
var drops []string
respond := func(subject string, f func(m *nats.Msg) any) {
sub, err := nc.Subscribe(subject, func(m *nats.Msg) {
raw, _ := json.Marshal(map[string]any{"result": f(m), "node": "n0"})
_ = m.Respond(raw)
})
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = sub.Unsubscribe() })
}
respond("mesh.seat.node-engine.call.compose-candidate.*.*", func(m *nats.Msg) any {
node := strings.Split(m.Subject, ".")[5]
var in nodeengine.ComposeCandidateInput
_ = json.Unmarshal(m.Data, &in)
mu.Lock()
inputsAsked[in.Candidate+" "+node]++
mu.Unlock()
return map[string]any{"candidate": in.Candidate, "node": node, "verdict": "composed", "input": map[string]any{"node": node}, "modules": []any{}, "own": map[string]any{}, "took": 1, "v": 1}
})
respond("mesh.seat.mesh-catalog.call.entry.*", func(*nats.Msg) any { return meshcatalog.EntryAnswer{Entry: engine} })
respond("mesh.seat.node-build-go.call.drop.*", func(m *nats.Msg) any {
var in nodebuildgo.DropInput
_ = json.Unmarshal(m.Data, &in)
mu.Lock()
drops = append(drops, in.ID)
mu.Unlock()
return map[string]any{}
})
respond("mesh.seat.mesh-hand-acts.call.hand-act.*", func(*nats.Msg) any { return map[string]any{"log": "hand-act-1"} })
// The ends are read from the stream through a durable consumer, as the runner hears them: an end the bus drops as a
// repeated message id is never heard (A-470q).
ended := make(chan meshwants.StepEnded, 32)
runner, err := b.js.CreateOrUpdateConsumer(b.ctx, "EVENTS", jetstream.ConsumerConfig{Durable: "runner-stand-in",
FilterSubject: "mesh.seat.mesh-wants.event.step-ended.n1", AckPolicy: jetstream.AckExplicitPolicy, DeliverPolicy: jetstream.DeliverNewPolicy})
if err != nil {
t.Fatal(err)
}
heard, err := runner.Consume(func(m jetstream.Msg) {
var e meshwants.StepEnded
_ = json.Unmarshal(m.Data(), &e)
_ = m.Ack()
ended <- e
})
if err != nil {
t.Fatal(err)
}
t.Cleanup(heard.Stop)
subscribed(t, nc)
from := b.journal.mark()
_, stop := b.startStoppable(t)
b.journal.await(t, from, "the composition step is heard on", "the composition step served", 30*time.Second)
select {
case <-b.seatListRead:
case <-time.After(30 * time.Second):
t.Fatal("the assigner did not read the seat list within 30 s")
}
queue, _ := buildstep.AskStream(buildstep.AskGo)
worker, err := b.js.CreateOrUpdateConsumer(b.ctx, queue, jetstream.ConsumerConfig{Durable: "stand-in", FilterSubject: "mesh.seat.node-build-go.accept.check.mesh-wants",
AckPolicy: jetstream.AckExplicitPolicy})
if err != nil {
t.Fatal(err)
}
step := func(id, module, run string, rerun bool) {
t.Helper()
s := meshwants.StepRun{Pipeline: "main.3." + module, Step: meshwants.StepRunStepComposition, Module: module, Branch: trunkBranch, Revision: 3,
Commit: commit, Fingerprint: "src2:" + module, Run: run, Rerun: rerun}
if run != "" {
s.Pipeline = "run-" + run + "." + module
}
if rerun {
s.By = "op@n1"
}
raw, _ := json.Marshal(s)
msg := nats.NewMsg(meshwants.AcceptSubjectStep)
msg.Header.Set("x-event-id", id)
msg.Data = raw
if err := nc.PublishMsg(msg); err != nil {
t.Fatal(err)
}
}
type asked struct {
ask buildstep.StepAsk
machine string
}
// asks takes n run asks of the stand-in build seat, each with the machine its request composes.
asks := func(n int) []asked {
t.Helper()
var out []asked
for range n {
got, err := worker.Next(jetstream.FetchMaxWait(20 * time.Second))
if err != nil {
t.Fatalf("%d of %d run asks put: %v", len(out), n, err)
}
_ = got.Ack()
a, err := buildstep.DecodeStepAsk(got.Data())
if err != nil || a.Run == nil {
t.Fatalf("the run ask %s does not read: %v", got.Data(), err)
}
req, err := composer.Decode(readInput(t, nc, a.Run.Input))
if err != nil || len(req.Machines) != 1 || req.BlameTries != 20 {
t.Fatalf("the request of %s: %+v %v", a.ID, req, err)
}
out = append(out, asked{a, req.Machines[0]})
}
return out
}
noMoreAsks := func(what string) {
t.Helper()
if got, err := worker.Next(jetstream.FetchMaxWait(300 * time.Millisecond)); err == nil {
t.Fatalf("%s: one more run ask is put: %s", what, got.Data())
}
}
say := func(event string, node string, body any) {
t.Helper()
raw, _ := json.Marshal(body)
if _, err := b.js.Publish(b.ctx, "mesh.seat.node-build-go.event."+event+"."+node, raw); err != nil {
t.Fatal(err)
}
}
answer := func(a asked, machine string) {
t.Helper()
v := fmt.Sprintf(`{"v":%d,"cases":[{"name":"change","manifests":{},"machines":{%q:%s}}]}`, composer.Version, a.machine, machine)
say("step-ended", "b1", buildstep.StepEnded{V: 1, ID: a.ask.ID, Ask: a.ask.ID, Asker: runAsker, Node: "b1", Outcome: buildstep.OutcomePassed, Output: v, At: time.Now().UTC()})
}
ends := func(n int, what string) map[string]meshwants.StepEnded {
t.Helper()
out := map[string]meshwants.StepEnded{}
for len(out) < n {
select {
case e := <-ended:
out[e.ID] = e
case <-time.After(20 * time.Second):
t.Fatalf("%s: %d of %d steps ended: %+v", what, len(out), n, out)
}
}
return out
}
runKeys := func(candidate string) []string {
all, _ := b.buckets[buckets.CompositionRuns()].Keys(b.ctx)
var out []string
for _, k := range all {
if isRunKey(k) && strings.HasPrefix(k, keysPart(candidate)+".") {
out = append(out, k)
}
}
return out
}
// readers waits, through a watch of the key, until the run of key has n readers.
readers := func(key string, n int) {
t.Helper()
w, err := b.buckets[buckets.CompositionRuns()].Watch(b.ctx, key)
if err != nil {
t.Fatal(err)
}
defer func() { _ = w.Stop() }()
deadline := time.After(10 * time.Second)
for {
select {
case e := <-w.Updates():
var run meshwants.CompositionRun
if e != nil && json.Unmarshal(e.Value(), &run) == nil && len(run.Readers) == n {
return
}
case <-deadline:
t.Fatalf("the run %s has not %d readers", key, n)
}
}
}
blames := `{"composes":false,"reason":"two modules own one path","blamed":[{"module":"a","problem":"/etc/x is a's and b's"},{"module":"b","problem":"/etc/x is a's and b's"}]}`
composes := `{"composes":true,"blamed":[]}`
// One candidate, three steps, two machines: two runs.
for _, m := range []string{"a", "b", "c"} {
step("step-"+m, m, "", false)
}
first := asks(2)
noMoreAsks("three steps of one candidate")
for _, a := range first {
answer(a, map[string]string{"n1": composes, "n2": blames}[a.machine])
}
got := ends(3, "the three steps")
if a, bb, c := got["step-a"], got["step-b"], got["step-c"]; a.Outcome != meshwants.StepEndedOutcomeFailed || !strings.Contains(a.Said, "n2 holds a") ||
bb.Outcome != meshwants.StepEndedOutcomeFailed || c.Outcome != meshwants.StepEndedOutcomePassed || len(c.Warnings) != 2 ||
!strings.Contains(c.Warnings[0], "n2 holds a") || len(c.Shared) != 2 || c.Shared[0].Ask == c.Shared[1].Ask {
t.Fatalf("the steps of one candidate: %+v", got)
}
if keys := runKeys("main.3"); len(keys) != 2 || inputsAsked["main.3 n1"] != 1 || inputsAsked["main.3 n2"] != 1 {
t.Fatalf("keys %v, inputs asked %v: want one run and one input per machine", keys, inputsAsked)
}
// Another candidate (a run on request): its own runs; an old composer and a machine blaming none fail.
step("step-r1", "a", "r1", false)
for _, a := range asks(2) {
answer(a, map[string]string{"n1": `{"composes":false,"reason":"x"}`, "n2": `{"composes":false,"reason":"own","blamed":[]}`}[a.machine])
}
if e := ends(1, "another candidate")["step-r1"]; e.Outcome != meshwants.StepEndedOutcomeFailed || !strings.Contains(e.Said, "n2 does not compose and blames no module") ||
len(e.Warnings) != 1 || e.Warnings[0] != "n1: the composer names no culprits" {
t.Fatalf("a step of another candidate: %+v", e)
}
// The bound: n1's ask taken and saying progress past the bound, then silent; n2's never taken.
from = b.journal.mark()
step("step-r2", "a", "r2", false)
bound := asks(2)
var n1 asked
for _, a := range bound {
if a.machine == "n1" {
n1 = a
}
}
say("ask-taken", "b1", nodebuildgo.AskTaken{Ask: n1.ask.ID, Asker: runAsker, At: time.Now().UTC(), Node: "b1"})
for i := range 3 { // past the 2 s bound, each beat within it
select {
case e := <-ended:
t.Fatalf("a run saying its progress ended: %+v", e)
case <-time.After(time.Second):
}
if i == 0 { // the bound holds its run's ask, published at the key's latest revision with its own time to live (A-470n)
runs := b.buckets[buckets.CompositionRuns()]
all, _ := runs.Keys(b.ctx)
held := ""
for _, k := range all {
if strings.HasPrefix(k, runBoundPrefix) {
if e, err := runs.Get(b.ctx, k); err == nil {
var v boundValue
_ = json.Unmarshal(e.Value(), &v)
held += v.Ask
}
}
}
if held != n1.ask.ID {
t.Fatalf("the bound keys hold the asks %q, want n1's %s alone", held, n1.ask.ID)
}
}
say("step-progress", "b1", nodebuildgo.StepProgress{Ask: n1.ask.ID, Asker: runAsker, At: time.Now().UTC(), Phase: "run"})
}
e := ends(1, "the bounds")["step-r2"]
if e.Outcome != meshwants.StepEndedOutcomeCouldNotRun || !strings.Contains(e.Said, "said no progress within 2s (composition-step-bound)") ||
!strings.Contains(e.Said, "taken by no build node within 15s (run-untaken-bound)") {
t.Fatalf("past the bounds: %+v", e)
}
mu.Lock()
dropped := len(drops)
mu.Unlock()
if dropped != 2 {
t.Fatalf("%d asks dropped past their bounds, want 2", dropped)
}
answer(n1, composes) // heard after the bound: ignored and journaled
b.journal.await(t, from, "the step-ended of the run ask "+n1.ask.ID+" is ignored", "a late step-ended", 10*time.Second)
// A rerun of a: n1 it passed keeps its verdict; n2 it failed starts one run carrying b, which moves.
step("step-a2", "a", "", true)
again := asks(1)
noMoreAsks("a rerun of a")
if again[0].machine != "n2" {
t.Fatalf("the rerun composes %s, want only the machine a did not pass", again[0].machine)
}
answer(again[0], composes)
moved := ends(2, "the rerun")
if a2, bb := moved["step-a2"], moved["step-b"]; a2.Outcome != meshwants.StepEndedOutcomePassed || bb.Outcome != meshwants.StepEndedOutcomePassed ||
!bb.Rerun || bb.By != "op@n1" || len(bb.Shared) != 2 || bb.Shared[1].Ask != again[0].ask.ID {
t.Fatalf("after the rerun (asked again %s): a2 %+v; b %+v", again[0].ask.ID, moved["step-a2"].Shared, moved["step-b"])
}
// A superseded step leaves the runs; its other reader goes on.
step("step-r3a", "a", "r3", false)
step("step-r3b", "b", "r3", false)
sup := asks(2)
readers(runKeyOf("run-r3", "n1"), 2)
readers(runKeyOf("run-r3", "n2"), 2)
pe, _ := json.Marshal(mpr.PipelineEnded{V: 1, Pipeline: "run-r3.a", Module: "a", Branch: trunkBranch, Commit: commit, Outcome: mpr.PipelineEndedOutcomeSuperseded, Run: "r3", Steps: []mpr.StepResult{}})
if _, err := b.js.Publish(b.ctx, "mesh.seat."+mpr.Seat+".event."+mpr.EventPipelineEnded+".n1", pe); err != nil {
t.Fatal(err)
}
for _, a := range sup {
answer(a, composes)
}
if got := ends(1, "the superseded run"); got["step-r3b"].Outcome != meshwants.StepEndedOutcomePassed {
t.Fatalf("the other reader: %+v", got)
}
select {
case e := <-ended:
t.Fatalf("a superseded step said its end: %+v", e)
case <-time.After(200 * time.Millisecond):
}
// Restarted during a run: the new assigner takes it up by its ask.
step("step-r4", "c", "r4", false)
restart := asks(2)
stop()
for _, a := range restart {
answer(a, composes)
}
from = b.journal.mark()
b.startStoppable(t)
b.journal.await(t, from, "is taken up again by its runs (ask step-r4)", "the take-up", 30*time.Second)
if e := ends(1, "the take-up")["step-r4"]; e.Outcome != meshwants.StepEndedOutcomePassed || len(e.Shared) != 2 {
t.Fatalf("the step taken up: %+v", e)
}
// compositions-drop: the readers ended naming the drop, the key deleted.
step("step-r5", "a", "r5", false)
asks(2)
body, _ := json.Marshal(meshwants.CompositionsDropInput{Candidate: "run-r5", Machine: "n1", Why: "stuck"})
reply, err := nc.Request("mesh.seat.mesh-wants.call.compositions-drop.n1~mesh-delivery-pipeline", body, 10*time.Second)
if err != nil || !strings.Contains(string(reply.Data), `"ended":[`) {
t.Fatalf("compositions-drop answered %s %v", reply.Data, err)
}
if e := ends(1, "the drop")["step-r5"]; e.Outcome != meshwants.StepEndedOutcomeCouldNotRun || !strings.Contains(e.Said, "dropped by hand: stuck") {
t.Fatalf("a reader of the dropped run: %+v", e)
}
if keys := runKeys("run-r5"); len(keys) != 1 {
t.Errorf("the keys of run-r5 after the drop: %v, want n2's alone", keys)
}
// At the cap of 1 MiB no run starts: the step ends could not run, naming the fill.
for i := range 2 {
if _, err := b.buckets[buckets.CompositionRuns()].Put(b.ctx, fmt.Sprintf("fill%d", i), make([]byte, 700<<10)); err != nil {
t.Fatal(err)
}
}
step("step-r6", "a", "r6", false)
if e := ends(1, "the cap")["step-r6"]; e.Outcome != meshwants.StepEndedOutcomeCouldNotRun || !strings.Contains(e.Said, "is at its cap of 1 MiB") {
t.Fatalf("at the cap: %+v", e)
}
}
@@ -1,642 +0,0 @@
package main
import (
"context"
"encoding/json"
"fmt"
"maps"
"os"
"path/filepath"
"slices"
"strings"
"sync"
"testing"
"time"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
)
// runsMem is the state composition-runs in memory, its bound keys' times to live kept beside.
type runsMem struct {
mu sync.Mutex
vals map[string][]byte
revs map[string]uint64
ttl map[string]time.Duration
bytes int64
n uint64
// beforeDel, when set, runs once before the next delete: another write landing between a read and the delete.
beforeDel func()
// beforeArm, when set, runs once before the next arm: another write landing between a read and the arm.
beforeArm func()
}
func (k *runsMem) another() {
k.mu.Lock()
f := k.beforeDel
k.beforeDel = nil
k.mu.Unlock()
if f != nil {
f()
}
}
func newRunsMem() *runsMem {
return &runsMem{vals: map[string][]byte{}, revs: map[string]uint64{}, ttl: map[string]time.Duration{}}
}
func (k *runsMem) get(key string) ([]byte, uint64, error) {
k.mu.Lock()
defer k.mu.Unlock()
return k.vals[key], k.revs[key], nil
}
func (k *runsMem) create(key string, v []byte) (uint64, error) {
k.mu.Lock()
defer k.mu.Unlock()
if _, ok := k.vals[key]; ok {
return 0, errRunChanged
}
k.n++
k.vals[key], k.revs[key] = v, k.n
return k.n, nil
}
func (k *runsMem) update(key string, v []byte, rev uint64) (uint64, error) {
k.mu.Lock()
defer k.mu.Unlock()
if k.revs[key] != rev {
return 0, errRunChanged
}
k.n++
k.vals[key], k.revs[key] = v, k.n
return k.n, nil
}
func (k *runsMem) delAt(key string, rev uint64) error {
k.another()
k.mu.Lock()
defer k.mu.Unlock()
if _, ok := k.vals[key]; !ok {
return nil
}
if k.revs[key] != rev {
return errRunChanged
}
delete(k.vals, key)
delete(k.revs, key)
delete(k.ttl, key)
return nil
}
func (k *runsMem) keys() ([]string, error) {
k.mu.Lock()
defer k.mu.Unlock()
var out []string
for key := range k.vals {
out = append(out, key)
}
slices.Sort(out)
return out, nil
}
func (k *runsMem) arm(key, ask string, ttl time.Duration) (uint64, error) {
k.mu.Lock()
f := k.beforeArm
k.beforeArm = nil
k.mu.Unlock()
if f != nil {
f()
}
raw, _ := json.Marshal(boundValue{Bound: key, Ask: ask})
k.mu.Lock()
defer k.mu.Unlock()
k.n++
k.vals[key], k.revs[key], k.ttl[key] = raw, k.n, ttl
return k.n, nil
}
// expire ends a bound key's time to live: the key gone, as the bus's marker leaves it.
func (k *runsMem) expire(key string) {
k.mu.Lock()
defer k.mu.Unlock()
delete(k.vals, key)
delete(k.revs, key)
delete(k.ttl, key)
}
func (k *runsMem) used() (int64, error) {
k.mu.Lock()
defer k.mu.Unlock()
if k.bytes > 0 {
return k.bytes, nil
}
var n int64
for _, v := range k.vals {
n += int64(len(v))
}
return n, nil
}
// Design 47 G §718, §719, §720, A-468, A-469d, A-469x: a run's key is `<candidate key>.<machine>`, each part escaped; two
// steps of one candidate joining at once start one run and the second reads it; a run that could not run is replaced; at the
// cap no run starts, naming the fill; a run is deleted composition-runs-kept after its end; the settings are read from their files.
func TestACompositionRunIsOneKeyPerCandidateAndMachine(t *testing.T) {
now := time.Date(2026, 11, 17, 12, 0, 0, 0, time.UTC)
step := func(id, module string) meshwants.CompositionReader {
return readerOf(id, meshwants.StepRun{Pipeline: "main.3." + module, Step: meshwants.StepRunStepComposition, Module: module, Branch: "main", Revision: 3})
}
t.Run("key", func(t *testing.T) {
if k := runKeyOf("main.3", "n1"); k != "main=2e3.n1" {
t.Errorf("the key is %q, want each part escaped as a key's name", k)
}
if isRunKey(runBoundPrefix+"main=2e3.n1") || isRunKey(runUntakenPrefix+"main=2e3.n1") || !isRunKey("main=2e3.n1") {
t.Error("a bound key is taken for a run's or a run's for a bound key")
}
})
t.Run("one run for two steps at once", func(t *testing.T) {
kv := newRunsMem()
r := newCompositionRuns(kv, defaultRunSettings, func() time.Time { return now }, t.Logf)
var wg sync.WaitGroup
got := make([]joined, 2)
for i, m := range []string{"a", "b"} {
wg.Add(1)
go func() {
defer wg.Done()
j, err := r.join(step("ask-"+m, m), "main.3", "n1", "run-"+m)
if err != nil {
t.Error(err)
}
got[i] = j
}()
}
wg.Wait()
runKeys := func() []string {
all, _ := kv.keys()
return slices.DeleteFunc(all, func(k string) bool { return !isRunKey(k) })
}
if got[0].started == got[1].started {
t.Fatalf("started %v and %v: want exactly one step to start the run", got[0].started, got[1].started)
}
keys := runKeys()
run, _, _, _ := r.read(runKeyOf("main.3", "n1"))
if len(keys) != 1 || len(run.Readers) != 2 || run.Candidate != "main.3" || run.Machine != "n1" || run.Started == nil {
t.Errorf("keys %v, run %+v: want one key read by both", keys, run)
}
if j, _ := r.join(step("ask-a", "a"), "main.3", "n1", "run-c"); j.started || len(j.run.Readers) != 2 {
t.Errorf("a reader joining again is added again or starts a run: %+v", j)
}
})
t.Run("could not run is never read", func(t *testing.T) {
kv := newRunsMem()
r := newCompositionRuns(kv, defaultRunSettings, func() time.Time { return now }, t.Logf)
if _, err := r.join(step("ask-a", "a"), "main.3", "n1", "run-1"); err != nil {
t.Fatal(err)
}
key := runKeyOf("main.3", "n1")
_, _, _ = r.change(key, func(c *meshwants.CompositionRun) bool {
c.Ended, c.Outcome = &now, meshwants.CompositionRunOutcomeCouldNotRun
return true
})
j, err := r.join(step("ask-b", "b"), "main.3", "n1", "run-2")
if err != nil || !j.started || j.run.Ask != "run-2" || len(j.run.Readers) != 1 || j.run.Ended != nil {
t.Errorf("after a could not run: %+v %v, want a new run read by its starter alone", j, err)
}
_, _, _ = r.change(key, func(c *meshwants.CompositionRun) bool {
c.Ended, c.Outcome = &now, meshwants.CompositionRunOutcomePassed
return true
})
if j, _ := r.join(step("ask-c", "c"), "main.3", "n1", "run-3"); j.started || j.run.Ask != "run-2" {
t.Errorf("a passed run is not read: %+v", j)
}
if j, _ := r.join(step("ask-d", "d"), "main.4", "n1", "run-4"); !j.started {
t.Errorf("a step of another candidate reads the first's run: %+v", j)
}
})
t.Run("cap", func(t *testing.T) {
kv := newRunsMem()
kv.bytes = 16 << 20
r := newCompositionRuns(kv, defaultRunSettings, func() time.Time { return now }, t.Logf)
_, err := r.join(step("ask-a", "a"), "main.3", "n1", "run-1")
if err == nil || !strings.Contains(err.Error(), "at its cap of 16 MiB (16777216 bytes kept)") {
t.Errorf("at the cap: %v, want the fill named", err)
}
})
t.Run("kept", func(t *testing.T) {
kv := newRunsMem()
at := now
r := newCompositionRuns(kv, defaultRunSettings, func() time.Time { return at }, t.Logf)
for _, c := range []string{"main.1", "main.2"} {
if _, err := r.join(step("ask-"+c, "a"), c, "n1", "run-"+c); err != nil {
t.Fatal(err)
}
}
old := now.Add(-25 * time.Hour)
_, _, _ = r.change(runKeyOf("main.1", "n1"), func(c *meshwants.CompositionRun) bool { c.Ended, c.Outcome = &old, "passed"; return true })
_, _ = kv.arm(runBoundPrefix+runKeyOf("main.1", "n1"), "run-main.1", time.Minute)
recent := now.Add(-time.Hour)
_, _, _ = r.change(runKeyOf("main.2", "n1"), func(c *meshwants.CompositionRun) bool { c.Ended, c.Outcome = &recent, "passed"; return true })
if _, err := r.join(step("ask-x", "a"), "main.9", "n1", "run-9"); err != nil {
t.Fatal(err)
}
keys, _ := kv.keys()
for _, k := range []string{runBoundPrefix, runUntakenPrefix} {
if v, _, _ := kv.get(k + runKeyOf("main.1", "n1")); v != nil {
t.Errorf("the key %s stays past its run", k+runKeyOf("main.1", "n1"))
}
}
keys = slices.DeleteFunc(keys, func(k string) bool { return !isRunKey(k) })
if want := []string{runKeyOf("main.2", "n1"), runKeyOf("main.9", "n1")}; !slices.Equal(keys, want) {
t.Errorf("keys %v, want %v: a run ended more than a day ago goes with its bound key at the next run", keys, want)
}
})
t.Run("a listing that misses a key being written", func(t *testing.T) {
kv := &missingRuns{runsMem: newRunsMem()}
r := newCompositionRuns(kv, defaultRunSettings, func() time.Time { return now }, t.Logf)
if _, err := r.join(step("ask-a", "a"), "main.3", "n1", "run-1"); err != nil {
t.Fatal(err)
}
kv.hide = runKeyOf("main.3", "n1")
if key, _, _ := r.keyOfAsk("run-1"); key != "" {
t.Fatalf("the stand's listing does not hide the key")
}
r.saw(kv.hide, true) // as the state's watch hands it
if key, _, _ := r.keyOfAsk("run-1"); key != kv.hide {
t.Errorf("the run of ask run-1 is not found by the keys the watch handed: %q", key)
}
r.saw(kv.hide, false)
if keys, _ := r.keys(); slices.Contains(keys, kv.hide) {
t.Errorf("a key the watch saw deleted is still listed: %v", keys)
}
})
// No delete lands on a value newer than the one it was judged from (the kind test 109 met in test-runs): a rerun's run
// replacing the key, and its untaken bound, stand.
for _, c := range []struct {
name string
ended time.Time
act func(r *compositionRuns, key string) error
}{
{"the sweep of a run replaced meanwhile", now.Add(-25 * time.Hour), func(r *compositionRuns, _ string) error { return r.sweep() }},
{"the end of a run replaced meanwhile", time.Time{}, func(r *compositionRuns, key string) error {
_, err := r.endRun(key, "run-1", meshwants.CompositionRunOutcomePassed, "", nil)
return err
}},
} {
t.Run(c.name, func(t *testing.T) {
kv := newRunsMem()
r := newCompositionRuns(kv, defaultRunSettings, func() time.Time { return now }, t.Logf)
if _, err := r.join(step("ask-a", "a"), "main.3", "n1", "run-1"); err != nil {
t.Fatal(err)
}
key := runKeyOf("main.3", "n1")
if !c.ended.IsZero() {
_, _, _ = r.change(key, func(run *meshwants.CompositionRun) bool { run.Ended, run.Outcome = &c.ended, "failed"; return true })
}
_, _ = kv.arm(runBoundPrefix+key, "run-1", time.Minute)
kv.beforeDel = func() { // a rerun replaces the key and arms its untaken bound
for {
raw, rev, _ := kv.get(key)
var run meshwants.CompositionRun
_ = json.Unmarshal(raw, &run)
if run.Ended == nil {
at := now
run.Ended, run.Outcome = &at, "failed"
}
fresh, _ := json.Marshal(meshwants.CompositionRun{Candidate: "main.3", Machine: "n1", Ask: "run-2", Started: &now, Readers: run.Readers})
if _, err := kv.update(key, fresh, rev); err == nil {
break
}
}
_, _ = kv.arm(runUntakenPrefix+key, "run-2", time.Hour)
}
if err := c.act(r, key); err != nil {
t.Fatal(err)
}
if run, _, ok, _ := r.read(key); !ok || run.Ask != "run-2" || run.Ended != nil {
t.Errorf("the key holds %+v (stands %v), want the rerun's running run-2", run, ok)
}
if v, _, _ := kv.get(runUntakenPrefix + key); v == nil {
t.Error("the untaken bound the rerun's run armed is deleted")
}
})
}
t.Run("settings", func(t *testing.T) {
dir := t.TempDir()
files := map[string]string{"MESH_ASSIGNER_COMPOSITION_RUNS_CAP_MIB_FILE": "8", "MESH_ASSIGNER_COMPOSITION_RUNS_KEPT_FILE": "2h",
"MESH_ASSIGNER_COMPOSITION_STEP_BOUND_FILE": "2s", "MESH_ASSIGNER_RUN_UNTAKEN_BOUND_FILE": "", "MESH_ASSIGNER_BLAME_TRIES_FILE": "5"}
env := map[string]string{}
for name, content := range files {
p := filepath.Join(dir, name)
if err := os.WriteFile(p, []byte(content), 0o600); err != nil {
t.Fatal(err)
}
env[name] = p
}
s, err := runSettingsOf(func(k string) string { return env[k] })
if err != nil || s.capBytes != 8<<20 || s.kept != 2*time.Hour || s.stepBound != 2*time.Second || s.untaken != 24*time.Hour || s.blameTries != 5 {
t.Errorf("settings %+v %v", s, err)
}
_ = os.WriteFile(env["MESH_ASSIGNER_COMPOSITION_STEP_BOUND_FILE"], []byte("30"), 0o600)
if _, err := runSettingsOf(func(k string) string { return env[k] }); err == nil || !strings.Contains(err.Error(), "not a positive Go duration") {
t.Errorf("a bound of no unit: %v", err)
}
if s, err := runSettingsOf(func(string) string { return "" }); err != nil || s != defaultRunSettings {
t.Errorf("no files: %+v %v, want the defaults", s, err)
}
})
raw, _ := json.Marshal(step("x", "a"))
if !strings.Contains(string(raw), `"id":"x"`) {
t.Errorf("a reader names no ask id: %s", raw)
}
}
// Design 47 G §720, A-466s, A-468b, A-469y, A-470: a run's untaken bound is armed at its start and each reader said waiting for a
// build node; ask-taken arms the bound of composition-step-bound and ends the waits ran; each step-progress arms it again; the
// bound's expiry drops the ask and ends the run could not run naming the ask and the bound, a step-ended heard later ignored and
// journaled; the untaken bound's expiry ends it naming run-untaken-bound, the waits superseded; the readers of a run with no end
// are taken up at the start.
func TestACompositionRunsBoundTimesOwedProgress(t *testing.T) {
now := time.Date(2026, 11, 17, 12, 0, 0, 0, time.UTC)
reader := func(id string) meshwants.CompositionReader {
return readerOf(id, meshwants.StepRun{Pipeline: "main.3." + id, Step: meshwants.StepRunStepComposition, Module: id, Branch: "main", Revision: 3})
}
type said struct{ event, how, module string }
setup := func(t *testing.T) (*runsMem, *compositionRuns, *[]said, *[]string, *[]string) {
kv := newRunsMem()
var saids []said
var drops, journal []string
var mu sync.Mutex
r := newCompositionRuns(kv, defaultRunSettings, func() time.Time { return now }, func(f string, a ...any) {
mu.Lock()
journal = append(journal, fmt.Sprintf(f, a...))
mu.Unlock()
})
r.say = func(event string, body any, _ string) error {
w := body.(meshwants.Wait)
if w.On != waitOnBuildNode || w.Step != "pipeline" || w.Owner != meshwants.Seat || w.Node == "" {
t.Errorf("the wait said %+v", w)
}
saids = append(saids, said{event, w.How, w.Module})
return nil
}
r.drop = func(ask, why string) error { drops = append(drops, ask+": "+why); return nil }
return kv, r, &saids, &drops, &journal
}
key := runKeyOf("main.3", "n1")
t.Run("taken, progress, bound", func(t *testing.T) {
kv, r, saids, drops, journal := setup(t)
if _, err := r.join(reader("a"), "main.3", "n1", "run-1"); err != nil {
t.Fatal(err)
}
if _, err := r.join(reader("b"), "main.3", "n1", "run-2"); err != nil {
t.Fatal(err)
}
if kv.ttl[runUntakenPrefix+key] != 24*time.Hour || !slices.Equal(*saids, []said{{meshwants.EventWaitHeld, "", "a"}, {meshwants.EventWaitHeld, "", "b"}}) {
t.Fatalf("at the start: untaken %v, said %v", kv.ttl, *saids)
}
if err := r.heardAskTaken(nodebuildgo.AskTaken{Ask: "run-1", Asker: "someone-else"}); err != nil || kv.ttl[runBoundPrefix+key] != 0 {
t.Fatalf("another asker's ask-taken arms a bound: %v %v", kv.ttl, err)
}
if err := r.heardAskTaken(nodebuildgo.AskTaken{Ask: "run-1", Asker: runAsker, Node: "b1"}); err != nil {
t.Fatal(err)
}
_, untaken, _ := kv.get(runUntakenPrefix + key)
if kv.ttl[runBoundPrefix+key] != 30*time.Minute || untaken != 0 || len(*saids) != 4 || (*saids)[2].how != "ran" || (*saids)[3].how != "ran" {
t.Fatalf("at ask-taken: %v, said %v", kv.ttl, *saids)
}
before := kv.revs[runBoundPrefix+key]
if err := r.heardProgress(nodebuildgo.StepProgress{Ask: "run-1", Asker: runAsker}); err != nil || kv.revs[runBoundPrefix+key] == before {
t.Fatalf("a step-progress does not arm the bound again: %v", err)
}
kv.expire(runBoundPrefix + key)
r.expired(runBoundPrefix + key)
run, _, _, _ := r.read(key)
if run.Outcome != meshwants.CompositionRunOutcomeCouldNotRun || run.Said != "n1: its run ask run-1 said no progress within 30m0s (composition-step-bound)" ||
len(*drops) != 1 || !strings.HasPrefix((*drops)[0], "run-1: ") {
t.Fatalf("past the bound: %+v, drops %v", run, *drops)
}
if got := judgeRun("a", run); got.couldNot != run.Said {
t.Errorf("a reader of the run past its bound: %+v", got)
}
late := buildstep.StepEnded{Ask: "run-1", Asker: runAsker, Outcome: buildstep.OutcomePassed, Output: "{}"}
if err := r.heardEnd(late, func(e buildstep.StepEnded) ([]byte, error) { return []byte(e.Output), nil }); err != nil {
t.Fatal(err)
}
if again, _, _, _ := r.read(key); again.Said != run.Said || !slices.ContainsFunc(*journal, func(l string) bool {
return strings.Contains(l, "step-ended of the run ask run-1 is ignored") && strings.Contains(l, key)
}) {
t.Errorf("a step-ended after the bound: %+v, journal %v", again, *journal)
}
if keys, _ := kv.keys(); !slices.Equal(keys, []string{key}) {
t.Errorf("the bound keys stay past the run's end: %v", keys)
}
})
t.Run("untaken", func(t *testing.T) {
kv, r, saids, drops, _ := setup(t)
if _, err := r.join(reader("a"), "main.3", "n1", "run-1"); err != nil {
t.Fatal(err)
}
kv.expire(runUntakenPrefix + key)
r.expired(runUntakenPrefix + key)
run, _, _, _ := r.read(key)
if run.Outcome != meshwants.CompositionRunOutcomeCouldNotRun || run.Said != "n1: its run ask run-1 was taken by no build node within 24h0m0s (run-untaken-bound)" ||
len(*drops) != 1 || !slices.Equal(*saids, []said{{meshwants.EventWaitHeld, "", "a"}, {meshwants.EventWaitEnded, "superseded", "a"}}) {
t.Errorf("untaken past its bound: %+v, drops %v, said %v", run, *drops, *saids)
}
r.expired(runUntakenPrefix + key) // a run ended takes nothing
if len(*drops) != 1 {
t.Errorf("an ended run is dropped again: %v", *drops)
}
})
// A-470n, A-470o: a bound holds its run's ask, armed for it alone; a late signal of a run replaced moves nothing and is
// journaled; an expiry ends the run only while its bound is still absent and the run taken (bound.) or untaken (untaken.).
rerun := func(kv *runsMem, r *compositionRuns) func() { // a rerun replaces run-1 by run-2, its bounds gone, run-2's untaken armed
return func() {
_, _, _ = r.change(key, func(c *meshwants.CompositionRun) bool { c.Ask = "run-2"; return true })
kv.expire(runBoundPrefix + key)
_, _ = kv.arm(runUntakenPrefix+key, "run-2", time.Hour)
}
}
taken := func(t *testing.T, r *compositionRuns) {
if err := r.heardAskTaken(nodebuildgo.AskTaken{Ask: "run-1", Asker: runAsker, Node: "b1"}); err != nil {
t.Fatal(err)
}
}
for _, c := range []struct {
name string
act func(t *testing.T, kv *runsMem, r *compositionRuns)
ended bool
bound string // the ask the bound key holds, "" for none
untaken string
journal string
}{
{"a late ask-taken of a run replaced as it is heard", func(t *testing.T, kv *runsMem, r *compositionRuns) {
kv.beforeArm = rerun(kv, r)
taken(t, r)
}, false, "", "run-2", "the ask-taken of the run ask run-1 is late"},
{"a late step-progress of a run replaced as it is heard", func(t *testing.T, kv *runsMem, r *compositionRuns) {
taken(t, r)
kv.beforeArm = rerun(kv, r)
if err := r.heardProgress(nodebuildgo.StepProgress{Ask: "run-1", Asker: runAsker}); err != nil {
t.Fatal(err)
}
kv.expire(runBoundPrefix + key) // a bound left standing would end run-2 here
r.expired(runBoundPrefix + key)
}, false, "", "run-2", "the step-progress of the run ask run-1 is late"},
{"a step-progress of a run gone", func(t *testing.T, kv *runsMem, r *compositionRuns) {
taken(t, r)
rerun(kv, r)()
if err := r.heardProgress(nodebuildgo.StepProgress{Ask: "run-1", Asker: runAsker}); err != nil {
t.Fatal(err)
}
}, false, "", "run-2", "the step-progress of the run ask run-1 is late"},
{"an expiry heard after the bound was armed again", func(t *testing.T, kv *runsMem, r *compositionRuns) {
taken(t, r)
r.expired(runBoundPrefix + key)
}, false, "run-1", "", "the expiry of " + runBoundPrefix + key + " ends nothing"},
{"the bound's expiry of a run not taken", func(t *testing.T, kv *runsMem, r *compositionRuns) {
_, _ = kv.arm(runBoundPrefix+key, "run-1", time.Minute)
kv.expire(runBoundPrefix + key)
r.expired(runBoundPrefix + key)
}, false, "", "run-1", "the expiry of " + runBoundPrefix + key + " ends nothing"},
{"the untaken expiry of a run taken", func(t *testing.T, kv *runsMem, r *compositionRuns) {
_, _ = kv.arm(runBoundPrefix+key, "run-1", time.Minute)
kv.expire(runUntakenPrefix + key)
r.expired(runUntakenPrefix + key)
}, false, "run-1", "", "the expiry of " + runUntakenPrefix + key + " ends nothing"},
{"an end of another ask", func(t *testing.T, kv *runsMem, r *compositionRuns) {
if ended, err := r.end(key, "run-0", meshwants.CompositionRunOutcomePassed, "", nil); ended || err != nil {
t.Errorf("the end of run-0 ended the run of run-1: %v %v", ended, err)
}
}, false, "", "run-1", ""},
{"the ask-taken of the run's own ask", func(t *testing.T, kv *runsMem, r *compositionRuns) {
taken(t, r)
}, false, "run-1", "", ""},
} {
t.Run(c.name, func(t *testing.T) {
kv, r, _, _, journal := setup(t)
if _, err := r.join(reader("a"), "main.3", "n1", "run-1"); err != nil {
t.Fatal(err)
}
c.act(t, kv, r)
run, _, _, _ := r.read(key)
if (run.Ended != nil) != c.ended {
t.Errorf("the run ended %v (%s), want %v", run.Ended != nil, run.Said, c.ended)
}
for prefix, want := range map[string]string{runBoundPrefix: c.bound, runUntakenPrefix: c.untaken} {
if held, _, _, _ := r.boundOf(prefix + key); held != want {
t.Errorf("%s holds the ask %q, want %q", prefix+key, held, want)
}
}
if c.journal != "" && !slices.ContainsFunc(*journal, func(l string) bool { return strings.Contains(l, c.journal) }) {
t.Errorf("the journal %v does not say %q", *journal, c.journal)
}
})
}
// A-469y, A-470u: a step asked again that joins a running run not yet taken, beside a reader or in its pipeline's former
// reader's place, says its own wait held, as join does; the former reader's wait, and that of a reader leaving, ends at once
// superseded.
held := func(m string) said { return said{meshwants.EventWaitHeld, "", m} }
gone := func(m string) said { return said{meshwants.EventWaitEnded, "superseded", m} }
for _, c := range []struct {
name string
act func(t *testing.T, r *compositionRuns)
want []said
}{
{"a rerun joins", func(t *testing.T, r *compositionRuns) {
again := reader("b")
again.Rerun = true
if _, err := r.rejoin(again, "main.3", "n1", "run-2"); err != nil {
t.Fatal(err)
}
}, []said{held("a"), held("b")}},
{"a rerun takes its former place", func(t *testing.T, r *compositionRuns) {
again := reader("a2")
again.Pipeline, again.Module, again.Rerun = "main.3.a", "a2", true
if _, err := r.rejoin(again, "main.3", "n1", "run-2"); err != nil {
t.Fatal(err)
}
}, []said{held("a"), gone("a"), held("a2")}},
{"a reader leaves", func(t *testing.T, r *compositionRuns) {
if _, err := r.join(reader("b"), "main.3", "n1", "run-2"); err != nil {
t.Fatal(err)
}
if _, err := r.leave("main.3.a", nil); err != nil {
t.Fatal(err)
}
}, []said{held("a"), held("b"), gone("a")}},
} {
t.Run(c.name, func(t *testing.T) {
_, r, saids, _, _ := setup(t)
if _, err := r.join(reader("a"), "main.3", "n1", "run-1"); err != nil {
t.Fatal(err)
}
c.act(t, r)
if !slices.Equal(*saids, c.want) {
t.Errorf("said %v, want %v", *saids, c.want)
}
})
}
t.Run("taken up at the start", func(t *testing.T) {
_, r, _, _, _ := setup(t)
for _, c := range []struct{ id, machine string }{{"a", "n1"}, {"a", "n2"}, {"b", "n1"}} {
if _, err := r.join(reader(c.id), "main.3", c.machine, "run-"+c.machine); err != nil {
t.Fatal(err)
}
}
ended := now
_, _, _ = r.change(runKeyOf("main.3", "n2"), func(c *meshwants.CompositionRun) bool { c.Ended, c.Outcome = &ended, "passed"; return true })
// A rerun's run carrying readers whose steps said their end (A-470l, A-470m): c passed and is not taken up, d is marked rerun.
_, _, _ = r.change(runKeyOf("main.3", "n1"), func(run *meshwants.CompositionRun) bool {
for _, id := range []string{"c", "d"} {
c := reader(id)
c.Ended, c.Rerun = &ended, id == "d"
run.Readers = append(run.Readers, c)
}
return true
})
readers, err := r.readersOfOpen()
if err != nil || len(readers) != 3 || readers[0].ID != "a" || readers[1].ID != "b" || readers[2].ID != "d" {
t.Fatalf("the readers taken up: %+v %v", readers, err)
}
if keys, _ := r.keysOf("main.3", "a"); !slices.Equal(keys, []string{runKeyOf("main.3", "n1"), runKeyOf("main.3", "n2")}) {
t.Errorf("the runs of reader a: %v", keys)
}
// A-470q: a reader with no `ended` says its first end under its ask's id; d, marked rerun, says its end again under
// `<id>.<the open run's ask>`.
ids := make(chan [2]string, 3)
steps := &compositionSteps{node: "n1", now: func() time.Time { return now }, logf: t.Logf,
resume: func(context.Context, string, meshwants.StepRun) stepOutcome {
return stepOutcome{Outcome: meshwants.StepEndedOutcomePassed}
},
say: func(_ string, body any, id string) error {
ids <- [2]string{body.(meshwants.StepEnded).ID, id}
return nil
}}
steps.takeUp(context.Background(), r)
got := map[string]string{}
for range 3 {
select {
case e := <-ids:
got[e[0]] = e[1]
case <-time.After(5 * time.Second):
t.Fatalf("the ends of the take-up: %v", got)
}
}
if want := map[string]string{"a": "a", "b": "b", "d": "d.run-n1"}; !maps.Equal(got, want) {
t.Errorf("the message ids of the take-up's ends: %v, want %v", got, want)
}
})
}
// missingRuns is runsMem whose listing misses one key, as a bucket's listing misses a key being written at that moment.
type missingRuns struct {
*runsMem
hide string
}
func (k *missingRuns) keys() ([]string, error) {
all, err := k.runsMem.keys()
return slices.DeleteFunc(all, func(key string) bool { return key == k.hide }), err
}
@@ -1,162 +0,0 @@
package main
import (
"encoding/json"
"errors"
"fmt"
"slices"
"strings"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/process"
)
// The composition runs' read and drop (design 47 G §728; A-469f, A-469u, A-469v): the seat mesh-wants' verbs `compositions`
// (class read) and `compositions-drop` (class act, §391's default callers), answered by the assigner's process, which keeps the
// runs and says the readers' ends; the module's tools `assigner_compositions` and `assigner_compositions_drop` call them.
// The tools that call the two verbs.
const (
compositionsTool = "assigner_compositions"
compositionsDropTool = "assigner_compositions_drop"
)
// compositions answers each composition run the state keeps, under §719's field names plus its key, with the candidate and the
// machine named only those, and the state's fill against its cap.
func (r *compositionRuns) compositions(in meshwants.CompositionsInput) (meshwants.CompositionsAnswer, error) {
all, err := r.keys()
if err != nil {
return meshwants.CompositionsAnswer{}, fmt.Errorf("the state %s is not listed: %w", "composition-runs", err)
}
slices.Sort(all)
out := meshwants.CompositionsAnswer{Runs: []meshwants.CompositionRun{}}
for _, key := range all {
if !isRunKey(key) {
continue
}
run, _, ok, err := r.read(key)
if err != nil || !ok || (in.Candidate != "" && run.Candidate != in.Candidate) || (in.Machine != "" && run.Machine != in.Machine) {
continue
}
run.Key = key
out.Runs = append(out.Runs, run)
}
out.Fill, err = r.fill()
return out, err
}
// drop drops one composition run by hand (§728): the act and why written in the hand-act log first (record), then a running run's
// ask dropped, each of its readers' waits stopped and, for each reader whose end is owed (no `ended`, or marked rerun), its end
// said could not run naming the drop, once (A-470r), and the key deleted with its bound keys. It answers the entry deleted and
// the readers it ended.
func (c *compositionSteps) drop(r *compositionRuns, record func(meshhandactsclient.Request) error, caller string, in meshwants.CompositionsDropInput) (meshwants.CompositionsDropAnswer, error) {
why := strings.TrimSpace(in.Why)
if in.Candidate == "" || in.Machine == "" || why == "" {
return meshwants.CompositionsDropAnswer{}, errors.New("name the candidate, the machine and why the composition run is dropped")
}
key := runKeyOf(in.Candidate, in.Machine)
run, _, ok, err := r.read(key)
if err != nil {
return meshwants.CompositionsDropAnswer{}, err
}
if !ok {
return meshwants.CompositionsDropAnswer{}, fmt.Errorf("no composition run of %s on %s is kept", in.Candidate, in.Machine)
}
if err := record(meshhandactsclient.Request{What: meshwants.VerbCompositionsDrop + " " + key, Why: why + " (" + caller + ")", Cause: "compositions-drop"}); err != nil {
return meshwants.CompositionsDropAnswer{}, fmt.Errorf("the hand act was not recorded, so nothing was dropped: %w", err)
}
said := fmt.Sprintf("%s: the composition run %s was dropped by hand: %s", in.Machine, key, why)
answer := meshwants.CompositionsDropAnswer{Run: run}
answer.Run.Key = key
if run.Ended == nil {
if r.drop != nil {
if err := r.drop(run.Ask, said); err != nil {
r.logf("the drop of the run ask %s of the composition run %s is not answered: %v", run.Ask, key, err)
}
}
ids := make([]string, 0, len(run.Readers))
for _, reader := range run.Readers {
ids = append(ids, reader.ID)
}
c.leave(ids) // their waits say no end of their own
// The run ends without moving its readers: the drop says each owed end itself, once (A-470r).
if _, err := r.endRunMoving(key, run.Ask, meshwants.CompositionRunOutcomeCouldNotRun, said, nil, r.untaken(key, run.Ask), false); err != nil {
return answer, err
}
shared := []meshwants.Shared{{Machine: run.Machine, Run: key, Ask: run.Ask}}
for _, reader := range run.Readers {
if reader.Ended != nil && !reader.Rerun {
continue // it ended passed already and keeps its verdict (A-470r)
}
again := "" // a reader whose step said its end already says it again under the dropped run's ask (A-470q)
if reader.Ended != nil {
again = run.Ask
}
c.sayEnd(reader.ID, stepRunOf(reader), nil, stepOutcome{Outcome: meshwants.StepEndedOutcomeCouldNotRun, Said: said, Shared: shared}, again)
answer.Ended = append(answer.Ended, reader)
}
}
// The dropped run's key goes while it holds that run: a run started on it since stands.
err = r.deleteRun(key, func(c meshwants.CompositionRun) bool { return c.Ask == run.Ask })
r.heard()
return answer, err
}
// compositionVerbs are the process's handlers of the seat mesh-wants' verbs compositions and compositions-drop.
func compositionVerbs(r *compositionRuns, c *compositionSteps, record func(meshhandactsclient.Request) error) map[string]process.SeatHandler {
return map[string]process.SeatHandler{
meshwants.VerbCompositions: func(_ string, raw json.RawMessage) (any, error) {
var in meshwants.CompositionsInput
if len(raw) > 0 {
if err := json.Unmarshal(raw, &in); err != nil {
return nil, fmt.Errorf("the arguments are not JSON: %w", err)
}
}
return r.compositions(in)
},
meshwants.VerbCompositionsDrop: func(caller string, raw json.RawMessage) (any, error) {
var in meshwants.CompositionsDropInput
if err := json.Unmarshal(raw, &in); err != nil {
return nil, fmt.Errorf("the arguments are not JSON: %w", err)
}
return c.drop(r, record, caller, in)
},
}
}
// compositionsTools are the module's tools of §728's names, each calling its verb (A-469f).
func compositionsTools(ask func(string, any) (json.RawMessage, error)) []stdio.Tool {
call := func(verb string, args map[string]any, names ...string) (any, error) {
in := map[string]any{}
for _, n := range names {
if v := arg(args, n); v != "" {
in[n] = v
}
}
raw, err := ask("seat:"+meshwants.Seat+"."+verb, in)
if err != nil {
return nil, err
}
var out any
if err := json.Unmarshal(raw, &out); err != nil {
return nil, fmt.Errorf("%s answered what is not JSON: %w", verb, err)
}
return out, nil
}
return []stdio.Tool{
{Name: compositionsTool, Description: "The composition runs the assigner keeps, one per candidate and machine (design 47 G §719, §728): each with " +
"its program, input digest, ask, readers, outcome and verdict, and the state's fill against its cap; with candidate or machine, only those. Read-only.",
Input: schema(map[string]string{"candidate": "the candidate's key: <branch>.<revision>, or run-<id>", "machine": "a machine"}, nil),
Run: func(args map[string]any) (any, error) {
return call(meshwants.VerbCompositions, args, "candidate", "machine")
}},
{Name: compositionsDropTool, Description: "Drop one composition run by hand (design 47 G §728): written in the hand-act log first, then its ask " +
"is dropped, each reader ended could not run naming the drop, and the key deleted.",
Input: schema(map[string]string{"candidate": "the candidate's key", "machine": "the machine", "why": "why the run is dropped"}, []string{"candidate", "machine", "why"}),
Run: func(args map[string]any) (any, error) {
return call(meshwants.VerbCompositionsDrop, args, "candidate", "machine", "why")
}},
}
}
@@ -1,91 +0,0 @@
package main
import (
"encoding/json"
"errors"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
)
// Design 47 G §728, A-469f, A-469u, A-469v: compositions answers each run with its key and the fill, only those of a candidate or
// machine when named; compositions-drop writes the hand act first (nothing is dropped when it is not recorded), drops a running
// run's ask, ends each reader could not run naming the drop and deletes the key with its bound keys; the tools call the verbs.
func TestTheCompositionRunsAreReadAndDroppedByTheirVerbs(t *testing.T) {
now := time.Date(2026, 11, 17, 12, 0, 0, 0, time.UTC)
kv := newRunsMem()
r := newCompositionRuns(kv, defaultRunSettings, func() time.Time { return now }, t.Logf)
var drops []string
r.drop = func(ask, why string) error { drops = append(drops, ask); return nil }
ends := newSaidEvents()
var endIDs []string // the message id of each end said
c := &compositionSteps{node: "n1", now: func() time.Time { return now }, logf: t.Logf, say: func(event string, body any, id string) error {
endIDs = append(endIDs, id)
return ends.say(event, body, id)
}}
moves := 0
move := func(meshwants.CompositionRun) { moves++ }
r.moved.Store(&move)
for _, m := range []string{"n1", "n2"} {
if _, err := r.join(readerOf("ask-a", meshwants.StepRun{Pipeline: "main.3.a", Step: meshwants.StepRunStepComposition, Module: "a"}), "main.3", m, "run-"+m); err != nil {
t.Fatal(err)
}
}
// A-470r: n2's run carries a reader that ended passed (p) and one marked rerun (q); only q's end is owed, said once.
_, _, _ = r.change(runKeyOf("main.3", "n2"), func(run *meshwants.CompositionRun) bool {
for _, id := range []string{"ask-p", "ask-q"} {
c := readerOf(id, meshwants.StepRun{Pipeline: "main.3." + id, Step: meshwants.StepRunStepComposition, Module: id})
c.Ended, c.Rerun = &now, id == "ask-q"
run.Readers = append(run.Readers, c)
}
return true
})
var recorded []meshhandactsclient.Request
record := func(q meshhandactsclient.Request) error { recorded = append(recorded, q); return nil }
verbs := compositionVerbs(r, c, record)
out, err := verbs[meshwants.VerbCompositions]("op", json.RawMessage(`{"machine":"n2"}`))
read, _ := out.(meshwants.CompositionsAnswer)
if err != nil || len(read.Runs) != 1 || read.Runs[0].Key != runKeyOf("main.3", "n2") || read.Runs[0].Ask != "run-n2" || len(read.Runs[0].Readers) != 3 ||
read.Fill.Store != "composition-runs" || read.Fill.Cap != 16<<20 || read.Fill.Bytes == 0 {
t.Fatalf("compositions = %+v %v", out, err)
}
refuse := compositionVerbs(r, c, func(meshhandactsclient.Request) error { return errors.New("no holder") })
if _, err := refuse[meshwants.VerbCompositionsDrop]("op", json.RawMessage(`{"candidate":"main.3","machine":"n2","why":"stuck"}`)); err == nil ||
!strings.Contains(err.Error(), "nothing was dropped") || len(drops) != 0 {
t.Fatalf("a drop not recorded: %v, drops %v", err, drops)
}
out, err = verbs[meshwants.VerbCompositionsDrop]("op@n1", json.RawMessage(`{"candidate":"main.3","machine":"n2","why":"stuck"}`))
dropped, _ := out.(meshwants.CompositionsDropAnswer)
if err != nil || dropped.Run.Key != runKeyOf("main.3", "n2") || len(dropped.Ended) != 2 || dropped.Ended[0].ID != "ask-a" || dropped.Ended[1].ID != "ask-q" ||
moves != 0 || len(endIDs) != 2 || endIDs[0] != "ask-a" || endIDs[1] != "ask-q.run-n2" ||
len(recorded) != 1 || recorded[0].Why != "stuck (op@n1)" || len(drops) != 1 || drops[0] != "run-n2" {
t.Fatalf("compositions-drop = %+v %v, recorded %+v, drops %v, moves %d, ends %v", out, err, recorded, drops, moves, endIDs)
}
end, _ := ends.said[meshwants.EventStepEnded+".n1"].(meshwants.StepEnded)
if end.ID != "ask-q" || end.Outcome != meshwants.StepEndedOutcomeCouldNotRun || !strings.Contains(end.Said, "dropped by hand: stuck") ||
len(end.Shared) != 1 || end.Shared[0].Ask != "run-n2" {
t.Errorf("the reader's end %+v", end)
}
if keys, _ := kv.keys(); len(keys) != 2 || keys[0] != runKeyOf("main.3", "n1") {
t.Errorf("keys after the drop %v, want n1's run and its untaken key", keys)
}
// The tools call the verbs with what they were given.
var asked []string
tools := compositionsTools(func(key string, body any) (json.RawMessage, error) {
raw, _ := json.Marshal(body)
asked = append(asked, key+" "+string(raw))
return json.RawMessage(`{"runs":[],"fill":{"store":"composition-runs","bytes":0,"cap":1}}`), nil
})
for _, tool := range tools {
if _, err := tool.Run(map[string]any{"candidate": "main.3", "machine": "n1", "why": "x"}); err != nil {
t.Fatal(err)
}
}
if len(tools) != 2 || tools[0].Name != compositionsTool || tools[1].Name != compositionsDropTool ||
asked[0] != `seat:mesh-wants.compositions {"candidate":"main.3","machine":"n1"}` || asked[1] != `seat:mesh-wants.compositions-drop {"candidate":"main.3","machine":"n1","why":"x"}` {
t.Errorf("the tools asked %v", asked)
}
}
@@ -23,8 +23,6 @@ type stepOutcome struct {
Outcome string
Said string
Warnings []string
// Shared names per machine composed the composition run the step read and its ask (design 47 G §723, §729).
Shared []meshwants.Shared
}
// compositionSteps takes the composition steps and says their ends.
@@ -32,21 +30,14 @@ type compositionSteps struct {
node string
// say says one event of the seat mesh-wants (wantsPublisher).
say publisher
// run runs one step and answers what it came to; resume judges a step taken up at the start from its runs (takeUp).
run func(ctx context.Context, id string, run meshwants.StepRun) stepOutcome
resume func(ctx context.Context, id string, run meshwants.StepRun) stepOutcome
// ended writes, once the step asked under id said its end, that time on its reader in each run naming it (A-470m); nil
// writes none.
ended func(id string, run meshwants.StepRun, at time.Time)
now func() time.Time
logf func(string, ...any)
// run runs one step and answers what it came to.
run func(ctx context.Context, id string, run meshwants.StepRun) stepOutcome
now func() time.Time
logf func(string, ...any)
// asker is the name the step's progress names as its asker, read when it is said (stepAskerOf).
asker func() string
// ticks is a ticker of every d, its channel and its stop (time.NewTicker); nil is the clock's. A test hands its own.
ticks func(d time.Duration) (<-chan time.Time, func())
mu sync.Mutex
// stops ends the wait of a running step by its ask's id, when its pipeline is superseded or cancelled (leave).
stops map[string]context.CancelFunc
}
// stepAskerOf is the asker a composition step's progress names: the one caller the seat mesh-wants' definition lists for its
@@ -78,27 +69,10 @@ func stepAskerOf(definition func() (seatdef.Definition, bool)) func() string {
// progressEvery is how often a running composition step says its progress (design 47 G §496, A-262).
const progressEvery = time.Minute
// stepPhase is the phase a running step is in: fetch, build, run or report (§496), set by the run through its context, and the
// composition runs it waits on (§723, §729).
// stepPhase is the phase a running step is in: fetch, build, run or report (§496), set by the run through its context.
type stepPhase struct {
mu sync.Mutex
phase string
shared []meshwants.Shared
}
// setShared names the runs the step ctx runs waits on, when it runs one.
func setShared(ctx context.Context, shared []meshwants.Shared) {
if p, ok := ctx.Value(phaseKey{}).(*stepPhase); ok {
p.mu.Lock()
p.shared = shared
p.mu.Unlock()
}
}
func (p *stepPhase) waitsOn() []meshwants.Shared {
p.mu.Lock()
defer p.mu.Unlock()
return p.shared
mu sync.Mutex
phase string
}
type phaseKey struct{}
@@ -126,97 +100,23 @@ func (c *compositionSteps) take(ctx context.Context, id string, data []byte) boo
c.logf("a step ask that is no composition step with an id is left: %v", err)
return false
}
go c.say1(ctx, id, run, c.run, "")
go c.runOne(ctx, id, run)
return true
}
// say1 runs one step with how and says its end, naming in `shared` the runs it read (§723); again is, for an end said again, the
// ask of the run whose end makes it say again (sayEnd). A step whose wait was ended by its pipeline's supersede or cancel (leave)
// says nothing: the runner ended it already.
func (c *compositionSteps) say1(ctx context.Context, id string, run meshwants.StepRun, how func(context.Context, string, meshwants.StepRun) stepOutcome, again string) {
// runOne runs one step and says its end.
func (c *compositionSteps) runOne(ctx context.Context, id string, run meshwants.StepRun) {
started := c.now().UTC()
ctx, stopWait := context.WithCancel(ctx)
c.mu.Lock()
if c.stops == nil {
c.stops = map[string]context.CancelFunc{}
}
if c.stops[id] != nil { // the step asked under id is running already: it says its own end
c.mu.Unlock()
stopWait()
return
}
c.stops[id] = stopWait
c.mu.Unlock()
defer func() {
c.mu.Lock()
delete(c.stops, id)
c.mu.Unlock()
stopWait()
}()
phase := &stepPhase{phase: meshwants.StepProgressEventPhaseFetch}
stop := c.sayProgress(id, phase)
o := how(context.WithValue(ctx, phaseKey{}, phase), id, run)
o := c.run(context.WithValue(ctx, phaseKey{}, phase), id, run)
stop()
if ctx.Err() != nil {
c.logf("the composition step %s of %s says no end: its wait ended (%v)", id, run.Pipeline, context.Cause(ctx))
return
}
c.sayEnd(id, run, &started, o, again)
}
// sayEnd says the end of the step asked under id as the seat's step-ended.<node>, its id the ask's; once said, its time is
// written as its reader's `ended` (A-470m). Its first end's message id is the ask's id; an end said again is published under
// `<ask id>.<again>`, again the ask of the run whose end makes it say again, so the bus's deduplication never drops it (A-470q).
func (c *compositionSteps) sayEnd(id string, run meshwants.StepRun, started *time.Time, o stepOutcome, again string) {
ended := c.now().UTC()
if started == nil {
started = &ended
}
end := meshwants.StepEnded{V: 1, ID: id, Pipeline: run.Pipeline, Step: meshwants.StepEndedStepComposition, Module: run.Module,
Commit: run.Commit, Fingerprint: run.Fingerprint, Node: c.node, Outcome: o.Outcome, Said: o.Said, Warnings: o.Warnings,
Rerun: run.Rerun, By: run.By, Started: started, Ended: &ended, Shared: o.Shared}
msgID := id
if again != "" {
msgID = id + "." + again
}
if err := c.say(meshwants.EventStepEnded+"."+c.node, end, msgID); err != nil {
Rerun: run.Rerun, By: run.By, Started: &started, Ended: &ended}
if err := c.say(meshwants.EventStepEnded+"."+c.node, end, id); err != nil {
c.logf("saying the end of the composition step %s: %v", id, err)
return
}
if c.ended != nil {
c.ended(id, run, ended)
}
}
// move says the end again of each reader a rerun carried into an ended run, marked rerun (§727, A-470d): its machines judged
// from the candidate's keys naming it, the run's new ask in `shared`. A reader running here says its own end.
func (c *compositionSteps) move(ctx context.Context) func(run meshwants.CompositionRun) {
return func(run meshwants.CompositionRun) {
for _, reader := range run.Readers {
if !reader.Rerun || c.running(reader.ID) {
continue
}
c.logf("the composition step %s of %s moves with the composition run of %s (ask %s)", reader.ID, reader.Pipeline, run.Machine, run.Ask)
go c.say1(ctx, reader.ID, stepRunOf(reader), c.resume, run.Ask) // said again under the moved run's ask (A-470q)
}
}
}
// running is whether the step asked under id runs here.
func (c *compositionSteps) running(id string) bool {
c.mu.Lock()
defer c.mu.Unlock()
return c.stops[id] != nil
}
// leave ends the waits of the steps asked under ids (§723: a superseded or cancelled step leaves its runs).
func (c *compositionSteps) leave(ids []string) {
c.mu.Lock()
defer c.mu.Unlock()
for _, id := range ids {
if stop := c.stops[id]; stop != nil {
stop()
}
}
}
@@ -239,7 +139,7 @@ func (c *compositionSteps) sayProgress(id string, phase *stepPhase) (stop func()
return
case <-tick:
at := c.now().UTC()
p := meshwants.StepProgressEvent{Ask: id, Asker: c.asker(), Phase: phase.get(), At: at, Shared: phase.waitsOn()}
p := meshwants.StepProgressEvent{Ask: id, Asker: c.asker(), Phase: phase.get(), At: at}
if err := c.say(meshwants.EventStepProgress+"."+c.node, p, id+".progress."+at.Format(time.RFC3339Nano)); err != nil {
c.logf("saying the progress of the composition step %s: %v", id, err)
}
@@ -146,9 +146,6 @@ func TestTheCompositionStepComposesEachMachineThroughTheBuildSeat(t *testing.T)
if err != nil || len(req.Machines) != 1 {
t.Fatalf("the request of %s: %+v %v", a.ID, req, err)
}
if len(req.Seats) == 0 { // the revision names no definition: the request carries the trunk seat list's (A-467)
t.Errorf("the request of %s carries no seat definition", a.ID)
}
node := req.Machines[0]
programs[node] = a.Run.Program.Commit
m := composer.Machine{Composes: true}
@@ -168,8 +165,7 @@ func TestTheCompositionStepComposesEachMachineThroughTheBuildSeat(t *testing.T)
t.Fatal("the composition step said no end")
}
if e.ID != "step-ask-1" || e.Outcome != meshwants.StepEndedOutcomeFailed || !strings.Contains(e.Said, "n2 does not compose") ||
strings.Contains(e.Said, "n1") || len(e.Warnings) != 2 || !strings.Contains(e.Warnings[0], "n3 is away") ||
e.Warnings[1] != "n2: the composer names no culprits" {
strings.Contains(e.Said, "n1") || len(e.Warnings) != 1 || !strings.Contains(e.Warnings[0], "n3 is away") {
t.Fatalf("the step ended %+v", e)
}
if programs["n1"] != engineCommit || programs["n2"] != engineCommit {
@@ -181,24 +177,9 @@ func TestTheCompositionStepComposesEachMachineThroughTheBuildSeat(t *testing.T)
t.Error("the machine away was asked for its input")
}
}
// The run's end comes before the input's delete (A-470o, A-470p): the inputs go after the step's end, heard by their watch.
inputs := b.buckets[buckets.Inputs()]
w, err := inputs.WatchAll(b.ctx)
if err != nil {
t.Fatal(err)
}
defer func() { _ = w.Stop() }()
deadline := time.After(10 * time.Second)
for {
keys, _ := inputs.Keys(b.ctx)
if len(keys) == 0 {
break
}
select {
case <-w.Updates():
case <-deadline:
t.Fatalf("the inputs stay kept past their step-ended: %v", keys)
}
keys, _ := b.buckets[buckets.Inputs()].Keys(b.ctx)
if len(keys) != 0 {
t.Errorf("the inputs stay kept past their step-ended: %v", keys)
}
}
@@ -4,8 +4,6 @@ import (
"context"
"encoding/json"
"git.novox.be/novox/mesh-sdk/go/seatdef"
"slices"
"strings"
"sync"
"testing"
"time"
@@ -42,8 +40,7 @@ func TestTheCompositionStepIsTakenAndItsEndSaid(t *testing.T) {
c := &compositionSteps{node: "n1", say: said.say, now: func() time.Time { return at }, logf: t.Logf,
run: func(_ context.Context, id string, run meshwants.StepRun) stepOutcome {
asked = run
return stepOutcome{Outcome: meshwants.StepEndedOutcomeFailed, Said: "a and b own /etc/x", Warnings: []string{"n3 asleep"},
Shared: []meshwants.Shared{{Machine: "n1", Run: "main=2e4.n1", Ask: "run-1"}}}
return stepOutcome{Outcome: meshwants.StepEndedOutcomeFailed, Said: "a and b own /etc/x", Warnings: []string{"n3 asleep"}}
}}
other, _ := json.Marshal(map[string]any{"pipeline": "main.4.m", "step": "build", "module": "m"})
if c.take(context.Background(), "ask-0", other) {
@@ -70,8 +67,7 @@ func TestTheCompositionStepIsTakenAndItsEndSaid(t *testing.T) {
if asked.Module != "m" || end.ID != "ask-1" || end.Pipeline != "main.4.m" || end.Step != meshwants.StepEndedStepComposition ||
end.Module != "m" || end.Commit != "c0ffee" || end.Fingerprint != "fp1" || end.Node != "n1" ||
end.Outcome != meshwants.StepEndedOutcomeFailed || end.Said != "a and b own /etc/x" || len(end.Warnings) != 1 ||
!end.Rerun || end.By != "lead@n1" || end.Started == nil || end.Ended == nil ||
len(end.Shared) != 1 || end.Shared[0] != (meshwants.Shared{Machine: "n1", Run: "main=2e4.n1", Ask: "run-1"}) { // §723, §729
!end.Rerun || end.By != "lead@n1" || end.Started == nil || end.Ended == nil {
t.Fatalf("step-ended %+v", end)
}
}
@@ -109,7 +105,6 @@ func TestARunningCompositionStepSaysItsProgress(t *testing.T) {
},
run: func(ctx context.Context, _ string, _ meshwants.StepRun) stepOutcome {
setPhase(ctx, meshwants.StepProgressEventPhaseRun)
setShared(ctx, []meshwants.Shared{{Machine: "n1", Run: "main=2e4.n1", Ask: "run-1"}})
<-release
return stepOutcome{Outcome: meshwants.StepEndedOutcomePassed}
}}
@@ -127,8 +122,7 @@ func TestARunningCompositionStepSaysItsProgress(t *testing.T) {
said.mu.Lock()
p, ok := said.said[progress].(meshwants.StepProgressEvent)
said.mu.Unlock()
if !ok || p.Ask != "ask-1" || p.Asker != "mesh-pipeline-runner" || p.Phase != meshwants.StepProgressEventPhaseRun || !p.At.Equal(at) ||
len(p.Shared) != 1 || p.Shared[0].Ask != "run-1" { // the runs the step waits on (§723)
if !ok || p.Ask != "ask-1" || p.Asker != "mesh-pipeline-runner" || p.Phase != meshwants.StepProgressEventPhaseRun || !p.At.Equal(at) {
t.Fatalf("step-progress %+v", p)
}
close(release)
@@ -141,70 +135,3 @@ func TestARunningCompositionStepSaysItsProgress(t *testing.T) {
t.Fatal("the ticker is not stopped when the step has ended")
}
}
// Design 47 G §723, A-469n, A-469z: a superseded or cancelled pipeline's composition step leaves the runs it reads and says no end;
// a run another pipeline's step reads goes on for it; a run left with no reader has its ask dropped and ends could not run, "no
// reader is left", so a later step starts a new run.
func TestASupersededStepLeavesItsRuns(t *testing.T) {
now := time.Date(2026, 11, 17, 12, 0, 0, 0, time.UTC)
r := newCompositionRuns(newRunsMem(), defaultRunSettings, func() time.Time { return now }, t.Logf)
var drops []string
r.drop = func(ask, why string) error { drops = append(drops, ask+": "+why); return nil }
step := func(pipeline string) meshwants.StepRun {
return meshwants.StepRun{Pipeline: pipeline, Step: meshwants.StepRunStepComposition, Module: pipeline, Branch: "main", Revision: 3}
}
said := newSaidEvents()
c := &compositionSteps{node: "n1", say: said.say, now: func() time.Time { return now }, logf: t.Logf,
run: func(ctx context.Context, id string, s meshwants.StepRun) stepOutcome {
j, err := r.join(readerOf(id, s), "main.3", "n1", "run-"+id)
if err != nil {
t.Error(err)
}
r.await(ctx, []string{j.key})
return stepOutcome{Outcome: meshwants.StepEndedOutcomePassed}
}}
body := func(s meshwants.StepRun) []byte { raw, _ := json.Marshal(s); return raw }
for _, p := range []string{"main.3.a", "main.3.b"} {
if !c.take(context.Background(), "ask-"+p, body(step(p))) {
t.Fatal("not taken")
}
}
key := runKeyOf("main.3", "n1")
for {
wake := r.wake()
if run, _, _, _ := r.read(key); len(run.Readers) == 2 {
break
}
<-wake
}
ids, err := r.leave("main.3.a", c.leave)
run, _, _, _ := r.read(key)
if err != nil || !slices.Equal(ids, []string{"ask-main.3.a"}) || run.Ended != nil || len(run.Readers) != 1 || len(drops) != 0 {
t.Fatalf("after a's supersede: %v %v, run %+v, drops %v: want the run going on for b", ids, err, run, drops)
}
_, err = r.leave("main.3.b", c.leave)
run, _, _, _ = r.read(key)
if err != nil || run.Outcome != meshwants.CompositionRunOutcomeCouldNotRun || !strings.Contains(run.Said, "no reader is left") || len(drops) != 1 ||
!strings.HasPrefix(drops[0], run.Ask+": no reader is left") {
t.Fatalf("after b's supersede: %v, run %+v, drops %v", err, run, drops)
}
deadline := time.After(5 * time.Second)
for {
c.mu.Lock()
n := len(c.stops)
c.mu.Unlock()
if n == 0 {
break
}
select {
case <-deadline:
t.Fatal("the steps that left still wait")
case <-time.After(time.Millisecond):
}
}
said.mu.Lock()
defer said.mu.Unlock()
if _, ok := said.said[meshwants.EventStepEnded+".n1"]; ok {
t.Errorf("a step that left says its end: %v", said.said)
}
}
@@ -123,7 +123,6 @@ func hearDelivery(ctx context.Context, js jetstream.JetStream, w *world, wake fu
m, at, err := presenceOf(data)
if err == nil {
w.heardPresence(m, true, at)
logf("%s is away (machine-away): a composition step does not compose it until its machine-back", m)
}
return err
}},
@@ -131,7 +130,6 @@ func hearDelivery(ctx context.Context, js jetstream.JetStream, w *world, wake fu
m, at, err := presenceOf(data)
if err == nil {
w.heardPresence(m, false, at)
logf("%s is back (machine-back): a composition step composes it again", m)
}
return err
}},
@@ -16,7 +16,6 @@ import (
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
stdio "git.novox.be/novox/mesh-sdk/go"
@@ -25,17 +24,14 @@ import (
"git.novox.be/novox/mesh-sdk/go/keys"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/stateobjects"
"mesh-delivery-assigner/internal/buckets"
)
// A step's input by its digest (design 47 G §354, §712, A-237, A-243): the assigner keeps the input of each composition step
// request it asked a build seat to run in its state `inputs`, one key per input, its sha256 alone (A-465a), the value the input's bytes in
// request it asked a build seat to run in its state `inputs`, one key `<time>.<digest>` per input, the value the input's bytes in
// base64 with the state's object store `inputs-objects` holding one over a key's limit (mesh-sdk stateobjects), and the seat
// mesh-wants' verb `input` answers one by its digest in chunks, to the holders of the build seats alone (the verb's callers, §338).
// An input is kept once per content: every ask that names its digest shares it (A-465a). It is deleted, with its object, when the
// step-ended of the last ask that names it is heard, or a day after its last write, whichever is first, the day judged at each new
// ask over at most 20 of the oldest inputs. `assigner_inputs` reads them. This mirrors the indexer's
// An input is deleted when the build seat's step-ended of its ask is heard, or a day after its write, whichever is first, the day
// judged at each new ask over at most 20 of the oldest keys. `assigner_inputs` reads them. This mirrors the indexer's
// (mesh-delivery-indexer internal/index/run_inputs.go): the two are separate modules' code, owed to mesh-sdk.
// inputsTool is the tool that reads the kept inputs.
@@ -48,39 +44,10 @@ const (
inputsSwept = 20
inputsTime = "20060102T150405Z"
// inputsAskPrefix opens the keys that name an input by the id of the ask it was kept for: ask.<id>, the id escaped as a name,
// holding the digest of the input the ask uses, so the build seat's step-ended of that ask finds it.
// holding the input's own key, so the build seat's step-ended of that ask finds it.
inputsAskPrefix = "ask."
)
// inputAsk is what an ask's key holds (A-465b): the digest of the input the ask uses and the time the ask wrote it, in inputsTime.
type inputAsk struct {
Digest string `json:"digest"`
At string `json:"at"`
}
// readAsk reads an ask's key: the JSON form, or the bare key of the old form `<time>.<digest>` an earlier assigner wrote, which
// carries its time in the key itself.
func readAsk(raw []byte) (digest string, at time.Time) {
var a inputAsk
if json.Unmarshal(raw, &a) == nil && a.Digest != "" {
at, _ = time.Parse(inputsTime, a.At)
return a.Digest, at
}
digest = string(raw)
t, _, _ := strings.Cut(digest, ".")
at, _ = time.Parse(inputsTime, t)
return digest, at
}
// isDigest is whether a key is an input's own key: 64 hex digits.
func isDigest(key string) bool {
if len(key) != 64 {
return false
}
_, err := hex.DecodeString(key)
return err == nil
}
// digestOf is an input's digest as the ask names it: its sha256 in hex.
func digestOf(b []byte) string { sum := sha256.Sum256(b); return hex.EncodeToString(sum[:]) }
@@ -96,9 +63,8 @@ type inputKeeper struct {
used func() (int64, error)
}
// keep keeps the exact bytes of the input of the ask under their sha256, after sweeping, and names the digest and the time in the
// ask's key; it answers the digest. The same bytes asked again are the same one input (A-465a, A-465b). The ask's key is written
// first, so a sweep never finds the input unnamed.
// keep keeps the exact bytes of the input of the ask under their write time and sha256, after sweeping the keys more than a day old
// among the 20 oldest, and names it by the ask's id; it answers the digest. The same bytes written again in the same second are one key.
func (k *inputKeeper) keep(ask string, body []byte) (string, error) {
if ask == "" {
return "", errors.New("name the ask the input is kept for")
@@ -106,43 +72,29 @@ func (k *inputKeeper) keep(ask string, body []byte) (string, error) {
if err := k.sweep(); err != nil {
k.logf("inputs: the keys older than a day are not all deleted: %v", err)
}
digest := digestOf(body)
raw, err := json.Marshal(inputAsk{Digest: digest, At: k.now().UTC().Format(inputsTime)})
if err != nil {
return "", err
}
askKey := inputsAskPrefix + keys.EscapeName(ask)
if err := k.vals.KV.Create(askKey, raw); err != nil && !errors.Is(err, stateobjects.ErrChanged) {
return "", err
}
have, err := k.vals.KV.Get(digest)
if err != nil {
return "", err
}
if have != nil {
return digest, nil
}
if k.used != nil {
used, err := k.used()
if err == nil && used+int64(len(body)) > inputsCap {
err = fmt.Errorf("the state %s is at its cap of %d MiB (%d bytes kept), so the ask's input of %d bytes is refused",
buckets.InputsState, inputsCap>>20, used, len(body))
} else if err != nil {
err = fmt.Errorf("the fill of the state %s is not read, so no input is kept: %w", buckets.InputsState, err)
}
if err != nil {
_ = k.vals.KV.Delete(askKey)
return "", err
return "", fmt.Errorf("the fill of the state %s is not read, so no input is kept: %w", meshwants.StateInputs, err)
}
if used+int64(len(body)) > inputsCap {
return "", fmt.Errorf("the state %s is at its cap of %d MiB (%d bytes kept), so the ask's input of %d bytes is refused",
meshwants.StateInputs, inputsCap>>20, used, len(body))
}
}
if err := k.vals.Put(digest, base64.StdEncoding.EncodeToString(body)); err != nil && !errors.Is(err, stateobjects.ErrChanged) {
digest := digestOf(body)
key := meshwants.InputsByTimeDigestKey(k.now().UTC().Format(inputsTime), digest)
if err := k.vals.Put(key, base64.StdEncoding.EncodeToString(body)); err != nil && !errors.Is(err, stateobjects.ErrChanged) {
return "", err
}
if err := k.vals.KV.Create(inputsAskPrefix+keys.EscapeName(ask), []byte(key)); err != nil && !errors.Is(err, stateobjects.ErrChanged) {
return "", err
}
return digest, nil
}
// ended removes the ask's key when the build seat's step-ended of that ask is heard, and deletes the input it named, with its
// object, when no other ask's key names it (A-465a); an ask that kept none changes nothing.
// ended deletes the input kept for an ask when the build seat's step-ended of that ask is heard; an ask that kept none, or whose
// input a sweep deleted, changes nothing.
func (k *inputKeeper) ended(ask string) error {
if ask == "" {
return nil
@@ -152,165 +104,66 @@ func (k *inputKeeper) ended(ask string) error {
if err != nil || e == nil {
return err
}
digest, _ := readAsk(e.Value)
if err := k.vals.KV.Delete(askKey); err != nil {
if err := k.vals.KV.Delete(string(e.Value)); err != nil {
return err
}
named, err := k.named()
if err != nil {
return err
}
if named[digest] {
return nil // another running ask still names it
}
return k.deleteInput(digest)
return k.vals.KV.Delete(askKey)
}
// named is the inputs some ask's key names now.
func (k *inputKeeper) named() (map[string]bool, error) {
all, err := k.vals.KV.Keys()
if err != nil {
return nil, err
}
named := map[string]bool{}
for _, key := range all {
if !strings.HasPrefix(key, inputsAskPrefix) {
continue
}
o, err := k.vals.KV.Get(key)
if err != nil {
return nil, err
}
if o != nil {
d, _ := readAsk(o.Value)
named[d] = true
}
}
return named, nil
}
// deleteInput deletes an input's key and the object that key names, when it names one (A-465): an owner that ends its own key
// ends its object. A key already gone leaves nothing to remove here; the start and assigner_inputs_drop remove what a key left.
func (k *inputKeeper) deleteInput(key string) error {
e, err := k.vals.KV.Get(key)
if err != nil {
return err
}
if err := k.vals.KV.Delete(key); err != nil {
return err
}
if e == nil {
return nil
}
if ref, ok := stateobjects.ReadRef(e.Value); ok {
if err := k.vals.Objects.Delete(ref.Object); err != nil {
return fmt.Errorf("the object %s of %s: %w", ref.Object, key, err)
}
}
return nil
}
// dropKeyGone removes every object of the state whose key is gone and answers their count and bytes (A-465).
func (k *inputKeeper) dropKeyGone() (dropped []stateobjects.Kept, bytes int64, err error) {
kept, err := k.vals.Kept()
if err != nil {
return nil, 0, err
}
names, err := k.vals.DropKeyGone()
gone := map[string]bool{}
for _, n := range names {
gone[n] = true
}
for _, o := range kept {
if gone[o.Name] {
dropped = append(dropped, o)
bytes += o.Size
}
}
return dropped, bytes, err
}
// dropAtStart removes the objects whose key is gone at the program's start, said with their count and bytes (A-465). An object
// store not on the bus yet is said and left.
func (k *inputKeeper) dropAtStart() {
dropped, bytes, err := k.dropKeyGone()
if err != nil {
k.logf("inputs: the objects of deleted keys are not all removed at the start (%d objects, %d bytes removed): %v", len(dropped), bytes, err)
return
}
k.logf("inputs: %d objects of deleted keys removed at the start, %d bytes", len(dropped), bytes)
}
// sweep deletes, among the 20 oldest ask keys, those written more than a day ago; every input of the digest form no ask's key names
// now, with its object; a key of the old form `<time>.<digest>` a day after the time it carries; and an old-form ask key whose
// input is gone (A-465b).
// sweep deletes, among the 20 oldest keys, those written more than a day ago, and a name by ask whose input is gone.
func (k *inputKeeper) sweep() error {
all, err := k.vals.KV.Keys()
if err != nil {
return err
}
var errs []error
type askAt struct {
key string
at time.Time
}
var asks []askAt
var inputs []string
var inputs, asks []string
for _, key := range all {
if strings.HasPrefix(key, inputsAskPrefix) {
e, err := k.vals.KV.Get(key)
if err != nil || e == nil {
continue
}
d, at := readAsk(e.Value)
if !isDigest(d) {
if gone, err := k.vals.KV.Get(d); err == nil && gone == nil {
errs = append(errs, k.vals.KV.Delete(key))
}
}
asks = append(asks, askAt{key, at})
continue
}
if !isDigest(key) {
t, _, _ := strings.Cut(key, ".")
if at, err := time.Parse(inputsTime, t); err == nil && k.now().Sub(at) >= inputsKeptFor {
errs = append(errs, k.deleteInput(key))
}
continue
}
inputs = append(inputs, key)
}
slices.SortFunc(asks, func(a, b askAt) int { return a.at.Compare(b.at) })
if len(asks) > inputsSwept {
asks = asks[:inputsSwept]
}
for _, a := range asks {
if !a.at.IsZero() && k.now().Sub(a.at) >= inputsKeptFor {
errs = append(errs, k.vals.KV.Delete(a.key))
asks = append(asks, key)
} else {
inputs = append(inputs, key)
}
}
named, err := k.named()
if err != nil {
return errors.Join(append(errs, err)...)
slices.Sort(inputs) // the write time leads the key, so the oldest come first
if len(inputs) > inputsSwept {
inputs = inputs[:inputsSwept]
}
for _, key := range inputs {
if !named[key] {
errs = append(errs, k.deleteInput(key))
t, _, ok := meshwants.ParseInputsByTimeDigestKey(key)
at, perr := time.Parse(inputsTime, t)
if !ok || perr != nil || k.now().Sub(at) < inputsKeptFor {
continue
}
if err := k.vals.KV.Delete(key); err != nil {
errs = append(errs, err)
}
}
for _, key := range asks {
if e, err := k.vals.KV.Get(key); err == nil && e != nil {
if gone, err := k.vals.KV.Get(string(e.Value)); err == nil && gone == nil {
if err := k.vals.KV.Delete(key); err != nil {
errs = append(errs, err)
}
}
}
}
return errors.Join(errs...)
}
// keyOf is the key of the input of the digest, or none: the digest itself.
// keyOf is the key of the input of the digest, or none.
func (k *inputKeeper) keyOf(digest string) (string, error) {
if !isDigest(digest) {
return "", nil
}
e, err := k.vals.KV.Get(digest)
if err != nil || e == nil {
all, err := k.vals.KV.Keys()
if err != nil {
return "", err
}
return digest, nil
slices.Sort(all)
for _, key := range all {
if _, d, ok := meshwants.ParseInputsByTimeDigestKey(key); ok && d == digest && !strings.HasPrefix(key, inputsAskPrefix) {
return key, nil
}
}
return "", nil
}
// body is the exact bytes of the input of the digest, checked against it, whatever its size.
@@ -320,7 +173,7 @@ func (k *inputKeeper) body(digest string) ([]byte, error) {
}
key, err := k.keyOf(digest)
if err != nil {
return nil, fmt.Errorf("the state %s cannot be read: %w", buckets.InputsState, err)
return nil, fmt.Errorf("the state %s cannot be read: %w", meshwants.StateInputs, err)
}
if key == "" {
return nil, fmt.Errorf("the assigner keeps no input with the digest %s", digest)
@@ -388,7 +241,7 @@ func (o *inputObjects) open() (stateobjects.Objects, error) {
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
store, err := o.js.ObjectStore(ctx, catalogue.BusName(o.module, catalogue.StateObjectsName(buckets.InputsState)))
store, err := o.js.ObjectStore(ctx, catalogue.BusName(o.module, catalogue.StateObjectsName(meshwants.StateInputs)))
if errors.Is(err, jetstream.ErrBucketNotFound) || errors.Is(err, jetstream.ErrStreamNotFound) {
return nil, fmt.Errorf("the object store of the state inputs is not on the bus yet: %w", process.ErrNotYet)
}
@@ -439,7 +292,7 @@ func inputsOver(kv jetstream.KeyValue, js jetstream.JetStream, module string, lo
return nil
}
objects := &inputObjects{js: js, module: module}
v := stateobjects.Values{KV: stateobjects.BusKV{KV: kv}, Objects: objects, Store: catalogue.StateObjectsName(buckets.InputsState)}
v := stateobjects.Values{KV: stateobjects.BusKV{KV: kv}, Objects: objects, Store: catalogue.StateObjectsName(meshwants.StateInputs)}
used := func() (int64, error) {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
@@ -480,21 +333,12 @@ func inputsRead(k *inputKeeper) stdio.Tool {
return nil, fmt.Errorf("the inputs cannot be listed: %w", err)
}
slices.Sort(all)
lastAsk := map[string]string{} // the latest write time an ask's key names for each input
for _, key := range all {
if e, err := k.vals.KV.Get(key); err == nil && e != nil && strings.HasPrefix(key, inputsAskPrefix) {
if d, at := readAsk(e.Value); !at.IsZero() && at.Format(inputsTime) > lastAsk[d] {
lastAsk[d] = at.Format(inputsTime)
}
}
}
out := []inputSaid{}
for _, key := range all {
d := key
if !isDigest(key) || (want != "" && d != want) {
t, d, ok := meshwants.ParseInputsByTimeDigestKey(key)
if !ok || strings.HasPrefix(key, inputsAskPrefix) || (want != "" && d != want) {
continue
}
t := lastAsk[key]
var text string
if found, err := k.vals.Get(key, &text); err != nil || !found {
continue
@@ -508,74 +352,26 @@ func inputsRead(k *inputKeeper) stdio.Tool {
}
out = append(out, said)
}
gone := []map[string]any{}
var goneBytes int64
answer := map[string]any{"inputs": out}
kept, err := k.vals.Kept()
if err != nil {
// never an empty list for objects that could not be read (A-465)
answer["key-gone-error"] = fmt.Sprintf("the objects of the state inputs cannot be listed, so key-gone objects are not known: %v", err)
return answer, nil
}
for _, o := range kept {
if o.KeyGone {
gone = append(gone, map[string]any{"object": o.Name, "key": o.Key, "size": o.Size})
goneBytes += o.Size
}
}
answer["key-gone"], answer["key-gone-bytes"] = gone, goneBytes
return answer, nil
return map[string]any{"inputs": out}, nil
}}
}
// inputsDropTool is the tool that removes the objects whose key is gone.
const inputsDropTool = "assigner_inputs_drop"
// inputsDrop is the tool assigner_inputs_drop (A-465): a hand act, recorded first, that removes the objects of the state inputs
// whose key is gone now and answers what it removed.
func inputsDrop(k *inputKeeper, ask func(string, any) (json.RawMessage, error)) stdio.Tool {
return stdio.Tool{Name: inputsDropTool, Description: "Remove the objects of the assigner's state inputs whose key is gone now (design 47 G A-465), " +
"and answer which and how many bytes. An object a key names is never removed. Recorded as a hand act; assigner_inputs lists them as key-gone.",
Input: schema(map[string]string{"why": "why the objects of deleted keys are dropped now"}, []string{"why"}), Run: func(args map[string]any) (any, error) {
why := strings.TrimSpace(arg(args, "why"))
if why == "" {
return nil, errors.New("say why the objects of deleted keys are dropped")
}
taken, err := meshhandactsclient.Recorder{Ask: ask}.Record(meshhandactsclient.Request{What: inputsDropTool, Why: why + " (" + stdio.CallerOf(args) + ")", Cause: "inputs-drop"})
if err != nil {
return nil, fmt.Errorf("the hand act was not recorded, so nothing was removed: %w", err)
}
dropped, bytes, err := k.dropKeyGone()
out := []map[string]any{}
for _, o := range dropped {
out = append(out, map[string]any{"object": o.Name, "key": o.Key, "size": o.Size})
}
return map[string]any{"dropped": out, "count": len(out), "bytes": bytes, "hand-act": taken.Log}, err
}}
}
// heardStepEnded hands the end a build seat's step-ended names to the composition run of that ask (A-240, §719), which writes it
// in the run's key, and then deletes the input kept for the ask (design 47 G §354): the run's end first, so no other act delays
// it past its bound (A-470o, A-470p). One whose run's end cannot be written, or whose input cannot be deleted, is given again by
// the bus; its redelivery finds the run ended, said as ignored for the run, and still deletes the input.
func heardStepEnded(k *inputKeeper, runEnded func(buildstep.StepEnded) error, m jetstream.Msg, logf func(string, ...any)) {
// heardStepEnded deletes the input kept for the ask a build seat's step-ended names (design 47 G §354), and hands the end to the
// composition run that waits for that ask (A-240); one whose input cannot be deleted is given again by the bus.
func heardStepEnded(k *inputKeeper, ends *runEnds, m jetstream.Msg, logf func(string, ...any)) {
var e buildstep.StepEnded
if err := json.Unmarshal(m.Data(), &e); err != nil {
logf("%s holds what is no step-ended, left: %v", m.Subject(), err)
_ = m.Ack()
return
}
if runEnded != nil {
if err := runEnded(e); err != nil {
logf("the composition run of the ask %s is not ended, so the bus gives its step-ended again: %v", e.Ask, err)
_ = m.Nak()
return
}
}
if err := k.ended(e.Ask); err != nil {
logf("inputs: the input of the ask %s could not be deleted, so the bus gives it again: %v", e.Ask, err)
_ = m.Nak()
return
}
if ends != nil {
ends.heard(e)
}
_ = m.Ack()
}
@@ -3,10 +3,8 @@ package main
import (
"bytes"
"crypto/rand"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"sort"
"strings"
"sync"
@@ -16,7 +14,6 @@ import (
"github.com/nats-io/nats.go"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-sdk/go/buscalls"
"git.novox.be/novox/mesh-sdk/go/stateobjects"
)
@@ -25,8 +22,6 @@ type kvMem struct {
mu sync.Mutex
vals map[string][]byte
revs map[string]uint64
// delErr, when set, fails the next Delete once.
delErr error
}
func newKVMem() *kvMem { return &kvMem{vals: map[string][]byte{}, revs: map[string]uint64{}} }
@@ -61,10 +56,6 @@ func (k *kvMem) Update(key string, v []byte, rev uint64) error {
func (k *kvMem) Delete(key string) error {
k.mu.Lock()
defer k.mu.Unlock()
if err := k.delErr; err != nil {
k.delErr = nil
return err
}
delete(k.vals, key)
return nil
}
@@ -80,9 +71,8 @@ func (k *kvMem) Keys() ([]string, error) {
}
type objectsMem struct {
mu sync.Mutex
objs map[string][]byte
listErr error // List fails with it
mu sync.Mutex
objs map[string][]byte
}
func (*objectsMem) MostBytes() int { return 0 }
@@ -101,24 +91,8 @@ func (o *objectsMem) Get(name string) ([]byte, error) {
}
return d, nil
}
func (o *objectsMem) Delete(name string) error {
o.mu.Lock()
defer o.mu.Unlock()
delete(o.objs, name)
return nil
}
func (o *objectsMem) List() ([]stateobjects.Object, error) {
o.mu.Lock()
defer o.mu.Unlock()
if o.listErr != nil {
return nil, o.listErr
}
var out []stateobjects.Object
for n, d := range o.objs {
out = append(out, stateobjects.Object{Name: n, Size: int64(len(d))})
}
return out, nil
}
func (o *objectsMem) Delete(string) error { return nil }
func (o *objectsMem) List() ([]stateobjects.Object, error) { return nil, nil }
// lineMem is a core bus in memory: a chunk published is kept, and its acknowledgement heard at once.
type lineMem struct {
@@ -254,250 +228,4 @@ func TestAnInputIsDeletedAtItsStepEndedOrAfterADay(t *testing.T) {
if _, err := k.body(d2); err == nil {
t.Error("an input a day old is still kept at the next ask")
}
// A-470o, A-470p: the step-ended's run end is written before the input is deleted; a delete that fails asks the event again,
// whose redelivery finds the run ended and still deletes the input; a run end not written asks it again with the input kept.
for _, c := range []struct {
name string
delErr error
runErr error
naked bool
runs int
deleted bool
}{
{name: "the run's end before the input's delete", runs: 1, deleted: true},
{name: "an input not deleted asks the event again", delErr: errors.New("the bus is away"), naked: true, runs: 2, deleted: true},
{name: "a run's end not written asks the event again", runErr: errors.New("the run is not read"), naked: true, runs: 1},
} {
t.Run(c.name, func(t *testing.T) {
k := testKeeper(&objectsMem{objs: map[string][]byte{}}, time.Now)
d, _ := k.keep("ask/run", []byte(`{"heads":"abc"}`))
k.vals.KV.(*kvMem).delErr = c.delErr
runs := 0
runEnded := func(e buildstep.StepEnded) error {
runs++
if _, err := k.body(d); err != nil {
t.Errorf("the run's end is written after the input's delete (%d)", runs)
}
return c.runErr
}
raw, _ := json.Marshal(buildstep.StepEnded{Ask: "ask/run"})
m := &heardMsg{data: raw, subject: "step-ended"}
heardStepEnded(k, runEnded, m, t.Logf)
if m.naked != c.naked || m.acked == c.naked {
t.Fatalf("acked %v, naked %v, want naked %v", m.acked, m.naked, c.naked)
}
if m.naked && c.delErr != nil { // the redelivery
m = &heardMsg{data: raw, subject: "step-ended"}
heardStepEnded(k, func(e buildstep.StepEnded) error { runs++; return nil }, m, t.Logf)
}
if _, err := k.body(d); runs != c.runs || (err != nil) != c.deleted {
t.Errorf("the run's end heard %d times (want %d), the input deleted %v (want %v)", runs, c.runs, err != nil, c.deleted)
}
})
}
}
// Design 47 G A-465: an input the assigner ends itself (the step-ended of its ask, the sweep of a day-old key) ends its object
// too; the objects of keys another part deleted are removed by dropKeyGone and by the tool assigner_inputs_drop, which lists
// them as key-gone first, and never removes an object a key names.
func TestAnEndedInputEndsItsObjectAndTheDropToolRemovesWhatAKeyLeft(t *testing.T) {
now := time.Date(2026, 11, 17, 10, 0, 0, 0, time.UTC)
objs := &objectsMem{objs: map[string][]byte{}}
k := testKeeper(objs, func() time.Time { return now })
big := func(b byte) []byte { return bytes.Repeat([]byte{b}, 2<<20) }
if _, err := k.keep("ask-one", big('a')); err != nil {
t.Fatal(err)
}
if _, err := k.keep("ask-two", big('b')); err != nil {
t.Fatal(err)
}
if len(objs.objs) != 2 {
t.Fatalf("objects = %d, want 2", len(objs.objs))
}
if err := k.ended("ask-one"); err != nil || len(objs.objs) != 1 {
t.Fatalf("the step-ended left %d objects (%v), want 1", len(objs.objs), err)
}
now = now.Add(25 * time.Hour)
if _, err := k.keep("ask-three", big('c')); err != nil {
t.Fatal(err)
}
if len(objs.objs) != 1 {
t.Fatalf("after the sweep of the day-old input %d objects, want only the new one", len(objs.objs))
}
// An object whose key another part deleted is listed as key-gone and removed by the tool; a live one stays.
keys, _ := k.vals.KV.Keys()
for _, key := range keys {
if !strings.HasPrefix(key, inputsAskPrefix) {
if err := k.vals.KV.Delete(key); err != nil {
t.Fatal(err)
}
}
}
live := []byte(`{"heads":"x"}`)
if _, err := k.keep("ask-four", append(live, make([]byte, 2<<20)...)); err != nil {
t.Fatal(err)
}
list, err := inputsRead(k).Run(map[string]any{})
if err != nil || len(list.(map[string]any)["key-gone"].([]map[string]any)) != 1 {
t.Fatalf("assigner_inputs key-gone = %+v %v, want the one object of the deleted key", list, err)
}
ask := func(string, any) (json.RawMessage, error) {
return json.RawMessage(`{"answer":{"log":"hand-acts"}}`), nil
}
if _, err := inputsDrop(k, ask).Run(map[string]any{}); err == nil {
t.Error("dropped with no reason")
}
out, err := inputsDrop(k, ask).Run(map[string]any{"why": "a key left its object"})
if err != nil || out.(map[string]any)["count"] != 1 || len(objs.objs) != 1 {
t.Fatalf("assigner_inputs_drop = %+v %v, %d objects left: want one removed and the live one kept", out, err, len(objs.objs))
}
// The start drops what a key left, said with its count and bytes.
if _, err := k.keep("ask-five", live); err != nil {
t.Fatal(err)
}
for _, key := range keysOf(t, k) {
if !strings.HasPrefix(key, inputsAskPrefix) {
_ = k.vals.KV.Delete(key)
}
}
var logged []string
k.logf = func(f string, a ...any) { logged = append(logged, fmt.Sprintf(f, a...)) }
before := len(objs.objs)
k.dropAtStart()
if len(objs.objs) != 0 || before == 0 || len(logged) != 1 || !strings.Contains(logged[0], fmt.Sprintf("%d objects of deleted keys removed at the start", before)) || !strings.Contains(logged[0], " bytes") {
t.Errorf("the start left %d of %d objects and said %q: want every key-gone object removed and its count and bytes said", len(objs.objs), before, logged)
}
}
func keysOf(t *testing.T, k *inputKeeper) []string {
t.Helper()
keys, err := k.vals.KV.Keys()
if err != nil {
t.Fatal(err)
}
return keys
}
// Design 47 G A-465: assigner_inputs says why key-gone objects are not known when the objects cannot be listed, and never
// answers them as none.
func TestAssignerInputsSaysWhenTheKeyGoneObjectsCannotBeListed(t *testing.T) {
objs := &objectsMem{objs: map[string][]byte{}, listErr: errors.New("the object store does not answer")}
k := testKeeper(objs, time.Now)
out, err := inputsRead(k).Run(map[string]any{})
if err != nil {
t.Fatal(err)
}
m := out.(map[string]any)
if msg, _ := m["key-gone-error"].(string); !strings.Contains(msg, "does not answer") {
t.Errorf("answer %+v: want key-gone-error naming the failure", m)
}
if _, has := m["key-gone"]; has {
t.Errorf("answer %+v: an unreadable store is not answered as no key-gone objects", m)
}
}
// Design 47 G A-465a: an input is kept once per content under its sha256 alone; every ask names the digest it uses, and the input
// goes, with its object, at the step-ended of the last ask that names it, or a day after its last write; a request the cap still
// refuses says the fill; an input of the old form (a key that is no digest) is ended with its object.
func TestAnInputIsKeptOncePerContentAndEndsWithItsLastAsk(t *testing.T) {
now := time.Date(2026, 11, 17, 10, 0, 0, 0, time.UTC)
objs := &objectsMem{objs: map[string][]byte{}}
k := testKeeper(objs, func() time.Time { return now })
body := bytes.Repeat([]byte{'n'}, 2<<20)
var digests []string
for _, ask := range []string{"ask-1", "ask-2", "ask-3"} {
d, err := k.keep(ask, body)
if err != nil {
t.Fatal(err)
}
digests = append(digests, d)
}
if digests[0] != digestOf(body) || digests[1] != digests[0] || digests[2] != digests[0] {
t.Fatalf("digests = %v, want the sha256 of the bytes each time", digests)
}
if _, err := k.vals.KV.Get(digests[0]); err != nil {
t.Fatal(err)
}
if e, _ := k.vals.KV.Get(digests[0]); e == nil || len(objs.objs) != 1 {
t.Fatalf("one key by the digest and one object wanted, got entry %v and %d objects", e, len(objs.objs))
}
for _, key := range keysOf(t, k) {
if !strings.HasPrefix(key, inputsAskPrefix) && key != digests[0] {
t.Errorf("a key %q beside the one input", key)
}
}
if e, _ := k.vals.KV.Get(inputsAskPrefix + "ask-2"); e == nil || !strings.Contains(string(e.Value), digests[0]) || !strings.Contains(string(e.Value), "20261117T100000Z") {
t.Errorf("the ask does not name the digest it uses: %v", e)
}
for _, ask := range []string{"ask-1", "ask-2"} {
if err := k.ended(ask); err != nil {
t.Fatal(err)
}
if _, err := k.body(digests[0]); err != nil {
t.Fatalf("the input went at the end of %s while ask-3 still names it: %v", ask, err)
}
}
if err := k.ended("ask-3"); err != nil {
t.Fatal(err)
}
if _, err := k.body(digests[0]); err == nil || len(objs.objs) != 0 || len(keysOf(t, k)) != 0 {
t.Errorf("the last step-ended left the input (%v), %d objects, keys %v", err, len(objs.objs), keysOf(t, k))
}
// a day after its last write, a renewed write counted
if _, err := k.keep("ask-4", body); err != nil {
t.Fatal(err)
}
now = now.Add(20 * time.Hour)
if _, err := k.keep("ask-5", body); err != nil {
t.Fatal(err)
}
now = now.Add(20 * time.Hour)
if _, err := k.keep("ask-6", []byte("other")); err != nil {
t.Fatal(err)
}
if _, err := k.body(digests[0]); err != nil {
t.Errorf("an input written again 20 hours ago was swept: %v", err)
}
now = now.Add(25 * time.Hour)
if _, err := k.keep("ask-7", []byte("another")); err != nil {
t.Fatal(err)
}
if _, err := k.body(digests[0]); err == nil || len(objs.objs) != 0 {
t.Errorf("an input a day after its last write is kept (%v), %d objects", err, len(objs.objs))
}
// an input of the old form goes a day after the time its key carries, with its object, never earlier
old := "20261117T090000Z." + digestOf([]byte("old"))
if err := k.vals.Put(old, base64.StdEncoding.EncodeToString(bytes.Repeat([]byte{'o'}, 2<<20))); err != nil {
t.Fatal(err)
}
recent := now.Add(-time.Hour).Format(inputsTime) + "." + digestOf([]byte("recent"))
if err := k.vals.Put(recent, "cmVjZW50"); err != nil {
t.Fatal(err)
}
if err := k.sweep(); err != nil {
t.Fatal(err)
}
if e, _ := k.vals.KV.Get(old); e != nil || len(objs.objs) != 0 {
t.Errorf("the input of the old form is kept (%v) with %d objects", e, len(objs.objs))
}
if e, _ := k.vals.KV.Get(recent); e == nil {
t.Error("an input of the old form under a day old was deleted early")
}
// a new content the cap refuses says the fill; the same content, already kept, needs no room
full := newTestKeeperWithFill(objs, now, inputsCap-10)
if _, err := full.keep("ask-8", []byte("fits")); err != nil {
t.Fatalf("a content that fits is refused: %v", err)
}
if _, err := full.keep("ask-9", []byte("does not fit at all")); err == nil || !strings.Contains(err.Error(), "bytes kept") {
t.Errorf("the refusal does not name the fill: %v", err)
}
if _, err := full.keep("ask-10", []byte("fits")); err != nil {
t.Errorf("the content already kept is refused at the cap: %v", err)
}
}
func newTestKeeperWithFill(objs *objectsMem, now time.Time, used int64) *inputKeeper {
k := testKeeper(objs, func() time.Time { return now })
k.used = func() (int64, error) { return used, nil }
return k
}
@@ -69,13 +69,11 @@ import (
type bed struct {
// seatListRead is closed once the assigner read the seat list (serveSeatList).
seatListRead <-chan struct{}
// settings are the files of settings a test sets, by the variable naming each (as the manifest's resources fill them).
settings map[string]string
srv *server.Server
js jetstream.JetStream
ctx context.Context
dir string
buckets map[string]jetstream.KeyValue
srv *server.Server
js jetstream.JetStream
ctx context.Context
dir string
buckets map[string]jetstream.KeyValue
// placed is, by node, the modules the bed's assignments names on it (place).
placed map[string][]string
// journal is what the programs the bed starts write in their journal.
@@ -171,7 +169,7 @@ func startBed(t *testing.T, leaveOut string) *bed {
continue
}
cfg := jetstream.KeyValueConfig{Bucket: name, History: 5}
if name == plc.PulsesBucket() || name == buckets.CompositionRuns() {
if name == plc.PulsesBucket() {
cfg.History, cfg.LimitMarkerTTL = 1, time.Minute // a time to live per key, as the controller makes it (key-ttl)
}
kv, err := js.CreateKeyValue(ctx, cfg)
@@ -180,11 +178,6 @@ func startBed(t *testing.T, leaveOut string) *bed {
}
b.buckets[name] = kv
}
// The object store beside the state inputs, which the grants part makes from the manifest's objects: a composition request that
// carries the whole seat list (A-467) is too big for the state's value and is kept there.
if _, err := js.CreateObjectStore(ctx, jetstream.ObjectStoreConfig{Bucket: catalogue.BusName("mesh-delivery-assigner", catalogue.StateObjectsName(buckets.InputsState))}); err != nil {
t.Fatal(err)
}
// The builder's module log, as the runtime raises it (ADR 0297): the running commits are read from it (ADR 0571 clause 1).
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: buscompose.LogStreamName(builds.BuilderModule, builds.BuilderLog),
Subjects: []string{buscompose.LogSubject(builds.BuilderModule, builds.BuilderLog) + ".>"}, AllowDirect: true}); err != nil {
@@ -303,13 +296,6 @@ func (b *bed) startStoppable(t *testing.T) (<-chan error, func()) {
t.Fatal(err)
}
env := map[string]string{"MESH_BROKER_FILE": broker, "MESH_ASSIGNER_OUTSIDE_SETTLE_FILE": settle}
for name, value := range b.settings {
file := filepath.Join(b.dir, name)
if err := os.WriteFile(file, []byte(value), 0o600); err != nil {
t.Fatal(err)
}
env[name] = file
}
ctx, stop := context.WithCancel(b.ctx)
done := make(chan error, 1)
go func() {
@@ -1196,10 +1182,9 @@ func TestABackupAnswerIsWrittenIntoTheNodesWant(t *testing.T) {
b := startBed(t, "")
fNew := meshcatalogclient.SourcePrefix2 + strings.Repeat("b", 64)
b.fixture(t, "web", fNew, `{"module":"web","version":"1"}`)
fRun := meshcatalogclient.SourcePrefix2 + strings.Repeat("a", 64) // a held have names the build it keeps running
b.compose(t, "ace", 'b')
b.putWant(t, "ace", "web", mwc.Want{Fp: fNew, Self: true}) // a placed node's want names the build alone
held, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fRun, Phase: nodeengineclient.PhaseHeld, Health: nodeengineclient.HealthWaits, Detail: "no space left"})
held, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseHeld, Health: nodeengineclient.HealthWaits, Detail: "no space left"})
b.putJSON(t, buckets.Have, "ace.web", held)
b.start(t)
akey, _ := asks.BackupAskKey("ace", "web", fNew)
@@ -1218,20 +1203,8 @@ func TestABackupAnswerIsWrittenIntoTheNodesWant(t *testing.T) {
if w1.Fp != fNew || w1.Composed != "" || !w1.Self {
t.Errorf("the want = %+v", w1)
}
// the node retries: its have is backing up, not held. The ask stands and its condition is not cleared (A-453a).
backing, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseBackingUp, Health: nodeengineclient.HealthWaits})
b.putJSON(t, buckets.Have, "ace.web", backing)
time.Sleep(time.Second)
if !b.hasKey(buckets.Asks(), akey) {
t.Fatal("the backup ask was taken away while the node backs up again")
}
for k, c := range b.conditionsSaid(t) {
if strings.Contains(k, "backup") && c.State == occ.StateCleared {
t.Fatalf("the condition %s was cleared while the node backs up again", k)
}
}
// the backup fails again: a new have value, asked again under the next number
again, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fRun, Phase: nodeengineclient.PhaseHeld, Health: nodeengineclient.HealthWaits, Detail: "no space left, again"})
// the node retries and the backup fails again: a new have value, asked again under the next number
again, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseHeld, Health: nodeengineclient.HealthWaits, Detail: "no space left, again"})
b.putJSON(t, buckets.Have, "ace.web", again)
a2 := b.askNow(t, akey, 2)
if a2.Ask.ID == a.Ask.ID {
@@ -9,10 +9,8 @@ package main
import (
"context"
"fmt"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-sdk/go/buscalls"
"io"
"log"
@@ -32,6 +30,7 @@ import (
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshwatchdogs"
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
@@ -47,9 +46,6 @@ import (
plc "mesh-delivery-assigner/internal/assignment"
)
// composeCallBound is how long the program's calls wait for an answer (main).
const composeCallBound = 3 * time.Minute
func main() {
if os.Getenv(process.BrokerFileEnv) == "" {
// Started by the tool runner, with no credential of its own: the put's tools (ADR 0546), served through the runtime,
@@ -60,10 +56,6 @@ func main() {
}
return
}
// A call waits up to composeCallBound for its answer: compose-candidate composes a whole revision in shadow on each node and
// answered in about 34 s on g14 and shanks (2026-11-16 23:27), past the SDK's 30 s, so every composition step could not run.
// A responder that is absent still answers at once. To be replaced by a bound per verb from the seat's definition.
process.CallTimeout = composeCallBound
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
if err := run(ctx, os.Getenv, os.Stderr); err != nil {
@@ -90,7 +82,7 @@ func ownBuckets() []string {
return []string{buckets.Asks(), buckets.Differences(),
buckets.Puts(), buckets.Wants(), plc.AssignmentsBucket(), plc.ProposalsBucket(),
plc.CandidatesBucket(), plc.PulsesBucket(), assign.FindingsBucket(), assign.PoliciesBucket(), plc.SeatHoldersBucket(),
engineReadsBucket(), retiredBucket(), copiesBucket(), writtenBucket(), buckets.Waits(), buckets.PutWaits(), buckets.DeliveryWaits(), buckets.DeliveredAgain(), buckets.Inputs(), buckets.CompositionRuns(), pinsBucket()}
engineReadsBucket(), retiredBucket(), copiesBucket(), writtenBucket(), buckets.Waits(), buckets.PutWaits(), buckets.DeliveryWaits(), buckets.DeliveredAgain(), buckets.Inputs(), pinsBucket()}
}
// engineReadsState is the assigner's state of what each node's engine states it reads (ADR 0618 §3), one key `<node>`, as
@@ -134,12 +126,11 @@ var keptStates = []announce.State{
{Name: copiesState, Readable: true},
{Name: writtenState, Readable: true},
{Name: buckets.WaitsState, Readable: true},
{Name: buckets.PutWaitsState, Readable: true}, // the put waits said (putwaits.go, design 47 G A-211)
{Name: buckets.DeliveryWaitsState, Readable: true}, // the delivery waits said (delivered.go, design 47 G A-246)
{Name: buckets.DeliveredAgainState, Readable: true}, // the asks of assigner_delivered_again (delivered.go, design 47 G A-250)
{Name: buckets.InputsState, Readable: true}, // read by the tool process alone: the verb input and assigner_inputs (design 47 G §354)
{Name: buckets.CompositionRunsState, Readable: true}, // the composition runs, read by the process and its tools (design 47 G §719, A-468c)
{Name: pinsState, Readable: true}, // the records pinning each assignment graph revision (design 47 G §579)
{Name: buckets.PutWaitsState, Readable: true}, // the put waits said (putwaits.go, design 47 G A-211)
{Name: buckets.DeliveryWaitsState, Readable: true}, // the delivery waits said (delivered.go, design 47 G A-246)
{Name: buckets.DeliveredAgainState, Readable: true}, // the asks of assigner_delivered_again (delivered.go, design 47 G A-250)
{Name: buckets.InputsState, Readable: true}, // read by the tool process alone: the verb input and assigner_inputs (design 47 G §354)
{Name: pinsState, Readable: true}, // the records pinning each assignment graph revision (design 47 G §579)
}
// kvSet is every bucket the assigner opened, by name.
@@ -307,7 +298,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// against the seat's definition it holds (A-186).
gate := seatlist.NewPart("assigner", seatListAgainCallers)
if inputs != nil {
inputs.dropAtStart() // the objects of keys another part deleted (A-465)
stopInput, err := said.AnswerSeat(meshwants.Seat, seatlist.Definition(gate.Reader, meshwants.Seat), map[string]process.SeatHandler{meshwants.VerbInput: inputs.verb(conn, buscalls.SeatAckWithin)})
if err != nil {
return fmt.Errorf("answering the verb input: %w", err)
@@ -526,35 +516,17 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
gate.Follow(seatlist.ProcessBus(bus.Call, hub), wake, logger.Printf)
}
// The build seat's step-ended of an ask deletes the input kept for it (design 47 G §354) and ends the composition run that waits
// for it (A-240). The machines a composition step does not compose are the watchman's machine-away, heard once, in the world (§64, §389).
// The composition runs (design 47 G §718, §719), watched in their own state; a build seat's step-ended of a run's ask ends
// the run in its key.
var runs *compositionRuns
if kv := kvs[buckets.CompositionRuns()]; kv != nil {
set, err := runSettingsOf(getenv)
if err != nil {
// for it (A-240); the watchman's machine-away and machine-back say which machines a composition step does not compose (§64,
// §389). Registered before the hub starts.
ends, away := &runEnds{}, &awayNodes{}
if inputs != nil {
if err := hub.Handle(nodebuildgo.SubjectStepEndedAll, func(m jetstream.Msg) { heardStepEnded(inputs, ends, m, logger.Printf) }); err != nil {
return err
}
runs = newCompositionRuns(busRuns{ctx, js, kv}, set, time.Now, logger.Printf)
if js != nil {
runs.say = wantsPublisher(ctx, js, thisNode)
}
if bus != nil {
runs.drop = dropOver(bus.CallKept)
}
// The build seats' ask-taken and step-progress begin a run's clock (§720, A-466q).
if err := hearRunBounds(hub, runs, logger.Printf); err != nil {
logger.Printf("the build seats' ask-taken and step-progress are not heard, so a composition run is ended by no bound: %v", err)
}
}
if inputs != nil {
var runEnded func(buildstep.StepEnded) error
if runs != nil && bus != nil {
read := func(e buildstep.StepEnded) ([]byte, error) { return e.ReadOutput(buildstep.AskGo, bus.CallFile) }
runEnded = func(e buildstep.StepEnded) error { return runs.heardEnd(e, read) }
}
if err := hub.Handle(nodebuildgo.SubjectStepEndedAll, func(m jetstream.Msg) { heardStepEnded(inputs, runEnded, m, logger.Printf) }); err != nil {
return err
for subject, isAway := range map[string]bool{meshwatchdogs.SubjectMachineAwayAll: true, meshwatchdogs.SubjectMachineBackAll: false} {
if err := hub.Handle(subject, func(m jetstream.Msg) { heardPresence(away, isAway, m, logger.Printf) }); err != nil {
logger.Printf("%s is not heard, so a composition step asks a machine that sleeps: %v", subject, err)
}
}
// The indexer's manifests, modules and module graph, through its first views and views family, never its buckets
@@ -702,40 +674,11 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
p.acts = newPutActs(bus, wake, logger.Printf)
}
// The composition step (design 47 G §22, §354): the seat mesh-wants' accepted kind step, run by one run ask per machine.
if bus != nil && js != nil && inputs != nil && p.publishWants != nil && runs != nil {
run := &compositionRun{runs: runs, inputs: &compositionInputs{machines: w.machinesOf, callFile: bus.CallFile}, keeper: inputs, reads: &compositionReads{}, call: bus.CallKept,
trunk: trunkOf(iv, w.trunkName), running: w.runningOf, away: func(machine string) bool { return w.awayNow()[machine] }, put: runAskPublisher(js), ask: bus.CallFile, now: time.Now, logf: logger.Printf}
steps := &compositionSteps{node: thisNode, say: p.publishWants, run: run.run, resume: run.resume, ended: runs.stepEnded, now: time.Now, logf: logger.Printf,
if bus != nil && js != nil && inputs != nil && p.publishWants != nil {
run := &compositionRun{inputs: &compositionInputs{machines: w.machinesOf, callFile: bus.CallFile}, keeper: inputs, call: bus.CallKept,
trunk: trunkOf(iv, w.trunkName), running: w.runningOf, away: away, put: runAskPublisher(js), ends: ends, ask: bus.CallFile, now: time.Now, logf: logger.Printf}
steps := &compositionSteps{node: thisNode, say: p.publishWants, run: run.run, now: time.Now, logf: logger.Printf,
asker: stepAskerOf(seatlist.Definition(gate.Reader, meshwants.Seat))}
move := steps.move(ctx) // the readers a rerun carried (§727, A-470d)
runs.moved.Store(&move)
// The state's watch wakes the steps waiting on a run and hears a bound's expiry; the runs standing at the start are taken up
// once the values that stand are read (§719, §720).
if err := watchRuns(ctx, kvs[buckets.CompositionRuns()], runs, runs.expired, func() { steps.takeUp(ctx, runs) }); err != nil {
return fmt.Errorf("the state %s is not watched: %w", buckets.CompositionRunsState, err)
}
// The runs' read and drop, the seat mesh-wants' verbs compositions and compositions-drop (§728, A-469f), answered here
// where the runs are kept and the readers' ends said; the drop's hand act recorded through the seat mesh-hand-acts first.
if e, ok := bus.(emitting); ok {
record := func(q meshhandactsclient.Request) error {
_, err := meshhandactsclient.Recorder{Ask: bus.Call}.Record(q)
return err
}
stopVerbs, err := e.AnswerSeat(meshwants.Seat, seatlist.Definition(gate.Reader, meshwants.Seat), compositionVerbs(runs, steps, record))
if err != nil {
logger.Printf("the verbs %s and %s are not answered until the assigner starts again: %v", meshwants.VerbCompositions, meshwants.VerbCompositionsDrop, err)
} else {
go func() { <-ctx.Done(); stopVerbs() }()
}
}
// A superseded or cancelled pipeline's composition step leaves the runs it reads (§723, A-469n).
if p.ends != nil {
left := func(pipeline string) error {
_, err := runs.leave(pipeline, steps.leave)
return err
}
p.ends.left.Store(&left)
}
if err := serveCompositionSteps(ctx, nc, steps); err != nil {
logger.Printf("the composition step is not heard, so no composition step runs until the assigner starts again: %v", err)
} else {
@@ -45,9 +45,7 @@ type passer struct {
kvs kvSet
w *world
logf func(string, ...any)
// unread is the set of modules whose running build's manifest served last could not read, said once per change of it.
unread saidSet
sdk string
sdk string
// say publishes a condition to the operator; said is, by condition key and state, what this process has said, so a
// pass says a condition once and a restart says it again (the router keeps one per key).
say func(oc.Condition) error
@@ -841,24 +839,6 @@ func (p *passer) staleAsk(key string, trunk map[string]meshcatalog.ModulesByModu
if w.Fp != k.FP {
return "its module's want on " + k.Node + " names " + w.Fp + " now"
}
// A backup ask is about a failed backup: a have that is no longer held, or that now carries a wait (a hold of the
// composition, not a failed backup), takes it away (design 47 G A-453). A have not read yet is not stale.
if k.Kind == asks.AskKindBackup {
if hkey, err := nodeengineclient.HaveKey(k.Node, k.Module); err == nil {
if raw := p.w.value(buckets.Have, hkey); raw != nil {
if h, err := nodeengineclient.DecodeHave(raw); err == nil {
// Past the hold is running, removed, or (a have not held) at another fingerprint, a held have naming the build it keeps running; a have mid-move (backing up again after an answer,
// applying, checking) is not (design 47 G A-453a): the ask stays, and asks again under the next number.
if h.Phase == nodeengineclient.PhaseRunning || h.Phase == nodeengineclient.PhaseRemoved || (h.Phase != nodeengineclient.PhaseHeld && h.Fp != k.FP) {
return "its module's have on " + k.Node + " has moved past the hold"
}
if h.Wait != nil {
return "its module's have on " + k.Node + " carries a wait (" + h.Wait.Reason + "), not a failed backup"
}
}
}
}
}
case asks.AskKindHold, asks.AskKindRestore:
m, on := trunk[k.Module]
if !on {
@@ -1085,8 +1065,6 @@ func (p *passer) recordAsk(ctx context.Context, module, fp string, need assign.R
// after "back up again" and is answered "again" again changes the key and the node retries. That the backup failed again is seen
// in a new have value after the answer was written.
//
// Only a held have with no wait is a failed backup (design 47 G A-453): the node-engine's other holds carry a wait.
//
// A have that holds a revert (reverts, which answers its `<node>.<module>` in reverting) is left to the revert ask.
func (p *passer) backups(ctx context.Context, reverting map[string]bool) {
wanted := p.w.wantsOf()
@@ -1099,9 +1077,6 @@ func (p *passer) backups(ctx context.Context, reverting map[string]bool) {
if err != nil || have.Phase != nodeengineclient.PhaseHeld || reverting[node+"."+module] {
continue
}
if have.Wait != nil {
continue // a hold of the composition with its wait, not a failed backup (design 47 G A-453)
}
want, ok := wanted[module][node]
if !ok || want.Removed {
continue
@@ -160,25 +160,16 @@ func TestAHoldAskedAgainAfterALiftAndARestartHasANewID(t *testing.T) {
for _, c := range []struct {
name string
want *mwc.Want
have *nodeengine.HaveByNodeModuleValue // A-453: the module's have on the node
stale bool
}{
{"its want names another fingerprint", &mwc.Want{Fp: fpOther, Self: true}, nil, true},
{"its want is removed", &mwc.Want{Removed: true}, nil, true},
{"its want names the fingerprint in another composition", &mwc.Want{Fp: fpTest, Composed: "sha256:" + strings.Repeat("e", 64)}, nil, false},
{"no want yet", nil, nil, false},
// A-453: only a held have with no wait is a failed backup; a hold that carries its wait, or a have no longer held, takes the ask away.
{"its have is held with no wait", &mwc.Want{Fp: fpTest, Self: true}, &nodeengine.HaveByNodeModuleValue{Fp: fpTest, Phase: nodeengineclient.PhaseHeld, Health: nodeengineclient.HealthWaits}, false},
{"its have is held with a wait", &mwc.Want{Fp: fpTest, Self: true}, &nodeengine.HaveByNodeModuleValue{Fp: fpTest, Phase: nodeengineclient.PhaseHeld, Health: nodeengineclient.HealthWaits,
Wait: &nodeengine.Wait{Step: nodeengine.WaitStepCompose, Reason: "entry-missing", Since: now}}, true},
{"its have runs", &mwc.Want{Fp: fpTest, Self: true}, &nodeengine.HaveByNodeModuleValue{Fp: fpTest, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}, true},
{"its have backs up again (A-453a)", &mwc.Want{Fp: fpTest, Self: true}, &nodeengine.HaveByNodeModuleValue{Fp: fpTest, Phase: nodeengineclient.PhaseBackingUp, Health: nodeengineclient.HealthWaits}, false},
{"its have is held, naming the build it keeps running", &mwc.Want{Fp: fpTest, Self: true}, &nodeengine.HaveByNodeModuleValue{Fp: fpOther, Phase: nodeengineclient.PhaseHeld, Health: nodeengineclient.HealthWaits}, false},
{"its have applies at another fingerprint", &mwc.Want{Fp: fpTest, Self: true}, &nodeengine.HaveByNodeModuleValue{Fp: fpOther, Phase: nodeengineclient.PhaseApplying, Health: nodeengineclient.HealthWaits}, true},
{"its want names another fingerprint", &mwc.Want{Fp: fpOther, Self: true}, true},
{"its want is removed", &mwc.Want{Removed: true}, true},
{"its want names the fingerprint in another composition", &mwc.Want{Fp: fpTest, Composed: "sha256:" + strings.Repeat("e", 64)}, false},
{"no want yet", nil, false},
} {
t.Run("ended backup ask: "+c.name, func(t *testing.T) {
bkv := &memKV{values: map[string][]byte{}}
pb := newPasser(kvSet{buckets.Asks(): bkv, buckets.Wants(): &memKV{values: map[string][]byte{}}, buckets.Have: &memKV{values: map[string][]byte{}}}, now)
pb := newPasser(kvSet{buckets.Asks(): bkv, buckets.Wants(): &memKV{values: map[string][]byte{}}}, now)
ba := assign.BackupAsk(backKey, "ace", "web", fpTest, "sha256:"+strings.Repeat("d", 64), "w", 2, now)
bv, err := assign.EncodeAsk(backKey, ba)
if err != nil {
@@ -190,14 +181,6 @@ func TestAHoldAskedAgainAfterALiftAndARestartHasANewID(t *testing.T) {
raw, _ := mwc.EncodeWant(*c.want)
pb.w.put(buckets.Wants(), wkey, raw)
}
if c.have != nil {
hkey, _ := nodeengineclient.HaveKey("ace", "web")
raw, err := nodeengineclient.EncodeHave(*c.have)
if err != nil {
t.Fatal(err)
}
pb.w.put(buckets.Have, hkey, raw)
}
_, bask, _ := assign.DecodeAsk(backKey, bv)
body, err := tellerevents.EncodeEnded(tellerevents.Ended{Key: backKey, Ask: bask.ID, Outcome: occ.OutcomeExpired})
if err != nil {
@@ -1044,7 +1027,7 @@ func TestAsksOfAFingerprintTheTrunkLeftBehindAreTakenAway(t *testing.T) {
{"the trunk moved on", trunkOf(fpNew), []string{holdOld, holdGone, restOld, recOld}, false, nil},
{"the backup's want names another fingerprint", trunkOf(fpNew), []string{holdOld, holdGone, restOld, recOld, backOld}, false, &mwc.Want{Fp: fpNew, Self: true}},
{"the backup's want is removed", trunkOf(fpNew), []string{holdOld, holdGone, restOld, recOld, backOld}, false, &mwc.Want{Removed: true}},
{"the backup's want names the fingerprint in another composition (its have waits for a yes: a mid-move have is not past the hold, A-453a)", trunkOf(fpNew), []string{holdOld, holdGone, restOld, recOld}, false, &mwc.Want{Fp: fpOld, Composed: digNew}},
{"the backup's want names the fingerprint in another composition", trunkOf(fpNew), []string{holdOld, holdGone, restOld, recOld}, false, &mwc.Want{Fp: fpOld, Composed: digNew}},
{"the trunk is at the old fingerprint", trunkOf(fpOld), []string{holdNew, holdGone, restNew, recOld}, false, nil},
{"the trunk's fingerprint is not read yet", trunkOf(""), []string{holdGone}, false, nil},
{"the composition is not read yet", trunkOf(fpNew), []string{holdOld, holdGone, restOld}, true, nil},
@@ -1434,17 +1417,6 @@ func aWantWaitsOnTheVerbEdgesTheGraphKeeps(t *testing.T) {
t.Errorf("served(%s) = %v, want %v: the running server's manifest serves w alone, late's is not read", v, served(v), want)
}
}
// The modules whose running build's manifest is not read are said once per change of their set (G-23).
var said []string
p.logf = func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }
lateOnly := assign.Nodes{Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{"g14": {"server": at(fp("f")), "late": at(fp("2"))}}}
p.served(g, lateOnly)
p.served(g, lateOnly)
p.served(g, assign.Nodes{Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{"g14": {"late": at(fp("2")), "loop": at(fp("1"))}}})
p.served(g, lateOnly)
if len(said) != 2 || !strings.Contains(said[0], "[loop late]") && !strings.Contains(said[0], "[late loop]") || !strings.Contains(said[1], "[late]") {
t.Errorf("the unread manifests were said %q, want once per change of the set", said)
}
// A-426a: a healthy build held (no catalog entry of it) or waiting for a yes serves what it serves, said once; one held unhealthy,
// or whose running statement says it stopped, does not.
held := at(fp("2"))
@@ -153,9 +153,6 @@ type pipelineEnds struct {
firsts map[string]string
// runEnded is told a run on request's group-ended, its group `run-<id>`, to drop the run's candidate (A-235); nil drops nothing.
runEnded func(group string) error
// left is told the key of a pipeline ended superseded or cancelled, whose composition steps leave the runs they read
// (design 47 G §723, A-469n); none tells nothing. Set once the composition steps are served, after the hub starts.
left atomic.Pointer[func(pipeline string) error]
// onBranches is told the branches a registration names, for the catalog half of each to be followed (design 47 G §14, §195).
onBranches func([]string)
// trunk names the first watched branch, the trunk's (design 47 G §507), the branch of a module registered with none.
@@ -288,13 +285,6 @@ func (e *pipelineEnds) branchOf(module, placed string) string {
// §29, §64).
func (e *pipelineEnds) hearPipeline(m jetstream.Msg) {
p, err := mprc.PipelineEnded(m.Data())
if left := e.left.Load(); err == nil && left != nil && (p.Outcome == mpr.PipelineEndedOutcomeSuperseded || p.Outcome == mpr.PipelineEndedOutcomeCancelled) {
if err := (*left)(p.Pipeline); err != nil {
e.logf("%s: the composition steps of %s do not all leave their runs: %v; it is heard again", m.Subject(), p.Pipeline, err)
_ = m.Nak()
return
}
}
switch {
case err != nil:
e.logf("%s is not taken: %v", m.Subject(), err)
@@ -133,22 +133,4 @@ func TestAHeldRevertIsAskedAndALiveAnswerIsWrittenIntoTheWant(t *testing.T) {
if _, raised := findings[fkey]; raised || p.w.has(buckets.Asks(), rkey) {
t.Fatalf("a newer want left the ask (%v) or the finding (%v)", p.w.has(buckets.Asks(), rkey), raised)
}
// A-453: only a held have with no wait is a failed backup and asked; the node-engine's other holds carry a wait.
ukey, _ := asks.BackupAskKey("ace", "web", fpTest)
for _, withWait := range []bool{true, false} {
h := nodeengine.HaveByNodeModuleValue{Fp: fpTest, Phase: nodeengineclient.PhaseHeld, Health: nodeengineclient.HealthWaits, Detail: "no space left"}
if withWait {
h.Wait = &nodeengine.Wait{Step: nodeengine.WaitStepCompose, Reason: "entry-missing", Since: time.Now()}
}
raw, err := nodeengineclient.EncodeHave(h)
if err != nil {
t.Fatal(err)
}
p.w.put(buckets.Have, hkey, raw)
pass()
if got := p.w.has(buckets.Asks(), ukey); got == withWait {
t.Fatalf("a held have with a wait = %v: backup ask asked = %v", withWait, got)
}
}
}
@@ -93,11 +93,6 @@ type seatTools struct {
// seats is the seat list the bundle reads (seatlist.go): until it is read, propose judges no mesh seat the bootstrapped
// definitions do not name as held once (ADR 0636 §1). Nil reads nothing.
seatList *seatlist.Part
// watched is the tools process's watched copy of the state seat-holders (tools.go wants.watched, which gate.Holders
// also reads: A-298), read by the verb assignments with seats instead of a bus get per key (issue 1982: under load
// that took about 14 s, where the watched copy answers at once). Nil, or a call it cannot answer, falls back to a
// direct listing.
watched *stdio.WatchedStates
}
func newSeatTools(st seatStore, ask func(string, any) (json.RawMessage, error)) *seatTools {
@@ -559,43 +554,25 @@ func (s *seatTools) tools() []stdio.Tool {
return out
}
// seatHolders is every key of the state seat-holders and its raw value: from the watched copy (s.watched) when
// there is one and it answers, else a direct listing, key by key, as the verb answered before (issue 1982).
func (s *seatTools) seatHolders() (map[string][]byte, error) {
if s.watched != nil {
if vals, err := s.watched.Values(mma.StateSeatHolders); err == nil {
return vals, nil
}
}
// seats is the verb assignments with seats (ADR 0571 §4): who holds each seat and on which node, as the state seat-holders
// holds it; with node, the keys of seats held on that node.
func (s *seatTools) seats(node string) (any, error) {
keys, err := s.st.keys(mma.StateSeatHolders)
if err != nil {
return nil, fmt.Errorf("seat-holders cannot be listed: %w", err)
}
out := make(map[string][]byte, len(keys))
sort.Strings(keys)
out := []mma.SeatHolders{}
for _, key := range keys {
raw, _, found, err := s.st.get(mma.StateSeatHolders, key)
if err != nil {
return nil, fmt.Errorf("seat-holders %s cannot be read: %w", key, err)
}
if found {
out[key] = raw
if !found {
continue
}
}
return out, nil
}
// seats is the verb assignments with seats (ADR 0571 §4): who holds each seat and on which node, as the state seat-holders
// holds it; with node, the keys of seats held on that node.
func (s *seatTools) seats(node string) (any, error) {
raws, err := s.seatHolders()
if err != nil {
return nil, err
}
keys := slices.Sorted(maps.Keys(raws))
out := []mma.SeatHolders{}
for _, key := range keys {
var sh mma.SeatHolders
if json.Unmarshal(raws[key], &sh) != nil {
if json.Unmarshal(raw, &sh) != nil {
continue
}
if node != "" && sh.Node != node && !slices.ContainsFunc(sh.Holders, func(h mma.SeatHoldersHoldersItem) bool { return h.Node == node }) {
@@ -352,53 +352,6 @@ func TestTheSeatsVerbs(t *testing.T) {
t.Errorf("seats on ace = %+v, %v", out, err)
}
})
t.Run("issue 1982: assignments with seats reads the tools process's watched copy of seat-holders, the store only while it cannot", func(t *testing.T) {
seeded := map[string][]byte{}
put := func(seat, node string, holders ...mma.SeatHoldersHoldersItem) {
raw, err := json.Marshal(mma.SeatHolders{Seat: seat, Node: node, Holders: holders})
if err != nil {
t.Fatal(err)
}
seeded[seat] = raw
}
put("mesh-broker", "", mma.SeatHoldersHoldersItem{Module: "nats", Node: "ace"})
put("node-shell.g14", "g14")
st := &countingSeatStore{memStore: memStore{mma.StateSeatHolders: &memKV{values: seeded}}}
sv := newSeatTools(st, nil)
watches := 0
sv.watched = stdio.WatchedStatesOver(func(state string, handle func(stdio.StateChange) error) error {
watches++
for k, v := range seeded {
if err := handle(stdio.StateChange{Key: k, Op: stdio.OpPut, Value: v, Current: true}); err != nil {
return err
}
}
return nil
}, func(state, key string) ([]byte, bool, error) {
v, ok := seeded[key]
return v, ok, nil
})
out, err := sv.seats("")
if err != nil {
t.Fatal(err)
}
if got := out.(map[string]any)["seats"].([]mma.SeatHolders); len(got) != 2 || watches != 1 || st.keysCalls != 0 || st.getCalls != 0 {
t.Errorf("seats from the watched copy = %+v, watched %d time(s), the store's own keys %d get %d (want none)",
got, watches, st.keysCalls, st.getCalls)
}
// The lead's answer on issue 1982: the verb falls back to the store's own listing only while the watched copy
// cannot be read (here, its watch refused); the answer's form does not change.
sv.watched = stdio.WatchedStatesOver(func(string, func(stdio.StateChange) error) error {
return errors.New("refused")
}, func(state, key string) ([]byte, bool, error) { return nil, false, nil })
out, err = sv.seats("")
if err != nil {
t.Fatal(err)
}
if got := out.(map[string]any)["seats"].([]mma.SeatHolders); len(got) != 2 || st.keysCalls != 1 || st.getCalls != 2 {
t.Errorf("seats falling back to the store = %+v, the store's own keys %d get %d (want 1, 2)", got, st.keysCalls, st.getCalls)
}
})
t.Run("research 072 row 37: a node is written as its keys, and history reads the node key", func(t *testing.T) {
ctx := context.Background()
kv := &revKV{values: map[string][]byte{}, revs: map[string]uint64{}}
@@ -797,23 +750,6 @@ func readManifest() (catalogue.Manifest, error) {
return catalogue.Decode(raw)
}
// countingSeatStore is a memStore that counts its calls to keys and get, to prove a reader takes the state from the
// watched copy and the store only while that copy cannot be read (issue 1982).
type countingSeatStore struct {
memStore
keysCalls, getCalls int
}
func (c *countingSeatStore) keys(state string) ([]string, error) {
c.keysCalls++
return c.memStore.keys(state)
}
func (c *countingSeatStore) get(state, key string) ([]byte, uint64, bool, error) {
c.getCalls++
return c.memStore.get(state, key)
}
// countingKV is a memKV that counts its puts.
type countingKV struct {
memKV
@@ -419,13 +419,9 @@ func serveTools() error {
wants.watched = stdio.NewWatchedStates() // the large states read whole from one watched copy (ADR 0580 clause 3)
// The callers' seat entries are resolved from the assigner's own seat-holders, its watched copy (A-298).
gate.Holders(seatlist.HoldersIn(func() (map[string][]byte, error) { return wants.watched.Values(mma.StateSeatHolders) }))
// The verb assignments with seats answers from the same watched copy, so every caller of it is fast too (issue 1982).
seatVerbs.watched = wants.watched
wants.revisions = runtimeObjects{busstate.ObjectsOf(assigngraph.RevisionsObjects)}
tools = append(tools, wants.tools()...)
keeper := &inputKeeper{vals: stateobjects.ForState(buckets.InputsState), now: time.Now, logf: func(string, ...any) {}}
tools = append(tools, inputsRead(keeper), inputsDrop(keeper, stdio.Ask))
tools = append(tools, compositionsTools(stdio.Ask)...) // the composition runs' read and drop, through their verbs (§728)
tools = append(tools, inputsRead(&inputKeeper{vals: stateobjects.ForState(buckets.InputsState), now: time.Now, logf: func(string, ...any) {}}))
tools = append(tools, viewTools(os.Getenv, keptSource, time.Now)...)
// What each node's engine reads, as the assigner uses it (ADR 0618 §3): its own state engine-reads, asked again through it.
tools = append(tools, enginereads.Tool(engineReadsTool, "the assigner", enginereads.KeptState(stdio.State(engineReadsState)), false, time.Now))
@@ -2,8 +2,6 @@ package main
import (
"slices"
"strings"
"sync"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
@@ -55,31 +53,12 @@ func (p *passer) served(g graph.Graph, n assign.Nodes) func(string) bool {
}
served, unread := graph.Served(servingHaves(n, p.countedServing), func(module, fp string) ([]string, bool) { return p.servesOf(module, fp) },
func(module string) []string { return g[module].Serves })
if p.unread.changed(unread) && len(unread) > 0 {
if len(unread) > 0 {
p.logf("the manifest of the running build of %v cannot be read, so what the module graph says they serve is counted as served", unread)
}
return served
}
// saidSet is the last set of names said, so a pass says a set again only when it changed.
type saidSet struct {
mu sync.Mutex
last string
}
// changed keeps names as the set said last and says whether it differs from the one before.
func (s *saidSet) changed(names []string) bool {
sorted := slices.Sorted(slices.Values(names))
key := strings.Join(sorted, " ")
s.mu.Lock()
defer s.mu.Unlock()
if key == s.last {
return false
}
s.last = key
return true
}
// offering is, for this pass, whether a build whose manifest offers a provision runs healthy on a node present (ADR 0603
// §5, ADR 0614 §5): every running healthy have on an online node, its build's manifest read as servesOf reads it. A have
// waiting for a person's yes on a newer build counts too: it names the build that still runs and that build's health, and
@@ -29,9 +29,6 @@ const (
PutWaitsState = "put-waits"
// InputsState holds the inputs of the composition steps, `<time>.<digest>`, which the seat mesh-wants' verb input answers (design 47 G §354).
InputsState = "inputs"
// CompositionRunsState holds the composition runs, one key `<candidate key>.<machine>` per run and its bound keys
// `bound.<run key>` and `untaken.<run key>` (design 47 G §719, §720, A-470).
CompositionRunsState = "composition-runs"
// PoliciesState is the state of the policies a person set for a module's upgrade.
PoliciesState = "policies"
)
@@ -40,17 +37,16 @@ const (
// constant of its name (design 47 G A-296, §428).
func own(state string) string { return plc.Module() + "_" + state }
func Wants() string { return own(wants.StateWants) }
func Differences() string { return own(wants.StateDifferences) }
func Asks() string { return own("asks") }
func Puts() string { return own("puts") }
func Waits() string { return own(WaitsState) }
func DeliveredAgain() string { return own(DeliveredAgainState) }
func DeliveryWaits() string { return own(DeliveryWaitsState) }
func PutWaits() string { return own(PutWaitsState) }
func Inputs() string { return own(InputsState) }
func CompositionRuns() string { return own(CompositionRunsState) }
func Policies() string { return own(PoliciesState) }
func Wants() string { return own(wants.StateWants) }
func Differences() string { return own(wants.StateDifferences) }
func Asks() string { return own("asks") }
func Puts() string { return own("puts") }
func Waits() string { return own(WaitsState) }
func DeliveredAgain() string { return own(DeliveredAgainState) }
func DeliveryWaits() string { return own(DeliveryWaitsState) }
func PutWaits() string { return own(PutWaitsState) }
func Inputs() string { return own(InputsState) }
func Policies() string { return own(PoliciesState) }
// AsksHistory is the revisions the bucket of asks keeps per key.
const AsksHistory = 5
@@ -31,8 +31,7 @@ func TestTheAssignersOwnBucketsTakeTheModuleItRunsAs(t *testing.T) {
"Wants": {Wants(), "some-assigner_wants"}, "Differences": {Differences(), "some-assigner_differences"},
"Asks": {Asks(), "some-assigner_asks"}, "Puts": {Puts(), "some-assigner_puts"}, "Policies": {Policies(), "some-assigner_policies"},
"Waits": {Waits(), "some-assigner_waits"}, "Inputs": {Inputs(), "some-assigner_inputs"},
"CompositionRuns": {CompositionRuns(), "some-assigner_composition-runs"},
"Assignments": {plc.AssignmentsBucket(), "some-assigner_assignments"}, "SeatHolders": {plc.SeatHoldersBucket(), "some-assigner_seat-holders"},
"Assignments": {plc.AssignmentsBucket(), "some-assigner_assignments"}, "SeatHolders": {plc.SeatHoldersBucket(), "some-assigner_seat-holders"},
} {
if c[0] != c[1] {
t.Errorf("%s = %q, want %q", name, c[0], c[1])
@@ -70,7 +70,7 @@ func TestManifest(t *testing.T) {
}
if want := map[string]int{"asks": 5, "differences": 64, "puts": 20, "wants": 20,
"assignments": 64, "placement": 64, "copies": 1, "proposals": 5, "candidates": 5, "pulses": 0, "findings": 5, "policies": 20, "seat-holders": 5, views.MarksState: 1, views.StatusState: 1,
"engine-reads": 1, "written": 5, "waits": 5, "put-waits": 1, "delivery-waits": 1, "delivered-again": 1, "inputs": 1, "composition-runs": 1, "pins": 5}; !reflect.DeepEqual(states, want) {
"engine-reads": 1, "written": 5, "waits": 5, "put-waits": 1, "delivery-waits": 1, "delivered-again": 1, "inputs": 1, "pins": 5}; !reflect.DeepEqual(states, want) {
t.Errorf("state = %v, want %v", states, want)
}
if _, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 {
@@ -93,7 +93,7 @@ func TestManifest(t *testing.T) {
// revision, entry and difference from its revisions kept as objects in the old section, form 1, capped at 512 MiB.
if len(m.Claims) != 2 || m.Claims[0].Name != meshmoduleassigner.Seat || m.Claims[0].At() != catalogue.ScopeMesh ||
!slices.Equal(m.Claims[0].Serves, []string{"assignments", "proposals", "propose", "withdraw", "again", "placement"}) ||
m.Claims[1].Name != meshwants.Seat || m.Claims[1].At() != catalogue.ScopeMesh || !slices.Equal(m.Claims[1].Serves, []string{"upgrade", "bus", "revision", "entry", "difference", "input", "compositions", "compositions-drop"}) {
m.Claims[1].Name != meshwants.Seat || m.Claims[1].At() != catalogue.ScopeMesh || !slices.Equal(m.Claims[1].Serves, []string{"upgrade", "bus", "revision", "entry", "difference", "input"}) {
t.Errorf("the assigner claims %+v, want mesh-module-assigner with its five verbs and mesh-wants with its five", m.Claims)
}
if len(m.Objects) != 1 || m.Objects[0].Name != "assignment-revisions" || m.Objects[0].MaxMiB != 512 || m.Objects[0].Form != 1 {
@@ -103,7 +103,7 @@ func TestManifest(t *testing.T) {
// ADR 0621 §1: the first-view tools assignments, seat-holders, wants and asks (§717) and design 25 § 12.2's six read tools, of the
// program's bundle, which runs in a runtime of its own on the assigner's account; ADR 0618 §3: and assigner_engine_reads,
// which lists and asks again what each node's engine reads.
if !slices.Equal(m.Tools, []string{"assigner_put", "assigner_release", "assigner_puts", "assigner_waits", "assigner_inputs", "assigner_inputs_drop", "assigner_compositions", "assigner_compositions_drop", "assignments", "seat-holders", "wants", "asks",
if !slices.Equal(m.Tools, []string{"assigner_put", "assigner_release", "assigner_puts", "assigner_waits", "assigner_inputs", "assignments", "seat-holders", "wants", "asks",
"assigner_read_assignments", "assigner_read_seat_holders", "assigner_read_wants", "assigner_read_findings", "assigner_read_differences", "assigner_read_copies",
"assigner_first_views", "assigner_first_view_again", "assigner_engine_reads", "assigner_seat_list", "assigner_seat_list_again", "assigner_ask_again", "assigner_written", "assigner_why", "assigner_pins", "assigner_unpin", "assigner_delivered_again"}) {
t.Errorf("the assigner serves tools %v, want its three put tools, three first views, six read tools, assigner_engine_reads, the seat list's two (ADR 0636 §7) assigner_ask_again (ADR 0668 §5) and assigner_why (design 47 G A-439)", m.Tools)
@@ -137,7 +137,7 @@ func TestManifest(t *testing.T) {
// ADR 0618 §4: and it hears each node's engine say its reads changed; ADR 0636 §4 (issue 1966): and the seat list's two events;
// issue 1966 note 151: and the teller's ask-ended, its subject the one the assigner hears; §376, §389: its own version-delivered and
// the watchman's machine-away and machine-back, read whole at its start.
if !slices.Equal(m.Consumes, []string{"mesh-conditions.findings-asked", "mesh-conditions.condition-raised", "mesh-conditions.condition-changed", "mesh-conditions.condition-cleared", "mesh-catalog.catalog-revision", "mesh-catalog.registration-changed", "mesh-delivery.source-moved", "mesh-catalog.version-released", "mesh-wants.version-delivered", "mesh-watchdogs.machine-away", "mesh-watchdogs.machine-back", "mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", builds.BuilderModule + "." + builds.BuildsStreamEvents, buckets.IndexerModule + ".take-in.taken", buckets.TellerModule + "." + tellerevents.AskEnded, buckets.TellerModule + ".views." + buckets.AnswersState, "mesh-pipeline-runner.pipeline-ended", "mesh-pipeline-runner.group-ended", "node-build-go.step-ended", "node-build-go.ask-taken", "node-build-go.step-progress", "node-engine.manifest-reads-changed", "node-engine.running", "mesh-delivery.pinned", "mesh-delivery.freed", "mesh-delivery.heads-replaced", "mesh-delivery.pins-stated"}) {
if !slices.Equal(m.Consumes, []string{"mesh-conditions.findings-asked", "mesh-conditions.condition-raised", "mesh-conditions.condition-changed", "mesh-conditions.condition-cleared", "mesh-catalog.catalog-revision", "mesh-catalog.registration-changed", "mesh-delivery.source-moved", "mesh-catalog.version-released", "mesh-wants.version-delivered", "mesh-watchdogs.machine-away", "mesh-watchdogs.machine-back", "mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", builds.BuilderModule + "." + builds.BuildsStreamEvents, buckets.IndexerModule + ".take-in.taken", buckets.TellerModule + "." + tellerevents.AskEnded, buckets.TellerModule + ".views." + buckets.AnswersState, "mesh-pipeline-runner.pipeline-ended", "mesh-pipeline-runner.group-ended", "node-build-go.step-ended", "node-engine.manifest-reads-changed", "node-engine.running", "mesh-delivery.pinned", "mesh-delivery.freed", "mesh-delivery.heads-replaced", "mesh-delivery.pins-stated"}) {
t.Errorf("consumes = %v, want the catalog half's revisions (design 47 G §195), the teller's ask-ended, the engines' reads changed and what they run", m.Consumes)
}
if want := []string{"mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules",
@@ -147,8 +147,7 @@ func TestManifest(t *testing.T) {
"seat:mesh-delivery.source", // §507: the branches the gate keeps a source revision of, before the first registrations
"seat:mesh-hand-acts.hand-act", "seat:mesh-hand-acts.hand-acts", "seat:mesh-hand-acts.warranted", // ADR 0565 §9, §11: the seat alone
"seat:node-backup.backed-up", "seat:node-backup.now",
"seat:node-build-go.accept.check", // A-231
"seat:node-build-go.drop", // §720, A-469s: a composition run past its bound drops its ask: the composition run's ask on the Go build seat's queue check (accept.check.<asker>)
"seat:node-build-go.accept.check", // A-231: the composition run's ask on the Go build seat's queue check (accept.check.<asker>)
"seat:node-build-go.output", // A-254: a composition run's answer kept as the build seat's object
"seat:node-engine.compose-candidate", // §354, A-226: each machine's composition input
"seat:node-engine.manifest-reads", // ADR 0618 §3
@@ -168,21 +167,13 @@ func TestManifest(t *testing.T) {
}
file, _ := os.ReadFile("module.json")
if err := json.Unmarshal(file, &raw); err != nil || raw.Config["outside-wait-minutes"].Default != float64(30) || raw.Config["outside-settle-seconds"].Default != float64(90) ||
raw.Config["ask-operator-for"].Default != "take-off,remove-node,stops" || len(raw.Config) != 8 ||
raw.Config["composition-runs-cap-mib"].Default != float64(16) || raw.Config["composition-runs-kept"].Default != "24h" ||
raw.Config["composition-step-bound"].Default != "30m" || raw.Config["run-untaken-bound"].Default != "24h" || raw.Config["blame-tries"].Default != float64(20) {
raw.Config["ask-operator-for"].Default != "take-off,remove-node,stops" || len(raw.Config) != 3 {
t.Errorf("config = %+v (%v), want outside-wait-minutes of 30, outside-settle-seconds of 90 and ask-operator-for of take-off,remove-node,stops (A-425a)", raw.Config, err)
}
for _, f := range []struct{ env, path, key string }{
{"MESH_ASSIGNER_OUTSIDE_WAIT_FILE", "${dir:state}/outside-wait-minutes", "outside-wait-minutes"},
{"MESH_ASSIGNER_OUTSIDE_SETTLE_FILE", "${dir:state}/outside-settle-seconds", "outside-settle-seconds"},
{"MESH_ASSIGNER_ASK_OPERATOR_FOR_FILE", "${dir:state}/ask-operator-for", "ask-operator-for"},
// design 47 G §719, §720, §721, A-469x: the composition runs' settings
{"MESH_ASSIGNER_COMPOSITION_RUNS_CAP_MIB_FILE", "${dir:state}/composition-runs-cap-mib", "composition-runs-cap-mib"},
{"MESH_ASSIGNER_COMPOSITION_RUNS_KEPT_FILE", "${dir:state}/composition-runs-kept", "composition-runs-kept"},
{"MESH_ASSIGNER_COMPOSITION_STEP_BOUND_FILE", "${dir:state}/composition-step-bound", "composition-step-bound"},
{"MESH_ASSIGNER_RUN_UNTAKEN_BOUND_FILE", "${dir:state}/run-untaken-bound", "run-untaken-bound"},
{"MESH_ASSIGNER_BLAME_TRIES_FILE", "${dir:state}/blame-tries", "blame-tries"},
} {
if !strings.Contains(string(file), `"`+f.env+`": "`+f.path+`"`) || !strings.Contains(string(file), `"path": "`+f.path+`"`) ||
!strings.Contains(string(file), `"content": "${config:`+f.key+`}"`) {
+2 -90
View File
@@ -25,9 +25,7 @@
"revision",
"entry",
"difference",
"input",
"compositions",
"compositions-drop"
"input"
]
}
],
@@ -61,8 +59,6 @@
"mesh-pipeline-runner.pipeline-ended",
"mesh-pipeline-runner.group-ended",
"node-build-go.step-ended",
"node-build-go.ask-taken",
"node-build-go.step-progress",
"node-engine.manifest-reads-changed",
"node-engine.running",
"mesh-delivery.pinned",
@@ -86,7 +82,6 @@
"seat:node-backup.backed-up",
"seat:node-backup.now",
"seat:node-build-go.output",
"seat:node-build-go.drop",
"seat:node-engine.compose-candidate",
"seat:node-engine.manifest-reads",
"seat:operator-channel.asks",
@@ -117,9 +112,6 @@
"assigner_puts",
"assigner_waits",
"assigner_inputs",
"assigner_inputs_drop",
"assigner_compositions",
"assigner_compositions_drop",
"assignments",
"seat-holders",
"wants",
@@ -165,7 +157,6 @@
{ "name": "delivery-waits", "history": 1 },
{ "name": "delivered-again", "history": 1 },
{ "name": "inputs", "history": 1, "max-mib": 64 },
{ "name": "composition-runs", "history": 1, "key-ttl": true, "max-mib": 64 },
{ "name": "pins", "history": 5, "max-mib": 4 }
],
"config": {
@@ -190,40 +181,6 @@
"description": "the change kinds of a proposal the assigner asks the operator before it writes it, separated by commas: place, take-off, pin, unpin, hold, unhold, decide, config, add-node, remove-node, and stops for a verdict whose answers say a node would stop running a module (design 47 G A-425, A-425a). A proposal none of whose changes is of a kind named is applied at once when its verdict passes, said in the log and by assignments-changed. The default is take-off,remove-node,stops; empty asks for nothing.",
"trusted": false,
"default": "take-off,remove-node,stops"
},
"composition-runs-cap-mib": {
"type": "number",
"description": "the cap of the state composition-runs in MiB, the process refusing a new composition run at it with its steps ending could not run naming the fill (design 47 G section 719, A-469d); its maximum is the manifest's max-mib.",
"minimum": 1,
"maximum": 64,
"trusted": false,
"default": 16
},
"composition-runs-kept": {
"type": "string",
"description": "how long a composition run's key is kept after it ended, a Go duration such as 24h (design 47 G section 719, A-469x).",
"trusted": false,
"default": "24h"
},
"composition-step-bound": {
"type": "string",
"description": "how long a taken composition run's ask may go without saying its progress before the run ends could not run, a Go duration such as 30m; written again at each step-progress (design 47 G section 720, A-469x).",
"trusted": false,
"default": "30m"
},
"run-untaken-bound": {
"type": "string",
"description": "how long a composition run's ask may wait for a build node to take it before the run ends could not run, a Go duration such as 24h (design 47 G section 720, A-466s, A-469x).",
"trusted": false,
"default": "24h"
},
"blame-tries": {
"type": "number",
"description": "the most compositions a machine's composer makes to find the modules it blames, carried in the run's request as blame_tries (design 47 G section 721, A-466r).",
"minimum": 1,
"maximum": 1000,
"trusted": false,
"default": 20
}
},
"objects": [
@@ -273,46 +230,6 @@
"mode": "0600",
"content": "${config:ask-operator-for}"
},
{
"id": "composition-runs-cap-mib",
"type": "file",
"path": "${dir:state}/composition-runs-cap-mib",
"owner": "mesh-delivery-assigner",
"mode": "0600",
"content": "${config:composition-runs-cap-mib}"
},
{
"id": "composition-runs-kept",
"type": "file",
"path": "${dir:state}/composition-runs-kept",
"owner": "mesh-delivery-assigner",
"mode": "0600",
"content": "${config:composition-runs-kept}"
},
{
"id": "composition-step-bound",
"type": "file",
"path": "${dir:state}/composition-step-bound",
"owner": "mesh-delivery-assigner",
"mode": "0600",
"content": "${config:composition-step-bound}"
},
{
"id": "run-untaken-bound",
"type": "file",
"path": "${dir:state}/run-untaken-bound",
"owner": "mesh-delivery-assigner",
"mode": "0600",
"content": "${config:run-untaken-bound}"
},
{
"id": "blame-tries",
"type": "file",
"path": "${dir:state}/blame-tries",
"owner": "mesh-delivery-assigner",
"mode": "0600",
"content": "${config:blame-tries}"
},
{
"id": "assigner",
"type": "process",
@@ -325,12 +242,7 @@
"MESH_BROKER_FILE": "${dir:state}/broker",
"MESH_ASSIGNER_OUTSIDE_WAIT_FILE": "${dir:state}/outside-wait-minutes",
"MESH_ASSIGNER_OUTSIDE_SETTLE_FILE": "${dir:state}/outside-settle-seconds",
"MESH_ASSIGNER_ASK_OPERATOR_FOR_FILE": "${dir:state}/ask-operator-for",
"MESH_ASSIGNER_COMPOSITION_RUNS_CAP_MIB_FILE": "${dir:state}/composition-runs-cap-mib",
"MESH_ASSIGNER_COMPOSITION_RUNS_KEPT_FILE": "${dir:state}/composition-runs-kept",
"MESH_ASSIGNER_COMPOSITION_STEP_BOUND_FILE": "${dir:state}/composition-step-bound",
"MESH_ASSIGNER_RUN_UNTAKEN_BOUND_FILE": "${dir:state}/run-untaken-bound",
"MESH_ASSIGNER_BLAME_TRIES_FILE": "${dir:state}/blame-tries"
"MESH_ASSIGNER_ASK_OPERATOR_FOR_FILE": "${dir:state}/ask-operator-for"
},
"user": "mesh-delivery-assigner",
"health": {
@@ -20,7 +20,7 @@ func tcs(language, version string, nodes ...string) []buildstep.NodeToolchain {
}
func chooseTakesTheAgreedNodesWithFewestRunningStepsTiesByName(t *testing.T) { // 1607/AC4, ADR 0478 clause 3
entries := tcs("go", "go1.27", "novox", "ace", "g14", "shanks")
entries := tcs("go", "go1.27.1", "novox", "ace", "g14", "shanks")
nodes, tc, err := ChooseNodes("go", entries, placesRunning(map[string]int{"novox": 3, "ace": 1, "g14": 1, "shanks": 0}), nil, 2)
if err != nil {
t.Fatal(err)
@@ -28,13 +28,13 @@ func chooseTakesTheAgreedNodesWithFewestRunningStepsTiesByName(t *testing.T) { /
if len(nodes) != 2 || nodes[0] != "shanks" || nodes[1] != "ace" {
t.Errorf("nodes = %v, want [shanks ace]: fewest steps first, a tie by name", nodes)
}
if tc.Version != "go1.27" || tc.Recipe != "1" || tc.Language != "go" {
if tc.Version != "go1.27.1" || tc.Recipe != "1" || tc.Language != "go" {
t.Errorf("toolchain = %+v", tc)
}
}
func chooseWithOneNodeAnswersOneNode(t *testing.T) { // 1607/AC4
nodes, _, err := ChooseNodes("go", tcs("go", "go1.27", "novox"), nil, nil, 2)
nodes, _, err := ChooseNodes("go", tcs("go", "go1.27.1", "novox"), nil, nil, 2)
if err != nil || len(nodes) != 1 || nodes[0] != "novox" {
t.Errorf("nodes = %v, err %v, want [novox]", nodes, err)
}
@@ -45,9 +45,9 @@ func chooseFailsNamingTheLanguageAndTheVersionsWhenNoNodeQualifies(t *testing.T)
if err == nil || !strings.Contains(err.Error(), "typescript") || !strings.Contains(err.Error(), buildstep.ToolchainNoReport) {
t.Errorf("no report: error = %v, want it to name typescript and %q", err, buildstep.ToolchainNoReport)
}
entries := append(tcs("go", "go1.27", "novox"), tcs("go", "go1.26.0", "ace")...)
entries := append(tcs("go", "go1.27.1", "novox"), tcs("go", "go1.26.0", "ace")...)
_, _, err = ChooseNodes("go", entries, nil, nil, 2)
if err == nil || !strings.Contains(err.Error(), "go1.27") || !strings.Contains(err.Error(), "go1.26") || !strings.Contains(err.Error(), "go") {
if err == nil || !strings.Contains(err.Error(), "go1.27.1") || !strings.Contains(err.Error(), "go1.26.0") || !strings.Contains(err.Error(), "go") {
t.Errorf("disagreeing nodes: error = %v, want both versions named", err)
}
}
@@ -67,17 +67,17 @@ func agreeNeedsEqualDigestsAndNamesBothNodesWhenTheyDiffer(t *testing.T) { // 16
}
func chooseSkipsNodesThatAreNotOnlineAndFailsNamingThemWhenNoneIs(t *testing.T) { // 1607 review 3, ADR 0478 clause 3
entries := tcs("go", "go1.27", "ace", "g14", "novox")
entries := tcs("go", "go1.27.1", "ace", "g14", "novox")
online := func(n string) bool { return n != "g14" } // a sleeping laptop
nodes, tc, err := ChooseNodes("go", entries, placesRunning(map[string]int{"novox": 5}), online, 2)
if err != nil || len(nodes) != 2 || nodes[0] != "ace" || nodes[1] != "novox" || tc.Version != "go1.27" {
if err != nil || len(nodes) != 2 || nodes[0] != "ace" || nodes[1] != "novox" || tc.Version != "go1.27.1" {
t.Errorf("nodes = %v, toolchain %+v, err %v, want [ace novox] with g14 left out although it runs nothing", nodes, tc, err)
}
_, tc, err = ChooseNodes("go", entries, nil, func(string) bool { return false }, 2)
if err == nil || !strings.Contains(err.Error(), "online") || !strings.Contains(err.Error(), "g14") || !strings.Contains(err.Error(), "go") {
t.Errorf("none online: error = %v, want it to name the nodes that report and that none is online", err)
}
if tc.Version != "go1.27" {
if tc.Version != "go1.27.1" {
t.Errorf("none online: the agreed toolchain %+v is still answered", tc)
}
}
@@ -88,7 +88,7 @@ func placesRunning(running map[string]int) *Places {
}
func choosePrefersNodesBelowTheirCapacity(t *testing.T) { // round 3
entries := tcs("go", "go1.27", "ace", "novox", "shanks")
entries := tcs("go", "go1.27.1", "ace", "novox", "shanks")
// shanks has capacity 3 with 2 running: room; ace has the default 1 with 1 running: none; novox has none running.
p := placesRunning(map[string]int{"shanks": 2, "ace": 1})
p.Capacity = map[string]int{"shanks": 3}
@@ -58,7 +58,7 @@ func TestToolchainGate(t *testing.T) {
t.Run("not placed on ace: its report is no builder's", func(t *testing.T) {
gate, set, said := start(t, true)
key, value := gateReport(t, "ace", "v26.11.0", old)
key, value := gateReport(t, "ace", "v26.12.0", old)
gate.Toolchain(testNode, key, value, false)
if a := buildstep.ToolchainAgreement(set.Entries("typescript")); len(a.SittingOut) != 1 || a.SittingOut[0].Node != "ace" { // ADR 0584: the majority is agreed, ace's report counts as the one that sits out
t.Fatalf("before the assignments is settled every report counts; got %+v", a)
@@ -68,7 +68,7 @@ func TestToolchainGate(t *testing.T) {
if !gate.Settled() {
t.Fatal("settling left ace's report out, so what was passed on changed")
}
if v, none := agreedOf(set); v != "v26.12" || none != "" {
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
t.Fatalf("the placed nodes agree on v26.12.1; got %q, %q", v, none)
}
if len(*said) != 1 || !strings.Contains((*said)[0], "does not place mesh-build-node on ace") {
@@ -80,14 +80,14 @@ func TestToolchainGate(t *testing.T) {
// Placed on ace with a report written since: it counts, and the builders really disagree.
k, v = placing(t, "ace", testGo, testNode)
gate.Assignments(k, v, false, old.Add(-time.Minute))
if a := buildstep.ToolchainAgreement(set.Entries("typescript")); a.Toolchain.Version != "v26.12" || len(a.SittingOut) != 1 || a.SittingOut[0].Version != "v26.11" {
if a := buildstep.ToolchainAgreement(set.Entries("typescript")); a.Toolchain.Version != "v26.12.1" || len(a.SittingOut) != 1 || a.SittingOut[0].Version != "v26.12.0" {
t.Fatalf("builders that run different versions: the majority is agreed and ace sits out; got %+v", a)
}
// Taken off ace again (its key deleted): its report goes again.
if !gate.Assignments(k, nil, true, placed) {
t.Fatal("taking ace's assignments away passed nothing on")
}
if v, none := agreedOf(set); v != "v26.12" || none != "" {
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
t.Fatalf("got %q, %q", v, none)
}
// A report deleted from the bucket goes whether its node is placed or not.
@@ -131,7 +131,7 @@ func TestToolchainGate(t *testing.T) {
if gate.Assignments(k, v, false, placed) {
t.Fatal("assignments with no report passed something on")
}
key, value := gateReport(t, "ace", "v26.11.0", c.at)
key, value := gateReport(t, "ace", "v26.12.0", c.at)
if got := gate.Toolchain(testNode, key, value, false); got != c.counts {
t.Fatalf("ace's report passed on: %v, want %v", got, c.counts)
}
@@ -143,7 +143,7 @@ func TestToolchainGate(t *testing.T) {
if c.counts {
return
}
if v, none := agreedOf(set); v != "v26.12" || none != "" {
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
t.Fatalf("ace's left report is no builder's; got %q, %q", v, none)
}
if len(*said) != 1 || !strings.Contains((*said)[0], "before mesh-build-node was placed on ace at 2026-10-25T19:56:45Z") {
@@ -158,7 +158,7 @@ func TestToolchainGate(t *testing.T) {
t.Run("no assignments held: every report counts", func(t *testing.T) {
gate, set, _ := start(t, false)
key, value := gateReport(t, "ace", "v26.11.0", old)
key, value := gateReport(t, "ace", "v26.12.0", old)
gate.Toolchain(testNode, key, value, false)
gate.Settled()
if n := len(set.Entries("typescript")); n != 4 {
@@ -183,7 +183,7 @@ func TestToolchainGate(t *testing.T) {
for _, node := range append([]string{"ace"}, three...) {
key, value := gateReport(t, node, "v26.12.1", placed.Add(time.Minute))
if node == "ace" {
key, value = gateReport(t, node, "v26.11.0", old)
key, value = gateReport(t, node, "v26.12.0", old)
}
gate.Toolchain(testNode, key, value, false)
}
@@ -69,7 +69,7 @@ type AgreedDigest struct {
func Digests(entries []buildstep.NodeToolchain, agreed buildstep.Toolchain) []AgreedDigest {
by := map[string][]string{}
for _, e := range entries {
if buildstep.LanguageVersion(e.Version) == agreed.Version && e.Recipe == agreed.Recipe && e.Digest != "" {
if e.Version == agreed.Version && e.Recipe == agreed.Recipe && e.Digest != "" {
by[e.Digest] = append(by[e.Digest], e.Node)
}
}
@@ -41,7 +41,7 @@ func TestDigestsOfTheAgreedVersion(t *testing.T) {
{Node: "novox", Toolchain: tc("go", "1.26.0", "sha256:b")}, {Node: "ace", Toolchain: tc("go", "1.26.0", "sha256:a")},
{Node: "g14", Toolchain: tc("go", "1.26.0", "sha256:a")}, {Node: "shanks", Toolchain: tc("go", "1.25.0", "sha256:c")},
{Node: "old", Toolchain: tc("go", "1.26.0", "")}}
got := Digests(entries, buildstep.ToolchainAgreement(entries[:4]).Toolchain)
got := Digests(entries, tc("go", "1.26.0", ""))
want := []AgreedDigest{{Digest: "sha256:a", Nodes: []string{"ace", "g14"}}, {Digest: "sha256:b", Nodes: []string{"novox"}}}
if !reflect.DeepEqual(got, want) {
t.Errorf("digests = %+v, want %+v", got, want)
@@ -434,7 +434,7 @@ func main() {
Newest: newestVia(stdio.Ask), Steps: r}))
tools = append(tools, reservedTools(rr, buildAgain, time.Now, meshhandactsclient.Recorder{Ask: stdio.Ask})...)
qs, as := stdio.State(versions.PublishQueueState), stdio.State(versions.ActedState)
tools = append(tools, republishTools(readerOf(qs), readerOf(as), readerOf(reasks), buildAgain, time.Now, meshhandactsclient.Recorder{Ask: stdio.Ask})...)
tools = append(tools, republishTools(readerOf(qs), readerOf(as), buildAgain, time.Now, meshhandactsclient.Recorder{Ask: stdio.Ask})...)
ws := stdio.State(build.WaitsState)
wr := versions.Reader{Keys: ws.Keys, Get: func(key string) (json.RawMessage, bool, error) {
e, err := ws.Get(key)
@@ -28,7 +28,7 @@ func readerOf(s stdio.KeptState) versions.Reader {
// republishTools are builder_publish_again and builder_publish_queue (design 47 G §588). The publish again is a hand act:
// it is recorded in the hand-act log first, then written to reasks for the builder, which owns the queue, to answer
// with the versions it queued.
func republishTools(queue, acted, reasks versions.Reader, ask queueAsker, now func() time.Time, acts meshhandactsclient.Recorder) []stdio.Tool {
func republishTools(queue, acted versions.Reader, ask queueAsker, now func() time.Time, acts meshhandactsclient.Recorder) []stdio.Tool {
str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) }
return []stdio.Tool{
{Name: versions.PublishAgainTool, Description: versions.PublishAgainDescription,
@@ -60,6 +60,6 @@ func republishTools(queue, acted, reasks versions.Reader, ask queueAsker, now fu
}},
{Name: versions.PublishQueueTool, Description: versions.PublishQueueDescription,
Input: map[string]any{"type": "object", "properties": map[string]any{}},
Run: func(map[string]any) (any, error) { return versions.AnswerQueue(queue, acted, reasks) }},
Run: func(map[string]any) (any, error) { return versions.AnswerQueue(queue, acted) }},
}
}
@@ -13,7 +13,7 @@ import (
"mesh-delivery-builder/internal/versions"
)
// reservedTools are builder_reserved, builder_reserved_drop and builder_reserve_done (A-455) (design 47 G section 369), served as tools of the bundle. The drop is a hand act: it is recorded in the hand-act log first, then written to
// reservedTools are builder_reserved and builder_reserved_drop (design 47 G section 369), served as tools of the bundle. The drop is a hand act: it is recorded in the hand-act log first, then written to
// reasks for the builder, which owns the reservations, to answer.
//
// The seat's verb reserved is the builder's process's (design 47 G A-115, A-127), not the bundle's.
@@ -51,32 +51,5 @@ func reservedTools(rr versions.Reader, ask queueAsker, now func() time.Time, act
}
return map[string]any{"key": key, "answer": a.Answer}, nil
}},
{Name: versions.ReserveDoneTool, Description: versions.ReserveDoneDescription,
Input: map[string]any{"type": "object", "required": []string{"module", "fp", "why"}, "properties": map[string]any{
"module": map[string]any{"type": "string", "description": "the module"},
"fp": map[string]any{"type": "string", "description": "the fingerprint of its done trunk build"},
"why": map[string]any{"type": "string", "description": "why, in plain words"}}},
Run: func(args map[string]any) (any, error) {
key, value, err := build.NewReserveDone(str(args, "module"), str(args, "fp"), str(args, "why"), stdio.CallerOf(args), now())
if err != nil {
return nil, err
}
if _, err := acts.Record(meshhandactsclient.Request{What: fmt.Sprintf("reserved a version for the done build of %s at %s by hand", str(args, "module"), str(args, "fp")),
Why: str(args, "why"), Cause: versions.ReserveDoneTool, Verb: "reserve-done"}); err != nil {
return nil, fmt.Errorf("the hand act was not recorded through mesh-hand-acts, so nothing was reserved: %w", err)
}
raw, err := ask(key, value)
if err != nil {
return nil, err
}
a, err := build.DecodeReAsk(raw)
if err != nil {
return nil, err
}
if a.Refused {
return nil, errors.New(a.Answer)
}
return map[string]any{"key": key, "answer": a.Answer}, nil
}},
}
}
@@ -2,7 +2,6 @@ package main
import (
"encoding/json"
"errors"
"strings"
"testing"
"time"
@@ -45,56 +44,3 @@ func TestWaitsToolsListTheWaitsAndReleaseAfterTheHandAct(t *testing.T) {
t.Fatalf("release: %v, %d recorded, asked %+v", err, recorded, asked)
}
}
// Design 47 G A-455: builder_reserve_done records the hand act first and only then asks the builder through reasks; the builder's
// answer (the version and its queued publish) is the tool's answer, its refusal the tool's error, and an act that was not
// recorded asks nothing.
func TestReserveDoneRecordsTheHandActFirstAndAnswersTheBuilders(t *testing.T) {
var order []string
var asked []build.ReAsk
reply := func(refuse bool) func(string, []byte) (json.RawMessage, error) {
return func(key string, value []byte) (json.RawMessage, error) {
order = append(order, "asked")
a, _ := build.DecodeReAsk(value)
asked = append(asked, a)
a.Answered, a.Answer, a.Refused = time.Now(), "reserved 0.1.13 for node-tools and queued 1 versions to publish: node-tools@0.1.13", refuse
if refuse {
a.Answer = "the build key is not done"
}
return json.Marshal(a)
}
}
record := func(fail bool) meshhandactsclient.Recorder {
return meshhandactsclient.Recorder{Ask: func(string, any) (json.RawMessage, error) {
order = append(order, "recorded")
if fail {
return nil, errors.New("no responders")
}
return json.RawMessage(`{}`), nil
}}
}
run := func(ask func(string, []byte) (json.RawMessage, error), acts meshhandactsclient.Recorder, args map[string]any) (any, error) {
for _, tl := range reservedTools(versions.Reader{}, ask, time.Now, acts) {
if tl.Name == versions.ReserveDoneTool {
return tl.Run(args)
}
}
t.Fatal("the bundle serves no builder_reserve_done")
return nil, nil
}
args := map[string]any{"module": "node-tools", "fp": "src2:7df8", "why": "its build ended done with no version"}
out, err := run(reply(false), record(false), args)
if err != nil || len(order) != 2 || order[0] != "recorded" || order[1] != "asked" || asked[0].Mode != build.ModeReserveDone || asked[0].Name != "node-tools" || asked[0].FP != "src2:7df8" {
t.Fatalf("answer %v, %v; order %v, asked %+v", out, err, order, asked)
}
if ans := out.(map[string]any)["answer"].(string); !strings.Contains(ans, "reserved 0.1.13") || !strings.Contains(ans, "queued 1 versions") {
t.Errorf("the answer %q names no version and queued publish", ans)
}
if _, err := run(reply(true), record(false), args); err == nil || !strings.Contains(err.Error(), "not done") {
t.Errorf("the builder's refusal is not the tool's error: %v", err)
}
order = nil
if _, err := run(reply(false), record(true), args); err == nil || len(order) != 1 || order[0] != "recorded" {
t.Errorf("an unrecorded act was asked of the builder: %v, %v", err, order)
}
}
@@ -369,7 +369,7 @@ func (b *bed) module(t *testing.T, name, fixture, path string) string {
}
// bothToolchains are the toolchains the bed's nodes report, as a module's fingerprint names them.
var bothToolchains = map[string]builds.ToolchainUsed{"go": {Version: "go1.27", Recipe: "1"}, "typescript": {Version: "tsc5", Recipe: "1"}}
var bothToolchains = map[string]builds.ToolchainUsed{"go": {Version: "go1.27.1", Recipe: "1"}, "typescript": {Version: "tsc5", Recipe: "1"}}
// moduleWith is module with the toolchains its fingerprint was computed with.
func (b *bed) moduleWith(t *testing.T, name, fixture, path string, toolchains map[string]builds.ToolchainUsed) string {
@@ -546,7 +546,7 @@ func TestBuilderMakesADoneKeyWithTheDigests(t *testing.T) { // 1607/AC7
t.Errorf("artifact = %+v, want the digest every node built and the reference its push answered (ADR 0498 clause 1)", a)
}
}
if v.Toolchains["go"].Version != "go1.27" || v.Toolchains["typescript"].Version != "tsc5" {
if v.Toolchains["go"].Version != "go1.27.1" || v.Toolchains["typescript"].Version != "tsc5" {
t.Errorf("toolchains = %+v", v.Toolchains)
}
// A-115: the builder's process answers the seat mesh-builds' verb reserved over the bus; the reservation of a module it never
@@ -750,7 +750,7 @@ func TestAFailedKeyIsBuiltAgainAsANewRevisionFromANewModulesValueWithTheSameFing
}
}
// TestAToolchainChangeDoesNotBuildTheOldFingerprint: the fingerprint names go1.27.1, the nodes then agree on go1.28.0 (a
// TestAToolchainChangeDoesNotBuildTheOldFingerprint: the fingerprint names go1.27.1, the nodes then agree on go1.27.2 (a
// report of both nodes), and the builder writes no key for the old fingerprint although it was woken by the report; a
// module value naming go1.27.2 (the new fingerprint, computed by the indexer) is built.
func TestAToolchainChangeDoesNotBuildTheOldFingerprint(t *testing.T) { // 1615/AC4
@@ -765,15 +765,15 @@ func TestAToolchainChangeDoesNotBuildTheOldFingerprint(t *testing.T) { // 1615/A
if err := st.Purge(b.ctx, jetstream.WithPurgeSubject("mesh.seat.node-build-go.event.toolchain-reported.>")); err != nil {
t.Fatal(err)
}
b.report(t, "mesh-build-go", "n1", "go", "go1.28.0", "1")
b.report(t, "mesh-build-go", "n2", "go", "go1.28.0", "1")
b.module(t, "netcheck", "catalogue-a67f0196-netcheck.module.json", "modules/netcheck") // names go1.27
b.report(t, "mesh-build-go", "n1", "go", "go1.27.2", "1")
b.report(t, "mesh-build-go", "n2", "go", "go1.27.2", "1")
b.module(t, "netcheck", "catalogue-a67f0196-netcheck.module.json", "modules/netcheck") // names go1.27.1
b.start(t, "n1", nil)
// Another module, whose fingerprint names the new toolchain, is built; by then the old one has been looked at.
b.moduleCommit(t, "netcheck2", "catalogue-a67f0196-netcheck.module.json", "modules/netcheck",
map[string]builds.ToolchainUsed{"go": {Version: "go1.28", Recipe: "1"}, "typescript": {Version: "tsc5", Recipe: "1"}}, commit)
map[string]builds.ToolchainUsed{"go": {Version: "go1.27.2", Recipe: "1"}, "typescript": {Version: "tsc5", Recipe: "1"}}, commit)
b.await(t, "netcheck2", isState(builds.BuildDone))
// The builder never agreed on go1.27, not for a moment: that is what would build the old fingerprint.
// The builder never agreed on go1.27.1, not for a moment: that is what would build the old fingerprint.
info, err := st.Info(b.ctx)
if err != nil {
t.Fatal(err)
@@ -786,12 +786,12 @@ func TestAToolchainChangeDoesNotBuildTheOldFingerprint(t *testing.T) { // 1615/A
if err != nil {
t.Fatal(err)
}
if strings.HasSuffix(m.Subject, ".toolchain-agreed") && strings.Contains(string(m.Data), `"version":"go1.27"`) {
t.Errorf("the builder said the nodes agree on go1.27: %s", m.Data)
if strings.HasSuffix(m.Subject, ".toolchain-agreed") && strings.Contains(string(m.Data), `"version":"go1.27.1"`) {
t.Errorf("the builder said the nodes agree on go1.27.1: %s", m.Data)
}
}
if _, err := b.builds.Get(b.ctx, mustKey(t, "netcheck")); err == nil {
t.Error("the old fingerprint, naming go1.27, was built with go1.28.0; the log: " + b.out.String())
t.Error("the old fingerprint, naming go1.27.1, was built with go1.27.2; the log: " + b.out.String())
}
if n := b.fake.count("mesh-build-go.build_go_build"); n != 1 {
t.Errorf("build_go_build was called %d times, want 1 (the new fingerprint on one node only)", n)
@@ -234,36 +234,17 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
acts := versions.Acts{Numberer: numberer, DropReserved: reserved.dropped, Reconsider: b.Reconsider}
// A reserved version left unpublished is published again, by the builder on an event or by a person's tool (design 47 G
// §588): its states publish-queue and acted, bound once the bus holds them.
republishMax, err := republishMaxOf(getenv("MESH_BUILDER_REPUBLISH_MAX_FILE"))
if err != nil {
return err
}
queueKV := process.AwaitBucket(ctx, js, conn, own.queue, "no version is published again on an event or by builder_publish_again", logger.Printf)
actedKV := process.AwaitBucket(ctx, js, conn, own.acted, "no version is published again on an event or by builder_publish_again", logger.Printf)
republish := &versions.Republisher{Reserved: reserved.r, Queue: awaitedQueue{awaitedReserved{awaitedVersions{b: queueKV, bucket: own.queue}}},
Entry: entryVia(calls.Call), Send: sendVia(calls.Call),
Unreleased: reserved.unreleasedWords, Logf: logger.Printf,
Acted: awaitedReserved{awaitedVersions{b: actedKV, bucket: own.acted}}, Entry: entryVia(calls.Call), Send: sendVia(calls.Call),
Unreleased: reserved.unreleasedWords, Max: republishMax, Logf: logger.Printf,
Holders: holders} // a registry seat's holder from the seat-holders view, from the start (design 47 G A-416, A-416a)
publishSet, err := publishSettingsOf(getenv)
if err != nil {
return err
}
releasedKept, err := numberFile(getenv("MESH_BUILDER_RELEASED_KEPT_FILE"), versions.DefaultReleasedKept)
if err != nil {
return err
}
reasksKV := process.AwaitBucket(ctx, js, conn, builds.StateBucket(cred.Module, build.ReAsksState), "builder_ask_again is not heard; an ask that can no longer be answered is still asked again by itself", logger.Printf)
// A version's publish is the builder's own work until its release is written (design 47 G A-457a): an item of the re-ask state.
items := &publishItems{reasks: reasksKV.KV, set: publishSet, now: time.Now, logf: logger.Printf,
reserved: func(ctx context.Context, module, version string) (versions.ReservedVersion, bool, error) {
return reserved.r.Of(ctx, module, version)
},
republish: republish.Republish, endedWith: reserved.unreleased,
allDone: func(ctx context.Context) ([]versions.ReservedVersion, error) {
all, err := reserved.r.All(ctx)
return builtOf(all), err
}}
reasksKV.Ready(func(jetstream.KeyValue) { items.AskAllDone(ctx) })
wirePublishItems(reserved, items, awaitedReserved{awaitedVersions{b: actedKV, bucket: own.acted}}, releasedKept)
acts.Republish = republish.Republish
// builder_reserve_done: a done trunk build that reserved none is given a version, and its publish queued as publish again does.
acts.ReserveDone = reserveDoneAct(b.ReserveDone, republish.Republish, logger.Printf)
again := &republishing{rp: republish}
// The operator is asked once a key's event builds are spent (design 47 G §568): the builder's asker on the seat operator-channel.
decided := &decisions{subject: oc.SubjectDecided(cred.Module)}
@@ -405,6 +386,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// And the four registry seats' published, refused and removed (DLV-B4), which move a numbered version.
seatHeard := seatHearer{n: numberer, b: b, logf: logger.Printf, pub: pubs, res: reserved}
events := func(m jetstream.Msg) {
if again.heard(ctx, m, logger.Printf) { // the publish again's own subjects, left for the bus to give again when its state cannot take them
return
}
if decided.heard(m) { // the operator's answer to an ask about a key (§568)
return
}
@@ -477,8 +461,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// A person asks a key's build again with builder_ask_again, which writes the reasks state (issue 1945): watched, as
// every other input. The builder asks again by itself without it.
// A state not on the bus yet at this start is watched once it appears (issue 1790).
reasksKV.Ready(func(kv jetstream.KeyValue) {
eachReAsk := func(e jetstream.KeyValueEntry) { reAsked(ctx, b, acts, items, kv, e, logger.Printf) }
process.AwaitBucket(ctx, js, conn, builds.StateBucket(cred.Module, build.ReAsksState), "builder_ask_again is not heard; an ask that can no longer be answered is still asked again by itself", logger.Printf).Ready(func(kv jetstream.KeyValue) {
eachReAsk := func(e jetstream.KeyValueEntry) { reAsked(ctx, b, acts, kv, e, logger.Printf) }
if rw, err := settle(ctx, kv, eachReAsk); err != nil {
process.NotWatched(logger.Printf, builds.StateBucket(cred.Module, build.ReAsksState), err, "builder_ask_again is not heard until the builder starts again")
} else {
@@ -1,216 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"io/fs"
"os"
"strconv"
"strings"
"time"
"github.com/nats-io/nats.go/jetstream"
"mesh-delivery-builder/internal/build"
"mesh-delivery-builder/internal/versions"
)
// **A version's publish is the builder's own work until its release is written** (design 47 G A-457a, §588): each reservation of a
// done build is a publish item in the builder's re-ask state, the mechanism builder_reserve_done and publish again use. It is
// asked when the build ends and at the builder's start; each try queues the publish and writes the item again with one more try
// and a longer pause; the release deletes it; after its tries it ends loudly and stays readable in builder_publish_queue.
// publishSettings are the settings publish-pause, publish-pause-max and publish-tries.
type publishSettings struct {
Pause, PauseMax time.Duration
Tries int
}
// defaultPublishSettings are the settings' defaults: 30 s, 30 min and 8 tries.
var defaultPublishSettings = publishSettings{Pause: 30 * time.Second, PauseMax: 30 * time.Minute, Tries: 8}
// pauseAfter is the pause after the try-th try: the setting doubled per try, up to the maximum.
func (s publishSettings) pauseAfter(try int) time.Duration {
d := s.Pause
for i := 1; i < try && d < s.PauseMax; i++ {
d *= 2
}
return min(d, s.PauseMax)
}
// publishItems asks, tries and ends the publish items.
type publishItems struct {
reasks func() jetstream.KeyValue // the re-ask state, nil until it is bound
reserved func(ctx context.Context, module, version string) (versions.ReservedVersion, bool, error)
republish func(ctx context.Context, module, version string) ([]string, error)
endedWith func(ctx context.Context, v versions.ReservedVersion, why string) // raises the finding version-unreleased
allDone func(ctx context.Context) ([]versions.ReservedVersion, error)
set publishSettings
now func() time.Time
after func(time.Duration) <-chan time.Time
logf func(string, ...any)
}
func (p *publishItems) wait(ctx context.Context, d time.Duration) bool {
if d <= 0 {
return ctx.Err() == nil
}
after := p.after
if after == nil {
after = time.After
}
select {
case <-after(d):
return true
case <-ctx.Done():
return false
}
}
// Ask writes the publish item of a version whose build is done, when none stands: its first try is now.
func (p *publishItems) Ask(ctx context.Context, module, version string) error {
kv := p.reasks()
if kv == nil {
return errors.New("the re-ask state is not on the bus yet")
}
raw, err := json.Marshal(build.ReAsk{V: 1, Why: "the build ended with its version unreleased", By: "the builder", At: p.now().UTC(), Mode: build.ModePublish, Module: module, Version: version, NextTry: p.now().UTC()})
if err != nil {
return err
}
if _, err := kv.Create(ctx, versions.PublishItemKey(module, version), raw); err != nil && !errors.Is(err, jetstream.ErrKeyExists) {
return fmt.Errorf("the publish item of %s %s cannot be written: %w", module, version, err)
}
return nil
}
// AskAllDone asks the publish item of every reservation whose build is done and that has no release: at the builder's start, so
// what stood before this change moves too, whatever its artifacts.
func (p *publishItems) AskAllDone(ctx context.Context) {
vs, err := p.allDone(ctx)
if err != nil {
p.logf("the reservations of done builds are not asked to publish at the start: %v", err)
return
}
for _, v := range vs {
if err := p.Ask(ctx, v.Module, v.Version); err != nil {
p.logf("%v", err)
}
}
}
// Done deletes a version's publish item: its release is written.
func (p *publishItems) Done(ctx context.Context, module, version string) {
kv := p.reasks()
if kv == nil {
return
}
if err := kv.Delete(ctx, versions.PublishItemKey(module, version)); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
p.logf("the publish item of %s %s is not deleted: %v", module, version, err)
}
}
// Try waits for the item's next try, then, while the version is still reserved, queues its publish and writes the item again with
// one more try and the pause after it; once the tries are spent it ends the item loudly. A version no longer reserved (released,
// dropped or taken over) has its item deleted; one whose artifacts are published keeps it until its release is written.
func (p *publishItems) Try(ctx context.Context, kv jetstream.KeyValue, e jetstream.KeyValueEntry, r build.ReAsk) {
if !p.wait(ctx, r.NextTry.Sub(p.now())) {
return
}
cur, err := kv.Get(ctx, e.Key())
if err != nil || cur.Revision() != e.Revision() {
return // deleted by the release, or written again
}
v, held, err := p.reserved(ctx, r.Module, r.Version)
if err != nil {
p.logf("%s %s: its reservation cannot be read, the publish item waits: %v", r.Module, r.Version, err)
return
}
if !held { // dropped, taken over or released: never because its artifacts are published (A-456b)
_ = kv.Delete(ctx, e.Key())
return
}
if r.Tries >= p.set.Tries {
r.Answer = fmt.Sprintf("the publish of %s %s was tried %d times and the version is not released; builder_publish_again tries it again", r.Module, r.Version, r.Tries)
r.Answered = p.now().UTC()
p.logf("%s: %s", e.Key(), r.Answer)
if p.endedWith != nil {
p.endedWith(ctx, v, "the publish ended after "+fmt.Sprint(r.Tries)+" tries")
}
} else {
r.Tries++
if _, err := p.republish(ctx, r.Module, r.Version); err != nil {
p.logf("%s %s: try %d queued no publish: %v", r.Module, r.Version, r.Tries, err)
}
r.NextTry = p.now().UTC().Add(p.set.pauseAfter(r.Tries))
}
data, err := json.Marshal(r)
if err != nil {
return
}
if _, err := kv.Update(ctx, e.Key(), data, e.Revision()); err != nil && !errors.Is(err, jetstream.ErrKeyExists) && !errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
p.logf("%s: the publish item could not be written: %v", e.Key(), err)
}
}
// numberFile reads a setting's number from the file the manifest declares: no file or an empty one is def; a content that is no
// number of at least one is an error naming the file.
func numberFile(path string, def int) (int, error) {
if path == "" {
return def, nil
}
raw, err := os.ReadFile(path)
if errors.Is(err, fs.ErrNotExist) {
return def, nil
}
if err != nil {
return 0, err
}
v := strings.TrimSpace(string(raw))
if v == "" {
return def, nil
}
n, err := strconv.Atoi(v)
if err != nil || n < 1 {
return 0, fmt.Errorf("%s holds %q, which is not a number of at least one", path, v)
}
return n, nil
}
// publishSettingsOf reads publish-pause (seconds), publish-pause-max (minutes) and publish-tries from their files.
func publishSettingsOf(getenv func(string) string) (publishSettings, error) {
s := defaultPublishSettings
pause, err := numberFile(getenv("MESH_BUILDER_PUBLISH_PAUSE_FILE"), int(s.Pause/time.Second))
if err != nil {
return s, err
}
pmax, err := numberFile(getenv("MESH_BUILDER_PUBLISH_PAUSE_MAX_FILE"), int(s.PauseMax/time.Minute))
if err != nil {
return s, err
}
tries, err := numberFile(getenv("MESH_BUILDER_PUBLISH_TRIES_FILE"), s.Tries)
if err != nil {
return s, err
}
return publishSettings{Pause: time.Duration(pause) * time.Second, PauseMax: time.Duration(pmax) * time.Minute, Tries: tries}, nil
}
// wirePublishItems joins the reservations to the publish items (design 47 G A-457a, A-456a, A-456b): a done build asks its item, a
// release, a drop and a takeover end it, and a release is recorded in releases, the last kept of each module.
func wirePublishItems(rv *reservations, items *publishItems, releases versions.ReserveStore, kept int) {
rv.built, rv.released = items.Ask, items.Done
rv.r.ReleasedKept, rv.r.Releases = kept, releases
}
// builtOf are the reservations whose build is done, whatever their artifacts' state: each is asked its publish item at the start
// (design 47 G A-456b), which goes only with its release.
func builtOf(all []versions.ReservedVersion) []versions.ReservedVersion {
var out []versions.ReservedVersion
for _, v := range all {
if !v.BuiltAt.IsZero() {
out = append(out, v)
}
}
return out
}
@@ -1,276 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"mesh-delivery-builder/internal/build"
"mesh-delivery-builder/internal/versions"
)
// itemKV is the re-ask state in memory: one revision per key.
type itemKV struct {
jetstream.KeyValue
vals map[string][]byte
revs map[string]uint64
rev uint64
}
type itemEntry struct {
jetstream.KeyValueEntry
key string
value []byte
rev uint64
}
func (e itemEntry) Key() string { return e.key }
func (e itemEntry) Value() []byte { return e.value }
func (e itemEntry) Revision() uint64 { return e.rev }
func (e itemEntry) Operation() jetstream.KeyValueOp { return jetstream.KeyValuePut }
func (k *itemKV) entry(key string) jetstream.KeyValueEntry {
return itemEntry{key: key, value: k.vals[key], rev: k.revs[key]}
}
func (k *itemKV) Get(_ context.Context, key string) (jetstream.KeyValueEntry, error) {
if _, ok := k.vals[key]; !ok {
return nil, jetstream.ErrKeyNotFound
}
return k.entry(key), nil
}
func (k *itemKV) Create(_ context.Context, key string, v []byte, _ ...jetstream.KVCreateOpt) (uint64, error) {
if _, ok := k.vals[key]; ok {
return 0, jetstream.ErrKeyExists
}
k.rev++
k.vals[key], k.revs[key] = v, k.rev
return k.rev, nil
}
func (k *itemKV) Update(_ context.Context, key string, v []byte, rev uint64) (uint64, error) {
if k.revs[key] != rev {
return 0, jetstream.ErrKeyExists
}
k.rev++
k.vals[key], k.revs[key] = v, k.rev
return k.rev, nil
}
func (k *itemKV) Delete(_ context.Context, key string, _ ...jetstream.KVDeleteOpt) error {
delete(k.vals, key)
return nil
}
func decodeItem(t *testing.T, raw []byte) build.ReAsk {
t.Helper()
r, err := build.DecodeReAsk(raw)
if err != nil {
t.Fatal(err)
}
return r
}
// TestAPublishItemIsTriedWithAGrowingPauseAndEndsLoudly: design 47 G A-457a — a version's publish item is asked once, each try queues
// the publish and writes the item again with one more try and the pause doubled up to its maximum, the release deletes it, and
// after the setting's tries it ends with its words, raises the condition and stays; a version no longer reserved has it deleted.
func TestAPublishItemIsTriedWithAGrowingPauseAndEndsLoudly(t *testing.T) {
ctx := context.Background()
kv := &itemKV{vals: map[string][]byte{}, revs: map[string]uint64{}}
now := time.Date(2026, 11, 17, 1, 0, 0, 0, time.UTC)
var waited []time.Duration
var queued, ended []string
reserved := true
v := versions.ReservedVersion{Module: "m", Version: "0.1.2", Artifacts: map[string]string{"m:a": "sha256:a"}, BuiltAt: now}
p := &publishItems{reasks: func() jetstream.KeyValue { return kv }, set: publishSettings{Pause: 30 * time.Second, PauseMax: 2 * time.Minute, Tries: 4},
now: func() time.Time { return now }, logf: t.Logf,
after: func(d time.Duration) <-chan time.Time {
waited = append(waited, d)
now = now.Add(d)
c := make(chan time.Time, 1)
c <- now
return c
},
reserved: func(_ context.Context, _, ver string) (versions.ReservedVersion, bool, error) {
if ver == "0.3.0" { // every artifact published, its release not written yet
return versions.ReservedVersion{Module: "m", Version: ver, Artifacts: map[string]string{"m:a": "sha256:a"}, Published: map[string]bool{"m:a": true}, BuiltAt: now}, true, nil
}
return v, reserved, nil
},
republish: func(_ context.Context, m, ver string) ([]string, error) {
queued = append(queued, m+" "+ver)
return nil, errors.New("no holder")
},
endedWith: func(_ context.Context, v versions.ReservedVersion, why string) {
ended = append(ended, v.Version+": "+why)
}}
key := versions.PublishItemKey("m", "0.1.2")
if err := p.Ask(ctx, "m", "0.1.2"); err != nil || p.Ask(ctx, "m", "0.1.2") != nil || len(kv.vals) != 1 {
t.Fatalf("asked: %v, %d items, want one item however often it is asked", err, len(kv.vals))
}
for i := 0; i < 5; i++ {
e, _ := kv.Get(ctx, key)
p.Try(ctx, kv, e, decodeItem(t, e.Value()))
}
got := decodeItem(t, kv.vals[key])
if len(queued) != 4 || got.Tries != 4 || got.Answer == "" || got.Answered.IsZero() || len(ended) != 1 || !strings.Contains(ended[0], "4 tries") {
t.Fatalf("queued %v, item %+v, ended %v: want four tries, then the end said with its words and the condition", queued, got, ended)
}
if want := []time.Duration{30 * time.Second, time.Minute, 2 * time.Minute, 2 * time.Minute}; len(waited) != len(want) || waited[0] != want[0] || waited[1] != want[1] || waited[2] != want[2] || waited[3] != want[3] {
t.Errorf("pauses before the tries and the end = %v, want %v (doubled, capped)", waited, want)
}
// An item whose artifacts are all published stays until its release is written, and is tried (A-456b).
queued = nil
_ = p.Ask(ctx, "m", "0.3.0")
pk := versions.PublishItemKey("m", "0.3.0")
pe, _ := kv.Get(ctx, pk)
p.Try(ctx, kv, pe, decodeItem(t, pe.Value()))
if _, ok := kv.vals[pk]; !ok || len(queued) != 1 || decodeItem(t, kv.vals[pk]).Tries != 1 {
t.Errorf("an item whose artifacts are published was deleted or not tried: %v, queued %v", kv.vals, queued)
}
kv.vals, kv.revs = map[string][]byte{}, map[string]uint64{}
// At the start every reservation of a done build with no release is asked, one item each.
kv.vals, kv.revs = map[string][]byte{}, map[string]uint64{}
p.allDone = func(context.Context) ([]versions.ReservedVersion, error) {
published := versions.ReservedVersion{Module: "n", Version: "0.2.0", Artifacts: map[string]string{"n:a": "sha256:a"}, Published: map[string]bool{"n:a": true}, BuiltAt: now}
return builtOf([]versions.ReservedVersion{v, published, {Module: "o", Version: "0.1.0"}}), nil // the last build has not ended
}
p.AskAllDone(ctx)
if len(kv.vals) != 2 {
t.Errorf("the start asked %d items, want one per done unreleased reservation", len(kv.vals))
}
// A released version's item is deleted; a version no longer reserved has it deleted at its next try.
p.Done(ctx, "m", "0.1.2")
p.Done(ctx, "n", "0.2.0")
if len(kv.vals) != 0 {
t.Errorf("the release left %v", kv.vals)
}
_ = p.Ask(ctx, "m", "0.1.2")
reserved = false
e, _ := kv.Get(ctx, key)
p.Try(ctx, kv, e, decodeItem(t, e.Value()))
if len(kv.vals) != 0 {
t.Errorf("a dropped reservation left its item %v", kv.vals)
}
}
// TestThePublishSettingsAreReadFromTheirFiles: design 47 G A-457a — the three settings, 30 s, 30 min and 8 tries unless set, and a
// content that is no number is refused naming its file.
func TestThePublishSettingsAreReadFromTheirFiles(t *testing.T) {
dir := t.TempDir()
write := func(name, v string) string {
p := filepath.Join(dir, name)
if err := os.WriteFile(p, []byte(v), 0o600); err != nil {
t.Fatal(err)
}
return p
}
env := map[string]string{}
got, err := publishSettingsOf(func(k string) string { return env[k] })
if err != nil || got != defaultPublishSettings || got.Pause != 30*time.Second || got.PauseMax != 30*time.Minute || got.Tries != 8 {
t.Fatalf("defaults = %+v (%v)", got, err)
}
env["MESH_BUILDER_PUBLISH_PAUSE_FILE"], env["MESH_BUILDER_PUBLISH_PAUSE_MAX_FILE"], env["MESH_BUILDER_PUBLISH_TRIES_FILE"] = write("p", "5\n"), write("m", "2"), write("t", "3")
if got, err = publishSettingsOf(func(k string) string { return env[k] }); err != nil || got != (publishSettings{5 * time.Second, 2 * time.Minute, 3}) {
t.Errorf("set = %+v (%v)", got, err)
}
env["MESH_BUILDER_PUBLISH_TRIES_FILE"] = write("bad", "many")
if _, err = publishSettingsOf(func(k string) string { return env[k] }); err == nil || !strings.Contains(err.Error(), "bad") {
t.Errorf("a content that is no number: %v", err)
}
}
// TestThePublishItemsAreWiredToTheReservationsAndTheReAskWatch: design 47 G A-457a, A-456b — a mode-publish value of the re-ask state
// reaches the item's try through reAsked; a done build gets an item; the catalog's version-released, a drop and a takeover delete it
// and the release is kept by module.
func TestThePublishItemsAreWiredToTheReservationsAndTheReAskWatch(t *testing.T) {
ctx := context.Background()
kv := &itemKV{vals: map[string][]byte{}, revs: map[string]uint64{}}
st := &memFindings{vals: map[string][]byte{}}
found := &findings{module: builds.BuilderModule, w: st, logf: t.Logf, now: func() time.Time { return time.Unix(0, 0) }}
call := func(string, any) (json.RawMessage, error) {
return json.Marshal(meshcatalog.VersionsAnswer{Module: "m"})
}
rv := newReservations(&memReserve{vals: map[string][]byte{}, revs: map[string]uint64{}}, call, func() jetstream.KeyValue { return &boundsKV{keys: map[string]bool{}} }, found, nil, t.Logf)
rv.filled.Store(true)
tried := make(chan string, 4)
items := &publishItems{reasks: func() jetstream.KeyValue { return kv }, set: defaultPublishSettings, now: time.Now, logf: t.Logf,
reserved: func(ctx context.Context, m, v string) (versions.ReservedVersion, bool, error) {
return rv.r.Of(ctx, m, v)
},
republish: func(_ context.Context, m, v string) ([]string, error) { tried <- m + " " + v; return nil, nil }}
releases := &memReserve{vals: map[string][]byte{}, revs: map[string]uint64{}}
wirePublishItems(rv, items, releases, 2)
done := func(fp string) versions.ReservedVersion {
v, err := rv.Reserve(ctx, "m", "0.1", []string{"m:a"}, versions.Source{Fingerprint: fp})
if err != nil {
t.Fatal(err)
}
if err := rv.Built(ctx, "m", v.Version, map[string]string{"m:a": "sha256:a"}, nil); err != nil {
t.Fatal(err)
}
return v
}
first := done("src2:one")
key := versions.PublishItemKey("m", first.Version)
if _, ok := kv.vals[key]; !ok {
t.Fatalf("a done build has no publish item: %v", kv.vals)
}
// A build whose artifacts are all published still gets its item, until its release is written (A-456b).
if _, _, err := rv.r.Published(ctx, versions.NameOf("m", "a"), first.Version, "sha256:a"); err != nil {
t.Fatal(err)
}
items.Done(ctx, "m", first.Version)
if err := rv.Built(ctx, "m", first.Version, map[string]string{"m:a": "sha256:a"}, nil); err != nil {
t.Fatal(err)
}
if _, ok := kv.vals[key]; !ok {
t.Fatalf("a build with every artifact published has no item: %v", kv.vals)
}
// The watch hands the item to its try, which queues the publish.
e, _ := kv.Get(ctx, key)
reAsked(ctx, nil, versions.Acts{}, items, kv, e, t.Logf)
select {
case got := <-tried:
if got != "m "+first.Version {
t.Errorf("tried %q", got)
}
case <-time.After(10 * time.Second):
t.Fatal("a mode-publish value was not handed to the item's try")
}
// A takeover by another build deletes the older's item and asks the newer's.
second := done("src2:two")
if _, ok := kv.vals[key]; ok {
t.Errorf("the taken-over reservation kept its item")
}
secondKey := versions.PublishItemKey("m", second.Version)
if _, ok := kv.vals[secondKey]; !ok {
t.Fatalf("the newer build has no item: %v", kv.vals)
}
// version-released deletes the item and keeps the release.
msg := &wakeMsg{subject: meshcatalog.SubjectVersionReleased, data: []byte(`{"v":1,"module":"m","version":"` + second.Version + `","source":1,"artifacts":[]}`)}
if !rv.heard(ctx, msg) || !msg.acked {
t.Fatal("version-released was not taken in")
}
if _, ok := kv.vals[secondKey]; ok {
t.Errorf("a released version kept its publish item")
}
if ok, _ := rv.ReleasedFor(ctx, "m", "src2:two"); !ok {
t.Errorf("the release is not kept by fingerprint")
}
// A drop deletes the item too.
third := done("src2:three")
if _, err := rv.dropped(ctx, "m", "0.1", "op", "never", time.Unix(0, 0)); err != nil {
t.Fatal(err)
}
if _, ok := kv.vals[versions.PublishItemKey("m", third.Version)]; ok {
t.Errorf("a dropped reservation kept its item")
}
}
@@ -33,7 +33,7 @@ const buildAgainBound = 15 * time.Second
// reAsked answers one value of the reasks state (the tool builder_ask_again, issue 1945): a value not yet answered asks the
// key's waiting lanes again and is written back, by compare-and-set at its revision, with what the builder did. A value
// answered already (this builder's own write) or deleted is left.
func reAsked(ctx context.Context, b askAgainer, acts versions.Acts, items *publishItems, kv jetstream.KeyValue, e jetstream.KeyValueEntry, logf func(string, ...any)) {
func reAsked(ctx context.Context, b askAgainer, acts versions.Acts, kv jetstream.KeyValue, e jetstream.KeyValueEntry, logf func(string, ...any)) {
if e.Operation() != jetstream.KeyValuePut {
return
}
@@ -45,13 +45,6 @@ func reAsked(ctx context.Context, b askAgainer, acts versions.Acts, items *publi
if r.Answer != "" {
return
}
if r.Mode == build.ModePublish {
// A version's publish item (design 47 G A-457a): its try waits for its pause, so it is made beside the watch.
if items != nil {
go items.Try(ctx, kv, e, r)
}
return
}
if r.Mode == build.ModeBuild {
// The answer comes when the claim and the asks are written, so it is made beside the watch, not in it.
go builtAgain(ctx, b, kv, e, r, logf)
@@ -65,7 +58,7 @@ func reAsked(ctx context.Context, b askAgainer, acts versions.Acts, items *publi
released(ctx, b, kv, e, r, logf)
return
}
if r.Mode == build.ModeRepublish || r.Mode == build.ModeFloorAgain || r.Mode == build.ModeOldFormDrop || r.Mode == build.ModeReservedDrop || r.Mode == build.ModeReserveDone {
if r.Mode == build.ModeRepublish || r.Mode == build.ModeFloorAgain || r.Mode == build.ModeOldFormDrop || r.Mode == build.ModeReservedDrop {
// A version's hand act (design 51 *Tools of the builder*) may build again, so it is answered beside the watch.
go versionActed(ctx, acts, kv, e, r, logf)
return
@@ -208,12 +201,6 @@ func versionActed(ctx context.Context, acts versions.Acts, kv jetstream.KeyValue
r.Answer = "queued " + fmt.Sprint(len(r.Queued)) + " versions to publish again: " + strings.Join(r.Queued, ", ") + "; builder_publish_queue shows them"
}
}
} else if r.Mode == build.ModeReserveDone {
if acts.ReserveDone == nil {
err = errors.New("this builder keeps no reservations")
} else {
r.Answer, err = acts.ReserveDone(ctx, r.Name, r.FP, r.By, r.Why, r.At)
}
} else if r.Mode == build.ModeReservedDrop {
if acts.DropReserved == nil {
err = errors.New("this builder keeps no reservations")
@@ -3,11 +3,17 @@ package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"io/fs"
"os"
"strconv"
"strings"
"sync/atomic"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/gopackageregistry"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
@@ -71,8 +77,32 @@ func publishBody(p versions.Publish) gopackageregistry.PublishInput {
return gopackageregistry.PublishInput{Commit: p.Commit, Digest: p.Digest, Fingerprint: p.Fingerprint, Kind: p.Kind, Name: p.Name, Transfer: p.Transfer, Version: p.Version}
}
// republishing starts the publish again once both its states are bound: the queue is taken up again (design 47 G §588). A version's
// publish is asked when its build ends and at the start (A-457a), never waited on a running or manifest-reads-changed.
// republishMaxOf reads the setting republish-max-versions from the file the manifest declares (design 47 G §633, A-71's
// form): no file or an empty one is the default; a content that is no positive number is an error naming the file.
func republishMaxOf(path string) (int, error) {
if path == "" {
return versions.DefaultRepublishMax, nil
}
raw, err := os.ReadFile(path)
if errors.Is(err, fs.ErrNotExist) {
return versions.DefaultRepublishMax, nil
}
if err != nil {
return 0, err
}
v := strings.TrimSpace(string(raw))
if v == "" {
return versions.DefaultRepublishMax, nil
}
n, err := strconv.Atoi(v)
if err != nil || n < 1 {
return 0, fmt.Errorf("%s holds %q, which is not a number of versions", path, v)
}
return n, nil
}
// republishing starts the publish again once both its states are bound: the queue is taken up again and the last running
// of every node's module is read (§556), acted on where its sequence is newer than the kept one.
type republishing struct {
rp *versions.Republisher
ready atomic.Int32
@@ -86,5 +116,32 @@ func (g *republishing) bound(ctx context.Context, js jetstream.JetStream, logf f
if err := g.rp.Resume(ctx); err != nil {
logf("the queued publishes are not taken up again: %v", err)
}
err := lastPerSubject(ctx, js, nodeengine.StreamRunning, nodeengine.SubjectRunningAll, func(subject string, data []byte, seq uint64) {
if _, err := g.rp.Heard(ctx, subject, data, seq); err != nil {
logf("the running %s is not acted on at the start: %v", subject, err)
}
})
if err != nil {
logf("the running of the node-engine seat is not read for the publish again at the start: %v", err)
}
}
}
// heard hands a message of the events consumer to the publish again: one of its subjects whose state cannot take it is left
// for the bus to give again (Nak); every message goes on to the other parts.
func (g *republishing) heard(ctx context.Context, m jetstream.Msg, logf func(string, ...any)) (stop bool) {
meta, err := m.Metadata()
if err != nil {
return false
}
handled, err := g.rp.Heard(ctx, m.Subject(), m.Data(), meta.Sequence.Stream)
if handled && err != nil {
logf("%v; heard again", err)
if nakErr := m.Nak(); nakErr != nil {
logf("leaving %s for the bus to give again: %v", m.Subject(), nakErr)
}
givenUp(m, "its event", err, logf)
return true
}
return false
}
@@ -4,6 +4,8 @@ import (
"context"
"encoding/json"
"errors"
"os"
"path/filepath"
"strings"
"testing"
"time"
@@ -57,6 +59,18 @@ func TestRepublishSend(t *testing.T) {
t.Errorf("%s: asked %q, error %v, want %q", name, asked, err, c.want)
}
}
dir := t.TempDir()
for content, want := range map[string]int{"": 50, "7": 7} {
path := filepath.Join(dir, "max")
_ = os.WriteFile(path, []byte(content), 0o600)
if got, err := republishMaxOf(path); err != nil || got != want {
t.Errorf("republishMaxOf(%q) = %d, %v, want %d", content, got, err, want)
}
}
_ = os.WriteFile(filepath.Join(dir, "bad"), []byte("0"), 0o600)
if _, err := republishMaxOf(filepath.Join(dir, "bad")); err == nil {
t.Error("a setting of 0 is accepted")
}
if _, _, err := build.NewRepublish("", "0.1.3", "why", "me", versionsNow()); err == nil {
t.Error("a version with no module is accepted")
}
@@ -40,16 +40,13 @@ func (s awaitedReserved) Delete(ctx context.Context, key string) error {
// reservations is the builder's reserving: the Reserver, the fill the catalog said, the bounds of a built reservation's
// publish and the findings.
type reservations struct {
r *versions.Reserver
// built asks a done build's publish (design 47 G A-457a), released ends its item; nil are none.
built func(ctx context.Context, module, version string) error
released func(ctx context.Context, module, version string)
filled atomic.Bool
bounds func() jetstream.KeyValue
found *findings
bound time.Duration
wake func(context.Context)
logf func(string, ...any)
r *versions.Reserver
filled atomic.Bool
bounds func() jetstream.KeyValue
found *findings
bound time.Duration
wake func(context.Context)
logf func(string, ...any)
// call asks a seat's verb: the git seat's tag (design 47 G §370).
call toolCaller
// say says package-published on the seat mesh-builds; nil says none.
@@ -75,41 +72,11 @@ func newReservations(store versions.ReserveStore, call toolCaller, bounds func()
}
return out, nil
}}
rv.r.Took = rv.took
return rv
}
// took is a build's takeover of the reservation of an older build never released (design 47 G A-455): the older version is
// dropped, its bound ended and its finding cleared (the newer build's reservation has its own), and the builder's log says so.
func (rv *reservations) took(ctx context.Context, older, newer versions.ReservedVersion) {
if older.Fingerprint == newer.Fingerprint {
rv.logf("%s: the build of %s reserves again: %s takes the place of %s", newer.Module, newer.Fingerprint, newer.Version, older.Version)
} else {
rv.logf("%s: the reservation of %s (build of %s, never released) is dropped: the build of %s takes the series over and reserves %s", newer.Module, older.Version, older.Fingerprint, newer.Fingerprint, newer.Version)
}
if rv.released != nil {
rv.released(ctx, older.Module, older.Version) // its publish item goes with its reservation (A-456b)
}
rv.dropBound(ctx, versions.ReservedKey(older.Module, older.Series))
rv.found.clear(ctx, versions.VersionUnreleased, meshconditionsclient.ScopeModule, older.Module)
}
var _ build.Reservations = (*reservations)(nil)
// ReservedFor is the reservation made for the build of fingerprint.
func (rv *reservations) ReservedFor(ctx context.Context, module, fingerprint string) (versions.ReservedVersion, bool, error) {
all, err := rv.r.All(ctx)
if err != nil {
return versions.ReservedVersion{}, false, err
}
for _, v := range all {
if v.Module == module && v.Fingerprint == fingerprint {
return v, true, nil
}
}
return versions.ReservedVersion{}, false, nil
}
func (rv *reservations) Filled() bool { return rv.filled.Load() }
func (rv *reservations) Reserve(ctx context.Context, module, series string, artifacts []string, src versions.Source) (versions.ReservedVersion, error) {
@@ -129,11 +96,6 @@ func (rv *reservations) Built(ctx context.Context, module, version string, diges
return err
}
rv.arm(ctx, cur)
if rv.built != nil {
if err := rv.built(ctx, module, version); err != nil {
rv.logf("%v", err)
}
}
// A published heard before the digests were recorded marks its artifact now, said as one heard after (design 47 G A-415).
marked, unmatched, err := rv.r.Kept(ctx, module, version)
if err != nil {
@@ -141,10 +103,6 @@ func (rv *reservations) Built(ctx context.Context, module, version string, diges
}
for _, h := range unmatched {
rv.logf("a published of %s@%s kept on %s %s marks no artifact: its digest %s is none of the built digests %v", h.Name, h.Version, module, version, h.Digest, cur.Artifacts)
// A build whose recorded digest differs from what the holder holds at its version fails loudly at once: the version is taken
// and no artifact is marked for it, so it is never released (design 47 G A-490j).
rv.unreleased(ctx, cur, fmt.Sprintf("the holder holds %s@%s under digest %s, which no artifact of the build carries (%v): the version is taken and is never released",
h.Name, h.Version, h.Digest, cur.Artifacts))
}
for _, m := range marked {
rv.sayPublished(ctx, m, m.Reserved.Artifacts[m.Artifact])
@@ -249,9 +207,6 @@ func (rv *reservations) heard(ctx context.Context, m jetstream.Msg) bool {
for _, k := range gone {
rv.dropBound(ctx, k)
}
if rv.released != nil {
rv.released(ctx, ev.Module, ev.Version)
}
rv.found.clear(ctx, versions.VersionUnreleased, meshconditionsclient.ScopeModule, ev.Module)
default:
return false
@@ -294,14 +249,10 @@ func (rv *reservations) sayPublished(ctx context.Context, m versions.Marked, dig
// dropped is builder_reserved_drop: the reservation is deleted and its bound ended.
func (rv *reservations) dropped(ctx context.Context, module, series, by, why string, at time.Time) (string, error) {
cur, held, _ := rv.r.At(ctx, versions.ReservedKey(module, series))
ok, err := rv.r.Drop(ctx, module, series)
if err != nil {
return "", err
}
if ok && held && rv.released != nil {
rv.released(ctx, module, cur.Version) // its publish item goes with its reservation (A-456b)
}
if !ok {
return "", fmt.Errorf("%s holds no reservation of %s", module, series)
}
@@ -359,27 +310,3 @@ func (rv *reservations) noMark(ctx context.Context, o versions.Outcome) string {
}
return strings.Join(out, "; ")
}
// reserveDoneAct is builder_reserve_done's act (design 47 G A-455): the next version of the series is reserved for a done trunk build
// that reserved none, and its publish is queued as builder_publish_again queues it. Its answer says the version and the queued
// publishes; a reservation whose publish could not be queued is said so, with the tool that queues it.
func reserveDoneAct(reserve func(ctx context.Context, module, fp string) (versions.ReservedVersion, error),
republish func(ctx context.Context, module, version string) ([]string, error), logf func(string, ...any)) func(ctx context.Context, module, fp, by, why string, at time.Time) (string, error) {
return func(ctx context.Context, module, fp, by, why string, at time.Time) (string, error) {
res, err := reserve(ctx, module, fp)
if err != nil {
return "", err
}
logf("%s at %s: version %s is reserved by hand (%s, %s) for the done build", module, fp, res.Version, by, why)
queued, err := republish(ctx, module, res.Version)
if err != nil {
return "", fmt.Errorf("%s is reserved for %s at %s, but its publish is not queued (builder_publish_again queues it): %w", res.Version, module, fp, err)
}
return fmt.Sprintf("reserved %s for %s at %s and queued %d versions to publish: %s; builder_publish_queue shows them", res.Version, module, fp, len(queued), strings.Join(queued, ", ")), nil
}
}
// ReleasedFor says whether a version was released for the build of fingerprint (design 47 G A-456a).
func (rv *reservations) ReleasedFor(ctx context.Context, module, fingerprint string) (bool, error) {
return rv.r.ReleasedFor(ctx, module, fingerprint)
}
@@ -5,7 +5,6 @@ import (
"encoding/json"
"fmt"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"slices"
"strings"
"testing"
"time"
@@ -17,7 +16,6 @@ import (
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"mesh-delivery-builder/internal/build"
"mesh-delivery-builder/internal/versions"
)
@@ -91,9 +89,6 @@ func TestTheCatalogsWordsMoveTheReservations(t *testing.T) {
}
rv := newReservations(&memReserve{vals: map[string][]byte{}, revs: map[string]uint64{}}, call, func() jetstream.KeyValue { return bounds }, found,
func(context.Context) { woke++ }, t.Logf)
var items []string // design 47 G A-457a: the end of a build asks its publish, the release ends it
rv.built = func(_ context.Context, m, v string) error { items = append(items, "asked "+m+" "+v); return nil }
rv.released = func(_ context.Context, m, v string) { items = append(items, "ended "+m+" "+v) }
if _, err := rv.Reserve(ctx, "m", "1.4", []string{"m:a"}, versions.Source{}); err == nil || rv.Filled() {
t.Fatalf("a version was reserved before versions-filled (%v)", err)
}
@@ -107,14 +102,9 @@ func TestTheCatalogsWordsMoveTheReservations(t *testing.T) {
if err != nil || res.Version != "1.4.3" {
t.Fatalf("reserved %q (%v), want 1.4.3 after the catalog's 1.4.2", res.Version, err)
}
items = nil
if err := rv.Built(ctx, "m", "1.4.3", map[string]string{"m:a": "sha256:a", "m:b": "sha256:b"}, nil); err != nil {
t.Fatal(err)
}
// The build's end asks the publish item once, the one mechanism (design 47 G A-457a, A-473).
if len(items) != 1 || items[0] != "asked m 1.4.3" {
t.Fatalf("the build's end asked %v, want the publish item of m 1.4.3 once (A-457a, A-473)", items)
}
boundKey := versions.ReservedBoundKey("m", "1.4")
if !bounds.keys[boundKey] {
t.Fatalf("the publish's bound is not armed: %v", bounds.keys)
@@ -138,9 +128,6 @@ func TestTheCatalogsWordsMoveTheReservations(t *testing.T) {
if !rv.heard(ctx, released) || !released.acked {
t.Fatalf("version-released was not taken in (acked %v)", released.acked)
}
if strings.Join(items, ", ") != "asked m 1.4.3, ended m 1.4.3" {
t.Errorf("publish items %v, want the build's end to ask the publish and the release to end it", items)
}
if all, _ := rv.r.All(ctx); len(all) != 0 || bounds.keys[boundKey] || len(st.vals) != 0 {
t.Errorf("after version-released: reservations %v, bounds %v, findings %v; want none", all, bounds.keys, st.vals)
}
@@ -221,14 +208,13 @@ func TestAReleasedVersionIsTaggedAtTheCommitItWasBuiltFrom(t *testing.T) {
// is said in the log.
func TestAPublishedHeardBeforeTheDigestsIsSaidWhenTheBuildRecordsThem(t *testing.T) {
ctx := context.Background()
fs := &memFindings{vals: map[string][]byte{}}
var logs []string
logf := func(f string, a ...any) { logs = append(logs, fmt.Sprintf(f, a...)) }
none := func(string, any) (json.RawMessage, error) {
return json.Marshal(meshcatalog.VersionsAnswer{Module: "m"})
}
rv := newReservations(&memReserve{vals: map[string][]byte{}, revs: map[string]uint64{}}, none, func() jetstream.KeyValue { return &boundsKV{keys: map[string]bool{}} },
&findings{module: builds.BuilderModule, w: fs, logf: t.Logf, now: time.Now}, nil, logf)
&findings{module: builds.BuilderModule, w: &memFindings{vals: map[string][]byte{}}, logf: t.Logf, now: time.Now}, nil, logf)
var said []meshbuilds.PackagePublished
rv.say = func(_ context.Context, p meshbuilds.PackagePublished) error { said = append(said, p); return nil }
rv.filled.Store(true)
@@ -250,137 +236,9 @@ func TestAPublishedHeardBeforeTheDigestsIsSaidWhenTheBuildRecordsThem(t *testing
if log := strings.Join(logs, "\n"); !strings.Contains(log, "is kept on") || !strings.Contains(log, "m-x@"+res.Version+" kept on m "+res.Version+" marks no artifact") {
t.Fatalf("log = %q, want the published kept and the one never matching said", logs)
}
// A-490j: the one that never matched is raised at once, naming the version taken and the holder's digest.
raised := ""
for _, v := range fs.vals {
raised += string(v)
}
if !strings.Contains(raised, "sha256:x") || !strings.Contains(raised, "is taken and is never released") {
t.Fatalf("no finding raised for the digest the holder holds: %s", raised)
}
// Taken in once: a published of m:a heard again says nothing more.
rv.published(ctx, versions.Outcome{Name: "m-a", Version: res.Version, Digest: "sha256:a"})
if len(said) != 1 {
t.Fatalf("package-published said again: %+v", said)
}
}
// TestABuildTakingOverAnOlderReservationDropsItsBoundAndSaysSo: design 47 G A-455 — a build that reserves while the series is
// reserved by an older build never released drops the older one's bound and finding and says so in the builder's journal.
func TestABuildTakingOverAnOlderReservationDropsItsBoundAndSaysSo(t *testing.T) {
ctx := context.Background()
st := &memFindings{vals: map[string][]byte{}}
found := &findings{module: builds.BuilderModule, w: st, logf: t.Logf, now: func() time.Time { return time.Unix(0, 0) }}
bounds := &boundsKV{keys: map[string]bool{}}
var journal []string
call := func(string, any) (json.RawMessage, error) {
return json.Marshal(meshcatalog.VersionsAnswer{Module: "m"})
}
rv := newReservations(&memReserve{vals: map[string][]byte{}, revs: map[string]uint64{}}, call, func() jetstream.KeyValue { return bounds }, found,
nil, func(f string, a ...any) { journal = append(journal, fmt.Sprintf(f, a...)) })
rv.filled.Store(true)
older, err := rv.Reserve(ctx, "m", "0.1", []string{"m:a"}, versions.Source{Fingerprint: "fp-old"})
if err != nil {
t.Fatal(err)
}
if err := rv.Built(ctx, "m", older.Version, map[string]string{"m:a": "sha256:a"}, nil); err != nil {
t.Fatal(err)
}
rv.unreleased(ctx, older, "nothing within the bound")
if !bounds.keys[versions.ReservedBoundKey("m", "0.1")] || len(st.vals) != 1 {
t.Fatalf("the older reservation's bound %v or finding %v is not there", bounds.keys, st.vals)
}
newer, err := rv.Reserve(ctx, "m", "0.1", []string{"m:a"}, versions.Source{Fingerprint: "fp-new"})
if err != nil || newer.Version != "0.1.1" {
t.Fatalf("reserved %q (%v), want 0.1.1", newer.Version, err)
}
if len(bounds.keys) != 0 || len(st.vals) != 0 {
t.Errorf("after the takeover: bounds %v, findings %v; want none", bounds.keys, st.vals)
}
if !slices.ContainsFunc(journal, func(l string) bool {
return strings.Contains(l, "0.1.0") && strings.Contains(l, "dropped") && strings.Contains(l, "0.1.1")
}) {
t.Errorf("journal %q does not say the reservation of 0.1.0 was dropped for 0.1.1", journal)
}
if got, ok, _ := rv.ReservedFor(ctx, "m", "fp-new"); !ok || got.Version != "0.1.1" {
t.Errorf("reserved for fp-new: %+v %v", got, ok)
}
}
// ansKV records the value versionActed writes back to a reasks key.
type ansKV struct {
jetstream.KeyValue
wrote chan []byte
}
func (a *ansKV) Update(_ context.Context, _ string, v []byte, _ uint64) (uint64, error) {
a.wrote <- v
return 2, nil
}
type ansEntry struct {
jetstream.KeyValueEntry
key string
value []byte
}
func (e ansEntry) Key() string { return e.key }
func (e ansEntry) Revision() uint64 { return 1 }
func (e ansEntry) Value() []byte { return e.value }
func (e ansEntry) Operation() jetstream.KeyValueOp { return jetstream.KeyValuePut }
// TestAReserveDoneIsAnsweredWithTheVersionAndItsQueuedPublish: design 47 G A-455 — the reasks value of builder_reserve_done reaches
// the act, which reserves for the done build and queues the publish as publish again does, and its answer names both; a
// refusal of the reserving queues nothing and is the answer; a publish that cannot be queued says the version is reserved.
func TestAReserveDoneIsAnsweredWithTheVersionAndItsQueuedPublish(t *testing.T) {
ctx := context.Background()
var queued []string
var logged []string
logf := func(f string, a ...any) { logged = append(logged, fmt.Sprintf(f, a...)) }
reserve := func(_ context.Context, module, fp string) (versions.ReservedVersion, error) {
if fp == "refused" {
return versions.ReservedVersion{}, fmt.Errorf("the build key of %s at %s is not done", module, fp)
}
return versions.ReservedVersion{Module: module, Version: "0.1.13", Fingerprint: fp}, nil
}
republish := func(_ context.Context, module, version string) ([]string, error) {
if module == "nope" {
return nil, fmt.Errorf("the queue is full")
}
queued = append(queued, module+"@"+version)
return []string{module + "@" + version}, nil
}
acts := versions.Acts{ReserveDone: reserveDoneAct(reserve, republish, logf)}
for _, c := range []struct {
module, fp, want string
refused bool
}{
{"node-tools", "src2:7df8", "reserved 0.1.13 for node-tools at src2:7df8 and queued 1 versions to publish: node-tools@0.1.13", false},
{"node-tools", "refused", "not done", true},
{"nope", "src2:1", "0.1.13 is reserved for nope at src2:1, but its publish is not queued", true},
} {
value, err := json.Marshal(build.ReAsk{V: 1, Why: "w", By: "me", Mode: build.ModeReserveDone, Name: c.module, FP: c.fp})
if err != nil {
t.Fatal(err)
}
kv := &ansKV{wrote: make(chan []byte, 1)}
// reAsked sends a reserve-done to the version acts (b is not asked), and the answer is written back to the key.
reAsked(ctx, nil, acts, nil, kv, ansEntry{key: "reserve-done." + c.module, value: value}, logf)
var wrote []byte
select {
case wrote = <-kv.wrote:
case <-time.After(10 * time.Second):
t.Fatalf("%s at %s: no answer was written to the key", c.module, c.fp)
}
var got build.ReAsk
if err := json.Unmarshal(wrote, &got); err != nil || got.Refused != c.refused || !strings.Contains(got.Answer, c.want) || got.Answered.IsZero() {
t.Errorf("%s at %s: written %s (%v), want refused=%v containing %q", c.module, c.fp, wrote, err, c.refused, c.want)
}
}
if !slices.Equal(queued, []string{"node-tools@0.1.13"}) {
t.Errorf("queued %v, want only node-tools@0.1.13", queued)
}
if !slices.ContainsFunc(logged, func(l string) bool { return strings.Contains(l, "0.1.13 is reserved by hand") }) {
t.Errorf("the reserving by hand is not said in the log: %q", logged)
}
}
@@ -111,7 +111,7 @@ func TestTheBuilderKeepsTheToolchainsItHeardAndForgetsThemOnStopped(t *testing.T
if len(keys) != 3 || keys[0] != "ace.go" {
t.Fatalf("the state holds %v, want the three nodes' keys <node>.go", keys)
}
if len(r.said) != 1 || r.said[0].Version != "1.26" || r.said[0].Language != "go" {
if len(r.said) != 1 || r.said[0].Version != "1.26.0" || r.said[0].Language != "go" {
t.Fatalf("toolchain-agreed said %+v, want one for go 1.26.0 (said only when the version changes)", r.said)
}
// The first report agreed it with one digest; a later digest of the same version is not said again.
@@ -120,13 +120,13 @@ func TestTheBuilderKeepsTheToolchainsItHeardAndForgetsThemOnStopped(t *testing.T
}
// A restarted builder reads the keys back and agrees the same version with no new report; the digests it says are all three nodes'.
again := newHeardRig(t, state)
if len(again.said) != 1 || again.said[0].Version != "1.26" || len(again.said[0].Digests) != 2 || again.said[0].Digests[0].Nodes[1] != "g14" {
if len(again.said) != 1 || again.said[0].Version != "1.26.0" || len(again.said[0].Digests) != 2 || again.said[0].Digests[0].Nodes[1] != "g14" {
t.Fatalf("a restarted builder said %+v, want go 1.26.0 with the digests of its three nodes", again.said)
}
// A version change is said; the stopped build module of ace takes ace's key out, a placement taking nothing.
again.report(t, "node-build-go", "ace", "go", "1.27.0", da)
again.report(t, "node-build-go", "g14", "go", "1.27.0", da)
if len(again.said) != 2 || again.said[1].Version != "1.27" {
if len(again.said) != 2 || again.said[1].Version != "1.27.0" {
t.Fatalf("a changed agreed version was not said: %+v", again.said)
}
at := time.Now()
@@ -88,7 +88,7 @@ func (b *Builder) handEnded(key, why string) {
// BuildAgain builds the failed key of module at fp again, as the builder does by itself: the key is written building as a
// new revision and the build module asks are put on the build seats. It answers once the asks are put, or at bound. A key
// left building that no look of this builder holds is taken as it stands and its asks put if they never were (issue 1966).
// A key done (unless its reserved version is not released) or queued is refused with why: done (nothing to do) or waiting (builder_ask_again asks its waiting asks again).
// A key done or queued is refused with why: done (nothing to do) or waiting (builder_ask_again asks its waiting asks again).
// ctx is the builder's own context: the build it starts lives on it.
func (b *Builder) BuildAgain(ctx context.Context, module, fp, why, by string, bound time.Duration) BuildAgainResult {
b.init()
@@ -109,11 +109,7 @@ func (b *Builder) BuildAgain(ctx context.Context, module, fp, why, by string, bo
}
switch {
case cur.State == builds.BuildDone:
// A done key whose version is not released is built again, so the build's own publish runs (A-490); one released, or
// that reserves no version, has nothing to do.
if why := b.rebuildRefused(ctx, module, fp); why != "" {
return BuildAgainResult{Refused: fmt.Sprintf("%s is done: nothing to do (%s)", key, why)}
}
return BuildAgainResult{Refused: fmt.Sprintf("%s is done: nothing to do", key)}
case cur.State == builds.BuildQueued:
return BuildAgainResult{Refused: fmt.Sprintf("%s is waiting (%s: %s): %s asks its waiting asks again", key, cur.State, cur.Step, AskAgainTool)}
case cur.State == builds.BuildBuilding:
@@ -200,29 +196,3 @@ func boolInt(b bool) int {
}
return 0
}
// rebuildRefused says why a done key of module at fp is not built again by hand: empty when its build is the first watched
// branch's trunk build and its reserved version is not released (a publish refused leaves no copy to publish, A-490).
func (b *Builder) rebuildRefused(ctx context.Context, module, fp string) string {
if b.Reserved == nil {
return "this builder reserves no version"
}
key, err := builds.BuilderBuildKey(module, fp)
if err != nil {
return err.Error()
}
b.mu.Lock()
trunk, first := b.trunkFP[key], b.origin[key].First || b.origin[key] == (Origin{})
b.mu.Unlock()
if !trunk || !first {
return "only the first watched branch's trunk build reserves a version"
}
released, err := b.Reserved.ReleasedFor(ctx, module, fp)
if err != nil {
return fmt.Sprintf("whether its version is released cannot be read: %v", err)
}
if released {
return "its version is released"
}
return ""
}
@@ -54,9 +54,6 @@ type ClaimOptions struct {
// ByHand is a person's hand act (BuildAgain): a key failed on a transport fault is taken again only by it, or by a builder
// that has just started (TakeOver), never by the builder's own retake (ADR 0664 §3).
ByHand bool
// Rebuild takes a done key again as a new revision and builds it whole: a person's BuildAgain of a key whose reserved version
// is not released (design 47 G A-490: a package whose build's publish was refused is built again, never published from a copy).
Rebuild bool
// ByEvent is the builder's retake on an event that says a transport fault's cause passed (design 47 G §568): it takes a key
// failed on a transport fault, as a new revision that counts one more of EventBuilds.
ByEvent bool
@@ -192,11 +189,6 @@ func (c Claimer) Claim(ctx context.Context, module, fingerprint, manifest string
}
switch {
case cur.State == builds.BuildDone:
if opt.Rebuild {
again := building
again.Step = "built again by hand: its reserved version is not released"
return c.write(ctx, key, again, func(data []byte) (uint64, error) { return c.Store.Update(ctx, key, data, rev) })
}
if Unreferenced(cur) {
return c.write(ctx, key, building, func(data []byte) (uint64, error) { return c.Store.Update(ctx, key, data, rev) })
}
@@ -2,7 +2,6 @@ package build
import (
"context"
"errors"
"fmt"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
"git.novox.be/novox/mesh-sdk/go/catalogue"
@@ -82,15 +81,6 @@ func (b *Builder) number(ctx context.Context, module string, m meshcatalog.Modul
b.logf("%s at %s: %v", module, m.Fp2, fmt.Errorf("not numbered, the versions cannot be used: %w", err))
return versions.Numbering{}, true
}
if done != nil && len(got.Versions) > 0 && b.Reserved != nil {
// A late look numbered a done build: G §349 retires numbering at a look, so one with no reservation and no release is
// said aloud with the recovery that reserves it (A-455), never taken in silently.
_, reserved, rerr := b.Reserved.ReservedFor(ctx, module, m.Fp2)
released, derr := b.Reserved.ReleasedFor(ctx, module, m.Fp2)
if rerr == nil && derr == nil && !reserved && !released {
b.logf("%s at %s: a late look numbered this done build (%v) and no version is reserved for it: builder_reserve_done reserves one and queues its publish", module, m.Fp2, got.Versions)
}
}
if got.Refused != "" {
b.logf("%s at %s: %s", module, m.Fp2, got.Refused)
return got, true
@@ -149,10 +139,6 @@ type Reservations interface {
// Built records the digests of the artifacts built under the reserved version, by reference, and what each resolved and was
// built with (design 47 G §466, §673).
Built(ctx context.Context, module, version string, digests map[string]string, deps map[string][]meshbuilds.Dependency) error
// ReservedFor is the reservation made for the build of fingerprint, and whether there is one.
ReservedFor(ctx context.Context, module, fingerprint string) (versions.ReservedVersion, bool, error)
// ReleasedFor says whether a version was released for the build of fingerprint, read from the builder's own record (A-456a).
ReleasedFor(ctx context.Context, module, fingerprint string) (bool, error)
}
// reserves says whether a look at a trunk build is one that reserves: the build has a series of the form MAJOR.MINOR
@@ -169,11 +155,6 @@ func (b *Builder) reserves(ctx context.Context, module string, m meshcatalog.Mod
return b.doneDigests(ctx, module, m.Fp2) == nil
}
// reservesVersion says whether a plan's build reserves a version: it has artifacts and a MAJOR.MINOR series (§368).
func (b *Builder) reservesVersion(plan Plan) bool {
return len(plan.Tasks) > 0 && strings.Count(plan.Series, ".") == 1
}
// waitFill holds a build that would reserve until the catalog says versions-filled, saying wait-held naming the fill once
// (design 47 G §369, §372). It answers whether the build waits.
func (b *Builder) waitFill(module string, m meshcatalog.ModulesByModuleValue) bool {
@@ -192,13 +173,13 @@ func (b *Builder) waitFill(module string, m meshcatalog.ModulesByModuleValue) bo
}
// reserve reserves the version of the build that starts; a reservation that cannot be made is said and the build goes on
// unreserved: the caller fails the key, which a look takes again (A-490).
func (b *Builder) reserve(ctx context.Context, module string, m meshcatalog.ModulesByModuleValue, plan Plan, o Origin, rebuild bool) (versions.ReservedVersion, bool) {
// unreserved, as an unnumbered build does.
func (b *Builder) reserve(ctx context.Context, module string, m meshcatalog.ModulesByModuleValue, plan Plan, o Origin) (versions.ReservedVersion, bool) {
refs := make([]string, 0, len(plan.Tasks))
for _, t := range plan.Tasks {
refs = append(refs, module+":"+t.Artifact)
}
src := versions.Source{Repository: m.Repository, Commit: m.Commit, Fingerprint: m.Fp2, Branch: o.Branch, Revision: o.Source, Rebuild: rebuild}
src := versions.Source{Repository: m.Repository, Commit: m.Commit, Fingerprint: m.Fp2, Branch: o.Branch, Revision: o.Source}
for _, t := range plan.Tasks {
src.GoPackage = src.GoPackage || (t.Kind == catalogue.ArtifactPackage && t.Language == "go")
}
@@ -210,70 +191,3 @@ func (b *Builder) reserve(ctx context.Context, module string, m meshcatalog.Modu
b.logf("%s: version %s is reserved for this build", module, res.Version)
return res, true
}
// ReserveDone is builder_reserve_done (design 47 G A-455): a done trunk build of the first watched branch that reserved no
// version (its key is done, so a look never reserves for it, and a done key cannot be built again) reserves the next version of
// its series now, with the digests the done key holds, as a build that started would, and its build ends with every digest
// recorded. It answers the reserved version; the caller queues its publish (versions.Republisher). A fingerprint whose key is not
// done, whose entry no watched branch carries now, that is no first branch's, or that holds a reservation already is refused, saying which.
func (b *Builder) ReserveDone(ctx context.Context, module, fp string) (versions.ReservedVersion, error) {
b.init()
if b.Reserved == nil {
return versions.ReservedVersion{}, errors.New("this builder reserves no versions")
}
key, err := builds.BuilderBuildKey(module, fp)
if err != nil {
return versions.ReservedVersion{}, err
}
b.mu.Lock()
var m meshcatalog.ModulesByModuleValue
found := false
for hk, v := range b.heard {
if strings.HasSuffix(hk, "\x00"+module) && v.Fp2 == fp {
m, found = v, true
}
}
trunk, first := b.trunkFP[key], b.origin[key].First
b.mu.Unlock()
switch {
case !found:
return versions.ReservedVersion{}, fmt.Errorf("no watched branch carries an entry of %s at %s now, so there is nothing to reserve it from", module, fp)
case !trunk || !first:
return versions.ReservedVersion{}, fmt.Errorf("%s at %s is not the first watched branch's build: only that one reserves a version (§371)", module, fp)
}
digests := b.doneDigests(ctx, module, fp)
if digests == nil {
return versions.ReservedVersion{}, fmt.Errorf("the build key of %s at %s is not done", module, fp)
}
plan, planErr, _ := b.planOf(ctx, m.Manifest)
switch {
case planErr != nil:
return versions.ReservedVersion{}, fmt.Errorf("the plan of %s at %s cannot be read: %w", module, fp, planErr)
case len(plan.Tasks) == 0 || strings.Count(plan.Series, ".") != 1:
return versions.ReservedVersion{}, fmt.Errorf("%s at %s builds nothing or has no MAJOR.MINOR series (%q): it reserves no version", module, fp, plan.Series)
}
if cur, ok, err := b.Reserved.ReservedFor(ctx, module, fp); err != nil {
return versions.ReservedVersion{}, err
} else if ok {
return versions.ReservedVersion{}, fmt.Errorf("%s at %s holds the reservation of %s already", module, fp, cur.Version)
}
if !b.Reserved.Filled() {
return versions.ReservedVersion{}, versions.ErrNotFilled
}
res, ok := b.reserve(ctx, module, m, plan, b.originOf(key), false)
if !ok {
return versions.ReservedVersion{}, fmt.Errorf("no version could be reserved for %s at %s (the builder's journal says why)", module, fp)
}
refs := map[string]string{}
for _, t := range plan.Tasks {
d, have := digests[t.Artifact]
if !have {
return res, fmt.Errorf("%s is reserved for %s at %s, but the done key holds no digest of %s", res.Version, module, fp, t.Artifact)
}
refs[module+":"+t.Artifact] = d
}
if err := b.Reserved.Built(ctx, module, res.Version, refs, nil); err != nil {
return res, fmt.Errorf("%s is reserved for %s at %s, but its digests are not recorded: %w", res.Version, module, fp, err)
}
return res, nil
}
@@ -3,7 +3,6 @@ package build
import (
"context"
"encoding/json"
"errors"
"fmt"
"reflect"
"slices"
@@ -23,39 +22,17 @@ import (
// fakeReservations answers a fill and keeps what the builder reserved and recorded.
type fakeReservations struct {
filled bool
released map[string]bool
reserved [][]string
series []string
built map[string]string
sources []versions.Source
deps map[string][]meshbuilds.Dependency
next string // the version Reserve answers when set, else the series' .5
fail error // the reservation cannot be made (the catalog's versions cannot be read)
}
func (f *fakeReservations) ReleasedFor(_ context.Context, _, fp string) (bool, error) {
return f.released[fp], nil
}
func (f *fakeReservations) Filled() bool { return f.filled }
func (f *fakeReservations) ReservedFor(_ context.Context, _, fp string) (versions.ReservedVersion, bool, error) {
for _, s := range f.sources {
if s.Fingerprint == fp {
return versions.ReservedVersion{Version: "held"}, true, nil
}
}
return versions.ReservedVersion{}, false, nil
}
func (f *fakeReservations) Reserve(_ context.Context, module, series string, artifacts []string, src versions.Source) (versions.ReservedVersion, error) {
if f.fail != nil {
return versions.ReservedVersion{}, f.fail
}
f.reserved, f.series, f.sources = append(f.reserved, artifacts), append(f.series, series), append(f.sources, src)
if f.next != "" {
return versions.ReservedVersion{Module: module, Series: series, Version: f.next}, nil
}
return versions.ReservedVersion{Module: module, Series: series, Version: series + ".5"}, nil
}
@@ -261,128 +238,3 @@ func TestAModuleWhoseOnlyArtifactIsAGoPackageIsBuiltUnderItsReservedVersion(t *t
})
}
}
// TestAReservingTrunkBuildNeverEndsDoneWithoutItsVersion: design 47 G A-490 (a package whose build's publish was refused is built
// again, never published from a copy; the kind met three times on 2026-11-19: a trunk build ended done with "no version is
// reserved") — a reservation that cannot be made fails the key with why and builds nothing, so a look or builder_build_again takes
// it again; and a done key whose version is not released is built again by hand under a reserved version, a released one is refused.
func TestAReservingTrunkBuildNeverEndsDoneWithoutItsVersion(t *testing.T) {
setup := func(t *testing.T, res *fakeReservations) *rig {
r := newRig(t, goBundle)
r.mf.byDigest[mdigest('a')] = catalogue.Manifest{Module: "m", Version: "0.1", Build: &catalogue.Build{Artifacts: []catalogue.Artifact{goBundle}}}
r.b.Reserved = res
return r
}
t.Run("a reservation that cannot be made fails the key and asks nothing", func(t *testing.T) {
r := setup(t, &fakeReservations{filled: true, fail: errors.New("the catalog's versions cannot be read")})
v := r.consider(t, "modules/m")
if v.State != builds.BuildFailed || !strings.Contains(v.Error, "no version is reserved") || len(r.asks.Asks()) != 0 {
t.Fatalf("key %+v with %d asks; want failed saying no version is reserved, nothing asked", v, len(r.asks.Asks()))
}
})
for _, c := range []struct {
name string
released bool
standing bool // a reservation of 0.1.0 stands, and the catalog now answers 0.1.16 released
refuse string
}{
{name: "a standing 0.1.0 reservation is not kept: the rebuild reserves afresh, 0.1.17 above the released 0.1.16", standing: true},
{name: "a done key whose version is not released is built again under a reserved version"},
{name: "a done key whose version is released is refused", released: true, refuse: "its version is released"},
} {
t.Run(c.name, func(t *testing.T) {
r := setup(t, nil)
r.b.Reserved = nil
if v := r.consider(t, "modules/m"); v.State != builds.BuildDone {
t.Fatalf("first build = %+v, want done", v)
}
res := &fakeReservations{filled: true, released: map[string]bool{fp1: c.released}}
if c.standing {
res.sources, res.next = []versions.Source{{Fingerprint: fp1}}, "0.1.17" // ReservedFor answers a held reservation
}
r.b.Reserved = res
asked := len(r.asks.Asks())
got := r.b.BuildAgain(context.Background(), "m", fp1, "its publish was refused", "shanks.lead", 2*time.Second)
r.b.Wait()
if c.refuse != "" {
if !strings.Contains(got.Refused, c.refuse) || len(r.asks.Asks()) != asked {
t.Fatalf("answer %+v; want refused with %q and nothing asked", got, c.refuse)
}
return
}
if got.Refused != "" || len(r.asks.Asks()) == asked || len(res.reserved) != 1 {
t.Fatalf("answer %+v with %d new asks, reserved %v; want built again under one reserved version", got, len(r.asks.Asks())-asked, res.reserved)
}
want := "0.1.5"
if c.standing {
want = "0.1.17"
}
if _, ok := res.built[want]; !ok || len(res.reserved) != 1 {
t.Fatalf("built %v after %d reservations; want the digests recorded under the freshly reserved %s so the build's end asks its publish", res.built, len(res.reserved), want)
}
})
}
}
// realReserve is the real versions.Reserver over a map, answering the builder's Reservations as the running builder's does.
type realReserve struct {
*versions.Reserver
reservations map[string][]byte
}
func (r *realReserve) Filled() bool { return true }
func (r *realReserve) ReleasedFor(context.Context, string, string) (bool, error) {
return false, nil
}
func (r *realReserve) ReservedFor(ctx context.Context, module, fp string) (versions.ReservedVersion, bool, error) {
all, err := r.All(ctx)
for _, v := range all {
if v.Module == module && v.Fingerprint == fp {
return v, true, err
}
}
return versions.ReservedVersion{}, false, err
}
func (r *realReserve) Built(ctx context.Context, module, version string, digests map[string]string, _ map[string][]meshbuilds.Dependency) error {
for ref, d := range digests {
if _, err := r.Reserver.Built(ctx, module, version, ref, d, nil); err != nil {
return err
}
}
return nil
}
type reserveMap struct{ *memStore }
func (m reserveMap) KeysOf(ctx context.Context, _ string) ([]string, error) { return m.Keys(ctx) }
func (m reserveMap) Delete(_ context.Context, key string) error {
m.mu.Lock()
defer m.mu.Unlock()
delete(m.vals, key)
return nil
}
// TestABuildAgainReservesAboveTheCatalogsHighestNotItsOwnStandingNumber: design 47 G A-490b, at the build, through the real
// reserver — a done key whose own fingerprint has an unreleased 0.1.30 standing, the catalog's highest 0.1.16, is built again
// under 0.1.17 (never 0.1.31), the standing reservation taken over.
func TestABuildAgainReservesAboveTheCatalogsHighestNotItsOwnStandingNumber(t *testing.T) {
r := newRig(t, goBundle)
r.mf.byDigest[mdigest('a')] = catalogue.Manifest{Module: "m", Version: "0.1", Build: &catalogue.Build{Artifacts: []catalogue.Artifact{goBundle}}}
if v := r.consider(t, "modules/m"); v.State != builds.BuildDone {
t.Fatalf("first build = %+v, want done", v)
}
store := reserveMap{newMem()}
real := &realReserve{Reserver: &versions.Reserver{Store: store, Filled: func() bool { return true },
ReleasedOf: func(context.Context, string) ([]string, error) { return []string{"0.1.16"}, nil }}}
raw, _ := json.Marshal(versions.ReservedVersion{V: 1, Module: "m", Series: "0.1", Version: "0.1.30", Fingerprint: fp1, Artifacts: map[string]string{"m:tool": "d"}})
if _, err := store.Create(context.Background(), versions.ReservedKey("m", "0.1"), raw); err != nil {
t.Fatal(err)
}
r.b.Reserved = real
got := r.b.BuildAgain(context.Background(), "m", fp1, "its publish was refused", "shanks.lead", 2*time.Second)
r.b.Wait()
cur, ok, err := real.Of(context.Background(), "m", "0.1.17")
if got.Refused != "" || err != nil || !ok || cur.Fingerprint != fp1 {
t.Fatalf("answer %+v; reservation of 0.1.17 = %+v (%v, %v); want built again under 0.1.17, the catalog's highest 0.1.16 plus one", got, cur, ok, err)
}
}
@@ -1,56 +0,0 @@
package build
import (
"context"
"slices"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
)
// TestADoneTrunkBuildThatReservedNoneIsGivenAVersionByHand: design 47 G A-455 — builder_reserve_done reserves the series' next
// version for a done key that reserved none, with the digests the key holds; a key not done, a fingerprint no entry carries and
// a build that holds a reservation already are refused; the value it writes is a reasks value of its own key.
func TestADoneTrunkBuildThatReservedNoneIsGivenAVersionByHand(t *testing.T) {
ctx := context.Background()
r := newRig(t, goBundle)
r.mf.byDigest[mdigest('a')] = catalogue.Manifest{Module: "m", Version: "0.1", Build: &catalogue.Build{Artifacts: []catalogue.Artifact{goBundle}}}
res := &fakeReservations{filled: true}
r.b.Reserved = res
if _, err := r.b.ReserveDone(ctx, "m", fp1); err == nil {
t.Fatal("a version was reserved for a key that is not built")
}
// The build ran while the builder reserved nothing (as the live key did): done, with no reservation.
r.b.Reserved = nil
if v := r.consider(t, "modules/m"); v.State != builds.BuildDone {
t.Fatalf("build = %+v, want done", v)
}
r.b.Reserved = res
if _, err := r.b.ReserveDone(ctx, "m", "src2:"+strings.Repeat("9", 64)); err == nil || !strings.Contains(err.Error(), "no watched branch carries") {
t.Errorf("a fingerprint no entry carries: %v", err)
}
got, err := r.b.ReserveDone(ctx, "m", fp1)
if err != nil || got.Version != "0.1.5" || !slices.Equal(res.series, []string{"0.1"}) || !slices.Equal(res.reserved[0], []string{"m:tool"}) {
t.Fatalf("reserved %+v (%v) in %v for %v, want 0.1.5 of m:tool in 0.1", got, err, res.series, res.reserved)
}
if res.built["m:tool"] != digest || res.sources[0].Fingerprint != fp1 {
t.Errorf("recorded %v from %+v, want the done key's digest for the entry %s", res.built, res.sources, fp1)
}
if _, err := r.b.ReserveDone(ctx, "m", fp1); err == nil || !strings.Contains(err.Error(), "reservation") {
t.Errorf("a build that holds a reservation was given another: %v", err)
}
at := time.Unix(0, 0)
key, _, err := NewReserveDone("m", fp1, "why", "op", at)
if err != nil || !strings.HasPrefix(key, "reserve-done.") {
t.Errorf("key %q (%v)", key, err)
}
for _, args := range [][3]string{{"", fp1, "why"}, {"m", "", "why"}, {"m", fp1, " "}} {
if _, _, err := NewReserveDone(args[0], args[1], args[2], "op", at); err == nil {
t.Errorf("%q made a value", args)
}
}
}
@@ -617,10 +617,7 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB
}
// A key built again by hand is taken over as it stands too: one left building by a look that ended without its write
// (issue 1966) is the hand tool's to finish.
// A done key a person builds again whose reserved version is not released is built whole, under a version reserved afresh (A-490b).
rebuild := byHand && trunk && planErr == nil && !nothing && b.Reserved != nil && b.doneDigests(ctx, module, m.Fp2) != nil &&
b.rebuildRefused(ctx, module, m.Fp2) == "" && b.reservesVersion(plan)
cl, err := b.Claimer.Claim(ctx, module, m.Fp2, m.Manifest, ClaimOptions{Rebuild: rebuild, TakeOver: first || byHand, ByHand: byHand, ByEvent: byEvent, ImageBuilder: imageSeat, Seats: b.Seats, Trunk: trunk, Published: b.isPublished,
cl, err := b.Claimer.Claim(ctx, module, m.Fp2, m.Manifest, ClaimOptions{TakeOver: first || byHand, ByHand: byHand, ByEvent: byEvent, ImageBuilder: imageSeat, Seats: b.Seats, Trunk: trunk, Published: b.isPublished,
Stored: b.storedOnce(key)})
if err != nil {
end("the claim was refused: %v", err)
@@ -658,22 +655,11 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB
}
var reserved versions.ReservedVersion
hasReserved := false
if rebuild && cl.Build.State == builds.BuildBuilding {
// Reserved afresh, above the highest the catalog's versions answers, taking over the older unreleased reservation of this
// fingerprint (A-455), never its standing number kept (A-490b); the build's own publish then runs.
if b.Reserved.Filled() {
reserved, hasReserved = b.reserve(ctx, module, m, plan, b.originOf(key), true)
}
reserving = true
} else if reserving && cl.Build.State != builds.BuildDone {
reserved, hasReserved = b.reserve(ctx, module, m, plan, b.originOf(key), false)
if reserving && cl.Build.State != builds.BuildDone {
reserved, hasReserved = b.reserve(ctx, module, m, plan, b.originOf(key))
}
var result builds.BuilderBuild
if reserving && !hasReserved && cl.Build.State != builds.BuildDone {
// A trunk build that reserves is never built without its version: it would end done and publish nothing, and its key would
// stay done, never built again (A-490). The key fails with why and is taken again at the next look or by builder_build_again.
result = failed("no version is reserved for this build: the builder's journal says why, and it is built again once one can be")
} else if numbering.Refused != "" && cl.Build.State != builds.BuildDone {
if numbering.Refused != "" && cl.Build.State != builds.BuildDone {
// Refused, naming the state it could not read, and built against no guess (ADR 0666 §15; design 47, test 70): the key
// fails with why, and is taken again at the next look.
result = failed("%s", numbering.Refused)
@@ -197,7 +197,7 @@ func newRig(t *testing.T, artifacts ...catalogue.Artifact) *rig {
}
// agreedToolchains are what the rig's nodes report, and so what a fingerprint computed now names.
var agreedToolchains = map[string]builds.ToolchainUsed{"go": {Version: "go1.27", Recipe: "1"}, "typescript": {Version: "tsc5", Recipe: "1"}}
var agreedToolchains = map[string]builds.ToolchainUsed{"go": {Version: "go1.27.1", Recipe: "1"}, "typescript": {Version: "tsc5", Recipe: "1"}}
func moduleValue(t *testing.T, path string) []byte { return moduleValueWith(t, path, agreedToolchains) }
@@ -301,7 +301,7 @@ func TestABundleAndAnArchiveAreBuiltOnOneNodeAndDone(t *testing.T) { // 1607/AC7
if !slices.Equal(v.Needs, []string{"form:artifact-store"}) || v.Artifacts[0].References != nil {
t.Errorf("needs = %v, references = %v", v.Needs, v.Artifacts[0].References)
}
if got := v.Toolchains["go"]; got.Version != "go1.27" || got.Recipe != "1" {
if got := v.Toolchains["go"]; got.Version != "go1.27.1" || got.Recipe != "1" {
t.Errorf("toolchains = %+v, want go and typescript named", v.Toolchains)
}
if got := v.Toolchains["typescript"]; got.Version != "tsc5" {
@@ -468,19 +468,9 @@ func TestATrunkImageCallCarriesPublish(t *testing.T) { // 1705b; design 51 *The
t.Errorf("a named package's done build: numbered %d more times with %v, %d image builds; want once with img %s and no build",
nums.numbered-looks, nums.done, len(r.bm.callsOf("build_image_build")), digest)
}
// A late look that numbers a done build with no reservation is said aloud, naming builder_reserve_done (G §349, A-455).
late := &logged{}
r.b.Logf, r.b.Reserved = late.logf, &fakeReservations{filled: true}
r.b.PackageNamed(context.Background(), "m")
r.b.Wait()
if !late.has("late look numbered", "builder_reserve_done") {
t.Errorf("a late look numbered a done build with no reservation and said %v", late.lines)
}
r.b.Reserved = nil
looks = nums.numbered
r.b.PackageNamed(context.Background(), "other")
r.b.Wait()
if nums.numbered != looks {
if nums.numbered != looks+1 {
t.Errorf("a module the builder does not know was looked at")
}
// D-B5 (ADR 0666 §15; design 47, test 70): a build whose published packages cannot be read is refused, naming the
@@ -694,32 +684,32 @@ func (d *seatDrops) get() []string {
func TestAKeyIsBuiltOnlyWithTheToolchainItsFingerprintNamesAndNothingIsWrittenOtherwise(t *testing.T) { // 1615/AC4
r := newRig(t, goBundle)
old := map[string]builds.ToolchainUsed{"go": {Version: "go1.26", Recipe: "1"}}
old := map[string]builds.ToolchainUsed{"go": {Version: "go1.26.0", Recipe: "1"}}
r.b.Module(context.Background(), "m", moduleValueWith(t, "modules/m", old))
r.b.Wait()
if len(r.st.vals) != 0 || len(r.bm.calls) != 0 {
t.Run("a recipe that moves builds the module again under its new fingerprint", aRecipeThatMovesBuildsTheModuleAgainUnderItsNewFingerprint)
t.Errorf("keys %v and %d calls for a fingerprint of go1.26 while the nodes agree on go1.27, want neither", r.st.vals, len(r.bm.calls))
t.Errorf("keys %v and %d calls for a fingerprint of go1.26.0 while the nodes agree on go1.27.1, want neither", r.st.vals, len(r.bm.calls))
}
// A recipe that differs is a different toolchain too, and a value that names none (written before) is not built.
r.b.Module(context.Background(), "m", moduleValueWith(t, "modules/m", map[string]builds.ToolchainUsed{"go": {Version: "go1.27", Recipe: "0"}}))
r.b.Module(context.Background(), "m", moduleValueWith(t, "modules/m", map[string]builds.ToolchainUsed{"go": {Version: "go1.27.1", Recipe: "0"}}))
r.b.Module(context.Background(), "m", moduleValueWith(t, "modules/m", nil))
r.b.Wait()
if len(r.st.vals) != 0 || len(r.bm.calls) != 0 {
t.Errorf("keys %v and %d calls for a recipe that differs or a value that names no toolchain", r.st.vals, len(r.bm.calls))
}
// The toolchain moves under a key that is not built yet: the old fingerprint stays unbuilt, the new one builds.
r.tools.Set("ace", tcOf("go", "go1.28.0"))
r.tools.Set("novox", tcOf("go", "go1.28.0"))
r.tools.Set("ace", tcOf("go", "go1.27.2"))
r.tools.Set("novox", tcOf("go", "go1.27.2"))
r.b.Reconsider(context.Background())
r.b.Wait()
if len(r.st.vals) != 0 {
t.Errorf("keys %v: the old fingerprint was built with the new toolchain", r.st.vals)
}
r.b.Module(context.Background(), "m", moduleValueWith(t, "modules/m", map[string]builds.ToolchainUsed{"go": {Version: "go1.28", Recipe: "1"}}))
r.b.Module(context.Background(), "m", moduleValueWith(t, "modules/m", map[string]builds.ToolchainUsed{"go": {Version: "go1.27.2", Recipe: "1"}}))
r.b.Wait()
if v := valueOf(t, r.st, r.key); v.State != builds.BuildDone || v.Toolchains["go"].Version != "go1.28" {
t.Errorf("build = %+v, want done with go1.28", v)
if v := valueOf(t, r.st, r.key); v.State != builds.BuildDone || v.Toolchains["go"].Version != "go1.27.2" {
t.Errorf("build = %+v, want done with go1.27.2", v)
}
}
@@ -740,7 +730,7 @@ func aRecipeThatMovesBuildsTheModuleAgainUnderItsNewFingerprint(t *testing.T) {
t.Errorf("%d calls, was %d: the first fingerprint was built again for the new recipe", len(r.bm.calls), calls)
}
data, err := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-catalog", Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20", Fp: "src1:" + hex64('f'), Fp2: fp2,
Manifest: mdigest('a'), Path: "modules/m", Toolchains: CatalogToolchains(map[string]builds.ToolchainUsed{"go": {Version: "go1.27", Recipe: "2"}})})
Manifest: mdigest('a'), Path: "modules/m", Toolchains: CatalogToolchains(map[string]builds.ToolchainUsed{"go": {Version: "go1.27.1", Recipe: "2"}})})
if err != nil {
t.Fatal(err)
}
@@ -67,7 +67,7 @@ func (b *Builder) buildEntry(ctx context.Context, e meshbuilds.BuildEntry) (outc
defer b.dropEnd(key, ended)
if done, failedWords, found := b.keyEnd(ctx, key); found {
if done {
return b.passedWithItsVersion(ctx, entry)
return meshbuilds.StepEndedOutcomePassed, ""
}
return meshbuilds.StepEndedOutcomeFailed, failedWords
}
@@ -77,7 +77,7 @@ func (b *Builder) buildEntry(ctx context.Context, e meshbuilds.BuildEntry) (outc
select {
case log := <-ended:
if log.Outcome == builds.BuildDone {
return b.passedWithItsVersion(ctx, entry)
return meshbuilds.StepEndedOutcomePassed, ""
}
return meshbuilds.StepEndedOutcomeFailed, fmt.Sprintf("%s at %s: %s %s", log.Module, log.FP, log.Outcome, log.Error)
case <-ctx.Done():
@@ -85,46 +85,6 @@ func (b *Builder) buildEntry(ctx context.Context, e meshbuilds.BuildEntry) (outc
}
}
// passedWithItsVersion is the verdict of a build that ended done (design 47 G A-456): a build that reserves a version (a build
// of the first watched branch's trunk, with a MAJOR.MINOR series and artifacts) passes only when a version is reserved for its
// fingerprint; one that ended done with none fails loudly, naming builder_reserve_done, so no pipeline says passed for a build
// that can never be released. A build that reserves none (a candidate's, a series of before G) passes as it did.
func (b *Builder) passedWithItsVersion(ctx context.Context, entry meshcatalog.CatalogEntry) (outcome, said string) {
if b.Reserved == nil {
return meshbuilds.StepEndedOutcomePassed, ""
}
key, err := builds.BuilderBuildKey(entry.Module, entry.Fp)
if err != nil {
return meshbuilds.StepEndedOutcomeCouldNotRun, err.Error()
}
b.mu.Lock()
reserving := b.trunkFP[key] && b.origin[key].First
b.mu.Unlock()
if !reserving {
return meshbuilds.StepEndedOutcomePassed, ""
}
plan, planErr, _ := b.planOf(ctx, meshcatalog.BareManifestDigest(entry.ManifestDigest))
switch {
case planErr != nil:
return meshbuilds.StepEndedOutcomeCouldNotRun, fmt.Sprintf("%s at %s: the plan cannot be read to judge its version: %v", entry.Module, entry.Fp, planErr)
case len(plan.Tasks) == 0 || strings.Count(plan.Series, ".") != 1:
return meshbuilds.StepEndedOutcomePassed, ""
}
_, held, err := b.Reserved.ReservedFor(ctx, entry.Module, entry.Fp)
switch {
case err != nil:
return meshbuilds.StepEndedOutcomeCouldNotRun, fmt.Sprintf("%s at %s: the reservations cannot be read: %v", entry.Module, entry.Fp, err)
case !held:
if released, err := b.Reserved.ReleasedFor(ctx, entry.Module, entry.Fp); err != nil {
return meshbuilds.StepEndedOutcomeCouldNotRun, fmt.Sprintf("%s at %s: the releases cannot be read: %v", entry.Module, entry.Fp, err)
} else if released {
return meshbuilds.StepEndedOutcomePassed, "" // a released build counts as reserved (A-456a)
}
return meshbuilds.StepEndedOutcomeFailed, fmt.Sprintf("%s: built at %s, but no version is reserved for it; builder_reserve_done reserves one for the done build", entry.Module, entry.Fp)
}
return meshbuilds.StepEndedOutcomePassed, ""
}
// keyEnd reads the build key: whether it is done, and its error when it failed for good; found only for a key that has ended.
func (b *Builder) keyEnd(ctx context.Context, key string) (done bool, failed string, found bool) {
raw, _, ok, err := b.Claimer.Store.Get(ctx, key)
@@ -11,8 +11,6 @@ import (
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"mesh-delivery-builder/internal/versions"
)
// The runner's build step ask is the builder's one entry point for a candidate (design 47 G A-97): each entry is read by its
@@ -76,48 +74,3 @@ func TestAStepAskIsWokenByItsBuildsEnd(t *testing.T) {
t.Errorf("waits left: %v", r.b.stepWaits)
}
}
// A build step passes a build that reserves a version only when a version is reserved for its fingerprint (design 47 G A-456):
// a trunk build that ended done with none fails naming builder_reserve_done, and passes once one is reserved for it; a build
// that reserves none (a manifest of before G, version "1") passes as it did.
func TestABuildStepPassesAReservingBuildOnlyWithItsReservedVersion(t *testing.T) {
for _, c := range []struct {
name, version string
reserve, released, unreadable bool
want, said string
}{
{"no version reserved", "0.1", false, false, false, meshbuilds.StepEndedOutcomeFailed, "built at " + fp1 + ", but no version is reserved for it"},
{"a version reserved", "0.1", true, false, false, meshbuilds.StepEndedOutcomePassed, ""},
{"a released build whose reservation is gone", "0.1", false, true, false, meshbuilds.StepEndedOutcomePassed, ""},
{"the plan cannot be read", "0.1", false, false, true, meshbuilds.StepEndedOutcomeCouldNotRun, "plan cannot be read"},
{"a series of before G", "1", false, false, false, meshbuilds.StepEndedOutcomePassed, ""},
} {
t.Run(c.name, func(t *testing.T) {
r := newRig(t, goBundle)
r.mf.byDigest[mdigest('a')] = catalogue.Manifest{Module: "m", Version: c.version, Build: &catalogue.Build{Artifacts: []catalogue.Artifact{goBundle}}}
res := &fakeReservations{filled: true, released: map[string]bool{fp1: c.released}}
if v := r.consider(t, "modules/m"); v.State != builds.BuildDone { // built while the builder reserved nothing, as the live key was
t.Fatalf("build = %+v, want done", v)
}
r.b.Reserved = res
if c.reserve {
res.sources = append(res.sources, versions.Source{Fingerprint: fp1})
}
r.b.StepEntry = func(context.Context, string) (meshcatalog.CatalogEntry, error) {
return meshcatalog.CatalogEntry{Module: "m", Repository: "novox/m", Commit: "c1", Fp: fp1, ManifestDigest: digestOf(c.unreadable), Path: "modules/m"}, nil
}
end := r.b.BuildStep(t.Context(), meshbuilds.StepRun{Pipeline: "main.1.m", Step: meshbuilds.StepRunStepBuild, Entries: []meshbuilds.BuildEntry{{Module: "m", Fingerprint: fp1}}})
if end.Outcome != c.want || !strings.Contains(end.Said, c.said) || (c.want == meshbuilds.StepEndedOutcomeFailed && !strings.Contains(end.Said, "builder_reserve_done")) {
t.Errorf("end = %+v, want %s saying %q", end, c.want, c.said)
}
})
}
}
// digestOf is the manifest digest a step's entry names: the rig's manifest, or one no store holds, whose plan cannot be read.
func digestOf(unreadable bool) string {
if unreadable {
return mdigest('z')
}
return mdigest('a')
}
@@ -61,15 +61,8 @@ type ReAsk struct {
Name string `json:"name,omitempty"`
Version string `json:"version,omitempty"`
Queued []string `json:"queued,omitempty"`
// Tries and NextTry are a ModePublish value's (design 47 G A-457a): the publishes tried so far, and when the next is.
Tries int `json:"tries,omitempty"`
NextTry time.Time `json:"next-try,omitzero"`
}
// ModePublish is the Mode of a reasks value that is a version's publish item (design 47 G A-457a): Module and Version name it,
// Tries and NextTry say where it stands; it is written by the builder itself, when a build ends and at its start.
const ModePublish = "publish"
// ModeRepublish is the Mode of a reasks value of builder_publish_again (design 47 G §588): Module and Version, each
// optional, name what to publish again; the builder answers with the versions it queued (Queued).
const ModeRepublish = "republish"
@@ -144,24 +137,6 @@ func NewReservedDrop(module, series, why, by string, at time.Time) (string, []by
return "reserved." + versions.ReservedKey(module, series), data, err
}
// ModeReserveDone is the Mode of a reasks value of builder_reserve_done (design 47 G A-455): Name is the module and FP the
// fingerprint of a done trunk build that reserved no version.
const ModeReserveDone = "reserve-done"
// NewReserveDone is the value builder_reserve_done writes: the key `reserve-done.<module>.<fingerprint>` (never a build key) and
// the value.
func NewReserveDone(module, fp, why, by string, at time.Time) (string, []byte, error) {
module, fp = strings.TrimSpace(module), strings.TrimSpace(fp)
if module == "" || fp == "" {
return "", nil, errors.New("module and fp are required")
}
if strings.TrimSpace(why) == "" {
return "", nil, errors.New("say why a version is reserved for the done build (why)")
}
data, err := json.Marshal(ReAsk{V: 1, Why: strings.TrimSpace(why), By: by, At: at.UTC(), Mode: ModeReserveDone, Name: module, FP: fp})
return "reserve-done." + versions.EscapeToken(module) + "." + versions.EscapeToken(fp), data, err
}
// ModeBuild is the Mode of a reasks value that asks the key to be built again, not its waiting asks to be asked again.
const ModeBuild = "build"
@@ -41,7 +41,7 @@ func (w *memWaits) End(_ context.Context, module, fp, how string) bool {
// built and ended `ran` when the agreed toolchain moves to that version; and one held is built with the running build
// module by Release and ended `released`.
func TestAnEntryWhoseFingerprintNamesAnotherToolchainWaitsAndIsBuiltWhenItIsAgreedOrReleased(t *testing.T) {
old := map[string]builds.ToolchainUsed{"go": {Version: "go1.26", Recipe: "1"}}
old := map[string]builds.ToolchainUsed{"go": {Version: "go1.26.0", Recipe: "1"}}
t.Run("ran", func(t *testing.T) {
r := newRig(t, goBundle)
w := &memWaits{}
@@ -52,7 +52,7 @@ func TestAnEntryWhoseFingerprintNamesAnotherToolchainWaitsAndIsBuiltWhenItIsAgre
t.Fatalf("held %v, keys %v: want one wait and nothing written", w.held, r.st.vals)
}
for _, h := range w.held {
if h.On() != "toolchain:go@go1.26" {
if h.On() != "toolchain:go@go1.26.0" {
t.Errorf("the wait is on %s", h.On())
}
}
@@ -75,7 +75,7 @@ func TestAnEntryWhoseFingerprintNamesAnotherToolchainWaitsAndIsBuiltWhenItIsAgre
t.Fatalf("release refused: %s", why)
}
r.b.Wait()
if got := valueOf(t, r.st, r.key); got.State != builds.BuildDone || got.Toolchains["go"].Version != "go1.27" || len(w.ended) != 1 || w.ended[0] != WaitReleased {
if got := valueOf(t, r.st, r.key); got.State != builds.BuildDone || got.Toolchains["go"].Version != "go1.27.1" || len(w.ended) != 1 || w.ended[0] != WaitReleased {
t.Errorf("build %+v, ended %v: want done with the running toolchain and a wait ended released", got, w.ended)
}
if why := r.b.Release(context.Background(), "m", fp1, "again", "operator"); why == "" {
@@ -24,9 +24,6 @@ type Acts struct {
Republish func(ctx context.Context, module, version string) ([]string, error)
// DropOldForm drops a retired node from the old form's count by hand (builder_old_form): its answer, or why not.
DropOldForm func(ctx context.Context, node, by, why string, at time.Time) (string, error)
// ReserveDone reserves the next version for a done trunk build that reserved none and queues its publish (builder_reserve_done):
// its answer, or why not.
ReserveDone func(ctx context.Context, module, fp, by, why string, at time.Time) (string, error)
// DropReserved drops a module's reservation of one MAJOR.MINOR by hand (builder_reserved_drop): its answer, or why not.
DropReserved func(ctx context.Context, module, series, by, why string, at time.Time) (string, error)
}
@@ -5,20 +5,22 @@ import (
"encoding/json"
"errors"
"fmt"
"slices"
"sort"
"strings"
"sync"
"time"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
)
// **A reserved version of a done build is published by the builder until its release is written** (design 47 G §588, A-457a): the
// publish is an item of the builder's re-ask state, asked when the build ends and at its start, tried with a growing pause and
// ended loudly after its tries; each registry seat's holder, as the seat-holders view names it (A-416, A-416a), has one queue of
// versions, kept in the state `publish-queue`, sent one at a time. A send is the registry seat's `publish` with the digest and no
// parts (A-73): the holder takes the bytes from the artifact store by digest, and refuses what no store holds. Nothing waits on a
// node-engine `running` or `manifest-reads-changed` any more.
// **A reserved version left unpublished is published again by the builder, on an event or by a person's tool** (design 47
// G §588, ADR 0674): on the node-engine seat's `running` of a module whose catalog graph entry claims the registry seat
// of the missing artifacts' kind, and on its `manifest-reads-changed` with `why` `returned`, at each registry seat's holder
// the seat-holders view names (A-416, A-416a: known from the start, not from the running heard). Each message is acted on once, by its stream sequence (state `acted`); each holder has one queue of versions,
// kept in the state `publish-queue`, sent one at a time. A send is the registry seat's `publish` with the digest and no
// parts (A-73): the holder takes the bytes from the artifact store by digest, and refuses what no store holds.
const (
// PublishQueueState is the builder's old state section of the queues, capped at PublishQueueMiB.
@@ -29,6 +31,8 @@ const (
ActedMiB = 1
// PublishQueueTool is builder_publish_queue.
PublishQueueTool = "builder_publish_queue"
// DefaultRepublishMax is the setting republish-max-versions unless set (§633).
DefaultRepublishMax = 50
// QueueFull is what version-unreleased says of a version the queue's cap refused.
QueueFull = "queue full"
// NotStoredWords opens what a refused artifact is said as.
@@ -102,14 +106,17 @@ type Publish struct {
type Republisher struct {
Reserved *Reserver
Queue QueueStore
Acted ReserveStore
// Entry reads the catalog graph entry of a fingerprint (the seat mesh-catalog's verb entry).
Entry func(ctx context.Context, fingerprint string) (meshcatalog.CatalogEntry, error)
// Send sends one publish to a registry seat; an error is the holder's refusal or the call's fault, in its words.
Send func(ctx context.Context, p Publish) error
// Unreleased says version-unreleased about a version with the holder's words.
Unreleased func(ctx context.Context, v ReservedVersion, why string)
Now func() time.Time
Logf func(string, ...any)
// Max is the setting republish-max-versions: the most versions one event queues.
Max int
Now func() time.Time
Logf func(string, ...any)
// Go starts the sending of a holder's queue; nil starts a goroutine.
Go func(func())
// Holders answers a registry seat's holders: the catalog's SeatHolders, the part's one holder function that its FollowSeat
@@ -152,6 +159,97 @@ func (r *Republisher) logf(f string, a ...any) {
// ActedKey is the acted key of a subject.
func ActedKey(subject string) string { return EscapeToken(subject) }
// Heard takes one message of the node-engine seat's running or manifest-reads-changed in, at stream sequence seq: it acts
// on it only when its sequence is newer than the one kept for its subject, and keeps it after. A subject that is neither is
// not its; an error leaves the message not acted on, so it is read again.
func (r *Republisher) Heard(ctx context.Context, subject string, data []byte, seq uint64) (bool, error) {
running := strings.HasPrefix(subject, strings.TrimSuffix(nodeengine.SubjectRunningAll, "*.*"))
changed := strings.HasPrefix(subject, "mesh.seat.node-engine.event."+nodeengine.EventManifestReadsChanged+".")
if !running && !changed {
return false, nil
}
key := ActedKey(subject)
raw, rev, found, err := r.Acted.Get(ctx, key)
if err != nil {
return true, fmt.Errorf("the acted sequence of %s cannot be read: %w", subject, err)
}
if found {
var a Acted
if json.Unmarshal(raw, &a) == nil && a.Sequence >= seq {
return true, nil
}
}
if running {
err = r.running(ctx, data)
} else {
err = r.returned(ctx, data)
}
if err != nil {
return true, err
}
val, _ := json.Marshal(Acted{Sequence: seq})
if found {
_, err = r.Acted.Update(ctx, key, val, rev)
} else {
_, err = r.Acted.Create(ctx, key, val)
}
if err != nil && !errors.Is(err, ErrConflict) {
return true, fmt.Errorf("the acted sequence of %s cannot be kept: %w", subject, err)
}
return true, nil
}
// registrySeatsOf are the registry seats the module of the catalog entry fingerprint claims, by their publish verb.
func (r *Republisher) registrySeatsOf(ctx context.Context, fingerprint string) ([]string, error) {
e, err := r.Entry(ctx, fingerprint)
if err != nil {
return nil, err
}
var out []string
for _, seat := range RegistrySeats {
if slices.Contains(e.Serves, seat+"."+VerbPublish) {
out = append(out, seat)
}
}
return out, nil
}
// running takes a running in: one of a module whose entry claims a registry seat queues the versions waiting on it (§588). It
// is not how a holder is known: Holders answers that (design 47 G A-416).
func (r *Republisher) running(ctx context.Context, data []byte) error {
var ev nodeengine.Running
if err := json.Unmarshal(data, &ev); err != nil || ev.Module == "" || ev.Node == "" {
r.logf("a running that does not read is not acted on: %v", err)
return nil
}
if ev.Stopped || ev.Fingerprint == "" {
return nil
}
seats, err := r.registrySeatsOf(ctx, ev.Fingerprint)
if err != nil {
return fmt.Errorf("the catalog entry %s of %s cannot be read: %w", ev.Fingerprint, ev.Module, err)
}
if len(seats) == 0 {
return nil
}
_, err = r.queueEvent(ctx, map[string][]string{ev.Module: seats})
return err
}
// heldSeats are the registry seats' holders as Holders answers them: holder module -> the registry seats it holds.
func (r *Republisher) heldSeats() map[string][]string {
held := map[string][]string{}
if r.Holders == nil {
return held
}
for _, seat := range RegistrySeats {
for _, h := range r.Holders(seat) {
held[h] = append(held[h], seat)
}
}
return held
}
// holdersFn answers a seat's holders from Holders.
func (r *Republisher) holdersFn(seat string) []string {
if r.Holders == nil {
@@ -160,12 +258,59 @@ func (r *Republisher) holdersFn(seat string) []string {
return r.Holders(seat)
}
// returned takes a manifest-reads-changed in: only one with why returned acts, for every registry seat's holder Holders
// answers (design 47 G A-416a); a version queued at a holder already is not queued again.
func (r *Republisher) returned(ctx context.Context, data []byte) error {
var ev nodeengine.ManifestReadsChanged
if err := json.Unmarshal(data, &ev); err != nil || ev.Node == "" {
r.logf("a manifest-reads-changed that does not read is not acted on: %v", err)
return nil
}
if ev.Why != nodeengine.ManifestReadsChangedWhyReturned {
return nil
}
holders := r.heldSeats()
if len(holders) == 0 {
return nil
}
_, err := r.queueEvent(ctx, holders)
return err
}
// view is a reserved version with the artifacts of it a holder would publish.
type view struct {
v ReservedVersion
arts map[string]meshcatalog.Artifact // reference -> its catalog artifact
}
// queueEvent queues, for each holder (module -> the seats it claims), the unreleased versions whose missing artifacts
// that holder's seats take: newest keys first, at most Max versions in all (one key per module and MAJOR.MINOR is the
// reservation's own).
func (r *Republisher) queueEvent(ctx context.Context, holders map[string][]string) ([]string, error) {
over, err := r.Reserved.Overdue(ctx, nil)
if err != nil {
return nil, err
}
var vs []ReservedVersion
for _, o := range over {
if v, ok, err := r.Reserved.Of(ctx, o.Module, o.Version); err == nil && ok {
vs = append(vs, v)
}
}
return r.queue(ctx, vs, func(seat string) []string { return holdersOf(holders, seat) }, r.Max)
}
func holdersOf(holders map[string][]string, seat string) []string {
var out []string
for m, seats := range holders {
if slices.Contains(seats, seat) {
out = append(out, m)
}
}
sort.Strings(out)
return out
}
// queue queues vs newest first, at most max of them (0 is no limit), each artifact to the first holder holdersOf names for
// its seat. It answers the versions queued, `<module> <version>`.
func (r *Republisher) queue(ctx context.Context, vs []ReservedVersion, holdersOf func(seat string) []string, max int) ([]string, error) {
@@ -304,34 +449,11 @@ func (r *Republisher) Republish(ctx context.Context, module, version string) ([]
type Queued struct {
Queue []QueuedPublish `json:"queue"`
Acted map[string]uint64 `json:"acted"`
// Items are the publish items (design 47 G A-457a), each with its tries and its next try; an item that ended says why.
Items []PublishItem `json:"items"`
}
// PublishItemKeyPrefix opens the key of a publish item in the builder's re-ask state.
const PublishItemKeyPrefix = "publish."
// PublishItem is the builder's own record of a version's publish (design 47 G A-457a): a value of the re-ask state, its tries and
// its next try. It is asked when the build ends and at the builder's start, rewritten with one more try and a longer pause after
// each, and deleted when the version is released; after the setting's number of tries it ends, saying so (Answer), and stays.
type PublishItem struct {
Module string `json:"module"`
Version string `json:"version"`
Tries int `json:"tries"`
NextTry time.Time `json:"next-try"`
Why string `json:"why,omitempty"`
Answer string `json:"answer,omitempty"`
Answered time.Time `json:"answered,omitzero"`
}
// PublishItemKey is the key of module's version's publish item.
func PublishItemKey(module, version string) string {
return PublishItemKeyPrefix + EscapeToken(module) + "." + EscapeToken(version)
}
// AnswerQueue is builder_publish_queue over the two sections as the tools bundle reads them.
func AnswerQueue(queue, acted, reasks Reader) (Queued, error) {
out := Queued{Queue: []QueuedPublish{}, Acted: map[string]uint64{}, Items: []PublishItem{}}
func AnswerQueue(queue, acted Reader) (Queued, error) {
out := Queued{Queue: []QueuedPublish{}, Acted: map[string]uint64{}}
keys, err := queue.Keys()
if err != nil {
return out, fmt.Errorf("the state %s cannot be listed: %w", PublishQueueState, err)
@@ -366,37 +488,18 @@ func AnswerQueue(queue, acted, reasks Reader) (Queued, error) {
out.Acted[k] = a.Sequence
}
}
rkeys, err := reasks.Keys()
if err != nil {
return out, fmt.Errorf("the re-ask state cannot be listed: %w", err)
}
for _, k := range rkeys {
if !strings.HasPrefix(k, PublishItemKeyPrefix) {
continue
}
raw, ok, err := reasks.Get(k)
if err != nil || !ok {
continue
}
var it PublishItem
if json.Unmarshal(raw, &it) == nil {
out.Items = append(out.Items, it)
}
}
sort.SliceStable(out.Items, func(i, j int) bool { return out.Items[i].NextTry.Before(out.Items[j].NextTry) })
return out, nil
}
// PublishQueueDescription is builder_publish_queue's description.
const PublishQueueDescription = "The versions the builder will publish again (its state " + PublishQueueState + ", design 47 G section 588): per registry " +
"seat holder, in the order they are sent, each with its artifacts' digests, when it was queued and whether its publish has begun; and the " +
"keys of the state " + ActedState + ", the last stream sequence the builder acted on for each subject; and the publish items (design 47 G " +
"A-457a), each version's tries and next try, and why one ended after its tries. Read-only."
"keys of the state " + ActedState + ", the last stream sequence the builder acted on for each running and manifest-reads-changed subject. Read-only."
// PublishAgainDescription is builder_publish_again's description.
const PublishAgainDescription = "Publish reserved versions again (design 47 G section 588), which builds nothing: with module and version, that reserved " +
"version's missing artifacts; with module alone, each reserved version of that module whose artifacts are not all published; with neither, every " +
"version the finding version-unreleased names, newest first. It queues them as the builder's publish items do, and answers at once " +
"version the finding version-unreleased names, newest first. It queues them as the builder does on a registry holder's running, and answers at once " +
"with the versions it queued; each outcome is said as package-published, or left in version-unreleased with the holder's words. A hand act, " +
"recorded in the hand-act log."
@@ -9,6 +9,7 @@ import (
"testing"
"time"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
)
@@ -25,6 +26,7 @@ type republishBed struct {
*Republisher
reserved *reserveMem
queue *queueMem
acted *reserveMem
now time.Time
sent []Publish
said []string
@@ -33,11 +35,11 @@ type republishBed struct {
holders map[string][]string
}
func newRepublishBed(t *testing.T) *republishBed {
func newRepublishBed(t *testing.T, max int) *republishBed {
t.Helper()
b := &republishBed{reserved: &reserveMem{mem: newMem()}, queue: &queueMem{reserveMem: &reserveMem{mem: newMem()}}, now: time.Date(2026, 11, 11, 12, 0, 0, 0, time.UTC)}
b := &republishBed{reserved: &reserveMem{mem: newMem()}, queue: &queueMem{reserveMem: &reserveMem{mem: newMem()}}, acted: &reserveMem{mem: newMem()}, now: time.Date(2026, 11, 11, 12, 0, 0, 0, time.UTC)}
res := &Reserver{Store: b.reserved, Now: func() time.Time { return b.now }, ReleasedOf: func(context.Context, string) ([]string, error) { return nil, nil }}
b.Republisher = &Republisher{Reserved: res, Queue: b.queue, Now: func() time.Time { return b.now }, Go: func(f func()) { f() },
b.Republisher = &Republisher{Reserved: res, Queue: b.queue, Acted: b.acted, Max: max, Now: func() time.Time { return b.now }, Go: func(f func()) { f() },
Entry: func(_ context.Context, fp string) (meshcatalog.CatalogEntry, error) {
if fp == "src2:holder" {
return meshcatalog.CatalogEntry{Serves: []string{SeatArtifacts + ".publish", SeatArtifacts + ".versions"}}, nil
@@ -80,32 +82,77 @@ func (b *republishBed) reserve(t *testing.T, module string) ReservedVersion {
return v
}
func running(node, module, fp string, stopped bool) (string, []byte) {
data, _ := json.Marshal(nodeengine.Running{Node: node, Module: module, Fingerprint: fp, Stopped: stopped, At: time.Unix(1, 0)})
return nodeengine.SubjectRunning(node, module), data
}
// TestRepublish: design 47 G §588 (A-73, A-76): a reserved version left unpublished is published again, on a registry
// holder's running and on its node's returned, once per message by its stream sequence, as a publish by digest with no parts,
// at most the setting's number per event, newest first; an artifact no store holds is said once and not tried again.
func TestRepublish(t *testing.T) {
ctx := context.Background()
t.Run("a done build is published as a digest with no parts, to its seat's holder", func(t *testing.T) {
b := newRepublishBed(t)
t.Run("running of a holder, once per sequence", func(t *testing.T) {
b := newRepublishBed(t, 50)
v := b.reserve(t, "m")
b.holders = map[string][]string{SeatArtifacts: {"oci"}}
if v.Fingerprint != "src2:m" {
t.Fatalf("the reservation keeps the entry's fingerprint: %q", v.Fingerprint)
}
if q, err := b.Republish(ctx, "", ""); err != nil || len(b.sent) != 1 || fmt.Sprint(q) != "[m "+v.Version+"]" {
t.Fatalf("queued %v sent %d (%v), want 1", q, len(b.sent), err)
subject, data := running("n1", "oci", "src2:holder", false)
for _, seq := range []uint64{5, 4} { // 4 is older than the acted 5: it starts nothing
if _, err := b.Heard(ctx, subject, data, seq); err != nil || len(b.sent) != 1 {
t.Fatalf("seq %d: sent %d (%v), want 1", seq, len(b.sent), err)
}
}
p := b.sent[0]
if p.Seat != SeatArtifacts || p.Name != "m/prog" || p.Kind != "bundle" || p.Version != v.Version || p.Digest != "sha256:aa" || p.Fingerprint != "src2:m" ||
p.Transfer != "republish-m-"+strings.ReplaceAll(v.Version, ".", "-")+"-prog" || strings.ContainsAny(p.Transfer, "/. *>") {
t.Errorf("sent %+v, want the bundle by digest to the oci seat, the Go package left to its own seat's holder", p)
}
if _, err := b.Heard(ctx, subject, data, 5); err != nil || len(b.sent) != 1 {
t.Errorf("the same sequence read again sent %d, want still 1", len(b.sent))
}
if _, err := b.Heard(ctx, subject, data, 6); err != nil || len(b.sent) != 2 {
t.Errorf("a newer sequence sent %d, want 2", len(b.sent))
}
if keys, _ := b.queue.KeysOf(ctx, AllFilter); len(keys) != 0 {
t.Errorf("a key stays in the queue after its outcome: %v", keys)
}
})
t.Run("returned acts at the seat's holder, started does not", func(t *testing.T) {
b := newRepublishBed(t, 50)
b.reserve(t, "m")
b.holders = map[string][]string{SeatArtifacts: {"oci"}}
for i, why := range []string{nodeengine.ManifestReadsChangedWhyStarted, "", nodeengine.ManifestReadsChangedWhyReturned} {
ev, _ := json.Marshal(nodeengine.ManifestReadsChanged{Node: "n1", Why: why})
handled, err := b.Heard(ctx, nodeengine.SubjectManifestReadsChanged("n1"), ev, uint64(10+i))
if !handled || err != nil {
t.Fatalf("%q: handled %v, %v", why, handled, err)
}
if want := map[string]int{"started": 0, "": 0, "returned": 1}[why]; len(b.sent) != want {
t.Errorf("why %q: sent %d, want %d", why, len(b.sent), want)
}
}
if handled, _ := b.Heard(ctx, "mesh.seat.node-engine.event.composed.n1", nil, 1); handled {
t.Error("a subject that is neither is taken as its own")
}
})
t.Run("the setting limits an event, newest first", func(t *testing.T) {
b := newRepublishBed(t, 2)
var versions []string
for _, m := range []string{"m1", "m2", "m3"} {
versions = append(versions, b.reserve(t, m).Module)
}
subject, data := running("n1", "oci", "src2:holder", false)
if _, err := b.Heard(ctx, subject, data, 1); err != nil {
t.Fatal(err)
}
if len(b.sent) != 2 || !strings.HasPrefix(b.sent[0].Name, "m3/") || !strings.HasPrefix(b.sent[1].Name, "m2/") {
t.Errorf("sent %+v, want the two newest, m3 then m2", b.sent)
}
})
t.Run("an entry that cannot be read stops neither the other versions nor the drain", func(t *testing.T) {
b := newRepublishBed(t)
b := newRepublishBed(t, 50)
b.reserve(t, "m1")
b.reserve(t, "m2") // newest: queued before m1's entry is met
entry := b.Entry
@@ -115,8 +162,8 @@ func TestRepublish(t *testing.T) {
}
return entry(ctx, fp)
}
b.holders = map[string][]string{SeatArtifacts: {"oci"}}
if _, err := b.Republish(ctx, "", ""); err == nil || !strings.Contains(err.Error(), "src2:m1") {
subject, data := running("n1", "oci", "src2:holder", false)
if _, err := b.Heard(ctx, subject, data, 1); err == nil || !strings.Contains(err.Error(), "src2:m1") {
t.Errorf("the unreadable entry is not said: %v", err)
}
if len(b.sent) != 1 || !strings.HasPrefix(b.sent[0].Name, "m2/") {
@@ -124,39 +171,39 @@ func TestRepublish(t *testing.T) {
}
})
t.Run("an artifact no store holds is said once and not tried again", func(t *testing.T) {
b := newRepublishBed(t)
b := newRepublishBed(t, 50)
b.reserve(t, "m")
b.refuse = "oci-artifact-registry's publish of m/prog: not stored: no blob sha256:aa"
b.holders = map[string][]string{SeatArtifacts: {"oci"}}
_, _ = b.Republish(ctx, "", "")
_, _ = b.Republish(ctx, "", "")
subject, data := running("n1", "oci", "src2:holder", false)
_, _ = b.Heard(ctx, subject, data, 1)
_, _ = b.Heard(ctx, subject, data, 2)
if len(b.sent) != 1 || len(b.said) != 1 || !strings.Contains(b.said[0], "not stored: ") {
t.Errorf("sent %d, said %v: want one try and one saying of not stored", len(b.sent), b.said)
}
})
t.Run("a refusal in other words is said each time", func(t *testing.T) {
b := newRepublishBed(t)
b := newRepublishBed(t, 50)
b.reserve(t, "m")
b.refuse = "the holder is full"
b.holders = map[string][]string{SeatArtifacts: {"oci"}}
_, _ = b.Republish(ctx, "", "")
_, _ = b.Republish(ctx, "", "")
subject, data := running("n1", "oci", "src2:holder", false)
_, _ = b.Heard(ctx, subject, data, 1)
_, _ = b.Heard(ctx, subject, data, 2)
if len(b.sent) != 2 || len(b.said) != 2 {
t.Errorf("sent %d, said %v: want both events tried and said", len(b.sent), b.said)
}
})
t.Run("a queue past its cap says queue full", func(t *testing.T) {
b := newRepublishBed(t)
b := newRepublishBed(t, 50)
b.reserve(t, "m")
b.queue.bytes = PublishQueueMiB << 20
b.holders = map[string][]string{SeatArtifacts: {"oci"}}
_, _ = b.Republish(ctx, "", "")
subject, data := running("n1", "oci", "src2:holder", false)
_, _ = b.Heard(ctx, subject, data, 1)
if len(b.sent) != 0 || len(b.said) != 1 || !strings.HasSuffix(b.said[0], QueueFull) {
t.Errorf("sent %d, said %v, want nothing sent and queue full said", len(b.sent), b.said)
}
})
t.Run("a restart sends the queued again, a begun one anew", func(t *testing.T) {
b := newRepublishBed(t)
b := newRepublishBed(t, 50)
v := b.reserve(t, "m")
raw, _ := json.Marshal(QueuedPublish{V: 1, Holder: "oci", Module: "m", Version: v.Version, Artifacts: map[string]string{"m:prog": "sha256:aa"}, Queued: b.now, Begun: true})
if _, err := b.queue.Create(ctx, QueueKey("oci", "m", v.Version), raw); err != nil {
@@ -169,7 +216,7 @@ func TestRepublish(t *testing.T) {
t.Run("after a fresh start the tool queues at the seat's holder, no running heard", func(t *testing.T) {
// Design 47 G A-416, A-416a: the holder is the one the seat-holders view names from the start, not one learnt from a
// running heard since; a seat with no holder queues nothing.
b := newRepublishBed(t)
b := newRepublishBed(t, 1)
b.reserve(t, "m1")
b.reserve(t, "m2")
if q, err := b.Republish(ctx, "", ""); err != nil || len(q) != 0 {
@@ -183,16 +230,12 @@ func TestRepublish(t *testing.T) {
if _, err := b.Republish(ctx, "m1", "0.1.9"); err == nil {
t.Error("a version no reservation holds is not refused")
}
b.holders = map[string][]string{SeatArtifacts: {"oci"}}
_, _ = b.Republish(ctx, "", "")
items := Reader{Keys: func() ([]string, error) { return []string{"publish.m.0.1.0", "reserve-done.m.x"}, nil },
Get: func(k string) (json.RawMessage, bool, error) {
return json.RawMessage(`{"mode":"publish","module":"m","version":"0.1.0","tries":3,"next-try":"2026-11-17T01:00:00Z","why":"w"}`), true, nil
}}
none := Reader{Keys: func() ([]string, error) { return nil, nil }}
answer, err := AnswerQueue(none, none, items)
if err != nil || len(answer.Items) != 1 || answer.Items[0].Tries != 3 || answer.Items[0].Version != "0.1.0" {
t.Errorf("builder_publish_queue items = %+v (%v), want the one publish item with its tries", answer.Items, err)
subject, data := running("n1", "oci", "src2:holder", false)
_, _ = b.Heard(ctx, subject, data, 1)
answer, err := AnswerQueue(Reader{Keys: func() ([]string, error) { return nil, nil }}, Reader{Keys: func() ([]string, error) { return b.acted.KeysOf(ctx, AllFilter) },
Get: func(k string) (json.RawMessage, bool, error) { v, _, ok, err := b.acted.Get(ctx, k); return v, ok, err }})
if err != nil || answer.Acted[ActedKey(subject)] != 1 {
t.Errorf("builder_publish_queue acted = %v (%v), want the running's sequence 1", answer.Acted, err)
}
})
}
@@ -28,8 +28,6 @@ const (
// ReservedTool is builder_reserved, ReservedDropTool builder_reserved_drop.
ReservedTool = "builder_reserved"
ReservedDropTool = "builder_reserved_drop"
// ReserveDoneTool is builder_reserve_done (design 47 G A-455).
ReserveDoneTool = "builder_reserve_done"
// UnreleasedBound is the publish's bound: past it a built reservation with an artifact unpublished is the finding
// version-unreleased (§369).
UnreleasedBound = 10 * time.Minute
@@ -91,9 +89,6 @@ type Source struct {
// that catalog revision (design 47 G §371, A-57), kept on the reservation for the publication.
Branch string
Revision int64
// Rebuild marks a build again of a done key whose version is not released: the standing reservation of Fingerprint is replaced
// and is no floor, so the number is the next above the catalog's highest alone (design 47 G A-490b).
Rebuild bool
}
// ReservedKey is the key of module's reservation of series (MAJOR.MINOR).
@@ -115,16 +110,7 @@ type Reserver struct {
ReleasedOf func(ctx context.Context, module string) ([]string, error)
// Filled says the catalog said versions-filled.
Filled func() bool
// Took is told when a build took over the series' reservation of an older build never released (design 47 G A-455): the
// older reservation, which no want follows any more, is gone from the key, replaced by the newer's. Nil is told none.
Took func(ctx context.Context, older, newer ReservedVersion)
// Releases is the builder's own record of the versions it saw released, by module and fingerprint (design 47 G A-456a): the
// reservation is deleted at release, so a rerun of a released build's step reads it here, never from the catalog. Nil keeps none.
Releases ReserveStore
// ReleasedKept is how many releases per module the record keeps, the oldest dropped (the setting released-kept; A-456b); 0 is
// DefaultReleasedKept.
ReleasedKept int
Now func() time.Time
Now func() time.Time
}
func (r *Reserver) now() time.Time {
@@ -171,7 +157,7 @@ func (r *Reserver) Reserve(ctx context.Context, module, series string, artifacts
return ReservedVersion{}, err
}
next := high
if found && !(src.Rebuild && cur.Fingerprint == src.Fingerprint) {
if found {
if _, p, err := SplitVersion(cur.Version); err == nil && p > next {
next = p
}
@@ -196,9 +182,6 @@ func (r *Reserver) Reserve(ctx context.Context, module, series string, artifacts
if err != nil {
return ReservedVersion{}, fmt.Errorf("%s cannot be reserved: %w", key, err)
}
if found && r.Took != nil {
r.Took(ctx, cur, out)
}
return out, nil
}
}
@@ -257,82 +240,6 @@ func allSet(m map[string]string) bool {
return true
}
// DefaultReleasedKept is the setting released-kept unless set (design 47 G A-456b).
const DefaultReleasedKept = 16
// ReleasedKey is the key of a module's record of its last releases (design 47 G A-456a, A-456b), in the state acted.
func ReleasedKey(module string) string { return "released." + EscapeToken(module) }
// ReleasedRecord is that key's value: the module's last releases, oldest first.
type ReleasedRecord struct {
Kept []ReleasedBuild `json:"kept"`
}
// ReleasedBuild is one release: the fingerprint of the build and its version.
type ReleasedBuild struct {
Fingerprint string `json:"fingerprint"`
Version string `json:"version"`
}
// ReleasedFor says whether the builder saw a version released for the build of fingerprint among the module's last releases.
func (r *Reserver) ReleasedFor(ctx context.Context, module, fingerprint string) (bool, error) {
if r.Releases == nil {
return false, nil
}
raw, _, found, err := r.Releases.Get(ctx, ReleasedKey(module))
if err != nil || !found {
return false, err
}
var rec ReleasedRecord
if json.Unmarshal(raw, &rec) != nil {
return false, nil
}
for _, k := range rec.Kept {
if k.Fingerprint == fingerprint {
return true, nil
}
}
return false, nil
}
// keepRelease adds a release to the module's record, dropping the oldest past ReleasedKept.
func (r *Reserver) keepRelease(ctx context.Context, module, fingerprint, version string) error {
n := r.ReleasedKept
if n < 1 {
n = DefaultReleasedKept
}
for {
raw, rev, found, err := r.Releases.Get(ctx, ReleasedKey(module))
if err != nil {
return err
}
var rec ReleasedRecord
if found {
_ = json.Unmarshal(raw, &rec)
}
kept := rec.Kept[:0:0]
for _, k := range rec.Kept {
if k.Fingerprint != fingerprint {
kept = append(kept, k)
}
}
kept = append(kept, ReleasedBuild{Fingerprint: fingerprint, Version: version})
if len(kept) > n {
kept = kept[len(kept)-n:]
}
out, _ := json.Marshal(ReleasedRecord{Kept: kept})
if found {
_, err = r.Releases.Update(ctx, ReleasedKey(module), out, rev)
} else {
_, err = r.Releases.Create(ctx, ReleasedKey(module), out)
}
if errors.Is(err, ErrConflict) {
continue
}
return err
}
}
// Released takes the catalog's version-released in: the key of that version is deleted, and every key of module's other
// series (the manifest moved on). A key holding a newer reserved version of the same series stays. It answers the keys
// deleted.
@@ -354,11 +261,6 @@ func (r *Reserver) Released(ctx context.Context, module, version string) ([]stri
if !found || (cur.Series == series && cur.Version != version) {
continue
}
if cur.Version == version && cur.Fingerprint != "" && r.Releases != nil {
if err := r.keepRelease(ctx, module, cur.Fingerprint, version); err != nil {
return gone, fmt.Errorf("the release of %s %s cannot be recorded: %w", module, version, err)
}
}
if err := r.Store.Delete(ctx, k); err != nil {
return gone, fmt.Errorf("%s cannot be deleted: %w", k, err)
}
@@ -720,10 +622,6 @@ const (
"seat said published. With module, that module's only. A key goes when the catalog says version-released for it. Read-only."
ReservedDropDescription = "Drop a module's reservation of one MAJOR.MINOR by hand (design 47 G section 369), when its version will never be " +
"released; the next build takes the next after the highest released. A hand act: it is recorded in the hand-act log."
ReserveDoneDescription = "Reserve a version for a done trunk build that reserved none (design 47 G A-455): given module and fp, the next version " +
"of the series is reserved for that fingerprint with the digests of the build's artifacts, and its publish is queued as builder_publish_again " +
"queues it; builds nothing. For a done key that cannot be built again. Refused when the key is not done, is no first watched branch's, or " +
"holds a reservation already. A hand act: it is recorded in the hand-act log."
// ReservedVerb is the seat mesh-builds' verb served as a tool of the same name.
ReservedVerb = meshbuilds.VerbReserved
)
@@ -58,14 +58,7 @@ func TestReservation(t *testing.T) {
if err != nil || first.Version != "0.1.20" {
t.Fatalf("first build reserves %q (%v), want 0.1.20 after the highest released 0.1.19", first.Version, err)
}
var tookOlder, tookNewer []string
r.Took = func(_ context.Context, older, newer ReservedVersion) {
tookOlder, tookNewer = append(tookOlder, older.Version), append(tookNewer, newer.Version)
}
second, err := r.Reserve(ctx, "m", "0.1", arts, Source{})
if len(tookOlder) != 1 || tookOlder[0] != "0.1.20" || tookNewer[0] != "0.1.21" {
t.Fatalf("takeover of the older reservation told %v by %v, want 0.1.20 by 0.1.21 (design 47 G A-455)", tookOlder, tookNewer)
}
if err != nil || second.Version != "0.1.21" {
t.Fatalf("second build reserves %q (%v), want 0.1.21: never the first's", second.Version, err)
}
@@ -173,83 +166,3 @@ func TestReservationIsReadAndMarkedPublished(t *testing.T) {
t.Errorf("the bound key does not name the reservation back: %q %v", key, ok)
}
}
// TestAReleaseIsRecordedByFingerprintBeforeItsReservationGoes: design 47 G A-456a — the version-released of a version records
// that the build of its fingerprint was released, so a rerun of its step finds it after the reservation is deleted.
func TestAReleaseIsRecordedByFingerprintBeforeItsReservationGoes(t *testing.T) {
ctx := context.Background()
rel := &reserveMem{mem: newMem()}
r := &Reserver{Store: &reserveMem{mem: newMem()}, Releases: rel, ReleasedKept: 2, Filled: func() bool { return true },
ReleasedOf: func(context.Context, string) ([]string, error) { return nil, nil }}
v, err := r.Reserve(ctx, "m", "0.1", []string{"m:a"}, Source{Fingerprint: "src2:aa"})
if err != nil {
t.Fatal(err)
}
if ok, _ := r.ReleasedFor(ctx, "m", "src2:aa"); ok {
t.Fatal("a build not released is said released")
}
if _, err := r.Released(ctx, "m", v.Version); err != nil {
t.Fatal(err)
}
if ok, err := r.ReleasedFor(ctx, "m", "src2:aa"); !ok || err != nil {
t.Errorf("released for the fingerprint: %v (%v), want true", ok, err)
}
if ok, _ := r.ReleasedFor(ctx, "m", "src2:bb"); ok {
t.Error("another fingerprint is said released")
}
if all, _ := r.All(ctx); len(all) != 0 {
t.Errorf("reservations left: %v", all)
}
// One key per module holds the last releases, the oldest dropped (A-456b).
for _, fp := range []string{"src2:b1", "src2:b2"} {
n, err := r.Reserve(ctx, "m", "0.1", []string{"m:a"}, Source{Fingerprint: fp})
if err != nil {
t.Fatal(err)
}
if _, err := r.Released(ctx, "m", n.Version); err != nil {
t.Fatal(err)
}
}
if keys, _ := rel.KeysOf(ctx, AllFilter); len(keys) != 1 {
t.Errorf("release keys %v, want one per module", keys)
}
if old, _ := r.ReleasedFor(ctx, "m", "src2:aa"); old {
t.Error("the oldest release is kept past released-kept")
}
if ok, _ := r.ReleasedFor(ctx, "m", "src2:b2"); !ok {
t.Error("the newest release is not kept")
}
}
// TestARebuildReservesAboveTheCatalogsHighestWhateverItsStandingNumberWas: design 47 G A-490b — a build again of a done key replaces
// its fingerprint's standing reservation (and tells Took) and numbers above the catalog's highest alone, whether the standing number
// is below it or above it; another fingerprint's standing reservation stays a floor.
func TestARebuildReservesAboveTheCatalogsHighestWhateverItsStandingNumberWas(t *testing.T) {
ctx := context.Background()
for _, c := range []struct {
name, standing, standingFp string
want string
}{
{name: "standing below the catalog's highest", standing: "0.1.0", standingFp: "fp", want: "0.1.17"},
{name: "standing above the catalog's highest", standing: "0.1.30", standingFp: "fp", want: "0.1.17"},
{name: "another fingerprint's standing number stays a floor", standing: "0.1.30", standingFp: "other", want: "0.1.31"},
} {
t.Run(c.name, func(t *testing.T) {
store := &reserveMem{mem: newMem()}
r := &Reserver{Store: store, Filled: func() bool { return true },
ReleasedOf: func(context.Context, string) ([]string, error) { return []string{"0.1.16"}, nil }}
raw, _ := json.Marshal(ReservedVersion{V: 1, Module: "m", Series: "0.1", Version: c.standing, Fingerprint: c.standingFp, Artifacts: map[string]string{"m:a": "d"}})
if _, err := store.Create(ctx, ReservedKey("m", "0.1"), raw); err != nil {
t.Fatal(err)
}
var took []string
r.Took = func(_ context.Context, older, newer ReservedVersion) {
took = append(took, older.Version+">"+newer.Version)
}
got, err := r.Reserve(ctx, "m", "0.1", []string{"m:a"}, Source{Fingerprint: "fp", Rebuild: true})
if err != nil || got.Version != c.want || len(took) != 1 || took[0] != c.standing+">"+c.want {
t.Fatalf("reserved %q (%v), took %v; want %s replacing %s", got.Version, err, took, c.want, c.standing)
}
})
}
}
@@ -175,7 +175,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
}
// 1945: the builder's tools (builder_seat_queue: note 62), served by its tools bundle, and the event it says when it asks an ask again.
if want := []string{"builder_announce_again", "builder_ask_again", "builder_bounds", "builder_build", "builder_build_again", "builder_builds_since", "builder_engine_reads", "builder_first_view_again", "builder_first_views",
"builder_floor_again", "builder_floor_waits", "builder_old_form", "builder_publish_again", "builder_publish_queue", "builder_release", "builder_reserve_done", "builder_reserved", "builder_reserved_drop", "builder_seat_list", "builder_seat_list_again", "builder_seat_queue", "builder_stuck", "builder_toolchains", "builder_toolchains_drop", "builder_versions", "builder_waits"}; !reflect.DeepEqual(sorted(m.Tools), want) {
"builder_floor_again", "builder_floor_waits", "builder_old_form", "builder_publish_again", "builder_publish_queue", "builder_release", "builder_reserved", "builder_reserved_drop", "builder_seat_list", "builder_seat_list_again", "builder_seat_queue", "builder_stuck", "builder_toolchains", "builder_toolchains_drop", "builder_versions", "builder_waits"}; !reflect.DeepEqual(sorted(m.Tools), want) {
t.Errorf("tools = %v, want %v", m.Tools, want)
}
// Design 47 G §672: the toolchain waits' stream, kept 7 days within 16 MiB, and the bound's setting, 60 minutes.
@@ -185,9 +185,8 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
m.Streams[2].Name != "mesh-delivery-builder-findings" || !reflect.DeepEqual(m.Streams[2].Events, []string{"finding-written.*", "finding-cleared.*", "findings-stated.*"}) {
t.Errorf("streams = %+v, want waits with wait-held.** and wait-ended.**, 7 days, 16 MiB, the builds' stream (build.ended.**, 7 days, kept last per subject: design 47 G A-181) and the findings' stream", m.Streams)
}
if c, ok := m.Config["toolchain-wait-minutes"]; !ok || c.Default != float64(60) || len(m.Config) != 5 ||
m.Config["publish-pause"].Default != float64(30) || m.Config["publish-pause-max"].Default != float64(30) || m.Config["publish-tries"].Default != float64(8) {
t.Errorf("config = %+v, want toolchain-wait-minutes of 60, the publish settings 30 s, 30 min and 8 tries (design 47 G §588, §633, A-457a)", m.Config)
if c, ok := m.Config["toolchain-wait-minutes"]; !ok || c.Default != float64(60) || len(m.Config) != 2 || m.Config["republish-max-versions"].Default != float64(50) {
t.Errorf("config = %+v, want toolchain-wait-minutes of 60 and republish-max-versions of 50 (design 47 G §588, §633)", m.Config)
}
// Design 47 G §349: version.numbered is no longer said; old-form-closed on the family version, apart from the builds' ends.
if want := []string{"build.asked-again", "version.old-form-closed"}; !reflect.DeepEqual(m.Emits, want) {
@@ -240,7 +239,7 @@ func TestManifestProcess(t *testing.T) { // 1607/AC1
t.Errorf("process = %v", p)
}
env, _ := p["env"].(map[string]any)
if len(env) != 6 || env["MESH_BUILDER_PUBLISH_TRIES_FILE"] != "${dir:state}/publish-tries" || env["MESH_BROKER_FILE"] != "${dir:state}/broker" || env["MESH_BUILDER_TOOLCHAIN_WAIT_FILE"] != "${dir:state}/toolchain-wait-minutes" {
if len(env) != 3 || env["MESH_BROKER_FILE"] != "${dir:state}/broker" || env["MESH_BUILDER_REPUBLISH_MAX_FILE"] != "${dir:state}/republish-max-versions" || env["MESH_BUILDER_TOOLCHAIN_WAIT_FILE"] != "${dir:state}/toolchain-wait-minutes" {
t.Errorf("env = %v, want the broker file and the path of the toolchain wait's file, never the setting itself (design 47 G §672, A-71)", p["env"])
}
for k, v := range env {
+8 -60
View File
@@ -29,7 +29,6 @@
"builder_reserved",
"builder_release",
"builder_reserved_drop",
"builder_reserve_done",
"builder_seat_queue",
"builder_stuck",
"builder_toolchains",
@@ -147,37 +146,13 @@
"trusted": false,
"default": 60
},
"publish-pause": {
"republish-max-versions": {
"type": "number",
"description": "seconds before a version's second publish try, doubled per try up to publish-pause-max (design 47 G A-457a).",
"description": "the most versions one event of a registry holder queues to publish again, newest first (design 47 G section 588); the rest wait for the next event or builder_publish_again.",
"minimum": 1,
"maximum": 3600,
"maximum": 10000,
"trusted": false,
"default": 30
},
"publish-pause-max": {
"type": "number",
"description": "minutes, the most a pause between a version's publish tries grows to (design 47 G A-457a).",
"minimum": 1,
"maximum": 1440,
"trusted": false,
"default": 30
},
"released-kept": {
"type": "number",
"description": "how many of a module's last releases the builder keeps, by fingerprint, to judge a rerun of a released build's step (design 47 G A-456b); the oldest is dropped.",
"minimum": 1,
"maximum": 1000,
"trusted": false,
"default": 16
},
"publish-tries": {
"type": "number",
"description": "how many times the builder tries a done build's publish before it ends loudly (log and condition) and leaves it readable in builder_publish_queue (design 47 G A-457a).",
"minimum": 1,
"maximum": 100,
"trusted": false,
"default": 8
"default": 50
}
},
"logs": [
@@ -224,36 +199,12 @@
"content": "${config:toolchain-wait-minutes}"
},
{
"id": "publish-pause",
"id": "republish-max",
"type": "file",
"path": "${dir:state}/publish-pause",
"path": "${dir:state}/republish-max-versions",
"owner": "mesh-delivery-builder",
"mode": "0600",
"content": "${config:publish-pause}"
},
{
"id": "publish-pause-max",
"type": "file",
"path": "${dir:state}/publish-pause-max",
"owner": "mesh-delivery-builder",
"mode": "0600",
"content": "${config:publish-pause-max}"
},
{
"id": "released-kept",
"type": "file",
"path": "${dir:state}/released-kept",
"owner": "mesh-delivery-builder",
"mode": "0600",
"content": "${config:released-kept}"
},
{
"id": "publish-tries",
"type": "file",
"path": "${dir:state}/publish-tries",
"owner": "mesh-delivery-builder",
"mode": "0600",
"content": "${config:publish-tries}"
"content": "${config:republish-max-versions}"
},
{
"id": "builder",
@@ -269,10 +220,7 @@
"env": {
"MESH_BROKER_FILE": "${dir:state}/broker",
"MESH_BUILDER_TOOLCHAIN_WAIT_FILE": "${dir:state}/toolchain-wait-minutes",
"MESH_BUILDER_PUBLISH_PAUSE_FILE": "${dir:state}/publish-pause",
"MESH_BUILDER_PUBLISH_PAUSE_MAX_FILE": "${dir:state}/publish-pause-max",
"MESH_BUILDER_PUBLISH_TRIES_FILE": "${dir:state}/publish-tries",
"MESH_BUILDER_RELEASED_KEPT_FILE": "${dir:state}/released-kept"
"MESH_BUILDER_REPUBLISH_MAX_FILE": "${dir:state}/republish-max-versions"
},
"user": "mesh-delivery-builder",
"health": {
@@ -422,7 +422,7 @@ func RequestDropTools(s check.StateDrop, acts views.HandActs) []stdio.Tool {
key, _ := args["check"].(string)
why, _ := args["why"].(string)
return check.DropRequests(s, key, why, func(what string) error {
return acts.RecordHandAct(views.HandAct{What: what, Why: why, Cause: "requests-drop", Verb: check.RequestsDropTool})
return acts.RecordHandAct(views.HandAct{What: what, Why: why, Verb: check.RequestsDropTool})
})
}}}
}
@@ -555,8 +555,8 @@ func TestOnlyThePackageWithoutAKeptVerdictRuns(t *testing.T) { // 1288/AC1
t.Fatalf("verdict %+v, asks %+v: want a pass of three asks, the first naming no need", out.Verdict, asks)
}
for _, a := range asks { // ADR 0584: every check ask names the agreed toolchain, so a node off it declines
if a.Check.Toolchain == nil || a.Check.Toolchain.Version != "go1.27" || a.Check.Toolchain.Recipe != "7" {
t.Errorf("a check ask names toolchain %+v, want go1.27 recipe 7", a.Check.Toolchain)
if a.Check.Toolchain == nil || a.Check.Toolchain.Version != "go1.27.1" || a.Check.Toolchain.Recipe != "7" {
t.Errorf("a check ask names toolchain %+v, want go1.27.1 recipe 7", a.Check.Toolchain)
}
}
for _, a := range asks[1:] {
@@ -1191,7 +1191,7 @@ func TestACheckIsOneAskAndAReJudgeGoesOnTheRecheckStream(t *testing.T) {
t.Fatalf("the job = %+v", asks[0].Check)
}
// B2g: the job carries the toolchain the checker agreed (ADR 0584), with its language, so a node on another declines it.
if tc := asks[0].Check.Toolchain; tc == nil || *tc != (buildstep.ToolchainUsed{Language: "go", Version: "go1.27", Recipe: "7"}) {
if tc := asks[0].Check.Toolchain; tc == nil || *tc != (buildstep.ToolchainUsed{Language: "go", Version: "go1.27.1", Recipe: "7"}) {
t.Errorf("the job's toolchain = %+v, want the agreed go toolchain", tc)
}
cs := caseOf(t, pullOf(repoA, 7, head1))
@@ -15,20 +15,18 @@ import (
const Seat = "mesh-pipeline-runner"
const (
VerbRun = "run"
VerbStart = "start"
VerbShow = "show"
VerbList = "list"
VerbStands = "stands"
VerbWhyCarried = "why-carried"
VerbKept = "kept"
VerbReview = "review"
VerbRerun = "rerun"
VerbAccept = "accept"
VerbCancel = "cancel"
VerbAskAgain = "ask-again"
VerbTestRuns = "test-runs"
VerbTestRunsDrop = "test-runs-drop"
VerbRun = "run"
VerbStart = "start"
VerbShow = "show"
VerbList = "list"
VerbStands = "stands"
VerbWhyCarried = "why-carried"
VerbKept = "kept"
VerbReview = "review"
VerbRerun = "rerun"
VerbAccept = "accept"
VerbCancel = "cancel"
VerbAskAgain = "ask-again"
)
const (
@@ -47,7 +45,7 @@ const (
// Verbs is every verb the seat serves, in its definition's order: a new copy each call.
func Verbs() []string {
return []string{VerbRun, VerbStart, VerbShow, VerbList, VerbStands, VerbWhyCarried, VerbKept, VerbReview, VerbRerun, VerbAccept, VerbCancel, VerbAskAgain, VerbTestRuns, VerbTestRunsDrop}
return []string{VerbRun, VerbStart, VerbShow, VerbList, VerbStands, VerbWhyCarried, VerbKept, VerbReview, VerbRerun, VerbAccept, VerbCancel, VerbAskAgain}
}
// Events is every event published on the seat's subject: a new copy each call.
@@ -598,26 +596,6 @@ type RunStarted struct {
V int64 `json:"v"`
}
// SharedAskedBy is the step that asked the run, for a test step reading another's run.
type SharedAskedBy struct {
// its artifact, <module>:<artifact>
Artifact string `json:"artifact"`
// its pipeline's key
Pipeline string `json:"pipeline"`
}
// Shared is a run whose result a step took or waits on, asked once for every step that reads it (design 47 G §729, A-469t): for a composition step one per machine composed, `run` the `composition-runs` key and `ask` its run ask's id; for a test step one with no machine, `run` the `test-runs` key, `ask` its build seat ask's id and `asked-by` the step that asked the run, left out for that step itself.
type Shared struct {
// the id of the run's ask of the build seat
Ask string `json:"ask"`
// the step that asked the run, for a test step reading another's run
AskedBy *SharedAskedBy `json:"asked-by,omitempty"`
// the machine composed, for a composition run
Machine string `json:"machine,omitempty"`
// the run's key: in the assigner's composition-runs, or the runner's test-runs
Run string `json:"run"`
}
// StepEvent is a step's start or end: the step, its status, its outcome, its fingerprint, its test counts, what a carried result was carried from, and for a rerun that it is one and its proved caller; a bypassed step too.
type StepEvent struct {
At *time.Time `json:"at,omitempty"`
@@ -667,9 +645,7 @@ type StepResult struct {
// the step was asked again by the runner's rerun
Rerun bool `json:"rerun,omitempty"`
// the seat that serves the step, whose holder alone may end it (ADR 0666 section 21)
Seat string `json:"seat,omitempty"`
// the run the step read or asked for every step that reads it: a composition step's per machine, as mesh-wants' step-ended names them; a test step's one, with asked-by when another step asked it (design 47 G §726, §729; A-468d, A-468e, A-469t)
Shared []Shared `json:"shared,omitempty"`
Seat string `json:"seat,omitempty"`
Started *time.Time `json:"started,omitempty"`
// the step's status in the module's policy
Status string `json:"status"`
@@ -722,56 +698,6 @@ type TestCounts struct {
Unit int64 `json:"unit,omitempty"`
}
// TestRun is one test run of a candidate, level and tested packages, a key of the runner's state test-runs (design 47 G §725, §728; A-469u).
type TestRun struct {
// the id of the build seat's ask
Ask string `json:"ask,omitempty"`
// the share digest (design 47 G §724)
Digest string `json:"digest"`
Ended *time.Time `json:"ended,omitempty"`
// why the packages are not named: the run's output is gone
Gone string `json:"gone,omitempty"`
// the key, <candidate key>.<level>.<digest>, each part escaped as a key's name
Key string `json:"key"`
Level string `json:"level"`
// the node whose holder took the ask
Node string `json:"node,omitempty"`
Outcome string `json:"outcome,omitempty"`
// the run's output object, by its name in test-runs-objects
Output string `json:"output,omitempty"`
// the packages it ran, read from its output object
Packages []TestRunPackage `json:"packages,omitempty"`
// the steps reading the run
Readers []TestRunReader `json:"readers,omitempty"`
// the outcome's words
Said string `json:"said,omitempty"`
// the build seat asked
Seat string `json:"seat,omitempty"`
Started *time.Time `json:"started,omitempty"`
Tests *TestCounts `json:"tests,omitempty"`
}
// The values of TestRun's level.
const (
TestRunLevelLint = "lint"
TestRunLevelUnit = "unit"
TestRunLevelIntegration = "integration"
)
// TestRunPackage is one package a test run ran, with its digest (design 47 G §724).
type TestRunPackage struct {
Digest string `json:"digest"`
Package string `json:"package"`
}
// TestRunReader is a test step reading a test run (design 47 G §725).
type TestRunReader struct {
// its artifact, <module>:<artifact>
Artifact string `json:"artifact"`
// its pipeline's key
Pipeline string `json:"pipeline"`
}
// ToolchainUsed is the toolchain a build was made with: the language, the installed version and the build module's recipe.
type ToolchainUsed struct {
Language string `json:"language"`
@@ -782,8 +708,6 @@ type ToolchainUsed struct {
// Wait is a step's wait, held and ended; a wait on a person is said as waiting on that person, never a stall (ADR 0283).
type Wait struct {
Artifact string `json:"artifact,omitempty"`
// the step's own ask, where it has one (design 47 G §731)
Ask string `json:"ask,omitempty"`
// the watched branch, or the branch of a run on request
Branch string `json:"branch"`
// the commit the module's entry was last changed at
@@ -802,10 +726,8 @@ type Wait struct {
// the catalog graph revision of the branch the pipeline runs for
Revision int64 `json:"revision,omitempty"`
// the run on request's id, for a pipeline of a run; such a pipeline delivers nothing
Run string `json:"run,omitempty"`
// the run the step waits on, where the runner knows it: a test step's run key and ask (design 47 G §731, A-469v)
Shared []Shared `json:"shared,omitempty"`
Since time.Time `json:"since"`
Run string `json:"run,omitempty"`
Since time.Time `json:"since"`
// the source revision of the watched branch; the pipeline is keyed by it (ADR 0666 section 9)
Source int64 `json:"source,omitempty"`
// a pipeline step a wait inside a pipeline is in
@@ -1055,44 +977,6 @@ type AskAgainAnswer struct {
Result StepResult `json:"result"`
}
// TestRunsInput is what test-runs takes.
type TestRunsInput struct {
// the candidate's key
Candidate string `json:"candidate,omitempty"`
Digest string `json:"digest,omitempty"`
Level string `json:"level,omitempty"`
Module string `json:"module,omitempty"`
}
// The values of TestRunsInput's level.
const (
TestRunsInputLevelLint = "lint"
TestRunsInputLevelUnit = "unit"
TestRunsInputLevelIntegration = "integration"
)
// TestRunsAnswer is what test-runs answers.
type TestRunsAnswer struct {
Fill Fill `json:"fill"`
Runs []TestRun `json:"runs"`
}
// TestRunsDropInput is what test-runs-drop takes.
type TestRunsDropInput struct {
// the run's key in test-runs
Key string `json:"key"`
// why, written with the caller the runtime proves in the hand-act log
Why string `json:"why"`
}
// TestRunsDropAnswer is what test-runs-drop answers.
type TestRunsDropAnswer struct {
// the readers it ended
Ended []TestRunReader `json:"ended,omitempty"`
// the entry deleted
Run TestRun `json:"run"`
}
// PipelineStartedEvent is the body of the event pipeline-started.
type PipelineStartedEvent = PipelineStarted
@@ -37,12 +37,6 @@ func TestFixtures(t *testing.T) {
decode: func(b []byte) (any, error) { var v WaitHeldEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-pipeline-runner\\.event\\.wait-held\\.[^.*> ]+\\.[^.*> ]+\\.[^.*> ]+$")},
{name: "the wait ended answered", subject: "mesh.seat.mesh-pipeline-runner.event.wait-ended.nats.main.0123456789abcdef0123456789abcdef01234567", given: "{\"branch\":\"main\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"how\":\"answered\",\"module\":\"nats\",\"on\":\"person\",\"pipeline\":\"main.12.nats\",\"revision\":12,\"since\":\"2026-11-10T10:10:00Z\",\"source\":3589,\"step\":\"approval\",\"v\":1,\"who\":\"the operator\"}", wire: "{\"branch\":\"main\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"how\":\"answered\",\"module\":\"nats\",\"on\":\"person\",\"pipeline\":\"main.12.nats\",\"revision\":12,\"since\":\"2026-11-10T10:10:00Z\",\"source\":3589,\"step\":\"approval\",\"v\":1,\"who\":\"the operator\"}",
decode: func(b []byte) (any, error) { var v WaitEndedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-pipeline-runner\\.event\\.wait-ended\\.[^.*> ]+\\.[^.*> ]+\\.[^.*> ]+$")},
{name: "C1/AC2 a test step started reading a test run, naming its run and ask", subject: "mesh.seat.mesh-pipeline-runner.event.step-started.nats.main.0123456789abcdef0123456789abcdef01234567", given: "{\"at\":\"2026-11-17T10:00:00Z\",\"branch\":\"main\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"pipeline\":\"main.12.nats\",\"result\":{\"artifact\":\"nats:tools\",\"fingerprint\":\"fp-0123\",\"outcome\":\"running\",\"seat\":\"node-build-go\",\"shared\":[{\"ask\":\"t1a2\",\"run\":\"main.12.unit.d1\"}],\"status\":\"required\",\"step\":\"unit\"},\"revision\":12,\"source\":3589,\"v\":1}", wire: "{\"at\":\"2026-11-17T10:00:00Z\",\"branch\":\"main\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"pipeline\":\"main.12.nats\",\"result\":{\"artifact\":\"nats:tools\",\"fingerprint\":\"fp-0123\",\"outcome\":\"running\",\"seat\":\"node-build-go\",\"shared\":[{\"ask\":\"t1a2\",\"run\":\"main.12.unit.d1\"}],\"status\":\"required\",\"step\":\"unit\"},\"revision\":12,\"source\":3589,\"v\":1}",
decode: func(b []byte) (any, error) { var v StepStartedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-pipeline-runner\\.event\\.step-started\\.[^.*> ]+\\.[^.*> ]+\\.[^.*> ]+$")},
{name: "C1/AC2 a test step ended with the run another step asked", subject: "mesh.seat.mesh-pipeline-runner.event.step-ended.nats.main.0123456789abcdef0123456789abcdef01234567", given: "{\"at\":\"2026-11-17T10:05:00Z\",\"branch\":\"main\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"pipeline\":\"main.12.nats\",\"result\":{\"artifact\":\"nats:tools\",\"fingerprint\":\"fp-0123\",\"node\":\"anchor\",\"outcome\":\"failed\",\"seat\":\"node-build-go\",\"shared\":[{\"ask\":\"t1a2\",\"asked-by\":{\"artifact\":\"nats:program\",\"pipeline\":\"main.12.nats\"},\"run\":\"main.12.unit.d1\"}],\"status\":\"required\",\"step\":\"unit\",\"tests\":{\"unit\":42}},\"revision\":12,\"source\":3589,\"v\":1}", wire: "{\"at\":\"2026-11-17T10:05:00Z\",\"branch\":\"main\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"pipeline\":\"main.12.nats\",\"result\":{\"artifact\":\"nats:tools\",\"fingerprint\":\"fp-0123\",\"node\":\"anchor\",\"outcome\":\"failed\",\"seat\":\"node-build-go\",\"shared\":[{\"ask\":\"t1a2\",\"asked-by\":{\"artifact\":\"nats:program\",\"pipeline\":\"main.12.nats\"},\"run\":\"main.12.unit.d1\"}],\"status\":\"required\",\"step\":\"unit\",\"tests\":{\"unit\":42}},\"revision\":12,\"source\":3589,\"v\":1}",
decode: func(b []byte) (any, error) { var v StepEndedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-pipeline-runner\\.event\\.step-ended\\.[^.*> ]+\\.[^.*> ]+\\.[^.*> ]+$")},
{name: "C1 a composition step ended with mesh-wants' runs as they came", subject: "mesh.seat.mesh-pipeline-runner.event.step-ended.nats.main.0123456789abcdef0123456789abcdef01234567", given: "{\"at\":\"2026-11-17T10:05:00Z\",\"branch\":\"main\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"pipeline\":\"main.12.nats\",\"result\":{\"fingerprint\":\"fp-0123\",\"node\":\"laptop\",\"outcome\":\"passed\",\"seat\":\"mesh-wants\",\"shared\":[{\"ask\":\"r1a2b3\",\"machine\":\"anchor\",\"run\":\"main.12.anchor\"}],\"status\":\"required\",\"step\":\"composition\"},\"revision\":12,\"source\":3589,\"v\":1}", wire: "{\"at\":\"2026-11-17T10:05:00Z\",\"branch\":\"main\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"pipeline\":\"main.12.nats\",\"result\":{\"fingerprint\":\"fp-0123\",\"node\":\"laptop\",\"outcome\":\"passed\",\"seat\":\"mesh-wants\",\"shared\":[{\"ask\":\"r1a2b3\",\"machine\":\"anchor\",\"run\":\"main.12.anchor\"}],\"status\":\"required\",\"step\":\"composition\"},\"revision\":12,\"source\":3589,\"v\":1}",
decode: func(b []byte) (any, error) { var v StepEndedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-pipeline-runner\\.event\\.step-ended\\.[^.*> ]+\\.[^.*> ]+\\.[^.*> ]+$")},
{name: "a pipeline kept", subject: "", given: "{\"branch\":\"main\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"end-count\":1,\"ended\":\"2026-11-10T10:20:00Z\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"outcome\":\"passed\",\"pipeline\":\"main.12.nats\",\"revision\":12,\"source\":3589,\"started\":\"2026-11-10T10:00:00Z\",\"steps\":[{\"artifact\":\"nats:program\",\"ask\":\"m2kx9a1b3c4d5e\",\"fingerprint\":\"fp-0123\",\"node\":\"anchor\",\"outcome\":\"passed\",\"seat\":\"node-build-go\",\"status\":\"required\",\"step\":\"lint\"},{\"artifact\":\"nats:program\",\"carried-from\":\"main.11.nats\",\"fingerprint\":\"fp-0123\",\"outcome\":\"carried\",\"status\":\"required\",\"step\":\"unit\",\"tests\":{\"unit\":42}},{\"artifact\":\"nats:program\",\"fingerprint\":\"fp-0123\",\"node\":\"home-server\",\"outcome\":\"failed\",\"seat\":\"node-build-go\",\"status\":\"optional\",\"step\":\"integration\",\"tests\":{\"integration\":3},\"why\":\"TestTheBusWhole failed\"},{\"outcome\":\"passed\",\"seat\":\"mesh-wants\",\"status\":\"required\",\"step\":\"composition\"},{\"outcome\":\"bypassed\",\"status\":\"bypass\",\"step\":\"review\"},{\"outcome\":\"passed\",\"status\":\"required\",\"step\":\"approval\"}],\"v\":1}", wire: "{\"branch\":\"main\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"end-count\":1,\"ended\":\"2026-11-10T10:20:00Z\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"outcome\":\"passed\",\"pipeline\":\"main.12.nats\",\"revision\":12,\"source\":3589,\"started\":\"2026-11-10T10:00:00Z\",\"steps\":[{\"artifact\":\"nats:program\",\"ask\":\"m2kx9a1b3c4d5e\",\"fingerprint\":\"fp-0123\",\"node\":\"anchor\",\"outcome\":\"passed\",\"seat\":\"node-build-go\",\"status\":\"required\",\"step\":\"lint\"},{\"artifact\":\"nats:program\",\"carried-from\":\"main.11.nats\",\"fingerprint\":\"fp-0123\",\"outcome\":\"carried\",\"status\":\"required\",\"step\":\"unit\",\"tests\":{\"unit\":42}},{\"artifact\":\"nats:program\",\"fingerprint\":\"fp-0123\",\"node\":\"home-server\",\"outcome\":\"failed\",\"seat\":\"node-build-go\",\"status\":\"optional\",\"step\":\"integration\",\"tests\":{\"integration\":3},\"why\":\"TestTheBusWhole failed\"},{\"outcome\":\"passed\",\"seat\":\"mesh-wants\",\"status\":\"required\",\"step\":\"composition\"},{\"outcome\":\"bypassed\",\"status\":\"bypass\",\"step\":\"review\"},{\"outcome\":\"passed\",\"status\":\"required\",\"step\":\"approval\"}],\"v\":1}",
decode: func(b []byte) (any, error) {
var v PipelinesByBranchRevisionModuleValue
@@ -73,12 +67,6 @@ func TestFixtures(t *testing.T) {
decode: func(b []byte) (any, error) { var v StandsAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil},
{name: "kept answers by digest", subject: "", given: "{\"fill\":{\"bytes\":300,\"cap\":134217728,\"store\":\"kept\"},\"kept\":[{\"digest\":\"sha256:aa\",\"level\":\"unit\",\"package\":\"git.novox.be/novox/nats\",\"pipeline\":\"main.12.nats\"}]}", wire: "{\"fill\":{\"bytes\":300,\"cap\":134217728,\"store\":\"kept\"},\"kept\":[{\"digest\":\"sha256:aa\",\"level\":\"unit\",\"package\":\"git.novox.be/novox/nats\",\"pipeline\":\"main.12.nats\"}]}",
decode: func(b []byte) (any, error) { var v KeptAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil},
{name: "C1/AC3 stands answers a waiting test step with its ask and the run it waits on", subject: "", given: "{\"stands\":[{\"artifact\":\"nats:tools\",\"ask\":\"t1a2\",\"branch\":\"main\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"on\":\"build node\",\"pipeline\":\"main.12.nats\",\"revision\":12,\"shared\":[{\"ask\":\"t1a2\",\"run\":\"main.12.unit.d1\"}],\"since\":\"2026-11-17T10:00:00Z\",\"source\":3589,\"step\":\"unit\",\"v\":1}]}", wire: "{\"stands\":[{\"artifact\":\"nats:tools\",\"ask\":\"t1a2\",\"branch\":\"main\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"module\":\"nats\",\"on\":\"build node\",\"pipeline\":\"main.12.nats\",\"revision\":12,\"shared\":[{\"ask\":\"t1a2\",\"run\":\"main.12.unit.d1\"}],\"since\":\"2026-11-17T10:00:00Z\",\"source\":3589,\"step\":\"unit\",\"v\":1}]}",
decode: func(b []byte) (any, error) { var v StandsAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil},
{name: "C1 test-runs answers each run with its packages, readers and the fill", subject: "", given: "{\"fill\":{\"bytes\":2048,\"cap\":16777216,\"store\":\"test-runs\"},\"runs\":[{\"ask\":\"t1a2\",\"digest\":\"d1\",\"ended\":\"2026-11-17T10:05:00Z\",\"key\":\"main.12.unit.d1\",\"level\":\"unit\",\"node\":\"anchor\",\"outcome\":\"failed\",\"output\":\"t1a2\",\"packages\":[{\"digest\":\"sha256:aa\",\"package\":\"git.novox.be/novox/nats/internal/x\"}],\"readers\":[{\"artifact\":\"nats:program\",\"pipeline\":\"main.12.nats\"},{\"artifact\":\"nats:tools\",\"pipeline\":\"main.12.nats\"}],\"said\":\"1 test failed\",\"seat\":\"node-build-go\",\"started\":\"2026-11-17T10:00:00Z\",\"tests\":{\"unit\":42}},{\"ask\":\"t1a2\",\"digest\":\"d1\",\"ended\":\"2026-11-17T10:05:00Z\",\"gone\":\"the run's output is gone\",\"key\":\"main.12.unit.d1\",\"level\":\"unit\",\"node\":\"anchor\",\"outcome\":\"failed\",\"output\":\"t1a2\",\"readers\":[{\"artifact\":\"nats:program\",\"pipeline\":\"main.12.nats\"},{\"artifact\":\"nats:tools\",\"pipeline\":\"main.12.nats\"}],\"said\":\"1 test failed\",\"seat\":\"node-build-go\",\"started\":\"2026-11-17T10:00:00Z\",\"tests\":{\"unit\":42}}]}", wire: "{\"fill\":{\"bytes\":2048,\"cap\":16777216,\"store\":\"test-runs\"},\"runs\":[{\"ask\":\"t1a2\",\"digest\":\"d1\",\"ended\":\"2026-11-17T10:05:00Z\",\"key\":\"main.12.unit.d1\",\"level\":\"unit\",\"node\":\"anchor\",\"outcome\":\"failed\",\"output\":\"t1a2\",\"packages\":[{\"digest\":\"sha256:aa\",\"package\":\"git.novox.be/novox/nats/internal/x\"}],\"readers\":[{\"artifact\":\"nats:program\",\"pipeline\":\"main.12.nats\"},{\"artifact\":\"nats:tools\",\"pipeline\":\"main.12.nats\"}],\"said\":\"1 test failed\",\"seat\":\"node-build-go\",\"started\":\"2026-11-17T10:00:00Z\",\"tests\":{\"unit\":42}},{\"ask\":\"t1a2\",\"digest\":\"d1\",\"ended\":\"2026-11-17T10:05:00Z\",\"gone\":\"the run's output is gone\",\"key\":\"main.12.unit.d1\",\"level\":\"unit\",\"node\":\"anchor\",\"outcome\":\"failed\",\"output\":\"t1a2\",\"readers\":[{\"artifact\":\"nats:program\",\"pipeline\":\"main.12.nats\"},{\"artifact\":\"nats:tools\",\"pipeline\":\"main.12.nats\"}],\"said\":\"1 test failed\",\"seat\":\"node-build-go\",\"started\":\"2026-11-17T10:00:00Z\",\"tests\":{\"unit\":42}}]}",
decode: func(b []byte) (any, error) { var v TestRunsAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil},
{name: "C1 test-runs-drop answers the run it deleted and the readers it ended", subject: "", given: "{\"ended\":[{\"artifact\":\"nats:program\",\"pipeline\":\"main.12.nats\"},{\"artifact\":\"nats:tools\",\"pipeline\":\"main.12.nats\"}],\"run\":{\"ask\":\"t1a2\",\"digest\":\"d1\",\"ended\":\"2026-11-17T10:05:00Z\",\"key\":\"main.12.unit.d1\",\"level\":\"unit\",\"node\":\"anchor\",\"outcome\":\"failed\",\"output\":\"t1a2\",\"packages\":[{\"digest\":\"sha256:aa\",\"package\":\"git.novox.be/novox/nats/internal/x\"}],\"readers\":[{\"artifact\":\"nats:program\",\"pipeline\":\"main.12.nats\"},{\"artifact\":\"nats:tools\",\"pipeline\":\"main.12.nats\"}],\"said\":\"1 test failed\",\"seat\":\"node-build-go\",\"started\":\"2026-11-17T10:00:00Z\",\"tests\":{\"unit\":42}}}", wire: "{\"ended\":[{\"artifact\":\"nats:program\",\"pipeline\":\"main.12.nats\"},{\"artifact\":\"nats:tools\",\"pipeline\":\"main.12.nats\"}],\"run\":{\"ask\":\"t1a2\",\"digest\":\"d1\",\"ended\":\"2026-11-17T10:05:00Z\",\"key\":\"main.12.unit.d1\",\"level\":\"unit\",\"node\":\"anchor\",\"outcome\":\"failed\",\"output\":\"t1a2\",\"packages\":[{\"digest\":\"sha256:aa\",\"package\":\"git.novox.be/novox/nats/internal/x\"}],\"readers\":[{\"artifact\":\"nats:program\",\"pipeline\":\"main.12.nats\"},{\"artifact\":\"nats:tools\",\"pipeline\":\"main.12.nats\"}],\"said\":\"1 test failed\",\"seat\":\"node-build-go\",\"started\":\"2026-11-17T10:00:00Z\",\"tests\":{\"unit\":42}}}",
decode: func(b []byte) (any, error) { var v TestRunsDropAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil},
}
for _, c := range cases {
v, err := c.decode([]byte(c.given))
@@ -15,14 +15,12 @@ import (
const Seat = "mesh-wants"
const (
VerbUpgrade = "upgrade"
VerbBus = "bus"
VerbRevision = "revision"
VerbEntry = "entry"
VerbDifference = "difference"
VerbInput = "input"
VerbCompositions = "compositions"
VerbCompositionsDrop = "compositions-drop"
VerbUpgrade = "upgrade"
VerbBus = "bus"
VerbRevision = "revision"
VerbEntry = "entry"
VerbDifference = "difference"
VerbInput = "input"
)
const (
@@ -46,7 +44,7 @@ const (
// Verbs is every verb the seat serves, in its definition's order: a new copy each call.
func Verbs() []string {
return []string{VerbUpgrade, VerbBus, VerbRevision, VerbEntry, VerbDifference, VerbInput, VerbCompositions, VerbCompositionsDrop}
return []string{VerbUpgrade, VerbBus, VerbRevision, VerbEntry, VerbDifference, VerbInput}
}
// Events is every event published on the seat's subject: a new copy each call.
@@ -127,6 +125,7 @@ const (
StateDifferences = "differences"
StateCompositions = "compositions"
StateAssignmentRevisions = "assignment-revisions"
StateInputs = "inputs"
)
// WantsByWantNodeModuleKey is the key of one value of the state wants under the form "want.<node>.<module>".
@@ -742,6 +741,42 @@ func DecodeAssignmentRevisionsByEntryDigestValue(raw []byte) (AssignmentRevision
return v, nil
}
// InputsByTimeDigestKey is the key of one value of the state inputs under the form "<time>.<digest>".
func InputsByTimeDigestKey(time, digest string) string { return time + "." + digest }
// ParseInputsByTimeDigestKey reads the parts of a key of the form "<time>.<digest>", in the order of the form; the last result is false for a key of another form.
func ParseInputsByTimeDigestKey(key string) (string, string, bool) {
rest := key
v0, after0, found0 := strings.Cut(rest, ".")
if !found0 || v0 == "" {
return "", "", false
}
rest = after0
if rest == "" {
return "", "", false
}
v2 := rest
rest = ""
if rest != "" {
return "", "", false
}
return v0, v2, true
}
// DecodeInputsByTimeDigestValue reads one value of the state inputs under the form "<time>.<digest>", refusing a field InputsByTimeDigestValue does not know and any text after the value.
func DecodeInputsByTimeDigestValue(raw []byte) (InputsByTimeDigestValue, error) {
var v InputsByTimeDigestValue
dec := json.NewDecoder(bytes.NewReader(raw))
dec.DisallowUnknownFields()
if err := dec.Decode(&v); err != nil {
return v, err
}
if dec.More() {
return v, errors.New("text after the value")
}
return v, nil
}
// Ask is an ask the assigner puts to a person: the router's ask whole, as the assigner composes it, and how many times it was written at its key.
type Ask struct {
// the seat operator-channel's ask, whole
@@ -832,79 +867,6 @@ type Change struct {
To string `json:"to,omitempty"`
}
// CompositionReader is a composition step reading a composition run: its step, as the runner asked it, with its ask's id (design 47 G §719, A-468).
type CompositionReader struct {
// the watched branch, or the branch of a run on request
Branch string `json:"branch"`
// for a rerun, the caller the runtime proved
By string `json:"by,omitempty"`
// the commit the step runs at
Commit string `json:"commit"`
// when the reader's step said its end; carried into a rerun's run, so a take-up tells a reader that passed from one that joined later (A-470m)
Ended *time.Time `json:"ended,omitempty"`
// the module's build-input fingerprint at the commit
Fingerprint string `json:"fingerprint"`
// the step's ask's id
ID string `json:"id"`
Module string `json:"module"`
// the runner's pipeline key, <branch>.<revision>.<module>, a branch escaped as a key's name is
Pipeline string `json:"pipeline"`
// the step is asked again by the runner's rerun
Rerun bool `json:"rerun,omitempty"`
// the catalog graph revision of the branch the pipeline runs for
Revision int64 `json:"revision,omitempty"`
// the run on request's id: its composition writes no want (ADR 0666 section 18)
Run string `json:"run,omitempty"`
// the source revision of the watched branch; absent for a run on request
Source int64 `json:"source,omitempty"`
Step string `json:"step"`
}
// The values of CompositionReader's step.
const (
CompositionReaderStepComposition = "composition"
)
// CompositionRunProgram is the composer program, chosen once when the run started.
type CompositionRunProgram struct {
Commit string `json:"commit,omitempty"`
Directory string `json:"directory,omitempty"`
// the node-engine fingerprint it was chosen from
Fingerprint string `json:"fingerprint,omitempty"`
Repository string `json:"repository,omitempty"`
}
// CompositionRun is one composition run of a candidate and a machine, a key of the assigner's state composition-runs (design 47 G §719, §728; A-469u).
type CompositionRun struct {
// the id of the build seat's run ask
Ask string `json:"ask,omitempty"`
// the candidate's key: <branch>.<revision>, or run-<id> for a run on request
Candidate string `json:"candidate"`
Ended *time.Time `json:"ended,omitempty"`
// the digest of the kept request
Input string `json:"input,omitempty"`
// the key, <candidate key>.<machine>, each part escaped as a key's name
Key string `json:"key"`
Machine string `json:"machine"`
Outcome string `json:"outcome,omitempty"`
// the composer program, chosen once when the run started
Program *CompositionRunProgram `json:"program,omitempty"`
// the composition steps reading the run
Readers []CompositionReader `json:"readers,omitempty"`
// the outcome's words
Said string `json:"said,omitempty"`
Started *time.Time `json:"started,omitempty"`
// that machine's composer verdict (design 47 G §721)
Verdict map[string]any `json:"verdict,omitempty"`
}
// The values of CompositionRun's outcome.
const (
CompositionRunOutcomePassed = "passed"
CompositionRunOutcomeFailed = "failed"
CompositionRunOutcomeCouldNotRun = "could not run"
)
// DifferenceRefusedItem is a type of the contract.
type DifferenceRefusedItem struct {
Module string `json:"module"`
@@ -982,26 +944,6 @@ type NodeEntryRef struct {
Node string `json:"node"`
}
// SharedAskedBy is the step that asked the run, for a test step reading another's run.
type SharedAskedBy struct {
// its artifact, <module>:<artifact>
Artifact string `json:"artifact"`
// its pipeline's key
Pipeline string `json:"pipeline"`
}
// Shared is a run whose result a step took or waits on, asked once for every step that reads it (design 47 G §729, A-469t): for a composition step one per machine composed, `run` the `composition-runs` key and `ask` its run ask's id; for a test step one with no machine, `run` the `test-runs` key, `ask` its build seat ask's id and `asked-by` the step that asked the run, left out for that step itself.
type Shared struct {
// the id of the run's ask of the build seat
Ask string `json:"ask"`
// the step that asked the run, for a test step reading another's run
AskedBy *SharedAskedBy `json:"asked-by,omitempty"`
// the machine composed, for a composition run
Machine string `json:"machine,omitempty"`
// the run's key: in the assigner's composition-runs, or the runner's test-runs
Run string `json:"run"`
}
// StepEnded is the composition step's end, said by the assigner on the node it runs on (ADR 0666 sections 7, 21): the runner takes it only from that holder's account, naming the ask, the commit and the fingerprint it was asked at.
type StepEnded struct {
// for a rerun, the caller the runtime proved
@@ -1019,9 +961,7 @@ type StepEnded struct {
Pipeline string `json:"pipeline"`
Rerun bool `json:"rerun,omitempty"`
// why it failed or could not run: the two modules owning one path, the part not heard
Said string `json:"said,omitempty"`
// per machine composed, the composition run the step read, its ask and so where its verdict was taken (design 47 G §723, §729)
Shared []Shared `json:"shared,omitempty"`
Said string `json:"said,omitempty"`
Started *time.Time `json:"started,omitempty"`
Step string `json:"step"`
// the layout version
@@ -1211,36 +1151,6 @@ type InputAnswer struct {
Digest string `json:"digest"`
}
// CompositionsInput is what compositions takes.
type CompositionsInput struct {
// the candidate's key
Candidate string `json:"candidate,omitempty"`
Machine string `json:"machine,omitempty"`
}
// CompositionsAnswer is what compositions answers.
type CompositionsAnswer struct {
Fill Fill `json:"fill"`
Runs []CompositionRun `json:"runs"`
}
// CompositionsDropInput is what compositions-drop takes.
type CompositionsDropInput struct {
// the candidate's key
Candidate string `json:"candidate"`
Machine string `json:"machine"`
// why, written with the caller the runtime proves in the hand-act log
Why string `json:"why"`
}
// CompositionsDropAnswer is what compositions-drop answers.
type CompositionsDropAnswer struct {
// the readers it ended
Ended []CompositionReader `json:"ended,omitempty"`
// the entry deleted
Run CompositionRun `json:"run"`
}
// StepAccept is the body of step.
type StepAccept = StepRun
@@ -1324,8 +1234,6 @@ type StepProgressEvent struct {
Asker string `json:"asker"`
At time.Time `json:"at"`
Phase string `json:"phase"`
// the composition runs the step waits on, by machine, each with its ask (design 47 G §723, §729)
Shared []Shared `json:"shared,omitempty"`
}
// The values of StepProgressEvent's phase.
@@ -1447,3 +1355,6 @@ type AssignmentRevisionsByCandidateGroupHeadsValue = AssignmentRevision
// AssignmentRevisionsByEntryDigestValue is one value of the state assignment-revisions under the key entry.<digest>.
type AssignmentRevisionsByEntryDigestValue = AssignmentEntry
// InputsByTimeDigestValue is one value of the state inputs under the key <time>.<digest>.
type InputsByTimeDigestValue = string
@@ -59,10 +59,6 @@ func TestFixtures(t *testing.T) {
decode: func(b []byte) (any, error) { var v WaitHeldEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-wants\\.event\\.wait-held$")},
{name: "an outside wait ended by the operator's Go on", subject: "mesh.seat.mesh-wants.event.wait-ended", given: "{\"fingerprint\":\"src2:aaaa\",\"how\":\"go-on\",\"module\":\"nats\",\"node\":\"anchor\",\"on\":\"silent:home-server\",\"owner\":\"mesh-delivery-assigner\",\"since\":\"2026-11-11T10:00:00Z\",\"step\":\"outside\"}", wire: "{\"fingerprint\":\"src2:aaaa\",\"how\":\"go-on\",\"module\":\"nats\",\"node\":\"anchor\",\"on\":\"silent:home-server\",\"owner\":\"mesh-delivery-assigner\",\"since\":\"2026-11-11T10:00:00Z\",\"step\":\"outside\"}",
decode: func(b []byte) (any, error) { var v WaitEndedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-wants\\.event\\.wait-ended$")},
{name: "C1/AC1 a reading composition step ended, naming per machine the run it read and its ask", subject: "mesh.seat.mesh-wants.event.step-ended.laptop", given: "{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"id\":\"m2kx9a1b3c4d6c\",\"module\":\"nats\",\"node\":\"laptop\",\"outcome\":\"passed\",\"pipeline\":\"main.12.nats\",\"shared\":[{\"ask\":\"r1a2b3\",\"machine\":\"anchor\",\"run\":\"main.12.anchor\"},{\"ask\":\"r4d5e6\",\"machine\":\"home-server\",\"run\":\"main.12.home-server\"}],\"step\":\"composition\",\"v\":1,\"warnings\":[\"anchor holds xclip: xclip and demo both own /etc/x\"]}", wire: "{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"id\":\"m2kx9a1b3c4d6c\",\"module\":\"nats\",\"node\":\"laptop\",\"outcome\":\"passed\",\"pipeline\":\"main.12.nats\",\"shared\":[{\"ask\":\"r1a2b3\",\"machine\":\"anchor\",\"run\":\"main.12.anchor\"},{\"ask\":\"r4d5e6\",\"machine\":\"home-server\",\"run\":\"main.12.home-server\"}],\"step\":\"composition\",\"v\":1,\"warnings\":[\"anchor holds xclip: xclip and demo both own /etc/x\"]}",
decode: func(b []byte) (any, error) { var v StepEndedEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-wants\\.event\\.step-ended\\.[^.*> ]+$")},
{name: "C1/AC1 a composition step's progress, naming the runs it waits on", subject: "mesh.seat.mesh-wants.event.step-progress.anchor", given: "{\"ask\":\"a1b2\",\"asker\":\"assigner\",\"at\":\"2026-11-17T10:01:00Z\",\"phase\":\"run\",\"shared\":[{\"ask\":\"r1a2b3\",\"machine\":\"anchor\",\"run\":\"main.12.anchor\"}]}", wire: "{\"ask\":\"a1b2\",\"asker\":\"assigner\",\"at\":\"2026-11-17T10:01:00Z\",\"phase\":\"run\",\"shared\":[{\"ask\":\"r1a2b3\",\"machine\":\"anchor\",\"run\":\"main.12.anchor\"}]}",
decode: func(b []byte) (any, error) { var v StepProgressEvent; err := json.Unmarshal(b, &v); return v, err }, subjectIs: regexp.MustCompile("^mesh\\.seat\\.mesh-wants\\.event\\.step-progress\\.[^.*> ]+$")},
{name: "a want", subject: "", given: "{\"fp\":\"ab12\",\"rule\":3,\"v\":1}", wire: "{\"fp\":\"ab12\",\"rule\":3,\"v\":1}",
decode: func(b []byte) (any, error) {
var v WantsByWantNodeModuleValue
@@ -145,12 +141,6 @@ func TestFixtures(t *testing.T) {
decode: func(b []byte) (any, error) { var v InputInput; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil},
{name: "input answers how many chunks carry the exact bytes", subject: "", given: "{\"chunks\":3,\"digest\":\"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\"}", wire: "{\"chunks\":3,\"digest\":\"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\"}",
decode: func(b []byte) (any, error) { var v InputAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil},
{name: "C1 compositions answers each run with its readers, verdict and the fill", subject: "", given: "{\"fill\":{\"bytes\":4096,\"cap\":16777216,\"store\":\"composition-runs\"},\"runs\":[{\"ask\":\"r1a2b3\",\"candidate\":\"main.12\",\"ended\":\"2026-11-17T10:03:00Z\",\"input\":\"sha256:0123\",\"key\":\"main.12.anchor\",\"machine\":\"anchor\",\"outcome\":\"passed\",\"program\":{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"directory\":\"cmd/mesh-compose\",\"fingerprint\":\"src2:aaaa\",\"repository\":\"novox/mesh-host\"},\"readers\":[{\"branch\":\"main\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"id\":\"m2kx9a1b3c4d6c\",\"module\":\"nats\",\"pipeline\":\"main.12.nats\",\"revision\":12,\"step\":\"composition\"}],\"started\":\"2026-11-17T10:00:00Z\",\"verdict\":{\"blamed\":[],\"composes\":true}}]}", wire: "{\"fill\":{\"bytes\":4096,\"cap\":16777216,\"store\":\"composition-runs\"},\"runs\":[{\"ask\":\"r1a2b3\",\"candidate\":\"main.12\",\"ended\":\"2026-11-17T10:03:00Z\",\"input\":\"sha256:0123\",\"key\":\"main.12.anchor\",\"machine\":\"anchor\",\"outcome\":\"passed\",\"program\":{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"directory\":\"cmd/mesh-compose\",\"fingerprint\":\"src2:aaaa\",\"repository\":\"novox/mesh-host\"},\"readers\":[{\"branch\":\"main\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"id\":\"m2kx9a1b3c4d6c\",\"module\":\"nats\",\"pipeline\":\"main.12.nats\",\"revision\":12,\"step\":\"composition\"}],\"started\":\"2026-11-17T10:00:00Z\",\"verdict\":{\"blamed\":[],\"composes\":true}}]}",
decode: func(b []byte) (any, error) { var v CompositionsAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil},
{name: "C1 compositions-drop takes a candidate, a machine and why", subject: "", given: "{\"candidate\":\"main.12\",\"machine\":\"anchor\",\"why\":\"its ask never came back\"}", wire: "{\"candidate\":\"main.12\",\"machine\":\"anchor\",\"why\":\"its ask never came back\"}",
decode: func(b []byte) (any, error) { var v CompositionsDropInput; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil},
{name: "C1 compositions-drop answers the run it deleted and the readers it ended", subject: "", given: "{\"ended\":[{\"branch\":\"main\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"id\":\"m2kx9a1b3c4d6c\",\"module\":\"nats\",\"pipeline\":\"main.12.nats\",\"revision\":12,\"step\":\"composition\"}],\"run\":{\"ask\":\"r1a2b3\",\"candidate\":\"main.12\",\"ended\":\"2026-11-17T10:03:00Z\",\"input\":\"sha256:0123\",\"key\":\"main.12.anchor\",\"machine\":\"anchor\",\"outcome\":\"passed\",\"program\":{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"directory\":\"cmd/mesh-compose\",\"fingerprint\":\"src2:aaaa\",\"repository\":\"novox/mesh-host\"},\"readers\":[{\"branch\":\"main\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"id\":\"m2kx9a1b3c4d6c\",\"module\":\"nats\",\"pipeline\":\"main.12.nats\",\"revision\":12,\"step\":\"composition\"}],\"started\":\"2026-11-17T10:00:00Z\",\"verdict\":{\"blamed\":[],\"composes\":true}}}", wire: "{\"ended\":[{\"branch\":\"main\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"id\":\"m2kx9a1b3c4d6c\",\"module\":\"nats\",\"pipeline\":\"main.12.nats\",\"revision\":12,\"step\":\"composition\"}],\"run\":{\"ask\":\"r1a2b3\",\"candidate\":\"main.12\",\"ended\":\"2026-11-17T10:03:00Z\",\"input\":\"sha256:0123\",\"key\":\"main.12.anchor\",\"machine\":\"anchor\",\"outcome\":\"passed\",\"program\":{\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"directory\":\"cmd/mesh-compose\",\"fingerprint\":\"src2:aaaa\",\"repository\":\"novox/mesh-host\"},\"readers\":[{\"branch\":\"main\",\"commit\":\"0123456789abcdef0123456789abcdef01234567\",\"fingerprint\":\"fp-0123\",\"id\":\"m2kx9a1b3c4d6c\",\"module\":\"nats\",\"pipeline\":\"main.12.nats\",\"revision\":12,\"step\":\"composition\"}],\"started\":\"2026-11-17T10:00:00Z\",\"verdict\":{\"blamed\":[],\"composes\":true}}}",
decode: func(b []byte) (any, error) { var v CompositionsDropAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil},
}
for _, c := range cases {
v, err := c.decode([]byte(c.given))
@@ -313,104 +313,5 @@
"how": "answered",
"fingerprint": "fp-0123"
}
},
{
"name": "C1/AC2 a test step started reading a test run, naming its run and ask",
"of": "event:step-started",
"subject": "mesh.seat.mesh-pipeline-runner.event.step-started.nats.main.0123456789abcdef0123456789abcdef01234567",
"given": {
"pipeline": "main.12.nats",
"module": "nats",
"branch": "main",
"revision": 12,
"source": 3589,
"commit": "0123456789abcdef0123456789abcdef01234567",
"v": 1,
"fingerprint": "fp-0123",
"result": {
"step": "unit",
"artifact": "nats:tools",
"status": "required",
"outcome": "running",
"fingerprint": "fp-0123",
"seat": "node-build-go",
"shared": [
{
"run": "main.12.unit.d1",
"ask": "t1a2"
}
]
},
"at": "2026-11-17T10:00:00Z"
}
},
{
"name": "C1/AC2 a test step ended with the run another step asked",
"of": "event:step-ended",
"subject": "mesh.seat.mesh-pipeline-runner.event.step-ended.nats.main.0123456789abcdef0123456789abcdef01234567",
"given": {
"pipeline": "main.12.nats",
"module": "nats",
"branch": "main",
"revision": 12,
"source": 3589,
"commit": "0123456789abcdef0123456789abcdef01234567",
"v": 1,
"fingerprint": "fp-0123",
"result": {
"step": "unit",
"artifact": "nats:tools",
"status": "required",
"outcome": "failed",
"fingerprint": "fp-0123",
"seat": "node-build-go",
"node": "anchor",
"tests": {
"unit": 42
},
"shared": [
{
"run": "main.12.unit.d1",
"ask": "t1a2",
"asked-by": {
"pipeline": "main.12.nats",
"artifact": "nats:program"
}
}
]
},
"at": "2026-11-17T10:05:00Z"
}
},
{
"name": "C1 a composition step ended with mesh-wants' runs as they came",
"of": "event:step-ended",
"subject": "mesh.seat.mesh-pipeline-runner.event.step-ended.nats.main.0123456789abcdef0123456789abcdef01234567",
"given": {
"pipeline": "main.12.nats",
"module": "nats",
"branch": "main",
"revision": 12,
"source": 3589,
"commit": "0123456789abcdef0123456789abcdef01234567",
"v": 1,
"fingerprint": "fp-0123",
"result": {
"step": "composition",
"status": "required",
"outcome": "passed",
"fingerprint": "fp-0123",
"seat": "mesh-wants",
"node": "laptop",
"shared": [
{
"machine": "anchor",
"run": "main.12.anchor",
"ask": "r1a2b3"
}
]
},
"at": "2026-11-17T10:05:00Z"
}
}
]
@@ -143,154 +143,5 @@
"cap": 134217728
}
}
},
{
"name": "C1/AC3 stands answers a waiting test step with its ask and the run it waits on",
"of": "verb:stands.answer",
"given": {
"stands": [
{
"pipeline": "main.12.nats",
"module": "nats",
"branch": "main",
"revision": 12,
"source": 3589,
"commit": "0123456789abcdef0123456789abcdef01234567",
"v": 1,
"fingerprint": "fp-0123",
"step": "unit",
"artifact": "nats:tools",
"on": "build node",
"since": "2026-11-17T10:00:00Z",
"ask": "t1a2",
"shared": [
{
"run": "main.12.unit.d1",
"ask": "t1a2"
}
]
}
]
}
},
{
"name": "C1 test-runs answers each run with its packages, readers and the fill",
"of": "verb:test-runs.answer",
"given": {
"runs": [
{
"key": "main.12.unit.d1",
"level": "unit",
"digest": "d1",
"seat": "node-build-go",
"ask": "t1a2",
"node": "anchor",
"started": "2026-11-17T10:00:00Z",
"readers": [
{
"pipeline": "main.12.nats",
"artifact": "nats:program"
},
{
"pipeline": "main.12.nats",
"artifact": "nats:tools"
}
],
"ended": "2026-11-17T10:05:00Z",
"outcome": "failed",
"said": "1 test failed",
"tests": {
"unit": 42
},
"output": "t1a2",
"packages": [
{
"package": "git.novox.be/novox/nats/internal/x",
"digest": "sha256:aa"
}
]
},
{
"key": "main.12.unit.d1",
"level": "unit",
"digest": "d1",
"seat": "node-build-go",
"ask": "t1a2",
"node": "anchor",
"started": "2026-11-17T10:00:00Z",
"readers": [
{
"pipeline": "main.12.nats",
"artifact": "nats:program"
},
{
"pipeline": "main.12.nats",
"artifact": "nats:tools"
}
],
"ended": "2026-11-17T10:05:00Z",
"outcome": "failed",
"said": "1 test failed",
"tests": {
"unit": 42
},
"output": "t1a2",
"gone": "the run's output is gone"
}
],
"fill": {
"store": "test-runs",
"bytes": 2048,
"cap": 16777216
}
}
},
{
"name": "C1 test-runs-drop answers the run it deleted and the readers it ended",
"of": "verb:test-runs-drop.answer",
"given": {
"run": {
"key": "main.12.unit.d1",
"level": "unit",
"digest": "d1",
"seat": "node-build-go",
"ask": "t1a2",
"node": "anchor",
"started": "2026-11-17T10:00:00Z",
"readers": [
{
"pipeline": "main.12.nats",
"artifact": "nats:program"
},
{
"pipeline": "main.12.nats",
"artifact": "nats:tools"
}
],
"ended": "2026-11-17T10:05:00Z",
"outcome": "failed",
"said": "1 test failed",
"tests": {
"unit": 42
},
"output": "t1a2",
"packages": [
{
"package": "git.novox.be/novox/nats/internal/x",
"digest": "sha256:aa"
}
]
},
"ended": [
{
"pipeline": "main.12.nats",
"artifact": "nats:program"
},
{
"pipeline": "main.12.nats",
"artifact": "nats:tools"
}
]
}
}
]
@@ -495,93 +495,6 @@
]
},
"class": "act"
},
{
"name": "test-runs",
"class": "read",
"description": "Each test run the runner keeps (design 47 G §728): its packages read from its output object, ask, node, readers, outcome and counts, and the state's fill against its cap; with candidate, level, digest or module, only those.",
"input": {
"type": "object",
"properties": {
"candidate": {
"type": "string",
"description": "the candidate's key"
},
"level": {
"type": "string",
"enum": [
"lint",
"unit",
"integration"
]
},
"digest": {
"type": "string"
},
"module": {
"type": "string"
}
}
},
"answer": {
"type": "object",
"properties": {
"runs": {
"type": "array",
"items": {
"$ref": "#/defs/TestRun"
}
},
"fill": {
"$ref": "#/defs/Fill"
}
},
"required": [
"runs",
"fill"
]
}
},
{
"name": "test-runs-drop",
"class": "act",
"description": "Drop one test run by hand (design 47 G §728): the act and why are written in the hand-act log first, then the run's ask is dropped, each reader ended could not run naming the drop, and the key deleted with the run's output object.",
"input": {
"type": "object",
"properties": {
"key": {
"type": "string",
"description": "the run's key in test-runs"
},
"why": {
"type": "string",
"description": "why, written with the caller the runtime proves in the hand-act log"
}
},
"required": [
"key",
"why"
]
},
"answer": {
"type": "object",
"properties": {
"run": {
"$ref": "#/defs/TestRun",
"description": "the entry deleted"
},
"ended": {
"type": "array",
"items": {
"$ref": "#/defs/TestRunReader"
},
"description": "the readers it ended"
}
},
"required": [
"run"
]
}
}
],
"events": [
@@ -974,13 +887,6 @@
"ended": {
"type": "string",
"format": "date-time"
},
"shared": {
"type": "array",
"items": {
"$ref": "#/defs/Shared"
},
"description": "the run the step read or asked for every step that reads it: a composition step's per machine, as mesh-wants' step-ended names them; a test step's one, with asked-by when another step asked it (design 47 G §726, §729; A-468d, A-468e, A-469t)"
}
},
"description": "one step of a pipeline: its status in the policy and its outcome (design 47 *Pipeline steps and the policy*)",
@@ -1588,17 +1494,6 @@
"how": {
"type": "string",
"description": "for its end, how it ended: answered, ran, superseded"
},
"ask": {
"type": "string",
"description": "the step's own ask, where it has one (design 47 G §731)"
},
"shared": {
"type": "array",
"items": {
"$ref": "#/defs/Shared"
},
"description": "the run the step waits on, where the runner knows it: a test step's run key and ask (design 47 G §731, A-469v)"
}
},
"description": "a step's wait, held and ended; a wait on a person is said as waiting on that person, never a stall (ADR 0283)",
@@ -1682,159 +1577,6 @@
"what",
"at"
]
},
"Shared": {
"type": "object",
"description": "a run whose result a step took or waits on, asked once for every step that reads it (design 47 G §729, A-469t): for a composition step one per machine composed, `run` the `composition-runs` key and `ask` its run ask's id; for a test step one with no machine, `run` the `test-runs` key, `ask` its build seat ask's id and `asked-by` the step that asked the run, left out for that step itself",
"properties": {
"machine": {
"type": "string",
"description": "the machine composed, for a composition run"
},
"run": {
"type": "string",
"description": "the run's key: in the assigner's composition-runs, or the runner's test-runs"
},
"ask": {
"type": "string",
"description": "the id of the run's ask of the build seat"
},
"asked-by": {
"type": "object",
"description": "the step that asked the run, for a test step reading another's run",
"properties": {
"pipeline": {
"type": "string",
"description": "its pipeline's key"
},
"artifact": {
"type": "string",
"description": "its artifact, <module>:<artifact>"
}
},
"required": [
"pipeline",
"artifact"
]
}
},
"required": [
"run",
"ask"
]
},
"TestRunReader": {
"type": "object",
"description": "a test step reading a test run (design 47 G §725)",
"properties": {
"pipeline": {
"type": "string",
"description": "its pipeline's key"
},
"artifact": {
"type": "string",
"description": "its artifact, <module>:<artifact>"
}
},
"required": [
"pipeline",
"artifact"
]
},
"TestRunPackage": {
"type": "object",
"description": "one package a test run ran, with its digest (design 47 G §724)",
"properties": {
"package": {
"type": "string"
},
"digest": {
"type": "string"
}
},
"required": [
"package",
"digest"
]
},
"TestRun": {
"type": "object",
"description": "one test run of a candidate, level and tested packages, a key of the runner's state test-runs (design 47 G §725, §728; A-469u)",
"properties": {
"key": {
"type": "string",
"description": "the key, <candidate key>.<level>.<digest>, each part escaped as a key's name"
},
"level": {
"type": "string",
"enum": [
"lint",
"unit",
"integration"
]
},
"digest": {
"type": "string",
"description": "the share digest (design 47 G §724)"
},
"seat": {
"type": "string",
"description": "the build seat asked"
},
"ask": {
"type": "string",
"description": "the id of the build seat's ask"
},
"node": {
"type": "string",
"description": "the node whose holder took the ask"
},
"started": {
"type": "string",
"format": "date-time"
},
"readers": {
"type": "array",
"items": {
"$ref": "#/defs/TestRunReader"
},
"description": "the steps reading the run"
},
"ended": {
"type": "string",
"format": "date-time"
},
"outcome": {
"type": "string"
},
"said": {
"type": "string",
"description": "the outcome's words"
},
"tests": {
"$ref": "#/defs/TestCounts"
},
"output": {
"type": "string",
"description": "the run's output object, by its name in test-runs-objects"
},
"packages": {
"type": "array",
"items": {
"$ref": "#/defs/TestRunPackage"
},
"description": "the packages it ran, read from its output object"
},
"gone": {
"type": "string",
"description": "why the packages are not named: the run's output is gone"
}
},
"required": [
"key",
"level",
"digest"
]
}
}
}
@@ -284,54 +284,5 @@
"since": "2026-11-11T10:00:00Z",
"how": "go-on"
}
},
{
"name": "C1/AC1 a reading composition step ended, naming per machine the run it read and its ask",
"of": "event:step-ended",
"subject": "mesh.seat.mesh-wants.event.step-ended.laptop",
"given": {
"v": 1,
"id": "m2kx9a1b3c4d6c",
"pipeline": "main.12.nats",
"step": "composition",
"module": "nats",
"commit": "0123456789abcdef0123456789abcdef01234567",
"fingerprint": "fp-0123",
"node": "laptop",
"outcome": "passed",
"warnings": [
"anchor holds xclip: xclip and demo both own /etc/x"
],
"shared": [
{
"machine": "anchor",
"run": "main.12.anchor",
"ask": "r1a2b3"
},
{
"machine": "home-server",
"run": "main.12.home-server",
"ask": "r4d5e6"
}
]
}
},
{
"name": "C1/AC1 a composition step's progress, naming the runs it waits on",
"of": "event:step-progress",
"subject": "mesh.seat.mesh-wants.event.step-progress.anchor",
"given": {
"asker": "assigner",
"ask": "a1b2",
"phase": "run",
"at": "2026-11-17T10:01:00Z",
"shared": [
{
"machine": "anchor",
"run": "main.12.anchor",
"ask": "r1a2b3"
}
]
}
}
]
@@ -175,109 +175,5 @@
"chunks": 3,
"digest": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
}
},
{
"name": "C1 compositions answers each run with its readers, verdict and the fill",
"of": "verb:compositions.answer",
"given": {
"runs": [
{
"key": "main.12.anchor",
"candidate": "main.12",
"machine": "anchor",
"program": {
"repository": "novox/mesh-host",
"commit": "0123456789abcdef0123456789abcdef01234567",
"directory": "cmd/mesh-compose",
"fingerprint": "src2:aaaa"
},
"input": "sha256:0123",
"ask": "r1a2b3",
"started": "2026-11-17T10:00:00Z",
"readers": [
{
"id": "m2kx9a1b3c4d6c",
"pipeline": "main.12.nats",
"step": "composition",
"module": "nats",
"branch": "main",
"commit": "0123456789abcdef0123456789abcdef01234567",
"revision": 12,
"fingerprint": "fp-0123"
}
],
"ended": "2026-11-17T10:03:00Z",
"outcome": "passed",
"verdict": {
"composes": true,
"blamed": []
}
}
],
"fill": {
"store": "composition-runs",
"bytes": 4096,
"cap": 16777216
}
}
},
{
"name": "C1 compositions-drop takes a candidate, a machine and why",
"of": "verb:compositions-drop.input",
"given": {
"candidate": "main.12",
"machine": "anchor",
"why": "its ask never came back"
}
},
{
"name": "C1 compositions-drop answers the run it deleted and the readers it ended",
"of": "verb:compositions-drop.answer",
"given": {
"run": {
"key": "main.12.anchor",
"candidate": "main.12",
"machine": "anchor",
"program": {
"repository": "novox/mesh-host",
"commit": "0123456789abcdef0123456789abcdef01234567",
"directory": "cmd/mesh-compose",
"fingerprint": "src2:aaaa"
},
"input": "sha256:0123",
"ask": "r1a2b3",
"started": "2026-11-17T10:00:00Z",
"readers": [
{
"id": "m2kx9a1b3c4d6c",
"pipeline": "main.12.nats",
"step": "composition",
"module": "nats",
"branch": "main",
"commit": "0123456789abcdef0123456789abcdef01234567",
"revision": 12,
"fingerprint": "fp-0123"
}
],
"ended": "2026-11-17T10:03:00Z",
"outcome": "passed",
"verdict": {
"composes": true,
"blamed": []
}
},
"ended": [
{
"id": "m2kx9a1b3c4d6c",
"pipeline": "main.12.nats",
"step": "composition",
"module": "nats",
"branch": "main",
"commit": "0123456789abcdef0123456789abcdef01234567",
"revision": 12,
"fingerprint": "fp-0123"
}
]
}
}
]
@@ -202,86 +202,6 @@
]
},
"chunks": true
},
{
"name": "compositions",
"class": "read",
"description": "Each composition run the assigner keeps (design 47 G \u00a7728): its program, input digest, ask, readers, outcome and verdict, and the state's fill against its cap; with candidate or machine, only those.",
"input": {
"type": "object",
"properties": {
"candidate": {
"type": "string",
"description": "the candidate's key"
},
"machine": {
"type": "string"
}
}
},
"answer": {
"type": "object",
"properties": {
"runs": {
"type": "array",
"items": {
"$ref": "#/defs/CompositionRun"
}
},
"fill": {
"$ref": "#/defs/Fill"
}
},
"required": [
"runs",
"fill"
]
}
},
{
"name": "compositions-drop",
"class": "act",
"description": "Drop one composition run by hand (design 47 G \u00a7728): the act and why are written in the hand-act log first, then the run's ask is dropped, each reader ended could not run naming the drop, and the key deleted.",
"input": {
"type": "object",
"properties": {
"candidate": {
"type": "string",
"description": "the candidate's key"
},
"machine": {
"type": "string"
},
"why": {
"type": "string",
"description": "why, written with the caller the runtime proves in the hand-act log"
}
},
"required": [
"candidate",
"machine",
"why"
]
},
"answer": {
"type": "object",
"properties": {
"run": {
"$ref": "#/defs/CompositionRun",
"description": "the entry deleted"
},
"ended": {
"type": "array",
"items": {
"$ref": "#/defs/CompositionReader"
},
"description": "the readers it ended"
}
},
"required": [
"run"
]
}
}
],
"accepts": [
@@ -550,13 +470,6 @@
"at": {
"type": "string",
"format": "date-time"
},
"shared": {
"type": "array",
"items": {
"$ref": "#/defs/Shared"
},
"description": "the composition runs the step waits on, by machine, each with its ask (design 47 G \u00a7723, \u00a7729)"
}
},
"required": [
@@ -895,6 +808,22 @@
"description": "an entry, written once, removed when no kept revision lists it"
}
]
},
{
"name": "inputs",
"template": "state",
"history": 1,
"description": "a step's input, named by its write time and its sha256; a value over one chunk is held in the state's objects store (design 47 G sections 354, 712)",
"key-forms": [
{
"key": "<time>.<digest>",
"value": {
"type": "string",
"description": "the input's bytes, base64"
},
"description": "one input"
}
]
}
],
"defs": {
@@ -1505,13 +1434,6 @@
"ended": {
"type": "string",
"format": "date-time"
},
"shared": {
"type": "array",
"items": {
"$ref": "#/defs/Shared"
},
"description": "per machine composed, the composition run the step read, its ask and so where its verdict was taken (design 47 G \u00a7723, \u00a7729)"
}
},
"required": [
@@ -1566,195 +1488,6 @@
"required": [
"ask"
]
},
"Shared": {
"type": "object",
"description": "a run whose result a step took or waits on, asked once for every step that reads it (design 47 G \u00a7729, A-469t): for a composition step one per machine composed, `run` the `composition-runs` key and `ask` its run ask's id; for a test step one with no machine, `run` the `test-runs` key, `ask` its build seat ask's id and `asked-by` the step that asked the run, left out for that step itself",
"properties": {
"machine": {
"type": "string",
"description": "the machine composed, for a composition run"
},
"run": {
"type": "string",
"description": "the run's key: in the assigner's composition-runs, or the runner's test-runs"
},
"ask": {
"type": "string",
"description": "the id of the run's ask of the build seat"
},
"asked-by": {
"type": "object",
"description": "the step that asked the run, for a test step reading another's run",
"properties": {
"pipeline": {
"type": "string",
"description": "its pipeline's key"
},
"artifact": {
"type": "string",
"description": "its artifact, <module>:<artifact>"
}
},
"required": [
"pipeline",
"artifact"
]
}
},
"required": [
"run",
"ask"
]
},
"CompositionReader": {
"type": "object",
"description": "a composition step reading a composition run: its step, as the runner asked it, with its ask's id (design 47 G \u00a7719, A-468)",
"properties": {
"id": {
"type": "string",
"description": "the step's ask's id"
},
"pipeline": {
"type": "string",
"description": "the runner's pipeline key, <branch>.<revision>.<module>, a branch escaped as a key's name is"
},
"step": {
"type": "string",
"enum": [
"composition"
]
},
"module": {
"type": "string"
},
"branch": {
"type": "string",
"description": "the watched branch, or the branch of a run on request"
},
"commit": {
"type": "string",
"description": "the commit the step runs at"
},
"revision": {
"type": "integer",
"description": "the catalog graph revision of the branch the pipeline runs for"
},
"source": {
"type": "integer",
"description": "the source revision of the watched branch; absent for a run on request"
},
"fingerprint": {
"type": "string",
"description": "the module's build-input fingerprint at the commit"
},
"run": {
"type": "string",
"description": "the run on request's id: its composition writes no want (ADR 0666 section 18)"
},
"rerun": {
"type": "boolean",
"description": "the step is asked again by the runner's rerun"
},
"by": {
"type": "string",
"description": "for a rerun, the caller the runtime proved"
},
"ended": {
"type": "string",
"format": "date-time",
"description": "when the reader's step said its end; carried into a rerun's run, so a take-up tells a reader that passed from one that joined later (A-470m)"
}
},
"required": [
"id",
"pipeline",
"step",
"module",
"branch",
"commit",
"fingerprint"
]
},
"CompositionRun": {
"type": "object",
"description": "one composition run of a candidate and a machine, a key of the assigner's state composition-runs (design 47 G \u00a7719, \u00a7728; A-469u)",
"properties": {
"key": {
"type": "string",
"description": "the key, <candidate key>.<machine>, each part escaped as a key's name"
},
"candidate": {
"type": "string",
"description": "the candidate's key: <branch>.<revision>, or run-<id> for a run on request"
},
"machine": {
"type": "string"
},
"program": {
"type": "object",
"description": "the composer program, chosen once when the run started",
"properties": {
"repository": {
"type": "string"
},
"commit": {
"type": "string"
},
"directory": {
"type": "string"
},
"fingerprint": {
"type": "string",
"description": "the node-engine fingerprint it was chosen from"
}
}
},
"input": {
"type": "string",
"description": "the digest of the kept request"
},
"ask": {
"type": "string",
"description": "the id of the build seat's run ask"
},
"started": {
"type": "string",
"format": "date-time"
},
"readers": {
"type": "array",
"items": {
"$ref": "#/defs/CompositionReader"
},
"description": "the composition steps reading the run"
},
"ended": {
"type": "string",
"format": "date-time"
},
"outcome": {
"type": "string",
"enum": [
"passed",
"failed",
"could not run"
]
},
"said": {
"type": "string",
"description": "the outcome's words"
},
"verdict": {
"type": "object",
"description": "that machine's composer verdict (design 47 G \u00a7721)"
}
},
"required": [
"key",
"candidate",
"machine"
]
}
},
"acts": [
@@ -17,8 +17,6 @@ export declare const verbs: {
readonly accept: "accept";
readonly cancel: "cancel";
readonly askAgain: "ask-again";
readonly testRuns: "test-runs";
readonly testRunsDrop: "test-runs-drop";
};
export declare const events: {
@@ -318,26 +316,6 @@ export interface RunStarted {
"v": number;
}
/** the step that asked the run, for a test step reading another's run */
export interface SharedAskedBy {
/** its artifact, <module>:<artifact> */
"artifact": string;
/** its pipeline's key */
"pipeline": string;
}
/** a run whose result a step took or waits on, asked once for every step that reads it (design 47 G §729, A-469t): for a composition step one per machine composed, `run` the `composition-runs` key and `ask` its run ask's id; for a test step one with no machine, `run` the `test-runs` key, `ask` its build seat ask's id and `asked-by` the step that asked the run, left out for that step itself */
export interface Shared {
/** the id of the run's ask of the build seat */
"ask": string;
/** the step that asked the run, for a test step reading another's run */
"asked-by"?: SharedAskedBy;
/** the machine composed, for a composition run */
"machine"?: string;
/** the run's key: in the assigner's composition-runs, or the runner's test-runs */
"run": string;
}
/** a step's start or end: the step, its status, its outcome, its fingerprint, its test counts, what a carried result was carried from, and for a rerun that it is one and its proved caller; a bypassed step too */
export interface StepEvent {
"at"?: string;
@@ -388,8 +366,6 @@ export interface StepResult {
"rerun"?: boolean;
/** the seat that serves the step, whose holder alone may end it (ADR 0666 section 21) */
"seat"?: string;
/** the run the step read or asked for every step that reads it: a composition step's per machine, as mesh-wants' step-ended names them; a test step's one, with asked-by when another step asked it (design 47 G §726, §729; A-468d, A-468e, A-469t) */
"shared"?: Shared[];
"started"?: string;
/** the step's status in the module's policy */
"status": "required" | "optional" | "bypass";
@@ -405,49 +381,6 @@ export interface TestCounts {
"unit"?: number;
}
/** one test run of a candidate, level and tested packages, a key of the runner's state test-runs (design 47 G §725, §728; A-469u) */
export interface TestRun {
/** the id of the build seat's ask */
"ask"?: string;
/** the share digest (design 47 G §724) */
"digest": string;
"ended"?: string;
/** why the packages are not named: the run's output is gone */
"gone"?: string;
/** the key, <candidate key>.<level>.<digest>, each part escaped as a key's name */
"key": string;
"level": "lint" | "unit" | "integration";
/** the node whose holder took the ask */
"node"?: string;
"outcome"?: string;
/** the run's output object, by its name in test-runs-objects */
"output"?: string;
/** the packages it ran, read from its output object */
"packages"?: TestRunPackage[];
/** the steps reading the run */
"readers"?: TestRunReader[];
/** the outcome's words */
"said"?: string;
/** the build seat asked */
"seat"?: string;
"started"?: string;
"tests"?: TestCounts;
}
/** one package a test run ran, with its digest (design 47 G §724) */
export interface TestRunPackage {
"digest": string;
"package": string;
}
/** a test step reading a test run (design 47 G §725) */
export interface TestRunReader {
/** its artifact, <module>:<artifact> */
"artifact": string;
/** its pipeline's key */
"pipeline": string;
}
/** the toolchain a build was made with: the language, the installed version and the build module's recipe */
export interface ToolchainUsed {
"language": string;
@@ -458,8 +391,6 @@ export interface ToolchainUsed {
/** a step's wait, held and ended; a wait on a person is said as waiting on that person, never a stall (ADR 0283) */
export interface Wait {
"artifact"?: string;
/** the step's own ask, where it has one (design 47 G §731) */
"ask"?: string;
/** the watched branch, or the branch of a run on request */
"branch": string;
/** the commit the module's entry was last changed at */
@@ -479,8 +410,6 @@ export interface Wait {
"revision"?: number;
/** the run on request's id, for a pipeline of a run; such a pipeline delivers nothing */
"run"?: string;
/** the run the step waits on, where the runner knows it: a test step's run key and ask (design 47 G §731, A-469v) */
"shared"?: Shared[];
"since": string;
/** the source revision of the watched branch; the pipeline is keyed by it (ADR 0666 section 9) */
"source"?: number;
@@ -665,37 +594,6 @@ export interface AskAgainAnswer {
"result": StepResult;
}
/** is what test-runs takes. */
export interface TestRunsInput {
/** the candidate's key */
"candidate"?: string;
"digest"?: string;
"level"?: "lint" | "unit" | "integration";
"module"?: string;
}
/** is what test-runs answers. */
export interface TestRunsAnswer {
"fill": Fill;
"runs": TestRun[];
}
/** is what test-runs-drop takes. */
export interface TestRunsDropInput {
/** the run's key in test-runs */
"key": string;
/** why, written with the caller the runtime proves in the hand-act log */
"why": string;
}
/** is what test-runs-drop answers. */
export interface TestRunsDropAnswer {
/** the readers it ended */
"ended"?: TestRunReader[];
/** the entry deleted */
"run": TestRun;
}
/** is the body of the event pipeline-started. */
export type PipelineStartedEvent = PipelineStarted;
@@ -17,8 +17,6 @@ export const verbs = {
accept: "accept",
cancel: "cancel",
askAgain: "ask-again",
testRuns: "test-runs",
testRunsDrop: "test-runs-drop",
};
export const events = {
+10 -117
View File
@@ -11,8 +11,6 @@ export declare const verbs: {
readonly entry: "entry";
readonly difference: "difference";
readonly input: "input";
readonly compositions: "compositions";
readonly compositionsDrop: "compositions-drop";
};
export declare const accepts: {
@@ -80,6 +78,7 @@ export declare const states: {
readonly differences: "differences";
readonly compositions: "compositions";
readonly assignmentRevisions: "assignment-revisions";
readonly inputs: "inputs";
};
/** The key of one value of the state wants under the form "want.<node>.<module>". */
@@ -172,6 +171,12 @@ export declare function assignmentRevisionsByEntryDigestKey(digest: string): str
/** The parts of a key of the form "entry.<digest>", or undefined for a key of another form. */
export declare function parseAssignmentRevisionsByEntryDigestKey(_key: string): { digest: string } | undefined;
/** The key of one value of the state inputs under the form "<time>.<digest>". */
export declare function inputsByTimeDigestKey(time: string, digest: string): string;
/** The parts of a key of the form "<time>.<digest>", or undefined for a key of another form. */
export declare function parseInputsByTimeDigestKey(_key: string): { time: string; digest: string } | undefined;
/** an ask the assigner puts to a person: the router's ask whole, as the assigner composes it, and how many times it was written at its key */
export interface Ask {
/** the seat operator-channel's ask, whole */
@@ -260,67 +265,6 @@ export interface Change {
"to"?: string;
}
/** a composition step reading a composition run: its step, as the runner asked it, with its ask's id (design 47 G §719, A-468) */
export interface CompositionReader {
/** the watched branch, or the branch of a run on request */
"branch": string;
/** for a rerun, the caller the runtime proved */
"by"?: string;
/** the commit the step runs at */
"commit": string;
/** when the reader's step said its end; carried into a rerun's run, so a take-up tells a reader that passed from one that joined later (A-470m) */
"ended"?: string;
/** the module's build-input fingerprint at the commit */
"fingerprint": string;
/** the step's ask's id */
"id": string;
"module": string;
/** the runner's pipeline key, <branch>.<revision>.<module>, a branch escaped as a key's name is */
"pipeline": string;
/** the step is asked again by the runner's rerun */
"rerun"?: boolean;
/** the catalog graph revision of the branch the pipeline runs for */
"revision"?: number;
/** the run on request's id: its composition writes no want (ADR 0666 section 18) */
"run"?: string;
/** the source revision of the watched branch; absent for a run on request */
"source"?: number;
"step": "composition";
}
/** the composer program, chosen once when the run started */
export interface CompositionRunProgram {
"commit"?: string;
"directory"?: string;
/** the node-engine fingerprint it was chosen from */
"fingerprint"?: string;
"repository"?: string;
}
/** one composition run of a candidate and a machine, a key of the assigner's state composition-runs (design 47 G §719, §728; A-469u) */
export interface CompositionRun {
/** the id of the build seat's run ask */
"ask"?: string;
/** the candidate's key: <branch>.<revision>, or run-<id> for a run on request */
"candidate": string;
"ended"?: string;
/** the digest of the kept request */
"input"?: string;
/** the key, <candidate key>.<machine>, each part escaped as a key's name */
"key": string;
"machine": string;
"outcome"?: "passed" | "failed" | "could not run";
/** the composer program, chosen once when the run started */
"program"?: CompositionRunProgram;
/** the composition steps reading the run */
"readers"?: CompositionReader[];
/** the outcome's words */
"said"?: string;
"started"?: string;
/** that machine's composer verdict (design 47 G §721) */
"verdict"?: Record<string, unknown>;
}
export interface DifferenceRefusedItem {
"module": string;
"why": string;
@@ -396,26 +340,6 @@ export interface NodeEntryRef {
"node": string;
}
/** the step that asked the run, for a test step reading another's run */
export interface SharedAskedBy {
/** its artifact, <module>:<artifact> */
"artifact": string;
/** its pipeline's key */
"pipeline": string;
}
/** a run whose result a step took or waits on, asked once for every step that reads it (design 47 G §729, A-469t): for a composition step one per machine composed, `run` the `composition-runs` key and `ask` its run ask's id; for a test step one with no machine, `run` the `test-runs` key, `ask` its build seat ask's id and `asked-by` the step that asked the run, left out for that step itself */
export interface Shared {
/** the id of the run's ask of the build seat */
"ask": string;
/** the step that asked the run, for a test step reading another's run */
"asked-by"?: SharedAskedBy;
/** the machine composed, for a composition run */
"machine"?: string;
/** the run's key: in the assigner's composition-runs, or the runner's test-runs */
"run": string;
}
/** the composition step's end, said by the assigner on the node it runs on (ADR 0666 sections 7, 21): the runner takes it only from that holder's account, naming the ask, the commit and the fingerprint it was asked at */
export interface StepEnded {
/** for a rerun, the caller the runtime proved */
@@ -434,8 +358,6 @@ export interface StepEnded {
"rerun"?: boolean;
/** why it failed or could not run: the two modules owning one path, the part not heard */
"said"?: string;
/** per machine composed, the composition run the step read, its ask and so where its verdict was taken (design 47 G §723, §729) */
"shared"?: Shared[];
"started"?: string;
"step": "composition";
/** the layout version */
@@ -584,36 +506,6 @@ export interface InputAnswer {
"digest": string;
}
/** is what compositions takes. */
export interface CompositionsInput {
/** the candidate's key */
"candidate"?: string;
"machine"?: string;
}
/** is what compositions answers. */
export interface CompositionsAnswer {
"fill": Fill;
"runs": CompositionRun[];
}
/** is what compositions-drop takes. */
export interface CompositionsDropInput {
/** the candidate's key */
"candidate": string;
"machine": string;
/** why, written with the caller the runtime proves in the hand-act log */
"why": string;
}
/** is what compositions-drop answers. */
export interface CompositionsDropAnswer {
/** the readers it ended */
"ended"?: CompositionReader[];
/** the entry deleted */
"run": CompositionRun;
}
/** is the body of step. */
export type StepAccept = StepRun;
@@ -697,8 +589,6 @@ export interface StepProgressEvent {
"asker": string;
"at": string;
"phase": "fetch" | "build" | "run" | "report";
/** the composition runs the step waits on, by machine, each with its ask (design 47 G §723, §729) */
"shared"?: Shared[];
}
/** is one value of the state wants under the key want.<node>.<module>. */
@@ -802,3 +692,6 @@ export type AssignmentRevisionsByCandidateGroupHeadsValue = AssignmentRevision;
/** is one value of the state assignment-revisions under the key entry.<digest>. */
export type AssignmentRevisionsByEntryDigestValue = AssignmentEntry;
/** is one value of the state inputs under the key <time>.<digest>. */
export type InputsByTimeDigestValue = string;
@@ -11,8 +11,6 @@ export const verbs = {
entry: "entry",
difference: "difference",
input: "input",
compositions: "compositions",
compositionsDrop: "compositions-drop",
};
export const accepts = {
@@ -86,6 +84,7 @@ export const states = {
differences: "differences",
compositions: "compositions",
assignmentRevisions: "assignment-revisions",
inputs: "inputs",
};
/** The key of one value of the state wants under the form "want.<node>.<module>". */
@@ -404,3 +403,22 @@ export function parseAssignmentRevisionsByEntryDigestKey(_key) {
return { digest };
}
/** The key of one value of the state inputs under the form "<time>.<digest>". */
export function inputsByTimeDigestKey(time, digest) {
return time + "." + digest;
}
/** The parts of a key of the form "<time>.<digest>", or undefined for a key of another form. */
export function parseInputsByTimeDigestKey(_key) {
let _rest = _key;
const _at0 = _rest.indexOf(".");
if (_at0 <= 0) return undefined;
const time = _rest.slice(0, _at0);
_rest = _rest.slice(_at0 + 1);
if (_rest === "") return undefined;
const digest = _rest;
_rest = "";
if (_rest !== "") return undefined;
return { time, digest };
}
@@ -1,6 +1,8 @@
package main
import (
"fmt"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
stdio "git.novox.be/novox/mesh-sdk/go"
@@ -13,43 +15,59 @@ import (
// catalogRead reads the indexer's view of the trunk (its modules and module graph, from the newest catalog graph
// revision) and its manifests state for the seat's verbs.
func catalogRead(modules views.Source, manifest func(digest string) (string, error), nodes views.Source, seatDefinitions, marks views.Source) index.CatalogRead {
func catalogRead(modules views.Source, manifests stdio.KeptState, nodes views.Source, seatDefinitions, marks views.Source) index.CatalogRead {
return index.CatalogRead{
SeatDefinitions: seatDefinitions, Marks: marks,
Modules: func() (map[string]meshcatalog.ModulesByModuleValue, error) {
values, err := index.SourceValues(modules)
keys, err := modules.Keys()
if err != nil {
return nil, err
}
out := map[string]meshcatalog.ModulesByModuleValue{}
for k, v := range values {
if m, err := meshcatalogclient.DecodeModule(v); err == nil {
for _, k := range keys {
e, ok, err := modules.Get(k)
if err != nil {
return nil, fmt.Errorf("modules.%s: %w", k, err)
}
if !ok {
continue
}
if m, err := meshcatalogclient.DecodeModule(e.Value); err == nil {
out[k] = m
}
}
return out, nil
},
Module: func(name string) (meshcatalog.ModulesByModuleValue, bool, error) {
key, err := meshcatalogclient.ModulesKey(name)
Manifest: func(digest string) (string, error) {
key, err := meshcatalogclient.ManifestsKey(digest)
if err != nil {
return meshcatalog.ModulesByModuleValue{}, false, nil // a name no key holds is a module the trunk holds none of
return "", err
}
e, ok, err := modules.Get(key)
if err != nil || !ok {
return meshcatalog.ModulesByModuleValue{}, false, err
e, err := manifests.Get(key)
if err != nil {
return "", err
}
m, err := meshcatalogclient.DecodeModule(e.Value)
return m, err == nil, nil
if e == nil {
return "", fmt.Errorf("the manifests state holds no %s", digest)
}
kept, err := meshcatalogclient.DecodeManifest(e.Value)
return kept.Manifest, err
},
Manifest: manifest,
Graph: func() (graph.Graph, error) {
values, err := index.SourceValues(nodes)
keys, err := nodes.Keys()
if err != nil {
return nil, err
}
g := graph.Graph{}
for k, v := range values {
if n, err := graph.Decode(v); err == nil {
for _, k := range keys {
e, ok, err := nodes.Get(k)
if err != nil {
return nil, fmt.Errorf("graph.%s: %w", k, err)
}
if !ok {
continue
}
if n, err := graph.Decode(e.Value); err == nil {
g[k] = n
}
}
@@ -59,27 +77,7 @@ func catalogRead(modules views.Source, manifest func(digest string) (string, err
}
// catalogTools are the seat's verbs over the indexer's own states, served beside the indexer's own tools, since the indexer's manifest claims the seat (ADR 0604 §2).
// Every state a verb reads whole is answered from memory (the view of the trunk held, or a watched copy), so a call costs no
// round trip per key: four node-engines ask every module at once.
func catalogTools(state func(string) views.Source, kept *index.KeptTrunk, watched *stdio.WatchedStates) []stdio.Tool {
return index.CatalogTools(catalogRead(state("modules"), kept.Manifest, state(graph.State),
watchedSource{watched, index.SeatDefinitionsState}, keptSource(views.MarksState)))
func catalogTools(state func(string) views.Source) []stdio.Tool {
return index.CatalogTools(catalogRead(state("modules"), stdio.State("manifests"), state(graph.State),
keptSource(index.SeatDefinitionsState), keptSource(views.MarksState)))
}
// watchedSource is a state as its watched copy holds it (stdio.WatchedStates), as a Source; it offers every value in one read.
type watchedSource struct {
w *stdio.WatchedStates
state string
}
func (s watchedSource) Keys() ([]string, error) { return s.w.Keys(s.state) }
func (s watchedSource) Get(key string) (*views.Entry, bool, error) {
v, ok, err := s.w.Get(s.state, key)
if err != nil || !ok {
return nil, false, err
}
return &views.Entry{Value: v}, true, nil
}
func (s watchedSource) Values() (map[string][]byte, error) { return s.w.Values(s.state) }
@@ -17,11 +17,9 @@ import (
"time"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/busstate"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/views"
"mesh-delivery-indexer/internal/index"
)
@@ -288,30 +286,13 @@ func main() {
tools := Tools(asks, time.Now, stdio.State(index.VerdictsState), heldCounts(stdio.State(index.OldSectionsState)), acts)
// The modules and the module graph are the newest catalog graph revision's (milestone 47/G): revision.go.
revisions := bundleObjects{busstate.ObjectsOf(index.CatalogRevisionsObjects)}
// The view of the trunk is read from the indexer's own objects once when the runtime starts and again each time the seat's
// event catalog-revision is heard (design 47 G A-487b), through the bundle's view path.
say := func(format string, a ...any) {
trunk := index.TrunkStates{Objects: revisions, Under: keptSource, Kept: &index.KeptTrunk{Say: func(format string, a ...any) {
fmt.Fprintf(os.Stderr, "[mesh-delivery-indexer-tools] "+format+"\n", a...)
}
kept := &index.KeptTrunk{Say: say}
trunk := index.TrunkStates{Objects: revisions, Under: keptSource, Kept: kept}
follow := &views.Reader{Module: os.Getenv(stdio.ServedAs), Logf: say, Acts: acts}
if err := follow.Adopt(trunkFollower(kept, revisions), meshcatalog.Seat, meshcatalog.EventCatalogRevision, func(ctx context.Context, data []byte) error {
var ev meshcatalog.CatalogRevisionEvent
if err := json.Unmarshal(data, &ev); err != nil {
return nil // not a revision event: offering it again changes nothing
}
return kept.Heard(ctx, revisions, ev)
}); err != nil {
fmt.Fprintf(os.Stderr, "[mesh-delivery-indexer-tools] %v\n", err)
os.Exit(1)
}
follow.Start(context.Background())
watched := stdio.NewWatchedStates()
tools = append(tools, catalogTools(trunk.State, kept, watched)...)
}}}
tools = append(tools, catalogTools(trunk.State)...)
// The seat mesh-catalog's verb revision, a first view of the catalog half answered in this runtime of its own to exactly
// the parts whose entry declares it (novox/hq ADR 0663).
tools = append(tools, index.RevisionTool(revisions, kept, os.Getenv(stdio.ServedAs)), index.EntryTool(revisions, keptRead{stdio.State(index.EntryIndexState)}), index.EntriesTool(revisions, keptRead{stdio.State(index.EntryIndexState)}, keptRead{stdio.State(index.NamedState)}), index.VersionsTool(keptRead{stdio.State(index.VersionsState)}),
tools = append(tools, index.RevisionTool(revisions, os.Getenv(stdio.ServedAs)), index.EntryTool(revisions, keptRead{stdio.State(index.EntryIndexState)}), index.EntriesTool(revisions, keptRead{stdio.State(index.EntryIndexState)}, keptRead{stdio.State(index.NamedState)}), index.VersionsTool(keptRead{stdio.State(index.VersionsState)}),
index.OutdatedTool(&index.Indexer{Versions: keptRead{stdio.State(index.VersionsState)}, Heard: keptRead{stdio.State(index.HeardState)}, Revisions: revisions}),
index.InputsReadTool(keptRead{stdio.State(index.InputsState)}, stateobjects.ForState(index.InputsState).Objects))
tools = append(tools, index.CatalogRevisionTools(revisions, asks, time.Now, func(what, why string) error {
@@ -337,26 +318,18 @@ func main() {
})...)
tools = append(tools, viewTools(os.Getenv, trunk.State, time.Now)...)
tools = append(tools, index.PendingTools(keptRead{stdio.State(index.PendingState)}, keptRead{stdio.State(index.OwnBuildStateName)}, asks, time.Now)...)
tools = append(tools, index.ValidationTools(asks, time.Now, func(what, why string) error {
_, err := acts.Record(meshhandactsclient.Request{What: what, Why: why, Cause: index.ValidateAgainTool, Verb: "validate-again"})
return err
})...)
tools = append(tools, index.FingerprintsTools(asks, time.Now, func(what, why string) error {
_, err := acts.Record(meshhandactsclient.Request{What: what, Why: why, Cause: index.FingerprintsAgainTool, Verb: "fingerprints-again"})
return err
}, 10*time.Second)...)
tools = append(tools, configReadersTool(watched, trunk.State("modules")), busStateTool(trunk.State), requirementsTool(keptSource),
tools = append(tools, configReadersTool(stdio.NewWatchedStates(), trunk.State("modules")), busStateTool(trunk.State), requirementsTool(keptSource),
packageUsersTool(trunk.State), seatDefinitionTool(keptSource), outOfStepTool(os.Getenv(stdio.ServedAs), keptSource))
if err := stdio.Serve("", tools); err != nil {
fmt.Fprintf(os.Stderr, "[mesh-delivery-indexer-tools] %v\n", err)
os.Exit(1)
}
}
// trunkFollower is the bundle's view of the trunk as a follower of the reader: its first view is the one read of the
// indexer's objects the start makes, and answers no value (the picture is KeptTrunk's, not the reader's).
func trunkFollower(kept *index.KeptTrunk, revisions index.Objects) *views.Follower {
return &views.Follower{Owner: meshcatalog.Seat, State: "catalog-half", Tool: "catalog-revisions",
Ask: func(ctx context.Context, _ string) (views.Answer, error) {
if err := kept.Reread(ctx, revisions); err != nil {
return views.Answer{}, err
}
return views.Answer{V: views.Version, Owner: meshcatalog.Seat, State: "catalog-half", Created: time.Unix(1, 0).UTC(), Values: map[string]json.RawMessage{}, At: time.Now()}, nil
}}
}
@@ -1,120 +0,0 @@
//go:build integration
package main
import (
"context"
"encoding/json"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-sdk/go/busretry"
"mesh-delivery-indexer/internal/index"
)
// TestIndexerFillsTheFloorOfAModuleWithTags (design 47 G §412 as A-490c to A-490h amend it): the whole indexer on a real bus, the
// seat git and the package holders stood in. A module of the trunk with a Go package and no version in `versions` stays unfilled
// while the forge does not answer, and is filled from the highest of its tags and the holder's versions once the forge answers,
// with no restart and no catalog revision: the kept call gives up after its bound (one second here, so that it is spent before the
// forge answers) and the fill asks it again at once (A-490i, ADR 0668 §3). The finding after five minutes is in the unit tests (the
// five minutes are not waited for here).
func TestIndexerFillsTheFloorOfAModuleWithTags(t *testing.T) {
t.Cleanup(busretry.Waits(time.Second, 50*time.Millisecond, 200*time.Millisecond)) // the form gives up after one second
const pkg = "git.novox.be/novox/mesh-delivery/modules/" + testIndexer
var serveTags func()
var prefix atomic.Value
js, ctx, _, _ := startIndexer(t, func(context.Context, buildLog) {}, func(ctx context.Context, js jetstream.JetStream) {
for _, bucket := range []string{index.Bucket(testIndexer, index.VersionsState), index.Bucket(testIndexer, index.HeardState)} {
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: bucket, History: 1}); err != nil {
t.Fatal(err)
}
}
// the module's artifacts are Go packages here, so that the indexer's own entries carry their names (read from go.mod, §676)
e := trunkIndexerEntryOf(t, ctx, js, func(raw []byte) []byte {
var m map[string]any
if err := json.Unmarshal(raw, &m); err != nil {
t.Fatal(err)
}
m["build"] = map[string]any{"artifacts": []any{map[string]any{"name": "go", "kind": "package", "language": "go"}}}
out, _ := json.Marshal(m)
return out
})
for i := range e.Artifacts {
e.Artifacts[i].Package = &meshcatalog.ArtifactPackage{Ecosystem: index.EcosystemGo, Name: pkg}
}
seedBuildModules(t, ctx, js, map[string]meshcatalog.CatalogEntry{testIndexer: e})
reply := func(m *nats.Msg, body map[string]any) {
body["node"] = "n0"
raw, _ := json.Marshal(body)
_ = m.Respond(raw)
}
serve := func(subject string, h func(m *nats.Msg)) {
if _, err := js.Conn().Subscribe(subject, h); err != nil {
t.Fatal(err)
}
}
// the forge answers no one until the test lets it (nobody serves the verb: a transport fault, which the kept call asks again)
serveTags = func() {
serve("mesh.seat.git.tool.tags", func(m *nats.Msg) {
var in git.TagsInput
_ = json.Unmarshal(m.Data, &in)
prefix.Store(in.Prefix)
reply(m, map[string]any{"result": git.TagsAnswer{Repository: in.Repository, Tags: []string{in.Prefix + "0.4.2", in.Prefix + "0.4.10", "unrelated"}}})
})
}
// the package's name is read from the revision's own files (§676): its go.mod
serve("mesh.seat.git.tool.file", func(m *nats.Msg) {
var in git.FileInput
_ = json.Unmarshal(m.Data, &in)
if strings.HasSuffix(in.Path, "go.mod") {
reply(m, map[string]any{"result": git.FileAnswer{Path: in.Path, Ref: in.Ref, Content: "module " + pkg + "\n\ngo 1.26\n"}})
return
}
reply(m, map[string]any{"result": git.FileAnswer{Path: in.Path, Ref: in.Ref, Absent: true}})
})
serve("mesh.seat.go-package-registry.tool.versions", func(m *nats.Msg) {
reply(m, map[string]any{"result": map[string]any{"name": pkg, "versions": []any{}}})
})
})
versions, err := js.KeyValue(ctx, index.Bucket(testIndexer, index.VersionsState))
if err != nil {
t.Fatal(err)
}
// the forge is asked at the start and does not answer: the form gives up and the module's clock is kept in the state heard
heard, err := js.KeyValue(ctx, index.Bucket(testIndexer, index.HeardState))
if err != nil {
t.Fatal(err)
}
for deadline := time.Now().Add(30 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
if _, err := heard.Get(ctx, index.FloorSinceKey(testIndexer)); err == nil {
break
}
}
if _, err := heard.Get(ctx, index.FloorSinceKey(testIndexer)); err != nil {
t.Fatalf("the unanswered floor read of the module is not kept within 30 s: %v", err)
}
if keys, _ := versions.Keys(ctx); len(keys) != 0 {
t.Fatalf("versions holds %v while the forge does not answer, want nothing", keys)
}
serveTags()
var got []string
for deadline := time.Now().Add(30 * time.Second); time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) {
if got, _ = versions.Keys(ctx); len(got) > 0 {
break
}
}
if len(got) != 1 || !strings.Contains(got[0], "0.4.10") {
t.Fatalf("versions holds %v within 30 s of the forge answering, want the floor 0.4.10 of its tags", got)
}
if p, _ := prefix.Load().(string); p != "modules/"+testIndexer+"/v" {
t.Errorf("the tags were read with the prefix %q, want the package's directory and /v", p)
}
}
@@ -22,6 +22,7 @@ import (
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer"
"github.com/nats-io/nats-server/v2/server"
"github.com/nats-io/nats.go"
@@ -96,7 +97,11 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15
// The forge bridge's tool bridge_merged_message (issue 1585): answers the merged message published for a merge, by
// repository@merge, as the bridge does; a restarted indexer holds none in memory and asks it.
var bridgeMessages sync.Map
js, ctx, candidates, _, _ := startIndexerRun(t, func(ctx context.Context, builds buildLog) {
// The build seat node-build-go's stand-in (design 47 G §712, §713): it hears the indexer's validation asks and ends each with
// the manifest validator's verdict; refusing names, by the commit the job runs at, the manifest a revision's change refuses.
var validations *standIn
var refusing sync.Map
js, ctx, candidates, _, restart := startIndexerRun(t, func(ctx context.Context, builds buildLog) {
// The builder's log entries of two builds' ends (ADR 0571 clause 1), as the builder appends them.
builds.end(t, "novox/mesh-delivery", "a", "modules/a")
builds.end(t, "novox/mesh-catalog", "c", "modules/c")
@@ -123,8 +128,9 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15
if _, err := tc.Put(ctx, key, value); err != nil {
t.Fatal(err)
}
// The trunk holds the indexer's entry, taken in at an earlier commit with its own manifest; the take-in asks no validation (A-476 a).
// The trunk holds the indexer's entry, taken in at an earlier commit with its own manifest; the validation names no Go module (A-268).
seedBuildModules(t, ctx, js, map[string]meshcatalog.CatalogEntry{testIndexer: trunkIndexerEntry(t, ctx, js)})
validations = standInBuildSeat(t, ctx, js, &refusing)
// The node-engine's composition on n1 (process.Conn.Call's subjects): the build of the indexer that runs there.
answer := func(m *nats.Msg) {
raw, _ := json.Marshal(map[string]any{"result": map[string]any{"runs": map[string]string{"fp": *running.Load()}}, "node": "n1"})
@@ -275,13 +281,17 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15
}
return takein.TakeIn{}, false
}
// A-476 a, b: the revision changing the indexer's own tree is taken in at once, with the build this program runs and no
// validation ask: it is never said pending, and the build the revision names is not waited for.
if taken, ok := said(takein.TakeInTaken, m1Key); !ok || len(taken.Merges) != 1 || taken.Merges[0].Merge != m1.Merge {
t.Fatalf("take-in.taken.%s = %+v (said %v) within 20 s, want the revision taken in at once", m1Key, taken, ok)
held, ok := said(takein.TakeInPending, m1Key)
if !ok || held.What != takein.TakeInRevision || held.WaitsFor == "" || held.WaitsFor == first || len(held.Merges) != 1 {
t.Fatalf("take-in.pending.%s = %+v (said %v) within 20 s, want the revision waiting for another build than %s", m1Key, held, ok, first)
}
if held := saidAll(takein.TakeInPending, m1Key); len(held) != 0 {
t.Errorf("the revision was said pending: %+v", held)
named := held.WaitsFor
// IX5 (§712): the revision was validated first, by one run ask whose job names the indexer entry's repository, path and
// the revision's head, and no heads; accepted, it is never asked again while it waits for the named indexer.
if asks := validations.asked(); len(asks) != 1 || asks[0].Run == nil || asks[0].Kind != sdkstep.AskCheck ||
asks[0].Run.Program.Repository != m1.Repository || asks[0].Run.Program.Dir != "modules/"+ix || asks[0].Run.Program.Commit != m1.Merge ||
len(asks[0].Run.Heads) != 0 || asks[0].Run.Program.Package != "./cmd/mesh-judge" {
t.Fatalf("validation asks = %+v, want one run of the indexer's manifest validator at %s", asks, m1.Merge)
}
get := func(bucket, key string) ([]byte, bool) {
t.Helper()
@@ -302,10 +312,11 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15
}
return e.Value(), true
}
// Only what building and moving the named indexer needs is written.
raw, ok := get(index.ModulesState, ix)
own, err := meshcatalogclient.DecodeModule(raw)
if !ok || err != nil || own.Commit != m1.Merge {
t.Fatalf("modules.%s = %+v (%v), want the indexer at the merge", ix, own, err)
if !ok || err != nil || own.Fp2 != named || own.Commit != m1.Merge {
t.Fatalf("modules.%s = %+v (%v), want the named build %s at the merge", ix, own, err, named)
}
if _, ok := get(index.Bucket(testIndexer, index.ManifestsState), own.Manifest); !ok {
t.Errorf("manifests.%s is not written", own.Manifest)
@@ -314,11 +325,16 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15
t.Errorf("the verdict on the indexer's manifest %s is not written", own.Manifest)
}
repoKey, _ := trunk.RepositoryKey(m1.Repository)
if raw, ok := get(index.ModulesState, "a"); !ok || !strings.Contains(string(raw), m1.Merge) {
t.Errorf("modules.a = %s, want it at the merge %s", raw, m1.Merge)
if _, ok := get(index.ModulesState, "a"); ok {
t.Errorf("modules.a is written while the revision is pending")
}
if _, ok := get(index.Bucket(testIndexer, index.TakenState), repoKey); !ok {
t.Errorf("taken.%s is not written", repoKey)
if _, ok := get(index.Bucket(testIndexer, index.TakenState), repoKey); ok {
t.Errorf("taken.%s is written while the revision is pending", repoKey)
}
if e, err := candidates.Get(ctx, "feat/x"); err != nil {
t.Errorf("the candidate feat/x is gone while the merge of #12 is pending: %v", err)
} else if c, err := index.DecodeCandidate(e.Value()); err != nil || len(c.Pulls) != 2 {
t.Errorf("the candidate feat/x = %+v while the merge of #12 is pending, want its two pull requests", c)
}
// A group's first member arrives meanwhile; its end is still to come (the knowledge base's remark: it stays across the restart).
@@ -349,13 +365,34 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15
}
var w []takein.PendingWait
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
if w = waits(); len(w) == 1 && w[0].Said != "" {
if w = waits(); len(w) == 2 && w[0].WaitsFor == named && w[1].Said != "" {
break
}
}
if len(w) != 1 || w[0].Key != groupKey || w[0].What != takein.TakeInGroup || len(w[0].Merges) != 1 || len(w[0].Keys) != 1 || w[0].Keys[0] != g1Key ||
w[0].Said != takein.TakeInPendingEvent {
t.Fatalf("indexer_pending = %+v, want only the group gathered, its first member %s", w, g1Key)
if len(w) != 2 || w[0].Key != m1Key || w[0].WaitsFor != named || !slices.Contains(w[0].Holds, "a") || !slices.Contains(w[0].Holds, ix) ||
w[1].Key != groupKey || w[1].What != takein.TakeInGroup || len(w[1].Merges) != 1 || w[1].Said != takein.TakeInPendingEvent {
t.Fatalf("indexer_pending = %+v, want the revision %s waiting for %s holding a and the indexer, then the group gathered", w, m1Key, named)
}
// The named indexer starts (the node-engine now runs that build): it takes the revision in, and leaves the group.
running.Store(&named)
restart()
if taken, ok := said(takein.TakeInTaken, m1Key); !ok || len(taken.Merges) != 1 || taken.Merges[0].Merge != m1.Merge {
t.Fatalf("take-in.taken.%s = %+v (said %v) within 20 s of the start of %s", m1Key, taken, ok, named)
}
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
if w = waits(); len(w) == 1 {
break
}
}
if raw, ok := get(index.ModulesState, "a"); !ok || !strings.Contains(string(raw), m1.Merge) {
t.Errorf("modules.a = %s after the take-in at the start, want it at the merge %s", raw, m1.Merge)
}
if _, ok := get(index.Bucket(testIndexer, index.TakenState), repoKey); !ok {
t.Errorf("taken.%s is not written by the take-in at the start", repoKey)
}
if len(w) != 1 || w[0].Key != groupKey || len(w[0].Keys) != 1 || w[0].Keys[0] != g1Key {
t.Fatalf("indexer_pending after the start = %+v, want only the group gathered, its first member %s", w, g1Key)
}
if _, ok := get(index.ModulesState, "c"); ok {
t.Errorf("modules.c is written while its group is gathered")
@@ -388,21 +425,32 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15
t.Errorf("the group's first member %s was said pending as a revision of its own", g1Key)
}
// IX5 (§712, §713): a merge whose changed manifest gains a problem the trunk case lacks stays pending, refused naming it,
// said in take-in.pending; nothing of it is taken in.
m3 := forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/y", Number: 13, Head: h1,
Merge: "3333333333333333333333333333333333333333", Root: "r0", Trees: map[string]string{"modules/a": "tree-a-3"},
Manifests: map[string]string{"modules/a": `{"module":"a","version":"2.0"}`}}
refusing.Store(m3.Merge, "a")
m3Key := takein.PendingKey(merged(m3))
if refused, ok := said(takein.TakeInPending, m3Key); !ok || !strings.Contains(refused.Refused, "a: the stand-in's problem") || refused.Failed != "" {
t.Fatalf("take-in.pending.%s = %+v (said %v) within 20 s, want the revision refused naming a's problem", m3Key, refused, ok)
}
if w = waits(); len(w) != 1 || w[0].Key != m3Key || w[0].Validation == nil || w[0].Validation.Accepted != 0 || len(validations.asked()) != 3 {
t.Fatalf("indexer_pending = %+v, asks %d: want the refused revision pending with its validation, asked once after m1 and the group", w, len(validations.asked()))
}
if raw, ok := get(index.ModulesState, "a"); !ok || strings.Contains(string(raw), m3.Merge) {
t.Errorf("modules.a = %s: the refused revision was taken in", raw)
}
}
// trunkIndexerEntry keeps the indexer's own manifest by its digest, accepted, and answers its trunk entry at an earlier commit.
func trunkIndexerEntry(t *testing.T, ctx context.Context, js jetstream.JetStream) meshcatalog.CatalogEntry {
return trunkIndexerEntryOf(t, ctx, js, func(raw []byte) []byte { return raw })
}
// trunkIndexerEntryOf is trunkIndexerEntry with the manifest's bytes changed by edit first.
func trunkIndexerEntryOf(t *testing.T, ctx context.Context, js jetstream.JetStream, edit func([]byte) []byte) meshcatalog.CatalogEntry {
t.Helper()
raw, err := os.ReadFile("../../module.json")
if err != nil {
t.Fatal(err)
}
raw = edit(raw)
var m catalogue.Manifest
if err := json.Unmarshal(raw, &m); err != nil {
t.Fatal(err)
@@ -427,6 +475,63 @@ func trunkIndexerEntryOf(t *testing.T, ctx context.Context, js jetstream.JetStre
return e
}
// standIn is the build seat node-build-go as the indexer's validation meets it: the asks it heard, oldest first.
type standIn struct {
mu sync.Mutex
asks []sdkstep.StepAsk
}
func (s *standIn) asked() []sdkstep.StepAsk {
s.mu.Lock()
defer s.mu.Unlock()
return slices.Clone(s.asks)
}
// standInBuildSeat keeps the seat's check queue as a stream and hears the indexer's asks on it (accept.check.indexer): each is
// ended at once with step-ended on the seat's family, its answer inline (A-254), the program's last line: the manifest validator's verdict over both cases,
// the change case refusing the manifest refusing names for the job's commit.
func standInBuildSeat(t *testing.T, ctx context.Context, js jetstream.JetStream, refusing *sync.Map) *standIn {
t.Helper()
subject, err := sdkstep.AskSubjectBy(sdkstep.AskGo, sdkstep.AskCheck, index.ValidationAsker)
if err != nil || subject != "mesh.seat.node-build-go.accept.check.mesh-catalog" {
t.Fatalf("the validation's subject %q %v, want the indexer's seat name its asker (A-163, A-232)", subject, err)
}
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "SEAT_NODE_BUILD_GO_CHECK", Subjects: []string{"mesh.seat.node-build-go.accept.check.>"}}); err != nil {
t.Fatal(err)
}
// The indexer's state inputs and its objects, where the validation's request is kept for the build seat's holder (§354).
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.Bucket(testIndexer, index.InputsState), History: 1}); err != nil {
t.Fatal(err)
}
if _, err := js.CreateObjectStore(ctx, jetstream.ObjectStoreConfig{Bucket: catalogue.BusName(testIndexer, catalogue.StateObjectsName(index.InputsState))}); err != nil {
t.Fatal(err)
}
s := &standIn{}
if _, err := js.Conn().Subscribe(subject, func(m *nats.Msg) {
a, err := sdkstep.DecodeStepAsk(m.Data)
if err != nil || a.Run == nil {
t.Errorf("the stand-in build seat heard what is no run ask: %v", err)
return
}
s.mu.Lock()
s.asks = append(s.asks, a)
s.mu.Unlock()
change := composer.CaseVerdict{Name: "change", Manifests: map[string]composer.ManifestRead{}}
if module, ok := refusing.Load(a.Run.Program.Commit); ok {
change.Manifests[module.(string)] = composer.ManifestRead{Refused: "the stand-in's problem"}
}
verdict, _ := json.Marshal(composer.Verdict{V: 1, Cases: []composer.CaseVerdict{change, {Name: judge.TrunkCase, Manifests: map[string]composer.ManifestRead{}}}})
end, _ := json.Marshal(sdkstep.StepEnded{V: 1, ID: a.ID, Ask: a.ID, Asker: index.ValidationAsker, Node: "n1", Outcome: sdkstep.OutcomePassed,
Output: string(verdict), At: time.Now().UTC()})
if _, err := js.Publish(ctx, nodebuildgo.SubjectStepEnded("n1"), end); err != nil {
t.Errorf("the stand-in build seat could not say step-ended: %v", err)
}
}); err != nil {
t.Fatal(err)
}
return s
}
// jsSource is a bucket as a views.Source.
type jsSource struct {
ctx context.Context
@@ -922,17 +1027,6 @@ func TestIndexerWritesModulesAtAMerge(t *testing.T) { // 1239/AC9; I1, I3: the m
if asks, err = js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.Bucket(testIndexer, index.TakeInsState), History: 1}); err != nil {
t.Fatal(err)
}
// A want and a running build said before the start: the indexer removes no entry until both streams were read whole and
// not empty (design 47 G §675, 5188d36a), so the drop below removes entry.dx only with them.
fp := "src2:" + strings.Repeat("b", 64)
at := time.Now().UTC()
want, _ := json.Marshal(meshwants.WantedEvent{V: 1, Node: "n1", Module: "avahi", Fp: fp, Revision: 1, At: &at})
run, _ := json.Marshal(nodeengine.Running{V: 1, Node: "n2", Module: "avahi", Fingerprint: fp, At: at})
for subject, data := range map[string][]byte{meshwants.SubjectWanted("n1", "avahi"): want, nodeengine.SubjectRunning("n2", "avahi"): run} {
if _, err := js.Publish(ctx, subject, data); err != nil {
t.Fatal(err)
}
}
})
const merge = "89abcdef0123456789abcdef0123456789abcdef"
data, err := forgemsg.EncodeForge(forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12,
@@ -1424,8 +1518,7 @@ func TestIndexerCountsOnlyThePlacedBuildNodesToolchains(t *testing.T) {
t.Fatal(err)
}
toolchains = kv
// A-471: a toolchain is agreed by its language version (go1.27), not its patch; n2 reports another language version.
for node, version := range map[string]string{"n1": "go1.27", "n2": "go1.28.1"} {
for node, version := range map[string]string{"n1": "go1.27.3", "n2": "go1.27.2"} {
key, _ := index.ToolchainKey(node, "go")
value, err := index.EncodeToolchain(nodebuildgo.Toolchain{Language: "go", Version: version, Recipe: "1", At: ptr(time.Now())})
if err != nil {
@@ -1469,15 +1562,15 @@ func TestIndexerCountsOnlyThePlacedBuildNodesToolchains(t *testing.T) {
return got
}
agreed := func(m meshcatalog.ModulesByModuleValue) bool {
return m.Fp2 != "" && m.Toolchains["go"].Version == "go1.27"
return m.Fp2 != "" && m.Toolchains["go"].Version == "go1.27.3"
}
held := read(agreed)
if !agreed(held) {
t.Fatalf("within 20 s avahi has fp2 %q, none2 %q, toolchains %v: want a fingerprint of n1's go1.27 alone", held.Fp2, held.None2, held.Toolchains)
t.Fatalf("within 20 s avahi has fp2 %q, none2 %q, toolchains %v: want a fingerprint of n1's go1.27.3 alone", held.Fp2, held.None2, held.Toolchains)
}
place(ctx, t, "n2", "mesh-build-go")
key, _ := index.ToolchainKey("n2", "go")
value, err := index.EncodeToolchain(nodebuildgo.Toolchain{Language: "go", Version: "go1.28.1", Recipe: "1", At: ptr(time.Now().Add(time.Second))})
value, err := index.EncodeToolchain(nodebuildgo.Toolchain{Language: "go", Version: "go1.27.2", Recipe: "1", At: ptr(time.Now().Add(time.Second))})
if err != nil {
t.Fatal(err)
}
@@ -1500,7 +1593,7 @@ func TestIndexerCountsOnlyThePlacedBuildNodesToolchains(t *testing.T) {
t.Fatal(err)
}
disagree := func(m meshcatalog.ModulesByModuleValue) bool {
return m.Fp2 == "" && strings.HasPrefix(m.None2, "builders disagree: go1.27 recipe 1 on n1; go1.28 recipe 1 on n2")
return m.Fp2 == "" && strings.HasPrefix(m.None2, "builders disagree: go1.27.2 recipe 1 on n2")
}
if m := read(disagree); !disagree(m) {
t.Fatalf("within 20 s of a merge after n2's assignments avahi has fp2 %q, none2 %q: want the builders to disagree", m.Fp2, m.None2)
@@ -479,7 +479,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// The take-in step (ADR 0633): its state pending and its stream take-in, both this delivery's, bound when they appear;
// until then every merge is taken in at once, as before.
Pending: awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.PendingState), "no merge is held for the indexer its revision names: each is taken in at once", logger.Printf)},
SayTakeIn: sayTakeIn(js, cred)}
SayTakeIn: sayTakeIn(js, cred),
// The validation's run ask (design 47 G §713, A-247, A-248), put under the indexer's own asker.
ValidationAsks: buildrun.Asker{Name: index.ValidationAsker, Publisher: buildrun.JetStreamAsks{JS: js}},
// The build seat's verb output, for a validation's answer kept as an object (A-254).
AskVerb: calls.CallFile}
// The registrations and the changes that wait for their warrant (design 47 G §2, §667): their states are bound when they
// appear; a warrant is heard on the router's decided event for this module, below.
registry := newRegistry(js, conn, cred, calls.Call,
@@ -491,7 +495,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
ix.Registry = registry
named.Changed = ix.RemoveUnlisted
// The first fill of the versions (design 47 G §372): at the start, and again at each catalog revision kept until it is made.
fill := &versionsFill{ix: ix, ctx: ctx, ask: versionsVia(calls.CallKeptWhileOpen, ix.ModuleInNewestView, logger.Printf), floor: index.FloorSources{Tags: tagsVia(calls.CallKept), Holder: holderVia(calls.CallKept)}, logf: logger.Printf}
fill := &versionsFill{ix: ix, ctx: ctx, ask: versionsVia(calls.CallKeptWhileOpen, ix.ModuleInNewestView, logger.Printf), logf: logger.Printf}
fillFollow.Store(fill)
fill.kick()
ix.SeedModules = func(ctx context.Context) ([]index.SeedModule, error) {
@@ -206,7 +206,6 @@ func TestAMeshSDKMergeMovesOnlyTheModulesImportingAChangedPackage(t *testing.T)
if pub.Commit != m2 || len(pub.Packages) != 3 {
t.Fatalf("packages.%s = %+v within 20 s, want the merge %s with a digest of go, announce and buildstep", key, pub, m2)
}
// A-476a: mesh-sdk's merge moves importer, of another repository, which imports a package it moved.
moved := module("importer", func(m meshcatalog.ModulesByModuleValue) bool { return m.Fp2 != importer.Fp2 })
if moved.Packages[sdkGo] != m2 {
t.Errorf("importer moved to %+v, want it built against %s", moved.Packages, m2)
@@ -28,6 +28,7 @@ type takerIn interface {
NamedDrop(ctx context.Context, fingerprint, member, by, why string) index.TakenIn
Unpin(ctx context.Context, revision, record, by, why string) index.TakenIn
DropRunByHand(ctx context.Context, run, by, why string) index.TakenIn
ValidateAgainByHand(ctx context.Context, key, by, why string) index.TakenIn
RegistryAct(ctx context.Context, a index.TakeInAsk) index.TakenIn
}
@@ -74,6 +75,9 @@ func takenIn(ctx context.Context, ix takerIn, kv jetstream.KeyValue, e jetstream
} else if a.Merge != "" {
asked, r = "hand take-in of a merge", ix.TakeInMerge(ctx, a.MergeRepository, a.Merge, a.Sequence, a.By, a.Why)
a.Module = r.Module
} else if a.Pending != "" && a.ValidateAgain {
asked, r = "validation asked again", ix.ValidateAgainByHand(ctx, a.Pending, a.By, a.Why)
a.Module = a.Pending
} else if a.Pending != "" && a.Drop {
asked, r = "drop of a pending merge", ix.DropPending(ctx, a.Pending, a.By, a.Why, a.Abandon)
a.Module = a.Pending
@@ -4,23 +4,20 @@ import (
"context"
"encoding/json"
"fmt"
"strings"
"sync/atomic"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/gopackageregistry"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/npmpackageregistry"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-indexer/internal/index"
@@ -158,6 +155,15 @@ func heardStepEnded(ctx context.Context, ix *index.Indexer, m jetstream.Msg, log
_ = m.Ack()
return
}
// The end of a validation ask resumes its pending revision (design 47 G §713); its own fault is given again by the bus.
var end buildstep.StepEnded
if err := json.Unmarshal(m.Data(), &end); err == nil {
if err := ix.HearValidationEnded(ctx, end); err != nil {
logf("take-in: the end of the validation ask %s is not taken in, so the bus gives it again: %v", end.Ask, err)
leftForBus(m, "the end of the validation ask "+end.Ask, err, logf)
return
}
}
if err := ix.HearInputEnded(ctx, e); err != nil {
logf("inputs: the input of the ask %s could not be deleted, so the bus gives it again: %v", e.Ask, err)
leftForBus(m, "the step-ended of the ask "+e.Ask, err, logf)
@@ -192,11 +198,10 @@ func versionsVia(call openCaller, inView func(ctx context.Context, module string
// versionsFill is the first fill of the versions in the background (design 47 G §372): started at the start and at each
// catalog revision kept, until it is made; one runs at a time, and a fill that fails is said and made again at the next.
type versionsFill struct {
ix *index.Indexer
ctx context.Context
ask index.VersionsAsker
floor index.FloorSources // what the floor reads (A-490g); a nil reader fills none
logf func(string, ...any)
ix *index.Indexer
ctx context.Context
ask index.VersionsAsker
logf func(string, ...any)
}
func (f *versionsFill) kick() {
@@ -205,28 +210,6 @@ func (f *versionsFill) kick() {
f.logf("versions: the first fill waits: %v", err)
}
}()
go f.fillFloors()
}
// fillFloors fills the floor of each module the versions hold none of, each on its own so that one unanswered read holds no other
// module's (A-490d, A-490f, A-490i): at every start and every revision, not waiting for the builder's half. A read is a kept call
// asked again until it answers, and the indexer holds no timer of its own (ADR 0437 §8).
func (f *versionsFill) fillFloors() {
if f.floor.Tags == nil && f.floor.Holder == nil {
return
}
modules, err := f.ix.FloorModules(f.ctx)
if err != nil {
f.logf("versions: the floor waits: %v", err)
return
}
for _, m := range modules {
go func() {
if err := f.ix.FillFloor(f.ctx, m, f.floor); err != nil {
f.logf("versions: the floor of %s waits for the next start or revision: %v", m, err)
}
}()
}
}
// afterRevision runs the fill's kick after each catalog revision kept, once the fill is set.
@@ -294,49 +277,3 @@ func runningWholeVia(js jetstream.JetStream) index.RunningReader {
}
}
}
// tagsVia reads every tag of a repository that starts with a prefix from the seat git's verb tags as a kept call, page by
// page until none is more (design 47 G §412, A-490c: protected or not).
func tagsVia(call toolCaller) index.TagsReader {
return func(_ context.Context, repository, prefix string) ([]string, error) {
var out []string
for page := int64(1); ; page++ {
var a git.TagsAnswer
in := git.TagsInput{Repository: trunk.RepositoryName(repository), Prefix: prefix, Page: page}
if !strings.Contains(in.Repository, "/") {
return nil, fmt.Errorf("%q is not <owner>/<name>", repository)
}
if err := askGit(call, git.VerbTags, in, &a); err != nil {
return nil, err
}
out = append(out, a.Tags...)
if !a.More {
return out, nil
}
}
}
}
// holderVia reads the versions a package holder answers for a package: the seat npm-package-registry's or go-package-registry's
// verb versions (A-490g), as a kept call.
func holderVia(call toolCaller) index.HolderReader {
return func(_ context.Context, ecosystem, name string) ([]string, error) {
seat := npmpackageregistry.Seat
if ecosystem == index.EcosystemGo {
seat = gopackageregistry.Seat
}
raw, err := call("seat:"+seat+"."+npmpackageregistry.VerbVersions, npmpackageregistry.VersionsInput{Name: name, Kind: npmpackageregistry.ArtifactKindPackage})
if err != nil {
return nil, err
}
var a npmpackageregistry.VersionsAnswer
if err := json.Unmarshal(raw, &a); err != nil {
return nil, fmt.Errorf("the answer of %s.versions is not a package's versions: %w", seat, err)
}
var out []string
for _, v := range a.Versions {
out = append(out, v.Version)
}
return out, nil
}
}
@@ -1,190 +0,0 @@
package index
import (
"context"
"errors"
"fmt"
"maps"
"slices"
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk"
)
// A merge the indexer never took in (design 47 G A-463, amended by A-464): at its start and on every source revision the
// gate's source names, the indexer compares each repository's `taken` merge with the head that source names. A repository
// whose head is newer than its taken merge (the taken merge is that head's ancestor, or no merge is taken) is taken in as
// its merged message would have been (the catch-up of indexer_take_in, for that repository), said in the log, with the
// finding merge-untaken raised while it stays behind and cleared once it is taken in. A head the taken merge contains (the
// source not yet moved to a merge just taken in) is left, and a head the taken merge neither contains nor is in is said.
// UntakenMergeKind is the finding of a repository whose head the gate's source names but whose merge the indexer never took in.
const UntakenMergeKind = "merge-untaken"
func untakenFindingID(own, key string) string { return own + ".untaken." + key }
// kickBehind runs the comparison in the background, one at a time: a source revision that arrives while one runs makes it
// run once more, on the latest heads. It runs after the first start's catch-up, which takes every head in itself.
func (ix *Indexer) kickBehind(ctx context.Context) {
if ix.Taken == nil || ix.Message == nil || ix.Modules == nil {
return
}
if ix.catchingUp.Load() {
ix.behindWaiting.Store(true) // compared again when that catch-up ends (A-463a)
return
}
if ix.behindRunning.Swap(true) {
ix.behindAgain.Store(true)
return
}
ix.catchUpWG.Add(1)
go func() {
defer ix.catchUpWG.Done()
for {
ix.behindAgain.Store(false)
ix.TakeInBehind(ctx)
if !ix.behindAgain.Load() {
ix.behindRunning.Store(false)
if !ix.behindAgain.Load() || ix.behindRunning.Swap(true) {
return
}
}
}
}()
}
// TakeInBehind compares each repository's taken merge with the head the latest source names, takes in each repository
// that is behind and answers the repositories it took in.
func (ix *Indexer) TakeInBehind(ctx context.Context) []string {
heads := ix.sourceHeads()
if ix.Taken == nil || ix.Message == nil || ix.Modules == nil || len(heads) == 0 {
return nil
}
behind := map[string]string{}
taken := map[string]string{}
held := ix.pendingHeads(ctx)
for _, repository := range slices.Sorted(maps.Keys(heads)) {
head := heads[repository]
key, err := trunk.RepositoryKey(repository)
if head == "" || err != nil {
continue
}
value, _, found, err := ix.Taken.Get(ctx, key)
if errors.Is(err, process.ErrNotYet) {
return nil // the state is not on the bus yet: the next source revision compares again
}
if err != nil {
ix.logf("merges: taken.%s cannot be read, so %s is not compared with its head: %v", key, repository, err)
continue
}
merge := ""
if found {
t, err := takein.DecodeTaken(value)
if err != nil {
ix.logf("merges: taken.%s holds what is not a taken merge, left: %v", key, err)
continue
}
merge = t.Merge
}
switch {
case held[head]:
// A head the pending holds (a member waiting for its group's end) is taken in by that path, whole with
// its group: it is not behind (A-463a).
ix.clearUntaken(ctx, key, "the head of "+repository+" waits in the indexer's pending")
case merge == head:
ix.clearUntaken(ctx, key, "the indexer took in the head of "+repository)
case merge == "":
behind[repository], taken[repository] = head, ""
case ix.Contains == nil:
ix.logf("merges: %s is taken in at %.12s and its head is %.12s, but the indexer reads no ancestry of the forge, so which is newer is not said", repository, merge, head)
default:
inHead, err := ix.Contains(ctx, repository, merge, head)
if err != nil {
ix.logf("merges: whether %.12s is in %.12s of %s cannot be read: %v", merge, head, repository, err)
continue
}
if inHead {
behind[repository], taken[repository] = head, merge
continue
}
ix.clearUntaken(ctx, key, "the head of "+repository+" is in the merge taken in")
if headIn, err := ix.Contains(ctx, repository, head, merge); err == nil && !headIn {
ix.logf("merges: %s is taken in at %.12s and its head is %.12s, and neither holds the other: left", repository, merge, head)
}
}
}
for _, repository := range slices.Sorted(maps.Keys(behind)) {
ix.sayUntaken(ctx, repository, taken[repository], behind[repository])
ix.logf("merges: %s is taken in at %.12s but the gate's source names %.12s: taken in now", repository, taken[repository], behind[repository])
}
if len(behind) == 0 {
return nil
}
unread := ix.takeInHeads(ctx, behind)
var done []string
for _, repository := range slices.Sorted(maps.Keys(behind)) {
if slices.Contains(unread, repository) {
continue
}
done = append(done, repository)
key, _ := trunk.RepositoryKey(repository)
ix.clearUntaken(ctx, key, fmt.Sprintf("%s taken in at %.12s", repository, behind[repository]))
ix.logf("merges: %s taken in at %.12s", repository, behind[repository])
}
if len(done) > 0 {
if err := ix.KeepView(ctx); err != nil {
ix.logf("merges: the view could not be kept after %v were taken in: %v", done, err)
}
}
return done
}
func (ix *Indexer) sayUntaken(ctx context.Context, repository, merge, head string) {
if ix.Findings == nil {
return
}
key, _ := trunk.RepositoryKey(repository)
taken := "no merge"
if merge != "" {
taken = fmt.Sprintf("%.12s", merge)
}
f := ix.observed(mcc.ScopeModule, untakenFindingID(ix.Module, key), UntakenMergeKind,
fmt.Sprintf("the indexer took in %s at %s, and the gate's source names the head %.12s", repository, taken, head),
"A merge the indexer never took in",
"A merge into a watched branch was never heard or applied, so no package or module of the repository follows its trunk. The indexer takes the head in itself and clears this; indexer_take_in with catch-up does it by hand.",
UntakenMergeKind)
if err := ix.Findings.Write(ctx, f); err != nil {
ix.logf("merges: the finding %s for %s is not written: %v", UntakenMergeKind, repository, err)
}
}
func (ix *Indexer) clearUntaken(ctx context.Context, key, why string) {
if ix.Findings == nil {
return
}
if err := ix.clearFinding(ctx, mcc.ScopeModule, untakenFindingID(ix.Module, key), UntakenMergeKind, why); err != nil {
ix.logf("merges: the finding %s for %s is not cleared: %v", UntakenMergeKind, key, err)
}
}
// pendingHeads are the merge commits the indexer's pending holds (A-463a).
func (ix *Indexer) pendingHeads(ctx context.Context) map[string]bool {
out := map[string]bool{}
if ix.Pending == nil || !ix.pendingOnBus(ctx) {
return out
}
entries, err := ReadPending(ctx, ix.Pending, ix.logf)
if err != nil {
ix.logf("merges: the pending state cannot be read, so every head is compared: %v", err)
return out
}
for _, e := range entries {
if e.Forge.Merge != "" {
out[e.Forge.Merge] = true
}
}
return out
}
@@ -1,112 +0,0 @@
package index
import (
"context"
"encoding/json"
"testing"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk"
)
// Design 47 G A-463: a merge the indexer never took in is noticed at a source revision and taken in as its merged message
// would have been, with a finding raised while it stays behind and cleared once it is taken in.
func TestAMergeTheIndexerNeverTookInIsTakenInAndSaidWhileItIsBehind(t *testing.T) {
ctx := context.Background()
b := newTakeInBed(t)
b.ix.Taken, b.ix.Findings = newMem(), newFindings()
key, _ := trunk.RepositoryKey(catalogueRepo)
old, head := sha('7'), sha('8')
data, _ := takein.EncodeTaken(takein.Taken{Repository: trunk.RepositoryName(catalogueRepo), Merge: old})
b.ix.Taken.(*memStore).put(key, data)
ancestry := map[[2]string]bool{{old, head}: true} // old is in head
b.ix.Contains = func(_ context.Context, _, commit, in string) (bool, error) {
return commit == in || ancestry[[2]string{commit, in}], nil
}
b.ix.source = sourceNow{rev: 3, heads: map[string]string{catalogueRepo: head}}
// The head has no merged message yet: nothing is taken in and the finding stays.
if done := b.ix.TakeInBehind(ctx); len(done) != 0 || len(b.ix.Findings.Held()) != 1 || b.ix.Findings.Held()[0].Kind != UntakenMergeKind {
t.Fatalf("a head whose message is not read: taken in %v, findings %+v: want none taken in and %s raised", done, b.ix.Findings.Held(), UntakenMergeKind)
}
// Its message is there: it is taken in, its taken key moves and the finding is cleared.
b.publish(t, b.merge(30, head, "tree-avahi-3", ""))
if done := b.ix.TakeInBehind(ctx); len(done) != 1 || len(b.ix.Findings.Held()) != 0 || b.commit(t, "avahi") != head {
t.Fatalf("a head with its message: taken in %v, findings %+v, avahi at %s: want it taken in at %s and the finding cleared", done, b.ix.Findings.Held(), b.commit(t, "avahi"), head)
}
if v, _, _, _ := b.ix.Taken.Get(ctx, key); string(v) == string(data) {
t.Errorf("taken.%s still names %s", key, old)
}
// A head the taken merge already holds (the source not yet moved to a merge just taken in) is left.
b.ix.source = sourceNow{rev: 4, heads: map[string]string{catalogueRepo: old}}
if done := b.ix.TakeInBehind(ctx); len(done) != 0 {
t.Errorf("a head older than the taken merge: taken in %v, want none", done)
}
// A source revision starts the comparison by itself (no timer): the taken key is put back to the old merge and the source
// names the head again.
b.ix.Taken.(*memStore).put(key, data)
if err := applyS(b.ix, ctx, 5, sourceOf(catalogueRepo, head)); err != nil {
t.Fatal(err)
}
b.ix.catchUpWG.Wait()
if v, _, _, _ := b.ix.Taken.Get(ctx, key); string(v) == string(data) {
t.Errorf("a source revision naming %s left taken.%s at %s", head, key, old)
}
}
// Design 47 G A-463a: a head the pending holds (a group member waiting for its group's end) is not behind: it raises no finding
// and is not taken in alone.
func TestAHeadThePendingHoldsIsNotBehind(t *testing.T) {
ctx := context.Background()
b := newTakeInBed(t)
b.ix.Taken, b.ix.Findings = newMem(), newFindings()
key, _ := trunk.RepositoryKey(catalogueRepo)
old, head := sha('7'), sha('8')
data, _ := takein.EncodeTaken(takein.Taken{Repository: trunk.RepositoryName(catalogueRepo), Merge: old})
b.ix.Taken.(*memStore).put(key, data)
b.ix.Contains = func(_ context.Context, _, commit, in string) (bool, error) { return commit == in || commit == old, nil }
b.publish(t, b.merge(30, head, "tree-avahi-3", ""))
// The head waits in pending as a member of a group not ended.
held := b.merge(30, head, "tree-avahi-3", "")
raw, _ := forgemsg.EncodeForge(held)
pv, _ := json.Marshal(Pending{V: 1, Forge: raw, Group: "feat", Since: time.Now(), LastWriter: "x"})
b.pending.put(takein.PendingKey(1), pv)
b.ix.source = sourceNow{rev: 3, heads: map[string]string{catalogueRepo: head}}
if done := b.ix.TakeInBehind(ctx); len(done) != 0 || len(b.ix.Findings.Held()) != 0 || b.commit(t, "avahi") == head {
t.Fatalf("a head the pending holds: taken in %v, findings %+v, avahi at %s: want none taken in and none raised", done, b.ix.Findings.Held(), b.commit(t, "avahi"))
}
}
// Design 47 G A-463a: a source revision heard while a catch-up runs is compared again when that catch-up ends.
func TestASourceRevisionHeardDuringACatchUpIsComparedWhenItEnds(t *testing.T) {
ctx := context.Background()
b := newTakeInBed(t)
b.ix.Taken, b.ix.Findings = newMem(), newFindings()
key, _ := trunk.RepositoryKey(catalogueRepo)
h1, h2 := sha('8'), sha('9')
b.ix.Contains = func(_ context.Context, _, commit, in string) (bool, error) {
return commit == in || (commit == h1 && in == h2), nil
}
b.publish(t, b.merge(30, h1, "tree-avahi-3", ""))
b.publish(t, b.merge(31, h2, "tree-avahi-4", ""))
b.ix.source = sourceNow{rev: 3, heads: map[string]string{catalogueRepo: h1}}
inner := b.ix.Message
heard := false
b.ix.Message = func(c context.Context, r, m string, n int) ([]byte, error) {
if !heard {
heard = true // the catch-up of h1 runs: a new source revision is heard now
if err := applyS(b.ix, ctx, 4, sourceOf(catalogueRepo, h2)); err != nil {
t.Error(err)
}
}
return inner(c, r, m, n)
}
b.ix.CatchUp(ctx)
b.ix.catchUpWG.Wait()
v, _, _, _ := b.ix.Taken.Get(ctx, key)
if got, _ := takein.DecodeTaken(v); got.Merge != h2 {
t.Errorf("taken after the catch-up = %s, want %s: the revision heard during it is compared afterwards\n%s", got.Merge, h2, b.logs)
}
}
@@ -6,6 +6,7 @@ import (
"context"
"errors"
"fmt"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
"git.novox.be/novox/mesh-sdk/go/stateobjects"
"reflect"
"slices"
@@ -174,7 +175,6 @@ type Indexer struct {
previews map[string]meshdeliverychecker.PreviewsByRepositoryHeadTrunkValue // by previews key (previews.go)
toolchains map[string]map[string]nodebuildgo.Toolchain // language, then node (toolchains.go)
buildModules map[string]BuildModule // by language: the build modules whose toolchains are taken in (buildmodules.go)
floors sync.Map // the modules whose floor FillFloor reads now (tag_floor.go), one fill each
versionsFilling atomic.Bool // a FillVersions runs (versions_fill.go)
stepsSince map[string]time.Time // by build seat: since when its steps is unanswered (seed_steps.go)
tried map[string]bool // repository, commit and directory a registered module's manifest was tried at (registered.go)
@@ -200,19 +200,16 @@ type Indexer struct {
keepMu sync.Mutex // one catalog graph revision kept at a time (keepEntries)
// Pins is the state pins (pins.go); nil keeps none. PinsPending says how many messages the indexer's durable consumer of the
// events stream has not been given: a revision is removed only while none is pending (§579).
Pins Store
PinsPending func(ctx context.Context) (uint64, error)
namesMu sync.Mutex // packageNames
packageNames map[string]string // the package names read through the forge, by repository, commit, path and artifact (artifacts.go)
catchingUp atomic.Bool // the first start's catch-up runs: its merges keep no revision until it ends (CatchUp)
behindRunning atomic.Bool // a comparison of the taken merges with the source's heads runs (kickBehind, A-463)
behindWaiting atomic.Bool // a source revision came while a catch-up ran (A-463a)
behindAgain atomic.Bool // a source revision came while it ran
catchUpWG sync.WaitGroup // the first start's catch-up in the background (kickCatchUp), for tests to wait on
ownFP string // the fingerprint of the build this program runs, fixed once at the start (FixOwnBuild)
ownFixed bool // FixOwnBuild asked already: it asks once
ownSince time.Time // when the build was asked and not known, with ownWhy
ownWhy string
Pins Store
PinsPending func(ctx context.Context) (uint64, error)
namesMu sync.Mutex // packageNames
packageNames map[string]string // the package names read through the forge, by repository, commit, path and artifact (artifacts.go)
catchingUp atomic.Bool // the first start's catch-up runs: its merges keep no revision until it ends (CatchUp)
catchUpWG sync.WaitGroup // the first start's catch-up in the background (kickCatchUp), for tests to wait on
ownFP string // the fingerprint of the build this program runs, fixed once at the start (FixOwnBuild)
ownFixed bool // FixOwnBuild asked already: it asks once
ownSince time.Time // when the build was asked and not known, with ownWhy
ownWhy string
// OwnBuildState keeps which build this process knows it runs, or since when and why not (FixOwnBuild); nil keeps none.
OwnBuildState Store
// Heard keeps what the indexer heard and compares later (A-80): the time a call is unanswered since (steps-since, call-since),
@@ -234,6 +231,12 @@ type Indexer struct {
Pending Store
// SayTakeIn publishes a take-in event (take-in.pending, take-in.taken; ADR 0633 §5); nil says none.
SayTakeIn func(ctx context.Context, t takein.TakeIn) error
// ValidationAsks puts the validation's run ask (design 47 G §713, A-247: buildrun.Asker, named as the indexer's asker,
// A-248); nil validates nothing, and a revision is taken in as before.
ValidationAsks ValidationAsks
// AskVerb asks a seat verb as the runtime calls it: the build seat's verb output, for a run's answer the seat keeps as an
// object (design 47 G A-254, buildstep.StepEnded.ReadOutput); nil reads none, and such an answer could not validate.
AskVerb buildstep.AskVerb
// Runs is the state runs: the runs on request the indexer holds a candidate for (runcandidates.go, A-174); nil keeps none.
Runs Store
runMu sync.Mutex // one run event at a time (runcandidates.go)
@@ -25,9 +25,6 @@ import (
type CatalogRead struct {
// Modules is every value of the modules state, by module.
Modules func() (map[string]meshcatalog.ModulesByModuleValue, error)
// Module is the one value of the modules state of that module, and whether the state holds it: the verb module reads
// the one module it is asked, never every module (design 47 G A-487).
Module func(name string) (meshcatalog.ModulesByModuleValue, bool, error)
// Manifest is the manifest kept under a digest in the manifests state, as its JSON.
Manifest func(digest string) (string, error)
// Graph is the module graph, by module.
@@ -54,26 +51,17 @@ func (c CatalogRead) CatalogModulesAnswer() (catalogue.CatalogModules, error) {
return catalogue.CatalogModules{Modules: out}, nil
}
// module is one module's value: through Module when set, else out of every module's.
func (c CatalogRead) module(name string) (meshcatalog.ModulesByModuleValue, bool, error) {
if c.Module != nil {
return c.Module(name)
}
mods, err := c.Modules()
m, ok := mods[name]
return m, ok, err
}
// CatalogModuleAnswer is the verb module: one module whole, its manifest included.
func (c CatalogRead) CatalogModuleAnswer(name string) (catalogue.CatalogOne, error) {
name = strings.TrimSpace(name)
if name == "" {
return catalogue.CatalogOne{}, fmt.Errorf("name the module")
}
m, ok, err := c.module(name)
mods, err := c.Modules()
if err != nil {
return catalogue.CatalogOne{}, fmt.Errorf("the indexer's modules cannot be read: %w", err)
}
m, ok := mods[name]
if !ok {
return catalogue.CatalogOne{}, fmt.Errorf("the trunk holds no module %s", name)
}
@@ -45,7 +45,7 @@ func TestACatalogRevisionIsKeptPerWatchedBranch(t *testing.T) {
t.Errorf("said %+v, want main 1, then the escaped branch 1 and 2", said)
}
answer := func(branch string, revision int64) meshcatalog.RevisionAnswer {
a, err := Revision(ctx, ix.Revisions, nil, "indexer", "reader", meshcatalog.RevisionInput{Branch: branch, Revision: revision})
a, err := Revision(ctx, ix.Revisions, "indexer", "reader", meshcatalog.RevisionInput{Branch: branch, Revision: revision})
if err != nil {
t.Fatalf("revision of %q: %v", branch, err)
}
@@ -66,7 +66,7 @@ func TestACatalogRevisionIsKeptPerWatchedBranch(t *testing.T) {
// A-205: `source` answers the branch's revision taken in from that source revision; a candidate's group alone answers its
// newest revision, whatever its heads.
bySource := func(branch string, source int64) (meshcatalog.RevisionAnswer, error) {
return Revision(ctx, ix.Revisions, nil, "indexer", "reader", meshcatalog.RevisionInput{Branch: branch, Source: source})
return Revision(ctx, ix.Revisions, "indexer", "reader", meshcatalog.RevisionInput{Branch: branch, Source: source})
}
if a, err := bySource("main", 5); err != nil || a.Revision.Revision != 1 || a.Entries[0].Commit != "c1" {
t.Errorf("main's source 5 answered %+v (%v)", a.Revision, err)
@@ -89,10 +89,10 @@ func TestACatalogRevisionIsKeptPerWatchedBranch(t *testing.T) {
t.Fatal(err)
}
}
if a, err := Revision(ctx, ix.Revisions, nil, "indexer", "reader", meshcatalog.RevisionInput{Group: "run-ab"}); err != nil || a.Revision.Revision != 4 || a.Revision.Heads != "h2" || a.Entries[0].Commit != "c9" {
if a, err := Revision(ctx, ix.Revisions, "indexer", "reader", meshcatalog.RevisionInput{Group: "run-ab"}); err != nil || a.Revision.Revision != 4 || a.Revision.Heads != "h2" || a.Entries[0].Commit != "c9" {
t.Errorf("a group alone answered %+v (%v), want its newest revision 4", a.Revision, err)
}
if _, err := Revision(ctx, ix.Revisions, nil, "indexer", "reader", meshcatalog.RevisionInput{Group: "run-none"}); err == nil {
if _, err := Revision(ctx, ix.Revisions, "indexer", "reader", meshcatalog.RevisionInput{Group: "run-none"}); err == nil {
t.Error("a candidate never kept was answered")
}
}
@@ -123,7 +123,7 @@ func TestTheNodeEngineReadsTheRevisionAsTheModuleHoldingItsSeat(t *testing.T) {
if err := ix.KeepBranchEntries(ctx, "main", 5, map[string]meshcatalog.CatalogEntry{"mesh-host": engine}); err != nil {
t.Fatal(err)
}
_, err := Revision(ctx, ix.Revisions, nil, "indexer", c.caller, meshcatalog.RevisionInput{Branch: "main"})
_, err := Revision(ctx, ix.Revisions, "indexer", c.caller, meshcatalog.RevisionInput{Branch: "main"})
if (err == nil) != c.want {
t.Errorf("caller %q: err %v, want answered %v", c.caller, err, c.want)
}
@@ -1,72 +0,0 @@
package index
import (
"context"
"encoding/json"
"fmt"
"strings"
"sync/atomic"
"testing"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-sdk/go/views"
)
// listingObjects counts the listings of the bucket: each costs the whole bucket (56 ms at 11 000 objects, more through the runtime).
type listingObjects struct {
*memObjects
listings atomic.Int64
}
func (l *listingObjects) Names(ctx context.Context) ([]string, error) {
l.listings.Add(1)
return l.memObjects.Names(ctx)
}
// A seat verb that answers from the whole view of the trunk (modules, provides, dependents, module) reads the catalog-revisions
// bucket's listing once per call, never once per module: the node-engines ask `module` for every module at once, and 33 asks
// in 5 minutes passed the tool runner's 30 s (2026-11-17).
func TestTrunkViewIsReadOnceForAWholeAnswer(t *testing.T) {
objects := &listingObjects{memObjects: newMemObjects(0)}
const modules = 40
var refs []meshcatalog.EntryRef
for m := 0; m < modules; m++ {
name := fmt.Sprintf("mod%02d", m)
digest := fmt.Sprintf("%064x", m)
e := meshcatalog.CatalogEntry{Module: name, Repository: "o/r", Path: "m/" + name, Commit: "c", Fp: "src2:" + strings.Repeat("a", 64),
Manifest: map[string]any{"name": name}, ManifestDigest: "sha256:" + digest}
b, _ := json.Marshal(e)
if err := objects.Put(context.Background(), EntryObject(digest), b); err != nil {
t.Fatal(err)
}
refs = append(refs, meshcatalog.EntryRef{Module: name, Digest: digest})
}
b, _ := json.Marshal(meshcatalog.CatalogRevision{Kind: "trunk", Revision: 1, Entries: refs})
if err := objects.Put(context.Background(), "trunk.1", b); err != nil {
t.Fatal(err)
}
ts := TrunkStates{Objects: objects, Under: func(string) views.Source { return nil }, Kept: &KeptTrunk{}}
for _, state := range []string{"modules", "graph"} {
objects.listings.Store(0)
values, err := SourceValues(ts.State(state))
if err != nil || len(values) != modules {
t.Fatalf("the %s of the view = %d values (%v), want %d", state, len(values), err, modules)
}
if n := objects.listings.Load(); n > 2 {
t.Errorf("a whole read of the %s listed the bucket %d times, want at most 2 (once per call, not once per module)", state, n)
}
}
if _, err := meshcatalogclient.DecodeModule(mustValue(t, ts)); err != nil {
t.Errorf("a value of the view does not decode: %v", err)
}
}
func mustValue(t *testing.T, ts TrunkStates) []byte {
t.Helper()
v, err := SourceValues(ts.State("modules"))
if err != nil {
t.Fatal(err)
}
return v["mod00"]
}
@@ -182,7 +182,7 @@ func newestOf(ctx context.Context, o Objects, branch string) (meshcatalog.Catalo
name := RevisionObject(meshcatalog.CatalogRevision{Branch: branch, Revision: newest})
data, err := o.Get(ctx, name)
if err != nil {
return meshcatalog.CatalogRevision{Branch: branch, Revision: newest}, false, err // the number that failed, for readNewestOf
return meshcatalog.CatalogRevision{}, false, err
}
var rev meshcatalog.CatalogRevision
if err := json.Unmarshal(data, &rev); err != nil {
@@ -191,31 +191,6 @@ func newestOf(ctx context.Context, o Objects, branch string) (meshcatalog.Catalo
return rev, true, nil
}
// readNewestOf is the newest revision of a branch and its entries whole (design 47 G A-487d). The pin judge removes a revision
// as soon as a newer one is kept, so a revision whose list or entry is removed under the read is read again as the newest,
// while the newest is newer than the one that failed; the same revision failing twice is the read's failure. Progress bounds
// the loop. No revision kept is false.
func readNewestOf(ctx context.Context, o Objects, branch string) (meshcatalog.CatalogRevision, []meshcatalog.CatalogEntry, bool, error) {
var failed int64
for {
newest, any, err := newestOf(ctx, o, branch)
if err == nil && !any {
return newest, nil, false, nil
}
var entries []meshcatalog.CatalogEntry
if err == nil {
_, entries, err = ReadRevision(ctx, o, RevisionObject(newest))
if err == nil {
return newest, entries, true, nil
}
}
if !errors.Is(err, ErrNoObject) || newest.Revision == 0 || newest.Revision <= failed {
return newest, nil, false, err
}
failed = newest.Revision
}
}
// changedEntries are the modules whose entry differs between two revisions' lists, one gone or added included, by name.
func changedEntries(before, after []meshcatalog.EntryRef) []string {
was := map[string]string{}
@@ -463,17 +438,15 @@ func TrunkView(ctx context.Context, o Objects) (modules, nodes Store, at int64,
// BranchView is TrunkView of a watched branch: the view read from the newest revision the bucket keeps of that branch
// (design 47 G §291, A-93), empty and at 0 when it keeps none.
func BranchView(ctx context.Context, o Objects, branch string) (modules, nodes Store, at int64, err error) {
newest, entries, any, err := readNewestOf(ctx, o, branch)
if err != nil || !any {
return newOverlay(nil), newOverlay(nil), 0, err
}
return viewOfEntries(newest.Revision, entries)
}
// viewOfEntries is the view of the entries of one trunk revision, as BranchView reads them.
func viewOfEntries(revision int64, entries []meshcatalog.CatalogEntry) (modules, nodes Store, at int64, err error) {
view, held := newOverlay(nil), newOverlay(nil)
ctx, newest := context.Background(), meshcatalog.CatalogRevision{Revision: revision}
newest, any, err := newestOf(ctx, o, branch)
if err != nil || !any {
return view, held, 0, err
}
_, entries, err := ReadRevision(ctx, o, RevisionObject(newest))
if err != nil {
return nil, nil, 0, err
}
for _, e := range entries {
key, err := meshcatalogclient.ModulesKey(e.Module)
if err != nil {
@@ -523,106 +496,14 @@ type TrunkStates struct {
Kept *KeptTrunk
}
// KeptTrunk is the view of the newest trunk revision, held in memory (design 47 G A-487, A-487b): it reads the newest
// revision from the indexer's own objects catalog-revisions once when the runtime starts (Reread) and again each time it
// hears the seat's event catalog-revision of a newer trunk revision (Heard); a read answers from the view held and never
// lists the bucket. Say, when set, says each read of a revision with its number, its module count and how long it took.
// KeptTrunk is the view of the trunk as last read, at its revision, shared by every read of a TrunkStates; Say, when set,
// says each read of a revision and how long it took.
type KeptTrunk struct {
Say func(format string, a ...any)
read sync.Mutex // one re-read at a time; a read of the view never holds it
mu sync.Mutex
started bool
at int64
mods Store
held Store
rev meshcatalog.CatalogRevision
entries []meshcatalog.CatalogEntry
// manifests are the manifests the revision's entries carry whole, as JSON, by their digest as a module's value names it
// (A-487c: the verbs module and provides answer them from here, never from the manifests state).
manifests map[string]string
}
// Reread reads the newest trunk revision of the bucket and holds its view; a bucket keeping none holds an empty view at 0.
func (k *KeptTrunk) Reread(ctx context.Context, o Objects) error {
k.read.Lock()
defer k.read.Unlock()
return k.reread(ctx, o)
}
func (k *KeptTrunk) reread(ctx context.Context, o Objects) error {
began := time.Now()
fail := func(err error) error {
if k.Say != nil {
k.Say("the view of the trunk is BEHIND, its re-read failed after %s (the view held stays, the next heard catalog-revision reads again): %v", time.Since(began).Round(time.Millisecond), err)
}
return err
}
newest, entries, any, err := readNewestOf(ctx, o, TrunkBranch)
if err != nil {
return fail(err)
}
mods, held, at := Store(newOverlay(nil)), Store(newOverlay(nil)), int64(0)
if any {
if mods, held, at, err = viewOfEntries(newest.Revision, entries); err != nil {
return fail(err)
}
}
manifests := map[string]string{}
for _, e := range entries {
if d := meshcatalog.BareManifestDigest(e.ManifestDigest); d != "" && e.Manifest != nil {
if raw, err := json.Marshal(e.Manifest); err == nil {
manifests[d] = string(raw)
}
}
}
k.mu.Lock()
was := k.at
k.mods, k.held, k.at, k.started, k.rev, k.entries, k.manifests = mods, held, at, true, newest, entries, manifests
k.mu.Unlock()
if k.Say != nil {
n, _ := mods.Keys(ctx)
k.Say("the view of the trunk: read revision %d (was %d), %d modules, in %s", at, was, len(n), time.Since(began).Round(time.Millisecond))
}
return nil
}
// Newest is the newest trunk revision held, with a copy of its entries; false when the bucket kept none at the last read.
func (k *KeptTrunk) Newest(ctx context.Context, o Objects) (meshcatalog.CatalogRevision, []meshcatalog.CatalogEntry, bool, error) {
if _, _, _, err := k.current(ctx, o); err != nil {
return meshcatalog.CatalogRevision{}, nil, false, err
}
k.mu.Lock()
defer k.mu.Unlock()
return k.rev, slices.Clone(k.entries), k.at != 0, nil
}
// Manifest is the manifest of that digest (as a module's value names it) the held revision's entries carry.
func (k *KeptTrunk) Manifest(digest string) (string, error) {
k.mu.Lock()
defer k.mu.Unlock()
raw, ok := k.manifests[digest]
if !ok {
return "", fmt.Errorf("the newest trunk revision's entries carry no manifest %s", digest)
}
return raw, nil
}
// Heard is the seat's event catalog-revision: one of a trunk revision newer than the one held wakes one re-read of the
// objects; any other event is no change. A failed re-read is an error, so the event is offered again.
func (k *KeptTrunk) Heard(ctx context.Context, o Objects, e meshcatalog.CatalogRevisionEvent) error {
if e.Kind != "trunk" || !isTrunkBranch(e.Branch) {
return nil
}
k.read.Lock()
defer k.read.Unlock()
k.mu.Lock()
held := k.started && e.Revision <= k.at
k.mu.Unlock()
if held {
return nil
}
return k.reread(ctx, o)
Say func(format string, a ...any)
mu sync.Mutex
at int64
mods Store
held Store
}
// read is the view's modules and nodes, and the revision they were read at; none kept yet is ErrNoRevision.
@@ -637,37 +518,36 @@ func (t TrunkStates) read(nodes bool) (Store, int64, error) {
return mods, at, err
}
// view is the view of the trunk: the one held while the follower's half is still at its revision, else made from the
// half and held.
// view is the view of the newest trunk revision: the one kept when it is still the newest, else read whole and kept.
func (t TrunkStates) view() (modules, nodes Store, at int64, err error) {
ctx := context.Background()
if t.Kept == nil {
return TrunkView(context.Background(), t.Objects)
return TrunkView(ctx, t.Objects)
}
return t.Kept.current(context.Background(), t.Objects)
}
// current is the view held; before the start's read it reads the newest revision once.
func (k *KeptTrunk) current(ctx context.Context, o Objects) (Store, Store, int64, error) {
k.mu.Lock()
started := k.started
k.mu.Unlock()
if !started {
k.read.Lock()
k.mu.Lock()
started = k.started
k.mu.Unlock()
var err error
if !started {
err = k.reread(ctx, o)
}
k.read.Unlock()
if err != nil {
return nil, nil, 0, err
}
k := t.Kept
newest, any, err := newestOf(ctx, t.Objects, TrunkBranch)
if err != nil || !any {
return newOverlay(nil), newOverlay(nil), 0, err
}
k.mu.Lock()
defer k.mu.Unlock()
return k.mods, k.held, k.at, nil
if k.mods != nil && k.at == newest.Revision {
return k.mods, k.held, k.at, nil
}
started := time.Now()
mods, held, at, err := TrunkView(ctx, t.Objects)
if err != nil {
if k.Say != nil {
k.Say("the view of the trunk: revision %d cannot be read after %s: %v", newest.Revision, time.Since(started).Round(time.Millisecond), err)
}
return nil, nil, 0, err
}
if k.Say != nil {
n, _ := mods.Keys(ctx)
k.Say("the view of the trunk: read revision %d (was %d), %d modules, in %s", at, k.at, len(n), time.Since(started).Round(time.Millisecond))
}
k.mods, k.held, k.at = mods, held, at
return mods, held, at, nil
}
// state is the source of a state by its name: modules and graph from the revision, their marks at its number, every
@@ -708,57 +588,6 @@ func (r revisionSource) Get(key string) (*views.Entry, bool, error) {
return &views.Entry{Value: value, Revision: uint64(at)}, true, nil
}
// Values is every value of the modules (or the module graph) of the newest trunk revision from one read of it: a seat verb
// that answers from the whole view asks which revision is newest once, never once per key (each ask lists the whole
// catalog-revisions bucket: 6 s for the 110 modules of a bucket of 11 000 objects, and 35 s with 12 callers at once).
func (r revisionSource) Values() (map[string][]byte, error) {
s, _, err := r.t.read(r.nodes)
if err != nil {
return nil, err
}
ctx := context.Background()
keys, err := s.Keys(ctx)
if err != nil {
return nil, err
}
out := make(map[string][]byte, len(keys))
for _, k := range keys {
value, _, found, err := s.Get(ctx, k)
if err != nil {
return nil, err
}
if found {
out[k] = value
}
}
return out, nil
}
// SourceValues is every key's value of a source: in one read where the source offers it (the view of the trunk, a
// watched copy of a state), else a get per key.
func SourceValues(src views.Source) (map[string][]byte, error) {
if all, ok := src.(interface {
Values() (map[string][]byte, error)
}); ok {
return all.Values()
}
keys, err := src.Keys()
if err != nil {
return nil, err
}
out := make(map[string][]byte, len(keys))
for _, k := range keys {
e, ok, err := src.Get(k)
if err != nil {
return nil, fmt.Errorf("%s: %w", k, err)
}
if ok && e != nil {
out[k] = e.Value
}
}
return out, nil
}
// Page is the values in key order after a key from one read of the revision, as many as fit maxBytes and at least one
// (views.Pager): a first view's page reads the revision once, never once per key.
func (r revisionSource) Page(after string, maxBytes int, _ time.Duration) ([]views.KeyedEntry, string, error) {
@@ -60,7 +60,7 @@ func (ix *Indexer) FingerprintsAgain(ctx context.Context, module string) TakenIn
ix.freshMu.Unlock()
// The path of a merge's recompute: the modules, then the candidates their fingerprints reach (recomputeWith), then the
// pending merges a changed fingerprint settles (PendingAgain, as after a toolchain report).
err = ix.recomputeWith(ctx, nil, nil, "")
err = ix.recomputeWith(ctx, nil, nil)
ix.freshMu.Lock()
ix.fresh = nil
ix.freshMu.Unlock()
@@ -1,325 +0,0 @@
package index
import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"sync/atomic"
"testing"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-sdk/go/views"
)
// listedObjects counts the listings of its bucket.
type listedObjects struct {
*memObjects
n atomic.Int64
fail atomic.Bool
}
func (l *listedObjects) Names(ctx context.Context) ([]string, error) {
l.n.Add(1)
if l.fail.Load() {
return nil, errors.New("the bucket does not answer")
}
return l.memObjects.Names(ctx)
}
func keptDigest(name string) string {
return "sha256:" + fmt.Sprintf("%064x", len(name))
}
// A-487, A-487b, A-487c: the view of the trunk is read from the bucket once at the start and once for each heard event of a newer
// trunk revision; 100 reads between events list the bucket zero times.
func TestTheKeptTrunkIsReadAtTheStartAndOncePerHeardRevision(t *testing.T) {
objects := &listedObjects{memObjects: newMemObjects(0)}
keep := func(revision int64, modules ...string) {
t.Helper()
rev, entries := meshcatalog.CatalogRevision{Revision: revision, Kind: "trunk"}, map[string][]byte{}
for _, name := range modules {
data, _ := json.Marshal(meshcatalog.CatalogEntry{Module: name, Repository: catalogueRepo, None: "not built", ManifestDigest: keptDigest(name), Manifest: map[string]any{"module": name}})
digest := fmt.Sprintf("%d-%s", revision, name)
entries[digest] = data
rev.Entries = append(rev.Entries, meshcatalog.EntryRef{Module: name, Digest: digest})
}
if err := KeepRevision(context.Background(), objects, rev, entries); err != nil {
t.Fatal(err)
}
}
said := []string{}
kept := &KeptTrunk{Say: func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }}
modules := TrunkStates{Objects: objects, Under: func(string) views.Source { return nil }, Kept: kept}.State("modules")
keep(1, "avahi")
if err := kept.Reread(context.Background(), objects); err != nil { // the start's read
t.Fatal(err)
}
listed := objects.n.Load()
for i := 0; i < 50; i++ {
if _, _, err := modules.Get("avahi"); err != nil {
t.Fatal(err)
}
if _, err := modules.Keys(); err != nil {
t.Fatal(err)
}
}
if got := objects.n.Load() - listed; got != 0 {
t.Errorf("100 reads listed the bucket %d times, want none", got)
}
// a newer revision kept is no reason to read: only the event says it
keep(2, "avahi", "ca-trust")
listed = objects.n.Load()
if keys, _ := modules.Keys(); len(keys) != 1 {
t.Errorf("the view holds %v before the event, want revision 1's one module", keys)
}
for _, e := range []meshcatalog.CatalogRevisionEvent{{Kind: "candidate", Revision: 9}, {Kind: "trunk", Branch: "other", Revision: 9}, {Kind: "trunk", Revision: 1}} {
if err := kept.Heard(context.Background(), objects, e); err != nil {
t.Errorf("%+v: %v", e, err)
}
}
if got := objects.n.Load() - listed; got != 0 {
t.Errorf("events of a candidate, another branch and the revision held listed the bucket %d times, want none", got)
}
if err := kept.Heard(context.Background(), objects, meshcatalog.CatalogRevisionEvent{Kind: "trunk", Branch: "main", Revision: 2}); err != nil {
t.Fatal(err)
}
if got := objects.n.Load() - listed; got != 1 {
t.Errorf("a heard catalog-revision listed the bucket %d times, want exactly one re-read", got)
}
if e, ok, err := modules.Get("ca-trust"); err != nil || !ok || e.Revision != 2 {
t.Errorf("after the event ca-trust = %+v %v (%v), want it at revision 2", e, ok, err)
}
// A-487c: the manifests by digest and the verb revision asked of the trunk's newest answer from the view held.
listed = objects.n.Load()
if raw, err := kept.Manifest(meshcatalog.BareManifestDigest(keptDigest("ca-trust"))); err != nil || raw != `{"module":"ca-trust"}` {
t.Errorf("the held manifest = %q (%v), want the entry's manifest", raw, err)
}
if a, err := Revision(context.Background(), objects, kept, "avahi", "avahi", meshcatalog.RevisionInput{}); err != nil || a.Revision.Revision != 2 || len(a.Entries) != 2 {
t.Errorf("revision of the newest answered %+v (%v), want revision 2 whole", a, err)
}
if got := objects.n.Load() - listed; got != 0 {
t.Errorf("the manifest and the verb revision listed the bucket %d times, want none", got)
}
if len(said) != 2 || !strings.Contains(said[1], "read revision 2 (was 1), 2 modules, in ") {
t.Errorf("said %q, want each read with its revision, module count and time", said)
}
// A-487b: a re-read that fails is said loudly, the view held stays, and the next heard event reads again.
keep(3, "avahi", "ca-trust", "dbus")
objects.fail.Store(true)
if err := kept.Heard(context.Background(), objects, meshcatalog.CatalogRevisionEvent{Kind: "trunk", Revision: 3}); err == nil {
t.Error("a re-read that failed was taken")
}
if last := said[len(said)-1]; !strings.Contains(last, "BEHIND") {
t.Errorf("a failed re-read said %q, want it said loudly", last)
}
if keys, _ := modules.Keys(); len(keys) != 2 {
t.Errorf("the view held after a failed re-read has %v, want revision 2's two modules", keys)
}
objects.fail.Store(false)
if err := kept.Heard(context.Background(), objects, meshcatalog.CatalogRevisionEvent{Kind: "trunk", Revision: 3}); err != nil {
t.Fatal(err)
}
if keys, _ := modules.Keys(); len(keys) != 3 {
t.Errorf("the next heard event left the view at %v, want revision 3's three modules", keys)
}
}
// A-487: the verb module reads the one module it is asked.
func TestTheVerbModuleReadsOneModule(t *testing.T) {
c := CatalogRead{
Modules: func() (map[string]meshcatalog.ModulesByModuleValue, error) {
return nil, errors.New("every module was read")
},
Module: func(name string) (meshcatalog.ModulesByModuleValue, bool, error) {
return meshcatalog.ModulesByModuleValue{Manifest: "d1"}, name == "avahi", nil
},
Manifest: func(string) (string, error) { return `{"module":"avahi"}`, nil },
}
if one, err := c.CatalogModuleAnswer("avahi"); err != nil || one.Module.Module != "avahi" {
t.Errorf("module avahi = %+v (%v)", one, err)
}
if _, err := c.CatalogModuleAnswer("nope"); err == nil || !strings.Contains(err.Error(), "holds no module nope") {
t.Errorf("module nope = %v, want the trunk holds none", err)
}
}
// goneUnderRead is a bucket whose pin judge runs while the first entry of the revision named newest is read: a newer revision
// is kept and the one named is removed, its list and its entries, as the indexer's judge does.
type goneUnderRead struct {
*memObjects
once atomic.Bool
prune func()
on string // when set, only the read of this entry runs the pruning
}
func (g *goneUnderRead) Get(ctx context.Context, name string) ([]byte, error) {
if strings.HasPrefix(name, "entry.") && (g.on == "" || name == g.on) && g.once.CompareAndSwap(false, true) {
g.prune()
}
return g.memObjects.Get(ctx, name)
}
// A revision removed while it is read (take-ins keep one every few seconds, each removing the last) is read again as the newest
// one, never failed: on novox the first view of the catalog half never came while revisions 487 to 493 were kept.
func TestTheKeptTrunkReadsTheNewerRevisionWhenTheOneNamedIsRemovedUnderIt(t *testing.T) {
mem := newMemObjects(0)
keep := func(revision int64) {
data, _ := json.Marshal(meshcatalog.CatalogEntry{Module: "avahi", Repository: catalogueRepo, None: "not built"})
digest := fmt.Sprintf("d%d", revision)
rev := meshcatalog.CatalogRevision{Revision: revision, Kind: "trunk", Entries: []meshcatalog.EntryRef{{Module: "avahi", Digest: digest}}}
if err := KeepRevision(context.Background(), mem, rev, map[string][]byte{digest: data}); err != nil {
t.Fatal(err)
}
}
keep(1)
objects := &goneUnderRead{memObjects: mem}
objects.prune = func() {
keep(2)
_ = mem.Delete(context.Background(), RevisionObject(meshcatalog.CatalogRevision{Revision: 1, Kind: "trunk"}))
_ = mem.Delete(context.Background(), EntryObject("d1"))
}
kept := &KeptTrunk{}
if err := kept.Reread(context.Background(), objects); err != nil {
t.Fatalf("the read failed when the revision named was removed under it: %v", err)
}
if rev, _, ok, err := kept.Newest(context.Background(), objects); err != nil || !ok || rev.Revision != 2 {
t.Errorf("the view holds revision %d (%v, %v), want 2", rev.Revision, ok, err)
}
}
// A-487d: the same revision failing twice is the read's failure, said as it is; a read by number never reads another.
func TestTheKeptTrunkFailsWhenTheSameRevisionIsGoneTwice(t *testing.T) {
mem := newMemObjects(0)
data, _ := json.Marshal(meshcatalog.CatalogEntry{Module: "avahi", Repository: catalogueRepo, None: "not built"})
rev := meshcatalog.CatalogRevision{Revision: 1, Kind: "trunk", Entries: []meshcatalog.EntryRef{{Module: "avahi", Digest: "d1"}}}
if err := KeepRevision(context.Background(), mem, rev, map[string][]byte{"d1": data}); err != nil {
t.Fatal(err)
}
_ = mem.Delete(context.Background(), EntryObject("d1")) // revision 1 stays listed, its entry lost, no newer revision
for name, read := range map[string]func() error{
"kept trunk": func() error { return (&KeptTrunk{}).Reread(context.Background(), mem) },
"TrunkView": func() error { _, _, _, err := TrunkView(context.Background(), mem); return err },
"Outdated": func() error {
_, err := outdatedOfAvahi(mem).Outdated(context.Background(), meshcatalog.OutdatedInput{})
return err
},
} {
if err := read(); !errors.Is(err, ErrNoObject) {
t.Errorf("%s: %v, want the lost entry said (no such object)", name, err)
}
}
}
// A-487d: BranchView and TrunkView name the newest again when the one named is removed under the read.
func TestTrunkViewReadsTheNewerRevisionWhenTheOneNamedIsRemovedUnderIt(t *testing.T) {
mem := newMemObjects(0)
keep := func(revision int64) {
data, _ := json.Marshal(meshcatalog.CatalogEntry{Module: "avahi", Repository: catalogueRepo, None: "not built"})
digest := fmt.Sprintf("d%d", revision)
rev := meshcatalog.CatalogRevision{Revision: revision, Kind: "trunk", Entries: []meshcatalog.EntryRef{{Module: "avahi", Digest: digest}}}
if err := KeepRevision(context.Background(), mem, rev, map[string][]byte{digest: data}); err != nil {
t.Fatal(err)
}
}
keep(1)
objects := &goneUnderRead{memObjects: mem}
objects.prune = func() {
keep(2)
_ = mem.Delete(context.Background(), RevisionObject(meshcatalog.CatalogRevision{Revision: 1, Kind: "trunk"}))
_ = mem.Delete(context.Background(), EntryObject("d1"))
}
if _, _, at, err := TrunkView(context.Background(), objects); err != nil || at != 2 {
t.Errorf("TrunkView read revision %d (%v), want 2", at, err)
}
}
// A-487d: every reader of the newest revision in the indexer program reads the newest again when the revision it named is
// removed under it (ModuleInNewestView, SeedEntryHeld, entryOf, SeedModulesFrom, FillVersions, the tool indexer_catalog_revision
// and the verb revision, trunk's and a watched branch's).
func TestTheNewestRevisionReadersReadAgainWhenTheRevisionIsRemovedUnderThem(t *testing.T) {
ctx := context.Background()
cases := map[string]struct {
branch string
read func(ix *Indexer) (bool, error)
}{
"ModuleInNewestView": {"", func(ix *Indexer) (bool, error) { return ix.ModuleInNewestView(ctx, "avahi") }},
"SeedEntryHeld": {"", func(ix *Indexer) (bool, error) { return ix.SeedEntryHeld(ctx, "avahi", "fp2") }},
"entryOf": {"", func(ix *Indexer) (bool, error) {
e, err := ix.entryOf(ctx, "avahi")
return e.Fp == "fp2", err
}},
"SeedModulesFrom": {"", func(ix *Indexer) (bool, error) { _, err := ix.SeedModulesFrom(ctx, nil, nil); return err == nil, err }},
"FillVersions": {"", func(ix *Indexer) (bool, error) {
ix.Versions, ix.Heard = MemoryStore(), MemoryStore()
err := ix.FillVersions(ctx, func(context.Context, string, string) (BuilderVersionsAnswer, error) {
return BuilderVersionsAnswer{}, nil
})
return err == nil, err
}},
"indexer_catalog_revision": {"", func(ix *Indexer) (bool, error) {
var tool map[string]any
for _, tl := range CatalogRevisionTools(ix.Revisions, nil, time.Now, nil) {
if tl.Name == CatalogRevisionTool {
got, err := tl.Run(map[string]any{})
if err != nil {
return false, err
}
tool = got.(map[string]any)
}
}
return tool != nil && tool["name"] == "trunk.2", nil
}},
"Outdated": {"", func(ix *Indexer) (bool, error) {
a, err := outdatedOfAvahi(ix.Revisions).Outdated(ctx, meshcatalog.OutdatedInput{})
return len(a.Behind) == 1 && a.Behind[0].Kind == 5 && strings.Contains(a.Behind[0].Newest, "fp2"), err
}},
"Revision": {"", func(ix *Indexer) (bool, error) {
a, err := Revision(ctx, ix.Revisions, nil, "avahi", "avahi", meshcatalog.RevisionInput{})
return a.Revision != nil && a.Revision.Revision == 2, err
}},
"Revision of a branch": {"dev", func(ix *Indexer) (bool, error) {
a, err := Revision(ctx, ix.Revisions, nil, "avahi", "avahi", meshcatalog.RevisionInput{Branch: "dev"})
return a.Revision != nil && a.Revision.Revision == 2 && len(a.Entries) == 1 && a.Entries[0].Fp == "fp2", err
}},
}
for name, c := range cases {
t.Run(name, func(t *testing.T) {
mem := newMemObjects(0)
keep := func(branch string, revision int64) {
data, _ := json.Marshal(meshcatalog.CatalogEntry{Module: "avahi", Repository: catalogueRepo, None: "not built", Fp: fmt.Sprintf("fp%d", revision)})
digest := fmt.Sprintf("%s-d%d", branch, revision)
rev := meshcatalog.CatalogRevision{Branch: branch, Revision: revision, Kind: "trunk", Entries: []meshcatalog.EntryRef{{Module: "avahi", Digest: digest}}}
if err := KeepRevision(ctx, mem, rev, map[string][]byte{digest: data}); err != nil {
t.Fatal(err)
}
}
keep("", 1)
if c.branch != "" { // the trunk stays at revision 1; the branch's revision 1 is removed under the read
keep("", 2)
keep(c.branch, 1)
}
objects := &goneUnderRead{memObjects: mem}
pruned := c.branch
objects.on = EntryObject(fmt.Sprintf("%s-d1", pruned))
objects.prune = func() {
keep(pruned, 2)
_ = mem.Delete(ctx, RevisionObject(meshcatalog.CatalogRevision{Branch: pruned, Revision: 1, Kind: "trunk"}))
_ = mem.Delete(ctx, objects.on)
}
if ok, err := c.read(&Indexer{Revisions: objects}); err != nil || !ok {
t.Errorf("answered %v (%v), want revision 2's entry", ok, err)
}
})
}
}
// outdatedOfAvahi is an indexer whose state versions holds avahi at a kind-5 edge, so Outdated reads the trunk's entry of avahi.
func outdatedOfAvahi(o Objects) *Indexer {
ix := &Indexer{Versions: MemoryStore(), Heard: MemoryStore(), Revisions: o}
data, _ := json.Marshal(meshcatalog.VersionEntry{V: 1, Module: "avahi", Version: "0.1.1", Edges: []meshcatalog.VersionEdge{{Kind: 5, Part: "actor:program"}}})
_ = ix.Versions.Create(context.Background(), meshcatalog.VersionsByModuleVersionKey("avahi", "0.1.1"), data)
return ix
}

Some files were not shown because too many files have changed in this diff Show More