novox/mesh-delivery
10
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch 'feat/47-G-fix11-b2f' into feat/47-G-join11

Browse Source
This commit is contained in:
jschoubben
2026-11-12 10:32:18 +01:00
parent d89a6bacc6 86af4c29c4
commit c28d12330e
77 changed files with 3935 additions and 1696 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/buildended.go
+51 -19
View File
@@ -1,26 +1,25 @@
package main
import (
"context"
"encoding/json"
"fmt"
"sync"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/process"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"mesh-delivery-assigner/internal/assign"
"mesh-delivery-assigner/internal/assignment"
)
// What a want names of a build is heard, never read from the builder's state (design 47 G §379, §526, A-94, A-96; ADR 0596 §2): the
// builder's event `build.ended` (a done build's parts by digest, its keeping node and its branch) and the catalog's
// `version-released` (a released version and each part's digest, with the source revision it was built from), each kept as heard.
// buildEndedSubjects is the builder's event build.ended on its own stream, as the manifest's consumes names it
// (`mesh-delivery-builder.build.ended`), each module and fingerprint after it.
var buildEndedSubjects = process.ModuleStreamSubject(builds.BuilderModule, builds.BuildEndedEvent+".>")
// builtEnds are the done builds' ends heard, by module and fingerprint (hearBuildEnded).
type builtEnds struct {
mu sync.Mutex
@@ -38,6 +37,9 @@ func (b *builtEnds) put(data []byte) error {
if e.Module == "" || e.Fingerprint == "" {
return fmt.Errorf("a build's end naming no module or fingerprint: %s", data)
}
if e.Outcome == "" { // an end said before A-162 is a done build's
e.Outcome = builds.BuildDone
}
b.mu.Lock()
defer b.mu.Unlock()
if b.ends == nil {
@@ -47,6 +49,14 @@ func (b *builtEnds) put(data []byte) error {
return nil
}
// endOf is the end heard of module's build at fp, done or failed (A-162): none while the build has not ended.
func (b *builtEnds) endOf(module, fp string) (builds.BuildEnded, bool) {
b.mu.Lock()
defer b.mu.Unlock()
e, ok := b.ends[b.key(module, fp)]
return e, ok
}
// partsOf is the parts of module's build at fp as its end names them (each artifact's digest by `<module>:<artifact>`), with the
// node that keeps it, for a kept build or a build of another watched branch; false for a build of the trunk not kept, whose
// parts the version's release says (releasedVersions).
@@ -68,6 +78,8 @@ func (b *builtEnds) partsOf(module, fp string) (assign.KeptBuild, bool) {
type releasedVersions struct {
mu sync.Mutex
seen map[string]assign.KeptBuild
// newest is, by module, the last version-released heard of it (delivered.go).
newest map[string]releaseHeard
}
func (r *releasedVersions) key(module string, source int64) string {
@@ -93,6 +105,14 @@ func (r *releasedVersions) put(data []byte) error {
r.seen = map[string]assign.KeptBuild{}
}
r.seen[r.key(e.Module, e.Source)] = assign.KeptBuild{Version: e.Version, Parts: parts}
if r.newest == nil {
r.newest = map[string]releaseHeard{}
}
h := releaseHeard{Version: e.Version, Parts: parts, Source: e.Source}
if e.At != nil {
h.At = *e.At
}
r.newest[e.Module] = h // the stream's order is the order of release
return nil
}
@@ -104,21 +124,33 @@ func (r *releasedVersions) of(module string, source int64) (assign.KeptBuild, bo
return k, ok
}
// hearBuildsAndReleases registers the hub's takers of build.ended and version-released, each waking the pass; before the hub starts.
func hearBuildsAndReleases(hub *process.EventHub, ends *builtEnds, released *releasedVersions, wake func(), logf func(string, ...any)) error {
taker := func(put func([]byte) error) func(jetstream.Msg) {
return func(m jetstream.Msg) {
if err := put(m.Data()); err != nil {
logf("%s is not kept: %v", m.Subject(), err)
}
if err := m.Ack(); err != nil {
logf("acknowledging %s: %v", m.Subject(), err)
}
wake()
// buildsStreamName is the builder's stream of build ends as the bus names it: kept last per subject (one end per module and
// fingerprint, its manifest's `last-per-subject`, design 47 G A-181, §703).
var buildsStreamName = catalogue.StreamBusName(builds.BuildsStream)
// hearBuilds reads the builder's stream of build ends whole at the start and then follows it (design 47 G A-181, §556): what was
// built, and a failed build's error, survive the assigner's restart, and it reads none of the builder's buckets. The read ends before
// it returns, so the first pass knows every build; a stream it cannot hear is said, and no want names a build until the assigner
// starts again.
func hearBuilds(ctx context.Context, js jetstream.JetStream, ends *builtEnds, wake func(), findings assign.Findings, logf func(string, ...any)) {
acct := process.Account{Node: thisNode, Module: assignment.Module}
nh := notHeardWriter(findings, logf)
cc, err := acct.HearStream(ctx, js, buildsStreamName, func(m jetstream.Msg) {
if err := ends.put(m.Data()); err != nil {
logf("%s is not kept: %v", m.Subject(), err)
}
if err := m.Ack(); err != nil {
logf("acknowledging %s: %v", m.Subject(), err)
}
wake()
}, logf)
if err != nil {
nh.NotHeard(buildsStreamName, "what the builder built is not heard until the assigner starts again, so no want names a build's parts or its keeping node and no put goes on", err)
return
}
if err := hub.Handle(buildEndedSubjects, taker(ends.put)); err != nil {
return err
}
return hub.Handle(meshcatalog.SubjectVersionReleased, taker(released.put))
nh.Heard(buildsStreamName)
go func() {
<-ctx.Done()
cc.Stop()
}()
}
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/buildended_test.go
+22
View File
@@ -86,3 +86,25 @@ func TestATrunkEntryTakesItsVersionFromTheReleaseOfTheKeptRevisionsSource(t *tes
t.Error("no revision kept names a release")
}
}
// A-162: the assigner knows a failed build from its `build.ended`: the error whole; a done end or none heard is not failed.
func TestTheStandingOfABuildIsHeardFromItsEnd(t *testing.T) {
w := newWorld()
say := func(e builds.BuildEnded) {
raw, _ := json.Marshal(e)
if err := w.ends.put(raw); err != nil {
t.Fatal(err)
}
}
if failed, _ := w.buildStanding("web", "fp1"); failed {
t.Error("a build nothing was heard of failed")
}
say(builds.BuildEnded{Module: "web", Fingerprint: "fp1", Outcome: builds.BuildFailed, Error: "transport fault: connection"})
if failed, why := w.buildStanding("web", "fp1"); !failed || why != "transport fault: connection" {
t.Errorf("failed end: failed %v why %q", failed, why)
}
say(builds.BuildEnded{Module: "web", Fingerprint: "fp1", Outcome: builds.BuildDone})
if failed, _ := w.buildStanding("web", "fp1"); failed {
t.Error("a later done end leaves the build failed")
}
}
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/buildends_helper_test.go
+24
View File
@@ -0,0 +1,24 @@
package main
import (
"encoding/json"
"testing"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
)
// endDone is the builder's `build.ended` of a done build of module at fp heard by w, naming the digest of its manifest (design 47 G
// A-181: what was built is heard, never read from the builder's bucket).
func endDone(t testing.TB, w *world, module, fp, manifest string) {
t.Helper()
data, err := json.Marshal(builds.BuildEnded{Module: module, Fingerprint: fp, Outcome: builds.BuildDone, Manifest: manifest})
if err != nil {
t.Fatal(err)
}
if err := w.ends.put(data); err != nil {
t.Fatal(err)
}
}
// trunkBranch is the first watched branch the tests give the world: the code names none (design 47 G §507).
const trunkBranch = "main"
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/candidates.go
+113
View File
@@ -0,0 +1,113 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"maps"
"strings"
mcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-sdk/go/buscalls"
"mesh-delivery-assigner/internal/assigngraph"
)
// A run's candidate assignment graph (design 47 G §189, §190, A-201, A-235): the candidate `run-<id>`'s catalog graph revision,
// handed by meshcatalogclient's candidates hook, is read whole through the catalog seat's verb revision; the assignment graph is
// computed from it in place of its branch's half, beside every other followed branch's, and kept under candidate.<group>.<heads>;
// the runner's group-ended of the run drops it. It writes no want and says no event; the gate's heads-replaced drops nothing.
// runGroupPrefix is a run on request's candidate group before its id, `run-<id>` (A-160).
const runGroupPrefix = "run-"
// readCandidateHalf reads a candidate's catalog graph revision whole through the seat's verb revision, page by page, with the
// candidate's group and heads and the revision its event names.
func readCandidateHalf(call mcc.Call, ev meshcatalog.CatalogRevisionEvent) (mcc.Half, error) {
half := mcc.Half{Entries: map[string]meshcatalog.CatalogEntry{}}
after := ""
for {
a, err := askCandidatePage(call, meshcatalog.RevisionInput{Branch: ev.Branch, Group: ev.Group, Heads: ev.Heads, Revision: ev.Revision, After: after})
if err != nil {
return half, err
}
if a.Revision == nil || a.Revision.Revision != ev.Revision || a.Revision.Group != ev.Group {
return half, fmt.Errorf("%s answered %+v for the candidate %s's revision %d", mcc.RevisionVerb, a.Revision, ev.Group, ev.Revision)
}
half.Revision = *a.Revision
for _, e := range a.Entries {
half.Entries[e.Module] = e
}
if a.Next == "" {
return half, nil
}
after = a.Next
}
}
// keepCandidate computes and keeps the candidate's assignment graph of a run's catalog-revision; another candidate's (a try's, a
// group's) is passed over.
func (p *passer) keepCandidate(ctx context.Context, call mcc.Call, ev meshcatalog.CatalogRevisionEvent) error {
if !strings.HasPrefix(ev.Group, runGroupPrefix) {
return nil
}
g := p.graph
if g == nil || g.objects == nil || g.halves == nil {
return errors.New("the assigner keeps no assignment graph, so no candidate's is kept")
}
half, err := readCandidateHalf(call, ev)
if err != nil {
return fmt.Errorf("the candidate %s's revision %d is not read, so its assignment graph is not kept: %w", ev.Group, ev.Revision, err)
}
in := p.graphInputs()
in.Halves = maps.Clone(in.Halves)
in.Halves[half.Revision.Branch] = half
c := assigngraph.Compute(in)
for _, u := range c.Uncomputed {
p.logf("the candidate %s's assignment graph leaves out %s", ev.Group, u)
}
rev, fresh, err := assigngraph.KeepCandidate(ctx, g.objects, c, ev.Group, ev.Heads, meshcatalog.EventCatalogRevision, p.clock())
if err != nil {
return fmt.Errorf("the candidate %s's assignment graph is not kept: %w", ev.Group, err)
}
if fresh {
p.logf("the candidate %s's assignment graph revision %d is kept, from its catalog graph revision %d", ev.Group, rev.Revision, ev.Revision)
}
return nil
}
// dropCandidate removes a run's candidate assignment graph on the runner's group-ended of that run (A-235).
func (p *passer) dropCandidate(ctx context.Context, group string) error {
g := p.graph
if g == nil || g.objects == nil {
return nil
}
o, ok := g.objects.(assigngraph.Remover)
if !ok {
return fmt.Errorf("the objects %s remove nothing, so the candidate %s is kept", assigngraph.RevisionsObjects, group)
}
n, err := assigngraph.DropCandidate(ctx, o, group)
if err == nil && n > 0 {
p.logf("the candidate %s's assignment graph is dropped: its run ended", group)
}
return err
}
// askCandidatePage is one page of a candidate's revision as the seat's verb revision answers it.
func askCandidatePage(call mcc.Call, in meshcatalog.RevisionInput) (meshcatalog.RevisionAnswer, error) {
raw, err := call(mcc.RevisionVerb, in)
if err != nil {
return meshcatalog.RevisionAnswer{}, err
}
answer, said, ok := buscalls.Unwrap(raw)
if !ok {
return meshcatalog.RevisionAnswer{}, fmt.Errorf("%s answered: %s", mcc.RevisionVerb, said)
}
var a meshcatalog.RevisionAnswer
if err := json.Unmarshal(answer, &a); err != nil {
return a, fmt.Errorf("%s answered no revision: %w", mcc.RevisionVerb, err)
}
return a, nil
}
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/delivered.go
+451
View File
@@ -0,0 +1,451 @@
package main
import (
"context"
"encoding/json"
"fmt"
"slices"
"sort"
"strings"
"sync"
"time"
"github.com/nats-io/nats.go/jetstream"
wd "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshwatchdogs"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-sdk/go/keys"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-assigner/internal/assign"
"mesh-delivery-assigner/internal/assignment"
"mesh-delivery-assigner/internal/buckets"
)
// A delivery is waited for mesh-wide (design 47 G §376, §389, A-189, A-246). At its start the assigner reads whole, from their first
// message, the streams of the catalog seat's version-released and of its own version-delivered (mesh-sdk process.FromFirst, A-233),
// and the watchman's machine-away and machine-back, never its state; then it hears them. A released version of a module is said
// version-delivered on the seat mesh-wants once, when every present machine placing it on the branch it is released from runs, healthy,
// the want the assigner wrote naming that version; a version no present machine places is not delivered. A dependent whose want
// changes fingerprint waits, by an edge of kind 2, 4 or 6, for its dependency's newest released version not yet said delivered, on
// the branch it is released from; on another branch no version is released and nothing waits. The wait is said wait-held, step
// delivery, bound 60 minutes from the version's release, kept in the state delivery-waits under the dependent's name, and said
// wait-ended when the version is delivered (ran) or released by hand (assigner_release, released).
// deliveryWaitBound is a dependent's delivery wait's bound, from its dependency's version-released.
const deliveryWaitBound = 60 * time.Minute
// Wait-ended's how of a delivery wait.
const (
deliveryWaitRan = "ran"
deliveryWaitReleased = "released"
)
// releaseHeard is one version-released heard: the version, each part's digest, the source revision and when it was said.
type releaseHeard struct {
Version string
Parts map[string]string
Source int64
At time.Time
}
// newestOf is the newest version of module released, the last heard of it in the stream's order.
func (r *releasedVersions) newestOf(module string) (releaseHeard, bool) {
r.mu.Lock()
defer r.mu.Unlock()
h, ok := r.newest[module]
return h, ok
}
// deliveredVersions are the version-delivered said, the assigner's own, by module and version: each is said once.
type deliveredVersions struct {
mu sync.Mutex
said map[string]bool
}
func (d *deliveredVersions) key(module, version string) string { return module + "@" + version }
// put takes one version-delivered in.
func (d *deliveredVersions) put(data []byte) error {
var e meshwants.VersionDeliveredEvent
if err := json.Unmarshal(data, &e); err != nil {
return fmt.Errorf("a version-delivered that is none: %w", err)
}
if e.Module == "" || e.Version == "" {
return fmt.Errorf("a version-delivered naming no module or version: %s", data)
}
d.mark(e.Module, e.Version)
return nil
}
func (d *deliveredVersions) mark(module, version string) {
d.mu.Lock()
defer d.mu.Unlock()
if d.said == nil {
d.said = map[string]bool{}
}
d.said[d.key(module, version)] = true
}
// has says whether version-delivered of module's version was said.
func (d *deliveredVersions) has(module, version string) bool {
d.mu.Lock()
defer d.mu.Unlock()
return d.said[d.key(module, version)]
}
// presenceOf reads one machine-away or machine-back: the machine and when it was said.
func presenceOf(data []byte) (string, time.Time, error) {
var e wd.MachineAwayEvent // machine-back has the same fields
if err := json.Unmarshal(data, &e); err != nil {
return "", time.Time{}, err
}
if e.Machine == "" {
return "", time.Time{}, fmt.Errorf("it names no machine: %s", data)
}
return e.Machine, e.At, nil
}
// hearDelivery reads whole, then follows, the four streams a delivery is judged from: version-released, version-delivered,
// machine-away and machine-back. A stream not heard is said as the assigner's finding, and what it carries is not known until
// the assigner starts again.
func hearDelivery(ctx context.Context, js jetstream.JetStream, w *world, wake func(), findings assign.Findings, logf func(string, ...any)) {
acct := process.Account{Node: thisNode, Module: assignment.Module}
nh := notHeardWriter(findings, logf)
for _, s := range []struct {
stream, words string
take func([]byte) error
}{
{meshcatalog.StreamVersionReleased, "no want names a released version and no delivery is judged", w.released.put},
{meshwants.StreamVersionDelivered, "a version said delivered before is said again once it is judged", w.delivered.put},
{wd.StreamMachineAway, "no machine is known away, so a delivery waits for a machine that is away", func(data []byte) error {
m, at, err := presenceOf(data)
if err == nil {
w.heardPresence(m, true, at)
}
return err
}},
{wd.StreamMachineBack, "a machine once away is not known back", func(data []byte) error {
m, at, err := presenceOf(data)
if err == nil {
w.heardPresence(m, false, at)
}
return err
}},
} {
take := s.take
cc, err := acct.HearSeatEvents(ctx, js, s.stream, func(m jetstream.Msg) {
if err := take(m.Data()); err != nil {
logf("%s is not kept: %v", m.Subject(), err)
}
if err := m.Ack(); err != nil {
logf("acknowledging %s: %v", m.Subject(), err)
}
wake()
}, logf, process.FromFirst())
if err != nil {
nh.NotHeard(s.stream, s.words+" until the assigner starts again", err)
continue
}
nh.Heard(s.stream)
go func() {
<-ctx.Done()
cc.Stop()
}()
}
}
// releaseBranch is the branch module's versions are released from: the first watched branch of its registration (§371).
func (p *passer) releaseBranch(module string) string {
if p.ends.following() {
return p.ends.branchOf(module, "")
}
return p.w.trunkName()
}
// placing is every machine placing module on the branch its versions are released from, sorted.
func (p *passer) placing(module string) []string {
branch := p.releaseBranch(module)
var out []string
for node, placed := range p.w.placedBranches()[module] {
if p.ends.following() {
placed = p.ends.branchOf(module, placed)
} else {
placed = p.w.trunkName() // the trunk-only form: every placement runs the trunk's
}
if placed == branch {
out = append(out, node)
}
}
sort.Strings(out)
return out
}
// deliveryStanding is where module's version stands: the present placing machines not yet running it healthy as its want names
// it, and the placing machines away.
func (p *passer) deliveryStanding(module, version string) (notRunning, away []string, placing int) {
gone := p.w.awayNow()
n := assign.Nodes{Running: p.w.runningNow()}
wanted := p.w.wantsOf()[module]
nodes := p.placing(module)
for _, node := range nodes {
if gone[node] {
away = append(away, node)
continue
}
want, ok := wanted[node]
if !ok || want.Removed || want.Version != version || !n.RunsHealthy(node, module, want.Fp) {
notRunning = append(notRunning, node)
}
}
return notRunning, away, len(nodes)
}
// judgeDelivered says version-delivered of each module's newest released version not said yet, once every present machine placing
// it runs it healthy; judged again at each pass, which each running, machine-away, machine-back and placement change wakes.
func (p *passer) judgeDelivered() {
for _, module := range p.releasedModules() {
rel, _ := p.w.released.newestOf(module)
if p.w.delivered.has(module, rel.Version) {
continue
}
notRunning, away, placing := p.deliveryStanding(module, rel.Version)
if placing == 0 || len(away) == placing || len(notRunning) > 0 {
continue
}
if err := p.sayDelivered(module, rel, away); err != nil {
p.logf("version-delivered of %s %s is not said, and is judged again at the next pass: %v", module, rel.Version, err)
}
}
}
// sayDelivered says version-delivered of module's release rel, with the machines away, and keeps it said.
func (p *passer) sayDelivered(module string, rel releaseHeard, away []string) error {
at := p.clock().UTC()
body := meshwants.VersionDeliveredEvent{V: keys.Version, Module: module, Version: rel.Version, Parts: rel.Parts, Away: away, At: &at}
if err := p.wantsSay(meshwants.EventVersionDelivered, body, fmt.Sprintf("%s.%s.%s.%s", meshwants.EventVersionDelivered, module, rel.Version, at.Format(time.RFC3339Nano))); err != nil {
return err
}
p.w.delivered.mark(module, rel.Version)
return nil
}
// releasedModules is every module a version-released was heard of, sorted.
func (p *passer) releasedModules() []string {
r := &p.w.released
r.mu.Lock()
defer r.mu.Unlock()
out := make([]string, 0, len(r.newest))
for m := range r.newest {
out = append(out, m)
}
sort.Strings(out)
return out
}
// deliveryWait is what a dependent waits for: its dependency, the version and the machines not yet running it.
type deliveryWait struct {
dependency string
rel releaseHeard
notRunning []string
away []string
}
// waitsForDelivery says whether module, whose want on its nodes changes to fp on branch, waits for a dependency's delivery: by
// each edge of kind 2, 4 or 6 into it in branch's catalog half, the dependency's newest released version on branch not yet said
// delivered. A want that changes no fingerprint waits for nothing; on a branch no version is released from, nothing waits.
func (p *passer) waitsForDelivery(module, fp, branch string, nodes []string) (deliveryWait, bool) {
wanted := p.w.wantsOf()[module]
changes := false
for _, node := range nodes {
if w, ok := wanted[node]; !ok || w.Removed || w.Fp != fp {
changes = true
}
}
if !changes || fp == "" {
return deliveryWait{}, false
}
var deps []string
for _, e := range p.w.edgesOf(branch, module) {
if e.Kind != 2 && e.Kind != 4 && e.Kind != 6 {
continue
}
dep, _, _ := strings.Cut(e.From, ":")
if dep != "" && dep != module && !slices.Contains(deps, dep) {
deps = append(deps, dep)
}
}
sort.Strings(deps)
for _, dep := range deps {
if p.releaseBranch(dep) != branch {
continue
}
rel, ok := p.w.released.newestOf(dep)
if !ok || p.w.delivered.has(dep, rel.Version) {
continue
}
notRunning, away, _ := p.deliveryStanding(dep, rel.Version)
return deliveryWait{dependency: dep, rel: rel, notRunning: notRunning, away: away}, true
}
return deliveryWait{}, false
}
// words is a delivery wait in plain words, as assigner_why_not and the status view say it.
func (d deliveryWait) words() string {
if len(d.notRunning) == 0 && len(d.away) > 0 {
return fmt.Sprintf("waiting for %s to be back", strings.Join(d.away, ", "))
}
return fmt.Sprintf("waiting for delivery of %s %s on %s", d.dependency, d.rel.Version, strings.Join(d.notRunning, ", "))
}
// deliveryWaitBody is the wait-held of module's delivery wait d.
func deliveryWaitBody(module, fp string, d deliveryWait, since time.Time) meshwants.Wait {
from := d.rel.At.UTC()
if from.IsZero() {
from = since.UTC()
}
return meshwants.Wait{Step: meshwants.WaitStepDelivery, Owner: meshwants.Seat, Module: module, Version: fp, Part: d.dependency,
On: d.dependency + "@" + d.rel.Version, Machines: d.notRunning, Since: since.UTC(), ClockFrom: &from, Bound: deliveryWaitBound.String(),
Words: d.words()}
}
// deliveryWaiting is one delivery wait the state delivery-waits keeps, by its dependent's name: the wait-held said, and a release
// by hand written by assigner_release.
type deliveryWaiting struct {
meshwants.Wait
// Released is set by assigner_release: the wait ends released and the dependent's want is written.
Released bool `json:"released,omitempty"`
}
// saidDeliveryWaits is every delivery wait the state delivery-waits holds, by module; a value that does not decode is said and left.
func (p *passer) saidDeliveryWaits() map[string]deliveryWaiting {
out := map[string]deliveryWaiting{}
for key, value := range p.w.read(buckets.DeliveryWaits) {
var w deliveryWaiting
if err := json.Unmarshal(value, &w); err != nil || w.Module == "" {
p.logf("%s %s holds no delivery wait, left: %v", buckets.DeliveryWaits, key, err)
continue
}
out[w.Module] = w
}
return out
}
// holdForDelivery says whether module's want, changing to fp on branch, is held for a dependency's delivery this pass: it says
// wait-held once and keeps it; a wait released by hand holds nothing; a wait that no longer holds is ended ran.
func (p *passer) holdForDelivery(ctx context.Context, module, fp, branch string, nodes []string) bool {
if p.kvs[buckets.DeliveryWaits] == nil {
return false
}
key := keys.EscapeName(module)
said, was := p.saidDeliveryWaits()[module]
d, waits := p.waitsForDelivery(module, fp, branch, nodes)
switch {
case was && said.Released:
p.endDeliveryWait(ctx, key, said, deliveryWaitReleased) // the want is written now, so it changes nothing at the next pass
return false
case !waits:
if was {
p.endDeliveryWait(ctx, key, said, deliveryWaitRan)
}
return false
}
body := deliveryWaitBody(module, fp, d, p.clock())
if was && said.On == body.On && said.Version == fp && slices.Equal(said.Machines, body.Machines) {
return true
}
if was {
body.Since = said.Since
}
id := fmt.Sprintf("%s.delivery.%s.%s.%s", key, fp, body.On, strings.Join(body.Machines, ","))
if err := p.wantsSay(meshwants.EventWaitHeld, body, id); err != nil {
p.logf("the delivery wait of %s is not said, and is said at the next pass: %v", module, err)
return true
}
kept := deliveryWaiting{Wait: body}
p.put(ctx, buckets.DeliveryWaits, key, func() ([]byte, error) { return json.Marshal(kept) })
return true
}
// endDeliveryWait says wait-ended of a delivery wait with how and removes it from the state.
func (p *passer) endDeliveryWait(ctx context.Context, key string, w deliveryWaiting, how string) {
body := w.Wait
body.How = how
if err := p.wantsSay(meshwants.EventWaitEnded, body, key+".delivery."+body.Version+"."+how); err != nil {
p.logf("the end of the delivery wait of %s is not said, and is said at the next pass: %v", w.Module, err)
return
}
p.remove(ctx, buckets.DeliveryWaits, key)
}
// againAsk is one ask of assigner_delivered_again, kept in the state delivered-again under its module (A-250): the version to say
// delivered again, who asked and why, and, while it does not hold, why not.
type againAsk struct {
Module string `json:"module"`
Version string `json:"version"`
By string `json:"by"`
Why string `json:"why"`
Waits string `json:"waits,omitempty"`
}
// releaseOf is the release heard of module's version.
func (r *releasedVersions) releaseOf(module, version string) (releaseHeard, bool) {
r.mu.Lock()
defer r.mu.Unlock()
if h, ok := r.newest[module]; ok && h.Version == version {
return h, true
}
for key, k := range r.seen {
if strings.HasPrefix(key, module+"@") && k.Version == version {
return releaseHeard{Version: version, Parts: k.Parts}, true
}
}
return releaseHeard{}, false
}
// deliveredAgain takes each ask of assigner_delivered_again, heard at once through the watch of the state delivered-again: it says
// version-delivered again when every present machine placing the module runs that version healthy and removes the ask, else keeps
// it with why it does not hold, which assigner_waits reads.
func (p *passer) deliveredAgain(ctx context.Context) {
if p.kvs[buckets.DeliveredAgain] == nil {
return
}
for key, value := range p.w.read(buckets.DeliveredAgain) {
var a againAsk
if err := json.Unmarshal(value, &a); err != nil || a.Version == "" {
p.logf("%s %s holds no ask, left: %v", buckets.DeliveredAgain, key, err)
continue
}
module := a.Module
if module == "" {
module = key
}
waits := ""
rel, ok := p.w.released.releaseOf(module, a.Version)
notRunning, away, placing := p.deliveryStanding(module, a.Version)
switch {
case !ok:
waits = fmt.Sprintf("no version-released of %s %s is heard", module, a.Version)
case placing == 0:
waits = fmt.Sprintf("no machine places %s", module)
case len(away) == placing:
waits = fmt.Sprintf("every machine placing %s is away: %s", module, strings.Join(away, ", "))
case len(notRunning) > 0:
waits = fmt.Sprintf("%s %s does not run healthy on %s", module, a.Version, strings.Join(notRunning, ", "))
}
if waits == "" {
if err := p.sayDelivered(module, rel, away); err != nil {
p.logf("version-delivered of %s %s asked again is not said, and is asked at the next pass: %v", module, a.Version, err)
continue
}
p.logf("version-delivered of %s %s said again, asked by %s: %s", module, a.Version, a.By, a.Why)
p.remove(ctx, buckets.DeliveredAgain, key)
continue
}
if a.Waits != waits {
a.Waits = waits
p.put(ctx, buckets.DeliveredAgain, key, func() ([]byte, error) { return json.Marshal(a) })
}
}
}
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/delivered_test.go
+188
View File
@@ -0,0 +1,188 @@
package main
import (
"context"
"encoding/json"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"mesh-delivery-assigner/internal/buckets"
)
// Design 47 G §376, §389, §498, A-189, A-246: a released version is said version-delivered once, when every present machine placing
// it runs healthy the want naming it, a machine away named and not waited for, none when every placing machine is away; a dependent
// whose want changes fingerprint waits by an edge of kind 2, 4 or 6 for its dependency's newest released version not yet said
// delivered, said wait-held (step delivery, 60 minutes from the release) once and kept in delivery-waits, and wait-ended ran once
// the version is delivered; a kind-1 edge, or a dependency with no version released, waits for nothing; one released by hand ends
// released and its want is written.
func TestADependentWaitsForItsDependencysDelivery(t *testing.T) {
ctx := context.Background()
at := time.Date(2026, 11, 12, 9, 0, 0, 0, time.UTC)
waits := &memKV{values: map[string][]byte{}}
again := &memKV{values: map[string][]byte{}}
p := newPasser(kvSet{buckets.Wants: &memKV{values: map[string][]byte{}}, buckets.DeliveryWaits: waits, buckets.DeliveredAgain: again}, at)
p.logf = t.Logf
var events []said
p.publishWants = func(event string, body any, id string) error {
raw, _ := json.Marshal(body)
events = append(events, said{event, id, raw})
return nil
}
p.w.runningHeard = true
placeOn(t, p.w, "ace", "db", "app")
placeOn(t, p.w, "g14", "db")
fp := func(c string) string { return meshcatalogclient.SourcePrefix2 + strings.Repeat(c, 64) }
want := func(node, module, fp, version string) {
key, _ := mwc.WantKey(node, module)
raw, err := mwc.EncodeWant(mwc.Want{Fp: fp, Version: version, Self: true})
if err != nil {
t.Fatal(err)
}
p.w.put(buckets.Wants, key, raw)
}
runs := func(node, module, fp string) {
raw, _ := json.Marshal(map[string]any{"v": 1, "node": node, "module": module, "fingerprint": fp, "at": at})
if err := p.w.hearRunning("running", raw); err != nil {
t.Fatal(err)
}
}
source := int64(3)
release := func(module, version string) {
source++
raw, _ := json.Marshal(meshcatalog.VersionReleasedEvent{V: 1, Module: module, Version: version, Source: source, At: &at,
Artifacts: []meshcatalog.VersionArtifact{{Artifact: "program", Digest: "sha256:d"}}})
if err := p.w.released.put(raw); err != nil {
t.Fatal(err)
}
}
saidOf := func(event string) []said {
var out []said
for _, e := range events {
if e.event == event {
out = append(out, e)
}
}
return out
}
want("ace", "app", fp("1"), "")
// No version of db is released: app's new fingerprint waits for nothing.
p.w.putEdges(trunkBranch, "app", []meshcatalog.Edge{{Kind: 4, From: "db:program", To: "app:program"}})
if p.holdForDelivery(ctx, "app", fp("2"), trunkBranch, []string{"ace"}) {
t.Fatal("app waits with no version of db released")
}
release("db", "1.1.0")
want("ace", "db", fp("d"), "1.1.0")
want("g14", "db", fp("d"), "1.1.0")
runs("ace", "db", fp("d"))
// A kind-1 edge orders nothing.
p.w.putEdges(trunkBranch, "app", []meshcatalog.Edge{{Kind: 1, From: "db:program", To: "app:program"}})
if p.holdForDelivery(ctx, "app", fp("2"), trunkBranch, []string{"ace"}) {
t.Fatal("a kind-1 edge holds app")
}
p.w.putEdges(trunkBranch, "app", []meshcatalog.Edge{{Kind: 4, From: "db:program", To: "app:program"}})
// app's want unchanged waits for nothing.
if p.holdForDelivery(ctx, "app", fp("1"), trunkBranch, []string{"ace"}) {
t.Fatal("app's unchanged want waits")
}
// g14 places db and does not run 1.1.0 yet: not delivered, and app's new version waits, said once.
p.judgeDelivered()
if !p.holdForDelivery(ctx, "app", fp("2"), trunkBranch, []string{"ace"}) || !p.holdForDelivery(ctx, "app", fp("2"), trunkBranch, []string{"ace"}) {
t.Fatal("app does not wait for db 1.1.0")
}
if len(saidOf(meshwants.EventVersionDelivered)) != 0 {
t.Fatalf("db is said delivered while g14 does not run it: %+v", events)
}
held := saidOf(meshwants.EventWaitHeld)
if len(held) != 1 {
t.Fatalf("wait-held said %d times, want once", len(held))
}
var w meshwants.Wait
_ = json.Unmarshal(held[0].body, &w)
if w.Step != meshwants.WaitStepDelivery || w.Part != "db" || w.Module != "app" || w.Version != fp("2") || w.Bound != "1h0m0s" ||
w.ClockFrom == nil || !w.ClockFrom.Equal(at) || len(w.Machines) != 1 || w.Machines[0] != "g14" || w.Words != "waiting for delivery of db 1.1.0 on g14" {
t.Errorf("the wait said is %+v", w)
}
if _, ok := waits.values["app"]; !ok {
t.Errorf("the wait is not kept in delivery-waits: %v", waits.values)
}
// g14 away: every present machine runs it, so it is delivered, g14 named away; once only.
p.w.heardPresence("g14", true, at)
p.judgeDelivered()
p.judgeDelivered()
delivered := saidOf(meshwants.EventVersionDelivered)
var d meshwants.VersionDeliveredEvent
if len(delivered) != 1 || json.Unmarshal(delivered[0].body, &d) != nil || d.Module != "db" || d.Version != "1.1.0" ||
len(d.Away) != 1 || d.Away[0] != "g14" || d.Parts["db:program"] != "sha256:d" {
t.Fatalf("version-delivered said %+v", delivered)
}
// The wait ends ran, and app's want goes on.
if p.holdForDelivery(ctx, "app", fp("2"), trunkBranch, []string{"ace"}) {
t.Fatal("app still waits once db 1.1.0 is delivered")
}
ended := saidOf(meshwants.EventWaitEnded)
if len(ended) != 1 || json.Unmarshal(ended[0].body, &w) != nil || w.How != deliveryWaitRan {
t.Errorf("wait-ended said %+v", ended)
}
if _, ok := waits.values["app"]; ok {
t.Error("the ended wait is kept")
}
// A newer release with every placing machine away is not delivered, and its wait says the machines it waits to be back.
release("db", "1.2.0")
p.w.heardPresence("ace", true, at)
p.judgeDelivered()
if len(saidOf(meshwants.EventVersionDelivered)) != 1 {
t.Error("a version is said delivered while every placing machine is away")
}
if !p.holdForDelivery(ctx, "app", fp("3"), trunkBranch, []string{"ace"}) {
t.Fatal("app does not wait for db 1.2.0")
}
held = saidOf(meshwants.EventWaitHeld)
if json.Unmarshal(held[len(held)-1].body, &w) != nil || w.Words != "waiting for ace, g14 to be back" {
t.Errorf("the wait while every machine is away says %q", w.Words)
}
// Released by hand (assigner_release writes it): the wait ends released and app's want is written.
var kept deliveryWaiting
_ = json.Unmarshal(waits.values["app"], &kept)
kept.Released = true
raw, _ := json.Marshal(kept)
p.w.put(buckets.DeliveryWaits, "app", raw)
if p.holdForDelivery(ctx, "app", fp("3"), trunkBranch, []string{"ace"}) {
t.Fatal("a released wait holds app")
}
ended = saidOf(meshwants.EventWaitEnded)
if json.Unmarshal(ended[len(ended)-1].body, &w) != nil || w.How != deliveryWaitReleased {
t.Errorf("the released wait ended %+v", w)
}
// Presence: the later of machine-away and machine-back holds, whichever stream is read first.
p.w.heardPresence("ace", false, at.Add(time.Minute))
p.w.heardPresence("ace", true, at)
if p.w.awayNow()["ace"] || !p.w.online()["ace"] {
t.Error("an older machine-away read after a newer machine-back leaves ace away")
}
// A-250: an ask of assigner_delivered_again of 1.1.0, which ace (present) runs and g14 (away) does not, is said again and taken
// away; one of 1.2.0, which no machine runs, is kept with why.
ask := func(version string) {
raw, _ := json.Marshal(againAsk{Module: "db", Version: version, By: "op", Why: "the catalog missed it"})
p.w.put(buckets.DeliveredAgain, "db", raw)
}
before := len(saidOf(meshwants.EventVersionDelivered))
ask("1.1.0")
p.deliveredAgain(ctx)
if len(saidOf(meshwants.EventVersionDelivered)) != before+1 || p.w.has(buckets.DeliveredAgain, "db") {
t.Errorf("the ask of 1.1.0 said %d version-delivered and is kept %v", len(saidOf(meshwants.EventVersionDelivered))-before, p.w.has(buckets.DeliveredAgain, "db"))
}
ask("1.2.0")
p.deliveredAgain(ctx)
var a againAsk
if json.Unmarshal(p.w.value(buckets.DeliveredAgain, "db"), &a) != nil || a.Waits != "db 1.2.0 does not run healthy on ace" ||
len(saidOf(meshwants.EventVersionDelivered)) != before+1 {
t.Errorf("the ask of 1.2.0 = %+v", a)
}
}
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/graphkept.go
+20 -8
View File
@@ -34,6 +34,8 @@ type graphKeeper struct {
source int64
ok bool
said string
// pins removes a revision that ceased to be current once nothing pins it (pins.go); nil keeps every revision.
pins *pinKeeper
}
// keepGraph computes the revision of this pass and keeps it when the pass's inputs differ from the last computed.
@@ -42,13 +44,7 @@ func (p *passer) keepGraph(ctx context.Context) {
if g == nil || g.objects == nil || g.halves == nil {
return
}
in := assigngraph.Inputs{Halves: g.halves(), Placements: p.w.assignments(), Capabilities: p.capabilityDigests()}
// With no pipeline ends kept, the wants are the trunk-only form (pass.go): a placement naming no branch runs the trunk's.
in.FirstBranch = func(string) string { return trunkBranch }
if p.ends != nil {
in.FirstBranch = func(module string) string { return p.ends.branchOf(module, "") }
}
c := assigngraph.Compute(in)
c := assigngraph.Compute(p.graphInputs())
for _, u := range c.Uncomputed {
if g.said != u {
g.said = u
@@ -66,19 +62,35 @@ func (p *passer) keepGraph(ctx context.Context) {
g.last, g.has, g.kept, g.ok = c.Revision, true, rev, true
g.source = 0
for _, cat := range rev.Catalogs {
if cat.Branch == trunkBranch {
if cat.Branch == p.w.trunkName() {
g.source = cat.Source
}
}
if !fresh {
return
}
if g.pins != nil {
g.pins.judge() // the revision before ceased to be current
}
id := fmt.Sprintf("%s-%s-%d", meshwants.EventAssignmentRevision, rev.Kind, rev.Revision)
if err := p.wantsSay(meshwants.EventAssignmentRevision, assigngraph.Event(rev), id); err != nil {
p.logf("the assignment revision %d is kept but not said: %v", rev.Revision, err)
}
}
// graphInputs are what the pass's assignment graph is computed from: every followed branch's catalog half, the placements, each
// node's capability reading and the first branch of each registration.
func (p *passer) graphInputs() assigngraph.Inputs {
in := assigngraph.Inputs{Halves: p.graph.halves(), Placements: p.w.assignments(), Capabilities: p.capabilityDigests()}
// With no pipeline ends kept, the wants are the trunk-only form (pass.go): a placement naming no branch runs the trunk's.
first := p.w.trunkName()
in.FirstBranch = func(string) string { return first }
if p.ends != nil {
in.FirstBranch = func(module string) string { return p.ends.branchOf(module, "") }
}
return in
}
// capabilityDigests is the digest of each node's capability reading, the statement its engine made, by node.
func (p *passer) capabilityDigests() []meshwants.AssignmentRevisionCapabilitiesItem {
if p.reads == nil {
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/graphkept_test.go
+44
View File
@@ -12,6 +12,7 @@ import (
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"mesh-delivery-assigner/internal/assigngraph"
"mesh-delivery-assigner/internal/buckets"
)
@@ -77,4 +78,47 @@ func TestTheWantsAreSaidWantedFromTheKeptAssignmentGraphRevision(t *testing.T) {
if err := json.Unmarshal(events[len(events)-1].body, &w); err != nil || len(events) != 3 || !w.Removed || w.Fp != fpTest {
t.Errorf("a removing want said %+v (%v), %d events", w, err, len(events))
}
// A-201, A-235: a run's candidate catalog revision, read whole through the seat's verb revision, keeps the run's assignment
// graph under candidate.<group>.<heads> in place of its branch's half; it says nothing and writes no want, a try's candidate
// keeps none, and the run's group-ended drops it.
before, puts := len(events), kv.puts
ev := meshcatalog.CatalogRevisionEvent{Kind: "candidate", Branch: trunkBranch, Group: "run-7", Heads: "h1", Revision: 5}
var asked []meshcatalog.RevisionInput
call := func(key string, body any) (json.RawMessage, error) {
in := body.(meshcatalog.RevisionInput)
asked = append(asked, in)
rev := meshcatalog.CatalogRevision{Branch: trunkBranch, Kind: "candidate", Group: "run-7", Heads: "h1", Revision: 5, Source: 9,
Entries: []meshcatalog.EntryRef{{Module: "web", Digest: "sha256:" + strings.Repeat("b", 64)}}}
a := meshcatalog.RevisionAnswer{Revision: &rev, Entries: []meshcatalog.CatalogEntry{{Module: "web", Fp: "src2:w2"}}}
return json.Marshal(a)
}
if err := p.keepCandidate(ctx, call, meshcatalog.CatalogRevisionEvent{Kind: "candidate", Branch: trunkBranch, Group: "try-3", Heads: "h", Revision: 2}); err != nil || len(asked) != 0 {
t.Errorf("a try's candidate asked %v: %v", asked, err)
}
if err := p.keepCandidate(ctx, call, ev); err != nil {
t.Fatal(err)
}
if len(asked) != 1 || asked[0].Group != "run-7" || asked[0].Heads != "h1" || asked[0].Revision != 5 {
t.Errorf("the candidate's revision was asked as %+v", asked)
}
a, err := assigngraph.Revision(ctx, objects, meshwants.RevisionInput{Group: "run-7"})
if err != nil || a.Revision.Kind != "candidate" || len(a.Revision.Catalogs) != 1 || a.Revision.Catalogs[0].Revision != 5 {
t.Errorf("the run's assignment graph is %+v: %v", a.Revision, err)
}
if e, err := assigngraph.Entry(ctx, objects, meshwants.EntryInput{Group: "run-7", Node: "ace", Module: "web"}); err != nil || e.Entry.Fp != "src2:w2" {
t.Errorf("the run's entry of web is %+v: %v", e.Entry, err)
}
if len(events) != before || kv.puts != puts {
t.Errorf("a candidate said %d events and wrote %d wants", len(events)-before, kv.puts-puts)
}
if err := p.dropCandidate(ctx, "run-7"); err != nil {
t.Fatal(err)
}
if _, ok := objects[assigngraph.CandidateObject("run-7", "h1")]; ok {
t.Error("the run's group-ended left its candidate kept")
}
if _, err := assigngraph.Revision(ctx, objects, meshwants.RevisionInput{}); err != nil {
t.Errorf("dropping the run's candidate dropped trunk: %v", err)
}
}
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/halves.go
+74 -38
View File
@@ -16,11 +16,12 @@ import (
"git.novox.be/novox/mesh-sdk/go/views"
)
// halves follows the catalog half of each watched branch through the seat mesh-catalog (design 47 G §14, §184, §195): one
// meshcatalogclient.Catalog per branch, adopted through the module's views reader (A-26), which asks the first view of the verb
// `revision`, hears each `catalog-revision` of that branch and raises and clears the follower's findings; each entry is taken
// into the world as the value and the graph node the rules read. Of the view the assigner reads only the catalog half, never its
// own output back (§195).
// halves follows the catalog half of each watched branch through the seat mesh-catalog (design 47 G §14, §184, §195, §507, §532): the
// branch follower of meshcatalogclient, adopted on the module's views reader (A-26), follows the branches the registrations name
// and, before the first registrations, the branches the gate's verb `source` answers and its event `source-moved` brings; each
// branch's view asks the verb `revision`, hears each `catalog-revision` of that branch and raises and clears the follower's findings;
// each entry is taken into the world as the value and the graph node the rules read. Of the view the assigner reads only the catalog
// half, never its own output back (§195).
type halves struct {
w *world
call mcc.Call
@@ -29,42 +30,70 @@ type halves struct {
logf func(string, ...any)
ctx context.Context
mu sync.Mutex
cats map[string]*mcc.Catalog
// settled is told once when a branch's first view is answered, for the first pass to wait on (the trunk's alone).
branches *mcc.Branches
mu sync.Mutex
// settled is told once when the first watched branch's first view is answered, for the first pass to wait on.
settled chan<- struct{}
told bool
}
// follow adopts the catalog of branch through the reader, once; true when it was made now.
func (h *halves) follow(branch string) bool {
h.mu.Lock()
if h.cats[branch] != nil {
h.mu.Unlock()
return false
}
c, _, err := mcc.WireCatalogProcess(h.reader, h.call, branch, func(module string, entry json.RawMessage, deleted bool, _ time.Time) {
h.changed(branch, module, entry, deleted)
})
// follow makes the branch follower on the reader; the reader's start asks the first views.
func (h *halves) follow() error {
b, err := mcc.FollowBranchesProcess(h.reader, h.call)
if err != nil {
h.mu.Unlock()
h.logf("the catalog half of branch %q is not followed: %v", branch, err)
return false
return err
}
if h.cats == nil {
h.cats = map[string]*mcc.Catalog{}
b.Logf = h.logf
b.Changed = func(branch, module string, entry json.RawMessage, deleted bool, _ time.Time) {
h.changed(branch, module, entry, deleted)
}
h.cats[branch] = c
b.Adopted = h.adopted
h.mu.Lock()
h.branches = b
h.mu.Unlock()
if branch == trunkBranch && h.settled != nil {
go func() {
select {
case <-c.Ready():
h.settled <- struct{}{}
case <-h.ctx.Done():
}
}()
return nil
}
// first is the first watched branch followed, the trunk's: "" while none is.
func (h *halves) first() string {
h.mu.Lock()
b := h.branches
h.mu.Unlock()
if b == nil {
return ""
}
return true
if f := b.Followed(); len(f) > 0 {
return f[0]
}
return ""
}
// adopted is told of each branch whose view the follower adopted: the first branch is the trunk's, and the first pass waits for its
// first view.
func (h *halves) adopted(branch string, c *mcc.Catalog) {
first := h.first()
if first == "" {
return
}
h.w.setTrunk(first)
h.wake()
if branch != first || h.settled == nil {
return
}
h.mu.Lock()
told := h.told
h.told = true
h.mu.Unlock()
if told {
return
}
go func() {
select {
case <-c.Ready():
h.settled <- struct{}{}
case <-h.ctx.Done():
}
}()
}
// changed takes one entry of branch's half into the world, or a module taken out of it.
@@ -81,6 +110,7 @@ func (h *halves) changed(branch, module string, raw json.RawMessage, deleted boo
}
v, n, m := fromEntry(e)
h.w.putModule(branch, module, v, n, m)
h.w.putEdges(branch, module, e.Edges)
}
// fromEntry is a catalog entry as the value and the graph node the rules read, and its manifest parsed once under the digest of
@@ -114,11 +144,17 @@ func fromEntry(e meshcatalog.CatalogEntry) (meshcatalog.ModulesByModuleValue, gr
// views is the catalog graph revision held now of each followed branch whose first view is answered.
func (h *halves) views() map[string]mcc.Half {
h.mu.Lock()
defer h.mu.Unlock()
out := make(map[string]mcc.Half, len(h.cats))
for branch, c := range h.cats {
if half, ok := c.View(); ok {
out[branch] = half
b := h.branches
h.mu.Unlock()
out := map[string]mcc.Half{}
if b == nil {
return out
}
for _, branch := range b.Followed() {
if c, ok := b.Catalog(branch); ok {
if half, ok := c.View(); ok {
out[branch] = half
}
}
}
return out
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go
+173 -26
View File
@@ -8,6 +8,8 @@ import (
"errors"
"fmt"
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
wd "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshwatchdogs"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-sdk/go/keys"
"io"
"maps"
@@ -44,6 +46,7 @@ import (
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
"git.novox.be/novox/mesh-contracts/provisioning/go/clients/meshprovisionerclient"
mdseat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/announce"
"git.novox.be/novox/mesh-sdk/go/buscompose"
@@ -176,7 +179,34 @@ func startBed(t *testing.T, leaveOut string) *bed {
Subjects: []string{buscompose.LogSubject(builds.BuilderModule, builds.BuilderLog) + ".>"}, AllowDirect: true}); err != nil {
t.Fatal(err)
}
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "EVENTS", Subjects: []string{"mesh.mod.*.event.>", "mesh.seat.*.event.>"}}); err != nil {
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "EVENTS", Subjects: []string{"mesh.mod.*.event.>", "mesh.seat.mesh-catalog.event.>", "mesh.seat.mesh-conditions.event.>", "mesh.seat.mesh-module-assigner.event.>", enginereads.SubjectPrefix + "*"}}); err != nil {
t.Fatal(err)
}
// The node-engine seat's stream of `running`, kept last per subject, which the assigner hears the health of what runs from (§520).
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: nodeengine.StreamRunning, Subjects: []string{nodeengine.SubjectRunningAll}, MaxMsgsPerSubject: 1}); err != nil {
t.Fatal(err)
}
if _, err := js.CreateOrUpdateConsumer(ctx, nodeengine.StreamRunning, jetstream.ConsumerConfig{Durable: "n1_mesh-delivery-assigner", AckPolicy: jetstream.AckExplicitPolicy}); err != nil {
t.Fatal(err)
}
// The watchman's machine-away and machine-back and the assigner's own version-delivered, kept by age, which it reads whole from their
// first message at its start and then follows through its durable (design 47 G §376, §389, A-189, A-233).
for _, k := range []struct{ stream, subject string }{{wd.StreamMachineAway, wd.SubjectMachineAwayAll}, {wd.StreamMachineBack, wd.SubjectMachineBackAll},
{meshwants.StreamVersionDelivered, meshwants.SubjectVersionDelivered}} {
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: k.stream, Subjects: []string{k.subject}, MaxAge: 90 * 24 * time.Hour}); err != nil {
t.Fatal(err)
}
if _, err := js.CreateOrUpdateConsumer(ctx, k.stream, jetstream.ConsumerConfig{Durable: "n1_mesh-delivery-assigner", AckPolicy: jetstream.AckExplicitPolicy}); err != nil {
t.Fatal(err)
}
}
// The builder's stream of build ends, kept last per subject (its manifest's `last-per-subject`, design 47 G A-181), which the assigner
// reads whole at its start and follows through its durable, as the grants part makes it (from the last of each subject).
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: buildsStreamName, Subjects: []string{process.ModuleStreamSubject(builds.BuilderModule, builds.BuildEndedEvent+".>")}, MaxMsgsPerSubject: 1}); err != nil {
t.Fatal(err)
}
if _, err := js.CreateOrUpdateConsumer(ctx, buildsStreamName, jetstream.ConsumerConfig{Durable: "n1_mesh-delivery-assigner", AckPolicy: jetstream.AckExplicitPolicy,
FilterSubjects: []string{process.ModuleStreamSubject(builds.BuilderModule, builds.BuildEndedEvent+".>")}, DeliverPolicy: jetstream.DeliverLastPerSubjectPolicy}); err != nil {
t.Fatal(err)
}
// The assigner's own stream of its findings' events (design 47 G §383), which its finding writer says into.
@@ -187,7 +217,8 @@ func startBed(t *testing.T, leaveOut string) *bed {
// event names the engine's machine last, so the filter is the event with every machine (mesh-grants buscompose
// machineEvent; the grants part's own test holds the derivation).
if _, err := js.CreateOrUpdateConsumer(ctx, "EVENTS", jetstream.ConsumerConfig{Durable: "n1_mesh-delivery-assigner",
FilterSubjects: []string{enginereads.SubjectPrefix + "*", "mesh.mod." + buckets.IndexerModule + ".event.views.>", meshcatalog.SubjectCatalogRevisionAll}, AckPolicy: jetstream.AckExplicitPolicy}); err != nil {
FilterSubjects: []string{enginereads.SubjectPrefix + "*", "mesh.mod." + buckets.IndexerModule + ".event.views.>",
"mesh.mod." + buckets.TellerModule + ".event.views.>", meshcatalog.SubjectCatalogRevisionAll}, AckPolicy: jetstream.AckExplicitPolicy}); err != nil {
t.Fatal(err)
}
// The operator-channel seat's stream of conditions, which the assigner says its conditions on.
@@ -196,6 +227,8 @@ func startBed(t *testing.T, leaveOut string) *bed {
}
// The indexer as the owner of its manifests and modules (DLV-F2r), with the real owner's side.
serveIndexer(t, b, nc)
// The teller as the owner of its answers (A-175).
serveTeller(t, b, nc)
// The seat mesh-catalog's catalog half (design 47 G §194), which the assigner follows per branch.
serveCatalog(t, b, nc)
// The assigner's views family is its ordinary events on EVENTS (ADR 0635 §5); its marks are its state.
@@ -356,7 +389,7 @@ func TestTheAssignerDoesNotStartWithoutABucketItReads(t *testing.T) {
}
for deadline := time.Now().Add(15 * time.Second); ; time.Sleep(200 * time.Millisecond) {
// A change of a bucket it watches starts a pass, which opens the puts' bucket.
if _, err := b.buckets[buckets.Presence].Put(b.ctx, "n1", []byte(time.Now().String())); err != nil {
if _, err := b.buckets[buckets.Have].Put(b.ctx, "n1.pulse", []byte(time.Now().String())); err != nil {
t.Fatal(err)
}
if e, err := putsKV.Get(b.ctx, pkey); err == nil {
@@ -407,11 +440,38 @@ func TestTheAssignerDoesNotStartWithoutABucketItReads(t *testing.T) {
}
// putJSON writes a value into one of the bed's buckets.
// putBuild says the builder's `build.ended` of the build a builder's key and value (the old form of a done build, as the tests write it) name,
// on the builder's stream (design 47 G A-181): the assigner hears what was built, it reads no bucket of the builder.
func (b *bed) putBuild(t *testing.T, key string, value []byte) {
t.Helper()
module, fp, err := builds.ParseBuilderBuildKey(key)
built, derr := builds.DecodeBuilderBuild(value)
if err != nil || derr != nil {
t.Fatalf("%s: %v %v", key, err, derr)
}
raw, _ := json.Marshal(builds.BuildEnded{Module: module, Fingerprint: fp, Outcome: built.State, Error: built.Error, Manifest: built.Manifest, Needs: built.Needs})
if _, err := b.js.Publish(b.ctx, process.ModuleStreamSubject(builds.BuilderModule, builds.BuildEndedEventName(module, fp)), raw); err != nil {
t.Fatal(err)
}
}
func (b *bed) putJSON(t *testing.T, bucket, key string, value []byte) {
t.Helper()
if _, err := b.buckets[bucket].Put(b.ctx, key, value); err != nil {
t.Fatalf("%s %s: %v", bucket, key, err)
}
if bucket == buckets.Have { // the node-engine says `running` with each have it writes (§520): the assigner reads health from it alone
node, module, isNode, err := nodeengineclient.ParseHaveKey(key)
h, derr := nodeengineclient.DecodeHave(value)
if err != nil || isNode || derr != nil || h.Phase != nodeengineclient.PhaseRunning {
return
}
ok := h.Health == nodeengineclient.HealthOK
raw, _ := json.Marshal(nodeengine.Running{At: time.Now(), Node: node, Module: module, Fingerprint: h.Fp, Healthy: &ok})
if _, err := b.js.Publish(b.ctx, nodeengine.SubjectRunning(node, module), raw); err != nil {
t.Fatal(err)
}
}
}
// place adds module to the seat's assignments of node, written as the seat writes it: one node key and a key per module
@@ -510,7 +570,7 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) {
t.Fatal(err)
}
bkey, _ := builds.BuilderBuildKey("web", fNew)
b.putJSON(t, buckets.Builds, bkey, built)
b.putBuild(t, bkey, built)
for _, fp := range []string{fNew, fOld} {
gkey, _ := grantsstate.PartGrantKey("web", fp)
grant, _ := grantsstate.EncodeGrant(grantsstate.Grant{})
@@ -519,8 +579,6 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) {
runs := map[string]string{"ace": fOld, "g14": "build-9", "shanks": fOld}
for _, node := range []string{"ace", "g14", "shanks"} {
b.compose(t, node, 'c')
on, _ := presenceValue(true)
b.putJSON(t, buckets.Presence, node, on)
have, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: runs[node], Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK})
b.putJSON(t, buckets.Have, node+".web", have)
b.place(t, node, "web")
@@ -660,7 +718,7 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) {
for _, fp := range fps {
built, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildDone, By: "builder.novox.1", Step: "pushed", Manifest: row.digest})
bkey, _ := builds.BuilderBuildKey(module, fp)
b.putJSON(t, buckets.Builds, bkey, built)
b.putBuild(t, bkey, built)
gkey, _ := grantsstate.PartGrantKey(module, fp)
grant, _ := grantsstate.EncodeGrant(grantsstate.Grant{})
b.putJSON(t, buckets.Grants, gkey, grant)
@@ -736,7 +794,7 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) {
t.Fatal(err)
}
bkey, _ := builds.BuilderBuildKey("forms", fp)
b.putJSON(t, buckets.Builds, bkey, built)
b.putBuild(t, bkey, built)
gkey, _ := grantsstate.PartGrantKey("forms", fp)
grant, _ := grantsstate.EncodeGrant(grantsstate.Grant{})
b.putJSON(t, buckets.Grants, gkey, grant)
@@ -847,8 +905,6 @@ func TestAHeldFingerprintIsAskedAbout(t *testing.T) {
digest := b.manifest(t, "web", `{"module":"web","version":"1"}`)
mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fNew, Manifest: digest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath})
b.putModule(t, "web", mod)
on, _ := presenceValue(true)
b.putJSON(t, buckets.Presence, "ace", on)
failed, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed, Detail: "connection refused"})
b.putJSON(t, buckets.Have, "ace.web", failed)
b.place(t, "ace", "web")
@@ -905,8 +961,6 @@ func TestAFailedDataModuleIsAskedToRestore(t *testing.T) {
digest := b.manifest(t, "mail", `{"module":"mail","version":"1","resources":[{"id":"data","type":"directory","path":"/var/lib/mail"}],"data":{"own":[{"id":"mail","path":"${dir:data}","class":"irreplaceable","backup":"copy"}]}}`)
mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fNew, Manifest: digest, Repository: "novox/mail", Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath})
b.putModule(t, "mail", mod)
on, _ := presenceValue(true)
b.putJSON(t, buckets.Presence, "ace", on)
failed, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed, Detail: "connection refused"})
b.putJSON(t, buckets.Have, "ace.mail", failed)
wantKey, _ := mwc.WantKey("ace", "mail")
@@ -945,8 +999,6 @@ func TestAFailedDataModuleIsAskedToRestore(t *testing.T) {
func TestTheAssignerReplacedResumesWithNothingLostOrRepeated(t *testing.T) {
b := startBed(t, "")
const modules = 40
on, _ := presenceValue(true)
b.putJSON(t, buckets.Presence, "ace", on)
names := make([]string, modules)
for i := range names {
name := fmt.Sprintf("mod%02d", i)
@@ -960,7 +1012,7 @@ func TestTheAssignerReplacedResumesWithNothingLostOrRepeated(t *testing.T) {
t.Fatal(err)
}
bkey, _ := builds.BuilderBuildKey(name, fp)
b.putJSON(t, buckets.Builds, bkey, built)
b.putBuild(t, bkey, built)
gkey, _ := grantsstate.PartGrantKey(name, fp)
grant, _ := grantsstate.EncodeGrant(grantsstate.Grant{})
b.putJSON(t, buckets.Grants, gkey, grant)
@@ -1003,12 +1055,10 @@ func (b *bed) fixture(t *testing.T, name, fp, manifest string) {
b.putModule(t, name, mod)
built, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildDone, By: "builder.novox.1", Step: "pushed", Manifest: digest})
bkey, _ := builds.BuilderBuildKey(name, fp)
b.putJSON(t, buckets.Builds, bkey, built)
b.putBuild(t, bkey, built)
gkey, _ := grantsstate.PartGrantKey(name, fp)
grant, _ := grantsstate.EncodeGrant(grantsstate.Grant{})
b.putJSON(t, buckets.Grants, gkey, grant)
on, _ := presenceValue(true)
b.putJSON(t, buckets.Presence, "ace", on)
b.place(t, "ace", name)
}
@@ -1326,6 +1376,13 @@ func (s bedStore) history(string, string) ([]stdio.StateRevision, error) { retur
// and declined, writes nothing and is withdrawn.
func TestAProposalIsAnsweredApprovedAndWritten(t *testing.T) {
b := startBed(t, "")
// Both machines are away throughout (§389): a machine the watchman never said away is present.
for _, node := range []string{"anchor", "laptop"} {
raw, _ := json.Marshal(wd.MachineAwayEvent{V: 1, Machine: node, At: time.Now(), LastHeard: time.Now()})
if _, err := b.js.Publish(b.ctx, wd.SubjectMachineAway(node), raw); err != nil {
t.Fatal(err)
}
}
if _, err := b.js.CreateStream(b.ctx, jetstream.StreamConfig{Name: "MOD_" + assignment.Module + "_assignments",
Subjects: assignmentsSubjects()}); err != nil { // the seat's four events: the views family has a stream of its own
t.Fatal(err)
@@ -1664,13 +1721,32 @@ var catalogLate chan struct{}
// read by the assigner's durable consumer; and its first-view tools of the three answered from the buckets at their marks.
func serveIndexer(t *testing.T, b *bed, nc *nats.Conn) {
t.Helper()
marks, err := b.js.CreateKeyValue(b.ctx, jetstream.KeyValueConfig{Bucket: buckets.IndexerModule + "_" + views.MarksState, History: 1})
tools := map[string][2]string{}
for state, bucket := range indexerFirstViews {
tools[state] = [2]string{state, bucket}
}
serveOwner(t, b, nc, buckets.IndexerModule, tools, catalogLate)
}
// serveTeller is the teller as the owner of its answers (design 47 G A-175, A-178): its bucket, which a test fills as the
// teller writes it, said by its views family, and its read tool teller_read_answers answering the first view.
func serveTeller(t *testing.T, b *bed, nc *nats.Conn) {
t.Helper()
serveOwner(t, b, nc, buckets.TellerModule, map[string][2]string{buckets.AnswersReadTool: {buckets.AnswersState, buckets.Answers}}, nil)
}
// serveOwner serves an owner's states as the owner does: each state's bucket, its views family on EVENTS, and the first-view
// tool of each (by tool, its state and bucket) answered from the bucket at its mark. While late is open every ask fails.
func serveOwner(t *testing.T, b *bed, nc *nats.Conn, owner string, tools map[string][2]string, late chan struct{}) {
t.Helper()
marks, err := b.js.CreateKeyValue(b.ctx, jetstream.KeyValueConfig{Bucket: owner + "_" + views.MarksState, History: 1})
if err != nil {
t.Fatal(err)
}
em := views.Emitter{Owner: buckets.IndexerModule, Subject: process.Account{Node: "n1", Module: buckets.IndexerModule}.EventSubject,
em := views.Emitter{Owner: owner, Subject: process.Account{Node: "n1", Module: owner}.EventSubject,
Publish: views.JetStreamPublish(b.js), Marks: marks}
for state, bucket := range indexerFirstViews {
for tool, sb := range tools {
state, bucket := sb[0], sb[1]
kv, err := b.js.CreateKeyValue(b.ctx, jetstream.KeyValueConfig{Bucket: bucket, History: 5})
if err != nil {
t.Fatal(err)
@@ -1679,13 +1755,12 @@ func serveIndexer(t *testing.T, b *bed, nc *nats.Conn) {
if err := em.Start(b.ctx, state, kv, nil); err != nil {
t.Fatal(err)
}
late := catalogLate
sub, err := nc.Subscribe("mesh.mod.mesh-delivery-indexer.call."+state+".>", func(m *nats.Msg) {
if late != nil { // its runtime comes up after the assigner started (DLV-F5r): until then every ask fails, as on a down indexer
sub, err := nc.Subscribe("mesh.mod."+owner+".call."+tool+".>", func(m *nats.Msg) {
if late != nil { // its runtime comes up after the assigner started (DLV-F5r): until then every ask fails, as on a down owner
select {
case <-late:
default:
_ = m.Respond([]byte(`{"error":"the indexer's runtime is not up"}`))
_ = m.Respond([]byte(`{"error":"the owner's runtime is not up"}`))
return
}
}
@@ -1693,7 +1768,7 @@ func serveIndexer(t *testing.T, b *bed, nc *nats.Conn) {
After string `json:"after"`
}
_ = json.Unmarshal(m.Data, &args)
a, err := views.FirstView(buckets.IndexerModule, state, bedSource{b.ctx, kv}, bedSource{b.ctx, marks}, args.After, time.Now())
a, err := views.FirstView(owner, state, bedSource{b.ctx, kv}, bedSource{b.ctx, marks}, args.After, time.Now())
out := map[string]any{"result": a, "node": "n0"}
if err != nil {
out = map[string]any{"error": err.Error()}
@@ -1911,6 +1986,29 @@ func serveCatalog(t *testing.T, b *bed, nc *nats.Conn) {
raw, _ := json.Marshal(map[string]any{"result": meshcatalog.RevisionAnswer{Revision: &rev, Entries: entries}, "node": "n0"})
_ = m.Respond(raw)
}
// The gate's verb `source` given no branch (design 47 G §507, §690): the branches it keeps a source revision of, which the assigner
// follows before the first registrations, and the trunk's `main` first.
source := func(m *nats.Msg) {
b.cat.mu.Lock()
defer b.cat.mu.Unlock()
names := map[string]bool{"main": true}
for branch := range b.cat.rev {
names[branch] = true
}
var escaped []string
for _, n := range slices.Sorted(maps.Keys(names)) {
escaped = append(escaped, keys.EscapeName(n))
}
raw, _ := json.Marshal(map[string]any{"result": mdseat.SourceAnswer{Branches: escaped}, "node": "n0"})
_ = m.Respond(raw)
}
for _, subject := range []string{"mesh.seat.mesh-delivery.tool.source", "mesh.seat.mesh-delivery.call.source.*", "mesh.seat.mesh-delivery.tool.source.*"} {
sub, err := nc.Subscribe(subject, source)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = sub.Unsubscribe() })
}
for _, subject := range []string{"mesh.seat.mesh-catalog.tool.revision", "mesh.seat.mesh-catalog.call.revision.*", "mesh.seat.mesh-catalog.tool.revision.*"} {
sub, err := nc.Subscribe(subject, answer)
if err != nil {
@@ -1920,3 +2018,52 @@ func serveCatalog(t *testing.T, b *bed, nc *nats.Conn) {
}
subscribed(t, nc)
}
// A-181: the assigner reads the builder's stream of build ends whole at its start (the last of each fingerprint, a failed one's error
// among them) though its durable is made after them, then follows it: what was built survives its restart, and it reads no builder bucket.
func TestTheAssignerReadsTheBuildersEndsWholeAtItsStart(t *testing.T) {
b := startBed(t, "")
thisNode = "n1"
b.putBuild(t, mustBuildKey(t, "web", "src2:"+strings.Repeat("a", 64)), mustEncodeBuild(t, builds.BuilderBuild{State: builds.BuildFailed, Error: "first try"}))
b.putBuild(t, mustBuildKey(t, "web", "src2:"+strings.Repeat("a", 64)), mustEncodeBuild(t, builds.BuilderBuild{State: builds.BuildDone, Manifest: strings.Repeat("d", 64)}))
b.putBuild(t, mustBuildKey(t, "api", "src2:"+strings.Repeat("b", 64)), mustEncodeBuild(t, builds.BuilderBuild{State: builds.BuildFailed, Error: "transport fault: x"}))
var ends builtEnds
woken := make(chan struct{}, 8)
hearBuilds(b.ctx, b.js, &ends, func() { woken <- struct{}{} }, assign.Findings{}, t.Logf)
if e, ok := ends.endOf("web", "src2:"+strings.Repeat("a", 64)); !ok || e.Outcome != builds.BuildDone || e.Manifest != strings.Repeat("d", 64) {
t.Fatalf("the read of the stream left web's end as %+v, %v: want the last, done", e, ok)
}
if e, ok := ends.endOf("api", "src2:"+strings.Repeat("b", 64)); !ok || e.Outcome != builds.BuildFailed || e.Error != "transport fault: x" {
t.Fatalf("the read of the stream left api's end as %+v, %v", e, ok)
}
b.putBuild(t, mustBuildKey(t, "api", "src2:"+strings.Repeat("c", 64)), mustEncodeBuild(t, builds.BuilderBuild{State: builds.BuildDone, Manifest: strings.Repeat("d", 64)}))
deadline := time.After(10 * time.Second)
for {
if _, ok := ends.endOf("api", "src2:"+strings.Repeat("c", 64)); ok {
return
}
select {
case <-woken:
case <-deadline:
t.Fatal("an end said after the read was not followed")
}
}
}
func mustBuildKey(t *testing.T, module, fp string) string {
t.Helper()
k, err := builds.BuilderBuildKey(module, fp)
if err != nil {
t.Fatal(err)
}
return k
}
func mustEncodeBuild(t *testing.T, b builds.BuilderBuild) []byte {
t.Helper()
raw, err := builds.EncodeBuilderBuild(b)
if err != nil {
t.Fatal(err)
}
return raw
}
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go
+70 -34
View File
@@ -29,6 +29,7 @@ import (
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-sdk/go/announce"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/seatlist"
@@ -62,13 +63,8 @@ func main() {
// holds the two lists equal), so the controller's broker grants exactly these.
var readBuckets = []string{
buckets.Grants,
buckets.Answers,
buckets.Builds,
buckets.Presence,
buckets.Have,
buckets.Provisions,
buckets.Source,
buckets.Merges,
}
// readFamilies are the states kept one bucket per node the assigner reads, each named in its manifest's reads: every
@@ -80,7 +76,7 @@ var readFamilies = []string{buckets.ComposedRead}
var ownBuckets = []string{buckets.Asks, buckets.Differences,
buckets.Puts, buckets.Wants, plc.AssignmentsBucket, plc.ProposalsBucket,
plc.CandidatesBucket, plc.PulsesBucket, assign.FindingsBucket, assign.PoliciesBucket, plc.SeatHoldersBucket,
engineReadsBucket, retiredBucket, copiesBucket, writtenBucket, buckets.Waits, buckets.Inputs}
engineReadsBucket, retiredBucket, copiesBucket, writtenBucket, buckets.Waits, buckets.PutWaits, buckets.DeliveryWaits, buckets.DeliveredAgain, buckets.Inputs, pinsBucket}
// engineReadsState is the assigner's state of what each node's engine states it reads (ADR 0618 §3), one key `<node>`, as
// the verb manifest-reads last answered it or the fallback read it: what assigner_engine_reads lists and asks again.
@@ -121,7 +117,11 @@ var keptStates = []announce.State{
{Name: copiesState, Readable: true},
{Name: writtenState, Readable: true},
{Name: buckets.WaitsState, Readable: true},
{Name: buckets.InputsState, Readable: true}, // read by the tool process alone: the verb input and assigner_inputs (design 47 G §354)
{Name: buckets.PutWaitsState, Readable: true}, // the put waits said (putwaits.go, design 47 G A-211)
{Name: buckets.DeliveryWaitsState, Readable: true}, // the delivery waits said (delivered.go, design 47 G A-246)
{Name: buckets.DeliveredAgainState, Readable: true}, // the asks of assigner_delivered_again (delivered.go, design 47 G A-250)
{Name: buckets.InputsState, Readable: true}, // read by the tool process alone: the verb input and assigner_inputs (design 47 G §354)
{Name: pinsState, Readable: true}, // the records pinning each assignment graph revision (design 47 G §579)
}
// kvSet is every bucket the assigner opened, by name.
@@ -168,12 +168,6 @@ func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any
logf("the bucket %s cannot be opened (%v): the assigner's conditions are said straight to the router until it starts again", name, err)
continue
}
if (name == buckets.Source || name == buckets.Merges) && err != nil {
// The gate's source and merges (ADR 0601 §1): without them no release is known to carry a build, so every move of
// a record module is asked of a person, as before.
logf("the bucket %s cannot be opened (%v): no release's Approve answers a record ask until it is bound", name, err)
continue
}
if name == engineReadsBucket && err != nil {
// New with ADR 0618 §3: without it the statements the verb answers are kept in memory only, and
// assigner_engine_reads lists nothing.
@@ -185,12 +179,32 @@ func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any
logf("the bucket %s cannot be opened (%v): it is read by the tool process alone", name, err)
continue
}
if name == pinsBucket && err != nil {
// New with the pins (design 47 G §579): without it no pin is kept and no revision is removed.
logf("the bucket %s cannot be opened (%v): no pin is kept and no revision removed until it starts again", name, err)
continue
}
if name == buckets.Waits && err != nil {
// New with the wait on an outside cause (design 47 G §706): without it no wait is made, and a failing check is judged
// as before.
logf("the bucket %s cannot be opened (%v): no outside wait is kept until the assigner starts again", name, err)
continue
}
if name == buckets.DeliveredAgain && err != nil {
// New with A-250: without it assigner_delivered_again is not taken until the assigner starts again.
logf("the bucket %s cannot be opened (%v): no version is said delivered again until the assigner starts again", name, err)
continue
}
if name == buckets.DeliveryWaits && err != nil {
// New with A-246: without it no dependent waits for its dependency's delivery until the assigner starts again.
logf("the bucket %s cannot be opened (%v): no delivery wait is kept until the assigner starts again", name, err)
continue
}
if name == buckets.PutWaits && err != nil {
// New with A-211: without it no put's wait is said until the assigner starts again.
logf("the bucket %s cannot be opened (%v): no put's wait is said until the assigner starts again", name, err)
continue
}
if name == retiredBucket && err != nil {
// The retired state is read only for the copy into `assignments` (migrate.go): a mesh made after the switch has
// none, and nothing is copied.
@@ -215,7 +229,7 @@ func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any
func startsWithout(name string) bool {
switch name {
case plc.SeatHoldersBucket, buckets.Provisions, assign.PoliciesBucket, assign.FindingsBucket,
buckets.Puts, buckets.Source, buckets.Merges, engineReadsBucket, retiredBucket, writtenBucket, buckets.Waits, buckets.Inputs:
buckets.Puts, engineReadsBucket, retiredBucket, writtenBucket, buckets.Waits, buckets.PutWaits, buckets.DeliveryWaits, buckets.DeliveredAgain, buckets.Inputs, pinsBucket:
return true
}
return slices.Contains(seatBuckets, name) || name == buckets.Grants
@@ -321,18 +335,18 @@ type emitting struct{ *process.Conn }
func (e emitting) Emit(eventType string, body any) error { return e.Conn.Emit(eventType, body) }
// optionalEnds is what stops when the watch of a bucket that never gates the first pass ends.
var optionalEnds = map[string]string{buckets.Puts: "no put is taken further", buckets.Answers: "no ask is answered"}
// watched are the buckets whose every value a pass reads; the manifests are read by digest when a pass needs one, and
// watched only for keys, so a manifest that arrives wakes a composition that waits for it.
var watched = []string{
buckets.Grants,
buckets.Answers,
buckets.Builds,
plc.AssignmentsBucket,
plc.ProposalsBucket,
plc.CandidatesBucket,
plc.PulsesBucket,
copiesBucket,
buckets.Presence,
buckets.Have,
buckets.Asks,
buckets.Differences,
@@ -340,11 +354,12 @@ var watched = []string{
assign.PoliciesBucket,
buckets.Provisions,
plc.SeatHoldersBucket,
buckets.Source,
buckets.Merges,
engineReadsBucket,
writtenBucket,
buckets.Waits,
buckets.PutWaits,
buckets.DeliveryWaits,
buckets.DeliveredAgain, // its watch wakes a pass at once on an ask of assigner_delivered_again (A-250)
}
// follow watches every bucket a pass reads and makes a pass at start, once every watch handed what stands, and after
@@ -377,7 +392,7 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
return fmt.Errorf("the bucket %s cannot be watched: %w", name, err)
}
// The puts' watch never stops the wants (ADR 0546): when it ends, the puts are no longer heard, said in the journal.
optional := name == buckets.Puts
optional := name == buckets.Puts || name == buckets.Answers
go func() {
defer kw.Stop()
for {
@@ -388,7 +403,7 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
case e, ok := <-kw.Updates():
if !ok {
if optional {
logger.Printf("the watch of %s ended: no put is taken further until the assigner starts again", name)
logger.Printf("the watch of %s ended: %s until the assigner starts again", name, optionalEnds[name])
return
}
errs <- fmt.Errorf("the watch of %s ended", name)
@@ -455,8 +470,8 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
}
started++
var opts []jetstream.WatchOpt
if name == buckets.Have || name == buckets.Source {
// what each node ran healthy before, for a set-back; and the release behind each source revision (ADR 0601 §1)
if name == buckets.Have {
// what each node ran healthy before, for a set-back
opts = append(opts, jetstream.IncludeHistory())
}
if err := watch(name, opts...); err != nil {
@@ -477,6 +492,14 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
if iv != nil {
iv.hub = hub
started += iv.start(ctx, w, wake, settled)
// The teller's answers' picture is watched as a bucket is, but never gates the first pass: a teller that does not
// answer its first view only leaves the asks unanswered, said by the reader's finding (A-175).
if iv.answers != nil {
kvs[buckets.Answers] = iv.answers
if err := watch(buckets.Answers); err != nil {
logger.Printf("%v: no ask is answered until the assigner starts again", err)
}
}
}
// The seat's states arrive with the same delivery as this assigner (ADR 0567 §1): one not on the bus at this start is
// bound once the bus says it was created, and watched from then on, with no restart (issue 1790's kind). Until then no
@@ -547,7 +570,26 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
if js != nil {
p.publishWants = wantsPublisher(ctx, js, thisNode)
if iv != nil && iv.h != nil {
p.graph = &graphKeeper{objects: newBusObjects(js, process.Account{Node: thisNode, Module: assignment.Module}, assigngraph.RevisionsObjects), halves: iv.h.views}
objects := newBusObjects(js, process.Account{Node: thisNode, Module: assignment.Module}, assigngraph.RevisionsObjects)
p.graph = &graphKeeper{objects: objects, halves: iv.h.views}
keep := func(ctx context.Context, ev meshcatalog.CatalogRevisionEvent) error {
return p.keepCandidate(ctx, iv.call, ev)
}
iv.candidates.Store(&keep)
// The pins of the revisions (pins.go, design 47 G §579): the gate's pinned, freed, heads-replaced and pins-stated
// on the hub, registered before it starts.
if kv := kvs[pinsBucket]; kv != nil {
k := &pinKeeper{ctx: ctx, st: kvEnds{ctx, kv}, objects: objects, inUse: inUseOf(w), findings: findings, now: time.Now, logf: logger.Printf}
if err := k.made(); err != nil {
logger.Printf("the state %s is not marked made, so no revision is removed until it is: %v", pinsState, err)
}
if err := hearPins(hub, k); err != nil {
logger.Printf("a pin is not heard, so no revision is removed until the assigner starts again: %v", err)
} else {
p.graph.pins = k
k.watchUnpins(kv)
}
}
}
}
// The node-engine seat's manifest-reads-changed, for one node (ADR 0618 §4): that node is asked again, with no timer.
@@ -564,13 +606,8 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
// The runner's pipeline-ended and group-ended (design 47 G §29): kept in the state written, which the wants follow
// once the first registrations are written (§95, §353).
p.ends = newPipelineEnds(kvEnds{ctx, kv}, wake, logger.Printf)
if iv != nil && iv.h != nil {
p.ends.onBranches = func(branches []string) {
for _, b := range branches {
iv.h.follow(b)
}
}
}
p.ends.trunk = w.trunkName
p.ends.runEnded = func(group string) error { return p.dropCandidate(ctx, group) }
err := hearEnds(hub, p.ends)
if err == nil && bus != nil {
err = followRegistrations(ctx, hub, bus.Call, p.ends)
@@ -579,9 +616,8 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
logger.Printf("a pipeline's end or a registration is not heard, so the wants keep the trunk-only form until the assigner starts again: %v", err)
}
}
if err := hearBuildsAndReleases(hub, &w.ends, &w.released, wake, logger.Printf); err != nil {
logger.Printf("a build's end or a version's release is not heard, so no want names a build's parts, its keeping node or its version until the assigner starts again: %v", err)
}
hearDelivery(ctx, js, w, wake, findings, logger.Printf) // version-released, version-delivered, machine-away and machine-back
hearBuilds(ctx, js, &w.ends, wake, findings, logger.Printf)
hearReadsChanged(ctx, js, hub, reads, findings, gate.EventsLive, logger.Printf)
hearRunning(ctx, js, w, wake, findings, logger.Printf)
}
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/objects.go
+12
View File
@@ -87,3 +87,15 @@ func (b *busObjects) Names(ctx context.Context) ([]string, error) {
}
return names, nil
}
// Delete removes the object of that name; one not held is no error (design 47 G §579: a revision nothing pins is removed).
func (b *busObjects) Delete(ctx context.Context, name string) error {
o, err := b.open(ctx)
if err != nil {
return err
}
if err := o.Delete(ctx, name); err != nil && !errors.Is(err, jetstream.ErrObjectNotFound) {
return err
}
return nil
}
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/outside.go
+1 -1
View File
@@ -269,7 +269,7 @@ func (p *passer) outsideReview(ctx context.Context, module, fp string, nodes []s
dirty := o.dirty[node]
cleared := prev != nil && prev.Cause != "" && o.cleared[node+"|"+prev.Cause]
o.mu.Unlock()
failing, healthy := ok && h.Fp == fp && assign.Failed(h), ok && assign.Healthy(h, fp)
failing, healthy := ok && h.Fp == fp && assign.Failed(h), n.RunsHealthy(node, module, fp)
if !failing && !healthy && prev == nil {
continue
}
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/outside_test.go
+37 -1
View File
@@ -66,6 +66,8 @@ func (r *outsideRig) review(keeperSelf bool, moved func(string, string) bool) as
r.t.Helper()
n := assign.Nodes{Online: map[string]bool{"ace": true}, Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{
"ace": {"nats": {Fp: fpTest, Phase: nodeengineclient.PhaseRunning, Health: r.health}}}}
healthy := r.health == nodeengineclient.HealthOK
n.Running = map[string]map[string]nodeengine.Running{"ace": {"nats": {Node: "ace", Module: "nats", Fingerprint: fpTest, Healthy: &healthy}}}
if moved == nil {
moved = func(string, string) bool { return false }
}
@@ -257,7 +259,7 @@ func TestTheToolsReadAndReleaseAnOutsideWait(t *testing.T) {
r.cond("machine.hub.unreachable", "ace")
r.review(false, nil)
var acts []string
s := newPutTools(memStore{waitsState: &memKV{values: map[string][]byte{}}}, func(key string, body any) (json.RawMessage, error) {
s := newPutTools(memStore{waitsState: &memKV{values: map[string][]byte{}}, buckets.DeliveryWaitsState: &memKV{values: map[string][]byte{}}, buckets.DeliveredAgainState: &memKV{values: map[string][]byte{}}}, func(key string, body any) (json.RawMessage, error) {
acts = append(acts, key)
return json.RawMessage(`{"ok":true,"log":"logged"}`), nil
})
@@ -294,4 +296,38 @@ func TestTheToolsReadAndReleaseAnOutsideWait(t *testing.T) {
if r.bodies[len(r.bodies)-1].How != "go-on" || n.Haves["ace"]["nats"].Health != nodeengineclient.HealthOK {
t.Errorf("last %+v health %s", r.bodies[len(r.bodies)-1], n.Haves["ace"]["nats"].Health)
}
// A-246: a dependent's delivery wait is listed beside them, and released by its module and step delivery, its hand act first.
dw, _ := json.Marshal(deliveryWaiting{Wait: meshwants.Wait{Step: meshwants.WaitStepDelivery, Module: "app", Part: "db", On: "db@1.1.0", Machines: []string{"g14"}}})
s.st.put(buckets.DeliveryWaitsState, "app", dw)
got, err = s.doWaits(map[string]any{"node": "g14"})
if d := got.(map[string]any)["delivery"].([]deliveryWaiting); err != nil || len(d) != 1 || d[0].Module != "app" {
t.Errorf("the delivery waits listed = %+v %v", got, err)
}
if _, err := s.doRelease(map[string]any{"module": "app", "step": "build", "why": "x"}, "shanks.node-tools"); err == nil {
t.Error("a release of another step is taken")
}
if _, err := s.doRelease(map[string]any{"module": "web", "step": "delivery", "why": "x"}, "shanks.node-tools"); err == nil {
t.Error("a release of a module waiting for no delivery is taken")
}
before := len(acts)
if _, err := s.doRelease(map[string]any{"module": "app", "step": "delivery", "why": "db is fine as it runs"}, "shanks.node-tools"); err != nil {
t.Fatal(err)
}
raw, _, _, _ = s.st.get(buckets.DeliveryWaitsState, "app")
var released deliveryWaiting
if json.Unmarshal(raw, &released) != nil || !released.Released || !strings.Contains(released.By, "db is fine") || len(acts) != before+1 {
t.Errorf("the delivery wait after its release = %+v, hand acts %v", released, acts[before:])
}
// A-250: assigner_delivered_again records its hand act, writes the ask and answers asked; assigner_waits lists it.
if _, err := s.doDeliveredAgain(map[string]any{"module": "db", "version": "1.1.0"}, "op"); err == nil {
t.Error("an ask with no why is taken")
}
ans, err := s.doDeliveredAgain(map[string]any{"module": "db", "version": "1.1.0", "why": "the catalog missed it"}, "op")
if err != nil || ans.(map[string]any)["asked"] != "db" || len(acts) != before+2 {
t.Fatalf("assigner_delivered_again = %+v %v", ans, err)
}
got, _ = s.doWaits(map[string]any{})
if a := got.(map[string]any)["delivered-again"].([]againAsk); len(a) != 1 || a[0].Version != "1.1.0" || a[0].By != "op" {
t.Errorf("the asks listed = %+v", a)
}
}
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/outside_tools.go
+108 -1
View File
@@ -1,6 +1,7 @@
package main
import (
"encoding/json"
"errors"
"fmt"
"slices"
@@ -9,7 +10,9 @@ import (
"time"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/keys"
"mesh-delivery-assigner/internal/buckets"
"mesh-delivery-assigner/internal/outside"
@@ -58,7 +61,111 @@ func (s *putTools) doWaits(args map[string]any) (any, error) {
}
out = append(out, said)
}
return map[string]any{"waits": out}, nil
// And each dependent's wait for its dependency's delivery (design 47 G §376, A-246), from the state delivery-waits.
delivery := []deliveryWaiting{}
dkeys, err := s.st.keys(buckets.DeliveryWaitsState)
if err != nil {
return nil, fmt.Errorf("the delivery waits cannot be listed: %w", err)
}
sort.Strings(dkeys)
for _, key := range dkeys {
raw, _, found, err := s.st.get(buckets.DeliveryWaitsState, key)
if err != nil || !found {
continue
}
var w deliveryWaiting
if json.Unmarshal(raw, &w) != nil || (arg(args, "module") != "" && w.Module != arg(args, "module")) ||
(arg(args, "node") != "" && !slices.Contains(w.Machines, arg(args, "node"))) {
continue
}
delivery = append(delivery, w)
}
// And each ask of assigner_delivered_again not yet holding, with why (A-250).
again := []againAsk{}
akeys, err := s.st.keys(buckets.DeliveredAgainState)
if err != nil {
return nil, fmt.Errorf("the asks to say a version delivered again cannot be listed: %w", err)
}
sort.Strings(akeys)
for _, key := range akeys {
raw, _, found, err := s.st.get(buckets.DeliveredAgainState, key)
var a againAsk
if err != nil || !found || json.Unmarshal(raw, &a) != nil || (arg(args, "module") != "" && a.Module != arg(args, "module")) {
continue
}
again = append(again, a)
}
return map[string]any{"waits": out, "delivery": delivery, "delivered-again": again}, nil
}
// toolDeliveredAgain is the tool that asks version-delivered said again for one version (design 47 G §376, A-250).
const toolDeliveredAgain = "assigner_delivered_again"
// doDeliveredAgain is assigner_delivered_again: the hand act is recorded first, then the ask is written into the state
// delivered-again, whose watch the assigner's process hears at once; it answers that it is asked.
func (s *putTools) doDeliveredAgain(args map[string]any, by string) (any, error) {
module, version, why := arg(args, "module"), arg(args, "version"), arg(args, "why")
if module == "" || version == "" || why == "" {
return nil, errors.New("assigner_delivered_again names the module, its version and why: it is kept in the hand-act log")
}
rec := meshhandactsclient.Recorder{Ask: s.ask}
taken, err := rec.Record(meshhandactsclient.Request{What: fmt.Sprintf("assigner_delivered_again %s %s", module, version), Why: why,
Cause: "version-delivered", Verb: "mesh-wants.assigner_delivered_again"})
if err != nil {
return nil, fmt.Errorf("the hand act could not be recorded, so nothing was asked: %w", err)
}
data, err := json.Marshal(againAsk{Module: module, Version: version, By: by, Why: why})
if err != nil {
return nil, err
}
if err := s.st.put(buckets.DeliveredAgainState, keys.EscapeName(module), data); err != nil {
return nil, fmt.Errorf("the ask of %s %s is not written: %w", module, version, err)
}
return map[string]any{"asked": module, "version": version, "hand-act": taken.Log,
"follow": "the assigner says version-delivered again once every present machine placing it runs it healthy; assigner_waits says why it does not yet"}, nil
}
// deliveredAgainTool is assigner_delivered_again as the runtime serves it.
func (s *putTools) deliveredAgainTool() stdio.Tool {
return stdio.Tool{Name: toolDeliveredAgain, Description: "Say version-delivered again for one version of a module, for the operator and the healers " +
"(design 47 G §376): written in the hand-act log first, then asked of the assigner, which says it once every present machine placing " +
"the module runs that version healthy; until then assigner_waits lists the ask with why it does not hold. Answers that it is asked.",
Input: schema(map[string]string{"module": "the module", "version": "its MAJOR.MINOR.PATCH", "why": "why, kept in the hand-act log"},
[]string{"module", "version", "why"}),
Run: func(a map[string]any) (any, error) { return s.doDeliveredAgain(a, stdio.CallerOf(a)) }}
}
// doReleaseDelivery is assigner_release with step delivery: the hand act is recorded first, then the release is written on the
// module's delivery wait, which the assigner's process ends released at its next pass, writing the module's want.
func (s *putTools) doReleaseDelivery(args map[string]any, by string) (any, error) {
module, why := arg(args, "module"), arg(args, "why")
if why == "" {
return nil, errors.New("release says why: it is kept in the hand-act log")
}
key := keys.EscapeName(module)
raw, _, found, err := s.st.get(buckets.DeliveryWaitsState, key)
if err != nil {
return nil, fmt.Errorf("the delivery wait of %s is not read: %w", module, err)
}
var w deliveryWaiting
if !found || json.Unmarshal(raw, &w) != nil || w.Released {
return nil, fmt.Errorf("%s waits for no delivery: assigner_waits lists the delivery waits", module)
}
rec := meshhandactsclient.Recorder{Ask: s.ask}
taken, err := rec.Record(meshhandactsclient.Request{What: fmt.Sprintf("assigner_release %s step delivery (waits on %s)", module, w.On), Why: why,
Cause: "delivery-wait", Verb: "mesh-wants.assigner_release"})
if err != nil {
return nil, fmt.Errorf("the hand act could not be recorded, so nothing was released: %w", err)
}
w.Released, w.By = true, by+": "+why
data, err := json.Marshal(w)
if err != nil {
return nil, err
}
if err := s.st.put(buckets.DeliveryWaitsState, key, data); err != nil {
return nil, fmt.Errorf("the release of %s's delivery wait is not written: %w", module, err)
}
return map[string]any{"released": module, "step": meshwants.WaitStepDelivery, "by": by, "hand-act": taken.Log}, nil
}
// doReleaseWait is assigner_release with how: the hand act is recorded first, then the release is written on the wait of node and
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go
+14 -17
View File
@@ -205,6 +205,8 @@ func (p *passer) clock() time.Time {
func (p *passer) pass(ctx context.Context) {
p.assignmentsSeat(ctx)
p.keepGraph(ctx)
p.judgeDelivered()
p.deliveredAgain(ctx)
p.wants(ctx)
p.puts(ctx)
p.nodeWants(ctx)
@@ -247,7 +249,7 @@ func (p *passer) wants(ctx context.Context) {
views[branch] = v
return v
}
nodes := viewOf(trunkBranch).nodes
nodes := viewOf(p.w.trunkName()).nodes
p.engineReads(nodes)
nodesOf := map[string][]string{}
for node, modules := range assigned {
@@ -303,6 +305,10 @@ func (p *passer) wants(ctx context.Context) {
if fp != "" && !p.ends.mayWant(name, fp, branches, wantedOf[name]) {
return
}
// A dependent's new version waits for its dependency's newest released version to be delivered (§376, A-189).
if p.holdForDelivery(ctx, name, fp, g.branch, g.nodes) {
return
}
m := assign.Module{Name: name, FP: fp, Manifest: manifests[name], Needs: trunk[name].Needs, Chosen: chosenOf(chosen, name), Provisions: provisionsOf(name), Answering: answering(manifests, nodes), Offered: offered(manifests), Nodes: nodesOf[name], Now: time.Now(), Puts: puts[name], FormNeeds: p.w.buildNeeds(name, fp),
Self: func(node string) bool { return placed[node] },
Composed: func(node, fp string) (assign.ComposedDigest, bool) {
@@ -319,8 +325,7 @@ func (p *passer) wants(ctx context.Context) {
Allowed: func(node, digest string) bool {
key, err := asks.RecordAskKey(node, name, digest)
return err == nil && p.answeredWith(key, assign.OptionYes)
},
Release: func(fp string) (string, bool) { return p.w.release(name, fp) }}
}}
if fp != "" {
held, _ := mwc.HeldKey(name, fp)
grant, _ := grantsstate.PartGrantKey(name, fp)
@@ -329,6 +334,7 @@ func (p *passer) wants(ctx context.Context) {
m.Released = p.answeredWith(hold, "release")
m.Kept = p.answeredWith(hold, "keep")
m.Built, m.Held, m.Granted = p.w.built(name, fp), p.w.has(buckets.Wants, held) && !m.Released, p.w.has(grants, grant)
m.BuildFailed, m.BuildError = p.w.buildStanding(name, fp)
if raw := p.w.value(grants, grant); raw != nil {
if g, err := grantsstate.DecodeGrant(raw); err == nil {
m.GrantedOn = func(node string) bool { return assign.GrantNamesNode(g, node, name) }
@@ -404,7 +410,7 @@ func (p *passer) wants(ctx context.Context) {
func (p *passer) movedTo(wanted map[string]map[string]mwc.Want, n assign.Nodes) func(owner, node string) bool {
return func(owner, node string) bool {
w, ok := wanted[owner][node]
return ok && !w.Removed && !assign.Healthy(n.Haves[node][owner], w.Fp)
return ok && !w.Removed && !n.RunsHealthy(node, owner, w.Fp)
}
}
@@ -1060,9 +1066,8 @@ func (p *passer) backupAsk(ctx context.Context, key, node, module string, want m
// yeses writes a placed node's wants naming the build alone, and sees to the moves a node that composes its own judged a
// person must allow (ADR 0547 §3): for a have of phase
// waiting-yes it raises the record ask once, naming the composition waiting and the node's judgement, and on a yes writes
// the want again naming it; a yes once written is kept while the want stays at its build, so the node is not asked again. A
// move of the build alone (the have says so) to a build an approved release carries is given its yes by that Approve, the
// want naming the release, with no ask (ADR 0601 §1).
// the want again naming it; a yes once written is kept while the want stays at its build, so the node is not asked again. No
// release's Approve is a yes: the Approve per merge goes and the assigner reads no `merges` (design 47 G §431).
func (p *passer) yeses(ctx context.Context, module string, m assign.Module, before map[string]mwc.Want, out *assign.Outcome) {
for _, node := range m.Nodes {
if m.Self == nil || !m.Self(node) {
@@ -1091,14 +1096,6 @@ func (p *passer) yeses(ctx context.Context, module string, m assign.Module, befo
switch {
case m.Allowed != nil && m.Allowed(node, h.Waits):
yes, release = h.Waits, ""
case h.BuildAlone && h.Fp != w.Fp && m.Release != nil:
// A move of the build alone, to the build an approved release carries: that Approve is the yes (ADR 0601 §1),
// named in the want. A move that changes a setting or a secret too, or one the node could not tell, is asked.
if r, ok := m.Release(w.Fp); ok {
yes, release = h.Waits, r
break
}
p.recordAsk(ctx, module, w.Fp, assign.RecordNeed{Node: node, Digest: h.Waits, Judgement: h.Judgement, Words: h.Detail})
default:
p.recordAsk(ctx, module, w.Fp, assign.RecordNeed{Node: node, Digest: h.Waits, Judgement: h.Judgement, Words: h.Detail})
}
@@ -1408,14 +1405,14 @@ type group struct {
func (p *passer) groups(module string, nodes []string, placed map[string]string) []group {
by := map[string][]string{}
for _, node := range nodes {
b := trunkBranch
b := p.w.trunkName()
if p.ends.following() {
b = p.ends.branchOf(module, placed[node])
}
by[b] = append(by[b], node)
}
if len(by) == 0 {
b := trunkBranch
b := p.w.trunkName()
if p.ends.following() {
b = p.ends.branchOf(module, "")
}
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go
+44 -96
View File
@@ -16,14 +16,11 @@ import (
"time"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
wdc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshwatchdogsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
mdc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient"
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
mdseat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-teller/tellerevents"
"git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate"
@@ -70,7 +67,9 @@ func (m *memKV) Delete(_ context.Context, key string, _ ...jetstream.KVDeleteOpt
// newPasser is a pass on a fresh world, as after a start, writing to the buckets given, at the clock given.
func newPasser(kvs kvSet, at time.Time) *passer {
return &passer{kvs: kvs, w: newWorld(), logf: func(string, ...any) {}, said: map[string]bool{}, answeredAt: map[string]uint64{},
w := newWorld()
w.setTrunk(trunkBranch)
return &passer{kvs: kvs, w: w, logf: func(string, ...any) {}, said: map[string]bool{}, answeredAt: map[string]uint64{},
now: func() time.Time { return at }}
}
@@ -301,8 +300,8 @@ func TestAnAnswerCountsOnlyForTheAskingItAnswered(t *testing.T) {
return json.Marshal(nodeengine.ManifestReadsAnswer{V: 1, Node: node, ManifestReads: []string{"form:new"}, StatedAt: time.Now().Format(time.RFC3339Nano)})
}
p := &passer{w: w, reads: &enginereads.Keeper{Call: call, Fallback: w.recordReads, Sync: true}}
on, _ := presenceValue(false)
w.put(buckets.Presence, "ace", on)
on := false
w.heardPresence("ace", !on, time.Now())
n = w.nodes(nil, nil)
p.engineReads(n)
if !slices.Equal(n.Reads["ace"], []string{"form:new"}) || len(n.Reads["g14"]) != 0 || asked["ace"] != 1 {
@@ -312,8 +311,8 @@ func TestAnAnswerCountsOnlyForTheAskingItAnswered(t *testing.T) {
t.Errorf("the statements are %+v", s)
}
p.engineReads(w.nodes(nil, nil))
on, _ = presenceValue(true)
w.put(buckets.Presence, "ace", on)
on = true
w.heardPresence("ace", !on, time.Now())
p.engineReads(w.nodes(nil, nil))
if asked["ace"] != 2 {
t.Errorf("ace, back online, was asked %d times, want 2", asked["ace"])
@@ -337,16 +336,17 @@ func TestAnAnswerCountsOnlyForTheAskingItAnswered(t *testing.T) {
Keys: func() ([]string, error) { return slices.Collect(maps.Keys(held)), nil }},
reads: &enginereads.Keeper{Call: silent, Sync: true, Now: func() time.Time { return clock }}}
key := mcc.Key(mcc.ScopeCall, enginereads.CallID("ace"), mcc.KindCallUnanswered)
w.heardPresence("g14", true, time.Now()) // g14 away throughout: a machine is present unless the watchman said it away (§389)
before := asked["ace"]
q.engineReads(w.nodes(nil, nil))
clock = clock.Add(6 * time.Minute)
off, _ := presenceValue(false)
w.put(buckets.Presence, "ace", off)
off := false
w.heardPresence("ace", !off, time.Now())
q.engineReads(w.nodes(nil, nil))
if len(held) != 0 {
t.Errorf("a finding was written while ace is away: %v", slices.Collect(maps.Keys(held)))
}
w.put(buckets.Presence, "ace", on)
w.heardPresence("ace", !on, time.Now())
q.engineReads(w.nodes(nil, nil))
if _, ok := held[key]; !ok || asked["ace"]-before < 3 {
t.Errorf("ace unanswered six minutes, heard: findings %v, asked %d times", slices.Collect(maps.Keys(held)), asked["ace"]-before)
@@ -560,18 +560,14 @@ func putOfTheBus(t *testing.T) {
mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fp, Manifest: mdigest, Repository: "novox/mesh-catalog", Commit: strings.Repeat("c", 40)})
putModuleRaw(p.w, trunkBranch, "nats", mod)
p.w.manifests[mdigest] = parsed{module: "nats", manifest: catalogue.Manifest{Module: "nats", Provides: []catalogue.Offer{{Name: "mesh-bus"}}}}
bkey, _ := builds.BuilderBuildKey("nats", fp)
built, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildDone, Manifest: mdigest})
world(buckets.Builds, bkey, built)
endDone(t, p.w, "nats", fp, mdigest)
hkey, _ := nodeengineclient.HaveKey("novox", "nats")
before, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: meshcatalogclient.SourcePrefix2 + strings.Repeat("a", 64), Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK})
world(buckets.Have, hkey, before)
buildKV := mem()
buildKV.values[bkey] = built
// Issue 1966: newest is asked of the indexer's first view of modules, never read from its bus state.
indexer := &fakeIndexer{views: map[string]map[string][]byte{"mesh-delivery-indexer.modules": {"mesh-catalog": mod, "nats": mod}}}
s := newPutTools(memStore{putsState: putsKV, builds.BuilderModule + ".builds": buildKV}, func(key string, body any) (json.RawMessage, error) {
s := newPutTools(memStore{putsState: putsKV}, func(key string, body any) (json.RawMessage, error) {
if raw, ok, err := indexer.answer(key, body); ok {
return raw, err
}
@@ -582,10 +578,10 @@ func putOfTheBus(t *testing.T) {
answerWithin = 0
defer func() { answerWithin = was }()
if _, err := s.doPut(map[string]any{"node": "novox", "module": "nats", "version": "fd497ab8ffff", "why": "x"}, "shanks.node-tools"); err == nil {
t.Error("a prefix of a fingerprint the builder never built was taken")
t.Error("a prefix of a fingerprint was taken: a put names the whole fingerprint (A-181)")
}
if got, err := s.version("nats", "fd497ab80000"); err != nil || got != fp {
t.Errorf("the prefix of the fingerprint built reads %q, %v", got, err)
if _, err := s.version("nats", "fd497ab80000"); err == nil {
t.Error("the prefix of the fingerprint built was read: a put names the whole fingerprint (A-181)")
}
if got, err := s.version("nats", "newest"); err != nil || got != fp || indexer.asks["mesh-delivery-indexer.modules"] != 2 {
t.Errorf("newest reads %q, %v, asking %d pages of the indexer's first view (want its 2)", got, err, indexer.asks["mesh-delivery-indexer.modules"])
@@ -811,14 +807,14 @@ func testThePassGathersTheModulesOnNeitherList(t *testing.T) {
p := newPasser(kvSet{}, time.Date(2026, 10, 25, 21, 30, 0, 0, time.UTC))
var logged []string
p.logf = func(f string, a ...any) { logged = append(logged, fmt.Sprintf(f, a...)) }
notDone := assign.Outcome{Rules: map[string]assign.Holding{"shanks": {Rule: mwc.RuleBuildDone, Words: "the build of rofi at 4e7b118e is not done"}}}
notDone := assign.Outcome{Rules: map[string]assign.Holding{"shanks": {Rule: mwc.RuleBuildDone, Words: "the build of rofi at 4e7b118e is being built"}}}
wanted := assign.Outcome{Wants: map[string]mwc.Want{"shanks": {Fp: fpTest, Composed: "sha256:" + strings.Repeat("c", 64)}}}
for i, c := range []struct {
before map[string]mwc.Want
out assign.Outcome
line string // the line said in this pass, "" for none
}{
{nil, notDone, "rofi on shanks gets no want: rule 1: the build of rofi at 4e7b118e is not done"},
{nil, notDone, "rofi on shanks gets no want: rule 1: the build of rofi at 4e7b118e is being built"},
{nil, notDone, ""},
{map[string]mwc.Want{"shanks": {Removed: true}}, notDone, ""},
{nil, assign.Outcome{Rules: map[string]assign.Holding{"shanks": {Rule: mwc.RuleGrant, Words: "no credential"}}}, "rofi on shanks gets no want: rule 6: no credential"},
@@ -903,8 +899,8 @@ func TestTheFirstPassAfterAStartClearsWhatAPreviousProcessRaised(t *testing.T) {
mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fpTest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath})
putModuleRaw(p.w, trunkBranch, "web", mod)
placeOn(t, p.w, "g14", "web")
on, _ := presenceValue(true)
p.w.put(buckets.Presence, "g14", on)
on := true
p.w.heardPresence("g14", !on, time.Now())
have, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fpTest, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK})
p.w.put(buckets.Have, "g14.web", have)
key, _ := mwc.WantKey("g14", "web")
@@ -941,8 +937,8 @@ func testAHeldModuleRaisesNoNeitherAndClearsAStandingOne(t *testing.T) {
mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fpTest, Repository: "novox/resolved", Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath})
putModuleRaw(p.w, trunkBranch, "systemd-resolved", mod)
placeOn(t, p.w, "shanks", "systemd-resolved")
on, _ := presenceValue(true)
p.w.put(buckets.Presence, "shanks", on)
on := true
p.w.heardPresence("shanks", !on, time.Now())
have, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fpTest, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed})
p.w.put(buckets.Have, "shanks.systemd-resolved", have)
hkey, _ := mwc.HeldKey("systemd-resolved", fpTest)
@@ -1168,70 +1164,29 @@ func TestARecordAskOfAHeldBackNodeSurvivesPassesAndItsYesMovesTheWant(t *testing
aReleasesApproveAnswersTheMoveOfItsBuildAlone(t)
}
// ADR 0601 §1: a placed node's have waiting for a yes to a move of the build alone, to a build an approved release carries, is
// given its yes by that Approve, the want naming the release, and no record ask is raised; a move that changes a setting or a
// secret too, one at the build it runs, and one no approved release carries are asked of a person as before.
// Design 47 G §431: no release's Approve is a yes (the assigner reads no `merges`): a placed node's have waiting for a yes to a move
// of the build alone is asked of a person, the want naming no release and no yes.
func aReleasesApproveAnswersTheMoveOfItsBuildAlone(t *testing.T) {
fpOld := meshcatalogclient.SourcePrefix2 + strings.Repeat("a", 64)
waiting, was := "sha256:"+strings.Repeat("e", 64), "sha256:"+strings.Repeat("d", 64)
commit := strings.Repeat("1", 40)
for _, c := range []struct {
name string
alone bool
at string // the fingerprint the have runs
mergedAt string // the commit the release merged web at; "" for no merge of the release
fpSrc int64 // the source revision at which web's fingerprint last changed
yes, asked bool
}{
{"the build alone, its release approved", true, fpOld, commit, 7, true, false},
{"a setting changed too", false, fpOld, commit, 7, false, true},
{"a setting changed at the build it runs", true, fpTest, commit, 7, false, true},
{"no approved release carries it", true, fpOld, "", 7, false, true},
{"the release merged another commit", true, fpOld, strings.Repeat("2", 40), 7, false, true},
// Its fingerprint last moved at another release's revision, which merged nothing of web: that Approve answers nothing.
{"a fingerprint another release moved", true, fpOld, commit, 8, false, true},
// Review rounds 1 and 2: moved by a recompute after the release (a toolchain, a package, a base) with no merge, the
// indexer writes fp-src 0, not known, though Src is still the release's: asked of a person.
{"a fingerprint moved by a recompute after the release", true, fpOld, commit, 0, false, true},
} {
asksKV := &memKV{values: map[string][]byte{}}
p := newPasser(kvSet{buckets.Asks: asksKV}, time.Now())
p.logf = t.Logf
mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fpTest, Repository: "novox/web", Commit: commit, Src: 7, FpSrc: c.fpSrc, Path: meshcatalogclient.RootPath})
putModuleRaw(p.w, trunkBranch, "web", mod)
src, _ := mdc.EncodeSource(mdseat.SourceByBranchValue{Group: "feat/web", Heads: map[string]string{"novox/web": commit}})
p.w.putEntry(buckets.Source, mdc.SourceKey(trunkBranch), src, 7, time.Now())
later, _ := mdc.EncodeSource(mdseat.SourceByBranchValue{Group: "feat/other", Heads: map[string]string{"novox/web": commit, "novox/x": commit}})
p.w.putEntry(buckets.Source, mdc.SourceKey(trunkBranch), later, 8, time.Now()) // a later merge leaves revision 7's release
if c.mergedAt != "" {
merge, err := mdc.EncodeMerge(mdseat.MergesByGroupValue{Group: "feat/web", Ask: "merge.feat_web.0123456789ab", Pulls: []mdseat.MergesByGroupValuePullsItem{
{Repository: "novox/web", Number: 3, Head: strings.Repeat("3", 40), State: string(mdc.MergeMerged), Commit: c.mergedAt}}})
if err != nil {
t.Fatal(err)
}
p.w.put(buckets.Merges, "feat/web", merge)
other, err := mdc.EncodeMerge(mdseat.MergesByGroupValue{Group: "feat/other", Ask: "merge.feat_other.0123456789ab", Pulls: []mdseat.MergesByGroupValuePullsItem{
{Repository: "novox/x", Number: 4, Head: strings.Repeat("4", 40), State: string(mdc.MergeMerged), Commit: commit}}})
if err != nil {
t.Fatal(err)
}
p.w.put(buckets.Merges, "feat/other", other)
}
have, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: c.at, Composed: was, Phase: nodeengineclient.PhaseWaitingYes, Health: nodeengineclient.HealthOK,
Judgement: assign.JudgedRecreates, Waits: waiting, Detail: "it recreates web.server", BuildAlone: c.alone})
p.w.put(buckets.Have, "g14.web", have)
m := assign.Module{Name: "web", FP: fpTest, Nodes: []string{"g14"}, Self: func(string) bool { return true },
Release: func(fp string) (string, bool) { return p.w.release("web", fp) }}
out := assign.Outcome{Wants: map[string]mwc.Want{"g14": {Fp: fpTest, Self: true}}}
p.yeses(context.Background(), "web", m, map[string]mwc.Want{"g14": {Fp: fpOld, Self: true}}, &out)
w := out.Wants["g14"]
if got := w.Yes == waiting && w.Release == "feat/web"; got != c.yes || (!c.yes && (w.Yes != "" || w.Release != "")) {
t.Errorf("%s: the want is %+v", c.name, w)
}
akey, _ := asks.RecordAskKey("g14", "web", waiting)
if _, there := asksKV.values[akey]; there != c.asked {
t.Errorf("%s: a record ask raised %v, want %v", c.name, there, c.asked)
}
asksKV := &memKV{values: map[string][]byte{}}
p := newPasser(kvSet{buckets.Asks: asksKV}, time.Now())
p.logf = t.Logf
mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fpTest, Repository: "novox/web", Commit: commit, Src: 7, FpSrc: 7, Path: meshcatalogclient.RootPath})
putModuleRaw(p.w, trunkBranch, "web", mod)
have, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fpOld, Composed: was, Phase: nodeengineclient.PhaseWaitingYes, Health: nodeengineclient.HealthOK,
Judgement: assign.JudgedRecreates, Waits: waiting, Detail: "it recreates web.server", BuildAlone: true})
p.w.put(buckets.Have, "g14.web", have)
m := assign.Module{Name: "web", FP: fpTest, Nodes: []string{"g14"}, Self: func(string) bool { return true }}
out := assign.Outcome{Wants: map[string]mwc.Want{"g14": {Fp: fpTest, Self: true}}}
p.yeses(context.Background(), "web", m, map[string]mwc.Want{"g14": {Fp: fpOld, Self: true}}, &out)
if w := out.Wants["g14"]; w.Yes != "" || w.Release != "" {
t.Errorf("the want is %+v: no release's Approve is a yes", w)
}
akey, _ := asks.RecordAskKey("g14", "web", waiting)
if _, there := asksKV.values[akey]; !there {
t.Error("no record ask was raised for the move of the build alone")
}
}
@@ -1265,8 +1220,8 @@ func aPlacedNodesWantsNameTheBuildAloneAndItsYesComesFromItsHave(t *testing.T) {
placeOn(t, p.w, "g14", "web")
pl, _ := mmac.EncodeAssignments(mma.Assignments{Node: "g14", Modules: []mma.PlacedModule{{Module: "web"}}})
put(plc.AssignmentsBucket, "assignments.g14", pl)
on, _ := presenceValue(true)
put(buckets.Presence, "g14", on)
on := true
p.w.heardPresence("g14", !on, time.Now())
have, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fpOld, Composed: was, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK})
put(buckets.Have, "g14.web", have)
wkey, _ := mwc.WantKey("g14", "web")
@@ -1407,9 +1362,7 @@ func aWantWaitsOnTheVerbEdgesTheGraphKeeps(t *testing.T) {
t.Fatal(err)
}
p.w.manifests[digest] = parsed{module: "server", manifest: parsedServer}
build, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildDone, Manifest: digest})
key, _ := builds.BuilderBuildKey("server", fp("f"))
p.w.put(buckets.Builds, key, build)
endDone(t, p.w, "server", fp("f"), digest)
running := nodeengine.HaveByNodeModuleValue{Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
at := func(f string) nodeengine.HaveByNodeModuleValue { h := running; h.Fp = f; return h }
served := p.served(g, assign.Nodes{Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{
@@ -1450,11 +1403,6 @@ func mustAskID(a asks.Ask) string {
return ask.ID
}
// presenceValue is a node's presence as the watchman writes it.
func presenceValue(online bool) ([]byte, error) {
return json.Marshal(wdc.NodePresence{V: 1, Online: online})
}
// Issue 1966 (nats on novox): a yes given for a record ask was lost because the same pass took the ask away as stale before the
// want named the yes, and an ask taken away and written again was never asked again because it carried the id the router kept.
// Given a record ask of a placed node, then (a) a have mid-move, or running with the operator's yes unwritten, leaves the ask,
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pins.go
+538
View File
@@ -0,0 +1,538 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"slices"
"sort"
"strconv"
"strings"
"sync"
"time"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
mdseat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-assigner/internal/assign"
"mesh-delivery-assigner/internal/assigngraph"
"mesh-delivery-assigner/internal/assignment"
)
// The pins of the assignment graph revisions (design 47 G §153, §178, §216, §579; A-169): the gate says on the seat
// mesh-delivery which revisions a delivery record starts (`pinned`) and stops (`freed`) pointing at, and once at each of its
// starts `pins-stated`. The assigner hears them on its durable consumer of the events stream and keeps, in its old `state`
// section `pins` (4 MiB), one key `trunk.<n>` per assignment graph revision whose value is the list of the records pinning
// it, added or removed by compare-and-set before the event is acknowledged; a record heard twice pins once. Two more keys:
// `made`, written once when the state is first used, and `pins-stated`, the `at` of the last pins-stated applied. A revision
// is removed, its object and its key and the entry objects no kept revision lists and no want or running build names
// (§179), when its list is empty and it is not current (the newest the assigner wrote), only while the consumer is caught up
// and once a pins-stated said after `made` was applied; judged on a freed, on a revision ceasing to be current, on being
// caught up, and at start once both hold. Never on a timer, and never by reading the gate's state.
const (
pinsState = "pins"
pinsBucket = assignment.Module + "_" + pinsState
pinsTool = "assigner_pins"
unpinTool = "assigner_unpin"
pinsMadeKey = "made"
pinsStatedKey = "pins-stated"
pinsCapBytes = 4 << 20
pinMissingKind = "pin-missing"
)
// PinsValue is one value of the state pins: the records that pin the revision, and whether the revision was no longer held
// when a record pinned it.
type PinsValue struct {
Records []string `json:"records"`
Missing bool `json:"missing,omitempty"`
}
// pinKey is the key of an assignment graph revision, its object's own name (A-169).
func pinKey(revision int64) string { return assigngraph.TrunkObject(revision) }
// pinRevision is the revision a key of the state names, and whether it names one.
func pinRevision(key string) (int64, bool) {
n, ok := strings.CutPrefix(key, "trunk.")
if !ok {
return 0, false
}
r, err := strconv.ParseInt(n, 10, 64)
return r, err == nil && r > 0
}
// pinObjects is the object bucket of the revisions as the pins remove from it.
type pinObjects interface {
assigngraph.Objects
Delete(ctx context.Context, name string) error
}
// pinKeeper keeps the pins and removes what nothing pins.
type pinKeeper struct {
ctx context.Context
st endStore
objects pinObjects
// inUse says a want or a running build names the module's fingerprint (§179).
inUse func(module, fp string) bool
findings assign.Findings
now func() time.Time
logf func(string, ...any)
mu sync.Mutex
// caught is whether the last event applied left none of the consumer's messages pending.
caught bool
}
// edit applies change to the value at key by compare-and-set, again while another write comes between.
func (k *pinKeeper) edit(key string, change func(v PinsValue, found bool) (PinsValue, bool)) error {
for {
raw, rev, err := k.st.get(key)
if err != nil {
return err
}
var v PinsValue
if raw != nil {
if err := json.Unmarshal(raw, &v); err != nil {
return fmt.Errorf("the pins of %s do not read: %w", key, err)
}
}
next, write := change(v, raw != nil)
if !write {
return nil
}
value, _ := json.Marshal(next)
if err := k.st.swap(key, value, rev); errors.Is(err, errSwapped) {
continue
} else {
return err
}
}
}
// held says the revision's object is kept.
func (k *pinKeeper) held(revision int64) (bool, error) {
_, err := k.objects.Get(k.ctx, assigngraph.TrunkObject(revision))
if errors.Is(err, assigngraph.ErrNoObject) {
return false, nil
}
return err == nil, err
}
// pin adds record to each assignment graph revision of p; a revision no longer held is written missing, with the finding
// pin-missing naming the record.
func (k *pinKeeper) pin(p mdseat.Pins) error {
for _, r := range p.Assignment {
held, err := k.held(r)
if err != nil {
return err
}
err = k.edit(pinKey(r), func(v PinsValue, _ bool) (PinsValue, bool) {
if slices.Contains(v.Records, p.Record) && (held || v.Missing) {
return v, false
}
if !slices.Contains(v.Records, p.Record) {
v.Records = append(v.Records, p.Record)
slices.Sort(v.Records)
}
v.Missing = v.Missing || !held
return v, true
})
if err != nil {
return err
}
if !held {
k.missing(r, p.Record)
}
}
return nil
}
// free removes record from each assignment graph revision of p; a freed applied twice changes nothing.
func (k *pinKeeper) free(p mdseat.Pins) error {
for _, r := range p.Assignment {
err := k.edit(pinKey(r), func(v PinsValue, found bool) (PinsValue, bool) {
i := slices.Index(v.Records, p.Record)
if !found || i < 0 {
return v, false
}
v.Records = slices.Delete(v.Records, i, i+1)
return v, true
})
if err != nil {
return err
}
}
return nil
}
// missing writes the finding pin-missing, scope the assigner, id the revision.
func (k *pinKeeper) missing(revision int64, record string) {
if k.findings.Put == nil {
k.logf("the revision %d the record %s pins is not held, and no finding writer is wired to say so", revision, record)
return
}
o := assign.Observed{Scope: assignment.Module, ID: strconv.FormatInt(revision, 10), Kind: pinMissingKind,
Summary: fmt.Sprintf("the delivery record %s pins the assignment graph revision %d, which the assigner no longer holds", record, revision),
Headline: "A delivery record pins a revision the assigner no longer holds",
Explanation: fmt.Sprintf("The record %s pinned revision %d after it was removed; %s clears it.", record, revision, unpinTool),
Source: assignment.Module}
if err := k.findings.Put(o.Finding(k.now())); err != nil {
k.logf("the finding %s of revision %d is not written: %v", pinMissingKind, revision, err)
}
}
// made writes the key made once, create-only; one there already stays.
func (k *pinKeeper) made() error {
raw, _, err := k.st.get(pinsMadeKey)
if err != nil || raw != nil {
return err
}
value, _ := json.Marshal(k.now().UTC())
if err := k.st.swap(pinsMadeKey, value, 0); err != nil && !errors.Is(err, errSwapped) {
return err
}
return nil
}
// timeAt is the time a key holds; false when it holds none.
func (k *pinKeeper) timeAt(key string) (time.Time, bool, error) {
raw, _, err := k.st.get(key)
if err != nil || raw == nil {
return time.Time{}, false, err
}
var t time.Time
if err := json.Unmarshal(raw, &t); err != nil {
return t, false, fmt.Errorf("the key %s does not read: %w", key, err)
}
return t, true, nil
}
// stated keeps the at of a pins-stated applied.
func (k *pinKeeper) stated(at time.Time) error {
for {
raw, rev, err := k.st.get(pinsStatedKey)
if err != nil {
return err
}
var was time.Time
if raw != nil && json.Unmarshal(raw, &was) == nil && !at.After(was) {
return nil
}
value, _ := json.Marshal(at.UTC())
if err := k.st.swap(pinsStatedKey, value, rev); errors.Is(err, errSwapped) {
continue
} else {
return err
}
}
}
// statedSinceMade says a pins-stated said after the state was made was applied.
func (k *pinKeeper) statedSinceMade() (bool, error) {
made, ok, err := k.timeAt(pinsMadeKey)
if err != nil || !ok {
return false, err
}
at, ok, err := k.timeAt(pinsStatedKey)
if err != nil || !ok {
return false, err
}
return at.After(made), nil
}
// hear applies one of the seat mesh-delivery's events and acknowledges it only once applied.
func (k *pinKeeper) hear(m jetstream.Msg) {
err := k.apply(m.Subject(), m.Data())
if err != nil {
k.logf("%s: %v; it is heard again", m.Subject(), err)
_ = m.Nak()
return
}
_ = m.Ack()
caught := false
if meta, merr := m.Metadata(); merr == nil {
caught = meta.NumPending == 0
}
k.mu.Lock()
was := k.caught
k.caught = caught
k.mu.Unlock()
if caught && (!was || m.Subject() != mdseat.SubjectPinned) {
k.judge() // on a freed, on a candidate's heads replaced, on pins-stated, and on being caught up
}
}
// apply takes one event's body.
func (k *pinKeeper) apply(subject string, data []byte) error {
switch subject {
case mdseat.SubjectPinned, mdseat.SubjectFreed:
var p mdseat.Pins
if err := json.Unmarshal(data, &p); err != nil || p.Record == "" {
k.logf("%s holds no pins naming a record, and is not taken: %v", subject, err)
return nil
}
if subject == mdseat.SubjectPinned {
return k.pin(p)
}
return k.free(p)
case mdseat.SubjectPinsStated:
var s mdseat.PinsStatedEvent
if err := json.Unmarshal(data, &s); err != nil || s.At.IsZero() {
k.logf("%s holds no at, and is not taken: %v", subject, err)
return nil
}
return k.stated(s.At)
}
return nil // heads-replaced: a candidate's revision stops being current, judged after
}
// judge removes every revision neither pinned nor current, when the rule allows a removal at all.
func (k *pinKeeper) judge() {
k.mu.Lock()
caught := k.caught
k.mu.Unlock()
if !caught {
return
}
ok, err := k.statedSinceMade()
if err != nil {
k.logf("the pins are not judged: %v", err)
return
}
if !ok {
return // no pins-stated applied since the state was made: a pin said before may not be heard yet
}
if err := k.remove(); err != nil {
k.logf("the revisions nothing pins are not removed: %v", err)
}
}
// remove removes each kept revision that is not current and that no record pins, and then the entries left unlisted.
func (k *pinKeeper) remove() error {
names, err := k.objects.Names(k.ctx)
if err != nil {
return err
}
var kept []int64
for _, n := range names {
if r, ok := pinRevision(n); ok {
kept = append(kept, r)
}
}
if len(kept) == 0 {
return nil
}
current := slices.Max(kept)
listed := map[string]bool{}
removed := 0
for _, r := range kept {
raw, _, err := k.st.get(pinKey(r))
if err != nil {
return err
}
var v PinsValue
if raw != nil {
if err := json.Unmarshal(raw, &v); err != nil {
return fmt.Errorf("the pins of %d do not read: %w", r, err)
}
}
if r == current || len(v.Records) > 0 {
rev, err := assigngraph.ReadRevision(k.ctx, k.objects, r)
if err != nil {
return err
}
for _, e := range rev.Entries {
listed[e.Digest] = true
}
continue
}
if err := k.objects.Delete(k.ctx, assigngraph.TrunkObject(r)); err != nil {
return fmt.Errorf("revision %d: %w", r, err)
}
if raw != nil {
if err := k.st.drop(pinKey(r)); err != nil {
return fmt.Errorf("the pins of %d: %w", r, err)
}
}
removed++
}
if removed == 0 {
return nil
}
// A run's candidate kept now lists its entries too, until its group-ended drops it (A-201, A-235).
for _, n := range names {
group, heads, ok := meshwants.ParseAssignmentRevisionsByCandidateGroupHeadsKey(n)
if !ok {
continue
}
rev, err := assigngraph.ReadCandidate(k.ctx, k.objects, group, heads)
if err != nil {
return err
}
for _, e := range rev.Entries {
listed[e.Digest] = true
}
}
for _, n := range names {
digest, ok := strings.CutPrefix(n, "entry.")
if !ok || listed[digest] {
continue
}
e, err := assigngraph.ReadEntry(k.ctx, k.objects, digest)
if err != nil {
return err
}
if k.inUse != nil && k.inUse(e.Module, e.Fp) {
continue
}
if err := k.objects.Delete(k.ctx, n); err != nil {
return fmt.Errorf("the entry %s: %w", digest, err)
}
}
k.logf("%d assignment graph revisions no record pins and not current are removed", removed)
return nil
}
// hearPins registers the seat mesh-delivery's four events on the assigner's hub.
func hearPins(hub *process.EventHub, k *pinKeeper) error {
for _, s := range []string{mdseat.SubjectPinned, mdseat.SubjectFreed, mdseat.SubjectHeadsReplaced, mdseat.SubjectPinsStated} {
if err := hub.Handle(s, k.hear); err != nil {
return fmt.Errorf("%s: %w", s, err)
}
}
return nil
}
// pinsTools are assigner_pins and assigner_unpin over the runtime's states.
func pinsTools(st store, ask func(string, any) (json.RawMessage, error)) []stdio.Tool {
return []stdio.Tool{{Name: pinsTool, Description: "the assignment graph revisions the delivery records pin (design 47 G §579): per revision " +
"the records that pin it and whether it was missing when pinned, the newest revision kept (current, never removed), when the state " +
"was made, the last pins-stated applied, and the state's fill against its cap of 4 MiB.",
Run: func(map[string]any) (any, error) { return readPins(st) }},
{Name: unpinTool, Description: "Remove one record from a revision's pins, for a freed that was lost (design 47 G §579); written in the " +
"hand-act log first. The removal rule then applies.",
Input: schema(map[string]string{"revision": "the assignment graph revision's number", "record": "the delivery record's id",
"why": "why it is removed by hand"}, []string{"revision", "record", "why"}),
Run: func(a map[string]any) (any, error) {
record, _ := a["record"].(string)
why, _ := a["why"].(string)
return unpin(st, ask, fmt.Sprint(a["revision"]), record, why, stdio.CallerOf(a))
}}}
}
// readPins is what assigner_pins answers.
func readPins(st store) (any, error) {
ks, err := st.keys(pinsState)
if err != nil {
return nil, err
}
sort.Strings(ks)
revisions := map[string]PinsValue{}
out := map[string]any{"revisions": revisions}
var newest int64
bytes := 0
for _, key := range ks {
raw, _, found, err := st.get(pinsState, key)
if err != nil || !found {
continue
}
bytes += len(key) + len(raw)
if r, ok := pinRevision(key); ok {
var v PinsValue
if json.Unmarshal(raw, &v) == nil {
revisions[key] = v
}
newest = max(newest, r)
continue
}
out[key] = json.RawMessage(raw)
}
out["fill"] = meshwants.Fill{Store: pinsState, Bytes: int64(bytes), Cap: pinsCapBytes}
if newest > 0 {
out["newest-pinned"] = newest
}
return out, nil
}
// unpin removes one record from one revision's list, the hand act recorded first.
func unpin(st store, ask func(string, any) (json.RawMessage, error), revision, record, why, by string) (any, error) {
r, err := strconv.ParseInt(strings.TrimPrefix(revision, "trunk."), 10, 64)
switch {
case err != nil || r <= 0:
return nil, fmt.Errorf("revision %q is no assignment graph revision's number", revision)
case record == "" || why == "":
return nil, errors.New("name the record and say why")
case by == "" || by == stdio.Unproven:
return nil, errors.New("an unpin names who asks it, and the runtime proved no caller")
}
key := pinKey(r)
raw, _, found, err := st.get(pinsState, key)
if err != nil {
return nil, err
}
var v PinsValue
if found {
if err := json.Unmarshal(raw, &v); err != nil {
return nil, err
}
}
i := slices.Index(v.Records, record)
if i < 0 {
return nil, fmt.Errorf("the record %s does not pin revision %d: %s lists its pins", record, r, pinsTool)
}
taken, err := meshhandactsclient.Recorder{Ask: ask}.Record(meshhandactsclient.Request{What: unpinTool + " " + key + " " + record,
Why: why + " (" + by + ")", Cause: "unpin"})
if err != nil {
return nil, fmt.Errorf("the hand act is not recorded, so nothing is removed: %w", err)
}
v.Records = slices.Delete(v.Records, i, i+1)
value, _ := json.Marshal(v)
if err := st.put(pinsState, key, value); err != nil {
return nil, err
}
return map[string]any{"revision": r, "unpinned": record, "records": v.Records, "hand-act": taken.Log}, nil
}
// watchUnpins judges again when a revision's pins change in the state itself, as assigner_unpin writes them from the tool
// process: the assigner's own state, heard as it changes, never on a timer.
func (k *pinKeeper) watchUnpins(kv jetstream.KeyValue) {
w, err := kv.WatchAll(k.ctx, jetstream.UpdatesOnly())
if err != nil {
k.logf("the state %s is not watched, so a hand unpin is judged at the next freed or revision: %v", pinsState, err)
return
}
go func() {
defer func() { _ = w.Stop() }()
for e := range w.Updates() {
if e == nil {
continue
}
if _, ok := pinRevision(e.Key()); ok && e.Operation() == jetstream.KeyValuePut {
k.judge()
}
}
}()
}
// inUseOf says a want or a running build names the module's fingerprint, as the world holds them now.
func inUseOf(w *world) func(module, fp string) bool {
return func(module, fp string) bool {
for _, want := range w.wantsOf()[module] {
if want.Fp == fp {
return true
}
}
for _, mods := range w.runningNow() {
if r, ok := mods[module]; ok && r.Fingerprint == fp {
return true
}
}
return false
}
}
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pins_test.go
+119
View File
@@ -0,0 +1,119 @@
package main
import (
"context"
"encoding/json"
"testing"
"time"
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
mdseat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"mesh-delivery-assigner/internal/assign"
"mesh-delivery-assigner/internal/assigngraph"
)
func (m memObjects) Delete(_ context.Context, name string) error { delete(m, name); return nil }
// The pins of the assignment graph revisions (design 47 G §153, §178, §579; A-169): a revision two records pin stays until
// both free it; a record pinning twice pins once; the current revision is never removed; nothing is removed before a
// pins-stated said after the state was made, nor while the consumer is behind; a revision no longer held is pinned missing,
// with the finding pin-missing; an entry another kept revision lists, or a want or a running build names, stays.
func TestPinsKeepARevisionWhileARecordPinsIt(t *testing.T) {
objects := memObjects{}
entry := func(module, fp string) string {
e := meshwants.AssignmentEntry{Node: "n1", Module: module, Fp: fp}
d := assigngraph.EntryDigest(e)
raw, _ := json.Marshal(e)
objects[assigngraph.EntryObject(d)] = raw
return d
}
shared, old, running := entry("a", "fp-a"), entry("b", "fp-b1"), entry("c", "fp-c1")
revision := func(n int64, digests ...string) {
rev := meshwants.AssignmentRevision{Revision: n}
for _, d := range digests {
rev.Entries = append(rev.Entries, meshwants.NodeEntryRef{Node: "n1", Digest: d})
}
raw, _ := json.Marshal(rev)
objects[assigngraph.TrunkObject(n)] = raw
}
// A run's candidate lists an entry no trunk revision lists (A-201): it stays while the candidate is kept.
cand := entry("d", "fp-d2")
candRaw, _ := json.Marshal(meshwants.AssignmentRevision{Revision: 1, Kind: "candidate", Group: "run-7", Heads: "h1",
Entries: []meshwants.NodeEntryRef{{Node: "n1", Module: "d", Digest: cand}}})
objects[assigngraph.CandidateObject("run-7", "h1")] = candRaw
revision(1, shared, old, running)
revision(2, shared, old)
revision(3, shared)
var found []mcseat.Finding
clock := time.Date(2026, 11, 12, 10, 0, 0, 0, time.UTC)
k := &pinKeeper{ctx: context.Background(), st: newMemEnds(), objects: objects, now: func() time.Time { return clock },
logf: t.Logf,
inUse: func(module, fp string) bool { return module == "c" && fp == "fp-c1" },
findings: assign.Findings{Put: func(f mcseat.Finding) error { found = append(found, f); return nil }}}
if err := k.made(); err != nil {
t.Fatal(err)
}
say := func(subject string, body any) {
t.Helper()
raw, _ := json.Marshal(body)
if err := k.apply(subject, raw); err != nil {
t.Fatal(err)
}
k.judge()
}
pins := func(record string, revs ...int64) mdseat.Pins { return mdseat.Pins{Record: record, Assignment: revs} }
has := func(n int64) bool { _, ok := objects[assigngraph.TrunkObject(n)]; return ok }
k.caught = true
say(mdseat.SubjectPinned, pins("r1", 1))
say(mdseat.SubjectPinned, pins("r1", 1))
say(mdseat.SubjectPinned, pins("r2", 1))
say(mdseat.SubjectFreed, pins("r9", 2)) // nothing pins 2, but no pins-stated since made: nothing removed
if !has(2) {
t.Fatal("revision 2 was removed before a pins-stated said after the state was made")
}
k.caught = false
say(mdseat.SubjectPinsStated, mdseat.PinsStatedEvent{At: clock.Add(time.Minute)})
if !has(2) {
t.Fatal("revision 2 was removed while the consumer was behind")
}
k.caught = true
say(mdseat.SubjectFreed, pins("r1", 1))
if has(2) {
t.Error("revision 2, neither pinned nor current, is kept")
}
if !has(1) || !has(3) {
t.Error("revision 1 (pinned by r2) or 3 (current) was removed")
}
raw, _, _ := k.st.get(pinKey(1))
if string(raw) != `{"records":["r2"]}` {
t.Errorf("the pins of 1 = %s, want r2 alone (r1 pinned twice pins once, and freed once)", raw)
}
say(mdseat.SubjectFreed, pins("r2", 1))
if has(1) {
t.Error("revision 1 is kept once both records freed it")
}
if _, ok := objects[assigngraph.EntryObject(old)]; ok {
t.Error("the entry no kept revision lists and nothing names is kept")
}
if _, ok := objects[assigngraph.EntryObject(running)]; !ok {
t.Error("the entry of a running build is removed")
}
if _, ok := objects[assigngraph.EntryObject(shared)]; !ok {
t.Error("the entry the current revision lists is removed")
}
if _, ok := objects[assigngraph.EntryObject(cand)]; !ok {
t.Error("the entry a run's kept candidate lists is removed")
}
say(mdseat.SubjectHeadsReplaced, mdseat.HeadsReplacedEvent{})
if _, ok := objects[assigngraph.CandidateObject("run-7", "h1")]; !ok {
t.Error("the gate's heads-replaced dropped a run's candidate")
}
say(mdseat.SubjectPinned, pins("r3", 2))
raw, _, _ = k.st.get(pinKey(2))
if string(raw) != `{"records":["r3"],"missing":true}` || len(found) != 1 || found[0].Kind != pinMissingKind || found[0].ID != "2" {
t.Errorf("a pin of a revision no longer held = %s, findings %+v", raw, found)
}
}
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines.go
+21 -2
View File
@@ -151,8 +151,12 @@ type pipelineEnds struct {
// firsts is, by module, the first watched branch of its registration, the branch of a placement that names none (design
// 47 G §14).
firsts map[string]string
// runEnded is told a run on request's group-ended, its group `run-<id>`, to drop the run's candidate (A-235); nil drops nothing.
runEnded func(group string) error
// onBranches is told the branches a registration names, for the catalog half of each to be followed (design 47 G §14, §195).
onBranches func([]string)
// trunk names the first watched branch, the trunk's (design 47 G §507), the branch of a module registered with none.
trunk func() string
}
func newPipelineEnds(st endStore, wake func(), logf func(string, ...any)) *pipelineEnds {
@@ -225,7 +229,10 @@ func (e *pipelineEnds) branchOf(module, placed string) string {
if b := e.firsts[module]; b != "" {
return b
}
return trunkBranch
if e.trunk != nil {
return e.trunk()
}
return ""
}
// hearPipeline takes a pipeline-ended. A pipeline of a group writes nothing on its own end, and a run on request delivers
@@ -246,7 +253,8 @@ func (e *pipelineEnds) hearPipeline(m jetstream.Msg) {
_ = m.Ack()
}
// hearGroup takes a group-ended: each member's end, passed only if the group passed.
// hearGroup takes a group-ended: each member's end, passed only if the group passed. A run on request's (group `run-<id>`) delivers
// nothing: it drops the run's candidate assignment graph (A-235).
func (e *pipelineEnds) hearGroup(m jetstream.Msg) {
g, err := mprc.GroupEnded(m.Data())
if err != nil {
@@ -254,6 +262,17 @@ func (e *pipelineEnds) hearGroup(m jetstream.Msg) {
_ = m.Ack()
return
}
if strings.HasPrefix(g.Group, runGroupPrefix) {
if e.runEnded != nil {
if err := e.runEnded(g.Group); err != nil {
e.logf("%s: %v; it is heard again", m.Subject(), err)
_ = m.Nak()
return
}
}
_ = m.Ack()
return
}
for _, p := range g.Pipelines {
if err := e.take(p, g.Outcome == mpr.GroupEndedOutcomePassed && requiredPassed(p)); err != nil {
e.logf("%s: %v; it is heard again", m.Subject(), err)
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines_test.go
+52 -11
View File
@@ -20,8 +20,8 @@ import (
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate"
@@ -55,6 +55,19 @@ func (m *memEnds) swap(key string, v []byte, rev uint64) error {
return nil
}
// heardMsg is one message as a handler of the hub is handed it, saying whether it was acknowledged or given back.
type heardMsg struct {
jetstream.Msg
data []byte
subject string
acked, naked bool
}
func (m *heardMsg) Data() []byte { return m.data }
func (m *heardMsg) Subject() string { return m.subject }
func (m *heardMsg) Ack() error { m.acked = true; return nil }
func (m *heardMsg) Nak() error { m.naked = true; return nil }
func pipe(module, branch string, source, count int64, fp, outcome string, steps ...mpr.StepResult) mpr.PipelineEnded {
return mpr.PipelineEnded{Module: module, Branch: branch, Source: source, EndCount: count, Fingerprint: fp, Outcome: outcome, Steps: steps,
Pipeline: branch + "." + module, Commit: "c"}
@@ -112,6 +125,35 @@ func TestAWantFollowsAPassedPipeline(t *testing.T) {
t.Errorf("%s: may want = %v, want %v", c.name, got, c.may)
}
}
// A run on request's group-ended (group run-<id>, A-235) writes no want and drops the run's candidate; a drop that fails is
// heard again.
{
e := newPipelineEnds(newMemEnds(), func() {}, t.Logf)
e.on.Store(true)
var dropped []string
fail := false
e.runEnded = func(group string) error {
dropped = append(dropped, group)
if fail {
return fmt.Errorf("the bucket is not open")
}
return nil
}
raw, _ := json.Marshal(mpr.GroupEnded{V: 1, Branch: "main", Group: "run-7", Outcome: mpr.GroupEndedOutcomePassed,
Pipelines: []mpr.PipelineEnded{func() mpr.PipelineEnded { p := pipe("web", "main", 5, 1, fp1, pass); p.Run = "7"; return p }()}})
m := &heardMsg{data: raw, subject: "mesh.seat.mesh-pipeline-runner.event.group-ended.x"}
e.hearGroup(m)
if e.mayWant("web", fp1, []string{"main"}, nil) || !slices.Equal(dropped, []string{"run-7"}) || !m.acked {
t.Errorf("a run's group-ended: may want %v, dropped %v, acked %v; want no want, run-7 dropped, acked",
e.mayWant("web", fp1, []string{"main"}, nil), dropped, m.acked)
}
fail = true
m = &heardMsg{data: raw, subject: m.subject}
e.hearGroup(m)
if !m.naked {
t.Error("a run's candidate not dropped is not heard again")
}
}
// A want that names the fingerprint already is not held by a later failure (a node placed afterwards gets it).
e := newPipelineEnds(newMemEnds(), func() {}, t.Logf)
e.on.Store(true)
@@ -154,9 +196,7 @@ func TestAPassWritesNoWantWithoutAPassedPipeline(t *testing.T) {
mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fp, Manifest: mdigest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath})
putModuleRaw(p.w, trunkBranch, "web", mod)
p.w.manifests[mdigest] = parsed{module: "web", manifest: catalogue.Manifest{Module: "web"}}
bkey, _ := builds.BuilderBuildKey("web", fp)
built, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildDone, Manifest: mdigest})
p.w.put(buckets.Builds, bkey, built)
endDone(t, p.w, "web", fp, mdigest)
gkey, _ := grantsstate.PartGrantKey("web", fp)
grant, _ := grantsstate.EncodeGrant(grantsstate.Grant{})
p.w.put(buckets.Grants, gkey, grant)
@@ -165,10 +205,11 @@ func TestAPassWritesNoWantWithoutAPassedPipeline(t *testing.T) {
p.w.put(buckets.Have, "g14.web", have)
composes(t, p.w, "g14", digest)
placeOn(t, p.w, "g14", "web")
on, _ := presenceValue(true)
p.w.put(buckets.Presence, "g14", on)
on := true
p.w.heardPresence("g14", !on, time.Now())
if c.follow {
p.ends = newPipelineEnds(newMemEnds(), func() {}, t.Logf)
p.ends.trunk = p.w.trunkName
p.ends.on.Store(true)
if c.passed {
_ = p.ends.take(pipe("web", trunkBranch, 5, 1, fp, mpr.PipelineEndedOutcomePassed), true)
@@ -196,6 +237,7 @@ func TestTheWantsSwitchOnTheFirstRegistration(t *testing.T) {
st := newMemEnds()
woken := 0
e := newPipelineEnds(st, func() { woken++ }, t.Logf)
e.trunk = func() string { return trunkBranch }
_ = e.take(pipe("web", "main", 5, 1, "fp1", mpr.PipelineEndedOutcomeFailed), false)
_ = e.take(pipe("web", "beta", 5, 1, "fp1", mpr.PipelineEndedOutcomePassed), true)
if e.following() || !e.mayWant("web", "fp1", []string{"main"}, nil) {
@@ -225,12 +267,13 @@ func TestACommitOnBetaMovesOnlyTheNodeThatChoseBeta(t *testing.T) {
p := newPasser(kvSet{buckets.Wants: wantsKV, buckets.Asks: &memKV{values: map[string][]byte{}}}, time.Now())
p.say = func(oc.Condition) error { return nil }
p.ends = newPipelineEnds(newMemEnds(), func() {}, t.Logf)
p.ends.trunk = p.w.trunkName
p.ends.registered(mpcat.Registration{Module: "web", Branches: []string{"main", "beta"}}, false)
digest := "sha256:" + strings.Repeat("c", 64)
on, _ := presenceValue(true)
on := true
for _, node := range []string{"g14", "ace"} {
composes(t, p.w, node, digest)
p.w.put(buckets.Presence, node, on)
p.w.heardPresence(node, !on, time.Now())
}
for node, branch := range map[string]string{"g14": "", "ace": "beta"} {
pl := mma.Assignments{Node: node, Modules: []mma.PlacedModule{{Module: "web", Branch: branch}}}
@@ -251,9 +294,7 @@ func TestACommitOnBetaMovesOnlyTheNodeThatChoseBeta(t *testing.T) {
mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: f, Manifest: mdigest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath})
putModuleRaw(p.w, branch, "web", mod)
p.w.manifests[mdigest] = parsed{module: "web", manifest: catalogue.Manifest{Module: "web"}}
bkey, _ := builds.BuilderBuildKey("web", f)
built, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildDone, Manifest: mdigest})
p.w.put(buckets.Builds, bkey, built)
endDone(t, p.w, "web", f, mdigest)
gkey, _ := grantsstate.PartGrantKey("web", f)
grant, _ := grantsstate.EncodeGrant(grantsstate.Grant{})
p.w.put(buckets.Grants, gkey, grant)
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/puts.go
+3 -9
View File
@@ -14,7 +14,6 @@ import (
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient"
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate"
@@ -107,6 +106,7 @@ func (p *passer) puts(ctx context.Context) {
}
p.placementPuts(ctx)
all := p.putsByModule()
defer func() { p.putWaits(ctx, p.putsByModule()) }()
if len(all) == 0 {
return
}
@@ -184,14 +184,8 @@ func (p *passer) putWorld(node, module string, put puts.Put, manifest catalogue.
w.Assigned = true
}
}
if bkey, err := builds.BuilderBuildKey(module, put.FP); err == nil {
if raw := p.w.value(buckets.Builds, bkey); raw != nil {
if b, err := builds.DecodeBuilderBuild(raw); err == nil {
w.Build, w.BuildWords = b.State, b.Error
} else {
w.Build = "unreadable: " + err.Error()
}
}
if e, ok := p.w.ends.endOf(module, put.FP); ok { // the build's end heard (A-181): none heard, no state
w.Build, w.BuildWords = e.Outcome, e.Error
}
w.Composed, w.Granted = p.composedFor(node, module, put.FP)
if d := w.Composed; d != nil {
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/putwaits.go
+116
View File
@@ -0,0 +1,116 @@
package main
import (
"context"
"encoding/json"
"fmt"
"time"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"mesh-delivery-assigner/internal/assignment"
"mesh-delivery-assigner/internal/buckets"
"mesh-delivery-assigner/internal/puts"
)
// A put without hold that stands is said as a wait (design 47 G §498, main file *A put without hold that stands*): when its
// want is written the assigner says `wait-held` on the seat mesh-wants, step put, with the node, the module, the fingerprint
// and since when, bound 30 minutes; and `wait-ended` when it ends: ran when its node runs it, released by assigner_release,
// superseded by a put asked again under its key. The watchman times it; a held put waits on nobody and is said as none.
// putWaitBound is the bound of a put without hold's wait.
const putWaitBound = 30 * time.Minute
// Wait-ended's how of a put's wait.
const (
putWaitRan = "ran"
putWaitReleased = "released"
putWaitSuperseded = "superseded"
)
// putWaits says whether p is a put that waits: one without hold whose want is written and not yet run.
func putWaits(p puts.Put) bool {
return !p.Hold && p.Step == puts.Wanted
}
// putWaitBody is the wait of the put p of module on node.
func putWaitBody(node, module string, p puts.Put) meshwants.Wait {
since := p.StandsSince().UTC()
return meshwants.Wait{Step: meshwants.WaitStepPut, Owner: assignment.Module, Node: node, Module: module, Fingerprint: p.FP,
Since: since, ClockFrom: &since, Bound: putWaitBound.String(),
Words: fmt.Sprintf("the put of %s at %s on %s waits for %s to run it", module, shortFP(p.FP), node, node)}
}
// putWaiting is a put whose wait-held was said, as the state put-waits keeps it under the put's key (design 47 G A-204,
// A-211): its identity and the put as it was then.
type putWaiting struct {
ID string `json:"id"`
Put puts.Put `json:"put"`
}
// saidPutWaits is every put wait the state put-waits holds, by key; a value that does not decode is said and left.
func (p *passer) saidPutWaits() map[string]putWaiting {
out := map[string]putWaiting{}
for key, value := range p.w.read(buckets.PutWaits) {
var w putWaiting
if err := json.Unmarshal(value, &w); err != nil || w.ID == "" {
p.logf("%s %s holds no put wait, left: %v", buckets.PutWaits, key, err)
continue
}
out[key] = w
}
return out
}
// putWaits says wait-ended of each put whose said wait the state put-waits keeps and that no longer waits, whether it ended
// while this process ran or while the assigner was stopped (A-204), and wait-held of each put that waits and was not said;
// one not said, or not kept, is said at the next pass. Without the state no put's wait is said.
func (p *passer) putWaits(ctx context.Context, all map[string]map[string]puts.Put) {
if p.kvs[buckets.PutWaits] == nil {
return
}
waiting := p.saidPutWaits()
for key, was := range waiting {
node, module, err := puts.ParseKey(key)
if err != nil {
continue
}
cur, there := all[module][node]
same := there && identity(key, cur) == was.ID
if same && putWaits(cur) {
continue
}
how := putWaitSuperseded
if same && cur.Step == puts.Done {
how = putWaitRan
} else if same && cur.Step == puts.Released {
how = putWaitReleased
}
body := putWaitBody(node, module, was.Put)
body.How = how
if err := p.wantsSay(meshwants.EventWaitEnded, body, was.ID+".put."+how); err != nil {
p.logf("the end of the put %s's wait is not said, and is said at the next pass: %v", key, err)
continue
}
p.remove(ctx, buckets.PutWaits, key)
delete(waiting, key)
}
for module, byNode := range all {
for node, put := range byNode {
key, err := puts.Key(node, module)
if err != nil || !putWaits(put) {
continue
}
id := identity(key, put)
if w, said := waiting[key]; said && w.ID == id {
continue
}
if err := p.wantsSay(meshwants.EventWaitHeld, putWaitBody(node, module, put), id+".put"); err != nil {
p.logf("the put %s's wait is not said, and is said at the next pass: %v", key, err)
continue
}
kept := putWaiting{ID: id, Put: put}
p.put(ctx, buckets.PutWaits, key, func() ([]byte, error) { return json.Marshal(kept) })
}
}
}
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/putwaits_test.go
+92
View File
@@ -0,0 +1,92 @@
package main
import (
"context"
"testing"
"time"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"mesh-delivery-assigner/internal/buckets"
"mesh-delivery-assigner/internal/puts"
)
// Design 47 G §498 and the main file's *A put without hold that stands*: a put without hold whose want is written is said as
// wait-held, step put, with its node, module, fingerprint, since and a 30-minute bound, and wait-ended when it runs, is
// released or is put again; a held put is said as no wait; assigner_puts answers since and bound. A-204, A-211: the said
// waits are kept in the state put-waits, so a put that waits at the start was said before and is not said again, and one
// that ran while the assigner was stopped gets its wait-ended at the next start.
func TestAPutWithoutHoldIsSaidAsAWait(t *testing.T) {
at := time.Date(2026, 11, 12, 8, 0, 0, 0, time.UTC)
kept := &memKV{values: map[string][]byte{}}
var events []string
var bodies []meshwants.Wait
start := func() *passer { // a process's start: a fresh world that hands what the state holds
p := newPasser(kvSet{buckets.PutWaits: kept}, at)
for k, v := range kept.values {
p.w.put(buckets.PutWaits, k, v)
}
p.acts = newPutActs(&fakeBus{calls: make(chan string, 8)}, func() {}, func(string, ...any) {})
p.publishWants = func(event string, body any, id string) error {
events, bodies = append(events, event), append(bodies, body.(meshwants.Wait))
return nil
}
return p
}
ctx := context.Background()
asked := puts.Put{FP: fpTest, Why: "a try", At: at}.Said(at, puts.Asked, "asked")
wanted := asked.Said(at.Add(time.Minute), puts.Wanted, "the want is written").Said(at.Add(2*time.Minute), puts.Wanted, "its check failed")
held := wanted
held.Hold = true
p := start()
p.putWaits(ctx, map[string]map[string]puts.Put{"nats": {"ace": wanted}})
if len(events) != 1 || events[0] != meshwants.EventWaitHeld || kept.values["ace.nats"] == nil {
t.Fatalf("a put that waits said %v, kept %q", events, kept.values)
}
events, bodies = nil, nil
// At the next start the put still waits: it was said before, nothing is said of it.
p = start()
p.putWaits(ctx, map[string]map[string]puts.Put{"nats": {"ace": wanted}})
if len(events) != 0 {
t.Fatalf("a put waiting at the start was said again: %v", events)
}
// It ran while the assigner was stopped: at the next start, wait-ended, ran, and the state forgets it.
p = start()
p.putWaits(ctx, map[string]map[string]puts.Put{"nats": {"ace": wanted.Said(at.Add(3*time.Minute), puts.Done, "runs")}})
if len(events) != 1 || events[0] != meshwants.EventWaitEnded || bodies[0].How != putWaitRan || bodies[0].Step != meshwants.WaitStepPut ||
kept.values["ace.nats"] != nil {
t.Fatalf("a put that ran while stopped said %v %+v, kept %q", events, bodies, kept.values)
}
events, bodies = nil, nil
// A new put's want is written: wait-held, since the want was written, bound 30m; a held put says nothing.
again := wanted
again.At = at.Add(time.Hour)
p.putWaits(ctx, map[string]map[string]puts.Put{"nats": {"ace": again}, "web": {"ace": held}})
if len(events) != 1 || events[0] != meshwants.EventWaitHeld {
t.Fatalf("a put's written want said %v", events)
}
b := bodies[0]
if b.Node != "ace" || b.Module != "nats" || b.Fingerprint != fpTest || !b.Since.Equal(at.Add(time.Minute)) || b.ClockFrom == nil ||
!b.ClockFrom.Equal(b.Since) || b.Bound != "30m0s" || b.Owner != "mesh-delivery-assigner" {
t.Errorf("the put's wait is %+v", b)
}
// Said once.
p.putWaits(ctx, map[string]map[string]puts.Put{"nats": {"ace": again}, "web": {"ace": held}})
if len(events) != 1 {
t.Errorf("a put's wait was said again: %v", events)
}
// Put again under its key: the first wait ends superseded, the new one is held.
newer := asked
newer.At = at.Add(2 * time.Hour)
p.putWaits(ctx, map[string]map[string]puts.Put{"nats": {"ace": newer}})
if len(events) != 2 || events[1] != meshwants.EventWaitEnded || bodies[1].How != putWaitSuperseded || bodies[1].Fingerprint != fpTest {
t.Errorf("a put asked again said %v %+v", events, bodies)
}
// The answer of assigner_puts.
if a := answerOf("ace", "nats", wanted); !a.Since.Equal(at.Add(time.Minute)) || a.Bound != "30m0s" {
t.Errorf("a put that waits answers since %v, bound %q", a.Since, a.Bound)
}
if a := answerOf("ace", "web", held); !a.Since.Equal(at.Add(time.Minute)) || a.Bound != "" {
t.Errorf("a held put answers since %v, bound %q", a.Since, a.Bound)
}
}
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tools.go
+38 -36
View File
@@ -17,9 +17,9 @@ import (
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/catalogkey"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/seatlist"
@@ -72,14 +72,11 @@ func (s *putTools) hear(key string, value []byte) {
s.mu.Unlock()
}
// fullFP is a fingerprint of the new form, whole; prefix at least its first twelve hex digits.
var (
fullFP = regexp.MustCompile(`^src2:[0-9a-f]{64}$`)
prefix = regexp.MustCompile(`^(src2:)?[0-9a-f]{12,63}$`)
)
// fullFP is a fingerprint of the new form, whole.
var fullFP = regexp.MustCompile(`^src2:[0-9a-f]{64}$`)
// version reads the version a put names: newest (the indexer's fingerprint at the trunk), a whole fingerprint, or the start
// of one the builder built, which must name one build alone.
// version reads the version a put names: newest (the indexer's fingerprint at the trunk) or a whole fingerprint (design 47 G A-181:
// a put names the whole fingerprint, the builder's builds being no bucket to list).
func (s *putTools) version(module, version string) (string, error) {
v := strings.ToLower(strings.TrimSpace(version))
switch {
@@ -103,29 +100,8 @@ func (s *putTools) version(module, version string) (string, error) {
return m.Fp2, nil
case fullFP.MatchString(v):
return v, nil
case prefix.MatchString(v):
start := strings.TrimPrefix(v, meshcatalogclient.SourcePrefix2)
keys, err := s.st.keys(builds.BuilderModule + ".builds")
if err != nil {
return "", fmt.Errorf("the builder's builds cannot be listed: %w", err)
}
var found []string
for _, k := range keys {
m, fp, err := builds.ParseBuilderBuildKey(k)
if err == nil && m == module && strings.HasPrefix(strings.TrimPrefix(fp, meshcatalogclient.SourcePrefix2), start) {
found = append(found, fp)
}
}
switch len(found) {
case 0:
return "", fmt.Errorf("the builder built no %s whose fingerprint starts %s: give it whole", module, start)
case 1:
return found[0], nil
}
sort.Strings(found)
return "", fmt.Errorf("%s starts %d fingerprints of %s the builder built (%s): give more of it", start, len(found), module, strings.Join(found, ", "))
}
return "", fmt.Errorf("%q is no version: newest, a fingerprint (src2: and 64 hex digits) or at least its first 12 hex digits", version)
return "", fmt.Errorf("%q is no version: newest, a fingerprint (src2: and 64 hex digits) whole", version)
}
// putAnswer is what assigner_put and assigner_puts answer of one put.
@@ -137,6 +113,19 @@ type putAnswer struct {
// Older says a put that has not ended stands at an older version than the newest build, and which that is: it pins the
// node's want until it ends or is released.
Older string `json:"older-than-newest,omitempty"`
// Since is when the put came to stand at its step, held or not; Bound is the bound of a put without hold whose want is
// written, the wait the watchman times and seeds from here (design 47 G §498, main file *A put without hold that stands*).
Since time.Time `json:"since"`
Bound string `json:"bound,omitempty"`
}
// answerOf is what assigner_put and assigner_puts answer of the put p of module on node.
func answerOf(node, module string, p puts.Put) putAnswer {
a := putAnswer{Node: node, Module: module, Put: p, Since: p.StandsSince()}
if putWaits(p) {
a.Bound = putWaitBound.String()
}
return a
}
// doPut is the tool assigner_put.
@@ -168,7 +157,9 @@ func (s *putTools) doPut(args map[string]any, by string) (any, error) {
} else if found {
if before, err := puts.Decode(raw); err == nil && !puts.Ended(before.Step) {
if before.FP == fp && before.Hold == hold {
return putAnswer{Node: node, Module: module, Put: before, Follow: followPut(node, module)}, nil
a := answerOf(node, module, before)
a.Follow = followPut(node, module)
return a, nil
}
words += fmt.Sprintf("; it replaces the put of %s at %s, which stood at %s", module, shortFP(before.FP), before.Step)
}
@@ -182,7 +173,9 @@ func (s *putTools) doPut(args map[string]any, by string) (any, error) {
return nil, fmt.Errorf("the put of %s on %s is not written: %w", module, node, err)
}
s.hear(key, data)
return putAnswer{Node: node, Module: module, Put: s.await(key, p), Follow: followPut(node, module)}, nil
a := answerOf(node, module, s.await(key, p))
a.Follow = followPut(node, module)
return a, nil
}
// await is the put at key once the assigner's process took it up past its asking, or as it stands after answerWithin.
@@ -217,6 +210,12 @@ func (s *putTools) doRelease(args map[string]any, by string) (any, error) {
if arg(args, "how") != "" {
return s.doReleaseWait(args, by) // a wait on an outside cause (outside_tools.go, design 47 G §706)
}
if step := arg(args, "step"); step != "" {
if step != meshwants.WaitStepDelivery {
return nil, fmt.Errorf("step is %s, the one wait released by its step, not %q", meshwants.WaitStepDelivery, step)
}
return s.doReleaseDelivery(args, by) // a dependent's delivery wait (delivered.go, design 47 G §376, A-246)
}
node, module, why := arg(args, "node"), arg(args, "module"), arg(args, "why")
key, err := puts.Key(node, module)
if err != nil {
@@ -248,7 +247,7 @@ func (s *putTools) doRelease(args map[string]any, by string) (any, error) {
return nil, fmt.Errorf("the release of %s on %s is not written: %w", module, node, err)
}
s.hear(key, data)
return putAnswer{Node: node, Module: module, Put: p}, nil
return answerOf(node, module, p), nil
}
// doPuts is the tool assigner_puts.
@@ -274,7 +273,7 @@ func (s *putTools) doPuts(args map[string]any) (any, error) {
if err != nil {
continue
}
a := putAnswer{Node: node, Module: module, Put: p}
a := answerOf(node, module, p)
if !puts.Ended(p.Step) {
if !asked {
asked = true
@@ -349,10 +348,12 @@ func (s *putTools) tools() []stdio.Tool {
"rules write the machine's want again from their next pass. Nothing changes on the machine until they move it.",
Input: schema(map[string]string{"node": "the machine", "module": "the module", "why": "why, kept with the put",
"how": "with how, the wait of the delivery on an outside cause (assigner_waits) is released, not a put: go-on (the module counts healthy there), " +
"failed (the failing check is the delivery's) or rejudge (judge it again now); written in the hand-act log first"},
[]string{"node", "module", "why"}),
"failed (the failing check is the delivery's) or rejudge (judge it again now); written in the hand-act log first",
"step": "delivery: the module's wait for its dependency's delivery (assigner_waits) is released, its want written now; no machine is named; written in the hand-act log first"},
[]string{"module", "why"}),
Run: func(a map[string]any) (any, error) { return s.doRelease(a, stdio.CallerOf(a)) }},
s.waitsTool(),
s.deliveredAgainTool(),
{Name: toolPuts, Description: "Every put and where it stands (novox/hq ADR 0546): its version, hold, why, each step with " +
"its time and words. Narrowed by machine or module. Read-only.",
Input: schema(map[string]string{"node": "one machine", "module": "one module"}, nil),
@@ -417,6 +418,7 @@ func serveTools() error {
// What each node's engine reads, as the assigner uses it (ADR 0618 §3): its own state engine-reads, asked again through it.
tools = append(tools, enginereads.Tool(engineReadsTool, "the assigner", enginereads.KeptState(stdio.State(engineReadsState)), false, time.Now))
tools = append(tools, writtenTools(runtimeStore{})...)
tools = append(tools, pinsTools(runtimeStore{}, stdio.Ask)...)
tools = append(tools, askAgainTools(runtimeStore{}, stdio.Ask, time.Now)...)
return stdio.Serve("", tools)
}
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/verbs.go
+5 -14
View File
@@ -1,7 +1,6 @@
package main
import (
"mesh-delivery-assigner/internal/buckets"
"slices"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
@@ -94,22 +93,14 @@ func (p *passer) servesOf(module, fp string) ([]string, bool) {
return graph.ServesOf(m), true
}
// manifestOf is the manifest of module's build at fp (the builder's build names its digest in the indexer's manifests),
// and whether it could be read.
// manifestOf is the manifest of module's build at fp (the build's end names its digest, A-181, which the indexer's manifests
// hold), and whether it could be read.
func (p *passer) manifestOf(module, fp string) (catalogue.Manifest, bool) {
key, err := builds.BuilderBuildKey(module, fp)
if err != nil {
e, ok := p.w.ends.endOf(module, fp)
if !ok || e.Outcome != builds.BuildDone || e.Manifest == "" {
return catalogue.Manifest{}, false
}
raw := p.w.value(buckets.Builds, key)
if raw == nil {
return catalogue.Manifest{}, false
}
b, err := builds.DecodeBuilderBuild(raw)
if err != nil || b.Manifest == "" {
return catalogue.Manifest{}, false
}
got := p.w.manifest(b.Manifest)
got := p.w.manifest(e.Manifest)
if got.err != "" {
return catalogue.Manifest{}, false
}
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/views.go
+28 -1
View File
@@ -3,15 +3,19 @@ package main
import (
"context"
"encoding/json"
"fmt"
"mesh-delivery-assigner/internal/assignment"
"mesh-delivery-assigner/internal/buckets"
"slices"
"sync/atomic"
"time"
"github.com/nats-io/nats.go/jetstream"
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
mcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/process"
@@ -122,6 +126,12 @@ type catalogViews struct {
hub *process.EventHub
h *halves
// candidates is told each candidate's catalog-revision (A-201), once the passer that keeps its assignment graph is made;
// before that a candidate's revision is said not taken.
candidates atomic.Pointer[func(context.Context, meshcatalog.CatalogRevisionEvent) error]
// answers is the picture of the teller's answers (design 47 G A-175, A-178): its first view through teller_read_answers,
// then its views family; nil when it is not followed.
answers *views.Picture
}
// firstViewBound and firstViewWait are each first view's bound before its condition is raised, and its first wait
@@ -136,7 +146,24 @@ func (iv *catalogViews) start(ctx context.Context, w *world, wake func(), settle
reader.Findings = func() views.WireWriter { return iv.writer }
}
iv.h = &halves{w: w, call: iv.call, reader: reader, wake: wake, logf: iv.logf, ctx: ctx, settled: settled}
iv.h.follow(trunkBranch)
if err := iv.h.follow(); err != nil {
iv.logf("the catalog half is not followed, so no module is placed until the assigner starts again: %v", err)
}
// Each candidate's catalog graph revision, heard on the reader's one subscription of catalog-revision (A-201, A-185).
if err := mcc.WireCandidatesProcess(reader, func(ctx context.Context, ev meshcatalog.CatalogRevisionEvent) error {
if f := iv.candidates.Load(); f != nil {
return (*f)(ctx, ev)
}
return fmt.Errorf("the candidate %s's revision %d is heard before the assigner can keep its assignment graph", ev.Group, ev.Revision)
}); err != nil {
iv.logf("no candidate's catalog graph revision is heard, so no run's assignment graph is kept until the assigner starts again: %v", err)
}
// The teller's answers to the assigner's asks, followed as every owner's state is (A-175), never its bucket.
if pic, err := reader.Follow(buckets.TellerModule, buckets.AnswersState, buckets.TellerModule+"."+buckets.AnswersReadTool); err != nil {
iv.logf("the teller's answers are not followed, so no ask is answered until the assigner starts again: %v", err)
} else {
iv.answers = pic
}
reader.Start(ctx, iv.js, iv.account, iv.status)
return 1
}
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go
+103 -93
View File
@@ -12,13 +12,11 @@ import (
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
wdc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshwatchdogsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
mdc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient"
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/graph"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk"
"git.novox.be/novox/mesh-sdk/go/keys"
mmac "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
@@ -45,12 +43,19 @@ type world struct {
ends builtEnds
// released are the catalog's version-released heard, by module and source revision (buildended.go).
released releasedVersions
// delivered are the assigner's own version-delivered heard and said (delivered.go).
delivered deliveredVersions
// edges are, by watched branch and module, the edges into the module's artifacts its catalog entry names (§374's kinds).
edges map[string]map[string][]meshcatalog.Edge
// assignments is the seat's state assignments assembled node by node from its keys in either form (mesh-sdk
// mmac.Keys; research 072 row 37): the one reader of its keys in the assigner.
assignmentKeys *mmac.Keys
// sourceGroups is, by revision of the gate's source.main, the release (group) that moved it, from every revision its
// history keeps (watched with history): a build's source revision names the release that carries it (ADR 0601 §1).
sourceGroups map[uint64]string
// first is the first watched branch the catalog half is followed of, the trunk's: the branch of the trunk-only want, read from the
// branches the registrations name, before the first registrations from those the gate's `source` answers (design 47 G §507, §532),
// and never fixed in code (§7).
first string
// presence is, by machine, the last machine-away or machine-back the watchman said of it (design 47 G §389).
presence map[string]presenceSaid
// running is, by `<node>.<module>`, the last `running` the node-engine seat said of it, kept from its stream by the assigner's
// durable consumer (design 47 G §199, §309); runningHeard says that consumer is bound.
running map[string]nodeengine.Running
@@ -77,7 +82,7 @@ type parsed struct {
func newWorld() *world {
return &world{values: map[string]map[string][]byte{}, manifests: map[string]parsed{}, healthy: map[string][]string{}, changes: map[string]uint64{},
meta: map[string]entryMeta{}, assignmentKeys: mmac.NewKeys(),
sourceGroups: map[uint64]string{}, running: map[string]nodeengine.Running{},
running: map[string]nodeengine.Running{},
modules: map[string]map[string]meshcatalog.ModulesByModuleValue{}, graphs: map[string]map[string]graph.Node{}}
}
@@ -94,44 +99,6 @@ func (w *world) putEntry(bucket, key string, value []byte, revision uint64, at t
if bucket == plc.AssignmentsBucket {
_, _ = w.assignmentKeys.Put(key, value, revision, false) // put took it already; this is its revision
}
if bucket == buckets.Source && key == mdc.SourceKey(trunkBranch) {
if s, err := mdc.DecodeSource(value); err == nil && s.Group != "" {
w.sourceGroups[revision] = s.Group
}
}
}
// release is the release whose Approve carries module's build fp (ADR 0601 §1), and whether one does: the trunk's value of
// the module names fp, the source revision at which that fingerprint last changed (FPSrc, never the Src of its last merge,
// which a recompute with no merge keeps) names the release that moved it, and the gate's merge of that release, which it
// writes on the operator's Approve, holds the module's own trunk merge among its merged pull requests. Any of them
// unknown: none, and the move is asked of a person.
func (w *world) release(module, fp string) (string, bool) {
t, ok := w.trunk()[module]
if !ok || fp == "" || t.Fp2 != fp || t.FpSrc == 0 || t.Commit == "" {
return "", false
}
w.mu.Lock()
group := w.sourceGroups[uint64(t.FpSrc)]
w.mu.Unlock()
key, err := mdc.MergesKey(group)
if group == "" || err != nil {
return "", false
}
raw := w.value(buckets.Merges, key)
if raw == nil {
return "", false
}
merge, err := mdc.DecodeMerge(raw)
if err != nil {
return "", false
}
for _, pull := range merge.Pulls {
if pull.State == string(mdc.MergeMerged) && pull.Commit == t.Commit && trunk.SameRepository(pull.Repository, t.Repository) {
return group, true
}
}
return "", false
}
// metaOf is the revision and time of the value a bucket holds at key, as the watch said them.
@@ -267,8 +234,8 @@ func (w *world) branches() []string {
w.mu.Lock()
defer w.mu.Unlock()
out := slices.Sorted(maps.Keys(w.modules))
if i := slices.Index(out, trunkBranch); i > 0 {
out = slices.Insert(slices.Delete(out, i, i+1), 0, trunkBranch)
if i := slices.Index(out, w.first); i > 0 {
out = slices.Insert(slices.Delete(out, i, i+1), 0, w.first)
}
return out
}
@@ -307,6 +274,27 @@ func (w *world) dropModule(branch, module string) {
defer w.mu.Unlock()
delete(w.modules[branch], module)
delete(w.graphs[branch], module)
delete(w.edges[branch], module)
}
// putEdges keeps the edges into module's artifacts that branch's catalog entry names.
func (w *world) putEdges(branch, module string, edges []meshcatalog.Edge) {
w.mu.Lock()
defer w.mu.Unlock()
if w.edges == nil {
w.edges = map[string]map[string][]meshcatalog.Edge{}
}
if w.edges[branch] == nil {
w.edges[branch] = map[string][]meshcatalog.Edge{}
}
w.edges[branch][module] = edges
}
// edgesOf is the edges into module's artifacts in branch's catalog half.
func (w *world) edgesOf(branch, module string) []meshcatalog.Edge {
w.mu.Lock()
defer w.mu.Unlock()
return w.edges[branch][module]
}
// holdsBranch says whether the catalog half of branch has been taken in (even empty).
@@ -332,18 +320,37 @@ func (w *world) records() map[string]nodeengine.NodeRecord {
return out
}
// presenceOf reads one key of the watchman's presence: a node's key (`<node>`) with the node's online or not; the keys of
// the parts (`part.<part>.<instance>`) are not a node's and are none here.
func presenceOf(key string, value []byte) (node string, online, ok bool) {
node, part, _, err := wdc.ParsePresenceKey(key)
if err != nil || node == "" || part != "" {
return "", false, false
// presenceSaid is the last of the watchman's machine-away and machine-back heard of a machine (design 47 G §389).
type presenceSaid struct {
away bool
at time.Time
}
// heardPresence takes one machine-away (away) or machine-back of machine said at at: the later of the two holds, so the two
// streams, read whole in either order, leave each machine as its last event says.
func (w *world) heardPresence(machine string, away bool, at time.Time) {
w.mu.Lock()
defer w.mu.Unlock()
if w.presence == nil {
w.presence = map[string]presenceSaid{}
}
p, err := wdc.DecodeNodePresence(value)
if err != nil {
return "", false, false
if was, ok := w.presence[machine]; ok && at.Before(was.at) {
return
}
return node, p.Online, true
w.presence[machine] = presenceSaid{away: away, at: at}
}
// awayNow is every machine whose last presence event heard is machine-away.
func (w *world) awayNow() map[string]bool {
w.mu.Lock()
defer w.mu.Unlock()
out := map[string]bool{}
for m, p := range w.presence {
if p.away {
out[m] = true
}
}
return out
}
// seatHeard says a machine holding seat (a seat held once) is online as the watchman holds it: the callee of a call to
@@ -364,13 +371,13 @@ func (w *world) seatHeard(seat string) bool {
return false
}
// online is, by node, whether the watchman holds the node online.
// online is, by node the assigner hears of, whether it is present: a machine is away from the watchman's machine-away until its
// machine-back, and present otherwise (design 47 G §389); the watchman's state is not read (ADR 0596 §2).
func (w *world) online() map[string]bool {
away := w.awayNow()
out := map[string]bool{}
for key, value := range w.read(buckets.Presence) {
if node, online, ok := presenceOf(key, value); ok {
out[node] = online
}
for _, node := range w.knownNodes() {
out[node] = !away[node]
}
return out
}
@@ -399,11 +406,7 @@ func (w *world) wantsOf() map[string]map[string]mwc.Want {
func (w *world) nodes(manifests map[string]catalogue.Manifest, assigned map[string][]string) assign.Nodes {
n := assign.Nodes{Online: map[string]bool{}, Running: w.runningNow(), Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{}, Providers: map[string]map[string][]string{},
Reads: map[string][]string{}}
for key, value := range w.read(buckets.Presence) {
if node, online, ok := presenceOf(key, value); ok {
n.Online[node] = online
}
}
n.Online = w.online()
for key, value := range w.read(buckets.Have) {
node, module, isNode, err := nodeengineclient.ParseHaveKey(key)
if err != nil {
@@ -459,34 +462,28 @@ func (w *world) has(bucket, key string) bool {
return ok
}
// built says whether the builder's build of a module at a fingerprint is done.
// built says whether the builder's build of a module at a fingerprint is done, from its `build.ended` heard (design 47 G A-181).
func (w *world) built(module, fp string) bool {
key, err := builds.BuilderBuildKey(module, fp)
if err != nil {
return false
}
w.mu.Lock()
value, ok := w.values[buckets.Builds][key]
w.mu.Unlock()
if !ok {
return false
}
b, err := builds.DecodeBuilderBuild(value)
return err == nil && b.State == builds.BuildDone
e, ok := w.ends.endOf(module, fp)
return ok && e.Outcome == builds.BuildDone
}
// buildNeeds are the reference forms the builder's done build of module at fp is written in (the build key's needs, ADR
// 0618 §1): nil when it is not done or names none.
// buildStanding says whether the builder's `build.ended` of module at fp said the build failed, and its error whole (design 47 G
// §575, A-162): the words of a hold under rule 1.
func (w *world) buildStanding(module, fp string) (failed bool, words string) {
if e, ok := w.ends.endOf(module, fp); ok && e.Outcome == builds.BuildFailed {
return true, e.Error
}
return false, ""
}
// buildNeeds are the reference forms the builder's done build of module at fp is written in (the build's end says them, ADR
// 0618 §1, A-181): nil when it is not done or names none.
func (w *world) buildNeeds(module, fp string) []string {
key, err := builds.BuilderBuildKey(module, fp)
if err != nil {
return nil
if e, ok := w.ends.endOf(module, fp); ok && e.Outcome == builds.BuildDone {
return e.Needs
}
b, err := builds.DecodeBuilderBuild(w.value(buckets.Builds, key))
if err != nil || b.State != builds.BuildDone {
return nil
}
return b.Needs
return nil
}
// recordReads is what node's record `<node>.node` states its engine reads (ADR 0614 §4), and whether the assigner holds
@@ -496,7 +493,7 @@ func (w *world) recordReads(node string) ([]string, bool) {
return r.ManifestReads, ok
}
// knownNodes is every node the assigner hears of: placed, with a record, or with a presence.
// knownNodes is every node the assigner hears of: placed, with a record, or named by machine-away or machine-back.
func (w *world) knownNodes() []string {
set := map[string]bool{}
for node := range w.placedNodes() {
@@ -505,9 +502,11 @@ func (w *world) knownNodes() []string {
for node := range w.records() {
set[node] = true
}
for node := range w.online() {
w.mu.Lock()
for node := range w.presence {
set[node] = true
}
w.mu.Unlock()
return slices.Sorted(maps.Keys(set))
}
@@ -582,8 +581,19 @@ func (w *world) unplacedNodes() []string {
return out
}
// trunkBranch is the branch of the trunk-only want, kept until the first registrations (design 47 G §95, §353).
const trunkBranch = "main"
// trunkName is the first watched branch, the trunk's (design 47 G §507, §353): "" until the first view of the catalog half is adopted.
func (w *world) trunkName() string {
w.mu.Lock()
defer w.mu.Unlock()
return w.first
}
// setTrunk says which watched branch is the first.
func (w *world) setTrunk(branch string) {
w.mu.Lock()
defer w.mu.Unlock()
w.first = branch
}
// placedBranches is, by module and node, the watched branch each placement names ("" when it names none).
func (w *world) placedBranches() map[string]map[string]string {
modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world_test.go
+31
View File
@@ -3,6 +3,8 @@ package main
import (
"context"
"mesh-delivery-assigner/internal/buckets"
"os"
"path/filepath"
"strings"
"testing"
"time"
@@ -81,3 +83,32 @@ func putModuleRaw(w *world, branch, module string, raw []byte) {
}
w.putModule(branch, module, v, graph.Node{FP: v.Fp2, Repository: v.Repository}, nil)
}
// Design 47 G §507, §532, §7: the first watched branch the catalog half is followed of is the trunk's, whatever its name, and no part of the
// assigner's code names a branch: the world's trunk comes first, and no non-test source holds the string "main".
func TestTheTrunkIsTheFirstWatchedBranchAndNoCodeNamesOne(t *testing.T) {
w := newWorld()
w.setTrunk("develop")
for _, b := range []string{"beta", "develop", "alpha"} {
w.putModule(b, "web", meshcatalog.ModulesByModuleValue{Fp2: "fp"}, graph.Node{FP: "fp"}, nil)
}
if got := w.branches(); strings.Join(got, ",") != "develop,alpha,beta" {
t.Errorf("branches = %v, want the first watched first, then by name", got)
}
files, err := filepath.Glob("*.go")
if err != nil {
t.Fatal(err)
}
for _, f := range files {
if strings.HasSuffix(f, "_test.go") {
continue
}
src, err := os.ReadFile(f)
if err != nil {
t.Fatal(err)
}
if strings.Contains(string(src), `"main"`) {
t.Errorf("%s names the branch main in code (design 47 G §507)", f)
}
}
}
modules/mesh-delivery-assigner/internal/assign/wants.go
+28 -24
View File
@@ -26,11 +26,15 @@ import (
// trunk (the indexer's modules), whether the builder's build of F is done, whether F is held, whether the controller
// issued its credential for F, its manifest at F, and the nodes it is assigned to.
type Module struct {
Name string
FP string
Built bool
Held bool
Granted bool
Name string
FP string
Built bool
// BuildFailed and BuildError say a build that ended failed, with the error as the builder's `build.ended` carries it whole
// (design 47 G §575, A-162): the words of a hold under rule 1.
BuildFailed bool
BuildError string
Held bool
Granted bool
// GrantedAt says whether the controller issued the module's credential for another fingerprint than FP: the first want of a node
// that runs an older one names it (issue 1945). Nil: none.
GrantedAt func(fp string) bool
@@ -69,10 +73,6 @@ type Module struct {
// Allowed says whether the operator answered yes to the record ask about the move of this module on node to the composition
// with this digest (ADR 0539 §14). Nil: none did.
Allowed func(node, digest string) bool
// Release names the release whose Approve carries the build fp, as mesh-delivery's gate holds it for the release's heads,
// read by the build's source revision (ADR 0601 §1): that Approve is the yes to a move of the build alone. False when no
// approved release is known to carry it. Nil: none is read.
Release func(fp string) (string, bool)
// Released is a hold on FP the operator released (ADR 0539 §19): FP is not held again by a health check that failed.
Released bool
// Restored says whether the operator answered "restore" to the restore ask of this data module on node: it is then set back
@@ -159,9 +159,6 @@ func healthy(h nodeengine.HaveByNodeModuleValue, fp string) bool {
return h.Fp == fp && h.Phase == nodeengineclient.PhaseRunning && h.Health == nodeengineclient.HealthOK
}
// Healthy says whether a have runs fp healthy.
func Healthy(h nodeengine.HaveByNodeModuleValue, fp string) bool { return healthy(h, fp) }
// Failed says whether a have's health check failed once the node is done moving.
func Failed(h nodeengine.HaveByNodeModuleValue) bool { return failed(h) }
@@ -191,7 +188,10 @@ func Holds(m Module, node string, n Nodes) (bool, int, string) {
func holds(m Module, node string, n Nodes, ordering bool) (bool, int, string) {
switch {
case !m.Built:
return false, mwc.RuleBuildDone, fmt.Sprintf("the build of %s at %s is not done", m.Name, short(m.FP))
if m.BuildFailed {
return false, mwc.RuleBuildDone, fmt.Sprintf("the build of %s at %s failed: %s", m.Name, short(m.FP), m.BuildError)
}
return false, mwc.RuleBuildDone, fmt.Sprintf("the build of %s at %s is being built", m.Name, short(m.FP))
case m.Held:
return false, mwc.RuleNotHeld, fmt.Sprintf("%s is held at %s", m.Name, short(m.FP))
case m.MachineHeld != nil && m.MachineHeld(node):
@@ -202,7 +202,7 @@ func holds(m Module, node string, n Nodes, ordering bool) (bool, int, string) {
switch {
case canary == "":
return false, mwc.RuleCanary, fmt.Sprintf("none of the nodes of %s is online, so it has no canary", m.Name)
case node != canary && !healthy(n.Haves[canary][m.Name], m.FP):
case node != canary && !n.RunsHealthy(canary, m.Name, m.FP):
return false, mwc.RuleCanary, fmt.Sprintf("the canary %s does not run %s healthy yet", canary, short(m.FP))
}
}
@@ -322,15 +322,19 @@ func (m Module) MissingProvisions(node string) []string {
return out
}
// runsHealthy says whether module runs healthy on node: from G the node-engine seat's last `running` of it names a build and
// does not say it stopped; with the seat's events not heard, the have's phase and health.
// runsHealthy says whether module runs healthy on node: the node-engine seat's last `running` of it names a build, does not say it
// stopped and does not say `healthy` false (design 47 G §520); nothing heard of it counts as not healthy.
func (n Nodes) runsHealthy(node, module string) bool {
if n.Running != nil {
r, ok := n.Running[node][module]
return ok && !r.Stopped && r.Fingerprint != ""
}
h, ok := n.Haves[node][module]
return ok && h.Phase == nodeengineclient.PhaseRunning && h.Health == nodeengineclient.HealthOK
r, ok := n.Running[node][module]
return ok && !r.Stopped && r.Fingerprint != "" && (r.Healthy == nil || *r.Healthy)
}
// RunsHealthy says whether the build of module at fp runs healthy on node, from the node-engine seat's last `running` of it alone
// (design 47 G §520): a `running` with no `healthy`, said by a node-engine of before G, reads as healthy while it stands, and a
// module of which nothing was heard counts as not healthy. The have is never read for it.
func (n Nodes) RunsHealthy(node, module, fp string) bool {
r, ok := n.Running[node][module]
return ok && n.runsHealthy(node, module) && r.Fingerprint == fp
}
// providedHealthy is rule 5 for one provision: some module providing it runs healthy on the node, or, assigned
@@ -702,7 +706,7 @@ func healthyElsewhere(m Module, n Nodes, node string) []string {
for _, other := range slices.Sorted(slices.Values(m.Nodes)) {
// A witness is online: a node that sleeps keeps the have it last wrote, and a stale have is no evidence that F runs
// healthy now.
if other != node && n.Online[other] && healthy(n.Haves[other][m.Name], m.FP) {
if other != node && n.Online[other] && n.RunsHealthy(other, m.Name, m.FP) {
out = append(out, other)
}
}
@@ -750,7 +754,7 @@ func allRunHealthy(m Module, n Nodes) bool {
if !ok || (h.Phase != nodeengineclient.PhaseRunning && h.Phase != nodeengineclient.PhaseFailed) {
continue
}
if !healthy(h, m.FP) {
if !n.RunsHealthy(node, m.Name, m.FP) {
return false
}
ran++
modules/mesh-delivery-assigner/internal/assign/wants_test.go
+155 -85
View File
@@ -60,7 +60,7 @@ func wantedOld() map[string]mwc.Want {
// g14 follows at once: step 0, ADR 0528 clause 3).
func TestEveryRuleHoldsSoTheWantsMove(t *testing.T) {
m, n := ready()
out := Wants(m, n, wantedOld(), nil)
out := Wants(m, heardOfHaves(n), wantedOld(), nil)
for _, node := range []string{"ace", "g14"} {
if out.Wants[node].Fp != fNew || out.Wants[node].Rule != 0 || out.Wants[node].Composed != dig(fNew) {
t.Errorf("%s's want = %+v, want %s", node, out.Wants[node], fNew)
@@ -89,7 +89,7 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) {
} {
m, n := ready()
breakIt(&m, &n)
out := Wants(m, n, wantedOld(), nil)
out := Wants(m, heardOfHaves(n), wantedOld(), nil)
if w, ok := out.Wants["g14"]; ok && w.Fp == fNew {
t.Errorf("rule %d broken: g14's want moved to F", rule)
}
@@ -110,7 +110,7 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) {
}, nil} {
m, n := ready()
m.Provisions = all
if out := Wants(m, n, wantedOld(), nil); out.Wants["g14"].Fp != fNew {
if out := Wants(m, heardOfHaves(n), wantedOld(), nil); out.Wants["g14"].Fp != fNew {
t.Errorf("provided provisions held g14's want: %+v (%s)", out.Wants["g14"], out.Rules["g14"].Words)
}
}
@@ -161,7 +161,7 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) {
m, n := ready()
m.Name, m.Needs, n.Reads, m.FormNeeds = c.name, c.needs, c.reads, c.forms
n.Haves["ace"][c.name], n.Haves["g14"][c.name] = n.Haves["ace"]["mail"], n.Haves["g14"]["mail"]
out := Wants(m, n, wantedOld(), nil)
out := Wants(m, heardOfHaves(n), wantedOld(), nil)
if moved := out.Wants["g14"].Fp == fNew; moved == c.held {
t.Errorf("%s: g14's want %+v, held by %+v", c.why, out.Wants["g14"], out.Rules["g14"])
}
@@ -208,7 +208,7 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) {
if c.holds == graph.BusState {
words = graph.BusStateWaitWords("mail", "g14")
}
out := Wants(m, n, wantedOld(), nil)
out := Wants(m, heardOfHaves(n), wantedOld(), nil)
if c.holds == "" {
if out.Wants["g14"].Fp != fNew {
t.Errorf("%s: held %+v", c.name, out.Rules["g14"])
@@ -235,7 +235,7 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) {
return map[string]meshprovisionerclient.Provision{node + ".mail.oidc": {Provision: "oidc", State: c.state},
node + ".mail.database": {Provision: "database", State: c.older, Why: "the operator's answer"}}
}
out := Wants(m, n, wantedOld(), nil)
out := Wants(m, heardOfHaves(n), wantedOld(), nil)
if got := out.Wants["g14"].Fp == fNew; got != c.moves || (!c.moves && (!strings.Contains(out.Rules["g14"].Words, "oidc") || strings.Contains(out.Rules["g14"].Words, "database"))) {
t.Errorf("%s: want %+v, held by %q", c.name, out.Wants["g14"], out.Rules["g14"].Words)
}
@@ -280,7 +280,7 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) {
m.Provisions = func(string) map[string]meshprovisionerclient.Provision {
return map[string]meshprovisionerclient.Provision{}
}
out := Wants(m, n, wantedOld(), nil)
out := Wants(m, heardOfHaves(n), wantedOld(), nil)
if c.missing == "" {
if out.Wants["g14"].Fp != fNew || len(m.MissingProvisions("g14")) != 0 {
t.Errorf("%s: held %+v", c.name, out.Rules["g14"])
@@ -307,7 +307,7 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) {
m.Provisions = func(string) map[string]meshprovisionerclient.Provision {
return map[string]meshprovisionerclient.Provision{}
}
out := Wants(m, n, wantedOld(), nil)
out := Wants(m, heardOfHaves(n), wantedOld(), nil)
if got := out.Wants["g14"].Fp == fNew; got != c.offered || (!c.offered && (out.Rules["g14"].Rule != mwc.RuleProvisions ||
!strings.Contains(out.Rules["g14"].Words, "no module offers database yet") || !strings.Contains(out.Rules["g14"].Words, "past its bound"))) {
t.Errorf("%s: want %+v, held by %+v", c.name, out.Wants["g14"], out.Rules["g14"])
@@ -342,7 +342,7 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) {
if c.have != nil {
n.Haves["novox"][c.module] = *c.have
}
out := Wants(m, n, c.before, nil)
out := Wants(m, heardOfHaves(n), c.before, nil)
w, written := out.Wants["novox"]
if !written {
w = c.before["novox"]
@@ -431,7 +431,7 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) {
if c.name == "placed" {
before["g14"] = mwc.Want{Fp: fNew, Composed: dig(fNew)}
}
out := Wants(m, n, before, nil)
out := Wants(m, heardOfHaves(n), before, nil)
if got := out.Wants["g14"]; !reflect.DeepEqual(got, c.want) {
t.Errorf("%s: g14's want = %+v, want %+v", c.name, got, c.want)
}
@@ -453,7 +453,7 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) {
m, n := ready()
grant := grantsstate.Grant{Accounts: []string{"ace.mail", "ace.mail.send"}}
m.GrantedOn = func(node string) bool { return GrantNamesNode(grant, node, "mail") }
out := Wants(m, n, wantedOld(), nil)
out := Wants(m, heardOfHaves(n), wantedOld(), nil)
if w := out.Wants["ace"]; w.Fp != fNew {
t.Errorf("ace, whose account the grant names, wants %+v", w)
}
@@ -486,7 +486,7 @@ func TestAnOfflineCanaryIsPassedOver(t *testing.T) {
if c := Canary(m.Nodes, n.Online); c != "g14" {
t.Fatalf("canary = %q, want g14", c)
}
out := Wants(m, n, wantedOld(), nil)
out := Wants(m, heardOfHaves(n), wantedOld(), nil)
if out.Wants["g14"].Fp != fNew {
t.Errorf("the canary g14's want = %+v", out.Wants["g14"])
}
@@ -500,7 +500,7 @@ func TestAFailedHealthHoldsAndSetsBack(t *testing.T) {
m, n := ready()
n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed, Detail: "connection refused"}
m.Now = time.Date(2026, 10, 25, 1, 0, 0, 0, time.UTC)
out := Wants(m, n, map[string]mwc.Want{"ace": {Fp: fNew, Composed: dig(fNew)}, "g14": {Fp: fOld, Composed: dig(fOld)}}, map[string]string{"ace": fOld})
out := Wants(m, heardOfHaves(n), map[string]mwc.Want{"ace": {Fp: fNew, Composed: dig(fNew)}, "g14": {Fp: fOld, Composed: dig(fOld)}}, map[string]string{"ace": fOld})
// 1906/H3: the held record keeps its cause in words: which machine (its node), which health, when, and the node's words.
if out.Held == nil || out.Held.Node != "ace" || out.Held.Words != "health failed, phase running, seen at 2026-10-25T01:00:00Z: connection refused" {
t.Fatalf("held = %+v", out.Held)
@@ -516,7 +516,7 @@ func TestAFailedHealthHoldsAndSetsBack(t *testing.T) {
// back, the failing node keeps its want, and the outcome names it (on ace a newly assigned module's first check held a version healthy for an hour on three nodes).
n.Haves["g14"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
before := map[string]mwc.Want{"ace": {Fp: fNew, Composed: dig(fNew)}, "g14": {Fp: fNew, Composed: dig(fNew)}}
out = Wants(m, n, before, map[string]string{"ace": fOld, "g14": fOld})
out = Wants(m, heardOfHaves(n), before, map[string]string{"ace": fOld, "g14": fOld})
if out.Held != nil || len(out.Wants) != 0 {
t.Fatalf("held %+v, wants %+v: want no hold and no want changed", out.Held, out.Wants)
}
@@ -534,7 +534,7 @@ func TestAFailedHealthHoldsAndSetsBack(t *testing.T) {
n.Online["novox"] = true
n.Haves["novox"] = map[string]nodeengine.HaveByNodeModuleValue{"mail": {Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}}
before["novox"] = mwc.Want{Fp: fOld, Composed: dig(fOld)}
out = Wants(three, n, before, map[string]string{"ace": fOld, "g14": fOld, "novox": fOld})
out = Wants(three, heardOfHaves(n), before, map[string]string{"ace": fOld, "g14": fOld, "novox": fOld})
if _, alone := out.NodeFailed["g14"]; !alone {
t.Errorf("g14 does not fail alone: %+v", out.NodeFailed)
}
@@ -547,7 +547,7 @@ func TestAFailedHealthHoldsAndSetsBack(t *testing.T) {
// A node that runs F healthy with no want, while g14 fails it alone, gets its want at F and is on no neither list (issue 1945).
n.Haves["novox"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
delete(before, "novox")
out = Wants(three, n, before, map[string]string{"ace": fOld, "g14": fOld, "novox": fOld})
out = Wants(three, heardOfHaves(n), before, map[string]string{"ace": fOld, "g14": fOld, "novox": fOld})
if w := out.Wants["novox"]; w.Fp != fNew || len(out.Neither) != 0 {
t.Errorf("novox running F with no want: want %+v, neither %+v; want its want at F and no neither", w, out.Neither)
}
@@ -569,7 +569,7 @@ func TestAFailedHealthHoldsAndSetsBack(t *testing.T) {
n.Haves["novox"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
}
before["novox"] = mwc.Want{Fp: fNew, Composed: dig(fNew)}
out = Wants(three, n, before, map[string]string{"ace": fOld, "g14": fOld, "novox": fOld})
out = Wants(three, heardOfHaves(n), before, map[string]string{"ace": fOld, "g14": fOld, "novox": fOld})
if out.Held == nil || out.Held.Node != "ace" || len(out.NodeFailed) != 0 {
t.Errorf("%s: held %+v, node failed %+v: want F held at ace", c.name, out.Held, out.NodeFailed)
}
@@ -584,7 +584,7 @@ func TestADataModuleIsNotSetBack(t *testing.T) {
m, n := ready()
m.Manifest.Data = &catalogue.Data{Own: []catalogue.DataItem{{ID: "mail", Class: catalogue.ClassIrreplaceable}}}
n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed}
out := Wants(m, n, map[string]mwc.Want{"ace": {Fp: fNew}}, map[string]string{"ace": fOld})
out := Wants(m, heardOfHaves(n), map[string]mwc.Want{"ace": {Fp: fNew}}, map[string]string{"ace": fOld})
if out.Held == nil {
t.Fatal("F was not held")
}
@@ -601,7 +601,7 @@ func TestADataModuleIsNotSetBack(t *testing.T) {
func TestAModuleWithoutDataIsNotAskedToRestore(t *testing.T) {
m, n := ready()
n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed}
out := Wants(m, n, map[string]mwc.Want{"ace": {Fp: fNew}}, map[string]string{"ace": fOld})
out := Wants(m, heardOfHaves(n), map[string]mwc.Want{"ace": {Fp: fNew}}, map[string]string{"ace": fOld})
if len(out.Restore) != 0 {
t.Errorf("restore asks = %v", out.Restore)
}
@@ -612,7 +612,7 @@ func TestAModuleWithoutDataIsNotAskedToRestore(t *testing.T) {
func TestALeavingNodeIsRemovedAndNothingIsWrittenTwice(t *testing.T) {
m, n := ready()
m.Nodes = []string{"ace"}
out := Wants(m, n, map[string]mwc.Want{"ace": {V: 1, Fp: fNew, Composed: dig(fNew)}, "g14": {Fp: fOld, Composed: dig(fOld)}}, nil)
out := Wants(m, heardOfHaves(n), map[string]mwc.Want{"ace": {V: 1, Fp: fNew, Composed: dig(fNew)}, "g14": {Fp: fOld, Composed: dig(fOld)}}, nil)
if w := out.Wants["g14"]; !w.Removed {
t.Errorf("g14's want = %+v, want removed", w)
}
@@ -632,7 +632,7 @@ func TestAWantWaitsForItsCompositionAndNamesItsDigest(t *testing.T) {
}
return composedEverywhere(node, fp)
}
out := Wants(m, n, wantedOld(), nil)
out := Wants(m, heardOfHaves(n), wantedOld(), nil)
for node, w := range out.Wants {
if w.Fp == fNew {
t.Errorf("%s's want moved to F before its composition exists: %+v", node, w)
@@ -642,11 +642,11 @@ func TestAWantWaitsForItsCompositionAndNamesItsDigest(t *testing.T) {
t.Errorf("ace is held back by %+v, want rule 7 naming the node", r)
}
// and a node that never had a want gets none
if out := Wants(m, n, nil, nil); len(out.Wants) != 0 {
if out := Wants(m, heardOfHaves(n), nil, nil); len(out.Wants) != 0 {
t.Errorf("wants without a composition: %+v", out.Wants)
}
have["ace|"+fNew] = true
out = Wants(m, n, wantedOld(), nil)
out = Wants(m, heardOfHaves(n), wantedOld(), nil)
if w := out.Wants["ace"]; w.Fp != fNew || w.Composed != dig(fNew) {
t.Errorf("ace's want = %+v, want F naming the composition's digest", w)
}
@@ -664,14 +664,14 @@ func TestARevisionAtTheSameFingerprintIsAWantByTheSameRules(t *testing.T) {
return ComposedDigest{Digest: revised, Judgement: JudgedNeither}, true
}
running := map[string]mwc.Want{"ace": {Fp: fNew, Composed: dig(fNew)}, "g14": {Fp: fNew, Composed: dig(fNew)}}
out := Wants(m, n, running, nil)
out := Wants(m, heardOfHaves(n), running, nil)
for _, node := range []string{"ace", "g14"} {
if w := out.Wants[node]; w.Fp != fNew || w.Composed != revised {
t.Errorf("%s's want = %+v, want F naming the revised digest", node, w)
}
}
m.Granted = false
out = Wants(m, n, running, nil)
out = Wants(m, heardOfHaves(n), running, nil)
if w, ok := out.Wants["ace"]; ok && w.Composed == revised {
t.Errorf("a revision was written while a rule holds: %+v", w)
}
@@ -679,7 +679,7 @@ func TestARevisionAtTheSameFingerprintIsAWantByTheSameRules(t *testing.T) {
// A want held back at an earlier fingerprint follows the new revision of that fingerprint's kept composition, which the
// controller composed again with today's config (novox/hq ADR 0545: on g14 a capacity of 0 after a rollback), while
// the rules still hold F; for a record module whose revision recreates a container, only after the ask.
out = Wants(m, n, wantedOld(), nil)
out = Wants(m, heardOfHaves(n), wantedOld(), nil)
if w := out.Wants["g14"]; w.Fp != fOld || w.Composed != revised || w.Newest != fNew {
t.Errorf("g14, held back at its earlier fingerprint, wants %+v; want it at that fingerprint naming the revised digest", w)
}
@@ -687,7 +687,7 @@ func TestARevisionAtTheSameFingerprintIsAWantByTheSameRules(t *testing.T) {
return ComposedDigest{Digest: revised, Judgement: JudgedRecreates, Policy: catalogue.PolicyRecord,
Against: dig(fOld), Words: "recreates"}, true
}
out = Wants(m, n, wantedOld(), nil)
out = Wants(m, heardOfHaves(n), wantedOld(), nil)
if w := out.Wants["g14"]; w.Composed == revised {
t.Errorf("a recorded module's revision that recreates a container was written with no ask: %+v", w)
}
@@ -707,12 +707,12 @@ func TestASetBackNamesTheOlderCompositionOrStays(t *testing.T) {
}
return composedEverywhere(node, fp)
}
out := Wants(m, n, running, map[string]string{"ace": fOld})
out := Wants(m, heardOfHaves(n), running, map[string]string{"ace": fOld})
if w := out.Wants["ace"]; w.Fp == fOld {
t.Errorf("set back to a fingerprint whose composition is gone: %+v", w)
}
m.Composed = composedEverywhere
out = Wants(m, n, running, map[string]string{"ace": fOld})
out = Wants(m, heardOfHaves(n), running, map[string]string{"ace": fOld})
if w := out.Wants["ace"]; w.Fp != fOld || w.Composed != dig(fOld) {
t.Errorf("set-back want = %+v", w)
}
@@ -741,7 +741,7 @@ func TestARecordModuleWaitsForItsAskWhenTheMoveIsNotJudgedNeither(t *testing.T)
for _, j := range []Judgement{JudgedRecreates, JudgedMovesData, JudgedCannot} {
m, n, wanted := movingTo(j)
recording(&m)
out := Wants(m, n, wanted, nil)
out := Wants(m, heardOfHaves(n), wanted, nil)
if w, ok := out.Wants["ace"]; ok {
t.Errorf("%s: ace's want was written without the ask's yes: %+v", j, w)
}
@@ -756,11 +756,11 @@ func TestARecordModuleWaitsForItsAskWhenTheMoveIsNotJudgedNeither(t *testing.T)
}
// a new revision at the same fingerprint is asked too
same := map[string]mwc.Want{"ace": {Fp: fNew, Composed: dig(fOld)}}
if out := Wants(m, n, same, nil); len(out.Record) == 0 || len(out.Wants) != 0 && out.Wants["ace"].Composed == dig(fNew) {
if out := Wants(m, heardOfHaves(n), same, nil); len(out.Record) == 0 || len(out.Wants) != 0 && out.Wants["ace"].Composed == dig(fNew) {
t.Errorf("%s: a new revision at the same fingerprint: %+v", j, out)
}
m.Allowed = func(node, digest string) bool { return node == "ace" && digest == dig(fNew) }
out = Wants(m, n, wanted, nil)
out = Wants(m, heardOfHaves(n), wanted, nil)
if w := out.Wants["ace"]; w.Fp != fNew || w.Composed != dig(fNew) {
t.Errorf("%s: after the yes ace's want = %+v", j, w)
}
@@ -772,25 +772,25 @@ func TestARecordModuleWaitsForItsAskWhenTheMoveIsNotJudgedNeither(t *testing.T)
func TestAMoveJudgedNeitherOrNeverRunOrNotRecordedIsWrittenAtOnce(t *testing.T) {
m, n, wanted := movingTo(JudgedNeither)
recording(&m)
if out := Wants(m, n, wanted, nil); out.Wants["ace"].Fp != fNew || len(out.Record) != 0 {
if out := Wants(m, heardOfHaves(n), wanted, nil); out.Wants["ace"].Fp != fNew || len(out.Record) != 0 {
t.Errorf("neither: %+v", out)
}
m, n, _ = movingTo(JudgedRecreates)
recording(&m)
m.Nodes = []string{"ace"} // g14 runs it with no want, so its first want is asked (1945/AC19): this case is ace's alone
delete(n.Haves["ace"], "mail")
if out := Wants(m, n, nil, nil); out.Wants["ace"].Fp != fNew || len(out.Record) != 0 {
if out := Wants(m, heardOfHaves(n), nil, nil); out.Wants["ace"].Fp != fNew || len(out.Record) != 0 {
t.Errorf("a node that never ran it: %+v", out)
}
// but one that runs an older fingerprint with no want yet is asked like any other (one running F is not: N1 below)
m, n, _ = movingTo(JudgedCannot)
recording(&m)
n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
if out := Wants(m, n, nil, nil); out.Wants["ace"].Fp == fNew || len(out.Record) == 0 {
if out := Wants(m, heardOfHaves(n), nil, nil); out.Wants["ace"].Fp == fNew || len(out.Record) == 0 {
t.Errorf("a node that runs it with no want: %+v", out)
}
m, n, wanted = movingTo(JudgedRecreates)
if out := Wants(m, n, wanted, nil); out.Wants["ace"].Fp != fNew || len(out.Record) != 0 {
if out := Wants(m, heardOfHaves(n), wanted, nil); out.Wants["ace"].Fp != fNew || len(out.Record) != 0 {
t.Errorf("a rolled module: %+v", out)
}
}
@@ -802,11 +802,11 @@ func TestTheBusMovesByTheRulesAndItsUserListRevisionIsWrittenAtOnce(t *testing.T
m.Manifest.Provides = []catalogue.Offer{{Name: "mesh-bus"}}
// ADR 0601 §1, §2: the bus module's want is written by the rules, as any module's, and no snapshot is asked for it: its node
// backs its streams up before it applies.
if out := Wants(m, n, wantedOld(), nil); out.Wants["ace"].Fp != fNew || len(out.Record) != 0 || len(out.Neither) != 0 {
if out := Wants(m, heardOfHaves(n), wantedOld(), nil); out.Wants["ace"].Fp != fNew || len(out.Record) != 0 || len(out.Neither) != 0 {
t.Errorf("the bus did not move by the rules: %+v", out)
}
running := map[string]mwc.Want{"ace": {Fp: fNew, Composed: dig(fOld)}, "g14": {Fp: fNew, Composed: dig(fOld)}}
out := Wants(m, n, running, nil)
out := Wants(m, heardOfHaves(n), running, nil)
if w := out.Wants["ace"]; w.Fp != fNew || w.Composed != dig(fNew) {
t.Errorf("a user-list revision of the bus: ace's want = %+v", w)
}
@@ -835,7 +835,7 @@ func TestTheBusMovesByTheRulesAndItsUserListRevisionIsWrittenAtOnce(t *testing.T
if c.step == "" {
m.Puts = nil
}
out := Wants(m, n, c.wanted, nil)
out := Wants(m, heardOfHaves(n), c.wanted, nil)
got, wrote := out.Wants["ace"]
switch {
case c.want == nil && wrote:
@@ -852,7 +852,7 @@ func TestTheBusMovesByTheRulesAndItsUserListRevisionIsWrittenAtOnce(t *testing.T
held := put
held.Step = puts.Holding
m.Puts = map[string]puts.Put{"ace": held}
if w := Wants(m, n, map[string]mwc.Want{"ace": PutWant(held)}, nil).Wants["ace"]; w.Composed != userList || !w.Put || !w.Hold {
if w := Wants(m, heardOfHaves(n), map[string]mwc.Want{"ace": PutWant(held)}, nil).Wants["ace"]; w.Composed != userList || !w.Put || !w.Hold {
t.Errorf("a held bus did not take its user-list revision as a put: %+v", w)
}
m.Puts = nil
@@ -983,14 +983,14 @@ func stepsOfAPut(t *testing.T) {
func TestTheBackupAnswerStaysInTheWantWhileItsTargetStays(t *testing.T) {
m, n := ready()
running := map[string]mwc.Want{"ace": {Fp: fNew, Composed: dig(fNew), Backup: mwc.BackupSkip, Words: BackupAnswerWords("skip", 0)}}
if w, ok := Wants(m, n, running, nil).Wants["ace"]; ok {
if w, ok := Wants(m, heardOfHaves(n), running, nil).Wants["ace"]; ok {
t.Errorf("the answer was erased: %+v", w)
}
// A placed node composes its own: its want names the build alone, and the answer stays in it all the same (the digest its
// have names is not in the want, so a want kept for its digest would erase the answer at every pass).
m.Self = func(string) bool { return true }
placed := map[string]mwc.Want{"ace": {Fp: fNew, Self: true, Backup: mwc.BackupSkip, Words: BackupAnswerWords("skip", 0)}}
if w, ok := Wants(m, n, placed, nil).Wants["ace"]; ok {
if w, ok := Wants(m, heardOfHaves(n), placed, nil).Wants["ace"]; ok {
t.Errorf("the answer in a placed node's want was erased: %+v", w)
}
}
@@ -1001,7 +1001,7 @@ func TestAReleasedHoldIsNotHeldAgain(t *testing.T) {
m, n := ready()
n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed, Detail: "x"}
m.Released = true
out := Wants(m, n, map[string]mwc.Want{"ace": {Fp: fOld, Composed: dig(fOld), Newest: fNew, Rule: mwc.RuleNotHeld}}, map[string]string{"ace": fOld})
out := Wants(m, heardOfHaves(n), map[string]mwc.Want{"ace": {Fp: fOld, Composed: dig(fOld), Newest: fNew, Rule: mwc.RuleNotHeld}}, map[string]string{"ace": fOld})
if out.Held != nil {
t.Errorf("a released fingerprint was held again: %+v", out.Held)
}
@@ -1018,11 +1018,11 @@ func TestARestoreAnswerSetsADataModuleBack(t *testing.T) {
n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed}
running := map[string]mwc.Want{"ace": {Fp: fNew, Composed: dig(fNew)}}
m.Restored = func(node string) bool { return false }
if w, ok := Wants(m, n, running, map[string]string{"ace": fOld}).Wants["ace"]; ok && w.Fp == fOld {
if w, ok := Wants(m, heardOfHaves(n), running, map[string]string{"ace": fOld}).Wants["ace"]; ok && w.Fp == fOld {
t.Errorf("set back without the operator's restore: %+v", w)
}
m.Restored = func(node string) bool { return node == "ace" }
if w := Wants(m, n, running, map[string]string{"ace": fOld}).Wants["ace"]; w.Fp != fOld || w.Composed != dig(fOld) {
if w := Wants(m, heardOfHaves(n), running, map[string]string{"ace": fOld}).Wants["ace"]; w.Fp != fOld || w.Composed != dig(fOld) {
t.Errorf("after restore ace's want = %+v", w)
}
}
@@ -1040,22 +1040,22 @@ func TestTheRecordPolicyIsTheOneTheControllerHolds(t *testing.T) {
}
}
with(catalogue.PolicyRecord, "person") // a roll module the operator set to record
if out := Wants(m, n, wanted, nil); len(out.Record) == 0 || out.Wants["ace"].Fp == fNew {
if out := Wants(m, heardOfHaves(n), wanted, nil); len(out.Record) == 0 || out.Wants["ace"].Fp == fNew {
t.Errorf("the operator's record policy was not honoured: %+v", out)
}
recording(&m) // the manifest says record, the operator chose roll
with("roll", "person")
if out := Wants(m, n, wanted, nil); len(out.Record) != 0 || out.Wants["ace"].Fp != fNew {
if out := Wants(m, heardOfHaves(n), wanted, nil); len(out.Record) != 0 || out.Wants["ace"].Fp != fNew {
t.Errorf("the operator's roll policy was not honoured: %+v", out)
}
with("", "") // made by hand: the manifest decides
if out := Wants(m, n, wanted, nil); len(out.Record) == 0 {
if out := Wants(m, heardOfHaves(n), wanted, nil); len(out.Record) == 0 {
t.Errorf("a hand-made composition fell through the manifest's record policy: %+v", out)
}
// 1953/D4 (ADR 0565 §7): a person's choice at the assigner is over the controller's beside the composition.
with(catalogue.PolicyRecord, "person")
m.Chosen = &catalogue.PersonPolicy{V: catalogue.PersonPolicyVersion, Module: m.Name, Policy: catalogue.PolicyRoll, Why: "w", By: "op"}
if out := Wants(m, n, wanted, nil); len(out.Record) != 0 || out.Wants["ace"].Fp != fNew {
if out := Wants(m, heardOfHaves(n), wanted, nil); len(out.Record) != 0 || out.Wants["ace"].Fp != fNew {
t.Errorf("a person's roll at the assigner was not honoured over the controller's record: %+v", out)
}
}
@@ -1066,11 +1066,11 @@ func TestTheBusWithNoWantIsComparedWithWhatItRuns(t *testing.T) {
m, n, _ := movingTo(JudgedNeither)
m.Manifest.Provides = []catalogue.Offer{{Name: "mesh-bus"}}
n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
if w, ok := Wants(m, n, nil, nil).Wants["ace"]; !ok || w.Fp != fNew {
if w, ok := Wants(m, heardOfHaves(n), nil, nil).Wants["ace"]; !ok || w.Fp != fNew {
t.Errorf("the bus runs F with no want and was not given one: %+v %v", w, ok)
}
n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
if w, ok := Wants(m, n, nil, nil).Wants["ace"]; !ok || w.Fp != fNew {
if w, ok := Wants(m, heardOfHaves(n), nil, nil).Wants["ace"]; !ok || w.Fp != fNew {
t.Errorf("the bus running an older fingerprint with no want was not given its want by the rules: %+v %v", w, ok)
}
}
@@ -1096,7 +1096,7 @@ func TestAUserListOnlyRevisionOfTheBusIsWrittenAtOnceBeforeTheRecordHold(t *test
"a want at the fingerprint it runs": {"ace": {Fp: fOld, Composed: dig(fOld)}},
"no readable want": nil,
} {
out := Wants(m, n, wanted, nil)
out := Wants(m, heardOfHaves(n), wanted, nil)
if w := out.Wants["ace"]; w.Fp != fOld || w.Composed != userList {
t.Errorf("%s: ace's want = %+v, want the running fingerprint naming the user-list revision", name, w)
}
@@ -1106,7 +1106,7 @@ func TestAUserListOnlyRevisionOfTheBusIsWrittenAtOnceBeforeTheRecordHold(t *test
}
// the same flag on a module that is not the bus changes nothing
m.Manifest.Provides = nil
if out := Wants(m, n, map[string]mwc.Want{"ace": {Fp: fOld, Composed: dig(fOld)}}, nil); out.Wants["ace"].Composed == userList {
if out := Wants(m, heardOfHaves(n), map[string]mwc.Want{"ace": {Fp: fOld, Composed: dig(fOld)}}, nil); out.Wants["ace"].Composed == userList {
t.Errorf("a module that is not the bus was written at once: %+v", out.Wants["ace"])
}
}
@@ -1129,7 +1129,7 @@ func TestAHaveFromBeforeTheSwitchGetsItsWantWithNoRecordHold(t *testing.T) {
}
m.Composed = unchanged
n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK} // no composition named
out := Wants(m, n, nil, nil)
out := Wants(m, heardOfHaves(n), nil, nil)
if w := out.Wants["ace"]; w.Fp != fNew || w.Composed != dig(fNew) {
t.Errorf("%s: ace's want = %+v, want F naming its composition", name, w)
}
@@ -1146,7 +1146,7 @@ func TestAHaveFromBeforeTheSwitchGetsItsWantWithNoRecordHold(t *testing.T) {
m2.Manifest.Provides = []catalogue.Offer{{Name: "mesh-bus"}}
}
n2.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
out := Wants(m2, n2, nil, nil)
out := Wants(m2, heardOfHaves(n2), nil, nil)
if j == JudgedNeither {
if out.Wants["ace"].Fp != fNew || len(out.Record) != 0 {
t.Errorf("%s, %s: %+v", name, what, out)
@@ -1162,12 +1162,12 @@ func TestAHaveFromBeforeTheSwitchGetsItsWantWithNoRecordHold(t *testing.T) {
return ComposedDigest{Digest: dig(fp), Judgement: JudgedRecreates, Against: dig(fOld), Words: "judged"}, true
}
n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Composed: dig(fOld), Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
if out := Wants(m, n, nil, nil); out.Wants["ace"].Fp == fNew && len(out.Record) == 0 {
if out := Wants(m, heardOfHaves(n), nil, nil); out.Wants["ace"].Fp == fNew && len(out.Record) == 0 {
t.Errorf("%s: a have naming a composition skipped the hold: %+v", name, out)
}
// a new fingerprint still asks (the bus is barred instead)
n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
out = Wants(m, n, nil, nil)
out = Wants(m, heardOfHaves(n), nil, nil)
if w, ok := out.Wants["ace"]; ok {
t.Errorf("%s: a new fingerprint was written with no yes: %+v", name, w)
}
@@ -1181,7 +1181,7 @@ func TestAHaveFromBeforeTheSwitchGetsItsWantWithNoRecordHold(t *testing.T) {
func TestAHoldOfAFailureWithoutWordsSaysSo(t *testing.T) {
m, n := ready()
n.Haves["ace"]["mail"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed}
out := Wants(m, n, map[string]mwc.Want{"ace": {Fp: fNew, Composed: dig(fNew)}}, map[string]string{"ace": fOld})
out := Wants(m, heardOfHaves(n), map[string]mwc.Want{"ace": {Fp: fNew, Composed: dig(fNew)}}, map[string]string{"ace": fOld})
if out.Held == nil || out.Held.Words != "health failed, phase running: the node gave no words" {
t.Fatalf("held = %+v", out.Held)
}
@@ -1198,7 +1198,7 @@ func TestAHoldIsLiftedOnceEveryNodeRunsItHealthy(t *testing.T) {
return m, n
}
m, n := everywhere()
out := Wants(m, n, wantedOld(), map[string]string{"ace": fOld, "g14": fOld})
out := Wants(m, heardOfHaves(n), wantedOld(), map[string]string{"ace": fOld, "g14": fOld})
if !out.Lifted || out.Held != nil {
t.Fatalf("lifted = %v, held = %+v: want the hold lifted", out.Lifted, out.Held)
}
@@ -1210,7 +1210,7 @@ func TestAHoldIsLiftedOnceEveryNodeRunsItHealthy(t *testing.T) {
m, n = everywhere()
delete(n.Haves["g14"], "mail")
if out := Wants(m, n, wantedOld(), nil); !out.Lifted {
if out := Wants(m, heardOfHaves(n), wantedOld(), nil); !out.Lifted {
t.Error("a node that does not run the module kept the hold")
}
@@ -1227,7 +1227,7 @@ func TestAHoldIsLiftedOnceEveryNodeRunsItHealthy(t *testing.T) {
} {
m, n := everywhere()
change(&m, &n)
if out := Wants(m, n, wantedOld(), map[string]string{"ace": fOld, "g14": fOld}); out.Lifted {
if out := Wants(m, heardOfHaves(n), wantedOld(), map[string]string{"ace": fOld, "g14": fOld}); out.Lifted {
t.Errorf("%s: the hold was lifted", name)
}
}
@@ -1258,7 +1258,7 @@ func x11() (Module, Nodes) {
// composition, never the newest, and says the newest it is held back from; the canary is never passed over.
func TestANodeRunningAModuleWithNoWantGetsItsFirstWantAtWhatItRuns(t *testing.T) {
m, n := x11()
out := Wants(m, n, nil, nil)
out := Wants(m, heardOfHaves(n), nil, nil)
// 1960: the providers rule holds no want: the canary g14 gets F, its provider's health unknown said
if w := out.Wants["g14"]; w.Fp != fNew || out.Unreached["g14"] == "" {
t.Errorf("g14's want = %+v (unreached %q), want F, the provider said", w, out.Unreached["g14"])
@@ -1273,13 +1273,13 @@ func TestANodeRunningAModuleWithNoWantGetsItsFirstWantAtWhatItRuns(t *testing.T)
m, n = x11()
n.Haves["g14"]["xorg"] = nodeengine.HaveByNodeModuleValue{Fp: fpOf('x'), Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
n.Haves["shanks"]["xorg"] = n.Haves["g14"]["xorg"]
if w := Wants(m, n, nil, nil).Wants["shanks"]; w.Fp != fOld || w.Rule != mwc.RuleCanary {
if w := Wants(m, heardOfHaves(n), nil, nil).Wants["shanks"]; w.Fp != fOld || w.Rule != mwc.RuleCanary {
t.Errorf("shanks held back by the canary with no want: %+v, want at fOld by rule 3", w)
}
// a node already running F gets its want at F; a removed want is no want either
m, n = x11()
n.Haves["g14"]["screen-lock"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthUnknown}
out = Wants(m, n, map[string]mwc.Want{"g14": {Removed: true}}, nil)
out = Wants(m, heardOfHaves(n), map[string]mwc.Want{"g14": {Removed: true}}, nil)
if w := out.Wants["g14"]; w.Fp != fNew || w.Composed != dig(fNew) || w.Removed {
t.Errorf("g14 running F with a removed want = %+v, want F", w)
}
@@ -1301,7 +1301,7 @@ func TestNoFirstWantWithoutTheCompositionAndGrantOfWhatItRuns(t *testing.T) {
} {
m, n := x11()
change(&m, &n)
out := Wants(m, n, nil, nil)
out := Wants(m, heardOfHaves(n), nil, nil)
if what == "an old-form build" {
// the canary g14 does not run F healthy: no want, on neither list by rule 3 (issue 1945)
if w, ok := out.Wants["shanks"]; ok {
@@ -1322,7 +1322,7 @@ func TestNoFirstWantWithoutTheCompositionAndGrantOfWhatItRuns(t *testing.T) {
}
m, n := x11()
before := map[string]mwc.Want{"g14": {Fp: fOld, Composed: dig(fOld)}, "shanks": {Fp: fOld, Composed: dig(fOld)}}
out := Wants(m, n, before, nil)
out := Wants(m, heardOfHaves(n), before, nil)
if len(out.Neither) != 0 {
t.Errorf("nodes holding a want named on neither list: %+v", out.Neither)
}
@@ -1341,7 +1341,7 @@ func TestNoFirstWantWithoutTheCompositionAndGrantOfWhatItRuns(t *testing.T) {
func TestANewAssignmentWaitingIsNotOnNeitherList(t *testing.T) {
m, n := x11()
delete(n.Haves["shanks"], "screen-lock")
out := Wants(m, n, nil, nil)
out := Wants(m, heardOfHaves(n), nil, nil)
// 1960: the canary orders moves, never a first want: the new assignment is wanted at F
if w := out.Wants["shanks"]; w.Fp != fNew {
t.Errorf("a new assignment while the canary does not run F: %+v, want F", w)
@@ -1350,7 +1350,7 @@ func TestANewAssignmentWaitingIsNotOnNeitherList(t *testing.T) {
t.Errorf("a new assignment is named on neither list: %+v", out.Neither)
}
m.Built = false
if out := Wants(m, n, nil, nil); len(out.Neither) != 1 || out.Neither["g14"].Rule != mwc.RuleBuildDone {
if out := Wants(m, heardOfHaves(n), nil, nil); len(out.Neither) != 1 || out.Neither["g14"].Rule != mwc.RuleBuildDone {
t.Errorf("with no build: neither = %+v, want g14 alone (it runs the module), by rule 1", out.Neither)
}
// 1945 (the assigner holds a first want back silently): with no build, a node whose want was removed and that does not run
@@ -1364,12 +1364,12 @@ func TestANewAssignmentWaitingIsNotOnNeitherList(t *testing.T) {
}{
{"no key: nothing written", nil, map[string]mwc.Want{}},
{"a removed want: it says why", map[string]mwc.Want{"shanks": gone},
map[string]mwc.Want{"shanks": {Removed: true, Words: "no want: rule 1: the build of screen-lock at " + short(m.FP) + " is not done"}}},
{"a removed want that says why already: nothing written", map[string]mwc.Want{"shanks": {Removed: true, Words: "no want: rule 1: the build of screen-lock at " + short(m.FP) + " is not done"}},
map[string]mwc.Want{"shanks": {Removed: true, Words: "no want: rule 1: the build of screen-lock at " + short(m.FP) + " is being built"}}},
{"a removed want that says why already: nothing written", map[string]mwc.Want{"shanks": {Removed: true, Words: "no want: rule 1: the build of screen-lock at " + short(m.FP) + " is being built"}},
map[string]mwc.Want{}},
{"a node that runs it: on neither list, its removed want left", map[string]mwc.Want{"g14": gone}, map[string]mwc.Want{}},
} {
out := Wants(m, n, c.before, nil)
out := Wants(m, heardOfHaves(n), c.before, nil)
if len(out.Wants) != len(c.want) {
t.Errorf("%s: wants = %+v, want %+v", c.name, out.Wants, c.want)
continue
@@ -1381,7 +1381,7 @@ func TestANewAssignmentWaitingIsNotOnNeitherList(t *testing.T) {
}
}
m.FP = ""
if out := Wants(m, n, nil, nil); len(out.Neither) != 0 {
if out := Wants(m, heardOfHaves(n), nil, nil); len(out.Neither) != 0 {
t.Errorf("a module of the node's own composition named on neither list: %+v", out.Neither)
}
}
@@ -1404,7 +1404,7 @@ func TestAnOldFormNodeGetsItsFirstWantAtFOnlyUnderTheCanary(t *testing.T) {
providersHealthy(n)
oldForm(n, "shanks")
n.Haves["g14"]["screen-lock"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
out := Wants(m, n, nil, nil)
out := Wants(m, heardOfHaves(n), nil, nil)
if w := out.Wants["shanks"]; w.Fp != fNew || w.Composed != dig(fNew) || w.Newest != "" || w.Rule != 0 {
t.Errorf("canary healthy on F: shanks's want = %+v, want F", w)
}
@@ -1416,7 +1416,7 @@ func TestAnOldFormNodeGetsItsFirstWantAtFOnlyUnderTheCanary(t *testing.T) {
providersHealthy(n)
oldForm(n, "shanks")
n.Haves["g14"]["screen-lock"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthUnknown}
out = Wants(m, n, nil, nil)
out = Wants(m, heardOfHaves(n), nil, nil)
if w, ok := out.Wants["shanks"]; ok {
t.Errorf("canary not healthy on F: shanks got a want: %+v", w)
}
@@ -1427,7 +1427,7 @@ func TestAnOldFormNodeGetsItsFirstWantAtFOnlyUnderTheCanary(t *testing.T) {
m, n = x11()
oldForm(n, "shanks")
n.Haves["g14"]["screen-lock"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
out = Wants(m, n, nil, nil)
out = Wants(m, heardOfHaves(n), nil, nil)
if w := out.Wants["shanks"]; w.Fp != fNew || out.Unreached["shanks"] == "" {
t.Errorf("providers not healthy: shanks's want = %+v (unreached %q), want F, the provider said", w, out.Unreached["shanks"])
}
@@ -1440,14 +1440,14 @@ func TestAnOldFormNodeGetsItsFirstWantAtFOnlyUnderTheCanary(t *testing.T) {
oldForm(n, "shanks")
n.Haves["g14"]["screen-lock"] = nodeengine.HaveByNodeModuleValue{Fp: fNew, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
m.Granted = false
if w, ok := Wants(m, n, nil, nil).Wants["shanks"]; ok {
if w, ok := Wants(m, heardOfHaves(n), nil, nil).Wants["shanks"]; ok {
t.Errorf("F not granted: shanks got a want: %+v", w)
}
// the old-form node is the canary itself: F once its providers run healthy
m, n = x11()
providersHealthy(n)
oldForm(n, "g14")
if w := Wants(m, n, nil, nil).Wants["g14"]; w.Fp != fNew || w.Composed != dig(fNew) {
if w := Wants(m, heardOfHaves(n), nil, nil).Wants["g14"]; w.Fp != fNew || w.Composed != dig(fNew) {
t.Errorf("old-form canary: g14's want = %+v, want F", w)
}
}
@@ -1462,7 +1462,7 @@ func TestAFailedHealthWhileTheNodeMovesHoldsNothing(t *testing.T) {
n := Nodes{Online: map[string]bool{"novox": true}, Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{
"novox": {"mesh-controller": {Fp: fNew, Phase: phase, Health: nodeengineclient.HealthFailed}}}}
before := map[string]mwc.Want{"novox": {Fp: fNew, Composed: dig(fNew)}}
if out := Wants(m, n, before, map[string]string{"novox": fOld}); out.Held != nil {
if out := Wants(m, heardOfHaves(n), before, map[string]string{"novox": fOld}); out.Held != nil {
t.Errorf("phase %s, health failed: held %+v, want no hold while the node moves", phase, out.Held)
}
}
@@ -1472,7 +1472,7 @@ func TestAFailedHealthWhileTheNodeMovesHoldsNothing(t *testing.T) {
n := Nodes{Online: map[string]bool{"novox": true}, Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{
"novox": {"mesh-controller": {Fp: fNew, Phase: phase, Health: nodeengineclient.HealthFailed}}}}
before := map[string]mwc.Want{"novox": {Fp: fNew, Composed: dig(fNew)}}
if out := Wants(m, n, before, map[string]string{"novox": fOld}); out.Held == nil || out.Held.Node != "novox" {
if out := Wants(m, heardOfHaves(n), before, map[string]string{"novox": fOld}); out.Held == nil || out.Held.Node != "novox" {
t.Errorf("phase %s, health failed: held %+v, want a hold on novox", phase, out.Held)
}
}
@@ -1491,7 +1491,7 @@ func TestAHoldIsLiftedOverHavesThatAreNotRunning(t *testing.T) {
}
for _, phase := range []string{nodeengineclient.PhaseRemoved, nodeengineclient.PhaseHeld, nodeengineclient.PhaseApplying, nodeengineclient.PhaseBackingUp, nodeengineclient.PhaseChecking} {
m, n := setup(phase, fOld, nodeengineclient.HealthUnknown)
if out := Wants(m, n, nil, nil); !out.Lifted {
if out := Wants(m, heardOfHaves(n), nil, nil); !out.Lifted {
t.Errorf("g14's have in phase %s kept the hold up", phase)
}
}
@@ -1500,14 +1500,14 @@ func TestAHoldIsLiftedOverHavesThatAreNotRunning(t *testing.T) {
"failed": {Fp: fNew, Phase: nodeengineclient.PhaseFailed, Health: nodeengineclient.HealthFailed},
} {
m, n := setup(h.Phase, h.Fp, h.Health)
if out := Wants(m, n, nil, nil); out.Lifted {
if out := Wants(m, heardOfHaves(n), nil, nil); out.Lifted {
t.Errorf("g14 %s: the hold was lifted", what)
}
}
// a failed health on a removed have holds nothing
m, n := setup(nodeengineclient.PhaseRemoved, fNew, nodeengineclient.HealthFailed)
m.Held = false
if out := Wants(m, n, nil, nil); out.Held != nil {
if out := Wants(m, heardOfHaves(n), nil, nil); out.Held != nil {
t.Errorf("a removed have with a failed health held F: %+v", out.Held)
}
}
@@ -1520,7 +1520,7 @@ func TestAFirstWantAfterARemovedWantGoesThroughTheRecordAsk(t *testing.T) {
m.Composed = func(_, fp string) (ComposedDigest, bool) {
return ComposedDigest{Digest: dig(fp), Judgement: JudgedRecreates, Policy: catalogue.PolicyRecord}, true
}
out := Wants(m, n, map[string]mwc.Want{"shanks": {Removed: true}}, nil)
out := Wants(m, heardOfHaves(n), map[string]mwc.Want{"shanks": {Removed: true}}, nil)
if w, ok := out.Wants["shanks"]; ok {
t.Errorf("shanks's first want after a removed want was written with no record ask: %+v", w)
}
@@ -1550,7 +1550,7 @@ func TestAHeldModuleWithNoWantIsNotOnNeitherList(t *testing.T) {
} {
m, n := x11()
change(&m, &n)
out := Wants(m, n, nil, nil)
out := Wants(m, heardOfHaves(n), nil, nil)
if len(out.Neither) != 0 {
t.Errorf("%s: a held module is on neither list: %+v", what, out.Neither)
}
@@ -1602,9 +1602,79 @@ func TestAWantNamesAKeptBuildsPartsAndNodeForANodeThatReadsThem(t *testing.T) {
m, n := ready()
n.Reads = map[string][]string{"ace": row.reads, "g14": row.reads}
m.KeptBuilds = row.kept
w := Wants(m, n, wantedOld(), nil).Wants["ace"]
w := Wants(m, heardOfHaves(n), wantedOld(), nil).Wants["ace"]
if w.Fp != fNew || (w.KeptOn == "g14") != row.wanted || (w.Version == "1.2.3") != row.wanted || (len(w.Parts) == 1) != row.wanted {
t.Errorf("%s: ace's want = %+v", name, w)
}
}
}
// heardOfHaves gives each have that runs the `running` the node-engine seat says of it (design 47 G §520: health is read from
// `running` alone), for a fixture written as haves; a fixture that names its own `running` is left as it is.
func heardOfHaves(n Nodes) Nodes {
if n.Running != nil {
return n
}
n.Running = map[string]map[string]nodeengine.Running{}
for node, byModule := range n.Haves {
n.Running[node] = map[string]nodeengine.Running{}
for module, h := range byModule {
if h.Phase == nodeengineclient.PhaseRunning && h.Fp != "" {
ok := h.Health == nodeengineclient.HealthOK
n.Running[node][module] = nodeengine.Running{Node: node, Module: module, Fingerprint: h.Fp, Healthy: &ok}
}
}
}
return n
}
// §520: health is read from the node-engine seat's `running` alone, whatever the have says, and a build of which nothing was heard
// is not healthy.
func TestHealthIsReadFromRunningAlone(t *testing.T) {
yes, no := true, false
say := func(node, module, fp string, healthy *bool) nodeengine.Running {
return nodeengine.Running{Node: node, Module: module, Fingerprint: fp, Healthy: healthy}
}
for _, c := range []struct {
name string
running map[string]map[string]nodeengine.Running
rule int
}{
{"all healthy by running", map[string]map[string]nodeengine.Running{"ace": {"mail": say("ace", "mail", fNew, &yes), "store": say("ace", "store", fpOf('s'), &yes)}}, 0},
{"a running with no healthy reads healthy", map[string]map[string]nodeengine.Running{"ace": {"mail": say("ace", "mail", fNew, nil), "store": say("ace", "store", fpOf('s'), nil)}}, 0},
{"the canary says unhealthy, its have says healthy", map[string]map[string]nodeengine.Running{"ace": {"mail": say("ace", "mail", fNew, &no), "store": say("ace", "store", fpOf('s'), &yes)}}, mwc.RuleCanary},
{"nothing heard of the canary", map[string]map[string]nodeengine.Running{"ace": {"store": say("ace", "store", fpOf('s'), &yes)}}, mwc.RuleCanary},
} {
m, n := ready()
n.Running = c.running
if got := Wants(m, n, wantedOld(), nil).Rules["g14"].Rule; got != c.rule {
t.Errorf("%s: g14 held by rule %d, want %d", c.name, got, c.rule)
}
}
// rule 5: a provider that says unhealthy, or that nothing was heard of, holds no want but leaves the consumer unreached
m, n := ready()
n.Running = map[string]map[string]nodeengine.Running{"ace": {"mail": say("ace", "mail", fNew, &yes), "store": say("ace", "store", fpOf('s'), &no)}}
if out := Wants(m, n, wantedOld(), nil); len(out.Unreached) == 0 {
t.Errorf("a provider saying unhealthy: no module unreached, %+v", out)
}
}
// §575: a hold under rule 1 says how the build stands: failed with the error whole, or being built.
func TestRuleOneSaysHowTheBuildStands(t *testing.T) {
for _, c := range []struct {
name string
failed bool
err string
want string
}{
{"a build that runs", false, "", "the build of mail at " + short(fNew) + " is being built"},
{"a failed build", true, "transport fault: no responders at mesh-build-go", "the build of mail at " + short(fNew) + " failed: transport fault: no responders at mesh-build-go"},
} {
m, n := ready()
m.Built, m.BuildFailed, m.BuildError = false, c.failed, c.err
held, rule, words := Holds(m, "g14", n)
if held || rule != mwc.RuleBuildDone || words != c.want {
t.Errorf("%s: held %v rule %d words %q, want %q", c.name, held, rule, words, c.want)
}
}
}
modules/mesh-delivery-assigner/internal/assigngraph/candidate.go
+138
View File
@@ -0,0 +1,138 @@
package assigngraph
import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
)
// A candidate's assignment graph (design 47 G §189, §190, A-201, A-235): computed per run on request from the candidate's catalog
// graph revision, kept apart from trunk's under the seat's key form candidate.<group>.<heads>, the group `run-<id>`, and dropped on
// the runner's group-ended of that run; it writes no want, and the gate's heads-replaced drops nothing.
// CandidateObject is the object name of a candidate's revision (the seat's key form candidate.<group>.<heads>).
func CandidateObject(group, heads string) string {
return meshwants.AssignmentRevisionsByCandidateGroupHeadsKey(group, heads)
}
// KeepCandidate keeps c as the candidate's revision of group and heads, numbered after the one kept of it, unless that one lists
// the same entries from the same inputs; its entry objects are written first, an entry trunk shares stored once.
func KeepCandidate(ctx context.Context, o Objects, c Computed, group, heads, cause string, at time.Time) (meshwants.AssignmentRevision, bool, error) {
if group == "" || heads == "" {
return meshwants.AssignmentRevision{}, false, fmt.Errorf("a candidate's revision names its group and its heads, not %q and %q", group, heads)
}
was, err := readCandidateObject(ctx, o, CandidateObject(group, heads))
has := err == nil
if err != nil && !errors.Is(err, ErrNoObject) {
return was, false, err
}
if has && Same(was, c.Revision) {
return was, false, nil
}
rev := c.Revision
rev.Kind, rev.Group, rev.Heads, rev.Cause = meshcatalogclient.KindCandidate, group, heads, cause
rev.Revision = was.Revision + 1
t := at.UTC()
rev.At = &t
if err := writeEntries(ctx, o, c, rev); err != nil {
return was, false, err
}
list, err := json.Marshal(rev)
if err != nil {
return was, false, err
}
if err := o.Put(ctx, CandidateObject(group, heads), list); err != nil {
return was, false, fmt.Errorf("the candidate %s's revision in %s: %w", group, RevisionsObjects, err)
}
return rev, true, nil
}
// ReadCandidate is the candidate's revision kept of group, of heads when named, else the newest kept of the group; ErrNoCandidate
// when none is kept.
func ReadCandidate(ctx context.Context, o Objects, group, heads string) (meshwants.AssignmentRevision, error) {
if heads != "" {
rev, err := readCandidateObject(ctx, o, CandidateObject(group, heads))
if errors.Is(err, ErrNoObject) {
err = fmt.Errorf("%w (group %q, heads %q)", ErrNoCandidate, group, heads)
}
return rev, err
}
names, err := CandidateNames(ctx, o, group)
if err != nil {
return meshwants.AssignmentRevision{}, err
}
var newest meshwants.AssignmentRevision
found := false
for _, n := range names {
rev, err := readCandidateObject(ctx, o, n)
if err != nil {
return rev, err
}
if !found || rev.At != nil && newest.At != nil && rev.At.After(*newest.At) {
newest, found = rev, true
}
}
if !found {
return newest, fmt.Errorf("%w (group %q)", ErrNoCandidate, group)
}
return newest, nil
}
// CandidateNames is the object name of every revision kept of the candidate group; every candidate's when group is "".
func CandidateNames(ctx context.Context, o Objects, group string) ([]string, error) {
names, err := o.Names(ctx)
if err != nil {
return nil, fmt.Errorf("the objects %s: %w", RevisionsObjects, err)
}
var out []string
for _, n := range names {
g, _, ok := meshwants.ParseAssignmentRevisionsByCandidateGroupHeadsKey(n)
if ok && (group == "" || g == group) {
out = append(out, n)
}
}
return out, nil
}
// Remover is an object bucket that also removes an object.
type Remover interface {
Objects
// Delete removes the object of that name; one not held is no error.
Delete(ctx context.Context, name string) error
}
// DropCandidate removes every revision kept of the candidate group (its run's group-ended, A-235); the entries it alone listed are
// removed with the next revisions nothing pins. It says how many it removed.
func DropCandidate(ctx context.Context, o Remover, group string) (int, error) {
if strings.TrimSpace(group) == "" {
return 0, errors.New("a candidate to drop names its group")
}
names, err := CandidateNames(ctx, o, group)
if err != nil {
return 0, err
}
for i, n := range names {
if err := o.Delete(ctx, n); err != nil {
return i, fmt.Errorf("the candidate %s's revision %s: %w", group, n, err)
}
}
return len(names), nil
}
func readCandidateObject(ctx context.Context, o Objects, name string) (meshwants.AssignmentRevision, error) {
var rev meshwants.AssignmentRevision
data, err := o.Get(ctx, name)
if err != nil {
return rev, err
}
if err := json.Unmarshal(data, &rev); err != nil {
return rev, fmt.Errorf("the candidate's revision %s: %w", name, err)
}
return rev, nil
}
modules/mesh-delivery-assigner/internal/assigngraph/compute.go
+35 -1
View File
@@ -58,13 +58,16 @@ type placedAt struct {
entry meshwants.AssignmentEntry
langs map[string]bool
acts []actingRule
// provides is what its catalog entry provides; open its requirements its placement neither binds nor its node pins.
provides, open []string
}
type actingRule struct{ seat, step, on, bound string }
// Compute is the assignment graph revision of in: one entry per module placed on a node whose branch's catalog graph
// holds it, its version the build the placement names or else the catalog entry's fingerprint (design 47 lines 909-914),
// with the edges of provisioning to the providers its placement binds or its node pins, and of acting from each acting
// with the edges of provisioning to the providers its placement binds or its node pins, else the provider on its node or
// the only provider (an entry held naming the candidates where none is), and of acting from each acting
// part's version to each version its rule names (design 47 *An acting part's new version acts first*: "the assigner
// gives an edge from the acting part's version to each version its rule names").
func Compute(in Inputs) Computed {
@@ -115,6 +118,7 @@ func Compute(in Inputs) Computed {
for _, r := range ce.Acts {
p.acts = append(p.acts, actingRule{seat: r.Seat, step: r.Step, on: r.On, bound: r.Bound})
}
p.provides = ce.Provides
for _, r := range ce.Requires {
prov, ok := m.Bindings[r]
if !ok {
@@ -122,12 +126,42 @@ func Compute(in Inputs) Computed {
}
if ok {
p.entry.Edges = append(p.entry.Edges, meshwants.Edge{Kind: EdgeProvisioning, Node: prov.Node, Module: prov.Module, Provision: r})
} else {
p.open = append(p.open, r)
}
}
placed = append(placed, p)
branches[branch] = true
}
}
// A requirement neither bound nor pinned points, by design 27's order (design 47 G §357), to the provider on the
// consumer's node, else to the only provider; otherwise the entry is held, naming the candidates, and nothing is guessed.
for _, p := range placed {
for _, r := range p.open {
var here, all []*placedAt
for _, q := range placed {
if q != p && slices.Contains(q.provides, r) {
all = append(all, q)
if q.node == p.node {
here = append(here, q)
}
}
}
switch {
case len(here) == 1:
p.entry.Edges = append(p.entry.Edges, meshwants.Edge{Kind: EdgeProvisioning, Node: here[0].node, Module: here[0].placed.Module, Provision: r})
case len(here) == 0 && len(all) == 1:
p.entry.Edges = append(p.entry.Edges, meshwants.Edge{Kind: EdgeProvisioning, Node: all[0].node, Module: all[0].placed.Module, Provision: r})
default:
h := meshwants.Held{Provision: r}
for _, q := range all {
h.Candidates = append(h.Candidates, meshwants.Edge{Kind: EdgeProvisioning, Node: q.node, Module: q.placed.Module, Fp: q.entry.Fp, Provision: r})
}
sortEdges(h.Candidates)
p.entry.Held = append(p.entry.Held, h)
}
}
}
at := map[string]*placedAt{}
for _, p := range placed {
at[p.node+"/"+p.placed.Module] = p
modules/mesh-delivery-assigner/internal/assigngraph/graph_test.go
+139 -3
View File
@@ -91,11 +91,16 @@ func TestTheAssignmentGraphIsComputedFromItsInputsAndKept(t *testing.T) {
if fmt.Sprint(webA.Edges) != fmt.Sprint(want) {
t.Errorf("web on anchor's edges:\n got %+v\nwant %+v", webA.Edges, want)
}
// laptop's web: its own node's node-engine composes it, the build module placed once builds it, and no binding or pin
// names a provider there.
if e := webL.Edges; len(e) != 2 || e[0].Module != "build-go" || e[0].Node != "anchor" || e[1].Module != "node-engine" || e[1].Node != "laptop" {
// laptop's web: its own node's node-engine composes it, the build module placed once builds it, and with no binding or
// pin its database is the only provider's (design 47 G §357).
if e := webL.Edges; len(e) != 3 || e[0].Module != "build-go" || e[0].Node != "anchor" || e[1].Module != "node-engine" || e[1].Node != "laptop" ||
e[2] != (meshwants.Edge{Kind: EdgeProvisioning, Node: "anchor", Module: "postgres", Fp: "src2:p1", Provision: "database"}) {
t.Errorf("web on laptop's edges: %+v", e)
}
// Its secrets no placed module provides: held on both nodes, naming no candidate.
if h := webA.Held; len(h) != 1 || h[0].Provision != "secrets" || len(h[0].Candidates) != 0 {
t.Errorf("web on anchor is held on %+v", h)
}
if e := byNode["anchor/postgres"].Edges; len(e) != 1 || e[0].Module != "node-engine" {
t.Errorf("postgres builds no Go, so only the node-engine acts on it: %+v", e)
}
@@ -149,6 +154,63 @@ func TestTheAssignmentGraphIsComputedFromItsInputsAndKept(t *testing.T) {
}
}
// Design 47 G §357 and A-167: a requirement neither bound nor pinned points to the provider on the consumer's node, else
// to the only provider; otherwise the entry is held, naming the candidates, and nothing is guessed.
func TestARequirementPointsByTheOrderOrIsHeld(t *testing.T) {
h := half("main", 3,
meshcatalog.CatalogEntry{Module: "kv", Fp: "src2:k1", Provides: []string{"cache"}},
meshcatalog.CatalogEntry{Module: "web", Fp: "src2:w1", Requires: []string{"cache"}})
for _, c := range []struct {
name string
kv []string // the nodes kv is placed on
pin bool // web's node pins kv on laptop
edge string // the node of the provisioning edge, "" for none
held string // the candidates' nodes of the hold, "-" for no hold
}{
{name: "the provider on the consumer's node", kv: []string{"anchor", "laptop"}, edge: "anchor", held: "-"},
{name: "the only provider", kv: []string{"laptop"}, edge: "laptop", held: "-"},
{name: "a pin before the node's own", kv: []string{"anchor", "laptop"}, pin: true, edge: "laptop", held: "-"},
{name: "two providers elsewhere: held", kv: []string{"laptop", "server"}, held: "laptop,server"},
{name: "none: held naming none", held: ""},
} {
t.Run(c.name, func(t *testing.T) {
pl := map[string]mma.Assignments{"anchor": {Node: "anchor", Modules: []mma.PlacedModule{{Module: "web"}}}}
if c.pin {
pl["anchor"] = mma.Assignments{Node: "anchor", Modules: pl["anchor"].Modules, Pins: map[string]mma.Provider{"cache": {Node: "laptop", Module: "kv"}}}
}
for _, n := range c.kv {
a := pl[n]
a.Node, a.Modules = n, append(a.Modules, mma.PlacedModule{Module: "kv"})
pl[n] = a
}
out := Compute(Inputs{Halves: map[string]meshcatalogclient.Half{"main": h}, Placements: pl, FirstBranch: func(string) string { return "main" }})
var web meshwants.AssignmentEntry
for _, ref := range out.Revision.Entries {
if ref.Node == "anchor" && ref.Module == "web" {
web = out.Entries[ref.Digest]
}
}
edge := ""
for _, e := range web.Edges {
if e.Kind == EdgeProvisioning && e.Provision == "cache" {
edge = e.Node
}
}
held := "-"
if len(web.Held) == 1 && web.Held[0].Provision == "cache" {
var nodes []string
for _, cand := range web.Held[0].Candidates {
nodes = append(nodes, cand.Node)
}
held = strings.Join(nodes, ",")
}
if edge != c.edge || held != c.held || len(web.Held) > 1 {
t.Errorf("web points to %q, held on %q (%+v); want %q, %q", edge, held, web.Held, c.edge, c.held)
}
})
}
}
// Design 47 line 1385: a whole assignment graph revision of about 120 modules on four nodes is under the bus's 256 KiB
// value limit.
func TestARevisionOf120ModulesOnFourNodesFitsOneValue(t *testing.T) {
@@ -174,3 +236,77 @@ func TestARevisionOf120ModulesOnFourNodesFitsOneValue(t *testing.T) {
t.Errorf("%d entries in %d bytes", len(c.Revision.Entries), len(raw))
}
}
func (m memObjects) Delete(_ context.Context, name string) error {
delete(m, name)
return nil
}
// A-201, A-235, §186, §189: a run's candidate assignment graph is kept apart from trunk's under candidate.<group>.<heads>, served by
// revision, entry and difference against trunk, and dropped by its group alone.
func TestARunsCandidateGraphIsKeptApartServedAndDropped(t *testing.T) {
ctx := context.Background()
at := time.Date(2026, 11, 12, 9, 0, 0, 0, time.UTC)
placed := map[string]mma.Assignments{"ace": {Modules: []mma.PlacedModule{{Module: "web"}}}}
first := func(string) string { return "main" }
o := memObjects{}
trunk := Compute(Inputs{Halves: map[string]meshcatalogclient.Half{"main": half("main", 3, meshcatalog.CatalogEntry{Module: "web", Fp: "src2:w1"})},
Placements: placed, FirstBranch: first})
if _, _, err := Keep(ctx, o, trunk, "catalog-revision", at); err != nil {
t.Fatal(err)
}
cand := Compute(Inputs{Halves: map[string]meshcatalogclient.Half{"main": half("main", 4, meshcatalog.CatalogEntry{Module: "web", Fp: "src2:w2"})},
Placements: placed, FirstBranch: first})
if _, _, err := KeepCandidate(ctx, o, cand, "", "h1", "catalog-revision", at); err == nil {
t.Error("a candidate naming no group was kept")
}
rev, fresh, err := KeepCandidate(ctx, o, cand, "run-7", "h1", "catalog-revision", at)
if err != nil || !fresh || rev.Kind != meshcatalogclient.KindCandidate || rev.Group != "run-7" || rev.Revision != 1 {
t.Fatalf("kept %+v fresh %v: %v", rev, fresh, err)
}
if _, ok := o[CandidateObject("run-7", "h1")]; !ok {
t.Fatalf("the candidate is not under candidate.run-7.h1: %v", o)
}
if _, fresh, _ := KeepCandidate(ctx, o, cand, "run-7", "h1", "catalog-revision", at); fresh {
t.Error("the same candidate kept again is a new revision")
}
newest, has, err := Newest(ctx, o)
if err != nil || !has || newest.Kind != meshcatalogclient.KindTrunk || newest.Revision != 1 {
t.Errorf("trunk's newest is %+v (%v, %v): the candidate is not kept apart", newest, has, err)
}
for _, in := range []meshwants.RevisionInput{{Group: "run-7"}, {Group: "run-7", Heads: "h1"}} {
a, err := Revision(ctx, o, in)
if err != nil || a.Revision.Group != "run-7" {
t.Errorf("revision %+v answered %+v: %v", in, a.Revision, err)
}
}
if _, err := Revision(ctx, o, meshwants.RevisionInput{Group: "run-8"}); !errors.Is(err, ErrNoCandidate) {
t.Errorf("an unkept candidate answered %v", err)
}
e, err := Entry(ctx, o, meshwants.EntryInput{Group: "run-7", Node: "ace", Module: "web"})
if err != nil || e.Entry.Fp != "src2:w2" {
t.Errorf("the candidate's entry is %+v: %v", e.Entry, err)
}
d, err := Difference(ctx, o, meshwants.DifferenceInput{Group: "run-7"})
if err != nil || len(d.Changes) != 1 || d.Changes[0].From != "src2:w1" || d.Changes[0].To != "src2:w2" {
t.Errorf("the candidate against trunk is %+v: %v", d, err)
}
if _, err := Difference(ctx, o, meshwants.DifferenceInput{}); err == nil {
t.Error("difference with none of from, to and group is not refused")
}
if _, _, err := KeepCandidate(ctx, o, cand, "run-9", "h2", "catalog-revision", at); err != nil {
t.Fatal(err)
}
if n, err := DropCandidate(ctx, o, "run-7"); err != nil || n != 1 {
t.Errorf("dropped %d: %v", n, err)
}
if _, err := ReadCandidate(ctx, o, "run-7", ""); !errors.Is(err, ErrNoCandidate) {
t.Errorf("the dropped candidate reads: %v", err)
}
if _, err := ReadCandidate(ctx, o, "run-9", ""); err != nil {
t.Errorf("another run's candidate went with it: %v", err)
}
if _, has, _ := Newest(ctx, o); !has {
t.Error("dropping a candidate dropped trunk")
}
}
modules/mesh-delivery-assigner/internal/assigngraph/keep.go
+33 -25
View File
@@ -103,36 +103,13 @@ func Keep(ctx context.Context, o Objects, c Computed, cause string, at time.Time
if has && Same(newest, c.Revision) {
return newest, false, nil
}
names, err := o.Names(ctx)
if err != nil {
return newest, false, fmt.Errorf("the objects %s: %w", RevisionsObjects, err)
}
held := make(map[string]bool, len(names))
for _, n := range names {
held[n] = true
}
rev := c.Revision
rev.Revision = newest.Revision + 1
rev.Cause = cause
t := at.UTC()
rev.At = &t
for _, ref := range rev.Entries {
name := EntryObject(ref.Digest)
if held[name] {
continue
}
e, ok := c.Entries[ref.Digest]
if !ok {
return newest, false, fmt.Errorf("revision %d lists the entry of %s/%s (%s), which it does not carry", rev.Revision, ref.Node, ref.Module, ref.Digest)
}
data, err := json.Marshal(e)
if err != nil {
return newest, false, err
}
if err := o.Put(ctx, name, data); err != nil {
return newest, false, fmt.Errorf("the entry of %s/%s in %s: %w", ref.Node, ref.Module, RevisionsObjects, err)
}
held[name] = true
if err := writeEntries(ctx, o, c, rev); err != nil {
return newest, false, err
}
list, err := json.Marshal(rev)
if err != nil {
@@ -144,6 +121,37 @@ func Keep(ctx context.Context, o Objects, c Computed, cause string, at time.Time
return rev, true, nil
}
// writeEntries writes each entry object rev lists that the bucket does not hold yet, from c, before its list is kept.
func writeEntries(ctx context.Context, o Objects, c Computed, rev meshwants.AssignmentRevision) error {
names, err := o.Names(ctx)
if err != nil {
return fmt.Errorf("the objects %s: %w", RevisionsObjects, err)
}
held := make(map[string]bool, len(names))
for _, n := range names {
held[n] = true
}
for _, ref := range rev.Entries {
name := EntryObject(ref.Digest)
if held[name] {
continue
}
e, ok := c.Entries[ref.Digest]
if !ok {
return fmt.Errorf("revision %d lists the entry of %s/%s (%s), which it does not carry", rev.Revision, ref.Node, ref.Module, ref.Digest)
}
data, err := json.Marshal(e)
if err != nil {
return err
}
if err := o.Put(ctx, name, data); err != nil {
return fmt.Errorf("the entry of %s/%s in %s: %w", ref.Node, ref.Module, RevisionsObjects, err)
}
held[name] = true
}
return nil
}
// Event is the seat's event assignment-revision of a kept revision.
func Event(rev meshwants.AssignmentRevision) meshwants.AssignmentRevisionEvent {
return meshwants.AssignmentRevisionEvent{V: rev.V, Revision: rev.Revision, Kind: rev.Kind, Cause: rev.Cause, Group: rev.Group,
modules/mesh-delivery-assigner/internal/assigngraph/verbs.go
+21 -5
View File
@@ -10,16 +10,23 @@ import (
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
)
// ErrNoCandidate answers a verb asked of a candidate's revision: the assigner keeps none (the candidate's graph and the
// pins from the gate are milestone 47/G's W5.9, held with research 086's fold).
var ErrNoCandidate = errors.New("the assigner keeps no candidate's assignment graph revision")
// ErrNoCandidate answers a verb asked of a candidate's revision the assigner keeps none of: a run's candidate is kept from its
// catalog graph revision until its run's group-ended (A-201, A-235).
var ErrNoCandidate = errors.New("the assigner keeps no such candidate's assignment graph revision")
// ErrNoRevision answers a verb asked while no trunk revision is kept yet.
var ErrNoRevision = errors.New("no assignment graph revision is kept yet")
func revisionOf(ctx context.Context, o Objects, revision int64, group, heads string) (meshwants.AssignmentRevision, error) {
if group != "" || heads != "" {
return meshwants.AssignmentRevision{}, fmt.Errorf("%w (group %q, heads %q)", ErrNoCandidate, group, heads)
if group == "" {
return meshwants.AssignmentRevision{}, fmt.Errorf("a candidate's revision is asked by its group, with or without its heads, not by heads %q alone", heads)
}
rev, err := ReadCandidate(ctx, o, group, heads)
if err == nil && revision > 0 && rev.Revision != revision {
err = fmt.Errorf("%w (group %q, revision %d: the one kept is %d)", ErrNoCandidate, group, revision, rev.Revision)
}
return rev, err
}
if revision > 0 {
return ReadRevision(ctx, o, revision)
@@ -66,7 +73,16 @@ func Entry(ctx context.Context, o Objects, in meshwants.EntryInput) (meshwants.E
// differs, with the edges that moved, and the nodes it touches.
func Difference(ctx context.Context, o Objects, in meshwants.DifferenceInput) (meshwants.Difference, error) {
if in.Group != "" || in.Heads != "" {
return meshwants.Difference{}, fmt.Errorf("%w (group %q, heads %q)", ErrNoCandidate, in.Group, in.Heads)
// A candidate against trunk (§186, §187): from the trunk revision named, else the newest kept.
to, err := revisionOf(ctx, o, 0, in.Group, in.Heads)
if err != nil {
return meshwants.Difference{}, err
}
from, err := revisionOf(ctx, o, in.From, "", "")
if err != nil {
return meshwants.Difference{}, err
}
return Compare(ctx, o, from, to)
}
if in.From <= 0 || in.To <= 0 {
return meshwants.Difference{}, errors.New("difference takes the two kept revisions it compares, from and to")
modules/mesh-delivery-assigner/internal/buckets/buckets.go
+18 -12
View File
@@ -6,10 +6,8 @@ package buckets
import (
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
watchdogs "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshwatchdogs"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
delivery "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
wants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate"
@@ -25,6 +23,18 @@ const (
// Waits holds the waits on an outside cause, one key `<node>.<module>.<fingerprint hex>` (design 47 G §706).
WaitsState = "waits"
Waits = plc.Module + "_" + WaitsState
// DeliveredAgain holds each ask of assigner_delivered_again, one key `<module>`, its value the version, who asked and why, and
// why it does not hold yet (design 47 G §376, A-250).
DeliveredAgainState = "delivered-again"
DeliveredAgain = plc.Module + "_" + DeliveredAgainState
// DeliveryWaits holds each dependent's delivery wait the assigner said, one key `<module>`, its value the wait-held said
// (design 47 G §376, §498, A-246).
DeliveryWaitsState = "delivery-waits"
DeliveryWaits = plc.Module + "_" + DeliveryWaitsState
// PutWaits holds each put wait the assigner said, one key `<node>.<module>`, its value the wait's id and the put as said
// (design 47 G A-204, A-211), so a put that ends while the assigner is stopped gets its wait-ended at the next start.
PutWaitsState = "put-waits"
PutWaits = plc.Module + "_" + PutWaitsState
// Inputs holds the inputs of the composition steps, `<time>.<digest>`, which the seat mesh-wants' verb input answers (design 47 G §354).
InputsState = "inputs"
Inputs = plc.Module + "_" + InputsState
@@ -56,18 +66,14 @@ const (
// is its state provisions.
ProvisionerModule = "mesh-provisioner"
Provisions = ProvisionerModule + "_provisions"
// Presence holds each part's presence and the node's.
Presence = "mesh-delivery-watchman_" + watchdogs.StatePresence
// Source and Merges are the gate's.
Source = "mesh-delivery-gate_" + delivery.StateSource
Merges = "mesh-delivery-gate_" + delivery.StateMerges
// TellerModule is the module whose hand the operator's answers pass: the one that holds each ask's answer, and the
// name the hand-act log records a warrant under. Held equal to the manifest's reads by its test.
TellerModule = "mesh-delivery-teller"
// Answers holds the teller's answer to each ask.
Answers = TellerModule + "_answers"
// Builds is the builder's.
Builds = builds.BuilderBuildsBucket
// AnswersState is the teller's state of its answer to each ask, followed through its first view by AnswersReadTool and its
// views family `views.answers` (design 47 G A-175, A-178), never its bucket; Answers is the picture's name, `<owner>_<state>`.
AnswersState = "answers"
AnswersReadTool = "teller_read_answers"
Answers = TellerModule + "_" + AnswersState
// Grants is the grants part's.
Grants = grantsstate.PartGrantsBucket
)
modules/mesh-delivery-assigner/internal/buckets/buckets_test.go
+3 -4
View File
@@ -8,10 +8,9 @@ func TestBucketsAreTheEarlierNames(t *testing.T) {
for name, c := range map[string][2]string{
"Have": {Have, "node-engine_have"}, "ComposedRead": {ComposedRead, "mesh-host.composed"},
"Modules": {Modules, "mesh-delivery-indexer_modules"}, "Manifests": {Manifests, "mesh-delivery-indexer_manifests"},
"Presence": {Presence, "mesh-delivery-watchman_presence"}, "Source": {Source, "mesh-delivery-gate_source"},
"Merges": {Merges, "mesh-delivery-gate_merges"}, "Answers": {Answers, "mesh-delivery-teller_answers"},
"Builds": {Builds, "mesh-delivery-builder_builds"}, "Grants": {Grants, "mesh-grants_grants"},
"Wants": {Wants, "mesh-delivery-assigner_wants"}, "Differences": {Differences, "mesh-delivery-assigner_differences"},
"Answers": {Answers, "mesh-delivery-teller_answers"},
"Grants": {Grants, "mesh-grants_grants"},
"Wants": {Wants, "mesh-delivery-assigner_wants"}, "Differences": {Differences, "mesh-delivery-assigner_differences"},
"Asks": {Asks, "mesh-delivery-assigner_asks"}, "Puts": {Puts, "mesh-delivery-assigner_puts"},
"Policies": {Policies, "mesh-delivery-assigner_policies"}, "Provisions": {Provisions, "mesh-provisioner_provisions"}, "TellerModule": {TellerModule, "mesh-delivery-teller"},
} {
modules/mesh-delivery-assigner/internal/puts/puts.go
+10
View File
@@ -97,6 +97,16 @@ func (p Put) Said(at time.Time, step, words string) Put {
return p
}
// StandsSince is when p came to stand at its step: the time of the first of its newest steps at that step (a step's words
// may change while it stands, each change a step of its own), or when it was asked when its steps say none.
func (p Put) StandsSince() time.Time {
since := p.At
for i := len(p.Steps) - 1; i >= 0 && p.Steps[i].Step == p.Step; i-- {
since = p.Steps[i].At
}
return since
}
var digestForm = regexp.MustCompile(`^sha256:[0-9a-f]{64}$`)
func (p Put) check() error {
modules/mesh-delivery-assigner/manifest_test.go
+14 -8
View File
@@ -43,17 +43,20 @@ func TestManifest(t *testing.T) {
reads := slices.Clone(m.Reads)
slices.Sort(reads)
// ADR 0593 §8: it composes nothing, so it reads no candidate, held or resolved value, compose step or build log; it reads
// every node's published compositions to compare them, and the gate's source and merges, for the release that carries a build (ADR 0601 §1).
// every node's published compositions to compare them,.
// DLV-F2r (ADR 0615 §1, ADR 0631 §1): the indexer's manifests, modules and module graph (whose verb edges a want waits
// on, ADR 0614 §2, §3) are read through its first views and views family, none of the indexer's states by reads.
want := []string{"mesh-delivery-builder.builds", "mesh-delivery-gate.merges", "mesh-delivery-gate.source",
"mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "mesh-grants.grants", "mesh-host.composed",
// A-181: what was built is heard from the builder's stream of ends, so the builder's builds bucket is no read.
// §431, §507: the gate's source is read through its verb `source` and its event `source-moved`, and no `merges` is read.
// A-175, A-178: the teller's answers are followed through its first view and its views family, never its bucket.
// §389: the watchman's machine-away and machine-back are consumed, never its presence.
want := []string{"mesh-grants.grants", "mesh-host.composed",
"mesh-provisioner.provisions", "node-engine.have"}
if !reflect.DeepEqual(reads, want) {
t.Errorf("reads = %v, want exactly %v", reads, want)
}
// the buckets the program names are the manifest's reads, `<module>.<state>` as `<module>_<state>`
named := []string{buckets.Builds, buckets.Merges, buckets.Source, buckets.Answers, buckets.Presence, buckets.Grants,
named := []string{buckets.Grants,
buckets.ComposedRead, "mesh-provisioner_provisions", buckets.Have}
slices.Sort(named)
for i, r := range want {
@@ -67,7 +70,7 @@ func TestManifest(t *testing.T) {
}
if want := map[string]int{"asks": 5, "differences": 64, "puts": 20, "wants": 20,
"assignments": 64, "placement": 64, "copies": 1, "proposals": 5, "candidates": 5, "pulses": 0, "findings": 5, "policies": 20, "seat-holders": 5, views.MarksState: 1, views.StatusState: 1,
"engine-reads": 1, "written": 5, "waits": 5, "inputs": 1}; !reflect.DeepEqual(states, want) {
"engine-reads": 1, "written": 5, "waits": 5, "put-waits": 1, "delivery-waits": 1, "delivered-again": 1, "inputs": 1, "pins": 5}; !reflect.DeepEqual(states, want) {
t.Errorf("state = %v, want %v", states, want)
}
if _, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 {
@@ -102,7 +105,7 @@ func TestManifest(t *testing.T) {
// which lists and asks again what each node's engine reads.
if !slices.Equal(m.Tools, []string{"assigner_put", "assigner_release", "assigner_puts", "assigner_waits", "assigner_inputs", "assignments", "seat-holders", "wants",
"assigner_read_assignments", "assigner_read_seat_holders", "assigner_read_wants", "assigner_read_findings", "assigner_read_asks", "assigner_read_differences", "assigner_read_copies",
"assigner_first_views", "assigner_first_view_again", "assigner_engine_reads", "assigner_seat_list", "assigner_seat_list_again", "assigner_ask_again", "assigner_written"}) {
"assigner_first_views", "assigner_first_view_again", "assigner_engine_reads", "assigner_seat_list", "assigner_seat_list_again", "assigner_ask_again", "assigner_written", "assigner_pins", "assigner_unpin", "assigner_delivered_again"}) {
t.Errorf("the assigner serves tools %v, want its three put tools, three first views, six read tools, assigner_engine_reads, the seat list's two (ADR 0636 §7) and assigner_ask_again (ADR 0668 §5)", m.Tools)
}
if m.RunsAs != m.Module || m.SecretsOwner != m.RunsAs {
@@ -129,13 +132,16 @@ func TestManifest(t *testing.T) {
}
}
// ADR 0618 §4: and it hears each node's engine say its reads changed; ADR 0636 §4 (issue 1966): and the seat list's two events;
// issue 1966 note 151: and the teller's ask-ended, its subject the one the assigner hears.
if !slices.Equal(m.Consumes, []string{"mesh-conditions.findings-asked", "mesh-conditions.condition-raised", "mesh-conditions.condition-changed", "mesh-conditions.condition-cleared", "mesh-catalog.catalog-revision", "mesh-catalog.registration-changed", "mesh-catalog.version-released", "mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", builds.BuilderModule + "." + builds.BuildEndedEvent, buckets.TellerModule + "." + tellerevents.AskEnded, "mesh-pipeline-runner.pipeline-ended", "mesh-pipeline-runner.group-ended", "node-engine.manifest-reads-changed", "node-engine.running"}) {
// issue 1966 note 151: and the teller's ask-ended, its subject the one the assigner hears; §376, §389: its own version-delivered and
// the watchman's machine-away and machine-back, read whole at its start.
if !slices.Equal(m.Consumes, []string{"mesh-conditions.findings-asked", "mesh-conditions.condition-raised", "mesh-conditions.condition-changed", "mesh-conditions.condition-cleared", "mesh-catalog.catalog-revision", "mesh-catalog.registration-changed", "mesh-delivery.source-moved", "mesh-catalog.version-released", "mesh-wants.version-delivered", "mesh-watchdogs.machine-away", "mesh-watchdogs.machine-back", "mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", builds.BuilderModule + "." + builds.BuildsStreamEvents, buckets.TellerModule + "." + tellerevents.AskEnded, buckets.TellerModule + ".views." + buckets.AnswersState, "mesh-pipeline-runner.pipeline-ended", "mesh-pipeline-runner.group-ended", "node-engine.manifest-reads-changed", "node-engine.running", "mesh-delivery.pinned", "mesh-delivery.freed", "mesh-delivery.heads-replaced", "mesh-delivery.pins-stated"}) {
t.Errorf("consumes = %v, want the catalog half's revisions (design 47 G §195), the teller's ask-ended, the engines' reads changed and what they run", m.Consumes)
}
if want := []string{"mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules",
buckets.TellerModule + "." + buckets.AnswersReadTool, // A-175, A-178: the teller's answers' first view
"seat:mesh-catalog.registrations", "seat:mesh-catalog.revision", "seat:mesh-catalog.seat-definitions", // ADR 0636 §4: the seat list
"seat:mesh-conditions.conditions", // design 47 G §706: the conditions read at each judging of an outside wait
"seat:mesh-delivery.source", // §507: the branches the gate keeps a source revision of, before the first registrations
"seat:mesh-hand-acts.hand-act", "seat:mesh-hand-acts.hand-acts", "seat:mesh-hand-acts.warranted", // ADR 0565 §9, §11: the seat alone
"seat:node-backup.backed-up", "seat:node-backup.now",
"seat:node-engine.manifest-reads", // ADR 0618 §3
modules/mesh-delivery-assigner/module.json
+22 -9
View File
@@ -30,11 +30,6 @@
}
],
"reads": [
"mesh-delivery-builder.builds",
"mesh-delivery-gate.merges",
"mesh-delivery-gate.source",
"mesh-delivery-teller.answers",
"mesh-delivery-watchman.presence",
"mesh-grants.grants",
"mesh-host.composed",
"mesh-provisioner.provisions",
@@ -50,23 +45,34 @@
"mesh-conditions.condition-cleared",
"mesh-catalog.catalog-revision",
"mesh-catalog.registration-changed",
"mesh-delivery.source-moved",
"mesh-catalog.version-released",
"mesh-wants.version-delivered",
"mesh-watchdogs.machine-away",
"mesh-watchdogs.machine-back",
"mesh-catalog.seat-definition-changed",
"mesh-catalog.seat-definition-removed",
"mesh-delivery-builder.build.ended",
"mesh-delivery-builder.build.ended.**",
"mesh-delivery-teller.ask-ended",
"mesh-delivery-teller.views.answers",
"mesh-pipeline-runner.pipeline-ended",
"mesh-pipeline-runner.group-ended",
"node-engine.manifest-reads-changed",
"node-engine.running"
"node-engine.running",
"mesh-delivery.pinned",
"mesh-delivery.freed",
"mesh-delivery.heads-replaced",
"mesh-delivery.pins-stated"
],
"invokes": [
"mesh-delivery-indexer.manifests",
"mesh-delivery-indexer.modules",
"mesh-delivery-teller.teller_read_answers",
"seat:mesh-catalog.registrations",
"seat:mesh-catalog.revision",
"seat:mesh-catalog.seat-definitions",
"seat:mesh-conditions.conditions",
"seat:mesh-delivery.source",
"seat:mesh-hand-acts.hand-act",
"seat:mesh-hand-acts.hand-acts",
"seat:mesh-hand-acts.warranted",
@@ -116,7 +122,10 @@
"assigner_seat_list",
"assigner_seat_list_again",
"assigner_ask_again",
"assigner_written"
"assigner_written",
"assigner_pins",
"assigner_unpin",
"assigner_delivered_again"
],
"state": [
{ "name": "assignments", "history": 64 },
@@ -137,7 +146,11 @@
{ "name": "engine-reads", "history": 1 },
{ "name": "written", "history": 5 },
{ "name": "waits", "history": 5 },
{ "name": "inputs", "history": 1 }
{ "name": "put-waits", "history": 1 },
{ "name": "delivery-waits", "history": 1 },
{ "name": "delivered-again", "history": 1 },
{ "name": "inputs", "history": 1 },
{ "name": "pins", "history": 5, "max-mib": 4 }
],
"config": {
"outside-wait-minutes": {
modules/mesh-delivery-builder/builds/ended.go
+16 -8
View File
@@ -17,11 +17,19 @@ const (
// BuildEnded is the body of the event build.ended: the module, the fingerprint, each artifact made with its digest, the build
// node that keeps a kept build (none for one not kept) and the watched branch a build of a candidate is for (none for the trunk).
type BuildEnded struct {
Module string `json:"module"`
Fingerprint string `json:"fingerprint"`
Made map[string]string `json:"made,omitempty"`
KeptOn string `json:"kept-on,omitempty"`
Branch string `json:"branch,omitempty"`
Module string `json:"module"`
Fingerprint string `json:"fingerprint"`
// Outcome is done or failed (design 47 G §575, A-162): the event is said for every end that is one of these; Error is why a
// failed build failed, as its key holds it (`transport fault:` included, §568), none for a done one.
Outcome string `json:"outcome"`
Error string `json:"error,omitempty"`
// Manifest is the digest of the manifest built and Needs the reader capabilities its reference forms need (ADR 0618 §1), of a
// done build: what a reader of the end asks of the builder's key otherwise.
Manifest string `json:"manifest,omitempty"`
Needs []string `json:"needs,omitempty"`
Made map[string]string `json:"made,omitempty"`
KeptOn string `json:"kept-on,omitempty"`
Branch string `json:"branch,omitempty"`
}
// BuildEndedEventName is the event's name for a module and fingerprint: `build.ended.<module>.<fingerprint>`, each escaped.
@@ -29,15 +37,15 @@ func BuildEndedEventName(module, fp string) string {
return BuildEndedEvent + "." + keys.EscapeName(module) + "." + keys.EscapeName(fp)
}
// BuildEndedOf is the event of a log entry of a build's end: false for any entry but the end of a done build, which says no
// BuildEndedOf is the event of a log entry of a build's end: false for any entry but the end of a done or a failed build. A done end says no
// digest of any part.
func BuildEndedOf(e BuildLogEntry) (BuildEnded, bool) {
if e.Event != BuildEndedEvent || e.Outcome != BuildDone || e.Module == "" || e.FP == "" {
if e.Event != BuildEndedEvent || (e.Outcome != BuildDone && e.Outcome != BuildFailed) || e.Module == "" || e.FP == "" {
return BuildEnded{}, false
}
made := map[string]string{}
for _, a := range e.Made {
made[a.Name] = a.Digest
}
return BuildEnded{Module: e.Module, Fingerprint: e.FP, Made: made, KeptOn: e.KeptOn, Branch: e.Ref}, true
return BuildEnded{Module: e.Module, Fingerprint: e.FP, Outcome: e.Outcome, Error: e.Error, Manifest: e.Manifest, Needs: e.Needs, Made: made, KeptOn: e.KeptOn, Branch: e.Ref}, true
}
modules/mesh-delivery-builder/builds/ended_test.go
+12 -2
View File
@@ -11,7 +11,7 @@ func TestADoneBuildsEndIsTheEventBuildEnded(t *testing.T) {
made := []BuiltArtifact{{Name: "program", Kind: "program", Digest: "sha256:1"}}
e := BuildLogEntry{Event: BuildEndedEvent, Outcome: BuildDone, Module: "node-engine", FP: "src2:a", Ref: "cand", KeptOn: "ace", Made: made}
got, ok := BuildEndedOf(e)
want := BuildEnded{Module: "node-engine", Fingerprint: "src2:a", Made: map[string]string{"program": "sha256:1"}, KeptOn: "ace", Branch: "cand"}
want := BuildEnded{Module: "node-engine", Fingerprint: "src2:a", Outcome: BuildDone, Made: map[string]string{"program": "sha256:1"}, KeptOn: "ace", Branch: "cand"}
if !ok || !reflect.DeepEqual(got, want) {
t.Errorf("event = %+v, %v", got, ok)
}
@@ -20,7 +20,7 @@ func TestADoneBuildsEndIsTheEventBuildEnded(t *testing.T) {
}
for name, mut := range map[string]func(*BuildLogEntry){
"started": func(e *BuildLogEntry) { e.Event = BuildStartedEvent },
"failed": func(e *BuildLogEntry) { e.Outcome = BuildFailed },
"queued": func(e *BuildLogEntry) { e.Outcome = BuildQueued },
"no fp": func(e *BuildLogEntry) { e.FP = "" },
} {
c := e
@@ -30,3 +30,13 @@ func TestADoneBuildsEndIsTheEventBuildEnded(t *testing.T) {
}
}
}
// Design 47 G §575, A-162: a failed build's end is said too, with its outcome and its error whole.
func TestAFailedBuildsEndIsSaidWithItsError(t *testing.T) {
e := BuildLogEntry{Event: BuildEndedEvent, Outcome: BuildFailed, Module: "web", FP: "src2:b", Error: "transport fault: no responders at mesh-build-go"}
got, ok := BuildEndedOf(e)
want := BuildEnded{Module: "web", Fingerprint: "src2:b", Outcome: BuildFailed, Error: e.Error, Made: map[string]string{}}
if !ok || !reflect.DeepEqual(got, want) {
t.Errorf("event = %+v, %v", got, ok)
}
}
modules/mesh-delivery-builder/builds/log.go
+3
View File
@@ -83,6 +83,9 @@ type BuildLogEntry struct {
// narration names neither.
Error string `json:"error,omitempty"`
Step string `json:"step,omitempty"`
// Needs are, at a done build's end, the reader capabilities its reference forms need (BuilderBuild.Needs, ADR 0618 §1): the end
// event carries them (design 47 G §575, A-162).
Needs []string `json:"needs,omitempty"`
// Line is a narration's words, Why what it narrates (asked, waiting, stored, refused): a build.said entry's alone.
Line string `json:"line,omitempty"`
Why string `json:"why,omitempty"`
modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go
+11 -2
View File
@@ -594,7 +594,7 @@ func TestBuilderMakesADoneKeyWithTheDigests(t *testing.T) { // 1607/AC7
t.Errorf("no build.ended of netcheck's build: %v", err)
} else {
var ev builds.BuildEnded
if err := json.Unmarshal(raw.Data, &ev); err != nil || ev.Module != "netcheck" || ev.Fingerprint != fp || len(ev.Made) != 2 {
if err := json.Unmarshal(raw.Data, &ev); err != nil || ev.Module != "netcheck" || ev.Fingerprint != fp || ev.Outcome != builds.BuildDone || len(ev.Made) != 2 {
t.Errorf("build.ended = %s (%v), want netcheck at its fingerprint with two artifacts' digests", raw.Data, err)
}
}
@@ -656,9 +656,18 @@ func TestBuilderMakesADoneKeyWithTheDigests(t *testing.T) { // 1607/AC7
b.fake.hang.Store(&release)
third := "src2:" + strings.Repeat("4", 64)
b.moduleFP(t, "netcheck", netcheck, "modules/netcheck", bothToolchains, commit, third, 3)
if f := b.awaitFP(t, "netcheck", third, isState(builds.BuildFailed)); !strings.Contains(f.Error, "did not end within") {
f := b.awaitFP(t, "netcheck", third, isState(builds.BuildFailed))
if !strings.Contains(f.Error, "did not end within") {
t.Errorf("the failed build = %+v, want its bound run out", f)
}
// A-162: the failed end is said as build.ended too, with its error whole.
if st, err := b.js.Stream(b.ctx, "BUILDER_BUILDS"); err != nil {
t.Fatal(err)
} else if raw, err := st.GetLastMsgForSubject(b.ctx, "mesh.stream.mesh-delivery-builder.event."+builds.BuildEndedEventName("netcheck", third)); err != nil {
t.Errorf("no build.ended of netcheck's failed build: %v", err)
} else if ev := (builds.BuildEnded{}); json.Unmarshal(raw.Data, &ev) != nil || ev.Outcome != builds.BuildFailed || ev.Error != f.Error {
t.Errorf("build.ended = %s, want the failed outcome with the error %q", raw.Data, f.Error)
}
failed := meshconditionsclient.Key(meshconditionsclient.ScopeModule, "netcheck", BuildFailed)
reg.awaitFinding(t, failed, true)
b.fake.hang.Store(nil)
modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reasks.go
+2 -2
View File
@@ -161,9 +161,9 @@ func appendLog(ctx context.Context, js jetstream.JetStream, cred process.Account
return nil
}
// emitBuildEnded says the builder's event build.ended of a done build, on the builder's own stream (design 47 G §379, A-96):
// emitBuildEnded says the builder's event build.ended of a done or failed build (A-162), on the builder's own stream (design 47 G §379, A-96):
// mesh.stream.<module>.event.build.ended.<module>.<fingerprint>, published where the log entry of the end is written. An
// entry that is not a done build's end says none. The id makes a repeated end one event.
// entry that is not a done or failed build's end says none. The id makes a repeated end one event.
func emitBuildEnded(ctx context.Context, js jetstream.JetStream, cred process.Account, e builds.BuildLogEntry) error {
body, ok := builds.BuildEndedOf(e)
if !ok {
modules/mesh-delivery-builder/internal/build/run.go
+1
View File
@@ -1214,6 +1214,7 @@ func LogEnded(module string, m meshcatalog.ModulesByModuleValue, ref, by string,
e.KeptOn = result.KeptOn
e.Imports = append([]string(nil), result.Imports...) // the indexer keeps them on the module (novox/hq ADR 0593 §2)
e.Error, e.Step = result.Error, result.Step // a failed build's reason, not only its outcome
e.Needs = append([]string(nil), result.Needs...)
return e
}
modules/mesh-delivery-builder/manifest_test.go
+7 -6
View File
@@ -53,10 +53,11 @@ type manifest struct {
Consumes []string `json:"consumes"`
State []state `json:"state"`
Streams []struct {
Name string `json:"name"`
Events []string `json:"events"`
MaxAgeDays int `json:"max-age-days"`
MaxMiB int `json:"max-mib"`
Name string `json:"name"`
Events []string `json:"events"`
MaxAgeDays int `json:"max-age-days"`
MaxMiB int `json:"max-mib"`
LastPerSubject bool `json:"last-per-subject"`
} `json:"streams"`
Logs []struct {
Name string `json:"name"`
@@ -177,9 +178,9 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
// Design 47 G §672: the toolchain waits' stream, kept 7 days within 16 MiB, and the bound's setting, 60 minutes.
// Design 47 G A-30, §383: and its own findings' events, which its finding writer says.
if len(m.Streams) != 3 || m.Streams[0].Name != "waits" || !reflect.DeepEqual(m.Streams[0].Events, []string{"wait-held.**", "wait-ended.**"}) || m.Streams[0].MaxAgeDays != 7 || m.Streams[0].MaxMiB != 16 ||
m.Streams[1].Name != builds.BuildsStream || !reflect.DeepEqual(m.Streams[1].Events, []string{builds.BuildsStreamEvents}) || m.Streams[1].MaxAgeDays != 7 ||
m.Streams[1].Name != builds.BuildsStream || !reflect.DeepEqual(m.Streams[1].Events, []string{builds.BuildsStreamEvents}) || m.Streams[1].MaxAgeDays != 7 || !m.Streams[1].LastPerSubject ||
m.Streams[2].Name != "mesh-delivery-builder-findings" || !reflect.DeepEqual(m.Streams[2].Events, []string{"finding-written.*", "finding-cleared.*", "findings-stated.*"}) {
t.Errorf("streams = %+v, want waits with wait-held.** and wait-ended.**, 7 days, 16 MiB, the builds' stream (build.ended.**, 7 days) and the findings' stream", m.Streams)
t.Errorf("streams = %+v, want waits with wait-held.** and wait-ended.**, 7 days, 16 MiB, the builds' stream (build.ended.**, 7 days, kept last per subject: design 47 G A-181) and the findings' stream", m.Streams)
}
if c, ok := m.Config["toolchain-wait-minutes"]; !ok || c.Default != float64(60) || len(m.Config) != 2 || m.Config["republish-max-versions"].Default != float64(50) {
t.Errorf("config = %+v, want toolchain-wait-minutes of 60 and republish-max-versions of 50 (design 47 G §588, §633)", m.Config)
modules/mesh-delivery-builder/module.json
+1 -1
View File
@@ -130,7 +130,7 @@
],
"streams": [
{ "name": "waits", "events": ["wait-held.**", "wait-ended.**"], "max-age-days": 7, "max-mib": 16 },
{ "name": "mesh-delivery-builder-builds", "events": ["build.ended.**"], "max-age-days": 7, "max-mib": 16 },
{ "name": "mesh-delivery-builder-builds", "events": ["build.ended.**"], "max-age-days": 7, "max-mib": 16, "last-per-subject": true },
{ "name": "mesh-delivery-builder-findings", "events": ["finding-written.*", "finding-cleared.*", "findings-stated.*"], "max-age-days": 7, "max-mib": 16 }
],
"config": {
modules/mesh-delivery-contracts/go/clients/meshcatalogclient/branches.go
+6
View File
@@ -68,6 +68,9 @@ type Branches struct {
Entries *Entries
// Changed, when not nil, is told of each change to a branch's view (module, its entry, deleted), set before Wire.
Changed func(branch, module string, entry json.RawMessage, deleted bool, at time.Time)
// Adopted, when not nil, is told on a goroutine of its own of each branch whose view was adopted, with that view (a part that waits for
// its first branch's first view, design 47 G §507), set before Wire.
Adopted func(branch string, c *Catalog)
// Logf, when not nil, says each view adopted and dropped.
Logf func(format string, args ...any)
@@ -250,6 +253,9 @@ func (b *Branches) reconcile() {
b.catalogs[e] = c
b.order = append(b.order, e)
b.logf("meshcatalogclient: the branch %q is followed", want[e])
if b.Adopted != nil {
go b.Adopted(branch, c)
}
}
for _, e := range slices.Clone(b.order) {
if _, ok := want[e]; ok {
modules/mesh-delivery-contracts/go/clients/meshcatalogclient/branches_test.go
+18
View File
@@ -126,6 +126,12 @@ func TestBranchesFollowSourceThenRegistrations(t *testing.T) {
t.Fatal(err)
}
b.Logf = func(format string, args ...any) { said <- fmt.Sprintf(format, args...) }
adopted := make(chan string, 8)
b.Adopted = func(branch string, c *Catalog) {
if c.Branch == branch {
adopted <- branch
}
}
want := func(names ...string) {
t.Helper()
if got := w.names(); !slices.Equal(got, names) {
@@ -148,6 +154,18 @@ func TestBranchesFollowSourceThenRegistrations(t *testing.T) {
t.Fatal(err)
}
waitFor(`"feat/x" is followed`)
told := []string{} // each adopted branch is told, with its view (a part waits for its first one's first view), in no set order
for range 2 {
select {
case got := <-adopted:
told = append(told, got)
case <-ctx.Done():
t.Fatalf("told of %v, want main and feat/x", told)
}
}
if slices.Sort(told); !slices.Equal(told, []string{"feat/x", "main"}) {
t.Fatalf("told of %v", told)
}
if got := b.Followed(); !slices.Equal(got, []string{"main", "feat/x"}) {
t.Fatalf("followed = %v", got)
}
modules/mesh-delivery-contracts/go/seats/meshwants/contract.go
+10 -1
View File
@@ -472,7 +472,9 @@ type AssignmentEntry struct {
Catalog string `json:"catalog"`
Edges []Edge `json:"edges,omitempty"`
// the fingerprint it is to run
Fp string `json:"fp"`
Fp string `json:"fp"`
// the requirements no step of the order resolves (design 47 G section 357): the entry is held, naming the candidates, and nothing is guessed
Held []Held `json:"held,omitempty"`
Module string `json:"module"`
Node string `json:"node"`
// the layout version
@@ -582,6 +584,13 @@ type Fill struct {
Store string `json:"store"`
}
// Held is one requirement an entry is held on, for want of a provider.
type Held struct {
// a provisioning edge to each placed provider of it, none chosen; none when no placed module provides it
Candidates []Edge `json:"candidates,omitempty"`
Provision string `json:"provision"`
}
// NodeEntryRef is a type of the contract.
type NodeEntryRef struct {
// the entry's digest, its object's name
modules/mesh-delivery-contracts/seats/mesh-wants/seat.json
+26
View File
@@ -825,6 +825,13 @@
"items": {
"$ref": "#/defs/Edge"
}
},
"held": {
"type": "array",
"description": "the requirements no step of the order resolves (design 47 G section 357): the entry is held, naming the candidates, and nothing is guessed",
"items": {
"$ref": "#/defs/Held"
}
}
},
"required": [
@@ -835,6 +842,25 @@
"catalog"
]
},
"Held": {
"type": "object",
"description": "one requirement an entry is held on, for want of a provider",
"properties": {
"provision": {
"type": "string"
},
"candidates": {
"type": "array",
"description": "a provisioning edge to each placed provider of it, none chosen; none when no placed module provides it",
"items": {
"$ref": "#/defs/Edge"
}
}
},
"required": [
"provision"
]
},
"NodeEntryRef": {
"type": "object",
"properties": {
modules/mesh-delivery-contracts/ts/seats/meshwants/index.d.ts
Vendored
+9
View File
@@ -143,6 +143,8 @@ export interface AssignmentEntry {
"edges"?: Edge[];
/** the fingerprint it is to run */
"fp": string;
/** the requirements no step of the order resolves (design 47 G section 357): the entry is held, naming the candidates, and nothing is guessed */
"held"?: Held[];
"module": string;
"node": string;
/** the layout version */
@@ -250,6 +252,13 @@ export interface Fill {
"store": string;
}
/** one requirement an entry is held on, for want of a provider */
export interface Held {
/** a provisioning edge to each placed provider of it, none chosen; none when no placed module provides it */
"candidates"?: Edge[];
"provision": string;
}
export interface NodeEntryRef {
/** the entry's digest, its object's name */
"digest": string;
modules/mesh-delivery-indexer/internal/index/candidates.go
+1 -1
View File
@@ -47,7 +47,7 @@ type Store interface {
// trunkLike are the branches that are no group: a pull request from one is nobody's delivery group (the rule of
// mesh-catalog modules/mesh-delivery/cmd/mesh-delivery/holder.go:321).
var trunkLike = map[string]bool{"main": true, "master": true, "trunk": true, "": true}
var trunkLike = map[string]bool{TrunkBranch: true, "master": true, "trunk": true, "": true}
// writeTries bounds the reads again after a conflict; a message that still conflicts is left to the server to deliver again.
const writeTries = 8
modules/mesh-delivery-indexer/internal/index/trunk_named_test.go
+37
View File
@@ -0,0 +1,37 @@
package index
import (
"os"
"path/filepath"
"strings"
"testing"
)
// A-93, design 47 G §7: the trunk `main` is named once in the indexer, as TrunkBranch; no other non-test source of the module holds the
// string, so the first watched branch changes in one place.
func TestTheTrunkIsNamedOnceInTheIndexer(t *testing.T) {
var named []string
for _, dir := range []string{".", "../../cmd/mesh-delivery-indexer", "../judge"} {
files, err := filepath.Glob(filepath.Join(dir, "*.go"))
if err != nil {
t.Fatal(err)
}
for _, f := range files {
if strings.HasSuffix(f, "_test.go") {
continue
}
src, err := os.ReadFile(f)
if err != nil {
t.Fatal(err)
}
for _, line := range strings.Split(string(src), "\n") {
if t := strings.TrimSpace(line); strings.Contains(line, `"main"`) && !strings.HasPrefix(t, "//") {
named = append(named, filepath.Base(f))
}
}
}
}
if len(named) != 1 {
t.Errorf(`"main" is named %d times in code (%v), want once: TrunkBranch`, len(named), named)
}
}
modules/mesh-delivery-teller/cmd/mesh-delivery-teller-tools/main.go
+14
View File
@@ -104,6 +104,19 @@ func AnswerTools(s answers) []stdio.Tool {
}}}
}
// ReadAnswersTool is the teller's ordinary read tool of its state answers (design 47 G A-175, A-178; mesh-sdk views.ReadTool):
// what the view follows, first by it and then by the teller's views family views.answers. Its marks are the state teller-marks.
const ReadAnswersTool = "teller_read_answers"
// marksState is the teller's state of what its views family said each state up to (A-178).
const marksState = "teller-marks"
// ReadAnswersTools is the tool teller_read_answers over the state answers and its marks.
func ReadAnswersTools(values, marks views.Source, now func() time.Time) []stdio.Tool {
return []stdio.Tool{views.ReadTool("mesh-delivery-teller", ReadAnswersTool, "answers",
"the operator's answers to the assigner's asks, by the ask's key.", values, marks, now)}
}
// WidensSaidTool reads the teller's state widens-said (design 47 G §335).
const WidensSaidTool = "teller_widens_said"
@@ -205,6 +218,7 @@ func main() {
tools := append(Tools(askEnd(stdio.State(ends.State), 25*time.Second), time.Now),
views.StatusTools("mesh-delivery-teller", "teller_first_views", "teller_first_view_again", stdio.State(views.StatusState), meshhandactsclient.Recorder{Ask: stdio.Ask}, time.Now)...)
tools = append(tools, AnswerTools(stdio.State("answers"))...)
tools = append(tools, ReadAnswersTools(views.KeptSource{State: stdio.State("answers")}, views.KeptSource{State: stdio.State(marksState)}, time.Now)...)
tools = append(tools, WidensSaidTools(stdio.State(widensSaidState))...)
if err := stdio.Serve("", tools); err != nil {
fmt.Fprintf(os.Stderr, "[mesh-delivery-teller-tools] %v\n", err)
modules/mesh-delivery-teller/cmd/mesh-delivery-teller-tools/main_test.go
+35
View File
@@ -4,8 +4,10 @@ import (
"encoding/json"
"fmt"
"testing"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/views"
)
type fakeAnswers map[string]string
@@ -64,3 +66,36 @@ func TestTellerWidensSaidReadsOneKeyOrEveryKey(t *testing.T) {
t.Errorf("every key: %s, %v", raw, err)
}
}
// teller_read_answers is the teller's ordinary read tool of its state answers (A-175, A-178): the key's value and revision, the
// keys, over the answers state and the marks state teller-marks.
type entries map[string]views.Entry
func (e entries) Keys() ([]string, error) {
var out []string
for k := range e {
out = append(out, k)
}
return out, nil
}
func (e entries) Get(key string) (*views.Entry, bool, error) {
v, ok := e[key]
return &v, ok, nil
}
func TestTellerReadAnswersIsTheViewsReadToolOfTheAnswersState(t *testing.T) {
values := entries{"hold.k1": {Value: json.RawMessage(`{"option":"keep"}`), Revision: 3}}
tool := ReadAnswersTools(values, entries{}, time.Now)[0]
if tool.Name != "teller_read_answers" {
t.Fatalf("the tool is %s", tool.Name)
}
out, err := tool.Run(map[string]any{"key": "hold.k1"})
raw, _ := json.Marshal(out)
if err != nil || string(raw) != `{"owner":"mesh-delivery-teller","state":"answers","key":"hold.k1","found":true,"value":{"option":"keep"},"revision":3}` {
t.Errorf("one key: %s, %v", raw, err)
}
if marksState != "teller-marks" {
t.Errorf("the marks state is %s", marksState)
}
}
modules/mesh-delivery-teller/cmd/mesh-delivery-teller/assignerasks.go
+4 -4
View File
@@ -25,9 +25,9 @@ import (
// of these asks needs proof of who answered (mesh-sdk go/delivery AskNeedsProof).
// mostAssignerOpen is how many of the assigner's asks the teller keeps open at the router at once. The router keeps at most
// occ.MostOpen open asks of one asker, and the teller asks both the gate's merge asks and the assigner's as one asker, so a burst
// of holds (seven at once on 2026-10-25) would take all the room and a merge Approve could not be asked; one place is always
// left for a merge ask. An assigner ask beyond these waits in the wait for room and is asked when one of them ends.
// occ.MostOpen open asks of one asker, and a burst of holds (seven at once on 2026-10-25) would take all the room; one place
// stays free (it was left for the merge ask, which went with the Approve per merge, design 47 G §653). An assigner ask beyond
// these waits in the wait for room and is asked when one of them ends.
const mostAssignerOpen = occ.MostOpen - 1
// assignerOpenRecord is the key of the router ids of the assigner's asks the teller asked.
@@ -89,7 +89,7 @@ func (t *Teller) waitForAssignerRoom(key, id string) {
}
}
// isAssignerKey says whether an ask key is one of the assigner's, not the gate's merge ask.
// isAssignerKey says whether an ask key is one of the assigner's, not the merge ask's key of before the Approve per merge went.
func isAssignerKey(key string) bool {
k, err := asks.ParseAnyAskKey(key)
return err == nil && k.Kind != meshdeliveryclient.AskKindMerge
modules/mesh-delivery-teller/cmd/mesh-delivery-teller/assignerasks_test.go
+11 -45
View File
@@ -151,22 +151,20 @@ func TestAnAssignerAskAnsweredInTheViewOrTakenAwayIsTakenBack(t *testing.T) {
}
}
// 1906/H1: an assigner ask refused for want of room waits and is asked again when another of the teller's asks ends, as a merge
// ask does; a refusal for its words is said as a condition about a delivery ask, not a merge.
// 1906/H1: an assigner ask refused for want of room waits and is asked again when another of the teller's asks ends; a refusal for
// its words is said as a condition about a delivery ask.
func TestAnAssignerAskWaitsForRoomAndARefusalIsSaidAsADeliveryAsk(t *testing.T) {
anAssignerAskWrittenAgainAfterBeingTakenAwayIsAskedAgainUnderANewAsking(t)
b := newBed(t)
assigner, _ := withView(b)
_, first := b.askOf(t, 1, 1)
b.askOf(t, 2, 2)
b.askOf(t, 3, 3)
_, first := b.holdOf(t, 1, 1)
key, value, ask := holdAsk(t)
assigner[key] = value
b.t.OnAssignerAsk(key)
b.refuse(t, ask.ID, roomRefusal)
b.deliver(t, warrantFor(first.Ask, "approve"))
b.deliver(t, warrantFor(first, "keep"))
got := b.askedIDs()
if len(got) != 5 || got[4] != ask.ID {
if len(got) != 3 || got[2] != ask.ID {
t.Fatalf("asks = %v, want the assigner's %s asked again", got, ask.ID)
}
@@ -181,60 +179,28 @@ func TestAnAssignerAskWaitsForRoomAndARefusalIsSaidAsADeliveryAsk(t *testing.T)
}
// ackFor is the router's warrant for an option of level acknowledge, as the assigner's asks offer.
func ackFor(a oc.Ask, option string) oc.Warrant {
w := warrantFor(a, option)
w.Level, w.Label = occ.Acknowledge, option
return w
}
func ackFor(a oc.Ask, option string) oc.Warrant { return warrantFor(a, option) }
// holdAskOf is the assigner's hold ask of module plex at the n-th fingerprint, shaped as holdAsk.
func holdAskOf(t *testing.T, n int) (string, []byte, oc.Ask) {
t.Helper()
key, err := asks.HoldAskKey("plex", "src2:"+strings.Repeat(string(rune('0'+n)), 64))
if err != nil {
t.Fatal(err)
}
_, _, ask := holdAsk(t)
ask.ID = meshdeliveryclient.AskID(key)
value, err := asks.EncodeAsk(key, asks.Ask{Ask: rawAsk(t, ask)})
if err != nil {
t.Fatal(err)
}
return key, value, ask
}
// 1906/H5: given seven holds at once (as on 2026-10-25), when they are raised, then at most two are open at the router, a merge
// ask is still asked, and each hold answered lets the next waiting one be asked.
func TestABurstOfHoldsLeavesRoomForAMergeAsk(t *testing.T) {
// 1906/H5: given seven holds at once (as on 2026-10-25), when they are raised, then at most mostAssignerOpen are open at the
// router, and each hold answered lets the next waiting one be asked.
func TestABurstOfHoldsAsksNoMoreThanTheMostOpenAndTheNextWhenOneEnds(t *testing.T) {
b := newBed(t)
assigner, _ := withView(b)
var held []oc.Ask
for n := 0; n < 7; n++ {
key, value, ask := holdAskOf(t, n)
assigner[key] = value
b.t.OnAssignerAsk(key)
_, ask := b.holdOf(t, n, 1)
held = append(held, ask)
}
if got := b.askedIDs(); len(got) != mostAssignerOpen || got[0] != held[0].ID || got[1] != held[1].ID {
t.Fatalf("asked %v, want the first %d holds only; log:\n%s", got, mostAssignerOpen, b.logs.String())
}
_, merge := b.askOf(t, 1, 1)
if got := b.askedIDs(); len(got) != mostAssignerOpen+1 || got[mostAssignerOpen] != merge.Ask.ID {
t.Fatalf("asked %v, want the merge ask asked beside the holds", got)
}
if len(b.dropped) != 0 || len(b.conds) != 0 {
t.Errorf("router dropped %v, conditions %+v", b.dropped, b.conds)
}
b.deliver(t, ackFor(held[0], "release"))
got := b.askedIDs()
if len(got) != mostAssignerOpen+2 || got[len(got)-1] != held[2].ID {
if len(got) != mostAssignerOpen+1 || got[len(got)-1] != held[2].ID {
t.Fatalf("asked %v after a hold was answered, want the third hold asked next; log:\n%s", got, b.logs.String())
}
// The merge ask answered frees its own place, and no hold takes it.
b.deliver(t, warrantFor(merge.Ask, "approve"))
if got := b.askedIDs(); len(got) != mostAssignerOpen+2 {
t.Errorf("asked %v after the merge ask was answered, want no further hold", got)
}
}
// Issue 1966 (nats on novox): an assigner ask taken back when its key went is asked again when the assigner writes the key
modules/mesh-delivery-teller/cmd/mesh-delivery-teller/integration_test.go
+80 -255
View File
@@ -25,25 +25,11 @@ import (
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-view/viewstate"
"mesh-delivery-teller/internal/ends"
)
// busConf is the shape of the composed accounts file (mesh-controller internal/broker): one account with JetStream, the
// controller's user and the teller's own principal <node>.<module>.
const busConf = `
listen: 127.0.0.1:-1
jetstream { store_dir: %q }
accounts {
MESH {
jetstream: enabled
users = [
{ user: "ctl", password: "pw" }
{ user: "n1.mesh-delivery-teller", password: "pw" }
]
}
}
`
type syncBuffer struct {
mu sync.Mutex
b strings.Builder
@@ -56,82 +42,6 @@ func (s *syncBuffer) Write(p []byte) (int, error) {
}
func (s *syncBuffer) String() string { s.mu.Lock(); defer s.mu.Unlock(); return s.b.String() }
// TestTellerWaitsForTheGate starts the program the way the node does (its entry, run, with MESH_BROKER_FILE) against an
// embedded bus where mesh-delivery-gate_asks does not exist: it keeps running until stopped, says once that it waits for
// the gate's asks, and writes and publishes nothing (the revised 1240/AC2: the watch is release 3d's).
func TestTellerWaitsForTheGate(t *testing.T) { // 1240/AC2
dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "t")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { os.RemoveAll(dir) })
conf := filepath.Join(dir, "bus.conf")
if err := os.WriteFile(conf, []byte(fmt.Sprintf(busConf, filepath.Join(dir, "js"))), 0o600); err != nil {
t.Fatal(err)
}
opts, err := server.ProcessConfigFile(conf)
if err != nil {
t.Fatal(err)
}
opts.NoLog, opts.NoSigs = true, true
srv, err := server.NewServer(opts)
if err != nil {
t.Fatal(err)
}
go srv.Start()
if !srv.ReadyForConnections(10 * time.Second) {
t.Fatal("the bus did not come up")
}
t.Cleanup(srv.Shutdown)
ctl, err := nats.Connect(srv.ClientURL(), nats.UserInfo("ctl", "pw"))
if err != nil {
t.Fatal(err)
}
t.Cleanup(ctl.Close)
js, err := jetstream.New(ctl)
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
broker := filepath.Join(t.TempDir(), "broker")
cred, _ := json.Marshal(process.Account{URL: srv.ClientURL(), Node: "n1", Module: "mesh-delivery-teller", User: "n1.mesh-delivery-teller", Password: "pw"})
if err := os.WriteFile(broker, cred, 0o600); err != nil {
t.Fatal(err)
}
env := map[string]string{"MESH_BROKER_FILE": broker}
logs := &syncBuffer{}
rn := startRunner(ctx, func(c context.Context) error { return run(c, func(k string) string { return env[k] }, logs) })
t.Cleanup(func() { rn.finish(t, 10*time.Second) })
for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline) && !strings.Contains(logs.String(), "waiting for the gate's asks"); time.Sleep(20 * time.Millisecond) {
}
if n := strings.Count(logs.String(), "waiting for the gate's asks"); n != 1 {
t.Fatalf("the teller said it waits %d times, want once: %q", n, logs.String())
}
select {
case err := <-rn.done:
t.Fatalf("the teller stopped: %v", err)
case <-time.After(500 * time.Millisecond):
}
if n := strings.Count(logs.String(), "waiting for the gate's asks"); n != 1 {
t.Errorf("the teller said it waits %d times after a while, want once", n)
}
// It writes and publishes nothing: no stream, no bucket exists.
names := js.StreamNames(ctx)
for name := range names.Name() {
t.Errorf("a stream %s exists: the teller wrote something", name)
}
if _, err := js.KeyValue(ctx, "mesh-delivery-gate_asks"); err == nil {
t.Error("the gate's asks bucket exists")
}
if err := rn.finish(t, 10*time.Second); err != nil {
t.Errorf("the teller stopped with %v, want a clean stop", err)
}
}
// The principals of the composed accounts file for the teller's tests, with the permissions mesh-controller composes for
// them (internal/broker): a module's own state is the read grants of its bucket and `$KV.<bucket>.>` to write it
// (state.go stateGrants, readGrants), a read of another module's bucket is the read grants alone, and a module that uses
@@ -171,9 +81,8 @@ func user(name string, pub, sub []string) string {
const (
tellerAnswers = "mesh-delivery-teller_answers"
tellerAsked = "mesh-delivery-teller_asked"
gateAsksKV = "mesh-delivery-gate_asks"
assignerAsks = "mesh-delivery-assigner_asks"
viewAnswers = "mesh-delivery-view_answers"
viewAnswers = viewstate.ViewAnswersBucket
tellerConsumer = "n1_mesh-delivery-teller"
seatAsk = "mesh.seat.operator-channel.accept.ask.mesh-delivery-teller"
seatCancel = "mesh.seat.operator-channel.accept.cancel.mesh-delivery-teller"
@@ -187,7 +96,7 @@ const (
func fullConf(dir string) string {
// ends is the teller's own state (manifest state entry ends), which the controller makes before it starts: a bed
// without it held the start ten seconds on a read the bus never answered, and the tests raced that stall.
teller := append(append(append(append(keepsOf(tellerAnswers), keepsOf(tellerAsked)...), keepsOf(ends.Bucket)...), readOf(gateAsksKV)...),
teller := append(append(append(keepsOf(tellerAnswers), keepsOf(tellerAsked)...), keepsOf(ends.Bucket)...),
seatAsk, seatCancel, seatCondition, "$JS.API.DIRECT.GET.KV_"+routerAsksBucket+".$KV."+routerAsksBucket+".mesh-delivery-teller.>")
teller = append(teller, keepsOf(widensSaidBucket)...) // design 47 G §335: the state widens-said
// The durable consumer on EVENTS the controller makes for the seat it uses, pulled and acknowledged (issue 1126).
@@ -195,14 +104,15 @@ func fullConf(dir string) string {
// The finding writer finds the stream of the keeper's findings-asked (design 47 G §384): the bed has none, so it is
// said and the findings stay in the state.
teller = append(teller, "$JS.API.STREAM.NAMES")
teller = append(teller, "mesh.mod.mesh-delivery-teller.event.answered") // the manifest's emits (ADR 0596 §2)
teller = append(teller, readOf(viewAnswers)...) // issue 1297 B10: the manifest's reads
// The assigner's asks by its read tool and its views family (ADR 0615, DLV-M3), and the teller's own first-views and
// findings; the bucket itself is no longer the teller's to read.
teller = append(append(append(teller, viewstest.Grants(assignerModule, assignerModule+".assigner_read_asks", tellerConsumer)...), keepsOf(firstViewsBucket)...), keepsOf(findingsBucket)...)
view := append(append(append(readOf(tellerAnswers), readOf(gateAsksKV)...), readOf(assignerAsks)...), keepsOf(viewAnswers)...)
// The manifest's emits (ADR 0596 §2; design 47 G A-175: its answers as its views family) and its marks state (A-178).
teller = append(teller, "mesh.mod.mesh-delivery-teller.event.answered", "mesh.mod.mesh-delivery-teller.event.ask-ended", "mesh.mod.mesh-delivery-teller.event.views.answers")
teller = append(teller, keepsOf(marksBucket)...)
// The assigner's asks and the view's answers, each by its read tool and its views family (ADR 0615, DLV-M3; A-175), and the
// teller's own first-views and findings; the buckets themselves are no longer the teller's to read.
teller = append(append(append(teller, viewstest.Grants(assignerModule, assignerModule+".assigner_read_asks", tellerConsumer)...), viewstest.Grants(viewModule, viewModule+"."+viewstate.ViewAnswersReadTool, tellerConsumer)...), keepsOf(firstViewsBucket)...)
teller = append(teller, keepsOf(findingsBucket)...)
view := append(append(readOf(tellerAnswers), readOf(assignerAsks)...), keepsOf(viewAnswers)...)
assigner := keepsOf(assignerAsks)
tools := append(keepsOf(gateAsksKV), readOf(tellerAnswers)...)
return fmt.Sprintf(`
listen: 127.0.0.1:-1
jetstream { store_dir: %q }
@@ -211,14 +121,13 @@ accounts {
jetstream: enabled
users = [
{ user: "ctl", password: "pw" }
%s%s%s%s ]
%s%s%s ]
}
}
`, filepath.Join(dir, "js"),
user("n1.mesh-delivery-teller", teller, nil),
user("n1.mesh-delivery-view", view, nil),
user("n1.mesh-delivery-assigner", assigner, nil),
user("n1.node-tools", tools, nil))
user("n1.mesh-delivery-assigner", assigner, nil))
}
type fullBed struct {
@@ -229,7 +138,7 @@ type fullBed struct {
}
// startFull starts the bus with the principals above and what the controller makes before any module starts: the buckets
// of the gate's asks, the teller's answers and asked, the router's work queue SEAT_OPERATOR_CHANNEL.
// of the assigner's asks, the view's answers, the teller's answers, asked and marks, the router's work queue SEAT_OPERATOR_CHANNEL.
func startFull(t *testing.T) *fullBed {
t.Helper()
dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "t")
@@ -266,7 +175,7 @@ func startFull(t *testing.T) *fullBed {
}
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
t.Cleanup(cancel)
for name, history := range map[string]uint8{gateAsksKV: gateAsksHistory, tellerAnswers: answersHistory, tellerAsked: 1, ends.Bucket: 1, routerAsksBucket: 1, assignerAsks: assignerAsksHistory, viewAnswers: viewAnswersHistory, firstViewsBucket: 1, findingsBucket: 1, widensSaidBucket: 1} {
for name, history := range map[string]uint8{tellerAnswers: answersHistory, tellerAsked: 1, ends.Bucket: 1, routerAsksBucket: 1, assignerAsks: 5, viewAnswers: 5, marksBucket: 1, firstViewsBucket: 1, findingsBucket: 1, widensSaidBucket: 1} {
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: name, History: history}); err != nil {
t.Fatal(err)
}
@@ -284,6 +193,8 @@ func startFull(t *testing.T) *fullBed {
}
// The assigner as the owner of its asks: its read tool and its views family (ADR 0615).
viewstest.Serve(t, ctx, js, ctl, assignerModule, "asks", assignerModule+".assigner_read_asks", tellerConsumer)
// The view as the owner of its answers, as the teller follows them (A-175, A-178).
viewstest.Serve(t, ctx, js, ctl, viewModule, viewstate.ViewAnswersState, viewModule+"."+viewstate.ViewAnswersReadTool, tellerConsumer)
return &fullBed{srv: srv, ctl: ctl, js: js, ctx: ctx}
}
@@ -332,29 +243,29 @@ func (b *fullBed) put(t *testing.T, js jetstream.JetStream, bucket, key string)
return err
}
// TestCredentialsWriteOnlyTheirOwnBuckets: the teller's credential is refused writing the gate's asks, and the gate's
// (the tool runner's) and the view's are refused writing the teller's answers; each writes where it may, so a refusal is the
// permissions' and not a bus that does not take writes (1343/AC4, ADR 0437 §6's check).
// TestCredentialsWriteOnlyTheirOwnBuckets: the view's credential, which reads the teller's answers, is refused writing them; each
// credential writes its own buckets, so a refusal is the permissions' and not a bus that does not take writes (1343/AC4, ADR
// 0437 §6's check). The teller reads neither the assigner's asks nor the view's answers: it follows them.
func TestCredentialsWriteOnlyTheirOwnBuckets(t *testing.T) { // 1343/AC4
b := startFull(t)
teller, view, tools := b.as(t, "n1.mesh-delivery-teller"), b.as(t, "n1.mesh-delivery-view"), b.as(t, "n1.node-tools")
if err := b.put(t, teller, tellerAnswers, "merge.g.0123456789ab"); err != nil {
t.Errorf("the teller cannot write its own answers: %v", err)
teller, view, assigner := b.as(t, "n1.mesh-delivery-teller"), b.as(t, "n1.mesh-delivery-view"), b.as(t, "n1.mesh-delivery-assigner")
for _, c := range []struct {
who jetstream.JetStream
bucket string
key string
}{{teller, tellerAnswers, "hold.a"}, {teller, marksBucket, "answers"}, {assigner, assignerAsks, "hold.a"}, {view, viewAnswers, "hold.a.view"}} {
if err := b.put(t, c.who, c.bucket, c.key); err != nil {
t.Errorf("a credential cannot write its own %s: %v", c.bucket, err)
}
}
if err := b.put(t, tools, gateAsksKV, "merge.g.0123456789ab"); err != nil {
t.Errorf("the gate's account cannot write the gate's asks: %v", err)
}
if err := b.put(t, teller, gateAsksKV, "merge.g.ffffffffffff"); err == nil {
t.Error("the teller's credential wrote the gate's asks")
}
if err := b.put(t, tools, tellerAnswers, "merge.g.ffffffffffff"); err == nil {
t.Error("the gate's account wrote the teller's answers")
}
if err := b.put(t, view, tellerAnswers, "merge.g.ffffffffffff"); err == nil {
t.Error("the view's credential wrote the teller's answers")
}
// What was refused is not there.
for _, c := range []struct{ bucket, key string }{{gateAsksKV, "merge.g.ffffffffffff"}, {tellerAnswers, "merge.g.ffffffffffff"}} {
for _, c := range []struct {
who jetstream.JetStream
bucket string
key string
}{{view, tellerAnswers, "hold.f"}} {
if err := b.put(t, c.who, c.bucket, c.key); err == nil {
t.Errorf("a credential wrote %s, which is not its own", c.bucket)
}
kv, _ := b.js.KeyValue(b.ctx, c.bucket)
if _, err := kv.Get(b.ctx, c.key); err == nil {
t.Errorf("%s holds %s though its write was refused", c.bucket, c.key)
@@ -362,14 +273,14 @@ func TestCredentialsWriteOnlyTheirOwnBuckets(t *testing.T) { // 1343/AC4
}
}
// TestTellerCarriesAnAskAndWritesTheAnswer starts the program on the composed principal against the bus: the gate's account
// writes an ask, the teller publishes it to the seat's work queue under its own name, the router (the test) says the
// warrant on the teller's decided subject, and the answer is under the ask's key in the teller's bucket; withdrawing a
// second ask cancels it (1343/AC1, AC2).
// TestTellerCarriesAnAskAndWritesTheAnswer starts the program on the composed principal against the bus: the assigner writes
// a hold ask, the teller publishes it to the seat's work queue under its own name, the router (the test) says the warrant on
// the teller's decided subject, and the answer is under the ask's key in the teller's bucket and said as the event answered
// and on the views family views.answers (A-175); taking a second ask away cancels it (1343/AC1, AC2, 1906/H1, H2).
func TestTellerCarriesAnAskAndWritesTheAnswer(t *testing.T) { // 1343/AC1, AC2
t.Cleanup(nowIs(time.Now().UTC()))
b := startFull(t)
tools := b.as(t, "n1.node-tools")
assigner := b.as(t, "n1.mesh-delivery-assigner")
broker := filepath.Join(t.TempDir(), "broker")
cred, _ := json.Marshal(process.Account{URL: b.srv.ClientURL(), Node: "n1", Module: "mesh-delivery-teller", User: "n1.mesh-delivery-teller", Password: "pw"})
if err := os.WriteFile(broker, cred, 0o600); err != nil {
@@ -408,8 +319,8 @@ func TestTellerCarriesAnAskAndWritesTheAnswer(t *testing.T) { // 1343/AC1, AC2
t.Fatalf("the router's record read as the teller: %v, %v; want it", rec, err)
}
key, value, ga := gateAsk(t, twoHeads(), twoHeads())
kv, err := tools.KeyValue(b.ctx, gateAsksKV)
key, value, ask := holdAsk(t)
kv, err := assigner.KeyValue(b.ctx, assignerAsks)
if err != nil {
t.Fatal(err)
}
@@ -432,11 +343,11 @@ func TestTellerCarriesAnAskAndWritesTheAnswer(t *testing.T) { // 1343/AC1, AC2
t.Fatalf("no ask on the seat's queue: %v\n%s", err, logs.String())
}
var asked oc.Ask
if err := json.Unmarshal(raw.Data, &asked); err != nil || occ.Digest(asked) != occ.Digest(ga.Ask) {
t.Fatalf("the ask on the queue is %+v (%v), want the gate's", asked, err)
if err := json.Unmarshal(raw.Data, &asked); err != nil || occ.Digest(asked) != occ.Digest(ask) {
t.Fatalf("the ask on the queue is %+v (%v), want the assigner's", asked, err)
}
// The router says the warrant.
w, _ := json.Marshal(warrantFor(ga.Ask, "approve"))
w, _ := json.Marshal(ackFor(ask, "release"))
if err := b.ctl.Publish(seatDecided, w); err != nil {
t.Fatal(err)
}
@@ -453,10 +364,10 @@ func TestTellerCarriesAnAskAndWritesTheAnswer(t *testing.T) { // 1343/AC1, AC2
if e == nil {
t.Fatalf("no answer under %s: %v\n%s", key, err, logs.String())
}
if a, err := meshdeliveryclient.DecodeAnswer(e.Value()); err != nil || a.Option != "approve" {
if a, err := meshdeliveryclient.DecodeAnswer(e.Value()); err != nil || a.Option != "release" {
t.Errorf("answer %+v err %v", a, err)
}
// The answer is said as the event answered, which the gate hears (ADR 0596 §2).
// The answer is said as the event answered (ADR 0596 §2) and on the views family views.answers (A-175).
events, err := b.js.Stream(b.ctx, process.EventsStream)
if err != nil {
t.Fatal(err)
@@ -470,17 +381,28 @@ func TestTellerCarriesAnAskAndWritesTheAnswer(t *testing.T) { // 1343/AC1, AC2
if said == nil {
t.Fatalf("the answer was not said as the event answered: %v\n%s", err, logs.String())
}
if got := string(said.Data); !strings.Contains(got, `"key":"`+key+`"`) || !strings.Contains(got, `"option":"approve"`) {
if got := string(said.Data); !strings.Contains(got, `"key":"`+key+`"`) || !strings.Contains(got, `"option":"release"`) {
t.Errorf("the event answered is %s", got)
}
// A second ask, withdrawn by the gate, is cancelled at the router.
other := []meshdeliveryclient.Head{{Repository: "novox/mesh-delivery", Number: 13, Head: headB}}
key2, value2, ga2 := gateAsk(t, other, other)
if vs, err := events.GetLastMsgForSubject(b.ctx, "mesh.mod.mesh-delivery-teller.event.views.answers"); err != nil || !strings.Contains(string(vs.Data), key) {
t.Errorf("the answer was not said on the views family views.answers: %v\n%s", err, logs.String())
}
// A second ask, taken away by the assigner, is cancelled at the router.
key2, err := asks.HoldAskKey("plex", "src2:"+strings.Repeat("3", 64))
if err != nil {
t.Fatal(err)
}
ask2 := ask
ask2.ID = meshdeliveryclient.AskID(key2)
value2, err := asks.EncodeAsk(key2, asks.Ask{Ask: rawAsk(t, ask2)})
if err != nil {
t.Fatal(err)
}
if _, err := kv.Put(b.ctx, key2, value2); err != nil {
t.Fatal(err)
}
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
if _, err := seat.GetLastMsgForSubject(b.ctx, seatAsk); err == nil && strings.Contains(logs.String(), ga2.Ask.ID) {
if strings.Contains(logs.String(), ask2.ID+" is asked") {
break
}
}
@@ -492,19 +414,19 @@ func TestTellerCarriesAnAskAndWritesTheAnswer(t *testing.T) { // 1343/AC1, AC2
break
}
}
if err != nil || string(raw.Data) != `{"id":"`+ga2.Ask.ID+`"}` {
t.Errorf("the withdrawn ask was not cancelled: %v %v\n%s", raw, err, logs.String())
if err != nil || string(raw.Data) != `{"id":"`+ask2.ID+`"}` {
t.Errorf("the ask taken away was not cancelled: %v %v\n%s", raw, err, logs.String())
}
}
// TestTellerAsksAgainAnAskRefusedForWantOfRoom starts the program against the bus: the router (the test) refuses the gate's
// TestTellerAsksAgainAnAskRefusedForWantOfRoom starts the program against the bus: the router (the test) refuses the assigner's
// ask for want of room, as the live router did on 2026-10-21 (issue 1610), and then says another of the teller's asks
// lapsed; the refused ask is on the seat's work queue again, which the bus's duplicate window would have dropped under its
// first message id (1610/AC1 on the real bus).
func TestTellerAsksAgainAnAskRefusedForWantOfRoom(t *testing.T) { // 1610/AC1
t.Cleanup(nowIs(time.Now().UTC()))
b := startFull(t)
tools := b.as(t, "n1.node-tools")
assigner := b.as(t, "n1.mesh-delivery-assigner")
broker := filepath.Join(t.TempDir(), "broker")
cred, _ := json.Marshal(process.Account{URL: b.srv.ClientURL(), Node: "n1", Module: "mesh-delivery-teller", User: "n1.mesh-delivery-teller", Password: "pw"})
if err := os.WriteFile(broker, cred, 0o600); err != nil {
@@ -514,7 +436,7 @@ func TestTellerAsksAgainAnAskRefusedForWantOfRoom(t *testing.T) { // 1610/AC1
logs := &syncBuffer{}
rn := startRunner(b.ctx, func(c context.Context) error { return run(c, func(k string) string { return env[k] }, logs) })
t.Cleanup(func() { rn.finish(t, 10*time.Second) })
kv, err := tools.KeyValue(b.ctx, gateAsksKV)
kv, err := assigner.KeyValue(b.ctx, assignerAsks)
if err != nil {
t.Fatal(err)
}
@@ -547,31 +469,30 @@ func TestTellerAsksAgainAnAskRefusedForWantOfRoom(t *testing.T) { // 1610/AC1
}
}
key, value, held := gateAsk(t, twoHeads(), twoHeads())
key, value, held := holdAskAt(t, 1, 1)
if _, err := kv.Put(b.ctx, key, value); err != nil {
t.Fatal(err)
}
first := lastAsk(held.Ask.ID, 0)
say(oc.Warrant{Ask: held.Ask.ID, Asker: tellerModule, Outcome: occ.OutcomeRefused, Words: roomRefusal, At: occ.FormatTime(now)})
first := lastAsk(held.ID, 0)
say(oc.Warrant{Ask: held.ID, Asker: tellerModule, Outcome: occ.OutcomeRefused, Words: roomRefusal, At: occ.FormatTime(now)})
for deadline := time.Now().Add(10 * time.Second); !strings.Contains(logs.String(), "for want of room"); time.Sleep(50 * time.Millisecond) {
if time.Now().After(deadline) {
t.Fatalf("the refusal was not heard\n%s", logs.String())
}
}
other := []meshdeliveryclient.Head{{Repository: "novox/mesh-delivery", Number: 13, Head: headB}}
key2, value2, ga2 := gateAsk(t, other, other)
key2, value2, ga2 := holdAskAt(t, 2, 1)
if _, err := kv.Put(b.ctx, key2, value2); err != nil {
t.Fatal(err)
}
second := lastAsk(ga2.Ask.ID, first)
second := lastAsk(ga2.ID, first)
// The other ask lapses at the router: there is room, and the refused ask is asked again.
say(oc.Warrant{Ask: ga2.Ask.ID, Asker: tellerModule, Outcome: occ.OutcomeExpired, At: occ.FormatTime(now), AskDigest: occ.Digest(ga2.Ask)})
lastAsk(held.Ask.ID, second)
say(oc.Warrant{Ask: ga2.ID, Asker: tellerModule, Outcome: occ.OutcomeExpired, At: occ.FormatTime(now), AskDigest: occ.Digest(ga2)})
lastAsk(held.ID, second)
}
// TestTellerFillsAnAskFromTheViewsAnswerOnce starts the program against the bus with the permissions above: the assigner's
// fixture hold ask is written, then the view's answer under `<ask key>.view`; the answer is under the ask's own key, once
// (rewriting the view's answer does not write a second), and a view answer under the merge's key fills nothing
// (rewriting the view's answer does not write a second), and a view answer under a key that is no assigner's (a merge's) fills nothing
// (1297/B10 AC: a view answer fills the ask once; an ask needing proof is not answered from a view key).
func TestTellerFillsAnAskFromTheViewsAnswerOnce(t *testing.T) { // 1297/B10 AC
t.Cleanup(nowIs(time.Now().UTC()))
@@ -629,7 +550,7 @@ func TestTellerFillsAnAskFromTheViewsAnswerOnce(t *testing.T) { // 1297/B10 AC
t.Errorf("the answer was written a second time: %v", err)
}
// An answer under the merge's key fills nothing.
mkey, mvalue, _ := gateAsk(t, twoHeads(), twoHeads())
mkey, mvalue := "merge.g.0123456789ab", value
if _, err := askKV.Put(b.ctx, mkey, mvalue); err != nil {
t.Fatal(err)
}
@@ -642,102 +563,6 @@ func TestTellerFillsAnAskFromTheViewsAnswerOnce(t *testing.T) { // 1297/B10 AC
}
}
// TestTellerRaisesAnAssignerAskAndWritesTheOperatorsAnswer starts the program against the bus: the assigner's hold ask, put in
// its bucket, is on the operator channel's queue as the teller's ask; the router's warrant for "release" is written under the
// ask's key, where the assigner reads it; a second ask the assigner takes away (a hold it lifted) is cancelled at the router
// (1906/H1, H2 on the real bus).
func TestTellerRaisesAnAssignerAskAndWritesTheOperatorsAnswer(t *testing.T) {
t.Cleanup(nowIs(time.Now().UTC()))
b := startFull(t)
assigner := b.as(t, "n1.mesh-delivery-assigner")
broker := filepath.Join(t.TempDir(), "broker")
cred, _ := json.Marshal(process.Account{URL: b.srv.ClientURL(), Node: "n1", Module: "mesh-delivery-teller", User: "n1.mesh-delivery-teller", Password: "pw"})
if err := os.WriteFile(broker, cred, 0o600); err != nil {
t.Fatal(err)
}
env := map[string]string{"MESH_BROKER_FILE": broker}
logs := &syncBuffer{}
rn := startRunner(b.ctx, func(c context.Context) error { return run(c, func(k string) string { return env[k] }, logs) })
t.Cleanup(func() { rn.finish(t, 10*time.Second) })
key, value, ask := holdAsk(t)
askKV, err := assigner.KeyValue(b.ctx, assignerAsks)
if err != nil {
t.Fatal(err)
}
b.listening(t) // the program hears warrants and watches
if _, err := askKV.Put(b.ctx, key, value); err != nil {
t.Fatal(err)
}
seat, err := b.js.Stream(b.ctx, "SEAT_OPERATOR_CHANNEL")
if err != nil {
t.Fatal(err)
}
var raw *jetstream.RawStreamMsg
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
if raw, err = seat.GetLastMsgForSubject(b.ctx, seatAsk); err == nil {
break
}
}
if raw == nil {
t.Fatalf("the assigner's ask is not on the seat's queue: %v\n%s", err, logs.String())
}
var asked oc.Ask
if err := json.Unmarshal(raw.Data, &asked); err != nil || asked.ID != ask.ID || occ.Digest(asked) != occ.Digest(ask) {
t.Fatalf("the ask on the queue is %+v (%v), want the assigner's", asked, err)
}
w, _ := json.Marshal(ackFor(ask, "release"))
if err := b.ctl.Publish(seatDecided, w); err != nil {
t.Fatal(err)
}
answers, err := b.js.KeyValue(b.ctx, tellerAnswers)
if err != nil {
t.Fatal(err)
}
var e jetstream.KeyValueEntry
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
if e, err = answers.Get(b.ctx, answerKey(key)); err == nil {
break
}
}
if e == nil {
t.Fatalf("no answer under %s: %v\n%s", key, err, logs.String())
}
if a, err := meshdeliveryclient.DecodeAnswer(e.Value()); err != nil || a.Option != "release" || answeredAsking(e.Value()) != ask.ID {
t.Fatalf("answer %+v err %v", a, err)
}
key2, err := asks.HoldAskKey("plex", "src2:"+strings.Repeat("3", 64))
if err != nil {
t.Fatal(err)
}
ask2 := ask
ask2.ID = meshdeliveryclient.AskID(key2)
value2, err := asks.EncodeAsk(key2, asks.Ask{Ask: rawAsk(t, ask2)})
if err != nil {
t.Fatal(err)
}
if _, err := askKV.Put(b.ctx, key2, value2); err != nil {
t.Fatal(err)
}
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
if strings.Contains(logs.String(), ask2.ID+" is asked") {
break
}
}
if err := askKV.Delete(b.ctx, key2); err != nil {
t.Fatal(err)
}
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
if raw, err = seat.GetLastMsgForSubject(b.ctx, seatCancel); err == nil {
break
}
}
if err != nil || string(raw.Data) != `{"id":"`+ask2.ID+`"}` {
t.Errorf("the ask taken away was not cancelled: %v %v\n%s", raw, err, logs.String())
}
}
// TestTellerTakesAChangedHoldAnswerFromTheView starts the program against the bus: a hold ask answered "keep" earlier, and the
// view's answer "release" written now, replace the answer of that asking with "release" (1906/N23 on the real bus).
func TestTellerTakesAChangedHoldAnswerFromTheView(t *testing.T) {
modules/mesh-delivery-teller/cmd/mesh-delivery-teller/kv.go
+5 -4
View File
@@ -101,13 +101,14 @@ func (a kvAnswers) Write(key string, value []byte, rev uint64, found bool) (bool
return err == nil, err
}
// kvGate reads the gate's bucket asks, which the teller may watch and read and never write.
type kvGate struct {
// kvReader reads the picture of another module's state the teller follows (the assigner's asks, the view's answers), which it
// may watch and read and never write.
type kvReader struct {
ctx context.Context
kv jetstream.KeyValue
}
func (g kvGate) Get(key string) ([]byte, bool, error) {
func (g kvReader) Get(key string) ([]byte, bool, error) {
e, err := g.kv.Get(g.ctx, key)
if errors.Is(err, jetstream.ErrKeyNotFound) {
return nil, false, nil
@@ -119,7 +120,7 @@ func (g kvGate) Get(key string) ([]byte, bool, error) {
}
// GetAt is Get with the time the value was written.
func (g kvGate) GetAt(key string) ([]byte, time.Time, bool, error) {
func (g kvReader) GetAt(key string) ([]byte, time.Time, bool, error) {
e, err := g.kv.Get(g.ctx, key)
if errors.Is(err, jetstream.ErrKeyNotFound) {
return nil, time.Time{}, false, nil
modules/mesh-delivery-teller/cmd/mesh-delivery-teller/main.go
+31 -42
View File
@@ -1,15 +1,14 @@
// Command mesh-delivery-teller is delivery's teller (novox/hq ADR 0437 §4, §6, §9; issues 1240, 1343). It carries the
// gate's merge asks to the operator and writes the answers back: it watches the gate's asks, asks each of the router as
// itself, and on the router's warrant, once it holds, writes the answer under the ask's key. Where the gate's asks
// bucket does not exist yet it says once that it waits for the gate's asks and does nothing: waiting for an absent bucket
// needs a timer (ADR 0437 §8 lets the parts hold none), so it is started again with the gate. It serves no request
// subject: its one tool, teller_end_ask (the program mesh-delivery-teller-tools, issue 774), asks through its state ends,
// which it watches.
// Command mesh-delivery-teller is delivery's teller (novox/hq ADR 0437 §4, §6, §9; issues 1240, 1343; design 47 G §653,
// A-172). It carries the assigner's asks to the operator and writes the answers back: it follows the assigner's asks (the
// first view of its read tool, then its views family), asks each of the router as itself, and on the router's warrant, once
// it holds, writes the answer under the ask's key, and says its state answers as its views family (A-175). It follows the
// view's answers the same way and fills an ask from the operator's answer in the view. It serves no request subject: its
// one tool beside the readers, teller_end_ask (the program mesh-delivery-teller-tools, issue 774), asks through its state
// ends, which it watches.
package main
import (
"context"
"errors"
"fmt"
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
"io"
@@ -25,6 +24,8 @@ import (
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-view/viewstate"
"mesh-delivery-teller/internal/ends"
)
@@ -49,16 +50,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
return err
}
defer conn.Close()
gate, err := js.KeyValue(ctx, gateAsksBucket)
if errors.Is(err, jetstream.ErrBucketNotFound) {
logger.Print(waiting)
<-ctx.Done()
return nil
}
if err != nil {
// Essential: the gate's asks are what the teller tells (issue 1945: the error names the grant).
return process.EssentialErr("teller", gateAsksBucket, "tell any of the gate's asks", err)
}
// The answers it writes and what it asked: its own work, so each stops the start, naming its grant.
var kvs [3]jetstream.KeyValue
for i, name := range []string{answersBucket, askedBucket, widensSaidBucket} {
@@ -85,7 +76,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
defer calls.Close()
client := &occ.Client{Module: tellerModule, State: state, Publish: publish,
Record: recordVia(calls.Call, calls.Ask), Subscribe: decidedVia(hub.Handle, logger.Printf)}
t := &Teller{Asker: client, Store: state, Answers: kvAnswers{ctx: ctx, kv: kvs[0]}, Gate: kvGate{ctx: ctx, kv: gate},
t := &Teller{Asker: client, Store: state, Answers: kvAnswers{ctx: ctx, kv: kvs[0]},
Say: sayVia(ctx, js, tellerModule), Logf: logger.Printf, Publish: publish,
// The teller's own events (its manifest's emits), on EVENTS under its module's subject.
Emit: func(event string, body []byte, id string) error {
@@ -111,8 +102,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
if err := client.Listen(t.Answer, t.Said); err != nil {
return fmt.Errorf("hearing the router's warrants: %w", err)
}
// The view's answers to the assigner's asks (issue 1297 B10): both buckets are the controller's to create, and a bucket not
// there is said and leaves the teller without that path rather than without the gate's asks (a wait for it needs a timer).
failed := make(chan error, 4)
t.Answered = func(key, askID string, answer []byte) error {
return calls.Emit(answeredEvent, Answered{Key: key, Ask: askID, Answer: answer}, stdio.EmitOptions{MessageID: answeredID(key, askID)})
@@ -129,25 +118,33 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
return err
}
assignerKV := jetstream.KeyValue(asks)
// The view's answers to the assigner's asks (issue 1297 B10, A-175): followed as the assigner's asks are, never its bucket.
answered, err := reader.Follow(viewModule, viewstate.ViewAnswersState, viewModule+"."+viewstate.ViewAnswersReadTool)
if err != nil {
return err
}
viewKV := jetstream.KeyValue(answered)
reader.Start(ctx, js, cred, process.AwaitBucket(ctx, js, conn, firstViewsBucket, "teller_first_views reads nothing and teller_first_view_again is not heard", logger.Printf))
// Every part has registered: the hub pulls the module's durable.
if err := hub.Start(ctx, js, cred); err != nil {
return fmt.Errorf("the consumer %s on %s cannot be read (%v): the teller hears no answer without it, so it stops", cred.EventConsumerName(), process.EventsStream, err)
}
viewKV := process.OptionalBucket(ctx, js, viewAnswersBucket, teller, logger.Printf)
// The teller says its own answers as its views family (A-175, A-178): once its marks state is bound.
process.AwaitBucket(ctx, js, conn, marksBucket, "no answer is said on the views family, so the view waits on its first view", logger.Printf).Ready(func(mk jetstream.KeyValue) {
em := views.Emitter{Owner: tellerModule, Subject: cred.EventSubject, Publish: views.JetStreamPublish(js), Marks: mk, Findings: writer, Logf: logger.Printf}
if err := em.Start(ctx, answersState, kvs[0], failed); err != nil {
logger.Printf("%v: the view asks the first view of the answers at each change until the teller starts again", err)
}
})
// The assigner's asks are raised on the operator channel whether or not the view's answers can be read (issue 1906).
if viewKV != nil {
t.View = kvGate{ctx: ctx, kv: viewKV}
}
t.Assigner = kvGate{ctx: ctx, kv: assignerKV}
t.View = kvReader{ctx: ctx, kv: viewKV}
t.Assigner = kvReader{ctx: ctx, kv: assignerKV}
go func() {
failed <- watchAsks(ctx, assignerKV, func(key string, _ uint64, _ []byte) { t.OnAssignerAsk(key) }, t.OnAssignerAskGone, nil)
failed <- watchAsks(ctx, assignerKV, func(key string, _ uint64, _ []byte) { t.OnAssignerAsk(key) }, t.OnAssignerAskGone, t.OnCurrent)
}()
go func() {
failed <- watchAsks(ctx, viewKV, func(key string, _ uint64, _ []byte) { t.OnViewAnswer(key) }, func(string) {}, nil)
}()
if viewKV != nil {
go func() {
failed <- watchAsks(ctx, viewKV, func(key string, _ uint64, _ []byte) { t.OnViewAnswer(key) }, func(string) {}, nil)
}()
}
// A person ends one ask by hand with teller_end_ask, which writes the state ends (issue 774): watched as the gate's asks
// are, once it is on the bus.
process.AwaitBucket(ctx, js, conn, ends.Bucket, "teller_end_ask is not answered; the asks still end when they are answered, lapse or their key is taken away", logger.Printf).Ready(func(kv jetstream.KeyValue) {
@@ -157,11 +154,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
}
}()
})
go func() {
if err := watchAsks(ctx, gate, t.OnAsk, t.OnAskGone, t.OnCurrent); err != nil {
failed <- fmt.Errorf("watching the gate's asks: %w", err)
}
}()
select {
case <-ctx.Done():
return nil
@@ -173,9 +165,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
}
}
// teller is what an optional bucket of the teller leaves out.
const teller = "the asks and answers it holds are not carried until the teller is started again with it"
// askedBucket is the teller's state bucket asked, `<module>_<name>`: what it asked of the router and which condition it
// raised (manifest state entry asked, novox/hq ADR 0437 §4).
const askedBucket = tellerModule + "_asked"
@@ -190,5 +179,5 @@ var (
// assignerAsksState is the assigner's state of its asks (assignerAsksBucket).
const assignerAsksState = "asks"
// waiting is said once at start.
const waiting = "waiting for the gate's asks, whose bucket does not exist yet; the teller is started again with the gate, and nothing is read, written or published until then"
// marksBucket is the teller's state of what its views family said its answers up to (A-178).
const marksBucket = tellerModule + "_" + marksState
modules/mesh-delivery-teller/cmd/mesh-delivery-teller/names.go
+11 -18
View File
@@ -2,31 +2,24 @@ package main
import "encoding/json"
// The holders and buckets the teller names, taken from its own manifest (module.json: its name, its `reads`) and the
// states it keeps itself; a unit row checks them against it (TestTheNamesAreTheManifests). No contract names a holder
// (novox/hq ADR 0625 §4, ADR 0634 §1).
// The holders and buckets the teller names, taken from its own manifest (module.json: its name, what it invokes and
// consumes) and the states it keeps itself; a unit row checks them against it (TestTheNamesAreTheManifests). No contract
// names a holder (novox/hq ADR 0625 §4, ADR 0634 §1).
const (
tellerModule = "mesh-delivery-teller"
assignerModule = "mesh-delivery-assigner"
gateModule = "mesh-delivery-gate"
viewModule = "mesh-delivery-view"
// gateAsksBucket is the gate's state asks, read by the teller (reads: mesh-delivery-gate.asks).
gateAsksBucket = gateModule + "_asks"
// viewAnswersBucket is the view's state answers, read by the teller (reads: mesh-delivery-view.answers).
viewAnswersBucket = viewModule + "_answers"
// answersBucket is the teller's own state answers, which the gate reads.
answersBucket = tellerModule + "_answers"
// answersState is the teller's own state answers, said as its views family views.answers, and marksState the state of
// what that family said it up to.
answersState = "answers"
marksState = "teller-marks"
// answersBucket is the teller's own state answers.
answersBucket = tellerModule + "_" + answersState
// widensSaidBucket is the teller's own state widens-said, the last commit it said a widening of per module and branch.
widensSaidBucket = tellerModule + "_" + widensSaidState
// assignerAsksBucket is the assigner's state asks (the first view of assigner_read_asks stands in for it).
assignerAsksBucket = assignerModule + "_asks"
// The revisions each keeps per key.
gateAsksHistory = 5
answersHistory = 5
assignerAsksHistory = 5
viewAnswersHistory = 5
// The revisions the teller's answers keep per key.
answersHistory = 5
)
// answerKey is the key, in the teller's answers, of its answer to the ask at askKey: the same string.
modules/mesh-delivery-teller/cmd/mesh-delivery-teller/names_test.go
+26 -11
View File
@@ -5,19 +5,25 @@ import (
"os"
"slices"
"testing"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-view/viewstate"
)
// TestTheNamesAreTheManifests: the holders and buckets the teller names are its manifest's (its module, its `reads`, its
// own state answers), since no contract names a holder (novox/hq ADR 0634 §1).
// TestTheNamesAreTheManifests: the holders, tools and buckets the teller names are its manifest's (its module, what it invokes
// and consumes, its own states answers and teller-marks, no `reads`), since no contract names a holder (novox/hq ADR 0634 §1;
// design 47 G §653, A-175, A-178).
func TestTheNamesAreTheManifests(t *testing.T) {
raw, err := os.ReadFile("../../module.json")
if err != nil {
t.Fatal(err)
}
var m struct {
Module string `json:"module"`
Reads []string `json:"reads"`
State []struct {
Module string `json:"module"`
Reads []string `json:"reads"`
Invokes []string `json:"invokes"`
Consumes []string `json:"consumes"`
Emits []string `json:"emits"`
State []struct {
Name string `json:"name"`
} `json:"state"`
}
@@ -27,16 +33,25 @@ func TestTheNamesAreTheManifests(t *testing.T) {
if m.Module != tellerModule {
t.Errorf("the manifest's module is %q, the teller names itself %q", m.Module, tellerModule)
}
for _, read := range []string{gateModule + ".asks", viewModule + ".answers"} {
if !slices.Contains(m.Reads, read) {
t.Errorf("the teller reads %s, its manifest's reads are %v", read, m.Reads)
}
if len(m.Reads) != 0 {
t.Errorf("the teller reads no other module's state, its manifest's reads are %v", m.Reads)
}
if want := viewModule + "." + viewstate.ViewAnswersReadTool; !slices.Contains(m.Invokes, want) {
t.Errorf("the teller follows the view's answers by %s, its manifest's invokes are %v", want, m.Invokes)
}
if want := viewModule + ".views." + viewstate.ViewAnswersState; !slices.Contains(m.Consumes, want) {
t.Errorf("the teller follows %s, its manifest's consumes are %v", want, m.Consumes)
}
if want := "views." + answersState; !slices.Contains(m.Emits, want) {
t.Errorf("the teller says %s, its manifest's emits are %v", want, m.Emits)
}
var own []string
for _, s := range m.State {
own = append(own, tellerModule+"_"+s.Name)
}
if !slices.Contains(own, answersBucket) {
t.Errorf("the teller writes %s, its manifest's states are %v", answersBucket, own)
for _, want := range []string{answersBucket, marksBucket} {
if !slices.Contains(own, want) {
t.Errorf("the teller keeps %s, its manifest's states are %v", want, own)
}
}
}
modules/mesh-delivery-teller/cmd/mesh-delivery-teller/room_test.go
+73 -93
View File
@@ -6,8 +6,6 @@ import (
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
"strings"
"testing"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient"
)
// roomRefusal is the router's words when an asker already holds occ.MostOpen open asks, as the live router said them to
@@ -16,20 +14,6 @@ import (
// modules/messenger cmd/messenger/asks.go:529-531).
const roomRefusal = "its asker already has 3 open asks"
// headsN is one pull request's head, distinct per n, so each n is an ask of its own key.
func headsN(n int) []meshdeliveryclient.Head {
return []meshdeliveryclient.Head{{Repository: "novox/mesh-delivery", Number: 10 + n, Head: strings.Repeat(string(rune('a'+n)), 40)}}
}
// askOf puts the gate's key for n and hands it to the teller, as the watch would.
func (b *bed) askOf(t *testing.T, n int, rev uint64) (string, gateAsked) {
t.Helper()
key, value, ga := gateAsk(t, headsN(n), headsN(n))
b.gate[key] = value
b.t.OnAsk(key, rev, value)
return key, ga
}
// refuse is the router's refusal of an ask: it keeps no record of a refused ask (asks.go refuseAsk only tells its asker),
// so the id may be asked again.
func (b *bed) refuse(t *testing.T, id, words string) {
@@ -40,9 +24,9 @@ func (b *bed) refuse(t *testing.T, id, words string) {
}
// expire is the router's word that an ask of the teller's lapsed with nobody's answer: one of its asks ended.
func (b *bed) expire(t *testing.T, ga gateAsked) {
func (b *bed) expire(t *testing.T, ask oc.Ask) {
t.Helper()
b.deliver(t, oc.Warrant{Ask: ga.Ask.ID, Asker: tellerModule, Outcome: occ.OutcomeExpired, At: occ.FormatTime(now), AskDigest: occ.Digest(ga.Ask)})
b.deliver(t, oc.Warrant{Ask: ask.ID, Asker: tellerModule, Outcome: occ.OutcomeExpired, At: occ.FormatTime(now), AskDigest: occ.Digest(ask)})
}
// askedIDs is the router ids of every ask the bus carried, in order.
@@ -58,87 +42,84 @@ func (b *bed) askedIDs() []string {
return out
}
// The live fault of issue 1610: three asks open, a fourth refused for want of room, and the operator then answers one of
// the three. The fourth is asked again, under its own router id and a new bus message id, and nothing is said as a fault.
// The live fault of issue 1610: asks open, one refused by the router for want of room, and the operator then answers one of the
// open. The refused is asked again, under its own router id and a new bus message id, and nothing is said as a fault.
func TestAnAskRefusedForWantOfRoomIsAskedAgainWhenAnotherEnds(t *testing.T) { // 1610/AC1
b := newBed(t)
_, first := b.askOf(t, 1, 1)
b.askOf(t, 2, 2)
b.askOf(t, 3, 3)
_, fourth := b.askOf(t, 4, 4)
b.refuse(t, fourth.Ask.ID, roomRefusal)
_, first := b.holdOf(t, 1, 1)
key2, second := b.holdOf(t, 2, 1)
b.refuse(t, second.ID, roomRefusal)
if len(b.conds) != 0 {
t.Errorf("a refusal for want of room is said as a fault before it was asked again: %+v", b.conds)
}
if got := b.askedIDs(); len(got) != 4 {
if got := b.askedIDs(); len(got) != 2 {
t.Fatalf("asked %v before room came", got)
}
b.deliver(t, warrantFor(first.Ask, "approve"))
b.deliver(t, warrantFor(first, "keep"))
got := b.askedIDs()
if len(got) != 5 || got[4] != fourth.Ask.ID {
t.Fatalf("asks = %v, want the refused %s asked again once the first was answered", got, fourth.Ask.ID)
if len(got) != 3 || got[2] != second.ID {
t.Fatalf("asks = %v, want the refused %s asked again once the first was answered", got, second.ID)
}
pubs := b.askedOf(oc.AcceptSubjectAsk(tellerModule))
if pubs[4].id == pubs[3].id {
t.Errorf("asked again under the bus message id %s of its first asking: the bus drops it as a duplicate", pubs[4].id)
if pubs[2].id == pubs[1].id {
t.Errorf("asked again under the bus message id %s of its first asking: the bus drops it as a duplicate", pubs[2].id)
}
if len(b.busDropped) != 0 || len(b.dropped) != 0 || len(b.conds) != 0 {
t.Errorf("bus dropped %v, router dropped %v, conditions %+v", b.busDropped, b.dropped, b.conds)
}
// Taken this time: an answer to it writes the answer as for any ask.
key4, _ := meshdeliveryclient.MergeAskKey("g", headsN(4))
b.deliver(t, warrantFor(fourth.Ask, "approve"))
if _, ok := b.answers.written[answerKey(key4)]; !ok {
b.deliver(t, warrantFor(second, "keep"))
if _, ok := b.answers.written[answerKey(key2)]; !ok {
t.Errorf("the answer to the ask asked again was not written; logs:\n%s", b.logs.String())
}
// And it is asked no more: its answer ends its wait.
_, second := b.askOf(t, 5, 5)
b.expire(t, second)
if got := b.askedIDs(); len(got) != 6 || got[5] == fourth.Ask.ID {
t.Errorf("asks after the answer = %v, want only the fifth asked", got)
_, third := b.holdOf(t, 3, 1)
b.expire(t, third)
if got := b.askedIDs(); len(got) != 4 || got[3] == second.ID {
t.Errorf("asks after the answer = %v, want only the third asked", got)
}
}
func TestAnAskRefusedForWantOfRoomIsNotAskedAgainOnceTheGateMovedOn(t *testing.T) { // 1610/AC2
func TestAnAskRefusedForWantOfRoomIsNotAskedAgainOnceTheAssignerMovedOn(t *testing.T) { // 1610/AC2
b := newBed(t)
_, first := b.askOf(t, 1, 1)
key, gone := b.askOf(t, 2, 2)
b.refuse(t, gone.Ask.ID, roomRefusal)
// Withdrawn by the gate (a head moved on) before room came.
delete(b.gate, key)
b.t.OnAskGone(key)
key3, g3 := b.askOf(t, 3, 3)
b.refuse(t, g3.Ask.ID, roomRefusal)
// Rewritten by the gate under its next asking before room came: the later asking is asked under its own id, and the
_, first := b.holdOf(t, 1, 1)
key, gone := b.holdOf(t, 2, 1)
b.refuse(t, gone.ID, roomRefusal)
// Taken away by the assigner (the hold lifted) before room came.
delete(b.assigner, key)
b.t.OnAssignerAskGone(key)
key3, g3 := b.holdOf(t, 3, 1)
b.refuse(t, g3.ID, roomRefusal)
// Rewritten by the assigner under its next asking before room came: the later asking is asked under its own id, and the
// refused earlier asking is not asked again.
_, v3b, g3b := gateAskN(t, headsN(3), headsN(3), 2)
b.gate[key3] = v3b
b.t.OnAsk(key3, 4, v3b)
_, v3b, g3b := holdAskAt(t, 3, 2)
b.assigner[key3] = v3b
b.t.OnAssignerAsk(key3)
before := len(b.askedIDs())
if ids := b.askedIDs(); ids[before-1] != g3b.Ask.ID {
t.Fatalf("asks = %v, want the next asking %s asked", ids, g3b.Ask.ID)
if ids := b.askedIDs(); ids[before-1] != g3b.ID {
t.Fatalf("asks = %v, want the next asking %s asked", ids, g3b.ID)
}
b.deliver(t, warrantFor(first.Ask, "approve"))
b.deliver(t, warrantFor(first, "keep"))
b.expire(t, g3b)
if after := b.askedIDs()[before:]; len(after) != 0 {
t.Errorf("asked again %v, which the gate no longer asks", after)
t.Errorf("asked again %v, which the assigner no longer asks", after)
}
}
func TestAnAskRefusedForWantOfRoomTooOftenIsSaidAsACondition(t *testing.T) { // 1610/AC3
b := newBed(t)
_, held := b.askOf(t, 1, 1)
_, held := b.holdOf(t, 1, 1)
for i := 0; i < mostRoomRefusals; i++ {
if len(b.conds) != 0 {
t.Fatalf("a condition after %d refusals: %+v", i, b.conds)
}
b.refuse(t, held.Ask.ID, roomRefusal)
_, other := b.askOf(t, 2+i, uint64(2+i))
b.refuse(t, held.ID, roomRefusal)
_, other := b.holdOf(t, 2+i, 1)
b.expire(t, other)
}
var again int
for _, id := range b.askedIDs() {
if id == held.Ask.ID {
if id == held.ID {
again++
}
}
@@ -149,7 +130,7 @@ func TestAnAskRefusedForWantOfRoomTooOftenIsSaidAsACondition(t *testing.T) { //
if len(b.conds) != 1 || b.conds[0].State != occ.StateRaised || !strings.Contains(b.conds[0].Summary, "open asks") {
t.Fatalf("conditions = %+v", b.conds)
}
_, last := b.askOf(t, 20, 20)
_, last := b.holdOf(t, 8, 1)
b.expire(t, last)
if len(b.askedIDs()) != asked+1 {
t.Errorf("asked again after the bound: %v", b.askedIDs())
@@ -158,47 +139,47 @@ func TestAnAskRefusedForWantOfRoomTooOftenIsSaidAsACondition(t *testing.T) { //
func TestAnAskWaitingForRoomIsAskedAgainAtStart(t *testing.T) { // 1610/AC4
b := newBed(t)
key, held := b.askOf(t, 1, 1)
b.refuse(t, held.Ask.ID, roomRefusal)
key, held := b.holdOf(t, 1, 1)
b.refuse(t, held.ID, roomRefusal)
// The teller starts again: a new teller on the same buckets, and the watch hands over the key it holds.
b2 := newBed(t)
b2.store, b2.answers, b2.gate = b.store, b.answers, b.gate
b2.store, b2.answers, b2.assigner = b.store, b.answers, b.assigner
b2.client.State = b.store
b2.t.Store, b2.t.Answers, b2.t.Gate = b.store, b.answers, b.gate
b2.t.OnAsk(key, 1, b.gate[key])
b2.t.Store, b2.t.Answers, b2.t.Assigner = b.store, b.answers, b.assigner
b2.t.OnAssignerAsk(key)
if len(b2.askedIDs()) != 0 {
t.Fatalf("asked before the watch handed over its keys: %v", b2.askedIDs())
}
b2.t.OnCurrent()
if got := b2.askedIDs(); len(got) != 1 || got[0] != held.Ask.ID {
t.Errorf("asks at start = %v, want %s asked again", got, held.Ask.ID)
if got := b2.askedIDs(); len(got) != 1 || got[0] != held.ID {
t.Errorf("asks at start = %v, want %s asked again", got, held.ID)
}
}
func TestAnAskRefusedForItsWordsIsSaidAtOnceAndNotAskedAgain(t *testing.T) { // 1610/AC5
b := newBed(t)
_, first := b.askOf(t, 1, 1)
_, bad := b.askOf(t, 2, 2)
b.refuse(t, bad.Ask.ID, "its words carry an address, which may not leave the mesh")
_, first := b.holdOf(t, 1, 1)
_, bad := b.holdOf(t, 2, 1)
b.refuse(t, bad.ID, "its words carry an address, which may not leave the mesh")
if len(b.conds) != 1 {
t.Fatalf("conditions = %+v", b.conds)
}
b.deliver(t, warrantFor(first.Ask, "approve"))
b.deliver(t, warrantFor(first, "keep"))
if got := b.askedIDs(); len(got) != 2 {
t.Errorf("asks = %v: an ask refused for its words was asked again", got)
}
}
// restart is a new teller on the same buckets and the same bus (its message ids still in the bus's duplicate window), and
// the router's records as they stand; the watch hands over the gate's keys, then says it has handed over all.
// the router's records as they stand; the watch hands over the assigner's keys, then says it has handed over all.
func (b *bed) restart(t *testing.T) *bed {
t.Helper()
n := newBed(t)
n.store, n.answers, n.gate, n.records, n.msgIDs, n.seenIDs = b.store, b.answers, b.gate, b.records, b.msgIDs, b.seenIDs
n.store, n.answers, n.assigner, n.records, n.msgIDs, n.seenIDs = b.store, b.answers, b.assigner, b.records, b.msgIDs, b.seenIDs
n.client.State = b.store
n.t.Store, n.t.Answers, n.t.Gate = b.store, b.answers, b.gate
for key, value := range b.gate {
n.t.OnAsk(key, 1, value)
n.t.Store, n.t.Answers, n.t.Assigner = b.store, b.answers, b.assigner
for key := range b.assigner {
n.t.OnAssignerAsk(key)
}
n.t.OnCurrent()
return n
@@ -209,22 +190,21 @@ func (b *bed) restart(t *testing.T) *bed {
// so no record there means it was not taken.
func TestAnAskAskedAgainWhoseRefusalWasMissedIsAskedAgainAtStart(t *testing.T) { // 1610/AC4
b := newBed(t)
b.askOf(t, 1, 1)
_, held := b.askOf(t, 2, 2)
b.refuse(t, held.Ask.ID, roomRefusal)
_, other := b.askOf(t, 3, 3)
b.expire(t, other)
b.records[other.Ask.ID] = `{"state":"expired"}`
if got := b.askedIDs(); got[len(got)-1] != held.Ask.ID {
t.Fatalf("asks = %v, want %s asked again", got, held.Ask.ID)
_, held := b.holdOf(t, 2, 1)
b.refuse(t, held.ID, roomRefusal)
_, other := b.holdOf(t, 3, 1)
b.deliver(t, warrantFor(other, "keep"))
b.records[other.ID] = `{"state":"chosen"}`
if got := b.askedIDs(); got[len(got)-1] != held.ID {
t.Fatalf("asks = %v, want %s asked again", got, held.ID)
}
// Refused again, and the teller does not hear it: the router forgets it.
delete(b.seenIDs, held.Ask.ID)
delete(b.records, held.Ask.ID)
delete(b.seenIDs, held.ID)
delete(b.records, held.ID)
n := b.restart(t)
got := n.askedIDs()
if len(got) != 1 || got[0] != held.Ask.ID {
t.Fatalf("asks after the start = %v, want %s asked again; logs:\n%s", got, held.Ask.ID, n.logs.String())
if len(got) != 1 || got[0] != held.ID {
t.Fatalf("asks after the start = %v, want %s asked again; logs:\n%s", got, held.ID, n.logs.String())
}
if len(n.busDropped) != 0 {
t.Errorf("the bus dropped %v as a duplicate", n.busDropped)
@@ -234,13 +214,13 @@ func TestAnAskAskedAgainWhoseRefusalWasMissedIsAskedAgainAtStart(t *testing.T) {
// The same for a first asking: kept open by the teller, its refusal never heard, and no record at the router.
func TestAFirstAskingWhoseRefusalWasMissedIsAskedAgainAtStart(t *testing.T) { // 1610/AC4
b := newBed(t)
_, held := b.askOf(t, 1, 1)
delete(b.seenIDs, held.Ask.ID) // refused for room, unheard: the router keeps no record of it
delete(b.records, held.Ask.ID)
_, held := b.holdOf(t, 1, 1)
delete(b.seenIDs, held.ID) // refused for room, unheard: the router keeps no record of it
delete(b.records, held.ID)
n := b.restart(t)
got := n.askedIDs()
if len(got) != 1 || got[0] != held.Ask.ID {
t.Fatalf("asks after the start = %v, want %s asked again; logs:\n%s", got, held.Ask.ID, n.logs.String())
if len(got) != 1 || got[0] != held.ID {
t.Fatalf("asks after the start = %v, want %s asked again; logs:\n%s", got, held.ID, n.logs.String())
}
if len(n.busDropped) != 0 {
t.Errorf("the bus dropped %v as a duplicate", n.busDropped)
modules/mesh-delivery-teller/cmd/mesh-delivery-teller/teller.go
+45 -265
View File
@@ -5,7 +5,6 @@ import (
"errors"
"fmt"
"slices"
"sort"
"strings"
"sync"
"time"
@@ -17,45 +16,14 @@ import (
stdio "git.novox.be/novox/mesh-sdk/go"
)
// The teller carries the gate's merge asks to the operator and writes the operator's answer back (novox/hq ADR 0437
// §6, §9). It reads the gate's asks, publishes each to the router unchanged as the asker mesh-delivery-teller
// (mesh-sdk go/asker), withdraws it when the gate takes its key away, and on the warrant the router says to it
// holds the warrant to the ask (ADR 0259 §6) and writes the answer under the ask's own key by compare-and-set. A
// warrant that does not hold writes no answer and is said as a condition. It holds no timer and serves no request:
// every reaction is a revision of the gate's key or a warrant.
// The teller carries the assigner's asks to the operator and writes the operator's answer back (novox/hq ADR 0437
// §6, §9; design 47 G §653, A-172: the merge ask went with the Approve per merge). It publishes each of the assigner's
// asks to the router unchanged as the asker mesh-delivery-teller (mesh-sdk go/asker), withdraws it when the assigner takes
// its key away, and on the warrant the router says to it holds the warrant to the ask (ADR 0259 §6) and writes the
// answer under the ask's own key by compare-and-set. A warrant that does not hold writes no answer and is said as a
// condition. It holds no timer and serves no request: every reaction is a revision of a key or a warrant.
// gateAsked is the gate's ask with the router's ask decoded from its JSON (the seat's ask value carries operator-channel's ask as
// JSON, novox/hq ADR 0627 §4).
type gateAsked struct {
Group string
Heads []meshdeliveryclient.Head
Asking int
Ask oc.Ask
}
// decodeGateAsk reads the gate's ask value and the router's ask in it, as meshdeliveryclient.DecodeAsk did before the ask was JSON.
func decodeGateAsk(value []byte) (gateAsked, error) {
ga, err := meshdeliveryclient.DecodeAsk(value)
if err != nil {
return gateAsked{}, err
}
heads := meshdeliveryclient.HeadsOf(ga.Heads)
key, err := meshdeliveryclient.MergeAskKey(ga.Group, heads)
if err != nil {
return gateAsked{}, err
}
raw, err := json.Marshal(ga.Ask)
if err != nil {
return gateAsked{}, err
}
a, err := occ.Carried(raw, meshdeliveryclient.AskingID(key, int(ga.Asking)))
if err != nil {
return gateAsked{}, err
}
return gateAsked{Group: ga.Group, Heads: heads, Asking: int(ga.Asking), Ask: a}, nil
}
// store is where the teller keeps what it asked: the asker's own records (`ask.<id>`) and, beside them, which gate key
// store is where the teller keeps what it asked: the asker's own records (`ask.<id>`) and, beside them, which ask key
// each ask was made for and at which revision of it (`key.<id>`), and which conditions it has raised (`cond.<id>`).
type store interface {
occ.Store
@@ -79,8 +47,9 @@ type answers interface {
Write(key string, value []byte, rev uint64, found bool) (written bool, err error)
}
// gateAsks reads the gate's bucket asks.
type gateAsks interface {
// asksReader reads the picture of a state another module keeps (the assigner's asks, the view's answers), as the
// reader of mesh-sdk's views gives it.
type asksReader interface {
Get(key string) (value []byte, found bool, err error)
}
@@ -89,11 +58,10 @@ type Teller struct {
Asker routerAsks
Store store
Answers answers
Gate gateAsks
// Assigner is the assigner's bucket asks and View the view's bucket answers (issue 1297 B10); with either nil the teller
// answers nothing from the view.
Assigner gateAsks
View gateAsks
// Assigner is the picture of the assigner's asks and View the picture of the view's answers (issue 1297 B10); with
// either nil the teller answers nothing from the view.
Assigner asksReader
View asksReader
// Say publishes a condition to the operator (mesh.seat.operator-channel.accept.condition.<module>).
Say func(c oc.Condition) error
Logf func(format string, args ...any)
@@ -118,8 +86,8 @@ func (t *Teller) logf(format string, args ...any) {
}
}
// seen is what the teller keeps beside an ask it made. Under `key.<router id>` it names the gate key the asking was made for;
// under `gate.<gate key>` it names the router id of the latest asking of that key and the revision of the key it was made
// seen is what the teller keeps beside an ask it made. Under `key.<router id>` it names the ask key (the assigner's) the asking was made for;
// under `gate.<ask key>` it names the router id of the latest asking of that key and the revision of the key it was made
// at, so a replay of the revision asks nothing again and an overwrite with a new id is known for a new asking.
type seen struct {
Key string `json:"key,omitempty"`
@@ -135,7 +103,7 @@ func recordKey(id string) string { return "key." + id }
func keptKey(id string) string { return "ask." + id }
func condKey(id string) string { return "cond." + id }
// condID is what a condition of an ask is kept and named by: the gate key it was asked for, so the asking that follows a bad
// condID is what a condition of an ask is kept and named by: the ask key it was asked for, so the asking that follows a bad
// answer clears the condition of the one before it. An id the teller holds no key for is its own.
func (t *Teller) condID(id string) string {
if s, e, _ := t.seenOf(id); e != nil && s.Key != "" {
@@ -156,7 +124,7 @@ func (t *Teller) seenAt(rk string) (seen, *stdio.StateEntry, error) {
return s, e, nil
}
// seenOf is the record of a router id: the gate key it was asked for.
// seenOf is the record of a router id: the ask key it was asked for.
func (t *Teller) seenOf(id string) (seen, *stdio.StateEntry, error) { return t.seenAt(recordKey(id)) }
func (t *Teller) keptOf(id string) (*occ.Kept, uint64, error) {
@@ -174,7 +142,7 @@ func (t *Teller) keptOf(id string) (*occ.Kept, uint64, error) {
// open says whether an ask is made and neither answered nor ended.
func open(k *occ.Kept) bool { return k != nil && !k.Acted && k.Ended == "" }
// note keeps that the asking with this router id is the latest of the gate key, made at revision rev of it.
// note keeps that the asking with this router id is the latest of the ask key, made at revision rev of it.
func (t *Teller) note(id, key string, rev uint64, prior *stdio.StateEntry) error {
if _, err := t.Store.Create(recordKey(id), seen{Key: key, Rev: rev}); err != nil && !errors.Is(err, stdio.ErrChanged) {
return err
@@ -188,82 +156,6 @@ func (t *Teller) note(id, key string, rev uint64, prior *stdio.StateEntry) error
return err
}
// OnAsk handles the gate's asks key put at revision rev. Each asking has its own router id, which is the value's ask.id and
// never one recomputed from the key (mesh-sdk go/delivery Ask.Asking, AskingID): the router keeps an ended ask and drops a
// second ask under its id. So the ask is published once per id; a key overwritten at a higher revision under a new id is a
// new asking, and the earlier asking's router ask is cancelled first. A key whose value is not the ask its name says, one
// already answered, and a revision the teller has handled are left. An ask still open and unchanged is only noted (and its
// warrant read from the router, in case it came while the teller was away). A value that changed its words under the id
// it was asked under is cancelled and not asked again: the router would drop it, so the gate must give it a new asking.
func (t *Teller) OnAsk(key string, rev uint64, value []byte) {
t.mu.Lock()
defer t.mu.Unlock()
ga, err := decodeGateAsk(value)
if err != nil {
t.logf("the gate's key %s at revision %d is not an ask, and is left: %v", key, rev, err)
return
}
if want, err := meshdeliveryclient.MergeAskKey(ga.Group, ga.Heads); err != nil || want != key {
t.logf("the gate's key %s at revision %d holds the ask of %q, not its own, and is left (%v)", key, rev, want, err)
return
}
id := ga.Ask.ID
if raw, _, has, err := t.Answers.Get(key); err != nil {
t.logf("%s: the answers could not be read, so the ask is left: %v", key, err)
return
} else if has && answeredAsking(raw) == id {
return // this asking is answered; an earlier asking's answer does not stop a later asking
}
prior, grec, err := t.seenAt(gateRecordKey(key))
if err != nil {
t.logf("%s: %v", key, err)
return
}
if grec != nil && rev <= prior.Rev {
if k, _, err := t.keptOf(prior.ID); err == nil && open(k) {
t.catchUp(prior.ID)
}
return
}
if grec != nil && prior.ID != id {
// A new asking of the group at these heads: the earlier one is taken back.
t.cancelOpen(key, prior.ID)
}
k, _, err := t.keptOf(id)
if err != nil {
t.logf("%s: %v", key, err)
return
}
if k != nil {
switch {
case open(k) && occ.Digest(k.Ask) == occ.Digest(ga.Ask):
_ = t.note(id, key, rev, grec)
t.catchUp(id)
case open(k):
t.cancelOpen(key, id)
_ = t.note(id, key, rev, grec)
t.logf("%s: the ask %s changed its words at revision %d under the same id: cancelled, and not asked again, for the router drops a second ask under an id (the gate gives a new asking a new id)", key, id, rev)
default:
_ = t.note(id, key, rev, grec)
t.logf("%s: the ask %s ended, and the gate wrote it again under the same id at revision %d: not asked again, the router would drop it", key, id, rev)
}
return
}
if err := t.note(id, key, rev, grec); err != nil {
t.logf("%s: the record of the ask %s could not be kept, so it is left: %v", key, id, err)
return
}
if err := t.Asker.Ask(ga.Ask); err != nil {
t.logf("%s at revision %d: the ask %s was not asked: %v", key, rev, id, err)
if kept, _, _ := t.keptOf(id); open(kept) {
// Kept but not published: forgotten, so the next revision of the key asks it.
_ = t.Store.Delete(keptKey(id))
}
return
}
t.logf("%s at revision %d: the ask %s is asked of the operator (asking %d)", key, rev, id, ga.Asking)
}
// cancelOpen takes an open ask back at the router and marks it ended here, since the router's word that it ended may come
// later. An ask that is not open is left.
func (t *Teller) cancelOpen(key, id string) {
@@ -286,25 +178,6 @@ func (t *Teller) cancelOpen(key, id string) {
t.logf("%s: the ask %s is cancelled", key, id)
}
// OnAskGone handles the gate's asks key taken away (a withdrawn ask): the latest asking's router ask is cancelled, and an
// answer to it merges nothing.
func (t *Teller) OnAskGone(key string) {
t.mu.Lock()
defer t.mu.Unlock()
if _, _, _, err := meshdeliveryclient.ParseAskKey(key); err != nil {
t.logf("the gate's key %s was taken away and is no ask key, and is left: %v", key, err)
return
}
prior, grec, err := t.seenAt(gateRecordKey(key))
if err != nil || grec == nil {
if err != nil {
t.logf("%s: %v", key, err)
}
return
}
t.cancelOpen(key, prior.ID)
}
// catchUp reads the router's record of an ask the teller made and hands its warrant on, as the router's event would:
// a warrant that came while the teller was away.
func (t *Teller) catchUp(id string) {
@@ -327,34 +200,6 @@ func (t *Teller) catchUp(id string) {
t.said(h, *w, err)
}
// mergeAct is what the approve option of the gate's ask binds: the ask's verb (meshdeliveryclient.AskVerb: mesh-delivery.merge; the settle ask
// went with the verb settle, design 47 G §431), this ask, the answer, and the heads it covers.
// It is spelled as the gate composes it (mesh-catalog modules/mesh-delivery-gate approval.go mergeAct, from
// modules/mesh-delivery approval.go:63-65) and as mesh-controller verifies it (cmd/mesh-controller/moved.go:98-106).
func mergeAct(verb, ask, option string, heads map[string]string) occ.Act {
var out []string
for id, sha := range heads {
out = append(out, id+"="+sha)
}
sort.Strings(out)
return occ.Act{"verb": verb, "ask": ask, "option": option, "heads": strings.Join(out, ",")}
}
// headsOf is the heads an ask covers as an act names them: a delivery's id, `owner/repository@<twelve characters of its
// commit>` with the repository in lower case, as the gate's IDOf spells it (mesh-catalog modules/mesh-delivery-gate model.go),
// and its whole commit.
func headsOf(heads []meshdeliveryclient.Head) map[string]string {
out := map[string]string{}
for _, h := range heads {
short := h.Head
if len(short) > 12 {
short = short[:12]
}
out[strings.ToLower(h.Repository)+"@"+short] = h.Head
}
return out
}
// answeredAsking is the router id of the asking an answer is for: its warrant's id before the "@". An answer that cannot be
// read answers no asking.
func answeredAsking(raw []byte) string {
@@ -376,47 +221,20 @@ func warrantID(w oc.Warrant) string {
}
// Answer is the act of a warrant that holds the ask (occ.Client.Take has checked its asker, the ask's digest, the
// option and its level, and the time): the option's bound act must be the act of this ask and the heads it covers,
// and then the answer is written under the ask's key, once. An error is a warrant that does not hold here.
// option and its level, and the time): the assigner's ask must still stand (answerAssigner), and then the answer is
// written under the ask's key, once. An error is a warrant that does not hold here.
func (t *Teller) Answer(o oc.Option, w oc.Warrant) error {
s, srec, err := t.seenOf(w.Ask)
if err != nil {
return err
}
if srec == nil {
return fmt.Errorf("the teller holds no gate key for the ask %s", w.Ask)
return fmt.Errorf("the teller holds no key for the ask %s", w.Ask)
}
if isAssignerKey(s.Key) {
return t.answerAssigner(s.Key, o, w)
if !isAssignerKey(s.Key) {
return fmt.Errorf("the ask %s was asked for %s, which is no ask of the assigner: the teller carries no merge ask", w.Ask, s.Key)
}
raw, found, err := t.Gate.Get(s.Key)
if err != nil {
return fmt.Errorf("the gate's ask %s could not be read: %w", s.Key, err)
}
if !found {
t.logf("%s: the ask %s was answered and the gate has taken it away: an answer to a withdrawn ask merges nothing", s.Key, w.Ask)
return nil
}
ga, err := decodeGateAsk(raw)
if err != nil {
return fmt.Errorf("the gate's ask %s is not an ask: %w", s.Key, err)
}
if ga.Ask.ID != w.Ask {
t.logf("%s: the ask %s was answered, and the gate has asked again under %s: an answer to an earlier asking merges nothing", s.Key, w.Ask, ga.Ask.ID)
return nil
}
if occ.Digest(ga.Ask) != w.AskDigest {
return fmt.Errorf("the gate's ask %s is no longer the ask the operator answered (its digest is %s, the warrant's %s)",
s.Key, occ.Digest(ga.Ask), w.AskDigest)
}
if err := occ.Performs(o, mergeAct(meshdeliveryclient.AskVerb(ga.Group), w.Ask, o.ID, headsOf(ga.Heads))); err != nil {
return err
}
body, err := meshdeliveryclient.EncodeAnswer(meshdeliveryclient.Answer{Option: o.ID, Warrant: warrantID(w), Words: occ.Says(w)})
if err != nil {
return err
}
return t.writeAnswer(s.Key, w.Ask, body, occ.Says(w))
return t.answerAssigner(s.Key, o, w)
}
// writeAnswer writes the answer to the asking askID under the ask's key by compare-and-set: created when the key has no answer;
@@ -500,10 +318,10 @@ func (t *Teller) saidOnce(h occ.Handled, w oc.Warrant, err error) {
t.clear(w.Ask)
case h == occ.Refused || (h == occ.Acted && err != nil):
t.logf("the warrant of %s does not hold, and no answer is written: %v", w.Ask, err)
t.raise(w.Ask, t.kindOf(w.Ask).answerHeadline, "its answer did not hold here: "+oneLine(err))
t.raise(w.Ask, assignerWords.answerHeadline, "its answer did not hold here: "+oneLine(err))
case h == occ.Recorded && w.Outcome == occ.OutcomeRefused:
t.logf("the router refused the ask %s: %s", w.Ask, w.Words)
t.raise(w.Ask, t.kindOf(w.Ask).refusedHeadline, "the router refused it: "+w.Words)
t.raise(w.Ask, assignerWords.refusedHeadline, "the router refused it: "+w.Words)
case err != nil:
t.logf("the warrant of %s: %v", w.Ask, err)
}
@@ -523,30 +341,19 @@ func oneLine(err error) string {
// conditionKey is the key of the condition about an ask.
func conditionKey(id string) string { return tellerModule + ".merge-ask." + id }
// askWords are the words the teller's conditions use for one kind of ask: the gate's merge ask, or one of the assigner's.
// askWords are the words the teller's conditions use for an ask: one of the assigner's.
type askWords struct {
scope, noun, needs, nothing, resolved string
answerHeadline, refusedHeadline, unaskedHeadline string
}
var (
mergeWords = askWords{scope: "merge", noun: "The merge ask ", needs: "to ask for the Approve again", nothing: "Nothing was merged.",
resolved: "A good answer to the merge ask was written.", answerHeadline: "A merge Approve did not hold",
refusedHeadline: "A merge ask was refused", unaskedHeadline: "A merge ask could not be asked"}
assignerWords = askWords{scope: "delivery", noun: "The delivery ask ", needs: "to answer it in the delivery view",
nothing: "Nothing was changed on a machine.", resolved: "A good answer to the delivery ask was written.",
answerHeadline: "An answer to a delivery ask did not hold", refusedHeadline: "A delivery ask was refused",
unaskedHeadline: "A delivery ask could not be asked"}
)
// kindOf is the words for the ask with this router id, by the key it was asked for.
func (t *Teller) kindOf(askID string) askWords {
if isAssignerKey(t.condID(askID)) {
return assignerWords
}
return mergeWords
}
func (t *Teller) raise(askID, headline, why string) {
if t.Say == nil {
return
@@ -558,7 +365,7 @@ func (t *Teller) raise(askID, headline, why string) {
}
t.logf("the condition of %s could not be kept: %v", id, err)
}
k := t.kindOf(askID)
k := assignerWords
needs := k.needs
c := oc.Condition{Key: conditionKey(id), Severity: occ.SeverityWarning, Subject: oc.ConditionSubject{Scope: k.scope, ID: id}, State: occ.StateRaised,
Summary: k.noun + id + ": " + why, Headline: headline, Needs: needs,
@@ -579,7 +386,7 @@ func (t *Teller) clear(askID string) {
if err != nil || e == nil {
return
}
c := oc.Condition{Key: conditionKey(id), Subject: oc.ConditionSubject{Scope: t.kindOf(askID).scope, ID: id}, State: occ.StateCleared}
c := oc.Condition{Key: conditionKey(id), Subject: oc.ConditionSubject{Scope: assignerWords.scope, ID: id}, State: occ.StateCleared}
if err := t.Say(c); err != nil {
t.logf("the condition of %s could not be cleared: %v", id, err)
return
@@ -594,8 +401,8 @@ func (t *Teller) clear(askID string) {
// `room`, and each time one of the teller's asks ends at the router (a warrant of any outcome: the room it held is free)
// the first that waits is asked again, under its own router id and a bus message id of its own (the bus drops a second
// message under an id it holds). It is asked again at most mostRoomRefusals times; refused that often it is said as a
// condition, as any refusal is. An ask the gate took away or asked again under a newer asking meanwhile, or one already
// answered, is the gate's no more, and is dropped from the wait. The teller holds no timer: an end, and its start, are the
// condition, as any refusal is. An ask the assigner took away or asked again under a newer asking meanwhile, or one already
// answered, is the assigner's no more, and is dropped from the wait. The teller holds no timer: an end, and its start, are the
// events that ask again.
// mostRoomRefusals is how many times an ask is refused for want of room before it is said as a condition and asked no
@@ -605,7 +412,7 @@ const mostRoomRefusals = 6
// roomRecord is the key of the asks that wait for room, in the order they were refused.
const roomRecord = "room"
// waitingAsk is one ask that waits for room: the gate key it was asked for, its router id, how often the router refused
// waitingAsk is one ask that waits for room: the ask key it was asked for, its router id, how often the router refused
// it for want of room, and whether it is asked again and its word not yet heard.
type waitingAsk struct {
Key string `json:"key"`
@@ -680,7 +487,7 @@ func (t *Teller) roomRefused(w oc.Warrant) {
return
}
t.logf("the router refused the ask %s for want of room %d times (%s): it is asked no more, and said", w.Ask, n, w.Words)
t.raise(w.Ask, t.kindOf(w.Ask).unaskedHeadline, fmt.Sprintf("the router refused it %d times for want of room: %s", n, w.Words))
t.raise(w.Ask, assignerWords.unaskedHeadline, fmt.Sprintf("the router refused it %d times for want of room: %s", n, w.Words))
return
}
if t.keepWaiting(ws, rec) != nil {
@@ -703,7 +510,7 @@ func (t *Teller) endWait(id string) {
_ = t.keepWaiting(slices.Delete(ws, i, i+1), rec)
}
// OnCurrent is the watch of the gate's asks having handed over every key it holds now, at the teller's start: an ask that
// OnCurrent is the watch of the assigner's asks having handed over every key it holds now, at the teller's start: an ask that
// waited for room while the teller was away is asked again (the asks that held the room may have ended meanwhile).
//
// An ask asked again whose word the teller did not hear (the router's warrants come on a plain subscription) waits once
@@ -754,7 +561,7 @@ func (t *Teller) unheard(id string) {
}
}
// askWaiting asks again the first n asks that wait for room and are not asked yet, dropping on the way every one the gate
// askWaiting asks again the first n asks that wait for room and are not asked yet, dropping on the way every one the assigner
// no longer asks. One at an end: one ask ended, so there is room for one.
func (t *Teller) askWaiting(n int) {
ws, rec, err := t.waiting()
@@ -777,9 +584,8 @@ func (t *Teller) askWaiting(n int) {
i++
continue
}
assigner := isAssignerKey(x.Key)
if assigner && !t.assignerRoom() {
i++ // the assigner's asks hold their most open: it waits, and a merge ask after it may be asked
if isAssignerKey(x.Key) && !t.assignerRoom() {
i++ // the assigner's asks hold their most open: it waits
continue
}
a, why := t.stillAsked(x)
@@ -794,9 +600,7 @@ func (t *Teller) askWaiting(n int) {
return
}
ws[i].Asked = true
if assigner {
t.assignerAsked(x.ID)
}
t.assignerAsked(x.ID)
t.logf("%s: the ask %s is asked of the operator again (refused %d times for want of room)", x.Key, x.ID, x.Refusals)
if n--; n <= 0 {
return
@@ -805,40 +609,16 @@ func (t *Teller) askWaiting(n int) {
}
}
// stillAsked is the gate's ask a waiting ask is for, when the gate still asks it under the same id and it is unanswered;
// otherwise why it is not.
// stillAsked is the assigner's ask a waiting ask is for, when the assigner still asks it under the same id and it is
// unanswered; otherwise why it is not. A waiting ask of a key that is no ask of the assigner (a merge ask kept before the
// Approve per merge went) is dropped.
func (t *Teller) stillAsked(x waitingAsk) (oc.Ask, string) {
if x.Key == "" {
return oc.Ask{}, "the teller holds no gate key for it"
return oc.Ask{}, "the teller holds no key for it"
}
if isAssignerKey(x.Key) {
return t.stillAskedByAssigner(x)
if !isAssignerKey(x.Key) {
return oc.Ask{}, "it is no ask of the assigner, and the teller carries no merge ask"
}
raw, found, err := t.Gate.Get(x.Key)
if err != nil {
return oc.Ask{}, "the gate's ask could not be read: " + err.Error()
}
if !found {
return oc.Ask{}, "the gate has taken it away"
}
ga, err := decodeGateAsk(raw)
if err != nil {
return oc.Ask{}, "the gate's key holds no ask: " + err.Error()
}
if ga.Ask.ID != x.ID {
return oc.Ask{}, "the gate asks again under " + ga.Ask.ID
}
if cur, _, has, err := t.Answers.Get(answerKey(x.Key)); err == nil && has && answeredAsking(cur) == x.ID {
return oc.Ask{}, "it is answered"
}
if err := occ.Check(ga.Ask, t.now()); err != nil {
return oc.Ask{}, "it no longer holds: " + err.Error()
}
return ga.Ask, ""
}
// stillAskedByAssigner is stillAsked for one of the assigner's asks.
func (t *Teller) stillAskedByAssigner(x waitingAsk) (oc.Ask, string) {
a, found, err := t.assignerAsk(x.Key)
switch {
case err != nil:
modules/mesh-delivery-teller/cmd/mesh-delivery-teller/teller_test.go
+147 -314
View File
@@ -1,9 +1,7 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
@@ -12,19 +10,11 @@ import (
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/process"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient"
meshdelivery "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
stdio "git.novox.be/novox/mesh-sdk/go"
)
const (
headA = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
headB = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
)
var now = time.Date(2026, 10, 20, 9, 0, 0, 0, time.UTC)
// nowIs sets the tests' clock now to at and returns what puts it back; a test sets it only as t.Cleanup(nowIs(at)), so no
@@ -119,10 +109,11 @@ type bed struct {
client *occ.Client
store *memStore
answers *memAnswers
gate memGate
pubs []published
conds []oc.Condition
records map[string]string
// assigner and view are the pictures of the assigner's asks and the view's answers the teller follows.
assigner, view memGate
pubs []published
conds []oc.Condition
records map[string]string
// seenIDs are the ids the fake router has taken an ask under, and dropped the asks it took a second time under: the
// messenger keeps an ask after it ends and returns for an ask whose id it holds (mesh-catalog modules/messenger
// cmd/messenger/asks.go:487-489, keepAsk), so a reused id reaches no phone.
@@ -140,7 +131,7 @@ type bed struct {
func newBed(t *testing.T) *bed {
t.Helper()
b := &bed{store: &memStore{entries: map[string]*stdio.StateEntry{}}, answers: &memAnswers{written: map[string][]byte{}, revs: map[string]uint64{}},
gate: memGate{}, records: map[string]string{}, seenIDs: map[string]bool{}, msgIDs: map[string]bool{}}
assigner: memGate{}, view: memGate{}, records: map[string]string{}, seenIDs: map[string]bool{}, msgIDs: map[string]bool{}}
b.client = &occ.Client{Module: tellerModule, State: b.store, Now: func() time.Time { return now },
Publish: func(subject string, body any, id string) error {
raw, err := json.Marshal(body)
@@ -173,7 +164,7 @@ func newBed(t *testing.T) *bed {
},
Subscribe: func(_ string, handle func(stdio.Envelope) error) error { b.hear = handle; return nil },
}
b.t = &Teller{Asker: b.client, Store: b.store, Answers: b.answers, Gate: b.gate, Publish: b.client.Publish, Now: func() time.Time { return now },
b.t = &Teller{Asker: b.client, Store: b.store, Answers: b.answers, Assigner: b.assigner, View: b.view, Publish: b.client.Publish, Now: func() time.Time { return now },
Say: func(c oc.Condition) error {
if _, err := occ.EncodeCondition(c); err != nil {
t.Errorf("a condition the router would refuse: %v", err)
@@ -201,41 +192,31 @@ func newBed(t *testing.T) *bed {
return b
}
// gateAsk is an ask as the gate composes it: the router's ask with the approve and decline options binding the act
// `mesh-delivery.merge` for this ask and the heads (mesh-catalog modules/mesh-delivery approval.go:123-191 composeMergeAsk
// and :63-65 mergeAct), under the id its key gives (mesh-sdk go/delivery AskID). boundTo are the heads the options bind,
// which are the ask's own unless a test says another.
func gateAsk(t *testing.T, heads, boundTo []meshdeliveryclient.Head) (key string, value []byte, ask gateAsked) {
// holdOf is the assigner's hold ask of module plex at the n-th fingerprint, written at its asking-th writing (its router id is
// AskingID(key, asking)), put in the bed's picture of the assigner's asks and handed to the teller as the watch would.
func (b *bed) holdOf(t *testing.T, n, asking int) (string, oc.Ask) {
t.Helper()
return gateAskN(t, heads, boundTo, 1)
key, value, ask := holdAskAt(t, n, asking)
b.assigner[key] = value
b.t.OnAssignerAsk(key)
return key, ask
}
// gateAskN is the asking-th writing of the group's ask at these heads: its router id is AskingID(key, asking) (mesh-sdk
// go/delivery Ask.Asking), the first asking's the key's own.
func gateAskN(t *testing.T, heads, boundTo []meshdeliveryclient.Head, asking int) (key string, value []byte, ask gateAsked) {
// holdAskAt is the assigner's hold ask at the n-th fingerprint and the asking-th writing, shaped as the assigner composes it
// (two options of level acknowledge that bind no act, so the router takes it).
func holdAskAt(t *testing.T, n, asking int) (string, []byte, oc.Ask) {
t.Helper()
return gateAskOf(t, "g", "mesh-delivery.merge", heads, boundTo, asking)
}
// gateAskOf is gateAskN for a group whose options bind the verb given.
func gateAskOf(t *testing.T, group, verb string, heads, boundTo []meshdeliveryclient.Head, asking int) (key string, value []byte, ask gateAsked) {
t.Helper()
key, err := meshdeliveryclient.MergeAskKey(group, heads)
key, err := asks.HoldAskKey("plex", "src2:"+strings.Repeat(string(rune('0'+n)), 64))
if err != nil {
t.Fatal(err)
}
id := meshdeliveryclient.AskingID(key, asking)
a := oc.Ask{ID: id, Headline: "Merge into main: novox/mesh-delivery#12", Explanation: "Merges two pull requests.\nReview: approved.",
Who: occ.Operator, Expires: occ.FormatTime(now.Add(occ.ApproveLasts - 10*time.Minute)), OnExpiry: "the merge stays held", About: "delivery.mesh-delivery.12"}
for _, o := range []struct{ id, label, does string }{{"approve", "Approve", "merges"}, {"decline", "Decline", "merges nothing"}} {
binds, err := occ.ActDigest(mergeAct(verb, id, o.id, headsOf(boundTo)))
if err != nil {
t.Fatal(err)
}
a.Options = append(a.Options, oc.Option{ID: o.id, Label: o.label, Does: o.does, Level: occ.Approve, Binds: binds})
_, _, ask := holdAsk(t)
ask.ID = meshdeliveryclient.AskingID(key, asking)
value, err := asks.EncodeAsk(key, asks.Ask{Ask: rawAsk(t, ask), Asking: asking})
if err != nil {
t.Fatal(err)
}
ask = gateAsked{Group: group, Heads: heads, Asking: asking, Ask: a}
return key, encodeGate(t, ask), ask
return key, value, ask
}
// rawAsk is the router's ask as the JSON a gate or assigner value carries (the seat's ask value).
@@ -248,31 +229,9 @@ func rawAsk(t *testing.T, a oc.Ask) json.RawMessage {
return b
}
// encodeGate is the gate's value for ga, written as the gate writes it: the router's ask checked against its key and asking
// (an id other than the one they give, no option), then carried as JSON.
func encodeGate(t *testing.T, ga gateAsked) []byte {
t.Helper()
key, err := meshdeliveryclient.MergeAskKey(ga.Group, ga.Heads)
if err != nil {
t.Fatal(err)
}
if want := meshdeliveryclient.AskingID(key, ga.Asking); ga.Ask.ID != want || len(ga.Ask.Options) == 0 {
t.Fatalf("the ask's id %q, want %q, with %d options", ga.Ask.ID, want, len(ga.Ask.Options))
}
value, err := gateValue(ga.Group, ga.Heads, ga.Asking, rawAsk(t, ga.Ask))
if err != nil {
t.Fatal(err)
}
return value
}
func twoHeads() []meshdeliveryclient.Head {
return []meshdeliveryclient.Head{{Repository: "novox/mesh-delivery", Number: 12, Head: headA}, {Repository: "novox/mesh-sdk", Number: 4, Head: headB}}
}
// warrantFor is the router's warrant for a chosen option of an ask, as the router verified the operator.
func warrantFor(a oc.Ask, option string) oc.Warrant {
return oc.Warrant{Ask: a.ID, Asker: tellerModule, Outcome: occ.OutcomeChosen, Option: option, Label: "Approve", Level: occ.Approve,
return oc.Warrant{Ask: a.ID, Asker: tellerModule, Outcome: occ.OutcomeChosen, Option: option, Label: option, Level: occ.Acknowledge,
By: &oc.Person{Who: occ.Operator, Kind: "telegram", Identity: "42", Verified: "user id"}, Channel: "telegram",
At: occ.FormatTime(now.Add(time.Hour)), AskDigest: occ.Digest(a)}
}
@@ -295,72 +254,51 @@ func (b *bed) askedOf(subject string) []published {
return out
}
func TestAnAsksKeyBecomesOneRouterAskAndDeletingItCancelsIt(t *testing.T) { // 1343/AC1
t.Run("1945: a refused bucket is said, or stops the start naming its grant", func(t *testing.T) {
refusal := errors.New("nats: permissions violation for publish to $JS.API.STREAM.INFO.KV_x")
var said []string
if kv := process.OptionalBucket(context.Background(), refusingBus{refusal}, "x", "x is not heard", func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }); kv != nil || len(said) != 1 || !strings.Contains(said[0], "permissions violation") {
t.Errorf("an optional bucket refused: %v, said %v", kv, said)
}
if _, err := process.EssentialBucket(context.Background(), refusingBus{refusal}, "teller", "x", "work"); err == nil || !strings.Contains(err.Error(), "grant of x") {
t.Errorf("an essential bucket refused: %v, want its grant named", err)
}
})
func TestAnAssignersKeyBecomesOneRouterAskAndDeletingItCancelsIt(t *testing.T) { // 1343/AC1
b := newBed(t)
key, value, ga := gateAsk(t, twoHeads(), twoHeads())
b.gate[key] = value
b.t.OnAsk(key, 3, value)
key, value, ask := holdAskAt(t, 1, 1)
b.assigner[key] = value
b.t.OnAssignerAsk(key)
asked := b.askedOf(oc.AcceptSubjectAsk(tellerModule))
if len(asked) != 1 || asked[0].id != ga.Ask.ID {
t.Fatalf("published %+v, want one ask under %s with id %s", b.pubs, tellerModule, ga.Ask.ID)
if len(asked) != 1 || asked[0].id != ask.ID {
t.Fatalf("published %+v, want one ask under %s with id %s", b.pubs, tellerModule, ask.ID)
}
var got oc.Ask
if err := json.Unmarshal(asked[0].body, &got); err != nil {
t.Fatal(err)
}
if occ.Digest(got) != occ.Digest(ga.Ask) || got.Headline != ga.Ask.Headline || got.Explanation != ga.Ask.Explanation || got.ID != meshdeliveryclient.AskID(key) {
t.Errorf("the router's ask differs from the gate's: %+v", got)
}
b.t.OnAsk(key, 3, value) // the revision seen again, as at a restart
b.t.OnAsk(key, 4, value) // the key written again with the same ask
b.t.OnAssignerAsk(key) // the key written again with the same ask, or heard again at a restart
if n := len(b.askedOf(oc.AcceptSubjectAsk(tellerModule))); n != 1 {
t.Errorf("the ask was published %d times", n)
}
delete(b.gate, key)
b.t.OnAskGone(key)
delete(b.assigner, key)
b.t.OnAssignerAskGone(key)
cancels := b.askedOf(oc.AcceptSubjectCancel(tellerModule))
if len(cancels) != 1 || string(cancels[0].body) != `{"id":"`+ga.Ask.ID+`"}` {
if len(cancels) != 1 || string(cancels[0].body) != `{"id":"`+ask.ID+`"}` {
t.Fatalf("cancels = %+v", cancels)
}
b.t.OnAskGone(key)
b.t.OnAssignerAskGone(key)
if n := len(b.askedOf(oc.AcceptSubjectCancel(tellerModule))); n != 1 {
t.Errorf("the ask was cancelled %d times", n)
}
// Issue 774 (2026-10-31): two merge asks of pull requests merged by another path held two of the router's three places,
// their keys never taken away, and a fourth ask was refused for want of room. teller_end_ask ends one by hand, named by its
// gate key or its router id; the router's word that it ended asks the one waiting for room again.
// Issue 774: teller_end_ask ends one by hand, named by its key or its router id; the router's word that it ended asks the
// one waiting for room again (the assigner's asks hold at most mostAssignerOpen places, the third waits).
for _, c := range []struct {
name string
by func(key string, ga gateAsked) string
by func(key string, ask oc.Ask) string
}{
{"774: ended by hand, named by its gate key", func(key string, _ gateAsked) string { return key }},
{"774: ended by hand, named by its router id", func(_ string, ga gateAsked) string { return ga.Ask.ID }},
{"774: ended by hand, named by its key", func(key string, _ oc.Ask) string { return key }},
{"774: ended by hand, named by its router id", func(_ string, ask oc.Ask) string { return ask.ID }},
} {
t.Run(c.name, func(t *testing.T) {
b := newBed(t)
k1, first := b.askOf(t, 1, 1)
b.askOf(t, 2, 2)
b.askOf(t, 3, 3)
_, fourth := b.askOf(t, 4, 4)
b.refuse(t, fourth.Ask.ID, roomRefusal)
r := b.t.EndAsk(c.by(k1, first), "its pull request merged without the Approve", "lead@shanks")
k1, first := b.holdOf(t, 1, 1)
b.holdOf(t, 2, 1)
_, third := b.holdOf(t, 3, 1)
r := b.t.EndAsk(c.by(k1, first), "the hold no longer holds", "lead@shanks")
cancels := b.askedOf(oc.AcceptSubjectCancel(tellerModule))
if !r.Ended || r.ID != first.Ask.ID || r.Key != k1 || len(cancels) != 1 || string(cancels[0].body) != `{"id":"`+first.Ask.ID+`"}` {
if !r.Ended || r.ID != first.ID || r.Key != k1 || len(cancels) != 1 || string(cancels[0].body) != `{"id":"`+first.ID+`"}` {
t.Fatalf("end = %+v, cancels %+v", r, cancels)
}
b.deliver(t, oc.Warrant{Ask: first.Ask.ID, Asker: tellerModule, Outcome: occ.OutcomeCancelled, At: occ.FormatTime(now), AskDigest: occ.Digest(first.Ask)})
if got := b.askedIDs(); len(got) != 5 || got[4] != fourth.Ask.ID {
t.Errorf("asks = %v, want the refused %s asked again once the ask ended by hand left", got, fourth.Ask.ID)
b.deliver(t, oc.Warrant{Ask: first.ID, Asker: tellerModule, Outcome: occ.OutcomeCancelled, At: occ.FormatTime(now), AskDigest: occ.Digest(first)})
if got := b.askedIDs(); len(got) != 3 || got[2] != third.ID {
t.Errorf("asks = %v, want the waiting %s asked once the ask ended by hand left", got, third.ID)
}
if again := b.t.EndAsk(k1, "again", "lead@shanks"); again.Ended || len(b.askedOf(oc.AcceptSubjectCancel(tellerModule))) != 1 {
t.Errorf("a second end = %+v: an ended ask is left", again)
@@ -369,62 +307,22 @@ func TestAnAsksKeyBecomesOneRouterAskAndDeletingItCancelsIt(t *testing.T) { // 1
}
t.Run("774: an ask waiting for room is dropped from the wait; a name the teller holds no ask under is said", func(t *testing.T) {
b := newBed(t)
_, first := b.askOf(t, 1, 1)
b.askOf(t, 2, 2)
b.askOf(t, 3, 3)
_, fourth := b.askOf(t, 4, 4)
b.refuse(t, fourth.Ask.ID, roomRefusal)
if r := b.t.EndAsk(fourth.Ask.ID, "asked about what no longer holds", "lead@shanks"); !r.Ended || len(b.askedOf(oc.AcceptSubjectCancel(tellerModule))) != 0 {
_, first := b.holdOf(t, 1, 1)
b.holdOf(t, 2, 1)
_, third := b.holdOf(t, 3, 1)
if r := b.t.EndAsk(third.ID, "asked about what no longer holds", "lead@shanks"); !r.Ended || len(b.askedOf(oc.AcceptSubjectCancel(tellerModule))) != 0 {
t.Fatalf("end of a waiting ask = %+v", r)
}
b.deliver(t, warrantFor(first.Ask, "approve"))
if got := b.askedIDs(); len(got) != 4 {
b.deliver(t, warrantFor(first, "keep"))
if got := b.askedIDs(); len(got) != 2 {
t.Errorf("asks = %v: the ask ended while it waited was asked again", got)
}
if r := b.t.EndAsk("merge.nothing.here", "a typo", "lead@shanks"); r.Ended || !strings.Contains(r.Says, "holds no ask") {
if r := b.t.EndAsk("hold.nothing.here", "a typo", "lead@shanks"); r.Ended || !strings.Contains(r.Says, "holds no ask") {
t.Errorf("end of an unknown ask = %+v", r)
}
})
}
func TestAWarrantThatHoldsWritesOneAnswerAndASecondWritesNone(t *testing.T) { // 1343/AC2
b := newBed(t)
key, value, ga := gateAsk(t, twoHeads(), twoHeads())
b.gate[key] = value
b.t.OnAsk(key, 1, value)
w := warrantFor(ga.Ask, "approve")
b.deliver(t, w)
raw, ok := b.answers.written[answerKey(key)]
if !ok || len(b.answers.written) != 1 {
t.Fatalf("answers = %v", b.answers.written)
}
a, err := meshdeliveryclient.DecodeAnswer(raw)
if err != nil || a.Option != "approve" || a.Warrant != warrantID(w) || !strings.Contains(a.Words, "chose Approve") {
t.Errorf("answer %+v err %v", a, err)
}
b.deliver(t, w) // the same warrant again
w2 := w
w2.At = occ.FormatTime(now.Add(time.Hour + time.Minute)) // a second warrant for the same ask
b.deliver(t, w2)
if b.answers.writes != 1 {
t.Errorf("the answer was written %d times, want once", b.answers.writes)
}
if len(b.conds) != 0 {
t.Errorf("conditions said: %+v", b.conds)
}
// design 47 G §431: the settle ask went with the verb settle, so an ask whose options bind it holds no more.
for verb, holds := range map[string]bool{"mesh-delivery.settle": false, "mesh-delivery.merge": true} {
b := newBed(t)
key, value, ga := gateAskOf(t, "g", verb, twoHeads(), twoHeads(), 1)
b.gate[key] = value
b.t.OnAsk(key, 1, value)
b.deliver(t, warrantFor(ga.Ask, "approve"))
if _, written := b.answers.written[answerKey(key)]; written != holds {
t.Errorf("an ask binding %s: answer written %v, want %v", verb, written, holds)
}
}
}
// ADR 0596 §2: the teller says answered after each answer it writes, under one message id per ask, and says it again for an
// answer written whose event was not said.
func TestEveryAnswerWrittenIsSaidAsTheEventAnswered(t *testing.T) {
@@ -438,89 +336,95 @@ func TestEveryAnswerWrittenIsSaidAsTheEventAnswered(t *testing.T) {
said = append(said, answeredID(key, askID)+" "+string(answer))
return nil
}
key, value, ga := gateAsk(t, twoHeads(), twoHeads())
b.gate[key] = value
b.t.OnAsk(key, 1, value)
w := warrantFor(ga.Ask, "approve")
b.deliver(t, w)
key, ask := b.holdOf(t, 1, 1)
b.deliver(t, warrantFor(ask, "keep"))
if len(said) != 0 || b.answers.writes != 1 {
t.Fatalf("said %v with the bus down, written %d times", said, b.answers.writes)
}
fail = false
if err := b.t.writeAnswer(key, ga.Ask.ID, b.answers.written[answerKey(key)], "again"); err != nil { // the answer is there; its event was not said
if err := b.t.writeAnswer(key, ask.ID, b.answers.written[answerKey(key)], "again"); err != nil { // the answer is there; its event was not said
t.Fatal(err)
}
if len(said) != 1 || b.answers.writes != 1 {
t.Fatalf("said %v, written %d times; want the event said once and the answer written once", said, b.answers.writes)
}
raw := b.answers.written[answerKey(key)]
if want := answeredID(key, ga.Ask.ID) + " " + string(raw); said[0] != want {
if want := answeredID(key, ask.ID) + " " + string(raw); said[0] != want {
t.Errorf("said %q, want %q", said[0], want)
}
}
func TestAWarrantThatDoesNotHoldWritesNoAnswerAndRaisesTheCondition(t *testing.T) { // 1343/AC3
other := []meshdeliveryclient.Head{{Repository: "novox/mesh-delivery", Number: 12, Head: headB}, {Repository: "novox/mesh-sdk", Number: 4, Head: headB}}
cases := map[string]func(t *testing.T) (*bed, string, oc.Warrant){
"heads that differ from the ones the option bound": func(t *testing.T) (*bed, string, oc.Warrant) {
b := newBed(t)
key, value, ga := gateAsk(t, twoHeads(), other) // the options bind other heads than the ask covers
b.gate[key] = value
b.t.OnAsk(key, 1, value)
return b, key, warrantFor(ga.Ask, "approve")
},
"an asker that differs": func(t *testing.T) (*bed, string, oc.Warrant) {
b := newBed(t)
key, value, ga := gateAsk(t, twoHeads(), twoHeads())
b.gate[key] = value
b.t.OnAsk(key, 1, value)
w := warrantFor(ga.Ask, "approve")
w.Asker = "mesh-delivery"
return b, key, w
},
"a digest that differs": func(t *testing.T) (*bed, string, oc.Warrant) {
b := newBed(t)
key, value, ga := gateAsk(t, twoHeads(), twoHeads())
b.gate[key] = value
b.t.OnAsk(key, 1, value)
w := warrantFor(ga.Ask, "approve")
w.AskDigest = "sha256:0000"
return b, key, w
},
cases := map[string]func(w *oc.Warrant){
"an asker that differs": func(w *oc.Warrant) { w.Asker = "mesh-delivery" },
"a digest that differs": func(w *oc.Warrant) { w.AskDigest = "sha256:0000" },
"an option the ask lacks": func(w *oc.Warrant) { w.Option = "delete" },
}
for name, mk := range cases {
for name, change := range cases {
t.Run(name, func(t *testing.T) {
b, key, w := mk(t)
b := newBed(t)
key, ask := b.holdOf(t, 1, 1)
w := warrantFor(ask, "keep")
change(&w)
b.deliver(t, w)
if len(b.answers.written) != 0 {
t.Errorf("an answer was written: %v", b.answers.written)
}
if len(b.conds) != 1 || b.conds[0].State != occ.StateRaised || b.conds[0].Subject.ID != key ||
!strings.HasPrefix(b.conds[0].Explanation, occ.NeedsYou) {
if len(b.conds) != 1 || b.conds[0].State != occ.StateRaised || b.conds[0].Subject.ID != key || b.conds[0].Subject.Scope != "delivery" ||
!strings.HasPrefix(b.conds[0].Explanation, occ.NeedsYou) || strings.Contains(b.conds[0].Explanation, "merge") {
t.Fatalf("conditions = %+v", b.conds)
}
})
}
}
// design 47 G §653, A-172: the teller carries no merge ask. A router ask it holds under a key that is no assigner key (a merge
// ask kept before the Approve per merge went) answers nothing, and a merge ask waiting for room is dropped from the wait.
func TestAMergeAskOfBeforeTheApprovePerMergeWentAnswersNothingAndWaitsNoMore(t *testing.T) {
b := newBed(t)
_, _, ask := holdAskAt(t, 1, 1)
mergeKey := "merge.g.0123456789ab"
ask.ID = meshdeliveryclient.AskID(mergeKey)
if err := b.t.note(ask.ID, mergeKey, 3, nil); err != nil {
t.Fatal(err)
}
if err := b.client.Ask(ask); err != nil {
t.Fatal(err)
}
if ws, rec, _ := b.t.waiting(); rec != nil || len(ws) != 0 {
t.Fatalf("the wait holds %v", ws)
}
b.refuse(t, ask.ID, roomRefusal) // it waits for room, as a merge ask did
b.deliver(t, warrantFor(ask, "keep"))
if len(b.answers.written) != 0 {
t.Errorf("a merge ask was answered: %v", b.answers.written)
}
b.t.OnCurrent() // the start: the wait is read, and the merge ask is dropped, not asked again
if got := b.askedIDs(); len(got) != 1 {
t.Errorf("asks = %v: a merge ask was asked again", got)
}
if ws, _, _ := b.t.waiting(); len(ws) != 0 {
t.Errorf("the wait still holds %+v", ws)
}
}
func TestAGoodAnswerClearsTheConditionOfAnAskAskedAgain(t *testing.T) { // 1343/AC3 (cleared when a good answer is written)
b := newBed(t)
other := []meshdeliveryclient.Head{{Repository: "novox/mesh-delivery", Number: 12, Head: headB}, {Repository: "novox/mesh-sdk", Number: 4, Head: headB}}
key, bad, gaBad := gateAsk(t, twoHeads(), other)
b.gate[key] = bad
b.t.OnAsk(key, 1, bad)
b.deliver(t, warrantFor(gaBad.Ask, "approve"))
key, bad := b.holdOf(t, 1, 1)
w := warrantFor(bad, "keep")
w.AskDigest = "sha256:0000"
b.deliver(t, w)
if len(b.conds) != 1 || b.conds[0].State != occ.StateRaised {
t.Fatalf("conditions = %+v", b.conds)
}
// The gate writes the ask again, right this time, at a new revision of its key: the second asking, under a new id.
_, good, gaGood := gateAskN(t, twoHeads(), twoHeads(), 2)
b.gate[key] = good
b.t.OnAsk(key, 2, good)
// The assigner writes the ask again at a new asking, under a new id.
_, value, good := holdAskAt(t, 1, 2)
b.assigner[key] = value
b.t.OnAssignerAsk(key)
if n := len(b.askedOf(oc.AcceptSubjectAsk(tellerModule))); n != 2 {
t.Fatalf("the ask was published %d times, want again at the new revision", n)
t.Fatalf("the ask was published %d times, want again at the new asking", n)
}
b.deliver(t, warrantFor(gaGood.Ask, "approve"))
b.deliver(t, warrantFor(good, "keep"))
if _, ok := b.answers.written[answerKey(key)]; !ok {
t.Fatalf("no answer: %v\n%s", b.answers.written, b.logs.String())
}
@@ -531,12 +435,10 @@ func TestAGoodAnswerClearsTheConditionOfAnAskAskedAgain(t *testing.T) { // 1343/
func TestAWarrantComingWhileTheTellerWasAwayIsReadFromTheRouter(t *testing.T) {
b := newBed(t)
key, value, ga := gateAsk(t, twoHeads(), twoHeads())
b.gate[key] = value
b.t.OnAsk(key, 1, value)
w, _ := json.Marshal(warrantFor(ga.Ask, "approve"))
b.records[ga.Ask.ID] = `{"state":"chosen","warrant":` + string(w) + `}`
b.t.OnAsk(key, 1, value) // the teller starts again and reads the key at the same revision
key, ask := b.holdOf(t, 1, 1)
w, _ := json.Marshal(warrantFor(ask, "keep"))
b.records[ask.ID] = `{"state":"chosen","warrant":` + string(w) + `}`
b.t.OnAssignerAsk(key) // the teller starts again and reads the key
if _, ok := b.answers.written[answerKey(key)]; !ok {
t.Errorf("no answer: %s", b.logs.String())
}
@@ -544,34 +446,32 @@ func TestAWarrantComingWhileTheTellerWasAwayIsReadFromTheRouter(t *testing.T) {
func TestARewriteWithAskingTwoCancelsAskingOneAndAsksUnderTheNewID(t *testing.T) { // G2 of review round 1
b := newBed(t)
key, one, g1 := gateAskN(t, twoHeads(), twoHeads(), 1)
b.gate[key] = one
b.t.OnAsk(key, 1, one)
_, two, g2 := gateAskN(t, twoHeads(), twoHeads(), 2)
if g1.Ask.ID == g2.Ask.ID {
key, g1 := b.holdOf(t, 1, 1)
_, two, g2 := holdAskAt(t, 1, 2)
if g1.ID == g2.ID {
t.Fatal("the two askings share an id")
}
b.gate[key] = two
b.t.OnAsk(key, 2, two)
b.assigner[key] = two
b.t.OnAssignerAsk(key)
cancels := b.askedOf(oc.AcceptSubjectCancel(tellerModule))
if len(cancels) != 1 || string(cancels[0].body) != `{"id":"`+g1.Ask.ID+`"}` {
if len(cancels) != 1 || string(cancels[0].body) != `{"id":"`+g1.ID+`"}` {
t.Fatalf("cancels = %+v, want asking 1's router ask", cancels)
}
asked := b.askedOf(oc.AcceptSubjectAsk(tellerModule))
if len(asked) != 2 || asked[1].id != g2.Ask.ID {
t.Fatalf("asks = %+v, want the second under %s", asked, g2.Ask.ID)
if len(asked) != 2 || asked[1].id != g2.ID {
t.Fatalf("asks = %+v, want the second under %s", asked, g2.ID)
}
if len(b.dropped) != 0 {
t.Errorf("the router dropped asks under a reused id: %v", b.dropped)
}
// The same revision again, and a warrant for the first asking (the gate has moved on), change nothing.
b.t.OnAsk(key, 2, two)
b.deliver(t, warrantFor(g1.Ask, "approve"))
// The same key again, and a warrant for the first asking (the assigner has moved on), change nothing.
b.t.OnAssignerAsk(key)
b.deliver(t, warrantFor(g1, "keep"))
if len(b.answers.written) != 0 || len(b.askedOf(oc.AcceptSubjectAsk(tellerModule))) != 2 {
t.Errorf("answers %v asks %d", b.answers.written, len(b.askedOf(oc.AcceptSubjectAsk(tellerModule))))
}
// The warrant of the second names it: its answer's warrant is the router id and the time.
w := warrantFor(g2.Ask, "approve")
w := warrantFor(g2, "keep")
b.deliver(t, w)
raw, ok := b.answers.written[answerKey(key)]
a, err := meshdeliveryclient.DecodeAnswer(raw)
@@ -580,65 +480,20 @@ func TestARewriteWithAskingTwoCancelsAskingOneAndAsksUnderTheNewID(t *testing.T)
}
}
func TestAnAskWhoseWordsChangedUnderTheSameIDIsCancelledAndNotAskedAgain(t *testing.T) {
b := newBed(t)
key, value, ga := gateAsk(t, twoHeads(), twoHeads())
b.t.OnAsk(key, 1, value)
changed := ga
changed.Ask.Explanation = "Merges two pull requests, now with a third point.\nReview: approved."
v2, err := gateValue(changed.Group, changed.Heads, changed.Asking, rawAsk(t, changed.Ask))
if err != nil {
t.Fatal(err)
}
b.t.OnAsk(key, 2, v2)
if n := len(b.askedOf(oc.AcceptSubjectCancel(tellerModule))); n != 1 {
t.Errorf("cancels = %d", n)
}
if n := len(b.askedOf(oc.AcceptSubjectAsk(tellerModule))); n != 1 || len(b.dropped) != 0 {
t.Errorf("asks = %d dropped %v: the router would drop a second ask under the id", n, b.dropped)
}
}
func TestACapitalisedRepositoryIsSpelledInLowerCaseInTheAct(t *testing.T) { // T1 of review round 1
b := newBed(t)
heads := []meshdeliveryclient.Head{{Repository: "novox/Mesh-Delivery", Number: 12, Head: headA}}
key, err := meshdeliveryclient.MergeAskKey("g", heads)
if err != nil {
t.Fatal(err)
}
id := meshdeliveryclient.AskID(key)
// The act as the gate binds it: its IDOf lowercases the repository (mesh-catalog modules/mesh-delivery-gate model.go).
binds, err := occ.ActDigest(occ.Act{"verb": "mesh-delivery.merge", "ask": id, "option": "approve", "heads": "novox/mesh-delivery@" + headA[:12] + "=" + headA})
if err != nil {
t.Fatal(err)
}
a := oc.Ask{ID: id, Headline: "Merge into main: Mesh-Delivery#12", Explanation: "Merges.", Who: occ.Operator, Expires: occ.FormatTime(now.Add(occ.ApproveLasts - 10*time.Minute)),
OnExpiry: "the merge stays held", Options: []oc.Option{{ID: "approve", Label: "Approve", Does: "merges", Level: occ.Approve, Binds: binds}}}
value := encodeGate(t, gateAsked{Group: "g", Heads: heads, Asking: 0, Ask: a})
b.gate[key] = value
b.t.OnAsk(key, 1, value)
b.deliver(t, warrantFor(a, "approve"))
if _, ok := b.answers.written[answerKey(key)]; !ok {
t.Errorf("no answer for the gate's spelling: %s", b.logs.String())
}
}
func TestAKeyWhoseValueIsNotItsOwnAskIsLeft(t *testing.T) {
b := newBed(t)
_, value, _ := gateAsk(t, twoHeads(), twoHeads())
b.t.OnAsk("merge.g.000000000000", 1, value)
b.t.OnAsk("merge.g.000000000000", 2, []byte(`{"v":1`))
if len(b.pubs) != 0 {
t.Errorf("published %+v", b.pubs)
key, _, _ := holdAskAt(t, 1, 1)
b.assigner[key] = []byte(`{"v":1`)
b.t.OnAssignerAsk(key)
if len(b.pubs) != 0 || !strings.Contains(b.logs.String(), "the ask is not raised") {
t.Errorf("published %+v; log:\n%s", b.pubs, b.logs.String())
}
}
func TestARefusedAskIsSaidAsACondition(t *testing.T) { // ADR 0437 §9
b := newBed(t)
key, value, ga := gateAsk(t, twoHeads(), twoHeads())
b.t.OnAsk(key, 1, value)
w := oc.Warrant{Ask: ga.Ask.ID, Asker: tellerModule, Outcome: occ.OutcomeRefused, Words: "no channel can send", At: occ.FormatTime(now)}
b.deliver(t, w)
_, ask := b.holdOf(t, 1, 1)
b.deliver(t, oc.Warrant{Ask: ask.ID, Asker: tellerModule, Outcome: occ.OutcomeRefused, Words: "no channel can send", At: occ.FormatTime(now)})
if len(b.conds) != 1 || b.conds[0].State != occ.StateRaised || !strings.Contains(b.conds[0].Summary, "no channel can send") {
t.Errorf("conditions = %+v", b.conds)
}
@@ -646,26 +501,23 @@ func TestARefusedAskIsSaidAsACondition(t *testing.T) { // ADR 0437 §9
func TestAnAnswerToAnEarlierAskingDoesNotBlockTheNextAsking(t *testing.T) { // R2-1 of review round 2
b := newBed(t)
key, one, g1 := gateAskN(t, twoHeads(), twoHeads(), 1)
b.gate[key] = one
b.t.OnAsk(key, 1, one)
b.deliver(t, warrantFor(g1.Ask, "approve"))
key, g1 := b.holdOf(t, 1, 1)
b.deliver(t, warrantFor(g1, "keep"))
first, ok := b.answers.written[answerKey(key)]
if a, err := meshdeliveryclient.DecodeAnswer(first); !ok || err != nil || !strings.HasPrefix(a.Warrant, g1.Ask.ID+"@") {
if a, err := meshdeliveryclient.DecodeAnswer(first); !ok || err != nil || !strings.HasPrefix(a.Warrant, g1.ID+"@") {
t.Fatalf("asking 1's answer: %+v %v %v", a, ok, err)
}
// The gate rewrites the key at asking 2 (a review record changed). Asking 1's answer stays in the bucket.
_, two, g2 := gateAskN(t, twoHeads(), twoHeads(), 2)
b.gate[key] = two
b.t.OnAsk(key, 2, two)
// The assigner writes the key again at asking 2. Asking 1's answer stays in the bucket.
_, two, g2 := holdAskAt(t, 1, 2)
b.assigner[key] = two
b.t.OnAssignerAsk(key)
asked := b.askedOf(oc.AcceptSubjectAsk(tellerModule))
if len(asked) != 2 || asked[1].id != g2.Ask.ID {
t.Fatalf("asks = %+v, want asking 2 asked under %s", asked, g2.Ask.ID)
if len(asked) != 2 || asked[1].id != g2.ID {
t.Fatalf("asks = %+v, want asking 2 asked under %s", asked, g2.ID)
}
w := warrantFor(g2.Ask, "approve")
w := warrantFor(g2, "release")
b.deliver(t, w)
raw := b.answers.written[answerKey(key)]
a, err := meshdeliveryclient.DecodeAnswer(raw)
a, err := meshdeliveryclient.DecodeAnswer(b.answers.written[answerKey(key)])
if err != nil || a.Warrant != warrantID(w) {
t.Fatalf("the key holds %+v (%v), want asking 2's answer", a, err)
}
@@ -673,27 +525,8 @@ func TestAnAnswerToAnEarlierAskingDoesNotBlockTheNextAsking(t *testing.T) { // R
w2 := w
w2.At = occ.FormatTime(now.Add(time.Hour + time.Minute))
b.deliver(t, w2)
b.t.OnAsk(key, 2, two)
b.t.OnAsk(key, 3, two)
b.t.OnAssignerAsk(key)
if b.answers.writes != 2 || len(b.askedOf(oc.AcceptSubjectAsk(tellerModule))) != 2 || len(b.dropped) != 0 {
t.Errorf("writes %d asks %d dropped %v", b.answers.writes, len(b.askedOf(oc.AcceptSubjectAsk(tellerModule))), b.dropped)
}
}
// refusingBus refuses every bucket with err (issue 1945).
type refusingBus struct{ err error }
func (r refusingBus) KeyValue(context.Context, string) (jetstream.KeyValue, error) { return nil, r.err }
// gateValue is the gate's ask value for the group, heads and asking, carrying the router's ask as its JSON.
func gateValue(group string, heads []meshdeliveryclient.Head, asking int, ask json.RawMessage) ([]byte, error) {
var asked any
if err := json.Unmarshal(ask, &asked); err != nil {
return nil, err
}
var items []meshdelivery.AsksByMergeGroupHeadsValueHeadsItem
for _, h := range heads {
items = append(items, meshdelivery.AsksByMergeGroupHeadsValueHeadsItem{Repository: h.Repository, Number: int64(h.Number), Head: h.Head})
}
return meshdeliveryclient.EncodeAsk(meshdelivery.AsksByMergeGroupHeadsValue{Group: group, Heads: items, Asking: int64(asking), Ask: asked})
}
modules/mesh-delivery-teller/cmd/mesh-delivery-teller/viewanswers.go
+5 -4
View File
@@ -12,11 +12,12 @@ import (
// The view's answers (novox/hq ADR 0528 clause 6, ADR 0437 §10, issue 1297 B10). The operator may answer, in the view, the
// asks of the assigner that need no proof of who answered (the hold of a fingerprint and the restore of a data module set back,
// mesh-sdk go/delivery AskNeedsProof); the view writes its answer under `<ask key>.view` in its bucket answers, and the teller,
// which alone fills an ask's answer, takes it from there as it takes a warrant from the phone: under the ask's own key, once,
// by compare-and-set. The merge's ask needs proof (ADR 0421 §1, issue 953) and is never answered from a view key.
// mesh-sdk go/delivery AskNeedsProof); the view writes its answer under `<ask key>.view` in its state answers, and the teller,
// which alone fills an ask's answer, takes it from the picture it follows of that state (the view's views family, first view
// by view_read_answers; design 47 G A-175) as it takes a warrant from the phone: under the ask's own key, once, by
// compare-and-set.
// OnViewAnswer handles a key put in the view's bucket answers. A key that is no view answer key, or is the answer to an ask that
// OnViewAnswer handles a key put in the followed picture of the view's answers. A key that is no view answer key, or is the answer to an ask that
// needs proof of who answered, is left and said.
func (t *Teller) OnViewAnswer(key string) {
t.mu.Lock()
modules/mesh-delivery-teller/cmd/mesh-delivery-teller/viewanswers_test.go
+6 -9
View File
@@ -47,12 +47,8 @@ func viewAnswer(t *testing.T, option, words string) []byte {
return v
}
// withView gives the bed's teller the assigner's asks and the view's answers, both empty.
func withView(b *bed) (assigner, view memGate) {
assigner, view = memGate{}, memGate{}
b.t.Assigner, b.t.View = assigner, view
return assigner, view
}
// withView is the bed's pictures of the assigner's asks and the view's answers, both empty.
func withView(b *bed) (assigner, view memGate) { return b.assigner, b.view }
func TestAViewAnswerFillsTheAskOnce(t *testing.T) { // 1297/B10 AC
b := newBed(t)
@@ -96,14 +92,15 @@ func TestAViewAnswerHeardBeforeItsAskFillsItWhenTheAskComes(t *testing.T) { // 1
func TestAnAskNeedingProofIsNotAnsweredFromAViewKey(t *testing.T) { // 1297/B10 AC
b := newBed(t)
assigner, view := withView(b)
key, value, _ := gateAsk(t, twoHeads(), twoHeads())
// Even with the merge's ask under the assigner's bucket and a view answer under its key, nothing is written.
// A key of a kind of ask that needs proof of who answered (the merge's, issue 953) with a view answer under it: nothing is written.
key := "merge.g.0123456789ab"
_, value, _ := holdAsk(t)
assigner[key] = value
view[key+".view"] = viewAnswer(t, "approve", "")
b.t.OnViewAnswer(key + ".view")
b.t.OnAssignerAsk(key)
if b.answers.writes != 0 || len(b.answers.written) != 0 {
t.Fatalf("the merge's ask was answered from the view: %v", b.answers.written)
t.Fatalf("an ask needing proof was answered from the view: %v", b.answers.written)
}
if !strings.Contains(b.logs.String(), "issue 953") {
t.Errorf("the refusal does not say why:\n%s", b.logs.String())
modules/mesh-delivery-teller/manifest_test.go
+11 -15
View File
@@ -92,28 +92,24 @@ func TestManifestDeclarations(t *testing.T) { // 1240/AC1
t.Errorf("module = %q", m.Module)
}
// answers is the teller's answer to each ask; asked is what it asked of the router, which condition it raised and which
// gate key and revision each ask was made at (issue 1343); ends is the ends asked by hand with teller_end_ask (issue 774).
// ask key and revision each ask was made at (issue 1343); ends is the ends asked by hand with teller_end_ask (issue 774).
// first-views is where its first view of the assigner's asks stands and findings its condition past the bound (ADR 0615).
if want := []state{{Name: "answers", History: 5}, {Name: "asked", History: 1}, {Name: "ends", History: 1}, {Name: "first-views", History: 1}, {Name: "widens-said", History: 1}}; !reflect.DeepEqual(m.State, want) {
if want := []state{{Name: "answers", History: 5}, {Name: "asked", History: 1}, {Name: "ends", History: 1}, {Name: "first-views", History: 1}, {Name: "widens-said", History: 1}, {Name: "teller-marks", History: 1}}; !reflect.DeepEqual(m.State, want) {
t.Errorf("state = %+v, want %+v", m.State, want)
}
// The view's answers to the assigner's asks are read so the view's answer fills an ask that needs no proof (1297/B10); the
// assigner's asks are its state, asked by its read tool and followed by its views family, never read (ADR 0615 §1, DLV-M3).
if want := []string{"mesh-delivery-gate.asks", "mesh-delivery-view.answers"}; !reflect.DeepEqual(sorted(m.Reads), want) {
t.Errorf("reads = %v, want %v", m.Reads, want)
}
for _, r := range m.Reads {
if strings.HasPrefix(r, "mesh-delivery-indexer.") || strings.HasPrefix(r, "mesh-delivery-assigner.") {
t.Errorf("reads %s: the indexer's and the assigner's state is read by their tools and views families alone", r)
}
// design 47 G §653, A-172: the teller reads no other module's state: the assigner's asks and the view's answers are the
// owners' states, asked by their read tools and followed by their views families (ADR 0615 §1, DLV-M3; A-175).
if len(m.Reads) != 0 {
t.Errorf("reads = %v, want none", m.Reads)
}
// Its own states answers and teller-marks are said as its views family views.answers, read by teller_read_answers (A-175, A-178).
// The join: its first view asked again by hand is a hand act, recorded through the seat (views.StatusTools).
if want := []string{"mesh-delivery-assigner.assigner_read_asks", "seat:mesh-hand-acts.hand-act", "seat:mesh-catalog.registrations", "seat:operator-channel.notify"}; !reflect.DeepEqual(m.Invokes, want) {
if want := []string{"mesh-delivery-assigner.assigner_read_asks", "mesh-delivery-view.view_read_answers", "seat:mesh-hand-acts.hand-act", "seat:mesh-catalog.registrations", "seat:operator-channel.notify"}; !reflect.DeepEqual(m.Invokes, want) {
t.Errorf("invokes = %v, want %v", m.Invokes, want)
}
// design 47 G §335 (TL1): the widening of a pipeline end is told through the seat operator-channel and kept in widens-said.
// design 47 G §387: the keeper's ask for the findings is heard, and the finding stream is declared (§383).
if want := []string{"mesh-conditions.findings-asked", "mesh-delivery-assigner.views.asks", "mesh-catalog.registration-changed", "mesh-pipeline-runner.pipeline-ended", "mesh-delivery.recovery-merged", "mesh-delivery.record-opened"}; !reflect.DeepEqual(m.Consumes, want) {
if want := []string{"mesh-conditions.findings-asked", "mesh-delivery-assigner.views.asks", "mesh-delivery-view.views.answers", "mesh-catalog.registration-changed", "mesh-pipeline-runner.pipeline-ended", "mesh-delivery.recovery-merged", "mesh-delivery.record-opened"}; !reflect.DeepEqual(m.Consumes, want) {
t.Errorf("consumes = %v, want %v", m.Consumes, want)
}
if want := []string{"operator-channel"}; !reflect.DeepEqual(sorted(m.Uses), want) {
@@ -126,12 +122,12 @@ func TestManifestDeclarations(t *testing.T) { // 1240/AC1
t.Errorf("secrets-owner = %q", m.SecretsOwner)
}
// 774: teller_end_ask, served by its tools bundle through the state ends; the process serves none.
if want := []string{"teller_end_ask", "teller_answers", "teller_first_views", "teller_first_view_again", "teller_widens_said"}; !reflect.DeepEqual(m.Tools, want) {
if want := []string{"teller_end_ask", "teller_answers", "teller_read_answers", "teller_first_views", "teller_first_view_again", "teller_widens_said"}; !reflect.DeepEqual(m.Tools, want) {
t.Errorf("tools = %v, want %v", m.Tools, want)
}
// Its two events: each answer it writes (ADR 0596 §2), and an assigner ask ended at the channel without an answer, heard by
// the assigner (issue 1966, tellerevents).
if want := []string{"answered", tellerevents.AskEnded}; !reflect.DeepEqual(m.Emits, want) {
if want := []string{"answered", tellerevents.AskEnded, "views.answers"}; !reflect.DeepEqual(m.Emits, want) {
t.Errorf("emits = %v, want %v", m.Emits, want)
}
// design 47 G §383, §387: the stream <module>-findings, 7 days and 16 MiB; §368: version is MAJOR.MINOR.
modules/mesh-delivery-teller/module.json
+9 -5
View File
@@ -5,12 +5,14 @@
"tools": [
"teller_end_ask",
"teller_answers",
"teller_read_answers",
"teller_first_views",
"teller_first_view_again",
"teller_widens_said"
],
"invokes": [
"mesh-delivery-assigner.assigner_read_asks",
"mesh-delivery-view.view_read_answers",
"seat:mesh-hand-acts.hand-act",
"seat:mesh-catalog.registrations",
"seat:operator-channel.notify"
@@ -18,6 +20,7 @@
"consumes": [
"mesh-conditions.findings-asked",
"mesh-delivery-assigner.views.asks",
"mesh-delivery-view.views.answers",
"mesh-catalog.registration-changed",
"mesh-pipeline-runner.pipeline-ended",
"mesh-delivery.recovery-merged",
@@ -25,7 +28,8 @@
],
"emits": [
"answered",
"ask-ended"
"ask-ended",
"views.answers"
],
"state": [
{
@@ -47,12 +51,12 @@
{
"name": "widens-said",
"history": 1
},
{
"name": "teller-marks",
"history": 1
}
],
"reads": [
"mesh-delivery-gate.asks",
"mesh-delivery-view.answers"
],
"uses": [
"operator-channel"
],
modules/mesh-delivery-view/viewstate/reach.go
+5 -1
View File
@@ -136,8 +136,12 @@ func SameReach(a, b Reach) bool {
// ViewAnswersBucket is the delivery view's state `answers`: one key per ask it answered, and ViewAnswersHistory the revisions it
// keeps per key.
const (
ViewAnswersBucket = ViewModule + "_answers"
ViewAnswersState = "answers"
ViewAnswersBucket = ViewModule + "_" + ViewAnswersState
ViewAnswersHistory = 5
// ViewAnswersReadTool is the view's read tool of its state answers (mesh-sdk views.ReadTool), whose changes it says on its
// views family as views.answers; its marks are its state view-marks (views.MarksState) (design 47 G A-178).
ViewAnswersReadTool = "view_read_answers"
)
// ViewAnswer is the value of the view's answer to an ask of the assigner: the asking it answers, the option chosen and the words
modules/mesh-grants/internal/buscompose/modstreams.go
+18 -3
View File
@@ -49,6 +49,9 @@ type ModuleStream struct {
// History says the stream keeps its history (novox/hq ADR 0592 §6, `retention: history`): every message, with no
// maximum age, a new one refused when it is full rather than its oldest dropped.
History bool
// LastPerSubject says the stream keeps only the last message of each subject (`last-per-subject`, design 47 G §703): a part that
// follows it reads it whole at each start (§556).
LastPerSubject bool
}
// Subjects are the subjects the stream takes.
@@ -93,6 +96,14 @@ func (s ModuleStream) Stream() Stream {
out.Why += "; heard only by the consumers its trusted config value grants"
}
}
if s.LastPerSubject {
out.MaxMsgsPerSubject = 1
out.Why = fmt.Sprintf("%s's own stream kept last per subject (design 47 G §703): the newest message of each subject, at most %d MiB, "+
"read whole by the part that follows it at each start (§556)", s.Module, s.MaxMiB)
if s.Restricted {
out.Why += "; heard only by the consumers its trusted config value grants"
}
}
if s.WorkQueue {
out.Retention = RetentionWorkQueue
out.DiscardNew = true
@@ -251,14 +262,18 @@ func heardGrants(p Principal, hears []Heard) []string {
"$JS.API.CONSUMER.INFO."+h.Stream+"."+durable,
"$JS.API.CONSUMER.MSG.NEXT."+h.Stream+"."+durable,
"$JS.ACK."+h.Stream+"."+durable+".>")
if strings.HasPrefix(h.Stream, seatdef.KeptStreamPrefix) {
// A seat's kept event is heard through process.HearSeatEvents, which reads the stream's own info to see whether it is
seatKept := strings.HasPrefix(h.Stream, seatdef.KeptStreamPrefix)
if seatKept || h.LastPerSubject {
// A seat's kept event, and a module's own stream kept last per subject, are heard through process.HearSeatEvents and
// process.HearStream, which reads the stream's own info to see whether it is
// kept last per subject (design 47 G §641, A-19): the info of this stream alone.
out = append(out, "$JS.API.STREAM.INFO."+h.Stream)
}
if h.LastPerSubject {
if seatKept || h.LastPerSubject {
// A stream kept last per subject is read whole at each start through an ordered consumer, made for that one read
// and with no durable name: its creation, its next message, its info and the deletion of its own (design 47 G §556, A-4).
// So is every seat's kept event, whatever its keep: one kept by age is read whole from its first message with
// process.FromFirst, with no manifest word (A-233).
out = append(out,
"$JS.API.CONSUMER.CREATE."+h.Stream, "$JS.API.CONSUMER.CREATE."+h.Stream+".>",
"$JS.API.CONSUMER.MSG.NEXT."+h.Stream+".*", "$JS.API.CONSUMER.INFO."+h.Stream+".*",
modules/mesh-grants/internal/buscompose/orderedread_test.go
+12
View File
@@ -20,6 +20,14 @@ func TestAReaderOfAStreamKeptLastPerSubjectIsGrantedItsOrderedConsumer(t *testin
if slices.Contains(plain, ordered[0]) {
t.Errorf("a reader of another stream is granted an ordered consumer: %v", plain)
}
// A seat's kept event kept by age is read whole from its first message (process.FromFirst, A-233): its ordered consumer too.
byAge := heardGrants(p, []Heard{{Stream: "SEAT_mesh-catalog_version-released", Filters: []string{"f"}}})
for _, w := range []string{"$JS.API.CONSUMER.CREATE.SEAT_mesh-catalog_version-released", "$JS.API.CONSUMER.MSG.NEXT.SEAT_mesh-catalog_version-released.*",
"$JS.API.CONSUMER.INFO.SEAT_mesh-catalog_version-released.*", "$JS.API.CONSUMER.DELETE.SEAT_mesh-catalog_version-released.*"} {
if !slices.Contains(byAge, w) {
t.Errorf("a reader of a seat's event kept by age is not granted %s: %v", w, byAge)
}
}
}
// A reader of a seat's kept event is granted the info of that stream, which process.HearSeatEvents reads to see whether the
@@ -30,6 +38,10 @@ func TestAReaderOfASeatsKeptEventIsGrantedItsStreamInfo(t *testing.T) {
if !slices.Contains(kept, "$JS.API.STREAM.INFO.SEAT_mesh-delivery_owed") {
t.Errorf("a reader of a seat's kept event is not granted its stream's info: %v", kept)
}
own := heardGrants(p, []Heard{{Stream: "BUILDER_BUILDS", Filters: []string{"f"}, LastPerSubject: true}})
if !slices.Contains(own, "$JS.API.STREAM.INFO.BUILDER_BUILDS") {
t.Errorf("a reader of a module's stream kept last per subject is not granted its info: %v", own)
}
other := heardGrants(p, []Heard{{Stream: "mesh-x-events", Filters: []string{"f"}}})
if slices.Contains(other, "$JS.API.STREAM.INFO.mesh-x-events") {
t.Errorf("a reader of a module's stream is granted its info: %v", other)
modules/mesh-grants/internal/grants/defined_test.go
+17
View File
@@ -25,3 +25,20 @@ func TestAStarEmitterBeforeAFindingEventReachesEachDeclaringStream(t *testing.T)
t.Fatalf("with no event defined, heard %+v", hears)
}
}
// Design 47 G §703, §556, A-181: a stream a manifest declares kept last per subject is asked of the bus so (one message per subject),
// and what hears it is composed to read it whole.
func TestAStreamKeptLastPerSubjectIsComposedAndHeardSo(t *testing.T) {
shelf := map[string]catalogue.Manifest{
"builder": {Module: "builder", Streams: []catalogue.StreamDeclaration{{Name: "builder-builds", Events: []string{"build.ended.**"}, LastPerSubject: true}}},
}
none := func(string, string) []any { return nil }
reader := catalogue.Manifest{Module: "reader", Consumes: []string{"builder.build.ended.**"}}
hears, _, _ := hearsOf(reader, shelf, "ace", none, reader.Consumes, nil)
if len(hears) != 1 || !hears[0].LastPerSubject {
t.Fatalf("heard %+v, want one stream read whole", hears)
}
if got := streamsOf(shelf["builder"]); len(got) != 1 || got[0].Stream().MaxMsgsPerSubject != 1 {
t.Fatalf("composed %+v, want one message per subject", got)
}
}
modules/mesh-grants/internal/grants/records.go
+2 -2
View File
@@ -419,7 +419,7 @@ func streamsOf(m catalogue.Manifest) []buscompose.ModuleStream {
}
out = append(out, buscompose.ModuleStream{Module: m.Module, Name: s.BusName(), Events: events,
MaxAgeDays: s.Days(), MaxMiB: s.Cap(), Restricted: s.Grant != nil,
WorkQueue: s.Retention == catalogue.StreamRetentionWorkQueue, Duplicates: s.Duplicates, History: s.KeepsHistory()})
WorkQueue: s.Retention == catalogue.StreamRetentionWorkQueue, Duplicates: s.Duplicates, History: s.KeepsHistory(), LastPerSubject: s.LastPerSubject})
}
return out
}
@@ -495,7 +495,7 @@ func hearsOf(m catalogue.Manifest, declared map[string]catalogue.Manifest, node
parts, touches, onEvents := catalogue.Hearing(reading, declared, node, values, defined)
var out []buscompose.Heard
for _, p := range parts {
h := buscompose.Heard{Stream: p.Stream.BusName(), Emitter: p.Emitter, Restricted: p.Stream.Grant != nil}
h := buscompose.Heard{Stream: p.Stream.BusName(), Emitter: p.Emitter, Restricted: p.Stream.Grant != nil, LastPerSubject: p.Stream.LastPerSubject}
for _, pattern := range p.Patterns {
h.Filters = append(h.Filters, buscompose.ModuleStreamSubject(p.Emitter, sdkbc.WirePattern(pattern)))
}