Compare commits
29
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b1f25f91d5 | ||
|
|
d281faeeb7 | ||
|
|
5dba7fe68b | ||
|
|
f0f28c56a4 | ||
|
|
20a7bd0f3c | ||
|
|
0b1ce08985 | ||
|
|
f053e3b041 | ||
|
|
d428650ce4 | ||
|
|
351eda0867 | ||
|
|
5e3d92ae33 | ||
|
|
e5e5c50e67 | ||
|
|
8167d825ed | ||
|
|
ac95e5628c | ||
|
|
bed9b61d5a | ||
|
|
67ef449230 | ||
|
|
8a52b6f801 | ||
|
|
380d4781e6 | ||
|
|
7534bf5c07 | ||
|
|
08fe40f233 | ||
|
|
27e96a2a1b | ||
|
|
f98d5ec748 | ||
|
|
abb7cfcf5d | ||
|
|
3956c0cf94 | ||
|
|
403fb3068b | ||
|
|
e54bf7f346 | ||
|
|
f95a164941 | ||
|
|
0cf419619e | ||
|
|
8b6f6b22d4 | ||
|
|
9cddb5921a |
@@ -5,6 +5,7 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -12,6 +13,8 @@ import (
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
mmac "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
|
||||
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
|
||||
ne "git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
mcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
meshcatalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
@@ -19,6 +22,7 @@ import (
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
)
|
||||
|
||||
// The run of a composition step (design 47 G §354, §63, §64, §718; A-226, A-238, A-240, A-252, A-254, A-261): the step reads one
|
||||
@@ -53,6 +57,18 @@ type compositionRun struct {
|
||||
call func(key string, body any) (json.RawMessage, error)
|
||||
// trunk is the trunk's catalog half as the assigner holds it.
|
||||
trunk func() (mcc.Half, bool)
|
||||
// newest is the newest catalog half the assigner's views hold of each watched branch (halves.views), and watched a module's
|
||||
// watched branches in its registration's order (halves.watchedOf): a machine's composition takes each module's manifest across
|
||||
// them (design 47 H-A-143, G §532). Nil: the step's revision alone.
|
||||
newest func() map[string]mcc.Half
|
||||
watched func(module string) []string
|
||||
// assigned is the modules the assignments place on a machine (world.assignments), for the modules no watched branch lists;
|
||||
// assignment is the machine's assignments whole, which the case composes with those modules left out (H-A-148).
|
||||
assigned func(node string) []string
|
||||
assignment func(node string) (mma.Assignments, bool)
|
||||
// findings is where the step raises the finding assignment-unlisted of each assignment it leaves out (H-A-149); the pass over
|
||||
// the assignments clears it.
|
||||
findings assign.Findings
|
||||
// running is the last `running` heard of a module on a node (world.runningOf), never the node-engine's bucket.
|
||||
running func(node, module string) (ne.Running, bool)
|
||||
// away says the watchman holds machine away (machine-away, not back since): the one presence the assigner keeps (world.awayNow),
|
||||
@@ -209,6 +225,9 @@ func (p *runPrep) get(ctx context.Context) error {
|
||||
p.err = errors.New("the step's catalog revision is not read: " + err.Error())
|
||||
return
|
||||
}
|
||||
if p.r.newest != nil && p.r.watched != nil {
|
||||
half = acrossBranches(p.step, half, p.r.newest(), p.r.watched)
|
||||
}
|
||||
if p.seats, p.err = p.r.seats(ctx, half.Revision.Definitions); p.err != nil {
|
||||
return
|
||||
}
|
||||
@@ -229,6 +248,21 @@ func (r *compositionRun) start(ctx context.Context, key string, run meshwants.Co
|
||||
r.logf("the composition run %s is not ended: %v", key, err)
|
||||
}
|
||||
}
|
||||
// The one set of unlisted assignments, read once at the run's start and carried on the run to the leaving-out, the finding and
|
||||
// every step's judgement, the resume's too (design 47 H-A-152, H-A-153).
|
||||
unlisted := r.unlistedOn(node)
|
||||
if len(unlisted) > 0 {
|
||||
if _, _, err := r.runs.change(key, func(c *meshwants.CompositionRun) bool {
|
||||
if c.Ask != ask || c.Ended != nil {
|
||||
return false
|
||||
}
|
||||
c.Unlisted = unlisted
|
||||
return true
|
||||
}); err != nil {
|
||||
end(node + ": the run's unlisted assignments are not written: " + err.Error())
|
||||
return
|
||||
}
|
||||
}
|
||||
if err := prep.get(ctx); err != nil {
|
||||
end(node + ": " + err.Error())
|
||||
return
|
||||
@@ -243,8 +277,15 @@ func (r *compositionRun) start(ctx context.Context, key string, run meshwants.Co
|
||||
end(node + ": " + err.Error())
|
||||
return
|
||||
}
|
||||
cs := composer.Case{Name: changeCase, Manifests: prep.manifests, None: prep.none}
|
||||
if left, err := r.leftOut(node, unlisted); err != nil {
|
||||
end(node + ": " + err.Error())
|
||||
return
|
||||
} else if left != nil {
|
||||
cs.Assignments = left
|
||||
}
|
||||
req := composer.Request{V: composer.Version, Input: in.Body, Depth: composer.DepthCompose, Machines: []string{node},
|
||||
Cases: []composer.Case{{Name: changeCase, Manifests: prep.manifests, None: prep.none}}, Seats: prep.seats,
|
||||
Cases: []composer.Case{cs}, Seats: prep.seats,
|
||||
BlameTries: r.runs.set.blameTries} // the composer's bound of its search for culprits (§721, A-466r)
|
||||
raw, err := json.Marshal(req)
|
||||
if err != nil {
|
||||
@@ -289,7 +330,29 @@ const notTried = "not tried: the search reached its bound"
|
||||
// could not run where the module was not tried (A-470b); a machine that does not compose and blames no module fails every step on
|
||||
// it, on the node's own account; a verdict with no `blamed` (a composer before §721) is judged as before, with the warning "the
|
||||
// composer names no culprits"; otherwise the machine passes, with a warning for each other module blamed there.
|
||||
//
|
||||
// The run's unlisted are the modules the machine is assigned that no watched branch's newest revision lists, left out of its
|
||||
// composition, as the run's start read them (H-A-152, H-A-153): each is said on every path, in the machine's failed or
|
||||
// could-not-run words where one is set, else as a warning, never the machine's failure and never dropped in silence (H-A-143,
|
||||
// H-A-144, H-A-148).
|
||||
func judgeRun(module string, run meshwants.CompositionRun) machineEnd {
|
||||
end := judgeVerdict(module, run)
|
||||
for _, u := range run.Unlisted {
|
||||
said := run.Machine + " is assigned " + u + ", which no watched branch lists (H-A-143)"
|
||||
switch {
|
||||
case end.failed != "":
|
||||
end.failed += "; " + said
|
||||
case end.couldNot != "":
|
||||
end.couldNot += "; " + said
|
||||
default:
|
||||
end.warnings = append(end.warnings, said)
|
||||
}
|
||||
}
|
||||
return end
|
||||
}
|
||||
|
||||
// judgeVerdict is judgeRun before the unlisted modules are said.
|
||||
func judgeVerdict(module string, run meshwants.CompositionRun) machineEnd {
|
||||
node := run.Machine
|
||||
if run.Ended == nil {
|
||||
return machineEnd{couldNot: node + ": the step stopped before its composition run " + runKeyOf(run.Candidate, node) + " ended"}
|
||||
@@ -503,6 +566,135 @@ func caseOf(half mcc.Half) (map[string]composer.Manifest, map[string]string, err
|
||||
return manifests, none, nil
|
||||
}
|
||||
|
||||
// acrossBranches is the catalog half a machine's composition reads (design 47 H-A-143, H-A-144, G §532): each module's entry from
|
||||
// the newest revision of the first of its watched branches, in its registration's order, that lists it, none for a module with no
|
||||
// watched branches; the candidate's revision replaces its
|
||||
// branch's newest for the modules it moves (its movers and the step's module), and the whole of it where it is that branch's newest
|
||||
// or the branch's is not held. The half keeps the candidate's revision, its definitions with it.
|
||||
func acrossBranches(step meshwants.StepRun, candidate mcc.Half, newest map[string]mcc.Half, watched func(module string) []string) mcc.Half {
|
||||
halves := make(map[string]mcc.Half, len(newest)+1)
|
||||
for b, h := range newest {
|
||||
halves[b] = h
|
||||
}
|
||||
if own, held := newest[step.Branch]; !held || candidate.Revision.Revision >= own.Revision.Revision {
|
||||
halves[step.Branch] = candidate
|
||||
} else {
|
||||
mix := mcc.Half{Revision: own.Revision, Entries: maps.Clone(own.Entries)}
|
||||
moved := []string{step.Module}
|
||||
for _, m := range candidate.Revision.Movers {
|
||||
moved = append(moved, m.Module)
|
||||
}
|
||||
for _, m := range moved {
|
||||
if e, ok := candidate.Entries[m]; ok {
|
||||
mix.Entries[m] = e
|
||||
} else {
|
||||
delete(mix.Entries, m)
|
||||
}
|
||||
}
|
||||
halves[step.Branch] = mix
|
||||
}
|
||||
out := mcc.Half{Revision: candidate.Revision, Entries: map[string]meshcatalog.CatalogEntry{}}
|
||||
for _, h := range halves {
|
||||
for name := range h.Entries {
|
||||
if _, done := out.Entries[name]; done {
|
||||
continue
|
||||
}
|
||||
for _, b := range watched(name) { // none: no watched branch lists it, it is unlisted (H-A-144)
|
||||
if e, ok := halves[b].Entries[name]; ok {
|
||||
out.Entries[name] = e
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// unlistedOn is the one set of unlisted assignments of node (H-A-152): the modules the assignments place there that the newest
|
||||
// revision of none of their watched branches lists, a module with no watched branches among them (H-A-143, H-A-144), by name.
|
||||
// The candidate never enters it, so a module the change drops while still assigned stays in the composition and fails it as the
|
||||
// change's own fault (ADR 0578 §2). Where the newest revisions are not followed yet nothing is unlisted: none while no branch's
|
||||
// newest is held, and not a module one of whose watched branches' newest is not held yet.
|
||||
func (r *compositionRun) unlistedOn(node string) []string {
|
||||
if r.assigned == nil || r.newest == nil || r.watched == nil {
|
||||
return nil
|
||||
}
|
||||
newest := r.newest()
|
||||
if len(newest) == 0 {
|
||||
return nil
|
||||
}
|
||||
var out []string
|
||||
for _, m := range r.assigned(node) {
|
||||
watched := r.watched(m)
|
||||
if !listedOn(newest, watched, m) && !slices.ContainsFunc(watched, func(b string) bool { _, held := newest[b]; return !held }) {
|
||||
out = append(out, m)
|
||||
}
|
||||
}
|
||||
slices.Sort(out)
|
||||
return slices.Compact(out)
|
||||
}
|
||||
|
||||
// leftOut is the case's assignments of node with each module of the run's unlisted set left out, or nil where none is (the case
|
||||
// then composes the input's): a stale assignment is a fault of the assignments, not of the change judged, so it is left out of the
|
||||
// composition, said in the journal, raised as assignment-unlisted and, at the judgement, said as unlisted from the same set
|
||||
// (design 47 H-A-148, H-A-149, H-A-152, H-A-143; ADR 0578 §2).
|
||||
func (r *compositionRun) leftOut(node string, set []string) (map[string]json.RawMessage, error) {
|
||||
if r.assignment == nil || len(set) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
a, ok := r.assignment(node)
|
||||
if !ok {
|
||||
return nil, nil
|
||||
}
|
||||
var unlisted []string
|
||||
for _, m := range a.Modules {
|
||||
if slices.Contains(set, m.Module) {
|
||||
unlisted = append(unlisted, m.Module)
|
||||
}
|
||||
}
|
||||
if len(unlisted) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
a.Modules = slices.DeleteFunc(slices.Clone(a.Modules), func(m mma.PlacedModule) bool { return slices.Contains(unlisted, m.Module) })
|
||||
raw, err := mmac.EncodeAssignments(a)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("its assignments with %s left out cannot be encoded: %w", strings.Join(unlisted, ", "), err)
|
||||
}
|
||||
r.logf("%s: the composition leaves out %s, assigned there and listed on no watched branch (H-A-148)", node, strings.Join(unlisted, ", "))
|
||||
for _, m := range unlisted {
|
||||
r.raiseUnlisted(node, m)
|
||||
}
|
||||
return map[string]json.RawMessage{node: raw}, nil
|
||||
}
|
||||
|
||||
// raiseUnlisted writes the finding assignment-unlisted of module on node (H-A-149); one not written is said in the journal.
|
||||
func (r *compositionRun) raiseUnlisted(node, module string) {
|
||||
if r.findings.Put == nil {
|
||||
return
|
||||
}
|
||||
now := r.now
|
||||
if now == nil {
|
||||
now = time.Now
|
||||
}
|
||||
f, err := assign.UnlistedFinding(node, module, now())
|
||||
if err == nil {
|
||||
err = r.findings.Put(f)
|
||||
}
|
||||
if err != nil {
|
||||
r.logf("the finding %s is not written: %v", assign.UnlistedKey(node, module), err)
|
||||
}
|
||||
}
|
||||
|
||||
// listedOn says whether the newest revision of one of module's watched branches lists it (H-A-143, H-A-144).
|
||||
func listedOn(newest map[string]mcc.Half, watched []string, module string) bool {
|
||||
for _, b := range watched {
|
||||
if _, ok := newest[b].Entries[module]; ok {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// nodeEngineEntry is the entry of the module that claims the seat node-engine, or false.
|
||||
func nodeEngineEntry(entries map[string]meshcatalog.CatalogEntry) (meshcatalog.CatalogEntry, bool) {
|
||||
for _, name := range slices.Sorted(mapsKeys(entries)) {
|
||||
@@ -616,3 +808,32 @@ func runAskPublisher(js jetstream.JetStream) func(ctx context.Context, a buildst
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// newestOf is the newest catalog half the views hold of each watched branch, or nil while the catalog half is not followed.
|
||||
func (iv *catalogViews) newestOf() func() map[string]mcc.Half {
|
||||
if iv == nil {
|
||||
return nil
|
||||
}
|
||||
return func() map[string]mcc.Half {
|
||||
if iv.h == nil {
|
||||
return map[string]mcc.Half{}
|
||||
}
|
||||
return iv.h.views()
|
||||
}
|
||||
}
|
||||
|
||||
// listed says whether one of module's watched branches lists it in the newest revision the views hold (H-A-149's clear).
|
||||
func (iv *catalogViews) listed(module string) bool {
|
||||
if iv == nil || iv.h == nil {
|
||||
return false
|
||||
}
|
||||
return listedOn(iv.h.views(), iv.h.watchedOf(module), module)
|
||||
}
|
||||
|
||||
// watchedOf is a module's watched branches in its registration's order (halves.watchedOf); none while the half is not followed.
|
||||
func (iv *catalogViews) watchedOf(module string) []string {
|
||||
if iv == nil || iv.h == nil {
|
||||
return nil
|
||||
}
|
||||
return iv.h.watchedOf(module)
|
||||
}
|
||||
|
||||
@@ -4,18 +4,24 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"maps"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
mmac "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
|
||||
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
|
||||
ne "git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
|
||||
mcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
meshcatalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -259,6 +265,104 @@ func TestTheCompositionStepRunsOneAskPerMachine(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Another watched branch's app is in every machine's case, and a module assigned there that no watched branch lists is said in
|
||||
// the machine's reason (H-A-143, §532).
|
||||
across := newRunStand(t, "src2:e1", "src2:e1", "d1")
|
||||
across.r.newest = func() map[string]mcc.Half {
|
||||
return map[string]mcc.Half{"master": {Revision: meshcatalog.CatalogRevision{Branch: "master", Revision: 2},
|
||||
Entries: map[string]meshcatalog.CatalogEntry{"f1-circuits": {Module: "f1-circuits", Fp: "f1", Manifest: map[string]any{"module": "f1-circuits"}}}}}
|
||||
}
|
||||
across.r.watched = func(module string) []string {
|
||||
if module == "checker" || module == "lidarr" { // no registration names them; lidarr is on main's revision all the same (H-A-144)
|
||||
return nil
|
||||
}
|
||||
return []string{"main", "master"}
|
||||
}
|
||||
across.r.assigned = func(node string) []string { return []string{"m", "f1-circuits", "checker", "lidarr"} }
|
||||
if out := across.r.run(context.Background(), "ask-step", step); out.Outcome != meshwants.StepEndedOutcomeFailed ||
|
||||
!strings.Contains(out.Said, "g14 is assigned checker, which no watched branch lists (H-A-143)") ||
|
||||
!strings.Contains(out.Said, "g14 is assigned lidarr, which no watched branch lists (H-A-143)") {
|
||||
t.Errorf("the step across the watched branches came to %+v", out)
|
||||
}
|
||||
if c := across.requests["ace"].Cases[0]; string(c.Manifests["f1-circuits"].Raw) != `{"module":"f1-circuits"}` || len(c.Manifests) != 4 ||
|
||||
c.Manifests["lidarr"].Raw != nil {
|
||||
t.Errorf("the case across the watched branches holds %v", slices.Sorted(maps.Keys(c.Manifests)))
|
||||
}
|
||||
if c := across.requests["ace"].Cases[0]; c.Assignments != nil {
|
||||
t.Errorf("with no assignments held the case composes the input's, it carries %v", c.Assignments)
|
||||
}
|
||||
|
||||
// A stale assignment is left out of the case's assignments as well, said, and the machine composes; a listed module stays in
|
||||
// (H-A-148, ADR 0578 §2). The one set is the watched branches' newest revisions', carried on the run (H-A-152, H-A-153): a module
|
||||
// the candidate drops while the newest still lists it stays in, the change's own fault, and while no newest revision is followed
|
||||
// nothing is left out.
|
||||
dropsWeb := func() map[string]mcc.Half {
|
||||
h := across.r.newest()
|
||||
h["main"] = mcc.Half{Revision: meshcatalog.CatalogRevision{Branch: "main", Revision: 6},
|
||||
Entries: map[string]meshcatalog.CatalogEntry{"m": {Module: "m", Manifest: map[string]any{"module": "m"}},
|
||||
"web": {Module: "web", Fp: "w1", Manifest: map[string]any{"module": "web"}}}}
|
||||
return h
|
||||
}
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
assigned []string
|
||||
newest func() map[string]mcc.Half // nil: across's
|
||||
want []string // the case's modules of ace; nil: the input's assignments
|
||||
}{
|
||||
{"a stale assignment left out and said, the machine composes", []string{"m", "f1-circuits", "checker"}, nil, []string{"m", "f1-circuits"}},
|
||||
{"listed modules alone: the input's assignments", []string{"m", "f1-circuits"}, nil, nil},
|
||||
{"a module the candidate drops, the newest lists: kept", []string{"m", "f1-circuits", "web"}, dropsWeb, nil},
|
||||
{"no newest revision followed: nothing left out", []string{"m", "checker"}, func() map[string]mcc.Half { return map[string]mcc.Half{} }, nil},
|
||||
} {
|
||||
left := newRunStand(t, "src2:e1", "src2:e1", "d1")
|
||||
left.r.newest, left.r.watched = across.r.newest, across.r.watched
|
||||
if tc.newest != nil {
|
||||
left.r.newest = tc.newest
|
||||
}
|
||||
left.r.assigned = func(string) []string { return tc.assigned }
|
||||
left.r.assignment = func(node string) (mma.Assignments, bool) {
|
||||
a := mma.Assignments{Node: node}
|
||||
for _, m := range tc.assigned {
|
||||
a.Modules = append(a.Modules, mma.PlacedModule{Module: m})
|
||||
}
|
||||
return a, true
|
||||
}
|
||||
left.verdict = func(string) (composer.Machine, bool) { return composer.Machine{Composes: true}, true }
|
||||
raised := map[string][]byte{}
|
||||
left.r.findings = assign.Findings{Put: putInto(raised)}
|
||||
out := left.r.run(context.Background(), "ask-step", step)
|
||||
// H-A-149: the step that leaves an assignment out raises assignment-unlisted, scope delivery, id <node>.<module>, a warning.
|
||||
var f mcseat.Finding
|
||||
_ = json.Unmarshal(raised[assign.UnlistedKey("ace", "checker")], &f)
|
||||
if want := tc.want != nil; want != (f.Kind == assign.KindAssignmentUnlisted && f.Scope == "delivery" && f.ID == "ace.checker" &&
|
||||
f.Severity == mcseat.FindingSeverityWarning) || (!want && len(raised) != 0) {
|
||||
t.Errorf("%s: raised %v", tc.name, raised)
|
||||
}
|
||||
said := "ace is assigned checker, which no watched branch lists (H-A-143)"
|
||||
if out.Outcome != meshwants.StepEndedOutcomePassed || (tc.want != nil) != slices.Contains(out.Warnings, said) {
|
||||
t.Errorf("%s: the step came to %+v", tc.name, out)
|
||||
}
|
||||
if run, _, _, _ := left.runs.read(runKeyOf("main.7", "ace")); (tc.want != nil) != slices.Equal(run.Unlisted, []string{"checker"}) ||
|
||||
(tc.want == nil && run.Unlisted != nil) {
|
||||
t.Errorf("%s: the run carries the unlisted %v", tc.name, run.Unlisted)
|
||||
}
|
||||
raw, ok := left.requests["ace"].Cases[0].Assignments["ace"]
|
||||
if tc.want == nil {
|
||||
if ok {
|
||||
t.Errorf("%s: the case carries assignments %s", tc.name, raw)
|
||||
}
|
||||
continue
|
||||
}
|
||||
a, err := mmac.DecodeAssignments(raw)
|
||||
var got []string
|
||||
for _, m := range a.Modules {
|
||||
got = append(got, m.Module)
|
||||
}
|
||||
if err != nil || !slices.Equal(got, tc.want) {
|
||||
t.Errorf("%s: the case composes ace with %v (%v), want %v", tc.name, got, err, tc.want)
|
||||
}
|
||||
}
|
||||
|
||||
// At the inputs state's cap the ask is refused, naming the store (§354).
|
||||
full := newRunStand(t, "src2:e1", "src2:e1", "d1")
|
||||
full.r.keeper.used = func() (int64, error) { return inputsCap, nil }
|
||||
@@ -438,21 +542,35 @@ func TestAMachineIsJudgedByTheModulesItBlames(t *testing.T) {
|
||||
run meshwants.CompositionRun
|
||||
failed, couldNo string
|
||||
warnings []string
|
||||
unlisted []string
|
||||
}{
|
||||
{"blamed module fails", "a", runOf(blamesA), "ace holds a: /etc/x is a's and c's", "", nil},
|
||||
{"another module passes with a warning", "b", runOf(blamesA), "", "", []string{"ace holds a: /etc/x is a's and c's"}},
|
||||
{"blames no module fails both", "b", runOf(`{"composes":false,"reason":"the node's own composition","blamed":[]}`), "ace does not compose and blames no module: the node's own composition", "", nil},
|
||||
{"an old composer", "b", runOf(`{"composes":false,"reason":"x"}`), "ace does not compose: x", "", []string{"ace: the composer names no culprits"}},
|
||||
{"composes", "b", runOf(`{"composes":true,"blamed":[]}`), "", "", nil},
|
||||
{"blamed module fails", "a", runOf(blamesA), "ace holds a: /etc/x is a's and c's", "", nil, nil},
|
||||
{"another module passes with a warning", "b", runOf(blamesA), "", "", []string{"ace holds a: /etc/x is a's and c's"}, nil},
|
||||
{"blames no module fails both", "b", runOf(`{"composes":false,"reason":"the node's own composition","blamed":[]}`), "ace does not compose and blames no module: the node's own composition", "", nil, nil},
|
||||
{"an old composer", "b", runOf(`{"composes":false,"reason":"x"}`), "ace does not compose: x", "", []string{"ace: the composer names no culprits"}, nil},
|
||||
{"composes", "b", runOf(`{"composes":true,"blamed":[]}`), "", "", nil, nil},
|
||||
{"own module not tried", "a", runOf(`{"composes":false,"blamed":[{"module":"a","problem":"not tried: the search reached its bound"}]}`), "",
|
||||
"ace holds a untried: the composer's search for culprits reached its bound of blame_tries (A-470b)", nil},
|
||||
"ace holds a untried: the composer's search for culprits reached its bound of blame_tries (A-470b)", nil, nil},
|
||||
{"another not tried", "b", runOf(`{"composes":false,"blamed":[{"module":"a","problem":"not tried: the search reached its bound"}]}`), "", "",
|
||||
[]string{"ace holds a: not tried: the search reached its bound"}},
|
||||
{"not judged", "a", runOf(`{"composes":false,"not_judged":"no seat list","blamed":[]}`), "", "ace was not judged: no seat list", nil},
|
||||
{"could not run", "a", meshwants.CompositionRun{Machine: "ace", Ended: &at, Outcome: meshwants.CompositionRunOutcomeCouldNotRun, Said: "ace: gone"}, "", "ace: gone", nil},
|
||||
[]string{"ace holds a: not tried: the search reached its bound"}, nil},
|
||||
{"not judged", "a", runOf(`{"composes":false,"not_judged":"no seat list","blamed":[]}`), "", "ace was not judged: no seat list", nil, nil},
|
||||
{"could not run", "a", meshwants.CompositionRun{Machine: "ace", Ended: &at, Outcome: meshwants.CompositionRunOutcomeCouldNotRun, Said: "ace: gone"}, "", "ace: gone", nil, nil},
|
||||
// H-A-143: a module assigned there that no watched branch lists is said in the reason, or as a warning where it composes.
|
||||
{"an unlisted module in the reason", "b", runOf(`{"composes":false,"reason":"nothing provides \"photos\"","blamed":[]}`),
|
||||
"ace does not compose and blames no module: nothing provides \"photos\"; ace is assigned photos, which no watched branch lists (H-A-143)", "", nil, []string{"photos"}},
|
||||
{"does not compose, blames a module, an unlisted module", "a", runOf(`{"composes":false,"reason":"nothing provides \"photos\"","blamed":[{"module":"a","problem":"nothing provides \"photos\""}]}`),
|
||||
"ace holds a: nothing provides \"photos\"; ace is assigned photos, which no watched branch lists (H-A-143)", "", nil, []string{"photos"}},
|
||||
{"another module blamed, an unlisted module", "b", runOf(`{"composes":false,"blamed":[{"module":"a","problem":"x"}]}`), "", "",
|
||||
[]string{"ace holds a: x", "ace is assigned photos, which no watched branch lists (H-A-143)"}, []string{"photos"}},
|
||||
{"could not run, an unlisted module", "a", meshwants.CompositionRun{Machine: "ace", Ended: &at, Outcome: meshwants.CompositionRunOutcomeCouldNotRun, Said: "ace: gone"},
|
||||
"", "ace: gone; ace is assigned photos, which no watched branch lists (H-A-143)", nil, []string{"photos"}},
|
||||
{"an unlisted module where it composes", "b", runOf(`{"composes":true,"blamed":[]}`), "", "",
|
||||
[]string{"ace is assigned photos, which no watched branch lists (H-A-143)"}, []string{"photos"}},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
e := judgeRun(tc.module, tc.run)
|
||||
run := tc.run
|
||||
run.Unlisted = tc.unlisted // the set the run's start wrote (H-A-153)
|
||||
e := judgeRun(tc.module, run)
|
||||
if e.failed != tc.failed || e.couldNot != tc.couldNo || !slices.Equal(e.warnings, tc.warnings) {
|
||||
t.Errorf("judged %+v, want failed %q, could not run %q, warnings %q", e, tc.failed, tc.couldNo, tc.warnings)
|
||||
}
|
||||
@@ -572,3 +690,106 @@ func TestARerunMovesEveryReaderThatDidNotPass(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 H-A-143, G §532: a machine's composition takes each module's entry from the newest revision of the first of its
|
||||
// watched branches, in its registration's order, that lists it; the candidate's revision replaces its branch's newest for the
|
||||
// modules it moves, and whole where it is that branch's newest; the half keeps the candidate's revision.
|
||||
func TestACompositionTakesEachModuleAcrossTheWatchedBranches(t *testing.T) {
|
||||
entry := func(module, fp string) meshcatalog.CatalogEntry {
|
||||
return meshcatalog.CatalogEntry{Module: module, Fp: fp, Manifest: map[string]any{"module": module}}
|
||||
}
|
||||
half := func(branch string, revision int64, movers []string, entries ...meshcatalog.CatalogEntry) mcc.Half {
|
||||
h := mcc.Half{Revision: meshcatalog.CatalogRevision{Branch: branch, Revision: revision, Definitions: []meshcatalog.CatalogRevisionDefinitionsItem{{Seat: "s1", Digest: branch}}},
|
||||
Entries: map[string]meshcatalog.CatalogEntry{}}
|
||||
for _, m := range movers {
|
||||
h.Revision.Movers = append(h.Revision.Movers, meshcatalog.TakeInEntry{Module: m})
|
||||
}
|
||||
for _, e := range entries {
|
||||
h.Entries[e.Module] = e
|
||||
}
|
||||
return h
|
||||
}
|
||||
newest := map[string]mcc.Half{
|
||||
"main": half("main", 4, nil, entry("engine", "e1"), entry("a", "a1"), entry("both", "main")),
|
||||
"master": half("master", 9, nil, entry("f1-circuits", "f1")),
|
||||
"nox-mesh": half("nox-mesh", 3, nil, entry("link2pay", "l1"), entry("photos", "p1"), entry("both", "nox")),
|
||||
}
|
||||
registrations := map[string][]string{"both": {"nox-mesh", "main"}, "f1-circuits": {"master"}, "link2pay": {"nox-mesh"}, "photos": {"nox-mesh"}}
|
||||
registrations["engine"], registrations["a"] = []string{"main"}, []string{"main"}
|
||||
watched := func(module string) []string { return registrations[module] } // a module no registration names has none (H-A-144)
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
step meshwants.StepRun
|
||||
candidate mcc.Half
|
||||
newest map[string]mcc.Half
|
||||
want map[string]string // module -> fingerprint
|
||||
}{
|
||||
{"the newest of main, the others' apps beside it", meshwants.StepRun{Branch: "main", Module: "a", Revision: 5},
|
||||
half("main", 5, []string{"a"}, entry("engine", "e1"), entry("a", "a2"), entry("both", "main")), newest,
|
||||
map[string]string{"engine": "e1", "a": "a2", "both": "nox", "f1-circuits": "f1", "link2pay": "l1", "photos": "p1"}},
|
||||
{"an older candidate moves only its movers", meshwants.StepRun{Branch: "nox-mesh", Module: "link2pay", Revision: 2},
|
||||
half("nox-mesh", 2, []string{"photos"}, entry("link2pay", "l0"), entry("both", "nox0")), newest,
|
||||
map[string]string{"engine": "e1", "a": "a1", "both": "nox", "f1-circuits": "f1", "link2pay": "l0"}},
|
||||
{"a branch not yet held is the candidate's", meshwants.StepRun{Branch: "master", Module: "f1-circuits", Revision: 1},
|
||||
half("master", 1, nil, entry("f1-circuits", "f0")), map[string]mcc.Half{"main": newest["main"]},
|
||||
map[string]string{"engine": "e1", "a": "a1", "both": "main", "f1-circuits": "f0"}},
|
||||
{"no view held yet: the candidate alone", meshwants.StepRun{Branch: "main", Module: "a", Revision: 5},
|
||||
half("main", 5, nil, entry("a", "a2")), nil, map[string]string{"a": "a2"}},
|
||||
{"a module no registration names is not composed", meshwants.StepRun{Branch: "main", Module: "a", Revision: 5},
|
||||
half("main", 5, nil, entry("a", "a2"), entry("checker", "c1")), nil, map[string]string{"a": "a2"}},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got := acrossBranches(tc.step, tc.candidate, tc.newest, watched)
|
||||
fps := map[string]string{}
|
||||
for name, e := range got.Entries {
|
||||
fps[name] = e.Fp
|
||||
}
|
||||
if !maps.Equal(fps, tc.want) {
|
||||
t.Errorf("composed from %v, want %v", fps, tc.want)
|
||||
}
|
||||
if got.Revision.Revision != tc.candidate.Revision.Revision || len(got.Revision.Definitions) != 1 || got.Revision.Definitions[0].Digest != tc.step.Branch {
|
||||
t.Errorf("the half keeps revision %d with definitions %v, want the candidate's %d with its own", got.Revision.Revision, got.Revision.Definitions,
|
||||
tc.candidate.Revision.Revision)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 H-A-149: the pass over the assignments clears assignment-unlisted when the assignment is gone or one of the module's
|
||||
// watched branches lists it again, and keeps it while it stands; another finding is left alone.
|
||||
func TestThePassClearsAnUnlistedAssignmentsFinding(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
assigned []string // novox's modules
|
||||
listed bool
|
||||
kept bool
|
||||
}{
|
||||
{"still assigned and listed nowhere: kept", []string{"checker"}, false, true},
|
||||
{"the assignment gone: cleared", []string{"web"}, false, false},
|
||||
{"listed again: cleared", []string{"checker"}, true, false},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
w := newWorld()
|
||||
a := mma.Assignments{V: keys.Version, Node: "novox"}
|
||||
for _, m := range tc.assigned {
|
||||
a.Modules = append(a.Modules, mma.PlacedModule{Module: m})
|
||||
}
|
||||
putAssignments(w, a)
|
||||
findings := map[string][]byte{}
|
||||
f, _ := assign.UnlistedFinding("novox", "checker", time.Now())
|
||||
_ = putInto(findings)(f)
|
||||
other, _ := assign.UnplacedFinding("g14", time.Now())
|
||||
_ = putInto(findings)(other)
|
||||
p := &passer{w: w, logf: t.Logf, listed: func(string) bool { return tc.listed }, findings: assign.Findings{Put: putInto(findings),
|
||||
Delete: func(k string) error { delete(findings, k); return nil },
|
||||
Keys: func() ([]string, error) { return slices.Collect(maps.Keys(findings)), nil }}}
|
||||
p.clearUnlisted()
|
||||
if _, ok := findings[assign.UnlistedKey("novox", "checker")]; ok != tc.kept {
|
||||
t.Errorf("the finding kept %v, want %v", ok, tc.kept)
|
||||
}
|
||||
if _, ok := findings[assign.UnplacedKey("g14")]; !ok {
|
||||
t.Error("another finding is cleared")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -818,7 +818,7 @@ func (r *compositionRuns) leave(pipeline string, stop func(ids []string)) ([]str
|
||||
|
||||
// passedOn is whether the step of module passed on the machine of an ended run (§722): neither failed nor could not run there.
|
||||
func passedOn(module string, run meshwants.CompositionRun) bool {
|
||||
e := judgeRun(module, run)
|
||||
e := judgeVerdict(module, run) // the unlisted modules change only the words, never whether it passed
|
||||
return e.failed == "" && e.couldNot == ""
|
||||
}
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -13,6 +14,7 @@ import (
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/graph"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
)
|
||||
|
||||
@@ -191,3 +193,25 @@ func (h *halves) views() map[string]mcc.Half {
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// watchedOf is module's watched branches in its registration's order (the branch `register` named first, then each branch in the
|
||||
// order `watch` added it). Once registrations are held, a module none names has none: it is unlisted (H-A-144). Only before the
|
||||
// first registrations are held do the gate's source's branches stand in (the branches followed then), in the order the source
|
||||
// gives them, by their names escaped as its keys are (design 47 G §532, H-A-143, H-A-144).
|
||||
func (h *halves) watchedOf(module string) []string {
|
||||
h.mu.Lock()
|
||||
b := h.branches
|
||||
h.mu.Unlock()
|
||||
if b == nil {
|
||||
return nil
|
||||
}
|
||||
if b.Registrations != nil && len(b.Registrations.All()) > 0 {
|
||||
if reg, ok := b.Registrations.Registration(module); ok {
|
||||
return slices.Clone(reg.Branches)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
out := b.Followed()
|
||||
slices.SortStableFunc(out, func(x, y string) int { return strings.Compare(keys.EscapeName(x), keys.EscapeName(y)) })
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -3,6 +3,8 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"slices"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -48,3 +50,45 @@ func TestAMeshThatNamesNoBranchSettlesTheAssignersFirstPass(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 H-A-143, H-A-144, G §532: a module's watched branches are its registration's, in its order; once registrations are held a
|
||||
// module none names has none; before the first registrations the gate's source's branches stand in, in the source's order.
|
||||
func TestAModulesWatchedBranchesAreItsRegistrations(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer cancel()
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
regs []seat.Registration
|
||||
module string
|
||||
want []string
|
||||
}{
|
||||
{"its registration's order", []seat.Registration{{Module: "link2pay", Branches: []string{"nox-mesh", "main"}}}, "link2pay", []string{"nox-mesh", "main"}},
|
||||
{"no registration names it", []seat.Registration{{Module: "link2pay", Branches: []string{"nox-mesh", "main"}}}, "checker", nil},
|
||||
{"before the first registrations, the source's", []seat.Registration{}, "checker", []string{"main", "master"}},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
call := func(key string, _ any) (json.RawMessage, error) {
|
||||
switch key {
|
||||
case mcc.RegistrationsVerb:
|
||||
return json.Marshal(seat.RegistrationsAnswer{Registrations: tc.regs})
|
||||
case mcc.SourceVerb:
|
||||
return json.Marshal(map[string][]string{"branches": {"master", "main"}})
|
||||
}
|
||||
return nil, errors.New("not answered here")
|
||||
}
|
||||
reader := &views.ProcessReader{Module: "mesh-delivery-assigner", Call: call, Logf: t.Logf}
|
||||
h := &halves{w: newWorld(), call: call, reader: reader, wake: func() {}, logf: t.Logf, ctx: ctx}
|
||||
if err := h.follow(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, f := range reader.Followers()[:2] { // the registrations and the gate's source
|
||||
if err := f.Start(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if got := h.watchedOf(tc.module); !slices.Equal(got, tc.want) {
|
||||
t.Errorf("the watched branches of %s are %v, want %v", tc.module, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1769,6 +1769,11 @@ func serveCatalog(t *testing.T, b *bed, nc *nats.Conn) {
|
||||
}
|
||||
if kept, ok := b.cat.at[branch][in.Revision]; ok {
|
||||
rev := meshcatalog.CatalogRevision{Kind: "trunk", Branch: in.Branch, Revision: in.Revision}
|
||||
for k := range b.cat.at[branch] { // its base, the newest kept below it, as the take-in names it (H-A-154)
|
||||
if k < in.Revision && k > rev.Base {
|
||||
rev.Base = k
|
||||
}
|
||||
}
|
||||
for _, e := range kept {
|
||||
rev.Entries = append(rev.Entries, meshcatalog.EntryRef{Module: e.Module, Digest: e.ManifestDigest})
|
||||
}
|
||||
|
||||
@@ -581,7 +581,7 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
|
||||
}
|
||||
}
|
||||
p := &passer{kvs: kvs, w: w, logf: logger.Printf, say: say, findings: findings, said: map[string]bool{}, answeredAt: map[string]uint64{},
|
||||
reads: reads, seats: gate, askForFile: getenv("MESH_ASSIGNER_ASK_OPERATOR_FOR_FILE")}
|
||||
reads: reads, seats: gate, askForFile: getenv("MESH_ASSIGNER_ASK_OPERATOR_FOR_FILE"), listed: iv.listed}
|
||||
p.askOperatorFor() // the setting ask-operator-for said once at the start (design 47 G A-425a)
|
||||
// The setting tier-bound (design 47 H §751) and the one timer that wakes the pass at a tier's deadline (H-A-56).
|
||||
tierBound, err := tierBoundSetting(getenv)
|
||||
@@ -665,7 +665,7 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
|
||||
// The composition step (design 47 G §22, §354): the seat mesh-wants' accepted kind step, run by one run ask per machine.
|
||||
if bus != nil && js != nil && inputs != nil && p.publishWants != nil && runs != nil {
|
||||
run := &compositionRun{runs: runs, inputs: &compositionInputs{machines: w.machinesOf, callFile: bus.CallFile}, keeper: inputs, reads: &compositionReads{}, call: bus.CallKept,
|
||||
trunk: trunkOf(iv, w.trunkName), running: w.runningOf, away: func(machine string) bool { return w.awayNow()[machine] }, put: runAskPublisher(js), ask: bus.CallFile, now: time.Now, logf: logger.Printf}
|
||||
trunk: trunkOf(iv, w.trunkName), newest: iv.newestOf(), watched: iv.watchedOf, assigned: w.assignedOn, assignment: w.assignmentOf, findings: findings, running: w.runningOf, away: func(machine string) bool { return w.awayNow()[machine] }, put: runAskPublisher(js), ask: bus.CallFile, now: time.Now, logf: logger.Printf}
|
||||
steps := &compositionSteps{node: thisNode, say: p.publishWants, run: run.run, resume: run.resume,
|
||||
validate: run.validate, resumeValidate: run.resumeValidate, ended: runs.stepEnded, now: time.Now, logf: logger.Printf}
|
||||
move := steps.move(ctx) // the readers a rerun carried (§727, A-470d)
|
||||
|
||||
@@ -56,6 +56,8 @@ type passer struct {
|
||||
findings assign.Findings
|
||||
// unplaced are the nodes whose unplaced finding this process wrote.
|
||||
unplaced map[string]bool
|
||||
// listed says whether a module's watched branches list it (catalogViews.listed), for the clear of assignment-unlisted.
|
||||
listed func(module string) bool
|
||||
// answeredAt is, by the id of a backup ask, how many values the have had when its answer was written into the want.
|
||||
answeredAt map[string]uint64
|
||||
// now is the clock; nil is time.Now.
|
||||
@@ -208,6 +210,7 @@ func (p *passer) wants(ctx context.Context) {
|
||||
}))
|
||||
chosen, _ := assign.DecodePolicies(p.w.read(assign.PoliciesBucket()))
|
||||
defer p.unplacedFindings()
|
||||
defer p.clearUnlisted()
|
||||
defer func() { p.seatHolders(ctx, viewOf(p.w.trunkName()).manifests) }()
|
||||
p.clearStaleCalls()
|
||||
placed := p.w.placedNodes()
|
||||
@@ -655,6 +658,33 @@ func (p *passer) unplacedFindings() {
|
||||
}
|
||||
}
|
||||
|
||||
// clearUnlisted clears, in the pass over the assignments, each finding assignment-unlisted whose assignment is gone or whose module
|
||||
// one of its watched branches lists again, so the clear never waits for the next composition step on that machine (H-A-149).
|
||||
func (p *passer) clearUnlisted() {
|
||||
if p.findings.Keys == nil || p.findings.Delete == nil || p.listed == nil {
|
||||
return
|
||||
}
|
||||
keys, err := p.findings.Keys()
|
||||
if err != nil {
|
||||
p.logf("the findings assignment-unlisted are not read, so none is cleared in this pass: %v", err)
|
||||
return
|
||||
}
|
||||
assignments := p.w.assignments()
|
||||
for _, key := range keys {
|
||||
node, module, ok := assign.UnlistedOfKey(key)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
assigned := slices.ContainsFunc(assignments[node].Modules, func(m mma.PlacedModule) bool { return m.Module == module })
|
||||
if assigned && !p.listed(module) {
|
||||
continue
|
||||
}
|
||||
if err := p.findings.Delete(key); err != nil {
|
||||
p.logf("the finding %s is not cleared: %v", key, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// seatHolders writes the seat mesh-module-assigner's state seat-holders from every node's assignments (ADR 0571 §4): one key per
|
||||
// seat held once, `<seat>`, and one per node of a node seat, `<seat>.<node>`, written only where it differs, and a key no
|
||||
// assignments holds any more deleted. A pass follows each assignments change and each manifest the indexer takes in, so no
|
||||
|
||||
@@ -22,7 +22,7 @@ import (
|
||||
// `<pipeline>.validate` of the state composition-runs in the CompositionRun form with no machine, which every mover's step
|
||||
// reads: the cycle the revision's entries make (graph.RefusedCycle) ends it at once; else the manifest validator built from
|
||||
// the revision's own entry of the module holding the seat mesh-catalog, on the Go build seat as the composer is, runs in the
|
||||
// sandbox over the revision's manifests and the branch's previous revision's, its input kept and served on mesh-wants' input.
|
||||
// sandbox over the revision's manifests and its base's (H-A-154), its input kept and served on mesh-wants' input.
|
||||
// Every mover's step ends the same: failed `cycle` naming the cycle, failed `manifest-refused` naming every problem of a
|
||||
// manifest the revision alters or between manifests one it alters takes part in, or passed.
|
||||
|
||||
@@ -32,7 +32,7 @@ const (
|
||||
validatorPackage = "mesh-delivery-indexer/cmd/mesh-judge"
|
||||
// validateKeySuffix ends the key of a revision's validate run, `<pipeline>.validate` (H-A-108).
|
||||
validateKeySuffix = "." + meshwants.StepRunStepValidate
|
||||
// beforeCase is the validator's case of the branch's previous revision, beside changeCase (H-A-108 (3)).
|
||||
// beforeCase is the validator's case of the revision's base, beside changeCase (H-A-108 (3), H-A-151, H-A-154).
|
||||
beforeCase = "before"
|
||||
// The reasons a validate step fails with, the first word of its said (§745).
|
||||
reasonManifestRefused = "manifest-refused"
|
||||
@@ -112,16 +112,29 @@ func (r *compositionRun) judgeValidate(ctx context.Context, step meshwants.StepR
|
||||
if err != nil {
|
||||
return couldNotRun("the step's catalog revision is not read: " + err.Error())
|
||||
}
|
||||
before, err := r.before(ctx, step)
|
||||
before, err := r.before(ctx, step, half)
|
||||
if err != nil {
|
||||
return couldNotRun("the branch's previous catalog revision is not read: " + err.Error())
|
||||
return couldNotRun("the base of the case before is not read: " + err.Error())
|
||||
}
|
||||
// A revision that names no base (the branch's first take-in) is judged against an empty case before, and says so (H-A-151).
|
||||
empty := ""
|
||||
if half.Revision.Base <= 0 {
|
||||
empty = emptyBeforeWords
|
||||
}
|
||||
if problems := causedProblems(v.Change, v.Before, altered(half, before)); len(problems) > 0 {
|
||||
return stepOutcome{Outcome: meshwants.StepEndedOutcomeFailed, Said: reasonManifestRefused + ": " + strings.Join(problems, "; ")}
|
||||
said := reasonManifestRefused + ": " + strings.Join(problems, "; ")
|
||||
if empty != "" {
|
||||
said += " (" + empty + ")"
|
||||
}
|
||||
return stepOutcome{Outcome: meshwants.StepEndedOutcomeFailed, Said: said}
|
||||
}
|
||||
return stepOutcome{Outcome: meshwants.StepEndedOutcomePassed}
|
||||
return stepOutcome{Outcome: meshwants.StepEndedOutcomePassed, Said: empty}
|
||||
}
|
||||
|
||||
// emptyBeforeWords say, in a validate step's end, that its revision names no base: the branch's first take-in, judged against
|
||||
// an empty case before (H-A-151, H-A-154).
|
||||
const emptyBeforeWords = "the case before is empty: the revision names no base, the branch's first take-in"
|
||||
|
||||
// validateVerdict is a validate run's verdict as the run keeps it: the cycle the revision makes, or the validator's two cases.
|
||||
type validateVerdict struct {
|
||||
Cycle []meshcatalog.ArtifactRef `json:"cycle,omitempty"`
|
||||
@@ -148,15 +161,16 @@ func cycleWords(cycle []meshcatalog.ArtifactRef) string {
|
||||
return strings.Join(words, " → ")
|
||||
}
|
||||
|
||||
// before is the branch's catalog revision before the step's, read through the seat mesh-catalog's verb revision (H-A-108 (3));
|
||||
// none before the first.
|
||||
func (r *compositionRun) before(ctx context.Context, step meshwants.StepRun) (mcc.Half, error) {
|
||||
if step.Revision <= 1 {
|
||||
// before is the case before of the step's revision half: the revision its base names, the newest kept revision below it that
|
||||
// its take-in pinned, read through the seat mesh-catalog's verb revision (H-A-108 (3), H-A-151, H-A-154); empty where the
|
||||
// revision names no base, the branch's first take-in.
|
||||
func (r *compositionRun) before(ctx context.Context, step meshwants.StepRun, half mcc.Half) (mcc.Half, error) {
|
||||
if half.Revision.Base <= 0 {
|
||||
return mcc.Half{Entries: map[string]meshcatalog.CatalogEntry{}}, nil
|
||||
}
|
||||
prev := step
|
||||
prev.Revision--
|
||||
return r.revision(ctx, prev)
|
||||
base := step
|
||||
base.Revision = half.Revision.Base
|
||||
return r.revision(ctx, base)
|
||||
}
|
||||
|
||||
// altered are the modules whose manifest the revision adds or changes against the one before it.
|
||||
@@ -270,9 +284,9 @@ func (r *compositionRun) startValidate(ctx context.Context, key string, run mesh
|
||||
end(meshwants.CompositionRunOutcomePassed, "", verdict)
|
||||
return
|
||||
}
|
||||
before, err := r.before(ctx, step)
|
||||
before, err := r.before(ctx, step, half)
|
||||
if err != nil {
|
||||
couldNot("the branch's previous catalog revision is not read: " + err.Error())
|
||||
couldNot("the base of the case before is not read: " + err.Error())
|
||||
return
|
||||
}
|
||||
entry, ok := catalogEntry(half.Entries)
|
||||
|
||||
@@ -20,7 +20,9 @@ import (
|
||||
// run, `<pipeline>.validate` of the state composition-runs; a manifest the revision alters that names an unknown field fails
|
||||
// it `manifest-refused` naming the problem, a problem of a manifest it leaves as it was does not, and the validator is the
|
||||
// revision's own entry of the module holding the seat mesh-catalog, on the Go build seat; a revision whose entries make a cycle
|
||||
// fails it `cycle` naming the cycle, with no validator asked.
|
||||
// fails it `cycle` naming the cycle, with no validator asked. The case before is the revision its base names (H-A-151, H-A-154:
|
||||
// the newest kept below it, which the stand-in names as the indexer does); a revision naming no base, the branch's first take-in,
|
||||
// is judged against an empty case before, said in its end, and no revision below it is asked.
|
||||
func TestTheValidateStepRefusesAManifestProblemAndACycle(t *testing.T) {
|
||||
indexer := meshcatalog.CatalogEntry{Module: "indexer", Repository: "novox/mesh-delivery", Commit: commitE3, Path: "modules/mesh-delivery-indexer",
|
||||
Fp: "src2:ix", Serves: []string{"seat:" + meshcatalog.Seat + ".revision"}, Manifest: map[string]any{"module": "indexer"}, ManifestDigest: "d-ix"}
|
||||
@@ -28,9 +30,24 @@ func TestTheValidateStepRefusesAManifestProblemAndACycle(t *testing.T) {
|
||||
return meshcatalog.CatalogEntry{Module: module, Manifest: map[string]any{"module": module, "d": digest}, ManifestDigest: digest, Edges: edges}
|
||||
}
|
||||
revisions := map[int64][]meshcatalog.CatalogEntry{
|
||||
6: {indexer, entry("web", "d-web-1"), entry("db", "d-db")},
|
||||
7: {indexer, entry("web", "d-web-2"), entry("db", "d-db")},
|
||||
9: {indexer, entry("a", "d-a", meshcatalog.Edge{From: "a:x", To: "b:y", Kind: 3}, meshcatalog.Edge{From: "b:y", To: "a:x", Kind: 3}), entry("b", "d-b")},
|
||||
6: {indexer, entry("web", "d-web-1"), entry("db", "d-db")},
|
||||
7: {indexer, entry("web", "d-web-2"), entry("db", "d-db")},
|
||||
9: {indexer, entry("a", "d-a", meshcatalog.Edge{From: "a:x", To: "b:y", Kind: 3}, meshcatalog.Edge{From: "b:y", To: "a:x", Kind: 3}), entry("b", "d-b")},
|
||||
3: {indexer},
|
||||
12: {indexer, entry("web", "d-web-2"), entry("db", "d-db")},
|
||||
}
|
||||
// base is the newest kept revision below n, as the take-in names it; none below the first. Revision 12 names 7 with 8 to 11
|
||||
// never kept, so a case before read as the number minus one could not run (H-A-154).
|
||||
base := func(n int64) (b int64) {
|
||||
if n == 12 {
|
||||
return 7
|
||||
}
|
||||
for k := range revisions {
|
||||
if k < n && k > b {
|
||||
b = k
|
||||
}
|
||||
}
|
||||
return b
|
||||
}
|
||||
call := func(key string, body any) (json.RawMessage, error) {
|
||||
raw, _ := json.Marshal(body)
|
||||
@@ -42,7 +59,7 @@ func TestTheValidateStepRefusesAManifestProblemAndACycle(t *testing.T) {
|
||||
if !ok || in.Branch != "main" {
|
||||
return nil, errors.New("asked another revision")
|
||||
}
|
||||
return json.Marshal(meshcatalog.RevisionAnswer{Revision: &meshcatalog.CatalogRevision{Revision: in.Revision,
|
||||
return json.Marshal(meshcatalog.RevisionAnswer{Revision: &meshcatalog.CatalogRevision{Revision: in.Revision, Base: base(in.Revision),
|
||||
Definitions: []meshcatalog.CatalogRevisionDefinitionsItem{{Seat: "s1", Digest: "sd"}}}, Entries: entries})
|
||||
case seatDefinitionsKey:
|
||||
return json.Marshal(meshcatalog.SeatDefinitionsAnswer{Found: true, Entry: &meshcatalog.SeatDefinitionsAnswerEntry{Seat: "s1", Digest: "sd", Definition: map[string]any{"seat": "s1"}}})
|
||||
@@ -117,4 +134,17 @@ func TestTheValidateStepRefusesAManifestProblemAndACycle(t *testing.T) {
|
||||
if o.Outcome != meshwants.StepEndedOutcomeFailed || o.Said != reasonCycle+": a:x → b:y → a:x" || len(asks) != 1 {
|
||||
t.Errorf("a revision making a cycle ended %+v (%d validator asks)", o, len(asks))
|
||||
}
|
||||
// The stand-in validator answers its one verdict: judged against the empty case before, web's problem is the revision's.
|
||||
if o := r.validate(ctx, "ask-first", step("indexer", 3)); o.Outcome != meshwants.StepEndedOutcomeFailed ||
|
||||
o.Said != reasonManifestRefused+`: web: unknown field "colour" (`+emptyBeforeWords+")" || len(asks) != 2 || len(requests[1].Cases[1].Manifests) != 0 {
|
||||
t.Errorf("a revision naming no base ended %+v (%d validator asks), want it judged against an empty case before, said", o, len(asks))
|
||||
}
|
||||
// Revision 12 holds revision 7's manifests and names base 7: judged against it, web's problem is the revision's, db's old one
|
||||
// is not, and the case before holds base 7's manifests.
|
||||
if o := r.validate(ctx, "ask-base", step("web", 12)); o.Outcome != meshwants.StepEndedOutcomeFailed ||
|
||||
o.Said != reasonManifestRefused+`: web: unknown field "colour"` || len(asks) != 3 {
|
||||
t.Errorf("a revision naming base 7 below its number ended %+v (%d validator asks), want it judged against base 7", o, len(asks))
|
||||
} else if web := requests[2].Cases[1].Manifests["web"]; len(requests[2].Cases[1].Manifests) != 3 || !strings.Contains(string(web.Raw), "d-web-2") {
|
||||
t.Errorf("the case before of revision 12 is %+v, want base 7's manifests", requests[2].Cases[1].Manifests)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -553,3 +553,18 @@ func (w *world) placedBranches() map[string]map[string]string {
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// assignedOn is the modules the assignments place on node.
|
||||
func (w *world) assignedOn(node string) []string {
|
||||
var out []string
|
||||
for _, m := range w.assignments()[node].Modules {
|
||||
out = append(out, m.Module)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// assignmentOf is node's assignments, or false where none is held.
|
||||
func (w *world) assignmentOf(node string) (mma.Assignments, bool) {
|
||||
a, ok := w.assignments()[node]
|
||||
return a, ok
|
||||
}
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
package assign
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
)
|
||||
|
||||
// KindAssignmentUnlisted is a module assigned to a node that no watched branch lists, left out of that node's composition
|
||||
// (design 47 H-A-148, H-A-149): scope delivery, id `<node>.<module>`, severity warning.
|
||||
const (
|
||||
KindAssignmentUnlisted = "assignment-unlisted"
|
||||
ScopeAssignmentUnlisted = "delivery"
|
||||
)
|
||||
|
||||
// UnlistedKey is its finding's key, `delivery.<node>.<module>.assignment-unlisted`.
|
||||
func UnlistedKey(node, module string) string {
|
||||
return views.FindingKey(ScopeAssignmentUnlisted, KeyID(node+"."+module), KindAssignmentUnlisted)
|
||||
}
|
||||
|
||||
// UnlistedOfKey is the node and module of an assignment-unlisted finding's key, or false for another finding's.
|
||||
func UnlistedOfKey(key string) (node, module string, ok bool) {
|
||||
rest, ok := strings.CutPrefix(key, ScopeAssignmentUnlisted+".")
|
||||
if !ok {
|
||||
return "", "", false
|
||||
}
|
||||
if rest, ok = strings.CutSuffix(rest, "."+KindAssignmentUnlisted); !ok {
|
||||
return "", "", false
|
||||
}
|
||||
node, module, ok = strings.Cut(rest, ".")
|
||||
return node, module, ok && node != "" && module != "" && !strings.Contains(module, ".")
|
||||
}
|
||||
|
||||
// UnlistedFinding is the finding of module assigned to node and listed on none of its watched branches, raised by the composition
|
||||
// step that leaves the assignment out (H-A-149).
|
||||
func UnlistedFinding(node, module string, at time.Time) (mcseat.Finding, error) {
|
||||
summary := fmt.Sprintf("%s is assigned %s, which no watched branch lists, so its composition leaves the assignment out", node, module)
|
||||
return encode(Observed{Scope: ScopeAssignmentUnlisted, ID: node + "." + module, Kind: KindAssignmentUnlisted, Machine: node,
|
||||
Severity: mcseat.FindingSeverityWarning, Summary: summary, Headline: clipHeadline(module + " on " + node + " is listed nowhere"),
|
||||
Explanation: summary + ": a stale assignment is a fault of the assignments, not of the change judged (ADR 0578 §2). " +
|
||||
"Take it off, or register the module on a watched branch.",
|
||||
Source: "assigner." + KindAssignmentUnlisted}, at)
|
||||
}
|
||||
@@ -34,7 +34,7 @@ type Half struct {
|
||||
// ViewName is how the catalog half's revision is named in a part's first-view status (views.Status.View, WaitsFor).
|
||||
func ViewName(revision int64) string { return strconv.FormatInt(revision, 10) }
|
||||
|
||||
// catalogCreated stands for the view's Created in its follower: a revision view is never made again, so one fixed time,
|
||||
// catalogCreated stands for the view's Created in its follower until its numbering starts again (H-A-141): one fixed time,
|
||||
// never zero, lets the follower tell a revision heard from one it holds.
|
||||
var catalogCreated = time.Unix(0, 0).UTC()
|
||||
|
||||
@@ -57,6 +57,23 @@ type Catalog struct {
|
||||
paging *Half
|
||||
// waits is the revision waited for, 0 when none.
|
||||
waits int64
|
||||
// made is when the view's numbering last started again (H-A-141), the follower's Created; zero is catalogCreated.
|
||||
made time.Time
|
||||
}
|
||||
|
||||
// created is madeLocked, taking mu.
|
||||
func (c *Catalog) created() time.Time {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
return c.madeLocked()
|
||||
}
|
||||
|
||||
// madeLocked is the view's Created in its follower: catalogCreated until a numbering started again. The caller holds mu.
|
||||
func (c *Catalog) madeLocked() time.Time {
|
||||
if c.made.IsZero() {
|
||||
return catalogCreated
|
||||
}
|
||||
return c.made
|
||||
}
|
||||
|
||||
// NewCatalog is a Catalog of branch asking the seat through call. Nothing is asked before Start.
|
||||
@@ -146,21 +163,34 @@ func (c *Catalog) Hear(ctx context.Context, data []byte) error {
|
||||
if c.held != nil {
|
||||
held = c.held.Revision.Revision
|
||||
}
|
||||
if ev.Revision > c.waits {
|
||||
c.waits = ev.Revision
|
||||
}
|
||||
c.mu.Unlock()
|
||||
if ev.Revision <= held {
|
||||
if ev.Revision == held {
|
||||
c.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
restarted := ev.Revision < held
|
||||
if restarted {
|
||||
// A revision below the one the picture holds is the state made again: its numbering started anew (design 47 H
|
||||
// H-A-141). The view's made time moves on, so the follower forgets the revisions heard of the earlier numbering,
|
||||
// and what it waits for is this revision.
|
||||
c.made = time.Now().UTC()
|
||||
c.waits = ev.Revision
|
||||
} else if ev.Revision > c.waits {
|
||||
c.waits = ev.Revision
|
||||
}
|
||||
made := c.madeLocked()
|
||||
c.mu.Unlock()
|
||||
// The follower keeps the newest revision heard, so an ask that was running when it was heard is asked once more.
|
||||
c.Follower.Apply(views.Event{Owner: seat.Seat, State: c.Follower.State, Revision: uint64(ev.Revision), Created: catalogCreated, Gap: true})
|
||||
c.Follower.Apply(views.Event{Owner: seat.Seat, State: c.Follower.State, Revision: uint64(ev.Revision), Created: made, Gap: true})
|
||||
c.Follower.WaitFor(ctx, ViewName(ev.Revision))
|
||||
why := fmt.Sprintf("catalog revision %d of %s said, the view holds %d", ev.Revision, c.Branch, held)
|
||||
if restarted {
|
||||
why = fmt.Sprintf("catalog revision %d of %s said, below the %d the view holds: the numbering started again", ev.Revision, c.Branch, held)
|
||||
}
|
||||
if c.Follower.Logf != nil {
|
||||
c.Follower.Logf("meshcatalogclient: catalog revision %d of %s said, the view holds %d: %s is asked once more", ev.Revision, c.Branch, held, RevisionVerb)
|
||||
c.Follower.Logf("meshcatalogclient: %s: %s is asked once more", why, RevisionVerb)
|
||||
}
|
||||
go func() {
|
||||
_ = c.Follower.Gap(ctx, fmt.Sprintf("catalog revision %d of %s said, the view holds %d", ev.Revision, c.Branch, held))
|
||||
_ = c.Follower.Gap(ctx, why)
|
||||
}()
|
||||
return nil
|
||||
}
|
||||
@@ -177,7 +207,7 @@ func (c *Catalog) ask(_ context.Context, after string) (views.Answer, error) {
|
||||
a, err := askRevision(c.call, in)
|
||||
if errors.Is(err, errNoneYet) {
|
||||
// A branch with no revision yet is ready and empty, and nothing waits for it (§479, §495).
|
||||
return views.Answer{V: 1, Owner: seat.Seat, State: c.Follower.State, Created: catalogCreated, Values: map[string]json.RawMessage{}, At: time.Now()}, nil
|
||||
return views.Answer{V: 1, Owner: seat.Seat, State: c.Follower.State, Created: c.created(), Values: map[string]json.RawMessage{}, At: time.Now()}, nil
|
||||
}
|
||||
if err != nil {
|
||||
return views.Answer{}, err
|
||||
@@ -212,7 +242,7 @@ func (c *Catalog) ask(_ context.Context, after string) (views.Answer, error) {
|
||||
if holds {
|
||||
c.Follower.Holds(ViewName(rev))
|
||||
}
|
||||
return views.Answer{V: 1, Owner: seat.Seat, State: c.Follower.State, Revision: uint64(rev), Created: catalogCreated, Values: values, Next: a.Next, At: time.Now()}, nil
|
||||
return views.Answer{V: 1, Owner: seat.Seat, State: c.Follower.State, Revision: uint64(rev), Created: c.created(), Values: values, Next: a.Next, At: time.Now()}, nil
|
||||
}
|
||||
|
||||
// errNoneYet is the seat's answer `none-yet`: the branch has no revision yet (design 47 G §530).
|
||||
|
||||
@@ -188,3 +188,89 @@ func TestCandidatesReachTheHook(t *testing.T) {
|
||||
t.Errorf("the hook was handed %+v, want the candidate's revision 4 alone", hooked)
|
||||
}
|
||||
}
|
||||
|
||||
// H-A-141 (design 47 H §750, §754): a catalog-revision below the revision the view holds is the state made again, its
|
||||
// numbering started anew: said in the log with both revisions, and the first view asked again, so the view follows the new
|
||||
// numbering and is behind nothing; one equal to the held revision asks nothing; one above is taken in as before.
|
||||
func TestCatalogFollowsANumberingStartedAgain(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
f := &fakeRevisions{newest: map[string]int64{"main": 528}, asks: map[string]int{}}
|
||||
var logMu sync.Mutex
|
||||
var logged []string
|
||||
c, err := NewCatalog(f.call, "main")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c.Follower.Wait = time.Nanosecond
|
||||
c.Follower.Logf = func(format string, args ...any) {
|
||||
logMu.Lock()
|
||||
defer logMu.Unlock()
|
||||
logged = append(logged, fmt.Sprintf(format, args...))
|
||||
}
|
||||
came := make(chan string, 16) // the module of each entry that came, one per revision answered
|
||||
c.Follower.Changed = func(module string, _ json.RawMessage, deleted bool, _ time.Time) {
|
||||
if !deleted {
|
||||
came <- module
|
||||
}
|
||||
}
|
||||
if err := c.Start(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
<-c.Ready()
|
||||
<-came // the first view's entry
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
newest int64 // what the seat answers from now
|
||||
heard int64
|
||||
asks int // the asks of the first view after the event
|
||||
holds int64 // the revision the view holds after it
|
||||
said string // a line the log holds after it; "" none new
|
||||
restarting bool
|
||||
}{
|
||||
{name: "equal to the held revision: dropped", newest: 528, heard: 528, asks: 1, holds: 528},
|
||||
{name: "above: taken in", newest: 529, heard: 529, asks: 2, holds: 529, said: "catalog revision 529 of main said, the view holds 528"},
|
||||
{name: "below: the numbering started again", newest: 7, heard: 7, asks: 3, holds: 7, said: "catalog revision 7 of main said, below the 529 the view holds: the numbering started again", restarting: true},
|
||||
{name: "equal after the restart: dropped", newest: 7, heard: 7, asks: 3, holds: 7},
|
||||
{name: "above after the restart: taken in", newest: 8, heard: 8, asks: 4, holds: 8, said: "catalog revision 8 of main said, the view holds 7"},
|
||||
} {
|
||||
t.Run(row.name, func(t *testing.T) {
|
||||
logMu.Lock()
|
||||
before := len(logged)
|
||||
logMu.Unlock()
|
||||
f.set("main", row.newest)
|
||||
if err := c.Hear(ctx, revisionEvent("main", KindTrunk, row.heard)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if row.said != "" {
|
||||
select {
|
||||
case m := <-came:
|
||||
if want := fmt.Sprintf("m-of-main-%d", row.holds); m != want {
|
||||
t.Fatalf("the entry %s came, want %s", m, want)
|
||||
}
|
||||
case <-ctx.Done():
|
||||
t.Fatalf("the view never took revision %d in", row.holds)
|
||||
}
|
||||
}
|
||||
if h, ok := c.View(); !ok || h.Revision.Revision != row.holds {
|
||||
t.Fatalf("the view holds %+v, want revision %d", h.Revision, row.holds)
|
||||
}
|
||||
if n := f.asked("main"); n != row.asks {
|
||||
t.Errorf("the first view was asked %d times, want %d", n, row.asks)
|
||||
}
|
||||
st := c.Follower.Status()
|
||||
if st.Behind != "" || st.WaitsFor != "" {
|
||||
t.Errorf("the view is behind %q, waits for %q, holding %d", st.Behind, st.WaitsFor, st.Revision)
|
||||
}
|
||||
logMu.Lock()
|
||||
fresh := strings.Join(logged[before:], "\n")
|
||||
logMu.Unlock()
|
||||
if row.said == "" && strings.Contains(fresh, "meshcatalogclient:") {
|
||||
t.Errorf("an event the view holds was said: %s", fresh)
|
||||
}
|
||||
if row.said != "" && !strings.Contains(fresh, row.said) {
|
||||
t.Errorf("the log does not say %q: %s", row.said, fresh)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -711,6 +711,8 @@ type CatalogRevisionDefinitionsItem struct {
|
||||
type CatalogRevision struct {
|
||||
// when (RFC 3339 time)
|
||||
At *time.Time `json:"at,omitempty"`
|
||||
// the newest kept revision of its branch below it, which the take-in that wrote it pinned for its pipeline: the base of the validate step's case before (design 47 H-A-147, H-A-151, H-A-154); absent on the branch's first take-in
|
||||
Base int64 `json:"base,omitempty"`
|
||||
// the watched branch the revision is of
|
||||
Branch string `json:"branch,omitempty"`
|
||||
// the seat definitions it was judged with, each by its digest, read whole through seat-definitions
|
||||
|
||||
@@ -59,7 +59,7 @@ func TestFixtures(t *testing.T) {
|
||||
decode: func(b []byte) (any, error) { var v ModulesAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil},
|
||||
{name: "the module verb (captured: xclip)", subject: "", given: "{\"module\":{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"fp\":\"src2:b427bdde1f100e2f52490cfa9df0c5112dd3700226b4892349f0c14a15a4af56\",\"manifest\":{\"build\":{\"artifacts\":[{\"binary\":\"xclip-tools\",\"from\":\"cmd/xclip-tools\",\"kind\":\"bundle\",\"language\":\"go\",\"loads\":[\"xclip-tools\"],\"name\":\"tools\",\"system\":\"arch\"}]},\"capabilities\":[\"package-manager\"],\"module\":\"xclip\",\"resources\":[{\"id\":\"package\",\"package\":\"xclip\",\"type\":\"package\"}],\"tools\":[\"xclip_copy\",\"xclip_paste\",\"xclip_targets\",\"xclip_session\"],\"version\":\"1\"},\"module\":\"xclip\",\"path\":\"modules/xclip\",\"repository\":\"novox/mesh-catalog\"}}", wire: "{\"module\":{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"fp\":\"src2:b427bdde1f100e2f52490cfa9df0c5112dd3700226b4892349f0c14a15a4af56\",\"manifest\":{\"build\":{\"artifacts\":[{\"binary\":\"xclip-tools\",\"from\":\"cmd/xclip-tools\",\"kind\":\"bundle\",\"language\":\"go\",\"loads\":[\"xclip-tools\"],\"name\":\"tools\",\"system\":\"arch\"}]},\"capabilities\":[\"package-manager\"],\"module\":\"xclip\",\"resources\":[{\"id\":\"package\",\"package\":\"xclip\",\"type\":\"package\"}],\"tools\":[\"xclip_copy\",\"xclip_paste\",\"xclip_targets\",\"xclip_session\"],\"version\":\"1\"},\"module\":\"xclip\",\"path\":\"modules/xclip\",\"repository\":\"novox/mesh-catalog\"}}",
|
||||
decode: func(b []byte) (any, error) { var v ModuleAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil},
|
||||
{name: "a watched branch's catalog graph revision", subject: "", given: "{\"at\":\"2026-11-09T19:00:00Z\",\"branch\":\"main\",\"definitions\":[{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"seat\":\"node-build-go\"}],\"entries\":[{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"module\":\"mesh-build-go\"},{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"module\":\"xclip\"}],\"kind\":\"trunk\",\"movers\":[{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"fingerprint\":\"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\",\"module\":\"xclip\",\"repository\":\"novox/mesh-catalog\"}],\"revision\":12,\"source\":3589,\"trunk\":[{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"repository\":\"novox/mesh-catalog\"}],\"v\":1}", wire: "{\"at\":\"2026-11-09T19:00:00Z\",\"branch\":\"main\",\"definitions\":[{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"seat\":\"node-build-go\"}],\"entries\":[{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"module\":\"mesh-build-go\"},{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"module\":\"xclip\"}],\"kind\":\"trunk\",\"movers\":[{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"fingerprint\":\"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\",\"module\":\"xclip\",\"repository\":\"novox/mesh-catalog\"}],\"revision\":12,\"source\":3589,\"trunk\":[{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"repository\":\"novox/mesh-catalog\"}],\"v\":1}",
|
||||
{name: "a watched branch's catalog graph revision", subject: "", given: "{\"at\":\"2026-11-09T19:00:00Z\",\"base\":9,\"branch\":\"main\",\"definitions\":[{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"seat\":\"node-build-go\"}],\"entries\":[{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"module\":\"mesh-build-go\"},{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"module\":\"xclip\"}],\"kind\":\"trunk\",\"movers\":[{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"fingerprint\":\"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\",\"module\":\"xclip\",\"repository\":\"novox/mesh-catalog\"}],\"revision\":12,\"source\":3589,\"trunk\":[{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"repository\":\"novox/mesh-catalog\"}],\"v\":1}", wire: "{\"at\":\"2026-11-09T19:00:00Z\",\"base\":9,\"branch\":\"main\",\"definitions\":[{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"seat\":\"node-build-go\"}],\"entries\":[{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"module\":\"mesh-build-go\"},{\"digest\":\"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\",\"module\":\"xclip\"}],\"kind\":\"trunk\",\"movers\":[{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"fingerprint\":\"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\",\"module\":\"xclip\",\"repository\":\"novox/mesh-catalog\"}],\"revision\":12,\"source\":3589,\"trunk\":[{\"commit\":\"f469eb387a3015569d9b16c66ed418e9dd098f64\",\"repository\":\"novox/mesh-catalog\"}],\"v\":1}",
|
||||
decode: func(b []byte) (any, error) {
|
||||
var v CatalogRevisionsByBranchRevisionValue
|
||||
err := json.Unmarshal(b, &v)
|
||||
|
||||
@@ -899,6 +899,8 @@ type CompositionRun struct {
|
||||
// the outcome's words
|
||||
Said string `json:"said,omitempty"`
|
||||
Started *time.Time `json:"started,omitempty"`
|
||||
// the module names of the assignments left out of the machine's composition, those no watched branch's newest revision lists, written when the run starts and read by every step's judgement and by the resume (design 47 H-A-152, H-A-153)
|
||||
Unlisted []string `json:"unlisted,omitempty"`
|
||||
// that machine's composer verdict (design 47 G §721)
|
||||
Verdict map[string]any `json:"verdict,omitempty"`
|
||||
}
|
||||
|
||||
@@ -208,7 +208,8 @@
|
||||
"fingerprint": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
|
||||
"repository": "novox/mesh-catalog"
|
||||
}
|
||||
]
|
||||
],
|
||||
"base": 9
|
||||
}
|
||||
},
|
||||
{
|
||||
|
||||
@@ -1927,6 +1927,10 @@
|
||||
"$ref": "#/defs/TakeInEntry"
|
||||
},
|
||||
"description": "the movers of the take-in that wrote it, so a redelivered source-moved whose revision is kept says take-in.taken again from them, recomputing nothing (design 47 H §743, H-A-92); absent for one with no mover"
|
||||
},
|
||||
"base": {
|
||||
"type": "integer",
|
||||
"description": "the newest kept revision of its branch below it, which the take-in that wrote it pinned for its pipeline: the base of the validate step's case before (design 47 H-A-147, H-A-151, H-A-154); absent on the branch's first take-in"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
|
||||
@@ -1701,6 +1701,13 @@
|
||||
"verdict": {
|
||||
"type": "object",
|
||||
"description": "that machine's composer verdict (design 47 G §721)"
|
||||
},
|
||||
"unlisted": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
},
|
||||
"description": "the module names of the assignments left out of the machine's composition, those no watched branch's newest revision lists, written when the run starts and read by every step's judgement and by the resume (design 47 H-A-152, H-A-153)"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
|
||||
@@ -273,6 +273,8 @@ export interface CatalogRevisionDefinitionsItem {
|
||||
export interface CatalogRevision {
|
||||
/** when (RFC 3339 time) */
|
||||
"at"?: string;
|
||||
/** the newest kept revision of its branch below it, which the take-in that wrote it pinned for its pipeline: the base of the validate step's case before (design 47 H-A-147, H-A-151, H-A-154); absent on the branch's first take-in */
|
||||
"base"?: number;
|
||||
/** the watched branch the revision is of */
|
||||
"branch"?: string;
|
||||
/** the seat definitions it was judged with, each by its digest, read whole through seat-definitions */
|
||||
|
||||
@@ -319,6 +319,8 @@ export interface CompositionRun {
|
||||
/** the outcome's words */
|
||||
"said"?: string;
|
||||
"started"?: string;
|
||||
/** the module names of the assignments left out of the machine's composition, those no watched branch's newest revision lists, written when the run starts and read by every step's judgement and by the resume (design 47 H-A-152, H-A-153) */
|
||||
"unlisted"?: string[];
|
||||
/** that machine's composer verdict (design 47 G §721) */
|
||||
"verdict"?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
+87
-11
@@ -162,6 +162,8 @@ type bed struct {
|
||||
answer func(verb string, answer string)
|
||||
// calls are the verbs the stand-in was called with, in order, as `<seat>.<verb> <input>`.
|
||||
calls func() []string
|
||||
// called is signalled after each call the stand-in answered, its answer chosen.
|
||||
called <-chan struct{}
|
||||
// restart stops the bridge, runs between, and starts it again on the same bus.
|
||||
restart func(between func())
|
||||
}
|
||||
@@ -237,6 +239,7 @@ func startBed(t *testing.T, head string) *bed {
|
||||
answers["git."+verb] = string(gitfixture.Given(t, verb+" answers", "verb:"+verb+".answer"))
|
||||
}
|
||||
var called []string
|
||||
signal := make(chan struct{}, 1)
|
||||
for _, subject := range []string{"mesh.seat.*.call.*.*", "mesh.seat.*.tool.*"} {
|
||||
sub, err := ctl.Subscribe(subject, func(m *nats.Msg) {
|
||||
parts := strings.Split(m.Subject, ".")
|
||||
@@ -249,6 +252,10 @@ func startBed(t *testing.T, head string) *bed {
|
||||
answer = answers[verb+" "+in.Repository]
|
||||
}
|
||||
mu.Unlock()
|
||||
select {
|
||||
case signal <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
if answer == "" {
|
||||
answer = `{"error":"the stand-in holds no answer"}`
|
||||
}
|
||||
@@ -279,7 +286,7 @@ func startBed(t *testing.T, head string) *bed {
|
||||
start()
|
||||
t.Cleanup(func() { stop() })
|
||||
set := func(verb, answer string) { mu.Lock(); answers[verb] = answer; mu.Unlock() }
|
||||
return &bed{srv: srv, ctl: ctl, js: js, ctx: ctx, done: done, answer: set,
|
||||
return &bed{srv: srv, ctl: ctl, js: js, ctx: ctx, done: done, answer: set, called: signal,
|
||||
calls: func() []string { mu.Lock(); defer mu.Unlock(); return append([]string(nil), called...) },
|
||||
restart: func(between func()) {
|
||||
stop()
|
||||
@@ -289,6 +296,30 @@ func startBed(t *testing.T, head string) *bed {
|
||||
}}
|
||||
}
|
||||
|
||||
// callsOf is how many calls of the stand-in start with prefix.
|
||||
func callsOf(b *bed, prefix string) int {
|
||||
n := 0
|
||||
for _, c := range b.calls() {
|
||||
if strings.HasPrefix(c, prefix) {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// calledTimes waits until the stand-in answered n calls starting with prefix: a call's answer is chosen when it is counted.
|
||||
func calledTimes(t *testing.T, b *bed, prefix string, n int) {
|
||||
t.Helper()
|
||||
bound := time.After(20 * time.Second)
|
||||
for callsOf(b, prefix) < n {
|
||||
select {
|
||||
case <-b.called:
|
||||
case <-bound:
|
||||
t.Fatalf("%d calls %s within 20 s, want %d", callsOf(b, prefix), prefix, n)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// headAt sets the stand-in seat git so main of the catalog is at head, protected or not.
|
||||
func headAt(b *bed, head string, protected bool) {
|
||||
p := "true"
|
||||
@@ -363,7 +394,10 @@ func TestBridgeSaysOneHeadMovedPerMove(t *testing.T) {
|
||||
b.restart(func() { headAt(b, h4, true) })
|
||||
said("a restart after a move", forgemsg.HeadMoved{Head: h4, Before: h3, Start: true})
|
||||
held(t, b, h4)
|
||||
branchReads := callsOf(b, "git.branch ")
|
||||
b.restart(func() {}) // at the head held: nothing
|
||||
// The head moves only once the start has read the head held, so that read is not the move's (§739: start true).
|
||||
calledTimes(t, b, "git.branch ", branchReads+1)
|
||||
headAt(b, h5, true)
|
||||
move(t, b, "main")
|
||||
said("the next move, the first message after the restart at the head held", forgemsg.HeadMoved{Head: h5, Before: h4})
|
||||
@@ -372,13 +406,11 @@ func TestBridgeSaysOneHeadMovedPerMove(t *testing.T) {
|
||||
t.Errorf("the bridge asked %s; it asks the seat git nothing but branch", c)
|
||||
}
|
||||
}
|
||||
heads, err := b.js.KeyValue(b.ctx, headsBucket(forgeBridgeModule()))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if e, err := heads.Get(b.ctx, "branch.novox.mesh-catalog.main"); err != nil || string(e.Value()) != h5 {
|
||||
t.Errorf("the state heads holds %v (%v), want %s", e, err, h5)
|
||||
// One read per start and per move of main: the first view's registrations, said after a start, adopt no branch (§532).
|
||||
if n := callsOf(b, "git.branch "); n != 3+4 {
|
||||
t.Errorf("the bridge read the head %d times, want 7: once at each of 3 starts and once per move of main (4)", n)
|
||||
}
|
||||
held(t, b, h5) // kept after the message is written, so watched for, not read at once
|
||||
}
|
||||
|
||||
func heard(t *testing.T, b *bed) <-chan *nats.Msg {
|
||||
@@ -395,10 +427,16 @@ func heard(t *testing.T, b *bed) <-chan *nats.Msg {
|
||||
return ch
|
||||
}
|
||||
|
||||
// next is the next message heard on subject within a bound; the others are passed by.
|
||||
// next is the next message heard on subject within 20 s; the others are passed by.
|
||||
func next(t *testing.T, ch <-chan *nats.Msg, subject string) []byte {
|
||||
t.Helper()
|
||||
bound := time.After(20 * time.Second)
|
||||
return nextWithin(t, ch, subject, 20*time.Second)
|
||||
}
|
||||
|
||||
// nextWithin is next within bound.
|
||||
func nextWithin(t *testing.T, ch <-chan *nats.Msg, subject string, within time.Duration) []byte {
|
||||
t.Helper()
|
||||
bound := time.After(within)
|
||||
for {
|
||||
select {
|
||||
case m := <-ch:
|
||||
@@ -406,7 +444,7 @@ func next(t *testing.T, ch <-chan *nats.Msg, subject string) []byte {
|
||||
return m.Data
|
||||
}
|
||||
case <-bound:
|
||||
t.Fatalf("nothing on %s within 20 s", subject)
|
||||
t.Fatalf("nothing on %s within %s", subject, within)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
@@ -455,6 +493,30 @@ func TestBridgePerformsWhatIsOwedOnce(t *testing.T) {
|
||||
}
|
||||
return n
|
||||
}
|
||||
// The bridge's acknowledgements on its durable of the stream owed, heard as the bus hears them.
|
||||
acks := make(chan *nats.Msg, 16)
|
||||
ackSub, err := b.ctl.ChanSubscribe("$JS.ACK."+seat.StreamOwed+".n1_mesh-delivery-forge-bridge.>", acks)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = ackSub.Unsubscribe() })
|
||||
if err := b.ctl.Flush(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
acked := func(n int) {
|
||||
t.Helper()
|
||||
bound := time.After(20 * time.Second)
|
||||
for n > 0 {
|
||||
select {
|
||||
case m := <-acks:
|
||||
if string(m.Data) == "+ACK" {
|
||||
n--
|
||||
}
|
||||
case <-bound:
|
||||
t.Fatalf("the bridge did not acknowledge what it heard on %s within 20 s", seat.StreamOwed)
|
||||
}
|
||||
}
|
||||
}
|
||||
gateOwed := gitfixture.Seat(t, seat.Seat, "events", "owed is said, a status", "event:owed")[0]
|
||||
subject := seat.SubjectOwed("novox~mesh-sdk", "178")
|
||||
for range 2 {
|
||||
@@ -479,12 +541,26 @@ func TestBridgePerformsWhatIsOwedOnce(t *testing.T) {
|
||||
_ = json.Unmarshal(gateOwed, &other)
|
||||
other["id"] = "status.novox/mesh-sdk#178.aaaaaaaaaaaa.mesh/other"
|
||||
otherOwed, _ := json.Marshal(other)
|
||||
// The three acts heard are acknowledged before the stop: one stopped while its outcome is on its way is heard again
|
||||
// after the start, by design, which is not what this part of the test is about.
|
||||
acked(3)
|
||||
// An act said while the bridge stops may be handed to the stopped bridge's pull before the bus has seen it go: the bus
|
||||
// hands it again after its consumer's acknowledgement wait, its redelivery the only clock (H-A-48).
|
||||
owedConsumer, err := b.js.Consumer(b.ctx, seat.StreamOwed, "n1_mesh-delivery-forge-bridge")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ackWait := owedConsumer.CachedInfo().Config.AckWait
|
||||
b.restart(func() {
|
||||
if _, err := b.js.Publish(b.ctx, subject, otherOwed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
if e := ended(); e.Of != seat.Seat || e.Outcome != forgemsg.OwedPerformed {
|
||||
afterRestart, err := forgemsg.DecodeOwedEnded(nextWithin(t, ch, forgeSubject(forgemsg.OwedEndedEvent), ackWait+20*time.Second))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if e := afterRestart; e.Of != seat.Seat || e.Outcome != forgemsg.OwedPerformed {
|
||||
t.Errorf("the gate's status after the restart: %+v, want performed", e)
|
||||
}
|
||||
if n := count("git.status"); n != 2 {
|
||||
|
||||
@@ -115,24 +115,31 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
var regs *meshcatalogclient.Registrations
|
||||
turn := &sync.Mutex{}
|
||||
mv := &mover{call: call, pub: js, heads: heads, watched: watched, turn: turn, logger: logger}
|
||||
recompute := func() {
|
||||
// begun says the start's read of the heads covers the watched branches from now on; it and what is watched are set
|
||||
// under setting, so a branch added before begun is read by the start and one added after it is adopted.
|
||||
var setting sync.Mutex
|
||||
begun := false
|
||||
recompute := func() (added bool) {
|
||||
if regs != nil {
|
||||
watched.Set(registeredOf(regs.All()), unescaped(src.Branches()))
|
||||
added = watched.Set(registeredOf(regs.All()), unescaped(src.Branches()))
|
||||
}
|
||||
return added
|
||||
}
|
||||
started := make(chan struct{}) // closed once the watched branches are read
|
||||
regs, _, err = meshcatalogclient.WireRegistrationsProcess(reader, keptCall, func(mcseat.Registration, bool) {
|
||||
recompute()
|
||||
select {
|
||||
case <-started: // a branch registered while the bridge runs is adopted: its head read (§532)
|
||||
setting.Lock()
|
||||
adopt := recompute() && begun
|
||||
setting.Unlock()
|
||||
// Only a registration adding a watched branch while the bridge runs is adopted, its head read (§532); the first
|
||||
// view's registrations, said after the start, add none, so the start reads their heads once, with start true (§739).
|
||||
if adopt {
|
||||
go mv.readHeads(ctx)
|
||||
default:
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
src.Follower.Changed = func(string, json.RawMessage, bool, time.Time) { recompute() }
|
||||
src.Follower.Changed = func(string, json.RawMessage, bool, time.Time) { setting.Lock(); recompute(); setting.Unlock() }
|
||||
if _, err := reader.Adopt(src.Follower, sourcebranches.Seat, sourcebranches.EventSourceMoved, src.Hear); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -167,7 +174,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
return nil
|
||||
}
|
||||
}
|
||||
setting.Lock()
|
||||
recompute()
|
||||
begun = true
|
||||
setting.Unlock()
|
||||
close(started)
|
||||
// What the gate says it owes, on its seat's stream owed through this module's durable there (made from its consumes).
|
||||
// A stream not there yet is said, and the bridge goes on: its other work does not wait on the gate.
|
||||
|
||||
@@ -27,6 +27,23 @@ func TestAMoveOfAWatchedBranchIsOneHeadMoved(t *testing.T) {
|
||||
if !watchedOf("main").Watches("novox/other", "main") || watchedOf("main").Watches("novox/other", "beta") {
|
||||
t.Errorf("before the first registration the gate's source's branches are watched, of every repository")
|
||||
}
|
||||
// §532: only a registration adding a watched branch is adopted; the same registrations read again (a first view's
|
||||
// values said after the start) add none.
|
||||
regs := []Registered{{Origin: "novox/mesh-sdk", Branches: []string{"main", "beta"}}}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
regs []Registered
|
||||
want bool
|
||||
}{
|
||||
{"the same registrations again", regs, false},
|
||||
{"a branch fewer", []Registered{{Origin: "novox/mesh-sdk", Branches: []string{"main"}}}, false},
|
||||
{"a branch registered", regs, true},
|
||||
{"a repository registered", append(regs, Registered{Origin: "novox/other", Branches: []string{"main"}}), true},
|
||||
} {
|
||||
if got := w.Set(c.regs, []string{"main"}); got != c.want {
|
||||
t.Errorf("%s: added %v, want %v", c.name, got, c.want)
|
||||
}
|
||||
}
|
||||
|
||||
const head, before = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", "cccccccccccccccccccccccccccccccccccccccc"
|
||||
moved := Moved{Owner: "novox", Repo: "mesh-sdk", Branch: "feat/47-H"}
|
||||
|
||||
@@ -21,8 +21,10 @@ type Registered struct {
|
||||
Branches []string
|
||||
}
|
||||
|
||||
// Set takes what the bridge read: the registrations (none before the first) and the gate's source's branches by name.
|
||||
func (w *Watched) Set(regs []Registered, source []string) {
|
||||
// Set takes what the bridge read: the registrations (none before the first) and the gate's source's branches by name. It
|
||||
// says whether a branch is watched now that was not (§532: a registration adding a watched branch is adopted); the same
|
||||
// registrations read again add none.
|
||||
func (w *Watched) Set(regs []Registered, source []string) (added bool) {
|
||||
by := map[string][]string{}
|
||||
for _, r := range regs {
|
||||
for _, b := range r.Branches {
|
||||
@@ -33,8 +35,33 @@ func (w *Watched) Set(regs []Registered, source []string) {
|
||||
}
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
before := w.pairsLocked()
|
||||
w.byRepo, w.registered = by, len(regs) > 0
|
||||
w.source = slices.Clone(source)
|
||||
for p := range w.pairsLocked() {
|
||||
if !before[p] {
|
||||
added = true
|
||||
}
|
||||
}
|
||||
return added
|
||||
}
|
||||
|
||||
// pairsLocked is every watched branch as `<repository> <branch>`, the repository empty before the first registration
|
||||
// (the gate's source's branch, of every repository). The caller holds mu.
|
||||
func (w *Watched) pairsLocked() map[string]bool {
|
||||
out := map[string]bool{}
|
||||
if !w.registered {
|
||||
for _, b := range w.source {
|
||||
out[" "+b] = true
|
||||
}
|
||||
return out
|
||||
}
|
||||
for repo, branches := range w.byRepo {
|
||||
for _, b := range branches {
|
||||
out[repo+" "+b] = true
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Watches says whether a move of branch in repository is one the bridge writes.
|
||||
|
||||
@@ -49,10 +49,12 @@ import (
|
||||
)
|
||||
|
||||
// busConf is the shape of the composed accounts file (mesh-controller internal/broker): one account with JetStream, the
|
||||
// controller's user and the indexer's own principal <node>.<module>.
|
||||
// controller's user and the indexer's own principal <node>.<module>; and the bus's system account, whose disconnect events
|
||||
// tell a restart when the stopped indexer's connections are gone (restartWaits).
|
||||
const busConf = `
|
||||
listen: 127.0.0.1:-1
|
||||
jetstream { store_dir: %q }
|
||||
system_account: SYS
|
||||
accounts {
|
||||
MESH {
|
||||
jetstream: enabled
|
||||
@@ -61,6 +63,11 @@ accounts {
|
||||
{ user: "n1.mesh-delivery-indexer", password: "pw" }
|
||||
]
|
||||
}
|
||||
SYS {
|
||||
users = [
|
||||
{ user: "sys", password: "pw" }
|
||||
]
|
||||
}
|
||||
}
|
||||
`
|
||||
|
||||
@@ -420,12 +427,54 @@ func startIndexerRun(t *testing.T, put func(context.Context, buildLog), before .
|
||||
}
|
||||
start()
|
||||
t.Cleanup(func() { stop() })
|
||||
// restart stops the program and starts it again on the same bus, as the node-engine does with a new build.
|
||||
restart := func() { stop(); start() }
|
||||
// restart stops the program and starts it again on the same bus, as the node-engine does with a new build, once the bus has
|
||||
// closed every connection of the stopped one (restartWaits).
|
||||
gone := restartWaits(t, srv)
|
||||
restart := func() { gone(stop); start() }
|
||||
|
||||
return js, ctx, candidates, modules, restart
|
||||
}
|
||||
|
||||
// restartWaits gives a stop that returns only once the bus has closed every connection the indexer held, heard on the system
|
||||
// account's disconnect events, never by sleeping. A program's stop closes its connections on its side alone; until the bus
|
||||
// has closed them its pulls on a durable still count, and a source-moved said in between is delivered to the stopped
|
||||
// program, unacknowledged for the durable's whole acknowledgement wait, while the branch's durable delivers one message
|
||||
// at a time (design 47 H §743).
|
||||
func restartWaits(t *testing.T, srv *server.Server) func(stop func()) {
|
||||
t.Helper()
|
||||
sys, err := nats.Connect(srv.ClientURL(), nats.UserInfo("sys", "pw"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(sys.Close)
|
||||
return func(stop func()) {
|
||||
t.Helper()
|
||||
disconnects, err := sys.SubscribeSync("$SYS.ACCOUNT.MESH.DISCONNECT")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = disconnects.Unsubscribe() }()
|
||||
if err := sys.Flush(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
open, err := srv.Connz(&server.ConnzOptions{User: "n1.mesh-delivery-indexer"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stop()
|
||||
for left := open.NumConns; left > 0; {
|
||||
m, err := disconnects.NextMsg(waitBound)
|
||||
if err != nil {
|
||||
t.Fatalf("the bus closed %d of the stopped indexer's %d connections within %s: %v", open.NumConns-left, open.NumConns, waitBound, err)
|
||||
}
|
||||
var ev server.DisconnectEventMsg
|
||||
if json.Unmarshal(m.Data, &ev) == nil && ev.Client.User == "n1.mesh-delivery-indexer" {
|
||||
left--
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The build seats and their holders as the mesh has them (design 47:482-485 and 1583: a language's build module is the
|
||||
// module claiming the seat whose `acts` is step build on that language): the seat definitions as modules/mesh-delivery-contracts/seats/<seat>/
|
||||
// seat.json has them (acts included) and the manifests as the catalog's modules/mesh-build-go and mesh-build-node claim them.
|
||||
@@ -691,6 +740,7 @@ func TestIndexerBuildsARunsCandidateAndDropsItWhenEveryMembersPipelineEnded(t *t
|
||||
func TestIndexerCountsOnlyThePlacedBuildNodesToolchains(t *testing.T) {
|
||||
const goSum = "3b5028f0c1e9bde04dd766abd63eb5931bc516bae695cf46f00abcd8510b0381"
|
||||
var assignments, toolchains jetstream.KeyValue
|
||||
var heard *takeIns
|
||||
place := func(ctx context.Context, t *testing.T, node string, modules ...string) {
|
||||
p := meshmoduleassigner.Assignments{Node: node}
|
||||
for _, m := range modules {
|
||||
@@ -712,6 +762,7 @@ func TestIndexerCountsOnlyThePlacedBuildNodesToolchains(t *testing.T) {
|
||||
assignerLate = up
|
||||
t.Cleanup(func() { assignerLate = nil })
|
||||
js, ctx, _, modules, restart := startIndexerRun(t, func(context.Context, buildLog) {}, func(ctx context.Context, js jetstream.JetStream) {
|
||||
heard = hearTakeIns(t, js.Conn())
|
||||
registerAvahi(t, ctx, js, "main")
|
||||
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.ToolchainsBucket("mesh-build-go"), History: 1})
|
||||
if err != nil {
|
||||
@@ -762,6 +813,10 @@ func TestIndexerCountsOnlyThePlacedBuildNodesToolchains(t *testing.T) {
|
||||
if !agreed(held) {
|
||||
t.Fatalf("within 20 s avahi has fp2 %q, none2 %q, toolchains %v: want a fingerprint of n1's go1.27 alone", held.Fp2, held.None2, held.Toolchains)
|
||||
}
|
||||
// The module is written before its take-in ends: each restart below waits for the source-moved's acknowledgement, the
|
||||
// take-in's end (design 47 H §743), so the stop cancels no take-in and leaves none unacknowledged ahead of the next one
|
||||
// on the branch's durable, which delivers one message at a time.
|
||||
heard.acked(t, "main")
|
||||
place(ctx, t, "n2", "mesh-build-go")
|
||||
key, _ := index.ToolchainKey("n2", "go")
|
||||
value, err := index.EncodeToolchain(nodebuildgo.Toolchain{Language: "go", Version: "go1.28.1", Recipe: "1", At: ptr(time.Now().Add(time.Second))})
|
||||
@@ -789,6 +844,7 @@ func TestIndexerCountsOnlyThePlacedBuildNodesToolchains(t *testing.T) {
|
||||
if m := read(disagree); !disagree(m) {
|
||||
t.Fatalf("within 20 s of a merge after n2's assignments avahi has fp2 %q, none2 %q: want the builders to disagree", m.Fp2, m.None2)
|
||||
}
|
||||
heard.acked(t, "main")
|
||||
// The view moves only at a take-in (design 47 H-A-49, H-A-103): with n2's assignments taken away, the next take-in agrees.
|
||||
place(ctx, t, "n2")
|
||||
restart()
|
||||
|
||||
@@ -357,7 +357,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
registry.Pruned = ix.PruneBranchBuilds
|
||||
ix.Registry = registry
|
||||
// The gate's source revisions, on one durable per watched branch (design 47 H §743, H-A-90), bound below.
|
||||
sources := &sourceMoved{ctx: ctx, ix: ix, branches: watchedBranches(registry), bind: bindBranch(js, cred, logger.Printf), logf: logger.Printf}
|
||||
sources := &sourceMoved{ctx: ctx, ix: ix, branches: watchedBranches(registry), bind: bindBranch(js, cred, logger.Printf), source: sourceVia(calls.Call), logf: logger.Printf}
|
||||
defer sources.stop()
|
||||
named.Changed = ix.RemoveUnlisted
|
||||
// The first fill of the versions (design 47 G §372): at the start, and again at each catalog revision kept until it is made.
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"fmt"
|
||||
"slices"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
|
||||
@@ -20,17 +21,22 @@ import (
|
||||
// the event's stream, named <node>_<indexer>_<branch>, the branch escaped as a key's name is (H-A-90), each delivering one message
|
||||
// at a time (the grants part makes it so, H-A-30), acknowledged once its catalog revision and take-in.taken are written. The
|
||||
// watched branches are the registrations'; a newly watched branch's consumer is bound on the indexer's own
|
||||
// registration-changed, not only at its start.
|
||||
// registration-changed, not only at its start. At its start, and again whenever a branch's consumer cannot be bound, the
|
||||
// branch's current source revision is read through the seat mesh-delivery's verb source and taken in as a heard source-moved
|
||||
// of it would be, when the view keeps no revision of the branch at or above it (H-A-129, H-A-130): at a clean state the
|
||||
// consumers are made only once the indexer answers its modules view, which it does only after its first take-in.
|
||||
type sourceMoved struct {
|
||||
ctx context.Context
|
||||
ix *index.Indexer
|
||||
branches func(context.Context) ([]string, error) // the watched branches the registrations name
|
||||
bind func(ctx context.Context, branch string, each func(jetstream.Msg)) (stop func(), err error)
|
||||
source func(ctx context.Context, branch string) (meshdelivery.SourceAnswer, error) // the gate's source revision of a branch
|
||||
logf func(string, ...any)
|
||||
|
||||
mu sync.Mutex
|
||||
bound map[string]func() // by branch: the consumer's stop
|
||||
again sync.Mutex // one take-in again at a time
|
||||
mu sync.Mutex
|
||||
bound map[string]func() // by branch: the consumer's stop
|
||||
started bool // the start's refresh ran: every branch's source was read once
|
||||
again sync.Mutex // one take-in again at a time
|
||||
}
|
||||
|
||||
// watchedBranches are the branches every registration watches, each once, sorted.
|
||||
@@ -53,10 +59,40 @@ func watchedBranches(r *index.Registry) func(context.Context) ([]string, error)
|
||||
}
|
||||
}
|
||||
|
||||
// bindBranch binds the indexer's durable of one watched branch on the stream of source-moved through mesh-sdk.
|
||||
// sourceVia reads a watched branch's current source revision through the seat mesh-delivery's verb source, as the
|
||||
// indexer's own account (H-A-129): a call where an answer is needed. An answer naming why it could not be read is that
|
||||
// failure. The branch is given by its name, as the gate reads it (it escapes the name for its key itself).
|
||||
func sourceVia(call toolCaller) func(context.Context, string) (meshdelivery.SourceAnswer, error) {
|
||||
verb := "seat:" + meshdelivery.Seat + "." + meshdelivery.VerbSource
|
||||
return func(_ context.Context, branch string) (meshdelivery.SourceAnswer, error) {
|
||||
var a meshdelivery.SourceAnswer
|
||||
raw, err := call(verb, meshdelivery.SourceInput{Branch: branch})
|
||||
if err != nil {
|
||||
return a, err
|
||||
}
|
||||
if err := json.Unmarshal(raw, &a); err != nil {
|
||||
return a, fmt.Errorf("the answer of %s is not its seat's answer: %w", verb, err)
|
||||
}
|
||||
if a.Error != "" {
|
||||
return a, fmt.Errorf("%s: %s", verb, a.Error)
|
||||
}
|
||||
return a, nil
|
||||
}
|
||||
}
|
||||
|
||||
// bindBranch binds the indexer's durable of one watched branch on the stream of source-moved through mesh-sdk. One the bus does
|
||||
// not hold the indexer makes itself, as its own kept consumer, and then binds (design 47 H §743, H-A-134: the grants part
|
||||
// composes none until H4b); one the bus holds is bound as it stands, never made again.
|
||||
func bindBranch(js jetstream.JetStream, cred process.Account, logf func(string, ...any)) func(context.Context, string, func(jetstream.Msg)) (func(), error) {
|
||||
return func(ctx context.Context, branch string, each func(jetstream.Msg)) (func(), error) {
|
||||
cons, err := cred.NamedEventConsumer(ctx, js, meshdelivery.StreamSourceMoved, keys.EscapeName(branch))
|
||||
if errors.Is(err, jetstream.ErrConsumerNotFound) {
|
||||
cfg := branchConsumerConfig(cred, branch)
|
||||
if cons, err = js.CreateConsumer(ctx, meshdelivery.StreamSourceMoved, cfg); err != nil {
|
||||
return nil, fmt.Errorf("the consumer %s on %s is not there and is not made: %w", cfg.Durable, meshdelivery.StreamSourceMoved, err)
|
||||
}
|
||||
logf("source-moved: the consumer %s on %s was not there: the indexer made it (H-A-134)", cfg.Durable, meshdelivery.StreamSourceMoved)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -68,31 +104,107 @@ func bindBranch(js jetstream.JetStream, cred process.Account, logf func(string,
|
||||
}
|
||||
}
|
||||
|
||||
// branchConsumerConfig is the indexer's durable of one watched branch as the mesh gives a module's seat-event consumer
|
||||
// (mesh-sdk's SeatEventConsumerConfig), named <node>_<indexer>_<branch> as NamedEventConsumer binds it, filtered to the
|
||||
// branch's source-moved and delivering one message at a time, with the ack wait of 30 s, the at most 5 deliveries and the
|
||||
// start from now the grants part gives a module's consumer on a seat event's stream kept by age (mesh-grants buscompose
|
||||
// heardConsumer): the branch's current source revision is read at the start through the gate's source (H-A-129, H-A-130), so
|
||||
// its past source-moved are never taken in again (design 47 H §743, H-A-90, H-A-134).
|
||||
func branchConsumerConfig(cred process.Account, branch string) jetstream.ConsumerConfig {
|
||||
esc := keys.EscapeName(branch)
|
||||
cfg := cred.SeatEventConsumerConfig([]string{meshdelivery.SubjectSourceMoved(esc, "*")}, false)
|
||||
cfg.Durable = cred.EventConsumerName() + "_" + esc
|
||||
cfg.MaxAckPending = 1
|
||||
cfg.AckWait, cfg.MaxDeliver = branchAckWait, branchMaxDeliver
|
||||
cfg.DeliverPolicy = jetstream.DeliverNewPolicy
|
||||
return cfg
|
||||
}
|
||||
|
||||
// The ack wait and the deliveries of a branch's durable, as the grants part gives a module's consumers (H-A-134, H-A-133).
|
||||
const (
|
||||
branchAckWait = 30 * time.Second
|
||||
branchMaxDeliver = 5
|
||||
)
|
||||
|
||||
// refresh binds the consumer of every watched branch not bound yet: at the start and on each registration-changed. A consumer
|
||||
// not there yet is said and bound at the next registration-changed or start.
|
||||
// not there yet is said and bound at the next registration-changed or start. The start's refresh reads every watched branch's
|
||||
// source revision, a later one each branch whose consumer could not be bound (H-A-129), after the binds.
|
||||
func (s *sourceMoved) refresh() {
|
||||
branches, err := s.branches(s.ctx)
|
||||
if err != nil {
|
||||
s.logf("source-moved: the watched branches are not read, so no new branch's consumer is bound: %v", err)
|
||||
return
|
||||
}
|
||||
for _, branch := range s.bindAll(branches) {
|
||||
s.readSource(branch)
|
||||
}
|
||||
}
|
||||
|
||||
// bindAll binds the branches not bound yet and answers those whose source revision is to be read now: at the start every
|
||||
// watched branch, later each one whose consumer could not be bound.
|
||||
func (s *sourceMoved) bindAll(branches []string) (read []string) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.bound == nil {
|
||||
s.bound = map[string]func(){}
|
||||
}
|
||||
start := !s.started
|
||||
s.started = true
|
||||
for _, b := range branches {
|
||||
if _, ok := s.bound[b]; ok {
|
||||
if start {
|
||||
read = append(read, b)
|
||||
}
|
||||
continue
|
||||
}
|
||||
branch := b
|
||||
stop, err := s.bind(s.ctx, branch, func(m jetstream.Msg) { s.heard(branch, m) })
|
||||
if err != nil {
|
||||
s.logf("source-moved: the consumer of the branch %s is not bound, so its source revisions are not taken in until the next registration-changed or start: %v", branch, err)
|
||||
s.logf("source-moved: the consumer of the branch %s is not bound, so its later source revisions are not heard until the next registration-changed or start; its current one is read through the gate's source: %v", branch, err)
|
||||
read = append(read, branch)
|
||||
continue
|
||||
}
|
||||
s.bound[branch] = stop
|
||||
s.logf("source-moved: the consumer of the branch %s is bound", branch)
|
||||
if start {
|
||||
read = append(read, branch)
|
||||
}
|
||||
}
|
||||
return read
|
||||
}
|
||||
|
||||
// readSource reads a branch's current source revision through the gate's verb source and takes it in as a heard
|
||||
// source-moved of that revision would be (takeIn), when the view keeps no revision of the branch at or above it (H-A-129,
|
||||
// H-A-130: index.SourceToTake).
|
||||
func (s *sourceMoved) readSource(branch string) {
|
||||
if s.source == nil {
|
||||
return
|
||||
}
|
||||
a, err := s.source(s.ctx, branch)
|
||||
if err != nil {
|
||||
s.logf("source-moved: the gate's source revision of %s is not read, so it is taken in only once its source-moved is heard: %v", branch, err)
|
||||
return
|
||||
}
|
||||
if a.Revision <= 0 {
|
||||
s.logf("source-moved: the gate keeps no source revision of %s yet", branch)
|
||||
return
|
||||
}
|
||||
s.again.Lock()
|
||||
defer s.again.Unlock()
|
||||
e, take, err := s.ix.SourceToTake(s.ctx, branch, a)
|
||||
if err != nil {
|
||||
s.logf("source-moved: the catalog revisions of %s are not read, so the gate's source revision %d of it is not taken in now: %v", branch, a.Revision, err)
|
||||
return
|
||||
}
|
||||
if !take {
|
||||
return
|
||||
}
|
||||
s.logf("source-moved: source revision %d of %s is read through the gate's source and taken in now (start %t)", a.Revision, branch, e.Start)
|
||||
switch err := s.takeIn(branch, e); {
|
||||
case errors.Is(err, index.ErrOldForm):
|
||||
s.logf("source-moved: the gate's source revision %d of %s is refused, the old form: %v", e.Source, branch, err)
|
||||
case err != nil:
|
||||
s.logf("source-moved: the gate's source revision %d of %s is not taken in now: %v", e.Source, branch, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -109,10 +221,7 @@ func (s *sourceMoved) heard(branch string, m jetstream.Msg) {
|
||||
}
|
||||
s.again.Lock()
|
||||
defer s.again.Unlock()
|
||||
behind, err := s.ix.HoldBehind(s.ctx, e)
|
||||
if !behind && err == nil {
|
||||
_, err = s.ix.TakeInSource(s.ctx, e)
|
||||
}
|
||||
err := s.takeIn(branch, e)
|
||||
switch {
|
||||
case errors.Is(err, index.ErrOldForm):
|
||||
s.logf("source-moved of %s on %s is refused, the old form: %v", branch, m.Subject(), err)
|
||||
@@ -123,14 +232,25 @@ func (s *sourceMoved) heard(branch string, m jetstream.Msg) {
|
||||
return
|
||||
case err != nil:
|
||||
s.logf("source-moved: source revision %d of %s is held, not taken in: %v", e.Source, branch, err)
|
||||
case behind:
|
||||
s.logf("source-moved: source revision %d of %s is held behind the branch's earlier held ones", e.Source, branch)
|
||||
}
|
||||
if err := m.Ack(); err != nil {
|
||||
s.logf("source-moved: source revision %d of %s is taken in or held and its acknowledgement failed: %v", e.Source, branch, err)
|
||||
}
|
||||
}
|
||||
|
||||
// takeIn takes one source-moved of a branch in, or holds it behind the branch's earlier held ones, so its source revisions
|
||||
// are taken in in their order; the caller holds s.again. A heard one and one read through the gate's source go this one way.
|
||||
func (s *sourceMoved) takeIn(branch string, e meshdelivery.SourceMovedEvent) error {
|
||||
behind, err := s.ix.HoldBehind(s.ctx, e)
|
||||
if !behind && err == nil {
|
||||
_, err = s.ix.TakeInSource(s.ctx, e)
|
||||
}
|
||||
if behind && err == nil {
|
||||
s.logf("source-moved: source revision %d of %s is held behind the branch's earlier held ones", e.Source, branch)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// takeAgain takes every held source revision in again from the state held, each branch in its order, on the indexer's own room
|
||||
// (H-A-50) and at its start (H-A-114).
|
||||
func (s *sourceMoved) takeAgain() {
|
||||
|
||||
@@ -2,14 +2,20 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"mesh-delivery-indexer/internal/index"
|
||||
)
|
||||
|
||||
@@ -88,3 +94,143 @@ func TestAHeldSourceMovedIsSkippedByHand(t *testing.T) {
|
||||
t.Errorf("held %+v after the skip, want source 5 alone", hs)
|
||||
}
|
||||
}
|
||||
|
||||
// revisionObjects is the catalog revisions' object bucket in memory.
|
||||
type revisionObjects map[string][]byte
|
||||
|
||||
func (o revisionObjects) Put(_ context.Context, name string, data []byte) error {
|
||||
o[name] = data
|
||||
return nil
|
||||
}
|
||||
func (o revisionObjects) Get(_ context.Context, name string) ([]byte, error) {
|
||||
if d, ok := o[name]; ok {
|
||||
return d, nil
|
||||
}
|
||||
return nil, index.ErrNoObject
|
||||
}
|
||||
func (o revisionObjects) Delete(_ context.Context, name string) error { delete(o, name); return nil }
|
||||
func (o revisionObjects) Names(context.Context) ([]string, error) {
|
||||
var out []string
|
||||
for n := range o {
|
||||
out = append(out, n)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
func (o revisionObjects) Fill(context.Context) (int64, int64, error) { return 0, 1 << 30, nil }
|
||||
|
||||
// Design 47 H §743, §768, H-A-129: at a clean state no branch's consumer is there to bind; the start reads each watched
|
||||
// branch's source revision through the gate's verb source and takes it in, and a failed bind at a registration-changed reads
|
||||
// it again without taking it in twice. A start whose view keeps the revision reads it and takes nothing in.
|
||||
func TestAStartWithNoConsumerTakesTheGatesSourceRevisionIn(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
var taken []string
|
||||
objects := revisionObjects{}
|
||||
ix := &index.Indexer{Module: testIndexer, Revisions: objects, Modules: index.MemoryStore(), Graph: index.MemoryStore(),
|
||||
Held: &memStore{vals: map[string][]byte{}}, Logf: func(string, ...any) {},
|
||||
SayTakeIn: func(_ context.Context, event, id string, _ any) error {
|
||||
taken = append(taken, event+" "+id)
|
||||
return nil
|
||||
}}
|
||||
var asked []string
|
||||
newSources := func() *sourceMoved {
|
||||
return &sourceMoved{ctx: ctx, ix: ix,
|
||||
branches: func(context.Context) ([]string, error) { return []string{"main"}, nil },
|
||||
bind: func(context.Context, string, func(jetstream.Msg)) (func(), error) {
|
||||
return nil, errors.New("nats: API error: code=404 err_code=10014 description=consumer not found")
|
||||
},
|
||||
source: func(_ context.Context, branch string) (meshdelivery.SourceAnswer, error) {
|
||||
asked = append(asked, branch)
|
||||
return meshdelivery.SourceAnswer{Branch: branch, Revision: 1, Call: "0123456789abcdef"}, nil
|
||||
},
|
||||
logf: func(string, ...any) {}}
|
||||
}
|
||||
s := newSources()
|
||||
s.refresh() // the start
|
||||
if _, ok := objects[index.BranchObject("main", 1)]; !ok || !reflect.DeepEqual(taken, []string{"take-in.taken " + index.TakenID("main", 1)}) {
|
||||
t.Fatalf("the start kept %v and said %v; want main.1 taken in from the gate's source", objects, taken)
|
||||
}
|
||||
s.refresh() // a registration-changed, the consumer still not there
|
||||
newSources().refresh() // a start whose view keeps the revision
|
||||
if !reflect.DeepEqual(asked, []string{"main", "main", "main"}) || len(taken) != 1 {
|
||||
t.Errorf("asked the gate for %v and said %v; want main asked at each and taken in once", asked, taken)
|
||||
}
|
||||
}
|
||||
|
||||
// fakeBranchJS is the bus as bindBranch uses it: the source-moved stream's consumers by name, and the consumers made.
|
||||
type fakeBranchJS struct {
|
||||
jetstream.JetStream
|
||||
held map[string]*fakeBranchConsumer
|
||||
made []jetstream.ConsumerConfig
|
||||
}
|
||||
|
||||
func (f *fakeBranchJS) Consumer(_ context.Context, stream, name string) (jetstream.Consumer, error) {
|
||||
if c, ok := f.held[stream+"/"+name]; ok {
|
||||
return c, nil
|
||||
}
|
||||
return nil, jetstream.ErrConsumerNotFound
|
||||
}
|
||||
|
||||
func (f *fakeBranchJS) CreateConsumer(_ context.Context, stream string, cfg jetstream.ConsumerConfig) (jetstream.Consumer, error) {
|
||||
f.made = append(f.made, cfg)
|
||||
c := &fakeBranchConsumer{}
|
||||
f.held[stream+"/"+cfg.Durable] = c
|
||||
return c, nil
|
||||
}
|
||||
|
||||
// fakeBranchConsumer hands one source-moved to the hearing as soon as it is consumed.
|
||||
type fakeBranchConsumer struct {
|
||||
jetstream.Consumer
|
||||
heard int
|
||||
}
|
||||
|
||||
type fakeConsumeContext struct{ jetstream.ConsumeContext }
|
||||
|
||||
func (fakeConsumeContext) Stop() {}
|
||||
|
||||
func (c *fakeBranchConsumer) Consume(handle jetstream.MessageHandler, _ ...jetstream.PullConsumeOpt) (jetstream.ConsumeContext, error) {
|
||||
c.heard++
|
||||
handle(msgOf{f: &fakeMsg{subject: "mesh.seat.mesh-delivery.event.source-moved.main.2"}})
|
||||
return fakeConsumeContext{}, nil
|
||||
}
|
||||
|
||||
// Design 47 H §743, H-A-134: a branch's durable the bus does not hold the indexer makes, as a module's seat-event consumer is
|
||||
// made (SeatEventConsumerConfig), named <node>_<indexer>_<branch>, filtered to the branch, one message at a time, and then
|
||||
// hears on it; one the bus holds is bound, never made again.
|
||||
func TestABranchDurableNotThereIsMadeAndHeard(t *testing.T) {
|
||||
cred := process.Account{Node: "n1", Module: "mesh-delivery-indexer"}
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
held bool
|
||||
}{{"not there: made", false}, {"there: bound", true}} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
js := &fakeBranchJS{held: map[string]*fakeBranchConsumer{}}
|
||||
name := "n1_mesh-delivery-indexer_" + keys.EscapeName("main")
|
||||
var existing *fakeBranchConsumer
|
||||
if tc.held {
|
||||
existing = &fakeBranchConsumer{}
|
||||
js.held[meshdelivery.StreamSourceMoved+"/"+name] = existing
|
||||
}
|
||||
var heard []string
|
||||
stop, err := bindBranch(js, cred, func(string, ...any) {})(context.Background(), "main", func(m jetstream.Msg) { heard = append(heard, m.Subject()) })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stop()
|
||||
if len(heard) != 1 {
|
||||
t.Fatalf("heard %v, want the one source-moved the consumer hands", heard)
|
||||
}
|
||||
if tc.held {
|
||||
if len(js.made) != 0 || existing.heard != 1 {
|
||||
t.Fatalf("made %v and heard the held one %d times; want it bound, not made again", js.made, existing.heard)
|
||||
}
|
||||
return
|
||||
}
|
||||
want := cred.SeatEventConsumerConfig([]string{meshdelivery.SubjectSourceMoved(keys.EscapeName("main"), "*")}, false)
|
||||
want.Durable, want.MaxAckPending = name, 1
|
||||
want.AckWait, want.MaxDeliver, want.DeliverPolicy = 30*time.Second, 5, jetstream.DeliverNewPolicy
|
||||
if len(js.made) != 1 || !reflect.DeepEqual(js.made[0], want) {
|
||||
t.Fatalf("made %+v, want once %+v", js.made, want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -361,11 +361,13 @@ func absent(v []byte) bool { return v == nil }
|
||||
//
|
||||
// - main's first source revision (start true) is taken in with no mover and pins nothing; the next, moving avahi's tree, is one
|
||||
// revision main.11 carrying avahi as its one mover, pinned pipeline.main.11, said once as take-in.taken; the same source-moved
|
||||
// given again says the same from the kept movers and keeps no second revision;
|
||||
// - pipeline-started of main.12 pins it, a restart keeps main.11 while its pipeline runs and removes main.10;
|
||||
// given again says the same from the kept movers and keeps no second revision; main.10, the branch's previous revision, is
|
||||
// pinned pipeline.main.11 in the same write as validate's base (H-A-147);
|
||||
// - pipeline-started of main.12 pins it, a restart keeps main.11 and main.10 while main.11's pipeline runs;
|
||||
// - with the store full, main.13 is held, said once as take-in.pending, written whole in the state held and acknowledged
|
||||
// (H-A-114, H-A-118); nothing takes it in on a
|
||||
// timer: main.11's pipeline-ended frees its pin, the revision goes, the indexer says its room and takes main.13 in on it;
|
||||
// timer: main.11's pipeline-ended frees both its pins, main.10 goes (main.11 stays, main.12's base), the indexer says its
|
||||
// room and takes main.13 in on it;
|
||||
// - beta's take-in whose commit the forge cannot read is held while main's next goes on; the hand skip, with why, is recorded in
|
||||
// the hand-act log before it is asked, and take-in.taken says it skipped;
|
||||
// - a branch watched later is bound on registration-changed and its first revision taken in.
|
||||
@@ -429,6 +431,7 @@ func TestIndexerWhole(t *testing.T) {
|
||||
t.Fatalf("main.11 = %+v (%v), want it kept carrying the movers said", rev, err)
|
||||
}
|
||||
untilKey(t, ctx, pins, index.PinKey("main", 11), pinned("pipeline.main.11"))
|
||||
untilKey(t, ctx, pins, index.PinKey("main", 10), pinned("pipeline.main.11")) // validate's base, H-A-147
|
||||
info, err := objects.open(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -445,8 +448,8 @@ func TestIndexerWhole(t *testing.T) {
|
||||
t.Errorf("main.11 was written again (%v): a source-moved given again keeps no second revision", err)
|
||||
}
|
||||
|
||||
// pipeline-started pins its revision; a restart while main.11's pipeline runs keeps it, and the judge at the start removes
|
||||
// main.10, which nothing pins and which is no longer current.
|
||||
// pipeline-started pins its revision; a restart while main.11's pipeline runs keeps it and main.10, its validate base
|
||||
// (H-A-147), which the take-in pinned for it.
|
||||
sayMoved(t, ctx, js, "main", 12, c12, false)
|
||||
heard.next(t, "main", 12)
|
||||
if err := pins.Delete(ctx, index.PinKey("main", 12)); err != nil { // the take-in's own pin, so pipeline-started alone writes it below
|
||||
@@ -454,10 +457,16 @@ func TestIndexerWhole(t *testing.T) {
|
||||
}
|
||||
sayPipeline(t, ctx, js, false, "main", 12)
|
||||
untilKey(t, ctx, pins, index.PinKey("main", 12), pinned("pipeline.main.12"))
|
||||
// The restart waits for the take-ins' ends, their source-moved acknowledged (design 47 H §743: main 10, 11, 11 given
|
||||
// again and 12), so the stop cancels none and leaves none unacknowledged ahead of main 13.
|
||||
for range 4 {
|
||||
heard.acked(t, "main")
|
||||
}
|
||||
restart()
|
||||
untilRevisionGone(t, ctx, js, "main.10")
|
||||
if _, _, err := index.ReadRevision(ctx, objects, "main.11"); err != nil {
|
||||
t.Fatalf("main.11 is not kept across a restart while its pipeline runs: %v", err)
|
||||
for _, name := range []string{"main.10", "main.11"} {
|
||||
if _, _, err := index.ReadRevision(ctx, objects, name); err != nil {
|
||||
t.Fatalf("%s is not kept across a restart while main.11's pipeline runs: %v", name, err)
|
||||
}
|
||||
}
|
||||
|
||||
// The store full: held, said once, in the state held; taken in on the indexer's own room once a freed pin makes it.
|
||||
@@ -487,7 +496,8 @@ func TestIndexerWhole(t *testing.T) {
|
||||
t.Errorf("main 13 taken in on room said %+v, want avahi at %s", got, c13)
|
||||
}
|
||||
untilKey(t, ctx, held, keys.EscapeName("main"), absent)
|
||||
untilKey(t, ctx, pins, index.PinKey("main", 11), absent)
|
||||
untilKey(t, ctx, pins, index.PinKey("main", 11), pinned("pipeline.main.12")) // main.12's validate base stays (H-A-147)
|
||||
untilRevisionGone(t, ctx, js, "main.10") // the end freed both of main.11's pins
|
||||
if n := len(heard.pending); n != 0 {
|
||||
t.Errorf("take-in.pending said %d more times for one held source revision, want once", n)
|
||||
}
|
||||
|
||||
@@ -268,7 +268,7 @@ func (ix *Indexer) viewEntries(ctx context.Context) (map[string]meshcatalog.Cata
|
||||
// KeepBranchEntries keeps entries as the catalog revision of a watched branch taken in from its source revision src (design
|
||||
// 47 H §743, H-A-32): <branch>.<src>, the trunk's included, numbered by the source, and says catalog-revision naming the
|
||||
// branch. movers are the take-in's: kept on the revision (H-A-92), and the revision pinned for the take-in's pipeline,
|
||||
// pipeline.<branch>.<src>, in the same write when there is one (A-449a). A revision already kept under that name is not written
|
||||
// pipeline.<branch>.<src>, in the same write when there is one (A-449a), with the branch's previous revision (H-A-147). A revision already kept under that name is not written
|
||||
// again (read before write, under the branch's one consumer); ErrObjectsFull is a store too full to keep it.
|
||||
func (ix *Indexer) KeepBranchEntries(ctx context.Context, branch string, src int64, entries map[string]meshcatalog.CatalogEntry, movers []meshcatalog.TakeInEntry) error {
|
||||
return ix.keepBranchEntries(ctx, branch, src, entries, movers)
|
||||
@@ -305,6 +305,12 @@ func (ix *Indexer) keepBranchEntries(ctx context.Context, branch string, src int
|
||||
return err
|
||||
}
|
||||
rev.Movers = movers
|
||||
// The base of the validate step's case before: the newest revision kept before this write, which the take-in pins for its
|
||||
// pipeline below, named on the revision in this same write (H-A-147, H-A-151, H-A-154); none on the branch's first take-in.
|
||||
pinsBase := len(movers) > 0 && ix.Pins != nil && newest.Revision > 0 && newest.Revision != src
|
||||
if pinsBase {
|
||||
rev.Base = newest.Revision
|
||||
}
|
||||
changed := changedEntries(newest.Entries, rev.Entries)
|
||||
used, capacity, fillErr := ix.Revisions.Fill(ctx)
|
||||
if fillErr == nil && capacity > 0 && used+newBytes(ctx, ix.Revisions, rev, objects) > capacity {
|
||||
@@ -330,6 +336,16 @@ func (ix *Indexer) keepBranchEntries(ctx context.Context, branch string, src int
|
||||
ix.logf("pins: the revision %s is not pinned for %s: %v", BranchObject(branch, src), record, err)
|
||||
return fmt.Errorf("the pin of %s: %w", BranchObject(branch, src), err)
|
||||
}
|
||||
// The branch's previous revision, the newest kept before this write, pinned for the same pipeline: it is the base of
|
||||
// the validate step's case before (H-A-108 (3), H-A-147), kept as long as the pipeline that needs it.
|
||||
if pinsBase {
|
||||
if err := ix.pin(ctx, PinKey(branch, rev.Base), record); err != nil {
|
||||
ix.logf("pins: the previous revision %s is not pinned for %s: %v", BranchObject(branch, newest.Revision), record, err)
|
||||
return fmt.Errorf("the pin of %s: %w", BranchObject(branch, newest.Revision), err)
|
||||
}
|
||||
} else {
|
||||
ix.logf("pins: %s has no revision before it on its branch: none is pinned for %s as its validate base", BranchObject(branch, src), record)
|
||||
}
|
||||
}
|
||||
// The revision before this one in its branch ceases to be current (§579): it goes unless a record pins it.
|
||||
if removed, err := ix.judgePinsLocked(ctx); err != nil {
|
||||
@@ -337,6 +353,11 @@ func (ix *Indexer) keepBranchEntries(ctx context.Context, branch string, src int
|
||||
} else if len(removed) > 0 {
|
||||
ix.logf("pins: %d revisions removed (%s), no record pins them and none is current", len(removed), strings.Join(removed, ", "))
|
||||
}
|
||||
if dropped, err := ix.dropEarlierFormsLocked(ctx); err != nil {
|
||||
ix.logf("catalog graph: the revisions of the earlier forms (trunk.*, branch.*) are not dropped: %v", err)
|
||||
} else if len(dropped) > 0 {
|
||||
ix.logf("catalog graph: %d revisions of the earlier forms dropped (%s), no record pins them; each branch's newest revision is its current one", len(dropped), strings.Join(dropped, ", "))
|
||||
}
|
||||
if _, err := ix.removeUnlisted(ctx); err != nil {
|
||||
ix.logf("catalog graph: the entries nothing lists or names any more are not removed: %v", err) // at the first revision after none does (§675)
|
||||
}
|
||||
|
||||
@@ -72,7 +72,8 @@ func pinObject(key string) (string, bool) {
|
||||
}
|
||||
|
||||
// HeardPins applies one event of the seat mesh-pipeline-runner the indexer's durable consumers gave it (design 47 H §749): a
|
||||
// pipeline-started pins the revision it names under the record `pipeline.<pipeline>`, its pipeline-ended frees it. A body
|
||||
// pipeline-started pins the revision it names under the record `pipeline.<pipeline>`, its pipeline-ended frees it and every
|
||||
// other revision the record pins (H-A-147). A body
|
||||
// that is no such event, or names no revision, changes nothing; an error leaves the event unacknowledged, so the bus gives it
|
||||
// again; one applied twice changes nothing.
|
||||
func (ix *Indexer) HeardPins(ctx context.Context, started *runner.PipelineStarted, ended *runner.PipelineEnded) error {
|
||||
@@ -83,7 +84,34 @@ func (ix *Indexer) HeardPins(ctx context.Context, started *runner.PipelineStarte
|
||||
case started != nil && started.Revision > 0 && started.Pipeline != "":
|
||||
return ix.pin(ctx, PinKey(started.Branch, started.Revision), PipelineRecord(started.Pipeline))
|
||||
case ended != nil && ended.Revision > 0 && ended.Pipeline != "":
|
||||
return ix.unpin(ctx, PinKey(ended.Branch, ended.Revision), PipelineRecord(ended.Pipeline))
|
||||
record := PipelineRecord(ended.Pipeline)
|
||||
if err := ix.unpin(ctx, PinKey(ended.Branch, ended.Revision), record); err != nil {
|
||||
return err
|
||||
}
|
||||
return ix.unpinElsewhere(ctx, record)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// unpinElsewhere frees a record on every other key that lists it: the branch's previous revision the take-in pinned for the
|
||||
// same pipeline (H-A-147), which the pipeline-ended does not name.
|
||||
func (ix *Indexer) unpinElsewhere(ctx context.Context, record string) error {
|
||||
pinKeys, err := ix.Pins.Keys(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, k := range pinKeys {
|
||||
v, _, found, err := ix.Pins.Get(ctx, k)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var pv PinValue
|
||||
if !found || json.Unmarshal(v, &pv) != nil || !slices.Contains(pv.Records, record) {
|
||||
continue
|
||||
}
|
||||
if err := ix.unpin(ctx, k, record); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -253,25 +281,10 @@ func (ix *Indexer) judgePinsLocked(ctx context.Context) ([]string, error) {
|
||||
return nil, err
|
||||
}
|
||||
current := CurrentRevisions(names)
|
||||
pinned := map[string]bool{}
|
||||
pinKeys, err := ix.Pins.Keys(ctx)
|
||||
pinKeys, pinned, err := ix.pinnedObjects(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, k := range pinKeys {
|
||||
object, ok := pinObject(k)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
v, _, found, err := ix.Pins.Get(ctx, k)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var pv PinValue
|
||||
if found && json.Unmarshal(v, &pv) == nil && len(pv.Records) > 0 {
|
||||
pinned[object] = true
|
||||
}
|
||||
}
|
||||
var removed []string
|
||||
for _, n := range names {
|
||||
if _, _, ok := RevisionGroup(n); !ok || pinned[n] {
|
||||
@@ -301,6 +314,72 @@ func (ix *Indexer) judgePinsLocked(ctx context.Context) ([]string, error) {
|
||||
return removed, nil
|
||||
}
|
||||
|
||||
// pinnedObjects are the state pins' keys and the revision objects a record pins.
|
||||
func (ix *Indexer) pinnedObjects(ctx context.Context) ([]string, map[string]bool, error) {
|
||||
pinned := map[string]bool{}
|
||||
pinKeys, err := ix.Pins.Keys(ctx)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
for _, k := range pinKeys {
|
||||
object, ok := pinObject(k)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
v, _, found, err := ix.Pins.Get(ctx, k)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
var pv PinValue
|
||||
if found && json.Unmarshal(v, &pv) == nil && len(pv.Records) > 0 {
|
||||
pinned[object] = true
|
||||
}
|
||||
}
|
||||
return pinKeys, pinned, nil
|
||||
}
|
||||
|
||||
// earlierForm says whether a revision object is of the earlier model's forms, trunk.<n> or branch.<branch>.<n>, beside the
|
||||
// branch-keyed <branch>.<n> of design 47 H §743 (H-A-135).
|
||||
func earlierForm(name string) bool {
|
||||
if _, _, ok := RevisionGroup(name); !ok {
|
||||
return false
|
||||
}
|
||||
return strings.HasPrefix(name, "trunk.") || strings.HasPrefix(name, "branch.")
|
||||
}
|
||||
|
||||
// dropEarlierFormsLocked drops every revision object of the earlier forms (earlierForm) no record pins, at a branch's take-in
|
||||
// (design 47 H §743, H-A-135): the branch's newest revision is its current one, and an earlier trunk revision kept beside it
|
||||
// would be the newest trunk revision a reader hears. It drops none while the pins do not count yet; the caller holds keepMu.
|
||||
func (ix *Indexer) dropEarlierFormsLocked(ctx context.Context) ([]string, error) {
|
||||
if ix.PinsPending != nil {
|
||||
if pending, err := ix.PinsPending(ctx); err != nil || pending != 0 {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
names, err := ix.Revisions.Names(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
pinned := map[string]bool{}
|
||||
if ix.Pins != nil {
|
||||
if _, pinned, err = ix.pinnedObjects(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
var dropped []string
|
||||
for _, n := range names {
|
||||
if !earlierForm(n) || pinned[n] {
|
||||
continue
|
||||
}
|
||||
if err := ix.Revisions.Delete(ctx, n); err != nil {
|
||||
return dropped, fmt.Errorf("%s: %w", n, err)
|
||||
}
|
||||
dropped = append(dropped, n)
|
||||
}
|
||||
sort.Strings(dropped)
|
||||
return dropped, nil
|
||||
}
|
||||
|
||||
// PinDrop drops one pin of the state pins by hand, by its key <branch>.<revision> (indexer_pin_drop, design 47 H §765, H-A-57):
|
||||
// every record's, for a pin that names no running pipeline; the finding pin-missing of the key is cleared, and the removal
|
||||
// rule then applies.
|
||||
|
||||
@@ -3,6 +3,7 @@ package index
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"slices"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -102,3 +103,73 @@ func mustRecords(t *testing.T, v []byte) []string {
|
||||
slices.Sort(pv.Records)
|
||||
return pv.Records
|
||||
}
|
||||
|
||||
// H-A-147 (§743, §749, H-A-108 (3)): a take-in with a mover pins, in its write, the branch's previous revision (the newest kept
|
||||
// before the write) for the same pipeline record as its own, so the validate step's case before can read it while the pipeline
|
||||
// runs, and names it as its base on the revision it writes (H-A-151, H-A-154); a take-in with no revision before it pins only
|
||||
// its own and names no base; the pipeline's end frees both.
|
||||
func TestATakeInPinsTheBranchsPreviousRevisionForItsPipeline(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
reader := meshcatalog.CatalogEntry{Module: "reader", Repository: "novox/r", Manifest: map[string]any{}}
|
||||
mover := []meshcatalog.TakeInEntry{{Module: "reader", Commit: "c", Fingerprint: "f", Repository: "novox/r"}}
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
before []int64 // sources taken in with no mover before the take-in of source 5
|
||||
whileRuns []string
|
||||
afterTheEnd []string
|
||||
base int64 // the base main.5 names (H-A-154)
|
||||
}{
|
||||
{name: "the previous revision kept and pinned", before: []int64{3, 4}, whileRuns: []string{"main.4", "main.5"}, afterTheEnd: []string{"main.5"}, base: 4},
|
||||
{name: "the newest kept before, whatever its number", before: []int64{2}, whileRuns: []string{"main.2", "main.5"}, afterTheEnd: []string{"main.5"}, base: 2},
|
||||
{name: "no revision before it pins only its own", whileRuns: []string{"main.5"}, afterTheEnd: []string{"main.5"}},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
ix := &Indexer{Module: testSelf, Revisions: newMemObjects(1 << 20), Pins: newMem(), Findings: newFindings(),
|
||||
Now: func() time.Time { return time.Date(2026, 11, 20, 16, 0, 0, 0, time.UTC) },
|
||||
PinsPending: func(context.Context) (uint64, error) { return 0, nil }}
|
||||
keep := func(source int64, movers []meshcatalog.TakeInEntry) {
|
||||
t.Helper()
|
||||
e := reader
|
||||
e.Commit = fmt.Sprintf("c%d", source)
|
||||
if err := ix.KeepBranchEntries(ctx, "main", source, map[string]meshcatalog.CatalogEntry{"reader": e}, movers); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
kept := func() []string {
|
||||
n, _ := ix.Revisions.Names(ctx)
|
||||
n = slices.DeleteFunc(n, func(s string) bool { _, _, ok := RevisionGroup(s); return !ok })
|
||||
slices.Sort(n)
|
||||
return n
|
||||
}
|
||||
for _, s := range tc.before {
|
||||
keep(s, nil)
|
||||
}
|
||||
keep(5, mover)
|
||||
if rev, _, err := ReadRevision(ctx, ix.Revisions, "main.5"); err != nil || rev.Base != tc.base {
|
||||
t.Errorf("main.5 names the base %d (%v), want %d, the revision its take-in pinned (H-A-154)", rev.Base, err, tc.base)
|
||||
}
|
||||
if got := kept(); !slices.Equal(got, tc.whileRuns) {
|
||||
t.Errorf("while the pipeline runs: kept %v, want %v", got, tc.whileRuns)
|
||||
}
|
||||
for _, k := range tc.whileRuns {
|
||||
if v, _, _, _ := ix.Pins.Get(ctx, k); !slices.Equal(mustRecords(t, v), []string{"pipeline.main.5"}) {
|
||||
t.Errorf("the pin of %s holds %s, want the pipeline's record", k, v)
|
||||
}
|
||||
}
|
||||
if err := ix.HeardPins(ctx, nil, &runner.PipelineEnded{Branch: "main", Pipeline: "main.5", Revision: 5}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ix.JudgePins(ctx)
|
||||
if got := kept(); !slices.Equal(got, tc.afterTheEnd) {
|
||||
t.Errorf("after the pipeline's end: kept %v, want %v", got, tc.afterTheEnd)
|
||||
}
|
||||
if keys, _ := ix.Pins.Keys(ctx); len(keys) > 0 {
|
||||
for _, k := range keys {
|
||||
if v, _, _, _ := ix.Pins.Get(ctx, k); len(mustRecords(t, v)) > 0 {
|
||||
t.Errorf("after the pipeline's end the pin of %s holds %s, want none", k, v)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -61,7 +61,8 @@ type RevisionSummary struct {
|
||||
Heads string `json:"heads,omitempty"`
|
||||
At *time.Time `json:"at,omitempty"`
|
||||
Entries int `json:"entries"`
|
||||
// Current says it is the trunk's newest revision, which is kept whatever pins it.
|
||||
// Current says it is the newest revision of its watched branch (CurrentRevisions, §579; H §743, H-A-135: no longer the
|
||||
// newest trunk-kind revision over every branch, and never one of the earlier forms), which is kept whatever pins it.
|
||||
Current bool `json:"current,omitempty"`
|
||||
}
|
||||
|
||||
@@ -84,7 +85,6 @@ func ListRevisions(ctx context.Context, o Objects, candidates bool) (RevisionsAn
|
||||
return out, fmt.Errorf("the objects %s cannot be listed: %w", CatalogRevisionsObjects, err)
|
||||
}
|
||||
listed := map[string]bool{}
|
||||
var newest int64
|
||||
entries := []string{}
|
||||
for _, n := range names {
|
||||
if strings.HasPrefix(n, "entry.") {
|
||||
@@ -103,17 +103,16 @@ func ListRevisions(ctx context.Context, o Objects, candidates bool) (RevisionsAn
|
||||
for _, ref := range rev.Entries {
|
||||
listed[EntryObject(ref.Digest)] = true
|
||||
}
|
||||
if rev.Kind == "trunk" && rev.Revision > newest {
|
||||
newest = rev.Revision
|
||||
}
|
||||
if candidates && rev.Kind != "candidate" {
|
||||
continue
|
||||
}
|
||||
out.Revisions = append(out.Revisions, RevisionSummary{Name: n, Kind: rev.Kind, Revision: rev.Revision, Source: rev.Source,
|
||||
Group: rev.Group, Heads: rev.Heads, At: rev.At, Entries: len(rev.Entries)})
|
||||
}
|
||||
current := CurrentRevisions(names)
|
||||
for i := range out.Revisions {
|
||||
out.Revisions[i].Current = out.Revisions[i].Kind == "trunk" && out.Revisions[i].Revision == newest
|
||||
_, cur := current[out.Revisions[i].Name]
|
||||
out.Revisions[i].Current = cur && out.Revisions[i].Kind == "trunk" && !earlierForm(out.Revisions[i].Name)
|
||||
}
|
||||
slices.SortFunc(out.Revisions, func(a, b RevisionSummary) int { return strings.Compare(a.Name, b.Name) })
|
||||
out.Entries = len(entries)
|
||||
|
||||
@@ -142,3 +142,37 @@ func (ix *Indexer) sayTaken(ctx context.Context, t meshcatalog.TakeInTaken) erro
|
||||
func PendingID(branch string, source int64) string {
|
||||
return takein.TakeInPendingEvent + "." + keys.EscapeName(branch) + "." + strconv.FormatInt(source, 10)
|
||||
}
|
||||
|
||||
// SourceToTake is the source-moved a read of the gate's source revision of a branch, through the seat mesh-delivery's verb
|
||||
// source, takes in as a heard one of that revision would be (design 47 H H-A-129), and false when the view keeps a revision of
|
||||
// the branch at or above it, so a revision is never taken in twice. It is the branch's first take-in, start true, exactly
|
||||
// when the view keeps no revision of the branch; otherwise its movers are judged against the newest kept (H-A-130). Every
|
||||
// repository the answer names is a head: the movers are the fingerprint changes alone.
|
||||
func (ix *Indexer) SourceToTake(ctx context.Context, branch string, a meshdelivery.SourceAnswer) (meshdelivery.SourceMovedEvent, bool, error) {
|
||||
e := meshdelivery.SourceMovedEvent{V: 1, Branch: branch, Source: a.Revision, Call: a.Call,
|
||||
Heads: make([]meshdelivery.SourceMovedEventHeadsItem, 0, len(a.Repositories))}
|
||||
if a.Revision <= 0 {
|
||||
return e, false, nil
|
||||
}
|
||||
if ix.Revisions == nil {
|
||||
return e, false, errors.New("the indexer keeps no catalog revisions")
|
||||
}
|
||||
names, err := ix.Revisions.Names(ctx)
|
||||
if err != nil {
|
||||
return e, false, fmt.Errorf("the objects %s: %w", CatalogRevisionsObjects, err)
|
||||
}
|
||||
var kept int64
|
||||
for _, n := range names {
|
||||
if r, ok := BranchRevisionOf(branch, n); ok && r > kept {
|
||||
kept = r
|
||||
}
|
||||
}
|
||||
if kept >= a.Revision {
|
||||
return e, false, nil
|
||||
}
|
||||
e.Start = kept == 0
|
||||
for _, r := range a.Repositories {
|
||||
e.Heads = append(e.Heads, meshdelivery.SourceMovedEventHeadsItem{Repository: r.Repository, Commit: r.Head})
|
||||
}
|
||||
return e, true, nil
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
@@ -121,6 +122,39 @@ func TestAFirstSourceRevisionIsTakenInWithNoMover(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 H §743, H-A-135: at a branch's take-in the earlier model's revisions (trunk.<n>, branch.<branch>.<n>) no record
|
||||
// pins are dropped, one a record pins stays, and the branch's newest revision is the current one, no longer the newest trunk.
|
||||
func TestATakeInDropsTheEarlierFormsNoRecordPins(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
ix, objects, _, f := takeInIndexer(t)
|
||||
for _, n := range []string{"trunk.527", "trunk.528", "branch.master.1"} {
|
||||
_, rev, _ := RevisionGroup(n)
|
||||
raw, _ := json.Marshal(meshcatalog.CatalogRevision{Revision: rev, Kind: "trunk", Source: 1000 + rev})
|
||||
if err := objects.Put(ctx, n, raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := ix.pin(ctx, "trunk.527", "pipeline.old"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
start := meshdelivery.SourceMovedEvent{V: 1, Branch: "main", Source: 1, Call: "seed.operator.20261120", Start: true,
|
||||
Heads: []meshdelivery.SourceMovedEventHeadsItem{{Repository: f.Repository, Commit: f.Merge}}}
|
||||
if _, err := ix.TakeInSource(ctx, start); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a, err := ListRevisions(ctx, objects, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var got []string
|
||||
for _, r := range a.Revisions {
|
||||
got = append(got, fmt.Sprintf("%s current=%t", r.Name, r.Current))
|
||||
}
|
||||
if want := []string{"main.1 current=true", "trunk.527 current=false"}; !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("kept %v, want %v: the unpinned earlier forms dropped, the pinned one kept, main.1 current", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 H §769: the take-in of a source revision moving the catalogue repository reads `predefined` from that commit's
|
||||
// trees, and one changing a predefined key rewrites it.
|
||||
func TestATakeInOfTheCatalogsCommitRewritesPredefined(t *testing.T) {
|
||||
@@ -222,3 +256,35 @@ func TestATakeInOnAFullStoreIsHeldAndSaidOnce(t *testing.T) {
|
||||
t.Errorf("held = %+v after the skip, want none", held)
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 H §743, §768, H-A-129, H-A-130: the gate's source revision read through its verb source is taken in as a heard
|
||||
// source-moved of it would be: start true when the view keeps no revision of the branch, every repository it names a head;
|
||||
// one the view keeps at or above is not taken again; a later one is judged against the newest kept, start false.
|
||||
func TestASourceRevisionReadFromTheGateIsTakenInOnce(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
ix, _, said, f := takeInIndexer(t)
|
||||
answer := meshdelivery.SourceAnswer{Branch: "main", Revision: 1, Call: "0123456789abcdef",
|
||||
Repositories: []meshdelivery.SourceHead{{Repository: f.Repository, Trunk: "main", Head: f.Merge}}}
|
||||
e, take, err := ix.SourceToTake(ctx, "main", answer)
|
||||
want := meshdelivery.SourceMovedEvent{V: 1, Branch: "main", Source: 1, Call: answer.Call, Start: true,
|
||||
Heads: []meshdelivery.SourceMovedEventHeadsItem{{Repository: f.Repository, Commit: f.Merge}}}
|
||||
if err != nil || !take || !reflect.DeepEqual(e, want) {
|
||||
t.Fatalf("a branch the view keeps none of answered %+v, %t (%v); want %+v taken", e, take, err, want)
|
||||
}
|
||||
if taken, err := ix.TakeInSource(ctx, e); err != nil || len(taken.Entries) != 0 || taken.Revision != 1 {
|
||||
t.Fatalf("the first revision read from the gate said %+v (%v), want revision 1 with no mover", taken, err)
|
||||
}
|
||||
if _, take, err := ix.SourceToTake(ctx, "main", answer); take || err != nil {
|
||||
t.Errorf("a revision the view keeps is taken again (%v)", err)
|
||||
}
|
||||
if _, take, _ := ix.SourceToTake(ctx, "nox-mesh", meshdelivery.SourceAnswer{Branch: "nox-mesh"}); take {
|
||||
t.Error("a branch the gate keeps no revision of is taken in")
|
||||
}
|
||||
answer.Revision, answer.Call = 2, "1123456789abcdef"
|
||||
if e, take, err := ix.SourceToTake(ctx, "main", answer); err != nil || !take || e.Start || e.Source != 2 {
|
||||
t.Errorf("a later revision answered %+v, %t (%v); want revision 2 taken, start false", e, take, err)
|
||||
}
|
||||
if len(*said) != 1 {
|
||||
t.Errorf("said %d take-in.taken, want 1", len(*said))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -182,7 +182,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
|
||||
// Three invokes: the file of a registered module's manifest and lock files (registered.go), the forge's trees (the named paths' objects, since the indexer is the one reader of a manifest's paths:
|
||||
// ADR 0499 clauses 2 and 3, ADR 0495 clause 1; and the module directories of the trunk, issue 1900) and whether a
|
||||
// commit is in another (issue 1900).
|
||||
if want := []string{"seat:git.trees", "seat:git.tags", "seat:go-package-registry.versions", "seat:npm-package-registry.versions", "seat:git.file", "seat:git.contains", "seat:git.branch", "seat:mesh-module-assigner.assignments", "seat:mesh-module-assigner.placement", "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence", "seat:node-build-go.steps",
|
||||
if want := []string{"seat:git.trees", "seat:git.tags", "seat:go-package-registry.versions", "seat:npm-package-registry.versions", "seat:git.file", "seat:git.contains", "seat:git.branch", "seat:mesh-delivery.source", "seat:mesh-module-assigner.assignments", "seat:mesh-module-assigner.placement", "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence", "seat:node-build-go.steps",
|
||||
"seat:node-build-node.steps", "seat:node-build-image.steps", "mesh-delivery-builder.builder_versions"}; !reflect.DeepEqual(m.Invokes, want) {
|
||||
t.Errorf("invokes = %v, want %v", m.Invokes, want)
|
||||
}
|
||||
|
||||
@@ -152,6 +152,7 @@
|
||||
"seat:git.file",
|
||||
"seat:git.contains",
|
||||
"seat:git.branch",
|
||||
"seat:mesh-delivery.source",
|
||||
"seat:mesh-module-assigner.assignments",
|
||||
"seat:mesh-module-assigner.placement",
|
||||
"seat:mesh-hand-acts.hand-act",
|
||||
|
||||
@@ -647,19 +647,32 @@ func aRuntimeDeniedTheMarksHearsNoMarkedCall(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
asker, askerRefused := bus.dial("a.asker")
|
||||
// The caller-named subject is process.Call's first: mesh.mod.<module>.call.<tool>.<caller's token>.
|
||||
for _, subject := range []string{"mesh.mod.shell.tool.execute", "mesh.mod.shell.call.execute.asker~a", "mesh.mod.shell.tool.read"} {
|
||||
// The caller-named subject is process.Call's first: mesh.mod.<module>.call.<tool>.<caller's token>, the token the
|
||||
// composer grants (buscompose.CallerToken: `a~asker` for the user a.asker).
|
||||
for _, subject := range []string{"mesh.mod.shell.tool.execute", "mesh.mod.shell.call.execute." + buscompose.CallerToken("a.asker"), "mesh.mod.shell.tool.read"} {
|
||||
if err := asker.Publish(subject, []byte(`{}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// The server says a refused publish asynchronously, in the order of the publishes: a probe the asker may not publish
|
||||
// (another caller's token) is refused after every refusal of the calls above, so waiting for it waits on what is asserted.
|
||||
const probe = "mesh.mod.shell.call.execute.a~someone"
|
||||
if err := asker.Publish(probe, []byte(`{}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := asker.Flush(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case e := <-askerRefused:
|
||||
t.Fatalf("the asker may not call: %s", e)
|
||||
default:
|
||||
for probed := false; !probed; {
|
||||
select {
|
||||
case e := <-askerRefused:
|
||||
if !strings.Contains(e, `"`+probe+`"`) {
|
||||
t.Fatalf("the asker may not call: %s", e)
|
||||
}
|
||||
probed = true
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("the probe's refusal never came, so the calls' refusals are not known")
|
||||
}
|
||||
}
|
||||
select {
|
||||
case m := <-heard:
|
||||
|
||||
@@ -110,6 +110,34 @@ func (c creating) RaiseMaxBytes(stream string, to int64) (int64, bool, error) {
|
||||
return from, true, nil
|
||||
}
|
||||
|
||||
// SourceFilter brings the filter subject of a held stream's source from `source` to `filter` and changes nothing else
|
||||
// (design 47 H H-A-145): it reads the stream's configuration and updates with the rest as it was, so its messages stay. A
|
||||
// stream with no source from `source`, or one already at `filter`, is left as it is.
|
||||
func (c creating) SourceFilter(stream, source, filter string) (string, bool, error) {
|
||||
ctx, cancel := context.WithTimeout(c.ctx, requestWithin)
|
||||
defer cancel()
|
||||
s, err := c.js.Stream(ctx, stream)
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
cfg := s.CachedInfo().Config
|
||||
i := slices.IndexFunc(cfg.Sources, func(src *jetstream.StreamSource) bool { return src != nil && src.Name == source })
|
||||
if i < 0 || cfg.Sources[i].FilterSubject == filter {
|
||||
if i < 0 {
|
||||
return "", false, nil
|
||||
}
|
||||
return cfg.Sources[i].FilterSubject, false, nil
|
||||
}
|
||||
from := cfg.Sources[i].FilterSubject
|
||||
src := *cfg.Sources[i]
|
||||
src.FilterSubject = filter
|
||||
cfg.Sources[i] = &src
|
||||
if _, err := c.js.UpdateStream(ctx, cfg); err != nil {
|
||||
return from, false, err
|
||||
}
|
||||
return from, true, nil
|
||||
}
|
||||
|
||||
// AddFilters adds filter subjects to a held consumer and changes nothing else.
|
||||
func (c creating) AddFilters(stream, name string, add []string) error {
|
||||
ctx, cancel := context.WithTimeout(c.ctx, requestWithin)
|
||||
|
||||
@@ -147,7 +147,10 @@ func TestTheGrantsPartWhole(t *testing.T) {
|
||||
consumers := []any{}
|
||||
names := st.ConsumerNames(ctx)
|
||||
for name := range names.Name() {
|
||||
c, _ := st.Consumer(ctx, name)
|
||||
c, err := st.Consumer(ctx, name)
|
||||
if err != nil {
|
||||
continue // deleted between the listing and its read: the bus module's tool lists what it can read
|
||||
}
|
||||
cfg := c.CachedInfo().Config
|
||||
consumers = append(consumers, map[string]any{"name": name, "filters": cfg.FilterSubjects, "max_deliver": cfg.MaxDeliver})
|
||||
}
|
||||
@@ -611,6 +614,35 @@ func TestTheGrantsPartWhole(t *testing.T) {
|
||||
if again, _ := stream.Info(ctx); !slices.Equal(again.Config.Subjects, []string{"a.>", "b.>"}) {
|
||||
t.Errorf("subjects after adding a held one: %v", again.Config.Subjects)
|
||||
}
|
||||
// A kept stream's source filter is brought to its definition's in place: the stream is not made again, its settings
|
||||
// stay, and an equal filter or a source of another stream changes nothing (design 47 H H-A-145, §746).
|
||||
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "KEPTSRC", Subjects: []string{"ev.>"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kept, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "KEPT", MaxAge: time.Hour,
|
||||
Sources: []*jetstream.StreamSource{{Name: "KEPTSRC", FilterSubject: "ev.ended.*.*.*"}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
keptBefore, _ := kept.Info(ctx)
|
||||
for _, row := range []struct {
|
||||
source, filter, from, holds string
|
||||
changed bool
|
||||
}{
|
||||
{"KEPTSRC", "ev.ended.*.*", "ev.ended.*.*.*", "ev.ended.*.*", true},
|
||||
{"KEPTSRC", "ev.ended.*.*", "ev.ended.*.*", "ev.ended.*.*", false},
|
||||
{"OTHER", "ev.other", "", "ev.ended.*.*", false},
|
||||
} {
|
||||
from, changed, err := c.SourceFilter("KEPT", row.source, row.filter)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now, _ := kept.Info(ctx)
|
||||
if from != row.from || changed != row.changed || len(now.Config.Sources) != 1 || now.Config.Sources[0].FilterSubject != row.holds ||
|
||||
now.Config.MaxAge != keptBefore.Config.MaxAge || !now.Created.Equal(keptBefore.Created) {
|
||||
t.Errorf("source %s to %s: from %q changed %v, config %+v created %v", row.source, row.filter, from, changed, now.Config, now.Created)
|
||||
}
|
||||
}
|
||||
// The named asks' purge takes one subject's messages and no other's, and a named worker's delete is said done also
|
||||
// where the bus holds none (design 47 G §544).
|
||||
for _, subj := range []string{"b.one", "b.two"} {
|
||||
|
||||
@@ -202,6 +202,9 @@ func (b *fakeBus) AddSubjects(stream string, add []string) error {
|
||||
return errors.New("stream not found")
|
||||
}
|
||||
|
||||
// SourceFilter is the bus's update of a held stream's source filter: the fake holds no sources, so nothing changes.
|
||||
func (b *fakeBus) SourceFilter(string, string, string) (string, bool, error) { return "", false, nil }
|
||||
|
||||
// RaiseMaxBytes is the bus's raise of a held stream's size cap: only to a larger cap, an unbounded stream and every other
|
||||
// setting kept.
|
||||
func (b *fakeBus) RaiseMaxBytes(stream string, to int64) (int64, bool, error) {
|
||||
@@ -684,10 +687,10 @@ func TestGrants(t *testing.T) {
|
||||
if n := strings.Count(string(src), "CreateStream"); (path == createOnly && n != 1) || (path != createOnly && n != 0) {
|
||||
t.Errorf("%s holds CreateStream %d times: only create.go creates, once", path, n)
|
||||
}
|
||||
// The two updates of a stream: adding subjects to a held one (the operator, 2026-11-05) and raising its size cap
|
||||
// (2026-11-09), both in create.go.
|
||||
if n := strings.Count(string(src), "UpdateStream("); (path == createOnly && n != 2) || (path != createOnly && n != 0) {
|
||||
t.Errorf("%s holds UpdateStream %d times: only create.go updates a stream, twice, to add subjects and to raise a size cap", path, n)
|
||||
// The three updates of a stream: adding subjects to a held one (the operator, 2026-11-05), raising its size cap
|
||||
// (2026-11-09) and bringing a kept stream's source filter to its definition's (design 47 H H-A-145), all in create.go.
|
||||
if n := strings.Count(string(src), "UpdateStream("); (path == createOnly && n != 3) || (path != createOnly && n != 0) {
|
||||
t.Errorf("%s holds UpdateStream %d times: only create.go updates a stream, three times, to add subjects, to raise a size cap and to bring a kept stream's source filter", path, n)
|
||||
}
|
||||
// The one object bucket: the part's own, of the user list sealed to the bus node, in sealed.go (ADR 0547 clause 7).
|
||||
if path == sealedOnly {
|
||||
|
||||
@@ -1765,7 +1765,8 @@ const WatchmanModule = "mesh-delivery-watchman"
|
||||
// 2026-10-26 08:54): its create, and its information to read back what it holds. Each by its own name: a name that
|
||||
// is not one plain subject token is left out, since a wildcard there would be every stream. Its update is granted for the one
|
||||
// acts the part does on a held stream: adding the subjects a manifest's new events need (grants.Creator.AddSubjects) and
|
||||
// raising its size cap to the manifest's when larger (grants.Creator.RaiseMaxBytes; the operator, 2026-11-09); it never
|
||||
// raising its size cap to the manifest's when larger (grants.Creator.RaiseMaxBytes; the operator, 2026-11-09), and bringing
|
||||
// a kept stream's source filter to its definition's (grants.Creator.SourceFilter; design 47 H H-A-145); it never
|
||||
// removes a subject, lowers a limit or recreates the stream, and never deletes or purges one.
|
||||
func CreateStreamGrants(streams []string) []string {
|
||||
var out []string
|
||||
|
||||
@@ -21,6 +21,9 @@ func (c *askCreator) AddSubjects(string, []string) error { return nil }
|
||||
func (c *askCreator) RaiseMaxBytes(string, int64) (int64, bool, error) {
|
||||
return 0, false, nil
|
||||
}
|
||||
func (c *askCreator) SourceFilter(string, string, string) (string, bool, error) {
|
||||
return "", false, nil
|
||||
}
|
||||
|
||||
// A seat whose queues came with G (mesh-builds) gets its work-queue ask stream from the seat list, as the controller made one for
|
||||
// every seat before G; a seat that accepts nothing gets none, and a stream the bus holds is never made again (lead-47, 2026-11-15).
|
||||
|
||||
@@ -69,6 +69,10 @@ type Creator interface {
|
||||
// changes nothing else (the operator, 2026-11-09 09:07: "raise only"). It answers the cap it held and whether it
|
||||
// raised; a cap that is larger or equal, or an unbounded stream, is left as it is. It never lowers any limit.
|
||||
RaiseMaxBytes(stream string, to int64) (from int64, raised bool, err error)
|
||||
// SourceFilter brings the filter subject of a held stream's source from the stream `source` to `filter`, keeping its
|
||||
// messages and every other setting, the stream not made again (design 47 H H-A-145). It answers the filter it held and
|
||||
// whether it changed; a stream with no source from `source` is left as it is.
|
||||
SourceFilter(stream, source, filter string) (from string, changed bool, err error)
|
||||
}
|
||||
|
||||
// Raise is one held stream's size cap raised to the manifest's, said in the event and the journal.
|
||||
|
||||
@@ -1,8 +1,11 @@
|
||||
package grants
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"mesh-grants/internal/buscompose"
|
||||
)
|
||||
|
||||
@@ -39,3 +42,70 @@ func TestTheGrantsAccountMayCreateEachKeptEventStream(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// sourcing is a Creator holding kept streams by their source filter and message count: SourceFilter updates a held one in
|
||||
// place, so its messages stay; CreateDeclared makes one the bus lacks.
|
||||
type sourcing struct {
|
||||
Creator
|
||||
filters map[string]string // by stream, its source filter from EVENTS
|
||||
messages map[string]int
|
||||
made []string
|
||||
}
|
||||
|
||||
func (s *sourcing) SourceFilter(stream, source, filter string) (string, bool, error) {
|
||||
from, ok := s.filters[stream]
|
||||
if !ok || source != buscompose.EventsStream || from == filter {
|
||||
return from, false, nil
|
||||
}
|
||||
s.filters[stream] = filter
|
||||
return from, true, nil
|
||||
}
|
||||
|
||||
func (s *sourcing) CreateDeclared(cfg jetstream.StreamConfig) error {
|
||||
s.made = append(s.made, cfg.Name)
|
||||
s.filters[cfg.Name], s.messages[cfg.Name] = cfg.Sources[0].FilterSubject, 0
|
||||
return nil
|
||||
}
|
||||
|
||||
// H-A-145 (design 47 H §746): a held kept stream whose source filter the definition changed is brought to the definition's
|
||||
// filter in place, its messages kept and the change said with both filters; one at the definition's filter is untouched; one
|
||||
// the bus lacks is made.
|
||||
func TestAKeptStreamFollowsItsDefinitionsSourceFilter(t *testing.T) {
|
||||
const ended = "mesh.seat.mesh-pipeline-runner.event.pipeline-ended.*.*"
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
held string // the held stream's source filter; "" for a stream the bus lacks
|
||||
made bool
|
||||
moved string
|
||||
filter string
|
||||
messages, keptMsgs int
|
||||
}{
|
||||
{name: "held at the definition's filter: untouched", held: ended, filter: ended, messages: 604, keptMsgs: 604},
|
||||
{name: "held at an earlier filter: brought to the definition's, messages kept", held: ended + ".*", filter: ended, messages: 604, keptMsgs: 604,
|
||||
moved: "SEAT_mesh-pipeline-runner_pipeline-ended: source filter " + ended + ".* brought to " + ended + ", its messages kept"},
|
||||
{name: "not held: made", made: true, filter: ended},
|
||||
} {
|
||||
t.Run(row.name, func(t *testing.T) {
|
||||
k := buscompose.KeptEvent{Seat: "mesh-pipeline-runner", Event: "pipeline-ended", Stream: "SEAT_mesh-pipeline-runner_pipeline-ended", Filter: ended, MaxAgeDays: 90, MaxMiB: 192}
|
||||
c := &sourcing{filters: map[string]string{}, messages: map[string]int{}}
|
||||
held := map[string]Limits{}
|
||||
if row.held != "" {
|
||||
c.filters[k.Stream], c.messages[k.Stream] = row.held, row.messages
|
||||
held[k.Stream] = Limits{}
|
||||
}
|
||||
made, moved, notMade := createKept(c, &creations{}, []buscompose.KeptEvent{k}, held, false)
|
||||
if len(notMade) != 0 {
|
||||
t.Fatalf("not made: %v", notMade)
|
||||
}
|
||||
if (len(made) == 1) != row.made || len(c.made) != len(made) {
|
||||
t.Errorf("made %v (bus %v), want made %v", made, c.made, row.made)
|
||||
}
|
||||
if got := strings.Join(moved, "; "); got != row.moved {
|
||||
t.Errorf("moved %q, want %q", got, row.moved)
|
||||
}
|
||||
if c.filters[k.Stream] != row.filter || c.messages[k.Stream] != row.keptMsgs {
|
||||
t.Errorf("the stream holds filter %q and %d messages, want %q and %d", c.filters[k.Stream], c.messages[k.Stream], row.filter, row.keptMsgs)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -65,14 +65,24 @@ func keptOutOfHearing(kept []buscompose.KeptEvent, watches []buscompose.Seat, co
|
||||
}
|
||||
|
||||
// createKept makes each kept event's stream the bus lacks, from the shared events stream's messages of its subject, and
|
||||
// says what it made. A stream the bus holds is left as it is; one made at this seat list and gone since is made again only on
|
||||
// a hand assert (the same rule as a manifest's streams: made once per definition). held is what the bus held at this pass.
|
||||
func createKept(c Creator, done *creations, kept []buscompose.KeptEvent, held map[string]Limits, force bool) (made, notMade []string) {
|
||||
// says what it made. A stream the bus holds keeps its messages and is not made again; its source filter is brought to the
|
||||
// definition's in place when the definition changed the event's subject (design 47 H H-A-145, §746: the runner's
|
||||
// pipeline-ended moved from three parts to two and its kept stream took nothing in), said in moved with the held and the
|
||||
// composed filter. One made at this seat list and gone since is made again only on a hand assert (the same rule as a
|
||||
// manifest's streams: made once per definition). held is what the bus held at this pass.
|
||||
func createKept(c Creator, done *creations, kept []buscompose.KeptEvent, held map[string]Limits, force bool) (made, moved, notMade []string) {
|
||||
for _, k := range kept {
|
||||
cfg := k.StreamConfig()
|
||||
if _, ok := held[k.Stream]; ok {
|
||||
from, changed, err := c.SourceFilter(k.Stream, buscompose.EventsStream, k.Filter)
|
||||
switch {
|
||||
case err != nil:
|
||||
notMade = append(notMade, k.Stream+": its source filter is not brought to "+k.Filter+": "+err.Error()+" (tried again at the next pass)")
|
||||
case changed:
|
||||
moved = append(moved, k.Stream+": source filter "+from+" brought to "+k.Filter+", its messages kept")
|
||||
}
|
||||
continue
|
||||
}
|
||||
cfg := k.StreamConfig()
|
||||
digest := fmt.Sprintf("%v", cfg.Sources[0].FilterSubject) + fmt.Sprint(k.Last, k.MaxAgeDays, k.MaxMiB)
|
||||
if !force && done.madeAt(k.Stream) == digest {
|
||||
notMade = append(notMade, k.Stream+": made at this definition and gone since; grants_assert makes it again")
|
||||
@@ -87,7 +97,7 @@ func createKept(c Creator, done *creations, kept []buscompose.KeptEvent, held ma
|
||||
notMade = append(notMade, k.Stream+": "+err.Error()+" (tried again at the next event)")
|
||||
}
|
||||
}
|
||||
return made, notMade
|
||||
return made, moved, notMade
|
||||
}
|
||||
|
||||
// eventSubjectsOf is, by seat, each event's subject pattern (`running.<node>.<module>`) of the events whose subject carries
|
||||
|
||||
@@ -565,11 +565,21 @@ func (p *Part) Compose() Result {
|
||||
}
|
||||
forced := ask.Assert
|
||||
if p.Create != nil && heldErr == nil {
|
||||
made, notMade := createKept(p.Create, &p.made, in.Kept, held, forced != "")
|
||||
made, moved, notMade := createKept(p.Create, &p.made, in.Kept, held, forced != "")
|
||||
if len(made) > 0 {
|
||||
res.Asserted = append(res.Asserted, "the seats' kept events: created "+strings.Join(made, ", "))
|
||||
p.say(&res, EventAsserted, map[string]any{"module": Module, "created": made})
|
||||
}
|
||||
for _, m := range moved {
|
||||
// Said in the journal with both filters (H-A-145).
|
||||
res.Asserted = append(res.Asserted, "the seats' kept events: "+m)
|
||||
if p.Logf != nil {
|
||||
p.Logf("kept stream %s", m)
|
||||
}
|
||||
}
|
||||
if len(moved) > 0 {
|
||||
p.say(&res, EventAsserted, map[string]any{"module": Module, "source-filters": moved})
|
||||
}
|
||||
for _, why := range notMade {
|
||||
res.NotAsserted = append(res.NotAsserted, "kept event "+why)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user