Merge pull request 'G starts: grants composes the first rights from the trunk, the indexer answers at once and knows its build, unpublished packages build at the source commit' (#217) from fix/47-G-start-2 into main
This commit was merged in pull request #217.
This commit is contained in:
@@ -31,7 +31,7 @@ func (p *passer) actingOf(module string, released map[string]bool) func(node str
|
||||
}
|
||||
for _, e := range p.entryEdges(node, module) {
|
||||
if acting.Held(e.Step) && !released[acting.Key(node, module, e.Module)+"@"+e.Fp] {
|
||||
out = append(out, assign.ActingEdge{Node: e.Node, Part: e.Module, Fp: e.Fp, Step: e.Step})
|
||||
out = append(out, assign.ActingEdge{Node: e.Node, Part: e.Module, Fp: e.Fp, Step: e.Step, Seat: e.Seat})
|
||||
}
|
||||
}
|
||||
return out
|
||||
@@ -103,7 +103,10 @@ func (p *passer) actingWaits(ctx context.Context, n assign.Nodes, wanted map[str
|
||||
continue
|
||||
}
|
||||
key := acting.Key(ref.Node, ref.Module, e.Module)
|
||||
if n.Runs(assign.ActingEdge{Node: e.Node, Part: e.Module, Fp: e.Fp, Step: e.Step}) {
|
||||
if took, runs := n.ActingRuns(assign.ActingEdge{Node: e.Node, Part: e.Module, Fp: e.Fp, Step: e.Step, Seat: e.Seat}); runs {
|
||||
if took != e.Fp && kept[key] != nil {
|
||||
p.logf("the wait of %s on %s for %s ends: %s runs %s, which serves the seat %s, though its newest is %s (design 47 G A-426)", ref.Module, ref.Node, e.Module, e.Node, took, e.Seat, e.Fp)
|
||||
}
|
||||
continue
|
||||
}
|
||||
standing[key] = true
|
||||
@@ -124,6 +127,7 @@ func (p *passer) actingWaits(ctx context.Context, n assign.Nodes, wanted map[str
|
||||
continue
|
||||
}
|
||||
p.put(ctx, buckets.Waits(), key, func() ([]byte, error) { return acting.Encode(w) })
|
||||
p.logf("the want of %s on %s is held: its acting part %s (step %s) does not run the version %s on %s; the wait %s ends when it does, or by assigner_release", ref.Module, ref.Node, e.Module, e.Step, e.Fp, e.Node, key)
|
||||
}
|
||||
}
|
||||
p.actingCycles(g.computed.Entries, engines, held, n)
|
||||
@@ -138,7 +142,9 @@ func (p *passer) actingWaits(ctx context.Context, n assign.Nodes, wanted map[str
|
||||
how := acting.HowRan
|
||||
if w.Released {
|
||||
how = acting.HowReleased
|
||||
} else if e, ok := p.edgeOf(w); !ok || !n.Runs(e) {
|
||||
} else if e, ok := p.edgeOf(w); !ok {
|
||||
how = acting.HowReplaced
|
||||
} else if _, runs := n.ActingRuns(e); !runs {
|
||||
how = acting.HowReplaced
|
||||
}
|
||||
if !w.EndSaid {
|
||||
@@ -184,7 +190,7 @@ func (p *passer) actingCycles(entries map[string]seat.AssignmentEntry, engines m
|
||||
released := p.actingReleased()
|
||||
cycles := assigngraph.HeldCycles(slices.Collect(maps.Values(entries)), func(e seat.AssignmentEntry, edge seat.Edge) bool {
|
||||
return held[e.Node+"/"+e.Module] && engines[e.Node] != e.Module && edge.Kind == "acting" && acting.Held(edge.Step) &&
|
||||
!released[acting.Key(e.Node, e.Module, edge.Module)+"@"+edge.Fp] && !n.Runs(assign.ActingEdge{Node: edge.Node, Part: edge.Module, Fp: edge.Fp, Step: edge.Step})
|
||||
!released[acting.Key(e.Node, e.Module, edge.Module)+"@"+edge.Fp] && !actingRunsOf(n, edge)
|
||||
})
|
||||
raised := map[string]bool{}
|
||||
for _, at := range slices.Sorted(maps.Keys(cycles)) {
|
||||
@@ -223,3 +229,9 @@ func (p *passer) actingCycles(entries map[string]seat.AssignmentEntry, engines m
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// actingRunsOf says whether an assignment edge's acting part runs a build serving it (A-426).
|
||||
func actingRunsOf(n assign.Nodes, edge seat.Edge) bool {
|
||||
_, runs := n.ActingRuns(assign.ActingEdge{Node: edge.Node, Part: edge.Module, Fp: edge.Fp, Step: edge.Step, Seat: edge.Seat})
|
||||
return runs
|
||||
}
|
||||
|
||||
@@ -36,7 +36,9 @@ func TestAnEntryWaitsForItsActingPartAndTheWaitIsKeptSaidAndEnded(t *testing.T)
|
||||
p.graph = &graphKeeper{}
|
||||
p.graph.computed.Revision.Entries = []meshwants.NodeEntryRef{{Node: "ace", Module: "nats", Digest: "d1"}}
|
||||
p.graph.computed.Entries = map[string]meshwants.AssignmentEntry{"d1": entry}
|
||||
n := assign.Nodes{Online: map[string]bool{"ace": true}, Running: map[string]map[string]nodeengine.Running{"ace": {"mesh-host": {Fingerprint: "src2:old"}}}}
|
||||
// The old build of mesh-host serves no seat of the edge here, so the entry waits (A-426: one serving it ends the wait).
|
||||
notServing := func(nodeengine.Running, string) (bool, bool) { return false, true }
|
||||
n := assign.Nodes{Online: map[string]bool{"ace": true}, Running: map[string]map[string]nodeengine.Running{"ace": {"mesh-host": {Fingerprint: "src2:old"}}}, Serves: notServing}
|
||||
wanted := map[string]map[string]mwc.Want{"nats": {"ace": {Fp: "src2:older"}}}
|
||||
ctx := context.Background()
|
||||
|
||||
@@ -149,7 +151,7 @@ func TestTheNodeEngineIsWantedFirstAndAWantHeldOnACycleIsAFinding(t *testing.T)
|
||||
}
|
||||
p.graph.computed.Entries = entries
|
||||
n := assign.Nodes{Online: map[string]bool{"ace": true}, Running: map[string]map[string]nodeengine.Running{"ace": {
|
||||
"mesh-host": {Fingerprint: "src2:old"}, "assigner": {Fingerprint: "src2:old"}}}}
|
||||
"mesh-host": {Fingerprint: "src2:old"}, "assigner": {Fingerprint: "src2:old"}}}, Serves: func(nodeengine.Running, string) (bool, bool) { return false, true }}
|
||||
wanted := map[string]map[string]mwc.Want{}
|
||||
if got := p.actingOf("mesh-host", nil)("ace"); len(got) != 0 {
|
||||
t.Errorf("the node-engine's want is held by %+v, want by no acting edge", got)
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io/fs"
|
||||
"os"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
mmac "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
|
||||
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
|
||||
|
||||
"mesh-delivery-assigner/internal/judge"
|
||||
)
|
||||
|
||||
// Which proposals the assigner asks of the operator is its setting ask-operator-for (design 47 G A-425, A-425a): a list of change
|
||||
// kinds, set at mesh level with mesh-config's config, its default take-off, remove-node and stops. A proposal none of whose changes
|
||||
// is listed is applied at once when its verdict passes, said in the log and by assignments-changed; one with a listed kind is
|
||||
// asked as before. The setting is read from the file the manifest declares with the content ${config:ask-operator-for}, which the
|
||||
// environment names (A-112), at every pass, and a change of it is said.
|
||||
|
||||
// KindStops is the change kind of a verdict whose answers say a node would stop running a module.
|
||||
const KindStops = "stops"
|
||||
|
||||
// defaultAskOperatorFor is the setting's default.
|
||||
var defaultAskOperatorFor = []string{mmac.OpTakeOff, mmac.OpRemoveNode, KindStops}
|
||||
|
||||
// askKinds are the kinds the setting may name.
|
||||
var askKinds = []string{mmac.OpPlace, mmac.OpTakeOff, mmac.OpPin, mmac.OpUnpin, mmac.OpHold, mmac.OpUnhold, mmac.OpDecide, mmac.OpConfig,
|
||||
mmac.OpAddNode, mmac.OpRemoveNode, KindStops}
|
||||
|
||||
// parseAskOperatorFor reads the setting's value: kinds separated by commas or spaces; empty is the default. A kind it does not
|
||||
// know is answered apart and named nowhere in the list.
|
||||
func parseAskOperatorFor(raw string) (kinds, unknown []string) {
|
||||
fields := strings.FieldsFunc(raw, func(r rune) bool { return r == ',' || r == ' ' || r == '\n' || r == '\t' })
|
||||
if len(fields) == 0 {
|
||||
return slices.Clone(defaultAskOperatorFor), nil
|
||||
}
|
||||
for _, f := range fields {
|
||||
f = strings.ToLower(strings.TrimSpace(f))
|
||||
switch {
|
||||
case !slices.Contains(askKinds, f):
|
||||
unknown = append(unknown, f)
|
||||
case !slices.Contains(kinds, f):
|
||||
kinds = append(kinds, f)
|
||||
}
|
||||
}
|
||||
slices.Sort(kinds)
|
||||
return kinds, unknown
|
||||
}
|
||||
|
||||
// readAskOperatorFor reads the setting's file: the default for no path or no file.
|
||||
func readAskOperatorFor(path string) (kinds, unknown []string, err error) {
|
||||
if path == "" {
|
||||
k, u := parseAskOperatorFor("")
|
||||
return k, u, nil
|
||||
}
|
||||
raw, err := os.ReadFile(path)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
k, u := parseAskOperatorFor("")
|
||||
return k, u, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
k, u := parseAskOperatorFor(string(raw))
|
||||
return k, u, nil
|
||||
}
|
||||
|
||||
// askOperatorFor is the setting as this pass read it: the file read again each pass, a change said once, a file that cannot be
|
||||
// read keeping what was read last (the default before any).
|
||||
func (p *passer) askOperatorFor() []string {
|
||||
kinds, unknown, err := readAskOperatorFor(p.askForFile)
|
||||
if err != nil {
|
||||
if p.askFor == nil {
|
||||
p.askFor = slices.Clone(defaultAskOperatorFor)
|
||||
}
|
||||
p.logf("the setting ask-operator-for cannot be read from %s, so %s stands: %v", p.askForFile, strings.Join(p.askFor, ", "), err)
|
||||
return p.askFor
|
||||
}
|
||||
if !slices.Equal(kinds, p.askFor) {
|
||||
words := "the setting ask-operator-for is " + strings.Join(kinds, ", ") + ": a proposal with a change of another kind only is applied with no ask"
|
||||
if len(unknown) > 0 {
|
||||
words += "; kinds it names that are none are left out: " + strings.Join(unknown, ", ")
|
||||
}
|
||||
p.logf("%s", words)
|
||||
p.askFor = kinds
|
||||
}
|
||||
return p.askFor
|
||||
}
|
||||
|
||||
// asksFor says whether a proposal of these change kinds is asked of the operator.
|
||||
func (p *passer) asksFor(kinds []string) bool {
|
||||
set := p.askOperatorFor()
|
||||
for _, k := range kinds {
|
||||
if slices.Contains(set, k) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// proposalKinds are the change kinds of a proposal whose verdict is v: each change's op, and stops when an answer says a node
|
||||
// would stop running a module.
|
||||
func proposalKinds(prop mma.Proposal, v mma.Verdict) []string {
|
||||
var out []string
|
||||
for _, c := range prop.Changes {
|
||||
if c.Op != "" && !slices.Contains(out, c.Op) {
|
||||
out = append(out, c.Op)
|
||||
}
|
||||
}
|
||||
if len(judge.Stops(v)) > 0 {
|
||||
out = append(out, KindStops)
|
||||
}
|
||||
slices.Sort(out)
|
||||
return out
|
||||
}
|
||||
@@ -4,6 +4,9 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"maps"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
@@ -97,7 +100,7 @@ func (r *releasedVersions) put(data []byte) error {
|
||||
}
|
||||
parts := map[string]string{}
|
||||
for _, a := range e.Artifacts {
|
||||
parts[e.Module+":"+a.Artifact] = a.Digest
|
||||
parts[partRef(e.Module, a.Artifact)] = a.Digest
|
||||
}
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
@@ -116,6 +119,41 @@ func (r *releasedVersions) put(data []byte) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// withParts is the version of module whose release named exactly parts (`<module>:<artifact>` -> digest), the newest such when
|
||||
// more than one did; false when none did or parts is empty (design 47 G A-421).
|
||||
func (r *releasedVersions) withParts(module string, parts map[string]string) (assign.KeptBuild, bool) {
|
||||
if len(parts) == 0 {
|
||||
return assign.KeptBuild{}, false
|
||||
}
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
var out assign.KeptBuild
|
||||
found := false
|
||||
for key, k := range r.seen {
|
||||
if m, _, ok := strings.Cut(key, "@"); !ok || m != module || !maps.Equal(k.Parts, parts) {
|
||||
continue
|
||||
}
|
||||
if !found || compareVersion(k.Version, out.Version) > 0 {
|
||||
out, found = k, true
|
||||
}
|
||||
}
|
||||
return out, found
|
||||
}
|
||||
|
||||
// madeOf is what module's done build at fp made, by `<module>:<artifact>`, as its build.ended names it; none for a build not heard
|
||||
// or not done (design 47 G A-421).
|
||||
func (b *builtEnds) madeOf(module, fp string) map[string]string {
|
||||
e, ok := b.endOf(module, fp)
|
||||
if !ok || e.Outcome != builds.BuildDone {
|
||||
return nil
|
||||
}
|
||||
parts := map[string]string{}
|
||||
for name, digest := range e.Made {
|
||||
parts[module+":"+name] = digest
|
||||
}
|
||||
return parts
|
||||
}
|
||||
|
||||
// of is the version of module released from the source revision, with its parts' digests; false before it is heard.
|
||||
func (r *releasedVersions) of(module string, source int64) (assign.KeptBuild, bool) {
|
||||
r.mu.Lock()
|
||||
@@ -143,7 +181,7 @@ func hearBuilds(ctx context.Context, js jetstream.JetStream, ends *builtEnds, wa
|
||||
logf("acknowledging %s: %v", m.Subject(), err)
|
||||
}
|
||||
wake()
|
||||
}, logf)
|
||||
}, logf, process.FromFirst()) // read whole from its first message whether the bus keeps it last per subject or by age (A-181, A-233)
|
||||
if err != nil {
|
||||
nh.NotHeard(buildsStreamName, "what the builder built is not heard until the assigner starts again, so no want names a build's parts or its keeping node and no put goes on", err)
|
||||
return
|
||||
@@ -168,3 +206,30 @@ func hearTaken(hub *process.EventHub, taken *takenSources, wake func(), logf fun
|
||||
wake()
|
||||
})
|
||||
}
|
||||
|
||||
// compareVersion orders two MAJOR.MINOR.PATCH versions by their numbers; a part that is no number counts as 0.
|
||||
func compareVersion(a, b string) int {
|
||||
pa, pb := strings.Split(a, "."), strings.Split(b, ".")
|
||||
for i := 0; i < 3; i++ {
|
||||
var x, y int
|
||||
if i < len(pa) {
|
||||
x, _ = strconv.Atoi(pa[i])
|
||||
}
|
||||
if i < len(pb) {
|
||||
y, _ = strconv.Atoi(pb[i])
|
||||
}
|
||||
if x != y {
|
||||
return x - y
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// partRef is a release's artifact as a part's key, `<module>:<artifact>`: the contract's VersionArtifact names the artifact
|
||||
// reference whole (an ArtifactRef), as the catalog's version-released says it; a bare name is taken as the module's (A-96).
|
||||
func partRef(module, artifact string) string {
|
||||
if strings.Contains(artifact, ":") {
|
||||
return artifact
|
||||
}
|
||||
return module + ":" + artifact
|
||||
}
|
||||
|
||||
@@ -2,7 +2,9 @@ package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
|
||||
@@ -54,11 +56,12 @@ func TestAVersionReleasedHeardNamesItsVersionAndPartsBySource(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 G §379, A-96: the trunk's entry takes its version and parts from the version-released of the kept assignment graph
|
||||
// revision's source; a kept build from its end; no revision kept, or another fingerprint, names none.
|
||||
func TestATrunkEntryTakesItsVersionFromTheReleaseOfTheKeptRevisionsSource(t *testing.T) {
|
||||
// Design 47 G §379, A-96, A-419: the trunk's entry takes its version and parts from the version-released of the source its
|
||||
// fingerprint was built at (its fp-src), whatever source the kept assignment graph revision is at; a kept build from its end; no
|
||||
// revision kept, another fingerprint, an fp-src with no release or none names none, and an unheard release is said once.
|
||||
func TestATrunkEntryTakesItsVersionFromTheReleaseOfItsFingerprintsSource(t *testing.T) {
|
||||
w := newWorld()
|
||||
raw, _ := json.Marshal(meshcatalog.VersionReleasedEvent{Module: "mail", Source: 7, Version: "1.2.3", Artifacts: []meshcatalog.VersionArtifact{{Artifact: "image", Digest: "sha256:9"}}})
|
||||
raw, _ := json.Marshal(meshcatalog.VersionReleasedEvent{Module: "mail", Source: 7, Version: "1.2.3", Artifacts: []meshcatalog.VersionArtifact{{Artifact: "mail:image", Digest: "sha256:9"}}})
|
||||
if err := w.released.put(raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -66,8 +69,9 @@ func TestATrunkEntryTakesItsVersionFromTheReleaseOfTheKeptRevisionsSource(t *tes
|
||||
if err := w.ends.put(endRaw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p := &passer{w: w, graph: &graphKeeper{ok: true, source: 7}}
|
||||
named := p.namedBuild("mail", "t")
|
||||
var logs []string
|
||||
p := &passer{w: w, graph: &graphKeeper{ok: true, source: 7}, logf: func(f string, a ...any) { logs = append(logs, fmt.Sprintf(f, a...)) }}
|
||||
named := p.namedBuild("mail", "t", 7)
|
||||
if k, ok := named("t"); !ok || k.Version != "1.2.3" || k.Parts["mail:image"] != "sha256:9" || k.KeptOn != "" {
|
||||
t.Errorf("trunk entry = %+v, %v", k, ok)
|
||||
}
|
||||
@@ -77,12 +81,39 @@ func TestATrunkEntryTakesItsVersionFromTheReleaseOfTheKeptRevisionsSource(t *tes
|
||||
if _, ok := named("other"); ok {
|
||||
t.Error("another fingerprint names a release")
|
||||
}
|
||||
p.graph.source = 8
|
||||
if _, ok := named("t"); ok {
|
||||
t.Error("a source with no release names a version")
|
||||
p.graph.source = 8 // a later merge that did not rebuild mail
|
||||
if k, ok := named("t"); !ok || k.Version != "1.2.3" {
|
||||
t.Errorf("after a later merge the trunk entry = %+v, %v; want the release of its fingerprint's source", k, ok)
|
||||
}
|
||||
if _, ok := p.namedBuild("mail", "t", 8)("t"); ok {
|
||||
t.Error("an fp-src with no release names a version")
|
||||
}
|
||||
_, _ = p.namedBuild("mail", "t", 8)("t")
|
||||
if _, ok := p.namedBuild("mail", "t", 0)("t"); ok {
|
||||
t.Error("an entry naming no fp-src, whose build.ended is not heard, names a version")
|
||||
}
|
||||
// The release names each artifact by its whole reference, as the catalog says it, and the parts are keyed so (A-96); the
|
||||
// assigner reads the entry's fp-src from its inputs (A-419).
|
||||
// A-429: a released version's build counts as done for rule 1; a kept build (no version) and an unknown one do not.
|
||||
if !releasedBuild(p.namedBuild("mail", "t", 7), "t") || releasedBuild(p.namedBuild("mail", "t", 7), "k") || releasedBuild(p.namedBuild("mail", "t", 8), "t") || releasedBuild(nil, "t") {
|
||||
t.Error("rule 1's released build: the release of the entry's fingerprint counts, a kept build, another source or none does not")
|
||||
}
|
||||
if v, _, _ := fromEntry(meshcatalog.CatalogEntry{Module: "mail", Fp: "t", Inputs: &meshcatalog.EntryInputs{Src: 9, FpSrc: 7}}); v.FpSrc != 7 || v.Src != 9 {
|
||||
t.Errorf("the entry's module value carries fp-src %d and src %d, want 7 and 9", v.FpSrc, v.Src)
|
||||
}
|
||||
// A-421: an entry naming no fp-src takes the release whose parts are what its build.ended made.
|
||||
made, _ := json.Marshal(builds.BuildEnded{Module: "mail", Fingerprint: "t", Made: map[string]string{"image": "sha256:9"}})
|
||||
if err := w.ends.put(made); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if k, ok := p.namedBuild("mail", "t", 0)("t"); !ok || k.Version != "1.2.3" {
|
||||
t.Errorf("an entry naming no fp-src, its build made the released digests: %+v, %v", k, ok)
|
||||
}
|
||||
if len(logs) != 2 || !strings.Contains(logs[0], "the release of mail at fingerprint t is not heard: no version-released of it from source 8") || !strings.Contains(logs[1], "names no fp-src, and no version-released of it names the parts its build.ended made") {
|
||||
t.Errorf("logs = %q, want each unheard release said once, naming the module, the fingerprint and the source", logs)
|
||||
}
|
||||
p.graph.ok = false
|
||||
if _, ok := p.namedBuild("mail", "t")("t"); ok {
|
||||
if _, ok := p.namedBuild("mail", "t", 7)("t"); ok {
|
||||
t.Error("no revision kept names a release")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -28,18 +28,25 @@ func (p *passer) addedOf(node string, laid *mma.Assignments, current map[string]
|
||||
if p.ends.following() {
|
||||
branch = p.ends.branchOf(module, placed[module])
|
||||
}
|
||||
fp := p.w.trunkOf(branch)[module].Fp2
|
||||
entry := p.w.trunkOf(branch)[module]
|
||||
fp := entry.Fp2
|
||||
if fp == "" {
|
||||
p.skippedAdd(node, module, "", "the branch "+branch+" names no fingerprint of it")
|
||||
continue
|
||||
}
|
||||
k, ok := p.namedBuild(module, fp)(fp)
|
||||
k, ok := p.namedBuild(module, fp, entry.FpSrc)(fp)
|
||||
if !ok {
|
||||
p.skippedAdd(node, module, fp, "no version-released of it is heard for that fingerprint, so the candidate carries no version and no parts")
|
||||
continue
|
||||
}
|
||||
if out == nil {
|
||||
out = map[string]mma.AddedModule{}
|
||||
}
|
||||
out[module] = mma.AddedModule{Version: k.Version, Parts: k.Parts}
|
||||
added := mma.AddedModule{Version: k.Version, Parts: k.Parts}
|
||||
if k.Version != "" {
|
||||
added.Parts = nil // a released build is named by its version alone (design 47 G A-430, A-430a)
|
||||
}
|
||||
out[module] = added
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -48,3 +55,18 @@ func (p *passer) addedOf(node string, laid *mma.Assignments, current map[string]
|
||||
func (p *passer) candidateOf(id, node string, laid *mma.Assignments, current map[string]mma.Assignments) mma.CandidatesByProposalNodeValue {
|
||||
return mma.CandidatesByProposalNodeValue{Proposal: id, Assignments: laid, Added: p.addedOf(node, laid, current)}
|
||||
}
|
||||
|
||||
// skippedAdd says once per node, module and fingerprint why a module the candidate adds is not listed in `added`.
|
||||
func (p *passer) skippedAdd(node, module, fp, why string) {
|
||||
key := node + "/" + module + "@" + fp + ": " + why
|
||||
if p.skipped[key] {
|
||||
return
|
||||
}
|
||||
if p.skipped == nil {
|
||||
p.skipped = map[string]bool{}
|
||||
}
|
||||
p.skipped[key] = true
|
||||
if p.logf != nil {
|
||||
p.logf("the candidate adds %s to %s with no parts: %s (fingerprint %q)", module, node, why, fp)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,10 +17,10 @@ import (
|
||||
func TestACandidateThatAddsAModuleCarriesItsParts(t *testing.T) {
|
||||
w := newWorld()
|
||||
for _, m := range []string{"mail", "web", "db"} {
|
||||
w.putModule("main", m, meshcatalog.ModulesByModuleValue{Fp2: "fp-" + m}, graph.Node{}, nil)
|
||||
w.putModule("main", m, meshcatalog.ModulesByModuleValue{Fp2: "fp-" + m, FpSrc: 7}, graph.Node{}, nil)
|
||||
}
|
||||
w.first = "main"
|
||||
rel, _ := json.Marshal(meshcatalog.VersionReleasedEvent{Module: "mail", Source: 7, Version: "1.2.3", Artifacts: []meshcatalog.VersionArtifact{{Artifact: "image", Digest: "sha256:9"}}})
|
||||
rel, _ := json.Marshal(meshcatalog.VersionReleasedEvent{Module: "mail", Source: 7, Version: "1.2.3", Artifacts: []meshcatalog.VersionArtifact{{Artifact: "mail:image", Digest: "sha256:9"}}})
|
||||
if err := w.released.put(rel); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -29,12 +29,12 @@ func TestACandidateThatAddsAModuleCarriesItsParts(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
w.put(buckets.Wants(), "want.ace.web", []byte(`{}`))
|
||||
p := &passer{w: w, graph: &graphKeeper{ok: true, source: 7}}
|
||||
p := &passer{w: w, graph: &graphKeeper{ok: true, source: 9}} // a later merge: the release is matched by fp-src (A-419)
|
||||
laid := &mma.Assignments{Node: "ace", Modules: []mma.PlacedModule{{Module: "mail"}, {Module: "web"}, {Module: "db"}, {Module: "old"}}}
|
||||
current := map[string]mma.Assignments{"ace": {Node: "ace", Modules: []mma.PlacedModule{{Module: "old"}}}}
|
||||
c := p.candidateOf("p1", "ace", laid, current)
|
||||
want := map[string]mma.AddedModule{
|
||||
"mail": {Version: "1.2.3", Parts: map[string]string{"mail:image": "sha256:9"}},
|
||||
"mail": {Version: "1.2.3"}, // a released build is named by its version alone (A-430a)
|
||||
"db": {Parts: map[string]string{"db:program": "sha256:3"}},
|
||||
}
|
||||
if c.Proposal != "p1" || c.Assignments != laid || !reflect.DeepEqual(c.Added, want) {
|
||||
|
||||
@@ -3,18 +3,19 @@ package main
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
nec "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
|
||||
ne "git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
)
|
||||
|
||||
// The inputs of a composition step (design 47 G §354, A-243, A-252): one request per machine, each carrying that machine's input,
|
||||
@@ -28,11 +29,11 @@ const composeCandidateKey = "seat:" + ne.Seat + "." + ne.VerbComposeCandidate
|
||||
func candidateKeyOf(run meshwants.StepRun) (string, error) {
|
||||
switch {
|
||||
case run.Run != "":
|
||||
return "run-" + run.Run, nil
|
||||
return nec.RunKey(run.Run)
|
||||
case run.Branch == "" || run.Revision <= 0:
|
||||
return "", errors.New("the step names neither a run on request nor a branch's revision, so no candidate is named")
|
||||
}
|
||||
return keys.EscapeName(run.Branch) + "." + strconv.FormatInt(run.Revision, 10), nil
|
||||
return nec.RevisionKey(run.Branch, run.Revision) // built by the seat's client, never by hand (A-226b)
|
||||
}
|
||||
|
||||
// machinesFor are the machines of a step (A-252): the nodes whose placement of the module names the step's branch (a placement
|
||||
@@ -72,9 +73,11 @@ type nodeInput struct {
|
||||
// compositionInputs asks the machines of a step for their inputs.
|
||||
type compositionInputs struct {
|
||||
machines func(module, branch string) []string
|
||||
// call asks one node's seat verb (process.Conn.CallKeptOn).
|
||||
call func(node, key string, body any) (json.RawMessage, error)
|
||||
bound time.Duration
|
||||
// callFile asks a seat verb marked chunks, addressed to one node as `seat:<seat>.<verb>@<node>`, and answers the verb's answer with
|
||||
// the whole file received added as its member `file`, base64 (process.Conn.CallFile; design 47 G A-237b, A-237c, A-249). The call
|
||||
// token is the body's `call`.
|
||||
callFile func(key string, body any) (json.RawMessage, error)
|
||||
bound time.Duration
|
||||
}
|
||||
|
||||
// gather asks each machine of the step's module and branch for its input with the step's candidate key, and answers every machine in
|
||||
@@ -121,7 +124,13 @@ func (c *compositionInputs) one(ctx context.Context, bound time.Duration, node,
|
||||
}
|
||||
got := make(chan reply, 1)
|
||||
go func() {
|
||||
raw, err := c.call(node, composeCandidateKey, ne.ComposeCandidateInput{Candidate: key, Input: true})
|
||||
// The input is any size, so the verb sends it in chunks on the call token chosen here, and the file is the input (A-237b, A-237c).
|
||||
var bits [16]byte
|
||||
if _, err := rand.Read(bits[:]); err != nil {
|
||||
got <- reply{nil, err}
|
||||
return
|
||||
}
|
||||
raw, err := c.callFile(composeCandidateKey+"@"+node, ne.ComposeCandidateInput{Candidate: key, Input: true, Call: hex.EncodeToString(bits[:])})
|
||||
got <- reply{raw, err}
|
||||
}()
|
||||
var raw json.RawMessage
|
||||
@@ -148,17 +157,36 @@ func (c *compositionInputs) one(ctx context.Context, bound time.Duration, node,
|
||||
case a.Verdict == nec.Unready:
|
||||
in.Why = "unready: " + a.Refused
|
||||
return in
|
||||
case len(a.Input) == 0:
|
||||
case a.Chunks == 0 && len(a.Input) == 0:
|
||||
in.Why = "the engine answered no composition input"
|
||||
if a.Refused != "" {
|
||||
in.Why += ": " + a.Refused
|
||||
}
|
||||
return in
|
||||
}
|
||||
body, err := json.Marshal(a.Input)
|
||||
if err != nil {
|
||||
in.Why = "its composition input cannot be encoded again: " + err.Error()
|
||||
return in
|
||||
var body []byte
|
||||
if a.Chunks > 0 {
|
||||
// The composition input is the file the chunks carried, the engine's own bytes (numbers as written).
|
||||
var whole struct {
|
||||
File string `json:"file"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &whole); err != nil {
|
||||
in.Why = "its answer is not a compose-candidate answer: " + err.Error()
|
||||
return in
|
||||
}
|
||||
file, err := base64.StdEncoding.DecodeString(whole.File)
|
||||
if err != nil || len(file) == 0 {
|
||||
in.Why = fmt.Sprintf("the engine named %d chunks of composition input and the file received is empty or not base64: %v", a.Chunks, err)
|
||||
return in
|
||||
}
|
||||
body = file
|
||||
} else {
|
||||
// An answer with the input inline (an engine before A-237b, read until the checker is removed, A-237c, G-6 §321): the object, as it was.
|
||||
var err error
|
||||
if body, err = json.Marshal(a.Input); err != nil {
|
||||
in.Why = "its composition input cannot be encoded again: " + err.Error()
|
||||
return in
|
||||
}
|
||||
}
|
||||
in.Body = body
|
||||
return in
|
||||
|
||||
+46
-5
@@ -2,13 +2,16 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
nec "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
|
||||
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
)
|
||||
|
||||
@@ -21,12 +24,21 @@ func TestTheCandidateKeyOfAStep(t *testing.T) {
|
||||
}{
|
||||
{run: meshwants.StepRun{Run: "r1", Branch: "main"}, want: "run-r1"},
|
||||
{run: meshwants.StepRun{Branch: "feat/x", Revision: 42}, want: "feat/x.42"},
|
||||
{run: meshwants.StepRun{Branch: "main", Revision: 362}, want: "main.362"},
|
||||
{run: meshwants.StepRun{Branch: "main"}, fail: true},
|
||||
{run: meshwants.StepRun{Branch: "main", Revision: 100000000000}, fail: true}, // a number the client's key would read as a heads hash
|
||||
{run: meshwants.StepRun{Run: "a.b"}, fail: true}, // a run id the client refuses
|
||||
} {
|
||||
got, err := candidateKeyOf(c.run)
|
||||
if (err != nil) != c.fail || got != c.want {
|
||||
t.Errorf("%+v: %q, %v; want %q", c.run, got, err, c.want)
|
||||
}
|
||||
// A-226b: the node-engine reads every key the step names with the client's one parser.
|
||||
if !c.fail {
|
||||
if _, err := nec.ParseCandidateKey(got); err != nil {
|
||||
t.Errorf("%+v: the key %q is refused by the seat's parser: %v", c.run, got, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -59,8 +71,9 @@ func TestAStepsInputsAreAskedOfEachMachine(t *testing.T) {
|
||||
// gather asks the machines side by side, so the record of what each was asked is guarded.
|
||||
var mu sync.Mutex
|
||||
asked := map[string]string{}
|
||||
call := func(node, key string, body any) (json.RawMessage, error) {
|
||||
call := func(key string, body any) (json.RawMessage, error) {
|
||||
raw, _ := json.Marshal(body)
|
||||
key, node, _ := strings.Cut(key, "@") // addressed to one node (A-237b)
|
||||
mu.Lock()
|
||||
asked[node] = key + " " + string(raw)
|
||||
mu.Unlock()
|
||||
@@ -72,13 +85,15 @@ func TestAStepsInputsAreAskedOfEachMachine(t *testing.T) {
|
||||
}
|
||||
return json.RawMessage(`{"candidate":"run-r1","node":"` + node + `","verdict":"composed","input":{"n":12345678901234567890,"node":"` + node + `"},"modules":[],"own":{},"took":1,"v":1}`), nil
|
||||
}
|
||||
c := &compositionInputs{machines: func(module, branch string) []string { return []string{"ace", "g14", "novox", "shanks"} }, call: call, bound: time.Second}
|
||||
c := &compositionInputs{machines: func(module, branch string) []string { return []string{"ace", "g14", "novox", "shanks"} }, callFile: call, bound: time.Second}
|
||||
got, err := c.gather(context.Background(), meshwants.StepRun{Module: "m", Branch: "main", Run: "r1"}, func(node string) bool { return node == "shanks" })
|
||||
if err != nil || len(got) != 4 {
|
||||
t.Fatalf("%+v %v", got, err)
|
||||
}
|
||||
if want := `seat:node-engine.compose-candidate {"candidate":"run-r1","input":true}`; asked["ace"] != want {
|
||||
t.Errorf("asked %q, want %q", asked["ace"], want)
|
||||
// With a call token of 128 random bits (32 hex digits), the verb being marked chunks.
|
||||
if k, body, _ := strings.Cut(asked["ace"], " "); k != "seat:node-engine.compose-candidate" ||
|
||||
!regexp.MustCompile(`^\{"call":"[0-9a-f]{32}","candidate":"run-r1","input":true\}$`).MatchString(body) {
|
||||
t.Errorf("asked %q", asked["ace"])
|
||||
}
|
||||
if got[0].Node != "ace" || got[0].Why != "" || !strings.Contains(string(got[0].Body), "12345678901234567890") {
|
||||
t.Errorf("ace: %+v", got[0])
|
||||
@@ -89,8 +104,34 @@ func TestAStepsInputsAreAskedOfEachMachine(t *testing.T) {
|
||||
if _, was := asked["shanks"]; was || !got[3].Away {
|
||||
t.Errorf("the machine away is asked or not said away: %+v", got[3])
|
||||
}
|
||||
none := &compositionInputs{machines: func(string, string) []string { return nil }, call: call}
|
||||
none := &compositionInputs{machines: func(string, string) []string { return nil }, callFile: call}
|
||||
if _, err := none.gather(context.Background(), meshwants.StepRun{Module: "m", Run: "r1"}, nil); err == nil {
|
||||
t.Error("a step with no machine is gathered")
|
||||
}
|
||||
}
|
||||
|
||||
// A-237b: the composition input is any size, so a node names the chunks it sent and the assigner reads the file the chunked call
|
||||
// received as the input, its numbers as written; a node that names chunks and sends no file is said so.
|
||||
func TestAStepsInputIsTheFileOfTheNodesChunks(t *testing.T) {
|
||||
input := `{"n":12345678901234567890,"big":"` + strings.Repeat("x", 1<<20) + `"}`
|
||||
file := base64.StdEncoding.EncodeToString([]byte(input))
|
||||
call := func(key string, body any) (json.RawMessage, error) {
|
||||
_, node, _ := strings.Cut(key, "@")
|
||||
answer := `{"candidate":"run-r1","node":"` + node + `","verdict":"composed","modules":[],"own":{},"took":1,"v":1,"chunks":3`
|
||||
if node == "g14" {
|
||||
return json.RawMessage(answer + `}`), nil // names chunks, none received
|
||||
}
|
||||
return json.RawMessage(answer + `,"file":"` + file + `"}`), nil
|
||||
}
|
||||
c := &compositionInputs{machines: func(module, branch string) []string { return []string{"ace", "g14"} }, callFile: call, bound: time.Second}
|
||||
got, err := c.gather(context.Background(), meshwants.StepRun{Module: "m", Branch: "main", Run: "r1"}, func(string) bool { return false })
|
||||
if err != nil || len(got) != 2 {
|
||||
t.Fatalf("%+v %v", got, err)
|
||||
}
|
||||
if got[0].Why != "" || string(got[0].Body) != input {
|
||||
t.Errorf("ace: why %q, %d bytes, want the file received (%d bytes) as the input", got[0].Why, len(got[0].Body), len(input))
|
||||
}
|
||||
if !strings.Contains(got[1].Why, "chunks") || got[1].Body != nil {
|
||||
t.Errorf("g14: %+v", got[1])
|
||||
}
|
||||
}
|
||||
|
||||
@@ -378,7 +378,8 @@ func caseOf(half mcc.Half) (map[string]composer.Manifest, map[string]json.RawMes
|
||||
func nodeEngineEntry(entries map[string]meshcatalog.CatalogEntry) (meshcatalog.CatalogEntry, bool) {
|
||||
for _, name := range slices.Sorted(mapsKeys(entries)) {
|
||||
for _, s := range entries[name].Serves {
|
||||
if strings.HasPrefix(s, ne.Seat+".") {
|
||||
// The catalog writes a served seat verb as `seat:<seat>.<verb>` (as the compose words above are made).
|
||||
if strings.HasPrefix(strings.TrimPrefix(s, "seat:"), ne.Seat+".") {
|
||||
return entries[name], true
|
||||
}
|
||||
}
|
||||
@@ -410,7 +411,7 @@ func programOfEntry(e meshcatalog.CatalogEntry) (buildstep.RunProgram, error) {
|
||||
|
||||
// programs answers, per machine, the composer its run builds (A-226, A-243, A-252): where the revision's node-engine entry differs
|
||||
// from the trunk's, that build for every machine; else the build the machine's node-engine runs, as its `running` says, its entry
|
||||
// read through mesh-catalog's `entry`.
|
||||
// read through mesh-catalog's `entry`; a machine whose node-engine's `running` is not heard gets the revision's entry, said (A-451).
|
||||
func (r *compositionRun) programs(half mcc.Half) (func(node string) (buildstep.RunProgram, error), error) {
|
||||
engine, ok := nodeEngineEntry(half.Entries)
|
||||
if !ok {
|
||||
@@ -425,7 +426,9 @@ func (r *compositionRun) programs(half mcc.Half) (func(node string) (buildstep.R
|
||||
return func(node string) (buildstep.RunProgram, error) {
|
||||
running, ok := r.running(node, engine.Module)
|
||||
if !ok || running.Fingerprint == "" {
|
||||
return buildstep.RunProgram{}, fmt.Errorf("no build of %s is heard running there", engine.Module)
|
||||
r.logf("%s: no build of %s is heard running there, so its composer is the catalog revision's entry %s (A-451)", node,
|
||||
engine.Module, engine.Fp)
|
||||
return programOfEntry(engine)
|
||||
}
|
||||
mu.Lock()
|
||||
p, ok := byFp[running.Fingerprint]
|
||||
|
||||
@@ -71,7 +71,8 @@ func newRunStand(t *testing.T, revisionEngine, trunkEngine, seatDigest string) *
|
||||
}
|
||||
return nil, errors.New("no such verb " + key)
|
||||
}
|
||||
engines := func(node, _ string, _ any) (json.RawMessage, error) {
|
||||
engines := func(key string, _ any) (json.RawMessage, error) {
|
||||
_, node, _ := strings.Cut(key, "@") // addressed to one node (A-237b)
|
||||
return json.RawMessage(`{"candidate":"main.7","node":"` + node + `","verdict":"composed","input":{"node":"` + node + `"},"modules":[],"own":{},"took":1,"v":1}`), nil
|
||||
}
|
||||
running := map[string]string{"ace": "src2:e1", "g14": "src2:e2"}
|
||||
@@ -80,7 +81,7 @@ func newRunStand(t *testing.T, revisionEngine, trunkEngine, seatDigest string) *
|
||||
keeper := testKeeper(&objectsMem{objs: map[string][]byte{}}, time.Now)
|
||||
ends := &runEnds{}
|
||||
s.r = &compositionRun{
|
||||
inputs: &compositionInputs{machines: func(string, string) []string { return []string{"ace", "g14", "shanks"} }, call: engines, bound: time.Second},
|
||||
inputs: &compositionInputs{machines: func(string, string) []string { return []string{"ace", "g14", "shanks"} }, callFile: engines, bound: time.Second},
|
||||
keeper: keeper, call: call, ends: ends, away: away, now: time.Now, logf: t.Logf,
|
||||
trunk: func() (mcc.Half, bool) {
|
||||
return mcc.Half{Entries: map[string]meshcatalog.CatalogEntry{"engine": engine(trunkEngine, commitE1)}}, true
|
||||
@@ -117,7 +118,7 @@ func newRunStand(t *testing.T, revisionEngine, trunkEngine, seatDigest string) *
|
||||
// Design 47 G §354, §63, §64, A-226, A-238, A-240, A-243, A-252, A-254, A-261: one request and one run ask per machine, the request
|
||||
// depth compose with the one machine and one case `change`, its manifests, verdicts and seats from the step's revision; the
|
||||
// composer is the node-engine build each machine runs, its entry read through `entry`, or the revision's where it differs from the
|
||||
// trunk's; the run's verdict is read from its step-ended; a machine away passes with a warning, one not composing fails the step.
|
||||
// trunk's, or the revision's for a machine whose node-engine is not heard running (A-451); the run's verdict is read from its step-ended; a machine away passes with a warning, one not composing fails the step.
|
||||
func TestTheCompositionStepRunsOneAskPerMachine(t *testing.T) {
|
||||
step := meshwants.StepRun{Module: "m", Branch: "main", Revision: 7, Step: meshwants.StepRunStepComposition}
|
||||
s := newRunStand(t, "src2:e1", "src2:e1", "d1")
|
||||
@@ -159,6 +160,18 @@ func TestTheCompositionStepRunsOneAskPerMachine(t *testing.T) {
|
||||
t.Errorf("the change's node-engine does not compose every machine: %+v %+v %v", changed.asks["ace"].Run.Program, changed.asks["g14"].Run.Program, changed.entries)
|
||||
}
|
||||
|
||||
// No running of the node-engine is heard on g14: its composer is the revision's entry, and no entry is asked for it (A-451).
|
||||
unheard := newRunStand(t, "src2:e1", "src2:e1", "d1")
|
||||
unheard.r.running = func(node, module string) (ne.Running, bool) {
|
||||
return ne.Running{Node: node, Module: module, Fingerprint: "src2:e1"}, node == "ace" && module == "engine"
|
||||
}
|
||||
if out := unheard.r.run(context.Background(), "ask-step", step); out.Outcome != meshwants.StepEndedOutcomeFailed || !strings.Contains(out.Said, "g14 does not compose") {
|
||||
t.Fatalf("the step with g14's node-engine not heard came to %+v", out)
|
||||
}
|
||||
if unheard.asks["g14"].Run == nil || unheard.asks["g14"].Run.Program.Commit != commitE1 {
|
||||
t.Errorf("g14's composer with no running heard is %+v, want the revision's entry", unheard.asks["g14"].Run)
|
||||
}
|
||||
|
||||
// A seat digest neither the revision nor the trunk holds: could not run, naming the seat and the digest (A-261).
|
||||
other := newRunStand(t, "src2:e1", "src2:e1", "d2")
|
||||
if out := other.r.run(context.Background(), "ask-step", step); out.Outcome != meshwants.StepEndedOutcomeCouldNotRun ||
|
||||
|
||||
@@ -365,6 +365,7 @@ func (p *passer) holdForDelivery(ctx context.Context, module, fp, branch string,
|
||||
}
|
||||
kept := deliveryWaiting{Wait: body}
|
||||
p.put(ctx, buckets.DeliveryWaits(), key, func() ([]byte, error) { return json.Marshal(kept) })
|
||||
p.logf("%s at %s on %s waits for the delivery of its dependency %s on %s: the rules write no want of it until then (design 47 G §376)", module, shortHex(fp), branch, body.On, strings.Join(body.Machines, ", "))
|
||||
return true
|
||||
}
|
||||
|
||||
|
||||
@@ -178,15 +178,59 @@ func (p *passer) sayWanted(node, module, fp string, removed bool) {
|
||||
|
||||
// namedBuild answers what a want of module at fp names of its build (design 47 G §379, §526, A-96): a kept build's or another watched
|
||||
// branch's parts and keeping node from the builder's `build.ended`; for the trunk's entry (fp is the module's, trunkFP), the released
|
||||
// version and the parts of the `version-released` whose source is the one the kept assignment graph revision was computed from (A-67).
|
||||
func (p *passer) namedBuild(module, trunkFP string) func(fp string) (assign.KeptBuild, bool) {
|
||||
// version and the parts of the `version-released` of the module whose source is the one its fingerprint was built at, the entry's
|
||||
// `fp-src` (A-419: a merge that does not rebuild the module leaves it so), or for an entry naming none the one whose parts are the
|
||||
// digests its build.ended made (A-421). A trunk entry whose release is not heard is said, once, naming which was tried.
|
||||
func (p *passer) namedBuild(module, trunkFP string, fpSrc int64, unreleased ...bool) func(fp string) (assign.KeptBuild, bool) {
|
||||
return func(fp string) (assign.KeptBuild, bool) {
|
||||
if k, ok := p.w.ends.partsOf(module, fp); ok {
|
||||
return k, true
|
||||
}
|
||||
if fp != trunkFP || p.graph == nil || !p.graph.ok {
|
||||
if len(unreleased) > 0 && unreleased[0] && fp == trunkFP {
|
||||
// Wanted with no passed pipeline (the holder of mesh-catalog, the runner's first start): no pipeline released it,
|
||||
// so its want names the parts its build.ended made, until a release of it is heard (design 47 G §16, A-428).
|
||||
if k, ok := p.w.released.withParts(module, p.w.ends.madeOf(module, fp)); ok {
|
||||
return k, true
|
||||
}
|
||||
if made := p.w.ends.madeOf(module, fp); len(made) > 0 {
|
||||
return assign.KeptBuild{Parts: made}, true
|
||||
}
|
||||
}
|
||||
if fp != trunkFP {
|
||||
return assign.KeptBuild{}, false
|
||||
}
|
||||
return p.w.released.of(module, p.graph.source)
|
||||
if p.graph == nil || !p.graph.ok {
|
||||
p.unheardRelease(module, fp, "no assignment graph revision is kept yet, so no release is looked up")
|
||||
return assign.KeptBuild{}, false
|
||||
}
|
||||
if fpSrc == 0 {
|
||||
// No fp-src (its fingerprint moved with no merge): the release whose parts are what its build made (A-421).
|
||||
if k, ok := p.w.released.withParts(module, p.w.ends.madeOf(module, fp)); ok {
|
||||
return k, true
|
||||
}
|
||||
p.unheardRelease(module, fp, "its catalog entry names no fp-src, and no version-released of it names the parts its build.ended made")
|
||||
return assign.KeptBuild{}, false
|
||||
}
|
||||
k, ok := p.w.released.of(module, fpSrc)
|
||||
if !ok {
|
||||
p.unheardRelease(module, fp, fmt.Sprintf("no version-released of it from source %d, the one its fingerprint was built at, is heard", fpSrc))
|
||||
}
|
||||
return k, ok
|
||||
}
|
||||
}
|
||||
|
||||
// unheardRelease says once per module, fingerprint and why that a trunk entry's release is not heard, so its wants and candidates
|
||||
// carry no version and no parts (design 47 G A-419).
|
||||
func (p *passer) unheardRelease(module, fp, why string) {
|
||||
key := "release " + module + "@" + fp + ": " + why
|
||||
if p.skipped[key] {
|
||||
return
|
||||
}
|
||||
if p.skipped == nil {
|
||||
p.skipped = map[string]bool{}
|
||||
}
|
||||
p.skipped[key] = true
|
||||
if p.logf != nil {
|
||||
p.logf("the release of %s at fingerprint %s is not heard: %s; its wants and candidates carry no version and no parts", module, fp, why)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -51,9 +51,39 @@ func (h *halves) follow() error {
|
||||
h.mu.Lock()
|
||||
h.branches = b
|
||||
h.mu.Unlock()
|
||||
h.settleWhenNoBranch()
|
||||
return nil
|
||||
}
|
||||
|
||||
// settleWhenNoBranch tells the first pass once, when the registrations and the gate's source are answered and name no branch: the
|
||||
// trunk is none-yet and the world is empty, as the catalog client's Ready says it (design 47 G, A-372). A mesh that names a branch is
|
||||
// told by adopted, when that branch's first view is answered.
|
||||
func (h *halves) settleWhenNoBranch() {
|
||||
if h.settled == nil {
|
||||
return
|
||||
}
|
||||
h.mu.Lock()
|
||||
b := h.branches
|
||||
h.mu.Unlock()
|
||||
go func() {
|
||||
select {
|
||||
case <-b.FirstViews(h.ctx):
|
||||
case <-h.ctx.Done():
|
||||
return
|
||||
}
|
||||
if len(b.Followed()) > 0 || b.Ready() != nil {
|
||||
return
|
||||
}
|
||||
h.mu.Lock()
|
||||
told := h.told
|
||||
h.told = true
|
||||
h.mu.Unlock()
|
||||
if !told {
|
||||
h.settled <- struct{}{}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// first is the first watched branch followed, the trunk's: "" while none is.
|
||||
func (h *halves) first() string {
|
||||
h.mu.Lock()
|
||||
@@ -118,6 +148,11 @@ func (h *halves) changed(branch, module string, raw json.RawMessage, deleted boo
|
||||
func fromEntry(e meshcatalog.CatalogEntry) (meshcatalog.ModulesByModuleValue, graph.Node, *parsed) {
|
||||
v := meshcatalog.ModulesByModuleValue{Commit: e.Commit, Fp2: e.Fp, Needs: e.Needs, None2: e.None, Packages: e.Packages,
|
||||
Path: e.Path, Repository: e.Repository, Verdict: e.Verdict}
|
||||
// The source revisions it was taken in at and its fingerprint was built at: the release of a trunk build is matched by the
|
||||
// latter (design 47 G A-419).
|
||||
if e.Inputs != nil {
|
||||
v.Src, v.FpSrc = e.Inputs.Src, e.Inputs.FpSrc
|
||||
}
|
||||
n := graph.Node{FP: e.Fp, None: e.None, Repository: e.Repository, Path: e.Path, Bases: e.Bases, Provides: e.Provides, Requires: e.Requires,
|
||||
Awaits: e.Awaits, Invokes: e.Invokes, Serves: e.Serves, LeftOut: e.LeftOut}
|
||||
for p := range e.Packages {
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
mcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
)
|
||||
|
||||
// Design 47 G A-372: the assigner's wait for its catalog half asks the catalog client's Ready as well: a mesh whose registrations and
|
||||
// gate's source are answered and name no branch is none-yet, and the first pass is told so once; before they answer it is not told.
|
||||
func TestAMeshThatNamesNoBranchSettlesTheAssignersFirstPass(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer cancel()
|
||||
call := func(key string, _ any) (json.RawMessage, error) {
|
||||
if key == mcc.RegistrationsVerb {
|
||||
return json.Marshal(seat.RegistrationsAnswer{Registrations: []seat.Registration{}})
|
||||
}
|
||||
return json.Marshal(map[string][]string{"branches": {}})
|
||||
}
|
||||
for _, answered := range []bool{false, true} {
|
||||
settled := make(chan struct{}, 2)
|
||||
reader := &views.ProcessReader{Module: "mesh-delivery-assigner", Call: call, Logf: t.Logf}
|
||||
h := &halves{w: newWorld(), call: call, reader: reader, wake: func() {}, logf: t.Logf, ctx: ctx, settled: settled}
|
||||
if err := h.follow(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if answered {
|
||||
for _, f := range reader.Followers() {
|
||||
if err := f.Start(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
select {
|
||||
case <-settled:
|
||||
case <-ctx.Done():
|
||||
t.Fatal("a mesh that names no branch never settled the first pass")
|
||||
}
|
||||
}
|
||||
select {
|
||||
case <-settled:
|
||||
t.Errorf("answered %v: the first pass was told twice or before the first views", answered)
|
||||
default:
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -208,7 +208,7 @@ func startBed(t *testing.T, leaveOut string) *bed {
|
||||
}
|
||||
// The builder's stream of build ends, kept last per subject (its manifest's `last-per-subject`, design 47 G A-181), which the assigner
|
||||
// reads whole at its start and follows through its durable, as the grants part makes it (from the last of each subject).
|
||||
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: buildsStreamName, Subjects: []string{process.ModuleStreamSubject(builds.BuilderModule, builds.BuildEndedEvent+".>")}, MaxMsgsPerSubject: 1}); err != nil {
|
||||
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: buildsStreamName, Subjects: []string{process.ModuleStreamSubject(builds.BuilderModule, builds.BuildEndedEvent+".>")}, MaxAge: 168 * time.Hour}); err != nil { // kept by age, as the bus keeps it live (2026-11-15): read whole from its first message all the same (A-233)
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := js.CreateOrUpdateConsumer(ctx, buildsStreamName, jetstream.ConsumerConfig{Durable: "n1_mesh-delivery-assigner", AckPolicy: jetstream.AckExplicitPolicy,
|
||||
|
||||
@@ -587,7 +587,8 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
|
||||
}
|
||||
}
|
||||
p := &passer{kvs: kvs, w: w, logf: logger.Printf, say: say, findings: findings, said: map[string]bool{}, answeredAt: map[string]uint64{},
|
||||
reads: reads, seats: gate}
|
||||
reads: reads, seats: gate, askForFile: getenv("MESH_ASSIGNER_ASK_OPERATOR_FOR_FILE")}
|
||||
p.askOperatorFor() // the setting ask-operator-for said once at the start (design 47 G A-425a)
|
||||
// The wait on an outside cause (outside.go, design 47 G §706): the keeper's three events say when to judge again, and the
|
||||
// seat mesh-wants' wait-held and wait-ended are said; registered before the hub starts.
|
||||
if bus != nil && kvs[buckets.Waits()] != nil {
|
||||
@@ -674,7 +675,7 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
|
||||
}
|
||||
// The composition step (design 47 G §22, §354): the seat mesh-wants' accepted kind step, run by one run ask per machine.
|
||||
if bus != nil && js != nil && inputs != nil && p.publishWants != nil {
|
||||
run := &compositionRun{inputs: &compositionInputs{machines: w.machinesOf, call: bus.CallKeptOn}, keeper: inputs, call: bus.CallKept,
|
||||
run := &compositionRun{inputs: &compositionInputs{machines: w.machinesOf, callFile: bus.CallFile}, keeper: inputs, call: bus.CallKept,
|
||||
trunk: trunkOf(iv, w.trunkName), running: w.runningOf, away: away, put: runAskPublisher(js), ends: ends, ask: bus.CallFile, now: time.Now, logf: logger.Printf}
|
||||
steps := &compositionSteps{node: thisNode, say: p.publishWants, run: run.run, now: time.Now, logf: logger.Printf,
|
||||
asker: stepAskerOf(seatlist.Definition(gate.Reader, meshwants.Seat))}
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
)
|
||||
|
||||
// The module claiming the seat node-engine is found by its served verbs, in the catalog's form `seat:<seat>.<verb>`
|
||||
// and in the bare form (live, 2026-11-16: every composition step said "holds no module claiming the seat node-engine").
|
||||
func TestTheNodeEngineEntryIsFoundByItsServedVerbs(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
serves []string
|
||||
found bool
|
||||
}{
|
||||
{"the catalog's seat form", []string{"seat:node-engine.compose", "seat:node-engine.acts"}, true},
|
||||
{"the bare form", []string{"node-engine.compose"}, true},
|
||||
{"another seat", []string{"seat:node-engine-x.compose", "seat:mesh-wants.ask"}, false},
|
||||
} {
|
||||
e, ok := nodeEngineEntry(map[string]meshcatalog.CatalogEntry{"other": {Serves: []string{"seat:mesh-builds.ask"}}, "engine": {Serves: c.serves}})
|
||||
if ok != c.found || (ok && e.Serves[0] != c.serves[0]) {
|
||||
t.Errorf("%s: found %v (%v), want %v", c.name, ok, e.Serves, c.found)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"mesh-delivery-assigner/internal/buckets"
|
||||
"slices"
|
||||
@@ -23,8 +24,10 @@ import (
|
||||
"git.novox.be/novox/mesh-sdk/go/seatlist"
|
||||
"sync"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
|
||||
"git.novox.be/novox/mesh-contracts/provisioning/go/clients/meshprovisionerclient"
|
||||
mpr "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/graph"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
|
||||
@@ -47,6 +50,14 @@ type passer struct {
|
||||
// pass says a condition once and a restart says it again (the router keeps one per key).
|
||||
say func(oc.Condition) error
|
||||
said map[string]bool
|
||||
// askForFile is the file of the setting ask-operator-for, askFor the kinds read last (design 47 G A-425a).
|
||||
askForFile string
|
||||
askFor []string
|
||||
// skipped are the added modules whose absence from a candidate's added was said (skippedAdd), and the trunk entries whose
|
||||
// release is not heard (unheardRelease), once each.
|
||||
skipped map[string]bool
|
||||
// whys is, by key, the why of a want this process last wrote, the time aside (why.go).
|
||||
whys map[string]string
|
||||
// findings is the assigner's findings bucket (its said conditions, ADR 0564 §2).
|
||||
findings assign.Findings
|
||||
// calls is the wait of the kept manifest-reads calls (enginereads.ReadsCalls, ADR 0668 §3), made at the first pass.
|
||||
@@ -87,6 +98,7 @@ type passer struct {
|
||||
// laid again is not ended on a seat it does not name (ADR 0636 §1). Nil reads nothing.
|
||||
seats *seatlist.Part
|
||||
seatWaitSaid sync.Once
|
||||
derivedSaid map[string]bool // each seat-holders key last written as derived from a claim (A-400), so each is said once
|
||||
publish publisher
|
||||
saidProposed map[string]bool
|
||||
// copied says the retired state was copied into `assignments` by this process, or found empty (copyRetired).
|
||||
@@ -220,6 +232,9 @@ func (p *passer) pass(ctx context.Context) {
|
||||
p.deliveredAgain(ctx)
|
||||
if p.graph == nil || !p.graph.full { // a full assignment graph store keeps the last revision and writes no new want (design 47 G §181)
|
||||
p.wants(ctx)
|
||||
p.sayPass(ctx, "judged")
|
||||
} else {
|
||||
p.sayPass(ctx, "the assignment graph store is full: no want is written (design 47 G §181)")
|
||||
}
|
||||
p.puts(ctx)
|
||||
p.nodeWants(ctx)
|
||||
@@ -276,7 +291,7 @@ func (p *passer) wants(ctx context.Context) {
|
||||
missing := map[string]bool{}
|
||||
defer func() { p.provisionsMissing(ctx, missing) }()
|
||||
defer p.unplacedFindings()
|
||||
defer p.seatHolders(ctx)
|
||||
defer func() { p.seatHolders(ctx, viewOf(p.w.trunkName()).manifests) }()
|
||||
p.clearStaleCalls()
|
||||
placed := p.w.placedNodes()
|
||||
puts := p.putsByModule()
|
||||
@@ -320,10 +335,31 @@ func (p *passer) wants(ctx context.Context) {
|
||||
branches = []string{g.branch}
|
||||
}
|
||||
if fp != "" && !p.ends.mayWant(name, fp, branches, wantedOf[name]) {
|
||||
return
|
||||
if !runnerFirstStart(name, manifests, p.w.runningNow()) && !manifests[name].ClaimsSeat(meshcatalog.Seat) {
|
||||
none := assign.Outcome{Rules: noPipelineRules(g.nodes, name, fp, branches)}
|
||||
p.sayNoWant(name, g.nodes, wantedOf[name], none)
|
||||
p.sayWhy(ctx, name, g.nodes, wantWhy{Branch: g.branch, Fp: fp, Step: whyNoPipeline}, wantedOf[name], &none, nil)
|
||||
return
|
||||
}
|
||||
// The first start of the pipeline runner (design 47 G A-428): no runner runs to pass its pipeline, so its want is
|
||||
// written from the trunk's entry by the other rules, and said once. So is the holder of mesh-catalog's: every
|
||||
// pipeline runs on a revision it took in, and a revision naming a newer one is taken in only by that one (§16,
|
||||
// G-5's whole test of indexer and assigner: the new indexer is wanted and granted, starts, and takes it in).
|
||||
if key := "no-pipeline-start " + name + "@" + fp; !p.skipped[key] {
|
||||
if p.skipped == nil {
|
||||
p.skipped = map[string]bool{}
|
||||
}
|
||||
p.skipped[key] = true
|
||||
if manifests[name].ClaimsSeat(meshcatalog.Seat) {
|
||||
p.logf("%s at %s is wanted with no passed pipeline: it claims the seat %s, every pipeline runs on a revision it took in, and a revision naming it is taken in only by it (design 47 G §16)", name, shortHex(fp), meshcatalog.Seat)
|
||||
} else {
|
||||
p.logf("%s at %s is wanted with no passed pipeline: it claims the seat %s and no module holding that seat runs on any node, so none can pass one (design 47 G A-428)", name, shortHex(fp), mpr.Seat)
|
||||
}
|
||||
}
|
||||
}
|
||||
// A dependent's new version waits for its dependency's newest released version to be delivered (§376, A-189).
|
||||
if p.holdForDelivery(ctx, name, fp, g.branch, g.nodes) {
|
||||
p.sayWhy(ctx, name, g.nodes, wantWhy{Branch: g.branch, Fp: fp, Step: whyDelivery, Words: p.saidDeliveryWaits()[name].Wait.On}, wantedOf[name], nil, nil)
|
||||
return
|
||||
}
|
||||
m := assign.Module{Name: name, FP: fp, Manifest: manifests[name], Needs: trunk[name].Needs, Chosen: chosenOf(chosen, name), Provisions: provisionsOf(name), Answering: answering(manifests, nodes), Offered: offered(manifests), Nodes: nodesOf[name], Now: time.Now(), Puts: puts[name], FormNeeds: p.w.buildNeeds(name, fp),
|
||||
@@ -347,11 +383,17 @@ func (p *passer) wants(ctx context.Context) {
|
||||
held, _ := mwc.HeldKey(name, fp)
|
||||
grant, _ := grantsstate.PartGrantKey(name, fp)
|
||||
hold, _ := asks.HoldAskKey(name, fp)
|
||||
m.KeptBuilds = p.namedBuild(name, m.FP)
|
||||
noPipeline := runnerFirstStart(name, manifests, p.w.runningNow()) || manifests[name].ClaimsSeat(meshcatalog.Seat)
|
||||
m.KeptBuilds = p.namedBuild(name, m.FP, trunk[name].FpSrc, noPipeline)
|
||||
m.Released = p.answeredWith(hold, "release")
|
||||
m.Kept = p.answeredWith(hold, "keep")
|
||||
m.Built, m.Held, m.Granted = p.w.built(name, fp), p.w.has(buckets.Wants(), held) && !m.Released, p.w.has(grants, grant)
|
||||
m.BuildFailed, m.BuildError = p.w.buildStanding(name, fp)
|
||||
// A build whose version is released counts as done for rule 1; its build.ended counts only for a build not
|
||||
// released (design 47 G A-429), so a restart that lost the end loses nothing of a released build.
|
||||
if releasedBuild(m.KeptBuilds, fp) {
|
||||
m.Built, m.BuildFailed, m.BuildError = true, false, ""
|
||||
}
|
||||
if raw := p.w.value(grants, grant); raw != nil {
|
||||
if g, err := grantsstate.DecodeGrant(raw); err == nil {
|
||||
m.GrantedOn = func(node string) bool { return assign.GrantNamesNode(g, node, name) }
|
||||
@@ -377,6 +419,8 @@ func (p *passer) wants(ctx context.Context) {
|
||||
actingOf, orderOf := p.actingOf(name, released), p.orderActing(name, released, unread, func() assign.Nodes { return nodes })
|
||||
m.Acting = func(node string) []assign.ActingEdge { return append(actingOf(node), orderOf(node)...) }
|
||||
out := assign.Wants(m, nodes, wantedOf[name], last)
|
||||
p.sayWhy(ctx, name, m.Nodes, wantWhy{Branch: g.branch, Fp: fp, Step: whyRules}, wantedOf[name], &out, &m)
|
||||
p.sayParts(name, m, nodes, out, wantedOf[name])
|
||||
for _, node := range m.Nodes {
|
||||
for _, prov := range m.MissingProvisions(node) {
|
||||
missing[node+"."+name+"."+prov] = true
|
||||
@@ -1041,7 +1085,7 @@ func (p *passer) backups(ctx context.Context, reverting map[string]bool) {
|
||||
continue
|
||||
}
|
||||
current, currentAsk, there := assign.DecodeAsk(akey, p.w.read(buckets.Asks())[akey])
|
||||
if there != nil || assign.BackupAskDigest(current) != want.Composed {
|
||||
if there != nil || assign.BackupAskDigest(current) != assign.ShortDigest(want.Composed) {
|
||||
after := -1
|
||||
if there == nil {
|
||||
after = current.Asking
|
||||
@@ -1319,7 +1363,10 @@ func (p *passer) unplacedFindings() {
|
||||
// seat held once, `<seat>`, and one per node of a node seat, `<seat>.<node>`, written only where it differs, and a key no
|
||||
// assignments holds any more deleted. A pass follows each assignments change and each manifest the indexer takes in, so no
|
||||
// timer writes it.
|
||||
func (p *passer) seatHolders(ctx context.Context) {
|
||||
//
|
||||
// A seat no assignments holds is held by the first assigned module whose manifest of the trunk claims it (lead answer A-400):
|
||||
// manifests is the trunk's, by module. Each holder so derived is said once while it stays derived.
|
||||
func (p *passer) seatHolders(ctx context.Context, manifests map[string]catalogue.Manifest) {
|
||||
kv := p.kvs[plc.SeatHoldersBucket()]
|
||||
if kv == nil {
|
||||
return
|
||||
@@ -1336,7 +1383,20 @@ func (p *passer) seatHolders(ctx context.Context) {
|
||||
}
|
||||
return
|
||||
}
|
||||
want := plc.SeatHoldersOf(assignments)
|
||||
want, derived := plc.SeatHoldersWithClaimants(assignments, manifests)
|
||||
now := map[string]bool{}
|
||||
for _, d := range derived {
|
||||
now[d] = true
|
||||
if !p.derivedSaid[d] {
|
||||
p.logf("seat-holders: the seat %s: no holding on record, written as held by the assigned module whose manifest claims it (A-400)", d)
|
||||
}
|
||||
}
|
||||
for d := range p.derivedSaid {
|
||||
if !now[d] {
|
||||
p.logf("seat-holders: the seat %s is no longer derived (a holding is on record, or nobody assigned claims it)", d)
|
||||
}
|
||||
}
|
||||
p.derivedSaid = now
|
||||
have := p.w.read(plc.SeatHoldersBucket())
|
||||
for key, sh := range want {
|
||||
value, err := json.Marshal(sh)
|
||||
@@ -1469,3 +1529,75 @@ func wantedHere(wanted map[string]mwc.Want, own []string, groups []group, first
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// noPipelineRules is why each node of module gets no want while no pipeline passed at fp on its branches (design 47 G §29): said as
|
||||
// any rule's reason (sayNoWant), once.
|
||||
func noPipelineRules(nodes []string, module, fp string, branches []string) map[string]assign.Holding {
|
||||
out := map[string]assign.Holding{}
|
||||
for _, node := range nodes {
|
||||
out[node] = assign.Holding{Rule: mwc.RuleNotHeld, Words: fmt.Sprintf("no pipeline of %s passed at %s on %s, and a want follows a passed pipeline (design 47 G §29)",
|
||||
module, shortHex(fp), strings.Join(branches, ", "))}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// runnerFirstStart says whether module claims the seat mesh-pipeline-runner while no module claiming it runs on any node (design 47
|
||||
// G A-428): read from the node-engine seat's running statements the assigner hears, each module's claims from its manifest. With
|
||||
// no running heard yet it says false: the gate stands until the assigner knows.
|
||||
func runnerFirstStart(module string, manifests map[string]catalogue.Manifest, running map[string]map[string]nodeengine.Running) bool {
|
||||
if !manifests[module].ClaimsSeat(mpr.Seat) || running == nil {
|
||||
return false
|
||||
}
|
||||
for _, mods := range running {
|
||||
for m, r := range mods {
|
||||
if !r.Stopped && r.Fingerprint != "" && manifests[m].ClaimsSeat(mpr.Seat) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// releasedBuild says whether the build at fp is a released version's, as a want of it names it (namedBuild): such a build counts as
|
||||
// done for rule 1 whatever the builder's build.ended said or the assigner kept of it (design 47 G A-429).
|
||||
func releasedBuild(named func(fp string) (assign.KeptBuild, bool), fp string) bool {
|
||||
if named == nil {
|
||||
return false
|
||||
}
|
||||
k, ok := named(fp)
|
||||
return ok && k.Version != ""
|
||||
}
|
||||
|
||||
// sayParts says once per node, module and reason why a want the pass writes or keeps at the module's fingerprint names no parts
|
||||
// (design 47 G §379, §525, §526): whether the node's engine reads a want's parts (its record's capabilities, its manifest reads)
|
||||
// and whether the build has a release or a kept build the want could name.
|
||||
func (p *passer) sayParts(name string, m assign.Module, n assign.Nodes, out assign.Outcome, before map[string]mwc.Want) {
|
||||
for _, node := range m.Nodes {
|
||||
w, ok := out.Wants[node]
|
||||
if !ok {
|
||||
w, ok = before[node] // kept as it stands this pass
|
||||
}
|
||||
if !ok || w.Removed || w.Fp == "" || len(w.Parts) > 0 || w.Version != "" {
|
||||
continue
|
||||
}
|
||||
reads := n.ReadsParts(node)
|
||||
var k assign.KeptBuild
|
||||
found := false
|
||||
if m.KeptBuilds != nil {
|
||||
k, found = m.KeptBuilds(w.Fp)
|
||||
}
|
||||
why := fmt.Sprintf("its engine reads parts: %v (capabilities %v); a release or kept build of %s found: %v", reads, n.Capable[node], shortHex(w.Fp), found)
|
||||
if found && len(k.Parts) == 0 {
|
||||
why += ", naming no parts"
|
||||
}
|
||||
key := "parts " + node + "." + name + "@" + w.Fp + ": " + why
|
||||
if p.skipped[key] {
|
||||
continue
|
||||
}
|
||||
if p.skipped == nil {
|
||||
p.skipped = map[string]bool{}
|
||||
}
|
||||
p.skipped[key] = true
|
||||
p.logf("the want of %s on %s names no version and no parts: %s", name, node, why)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -10,6 +10,8 @@ import (
|
||||
"maps"
|
||||
"mesh-delivery-assigner/internal/buckets"
|
||||
"mesh-delivery-assigner/internal/puts"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -1327,6 +1329,31 @@ func TestAnApprovedAssignmentsCallsNoControllerVerb(t *testing.T) {
|
||||
if len(bus.calls) != 0 || !slices.Equal(said, []string{mma.EventAssignmentsChanged}) {
|
||||
t.Fatalf("calls = %d, said = %v", len(bus.calls), said)
|
||||
}
|
||||
// Design 47 G A-425a: the setting ask-operator-for says which change kinds are asked; its default is take-off, remove-node and
|
||||
// stops; a kind it does not know is left out; it is read from its file at each pass and a change of it is said.
|
||||
if k, u := parseAskOperatorFor(""); !slices.Equal(k, []string{"take-off", "remove-node", "stops"}) || u != nil {
|
||||
t.Errorf("the default = %v %v", k, u)
|
||||
}
|
||||
if k, u := parseAskOperatorFor("place, Stops,typo"); !slices.Equal(k, []string{"place", "stops"}) || !slices.Equal(u, []string{"typo"}) {
|
||||
t.Errorf("parsed %v, unknown %v", k, u)
|
||||
}
|
||||
file := filepath.Join(t.TempDir(), "ask-operator-for")
|
||||
var logs []string
|
||||
p.logf = func(f string, a ...any) { logs = append(logs, fmt.Sprintf(f, a...)) }
|
||||
p.askForFile = file
|
||||
if p.asksFor([]string{"place"}) || !p.asksFor([]string{"place", "take-off"}) {
|
||||
t.Error("with no file, a place alone is asked or a take-off is not")
|
||||
}
|
||||
if err := os.WriteFile(file, []byte("place"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !p.asksFor([]string{"place"}) || p.asksFor([]string{"take-off"}) || len(logs) != 2 || !strings.Contains(logs[1], "the setting ask-operator-for is place") {
|
||||
t.Errorf("after the setting changed to place: logs %q", logs)
|
||||
}
|
||||
stopping := mma.Verdict{Answers: map[string]map[string]any{"anchor": {"removed": []any{"db"}}}}
|
||||
if got := proposalKinds(mma.Proposal{Changes: []mma.ProposalChange{{Op: "place"}, {Op: "pin"}, {Op: "place"}}}, stopping); !slices.Equal(got, []string{"pin", "place", "stops"}) {
|
||||
t.Errorf("proposal kinds = %v, want pin, place and stops", got)
|
||||
}
|
||||
|
||||
var conds []oc.Condition
|
||||
p.say = func(c oc.Condition) error { conds = append(conds, c); return nil }
|
||||
@@ -1390,6 +1417,21 @@ func aWantWaitsOnTheVerbEdgesTheGraphKeeps(t *testing.T) {
|
||||
t.Errorf("served(%s) = %v, want %v: the running server's manifest serves w alone, late's is not read", v, served(v), want)
|
||||
}
|
||||
}
|
||||
// A-426a: a healthy build held (no catalog entry of it) or waiting for a yes serves what it serves, said once; one held unhealthy,
|
||||
// or whose running statement says it stopped, does not.
|
||||
held := at(fp("2"))
|
||||
held.Phase = nodeengineclient.PhaseHeld
|
||||
served = p.served(g, assign.Nodes{Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{"novox": {"late": held}}})
|
||||
if !served("late.t") {
|
||||
t.Error("a healthy held build of late does not serve late.t")
|
||||
}
|
||||
stopped := assign.Nodes{Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{"novox": {"late": held}},
|
||||
Running: map[string]map[string]nodeengine.Running{"novox": {"late": {Stopped: true}}}}
|
||||
sick := held
|
||||
sick.Health = "failed"
|
||||
if p.served(g, stopped)("late.t") || p.served(g, assign.Nodes{Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{"novox": {"late": sick}}})("late.t") {
|
||||
t.Error("a held build stopped, or not healthy, serves late.t")
|
||||
}
|
||||
// ADR 0614 §5: bus-state counts as offered only from a healthy build on a node present; the bus module's build on an
|
||||
// offline node offers nothing, and one whose manifest is not read counts its trunk manifest's offer. A-352: a have waiting
|
||||
// for a person's yes on a newer build names the build that still runs, healthy, and it offers bus-state too.
|
||||
|
||||
@@ -344,8 +344,12 @@ func hearEnds(hub *process.EventHub, e *pipelineEnds) error {
|
||||
|
||||
// writtenTools is the tool assigner_written: every key of the state written, with its end.
|
||||
func writtenTools(st store) []stdio.Tool {
|
||||
return []stdio.Tool{{Name: writtenTool, Description: "the pipeline ends the assigner's wants follow: per module and watched branch, the source revision and end count of the newest end heard, whether it passed and whether the wants were written from it.",
|
||||
Run: func(map[string]any) (any, error) {
|
||||
return []stdio.Tool{{Name: writtenTool, Description: "the pipeline ends the assigner's wants follow: per module and watched branch, the source revision and end count of the newest end heard, whether it passed and whether the wants were written from it; with why, why each want is what it is (as assigner_why).",
|
||||
Input: schema(map[string]string{"why": "true: why each want is what it is", "module": "with why: a module's name", "node": "with why: a node's name"}, nil, "why"),
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
if args["why"] == true || args["why"] == "true" {
|
||||
return whyTools(st)[0].Run(args) // a want's why (why.go), until the manifest naming assigner_why is taken in
|
||||
}
|
||||
ks, err := st.keys(writtenState)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -353,6 +357,9 @@ func writtenTools(st store) []stdio.Tool {
|
||||
sort.Strings(ks)
|
||||
out := map[string]End{}
|
||||
for _, k := range ks {
|
||||
if strings.HasPrefix(k, whyPrefix) {
|
||||
continue // a want's why (why.go), read by assigner_why
|
||||
}
|
||||
raw, _, ok, err := st.get(writtenState, k)
|
||||
if err != nil || !ok {
|
||||
continue
|
||||
|
||||
@@ -196,6 +196,17 @@ func TestAWantFollowsAPassedPipeline(t *testing.T) {
|
||||
t.Error("the runner's member that failed is wanted")
|
||||
}
|
||||
}
|
||||
// Design 47 G A-428: the runner's first start skips the gate while no module claiming the seat runs; another module never
|
||||
// does, and a runner running anywhere (stopped ones aside) closes it; with no running heard the gate stands.
|
||||
manifests := map[string]catalogue.Manifest{"runner": {Claims: []catalogue.Claim{{Name: mpr.Seat}}}, "web": {}}
|
||||
none := map[string]map[string]nodeengine.Running{"ace": {"web": {Fingerprint: fp1}}}
|
||||
if !runnerFirstStart("runner", manifests, none) || runnerFirstStart("web", manifests, none) || runnerFirstStart("runner", manifests, nil) {
|
||||
t.Error("the runner's first start: the runner skips the gate, another module or an unknown world does not")
|
||||
}
|
||||
if runnerFirstStart("runner", manifests, map[string]map[string]nodeengine.Running{"novox": {"runner": {Fingerprint: "fp-runner"}}}) ||
|
||||
!runnerFirstStart("runner", manifests, map[string]map[string]nodeengine.Running{"novox": {"runner": {Stopped: true}}}) {
|
||||
t.Error("a runner running closes the gate's opening; one stopped does not")
|
||||
}
|
||||
// A want that names the fingerprint already is not held by a later failure (a node placed afterwards gets it).
|
||||
e := newPipelineEnds(newMemEnds(), func() {}, t.Logf)
|
||||
e.on.Store(true)
|
||||
|
||||
@@ -266,6 +266,7 @@ func (p *passer) ask(ctx context.Context, prop mma.Proposal, akey, brief string)
|
||||
p.pulse(ctx, pk, assign.ExpiresAt(placed).Sub(p.clock()))
|
||||
}
|
||||
prop.State, prop.Ask = mmac.ProposalAsked, placed.ID
|
||||
p.logf("the proposal %s is asked of the operator: its ask %s is written under %s, which the teller raises on the operator channel", prop.ID, placed.ID, akey)
|
||||
return prop, true
|
||||
}
|
||||
|
||||
@@ -346,6 +347,14 @@ func (p *passer) stepProposal(ctx context.Context, prop mma.Proposal, rev uint64
|
||||
prop.State, prop.Ask = mmac.ProposalAnswering, ""
|
||||
case mdcc.VerdictPass:
|
||||
if prop.State != mmac.ProposalAsked || again {
|
||||
// A proposal none of whose changes the setting ask-operator-for names is applied at once, with no ask (A-425a).
|
||||
if kinds := proposalKinds(prop, v); !p.asksFor(kinds) {
|
||||
words := fmt.Sprintf("applied with no ask: its changes (%s) are none of the kinds the setting ask-operator-for names (%s); proposed by %s: %s",
|
||||
strings.Join(kinds, ", "), strings.Join(p.askOperatorFor(), ", "), prop.By, prop.Why)
|
||||
p.logf("the proposal %s is %s", prop.ID, words)
|
||||
p.apply(ctx, prop, rev, mdc.Answer{Words: words}, current, revs)
|
||||
return
|
||||
}
|
||||
var ok bool
|
||||
if prop, ok = p.ask(ctx, prop, akey, judge.Brief(prop, v)); !ok {
|
||||
return
|
||||
@@ -499,6 +508,12 @@ func (p *passer) warranted(ctx context.Context, prop mma.Proposal, rev uint64, a
|
||||
return
|
||||
}
|
||||
answer, _ := p.w.answer(akey)
|
||||
p.apply(ctx, prop, rev, answer, current, revs)
|
||||
}
|
||||
|
||||
// apply writes a proposal's laid candidates, its answer carried by assignments-changed: the operator's warranted answer, or the
|
||||
// words of a proposal applied with no ask (design 47 G A-425a).
|
||||
func (p *passer) apply(ctx context.Context, prop mma.Proposal, rev uint64, answer mdc.Answer, current map[string]mma.Assignments, revs map[string]uint64) {
|
||||
if moved(prop, revs) {
|
||||
p.layAgain(ctx, prop, rev, current, revs, "a key moved since it was approved, so its warrant is spent")
|
||||
return
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"fmt"
|
||||
"maps"
|
||||
"mesh-delivery-assigner/internal/buckets"
|
||||
"os"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
@@ -67,13 +68,17 @@ func indexerView(ask func(string, any) (json.RawMessage, error), state string) (
|
||||
}
|
||||
page := views.AskVia(ask, tool)
|
||||
out := map[string]json.RawMessage{}
|
||||
started, pages := time.Now(), 0
|
||||
for after := ""; ; {
|
||||
a, err := page(context.Background(), after)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "mesh-delivery-assigner: %s does not answer page %d after %s: %v\n", tool, pages+1, time.Since(started).Round(time.Millisecond), err)
|
||||
return nil, fmt.Errorf("%s does not answer: %w", tool, err)
|
||||
}
|
||||
pages++
|
||||
maps.Copy(out, a.Values)
|
||||
if a.Next == "" || a.Next == after {
|
||||
fmt.Fprintf(os.Stderr, "mesh-delivery-assigner: %s read, %d values in %d pages, %s\n", tool, len(out), pages, time.Since(started).Round(time.Millisecond))
|
||||
return out, nil
|
||||
}
|
||||
after = a.Next
|
||||
@@ -103,25 +108,18 @@ type keyed struct {
|
||||
// assignments reads every node's assignments and the revision of its node key (its legacy key's before it is converted),
|
||||
// assembled from its keys in either form (mmac.Keys; research 072 row 37).
|
||||
func (s *seatTools) assignments() (map[string]mma.Assignments, map[string]uint64, error) {
|
||||
keys, err := s.st.keys(mma.StateAssignments)
|
||||
all, err := readAll(s.st, mma.StateAssignments)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("assignments cannot be listed: %w", err)
|
||||
return nil, nil, fmt.Errorf("assignments cannot be read: %w", err)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
keys := slices.Sorted(maps.Keys(all))
|
||||
held := mmac.NewKeys()
|
||||
var bad []string
|
||||
for _, key := range keys {
|
||||
if _, _, _, err := mmac.ParseKey(key); err != nil {
|
||||
continue
|
||||
}
|
||||
raw, rev, found, err := s.st.get(mma.StateAssignments, key)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("assignments key %s cannot be read: %w", key, err)
|
||||
}
|
||||
if !found {
|
||||
continue
|
||||
}
|
||||
if _, err := held.Put(key, raw, rev, false); err != nil {
|
||||
if _, err := held.Put(key, all[key].raw, all[key].rev, false); err != nil {
|
||||
bad = append(bad, err.Error())
|
||||
}
|
||||
}
|
||||
@@ -145,20 +143,18 @@ func (s *seatTools) lays() (func(string) bool, func(string) bool, error) {
|
||||
held[name] = true
|
||||
}
|
||||
}
|
||||
haves, err := s.st.keys(haveState)
|
||||
haves, err := readAll(s.st, haveState)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("the nodes' records cannot be listed: %w", err)
|
||||
return nil, nil, fmt.Errorf("the nodes' records cannot be read: %w", err)
|
||||
}
|
||||
recorded := map[string]bool{}
|
||||
for _, k := range haves {
|
||||
for k, v := range haves {
|
||||
node, _, isNode, err := nodeengineclient.ParseHaveKey(k)
|
||||
if err != nil || !isNode {
|
||||
continue
|
||||
}
|
||||
if raw, _, found, err := s.st.get(haveState, k); err == nil && found {
|
||||
if h, err := nodeengineclient.DecodeHaveNode(raw); err == nil && h.Record != nil {
|
||||
recorded[node] = true
|
||||
}
|
||||
if h, err := nodeengineclient.DecodeHaveNode(v.raw); err == nil && h.Record != nil {
|
||||
recorded[node] = true
|
||||
}
|
||||
}
|
||||
return func(m string) bool { return held[m] }, func(n string) bool { return recorded[n] }, nil
|
||||
@@ -239,11 +235,21 @@ func (s *seatTools) doPropose(args map[string]any, by string) (any, error) {
|
||||
return nil, errors.New("the seed is gone with adopt: propose takes place, take-off, pin, unpin, hold, unhold, decide and config")
|
||||
}
|
||||
}
|
||||
started := time.Now()
|
||||
step := func(what string, err error) {
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "mesh-delivery-assigner: propose by %s: %s failed after %s: %v\n", by, what, time.Since(started).Round(time.Millisecond), err)
|
||||
return
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "mesh-delivery-assigner: propose by %s: %s at %s\n", by, what, time.Since(started).Round(time.Millisecond))
|
||||
}
|
||||
current, revs, err := s.assignments()
|
||||
step("assignments read", err)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
held, recorded, err := s.lays()
|
||||
step("the indexer's modules and the nodes' records read", err)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -259,11 +265,15 @@ func (s *seatTools) doPropose(args map[string]any, by string) (any, error) {
|
||||
findings = append(findings, refusal)
|
||||
}
|
||||
if len(findings) > 0 {
|
||||
step(fmt.Sprintf("refused (%d findings)", len(findings)), nil)
|
||||
return nil, fmt.Errorf("the proposal is refused: %s", strings.Join(findings, "; "))
|
||||
}
|
||||
step("judged", nil)
|
||||
if err := s.write(p); err != nil {
|
||||
step("the proposal "+p.ID+" written", err)
|
||||
return nil, err
|
||||
}
|
||||
step("the proposal "+p.ID+" written", nil)
|
||||
return proposed{Proposal: p, Follow: followProposal(p.ID)}, nil
|
||||
}
|
||||
|
||||
@@ -477,21 +487,17 @@ func (s *seatTools) doProposals(args map[string]any) (any, error) {
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the candidates cannot be listed: %w", err)
|
||||
}
|
||||
haves, err := s.st.keys(haveState)
|
||||
haves, err := readAll(s.st, haveState)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the nodes' haves cannot be listed: %w", err)
|
||||
return nil, fmt.Errorf("the nodes' haves cannot be read: %w", err)
|
||||
}
|
||||
answers := map[string]map[string]nodeengine.CandidateAnswer{} // by proposal, then node
|
||||
for _, k := range haves {
|
||||
for k, v := range haves {
|
||||
node, _, isNode, err := nodeengineclient.ParseHaveKey(k)
|
||||
if err != nil || !isNode {
|
||||
continue
|
||||
}
|
||||
raw, _, found, err := s.st.get(haveState, k)
|
||||
if err != nil || !found {
|
||||
continue
|
||||
}
|
||||
h, err := nodeengineclient.DecodeHaveNode(raw)
|
||||
h, err := nodeengineclient.DecodeHaveNode(v.raw)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -325,11 +325,11 @@ func TestTheSeatsVerbs(t *testing.T) {
|
||||
place("ace", mma.Holding{Claim: "mesh-broker", Scope: "mesh", Module: "nats"}, mma.Holding{Claim: "node-shell", Scope: "node", Module: "web"})
|
||||
place("g14", mma.Holding{Claim: "node-shell", Scope: "node", Module: "web"})
|
||||
ctx := context.Background()
|
||||
if p.seatHolders(ctx); writes != 0 {
|
||||
if p.seatHolders(ctx, nil); writes != 0 {
|
||||
t.Fatalf("seat-holders was written from an unread seat list: %v", kv.values)
|
||||
}
|
||||
p.seats = gateOver(t, false)
|
||||
p.seatHolders(ctx)
|
||||
p.seatHolders(ctx, nil)
|
||||
var broker mma.SeatHolders
|
||||
if err := json.Unmarshal(kv.values["mesh-broker"], &broker); err != nil || broker.Outside {
|
||||
t.Errorf("mesh-broker's holders %+v, %v: a seat the list names is no seat outside it", broker, err)
|
||||
@@ -337,12 +337,12 @@ func TestTheSeatsVerbs(t *testing.T) {
|
||||
if writes != 3 || kv.values["mesh-broker"] == nil || kv.values["node-shell.ace"] == nil || kv.values["node-shell.g14"] == nil {
|
||||
t.Fatalf("%d writes: %v", writes, kv.values)
|
||||
}
|
||||
p.seatHolders(ctx)
|
||||
p.seatHolders(ctx, nil)
|
||||
if writes != 3 {
|
||||
t.Errorf("an unchanged assignments wrote %d keys", writes-3)
|
||||
}
|
||||
place("g14") // g14 holds nothing now
|
||||
p.seatHolders(ctx)
|
||||
p.seatHolders(ctx, nil)
|
||||
if writes != 3 || kv.values["node-shell.g14"] != nil || kv.values["node-shell.ace"] == nil {
|
||||
t.Errorf("after g14's change: %d writes, %v", writes, kv.values)
|
||||
}
|
||||
@@ -531,6 +531,14 @@ func TestTheSeatsVerbs(t *testing.T) {
|
||||
t.Errorf("keyed again: findings %v", findings)
|
||||
}
|
||||
})
|
||||
// A whole state is read in pages where the store has them, never a key at a time (27 s for the assignments live,
|
||||
// 2026-11-14): the paged read answers what the key-by-key read answers, and asks no key alone.
|
||||
direct, err := readAll(st, haveState)
|
||||
paged := &pagedStore{memStore: st}
|
||||
viaPages, perr := readAll(paged, haveState)
|
||||
if err != nil || perr != nil || len(viaPages) != len(direct) || paged.pages == 0 || paged.gets != 0 {
|
||||
t.Errorf("readAll in pages = %d keys in %d pages with %d gets (%v, %v), want the %d keys of the direct read, no get", len(viaPages), paged.pages, paged.gets, err, perr, len(direct))
|
||||
}
|
||||
}
|
||||
|
||||
// wantsVerbs (novox/hq ADR 0565 §7, §8, §9): upgrade answers each module's policy and its source; a person's choice is
|
||||
@@ -943,3 +951,35 @@ func (f *fakeIndexer) answer(key string, body any) (json.RawMessage, bool, error
|
||||
raw, err := json.Marshal(a)
|
||||
return raw, true, err
|
||||
}
|
||||
|
||||
// pagedStore is a memStore whose states are read in pages of one key, counting pages and single gets.
|
||||
type pagedStore struct {
|
||||
memStore
|
||||
pages, gets int
|
||||
}
|
||||
|
||||
func (p *pagedStore) get(state, key string) ([]byte, uint64, bool, error) {
|
||||
p.gets++
|
||||
return p.memStore.get(state, key)
|
||||
}
|
||||
|
||||
func (p *pagedStore) page(state, after string) (stdio.StatePage, error) {
|
||||
p.pages++
|
||||
keys, err := p.memStore.keys(state)
|
||||
if err != nil {
|
||||
return stdio.StatePage{}, err
|
||||
}
|
||||
slices.Sort(keys)
|
||||
for i, k := range keys {
|
||||
if k <= after {
|
||||
continue
|
||||
}
|
||||
raw, rev, _, _ := p.memStore.get(state, k)
|
||||
pg := stdio.StatePage{Entries: []stdio.StateEntry{{Key: k, Value: raw, Revision: rev}}}
|
||||
if i < len(keys)-1 {
|
||||
pg.Next = k
|
||||
}
|
||||
return pg, nil
|
||||
}
|
||||
return stdio.StatePage{}, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A module a candidate adds with no release heard is said once, not each pass.
|
||||
func TestASkippedAddIsSaidOncePerModuleAndFingerprint(t *testing.T) {
|
||||
var logs []string
|
||||
p := &passer{logf: func(f string, a ...any) { logs = append(logs, strings.TrimSpace(f)) }}
|
||||
p.skippedAdd("n", "m", "fp", "why")
|
||||
p.skippedAdd("n", "m", "fp", "why")
|
||||
p.skippedAdd("n", "m", "fp2", "why")
|
||||
if len(logs) != 2 {
|
||||
t.Fatalf("said %d times, want 2: %v", len(logs), logs)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
)
|
||||
|
||||
// kept is one key's value and revision, as a whole state read gives it.
|
||||
type kept struct {
|
||||
raw []byte
|
||||
rev uint64
|
||||
}
|
||||
|
||||
// pager is a store that reads a state's values many to a request (stdio.KeptState.Page): the runtime's store.
|
||||
type pager interface {
|
||||
page(state, after string) (stdio.StatePage, error)
|
||||
}
|
||||
|
||||
// pageBytes and pageBudget bound one page of a whole state read, so its answer fits one bus message and its time.
|
||||
const (
|
||||
pageBytes = 768 * 1024
|
||||
pageBudget = 10 * time.Second
|
||||
)
|
||||
|
||||
func (runtimeStore) page(state, after string) (stdio.StatePage, error) {
|
||||
return stdio.State(state).Page(after, pageBytes, pageBudget)
|
||||
}
|
||||
|
||||
// readAll is every key of a state with its value and revision: in pages when the store has them, else key by key. Key by
|
||||
// key cost one bus request each, so the assignments (about 330 keys) took 27 s to read and a proposal outlasted its
|
||||
// caller's 30 s (2026-11-14), as the indexer's view of the trunk had the hour before.
|
||||
func readAll(st store, state string) (map[string]kept, error) {
|
||||
out := map[string]kept{}
|
||||
if p, ok := st.(pager); ok {
|
||||
for after := ""; ; {
|
||||
pg, err := p.page(state, after)
|
||||
if errors.Is(err, errors.ErrUnsupported) {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s cannot be read: %w", state, err)
|
||||
}
|
||||
for _, e := range pg.Entries {
|
||||
out[e.Key] = kept{raw: e.Value, rev: e.Revision}
|
||||
}
|
||||
if pg.Next == "" || pg.Next == after {
|
||||
return out, nil
|
||||
}
|
||||
after = pg.Next
|
||||
}
|
||||
}
|
||||
keys, err := st.keys(state)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s cannot be listed: %w", state, err)
|
||||
}
|
||||
for _, k := range keys {
|
||||
raw, rev, found, err := st.get(state, k)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s %s cannot be read: %w", state, k, err)
|
||||
}
|
||||
if found {
|
||||
out[k] = kept{raw: raw, rev: rev}
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -426,6 +426,7 @@ func serveTools() error {
|
||||
// What each node's engine reads, as the assigner uses it (ADR 0618 §3): its own state engine-reads, asked again through it.
|
||||
tools = append(tools, enginereads.Tool(engineReadsTool, "the assigner", enginereads.KeptState(stdio.State(engineReadsState)), false, time.Now))
|
||||
tools = append(tools, writtenTools(runtimeStore{})...)
|
||||
tools = append(tools, whyTools(runtimeStore{})...)
|
||||
tools = append(tools, pinsTools(runtimeStore{}, stdio.Ask)...)
|
||||
tools = append(tools, askAgainTools(runtimeStore{}, stdio.Ask, time.Now)...)
|
||||
return stdio.Serve("", tools)
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"slices"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
@@ -50,7 +51,7 @@ func (p *passer) served(g graph.Graph, n assign.Nodes) func(string) bool {
|
||||
if len(g) == 0 {
|
||||
return nil
|
||||
}
|
||||
served, unread := graph.Served(n.Haves, func(module, fp string) ([]string, bool) { return p.servesOf(module, fp) },
|
||||
served, unread := graph.Served(servingHaves(n, p.countedServing), func(module, fp string) ([]string, bool) { return p.servesOf(module, fp) },
|
||||
func(module string) []string { return g[module].Serves })
|
||||
if len(unread) > 0 {
|
||||
p.logf("the manifest of the running build of %v cannot be read, so what the module graph says they serve is counted as served", unread)
|
||||
@@ -108,3 +109,40 @@ func (p *passer) manifestOf(module, fp string) (catalogue.Manifest, bool) {
|
||||
}
|
||||
return got.manifest, true
|
||||
}
|
||||
|
||||
// servingHaves are the haves rule 8 counts (design 47 G A-426, A-426a): a node that runs a healthy build of a part serves what that
|
||||
// build serves, whatever its version and whether or not the catalog lists it, so a have held (its build has no catalog entry, a
|
||||
// put holds it) or waiting for a person's yes, whose build runs healthy, counts as running; a running statement that names a build
|
||||
// stopped, or not healthy, does not. counted is told each build so counted that its phase alone would have left out.
|
||||
func servingHaves(n assign.Nodes, counted func(node, module string, h nodeengine.HaveByNodeModuleValue)) map[string]map[string]nodeengine.HaveByNodeModuleValue {
|
||||
out := map[string]map[string]nodeengine.HaveByNodeModuleValue{}
|
||||
for node, haves := range n.Haves {
|
||||
out[node] = map[string]nodeengine.HaveByNodeModuleValue{}
|
||||
for module, h := range haves {
|
||||
if (h.Phase == nodeengineclient.PhaseHeld || h.Phase == nodeengineclient.PhaseWaitingYes) && h.Health == nodeengineclient.HealthOK {
|
||||
if r, ok := n.Running[node][module]; !ok || (!r.Stopped && (r.Healthy == nil || *r.Healthy)) {
|
||||
if counted != nil {
|
||||
counted(node, module, h)
|
||||
}
|
||||
h.Phase = nodeengineclient.PhaseRunning
|
||||
}
|
||||
}
|
||||
out[node][module] = h
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// countedServing says once per node, module and build that rule 8 counted a build whose have is not in phase running as serving
|
||||
// its verbs (A-426a): the rule says which build it counted.
|
||||
func (p *passer) countedServing(node, module string, h nodeengine.HaveByNodeModuleValue) {
|
||||
key := "serving " + node + "." + module + "@" + h.Fp + "." + h.Phase
|
||||
if p.skipped[key] {
|
||||
return
|
||||
}
|
||||
if p.skipped == nil {
|
||||
p.skipped = map[string]bool{}
|
||||
}
|
||||
p.skipped[key] = true
|
||||
p.logf("%s on %s runs %s healthy in phase %s: counted as serving what that build serves, whatever its version (design 47 G A-426a)", module, node, shortHex(h.Fp), h.Phase)
|
||||
}
|
||||
|
||||
@@ -71,21 +71,15 @@ func (s *wantsTools) values(state string) (map[string][]byte, error) {
|
||||
return s.direct(state)
|
||||
}
|
||||
|
||||
// direct reads a state key by key, for a small state of the seat's own.
|
||||
// direct reads a state whole, in pages where the runtime has them (readAll).
|
||||
func (s *wantsTools) direct(state string) (map[string][]byte, error) {
|
||||
keys, err := s.st.keys(state)
|
||||
all, err := readAll(s.st, state)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s cannot be listed: %w", state, err)
|
||||
return nil, err
|
||||
}
|
||||
out := map[string][]byte{}
|
||||
for _, k := range keys {
|
||||
raw, _, found, err := s.st.get(state, k)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s %s cannot be read: %w", state, k, err)
|
||||
}
|
||||
if found {
|
||||
out[k] = raw
|
||||
}
|
||||
out := make(map[string][]byte, len(all))
|
||||
for k, v := range all {
|
||||
out[k] = v.raw
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,155 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
)
|
||||
|
||||
// Why a want is what it is (lead-47 2026-11-15, the tools rule: novox's want of the indexer stood unwritten for a day and nothing
|
||||
// said why). Each pass writes, in the state written beside the pipeline ends, one key `why.<node>.<module>`: the step of the pass
|
||||
// that decided the want and what it read there. A key whose time stands still says the pass no longer judges that module.
|
||||
// assigner_why reads them.
|
||||
|
||||
// whyPrefix starts every key of a want's why in the state written; a pipeline end's key starts with a module's name.
|
||||
const whyPrefix = "why."
|
||||
|
||||
// whyTool is the tool that reads them.
|
||||
const whyTool = "assigner_why"
|
||||
|
||||
// The steps of the pass that decide a want.
|
||||
const (
|
||||
whyNoPipeline = "no-passed-pipeline" // the trunk's fingerprint passed no pipeline on the branch, and no want names it
|
||||
whyDelivery = "waits-for-delivery" // a dependency's newest released version is not delivered yet
|
||||
whyRules = "rules" // the six rules judged it
|
||||
)
|
||||
|
||||
// wantWhy is one key's value.
|
||||
type wantWhy struct {
|
||||
At time.Time `json:"at"`
|
||||
Branch string `json:"branch,omitempty"`
|
||||
// Fp is the fingerprint the trunk's entry names, the one the pass judged.
|
||||
Fp string `json:"fp,omitempty"`
|
||||
Step string `json:"step"`
|
||||
// Before is the want that stood before the pass, After the one the pass wrote (none: it kept Before).
|
||||
Before *mwc.Want `json:"before,omitempty"`
|
||||
After *mwc.Want `json:"after,omitempty"`
|
||||
// Rule is the rule holding the node back from Fp, with its words.
|
||||
Rule string `json:"rule,omitempty"`
|
||||
// What the rules read of the module.
|
||||
Built, BuildFailed, Held, Granted, Released, Kept bool `json:",omitempty"`
|
||||
BuildError string `json:"build-error,omitempty"`
|
||||
Acting []string `json:"acting,omitempty"`
|
||||
Words string `json:"words,omitempty"`
|
||||
}
|
||||
|
||||
func whyKey(node, module string) string { return whyPrefix + node + "." + module }
|
||||
|
||||
// sayWhy writes the why of module on each of nodes, once per change (the time aside): every put is a change its watchers hear.
|
||||
func (p *passer) sayWhy(ctx context.Context, module string, nodes []string, w wantWhy, before map[string]mwc.Want, out *assign.Outcome, m *assign.Module) {
|
||||
kv := p.kvs[writtenBucket()]
|
||||
if kv == nil {
|
||||
return
|
||||
}
|
||||
if p.whys == nil {
|
||||
p.whys = map[string]string{}
|
||||
}
|
||||
for _, node := range nodes {
|
||||
one := w
|
||||
one.At = time.Now().UTC()
|
||||
if b, ok := before[node]; ok {
|
||||
one.Before = &b
|
||||
}
|
||||
if out != nil {
|
||||
if a, ok := out.Wants[node]; ok {
|
||||
one.After = &a
|
||||
}
|
||||
if h, ok := out.Rules[node]; ok {
|
||||
one.Rule = h.Reason()
|
||||
}
|
||||
}
|
||||
if m != nil {
|
||||
one.Built, one.BuildFailed, one.BuildError, one.Held, one.Granted, one.Released, one.Kept = m.Built, m.BuildFailed, m.BuildError, m.Held, m.Granted, m.Released, m.Kept
|
||||
if m.Acting != nil {
|
||||
for _, e := range m.Acting(node) {
|
||||
one.Acting = append(one.Acting, e.Step+" "+e.Node+"/"+e.Part+"@"+shortHex(e.Fp)+" (seat "+e.Seat+")")
|
||||
}
|
||||
}
|
||||
}
|
||||
same := one
|
||||
same.At = time.Time{}
|
||||
cmp, _ := json.Marshal(same)
|
||||
key := whyKey(node, module)
|
||||
if p.whys[key] == string(cmp) {
|
||||
continue
|
||||
}
|
||||
raw, err := json.Marshal(one)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if _, err := kv.Put(ctx, key, raw); err != nil {
|
||||
p.logf("the why of %s on %s is not written: %v", module, node, err)
|
||||
continue
|
||||
}
|
||||
p.whys[key] = string(cmp)
|
||||
}
|
||||
}
|
||||
|
||||
// whyTools is the tool assigner_why: every want's why, or those of one module or one node.
|
||||
func whyTools(st store) []stdio.Tool {
|
||||
return []stdio.Tool{{Name: whyTool, Description: "why a want is what it is: per node and module, the step of the assigner's last pass that decided the want (no passed pipeline, waits for a dependency's delivery, or the six rules with the rule holding it back), the fingerprint the trunk names, the want before and the one written, and what the rules read (built, held, granted, acting waits). A time that stands still says the pass no longer judges that module. Arguments: module and node narrow it.",
|
||||
Input: schema(map[string]string{"module": "a module's name", "node": "a node's name"}, nil),
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
ks, err := st.keys(writtenState)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
module, node := arg(args, "module"), arg(args, "node")
|
||||
sort.Strings(ks)
|
||||
out := map[string]wantWhy{}
|
||||
for _, k := range ks {
|
||||
rest, ok := strings.CutPrefix(k, whyPrefix)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
n, mod, _ := strings.Cut(rest, ".")
|
||||
if (module != "" && mod != module) || (node != "" && n != node) {
|
||||
continue
|
||||
}
|
||||
raw, _, ok, err := st.get(writtenState, k)
|
||||
if err != nil || !ok {
|
||||
continue
|
||||
}
|
||||
var w wantWhy
|
||||
if json.Unmarshal(raw, &w) == nil {
|
||||
out[rest] = w
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}}}
|
||||
}
|
||||
|
||||
// sayPass writes, under `why.pass`, whether the last pass judged the wants at all, once per change of its words.
|
||||
func (p *passer) sayPass(ctx context.Context, words string) {
|
||||
kv := p.kvs[writtenBucket()]
|
||||
if kv == nil {
|
||||
return
|
||||
}
|
||||
if p.whys == nil {
|
||||
p.whys = map[string]string{}
|
||||
}
|
||||
if p.whys[whyPrefix+"pass"] == words {
|
||||
return
|
||||
}
|
||||
raw, _ := json.Marshal(wantWhy{At: time.Now().UTC(), Step: words})
|
||||
if _, err := kv.Put(ctx, whyPrefix+"pass", raw); err == nil {
|
||||
p.whys[whyPrefix+"pass"] = words
|
||||
}
|
||||
}
|
||||
@@ -407,8 +407,19 @@ func (w *world) wantsOf() map[string]map[string]mwc.Want {
|
||||
// nodes is what the six rules read of the nodes: presence, haves, and every provision's providers with their nodes.
|
||||
func (w *world) nodes(manifests map[string]catalogue.Manifest, assigned map[string][]string) assign.Nodes {
|
||||
n := assign.Nodes{Online: map[string]bool{}, Running: w.runningNow(), Haves: map[string]map[string]nodeengine.HaveByNodeModuleValue{}, Providers: map[string]map[string][]string{},
|
||||
Reads: map[string][]string{}}
|
||||
Reads: map[string][]string{}, Capable: map[string][]string{}}
|
||||
n.Online = w.online()
|
||||
// The build a running statement names serves an acting edge's seat when its manifest claims it (design 47 G A-426).
|
||||
n.Serves = func(r nodeengine.Running, seat string) (bool, bool) {
|
||||
if r.Manifest == "" {
|
||||
return false, false
|
||||
}
|
||||
got := w.manifest(r.Manifest)
|
||||
if got.err != "" {
|
||||
return false, false
|
||||
}
|
||||
return got.manifest.ClaimsSeat(seat), true
|
||||
}
|
||||
for key, value := range w.read(buckets.Have) {
|
||||
node, module, isNode, err := nodeengineclient.ParseHaveKey(key)
|
||||
if err != nil {
|
||||
@@ -418,6 +429,7 @@ func (w *world) nodes(manifests map[string]catalogue.Manifest, assigned map[stri
|
||||
// What the node's engine reads (ADR 0614 §4); a record that states none is read as the baseline.
|
||||
if h, err := nodeengineclient.DecodeHaveNode(value); err == nil && h.Record != nil {
|
||||
n.Reads[node] = h.Record.ManifestReads
|
||||
n.Capable[node] = h.Record.Capabilities // where a node-engine of G states want-parts (§525)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"fmt"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient"
|
||||
"regexp"
|
||||
"time"
|
||||
|
||||
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
|
||||
@@ -13,6 +14,10 @@ import (
|
||||
|
||||
// assignerAsk is the asking-th ask a, as the asks state holds it: the router's ask as JSON (asks.Ask.Ask).
|
||||
func assignerAsk(asking int, a oc.Ask) asks.Ask {
|
||||
a.Headline, a.Explanation, a.Details, a.OnExpiry = said(a.Headline), said(a.Explanation), said(a.Details), said(a.OnExpiry)
|
||||
for i := range a.Options {
|
||||
a.Options[i].Label, a.Options[i].Does = said(a.Options[i].Label), said(a.Options[i].Does)
|
||||
}
|
||||
raw, err := json.Marshal(a)
|
||||
if err != nil { // an oc.Ask holds only strings, bools and a slice of them: it cannot fail
|
||||
panic(fmt.Sprintf("assign: the ask %q does not marshal: %v", a.ID, err))
|
||||
@@ -20,6 +25,17 @@ func assignerAsk(asking int, a oc.Ask) asks.Ask {
|
||||
return asks.Ask{Asking: asking, Ask: raw}
|
||||
}
|
||||
|
||||
// longHex is a digest or fingerprint whole: 64 hexadecimal digits.
|
||||
var longHex = regexp.MustCompile(`[0-9a-f]{64}`)
|
||||
|
||||
// said is words as an ask carries them out of the mesh: each digest or fingerprint cut to its first 12 digits, which the
|
||||
// router lets pass, where it refuses a long hexadecimal string as a secret's shape (mesh-sdk outward; novox 2026-11-16 01:24:
|
||||
// every backup ask was refused for its "(composition sha256:…)" and the fingerprints in the node's words, and the operator
|
||||
// was never asked).
|
||||
func said(words string) string {
|
||||
return longHex.ReplaceAllStringFunc(words, func(h string) string { return h[:12] })
|
||||
}
|
||||
|
||||
// AskOf is the router's ask an asks-state value carries.
|
||||
func AskOf(a asks.Ask) (oc.Ask, error) {
|
||||
var ask oc.Ask
|
||||
|
||||
@@ -288,9 +288,10 @@ func NextBackupAsking(after int) int {
|
||||
return max(after+1, 2)
|
||||
}
|
||||
|
||||
var backupDigest = regexp.MustCompile(`\(composition (sha256:[0-9a-f]{64})\)`)
|
||||
var backupDigest = regexp.MustCompile(`\(composition (sha256:[0-9a-f]{12})\)`)
|
||||
|
||||
// BackupAskDigest is the digest of the composition a backup ask was made about, from its explanation; empty for an ask that names
|
||||
// BackupAskDigest is the digest of the composition a backup ask was made about, from its explanation, cut to its first 12 digits as
|
||||
// the ask says it (ShortDigest); empty for an ask that names
|
||||
// none. An answer is taken for the composition of the ask it answers, and a new revision at the same fingerprint is asked again
|
||||
// under a new number (ADR 0539 §16).
|
||||
func BackupAskDigest(a asks.Ask) string {
|
||||
@@ -304,6 +305,10 @@ func BackupAskDigest(a asks.Ask) string {
|
||||
return ""
|
||||
}
|
||||
|
||||
// ShortDigest is a composition's digest as a backup ask names it, its first 12 digits (said): the ask stays bound to its
|
||||
// composition, and a new composition is asked again.
|
||||
func ShortDigest(digest string) string { return said(digest) }
|
||||
|
||||
// BackupAnswerWords are the words of a want carrying the answer to the asking-th ask: each asking's answer is another want
|
||||
// value, so a backup that fails again after "back up again" and is answered "again" again changes the key and the node retries.
|
||||
func BackupAnswerWords(option string, asking int) string {
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-teller/tellerevents"
|
||||
"mesh-delivery-assigner/internal/assignment"
|
||||
"mesh-delivery-assigner/internal/puts"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -366,8 +367,13 @@ func TestTheRecordAndBackupAsksAreValid(t *testing.T) {
|
||||
if NextBackupAsking(-1) != 0 || NextBackupAsking(0) != 2 || NextBackupAsking(2) != 3 {
|
||||
t.Errorf("the numbers of the askings: %d %d %d", NextBackupAsking(-1), NextBackupAsking(0), NextBackupAsking(2))
|
||||
}
|
||||
if BackupAskDigest(b) != dgst {
|
||||
t.Errorf("the ask names composition %q, want %q", BackupAskDigest(b), dgst)
|
||||
if BackupAskDigest(b) != ShortDigest(dgst) || ShortDigest(dgst) == dgst {
|
||||
t.Errorf("the ask names composition %q, want %q", BackupAskDigest(b), ShortDigest(dgst))
|
||||
}
|
||||
// novox 2026-11-16: the router refuses a long hexadecimal string, so no word of an ask carries a digest or fingerprint whole.
|
||||
held := BackupAsk(bkey, "novox", "mail", fNew, dgst, "the kept data between "+fNew+" and "+fNew+" is not known", 0, now)
|
||||
if a := mustAsk(t, held); regexp.MustCompile(`[0-9a-f]{20,}`).MatchString(a.Headline + a.Explanation + a.Details) {
|
||||
t.Errorf("the ask carries a long hexadecimal string: %q", a.Explanation)
|
||||
}
|
||||
if BackupAnswerWords("again", 0) == BackupAnswerWords("again", 1) {
|
||||
t.Errorf("two answers of the same option are one want value")
|
||||
|
||||
@@ -79,6 +79,23 @@ func StepPut(w PutWorld) PutNext {
|
||||
if !w.Assigned {
|
||||
return say(puts.Refused, "%s is not assigned to %s: a put moves a module where it stands assigned (propose its assignments first)", w.Module, w.Node)
|
||||
}
|
||||
if p.Step != puts.Wanted && p.Step != puts.Holding && w.Build == "" {
|
||||
// Only where the builder's store holds no build of it: one it built goes the put's whole way (composition, the yes).
|
||||
if r := w.Running; r != nil && !r.Stopped && r.Fingerprint != "" && r.Fingerprint == p.FP {
|
||||
// The build the node runs now (design 47 G A-427): read from the node's running statement, held there with the
|
||||
// composition its have names, and the builder's store is not asked for it (a part started by hand has none there).
|
||||
if h := w.Have; h == nil || h.Fp != p.FP || h.Composed == "" {
|
||||
return say(puts.Refused, "%s runs %s at %s, and its have names no composition at that fingerprint, so the want cannot name what runs", w.Node, w.Module, fp)
|
||||
}
|
||||
p.Composed = w.Have.Composed
|
||||
words := fmt.Sprintf("the want of %s on %s names %s, the build it runs now (composition %s), from its running statement; the builder's store is not asked",
|
||||
w.Module, w.Node, fp, short(p.Composed))
|
||||
if p.Hold {
|
||||
words += ", held until released"
|
||||
}
|
||||
return PutNext{Put: applied(w, p.Said(w.Now, puts.Wanted, words))}
|
||||
}
|
||||
}
|
||||
switch w.Build {
|
||||
case builds.BuildDone:
|
||||
case builds.BuildFailed:
|
||||
@@ -86,8 +103,13 @@ func StepPut(w PutWorld) PutNext {
|
||||
case "":
|
||||
if p.FP != w.Newest {
|
||||
// The builder builds the trunk's newest fingerprint and the candidates', and takes no ask for another yet (ADR 0546
|
||||
// clause 3): an older fingerprint it never built, or whose key it removed, cannot be put.
|
||||
return say(puts.Refused, "the builder holds no build of %s at %s, and builds by itself only the trunk's newest (%s): put that, or a version it built", w.Module, fp, short(w.Newest))
|
||||
// clause 3): an older fingerprint it never built, or whose key it removed, cannot be put; the build the node runs now
|
||||
// can (A-427), and the refusal names it.
|
||||
runs := "states no running build of it"
|
||||
if r := w.Running; r != nil && !r.Stopped && r.Fingerprint != "" {
|
||||
runs = "runs " + short(r.Fingerprint)
|
||||
}
|
||||
return say(puts.Refused, "the builder holds no build of %s at %s, and builds by itself only the trunk's newest (%s), and %s %s by its running statement: put that, or a version the builder built", w.Module, fp, short(w.Newest), w.Node, runs)
|
||||
}
|
||||
return say(puts.Build, "waits for the builder's build of %s at %s, the trunk's newest, which it builds by itself", w.Module, fp)
|
||||
default:
|
||||
|
||||
@@ -108,7 +108,7 @@ type Module struct {
|
||||
}
|
||||
|
||||
// ActingEdge is one acting edge: Part's version Fp acts in Step on Node, before the entry moves.
|
||||
type ActingEdge struct{ Node, Part, Fp, Step string }
|
||||
type ActingEdge struct{ Node, Part, Fp, Step, Seat string }
|
||||
|
||||
// Runs says the part's version of the edge runs on its node, as the node-engine seat's `running` says it.
|
||||
func (n Nodes) Runs(e ActingEdge) bool {
|
||||
@@ -126,13 +126,19 @@ func (m Module) actingWaiting(node string, n Nodes) (string, bool) {
|
||||
return "", false
|
||||
}
|
||||
for _, e := range m.Acting(node) {
|
||||
if !n.Runs(e) {
|
||||
if _, runs := n.ActingRuns(e); !runs {
|
||||
return fmt.Sprintf("%s on %s waits in %s for %s at %s to run on %s", m.Name, node, e.Step, e.Part, short(e.Fp), e.Node), true
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
// ReadsParts says whether node's node-engine reads a want's version, parts and kept-on: it states WantParts among its record's
|
||||
// capabilities, where a node-engine of G states it (design 47 G §525), or, as read before, among its manifest reads.
|
||||
func (n Nodes) ReadsParts(node string) bool {
|
||||
return slices.Contains(n.Capable[node], WantParts) || slices.Contains(n.Reads[node], WantParts)
|
||||
}
|
||||
|
||||
// WantParts is the capability a node-engine of G states (ADR 0614 §4, design 47 G §525): it reads a want's `parts` and `kept-on`.
|
||||
const WantParts = "want-parts"
|
||||
|
||||
@@ -153,10 +159,29 @@ func (n Nodes) withKept(m Module, node string, w mwc.Want) mwc.Want {
|
||||
return w
|
||||
}
|
||||
k, ok := m.KeptBuilds(w.Fp)
|
||||
if !ok || !slices.Contains(n.Reads[node], WantParts) {
|
||||
if !ok {
|
||||
return w
|
||||
}
|
||||
w.Parts, w.KeptOn, w.Version = k.Parts, k.KeptOn, k.Version
|
||||
if k.Version != "" {
|
||||
// A want names the module and its version, never its artifacts' digests (design 47 G A-430, A-430a): the node-engine
|
||||
// resolves each artifact at that version through its kind's registry seat.
|
||||
w.Version, w.Parts = k.Version, nil
|
||||
if k.KeptOn != "" {
|
||||
w.KeptOn = k.KeptOn
|
||||
}
|
||||
// Bootstrap, until G is delivered (lead-47, 2026-11-15): a node whose node-engine reads parts gets them beside the
|
||||
// version, so it composes without asking a registry seat its grant does not yet admit (the grant follows the
|
||||
// node-engine build wanted there, which needs a pipeline, which needs the pipeline runner this composes).
|
||||
if n.ReadsParts(node) {
|
||||
w.Parts = k.Parts
|
||||
}
|
||||
return w
|
||||
}
|
||||
// A build with no version known to the assigner (another watched branch's, a kept one: build.ended names none) is named by its
|
||||
// parts and keeping node as before, for a node that reads them, until the builder says the version it numbered it under.
|
||||
if n.ReadsParts(node) {
|
||||
w.Parts, w.KeptOn = k.Parts, k.KeptOn
|
||||
}
|
||||
return w
|
||||
}
|
||||
|
||||
@@ -182,6 +207,49 @@ type Nodes struct {
|
||||
// Reads is, by node, the reader capabilities its node-engine states in its node record (nodeengine.NodeRecord.ManifestReads,
|
||||
// ADR 0614 §4). A node with none, or none read, is read as stating the baseline (catalogue.StatedManifestReads).
|
||||
Reads map[string][]string
|
||||
// Capable is, by node, the capabilities its node-engine states in its node record (nodeengine.NodeRecord.Capabilities): a
|
||||
// node-engine of G states WantParts there (design 47 G §525).
|
||||
Capable map[string][]string
|
||||
// Serves says whether the build of a running statement serves an acting edge's seat, from that build's manifest: known false
|
||||
// when its manifest cannot be read (design 47 G A-426). Nil: none is read.
|
||||
Serves func(r nodeengine.Running, seat string) (serves, known bool)
|
||||
}
|
||||
|
||||
// ActingRuns says whether an acting edge's part runs a build serving the edge's seat on its node, and which build (design 47 G
|
||||
// A-426): any running build of the part that serves the seat ends the wait, whatever version its newest is, so a wait on a version
|
||||
// not yet built is never held for that alone. A build whose manifest cannot be read counts as serving it when it is the part's
|
||||
// (the part holds the seat by its claim). Without the running statements the have is read as before: the edge's version only.
|
||||
func (n Nodes) ActingRuns(e ActingEdge) (string, bool) {
|
||||
if n.Running == nil {
|
||||
return e.Fp, n.Runs(e)
|
||||
}
|
||||
r, ok := n.Running[e.Node][e.Part]
|
||||
if !ok {
|
||||
// No running statement names the part on its node: its node-engine states running only of what it applied itself (a
|
||||
// part placed before, the node-engine itself, or an engine from before G states none), so the part is read from its
|
||||
// have: a build running there, its health check not failed (such an engine may write no health), ends the wait,
|
||||
// whatever its version, as a statement would (A-426). Read by the edge's version only, a part placed there waited for
|
||||
// ever (lead-47 2026-11-16: the indexer on novox for the assigner on ace and novox's own node-engine).
|
||||
if h, ok := n.Haves[e.Node][e.Part]; ok && h.Fp != "" && h.Phase == nodeengineclient.PhaseRunning && !failed(h) {
|
||||
return h.Fp, true
|
||||
}
|
||||
// The node's own node-engine writes that node's haves and keeps it heard: a node online runs its node-engine, whatever
|
||||
// phase or health its have of itself says (novox's said held, health unknown, while it composed every other module).
|
||||
if h, ok := n.Haves[e.Node][e.Part]; ok && e.Seat == nodeengine.Seat && n.Online[e.Node] {
|
||||
return h.Fp, true
|
||||
}
|
||||
return e.Fp, n.Runs(e)
|
||||
}
|
||||
if r.Stopped || r.Fingerprint == "" {
|
||||
return "", false
|
||||
}
|
||||
if r.Fingerprint == e.Fp || e.Seat == "" || n.Serves == nil {
|
||||
return r.Fingerprint, true
|
||||
}
|
||||
if serves, known := n.Serves(r, e.Seat); known && !serves {
|
||||
return "", false
|
||||
}
|
||||
return r.Fingerprint, true
|
||||
}
|
||||
|
||||
// healthy says whether a have runs fp healthy.
|
||||
@@ -670,6 +738,11 @@ func Wants(m Module, n Nodes, wants map[string]mwc.Want, lastHealthy map[string]
|
||||
if !write {
|
||||
continue
|
||||
}
|
||||
// Every want that names a build names its version (A-430a), whichever rule made or kept it: a want held at its fingerprint
|
||||
// by the rules, a node's own (self) want too.
|
||||
if !next.Removed && next.Fp != "" && next.Version == "" {
|
||||
next = n.withKept(m, node, next)
|
||||
}
|
||||
if !had || !mwc.SameWant(normal(before), normal(next)) {
|
||||
out.Wants[node] = next
|
||||
}
|
||||
|
||||
@@ -424,6 +424,17 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) {
|
||||
return composedEverywhere(node, fp)
|
||||
}
|
||||
}, want: mwc.Want{Fp: fNew, Self: true}},
|
||||
// A-430a: a placed node's want held at F by the rules names F's release, as a moved want does (novox's want of
|
||||
// mesh-delivery-pipeline stood as {fp, self}, the hold having rebuilt it without its version).
|
||||
{name: "placed, held at F, names its release", before: fNew, set: func(m *Module, n *Nodes) {
|
||||
atF(m, n)
|
||||
m.Held = true
|
||||
m.Self = func(node string) bool { return node == "g14" }
|
||||
m.KeptBuilds = func(fp string) (KeptBuild, bool) {
|
||||
return KeptBuild{Parts: map[string]string{"mail:image": "sha256:1"}, KeptOn: "g14", Version: "1.2.3"}, fp == fNew
|
||||
}
|
||||
}, rule: mwc.RuleNotHeld, want: mwc.Want{Fp: fNew, Self: true, Version: "1.2.3", KeptOn: "g14", Yes: dig(fNew),
|
||||
Words: "mail is held at " + short(fNew) + " and g14 ran no other fingerprint healthy to set it back to"}},
|
||||
} {
|
||||
m, n := ready()
|
||||
c.set(&m, &n)
|
||||
@@ -882,6 +893,21 @@ func stepsOfAPut(t *testing.T) {
|
||||
{"not assigned", func() PutWorld { w := bus(); w.Assigned = false; return w }, puts.Refused, "not assigned", false, false, false},
|
||||
{"no build yet of the newest", func() PutWorld { w := bus(); w.Build = ""; return w }, puts.Build, "builds by itself", false, false, false},
|
||||
{"no build of an older one", func() PutWorld { w := bus(); w.Build, w.Newest = "", fpOf('c'); return w }, puts.Refused, "holds no build", false, false, false},
|
||||
// A-427: the build the node runs now is put from its running statement, held with the have's composition, the
|
||||
// builder's store not asked (a part started by hand has no build there); a have naming no composition refuses it.
|
||||
{"the build the node runs now, not in the builder's store: held there", func() PutWorld {
|
||||
w := bus()
|
||||
w.Build, w.Newest = "", fpOf('c')
|
||||
w.Running = &nodeengine.Running{Node: "novox", Module: "nats", Fingerprint: fNew}
|
||||
w.Have = &nodeengine.HaveByNodeModuleValue{Fp: fNew, Composed: dig(fNew), Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}
|
||||
return w
|
||||
}, puts.Holding, "held there until released", false, false, false},
|
||||
{"the build the node runs now, its have naming no composition: refused", func() PutWorld {
|
||||
w := bus()
|
||||
w.Build, w.Newest = "", fpOf('c')
|
||||
w.Running = &nodeengine.Running{Node: "novox", Module: "nats", Fingerprint: fNew}
|
||||
return w
|
||||
}, puts.Refused, "names no composition", false, false, false},
|
||||
{"a failed build", func() PutWorld { w := bus(); w.Build, w.BuildWords = builds.BuildFailed, "go vet"; return w }, puts.Refused, "go vet", false, false, false},
|
||||
{"no composition yet: waits", func() PutWorld { w := bus(); w.Composed = nil; return w }, puts.Compose, "no composition of nats for novox", false, false, false},
|
||||
{"composed, grant not naming the node", func() PutWorld { w := bus(); w.Granted = false; return w }, puts.Compose, "does not name novox", false, false, false},
|
||||
@@ -1584,26 +1610,32 @@ func TestAProviderRunsHealthyByItsRunning(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 G §525, §526: a want at a kept build names its parts and the node that keeps it, only for a node whose engine states
|
||||
// the capability want-parts.
|
||||
// Design 47 G A-430, A-430a (over §525, §526): a want names the module's version, never its artifacts' digests, whichever node it
|
||||
// is for; a build the assigner knows no version of (another branch's, a kept one) is named by its parts and keeping node, as
|
||||
// before, for a node whose engine reads them (its record's capabilities or its manifest reads).
|
||||
func TestAWantNamesAKeptBuildsPartsAndNodeForANodeThatReadsThem(t *testing.T) {
|
||||
kept := KeptBuild{Parts: map[string]string{"mail:image": "sha256:1"}, KeptOn: "g14", Version: "1.2.3"}
|
||||
released := KeptBuild{Parts: map[string]string{"mail:image": "sha256:1"}, KeptOn: "g14", Version: "1.2.3"}
|
||||
unversioned := KeptBuild{Parts: map[string]string{"mail:image": "sha256:1"}, KeptOn: "g14"}
|
||||
for name, row := range map[string]struct {
|
||||
reads []string
|
||||
kept func(string) (KeptBuild, bool)
|
||||
wanted bool
|
||||
reads, capable []string
|
||||
kept func(string) (KeptBuild, bool)
|
||||
version string
|
||||
parts bool
|
||||
}{
|
||||
"states it": {[]string{WantParts}, func(fp string) (KeptBuild, bool) { return kept, fp == fNew }, true},
|
||||
"does not state": {[]string{"form:oci"}, func(fp string) (KeptBuild, bool) { return kept, fp == fNew }, false},
|
||||
"build not kept": {[]string{WantParts}, func(string) (KeptBuild, bool) { return KeptBuild{}, false }, false},
|
||||
"no kept builds": {[]string{WantParts}, nil, false},
|
||||
"another fingerp": {[]string{WantParts}, func(fp string) (KeptBuild, bool) { return kept, fp == fOld }, false},
|
||||
"released, reads parts": {[]string{WantParts}, nil, func(fp string) (KeptBuild, bool) { return released, fp == fNew }, "1.2.3", true},
|
||||
"released, reads none": {[]string{"form:oci"}, nil, func(fp string) (KeptBuild, bool) { return released, fp == fNew }, "1.2.3", false},
|
||||
"unversioned, capability": {[]string{"form:oci"}, []string{WantParts}, func(fp string) (KeptBuild, bool) { return unversioned, fp == fNew }, "", true},
|
||||
"unversioned, reads none": {[]string{"form:oci"}, nil, func(fp string) (KeptBuild, bool) { return unversioned, fp == fNew }, "", false},
|
||||
"build not kept": {[]string{WantParts}, nil, func(string) (KeptBuild, bool) { return KeptBuild{}, false }, "", false},
|
||||
"no kept builds": {[]string{WantParts}, nil, nil, "", false},
|
||||
"another fingerprint's release": {[]string{WantParts}, nil, func(fp string) (KeptBuild, bool) { return released, fp == fOld }, "", false},
|
||||
} {
|
||||
m, n := ready()
|
||||
n.Reads = map[string][]string{"ace": row.reads, "g14": row.reads}
|
||||
n.Capable = map[string][]string{"ace": row.capable}
|
||||
m.KeptBuilds = row.kept
|
||||
w := Wants(m, heardOfHaves(n), wantedOld(), nil).Wants["ace"]
|
||||
if w.Fp != fNew || (w.KeptOn == "g14") != row.wanted || (w.Version == "1.2.3") != row.wanted || (len(w.Parts) == 1) != row.wanted {
|
||||
if w.Fp != fNew || w.Version != row.version || (len(w.Parts) == 1) != row.parts {
|
||||
t.Errorf("%s: ace's want = %+v", name, w)
|
||||
}
|
||||
}
|
||||
@@ -1679,27 +1711,38 @@ func TestRuleOneSaysHowTheBuildStands(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 G §266, §268 (AS9): an entry whose acting edge names a step the assigner holds waits, its want held, until the acting part's
|
||||
// version runs on the edge's node, as the node-engine seat's `running` says it; then the want moves.
|
||||
// Design 47 G §266, §268 (AS9), A-426: an entry whose acting edge names a step the assigner holds waits, its want held, only while
|
||||
// the acting part runs nothing that serves the edge's seat on the edge's node: any build of it serving the seat (an older one, a
|
||||
// hand start) ends the wait at once, a build not serving it or a stopped part holds it, and a released wait is no edge.
|
||||
func TestAWantWaitsForItsActingPartsNewVersion(t *testing.T) {
|
||||
edge := ActingEdge{Node: "g14", Part: "mesh-host", Fp: fNew, Step: "compose"}
|
||||
run := func(fp string, stopped bool) Nodes {
|
||||
edge := ActingEdge{Node: "g14", Part: "mesh-host", Fp: fNew, Step: "compose", Seat: "node-engine"}
|
||||
run := func(fp string, stopped bool, serves bool) Nodes {
|
||||
_, n := ready()
|
||||
n.Running = map[string]map[string]nodeengine.Running{"g14": {"mesh-host": {Fingerprint: fp, Stopped: stopped}}}
|
||||
n.Running = map[string]map[string]nodeengine.Running{"g14": {"mesh-host": {Fingerprint: fp, Stopped: stopped, Manifest: "m"}}}
|
||||
n.Serves = func(nodeengine.Running, string) (bool, bool) { return serves, true }
|
||||
return n
|
||||
}
|
||||
m, _ := ready()
|
||||
m.Acting = func(string) []ActingEdge { return []ActingEdge{edge} }
|
||||
m.Nodes = []string{"g14"}
|
||||
out := Wants(m, run(fOld, false), wantedOld(), nil)
|
||||
out := Wants(m, run("", true, true), wantedOld(), nil)
|
||||
if out.Wants["g14"].Fp == fNew || out.Rules["g14"].Rule != mwc.RuleComposed || !strings.Contains(out.Rules["g14"].Words, "waits in compose for mesh-host") {
|
||||
t.Fatalf("the old version runs: want %+v rule %+v", out.Wants["g14"], out.Rules["g14"])
|
||||
t.Fatalf("the part stopped: want %+v rule %+v", out.Wants["g14"], out.Rules["g14"])
|
||||
}
|
||||
if out := Wants(m, run(fNew, false), wantedOld(), nil); out.Wants["g14"].Fp != fNew {
|
||||
if out := Wants(m, run(fOld, false, false), wantedOld(), nil); out.Wants["g14"].Fp == fNew {
|
||||
t.Errorf("an old build not serving the seat runs: want %+v", out.Wants["g14"])
|
||||
}
|
||||
if out := Wants(m, run(fOld, false, true), wantedOld(), nil); out.Wants["g14"].Fp != fNew {
|
||||
t.Errorf("an old build serving the seat runs: want %+v, want the new version moved", out.Wants["g14"])
|
||||
}
|
||||
if took, runs := run(fOld, false, true).ActingRuns(edge); !runs || took != fOld {
|
||||
t.Errorf("ActingRuns took %q, %v: want the old build it runs", took, runs)
|
||||
}
|
||||
if out := Wants(m, run(fNew, false, true), wantedOld(), nil); out.Wants["g14"].Fp != fNew {
|
||||
t.Errorf("the new version runs: want %+v", out.Wants["g14"])
|
||||
}
|
||||
m.Acting = func(string) []ActingEdge { return nil } // a released wait is no edge
|
||||
if out := Wants(m, run(fOld, false), wantedOld(), nil); out.Wants["g14"].Fp != fNew {
|
||||
if out := Wants(m, run("", true, true), wantedOld(), nil); out.Wants["g14"].Fp != fNew {
|
||||
t.Errorf("released: want %+v", out.Wants["g14"])
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
|
||||
client "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
|
||||
seat "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
)
|
||||
|
||||
@@ -356,5 +357,30 @@ func TestAssignments(t *testing.T) {
|
||||
if h, _ := client.HoldersOf(keyed, "node-files", "novox"); h != nil {
|
||||
t.Errorf("a node seat nobody holds there is held by %v", h)
|
||||
}
|
||||
|
||||
// A-400: a seat no holding names is held by the first assigned module whose manifest claims it; a holding on record wins.
|
||||
assigned := map[string]seat.Assignments{
|
||||
"ace": {Node: "ace", Modules: []seat.PlacedModule{{Module: "mesh-delivery-assigner"}, {Module: "mesh-files"}}},
|
||||
"novox": {Node: "novox", Modules: []seat.PlacedModule{{Module: "nats"}, {Module: "mesh-files"}}, Holds: []seat.Holding{{Claim: "mesh-broker", Scope: "mesh", Module: "nats"}}},
|
||||
}
|
||||
manifests := map[string]catalogue.Manifest{
|
||||
"mesh-delivery-assigner": {Claims: []catalogue.Claim{{Name: "mesh-module-assigner", Scope: "mesh"}}},
|
||||
"nats": {Claims: []catalogue.Claim{{Name: "mesh-broker", Scope: "mesh"}}},
|
||||
"mesh-files": {Claims: []catalogue.Claim{{Name: "node-files"}}},
|
||||
}
|
||||
with, derived := SeatHoldersWithClaimants(assigned, manifests)
|
||||
keyed = map[string][]byte{}
|
||||
for k, v := range with {
|
||||
keyed[k], _ = json.Marshal(v)
|
||||
}
|
||||
if h, err := client.HoldersOf(keyed, "mesh-module-assigner", ""); err != nil || len(h) != 1 || h[0].Module != "mesh-delivery-assigner" || h[0].Node != "ace" {
|
||||
t.Errorf("an unheld seat's claimant is %v, %v", h, err)
|
||||
}
|
||||
if h, _ := client.HoldersOf(keyed, "node-files", "novox"); len(h) != 1 || h[0].Module != "mesh-files" {
|
||||
t.Errorf("a node seat's claimant on novox is %v", h)
|
||||
}
|
||||
if h, _ := client.HoldersOf(keyed, "mesh-broker", ""); len(h) != 1 || h[0].Node != "novox" || len(derived) != 3 {
|
||||
t.Errorf("a held seat is %v; derived %v", h, derived)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -252,3 +252,54 @@ func sortedKeys[V any](m map[string]V) []string {
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// SeatHoldersWithClaimants is SeatHoldersOf over the assignments with, for each seat no holding names (a seat held once:
|
||||
// on no node; a seat of every node: on that node), the first module assigned to a node (nodes and modules in order) whose
|
||||
// manifest claims it, as its holder (lead answer A-400, the same rule as grants' A-398; the operator, 2026-11-14: who holds
|
||||
// a seat follows from the module graph and the assignments). derived lists each key so held, for its writer to say. Pure.
|
||||
func SeatHoldersWithClaimants(assignments map[string]seat.Assignments, manifests map[string]catalogue.Manifest) (out map[string]seat.SeatHolders, derived []string) {
|
||||
held := map[string]bool{} // "<seat>" for a seat held once, "<seat>.<node>" for a seat of every node
|
||||
keyOf := func(claim, scope, node string) string {
|
||||
name := claim
|
||||
if s, ok := catalogue.SeatNamed(claim); ok {
|
||||
name = s.Name
|
||||
}
|
||||
if scope == catalogue.ScopeNode {
|
||||
return name + "." + node
|
||||
}
|
||||
return name
|
||||
}
|
||||
for _, n := range sortedKeys(assignments) {
|
||||
for _, h := range assignments[n].Holds {
|
||||
held[keyOf(h.Claim, h.Scope, n)] = true
|
||||
}
|
||||
}
|
||||
with := map[string]seat.Assignments{}
|
||||
var added []string
|
||||
for _, n := range sortedKeys(assignments) {
|
||||
a := assignments[n]
|
||||
a.Holds = append([]seat.Holding(nil), a.Holds...)
|
||||
modules := make([]string, 0, len(a.Modules))
|
||||
for _, m := range a.Modules {
|
||||
modules = append(modules, m.Module)
|
||||
}
|
||||
sort.Strings(modules)
|
||||
for _, module := range modules {
|
||||
m, ok := manifests[module]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
for _, c := range m.Claims {
|
||||
k := keyOf(c.Name, c.At(), n)
|
||||
if held[k] {
|
||||
continue
|
||||
}
|
||||
held[k] = true
|
||||
a.Holds = append(a.Holds, seat.Holding{Claim: c.Name, Module: module, Scope: c.At()})
|
||||
added = append(added, k+" by "+module+" on "+n)
|
||||
}
|
||||
}
|
||||
with[n] = a
|
||||
}
|
||||
return SeatHoldersOf(with), added
|
||||
}
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
mdcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliverycheckerclient"
|
||||
"reflect"
|
||||
"slices"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -351,3 +352,16 @@ func answerOf(body map[string]any) nodeengine.CandidateAnswer {
|
||||
_ = json.Unmarshal(raw, &out)
|
||||
return out
|
||||
}
|
||||
|
||||
// Stops are the modules a node's answer to the verdict says it would stop running, as `<node>: <module>` (design 47 G A-425a: the
|
||||
// change kind `stops`).
|
||||
func Stops(v mma.Verdict) []string {
|
||||
var out []string
|
||||
for node, body := range v.Answers {
|
||||
for _, m := range answerOf(body).Removed {
|
||||
out = append(out, node+": "+m)
|
||||
}
|
||||
}
|
||||
slices.Sort(out)
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -105,8 +105,8 @@ func TestManifest(t *testing.T) {
|
||||
// which lists and asks again what each node's engine reads.
|
||||
if !slices.Equal(m.Tools, []string{"assigner_put", "assigner_release", "assigner_puts", "assigner_waits", "assigner_inputs", "assignments", "seat-holders", "wants", "asks",
|
||||
"assigner_read_assignments", "assigner_read_seat_holders", "assigner_read_wants", "assigner_read_findings", "assigner_read_differences", "assigner_read_copies",
|
||||
"assigner_first_views", "assigner_first_view_again", "assigner_engine_reads", "assigner_seat_list", "assigner_seat_list_again", "assigner_ask_again", "assigner_written", "assigner_pins", "assigner_unpin", "assigner_delivered_again"}) {
|
||||
t.Errorf("the assigner serves tools %v, want its three put tools, three first views, six read tools, assigner_engine_reads, the seat list's two (ADR 0636 §7) and assigner_ask_again (ADR 0668 §5)", m.Tools)
|
||||
"assigner_first_views", "assigner_first_view_again", "assigner_engine_reads", "assigner_seat_list", "assigner_seat_list_again", "assigner_ask_again", "assigner_written", "assigner_why", "assigner_pins", "assigner_unpin", "assigner_delivered_again"}) {
|
||||
t.Errorf("the assigner serves tools %v, want its three put tools, three first views, six read tools, assigner_engine_reads, the seat list's two (ADR 0636 §7) assigner_ask_again (ADR 0668 §5) and assigner_why (design 47 G A-439)", m.Tools)
|
||||
}
|
||||
if m.RunsAs != m.Module || m.SecretsOwner != m.RunsAs {
|
||||
t.Errorf("runs-as = %q: the bundle runs on the assigner's own account (ADR 0621 §1)", m.RunsAs)
|
||||
@@ -166,12 +166,14 @@ func TestManifest(t *testing.T) {
|
||||
} `json:"config"`
|
||||
}
|
||||
file, _ := os.ReadFile("module.json")
|
||||
if err := json.Unmarshal(file, &raw); err != nil || raw.Config["outside-wait-minutes"].Default != float64(30) || raw.Config["outside-settle-seconds"].Default != float64(90) || len(raw.Config) != 2 {
|
||||
t.Errorf("config = %+v (%v), want outside-wait-minutes of 30 and outside-settle-seconds of 90", raw.Config, err)
|
||||
if err := json.Unmarshal(file, &raw); err != nil || raw.Config["outside-wait-minutes"].Default != float64(30) || raw.Config["outside-settle-seconds"].Default != float64(90) ||
|
||||
raw.Config["ask-operator-for"].Default != "take-off,remove-node,stops" || len(raw.Config) != 3 {
|
||||
t.Errorf("config = %+v (%v), want outside-wait-minutes of 30, outside-settle-seconds of 90 and ask-operator-for of take-off,remove-node,stops (A-425a)", raw.Config, err)
|
||||
}
|
||||
for _, f := range []struct{ env, path, key string }{
|
||||
{"MESH_ASSIGNER_OUTSIDE_WAIT_FILE", "${dir:state}/outside-wait-minutes", "outside-wait-minutes"},
|
||||
{"MESH_ASSIGNER_OUTSIDE_SETTLE_FILE", "${dir:state}/outside-settle-seconds", "outside-settle-seconds"},
|
||||
{"MESH_ASSIGNER_ASK_OPERATOR_FOR_FILE", "${dir:state}/ask-operator-for", "ask-operator-for"},
|
||||
} {
|
||||
if !strings.Contains(string(file), `"`+f.env+`": "`+f.path+`"`) || !strings.Contains(string(file), `"path": "`+f.path+`"`) ||
|
||||
!strings.Contains(string(file), `"content": "${config:`+f.key+`}"`) {
|
||||
|
||||
@@ -129,6 +129,7 @@
|
||||
"assigner_seat_list_again",
|
||||
"assigner_ask_again",
|
||||
"assigner_written",
|
||||
"assigner_why",
|
||||
"assigner_pins",
|
||||
"assigner_unpin",
|
||||
"assigner_delivered_again"
|
||||
@@ -174,6 +175,12 @@
|
||||
"maximum": 3600,
|
||||
"trusted": false,
|
||||
"default": 90
|
||||
},
|
||||
"ask-operator-for": {
|
||||
"type": "string",
|
||||
"description": "the change kinds of a proposal the assigner asks the operator before it writes it, separated by commas: place, take-off, pin, unpin, hold, unhold, decide, config, add-node, remove-node, and stops for a verdict whose answers say a node would stop running a module (design 47 G A-425, A-425a). A proposal none of whose changes is of a kind named is applied at once when its verdict passes, said in the log and by assignments-changed. The default is take-off,remove-node,stops; empty asks for nothing.",
|
||||
"trusted": false,
|
||||
"default": "take-off,remove-node,stops"
|
||||
}
|
||||
},
|
||||
"objects": [
|
||||
@@ -215,6 +222,14 @@
|
||||
"mode": "0600",
|
||||
"content": "${config:outside-settle-seconds}"
|
||||
},
|
||||
{
|
||||
"id": "ask-operator-for",
|
||||
"type": "file",
|
||||
"path": "${dir:state}/ask-operator-for",
|
||||
"owner": "mesh-delivery-assigner",
|
||||
"mode": "0600",
|
||||
"content": "${config:ask-operator-for}"
|
||||
},
|
||||
{
|
||||
"id": "assigner",
|
||||
"type": "process",
|
||||
@@ -226,7 +241,8 @@
|
||||
"env": {
|
||||
"MESH_BROKER_FILE": "${dir:state}/broker",
|
||||
"MESH_ASSIGNER_OUTSIDE_WAIT_FILE": "${dir:state}/outside-wait-minutes",
|
||||
"MESH_ASSIGNER_OUTSIDE_SETTLE_FILE": "${dir:state}/outside-settle-seconds"
|
||||
"MESH_ASSIGNER_OUTSIDE_SETTLE_FILE": "${dir:state}/outside-settle-seconds",
|
||||
"MESH_ASSIGNER_ASK_OPERATOR_FOR_FILE": "${dir:state}/ask-operator-for"
|
||||
},
|
||||
"user": "mesh-delivery-assigner",
|
||||
"health": {
|
||||
|
||||
@@ -45,7 +45,7 @@ func (m *entryManifests) keep(e meshcatalog.CatalogEntry) error {
|
||||
if m.byFp == nil {
|
||||
m.byFp = map[string]catalogue.Manifest{}
|
||||
}
|
||||
m.byFp[e.ManifestDigest] = mf
|
||||
m.byFp[meshcatalog.BareManifestDigest(e.ManifestDigest)] = mf
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -15,10 +15,10 @@ func TestTheManifestOfAnEntryIsReadByItsDigest(t *testing.T) {
|
||||
if err := m.keep(e); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if mf, err := m.Manifest(context.Background(), "sha256:aa"); err != nil || mf.Module != "netcheck" {
|
||||
t.Errorf("Manifest(sha256:aa) = %+v, %v", mf, err)
|
||||
if mf, err := m.Manifest(context.Background(), "aa"); err != nil || mf.Module != "netcheck" {
|
||||
t.Errorf("Manifest(aa) = %+v, %v", mf, err)
|
||||
}
|
||||
if _, err := m.Manifest(context.Background(), "sha256:bb"); err == nil {
|
||||
if _, err := m.Manifest(context.Background(), "bb"); err == nil {
|
||||
t.Error("Manifest of a digest no entry holds: want an error")
|
||||
}
|
||||
if err := m.keep(meshcatalog.CatalogEntry{Module: "bare"}); err != nil || len(m.byFp) != 1 {
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
)
|
||||
|
||||
// lastKept reads whether the stream keeps a message on a subject: found false when the stream or the message is not there.
|
||||
type lastKept func(ctx context.Context, subject string) (found bool, err error)
|
||||
|
||||
// lastKeptOf reads the seat event's own kept stream (the contract's StreamVersionsFilled, kept by age) the way any kept event
|
||||
// is heard (design 47 G A-411, A-233): the SDK's hearing of it from its first message, whose start read hands every message
|
||||
// kept to the handler; the reading is stopped once the read ended, the hub's own hearing following the durable. Nothing of
|
||||
// EVENTS is read.
|
||||
func lastKeptOf(js jetstream.JetStream, acct process.Account, logf func(string, ...any)) lastKept {
|
||||
return func(ctx context.Context, subject string) (bool, error) {
|
||||
var found atomic.Bool
|
||||
cc, err := acct.HearSeatEvents(ctx, js, meshcatalog.StreamVersionsFilled, func(m jetstream.Msg) {
|
||||
if m.Subject() == subject {
|
||||
found.Store(true)
|
||||
}
|
||||
_ = m.Ack() // a message of the durable is taken as the hub takes it; an ordered read's has nothing to acknowledge
|
||||
}, logf, process.FromFirst())
|
||||
if err != nil {
|
||||
if errors.Is(err, jetstream.ErrStreamNotFound) {
|
||||
return false, nil
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
cc.Stop()
|
||||
return found.Load(), nil
|
||||
}
|
||||
}
|
||||
|
||||
// readFilled reads at the builder's start the last versions-filled the stream keeps (design 47 G §369, §372, A-411) and takes it as heard; the event heard later sets it as before. Nothing found leaves it unset, said; a
|
||||
// read that failed is said and tried again in the background with backoff, never holding the start.
|
||||
func (rv *reservations) readFilled(ctx context.Context, read lastKept, wait func(context.Context, time.Duration) bool) {
|
||||
const first, most = time.Second, time.Minute
|
||||
try := func() (done bool, err error) {
|
||||
found, err := read(ctx, meshcatalog.SubjectVersionsFilled)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if !found {
|
||||
rv.logf("the stream keeps no versions-filled yet (read at start): versions are reserved once the catalog says it")
|
||||
return true, nil
|
||||
}
|
||||
rv.filled.Store(true)
|
||||
rv.logf("the catalog said versions-filled (read at start): versions are reserved from now on")
|
||||
if rv.wake != nil {
|
||||
rv.wake(ctx)
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
if done, err := try(); done {
|
||||
return
|
||||
} else {
|
||||
rv.logf("the last versions-filled is not read at start: %v; read again in the background", err)
|
||||
}
|
||||
go func() {
|
||||
for d := first; ; d = min(d*2, most) {
|
||||
if !wait(ctx, d) {
|
||||
return
|
||||
}
|
||||
done, err := try()
|
||||
if done {
|
||||
return
|
||||
}
|
||||
rv.logf("the last versions-filled is not read: %v; read again in %s", err, min(d*2, most))
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// sleeps waits d, as a context's bound (ADR 0437 §8: the builder holds no timer); false when ctx ended first.
|
||||
func sleeps(ctx context.Context, d time.Duration) bool {
|
||||
bound, cancel := context.WithTimeout(ctx, d)
|
||||
defer cancel()
|
||||
<-bound.Done()
|
||||
return ctx.Err() == nil
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"reflect"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestTheLastVersionsFilledIsReadAtStart(t *testing.T) {
|
||||
boom := errors.New("bus down")
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
results []error // per read: nil found, boom fails
|
||||
notFound bool
|
||||
wantFilled bool
|
||||
wantReads int
|
||||
waits []time.Duration
|
||||
}{
|
||||
{"kept", []error{nil}, false, true, 1, nil},
|
||||
{"none kept", []error{nil}, true, false, 1, nil},
|
||||
{"read fails then kept", []error{boom, boom, nil}, false, true, 3, []time.Duration{time.Second, 2 * time.Second}},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
rv := newReservations(nil, nil, nil, nil, nil, t.Logf)
|
||||
var mu sync.Mutex
|
||||
woke := make(chan struct{}, 1)
|
||||
rv.wake = func(context.Context) { woke <- struct{}{} }
|
||||
reads := 0
|
||||
read := func(_ context.Context, subject string) (bool, error) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
err := tc.results[reads]
|
||||
reads++
|
||||
return !tc.notFound, err
|
||||
}
|
||||
var waits []time.Duration
|
||||
wait := func(_ context.Context, d time.Duration) bool {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
waits = append(waits, d)
|
||||
return true
|
||||
}
|
||||
rv.readFilled(context.Background(), read, wait)
|
||||
if tc.wantFilled {
|
||||
select {
|
||||
case <-woke:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("the read did not take the fill")
|
||||
}
|
||||
}
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if rv.Filled() != tc.wantFilled || reads != tc.wantReads {
|
||||
t.Fatalf("filled %v reads %d", rv.Filled(), reads)
|
||||
}
|
||||
if !reflect.DeepEqual(waits, tc.waits) {
|
||||
t.Fatalf("backoff %v, want %v", waits, tc.waits)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -14,6 +14,8 @@ import (
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
|
||||
sweep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshartifactsweep"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
)
|
||||
|
||||
// A done key whose blob the artifact store no longer holds is built again, by the builder whole against the bus
|
||||
@@ -185,3 +187,31 @@ func TestArtifactsGoneLeadsTheBuilderToBuildTheKeyAgain(t *testing.T) {
|
||||
b.fake.count("mesh-build-go.build_go_build"))
|
||||
}
|
||||
}
|
||||
|
||||
// The last versions-filled is read at the start from the seat event's own kept stream, kept by age, through the SDK's hearing
|
||||
// from the first message (design 47 G A-411), never from EVENTS: a message the stream keeps is found, an empty one is none.
|
||||
func TestTheLastVersionsFilledIsReadFromItsKeptStream(t *testing.T) {
|
||||
b := startBed(t)
|
||||
acct := process.Account{Node: "n1", Module: "mesh-delivery-builder"}
|
||||
if err := b.js.DeleteStream(b.ctx, eventsStream); err != nil { // the bus keeps a kept event off EVENTS: its subject is nowhere else
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := b.js.CreateOrUpdateStream(b.ctx, jetstream.StreamConfig{Name: meshcatalog.StreamVersionsFilled,
|
||||
Subjects: []string{meshcatalog.SubjectVersionsFilled}, MaxAge: 24 * time.Hour}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := b.js.CreateOrUpdateConsumer(b.ctx, meshcatalog.StreamVersionsFilled, jetstream.ConsumerConfig{
|
||||
Durable: acct.EventConsumerName(), AckPolicy: jetstream.AckExplicitPolicy, FilterSubject: meshcatalog.SubjectVersionsFilled}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
read := lastKeptOf(b.js, acct, t.Logf)
|
||||
if found, err := read(b.ctx, meshcatalog.SubjectVersionsFilled); err != nil || found {
|
||||
t.Fatalf("empty stream: found %v, %v", found, err)
|
||||
}
|
||||
if _, err := b.js.Publish(b.ctx, meshcatalog.SubjectVersionsFilled, []byte(`{"v":1}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if found, err := read(b.ctx, meshcatalog.SubjectVersionsFilled); err != nil || !found {
|
||||
t.Fatalf("kept message: found %v, %v", found, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -226,6 +226,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
return emitSeatEvent(context.WithoutCancel(ctx), js, cred, meshbuilds.SubjectPackagePublished, "package-published-"+p.Module+"-"+p.Version+"-"+p.Artifact, p)
|
||||
}
|
||||
b.Reserved = reserved
|
||||
// The last versions-filled the stream keeps is read at the start (design 47 G §369, §372): a restart does not lose it.
|
||||
reserved.readFilled(ctx, lastKeptOf(js, cred, logger.Printf), sleeps)
|
||||
// A state bound later numbers the builds looked at without it.
|
||||
versionsKV.Ready(func(jetstream.KeyValue) { b.Reconsider(ctx) })
|
||||
// The versions' hand acts (builder_publish_again, builder_floor_again) come through reasks, as builder_build_again.
|
||||
@@ -240,7 +242,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
actedKV := process.AwaitBucket(ctx, js, conn, own.acted, "no version is published again on an event or by builder_publish_again", logger.Printf)
|
||||
republish := &versions.Republisher{Reserved: reserved.r, Queue: awaitedQueue{awaitedReserved{awaitedVersions{b: queueKV, bucket: own.queue}}},
|
||||
Acted: awaitedReserved{awaitedVersions{b: actedKV, bucket: own.acted}}, Entry: entryVia(calls.Call), Send: sendVia(calls.Call),
|
||||
Unreleased: reserved.unreleasedWords, Max: republishMax, Logf: logger.Printf}
|
||||
Unreleased: reserved.unreleasedWords, Max: republishMax, Logf: logger.Printf,
|
||||
Holders: holders} // a registry seat's holder from the seat-holders view, from the start (design 47 G A-416, A-416a)
|
||||
acts.Republish = republish.Republish
|
||||
again := &republishing{rp: republish}
|
||||
// The operator is asked once a key's event builds are spent (design 47 G §568): the builder's asker on the seat operator-channel.
|
||||
@@ -408,7 +411,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// The seat mesh-builds' verb reserved, answered here (A-115, A-127): the reservations, the catalog half's carriers and the seat
|
||||
// mesh-catalog's versions.
|
||||
stopReserved, err := calls.AnswerSeat(meshbuilds.Seat, seatlist.Definition(seatGate.Reader, meshbuilds.Seat), map[string]process.SeatHandler{
|
||||
meshbuilds.VerbReserved: reservedVerb(ctx, reservedReader(ctx, reserved.r.Store), versions.CarriersFrom(versions.EntriesOf(branches)), catalogReleased(calls.Call))})
|
||||
meshbuilds.VerbReserved: reservedVerb(ctx, reservedReader(ctx, reserved.r.Store), versions.CarriersFrom(versions.EntriesOf(branches)), catalogReleased(calls.Call),
|
||||
func(name string) {
|
||||
logger.Printf("reserved: no entry carries the name %s; the entries looked at: %s", name, versions.LookedAt(versions.EntriesOf(branches)))
|
||||
})})
|
||||
if err != nil {
|
||||
return fmt.Errorf("answering the verb reserved: %w", err)
|
||||
}
|
||||
@@ -443,7 +449,13 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// Every part has registered: the hub pulls the module's durable. Where the consumer is not there yet the builder says so
|
||||
// and binds it at a later change (heard.Writer; ADR 0615 §5).
|
||||
_ = notHeard.Bind(eventsStream, sweep.EventArtifactsGone+", what wakes a floor, the registry seats' words, the views and manifest-reads-changed are not heard until the builder starts again", func() error {
|
||||
return hub.Start(ctx, js, cred)
|
||||
if err := hub.Start(ctx, js, cred); err != nil {
|
||||
return err
|
||||
}
|
||||
// And the kept seat events it handles (the build seats' toolchain-reported, the seat mesh-catalog's versions-filled
|
||||
// and version-released), each on its own stream: the tool runner reads its own twin there (lead answer A-397).
|
||||
hub.StartKept(ctx, js, cred)
|
||||
return nil
|
||||
})
|
||||
go func() { <-ctx.Done(); hub.Stop() }()
|
||||
// A person asks a key's build again with builder_ask_again, which writes the reasks state (issue 1945): watched, as
|
||||
|
||||
@@ -31,6 +31,8 @@ import (
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds/forms"
|
||||
sweep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshartifactsweep"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
|
||||
"mesh-delivery-builder/internal/build"
|
||||
"mesh-delivery-builder/internal/versions"
|
||||
@@ -226,6 +228,8 @@ type wakeMsg struct {
|
||||
acked bool
|
||||
naked bool
|
||||
delayed time.Duration
|
||||
// delivered is the delivery count Metadata says; 0 says no metadata.
|
||||
delivered uint64
|
||||
}
|
||||
|
||||
func (m *wakeMsg) Subject() string { return m.subject }
|
||||
@@ -237,6 +241,9 @@ func (m *wakeMsg) NakWithDelay(d time.Duration) error {
|
||||
return nil
|
||||
}
|
||||
func (m *wakeMsg) Metadata() (*jetstream.MsgMetadata, error) {
|
||||
if m.delivered > 0 {
|
||||
return &jetstream.MsgMetadata{NumDelivered: m.delivered}, nil
|
||||
}
|
||||
return nil, errors.New("no metadata")
|
||||
}
|
||||
func (m *wakeMsg) Headers() nats.Header {
|
||||
@@ -271,13 +278,39 @@ func aFloorsWakeIsHeard(t *testing.T) {
|
||||
if !h.heard(context.Background(), byDigest) || !byDigest.acked || !strings.Contains(logs.String(), "the floor of m/pkg is asked again: go-package-registry said published") {
|
||||
t.Fatalf("published: acked %v, log %q", byDigest.acked, logs.String())
|
||||
}
|
||||
refused := &wakeMsg{subject: "mesh.seat.oci-artifact-registry.event.refused", data: []byte(`{"name":"m/program","version":"0.1.3","digest":"sha256:bb","reason":"bytes differ"}`)}
|
||||
if !h.heard(context.Background(), refused) || refused.acked || !refused.naked {
|
||||
t.Fatalf("a refusal the state cannot take: acked %v, naked %v, log %q", refused.acked, refused.naked, logs.String())
|
||||
// Design 47 G §369, A-415: a published of a version the reservation numbered, which the state versions does not hold, is
|
||||
// taken: its artifact is marked and package-published said, never left for the bus to give again.
|
||||
var saidPub []meshbuilds.PackagePublished
|
||||
rv := newReservations(&memReserve{vals: map[string][]byte{}, revs: map[string]uint64{}}, func(string, any) (json.RawMessage, error) {
|
||||
return json.Marshal(meshcatalog.VersionsAnswer{Module: "m"})
|
||||
}, func() jetstream.KeyValue { return &boundsKV{keys: map[string]bool{}} },
|
||||
&findings{module: builds.BuilderModule, w: &memFindings{vals: map[string][]byte{}}, logf: t.Logf, now: time.Now}, nil, t.Logf)
|
||||
rv.say = func(_ context.Context, p meshbuilds.PackagePublished) error { saidPub = append(saidPub, p); return nil }
|
||||
rv.filled.Store(true)
|
||||
res, err := rv.Reserve(context.Background(), "m", "0.1", []string{"m:program"}, versions.Source{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := rv.Built(context.Background(), "m", res.Version, map[string]string{"m:program": "sha256:bb"}, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
h.res = rv
|
||||
published := &wakeMsg{subject: "mesh.seat.oci-artifact-registry.event.published", data: []byte(`{"name":"m/program","version":"` + res.Version + `","digest":"sha256:bb","kind":"bundle"}`)}
|
||||
if !h.heard(context.Background(), published) || !published.acked || published.naked || len(saidPub) != 1 || saidPub[0].Artifact != "m:program" {
|
||||
t.Fatalf("a published of a reserved version: acked %v, naked %v, said %+v, log %q", published.acked, published.naked, saidPub, logs.String())
|
||||
}
|
||||
if h.heard(context.Background(), &wakeMsg{subject: manifestAcceptedSubject}) {
|
||||
t.Fatal("manifest-accepted was taken as a seat's outcome")
|
||||
}
|
||||
// Design 47 G A-420: a message left for the bus is said given up on the last delivery its consumer allows, never before.
|
||||
for _, n := range []uint64{4, EventDeliveries} {
|
||||
logs.Reset()
|
||||
givenUp(&wakeMsg{subject: "mesh.seat.oci-artifact-registry.event.published", delivered: n}, "oci-artifact-registry's published of m/program@0.1.6", errors.New("down"), log.New(&logs, "", 0).Printf)
|
||||
said := strings.Contains(logs.String(), "gives up mesh.seat.oci-artifact-registry.event.published: oci-artifact-registry's published of m/program@0.1.6, delivered 5 times, the most its consumer allows; the last refusal: down")
|
||||
if said != (n == EventDeliveries) {
|
||||
t.Errorf("delivery %d: log %q", n, logs.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// noVersions is a state versions that holds nothing.
|
||||
|
||||
@@ -140,6 +140,7 @@ func (g *republishing) heard(ctx context.Context, m jetstream.Msg, logf func(str
|
||||
if nakErr := m.Nak(); nakErr != nil {
|
||||
logf("leaving %s for the bus to give again: %v", m.Subject(), nakErr)
|
||||
}
|
||||
givenUp(m, "its event", err, logf)
|
||||
return true
|
||||
}
|
||||
return false
|
||||
|
||||
@@ -96,6 +96,17 @@ func (rv *reservations) Built(ctx context.Context, module, version string, diges
|
||||
return err
|
||||
}
|
||||
rv.arm(ctx, cur)
|
||||
// A published heard before the digests were recorded marks its artifact now, said as one heard after (design 47 G A-415).
|
||||
marked, unmatched, err := rv.r.Kept(ctx, module, version)
|
||||
if err != nil {
|
||||
rv.logf("the published kept on %s %s are not taken in: %v", module, version, err)
|
||||
}
|
||||
for _, h := range unmatched {
|
||||
rv.logf("a published of %s@%s kept on %s %s marks no artifact: its digest %s is none of the built digests %v", h.Name, h.Version, module, version, h.Digest, cur.Artifacts)
|
||||
}
|
||||
for _, m := range marked {
|
||||
rv.sayPublished(ctx, m, m.Reserved.Artifacts[m.Artifact])
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -181,6 +192,7 @@ func (rv *reservations) heard(ctx context.Context, m jetstream.Msg) bool {
|
||||
if nakErr := m.Nak(); nakErr != nil {
|
||||
rv.logf("leaving %s for the bus to give again: %v", m.Subject(), nakErr)
|
||||
}
|
||||
givenUp(m, "the version-released of "+ev.Module+" "+ev.Version, err, rv.logf)
|
||||
return true
|
||||
}
|
||||
gone, err := rv.r.Released(ctx, ev.Module, ev.Version)
|
||||
@@ -189,6 +201,7 @@ func (rv *reservations) heard(ctx context.Context, m jetstream.Msg) bool {
|
||||
if nakErr := m.Nak(); nakErr != nil {
|
||||
rv.logf("leaving %s for the bus to give again: %v", m.Subject(), nakErr)
|
||||
}
|
||||
givenUp(m, "the version-released of "+ev.Module+" "+ev.Version, err, rv.logf)
|
||||
return true
|
||||
}
|
||||
for _, k := range gone {
|
||||
@@ -206,18 +219,31 @@ func (rv *reservations) heard(ctx context.Context, m jetstream.Msg) bool {
|
||||
|
||||
// published marks the artifacts of the reservation a registry seat's published names (its version and digest).
|
||||
func (rv *reservations) published(ctx context.Context, o versions.Outcome) {
|
||||
marked, err := rv.r.Published(ctx, o.Version, o.Digest)
|
||||
marked, kept, err := rv.r.Published(ctx, o.Name, o.Version, o.Digest)
|
||||
if err != nil {
|
||||
rv.logf("a published of %s@%s is not recorded in the reservations: %v", o.Name, o.Version, err)
|
||||
}
|
||||
if len(kept) > 0 {
|
||||
rv.logf("the published of %s@%s (digest %s, holder %s) is kept on %s until the build records its digests", o.Name, o.Version, o.Digest, o.Holder, strings.Join(kept, ", "))
|
||||
} else if len(marked) == 0 {
|
||||
rv.logf("package-published not said: the published of %s@%s (digest %s, holder %s) marks no artifact of a reservation: %s", o.Name, o.Version, o.Digest, o.Holder, rv.noMark(ctx, o))
|
||||
}
|
||||
// Each artifact the holder's published newly marked is said as the seat mesh-builds' package-published (design 47 G §52, §334).
|
||||
for _, m := range marked {
|
||||
if rv.say == nil {
|
||||
continue
|
||||
}
|
||||
if err := rv.say(ctx, m.PackagePublished(o.Digest, time.Now())); err != nil {
|
||||
rv.logf("the package-published of %s at %s is not said: %v", m.Artifact, m.Reserved.Version, err)
|
||||
}
|
||||
rv.sayPublished(ctx, m, o.Digest)
|
||||
}
|
||||
}
|
||||
|
||||
// sayPublished says the seat mesh-builds' package-published of an artifact a published marked.
|
||||
func (rv *reservations) sayPublished(ctx context.Context, m versions.Marked, digest string) {
|
||||
if rv.say == nil {
|
||||
rv.logf("package-published of %s at %s not said: no seat to say it on", m.Artifact, m.Reserved.Version)
|
||||
return
|
||||
}
|
||||
if err := rv.say(ctx, m.PackagePublished(digest, time.Now())); err != nil {
|
||||
rv.logf("the package-published of %s at %s is not said: %v", m.Artifact, m.Reserved.Version, err)
|
||||
} else {
|
||||
rv.logf("package-published said: %s at %s, digest %s", m.Artifact, m.Reserved.Version, digest)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -261,3 +287,26 @@ func (rv *reservations) tagged(ctx context.Context, ev meshcatalog.VersionReleas
|
||||
rv.logf("%s is tagged %s at %s", ev.Module, res.Tag, res.Commit)
|
||||
return nil
|
||||
}
|
||||
|
||||
// noMark says why a published marked nothing: which reservation holds that version, and what it holds of digests.
|
||||
func (rv *reservations) noMark(ctx context.Context, o versions.Outcome) string {
|
||||
version := strings.TrimPrefix(o.Version, "v")
|
||||
if version == "" || o.Digest == "" {
|
||||
return "it names no version or no digest (a by-digest publish)"
|
||||
}
|
||||
all, err := rv.r.All(ctx)
|
||||
if err != nil {
|
||||
return "the reservations cannot be read: " + err.Error()
|
||||
}
|
||||
var out []string
|
||||
for _, v := range all {
|
||||
if v.Version != version {
|
||||
continue
|
||||
}
|
||||
out = append(out, fmt.Sprintf("%s holds %d built digests %v, published %v (a published heard before the build recorded its digests is kept until it does)", v.Module, len(v.Artifacts), v.Artifacts, v.Published))
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return "no reservation holds version " + version
|
||||
}
|
||||
return strings.Join(out, "; ")
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -201,3 +202,43 @@ func TestAReleasedVersionIsTaggedAtTheCommitItWasBuiltFrom(t *testing.T) {
|
||||
t.Fatalf("the newer build's version: tags %+v; want go/v0.1.1 at b0b", tags)
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 G A-415: a published heard before the build recorded its digests is kept on its reservation, and when the build
|
||||
// records them it marks its artifact and package-published is said then; a kept one whose digest is none of the built digests
|
||||
// is said in the log.
|
||||
func TestAPublishedHeardBeforeTheDigestsIsSaidWhenTheBuildRecordsThem(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
var logs []string
|
||||
logf := func(f string, a ...any) { logs = append(logs, fmt.Sprintf(f, a...)) }
|
||||
none := func(string, any) (json.RawMessage, error) {
|
||||
return json.Marshal(meshcatalog.VersionsAnswer{Module: "m"})
|
||||
}
|
||||
rv := newReservations(&memReserve{vals: map[string][]byte{}, revs: map[string]uint64{}}, none, func() jetstream.KeyValue { return &boundsKV{keys: map[string]bool{}} },
|
||||
&findings{module: builds.BuilderModule, w: &memFindings{vals: map[string][]byte{}}, logf: t.Logf, now: time.Now}, nil, logf)
|
||||
var said []meshbuilds.PackagePublished
|
||||
rv.say = func(_ context.Context, p meshbuilds.PackagePublished) error { said = append(said, p); return nil }
|
||||
rv.filled.Store(true)
|
||||
res, err := rv.Reserve(ctx, "m", "1.4", []string{"m:a", "m:b"}, versions.Source{Branch: "main", Revision: 41})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rv.published(ctx, versions.Outcome{Name: "m-a", Version: res.Version, Digest: "sha256:a"})
|
||||
rv.published(ctx, versions.Outcome{Name: "m-x", Version: res.Version, Digest: "sha256:x"})
|
||||
if len(said) != 0 {
|
||||
t.Fatalf("package-published said before the digests are recorded: %+v", said)
|
||||
}
|
||||
if err := rv.Built(ctx, "m", res.Version, map[string]string{"m:a": "sha256:a", "m:b": "sha256:b"}, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(said) != 1 || said[0].Artifact != "m:a" || said[0].Version != res.Version || said[0].Digest != "sha256:a" || said[0].Branch != "main" || said[0].Source != 41 {
|
||||
t.Fatalf("package-published said %+v, want one of m:a at %s for main at source 41", said, res.Version)
|
||||
}
|
||||
if log := strings.Join(logs, "\n"); !strings.Contains(log, "is kept on") || !strings.Contains(log, "m-x@"+res.Version+" kept on m "+res.Version+" marks no artifact") {
|
||||
t.Fatalf("log = %q, want the published kept and the one never matching said", logs)
|
||||
}
|
||||
// Taken in once: a published of m:a heard again says nothing more.
|
||||
rv.published(ctx, versions.Outcome{Name: "m-a", Version: res.Version, Digest: "sha256:a"})
|
||||
if len(said) != 1 {
|
||||
t.Fatalf("package-published said again: %+v", said)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -28,8 +28,9 @@ func reservedReader(ctx context.Context, store versions.Store) versions.Reader {
|
||||
}}
|
||||
}
|
||||
|
||||
// reservedVerb is the handler of the verb reserved.
|
||||
func reservedVerb(ctx context.Context, rr versions.Reader, carriers func(context.Context, string) ([]versions.Built, error), released versions.CatalogVersions) process.SeatHandler {
|
||||
// reservedVerb is the handler of the verb reserved; missed is told each name no entry carries, for the builder's log (design 47 G
|
||||
// A-418).
|
||||
func reservedVerb(ctx context.Context, rr versions.Reader, carriers func(context.Context, string) ([]versions.Built, error), released versions.CatalogVersions, missed func(name string)) process.SeatHandler {
|
||||
return func(_ string, body json.RawMessage) (any, error) {
|
||||
var in meshbuilds.ReservedInput
|
||||
if len(body) > 0 {
|
||||
@@ -38,7 +39,11 @@ func reservedVerb(ctx context.Context, rr versions.Reader, carriers func(context
|
||||
}
|
||||
}
|
||||
if name := strings.TrimSpace(in.Name); name != "" {
|
||||
return versions.AnswerReservedByName(ctx, rr, carriers, released, name, in.Version)
|
||||
a, err := versions.AnswerReservedByName(ctx, rr, carriers, released, name, in.Version)
|
||||
if err == nil && missed != nil && strings.HasPrefix(a.Error, versions.NoEntryCarries) {
|
||||
missed(name)
|
||||
}
|
||||
return a, err
|
||||
}
|
||||
return versions.AnswerReservedVerb(rr, in.Module, in.Version)
|
||||
}
|
||||
|
||||
@@ -27,7 +27,8 @@ func TestTheProcessAnswersReserved(t *testing.T) {
|
||||
return json.Marshal(meshcatalog.VersionsAnswer{Versions: []meshcatalog.VersionEntry{{Module: "mesh-sdk", Version: "0.1.2",
|
||||
Artifacts: []meshcatalog.VersionArtifact{{Artifact: "mesh-sdk:ts", Digest: "sha256:dd"}}}}})
|
||||
})
|
||||
handle := reservedVerb(context.Background(), none, carriers, released)
|
||||
var missed []string
|
||||
handle := reservedVerb(context.Background(), none, carriers, released, func(name string) { missed = append(missed, name) })
|
||||
got, err := handle("", json.RawMessage(`{"name":"@novox/mesh-sdk","version":"0.1.2"}`))
|
||||
a, ok := got.(meshbuilds.ReservedAnswer)
|
||||
if err != nil || !ok || !a.Released || a.Module != "mesh-sdk" || len(a.Artifacts) != 1 || a.Artifacts[0].Digest != "sha256:dd" {
|
||||
@@ -36,6 +37,10 @@ func TestTheProcessAnswersReserved(t *testing.T) {
|
||||
if len(asked) != 1 || asked[0] != "seat:mesh-catalog.versions" {
|
||||
t.Errorf("the released versions were asked of %v, want the seat mesh-catalog's versions", asked)
|
||||
}
|
||||
// Design 47 G A-418: a name no entry carries is answered error and told for the builder's log.
|
||||
if got, err := handle("", json.RawMessage(`{"name":"nobody/program","version":"0.1.6"}`)); err != nil || got.(meshbuilds.ReservedAnswer).Error == "" || len(missed) != 1 || missed[0] != "nobody/program" {
|
||||
t.Errorf("a name no entry carries: %+v (%v), told %v", got, err, missed)
|
||||
}
|
||||
if _, err := handle("", json.RawMessage(`{"module":"mesh-sdk"}`)); err == nil {
|
||||
t.Error("a module with no reservation was answered")
|
||||
}
|
||||
|
||||
@@ -27,6 +27,7 @@ func (g *retaking) heard(ctx context.Context, m jetstream.Msg, logf func(string,
|
||||
if nakErr := m.Nak(); nakErr != nil {
|
||||
logf("leaving %s for the bus to give again: %v", m.Subject(), nakErr)
|
||||
}
|
||||
givenUp(m, "its event", err, logf)
|
||||
return true
|
||||
}
|
||||
return false
|
||||
@@ -67,6 +68,7 @@ func (d *decisions) hear(logf func(string, ...any)) func(pattern string, handle
|
||||
if nakErr := m.Nak(); nakErr != nil {
|
||||
logf("leaving %s for the bus to give again: %v", m.Subject(), nakErr)
|
||||
}
|
||||
givenUp(m, "the operator's warrant", err, logf)
|
||||
return
|
||||
}
|
||||
if err := m.Ack(); err != nil {
|
||||
|
||||
@@ -4,6 +4,8 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
@@ -82,11 +84,19 @@ func (h *seatHearer) heard(ctx context.Context, m jetstream.Msg) bool {
|
||||
id = m.Headers().Get("x-event-id")
|
||||
}
|
||||
wake, err := h.n.Heard(ctx, se.seat, se.event, id, o)
|
||||
if errors.Is(err, versions.ErrNotNumbered) {
|
||||
// The version is no entry of the state versions, design 51's numbering by name, which G keeps only until it is removed
|
||||
// (§372); the reservation numbers a version (§369, §52) and takes the outcome in (A-415). Heard again it would stay
|
||||
// unnumbered there, so it is taken, not left for the bus.
|
||||
h.logf("%v in the state versions; the reservations take it (design 47 G §369, A-415)", err)
|
||||
err = nil
|
||||
}
|
||||
if err != nil {
|
||||
h.logf("%v; heard again", err)
|
||||
h.logf("%v; heard again (delivery %s)", err, deliveryOf(m))
|
||||
if nakErr := m.Nak(); nakErr != nil {
|
||||
h.logf("leaving %s for the bus to give again: %v", m.Subject(), nakErr)
|
||||
}
|
||||
givenUp(m, fmt.Sprintf("%s's %s of %s@%s", se.seat, se.event, o.Name, o.Version), err, h.logf)
|
||||
return true
|
||||
}
|
||||
if h.pub != nil {
|
||||
@@ -103,6 +113,28 @@ func (h *seatHearer) heard(ctx context.Context, m jetstream.Msg) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// deliveryOf is the message's delivery count as the bus says it, so a message the bus stops giving after its consumer's
|
||||
// bound is readable in the log; "unknown" when it says none.
|
||||
func deliveryOf(m jetstream.Msg) string {
|
||||
md, err := m.Metadata()
|
||||
if err != nil || md == nil {
|
||||
return "unknown"
|
||||
}
|
||||
return strconv.FormatUint(md.NumDelivered, 10)
|
||||
}
|
||||
|
||||
// EventDeliveries is how many times the bus gives a message of the builder's events consumer: the MaxDeliver the grants part
|
||||
// composes a module's durable with (mesh-grants buscompose).
|
||||
const EventDeliveries = 5
|
||||
|
||||
// givenUp says, on the last delivery the events consumer allows, that the builder gives the message up, naming its subject, what
|
||||
// it carries and the last refusal (design 47 G A-420): past it the bus drops the message without a word.
|
||||
func givenUp(m jetstream.Msg, what string, err error, logf func(string, ...any)) {
|
||||
if md, merr := m.Metadata(); merr == nil && md != nil && md.NumDelivered >= EventDeliveries {
|
||||
logf("gives up %s: %s, delivered %d times, the most its consumer allows; the last refusal: %v", m.Subject(), what, md.NumDelivered, err)
|
||||
}
|
||||
}
|
||||
|
||||
func (h *seatHearer) ack(m jetstream.Msg) {
|
||||
if err := m.Ack(); err != nil {
|
||||
h.logf("acknowledging %s: %v", m.Subject(), err)
|
||||
|
||||
@@ -157,7 +157,9 @@ func (h *toolchainsHeard) Reported(ctx context.Context, m jetstream.Msg) bool {
|
||||
h.logf("%s is no toolchain, and is dropped: %v", subject, err)
|
||||
return true
|
||||
}
|
||||
if h.set.Set(e.Node, tc) {
|
||||
changed := h.set.Set(e.Node, tc)
|
||||
h.logf("toolchain heard on %s: %s %s of %s (changed: %v)", subject, e.Language, e.Version, e.Node, changed)
|
||||
if changed {
|
||||
h.keep(e.Node, tc)
|
||||
h.agree(ctx)
|
||||
h.changed(ctx)
|
||||
|
||||
@@ -42,6 +42,8 @@ type handWatch struct {
|
||||
shut bool
|
||||
// ended is set when the look ended before the asks were all put.
|
||||
ended bool
|
||||
// why is what the look that wrote nothing said in the builder's log.
|
||||
why string
|
||||
}
|
||||
|
||||
// settle closes done once the claim is written and every lane has put its ask. Call with b.mu held.
|
||||
@@ -73,8 +75,9 @@ func (b *Builder) handAsk(key, language, id string) {
|
||||
}
|
||||
|
||||
// handEnded ends the watch of a request whose look ended: it wrote no claim, or the build ended.
|
||||
func (b *Builder) handEnded(key string) {
|
||||
func (b *Builder) handEnded(key, why string) {
|
||||
b.hand(key, func(w *handWatch) {
|
||||
w.why = why
|
||||
if !w.shut {
|
||||
w.shut, w.ended = true, true
|
||||
close(w.done)
|
||||
@@ -158,11 +161,14 @@ func (b *Builder) BuildAgain(ctx context.Context, module, fp, why, by string, bo
|
||||
}
|
||||
b.mu.Lock()
|
||||
res := BuildAgainResult{Revision: w.rev, Asks: maps.Clone(w.asks)}
|
||||
ended, shut, lanes := w.ended, w.shut, w.lanes
|
||||
ended, shut, lanes, said := w.ended, w.shut, w.lanes, w.why
|
||||
b.mu.Unlock()
|
||||
switch {
|
||||
case res.Revision == 0 && shut:
|
||||
res.Refused = fmt.Sprintf("the builder looked at %s and wrote nothing (a toolchain moved, the claim was lost or could not be written): its log says why", key)
|
||||
if said != "" {
|
||||
res.Refused = fmt.Sprintf("the builder looked at %s and wrote nothing: %s", key, said)
|
||||
}
|
||||
case res.Revision == 0:
|
||||
res.Pending = fmt.Sprintf("the builder has not claimed %s within %s: it waits behind other builds; builder_stuck shows where it stands", key, bound)
|
||||
case ended && len(res.Asks) < max(lanes, 0)+boolInt(lanes < 0):
|
||||
|
||||
@@ -16,7 +16,7 @@ func TestAnEntryOfTheCatalogHalfIsTheFormTheBuilderBuildsBy(t *testing.T) {
|
||||
Inputs: &meshcatalog.EntryInputs{Src: 7, Toolchains: map[string]meshcatalog.ToolchainUsed{"go": {Version: "go1", Recipe: "1"}},
|
||||
Paths: map[string][]string{"lib": {"a/b", "c"}}}}
|
||||
got := ModuleOf(e)
|
||||
if got.Fp2 != "src2:f" || got.Manifest != "sha256:m" || got.Path != "modules/netcheck" || got.Commit != "abc" || got.Src != 7 ||
|
||||
if got.Fp2 != "src2:f" || got.Manifest != "m" || got.Path != "modules/netcheck" || got.Commit != "abc" || got.Src != 7 ||
|
||||
got.Toolchains["go"].Version != "go1" || got.Packages["p"] != "1" || !reflect.DeepEqual(got.Needs, []string{"n"}) ||
|
||||
got.Inputs2 == nil || !reflect.DeepEqual(got.Inputs2.Bases, []string{"base"}) ||
|
||||
!reflect.DeepEqual(got.Inputs2.Paths, map[string]string{"a/b": "lib", "c": "lib"}) {
|
||||
|
||||
@@ -253,8 +253,8 @@ func TestTheFetchOfAModuleNamingItsPathsCarriesItsDirectoryAndThosePaths(t *test
|
||||
}
|
||||
r2 := newRig(t, goBundle)
|
||||
r2.consider(t, "modules/m")
|
||||
if a := r2.bm.callsOf("build_go_fetch")[0].body.(map[string]any); a["dir"] != nil || a["paths"] != nil {
|
||||
t.Errorf("fetch args = %+v, want no dir or paths for a module naming none", a)
|
||||
if a := r2.bm.callsOf("build_go_fetch")[0].body.(map[string]any); a["dir"] != "modules/m" || a["paths"] != nil {
|
||||
t.Errorf("fetch args = %+v, want the module's directory and no paths for a module naming none (A-447)", a)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -270,7 +270,7 @@ func (b *Builder) take(ctx context.Context, o Origin, key string, m meshcatalog.
|
||||
// built against, its packages and needs, and the named paths of its artifacts.
|
||||
func ModuleOf(e meshcatalog.CatalogEntry) meshcatalog.ModulesByModuleValue {
|
||||
m := meshcatalog.ModulesByModuleValue{Repository: e.Repository, Path: e.Path, Commit: e.Commit, Fp2: e.Fp, None2: e.None,
|
||||
Manifest: e.ManifestDigest, Packages: e.Packages, Needs: e.Needs}
|
||||
Manifest: meshcatalog.BareManifestDigest(e.ManifestDigest), Packages: e.Packages, Needs: e.Needs}
|
||||
if len(e.Bases) > 0 {
|
||||
m.Inputs2 = &meshcatalog.ModulesByModuleValueInputs2{Bases: e.Bases}
|
||||
}
|
||||
@@ -562,14 +562,20 @@ func (b *Builder) startOnce(ctx context.Context, key, module string, m meshcatal
|
||||
// building by this builder: it takes it again when it starts (ADR 0478 clause 5).
|
||||
func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesByModuleValue) {
|
||||
key, _ := builds.BuilderBuildKey(module, m.Fp2)
|
||||
defer b.handEnded(key)
|
||||
whyEnded := ""
|
||||
defer func() { b.handEnded(key, whyEnded) }()
|
||||
defer b.unrelease(key)
|
||||
// end logs why a look wrote nothing and keeps it for the answer to a build asked again (design 47 G A-412).
|
||||
end := func(format string, args ...any) {
|
||||
whyEnded = fmt.Sprintf(format, args...)
|
||||
b.logf("%s at %s: %s", module, m.Fp2, whyEnded)
|
||||
}
|
||||
plan, planErr, read := b.planOf(ctx, m.Manifest)
|
||||
var noModule *NoBuildModule
|
||||
if errors.As(planErr, &noModule) {
|
||||
// The seat list or the catalog half does not name the build module yet: nothing is claimed, and the look is made again
|
||||
// when they do (Reconsider, at each change of either; design 47 G A-82).
|
||||
b.logf("%s at %s is not built yet: %v", module, m.Fp2, planErr)
|
||||
end("is not built yet: %v", planErr)
|
||||
return
|
||||
}
|
||||
nothing := read && planErr == nil && len(plan.Tasks) == 0
|
||||
@@ -580,7 +586,7 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB
|
||||
}
|
||||
// The fingerprint names a toolchain the build nodes no longer agree on: a new fingerprint is coming, and a build
|
||||
// now would be of this source with another toolchain (ADR 0584 clause 1, which changes ADR 0477 clause 2). Nothing is written.
|
||||
b.logf("%s at %s is not built: %s", module, m.Fp2, why)
|
||||
end("is not built: %s", why)
|
||||
return
|
||||
}
|
||||
if b.Waits != nil {
|
||||
@@ -599,12 +605,14 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB
|
||||
if trunk && planErr == nil && !nothing {
|
||||
var goOn bool
|
||||
if numbering, goOn = b.number(ctx, module, m, plan); !goOn {
|
||||
end("is not built yet: it waits for its package's floor, nothing is claimed")
|
||||
return
|
||||
}
|
||||
}
|
||||
// Only a build of the registration's first watched branch reserves and releases a version (design 47 G §371).
|
||||
reserving := trunk && b.originOf(key).First && planErr == nil && !nothing && b.reserves(ctx, module, m, plan)
|
||||
if reserving && b.waitFill(module, m) {
|
||||
end("is not built yet: wait-held, the catalog has not said versions-filled")
|
||||
return
|
||||
}
|
||||
// A key built again by hand is taken over as it stands too: one left building by a look that ended without its write
|
||||
@@ -612,10 +620,11 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB
|
||||
cl, err := b.Claimer.Claim(ctx, module, m.Fp2, m.Manifest, ClaimOptions{TakeOver: first || byHand, ByHand: byHand, ByEvent: byEvent, ImageBuilder: imageSeat, Seats: b.Seats, Trunk: trunk, Published: b.isPublished,
|
||||
Stored: b.storedOnce(key)})
|
||||
if err != nil {
|
||||
b.logf("%s: %v", module, err)
|
||||
end("the claim was refused: %v", err)
|
||||
return
|
||||
}
|
||||
if cl == nil {
|
||||
end("nothing was claimed: the key is held by a build or a look already, is done or failed for a reason not retried on this fingerprint, waits for a package, or was made by another builder first")
|
||||
return
|
||||
}
|
||||
b.handClaimed(key, cl.Rev)
|
||||
@@ -1038,11 +1047,12 @@ func (n *askNotes) note(ctx context.Context, language, id, step string) error {
|
||||
// is copied by its pinned reference with no fetch (ADR 0498 clause 4); a lane of copies alone fetches nothing. A lane of a
|
||||
// key taken over at start waits for the ask the key names and puts none.
|
||||
func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m meshcatalog.ModulesByModuleValue, module string, asked *askNotes) {
|
||||
req := sdkstep.Fetch{Repository: repository, Commit: m.Commit}
|
||||
// The module's directory goes with every fetch, so a Go build downloads and judges that module and what its go.mod
|
||||
// replaces alone, never another Go module of the tree (design 47 G, A-447).
|
||||
req := sdkstep.Fetch{Repository: repository, Commit: m.Commit, Dir: m.Path}
|
||||
if m.Inputs2 != nil && len(m.Inputs2.Paths) > 0 {
|
||||
// A module whose every artifact names its paths is built from those alone (ADR 0499 clause 4): the paths come from
|
||||
// the indexer's kept inputs, never from a manifest (the lead's decision L8 of R4e).
|
||||
req.Dir = m.Path
|
||||
for p := range m.Inputs2.Paths {
|
||||
req.Paths = append(req.Paths, p)
|
||||
}
|
||||
|
||||
@@ -902,7 +902,7 @@ func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) {
|
||||
key: &builds.BuilderBuild{State: builds.BuildFailed, Error: "ace: no tun"}},
|
||||
{name: "build again: a module with no manifest is refused", again: true, refuse: "names no manifest", edit: func(m *meshcatalog.ModulesByModuleValue) { m.Manifest = "" },
|
||||
key: &builds.BuilderBuild{State: builds.BuildFailed, Error: "ace: no tun"}},
|
||||
{name: "build again: a toolchain that moved writes nothing and is said at once", again: true, refuse: "wrote nothing",
|
||||
{name: "build again: a toolchain that moved writes nothing and says why at once", again: true, refuse: "wrote nothing: is not built:",
|
||||
edit: func(m *meshcatalog.ModulesByModuleValue) {
|
||||
m.Toolchains = CatalogToolchains(map[string]builds.ToolchainUsed{"go": {Version: "go1.0", Recipe: "1"}})
|
||||
},
|
||||
|
||||
@@ -71,8 +71,8 @@ func (b *Builder) buildEntry(ctx context.Context, e meshbuilds.BuildEntry) (outc
|
||||
}
|
||||
return meshbuilds.StepEndedOutcomeFailed, failedWords
|
||||
}
|
||||
m := meshcatalog.ModulesByModuleValue{Repository: entry.Repository, Commit: entry.Commit, Fp2: entry.Fp, Manifest: entry.ManifestDigest, Path: entry.Path,
|
||||
Toolchains: CatalogToolchains(b.agreedFor(ctx, entry.ManifestDigest)), Packages: entry.Packages}
|
||||
m := meshcatalog.ModulesByModuleValue{Repository: entry.Repository, Commit: entry.Commit, Fp2: entry.Fp, Manifest: meshcatalog.BareManifestDigest(entry.ManifestDigest), Path: entry.Path,
|
||||
Toolchains: CatalogToolchains(b.agreedFor(ctx, meshcatalog.BareManifestDigest(entry.ManifestDigest))), Packages: entry.Packages}
|
||||
b.consider(ctx, entry.Module, m)
|
||||
select {
|
||||
case log := <-ended:
|
||||
|
||||
@@ -2,6 +2,7 @@ package versions
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
@@ -38,7 +39,9 @@ func NamesFrom(all Entries) PackageNames {
|
||||
}
|
||||
|
||||
// CarriersFrom are the module artifacts a package's name in its ecosystem is the name of, as the entries hold them (design 47 G
|
||||
// §572, §478), each distinct module and artifact once, sorted; a name no entry carries answers none.
|
||||
// §572, §478), each distinct module and artifact once, sorted; a name no entry carries answers none. A bundle's or an archive's
|
||||
// name is `<module>/<artifact>` (NameOf), read from its artifact reference `<module>:<artifact>`: the entry gives it no
|
||||
// `package` (design 47 G A-418).
|
||||
func CarriersFrom(all Entries) func(ctx context.Context, name string) ([]Built, error) {
|
||||
return func(_ context.Context, name string) ([]Built, error) {
|
||||
seen := map[Built]bool{}
|
||||
@@ -46,7 +49,7 @@ func CarriersFrom(all Entries) func(ctx context.Context, name string) ([]Built,
|
||||
for _, e := range all() {
|
||||
for _, a := range e.Artifacts {
|
||||
artifact, ok := artifactOf(e.Module, a.Ref)
|
||||
if !ok || a.Package == nil || a.Package.Name != name || seen[Built{e.Module, artifact}] {
|
||||
if !ok || !carries(e.Module, artifact, a, name) || seen[Built{e.Module, artifact}] {
|
||||
continue
|
||||
}
|
||||
seen[Built{e.Module, artifact}] = true
|
||||
@@ -58,6 +61,39 @@ func CarriersFrom(all Entries) func(ctx context.Context, name string) ([]Built,
|
||||
}
|
||||
}
|
||||
|
||||
// carries says whether artifact a of module is named name: its package's name, or a bundle's or an archive's NameOf (A-418).
|
||||
func carries(module, artifact string, a meshcatalog.Artifact, name string) bool {
|
||||
if a.Package != nil {
|
||||
return a.Package.Name == name
|
||||
}
|
||||
return (a.Kind == KindBundle || a.Kind == KindArchive) && NameOf(module, artifact) == name
|
||||
}
|
||||
|
||||
// LookedAt says the entries a name was looked for in, for the log of a name no entry carries (design 47 G A-418): how many,
|
||||
// and each module's artifacts with the name each is carried as.
|
||||
func LookedAt(all Entries) string {
|
||||
entries := all()
|
||||
var out []string
|
||||
for _, e := range entries {
|
||||
for _, a := range e.Artifacts {
|
||||
artifact, ok := artifactOf(e.Module, a.Ref)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
switch {
|
||||
case a.Package != nil:
|
||||
out = append(out, a.Ref+" as "+a.Package.Name)
|
||||
case a.Kind == KindBundle || a.Kind == KindArchive:
|
||||
out = append(out, a.Ref+" as "+NameOf(e.Module, artifact))
|
||||
default:
|
||||
out = append(out, a.Ref+" ("+a.Kind+", named nothing)")
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return fmt.Sprintf("%d entries: %s", len(entries), strings.Join(out, ", "))
|
||||
}
|
||||
|
||||
// ModuleFrom is the module and artifact a package's name is built from, from the entries (CarriersFrom); more than one entry
|
||||
// carrying it is an error naming them.
|
||||
func ModuleFrom(all Entries) func(ctx context.Context, name string) (module, artifact string, found bool, err error) {
|
||||
|
||||
@@ -3,6 +3,7 @@ package versions
|
||||
import (
|
||||
"context"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
@@ -19,7 +20,8 @@ func named(module, artifact, name string) meshcatalog.CatalogEntry {
|
||||
func TestPackageNamesAreReadFromTheCatalogHalfsEntries(t *testing.T) {
|
||||
entries := func() []meshcatalog.CatalogEntry {
|
||||
return []meshcatalog.CatalogEntry{named("mesh-sdk", "go", "git.novox.be/novox/mesh-sdk/go"), named("netcheck", "lib", "netcheck"),
|
||||
named("twin-a", "lib", "twin"), named("twin-b", "lib", "twin"), {Module: "bare"}}
|
||||
named("twin-a", "lib", "twin"), named("twin-b", "lib", "twin"), {Module: "bare"},
|
||||
{Module: "pipe", Artifacts: []meshcatalog.Artifact{{Ref: "pipe:program", Kind: KindBundle}, {Ref: "pipe:img", Kind: KindImage}}}}
|
||||
}
|
||||
ctx := context.Background()
|
||||
for _, c := range []struct {
|
||||
@@ -48,6 +50,17 @@ func TestPackageNamesAreReadFromTheCatalogHalfsEntries(t *testing.T) {
|
||||
if _, _, _, err := ModuleFrom(entries)(ctx, "twin"); err == nil {
|
||||
t.Error("ModuleFrom(twin): two entries carry it, want an error")
|
||||
}
|
||||
// Design 47 G A-418: a bundle is carried as <module>/<artifact>, from its reference, with no package in its entry; a kind
|
||||
// that is neither a bundle nor an archive is not named so.
|
||||
if got, _ := CarriersFrom(entries)(ctx, "pipe/program"); !reflect.DeepEqual(got, []Built{{"pipe", "program"}}) {
|
||||
t.Errorf("CarriersFrom(pipe/program) = %v, want the bundle pipe:program", got)
|
||||
}
|
||||
if got, _ := CarriersFrom(entries)(ctx, "pipe/img"); len(got) != 0 {
|
||||
t.Errorf("CarriersFrom(pipe/img) = %v, want none: an image with no package is named nothing", got)
|
||||
}
|
||||
if looked := LookedAt(entries); !strings.Contains(looked, "6 entries") || !strings.Contains(looked, "pipe:program as pipe/program") {
|
||||
t.Errorf("LookedAt = %q, want the six entries and the bundle's name", looked)
|
||||
}
|
||||
got, _ := CarriersFrom(entries)(ctx, "twin")
|
||||
if want := []Built{{"twin-a", "lib"}, {"twin-b", "lib"}}; !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("CarriersFrom(twin) = %v, want %v", got, want)
|
||||
|
||||
@@ -17,8 +17,8 @@ import (
|
||||
|
||||
// **A reserved version left unpublished is published again by the builder, on an event or by a person's tool** (design 47
|
||||
// G §588, ADR 0674): on the node-engine seat's `running` of a module whose catalog graph entry claims the registry seat
|
||||
// of the missing artifacts' kind, and on its `manifest-reads-changed` with `why` `returned` of the node such a holder
|
||||
// runs on. Each message is acted on once, by its stream sequence (state `acted`); each holder has one queue of versions,
|
||||
// of the missing artifacts' kind, and on its `manifest-reads-changed` with `why` `returned`, at each registry seat's holder
|
||||
// the seat-holders view names (A-416, A-416a: known from the start, not from the running heard). Each message is acted on once, by its stream sequence (state `acted`); each holder has one queue of versions,
|
||||
// kept in the state `publish-queue`, sent one at a time. A send is the registry seat's `publish` with the digest and no
|
||||
// parts (A-73): the holder takes the bytes from the artifact store by digest, and refuses what no store holds.
|
||||
|
||||
@@ -41,6 +41,13 @@ const (
|
||||
VerbPublish = "publish"
|
||||
)
|
||||
|
||||
// RepublishTransfer is a publish again's transfer name, the builder's own (the registry part's publish: the caller names its
|
||||
// transfer): safe as a transfer id, which holds none of "/", ".", " ", "*" or ">" (mesh-sdk's parts receiver refuses them), so a
|
||||
// version's dots are dashes (design 47 G §588).
|
||||
func RepublishTransfer(module, version, artifact string) string {
|
||||
return strings.NewReplacer("/", "-", ".", "-", " ", "-", "*", "-", ">", "-", "@", "-").Replace("republish-" + module + "-" + version + "-" + artifact)
|
||||
}
|
||||
|
||||
// RegistrySeats are the four registry seats a version is published to.
|
||||
var RegistrySeats = []string{SeatImages, SeatArtifacts, SeatGoPackages, SeatNPM}
|
||||
|
||||
@@ -112,9 +119,11 @@ type Republisher struct {
|
||||
Logf func(string, ...any)
|
||||
// Go starts the sending of a holder's queue; nil starts a goroutine.
|
||||
Go func(func())
|
||||
// Holders answers a registry seat's holders: the catalog's SeatHolders, the part's one holder function that its FollowSeat
|
||||
// of the assignments takes too, read from the builder's start (design 47 G A-416, A-416a); nil answers none.
|
||||
Holders func(seat string) []string
|
||||
|
||||
mu sync.Mutex
|
||||
runs map[string]map[string][]string // node -> holder module -> the registry seats it claims
|
||||
busy map[string]bool
|
||||
// lastQueued is the newest time a version was queued at: a clock that has not moved between two versions (a coarse clock,
|
||||
// or a fixed one) must not make the queue's order the order of its keys, so each is queued after the last.
|
||||
@@ -205,8 +214,8 @@ func (r *Republisher) registrySeatsOf(ctx context.Context, fingerprint string) (
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// running takes a running in: a module stopped is forgotten; one running whose entry claims a registry seat is a holder on
|
||||
// its node, and its versions are queued.
|
||||
// running takes a running in: one of a module whose entry claims a registry seat queues the versions waiting on it (§588). It
|
||||
// is not how a holder is known: Holders answers that (design 47 G A-416).
|
||||
func (r *Republisher) running(ctx context.Context, data []byte) error {
|
||||
var ev nodeengine.Running
|
||||
if err := json.Unmarshal(data, &ev); err != nil || ev.Module == "" || ev.Node == "" {
|
||||
@@ -214,28 +223,12 @@ func (r *Republisher) running(ctx context.Context, data []byte) error {
|
||||
return nil
|
||||
}
|
||||
if ev.Stopped || ev.Fingerprint == "" {
|
||||
r.mu.Lock()
|
||||
delete(r.runs[ev.Node], ev.Module)
|
||||
r.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
seats, err := r.registrySeatsOf(ctx, ev.Fingerprint)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the catalog entry %s of %s cannot be read: %w", ev.Fingerprint, ev.Module, err)
|
||||
}
|
||||
r.mu.Lock()
|
||||
if r.runs == nil {
|
||||
r.runs = map[string]map[string][]string{}
|
||||
}
|
||||
if r.runs[ev.Node] == nil {
|
||||
r.runs[ev.Node] = map[string][]string{}
|
||||
}
|
||||
if len(seats) == 0 {
|
||||
delete(r.runs[ev.Node], ev.Module)
|
||||
} else {
|
||||
r.runs[ev.Node][ev.Module] = seats
|
||||
}
|
||||
r.mu.Unlock()
|
||||
if len(seats) == 0 {
|
||||
return nil
|
||||
}
|
||||
@@ -243,7 +236,30 @@ func (r *Republisher) running(ctx context.Context, data []byte) error {
|
||||
return err
|
||||
}
|
||||
|
||||
// returned takes a manifest-reads-changed in: only one with why returned acts, for every holder known to run on its node.
|
||||
// heldSeats are the registry seats' holders as Holders answers them: holder module -> the registry seats it holds.
|
||||
func (r *Republisher) heldSeats() map[string][]string {
|
||||
held := map[string][]string{}
|
||||
if r.Holders == nil {
|
||||
return held
|
||||
}
|
||||
for _, seat := range RegistrySeats {
|
||||
for _, h := range r.Holders(seat) {
|
||||
held[h] = append(held[h], seat)
|
||||
}
|
||||
}
|
||||
return held
|
||||
}
|
||||
|
||||
// holdersFn answers a seat's holders from Holders.
|
||||
func (r *Republisher) holdersFn(seat string) []string {
|
||||
if r.Holders == nil {
|
||||
return nil
|
||||
}
|
||||
return r.Holders(seat)
|
||||
}
|
||||
|
||||
// returned takes a manifest-reads-changed in: only one with why returned acts, for every registry seat's holder Holders
|
||||
// answers (design 47 G A-416a); a version queued at a holder already is not queued again.
|
||||
func (r *Republisher) returned(ctx context.Context, data []byte) error {
|
||||
var ev nodeengine.ManifestReadsChanged
|
||||
if err := json.Unmarshal(data, &ev); err != nil || ev.Node == "" {
|
||||
@@ -253,12 +269,7 @@ func (r *Republisher) returned(ctx context.Context, data []byte) error {
|
||||
if ev.Why != nodeengine.ManifestReadsChangedWhyReturned {
|
||||
return nil
|
||||
}
|
||||
r.mu.Lock()
|
||||
holders := map[string][]string{}
|
||||
for m, seats := range r.runs[ev.Node] {
|
||||
holders[m] = seats
|
||||
}
|
||||
r.mu.Unlock()
|
||||
holders := r.heldSeats()
|
||||
if len(holders) == 0 {
|
||||
return nil
|
||||
}
|
||||
@@ -311,6 +322,7 @@ func (r *Republisher) queue(ctx context.Context, vs []ReservedVersion, holdersOf
|
||||
break
|
||||
}
|
||||
if v.Fingerprint == "" || v.BuiltAt.IsZero() {
|
||||
r.logf("publish again: %s %s is not queued: it holds no fingerprint or its build has not ended with every digest", v.Module, v.Version)
|
||||
continue
|
||||
}
|
||||
e, err := r.Entry(ctx, v.Fingerprint)
|
||||
@@ -320,20 +332,28 @@ func (r *Republisher) queue(ctx context.Context, vs []ReservedVersion, holdersOf
|
||||
byHolder := map[string]map[string]string{}
|
||||
for _, ref := range v.Missing() {
|
||||
if _, no := v.Unstored[ref]; no {
|
||||
r.logf("publish again: %s %s: %s is not queued: a holder refused it as not stored", v.Module, v.Version, ref)
|
||||
continue
|
||||
}
|
||||
found := false
|
||||
for _, a := range e.Artifacts {
|
||||
seat, ok := SeatOfArtifact(a)
|
||||
if a.Ref != ref || !ok {
|
||||
continue
|
||||
}
|
||||
found = true
|
||||
if hs := holdersOf(seat); len(hs) > 0 {
|
||||
if byHolder[hs[0]] == nil {
|
||||
byHolder[hs[0]] = map[string]string{}
|
||||
}
|
||||
byHolder[hs[0]][ref] = v.Artifacts[ref]
|
||||
} else {
|
||||
r.logf("publish again: %s %s: %s is not queued: the registry seat %s has no holder in the seat-holders view", v.Module, v.Version, ref, seat)
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
r.logf("publish again: %s %s: %s is not queued: the catalog entry %s names no registry seat for it", v.Module, v.Version, ref, v.Fingerprint)
|
||||
}
|
||||
}
|
||||
did := false
|
||||
for holder, arts := range byHolder {
|
||||
@@ -361,6 +381,7 @@ func (r *Republisher) enqueue(ctx context.Context, v ReservedVersion, holder str
|
||||
if _, _, found, err := r.Queue.Get(ctx, key); err != nil {
|
||||
return false, err
|
||||
} else if found {
|
||||
r.logf("publish again: %s %s is queued at %s already", v.Module, v.Version, holder)
|
||||
return false, nil
|
||||
}
|
||||
raw, _ := json.Marshal(QueuedPublish{V: 1, Holder: holder, Module: v.Module, Version: v.Version, Artifacts: arts, Queued: r.queuedAt()})
|
||||
@@ -380,13 +401,14 @@ func (r *Republisher) enqueue(ctx context.Context, v ReservedVersion, holder str
|
||||
}
|
||||
return false, fmt.Errorf("%s cannot be queued: %w", key, err)
|
||||
}
|
||||
r.logf("publish again: %s %s is queued to publish at %s: %d artifacts", v.Module, v.Version, holder, len(arts))
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// Republish is builder_publish_again: with module and version, that reserved version's missing artifacts; with module
|
||||
// alone, each reserved version of it whose artifacts are not all published; with neither, every version the finding
|
||||
// version-unreleased names. It queues as the events do (no limit but the cap), for the holders known to run, and answers
|
||||
// the versions it queued.
|
||||
// version-unreleased names. It queues as the events do (no limit but the cap), at the holders Holders answers (design 47 G
|
||||
// A-416), and answers the versions it queued.
|
||||
func (r *Republisher) Republish(ctx context.Context, module, version string) ([]string, error) {
|
||||
var vs []ReservedVersion
|
||||
if module == "" {
|
||||
@@ -413,15 +435,7 @@ func (r *Republisher) Republish(ctx context.Context, module, version string) ([]
|
||||
return nil, fmt.Errorf("%s holds no reserved version%s with an artifact unpublished", module, map[bool]string{true: " " + version, false: ""}[version != ""])
|
||||
}
|
||||
}
|
||||
r.mu.Lock()
|
||||
known := map[string][]string{}
|
||||
for _, mods := range r.runs {
|
||||
for m, seats := range mods {
|
||||
known[m] = seats
|
||||
}
|
||||
}
|
||||
r.mu.Unlock()
|
||||
return r.queue(ctx, vs, func(seat string) []string { return holdersOf(known, seat) }, 0)
|
||||
return r.queue(ctx, vs, r.holdersFn, 0)
|
||||
}
|
||||
|
||||
// Queued are the queued versions, per holder in the order they are sent, and the acted keys with their sequences:
|
||||
@@ -605,9 +619,13 @@ func (r *Republisher) send(ctx context.Context, q QueuedPublish) {
|
||||
kind := art.Kind
|
||||
bound, cancel := context.WithTimeout(ctx, UnreleasedBound) // a context's bound, no timer (ADR 0437 §8)
|
||||
err := r.Send(bound, Publish{Seat: seat, Kind: kind, Name: name, Version: v.Version, Digest: q.Artifacts[ref], Fingerprint: v.Fingerprint,
|
||||
Commit: v.Commit, Transfer: "republish-" + q.Module + "-" + v.Version + "-" + strings.TrimPrefix(ref, q.Module+":")})
|
||||
Commit: v.Commit, Transfer: RepublishTransfer(q.Module, v.Version, strings.TrimPrefix(ref, q.Module+":"))})
|
||||
cancel()
|
||||
if err == nil {
|
||||
r.logf("publish again: %s %s: %s sent to %s as %s %s, digest %s; the package-published is said when the seat says published", q.Module, q.Version, ref, seat, name, v.Version, q.Artifacts[ref])
|
||||
}
|
||||
if err != nil {
|
||||
r.logf("publish again: %s %s: %s refused by %s: %v", q.Module, q.Version, ref, seat, err)
|
||||
words := err.Error()
|
||||
if strings.Contains(words, strings.TrimSuffix(NotStoredWords, ": ")) {
|
||||
if ok, merr := r.Reserved.MarkUnstored(ctx, q.Module, q.Version, ref, words); merr != nil {
|
||||
|
||||
@@ -31,6 +31,8 @@ type republishBed struct {
|
||||
sent []Publish
|
||||
said []string
|
||||
refuse string // the words Send fails with, empty for none
|
||||
// holders are the registry seats' holders the seat-holders view names (design 47 G A-416).
|
||||
holders map[string][]string
|
||||
}
|
||||
|
||||
func newRepublishBed(t *testing.T, max int) *republishBed {
|
||||
@@ -58,6 +60,7 @@ func newRepublishBed(t *testing.T, max int) *republishBed {
|
||||
b.said = append(b.said, v.Module+" "+v.Version+": "+why)
|
||||
},
|
||||
Logf: t.Logf}
|
||||
b.Holders = func(seat string) []string { return b.holders[seat] }
|
||||
return b
|
||||
}
|
||||
|
||||
@@ -102,7 +105,8 @@ func TestRepublish(t *testing.T) {
|
||||
}
|
||||
}
|
||||
p := b.sent[0]
|
||||
if p.Seat != SeatArtifacts || p.Name != "m/prog" || p.Kind != "bundle" || p.Version != v.Version || p.Digest != "sha256:aa" || p.Fingerprint != "src2:m" {
|
||||
if p.Seat != SeatArtifacts || p.Name != "m/prog" || p.Kind != "bundle" || p.Version != v.Version || p.Digest != "sha256:aa" || p.Fingerprint != "src2:m" ||
|
||||
p.Transfer != "republish-m-"+strings.ReplaceAll(v.Version, ".", "-")+"-prog" || strings.ContainsAny(p.Transfer, "/. *>") {
|
||||
t.Errorf("sent %+v, want the bundle by digest to the oci seat, the Go package left to its own seat's holder", p)
|
||||
}
|
||||
if _, err := b.Heard(ctx, subject, data, 5); err != nil || len(b.sent) != 1 {
|
||||
@@ -115,14 +119,10 @@ func TestRepublish(t *testing.T) {
|
||||
t.Errorf("a key stays in the queue after its outcome: %v", keys)
|
||||
}
|
||||
})
|
||||
t.Run("returned acts, started does not", func(t *testing.T) {
|
||||
t.Run("returned acts at the seat's holder, started does not", func(t *testing.T) {
|
||||
b := newRepublishBed(t, 50)
|
||||
b.reserve(t, "m")
|
||||
subject, data := running("n1", "oci", "src2:holder", true) // stopped: not a holder yet
|
||||
_, _ = b.Heard(ctx, subject, data, 1)
|
||||
subject, data = running("n1", "oci", "src2:holder", false)
|
||||
_, _ = b.Heard(ctx, subject, data, 2)
|
||||
b.sent = nil
|
||||
b.holders = map[string][]string{SeatArtifacts: {"oci"}}
|
||||
for i, why := range []string{nodeengine.ManifestReadsChangedWhyStarted, "", nodeengine.ManifestReadsChangedWhyReturned} {
|
||||
ev, _ := json.Marshal(nodeengine.ManifestReadsChanged{Node: "n1", Why: why})
|
||||
handled, err := b.Heard(ctx, nodeengine.SubjectManifestReadsChanged("n1"), ev, uint64(10+i))
|
||||
@@ -194,23 +194,25 @@ func TestRepublish(t *testing.T) {
|
||||
t.Errorf("resume sent %d (%v), want the begun version sent anew", len(b.sent), err)
|
||||
}
|
||||
})
|
||||
t.Run("the tool queues for a holder known to run", func(t *testing.T) {
|
||||
t.Run("after a fresh start the tool queues at the seat's holder, no running heard", func(t *testing.T) {
|
||||
// Design 47 G A-416, A-416a: the holder is the one the seat-holders view names from the start, not one learnt from a
|
||||
// running heard since; a seat with no holder queues nothing.
|
||||
b := newRepublishBed(t, 1)
|
||||
b.reserve(t, "m1")
|
||||
b.reserve(t, "m2")
|
||||
if q, err := b.Republish(ctx, "", ""); err != nil || len(q) != 0 {
|
||||
t.Errorf("no holder known: queued %v (%v), want none", q, err)
|
||||
t.Errorf("no holder named: queued %v (%v), want none", q, err)
|
||||
}
|
||||
subject, data := running("n1", "oci", "src2:holder", false)
|
||||
_, _ = b.Heard(ctx, subject, data, 1) // Max 1: only m2 sent
|
||||
b.sent = nil
|
||||
b.holders = map[string][]string{SeatArtifacts: {"oci"}}
|
||||
q, err := b.Republish(ctx, "", "")
|
||||
if err != nil || fmt.Sprint(q) != "[m2 0.1.0 m1 0.1.0]" || len(b.sent) != 2 {
|
||||
t.Errorf("the tool with neither queued %v (%v), sent %d: want both versions, with no limit", q, err, len(b.sent))
|
||||
if err != nil || fmt.Sprint(q) != "[m2 0.1.0 m1 0.1.0]" || len(b.sent) != 2 || b.sent[0].Seat != SeatArtifacts {
|
||||
t.Errorf("the tool with neither queued %v (%v), sent %+v: want both versions at oci, with no limit", q, err, b.sent)
|
||||
}
|
||||
if _, err := b.Republish(ctx, "m1", "0.1.9"); err == nil {
|
||||
t.Error("a version no reservation holds is not refused")
|
||||
}
|
||||
subject, data := running("n1", "oci", "src2:holder", false)
|
||||
_, _ = b.Heard(ctx, subject, data, 1)
|
||||
answer, err := AnswerQueue(Reader{Keys: func() ([]string, error) { return nil, nil }}, Reader{Keys: func() ([]string, error) { return b.acted.KeysOf(ctx, AllFilter) },
|
||||
Get: func(k string) (json.RawMessage, bool, error) { v, _, ok, err := b.acted.Get(ctx, k); return v, ok, err }})
|
||||
if err != nil || answer.Acted[ActedKey(subject)] != 1 {
|
||||
|
||||
@@ -66,6 +66,16 @@ type ReservedVersion struct {
|
||||
// Dependencies are, by artifact reference, what the step's result says each artifact resolved (kind 1) and was built with
|
||||
// (kind 3), kept for `package-published` (design 47 G §466, §673, A-57).
|
||||
Dependencies map[string][]meshbuilds.Dependency `json:"dependencies,omitempty"`
|
||||
// Kept are the registry seats' published heard while the build had not yet recorded every digest and whose digest marked
|
||||
// no artifact then: kept as heard, they mark their artifact when the build records its digests (design 47 G A-415).
|
||||
Kept []KeptPublished `json:"kept-published,omitempty"`
|
||||
}
|
||||
|
||||
// KeptPublished is a registry seat's published kept on a reservation, as heard (design 47 G A-415).
|
||||
type KeptPublished struct {
|
||||
Name string `json:"name"`
|
||||
Version string `json:"version"`
|
||||
Digest string `json:"digest"`
|
||||
}
|
||||
|
||||
// Source is what a build reserving a version is made from (design 47 G §370): the repository and commit the git tag is
|
||||
@@ -345,30 +355,38 @@ func (m Marked) PackagePublished(digest string, at time.Time) meshbuilds.Package
|
||||
|
||||
// Published takes a registry seat's published in: the artifacts of a reservation at version (a leading v is the Go form)
|
||||
// whose built digest it names are marked published. It answers each artifact it newly marked, with the reservation as marked.
|
||||
func (r *Reserver) Published(ctx context.Context, version, digest string) ([]Marked, error) {
|
||||
// A published that marks no artifact of a reservation at version whose build has not recorded every digest is kept on that
|
||||
// reservation as heard (name, version, digest), and the keys it is kept on are answered too (design 47 G A-415): Kept marks
|
||||
// it when the build records its digests.
|
||||
func (r *Reserver) Published(ctx context.Context, name, version, digest string) ([]Marked, []string, error) {
|
||||
version = strings.TrimPrefix(version, "v")
|
||||
if version == "" || digest == "" {
|
||||
return nil, nil
|
||||
return nil, nil, nil
|
||||
}
|
||||
keys, err := r.Store.KeysOf(ctx, AllFilter)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the reservations cannot be listed: %w", err)
|
||||
return nil, nil, fmt.Errorf("the reservations cannot be listed: %w", err)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
var marked []Marked
|
||||
var kept []string
|
||||
for _, k := range keys {
|
||||
again:
|
||||
cur, rev, found, err := r.get(ctx, k)
|
||||
if err != nil {
|
||||
return marked, err
|
||||
return marked, kept, err
|
||||
}
|
||||
if !found || cur.Version != version {
|
||||
continue
|
||||
}
|
||||
changed := false
|
||||
changed, matched, keep := false, false, false
|
||||
var arts []string
|
||||
for a, d := range cur.Artifacts {
|
||||
if d == digest && !cur.Published[a] {
|
||||
if d != digest {
|
||||
continue
|
||||
}
|
||||
matched = true
|
||||
if !cur.Published[a] {
|
||||
if cur.Published == nil {
|
||||
cur.Published = map[string]bool{}
|
||||
}
|
||||
@@ -377,6 +395,10 @@ func (r *Reserver) Published(ctx context.Context, version, digest string) ([]Mar
|
||||
}
|
||||
}
|
||||
sort.Strings(arts)
|
||||
if !matched && cur.BuiltAt.IsZero() && !cur.keeps(name, digest) {
|
||||
cur.Kept = append(cur.Kept, KeptPublished{Name: name, Version: version, Digest: digest})
|
||||
changed, keep = true, true
|
||||
}
|
||||
if !changed {
|
||||
continue
|
||||
}
|
||||
@@ -384,13 +406,76 @@ func (r *Reserver) Published(ctx context.Context, version, digest string) ([]Mar
|
||||
if _, err := r.Store.Update(ctx, k, raw, rev); errors.Is(err, ErrConflict) {
|
||||
goto again
|
||||
} else if err != nil {
|
||||
return marked, fmt.Errorf("%s cannot be written: %w", k, err)
|
||||
return marked, kept, fmt.Errorf("%s cannot be written: %w", k, err)
|
||||
}
|
||||
for _, a := range arts {
|
||||
marked = append(marked, Marked{Reserved: cur, Artifact: a})
|
||||
}
|
||||
if keep {
|
||||
kept = append(kept, k)
|
||||
}
|
||||
}
|
||||
return marked, kept, nil
|
||||
}
|
||||
|
||||
func (v ReservedVersion) keeps(name, digest string) bool {
|
||||
for _, h := range v.Kept {
|
||||
if h.Name == name && h.Digest == digest {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Kept takes the published kept on module's reservation of version in once its build recorded every digest (design 47 G
|
||||
// A-415): each kept published whose digest is an artifact's marks that artifact, as one heard after would, and every kept one
|
||||
// leaves the reservation. It answers the artifacts newly marked and the kept published whose digest matched no artifact.
|
||||
func (r *Reserver) Kept(ctx context.Context, module, version string) ([]Marked, []KeptPublished, error) {
|
||||
series, _, err := SplitVersion(version)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
key := ReservedKey(module, series)
|
||||
for {
|
||||
cur, rev, found, err := r.get(ctx, key)
|
||||
if err != nil || !found || cur.Version != version || cur.BuiltAt.IsZero() || len(cur.Kept) == 0 {
|
||||
return nil, nil, err
|
||||
}
|
||||
var arts []string
|
||||
var unmatched []KeptPublished
|
||||
for _, h := range cur.Kept {
|
||||
matched := false
|
||||
for a, d := range cur.Artifacts {
|
||||
if d != h.Digest {
|
||||
continue
|
||||
}
|
||||
matched = true
|
||||
if !cur.Published[a] {
|
||||
if cur.Published == nil {
|
||||
cur.Published = map[string]bool{}
|
||||
}
|
||||
cur.Published[a] = true
|
||||
arts = append(arts, a)
|
||||
}
|
||||
}
|
||||
if !matched {
|
||||
unmatched = append(unmatched, h)
|
||||
}
|
||||
}
|
||||
cur.Kept = nil
|
||||
sort.Strings(arts)
|
||||
raw, _ := json.Marshal(cur)
|
||||
if _, err := r.Store.Update(ctx, key, raw, rev); errors.Is(err, ErrConflict) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
return nil, nil, fmt.Errorf("%s cannot be written: %w", key, err)
|
||||
}
|
||||
var marked []Marked
|
||||
for _, a := range arts {
|
||||
marked = append(marked, Marked{Reserved: cur, Artifact: a})
|
||||
}
|
||||
return marked, unmatched, nil
|
||||
}
|
||||
return marked, nil
|
||||
}
|
||||
|
||||
// Of is module's reservation of series, and whether there is one; with an empty series the module's highest.
|
||||
@@ -496,7 +581,7 @@ func AnswerReservedByName(ctx context.Context, r Reader, carriers func(context.C
|
||||
}
|
||||
switch {
|
||||
case len(got) == 0:
|
||||
return meshbuilds.ReservedAnswer{Error: "no entry carries the name " + name}, nil
|
||||
return meshbuilds.ReservedAnswer{Error: NoEntryCarries + name}, nil
|
||||
case len(got) > 1:
|
||||
return meshbuilds.ReservedAnswer{Error: MoreThanOne(name, got)}, nil
|
||||
}
|
||||
@@ -527,6 +612,9 @@ func AnswerReservedByName(ctx context.Context, r Reader, carriers func(context.C
|
||||
return meshbuilds.ReservedAnswer{Error: fmt.Sprintf("%s (%s) has no reserved version", name, module)}, nil
|
||||
}
|
||||
|
||||
// NoEntryCarries opens the answer of reserved given a name no entry carries.
|
||||
const NoEntryCarries = "no entry carries the name "
|
||||
|
||||
// The descriptions of the reservation's tools.
|
||||
const (
|
||||
ReservedDescription = "The versions the builder reserved (its state reserved, design 47 G section 369): one per module and MAJOR.MINOR, " +
|
||||
|
||||
@@ -143,7 +143,7 @@ func TestReservationIsReadAndMarkedPublished(t *testing.T) {
|
||||
if got, err := AnswerReserved(rd, "other"); err != nil || len(got["reserved"].([]ReservedVersion)) != 0 {
|
||||
t.Errorf("builder_reserved of another module = %v (%v), want none", got, err)
|
||||
}
|
||||
marked, err := r.Published(ctx, "v1.4.3", "sha256:aa")
|
||||
marked, _, err := r.Published(ctx, "m-a", "v1.4.3", "sha256:aa")
|
||||
if err != nil || len(marked) != 1 {
|
||||
t.Fatalf("a published of 1.4.3 with a built digest marked %v (%v)", marked, err)
|
||||
}
|
||||
@@ -155,7 +155,7 @@ func TestReservationIsReadAndMarkedPublished(t *testing.T) {
|
||||
if wantSaid := (meshbuilds.PackagePublished{V: 1, Artifact: "m:a", At: &atUTC, Branch: "main", Source: 41, Module: "m", Version: "1.4.3", Digest: "sha256:aa", Dependencies: deps}); !reflect.DeepEqual(said, wantSaid) {
|
||||
t.Errorf("package-published = %+v, want %+v", said, wantSaid)
|
||||
}
|
||||
if marked, _ := r.Published(ctx, "1.4.3", "sha256:other"); len(marked) != 0 {
|
||||
if marked, _, _ := r.Published(ctx, "m-a", "1.4.3", "sha256:other"); len(marked) != 0 {
|
||||
t.Errorf("a published with another digest marked %v", marked)
|
||||
}
|
||||
cur, _, _ := r.At(ctx, ReservedKey("m", "1.4"))
|
||||
|
||||
@@ -104,6 +104,10 @@ func (v Versions) Number(ctx context.Context, c Claim) (Decision, Entry, error)
|
||||
return Decision{}, Entry{}, fmt.Errorf("%s: the versions kept changing under 16 claims", c.Name)
|
||||
}
|
||||
|
||||
// ErrNotNumbered is a version the state versions does not hold (design 51's numbering by name): a version the reservation
|
||||
// numbered (§369) is never held there.
|
||||
var ErrNotNumbered = errors.New("not numbered by the builder")
|
||||
|
||||
// Mark moves one version to a new state by compare-and-set, as the holders' events say (published, refused, removed):
|
||||
// change edits the entry read; a version the state does not hold is an error naming it.
|
||||
func (v Versions) Mark(ctx context.Context, name, version string, change func(*Entry)) (Entry, error) {
|
||||
@@ -114,7 +118,7 @@ func (v Versions) Mark(ctx context.Context, name, version string, change func(*E
|
||||
return Entry{}, fmt.Errorf("%s: %w", key, err)
|
||||
}
|
||||
if !found {
|
||||
return Entry{}, fmt.Errorf("%s@%s is not numbered by the builder", name, version)
|
||||
return Entry{}, fmt.Errorf("%s@%s is %w", name, version, ErrNotNumbered)
|
||||
}
|
||||
e, err := Decode(raw)
|
||||
if err != nil {
|
||||
|
||||
@@ -227,7 +227,7 @@ func (e EngineCompositions) inSandbox(ctx context.Context, c *Case, head, node s
|
||||
return a, err
|
||||
}
|
||||
}
|
||||
job := buildstep.ComposeJob{Fetch: buildstep.Fetch{Repository: nodeEngineRepo, Commit: head, Packages: packagesFor(e.Published, c, nodeEngineRepo)}, Dir: meshcatalogclient.RootPath,
|
||||
job := buildstep.ComposeJob{Fetch: buildstep.Fetch{Repository: nodeEngineRepo, Commit: head, Dir: meshcatalogclient.RootPath, Packages: packagesFor(e.Published, c, nodeEngineRepo)}, Dir: meshcatalogclient.RootPath,
|
||||
Package: composer.Package, Request: raw, Reader: "the node-engine head of " + c.Group}
|
||||
// The ask travels on the bus, whose message limit a whole request passes (1.2 MB for a large node): it goes packed.
|
||||
if err := job.PackRequest(raw); err != nil {
|
||||
|
||||
@@ -219,6 +219,9 @@ func TestTheCompositionIsMadeFromTheEnginesAnswers(t *testing.T) {
|
||||
t.Fatalf("asked %+v, jobs %+v: want the input asked and one job", e.asked, sb.jobs)
|
||||
}
|
||||
job := sb.jobs[0]
|
||||
if job.Fetch.Dir == "" || job.Fetch.Dir != job.Dir { // a build is for one module: its fetch names the directory (A-447b)
|
||||
t.Errorf("the compose job's fetch names the directory %q, want its module's %q", job.Fetch.Dir, job.Dir)
|
||||
}
|
||||
var req composer.Request
|
||||
if len(job.Request) == 0 || job.Request[0] != '"' {
|
||||
t.Errorf("the request went whole, not packed (issue 1966): %.60s", job.Request)
|
||||
|
||||
@@ -160,7 +160,7 @@ func (st *state) runJudge(ctx context.Context, req composer.Request, touched []s
|
||||
st.fail(st.goModule(), err.Error())
|
||||
return change, trunk, false
|
||||
}
|
||||
job := buildstep.ComposeJob{Fetch: buildstep.Fetch{Repository: DeliveryRepository, Commit: commit, Packages: versions}, Dir: JudgeDir,
|
||||
job := buildstep.ComposeJob{Fetch: buildstep.Fetch{Repository: DeliveryRepository, Commit: commit, Dir: JudgeDir, Packages: versions}, Dir: JudgeDir,
|
||||
Package: JudgePackage, Request: raw, Reader: "the indexer's judge at " + at}
|
||||
a := buildstep.StepAsk{ID: buildstep.NewID(), Kind: buildstep.AskCheck, Language: buildstep.AskGo, Asker: st.j.Module,
|
||||
At: st.j.now().UTC(), Compose: &job}
|
||||
|
||||
@@ -12,10 +12,16 @@ import (
|
||||
)
|
||||
|
||||
// Ready is nil once every watched branch has its first view (`none-yet` counting as one), else says what it waits for
|
||||
// (design 47 G §614, A-103): a part acts on nothing until then.
|
||||
// (design 47 G §614, A-103): a part acts on nothing until then. On a fresh mesh, where the registrations and the gate's source are both
|
||||
// answered and name no branch, it is nil too: the trunk is `none-yet` (A-368).
|
||||
func (b *Branches) Ready() error {
|
||||
followed := b.Followed()
|
||||
if len(followed) == 0 {
|
||||
// A fresh mesh: the registrations and the gate's source are both answered and name no branch, so the trunk is `none-yet`
|
||||
// and the view is empty (design 47 G, A-368).
|
||||
if b.namesNoBranch() {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("the catalog half names no watched branch yet (the registrations of the seat %s are asked until they answer)", seat.Seat)
|
||||
}
|
||||
var waiting []string
|
||||
@@ -34,6 +40,59 @@ func (b *Branches) Ready() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// namesNoBranch says the first view of the registrations is in and, while the gate's source is followed, so is its, and
|
||||
// neither names a branch (none is followed).
|
||||
func (b *Branches) namesNoBranch() bool {
|
||||
select {
|
||||
case <-b.Registrations.Ready():
|
||||
default:
|
||||
return false
|
||||
}
|
||||
// A first view closes before the follower's change is taken into the branches followed, so what the views hold is read too.
|
||||
for _, reg := range b.Registrations.All() {
|
||||
if len(reg.Branches) > 0 {
|
||||
return false
|
||||
}
|
||||
}
|
||||
b.mu.Lock()
|
||||
s := b.source
|
||||
b.mu.Unlock()
|
||||
if s == nil {
|
||||
return true
|
||||
}
|
||||
select {
|
||||
case <-s.Follower.Ready():
|
||||
default:
|
||||
return false
|
||||
}
|
||||
return len(s.Branches()) == 0
|
||||
}
|
||||
|
||||
// FirstViews is closed once the first views of the registrations and, while it is followed, of the gate's source are in: a part
|
||||
// that waits for the catalog half to settle looks at Followed and Ready once it is closed (design 47 G, A-372).
|
||||
func (b *Branches) FirstViews(ctx context.Context) <-chan struct{} {
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
select {
|
||||
case <-b.Registrations.Ready():
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
b.mu.Lock()
|
||||
s := b.source
|
||||
b.mu.Unlock()
|
||||
if s != nil {
|
||||
select {
|
||||
case <-s.Follower.Ready():
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
close(done)
|
||||
}()
|
||||
return done
|
||||
}
|
||||
|
||||
// Entry is a module's entry at the trunk: the module's own registration decides, so the branch followed first, which lists
|
||||
// the module, is the one read (design 47 G A-103). An error says a watched branch has no first view yet, never "no module".
|
||||
func (b *Branches) Entry(module string) (seat.CatalogEntry, bool, error) {
|
||||
|
||||
@@ -94,3 +94,81 @@ func TestTheBranchFollowerGivesEveryPartItsEntryHelpers(t *testing.T) {
|
||||
t.Error("a released entry kept its finding")
|
||||
}
|
||||
}
|
||||
|
||||
// A-368 (design 47 G, C6): on a fresh mesh, where neither the registrations nor the gate's source name a branch, the catalog half
|
||||
// counts the trunk as `none-yet`: Ready is nil once both are answered, and the view is empty (Entry finds no module); until
|
||||
// both answer, no part is ready.
|
||||
func TestAMeshThatNamesNoBranchIsReadyWithAnEmptyView(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
start []string // the followers started, by the name they are adopted under
|
||||
ready bool
|
||||
}{
|
||||
{"nothing answered", nil, false},
|
||||
{"only the registrations answered", []string{"mesh-catalog.registrations"}, false},
|
||||
{"only the gate's source answered", []string{"mesh-delivery.source"}, false},
|
||||
{"both answered", []string{"mesh-catalog.registrations", "mesh-delivery.source"}, true},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
seats := &fakeSeats{regs: map[string][]string{}}
|
||||
w := &fakeWiring{adopted: map[string]*views.Follower{}, hears: map[string]func(context.Context, []byte) error{}}
|
||||
b, err := NewBranches(w, seats.call)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, name := range c.start {
|
||||
f, _ := w.get(name)
|
||||
if err := f.Start(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := b.Ready(); (err == nil) != c.ready {
|
||||
t.Fatalf("Ready = %v, want ready %v", err, c.ready)
|
||||
}
|
||||
if c.ready {
|
||||
if e, found, err := b.Entry("m"); err != nil || found {
|
||||
t.Errorf("Entry = %v, %v, %v: want an empty view, no error", e, found, err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A-368: a first view closes before the follower's change reaches the branches followed, so a registration or a gate's source that
|
||||
// names a branch is no mesh that names none, though Followed holds nothing yet.
|
||||
func TestAFirstViewThatNamesABranchIsNotReadyBeforeItIsFollowed(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
source []string
|
||||
regs map[string][]string
|
||||
}{
|
||||
{"a registration names main", nil, map[string][]string{"nats": {"main"}}},
|
||||
{"the gate's source names main", []string{"main"}, map[string][]string{}},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
seats := &fakeSeats{source: c.source, regs: c.regs, version: 1}
|
||||
w := &fakeWiring{adopted: map[string]*views.Follower{}, hears: map[string]func(context.Context, []byte) error{}}
|
||||
b, err := NewBranches(w, seats.call)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, name := range []string{"mesh-catalog.registrations", "mesh-delivery.source"} {
|
||||
f, _ := w.get(name)
|
||||
f.Changed = nil // the change is not yet taken into the branches followed
|
||||
if err := f.Start(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if len(b.Followed()) != 0 {
|
||||
t.Fatalf("followed %v: the test means no branch is followed yet", b.Followed())
|
||||
}
|
||||
if err := b.Ready(); err == nil {
|
||||
t.Error("Ready said ready with a branch named and not yet followed")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1154,6 +1154,8 @@ type TakeInTaken struct {
|
||||
OwnBuildUnknown string `json:"own-build-unknown,omitempty"`
|
||||
// why the manifest of the indexer the revision names was refused
|
||||
Refused string `json:"refused,omitempty"`
|
||||
// the catalog graph revision of the branch the take-in made, which the pipelines of its entries start at (design 47 G section 476, A-448)
|
||||
Revision int64 `json:"revision,omitempty"`
|
||||
// when the wait began
|
||||
Since time.Time `json:"since"`
|
||||
// the source revision taken in (design 47 G section 476)
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
package meshcatalog
|
||||
|
||||
import "strings"
|
||||
|
||||
// manifestDigestPrefix is how a catalog entry's field manifest-digest begins, as every digest a record names (design 47 G §578).
|
||||
const manifestDigestPrefix = "sha256:"
|
||||
|
||||
// EntryManifestDigest is the entry's field manifest-digest for the module's digest bare, the 64 hex digits the module's
|
||||
// value and the key of `manifests` hold: `sha256:` and the digits (design 47 G §578). A digest already prefixed stays, and
|
||||
// none stays none.
|
||||
func EntryManifestDigest(bare string) string {
|
||||
if bare == "" || strings.HasPrefix(bare, manifestDigestPrefix) {
|
||||
return bare
|
||||
}
|
||||
return manifestDigestPrefix + bare
|
||||
}
|
||||
|
||||
// BareManifestDigest is the 64 hex digits of an entry's manifest-digest: the module's value and the key of `manifests`
|
||||
// (design 47 G §578). An entry taken in before G carries the digits alone, which is what its module value held (§609), and
|
||||
// stays as it is.
|
||||
func BareManifestDigest(field string) string { return strings.TrimPrefix(field, manifestDigestPrefix) }
|
||||
@@ -27,7 +27,6 @@ const (
|
||||
VerbAccept = "accept"
|
||||
VerbCancel = "cancel"
|
||||
VerbAskAgain = "ask-again"
|
||||
VerbRelease = "release"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -46,7 +45,7 @@ const (
|
||||
|
||||
// Verbs is every verb the seat serves, in its definition's order: a new copy each call.
|
||||
func Verbs() []string {
|
||||
return []string{VerbRun, VerbStart, VerbShow, VerbList, VerbStands, VerbWhyCarried, VerbKept, VerbReview, VerbRerun, VerbAccept, VerbCancel, VerbAskAgain, VerbRelease}
|
||||
return []string{VerbRun, VerbStart, VerbShow, VerbList, VerbStands, VerbWhyCarried, VerbKept, VerbReview, VerbRerun, VerbAccept, VerbCancel, VerbAskAgain}
|
||||
}
|
||||
|
||||
// Events is every event published on the seat's subject: a new copy each call.
|
||||
@@ -360,14 +359,6 @@ const (
|
||||
GroupEndedOutcomeCancelled = "cancelled"
|
||||
)
|
||||
|
||||
// Held is a pipeline held for the new version of the runner: its steps are placed and asked when the node-engine seat says that fingerprint runs healthy (design 47 G 65, 266, 534, A-195).
|
||||
type Held struct {
|
||||
// the fingerprint of the runner's own module the pipeline awaits running healthy (design 47 G 266, 276, A-195)
|
||||
Fingerprint string `json:"fingerprint"`
|
||||
// since when the pipeline is held, RFC 3339
|
||||
Since time.Time `json:"since"`
|
||||
}
|
||||
|
||||
// KeptResult is a package's passing result the runner keeps, by the digest of what its tests read (ADR 0666 section 8).
|
||||
type KeptResult struct {
|
||||
// sha256:<hex> of what the package's tests read
|
||||
@@ -452,7 +443,6 @@ type Pipeline struct {
|
||||
Fingerprint string `json:"fingerprint,omitempty"`
|
||||
// the gathered group, when the pipeline is its
|
||||
Group string `json:"group,omitempty"`
|
||||
Held *Held `json:"held,omitempty"`
|
||||
Module string `json:"module"`
|
||||
Outcome string `json:"outcome"`
|
||||
// yes when the module is over its test budget (design 47 G section 419)
|
||||
@@ -720,18 +710,16 @@ type Wait struct {
|
||||
Artifact string `json:"artifact,omitempty"`
|
||||
// the watched branch, or the branch of a run on request
|
||||
Branch string `json:"branch"`
|
||||
// the time the bound counts from (design 47 G section 266, section 269, section 498)
|
||||
ClockFrom *time.Time `json:"clock-from,omitempty"`
|
||||
// the commit the module's entry was last changed at
|
||||
Commit string `json:"commit,omitempty"`
|
||||
// the module's build-input fingerprint at the commit
|
||||
Fingerprint string `json:"fingerprint,omitempty"`
|
||||
// the gathered group, when the pipeline is its
|
||||
Group string `json:"group,omitempty"`
|
||||
// for its end, how it ended: answered, ran, released, superseded
|
||||
// for its end, how it ended: answered, ran, superseded
|
||||
How string `json:"how,omitempty"`
|
||||
Module string `json:"module"`
|
||||
// what the step waits on (design 47 *Health and going back*); the runner version is what a pipeline held for the runner's new version waits on (design 47 G 534, A-206)
|
||||
// what the step waits on (design 47 *Health and going back*)
|
||||
On string `json:"on"`
|
||||
// the pipeline's key, <branch>.<revision>.<module>, a branch escaped as a key's name is
|
||||
Pipeline string `json:"pipeline"`
|
||||
@@ -742,7 +730,7 @@ type Wait struct {
|
||||
Since time.Time `json:"since"`
|
||||
// the source revision of the watched branch; the pipeline is keyed by it (ADR 0666 section 9)
|
||||
Source int64 `json:"source,omitempty"`
|
||||
// a pipeline step a wait inside a pipeline is in; pipeline is the acting step of design 47 G 203, for a wait of the entries the runner holds for its new version (design 47 G 534)
|
||||
// a pipeline step a wait inside a pipeline is in
|
||||
Step string `json:"step"`
|
||||
V int64 `json:"v"`
|
||||
// for a wait on a person, who: the operator, a reviewer
|
||||
@@ -751,10 +739,9 @@ type Wait struct {
|
||||
|
||||
// The values of Wait's on.
|
||||
const (
|
||||
WaitOnPerson = "person"
|
||||
WaitOnBuildNode = "build node"
|
||||
WaitOnRevisionView = "revision view"
|
||||
WaitOnRunnerVersion = "runner version"
|
||||
WaitOnPerson = "person"
|
||||
WaitOnBuildNode = "build node"
|
||||
WaitOnRevisionView = "revision view"
|
||||
)
|
||||
|
||||
// The values of Wait's step.
|
||||
@@ -766,7 +753,6 @@ const (
|
||||
WaitStepComposition = "composition"
|
||||
WaitStepReview = "review"
|
||||
WaitStepApproval = "approval"
|
||||
WaitStepPipeline = "pipeline"
|
||||
)
|
||||
|
||||
// Widening is a part serving a step the change alters, or an acting rule, a grant or a credential it widens, computed by the running runner (ADR 0666 section 23).
|
||||
@@ -991,18 +977,6 @@ type AskAgainAnswer struct {
|
||||
Result StepResult `json:"result"`
|
||||
}
|
||||
|
||||
// ReleaseInput is what release takes.
|
||||
type ReleaseInput struct {
|
||||
// why, written with the caller the runtime proves in the hand-act log
|
||||
Why string `json:"why"`
|
||||
}
|
||||
|
||||
// ReleaseAnswer is what release answers.
|
||||
type ReleaseAnswer struct {
|
||||
// how many held pipelines were started
|
||||
Released int64 `json:"released"`
|
||||
}
|
||||
|
||||
// PipelineStartedEvent is the body of the event pipeline-started.
|
||||
type PipelineStartedEvent = PipelineStarted
|
||||
|
||||
|
||||
@@ -66,7 +66,8 @@
|
||||
"required": [
|
||||
"modules"
|
||||
]
|
||||
}
|
||||
},
|
||||
"class": "read"
|
||||
},
|
||||
{
|
||||
"name": "module",
|
||||
@@ -127,7 +128,8 @@
|
||||
"required": [
|
||||
"module"
|
||||
]
|
||||
}
|
||||
},
|
||||
"class": "read"
|
||||
},
|
||||
{
|
||||
"name": "provides",
|
||||
@@ -147,7 +149,8 @@
|
||||
"answer": {
|
||||
"type": "object",
|
||||
"description": "what provides answers, as its description says"
|
||||
}
|
||||
},
|
||||
"class": "read"
|
||||
},
|
||||
{
|
||||
"name": "dependents",
|
||||
@@ -167,7 +170,8 @@
|
||||
"answer": {
|
||||
"type": "object",
|
||||
"description": "what dependents answers, as its description says"
|
||||
}
|
||||
},
|
||||
"class": "read"
|
||||
},
|
||||
{
|
||||
"name": "seat-definitions",
|
||||
@@ -246,7 +250,8 @@
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"class": "read"
|
||||
},
|
||||
{
|
||||
"name": "revision",
|
||||
@@ -2954,6 +2959,10 @@
|
||||
},
|
||||
"description": "the merges it holds, oldest first"
|
||||
},
|
||||
"revision": {
|
||||
"type": "integer",
|
||||
"description": "the catalog graph revision of the branch the take-in made, which the pipelines of its entries start at (design 47 G section 476, A-448)"
|
||||
},
|
||||
"group": {
|
||||
"type": "string",
|
||||
"description": "the delivery group, for a group being gathered"
|
||||
|
||||
@@ -6,12 +6,6 @@
|
||||
"domain": "Delivery",
|
||||
"decision": "novox/hq ADR 0666",
|
||||
"description": "The pipeline runner: it runs each module's pipeline on each of its watched branches, asking each step of the seat that serves it and hearing its end, and says each pipeline's and each step's start and end (design 47 *Pipelines per module*). Held once for the mesh; a foundation part.",
|
||||
"acts": [
|
||||
{
|
||||
"step": "pipeline",
|
||||
"bound": "30m"
|
||||
}
|
||||
],
|
||||
"verbs": [
|
||||
{
|
||||
"name": "run",
|
||||
@@ -501,35 +495,6 @@
|
||||
]
|
||||
},
|
||||
"class": "act"
|
||||
},
|
||||
{
|
||||
"name": "release",
|
||||
"description": "The pipelines the runner holds for its new version started by hand with the running runner, whatever the new version's state; written in the hand-act log (design 47 G 276, A-206).",
|
||||
"input": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"why": {
|
||||
"type": "string",
|
||||
"description": "why, written with the caller the runtime proves in the hand-act log"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"why"
|
||||
]
|
||||
},
|
||||
"answer": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"released": {
|
||||
"type": "integer",
|
||||
"description": "how many held pipelines were started"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"released"
|
||||
]
|
||||
},
|
||||
"class": "act"
|
||||
}
|
||||
],
|
||||
"events": [
|
||||
@@ -719,25 +684,6 @@
|
||||
},
|
||||
"description": "how many tests a step ran and carried, per level (ADR 0666 section 8)"
|
||||
},
|
||||
"Held": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"since": {
|
||||
"type": "string",
|
||||
"format": "date-time",
|
||||
"description": "since when the pipeline is held, RFC 3339"
|
||||
},
|
||||
"fingerprint": {
|
||||
"type": "string",
|
||||
"description": "the fingerprint of the runner's own module the pipeline awaits running healthy (design 47 G 266, 276, A-195)"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"since",
|
||||
"fingerprint"
|
||||
],
|
||||
"description": "a pipeline held for the new version of the runner: its steps are placed and asked when the node-engine seat says that fingerprint runs healthy (design 47 G 65, 266, 534, A-195)"
|
||||
},
|
||||
"ToolchainUsed": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
@@ -1074,9 +1020,6 @@
|
||||
"type": "boolean",
|
||||
"description": "yes when the module is over its test budget (design 47 G section 419)"
|
||||
},
|
||||
"held": {
|
||||
"$ref": "#/defs/Held"
|
||||
},
|
||||
"widens": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
@@ -1524,10 +1467,9 @@
|
||||
"integration",
|
||||
"composition",
|
||||
"review",
|
||||
"approval",
|
||||
"pipeline"
|
||||
"approval"
|
||||
],
|
||||
"description": "a pipeline step a wait inside a pipeline is in; pipeline is the acting step of design 47 G 203, for a wait of the entries the runner holds for its new version (design 47 G 534)"
|
||||
"description": "a pipeline step a wait inside a pipeline is in"
|
||||
},
|
||||
"artifact": {
|
||||
"type": "string"
|
||||
@@ -1537,10 +1479,9 @@
|
||||
"enum": [
|
||||
"person",
|
||||
"build node",
|
||||
"revision view",
|
||||
"runner version"
|
||||
"revision view"
|
||||
],
|
||||
"description": "what the step waits on (design 47 *Health and going back*); the runner version is what a pipeline held for the runner's new version waits on (design 47 G 534, A-206)"
|
||||
"description": "what the step waits on (design 47 *Health and going back*)"
|
||||
},
|
||||
"who": {
|
||||
"type": "string",
|
||||
@@ -1552,12 +1493,7 @@
|
||||
},
|
||||
"how": {
|
||||
"type": "string",
|
||||
"description": "for its end, how it ended: answered, ran, released, superseded"
|
||||
},
|
||||
"clock-from": {
|
||||
"type": "string",
|
||||
"format": "date-time",
|
||||
"description": "the time the bound counts from (design 47 G section 266, section 269, section 498)"
|
||||
"description": "for its end, how it ended: answered, ran, superseded"
|
||||
}
|
||||
},
|
||||
"description": "a step's wait, held and ended; a wait on a person is said as waiting on that person, never a stall (ADR 0283)",
|
||||
|
||||
@@ -608,6 +608,8 @@ export interface TakeInTaken {
|
||||
"own-build-unknown"?: string;
|
||||
/** why the manifest of the indexer the revision names was refused */
|
||||
"refused"?: string;
|
||||
/** the catalog graph revision of the branch the take-in made, which the pipelines of its entries start at (design 47 G section 476, A-448) */
|
||||
"revision"?: number;
|
||||
/** when the wait began */
|
||||
"since": string;
|
||||
/** the source revision taken in (design 47 G section 476) */
|
||||
|
||||
@@ -17,7 +17,6 @@ export declare const verbs: {
|
||||
readonly accept: "accept";
|
||||
readonly cancel: "cancel";
|
||||
readonly askAgain: "ask-again";
|
||||
readonly release: "release";
|
||||
};
|
||||
|
||||
export declare const events: {
|
||||
@@ -154,14 +153,6 @@ export interface GroupEnded {
|
||||
"why"?: string;
|
||||
}
|
||||
|
||||
/** a pipeline held for the new version of the runner: its steps are placed and asked when the node-engine seat says that fingerprint runs healthy (design 47 G 65, 266, 534, A-195) */
|
||||
export interface Held {
|
||||
/** the fingerprint of the runner's own module the pipeline awaits running healthy (design 47 G 266, 276, A-195) */
|
||||
"fingerprint": string;
|
||||
/** since when the pipeline is held, RFC 3339 */
|
||||
"since": string;
|
||||
}
|
||||
|
||||
/** a package's passing result the runner keeps, by the digest of what its tests read (ADR 0666 section 8) */
|
||||
export interface KeptResult {
|
||||
/** sha256:<hex> of what the package's tests read */
|
||||
@@ -211,7 +202,6 @@ export interface Pipeline {
|
||||
"fingerprint"?: string;
|
||||
/** the gathered group, when the pipeline is its */
|
||||
"group"?: string;
|
||||
"held"?: Held;
|
||||
"module": string;
|
||||
"outcome": "running" | "passed" | "failed" | "refused" | "superseded" | "cancelled";
|
||||
/** yes when the module is over its test budget (design 47 G section 419) */
|
||||
@@ -403,19 +393,17 @@ export interface Wait {
|
||||
"artifact"?: string;
|
||||
/** the watched branch, or the branch of a run on request */
|
||||
"branch": string;
|
||||
/** the time the bound counts from (design 47 G section 266, section 269, section 498) */
|
||||
"clock-from"?: string;
|
||||
/** the commit the module's entry was last changed at */
|
||||
"commit"?: string;
|
||||
/** the module's build-input fingerprint at the commit */
|
||||
"fingerprint"?: string;
|
||||
/** the gathered group, when the pipeline is its */
|
||||
"group"?: string;
|
||||
/** for its end, how it ended: answered, ran, released, superseded */
|
||||
/** for its end, how it ended: answered, ran, superseded */
|
||||
"how"?: string;
|
||||
"module": string;
|
||||
/** what the step waits on (design 47 *Health and going back*); the runner version is what a pipeline held for the runner's new version waits on (design 47 G 534, A-206) */
|
||||
"on": "person" | "build node" | "revision view" | "runner version";
|
||||
/** what the step waits on (design 47 *Health and going back*) */
|
||||
"on": "person" | "build node" | "revision view";
|
||||
/** the pipeline's key, <branch>.<revision>.<module>, a branch escaped as a key's name is */
|
||||
"pipeline": string;
|
||||
/** the catalog graph revision of the branch the pipeline runs for */
|
||||
@@ -425,8 +413,8 @@ export interface Wait {
|
||||
"since": string;
|
||||
/** the source revision of the watched branch; the pipeline is keyed by it (ADR 0666 section 9) */
|
||||
"source"?: number;
|
||||
/** a pipeline step a wait inside a pipeline is in; pipeline is the acting step of design 47 G 203, for a wait of the entries the runner holds for its new version (design 47 G 534) */
|
||||
"step": "build" | "lint" | "unit" | "integration" | "composition" | "review" | "approval" | "pipeline";
|
||||
/** a pipeline step a wait inside a pipeline is in */
|
||||
"step": "build" | "lint" | "unit" | "integration" | "composition" | "review" | "approval";
|
||||
"v": number;
|
||||
/** for a wait on a person, who: the operator, a reviewer */
|
||||
"who"?: string;
|
||||
@@ -606,18 +594,6 @@ export interface AskAgainAnswer {
|
||||
"result": StepResult;
|
||||
}
|
||||
|
||||
/** is what release takes. */
|
||||
export interface ReleaseInput {
|
||||
/** why, written with the caller the runtime proves in the hand-act log */
|
||||
"why": string;
|
||||
}
|
||||
|
||||
/** is what release answers. */
|
||||
export interface ReleaseAnswer {
|
||||
/** how many held pipelines were started */
|
||||
"released": number;
|
||||
}
|
||||
|
||||
/** is the body of the event pipeline-started. */
|
||||
export type PipelineStartedEvent = PipelineStarted;
|
||||
|
||||
|
||||
@@ -17,7 +17,6 @@ export const verbs = {
|
||||
accept: "accept",
|
||||
cancel: "cancel",
|
||||
askAgain: "ask-again",
|
||||
release: "release",
|
||||
};
|
||||
|
||||
export const events = {
|
||||
|
||||
@@ -123,7 +123,7 @@ func Tools(ask asker, now func() time.Time, groupsRead func() ([]groups.Row, []s
|
||||
// number, branch and head through the seat git's verb pull (by number when given, else by the merge commit; a merge made
|
||||
// on the forge is read the same way), the rest from the forge through trees and files (bridge.MergedMessage), and, for a
|
||||
// member of a delivery group the seat mesh-delivery names (of), the group and its members as the process sets them.
|
||||
func MergedOf(of []seat.Group, repository, merge string, number int, trees bridge.Trees, files bridge.Files, forge bridge.Caller) (json.RawMessage, error) {
|
||||
func MergedOf(groupOf func(id string) ([]seat.Group, error), repository, merge string, number int, trees bridge.Trees, files bridge.Files, forge bridge.Caller) (json.RawMessage, error) {
|
||||
owner, repo, ok := strings.Cut(repository, "/")
|
||||
if !ok || owner == "" || repo == "" {
|
||||
return nil, fmt.Errorf("%q is not owner/name", repository)
|
||||
@@ -140,6 +140,10 @@ func MergedOf(of []seat.Group, repository, merge string, number int, trees bridg
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
of, err := groupOf(f.Branch) // the one group the pull request's branch names, never the whole list
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if g, ok := groups.Named(of, f.Branch, f.Repository, f.Number); ok {
|
||||
f.Group, f.Members = g.ID, groups.MembersOf(g)
|
||||
}
|
||||
@@ -251,7 +255,7 @@ func main() {
|
||||
performed, asks := stdio.State(groups.PerformedState), stdio.State(groups.EndAsksState)
|
||||
tools := Tools(askEnd(asks, served, 25*time.Second), time.Now,
|
||||
func() ([]groups.Row, []string, error) {
|
||||
of, err := groups.Read(context.Background(), askCaller)
|
||||
of, err := groups.ReadAll(context.Background(), askCaller)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
@@ -259,11 +263,7 @@ func main() {
|
||||
},
|
||||
func(repository, merge string, number int) (json.RawMessage, error) {
|
||||
ctx := context.Background()
|
||||
of, err := groups.Read(ctx, askCaller)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return MergedOf(of, repository, merge, number, bridge.TreesVia(ctx, askCaller), bridge.FilesVia(ctx, askCaller), askCaller)
|
||||
return MergedOf(func(id string) ([]seat.Group, error) { return groups.ReadGroup(ctx, askCaller, id) }, repository, merge, number, bridge.TreesVia(ctx, askCaller), bridge.FilesVia(ctx, askCaller), askCaller)
|
||||
}, askCaller)
|
||||
owedAsks := stdio.State(groups.OwedAsksState)
|
||||
recorder := meshhandactsclient.Recorder{Ask: stdio.Ask}
|
||||
|
||||
+2
-2
@@ -199,7 +199,7 @@ func TestTheBridgeToolsReadTheGroupsAndAskTheEndAgain(t *testing.T) {
|
||||
}, fail: "the forge is down"},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
raw, err := MergedOf(c.of, c.repository, c.merge, c.number, trees, files, c.api)
|
||||
raw, err := MergedOf(func(string) ([]seat.Group, error) { return c.of, nil }, c.repository, c.merge, c.number, trees, files, c.api)
|
||||
if c.fail != "" {
|
||||
if err == nil || !strings.Contains(err.Error(), c.fail) {
|
||||
t.Fatalf("err %v, want %q", err, c.fail)
|
||||
@@ -212,7 +212,7 @@ func TestTheBridgeToolsReadTheGroupsAndAskTheEndAgain(t *testing.T) {
|
||||
}
|
||||
})
|
||||
}
|
||||
if _, err := MergedOf(nil, "novox/a", c2, 9, func(string, string, string) (string, map[string]string, error) {
|
||||
if _, err := MergedOf(func(string) ([]seat.Group, error) { return nil, nil }, "novox/a", c2, 9, func(string, string, string) (string, map[string]string, error) {
|
||||
return "", nil, errors.New("the forge does not answer")
|
||||
}, files, forgeAnswers(t, 9, "", "merged", "fix/x", h2, c2, "main")); err == nil || !strings.Contains(err.Error(), "does not answer") {
|
||||
t.Errorf("a forge that does not answer: %v", err)
|
||||
|
||||
@@ -230,11 +230,21 @@ func Rows(of []seat.Group, published, ended map[string]string, asks map[string][
|
||||
|
||||
func sortRows(rows []Row) { sort.Slice(rows, func(i, j int) bool { return rows[i].Key < rows[j].Key }) }
|
||||
|
||||
// MembersOf are a group's members with their heads, in the group's order.
|
||||
// MembersOf are a group's members with their heads, in the group's order, each pull request once: a group the seat
|
||||
// answers with a member twice (feat/47-G-join11 named novox/mesh-sdk#189 twice, 2026-11-14) keeps the later head at the
|
||||
// first place, since a merged message naming a member twice is refused whole and no merge of that branch is taken in.
|
||||
func MembersOf(g seat.Group) []meshdeliveryclient.Head {
|
||||
out := make([]meshdeliveryclient.Head, 0, len(g.Members))
|
||||
at := map[string]int{}
|
||||
for _, m := range g.Members {
|
||||
out = append(out, meshdeliveryclient.Head{Repository: m.Repository, Number: int(m.Number), Head: m.Head})
|
||||
h := meshdeliveryclient.Head{Repository: m.Repository, Number: int(m.Number), Head: m.Head}
|
||||
k := fmt.Sprintf("%s#%d", m.Repository, m.Number)
|
||||
if i, twice := at[k]; twice {
|
||||
out[i] = h
|
||||
continue
|
||||
}
|
||||
at[k] = len(out)
|
||||
out = append(out, h)
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -255,12 +265,28 @@ func Named(of []seat.Group, id, repository string, number int) (seat.Group, bool
|
||||
return seat.Group{}, false
|
||||
}
|
||||
|
||||
// Read is every delivery group the seat mesh-delivery's verb groups answers, the ended ones too (all), called through call
|
||||
// by the verb's manifest key: the bridge reads a group from the seat, never from the gate's state (design 47 *A group's
|
||||
// merges are taken in together*). An answer saying an error, or one the contract cannot type, is an error: a failed read
|
||||
// is never read as no group.
|
||||
// Read is the open delivery groups the seat mesh-delivery's verb groups answers by default, called through call by the
|
||||
// verb's manifest key: the bridge reads a group from the seat, never from the gate's state (design 47 *A group's merges are
|
||||
// taken in together*). The ended ones are never asked whole: their list outgrows one bus message (lead: principle 7, live
|
||||
// 02:06); one is read by its id with ReadGroup. An answer saying an error, or one the contract cannot type, is an error: a
|
||||
// failed read is never read as no group.
|
||||
func Read(ctx context.Context, call func(ctx context.Context, key string, body any) (json.RawMessage, error)) ([]seat.Group, error) {
|
||||
raw, err := call(ctx, "seat:"+seat.Seat+"."+seat.VerbGroups, seat.GroupsInput{All: seat.GroupsInputAllTrue})
|
||||
return ask(ctx, call, seat.GroupsInput{})
|
||||
}
|
||||
|
||||
// ReadAll is every delivery group, the ended ones too (all). Its answer grows with every group that ended, so only a
|
||||
// tool a person asks reads it, never the take-in or the catch-up.
|
||||
func ReadAll(ctx context.Context, call func(ctx context.Context, key string, body any) (json.RawMessage, error)) ([]seat.Group, error) {
|
||||
return ask(ctx, call, seat.GroupsInput{All: seat.GroupsInputAllTrue})
|
||||
}
|
||||
|
||||
// ReadGroup is the one group id names, ended or not, as the verb answers with its group: none for a group it does not know.
|
||||
func ReadGroup(ctx context.Context, call func(ctx context.Context, key string, body any) (json.RawMessage, error), id string) ([]seat.Group, error) {
|
||||
return ask(ctx, call, seat.GroupsInput{Group: id})
|
||||
}
|
||||
|
||||
func ask(ctx context.Context, call func(ctx context.Context, key string, body any) (json.RawMessage, error), in seat.GroupsInput) ([]seat.Group, error) {
|
||||
raw, err := call(ctx, "seat:"+seat.Seat+"."+seat.VerbGroups, in)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the seat %s did not answer %s: %w", seat.Seat, seat.VerbGroups, err)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
package groups
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
|
||||
)
|
||||
|
||||
// The seat answers the whole list of groups with more than one bus message carries once many have ended (live 02:06);
|
||||
// a read of the open ones, or of one by its id, is answered; Read never asks for all (lead: principle 7).
|
||||
func TestReadsNeverAskTheEndedGroupsWhole(t *testing.T) {
|
||||
const bus = 1 << 20
|
||||
seatOf := func(_ context.Context, _ string, body any) (json.RawMessage, error) {
|
||||
in := body.(seat.GroupsInput)
|
||||
groups := []seat.Group{{ID: "feat/open", State: "gathering"}}
|
||||
switch {
|
||||
case in.Group != "":
|
||||
groups = []seat.Group{{ID: in.Group}}
|
||||
case in.All == seat.GroupsInputAllTrue:
|
||||
for i := 0; i < 3000; i++ {
|
||||
groups = append(groups, seat.Group{ID: "ended", Why: strings.Repeat("x", 500)})
|
||||
}
|
||||
}
|
||||
raw, _ := json.Marshal(seat.GroupsAnswer{Groups: groups})
|
||||
if len(raw) > bus {
|
||||
return nil, errors.New("the answer is more than the bus carries in one message")
|
||||
}
|
||||
return raw, nil
|
||||
}
|
||||
if got, err := Read(context.Background(), seatOf); err != nil || len(got) != 1 || got[0].ID != "feat/open" {
|
||||
t.Errorf("Read: %v %v; want the open group alone", got, err)
|
||||
}
|
||||
if got, err := ReadGroup(context.Background(), seatOf, "feat/ended"); err != nil || len(got) != 1 || got[0].ID != "feat/ended" {
|
||||
t.Errorf("ReadGroup: %v %v; want the group by its id", got, err)
|
||||
}
|
||||
if _, err := ReadAll(context.Background(), seatOf); err == nil {
|
||||
t.Errorf("ReadAll: want the seat's refusal to be an error, never read as no group")
|
||||
}
|
||||
// A member the seat names twice is said once, at its later head (2026-11-14: the take-in of every repository with a
|
||||
// branch of that group was refused whole).
|
||||
twice := seat.Group{ID: "feat/twice", Members: []seat.Member{{Repository: "o/a", Number: 1, Head: "old"}, {Repository: "o/b", Number: 2, Head: "b"}, {Repository: "o/a", Number: 1, Head: "new"}}}
|
||||
if got := MembersOf(twice); len(got) != 2 || got[0].Repository != "o/a" || got[0].Head != "new" || got[1].Repository != "o/b" {
|
||||
t.Errorf("MembersOf a group naming o/a#1 twice = %+v, want o/a#1 at new, then o/b#2", got)
|
||||
}
|
||||
}
|
||||
@@ -286,7 +286,9 @@ func main() {
|
||||
tools := Tools(asks, time.Now, stdio.State(index.VerdictsState), heldCounts(stdio.State(index.OldSectionsState)), acts)
|
||||
// The modules and the module graph are the newest catalog graph revision's (milestone 47/G): revision.go.
|
||||
revisions := bundleObjects{busstate.ObjectsOf(index.CatalogRevisionsObjects)}
|
||||
trunk := index.TrunkStates{Objects: revisions, Under: keptSource}
|
||||
trunk := index.TrunkStates{Objects: revisions, Under: keptSource, Kept: &index.KeptTrunk{Say: func(format string, a ...any) {
|
||||
fmt.Fprintf(os.Stderr, "[mesh-delivery-indexer-tools] "+format+"\n", a...)
|
||||
}}}
|
||||
tools = append(tools, catalogTools(trunk.State)...)
|
||||
// The seat mesh-catalog's verb revision, a first view of the catalog half answered in this runtime of its own to exactly
|
||||
// the parts whose entry declares it (novox/hq ADR 0663).
|
||||
@@ -310,7 +312,7 @@ func main() {
|
||||
return err
|
||||
})...)
|
||||
tools = append(tools, index.ConsumerTools(keptRead{stdio.State(process.ConsumersState)})...)
|
||||
tools = append(tools, index.RegistrationTools(keptRead{stdio.State(index.ProposalsState)}, asks, time.Now, func(cause, verb, what, why string) error {
|
||||
tools = append(tools, index.RegistrationTools(keptRead{stdio.State(index.RegistrationsState)}, keptRead{stdio.State(index.ProposalsState)}, asks, time.Now, func(cause, verb, what, why string) error {
|
||||
_, err := acts.Record(meshhandactsclient.Request{What: what, Why: why, Cause: cause, Verb: verb})
|
||||
return err
|
||||
})...)
|
||||
|
||||
+68
@@ -0,0 +1,68 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"github.com/nats-io/nats.go"
|
||||
"testing"
|
||||
|
||||
meshcatalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
)
|
||||
|
||||
func TestACatalogRevisionIsSaidOnTheSeatsSubjectWithItsBranch(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
e meshcatalog.CatalogRevisionEvent
|
||||
want string
|
||||
refused bool
|
||||
}{
|
||||
{"trunk", meshcatalog.CatalogRevisionEvent{Kind: "trunk", Branch: "main", Revision: 3}, "mesh.seat.mesh-catalog.event.catalog-revision.main", false},
|
||||
{"candidate", meshcatalog.CatalogRevisionEvent{Kind: "candidate", Branch: "fix/x=2ey", Revision: 1}, "mesh.seat.mesh-catalog.event.catalog-revision.fix/x=2ey", false},
|
||||
{"no branch", meshcatalog.CatalogRevisionEvent{Kind: "trunk", Revision: 3}, "", true},
|
||||
} {
|
||||
got, err := catalogRevisionSubject(c.e)
|
||||
if (err != nil) != c.refused || got != c.want {
|
||||
t.Errorf("%s: subject %q (%v), want %q refused=%v", c.name, got, err, c.want, c.refused)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The duplicate id names the branch, and the publish goes through the acknowledging publisher, its error returned.
|
||||
func TestACatalogRevisionIsPublishedAcknowledgedUnderAnIdNamingItsBranch(t *testing.T) {
|
||||
cred := process.Account{Module: "mesh-delivery-indexer", Node: "n1"}
|
||||
ev := func(branch string) meshcatalog.CatalogRevisionEvent {
|
||||
return meshcatalog.CatalogRevisionEvent{Kind: "candidate", Branch: branch, Revision: 7}
|
||||
}
|
||||
var ids []string
|
||||
var subjects []string
|
||||
publish := func(_ context.Context, m *nats.Msg, id string) error {
|
||||
ids = append(ids, id)
|
||||
subjects = append(subjects, m.Subject)
|
||||
if m.Header.Get(nats.MsgIdHdr) != id {
|
||||
t.Errorf("header id %q, passed id %q", m.Header.Get(nats.MsgIdHdr), id)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
say := sayCatalogRevision(cred, publish)
|
||||
for _, b := range []string{"main", "fix/x"} {
|
||||
if err := say(context.Background(), ev(b)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if len(ids) != 2 || ids[0] == ids[1] || !strings.Contains(ids[0], "main") || !strings.Contains(ids[1], "fix/x") {
|
||||
t.Errorf("ids %v: the same number on two branches must be two ids naming each branch", ids)
|
||||
}
|
||||
if subjects[0] != "mesh.seat.mesh-catalog.event.catalog-revision.main" {
|
||||
t.Errorf("subject %q", subjects[0])
|
||||
}
|
||||
boom := errors.New("stream refused")
|
||||
if err := sayCatalogRevision(cred, func(context.Context, *nats.Msg, string) error { return boom })(context.Background(), ev("main")); !errors.Is(err, boom) {
|
||||
t.Errorf("a publish the bus refused returned %v", err)
|
||||
}
|
||||
if err := say(context.Background(), meshcatalog.CatalogRevisionEvent{Kind: "trunk", Revision: 1}); err == nil {
|
||||
t.Error("an event with no branch was published")
|
||||
}
|
||||
}
|
||||
@@ -1064,7 +1064,7 @@ func TestIndexerWritesModulesAtAMerge(t *testing.T) { // 1239/AC9; I1, I3: the m
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if msg, err := events.GetLastMsgForSubject(ctx, "mesh.seat.mesh-catalog.event."+meshcatalog.EventCatalogRevision); err != nil ||
|
||||
if msg, err := events.GetLastMsgForSubject(ctx, meshcatalog.SubjectCatalogRevisionAll); err != nil ||
|
||||
!strings.Contains(string(msg.Data), fmt.Sprintf(`"revision":%d`, newest)) {
|
||||
t.Errorf("catalog-revision said: %v (%v): want revision %d", msg, err, newest)
|
||||
}
|
||||
|
||||
@@ -460,14 +460,14 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
ix := &index.Indexer{Module: cred.Module, NamedSets: named, Findings: writer, OldSections: awaitedStore{b: oldSections}, Unjudged: awaitedStore{b: unjudged}, Requirements: awaitedStore{b: requirements}, Store: kvStore{candidates}, Modules: modules, Manifests: kvStore{manifests}, Taken: awaitedStore{b: taken}, Builds: known,
|
||||
Verdicts: awaitedStore{b: verdicts}, SayRefused: sayRefused(conn, cred, logger.Printf), SayAccepted: sayAccepted(conn, cred),
|
||||
Predefined: awaitedStore{b: predefined}, ConfigCandidates: awaitedStore{b: configCandidates}, Logf: logger.Printf,
|
||||
Named: namedVia(calls.Call), Trunk: trunkVia(calls.Call), Contains: containsVia(calls.Call), Read: readVia(calls.Call),
|
||||
Named: namedVia(calls.Call), Trunk: trunkVia(calls.Call), BranchHead: branchHeadVia(calls.Call), Contains: containsVia(calls.Call), Read: readVia(calls.Call),
|
||||
ReadOpen: openVia(calls.Call), Packages: awaitedStore{b: packages, js: js, name: index.Bucket(cred.Module, index.PackagesState)}, Files: filesVia(calls.Call), GoSum: goSumVia(calls.Call), GoMod: goModVia(calls.Call),
|
||||
OwnBuild: ownBuildVia(calls.Call, cred.Module, cred.Node), OwnBuildState: awaitedStore{b: ownBuildState}, Heard: heardStore, Placement: placementVia(calls.CallKept), RunningWhole: runningWholeVia(js), Runs: awaitedStore{b: runs}, Graph: graph,
|
||||
SeatDefinitions: awaitedStore{b: seatDefs}, ReadFile: readFileVia(calls.Call), SaySeatDefinition: saySeats,
|
||||
Revisions: revisions, EntryIndex: awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.EntryIndexState), "no entry is found by its fingerprint", logger.Printf)}, // Message a kept call: a read, called again through a transport fault (ADR 0668 §1)
|
||||
// Each catalog revision may bring another holder of a build seat: the build modules are found again (buildmodules.go).
|
||||
// A new trunk revision that changes the modules is said on the views family of modules too (design 47 G A-360).
|
||||
SayCatalogRevision: afterRevision(refreshingBuildModules(sayingModulesView(sayCatalogRevision(conn, cred), cred, views.JetStreamPublish(js)), &buildFollow), &fillFollow),
|
||||
SayCatalogRevision: afterRevision(refreshingBuildModules(sayingModulesView(sayCatalogRevision(cred, views.JetStreamPublish(js)), cred, views.JetStreamPublish(js)), &buildFollow), &fillFollow),
|
||||
SayFill: sayFill(conn, cred),
|
||||
// The catalog's released versions (design 47 G §349): the state versions, bound when it appears, and the event version-released.
|
||||
Versions: awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, index.VersionsState), "no released version is kept, so no package-published makes a version", logger.Printf)},
|
||||
@@ -509,6 +509,14 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// node-engine names the successor while it replaces this process, and an old process that learnt it later would take
|
||||
// revisions in with its old judge.
|
||||
ix.FixOwnBuild(ctx)
|
||||
// Each step of the start says when it begins and how long it took, so a start that does not end is read in the journal
|
||||
// and not guessed (lead-47 2026-11-16: two starts on novox ran 25 minutes at one CPU with no line).
|
||||
step := func(name string) func() {
|
||||
began := time.Now()
|
||||
logger.Printf("start: %s", name)
|
||||
return func() { logger.Printf("start: %s done in %s", name, time.Since(began).Round(time.Millisecond)) }
|
||||
}
|
||||
done := step("reading the build log")
|
||||
// The log's entries at the start are read before any fingerprint is computed or registered module indexed (below); each
|
||||
// new one indexes the module it names when that module is registered and the bucket holds none (registered.go).
|
||||
var started atomic.Bool
|
||||
@@ -537,6 +545,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
"indexed from it, until the indexer starts again", builds.BuildLogRead, err)
|
||||
}
|
||||
started.Store(true)
|
||||
done()
|
||||
logger.Printf("start: the build log held %d entries at the start", len(startEnds))
|
||||
// The source revision of a watched branch (design 47 G §15, §115, §362): asked of the seat mesh-delivery's verb source as a
|
||||
// kept call, once at the start, before the handler of its event source-moved reads anything; a source-moved heard
|
||||
// meanwhile is read after.
|
||||
@@ -553,6 +563,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
}
|
||||
return nil
|
||||
}}
|
||||
done = step("the build modules and the seat holders")
|
||||
buildModules.refresh()
|
||||
// The assignments is the seat mesh-module-assigner's: read through its holder's first view and views family, never its bucket (DLV-F1r).
|
||||
assignmentsErrs := make(chan error)
|
||||
@@ -566,11 +577,15 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
seatHolders := &seatHoldersFollow{ctx: ctx, ix: ix, reader: reader, logf: logger.Printf}
|
||||
seatHolders.refresh()
|
||||
buildModules.seatHolders.Store(seatHolders)
|
||||
done()
|
||||
done = step("following the assignments")
|
||||
followAssignments(ctx, reader, seatHolders.holders(), process.AwaitBucket(ctx, js, conn, index.Bucket(cred.Module, views.StatusState), "indexer_first_views reads nothing and indexer_first_view_again is not heard", logger.Printf), cred, js, gate, func(ctx context.Context) {
|
||||
if err := ix.Recompute(ctx); err != nil {
|
||||
logger.Printf("writing the modules again after the assignments of a build module changed: %v", err)
|
||||
}
|
||||
}, logger.Printf)
|
||||
done()
|
||||
done = step("the toolchain watch and the previews")
|
||||
tools := newToolchainWatch(ctx, js, toolchainSink{
|
||||
put: gate.Toolchain,
|
||||
apply: func(ctx context.Context, module, key string, value []byte, deleted bool) error {
|
||||
@@ -597,26 +612,42 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// having no toolchain reported, and back).
|
||||
// The packages the held modules reach and the state does not hold yet are published at their trunk's taken merge, and
|
||||
// the imports the log's done builds reported are then kept, before the fingerprints are computed (ADR 0593 §2, §4).
|
||||
done()
|
||||
packages.Ready(func(jetstream.KeyValue) {
|
||||
done := step("publishing the packages at start")
|
||||
if err := ix.FillPackages(ctx); err != nil {
|
||||
logger.Printf("publishing the packages at start: %v", err)
|
||||
}
|
||||
done()
|
||||
})
|
||||
for _, le := range startEnds {
|
||||
done = step(fmt.Sprintf("the out-of-step findings of %d build ends", len(startEnds)))
|
||||
values := make([][]byte, len(startEnds))
|
||||
for i, le := range startEnds {
|
||||
ix.ApplyOutOfStepBuild(ctx, le.Value)
|
||||
if err := ix.ApplyBuildEnd(ctx, le.Value); err != nil {
|
||||
logger.Printf("%s entry %d: its imports are not kept: %v", builds.BuildLogRead, le.Sequence, err)
|
||||
values[i] = le.Value
|
||||
if (i+1)%50000 == 0 {
|
||||
logger.Printf("start: out-of-step findings: %d of %d entries", i+1, len(startEnds))
|
||||
}
|
||||
}
|
||||
done()
|
||||
if err := ix.ApplyBuildEndsAtStart(ctx, values, func(i int, err error) {
|
||||
logger.Printf("%s entry %d: its imports are not kept: %v", builds.BuildLogRead, startEnds[i].Sequence, err)
|
||||
}); err != nil {
|
||||
logger.Printf("keeping the view after the build ends at start: %v", err)
|
||||
}
|
||||
startEnds = nil
|
||||
done = step("computing the fingerprints")
|
||||
if err := ix.RecomputeAtStart(ctx); err != nil {
|
||||
logger.Printf("computing the new-form fingerprints at start: %v", err)
|
||||
}
|
||||
done()
|
||||
done = step("indexing the registered modules")
|
||||
// A module registered in the catalogue from its own repository gets its manifest indexed when the builder's log names
|
||||
// it and the bucket holds none (registered.go); the entries that follow are heard above.
|
||||
if err := ix.IndexRegisteredAll(ctx); err != nil {
|
||||
logger.Printf("indexing the registered modules at start: %v", err)
|
||||
}
|
||||
done()
|
||||
// A person takes a registered module in now with indexer_take_in, which writes the takeins state (issue 1945): watched,
|
||||
// as every other input. A module the forge could not read is also read again by itself on the next source revision and
|
||||
// merged message (index.RetryUnread).
|
||||
@@ -751,7 +782,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
logger.Printf("the open changes of the registrations are not all read again at the start: %v", err)
|
||||
}
|
||||
_ = notHeard.Bind(process.EventsStream, "no package's series moves, no warrant for a change of a registration is heard, no node's manifest-reads-changed and no assignments view is heard until the indexer starts again", func() error {
|
||||
if err := hub.Start(ctx, js, cred); err != nil {
|
||||
if err := startHearing(ctx, hub, js, cred); err != nil {
|
||||
stopped = cmp.Or(stopped, err)
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -180,6 +180,41 @@ func TestAMessageIsIndexedAndAcked(t *testing.T) { // 1238/AC2 (unit part)
|
||||
}
|
||||
}
|
||||
})
|
||||
// Design 47 G A-397: the program hears its kept seat events (package-published among them) on their own streams once its
|
||||
// durable on EVENTS is bound, never when that fails.
|
||||
for _, startErr := range []error{nil, errors.New("not composed yet")} {
|
||||
h := &startedHub{startErr: startErr}
|
||||
if err := startHearing(context.Background(), h, nil, process.Account{}); !errors.Is(err, startErr) || h.kept != (startErr == nil) {
|
||||
t.Errorf("start error %v: answered %v, kept streams bound %v", startErr, err, h.kept)
|
||||
}
|
||||
}
|
||||
// Design 47 G A-420: a package-published that cannot be kept is left for the bus, and on its last delivery said given up,
|
||||
// naming its subject, the artifact and version and the last refusal; before it, nothing more is said.
|
||||
for _, n := range []uint64{4, EventDeliveries} {
|
||||
var said []string
|
||||
m := &deliveredMsg{subject: "mesh.seat.mesh-builds.event.package-published", data: []byte(`{"artifact":"m:program","version":"0.1.6"}`), n: n}
|
||||
heardPackagePublished(context.Background(), &index.Indexer{}, m, func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) })
|
||||
givenUp := strings.Contains(strings.Join(said, "\n"), "gives up mesh.seat.mesh-builds.event.package-published: the package-published of m:program 0.1.6, delivered 5 times")
|
||||
if !m.naked || givenUp != (n == EventDeliveries) || (givenUp && !strings.Contains(strings.Join(said, "\n"), "the last refusal: the state versions is not kept here")) {
|
||||
t.Errorf("delivery %d: naked %v, said %q", n, m.naked, said)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// deliveredMsg is a message delivered n times, which records a Nak.
|
||||
type deliveredMsg struct {
|
||||
jetstream.Msg
|
||||
subject string
|
||||
data []byte
|
||||
n uint64
|
||||
naked bool
|
||||
}
|
||||
|
||||
func (d *deliveredMsg) Subject() string { return d.subject }
|
||||
func (d *deliveredMsg) Data() []byte { return d.data }
|
||||
func (d *deliveredMsg) Nak() error { d.naked = true; return nil }
|
||||
func (d *deliveredMsg) Metadata() (*jetstream.MsgMetadata, error) {
|
||||
return &jetstream.MsgMetadata{NumDelivered: d.n}, nil
|
||||
}
|
||||
|
||||
// opener is a bus that answers every bucket with err.
|
||||
@@ -270,3 +305,17 @@ func TestTheIndexerTakesAPushedMessage(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// startedHub records which of an event hub's starts were asked.
|
||||
type startedHub struct {
|
||||
startErr error
|
||||
kept bool
|
||||
}
|
||||
|
||||
func (h *startedHub) Start(context.Context, jetstream.JetStream, process.Account) error {
|
||||
return h.startErr
|
||||
}
|
||||
func (h *startedHub) StartKept(context.Context, jetstream.JetStream, process.Account) int {
|
||||
h.kept = true
|
||||
return 1
|
||||
}
|
||||
|
||||
@@ -260,3 +260,14 @@ func openVia(call toolCaller) func(ctx context.Context) ([]forgemsg.OpenPull, er
|
||||
return out, nil
|
||||
}
|
||||
}
|
||||
|
||||
// branchHeadVia is the head of a repository's branch through the seat git's verb branch.
|
||||
func branchHeadVia(call toolCaller) func(ctx context.Context, repository, branch string) (string, error) {
|
||||
return func(_ context.Context, repository, branch string) (string, error) {
|
||||
var a git.BranchAnswer
|
||||
if err := askGit(call, git.VerbBranch, git.BranchInput{Repository: repository, Branch: branch}, &a); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return a.Head, nil
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"fmt"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"mesh-delivery-indexer/internal/index"
|
||||
"slices"
|
||||
"strings"
|
||||
@@ -125,6 +126,28 @@ func TestTheTrunkIsTheDirectoriesTheForgeListsTrees(t *testing.T) { // 1900/AC3
|
||||
t.Errorf("%s: own build = %q %v, want %q", row.name, fp, err, row.want)
|
||||
}
|
||||
}
|
||||
// Design 47 G A-366: a process started with a fingerprint takes it and asks no engine; one started without asks the engine.
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
started process.Build
|
||||
want string
|
||||
wantAsked bool
|
||||
}{
|
||||
{"started with a fingerprint", process.Build{Fingerprint: "src2:env"}, "src2:env", false},
|
||||
{"started without one", process.Build{Unknown: process.OwnBuildUnknown}, "src2:abc", true},
|
||||
} {
|
||||
started := startedBuild
|
||||
startedBuild = func() process.Build { return row.started }
|
||||
defer func() { startedBuild = started }()
|
||||
called := false
|
||||
ask := func(string, any) (json.RawMessage, error) {
|
||||
called = true
|
||||
return json.RawMessage(`{"runs":{"fp":"src2:abc"}}`), nil
|
||||
}
|
||||
if fp, err := ownBuildVia(ask, "acme-indexer", "n1")(context.Background()); err != nil || fp != row.want || called != row.wantAsked {
|
||||
t.Errorf("%s: own build = %q %v, engine asked %v, want %q asked %v", row.name, fp, err, called, row.want, row.wantAsked)
|
||||
}
|
||||
}
|
||||
none := func(string, any) (json.RawMessage, error) { return json.RawMessage(`{}`), nil }
|
||||
if _, err := ownBuildVia(none, "acme-indexer", "n1")(context.Background()); err == nil {
|
||||
t.Error("an engine that says nothing runs gave a build")
|
||||
|
||||
@@ -125,24 +125,49 @@ func (b *busObjects) Fill(ctx context.Context) (int64, int64, error) {
|
||||
return int64(st.Size()), info.CachedInfo().Config.MaxBytes, nil
|
||||
}
|
||||
|
||||
// sayCatalogRevision publishes the seat mesh-catalog's event catalog-revision under the duplicate id of its kind and
|
||||
// number, so a revision said twice is heard once (design 47, Two graphs).
|
||||
func sayCatalogRevision(conn *nats.Conn, cred process.Account) func(context.Context, meshcatalog.CatalogRevisionEvent) error {
|
||||
return func(_ context.Context, e meshcatalog.CatalogRevisionEvent) error {
|
||||
data, err := json.Marshal(e)
|
||||
// catalogRevisionSubject is where an event says: the seat's catalog-revision.<branch> (the seat definition's subject), the
|
||||
// branch the event's own field names. An event with no branch is refused: the indexer names the branch of every revision it
|
||||
// keeps, and a bare subject is one the bus refuses.
|
||||
func catalogRevisionSubject(e meshcatalog.CatalogRevisionEvent) (string, error) {
|
||||
if e.Branch == "" {
|
||||
return "", fmt.Errorf("the catalog-revision of %s revision %d names no branch, so it has no subject", e.Kind, e.Revision)
|
||||
}
|
||||
return meshcatalog.SubjectCatalogRevision(e.Branch), nil
|
||||
}
|
||||
|
||||
// catalogRevisionMessage is the seat mesh-catalog's event catalog-revision as it is published, with its duplicate id: the
|
||||
// event's name, its branch, its kind and its number, so a revision said twice is heard once and the same number on two
|
||||
// branches (a candidate's and the trunk's) is two events (design 47 G A-409).
|
||||
func catalogRevisionMessage(cred process.Account, e meshcatalog.CatalogRevisionEvent) (*nats.Msg, string, error) {
|
||||
data, err := json.Marshal(e)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
subject, err := catalogRevisionSubject(e)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
msg := nats.NewMsg(subject)
|
||||
id := fmt.Sprintf("%s-%s-%s-%d%s", meshcatalog.EventCatalogRevision, e.Branch, e.Kind, e.Revision, e.Heads)
|
||||
msg.Header.Set("x-event-id", id)
|
||||
msg.Header.Set("x-source", cred.Module)
|
||||
msg.Header.Set("x-node", cred.Node)
|
||||
msg.Header.Set("x-time", time.Now().UTC().Format(time.RFC3339Nano))
|
||||
msg.Header.Set("content-type", "application/json")
|
||||
msg.Header.Set(nats.MsgIdHdr, id)
|
||||
msg.Data = data
|
||||
return msg, id, nil
|
||||
}
|
||||
|
||||
// sayCatalogRevision publishes the event through the bus's acknowledgement (views.JetStreamPublish), an error returned when
|
||||
// the bus did not keep it: a core publish says nothing of a stream that refused it (design 47 G A-409).
|
||||
func sayCatalogRevision(cred process.Account, publish func(context.Context, *nats.Msg, string) error) func(context.Context, meshcatalog.CatalogRevisionEvent) error {
|
||||
return func(ctx context.Context, e meshcatalog.CatalogRevisionEvent) error {
|
||||
msg, id, err := catalogRevisionMessage(cred, e)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
msg := nats.NewMsg(seatEventSubject(meshcatalog.EventCatalogRevision))
|
||||
id := fmt.Sprintf("%s-%s-%d%s", meshcatalog.EventCatalogRevision, e.Kind, e.Revision, e.Heads)
|
||||
msg.Header.Set("x-event-id", id)
|
||||
msg.Header.Set("x-source", cred.Module)
|
||||
msg.Header.Set("x-node", cred.Node)
|
||||
msg.Header.Set("x-time", time.Now().UTC().Format(time.RFC3339Nano))
|
||||
msg.Header.Set("content-type", "application/json")
|
||||
msg.Header.Set(nats.MsgIdHdr, id)
|
||||
msg.Data = data
|
||||
return conn.PublishMsg(msg)
|
||||
return publish(ctx, msg, id)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -5,6 +5,8 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
)
|
||||
|
||||
// compositionVerb is the node-engine seat's read verb that says what runs of a module on a machine (its Runs.FP), asked of
|
||||
@@ -12,9 +14,17 @@ import (
|
||||
// from the trunk's value of itself or the haves bucket.
|
||||
const compositionVerb = "seat:node-engine.composition"
|
||||
|
||||
// ownBuildVia asks the engine of node which fingerprint of the indexer runs there.
|
||||
// startedBuild is the build the process was started with (mesh-sdk process.OwnBuild: MESH_FINGERPRINT, set by the node-engine and
|
||||
// by a unit started by hand); a variable so a test starts the process with another.
|
||||
var startedBuild = process.OwnBuild
|
||||
|
||||
// ownBuildVia is the build of the indexer: the one it was started with, else what the engine of node says which fingerprint of the indexer runs there.
|
||||
func ownBuildVia(call toolCaller, module, node string) func(ctx context.Context) (string, error) {
|
||||
return func(context.Context) (string, error) {
|
||||
// The build the process was started with is the build; the engine is asked only when there is none (design 47 G A-366).
|
||||
if fp := startedBuild().Fingerprint; fp != "" {
|
||||
return fp, nil
|
||||
}
|
||||
raw, err := call(compositionVerb+"@"+node, map[string]any{"module": module})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("%s: %w", compositionVerb, err)
|
||||
|
||||
@@ -59,12 +59,19 @@ func newRegistry(js jetstream.JetStream, conn *nats.Conn, cred process.Account,
|
||||
}
|
||||
_, err = js.Publish(ctx, historySubject(cred.Module, r.Module), data, jetstream.WithMsgID(fmt.Sprintf("%s.%d", r.Module, r.Version)))
|
||||
if err == nil {
|
||||
// Past 80 % full the stream's fill is a finding (design 47 G §704).
|
||||
if st, serr := js.Stream(ctx, catalogue.StreamBusName(index.RegistrationHistoryStream)); serr == nil {
|
||||
if info, ierr := st.Info(ctx); ierr == nil {
|
||||
// Past 80 % full the stream's fill is a finding (design 47 G §704); read within a second, and said when it cannot be.
|
||||
fctx, cancel := context.WithTimeout(ctx, time.Second)
|
||||
st, serr := js.Stream(fctx, catalogue.StreamBusName(index.RegistrationHistoryStream))
|
||||
if serr == nil {
|
||||
var info *jetstream.StreamInfo
|
||||
if info, serr = st.Info(fctx); serr == nil {
|
||||
ix.HistoryFill(ctx, int64(info.State.Bytes), info.Config.MaxBytes)
|
||||
}
|
||||
}
|
||||
cancel()
|
||||
if serr != nil {
|
||||
logf("the fill of %s is not read, so its finding is not judged: %v", index.RegistrationHistoryStream, serr)
|
||||
}
|
||||
}
|
||||
return err
|
||||
},
|
||||
|
||||
@@ -48,6 +48,11 @@ func takenIn(ctx context.Context, ix takerIn, kv jetstream.KeyValue, e jetstream
|
||||
return
|
||||
}
|
||||
asked, r := "take-in", index.TakenIn{}
|
||||
logf("%s.%s: asked by %s (%s): started", index.TakeInsState, e.Key(), a.By, a.Why)
|
||||
started := time.Now()
|
||||
defer func() {
|
||||
logf("%s.%s: ended after %s", index.TakeInsState, e.Key(), time.Since(started).Round(time.Millisecond))
|
||||
}()
|
||||
if a.Registry != nil {
|
||||
asked, r = "registrations' act "+a.Registry.Verb, ix.RegistryAct(ctx, a)
|
||||
} else if a.Recompute {
|
||||
|
||||
@@ -48,6 +48,38 @@ func sayVersionReleased(conn *nats.Conn, cred process.Account) func(context.Cont
|
||||
}
|
||||
}
|
||||
|
||||
// hubStarter is the event hub as startHearing starts it.
|
||||
type hubStarter interface {
|
||||
Start(ctx context.Context, js jetstream.JetStream, cred process.Account) error
|
||||
StartKept(ctx context.Context, js jetstream.JetStream, cred process.Account) int
|
||||
}
|
||||
|
||||
// startHearing binds the indexer's durable on the shared events stream and then, beside it, its durable on each kept seat
|
||||
// event's own stream (SEAT_<seat>_<event>) whose event it handles: package-published, toolchain-agreed and the build seats'
|
||||
// step-ended are never on EVENTS, and its tool runner reads only its own twin there, so the program hears them only so
|
||||
// (design 47 G A-397; mesh-sdk process.EventHub.StartKept).
|
||||
func startHearing(ctx context.Context, hub hubStarter, js jetstream.JetStream, cred process.Account) error {
|
||||
if err := hub.Start(ctx, js, cred); err != nil {
|
||||
return err
|
||||
}
|
||||
hub.StartKept(ctx, js, cred)
|
||||
return nil
|
||||
}
|
||||
|
||||
// EventDeliveries is how many times the bus gives a message of the indexer's events consumer: the MaxDeliver the grants part
|
||||
// composes a module's durable with (mesh-grants buscompose).
|
||||
const EventDeliveries = 5
|
||||
|
||||
// leftForBus leaves a message the indexer could not take for the bus to give again, and on the last delivery its consumer allows
|
||||
// says that it gives the message up, naming its subject, what it carries and the last refusal (design 47 G A-420): past it the bus
|
||||
// drops the message without a word.
|
||||
func leftForBus(m jetstream.Msg, what string, err error, logf func(string, ...any)) {
|
||||
_ = m.Nak()
|
||||
if md, merr := m.Metadata(); merr == nil && md != nil && md.NumDelivered >= EventDeliveries {
|
||||
logf("gives up %s: %s, delivered %d times, the most its consumer allows; the last refusal: %v", m.Subject(), what, md.NumDelivered, err)
|
||||
}
|
||||
}
|
||||
|
||||
// heardPackagePublished takes one package-published of the seat mesh-builds into the catalog's versions (design 47 G §349):
|
||||
// one that does not decode is said and acknowledged; one that cannot be kept is left, so the bus gives it again.
|
||||
func heardPackagePublished(ctx context.Context, ix *index.Indexer, m jetstream.Msg, logf func(string, ...any)) {
|
||||
@@ -59,7 +91,7 @@ func heardPackagePublished(ctx context.Context, ix *index.Indexer, m jetstream.M
|
||||
}
|
||||
if err := ix.HearPackagePublished(ctx, p); err != nil {
|
||||
logf("versions: the package-published of %s %s could not be kept, so the bus gives it again: %v", p.Artifact, p.Version, err)
|
||||
_ = m.Nak()
|
||||
leftForBus(m, fmt.Sprintf("the package-published of %s %s", p.Artifact, p.Version), err, logf)
|
||||
return
|
||||
}
|
||||
_ = m.Ack()
|
||||
@@ -76,7 +108,7 @@ func heardToolchainAgreed(ctx context.Context, ix *index.Indexer, m jetstream.Ms
|
||||
}
|
||||
if err := ix.HearToolchainAgreed(ctx, t); err != nil {
|
||||
logf("versions: the toolchain-agreed of %s could not be kept, so the bus gives it again: %v", t.Language, err)
|
||||
_ = m.Nak()
|
||||
leftForBus(m, "the toolchain-agreed of "+t.Language, err, logf)
|
||||
return
|
||||
}
|
||||
_ = m.Ack()
|
||||
@@ -93,7 +125,7 @@ func heardVersionDelivered(ctx context.Context, ix *index.Indexer, m jetstream.M
|
||||
}
|
||||
if err := ix.HearVersionDelivered(ctx, d); err != nil {
|
||||
logf("versions: the version-delivered of %s %s could not be kept, so the bus gives it again: %v", d.Module, d.Version, err)
|
||||
_ = m.Nak()
|
||||
leftForBus(m, fmt.Sprintf("the version-delivered of %s %s", d.Module, d.Version), err, logf)
|
||||
return
|
||||
}
|
||||
_ = m.Ack()
|
||||
@@ -109,7 +141,7 @@ func heardWaitEnded(ctx context.Context, ix *index.Indexer, m jetstream.Msg, log
|
||||
}
|
||||
if err := ix.HearWaitEnded(ctx, w); err != nil {
|
||||
logf("versions: the wait-ended of %s for %s could not be kept, so the bus gives it again: %v", w.Part, w.Module, err)
|
||||
_ = m.Nak()
|
||||
leftForBus(m, fmt.Sprintf("the wait-ended of %s for %s", w.Part, w.Module), err, logf)
|
||||
return
|
||||
}
|
||||
_ = m.Ack()
|
||||
@@ -128,13 +160,13 @@ func heardStepEnded(ctx context.Context, ix *index.Indexer, m jetstream.Msg, log
|
||||
if err := json.Unmarshal(m.Data(), &end); err == nil {
|
||||
if err := ix.HearValidationEnded(ctx, end); err != nil {
|
||||
logf("take-in: the end of the validation ask %s is not taken in, so the bus gives it again: %v", end.Ask, err)
|
||||
_ = m.Nak()
|
||||
leftForBus(m, "the end of the validation ask "+end.Ask, err, logf)
|
||||
return
|
||||
}
|
||||
}
|
||||
if err := ix.HearInputEnded(ctx, e); err != nil {
|
||||
logf("inputs: the input of the ask %s could not be deleted, so the bus gives it again: %v", e.Ask, err)
|
||||
_ = m.Nak()
|
||||
leftForBus(m, "the step-ended of the ask "+e.Ask, err, logf)
|
||||
return
|
||||
}
|
||||
_ = m.Ack()
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user