Compare commits
34
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e4781edcd6 | ||
|
|
1db38fcc72 | ||
|
|
eba4309b19 | ||
|
|
a7fc37561e | ||
|
|
e93575ffe7 | ||
|
|
859f88b2f0 | ||
|
|
517a4b3cf1 | ||
|
|
7f88916863 | ||
|
|
5dee4cb263 | ||
|
|
77b683c1c1 | ||
|
|
60e32772a5 | ||
|
|
fd2ef6c7e8 | ||
|
|
71371df160 | ||
|
|
b2d5644c34 | ||
|
|
9598b92c39 | ||
|
|
f00d34673f | ||
|
|
87357d84bd | ||
|
|
08dde53e33 | ||
|
|
c772eb0683 | ||
|
|
a7b2fb0ffa | ||
|
|
ef086e0b75 | ||
|
|
a8adda1a8c | ||
|
|
e1f0c1a1cc | ||
|
|
bf24bb9740 | ||
|
|
814244fd5a | ||
|
|
442b03f61d | ||
|
|
9083287d57 | ||
|
|
d0436bb38c | ||
|
|
fe51379ee2 | ||
|
|
037cb9555e | ||
|
|
5c5136d6d1 | ||
|
|
c43e7a5d1c | ||
|
|
e04321acae | ||
|
|
4c2914194b |
@@ -95,15 +95,17 @@ type machineEnd struct {
|
||||
|
||||
// run runs one composition step and answers what it came to: per machine not away it joins or starts the candidate's
|
||||
// composition run (§718), waits for each run's end in its key, and judges each machine from it; failed when a machine does not
|
||||
// compose, could not run when a machine could not be composed or was not judged, else passed with a warning per machine away.
|
||||
// compose, could not run when a machine could not be composed or was not judged, nothing to run when no node places the module on
|
||||
// the branch (H-A-179), else passed with a warning per machine away.
|
||||
func (r *compositionRun) run(ctx context.Context, id string, step meshwants.StepRun) stepOutcome {
|
||||
candidate, err := candidateKeyOf(step)
|
||||
if err != nil {
|
||||
return couldNotRun(err.Error())
|
||||
}
|
||||
nodes := r.inputs.machines(step.Module, step.Branch)
|
||||
if len(nodes) == 0 {
|
||||
return couldNotRun(fmt.Sprintf("no node places %s on the branch %q, so the composition step has no machine to compose", step.Module, step.Branch))
|
||||
if len(nodes) == 0 { // a mover no machine holds on the branch has nothing to judge: nothing to run, passed (H-A-179, ADR 0682 §8)
|
||||
return stepOutcome{Outcome: meshwants.StepEndedOutcomeNothingToRun,
|
||||
Said: fmt.Sprintf("no node places %s on the branch %q, so the composition step has no machine to compose", step.Module, step.Branch)}
|
||||
}
|
||||
reader := readerOf(id, step)
|
||||
prep := &runPrep{r: r, step: step}
|
||||
|
||||
@@ -259,7 +259,11 @@ func TestTheCompositionStepRunsOneAskPerMachine(t *testing.T) {
|
||||
}
|
||||
none := newRunStand(t, "src2:e1", "src2:e1", "d1")
|
||||
none.r.inputs.machines = func(string, string) []string { return machines }
|
||||
if out := none.r.run(context.Background(), "ask-step", step); out.Outcome != meshwants.StepEndedOutcomeCouldNotRun ||
|
||||
outcome := meshwants.StepEndedOutcomeCouldNotRun
|
||||
if len(machines) == 0 { // a mover no machine holds: nothing to run, saying why (H-A-179)
|
||||
outcome = meshwants.StepEndedOutcomeNothingToRun
|
||||
}
|
||||
if out := none.r.run(context.Background(), "ask-step", step); out.Outcome != outcome ||
|
||||
!strings.Contains(out.Said, want) || (len(machines) > 0 && !strings.Contains(out.Said, "shanks is away")) || len(none.asks) != 0 {
|
||||
t.Errorf("%s came to %+v, %d asks", name, out, len(none.asks))
|
||||
}
|
||||
|
||||
@@ -119,6 +119,13 @@ func TestManifest(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
// H-A-186 (design 47 G §370): asks, which the gate, the teller and the view import, is a declared Go package of the
|
||||
// module, so every release publishes it; H-A-188: in the module's version series, 0.3.
|
||||
if !slices.ContainsFunc(m.Build.Artifacts, func(a catalogue.Artifact) bool {
|
||||
return a.Name == "asks" && a.Kind == catalogue.ArtifactPackage && a.Language == "go" && slices.Equal(a.Paths, []string{"asks"}) && a.Series == m.Version && m.Version == "0.3"
|
||||
}) {
|
||||
t.Errorf("artifacts = %+v, want the Go package asks of series 0.3, the module's", m.Build.Artifacts)
|
||||
}
|
||||
// ADR 0635 §5: its views family is its ordinary events, `views.<state>` among its emits, on no stream of its own.
|
||||
// The seat's four events are said on the seat's own subject (the seat's family), so the module keeps no stream for them
|
||||
// and lists none among its emits.
|
||||
|
||||
@@ -370,6 +370,15 @@
|
||||
"loads": [
|
||||
"mesh-delivery-assigner"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "asks",
|
||||
"kind": "package",
|
||||
"language": "go",
|
||||
"paths": [
|
||||
"asks"
|
||||
],
|
||||
"series": "0.3"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -1114,6 +1114,7 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m mes
|
||||
// The versions of the mesh packages the fingerprint was computed with (novox/hq ADR 0573 §2): the build module writes
|
||||
// them into the workspace, whatever the repository's go.mod names.
|
||||
ask.Packages = maps.Clone(m.Packages)
|
||||
maps.DeleteFunc(ask.Packages, func(_, v string) bool { return v == "" }) // a package with no released version is no pin (H-A-161)
|
||||
for _, fb := range req.Builds {
|
||||
delete(ask.Packages, fb.Package) // a package `builds` names is named in neither packages nor package (§465)
|
||||
}
|
||||
|
||||
@@ -332,10 +332,12 @@ func TestABundleAndAnArchiveAreBuiltOnOneNodeAndDone(t *testing.T) { // 1607/AC7
|
||||
func TestATrunkFingerprintIsOneAskOnTheDeliveryStream(t *testing.T) {
|
||||
r := newRig(t, goBundle)
|
||||
r.asks.Answer = nil // no node has pulled it yet
|
||||
// Issue 1585 (ADR 0573 §2): the ask carries the package versions the fingerprint was computed with.
|
||||
sdk := buildstep.PackageVersions{"git.novox.be/novox/mesh-sdk/go": hex40('5')}
|
||||
// Issue 1585 (ADR 0573 §2): the ask carries the module's pins; a package the entry keeps with no version (none
|
||||
// released, design 47 H-A-161) is no pin.
|
||||
sdk := buildstep.PackageVersions{"git.novox.be/novox/mesh-sdk/go": "v0.1.20"}
|
||||
data, err := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-catalog", Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20",
|
||||
Fp2: fp1, Manifest: mdigest('a'), Path: "modules/m", Toolchains: CatalogToolchains(agreedToolchains), Packages: sdk})
|
||||
Fp2: fp1, Manifest: mdigest('a'), Path: "modules/m", Toolchains: CatalogToolchains(agreedToolchains),
|
||||
Packages: map[string]string{"git.novox.be/novox/mesh-sdk/go": "v0.1.20", "git.novox.be/novox/mesh-contracts/go/seats/x": ""}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -266,10 +266,16 @@ func TestManifestProcess(t *testing.T) { // 1607/AC1
|
||||
if _, bus := p["bus"]; bus {
|
||||
t.Errorf("the process declares a bus: it runs on its own broker credential")
|
||||
}
|
||||
// Issue 1966's bootstrap: the package artifact builds is out of the manifest until the new builder runs; the follow-up
|
||||
// puts it back with its assertion (ADR 0634 §1, ADR 0606 §4: three artifacts, the third the package builds).
|
||||
if len(m.Build.Artifacts) != 2 {
|
||||
t.Fatalf("artifacts = %+v, want the program and the tools", m.Build.Artifacts)
|
||||
// H-A-186, design 47 G §370: a Go package another module imports is a declared package artifact of its module, so every
|
||||
// release publishes it: builds (ADR 0634 §1, ADR 0606 §4), imported by the build modules and the node-engine, and
|
||||
// buildrun, imported by the indexer, the view and the watchman.
|
||||
if len(m.Build.Artifacts) != 4 {
|
||||
t.Fatalf("artifacts = %+v, want the program, the tools and the packages builds and buildrun", m.Build.Artifacts)
|
||||
}
|
||||
for i, name := range []string{"builds", "buildrun"} {
|
||||
if a := m.Build.Artifacts[2+i]; a.Name != name || a.Kind != "package" || a.Language != "go" || !reflect.DeepEqual(a.Paths, []string{name}) || a.Series != "0.1" {
|
||||
t.Errorf("package artifact = %+v, want the Go package %s of series 0.1", a, name)
|
||||
}
|
||||
}
|
||||
if a := m.Build.Artifacts[1]; a.Name != "tools" || a.Kind != "bundle" || a.Binary != "mesh-delivery-builder-tools" || !reflect.DeepEqual(a.Loads, []string{"mesh-delivery-builder-tools"}) {
|
||||
t.Errorf("tools artifact = %+v", a) // 1945
|
||||
|
||||
@@ -334,6 +334,24 @@
|
||||
"loads": [
|
||||
"mesh-delivery-builder-tools"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "builds",
|
||||
"kind": "package",
|
||||
"language": "go",
|
||||
"paths": [
|
||||
"builds"
|
||||
],
|
||||
"series": "0.1"
|
||||
},
|
||||
{
|
||||
"name": "buildrun",
|
||||
"kind": "package",
|
||||
"language": "go",
|
||||
"paths": [
|
||||
"buildrun"
|
||||
],
|
||||
"series": "0.1"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -141,7 +141,7 @@ func main() {
|
||||
tools = append(tools, OwedTools(askState(owedAsks, groups.OwedAsksBucket(served), 25*time.Second, func(v []byte) bool {
|
||||
a, err := groups.DecodeOwedAsk(v)
|
||||
return err == nil && a.Done()
|
||||
}), func() ([]OwedRow, error) { return performedRows(performed) }, func(what, why string) error {
|
||||
}), func(f OwedFilter) ([]OwedRow, error) { return performedRows(performed, f) }, func(what, why string) error {
|
||||
_, err := recorder.Record(meshhandactsclient.Request{What: what, Why: why, Cause: "owed-dropped"})
|
||||
return err
|
||||
}, time.Now)...)
|
||||
|
||||
+43
-5
@@ -94,17 +94,55 @@ func TestTheBridgeTools(t *testing.T) {
|
||||
}
|
||||
return json.Marshal(a)
|
||||
}
|
||||
rows := []OwedRow{{Of: seat.Seat, ID: "a", Outcome: forgemsg.OwedPerformed, At: at}}
|
||||
rows := []OwedRow{
|
||||
{OwedEnded: forgemsg.OwedEnded{Of: seat.Seat, ID: "a", Outcome: forgemsg.OwedPerformed, At: at}, Repository: "novox/mesh-sdk", Number: 178},
|
||||
{OwedEnded: forgemsg.OwedEnded{Of: seat.Seat, ID: "b", Outcome: forgemsg.OwedPerformed, At: at}, Repository: "novox/mesh-sdk", Number: 179}}
|
||||
var filters []OwedFilter
|
||||
performed := func(f OwedFilter) ([]OwedRow, error) {
|
||||
filters = append(filters, f)
|
||||
var out []OwedRow
|
||||
for _, r := range rows {
|
||||
if f.holds(r) {
|
||||
out = append(out, r)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
var recorded []string
|
||||
recordErr := error(nil)
|
||||
record := func(what, why string) error { recorded = append(recorded, what+": "+why); return recordErr }
|
||||
tools := map[string]func(map[string]any) (any, error){}
|
||||
for _, tl := range OwedTools(ask, func() ([]OwedRow, error) { return rows, nil }, record, func() time.Time { return at }) {
|
||||
for _, tl := range OwedTools(ask, performed, record, func() time.Time { return at }) {
|
||||
tools[tl.Name] = tl.Run
|
||||
}
|
||||
out, err := tools[OwedTool](map[string]any{"of": seat.Seat})
|
||||
if m, _ := out.(map[string]any); err != nil || len(m["performed"].([]OwedRow)) != 1 || len(m["backlog"].([]groups.Backlog)) != 1 {
|
||||
t.Errorf("bridge_owed: %v %v", out, err)
|
||||
// bridge_owed reads one act or one pull request's acts, never the whole state (H-A-181).
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
args map[string]any
|
||||
ids []string
|
||||
}{
|
||||
{"one pull request", map[string]any{"repository": "novox/mesh-sdk", "number": float64(179)}, []string{"b"}},
|
||||
{"one act", map[string]any{"of": seat.Seat, "id": "a"}, []string{"a"}},
|
||||
{"the whole state", map[string]any{"of": seat.Seat}, nil},
|
||||
{"a repository with no number", map[string]any{"repository": "novox/mesh-sdk"}, nil},
|
||||
} {
|
||||
n := len(filters)
|
||||
out, err := tools[OwedTool](c.args)
|
||||
if c.ids == nil {
|
||||
if err == nil || len(filters) != n {
|
||||
t.Errorf("bridge_owed of %s: %v %v, read %d; want refused, nothing read", c.name, out, err, len(filters)-n)
|
||||
}
|
||||
continue
|
||||
}
|
||||
m, _ := out.(map[string]any)
|
||||
got, _ := m["performed"].([]OwedRow)
|
||||
var ids []string
|
||||
for _, r := range got {
|
||||
ids = append(ids, r.ID)
|
||||
}
|
||||
if err != nil || !reflect.DeepEqual(ids, c.ids) || len(m["backlog"].([]groups.Backlog)) != 1 {
|
||||
t.Errorf("bridge_owed of %s: %v %v, want %v", c.name, out, err, c.ids)
|
||||
}
|
||||
}
|
||||
if out, err := tools[OwedAgainTool](map[string]any{"of": seat.Seat, "id": "a"}); err != nil || out.(map[string]any)["outcome"] != forgemsg.OwedPerformed ||
|
||||
asked[len(asked)-1].Kind != groups.OwedAgain {
|
||||
|
||||
@@ -21,26 +21,50 @@ const (
|
||||
OwedDropTool = "bridge_owed_drop"
|
||||
)
|
||||
|
||||
// OwedRow is one act the bridge performed, as the state performed keeps it: the outcome said as owed.ended.
|
||||
type OwedRow = forgemsg.OwedEnded
|
||||
// OwedRow is one act the bridge performed, as the state performed keeps it: the outcome said as owed.ended, and the pull
|
||||
// request the act names (groups.Performed).
|
||||
type OwedRow struct {
|
||||
forgemsg.OwedEnded
|
||||
Repository string `json:"repository,omitempty"`
|
||||
Number int `json:"number,omitempty"`
|
||||
}
|
||||
|
||||
// performedRows reads every act the state performed holds; a key that is not an act (a group end's bookkeeping) is left.
|
||||
func performedRows(s stdio.KeptState) ([]OwedRow, error) {
|
||||
keys, err := s.Keys()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the bridge's performed state cannot be listed: %w", err)
|
||||
// OwedFilter is what bridge_owed reads of performed: one act (its id, of narrowing it to one key) or one pull request's
|
||||
// acts (repository and number), never the whole state (novox/hq design 47 H-A-181).
|
||||
type OwedFilter struct {
|
||||
Of, ID, Repository string
|
||||
Number int
|
||||
}
|
||||
|
||||
func (f OwedFilter) holds(r OwedRow) bool {
|
||||
return (f.Of == "" || r.Of == f.Of) && (f.ID == "" || r.ID == f.ID) &&
|
||||
(f.Repository == "" || (r.Repository == f.Repository && r.Number == f.Number))
|
||||
}
|
||||
|
||||
// performedRows reads the acts of performed f names, newest first: one key when f names the seat and the id, else each
|
||||
// key's value held to f. A key that is not an act (a group end's bookkeeping) is left.
|
||||
func performedRows(s stdio.KeptState, f OwedFilter) ([]OwedRow, error) {
|
||||
var keys []string
|
||||
if f.Of != "" && f.ID != "" {
|
||||
keys = []string{groups.PerformedKey(f.Of, f.ID)}
|
||||
} else {
|
||||
all, err := s.Keys()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the bridge's performed state cannot be listed: %w", err)
|
||||
}
|
||||
keys = all
|
||||
}
|
||||
var rows []OwedRow
|
||||
rows := []OwedRow{}
|
||||
for _, k := range keys {
|
||||
e, err := s.Get(k)
|
||||
if err != nil || e == nil {
|
||||
continue
|
||||
}
|
||||
var p struct {
|
||||
Ended OwedRow `json:"ended"`
|
||||
}
|
||||
var p groups.Performed
|
||||
if json.Unmarshal(e.Value, &p) == nil && p.Ended.Of != "" {
|
||||
rows = append(rows, p.Ended)
|
||||
if r := (OwedRow{OwedEnded: p.Ended, Repository: p.Repository, Number: p.Number}); f.holds(r) {
|
||||
rows = append(rows, r)
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Slice(rows, func(i, j int) bool { return rows[i].At.After(rows[j].At) })
|
||||
@@ -50,7 +74,7 @@ func performedRows(s stdio.KeptState) ([]OwedRow, error) {
|
||||
// OwedTools are bridge_owed, bridge_owed_again and bridge_owed_drop: ask writes an owed-asks value and waits for the
|
||||
// bridge's answer; performed reads the state performed; record writes a hand act in the seat mesh-hand-acts' log before
|
||||
// a drop is asked, so a drop is never made without its record.
|
||||
func OwedTools(ask func(key string, value []byte) ([]byte, error), performed func() ([]OwedRow, error), record func(what, why string) error, now func() time.Time) []stdio.Tool {
|
||||
func OwedTools(ask func(key string, value []byte) ([]byte, error), performed func(OwedFilter) ([]OwedRow, error), record func(what, why string) error, now func() time.Time) []stdio.Tool {
|
||||
str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) }
|
||||
act := map[string]any{
|
||||
"of": map[string]any{"type": "string", "description": "the seat that said the act: mesh-delivery (the gate)"},
|
||||
@@ -66,6 +90,12 @@ func OwedTools(ask func(key string, value []byte) ([]byte, error), performed fun
|
||||
}
|
||||
return groups.DecodeOwedAsk(v)
|
||||
}
|
||||
read := map[string]any{
|
||||
"repository": map[string]any{"type": "string", "description": "the pull request's repository, as owner/name; with number"},
|
||||
"number": map[string]any{"type": "integer", "description": "the pull request's number; with repository"}}
|
||||
for k, v := range act {
|
||||
read[k] = v
|
||||
}
|
||||
withWhy := map[string]any{"why": map[string]any{"type": "string", "description": "why it is dropped, in plain words"}}
|
||||
for k, v := range act {
|
||||
withWhy[k] = v
|
||||
@@ -74,22 +104,24 @@ func OwedTools(ask func(key string, value []byte) ([]byte, error), performed fun
|
||||
{Name: OwedTool,
|
||||
Description: "What the forge bridge performed of what the gate and the checker owe the forge (design 47): each act it said as " +
|
||||
"owed.ended, newest first, with its seat, id, pull request, outcome (performed, refused, could not be performed, " +
|
||||
"undecodable, dropped), the forge's words and when, from its state performed (kept 7 days); and the backlog of its " +
|
||||
"undecodable, dropped), the forge's words and when, from its state performed (kept 7 days), of one act (id, of " +
|
||||
"narrowing it) or of one pull request (repository and number), never the whole state; and the backlog of its " +
|
||||
"durable on each seat's stream owed: the last event applied, the events waiting and held, the age of the oldest. " +
|
||||
"of or id narrow it. Read-only.",
|
||||
Input: map[string]any{"type": "object", "properties": act},
|
||||
"Read-only.",
|
||||
Input: map[string]any{"type": "object", "properties": read},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
rows, err := performed()
|
||||
f := OwedFilter{Of: str(args, "of"), ID: str(args, "id"), Repository: str(args, "repository")}
|
||||
if n, ok := args["number"].(float64); ok {
|
||||
f.Number = int(n)
|
||||
}
|
||||
if f.ID == "" && (f.Repository == "" || f.Number <= 0) {
|
||||
return nil, fmt.Errorf("bridge_owed reads one act (id) or one pull request's acts (repository and number), never the whole state")
|
||||
}
|
||||
rows, err := performed(f)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
kept := []OwedRow{}
|
||||
for _, r := range rows {
|
||||
if (str(args, "of") == "" || r.Of == str(args, "of")) && (str(args, "id") == "" || r.ID == str(args, "id")) {
|
||||
kept = append(kept, r)
|
||||
}
|
||||
}
|
||||
out := map[string]any{"performed": kept}
|
||||
out := map[string]any{"performed": rows}
|
||||
if a, err := asked(groups.OwedBacklog, args); err != nil {
|
||||
out["backlog-error"] = err.Error()
|
||||
} else {
|
||||
@@ -100,8 +132,8 @@ func OwedTools(ask func(key string, value []byte) ([]byte, error), performed fun
|
||||
{Name: OwedAgainTool,
|
||||
Description: "Perform one act owed to the forge again now, by hand, whatever its last outcome (a refusal the forge no longer " +
|
||||
"makes, an act that could not be performed): the bridge calls the seat git's verb of the act under its id (safe to " +
|
||||
"repeat) and says its outcome as owed.ended. The act is the one the bridge kept, or, for the gate's, the seat " +
|
||||
"mesh-delivery's groups. Answers the outcome.",
|
||||
"repeat) and says its outcome as owed.ended. The act is read from the stream owed of the seat that said it, on the " +
|
||||
"subject of the pull request the bridge's state performed names (kept 7 days there). Answers the outcome.",
|
||||
Input: map[string]any{"type": "object", "required": []string{"of", "id"}, "properties": act},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
a, err := asked(groups.OwedAgain, args)
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strconv"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
)
|
||||
|
||||
// The act bridge_owed_again performs again is read from the stream owed of the seat that said it, on the subject of the
|
||||
// pull request performed names, the stream being the act's one copy (novox/hq design 47 G §162, §163, H-A-181).
|
||||
|
||||
// actReader reads the act the seat of said under id from its stream owed, on the subject of the pull request
|
||||
// (repository and number) the act names, for decode to say each message's id; found false when the stream no longer
|
||||
// holds it.
|
||||
type actReader interface {
|
||||
act(ctx context.Context, of, repository string, number int, id string, idOf func([]byte) string) (data []byte, found bool, err error)
|
||||
}
|
||||
|
||||
// actOf is the act of to perform again, read from its seat's stream owed on the pull request p names.
|
||||
func (o *owing) actOf(ctx context.Context, of string, p Performed, id string) ([]byte, error) {
|
||||
read, err := readerOf(of)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// A value of the form before H-A-182 names no pull request: every act the bridge decodes names its repository (a
|
||||
// status may name no number, said on `.0` as the gate says it), so one with none is the old form, never read again.
|
||||
if p.Repository == "" {
|
||||
return nil, fmt.Errorf("performed names no pull request for the act %s of %s: kept in its old form, it expires with the state's TTL", id, of)
|
||||
}
|
||||
if o.acts == nil {
|
||||
return nil, fmt.Errorf("the bridge reads no stream owed, so the act %s of %s cannot be performed again", id, of)
|
||||
}
|
||||
data, found, err := o.acts.act(ctx, of, p.Repository, p.Number, id, func(m []byte) string { c, _ := read(m); return c.ID })
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the act %s of %s cannot be read from %s's stream owed: %w", id, of, of, err)
|
||||
}
|
||||
if !found {
|
||||
return nil, fmt.Errorf("%s's stream owed no longer holds the act %s of %s#%d (kept 7 days)", of, id, p.Repository, p.Number)
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
// streamActs reads an act from the gate's stream owed (seat mesh-delivery's StreamOwed) by an ordered consumer filtered
|
||||
// on the pull request's subject, as the gate says it (SubjectOwed with the repository escaped): every message the
|
||||
// stream keeps there, the newest of the id kept.
|
||||
type streamActs struct{ js jetstream.JetStream }
|
||||
|
||||
func (s streamActs) act(ctx context.Context, of, repository string, number int, id string, idOf func([]byte) string) ([]byte, bool, error) {
|
||||
if of != seat.Seat {
|
||||
return nil, false, fmt.Errorf("the seat %q keeps no stream owed the bridge reads", of)
|
||||
}
|
||||
subject := seat.SubjectOwed(keys.EscapeName(repository), strconv.Itoa(number))
|
||||
st, err := s.js.Stream(ctx, seat.StreamOwed)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
info, err := st.Info(ctx, jetstream.WithSubjectFilter(subject))
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
held := info.State.Subjects[subject]
|
||||
if held == 0 {
|
||||
return nil, false, nil
|
||||
}
|
||||
c, err := st.OrderedConsumer(ctx, jetstream.OrderedConsumerConfig{FilterSubjects: []string{subject}})
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
var found []byte
|
||||
for range held {
|
||||
m, err := c.Next(jetstream.FetchContext(ctx))
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if idOf(m.Data()) == id {
|
||||
found = m.Data()
|
||||
}
|
||||
if md, err := m.Metadata(); err == nil && md.NumPending == 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
return found, found != nil, nil
|
||||
}
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -38,6 +39,24 @@ func (m memPerformed) Get(_ context.Context, key string) ([]byte, bool, error) {
|
||||
}
|
||||
func (m memPerformed) Put(_ context.Context, key string, v []byte) error { m[key] = v; return nil }
|
||||
|
||||
// memActs is a seat's stream owed by pull request subject (`<repository>#<number>`); asked records what was read.
|
||||
type memActs struct {
|
||||
held map[string][][]byte
|
||||
asked []string
|
||||
}
|
||||
|
||||
func (m *memActs) act(_ context.Context, of, repository string, number int, id string, idOf func([]byte) string) ([]byte, bool, error) {
|
||||
pr := fmt.Sprintf("%s %s#%d", of, repository, number)
|
||||
m.asked = append(m.asked, pr)
|
||||
var found []byte
|
||||
for _, d := range m.held[pr] {
|
||||
if idOf(d) == id {
|
||||
found = d
|
||||
}
|
||||
}
|
||||
return found, found != nil, nil
|
||||
}
|
||||
|
||||
// seatFake answers each seat verb from answers, by its key; one absent is not answered (an error, as a bound passed).
|
||||
type seatFake struct {
|
||||
calls []string
|
||||
@@ -109,9 +128,12 @@ func TestWhatIsOwedIsPerformedSaidAndKept(t *testing.T) {
|
||||
if v, ok := kept[performedKey(c.of, e.ID)]; ok != c.kept {
|
||||
t.Errorf("%s: kept %v (%s), want %v", c.name, ok, v, c.kept)
|
||||
} else if ok {
|
||||
// The outcome and when, and the pull request the act names, nothing more (§163, H-A-181, H-A-182).
|
||||
var p Performed
|
||||
if err := json.Unmarshal(v, &p); err != nil || p.Ended.Outcome != c.outcome || !json.Valid(p.Owed) {
|
||||
t.Errorf("%s: kept %s, want the outcome and the owed event", c.name, v)
|
||||
var fields map[string]json.RawMessage
|
||||
if err := json.Unmarshal(v, &p); err != nil || p.Ended.Outcome != c.outcome || p.Ended.At.IsZero() ||
|
||||
p.Repository != "novox/mesh-sdk" || p.Number != 178 || json.Unmarshal(v, &fields) != nil || len(fields) != 3 {
|
||||
t.Errorf("%s: kept %s, want the outcome, when, and the pull request novox/mesh-sdk#178, nothing more", c.name, v)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -142,6 +164,9 @@ func TestWhatIsOwedIsPerformedSaidAndKept(t *testing.T) {
|
||||
t.Run("asked again, dropped, and its backlog read by hand", func(t *testing.T) {
|
||||
f := &seatFake{answers: statusOK}
|
||||
o, pub, kept := newOwing(f)
|
||||
// The act is read again from the gate's stream owed on its pull request's subject, never from performed (H-A-181).
|
||||
acts := &memActs{held: map[string][][]byte{"mesh-delivery novox/mesh-sdk#178": {[]byte(`{`), gate("owed is said, a status")}}}
|
||||
o.acts = acts
|
||||
status := "status.novox/mesh-sdk#178.aaaaaaaaaaaa.mesh/merge-gate"
|
||||
if _, err := o.heard(context.Background(), seat.Seat, gate("owed is said, a status")); err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -153,11 +178,20 @@ func TestWhatIsOwedIsPerformedSaidAndKept(t *testing.T) {
|
||||
calls, said int
|
||||
outcome, answers string
|
||||
}{
|
||||
{"again, from performed", groups.OwedAsk{Kind: groups.OwedAgain, Of: seat.Seat, ID: status}, 2, 2, forgemsg.OwedPerformed, "performed"},
|
||||
{"again, from the gate's stream owed", groups.OwedAsk{Kind: groups.OwedAgain, Of: seat.Seat, ID: status}, 2, 2, forgemsg.OwedPerformed, "performed"},
|
||||
{"dropped", groups.OwedAsk{Kind: groups.OwedDrop, Of: seat.Seat, ID: status, Why: "set by hand", By: "lead@shanks"}, 2, 3, forgemsg.OwedDropped, "set by hand"},
|
||||
{"again, an act nobody holds", groups.OwedAsk{Kind: groups.OwedAgain, Of: seat.Seat, ID: "x"}, 2, 3, "", "holds no owed act"},
|
||||
{"the backlog", groups.OwedAsk{Kind: groups.OwedBacklog}, 2, 3, "", "backlog"},
|
||||
{"again, the stream no longer holding it", groups.OwedAsk{Kind: groups.OwedAgain, Of: seat.Seat, ID: status}, 2, 3, "", "no longer holds"},
|
||||
{"again, a value of the old form", groups.OwedAsk{Kind: groups.OwedAgain, Of: seat.Seat, ID: "old"}, 2, 3, "", "names no pull request"},
|
||||
} {
|
||||
switch c.name {
|
||||
case "again, the stream no longer holding it":
|
||||
acts.held = nil
|
||||
case "again, a value of the old form":
|
||||
// Kept before H-A-182: the outcome and the whole act, no pull request; the stream is not read for it.
|
||||
kept[performedKey(seat.Seat, "old")] = []byte(`{"ended":{"v":1,"of":"mesh-delivery","id":"old","outcome":"performed","at":"2026-11-09T21:00:00Z"},"owed":{"id":"old"}}`)
|
||||
}
|
||||
a := o.answer(context.Background(), c.ask, backlog)
|
||||
if !a.Done() || a.Outcome != c.outcome || !strings.Contains(a.Answer, c.answers) || len(f.calls) != c.calls || len(pub.data) != c.said {
|
||||
t.Errorf("%s: %+v, called %v, said %d; want %q naming %q, %d calls, %d said", c.name, a, f.calls, len(pub.data), c.outcome, c.answers, c.calls, c.said)
|
||||
@@ -167,8 +201,12 @@ func TestWhatIsOwedIsPerformedSaidAndKept(t *testing.T) {
|
||||
}
|
||||
}
|
||||
var p Performed
|
||||
if err := json.Unmarshal(kept[performedKey(seat.Seat, status)], &p); err != nil || p.Ended.Outcome != forgemsg.OwedDropped {
|
||||
t.Errorf("performed after the drop: %+v %v", p, err)
|
||||
if err := json.Unmarshal(kept[performedKey(seat.Seat, status)], &p); err != nil || p.Ended.Outcome != forgemsg.OwedDropped ||
|
||||
p.Repository != "novox/mesh-sdk" || p.Number != 178 {
|
||||
t.Errorf("performed after the drop: %+v %v; want dropped, its pull request kept", p, err)
|
||||
}
|
||||
if len(acts.asked) != 2 || acts.asked[0] != "mesh-delivery novox/mesh-sdk#178" {
|
||||
t.Errorf("the stream owed was read on %v, want twice on the pull request performed names", acts.asked)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
+22
-3
@@ -22,6 +22,7 @@ import (
|
||||
git "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git"
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"mesh-delivery-forge-bridge/internal/gitfixture"
|
||||
@@ -33,7 +34,8 @@ import (
|
||||
// no others, so a permission the bridge lacks fails a test here and not on the mesh (issue 1279).
|
||||
//
|
||||
// From module.json: the registrations and the gate's source-moved heard through its durable on EVENTS; branch-moved and
|
||||
// what the gate owes heard through its durable on each seat's own stream (SEAT_<seat>_<event>); the seat git's verbs, the
|
||||
// what the gate owes heard through its durable on each seat's own stream (SEAT_<seat>_<event>), and each seat's kept
|
||||
// event also read through an ordered consumer of its own (mesh-grants heardGrants, A-233; bridge_owed_again, H-A-181); the seat git's verbs, the
|
||||
// seat mesh-catalog's registrations, the seat mesh-delivery's source, the seat mesh-broker's show and the seat
|
||||
// mesh-hand-acts' hand-act called on mesh.seat.<seat>.call.<verb>.<token> (or its plain fallback); its own states heads
|
||||
// and owed-asks, and performed under the name its binding gives; its stream forge. `**` is written `>` on the wire. The
|
||||
@@ -82,6 +84,11 @@ accounts {
|
||||
"$JS.ACK.SEAT_mesh-delivery_owed.n1_mesh-delivery-forge-bridge.>"
|
||||
"$JS.API.CONSUMER.INFO.SEAT_mesh-delivery_owed.n1_mesh-delivery-forge-bridge"
|
||||
"$JS.API.CONSUMER.MSG.NEXT.SEAT_mesh-delivery_owed.n1_mesh-delivery-forge-bridge"
|
||||
"$JS.API.CONSUMER.CREATE.SEAT_mesh-delivery_owed"
|
||||
"$JS.API.CONSUMER.CREATE.SEAT_mesh-delivery_owed.>"
|
||||
"$JS.API.CONSUMER.MSG.NEXT.SEAT_mesh-delivery_owed.*"
|
||||
"$JS.API.CONSUMER.INFO.SEAT_mesh-delivery_owed.*"
|
||||
"$JS.API.CONSUMER.DELETE.SEAT_mesh-delivery_owed.*"
|
||||
"mesh.seat.mesh-broker.call.show.n1~mesh-delivery-forge-bridge"
|
||||
"mesh.seat.mesh-broker.tool.show"
|
||||
"mesh.seat.mesh-catalog.call.registrations.n1~mesh-delivery-forge-bridge"
|
||||
@@ -518,7 +525,9 @@ func TestBridgePerformsWhatIsOwedOnce(t *testing.T) {
|
||||
}
|
||||
}
|
||||
gateOwed := gitfixture.Seat(t, seat.Seat, "events", "owed is said, a status", "event:owed")[0]
|
||||
subject := seat.SubjectOwed("novox~mesh-sdk", "178")
|
||||
// On the pull request's subject as the gate says it (its repository escaped as a key part), where bridge_owed_again
|
||||
// reads the act again (H-A-181).
|
||||
subject := seat.SubjectOwed(keys.EscapeName("novox/mesh-sdk"), "178")
|
||||
for range 2 {
|
||||
if _, err := b.js.Publish(b.ctx, subject, gateOwed); err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -597,12 +606,22 @@ func TestBridgePerformsWhatIsOwedOnce(t *testing.T) {
|
||||
if a, err := groupkeys.DecodeOwedAsk(e.Value()); err == nil && a.Done() {
|
||||
answered = true
|
||||
if a.Outcome != forgemsg.OwedPerformed || count("git.status") != 3 {
|
||||
t.Errorf("bridge_owed_again: %+v, status asked %d times; want performed again", a, count("git.status"))
|
||||
t.Errorf("bridge_owed_again: %+v, status asked %d times; want performed again, read from the gate's stream owed", a, count("git.status"))
|
||||
}
|
||||
}
|
||||
case <-bound:
|
||||
t.Fatal("bridge_owed_again was not answered within 20 s")
|
||||
}
|
||||
}
|
||||
// performed keeps the outcome, when and the pull request, never the act (§163, H-A-181, H-A-182).
|
||||
kept, err := b.js.KeyValue(b.ctx, boundPerformed)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if e, err := kept.Get(b.ctx, performedKey(seat.Seat, g.ID)); err != nil {
|
||||
t.Errorf("performed of %s: %v", g.ID, err)
|
||||
} else if fields := (map[string]json.RawMessage{}); json.Unmarshal(e.Value(), &fields) != nil || len(fields) != 3 || fields["owed"] != nil {
|
||||
t.Errorf("performed of %s keeps %s, want the outcome, when and the pull request alone", g.ID, e.Value())
|
||||
}
|
||||
refusedRead(t, b, "mesh-delivery-gate_owed")
|
||||
}
|
||||
|
||||
@@ -108,7 +108,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
}
|
||||
performedKV := &boundBucket{binding: binding, local: performedLocal, js: js}
|
||||
// What the gate owes the forge, performed through the seat git (design 47 *What the gate owes the forge*).
|
||||
owe := &owing{call: call, pub: js, performed: kvPerformed{performedKV}, waits: binding.Waiting, now: time.Now, logger: logger}
|
||||
owe := &owing{call: call, pub: js, performed: kvPerformed{performedKV}, acts: streamActs{js: js}, waits: binding.Waiting, now: time.Now, logger: logger}
|
||||
keptCall := keptCallOf(ctx, call)
|
||||
reader := &views.ProcessReader{Module: forgeBridgeModule(), Call: keptCall, Logf: logger.Printf, Hub: hub}
|
||||
src := sourcebranches.New(sourcebranches.Call(keptCall))
|
||||
|
||||
@@ -2,7 +2,6 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
@@ -22,22 +21,14 @@ import (
|
||||
)
|
||||
|
||||
// The state performed (manifest `requires`, the template state, opened by its binding: bound.go) keeps one key per owed act it
|
||||
// performed, the act's outcome and when, with the owed event as heard (novox/hq design 47 *What the gate owes the
|
||||
// forge*). It is a saving and a read, never what keeps an act from happening twice: every act is safe to repeat at the
|
||||
// forge.
|
||||
// performed, in the form groups.Performed says (novox/hq design 47 G §163, H-A-181, H-A-182). It is a saving and a read,
|
||||
// never what keeps an act from happening twice: every act is safe to repeat at the forge.
|
||||
|
||||
// performedKey is the key of one act in performed: the seat that said it and its id, the id in unpadded base64url, since
|
||||
// an id may hold what a key may not (`#`, `/`).
|
||||
func performedKey(of, id string) string {
|
||||
return of + "." + base64.RawURLEncoding.EncodeToString([]byte(id))
|
||||
}
|
||||
// performedKey is the key of one act in performed (groups.PerformedKey).
|
||||
func performedKey(of, id string) string { return groups.PerformedKey(of, id) }
|
||||
|
||||
// Performed is one value of the state performed: the outcome said as owed.ended, and the owed event it answers, so
|
||||
// bridge_owed_again performs it again from here.
|
||||
type Performed struct {
|
||||
Ended forgemsg.OwedEnded `json:"ended"`
|
||||
Owed json.RawMessage `json:"owed,omitempty"`
|
||||
}
|
||||
// Performed is one value of the state performed.
|
||||
type Performed = groups.Performed
|
||||
|
||||
// performedStore is the state performed as the owing reads and writes it.
|
||||
type performedStore interface {
|
||||
@@ -70,6 +61,8 @@ type owing struct {
|
||||
call caller
|
||||
pub publisher
|
||||
performed performedStore
|
||||
// acts reads an act again from the stream owed of the seat that said it (H-A-181), for bridge_owed_again.
|
||||
acts actReader
|
||||
// waits, when set, says why the state performed cannot be used yet (its binding, design 47 G §577); empty when it can.
|
||||
waits func() string
|
||||
now func() time.Time
|
||||
@@ -90,12 +83,12 @@ func readerOf(of string) (func([]byte) (bridge.OwedCall, error), error) {
|
||||
func (o *owing) heard(ctx context.Context, of string, data []byte) (forgemsg.OwedEnded, error) {
|
||||
read, err := readerOf(of)
|
||||
if err != nil {
|
||||
return o.end(ctx, forgemsg.OwedEnded{Of: of, Outcome: forgemsg.OwedUndecodable, Words: err.Error()}, data)
|
||||
return o.end(ctx, forgemsg.OwedEnded{Of: of, Outcome: forgemsg.OwedUndecodable, Words: err.Error()}, "", 0)
|
||||
}
|
||||
c, err := read(data)
|
||||
if err != nil {
|
||||
return o.end(ctx, forgemsg.OwedEnded{Of: of, ID: c.ID,
|
||||
Outcome: forgemsg.OwedUndecodable, Words: err.Error()}, data)
|
||||
Outcome: forgemsg.OwedUndecodable, Words: err.Error()}, c.Repository, c.Number)
|
||||
}
|
||||
if w := o.waiting(); w != "" {
|
||||
return forgemsg.OwedEnded{Of: c.Of, ID: c.ID}, fmt.Errorf("the bridge %s", w)
|
||||
@@ -104,7 +97,7 @@ func (o *owing) heard(ctx context.Context, of string, data []byte) (forgemsg.Owe
|
||||
// said twice, or said again while its outcome was on its way: performed once, the outcome said once
|
||||
return forgemsg.OwedEnded{Of: c.Of, ID: c.ID}, nil
|
||||
}
|
||||
return o.perform(ctx, c, data)
|
||||
return o.perform(ctx, c)
|
||||
}
|
||||
|
||||
// hear is one message of a seat's stream owed: performed and acknowledged, or, when its outcome could not be said,
|
||||
@@ -127,19 +120,20 @@ func (o *owing) waiting() string {
|
||||
}
|
||||
|
||||
// perform calls the seat git's verb of one act and says what it came to.
|
||||
func (o *owing) perform(ctx context.Context, c bridge.OwedCall, data []byte) (forgemsg.OwedEnded, error) {
|
||||
func (o *owing) perform(ctx context.Context, c bridge.OwedCall) (forgemsg.OwedEnded, error) {
|
||||
answer, callErr := o.call(ctx, "seat:"+git.Seat+"."+c.Verb, c.Input)
|
||||
outcome, words := bridge.Outcome(answer, callErr)
|
||||
if outcome != forgemsg.OwedPerformed {
|
||||
o.logger.Printf("the owed %s %s of %s#%d (said by %s) is %s: %s", c.Act, c.ID, c.Repository, c.Number, c.Of, outcome, words)
|
||||
}
|
||||
return o.end(ctx, forgemsg.OwedEnded{Of: c.Of, ID: c.ID,
|
||||
Outcome: outcome, Words: words}, data)
|
||||
Outcome: outcome, Words: words}, c.Repository, c.Number)
|
||||
}
|
||||
|
||||
// end says an act's outcome as owed.ended on the stream forge, then keeps it in performed. An outcome not kept is
|
||||
// logged: the act may be performed again, which changes nothing at the forge.
|
||||
func (o *owing) end(ctx context.Context, e forgemsg.OwedEnded, owed []byte) (forgemsg.OwedEnded, error) {
|
||||
// end says an act's outcome as owed.ended on the stream forge, then keeps it in performed with the pull request the act
|
||||
// names (repository and number). An outcome not kept is logged: the act may be performed again, which changes nothing at
|
||||
// the forge.
|
||||
func (o *owing) end(ctx context.Context, e forgemsg.OwedEnded, repository string, number int) (forgemsg.OwedEnded, error) {
|
||||
e.At = o.now().UTC()
|
||||
data, err := forgemsg.EncodeOwedEnded(e)
|
||||
if err != nil {
|
||||
@@ -151,10 +145,7 @@ func (o *owing) end(ctx context.Context, e forgemsg.OwedEnded, owed []byte) (for
|
||||
if e.ID == "" {
|
||||
return e, nil // an undecodable act with no id is said, and has no key to keep
|
||||
}
|
||||
if !json.Valid(owed) {
|
||||
owed = nil
|
||||
}
|
||||
v, err := json.Marshal(Performed{Ended: e, Owed: owed})
|
||||
v, err := json.Marshal(Performed{Ended: e, Repository: repository, Number: number})
|
||||
if err == nil {
|
||||
err = o.performed.Put(ctx, performedKey(e.Of, e.ID), v)
|
||||
}
|
||||
@@ -175,19 +166,17 @@ func (o *owing) held(ctx context.Context, of, id string) bool {
|
||||
return found
|
||||
}
|
||||
|
||||
// owedOf is the owed event of an act to perform again, as performed keeps it.
|
||||
func (o *owing) owedOf(ctx context.Context, of, id string) ([]byte, error) {
|
||||
// kept is what performed holds of an act: its outcome and its pull request.
|
||||
func (o *owing) kept(ctx context.Context, of, id string) (Performed, error) {
|
||||
v, found, err := o.performed.Get(ctx, performedKey(of, id))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("performed cannot be read: %w", err)
|
||||
return Performed{}, fmt.Errorf("performed cannot be read: %w", err)
|
||||
}
|
||||
if found {
|
||||
var p Performed
|
||||
if err := json.Unmarshal(v, &p); err == nil && len(p.Owed) > 0 {
|
||||
return p.Owed, nil
|
||||
}
|
||||
var p Performed
|
||||
if !found || json.Unmarshal(v, &p) != nil {
|
||||
return Performed{}, fmt.Errorf("the bridge holds no owed act %s of %s in its state performed", id, of)
|
||||
}
|
||||
return nil, fmt.Errorf("the bridge holds no owed act %s of %s in its state performed", id, of)
|
||||
return p, nil
|
||||
}
|
||||
|
||||
// answer answers one owed ask: an act performed again now, whatever performed holds of it; an act dropped, said as
|
||||
@@ -199,20 +188,22 @@ func (o *owing) answer(ctx context.Context, a groups.OwedAsk, backlog func(conte
|
||||
a.Backlog, a.Answer = backlog(ctx), "the backlog of each seat's stream owed"
|
||||
return a
|
||||
}
|
||||
data, err := o.owedOf(ctx, a.Of, a.ID)
|
||||
p, err := o.kept(ctx, a.Of, a.ID)
|
||||
if err != nil {
|
||||
a.Answer = err.Error()
|
||||
return a
|
||||
}
|
||||
var e forgemsg.OwedEnded
|
||||
if a.Kind == groups.OwedDrop {
|
||||
e, err = o.end(ctx, forgemsg.OwedEnded{Of: a.Of, ID: a.ID, Outcome: forgemsg.OwedDropped, Words: a.Why + " (dropped by " + a.By + ")"}, data)
|
||||
e, err = o.end(ctx, forgemsg.OwedEnded{Of: a.Of, ID: a.ID, Outcome: forgemsg.OwedDropped, Words: a.Why + " (dropped by " + a.By + ")"}, p.Repository, p.Number)
|
||||
} else if read, rerr := readerOf(a.Of); rerr != nil {
|
||||
err = rerr
|
||||
} else if data, rerr := o.actOf(ctx, a.Of, p, a.ID); rerr != nil {
|
||||
err = rerr
|
||||
} else if c, rerr := read(data); rerr != nil {
|
||||
e, err = o.end(ctx, forgemsg.OwedEnded{Of: a.Of, ID: a.ID, Outcome: forgemsg.OwedUndecodable, Words: rerr.Error()}, data)
|
||||
e, err = o.end(ctx, forgemsg.OwedEnded{Of: a.Of, ID: a.ID, Outcome: forgemsg.OwedUndecodable, Words: rerr.Error()}, p.Repository, p.Number)
|
||||
} else {
|
||||
e, err = o.perform(ctx, c, data)
|
||||
e, err = o.perform(ctx, c)
|
||||
}
|
||||
if err != nil {
|
||||
a.Answer = err.Error()
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
package groups
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
)
|
||||
|
||||
// The state performed keeps one key per owed act the bridge performed, its outcome and when, and the pull request the
|
||||
// act names, nothing more (novox/hq design 47 G §163, H-A-181, H-A-182): the act itself stays in the stream owed of the
|
||||
// seat that said it, its one copy, where bridge_owed_again reads it again. The bridge writes it; its tools read it.
|
||||
|
||||
// PerformedKey is the key of one act in performed: the seat that said it and its id, the id in unpadded base64url, since
|
||||
// an id may hold what a key may not (`#`, `/`).
|
||||
func PerformedKey(of, id string) string {
|
||||
return of + "." + base64.RawURLEncoding.EncodeToString([]byte(id))
|
||||
}
|
||||
|
||||
// Performed is one value of the state performed (§163, H-A-182): the outcome said as owed.ended, with when, and the
|
||||
// repository and number of the pull request the act names, so bridge_owed reads one pull request's acts and
|
||||
// bridge_owed_again finds the act on its pull request's subject of the stream owed (H-A-181).
|
||||
type Performed struct {
|
||||
Ended forgemsg.OwedEnded `json:"ended"`
|
||||
Repository string `json:"repository,omitempty"`
|
||||
Number int `json:"number,omitempty"`
|
||||
}
|
||||
@@ -187,10 +187,13 @@ func TestManifestProcess(t *testing.T) { // 1235/AC1
|
||||
if _, bus := p["bus"]; bus {
|
||||
t.Errorf("the process declares a bus: it runs on its own broker credential")
|
||||
}
|
||||
// Issue 1966's bootstrap: the package artifact forgemsg is out of the manifest until the new builder runs; the follow-up
|
||||
// puts it back with its assertion (ADR 0573 §1: three artifacts, the third the package forgemsg).
|
||||
if len(m.Build.Artifacts) != 2 || m.Build.Artifacts[1].Name != "tools" || m.Build.Artifacts[1].From != "cmd/mesh-delivery-forge-bridge-tools" {
|
||||
t.Fatalf("artifacts = %+v, want the program and the tools bundle", m.Build.Artifacts)
|
||||
if len(m.Build.Artifacts) != 3 || m.Build.Artifacts[1].Name != "tools" || m.Build.Artifacts[1].From != "cmd/mesh-delivery-forge-bridge-tools" {
|
||||
t.Fatalf("artifacts = %+v, want the program, the tools bundle and the package forgemsg", m.Build.Artifacts)
|
||||
}
|
||||
// ADR 0573 §1, H-A-186 (design 47 G §370): the gate, the builder and the indexer import forgemsg, so the module declares
|
||||
// it as a package and every release publishes it.
|
||||
if a := m.Build.Artifacts[2]; a.Name != "forgemsg" || a.Kind != "package" || a.Language != "go" || len(a.Paths) != 1 || a.Paths[0] != "forgemsg" || a.Series != "0.1" {
|
||||
t.Errorf("package artifact = %+v", a)
|
||||
}
|
||||
a := m.Build.Artifacts[0]
|
||||
if st, err := os.Stat(a.From); err != nil || !st.IsDir() || a.Binary != "mesh-delivery-forge-bridge" || !strings.HasSuffix(a.From, a.Binary) {
|
||||
|
||||
@@ -101,6 +101,15 @@
|
||||
"loads": [
|
||||
"mesh-delivery-forge-bridge-tools"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "forgemsg",
|
||||
"kind": "package",
|
||||
"language": "go",
|
||||
"paths": [
|
||||
"forgemsg"
|
||||
],
|
||||
"series": "0.1"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -31,7 +31,6 @@ import (
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
@@ -303,8 +302,9 @@ func startIndexerRun(t *testing.T, put func(context.Context, buildLog), before .
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
t.Cleanup(cancel)
|
||||
// The gate's source-moved on its kept stream, with the indexer's durable of each watched branch the beds register
|
||||
// (watchBranch), and the streams of the seat events the indexer consumes on durables of its own: its registration-changed
|
||||
// and the runner's run-started, pipeline-started and pipeline-ended (design 47 H §743, §749), as the grants part makes them.
|
||||
// (watchBranch), and the streams of the seat events the indexer consumes on durables of its own: its registration-changed,
|
||||
// the runner's run-started, pipeline-started and pipeline-ended (design 47 H §743, §749) and the seat mesh-wants' pipeline-wanted
|
||||
// (H-A-170), as the grants part makes them.
|
||||
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: meshdelivery.StreamSourceMoved, Subjects: []string{meshdelivery.SubjectSourceMovedAll}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -313,7 +313,7 @@ func startIndexerRun(t *testing.T, put func(context.Context, buildLog), before .
|
||||
}
|
||||
for stream, subject := range map[string]string{meshcatalog.StreamRegistrationChanged: meshcatalog.SubjectRegistrationChanged,
|
||||
meshpipelinerunner.StreamRunStarted: meshpipelinerunner.SubjectRunStartedAll, meshpipelinerunner.StreamPipelineStarted: meshpipelinerunner.SubjectPipelineStartedAll,
|
||||
meshpipelinerunner.StreamPipelineEnded: meshpipelinerunner.SubjectPipelineEndedAll} {
|
||||
meshpipelinerunner.StreamPipelineEnded: meshpipelinerunner.SubjectPipelineEndedAll, meshwants.StreamPipelineWanted: meshwants.SubjectPipelineWantedAll} {
|
||||
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: stream, Subjects: []string{subject}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -646,91 +646,6 @@ func (l buildLog) end(t *testing.T, repo, module, path string) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestIndexerBuildsARunsCandidateAndDropsItWhenEveryMembersPipelineEnded (design 47 G §64, §67 test 64, §190; A-160, A-174):
|
||||
// the indexer starts with the runner's streams run-started and pipeline-ended and its durables on them, its state runs and a
|
||||
// registration of avahi; the runner says run-started of run 9 with avahi at a commit on a branch no registration watches;
|
||||
// within 20 s candidates.run-9 holds one pull of number 0 at that commit, avahi's manifest read through the seat git; the
|
||||
// indexer is restarted, the runner says avahi's pipeline of run 9 ended, and the candidate and the run's key go.
|
||||
func TestIndexerBuildsARunsCandidateAndDropsItWhenEveryMembersPipelineEnded(t *testing.T) {
|
||||
manifest, err := os.ReadFile("../../internal/index/testdata/catalogue-a67f0196-avahi.module.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
const goSum = "3b5028f0c1e9bde04dd766abd63eb5931bc516bae695cf46f00abcd8510b0381"
|
||||
const commit = "0123456789abcdef0123456789abcdef01234567"
|
||||
var runs jetstream.KeyValue
|
||||
js, ctx, candidates, _, restart := startIndexerRun(t, func(context.Context, buildLog) {}, func(ctx context.Context, js jetstream.JetStream) {
|
||||
var err error
|
||||
if runs, err = js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.Bucket(testIndexer, index.RunsState), History: 1}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
regs, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.Bucket(testIndexer, index.RegistrationsState), History: 1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reg, _ := json.Marshal(meshcatalog.Registration{V: 1, Module: "avahi", Origin: "novox/mesh-catalog", Path: "modules/avahi", Branches: []string{"main"}})
|
||||
if _, err := regs.Create(ctx, index.RegistrationKey("avahi"), reg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
seedBuildModules(t, ctx, js, nil)
|
||||
// The seat git as the forge module serves it: avahi's directory and its manifest at the run's commit.
|
||||
reply := func(m *nats.Msg, result any) {
|
||||
raw, _ := json.Marshal(map[string]any{"result": result, "node": "n0"})
|
||||
_ = m.Respond(raw)
|
||||
}
|
||||
if _, err := js.Conn().Subscribe("mesh.seat.git.tool.trees", func(m *nats.Msg) {
|
||||
reply(m, git.TreesAnswer{Root: "r0", Commit: commit, Trees: map[string]string{"modules/avahi": "newtree"}})
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := js.Conn().Subscribe("mesh.seat.git.tool.file", func(m *nats.Msg) {
|
||||
var a git.FileInput
|
||||
_ = json.Unmarshal(m.Data, &a)
|
||||
switch a.Path {
|
||||
case "modules/avahi/module.json":
|
||||
reply(m, git.FileAnswer{Path: a.Path, Ref: a.Ref, Content: string(manifest)})
|
||||
case "modules/avahi/go.sum":
|
||||
reply(m, git.FileAnswer{Path: a.Path, Ref: a.Ref, Content: goSum})
|
||||
default:
|
||||
reply(m, git.FileAnswer{Path: a.Path, Ref: a.Ref, Absent: true})
|
||||
}
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
started, _ := json.Marshal(meshpipelinerunner.RunStarted{V: 1, Run: "9", By: "lead@n1", Members: []meshpipelinerunner.Member{{Module: "avahi", Commit: commit, Branch: "feature/1966-x"}}})
|
||||
if _, err := js.Publish(ctx, meshpipelinerunner.SubjectRunStarted("avahi", "feature-1966-x", commit), started); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key, _ := meshcatalogclient.CandidatesKey("run-9")
|
||||
var c meshcatalog.CandidatesByGroupValue
|
||||
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||
if e, err := candidates.Get(ctx, key); err == nil {
|
||||
if c, err = index.DecodeCandidate(e.Value()); err == nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(c.Pulls) != 1 || c.Pulls[0].Number != 0 || c.Pulls[0].Head != commit || c.Pulls[0].Fps["avahi"].Manifest != meshcatalogclient.ManifestDigest(manifest) {
|
||||
t.Fatalf("candidates.run-9 = %+v within 20 s of run-started, want avahi at the run's commit with its manifest", c)
|
||||
}
|
||||
restart()
|
||||
ended, _ := json.Marshal(meshpipelinerunner.PipelineEnded{V: 1, Pipeline: "feature-1966-x.0", Branch: "feature/1966-x", Run: "9",
|
||||
Movers: []meshpipelinerunner.MoverEnded{{Module: "avahi", Steps: []meshpipelinerunner.StepResult{}}}, Outcome: meshpipelinerunner.PipelineEndedOutcomePassed})
|
||||
if _, err := js.Publish(ctx, meshpipelinerunner.SubjectPipelineEnded("feature-1966-x", "0"), ended); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gone := false
|
||||
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline) && !gone; time.Sleep(50 * time.Millisecond) {
|
||||
_, cerr := candidates.Get(ctx, key)
|
||||
_, rerr := runs.Get(ctx, "run-9")
|
||||
gone = errors.Is(cerr, jetstream.ErrKeyNotFound) && errors.Is(rerr, jetstream.ErrKeyNotFound)
|
||||
}
|
||||
if !gone {
|
||||
t.Error("the run's candidate or its runs key stays 20 s after its member's pipeline ended across a restart")
|
||||
}
|
||||
}
|
||||
|
||||
// TestIndexerCountsOnlyThePlacedBuildNodesToolchains (issue 1940, issue 774): the toolchains bucket holds the report of
|
||||
// n2, where the catalogue no longer places mesh-build-go, at another version than n1's, as ace's v26.12.0 stayed beside
|
||||
// the v26.12.1 of the nodes mesh-build-node runs on. The indexer gives the merged module a fingerprint from n1's
|
||||
|
||||
@@ -182,9 +182,9 @@ func containsVia(call toolCaller) func(ctx context.Context, repository, commit,
|
||||
var lockFiles = []string{"go.sum", "package-lock.json"}
|
||||
|
||||
// readVia reads a repository at a commit as the forge bridge does for a merged message: the seat git's trees, then the
|
||||
// module.json and lock files of the one directory dir by its file (a repository can hold hundreds). A manifest the forge
|
||||
// does not have is an error; a directory with no module.json in the trees is answered with no manifest; a lock file it does
|
||||
// not have is left out.
|
||||
// module.json and lock files of the one directory dir by its file (a repository can hold hundreds), and the go.mod beside a
|
||||
// go.sum for its mesh packages (Packages). A manifest the forge does not have is an error; a directory with no module.json in
|
||||
// the trees is answered with no manifest; a lock file it does not have is left out.
|
||||
func readVia(call toolCaller) func(ctx context.Context, repository, sha, dir string) (index.RepoRead, error) {
|
||||
return func(_ context.Context, repository, sha, dir string) (index.RepoRead, error) {
|
||||
a, err := treesOf(call, repository, git.TreesInput{Commit: sha})
|
||||
@@ -222,13 +222,20 @@ func readVia(call toolCaller) func(ctx context.Context, repository, sha, dir str
|
||||
out.Locks[dir] = map[string]string{}
|
||||
}
|
||||
out.Locks[dir][name] = hex.EncodeToString(sum[:])
|
||||
if name == "go.sum" {
|
||||
if p := fingerprint.MeshPackagesOf([]byte(c)); len(p) > 0 {
|
||||
if out.Packages == nil {
|
||||
out.Packages = map[string][]string{}
|
||||
}
|
||||
out.Packages[dir] = p // the mesh packages it uses, as the bridge reads them (ADR 0573 §2)
|
||||
if name != "go.sum" {
|
||||
continue
|
||||
}
|
||||
// A Go module's mesh packages are what its go.mod requires, at any version (design 47 H-A-157): its go.sum
|
||||
// names none required at the zero version (ADR 0573 §2).
|
||||
mod, absent, err := file(dir, "go.mod")
|
||||
if err != nil {
|
||||
return index.RepoRead{}, fmt.Errorf("%s: %w", path.Join(dir, "go.mod"), err)
|
||||
}
|
||||
if p := fingerprint.MeshRequiresOf([]byte(mod)); !absent && len(p) > 0 {
|
||||
if out.Packages == nil {
|
||||
out.Packages = map[string][]string{}
|
||||
}
|
||||
out.Packages[dir] = p
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -83,22 +83,29 @@ func TestTheTrunkIsTheDirectoriesTheForgeListsTrees(t *testing.T) { // 1900/AC3
|
||||
// or a run that can never be taken in, is terminated, any other failure is left for redelivery, and a taken one is acked.
|
||||
for _, row := range []struct {
|
||||
name, body string
|
||||
ended bool
|
||||
ended, wanted bool
|
||||
err error
|
||||
acked, termd int
|
||||
started, endedOf string
|
||||
}{
|
||||
{name: "a run started", body: `{"v":1,"run":"7","members":[{"module":"a","commit":"c"}],"by":"s"}`, acked: 1, started: "7"},
|
||||
{name: "a run's pipeline ended", body: `{"v":1,"pipeline":"p","module":"a","branch":"b","commit":"c","outcome":"passed","steps":[],"run":"7"}`, ended: true, acked: 1, endedOf: "7"},
|
||||
{name: "a pipeline's wants written to the end", body: `{"v":1,"pipeline":"main.12","branch":"main","source":12,"outcome":"tier-late"}`, wanted: true, acked: 1},
|
||||
{name: "not json", body: `nope`, termd: 1},
|
||||
{name: "a run never taken in", body: `{"v":1,"run":"7","members":[]}`, err: fmt.Errorf("%w: no member", index.ErrUnreadable), termd: 1, started: "7"},
|
||||
{name: "a bus fault", body: `{"v":1,"run":"7","members":[{"module":"a","commit":"c"}]}`, err: errors.New("the bus did not answer"), started: "7"},
|
||||
} {
|
||||
ix := &fakeRuns{err: row.err}
|
||||
m := &fakeMsg{subject: "x", data: []byte(row.body)}
|
||||
if row.ended {
|
||||
switch {
|
||||
case row.wanted:
|
||||
heardPipelineWanted(context.Background(), ix, m, t.Logf)
|
||||
if ix.wanted != "main.12 tier-late" {
|
||||
t.Errorf("%s: was handed %q, want main.12 tier-late", row.name, ix.wanted)
|
||||
}
|
||||
case row.ended:
|
||||
heardPipelineEnded(context.Background(), ix, m, t.Logf)
|
||||
} else {
|
||||
default:
|
||||
heardRunStarted(context.Background(), ix, m, t.Logf)
|
||||
}
|
||||
if m.acked != row.acked || m.termd != row.termd || ix.started != row.started || ix.ended != row.endedOf {
|
||||
@@ -120,6 +127,8 @@ func TestReadViaReadsManifestsAndLocks(t *testing.T) {
|
||||
return json.RawMessage(`{"path":"module.json","content":"{\"module\":\"snake\"}"}`), nil
|
||||
case "go.sum":
|
||||
return json.RawMessage(`{"path":"go.sum","content":"x"}`), nil
|
||||
case "go.mod": // a mesh package required at the zero version, which no go.sum line names (design 47 H-A-157)
|
||||
return json.RawMessage(`{"path":"go.mod","content":"module snake\n\nrequire (\n\tgit.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000\n)\n"}`), nil
|
||||
}
|
||||
return json.RawMessage(`{"path":"x","absent":true}`), nil
|
||||
}
|
||||
@@ -129,18 +138,21 @@ func TestReadViaReadsManifestsAndLocks(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got.Manifests) != 1 || len(files) != 3 { // the one directory asked: its manifest and two lock files, never the others
|
||||
if len(got.Manifests) != 1 || len(files) != 4 { // the one directory asked: its manifest, two lock files and the go.mod, never the others
|
||||
t.Fatalf("manifests = %v, files asked = %v", got.Manifests, files)
|
||||
}
|
||||
if got.Root != "r0" || got.Manifests[""] != `{"module":"snake"}` || len(got.Locks[""]) != 1 || got.Locks[""]["go.sum"] == "" {
|
||||
t.Fatalf("read = %+v", got)
|
||||
}
|
||||
if p := got.Packages[""]; len(p) != 1 || p[0] != "git.novox.be/novox/mesh-sdk/go" {
|
||||
t.Fatalf("mesh packages %v, want mesh-sdk, which the go.mod requires", got.Packages)
|
||||
}
|
||||
}
|
||||
|
||||
// fakeRuns records the run events handed to it and answers err.
|
||||
type fakeRuns struct {
|
||||
err error
|
||||
started, ended string
|
||||
err error
|
||||
started, ended, wanted string
|
||||
}
|
||||
|
||||
func (f *fakeRuns) StartRun(_ context.Context, e meshpipelinerunner.RunStarted) error {
|
||||
@@ -157,4 +169,9 @@ func (f *fakeRuns) HeardPins(context.Context, *meshpipelinerunner.PipelineStarte
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeRuns) HeardPipelineWanted(_ context.Context, branch string, source int64, outcome string) error {
|
||||
f.wanted = fmt.Sprintf("%s.%d %s", branch, source, outcome)
|
||||
return f.err
|
||||
}
|
||||
|
||||
func (f *fakeRuns) JudgePins(context.Context) {}
|
||||
|
||||
+10
-4
@@ -58,7 +58,8 @@ func untilModule(t *testing.T, ctx context.Context, js jetstream.JetStream, name
|
||||
}
|
||||
|
||||
// TestASourceRevisionMovingMeshSDKNamesItAndItsImporters (design 47 H §743; A-476a; ADR 0593 §2 to §4): the indexer whole against
|
||||
// the bus. Two Go modules of the catalogue, importer, whose go.sum requires mesh-sdk's Go module, and plain, which requires
|
||||
// the bus. Two Go modules of the catalogue, importer, whose go.mod requires mesh-sdk's Go module at the zero version (no
|
||||
// go.sum line names it, ADR 0573 §2; design 47 H-A-157), and plain, which requires
|
||||
// nothing of it, are taken in with mesh-sdk as main's first source revision. A source revision moving mesh-sdk to a commit
|
||||
// changing its buildstep package is taken in as one revision whose movers are mesh-sdk and importer, of another repository,
|
||||
// and not plain; the packages key of mesh-sdk/go names that commit with a digest per package.
|
||||
@@ -67,7 +68,7 @@ func TestASourceRevisionMovingMeshSDKNamesItAndItsImporters(t *testing.T) {
|
||||
sdkGo = "git.novox.be/novox/mesh-sdk/go"
|
||||
m1, m2 = "1111111111111111111111111111111111111111", "2222222222222222222222222222222222222222"
|
||||
catalogMerge = "3333333333333333333333333333333333333333"
|
||||
goSumContent = "git.novox.be/novox/mesh-sdk/go v0.1.14 h1:x=\ngit.novox.be/novox/mesh-sdk/go v0.1.14/go.mod h1:y=\n"
|
||||
goModContent = "module importer\n\nrequire git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000\n"
|
||||
sdkRoot1, sdk2 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
|
||||
)
|
||||
avahi, err := os.ReadFile("../../internal/index/testdata/catalogue-a67f0196-avahi.module.json")
|
||||
@@ -119,7 +120,8 @@ func TestASourceRevisionMovingMeshSDKNamesItAndItsImporters(t *testing.T) {
|
||||
forge.put(m2, sdkAt(sdk2, "f6", "7777777777777777777777777777777777777777"))
|
||||
forge.put(catalogMerge, repoAt{trees: map[string]string{"modules/importer": "t1", "modules/plain": "t2"},
|
||||
files: map[string]string{"modules/importer/module.json": named("importer"), "modules/plain/module.json": named("plain"),
|
||||
"modules/importer/go.sum": goSumContent, "modules/plain/go.sum": "golang.org/x/sys v0.1.0 h1:x=\n"}})
|
||||
"modules/importer/go.mod": goModContent, "modules/importer/go.sum": "golang.org/x/sys v0.1.0 h1:x=\n",
|
||||
"modules/plain/go.mod": "module plain\n\nrequire golang.org/x/sys v0.1.0\n", "modules/plain/go.sum": "golang.org/x/sys v0.1.0 h1:x=\n"}})
|
||||
regs := registerAvahi(t, ctx, js, "main")
|
||||
for _, r := range []meshcatalog.Registration{{V: 1, Module: "importer", Origin: "novox/mesh-catalog", Path: "modules/importer", Branches: []string{"main"}},
|
||||
{V: 1, Module: "plain", Origin: "novox/mesh-catalog", Path: "modules/plain", Branches: []string{"main"}},
|
||||
@@ -142,7 +144,11 @@ func TestASourceRevisionMovingMeshSDKNamesItAndItsImporters(t *testing.T) {
|
||||
say(10, true, meshdelivery.SourceMovedEventHeadsItem{Repository: "novox/mesh-sdk", Commit: m1}, meshdelivery.SourceMovedEventHeadsItem{Repository: "novox/mesh-catalog", Commit: catalogMerge})
|
||||
heard.next(t, "main", 10)
|
||||
heard.acked(t, "main")
|
||||
untilModule(t, ctx, js, "importer", func(m meshcatalog.ModulesByModuleValue) bool { return m.Fp2 != "" && m.Packages[sdkGo] == m1 })
|
||||
// mesh-sdk has released no version: importer keeps the package for its kind-1 edge, with no pin (H-A-161).
|
||||
untilModule(t, ctx, js, "importer", func(m meshcatalog.ModulesByModuleValue) bool {
|
||||
pin, kept := m.Packages[sdkGo]
|
||||
return m.Fp2 != "" && kept && pin == ""
|
||||
})
|
||||
plain := untilModule(t, ctx, js, "plain", func(m meshcatalog.ModulesByModuleValue) bool { return m.Fp2 != "" })
|
||||
if _, ok := plain.Packages[sdkGo]; ok {
|
||||
t.Fatalf("plain requires %s: %+v", sdkGo, plain.Packages)
|
||||
|
||||
@@ -6,15 +6,17 @@ import (
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
)
|
||||
|
||||
// The indexer pins a pipeline's revision on the runner's pipeline-started and frees it on its pipeline-ended, each heard on
|
||||
// a durable consumer of its own on that event's stream (design 47 H §749, runs.go): an event is applied to the state pins,
|
||||
// and only then acknowledged, so one said while the indexer was down is applied when it comes back.
|
||||
// The indexer pins a pipeline's revision on the runner's pipeline-started and frees it on its pipeline-ended, a passed one's release
|
||||
// pinned there and freed on the seat mesh-wants' pipeline-wanted of a newer pipeline (H-A-168, H-A-170), each heard on a durable
|
||||
// consumer of its own on that event's stream (design 47 H §749, runs.go): an event is applied to the state pins, and only then
|
||||
// acknowledged, so one said while the indexer was down is applied when it comes back.
|
||||
|
||||
// pinsStreams are the streams of the runner's events the indexer's pins follow.
|
||||
var pinsStreams = []string{meshpipelinerunner.StreamPipelineStarted, meshpipelinerunner.StreamPipelineEnded}
|
||||
// pinsStreams are the streams of the events the indexer's pins follow.
|
||||
var pinsStreams = []string{meshpipelinerunner.StreamPipelineStarted, meshpipelinerunner.StreamPipelineEnded, meshwants.StreamPipelineWanted}
|
||||
|
||||
// pendingOf says how many messages of the indexer's durable consumers of the pins' streams have not been given to it yet.
|
||||
func pendingOf(js jetstream.JetStream, cred process.Account) func(context.Context) (uint64, error) {
|
||||
|
||||
@@ -8,17 +8,20 @@ import (
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"mesh-delivery-indexer/internal/index"
|
||||
)
|
||||
|
||||
// runTaker is what the indexer does with the runner's events: a run on request's (index.Indexer.StartRun, EndRunPipeline),
|
||||
// and a pipeline's pins (HeardPins, design 47 H §749), the kept revisions judged after each pin freed (JudgePins).
|
||||
// and a pipeline's pins (HeardPins, design 47 H §749), a release's freed on the seat mesh-wants' pipeline-wanted (HeardPipelineWanted,
|
||||
// H-A-168, H-A-170), the kept revisions judged after each pin freed (JudgePins).
|
||||
type runTaker interface {
|
||||
StartRun(ctx context.Context, e meshpipelinerunner.RunStarted) error
|
||||
EndRunPipeline(ctx context.Context, e meshpipelinerunner.PipelineEnded) error
|
||||
HeardPins(ctx context.Context, started *meshpipelinerunner.PipelineStarted, ended *meshpipelinerunner.PipelineEnded) error
|
||||
HeardPipelineWanted(ctx context.Context, branch string, source int64, outcome string) error
|
||||
JudgePins(ctx context.Context)
|
||||
}
|
||||
|
||||
@@ -33,6 +36,7 @@ func hearRuns(ctx context.Context, js jetstream.JetStream, cred process.Account,
|
||||
{meshpipelinerunner.StreamRunStarted, func(m jetstream.Msg) { heardRunStarted(ctx, ix, m, logf) }},
|
||||
{meshpipelinerunner.StreamPipelineStarted, func(m jetstream.Msg) { heardPipelineStarted(ctx, ix, m, logf) }},
|
||||
{meshpipelinerunner.StreamPipelineEnded, func(m jetstream.Msg) { heardPipelineEnded(ctx, ix, m, logf) }},
|
||||
{meshwants.StreamPipelineWanted, func(m jetstream.Msg) { heardPipelineWanted(ctx, ix, m, logf) }},
|
||||
} {
|
||||
_ = notHeard.Bind(s.stream, "no run on request's candidate is written or dropped until the indexer starts again", func() error {
|
||||
cc, err := cred.HearSeatEvents(ctx, js, s.stream, s.each, logf)
|
||||
@@ -85,6 +89,23 @@ func heardPipelineEnded(ctx context.Context, ix runTaker, m delivered, logf func
|
||||
ix.JudgePins(ctx)
|
||||
}
|
||||
|
||||
// heardPipelineWanted takes one pipeline-wanted of the seat mesh-wants in: the pipeline's wants are written to the end, tier-late
|
||||
// included, so the older releases of the modules it moves free their revisions, and the kept revisions are judged (H-A-168,
|
||||
// H-A-170).
|
||||
func heardPipelineWanted(ctx context.Context, ix runTaker, m delivered, logf func(string, ...any)) {
|
||||
var e meshwants.PipelineWanted
|
||||
if err := json.Unmarshal(m.Data(), &e); err != nil || e.Branch == "" || e.Source <= 0 {
|
||||
logf("%s: not a pipeline-wanted naming a branch and a source, dropped: %v", m.Subject(), err)
|
||||
_ = m.Term()
|
||||
return
|
||||
}
|
||||
err := ix.HeardPipelineWanted(ctx, e.Branch, e.Source, e.Outcome)
|
||||
settleRun(m, err, "pipeline-wanted of "+e.Pipeline, logf)
|
||||
if err == nil {
|
||||
ix.JudgePins(ctx)
|
||||
}
|
||||
}
|
||||
|
||||
func settleRun(m delivered, err error, what string, logf func(string, ...any)) {
|
||||
switch {
|
||||
case err == nil:
|
||||
|
||||
+90
-14
@@ -7,6 +7,7 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -16,10 +17,12 @@ import (
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
@@ -185,12 +188,14 @@ func sayMoved(t *testing.T, ctx context.Context, js jetstream.JetStream, branch
|
||||
type takeIns struct {
|
||||
taken chan meshcatalog.TakeInTaken
|
||||
pending chan meshcatalog.TakeInPending
|
||||
acks chan string // the escaped branch whose source-moved the indexer acknowledged
|
||||
acks chan string // the escaped branch whose source-moved the indexer acknowledged
|
||||
runAcks chan struct{} // a run-started the indexer acknowledged
|
||||
}
|
||||
|
||||
func hearTakeIns(t *testing.T, nc *nats.Conn) *takeIns {
|
||||
t.Helper()
|
||||
h := &takeIns{taken: make(chan meshcatalog.TakeInTaken, 64), pending: make(chan meshcatalog.TakeInPending, 64), acks: make(chan string, 64)}
|
||||
h := &takeIns{taken: make(chan meshcatalog.TakeInTaken, 64), pending: make(chan meshcatalog.TakeInPending, 64), acks: make(chan string, 64),
|
||||
runAcks: make(chan struct{}, 64)}
|
||||
prefix := "$JS.ACK." + meshdelivery.StreamSourceMoved + "."
|
||||
acks, err := nc.Subscribe(prefix+"*.>", func(m *nats.Msg) {
|
||||
consumer, _, _ := strings.Cut(strings.TrimPrefix(m.Subject, prefix), ".")
|
||||
@@ -202,6 +207,15 @@ func hearTakeIns(t *testing.T, nc *nats.Conn) *takeIns {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = acks.Unsubscribe() })
|
||||
runAcks, err := nc.Subscribe("$JS.ACK."+meshpipelinerunner.StreamRunStarted+".n1_mesh-delivery-indexer.>", func(m *nats.Msg) {
|
||||
if string(m.Data) == "+ACK" || len(m.Data) == 0 {
|
||||
h.runAcks <- struct{}{}
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = runAcks.Unsubscribe() })
|
||||
sub, err := nc.Subscribe("mesh.mod."+testIndexer+".event.take-in.*", func(m *nats.Msg) {
|
||||
switch m.Subject {
|
||||
case "mesh.mod." + testIndexer + ".event.take-in.taken":
|
||||
@@ -251,6 +265,16 @@ func (h *takeIns) acked(t *testing.T, branch string) {
|
||||
}
|
||||
}
|
||||
|
||||
// runAcked waits for the indexer's acknowledgement of a run-started.
|
||||
func (h *takeIns) runAcked(t *testing.T) {
|
||||
t.Helper()
|
||||
select {
|
||||
case <-h.runAcks:
|
||||
case <-time.After(waitBound):
|
||||
t.Fatalf("no run-started acknowledged within %s", waitBound)
|
||||
}
|
||||
}
|
||||
|
||||
// nextPending is the next take-in.pending said, which must be of that branch and source.
|
||||
func (h *takeIns) nextPending(t *testing.T, branch string, source int64) meshcatalog.TakeInPending {
|
||||
t.Helper()
|
||||
@@ -339,17 +363,22 @@ func sayPipeline(t *testing.T, ctx context.Context, js jetstream.JetStream, ende
|
||||
return
|
||||
}
|
||||
raw, _ := json.Marshal(meshpipelinerunner.PipelineEnded{V: 1, Pipeline: pipeline, Branch: branch, Source: source, Revision: source,
|
||||
Outcome: meshpipelinerunner.PipelineEndedOutcomePassed, Movers: []meshpipelinerunner.MoverEnded{{Module: "avahi", Steps: []meshpipelinerunner.StepResult{}, Tier: 1}}})
|
||||
Outcome: meshpipelinerunner.PipelineEndedOutcomePassed, Movers: []meshpipelinerunner.MoverEnded{{Module: "avahi", Steps: []meshpipelinerunner.StepResult{}, Tier: 1,
|
||||
Version: "0.1." + s}}})
|
||||
if _, err := js.Publish(ctx, meshpipelinerunner.SubjectPipelineEnded(b, s), raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// pinned is whether a pins value names the record.
|
||||
func pinned(record string) func([]byte) bool {
|
||||
// pinned is whether a pins value names exactly the records, in any order.
|
||||
func pinned(records ...string) func([]byte) bool {
|
||||
return func(v []byte) bool {
|
||||
var p index.PinValue
|
||||
return v != nil && json.Unmarshal(v, &p) == nil && len(p.Records) == 1 && p.Records[0] == record
|
||||
if v == nil || json.Unmarshal(v, &p) != nil {
|
||||
return false
|
||||
}
|
||||
slices.Sort(p.Records)
|
||||
return slices.Equal(p.Records, slices.Sorted(slices.Values(records)))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -364,10 +393,14 @@ func absent(v []byte) bool { return v == nil }
|
||||
// given again says the same from the kept movers and keeps no second revision; main.10, the branch's previous revision, is
|
||||
// pinned pipeline.main.11 in the same write as validate's base (H-A-147);
|
||||
// - pipeline-started of main.12 pins it, a restart keeps main.11 and main.10 while main.11's pipeline runs;
|
||||
// - a run on request's run-started (run 9, avahi at main 12's commit on a branch no registration watches) writes
|
||||
// candidates.run-9, one pull of number 0 at that commit with avahi's manifest read through the seat git, and the run's key; across
|
||||
// the restart, the runner's pipeline-ended of its one member drops both (design 47 G §64, §190; A-160, A-174);
|
||||
// - with the store full, main.13 is held, said once as take-in.pending, written whole in the state held and acknowledged
|
||||
// (H-A-114, H-A-118); nothing takes it in on a
|
||||
// timer: main.11's pipeline-ended frees both its pins, main.10 goes (main.11 stays, main.12's base), the indexer says its
|
||||
// room and takes main.13 in on it;
|
||||
// timer: main.11's pipeline-ended frees both its pins, main.10 goes (main.11 stays, main.12's base and its passed release), the
|
||||
// indexer says its room and takes main.13 in on it; main.12's passed end and its pipeline-wanted free main.11's release, and
|
||||
// main.11 goes (H-A-168, H-A-170, H-A-173);
|
||||
// - beta's take-in whose commit the forge cannot read is held while main's next goes on; the hand skip, with why, is recorded in
|
||||
// the hand-act log before it is asked, and take-in.taken says it skipped;
|
||||
// - a branch watched later is bound on registration-changed and its first revision taken in.
|
||||
@@ -378,8 +411,8 @@ func TestIndexerWhole(t *testing.T) {
|
||||
)
|
||||
var forge *forgeCommits
|
||||
var heard *takeIns
|
||||
var pins, held, takeinsKV, regs jetstream.KeyValue
|
||||
js, ctx, _, _, restart := startIndexerRun(t, func(context.Context, buildLog) {}, func(ctx context.Context, js jetstream.JetStream) {
|
||||
var pins, held, takeinsKV, regs, runs jetstream.KeyValue
|
||||
js, ctx, candidates, _, restart := startIndexerRun(t, func(context.Context, buildLog) {}, func(ctx context.Context, js jetstream.JetStream) {
|
||||
heard = hearTakeIns(t, js.Conn())
|
||||
forge = serveCommits(t, js.Conn())
|
||||
for commit, tree := range map[string]string{c10: "t10", c11: "t11", c12: "t12", c13: "t13"} {
|
||||
@@ -389,7 +422,7 @@ func TestIndexerWhole(t *testing.T) {
|
||||
for _, b := range []struct {
|
||||
state string
|
||||
kv *jetstream.KeyValue
|
||||
}{{index.HeldState, &held}, {index.TakeInsState, &takeinsKV}} {
|
||||
}{{index.HeldState, &held}, {index.TakeInsState, &takeinsKV}, {index.RunsState, &runs}} {
|
||||
if *b.kv, err = js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.Bucket(testIndexer, b.state), History: 1}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -457,6 +490,25 @@ func TestIndexerWhole(t *testing.T) {
|
||||
}
|
||||
sayPipeline(t, ctx, js, false, "main", 12)
|
||||
untilKey(t, ctx, pins, index.PinKey("main", 12), pinned("pipeline.main.12"))
|
||||
// A run on request: its candidate and its key written on run-started, from avahi's files at the run's commit (A-160, A-174).
|
||||
manifest, err := os.ReadFile("../../internal/index/testdata/catalogue-a67f0196-avahi.module.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
started, _ := json.Marshal(meshpipelinerunner.RunStarted{V: 1, Run: "9", By: "lead@n1", Members: []meshpipelinerunner.Member{{Module: "avahi", Commit: c12, Branch: "feature/1966-x"}}})
|
||||
if _, err := js.Publish(ctx, meshpipelinerunner.SubjectRunStarted("avahi", "feature-1966-x", c12), started); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
runKey, _ := meshcatalogclient.CandidatesKey("run-9")
|
||||
untilKey(t, ctx, candidates, runKey, func(v []byte) bool {
|
||||
c, err := index.DecodeCandidate(v)
|
||||
return v != nil && err == nil && len(c.Pulls) == 1 && c.Pulls[0].Number == 0 && c.Pulls[0].Head == c12 &&
|
||||
c.Pulls[0].Fps["avahi"].Manifest == meshcatalogclient.ManifestDigest(manifest)
|
||||
})
|
||||
untilKey(t, ctx, runs, "run-9", func(v []byte) bool { return v != nil })
|
||||
// The candidate is written before its run-started is acknowledged: the restart waits for that too, so the stop leaves it
|
||||
// unacknowledged for no redelivery to write the candidate again after its run ended.
|
||||
heard.runAcked(t)
|
||||
// The restart waits for the take-ins' ends, their source-moved acknowledged (design 47 H §743: main 10, 11, 11 given
|
||||
// again and 12), so the stop cancels none and leaves none unacknowledged ahead of main 13.
|
||||
for range 4 {
|
||||
@@ -468,6 +520,14 @@ func TestIndexerWhole(t *testing.T) {
|
||||
t.Fatalf("%s is not kept across a restart while main.11's pipeline runs: %v", name, err)
|
||||
}
|
||||
}
|
||||
// The run's member's pipeline ends across the restart: its candidate and its key go (A-174).
|
||||
ended, _ := json.Marshal(meshpipelinerunner.PipelineEnded{V: 1, Pipeline: "feature-1966-x.0", Branch: "feature/1966-x", Run: "9",
|
||||
Movers: []meshpipelinerunner.MoverEnded{{Module: "avahi", Steps: []meshpipelinerunner.StepResult{}}}, Outcome: meshpipelinerunner.PipelineEndedOutcomePassed})
|
||||
if _, err := js.Publish(ctx, meshpipelinerunner.SubjectPipelineEnded("feature-1966-x", "0"), ended); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
untilKey(t, ctx, candidates, runKey, absent)
|
||||
untilKey(t, ctx, runs, "run-9", absent)
|
||||
|
||||
// The store full: held, said once, in the state held; taken in on the indexer's own room once a freed pin makes it.
|
||||
used, _, err := objects.Fill(ctx)
|
||||
@@ -484,7 +544,9 @@ func TestIndexerWhole(t *testing.T) {
|
||||
return v != nil && json.Unmarshal(v, &hs) == nil && len(hs) == 1 && hs[0].Source == 13 && hs[0].Why != "" && !hs[0].Since.IsZero() &&
|
||||
hs[0].Call != "" && len(hs[0].Heads) == 1 && hs[0].Heads[0].Commit == c13
|
||||
})
|
||||
// H-A-114: the held source-moved is acknowledged on the bus, kept only in the state held.
|
||||
// H-A-114: the held source-moved is acknowledged on the bus, kept only in the state held; its acknowledgement follows the
|
||||
// write, so it is waited for before the consumer is read.
|
||||
heard.acked(t, "main")
|
||||
if cons, err := testAccount.NamedEventConsumer(ctx, js, meshdelivery.StreamSourceMoved, keys.EscapeName("main")); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if info, err := cons.Info(ctx); err != nil || info.NumAckPending != 0 {
|
||||
@@ -496,8 +558,22 @@ func TestIndexerWhole(t *testing.T) {
|
||||
t.Errorf("main 13 taken in on room said %+v, want avahi at %s", got, c13)
|
||||
}
|
||||
untilKey(t, ctx, held, keys.EscapeName("main"), absent)
|
||||
untilKey(t, ctx, pins, index.PinKey("main", 11), pinned("pipeline.main.12")) // main.12's validate base stays (H-A-147)
|
||||
untilRevisionGone(t, ctx, js, "main.10") // the end freed both of main.11's pins
|
||||
// main.12's validate base stays (H-A-147), and main.11's passed pipeline's release pins it (H-A-168, H-A-170).
|
||||
untilKey(t, ctx, pins, index.PinKey("main", 11), pinned("pipeline.main.12", index.ReleaseRecord("main", "avahi")))
|
||||
untilRevisionGone(t, ctx, js, "main.10") // the end freed both of main.11's pins
|
||||
// main.12's passed pipeline releases avahi again; its pipeline-wanted, the seat mesh-wants' (its wants written to the end),
|
||||
// frees main.11's older release, and main.11 goes (H-A-168, H-A-170, H-A-173).
|
||||
sayPipeline(t, ctx, js, true, "main", 12)
|
||||
untilKey(t, ctx, pins, index.PinKey("main", 12), func(v []byte) bool {
|
||||
var p index.PinValue
|
||||
return v != nil && json.Unmarshal(v, &p) == nil && slices.Contains(p.Records, index.ReleaseRecord("main", "avahi"))
|
||||
})
|
||||
wantedRaw, _ := json.Marshal(meshwants.PipelineWanted{V: 1, Pipeline: "main.12", Branch: "main", Source: 12, Outcome: meshwants.PipelineWantedOutcomeRunning})
|
||||
if _, err := js.Publish(ctx, meshwants.SubjectPipelineWanted("main", "12"), wantedRaw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
untilKey(t, ctx, pins, index.PinKey("main", 11), absent)
|
||||
untilRevisionGone(t, ctx, js, "main.11")
|
||||
if n := len(heard.pending); n != 0 {
|
||||
t.Errorf("take-in.pending said %d more times for one held source revision, want once", n)
|
||||
}
|
||||
|
||||
@@ -135,14 +135,32 @@ func NamedTree(manifest string, paths map[string]string) (string, error) {
|
||||
return NamedTreePrefix + hex.EncodeToString(sum[:]), nil
|
||||
}
|
||||
|
||||
// MeshPackagesOf are the mesh Go modules a go.sum names (the module path of each line under
|
||||
// trunk.ForgeModulePrefix), sorted and each once; nil for none. They are what a module with no reported import set
|
||||
// takes the version line of (ADR 0593 §2).
|
||||
func MeshPackagesOf(goSum []byte) []string {
|
||||
// MeshRequiresOf are the mesh Go modules a go.mod requires (the module path of each `require` line under
|
||||
// trunk.ForgeModulePrefix, alone or in a block, at any version, the zero version and `// indirect` included), sorted and
|
||||
// each once; nil for none. They are a module's mesh packages from the take-in, with no build needed (design 47 H-A-157):
|
||||
// a go.sum names none required at the zero version (ADR 0573 §2), so it is never read for them.
|
||||
func MeshRequiresOf(goMod []byte) []string {
|
||||
seen := map[string]bool{}
|
||||
for _, line := range strings.Split(string(goSum), "\n") {
|
||||
inBlock := false
|
||||
for _, line := range strings.Split(string(goMod), "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if i := strings.Index(line, "//"); i >= 0 {
|
||||
line = strings.TrimSpace(line[:i])
|
||||
}
|
||||
switch {
|
||||
case line == "require (":
|
||||
inBlock = true
|
||||
continue
|
||||
case inBlock && line == ")":
|
||||
inBlock = false
|
||||
continue
|
||||
case strings.HasPrefix(line, "require "):
|
||||
line = strings.TrimSpace(strings.TrimPrefix(line, "require "))
|
||||
case !inBlock:
|
||||
continue
|
||||
}
|
||||
f := strings.Fields(line)
|
||||
if len(f) < 2 || !strings.HasPrefix(f[0], trunk.ForgeModulePrefix) {
|
||||
if len(f) != 2 || !strings.HasPrefix(f[0], trunk.ForgeModulePrefix) {
|
||||
continue
|
||||
}
|
||||
if _, _, ok := trunk.PackageRepository(f[0]); ok {
|
||||
@@ -405,41 +423,49 @@ func sortedKeys[V any](m map[string]V) []string { return slices.Sorted(maps.Keys
|
||||
// commit or no directory, is in missing, sorted; versions is nil when packages is empty.
|
||||
func HeldVersionsOf(packages []string, modules map[string]meshcatalog.ModulesByModuleValue) (versions sdkstep.PackageVersions, missing []string) {
|
||||
for _, pkg := range packages {
|
||||
repository, dir, ok := trunk.PackageRepository(pkg)
|
||||
commit, depth := "", -1
|
||||
if ok {
|
||||
for _, key := range slices.Sorted(maps.Keys(modules)) {
|
||||
m := modules[key]
|
||||
if !trunk.SameRepository(m.Repository, repository) || m.Commit == "" {
|
||||
continue
|
||||
}
|
||||
at := strings.Trim(m.Path, "/")
|
||||
switch {
|
||||
case at == "":
|
||||
continue // a value that names no directory says nothing of where the package lies
|
||||
case at == meshcatalogclient.RootPath:
|
||||
at = ""
|
||||
case dir != at && !strings.HasPrefix(dir, at+"/"):
|
||||
continue
|
||||
}
|
||||
if d := len(at); d > depth {
|
||||
commit, depth = m.Commit, d
|
||||
}
|
||||
}
|
||||
}
|
||||
if commit == "" {
|
||||
key := HolderOf(pkg, modules)
|
||||
if key == "" {
|
||||
missing = append(missing, pkg)
|
||||
continue
|
||||
}
|
||||
if versions == nil {
|
||||
versions = sdkstep.PackageVersions{}
|
||||
}
|
||||
versions[pkg] = commit
|
||||
versions[pkg] = modules[key].Commit
|
||||
}
|
||||
slices.Sort(missing)
|
||||
return versions, missing
|
||||
}
|
||||
|
||||
// HolderOf is the key of the held module of the package's repository whose directory holds the package's directory (the
|
||||
// deepest, a module at the repository's root holding every directory) and that names a commit; "" for none.
|
||||
func HolderOf(pkg string, modules map[string]meshcatalog.ModulesByModuleValue) string {
|
||||
repository, dir, ok := trunk.PackageRepository(pkg)
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
holder, depth := "", -1
|
||||
for _, key := range slices.Sorted(maps.Keys(modules)) {
|
||||
m := modules[key]
|
||||
if !trunk.SameRepository(m.Repository, repository) || m.Commit == "" {
|
||||
continue
|
||||
}
|
||||
at := strings.Trim(m.Path, "/")
|
||||
switch {
|
||||
case at == "":
|
||||
continue // a value that names no directory says nothing of where the package lies
|
||||
case at == meshcatalogclient.RootPath:
|
||||
at = ""
|
||||
case dir != at && !strings.HasPrefix(dir, at+"/"):
|
||||
continue
|
||||
}
|
||||
if d := len(at); d > depth {
|
||||
holder, depth = key, d
|
||||
}
|
||||
}
|
||||
return holder
|
||||
}
|
||||
|
||||
// digestOf is the content digest of the package at import path pkg in a published module, and whether it holds it
|
||||
// (its packages are sorted by import path).
|
||||
func digestOf(p meshcatalog.PackagesByModuleValue, pkg string) (string, bool) {
|
||||
@@ -455,3 +481,21 @@ func digestOf(p meshcatalog.PackagesByModuleValue, pkg string) (string, bool) {
|
||||
// NamedTreePrefix opens a module's tree when it is computed over the paths its artifacts name, not the directory's own
|
||||
// git tree (novox/hq ADR 0499 clause 2): the indexer alone computes it (NamedTree).
|
||||
const NamedTreePrefix = "named:"
|
||||
|
||||
// WithPins is the fingerprint fp with the pins a build of it is made against (design 47 H-A-167): fp itself when no
|
||||
// package is pinned, else `src2:` and the hex sha256 of fp and, sorted, the lines "pin <package> <version>" of each pin,
|
||||
// joined by a newline. A package kept with no version (none released) is no pin.
|
||||
func WithPins(fp string, pins sdkstep.PackageVersions) string {
|
||||
var lines []string
|
||||
for p, v := range pins {
|
||||
if v != "" {
|
||||
lines = append(lines, "pin "+p+" "+v)
|
||||
}
|
||||
}
|
||||
if fp == "" || len(lines) == 0 {
|
||||
return fp
|
||||
}
|
||||
sort.Strings(lines)
|
||||
sum := sha256.Sum256([]byte(fp + "\n" + strings.Join(lines, "\n")))
|
||||
return meshcatalogclient.SourcePrefix2 + hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
@@ -31,6 +31,20 @@ func versionLineTakesASiblingDirectoryByItsContent(t *testing.T) {
|
||||
if got := ReplacedSiblings(goMod, repo); len(got) != 1 || got[0] != buildrun {
|
||||
t.Fatalf("siblings %v, want buildrun alone (mesh-sdk is another repository, replaced by a version)", got)
|
||||
}
|
||||
// Its mesh packages are every mesh package its go.mod requires, at any version, the zero version included (design 47
|
||||
// H-A-157); a replace line or a module outside the forge is none.
|
||||
for _, c := range []struct {
|
||||
goMod string
|
||||
want []string
|
||||
}{
|
||||
{string(goMod), []string{buildrun, sdk}},
|
||||
{"module x\n\nrequire " + sdk + " v0.0.0-00010101000000-000000000000 // indirect\nrequire github.com/nats-io/nats.go v1.37.0\n", []string{sdk}},
|
||||
{"module x\n\nreplace " + sdk + " => ../sdk\n", nil},
|
||||
} {
|
||||
if got := MeshRequiresOf([]byte(c.goMod)); strings.Join(got, ",") != strings.Join(c.want, ",") {
|
||||
t.Errorf("mesh packages of %q: %v, want %v", c.goMod, got, c.want)
|
||||
}
|
||||
}
|
||||
published := func(digest string) map[string]meshcatalog.PackagesByModuleValue {
|
||||
return map[string]meshcatalog.PackagesByModuleValue{buildrun: {Module: buildrun, Repository: repo, Requires: "r1",
|
||||
Packages: []meshcatalog.PackagesByModuleValuePackagesItem{{Package: buildrun, Digest: digest}}}}
|
||||
|
||||
@@ -17,6 +17,11 @@ import (
|
||||
type form struct {
|
||||
fp, none string
|
||||
versions sdkstep.PackageVersions
|
||||
// stands: fp is the held fingerprint of the build that reported the imports, made with the pins built (its held
|
||||
// packages) from the import form raw; pinnedOf keeps it while the pins are those (H-A-172).
|
||||
stands bool
|
||||
raw string
|
||||
built map[string]string
|
||||
}
|
||||
|
||||
// formOf is the module's fingerprint from its kept inputs (bases already its bases' fingerprints) and toolchains, in the
|
||||
@@ -24,15 +29,16 @@ type form struct {
|
||||
//
|
||||
// - **The import form**, for a module whose last trunk build reported its imports (kept.Imports) when every import is
|
||||
// published (the packages state): the content digest of each imported package and the requirements' digest of each
|
||||
// mesh Go module holding one. Its versions are each holding module's published commit (another repository's only).
|
||||
// mesh Go module holding one. Its versions are each holding module's published commit (another repository's only); an
|
||||
// import set naming none of another repository while the go.mod requires one keeps the version line's (refined).
|
||||
// - **A change of form never rebuilds by itself** (§4): kept.ImportForm is the import-form fingerprint of the inputs
|
||||
// the build that reported the imports was built from. While the import form computes to it, nothing the build was
|
||||
// built from changed: the module keeps the fingerprint it holds (prev), in whatever form, and its versions.
|
||||
// Otherwise the import form is written, and ImportForm moves with it.
|
||||
// - **The version line** otherwise (no import set, or an import none publishes): the version of each mesh Go module of
|
||||
// another repository its go.sum names, read as the indexer always read it (fingerprint.HeldVersionsOf), and the
|
||||
// content of each mesh Go module of its own repository its go.mod replaces by a directory (fingerprint.VersionLine),
|
||||
// so a change of that directory alone moves it (review finding 2). Its versions are the other repositories' alone.
|
||||
// another repository its go.mod requires, at any version (design 47 H-A-157), read as the indexer always read it
|
||||
// (fingerprint.HeldVersionsOf), and the content of each mesh Go module of its own repository its go.mod replaces by a
|
||||
// directory (fingerprint.VersionLine), so a change of that directory alone moves it (review finding 2). Its versions are the other repositories' alone.
|
||||
// A package of another repository none of them publishes takes the commit the gate's source names for its repository
|
||||
// (heads, design 47 G A-365).
|
||||
// - **An accepted series** (DLV-I6): published and pins come from acceptedSeries, so a mesh Go module of another
|
||||
@@ -51,10 +57,12 @@ func formOf(module, repository string, kept *meshcatalog.ModulesByModuleValueInp
|
||||
case fp == "":
|
||||
return form{none: none}
|
||||
case kept.ImportForm == fp && prev.Fp2 != "":
|
||||
return form{fp: prev.Fp2, versions: prev.Packages} // the build that reported the imports stands
|
||||
// The build that reported the imports stands, and so do its versions.
|
||||
return form{fp: prev.Fp2, versions: refined(sdkstep.PackageVersions(prev.Packages), r.Versions, module, repository, kept, held, pins, heads),
|
||||
stands: true, raw: fp, built: prev.Packages}
|
||||
}
|
||||
kept.ImportForm = fp
|
||||
return form{fp: fp, versions: r.Versions}
|
||||
return form{fp: fp, versions: refined(r.Versions, nil, module, repository, kept, held, pins, heads)}
|
||||
}
|
||||
if logf != nil {
|
||||
logf("%s: no published package holds %s, so its fingerprint takes the version line", module, strings.Join(r.Missing, ", "))
|
||||
@@ -63,6 +71,34 @@ func formOf(module, repository string, kept *meshcatalog.ModulesByModuleValueInp
|
||||
// The fingerprint written here is the version line's, not the import form's: no import form is held for the inputs it was
|
||||
// computed from, so a later pass whose import form computes the same does not take this one for the build's (issue 1966).
|
||||
kept.ImportForm = ""
|
||||
versions := lineVersions(module, repository, kept, held, pins, heads, logf)
|
||||
in.Packages = fingerprint.VersionLine(repository, kept.Packages, versions, published)
|
||||
fp, none := fingerprint.Of(in)
|
||||
if fp == "" {
|
||||
return form{none: none}
|
||||
}
|
||||
return form{fp: fp, versions: versions}
|
||||
}
|
||||
|
||||
// refined are an import form's versions: the first of sets that names a package, else, while the module's go.mod
|
||||
// requires a mesh package of another repository (kept.Packages), the version line's, so a build's reported imports
|
||||
// refine the go.mod's set and never empty it (design 47 H-A-157); nil when nothing names one.
|
||||
func refined(versions, then sdkstep.PackageVersions, module, repository string, kept *meshcatalog.ModulesByModuleValueInputs2, held map[string]meshcatalog.ModulesByModuleValue, pins sdkstep.PackageVersions, heads map[string]string) sdkstep.PackageVersions {
|
||||
switch {
|
||||
case len(versions) > 0:
|
||||
return versions
|
||||
case len(then) > 0:
|
||||
return then
|
||||
case len(fingerprint.OtherPackages(kept.Packages, repository)) > 0:
|
||||
return lineVersions(module, repository, kept, held, pins, heads, nil)
|
||||
}
|
||||
return versions
|
||||
}
|
||||
|
||||
// lineVersions are the version line's versions of the mesh Go modules of other repositories the module's go.mod requires
|
||||
// (kept.Packages): each as the indexer has always read it, an accepted series' pin over it, else the commit the gate's
|
||||
// source names for its repository.
|
||||
func lineVersions(module, repository string, kept *meshcatalog.ModulesByModuleValueInputs2, held map[string]meshcatalog.ModulesByModuleValue, pins sdkstep.PackageVersions, heads map[string]string, logf func(string, ...any)) sdkstep.PackageVersions {
|
||||
others := fingerprint.OtherPackages(kept.Packages, repository)
|
||||
versions, unpublished := fingerprint.HeldVersionsOf(others, held)
|
||||
for p, v := range pins { // an accepted series' newest published version (acceptedSeries, DLV-I6)
|
||||
@@ -90,12 +126,7 @@ func formOf(module, repository string, kept *meshcatalog.ModulesByModuleValueInp
|
||||
if len(unpublished) > 0 && logf != nil {
|
||||
logf("%s: no module the indexer holds publishes %s, so its fingerprint takes no version of it", module, strings.Join(unpublished, ", "))
|
||||
}
|
||||
in.Packages = fingerprint.VersionLine(repository, kept.Packages, versions, published)
|
||||
fp, none := fingerprint.Of(in)
|
||||
if fp == "" {
|
||||
return form{none: none}
|
||||
}
|
||||
return form{fp: fp, versions: versions}
|
||||
return versions
|
||||
}
|
||||
|
||||
// inputsFrom is the fingerprint's inputs of module m from its kept inputs and the given toolchains, no package of
|
||||
|
||||
@@ -118,10 +118,39 @@ func TestImportsReportedMoveTheFingerprintOnlyWithAnImportedPackage(t *testing.T
|
||||
t.Fatal(err)
|
||||
}
|
||||
moved, _ := moduleOf(t, mods, "avahi")
|
||||
if moved.Fp2 == before.Fp2 || moved.Fp2 != moved.Inputs2.ImportForm || moved.Packages[meshSDK] != c3 {
|
||||
t.Fatalf("a change of an imported package: fp2 %q (was %q), import form %q, versions %v; want the import form at c3",
|
||||
if pin, kept := moved.Packages[meshSDK]; moved.Fp2 == before.Fp2 || moved.Fp2 != moved.Inputs2.ImportForm || !kept || pin != "" {
|
||||
t.Fatalf("a change of an imported package: fp2 %q (was %q), import form %q, versions %v; want the import form, mesh-sdk kept unpinned (none released, H-A-161)",
|
||||
moved.Fp2, before.Fp2, moved.Inputs2.ImportForm, moved.Packages)
|
||||
}
|
||||
|
||||
// mesh-sdk's module releases 0.1.97 while the registry has published nothing of its Go package: the module's release
|
||||
// number is never the package's pin (H-A-183), so avahi stays unpinned and where it was.
|
||||
ix.Versions = newMem()
|
||||
ix.Versions.(*memStore).put(meshcatalog.VersionsByModuleVersionKey("mesh-sdk", "0.1.97"), []byte(`{}`))
|
||||
if err := ix.Recompute(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m, _ := moduleOf(t, mods, "avahi"); m.Fp2 != moved.Fp2 || m.Packages[meshSDK] != "" {
|
||||
t.Fatalf("after mesh-sdk's module release alone: fp2 %q (was %q), versions %v; want it unpinned where it was", m.Fp2, moved.Fp2, m.Packages)
|
||||
}
|
||||
// The registry publishes its Go package at 0.1.20, another version than the module's release: the import form's
|
||||
// fingerprint takes that pin and moves, once (H-A-172, H-A-183).
|
||||
if _, err := ix.HearVersion(ctx, HeardVersion{Package: "mesh-sdk/go", Seat: "go-package-registry", Version: "0.1.20", Commit: c3, GoModule: meshSDK, Published: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for range 2 {
|
||||
if err := ix.Recompute(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if m, _ := moduleOf(t, mods, "avahi"); m.Fp2 == moved.Fp2 || m.Fp2 == "" || m.Packages[meshSDK] != "v0.1.20" || m.Inputs2.ImportForm != moved.Inputs2.ImportForm {
|
||||
t.Fatalf("after mesh-sdk's release: fp2 %q (was %q), versions %v, import form %q; want a new fingerprint pinned to v0.1.20, the import form kept",
|
||||
m.Fp2, moved.Fp2, m.Packages, m.Inputs2.ImportForm)
|
||||
} else if err := ix.Recompute(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if again, _ := moduleOf(t, mods, "avahi"); again.Fp2 != m.Fp2 {
|
||||
t.Fatalf("a pass with nothing moved moved the pinned fingerprint %q to %q", m.Fp2, again.Fp2)
|
||||
}
|
||||
}
|
||||
|
||||
// A build of a fingerprint the module no longer holds says nothing of its inputs now: its imports are not kept.
|
||||
@@ -256,8 +285,8 @@ func anAcceptedSeriesPinsItsNewest(t *testing.T) {
|
||||
}
|
||||
}
|
||||
before := step("trunk at c1", trunk(c1))
|
||||
if before.Packages[meshSDK] != c1 {
|
||||
t.Fatalf("with no series published, versions %v, want the commit form c1", before.Packages)
|
||||
if pin, kept := before.Packages[meshSDK]; !kept || pin != "" {
|
||||
t.Fatalf("with no series published, versions %v, want mesh-sdk kept with no pin, never the commit (H-A-161)", before.Packages)
|
||||
}
|
||||
step("the build reports its imports", func() error {
|
||||
return ix.ApplyBuildEnd(ctx, buildEnd(t, "avahi", before.Fp2, meshSDK, meshSDK+"/announce"))
|
||||
@@ -266,12 +295,19 @@ func anAcceptedSeriesPinsItsNewest(t *testing.T) {
|
||||
if pinned.Fp2 == before.Fp2 || pinned.Packages[meshSDK] != "v0.1.15" {
|
||||
t.Fatalf("after 0.1.15: fp2 %q (was %q), versions %v; want moved by c3's announce, pinned to v0.1.15", pinned.Fp2, before.Fp2, pinned.Packages)
|
||||
}
|
||||
// A publish in a series avahi does not accept pins nothing and moves nothing (H-A-184).
|
||||
if m := step("0.2.0 published from c1", hear("0.2.0", c1)); m.Fp2 != pinned.Fp2 || m.Packages[meshSDK] != "v0.1.15" {
|
||||
t.Errorf("a publish of a series avahi does not accept moved it: %q %v", m.Fp2, m.Packages)
|
||||
}
|
||||
if m := step("trunk at c2", trunk(c2)); m.Fp2 != pinned.Fp2 || m.Packages[meshSDK] != "v0.1.15" {
|
||||
t.Errorf("a trunk merge moved avahi off its accepted series' newest: %q %v", m.Fp2, m.Packages)
|
||||
}
|
||||
// mesh-sdk's module releases 0.3.0: a module's release number is never a package's pin (H-A-183).
|
||||
ix.Versions = newMem()
|
||||
ix.Versions.(*memStore).put(meshcatalog.VersionsByModuleVersionKey("mesh-sdk", "0.3.0"), []byte(`{}`))
|
||||
if m := step("0.3.0 released", func() error { return nil }); m.Packages[meshSDK] != "v0.1.15" {
|
||||
t.Errorf("a module's release pinned its package: %v", m.Packages)
|
||||
}
|
||||
}
|
||||
|
||||
// Issue 1966 (the indexer started while the bus's storage was down): a read of the packages state that failed, or that
|
||||
@@ -370,22 +406,36 @@ func anUnreadPackagesStateWritesNoFingerprintAndFingerprintsAreComputedAgain(t *
|
||||
head := strings.Repeat("c", 40)
|
||||
contracts := "git.novox.be/novox/mesh-contracts/go/seats/findings"
|
||||
held := map[string]meshcatalog.ModulesByModuleValue{"other": {Repository: "novox/mesh-sdk", Commit: strings.Repeat("d", 40), Path: "go"}}
|
||||
// A build's imports of its own repository alone (design 47 H-A-157): they refine the go.mod's set and never empty it.
|
||||
own := "git.novox.be/novox/mesh-catalog/modules/avahi/lib"
|
||||
published := map[string]meshcatalog.PackagesByModuleValue{own: {Module: own, Repository: "novox/mesh-catalog", Commit: head, Requires: "r1",
|
||||
Packages: []meshcatalog.PackagesByModuleValuePackagesItem{{Package: own, Digest: "d1"}}}}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
heads map[string]string
|
||||
held map[string]meshcatalog.ModulesByModuleValue
|
||||
want string
|
||||
name string
|
||||
heads map[string]string
|
||||
held map[string]meshcatalog.ModulesByModuleValue
|
||||
imports []string
|
||||
want string
|
||||
}{
|
||||
{"the source names its repository", map[string]string{"novox/mesh-contracts": head}, held, head},
|
||||
{"the source names another repository only", map[string]string{"novox/mesh-sdk": head}, held, ""},
|
||||
{"the source names its repository", map[string]string{"novox/mesh-contracts": head}, held, nil, head},
|
||||
{"the source names another repository only", map[string]string{"novox/mesh-sdk": head}, held, nil, ""},
|
||||
{"a module the indexer holds publishes it", map[string]string{"novox/mesh-contracts": head},
|
||||
map[string]meshcatalog.ModulesByModuleValue{"c": {Repository: "novox/mesh-contracts", Commit: strings.Repeat("e", 40), Path: "go/seats/findings"}}, strings.Repeat("e", 40)},
|
||||
map[string]meshcatalog.ModulesByModuleValue{"c": {Repository: "novox/mesh-contracts", Commit: strings.Repeat("e", 40), Path: "go/seats/findings"}}, nil, strings.Repeat("e", 40)},
|
||||
{"a build's imports naming none of another repository keep the go.mod's", map[string]string{"novox/mesh-contracts": head}, held, []string{own}, head},
|
||||
} {
|
||||
kept := meshcatalog.ModulesByModuleValueInputs2{Tree: "t", Packages: []string{contracts}}
|
||||
f := formOf("avahi", "novox/mesh-catalog", &kept, nil, meshcatalog.ModulesByModuleValue{}, c.held, nil, nil, c.heads, nil)
|
||||
kept := meshcatalog.ModulesByModuleValueInputs2{Tree: "t", Packages: []string{contracts}, Imports: c.imports}
|
||||
f := formOf("avahi", "novox/mesh-catalog", &kept, nil, meshcatalog.ModulesByModuleValue{}, c.held, published, nil, c.heads, nil)
|
||||
if got := f.versions[contracts]; got != c.want {
|
||||
t.Errorf("%s: version of %s = %q, want %q", c.name, contracts, got, c.want)
|
||||
}
|
||||
if c.imports == nil {
|
||||
continue
|
||||
}
|
||||
// The build that reported them stands, its held versions empty (a version line read from go.sum).
|
||||
f = formOf("avahi", "novox/mesh-catalog", &kept, nil, meshcatalog.ModulesByModuleValue{Fp2: "src2:" + strings.Repeat("b", 64)}, c.held, published, nil, c.heads, nil)
|
||||
if got := f.versions[contracts]; got != c.want || f.fp != "src2:"+strings.Repeat("b", 64) {
|
||||
t.Errorf("%s, the build standing: version of %s = %q (fp %q), want %q", c.name, contracts, got, f.fp, c.want)
|
||||
}
|
||||
}
|
||||
})
|
||||
t.Run("formOf clears the import form when it writes the version line", func(t *testing.T) {
|
||||
|
||||
@@ -108,7 +108,7 @@ func (ix *Indexer) applyMoving(ctx context.Context, f whole, drop bool, moves ma
|
||||
if next.Manifest == "" {
|
||||
next.None2 = "the manifest cannot be read: it cannot be kept as a value"
|
||||
} else {
|
||||
packages := ix.goSumPackages(ctx, f, dir, cur, has)
|
||||
packages := ix.goPackages(ctx, f, dir, cur, has)
|
||||
in := inputsOfNamed(name, raw, f.Trees[dir], f.Locks[dir], objects[dir], packages, ix.baseFP(ctx))
|
||||
next.Inputs2, next.None2, next.Paths = in.Kept, in.None2, in.ArtifactPaths
|
||||
if in.Kept != nil && has && cur.Inputs2 != nil {
|
||||
@@ -365,79 +365,72 @@ func unreadInputs(tree string, locks, paths map[string]string, packages []string
|
||||
return in
|
||||
}
|
||||
|
||||
// goSumPackages are the version line's packages of the merge's directory dir (ADR 0573 §2, ADR 0593 §2): the mesh Go
|
||||
// modules of other repositories its go.sum names, and those of its own repository its go.mod replaces by a directory
|
||||
// (siblingPackages), which no go.sum line names. A directory with no go.sum is no Go module and has none.
|
||||
func (ix *Indexer) goSumPackages(ctx context.Context, f whole, dir string, cur meshcatalog.ModulesByModuleValue, has bool) []string {
|
||||
others := ix.otherPackages(ctx, f, dir, cur, has)
|
||||
// goPackages are the version line's packages of the merge's directory dir (ADR 0593 §2; design 47 H-A-157): the mesh Go
|
||||
// modules of other repositories its go.mod requires, at any version, and those of its own repository its go.mod replaces
|
||||
// by a directory (siblingPackages, H-A-136's sibling rule). A directory with no go.sum is no Go module and has none.
|
||||
func (ix *Indexer) goPackages(ctx context.Context, f whole, dir string, cur meshcatalog.ModulesByModuleValue, has bool) []string {
|
||||
if f.Locks[dir]["go.sum"] == "" && len(f.Packages[dir]) == 0 {
|
||||
return others
|
||||
return nil
|
||||
}
|
||||
ix.requirementsOf(ctx, f, dir, cur, has)
|
||||
goMod, read := ix.goModOf(ctx, f, dir)
|
||||
others := fingerprint.OtherPackages(f.Packages[dir], f.Repository) // carried: what the go.mod requires, read with the tree
|
||||
switch {
|
||||
case len(others) > 0:
|
||||
case read:
|
||||
others = fingerprint.OtherPackages(fingerprint.MeshRequiresOf(goMod), f.Repository)
|
||||
case has && cur.Inputs2 != nil:
|
||||
others = fingerprint.OtherPackages(cur.Inputs2.Packages, f.Repository)
|
||||
}
|
||||
var siblings []string
|
||||
if read {
|
||||
siblings = fingerprint.ReplacedSiblings(goMod, f.Repository)
|
||||
} else if has && cur.Inputs2 != nil {
|
||||
for _, p := range cur.Inputs2.Packages {
|
||||
if repo, _, ok := trunk.PackageRepository(p); ok && trunk.SameRepository(repo, f.Repository) {
|
||||
siblings = append(siblings, p)
|
||||
}
|
||||
}
|
||||
}
|
||||
siblings := ix.siblingPackages(ctx, f, dir, cur, has)
|
||||
if len(siblings) == 0 {
|
||||
return others
|
||||
}
|
||||
return slices.Compact(slices.Sorted(slices.Values(append(slices.Clone(others), siblings...))))
|
||||
}
|
||||
|
||||
// siblingPackages are the mesh Go modules of the merge's own repository the go.mod of dir replaces by a directory, read
|
||||
// from the forge (GoMod); where it cannot be read, the held module's own-repository packages stand, said.
|
||||
func (ix *Indexer) siblingPackages(ctx context.Context, f whole, dir string, cur meshcatalog.ModulesByModuleValue, has bool) []string {
|
||||
held := func() []string {
|
||||
if !has || cur.Inputs2 == nil {
|
||||
return nil
|
||||
}
|
||||
var out []string
|
||||
for _, p := range cur.Inputs2.Packages {
|
||||
if repo, _, ok := trunk.PackageRepository(p); ok && trunk.SameRepository(repo, f.Repository) {
|
||||
out = append(out, p)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
// goModOf is the go.mod of the merge's directory dir, read from the forge (GoMod), and whether it was read; where it
|
||||
// cannot be, the held module's packages stand, said.
|
||||
func (ix *Indexer) goModOf(ctx context.Context, f whole, dir string) ([]byte, bool) {
|
||||
if ix.GoMod == nil {
|
||||
return held()
|
||||
return nil, false
|
||||
}
|
||||
raw, err := ix.GoMod(ctx, f.Repository, f.Merge, dir)
|
||||
if err != nil {
|
||||
ix.logf("%s at %s: the go.mod of %q cannot be read, so its module keeps the directory replaces it held: %v", f.Repository, f.Merge, dir, err)
|
||||
return held()
|
||||
ix.logf("%s at %s: the go.mod of %q cannot be read, so its module keeps the packages it held: %v", f.Repository, f.Merge, dir, err)
|
||||
return nil, false
|
||||
}
|
||||
return fingerprint.ReplacedSiblings(raw, f.Repository)
|
||||
return raw, true
|
||||
}
|
||||
|
||||
// otherPackages are the mesh Go modules of other repositories the go.sum of the merge's directory dir names: those the
|
||||
// message carries (an older bridge), else the held module's while its go.sum's hash is unchanged, else the go.sum read
|
||||
// from the forge (GoSum).
|
||||
func (ix *Indexer) otherPackages(ctx context.Context, f whole, dir string, cur meshcatalog.ModulesByModuleValue, has bool) []string {
|
||||
if carried := f.Packages[dir]; len(carried) > 0 {
|
||||
return fingerprint.OtherPackages(carried, f.Repository) // its own repository's come with its tree
|
||||
}
|
||||
// requirementsOf keeps what the go.sum of the merge's directory dir requires in the requirements state
|
||||
// (keepRequirements): read from the forge (GoSum) when its hash moved, or this process has not kept it yet, whatever
|
||||
// packages the reader carried. It gives no package: those are the go.mod's (H-A-157).
|
||||
func (ix *Indexer) requirementsOf(ctx context.Context, f whole, dir string, cur meshcatalog.ModulesByModuleValue, has bool) {
|
||||
hash := f.Locks[dir]["go.sum"]
|
||||
if hash == "" {
|
||||
return nil
|
||||
if hash == "" || ix.GoSum == nil {
|
||||
return
|
||||
}
|
||||
if has && cur.Inputs2 != nil && trunk.SameRepository(cur.Repository, f.Repository) && ix.requirementsKept(f.Repository, dir) {
|
||||
for _, h := range cur.Inputs2.Locks {
|
||||
if h == hash {
|
||||
return fingerprint.OtherPackages(cur.Inputs2.Packages, f.Repository)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
if ix.GoSum == nil {
|
||||
if has && cur.Inputs2 != nil {
|
||||
return fingerprint.OtherPackages(cur.Inputs2.Packages, f.Repository)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
raw, err := ix.GoSum(ctx, f.Repository, f.Merge, dir)
|
||||
if err != nil {
|
||||
ix.logf("%s at %s: the go.sum of %q cannot be read, so its module keeps the packages it held: %v", f.Repository, f.Merge, dir, err)
|
||||
if has && cur.Inputs2 != nil {
|
||||
return fingerprint.OtherPackages(cur.Inputs2.Packages, f.Repository)
|
||||
}
|
||||
return nil
|
||||
ix.logf("%s at %s: the go.sum of %q cannot be read, so what it requires is not kept: %v", f.Repository, f.Merge, dir, err)
|
||||
return
|
||||
}
|
||||
ix.keepRequirements(ctx, f.Repository, dir, raw)
|
||||
return fingerprint.OtherPackages(fingerprint.MeshPackagesOf(raw), f.Repository)
|
||||
}
|
||||
|
||||
@@ -401,8 +401,36 @@ func TestARootModuleHasTheDirectoryDot(t *testing.T) { // 1615/AC3: empty means
|
||||
}
|
||||
|
||||
func TestAMergeAddingAModuleDirectoryWritesItsValueWithNoBuildRecordAnywhere(t *testing.T) { // I1: issue 1851, ADR 0501 clause 6
|
||||
ix, mods, _ := manifestsIndexer(t) // no build record of any module
|
||||
avahi := catalogueManifest(t, "avahi")
|
||||
// Its go.mod requires mesh-sdk at the zero version (ADR 0573 §2): the go.mod's require lines are its mesh packages, with
|
||||
// no build, carried by the reader or read through GoMod (design 47 H-A-157). Its go.sum is still read for the
|
||||
// requirements state, carried packages or not, so an older version it names is kept there (and cleared when gone).
|
||||
for _, carried := range [][]string{nil, {meshSDK}} {
|
||||
ix, mods, _ := manifestsIndexer(t)
|
||||
ix.Requirements = newMem()
|
||||
ix.GoMod = func(context.Context, string, string, string) ([]byte, error) {
|
||||
return []byte("module avahi\n\nrequire " + meshSDK + " v0.0.0-00010101000000-000000000000\n"), nil
|
||||
}
|
||||
ix.GoSum = func(context.Context, string, string, string) ([]byte, error) {
|
||||
return []byte(meshSDK + " v0.1.14 h1:x=\n"), nil
|
||||
}
|
||||
f := withManifests(mergedMsg(catalogueRepo, map[string]string{"modules/avahi": "newtree"}), map[string]string{"modules/avahi": avahi},
|
||||
map[string]map[string]string{"modules/avahi": {"go.sum": avahiGoSum}})
|
||||
if carried != nil {
|
||||
f.Packages = map[string][]string{"modules/avahi": carried}
|
||||
}
|
||||
if err := ix.Apply(context.Background(), f); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, _ := moduleOf(t, mods, "avahi")
|
||||
if m.Inputs2 == nil || !slices.Equal(m.Inputs2.Packages, []string{meshSDK}) {
|
||||
t.Errorf("carried %v: mesh packages %+v, want mesh-sdk, which its go.mod requires at the zero version", carried, m.Inputs2)
|
||||
}
|
||||
if _, kept := ix.Requirements.(*memStore).vals[meshSDK]; !kept {
|
||||
t.Errorf("carried %v: the requirements state holds no %s, which the go.sum names", carried, meshSDK)
|
||||
}
|
||||
}
|
||||
ix, mods, _ := manifestsIndexer(t) // no build record of any module
|
||||
f := withManifests(mergedMsg(catalogueRepo, map[string]string{"modules/avahi": "newtree"}), map[string]string{"modules/avahi": avahi},
|
||||
map[string]map[string]string{"modules/avahi": {"go.sum": avahiGoSum}})
|
||||
if err := ix.Apply(context.Background(), f); err != nil {
|
||||
|
||||
@@ -14,14 +14,15 @@ import (
|
||||
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
runner "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
)
|
||||
|
||||
// The indexer keeps the records pinning each catalog graph revision and removes a revision only on an event (design 47 G §153,
|
||||
// §178, §189, §215, §216, §579, A-169, A-171): its state pins holds one key per revision, `<branch>.<revision>` (the watched branch
|
||||
// and the revision's number, the branch escaped as a name is), whose value lists the records that pin it: a take-in's and each
|
||||
// running pipeline's (design 47 H §743, §749).
|
||||
// and the revision's number, the branch escaped as a name is), whose value lists the records that pin it: a take-in's, each
|
||||
// running pipeline's (design 47 H §743, §749) and each passed pipeline's release until a newer one is the want (H-A-168, H-A-170).
|
||||
const (
|
||||
// PinsState is the state's name; PinsCapMiB is its cap in the manifest (about 1,800 keys of about 0.2 KiB).
|
||||
PinsState = "pins"
|
||||
@@ -84,6 +85,19 @@ func (ix *Indexer) HeardPins(ctx context.Context, started *runner.PipelineStarte
|
||||
case started != nil && started.Revision > 0 && started.Pipeline != "":
|
||||
return ix.pin(ctx, PinKey(started.Branch, started.Revision), PipelineRecord(started.Pipeline))
|
||||
case ended != nil && ended.Revision > 0 && ended.Pipeline != "":
|
||||
// A passed pipeline's release pins the revision its movers' entries live in, before the pipeline's own pin is freed,
|
||||
// one record per mover that released a version in it (H-A-168, H-A-170, H-A-173); a mover that released none, and a run
|
||||
// on request's pipeline, which delivers nothing, pin none.
|
||||
if ended.Outcome == runner.PipelineEndedOutcomePassed && ended.Run == "" {
|
||||
for _, m := range ended.Movers {
|
||||
if m.Version == "" {
|
||||
continue
|
||||
}
|
||||
if err := ix.pin(ctx, PinKey(ended.Branch, ended.Revision), ReleaseRecord(ended.Branch, m.Module)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
record := PipelineRecord(ended.Pipeline)
|
||||
if err := ix.unpin(ctx, PinKey(ended.Branch, ended.Revision), record); err != nil {
|
||||
return err
|
||||
@@ -93,6 +107,77 @@ func (ix *Indexer) HeardPins(ctx context.Context, started *runner.PipelineStarte
|
||||
return nil
|
||||
}
|
||||
|
||||
// ReleaseRecord is the record a passed pipeline's release pins its revision under, one per mover: `release.<branch>.<module>`,
|
||||
// each name escaped as a key's name is (design 47 H-A-168, H-A-170).
|
||||
func ReleaseRecord(branch, module string) string {
|
||||
return "release." + keys.EscapeName(branch) + "." + keys.EscapeName(module)
|
||||
}
|
||||
|
||||
// HeardPipelineWanted applies one pipeline-wanted of the seat mesh-wants, said when a pipeline's wants are written to the end
|
||||
// (design 47 H §751, H-A-168, H-A-170, H-A-173, H-A-174). Outcome running: a module whose release record the pipeline's own
|
||||
// kept revision holds released its newer version there, and that is the want everywhere, so the module's record is freed from
|
||||
// every revision of the branch below it; a mover the revision holds no release of (superseded, no version released) keeps its
|
||||
// older release's revision. Outcome tier-late frees nothing: its later tiers' wants are not written, though their release
|
||||
// records are held, and the same pipeline's later running frees them. A pipeline whose revision is not kept frees nothing and
|
||||
// says so; one applied twice changes nothing.
|
||||
func (ix *Indexer) HeardPipelineWanted(ctx context.Context, branch string, source int64, outcome string) error {
|
||||
if ix.Pins == nil || ix.Revisions == nil {
|
||||
return errors.New("the indexer keeps no state " + PinsState)
|
||||
}
|
||||
if outcome != meshwants.PipelineWantedOutcomeRunning {
|
||||
ix.logf("pins: pipeline-wanted of %s %d is %s: it frees no release until its wants are written everywhere (H-A-174)", branch, source, outcome)
|
||||
return nil
|
||||
}
|
||||
name, err := revisionOfSource(ctx, ix.Revisions, branch, source)
|
||||
if err != nil {
|
||||
ix.logf("pins: pipeline-wanted of %s %d frees no release: its revision is not kept: %v", branch, source, err)
|
||||
return nil
|
||||
}
|
||||
rev, _, err := ReadRevision(ctx, ix.Revisions, name)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s: %w", name, err)
|
||||
}
|
||||
own, _, found, err := ix.Pins.Get(ctx, PinKey(branch, rev.Revision))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var held PinValue
|
||||
if found {
|
||||
_ = json.Unmarshal(own, &held)
|
||||
}
|
||||
var released []meshcatalog.TakeInEntry
|
||||
for _, m := range rev.Movers {
|
||||
if slices.Contains(held.Records, ReleaseRecord(branch, m.Module)) {
|
||||
released = append(released, m)
|
||||
}
|
||||
}
|
||||
pinKeys, err := ix.Pins.Keys(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, k := range pinKeys {
|
||||
b, r, ok := ParsePinKey(k)
|
||||
if !ok || b != branch || r >= rev.Revision {
|
||||
continue
|
||||
}
|
||||
for _, m := range released {
|
||||
v, _, found, err := ix.Pins.Get(ctx, k)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var pv PinValue
|
||||
record := ReleaseRecord(branch, m.Module)
|
||||
if !found || json.Unmarshal(v, &pv) != nil || !slices.Contains(pv.Records, record) {
|
||||
continue
|
||||
}
|
||||
if err := ix.unpin(ctx, k, record); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// unpinElsewhere frees a record on every other key that lists it: the branch's previous revision the take-in pinned for the
|
||||
// same pipeline (H-A-147), which the pipeline-ended does not name.
|
||||
func (ix *Indexer) unpinElsewhere(ctx context.Context, record string) error {
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
runner "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
)
|
||||
|
||||
// Design 47 H §749 (G §579, A-169, A-171): the indexer pins a revision on a pipeline-started that names it and frees it on its
|
||||
@@ -92,6 +93,55 @@ func TestTheIndexerRemovesARevisionOnlyWhenNoRecordPinsItAndItIsNotCurrent(t *te
|
||||
if keys, _ := ix.Pins.Keys(ctx); slices.Contains(keys, "main.1") {
|
||||
t.Errorf("the state holds %v after the pin drop, want no main.1", keys)
|
||||
}
|
||||
// H-A-168, H-A-170, H-A-173: a passed pipeline's release keeps its revision past a later failed pipeline; a passed one whose
|
||||
// mover released no version pins nothing and its pipeline-wanted frees nothing; the next one that released a version frees
|
||||
// the older release on its pipeline-wanted running (its wants written to the end), not on a tier-late one (H-A-174), and the
|
||||
// revision goes.
|
||||
mover := []meshcatalog.TakeInEntry{{Module: "reader", Commit: "c", Fingerprint: "f", Repository: "novox/r"}}
|
||||
take := func(source int64) {
|
||||
t.Helper()
|
||||
e := reader
|
||||
e.Commit = fmt.Sprintf("r%d", source)
|
||||
if err := ix.KeepBranchEntries(ctx, "main", source, map[string]meshcatalog.CatalogEntry{"reader": e}, mover); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
end := func(source int64, outcome, version string) {
|
||||
t.Helper()
|
||||
e := runner.PipelineEnded{Branch: "main", Pipeline: fmt.Sprintf("main.%d", source), Revision: source, Source: source, Outcome: outcome,
|
||||
Movers: []runner.MoverEnded{{Module: "reader", Fingerprint: "f", Version: version}}}
|
||||
if err := ix.HeardPins(ctx, nil, &e); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ix.JudgePins(ctx)
|
||||
}
|
||||
wanted := func(source int64, outcome string) {
|
||||
t.Helper()
|
||||
if err := ix.HeardPipelineWanted(ctx, "main", source, outcome); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ix.JudgePins(ctx)
|
||||
}
|
||||
take(10)
|
||||
end(10, runner.PipelineEndedOutcomePassed, "0.1.10")
|
||||
take(11)
|
||||
end(11, runner.PipelineEndedOutcomeFailed, "")
|
||||
want("a passed release past a later failed pipeline", "main.10", "main.11")
|
||||
take(12)
|
||||
end(12, runner.PipelineEndedOutcomePassed, "")
|
||||
wanted(12, meshwants.PipelineWantedOutcomeRunning)
|
||||
want("a passed pipeline releasing no version frees no older release", "main.10", "main.12")
|
||||
take(13)
|
||||
end(13, runner.PipelineEndedOutcomePassed, "0.1.13")
|
||||
want("two releases, the unreleased one between them gone", "main.10", "main.13")
|
||||
wanted(13, meshwants.PipelineWantedOutcomeTierLate) // H-A-174: its later tiers' wants are not written, so it frees nothing
|
||||
want("a tier-late pipeline-wanted", "main.10", "main.13")
|
||||
wanted(13, meshwants.PipelineWantedOutcomeRunning)
|
||||
wanted(13, meshwants.PipelineWantedOutcomeRunning) // heard twice: changes nothing more
|
||||
want("the newer release written everywhere", "main.13")
|
||||
if v, _, _, _ := ix.Pins.Get(ctx, "main.13"); !slices.Equal(mustRecords(t, v), []string{ReleaseRecord("main", "reader")}) {
|
||||
t.Errorf("the pin of main.13 holds %s, want its release alone", v)
|
||||
}
|
||||
}
|
||||
|
||||
func mustRecords(t *testing.T, v []byte) []string {
|
||||
@@ -107,7 +157,8 @@ func mustRecords(t *testing.T, v []byte) []string {
|
||||
// H-A-147 (§743, §749, H-A-108 (3)): a take-in with a mover pins, in its write, the branch's previous revision (the newest kept
|
||||
// before the write) for the same pipeline record as its own, so the validate step's case before can read it while the pipeline
|
||||
// runs, and names it as its base on the revision it writes (H-A-151, H-A-154); a take-in with no revision before it pins only
|
||||
// its own and names no base; the pipeline's end frees both.
|
||||
// its own and names no base; the pipeline's end frees both. A passed pipeline's end pins its revision under its mover's release
|
||||
// record instead (H-A-168, H-A-170), so it stays when a later take-in makes it no longer current.
|
||||
func TestATakeInPinsTheBranchsPreviousRevisionForItsPipeline(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
reader := meshcatalog.CatalogEntry{Module: "reader", Repository: "novox/r", Manifest: map[string]any{}}
|
||||
@@ -117,11 +168,21 @@ func TestATakeInPinsTheBranchsPreviousRevisionForItsPipeline(t *testing.T) {
|
||||
before []int64 // sources taken in with no mover before the take-in of source 5
|
||||
whileRuns []string
|
||||
afterTheEnd []string
|
||||
base int64 // the base main.5 names (H-A-154)
|
||||
base int64 // the base main.5 names (H-A-154)
|
||||
passed bool // main.5's pipeline passed: its release pins main.5
|
||||
unreleased bool // its mover released no version there, so nothing pins main.5 (H-A-173)
|
||||
then []int64 // sources taken in with no mover after the end
|
||||
released []string
|
||||
}{
|
||||
{name: "the previous revision kept and pinned", before: []int64{3, 4}, whileRuns: []string{"main.4", "main.5"}, afterTheEnd: []string{"main.5"}, base: 4},
|
||||
{name: "the newest kept before, whatever its number", before: []int64{2}, whileRuns: []string{"main.2", "main.5"}, afterTheEnd: []string{"main.5"}, base: 2},
|
||||
{name: "no revision before it pins only its own", whileRuns: []string{"main.5"}, afterTheEnd: []string{"main.5"}},
|
||||
{name: "a passed pipeline's release keeps its revision once no longer current", before: []int64{4}, whileRuns: []string{"main.4", "main.5"},
|
||||
base: 4, passed: true, then: []int64{6}, afterTheEnd: []string{"main.5", "main.6"}, released: []string{"main.5"}},
|
||||
{name: "a passed pipeline whose mover released no version pins nothing", before: []int64{4}, whileRuns: []string{"main.4", "main.5"},
|
||||
base: 4, passed: true, unreleased: true, then: []int64{6}, afterTheEnd: []string{"main.6"}},
|
||||
{name: "a failed pipeline's revision goes once no longer current", before: []int64{4}, whileRuns: []string{"main.4", "main.5"},
|
||||
base: 4, then: []int64{6}, afterTheEnd: []string{"main.6"}},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
ix := &Indexer{Module: testSelf, Revisions: newMemObjects(1 << 20), Pins: newMem(), Findings: newFindings(),
|
||||
@@ -156,17 +217,32 @@ func TestATakeInPinsTheBranchsPreviousRevisionForItsPipeline(t *testing.T) {
|
||||
t.Errorf("the pin of %s holds %s, want the pipeline's record", k, v)
|
||||
}
|
||||
}
|
||||
if err := ix.HeardPins(ctx, nil, &runner.PipelineEnded{Branch: "main", Pipeline: "main.5", Revision: 5}); err != nil {
|
||||
end := runner.PipelineEnded{Branch: "main", Pipeline: "main.5", Revision: 5, Outcome: runner.PipelineEndedOutcomeFailed,
|
||||
Movers: []runner.MoverEnded{{Module: "reader", Fingerprint: "f", Version: "0.1.5"}}}
|
||||
if tc.unreleased {
|
||||
end.Movers[0].Version = ""
|
||||
}
|
||||
if tc.passed {
|
||||
end.Outcome = runner.PipelineEndedOutcomePassed
|
||||
}
|
||||
if err := ix.HeardPins(ctx, nil, &end); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ix.JudgePins(ctx)
|
||||
for _, s := range tc.then {
|
||||
keep(s, nil)
|
||||
}
|
||||
if got := kept(); !slices.Equal(got, tc.afterTheEnd) {
|
||||
t.Errorf("after the pipeline's end: kept %v, want %v", got, tc.afterTheEnd)
|
||||
}
|
||||
if keys, _ := ix.Pins.Keys(ctx); len(keys) > 0 {
|
||||
for _, k := range keys {
|
||||
if v, _, _, _ := ix.Pins.Get(ctx, k); len(mustRecords(t, v)) > 0 {
|
||||
t.Errorf("after the pipeline's end the pin of %s holds %s, want none", k, v)
|
||||
want := []string{}
|
||||
if slices.Contains(tc.released, k) {
|
||||
want = []string{ReleaseRecord("main", "reader")}
|
||||
}
|
||||
if v, _, _, _ := ix.Pins.Get(ctx, k); !slices.Equal(append([]string{}, mustRecords(t, v)...), want) {
|
||||
t.Errorf("after the pipeline's end the pin of %s holds %s, want %v", k, v, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -41,7 +41,7 @@ func (r Requirement) Versions() []string {
|
||||
return out
|
||||
}
|
||||
|
||||
// RequiredOf is, per mesh Go module a go.sum names (as fingerprint.MeshPackagesOf takes them), the highest version it
|
||||
// RequiredOf is, per mesh Go module a go.sum names (a line under trunk.ForgeModulePrefix whose path names a repository), the highest version it
|
||||
// names, its module line or its go.mod line alike.
|
||||
func RequiredOf(goSum []byte) map[string]string {
|
||||
out := map[string]string{}
|
||||
|
||||
@@ -108,7 +108,7 @@ func (ix *Indexer) SeedModulesFrom(ctx context.Context, defs map[string]seatdef.
|
||||
seat := seatOf[a.Ref]
|
||||
sa := SeedArtifact{Name: name, Seat: seat}
|
||||
in := nodebuildgo.StepsInput{Module: e.Module, Artifact: nodebuildgo.DetectedStepsByModuleArtifactKey(e.Module, name), Fingerprint: e.Fp,
|
||||
Fetch: &nodebuildgo.Fetch{Repository: e.Repository, Commit: e.Commit, Packages: maps.Clone(e.Packages)}}
|
||||
Fetch: &nodebuildgo.Fetch{Repository: e.Repository, Commit: e.Commit, Packages: PinsOf(e.Packages)}}
|
||||
// The module's directory and its named paths go together, only when the entry names its paths (ADR 0499 clause
|
||||
// 4); a request without them builds from the module's directory (sdk/buildstep FetchRequest).
|
||||
if paths := namedPaths(e, name); len(paths) > 0 {
|
||||
|
||||
@@ -24,7 +24,7 @@ func TestTheSeedAsksStepsOfTheBuildSeatsOfTheTrunkEntries(t *testing.T) {
|
||||
fp := "src2:" + strings.Repeat("a", 64)
|
||||
entries := map[string]meshcatalog.CatalogEntry{"avahi": {Module: "avahi", Repository: "novox/r", Commit: "c1", Path: "mods/avahi", Fp: fp, Manifest: map[string]any{},
|
||||
Artifacts: []meshcatalog.Artifact{{Ref: "avahi:program", Language: "go"}, {Ref: "avahi:image", Language: "go", Kind: "image"}},
|
||||
Packages: map[string]string{"git.novox.be/novox/mesh-sdk": "c9"},
|
||||
Packages: map[string]string{"git.novox.be/novox/mesh-sdk": "v0.1.9", "git.novox.be/novox/mesh-contracts/go": ""}, // the second none released (H-A-161)
|
||||
Inputs: &meshcatalog.EntryInputs{Paths: map[string][]string{"program": {"cmd"}}}}}
|
||||
rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: 3, Kind: "trunk"}, entries)
|
||||
if err != nil {
|
||||
@@ -71,9 +71,12 @@ func TestTheSeedAsksStepsOfTheBuildSeatsOfTheTrunkEntries(t *testing.T) {
|
||||
t.Errorf("the image artifact is %+v", a)
|
||||
}
|
||||
in := got[0]
|
||||
if in.Module != "avahi" || in.Artifact != "avahi.program" || in.Fingerprint != fp || in.Fetch == nil || in.Fetch.Repository != "novox/r" || in.Fetch.Commit != "c1" || in.Fetch.Dir != "mods/avahi" || len(in.Fetch.Paths) != 1 || in.Fetch.Packages["git.novox.be/novox/mesh-sdk"] != "c9" {
|
||||
if in.Module != "avahi" || in.Artifact != "avahi.program" || in.Fingerprint != fp || in.Fetch == nil || in.Fetch.Repository != "novox/r" || in.Fetch.Commit != "c1" || in.Fetch.Dir != "mods/avahi" || len(in.Fetch.Paths) != 1 || in.Fetch.Packages["git.novox.be/novox/mesh-sdk"] != "v0.1.9" {
|
||||
t.Errorf("the steps input is %+v", in)
|
||||
}
|
||||
if _, pinned := in.Fetch.Packages["git.novox.be/novox/mesh-contracts/go"]; pinned {
|
||||
t.Errorf("a package kept with no version is pinned in the fetch: %v", in.Fetch.Packages)
|
||||
}
|
||||
if len(w.Held()) != 0 {
|
||||
t.Fatal("a call unanswered for no time wrote its finding")
|
||||
}
|
||||
|
||||
@@ -9,7 +9,7 @@ type RepoRead struct {
|
||||
Trees map[string]string
|
||||
Manifests map[string]string
|
||||
Locks map[string]map[string]string
|
||||
// Packages are the mesh packages each directory's go.sum names (fingerprint.MeshPackagesOf, novox/hq ADR 0573 §2).
|
||||
// Packages are the mesh packages each directory's go.mod requires (fingerprint.MeshRequiresOf, design 47 H-A-157).
|
||||
Packages map[string][]string
|
||||
}
|
||||
|
||||
|
||||
@@ -16,6 +16,7 @@ import (
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
|
||||
"mesh-delivery-indexer/internal/fingerprint"
|
||||
"mesh-delivery-indexer/judge"
|
||||
)
|
||||
|
||||
@@ -152,11 +153,13 @@ func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]mesh
|
||||
}
|
||||
ix.recompute.Lock()
|
||||
defer ix.recompute.Unlock()
|
||||
return ix.recomputeLocked(ctx, overlay, w, repository)
|
||||
return ix.recomputeLocked(ctx, overlay, w, repository, nil)
|
||||
}
|
||||
|
||||
// recomputeLocked is recomputeModules with ix.recompute held by the caller (a hold that moves, holds.go).
|
||||
func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]meshcatalog.ModulesByModuleValue, w *mergeWrite, repository string) error {
|
||||
//
|
||||
// only, when not nil, names the modules the pass moves (a re-pin, H-A-165): every other stands as held.
|
||||
func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]meshcatalog.ModulesByModuleValue, w *mergeWrite, repository string, only map[string]bool) error {
|
||||
if ix.Manifests == nil {
|
||||
return ix.writeOverlay(ctx, overlay, nil, w)
|
||||
}
|
||||
@@ -287,18 +290,20 @@ func (ix *Indexer) recomputeLocked(ctx context.Context, overlay map[string]meshc
|
||||
r.none = strings.Join(missing, "; ")
|
||||
case noToolchain != "":
|
||||
r.none = noToolchain
|
||||
if fp, pkgs, ok := stillComputed(module, kept, k, toolchains, held, pub, pins, ix.sourceHeads()); ok {
|
||||
if fp, pkgs, ok := stillComputed(module, kept, k, toolchains, held, pub, pins, ix.sourceHeads(), series); ok {
|
||||
r = result{fp: fp, tcs: k.held.Toolchains, pkgs: pkgs}
|
||||
}
|
||||
default:
|
||||
f := formOf(module, k.cur.Repository, &kept, toolchains, k.held, held, pub, pins, ix.sourceHeads(), ix.logf)
|
||||
r.fp, r.none = f.fp, f.none
|
||||
if r.fp != "" {
|
||||
r.tcs, r.pkgs = toolchainsUsed(toolchains), f.versions
|
||||
// a pin is a released version, never a commit (H-A-161), and the version line's fingerprint takes it (H-A-167)
|
||||
r.fp, r.pkgs = pinnedOf(f, series, k.manifest)
|
||||
r.tcs = toolchainsUsed(toolchains)
|
||||
}
|
||||
}
|
||||
if _, merged := overlay[module]; !merged && repository != "" && !trunk.SameRepository(k.cur.Repository, repository) &&
|
||||
maps.Equal(k.held.Packages, map[string]string(r.pkgs)) {
|
||||
if _, merged := overlay[module]; !merged && (repository != "" && !trunk.SameRepository(k.cur.Repository, repository) &&
|
||||
!importsOf(r.pkgs, repository) || only != nil && !only[module]) {
|
||||
// A module registered for another repository is moved by this repository's commit only when it imports a package the
|
||||
// commit moved (A-476a, ADR 0593 §3); any other stands as held (A-476 c), and so it is for a base of this repository's.
|
||||
r = result{fp: k.held.Fp2, none: k.held.None2, tcs: k.held.Toolchains, pkgs: sdkstep.PackageVersions(k.held.Packages)}
|
||||
@@ -492,7 +497,7 @@ func hasContext(m catalogue.Manifest) bool {
|
||||
// with the toolchains it was computed with in place of the languages agreed on none, give it again, in its form. Any
|
||||
// other input that moved (a merge, a base, an agreed toolchain, an imported package) gives another fingerprint, and the
|
||||
// module has none until they agree.
|
||||
func stillComputed(module string, kept meshcatalog.ModulesByModuleValueInputs2, k *known, agreed map[string]meshcatalogclient.Toolchain2, held map[string]meshcatalog.ModulesByModuleValue, published map[string]meshcatalog.PackagesByModuleValue, pins sdkstep.PackageVersions, heads map[string]string) (string, sdkstep.PackageVersions, bool) {
|
||||
func stillComputed(module string, kept meshcatalog.ModulesByModuleValueInputs2, k *known, agreed map[string]meshcatalogclient.Toolchain2, held map[string]meshcatalog.ModulesByModuleValue, published map[string]meshcatalog.PackagesByModuleValue, pins sdkstep.PackageVersions, heads map[string]string, series map[string]SeriesPackage) (string, sdkstep.PackageVersions, bool) {
|
||||
if k.held.Fp2 == "" {
|
||||
return "", nil, false
|
||||
}
|
||||
@@ -509,7 +514,11 @@ func stillComputed(module string, kept meshcatalog.ModulesByModuleValueInputs2,
|
||||
toolchains[artifact] = meshcatalogclient.Toolchain2{Language: language, Version: used.Version, Recipe: used.Recipe}
|
||||
}
|
||||
f := formOf(module, k.held.Repository, &kept, toolchains, k.held, held, published, pins, heads, nil)
|
||||
return f.fp, f.versions, f.none == "" && f.fp == k.held.Fp2
|
||||
if f.none != "" || f.fp == "" {
|
||||
return "", nil, false
|
||||
}
|
||||
fp, pkgs := pinnedOf(f, series, k.manifest)
|
||||
return fp, pkgs, fp == k.held.Fp2
|
||||
}
|
||||
|
||||
// toolchainsUsed are the toolchains a fingerprint was computed with, by language (an artifact's toolchain is its language's).
|
||||
@@ -577,3 +586,41 @@ func (ix *Indexer) fillPaths(ctx context.Context) error {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// importsOf says whether pkgs name a package of repository: a pin is a released version (H-A-161), so a commit of
|
||||
// repository moves a module of another only through a package of it the module imports (A-476a).
|
||||
func importsOf(pkgs sdkstep.PackageVersions, repository string) bool {
|
||||
for p := range pkgs {
|
||||
if repo, _, ok := trunk.PackageRepository(p); ok && trunk.SameRepository(repo, repository) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Repin moves the pins of every module whose packages hold a package of module (a held module's directory holds it) to
|
||||
// module's newest released version, the pass of H-A-161 for those modules alone: a version-released of module starts it
|
||||
// (design 47 H-A-165), so pins move on the event and never wait for a merge or a hand act. It answers how many it named.
|
||||
func (ix *Indexer) Repin(ctx context.Context, module string) (int, error) {
|
||||
if ix.Modules == nil || ix.Manifests == nil {
|
||||
return 0, nil
|
||||
}
|
||||
held, err := ix.heldModules(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
only := map[string]bool{}
|
||||
for key, m := range held {
|
||||
for p := range m.Packages {
|
||||
if moduleOfKey(fingerprint.HolderOf(p, held)) == module {
|
||||
only[key] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(only) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
ix.recompute.Lock()
|
||||
defer ix.recompute.Unlock()
|
||||
return len(only), ix.recomputeLocked(ctx, nil, nil, "", only)
|
||||
}
|
||||
|
||||
@@ -17,6 +17,7 @@ import (
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
@@ -757,7 +758,7 @@ func TestAModuleKeepsTheToolchainsItsFingerprintWasComputedWith(t *testing.T) {
|
||||
t.Errorf("no published mesh-sdk: fp2 %q none2 %q", u.Fp2, u.None2)
|
||||
}
|
||||
tools, _ := moduleOf(t, mods, "mesh-tools")
|
||||
sdkMerge := func(commit, tree string) meshcatalog.ModulesByModuleValue {
|
||||
sdkMerge := func(commit, tree, pin string) meshcatalog.ModulesByModuleValue {
|
||||
t.Helper()
|
||||
f := whole{Repository: "novox/mesh-sdk", Branch: "feat/x", Head: h1,
|
||||
Merge: commit, Root: tree, Trees: map[string]string{"": tree}, Manifests: map[string]string{"": `{"module":"mesh-sdk","version":"1.0"}`}}
|
||||
@@ -765,9 +766,10 @@ func TestAModuleKeepsTheToolchainsItsFingerprintWasComputedWith(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A-476a: a commit of mesh-sdk moves avahi, of another repository, which imports its package, and leaves mesh-tools,
|
||||
// which imports nothing it moved.
|
||||
// which imports nothing it moved. Its pin is mesh-sdk's newest released version, never the commit; with none
|
||||
// released it has no pin, the package kept for its kind-1 edge (H-A-161).
|
||||
a, _ := moduleOf(t, mods, "avahi")
|
||||
if a.Fp2 == "" || !reflect.DeepEqual(a.Packages, map[string]string{sdk: commit}) {
|
||||
if a.Fp2 == "" || !reflect.DeepEqual(a.Packages, map[string]string{sdk: pin}) {
|
||||
t.Fatalf("after mesh-sdk's merge %.8s: fp2 %q none2 %q packages %v", commit, a.Fp2, a.None2, a.Packages)
|
||||
}
|
||||
if o, _ := moduleOf(t, mods, "mesh-tools"); o.Fp2 != tools.Fp2 || o.Packages != nil {
|
||||
@@ -781,8 +783,56 @@ func TestAModuleKeepsTheToolchainsItsFingerprintWasComputedWith(t *testing.T) {
|
||||
stale.Fp2 = "src2:" + strings.Repeat("5", 64)
|
||||
mods.put("mesh-tools", mustEncode(t, stale))
|
||||
tools = stale
|
||||
first := sdkMerge(strings.Repeat("1", 40), "tree-sdk-1")
|
||||
second := sdkMerge(strings.Repeat("2", 40), "tree-sdk-2")
|
||||
first := sdkMerge(strings.Repeat("1", 40), "tree-sdk-1", "")
|
||||
// mesh-sdk releases 0.1.20 (0.1.9 before it): the release re-pins avahi with no merge, its fingerprint moving with the
|
||||
// pin so it is built again against the release, and leaves mesh-tools, which holds none of it (H-A-165, H-A-167). A
|
||||
// re-pin that fails is no release acknowledged: the package-published delivered again re-pins.
|
||||
ix.Versions, ix.Heard = newMem(), newMem()
|
||||
ix.Versions.(*memStore).put(meshcatalog.VersionsByModuleVersionKey("mesh-sdk", "0.1.9"), []byte(`{}`))
|
||||
revisions := ix.Revisions
|
||||
ix.Revisions = newMemObjects(1 << 20)
|
||||
rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: 3, Kind: "trunk"}, map[string]meshcatalog.CatalogEntry{
|
||||
"mesh-sdk": {Module: "mesh-sdk", Manifest: map[string]any{}, Artifacts: []meshcatalog.Artifact{{Ref: "mesh-sdk:go"}}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := KeepRevision(context.Background(), ix.Revisions, rev, objects); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
release := meshbuilds.PackagePublished{Module: "mesh-sdk", Artifact: "mesh-sdk:go", Digest: "sha256:g", Version: "0.1.20", Source: 7, Branch: "main"}
|
||||
mods.keysErr = errors.New("the bus did not answer")
|
||||
if err := ix.HearPackagePublished(context.Background(), release); err == nil {
|
||||
t.Fatal("a release whose re-pin failed was acknowledged")
|
||||
}
|
||||
mods.keysErr = nil
|
||||
// The registry published mesh-sdk's Go package at 0.1.18, not at the module's release number 0.1.20, in the series
|
||||
// avahi accepts (the default 0.1), with no contents read: the pin is that version (H-A-183, H-A-185).
|
||||
ix.Packages = newMem()
|
||||
if _, err := ix.HearVersion(context.Background(), HeardVersion{Package: "mesh-sdk/go", Seat: "go-package-registry", Version: "0.1.18", Commit: strings.Repeat("1", 40), GoModule: sdk, Published: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ix.HearPackagePublished(context.Background(), release); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ix.Revisions = revisions
|
||||
if a, _ := moduleOf(t, mods, "avahi"); a.Fp2 == first.Fp2 || a.Fp2 == "" || !reflect.DeepEqual(a.Packages, map[string]string{sdk: "v0.1.18"}) {
|
||||
t.Fatalf("after mesh-sdk's release: fp2 %q (was %q) packages %v, want a new fingerprint pinned to v0.1.18", a.Fp2, first.Fp2, a.Packages)
|
||||
} else {
|
||||
// A 0.2.0 publish, of a series avahi does not accept, leaves it alone, contents or none (H-A-184, H-A-185).
|
||||
if _, err := ix.HearVersion(context.Background(), HeardVersion{Package: "mesh-sdk/go", Seat: "go-package-registry", Version: "0.2.0", Commit: strings.Repeat("1", 40), GoModule: sdk, Published: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := ix.Repin(context.Background(), "mesh-sdk"); err != nil { // the pass for avahi alone: mesh-tools is held stale here
|
||||
t.Fatal(err)
|
||||
}
|
||||
if b, _ := moduleOf(t, mods, "avahi"); b.Fp2 != a.Fp2 || b.Packages[sdk] != "v0.1.18" {
|
||||
t.Fatalf("a publish of another series moved avahi: fp2 %q (was %q) packages %v", b.Fp2, a.Fp2, b.Packages)
|
||||
}
|
||||
}
|
||||
if o, _ := moduleOf(t, mods, "mesh-tools"); o.Fp2 != tools.Fp2 {
|
||||
t.Errorf("a re-pin moved mesh-tools, which holds no package of mesh-sdk: %q (was %q)", o.Fp2, tools.Fp2)
|
||||
}
|
||||
second := sdkMerge(strings.Repeat("2", 40), "tree-sdk-2", "v0.1.18")
|
||||
if second.Fp2 == first.Fp2 {
|
||||
t.Errorf("a mesh-sdk trunk merge kept avahi's fingerprint %q", first.Fp2)
|
||||
}
|
||||
|
||||
@@ -6,14 +6,20 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
|
||||
"mesh-delivery-indexer/internal/fingerprint"
|
||||
)
|
||||
|
||||
// The catalog's released versions (design 47 G §349, §372, §374): the indexer keeps, for the seat mesh-catalog, each module's
|
||||
@@ -123,8 +129,9 @@ func (ix *Indexer) HearPackagePublished(ctx context.Context, p meshbuilds.Packag
|
||||
if raw, _, found, err := ix.Versions.Get(ctx, meshcatalog.VersionsByModuleVersionKey(p.Module, p.Version)); err != nil {
|
||||
return err
|
||||
} else if found && !isFloor(raw) {
|
||||
ix.logf("versions: %s %s is released already, so the package-published of %s is heard again and changes nothing", p.Module, p.Version, p.Artifact)
|
||||
return nil
|
||||
// A package-published delivered again after a re-pin failed (H-A-167): the re-pin is made again, nothing else.
|
||||
ix.logf("versions: %s %s is released already, so the package-published of %s is heard again and only re-pins", p.Module, p.Version, p.Artifact)
|
||||
return ix.repin(ctx, p.Module, p.Version)
|
||||
}
|
||||
entry, err := ix.entryOf(ctx, p.Module)
|
||||
if err != nil {
|
||||
@@ -234,6 +241,13 @@ func (ix *Indexer) releaseVersion(ctx context.Context, p meshbuilds.PackagePubli
|
||||
}
|
||||
ix.logf("versions: %s %s is released and version-released is said (%d artifacts, source %d)", p.Module, p.Version, len(v.Artifacts), v.Source)
|
||||
}
|
||||
if !late {
|
||||
// The release re-pins every module whose packages hold one of it (design 47 H-A-165); one that fails is not
|
||||
// acknowledged, so the package-published is delivered again (H-A-167).
|
||||
if err := ix.repin(ctx, p.Module, p.Version); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := ix.trimVersions(ctx, p.Module); err != nil {
|
||||
ix.logf("versions: %s's oldest versions are not deleted: %v", p.Module, err)
|
||||
}
|
||||
@@ -352,3 +366,84 @@ func isFloor(raw []byte) bool {
|
||||
var v meshcatalog.VersionEntry
|
||||
return json.Unmarshal(raw, &v) == nil && len(v.Artifacts) == 0
|
||||
}
|
||||
|
||||
// releasedPins are a module's pins (its catalog entry's packages, which the builder builds it against) from the versions its
|
||||
// fingerprint was computed with (design 47 H-A-161, H-A-183, H-A-184, H-A-185, G §370): each package is pinned to the
|
||||
// newest version the registry published for its path (the seats' published, kept in the package's series by HearVersion)
|
||||
// within the series the module accepts, the one its manifest m names or the default (Manifest.AcceptedSeries), whether
|
||||
// or not that series holds contents yet; a publish in another series pins nothing and moves nothing, and a module's
|
||||
// release number is never a pin. A package with none published in that series is kept with no version, so its kind-1
|
||||
// edge stands (H §746) and its build waits for the publish (H-A-165). nil for none.
|
||||
func releasedPins(versions sdkstep.PackageVersions, series map[string]SeriesPackage, m catalogue.Manifest) sdkstep.PackageVersions {
|
||||
if len(versions) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := sdkstep.PackageVersions{}
|
||||
for p := range versions {
|
||||
out[p] = ""
|
||||
for _, s := range series {
|
||||
if s.GoModule != p || s.Newest == nil || s.Series != m.AcceptedSeries(s.Name) {
|
||||
continue
|
||||
}
|
||||
v := strings.TrimPrefix(s.Newest.Version, "v")
|
||||
if _, ok := versionParts(v); ok && (out[p] == "" || compareVersions(v, strings.TrimPrefix(out[p], "v")) > 0) {
|
||||
out[p] = "v" + v
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// pinnedOf is a form's fingerprint and its pins (releasedPins): the pins are part of what a build is made of, so every
|
||||
// form's fingerprint takes them (design 47 H-A-167, H-A-172; ADR 0573 §4) and a re-pin moves it, the module a mover
|
||||
// built again against the release; a fingerprint with no pin is the form's as it is. The held fingerprint of a build that
|
||||
// stands is kept while its pins are the ones it was built with, else its import form takes the new pins.
|
||||
func pinnedOf(f form, series map[string]SeriesPackage, m catalogue.Manifest) (string, sdkstep.PackageVersions) {
|
||||
pins := releasedPins(f.versions, series, m)
|
||||
if f.stands {
|
||||
if maps.Equal(f.built, map[string]string(pins)) {
|
||||
return f.fp, pins
|
||||
}
|
||||
return fingerprint.WithPins(f.raw, pins), pins
|
||||
}
|
||||
return fingerprint.WithPins(f.fp, pins), pins
|
||||
}
|
||||
|
||||
// moduleOfKey is the module a modules key names; "" for none.
|
||||
func moduleOfKey(key string) string {
|
||||
if key == "" {
|
||||
return ""
|
||||
}
|
||||
if m, err := meshcatalogclient.ParseModulesKey(key); err == nil {
|
||||
return m
|
||||
}
|
||||
return key
|
||||
}
|
||||
|
||||
// PinsOf are the pins a build is asked with from a catalog entry's packages: each package with a version; one kept with
|
||||
// none (no released version, H-A-161) is no pin. nil for none.
|
||||
func PinsOf(packages map[string]string) sdkstep.PackageVersions {
|
||||
var out sdkstep.PackageVersions
|
||||
for p, v := range packages {
|
||||
if v == "" {
|
||||
continue
|
||||
}
|
||||
if out == nil {
|
||||
out = sdkstep.PackageVersions{}
|
||||
}
|
||||
out[p] = v
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// repin is Repin of a release of module at version, said in the journal; its failure is the caller's error.
|
||||
func (ix *Indexer) repin(ctx context.Context, module, version string) error {
|
||||
n, err := ix.Repin(ctx, module)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the modules pinned to %s are not re-pinned to %s: %w", module, version, err)
|
||||
}
|
||||
if n > 0 {
|
||||
ix.logf("versions: %d modules re-pinned to %s %s", n, module, version)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -42,6 +42,8 @@ type artifact struct {
|
||||
Binary string `json:"binary"`
|
||||
Loads []string `json:"loads"`
|
||||
Env map[string]string `json:"env"`
|
||||
Paths []string `json:"paths"`
|
||||
Series string `json:"series"`
|
||||
}
|
||||
|
||||
type stream struct {
|
||||
@@ -170,7 +172,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
|
||||
// internal/broker/state.go:62-93).
|
||||
// The registry seats' published fill each (§413: no consume of version.numbered) package's series (DLV-I5, design 51 *The versions*).
|
||||
if want := []string{"go-package-registry.published", "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "mesh-builds.package-published", "mesh-builds.toolchain-agreed", "mesh-catalog.catalog-revision", "mesh-catalog.registration-changed", "mesh-catalog.room", "mesh-conditions.findings-asked",
|
||||
"mesh-delivery.source-moved", "mesh-pipeline-runner.pipeline-ended", "mesh-pipeline-runner.pipeline-started", "mesh-pipeline-runner.run-started", "mesh-wants.wait-ended", "mesh-wants.wanted", "node-build-go.step-ended", "node-engine.running", "npm-package-registry.published", "operator-channel.decided", "seat:mesh-module-assigner.views.assignments", "seat:mesh-module-assigner.views.seat-holders"}; !reflect.DeepEqual(sorted(m.Consumes), want) {
|
||||
"mesh-delivery.source-moved", "mesh-pipeline-runner.pipeline-ended", "mesh-pipeline-runner.pipeline-started", "mesh-pipeline-runner.run-started", "mesh-wants.pipeline-wanted", "mesh-wants.wait-ended", "mesh-wants.wanted", "node-build-go.step-ended", "node-engine.running", "npm-package-registry.published", "operator-channel.decided", "seat:mesh-module-assigner.views.assignments", "seat:mesh-module-assigner.views.seat-holders"}; !reflect.DeepEqual(sorted(m.Consumes), want) {
|
||||
t.Errorf("consumes = %v, want %v", m.Consumes, want)
|
||||
}
|
||||
if b, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 || b != "${dir:state}/broker" {
|
||||
@@ -344,8 +346,12 @@ func TestManifestProcess(t *testing.T) { // 1238/AC1
|
||||
if _, bus := p["bus"]; bus {
|
||||
t.Errorf("the process declares a bus: it runs on its own broker credential")
|
||||
}
|
||||
if len(m.Build.Artifacts) != 2 {
|
||||
t.Fatalf("artifacts = %+v, want the program and the tools", m.Build.Artifacts)
|
||||
if len(m.Build.Artifacts) != 3 {
|
||||
t.Fatalf("artifacts = %+v, want the program, the tools and the package takein", m.Build.Artifacts)
|
||||
}
|
||||
// H-A-186 (design 47 G §370): takein, which the gate, the view and the watchman import, is a declared package.
|
||||
if a := m.Build.Artifacts[2]; a.Name != "takein" || a.Kind != "package" || a.Language != "go" || !reflect.DeepEqual(a.Paths, []string{"takein"}) || a.Series != "0.1" {
|
||||
t.Errorf("package artifact = %+v", a)
|
||||
}
|
||||
if a := m.Build.Artifacts[1]; a.Name != "tools" || a.Kind != "bundle" || a.Binary != "mesh-delivery-indexer-tools" || !reflect.DeepEqual(a.Loads, []string{"mesh-delivery-indexer-tools"}) {
|
||||
t.Errorf("tools artifact = %+v", a) // 1945
|
||||
|
||||
@@ -89,6 +89,7 @@
|
||||
"mesh-pipeline-runner.run-started",
|
||||
"mesh-pipeline-runner.pipeline-started",
|
||||
"mesh-pipeline-runner.pipeline-ended",
|
||||
"mesh-wants.pipeline-wanted",
|
||||
"node-engine.running",
|
||||
"operator-channel.decided",
|
||||
"npm-package-registry.published"
|
||||
@@ -222,6 +223,15 @@
|
||||
"loads": [
|
||||
"mesh-delivery-indexer-tools"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "takein",
|
||||
"kind": "package",
|
||||
"language": "go",
|
||||
"paths": [
|
||||
"takein"
|
||||
],
|
||||
"series": "0.1"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -67,6 +67,7 @@ func TestTheQueueRunsPipelinesInTurn(t *testing.T) {
|
||||
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: buscompose.LogStreamName(module, seat.StateStepLog), Subjects: []string{buscompose.LogSubject(module, seat.StateStepLog) + ".>"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
findingsStream(t, ctx, js, module)
|
||||
events, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: process.EventsStream, Subjects: []string{"mesh.mod.*.event.>", "mesh.seat.*.event.>"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -75,6 +75,7 @@ func TestTheRunnerPlacesAPipelineAsksItsBuildTakesItsEndAndAnswersAReview(t *tes
|
||||
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: buscompose.LogStreamName(module, seat.StateStepLog), Subjects: []string{buscompose.LogSubject(module, seat.StateStepLog) + ".>"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
findingsStream(t, ctx, js, module)
|
||||
events, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: process.EventsStream, Subjects: []string{"mesh.mod.*.event.>", "mesh.seat.*.event.>"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -283,6 +284,16 @@ func readPipeline(ctx context.Context, kv jetstream.KeyValue, key string, p *sea
|
||||
}
|
||||
|
||||
// waitFor asks cond until it holds or the test's context ends, the module's journal in the failure.
|
||||
// findingsStream makes the stream of the module's own finding events, as its manifest's streams ask the bus for
|
||||
// (`mesh.stream.<module>.event.>`), so a finding the runner writes, such as the unhandled-event count of the SDK's view
|
||||
// reader (H-A-166, H-A-169), is acknowledged at once and holds no event behind its 3 s wait for the bus.
|
||||
func findingsStream(t *testing.T, ctx context.Context, js jetstream.JetStream, module string) {
|
||||
t.Helper()
|
||||
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: module + "-findings", Subjects: []string{"mesh.stream." + module + ".event.>"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func waitFor(t *testing.T, ctx context.Context, logs *syncBuffer, cond func() bool) {
|
||||
t.Helper()
|
||||
for ctx.Err() == nil {
|
||||
|
||||
@@ -137,7 +137,9 @@ func components(movers []seat.Mover, after map[string][]string) map[string]int {
|
||||
var buildKinds = []int64{3, 1}
|
||||
|
||||
// awaited are the artifact references mover m's steps wait for in p (§746, H-A-51): the From of each edge of a build kind
|
||||
// into m from another mover of p, toolchain edges first, each once.
|
||||
// into m from another mover of p, toolchain edges first, each once. A toolchain edge (kind 3) from a build module whose
|
||||
// own new version is built from m is not awaited: what a build module's new version is built from is built by its
|
||||
// running version, never waited for (G §270, A-326), so the running build module builds m and the new one the rest.
|
||||
func awaited(p seat.Pipeline, m string, entries map[string]catalog.CatalogEntry) []catalog.Edge {
|
||||
var out []catalog.Edge
|
||||
for _, kind := range buildKinds {
|
||||
@@ -146,12 +148,35 @@ func awaited(p seat.Pipeline, m string, entries map[string]catalog.CatalogEntry)
|
||||
if x.Kind != kind || from == m || !Moves(p, from) || slices.ContainsFunc(out, func(e catalog.Edge) bool { return e.From == x.From }) {
|
||||
continue
|
||||
}
|
||||
if kind == 3 && builtFrom(p, from, entries)[m] {
|
||||
continue
|
||||
}
|
||||
out = append(out, x)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// builtFrom are the movers of p that module's new version is built from (G §270): the From mover of each edge of a build
|
||||
// kind into it, and what those are built from in turn.
|
||||
func builtFrom(p seat.Pipeline, module string, entries map[string]catalog.CatalogEntry) map[string]bool {
|
||||
seen := map[string]bool{}
|
||||
next := []string{module}
|
||||
for len(next) > 0 {
|
||||
cur := next[0]
|
||||
next = next[1:]
|
||||
for _, x := range entries[cur].Edges {
|
||||
from := moduleOf(x.From)
|
||||
if !slices.Contains(buildKinds, x.Kind) || from == cur || seen[from] || !Moves(p, from) {
|
||||
continue
|
||||
}
|
||||
seen[from] = true
|
||||
next = append(next, from)
|
||||
}
|
||||
}
|
||||
return seen
|
||||
}
|
||||
|
||||
// ready says whether mover m of p may be placed (§746, H-A-84): every artifact it awaits is in its From mover's published.
|
||||
// It answers the references still awaited, and the first From mover whose build ended without passing, so m never builds.
|
||||
func ready(p seat.Pipeline, m string, entries map[string]catalog.CatalogEntry) (bool, []string, string) {
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
||||
catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
@@ -124,3 +125,115 @@ func TestAMoversBuildWaitsForItsDependenciesInThePipeline(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 G §270, A-326 (H §746, H-A-83 (2)): what a build module's new version is built from is built by its running
|
||||
// version, never waited for: a mover the moving build module imports, directly or through another mover, does not await
|
||||
// that build module's toolchain edge, so a pipeline moving mesh-sdk or the contracts with the Go build module places them
|
||||
// at once (main.28, main.30 of 2026-11-20 waited on each other forever); every other mover still awaits the new one.
|
||||
func TestAToolchainEdgeIsNotAwaitedByTheBuildModulesOwnInputs(t *testing.T) {
|
||||
e := func(from, to string, kind int64) catalog.Edge { return catalog.Edge{From: from, To: to, Kind: kind} }
|
||||
entries := map[string]catalog.CatalogEntry{
|
||||
"mesh-sdk": {Edges: []catalog.Edge{e("mesh-build-go:files", "mesh-sdk:go", 3)}},
|
||||
"contracts": {Edges: []catalog.Edge{e("mesh-build-go:files", "contracts:go-a", 3), e("mesh-build-go:files", "contracts:go-b", 3)}},
|
||||
"delivery-sdk": {Edges: []catalog.Edge{e("mesh-build-go:files", "delivery-sdk:go", 3), e("mesh-sdk:go", "delivery-sdk:go", 1)}},
|
||||
"mesh-build-go": {Edges: []catalog.Edge{e("contracts:go-a", "mesh-build-go:tools", 1), e("delivery-sdk:go", "mesh-build-go:tools", 1), e("nats:tools", "mesh-build-go:files", 4)}},
|
||||
"zfs": {Edges: []catalog.Edge{e("mesh-build-go:files", "zfs:tools", 3), e("contracts:go-a", "zfs:tools", 1)}},
|
||||
"tool": {Edges: []catalog.Edge{e("builder:program", "tool:program", 3)}},
|
||||
"builder": {},
|
||||
}
|
||||
pipe := func(modules ...string) seat.Pipeline {
|
||||
var p seat.Pipeline
|
||||
for _, m := range modules {
|
||||
p.Movers = append(p.Movers, seat.Mover{Module: m})
|
||||
}
|
||||
return p
|
||||
}
|
||||
all := pipe("mesh-sdk", "contracts", "delivery-sdk", "mesh-build-go", "zfs")
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
p seat.Pipeline
|
||||
mover string
|
||||
want []string
|
||||
}{
|
||||
{"the build module's own package input awaits no toolchain edge", all, "contracts", nil},
|
||||
{"an input through another mover awaits no toolchain edge", all, "mesh-sdk", nil},
|
||||
{"an input that imports another input awaits only that package", all, "delivery-sdk", []string{"mesh-sdk:go"}},
|
||||
{"the build module awaits its inputs' packages", all, "mesh-build-go", []string{"contracts:go-a", "delivery-sdk:go"}},
|
||||
{"any other mover awaits the new build module first", all, "zfs", []string{"mesh-build-go:files", "contracts:go-a"}},
|
||||
{"a build module that is not built from the mover is awaited", pipe("tool", "builder"), "tool", []string{"builder:program"}},
|
||||
{"a build module that does not move is not awaited", pipe("tool"), "tool", nil},
|
||||
} {
|
||||
var got []string
|
||||
for _, x := range awaited(c.p, c.mover, entries) {
|
||||
got = append(got, x.From)
|
||||
}
|
||||
if !slices.Equal(got, c.want) {
|
||||
t.Errorf("%s: %s awaits %v, want %v", c.name, c.mover, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 H H-A-175, H-A-176 (G §270, A-326; H-A-101 (2), H-A-163): when every mover without steps waits only on movers
|
||||
// without steps, their steps are placed ended not asked, the why `cycle` naming the edges inside the cycle alone; a step
|
||||
// of another mover is asked as ever, and the pipeline ends by the end judgement, at once when no required step is open; a
|
||||
// cycle through a build module's own inputs is broken (G §270) and placed; a mover waiting on a placed mover only waits.
|
||||
func TestACycleAmongUnplacedMoversEndsThePipelineFailed(t *testing.T) {
|
||||
reg, base := placeFixture()
|
||||
reg.Policy = catalog.Policy{Steps: catalog.StepStatuses{Lint: "bypass", Unit: "bypass", Integration: "bypass", Validate: "bypass", Composition: "bypass"}}
|
||||
entry := func(m string, edges ...catalog.Edge) catalog.CatalogEntry {
|
||||
return catalog.CatalogEntry{Module: m, Artifacts: []catalog.Artifact{{Ref: m + ":go", Language: "go"}}, Edges: edges}
|
||||
}
|
||||
e := func(from, to string, kind int64) catalog.Edge { return catalog.Edge{From: from, To: to, Kind: kind} }
|
||||
cycle := []catalog.CatalogEntry{entry("a", e("b:go", "a:go", 1)), entry("b", e("a:go", "b:go", 1))}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
entries []catalog.CatalogEntry
|
||||
outcome string
|
||||
asked []string
|
||||
notAsked []string
|
||||
}{
|
||||
{"a cycle G §270 breaks is placed", []catalog.CatalogEntry{
|
||||
entry("mesh-sdk", e("mesh-build-go:files", "mesh-sdk:go", 3)),
|
||||
entry("mesh-build-go", e("mesh-sdk:go", "mesh-build-go:go", 1))}, seat.PipelineOutcomeRunning, []string{"mesh-sdk"}, nil},
|
||||
{"a cycle beside a free mover ends its own steps and the free one is asked",
|
||||
append(slices.Clone(cycle), entry("c"), entry("d", e("a:go", "d:go", 1))), seat.PipelineOutcomeRunning, []string{"c"}, []string{"a", "b", "d"}},
|
||||
{"a cycle alone ends the pipeline failed at once", cycle, seat.PipelineOutcomeFailed, nil, []string{"a", "b"}},
|
||||
{"a mover waiting on a placed mover waits", []catalog.CatalogEntry{
|
||||
entry("a", e("b:go", "a:go", 1)), entry("b")}, seat.PipelineOutcomeRunning, []string{"b"}, nil},
|
||||
} {
|
||||
entries := map[string]catalog.CatalogEntry{"mesh-build-go": base["mesh-build-go"]}
|
||||
p := seat.Pipeline{Pipeline: "main.4", Branch: "main", Source: 4, Outcome: seat.PipelineOutcomeRunning}
|
||||
for _, en := range c.entries {
|
||||
entries[en.Module] = en
|
||||
p.Movers = append(p.Movers, seat.Mover{Module: en.Module, Commit: "c", Fingerprint: "src2:" + en.Module})
|
||||
}
|
||||
s := &Starter{Name: "the-runner"}
|
||||
asks, _, ended, notAsked := s.placeMovers(&p, func(string) (catalog.Registration, bool) { return reg, true }, entries, time.Unix(1, 0))
|
||||
if p.Outcome != c.outcome || (c.outcome == seat.PipelineOutcomeFailed) != (len(ended) > 0) || (len(c.asked) > 0) != (len(asks) > 0) {
|
||||
t.Errorf("%s: pipeline %s (%s), %d asks, %d ended; want %s", c.name, p.Outcome, p.Why, len(asks), len(ended), c.outcome)
|
||||
}
|
||||
said := map[int]bool{}
|
||||
for _, i := range append(ended, notAsked...) {
|
||||
said[i] = true
|
||||
}
|
||||
var asked, ends []string
|
||||
for i, st := range p.Steps {
|
||||
switch {
|
||||
case st.Outcome == seat.StepResultOutcomeNotAsked && strings.HasPrefix(st.Why, "cycle: ") && said[i]:
|
||||
if !slices.Contains(ends, st.Module) {
|
||||
ends = append(ends, st.Module)
|
||||
}
|
||||
if st.Why != "cycle: a:go before b:go (kind 1); b:go before a:go (kind 1)" {
|
||||
t.Errorf("%s: %s's %s why %q, want the cycle's two edges alone", c.name, st.Module, st.Step, st.Why)
|
||||
}
|
||||
case open(st) || st.Outcome == seat.StepResultOutcomeWaiting:
|
||||
if !slices.Contains(asked, st.Module) {
|
||||
asked = append(asked, st.Module)
|
||||
}
|
||||
}
|
||||
}
|
||||
if !slices.Equal(asked, c.asked) || !slices.Equal(ends, c.notAsked) {
|
||||
t.Errorf("%s: open %v, ended not asked %v; want %v and %v", c.name, asked, ends, c.asked, c.notAsked)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -317,6 +317,7 @@ func (s *Starter) placeMovers(p *seat.Pipeline, reg func(module string) (catalog
|
||||
var asks []Asked
|
||||
var unregistered []string
|
||||
var notAsked []int
|
||||
var waits []seat.Mover
|
||||
for _, m := range p.Movers {
|
||||
if placed(*p, m.Module) {
|
||||
continue
|
||||
@@ -331,6 +332,7 @@ func (s *Starter) placeMovers(p *seat.Pipeline, reg func(module string) (catalog
|
||||
due, waiting, failed := ready(*p, m.Module, entries)
|
||||
if !due && failed == "" {
|
||||
s.logf("%s: %s waits for %s before its build", p.Pipeline, m.Module, strings.Join(waiting, ", "))
|
||||
waits = append(waits, m)
|
||||
continue
|
||||
}
|
||||
steps, a := Place(*p, m, r, entries, now, s.Name)
|
||||
@@ -367,6 +369,11 @@ func (s *Starter) placeMovers(p *seat.Pipeline, reg func(module string) (catalog
|
||||
}
|
||||
}
|
||||
}
|
||||
if steps, ended := cycled(p, waits, unregistered, reg, entries, now, s.Name); ended {
|
||||
return nil, unregistered, steps, notAsked
|
||||
} else {
|
||||
notAsked = append(notAsked, steps...)
|
||||
}
|
||||
// A mover placed after the validate ended has its composition asked, or not asked, at once (§745).
|
||||
more, _ := releaseCompositions(p, now)
|
||||
asks = append(asks, more...)
|
||||
@@ -374,6 +381,72 @@ func (s *Starter) placeMovers(p *seat.Pipeline, reg func(module string) (catalog
|
||||
return asks, unregistered, ended, notAsked
|
||||
}
|
||||
|
||||
// cycled ends not asked, at p's placing, the steps of every mover still without steps when each waits only on movers
|
||||
// without steps (H-A-175, H-A-176; G §270 and A-326 break a build module's own inputs before this): a cycle no build can
|
||||
// start, and the movers that wait only on it. Their steps are placed ended not asked (as H-A-101 (2) does), the why
|
||||
// `cycle` naming each artifact and edge kind whose two movers share a strongly connected group; no step already asked is
|
||||
// touched. The pipeline then ends by the end judgement of H-A-163 (endPipeline), failed once no required step is open,
|
||||
// its running steps running to their own end. It answers the steps it ended not asked, or, when the pipeline ended, every
|
||||
// step it ended, and whether the pipeline ended.
|
||||
func cycled(p *seat.Pipeline, waits []seat.Mover, unregistered []string, reg func(module string) (catalog.Registration, bool), entries map[string]catalog.CatalogEntry, now time.Time, name string) ([]int, bool) {
|
||||
if len(waits) == 0 || len(unregistered) > 0 {
|
||||
return nil, false
|
||||
}
|
||||
after := map[string][]string{}
|
||||
edges := map[string][]catalog.Edge{}
|
||||
for _, m := range waits {
|
||||
for _, x := range awaited(*p, m.Module, entries) {
|
||||
from := moduleOf(x.From)
|
||||
if placed(*p, from) {
|
||||
return nil, false
|
||||
}
|
||||
after[m.Module] = append(after[m.Module], from)
|
||||
edges[m.Module] = append(edges[m.Module], x)
|
||||
}
|
||||
}
|
||||
comp := components(waits, after)
|
||||
var on []string
|
||||
for _, m := range waits {
|
||||
for _, x := range edges[m.Module] {
|
||||
if comp[moduleOf(x.From)] == comp[m.Module] {
|
||||
on = append(on, fmt.Sprintf("%s before %s (kind %d)", x.From, x.To, x.Kind))
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(on)
|
||||
why := "cycle: " + strings.Join(on, "; ")
|
||||
var ended []int
|
||||
for _, m := range waits {
|
||||
r, _ := reg(m.Module)
|
||||
steps, _ := Place(*p, m, r, entries, now, name)
|
||||
for k := range steps {
|
||||
st := &steps[k]
|
||||
if st.Outcome == seat.StepResultOutcomeBypassed {
|
||||
continue
|
||||
}
|
||||
st.Outcome, st.Why, st.Ask, st.Shared, st.Ended = seat.StepResultOutcomeNotAsked, why, "", nil, &now
|
||||
p.Unlogged = append(p.Unlogged, endEntry(p.Pipeline, *st, now))
|
||||
ended = append(ended, len(p.Steps)+k)
|
||||
}
|
||||
p.Steps = append(p.Steps, steps...)
|
||||
}
|
||||
var approvals []int
|
||||
for i, st := range p.Steps {
|
||||
if st.Step == seat.StepResultStepApproval && st.Outcome == seat.StepResultOutcomeWaiting {
|
||||
approvals = append(approvals, i)
|
||||
}
|
||||
}
|
||||
if !endPipeline(p, now, nil) {
|
||||
return ended, false
|
||||
}
|
||||
for _, i := range approvals {
|
||||
if p.Steps[i].Outcome == seat.StepResultOutcomeNotAsked {
|
||||
ended = append(ended, i)
|
||||
}
|
||||
}
|
||||
return ended, true
|
||||
}
|
||||
|
||||
// sayNotAsked says the step-ended of each step of p placed not asked because its mover's dependency did not build, naming
|
||||
// the dependency in its why, so no step ends silently (H-A-101 (2)).
|
||||
func (s *Starter) sayNotAsked(p seat.Pipeline, steps []int) {
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
// keys), and the checker so at a delivery group member's head for a package new in the group, so the judge is handed one
|
||||
// name for each package whoever reads it (mesh-sdk composer.Request.Packages).
|
||||
// Design 47 H1, proof 2: this comment is the mesh-delivery-sdk half of a two-module change its pipeline delivers.
|
||||
// Design 47 H1, proof 2 again: the provider half of the edge mesh-delivery-sdk to mesh-build-node.
|
||||
package packagename
|
||||
|
||||
import (
|
||||
|
||||
@@ -38,9 +38,12 @@ type artifact struct {
|
||||
From string `json:"from"`
|
||||
Binary string `json:"binary"`
|
||||
Loads []string `json:"loads"`
|
||||
Paths []string `json:"paths"`
|
||||
Series string `json:"series"`
|
||||
}
|
||||
|
||||
type manifest struct {
|
||||
Packages map[string]string `json:"packages"`
|
||||
Module string `json:"module"`
|
||||
Version string `json:"version"`
|
||||
Slug string `json:"slug"`
|
||||
@@ -176,8 +179,16 @@ func TestManifestProcess(t *testing.T) { // 1240/AC1
|
||||
if _, bus := p["bus"]; bus {
|
||||
t.Errorf("the process declares a bus: it runs on its own broker credential")
|
||||
}
|
||||
if len(m.Build.Artifacts) != 2 {
|
||||
t.Fatalf("artifacts = %+v, want the program and the tools", m.Build.Artifacts)
|
||||
if len(m.Build.Artifacts) != 3 {
|
||||
t.Fatalf("artifacts = %+v, want the program, the tools and the package tellerevents", m.Build.Artifacts)
|
||||
}
|
||||
// H-A-188: the assigner is at 0.3, so its package asks, which this module imports, is accepted in that series.
|
||||
if got := m.Packages["git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"]; got != "0.3" || len(m.Packages) != 1 {
|
||||
t.Errorf("packages = %v, want asks accepted in the assigner's series 0.3", m.Packages)
|
||||
}
|
||||
// H-A-186 (design 47 G §370): tellerevents, which the assigner imports, is a declared package.
|
||||
if a := m.Build.Artifacts[2]; a.Name != "tellerevents" || a.Kind != "package" || a.Language != "go" || !reflect.DeepEqual(a.Paths, []string{"tellerevents"}) || a.Series != "0.1" {
|
||||
t.Errorf("package artifact = %+v", a)
|
||||
}
|
||||
a := m.Build.Artifacts[0]
|
||||
if st, err := os.Stat(a.From); err != nil || !st.IsDir() || a.Binary != "mesh-delivery-teller" || !strings.HasSuffix(a.From, a.Binary) {
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
{
|
||||
"module": "mesh-delivery-teller",
|
||||
"version": "0.1",
|
||||
"packages": {
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks": "0.3"
|
||||
},
|
||||
"slug": "teller",
|
||||
"tools": [
|
||||
"teller_end_ask",
|
||||
@@ -123,6 +126,15 @@
|
||||
"loads": [
|
||||
"mesh-delivery-teller-tools"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "tellerevents",
|
||||
"kind": "package",
|
||||
"language": "go",
|
||||
"paths": [
|
||||
"tellerevents"
|
||||
],
|
||||
"series": "0.1"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
@@ -70,6 +70,7 @@ type claim struct {
|
||||
}
|
||||
|
||||
type manifest struct {
|
||||
Packages map[string]string `json:"packages"`
|
||||
Claims []claim `json:"claims"`
|
||||
State []state `json:"state"`
|
||||
Module string `json:"module"`
|
||||
@@ -313,9 +314,12 @@ func TestManifestIsTakenAndReadsOnly(t *testing.T) { // 1134/AC7
|
||||
|
||||
func TestManifestIsWholeInItsRepository(t *testing.T) { // 1134/AC7
|
||||
m, _ := load(t)
|
||||
// Issue 1966's bootstrap: the package artifact viewstate is out of the manifest until the new builder runs; the
|
||||
// follow-up puts the count back at 5.
|
||||
if len(m.Build.Artifacts) != 4 {
|
||||
// H-A-188: the assigner is at 0.3, so its package asks, which this module imports, is accepted in that series.
|
||||
if got := m.Packages["git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"]; got != "0.3" || len(m.Packages) != 1 {
|
||||
t.Errorf("packages = %v, want asks accepted in the assigner's series 0.3", m.Packages)
|
||||
}
|
||||
// H-A-186 (design 47 G §370): viewstate, which the gate, the assigner and the teller import, is a declared package.
|
||||
if len(m.Build.Artifacts) != 5 || m.Build.Artifacts[4].Name != "viewstate" {
|
||||
t.Fatalf("artifacts = %+v", m.Build.Artifacts)
|
||||
}
|
||||
for _, a := range m.Build.Artifacts {
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
{
|
||||
"module": "mesh-delivery-view",
|
||||
"version": "0.1",
|
||||
"packages": {
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks": "0.3"
|
||||
},
|
||||
"moved": true,
|
||||
"slug": "view",
|
||||
"tools": [
|
||||
@@ -234,6 +237,15 @@
|
||||
"name": "files",
|
||||
"kind": "archive",
|
||||
"from": "files"
|
||||
},
|
||||
{
|
||||
"name": "viewstate",
|
||||
"kind": "package",
|
||||
"language": "go",
|
||||
"paths": [
|
||||
"viewstate"
|
||||
],
|
||||
"series": "0.1"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
@@ -648,6 +648,17 @@ func TestGrants(t *testing.T) {
|
||||
if _, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 || m.SecretsOwner != "mesh-grants" {
|
||||
t.Errorf("own-secrets %v owner %q: a credential of its own, broker", m.OwnSecrets, m.SecretsOwner)
|
||||
}
|
||||
// H-A-186 (design 47 G §370): grantsstate (imported by the assigner, the vault and the node-engine) and busgrants
|
||||
// (imported by the bus module and the watchman) are declared Go packages of the module, so every release publishes them.
|
||||
var packages []string
|
||||
for _, a := range m.Build.Artifacts {
|
||||
if a.Kind == catalogue.ArtifactPackage && a.Language == "go" && len(a.Paths) == 1 && a.Paths[0] == a.Name && a.Series == "0.1" {
|
||||
packages = append(packages, a.Name)
|
||||
}
|
||||
}
|
||||
if !slices.Equal(packages, []string{"grantsstate", "busgrants"}) {
|
||||
t.Errorf("Go packages %v, want grantsstate and busgrants of series 0.1", packages)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("no timer, no update or delete, a publish only of conditions, a create only of declared streams", func(t *testing.T) {
|
||||
|
||||
@@ -185,6 +185,24 @@
|
||||
"loads": [
|
||||
"mesh-grants"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "grantsstate",
|
||||
"kind": "package",
|
||||
"language": "go",
|
||||
"paths": [
|
||||
"grantsstate"
|
||||
],
|
||||
"series": "0.1"
|
||||
},
|
||||
{
|
||||
"name": "busgrants",
|
||||
"kind": "package",
|
||||
"language": "go",
|
||||
"paths": [
|
||||
"busgrants"
|
||||
],
|
||||
"series": "0.1"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user