diff --git a/modules/distribution/cmd/distribution-tools/bounded.go b/modules/distribution/cmd/distribution-tools/bounded.go new file mode 100644 index 000000000..7bf77f0dd --- /dev/null +++ b/modules/distribution/cmd/distribution-tools/bounded.go @@ -0,0 +1,123 @@ +package main + +import ( + "fmt" + "time" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/parts" + + "distribution/internal/holder" +) + +// stepBound bounds each bus operation of a publish well under the tool runner's 30 s call bound, so a hang is an error +// naming its step and the publish answers with it. +const stepBound = 10 * time.Second + +// step runs f, logs its start and its end with its time, and gives up on it after bound: the error names the step. A step +// given up on is not cancelled (the SDK's bus calls take no context): its call ends at the SDK's own bound of the ask, and +// the log says, by the step's name, that it was given up and later when the call behind it ended and how (A-443, A-443a). +func step[T any](bound time.Duration, what string, f func() (T, error)) (T, error) { + type res struct { + v T + err error + } + start := time.Now() + logf("step %s starts", what) + ch := make(chan res, 1) + go func() { v, err := f(); ch <- res{v, err} }() + select { + case r := <-ch: + logf("step %s ended after %s (%v)", what, time.Since(start).Round(time.Millisecond), r.err) + return r.v, r.err + case <-time.After(bound): + var zero T + err := fmt.Errorf("%s did not answer within %s", what, bound) + logf("step %s: %v; given up, its call is left to end at the SDK's own bound", what, err) + go func() { + r := <-ch + logf("step %s, given up after %s: its call ended after %s (%v)", what, bound, time.Since(start).Round(time.Millisecond), r.err) + }() + return zero, err + } +} + +func stepErr(bound time.Duration, what string, f func() error) error { + _, err := step(bound, what, func() (struct{}, error) { return struct{}{}, f() }) + return err +} + +// boundedObjects is the incoming object store with each operation bounded and logged. +type boundedObjects struct{ parts.Objects } + +func (b boundedObjects) Used() (uint64, error) { + return step(stepBound, "the incoming object store's fill (objects)", b.Objects.Used) +} +func (b boundedObjects) Put(name string, data []byte) error { + return stepErr(stepBound, "the put of a chunk in the incoming object store (objects)", func() error { return b.Objects.Put(name, data) }) +} +func (b boundedObjects) Get(name string) ([]byte, error) { + return step(stepBound, "the get of a chunk from the incoming object store (objects)", func() ([]byte, error) { return b.Objects.Get(name) }) +} +func (b boundedObjects) Delete(name string) error { + return stepErr(stepBound, "the delete of a chunk in the incoming object store (objects)", func() error { return b.Objects.Delete(name) }) +} + +// boundedTransfers is the transfers state with its reads and writes bounded and logged. +type boundedTransfers struct { + parts.Transfers[holder.Opening] +} + +func (b boundedTransfers) Get(id string) (parts.Transfer[holder.Opening], bool, error) { + type r struct { + t parts.Transfer[holder.Opening] + ok bool + } + x, err := step(stepBound, "the read of the transfer in the transfers state", func() (r, error) { + t, ok, err := b.Transfers.Get(id) + return r{t, ok}, err + }) + return x.t, x.ok, err +} +func (b boundedTransfers) Put(t parts.Transfer[holder.Opening]) error { + return stepErr(stepBound, "the write of the transfer in the transfers state", func() error { return b.Transfers.Put(t) }) +} +func (b boundedTransfers) Delete(id string) error { + return stepErr(stepBound, "the delete of the transfer in the transfers state", func() error { return b.Transfers.Delete(id) }) +} + +// boundedCap is the incoming cap read from its binding, bounded and logged. +func boundedCap(c func() (uint64, error)) func() (uint64, error) { + return func() (uint64, error) { + return step(stepBound, "the read of incoming's cap from its binding (objects)", c) + } +} + +// boundedArm is the idle key's write, bounded and logged. +func boundedArm(a func(string, time.Duration) error) func(string, time.Duration) error { + return func(id string, ttl time.Duration) error { + return stepErr(stepBound, "the write of the idle key in the idle state", func() error { return a(id, ttl) }) + } +} + +// boundedFinding is the findings writer's say moved off the publish's path: the receiver says the incoming fill on every +// open, and a publish never waits for it. One worker says them in order, each bounded and logged; one said while the worker +// is full is dropped and logged (the next open says the fill again). +func boundedFinding(f holder.Finding) holder.Finding { + type said struct { + kind, subject, words string + raised bool + } + queue := make(chan said, 64) + go func() { + for s := range queue { + _ = stepErr(stepBound, "the finding "+s.kind+" of "+s.subject, func() error { f(s.kind, s.subject, s.words, s.raised); return nil }) + } + }() + return func(kind, subject, words string, raised bool) { + select { + case queue <- said{kind, subject, words, raised}: + default: + logf("the finding %s of %s is not said now: the writer is behind, the next open says it again", kind, subject) + } + } +} diff --git a/modules/distribution/cmd/distribution-tools/bounded_test.go b/modules/distribution/cmd/distribution-tools/bounded_test.go new file mode 100644 index 000000000..64cec0079 --- /dev/null +++ b/modules/distribution/cmd/distribution-tools/bounded_test.go @@ -0,0 +1,70 @@ +package main + +import ( + "fmt" + "strings" + "sync" + "testing" + "time" +) + +// A bus operation that never answers becomes an error naming its step, within the bound. +func TestAHungStepAnswersWithItsName(t *testing.T) { + block := make(chan struct{}) + defer close(block) + _, err := step(10*time.Millisecond, "the write of the idle key", func() (int, error) { <-block; return 0, nil }) + if err == nil || !strings.Contains(err.Error(), "the write of the idle key did not answer within") { + t.Fatalf("err = %v", err) + } + if v, err := step(time.Second, "x", func() (int, error) { return 7, nil }); v != 7 || err != nil { + t.Fatalf("%d %v", v, err) + } +} + +// A step given up on is said given up by name, and the log says later when the call behind it ended and how (A-443a). +func TestAGivenUpStepSaysWhenItsCallEnded(t *testing.T) { + var mu sync.Mutex + var lines []string + ended := make(chan struct{}) + w := func(f string, a ...any) { + l := fmt.Sprintf(f, a...) + mu.Lock() + lines = append(lines, l) + mu.Unlock() + if strings.Contains(l, "the put of a chunk") && strings.Contains(l, "its call ended after") { + close(ended) + } + } + old := logTo.Swap(&w) + defer logTo.Store(old) + release := make(chan struct{}) + if _, err := step(10*time.Millisecond, "the put of a chunk", func() (int, error) { <-release; return 0, fmt.Errorf("bus closed") }); err == nil { + t.Fatal("not given up") + } + close(release) + <-ended + mu.Lock() + all := strings.Join(lines, "\n") + mu.Unlock() + if !strings.Contains(all, "given up") || !strings.Contains(all, "bus closed") { + t.Fatalf("log: %s", all) + } +} + +// The finding is said off the caller's path: a writer that never answers does not hold the call that says it. +func TestAFindingIsSaidOffThePath(t *testing.T) { + block := make(chan struct{}) + defer close(block) + say := boundedFinding(func(string, string, string, bool) { <-block }) + done := make(chan struct{}) + go func() { + say("incoming-full", "incoming", "w", false) + say("incoming-full", "incoming", "w", false) + close(done) + }() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("the call was held by the finding") + } +} diff --git a/modules/distribution/cmd/distribution-tools/main.go b/modules/distribution/cmd/distribution-tools/main.go index b0278ca9f..ae4a503b4 100644 --- a/modules/distribution/cmd/distribution-tools/main.go +++ b/modules/distribution/cmd/distribution-tools/main.go @@ -16,6 +16,7 @@ import ( "os" "strings" "sync" + "sync/atomic" "time" "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" @@ -29,7 +30,15 @@ import ( "distribution/internal/holder" ) -func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, "[distribution-tools] "+format+"\n", a...) } +// logTo is where logf writes; a test replaces it (atomically: a step given up on logs from its own goroutine). +var logTo atomic.Pointer[func(string, ...any)] + +func init() { + w := func(format string, a ...any) { fmt.Fprintf(os.Stderr, "[distribution-tools] "+format+"\n", a...) } + logTo.Store(&w) +} + +func logf(format string, a ...any) { (*logTo.Load())(format, a...) } // kept is one state of distribution's, through the runtime. type kept[T any] struct { @@ -249,10 +258,21 @@ func tools(v *holder.Verbs, windows kept[holder.Window]) []stdio.Tool { if err != nil { return nil, err } - a, err := v.Publish(seat, stdio.CallerOf(args), publishOf(in)) + caller := stdio.CallerOf(args) + start := time.Now() + logf("%s.publish received from %s: %s %s (%s, %d parts)", seat, caller, in.Name, in.Version, in.Kind, len(in.Parts)) + a, err := v.Publish(seat, caller, publishOf(in)) if err == nil { - learnPublishing(windows, stdio.CallerOf(args)) + // The window's learning asks the delivery view (up to 60 s, past the runner's 30 s call bound), + // so it never holds the answer. + go func() { + t := time.Now() + logf("learning the publishing build of %s starts (mesh-delivery-view.node_haves)", caller) + learnPublishing(windows, caller) + logf("learning the publishing build of %s ended after %s", caller, time.Since(t).Round(time.Millisecond)) + }() } + logf("%s.publish from %s answered after %s (error: %v)", seat, caller, time.Since(start).Round(time.Millisecond), err) return publishAnswer(a), err }}, stdio.Tool{Name: seat + "." + holder.VerbPublishPart, Description: "Send one chunk (at most 1 MiB) of one part of an open transfer, from the caller that opened it.", @@ -468,8 +488,8 @@ func main() { }() base, _ := url.Parse("http://127.0.0.1:5001") store := &holder.OCI{Base: base, Upstream: &holder.Upstream{}} - h := &holder.Holder{Objects: parts.BusObjects(holder.IncomingObjects), Builder: holder.Reserved{Ask: stdio.Ask}, Store: store, Say: say, Finding: findings.Say, - Transfers: parts.BusTransfers[holder.Opening](holder.TransfersState), Cap: parts.BusCap(holder.IncomingObjects), Record: recordAct, Idle: parts.BusArm(holder.IdleState)} + h := &holder.Holder{Logf: logf, Objects: boundedObjects{parts.BusObjects(holder.IncomingObjects)}, Builder: holder.Reserved{Ask: stdio.Ask}, Store: store, Say: say, Finding: boundedFinding(findings.Say), + Transfers: boundedTransfers{parts.BusTransfers[holder.Opening](holder.TransfersState)}, Cap: boundedCap(parts.BusCap(holder.IncomingObjects)), Record: recordAct, Idle: boundedArm(parts.BusArm(holder.IdleState))} files := &holder.Files{Container: or(os.Getenv("MESH_REGISTRY_CONTAINER"), "mesh-registry"), Run: holder.ExecRunner, UID: os.Getuid()} v := &holder.Verbs{Holder: h, Catalog: store, Holds: kept[holder.Hold]{state: "holds", key: holder.Hold.Key}, Mirrors: mirrors{}, Files: files, SweepHolder: func() (string, error) { return seatlist.HolderOf(stdio.Ask, holder.SweepSeat) }} diff --git a/modules/distribution/internal/holder/holder_test.go b/modules/distribution/internal/holder/holder_test.go index 1e202aab9..08d6aee84 100644 --- a/modules/distribution/internal/holder/holder_test.go +++ b/modules/distribution/internal/holder/holder_test.go @@ -71,6 +71,7 @@ func (s *fakeStore) Put(t PublishRequest, parts map[string][]byte) (string, erro return t.Name + "@" + t.Digest, nil } func (s *fakeStore) Held(string, string, string) (string, bool, error) { return "", false, nil } +func (s *fakeStore) Holds(r PublishRequest) (bool, error) { return s.has[r.Digest], nil } type said struct { seat, event string @@ -231,6 +232,21 @@ func TestTheHolderOfTheTwoImageSeats(t *testing.T) { return PublishRequest{Transfer: id, Name: "m/program", Version: "0.1.4", Kind: KindBundle, Digest: part.Digest, Parts: []Part{part}} } + t.Run("a publish says each of its waits with its time", func(t *testing.T) { + b := newBed(true) + var lines []string + b.h.Logf = func(f string, a ...any) { lines = append(lines, fmt.Sprintf(f, a...)) } + if _, err := b.h.Publish(ArtifactSeat, builder, req("tl")); err != nil { + t.Fatal(err) + } + all := strings.Join(lines, "\n") + for _, want := range []string{"the store's read of what it holds (the program's registry) starts", "(the program's registry) ended after", "the opening of the transfer on the bus"} { + if !strings.Contains(all, want) { + t.Errorf("no log line %q in:\n%s", want, all) + } + } + }) + t.Run("a transfer in parts, cut and resumed, hashed, numbered by the builder, published", func(t *testing.T) { b := newBed(true) a, err := b.h.Publish(ArtifactSeat, builder, req("t1")) @@ -650,30 +666,37 @@ func TestTheHolderOfTheTwoImageSeats(t *testing.T) { } } }) - t.Run("a late publish names a held bundle with no size: tagged at the store's size, nothing sent", func(t *testing.T) { + t.Run("a publish with a version and no parts tags the held digest at the store's size; one not held is not stored", func(t *testing.T) { + // Design 47 G A-417, A-73, A-50: the build stored the bundle untagged; the builder's publish of its version with that + // digest and no parts tags it and says published, nothing sent; a digest the store holds none of is refused as not + // stored, naming the digest. store := newFakeRegistry() sts := httptest.NewServer(store) defer sts.Close() base, _ := url.Parse(sts.URL) - o := &OCI{Base: base} + b := newBed(true) + b.h.Store = &OCI{Base: base} blob := []byte(strings.Repeat("b", 4321)) store.blobs["m/tools@"+digestOf(blob)] = blob - ref, err := o.Put(PublishRequest{Name: "m/tools", Version: "0.1.4", Kind: KindBundle, Digest: digestOf(blob), - Parts: []Part{{Digest: digestOf(blob)}}}, map[string][]byte{}) - if err != nil || ref != "m/tools@"+digestOf(blob) { - t.Fatalf("Put = %q, %v", ref, err) + got, err := b.h.Publish(ArtifactSeat, "x.mesh-delivery-builder", PublishRequest{Name: "m/tools", Version: "0.1.4", Kind: KindBundle, Digest: digestOf(blob), + Transfer: "republish-m-0-1-4-tools"}) + if err != nil || !got.Ended || len(b.said) != 1 || b.said[0].event != EventPublished || b.said[0].body.Reference != SeatReference(ArtifactSeat, "m/tools@"+digestOf(blob)) { + t.Fatalf("Publish = %+v, %v; said %+v", got, err, b.said) } var man struct { Layers []struct { - Size int64 `json:"size"` + Digest string `json:"digest"` + Size int64 `json:"size"` } `json:"layers"` } - if err := json.Unmarshal(store.manifests["m/tools:0.1.4"], &man); err != nil || len(man.Layers) != 1 || man.Layers[0].Size != int64(len(blob)) { + if err := json.Unmarshal(store.manifests["m/tools:0.1.4"], &man); err != nil || len(man.Layers) != 1 || man.Layers[0].Size != int64(len(blob)) || man.Layers[0].Digest != digestOf(blob) { t.Errorf("the version's manifest %s, want one layer of %d bytes", store.manifests["m/tools:0.1.4"], len(blob)) } - if _, err := o.Put(PublishRequest{Name: "m/tools", Version: "0.1.5", Kind: KindBundle, Digest: "sha256:" + strings.Repeat("e", 64), - Parts: []Part{{Digest: "sha256:" + strings.Repeat("e", 64)}}}, map[string][]byte{}); err == nil { - t.Error("a digest the store does not hold was tagged") + gone := "sha256:" + strings.Repeat("e", 64) + _, err = b.h.Publish(ArtifactSeat, "x.mesh-delivery-builder", PublishRequest{Name: "m/tools", Version: "0.1.5", Kind: KindBundle, Digest: gone, Transfer: "republish-m-0-1-5-tools"}) + if err == nil || !strings.Contains(err.Error(), "not stored: ") || !strings.Contains(err.Error(), gone) || store.manifests["m/tools:0.1.5"] != nil || + len(b.said) != 2 || b.said[1].event != EventRefused || !b.findings["publish-refused m/tools@0.1.5"] { + t.Errorf("a digest not held: %v; said %+v; findings %v", err, b.said, b.findings) } }) t.Run("mirror answers at once and says the copy's end on the seat's events", func(t *testing.T) { diff --git a/modules/distribution/internal/holder/oci.go b/modules/distribution/internal/holder/oci.go index 7b41d9feb..65abad32f 100644 --- a/modules/distribution/internal/holder/oci.go +++ b/modules/distribution/internal/holder/oci.go @@ -219,6 +219,25 @@ func (o *OCI) Held(name, version, digest string) (string, bool, error) { return name + "@" + d, true, nil } +// Holds is whether the store holds what r's digest names under r's name: an image's manifest by its digest, or a bundle's +// or an archive's blob. +func (o *OCI) Holds(r PublishRequest) (bool, error) { + if r.Kind != KindImage && r.Kind != KindUpstream { + return o.Has(r.Name, r.Digest) + } + resp, _, err := o.do(http.MethodHead, "/v2/"+r.Name+"/manifests/"+r.Digest, "", nil, accepted...) + if err != nil { + return false, err + } + switch resp.StatusCode { + case http.StatusOK: + return true, nil + case http.StatusNotFound: + return false, nil + } + return false, fmt.Errorf("the store answered %d for the manifest %s of %s", resp.StatusCode, r.Digest, r.Name) +} + // Versions are name's tags. func (o *OCI) Versions(name string) ([]string, error) { resp, b, err := o.do(http.MethodGet, "/v2/"+name+"/tags/list", "", nil) diff --git a/modules/distribution/internal/holder/transfer.go b/modules/distribution/internal/holder/transfer.go index bbd93b487..7b3bcba67 100644 --- a/modules/distribution/internal/holder/transfer.go +++ b/modules/distribution/internal/holder/transfer.go @@ -143,6 +143,9 @@ type Store interface { // Held is the reference's path of name at version when the store already holds digest under it: a rebuild of a held // fingerprint answers it and stores nothing. Held(name, version, digest string) (reference string, ok bool, err error) + // Holds says whether the store holds what a publish's digest names under its name: an image's manifest, or a bundle's or + // an archive's blob. The store is the artifact store (A-50): `artifact-store:///blobs/` is that blob. + Holds(r PublishRequest) (bool, error) } // SeatReference is a reference in the form the seat answers (novox/hq ADR 0605 §9): the seat's name, then the path the @@ -173,6 +176,20 @@ type Holder struct { // Idle writes a transfer's idle key with its own time to live, once (parts.BusArm over IdleState); the receiver // arms it on each open that leaves a transfer open. Nil arms nothing. Idle func(id string, ttl time.Duration) error + // Logf says each wait of a publish with its time (stderr in the bundle); nil says nothing. + Logf func(string, ...any) +} + +// timed runs f and logs its name and time when the holder logs. +func (h *Holder) timed(what string, f func() error) error { + if h.Logf == nil { + return f() + } + start := time.Now() + h.Logf("publish: %s starts", what) + err := f() + h.Logf("publish: %s ended after %s (%v)", what, time.Since(start).Round(time.Millisecond), err) + return err } func (h *Holder) now() time.Time { @@ -249,23 +266,83 @@ func (h *Holder) Publish(seat, caller string, r PublishRequest) (PublishAnswer, } // A rebuild of a held fingerprint answers the held version and stores nothing. if r.Version != "" { - if ref, ok, err := h.Store.Held(r.Name, r.Version, r.Digest); err != nil { + var ref string + var ok bool + err := h.timed("the store's read of what it holds (the program's registry)", func() (err error) { + ref, ok, err = h.Store.Held(r.Name, r.Version, r.Digest) + return err + }) + if err != nil { return PublishAnswer{}, fmt.Errorf("publish: the store cannot say whether it holds %s@%s: %w", r.Name, r.Version, err) } else if ok { - return PublishAnswer{Transfer: r.Transfer, Ended: true}, h.Say(seat, EventPublished, h.eventOf(seat, r, SeatReference(seat, ref), "")) + return PublishAnswer{Transfer: r.Transfer, Ended: true}, h.timed("the published event on the seat", func() error { + return h.Say(seat, EventPublished, h.eventOf(seat, r, SeatReference(seat, ref), "")) + }) } } + if r.Version != "" && len(r.Parts) == 0 { + return h.tagHeld(seat, r) + } wants := make([]parts.Want, 0, len(r.Parts)) for _, p := range r.Parts { wants = append(wants, parts.Want{Digest: p.Digest, Size: p.Size}) } - got, err := h.receiver().Open(r.Transfer, caller, wants, Opening{PublishRequest: r, Seat: seat}) + var got parts.Received + err := h.timed("the opening of the transfer on the bus (objects, transfer state, idle key)", func() (err error) { + got, err = h.receiver().Open(r.Transfer, caller, wants, Opening{PublishRequest: r, Seat: seat}) + return err + }) if err != nil { return answerOf(got), fmt.Errorf("publish: %w", err) } return answerOf(got), nil } +// tagHeld is a publish with a version, a digest and no parts (design 47 G A-417, A-73, A-50): what the store holds of that +// digest under the name (the build's untagged publish, the artifact store's blob) is checked against the builder's reservation +// and tagged with the version, and published is said; when the store holds none of it, the publish is refused as not stored, +// naming the digest. Nothing travels. +func (h *Holder) tagHeld(seat string, r PublishRequest) (PublishAnswer, error) { + var held bool + err := h.timed("the store's read of the digest it holds (the program's registry)", func() (err error) { + held, err = h.Store.Holds(r) + return err + }) + if err != nil { + return PublishAnswer{}, fmt.Errorf("publish: the store cannot say whether it holds %s of %s: %w", r.Digest, r.Name, err) + } + refuse := func(reason string) (PublishAnswer, error) { + h.Finding("publish-refused", r.Name+"@"+r.Version, reason, true) + if err := h.Say(seat, EventRefused, h.eventOf(seat, r, "", reason)); err != nil { + return PublishAnswer{}, fmt.Errorf("publish: %s, and the refused is not said: %w", reason, err) + } + return PublishAnswer{Transfer: r.Transfer, Ended: true}, errors.New("publish: " + reason) + } + if !held { + return refuse(fmt.Sprintf("not stored: the store holds no %s of %s, so version %s names nothing", r.Digest, r.Name, r.Version)) + } + ok, why, err := h.Builder.Recorded(r.Name, r.Version, r.Digest) + if err != nil { + return PublishAnswer{}, fmt.Errorf("publish: the reservation could not be read, so nothing is tagged: %w", err) + } + if !ok { + return refuse("the builder did not reserve it: " + why) + } + tag := r + tag.Parts = []Part{{Digest: r.Digest}} + var path string + if err := h.timed("the store's tag of the held digest (the program's registry)", func() (err error) { + path, err = h.Store.Put(tag, map[string][]byte{}) + return err + }); err != nil { + return refuse("the store did not tag it: " + err.Error()) + } + h.Finding("publish-refused", r.Name+"@"+r.Version, "", false) + return PublishAnswer{Transfer: r.Transfer, Ended: true}, h.timed("the published event on the seat", func() error { + return h.Say(seat, EventPublished, h.eventOf(seat, r, SeatReference(seat, path), "")) + }) +} + // PublishPart is `publish-part`: one chunk, from the caller that opened the transfer alone. func (h *Holder) PublishPart(seat, caller string, r PartRequest) (PublishAnswer, error) { got, err := h.receiver().Take(r.Transfer, caller, r.Digest, r.Sequence, r.Last, r.Data, func(t Transfer) error { diff --git a/modules/gitea/seat.ts b/modules/gitea/seat.ts index 8cd08bd95..1c0c97124 100644 --- a/modules/gitea/seat.ts +++ b/modules/gitea/seat.ts @@ -152,7 +152,10 @@ export function seatTools(gitea: GiteaClient, deps: SeatDeps): ToolDefinition[] verb(GIT_VERBS.branch, "The seat git's branch: a branch's head, and whether a protection rule refuses a direct push and admits only merges.", async (args): Promise => { const { owner, repo, full } = repositoryOf(args.repository); - const branch = String(args.branch ?? ""); + // No branch named is the repository's default branch, read from the forge, as the verb file reads it (A-399). + let branch = String(args.branch ?? ""); + if (branch === "") branch = String((await gitea.getRepo(owner, repo)).default_branch ?? ""); + if (branch === "") throw new Error(`the forge names no default branch of ${full}`); // The head first: a branch the forge does not hold is its 404, the kind not-found, never an unprotected branch. const head = await gitea.branchHead(owner, repo, branch); return { repository: full, branch, head, protected: pushRefused(await gitea.appliedProtection(owner, repo, branch)) }; diff --git a/modules/mesh-build-go/cmd/mesh-build-go/digest.go b/modules/mesh-build-go/cmd/mesh-build-go/digest.go index fe7c06c79..9aa9c4616 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/digest.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/digest.go @@ -260,7 +260,7 @@ func (m *Module) digestHere(ctx context.Context, workspace, level string, names namesHeader = []string{"node-names " + strings.Join(sortedNames, ",")} } - dirs, err := m.goModDirs(ctx, tree) + dirs, err := m.testDirs(ctx, tree) if err != nil { return DigestResult{}, err } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/fetch.go b/modules/mesh-build-go/cmd/mesh-build-go/fetch.go index b4d5ab854..04a18aadf 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/fetch.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/fetch.go @@ -20,6 +20,8 @@ import ( "sync" "time" + "golang.org/x/mod/modfile" + sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" dforgekey "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/forgekey" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/steprun" @@ -106,6 +108,8 @@ func (m *Module) fetch(ctx context.Context, args map[string]any) (res any, err e if err != nil { return nil, err } + // A fetch with no dir (the checker's check of a pull request, until G removes the checker; A-447e) fetches the tree + // whole: every go.mod of it. siblings, err := siblingsArg(args["beside"]) if err != nil { return nil, err @@ -165,6 +169,20 @@ func (m *Module) fetchInto(ctx context.Context, req sdkstep.FetchRequest, id str if err != nil { return nil, err } + if dir := cleanDir(req.Dir); dir != "" { + // A fetch asked with a module's directory is of that Go module and what its go.mod replaces with a local path, + // never another Go module of the tree (design 47 G, A-447). Said beside the tree, so every later step lists the + // same modules. + if dirs, err = scopedDirs(tree, dir, dirs); err != nil { + return nil, err + } + if err := os.WriteFile(filepath.Join(root, scopeMark), []byte(strings.Join(dirs, "\n")), 0o600); err != nil { + return nil, err + } + if err := os.WriteFile(filepath.Join(root, moduleDirMark), []byte(dir), 0o600); err != nil { + return nil, err + } + } res.Modules = dirs if base != "" && res.Merge == "" { return res, nil // a conflict: the tree is at the trunk, nothing of the head is built or tested @@ -434,7 +452,11 @@ func (m *Module) downloadAndVerify(ctx context.Context, dir, d string) error { } // goModDirs are the directories of the tree that hold a go.mod, relative to it, in order. +// When the fetch was asked with a module's directory (scopeMark), the steps after it see only the directories it chose. func (m *Module) goModDirs(ctx context.Context, tree string) ([]string, error) { + if b, err := os.ReadFile(filepath.Join(filepath.Dir(tree), scopeMark)); err == nil { + return strings.Fields(string(b)), nil + } r := m.Run(ctx, nil, "git", "-C", tree, "ls-files", "--", "go.mod", "*/go.mod") if r.Status != 0 || r.Err != "" { return nil, fmt.Errorf("git ls-files: %s", dforgekey.Said(r.Err+"\n"+r.Stderr)) @@ -454,6 +476,27 @@ func (m *Module) goModDirs(ctx context.Context, tree string) ([]string, error) { return dirs, nil } +// testDirs are the directories the unit and integration steps and the digest take: the chosen go.mod directories at or +// under the module's own directory. A directory the module's go.mod replaces from outside its own stays for download and +// resolution alone; its tests are another module's (design 47 G, A-447c). A workspace fetched with no dir keeps every go.mod. +func (m *Module) testDirs(ctx context.Context, tree string) ([]string, error) { + all, err := m.goModDirs(ctx, tree) + if err != nil { + return nil, err + } + own, ok := moduleDirOf(tree) + if !ok || own == "." || own == "" { + return all, nil + } + out := []string{} + for _, d := range all { + if d == own || strings.HasPrefix(d, own+"/") { + out = append(out, d) + } + } + return out, nil +} + // inTestdata says a path lies under a directory named testdata, whose go.mod files are fixtures Go ignores, not modules // of the tree (issue 1810). func inTestdata(path string) bool { @@ -608,3 +651,85 @@ func narrowed(tree string) bool { _, err := os.Stat(filepath.Join(filepath.Dir(tree), narrowedMark)) return err == nil } + +// scopeMark is the file beside a workspace's tree naming the Go module directories a fetch asked with a module's +// directory chose, one to a line (empty: none). +const scopeMark = "gomodules" + +// moduleDirMark is the file beside a workspace's tree naming the module directory the fetch was asked with: the one +// module.json a step compares generated files for (design 47 G, A-447c). +const moduleDirMark = "moduledir" + +// moduleDirOf is the module directory the workspace of tree was fetched for, ok false for a workspace with none. +func moduleDirOf(tree string) (string, bool) { + b, err := os.ReadFile(filepath.Join(filepath.Dir(tree), moduleDirMark)) + if err != nil { + return "", false + } + return string(b), true +} + +// cleanDir is a request's module directory in the form goModDirs lists, "" for none. +func cleanDir(dir string) string { + if strings.TrimSpace(dir) == "" { + return "" + } + return strings.Trim(filepath.ToSlash(filepath.Clean(dir)), "/") +} + +// scopedDirs are the go.mod directories of all that a module directory dir chooses: dir itself and each directory its +// go.mod replaces with a local path, followed through their own replaces (design 47 G, A-447), in order. A directory +// the tree holds no go.mod for is left out. A directory holding no go.mod chooses every go.mod under it, and one that +// chooses no Go module at all is an error naming it (A-447a). +func scopedDirs(tree, dir string, all []string) ([]string, error) { + have := map[string]bool{} + for _, d := range all { + have[d] = true + } + chosen := map[string]bool{} + todo := []string{dir} + if !have[dir] { + // A directory with no go.mod of its own (a module whose Go code lies in go/, a contracts module with a go.mod per + // seat) is every go.mod under it; the tree is already narrowed to the module's paths (A-447a). + todo = nil + for _, d := range all { + if dir == "." || strings.HasPrefix(d, dir+"/") { + todo = append(todo, d) + } + } + } + for len(todo) > 0 { + d := todo[0] + todo = todo[1:] + if chosen[d] || !have[d] { + continue + } + chosen[d] = true + file := filepath.Join(tree, filepath.FromSlash(d), "go.mod") + body, err := os.ReadFile(file) + if err != nil { + return nil, err + } + f, err := modfile.Parse(file, body, nil) // not ParseLax: that one ignores the replaces + if err != nil { + return nil, fmt.Errorf("%s: %w", file, err) + } + for _, r := range f.Replace { + if !modfile.IsDirectoryPath(r.New.Path) { + continue + } + target := filepath.ToSlash(filepath.Clean(filepath.Join(filepath.FromSlash(d), filepath.FromSlash(r.New.Path)))) + todo = append(todo, target) + } + } + out := []string{} + for _, d := range all { + if chosen[d] { + out = append(out, d) + } + } + if len(out) == 0 { + return nil, fmt.Errorf("the module directory %q holds no Go module (no go.mod in it or under it)", dir) + } + return out, nil +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/fetch_test.go b/modules/mesh-build-go/cmd/mesh-build-go/fetch_test.go index 75c8823c9..aeaa3e44f 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/fetch_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/fetch_test.go @@ -61,6 +61,13 @@ func (f *fakeStep) run(ctx context.Context, env []string, name string, args ...s os.MkdirAll(filepath.Dir(filepath.Join(args[i+1], name)), 0o755) os.WriteFile(filepath.Join(args[i+1], name), []byte(text), 0o644) } + // Every go.mod the fake lists exists, since a fetch asked with a directory reads its go.mod (A-447). + for _, l := range strings.Fields(f.gomods) { + if _, err := os.Stat(filepath.Join(args[i+1], l)); err != nil && f.files[l] == "" { + os.MkdirAll(filepath.Dir(filepath.Join(args[i+1], l)), 0o755) + os.WriteFile(filepath.Join(args[i+1], l), []byte("module fake\n"), 0o644) + } + } for _, v := range f.vendors { os.MkdirAll(filepath.Join(args[i+1], v, "vendor"), 0o755) os.WriteFile(filepath.Join(args[i+1], v, "vendor", "modules.txt"), nil, 0o644) @@ -91,7 +98,24 @@ func fetchModule(t *testing.T, f *fakeStep) (*Module, *keptStates) { return m, states } +// fetchTool is the fetch tool as a build module is asked, always with its module's directory (A-447b): the root of the +// tree unless the args name one. rawFetchTool is the tool as it is. func fetchTool(t *testing.T, m *Module) func(map[string]any) (any, error) { + t.Helper() + raw := rawFetchTool(t, m) + return func(args map[string]any) (any, error) { + if _, ok := args["dir"]; !ok { + named := map[string]any{"dir": "."} + for k, v := range args { + named[k] = v + } + args = named + } + return raw(args) + } +} + +func rawFetchTool(t *testing.T, m *Module) func(map[string]any) (any, error) { t.Helper() for _, tool := range m.Tools(m.StepTools()) { if tool.Name == "build_go_fetch" { @@ -107,7 +131,9 @@ var fetchArgs = map[string]any{"repository": "novox/mesh-sdk", "commit": commitS // 1485/AC1: git is asked through FetchCommit with the ssh remote, and go mod download and verify run in each go.mod // directory (not in one that vendors) with GOTOOLCHAIN=local, GOFLAGS=-mod=mod and the module's cache paths. func TestFetchAsksGitForTheSshRemoteAndDownloadsEachModule(t *testing.T) { - f := &fakeStep{gomods: "go.mod\nsub/dir/go.mod\nvend/go.mod\n", vendors: []string{"vend"}} + // The root go.mod replaces the other two by directory: a fetch asked with a module's directory chooses what it replaces. + f := &fakeStep{gomods: "go.mod\nsub/dir/go.mod\nvend/go.mod\n", vendors: []string{"vend"}, + files: map[string]string{"go.mod": "module fake\nreplace a => ./sub/dir\nreplace b => ./vend\n"}} m, states := fetchModule(t, f) if _, err := fetchTool(t, m)(fetchArgs); err != nil { t.Fatal(err) @@ -270,7 +296,8 @@ func TestAVerifyThatFailsEndsTheStepNamingTheDirectoryAndTheLine(t *testing.T) { return forgekey.Ran{} }} m, states := fetchModule(t, f) - if _, err := fetchTool(t, m)(fetchArgs); err != nil { + // Asked for the module of tools, as a build is for one module (A-447b). + if _, err := fetchTool(t, m)(map[string]any{"repository": "novox/mesh-sdk", "commit": commitSHA, "dir": "tools"}); err != nil { t.Fatal(err) } st := states.waitEnd(t) diff --git a/modules/mesh-build-go/cmd/mesh-build-go/gomoddirs_test.go b/modules/mesh-build-go/cmd/mesh-build-go/gomoddirs_test.go index dfd9bb678..1a7a8d429 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/gomoddirs_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/gomoddirs_test.go @@ -2,9 +2,13 @@ package main import ( "context" + "os" + "path/filepath" "reflect" + "strings" "testing" + sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" "git.novox.be/novox/mesh-sdk/go/forgekey" ) @@ -25,3 +29,123 @@ func TestGoModDirsLeaveOutTheFixturesUnderTestdata(t *testing.T) { t.Fatalf("go.mod directories = %v, want %v", got, want) } } + +// A-447: a fetch asked with a module's directory chooses that Go module and the directories its go.mod replaces with a +// local path, through their own replaces; the later steps list the same. The takein module of the indexer is no module +// of the assigner's build (live, 2026-11-16). +func TestAFetchAskedWithADirectoryChoosesItsModuleAndWhatItReplacesAlone(t *testing.T) { + all := []string{".", "modules/assigner", "modules/indexer", "modules/indexer/takein", "modules/lib", "modules/lib/deep", "modules/other", "rootless/x/go", "seats/a", "seats/b"} + mods := map[string]string{ + "modules/assigner": "module a\nreplace x => ../lib\nreplace (\n\ty => ../../outside\n\tz v1.0.0 => z v1.1.0\n)\n", + "modules/indexer": "module i\nreplace t => ./takein\n", + "modules/indexer/takein": "module t\n", + "modules/lib": "module l\nreplace d => ./deep\n", + "modules/lib/deep": "module d\n", + "modules/other": "module o\n", + ".": "module r\n", + "seats/a": "module sa\nreplace q => ../q\n", + "seats/b": "module sb\n", + "rootless/x/go": "module rx\n", + } + for _, c := range []struct { + dir string + want []string + }{ + {"modules/assigner", []string{"modules/assigner", "modules/lib", "modules/lib/deep"}}, + {"modules/indexer", []string{"modules/indexer", "modules/indexer/takein"}}, + {"modules/other/", []string{"modules/other"}}, + {"modules/lib/deep/", []string{"modules/lib/deep"}}, + {"seats", []string{"seats/a", "seats/b"}}, // a go.mod per seat, none in seats itself + {".", []string{"."}}, // the root holds a go.mod + {"rootless/x", []string{"rootless/x/go"}}, // Go code lies in go/ + {"modules/none", nil}, // chooses no Go module: an error naming it + } { + tree := t.TempDir() + for d, body := range mods { + if err := os.MkdirAll(filepath.Join(tree, d), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(tree, d, "go.mod"), []byte(body), 0o600); err != nil { + t.Fatal(err) + } + } + got, err := scopedDirs(tree, cleanDir(c.dir), all) + if c.want == nil { + if err == nil || !strings.Contains(err.Error(), c.dir) { + t.Errorf("dir %q chose %v, %v; want an error naming it", c.dir, got, err) + } + continue + } + if err != nil || !reflect.DeepEqual(got, c.want) { + t.Errorf("dir %q chooses %v, %v; want %v", c.dir, got, err, c.want) + } + } + // What the steps after the fetch list is what the fetch chose, beside the tree. + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, scopeMark), []byte("modules/assigner\nmodules/lib"), 0o600); err != nil { + t.Fatal(err) + } + m := &Module{} + got, err := m.goModDirs(context.Background(), filepath.Join(root, "tree")) + if err != nil || !reflect.DeepEqual(got, []string{"modules/assigner", "modules/lib"}) { + t.Errorf("the later steps list %v, %v", got, err) + } + if cleanDir("") != "" || cleanDir(".") != "." { + t.Errorf("cleanDir of nothing = %q, of . = %q", cleanDir(""), cleanDir(".")) + } +} + +// A-447e: a fetch with no dir (the checker's check of a pull request, until G removes the checker) fetches the tree whole: +// it chooses every go.mod of it. +func TestAFetchWithNoDirectoryChoosesEveryGoModOfTheTree(t *testing.T) { + f := &fakeStep{gomods: "go.mod\nsub/go.mod\n"} + m, states := fetchModule(t, f) + if _, err := rawFetchTool(t, m)(map[string]any{"repository": "novox/mesh-sdk", "commit": commitSHA}); err != nil { + t.Fatal(err) + } + st := states.waitEnd(t) + if st.State != sdkstep.StepPassed || !strings.Contains(string(st.Result), `"modules":[".","sub"]`) { + t.Fatalf("a fetch with no dir ended %s: %q %s", st.State, st.Reason, st.Result) + } +} + +// A-447c: the unit level of a module whose go.mod replaces ../lib tests the module alone; lib is downloaded and resolved +// for it, and its tests are another module's. +func TestTheUnitLevelTestsOnlyTheModulesOwnDirectoriesNotWhatItReplaces(t *testing.T) { + f := &fakeStep{} + m, _ := fetchModule(t, f) + root := filepath.Join(m.Work, "ws1") + tree := filepath.Join(root, "tree") + for d, body := range map[string]string{"mod": "module example.org/mod\nreplace example.org/lib => ../lib\n", "lib": "module example.org/lib\n"} { + if err := os.MkdirAll(filepath.Join(tree, d), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(tree, d, "go.mod"), []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + for name, body := range map[string]string{scopeMark: "lib\nmod", moduleDirMark: "mod"} { + if err := os.WriteFile(filepath.Join(root, name), []byte(body), 0o600); err != nil { + t.Fatal(err) + } + } + dirs, err := m.testDirs(context.Background(), tree) + if err != nil || !reflect.DeepEqual(dirs, []string{"mod"}) { + t.Fatalf("the test directories = %v, %v; want mod alone", dirs, err) + } + if _, err := m.runLevel(context.Background(), tree, levelOpts{workspace: "ws1", level: LevelUnit}); err != nil { + t.Fatal(err) + } + for _, l := range f.lines() { + if strings.Contains(l, "/tree/lib test") { + t.Errorf("the module's unit level tested lib: %s", l) + } + } + // No dir asked: every go.mod of the tree stays. + if err := os.Remove(filepath.Join(root, moduleDirMark)); err != nil { + t.Fatal(err) + } + if dirs, _ = m.testDirs(context.Background(), tree); len(dirs) != 2 { + t.Errorf("a workspace with no module directory tests %v, want every go.mod", dirs) + } +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/integration.go b/modules/mesh-build-go/cmd/mesh-build-go/integration.go index 86c9d6db5..cfa51ba98 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/integration.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/integration.go @@ -58,7 +58,7 @@ func (m *Module) integrationHere(ctx context.Context, workspace, tree string, pa defer os.RemoveAll(run) began := m.clock() - dirs, err := m.goModDirs(ctx, tree) + dirs, err := m.testDirs(ctx, tree) if err != nil { return nil, err } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/main.go b/modules/mesh-build-go/cmd/mesh-build-go/main.go index 1c404a996..7886aaca6 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/main.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/main.go @@ -87,8 +87,9 @@ func main() { go func() { <-stdio.Served() if err := findings.Start(ctx); err != nil { - logf("the finding writer: %v", err) - os.Exit(1) + // Said, and the module goes on serving: a finding writer that cannot hear its asks keeps its findings + // in its state alone, as the indexer does (lead answer A-394); its builds and steps never wait on it. + logf("the finding writer cannot start, its findings are kept in the state alone: %v", err) } }() } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/mirror_test.go b/modules/mesh-build-go/cmd/mesh-build-go/mirror_test.go index af3f63d02..a71ca19f0 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/mirror_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/mirror_test.go @@ -176,6 +176,10 @@ func TestTheWorkspaceIsSelfContainedAfterAFetchThroughTheMirror(t *testing.T) { return strings.TrimSpace(string(out)) } git(src, "init", "-q") + // A fetch is for one module (A-447b): the repository holds a module at its root. + if err := os.WriteFile(filepath.Join(src, "go.mod"), []byte("module example.org/x\n\ngo 1.26\n"), 0o644); err != nil { + t.Fatal(err) + } if err := os.WriteFile(filepath.Join(src, "f.txt"), []byte("one\n"), 0o644); err != nil { t.Fatal(err) } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/narrow_integration_test.go b/modules/mesh-build-go/cmd/mesh-build-go/narrow_integration_test.go index 2ce32371e..e7ef09577 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/narrow_integration_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/narrow_integration_test.go @@ -45,7 +45,7 @@ func TestAFetchWithNamedPathsNarrowsTheWorkspace(t *testing.T) { if strings.Join(res.Modules, ",") != "lib" { t.Errorf("the fetch answered the modules %v, want lib alone", res.Modules) } - _, st = fetched(t, m, states, map[string]any{"repository": "novox/narrow", "commit": head, "paths": []any{"cmd"}}) + _, st = fetched(t, m, states, map[string]any{"repository": "novox/narrow", "commit": head, "dir": "", "paths": []any{"cmd"}}) if st.State != sdkstep.StepFailed || !strings.Contains(st.Reason, "no module directory") { t.Errorf("paths without a directory: %s %q", st.State, st.Reason) } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/pins_test.go b/modules/mesh-build-go/cmd/mesh-build-go/pins_test.go index 17b5f20e8..5b957bc42 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/pins_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/pins_test.go @@ -50,7 +50,15 @@ func newPinHarness(t *testing.T, gomod string, vendor bool, pulls string) (*pinH for _, g := range strings.Fields(f.gomods) { d := filepath.Join(args[i+1], filepath.Dir(g)) os.MkdirAll(d, 0o755) - os.WriteFile(filepath.Join(d, "go.mod"), []byte(gomod), 0o644) + body := gomod + if g == "go.mod" { // the root replaces the other directories, so a fetch asked with it chooses them (A-447) + for _, o := range strings.Fields(f.gomods) { + if o != "go.mod" { + body += "\nreplace local/" + filepath.Dir(o) + " => ./" + filepath.Dir(o) + "\n" + } + } + } + os.WriteFile(filepath.Join(d, "go.mod"), []byte(body), 0o644) os.WriteFile(filepath.Join(d, "go.sum"), []byte("sum\n"), 0o644) } if vendor { diff --git a/modules/mesh-build-go/cmd/mesh-build-go/run.go b/modules/mesh-build-go/cmd/mesh-build-go/run.go index cdd204725..24c9e50eb 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/run.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/run.go @@ -36,7 +36,7 @@ func (m *Module) run(ctx context.Context, fetch steprun.StepFunc, a sdkstep.Step return nil, err } key := sdkstep.RunKey(job.Program, sdkstep.ToolchainUsed{Version: installed, Recipe: m.Recipe.Version()}, a.Packages) - fetchArgs := sdkstep.FetchRequest{Repository: job.Program.Repository, Commit: job.Program.Commit, Packages: a.Packages}.Args() + fetchArgs := sdkstep.FetchRequest{Repository: job.Program.Repository, Commit: job.Program.Commit, Dir: job.Program.Dir, Packages: a.Packages}.Args() exe, built, err := m.programOf(ctx, fetch, "programs", "checking program", key, fetchArgs, job.Program.Dir, job.Program.Package) if err != nil { return nil, err diff --git a/modules/mesh-build-go/cmd/mesh-build-go/test.go b/modules/mesh-build-go/cmd/mesh-build-go/test.go index 4ec9307df..e76b9b2c5 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/test.go @@ -109,18 +109,22 @@ func (m *Module) test(ctx context.Context, args map[string]any) (any, error) { } // compareGenerated regenerates the generated Go files of each contracts module in the tree (every module.json in the tree, in -// any folder: ADR 0634 §8; CAT-BG5) and fails the level on any out of step, Why naming each path. +// any folder: ADR 0634 §8; CAT-BG5; for a workspace fetched for a module's directory, that module's own alone) and fails the level on any out of step, Why naming each path. func compareGenerated(tree string, res *TestResult) error { dirs, err := steprun.ModuleDirs(tree) if err != nil { return err } for _, dir := range dirs { + // A step for a module compares its own module.json alone, never another module's of the repository (A-447c). + rel, _ := filepath.Rel(tree, dir) + if own, ok := moduleDirOf(tree); ok && filepath.ToSlash(rel) != own { + continue + } paths, err := steprun.OutOfStep(dir, steprun.GeneratedRootGo) if err != nil { return err } - rel, _ := filepath.Rel(tree, dir) for _, p := range paths { res.OutOfStep = append(res.OutOfStep, filepath.ToSlash(filepath.Join(rel, p))) } @@ -252,7 +256,7 @@ type levelOpts struct { // runLevel runs `go test -json` in each go.mod directory of the tree and judges the whole against the bounds. func (m *Module) runLevel(ctx context.Context, tree string, o levelOpts) (*TestResult, error) { - dirs, err := m.goModDirs(ctx, tree) + dirs, err := m.testDirs(ctx, tree) if err != nil { return nil, err } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/test_test.go b/modules/mesh-build-go/cmd/mesh-build-go/test_test.go index 0931f3dcd..779b4dba5 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/test_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/test_test.go @@ -340,3 +340,36 @@ func TestADirectoryThatVendorsIsReadFromItsVendorDirectory(t *testing.T) { t.Errorf("the last GOFLAGS of go test in a vendoring directory is %q", flags) } } + +// A-447c: a workspace fetched for a module's directory compares the generated files of that module's own module.json +// alone, never another module's of the repository. +func TestUnitLevelComparesOnlyTheOwnModulesGeneratedFiles(t *testing.T) { + m, states, ws, _ := workspaceFor(t, "empty") + tree := filepath.Join(m.Work, ws, "tree") + for _, folder := range []string{"domain", "other"} { + for p, text := range map[string]string{ + "module.json": `{"module":"c","build":{"artifacts":[{"name":"go","kind":"package","language":"go","paths":["go"],"generated-from":"."}]}}`, + "seats/one-store/seat.json": `{"format":1,"seat":"one-store","scope":"mesh","grain":"seat-list","domain":"Module","decision":"ADR 1","verbs":[{"name":"query","input":{"type":"object"}}]}`, + "go/go.mod": "module example.org/c/go\n", + "go/stray.go": "package stray\n", + } { + full := filepath.Join(tree, folder, filepath.FromSlash(p)) + _ = os.MkdirAll(filepath.Dir(full), 0o755) + if err := os.WriteFile(full, []byte(text), 0o644); err != nil { + t.Fatal(err) + } + } + } + if err := os.WriteFile(filepath.Join(m.Work, ws, moduleDirMark), []byte("domain"), 0o600); err != nil { + t.Fatal(err) + } + res, _ := runUnit(t, m, states, ws) + if len(res.OutOfStep) == 0 { + t.Fatalf("nothing out of step: %+v", res) + } + for _, p := range res.OutOfStep { + if !strings.HasPrefix(p, "domain/") { + t.Errorf("out of step = %v, want the module's own files under domain/ alone", res.OutOfStep) + } + } +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/toolchainreport.go b/modules/mesh-build-go/cmd/mesh-build-go/toolchainreport.go index 4ddfe4fd2..bee85bf34 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/toolchainreport.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/toolchainreport.go @@ -74,6 +74,8 @@ func putReporting(put steprun.Put, r *toolchainReporter, digest func(version str } if eerr := r.Report(key, rev, t); eerr != nil && say != nil { say("toolchain-reported could not be said: %v", eerr) + } else if eerr == nil && say != nil { + say("toolchain-reported said: %s %s (key %s, revision %d)", t.Language, t.Version, key, rev) } return rev, nil } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/verify_once_test.go b/modules/mesh-build-go/cmd/mesh-build-go/verify_once_test.go index 3305e5cfd..cf7beee85 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/verify_once_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/verify_once_test.go @@ -53,7 +53,7 @@ func TestGoModVerifyRunsOncePerGoModAndGoSumDigest(t *testing.T) { // Without bubblewrap the module cache is not read-only to a step: a pass is never reused. func TestAPassOfVerifyIsNotReusedWhenStepsAreNotShutOff(t *testing.T) { - f := &fakeStep{gomods: "a/go.mod\nb/go.mod\n", files: map[string]string{"a/go.mod": "m\n", "b/go.mod": "m\n"}} + f := &fakeStep{gomods: "a/go.mod\nb/go.mod\n", files: map[string]string{"a/go.mod": "module m\n", "b/go.mod": "module m\n"}} m, states := fetchModule(t, f) // no Bubblewrap fetchOnce(t, m, states, fetchArgs) fetchOnce(t, m, states, fetchArgs) @@ -65,7 +65,7 @@ func TestAPassOfVerifyIsNotReusedWhenStepsAreNotShutOff(t *testing.T) { // A verify that fails leaves no mark: the next fetch verifies again. func TestAFailedVerifyLeavesNoMark(t *testing.T) { fail := true - f := &fakeStep{gomods: "go.mod\n", files: map[string]string{"go.mod": "m\n"}} + f := &fakeStep{gomods: "go.mod\n", files: map[string]string{"go.mod": "module m\n"}} f.goRule = func(line string) forgekey.Ran { if strings.HasSuffix(line, " mod verify") && fail { return forgekey.Ran{Status: 1, Stdout: "x v1: dir has been modified\n"} @@ -88,7 +88,7 @@ func TestDownloadsRunSideBySideUpToTheBound(t *testing.T) { running, most := 0, 0 release := make(chan struct{}) // closed once two run at once: they overlap however slowly the machine starts them f := &fakeStep{gomods: "a/go.mod\nb/go.mod\nc/go.mod\nd/go.mod\n", files: map[string]string{ - "a/go.mod": "a\n", "b/go.mod": "b\n", "c/go.mod": "c\n", "d/go.mod": "d\n"}} + "a/go.mod": "module a\n", "b/go.mod": "module b\n", "c/go.mod": "module c\n", "d/go.mod": "module d\n"}} f.goRule = func(line string) forgekey.Ran { if strings.HasSuffix(line, " mod download") { mu.Lock() diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/deliveries.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/deliveries.go index 24780ce18..42f700bd5 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/deliveries.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/deliveries.go @@ -69,6 +69,8 @@ type DeliveryStore interface { type ForgeReads interface { OpenPulls() (gitseat.OpenPullsAnswer, error) Pull(repository string, number int64) (gitseat.PullAnswer, error) + // Branch is a repository branch's head (the seat git's verb branch). + Branch(repository, branch string) (gitseat.BranchAnswer, error) } // kvKept is the store over the runtime. diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/deliveries_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/deliveries_test.go index 7710e6fe1..7fd363538 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/deliveries_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/deliveries_test.go @@ -41,6 +41,7 @@ func (k *memKept) Delete(state, key string) error { delete(k.m[state], key); ret type fakeForge struct { open []gitseat.OpenPull pulls map[string]gitseat.Pull + heads map[string]string // "@" -> head fail bool } @@ -50,6 +51,16 @@ func (f *fakeForge) OpenPulls() (gitseat.OpenPullsAnswer, error) { } return gitseat.OpenPullsAnswer{Pulls: f.open}, nil } +func (f *fakeForge) Branch(repository, branch string) (gitseat.BranchAnswer, error) { + if f.fail { + return gitseat.BranchAnswer{}, errors.New("the forge is away") + } + head, ok := f.heads[repository+"@"+branch] + if !ok { + return gitseat.BranchAnswer{Error: "no such branch"}, nil + } + return gitseat.BranchAnswer{Branch: branch, Repository: repository, Head: head}, nil +} func (f *fakeForge) Pull(repository string, number int64) (gitseat.PullAnswer, error) { p, ok := f.pulls[fmt.Sprintf("%s#%d", repository, number)] if !ok { diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/gate.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/gate.go index 94de1c9b7..05222b46c 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/gate.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/gate.go @@ -23,11 +23,14 @@ type Holder struct { // Merges and Old are the old sections the switch reads (switch.go); Source is the gate's source (merges.go). Merges MergeStore // Kept is the gate's states deliveries and groups, and Forge the seat git's reads (deliveries.go, §716). - Kept DeliveryStore - Forge ForgeReads - dmu sync.Mutex - Old OldState - Source SourceStore + Kept DeliveryStore + Forge ForgeReads + // Catalog reads the seat mesh-catalog's registrations (sourcestart.go); startMu keeps one start of the sources at a time. + Catalog Registered + startMu sync.Mutex + dmu sync.Mutex + Old OldState + Source SourceStore // Owed says what the gate owes the forge as the seat's event owed (owed.go). Owed OwedSayer // Records keeps the delivery records on the gate's ledger (record.go, ADR 0656 §7); nil keeps none. diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go index 60984b018..1992a61e6 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go @@ -32,7 +32,7 @@ func (l *listening) get() string { l.mu.Lock(); defer l.mu.Unlock(); return l.n // The events this owner follows: the forge bridge's word on an act owed, and the step events the delivery records are built // from (consumer.go). -var followed = append(append([]string{owedEndedEvent}, stepPatterns()...), pullUpdatedEvent, pullMergedEvent, pullClosedEvent) +var followed = append(append([]string{owedEndedEvent}, stepPatterns()...), pullUpdatedEvent, pullMergedEvent, pullClosedEvent, registrationChangedEvent) func main() { settingsFile := os.Getenv("MESH_DELIVERY_GATE_SETTINGS") @@ -68,6 +68,7 @@ func main() { h.Old = kvOld{} h.Kept = kvKept{} h.Forge = seatGit{ask: stdio.Ask} + h.Catalog = seatCatalog{ask: stdio.Ask} h.Consumer = &StepConsumer{Records: h.Records, Families: stepFamilies, Now: h.Now, Logf: logf, Taken: h.TakenIn} // The gate's one held writer of its own findings (design 47 G A-39, A-37, A-40): made at once, started once the runtime serves. fw, err := NewFindingWriter() @@ -116,6 +117,8 @@ func run(h *Holder, l *listening) { } l.set("listening") logf("[mesh-delivery] listening: %s", strings.Join(followed, ", ")) + // A watched branch no source key holds starts from the forge, once at each start and at each registration change (A-401). + go startSourcesAtStart(h) // A missed pull event is put right from the forge's own record, once at each start (§716). if err := h.ReadForge(); err != nil { logf("[mesh-delivery] the open pull requests were not read from the forge, they are read again at the next start: %v", err) @@ -172,6 +175,10 @@ func take(h *Holder, e stdio.Envelope) error { if e.Key == pullMergedEvent { return errors.Join(h.DeliveryMerged(e.Body), h.pullEnded(e.Body, "merged")) } + if e.Key == registrationChangedEvent { + _, err := h.StartSources() + return err + } if e.Key == pullClosedEvent { return errors.Join(h.DeliveryClosed(e.Body), h.pullEnded(e.Body, "closed")) } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go index a679e1fa7..7e060757d 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go @@ -181,7 +181,7 @@ func TestItClaimsTheDeliverySeatAndFollowsWhatItHandles(t *testing.T) { } // It calls no controller verb and no forge verb: the forge's reads and writes are the bridge's (ADR 0437 §8; design 47 §592). // It uses operator-channel only to say its conditions (ADR 0437 §9, issue 1275): the teller asks no more (§431). - want := []string{"seat:git.open-pulls", "seat:git.pull", "seat:mesh-hand-acts.hand-act", WantsEntryVerb, EngineCompositionVerb} + want := []string{"seat:git.branch", "seat:git.open-pulls", "seat:git.pull", "seat:mesh-catalog.registrations", "seat:mesh-hand-acts.hand-act", WantsEntryVerb, EngineCompositionVerb} if !reflect.DeepEqual(m.Uses, []string{"operator-channel"}) { t.Fatalf("uses %v: operator-channel, for its conditions alone", m.Uses) } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/sourcestart.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/sourcestart.go new file mode 100644 index 000000000..f7538a412 --- /dev/null +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/sourcestart.go @@ -0,0 +1,236 @@ +package main + +// A watched branch starts from the forge, as a clone does (design 47 G A-401, §430, §15): at its start and on each +// registration-changed of the seat mesh-catalog, the gate writes source. for every watched branch the registrations name, +// each registered repository at the head of that branch, asked of the seat git's verb branch, the group left empty, and says +// source-moved. A repository whose head cannot be read is left out and said. A head a key holds is never changed here (A-403): from +// then on only a take-in moves it (A-34). + +import ( + "encoding/json" + "errors" + "fmt" + "sort" + "strconv" + "strings" + "time" + + "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" + gitseat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git" + meshcatalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" + "git.novox.be/novox/mesh-sdk/go/keys" +) + +// registrationChangedEvent is the seat mesh-catalog's event, as the gate's manifest consumes it. +var registrationChangedEvent = declared(meshcatalog.EventRegistrationChanged) + +// Registered is what the gate reads of the seat mesh-catalog: its registrations. +type Registered interface { + Registrations() (meshcatalog.RegistrationsAnswer, error) +} + +// seatCatalog reads the seat mesh-catalog through the tool runner. +type seatCatalog struct{ ask asking } + +func (s seatCatalog) Registrations() (meshcatalog.RegistrationsAnswer, error) { + var a meshcatalog.RegistrationsAnswer + raw, err := s.ask("seat:"+meshcatalog.Seat+"."+meshcatalog.VerbRegistrations, meshcatalog.RegistrationsInput{}) + if err != nil { + return a, err + } + answer, said, ok := answerOf(raw) + if !ok { + return a, fmt.Errorf("registrations refused: %s", lastLine(said)) + } + if err := json.Unmarshal(answer, &a); err != nil { + return a, fmt.Errorf("the answer of registrations is not readable: %w", err) + } + return a, nil +} + +// Branch is the seat git's verb branch: a repository's branch head. +func (s seatGit) Branch(repository, branch string) (gitseat.BranchAnswer, error) { + var a gitseat.BranchAnswer + return a, s.call(gitseat.VerbBranch, gitseat.BranchInput{Repository: repository, Branch: branch}, &a) +} + +// watchedRepositories are the repositories each watched branch is registered for, each once, sorted. +func watchedRepositories(regs []meshcatalog.Registration) map[string][]string { + seen := map[string]map[string]bool{} + for _, r := range regs { + if strings.TrimSpace(r.Origin) == "" { + continue + } + for _, b := range r.Branches { + if strings.TrimSpace(b) == "" { + continue + } + if seen[b] == nil { + seen[b] = map[string]bool{} + } + seen[b][r.Origin] = true + } + } + out := map[string][]string{} + for b, rs := range seen { + for r := range rs { + out[b] = append(out[b], r) + } + sort.Strings(out[b]) + } + return out +} + +// StartSources adds to the source key of every watched branch the registrations name each registered repository it lacks, and answers the +// branches it wrote. A registrations answer that cannot be read is an error (the event is offered again); a repository whose +// head cannot be read is said and left out. +func (h *Holder) StartSources() ([]string, error) { + started, _, err := h.startSources("") + return started, err +} + +// startSleep waits between the start's passes; a test replaces it. +var startSleep = time.Sleep + +// errRegistrationsUnread is a start pass's error when the registrations cannot be read. +var errRegistrationsUnread = errors.New("the registrations cannot be read") + +// startSourcesAtStart runs the start pass once (A-442), and asks again, as switchToRecords does (2 s doubling to a minute), +// only while the registrations cannot be read: the seat mesh-catalog may serve after the gate does. A repository a pass +// leaves out is said by the pass, and added at the next registration change or by gate_source_start (A-401). +func startSourcesAtStart(h *Holder) { + for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { + started, leftOut, err := h.startSources("") + if err == nil || !errors.Is(err, errRegistrationsUnread) { + h.logf("[mesh-delivery] the sources started from the forge at the start: %v; left out %v; %v", started, leftOut, err) + return + } + h.logf("[mesh-delivery] the registrations cannot be read yet (%v); asking again in %s", err, wait) + startSleep(wait) + } +} + +// startSources starts the branches no key holds, or only branch when it is named; it answers the branches written and the +// repositories left out as "@". +func (h *Holder) startSources(only string) (started, leftOut []string, err error) { + if h.Source == nil || h.Catalog == nil || h.Forge == nil { + return nil, nil, errors.New("this gate keeps no source, reads no registrations or reads no forge") + } + h.startMu.Lock() + defer h.startMu.Unlock() + regs, err := h.Catalog.Registrations() + if err != nil { + return nil, nil, fmt.Errorf("%w: %w", errRegistrationsUnread, err) + } + watched := watchedRepositories(regs.Registrations) + if only != "" { + if _, ok := watched[only]; !ok { + return nil, nil, fmt.Errorf("no registration names the branch %s", only) + } + } + names := make([]string, 0, len(watched)) + for b := range watched { + if only == "" || b == only { + names = append(names, b) + } + } + sort.Strings(names) + var errs []error + for _, branch := range names { + held, _, err := h.Source.Read(branch) + if err != nil { + errs = append(errs, err) + continue + } + heads := map[string]string{} + for _, repository := range watched[branch] { + if held != nil { + if _, has := held.Heads[repository]; has { + continue // a head a key holds moves by a take-in alone (A-34) + } + } + a, err := h.Forge.Branch(repository, branch) + if err == nil && a.Error != "" { + err = errors.New(a.Error) + } + if err == nil && a.Head == "" { + err = errors.New("the forge named no head") + } + if err != nil { + leftOut = append(leftOut, repository+"@"+branch) + h.logf("[mesh-delivery] source.%s starts without %s: its head cannot be read (%v); it is added at the next registration change or by gate_source_start", branch, repository, err) + continue + } + heads[repository] = a.Head + } + if len(heads) == 0 { + continue + } + wrote := false + rev, err := h.Source.Set(branch, func(prev *Source) (Source, bool) { + next := Source{Heads: map[string]string{}} + if prev != nil { + next.Group = prev.Group + for r, c := range prev.Heads { + next.Heads[r] = c + } + } + wrote = false + for r, c := range heads { + if _, has := next.Heads[r]; !has { // a held head is never changed + next.Heads[r] = c + wrote = true + } + } + return next, wrote + }) + if err != nil { + errs = append(errs, fmt.Errorf("source.%s cannot be started: %w", branch, err)) + continue + } + if !wrote { + continue + } + started = append(started, branch) + if h.SaySeat == nil { + continue + } + at := h.Now().UTC() + if err := h.SaySeat(seat.SubjectSourceMoved(keys.EscapeName(branch)), + seat.SourceMovedEvent{Branch: branch, Revision: int64(rev), At: &at}, + "source-moved."+keys.EscapeName(branch)+"."+strconv.FormatUint(rev, 10)); err != nil { + errs = append(errs, err) + } + } + return started, leftOut, errors.Join(errs...) +} + +func (h *Holder) logf(format string, a ...any) { + if h.Logf != nil { + h.Logf(format, a...) + } +} + +// SourceStart starts one watched branch by hand, after the hand-act log took it: the same as the start does. +func (h *Holder) SourceStart(branch, why, by string, acts handActs) (map[string]any, error) { + if strings.TrimSpace(branch) == "" || strings.TrimSpace(why) == "" { + return nil, errors.New("say the branch and why its source is started") + } + if h.Source == nil || h.Catalog == nil || h.Forge == nil { + return nil, errors.New("this gate keeps no source, reads no registrations or reads no forge") + } + if acts == nil { + return nil, errors.New("the hand-act log cannot be reached here, so nothing was started") + } + if _, err := acts.Record(meshhandactsclient.Request{What: "started the source of the branch " + branch, + Why: why, Cause: "gate_source_start", Verb: "source-start"}); err != nil { + return nil, fmt.Errorf("the hand act was not recorded through mesh-hand-acts, so nothing was started: %w", err) + } + started, leftOut, err := h.startSources(branch) + out := map[string]any{"branch": branch, "started": started, "left-out": leftOut, "by": by} + if err == nil && len(started) == 0 && len(leftOut) == 0 { + out["note"] = "the key names every registered repository already: only a take-in moves a head" + } + return out, err +} diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/sourcetools.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/sourcetools.go index 68c982da0..6d3f7586c 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/sourcetools.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/sourcetools.go @@ -79,5 +79,15 @@ func sourceTools(h *Holder, acts handActs) []stdio.Tool { Run: func(a map[string]any) (any, error) { return h.SourceAgain(strArg(a, "branch"), strArg(a, "why"), stdio.CallerOf(a), acts) }}, + {Name: "gate_source_start", + Description: "Start the source of one watched branch the registrations name and no source key holds yet, from the forge: each " + + "registered repository at the branch's head, the group empty, then source-moved; recorded in the hand-act log of the seat " + + "mesh-hand-acts first. A key the gate holds is never rewritten.", + Input: map[string]any{"type": "object", "properties": map[string]any{ + "branch": str("the watched branch"), "why": str("why it is started by hand, in plain words")}, + "required": []string{"branch", "why"}}, + Run: func(a map[string]any) (any, error) { + return h.SourceStart(strArg(a, "branch"), strArg(a, "why"), stdio.CallerOf(a), acts) + }}, } } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/sourcetools_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/sourcetools_test.go index 2094ff7b5..aa0b7063c 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/sourcetools_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/sourcetools_test.go @@ -1,8 +1,13 @@ package main import ( + "encoding/json" "errors" + "fmt" + meshcatalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" + stdio "git.novox.be/novox/mesh-sdk/go" + "reflect" "strings" "testing" "time" @@ -103,3 +108,204 @@ func TestTheSourceAnswerNamesTheBranchItWasAskedAndStatusSaysEveryBranch(t *test t.Fatalf("status says every branch's source: %+v", h.SourceAll()) } } + +type fakeCatalog struct { + regs []meshcatalog.Registration + err error +} + +func (f fakeCatalog) Registrations() (meshcatalog.RegistrationsAnswer, error) { + return meshcatalog.RegistrationsAnswer{Registrations: f.regs}, f.err +} + +// Design 47 G A-401: a watched branch starts from the forge as a clone does: each registered repository at the branch's head, +// the group empty, source-moved said; a repository whose head cannot be read is left out; a key held is never rewritten. +func TestAWatchedBranchStartsFromTheForge(t *testing.T) { + regs := []meshcatalog.Registration{ + {Module: "car-hunter", Origin: "o/car-hunter", Branches: []string{"main"}}, + {Module: "mesh-a", Origin: "o/mesh", Branches: []string{"main"}}, + {Module: "mesh-b", Origin: "o/mesh", Branches: []string{"main"}}, // the same repository twice: once + {Module: "f1", Origin: "o/f1-circuits", Branches: []string{"master"}}, + {Module: "l2p", Origin: "o/link2pay", Branches: []string{"nox-mesh"}}, + {Module: "ph", Origin: "o/photos", Branches: []string{"nox-mesh", "main"}}, + } + forge := &fakeForge{heads: map[string]string{ + "o/car-hunter@main": "c1", "o/mesh@main": "c2", "o/f1-circuits@master": "c3", "o/link2pay@nox-mesh": "c4", + "o/photos@main": "c6", // o/photos@nox-mesh cannot be read + }} + for _, c := range []struct { + name string + held map[string]Source + catalog fakeCatalog + only string + started []string + heads map[string]map[string]string + events int + wantErr bool + }{ + {name: "every branch no key holds", catalog: fakeCatalog{regs: regs}, + started: []string{"main", "master", "nox-mesh"}, events: 3, + heads: map[string]map[string]string{"main": {"o/car-hunter": "c1", "o/mesh": "c2", "o/photos": "c6"}, + "master": {"o/f1-circuits": "c3"}, "nox-mesh": {"o/link2pay": "c4"}}}, + {name: "a held key lacking repositories gets them, its heads unchanged", catalog: fakeCatalog{regs: regs}, + held: map[string]Source{"main": {Group: "g", Heads: map[string]string{"o/mesh": "old"}}}, + started: []string{"main", "master", "nox-mesh"}, events: 3, + heads: map[string]map[string]string{"main": {"o/mesh": "old", "o/car-hunter": "c1", "o/photos": "c6"}, "master": {"o/f1-circuits": "c3"}, "nox-mesh": {"o/link2pay": "c4"}}}, + {name: "a held key lacking one repository", catalog: fakeCatalog{regs: regs[:2]}, + held: map[string]Source{"main": {Group: "g", Heads: map[string]string{"o/mesh": "old"}}}, + started: []string{"main"}, events: 1, + heads: map[string]map[string]string{"main": {"o/mesh": "old", "o/car-hunter": "c1"}}}, + {name: "a held key naming all: nothing written or said", catalog: fakeCatalog{regs: regs[:2]}, + held: map[string]Source{"main": {Group: "g", Heads: map[string]string{"o/mesh": "old", "o/car-hunter": "older"}}}, + events: 0, + heads: map[string]map[string]string{"main": {"o/mesh": "old", "o/car-hunter": "older"}}}, + {name: "by hand, one branch", catalog: fakeCatalog{regs: regs}, only: "master", + started: []string{"master"}, events: 1, heads: map[string]map[string]string{"master": {"o/f1-circuits": "c3"}}}, + {name: "registrations unreadable", catalog: fakeCatalog{err: errors.New("away")}, wantErr: true}, + {name: "no registration names the branch", catalog: fakeCatalog{regs: regs}, only: "nope", wantErr: true}, + } { + t.Run(c.name, func(t *testing.T) { + src := &branchSource{memSource{vals: map[string]Source{}}} + for b, s := range c.held { + src.vals[b] = s + } + events := 0 + var logged []string + h := &Holder{Now: time.Now, Source: src, Forge: forge, Catalog: c.catalog, + Logf: func(f string, a ...any) { logged = append(logged, fmt.Sprintf(f, a...)) }, + SaySeat: func(subject string, body any, id string) error { + events++ + if ev := body.(seat.SourceMovedEvent); ev.Group != "" { + t.Errorf("a source-moved for started heads names no group (A-404), got %q", ev.Group) + } + return nil + }} + started, _, err := h.startSources(c.only) + if (err != nil) != c.wantErr { + t.Fatalf("err %v", err) + } + if c.wantErr { + return + } + if !reflect.DeepEqual(started, c.started) || events != c.events { + t.Fatalf("started %v (%d events), want %v (%d)", started, events, c.started, c.events) + } + for b, want := range c.heads { + if got := src.vals[b].Heads; !reflect.DeepEqual(got, want) { + t.Errorf("source.%s heads %v, want %v", b, got, want) + } + } + for b, h := range c.held { + if src.vals[b].Group != h.Group { + t.Errorf("source.%s keeps the group %q a take-in wrote, got %q", b, h.Group, src.vals[b].Group) + } + } + if c.only == "" && c.held == nil && !strings.Contains(strings.Join(logged, "\n"), "o/photos") { + t.Errorf("the repository left out was not said: %v", logged) + } + }) + } +} + +// Design 47 G A-401: gate_source_start is recorded in the hand-act log first, and starts nothing when the log refuses it. +func TestSourceStartByHandIsRecordedFirst(t *testing.T) { + src := &branchSource{memSource{vals: map[string]Source{}}} + h := &Holder{Now: time.Now, Source: src, Forge: &fakeForge{heads: map[string]string{"o/f1@master": "c3"}}, + Catalog: fakeCatalog{regs: []meshcatalog.Registration{{Module: "f1", Origin: "o/f1", Branches: []string{"master"}}}}} + acts := &handActsFake{refuse: errors.New("log down")} + if _, err := h.SourceStart("master", "newly watched", "me", acts); err == nil || len(src.vals) != 0 { + t.Fatalf("started without the log: %v %v", err, src.vals) + } + acts.refuse = nil + if _, err := h.SourceStart("master", "", "me", acts); err == nil { + t.Fatal("started with no reason") + } + if _, err := h.SourceStart("master", "newly watched", "me", acts); err != nil { + t.Fatal(err) + } + if len(acts.took) != 1 || acts.took[0].Cause != "gate_source_start" || src.vals["master"].Heads["o/f1"] != "c3" { + t.Fatalf("log %+v source %+v", acts.took, src.vals) + } +} + +// Design 47 G A-401: the registration-changed branch of take and the start pass both write the new branch's key and say +// source-moved; removing either leaves the branch unstarted. +func TestRegistrationChangedAndTheStartPassStartABranch(t *testing.T) { + regs := []meshcatalog.Registration{{Module: "f1", Origin: "o/f1", Branches: []string{"master"}}} + for _, c := range []struct { + name string + run func(h *Holder) error + }{ + {"registration-changed", func(h *Holder) error { + return take(h, stdio.Envelope{Key: registrationChangedEvent, Body: json.RawMessage(`{}`)}) + }}, + {"the start pass", func(h *Holder) error { startSourcesAtStart(h); return nil }}, + } { + t.Run(c.name, func(t *testing.T) { + if registrationChangedEvent == "" { + t.Fatal("the manifest does not consume registration-changed") + } + src := &branchSource{memSource{vals: map[string]Source{}}} + var subjects []string + h := &Holder{Now: time.Now, Source: src, Forge: &fakeForge{heads: map[string]string{"o/f1@master": "c3"}}, + Catalog: fakeCatalog{regs: regs}, Logf: func(string, ...any) {}, + SaySeat: func(subject string, body any, id string) error { subjects = append(subjects, subject); return nil }} + if err := c.run(h); err != nil { + t.Fatal(err) + } + if src.vals["master"].Heads["o/f1"] != "c3" || len(subjects) != 1 || !strings.HasSuffix(subjects[0], ".event.source-moved.master") { + t.Fatalf("source %+v events %v", src.vals, subjects) + } + }) + } +} + +// Design 47 G A-442: the start pass runs once; it asks again, doubling its wait, only while the registrations cannot be read, +// and a repository left out is said, not waited for. +func TestTheStartAsksAgainOnlyWhileTheRegistrationsCannotBeRead(t *testing.T) { + regs := []meshcatalog.Registration{{Module: "f1", Origin: "o/f1", Branches: []string{"master"}}} + for _, c := range []struct { + name string + unreadFor int // passes the registrations cannot be read + forgeFail bool // the forge cannot name the head + waits []time.Duration + started bool + }{ + {"registrations unread twice", 2, false, []time.Duration{2 * time.Second, 4 * time.Second}, true}, + {"a repository left out is not waited for", 0, true, nil, false}, + } { + t.Run(c.name, func(t *testing.T) { + var waits []time.Duration + old := startSleep + startSleep = func(d time.Duration) { waits = append(waits, d) } + defer func() { startSleep = old }() + src := &branchSource{memSource{vals: map[string]Source{}}} + forge := &fakeForge{fail: c.forgeFail, heads: map[string]string{"o/f1@master": "c3"}} + cat := &flakyCatalog{fakeCatalog: fakeCatalog{regs: regs}, failFor: c.unreadFor} + var said []string + h := &Holder{Now: time.Now, Source: src, Forge: forge, Catalog: cat, + Logf: func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }} + startSourcesAtStart(h) + if !reflect.DeepEqual(waits, c.waits) || (src.vals["master"].Heads["o/f1"] == "c3") != c.started { + t.Fatalf("waits %v source %+v", waits, src.vals) + } + if c.forgeFail && !strings.Contains(strings.Join(said, "\n"), "o/f1@master") { + t.Fatalf("the left-out repository is not said: %v", said) + } + }) + } +} + +// flakyCatalog cannot read the registrations for its first failFor asks. +type flakyCatalog struct { + fakeCatalog + failFor int +} + +func (f *flakyCatalog) Registrations() (meshcatalog.RegistrationsAnswer, error) { + if f.failFor > 0 { + f.failFor-- + return meshcatalog.RegistrationsAnswer{}, errors.New("not served yet") + } + return f.fakeCatalog.Registrations() +} diff --git a/modules/mesh-delivery-gate/module.json b/modules/mesh-delivery-gate/module.json index aad02e5cf..2d170922d 100644 --- a/modules/mesh-delivery-gate/module.json +++ b/modules/mesh-delivery-gate/module.json @@ -22,6 +22,7 @@ "git.pull-updated", "git.pull-merged", "git.pull-closed", + "mesh-catalog.registration-changed", "mesh-conditions.findings-asked" ], "emits": [ @@ -351,8 +352,10 @@ "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg": "0.1" }, "invokes": [ + "seat:git.branch", "seat:git.open-pulls", "seat:git.pull", + "seat:mesh-catalog.registrations", "seat:mesh-hand-acts.hand-act", "seat:mesh-wants.entry", "seat:node-engine.composition" @@ -379,7 +382,8 @@ "gate_record_repair", "gate_records", "gate_source", - "gate_source_again" + "gate_source_again", + "gate_source_start" ], "config": { "record-retention": { diff --git a/modules/messenger/cmd/messenger/answers_test.go b/modules/messenger/cmd/messenger/answers_test.go index 025eba1c4..d15449dfa 100644 --- a/modules/messenger/cmd/messenger/answers_test.go +++ b/modules/messenger/cmd/messenger/answers_test.go @@ -601,14 +601,14 @@ func TestAnAskIsRefusedInWordsToItsAsker(t *testing.T) { } } all := strings.Join(refused, "\n") - // A fourth offered ask is never refused (ADR 0666 §12): it is taken and the asker's oldest is held back. + // A fourth offered ask is never refused and none is held back (design 47 G A-424). for _, want := range []string{"d-1: the ask is refused: an ask that destroys lasts at most", "d-2: its words carry"} { if !strings.Contains(all, want) { t.Errorf("not refused: %q in\n%s", want, all) } } - if strings.Contains(all, "d-6") || store.asks["mesh-delivery.d-6"].State != StateOpen || store.asks["mesh-delivery.d-3"].State != StateHeldBack { - t.Errorf("the fourth ask was refused or the oldest not held back: d-6 %s, d-3 %s", store.asks["mesh-delivery.d-6"].State, store.asks["mesh-delivery.d-3"].State) + if strings.Contains(all, "d-6") || store.asks["mesh-delivery.d-6"].State != StateOpen || store.asks["mesh-delivery.d-3"].State != StateOpen { + t.Errorf("the fourth ask was refused or the oldest held back: d-6 %s, d-3 %s", store.asks["mesh-delivery.d-6"].State, store.asks["mesh-delivery.d-3"].State) } // A refused ask is kept as a closed record with its outcome and words (design 47 G §570), and `asks` lists it refused. if rec, kept := store.asks["mesh-delivery.d-1"]; !kept || rec.State != string(occ.OutcomeRefused) || rec.Warrant == nil || diff --git a/modules/messenger/cmd/messenger/asks.go b/modules/messenger/cmd/messenger/asks.go index 74a94cbf1..729ee7af9 100644 --- a/modules/messenger/cmd/messenger/asks.go +++ b/modules/messenger/cmd/messenger/asks.go @@ -563,9 +563,9 @@ func (h *Holder) TakeAsk(verb, asker string, body json.RawMessage) error { h.refuseAsk(asker, a, "its whole words carry "+refusal.String()+", which may not leave the mesh on any channel: withhold it") return nil } - // A fourth offered ask is taken, never refused: the asker's oldest offered one is held back once this one is recorded - // (holdBackOldest). An ask with the same `about` replaces the open one, a held-back one too. - open, replaced := 0, "" + // Every ask is offered when it is raised: the channel keeps no count of an asker's open asks and holds none back for room + // (design 47 G A-424). An ask with the same `about` replaces the open one, a held-back one of before too. + replaced := "" h.mu.Lock() for k, e := range h.asks { if e.rec.Asker != asker || !liveState(e.rec.State) { @@ -573,14 +573,9 @@ func (h *Holder) TakeAsk(verb, asker string, body json.RawMessage) error { } if a.About != "" && e.rec.Ask.About == a.About { replaced = k - continue - } - if e.rec.State != StateHeldBack { - open++ } } h.mu.Unlock() - holdBack := open >= occ.MostOpen rec := AskRecord{Asker: asker, Ask: a, State: StateOpen, Opened: now, Tickets: h.ticketsFor(a)} if len(rec.Tickets) == 0 { // Held, not refused (ADR 0397): the ask breaks no rule, and the operator is asked when a channel can carry it. @@ -611,9 +606,6 @@ func (h *Holder) TakeAsk(verb, asker string, body json.RawMessage) error { } else { h.keepAsk(key, rec, 0) } - if holdBack { - h.holdBackOldest(asker, key) - } if rec.State == StateHeld { h.Logf("[messenger] %s asks %q (%s): held, not shown: %s", asker, a.Headline, a.ID, rec.HeldWhy) return nil diff --git a/modules/messenger/cmd/messenger/held_test.go b/modules/messenger/cmd/messenger/held_test.go index b04abc95f..d666310e6 100644 --- a/modules/messenger/cmd/messenger/held_test.go +++ b/modules/messenger/cmd/messenger/held_test.go @@ -105,7 +105,8 @@ func TestAHeldAskExpiresAsAnyOpenAskDoes(t *testing.T) { } } -func TestAHeldAskCountsTowardTheOffersOfItsAskerAndTheOldestIsHeldBack(t *testing.T) { +// Design 47 G A-424: four held asks of one asker are taken and held for want of a channel, and none is held back for room. +func TestFourHeldAsksOfOneAskerAreHeldAndNoneHeldBack(t *testing.T) { h, _, _, c, store, w := newRouter(t) withoutVerifiedSender(h) for i := 1; i <= 4; i++ { @@ -114,8 +115,8 @@ func TestAHeldAskCountsTowardTheOffersOfItsAskerAndTheOldestIsHeldBack(t *testin submit(t, h, "mesh-delivery", a) c.pass(time.Second) } - if got := w.of("mesh-delivery"); len(got) != 0 || store.asks["mesh-delivery.d-1"].State != StateHeldBack || store.asks["mesh-delivery.d-4"].State != StateHeld { - t.Fatalf("%+v: a fourth is taken, never refused; the oldest is held back (%s)", got, store.asks["mesh-delivery.d-1"].State) + if got := w.of("mesh-delivery"); len(got) != 0 || store.asks["mesh-delivery.d-1"].State != StateHeld || store.asks["mesh-delivery.d-4"].State != StateHeld { + t.Fatalf("%+v: a fourth is taken, never refused, and none is held back (d-1 %s)", got, store.asks["mesh-delivery.d-1"].State) } } diff --git a/modules/messenger/cmd/messenger/holdback.go b/modules/messenger/cmd/messenger/holdback.go index b9aa3a018..d08d22b6c 100644 --- a/modules/messenger/cmd/messenger/holdback.go +++ b/modules/messenger/cmd/messenger/holdback.go @@ -5,14 +5,12 @@ import ( oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel" "sort" "time" - - occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient" ) -// An ask held back (novox/hq ADR 0666 §12, design 46 § 6): an asker has at most MostOpen asks offered at once; a fourth -// is offered and nothing is refused, so no ask is hidden: the asker's oldest offered ask is held back, its tickets -// withdrawn and its message edited to say it waits behind newer asks, and it is offered again, with new tickets, when -// one of the asker's offered asks ends. `held` is another state, an ask no ready channel can carry (ADR 0397). +// An ask held back (novox/hq ADR 0666 §12, design 46 § 6) was one kept open behind newer asks of its asker, offered again when +// one of them ended. The channel holds none back any more (design 47 G A-424): every ask is offered when it is raised, and one +// held back before the change is offered at the router's start (offerAllHeldBack) and by ask-again. `held` is another state, an +// ask no ready channel can carry (ADR 0397). // StateHeldBack is an ask kept open behind newer asks of its asker. const StateHeldBack = "held-back" @@ -90,50 +88,35 @@ func (h *Holder) freshAsk(key string) (AskRecord, uint64, bool) { return rec, rev, true } -// holdBackOldest holds back the asker's oldest offered ask other than except (the newest, just taken): its tickets -// are withdrawn, it stays open, and its own message is edited to say it waits. One held for want of a channel counts as offered -// and is held back the same way. The work lock is held. -func (h *Holder) holdBackOldest(asker, except string) { - h.mu.Lock() - key := h.pickLocked(asker, except, StateOpen, StateHeld) - h.mu.Unlock() - if key == "" { - return +// offerHeldBack offers the asker's held-back asks of before the change again (A-424): there is no room to wait for. The work +// lock is held. +func (h *Holder) offerHeldBack(asker string) { + for { + h.mu.Lock() + key := h.pickLocked(asker, "", StateHeldBack) + h.mu.Unlock() + if key == "" { + return + } + h.offerOneHeldBack(key) + if after := h.askOf(key); after == nil || after.rec.State == StateHeldBack { + return // not offered (its record changed meanwhile): left for the next start rather than tried forever + } } - rec, rev, ok := h.freshAsk(key) - if !ok || (rec.State != StateOpen && rec.State != StateHeld) { - return - } - now := h.Now() - rec.State, rec.HeldBackSince, rec.Tickets, rec.HeldSince, rec.HeldWhy = StateHeldBack, now, nil, time.Time{}, "" - if !h.saveHeld(key, rec, rev) { - return - } - h.Logf("[messenger] %s's ask %q (%s) is held back behind its newer asks: it is offered again when one ends", asker, rec.Ask.Headline, rec.Ask.ID) - h.mu.Lock() - r := h.open[rec.Record] - h.mu.Unlock() - if r == nil || r.Origin != "ask" { - return // a condition's message is the newer ask's to show - } - h.mu.Lock() - r.Answered = "Waits behind newer questions of " + asker + ".\nIt is offered again, with its answers, when one of them ends." - h.mu.Unlock() - h.edit(r, "answered", false) - h.persist(r) } -// offerHeldBack offers the asker's oldest held-back ask again when a slot is free: new tickets, its message edited back -// to the question. The work lock is held. -func (h *Holder) offerHeldBack(asker string) { - if h.offeredCount(asker) >= occ.MostOpen { - return - } +// offerAllHeldBack offers every ask held back before the change, at each tick of the router (A-424). The work lock is held. +func (h *Holder) offerAllHeldBack() { h.mu.Lock() - key := h.pickLocked(asker, "", StateHeldBack) + askers := map[string]bool{} + for _, e := range h.asks { + if e.rec.State == StateHeldBack { + askers[e.rec.Asker] = true + } + } h.mu.Unlock() - if key != "" { - h.offerOneHeldBack(key) + for asker := range askers { + h.offerHeldBack(asker) } } @@ -172,9 +155,9 @@ func (h *Holder) offerOneHeldBack(key string) { h.showAsk(key, rec, now) } -// AskAgain is the seat operator-channel's verb ask-again (G §394, §450): offer a held-back ask now, holding back the -// asker's oldest offered ask in its place when the asker already has the most offered. It answers the ask's state after it, -// or the error that says why it was not offered. +// AskAgain is the seat operator-channel's verb ask-again (G §394, §450): it offers an ask held back before the channel stopped +// holding any back (A-424), and holds back nothing in its place. It answers the ask's state after it, or the error that says why +// it was not offered. func (h *Holder) AskAgain(key string) oc.AskAgainAnswer { h.work.Lock() defer h.work.Unlock() @@ -186,10 +169,7 @@ func (h *Holder) AskAgain(key string) oc.AskAgainAnswer { case !ok: return oc.AskAgainAnswer{Error: fmt.Sprintf("this router holds no ask %q: its key is ., as operator-channel.asks lists", key)} case rec.State != StateHeldBack: - return oc.AskAgainAnswer{Error: fmt.Sprintf("the ask %s is %s, not held back: only a held-back ask is offered again", key, rec.State)} - } - if h.offeredCount(rec.Asker) >= occ.MostOpen { - h.holdBackOldest(rec.Asker, key) + return oc.AskAgainAnswer{Error: fmt.Sprintf("the ask %s is %s, not held back: only a held-back ask is offered again, and the channel holds none back any more", key, rec.State)} } h.offerOneHeldBack(key) got, _, _ := h.freshAsk(key) diff --git a/modules/messenger/cmd/messenger/holdback_test.go b/modules/messenger/cmd/messenger/holdback_test.go index 5b92f1d2f..9e0d8d4e1 100644 --- a/modules/messenger/cmd/messenger/holdback_test.go +++ b/modules/messenger/cmd/messenger/holdback_test.go @@ -9,100 +9,50 @@ import ( occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient" ) -// ADR 0666 §12, design 46 §6 and test 71: an asker has at most three asks offered at once; a fourth is offered and the -// oldest is held back (its tickets withdrawn, its message edited, its record open), offered again with new tickets -// when one of the three ends, offered now by messenger_ask_again; an ask with the same `about` replaces a held-back -// one, and a Decline ends an ask whose asker may ask again at once. -func TestNoAskIsHiddenTheOldestOfFourIsHeldBackAndOfferedAgain(t *testing.T) { - h, tg, _, c, store, w := newRouter(t) +// Design 47 G A-424: the channel keeps no count of an asker's open asks and holds none back: every ask is offered when it is +// raised, five of one asker at once too, and none is refused. An ask held back before the change is offered again at the +// router's next tick, with new tickets; ask-again offers one now and holds nothing back in its place. +func TestEveryAskIsOfferedAndOneHeldBackBeforeIsOfferedAgain(t *testing.T) { + h, _, _, c, store, w := newRouter(t) link(h, store, c, "42", c.now().Add(-2*time.Hour)) - ask := func(id, about string) { + state := func(id string) string { return store.asks["mesh-delivery."+id].State } + for i, id := range []string{"d-1", "d-2", "d-3", "d-4", "d-5"} { a := ackAsk(c, id) - a.About = about + a.About = "about-" + string(rune('1'+i)) submit(t, h, "mesh-delivery", a) c.pass(time.Second) settle(h, c) } - state := func(id string) string { return store.asks["mesh-delivery."+id].State } - for i, id := range []string{"d-1", "d-2", "d-3"} { - ask(id, "about-"+string(rune('1'+i))) - } - oldTicket := "" - for tk := range store.asks["mesh-delivery.d-1"].Tickets { - oldTicket = tk - } - ask("d-4", "about-4") - // M1: the fourth reaches the channel, the oldest is held back, none is refused. - if len(w.of("mesh-delivery")) != 0 || state("d-4") != StateOpen || state("d-1") != StateHeldBack { - t.Fatalf("refused %+v; d-4 %s, d-1 %s", w.of("mesh-delivery"), state("d-4"), state("d-1")) - } - if rec := store.asks["mesh-delivery.d-1"]; len(rec.Tickets) != 0 || rec.HeldBackSince.IsZero() { - t.Errorf("held back %+v: tickets withdrawn, since set", rec) - } - if m, ok := tg.edits["1"]; !ok || len(m.Actions) != 0 || !strings.Contains(m.Note, "Waits behind newer questions of mesh-delivery") { - t.Errorf("its message was not edited to say it waits: %v %+v", ok, m) - } - if _, err := h.Chosen("telegram", choiceOf(oldTicket, "42", c.now())); err == nil || !strings.Contains(err.Error(), "waits behind newer questions") || len(w.of("mesh-delivery")) != 0 { - t.Errorf("a tap of a withdrawn ticket: %v %+v", err, w.of("mesh-delivery")) - } - // M3: asks lists it with since when and the newer asks it waits behind; open includes it. - q, _ := AsksQueryOf(map[string]any{"key": "mesh-delivery.d-1"}) - raw, _ := json.Marshal(q.Narrow(h.AsksNow())) - for _, want := range []string{`"state":"held-back"`, `"held-since":"`, `"behind":["mesh-delivery.d-2","mesh-delivery.d-3","mesh-delivery.d-4"]`} { - if !strings.Contains(string(raw), want) { - t.Errorf("asks lacks %s: %s", want, raw) + for _, id := range []string{"d-1", "d-2", "d-3", "d-4", "d-5"} { + if state(id) != StateOpen || len(store.asks["mesh-delivery."+id].Tickets) == 0 { + t.Fatalf("%s is %s with %d tickets: every ask is offered", id, state(id), len(store.asks["mesh-delivery."+id].Tickets)) } } - if q, _ = AsksQueryOf(nil); !strings.Contains(string(mustJSON(q.Narrow(h.AsksNow()))), "mesh-delivery.d-1") { - t.Error("the open asks leave out the held-back one") - } - // M5: an ask with the same about replaces a held-back one; the three offered stay three, so the oldest of them is held back. - ask("d-5", "about-1") - if state("d-1") != string(occ.OutcomeReplaced) || state("d-2") != StateHeldBack || state("d-5") != StateOpen || state("d-3") != StateOpen { - t.Fatalf("after d-5 about the held-back ask's thing: d-1 %s d-2 %s d-3 %s d-5 %s", state("d-1"), state("d-2"), state("d-3"), state("d-5")) - } - // M2: when one of the offered ends, the oldest held back is offered again with new tickets and its message asks again. - if a := h.AskAgain("mesh-delivery.d-3"); !strings.Contains(a.Error, "not held back") { - t.Errorf("ask-again of an offered ask: %+v", a) - } - if err := h.TakeAsk("cancel", "mesh-delivery", []byte(`{"id":"d-3"}`)); err != nil { + // One held back before the change, as the store kept it. + key := "mesh-delivery.d-2" + got, rev, _ := store.GetAsk(key) + got.State, got.Tickets, got.HeldBackSince = StateHeldBack, nil, c.now() + rev, err := store.UpdateAsk(key, *got, rev) + if err != nil { t.Fatal(err) } - settle(h, c) - if rec := store.asks["mesh-delivery.d-2"]; rec.State != StateOpen || len(rec.Tickets) == 0 || !rec.HeldBackSince.IsZero() { - t.Fatalf("d-2 after d-3 ended: %+v", rec) - } - if m := tg.edits["2"]; len(m.Actions) == 0 || strings.Contains(m.Note, "Waits behind") { - t.Errorf("its message does not ask again: %+v", m) - } - // M4: ask-again by hand offers a held-back ask now, holding back the oldest offered one; the seat operator-channel's ask-again (G §450). - ask("d-6", "about-6") - ask("d-7", "about-7") - held := "" - for _, id := range []string{"d-2", "d-4", "d-5", "d-6", "d-7"} { - if state(id) == StateHeldBack { - held = id - } - } - if held == "" { - t.Fatal("no ask is held back after two more") + h.keepAsk(key, *got, rev) + if a := h.AskAgain("mesh-delivery.d-1"); !strings.Contains(a.Error, "not held back") { + t.Errorf("ask-again of an offered ask: %+v", a) } if a := h.AskAgain("mesh-delivery.d-none"); a.Error == "" || a.State != "" { t.Errorf("ask-again of an ask the router holds none of: %+v", a) } - if a := h.AskAgain("mesh-delivery." + held); a.Error != "" || a.State != string(StateOpen) || state(held) != StateOpen { - t.Fatalf("%s is %s; ask-again answered %+v", held, state(held), a) + h.Tick() + settle(h, c) + if rec := store.asks[key]; rec.State != StateOpen || len(rec.Tickets) == 0 || !rec.HeldBackSince.IsZero() { + t.Fatalf("the ask held back before the change, after a tick: %+v", rec) } - n := 0 - for _, id := range []string{"d-2", "d-4", "d-5", "d-6", "d-7"} { - if state(id) == StateHeldBack { - n++ + for _, id := range []string{"d-1", "d-3", "d-4", "d-5"} { + if state(id) != StateOpen { + t.Errorf("%s is %s after the held-back one was offered: nothing is held back in its place", id, state(id)) } } - if n != 2 || state("d-5") != StateHeldBack { - t.Errorf("%d asks held back after ask-again (d-5 %s): the oldest offered one took its place", n, state("d-5")) - } - // A Decline ends an ask; its asker may ask again at once, and nothing is refused. for _, wt := range w.of("mesh-delivery") { if wt.Outcome == occ.OutcomeRefused { t.Errorf("refused: %+v", wt) diff --git a/modules/messenger/cmd/messenger/holder.go b/modules/messenger/cmd/messenger/holder.go index d87b60e68..09debb721 100644 --- a/modules/messenger/cmd/messenger/holder.go +++ b/modules/messenger/cmd/messenger/holder.go @@ -778,6 +778,7 @@ func (h *Holder) Tick() { h.mu.Unlock() sort.Slice(recs, func(i, j int) bool { return recs[i].Raised.Before(recs[j].Raised) }) h.sweepAsks(now) + h.offerAllHeldBack() // held back before the channel stopped holding any back, offered at the first tick (design 47 G A-424) h.releaseHeld() h.offerUnsent() h.breakGlass() diff --git a/modules/nats/internal/keeper/keeper.go b/modules/nats/internal/keeper/keeper.go index 3b1af6d94..68788bed4 100644 --- a/modules/nats/internal/keeper/keeper.go +++ b/modules/nats/internal/keeper/keeper.go @@ -456,7 +456,8 @@ func (k *Keeper) statusWritten(s Status, err error) { // watched branch's newest revision is applied (Heard), a module of the new form that sends gets its sent store, a drift // between branches is said once, and a module in no watched branch's newest revision has its plans key deleted, its // entries then orphans. Before every watched branch has its first view it plans nothing, and says what it waits for in -// its status; the first plan checks what is held. +// its status, and an answer listing no module before one has listed any is the same (none-yet); the first plan checks what +// is held. func (k *Keeper) PlanCatalog() error { at := k.now() view, err := k.Catalog() @@ -466,6 +467,15 @@ func (k *Keeper) PlanCatalog() error { } k.mu.Lock() first := !k.viewed + // A catalog half with no module in it before any has been seen is a trunk with no revision yet (`none-yet`, design 47 + // G A-367): not yet viewed, so nothing is planned, no plan deleted and no entry made an orphan for want of knowledge. + if first && len(view.Modules) == 0 { + k.mu.Unlock() + k.setStatus(func(s *Status) { + s.FirstView.Asked, s.FirstView.Error = at, "the catalog half lists no module yet (none-yet): nothing is planned or orphaned until a revision lists one" + }) + return nil + } k.viewed = true k.mu.Unlock() var revision uint64 diff --git a/modules/nats/internal/keeper/keeper_test.go b/modules/nats/internal/keeper/keeper_test.go index b7c2aa305..500eb3d04 100644 --- a/modules/nats/internal/keeper/keeper_test.go +++ b/modules/nats/internal/keeper/keeper_test.go @@ -450,6 +450,9 @@ func TestTheKeeperMakesWhatAVerdictAsksForAndSaysIt(t *testing.T) { if len(bus.created) != 0 { t.Fatalf("created %v before the first view", bus.created) } + k.Catalog = func() (CatalogView, error) { + return CatalogView{Modules: []CatalogModule{{Accepted: Accepted{Module: "m", Digest: "d1", Revision: 3, BusState: []catalogue.BusStateEntry{drafts}}}}}, nil + } if err := k.PlanCatalog(); err != nil { t.Fatal(err) } @@ -541,6 +544,37 @@ func TestTheKeeperMakesWhatAVerdictAsksForAndSaysIt(t *testing.T) { t.Errorf("old sections %d: 8 MiB counted", k.Status().OldBytes) } }}, + // A-367: a trunk with no revision yet (none-yet) is not yet viewed: nothing is planned, deleted or orphaned. + {"a none-yet catalog half plans nothing and orphans nothing until a module is listed", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) { + k.Heard(Accepted{Module: "m", Revision: 9, BusState: []catalogue.BusStateEntry{drafts}}) + k.Catalog = func() (CatalogView, error) { + return CatalogView{Modules: []CatalogModule{{Accepted: Accepted{Module: "m", Revision: 9, BusState: []catalogue.BusStateEntry{drafts}}}}}, nil + } + _ = k.PlanCatalog() + k.mu.Lock() + k.viewed = false + k.mu.Unlock() + k.Catalog = func() (CatalogView, error) { return CatalogView{}, nil } + if err := k.PlanCatalog(); err != nil { + t.Fatal(err) + } + if _, ok := k.planned("m"); !ok { + t.Error("m's plan is deleted by a none-yet view") + } + if a := k.Held()["m"]; a.Made["drafts"].Orphaned { + t.Errorf("held %+v: drafts orphaned by a none-yet view", a) + } + if k.Viewed() || !strings.Contains(k.Status().FirstView.Error, "none-yet") { + t.Errorf("viewed %v, status %+v: none-yet is not a first view and says so", k.Viewed(), k.Status().FirstView) + } + k.Catalog = func() (CatalogView, error) { + return CatalogView{Modules: []CatalogModule{{Accepted: Accepted{Module: "m", Revision: 10, BusState: []catalogue.BusStateEntry{drafts}}}}}, nil + } + _ = k.PlanCatalog() + if !k.Viewed() || k.Status().FirstView.Error != "" { + t.Error("the first revision with entries is viewed and plans as before") + } + }}, // §538: a module in no watched branch's newest revision has its plans key deleted, its entries then orphans. {"a module no watched branch lists loses its plan, its entries orphans", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) { k.Catalog = func() (CatalogView, error) { @@ -1240,9 +1274,11 @@ func TestTheKeeperMakesWhatAVerdictAsksForAndSaysIt(t *testing.T) { } { t.Run(row.name, func(t *testing.T) { k, bus, _, out := newKeeper(CatalogView{}) - if err := k.PlanCatalog(); err != nil { - t.Fatal(err) - } + // The rows start after the first view: an empty answer is none-yet (A-367), so the view is marked answered here. + k.mu.Lock() + k.viewed = true + k.mu.Unlock() + k.setStatus(func(s *Status) { s.FirstView.Asked, s.FirstView.Answered = k.now(), k.now() }) row.run(t, k, bus, out) }) }