diff --git a/modules/mesh-delivery-indexer/internal/index/candidates.go b/modules/mesh-delivery-indexer/internal/index/candidates.go index defb93bc..a600e0f4 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidates.go +++ b/modules/mesh-delivery-indexer/internal/index/candidates.go @@ -166,6 +166,7 @@ type Indexer struct { mu sync.Mutex // guards source, toolchains and previews, which the watches and the forge messages all reach source sourceNow + saidSource map[string]uint64 // the newest source revision a take-in.taken named, by watched branch (nextSource) previews map[string]meshdeliverychecker.PreviewsByRepositoryHeadTrunkValue // by previews key (previews.go) toolchains map[string]map[string]nodebuildgo.Toolchain // language, then node (toolchains.go) buildModules map[string]BuildModule // by language: the build modules whose toolchains are taken in (buildmodules.go) diff --git a/modules/mesh-delivery-indexer/internal/index/pending.go b/modules/mesh-delivery-indexer/internal/index/pending.go index c2f9af9c..f3de90d9 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending.go +++ b/modules/mesh-delivery-indexer/internal/index/pending.go @@ -10,6 +10,7 @@ import ( "strings" "time" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" @@ -1167,14 +1168,71 @@ func (ix *Indexer) takeInWholeBy(ctx context.Context, entries []PendingEntry, ow } } ix.forget(claimed...) - ix.logf("take-in: %d merges taken in as one revision by %s%s, oldest %s", len(claimed), own, byWords(by), claimed[0].Key) + branch, src, took := ix.takenWhat(ctx, claimed, applied) + ix.logf("take-in: %d merges taken in as one revision by %s%s, oldest %s: source revision %d of %s, %d entries", len(claimed), own, byWords(by), claimed[0].Key, src, branch, len(took)) for _, t := range takenWaits(claimed) { t.By, t.Why = who, why + t.Branch, t.Source, t.Entries = branch, src, took ix.sayTakeIn(ctx, t) } return applied, nil } +// takenWhat is what one take-in took in (design 47 G §476, §493, §648): the watched branch its merges landed on (the +// trunk for a merge that names none, as before G), the source revision it makes of that branch (nextSource) and one entry +// per module it wrote, with the commit, fingerprint and repository the catalog entry holds now and the pull request of +// that repository's merge. A module with no fingerprint (no build) is no entry; one the view cannot read is said. +func (ix *Indexer) takenWhat(ctx context.Context, claimed []PendingEntry, applied Applied) (string, int64, []meshcatalog.TakeInEntry) { + branch := TrunkBranch + pulls := map[string]int64{} + for _, e := range claimed { + if e.Forge.Base != "" { + branch = e.Forge.Base + } + if e.Forge.Number != 0 { + pulls[strings.ToLower(e.Forge.Repository)] = int64(e.Forge.Number) + } + } + src := ix.nextSource(branch) + written := writtenModules(applied.Written) + if len(written) == 0 { + return branch, src, nil + } + view, err := ix.viewEntries(ctx) + if err != nil { + ix.logf("take-in: the entries of source revision %d of %s cannot be read, so its take-in.taken names none and starts no pipeline: %v", src, branch, err) + return branch, src, nil + } + var out []meshcatalog.TakeInEntry + for _, m := range slices.Sorted(slices.Values(written)) { + e, ok := view[m] + if !ok || e.Fp == "" { + ix.logf("take-in: %s is written in source revision %d of %s with no fingerprint (%s): no pipeline entry", m, src, branch, e.None) + continue + } + out = append(out, meshcatalog.TakeInEntry{Module: m, Commit: e.Commit, Fingerprint: e.Fp, Repository: e.Repository, + Pull: pulls[strings.ToLower(e.Repository)]}) + } + return branch, src, out +} + +// nextSource is the source revision a take-in of branch makes: one above the newest the indexer has seen for it (the gate's +// source-moved, the trunk's alone held here) or named in a take-in.taken before, which the gate's source. then moves +// to (A-34; the lead's answer A-431, principle 7). +func (ix *Indexer) nextSource(branch string) int64 { + ix.mu.Lock() + defer ix.mu.Unlock() + n := ix.saidSource[branch] + if isTrunkBranch(branch) && ix.source.rev > n { + n = ix.source.rev + } + if ix.saidSource == nil { + ix.saidSource = map[string]uint64{} + } + ix.saidSource[branch] = n + 1 + return int64(n + 1) +} + // takenWaits are the taken events of a take-in: every wait the entries were said under (saidWaits), every wait // indexer_pending names the claimed entries by now (WaitsOf: a revision by its oldest merge's key, a group by its g. key) and // the key of every claimed entry, since indexer_pending named it by that key earlier when it waited behind a group. So a wait diff --git a/modules/mesh-delivery-indexer/internal/index/pending_test.go b/modules/mesh-delivery-indexer/internal/index/pending_test.go index 43f3cbfb..b23be27e 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending_test.go +++ b/modules/mesh-delivery-indexer/internal/index/pending_test.go @@ -1007,6 +1007,16 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) if len(b.keys()) != 0 || b.commit(t, "avahi") != sha('d') || len(b.said) != 1 || b.said[0].Kind != takein.TakeInTaken || b.said[0].By != "lead@shanks" { t.Errorf("hand take-in: pending %v, avahi at %s, said %+v", b.keys(), b.commit(t, "avahi"), b.said) } + // §476, §493: a hand take-in's taken event carries the branch, the source revision it makes and its entries. + if len(b.said) == 1 { + got := b.said[0] + if got.Branch != TrunkBranch || got.Source <= 0 || len(got.Entries) != 1 || got.Entries[0].Module != "avahi" || + got.Entries[0].Commit != sha('d') || got.Entries[0].Fingerprint == "" { + t.Errorf("hand take-in said branch %q source %d entries %+v: want main, a source revision and avahi at its commit", got.Branch, got.Source, got.Entries) + } else if _, err := takein.EncodeTakeIn(got); err != nil { + t.Errorf("the taken event is refused by its own contract: %v", err) + } + } b.arrive(t, 4, b.merge(23, sha('e'), "tree-avahi-5", "tree-indexer-2")) if r := b.ix.DropPending(ctx, takein.PendingKey(4), "lead@shanks", "abandoned on the operator's word", true); !r.Stored || len(b.keys()) != 0 { t.Errorf("an abandoned drop: %+v, pending %v", r, b.keys()) diff --git a/modules/mesh-delivery-indexer/takein/takein.go b/modules/mesh-delivery-indexer/takein/takein.go index 056b521f..7b559d64 100644 --- a/modules/mesh-delivery-indexer/takein/takein.go +++ b/modules/mesh-delivery-indexer/takein/takein.go @@ -12,6 +12,7 @@ import ( "time" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-sdk/go/keys" ) @@ -147,6 +148,12 @@ type TakeIn struct { // Unknown is why the indexer holds a merge it does not take in although no indexer build is named: it cannot learn the build // it runs itself (ADR 0633 §1), so Since is when it began not to know. A pending revision names WaitsFor or Unknown. Unknown string `json:"own-build-unknown,omitempty"` + // Branch, Source and Entries are what a take-in took in (design 47 G §476, §493, §648): the watched branch, the source + // revision it makes of that branch, and one entry per module it wrote. Every taken event of one take-in carries the same; + // a taken event that ends a wait without a take-in (a group's end, a drop) carries none and starts no pipeline. + Branch string `json:"branch,omitempty"` + Source int64 `json:"source,omitempty"` + Entries []meshcatalog.TakeInEntry `json:"entries,omitempty"` } func (t TakeIn) check() error { @@ -194,6 +201,12 @@ func (t TakeIn) check() error { if t.Kind != TakeInTaken && (t.By != "" || t.Why != "" || t.Dropped) { return fmt.Errorf("delivery: only a taken event names the hand act that ended a wait") } + if t.Kind != TakeInTaken && (t.Source != 0 || len(t.Entries) > 0) { + return fmt.Errorf("delivery: only a taken event names the source revision and the entries taken in") + } + if len(t.Entries) > 0 && (t.Branch == "" || t.Source <= 0) { + return fmt.Errorf("delivery: the taken event %s names entries without its watched branch and source revision", t.Key) + } if (t.Why != "" || t.Dropped) && t.By == "" { return fmt.Errorf("delivery: a hand act's take-in event names who acted (by)") } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start.go b/modules/mesh-delivery-pipeline/internal/pipeline/start.go index 73ebf5a7..cde61cce 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/start.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start.go @@ -436,6 +436,11 @@ func (s *Starter) TakenIn(ev catalog.TakeInTaken, view func(branch string) (Half s.logf("the take-in %s was dropped by %s: it starts no pipeline", ev.Key, ev.By) return 0, nil } + if len(ev.Entries) == 0 { + // A taken event that took nothing in (a group's end, one said before G and read again) starts nothing (§493). + s.logf("the take-in %s names no entries: it starts no pipeline", ev.Key) + return 0, nil + } if ev.Branch == "" || ev.Source <= 0 { return 0, fmt.Errorf("a take-in.taken names its watched branch and source revision: key %q branch %q source %d", ev.Key, ev.Branch, ev.Source) } diff --git a/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go b/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go index 727e88f7..ac0d68b3 100644 --- a/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go +++ b/modules/mesh-delivery-pipeline/internal/pipeline/start_test.go @@ -118,9 +118,13 @@ func TestTakenInStarts(t *testing.T) { if n, err := s.TakenIn(catalog.TakeInTaken{Kind: "taken", Dropped: true, By: "op", Branch: "main", Source: 8, Entries: ev.Entries}, view); n != 0 || err != nil { t.Errorf("a hand drop started %d (%v)", n, err) } - if _, err := s.TakenIn(catalog.TakeInTaken{Kind: "taken", Branch: "main"}, view); err == nil { + if _, err := s.TakenIn(catalog.TakeInTaken{Kind: "taken", Branch: "main", Entries: ev.Entries}, view); err == nil { t.Error("a take-in naming no source revision is refused") } + // §493: a taken event with no entries (a group's end, one said before G) starts nothing and is not heard again. + if n, err := s.TakenIn(catalog.TakeInTaken{Kind: "taken", Key: "m.9"}, view); n != 0 || err != nil { + t.Errorf("a take-in with no entries started %d (%v)", n, err) + } } // A-327 (design 47 G §21, §23, §54): a pipeline started before its module has a registration starts with no step; the diff --git a/modules/mesh-delivery-sdk/graph/graph.go b/modules/mesh-delivery-sdk/graph/graph.go index 0b9e78a4..653904f6 100644 --- a/modules/mesh-delivery-sdk/graph/graph.go +++ b/modules/mesh-delivery-sdk/graph/graph.go @@ -171,9 +171,11 @@ func (g Node) check() error { } } } + // A left-out word is an invoked verb or a seat call word of a package the module builds against (A-361), which its + // own invokes need not name; so it is a verb word, invoked or not. for _, l := range g.LeftOut { - if !slices.Contains(g.Invokes, l) { - return fmt.Errorf("graph: the graph node leaves out %q, which it does not invoke", l) + if !isVerb(l) { + return fmt.Errorf("graph: the graph node leaves out %q, which is neither seat:. nor .", l) } } return nil diff --git a/modules/mesh-delivery-sdk/graph/graph_test.go b/modules/mesh-delivery-sdk/graph/graph_test.go index 0d6fc75d..86fedf53 100644 --- a/modules/mesh-delivery-sdk/graph/graph_test.go +++ b/modules/mesh-delivery-sdk/graph/graph_test.go @@ -255,17 +255,17 @@ func nodeValues(t *testing.T) { change func(*Node) word string }{ - "both": {func(g *Node) { g.None = "why" }, "both"}, - "neither": {func(g *Node) { g.FP = "" }, "neither"}, - "an old form": {func(g *Node) { g.FP = "src1:x" }, "src2:"}, - "a dotted base": {func(g *Node) { g.Bases = []string{"a.b"} }, "a.b"}, - "a foreign package": {func(g *Node) { g.Packages = []string{"example.com/x"} }, "example.com/x"}, - "a toolchain of no module": {func(g *Node) { g.Toolchains = map[string]Toolchain{"go": {Version: "go1", Recipe: "1"}} }, "build module"}, - "a toolchain of no recipe": {func(g *Node) { g.Toolchains = map[string]Toolchain{"go": {Module: "b", Version: "go1"}} }, "recipe"}, - "awaiting what it does not require": {func(g *Node) { g.Awaits = []string{"other"} }, "other"}, - "no repository": {func(g *Node) { g.Repository = "" }, "repositor"}, - "leaving out what it does not invoke": {func(g *Node) { g.LeftOut = []string{"x.y"} }, "x.y"}, - "an invoke of no verb": {func(g *Node) { g.Invokes = []string{"seat:nothing"} }, "seat:nothing"}, + "both": {func(g *Node) { g.None = "why" }, "both"}, + "neither": {func(g *Node) { g.FP = "" }, "neither"}, + "an old form": {func(g *Node) { g.FP = "src1:x" }, "src2:"}, + "a dotted base": {func(g *Node) { g.Bases = []string{"a.b"} }, "a.b"}, + "a foreign package": {func(g *Node) { g.Packages = []string{"example.com/x"} }, "example.com/x"}, + "a toolchain of no module": {func(g *Node) { g.Toolchains = map[string]Toolchain{"go": {Version: "go1", Recipe: "1"}} }, "build module"}, + "a toolchain of no recipe": {func(g *Node) { g.Toolchains = map[string]Toolchain{"go": {Module: "b", Version: "go1"}} }, "recipe"}, + "awaiting what it does not require": {func(g *Node) { g.Awaits = []string{"other"} }, "other"}, + "no repository": {func(g *Node) { g.Repository = "" }, "repositor"}, + "leaving out no verb": {func(g *Node) { g.LeftOut = []string{"seat:nothing"} }, "seat:nothing"}, + "an invoke of no verb": {func(g *Node) { g.Invokes = []string{"seat:nothing"} }, "seat:nothing"}, } { g := sampleNode() c.change(&g) @@ -273,6 +273,12 @@ func nodeValues(t *testing.T) { t.Errorf("a node with %s: %v, want it refused naming %q", name, err, c.word) } } + // A-361: a package's seat call word left out on a cycle is kept though the module's own invokes do not name it. + kept := sampleNode() + kept.LeftOut = []string{"seat:mesh-catalog.registrations"} + if _, err := Encode(kept); err != nil { + t.Errorf("a node leaving out a package's call word is refused: %v", err) + } } // readersOfTheSeatList (D-31): a module whose manifest invokes the seat mesh-catalog's seat-definitions waits, by the verb diff --git a/sdk/graph/graph.go b/sdk/graph/graph.go index c8086e93..47e35a3e 100644 --- a/sdk/graph/graph.go +++ b/sdk/graph/graph.go @@ -171,9 +171,11 @@ func (g Node) check() error { } } } + // A left-out word is an invoked verb or a seat call word of a package the module builds against (A-361), which its + // own invokes need not name; so it is a verb word, invoked or not. for _, l := range g.LeftOut { - if !slices.Contains(g.Invokes, l) { - return fmt.Errorf("graph: the graph node leaves out %q, which it does not invoke", l) + if !isVerb(l) { + return fmt.Errorf("graph: the graph node leaves out %q, which is neither seat:. nor .", l) } } return nil diff --git a/sdk/graph/graph_test.go b/sdk/graph/graph_test.go index 54379bba..3573f79a 100644 --- a/sdk/graph/graph_test.go +++ b/sdk/graph/graph_test.go @@ -255,17 +255,17 @@ func nodeValues(t *testing.T) { change func(*Node) word string }{ - "both": {func(g *Node) { g.None = "why" }, "both"}, - "neither": {func(g *Node) { g.FP = "" }, "neither"}, - "an old form": {func(g *Node) { g.FP = "src1:x" }, "src2:"}, - "a dotted base": {func(g *Node) { g.Bases = []string{"a.b"} }, "a.b"}, - "a foreign package": {func(g *Node) { g.Packages = []string{"example.com/x"} }, "example.com/x"}, - "a toolchain of no module": {func(g *Node) { g.Toolchains = map[string]Toolchain{"go": {Version: "go1", Recipe: "1"}} }, "build module"}, - "a toolchain of no recipe": {func(g *Node) { g.Toolchains = map[string]Toolchain{"go": {Module: "b", Version: "go1"}} }, "recipe"}, - "awaiting what it does not require": {func(g *Node) { g.Awaits = []string{"other"} }, "other"}, - "no repository": {func(g *Node) { g.Repository = "" }, "repositor"}, - "leaving out what it does not invoke": {func(g *Node) { g.LeftOut = []string{"x.y"} }, "x.y"}, - "an invoke of no verb": {func(g *Node) { g.Invokes = []string{"seat:nothing"} }, "seat:nothing"}, + "both": {func(g *Node) { g.None = "why" }, "both"}, + "neither": {func(g *Node) { g.FP = "" }, "neither"}, + "an old form": {func(g *Node) { g.FP = "src1:x" }, "src2:"}, + "a dotted base": {func(g *Node) { g.Bases = []string{"a.b"} }, "a.b"}, + "a foreign package": {func(g *Node) { g.Packages = []string{"example.com/x"} }, "example.com/x"}, + "a toolchain of no module": {func(g *Node) { g.Toolchains = map[string]Toolchain{"go": {Version: "go1", Recipe: "1"}} }, "build module"}, + "a toolchain of no recipe": {func(g *Node) { g.Toolchains = map[string]Toolchain{"go": {Module: "b", Version: "go1"}} }, "recipe"}, + "awaiting what it does not require": {func(g *Node) { g.Awaits = []string{"other"} }, "other"}, + "no repository": {func(g *Node) { g.Repository = "" }, "repositor"}, + "leaving out no verb": {func(g *Node) { g.LeftOut = []string{"seat:nothing"} }, "seat:nothing"}, + "an invoke of no verb": {func(g *Node) { g.Invokes = []string{"seat:nothing"} }, "seat:nothing"}, } { g := sampleNode() c.change(&g) @@ -273,6 +273,12 @@ func nodeValues(t *testing.T) { t.Errorf("a node with %s: %v, want it refused naming %q", name, err, c.word) } } + // A-361: a package's seat call word left out on a cycle is kept though the module's own invokes do not name it. + kept := sampleNode() + kept.LeftOut = []string{"seat:mesh-catalog.registrations"} + if _, err := Encode(kept); err != nil { + t.Errorf("a node leaving out a package's call word is refused: %v", err) + } } // readersOfTheSeatList (D-31): a module whose manifest invokes the seat mesh-catalog's seat-definitions waits, by the verb