Compare commits
39
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d51324df45 | ||
|
|
58512a9c6d | ||
|
|
1fd833bfe8 | ||
|
|
f9014452d3 | ||
|
|
61449249a8 | ||
|
|
61f7e89ad5 | ||
|
|
9713ceb84d | ||
|
|
93fc2f48a1 | ||
|
|
d5d53893a4 | ||
|
|
a0b2d98002 | ||
|
|
bc80683b9e | ||
|
|
420ae3d0e7 | ||
|
|
7c0e1d223f | ||
|
|
a10ef8995b | ||
|
|
dfc2f2aa98 | ||
|
|
ab33f015ec | ||
|
|
791c2a3d48 | ||
|
|
d36e0ea278 | ||
|
|
b6ed18197b | ||
|
|
45d091778c | ||
|
|
3a1937914e | ||
|
|
25fdf9920e | ||
|
|
323e1270cc | ||
|
|
e7e23c57cc | ||
|
|
a8387ebad8 | ||
|
|
1963d84a7a | ||
|
|
b71425867f | ||
|
|
c7039d81e1 | ||
|
|
f76da2ce9c | ||
|
|
7f6d418205 | ||
|
|
0aec975d19 | ||
|
|
9da79ef8c3 | ||
|
|
7743bd0d4b | ||
|
|
62268cf43d | ||
|
|
ea3c8ad7f6 | ||
|
|
d984f0a55d | ||
|
|
f00b6aff8b | ||
|
|
eff0d5cbf4 | ||
|
|
1ed7e34155 |
@@ -31,6 +31,7 @@ func TestALookComesBackForATierTheBusRefusedWhole(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
bus := false
|
||||
asksOfEveryModule(t) // the one ask left is the builder's own (ADR 0453 §5): this test is of what a walk does around it
|
||||
was := askABuild
|
||||
askABuild = func(_ context.Context, source buildSource, _, _ string) (string, error) {
|
||||
if !bus {
|
||||
@@ -95,6 +96,7 @@ func TestALookFailsTheTierTheBusRefusedPastTheBound(t *testing.T) {
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
asksOfEveryModule(t) // the one ask left is the builder's own (ADR 0453 §5): this test is of what a walk does around it
|
||||
was := askABuild
|
||||
askABuild = func(context.Context, buildSource, string, string) (string, error) {
|
||||
return "", fmt.Errorf("%w: nats: no responders available for request", inventory.ErrNoChain)
|
||||
@@ -114,11 +116,6 @@ func TestALookFailsTheTierTheBusRefusedPastTheBound(t *testing.T) {
|
||||
t.Fatalf("a bus silent past its bound did not fail the plan in words: %s %q", got.State, got.Note)
|
||||
}
|
||||
|
||||
// Rebuilt by hand while the bus still reloads: said as waiting, not as a failure and not with an empty reason.
|
||||
joined, said, err := joinAPlan(ctx, open, "a")
|
||||
if err != nil || !joined || !strings.Contains(said, "not asked yet: the bus did not answer") {
|
||||
t.Fatalf("a rebuild left for the bus is said as %v %q %v", joined, said, err)
|
||||
}
|
||||
// And retried by hand: the module is named as waiting, not as asked again.
|
||||
got, _ = inv.PlanByID(ctx, "plan-bus")
|
||||
if got.State == inventory.PlanFailed {
|
||||
|
||||
@@ -15,6 +15,15 @@ import (
|
||||
// seconds) is not a failed build. The module is left unasked and the plan building, and the next look asks it again; only a
|
||||
// module's own fault, or a bus silent for ten minutes, fails the plan.
|
||||
|
||||
// asksOfEveryModule makes every module a test names the builder's own, the one module a walk still asks build-agent
|
||||
// for (ADR 0453 §5): what a test of that ask's waits reads.
|
||||
func asksOfEveryModule(t *testing.T) {
|
||||
t.Helper()
|
||||
was := buildersOwn
|
||||
buildersOwn = func(string) bool { return true }
|
||||
t.Cleanup(func() { buildersOwn = was })
|
||||
}
|
||||
|
||||
func aTierOfThree() (*inventory.Plan, map[string]inventory.Entry) {
|
||||
p := &inventory.Plan{ID: "plan-1", State: inventory.PlanBuilding, Tiers: [][]string{{"a", "b", "c"}},
|
||||
Modules: map[string]*inventory.PlanModule{}}
|
||||
@@ -26,6 +35,7 @@ func aTierOfThree() (*inventory.Plan, map[string]inventory.Entry) {
|
||||
}
|
||||
|
||||
func TestABusThatDoesNotAnswerForAMomentDoesNotFailTheTier(t *testing.T) {
|
||||
asksOfEveryModule(t)
|
||||
p, byName := aTierOfThree()
|
||||
bus := false
|
||||
was := askABuild
|
||||
@@ -68,6 +78,7 @@ func TestABusThatDoesNotAnswerForAMomentDoesNotFailTheTier(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestABusSilentForTenMinutesFailsTheTierAsBefore(t *testing.T) {
|
||||
asksOfEveryModule(t)
|
||||
p, byName := aTierOfThree()
|
||||
was := askABuild
|
||||
askABuild = func(context.Context, buildSource, string, string) (string, error) {
|
||||
@@ -119,6 +130,7 @@ func TestAPlanWithAModuleWaitingForTheBusIsOneTheLookComesBackTo(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestAModulesOwnFaultStillFailsThePlan(t *testing.T) {
|
||||
asksOfEveryModule(t)
|
||||
p, byName := aTierOfThree()
|
||||
delete(byName, "a")
|
||||
askModule(t.Context(), p, "a", byName)
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// A retry of a plan failed past the bus's bound, whose module the bus still does not answer, names that module as waiting:
|
||||
// read while the plan still says failed, it was named nowhere, and the retry said "asked" with nothing after it (issue
|
||||
// 1403, met again once the hand rebuild no longer resumed the plan first).
|
||||
func TestARetryNamesTheModulesStillWaitingForTheBus(t *testing.T) {
|
||||
since := time.Now().UTC()
|
||||
p := inventory.Plan{ID: "plan-bus", State: inventory.PlanFailed, Tiers: [][]string{{"a"}},
|
||||
Modules: map[string]*inventory.PlanModule{"a": {BusWaitFrom: &since}}}
|
||||
waiting := busWaitingOnRetry(p)
|
||||
_, said := retryWords(p, []string{"a"}, nil, waiting)
|
||||
if !strings.Contains(said, "a not asked yet: the bus did not answer") || strings.Contains(said, "asked ;") {
|
||||
t.Fatalf("a retry left for the bus is said as %q", said)
|
||||
}
|
||||
}
|
||||
@@ -68,6 +68,7 @@ func TestAPlanGoesOnWithoutAMachineAwayWhenItsTierIsSent(t *testing.T) {
|
||||
var mu sync.Mutex
|
||||
var asks []string
|
||||
var sends [][]string
|
||||
asksOfEveryModule(t) // the one ask left is the builder's own (ADR 0453 §5): this test is of what a walk does around it
|
||||
wasAsk := askABuild
|
||||
askABuild = func(_ context.Context, source buildSource, _, _ string) (string, error) {
|
||||
mu.Lock()
|
||||
|
||||
@@ -232,8 +232,8 @@ func (f following) hearMerge(ctx context.Context, m link.SourceMoved, now time.T
|
||||
fmt.Print(nothingReadsIt(repository, m))
|
||||
return nil
|
||||
}
|
||||
// **A module the merge adds is asked for now** (novox/hq issue 1617), not when the batch is cut.
|
||||
askNewModulesAtHearing(ctx, inv, m, entries, read, facts.from)
|
||||
// A module the merge adds is registered from the builder's first done key of it, which the builder's write wakes
|
||||
// (novox/hq ADR 0501 §6): nothing is asked here.
|
||||
// **Its delivery group's word, before the plans are held** (ADR 0306 decision 2): a slow answer holds up no walk.
|
||||
own := ownPath(touches)
|
||||
waits, holding, err := groupWaitHolding(ctx, inv, m, deliverySeatHeld(entries))
|
||||
|
||||
+110
-296
@@ -28,112 +28,6 @@ import (
|
||||
// cheapest next step. That is how novox/hq ADR 0001 records `hal/sdk` reaching 34,636:
|
||||
// nothing in it was wrong, and no one edit was the one that should have been a new file.
|
||||
|
||||
// buildCommand builds a module from its source and records what came out.
|
||||
//
|
||||
// **Run where there is a container runtime**, which is why it is a command rather than something
|
||||
// the control plane does on its own: building needs to run things on a machine, and what the
|
||||
// control plane may send a machine is bounded by the declaration language. This is the shape the
|
||||
// builder module will take when it is given work over the broker; today a person runs it, and the
|
||||
// mesh records the result the same way either way.
|
||||
// buildOn rebuilds every module the mesh holds that stands on the named module's artifacts — the
|
||||
// rebuild a changed base needs, which nothing else asks for: their sources did not move, and
|
||||
// "behind" does not see a base that did (novox/hq 04-ISSUES/131). Bases first among them too.
|
||||
func buildOn(ctx context.Context, base string, wait time.Duration) error {
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
held, err := open.inventory.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
against, err := open.inventory.BuiltAgainst(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var on []inventory.Entry
|
||||
for _, e := range held {
|
||||
if standsOnModule(e, base, against) {
|
||||
on = append(on, e)
|
||||
}
|
||||
}
|
||||
if len(on) == 0 {
|
||||
fmt.Printf("nothing the mesh holds stands on %s\n", base)
|
||||
return nil
|
||||
}
|
||||
on = orderByBases(on, against)
|
||||
fmt.Printf("%d module(s) stand on %s:\n", len(on), base)
|
||||
var failed []string
|
||||
for _, e := range on {
|
||||
fmt.Printf("--- %s\n", e.Manifest.Module)
|
||||
source := buildSource{Repository: e.Source.Repository, Seat: e.Source.Seat}
|
||||
if err := buildOne(ctx, source, e.Source.Path, e.Source.Ref, wait); err != nil {
|
||||
fmt.Printf(" %v\n", err)
|
||||
failed = append(failed, e.Manifest.Module)
|
||||
}
|
||||
}
|
||||
if len(failed) > 0 {
|
||||
return fmt.Errorf("%d of %d could not be built: %s", len(failed), len(on), strings.Join(failed, ", "))
|
||||
}
|
||||
fmt.Printf("\n%d module(s) rebuilt on %s. `push --behind` sends them on\n", len(on), base)
|
||||
return nil
|
||||
}
|
||||
|
||||
func buildCommand(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("build", flag.ContinueOnError)
|
||||
ref := set.String("ref", "", "the branch, tag or commit to build")
|
||||
// A module is a repository and a path within it (novox/hq ADR 0069). Empty is the repository's
|
||||
// root, which is the ordinary case and why this is a flag rather than a second argument.
|
||||
path := set.String("path", "", "the module's directory inside the repository")
|
||||
wait := set.Duration("wait", 10*time.Minute, "how long to wait for a builder to answer")
|
||||
dryRun := set.Bool("dry-run", false, "build and print the manifest, recording nothing")
|
||||
// Every module whose source has moved, rather than one named repository.
|
||||
//
|
||||
// **The mirror of `push --behind`, and the same argument** (novox/hq ADR 0010): the mesh
|
||||
// already knows which modules are behind their source, so making a person read that list and
|
||||
// retype each repository is asking them to be the loop. Naming a repository and asking which
|
||||
// ones need building are different requests, so they are not combined.
|
||||
behind := set.Bool("behind", false, "every module the mesh holds older than its source has")
|
||||
on := set.String("on", "", "rebuild every module that stands on this module's artifacts — the rebuild a changed base needs")
|
||||
// A repository on the mesh's own forge, named by its path there (novox/hq ADR 0111). Without it
|
||||
// the repository is external, cloned exactly as given — see source.go.
|
||||
self := set.Bool("self", false, "the repository is a path on the forge holding the git seat")
|
||||
positionals, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if *on != "" {
|
||||
if len(positionals) != 0 || *behind || *self {
|
||||
return errors.New("build --on <module> names a base and nothing else")
|
||||
}
|
||||
return buildOn(ctx, *on, *wait)
|
||||
}
|
||||
|
||||
if *behind {
|
||||
if len(positionals) != 0 || *self {
|
||||
return errors.New("build <repository> or build --behind, not both: one names a " +
|
||||
"repository and the other asks which need building")
|
||||
}
|
||||
return buildBehind(ctx, *wait)
|
||||
}
|
||||
if len(positionals) != 1 {
|
||||
return errors.New("build <repository> [--self] [--path P] [--ref R] [--wait D] [--dry-run] | build --behind | build --on <module>")
|
||||
}
|
||||
source := buildSource{Repository: positionals[0]}
|
||||
if *self {
|
||||
if err := onASeat(source.Repository); err != nil {
|
||||
return err
|
||||
}
|
||||
source.Seat = gitSeat
|
||||
}
|
||||
|
||||
if *dryRun {
|
||||
return buildAndShow(ctx, source, *path, *ref, *wait)
|
||||
}
|
||||
return buildOne(ctx, source, *path, *ref, *wait)
|
||||
}
|
||||
|
||||
// buildFrom turns what a builder said into what the mesh keeps.
|
||||
//
|
||||
// **By digest and path, never by where it was pushed** (novox/hq 04-ISSUES/102). The builder
|
||||
@@ -355,86 +249,6 @@ func builderCommand(ctx context.Context, args []string) error {
|
||||
return issueOnTheNewBus(ctx, inv, m, node, address)
|
||||
}
|
||||
|
||||
// buildBehind builds every module the mesh holds older than its source has.
|
||||
//
|
||||
// **This is the loop novox/hq ADR 0010 replaced a pipeline with, closed.** The mesh already
|
||||
// records where each module came from and what its source last had; until this, a person read
|
||||
// that list and retyped each repository — which is a person being the loop, and the thing a
|
||||
// pipeline was doing before it was taken away.
|
||||
//
|
||||
// Each is built and recorded on its own. **One failing does not stop the others**, for the same
|
||||
// reason one broken module no longer blocks a machine's whole declaration: a mesh where one bad
|
||||
// repository holds back nine good ones is a mesh where nobody dares add the tenth.
|
||||
func buildBehind(ctx context.Context, wait time.Duration) error {
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
inv := open.inventory
|
||||
|
||||
held, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var stale []inventory.Entry
|
||||
for _, e := range held {
|
||||
if !e.Source.Current() {
|
||||
stale = append(stale, e)
|
||||
}
|
||||
}
|
||||
if len(stale) == 0 {
|
||||
// Said rather than doing nothing quietly: "nothing needed building" and "this did not
|
||||
// run" must never look the same.
|
||||
fmt.Println("every module the mesh holds is what its source last had")
|
||||
return nil
|
||||
}
|
||||
|
||||
fmt.Printf("%d module(s) behind their source:\n", len(stale))
|
||||
for _, e := range stale {
|
||||
fmt.Printf(" %s %s < %s\n",
|
||||
e.Manifest.Module, short(e.Source.BuiltFrom), short(e.Source.Head))
|
||||
}
|
||||
fmt.Println()
|
||||
|
||||
// Bases first: a module built before the module it stands on is built against the old one
|
||||
// and reports success (novox/hq 04-ISSUES/131).
|
||||
against, err := inv.BuiltAgainst(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
stale = orderByBases(stale, against)
|
||||
|
||||
var failed []string
|
||||
for _, e := range stale {
|
||||
fmt.Printf("--- %s\n", e.Manifest.Module)
|
||||
// Its own recorded ref, not its head commit: a module tracking a branch should be built
|
||||
// from that branch, and pinning to the commit the mesh happened to notice would quietly
|
||||
// turn a tracked branch into a pin.
|
||||
source := buildSource{Repository: e.Source.Repository, Seat: e.Source.Seat}
|
||||
if err := buildOne(ctx, source, e.Source.Path, e.Source.Ref, wait); err != nil {
|
||||
fmt.Printf(" %v\n", err)
|
||||
failed = append(failed, e.Manifest.Module)
|
||||
}
|
||||
}
|
||||
|
||||
if len(failed) > 0 {
|
||||
return fmt.Errorf("%d of %d could not be built: %s",
|
||||
len(failed), len(stale), strings.Join(failed, ", "))
|
||||
}
|
||||
fmt.Printf("\n%d module(s) built. `push --behind` sends them to the machines running them\n",
|
||||
len(stale))
|
||||
return nil
|
||||
}
|
||||
|
||||
// buildOne asks a build machine for one repository and records everything that came back.
|
||||
//
|
||||
// Separated from the command so `--behind` can walk a list without a second path to the same act.
|
||||
func buildOne(ctx context.Context, source buildSource, path, ref string, wait time.Duration) error {
|
||||
_, err := buildOneAsked(ctx, source, path, ref, wait, false, "build")
|
||||
return err
|
||||
}
|
||||
|
||||
// recordAsked keeps a build request once it is asked (novox/hq issue 325), with who asked it, in a store
|
||||
// opened for the purpose: the asks reach here from processes that hold none.
|
||||
func recordAsked(ctx context.Context, r inventory.BuildRequest) {
|
||||
@@ -511,7 +325,6 @@ func buildOneAsked(ctx context.Context, source buildSource, path, ref string, wa
|
||||
Seats: seatBases(ctx),
|
||||
DryRun: dryRun,
|
||||
}
|
||||
request.Reuse = reuseAsked(ctx, dryRun)
|
||||
fmt.Printf("asked for %s", source)
|
||||
if source.Seat != "" {
|
||||
fmt.Printf(" (%s)", repository)
|
||||
@@ -679,13 +492,6 @@ func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResu
|
||||
recorded.Ref = followedBranch(was.Ref)
|
||||
}
|
||||
}
|
||||
if err := namesNoInstallation(manifest); err != nil {
|
||||
return manifest, kept, fmt.Errorf("%s built %s (%s), and the mesh does not register it: %w",
|
||||
result.On, result.Repository, short(result.Commit), err)
|
||||
}
|
||||
// **Only from the repository the catalogue builds the module from** (novox/hq ADR 0266): else the trunk
|
||||
// below is the trunk of whatever repository was built, which may be one an agent made — and a module named
|
||||
// `sudo` from it would be what the next push sends. Another repository is the operator's, at the terminal.
|
||||
trunk := result.Trunk
|
||||
if was, err := inv.SourceOf(ctx, manifest.Module); err == nil && followedBranch(was.Ref) != "" {
|
||||
trunk = followedBranch(was.Ref)
|
||||
@@ -693,55 +499,111 @@ func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResu
|
||||
if trunk == "" {
|
||||
trunk = "main"
|
||||
}
|
||||
repoID, err := mayRegisterFrom(ctx, inv, manifest.Module, recorded, result.ID, result.Path, trunk)
|
||||
j := takenBuild{kept: kept, manifest: manifest, recorded: recorded, ask: byBuild(result.ID), path: result.Path,
|
||||
trunk: trunk, by: result.On, what: result.Repository, commit: result.Commit}
|
||||
// The checks only an outcome of build-agent needs: a key is a trunk build by construction (ADR 0501 §2), in no
|
||||
// scratch namespace, and carries no list of its own.
|
||||
j.published = func() error {
|
||||
// **Only a commit on the trunk is published** (novox/hq ADR 0238): a commit off its repository's
|
||||
// default branch — a pull request's head, a feature branch built by hand, a `rebuild` or `replay
|
||||
// --register` of one — is for checking, and is never a module's version; nothing could then send it.
|
||||
// The branch the module already follows is its trunk — never the branch a build was asked at, or a
|
||||
// build of a feature branch by name would make that branch its trunk.
|
||||
follows := ""
|
||||
if was, err := inv.SourceOf(ctx, manifest.Module); err == nil {
|
||||
follows = followedBranch(was.Ref)
|
||||
}
|
||||
if err := publishable(result, follows); err != nil {
|
||||
return err
|
||||
}
|
||||
// **Never a build in the scratch namespace** (novox/hq ADR 0239 §9, ADR 0350 §2): what a check builds is
|
||||
// registered only as the walk's copy in the module's own namespace.
|
||||
if scratch := inScratch(result.Made); scratch != "" {
|
||||
return fmt.Errorf("%s is in the scratch namespace, which only a walk's copy leaves (ADR 0350)", scratch)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
j.carried = func() error {
|
||||
// **The newest trunk build's list of predefined keys is published** (novox/hq ADR 0348 §7): a build of the
|
||||
// catalogue that may be registered keeps its list when its commit is newer than the one kept.
|
||||
carried, err := carriedList(result)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if carried != nil {
|
||||
if err := publishPredefined(ctx, inv, result, carried); err != nil {
|
||||
fmt.Printf("%s: the list of predefined keys it carried is not published: %v\n", result.ID, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
manifest, err = registerJudged(ctx, inv, j)
|
||||
return manifest, kept, err
|
||||
}
|
||||
|
||||
// takenBuild is a build the controller judges and registers, whatever told it of the build: build-agent's outcome (the
|
||||
// builder's own module's, until R4g, ADR 0453 §5) or the builder's done key (ADR 0501 §1). **One judging for both**:
|
||||
// the rules of who may register, the gate a build failed, the predefined keys and the rule that an unchanged build
|
||||
// source moves nothing are the same whichever part built it (ADR 0501 §1, §3).
|
||||
type takenBuild struct {
|
||||
kept inventory.Build
|
||||
manifest catalogue.Manifest
|
||||
recorded inventory.Source
|
||||
ask registrationAsk
|
||||
path string
|
||||
trunk string
|
||||
// by and what name the build in what is said: who built it, and what was built ("ace", "novox/mesh-catalog").
|
||||
by, what, commit string
|
||||
// published refuses a build that may not become a module's version (off the trunk, in the scratch namespace);
|
||||
// nil for a key, which is a trunk fingerprint's by construction (ADR 0501 §2).
|
||||
published func() error
|
||||
// carried publishes a list of predefined keys the build carried; nil when it carried none.
|
||||
carried func() error
|
||||
}
|
||||
|
||||
// registerJudged judges a recorded build and registers its module, or says why not.
|
||||
func registerJudged(ctx context.Context, inv *inventory.Inventory, j takenBuild) (catalogue.Manifest, error) {
|
||||
manifest, kept := j.manifest, j.kept
|
||||
notRegistered := func(err error) (catalogue.Manifest, error) {
|
||||
return manifest, fmt.Errorf("%s built %s (%s), recorded and not registered: %w", j.by, manifest.Module,
|
||||
short(j.commit), err)
|
||||
}
|
||||
if err := namesNoInstallation(manifest); err != nil {
|
||||
return manifest, fmt.Errorf("%s built %s (%s), and the mesh does not register it: %w",
|
||||
j.by, j.what, short(j.commit), err)
|
||||
}
|
||||
// **Only from the repository the catalogue builds the module from** (novox/hq ADR 0266): else the trunk
|
||||
// below is the trunk of whatever repository was built, which may be one an agent made — and a module named
|
||||
// `sudo` from it would be what the next push sends. Another repository is the operator's, at the terminal.
|
||||
repoID, err := mayRegisterFrom(ctx, inv, manifest.Module, j.recorded, j.ask, j.path, j.trunk)
|
||||
if err != nil {
|
||||
return manifest, kept, fmt.Errorf("%s built %s (%s), recorded and not registered: %w", result.On,
|
||||
manifest.Module, short(result.Commit), err)
|
||||
return notRegistered(err)
|
||||
}
|
||||
// **Only a commit on the trunk is published** (novox/hq ADR 0238): a commit off its repository's
|
||||
// default branch — a pull request's head, a feature branch built by hand, a `rebuild` or `replay
|
||||
// --register` of one — is for checking, and is never a module's version; nothing could then send it.
|
||||
// The branch the module already follows is its trunk — never the branch a build was asked at, or a
|
||||
// build of a feature branch by name would make that branch its trunk.
|
||||
follows := ""
|
||||
if was, err := inv.SourceOf(ctx, manifest.Module); err == nil {
|
||||
follows = followedBranch(was.Ref)
|
||||
}
|
||||
if err := publishable(result, follows); err != nil {
|
||||
return manifest, kept, fmt.Errorf("%s built %s (%s), recorded and not registered: %w", result.On,
|
||||
manifest.Module, short(result.Commit), err)
|
||||
}
|
||||
// **Never a build in the scratch namespace** (novox/hq ADR 0239 §9, ADR 0350 §2): what a check builds is
|
||||
// registered only as the walk's copy in the module's own namespace.
|
||||
if scratch := inScratch(result.Made); scratch != "" {
|
||||
return manifest, kept, fmt.Errorf("%s built %s (%s), recorded and not registered: %s is in the scratch "+
|
||||
"namespace, which only a walk's copy leaves (ADR 0350)", result.On, manifest.Module, short(result.Commit), scratch)
|
||||
if j.published != nil {
|
||||
if err := j.published(); err != nil {
|
||||
return notRegistered(err)
|
||||
}
|
||||
}
|
||||
// **A build that failed its gate is never registered again** (novox/hq ADR 0236): an outcome heard
|
||||
// twice, or replayed, would otherwise make the build a rollback put back what the module is again,
|
||||
// and the next push would send it.
|
||||
if failed, err := inv.GateFailed(ctx, kept.ID); err != nil {
|
||||
return manifest, kept, err
|
||||
return manifest, err
|
||||
} else if failed {
|
||||
return manifest, kept, fmt.Errorf("%s built %s (%s), which failed its gate on its first machine and was put "+
|
||||
"back: it is recorded and not registered again — a newer build is", result.On, manifest.Module,
|
||||
short(result.Commit))
|
||||
return manifest, fmt.Errorf("%s built %s (%s), which failed its gate on its first machine and was put "+
|
||||
"back: it is recorded and not registered again — a newer build is", j.by, manifest.Module,
|
||||
short(j.commit))
|
||||
}
|
||||
// **The newest trunk build's list of predefined keys is published** (novox/hq ADR 0348 §7): a build of the
|
||||
// catalogue that may be registered keeps its list when its commit is newer than the one kept. Then the module
|
||||
// is judged against the list the mesh publishes, as every manifest read again is: one naming a key that list
|
||||
// does not hold, or of another type, is refused here and never enters the catalogue.
|
||||
if carried, err := carriedList(result); err != nil {
|
||||
return manifest, kept, err
|
||||
} else if carried != nil {
|
||||
if err := publishPredefined(ctx, inv, result, carried); err != nil {
|
||||
fmt.Printf("%s: the list of predefined keys it carried is not published: %v\n", result.ID, err)
|
||||
// Then the module is judged against the list the mesh publishes, as every manifest read again is: one naming a
|
||||
// key that list does not hold, or of another type, is refused here and never enters the catalogue.
|
||||
if j.carried != nil {
|
||||
if err := j.carried(); err != nil {
|
||||
return manifest, err
|
||||
}
|
||||
}
|
||||
if manifest.NamesPredefined() {
|
||||
if _, err := catalogue.ParseManifest(kept.Manifest); err != nil {
|
||||
return manifest, kept, fmt.Errorf("%s built %s (%s), recorded and not registered: judged against the "+
|
||||
"list of predefined keys the mesh publishes, %w", result.On, manifest.Module, short(result.Commit), err)
|
||||
return notRegistered(fmt.Errorf("judged against the list of predefined keys the mesh publishes, %w", err))
|
||||
}
|
||||
}
|
||||
// **A build whose source is unchanged is never a move** (novox/hq issue 280): a rebuild made from
|
||||
@@ -751,35 +613,34 @@ func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResu
|
||||
if kept.SourceFingerprint != "" {
|
||||
stands, raw, err := inv.StandingBuild(ctx, manifest.Module, kept.ID)
|
||||
if err != nil {
|
||||
return manifest, kept, err
|
||||
return manifest, err
|
||||
}
|
||||
if stands != "" && stands != kept.ID && len(raw) > 0 {
|
||||
if same, err := catalogue.ParseManifest(raw); err == nil && same.Module == manifest.Module {
|
||||
if was, err := catalogue.ParseManifest(raw); err == nil && was.Module == manifest.Module {
|
||||
fmt.Printf("%s at %s was made from the source %s was: registered with its artifacts, no move\n",
|
||||
manifest.Module, short(result.Commit), stands)
|
||||
manifest = same
|
||||
manifest.Module, short(j.commit), stands)
|
||||
manifest = was
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := inv.RegisterModule(ctx, manifest, recorded); err != nil {
|
||||
if err := inv.RegisterModule(ctx, manifest, j.recorded); err != nil {
|
||||
if errors.Is(err, inventory.ErrSuperseded) {
|
||||
return manifest, kept, fmt.Errorf("%s built %s (%s), recorded and not registered: %w",
|
||||
result.On, manifest.Module, short(result.Commit), err)
|
||||
return notRegistered(err)
|
||||
}
|
||||
return manifest, kept, err
|
||||
return manifest, err
|
||||
}
|
||||
// A bus process the new definition gains gets its account now, on each machine assigned the module, so the
|
||||
// next push composes a real account (issue 1226).
|
||||
issueGainedProcessAccounts(ctx, inv, manifest)
|
||||
// Which repository it is registered from, by the forge's own id (novox/hq ADR 0266).
|
||||
if err := inv.SetSourceIdentity(ctx, manifest.Module, repoID); err != nil {
|
||||
return manifest, kept, err
|
||||
return manifest, err
|
||||
}
|
||||
// The keep set just moved, and new bytes just landed (novox/hq ADR 0189). Asked here rather
|
||||
// than on a timer of its own: this is the only moment either is true. Never fatal — the build
|
||||
// worked and the module is registered.
|
||||
collect(ctx, inv)
|
||||
return manifest, kept, nil
|
||||
return manifest, nil
|
||||
}
|
||||
|
||||
// inScratch is the first artifact a build made into the scratch namespace; empty when none is.
|
||||
@@ -820,64 +681,17 @@ func publishable(result link.BuildResult, follows string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// buildAndShow builds and prints the manifest without recording anything.
|
||||
func buildAndShow(ctx context.Context, source buildSource, path, ref string, wait time.Duration) error {
|
||||
repository, err := cloneFrom(ctx, source)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := verbMayAsk(ctx, source, repository, path); err != nil {
|
||||
return err
|
||||
}
|
||||
ident, err := openIdentity(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer ident.Close()
|
||||
server, err := connectLink(ctx, nil, nil, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer server.Close()
|
||||
|
||||
ask, err := askOverOn(buildSeatHeld(ctx))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer ask.Close()
|
||||
|
||||
result, err := ask.Submit(ctx, link.BuildRequest{
|
||||
ID: link.NewBuildID(time.Now()),
|
||||
Repository: repository, Path: path, Ref: ref,
|
||||
Held: heldBy(ctx), Seats: seatBases(ctx),
|
||||
DryRun: true,
|
||||
}, wait)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if result.Failed != "" {
|
||||
return fmt.Errorf("%s could not build %s:\n%s", result.On, result.Repository, result.Failed)
|
||||
}
|
||||
manifest, err := catalogue.ParseManifest(result.Manifest)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s built %s and what came back is not a manifest: %w",
|
||||
result.On, result.Repository, err)
|
||||
}
|
||||
body, err := json.MarshalIndent(manifest, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(string(body))
|
||||
return nil
|
||||
}
|
||||
|
||||
// answers is what the three questions came back with, read once.
|
||||
type answers struct {
|
||||
wrong []inventory.Doing
|
||||
nodes []inventory.Node
|
||||
quiet []inventory.Node
|
||||
behind map[string][]string
|
||||
sources map[string]inventory.Source
|
||||
// notAnnounced are the owners some module reads the state of that have announced nothing yet (novox/hq ADR 0495,
|
||||
// issue 1790): read from `reads` as before until they do; notAnnouncedWhy why that could not be read.
|
||||
notAnnounced []string
|
||||
notAnnouncedWhy string
|
||||
wrong []inventory.Doing
|
||||
nodes []inventory.Node
|
||||
quiet []inventory.Node
|
||||
behind map[string][]string
|
||||
sources map[string]inventory.Source
|
||||
// waiting is every machine not running what the mesh would send it.
|
||||
waiting []inventory.Machine
|
||||
// recordedWaits is, per machine, every move of a recorded module that waits there for a person (novox/hq ADR
|
||||
|
||||
@@ -373,22 +373,7 @@ type protectionRule struct {
|
||||
// branch, else the first glob rule, in the forge's order, that covers it (gitea's first matching rule). Never a
|
||||
// later rule that happens to be stronger: the forge does not read it.
|
||||
func judgedRule(rules []protectionRule, branch string) forgeFacts {
|
||||
var applied *protectionRule
|
||||
for i := range rules {
|
||||
if rules[i].Rule == branch {
|
||||
applied = &rules[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
if applied == nil {
|
||||
for i := range rules {
|
||||
if ruleCovers(rules[i].Rule, branch) {
|
||||
applied = &rules[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
switch r := applied; {
|
||||
switch r := appliedRule(rules, branch); {
|
||||
case r == nil:
|
||||
return forgeFacts{Why: fmt.Sprintf("no protection rule covers %s", branch)}
|
||||
case r.Push:
|
||||
@@ -401,6 +386,22 @@ func judgedRule(rules []protectionRule, branch string) forgeFacts {
|
||||
return forgeFacts{Guarded: true}
|
||||
}
|
||||
|
||||
// appliedRule is the one rule the forge applies to a branch: the rule named for it, else the first glob rule, in the
|
||||
// forge's order, that covers it; nil when none does.
|
||||
func appliedRule(rules []protectionRule, branch string) *protectionRule {
|
||||
for i := range rules {
|
||||
if rules[i].Rule == branch {
|
||||
return &rules[i]
|
||||
}
|
||||
}
|
||||
for i := range rules {
|
||||
if ruleCovers(rules[i].Rule, branch) {
|
||||
return &rules[i]
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ruleCovers says a protection rule's name — a branch, or a glob of them — covers a branch, as the forge reads it.
|
||||
func ruleCovers(rule, branch string) bool {
|
||||
if rule == branch {
|
||||
@@ -469,8 +470,39 @@ func (f *sourceForms) onTheForge(repository, seat string) (owner, name string, o
|
||||
// repository and path; or, for the build of a merge, when mesh-delivery announced it on the operator's Approve of
|
||||
// that head (novox/hq ADR 0420 §7 for a moved module, ADR 0432 for a new one), the forge's checks still holding. path is the module's directory as built; branch the trunk it is registered from.
|
||||
func mayRegisterFrom(ctx context.Context, inv *inventory.Inventory, module string, built inventory.Source,
|
||||
buildID, path, branch string) (int64, error) {
|
||||
return mayRegisterIn(ctx, inv, newSourceForms(ctx, inv), module, built, buildID, path, branch)
|
||||
ask registrationAsk, path, branch string) (int64, error) {
|
||||
return mayRegisterIn(ctx, inv, newSourceForms(ctx, inv), module, built, ask, path, branch)
|
||||
}
|
||||
|
||||
// registrationAsk is what a registration's warrant is read from (novox/hq ADR 0501 §3): the build an outcome answers,
|
||||
// whose kept request says whether it was asked at the terminal or for a merge (build-agent's outcome, the builder's own
|
||||
// module until R4g); or, for a take-in from the builder's done key, the merge the indexer's trunk value names and
|
||||
// whether the operator asked the registration at the controller's terminal (ADR 0501 §4). A key asks no build, so it
|
||||
// keeps no request.
|
||||
type registrationAsk struct {
|
||||
// Build is the build whose kept request is read; empty for a take-in from a key.
|
||||
Build string
|
||||
// Merge is the commit of the merge a key's module is registered from (the indexer's trunk value); empty for an
|
||||
// outcome, whose request names it.
|
||||
Merge string
|
||||
// Terminal is a key registration the operator asked at the controller's terminal for this repository and path.
|
||||
Terminal bool
|
||||
}
|
||||
|
||||
// byBuild is the warrant of a build's outcome: read from the request kept under its id.
|
||||
func byBuild(id string) registrationAsk { return registrationAsk{Build: id} }
|
||||
|
||||
// String is the ask as the registration's words name it.
|
||||
func (a registrationAsk) String() string {
|
||||
switch {
|
||||
case a.Build != "":
|
||||
return a.Build
|
||||
case a.Terminal:
|
||||
return "the registration asked at the terminal"
|
||||
case a.Merge != "":
|
||||
return fmt.Sprintf("the merge %.8s", a.Merge)
|
||||
}
|
||||
return "a key"
|
||||
}
|
||||
|
||||
// catalogueReader is the catalogue's manifests, as the router's records are found in them.
|
||||
@@ -490,15 +522,19 @@ type registrations interface {
|
||||
|
||||
// mayRegisterIn is mayRegisterFrom on what it reads of the mesh handed in.
|
||||
func mayRegisterIn(ctx context.Context, inv registrations, forms *sourceForms, module string, built inventory.Source,
|
||||
buildID, path, branch string) (int64, error) {
|
||||
ask registrationAsk, path, branch string) (int64, error) {
|
||||
buildID := ask.String()
|
||||
was, err := inv.SourceOf(ctx, module)
|
||||
isNew := errors.Is(err, inventory.ErrNoSuchModule)
|
||||
if err != nil && !isNew {
|
||||
return 0, err
|
||||
}
|
||||
terminal, err := askedHereFor(ctx, inv, forms, buildID, built, path)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
terminal := ask.Terminal
|
||||
if ask.Build != "" {
|
||||
var err error
|
||||
if terminal, err = askedHereFor(ctx, inv, forms, ask.Build, built, path); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
refuse := func(why string) (int64, error) {
|
||||
return 0, fmt.Errorf("%w: %s. A module is registered from such a source only by a build asked at the "+
|
||||
@@ -542,7 +578,7 @@ func mayRegisterIn(ctx context.Context, inv registrations, forms *sourceForms, m
|
||||
// The forge's protection and the repository's identity are checked below as for every registration.
|
||||
movedWhy := ""
|
||||
if why != "" && !terminal {
|
||||
accepted, err := approvedMove(ctx, inv, forms, buildID, built)
|
||||
accepted, err := approvedMove(ctx, inv, forms, ask, built)
|
||||
switch {
|
||||
case err == nil:
|
||||
movedWhy, why = accepted, ""
|
||||
@@ -662,37 +698,6 @@ func sourceWords(repository, seat string) string {
|
||||
return buildSource{Repository: repository, Seat: seat}.String()
|
||||
}
|
||||
|
||||
// firstBuildWarrant is what the controller checked before it asks the first build of the modules a merge adds to a
|
||||
// repository nothing is built from: the merge was made on the operator's Approve of its head (approvedAnnounced). It
|
||||
// travels in the context of that ask alone, in this process, so no verb carries one.
|
||||
type firstBuildWarrant struct {
|
||||
repository string // owner/name on the git seat
|
||||
paths map[string]bool
|
||||
why string
|
||||
}
|
||||
|
||||
type firstBuildWarrantKey struct{}
|
||||
|
||||
// withFirstBuildWarrant is ctx carrying the warrant for asking the first builds of paths in repository.
|
||||
func withFirstBuildWarrant(ctx context.Context, repository string, paths []string, why string) context.Context {
|
||||
w := &firstBuildWarrant{repository: repository, paths: map[string]bool{}, why: why}
|
||||
for _, p := range paths {
|
||||
w.paths[strings.Trim(p, "/")] = true
|
||||
}
|
||||
return context.WithValue(ctx, firstBuildWarrantKey{}, w)
|
||||
}
|
||||
|
||||
func firstBuildWarrantIn(ctx context.Context) *firstBuildWarrant {
|
||||
w, _ := ctx.Value(firstBuildWarrantKey{}).(*firstBuildWarrant)
|
||||
return w
|
||||
}
|
||||
|
||||
// covers says the warrant is for exactly this ask: the repository on the git seat, and a module path the merge holds.
|
||||
func (w *firstBuildWarrant) covers(source buildSource, path string) bool {
|
||||
return w != nil && source.Seat == gitSeat && strings.EqualFold(strings.Trim(source.Repository, "/"), w.repository) &&
|
||||
w.paths[strings.Trim(path, "/")]
|
||||
}
|
||||
|
||||
// verbMayAsk refuses, for a call through a verb, a build of a repository the catalogue builds no module from
|
||||
// (novox/hq ADR 0266): the build node would run what an agent wrote, and its outcome could never be
|
||||
// registered anyway. url is the repository as it is cloned. At the terminal anything may be asked.
|
||||
@@ -700,11 +705,6 @@ func verbMayAsk(ctx context.Context, source buildSource, url, path string) error
|
||||
if startedAtTheTerminal() {
|
||||
return nil
|
||||
}
|
||||
// **Or the first build of a module a merge adds on the operator's Approve** (novox/hq ADR 0432, issue 1146): the
|
||||
// controller's own ask, carrying the warrant it checked, for that repository and the module paths the merge holds.
|
||||
if firstBuildWarrantIn(ctx).covers(source, path) {
|
||||
return nil
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@@ -0,0 +1,139 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// builderOnTheBus is the builder's and the indexer's buckets on a test's own bus, which the controller's connection for
|
||||
// the moment (servingBus) reaches: what a take-in from a key reads (R4e, issue 1851).
|
||||
type builderOnTheBus struct {
|
||||
js *broker.JetStream
|
||||
builds, modules, manifests jetstream.KeyValue
|
||||
}
|
||||
|
||||
func aBuilderOnTheBus(t *testing.T) builderOnTheBus {
|
||||
t.Helper()
|
||||
js, err := broker.Dial(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
api, err := jetstream.New(js.Conn())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx := t.Context()
|
||||
b := builderOnTheBus{js: js}
|
||||
for _, c := range []struct {
|
||||
into *jetstream.KeyValue
|
||||
bucket string
|
||||
history uint8
|
||||
}{
|
||||
{&b.builds, sdkdelivery.BuilderBuildsBucket, sdkdelivery.BuilderBuildsHistory},
|
||||
{&b.modules, sdkdelivery.ModulesBucket, sdkdelivery.ModulesHistory},
|
||||
{&b.manifests, sdkdelivery.ManifestsBucket, 1},
|
||||
} {
|
||||
kv, err := api.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: c.bucket, History: c.history})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
*c.into = kv
|
||||
}
|
||||
servingBus.Store(js)
|
||||
t.Cleanup(func() { servingBus.Store(nil) })
|
||||
withTrunkList(t, true)
|
||||
// The indexer on this bus is read, not the "not held" a mesh with no bus reads (aMeshOnTheStore).
|
||||
wasIndexed := indexedModule
|
||||
indexedModule = theIndexedModule
|
||||
t.Cleanup(func() { indexedModule = wasIndexed })
|
||||
return b
|
||||
}
|
||||
|
||||
// indexed writes the indexer's trunk value of module at commit, naming fp, and its manifest: a module of one archive.
|
||||
func (b builderOnTheBus) indexed(t *testing.T, module, version, repository, path, commit, fp string) string {
|
||||
t.Helper()
|
||||
keys, trunk := aKeyFor(module, version, fp, repository, path, commit)
|
||||
raw := keys.manifests[trunk.Manifest]
|
||||
value, err := sdkdelivery.EncodeManifest(sdkdelivery.Manifest{Module: module, JSON: raw})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := b.manifests.Put(context.Background(), trunk.Manifest, value); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
trunk.FP = "src1:" + fp[len("src2:"):]
|
||||
encoded, err := sdkdelivery.EncodeModule(trunk)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := b.modules.Put(context.Background(), module, encoded); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return trunk.Manifest
|
||||
}
|
||||
|
||||
// built writes the builder's key of module at fp: done, its one archive named with a reference, or in another state.
|
||||
func (b builderOnTheBus) built(t *testing.T, module, version, fp, manifest, state string) {
|
||||
t.Helper()
|
||||
keys, _ := aKeyFor(module, version, fp, "", "", "")
|
||||
v := keys.keys[module].Build
|
||||
v.Manifest = manifest
|
||||
if state != sdkdelivery.BuildDone {
|
||||
v = sdkdelivery.BuilderBuild{State: state, By: "ace"}
|
||||
if state == sdkdelivery.BuildFailed {
|
||||
v.Error = "digests differ: ace sha256:1, g14 sha256:2"
|
||||
}
|
||||
}
|
||||
value, err := sdkdelivery.EncodeBuilderBuild(v)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key, err := sdkdelivery.BuilderBuildKey(module, fp)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := b.builds.Put(context.Background(), key, value); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// noBuildAsked fails the test if anything asks build-agent for a build (ADR 0501 §6, ADR 0503 §6).
|
||||
func noBuildAsked(t *testing.T) {
|
||||
t.Helper()
|
||||
was := askABuild
|
||||
askABuild = func(_ context.Context, _ buildSource, path, _ string) (string, error) {
|
||||
t.Errorf("a build was asked of build-agent, at %q", path)
|
||||
return "", context.Canceled
|
||||
}
|
||||
t.Cleanup(func() { askABuild = was })
|
||||
}
|
||||
|
||||
// asksWithPaths records every build a merge asks for, as repository, path and ref. The merge window's goroutine and
|
||||
// the test's own may ask at once, so the record is written under a lock (issue 1543).
|
||||
func asksWithPaths(t *testing.T) *[][3]string {
|
||||
t.Helper()
|
||||
asksOfEveryModule(t) // the one ask left is the builder's own (ADR 0453 §5): these tests are of what a walk does around it
|
||||
var asked [][3]string
|
||||
var mu sync.Mutex
|
||||
was := askABuild
|
||||
askABuild = func(_ context.Context, source buildSource, path, ref string) (string, error) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
asked = append(asked, [3]string{source.Repository, path, ref})
|
||||
return "b-" + path, nil
|
||||
}
|
||||
t.Cleanup(func() { askABuild = was })
|
||||
return &asked
|
||||
}
|
||||
|
||||
// theIndexedModule is the look at the indexer as the controller makes it, kept before any test replaces it.
|
||||
var theIndexedModule = indexedModule
|
||||
@@ -0,0 +1,112 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A12 (ADR 0453 §5, ADR 0498 §7; issue 1851): the one function every ask goes through refuses any module but the
|
||||
// builder's own, naming the builder's key it is taken from instead; the builder's own module passes the refusal.
|
||||
func TestBuildAgentIsAskedForTheBuildersOwnModuleAlone(t *testing.T) {
|
||||
was := askedModuleOf
|
||||
t.Cleanup(func() { askedModuleOf = was })
|
||||
askedModuleOf = func(_ context.Context, _ buildSource, path string) (string, error) {
|
||||
return strings.TrimPrefix(path, "modules/"), nil
|
||||
}
|
||||
_, err := askABuild(t.Context(), buildSource{Repository: "novox/mesh-catalog", Seat: "git"}, "modules/sudo", "main")
|
||||
if !errors.Is(err, errNotTheBuildersOwn) || !strings.Contains(err.Error(), "sudo is taken from the builder's done key") ||
|
||||
!strings.Contains(err.Error(), "mesh-delivery-builder_builds") {
|
||||
t.Fatalf("an ask of another module: %v", err)
|
||||
}
|
||||
if err := onlyTheBuildersOwn(t.Context(), buildSource{Repository: "novox/mesh-delivery", Seat: "git"},
|
||||
"modules/mesh-delivery-builder"); err != nil {
|
||||
t.Fatalf("the builder's own module is refused: %v", err)
|
||||
}
|
||||
askedModuleOf = func(context.Context, buildSource, string) (string, error) { return "", errors.New("no store") }
|
||||
if err := onlyTheBuildersOwn(t.Context(), buildSource{Repository: "novox/x"}, ""); !errors.Is(err, errNotTheBuildersOwn) {
|
||||
t.Fatalf("an ask whose module cannot be read went on: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A12: a walk of the builder's own module and one other asks build-agent for the first alone; the other is taken from
|
||||
// its key.
|
||||
func TestAWalkAsksBuildAgentForTheBuildersOwnModuleAlone(t *testing.T) {
|
||||
var asked []string
|
||||
was := askABuild
|
||||
askABuild = func(_ context.Context, _ buildSource, path, _ string) (string, error) {
|
||||
asked = append(asked, path)
|
||||
return "build-1", nil
|
||||
}
|
||||
t.Cleanup(func() { askABuild = was })
|
||||
p, byName := aWalkOfOne("sudo")
|
||||
p.Tiers = [][]string{{"mesh-delivery-builder", "sudo"}}
|
||||
byName["mesh-delivery-builder"] = byName["sudo"]
|
||||
e := byName["mesh-delivery-builder"]
|
||||
e.Source.Repository, e.Source.Path = "novox/mesh-delivery", "modules/mesh-delivery-builder"
|
||||
byName["mesh-delivery-builder"] = e
|
||||
for _, m := range p.Tiers[0] {
|
||||
askModule(t.Context(), p, m, byName)
|
||||
}
|
||||
if len(asked) != 1 || asked[0] != "modules/mesh-delivery-builder" {
|
||||
t.Fatalf("build-agent was asked for %v, want the builder's own module alone", asked)
|
||||
}
|
||||
if s := p.Modules["mesh-delivery-builder"]; s.Key || s.Build != "build-1" {
|
||||
t.Fatalf("the builder's own module is not asked: %+v", s)
|
||||
}
|
||||
if s := p.Modules["sudo"]; !s.Key || s.Build != "" {
|
||||
t.Fatalf("the other module is not taken from its key: %+v", s)
|
||||
}
|
||||
}
|
||||
|
||||
// ADR 0520 §1 (issue 1851): until R4g build-agent and mesh-tools, the toolchain image its builds stand on, are build-agent's
|
||||
// own, as the builder's module is: the builder writes no key of them, so a walk asks build-agent for them and reads no key;
|
||||
// mesh-tools-go (its key from mesh-build-image) and any other module of the same tier are still taken from their keys.
|
||||
func TestAWalkAsksBuildAgentForItselfAndMeshToolsAndReadsNoKeyOfThem(t *testing.T) {
|
||||
own := []string{"build-agent", "mesh-tools"}
|
||||
keyed := []string{"mesh-tools-go", "sudo"}
|
||||
wasModule := askedModuleOf
|
||||
askedModuleOf = func(_ context.Context, _ buildSource, path string) (string, error) {
|
||||
return strings.TrimPrefix(path, "modules/"), nil
|
||||
}
|
||||
t.Cleanup(func() { askedModuleOf = wasModule })
|
||||
for _, m := range own {
|
||||
if err := onlyTheBuildersOwn(t.Context(), buildSource{Repository: "novox/x", Seat: "git"}, "modules/"+m); err != nil {
|
||||
t.Fatalf("an ask of %s is refused: %v", m, err)
|
||||
}
|
||||
}
|
||||
if err := onlyTheBuildersOwn(t.Context(), buildSource{Repository: "novox/x", Seat: "git"}, "modules/mesh-tools-go"); !errors.Is(err, errNotTheBuildersOwn) {
|
||||
t.Fatalf("an ask of mesh-tools-go went on: %v", err)
|
||||
}
|
||||
var asked []string
|
||||
was := askABuild
|
||||
askABuild = func(_ context.Context, _ buildSource, path, _ string) (string, error) {
|
||||
asked = append(asked, path)
|
||||
return "build-" + strings.TrimPrefix(path, "modules/"), nil
|
||||
}
|
||||
t.Cleanup(func() { askABuild = was })
|
||||
p, byName := aWalkOfOne("sudo")
|
||||
p.Tiers = [][]string{append(append([]string{}, own...), keyed...)}
|
||||
for _, m := range append(append([]string{}, own...), keyed...) {
|
||||
e := byName["sudo"]
|
||||
e.Source.Path = "modules/" + m
|
||||
byName[m] = e
|
||||
}
|
||||
for _, m := range p.Tiers[0] {
|
||||
askModule(t.Context(), p, m, byName)
|
||||
}
|
||||
if strings.Join(asked, " ") != "modules/build-agent modules/mesh-tools" {
|
||||
t.Fatalf("build-agent was asked for %v", asked)
|
||||
}
|
||||
for _, m := range own {
|
||||
if s := p.Modules[m]; s.Key || s.Build != "build-"+m || s.State != "asked" {
|
||||
t.Fatalf("%s is not asked of build-agent, or is read from a key: %+v", m, s)
|
||||
}
|
||||
}
|
||||
for _, m := range keyed {
|
||||
if s := p.Modules[m]; !s.Key || s.Build != "" {
|
||||
t.Fatalf("%s is not taken from its key: %+v", m, s)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -241,10 +241,9 @@ func mergeAskPrefix(number int, head string) string {
|
||||
return fmt.Sprintf("merge-%d-%s-", number, head)
|
||||
}
|
||||
|
||||
// askersOfMerges are the askers a merge's warrant is read under: the teller, and mesh-delivery for a merge before release 3e
|
||||
// (ADR 0437 §6). Release 4's controller work stops accepting mesh-delivery (ADR 0450 §8); a merge asked under it earlier is
|
||||
// still found here until then.
|
||||
func askersOfMerges() []string { return []string{sdkdelivery.TellerModule, theDelivery} }
|
||||
// askersOfMerges are the askers a merge's warrant is read under: the teller alone (ADR 0437 §6), since R4e stopped taking
|
||||
// mesh-delivery's (ADR 0450 §8; issue 1318).
|
||||
func askersOfMerges() []string { return []string{sdkdelivery.TellerModule} }
|
||||
|
||||
// caughtUpWarrant judges a chosen merge ask of the router as the warrant of a merge: the ask is the operator's, the warrant
|
||||
// answers that very ask as it was published, and it is the operator's Approve proved by the channel it came through (ADR 0437
|
||||
@@ -391,9 +390,6 @@ var findMergeWarrant = func(ctx context.Context, inv catalogueReader, owner, rep
|
||||
if asker == sdkdelivery.TellerModule && !strings.HasPrefix(id, sdkdelivery.AskKindMerge+"-") {
|
||||
continue
|
||||
}
|
||||
if asker == theDelivery && !named {
|
||||
continue
|
||||
}
|
||||
ask, w, err := closedRecordOf(ctx, inv, asker, id)
|
||||
if err != nil {
|
||||
return "", false, "", err
|
||||
|
||||
@@ -69,12 +69,12 @@ func TestAForgeThatDoesNotAnswerIsAnErrorNotAnEmptyTrunk(t *testing.T) {
|
||||
|
||||
func TestACaughtUpMergeIsAcceptedOnTheSameWarrantAsAnAnnouncedOne(t *testing.T) {
|
||||
ok := approvedMerge(t)
|
||||
for _, asker := range []string{sdkdelivery.TellerModule, theDelivery} {
|
||||
w := ok.warrant
|
||||
w.Asker = asker
|
||||
if _, err := caughtUpWarrant(asker, &ok.ask, &w); err != nil {
|
||||
t.Errorf("the operator's Approve under %s was refused: %v", asker, err)
|
||||
}
|
||||
if _, err := caughtUpWarrant(sdkdelivery.TellerModule, &ok.ask, &ok.warrant); err != nil {
|
||||
t.Errorf("the operator's Approve under the teller was refused: %v", err)
|
||||
}
|
||||
// 1318 (ADR 0450 §8): mesh-delivery's warrant of before the gate is no longer taken.
|
||||
if _, err := caughtUpWarrant(theDelivery, &ok.ask, &ok.warrant); err == nil {
|
||||
t.Error("a warrant under mesh-delivery was accepted")
|
||||
}
|
||||
if _, err := caughtUpWarrant("keycloak", &ok.ask, &ok.warrant); err == nil {
|
||||
t.Error("a warrant of an asker the controller takes none from was accepted")
|
||||
@@ -229,34 +229,6 @@ func TestAGroupsWarrantIsFoundByTheAsksWordsForThePullRequestAndItsHead(t *testi
|
||||
}
|
||||
}
|
||||
|
||||
// Keycloak's merge was asked as mesh-delivery.merge-378-b9104454f9a5-1, before release 3e: the id names the pull request and
|
||||
// the first twelve digits of its head, and the warrant is found by them under mesh-delivery.
|
||||
func TestAWarrantAskedUnderMeshDeliveryIsFoundByThePullRequestAndItsHead(t *testing.T) {
|
||||
ok := approvedMerge(t)
|
||||
head := "b9104454f9a5e0c4d3b2a1f0e9d8c7b6a5948372"
|
||||
askID := "merge-378-b9104454f9a5-1"
|
||||
ask, w := ok.ask, ok.warrant
|
||||
ask.ID, w.Ask, w.Asker = askID, askID, theDelivery
|
||||
w.AskDigest = ask.Digest()
|
||||
withRouter(t, map[string]map[string]aRouterRecord{theDelivery: {
|
||||
"merge-377-aaaaaaaaaaaa-1": {},
|
||||
askID: {ask, w},
|
||||
}})
|
||||
why, found, reason, err := findMergeWarrant(t.Context(), nil, "novox", "mesh-catalog", 378, head)
|
||||
if err != nil || !found || !strings.Contains(why, askID) {
|
||||
t.Fatalf("the warrant was not found: %q %v %q %v", why, found, reason, err)
|
||||
}
|
||||
// Another head names another ask, and none is found.
|
||||
if _, found, _, _ := findMergeWarrant(t.Context(), nil, "novox", "mesh-catalog", 378, "ffffffffffff0000"); found {
|
||||
t.Fatal("another head found a warrant")
|
||||
}
|
||||
// A router that cannot be read is an error, never "no warrant".
|
||||
routerAskIDs = func(context.Context, catalogueReader, string) ([]string, error) { return nil, errors.New("no answer") }
|
||||
if _, found, _, err := findMergeWarrant(t.Context(), nil, "novox", "mesh-catalog", 378, head); found || err == nil {
|
||||
t.Fatalf("an unreadable router read as no warrant: %v %v", found, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A trunk shorter than a page, or with none, is read without a fault: the pages end when the forge's do.
|
||||
func TestAShortTrunkIsReadWholeAndAnEmptyOneIsNoMerges(t *testing.T) {
|
||||
was := trunkCommitsPage
|
||||
|
||||
@@ -51,55 +51,13 @@ func attachTheChain(ctx context.Context, inv *inventory.Inventory, server *link.
|
||||
fmt.Printf("chain: the builds are not laid out under their modules yet, and none is read until they are: %v\n", err)
|
||||
go layBuildsOutSoon(ctx, inv)
|
||||
}
|
||||
// And the requests the move of delivery 1 found never handed over, asked again once (askAgain), once it is serving.
|
||||
go askAgainSoon(ctx, inv)
|
||||
// The requests the move of delivery 1 found never handed over are not asked again: the controller asks no build
|
||||
// (novox/hq ADR 0501; issue 1851, A10), and the builder builds every fingerprint the indexer writes.
|
||||
// And what delivery 1 recorded of what each machine was sent, copied onto the keys of the assignments (CopySentBuilds).
|
||||
go copySentBuildsSoon(ctx, inv)
|
||||
return nil
|
||||
}
|
||||
|
||||
// askAgain asks once more for each build request the move found never handed over (novox/hq issue 981 note 7 Q5): the
|
||||
// requests a merge could not ask for, which a person's hand `build` would otherwise have to repeat. **At most once**: a
|
||||
// request is claimed before it is asked and settled after, so a crash between loses the ask, which is where it stood.
|
||||
func askAgain(ctx context.Context, inv *inventory.Inventory) {
|
||||
pending, err := inv.RequestsToAskAgain(ctx)
|
||||
if err != nil {
|
||||
fmt.Printf("chain: the requests never handed over cannot be read: %v\n", err)
|
||||
return
|
||||
}
|
||||
for _, r := range pending {
|
||||
if claimed, err := inv.ClaimToAskAgain(ctx, r.ID); err != nil || !claimed {
|
||||
if err != nil {
|
||||
fmt.Printf("chain: %s cannot be claimed to be asked again: %v\n", r.ID, err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
id, err := askABuild(ctx, buildSource{Repository: r.Repository, Seat: r.Seat}, r.Path, r.Ref)
|
||||
if err != nil {
|
||||
if serr := inv.AskedAgainAs(ctx, r.ID, "not asked again: "+err.Error()); serr != nil {
|
||||
fmt.Printf("chain: %s was not asked again, and that could not be kept: %v\n", r.ID, serr)
|
||||
}
|
||||
fmt.Printf("chain: %s was never handed over and cannot be asked again: %v\n", r.ID, err)
|
||||
continue
|
||||
}
|
||||
recordBuildRequest(ctx, inv, inventory.BuildRequest{ID: id, Repository: r.Repository, Seat: r.Seat, Path: r.Path,
|
||||
Ref: r.Ref, Commit: r.Commit, For: r.For})
|
||||
if err := inv.AskedAgainAs(ctx, r.ID, id); err != nil {
|
||||
fmt.Printf("chain: %s was asked again as %s, and that could not be kept: %v\n", r.ID, id, err)
|
||||
}
|
||||
fmt.Printf("chain: %s, never handed over, is asked again as %s\n", r.ID, id)
|
||||
}
|
||||
}
|
||||
|
||||
// askAgainSoon waits for the controller to be serving, which an ask needs, and then asks again.
|
||||
func askAgainSoon(ctx context.Context, inv *inventory.Inventory) {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-time.After(time.Minute):
|
||||
askAgain(ctx, inv)
|
||||
}
|
||||
}
|
||||
|
||||
// kindSentBuildsNotCopied is the condition of the one-time copy of node.sent_builds onto the assignments' keys that has not
|
||||
// succeeded: until it has, a module an assignment pulled in may be missing from what a machine is read to run.
|
||||
const kindSentBuildsNotCopied = "sent-builds-not-copied"
|
||||
|
||||
@@ -5,7 +5,6 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/builder"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
@@ -53,7 +52,7 @@ func forgetChecksOfWalk(ctx context.Context, inv *inventory.Inventory, p invento
|
||||
if p.WalkSaid != inventory.WalkEnded || p.Delivery == nil {
|
||||
return
|
||||
}
|
||||
for _, pull := range reuseAskOf(&p).Pulls {
|
||||
for _, pull := range pullsOfWalk(&p) {
|
||||
said, err := forgetChecks(ctx, inv, builder.PullRef{Owner: pull.Owner, Repo: pull.Repo, Number: pull.Number})
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "%s: its check builds stay until the week's sweep: %v\n", p.ID, err)
|
||||
@@ -62,26 +61,3 @@ func forgetChecksOfWalk(ctx context.Context, inv *inventory.Inventory, p invento
|
||||
fmt.Printf("%s: %s\n", p.ID, said)
|
||||
}
|
||||
}
|
||||
|
||||
// forgetChecksCommand is `forget-checks <owner> <repo> <number>`: the verb the delivery's owner asks when the forge
|
||||
// says a pull request closed unmerged (novox/hq ADR 0350 §5).
|
||||
func forgetChecksCommand(ctx context.Context, args []string) error {
|
||||
if len(args) != 3 {
|
||||
return errors.New("forget-checks <owner> <repo> <number>")
|
||||
}
|
||||
number, err := strconv.Atoi(args[2])
|
||||
if err != nil || number <= 0 {
|
||||
return fmt.Errorf("%q is not a pull request's number", args[2])
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
said, err := forgetChecks(ctx, open.inventory, builder.PullRef{Owner: args[0], Repo: args[1], Number: number})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(said)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -14,23 +14,12 @@ import (
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
func TestOnlyAWalksAskTakesAChecksBuildAndADryRunNever(t *testing.T) {
|
||||
func TestAWalkNamesThePullRequestsOfItsMergesOnce(t *testing.T) {
|
||||
p := &inventory.Plan{Delivery: &inventory.PlanDelivery{Merges: []inventory.PlanMerge{
|
||||
{Repository: "novox/shop", Number: 7}, {Repository: "http://forge.invalid/novox/web.git", Number: 3},
|
||||
{Repository: "novox/shop", Number: 7}, {Repository: "novox/shop"}}}}
|
||||
asked := reuseAskOf(p)
|
||||
if want := []link.PullRef{{Owner: "novox", Repo: "shop", Number: 7}, {Owner: "novox", Repo: "web", Number: 3}}; !reflect.DeepEqual(asked.Pulls, want) {
|
||||
t.Fatalf("the walk names %+v", asked.Pulls)
|
||||
}
|
||||
walk := withReuse(t.Context(), asked)
|
||||
if reuseAsked(walk, false) != asked {
|
||||
t.Fatal("a walk's ask does not carry its leave")
|
||||
}
|
||||
if reuseAsked(walk, true) != nil {
|
||||
t.Fatal("a dry run carries a leave to take a check's build")
|
||||
}
|
||||
if reuseAsked(t.Context(), false) != nil {
|
||||
t.Fatal("an ask that is not a walk's carries a leave")
|
||||
if want := []link.PullRef{{Owner: "novox", Repo: "shop", Number: 7}, {Owner: "novox", Repo: "web", Number: 3}}; !reflect.DeepEqual(pullsOfWalk(p), want) {
|
||||
t.Fatalf("the walk names %+v", pullsOfWalk(p))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -104,44 +104,6 @@ func TestAWalksBuildSaysWhetherItReusedOrBuilt(t *testing.T) {
|
||||
// build as it judges one the walk built. A scratch reference is refused. Needs the store (MESH_TEST_POSTGRES), and
|
||||
// is named in testdata/store-tests.
|
||||
|
||||
// A walk's leave names the builds its own pull requests' verdicts made at the head each merge merged — never an
|
||||
// earlier head's — read once per walk, and none when they cannot be read: the builder takes nothing a verdict does
|
||||
// not name (novox/hq ADR 0350 §2, issue 703).
|
||||
func TestAWalksLeaveNamesTheBuildsItsPullRequestsVerdictsMade(t *testing.T) {
|
||||
was := checkedBuildsOf
|
||||
forgetWalkVerdicts()
|
||||
t.Cleanup(func() { checkedBuildsOf = was; forgetWalkVerdicts() })
|
||||
var asked [][]link.PullRef
|
||||
pull := link.PullRef{Owner: "novox", Repo: "shop", Number: 7}
|
||||
atHead := link.CheckedBuild{Pull: pull, Check: "build-2", Head: "bbbbbbbbbbbb", Built: link.CheckBuilt{Module: "app", Key: "r1:2"}}
|
||||
earlier := link.CheckedBuild{Pull: pull, Check: "build-1", Head: "aaaaaaaaaaaa", Built: link.CheckBuilt{Module: "app", Key: "r1:1"}}
|
||||
checkedBuildsOf = func(_ context.Context, pulls []link.PullRef) ([]link.CheckedBuild, error) {
|
||||
asked = append(asked, pulls)
|
||||
return []link.CheckedBuild{atHead, earlier}, nil
|
||||
}
|
||||
pulls := []link.PullRef{pull}
|
||||
heads := map[link.PullRef]string{pull: "bbbbbbbbbbbb0123456789"}
|
||||
for range 3 {
|
||||
if got := reuseWithVerdicts(t.Context(), "plan-1", &link.ReuseAsk{Pulls: pulls}, heads); !reflect.DeepEqual(got.Builds,
|
||||
[]link.CheckedBuild{atHead}) {
|
||||
t.Fatalf("the leave names %+v, not only the merged head's build", got.Builds)
|
||||
}
|
||||
}
|
||||
if !reflect.DeepEqual(asked, [][]link.PullRef{pulls}) {
|
||||
t.Fatalf("the verdicts were read %d times for one walk: %v", len(asked), asked)
|
||||
}
|
||||
// A merge whose head was not said takes nothing.
|
||||
if got := reuseWithVerdicts(t.Context(), "plan-1", &link.ReuseAsk{Pulls: pulls}, nil); got.Builds != nil {
|
||||
t.Fatalf("a merge of no said head names %+v", got.Builds)
|
||||
}
|
||||
checkedBuildsOf = func(context.Context, []link.PullRef) ([]link.CheckedBuild, error) {
|
||||
return nil, errors.New("no bus")
|
||||
}
|
||||
if got := reuseWithVerdicts(t.Context(), "plan-2", &link.ReuseAsk{Pulls: pulls}, heads); got.Builds != nil {
|
||||
t.Fatalf("verdicts that could not be read named %+v", got.Builds)
|
||||
}
|
||||
}
|
||||
|
||||
// A check is handed only the builds earlier checks of its own pull request at its own head made: another pull
|
||||
// request's build, or an earlier head's, is never kept (novox/hq issue 703). None when they cannot be read.
|
||||
func TestACheckIsHandedOnlyItsOwnHeadsBuilds(t *testing.T) {
|
||||
@@ -170,24 +132,3 @@ func TestACheckIsHandedOnlyItsOwnHeadsBuilds(t *testing.T) {
|
||||
t.Fatalf("verdicts that could not be read named %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func forgetWalkVerdicts() {
|
||||
walkVerdicts.Lock()
|
||||
defer walkVerdicts.Unlock()
|
||||
walkVerdicts.read = map[string]verdictsRead{}
|
||||
}
|
||||
|
||||
// A walk's merged heads are the forge's word for each merge; a merge announced without its head, or a pull request
|
||||
// merged at two heads in one walk, has none, and its walk takes no check's build (novox/hq issue 703).
|
||||
func TestAWalksMergedHeadsAreTheForgesWord(t *testing.T) {
|
||||
merge := func(number int, head string) inventory.PlanMerge {
|
||||
return inventory.PlanMerge{Repository: "novox/shop", Number: number, Head: head}
|
||||
}
|
||||
p := &inventory.Plan{Delivery: &inventory.PlanDelivery{Merges: []inventory.PlanMerge{
|
||||
merge(7, "aaaaaaaaaaaa1"), merge(8, ""), merge(9, "bbbbbbbbbbbb"), merge(9, "cccccccccccc")}}}
|
||||
heads := mergedHeadsOf(p)
|
||||
shop := func(n int) link.PullRef { return link.PullRef{Owner: "novox", Repo: "shop", Number: n} }
|
||||
if heads[shop(7)] != "aaaaaaaaaaaa1" || heads[shop(8)] != "" || heads[shop(9)] != "" {
|
||||
t.Fatalf("merged heads %v", heads)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,165 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// **A delivery group's member is checked beside the other members' heads** (novox/hq ADR 0382, issue 867).
|
||||
//
|
||||
// A mesh-host change and a mesh-controller change that need each other could not both pass their own merge checks:
|
||||
// each was read beside the other's repository at the commit the mesh runs, which holds neither change. So the
|
||||
// controller asks mesh-delivery's `groups` once for every pull request's check it asks (§1), and names, in the ask,
|
||||
// the group's other open pull requests into a base of the same name, in a repository the build seat reads beside the
|
||||
// check — mesh-lab never — each with its whole head. The build seat reads those repositories at the sibling's merge (§2).
|
||||
// A pull request for which mesh-delivery answered and named no sibling is asked exactly as before (§7).
|
||||
|
||||
// wholeCommit is a commit as the forge says a head: forty hexadecimal characters.
|
||||
var wholeCommit = regexp.MustCompile(`^[0-9a-f]{40}$`)
|
||||
|
||||
// besideRepositories are the repositories (lower case, owner/repo) a check reads beside its own, by the directory each
|
||||
// is found under — the core's, as besideOf clones them, and mesh-sdk beside the controller; never mesh-lab, which is
|
||||
// always read at its trunk.
|
||||
func besideRepositories(entries []inventory.Entry) map[string]string {
|
||||
read := map[string]string{}
|
||||
for _, e := range entries {
|
||||
dir, core := coreModules[e.Manifest.Module]
|
||||
if !core || e.Provided || e.Source.Repository == "" {
|
||||
continue
|
||||
}
|
||||
read[nameOf(e.Source.Repository)] = dir
|
||||
if dir == "mesh-controller" && e.Source.Seat != "" {
|
||||
read[nameOf(siblingOf(e.Source.Repository, "mesh-sdk"))] = "mesh-sdk"
|
||||
}
|
||||
}
|
||||
delete(read, "")
|
||||
return read
|
||||
}
|
||||
|
||||
// nameOf is a recorded repository as owner/repo in lower case, empty when it names no owner.
|
||||
func nameOf(repository string) string {
|
||||
owner, repo := sourceOwner(repository), sourceName(repository)
|
||||
if owner == "" || repo == "" {
|
||||
return ""
|
||||
}
|
||||
return strings.ToLower(owner + "/" + repo)
|
||||
}
|
||||
|
||||
// siblingsFor is the pull request's siblings, as mesh-delivery's groups say them (ADR 0382 §1): the other members of
|
||||
// the group its head branch names that are open, unmerged and into a base of its own base's name, in a repository the
|
||||
// check reads beside it (beside: the directories the ask already names). The second answer is why they could not be
|
||||
// learned, empty when they could: the group's members carry no head or base — a mesh-delivery from before the rule.
|
||||
func siblingsFor(p link.PullUpdated, entries []inventory.Entry, beside map[string]link.CheckedOut,
|
||||
lines []groupsLine) ([]link.CheckSibling, string) {
|
||||
own := strings.ToLower(p.Owner + "/" + p.Repo)
|
||||
var group *groupsLine
|
||||
if p.Head == "" {
|
||||
// A recheck (mesh-delivery's `recheck` through `delivery check`) carries no head branch name: the group is the one
|
||||
// whose members hold this very pull request, by repository and number.
|
||||
learnable := false
|
||||
for i := range lines {
|
||||
for _, m := range lines[i].Details {
|
||||
learnable = true
|
||||
if p.Number != 0 && m.Number == p.Number && strings.ToLower(m.Repository) == own {
|
||||
group = &lines[i]
|
||||
}
|
||||
}
|
||||
}
|
||||
if group == nil {
|
||||
for _, l := range lines {
|
||||
if len(l.Details) == 0 && len(l.Members) > 0 && !learnable {
|
||||
return nil, "mesh-delivery named the groups' members without their pull requests (a mesh-delivery from before novox/hq ADR 0382): " +
|
||||
"the group of a pull request announced without its head branch cannot be found"
|
||||
}
|
||||
}
|
||||
return nil, "" // a lone pull request
|
||||
}
|
||||
} else {
|
||||
for i := range lines {
|
||||
if lines[i].ID == p.Head {
|
||||
group = &lines[i]
|
||||
}
|
||||
}
|
||||
if group == nil {
|
||||
return nil, ""
|
||||
}
|
||||
}
|
||||
if len(group.Details) == 0 && len(group.Members) > 0 {
|
||||
return nil, "mesh-delivery named the group's members without their heads and bases (a mesh-delivery from before novox/hq ADR 0382)"
|
||||
}
|
||||
read := besideRepositories(entries)
|
||||
var siblings []link.CheckSibling
|
||||
for _, m := range group.Details {
|
||||
repository := strings.ToLower(m.Repository)
|
||||
if repository == own {
|
||||
continue
|
||||
}
|
||||
if !m.Open {
|
||||
continue
|
||||
}
|
||||
if m.Head == "" || m.Base == "" {
|
||||
return nil, "mesh-delivery named " + m.ID + " without its head or base (a mesh-delivery from before novox/hq ADR 0382)"
|
||||
}
|
||||
dir, readBeside := read[repository]
|
||||
if m.Base != p.Base || !readBeside || dir == "mesh-lab" {
|
||||
continue
|
||||
}
|
||||
if _, there := beside[dir]; !there {
|
||||
continue
|
||||
}
|
||||
head := strings.ToLower(m.Head)
|
||||
if !wholeCommit.MatchString(head) {
|
||||
return nil, fmt.Sprintf("mesh-delivery named %s with a head that is not a whole commit (%q)", m.ID, m.Head)
|
||||
}
|
||||
siblings = append(siblings, link.CheckSibling{Repository: m.Repository, Number: m.Number, Head: head, Base: m.Base, Dir: dir})
|
||||
}
|
||||
sort.Slice(siblings, func(i, j int) bool { return siblings[i].Dir < siblings[j].Dir })
|
||||
return siblings, ""
|
||||
}
|
||||
|
||||
// withSiblings asks mesh-delivery's `groups` once and says in the request the pull request's siblings, or why they
|
||||
// could not be learned (ADR 0382 §1): a check never waits on mesh-delivery past groupAskWithin and never fails for it,
|
||||
// so a fix to mesh-delivery is still checked without it.
|
||||
func withSiblings(ctx context.Context, request *link.BuildRequest, p link.PullUpdated, entries []inventory.Entry) {
|
||||
lines, err := groupsAsked(ctx, deliverySeatHeld(entries))
|
||||
siblingsFrom(request, p, entries, lines, err)
|
||||
}
|
||||
|
||||
// siblingsFrom is withSiblings from mesh-delivery's answer already had.
|
||||
func siblingsFrom(request *link.BuildRequest, p link.PullUpdated, entries []inventory.Entry, lines []groupsLine, err error) {
|
||||
if request.Check == nil {
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
request.Check.SiblingsUnknown = fmt.Sprintf("mesh-delivery did not say the group's pull requests (%v): "+
|
||||
"the check reads no siblings", err)
|
||||
return
|
||||
}
|
||||
request.Check.Siblings, request.Check.SiblingsUnknown = siblingsFor(p, entries, request.Check.Beside, lines)
|
||||
}
|
||||
|
||||
// composedMember says the pull request is a member of a delivery group whose gate is the composed check's (novox/hq
|
||||
// ADR 0403 clauses 1 and 7): the group has two members or more and mesh-delivery says so on its line. The group is
|
||||
// found as siblingsFor finds it, by its head branch name, or, for a recheck that carries none, by the pull request's
|
||||
// repository and number — not by siblings, which name only what the check reads beside it.
|
||||
func composedMember(p link.PullUpdated, lines []groupsLine) bool {
|
||||
own := strings.ToLower(p.Owner + "/" + p.Repo)
|
||||
for _, l := range lines {
|
||||
found := p.Head != "" && l.ID == p.Head
|
||||
for _, m := range l.Details {
|
||||
if p.Head == "" && p.Number != 0 && m.Number == p.Number && strings.ToLower(m.Repository) == own {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if found {
|
||||
return l.ComposedGate && max(len(l.Details), len(l.Members)) >= 2
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -1,248 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// **A delivery group's member is checked beside the other members' heads** (novox/hq ADR 0382 §1, §7, issue 867): the
|
||||
// controller asks mesh-delivery's `groups` once for every check it asks, and names in the ask the group's other open
|
||||
// pull requests into a base of the same name, in a repository the build seat reads beside it — mesh-lab never.
|
||||
|
||||
const (
|
||||
headHost = "1111111111111111111111111111111111111111"
|
||||
headController = "2222222222222222222222222222222222222222"
|
||||
headSDK = "3333333333333333333333333333333333333333"
|
||||
headLab = "4444444444444444444444444444444444444444"
|
||||
headHq = "5555555555555555555555555555555555555555"
|
||||
)
|
||||
|
||||
func theCoreOnTheGitSeat() []inventory.Entry {
|
||||
onSeat := func(module, repository string) inventory.Entry {
|
||||
e := fromRepo(module, repository, "")
|
||||
e.Source.Seat = "git"
|
||||
return e
|
||||
}
|
||||
nats := onSeat("nats", "novox/mesh-catalog")
|
||||
nats.Source.Path = "modules/nats"
|
||||
held := catalogue.Manifest{Module: "mesh-delivery", Claims: []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}}
|
||||
return []inventory.Entry{onSeat("mesh-controller", "novox/mesh-controller"), onSeat("mesh-host", "novox/mesh-host"),
|
||||
onSeat("node-tools", "novox/mesh-tools"), nats, {Manifest: held, On: []string{"anchor"}}}
|
||||
}
|
||||
|
||||
func besideTheCore() map[string]link.CheckedOut {
|
||||
return map[string]link.CheckedOut{"mesh-controller": {Repository: "c"}, "mesh-host": {Repository: "h"},
|
||||
"mesh-tools": {Repository: "t"}, "mesh-catalog": {Repository: "k"}, "mesh-sdk": {Repository: "s"}, "mesh-lab": {Repository: "l"}}
|
||||
}
|
||||
|
||||
func member(repository string, number int, head, base string, open bool) memberLine {
|
||||
return memberLine{ID: repository + "@" + head[:12], Repository: repository, State: "checked", Number: number, Head: head,
|
||||
Base: base, Open: open}
|
||||
}
|
||||
|
||||
func group(id string, members ...memberLine) groupsLine {
|
||||
line := groupsLine{ID: id, Details: members}
|
||||
for _, m := range members {
|
||||
line.Members = append(line.Members, m.ID+" ("+m.State+")")
|
||||
}
|
||||
return line
|
||||
}
|
||||
|
||||
func TestASiblingIsTheGroupsOtherOpenPullRequestIntoABaseOfTheSameName(t *testing.T) {
|
||||
entries := theCoreOnTheGitSeat()
|
||||
pull := link.PullUpdated{Owner: "novox", Repo: "mesh-host", Number: 114, Base: "main", Head: "feat/x", Commit: headHost}
|
||||
for _, c := range []struct {
|
||||
what string
|
||||
lines []groupsLine
|
||||
want []string
|
||||
unknown string
|
||||
}{
|
||||
{"the other member, open, same base", []groupsLine{group("feat/x",
|
||||
member("novox/mesh-host", 114, headHost, "main", true), member("novox/mesh-controller", 346, headController, "main", true))},
|
||||
[]string{"mesh-controller=novox/mesh-controller#346@" + headController + " onto main"}, ""},
|
||||
{"a member into a base of another name is not one", []groupsLine{group("feat/x",
|
||||
member("novox/mesh-host", 114, headHost, "main", true), member("novox/mesh-controller", 346, headController, "release", true))},
|
||||
nil, ""},
|
||||
{"a member not open (merged or closed) is not one", []groupsLine{group("feat/x",
|
||||
member("novox/mesh-host", 114, headHost, "main", true), member("novox/mesh-controller", 346, headController, "main", false))},
|
||||
nil, ""},
|
||||
{"mesh-lab is never one", []groupsLine{group("feat/x",
|
||||
member("novox/mesh-host", 114, headHost, "main", true), member("novox/mesh-lab", 5, headLab, "main", true))},
|
||||
nil, ""},
|
||||
{"a repository the check reads nothing of is not one", []groupsLine{group("feat/x",
|
||||
member("novox/mesh-host", 114, headHost, "main", true), member("novox/hq", 9, headHq, "main", true))},
|
||||
nil, ""},
|
||||
{"mesh-sdk, read beside the controller, is one", []groupsLine{group("feat/x",
|
||||
member("novox/mesh-host", 114, headHost, "main", true), member("novox/mesh-sdk", 7, headSDK, "main", true))},
|
||||
[]string{"mesh-sdk=novox/mesh-sdk#7@" + headSDK + " onto main"}, ""},
|
||||
{"another group is none of its business", []groupsLine{group("feat/y",
|
||||
member("novox/mesh-host", 114, headHost, "main", true), member("novox/mesh-controller", 346, headController, "main", true))},
|
||||
nil, ""},
|
||||
{"a group of its own alone has none", []groupsLine{group("feat/x", member("novox/mesh-host", 114, headHost, "main", true))},
|
||||
nil, ""},
|
||||
{"an older mesh-delivery names no heads", []groupsLine{{ID: "feat/x",
|
||||
Members: []string{"novox/mesh-host@111111111111 (checked)", "novox/mesh-controller@222222222222 (checked)"}}},
|
||||
nil, "without their heads and bases"},
|
||||
{"a member without its base", []groupsLine{group("feat/x",
|
||||
member("novox/mesh-host", 114, headHost, "main", true), member("novox/mesh-controller", 346, headController, "", true))},
|
||||
nil, "without its head or base"},
|
||||
{"a head that is no commit", []groupsLine{group("feat/x",
|
||||
member("novox/mesh-host", 114, headHost, "main", true), member("novox/mesh-controller", 346, headController[:12]+"zzzzzzzzzzzzzzzzzzzzzzzzzzzz", "main", true))},
|
||||
nil, "not a whole commit"},
|
||||
} {
|
||||
got, unknown := siblingsFor(pull, entries, besideTheCore(), c.lines)
|
||||
var said []string
|
||||
for _, s := range got {
|
||||
said = append(said, s.Dir+"="+s.Repository+"#"+itoa(s.Number)+"@"+s.Head+" onto "+s.Base)
|
||||
}
|
||||
if strings.Join(said, ";") != strings.Join(c.want, ";") {
|
||||
t.Errorf("%s: siblings %v, wanted %v", c.what, said, c.want)
|
||||
}
|
||||
if (c.unknown == "") != (unknown == "") || !strings.Contains(unknown, c.unknown) {
|
||||
t.Errorf("%s: could not learn %q, wanted %q", c.what, unknown, c.unknown)
|
||||
}
|
||||
}
|
||||
// A pull request into a release branch has siblings into the release branch of the same name (ADR 0358 §1).
|
||||
release := pull
|
||||
release.Base = "feat/57"
|
||||
lines := []groupsLine{group("feat/x", member("novox/mesh-host", 114, headHost, "feat/57", true),
|
||||
member("novox/mesh-controller", 346, headController, "feat/57", true), member("novox/mesh-tools", 8, headLab, "main", true))}
|
||||
if got, _ := siblingsFor(release, entries, besideTheCore(), lines); len(got) != 1 || got[0].Dir != "mesh-controller" {
|
||||
t.Errorf("into a release branch: %+v", got)
|
||||
}
|
||||
// A recheck announces the pull request without its head branch: its group is the one holding it (repository, number).
|
||||
pull.Head = ""
|
||||
got, unknown := siblingsFor(pull, entries, besideTheCore(), lines)
|
||||
if len(got) != 1 || got[0].Dir != "mesh-tools" || unknown != "" {
|
||||
t.Errorf("a recheck of a group member: %+v %q", got, unknown)
|
||||
}
|
||||
// A pull request in no group, rechecked, is alone: no sibling, and nothing said.
|
||||
pull.Number = 999
|
||||
if got, unknown := siblingsFor(pull, entries, besideTheCore(), lines); got != nil || unknown != "" {
|
||||
t.Errorf("a lone pull request: %+v %q", got, unknown)
|
||||
}
|
||||
if got, unknown := siblingsFor(pull, entries, besideTheCore(), nil); got != nil || unknown != "" {
|
||||
t.Errorf("no groups at all: %+v %q", got, unknown)
|
||||
}
|
||||
// A mesh-delivery that names members without their pull requests cannot find the group of a recheck: it says so.
|
||||
older := []groupsLine{{ID: "feat/x", Members: []string{"novox/mesh-host@111111111111 (checked)"}}}
|
||||
pull.Number = 114
|
||||
if got, unknown := siblingsFor(pull, entries, besideTheCore(), older); got != nil || unknown == "" {
|
||||
t.Errorf("an older mesh-delivery: %+v %q", got, unknown)
|
||||
}
|
||||
}
|
||||
|
||||
func itoa(n int) string { b, _ := json.Marshal(n); return string(b) }
|
||||
|
||||
// Every check asks mesh-delivery's `groups` once; the ask names the sibling's repository, head and base.
|
||||
func TestEveryCheckAsksMeshDeliveryOnceAndNamesItsSiblings(t *testing.T) {
|
||||
entries := theCoreOnTheGitSeat()
|
||||
asked := 0
|
||||
was := groupsAsked
|
||||
t.Cleanup(func() { groupsAsked = was })
|
||||
groupsAsked = func(_ context.Context, held bool) ([]groupsLine, error) {
|
||||
asked++
|
||||
if !held {
|
||||
t.Error("the seat is held, yet it was asked as not held")
|
||||
}
|
||||
return []groupsLine{group("feat/x", member("novox/mesh-host", 114, headHost, "main", true),
|
||||
member("novox/mesh-controller", 346, headController, "main", true))}, nil
|
||||
}
|
||||
request := link.BuildRequest{Check: &link.CheckRequest{Owner: "novox", Repo: "mesh-host", Base: "main", Beside: besideTheCore()}}
|
||||
withSiblings(t.Context(), &request, link.PullUpdated{Owner: "novox", Repo: "mesh-host", Base: "main", Head: "feat/x"}, entries)
|
||||
if asked != 1 {
|
||||
t.Fatalf("mesh-delivery was asked %d times for one check", asked)
|
||||
}
|
||||
want := link.CheckSibling{Repository: "novox/mesh-controller", Number: 346, Head: headController, Base: "main", Dir: "mesh-controller"}
|
||||
if got := request.Check.Siblings; len(got) != 1 || got[0] != want || request.Check.SiblingsUnknown != "" {
|
||||
t.Fatalf("the ask names siblings %+v, unknown %q", got, request.Check.SiblingsUnknown)
|
||||
}
|
||||
body, _ := json.Marshal(request.Check)
|
||||
if !strings.Contains(string(body), `"siblings":[{"repository":"novox/mesh-controller","number":346,"head":"`+headController+
|
||||
`","base":"main","dir":"mesh-controller"}]`) {
|
||||
t.Fatalf("on the wire: %s", body)
|
||||
}
|
||||
}
|
||||
|
||||
// When mesh-delivery does not answer in time (or is not held, or answered without the fields), the check reads no
|
||||
// sibling and its request says it could not learn them, so a fix to mesh-delivery is still checked without it.
|
||||
func TestWhenMeshDeliveryDoesNotAnswerTheCheckReadsNoSiblingAndSaysSo(t *testing.T) {
|
||||
was := groupsAsked
|
||||
t.Cleanup(func() { groupsAsked = was })
|
||||
for _, c := range []struct {
|
||||
what string
|
||||
err error
|
||||
held bool
|
||||
}{{"it did not answer", context.DeadlineExceeded, true}, {"nothing serves it", errors.New("nothing serves mesh-delivery.groups"), true},
|
||||
{"no holder on record", errors.New("no holder of mesh-delivery is on record to say it"), false}} {
|
||||
entries := theCoreOnTheGitSeat()
|
||||
if !c.held {
|
||||
entries = entries[:4]
|
||||
}
|
||||
groupsAsked = func(_ context.Context, held bool) ([]groupsLine, error) {
|
||||
if held != c.held {
|
||||
t.Errorf("%s: held %v", c.what, held)
|
||||
}
|
||||
return nil, c.err
|
||||
}
|
||||
request := link.BuildRequest{Check: &link.CheckRequest{Owner: "novox", Repo: "mesh-host", Base: "main", Beside: besideTheCore()}}
|
||||
withSiblings(t.Context(), &request, link.PullUpdated{Owner: "novox", Repo: "mesh-host", Base: "main", Head: "feat/x"}, entries)
|
||||
if len(request.Check.Siblings) != 0 || !strings.Contains(request.Check.SiblingsUnknown, c.err.Error()) ||
|
||||
!strings.Contains(request.Check.SiblingsUnknown, "reads no siblings") {
|
||||
t.Errorf("%s: siblings %+v, unknown %q", c.what, request.Check.Siblings, request.Check.SiblingsUnknown)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **A pull request for which mesh-delivery answered and named no sibling is asked exactly as before** (ADR 0382 §7):
|
||||
// byte for byte the request it was without the rule.
|
||||
func TestAPullRequestWithNoSiblingIsAskedByteForByteAsBefore(t *testing.T) {
|
||||
entries := theCoreOnTheGitSeat()
|
||||
was := groupsAsked
|
||||
t.Cleanup(func() { groupsAsked = was })
|
||||
for _, answer := range [][]groupsLine{nil, {group("feat/other", member("novox/mesh-controller", 346, headController, "main", true))},
|
||||
{group("feat/x", member("novox/mesh-host", 114, headHost, "main", true), member("novox/hq", 3, headHq, "main", true))}} {
|
||||
groupsAsked = func(context.Context, bool) ([]groupsLine, error) { return answer, nil }
|
||||
build := func() link.BuildRequest {
|
||||
return link.BuildRequest{ID: "build-1", Repository: "r", Ref: headHost, Check: &link.CheckRequest{Owner: "novox",
|
||||
Repo: "mesh-host", Number: 114, Base: "main", Beside: besideTheCore(), Modules: []string{"mesh-host"}, Judge: "validator"}}
|
||||
}
|
||||
before, _ := json.Marshal(build())
|
||||
after := build()
|
||||
withSiblings(t.Context(), &after, link.PullUpdated{Owner: "novox", Repo: "mesh-host", Base: "main", Head: "feat/x"}, entries)
|
||||
got, _ := json.Marshal(after)
|
||||
if string(before) != string(got) {
|
||||
t.Fatalf("the request changed:\nbefore %s\nafter %s", before, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// What the build seat's verdict says of its siblings reaches the forge's holder in the controller's `checked`.
|
||||
func TestTheVerdictCarriesTheSiblingsAndWhereTheJudgeCameFrom(t *testing.T) {
|
||||
read := []link.CheckSibling{{Repository: "novox/mesh-controller", Number: 346, Head: headController, Base: "main", BaseCommit: headHost}}
|
||||
c := checkedOf(link.BuildResult{ID: "b", Ref: headHost, Checked: &link.CheckRequest{Owner: "novox", Repo: "mesh-host", Number: 114},
|
||||
Check: &link.CheckOutcome{Verdict: "pass", Summary: "ok", Siblings: read, JudgeFrom: "sibling novox/mesh-controller#346 merge abc",
|
||||
Validator: "the change's own validator"}})
|
||||
if len(c.Siblings) != 1 || c.Siblings[0] != read[0] || c.JudgeFrom == "" || c.Validator == "" {
|
||||
t.Fatalf("the verdict says %+v", c)
|
||||
}
|
||||
body, _ := json.Marshal(c)
|
||||
for _, want := range []string{`"siblings":[{"repository":"novox/mesh-controller","number":346,"head":"` + headController +
|
||||
`","base":"main","base_commit":"` + headHost + `"}]`, `"judge_from":"sibling novox/mesh-controller#346 merge abc"`} {
|
||||
if !strings.Contains(string(body), want) {
|
||||
t.Fatalf("the verdict lacks %s:\n%s", want, body)
|
||||
}
|
||||
}
|
||||
unknown := checkedOf(link.BuildResult{ID: "b", Check: &link.CheckOutcome{Verdict: "pass", SiblingsUnknown: "mesh-delivery did not answer"}})
|
||||
if body, _ := json.Marshal(unknown); !strings.Contains(string(body), `"siblings_unknown":"mesh-delivery did not answer"`) ||
|
||||
strings.Contains(string(body), `"siblings":`) {
|
||||
t.Fatalf("the verdict that could not learn them says %s", body)
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"io"
|
||||
"os"
|
||||
"path"
|
||||
@@ -161,39 +162,11 @@ func (f following) pullUpdated(ctx context.Context, p link.PullUpdated, w io.Wri
|
||||
return "", err
|
||||
}
|
||||
scope := pullScope(p, entries, read, edges)
|
||||
// **A cycle of edges fails the check, naming every module and edge** (novox/hq ADR 0419 clause 6), a lone pull
|
||||
// request's as a group's composed check is: no order exists for what it changes.
|
||||
alone := orderMember{ID: p.Owner + "/" + p.Repo + "@" + twelve(p.Commit), Repository: p.Owner + "/" + p.Repo, Base: p.Base,
|
||||
Head: p.Commit, Number: p.Number, Paths: p.Paths, PathsTruncated: p.PathsTruncated, Removed: p.Removed, BaseSHA: p.BaseSHA}
|
||||
if failing, cyclic, err := cycleVerdict(ctx, "", []orderMember{alone},
|
||||
map[string]orderReach{alone.ID: {Moved: scope.Modules}}, edges, entries); err != nil {
|
||||
return "", err
|
||||
} else if cyclic {
|
||||
failing.Commit, failing.Trunk = p.Commit, twelve(p.BaseSHA)
|
||||
fmt.Fprintf(w, "%s/%s#%d (%.8s): %s\n", p.Owner, p.Repo, p.Number, p.Commit, failing.Summary)
|
||||
sayChecked(ctx, w, failing)
|
||||
return "", nil
|
||||
}
|
||||
// **A member of a group whose gate is the composed check's runs its repository's check alone** (novox/hq ADR 0403
|
||||
// clause 1): mesh-delivery is asked once, and says so on the group's line. Told nothing, the pull request is
|
||||
// checked alone, gate included (clause 6).
|
||||
lines, groupsErr := groupsAsked(ctx, deliverySeatHeld(entries))
|
||||
composed := groupsErr == nil && composedMember(p, lines)
|
||||
// The change plan of the commit at hand (ADR 0238): what a merge of it would build and send, posted
|
||||
// with the verdict whatever the verdict is.
|
||||
plan := changePlanOf(p.Owner+"/"+p.Repo, p.Base, p.Commit, scope.Reach, entries, func(module string) (inventory.Upgrade, bool) {
|
||||
u, err := inv.UpgradeOf(ctx, module)
|
||||
return u, err == nil
|
||||
})
|
||||
fmt.Fprint(w, planText(plan))
|
||||
direct := directVerdict(p, plan)
|
||||
if composed {
|
||||
scope.Modules, scope.Dependents, scope.New, scope.Manifests = nil, nil, nil, nil
|
||||
composedGate(&direct)
|
||||
}
|
||||
// **Mesh-config is told what the head declares at every announced head of a pull request in a repository that holds
|
||||
// modules** (novox/hq ADR 0502 clauses 2 and 6, issue 1826), also when the head touches no module: a head that reverts
|
||||
// the change must drop the value held for it. Asked of nothing else here.
|
||||
// **The merge gate's checks stop** (novox/hq ADR 0503 §3, §5; issue 1299): the checker judges every repository whose
|
||||
// trunk requires `mesh/check`. A repository whose trunk requires `mesh/repo-check` (hq), whatever else it requires
|
||||
// (ADR 0513 §4), has its own merge-check.sh asked of build-agent, and said with that layer alone; any other is asked
|
||||
// nothing and said nothing. What the forge's protection requires is read, never listed (lead decision L11).
|
||||
// Mesh-config is still told what the head declares (ADR 0502), whatever judges it.
|
||||
this := pullAtOf(p)
|
||||
if this.Truncated {
|
||||
this.Paths, this.Truncated = truncatedPaths(entries, p.Owner, p.Repo, scope.Modules), false
|
||||
@@ -201,29 +174,65 @@ func (f following) pullUpdated(ctx context.Context, p link.PullUpdated, w io.Wri
|
||||
if scope.gated() || holdsModules(entries, p.Owner, p.Repo) {
|
||||
tellPullDeclares(ctx, w, []pullAt{this})
|
||||
}
|
||||
required, err := requiredStatusesOf(ctx, p.Owner, p.Repo, p.Base)
|
||||
switch {
|
||||
case !scope.gated() && !scope.Mesh:
|
||||
fmt.Fprintf(w, "%s/%s#%d (%.8s): %s, and the repository is not the mesh's: said, nothing run\n",
|
||||
p.Owner, p.Repo, p.Number, p.Commit, noModuleTouched)
|
||||
case err != nil:
|
||||
fmt.Fprintf(w, "%s/%s#%d (%.8s): what %s's protection requires cannot be read, so nothing is asked and nothing is "+
|
||||
"said: %v\n", p.Owner, p.Repo, p.Number, p.Commit, p.Base, err)
|
||||
return "", nil
|
||||
case !asksRepoCheck(required):
|
||||
fmt.Fprintf(w, "%s/%s#%d (%.8s): %s does not require %s, and %s; nothing is asked\n", p.Owner, p.Repo, p.Number,
|
||||
p.Commit, p.Base, sdkdelivery.StatusContextRepoCheck, checkerWords(required))
|
||||
return "", nil
|
||||
}
|
||||
return f.repoCheckAlone(ctx, w, p, scope, entries)
|
||||
}
|
||||
|
||||
// asksRepoCheck says what a trunk's protection has the controller ask: its repository's own merge-check.sh whenever the
|
||||
// trunk requires `mesh/repo-check`, whatever else it requires (ADR 0513 §4: a trunk requiring both has the checker judge
|
||||
// `mesh/check` and the repository check asked beside it), and nothing otherwise.
|
||||
func asksRepoCheck(required []string) bool {
|
||||
return slices.Contains(required, sdkdelivery.StatusContextRepoCheck)
|
||||
}
|
||||
|
||||
// checkerWords says who judges a trunk that requires no `mesh/repo-check`.
|
||||
func checkerWords(required []string) string {
|
||||
if slices.Contains(required, sdkdelivery.StatusContextCheck) {
|
||||
return "the checker judges its " + sdkdelivery.StatusContextCheck
|
||||
}
|
||||
return "does not require " + sdkdelivery.StatusContextCheck + " either, so nothing is said"
|
||||
}
|
||||
|
||||
// repoCheckAlone asks build-agent for a repository's own merge-check.sh alone, for a pull request of a repository whose
|
||||
// trunk requires `mesh/repo-check`, whether or not it also requires `mesh/check` (ADR 0503 §5, ADR 0513 §4): no gate, no module built, its verdict said with the
|
||||
// repo-check layer alone.
|
||||
func (f following) repoCheckAlone(ctx context.Context, w io.Writer, p link.PullUpdated, scope checkScope,
|
||||
entries []inventory.Entry) (string, error) {
|
||||
inv := f.open.inventory
|
||||
plan := changePlanOf(p.Owner+"/"+p.Repo, p.Base, p.Commit, scope.Reach, entries, func(module string) (inventory.Upgrade, bool) {
|
||||
u, err := inv.UpgradeOf(ctx, module)
|
||||
return u, err == nil
|
||||
})
|
||||
fmt.Fprint(w, planText(plan))
|
||||
direct := directVerdict(p, plan)
|
||||
switch {
|
||||
case !scope.Mesh:
|
||||
direct.RepoCheck = &link.CheckLayer{Verdict: "warning", Summary: "the repository is not the mesh's: nothing run"}
|
||||
fmt.Fprintf(w, "%s/%s#%d (%.8s): the repository is not the mesh's: said, nothing run\n", p.Owner, p.Repo, p.Number,
|
||||
p.Commit)
|
||||
sayChecked(ctx, w, direct)
|
||||
return "", nil
|
||||
case !scope.gated() && p.MergeCheckSaid && !p.MergeCheck:
|
||||
case p.MergeCheckSaid && !p.MergeCheck:
|
||||
direct.RepoCheck = &link.CheckLayer{Verdict: "warning", Summary: noMergeCheck}
|
||||
fmt.Fprintf(w, "%s/%s#%d (%.8s): %s; %s\n", p.Owner, p.Repo, p.Number, p.Commit, noModuleTouched, noMergeCheck)
|
||||
fmt.Fprintf(w, "%s/%s#%d (%.8s): %s\n", p.Owner, p.Repo, p.Number, p.Commit, noMergeCheck)
|
||||
sayChecked(ctx, w, direct)
|
||||
return "", nil
|
||||
}
|
||||
request, err := checkRequestFor(ctx, f.open, p, scope, entries)
|
||||
request, err := checkRequestFor(ctx, f.open, p, checkScope{Mesh: true, From: scope.From}, entries)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
request.Check.Plan = &plan
|
||||
request.Check.Composed = composed
|
||||
if !composed {
|
||||
carryHeld(ctx, w, &request, []pullAt{this})
|
||||
}
|
||||
// **The group's other open pull requests are read beside it** (novox/hq ADR 0382 §1).
|
||||
siblingsFrom(&request, p, entries, lines, groupsErr)
|
||||
seat := buildSeatHeld(ctx)
|
||||
ask, err := askOverOn(seat)
|
||||
if err != nil {
|
||||
@@ -233,23 +242,42 @@ func (f following) pullUpdated(ctx context.Context, p link.PullUpdated, w io.Wri
|
||||
if err := ask.Ask(ctx, request); err != nil {
|
||||
return "", err
|
||||
}
|
||||
// **A newer request replaces the one still waiting, and a newer key stops the one running** (novox/hq ADR 0406 §3, §4,
|
||||
// issue 974): the check queue keeps one request a pull request, and a running check watches its key. Nothing is asked
|
||||
// of the forge here (issue 726).
|
||||
what := "its own merge-check.sh alone: " + noModuleTouched
|
||||
if composed {
|
||||
what = "its own merge-check.sh alone: the gate is its group's composed check's"
|
||||
}
|
||||
if scope.gated() {
|
||||
what = fmt.Sprintf("the gate over %s (a merge would build %d module(s) in %d tier(s))",
|
||||
strings.Join(append(append([]string{}, scope.Modules...), prefixedAll("new:", scope.New)...), ", "),
|
||||
scope.Width, scope.Tiers)
|
||||
}
|
||||
fmt.Fprintf(w, "%s/%s#%d (%.8s): asked %s to check it before it merges — %s — as %s\n", p.Owner, p.Repo, p.Number,
|
||||
p.Commit, seat, what, request.ID)
|
||||
fmt.Fprintf(w, "%s/%s#%d (%.8s): asked %s to run its own merge-check.sh alone, as %s\n", p.Owner, p.Repo, p.Number,
|
||||
p.Commit, seat, request.ID)
|
||||
return request.ID, nil
|
||||
}
|
||||
|
||||
// requiredStatusesOf is what the forge's protection of a repository's branch requires before a merge, as the forge
|
||||
// picks the rule it applies (the one named for the branch, else the first glob covering it). A variable so a test needs
|
||||
// no forge.
|
||||
var requiredStatusesOf = func(ctx context.Context, owner, repo, branch string) ([]string, error) {
|
||||
var rules struct {
|
||||
Rules []protectionRule `json:"rules"`
|
||||
}
|
||||
js, err := aBus()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer js.Close()
|
||||
raw, _ := json.Marshal(map[string]any{"owner": owner, "repo": repo})
|
||||
answer, err := link.Ask(ctx, link.OverNATS{Conn: js.Conn()}, "gitea", "gitea_branch_protection_get", raw, 30*time.Second)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if answer.Error != "" {
|
||||
return nil, fmt.Errorf("gitea.gitea_branch_protection_get: %s", answer.Error)
|
||||
}
|
||||
if err := json.Unmarshal(answer.Result, &rules); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
r := appliedRule(rules.Rules, branch)
|
||||
if r == nil {
|
||||
// empty-on-error: not an error; no rule covers the branch, so it requires nothing, which the caller says
|
||||
return nil, nil
|
||||
}
|
||||
return r.RequiredStatuses, nil
|
||||
}
|
||||
|
||||
// composedWords is what a group member's own verdict says of its gate layer (novox/hq ADR 0403 clause 1).
|
||||
const composedWords = "judged by its group's composed check"
|
||||
|
||||
@@ -378,15 +406,6 @@ func trunkOf(s inventory.Source) string {
|
||||
return "main"
|
||||
}
|
||||
|
||||
// sourceName is the name of a recorded repository, a path on the git seat or a URL: novox/mesh-host → mesh-host.
|
||||
func sourceName(repository string) string {
|
||||
parts := strings.Split(strings.Trim(strings.TrimSuffix(repository, ".git"), "/"), "/")
|
||||
if len(parts) < 2 {
|
||||
return ""
|
||||
}
|
||||
return parts[len(parts)-1]
|
||||
}
|
||||
|
||||
// namedForTheCheck are the builds earlier checks of the same pull request at the same head named, read once for the
|
||||
// check (novox/hq issue 703): asked again because only the trunk moved, the check keeps what they built of a key
|
||||
// still the same. Another pull request's build, or one of an earlier head, is never kept. None when they cannot be
|
||||
@@ -676,6 +695,9 @@ func prefixedAll(prefix string, items []string) []string {
|
||||
|
||||
// sayChecked says a merge check's verdict on the bus, where the forge's holder hears it.
|
||||
func sayChecked(ctx context.Context, w io.Writer, c link.Checked) {
|
||||
// **No merge-gate layer from R4e, for any repository** (ADR 0503 §3, §5; issue 1299): the forge's holder writes
|
||||
// `mesh/repo-check` from the repo-check layer alone, and the checker owes `mesh/check`.
|
||||
c.Gate = nil
|
||||
repo := "none"
|
||||
if c.RepoCheck != nil {
|
||||
repo = strings.ToUpper(c.RepoCheck.Verdict) + " — " + c.RepoCheck.Summary
|
||||
@@ -684,8 +706,8 @@ func sayChecked(ctx context.Context, w io.Writer, c link.Checked) {
|
||||
if c.Trunk != "" {
|
||||
judged = " merged into trunk " + c.Trunk
|
||||
}
|
||||
fmt.Fprintf(w, "%s: %s/%s#%d at %.8s%s checked on %s: gate %s — %s; repository %s\n", c.ID, c.Owner, c.Repo, c.Number,
|
||||
c.Commit, judged, orSomewhere(c.On), strings.ToUpper(c.Verdict), c.Summary, repo)
|
||||
fmt.Fprintf(w, "%s: %s/%s#%d at %.8s%s checked on %s: repository %s\n", c.ID, c.Owner, c.Repo, c.Number,
|
||||
c.Commit, judged, orSomewhere(c.On), repo)
|
||||
if checkEvents == nil {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -1,97 +0,0 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// Acceptance 5 of milestone 2 (novox/hq ADR 0406 §3, issue 974), live, with the store and a real bus: two merges into one
|
||||
// repository within a minute, with two open pull requests, leave each pull request with exactly one queued check, by the
|
||||
// stream's rule. Each merge moves the trunk, and mesh-delivery asks the controller to check both open pull requests again
|
||||
// against it (`delivery check`, which is the controller's own path for a head announced); nothing moves a request, nothing
|
||||
// asks the forge, and the queue is read back whole.
|
||||
func TestTwoMergesWithinAMinuteLeaveEachOpenPullRequestOneQueuedCheck(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
js, _ := aBuildQueue(t)
|
||||
servingBus.Store(js)
|
||||
t.Cleanup(func() { servingBus.Store(nil) })
|
||||
holdTheDeliverySeat(t, open)
|
||||
was := askDeliveryOwner
|
||||
t.Cleanup(func() { askDeliveryOwner = was })
|
||||
askDeliveryOwner = func(_ context.Context, _ *nats.Conn, verb string, _ map[string]any) (json.RawMessage, error) {
|
||||
if verb != "groups" {
|
||||
t.Fatalf("asked the owner %s", verb)
|
||||
}
|
||||
return json.RawMessage(`[]`), nil
|
||||
}
|
||||
// The repository the two pull requests are into, as a module the mesh builds from it.
|
||||
if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: "1"},
|
||||
inventory.Source{Repository: "novox/mesh-controller", Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
heads := map[int]string{7: "7777777777777777777777777777777777777777", 8: "8888888888888888888888888888888888888888"}
|
||||
check := func(number int, trunk string) {
|
||||
t.Helper()
|
||||
p := orderMember{ID: fmt.Sprintf("novox/mesh-controller@%.12s", heads[number]), Repository: "novox/mesh-controller",
|
||||
Base: "main", Head: heads[number], Number: number, Paths: []string{"main.go"}, ModuleDirsSaid: true,
|
||||
BaseSHA: trunk, Ready: number == 8}.pull()
|
||||
p.MergeCheck, p.MergeCheckSaid = true, true
|
||||
if err := (following{open: open, oneShot: true}).PullUpdated(ctx, p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// Announced against the trunk as it stands, then asked again after each of two merges.
|
||||
for _, trunk := range []string{"1111111111111111111111111111111111111111", "2222222222222222222222222222222222222222",
|
||||
"3333333333333333333333333333333333333333"} {
|
||||
check(7, trunk)
|
||||
check(8, trunk)
|
||||
}
|
||||
|
||||
q, err := link.ReadQueue(ctx, js, link.TheBuildMachine)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
seen := map[int]string{}
|
||||
for _, a := range q.Asks {
|
||||
c := a.Request.Check
|
||||
if c == nil {
|
||||
t.Fatalf("%s is no check", a.ID)
|
||||
}
|
||||
if _, twice := seen[c.Number]; twice {
|
||||
t.Errorf("pull request %d has two queued checks", c.Number)
|
||||
}
|
||||
seen[c.Number] = c.Trunk
|
||||
if a.State != link.AskWaiting {
|
||||
t.Errorf("%s is %s", a.ID, a.State)
|
||||
}
|
||||
if want := "3333333333333333333333333333333333333333"; c.Trunk != want {
|
||||
t.Errorf("pull request %d is queued against %.8s; want the newest trunk %.8s", c.Number, c.Trunk, want)
|
||||
}
|
||||
}
|
||||
if len(seen) != 2 {
|
||||
t.Fatalf("queued checks for %v; want one for each of the two open pull requests", seen)
|
||||
}
|
||||
// By the streams' own count: the ready one on its stream, the other on its.
|
||||
for stream, want := range map[string]uint64{broker.ChecksReadyStream: 1, broker.ChecksOtherStream: 1} {
|
||||
info, err := js.Context().StreamInfo(stream)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if info.State.Msgs != want {
|
||||
t.Errorf("%s holds %d request(s); want %d", stream, info.State.Msgs, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// C1 (ADR 0503 §3, §5; issue 1299, lead decision L11), with the store and a real build queue: a pull request of a
|
||||
// repository whose trunk requires mesh/check asks build-agent nothing and says nothing; one whose trunk requires
|
||||
// mesh/repo-check alone (hq) has its own merge-check.sh asked, and every verdict said names no gate layer; a protection
|
||||
// that cannot be read asks and says nothing.
|
||||
func TestThePullRequestChecksAreAskedOnlyForARepositoryTheCheckerDoesNotJudge(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
js, _ := aBuildQueue(t)
|
||||
servingBus.Store(js)
|
||||
t.Cleanup(func() { servingBus.Store(nil) })
|
||||
said := &saidOnTheBus{}
|
||||
checkEvents = said
|
||||
t.Cleanup(func() { checkEvents = nil })
|
||||
if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: "1"},
|
||||
inventory.Source{Repository: "novox/mesh-controller", Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// hq is cloned as a module built from it is, from its URL: no forge is held in this test.
|
||||
if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "hq-docs", Version: "1"},
|
||||
inventory.Source{Repository: "http://forge.invalid/novox/hq.git", Ref: "main", Path: "docs", BuiltFrom: "c0",
|
||||
Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
required := map[string][]string{"novox/mesh-controller": {"mesh/check"}, "novox/hq": {"mesh/repo-check"}}
|
||||
was := requiredStatusesOf
|
||||
requiredStatusesOf = func(_ context.Context, owner, repo, _ string) ([]string, error) {
|
||||
if r, ok := required[owner+"/"+repo]; ok {
|
||||
return r, nil
|
||||
}
|
||||
return nil, errors.New("the forge did not answer")
|
||||
}
|
||||
t.Cleanup(func() { requiredStatusesOf = was })
|
||||
queued := func() int {
|
||||
q, err := link.ReadQueue(ctx, js, link.TheBuildMachine)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return len(q.Asks)
|
||||
}
|
||||
pull := func(repo string, number int, mergeCheck bool) link.PullUpdated {
|
||||
p := link.PullUpdated{Owner: "novox", Repo: repo, Number: number, Base: "main", Commit: "abcdefabcdefabcdefabcdefabcdefabcdefabcd",
|
||||
BaseSHA: "1111111111111111111111111111111111111111", Paths: []string{"main.go"}, ModuleDirsSaid: true}
|
||||
p.MergeCheck, p.MergeCheckSaid = mergeCheck, true
|
||||
return p
|
||||
}
|
||||
f := following{open: open, oneShot: true}
|
||||
|
||||
if err := f.PullUpdated(ctx, pull("mesh-controller", 7, true)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n := queued(); n != 0 || len(said.events) != 0 {
|
||||
t.Fatalf("0503 §3: a repository the checker judges asked %d check(s) and said %v", n, said.events)
|
||||
}
|
||||
if err := f.PullUpdated(ctx, pull("unread", 3, true)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n := queued(); n != 0 || len(said.events) != 0 {
|
||||
t.Fatalf("L11: a protection that cannot be read asked %d check(s) and said %v", n, said.events)
|
||||
}
|
||||
if err := f.PullUpdated(ctx, pull("hq", 9, true)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
q, err := link.ReadQueue(ctx, js, link.TheBuildMachine)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(q.Asks) != 1 || q.Asks[0].Request.Check == nil || len(q.Asks[0].Request.Check.Modules) != 0 ||
|
||||
q.Asks[0].Request.Check.Repo != "hq" {
|
||||
t.Fatalf("0503 §5: hq's repository check is not the one ask: %+v", q.Asks)
|
||||
}
|
||||
// A head with no merge-check.sh is said at once, with the repo-check layer alone.
|
||||
if err := f.PullUpdated(ctx, pull("hq", 10, false)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(said.events) != 1 {
|
||||
t.Fatalf("hq's head with no merge-check.sh was said %d time(s)", len(said.events))
|
||||
}
|
||||
var c link.Checked
|
||||
if err := json.Unmarshal(said.bodies[0], &c); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if c.Gate != nil || c.RepoCheck == nil || c.RepoCheck.Verdict != "warning" {
|
||||
t.Fatalf("0503 §5: the verdict said names a gate layer, or no repo-check layer: %+v", c)
|
||||
}
|
||||
// A trunk requiring both (hq, ADR 0513 §4): the checker judges mesh/check, and the repository check is still asked.
|
||||
required["novox/hq"] = []string{"mesh/check", "mesh/repo-check"}
|
||||
if err := f.PullUpdated(ctx, pull("hq", 11, true)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if q, err := link.ReadQueue(ctx, js, link.TheBuildMachine); err != nil || len(q.Asks) != 2 ||
|
||||
q.Asks[1].Request.Check == nil || q.Asks[1].Request.Check.Repo != "hq" {
|
||||
t.Fatalf("0513 §4: a trunk requiring both had no repository check asked: %+v %v", q.Asks, err)
|
||||
}
|
||||
}
|
||||
@@ -316,3 +316,21 @@ func TestARecheckedHeadsDirectVerdictNamesItsTrunk(t *testing.T) {
|
||||
|
||||
// A check stopped by its pull request's key moving on says no verdict (issue 974, review 2): an error said for it would name the
|
||||
// live head, and the newer request is the one that judges. A check that failed for any other reason still says its error.
|
||||
|
||||
// C2 (ADR 0513 §4, ADR 0503 §5; issue 1299): a trunk requiring mesh/repo-check has its repository check asked, whether or
|
||||
// not it also requires mesh/check (hq requires both); a trunk requiring mesh/check alone, or neither, is asked nothing.
|
||||
func TestARepositoryCheckIsAskedWheneverTheTrunkRequiresIt(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
required []string
|
||||
asks bool
|
||||
}{
|
||||
{[]string{"mesh/check", "mesh/repo-check"}, true},
|
||||
{[]string{"mesh/repo-check"}, true},
|
||||
{[]string{"mesh/check"}, false},
|
||||
{nil, false},
|
||||
} {
|
||||
if got := asksRepoCheck(c.required); got != c.asks {
|
||||
t.Fatalf("a trunk requiring %v asks its repository check: %v, want %v", c.required, got, c.asks)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,68 +0,0 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A member of a group whose gate is the composed check's is asked of the build seat with no module: its repository's
|
||||
// check alone (novox/hq ADR 0403 clause 1). With no word from mesh-delivery, or one that does not say the gate is
|
||||
// composed, it is checked alone with its gate (clause 6).
|
||||
|
||||
func askedForMember(t *testing.T, answer []groupsLine, answerErr error) link.CheckRequest {
|
||||
t.Helper()
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
js, _ := aBuildQueue(t)
|
||||
servingBus.Store(js)
|
||||
t.Cleanup(func() { servingBus.Store(nil) })
|
||||
if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: "1"},
|
||||
inventory.Source{Repository: "novox/mesh-controller", Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
was := groupsAsked
|
||||
groupsAsked = func(context.Context, bool) ([]groupsLine, error) { return answer, answerErr }
|
||||
t.Cleanup(func() { groupsAsked = was })
|
||||
p := link.PullUpdated{Owner: "novox", Repo: "mesh-controller", Number: 346, Base: "main", Head: "feat/x", Commit: headController,
|
||||
Paths: []string{"main.go"}, ModuleDirsSaid: true, MergeCheck: true, MergeCheckSaid: true}
|
||||
if err := (following{open: open, oneShot: true}).PullUpdated(ctx, p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
q, err := link.ReadQueue(ctx, js, link.TheBuildMachine)
|
||||
if err != nil || len(q.Asks) != 1 {
|
||||
t.Fatalf("asks %d: %v", len(q.Asks), err)
|
||||
}
|
||||
return *q.Asks[0].Request.Check
|
||||
}
|
||||
|
||||
func TestAComposedMembersCheckIsItsRepositoryChecksAlone(t *testing.T) {
|
||||
line := group("feat/x", member("novox/mesh-controller", 346, headController, "main", true), member("novox/mesh-catalog", 350, headHost, "main", true))
|
||||
line.ComposedGate = true
|
||||
c := askedForMember(t, []groupsLine{line}, nil)
|
||||
if len(c.Modules) != 0 || len(c.New) != 0 || len(c.Dependents) != 0 || !c.Composed {
|
||||
t.Fatalf("the member's check is asked as %+v; want no module and composed", c)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAMemberOfAGroupOfAnOlderMeshDeliveryKeepsItsOwnGate(t *testing.T) {
|
||||
line := group("feat/x", member("novox/mesh-controller", 346, headController, "main", true), member("novox/mesh-catalog", 350, headHost, "main", true))
|
||||
c := askedForMember(t, []groupsLine{line}, nil)
|
||||
if !strings.Contains(strings.Join(c.Modules, ","), "mesh-controller") || c.Composed {
|
||||
t.Fatalf("an older mesh-delivery's member is asked as %+v; want its own gate", c)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAMemberWhoseGroupCannotBeLearnedIsCheckedAloneWithItsGate(t *testing.T) {
|
||||
c := askedForMember(t, nil, context.DeadlineExceeded)
|
||||
if !strings.Contains(strings.Join(c.Modules, ","), "mesh-controller") || c.Composed ||
|
||||
!strings.Contains(c.SiblingsUnknown, "reads no siblings") {
|
||||
t.Fatalf("the member is asked as %+v; want its gate and why its siblings are unknown", c)
|
||||
}
|
||||
}
|
||||
@@ -1,50 +1,21 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A delivery group's gate is the composed check's, and its members' own checks run their repository's check alone
|
||||
// (novox/hq ADR 0403 clauses 1 and 7). Pure: no store, no bus.
|
||||
|
||||
// A member of a group whose line says composed_gate is a composed member; one whose group says nothing of it (an older
|
||||
// mesh-delivery), one of a group of a single pull request and a lone pull request are not.
|
||||
func TestAMemberOfAComposedGroupIsKnownByTheGroupsLine(t *testing.T) {
|
||||
two := func(composed bool) groupsLine {
|
||||
l := group("feat/x", member("novox/mesh-host", 114, headHost, "main", true), member("novox/mesh-controller", 346, headController, "main", true))
|
||||
l.ComposedGate = composed
|
||||
return l
|
||||
}
|
||||
p := link.PullUpdated{Owner: "novox", Repo: "mesh-host", Number: 114, Base: "main", Head: "feat/x", Commit: headHost}
|
||||
recheck := link.PullUpdated{Owner: "novox", Repo: "mesh-host", Number: 114, Base: "main", Commit: headHost}
|
||||
for name, c := range map[string]struct {
|
||||
p link.PullUpdated
|
||||
lines []groupsLine
|
||||
want bool
|
||||
}{
|
||||
"composed": {p, []groupsLine{two(true)}, true},
|
||||
"a recheck, by its number": {recheck, []groupsLine{two(true)}, true},
|
||||
"an older mesh-delivery": {p, []groupsLine{two(false)}, false},
|
||||
"another group": {p, []groupsLine{group("feat/y", member("novox/mesh-host", 1, headHost, "main", true))}, false},
|
||||
"no group": {p, nil, false},
|
||||
} {
|
||||
if got := composedMember(c.p, c.lines); got != c.want {
|
||||
t.Errorf("%s: composed %v, want %v", name, got, c.want)
|
||||
}
|
||||
}
|
||||
alone := group("feat/x", member("novox/mesh-host", 114, headHost, "main", true))
|
||||
alone.ComposedGate = true
|
||||
if composedMember(p, []groupsLine{alone}) {
|
||||
t.Error("a group of a single pull request is composed")
|
||||
}
|
||||
}
|
||||
// The heads of a group's members, by repository.
|
||||
const (
|
||||
headHost = "1111111111111111111111111111111111111111"
|
||||
headController = "2222222222222222222222222222222222222222"
|
||||
)
|
||||
|
||||
// A composed member's verdict never says its gate passed: the gate layer, and the verdict that is the gate's, read
|
||||
// "composed"; a failure or an error stays what it was.
|
||||
@@ -88,49 +59,3 @@ func TestTheComposedVerdictCarriesEveryMember(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Members whose manifests order their modules in a cycle fail the composed check, naming every module and edge, and no
|
||||
// ask is made of the build seat.
|
||||
func TestAGroupInACycleFailsItsComposedCheckWithoutABuild(t *testing.T) {
|
||||
members := []orderMember{
|
||||
{ID: "novox/x@" + headHost[:12], Repository: "novox/x", Base: "main", Head: headHost, Number: 7, Paths: []string{"module.json"}},
|
||||
{ID: "novox/y@" + headController[:12], Repository: "novox/y", Base: "main", Head: headController, Number: 9, Paths: []string{"module.json"}},
|
||||
}
|
||||
reach := map[string]orderReach{members[0].ID: {Moved: []string{"mx"}}, members[1].ID: {Moved: []string{"my"}}}
|
||||
was, wasParent := manifestAt, parentOf
|
||||
t.Cleanup(func() { manifestAt, parentOf = was, wasParent })
|
||||
parentOf = func(context.Context, string, string, string) (string, error) { return "", nil }
|
||||
manifestAt = func(_ context.Context, _, repo, _, ref string) (catalogue.Manifest, bool, error) {
|
||||
if ref == "main" {
|
||||
return catalogue.Manifest{}, false, nil // what the pull requests merge into holds no such manifest
|
||||
}
|
||||
if repo == "x" {
|
||||
return catalogue.Manifest{Module: "mx", DefinesSeats: []catalogue.SeatDeclaration{{Name: "s", Serves: []catalogue.Verb{{Name: "a"}}}},
|
||||
Invokes: []string{"seat:s.b"}}, true, nil
|
||||
}
|
||||
return catalogue.Manifest{Module: "my", DefinesSeats: []catalogue.SeatDeclaration{{Name: "s", Serves: []catalogue.Verb{{Name: "b"}}}},
|
||||
Invokes: []string{"seat:s.a"}}, true, nil
|
||||
}
|
||||
c, cyclic, err := cycleVerdict(context.Background(), "feat/cycle", members, reach, []inventory.Edge(nil), nil)
|
||||
if err != nil || !cyclic {
|
||||
t.Fatalf("no cycle found: %v", err)
|
||||
}
|
||||
if c.Verdict != "fail" || c.Group != "feat/cycle" || len(c.Members) != 2 || c.Gate == nil || c.Gate.Verdict != "fail" {
|
||||
t.Fatalf("the verdict is %+v", c)
|
||||
}
|
||||
for _, w := range []string{"mx", "my", "s.a", "s.b"} {
|
||||
if !strings.Contains(c.Summary, w) {
|
||||
t.Errorf("the cycle does not name %s: %s", w, c.Summary)
|
||||
}
|
||||
}
|
||||
// Members with no edge between them are no cycle.
|
||||
manifestsReadMu.Lock()
|
||||
clear(manifestsRead)
|
||||
manifestsReadMu.Unlock()
|
||||
manifestAt = func(context.Context, string, string, string, string) (catalogue.Manifest, bool, error) {
|
||||
return catalogue.Manifest{}, false, nil
|
||||
}
|
||||
if _, cyclic, err := cycleVerdict(context.Background(), "feat/cycle", members, reach, nil, nil); err != nil || cyclic {
|
||||
t.Fatalf("a cycle where there is none: %v %v", cyclic, err)
|
||||
}
|
||||
}
|
||||
|
||||
+15
-159
@@ -815,16 +815,14 @@ func deliveryPlanDocument(m orderMember, entries []inventory.Entry, read map[str
|
||||
// delivery walks [-n 50] [--plan <id>]
|
||||
func deliveryCommand(ctx context.Context, args []string) error {
|
||||
if len(args) == 0 {
|
||||
return errors.New("delivery plan|order|check|go|stop|walks")
|
||||
return errors.New("delivery order|check|go|stop|walks")
|
||||
}
|
||||
sub, rest := args[0], args[1:]
|
||||
if sub == "plan" {
|
||||
// Removed (ADR 0450 §8; issue 1318): refused by name before any flag is read, naming its replacement.
|
||||
return removedByName("delivery-plan")
|
||||
}
|
||||
set := flag.NewFlagSet("delivery "+sub, flag.ContinueOnError)
|
||||
repository := set.String("repository", "", "owner/repository")
|
||||
base := set.String("base", "main", "the branch it merges into")
|
||||
head := set.String("head", "", "the commit at hand")
|
||||
paths := set.String("paths", "", "the files it changes, comma-separated")
|
||||
moduleDirs := set.String("module-dirs", "", "the directories holding a module.json at the head, comma-separated")
|
||||
removed := set.String("removed", "", "the files it deletes, comma-separated")
|
||||
membersJSON := set.String("members", "", "a group's members, as JSON")
|
||||
membersFile := set.String("members-file", "", "a file of a group's members, as JSON: the same as --members, for a list "+
|
||||
"too long for an argument (novox/hq issue 1272)")
|
||||
@@ -872,19 +870,6 @@ func deliveryCommand(ctx context.Context, args []string) error {
|
||||
return enc.Encode(v)
|
||||
}
|
||||
switch sub {
|
||||
case "plan":
|
||||
if *repository == "" || *head == "" && *paths == "" {
|
||||
return errors.New("delivery plan --repository owner/repo --head <commit> --paths a,b")
|
||||
}
|
||||
m := changeMember(*repository, *base, *head, splitList(*paths), splitList(*removed), moduleDirsOf(*moduleDirs))
|
||||
entries, read, edges, err := theGraph(ctx, inv)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return answer(deliveryPlanDocument(m, entries, read, edges, func(module string) (inventory.Upgrade, bool) {
|
||||
u, err := inv.UpgradeOf(ctx, module)
|
||||
return u, err == nil
|
||||
}))
|
||||
case "order":
|
||||
if len(members) == 0 {
|
||||
return errors.New("delivery order --members <json>: a group has members")
|
||||
@@ -914,11 +899,9 @@ func deliveryCommand(ctx context.Context, args []string) error {
|
||||
if *group == "" || len(members) < 2 {
|
||||
return errors.New("delivery check --group <id> --members <json> (two heads or more), or --members <one head>")
|
||||
}
|
||||
id, err := askGroupCheck(ctx, open, *group, members)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return answer(map[string]any{"asked": id, "group": *group})
|
||||
// **The group's composed check is the checker's** (novox/hq ADR 0503 §3; issue 1299, C2): nothing is asked of
|
||||
// build-agent, and the answer says who judges it.
|
||||
return answer(groupCheckAnswer(*group))
|
||||
case "go":
|
||||
if len(positionals) != 1 {
|
||||
return errors.New("delivery go <plan> [--by <who>] [--why <text>]")
|
||||
@@ -1093,38 +1076,6 @@ func theGraph(ctx context.Context, inv *inventory.Inventory) ([]inventory.Entry,
|
||||
return entries, read, edges, nil
|
||||
}
|
||||
|
||||
// cycleVerdict is the failing verdict of a group whose modules are ordered in a cycle, and whether there is one: the
|
||||
// group's composed check, said without a build. The manifests' change is read from the forge; an answer it cannot give
|
||||
// is an error, never read as no cycle.
|
||||
func cycleVerdict(ctx context.Context, group string, members []orderMember, reach map[string]orderReach,
|
||||
edges []inventory.Edge, entries []inventory.Entry) (link.Checked, bool, error) {
|
||||
changes, err := changesOf(ctx, changeSourcesOf(members, entries, reach))
|
||||
if err != nil {
|
||||
return link.Checked{}, false, fmt.Errorf("what the group's members say of their manifests cannot be read: %w", err)
|
||||
}
|
||||
var moved []string
|
||||
for _, m := range members {
|
||||
moved = appendNew(moved, reach[m.ID].Moved...)
|
||||
}
|
||||
said := orderEdges(moved, changes, edges).CycleSaid()
|
||||
if said == "" {
|
||||
return link.Checked{}, false, nil
|
||||
}
|
||||
first := members[0].pull()
|
||||
c := link.Checked{ID: link.NewBuildID(time.Now()), Owner: first.Owner, Repo: first.Repo, Number: first.Number,
|
||||
Commit: first.Commit, Verdict: "fail", Summary: said, Report: said, Group: group,
|
||||
Gate: &link.CheckLayer{Verdict: "fail", Summary: said}}
|
||||
for _, m := range members {
|
||||
if group == "" {
|
||||
break // a lone pull request's verdict names no group and no member
|
||||
}
|
||||
p := m.pull()
|
||||
c.Members = append(c.Members, link.CheckedMember{Owner: p.Owner, Repo: p.Repo, Number: p.Number, Commit: p.Commit,
|
||||
Trunk: twelve(p.BaseSHA)})
|
||||
}
|
||||
return c, true, nil
|
||||
}
|
||||
|
||||
// twelve is a commit as twelve characters.
|
||||
func twelve(c string) string {
|
||||
if len(c) > 12 {
|
||||
@@ -1133,99 +1084,6 @@ func twelve(c string) string {
|
||||
return c
|
||||
}
|
||||
|
||||
// groupPrimary is the head a group's composed check is run from: the one moving the controller, which then
|
||||
// judges the group by itself; else the one moving the node-engine, whose validator judges; else the first.
|
||||
func groupPrimary(members []orderMember, reach map[string]orderReach) int {
|
||||
for _, want := range []string{"mesh-controller", "mesh-host"} {
|
||||
for i, m := range members {
|
||||
if slices.Contains(reach[m.ID].Moved, want) {
|
||||
return i
|
||||
}
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// askGroupCheck asks the build seat for a group's composed check: every head laid over the mesh in turn,
|
||||
// judged as one future state (novox/hq ADR 0239). The verdict comes back as `checked` with the group's id,
|
||||
// for mesh-delivery; the forge's holder sets no status from it.
|
||||
func askGroupCheck(ctx context.Context, open *stores, group string, members []orderMember) (string, error) {
|
||||
entries, read, edges, err := theGraph(ctx, open.inventory)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
reach := reachOfMembers(members, entries, read, edges)
|
||||
// **A cycle of edges fails the group's composed check, naming every module and edge** (novox/hq ADR 0419 clause 6),
|
||||
// before the build seat is asked: no composition of the heads has an order to be sent in.
|
||||
if failing, cyclic, err := cycleVerdict(ctx, group, members, reach, edges, entries); err != nil {
|
||||
return "", err
|
||||
} else if cyclic {
|
||||
sayChecked(ctx, os.Stderr, failing)
|
||||
return failing.ID, nil
|
||||
}
|
||||
primary := groupPrimary(members, reach)
|
||||
p := members[primary].pull()
|
||||
scope := pullScope(p, entries, read, edges)
|
||||
// The gate runs over what any member moves; a judge from the primary alone.
|
||||
for _, m := range members {
|
||||
r := reach[m.ID]
|
||||
scope.Modules = appendNew(scope.Modules, r.Moved...)
|
||||
scope.Dependents = appendNew(scope.Dependents, r.Dependents...)
|
||||
}
|
||||
request, err := checkRequestFor(ctx, open, p, scope, entries)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
request.Check.Group = group
|
||||
// **The values held for the keys the members introduce are told and carried** (novox/hq ADR 0502, issue 1826).
|
||||
var pulls []pullAt
|
||||
for i, s := range changeSourcesOf(members, entries, reach) {
|
||||
pulls = append(pulls, pullAt{changeSource: s, Number: members[i].pull().Number})
|
||||
}
|
||||
withHeldConfig(ctx, os.Stderr, &request, pulls, true)
|
||||
// A group's check is queued as ready when any member's pull request is marked ready.
|
||||
request.Check.Ready = slices.ContainsFunc(members, func(m orderMember) bool { return m.Ready })
|
||||
world, err := restOfTheMesh(ctx, open, shelfOf(entries), "")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
for i, m := range members {
|
||||
if i == primary {
|
||||
continue
|
||||
}
|
||||
mp := m.pull()
|
||||
// As the mesh clones a module built from it; a repository no module is built from, from the forge.
|
||||
source := inventory.Source{Seat: gitSeat, Repository: mp.Owner + "/" + mp.Repo}
|
||||
for _, e := range entries {
|
||||
if !e.Provided && sameRepository(e.Source.Repository, link.SourceMoved{Owner: mp.Owner, Repo: mp.Repo}) {
|
||||
source = e.Source
|
||||
break
|
||||
}
|
||||
}
|
||||
url := source.Repository
|
||||
if source.Seat != "" {
|
||||
url, err = clonedFromSeat(world, source.Seat, source.Repository)
|
||||
}
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("%s cannot be cloned for the group's check: %w", m.ID, err)
|
||||
}
|
||||
request.Check.Members = append(request.Check.Members, link.GroupMember{Owner: mp.Owner, Repo: mp.Repo,
|
||||
Number: mp.Number, Repository: url, Ref: mp.Commit, Paths: mp.Paths, Base: mp.Base, Trunk: mp.BaseSHA})
|
||||
}
|
||||
seat := buildSeatHeld(ctx)
|
||||
ask, err := askOverOn(seat)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer ask.Close()
|
||||
if err := ask.Ask(ctx, request); err != nil {
|
||||
return "", err
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "group %s: asked %s to check %d head(s) composed together, as %s\n", group, seat,
|
||||
len(members), request.ID)
|
||||
return request.ID, nil
|
||||
}
|
||||
|
||||
// appendNew appends what is not there yet.
|
||||
func appendNew(to []string, items ...string) []string {
|
||||
for _, i := range items {
|
||||
@@ -1236,15 +1094,6 @@ func appendNew(to []string, items ...string) []string {
|
||||
return to
|
||||
}
|
||||
|
||||
// shelfOf is the catalogue's manifests by name.
|
||||
func shelfOf(entries []inventory.Entry) map[string]catalogue.Manifest {
|
||||
shelf := map[string]catalogue.Manifest{}
|
||||
for _, e := range entries {
|
||||
shelf[e.Manifest.Module] = e.Manifest
|
||||
}
|
||||
return shelf
|
||||
}
|
||||
|
||||
// sayPlanMoved says a walk kept in a new state as the controller's `plan-moved`, the plan whole: what the
|
||||
// delivery it walks reads its steps from. Never in the way of the walk: said beside it, and a save that could
|
||||
// not be said is logged — the delivery's owner, which also asks for the walks it follows, finds it by
|
||||
@@ -1408,3 +1257,10 @@ func recheckOneHead(member string, answer func(any) error, ask func() (string, e
|
||||
}
|
||||
return answer(recheckAnswer(asked, member))
|
||||
}
|
||||
|
||||
// groupCheckAnswer is what `delivery check` of a group answers from R4e: who judges it, and that nothing was asked
|
||||
// (novox/hq ADR 0503 §3; issue 1299, C2).
|
||||
func groupCheckAnswer(group string) map[string]any {
|
||||
return map[string]any{"asked": "", "group": group, "judged-by": checkerModule,
|
||||
"said": "the checker judges a delivery group's heads merged together (ADR 0503 §3): nothing was asked"}
|
||||
}
|
||||
|
||||
@@ -41,7 +41,7 @@ func TestSayCheckedWritesTheSameLineToItsWriter(t *testing.T) {
|
||||
var w bytes.Buffer
|
||||
sayChecked(context.Background(), &w, link.Checked{ID: "c1", Owner: "novox", Repo: "x", Number: 7, Commit: "0123456789abcdef",
|
||||
Verdict: "pass", Summary: "fine"})
|
||||
want := "c1: novox/x#7 at 01234567 checked on a machine that did not say which: gate PASS — fine; repository none\n"
|
||||
want := "c1: novox/x#7 at 01234567 checked on a machine that did not say which: repository none\n"
|
||||
if w.String() != want {
|
||||
t.Fatalf("sayChecked said %q; want %q", w.String(), want)
|
||||
}
|
||||
|
||||
@@ -181,32 +181,3 @@ func TestATruncatedChangeIsReadAsChangingTheManifestsOfWhatItMoves(t *testing.T)
|
||||
t.Fatalf("the member's change is read as %+v", src)
|
||||
}
|
||||
}
|
||||
|
||||
// A lone pull request's cycle fails its check, naming every module and edge, with no group and no members.
|
||||
func TestALonePullRequestInACycleFailsWithNoGroup(t *testing.T) {
|
||||
was, wasParent := manifestAt, parentOf
|
||||
t.Cleanup(func() { manifestAt, parentOf = was, wasParent })
|
||||
manifestsReadMu.Lock()
|
||||
clear(manifestsRead)
|
||||
manifestsReadMu.Unlock()
|
||||
const head = "3333333333333333333333333333333333333333"
|
||||
parentOf = func(context.Context, string, string, string) (string, error) { return "", nil }
|
||||
manifestAt = func(_ context.Context, _, _, file, ref string) (catalogue.Manifest, bool, error) {
|
||||
if ref == "main" {
|
||||
return catalogue.Manifest{}, false, nil
|
||||
}
|
||||
if file == "modules/x/module.json" {
|
||||
return catalogue.Manifest{Module: "mx", DefinesSeats: []catalogue.SeatDeclaration{{Name: "s", Serves: []catalogue.Verb{{Name: "a"}}}},
|
||||
Invokes: []string{"seat:s.b"}}, true, nil
|
||||
}
|
||||
return catalogue.Manifest{Module: "my", DefinesSeats: []catalogue.SeatDeclaration{{Name: "s", Serves: []catalogue.Verb{{Name: "b"}}}},
|
||||
Invokes: []string{"seat:s.a"}}, true, nil
|
||||
}
|
||||
m := orderMember{ID: "novox/r@" + head[:12], Repository: "novox/r", Base: "main", Head: head, Number: 5,
|
||||
Paths: []string{"modules/x/module.json", "modules/y/module.json"}}
|
||||
c, cyclic, err := cycleVerdict(context.Background(), "", []orderMember{m}, map[string]orderReach{m.ID: {Moved: []string{"mx", "my"}}}, nil, nil)
|
||||
if err != nil || !cyclic || c.Verdict != "fail" || c.Group != "" || len(c.Members) != 0 || !strings.Contains(c.Summary, "s.a") ||
|
||||
!strings.Contains(c.Summary, "my") {
|
||||
t.Fatalf("the verdict is %+v (cyclic %v, %v)", c, cyclic, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -138,8 +138,6 @@ func TestTheDeliveryVerbsComposeTheirCommands(t *testing.T) {
|
||||
{"delivery-order", map[string]any{"members": "[]"}, []string{"delivery", "order", "--members", "[]"}},
|
||||
{"delivery-check", map[string]any{"group": "feat/x", "members": "[]"},
|
||||
[]string{"delivery", "check", "--group", "feat/x", "--members", "[]"}},
|
||||
{"delivery-plan", map[string]any{"repository": "novox/a", "paths": "x", "head": "c0"},
|
||||
[]string{"delivery", "plan", "--repository", "novox/a", "--paths", "x", "--head", "c0"}},
|
||||
{"plans", map[string]any{"go": "plan-1", "why": "down"}, []string{"plans", "go", "plan-1", "--why", "down"}},
|
||||
} {
|
||||
got, err := argvFor(c.verb, c.args)
|
||||
|
||||
@@ -35,16 +35,6 @@ func TestAMergeOfABaseTakesWhatStandsOnItAlong(t *testing.T) {
|
||||
t.Fatalf("%s was rebuilt and stands on nothing that moved (%v)", n, names)
|
||||
}
|
||||
}
|
||||
// The dependents come in base order when the merge orders them: the runtime, then shop, then
|
||||
// the plugin that stands on shop.
|
||||
ordered := orderByBases(append([]inventory.Entry{entry("mesh-tools")}, got...), against)
|
||||
pos := map[string]int{}
|
||||
for i, e := range ordered {
|
||||
pos[e.Manifest.Module] = i
|
||||
}
|
||||
if !(pos["mesh-tools"] < pos["shop"] && pos["shop"] < pos["shop-plugin"]) {
|
||||
t.Fatalf("not in base order: %v", ordered)
|
||||
}
|
||||
// Nothing moved: nothing follows.
|
||||
if more := dependentsOf(nil, entries, against); len(more) != 0 {
|
||||
t.Fatalf("with nothing moved, %d module(s) were rebuilt", len(more))
|
||||
|
||||
@@ -1,84 +0,0 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/identity"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// aForgeAndABuildSeat gives a test what a real ask of a build needs: the forge holding the git seat, assigned to anchor, so
|
||||
// the clone URL composes; the identity store; the build seat's queue on a real bus; and the environment of the serving controller, which is
|
||||
// never the controller's terminal (a verb's, in effect: startedAtTheTerminal is false).
|
||||
func aForgeAndABuildSeat(t *testing.T, open *stores) *broker.JetStream {
|
||||
t.Helper()
|
||||
js, _ := aBuildQueue(t)
|
||||
// The ask signs what it sends with the controller's identity, as the serving controller does.
|
||||
identity.ForTest(t)
|
||||
forge, err := catalogue.ParseManifest([]byte(`{"module":"gitea","version":"1",
|
||||
"claims":[{"name":"git","scope":"mesh"}],"provides":[{"name":"git","scope":"mesh"}],
|
||||
"serves":{"git":{"scheme":"http","port":3000}}}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
register(t, open, forge)
|
||||
if _, err := assign(t.Context(), open, "anchor", "gitea"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Setenv(servedVar, "1")
|
||||
return js
|
||||
}
|
||||
|
||||
// 1146/AC9, through the real ask (askABuild, buildOneAsked, verbMayAsk) and a real bus: the first build of a module a
|
||||
// merge adds on the operator's Approve is queued on the build seat; the same ask without that warrant, and the warrant's
|
||||
// ask for a path the merge does not hold, are refused by the guard of a build through a verb.
|
||||
func TestTheApprovedFirstBuildPassesTheGuardOfAVerbAndIsQueued(t *testing.T) {
|
||||
const head, commit = "a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1", "0177ea7a00000000"
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
js := aForgeAndABuildSeat(t, open)
|
||||
|
||||
a := aDeliveryMergeOf(t, head, map[string]string{"novox/mesh-delivery@a1a1a1a1a1a1": head})
|
||||
wasKept, wasAsk := keptDeliveryMerge, routerAskOf
|
||||
keptDeliveryMerge = func(context.Context, string, string, int, string) (deliveryMerge, bool, error) {
|
||||
return a.merge, true, nil
|
||||
}
|
||||
routerAskOf = func(context.Context, catalogueReader, string, string) (*asks.Ask, *asks.Warrant, error) {
|
||||
return &a.ask, &a.warrant, nil
|
||||
}
|
||||
t.Cleanup(func() { keptDeliveryMerge, routerAskOf = wasKept, wasAsk })
|
||||
|
||||
// Without the warrant the guard refuses, whatever the repository holds.
|
||||
_, err := askABuild(ctx, buildSource{Repository: "novox/mesh-delivery", Seat: "git"}, "", "main")
|
||||
var policy *heldAtTheTerminal
|
||||
if !errors.As(err, &policy) {
|
||||
t.Fatalf("1146/AC9: a first build asked without the warrant was not refused: %v", err)
|
||||
}
|
||||
// And a path the merge does not hold is refused on the warrant.
|
||||
warranted := withFirstBuildWarrant(ctx, "novox/mesh-delivery", []string{""}, "the Approve")
|
||||
if _, err := askABuild(warranted, buildSource{Repository: "novox/mesh-delivery", Seat: "git"}, "other", "main"); !errors.As(err, &policy) {
|
||||
t.Fatalf("1146/AC9: the warrant covered a path the merge does not hold: %v", err)
|
||||
}
|
||||
|
||||
m := link.SourceMoved{Owner: "novox", Repo: "mesh-delivery", Base: "main", HeadSHA: head, Commit: commit, Number: 7,
|
||||
MergedAt: time.Now().Add(-time.Hour).UTC().Format(time.RFC3339Nano), Paths: []string{"module.json", "main.go"},
|
||||
ModuleDirs: []string{"."}, ModuleDirsSaid: true}
|
||||
if err := (following{open: open}).SourceMoved(ctx, m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
queue, err := link.ReadQueue(ctx, js, link.TheBuildMachine)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(queue.Asks) != 1 {
|
||||
t.Fatalf("1146/AC9: the build queue holds %+v, want the one first build", queue)
|
||||
}
|
||||
}
|
||||
@@ -1,29 +0,0 @@
|
||||
package main
|
||||
|
||||
import "testing"
|
||||
|
||||
// 1146/AC9: the warrant the controller checked covers exactly the repository on the git seat and the module paths the
|
||||
// merge holds; nothing else passes verbMayAsk on it.
|
||||
func TestTheFirstBuildWarrantCoversOnlyItsRepositoryAndItsModulePaths(t *testing.T) {
|
||||
ctx := withFirstBuildWarrant(t.Context(), "novox/mesh-delivery", []string{"", "view"}, "the Approve")
|
||||
w := firstBuildWarrantIn(ctx)
|
||||
for _, c := range []struct {
|
||||
source buildSource
|
||||
path string
|
||||
covers bool
|
||||
}{
|
||||
{buildSource{Repository: "novox/mesh-delivery", Seat: "git"}, "", true},
|
||||
{buildSource{Repository: "novox/mesh-delivery", Seat: "git"}, "/view/", true},
|
||||
{buildSource{Repository: "novox/mesh-delivery", Seat: "git"}, "other", false},
|
||||
{buildSource{Repository: "agent/mesh-delivery", Seat: "git"}, "", false},
|
||||
{buildSource{Repository: "novox/mesh-delivery"}, "", false},
|
||||
{buildSource{Repository: "novox/mesh-delivery", Seat: "elsewhere"}, "", false},
|
||||
} {
|
||||
if got := w.covers(c.source, c.path); got != c.covers {
|
||||
t.Errorf("1146/AC9: %v at %q covered = %v, want %v", c.source, c.path, got, c.covers)
|
||||
}
|
||||
}
|
||||
if firstBuildWarrantIn(t.Context()).covers(buildSource{Repository: "novox/mesh-delivery", Seat: "git"}, "") {
|
||||
t.Error("1146/AC9: a context with no warrant covers an ask")
|
||||
}
|
||||
}
|
||||
@@ -47,7 +47,7 @@ func (f following) firstBuilds(ctx context.Context, m link.SourceMoved, entries
|
||||
case len(added) == 0:
|
||||
return false, nil
|
||||
}
|
||||
if err := askFirstBuilds(ctx, inv, m, now, added, why); err != nil {
|
||||
if err := askFirstBuilds(ctx, f.open, m, now, added, why); err != nil {
|
||||
return true, notNow(err)
|
||||
}
|
||||
return true, nil
|
||||
@@ -68,27 +68,35 @@ func keepFirstMerge(ctx context.Context, inv *inventory.Inventory, m link.Source
|
||||
return err
|
||||
}
|
||||
|
||||
// askFirstBuilds keeps the merge, asks the builds, and marks the merge as no longer waiting for them.
|
||||
func askFirstBuilds(ctx context.Context, inv *inventory.Inventory, m link.SourceMoved, now time.Time, added []string,
|
||||
// askFirstBuilds keeps the merge as answered and registers its modules from the builder's done keys (novox/hq ADR 0501
|
||||
// §3, §6): nothing asks their first builds. The merge kept is the registration's warrant (approvedMove reads it by the
|
||||
// commit the indexer's trunk value names); a module whose key is not done yet is registered when the key's write wakes
|
||||
// the take-in (new_modules.go).
|
||||
func askFirstBuilds(ctx context.Context, open *stores, m link.SourceMoved, now time.Time, added []string,
|
||||
why string) error {
|
||||
inv := open.inventory
|
||||
repository := m.Owner + "/" + m.Repo
|
||||
if err := keepFirstMerge(ctx, inv, m, now, false); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%s merged into %s (%.8s) on the operator's Approve (%s): the mesh builds nothing from it yet, so the "+
|
||||
"first build of %s is asked\n", repository, m.Base, m.Commit, why, strings.Join(added, ", "))
|
||||
// The ask carries the warrant just checked, for this repository and these module paths alone: the guard of a build
|
||||
// through a verb (verbMayAsk) lets exactly that through.
|
||||
paths := make([]string, 0, len(added))
|
||||
for _, dir := range added {
|
||||
if dir == "." {
|
||||
dir = ""
|
||||
}
|
||||
paths = append(paths, dir)
|
||||
fmt.Printf("%s merged into %s (%.8s) on the operator's Approve (%s): the mesh builds nothing from it yet, so %s "+
|
||||
"are registered from the builder's done keys, asking no build\n", repository, m.Base, m.Commit, why,
|
||||
strings.Join(added, ", "))
|
||||
// Answered: it no longer waits, so the word's arrival again does nothing twice.
|
||||
if err := inv.AnswerMerge(ctx, repository, m.Commit, inventory.WaitingRecord, false); err != nil {
|
||||
return err
|
||||
}
|
||||
askBuilds(withFirstBuildWarrant(ctx, repository, paths, why), inv, m, buildSource{Repository: repository, Seat: gitSeat}, added)
|
||||
// Answered: it no longer waits, so the word's arrival again asks nothing twice.
|
||||
return inv.AnswerMerge(ctx, repository, m.Commit, inventory.WaitingRecord, false)
|
||||
modules, err := newModulesOf(ctx, repository)
|
||||
switch {
|
||||
case err != nil:
|
||||
// Said, not failed: the merge is answered, and the look at every module when a controller starts, or the next
|
||||
// write of the builder or the indexer for one of them, registers them.
|
||||
fmt.Printf(" the indexer's modules of %s cannot be read now (%v): they are registered when the builder's next "+
|
||||
"write of one wakes the take-in\n", repository, err)
|
||||
case len(modules) > 0:
|
||||
inBackground(func() { takeNewModules(context.WithoutCancel(ctx), open, modules...) })
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// resumeFirstBuilds runs firstBuilds for the first merges waiting for the word mesh-delivery just announced for a pull
|
||||
@@ -126,7 +134,7 @@ func (f following) resumeFirstBuilds(ctx context.Context, d deliveryMerge) error
|
||||
case err != nil:
|
||||
return notNow(err)
|
||||
case len(added) > 0:
|
||||
if err := askFirstBuilds(ctx, inv, k.SourceMoved, time.Now().UTC(), added, why); err != nil {
|
||||
if err := askFirstBuilds(ctx, f.open, k.SourceMoved, time.Now().UTC(), added, why); err != nil {
|
||||
return notNow(err)
|
||||
}
|
||||
}
|
||||
@@ -174,7 +182,9 @@ func firstBuildsOf(ctx context.Context, inv catalogueReader, m link.SourceMoved,
|
||||
}
|
||||
|
||||
// heldFirstBuild says a first build was refused because its manifest uses a word the running build-agent does not read,
|
||||
// and its merge is kept waiting for a build-agent that reads it (novox/hq ADR 0482 §5, issue 1173).
|
||||
// and its merge is kept waiting for a build-agent that reads it (novox/hq ADR 0482 §5, issue 1173). Read from a merge kept
|
||||
// before R4e, never written since: nothing asks a first build (ADR 0501 §6), and a controller that starts answers such a
|
||||
// merge and registers its modules from their keys (answerHeldMerges).
|
||||
type heldFirstBuild struct {
|
||||
Module string `json:"module"`
|
||||
Path string `json:"path"`
|
||||
@@ -182,52 +192,6 @@ type heldFirstBuild struct {
|
||||
At time.Time `json:"at"`
|
||||
}
|
||||
|
||||
// judgeRefusedFirstBuild is the hold a first build's outcome puts its merge under: every module the merge holds already,
|
||||
// and this one, unless it is among them; nil unless the build failed on a word of its manifest the running build-agent
|
||||
// does not know, and was asked for a first merge, answered by the record of merges no plan answers or put back to waiting
|
||||
// by an earlier hold (a second refusal in one merge is held too, review of issue 1173).
|
||||
func judgeRefusedFirstBuild(result link.BuildResult, req inventory.BuildRequest, merge inventory.BatchedMerge) []heldFirstBuild {
|
||||
word := refusedWord(result.Failed)
|
||||
if word == "" || req.For != "merge" {
|
||||
return nil
|
||||
}
|
||||
var held []heldFirstBuild
|
||||
switch k := announcedOf(merge); {
|
||||
case merge.Plan == inventory.WaitingRecord:
|
||||
// Answered when its builds were asked: a hold from before (a build asked again) is over, and starts anew.
|
||||
case merge.Plan == "" && len(k.Held) > 0:
|
||||
held = k.Held
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
this := heldFirstBuild{Module: req.Name(), Path: strings.Trim(req.Path, "/"), Word: word, At: time.Now().UTC()}
|
||||
for i, h := range held {
|
||||
if strings.Trim(h.Path, "/") == this.Path {
|
||||
out := append([]heldFirstBuild(nil), held...)
|
||||
out[i] = this
|
||||
return out
|
||||
}
|
||||
}
|
||||
return append(append([]heldFirstBuild(nil), held...), this)
|
||||
}
|
||||
|
||||
// heldDirsOf is the directories a held merge asks again: its held modules' own, each once, "." for the repository's root.
|
||||
func heldDirsOf(k keptMerge) []string {
|
||||
var dirs []string
|
||||
seen := map[string]bool{}
|
||||
for _, h := range k.Held {
|
||||
dir := strings.Trim(h.Path, "/")
|
||||
if dir == "" {
|
||||
dir = "."
|
||||
}
|
||||
if !seen[dir] {
|
||||
seen[dir] = true
|
||||
dirs = append(dirs, dir)
|
||||
}
|
||||
}
|
||||
return dirs
|
||||
}
|
||||
|
||||
// unknownWord is how the strict reading of a manifest says a word is not one it knows: encoding/json's own words, which
|
||||
// the build-agent's answer carries whole. The build's answer has no typed field for it (Failed is the builder's words).
|
||||
var unknownWord = regexp.MustCompile(`json: unknown field "([^"]+)"`)
|
||||
@@ -240,55 +204,12 @@ func refusedWord(failed string) string {
|
||||
return ""
|
||||
}
|
||||
|
||||
// holdRefusedFirstBuild keeps a first build's merge waiting when the build was refused for a word the running build-agent
|
||||
// does not read (novox/hq ADR 0482 §5, issue 1173): no failed build is recorded, nothing expires the merge, and it is
|
||||
// named by a condition until a delivery of build-agent asks the build again. Answers whether the outcome was dealt with.
|
||||
func holdRefusedFirstBuild(ctx context.Context, inv *inventory.Inventory, result link.BuildResult) (bool, error) {
|
||||
if result.Failed == "" || refusedWord(result.Failed) == "" {
|
||||
return false, nil
|
||||
}
|
||||
req, found, err := inv.BuildRequestByID(ctx, result.ID)
|
||||
if err != nil || !found {
|
||||
return false, err
|
||||
}
|
||||
merge, found, err := inv.MergeOf(ctx, req.Repository, req.Commit)
|
||||
if err != nil || !found {
|
||||
return false, err
|
||||
}
|
||||
held := judgeRefusedFirstBuild(result, req, merge)
|
||||
if len(held) == 0 {
|
||||
return false, nil
|
||||
}
|
||||
k := announcedOf(merge)
|
||||
k.Held = held
|
||||
event, err := json.Marshal(k)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if err := inv.HoldMerge(ctx, merge.Repository, merge.Commit, event); err != nil {
|
||||
return false, err
|
||||
}
|
||||
this := held[len(held)-1]
|
||||
for _, h := range held {
|
||||
if h.Path == strings.Trim(req.Path, "/") {
|
||||
this = h
|
||||
}
|
||||
}
|
||||
fmt.Printf("%s: the first build of %s was refused for the word %q, which the running build-agent does not read; the "+
|
||||
"merge %.8s of %s is kept waiting for a build-agent that reads it (holding %s), and the build is asked again when "+
|
||||
"one is delivered\n", result.ID, this.Module, this.Word, merge.Commit, merge.Repository, heldNames(held))
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// probeFirstBuildsID is the probe that reads the first merges waiting for mesh-delivery's word.
|
||||
const probeFirstBuildsID = "D18"
|
||||
|
||||
// kindFirstBuildUnasked is what a first merge waiting too long for mesh-delivery's word raises.
|
||||
const kindFirstBuildUnasked = "first-build-unasked"
|
||||
|
||||
// kindFirstBuildHeld is what a first merge held for a build-agent that reads its manifest raises.
|
||||
const kindFirstBuildHeld = "first-build-held"
|
||||
|
||||
// firstBuildWaitBound is how long a first merge waits for mesh-delivery's announcement of the operator's Approve before
|
||||
// it is said: a bound in the sense of ADR 0426 §3, not a delay. Nothing waits on it: the word's arrival asks the builds
|
||||
// at once, and the merge is said, never dropped, once it is past this (issue 1146).
|
||||
@@ -300,10 +221,6 @@ func firstBuildObservations(waiting []inventory.BatchedMerge, now time.Time) []c
|
||||
var out []conditions.Observation
|
||||
for _, w := range waiting {
|
||||
k := announcedOf(w)
|
||||
if len(k.Held) > 0 {
|
||||
out = append(out, heldFirstBuildObservation(k))
|
||||
continue
|
||||
}
|
||||
if !k.FirstBuild || now.Sub(w.Heard) <= firstBuildWaitBound {
|
||||
continue
|
||||
}
|
||||
@@ -318,108 +235,43 @@ func firstBuildObservations(waiting []inventory.BatchedMerge, now time.Time) []c
|
||||
Headline: fmt.Sprintf("The first build of %s is not asked: no Approve was announced", repository),
|
||||
Explanation: fmt.Sprintf("The merge %.8s into %s adds modules the mesh builds nothing from yet. It registers them "+
|
||||
"only when mesh-delivery announces that it merged the pull request on your Approve of that head, and no such "+
|
||||
"announcement has come in %s. The merge is kept, and the builds are asked the moment the announcement is kept.",
|
||||
"announcement has come in %s. The merge is kept, and the modules are registered from the builder's keys the moment "+
|
||||
"the announcement is kept.",
|
||||
k.Commit, repository, for_),
|
||||
Resolved: "mesh-delivery announces the merge, or a hand `build` registers the modules",
|
||||
Resolved: "mesh-delivery announces the merge, or the operator registers the modules at the controller's terminal",
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// heldNames is the modules a merge holds, each with the word it waits for: "watchman (key-ttl), view (per-machine)".
|
||||
func heldNames(held []heldFirstBuild) string {
|
||||
names := make([]string, 0, len(held))
|
||||
for _, h := range held {
|
||||
names = append(names, fmt.Sprintf("%s (%q)", h.Module, h.Word))
|
||||
}
|
||||
return strings.Join(names, ", ")
|
||||
}
|
||||
|
||||
// heldFirstBuildObservation names a first merge held for a build-agent that reads its manifests: at once, since nothing
|
||||
// is lost and nothing times out, but the operator can see what the merge waits for. One condition names every module held.
|
||||
func heldFirstBuildObservation(k keptMerge) conditions.Observation {
|
||||
repository := k.Owner + "/" + k.Repo
|
||||
modules := make([]string, 0, len(k.Held))
|
||||
for _, h := range k.Held {
|
||||
modules = append(modules, h.Module)
|
||||
}
|
||||
held := heldNames(k.Held)
|
||||
return conditions.Observation{Scope: conditions.ScopeMerge, ID: fmt.Sprintf("%s_%.8s", repository, k.Commit),
|
||||
Kind: kindFirstBuildHeld, Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("the merge %.8s of %s adds modules whose manifests use words the running build-agent does not "+
|
||||
"read: %s; the merge waits for a build-agent that reads them", k.Commit, repository, held),
|
||||
Said: fmt.Sprintf("%s at %s waits for a build-agent that reads the words of %s", repository, k.Commit, held),
|
||||
Headline: fmt.Sprintf("The first build of %s waits for a newer build-agent", strings.Join(modules, ", ")),
|
||||
Explanation: fmt.Sprintf("The merge %.8s into %s adds modules whose manifests use words the build-agent now "+
|
||||
"running does not know: %s. The merge is kept, not failed: their first builds are asked again when a delivery "+
|
||||
"of build-agent ends.", k.Commit, repository, held),
|
||||
Resolved: "a delivery of build-agent ends and the first builds register the modules, or a hand `build` does",
|
||||
}
|
||||
}
|
||||
|
||||
// probeFirstBuilds is D18: no first merge waits past its bound for mesh-delivery's word unsaid.
|
||||
func probeFirstBuilds(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
waiting, err := d.open.inventory.WaitingMerges(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return firstBuildObservations(waiting, time.Now().UTC()), nil
|
||||
now := time.Now().UTC()
|
||||
return append(firstBuildObservations(waiting, now), newModuleObservations(now)...), nil
|
||||
}
|
||||
|
||||
// askHeldFirstBuildsAgain asks again the first build of each first merge held for a build-agent that reads its manifest,
|
||||
// when the plan that ended delivered build-agent (novox/hq issue 1173): once for each end, since the ask answers the merge
|
||||
// and a refusal that comes again holds it anew. Answers how many merges it asked for.
|
||||
func askHeldFirstBuildsAgain(ctx context.Context, open *stores, p inventory.Plan) int {
|
||||
if !deliversBuildAgent(p) {
|
||||
return 0
|
||||
}
|
||||
actingOnMerges.Lock()
|
||||
defer actingOnMerges.Unlock()
|
||||
inv := open.inventory
|
||||
// answerHeldMerges answers every merge a controller of before held for a build-agent that reads its new modules'
|
||||
// manifests (issue 1173): nothing asks their first builds any more, and the look at every module when this controller
|
||||
// starts registers them from their keys (ADR 0501 §6, §7).
|
||||
func answerHeldMerges(ctx context.Context, inv *inventory.Inventory) {
|
||||
waiting, err := inv.WaitingMerges(ctx)
|
||||
if err != nil {
|
||||
fmt.Printf("%s delivered build-agent, and the first merges held for it could not be read: %v\n", p.ID, err)
|
||||
return 0
|
||||
fmt.Printf("the merges held for a newer build-agent could not be read: %v\n", err)
|
||||
return
|
||||
}
|
||||
asked := 0
|
||||
for _, w := range waiting {
|
||||
k := announcedOf(w)
|
||||
if len(k.Held) == 0 {
|
||||
continue
|
||||
}
|
||||
// Its own held modules, not the rule of a first merge: a module of the merge that passed is registered by now,
|
||||
// and the repository no longer counts as one the mesh builds nothing from (review of issue 1173).
|
||||
why, err := approvedAnnounced(ctx, inv, k.Owner, k.Repo, keptMerge{SourceMoved: k.SourceMoved})
|
||||
if err != nil {
|
||||
fmt.Printf("%s delivered build-agent, and the merge %.8s of %s held for it asks no build now (%v): it stays "+
|
||||
"waiting\n", p.ID, w.Commit, w.Repository, err)
|
||||
continue
|
||||
}
|
||||
dirs := heldDirsOf(k)
|
||||
fmt.Printf("%s delivered build-agent: the first builds of %s are asked again\n", p.ID, heldNames(k.Held))
|
||||
paths := make([]string, 0, len(dirs))
|
||||
for _, dir := range dirs {
|
||||
if dir == "." {
|
||||
dir = ""
|
||||
if k := announcedOf(w); len(k.Held) > 0 {
|
||||
if err := inv.AnswerMerge(ctx, w.Repository, w.Commit, inventory.WaitingRecord, false); err != nil {
|
||||
fmt.Printf("the merge %.8s of %s held for a newer build-agent could not be answered: %v\n", w.Commit,
|
||||
w.Repository, err)
|
||||
continue
|
||||
}
|
||||
paths = append(paths, dir)
|
||||
fmt.Printf("the merge %.8s of %s was held for a newer build-agent; its modules are registered from the "+
|
||||
"builder's keys instead\n", w.Commit, w.Repository)
|
||||
}
|
||||
repository := k.Owner + "/" + k.Repo
|
||||
askBuilds(withFirstBuildWarrant(ctx, repository, paths, why), inv, k.SourceMoved,
|
||||
buildSource{Repository: repository, Seat: gitSeat}, dirs)
|
||||
// Answered: it no longer waits, so this end asks nothing twice; a refusal that comes again holds it anew.
|
||||
if err := inv.AnswerMerge(ctx, w.Repository, w.Commit, inventory.WaitingRecord, false); err != nil {
|
||||
fmt.Printf(" and the merge could not be marked answered: %v\n", err)
|
||||
continue
|
||||
}
|
||||
asked++
|
||||
}
|
||||
return asked
|
||||
}
|
||||
|
||||
// deliversBuildAgent says a delivery plan ended delivered with build-agent built in it: the reader a held first build
|
||||
// waits for is the one now running.
|
||||
func deliversBuildAgent(p inventory.Plan) bool {
|
||||
m := p.Modules["build-agent"]
|
||||
return p.WalkSaid == inventory.WalkEnded && p.Outcome() == inventory.WalkDelivered && m != nil && m.State == "built"
|
||||
}
|
||||
|
||||
@@ -1,221 +0,0 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// 1173/AC1, 1173/AC3 whole against the store: a first merge's build refused for a word the running build-agent does not
|
||||
// read keeps the merge waiting with no failed build recorded; a delivery plan that does not deliver build-agent asks
|
||||
// nothing; the one that ends delivering it asks the first build again once, and its pass registers the module.
|
||||
func TestAHeldFirstMergeIsAskedAgainWhenABuildAgentIsDelivered(t *testing.T) {
|
||||
const head, commit = "a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1", "0177ea7a00000000"
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
|
||||
a := aDeliveryMergeOf(t, head, map[string]string{"novox/mesh-delivery@a1a1a1a1a1a1": head})
|
||||
wasKept, wasAsk, realAsk := keptDeliveryMerge, routerAskOf, askABuild
|
||||
keptDeliveryMerge = func(context.Context, string, string, int, string) (deliveryMerge, bool, error) {
|
||||
return a.merge, true, nil
|
||||
}
|
||||
routerAskOf = func(context.Context, catalogueReader, string, string) (*asks.Ask, *asks.Warrant, error) {
|
||||
return &a.ask, &a.warrant, nil
|
||||
}
|
||||
var asked []string
|
||||
askABuild = func(_ context.Context, source buildSource, path, ref string) (string, error) {
|
||||
id := "build-" + string(rune('1'+len(asked)))
|
||||
asked = append(asked, id)
|
||||
return id, nil
|
||||
}
|
||||
t.Cleanup(func() { keptDeliveryMerge, routerAskOf, askABuild = wasKept, wasAsk, realAsk })
|
||||
|
||||
m := link.SourceMoved{Owner: "novox", Repo: "mesh-delivery", Base: "main", HeadSHA: head, Commit: commit, Number: 7,
|
||||
MergedAt: time.Now().Add(-time.Hour).UTC().Format(time.RFC3339Nano), Paths: []string{"module.json", "main.go"},
|
||||
ModuleDirs: []string{"."}, ModuleDirsSaid: true}
|
||||
if err := (following{open: open}).SourceMoved(ctx, m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(asked) != 1 {
|
||||
t.Fatalf("1173/AC3: the first merge asked %v, want the one first build", asked)
|
||||
}
|
||||
|
||||
// The running build-agent refuses the manifest's word.
|
||||
refused := onTrunk("build-1", "novox/mesh-delivery", "git", "", nil)
|
||||
refused.Failed, refused.Manifest, refused.Commit = refusedKeyTTL, nil, commit
|
||||
if err := (builds{inv: inv, open: open}).Built(ctx, refused); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, recorded, _ := inv.BuildByID(ctx, "build-1"); recorded {
|
||||
t.Fatal("1173/AC1: a refused first build was recorded as a failed build")
|
||||
}
|
||||
waiting, err := inv.WaitingMerges(ctx)
|
||||
if err != nil || len(waiting) != 1 || len(announcedOf(waiting[0]).Held) == 0 {
|
||||
t.Fatalf("1173/AC1: the merges waiting are %+v (%v), want the first merge, held", waiting, err)
|
||||
}
|
||||
if obs := firstBuildObservations(waiting, time.Now()); len(obs) != 1 || obs[0].Kind != kindFirstBuildHeld {
|
||||
t.Fatalf("1173/AC2: the held merge is named by %+v", obs)
|
||||
}
|
||||
|
||||
// A plan that ended without delivering build-agent asks nothing; one that delivered it asks once.
|
||||
other := inventory.Plan{ID: "p1", State: inventory.PlanDone, WalkSaid: inventory.WalkEnded,
|
||||
Modules: map[string]*inventory.PlanModule{"mesh-delivery": {State: "built"}}}
|
||||
if n := askHeldFirstBuildsAgain(ctx, open, other); n != 0 || len(asked) != 1 {
|
||||
t.Fatalf("1173/AC3: a plan that delivered no build-agent asked %d builds again (%v)", n, asked)
|
||||
}
|
||||
delivered := inventory.Plan{ID: "p2", State: inventory.PlanDone, WalkSaid: inventory.WalkEnded,
|
||||
Modules: map[string]*inventory.PlanModule{"build-agent": {State: "built"}}}
|
||||
if n := askHeldFirstBuildsAgain(ctx, open, delivered); n != 1 || len(asked) != 2 {
|
||||
t.Fatalf("1173/AC3: the plan that delivered build-agent asked %d builds again (%v), want one", n, asked)
|
||||
}
|
||||
if n := askHeldFirstBuildsAgain(ctx, open, delivered); n != 0 || len(asked) != 2 {
|
||||
t.Fatalf("1173/AC3: the same end asked again: %d (%v)", n, asked)
|
||||
}
|
||||
|
||||
// Its pass registers the module.
|
||||
pass := onTrunk("build-2", "novox/mesh-delivery", "git", "", map[string]any{"module": "mesh-delivery-watchman", "version": "1"})
|
||||
pass.Commit = commit
|
||||
if err := (builds{inv: inv, open: open}).Built(ctx, pass); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, known := inventoryHolds(t, inv, "mesh-delivery-watchman"); !known {
|
||||
t.Fatal("1173/AC3: the module is not registered after the first build passed")
|
||||
}
|
||||
}
|
||||
|
||||
// aTwoModuleFirstMerge is a first merge into novox/mesh-delivery adding the modules watchman and view, made on the
|
||||
// operator's Approve of its head, whose first builds are asked; asked says the build id asked for each path, in order.
|
||||
func aTwoModuleFirstMerge(t *testing.T, open *stores, commit string) *[][2]string {
|
||||
t.Helper()
|
||||
const head = "b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2"
|
||||
a := aDeliveryMergeOf(t, head, map[string]string{"novox/mesh-delivery@b2b2b2b2b2b2": head})
|
||||
wasKept, wasAsk, realAsk := keptDeliveryMerge, routerAskOf, askABuild
|
||||
keptDeliveryMerge = func(context.Context, string, string, int, string) (deliveryMerge, bool, error) {
|
||||
return a.merge, true, nil
|
||||
}
|
||||
routerAskOf = func(context.Context, catalogueReader, string, string) (*asks.Ask, *asks.Warrant, error) {
|
||||
return &a.ask, &a.warrant, nil
|
||||
}
|
||||
asked := &[][2]string{}
|
||||
askABuild = func(_ context.Context, _ buildSource, path, _ string) (string, error) {
|
||||
id := fmt.Sprintf("build-%d", len(*asked)+1)
|
||||
*asked = append(*asked, [2]string{path, id})
|
||||
return id, nil
|
||||
}
|
||||
t.Cleanup(func() { keptDeliveryMerge, routerAskOf, askABuild = wasKept, wasAsk, realAsk })
|
||||
m := link.SourceMoved{Owner: "novox", Repo: "mesh-delivery", Base: "main", HeadSHA: head, Commit: commit, Number: 7,
|
||||
MergedAt: time.Now().Add(-time.Hour).UTC().Format(time.RFC3339Nano),
|
||||
Paths: []string{"view/module.json", "view/main.go", "watchman/module.json", "watchman/main.go"},
|
||||
ModuleDirs: []string{"view", "watchman"}, ModuleDirsSaid: true}
|
||||
if err := (following{open: open}).SourceMoved(t.Context(), m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(*asked) != 2 {
|
||||
t.Fatalf("the first merge asked %v, want the two first builds", *asked)
|
||||
}
|
||||
return asked
|
||||
}
|
||||
|
||||
// idOf is the build asked last for path.
|
||||
func idOf(t *testing.T, asked [][2]string, path string) string {
|
||||
t.Helper()
|
||||
for i := len(asked) - 1; i >= 0; i-- {
|
||||
if asked[i][0] == path {
|
||||
return asked[i][1]
|
||||
}
|
||||
}
|
||||
t.Fatalf("no build asked for %s: %v", path, asked)
|
||||
return ""
|
||||
}
|
||||
|
||||
// built hands the controller a build's outcome: refused for the word, or passing as module.
|
||||
func built(t *testing.T, open *stores, id, path, commit, refusedFor, module string) {
|
||||
t.Helper()
|
||||
r := onTrunk(id, "novox/mesh-delivery", "git", path, map[string]any{"module": module, "version": "1"})
|
||||
r.Commit = commit
|
||||
if refusedFor != "" {
|
||||
r.Failed, r.Manifest = `this is not a module manifest: json: unknown field "`+refusedFor+`"`, nil
|
||||
}
|
||||
if err := (builds{inv: open.inventory, open: open}).Built(t.Context(), r); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
var buildAgentDelivered = inventory.Plan{ID: "p-ba", State: inventory.PlanDone, WalkSaid: inventory.WalkEnded,
|
||||
Modules: map[string]*inventory.PlanModule{"build-agent": {State: "built"}}}
|
||||
|
||||
// 1173/AC3 (review round 1, item 1): in a first merge of two modules where view passes and watchman is refused, the
|
||||
// held watchman is asked again when build-agent is delivered, though the repository builds view by then.
|
||||
func TestAHeldModuleIsAskedAgainAfterItsSiblingRegistered(t *testing.T) {
|
||||
const commit = "0177ea7b00000000"
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
asked := aTwoModuleFirstMerge(t, open, commit)
|
||||
|
||||
built(t, open, idOf(t, *asked, "view"), "view", commit, "", "mesh-delivery-view")
|
||||
if _, known := inventoryHolds(t, open.inventory, "mesh-delivery-view"); !known {
|
||||
t.Fatal("view is not registered after its first build passed")
|
||||
}
|
||||
refused := idOf(t, *asked, "watchman")
|
||||
built(t, open, refused, "watchman", commit, "key-ttl", "")
|
||||
if _, recorded, _ := open.inventory.BuildByID(ctx, refused); recorded {
|
||||
t.Fatal("1173/AC1: the refused first build of watchman was recorded as a failed build")
|
||||
}
|
||||
waiting, err := open.inventory.WaitingMerges(ctx)
|
||||
if err != nil || len(waiting) != 1 || len(announcedOf(waiting[0]).Held) != 1 {
|
||||
t.Fatalf("1173/AC1: the merges waiting are %+v (%v), want the first merge, holding watchman", waiting, err)
|
||||
}
|
||||
|
||||
if n := askHeldFirstBuildsAgain(ctx, open, buildAgentDelivered); n != 1 || len(*asked) != 3 || (*asked)[2][0] != "watchman" {
|
||||
t.Fatalf("1173/AC3: the delivery of build-agent asked %d merges again (%v), want watchman's first build alone", n, *asked)
|
||||
}
|
||||
built(t, open, idOf(t, *asked, "watchman"), "watchman", commit, "", "mesh-delivery-watchman")
|
||||
if _, known := inventoryHolds(t, open.inventory, "mesh-delivery-watchman"); !known {
|
||||
t.Fatal("1173/AC3: watchman is not registered after the first build asked again passed")
|
||||
}
|
||||
}
|
||||
|
||||
// 1173/AC1, AC2, AC3 (review round 1, item 2): in a first merge of two modules both refused, neither is recorded as a
|
||||
// failed build, the condition names both, and the delivery of build-agent asks both again.
|
||||
func TestTwoRefusedModulesOfOneFirstMergeAreBothHeld(t *testing.T) {
|
||||
const commit = "0177ea7c00000000"
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
asked := aTwoModuleFirstMerge(t, open, commit)
|
||||
|
||||
for _, path := range []string{"watchman", "view"} {
|
||||
id := idOf(t, *asked, path)
|
||||
built(t, open, id, path, commit, "key-ttl", "")
|
||||
if _, recorded, _ := open.inventory.BuildByID(ctx, id); recorded {
|
||||
t.Fatalf("1173/AC1: the refused first build of %s was recorded as a failed build", path)
|
||||
}
|
||||
}
|
||||
waiting, err := open.inventory.WaitingMerges(ctx)
|
||||
if err != nil || len(waiting) != 1 || len(announcedOf(waiting[0]).Held) != 2 {
|
||||
t.Fatalf("1173/AC1: the merges waiting are %+v (%v), want the first merge, holding both", waiting, err)
|
||||
}
|
||||
obs := firstBuildObservations(waiting, time.Now())
|
||||
if len(obs) != 1 || !strings.Contains(obs[0].Summary, "watchman") || !strings.Contains(obs[0].Summary, "view") {
|
||||
t.Fatalf("1173/AC2: the held merge is named by %+v, want one condition naming both", obs)
|
||||
}
|
||||
|
||||
if n := askHeldFirstBuildsAgain(ctx, open, buildAgentDelivered); n != 1 || len(*asked) != 4 {
|
||||
t.Fatalf("1173/AC3: the delivery of build-agent asked %d merges again (%v), want both first builds", n, *asked)
|
||||
}
|
||||
built(t, open, idOf(t, *asked, "watchman"), "watchman", commit, "", "mesh-delivery-watchman")
|
||||
built(t, open, idOf(t, *asked, "view"), "view", commit, "", "mesh-delivery-view")
|
||||
for _, module := range []string{"mesh-delivery-watchman", "mesh-delivery-view"} {
|
||||
if _, known := inventoryHolds(t, open.inventory, module); !known {
|
||||
t.Fatalf("1173/AC3: %s is not registered after the first build asked again passed", module)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,141 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
const refusedKeyTTL = `this is not a module manifest: a state is either a name or {name, history, ttl-seconds, per-machine}: ` +
|
||||
`json: unknown field "key-ttl"`
|
||||
|
||||
// aFirstMergeAnswered is a first merge as askFirstBuilds leaves it: answered by the record of merges no plan answers.
|
||||
func aFirstMergeAnswered() (inventory.BuildRequest, inventory.BatchedMerge) {
|
||||
req := inventory.BuildRequest{ID: "build-1", Repository: "novox/mesh-delivery", Seat: "git", Path: "watchman", Ref: "main",
|
||||
Commit: "0177ea7a00", For: "merge"}
|
||||
event, _ := json.Marshal(keptMerge{SourceMoved: link.SourceMoved{Owner: "novox", Repo: "mesh-delivery", Base: "main",
|
||||
Commit: "0177ea7a00", Number: 3, HeadSHA: "a1a1a1a1a1a1"}, FirstBuild: true})
|
||||
return req, inventory.BatchedMerge{Repository: "novox/mesh-delivery", Branch: "main", Commit: "0177ea7a00",
|
||||
Plan: inventory.WaitingRecord, Event: event}
|
||||
}
|
||||
|
||||
// 1173/AC1: a first build refused for a word its manifest uses and the running build-agent does not know holds its merge.
|
||||
func TestAFirstBuildRefusedForAnUnknownWordHoldsItsMerge(t *testing.T) {
|
||||
req, merge := aFirstMergeAnswered()
|
||||
held := judgeRefusedFirstBuild(link.BuildResult{ID: "build-1", Failed: refusedKeyTTL}, req, merge)
|
||||
if len(held) != 1 || held[0].Word != "key-ttl" || held[0].Module != "watchman" || held[0].Path != "watchman" {
|
||||
t.Fatalf("1173/AC1: a first build refused for \"key-ttl\" is judged %+v, want a hold on that word for watchman", held)
|
||||
}
|
||||
}
|
||||
|
||||
// 1173/AC2: a held first merge is named at once by one condition: the merge, the module, the word and what it waits for.
|
||||
func TestAHeldFirstMergeIsNamedByACondition(t *testing.T) {
|
||||
now := time.Date(2026, 10, 22, 14, 0, 0, 0, time.UTC)
|
||||
_, merge := aFirstMergeAnswered()
|
||||
var k keptMerge
|
||||
_ = json.Unmarshal(merge.Event, &k)
|
||||
k.Held = []heldFirstBuild{{Module: "watchman", Path: "watchman", Word: "key-ttl", At: now},
|
||||
{Module: "view", Path: "view", Word: "per-machine", At: now}}
|
||||
merge.Event, _ = json.Marshal(k)
|
||||
merge.Heard = now.Add(-time.Second)
|
||||
got := firstBuildObservations([]inventory.BatchedMerge{merge}, now)
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("1173/AC2: %d conditions for a held first merge, want one: %+v", len(got), got)
|
||||
}
|
||||
o := got[0]
|
||||
if o.Kind != kindFirstBuildHeld || o.Scope != conditions.ScopeMerge {
|
||||
t.Fatalf("1173/AC2: the condition is %+v", o)
|
||||
}
|
||||
for _, want := range []string{"0177ea7a", "watchman", "key-ttl", "view", "per-machine", "build-agent"} {
|
||||
if !strings.Contains(o.Summary, want) {
|
||||
t.Fatalf("1173/AC2: the summary %q does not name %q", o.Summary, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1173/AC4: a first build that fails for any other reason is not held.
|
||||
func TestAFirstBuildFailingForAnyOtherReasonIsNotHeld(t *testing.T) {
|
||||
req, merge := aFirstMergeAnswered()
|
||||
if judgeRefusedFirstBuild(link.BuildResult{Failed: refusedKeyTTL}, req, merge) == nil {
|
||||
t.Fatal("1173/AC4: the control, a refusal for an unknown word, is not held")
|
||||
}
|
||||
for _, c := range []struct {
|
||||
what string
|
||||
failed string
|
||||
change func(*inventory.BuildRequest, *inventory.BatchedMerge)
|
||||
}{
|
||||
{what: "a compile error", failed: "main.go:3: undefined: foo"},
|
||||
{what: "no failure", failed: ""},
|
||||
{what: "a missing field", failed: `json: cannot unmarshal string into Go struct field`},
|
||||
{what: "a build a person asked", failed: refusedKeyTTL, change: func(r *inventory.BuildRequest, _ *inventory.BatchedMerge) { r.For = "build" }},
|
||||
{what: "a merge a plan answers", failed: refusedKeyTTL, change: func(_ *inventory.BuildRequest, m *inventory.BatchedMerge) { m.Plan = "plan-7" }},
|
||||
{what: "a waiting merge held for nothing", failed: refusedKeyTTL, change: func(_ *inventory.BuildRequest, m *inventory.BatchedMerge) { m.Plan = "" }},
|
||||
} {
|
||||
r, m := req, merge
|
||||
if c.change != nil {
|
||||
c.change(&r, &m)
|
||||
}
|
||||
if held := judgeRefusedFirstBuild(link.BuildResult{Failed: c.failed}, r, m); held != nil {
|
||||
t.Fatalf("1173/AC4: %s is held: %+v", c.what, held)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1173/AC1, AC2 (review round 1, item 2): a second refusal in a merge already held is held too, its module added to the
|
||||
// hold, so no failed build is recorded and the condition names both.
|
||||
func TestASecondRefusalInAHeldMergeIsAddedToTheHold(t *testing.T) {
|
||||
req, merge := aFirstMergeAnswered()
|
||||
first := judgeRefusedFirstBuild(link.BuildResult{Failed: refusedKeyTTL}, req, merge)
|
||||
k := announcedOf(merge)
|
||||
k.Held = first
|
||||
merge.Event, _ = json.Marshal(k)
|
||||
merge.Plan = "" // HoldMerge put it back to waiting
|
||||
req.ID, req.Path = "build-2", "view"
|
||||
held := judgeRefusedFirstBuild(link.BuildResult{Failed: `json: unknown field "per-machine"`}, req, merge)
|
||||
if len(held) != 2 || held[0].Module != "watchman" || held[1].Module != "view" || held[1].Word != "per-machine" {
|
||||
t.Fatalf("1173/AC1: the second refusal in a held merge is judged %+v, want watchman and view held", held)
|
||||
}
|
||||
// The same module refused again is held once.
|
||||
k.Held = held
|
||||
merge.Event, _ = json.Marshal(k)
|
||||
if again := judgeRefusedFirstBuild(link.BuildResult{Failed: refusedKeyTTL}, req, merge); len(again) != 2 {
|
||||
t.Fatalf("1173/AC1: view refused again is held as %+v, want it once", again)
|
||||
}
|
||||
}
|
||||
|
||||
// 1173/AC3 (review round 1, item 1): a held merge is asked again for its own held modules' directories, not by the rule of
|
||||
// a first merge, which finds nothing once another module of the repository is registered.
|
||||
func TestAHeldMergeIsAskedAgainForItsHeldModulesAlone(t *testing.T) {
|
||||
k := keptMerge{Held: []heldFirstBuild{{Module: "watchman", Path: "watchman"}, {Module: "mesh-delivery", Path: ""},
|
||||
{Module: "watchman", Path: "/watchman/"}}}
|
||||
if got := heldDirsOf(k); strings.Join(got, ",") != "watchman,." {
|
||||
t.Fatalf("1173/AC3: a held merge asks again %v, want watchman and the root once each", got)
|
||||
}
|
||||
}
|
||||
|
||||
// 1173/AC3 (the part that needs no store): only a plan that ended delivered with build-agent built in it is the end a held
|
||||
// first build waits for.
|
||||
func TestOnlyADeliveredPlanOfBuildAgentIsWhatAHeldFirstBuildWaitsFor(t *testing.T) {
|
||||
plan := func(state, walk string, modules map[string]*inventory.PlanModule) inventory.Plan {
|
||||
return inventory.Plan{ID: "p", State: state, WalkSaid: walk, Modules: modules}
|
||||
}
|
||||
built := map[string]*inventory.PlanModule{"build-agent": {State: "built"}}
|
||||
if !deliversBuildAgent(plan(inventory.PlanDone, inventory.WalkEnded, built)) {
|
||||
t.Fatal("1173/AC3: a plan that delivered build-agent is not the end a held first build waits for")
|
||||
}
|
||||
for what, p := range map[string]inventory.Plan{
|
||||
"a plan still running": plan(inventory.PlanBuilding, "", built),
|
||||
"a plan that failed": plan(inventory.PlanFailed, inventory.WalkEnded, built),
|
||||
"a failed build-agent": plan(inventory.PlanDone, inventory.WalkEnded, map[string]*inventory.PlanModule{"build-agent": {State: "failed"}}),
|
||||
"another module's plan": plan(inventory.PlanDone, inventory.WalkEnded, map[string]*inventory.PlanModule{"mailu": {State: "built"}}),
|
||||
} {
|
||||
if deliversBuildAgent(p) {
|
||||
t.Fatalf("1173/AC3: %s is the end a held first build waits for", what)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -5,71 +5,48 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"slices"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// 1146/AC7, whole against a real bus and the store, in both orders of the two events (novox/hq ADR 0432, issue 1146):
|
||||
// mesh-delivery #7 merged on the operator's Approve into a repository the mesh builds nothing from. The forge's merge,
|
||||
// heard as the controller hears it (SourceMoved), and mesh-delivery's word, published on the subject the controller
|
||||
// consumes with the router's record of the ask in its bucket, come in either order; each way the first build of the
|
||||
// module the merge commit holds is asked exactly once, its outcome registers the module, and it is in the catalogue.
|
||||
// Without the Approve in the record nothing is asked. Store-backed: skips without one.
|
||||
func TestAMergeOnTheApproveIntoANewRepositoryAsksTheFirstBuildAndItsOutcomeRegistersTheModule(t *testing.T) {
|
||||
// 1146/AC7 on keys (ADR 0501 §3, §6; issue 1851, A5), whole against a real bus and the store, in both orders of the two
|
||||
// events: mesh-delivery #7 merged on the operator's Approve into a repository the mesh builds nothing from. The forge's
|
||||
// merge, heard as the controller hears it (SourceMoved), and mesh-delivery's word, published on the subject the
|
||||
// controller consumes with the router's record of the ask in its bucket, come in either order; each way the module the
|
||||
// merge commit holds is registered from the builder's done key of it, and nothing asks a build. Without the Approve in
|
||||
// the record it is not registered. Store-backed: skips without one.
|
||||
func TestAMergeOnTheApproveIntoANewRepositoryRegistersItsModuleFromItsKey(t *testing.T) {
|
||||
const head, commit = "a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1", "0177ea7a00000000"
|
||||
for _, c := range []struct {
|
||||
what string
|
||||
wordFirst bool
|
||||
approved bool // the router's record holds the operator's Approve of the merged head
|
||||
wantRegiter bool
|
||||
what string
|
||||
wordFirst bool
|
||||
approved bool // the router's record holds the operator's Approve of the merged head
|
||||
wantRegister bool
|
||||
}{
|
||||
{what: "the forge's merge before mesh-delivery's word", approved: true, wantRegiter: true},
|
||||
{what: "mesh-delivery's word before the forge's merge", wordFirst: true, approved: true, wantRegiter: true},
|
||||
{what: "the forge's merge before mesh-delivery's word", approved: true, wantRegister: true},
|
||||
{what: "mesh-delivery's word before the forge's merge", wordFirst: true, approved: true, wantRegister: true},
|
||||
{what: "a word whose record holds no Approve", wordFirst: true},
|
||||
} {
|
||||
t.Run(c.what, func(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
// The real ask, guard and queue included: only its answers are kept, so the build's outcome can be matched.
|
||||
aForgeAndABuildSeat(t, open)
|
||||
// Written by the controller's merge path on its own goroutine, read by the test: under one lock.
|
||||
var mu sync.Mutex
|
||||
var asks [][3]string
|
||||
var ids []string
|
||||
asked := func() [][3]string { mu.Lock(); defer mu.Unlock(); return slices.Clone(asks) }
|
||||
realAsk := askABuild
|
||||
askABuild = func(c context.Context, source buildSource, path, ref string) (string, error) {
|
||||
id, err := realAsk(c, source, path, ref)
|
||||
if err == nil {
|
||||
mu.Lock()
|
||||
asks = append(asks, [3]string{source.Repository, path, ref})
|
||||
ids = append(ids, id)
|
||||
mu.Unlock()
|
||||
}
|
||||
return id, err
|
||||
}
|
||||
t.Cleanup(func() { askABuild = realAsk })
|
||||
noBuildAsked(t)
|
||||
b := aBuilderOnTheBus(t)
|
||||
js := b.js
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "operator-channel", Version: "1",
|
||||
DefinesSeats: []catalogue.SeatDeclaration{{Name: broker.AsksSeat, Records: []string{"asks"}}}},
|
||||
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/operator-channel", BuiltFrom: "c0ffee"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
js, err := broker.Dial(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
if err := broker.AssertMeshStreams(js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -87,10 +64,14 @@ func TestAMergeOnTheApproveIntoANewRepositoryAsksTheFirstBuildAndItsOutcomeRegis
|
||||
a := aDeliveryMergeOf(t, head, map[string]string{"novox/mesh-delivery@a1a1a1a1a1a1": head})
|
||||
if c.approved {
|
||||
record, _ := json.Marshal(map[string]any{"state": "closed", "ask": a.ask, "warrant": a.warrant})
|
||||
if _, err := records.Put(ctx, theDelivery+"."+a.merge.AskID, record); err != nil {
|
||||
if _, err := records.Put(ctx, sdkdelivery.TellerModule+"."+a.merge.AskID, record); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// The indexer's value and the builder's done key of the module the merge commit holds.
|
||||
fp := fpOf('9')
|
||||
manifest := b.indexed(t, "mesh-delivery-view", "1", "novox/mesh-delivery", ".", commit, fp)
|
||||
b.built(t, "mesh-delivery-view", "1", fp, manifest, sdkdelivery.BuildDone)
|
||||
|
||||
following := following{open: open}
|
||||
server := link.ConnectNats(js, nil, noListener{})
|
||||
@@ -104,8 +85,6 @@ func TestAMergeOnTheApproveIntoANewRepositoryAsksTheFirstBuildAndItsOutcomeRegis
|
||||
done := make(chan struct{})
|
||||
go func() { defer close(done); _ = server.Serve(serving) }()
|
||||
t.Cleanup(func() { stop(); <-done })
|
||||
servingBus.Store(js)
|
||||
t.Cleanup(func() { servingBus.Store(nil) })
|
||||
|
||||
word := func() {
|
||||
t.Helper()
|
||||
@@ -133,37 +112,27 @@ func TestAMergeOnTheApproveIntoANewRepositoryAsksTheFirstBuildAndItsOutcomeRegis
|
||||
forge()
|
||||
} else {
|
||||
forge()
|
||||
if len(asked()) != 0 {
|
||||
t.Fatalf("1146/AC7: the merge asked %v before mesh-delivery's word was kept", asked())
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
if _, known := inventoryHolds(t, inv, "mesh-delivery-view"); known {
|
||||
t.Fatal("1146/AC7: registered before mesh-delivery's word was kept")
|
||||
}
|
||||
word()
|
||||
}
|
||||
if !c.wantRegiter {
|
||||
if len(asked()) != 0 {
|
||||
t.Fatalf("1146/AC7: a merge whose record holds no Approve asked %v", asked())
|
||||
if !c.wantRegister {
|
||||
time.Sleep(time.Second)
|
||||
if _, known := inventoryHolds(t, inv, "mesh-delivery-view"); known {
|
||||
t.Fatal("1146/AC7: a merge whose record holds no Approve registered its module")
|
||||
}
|
||||
return
|
||||
}
|
||||
want := [3]string{"novox/mesh-delivery", "", "main"}
|
||||
becomes(t, 10*time.Second, "the first build asked", func() bool { return len(asked()) >= 1 })
|
||||
if got := asked(); len(got) != 1 || got[0] != want {
|
||||
t.Fatalf("1146/AC7: the merge asked %v, want %v once", got, want)
|
||||
}
|
||||
mu.Lock()
|
||||
first := ids[0]
|
||||
mu.Unlock()
|
||||
build := onTrunk(first, "novox/mesh-delivery", "git", "", map[string]any{"module": "mesh-delivery-view", "version": "1"})
|
||||
build.Commit = commit
|
||||
if _, _, err := takeIn(ctx, inv, build); err != nil {
|
||||
t.Fatalf("1146/AC7: the outcome of the first build was refused: %v", err)
|
||||
}
|
||||
becomes(t, 10*time.Second, "the module registered", func() bool {
|
||||
_, known := inventoryHolds(t, inv, "mesh-delivery-view")
|
||||
return known
|
||||
})
|
||||
src, err := inv.SourceOf(ctx, "mesh-delivery-view")
|
||||
if err != nil || src.Repository != "novox/mesh-delivery" {
|
||||
t.Fatalf("1146/AC7: the module is built from %q (%v)", src.Repository, err)
|
||||
}
|
||||
if _, known := inventoryHolds(t, inv, "mesh-delivery-view"); !known {
|
||||
t.Fatal("1146/AC7: the module is not in the catalogue")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -64,7 +64,7 @@ func registerFrom(t *testing.T, forms *sourceForms) (int64, error) {
|
||||
mesh := &aMeshRegisteredFromTheURL{anEmptyMesh{asked: true,
|
||||
request: inventory.BuildRequest{ID: "build-1", Repository: aRepoOnTheForge, For: "build"}}, aRepoOnTheForge}
|
||||
return mayRegisterIn(t.Context(), mesh, forms, "mesh-delivery-view", inventory.Source{Repository: aRepoOnTheForge},
|
||||
"build-1", "", "main")
|
||||
byBuild("build-1"), "", "main")
|
||||
}
|
||||
|
||||
// 1277/AC2: an address unread twice, then read, is waited out and the forge is asked.
|
||||
@@ -112,7 +112,7 @@ func TestAKnownAddressAndAnotherHostIsStillNotOnTheForge(t *testing.T) {
|
||||
const elsewhere = "http://elsewhere.example/x/y.git"
|
||||
mesh := &aMeshRegisteredFromTheURL{anEmptyMesh{asked: true, request: inventory.BuildRequest{ID: "build-1", For: "build"}}, elsewhere}
|
||||
_, err := mayRegisterIn(t.Context(), mesh, sourceFormsRead(context.Background(), time.Now, read), "mesh-delivery-view",
|
||||
inventory.Source{Repository: elsewhere}, "build-1", "", "main")
|
||||
inventory.Source{Repository: elsewhere}, byBuild("build-1"), "", "main")
|
||||
if !errors.Is(err, errNotItsSource) || !strings.Contains(err.Error(), "is not on the mesh's forge") {
|
||||
t.Fatalf("1277/AC4: the refusal changed: %v", err)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
package main
|
||||
|
||||
import "testing"
|
||||
|
||||
// C2 (ADR 0503 §3; issue 1299): a delivery group's composed check is answered as the checker's, and nothing is asked
|
||||
// of build-agent: the command answers before any ask is composed.
|
||||
func TestTheGroupCheckNamesTheCheckerAndAsksNothing(t *testing.T) {
|
||||
a := groupCheckAnswer("feat/x")
|
||||
if a["judged-by"] != "mesh-delivery-checker" || a["asked"] != "" || a["group"] != "feat/x" {
|
||||
t.Fatalf("the group check answered %v", a)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,460 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"path"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
"github.com/novox/mesh-controller/internal/artifacts"
|
||||
"github.com/novox/mesh-controller/internal/builderkeys"
|
||||
"github.com/novox/mesh-controller/internal/buildreads"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The take-in from the builder's done key (novox/hq ADR 0501, ADR 0498 §8, ADR 0453 §5; issue 1851).
|
||||
//
|
||||
// From R4e the controller asks no build but the builder's own module's: a module of a merge is taken from the builder's
|
||||
// key for the fingerprint the indexer's trunk value of the module names (ADR 0501 §2). Only a done key whose every
|
||||
// artifact names a reference is taken (ADR 0498 §8); a key not done yet, one waiting for a reference, a templates
|
||||
// archive the store does not answer, or a list of predefined keys the forge does not answer is waited for and said,
|
||||
// never failed and never delivered from. A failed key fails, in the builder's own words. The take-in then records the
|
||||
// key as a build of today's record layout, so a controller of before reads it on a rollback, and judges it as every
|
||||
// build is judged (registerJudged).
|
||||
|
||||
// errKeyWaits says a take-in from a key waits: for the builder, a reference, the store or the forge. Never a failure.
|
||||
var errKeyWaits = errors.New("waits")
|
||||
|
||||
// errKeyFailed says the module's key failed: the builder's own words follow.
|
||||
var errKeyFailed = errors.New("the builder's key failed")
|
||||
|
||||
// keyReads is what a take-in from a key reads of the builder and the indexer (builderkeys.Reader, or a test's).
|
||||
type keyReads interface {
|
||||
Build(ctx context.Context, module, fp string) (builderkeys.Key, error)
|
||||
Manifest(ctx context.Context, digest string) (sdkdelivery.Manifest, error)
|
||||
}
|
||||
|
||||
// keyBuildID is the id a key's build is recorded under: the key and its revision, so the same revision recorded twice
|
||||
// is one record, and a rebuild of the same fingerprint (a new revision, ADR 0429 §18) another.
|
||||
func keyBuildID(k builderkeys.Key) string {
|
||||
part, err := sdkdelivery.FingerprintKeyPart(k.Fingerprint)
|
||||
if err != nil {
|
||||
part = strings.TrimPrefix(k.Fingerprint, sdkdelivery.SourcePrefix2)
|
||||
}
|
||||
return fmt.Sprintf("key-%s-%d", part, k.Revision)
|
||||
}
|
||||
|
||||
// readArchive is the archive a key's reference names, read from the artifact store and checked against its digest. A
|
||||
// variable so a test needs no store.
|
||||
var readArchive = func(ctx context.Context, inv *inventory.Inventory, reference string) ([]byte, error) {
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
address, err := artifactStoreAddress(ctx, inv, shelf, "")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if address == "" {
|
||||
return nil, errors.New("the artifact store is on no network the controller reaches")
|
||||
}
|
||||
return artifacts.Store{Address: address}.GetBlob(ctx, reference)
|
||||
}
|
||||
|
||||
// predefinedHead is the list of predefined keys at the head of the catalogue repository's trunk, and that head.
|
||||
type predefinedHead struct {
|
||||
List catalogue.PredefinedList
|
||||
Repository string
|
||||
Commit string
|
||||
// Generation orders two heads of the trunk: the head's commit time in seconds. A list kept from a build's
|
||||
// generation (its commit count) is always older than a head read here.
|
||||
Generation int
|
||||
}
|
||||
|
||||
// predefinedAtTrunk is the list of predefined keys at the head of the catalogue repository's trunk (ADR 0501 §8), read
|
||||
// from the forge. A variable so a test needs no forge.
|
||||
var predefinedAtTrunk = func(ctx context.Context, inv *inventory.Inventory) (predefinedHead, error) {
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return predefinedHead{}, err
|
||||
}
|
||||
repository := ""
|
||||
for _, e := range entries {
|
||||
if e.Source.Seat == gitSeat && path.Base(strings.Trim(e.Source.Repository, "/")) == catalogue.CatalogueRepository {
|
||||
repository = strings.Trim(e.Source.Repository, "/")
|
||||
break
|
||||
}
|
||||
}
|
||||
if repository == "" {
|
||||
return predefinedHead{}, fmt.Errorf("no module of the catalogue names the repository %s on the forge",
|
||||
catalogue.CatalogueRepository)
|
||||
}
|
||||
var heads []struct {
|
||||
SHA string `json:"sha"`
|
||||
Commit struct {
|
||||
Committer struct {
|
||||
Date time.Time `json:"date"`
|
||||
} `json:"committer"`
|
||||
} `json:"commit"`
|
||||
}
|
||||
if err := askGitea(ctx, "gitea_api", map[string]any{"path": "/repos/" + repository +
|
||||
"/commits?limit=1&stat=false&verification=false&files=false"}, &heads); err != nil {
|
||||
return predefinedHead{}, err
|
||||
}
|
||||
if len(heads) == 0 || heads[0].SHA == "" {
|
||||
return predefinedHead{}, fmt.Errorf("the forge named no head of %s's trunk", repository)
|
||||
}
|
||||
head := predefinedHead{Repository: repository, Commit: heads[0].SHA,
|
||||
Generation: int(heads[0].Commit.Committer.Date.Unix())}
|
||||
var file struct {
|
||||
Content string `json:"content"`
|
||||
Encoding string `json:"encoding"`
|
||||
}
|
||||
if err := askGitea(ctx, "gitea_api", map[string]any{"path": "/repos/" + repository + "/contents/" +
|
||||
catalogue.PredefinedFile + "?ref=" + head.Commit}, &file); err != nil {
|
||||
return predefinedHead{}, err
|
||||
}
|
||||
if file.Encoding != "base64" {
|
||||
return predefinedHead{}, fmt.Errorf("the forge answered %s at %.8s in the encoding %q", catalogue.PredefinedFile,
|
||||
head.Commit, file.Encoding)
|
||||
}
|
||||
raw, err := base64.StdEncoding.DecodeString(file.Content)
|
||||
if err != nil {
|
||||
return predefinedHead{}, err
|
||||
}
|
||||
if head.List, err = catalogue.ParsePredefined(raw); err != nil {
|
||||
return predefinedHead{}, fmt.Errorf("%s at %.8s is refused: %w", catalogue.PredefinedFile, head.Commit, err)
|
||||
}
|
||||
return head, nil
|
||||
}
|
||||
|
||||
// askGitea asks the forge's module one tool on the bus and reads its result into into.
|
||||
func askGitea(ctx context.Context, tool string, args map[string]any, into any) error {
|
||||
js, err := aBus()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer js.Close()
|
||||
raw, _ := json.Marshal(args)
|
||||
answer, err := link.Ask(ctx, link.OverNATS{Conn: js.Conn()}, "gitea", tool, raw, 30*time.Second)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if answer.Error != "" {
|
||||
return fmt.Errorf("gitea.%s: %s", tool, answer.Error)
|
||||
}
|
||||
var wrapped struct {
|
||||
Result json.RawMessage `json:"result"`
|
||||
}
|
||||
if err := json.Unmarshal(answer.Result, &wrapped); err != nil {
|
||||
return err
|
||||
}
|
||||
return json.Unmarshal(wrapped.Result, into)
|
||||
}
|
||||
|
||||
// keyTake is one take-in from a key: the module, the indexer's trunk value naming the key's fingerprint, and the
|
||||
// registration's warrant.
|
||||
type keyTake struct {
|
||||
Module string
|
||||
Trunk sdkdelivery.Module
|
||||
Ask registrationAsk
|
||||
}
|
||||
|
||||
// takeInKey takes a module in from the builder's key for the fingerprint its trunk value names: records the key as a
|
||||
// build and registers the module, or says why not. errKeyWaits when it waits, errKeyFailed when the key failed.
|
||||
func takeInKey(ctx context.Context, inv *inventory.Inventory, keys keyReads, t keyTake) (
|
||||
catalogue.Manifest, inventory.Build, error) {
|
||||
head, err := predefinedAtTrunk(ctx, inv)
|
||||
if err != nil {
|
||||
return catalogue.Manifest{}, inventory.Build{}, fmt.Errorf("%w: the list of predefined keys at the catalogue's "+
|
||||
"trunk head cannot be read, so %s is not judged: %v", errKeyWaits, t.Module, err)
|
||||
}
|
||||
keepPredefinedHead(ctx, inv, head)
|
||||
previous, err := inv.Builds(ctx, t.Module, 5)
|
||||
if err != nil {
|
||||
return catalogue.Manifest{}, inventory.Build{}, err
|
||||
}
|
||||
kept, manifest, err := buildOfKey(ctx, keys, t, keyContext{list: head.List, previous: previous,
|
||||
archive: func(ctx context.Context, reference string) ([]byte, error) { return readArchive(ctx, inv, reference) }})
|
||||
if err != nil {
|
||||
return catalogue.Manifest{}, kept, err
|
||||
}
|
||||
if err := inv.RecordBuild(ctx, kept); err != nil {
|
||||
return catalogue.Manifest{}, kept, err
|
||||
}
|
||||
recorded := inventory.Source{Repository: t.Trunk.Repository, Seat: gitSeat, Path: keyPath(t.Trunk),
|
||||
BuiltFrom: t.Trunk.Commit, Head: t.Trunk.Commit, Against: kept.Against}
|
||||
trunk := "main"
|
||||
if was, err := inv.SourceOf(ctx, manifest.Module); err == nil {
|
||||
recorded.Ref = followedBranch(was.Ref)
|
||||
if recorded.Ref != "" {
|
||||
trunk = recorded.Ref
|
||||
}
|
||||
}
|
||||
registered, err := registerJudged(ctx, inv, takenBuild{kept: kept, manifest: manifest, recorded: recorded,
|
||||
ask: t.Ask, path: recorded.Path, trunk: trunk, by: "the builder", what: t.Trunk.Repository,
|
||||
commit: t.Trunk.Commit})
|
||||
return registered, kept, err
|
||||
}
|
||||
|
||||
// keyContext is what a key's build is made with besides the key: the list of predefined keys its manifest is judged
|
||||
// against, the module's recent builds (the bases its last one stood on are carried) and how an archive is read.
|
||||
type keyContext struct {
|
||||
list catalogue.PredefinedList
|
||||
previous []inventory.Build
|
||||
archive func(ctx context.Context, reference string) ([]byte, error)
|
||||
}
|
||||
|
||||
// keyPath is a module's directory as the catalogue records it: empty for a module at its repository's root.
|
||||
func keyPath(m sdkdelivery.Module) string {
|
||||
if m.Path == sdkdelivery.RootPath {
|
||||
return ""
|
||||
}
|
||||
return strings.Trim(m.Path, "/")
|
||||
}
|
||||
|
||||
// buildOfKey reads the key of the trunk value's fingerprint and makes its build record and the manifest it resolves to
|
||||
// (ADR 0498 §8, ADR 0501 §1, §8): the module's manifest by the key's digest, judged against the list of predefined keys
|
||||
// at the catalogue's trunk head, resolved with the key's artifacts.
|
||||
func buildOfKey(ctx context.Context, keys keyReads, t keyTake, kc keyContext) (
|
||||
inventory.Build, catalogue.Manifest, error) {
|
||||
waits := func(format string, a ...any) (inventory.Build, catalogue.Manifest, error) {
|
||||
return inventory.Build{}, catalogue.Manifest{}, fmt.Errorf("%w: "+format, append([]any{errKeyWaits}, a...)...)
|
||||
}
|
||||
if t.Trunk.FP2 == "" {
|
||||
why := t.Trunk.None2
|
||||
if why == "" {
|
||||
why = "the indexer wrote no fingerprint of the new form"
|
||||
}
|
||||
return waits("%s at %.8s has no fingerprint to take a build by: %s", t.Module, t.Trunk.Commit, why)
|
||||
}
|
||||
k, err := keys.Build(ctx, t.Module, t.Trunk.FP2)
|
||||
if err != nil {
|
||||
return waits("%s's key cannot be read: %v", t.Module, err)
|
||||
}
|
||||
switch {
|
||||
case k.Failed():
|
||||
return inventory.Build{}, catalogue.Manifest{}, fmt.Errorf("%w: %s", errKeyFailed, k.Says())
|
||||
case !k.Done():
|
||||
return waits("%s", k.Says())
|
||||
case len(k.Unreferenced()) > 0:
|
||||
return waits("%s", k.Says())
|
||||
}
|
||||
src, err := keys.Manifest(ctx, k.Build.Manifest)
|
||||
if err != nil {
|
||||
return waits("%s's manifest: %v", t.Module, err)
|
||||
}
|
||||
manifest, err := catalogue.ParseManifestWith([]byte(src.JSON), kc.list)
|
||||
if err != nil {
|
||||
// **A word the running controller does not read waits for the controller that reads it** (ADR 0501 §7): the
|
||||
// key stays done, and the controller that reads the word registers it with no second build.
|
||||
if word := refusedWord(err.Error()); word != "" {
|
||||
return waits("%s's manifest uses the word %q, which this controller does not read: it is registered "+
|
||||
"when a controller that reads it runs", t.Module, word)
|
||||
}
|
||||
return inventory.Build{}, catalogue.Manifest{}, fmt.Errorf("the builder built %s and its manifest is "+
|
||||
"refused: %w", t.Module, err)
|
||||
}
|
||||
if manifest.Module != t.Module {
|
||||
return inventory.Build{}, catalogue.Manifest{}, fmt.Errorf("the key of %s names the manifest of %s",
|
||||
t.Module, manifest.Module)
|
||||
}
|
||||
built, err := builtOfKey(ctx, kc.archive, manifest, k)
|
||||
if err != nil {
|
||||
return waits("%v", err)
|
||||
}
|
||||
resolved, err := manifest.Resolve(built)
|
||||
if err != nil {
|
||||
return inventory.Build{}, catalogue.Manifest{}, fmt.Errorf("the builder built %s and it does not resolve: %w",
|
||||
t.Module, err)
|
||||
}
|
||||
raw, err := json.Marshal(resolved)
|
||||
if err != nil {
|
||||
return inventory.Build{}, catalogue.Manifest{}, err
|
||||
}
|
||||
kept := inventory.Build{ID: keyBuildID(k), Repository: t.Trunk.Repository, Module: t.Module,
|
||||
Commit: t.Trunk.Commit, On: k.Build.By, Path: keyPath(t.Trunk), Manifest: raw,
|
||||
SourceFingerprint: k.Fingerprint, At: time.Now().UTC()}
|
||||
// **Read from the manifest, fresh at every key** (the same rule build-agent says, internal/buildreads). Sources are
|
||||
// left unsaid but for a module whose every artifact names its paths: the planner then reads the module's whole
|
||||
// directory, over-planning and never under-planning, and a fingerprint that did not change moves nothing (R4e A9).
|
||||
// **Its build source where its every artifact names its paths** (novox/hq ADR 0499 §1, §5; issue 1758, B1): the
|
||||
// planner reads a merge's changed files against those paths and the manifest alone, so a change elsewhere in the
|
||||
// module's directory touches nothing it builds from.
|
||||
if own := pathsBuildSource(manifest, keyPath(t.Trunk)); own != nil {
|
||||
kept.Sources = []inventory.BuildSource{{Paths: own}}
|
||||
}
|
||||
for _, c := range buildreads.Of(manifest) {
|
||||
kept.Read = append(kept.Read, inventory.ReadRepository{Repository: c.Repository, Ref: c.Ref})
|
||||
}
|
||||
// **The bases its last build stood on, carried** (R4e, lead decision): what the collector keeps a mirrored base by.
|
||||
// A key names none; the module's newest successful build's are carried so the keep set does not change.
|
||||
for _, b := range kc.previous {
|
||||
if b.Worked() && b.Module == t.Module && b.ID != kept.ID {
|
||||
kept.Against = append([]string(nil), b.Against...)
|
||||
break
|
||||
}
|
||||
}
|
||||
for _, a := range k.Build.Artifacts {
|
||||
kept.Made = append(kept.Made, inventory.Artifact{Name: a.Name, Kind: kindOf(manifest, a),
|
||||
Reference: catalogue.Recorded(a.Reference)})
|
||||
}
|
||||
return kept, resolved, nil
|
||||
}
|
||||
|
||||
// kindOf is an artifact's kind: the key's, else the manifest's.
|
||||
func kindOf(m catalogue.Manifest, a sdkdelivery.BuiltArtifact) string {
|
||||
if a.Kind != "" || m.Build == nil {
|
||||
return a.Kind
|
||||
}
|
||||
for _, d := range m.Build.Artifacts {
|
||||
if d.Name == a.Name {
|
||||
return d.Kind
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// builtOfKey is each of a done key's artifacts as Resolve reads it: its reference and digest, a TypeScript bundle's
|
||||
// launchers by the build modules' one rule (mesh-sdk buildstep.Launchers), and, for the archive a module packs its
|
||||
// templates into, each template's text read back from the archive the store holds under the key's reference.
|
||||
func builtOfKey(ctx context.Context, archive func(context.Context, string) ([]byte, error), m catalogue.Manifest,
|
||||
k builderkeys.Key) (
|
||||
[]catalogue.Built, error) {
|
||||
declared := map[string]catalogue.Artifact{}
|
||||
if m.Build != nil {
|
||||
for _, a := range m.Build.Artifacts {
|
||||
declared[a.Name] = a
|
||||
}
|
||||
}
|
||||
templates := ""
|
||||
if m.Templated() {
|
||||
templates = catalogue.TemplatesArtifact(m)
|
||||
}
|
||||
var out []catalogue.Built
|
||||
for _, a := range k.Build.Artifacts {
|
||||
d := declared[a.Name]
|
||||
b := catalogue.Built{Name: a.Name, Kind: kindOf(m, a), Reference: catalogue.Recorded(a.Reference),
|
||||
Digest: a.Digest}
|
||||
if launchers := buildstep.Launchers(d.Language, d.Entrypoints); len(launchers) > 0 {
|
||||
b.Launchers = launchers
|
||||
}
|
||||
if a.Name == templates && templates != "" {
|
||||
body, err := archive(ctx, b.Reference)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the templates archive %q of %s cannot be read from the artifact store: %v",
|
||||
a.Name, m.Module, err)
|
||||
}
|
||||
if b.Texts, err = unpackTexts(body); err != nil {
|
||||
return nil, fmt.Errorf("the templates archive %q of %s cannot be read back: %v", a.Name, m.Module, err)
|
||||
}
|
||||
}
|
||||
out = append(out, b)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// unpackTexts is every file of a packed templates archive (gzip over tar), by its path inside it.
|
||||
func unpackTexts(body []byte) (map[string]string, error) {
|
||||
zipped, err := gzip.NewReader(bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer zipped.Close()
|
||||
reader := tar.NewReader(zipped)
|
||||
texts := map[string]string{}
|
||||
for {
|
||||
header, err := reader.Next()
|
||||
if errors.Is(err, io.EOF) {
|
||||
return texts, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if header.Typeflag != tar.TypeReg {
|
||||
continue
|
||||
}
|
||||
text, err := io.ReadAll(io.LimitReader(reader, 16<<20))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
texts[strings.TrimPrefix(header.Name, "./")] = string(text)
|
||||
}
|
||||
}
|
||||
|
||||
// keepPredefinedHead keeps the list read at the catalogue's trunk head when it is newer than the one kept, as a build of
|
||||
// the catalogue's did (ADR 0348 §7), and judges every manifest read again against it. Said, never fatal: the list read
|
||||
// is the one this take-in judges against either way.
|
||||
func keepPredefinedHead(ctx context.Context, inv *inventory.Inventory, head predefinedHead) {
|
||||
kept, err := inv.KeepPredefined(ctx, inventory.PredefinedRecord{Keys: head.List, Repository: head.Repository,
|
||||
Commit: head.Commit, Generation: head.Generation, Build: "the trunk head " + short(head.Commit)})
|
||||
if err != nil {
|
||||
fmt.Printf("the list of predefined keys at %s's trunk head %s is not kept: %v\n", head.Repository,
|
||||
short(head.Commit), err)
|
||||
return
|
||||
}
|
||||
if kept {
|
||||
fmt.Printf("the list of predefined keys at %s's trunk head %s is published: %s\n", head.Repository,
|
||||
short(head.Commit), strings.Join(head.List.Keys(), ", "))
|
||||
}
|
||||
if err := loadPredefined(ctx, inv); err != nil {
|
||||
fmt.Printf("the list of predefined keys kept cannot be read: %v\n", err)
|
||||
}
|
||||
}
|
||||
|
||||
// openBuilderKeys is the builder's and the indexer's buckets, read on a connection for the moment.
|
||||
func openBuilderKeys() (builderkeys.Reader, func(), error) {
|
||||
js, err := aBus()
|
||||
if err != nil {
|
||||
return builderkeys.Reader{}, nil, err
|
||||
}
|
||||
api, err := jetstream.New(js.Conn())
|
||||
if err != nil {
|
||||
js.Close()
|
||||
return builderkeys.Reader{}, nil, err
|
||||
}
|
||||
return builderkeys.Reader{JS: api}, js.Close, nil
|
||||
}
|
||||
|
||||
// pathsBuildSource is the build source of a module whose every artifact names the paths it is built from (ADR 0499
|
||||
// §1): its manifest and each path, relative to its repository's root, a path read as a file or everything below a
|
||||
// directory of that name (`<path>/**` holds both). Nil when an artifact names none: the module reads its directory.
|
||||
func pathsBuildSource(m catalogue.Manifest, dir string) []string {
|
||||
if m.Build == nil || len(m.Build.Artifacts) == 0 {
|
||||
return nil
|
||||
}
|
||||
in := func(p string) string {
|
||||
if dir == "" {
|
||||
return p
|
||||
}
|
||||
return dir + "/" + p
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
out := []string{in("module.json")}
|
||||
for _, a := range m.Build.Artifacts {
|
||||
if len(a.Paths) == 0 {
|
||||
return nil
|
||||
}
|
||||
for _, p := range a.Paths {
|
||||
entry := in(strings.Trim(path.Clean(p), "/")) + "/**"
|
||||
if !seen[entry] {
|
||||
seen[entry] = true
|
||||
out = append(out, entry)
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,251 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"github.com/novox/mesh-controller/internal/builderkeys"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// withTrunkList stands in for the catalogue's trunk head: an empty list of predefined keys, readable, or not.
|
||||
func withTrunkList(t *testing.T, readable bool) {
|
||||
t.Helper()
|
||||
was := predefinedAtTrunk
|
||||
predefinedAtTrunk = func(context.Context, *inventory.Inventory) (predefinedHead, error) {
|
||||
if !readable {
|
||||
return predefinedHead{}, errors.New("the forge did not answer")
|
||||
}
|
||||
list, err := catalogue.ParsePredefined([]byte(`{}`))
|
||||
return predefinedHead{List: list, Repository: "novox/mesh-catalog", Commit: "abcdef0123456789",
|
||||
Generation: 1 << 40}, err
|
||||
}
|
||||
t.Cleanup(func() { predefinedAtTrunk = was })
|
||||
}
|
||||
|
||||
// aKeyFor is a done key of module at fp, of one archive with a reference, and the indexer's trunk value naming it at
|
||||
// commit in repository/path.
|
||||
func aKeyFor(module, version, fp, repository, path, commit string) (keyBuckets, sdkdelivery.Module) {
|
||||
raw := `{"module":"` + module + `","version":"` + version + `","build":{"artifacts":[{"name":"a","kind":"archive","from":"files"}]},` +
|
||||
`"resources":[{"id":"a","type":"archive","path":"/opt/` + module + `","artifact":"a"}]}`
|
||||
digest := sdkdelivery.ManifestDigest([]byte(raw))
|
||||
blob := shaOf(module + version)
|
||||
k := builderkeys.Key{Module: module, Fingerprint: fp, Found: true, Revision: 1,
|
||||
Build: sdkdelivery.BuilderBuild{State: sdkdelivery.BuildDone, By: "ace", Manifest: digest,
|
||||
Artifacts: []sdkdelivery.BuiltArtifact{{Name: "a", Kind: "archive", Digest: blob,
|
||||
Reference: "artifact-store://" + module + "/a/blobs/" + blob}}}}
|
||||
return keyBuckets{keys: map[string]builderkeys.Key{module: k}, manifests: map[string]string{digest: raw}},
|
||||
sdkdelivery.Module{FP2: fp, Manifest: digest, Repository: repository, Commit: commit, Path: path}
|
||||
}
|
||||
|
||||
func fpOf(c byte) string { return "src2:" + strings.Repeat(string(c), 64) }
|
||||
|
||||
// A3, A4 (ADR 0498 §8, ADR 0501 §1-§3, §8): a done key of a trunk fingerprint records one build of today's layout,
|
||||
// whose artifacts are the key's, and registers the module under the same rules of who may register as before.
|
||||
func TestADoneKeyOfTheTrunkRegistersUnderTheSameRules(t *testing.T) {
|
||||
open := theCatalogue(t) // sudo from novox/mesh-catalog modules/sudo, mesh-host from novox/mesh-host
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
withTrunkList(t, true)
|
||||
|
||||
// A known module from the repository and path the catalogue builds it from: registered from its key.
|
||||
keys, trunk := aKeyFor("sudo", "2", fpOf('a'), "novox/mesh-catalog", "modules/sudo", "c0ffee0123456789")
|
||||
_, kept, err := takeInKey(ctx, inv, keys, keyTake{Module: "sudo", Trunk: trunk, Ask: registrationAsk{Merge: trunk.Commit}})
|
||||
if err != nil {
|
||||
t.Fatalf("0501 §1: a done key of sudo's trunk fingerprint was not taken in: %v", err)
|
||||
}
|
||||
if shelf, _ := inv.Catalogue(ctx); shelf["sudo"].Version != "2" {
|
||||
t.Fatalf("0501 §1: sudo is %q", shelf["sudo"].Version)
|
||||
}
|
||||
got, found, err := inv.BuildByID(ctx, kept.ID)
|
||||
if err != nil || !found || len(got.Made) != 1 || got.Made[0].Reference != kept.Made[0].Reference {
|
||||
t.Fatalf("0498 §8: the recorded build is %+v (%v)", got, err)
|
||||
}
|
||||
if fps, err := inv.BuildSourceFingerprints(ctx, "sudo"); err != nil || fps["c0ffee0123456789"] != fpOf('a') {
|
||||
t.Fatalf("0498 §8: the recorded build's fingerprint is %v (%v), not the key's", fps, err)
|
||||
}
|
||||
// Recording the same key again is one record.
|
||||
if _, _, err := takeInKey(ctx, inv, keys, keyTake{Module: "sudo", Trunk: trunk, Ask: registrationAsk{Merge: trunk.Commit}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
builds, _ := inv.Builds(ctx, "sudo", 20)
|
||||
n := 0
|
||||
for _, b := range builds {
|
||||
if b.ID == kept.ID {
|
||||
n++
|
||||
}
|
||||
}
|
||||
if n != 1 {
|
||||
t.Fatalf("0498 §8: one key taken in twice is %d records", n)
|
||||
}
|
||||
|
||||
// A known module from another repository: refused, as a build of it was (ADR 0266).
|
||||
other, otherTrunk := aKeyFor("sudo", "evil", fpOf('b'), "agent/sudo", "modules/sudo", "d00d0123456789ab")
|
||||
if _, _, err := takeInKey(ctx, inv, other, keyTake{Module: "sudo", Trunk: otherTrunk,
|
||||
Ask: registrationAsk{Merge: otherTrunk.Commit}}); !errors.Is(err, errNotItsSource) {
|
||||
t.Fatalf("0501 §3: a known module from another repository was taken in: %v", err)
|
||||
}
|
||||
// A new module from a repository the catalogue builds from: registered.
|
||||
zram, zramTrunk := aKeyFor("zram", "1", fpOf('c'), "novox/mesh-catalog", "modules/zram", "c0ffee0123456789")
|
||||
if _, _, err := takeInKey(ctx, inv, zram, keyTake{Module: "zram", Trunk: zramTrunk, Ask: registrationAsk{Merge: zramTrunk.Commit}}); err != nil {
|
||||
t.Fatalf("0501 §3: a new module from a built repository was refused: %v", err)
|
||||
}
|
||||
// A new module from a repository nothing is built from, with no Approve: refused.
|
||||
tools, toolsTrunk := aKeyFor("agent-tools", "1", fpOf('d'), "agent/tools", "", "e0e0e0e0e0e0e0e0")
|
||||
if _, _, err := takeInKey(ctx, inv, tools, keyTake{Module: "agent-tools", Trunk: toolsTrunk,
|
||||
Ask: registrationAsk{Merge: toolsTrunk.Commit}}); !errors.Is(err, errNotItsSource) {
|
||||
t.Fatalf("0501 §3: a new repository's module was taken in with no warrant: %v", err)
|
||||
}
|
||||
// At the terminal the operator registers it (ADR 0501 §4).
|
||||
if _, _, err := takeInKey(ctx, inv, tools, keyTake{Module: "agent-tools", Trunk: toolsTrunk,
|
||||
Ask: registrationAsk{Terminal: true}}); err != nil {
|
||||
t.Fatalf("0501 §4: the terminal's registration was refused: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// ADR 0501 §2: a key only a candidate names registers nothing: the trunk value names another fingerprint, whose key is
|
||||
// not there, and the take-in waits.
|
||||
func TestAKeyOnlyACandidateNamesRegistersNothing(t *testing.T) {
|
||||
open := theCatalogue(t)
|
||||
ctx := t.Context()
|
||||
withTrunkList(t, true)
|
||||
keys, trunk := aKeyFor("sudo", "3", fpOf('e'), "novox/mesh-catalog", "modules/sudo", "c0ffee0123456789")
|
||||
trunk.FP2 = fpOf('f') // the trunk names another fingerprint than the candidate's done key
|
||||
_, _, err := takeInKey(ctx, open.inventory, keys, keyTake{Module: "sudo", Trunk: trunk, Ask: registrationAsk{Merge: trunk.Commit}})
|
||||
if !errors.Is(err, errKeyWaits) {
|
||||
t.Fatalf("0501 §2: %v", err)
|
||||
}
|
||||
if shelf, _ := open.inventory.Catalogue(ctx); shelf["sudo"].Version != "1" {
|
||||
t.Fatalf("0501 §2: a candidate's key registered sudo %q", shelf["sudo"].Version)
|
||||
}
|
||||
}
|
||||
|
||||
// ADR 0501 §8: a take-in that cannot read the list of predefined keys at the catalogue's trunk head waits, says so and
|
||||
// registers nothing.
|
||||
func TestATakeInWithoutTheTrunksListWaits(t *testing.T) {
|
||||
open := theCatalogue(t)
|
||||
ctx := t.Context()
|
||||
withTrunkList(t, false)
|
||||
keys, trunk := aKeyFor("sudo", "4", fpOf('1'), "novox/mesh-catalog", "modules/sudo", "c0ffee0123456789")
|
||||
_, _, err := takeInKey(ctx, open.inventory, keys, keyTake{Module: "sudo", Trunk: trunk, Ask: registrationAsk{Merge: trunk.Commit}})
|
||||
if !errors.Is(err, errKeyWaits) || !strings.Contains(err.Error(), "predefined keys") {
|
||||
t.Fatalf("0501 §8: %v", err)
|
||||
}
|
||||
if shelf, _ := open.inventory.Catalogue(ctx); shelf["sudo"].Version != "1" {
|
||||
t.Fatalf("0501 §8: registered %q", shelf["sudo"].Version)
|
||||
}
|
||||
}
|
||||
|
||||
// A3 (the lead's decision on Against and Read): a key's record keeps the collector's keep set as it was and its read
|
||||
// repositories are its manifest's, fresh at every key.
|
||||
func TestAKeysRecordKeepsTheKeepSetAndReadsItsManifest(t *testing.T) {
|
||||
open := theCatalogue(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
withTrunkList(t, true)
|
||||
before, err := inv.KeepSet(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
raw := `{"module":"sudo","version":"5","build":{"artifacts":[{"name":"img","kind":"image","from":"Dockerfile",` +
|
||||
`"context":{"repository":"novox/elsewhere","seat":"git"}}]},"resources":[{"id":"run","type":"container","artifact":"img"}]}`
|
||||
digest := sdkdelivery.ManifestDigest([]byte(raw))
|
||||
k := builderkeys.Key{Module: "sudo", Fingerprint: fpOf('2'), Found: true, Revision: 1,
|
||||
Build: sdkdelivery.BuilderBuild{State: sdkdelivery.BuildDone, By: "g14", Manifest: digest,
|
||||
Artifacts: []sdkdelivery.BuiltArtifact{{Name: "img", Kind: "image", Digest: shaOf("img5"),
|
||||
Reference: "artifact-store://sudo/img@" + shaOf("img5")}}}}
|
||||
keys := keyBuckets{keys: map[string]builderkeys.Key{"sudo": k}, manifests: map[string]string{digest: raw}}
|
||||
trunk := sdkdelivery.Module{FP2: fpOf('2'), Manifest: digest, Repository: "novox/mesh-catalog",
|
||||
Commit: "c0ffee0123456789", Path: "modules/sudo"}
|
||||
if _, _, err := takeInKey(ctx, inv, keys, keyTake{Module: "sudo", Trunk: trunk, Ask: registrationAsk{Merge: trunk.Commit}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
after, err := inv.KeepSet(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for reference, why := range before {
|
||||
if !reflect.DeepEqual(after[reference], why) && !strings.Contains(strings.Join(after[reference], ","), strings.Join(why, ",")) {
|
||||
t.Errorf("the keep set changed for %s: %v, was %v", reference, after[reference], why)
|
||||
}
|
||||
}
|
||||
read, err := inv.ReadRepositories(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
found := false
|
||||
for _, r := range read["sudo"] {
|
||||
if !r.Own && r.Repository == "novox/elsewhere" {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
text, _ := json.Marshal(read["sudo"])
|
||||
t.Fatalf("the key's record does not read its manifest's context: %s", text)
|
||||
}
|
||||
}
|
||||
|
||||
// A9 (ADR 0453 §6, ADR 0499 consequences): at the switch, a key's build whose every artifact has the digest of the
|
||||
// build-agent build the mesh holds (`src1:`) is no move: it is registered with that build's artifacts, and a walk sends
|
||||
// nothing for it.
|
||||
func TestAKeyWithTheHeldBuildsDigestsIsNoMove(t *testing.T) {
|
||||
open := theCatalogue(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
withTrunkList(t, true)
|
||||
keys, trunk := aKeyFor("sudo", "6", fpOf('6'), "novox/mesh-catalog", "modules/sudo", "c0ffee0123456789")
|
||||
k := keys.keys["sudo"]
|
||||
held := inventory.Build{ID: "build-sudo-old", Module: "sudo", Repository: "novox/mesh-catalog", Path: "modules/sudo",
|
||||
Commit: "0ld0ld0ld0ld0ld0", SourceFingerprint: "src1:" + strings.Repeat("9", 64),
|
||||
Made: []inventory.Artifact{{Name: "a", Kind: "archive", Reference: k.Build.Artifacts[0].Reference}}}
|
||||
held.Manifest, _ = json.Marshal(catalogue.Manifest{Module: "sudo", Version: "held"})
|
||||
if err := inv.RecordBuild(ctx, held); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, kept, err := takeInKey(ctx, inv, keys, keyTake{Module: "sudo", Trunk: trunk, Ask: registrationAsk{Merge: trunk.Commit}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
same, err := inv.SameSourceCommits(ctx, "sudo", kept.ID)
|
||||
if err != nil || !same["0ld0ld0ld0ld0ld0"] {
|
||||
t.Fatalf("A9: the key's build does not stand on the held build of the same digests: %v %v", same, err)
|
||||
}
|
||||
if shelf, _ := inv.Catalogue(ctx); shelf["sudo"].Version != "held" {
|
||||
t.Fatalf("A9: sudo is registered as %q, not with the held build's artifacts", shelf["sudo"].Version)
|
||||
}
|
||||
}
|
||||
|
||||
// A13 against the store: the switch's list reads the build the mesh holds of each module and the builder's key of its
|
||||
// trunk fingerprint, and names each module unmoved, moved or not known.
|
||||
func TestTheSwitchsListReadsTheHeldBuildsAndTheKeys(t *testing.T) {
|
||||
open := theCatalogue(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
keys, trunk := aKeyFor("sudo", "1", fpOf('b'), "novox/mesh-catalog", "modules/sudo", "c0ffee0123456789")
|
||||
k := keys.keys["sudo"]
|
||||
if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-sudo-held", Module: "sudo", Repository: "novox/mesh-catalog",
|
||||
Commit: "c0ffee0123456789", Made: []inventory.Artifact{{Name: "a", Kind: "archive",
|
||||
Reference: k.Build.Artifacts[0].Reference}}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reads := planKeysOf{values: map[string]sdkdelivery.Module{"sudo@": trunk}, keys: map[string]builderkeys.Key{"sudo": k}}
|
||||
lines, err := switchList(ctx, inv, reads)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := map[string]string{}
|
||||
for _, l := range lines {
|
||||
got[l.Module] = l.Verdict
|
||||
}
|
||||
if got["sudo"] != switchUnmoved || got["mesh-host"] != switchUnknown {
|
||||
t.Fatalf("A13: the switch's list is %+v", lines)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,231 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"github.com/novox/mesh-controller/internal/builderkeys"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// keyBuckets is the builder's and the indexer's buckets as a take-in reads them, for a test: keys by module, manifests by
|
||||
// digest.
|
||||
type keyBuckets struct {
|
||||
keys map[string]builderkeys.Key
|
||||
manifests map[string]string
|
||||
}
|
||||
|
||||
func (k keyBuckets) Build(_ context.Context, module, fp string) (builderkeys.Key, error) {
|
||||
if key, ok := k.keys[module]; ok && key.Fingerprint == fp {
|
||||
return key, nil
|
||||
}
|
||||
return builderkeys.Key{Module: module, Fingerprint: fp}, nil
|
||||
}
|
||||
|
||||
func (k keyBuckets) Manifest(_ context.Context, digest string) (sdkdelivery.Manifest, error) {
|
||||
raw, ok := k.manifests[digest]
|
||||
if !ok {
|
||||
return sdkdelivery.Manifest{}, errors.New("the indexer holds no manifest " + digest)
|
||||
}
|
||||
return sdkdelivery.Manifest{Module: "x", JSON: raw}, nil
|
||||
}
|
||||
|
||||
const keyFP = "src2:" + "1111111111111111111111111111111111111111111111111111111111111111"
|
||||
|
||||
// aKeyOf is a done key of module x at keyFP for the manifest raw, with these artifacts.
|
||||
func aKeyOf(raw string, revision uint64, made ...sdkdelivery.BuiltArtifact) (keyBuckets, sdkdelivery.Module) {
|
||||
digest := sdkdelivery.ManifestDigest([]byte(raw))
|
||||
k := builderkeys.Key{Module: "x", Fingerprint: keyFP, Found: true, Revision: revision,
|
||||
Build: sdkdelivery.BuilderBuild{State: sdkdelivery.BuildDone, By: "ace", Manifest: digest, Artifacts: made}}
|
||||
trunk := sdkdelivery.Module{FP2: keyFP, Manifest: digest, Repository: "novox/mesh-catalog", Commit: "c0ffee0123456789",
|
||||
Path: "modules/x"}
|
||||
return keyBuckets{keys: map[string]builderkeys.Key{"x": k}, manifests: map[string]string{digest: raw}}, trunk
|
||||
}
|
||||
|
||||
const tsBundle = `{"module":"x","version":"1","tools":["x_read"],
|
||||
"build":{"artifacts":[{"name":"tools","kind":"bundle","language":"typescript","entrypoints":["dist/index.js"]},
|
||||
{"name":"img","kind":"image","from":"Dockerfile","context":{"repository":"novox/other","seat":"git"}}]},
|
||||
"resources":[{"id":"run","type":"container","artifact":"img"}]}`
|
||||
|
||||
func shaOf(b string) string {
|
||||
sum := sha256.Sum256([]byte(b))
|
||||
return "sha256:" + hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
// A3 (ADR 0498 §8, ADR 0501 §1): a done key whose every artifact names a reference is one recorded build, its artifacts
|
||||
// the key's, its launchers by the build modules' rule, its reads from its manifest, the bases its last build stood on
|
||||
// carried; recorded twice it is one record.
|
||||
func TestADoneKeyIsOneRecordedBuildOfItsArtifacts(t *testing.T) {
|
||||
keys, trunk := aKeyOf(tsBundle, 3,
|
||||
sdkdelivery.BuiltArtifact{Name: "tools", Kind: "bundle", Digest: shaOf("t"),
|
||||
Reference: "artifact-store://x/tools/blobs/" + shaOf("t")},
|
||||
sdkdelivery.BuiltArtifact{Name: "img", Kind: "image", Digest: shaOf("i"),
|
||||
Reference: "artifact-store://x/img@" + shaOf("i")})
|
||||
previous := []inventory.Build{{ID: "build-1", Module: "x", Against: []string{"artifact-store://base@sha256:aa"}}}
|
||||
take := keyTake{Module: "x", Trunk: trunk}
|
||||
kept, resolved, err := buildOfKey(t.Context(), keys, take, keyContext{previous: previous})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if kept.ID != "key-1111111111111111111111111111111111111111111111111111111111111111-3" {
|
||||
t.Errorf("the record's id is %s", kept.ID)
|
||||
}
|
||||
again, _, _ := buildOfKey(t.Context(), keys, take, keyContext{previous: previous})
|
||||
if again.ID != kept.ID {
|
||||
t.Errorf("one key recorded twice is two records: %s and %s", kept.ID, again.ID)
|
||||
}
|
||||
if kept.SourceFingerprint != keyFP || kept.Commit != trunk.Commit || kept.Path != "modules/x" || kept.On != "ace" ||
|
||||
kept.Module != "x" || kept.Repository != "novox/mesh-catalog" {
|
||||
t.Errorf("the record: %+v", kept)
|
||||
}
|
||||
if len(kept.Made) != 2 || kept.Made[0].Reference != "artifact-store://x/tools/blobs/"+shaOf("t") ||
|
||||
kept.Made[1].Reference != "artifact-store://x/img@"+shaOf("i") || kept.Made[1].Kind != "image" {
|
||||
t.Errorf("Made is not the key's artifacts: %+v", kept.Made)
|
||||
}
|
||||
if len(kept.Read) != 1 || kept.Read[0].Repository != "novox/other" {
|
||||
t.Errorf("Read is not the manifest's contexts: %+v", kept.Read)
|
||||
}
|
||||
if kept.Sources != nil || kept.Mirrored != nil {
|
||||
t.Errorf("a key's record says build sources or mirrored bases it cannot know: %+v %+v", kept.Sources, kept.Mirrored)
|
||||
}
|
||||
if len(kept.Against) != 1 || kept.Against[0] != "artifact-store://base@sha256:aa" {
|
||||
t.Errorf("the bases the last build stood on are not carried: %v", kept.Against)
|
||||
}
|
||||
if len(resolved.Bundles) != 1 || resolved.Bundles[0].Launchers["dist/index.js"] != "dist/index.serve.mjs" {
|
||||
t.Errorf("the bundle's launchers: %+v", resolved.Bundles)
|
||||
}
|
||||
var m map[string]any
|
||||
if err := json.Unmarshal(kept.Manifest, &m); err != nil || !strings.Contains(string(kept.Manifest), shaOf("i")) {
|
||||
t.Errorf("the recorded manifest names no image digest: %s", kept.Manifest)
|
||||
}
|
||||
}
|
||||
|
||||
// A3 (ADR 0498 §8): a key with an artifact naming no reference, a key not done and a missing key are waited for, said;
|
||||
// a failed key fails in the builder's words; a module with no fingerprint of the new form waits, naming why.
|
||||
func TestAKeyNotReadyWaitsAndAFailedKeyFails(t *testing.T) {
|
||||
noRef, trunk := aKeyOf(tsBundle, 1,
|
||||
sdkdelivery.BuiltArtifact{Name: "tools", Kind: "bundle", Digest: shaOf("t"),
|
||||
Reference: "artifact-store://x/tools/blobs/" + shaOf("t")},
|
||||
sdkdelivery.BuiltArtifact{Name: "img", Kind: "image", Digest: shaOf("i")})
|
||||
take := keyTake{Module: "x", Trunk: trunk}
|
||||
_, _, err := buildOfKey(t.Context(), noRef, take, keyContext{})
|
||||
if !errors.Is(err, errKeyWaits) || !strings.Contains(err.Error(), "waits for the reference of img") {
|
||||
t.Errorf("a key without a reference: %v", err)
|
||||
}
|
||||
building := noRef
|
||||
k := building.keys["x"]
|
||||
k.Build = sdkdelivery.BuilderBuild{State: sdkdelivery.BuildBuilding, By: "g14"}
|
||||
building.keys = map[string]builderkeys.Key{"x": k}
|
||||
if _, _, err := buildOfKey(t.Context(), building, take, keyContext{}); !errors.Is(err, errKeyWaits) ||
|
||||
!strings.Contains(err.Error(), "building on g14") {
|
||||
t.Errorf("a building key: %v", err)
|
||||
}
|
||||
if _, _, err := buildOfKey(t.Context(), keyBuckets{}, take, keyContext{}); !errors.Is(err, errKeyWaits) ||
|
||||
!strings.Contains(err.Error(), "no build of x") {
|
||||
t.Errorf("no key: %v", err)
|
||||
}
|
||||
k.Build = sdkdelivery.BuilderBuild{State: sdkdelivery.BuildFailed, Error: "digests differ: ace sha256:1, g14 sha256:2"}
|
||||
failed := keyBuckets{keys: map[string]builderkeys.Key{"x": k}}
|
||||
if _, _, err := buildOfKey(t.Context(), failed, take, keyContext{}); !errors.Is(err, errKeyFailed) ||
|
||||
!strings.Contains(err.Error(), "digests differ: ace sha256:1, g14 sha256:2") {
|
||||
t.Errorf("a failed key: %v", err)
|
||||
}
|
||||
none := take
|
||||
none.Trunk.FP2, none.Trunk.None2 = "", "no lock"
|
||||
if _, _, err := buildOfKey(t.Context(), noRef, none, keyContext{}); !errors.Is(err, errKeyWaits) ||
|
||||
!strings.Contains(err.Error(), "no lock") {
|
||||
t.Errorf("no fingerprint: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// ADR 0501 §7: a manifest word this controller does not read waits for the controller that reads it; never fails.
|
||||
func TestAManifestWordTheControllerDoesNotReadWaits(t *testing.T) {
|
||||
raw := `{"module":"x","version":"1","not-a-word-yet":true,"build":{"artifacts":[{"name":"a","kind":"archive","from":"files"}]},
|
||||
"resources":[{"id":"a","type":"archive","path":"/opt/x","artifact":"a"}]}`
|
||||
keys, trunk := aKeyOf(raw, 1, sdkdelivery.BuiltArtifact{Name: "a", Kind: "archive", Digest: shaOf("a"),
|
||||
Reference: "artifact-store://x/a/blobs/" + shaOf("a")})
|
||||
_, _, err := buildOfKey(t.Context(), keys, keyTake{Module: "x", Trunk: trunk}, keyContext{})
|
||||
if !errors.Is(err, errKeyWaits) || !strings.Contains(err.Error(), `"not-a-word-yet"`) {
|
||||
t.Fatalf("an unread word: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func packed(t *testing.T, files map[string]string) []byte {
|
||||
t.Helper()
|
||||
var buf bytes.Buffer
|
||||
z := gzip.NewWriter(&buf)
|
||||
w := tar.NewWriter(z)
|
||||
for name, text := range files {
|
||||
if err := w.WriteHeader(&tar.Header{Name: name, Mode: 0o644, Size: int64(len(text)), Typeflag: tar.TypeReg}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := w.Write([]byte(text)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := w.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := z.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
// A templated module's texts are read back from the archive the store holds under the key's reference; an archive the
|
||||
// store does not answer is waited for (the lead's Q1b).
|
||||
func TestATemplatedModulesTextsAreReadFromItsArchive(t *testing.T) {
|
||||
raw := `{"module":"x","version":"1","templates":"templates",
|
||||
"build":{"artifacts":[{"name":"templates","kind":"archive","from":"templates"}]},
|
||||
"resources":[{"id":"main","type":"file","path":"/etc/x.conf","mode":"0644","template":"templates/x.conf"}]}`
|
||||
body := packed(t, map[string]string{"x.conf": "listen 1\n"})
|
||||
sum := sha256.Sum256(body)
|
||||
digest := "sha256:" + hex.EncodeToString(sum[:])
|
||||
keys, trunk := aKeyOf(raw, 1, sdkdelivery.BuiltArtifact{Name: "templates", Kind: "archive", Digest: digest,
|
||||
Reference: "artifact-store://x/templates/blobs/" + digest})
|
||||
asked := ""
|
||||
archive := func(_ context.Context, reference string) ([]byte, error) { asked = reference; return body, nil }
|
||||
_, resolved, err := buildOfKey(t.Context(), keys, keyTake{Module: "x", Trunk: trunk}, keyContext{archive: archive})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if asked != "artifact-store://x/templates/blobs/"+digest {
|
||||
t.Errorf("the archive asked: %q", asked)
|
||||
}
|
||||
if text, _ := json.Marshal(resolved.Resources); !strings.Contains(string(text), "listen 1") {
|
||||
t.Errorf("the template's text is not in the file: %s", text)
|
||||
}
|
||||
gone := func(context.Context, string) ([]byte, error) { return nil, errors.New("the store does not hold it") }
|
||||
if _, _, err := buildOfKey(t.Context(), keys, keyTake{Module: "x", Trunk: trunk}, keyContext{archive: gone}); !errors.Is(err, errKeyWaits) ||
|
||||
!strings.Contains(err.Error(), "cannot be read from the artifact store") {
|
||||
t.Errorf("an archive the store does not answer: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// The launchers a key's bundle is resolved with are mesh-sdk's one rule (buildstep.Launchers), the names
|
||||
// mesh-build-node writes: a TypeScript bundle's entrypoint x.js runs as x.serve.mjs, a Go bundle has none.
|
||||
func TestALaunchersRuleIsTheBuildModules(t *testing.T) {
|
||||
m, err := catalogue.ParseManifest([]byte(tsBundle))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
k := builderkeys.Key{Build: sdkdelivery.BuilderBuild{Artifacts: []sdkdelivery.BuiltArtifact{{Name: "tools", Digest: "sha256:1",
|
||||
Reference: "artifact-store://x/tools/blobs/sha256:1"}}}}
|
||||
built, err := builtOfKey(t.Context(), nil, m, k)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if built[0].Launchers["dist/index.js"] != "dist/index.serve.mjs" || built[0].Kind != "bundle" {
|
||||
t.Fatalf("built: %+v", built[0])
|
||||
}
|
||||
}
|
||||
+44
-21
@@ -54,6 +54,8 @@ func run() error {
|
||||
return fmt.Errorf("no command given")
|
||||
}
|
||||
|
||||
// The bus's users are composed from what each module announced it keeps (novox/hq ADR 0495, issue 1790).
|
||||
inventory.Announcements = readAnnouncements
|
||||
// Every connection this process dials says what it is (novox/hq issue 327).
|
||||
broker.ConnectionName = connectionName(args[0], os.Getenv(verbVar))
|
||||
|
||||
@@ -73,8 +75,13 @@ func run() error {
|
||||
}
|
||||
|
||||
switch args[0] {
|
||||
case "build":
|
||||
return buildCommand(ctx, args[1:])
|
||||
case "switch":
|
||||
return switchCommand(ctx, args[1:])
|
||||
case "register":
|
||||
return registerCommand(ctx, args[1:])
|
||||
case "build", "rebuild":
|
||||
// **The hand build goes with the asks** (novox/hq ADR 0501 §5): refused by name, naming the replacement.
|
||||
return removedByName(args[0])
|
||||
case "builder":
|
||||
return builderCommand(ctx, args[1:])
|
||||
case "board":
|
||||
@@ -98,14 +105,12 @@ func run() error {
|
||||
return cancelCommand(ctx, args[1:])
|
||||
case "clear":
|
||||
return clearCommand(ctx, args[1:])
|
||||
case "rebuild":
|
||||
return rebuildCommand(ctx, args[1:])
|
||||
case "replay":
|
||||
return replayCommand(ctx, args[1:])
|
||||
case "kill":
|
||||
return killCommand(ctx, args[1:])
|
||||
case "forget-checks":
|
||||
return forgetChecksCommand(ctx, args[1:])
|
||||
return removedByName("forget-checks")
|
||||
case "try":
|
||||
return tryCommand(ctx, args[1:])
|
||||
case "pause", "resume":
|
||||
@@ -232,8 +237,8 @@ func run() error {
|
||||
}
|
||||
}
|
||||
|
||||
func usage() {
|
||||
fmt.Fprint(os.Stderr, `mesh-controller — the control plane
|
||||
// usageText is the terminal's help: every command it has, and none it refuses as removed (removedByName).
|
||||
var usageText = `mesh-controller — the control plane
|
||||
|
||||
On a node the operator types these as 'mesh-cli <command>' (in zsh, 'nox <command>'): asked
|
||||
through the node's engine, and run as the controller's terminal only on the control-node
|
||||
@@ -307,15 +312,14 @@ func usage() {
|
||||
operator key make [--out <file>] make the operator's sealing key, off the mesh; private half to the file only
|
||||
operator key set <public> [--replace] tell the mesh which operator key to seal to
|
||||
operator key show the operator key, and what it can recover
|
||||
build <repository> [--ref R] have a build machine build it, and record what came out
|
||||
build --behind build every module the mesh holds older than its source
|
||||
build --on <module> rebuild every module that stands on this module's artifacts, bases first
|
||||
register <module> <owner>/<repository> <path> register a module from a repository the catalogue does not
|
||||
build, from the builder's done key of its fingerprint (ADR 0501 §4)
|
||||
switch [--json] the switch's list: each module, the build it runs, and what the builder holds
|
||||
builds [<module>] what has been built lately, and what came of it
|
||||
queue [--json] every ask in the build queue: waiting, in flight (where, how long), dead
|
||||
cancel <id> drop a waiting or dead ask; recorded failed, cancelled by hand
|
||||
clear [--dead] cancel every waiting ask (and the dead ones); never one in flight
|
||||
rebuild <module|build-id> ask the module's source again, or that build's, under a new id
|
||||
replay <build-id> [--register [--older]] that build's commit again; a dry run unless --register
|
||||
replay <build-id> that build's commit again, a dry run: what a build of it makes
|
||||
kill <id> end a build where it runs; recorded failed, killed by hand
|
||||
pause [<node>] / resume [<node>] the build seat's holder there, or every holder, takes nothing new / again
|
||||
plans retry <id> ask a failed plan's failed builds again, and carry the plan on
|
||||
@@ -350,10 +354,13 @@ func usage() {
|
||||
version what this binary is
|
||||
|
||||
Each context reaches its own store through its own credential (novox/hq ADR 0008), named
|
||||
`+store.Variable("<context>")+` — or `+store.FileVariable("<context>")+`, naming a file that holds
|
||||
` + store.Variable("<context>") + ` — or ` + store.FileVariable("<context>") + `, naming a file that holds
|
||||
the same thing and keeps the password out of the environment. This process holds:
|
||||
|
||||
`)
|
||||
`
|
||||
|
||||
func usage() {
|
||||
fmt.Fprint(os.Stderr, usageText)
|
||||
for _, c := range held {
|
||||
fmt.Fprintf(os.Stderr, " %-12s database %-12s from %s\n",
|
||||
c.name, store.Database(c.name), store.Variable(c.name))
|
||||
@@ -408,13 +415,6 @@ func (b builds) Built(ctx context.Context, result link.BuildResult) error {
|
||||
fmt.Printf("%s: a dry run of %s on %s, not taken in\n", result.ID, result.Repository, result.Ref)
|
||||
return nil
|
||||
}
|
||||
// **A first build refused for a word the running build-agent does not read waits for one that does** (novox/hq ADR
|
||||
// 0482 §5, issue 1173): its merge is kept, named, and never failed; no failed build is recorded.
|
||||
if held, err := holdRefusedFirstBuild(ctx, b.inv, result); err != nil {
|
||||
return err
|
||||
} else if held {
|
||||
return nil
|
||||
}
|
||||
manifest, _, err := takeIn(ctx, b.inv, result)
|
||||
// The assignments pending on this build, made or ended (novox/hq issue 325), said in the daemon's log
|
||||
// after what became of the build.
|
||||
@@ -482,3 +482,26 @@ func connectionName(command, verb string) string {
|
||||
}
|
||||
return "mesh-controller command " + command
|
||||
}
|
||||
|
||||
// removedByName refuses a command the controller no longer has, naming what replaced it (novox/hq ADR 0501 §4, §5;
|
||||
// ADR 0429 §18; ADR 0503 §3; issue 1851, A11, and issue 1318).
|
||||
func removedByName(command string) error {
|
||||
replacement := map[string]string{
|
||||
"build": "the controller asks no build (ADR 0501 §5): the builder builds every fingerprint the indexer writes, a " +
|
||||
"module is registered from its done key, and a branch is tried with `try`; a module from a repository the " +
|
||||
"catalogue does not build is registered at the controller's terminal with `register`",
|
||||
"rebuild": "the controller asks no build (ADR 0501 §5): a failed key is built again by the builder as a new " +
|
||||
"revision (ADR 0429 §18), and a walk's Retry waits for it",
|
||||
"replay --register": "a build is registered only from the builder's done key of a trunk fingerprint (ADR 0501 " +
|
||||
"§1, §2); `replay` without --register still shows what a build of a recorded commit makes",
|
||||
"forget-checks": "build-agent makes no check build from R4e (ADR 0503 §3), so there is nothing to let go of",
|
||||
"delivery-plan": "a delivery's plan is read with `delivery-order` and the checker's verdict (ADR 0503 §3)",
|
||||
}[command]
|
||||
return &removedCommand{msg: fmt.Sprintf("`%s` is removed: %s", command, replacement)}
|
||||
}
|
||||
|
||||
// removedCommand is a command or verb the controller no longer has: a refusal that is the verb's whole answer, not a
|
||||
// verb of a newer build this one cannot run.
|
||||
type removedCommand struct{ msg string }
|
||||
|
||||
func (r *removedCommand) Error() string { return r.msg }
|
||||
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/overlay"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/licences"
|
||||
@@ -55,9 +56,18 @@ func aMeshOnTheStore(t *testing.T) *stores {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(open.Close)
|
||||
// The work this mesh started in the background ends before its stores close (cleanups run last first).
|
||||
t.Cleanup(background.Wait)
|
||||
// A condition store of its own, held in memory (novox/hq to-be 45 §2): status leads with what is
|
||||
// open, and a mesh with no bus would otherwise read as one whose conditions cannot be read.
|
||||
withConditionsInMemory(t)
|
||||
// The indexer holds no module unless a test puts one on its bus (aBuilderOnTheBus): an assignment's look at it
|
||||
// reads "not held" on a mesh with no bus of its own.
|
||||
wasIndexed := indexedModule
|
||||
indexedModule = func(context.Context, string) (sdkdelivery.Module, bool, error) {
|
||||
return sdkdelivery.Module{}, false, nil
|
||||
}
|
||||
t.Cleanup(func() { indexedModule = wasIndexed })
|
||||
for _, m := range provided {
|
||||
if err := open.inventory.Provide(t.Context(), m); err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -52,23 +52,22 @@ type deliveryMerge struct {
|
||||
Asker string `json:"asker,omitempty"`
|
||||
}
|
||||
|
||||
// askerOf is the asker whose warrant the merge is read under: the one the announcement names, mesh-delivery when it
|
||||
// names none.
|
||||
func (d deliveryMerge) askerOf() string {
|
||||
if d.Asker == "" {
|
||||
return theDelivery
|
||||
}
|
||||
return d.Asker
|
||||
}
|
||||
// askerOf is the asker whose warrant the merge is read under: the one the announcement names. One that names none is
|
||||
// from mesh-delivery of before the gate, whose warrant is no longer taken (novox/hq ADR 0450 §8; issue 1318).
|
||||
func (d deliveryMerge) askerOf() string { return d.Asker }
|
||||
|
||||
// askerAccepted is whether the controller takes a warrant from this asker for a merge: the teller, and mesh-delivery
|
||||
// until the gate takes the seat over (novox/hq ADR 0437 §6). Any other registers no move.
|
||||
// askerAccepted is whether the controller takes a warrant from this asker for a merge: the teller alone, since the gate
|
||||
// took the delivery seat over (novox/hq ADR 0437 §6, ADR 0450 §8; issue 1318). Any other registers no move.
|
||||
func askerAccepted(asker string) error {
|
||||
if asker == theDelivery || asker == sdkdelivery.TellerModule {
|
||||
if asker == sdkdelivery.TellerModule {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("the merge names the asker %q, and the controller takes a warrant only from %s and %s",
|
||||
asker, sdkdelivery.TellerModule, theDelivery)
|
||||
if asker == "" {
|
||||
return fmt.Errorf("the merge names no asker, as mesh-delivery's of before the gate did, and the controller takes "+
|
||||
"a warrant only from %s (ADR 0450 §8)", sdkdelivery.TellerModule)
|
||||
}
|
||||
return fmt.Errorf("the merge names the asker %q, and the controller takes a warrant only from %s (ADR 0450 §8)",
|
||||
asker, sdkdelivery.TellerModule)
|
||||
}
|
||||
|
||||
// parseDeliveryMerge reads an announcement, refusing one that names no pull request, head or ask.
|
||||
@@ -265,25 +264,32 @@ var errNotAMerge = errors.New("the build is not of a merge")
|
||||
// (ADR 0420 §7): the build is of a merge, that merge is a pull request mesh-delivery announced it made, and the
|
||||
// ask it made it on holds an Approve by the operator of that head. Answers why it was accepted; an error says why
|
||||
// it was not, and the refusal of ADR 0266 stands.
|
||||
func approvedMove(ctx context.Context, inv registrations, forms *sourceForms, buildID string,
|
||||
func approvedMove(ctx context.Context, inv registrations, forms *sourceForms, ask registrationAsk,
|
||||
built inventory.Source) (string, error) {
|
||||
r, found, err := inv.BuildRequestByID(ctx, buildID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
commit := ask.Merge
|
||||
if ask.Build != "" {
|
||||
r, found, err := inv.BuildRequestByID(ctx, ask.Build)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if !found || r.For != "merge" || r.Commit == "" {
|
||||
return "", errNotAMerge
|
||||
}
|
||||
commit = r.Commit
|
||||
}
|
||||
if !found || r.For != "merge" || r.Commit == "" {
|
||||
if commit == "" {
|
||||
return "", errNotAMerge
|
||||
}
|
||||
owner, name, onForge := forms.onTheForge(built.Repository, built.Seat)
|
||||
if !onForge {
|
||||
return "", errors.New("the repository is not on the mesh's forge")
|
||||
}
|
||||
merged, known, err := inv.MergeOf(ctx, owner+"/"+name, r.Commit)
|
||||
merged, known, err := inv.MergeOf(ctx, owner+"/"+name, commit)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if !known {
|
||||
return "", fmt.Errorf("the controller kept no merge of %s/%s at %.8s", owner, name, r.Commit)
|
||||
return "", fmt.Errorf("the controller kept no merge of %s/%s at %.8s", owner, name, commit)
|
||||
}
|
||||
return approvedAnnounced(ctx, inv, owner, name, announcedOf(merged))
|
||||
}
|
||||
|
||||
@@ -48,18 +48,19 @@ func TestAMergeNamingTheTellerIsMovedOnTheTellersWarrant(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// 1213/AC2: an announcement naming mesh-delivery, or none (today's module): read under mesh-delivery, registered.
|
||||
func TestAMergeNamingNoAskerIsMovedOnMeshDeliverysWarrant(t *testing.T) {
|
||||
// 1318 (ADR 0450 §8): a merge naming no asker, or the asker mesh-delivery of before the gate, registers no move; the
|
||||
// teller's is taken, and its warrant read under its name.
|
||||
func TestAMergeOfMeshDeliveryOrNoAskerMovesNothing(t *testing.T) {
|
||||
for _, named := range []string{"", theDelivery} {
|
||||
a := askedBy(t, named, theDelivery)
|
||||
if _, err := movedOnApprove(a.merge, &a.ask, &a.warrant); err != nil {
|
||||
t.Errorf("asker %q: refused: %v", named, err)
|
||||
if _, err := movedOnApprove(a.merge, &a.ask, &a.warrant); err == nil || !strings.Contains(err.Error(), "ADR 0450 §8") {
|
||||
t.Errorf("asker %q: %v", named, err)
|
||||
}
|
||||
}
|
||||
was := routerAskOf
|
||||
wasKept := keptDeliveryMerge
|
||||
t.Cleanup(func() { routerAskOf, keptDeliveryMerge = was, wasKept })
|
||||
a := askedBy(t, "", theDelivery)
|
||||
a := askedBy(t, sdkdelivery.TellerModule, sdkdelivery.TellerModule)
|
||||
keptDeliveryMerge = func(context.Context, string, string, int, string) (deliveryMerge, bool, error) {
|
||||
return a.merge, true, nil
|
||||
}
|
||||
@@ -71,8 +72,8 @@ func TestAMergeNamingNoAskerIsMovedOnMeshDeliverysWarrant(t *testing.T) {
|
||||
if _, err := approvedAnnounced(t.Context(), nil, "novox", "mesh-controller", announcedPull(7, a.merge.HeadSHA)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if readUnder != theDelivery {
|
||||
t.Errorf("with no asker named the warrant was read under %q, not %q", readUnder, theDelivery)
|
||||
if readUnder != sdkdelivery.TellerModule {
|
||||
t.Errorf("the warrant was read under %q, not %q", readUnder, sdkdelivery.TellerModule)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -68,7 +69,7 @@ func TestAModuleBuiltFromANewRepositoryIsRegisteredOnlyOnTheOperatorsApproveOnTh
|
||||
a.merge.Owner, a.merge.Repo, a.merge.Number = "novox", "mesh-host", 7
|
||||
if c.answer {
|
||||
record, _ := json.Marshal(map[string]any{"state": "closed", "ask": a.ask, "warrant": a.warrant})
|
||||
if _, err := records.Put(ctx, theDelivery+"."+a.merge.AskID, record); err != nil {
|
||||
if _, err := records.Put(ctx, sdkdelivery.TellerModule+"."+a.merge.AskID, record); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -21,7 +22,7 @@ func approvedMerge(t *testing.T) anApprovedMerge {
|
||||
heads := map[string]string{"novox/mesh-controller@aaaaaaaaaaaa": "a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1",
|
||||
"novox/mesh-catalog@bbbbbbbbbbbb": "b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2"}
|
||||
m := deliveryMerge{Owner: "novox", Repo: "mesh-controller", Number: 7, HeadSHA: "a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1",
|
||||
Group: "g1", AskID: "merge-g1", Option: optionApprove, Heads: heads}
|
||||
Group: "g1", AskID: "merge-g1", Option: optionApprove, Heads: heads, Asker: sdkdelivery.TellerModule}
|
||||
bind := func(option string, h map[string]string) string {
|
||||
d, err := asks.ActDigest(mergeAct(m.AskID, option, h))
|
||||
if err != nil {
|
||||
@@ -35,7 +36,7 @@ func approvedMerge(t *testing.T) anApprovedMerge {
|
||||
Options: []asks.Option{
|
||||
{ID: optionApprove, Label: "Approve", Does: "merges the group", Level: asks.Approve, Binds: bind(optionApprove, heads)},
|
||||
{ID: "decline", Label: "Decline", Does: "leaves it", Level: asks.Approve, Binds: bind("decline", heads)}}}
|
||||
w := asks.Warrant{Ask: m.AskID, Asker: theDelivery, Outcome: asks.OutcomeChosen, Option: optionApprove, Label: "Approve",
|
||||
w := asks.Warrant{Ask: m.AskID, Asker: sdkdelivery.TellerModule, Outcome: asks.OutcomeChosen, Option: optionApprove, Label: "Approve",
|
||||
Level: asks.Approve, By: &asks.Person{Who: asks.Operator, Kind: "telegram", Identity: "1", Verified: "user id"},
|
||||
At: at, AskDigest: ask.Digest()}
|
||||
return anApprovedMerge{merge: m, ask: ask, warrant: w}
|
||||
|
||||
@@ -116,7 +116,7 @@ func TestANewModuleIsRegisteredOnTheOperatorsApproveOfItsMergeAndRefusedOtherwis
|
||||
if c.built != nil {
|
||||
src = *c.built
|
||||
}
|
||||
id, err := mayRegisterIn(t.Context(), mesh, forms, "mesh-delivery-view", src, "build-1", "", "main")
|
||||
id, err := mayRegisterIn(t.Context(), mesh, forms, "mesh-delivery-view", src, byBuild("build-1"), "", "main")
|
||||
switch {
|
||||
case c.accepted && err != nil:
|
||||
t.Fatalf("%s: the Approved merge's new module was refused: %v", c.ac, err)
|
||||
@@ -157,7 +157,7 @@ func TestASecondNewModuleFromARepositoryIsJudgedAlongsideAndAsksNoWarrant(t *tes
|
||||
request: inventory.BuildRequest{ID: "build-2", Repository: "novox/mesh-delivery", Seat: "git", For: "build"}}}
|
||||
forms := &sourceForms{bases: map[string]string{}, base: func(string) string { return "http://forge.internal:20000" }}
|
||||
built := inventory.Source{Repository: "novox/mesh-delivery", Seat: "git"}
|
||||
if id, err := mayRegisterIn(t.Context(), mesh, forms, "mesh-delivery-watchman", built, "build-2", "", "main"); err != nil || id == 0 {
|
||||
if id, err := mayRegisterIn(t.Context(), mesh, forms, "mesh-delivery-watchman", built, byBuild("build-2"), "", "main"); err != nil || id == 0 {
|
||||
t.Fatalf("1146/AC5: a second module from a repository the catalogue builds from was refused (id %d): %v", id, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,37 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// 1617/AC2 (unit): a new module whose first build a merge's hearing asked is not asked again when its batch is cut,
|
||||
// and one whose ask failed, was never handed over or is long lost is.
|
||||
func TestAModuleWhoseBuildIsComingIsNotAskedAgain(t *testing.T) {
|
||||
now := time.Date(2026, 10, 21, 12, 0, 0, 0, time.UTC)
|
||||
source := buildSource{Repository: "http://forge/novox/mesh-catalog.git", Seat: "git"}
|
||||
asked := func(path string, mod func(*inventory.RequestOutcome)) inventory.RequestOutcome {
|
||||
r := inventory.RequestOutcome{BuildRequest: inventory.BuildRequest{ID: "b-" + path, Repository: source.Repository,
|
||||
Seat: "git", Path: path, For: "merge", At: now.Add(-3 * time.Minute)}}
|
||||
if mod != nil {
|
||||
mod(&r)
|
||||
}
|
||||
return r
|
||||
}
|
||||
requests := []inventory.RequestOutcome{
|
||||
asked("modules/go", nil),
|
||||
asked("modules/failed", func(r *inventory.RequestOutcome) { r.Heard, r.Failed = true, "no module.json" }),
|
||||
asked("modules/notasked", func(r *inventory.RequestOutcome) { r.NotAsked = "the bus is silent" }),
|
||||
asked("modules/lost", func(r *inventory.RequestOutcome) { r.At = now.Add(-2 * time.Hour) }),
|
||||
asked("modules/byhand", func(r *inventory.RequestOutcome) { r.For = "build" }),
|
||||
}
|
||||
got := unaskedDirs([]string{"modules/go", "modules/failed", "modules/notasked", "modules/lost", "modules/byhand",
|
||||
"modules/nodejs"}, requests, source, now)
|
||||
want := []string{"modules/failed", "modules/notasked", "modules/lost", "modules/byhand", "modules/nodejs"}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("asked again %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -69,7 +70,7 @@ func TestANewModuleFromANewRepositoryIsRegisteredOnlyOnTheOperatorsApproveOnTheB
|
||||
map[string]string{"novox/mesh-delivery@a1a1a1a1a1a1": "a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1"})
|
||||
if c.answer {
|
||||
record, _ := json.Marshal(map[string]any{"state": "closed", "ask": a.ask, "warrant": a.warrant})
|
||||
if _, err := records.Put(ctx, theDelivery+"."+a.merge.AskID, record); err != nil {
|
||||
if _, err := records.Put(ctx, sdkdelivery.TellerModule+"."+a.merge.AskID, record); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,134 +4,110 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// asksWithPaths records every build a merge asks for, as repository, path and ref. The merge window's goroutine and
|
||||
// the test's own may ask at once, so the record is written under a lock (issue 1543).
|
||||
func asksWithPaths(t *testing.T) *[][3]string {
|
||||
t.Helper()
|
||||
var asked [][3]string
|
||||
var mu sync.Mutex
|
||||
was := askABuild
|
||||
askABuild = func(_ context.Context, source buildSource, path, ref string) (string, error) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
asked = append(asked, [3]string{source.Repository, path, ref})
|
||||
return "b-" + path, nil
|
||||
}
|
||||
t.Cleanup(func() { askABuild = was })
|
||||
return &asked
|
||||
}
|
||||
|
||||
// The live case of 2026-10-07 (novox/hq issue 300), replayed: a merge of the catalogue adds one module's
|
||||
// directory and touches nothing else the mesh holds. Its delivery plan said "builds new: …, sent nowhere";
|
||||
// the merge said it changed nothing any module the mesh holds is built from, and built nothing, so the
|
||||
// module was never registered and `assign` refused it. The merge asks for its build, at the branch merged
|
||||
// into, from the repository as the mesh spells it — and opens no plan, since nothing it holds moved.
|
||||
func TestAMergeBuildsTheNewModuleItAdds(t *testing.T) {
|
||||
// ADR 0501 §6 (issue 1851, A5): the live case of 2026-10-07 (novox/hq issue 300), on keys. A merge of the catalogue adds
|
||||
// one module's directory and touches nothing else the mesh holds: nothing asks its first build, it opens no plan, and
|
||||
// the module is registered from the builder's first done key of it, assigned nowhere, with no build request written.
|
||||
func TestAMergeAddingAModuleEndsRegisteredFromItsKeyAndAsksNoBuild(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
asked := asksWithPaths(t)
|
||||
noBuildAsked(t)
|
||||
b := aBuilderOnTheBus(t)
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "networkmanager", Version: "1"},
|
||||
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/networkmanager", Ref: "main",
|
||||
BuiltFrom: "c0", Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "3da80a4b00",
|
||||
watching, stop := context.WithCancel(context.Background())
|
||||
t.Cleanup(stop)
|
||||
watchNewModules(watching, open)
|
||||
|
||||
const commit = "3da80a4b00000000"
|
||||
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: commit,
|
||||
Paths: []string{"modules/systemd-resolved/module.json", "modules/systemd-resolved/cmd/main.go"},
|
||||
ModuleDirs: []string{"modules/systemd-resolved"}, ModuleDirsSaid: true}
|
||||
if err := (following{open: open}).SourceMoved(ctx, m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := [3]string{"novox/mesh-catalog", "modules/systemd-resolved", "main"}
|
||||
if len(*asked) != 1 || (*asked)[0] != want {
|
||||
t.Fatalf("the merge adding modules/systemd-resolved asked for %v, not %v", *asked, want)
|
||||
}
|
||||
if plans, err := inv.OpenPlans(ctx); err != nil || len(plans) != 0 {
|
||||
t.Fatalf("a merge moving nothing the mesh holds opened a plan: %+v %v", plans, err)
|
||||
}
|
||||
// The indexer writes the module's trunk value; the builder builds it, then its key is done.
|
||||
fp := fpOf('7')
|
||||
manifest := b.indexed(t, "systemd-resolved", "1", "novox/mesh-catalog", "modules/systemd-resolved", commit, fp)
|
||||
b.built(t, "systemd-resolved", "1", fp, manifest, sdkdelivery.BuildBuilding)
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
if _, known := inventoryHolds(t, inv, "systemd-resolved"); known {
|
||||
t.Fatal("0501 §1: registered from a key still building")
|
||||
}
|
||||
if obs := newModuleObservations(time.Now().Add(time.Hour)); len(obs) != 1 || obs[0].Kind != kindNewModuleWaits {
|
||||
t.Fatalf("0501 §6: the new module waiting on its key past its bound is named by %+v", obs)
|
||||
}
|
||||
b.built(t, "systemd-resolved", "1", fp, manifest, sdkdelivery.BuildDone)
|
||||
becomes(t, 10*time.Second, "the new module registered", func() bool {
|
||||
_, known := inventoryHolds(t, inv, "systemd-resolved")
|
||||
return known
|
||||
})
|
||||
// The take-in forgets the wait just after the registration it made is seen.
|
||||
becomes(t, 5*time.Second, "the registered module no longer named waiting", func() bool {
|
||||
return len(newModuleObservations(time.Now().Add(time.Hour))) == 0
|
||||
})
|
||||
requests, err := inv.RequestsNamed(ctx, inventory.BuildRequest{Repository: "novox/mesh-catalog",
|
||||
Path: "modules/systemd-resolved"}.Name())
|
||||
if err != nil || len(requests) != 0 {
|
||||
t.Fatalf("0501 §6: a build request was written: %+v (%v)", requests, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A merge that changes a held module and adds a new one does both: the held one walks its plan, the new
|
||||
// one is asked for beside it. A merge adding nothing builds nothing new.
|
||||
func TestAMergeBuildsItsNewModuleBesideItsPlan(t *testing.T) {
|
||||
// ADR 0501 §7: a new module whose manifest uses a word this controller does not read is named waiting at once, never
|
||||
// failed, and registered from the same key, with no second build, when a controller that reads it looks.
|
||||
func TestANewModuleWithAnUnreadWordWaitsForTheControllerThatReadsIt(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
asked := asksWithPaths(t)
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"},
|
||||
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", Ref: "main",
|
||||
noBuildAsked(t)
|
||||
b := aBuilderOnTheBus(t)
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "networkmanager", Version: "1"},
|
||||
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/networkmanager", Ref: "main",
|
||||
BuiltFrom: "c0", Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "c1aaaaaaaa",
|
||||
Paths: []string{"modules/app/index.ts", "modules/fresh/module.json"},
|
||||
ModuleDirs: []string{"modules/app", "modules/fresh"}, ModuleDirsSaid: true}
|
||||
if err := (following{open: open}).SourceMoved(ctx, m); err != nil {
|
||||
fp := fpOf('8')
|
||||
raw := `{"module":"watchman","version":"1","not-a-word-yet":"5m","build":{"artifacts":[{"name":"a","kind":"archive","from":"files"}]},` +
|
||||
`"resources":[{"id":"a","type":"archive","path":"/opt/watchman","artifact":"a"}]}`
|
||||
digest := sdkdelivery.ManifestDigest([]byte(raw))
|
||||
value, _ := sdkdelivery.EncodeManifest(sdkdelivery.Manifest{Module: "watchman", JSON: raw})
|
||||
if _, err := b.manifests.Put(ctx, digest, value); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
paths := map[string]bool{}
|
||||
for _, a := range *asked {
|
||||
paths[a[1]] = true
|
||||
}
|
||||
if !paths["modules/fresh"] || !paths["modules/app"] || len(*asked) != 2 {
|
||||
t.Fatalf("asked %v; want the new module and the plan's first tier", *asked)
|
||||
}
|
||||
plans, err := inv.OpenPlans(ctx)
|
||||
if err != nil || len(plans) != 1 || plans[0].Modules["fresh"] != nil || plans[0].Modules["app"] == nil {
|
||||
t.Fatalf("the plan should walk app alone: %+v %v", plans, err)
|
||||
}
|
||||
|
||||
*asked = nil
|
||||
m2 := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "c2aaaaaaaa",
|
||||
Paths: []string{"README.md"}, ModuleDirsSaid: true}
|
||||
if err := (following{open: open}).SourceMoved(ctx, m2); err != nil {
|
||||
trunk, _ := sdkdelivery.EncodeModule(sdkdelivery.Module{FP: "src1:" + fp[5:], FP2: fp, Manifest: digest,
|
||||
Repository: "novox/mesh-catalog", Commit: "4444444400000000", Path: "modules/watchman"})
|
||||
if _, err := b.modules.Put(ctx, "watchman", trunk); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(*asked) != 0 {
|
||||
t.Fatalf("a merge adding no module asked for %v", *asked)
|
||||
b.built(t, "watchman", "1", fp, digest, sdkdelivery.BuildDone)
|
||||
takeNewModules(ctx, open, "watchman")
|
||||
if _, known := inventoryHolds(t, inv, "watchman"); known {
|
||||
t.Fatal("0501 §7: a manifest with an unread word was registered")
|
||||
}
|
||||
}
|
||||
|
||||
// 1617/AC1 (integration): a merge adding a module to a repository the mesh builds modules from, heard while a batch of
|
||||
// that repository is still assembling (its window running), asks the module's first build when it is heard, so the
|
||||
// module is registered while the batch waits; cutting the batch asks it no second time (1617/AC2).
|
||||
func TestAMergeAddingAModuleAsksItsBuildBeforeItsBatchIsCut(t *testing.T) {
|
||||
open := windowed(t)
|
||||
groupsWaiting(t, "pr/c1")
|
||||
asked := asksWithPaths(t)
|
||||
catalog := func(commit string, made time.Time, paths ...string) link.SourceMoved {
|
||||
return link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: commit,
|
||||
Head: "pr/" + commit, MergedAt: made.Format(time.RFC3339Nano), Paths: paths}
|
||||
obs := newModuleObservations(time.Now())
|
||||
if len(obs) != 1 || obs[0].Kind != kindNewModuleWaits || !strings.Contains(obs[0].Summary, `"not-a-word-yet"`) {
|
||||
t.Fatalf("0501 §7: the module waiting for a reader of its word is named by %+v", obs)
|
||||
}
|
||||
hear(t, open, catalog("c1", t0, "modules/app/index.ts"), t0)
|
||||
if len(*asked) != 0 {
|
||||
t.Fatalf("a merge adding no module asked %v", *asked)
|
||||
}
|
||||
hear(t, open, lone(catalog("c2", t0.Add(10*time.Second), "modules/go/packages.json", "modules/go/module.json")), t0.Add(10*time.Second))
|
||||
if ws, bs := walks(t, open); len(ws) != 0 || len(bs) != 1 {
|
||||
t.Fatalf("the batch should still be assembling: %d walk(s), %d batch(es)", len(ws), len(bs))
|
||||
}
|
||||
want := [3]string{"novox/mesh-catalog", "modules/go", "main"}
|
||||
if len(*asked) != 1 || (*asked)[0] != want {
|
||||
t.Fatalf("heard with its batch assembling, the merge asked %v, not %v", *asked, want)
|
||||
}
|
||||
cutAt(t, open, t0.Add(10*time.Second+90*time.Second))
|
||||
n := 0
|
||||
for _, a := range *asked {
|
||||
if a[1] == "modules/go" {
|
||||
n++
|
||||
builds, _ := inv.Builds(ctx, "watchman", 5)
|
||||
for _, b := range builds {
|
||||
if !b.Worked() {
|
||||
t.Fatalf("0501 §7: a failed build was recorded: %+v", b)
|
||||
}
|
||||
}
|
||||
if n != 1 {
|
||||
t.Fatalf("modules/go was asked %d times: %v", n, *asked)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,274 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"github.com/novox/mesh-controller/internal/builderkeys"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// A new module is registered from its first done key (novox/hq ADR 0501 §1, §6; issue 1851, A5).
|
||||
//
|
||||
// Nothing asks a new module's first build: the builder builds every fingerprint the indexer writes, a new module's
|
||||
// included (ADR 0429 §5). The controller registers a module the catalogue does not hold from the builder's done key of the
|
||||
// fingerprint the indexer's trunk value names, under the same rules of who may register as before (ADR 0501 §3): from a
|
||||
// repository the catalogue builds another module from, or on the operator's Approve of the merge the trunk value names
|
||||
// (a first merge, kept with mesh-delivery's word), or at the terminal (ADR 0501 §4). It is assigned nowhere.
|
||||
//
|
||||
// **No timer** (ADR 0426): a write of the builder or the indexer wakes it for that module (builderkeys.Watch), a first
|
||||
// merge's word wakes it for that repository, and a controller that starts looks at every module once (ADR 0501 §7: a
|
||||
// manifest the controller before could not read is registered when the controller that reads it runs).
|
||||
|
||||
// newModulesMu keeps one take-in of new modules at a time in this process.
|
||||
var newModulesMu sync.Mutex
|
||||
|
||||
// newModuleWait is a new module the controller could not register yet, and why: what the first-builds condition names.
|
||||
type newModuleWait struct {
|
||||
Module, Repository, Commit, Why string
|
||||
// Word is the manifest word this controller does not read, when that is why (ADR 0501 §7).
|
||||
Word string
|
||||
Since time.Time
|
||||
}
|
||||
|
||||
// newModulesWaiting are the new modules waiting, by module, in this process: a restart looks at every module again.
|
||||
var newModulesWaiting = struct {
|
||||
sync.Mutex
|
||||
by map[string]newModuleWait
|
||||
}{by: map[string]newModuleWait{}}
|
||||
|
||||
// takeNewModules looks at the modules the indexer holds that the catalogue does not, all of them or those named, and
|
||||
// registers each whose done key may register it. Each outcome is said once: a module still waiting for the same reason
|
||||
// is not said again.
|
||||
func takeNewModules(ctx context.Context, open *stores, only ...string) {
|
||||
newModulesMu.Lock()
|
||||
defer newModulesMu.Unlock()
|
||||
keys, closeKeys, err := openBuilderKeys()
|
||||
if err != nil {
|
||||
fmt.Printf("new modules: the builder's keys cannot be read: %v\n", err)
|
||||
return
|
||||
}
|
||||
defer closeKeys()
|
||||
for _, module := range takeNewModulesWith(ctx, open.inventory, keys, only...) {
|
||||
// An assignment waiting for it is made now (ADR 0501 §6; issue 1851, A10).
|
||||
for _, line := range settlePendingOnKey(ctx, open, module) {
|
||||
fmt.Println(line)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// takeNewModulesWith is takeNewModules on the buckets handed in; it answers the modules it registered.
|
||||
func takeNewModulesWith(ctx context.Context, inv *inventory.Inventory, keys newModuleReads, only ...string) []string {
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
fmt.Printf("new modules: the catalogue cannot be read: %v\n", err)
|
||||
// empty-on-error: said above; nothing is registered this time, and the next write of a key looks again
|
||||
return nil
|
||||
}
|
||||
held := map[string]bool{}
|
||||
for _, e := range entries {
|
||||
held[e.Manifest.Module] = true
|
||||
}
|
||||
values, unread, err := keys.Modules(ctx)
|
||||
if err != nil {
|
||||
fmt.Printf("new modules: the indexer's modules cannot be read: %v\n", err)
|
||||
// empty-on-error: said above; nothing is registered this time, and the next write of a key looks again
|
||||
return nil
|
||||
}
|
||||
for module, err := range unread {
|
||||
if !held[module] && (len(only) == 0 || slices.Contains(only, module)) {
|
||||
sayNewModuleWait(newModuleWait{Module: module, Why: "the indexer's value of it cannot be read: " + err.Error()})
|
||||
}
|
||||
}
|
||||
names := make([]string, 0, len(values))
|
||||
for module := range values {
|
||||
if !held[module] && (len(only) == 0 || slices.Contains(only, module)) {
|
||||
names = append(names, module)
|
||||
}
|
||||
}
|
||||
sort.Strings(names)
|
||||
var registered []string
|
||||
for _, module := range names {
|
||||
v := values[module]
|
||||
_, _, err := takeInKey(ctx, inv, keys, keyTake{Module: module, Trunk: v, Ask: registrationAsk{Merge: v.Commit}})
|
||||
switch {
|
||||
case err == nil:
|
||||
forgetNewModuleWait(module)
|
||||
registered = append(registered, module)
|
||||
fmt.Printf("%s is new in %s at %.8s: registered from the builder's done key, assigned nowhere\n", module,
|
||||
v.Repository, v.Commit)
|
||||
case errors.Is(err, errKeyWaits):
|
||||
w := newModuleWait{Module: module, Repository: v.Repository, Commit: v.Commit, Why: err.Error()}
|
||||
if word := wordIn(err.Error()); word != "" {
|
||||
w.Word = word
|
||||
}
|
||||
sayNewModuleWait(w)
|
||||
default:
|
||||
sayNewModuleWait(newModuleWait{Module: module, Repository: v.Repository, Commit: v.Commit,
|
||||
Why: "not registered: " + err.Error()})
|
||||
}
|
||||
}
|
||||
return registered
|
||||
}
|
||||
|
||||
// newModuleReads is what the take-in of new modules reads of the builder and the indexer.
|
||||
type newModuleReads interface {
|
||||
keyReads
|
||||
Modules(ctx context.Context) (map[string]sdkdelivery.Module, map[string]error, error)
|
||||
}
|
||||
|
||||
var _ newModuleReads = builderkeys.Reader{}
|
||||
|
||||
// wordIn is the manifest word a take-in's words say this controller does not read.
|
||||
func wordIn(said string) string {
|
||||
_, rest, ok := strings.Cut(said, "uses the word \"")
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
word, _, _ := strings.Cut(rest, "\"")
|
||||
return word
|
||||
}
|
||||
|
||||
// sayNewModuleWait keeps a new module waiting and says it, once for each reason.
|
||||
func sayNewModuleWait(w newModuleWait) {
|
||||
newModulesWaiting.Lock()
|
||||
defer newModulesWaiting.Unlock()
|
||||
was, known := newModulesWaiting.by[w.Module]
|
||||
if known && was.Why == w.Why && was.Commit == w.Commit {
|
||||
return
|
||||
}
|
||||
w.Since = time.Now().UTC()
|
||||
if known && was.Commit == w.Commit {
|
||||
w.Since = was.Since
|
||||
}
|
||||
newModulesWaiting.by[w.Module] = w
|
||||
fmt.Printf("%s is new at %.8s and not registered yet: %s\n", w.Module, w.Commit, firstLine(w.Why))
|
||||
}
|
||||
|
||||
func forgetNewModuleWait(module string) {
|
||||
newModulesWaiting.Lock()
|
||||
defer newModulesWaiting.Unlock()
|
||||
delete(newModulesWaiting.by, module)
|
||||
}
|
||||
|
||||
// kindNewModuleWaits is what a new module waiting for this controller's reader of its manifest, or waiting past its
|
||||
// bound, raises (ADR 0501 §6, §7: the first-builds condition reads the key, not a build request).
|
||||
const kindNewModuleWaits = "new-module-waits"
|
||||
|
||||
// newModuleWaitBound is how long a new module waits for its key before it is said: the builder builds a fingerprint in
|
||||
// minutes. Not a delay: its key's write registers it the moment it is done.
|
||||
const newModuleWaitBound = 30 * time.Minute
|
||||
|
||||
// newModuleObservations names the new modules waiting: one whose manifest uses a word this controller does not read at
|
||||
// once (nothing is lost, but the operator sees what waits for which controller), any other past the bound.
|
||||
func newModuleObservations(now time.Time) []conditions.Observation {
|
||||
newModulesWaiting.Lock()
|
||||
defer newModulesWaiting.Unlock()
|
||||
var out []conditions.Observation
|
||||
for _, w := range newModulesWaiting.by {
|
||||
if w.Word == "" && now.Sub(w.Since) <= newModuleWaitBound {
|
||||
continue
|
||||
}
|
||||
head := fmt.Sprintf("%s is not registered: %s", w.Module, firstLine(w.Why))
|
||||
resolved := "its key is done and its take-in registers it"
|
||||
if w.Word != "" {
|
||||
head = fmt.Sprintf("%s waits for a controller that reads the manifest word %q", w.Module, w.Word)
|
||||
resolved = "a controller that reads the word runs and registers it from the same key, with no second build"
|
||||
}
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMerge,
|
||||
ID: fmt.Sprintf("%s_%s", w.Module, short(w.Commit)), Kind: kindNewModuleWaits, Severity: conditions.Warning,
|
||||
Summary: head,
|
||||
Said: fmt.Sprintf("%s at %s waits since %s: %s", w.Module, w.Commit, w.Since.Format(time.RFC3339), w.Why),
|
||||
Headline: fmt.Sprintf("The new module %s is not registered yet", w.Module),
|
||||
Explanation: fmt.Sprintf("The merge %.8s of %s adds %s. The mesh registers it from the builder's done key of its "+
|
||||
"fingerprint, and has not yet: %s", w.Commit, w.Repository, w.Module, w.Why),
|
||||
Resolved: resolved,
|
||||
})
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID })
|
||||
return out
|
||||
}
|
||||
|
||||
// watchNewModules wakes the take-in of new modules for each module the builder or the indexer writes, until ctx ends, and
|
||||
// looks at every module once now. A bucket not there yet is said: the look at start and a first merge's word still work.
|
||||
func watchNewModules(ctx context.Context, open *stores) {
|
||||
inBackground(func() {
|
||||
answerHeldMerges(ctx, open.inventory)
|
||||
takeNewModules(ctx, open)
|
||||
})
|
||||
keys, closeKeys, err := openBuilderKeys()
|
||||
if err != nil {
|
||||
fmt.Printf("new modules: the builder's keys are not watched: %v\n", err)
|
||||
return
|
||||
}
|
||||
if err := builderkeys.Watch(ctx, keys.JS, func(module string) { wakeNewModule(ctx, open, module) }); err != nil {
|
||||
closeKeys()
|
||||
fmt.Printf("new modules: the builder's keys are not watched: %v\n", err)
|
||||
return
|
||||
}
|
||||
go func() {
|
||||
<-ctx.Done()
|
||||
closeKeys()
|
||||
}()
|
||||
}
|
||||
|
||||
// wakeNewModule takes module in when the catalogue does not hold it: never on the watch's own goroutine, which must keep
|
||||
// draining.
|
||||
func wakeNewModule(ctx context.Context, open *stores, module string) {
|
||||
inBackground(func() {
|
||||
entries, err := open.inventory.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
for _, e := range entries {
|
||||
if e.Manifest.Module == module {
|
||||
// Known: a plan waiting on its key is woken by the plans' own look.
|
||||
askToLook()
|
||||
return
|
||||
}
|
||||
}
|
||||
takeNewModules(ctx, open, module)
|
||||
})
|
||||
}
|
||||
|
||||
// background is the work the take-ins of keys and of announcements start beside the event that woke them, so a test
|
||||
// ends only once the work its mesh started has.
|
||||
var background sync.WaitGroup
|
||||
|
||||
// inBackground runs work beside the caller, counted in background.
|
||||
func inBackground(work func()) {
|
||||
background.Add(1)
|
||||
go func() {
|
||||
defer background.Done()
|
||||
work()
|
||||
}()
|
||||
}
|
||||
|
||||
// newModulesOf are the modules the indexer holds in a repository: what a first merge's word wakes.
|
||||
func newModulesOf(ctx context.Context, repository string) ([]string, error) {
|
||||
keys, closeKeys, err := openBuilderKeys()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer closeKeys()
|
||||
values, _, err := keys.Modules(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []string
|
||||
for module, v := range values {
|
||||
if strings.EqualFold(v.Repository, repository) {
|
||||
out = append(out, module)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out, nil
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ADR 0501 §6, §7 (issue 1851, A5b): the condition of a new module reads its key's take-in: one waiting for a controller
|
||||
// that reads its manifest's word is named at once, one waiting for its key only past the bound, each reason said once.
|
||||
func TestANewModuleWaitingIsNamedByWhatItWaitsFor(t *testing.T) {
|
||||
t.Cleanup(func() {
|
||||
newModulesWaiting.Lock()
|
||||
newModulesWaiting.by = map[string]newModuleWait{}
|
||||
newModulesWaiting.Unlock()
|
||||
})
|
||||
word := `waits: watchman's manifest uses the word "key-ttl", which this controller does not read: it is registered when a controller that reads it runs`
|
||||
sayNewModuleWait(newModuleWait{Module: "watchman", Commit: "c1", Why: word, Word: wordIn(word)})
|
||||
sayNewModuleWait(newModuleWait{Module: "view", Commit: "c1", Why: "waits: the build of view at aaaa is building on ace"})
|
||||
now := time.Now()
|
||||
obs := newModuleObservations(now)
|
||||
if len(obs) != 1 || !strings.Contains(obs[0].Summary, `"key-ttl"`) || obs[0].Kind != kindNewModuleWaits {
|
||||
t.Fatalf("at once, only the unread word is named: %+v", obs)
|
||||
}
|
||||
if obs := newModuleObservations(now.Add(newModuleWaitBound + time.Minute)); len(obs) != 2 {
|
||||
t.Fatalf("past the bound, both are named: %+v", obs)
|
||||
}
|
||||
forgetNewModuleWait("watchman")
|
||||
forgetNewModuleWait("view")
|
||||
if obs := newModuleObservations(now.Add(time.Hour)); len(obs) != 0 {
|
||||
t.Fatalf("registered modules are still named: %+v", obs)
|
||||
}
|
||||
}
|
||||
@@ -1,24 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/chain"
|
||||
)
|
||||
|
||||
// 1146: a merge's ask that failed is kept under an id the bus takes as a key, for a module at the repository's root
|
||||
// (".") as for one in a folder.
|
||||
func TestTheIdOfABuildNotAskedIsAKeyTheBusTakes(t *testing.T) {
|
||||
for dir, want := range map[string]string{
|
||||
".": "not-asked-0177ea7a",
|
||||
"modules/mesh-delivery-view": "not-asked-0177ea7a-modules-mesh-delivery-view",
|
||||
} {
|
||||
id := notAskedID("0177ea7a00000000", dir)
|
||||
if id != want {
|
||||
t.Errorf("notAskedID(%q) = %q, want %q", dir, id, want)
|
||||
}
|
||||
if !chain.ValidKey("req." + id) {
|
||||
t.Errorf("notAskedID(%q) = %q: the bus takes no key req.%s", dir, id, id)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -109,8 +109,8 @@ func TestAFeatureWithANewSeatVerbIsOneBatchAndOnePlanInOrderWithNoAfterLine(t *t
|
||||
}
|
||||
|
||||
// Issue 1049: a move delivered half before its second merge. The controller adds a module the catalogue removes, in one
|
||||
// group: the merges are one batch and one plan in either order, the module is built once and from the controller's
|
||||
// repository, and the catalogue's removal does not forget it.
|
||||
// group: the merges are one batch and one plan in either order, no build of the module is asked (it is registered from
|
||||
// the builder's key, ADR 0501 §6), and the catalogue's removal does not forget it.
|
||||
func TestAMoveOfAModuleAcrossTwoRepositoriesIsOnePlanAndBuildsItOnceFromTheNewOne(t *testing.T) {
|
||||
for name, controllerFirst := range map[string]bool{"controller_first": true, "catalogue_first": false} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
@@ -149,17 +149,12 @@ func TestAMoveOfAModuleAcrossTwoRepositoriesIsOnePlanAndBuildsItOnceFromTheNewOn
|
||||
if len(ws) != 1 || len(bs) != 0 {
|
||||
t.Fatalf("a move: %d walk(s), %d batch(es); want exactly one plan", len(ws), len(bs))
|
||||
}
|
||||
var fromController int
|
||||
// **No build of it is asked from either repository** (ADR 0501 §6, issue 1851): the move is registered from the
|
||||
// builder's done key of route-proxy in the controller's repository, on the operator's Approve of the group.
|
||||
for _, a := range *asked {
|
||||
if strings.Contains(a[0], "mesh-controller") && a[1] == "modules/route-proxy" {
|
||||
fromController++
|
||||
if a[1] == "modules/route-proxy" {
|
||||
t.Errorf("a build of route-proxy was asked: %v", a)
|
||||
}
|
||||
if strings.Contains(a[0], "mesh-catalog") && a[1] == "modules/route-proxy" {
|
||||
t.Errorf("route-proxy was built from the repository it left: %v", a)
|
||||
}
|
||||
}
|
||||
if fromController != 1 {
|
||||
t.Fatalf("route-proxy was built %d time(s) from the controller's repository: %v", fromController, *asked)
|
||||
}
|
||||
entries, err := open.inventory.Catalogued(ctx)
|
||||
if err != nil {
|
||||
|
||||
@@ -28,48 +28,6 @@ func stoodOn(edges map[string][]string) map[string][]string {
|
||||
return out
|
||||
}
|
||||
|
||||
// A module built before the module it stands on is built against the old one and reports success
|
||||
// (novox/hq 04-ISSUES/131). So bases come first, however the set arrived.
|
||||
func TestBasesAreBuiltBeforeWhatStandsOnThem(t *testing.T) {
|
||||
in := []inventory.Entry{entry("app"), entry("runtime"), entry("other"), entry("base")}
|
||||
edges := stoodOn(map[string][]string{"app": {"runtime"}, "runtime": {"base"}})
|
||||
got := orderByBases(in, edges)
|
||||
pos := map[string]int{}
|
||||
for i, e := range got {
|
||||
pos[e.Manifest.Module] = i
|
||||
}
|
||||
if !(pos["base"] < pos["runtime"] && pos["runtime"] < pos["app"]) {
|
||||
t.Fatalf("bases not first: %v", pos)
|
||||
}
|
||||
if len(got) != 4 {
|
||||
t.Fatalf("an entry was lost or doubled: %d", len(got))
|
||||
}
|
||||
// A base outside the set is not waited for: it is not being rebuilt.
|
||||
got = orderByBases([]inventory.Entry{entry("app")}, stoodOn(map[string][]string{"app": {"elsewhere"}}))
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("a dependency outside the set changed the set: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A module registered from its manifest and never built still names its bases there; once built,
|
||||
// the recorded edge is what says so. Both are read, and a module never stands on itself.
|
||||
func TestWhatStandsOnAModuleIsReadFromItsBuildOrItsManifest(t *testing.T) {
|
||||
built := entry("gitea")
|
||||
edges := stoodOn(map[string][]string{"gitea": {"mesh-tools"}})
|
||||
if !standsOnModule(built, "mesh-tools", edges) {
|
||||
t.Fatal("a recorded edge was not read")
|
||||
}
|
||||
if standsOnModule(built, "gitea", edges) || standsOnModule(built, "postgres", edges) {
|
||||
t.Fatal("an edge was invented")
|
||||
}
|
||||
fresh := inventory.Entry{Manifest: catalogue.Manifest{Module: "plex", Build: &catalogue.Build{
|
||||
On: []catalogue.BuildsOn{{Arg: "RUNTIME_BASE", Module: "mesh-tools", Artifact: "runtime"}},
|
||||
}}}
|
||||
if !standsOnModule(fresh, "mesh-tools", nil) {
|
||||
t.Fatal("a manifest's own base was not read")
|
||||
}
|
||||
}
|
||||
|
||||
// A merge names a repository the way the forge does; a source is recorded the way a build was
|
||||
// asked for. The two meet on owner/repo and branch, whichever form the record took.
|
||||
func TestAMergeMatchesTheSourcesBuiltFromIt(t *testing.T) {
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// A new controller's own grants reach the bus before it writes (novox/hq issue 1800).
|
||||
//
|
||||
// A controller's grants are composed by the controller binary itself (broker.PermissionsFor), so the controller that
|
||||
// walks a delivery cannot put its successor's grants in the bus's user list: the list that carries them was sent only
|
||||
// once the new controller passed its gate on its first machine (carryUserList). Until then the new controller wrote
|
||||
// buckets its account was not granted, and the bus refused them: plans-building on 2026-10-21, declared-packages on
|
||||
// 2026-10-22, and R4e's states-announced would be the third.
|
||||
//
|
||||
// So a controller that starts, holding the lease and before it starts anything that writes its buckets, sends the
|
||||
// machine holding the bus its user list when the list it composes is not the one that machine was last sent: the grants
|
||||
// step's send (release.go grantsStep), every build there kept at the one it runs, refused unsent when the declaration
|
||||
// would change more than the user list (errNotGrantsOnly). No build of a running plan reaches that machine before the
|
||||
// plan's own tier sends it (ADR 0503 §8). A send that cannot be made is said once, and the controller serves on.
|
||||
//
|
||||
// It does not ask heldMachines, as carryUserList does: every reason that gives is a build that would move with the
|
||||
// list (a policy, a plan not yet sent there, no gate passed), and here none can. The running plan that brought this
|
||||
// controller holds the bus's machine until its tier sends it; asking would leave the list behind exactly when 1800
|
||||
// meets it. The refusal unsent (errNotGrantsOnly) is the guard instead.
|
||||
|
||||
// ownGrantsCarrier is what the start's send reads and does: whether the bus's list is behind and on which machine, and
|
||||
// the grants-only send to it. Variables so a test needs no store and no bus.
|
||||
type ownGrantsCarrier struct {
|
||||
behind func(ctx context.Context) (holder string, behind bool, err error)
|
||||
send func(ctx context.Context, holder string) error
|
||||
}
|
||||
|
||||
// carry sends the machine holding the bus the controller's own user list when it is behind, and says what it did.
|
||||
func (c ownGrantsCarrier) carry(ctx context.Context) string {
|
||||
holder, behind, err := c.behind(ctx)
|
||||
switch {
|
||||
case err != nil:
|
||||
return fmt.Sprintf("whether the bus's user list carries this controller's grants cannot be read, so it is not sent "+
|
||||
"at start; a bucket it newly writes may be refused until the gate carries it: %v", err)
|
||||
case holder == "" || !behind:
|
||||
return ""
|
||||
}
|
||||
if err := c.send(ctx, holder); err != nil {
|
||||
return fmt.Sprintf("this controller's grants could not be sent to %s, which holds the bus, at its start; a bucket "+
|
||||
"it newly writes may be refused until the gate carries them: %v", holder, err)
|
||||
}
|
||||
return fmt.Sprintf("sent %s, which holds the bus, this controller's user list alone, every build there kept, "+
|
||||
"before it writes", holder)
|
||||
}
|
||||
|
||||
// startWithOwnGrants carries the controller's own grants to the bus and only then starts what writes (then).
|
||||
func startWithOwnGrants(ctx context.Context, c ownGrantsCarrier, then func()) {
|
||||
if said := c.carry(ctx); said != "" {
|
||||
fmt.Println(said)
|
||||
}
|
||||
then()
|
||||
}
|
||||
|
||||
// ownGrantsOf is the carrier of the serving controller: the grants step's reading and its grants-only send.
|
||||
func ownGrantsOf(open *stores) ownGrantsCarrier {
|
||||
return ownGrantsCarrier{
|
||||
behind: func(ctx context.Context) (string, bool, error) { return brokerBehindOf(ctx, open, nil) },
|
||||
send: func(ctx context.Context, holder string) error {
|
||||
_, err := sendRollout(keepingEveryBuild(withScope(ctx, sendScope{}), holder), open, []string{holder})
|
||||
return err
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// 1800 (1): a controller starting with the bus's list behind its own sends the bus's machine once, before anything
|
||||
// that writes its buckets starts.
|
||||
func TestAStartingControllerSendsItsGrantsBeforeItWrites(t *testing.T) {
|
||||
var order []string
|
||||
c := ownGrantsCarrier{
|
||||
behind: func(context.Context) (string, bool, error) { return "novox", true, nil },
|
||||
send: func(_ context.Context, holder string) error { order = append(order, "send "+holder); return nil },
|
||||
}
|
||||
startWithOwnGrants(t.Context(), c, func() { order = append(order, "writers") })
|
||||
if strings.Join(order, ", ") != "send novox, writers" {
|
||||
t.Fatalf("the start did %v", order)
|
||||
}
|
||||
}
|
||||
|
||||
// 1800 (3), (4): no send when the list is current or there is no bus with a list; one that cannot be read or sent is
|
||||
// said once, and the controller goes on.
|
||||
func TestAStartingControllerSendsNothingWhenTheListIsCurrentAndGoesOnWhenRefused(t *testing.T) {
|
||||
sends := 0
|
||||
send := func(context.Context, string) error { sends++; return nil }
|
||||
for _, c := range []ownGrantsCarrier{
|
||||
{behind: func(context.Context) (string, bool, error) { return "novox", false, nil }, send: send},
|
||||
{behind: func(context.Context) (string, bool, error) { return "", true, nil }, send: send},
|
||||
} {
|
||||
if said := c.carry(t.Context()); said != "" || sends != 0 {
|
||||
t.Fatalf("a current list sent %d and said %q", sends, said)
|
||||
}
|
||||
}
|
||||
refused := ownGrantsCarrier{behind: func(context.Context) (string, bool, error) { return "novox", true, nil },
|
||||
send: func(context.Context, string) error { return errNotGrantsOnly }}
|
||||
went := false
|
||||
startWithOwnGrants(t.Context(), refused, func() { went = true })
|
||||
if !went {
|
||||
t.Fatal("a refused send stopped the controller")
|
||||
}
|
||||
if said := refused.carry(t.Context()); !strings.Contains(said, "could not be sent to novox") ||
|
||||
!strings.Contains(said, errNotGrantsOnly.Error()) {
|
||||
t.Fatalf("a refused send is said as %q", said)
|
||||
}
|
||||
unread := ownGrantsCarrier{behind: func(context.Context) (string, bool, error) { return "", false, errors.New("no store") },
|
||||
send: send}
|
||||
if said := unread.carry(t.Context()); !strings.Contains(said, "cannot be read") || sends != 0 {
|
||||
t.Fatalf("an unread list: sent %d, said %q", sends, said)
|
||||
}
|
||||
}
|
||||
|
||||
// 1800 (2): the start's send is the grants step's: composed with every build of the bus's machine kept at the one it
|
||||
// runs, so the declaration differs from its last only by the user list (and is refused unsent otherwise,
|
||||
// errNotGrantsOnly, release.go), whatever a running plan registered since.
|
||||
func TestAStartingControllersSendKeepsEveryBuildOfTheBusMachine(t *testing.T) {
|
||||
was := sendRollout
|
||||
t.Cleanup(func() { sendRollout = was })
|
||||
var kept bool
|
||||
var named []string
|
||||
sendRollout = func(ctx context.Context, _ *stores, names []string) ([]string, error) {
|
||||
kept, named = everyBuildKept(ctx, "novox"), names
|
||||
return names, nil
|
||||
}
|
||||
if err := ownGrantsOf(nil).send(t.Context(), "novox"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !kept || len(named) != 1 || named[0] != "novox" {
|
||||
t.Fatalf("the start's send keeps every build: %v, names %v", kept, named)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// B1 (ADR 0499 §1, §5; issue 1758): a module whose every artifact names the paths it is built from is read from those
|
||||
// paths and its manifest alone. mesh-sdk's TypeScript package at its repository's root: a change to its go/ touches no
|
||||
// module, one to its src/ or its package.json touches it. A module with an artifact naming no paths reads its directory.
|
||||
func TestAModuleNamingItsPathsIsReadFromThemAlone(t *testing.T) {
|
||||
m, err := catalogue.ParseManifest([]byte(`{"module":"mesh-sdk","version":"1","build":{"artifacts":[{"name":"lib",
|
||||
"kind":"package","language":"typescript","paths":["src","package.json","package-lock.json","tsconfig.json"]}]},
|
||||
"resources":[]}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
own := pathsBuildSource(m, "")
|
||||
read := inventory.WithBuildSources(nil, []inventory.BuildSource{{Paths: own}})
|
||||
e := inventory.Entry{Manifest: m, Source: inventory.Source{Repository: "novox/mesh-sdk", Path: ""}}
|
||||
for file, reads := range map[string]bool{
|
||||
"go/delivery/builder.go": false, "go/go.mod": false, "README.md": false,
|
||||
"src/index.ts": true, "src/deep/x.ts": true, "package.json": true, "module.json": true,
|
||||
} {
|
||||
if got := readsFile(e, read, file); got != reads {
|
||||
t.Errorf("%s: read %v, want %v", file, got, reads)
|
||||
}
|
||||
}
|
||||
// In a module's directory, the paths are under it.
|
||||
if got := pathsBuildSource(m, "modules/sdk"); got[0] != "modules/sdk/module.json" || got[1] != "modules/sdk/src/**" {
|
||||
t.Errorf("paths under a directory: %v", got)
|
||||
}
|
||||
// An artifact naming no paths: the whole directory, as before.
|
||||
m.Build.Artifacts = append(m.Build.Artifacts, catalogue.Artifact{Name: "files", Kind: "archive", From: "files"})
|
||||
if got := pathsBuildSource(m, ""); got != nil {
|
||||
t.Errorf("a module with an artifact naming no paths is narrowed to %v", got)
|
||||
}
|
||||
}
|
||||
+116
-22
@@ -9,6 +9,9 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"github.com/novox/mesh-controller/internal/builderkeys"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
@@ -85,8 +88,30 @@ const (
|
||||
unknownModule = iota
|
||||
buildInFlight
|
||||
knownNotBuilt
|
||||
// keyComing is a module the indexer holds and the catalogue does not yet: registered from the builder's done key
|
||||
// of it (ADR 0501 §6), which nobody asks.
|
||||
keyComing
|
||||
)
|
||||
|
||||
// keyPendingPrefix opens the build a pending assignment waits for when it waits for a module's key: no build is asked
|
||||
// (ADR 0501 §5, §6; issue 1851, A10), so it names the module's key, not a build id.
|
||||
const keyPendingPrefix = "key:"
|
||||
|
||||
// indexedModule is the indexer's trunk value of a module the catalogue does not hold, found when it holds one. A
|
||||
// variable so a test needs no bus.
|
||||
var indexedModule = func(ctx context.Context, module string) (sdkdelivery.Module, bool, error) {
|
||||
keys, closeKeys, err := openBuilderKeys()
|
||||
if err != nil {
|
||||
return sdkdelivery.Module{}, false, err
|
||||
}
|
||||
defer closeKeys()
|
||||
v, err := keys.ModuleAt(ctx, module, "")
|
||||
if errors.Is(err, builderkeys.ErrNoValue) {
|
||||
return sdkdelivery.Module{}, false, nil
|
||||
}
|
||||
return v, err == nil, err
|
||||
}
|
||||
|
||||
// notHeld is where a module the catalogue does not hold stands: its case, the build request it was read
|
||||
// from, and the sentence that says it.
|
||||
type notHeld struct {
|
||||
@@ -96,7 +121,24 @@ type notHeld struct {
|
||||
}
|
||||
|
||||
// whereIs looks for a module the catalogue does not hold among the build requests the controller keeps.
|
||||
func whereIs(ctx context.Context, inv *inventory.Inventory, module string, now time.Time) (notHeld, error) {
|
||||
// notHeldReads is what whereIs reads of the records (*inventory.Inventory, or a test's).
|
||||
type notHeldReads interface {
|
||||
RequestsNamed(ctx context.Context, name string) ([]inventory.RequestOutcome, error)
|
||||
Catalogue(ctx context.Context) (map[string]catalogue.Manifest, error)
|
||||
RequestedNames(ctx context.Context) ([]string, error)
|
||||
}
|
||||
|
||||
func whereIs(ctx context.Context, inv notHeldReads, module string, now time.Time) (notHeld, error) {
|
||||
// **Its key is coming** (ADR 0501 §6): the indexer holds it, and the controller registers it from the builder's done
|
||||
// key of it. Read before the build requests of before R4e, which nothing writes any more.
|
||||
if v, held, err := indexedModule(ctx, module); err != nil {
|
||||
return notHeld{}, fmt.Errorf("whether the indexer holds %s cannot be read: %w", module, err)
|
||||
} else if held {
|
||||
return notHeld{kind: keyComing, request: inventory.RequestOutcome{BuildRequest: inventory.BuildRequest{
|
||||
ID: keyPendingPrefix + module, Repository: v.Repository, Seat: gitSeat, Path: keyPath(v)}},
|
||||
said: fmt.Sprintf("%s is not registered yet: the indexer holds it in %s at %.8s, and it is registered from "+
|
||||
"the builder's done key of it, which no one asks", module, v.Repository, v.Commit)}, nil
|
||||
}
|
||||
requests, err := inv.RequestsNamed(ctx, module)
|
||||
if err != nil {
|
||||
return notHeld{}, err
|
||||
@@ -169,6 +211,12 @@ func notRegistered(ctx context.Context, open *stores, node, module string, opts
|
||||
return "", err
|
||||
}
|
||||
switch where.kind {
|
||||
case keyComing:
|
||||
lead := where.said
|
||||
if opts.Build {
|
||||
lead += "\n build \"true\" asks nothing: the builder builds every fingerprint the indexer writes (ADR 0501 §6)"
|
||||
}
|
||||
return keepPending(ctx, open, node, module, where.request, waiting, lead)
|
||||
case buildInFlight:
|
||||
lead := where.said
|
||||
if opts.Build {
|
||||
@@ -183,30 +231,20 @@ func notRegistered(ctx context.Context, open *stores, node, module string, opts
|
||||
also = fmt.Sprintf("\n %s already has a pending assignment of %s, waiting for build %s; build "+
|
||||
"\"true\" asks for a new build and makes it wait for that one", node, module, waiting.Build)
|
||||
}
|
||||
return "", fmt.Errorf("%w: %s%s\n assign it again with build \"true\" to ask for its build and keep this "+
|
||||
"assignment until the build registers it; or `build` it (repository %s, path %s) and assign it "+
|
||||
"once it is registered", inventory.ErrNoSuchModule, where.said, also, last.Repository, orRoot(last.Path))
|
||||
return "", fmt.Errorf("%w: %s%s\n it is registered from the builder's done key once a merge adds it to %s "+
|
||||
"(path %s) and the indexer holds it, or at the controller's terminal; assign it then",
|
||||
inventory.ErrNoSuchModule, where.said, also, last.Repository, orRoot(last.Path))
|
||||
}
|
||||
if waiting != nil && waiting.State == inventory.PendingApplying {
|
||||
return "", fmt.Errorf("%s is being assigned %s now; nothing was asked", node, module)
|
||||
}
|
||||
source := buildSource{Repository: last.Repository, Seat: last.Seat}
|
||||
id, err := askABuild(ctx, source, last.Path, last.Ref)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("%s\n its build could not be asked for: %w; nothing was assigned", where.said, err)
|
||||
}
|
||||
asked := inventory.RequestOutcome{BuildRequest: inventory.BuildRequest{ID: id, Repository: last.Repository,
|
||||
Seat: last.Seat, Path: last.Path, Ref: last.Ref, For: "assign", At: now.UTC()}}
|
||||
recordBuildRequest(ctx, inv, asked.BuildRequest)
|
||||
lead := fmt.Sprintf("%s\n build %s of %s is asked for now; %s can be assigned once it registers it",
|
||||
where.said, id, requestSource(asked), module)
|
||||
return keepPending(ctx, open, node, module, asked, waiting, lead)
|
||||
// **No build is asked** (ADR 0501 §5, §6): a module the indexer does not hold has no key coming.
|
||||
return "", fmt.Errorf("%w: %s\n build \"true\" asks no build: the controller asks none (ADR 0501 §5); a module "+
|
||||
"is registered from the builder's done key once a merge adds it to a repository the catalogue builds from, or "+
|
||||
"at the controller's terminal, and the indexer holds no %s", inventory.ErrNoSuchModule, where.said, module)
|
||||
}
|
||||
answer := strings.TrimPrefix(where.said, inventory.ErrNoSuchModule.Error()) +
|
||||
"\n a module in a repository is built with `build` (its repository and path), then assigned"
|
||||
"\n a module in a repository is registered from the builder's done key of it, then assigned"
|
||||
if opts.Build {
|
||||
answer += "\n build \"true\" asks for nothing here: the controller has no build request saying where a " +
|
||||
"module of that name is"
|
||||
answer += "\n build \"true\" asks for nothing: the controller asks no build (ADR 0501 §5), and the indexer " +
|
||||
"holds no module of that name"
|
||||
}
|
||||
return "", fmt.Errorf("%w%s", inventory.ErrNoSuchModule, answer)
|
||||
}
|
||||
@@ -316,7 +354,7 @@ func clock(t time.Time) string { return t.Local().Format("2006-01-02 15:04:05 MS
|
||||
|
||||
// closestNames is up to three names near the one asked: registered modules and directories the controller
|
||||
// asked to build.
|
||||
func closestNames(ctx context.Context, inv *inventory.Inventory, name string) []string {
|
||||
func closestNames(ctx context.Context, inv notHeldReads, name string) []string {
|
||||
var candidates []string
|
||||
if shelf, err := inv.Catalogue(ctx); err == nil {
|
||||
for m := range shelf {
|
||||
@@ -535,6 +573,19 @@ func settlePending(ctx context.Context, open *stores, now time.Time) []string {
|
||||
}
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(p.Build, keyPendingPrefix) {
|
||||
why, err := keyPendingEnded(ctx, p.Module)
|
||||
if err != nil {
|
||||
say("the key %s waits for cannot be read: %v", p.Module, err)
|
||||
continue
|
||||
}
|
||||
if why != "" {
|
||||
if line := expirePending(ctx, inv, p, why); line != "" {
|
||||
said = append(said, line)
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
requests, err := inv.RequestsNamed(ctx, p.Module)
|
||||
if err != nil {
|
||||
say("the build requests for %s could not be read: %v", p.Module, err)
|
||||
@@ -744,3 +795,46 @@ func withdrawPending(ctx context.Context, inv *inventory.Inventory, node, module
|
||||
}
|
||||
return "", false, nil
|
||||
}
|
||||
|
||||
// keyPendingEnded is why a pending assignment waiting for a module's key ends: the indexer no longer holds the module,
|
||||
// or the builder's key of its fingerprint failed; empty while it is coming. A variable so a test needs no bus.
|
||||
var keyPendingEnded = func(ctx context.Context, module string) (string, error) {
|
||||
keys, closeKeys, err := openBuilderKeys()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer closeKeys()
|
||||
v, err := keys.ModuleAt(ctx, module, "")
|
||||
if errors.Is(err, builderkeys.ErrNoValue) {
|
||||
return fmt.Sprintf("the indexer no longer holds %s", module), nil
|
||||
}
|
||||
if err != nil || v.FP2 == "" {
|
||||
return "", err
|
||||
}
|
||||
k, err := keys.Build(ctx, module, v.FP2)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if k.Failed() {
|
||||
return k.Says(), nil
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
|
||||
// settlePendingOnKey assigns, now, every pending assignment of a module its done key just registered (ADR 0501 §6):
|
||||
// what an outcome of a build did for one waiting on that build.
|
||||
func settlePendingOnKey(ctx context.Context, open *stores, module string) []string {
|
||||
waiting, err := open.inventory.Pending(ctx, time.Time{})
|
||||
if err != nil {
|
||||
return []string{fmt.Sprintf("the pending assignments could not be read after %s was registered: %v", module, err)}
|
||||
}
|
||||
var said []string
|
||||
for _, p := range waiting {
|
||||
if p.Module == module {
|
||||
if line := applyPending(ctx, open, p, keyPendingPrefix+module); line != "" {
|
||||
said = append(said, line)
|
||||
}
|
||||
}
|
||||
}
|
||||
return said
|
||||
}
|
||||
|
||||
@@ -3,7 +3,6 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
@@ -13,6 +12,7 @@ import (
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
@@ -36,11 +36,13 @@ func aCatalogueMesh(t *testing.T) *stores {
|
||||
return open
|
||||
}
|
||||
|
||||
// mergeAdding keeps the build request a merge adding a module asked before R4e (issue 300): nothing asks one since (ADR
|
||||
// 0501 §6), and the pending assignments still read the ones kept. These tests are of what an assignment does with one.
|
||||
func mergeAdding(t *testing.T, open *stores, dir, commit string) {
|
||||
t.Helper()
|
||||
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: commit,
|
||||
Paths: []string{dir + "/module.json"}, ModuleDirs: []string{dir}, ModuleDirsSaid: true}
|
||||
if err := (following{open: open}).SourceMoved(t.Context(), m); err != nil {
|
||||
if err := open.inventory.RecordBuildRequest(t.Context(), inventory.BuildRequest{ID: "b-" + dir,
|
||||
Repository: "novox/mesh-catalog", Seat: "git", Path: dir, Ref: "main", Commit: commit, For: "merge",
|
||||
At: time.Now().UTC()}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
@@ -175,7 +177,7 @@ func TestAPendingAssignmentExpiresWhenItsBuildFails(t *testing.T) {
|
||||
t.Fatalf("a module whose build failed: %v", err)
|
||||
}
|
||||
for _, want := range []string{"sensors is known and not registered", "its last build, b-modules/sensors",
|
||||
"failed: go build: undefined: sensorsRead", "build \"true\"", "repository novox/mesh-catalog, path modules/sensors"} {
|
||||
"failed: go build: undefined: sensorsRead", "registered from the builder's done key", "novox/mesh-catalog (path modules/sensors)"} {
|
||||
if !strings.Contains(err.Error(), want) {
|
||||
t.Fatalf("the refusal does not say %q:\n%v", want, err)
|
||||
}
|
||||
@@ -225,100 +227,10 @@ func TestTheConditionClearsWhenTheModuleIsAssignedLater(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestAssignWithBuildAsksForAKnownModule(t *testing.T) {
|
||||
open := aCatalogueMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
if err := inv.RecordBuildRequest(ctx, inventory.BuildRequest{ID: "build-old", Repository: "novox/mesh-catalog",
|
||||
Seat: "git", Path: "modules/sensors", Ref: "main", For: "build", At: time.Now().Add(-time.Hour)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-old", Repository: "novox/mesh-catalog", Ref: "main",
|
||||
Path: "modules/sensors", On: "anchor", Failed: "no space left on device"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
asked := asksWithPaths(t)
|
||||
said, err := assignWith(ctx, open, "laptop", assignOptions{Build: true}, "sensors")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(*asked) != 1 || (*asked)[0] != [3]string{"novox/mesh-catalog", "modules/sensors", "main"} {
|
||||
t.Fatalf("asked %v", *asked)
|
||||
}
|
||||
for _, want := range []string{"known and not registered", "no space left on device",
|
||||
"build b-modules/sensors of novox/mesh-catalog@main (modules/sensors) is asked for now", "kept as pending"} {
|
||||
if !strings.Contains(said, want) {
|
||||
t.Fatalf("the answer does not say %q:\n%s", want, said)
|
||||
}
|
||||
}
|
||||
pending := pendingOf(t, open)
|
||||
if len(pending) != 1 || pending[0].Build != "b-modules/sensors" {
|
||||
t.Fatalf("pending: %+v", pending)
|
||||
}
|
||||
requests, _ := inv.RequestsNamed(ctx, "sensors")
|
||||
if len(requests) != 2 || requests[0].ID != "b-modules/sensors" || requests[0].For != "assign" {
|
||||
t.Fatalf("the request is not kept as assign's: %+v", requests)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAssignWithBuildRepointsTheWaitingAssignment(t *testing.T) {
|
||||
open := aCatalogueMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
if err := inv.RecordBuildRequest(ctx, inventory.BuildRequest{ID: "build-old", Repository: "novox/mesh-catalog",
|
||||
Seat: "git", Path: "modules/sensors", Ref: "main", For: "merge", At: time.Now().Add(-time.Hour)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-old", Repository: "novox/mesh-catalog", Ref: "main",
|
||||
Path: "modules/sensors", On: "anchor", Failed: "boom"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Waiting on the failed build, its expiry not yet settled.
|
||||
if _, err := inv.RecordPending(ctx, inventory.PendingAssignment{Node: "laptop", Module: "sensors",
|
||||
Build: "build-old", Repository: "novox/mesh-catalog", Path: "modules/sensors"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err := assign(ctx, open, "laptop", "sensors")
|
||||
if err == nil || !strings.Contains(err.Error(), "already has a pending assignment of sensors, waiting for build build-old") {
|
||||
t.Fatalf("known and not built with a pending assignment waiting: %v", err)
|
||||
}
|
||||
asked := asksWithPaths(t)
|
||||
said, err := assignWith(ctx, open, "laptop", assignOptions{Build: true}, "sensors")
|
||||
if err != nil || !strings.Contains(said, "which waited for build build-old, now waits for build b-modules/sensors") {
|
||||
t.Fatalf("assign with build: %q %v", said, err)
|
||||
}
|
||||
if len(*asked) != 1 {
|
||||
t.Fatalf("asked %v", *asked)
|
||||
}
|
||||
pending := pendingOf(t, open)
|
||||
if len(pending) != 1 || pending[0].Build != "b-modules/sensors" || pending[0].State != inventory.PendingWaiting {
|
||||
t.Fatalf("pending: %+v", pending)
|
||||
}
|
||||
// The new build in flight now: build "true" again asks nothing.
|
||||
if said, err := assignWith(ctx, open, "laptop", assignOptions{Build: true}, "sensors"); err != nil ||
|
||||
!strings.Contains(said, "no second build was asked") || len(*asked) != 1 {
|
||||
t.Fatalf("a second build true while the build runs: %q %v, asked %v", said, err, *asked)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAFailedAskIsNotABuildInFlight(t *testing.T) {
|
||||
open := aCatalogueMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
was := askABuild
|
||||
askABuild = func(context.Context, buildSource, string, string) (string, error) {
|
||||
return "", errors.New("cannot submit a build: nats: timeout")
|
||||
}
|
||||
t.Cleanup(func() { askABuild = was })
|
||||
mergeAdding(t, open, "modules/sensors", "3da80a4b00aa")
|
||||
_, err := assign(ctx, open, "laptop", "sensors")
|
||||
if err == nil || !strings.Contains(err.Error(), "the merge that added it (3da80a4b) could not ask for its build: cannot submit") {
|
||||
t.Fatalf("a merge that could not ask: %v", err)
|
||||
}
|
||||
if len(pendingOf(t, open)) != 0 {
|
||||
t.Fatal("a pending assignment waits on a build never asked")
|
||||
}
|
||||
|
||||
if err := inv.RecordBuildRequest(ctx, inventory.BuildRequest{ID: "build-waited", Repository: "novox/mesh-catalog",
|
||||
Seat: "git", Path: "modules/gauges", Ref: "main", For: "build"}); err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -326,7 +238,7 @@ func TestAFailedAskIsNotABuildInFlight(t *testing.T) {
|
||||
if err := inv.MarkNotAsked(ctx, "build-waited", "cannot submit a build: no responders"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = assign(ctx, open, "laptop", "gauges")
|
||||
_, err := assign(ctx, open, "laptop", "gauges")
|
||||
if err == nil || !strings.Contains(err.Error(), "build build-waited, asked by build, was not handed over") {
|
||||
t.Fatalf("a build not handed over: %v", err)
|
||||
}
|
||||
@@ -395,7 +307,7 @@ func TestAnUnknownModuleIsRefusedWithTheClosestNames(t *testing.T) {
|
||||
}
|
||||
for _, want := range []string{"no module of that name: sensors", "the catalogue holds no module of that name",
|
||||
"no build request the controller kept (the last 30 days)", "the closest names it holds: sensord",
|
||||
"asks for nothing here"} {
|
||||
"asks for nothing: the controller asks no build"} {
|
||||
if !strings.Contains(err.Error(), want) {
|
||||
t.Fatalf("the refusal does not say %q:\n%v", want, err)
|
||||
}
|
||||
@@ -565,59 +477,6 @@ func TestAPendingAssignmentNeverWaitsSilently(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestASecondAssignmentNotMadeKeepsItsCondition(t *testing.T) {
|
||||
open := aCatalogueMesh(t)
|
||||
ctx := t.Context()
|
||||
asked := 0
|
||||
was := askABuild
|
||||
askABuild = func(context.Context, buildSource, string, string) (string, error) {
|
||||
asked++
|
||||
return fmt.Sprintf("b-%d", asked), nil
|
||||
}
|
||||
t.Cleanup(func() { askABuild = was })
|
||||
mergeAdding(t, open, "modules/sensors", "3da80a4b00aa")
|
||||
if _, err := assign(ctx, open, "laptop", "sensors"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
failedBuild(t, open, "b-1")
|
||||
settlePending(ctx, open, time.Now())
|
||||
first := pendingOf(t, open)[0]
|
||||
if !conditionOpen(t, first) {
|
||||
t.Fatal("the first assignment not made raised nothing")
|
||||
}
|
||||
|
||||
if _, err := assignWith(ctx, open, "laptop", assignOptions{Build: true}, "sensors"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
failedBuild(t, open, "b-2")
|
||||
settlePending(ctx, open, time.Now())
|
||||
|
||||
var second inventory.PendingAssignment
|
||||
for _, p := range pendingOf(t, open) {
|
||||
if p.Build == "b-2" {
|
||||
second = p
|
||||
}
|
||||
}
|
||||
if second.State != inventory.PendingExpired {
|
||||
t.Fatalf("the second: %+v", second)
|
||||
}
|
||||
if !conditionOpen(t, second) {
|
||||
t.Fatal("the second assignment's condition was cleared in the tick that raised it")
|
||||
}
|
||||
if !conditionOpen(t, rowOf(t, open, first.ID)) {
|
||||
t.Fatal("the first was judged answered by a second that itself was not made")
|
||||
}
|
||||
// unassign takes both back, and the next tick clears both.
|
||||
if said, err := unassign(ctx, open, "laptop", "sensors"); err != nil || !strings.Contains(said, "b-1") ||
|
||||
!strings.Contains(said, "b-2") {
|
||||
t.Fatalf("unassign: %q %v", said, err)
|
||||
}
|
||||
settlePending(ctx, open, time.Now())
|
||||
if conditionOpen(t, first) || conditionOpen(t, second) {
|
||||
t.Fatal("a condition stayed open after both were taken back")
|
||||
}
|
||||
}
|
||||
|
||||
func TestARaisedRowIsKeptUntilItsConditionClears(t *testing.T) {
|
||||
open := aCatalogueMesh(t)
|
||||
ctx := t.Context()
|
||||
@@ -735,3 +594,31 @@ func TestABuildNoLongerWaitedForIsStillInFlight(t *testing.T) {
|
||||
t.Fatalf("past the bound: %+v", rows)
|
||||
}
|
||||
}
|
||||
|
||||
// A10 (ADR 0501 §6; issue 1851): an assignment of a module the indexer holds and the catalogue does not yet waits for
|
||||
// the builder's done key of it, asking nothing, and is made when that key registers the module.
|
||||
func TestAnAssignmentWaitsForTheKeyAndAsksNothing(t *testing.T) {
|
||||
open := aCatalogueMesh(t)
|
||||
ctx := t.Context()
|
||||
noBuildAsked(t)
|
||||
b := aBuilderOnTheBus(t)
|
||||
fp := fpOf('a')
|
||||
manifest := b.indexed(t, "sensors", "1", "novox/mesh-catalog", "modules/sensors", "3da80a4b00aa0000", fp)
|
||||
b.built(t, "sensors", "1", fp, manifest, sdkdelivery.BuildBuilding)
|
||||
said, err := assignWith(ctx, open, "laptop", assignOptions{Build: true}, "sensors")
|
||||
if err != nil || !strings.Contains(said, "registered from the builder's done key") || !strings.Contains(said, "kept as pending") {
|
||||
t.Fatalf("A10: the assignment of a module whose key is coming: %q %v", said, err)
|
||||
}
|
||||
if p := pendingOf(t, open); len(p) != 1 || p[0].Build != keyPendingPrefix+"sensors" {
|
||||
t.Fatalf("A10: pending %+v", p)
|
||||
}
|
||||
settlePending(ctx, open, time.Now().Add(time.Hour))
|
||||
if p := pendingOf(t, open); p[0].State != inventory.PendingWaiting {
|
||||
t.Fatalf("A10: a pending assignment waiting for a building key expired: %+v", p)
|
||||
}
|
||||
b.built(t, "sensors", "1", fp, manifest, sdkdelivery.BuildDone)
|
||||
takeNewModules(ctx, open, "sensors")
|
||||
if !slices.Contains(assignedTo(t, open, "laptop"), "sensors") {
|
||||
t.Fatalf("A10: the key registered sensors and the pending assignment was not made: %+v", pendingOf(t, open))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// A10 (ADR 0501 §6; issue 1851): a module the catalogue does not hold and the indexer does is one whose key is coming:
|
||||
// an assignment of it waits for that key, named by the module, with no build asked and no build request read.
|
||||
func TestAModuleTheIndexerHoldsIsWaitedForByItsKey(t *testing.T) {
|
||||
was := indexedModule
|
||||
indexedModule = func(_ context.Context, module string) (sdkdelivery.Module, bool, error) {
|
||||
return sdkdelivery.Module{Repository: "novox/mesh-catalog", Commit: "3da80a4b00aa", Path: "modules/" + module}, true, nil
|
||||
}
|
||||
t.Cleanup(func() { indexedModule = was })
|
||||
where, err := whereIs(t.Context(), noRequests{}, "sensors", time.Now())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if where.kind != keyComing || where.request.ID != keyPendingPrefix+"sensors" || where.request.Path != "modules/sensors" ||
|
||||
!strings.Contains(where.said, "registered from the builder's done key") {
|
||||
t.Fatalf("a module whose key is coming is read as %+v", where)
|
||||
}
|
||||
}
|
||||
|
||||
// noRequests is records with no build request and no module: what a mesh after R4e reads of a new module.
|
||||
type noRequests struct{}
|
||||
|
||||
func (noRequests) RequestsNamed(context.Context, string) ([]inventory.RequestOutcome, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (noRequests) Catalogue(context.Context) (map[string]catalogue.Manifest, error) { return nil, nil }
|
||||
func (noRequests) RequestedNames(context.Context) ([]string, error) { return nil, nil }
|
||||
@@ -0,0 +1,280 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/builder"
|
||||
"github.com/novox/mesh-controller/internal/builderkeys"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// A walk takes its modules from the builder's done keys (novox/hq ADR 0503 §6, ADR 0501 §1, ADR 0498 §8; issue 1851, A6
|
||||
// and A7).
|
||||
//
|
||||
// From R4e a walk asks build-agent for no module but the builder's own (ADR 0453 §5). A module of its tier is marked
|
||||
// taken from a key (PlanModule.Key) and settled by the look: its fingerprint is read from the indexer's trunk value
|
||||
// written for the commit the walk carries of its repository; the builder's key of that fingerprint, done with every
|
||||
// artifact referenced, is taken in (recorded and registered, off the plans' hold) and the module is built; a key not done
|
||||
// yet, or waiting for a reference, holds the tier and says so; a failed key fails the walk in the builder's own words.
|
||||
// **No timer**: the builder's and the indexer's writes wake the look (builderkeys.Watch, watchNewModules).
|
||||
|
||||
// isBuildersOwn says a module is build-agent's own: asked of build-agent on a merge and taken from its build record, never
|
||||
// from the builder's key, until R4g (ADR 0453 §5, ADR 0498 §7, ADR 0520 §1).
|
||||
func isBuildersOwn(module string) bool { return buildersOwn(module) }
|
||||
|
||||
// buildAgentsOwn are those modules, named (ADR 0520 §1): the builder's; build-agent itself, whose image's context is another
|
||||
// repository's; and mesh-tools, the toolchain image build-agent's builds stand on. The builder writes no key of any of them,
|
||||
// and a walk waiting for one would hold its tier for ever. mesh-tools-go is not among them: it builds from a Dockerfile in
|
||||
// its own directory on a pinned base, and has its key from mesh-build-image.
|
||||
var buildAgentsOwn = map[string]bool{sdkdelivery.BuilderModule: true, builder.AgentModule: true, "mesh-tools": true}
|
||||
|
||||
// buildersOwn is isBuildersOwn's rule: a variable only so a test of the asks left (their bus waits, their plan key) may
|
||||
// name modules of its own as build-agent's.
|
||||
var buildersOwn = func(module string) bool { return buildAgentsOwn[module] }
|
||||
|
||||
// planKeyReads is what a walk reads of the indexer and the builder (builderkeys.Reader, or a test's).
|
||||
type planKeyReads interface {
|
||||
ModuleAt(ctx context.Context, module, commit string) (sdkdelivery.Module, error)
|
||||
Build(ctx context.Context, module, fp string) (builderkeys.Key, error)
|
||||
}
|
||||
|
||||
// openPlanKeys opens what a walk reads of the indexer and the builder. A variable so a test needs no bus.
|
||||
var openPlanKeys = func() (planKeyReads, func(), error) {
|
||||
r, closeKeys, err := openBuilderKeys()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
return r, closeKeys, nil
|
||||
}
|
||||
|
||||
// takeFromKey marks a module of a walk's tier as taken from the builder's key: asked of nobody (ADR 0501 §6).
|
||||
func takeFromKey(p *inventory.Plan, state *inventory.PlanModule, e inventory.Entry, now time.Time) {
|
||||
state.BusWaitFrom = nil
|
||||
state.State = "asked"
|
||||
state.AskedAt = &now
|
||||
state.Key = true
|
||||
state.Build, state.Fingerprint = "", ""
|
||||
state.KeyAt = p.CommitOn(e.Source.Repository, followedBranch(e.Source.Ref))
|
||||
state.Why = "waits for the builder's key"
|
||||
state.Commit, state.BuiltAt, state.Made = "", nil, ""
|
||||
fmt.Printf(" tier %d: %s is taken from the builder's key, asking no build\n", p.Tier, e.Manifest.Module)
|
||||
}
|
||||
|
||||
// keyTakes are the take-ins of keys a look started and has not seen end, by key build id, and what each said when it
|
||||
// waits: kept in this process, so a look starts each once and a settle says why it waits.
|
||||
var keyTakes = struct {
|
||||
sync.Mutex
|
||||
running map[string]bool
|
||||
waits map[string]string
|
||||
// taken are the keys whose take-in ended registered, or found the module already at a later build (superseded).
|
||||
taken map[string]bool
|
||||
}{running: map[string]bool{}, waits: map[string]string{}, taken: map[string]bool{}}
|
||||
|
||||
// settleFromKeys settles every module of the tier taken from a key: built from its key's record, failed with the key's
|
||||
// words or the take-in's refusal, or still waiting, saying for what. It says whether it changed the plan. It reads only
|
||||
// the buckets and the records; a done key not recorded yet is taken in off the plans' hold, and its end asks the look.
|
||||
func settleFromKeys(ctx context.Context, open *stores, keys planKeyReads, p *inventory.Plan, tier []string) (bool, error) {
|
||||
return settleKeys(ctx, open.inventory, keys, p, tier, func(module string, trunk sdkdelivery.Module, id string) {
|
||||
inBackground(func() { takeKeyOffTheHold(context.WithoutCancel(ctx), open, module, trunk, id) })
|
||||
})
|
||||
}
|
||||
|
||||
// keyRecords is what settling from keys reads of the records (*inventory.Inventory, or a test's).
|
||||
type keyRecords interface {
|
||||
BuildByID(ctx context.Context, id string) (inventory.Build, bool, error)
|
||||
RefusedOf(ctx context.Context, ids []string) (map[string]string, error)
|
||||
SourceOf(ctx context.Context, module string) (inventory.Source, error)
|
||||
}
|
||||
|
||||
// settleKeys is settleFromKeys on the records handed in, starting a take-in with take.
|
||||
func settleKeys(ctx context.Context, inv keyRecords, keys planKeyReads, p *inventory.Plan, tier []string,
|
||||
take func(module string, trunk sdkdelivery.Module, id string)) (bool, error) {
|
||||
changed := false
|
||||
set := func(s *inventory.PlanModule, why string) {
|
||||
if s.Why != why {
|
||||
s.Why = why
|
||||
changed = true
|
||||
}
|
||||
}
|
||||
for _, m := range tier {
|
||||
s := p.Modules[m]
|
||||
if s == nil || !s.Key || s.State != "asked" {
|
||||
continue
|
||||
}
|
||||
trunk, err := keys.ModuleAt(ctx, m, s.KeyAt)
|
||||
if errors.Is(err, builderkeys.ErrNoValue) {
|
||||
set(s, fmt.Sprintf("waits for the indexer's value of %s at %s", m, orNow(s.KeyAt)))
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return changed, err
|
||||
}
|
||||
if trunk.FP2 == "" {
|
||||
why := trunk.None2
|
||||
if why == "" {
|
||||
why = "the indexer wrote no fingerprint of the new form"
|
||||
}
|
||||
set(s, fmt.Sprintf("waits: %s at %s has no fingerprint to take a build by: %s", m, orNow(s.KeyAt), why))
|
||||
continue
|
||||
}
|
||||
if s.Fingerprint != trunk.FP2 {
|
||||
s.Fingerprint, changed = trunk.FP2, true
|
||||
}
|
||||
k, err := keys.Build(ctx, m, trunk.FP2)
|
||||
if err != nil {
|
||||
return changed, err
|
||||
}
|
||||
switch {
|
||||
case k.Failed():
|
||||
// **A failed key fails the walk in the builder's own words** (`digests differ:`, `not stored:`, `not pinned:`,
|
||||
// `version taken:`), and a rebuild of it is a new revision the next walk takes.
|
||||
s.State, s.Why = "failed", k.Build.Error
|
||||
p.State = inventory.PlanFailed
|
||||
p.Note = fmt.Sprintf("%s failed to build in tier %d: %s", m, p.Tier, firstLine(k.Build.Error))
|
||||
changed = true
|
||||
continue
|
||||
case !k.Done() || len(k.Unreferenced()) > 0:
|
||||
set(s, "waits: "+k.Says())
|
||||
continue
|
||||
}
|
||||
id := keyBuildID(k)
|
||||
refused, err := inv.RefusedOf(ctx, []string{id})
|
||||
if err != nil {
|
||||
return changed, fmt.Errorf("the refused registrations cannot be read: %w", err)
|
||||
}
|
||||
for kept, why := range refusalsKeptFor([]string{id}) {
|
||||
refused[kept] = why
|
||||
}
|
||||
if why, ok := refused[id]; ok {
|
||||
s.State, s.Why, s.Build = "failed", why, id
|
||||
p.State = inventory.PlanFailed
|
||||
p.Note = fmt.Sprintf("%s failed to build in tier %d: %s", m, p.Tier, firstLine(why))
|
||||
changed = true
|
||||
continue
|
||||
}
|
||||
b, found, err := inv.BuildByID(ctx, id)
|
||||
if err != nil {
|
||||
return changed, err
|
||||
}
|
||||
if found && b.Worked() {
|
||||
keyTakes.Lock()
|
||||
taken := keyTakes.taken[id]
|
||||
keyTakes.Unlock()
|
||||
registered := taken
|
||||
if !registered {
|
||||
if registered, err = registeredAt(ctx, inv, m, trunk.Commit); err != nil {
|
||||
return changed, err
|
||||
}
|
||||
}
|
||||
if registered {
|
||||
at := b.At
|
||||
s.State, s.BuiltAt, s.Commit, s.Build = "built", &at, trunk.Commit, id
|
||||
s.Made = fmt.Sprintf("taken from the builder's key %s (revision %d), no build asked", short(trunk.FP2), k.Revision)
|
||||
s.Why = ""
|
||||
fmt.Printf("%s: %s taken from the builder's key %s\n", p.ID, m, short(trunk.FP2))
|
||||
changed = true
|
||||
continue
|
||||
}
|
||||
}
|
||||
keyTakes.Lock()
|
||||
running, waits := keyTakes.running[id], keyTakes.waits[id]
|
||||
if !running {
|
||||
keyTakes.running[id] = true
|
||||
}
|
||||
keyTakes.Unlock()
|
||||
if waits != "" {
|
||||
set(s, firstLine(waits))
|
||||
} else {
|
||||
set(s, "taking in the builder's done key "+short(trunk.FP2))
|
||||
}
|
||||
if !running {
|
||||
take(m, trunk, id)
|
||||
}
|
||||
}
|
||||
return changed, nil
|
||||
}
|
||||
|
||||
// registeredAt says the catalogue holds the module registered at the commit the walk takes it at: its key's take-in
|
||||
// registered it there, whether with the key's artifacts or, unchanged, with the build it stands on (no move). A key
|
||||
// recorded at an earlier commit (the same fingerprint at an older merge) is taken in again, to register it here.
|
||||
func registeredAt(ctx context.Context, inv keyRecords, module, commit string) (bool, error) {
|
||||
src, err := inv.SourceOf(ctx, module)
|
||||
if errors.Is(err, inventory.ErrNoSuchModule) {
|
||||
return false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return sameCommitID(src.BuiltFrom, commit), nil
|
||||
}
|
||||
|
||||
func sameCommitID(a, b string) bool {
|
||||
if a == "" || b == "" {
|
||||
return false
|
||||
}
|
||||
if a == b {
|
||||
return true
|
||||
}
|
||||
n := min(len(a), len(b))
|
||||
return n >= 7 && a[:n] == b[:n]
|
||||
}
|
||||
|
||||
func orNow(commit string) string {
|
||||
if commit == "" {
|
||||
return "its value now"
|
||||
}
|
||||
return short(commit)
|
||||
}
|
||||
|
||||
// takeKeyOffTheHold takes a walk's module in from its done key, off the plans' hold: a registration may sweep the
|
||||
// artifact store, and the plans are held by one look at a time. Its end asks the look, which settles the module from the
|
||||
// record; a refusal is kept by the key's build id, so the look fails the walk with it; a wait is said by the look.
|
||||
func takeKeyOffTheHold(ctx context.Context, open *stores, module string, trunk sdkdelivery.Module, id string) {
|
||||
defer func() {
|
||||
keyTakes.Lock()
|
||||
delete(keyTakes.running, id)
|
||||
keyTakes.Unlock()
|
||||
askToLook()
|
||||
}()
|
||||
keys, closeKeys, err := openBuilderKeys()
|
||||
if err != nil {
|
||||
keepKeyWait(id, "waits: the builder's keys cannot be read: "+err.Error())
|
||||
return
|
||||
}
|
||||
defer closeKeys()
|
||||
_, _, err = takeInKey(ctx, open.inventory, keys, keyTake{Module: module, Trunk: trunk,
|
||||
Ask: registrationAsk{Merge: trunk.Commit}})
|
||||
switch {
|
||||
case err == nil, errors.Is(err, inventory.ErrSuperseded):
|
||||
// Superseded is no failure: the module is already at what a later merge's key built, which a walk that carries
|
||||
// this merge is answered by, as a build-agent outcome superseded was.
|
||||
keepKeyWait(id, "")
|
||||
keyTakes.Lock()
|
||||
keyTakes.taken[id] = true
|
||||
keyTakes.Unlock()
|
||||
case errors.Is(err, errKeyWaits):
|
||||
keepKeyWait(id, err.Error())
|
||||
default:
|
||||
why := err.Error()
|
||||
if rerr := open.inventory.RecordRefused(ctx, id, module, why); rerr != nil {
|
||||
fmt.Printf("plans: %s's refusal could not be kept in the store, only in this controller: %v\n", module, rerr)
|
||||
}
|
||||
keepRefusalFor(id, why, time.Now())
|
||||
}
|
||||
}
|
||||
|
||||
func keepKeyWait(id, why string) {
|
||||
keyTakes.Lock()
|
||||
defer keyTakes.Unlock()
|
||||
if why == "" {
|
||||
delete(keyTakes.waits, id)
|
||||
return
|
||||
}
|
||||
keyTakes.waits[id] = why
|
||||
}
|
||||
@@ -0,0 +1,220 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"github.com/novox/mesh-controller/internal/builderkeys"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// planKeysOf is the indexer and the builder as a walk reads them, for a test: values by module and commit, keys by
|
||||
// module.
|
||||
type planKeysOf struct {
|
||||
values map[string]sdkdelivery.Module // by module + "@" + commit
|
||||
keys map[string]builderkeys.Key
|
||||
}
|
||||
|
||||
func (k planKeysOf) ModuleAt(_ context.Context, module, commit string) (sdkdelivery.Module, error) {
|
||||
if v, ok := k.values[module+"@"+commit]; ok {
|
||||
return v, nil
|
||||
}
|
||||
return sdkdelivery.Module{}, builderkeys.ErrNoValue
|
||||
}
|
||||
|
||||
func (k planKeysOf) Build(_ context.Context, module, fp string) (builderkeys.Key, error) {
|
||||
if key, ok := k.keys[module]; ok && key.Fingerprint == fp {
|
||||
return key, nil
|
||||
}
|
||||
return builderkeys.Key{Module: module, Fingerprint: fp}, nil
|
||||
}
|
||||
|
||||
// recordsOf is the build records and refusals a settle reads, for a test.
|
||||
type recordsOf struct {
|
||||
builds map[string]inventory.Build
|
||||
refused map[string]string
|
||||
sources map[string]inventory.Source
|
||||
}
|
||||
|
||||
func (r recordsOf) BuildByID(_ context.Context, id string) (inventory.Build, bool, error) {
|
||||
b, ok := r.builds[id]
|
||||
return b, ok, nil
|
||||
}
|
||||
|
||||
func (r recordsOf) RefusedOf(_ context.Context, ids []string) (map[string]string, error) {
|
||||
out := map[string]string{}
|
||||
for _, id := range ids {
|
||||
if why, ok := r.refused[id]; ok {
|
||||
out[id] = why
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (r recordsOf) SourceOf(_ context.Context, module string) (inventory.Source, error) {
|
||||
if s, ok := r.sources[module]; ok {
|
||||
return s, nil
|
||||
}
|
||||
return inventory.Source{}, inventory.ErrNoSuchModule
|
||||
}
|
||||
|
||||
func aWalkOfOne(module string) (*inventory.Plan, map[string]inventory.Entry) {
|
||||
p := &inventory.Plan{ID: "plan-1", State: inventory.PlanBuilding, Tiers: [][]string{{module}},
|
||||
Modules: map[string]*inventory.PlanModule{}, Repository: "novox/mesh-catalog", Commit: "c0ffee0123456789",
|
||||
Delivery: &inventory.PlanDelivery{Merges: []inventory.PlanMerge{{Repository: "novox/mesh-catalog", Commit: "c0ffee0123456789"}}}}
|
||||
byName := map[string]inventory.Entry{module: {Source: inventory.Source{Repository: "novox/mesh-catalog", Seat: "git",
|
||||
Path: "modules/" + module, Ref: "main"}}}
|
||||
return p, byName
|
||||
}
|
||||
|
||||
// A6 (ADR 0503 §6, ADR 0498 check §7): a walk whose module has a done key goes asked to built with no build asked: the
|
||||
// ask stub is never called, the key is taken in once, and `plans` says it was taken from the builder's key.
|
||||
func TestAWalkTakesADoneKeyAndAsksNoBuild(t *testing.T) {
|
||||
was := askABuild
|
||||
askABuild = func(context.Context, buildSource, string, string) (string, error) {
|
||||
t.Fatal("a walk asked build-agent for a module that is not the builder's own")
|
||||
return "", nil
|
||||
}
|
||||
t.Cleanup(func() { askABuild = was })
|
||||
p, byName := aWalkOfOne("sudo")
|
||||
askModule(t.Context(), p, "sudo", byName)
|
||||
s := p.Modules["sudo"]
|
||||
if s.State != "asked" || !s.Key || s.Build != "" || s.KeyAt != "c0ffee0123456789" {
|
||||
t.Fatalf("the module is not taken from a key: %+v", s)
|
||||
}
|
||||
fp := "src2:" + strings.Repeat("a", 64)
|
||||
k := builderkeys.Key{Module: "sudo", Fingerprint: fp, Found: true, Revision: 2, Build: sdkdelivery.BuilderBuild{
|
||||
State: sdkdelivery.BuildDone, Manifest: strings.Repeat("b", 64), Artifacts: []sdkdelivery.BuiltArtifact{{Name: "a",
|
||||
Digest: "sha256:1", Reference: "artifact-store://sudo/a/blobs/sha256:1"}}}}
|
||||
keys := planKeysOf{values: map[string]sdkdelivery.Module{"sudo@c0ffee0123456789": {FP2: fp, Commit: "c0ffee0123456789"}},
|
||||
keys: map[string]builderkeys.Key{"sudo": k}}
|
||||
records := recordsOf{builds: map[string]inventory.Build{}, sources: map[string]inventory.Source{}}
|
||||
var taken []string
|
||||
take := func(m string, _ sdkdelivery.Module, id string) { taken = append(taken, m+" "+id) }
|
||||
t.Cleanup(func() {
|
||||
keyTakes.Lock()
|
||||
keyTakes.running, keyTakes.waits, keyTakes.taken = map[string]bool{}, map[string]string{}, map[string]bool{}
|
||||
keyTakes.Unlock()
|
||||
})
|
||||
|
||||
changed, err := settleKeys(t.Context(), records, keys, p, p.Tiers[0], take)
|
||||
if err != nil || !changed || len(taken) != 1 || s.State != "asked" || s.Fingerprint != fp {
|
||||
t.Fatalf("a done key not recorded yet is taken in once: %v %v %v %+v", changed, err, taken, s)
|
||||
}
|
||||
// A second look while the take-in runs starts no second one.
|
||||
if _, err := settleKeys(t.Context(), records, keys, p, p.Tiers[0], take); err != nil || len(taken) != 1 {
|
||||
t.Fatalf("a second look took the key in again: %v %v", taken, err)
|
||||
}
|
||||
// The take-in recorded and registered it: the look settles it built from the record.
|
||||
id := keyBuildID(k)
|
||||
records.builds[id] = inventory.Build{ID: id, Module: "sudo", Commit: "c0ffee0123456789", At: time.Now()}
|
||||
records.sources["sudo"] = inventory.Source{BuiltFrom: "c0ffee0123456789"}
|
||||
if _, err := settleKeys(t.Context(), records, keys, p, p.Tiers[0], take); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if s.State != "built" || s.Build != id || !strings.Contains(s.Made, "taken from the builder's key") {
|
||||
t.Fatalf("the module is not built from its key: %+v", s)
|
||||
}
|
||||
if takesOutcome(*p, &inventory.PlanModule{Key: true, State: "asked", AskedAt: s.AskedAt}, time.Now(), "build-1") {
|
||||
t.Fatal("a build-agent outcome answers a module taken from a key")
|
||||
}
|
||||
}
|
||||
|
||||
// A7: a key building waits and says so, then moves on when done, with no timer and no ask; a failed key fails the walk
|
||||
// in the builder's words; a value the indexer has not written yet waits, named.
|
||||
func TestAWalkWaitsForAKeyAndFailsWithItsWords(t *testing.T) {
|
||||
asksOfNone := askABuild
|
||||
askABuild = func(context.Context, buildSource, string, string) (string, error) {
|
||||
t.Fatal("a walk asked a build")
|
||||
return "", nil
|
||||
}
|
||||
t.Cleanup(func() { askABuild = asksOfNone })
|
||||
t.Cleanup(func() {
|
||||
keyTakes.Lock()
|
||||
keyTakes.running, keyTakes.waits, keyTakes.taken = map[string]bool{}, map[string]string{}, map[string]bool{}
|
||||
keyTakes.Unlock()
|
||||
})
|
||||
p, byName := aWalkOfOne("view")
|
||||
askModule(t.Context(), p, "view", byName)
|
||||
s := p.Modules["view"]
|
||||
records := recordsOf{builds: map[string]inventory.Build{}}
|
||||
take := func(string, sdkdelivery.Module, string) {}
|
||||
keys := planKeysOf{values: map[string]sdkdelivery.Module{}, keys: map[string]builderkeys.Key{}}
|
||||
if _, err := settleKeys(t.Context(), records, keys, p, p.Tiers[0], take); err != nil ||
|
||||
!strings.Contains(s.Why, "waits for the indexer's value of view at c0ffee01") {
|
||||
t.Fatalf("no indexer value yet: %q %v", s.Why, err)
|
||||
}
|
||||
fp := "src2:" + strings.Repeat("c", 64)
|
||||
keys.values["view@c0ffee0123456789"] = sdkdelivery.Module{FP2: fp}
|
||||
keys.keys["view"] = builderkeys.Key{Module: "view", Fingerprint: fp, Found: true,
|
||||
Build: sdkdelivery.BuilderBuild{State: sdkdelivery.BuildBuilding, By: "g14"}}
|
||||
if _, err := settleKeys(t.Context(), records, keys, p, p.Tiers[0], take); err != nil ||
|
||||
s.State != "asked" || !strings.Contains(s.Why, "building on g14") || p.State != inventory.PlanBuilding {
|
||||
t.Fatalf("a building key: %+v %v", s, err)
|
||||
}
|
||||
keys.keys["view"] = builderkeys.Key{Module: "view", Fingerprint: fp, Found: true,
|
||||
Build: sdkdelivery.BuilderBuild{State: sdkdelivery.BuildFailed, Error: "not stored: tools: the store answered 500"}}
|
||||
if _, err := settleKeys(t.Context(), records, keys, p, p.Tiers[0], take); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if s.State != "failed" || p.State != inventory.PlanFailed || !strings.Contains(p.Note, "not stored: tools") {
|
||||
t.Fatalf("a failed key does not fail the walk in its words: %+v %q", s, p.Note)
|
||||
}
|
||||
}
|
||||
|
||||
// A7: a take-in refused (a module from another repository, a manifest refused) fails the walk with the refusal.
|
||||
func TestARefusedTakeInFailsTheWalk(t *testing.T) {
|
||||
t.Cleanup(func() {
|
||||
keyTakes.Lock()
|
||||
keyTakes.running, keyTakes.waits, keyTakes.taken = map[string]bool{}, map[string]string{}, map[string]bool{}
|
||||
keyTakes.Unlock()
|
||||
})
|
||||
p, byName := aWalkOfOne("sudo")
|
||||
askModule(t.Context(), p, "sudo", byName)
|
||||
fp := "src2:" + strings.Repeat("d", 64)
|
||||
k := builderkeys.Key{Module: "sudo", Fingerprint: fp, Found: true, Revision: 1, Build: sdkdelivery.BuilderBuild{
|
||||
State: sdkdelivery.BuildDone, Manifest: strings.Repeat("b", 64), Artifacts: []sdkdelivery.BuiltArtifact{{Name: "a",
|
||||
Digest: "sha256:1", Reference: "artifact-store://sudo/a/blobs/sha256:1"}}}}
|
||||
keys := planKeysOf{values: map[string]sdkdelivery.Module{"sudo@c0ffee0123456789": {FP2: fp}},
|
||||
keys: map[string]builderkeys.Key{"sudo": k}}
|
||||
records := recordsOf{refused: map[string]string{keyBuildID(k): "sudo is built from novox/mesh-catalog, and this build is of agent/sudo"}}
|
||||
if _, err := settleKeys(t.Context(), records, keys, p, p.Tiers[0], func(string, sdkdelivery.Module, string) {}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if p.State != inventory.PlanFailed || !strings.Contains(p.Note, "agent/sudo") {
|
||||
t.Fatalf("a refused take-in did not fail the walk: %s %q", p.State, p.Note)
|
||||
}
|
||||
}
|
||||
|
||||
// A6: a walk's module whose fingerprint did not change since an earlier merge has its key recorded at that merge's
|
||||
// commit: the walk takes it in again, so the module is registered at the commit the walk carries, before it is built.
|
||||
func TestAKeyRecordedAtAnEarlierCommitIsTakenInAgain(t *testing.T) {
|
||||
t.Cleanup(func() {
|
||||
keyTakes.Lock()
|
||||
keyTakes.running, keyTakes.waits, keyTakes.taken = map[string]bool{}, map[string]string{}, map[string]bool{}
|
||||
keyTakes.Unlock()
|
||||
})
|
||||
p, byName := aWalkOfOne("sudo")
|
||||
askModule(t.Context(), p, "sudo", byName)
|
||||
fp := "src2:" + strings.Repeat("e", 64)
|
||||
k := builderkeys.Key{Module: "sudo", Fingerprint: fp, Found: true, Revision: 1, Build: sdkdelivery.BuilderBuild{
|
||||
State: sdkdelivery.BuildDone, Manifest: strings.Repeat("b", 64), Artifacts: []sdkdelivery.BuiltArtifact{{Name: "a",
|
||||
Digest: "sha256:1", Reference: "artifact-store://sudo/a/blobs/sha256:1"}}}}
|
||||
keys := planKeysOf{values: map[string]sdkdelivery.Module{"sudo@c0ffee0123456789": {FP2: fp, Commit: "c0ffee0123456789"}},
|
||||
keys: map[string]builderkeys.Key{"sudo": k}}
|
||||
id := keyBuildID(k)
|
||||
records := recordsOf{builds: map[string]inventory.Build{id: {ID: id, Module: "sudo", Commit: "0ld0ld0ld0ld"}},
|
||||
sources: map[string]inventory.Source{"sudo": {BuiltFrom: "0ld0ld0ld0ld"}}}
|
||||
var taken []string
|
||||
if _, err := settleKeys(t.Context(), records, keys, p, p.Tiers[0], func(m string, _ sdkdelivery.Module, _ string) {
|
||||
taken = append(taken, m)
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if s := p.Modules["sudo"]; s.State != "asked" || len(taken) != 1 {
|
||||
t.Fatalf("a key recorded at an earlier commit settled the walk at that commit: %+v, taken %v", s, taken)
|
||||
}
|
||||
}
|
||||
@@ -209,10 +209,6 @@ func servingPlanSaved(ctx context.Context, bus link.Bus, open *stores) func(*inv
|
||||
keepBuildingKey(ctx, p)
|
||||
if p.WalkSaid == inventory.WalkEnded {
|
||||
go forgetChecksOfWalk(context.WithoutCancel(ctx), open.inventory, p)
|
||||
// A first build refused for a word only a newer build-agent reads is asked again (novox/hq issue 1173).
|
||||
if deliversBuildAgent(p) {
|
||||
go askHeldFirstBuildsAgain(context.WithoutCancel(ctx), open, p)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -402,11 +402,13 @@ func retryPlan(ctx context.Context, open *stores, id string) (string, error) {
|
||||
var asked []string
|
||||
for _, m := range failed {
|
||||
askModule(ctx, &p, m, byName)
|
||||
if s := p.Modules[m]; s.State == "asked" {
|
||||
if s := p.Modules[m]; s.State == "asked" && s.Key {
|
||||
asked = append(asked, m+" from the builder's key")
|
||||
} else if s.State == "asked" {
|
||||
asked = append(asked, m+" as "+s.Build)
|
||||
}
|
||||
}
|
||||
waiting := busWaitingIn(p)
|
||||
waiting := busWaitingOnRetry(p)
|
||||
note, said := retryWords(p, failed, asked, waiting)
|
||||
resumed(&p, note)
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
@@ -418,6 +420,14 @@ func retryPlan(ctx context.Context, open *stores, id string) (string, error) {
|
||||
return said, nil
|
||||
}
|
||||
|
||||
// busWaitingOnRetry are the modules a retry left waiting for the bus. Read before the plan is resumed, while it still says
|
||||
// failed, so read as the plan the retry resumes: a failed plan is not open, and busWaitingIn would name none, leaving the
|
||||
// answer "asked" with nothing after it (issue 1403).
|
||||
func busWaitingOnRetry(p inventory.Plan) []string {
|
||||
p.State = inventory.PlanBuilding
|
||||
return busWaitingIn(p)
|
||||
}
|
||||
|
||||
// retryWords are the plan's note and the verb's answer for a retry: what was asked again (as the builds it got), and what waits
|
||||
// for the bus. A retry whose modules all wait for the bus asked none, and says so rather than "asked" with nothing after it
|
||||
// (issue 1403).
|
||||
@@ -440,101 +450,6 @@ func retryWords(p inventory.Plan, failed, asked, waiting []string) (note, said s
|
||||
return note, said
|
||||
}
|
||||
|
||||
// joinAPlan asks a module again for the plan that holds it unbuilt or failed, if one does: open plans
|
||||
// first, then the most recent failed one that nothing newer supersedes. Says whether it joined one.
|
||||
func joinAPlan(ctx context.Context, open *stores, module string) (bool, string, error) {
|
||||
inv := open.inventory
|
||||
release, err := inv.HoldPlans(ctx, true)
|
||||
if err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
defer release()
|
||||
openPlans, err := inv.OpenPlans(ctx)
|
||||
if err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
recent, err := inv.RecentPlans(ctx, 20)
|
||||
if err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
p, found := planHolding(module, openPlans, recent)
|
||||
if !found {
|
||||
return false, "", nil
|
||||
}
|
||||
// **A walk that waits for its delivery's word asks nothing** (novox/hq ADR 0239, issue 754): a rebuild does not
|
||||
// join it, and its outcome is not the walk's. The walk asks its own tier once it has its word.
|
||||
if p.Waiting() {
|
||||
return false, fmt.Sprintf("not joining %s (%s at %s): it waits for %s's word, so this build is not its "+
|
||||
"outcome; once let go, the walk asks %s itself", p.ID, p.Repository, short(p.Commit), p.Delivery.Awaits,
|
||||
module), nil
|
||||
}
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
byName := map[string]inventory.Entry{}
|
||||
for _, e := range entries {
|
||||
byName[e.Manifest.Module] = e
|
||||
}
|
||||
was := p.State
|
||||
askModule(ctx, &p, module, byName)
|
||||
s := p.Modules[module]
|
||||
resumed(&p, fmt.Sprintf("tier %d: %s rebuilt by hand", p.Tier, module))
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
if s.State == "" && s.BusWaitFrom != nil {
|
||||
return true, busWaitWords(p, []string{module}), nil
|
||||
}
|
||||
if s.State != "asked" {
|
||||
return true, "", fmt.Errorf("%s could not be asked for %s: %s", module, p.ID, s.Why)
|
||||
}
|
||||
said := fmt.Sprintf("rebuild asked as %s, joining %s at tier %d (%s at %s): the plan takes this build as %s's outcome",
|
||||
s.Build, p.ID, p.Tier, p.Repository, short(p.Commit), module)
|
||||
switch {
|
||||
case was == inventory.PlanFailed && p.State == inventory.PlanBuilding:
|
||||
said += "; the plan had failed and builds again from this tier"
|
||||
case was == inventory.PlanFailed:
|
||||
said += "; the plan stays failed while " + strings.Join(failedIn(p), ", ") + " failed too — `plans retry " + p.ID + "` asks them"
|
||||
}
|
||||
return true, said, nil
|
||||
}
|
||||
|
||||
// planHolding is the plan a rebuild of a module joins: an open plan whose current tier holds it not
|
||||
// yet built, else the newest failed plan whose current tier does, and that no open plan of its
|
||||
// repository supersedes.
|
||||
func planHolding(module string, openPlans, recent []inventory.Plan) (inventory.Plan, bool) {
|
||||
holds := func(p inventory.Plan) bool {
|
||||
if p.Tier >= len(p.Tiers) {
|
||||
return false
|
||||
}
|
||||
for _, m := range p.Tiers[p.Tier] {
|
||||
if m == module {
|
||||
s := p.Modules[m]
|
||||
return s == nil || s.State != "built"
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
for _, p := range openPlans {
|
||||
if holds(p) {
|
||||
return p, true
|
||||
}
|
||||
}
|
||||
sorted := append([]inventory.Plan(nil), recent...)
|
||||
sort.SliceStable(sorted, func(i, j int) bool { return sorted[i].Created.After(sorted[j].Created) })
|
||||
for _, p := range sorted {
|
||||
if p.State != inventory.PlanFailed || !holds(p) {
|
||||
continue
|
||||
}
|
||||
if _, superseded := newerOpenPlan(p, openPlans); superseded {
|
||||
continue
|
||||
}
|
||||
return p, true
|
||||
}
|
||||
return inventory.Plan{}, false
|
||||
}
|
||||
|
||||
// retryRollouts sends each module whose rollout stopped to the machines it was first sent to, again,
|
||||
// records that send as the first anew, and sets the plan rolling: from there it goes on as the plan
|
||||
// would have — the rest sent once those report they applied it, the next tier after (ADR 0218).
|
||||
@@ -629,7 +544,9 @@ func retryTierWhole(ctx context.Context, open *stores, p *inventory.Plan, again
|
||||
for _, m := range again {
|
||||
sendAgain(p.Modules[m])
|
||||
askModule(ctx, p, m, byName)
|
||||
if s := p.Modules[m]; s.State == "asked" {
|
||||
if s := p.Modules[m]; s.State == "asked" && s.Key {
|
||||
asked = append(asked, m+" from the builder's key")
|
||||
} else if s.State == "asked" {
|
||||
asked = append(asked, m+" as "+s.Build)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -156,6 +156,7 @@ func TestASavedPlanPutsOrDeletesItsKey(t *testing.T) {
|
||||
|
||||
// The plan's key is there before its first build is asked, so a check cannot take a place between the ask and the plan's save.
|
||||
func TestThePlansKeyIsPutBeforeItsFirstBuildIsAsked(t *testing.T) {
|
||||
asksOfEveryModule(t)
|
||||
store := newFakeBuildingStore()
|
||||
var s buildingStore = store
|
||||
buildingKeys.Store(&s)
|
||||
|
||||
@@ -1,13 +1,10 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// flagsOf reads a command line's flags (and its first positional after --what-if) into a map.
|
||||
@@ -28,74 +25,6 @@ func flagsOf(argv []string) map[string]string {
|
||||
return out
|
||||
}
|
||||
|
||||
// **The plan of a change as data from `plans`** (novox/hq ADR 0450 §2, issue 1324 AC1): asked for one change
|
||||
// both ways, `plans` with data and `delivery-plan` answer the same plan in every field.
|
||||
func TestPlansWhatIfAsDataAnswersWhatDeliveryPlanAnswers(t *testing.T) {
|
||||
change := map[string]any{"repository": "novox/mesh-catalog", "paths": "modules/gitea/index.ts,modules/gitea/old.txt",
|
||||
"head": "abc123", "base": "main", "module-dirs": "modules/gitea", "removed": "modules/gitea/old.txt"}
|
||||
with := func(extra map[string]any) map[string]any {
|
||||
out := map[string]any{}
|
||||
for k, v := range change {
|
||||
out[k] = v
|
||||
}
|
||||
for k, v := range extra {
|
||||
out[k] = v
|
||||
}
|
||||
return out
|
||||
}
|
||||
plansArgv, err := argvFor("plans", with(map[string]any{"data": "true"}))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
deliveryArgv, err := argvFor("delivery-plan", change)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if plansArgv[0] != "plans" || !slicesHas(plansArgv, "--data") || !slicesHas(plansArgv, "--what-if") {
|
||||
t.Fatalf("plans with data became %q", strings.Join(plansArgv, " "))
|
||||
}
|
||||
p, d := flagsOf(plansArgv), flagsOf(deliveryArgv)
|
||||
for _, name := range []string{"paths", "head", "base", "module-dirs", "removed"} {
|
||||
if p[name] == "" || p[name] != d[name] {
|
||||
t.Errorf("--%s: plans says %q, delivery-plan %q", name, p[name], d[name])
|
||||
}
|
||||
}
|
||||
if p["what-if"] != d["repository"] {
|
||||
t.Errorf("repository: plans says %q, delivery-plan %q", p["what-if"], d["repository"])
|
||||
}
|
||||
|
||||
// Both commands build their document from the one function, from the same member.
|
||||
const src = "http://forge.internal:20000/novox/mesh-catalog.git"
|
||||
gitea := fromRepo("gitea", src, "modules/gitea")
|
||||
gitea.Source.BuiltFrom = "old"
|
||||
gitea.On = []string{"anchor", "laptop"}
|
||||
entries := []inventory.Entry{gitea}
|
||||
policy := func(string) (inventory.Upgrade, bool) { return inventory.Upgrade{RollOut: true}, true }
|
||||
member := func(f map[string]string, repository string) orderMember {
|
||||
return changeMember(repository, f["base"], f["head"], splitList(f["paths"]), splitList(f["removed"]),
|
||||
moduleDirsOf(f["module-dirs"]))
|
||||
}
|
||||
a := deliveryPlanDocument(member(p, p["what-if"]), entries, nil, nil, policy)
|
||||
b := deliveryPlanDocument(member(d, d["repository"]), entries, nil, nil, policy)
|
||||
ja, _ := json.Marshal(a)
|
||||
jb, _ := json.Marshal(b)
|
||||
if !reflect.DeepEqual(ja, jb) {
|
||||
t.Errorf("the two plans differ:\n%s\n%s", ja, jb)
|
||||
}
|
||||
plan := a["plan"].(interface{})
|
||||
raw, _ := json.Marshal(plan)
|
||||
var fields map[string]any
|
||||
_ = json.Unmarshal(raw, &fields)
|
||||
for _, want := range []string{"moved", "tiers", "machines", "summary", "class", "head", "base", "repository"} {
|
||||
if _, ok := fields[want]; !ok {
|
||||
t.Errorf("the plan has no %q: %s", want, raw)
|
||||
}
|
||||
}
|
||||
if fields["head"] != "abc123" || fields["base"] != "main" {
|
||||
t.Errorf("head and base are %v and %v", fields["head"], fields["base"])
|
||||
}
|
||||
}
|
||||
|
||||
// Without the data form the text answer is unchanged (AC2): no data flag, and the data-only arguments are not passed.
|
||||
func TestPlansWhatIfWithoutDataIsUnchanged(t *testing.T) {
|
||||
argv, err := argvFor("plans", map[string]any{"repository": "novox/x", "paths": "a,b"})
|
||||
@@ -114,20 +43,6 @@ func TestPlansWhatIfWithoutDataIsUnchanged(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// delivery-plan stays and answers as before (AC3, ADR 0450 §8).
|
||||
func TestDeliveryPlanStaysAsItWas(t *testing.T) {
|
||||
argv, err := argvFor("delivery-plan", map[string]any{"repository": "novox/x", "paths": "a", "head": "h"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := strings.Join(argv, " "); got != "delivery plan --repository novox/x --paths a --head h" {
|
||||
t.Errorf("delivery-plan reads %q", got)
|
||||
}
|
||||
if _, ok := catalogueVerb("delivery-plan"); !ok {
|
||||
t.Error("delivery-plan is gone")
|
||||
}
|
||||
}
|
||||
|
||||
func catalogueVerb(name string) (catalogue.Verb, bool) { return controllerVerb(name) }
|
||||
|
||||
func slicesHas(s []string, v string) bool {
|
||||
|
||||
@@ -2,7 +2,6 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"path"
|
||||
@@ -138,71 +137,6 @@ func tellPullDeclares(ctx context.Context, w io.Writer, pulls []pullAt) {
|
||||
}
|
||||
}
|
||||
|
||||
// heldFor asks mesh-config for the approved held values of these pull requests at their heads (ADR 0502 clause 4). Only
|
||||
// those of the pull requests asked about are carried. A failure is said to w and answers none.
|
||||
func heldFor(ctx context.Context, w io.Writer, pulls []pullAt) []link.HeldConfig {
|
||||
if len(pulls) == 0 {
|
||||
return nil
|
||||
}
|
||||
type pullHead struct {
|
||||
Pull string `json:"pull"`
|
||||
Head string `json:"head"`
|
||||
}
|
||||
asked := make([]pullHead, 0, len(pulls))
|
||||
for _, p := range pulls {
|
||||
asked = append(asked, pullHead{p.key(), p.New})
|
||||
}
|
||||
raw, err := meshConfigAsk(ctx, map[string]any{"action": "held", "pulls": asked})
|
||||
if err != nil {
|
||||
fmt.Fprintf(w, "mesh-config cannot say what it holds for these pull requests (%v); the check goes on without held values\n", err)
|
||||
return nil // empty-on-error: said above; a check without held values fails for want of the value and says so, it never passes on a guess
|
||||
}
|
||||
var answer struct {
|
||||
Held []link.HeldConfig `json:"held"`
|
||||
Skipped []struct {
|
||||
Pull string `json:"pull"`
|
||||
Why string `json:"why"`
|
||||
} `json:"skipped"`
|
||||
}
|
||||
if len(raw) == 0 {
|
||||
return nil
|
||||
}
|
||||
if err := json.Unmarshal(raw, &answer); err != nil {
|
||||
fmt.Fprintf(w, "mesh-config's held values cannot be read (%v); the check goes on without them\n", err)
|
||||
return nil // empty-on-error: said above; a check without held values fails for want of the value and says so, it never passes on a guess
|
||||
}
|
||||
for _, s := range answer.Skipped {
|
||||
fmt.Fprintf(w, "%s: mesh-config holds a value it does not carry: %s\n", s.Pull, s.Why)
|
||||
}
|
||||
var out []link.HeldConfig
|
||||
for _, h := range answer.Held {
|
||||
for _, p := range pulls {
|
||||
if h.Pull == p.key() {
|
||||
out = append(out, h)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// withHeldConfig tells mesh-config what these pull requests declare and puts the approved held values of them on the
|
||||
// check request (ADR 0502 clause 4). With carry false, a member of a group whose composed check composes, only the
|
||||
// declarations are told.
|
||||
func withHeldConfig(ctx context.Context, w io.Writer, request *link.BuildRequest, pulls []pullAt, carry bool) {
|
||||
tellPullDeclares(ctx, w, pulls)
|
||||
if carry {
|
||||
carryHeld(ctx, w, request, pulls)
|
||||
}
|
||||
}
|
||||
|
||||
// carryHeld puts the approved held values of these pull requests on the check request.
|
||||
func carryHeld(ctx context.Context, w io.Writer, request *link.BuildRequest, pulls []pullAt) {
|
||||
if request.Check != nil {
|
||||
request.Check.HeldConfig = heldFor(ctx, w, pulls)
|
||||
}
|
||||
}
|
||||
|
||||
// holdsModules is whether a module of the mesh is built from this repository.
|
||||
func holdsModules(entries []inventory.Entry, owner, repo string) bool {
|
||||
return slices.ContainsFunc(entries, func(e inventory.Entry) bool {
|
||||
|
||||
@@ -9,7 +9,6 @@ import (
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -153,50 +152,8 @@ func TestMeshConfigNotAnsweringIsSaidAndGoneOn(t *testing.T) {
|
||||
meshConfigAsk = func(context.Context, map[string]any) ([]byte, error) { return nil, io.ErrUnexpectedEOF }
|
||||
var said strings.Builder
|
||||
tellPullDeclares(context.Background(), &said, []pullAt{thePull()})
|
||||
got := heldFor(context.Background(), &said, []pullAt{thePull()})
|
||||
if len(got) != 0 || !strings.Contains(said.String(), "mesh-config") {
|
||||
t.Fatalf("held %v, said %q", got, said.String())
|
||||
}
|
||||
}
|
||||
|
||||
// 1826/AC2: the held values mesh-config answers for the check's pull requests are what the check request carries.
|
||||
func TestTheHeldValuesOfTheChecksPullRequestsAreAsked(t *testing.T) {
|
||||
calls := fakeConfig(t, map[string]string{"held": `{"held":[{"pull":"novox/mesh-catalog#434","head":"` + headSHA +
|
||||
`","node":"ace","module":"n8n","key":"media-gid","value":2000,"approved":"the operator"}]}`})
|
||||
other := pullAt{Number: 7, changeSource: changeSource{Owner: "novox", Repo: "mesh-controller", New: baseSHA}}
|
||||
got := heldFor(context.Background(), io.Discard, []pullAt{thePull(), other})
|
||||
if len(got) != 1 || got[0].Node != "ace" || got[0].Key != "media-gid" || string(got[0].Value) != "2000" {
|
||||
t.Fatalf("got %+v", got)
|
||||
}
|
||||
c := (*calls)[0]
|
||||
pulls, _ := json.Marshal(c["pulls"])
|
||||
if c["action"] != "held" || string(pulls) != `[{"pull":"novox/mesh-catalog#434","head":"`+headSHA+
|
||||
`"},{"pull":"novox/mesh-controller#7","head":"`+baseSHA+`"}]` {
|
||||
t.Fatalf("asked %v %s", c, pulls)
|
||||
}
|
||||
}
|
||||
|
||||
// 1826/AC2: a value held for a pull request the check is not of is never carried.
|
||||
func TestAHeldValueOfAnotherPullRequestIsNotCarried(t *testing.T) {
|
||||
fakeConfig(t, map[string]string{"held": `{"held":[{"pull":"novox/mesh-catalog#999","head":"x","node":"ace","module":"n8n","key":"k","value":1}]}`})
|
||||
if got := heldFor(context.Background(), io.Discard, []pullAt{thePull()}); len(got) != 0 {
|
||||
t.Fatalf("carried %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// 1826/AC2: the check request carries what mesh-config answered; a member of a composed group asks none.
|
||||
func TestTheCheckRequestCarriesTheHeldValues(t *testing.T) {
|
||||
fakeManifests(t, map[string]*catalogue.Manifest{baseSHA: manifest("n8n", nil), headSHA: manifest("n8n", nil)})
|
||||
fakeConfig(t, map[string]string{"held": `{"held":[{"pull":"novox/mesh-catalog#434","node":"ace","module":"n8n","key":"media-gid","value":2000}]}`})
|
||||
request := link.BuildRequest{Check: &link.CheckRequest{}}
|
||||
withHeldConfig(context.Background(), io.Discard, &request, []pullAt{thePull()}, true)
|
||||
if len(request.Check.HeldConfig) != 1 || request.Check.HeldConfig[0].Key != "media-gid" {
|
||||
t.Fatalf("the request carries %+v", request.Check.HeldConfig)
|
||||
}
|
||||
request = link.BuildRequest{Check: &link.CheckRequest{}}
|
||||
withHeldConfig(context.Background(), io.Discard, &request, []pullAt{thePull()}, false)
|
||||
if len(request.Check.HeldConfig) != 0 {
|
||||
t.Fatalf("a composed member carries %+v", request.Check.HeldConfig)
|
||||
if !strings.Contains(said.String(), "mesh-config") {
|
||||
t.Fatalf("said %q", said.String())
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -189,7 +189,9 @@ func serve(ctx context.Context) (err error) {
|
||||
// Open plans move on events (novox/hq ADR 0406 §5, issue 974): a tier is asked on the `built` and the send of the
|
||||
// tier before it, a walk ends on its machines' reports, a silent machine is a key that expired, and a plan left
|
||||
// by a replaced controller resumes from the first look, which this asks for. No timer looks every so often.
|
||||
startRolloutEvents(ctx, open, server.JetStream())
|
||||
// **Its own grants on the bus before it writes** (novox/hq issue 1800): a bucket this controller newly writes is
|
||||
// granted before the first write, not after its gate.
|
||||
startWithOwnGrants(ctx, ownGrantsOf(open), func() { startRolloutEvents(ctx, open, server.JetStream()) })
|
||||
// And the merge window (novox/hq ADR 0276, ADR 0407): a batch is cut into its walk when the window closes,
|
||||
// which no merge or outcome says. The window is two keys on the bus that expire and a watcher that hears it;
|
||||
// nothing polls (issue 973).
|
||||
@@ -286,6 +288,17 @@ func serve(ctx context.Context) (err error) {
|
||||
go sweepingDeclarationFiles(ctx, inv, bus.Conn)
|
||||
// And the key of each plan in its build phase, which holds the build seat's checks while a delivery builds (ADR 0483 clause 2).
|
||||
go keepingPlansBuilding(ctx, inv, bus.Conn)
|
||||
// And the modules the catalogue does not hold yet, registered from the builder's first done key of each, woken by
|
||||
// the builder's and the indexer's writes (novox/hq ADR 0501 §6, issue 1851): nothing asks a first build.
|
||||
watchNewModules(ctx, open)
|
||||
// And says what it keeps, and keeps what every module says it keeps (novox/hq ADR 0495, issue 1790): a changed word
|
||||
// sends the machine holding the bus its new user list.
|
||||
if err := announceOwnStates(ctx, server.JetStream()); err != nil {
|
||||
fmt.Printf("states: %v\n", err)
|
||||
}
|
||||
if err := keepAnnouncements(ctx, server.JetStream(), open); err != nil {
|
||||
fmt.Printf("states: the states the modules announce are not kept: %v\n", err)
|
||||
}
|
||||
// Every call carries the lease's epoch, and its record is written only under the lease (novox/hq
|
||||
// to-be 45 §6).
|
||||
link.Calls.UnderLease(func() (uint64, error) { return theLease.epoch(ctx) })
|
||||
@@ -1479,6 +1492,13 @@ func issueMemberships(ctx context.Context, open *stores, server *link.Server, se
|
||||
if reheard > 0 {
|
||||
fmt.Printf(" issued %d membership(s) again for what they hear of a module's stream\n", reheard)
|
||||
}
|
||||
// **And says again what it keeps** (novox/hq ADR 0495 clause 2, issue 1790): its word at start is refused until a push
|
||||
// carries its grant; kept only when it changed, so saying it at every push changes nothing else.
|
||||
if js := servingBus.Load(); js != nil {
|
||||
if err := announceOwnStates(ctx, js); err != nil {
|
||||
fmt.Printf(" states: %v\n", err)
|
||||
}
|
||||
}
|
||||
if len(refused) > 0 {
|
||||
// Returned, never passed over (novox/hq issue 249): the declarations of the machines they are
|
||||
// for are not sent, and the rollout that asked is tried again rather than waiting for a push.
|
||||
|
||||
+11
-191
@@ -477,89 +477,6 @@ func clearQueue(ctx context.Context, js *broker.JetStream, open *stores, seat st
|
||||
return b.String(), nil
|
||||
}
|
||||
|
||||
// rebuildCommand asks the module's current source again — or, given a build's id, that build's
|
||||
// repository, path and ref — under a new id.
|
||||
func rebuildCommand(ctx context.Context, args []string) error {
|
||||
if len(args) != 1 {
|
||||
return errors.New("rebuild <module | build id>")
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
inv := open.inventory
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var source buildSource
|
||||
var path, ref, module string
|
||||
if b, found, err := inv.BuildByID(ctx, args[0]); err != nil {
|
||||
return err
|
||||
} else if found {
|
||||
source, module = sourceOfBuild(b, entries)
|
||||
path, ref = b.Path, b.Ref
|
||||
// **A build at a commit is rebuilt at what its module follows now** (novox/hq ADR 0219, issue
|
||||
// 219): asked now, a rebuild of an old commit would be the newest ask of the module and roll
|
||||
// that commit out over everything since. Building that commit again is `replay`, which says
|
||||
// what it would do and refuses to register it while anything newer is asked or registered.
|
||||
if e, known := entryNamed(entries, module); known && ref != "" && followedBranch(ref) == "" {
|
||||
ref = followedBranch(e.Source.Ref)
|
||||
follows := ref
|
||||
if follows == "" {
|
||||
follows = "the repository's default branch"
|
||||
}
|
||||
fmt.Printf("%s was built at commit %s; a rebuild asks what %s follows now, %s — `replay %s` "+
|
||||
"builds that commit\n", b.ID, short(b.Ref), module, follows, b.ID)
|
||||
}
|
||||
} else if e, known := entryNamed(entries, args[0]); known {
|
||||
// As a plan asks it: the branch it follows, never a commit a build once named (issue 215).
|
||||
source = buildSource{Repository: e.Source.Repository, Seat: e.Source.Seat}
|
||||
path, ref, module = e.Source.Path, followedBranch(e.Source.Ref), e.Manifest.Module
|
||||
} else {
|
||||
return fmt.Errorf("%s is neither a module the catalogue holds nor a build the mesh recorded", args[0])
|
||||
}
|
||||
|
||||
// **A module a plan holds unbuilt, or failed, joins that plan** rather than running beside it: the
|
||||
// plan would ask it again, or stay failed on an outcome a rebuild has replaced. A walk waiting for its word
|
||||
// is not joined (novox/hq issue 754): the rebuild runs beside it, and says so.
|
||||
var beside string
|
||||
if module != "" {
|
||||
joined, said, err := joinAPlan(ctx, open, module)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if joined {
|
||||
fmt.Println(said)
|
||||
return nil
|
||||
}
|
||||
beside = said
|
||||
}
|
||||
id, err := askABuild(ctx, source, path, ref)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
recordAsked(ctx, inventory.BuildRequest{ID: id, Repository: source.Repository, Seat: source.Seat, Path: path,
|
||||
Ref: ref, For: "rebuild"})
|
||||
if beside != "" {
|
||||
fmt.Printf("rebuild asked as %s, beside the walk: %s\n", id, beside)
|
||||
return nil
|
||||
}
|
||||
fmt.Printf("rebuild asked as %s\n", id)
|
||||
return nil
|
||||
}
|
||||
|
||||
// entryNamed is the catalogue's entry for a module.
|
||||
func entryNamed(entries []inventory.Entry, name string) (inventory.Entry, bool) {
|
||||
for _, e := range entries {
|
||||
if e.Manifest.Module == name {
|
||||
return e, true
|
||||
}
|
||||
}
|
||||
return inventory.Entry{}, false
|
||||
}
|
||||
|
||||
// sourceOfBuild is where a recorded build's repository is asked from: the catalogued module's own
|
||||
// source when the build is of one — on its seat, so the outcome registers it as the mesh records it
|
||||
// (ADR 0111) — else the repository as it was cloned.
|
||||
@@ -576,14 +493,17 @@ func sourceOfBuild(b inventory.Build, entries []inventory.Entry) (buildSource, s
|
||||
// replayCommand asks a recorded build's repository and path again at the commit it built.
|
||||
func replayCommand(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("replay", flag.ContinueOnError)
|
||||
register := set.Bool("register", false, "register what it builds, as any build is")
|
||||
older := set.Bool("older", false, "with --register: even though a newer build of the module is registered")
|
||||
register := set.Bool("register", false, "removed (novox/hq ADR 0501 §5): refused by name")
|
||||
positionals, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// **Registering a replay goes with the asks** (novox/hq ADR 0429 §18, ADR 0501 §5): refused by name.
|
||||
if *register {
|
||||
return removedByName("replay --register")
|
||||
}
|
||||
if len(positionals) != 1 {
|
||||
return errors.New("replay <build id> [--register [--older]]")
|
||||
return errors.New("replay <build id>")
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
@@ -603,86 +523,15 @@ func replayCommand(ctx context.Context, args []string) error {
|
||||
return err
|
||||
}
|
||||
source, module := sourceOfBuild(b, entries)
|
||||
var history []inventory.Build
|
||||
if module != "" {
|
||||
if history, err = inv.Builds(ctx, module, 100); err != nil {
|
||||
return err
|
||||
}
|
||||
if b.Commit == "" {
|
||||
return fmt.Errorf("%s recorded no commit — it failed before it knew what it was building, so there is "+
|
||||
"nothing to replay", b.ID)
|
||||
}
|
||||
// What is asked of the module and not yet answered, when the replay would be registered.
|
||||
var outstanding []string
|
||||
if *register {
|
||||
js, err := dialTheBus()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
q, err := link.ReadQueue(ctx, js, buildSeatHeld(ctx))
|
||||
js.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
plans, err := inv.OpenPlans(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
outstanding = outstandingFor(module, b.Repository, b.Path, q, plans)
|
||||
}
|
||||
if err := replayRefusal(b, module, history, *register, *older, outstanding); err != nil {
|
||||
return err
|
||||
}
|
||||
asker := ""
|
||||
if *register {
|
||||
asker = "replay"
|
||||
}
|
||||
id, err := buildOneAsked(ctx, source, b.Path, b.Commit, 0, !*register, asker)
|
||||
id, err := buildOneAsked(ctx, source, b.Path, b.Commit, 0, true, "")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(replaySaid(b, module, id, *register))
|
||||
return nil
|
||||
}
|
||||
|
||||
// replayRefusal is why a replay is not asked, or nothing (novox/hq ADR 0219, issue 207).
|
||||
//
|
||||
// A replay re-asks a recorded build at the commit it built, under a new id — and an id is when it
|
||||
// was asked, which is what orders builds of one module (issue 219). So a registered replay of an
|
||||
// older commit is newer than everything since, and would roll that older commit out as the module's
|
||||
// current version: what issue 207 recorded happening by accident. It is therefore a dry run unless
|
||||
// --register says otherwise, and --register is refused when a newer build of a different commit is
|
||||
// registered, unless --older says that is the point.
|
||||
//
|
||||
// **And refused while anything newer is outstanding**, --older or not: an ask of the module in the
|
||||
// queue, or an open plan holding it unbuilt. Asked now, the replay would be newer than those asks,
|
||||
// and their builds — made from newer source — would be recorded and never registered (issue 219
|
||||
// orders by ask), the plan waiting on them sending the older commit instead.
|
||||
func replayRefusal(b inventory.Build, module string, history []inventory.Build, register, older bool,
|
||||
outstanding []string) error {
|
||||
if b.Commit == "" {
|
||||
return fmt.Errorf("%s recorded no commit — it failed before it knew what it was building, so there is "+
|
||||
"nothing to replay; `rebuild %s` asks its repository, path and ref again", b.ID, b.ID)
|
||||
}
|
||||
if older && !register {
|
||||
return errors.New("--older only says what --register may do; a dry run registers nothing")
|
||||
}
|
||||
if register && len(outstanding) > 0 {
|
||||
return fmt.Errorf("%s is asked and not yet answered — %s — and a registered replay asked now would "+
|
||||
"replace what those build (novox/hq issue 219). Wait for them, or `replay %s` without --register to look",
|
||||
orNone(module), strings.Join(outstanding, "; "), b.ID)
|
||||
}
|
||||
if !register || older {
|
||||
return nil
|
||||
}
|
||||
for _, h := range history {
|
||||
if h.ID == b.ID || !h.Worked() || h.Commit == b.Commit {
|
||||
continue
|
||||
}
|
||||
if h.AskedOrAt().After(b.AskedOrAt()) {
|
||||
return fmt.Errorf("a newer build of %s is registered — %s, from %s — and registering a replay of %s "+
|
||||
"would roll that older commit out as %s's current version (novox/hq issue 207). `replay %s` "+
|
||||
"without --register looks at it; --register --older registers it anyway",
|
||||
module, h.ID, short(h.Commit), short(b.Commit), module, b.ID)
|
||||
}
|
||||
}
|
||||
fmt.Println(replaySaid(b, module, id, false))
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -822,32 +671,3 @@ func holdersAmong(entries []inventory.Entry, seat string) []string {
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// outstandingFor is everything asked of a module and not yet answered: its asks in the build queue,
|
||||
// by repository and path, and every open plan holding it not yet built.
|
||||
func outstandingFor(module, repository, path string, q link.Queue, plans []inventory.Plan) []string {
|
||||
var out []string
|
||||
for _, a := range q.Asks {
|
||||
if repositoryMatches(a.Repository, repository) && a.Path == path {
|
||||
out = append(out, fmt.Sprintf("%s %s in the queue", a.ID, a.State))
|
||||
}
|
||||
}
|
||||
if module == "" {
|
||||
return out
|
||||
}
|
||||
for _, p := range plans {
|
||||
if !p.Open() {
|
||||
continue
|
||||
}
|
||||
for _, tier := range p.Tiers {
|
||||
for _, m := range tier {
|
||||
if m == module {
|
||||
if st := p.Modules[m]; st == nil || st.State != "built" {
|
||||
out = append(out, fmt.Sprintf("%s holds it not yet built", p.ID))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -71,44 +71,6 @@ func TestAFailedPlanIsRetriedAndGoesOnThroughItsLaterTiers(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestARebuildJoinsThePlanHoldingTheModule(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
asked := asksRecorded(t)
|
||||
twoTiers(t, open)
|
||||
before := time.Now().UTC().Add(-time.Hour)
|
||||
failed := inventory.Plan{ID: "plan-join", Repository: "novox/a", Branch: "main", Commit: "c0ffee",
|
||||
Created: before, State: inventory.PlanFailed, Tiers: [][]string{{"a"}, {"b"}},
|
||||
Note: "a failed to build in tier 0",
|
||||
Modules: map[string]*inventory.PlanModule{"a": {State: "failed", AskedAt: &before, Build: "build-1", Why: link.CancelledByHand}}}
|
||||
if err := open.inventory.SavePlan(ctx, &failed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := rebuildCommand(ctx, []string{"a"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p, err := open.inventory.PlanByID(ctx, failed.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if p.State != inventory.PlanBuilding || p.Modules["a"].State != "asked" || p.Modules["a"].Build == "build-1" {
|
||||
t.Fatalf("the rebuild did not join the failed plan: %s %+v", p.State, p.Modules["a"])
|
||||
}
|
||||
if len(*asked) != 1 {
|
||||
t.Fatalf("asked %v", *asked)
|
||||
}
|
||||
// b is in a tier not yet reached: nothing holds it in its current tier, so it is asked alone.
|
||||
if err := rebuildCommand(ctx, []string{"b"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if p, _ = open.inventory.PlanByID(ctx, failed.ID); p.Modules["b"] != nil {
|
||||
t.Fatalf("a module the plan has not reached joined it: %+v", p.Modules["b"])
|
||||
}
|
||||
if len(*asked) != 2 {
|
||||
t.Fatalf("asked %v", *asked)
|
||||
}
|
||||
}
|
||||
|
||||
func aBuildQueue(t *testing.T) (*broker.JetStream, func(id, repository string) link.BuildRequest) {
|
||||
t.Helper()
|
||||
url := testbus.URL(t)
|
||||
@@ -425,29 +387,6 @@ func TestAPlanIsAnsweredOnlyByTheBuildItAskedFor(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestARebuildOfACommitAsksWhatTheModuleFollows(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
asked := asksRecorded(t)
|
||||
twoTiers(t, open)
|
||||
var refs []string
|
||||
was := askABuild
|
||||
askABuild = func(c context.Context, source buildSource, path, ref string) (string, error) {
|
||||
refs = append(refs, ref)
|
||||
return was(c, source, path, ref)
|
||||
}
|
||||
if err := open.inventory.RecordBuild(ctx, inventory.Build{ID: "build-at-commit", Repository: "novox/a",
|
||||
Ref: "0123456789abcdef0123456789abcdef01234567", Module: "a", Commit: "0123456789abcdef0123456789abcdef01234567"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := rebuildCommand(ctx, []string{"build-at-commit"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(refs) != 1 || refs[0] != "main" || len(*asked) != 1 {
|
||||
t.Fatalf("asked %v at %v", *asked, refs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCancelOfAnAskNobodySaidIsWithdrawnWhenItStarts(t *testing.T) {
|
||||
js, ask := aBuildQueue(t)
|
||||
open := aMesh(t)
|
||||
|
||||
@@ -24,6 +24,7 @@ import (
|
||||
// asksRecorded makes every ask a plan makes return the next id in a row, and says which were asked.
|
||||
func asksRecorded(t *testing.T) *[]string {
|
||||
t.Helper()
|
||||
asksOfEveryModule(t) // the one ask left is the builder's own (ADR 0453 §5): these tests are of what a walk does around it
|
||||
var asked []string
|
||||
was := askABuild
|
||||
askABuild = func(_ context.Context, source buildSource, path, ref string) (string, error) {
|
||||
@@ -100,89 +101,6 @@ func TestRetryRefusesWhatItCannotResume(t *testing.T) {
|
||||
|
||||
// `rebuild` of a module a failed plan holds joins that plan; one held by nothing runs alone.
|
||||
|
||||
// A failed plan an open plan of its repository supersedes is not joined: the open one holds the module.
|
||||
func TestARebuildJoinsTheOpenPlanBeforeASupersededFailedOne(t *testing.T) {
|
||||
at := time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC)
|
||||
failed := inventory.Plan{ID: "plan-old", Repository: "novox/a", Branch: "main", Created: at, State: inventory.PlanFailed,
|
||||
Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{"a": {State: "failed"}}}
|
||||
newer := inventory.Plan{ID: "plan-new", Repository: "novox/a", Branch: "main", Created: at.Add(time.Hour),
|
||||
State: inventory.PlanBuilding, Tiers: [][]string{{"x"}, {"a"}}, Modules: map[string]*inventory.PlanModule{}}
|
||||
if _, found := planHolding("a", []inventory.Plan{newer}, []inventory.Plan{newer, failed}); found {
|
||||
t.Fatal("joined a failed plan a newer open one supersedes")
|
||||
}
|
||||
if p, found := planHolding("a", nil, []inventory.Plan{failed}); !found || p.ID != "plan-old" {
|
||||
t.Fatalf("did not join the failed plan holding it: %v %s", found, p.ID)
|
||||
}
|
||||
built := failed
|
||||
built.Modules = map[string]*inventory.PlanModule{"a": {State: "built"}}
|
||||
if _, found := planHolding("a", nil, []inventory.Plan{built}); found {
|
||||
t.Fatal("joined a plan that has the module built")
|
||||
}
|
||||
}
|
||||
|
||||
// replay is a dry run unless registered, and registering an older commit than one registered since
|
||||
// is refused unless --older says it is meant (novox/hq issue 207).
|
||||
func TestReplayRefusesToRollAnOlderCommitOutUnlessToldTo(t *testing.T) {
|
||||
asked := time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC)
|
||||
old := inventory.Build{ID: "build-old", Module: "a", Commit: "0ldc0mm1t", Asked: asked}
|
||||
newer := inventory.Build{ID: "build-new", Module: "a", Commit: "n3wc0mm1t", Asked: asked.Add(time.Hour)}
|
||||
history := []inventory.Build{newer, old}
|
||||
|
||||
if err := replayRefusal(old, "a", history, false, false, nil); err != nil {
|
||||
t.Errorf("a dry run was refused: %v", err)
|
||||
}
|
||||
err := replayRefusal(old, "a", history, true, false, nil)
|
||||
if err == nil || !strings.Contains(err.Error(), "build-new") || !strings.Contains(err.Error(), "--older") {
|
||||
t.Errorf("registering an older commit than the one registered was not refused: %v", err)
|
||||
}
|
||||
if err := replayRefusal(old, "a", history, true, true, nil); err != nil {
|
||||
t.Errorf("--register --older was refused: %v", err)
|
||||
}
|
||||
if err := replayRefusal(newer, "a", history, true, false, nil); err != nil {
|
||||
t.Errorf("registering the newest build again was refused: %v", err)
|
||||
}
|
||||
// A newer build of the same commit, or one that failed, is not a newer version to roll back from.
|
||||
same := inventory.Build{ID: "build-same", Module: "a", Commit: old.Commit, Asked: asked.Add(2 * time.Hour)}
|
||||
broken := inventory.Build{ID: "build-broken", Module: "a", Failed: "no", Asked: asked.Add(3 * time.Hour)}
|
||||
if err := replayRefusal(old, "a", []inventory.Build{broken, same, old}, true, false, nil); err != nil {
|
||||
t.Errorf("refused for a newer build of the same commit or a failed one: %v", err)
|
||||
}
|
||||
if err := replayRefusal(inventory.Build{ID: "build-x", Failed: "clone"}, "", nil, false, false, nil); err == nil {
|
||||
t.Error("a build that recorded no commit was replayed")
|
||||
}
|
||||
if err := replayRefusal(old, "a", history, false, true, nil); err == nil {
|
||||
t.Error("--older without --register was taken")
|
||||
}
|
||||
// **Nothing newer outstanding**, --older or not: an ask of the module in the queue, or an open plan
|
||||
// holding it unbuilt, would be replaced by a replay asked now (issue 219).
|
||||
q := link.Queue{Asks: []link.QueuedAsk{
|
||||
{ID: "build-queued", Repository: "https://forge.example/novox/a.git", State: link.AskWaiting},
|
||||
{ID: "build-elsewhere", Repository: "https://forge.example/novox/b.git", State: link.AskWaiting},
|
||||
{ID: "build-other-path", Repository: "https://forge.example/novox/a.git", Path: "sub", State: link.AskWaiting},
|
||||
}}
|
||||
plans := []inventory.Plan{
|
||||
{ID: "plan-holds", State: inventory.PlanBuilding, Tiers: [][]string{{"x"}, {"a"}}, Modules: map[string]*inventory.PlanModule{}},
|
||||
{ID: "plan-built", State: inventory.PlanRolling, Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{"a": {State: "built"}}},
|
||||
{ID: "plan-failed", State: inventory.PlanFailed, Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{}},
|
||||
}
|
||||
outstanding := outstandingFor("a", "novox/a", "", q, plans)
|
||||
if len(outstanding) != 2 || !strings.Contains(outstanding[0], "build-queued") || !strings.Contains(outstanding[1], "plan-holds") {
|
||||
t.Fatalf("outstanding: %v", outstanding)
|
||||
}
|
||||
if err := replayRefusal(old, "a", history, true, true, outstanding); err == nil || !strings.Contains(err.Error(), "build-queued") {
|
||||
t.Errorf("registered over an outstanding ask: %v", err)
|
||||
}
|
||||
if err := replayRefusal(old, "a", history, false, false, outstanding); err != nil {
|
||||
t.Errorf("a dry run was refused for what is outstanding: %v", err)
|
||||
}
|
||||
if said := replaySaid(old, "a", "build-1", false); !strings.Contains(said, "dry run") || !strings.Contains(said, "--register") {
|
||||
t.Errorf("a dry replay does not say what it is: %q", said)
|
||||
}
|
||||
if said := replaySaid(old, "a", "build-1", true); !strings.Contains(said, "registered") || !strings.Contains(said, "rolled out") {
|
||||
t.Errorf("a registered replay does not say what it does: %q", said)
|
||||
}
|
||||
}
|
||||
|
||||
// A plan waiting on builds of a seat whose every holder is paused says so and is not late; a seat
|
||||
// paused on some holders only is not a reason the plan is waiting.
|
||||
func TestAPlanWaitingOnAPausedSeatSaysSoAndIsNotLate(t *testing.T) {
|
||||
@@ -240,9 +158,7 @@ func TestTheQueueVerbsRunTheirCommands(t *testing.T) {
|
||||
{"cancel", map[string]any{"id": "build-1"}, []string{"cancel", "build-1"}},
|
||||
{"clear", nil, []string{"clear"}},
|
||||
{"clear", map[string]any{"dead": "true"}, []string{"clear", "--dead"}},
|
||||
{"rebuild", map[string]any{"what": "gitea"}, []string{"rebuild", "gitea"}},
|
||||
{"replay", map[string]any{"id": "build-1"}, []string{"replay", "build-1"}},
|
||||
{"replay", map[string]any{"id": "build-1", "register": "true", "older": "true"}, []string{"replay", "build-1", "--register", "--older"}},
|
||||
{"kill", map[string]any{"id": "build-1"}, []string{"kill", "build-1"}},
|
||||
{"pause", nil, []string{"pause"}},
|
||||
{"resume", map[string]any{"node": "ace"}, []string{"resume", "ace"}},
|
||||
|
||||
@@ -53,6 +53,11 @@ type meshStatus struct {
|
||||
// Per machine, and data. One node failing must never take the document away from a reader
|
||||
// asking about the others.
|
||||
Unresolved []machineUnresolved `json:"unresolved"`
|
||||
// NotAnnounced are the modules others read the state of that have not said what they keep (novox/hq ADR 0495,
|
||||
// issue 1790): their readers are granted from `reads` as before until they do. NotAnnouncedWhy is why it could not
|
||||
// be read.
|
||||
NotAnnounced []string `json:"not-announced,omitempty"`
|
||||
NotAnnouncedWhy string `json:"not-announced-why,omitempty"`
|
||||
// Network is why the private network could not be computed, when it could not; absent when it
|
||||
// could. Almost always a consequence of Unresolved: a node that does not resolve is not on the
|
||||
// network, and a mesh whose hub is that node has no hub.
|
||||
@@ -247,7 +252,8 @@ func statusAsJSON(asked answers) ([]byte, error) {
|
||||
out := meshStatus{Machines: len(nodes), Wrong: []machineDoing{},
|
||||
Quiet: []machineQuiet{}, Behind: []moduleBehind{}, Waiting: []machineWaiting{},
|
||||
Reported: []machineReported{}, Unresolved: []machineUnresolved{},
|
||||
Network: asked.network, Adopted: adoptedNodes(nodes), Plans: planStatuses(asked.plans, time.Now(), asked.paused, asked.tierBounds)}
|
||||
Network: asked.network, Adopted: adoptedNodes(nodes), Plans: planStatuses(asked.plans, time.Now(), asked.paused, asked.tierBounds),
|
||||
NotAnnounced: asked.notAnnounced, NotAnnouncedWhy: asked.notAnnouncedWhy}
|
||||
// In a stated order, so two readings of an unchanged mesh are the same document.
|
||||
untakenNodes := make([]string, 0, len(asked.untaken))
|
||||
for name := range asked.untaken {
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"github.com/novox/mesh-controller/internal/builderkeys"
|
||||
)
|
||||
|
||||
// The terminal's exception becomes a registration, not a build (novox/hq ADR 0501 §4; issue 1851, A11b).
|
||||
//
|
||||
// `register <module> <owner>/<repository> <path>`, at the controller's terminal alone: the operator names the module,
|
||||
// its repository and its path, and the controller registers it from the builder's done key of the fingerprint the
|
||||
// indexer's trunk value of that module names — the registration a build asked at the terminal used to make for a
|
||||
// repository the catalogue does not build (ADR 0432 §3, ADR 0420 §7). No verb reaches it (terminalOnlyCommands), and it
|
||||
// asks no build: a key not done is waited for, and said.
|
||||
func registerCommand(ctx context.Context, args []string) error {
|
||||
if len(args) != 3 {
|
||||
return errors.New("register <module> <owner>/<repository> <path> (the path \".\" for a module at the repository's root)")
|
||||
}
|
||||
if !startedAtTheTerminal() {
|
||||
return terminalRefusal("register is run at the controller's terminal only, never through a verb: it registers " +
|
||||
"a module from a repository the catalogue does not build, and whoever may call a verb includes agents " +
|
||||
"(novox/hq ADR 0266, ADR 0501 §4). Nothing was done")
|
||||
}
|
||||
module, repository, path := args[0], strings.Trim(args[1], "/"), strings.Trim(args[2], "/")
|
||||
if path == sdkdelivery.RootPath {
|
||||
path = ""
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
keys, closeKeys, err := openBuilderKeys()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer closeKeys()
|
||||
trunk, err := keys.ModuleAt(ctx, module, "")
|
||||
if errors.Is(err, builderkeys.ErrNoValue) {
|
||||
return fmt.Errorf("the indexer holds no trunk value of %s: a module is registered from the builder's key of a "+
|
||||
"merged trunk, and %s has none yet. Nothing was registered", module, module)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := sameModuleSource(module, repository, path, trunk); err != nil {
|
||||
return err
|
||||
}
|
||||
manifest, kept, err := takeInKey(ctx, open.inventory, keys, keyTake{Module: module, Trunk: trunk,
|
||||
Ask: registrationAsk{Terminal: true}})
|
||||
if errors.Is(err, errKeyWaits) {
|
||||
return fmt.Errorf("%s is not registered yet, and nothing was asked: %v. Run register again once its key is done",
|
||||
module, err)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%s %s registered from the builder's key %s (%s at %.8s), as asked at the controller's terminal; "+
|
||||
"assigned nowhere\n", manifest.Module, manifest.Version, short(kept.SourceFingerprint), repository, kept.Commit)
|
||||
return nil
|
||||
}
|
||||
|
||||
// sameModuleSource refuses a registration whose repository or path is not the one the indexer's trunk value names: the
|
||||
// operator names exactly what is registered (ADR 0501 §4).
|
||||
func sameModuleSource(module, repository, path string, trunk sdkdelivery.Module) error {
|
||||
if !strings.EqualFold(trunk.Repository, repository) || keyPath(trunk) != path {
|
||||
return fmt.Errorf("the indexer holds %s in %s at %q, not in %s at %q: the registration names exactly what it "+
|
||||
"registers. Nothing was registered", module, trunk.Repository, orRoot(keyPath(trunk)), repository, orRoot(path))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// A11b (ADR 0501 §4): the terminal's registration names exactly the module's repository and path the indexer holds,
|
||||
// and no verb reaches it.
|
||||
func TestTheTerminalsRegistrationNamesItsSourceExactlyAndNoVerbReachesIt(t *testing.T) {
|
||||
trunk := sdkdelivery.Module{Repository: "someone/app", Path: sdkdelivery.RootPath, Commit: "c0ffee"}
|
||||
if err := sameModuleSource("app", "someone/app", "", trunk); err != nil {
|
||||
t.Fatalf("the source the indexer holds is refused: %v", err)
|
||||
}
|
||||
if err := sameModuleSource("app", "someone/other", "", trunk); err == nil || !strings.Contains(err.Error(), "someone/app") {
|
||||
t.Fatalf("another repository is taken: %v", err)
|
||||
}
|
||||
if err := sameModuleSource("app", "someone/app", "modules/app", trunk); err == nil {
|
||||
t.Fatal("another path is taken")
|
||||
}
|
||||
// The generic verb only reads; and register is the terminal's only, whatever reads.
|
||||
if _, err := argvFor("command", map[string]any{"command": "register app someone/app ."}); err == nil {
|
||||
t.Fatal("a verb reaches the terminal's registration")
|
||||
}
|
||||
var refused *heldAtTheTerminal
|
||||
if err := terminalOnly([]string{"register", "app", "someone/app", "."}); !errors.As(err, &refused) {
|
||||
t.Fatalf("register is not the terminal's only: %v", err)
|
||||
}
|
||||
t.Setenv(verbVar, "command")
|
||||
var held *heldAtTheTerminal
|
||||
if err := registerCommand(context.Background(), []string{"app", "someone/app", "."}); !errors.As(err, &held) {
|
||||
t.Fatalf("register through a verb: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"os"
|
||||
"slices"
|
||||
"sort"
|
||||
@@ -390,6 +391,11 @@ func takesOutcome(p inventory.Plan, state *inventory.PlanModule, asked time.Time
|
||||
if p.Waiting() || state == nil || (state.Build == "" && state.AskedAt == nil) {
|
||||
return false
|
||||
}
|
||||
// **A module taken from the builder's key is answered by its key alone** (ADR 0501 §1): a build-agent outcome of it
|
||||
// is somebody else's build.
|
||||
if state.Key {
|
||||
return false
|
||||
}
|
||||
if state.Build != "" {
|
||||
return state.Build == id
|
||||
}
|
||||
@@ -405,40 +411,66 @@ var askABuild func(ctx context.Context, source buildSource, path, ref string) (s
|
||||
|
||||
func init() {
|
||||
askABuild = func(ctx context.Context, source buildSource, path, ref string) (string, error) {
|
||||
// **Only the builder's own module is asked of build-agent** (novox/hq ADR 0453 §5, ADR 0498 §7; issue 1851,
|
||||
// A12): refused here, in the one function every ask goes through, whoever calls it.
|
||||
if err := onlyTheBuildersOwn(ctx, source, path); err != nil {
|
||||
return "", err
|
||||
}
|
||||
// Kept by each asker with its own name once the ask is made (novox/hq issue 325).
|
||||
return buildOneAsked(ctx, source, path, ref, 0, false, "")
|
||||
}
|
||||
}
|
||||
|
||||
// reuseKey carries a walk's leave to take a check's build from askModule to the ask it makes (novox/hq ADR 0350 §2).
|
||||
type reuseKey struct{}
|
||||
// errNotTheBuildersOwn says an ask of build-agent was refused: from R4e build-agent builds the builder's own module
|
||||
// alone, and every other is taken from the builder's done key.
|
||||
var errNotTheBuildersOwn = errors.New("build-agent is asked for the builder's own module alone (ADR 0453 §5)")
|
||||
|
||||
// withReuse is ctx carrying a walk's leave to take a check's build.
|
||||
func withReuse(ctx context.Context, r *link.ReuseAsk) context.Context {
|
||||
return context.WithValue(ctx, reuseKey{}, r)
|
||||
// askedModuleOf is the module the catalogue builds from a source and path, empty when it builds none from it. A variable
|
||||
// so a test needs no store.
|
||||
var askedModuleOf = func(ctx context.Context, source buildSource, path string) (string, error) {
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer open.Close()
|
||||
entries, err := open.inventory.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
forms := newSourceForms(ctx, open.inventory)
|
||||
for _, e := range entries {
|
||||
if forms.same(e.Source.Repository, e.Source.Seat, source.Repository, source.Seat) &&
|
||||
strings.Trim(e.Source.Path, "/") == strings.Trim(path, "/") {
|
||||
return e.Manifest.Module, nil
|
||||
}
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
|
||||
// reuseIn is the leave ctx carries; nil for every ask but a walk's.
|
||||
func reuseIn(ctx context.Context) *link.ReuseAsk {
|
||||
r, _ := ctx.Value(reuseKey{}).(*link.ReuseAsk)
|
||||
return r
|
||||
}
|
||||
|
||||
// reuseAsked is the leave an ask carries: a walk's, and never a dry run's — a dry run registers nothing, so it
|
||||
// takes nothing either (novox/hq ADR 0350 §2, §6).
|
||||
func reuseAsked(ctx context.Context, dryRun bool) *link.ReuseAsk {
|
||||
if dryRun {
|
||||
// onlyTheBuildersOwn refuses an ask of any module but the builder's own, naming the key it is taken from instead.
|
||||
func onlyTheBuildersOwn(ctx context.Context, source buildSource, path string) error {
|
||||
module, err := askedModuleOf(ctx, source, path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: which module %s at %q is cannot be read, so nothing is asked: %v", errNotTheBuildersOwn,
|
||||
source, orRoot(path), err)
|
||||
}
|
||||
if isBuildersOwn(module) {
|
||||
return nil
|
||||
}
|
||||
return reuseIn(ctx)
|
||||
if module == "" {
|
||||
module = fmt.Sprintf("%s at %q", source, orRoot(path))
|
||||
}
|
||||
return fmt.Errorf("%w: %s is taken from the builder's done key of its fingerprint in %s (ADR 0501), and no build "+
|
||||
"of it is asked", errNotTheBuildersOwn, module, sdkdelivery.BuilderBuildsBucket)
|
||||
}
|
||||
|
||||
// reuseAskOf is a walk's leave to take a check's build: the pull requests of the merges it answers.
|
||||
func reuseAskOf(p *inventory.Plan) *link.ReuseAsk {
|
||||
r := &link.ReuseAsk{}
|
||||
// pullsOfWalk are the pull requests of the merges a walk answers, each once: what its end lets go of (forgetChecksOfWalk)
|
||||
// and what a check of one was (same_program.go).
|
||||
func pullsOfWalk(p *inventory.Plan) []link.PullRef {
|
||||
if p.Delivery == nil {
|
||||
return r
|
||||
return nil
|
||||
}
|
||||
var out []link.PullRef
|
||||
seen := map[link.PullRef]bool{}
|
||||
for _, m := range p.Delivery.Merges {
|
||||
pull, ok := pullOf(m)
|
||||
@@ -447,10 +479,10 @@ func reuseAskOf(p *inventory.Plan) *link.ReuseAsk {
|
||||
}
|
||||
if !seen[pull] {
|
||||
seen[pull] = true
|
||||
r.Pulls = append(r.Pulls, pull)
|
||||
out = append(out, pull)
|
||||
}
|
||||
}
|
||||
return r
|
||||
return out
|
||||
}
|
||||
|
||||
// checkedBuildsOf are the builds the build seat's checks of those pull requests said they made for their merges,
|
||||
@@ -468,76 +500,6 @@ var checkedBuildsOf = func(ctx context.Context, pulls []link.PullRef) ([]link.Ch
|
||||
return link.ReadCheckBuilds(ctx, js, buildSeatHeld(ctx), pulls, time.Now().Add(-builder.CheckBuildsKept))
|
||||
}
|
||||
|
||||
// mergedHeadsOf are the head commits the walk's merges merged, by pull request (novox/hq issue 703): a pull request
|
||||
// merged twice in one walk, or whose head the announcement did not say, has none, and takes no check's build.
|
||||
func mergedHeadsOf(p *inventory.Plan) map[link.PullRef]string {
|
||||
heads := map[link.PullRef]string{}
|
||||
if p.Delivery == nil {
|
||||
return heads
|
||||
}
|
||||
seen := map[link.PullRef]bool{}
|
||||
for _, m := range p.Delivery.Merges {
|
||||
pull, ok := pullOf(m)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if seen[pull] && heads[pull] != m.Head {
|
||||
heads[pull] = ""
|
||||
continue
|
||||
}
|
||||
seen[pull] = true
|
||||
heads[pull] = m.Head
|
||||
}
|
||||
return heads
|
||||
}
|
||||
|
||||
// walkVerdicts are the checks' builds a walk read, by the walk's plan: read once per walk (novox/hq issue 649), not
|
||||
// once per module it asks for. Kept a day; a walk lasts minutes.
|
||||
var walkVerdicts = struct {
|
||||
sync.Mutex
|
||||
read map[string]verdictsRead
|
||||
}{read: map[string]verdictsRead{}}
|
||||
|
||||
type verdictsRead struct {
|
||||
at time.Time
|
||||
builds []link.CheckedBuild
|
||||
}
|
||||
|
||||
// reuseWithVerdicts is a walk's leave with the builds its pull requests' verdicts name: the only builds the builder
|
||||
// may take (ADR 0350 §2), and **only those of a check of the head each merge merged** (novox/hq issue 703): a build
|
||||
// a check made at an earlier head of the same pull request is never registered. Read once per walk. When they
|
||||
// cannot be read the leave names none, and the walk builds, saying why.
|
||||
func reuseWithVerdicts(ctx context.Context, walk string, r *link.ReuseAsk, heads map[link.PullRef]string) *link.ReuseAsk {
|
||||
walkVerdicts.Lock()
|
||||
read, known := walkVerdicts.read[walk]
|
||||
walkVerdicts.Unlock()
|
||||
if !known || walk == "" {
|
||||
builds, err := checkedBuildsOf(ctx, r.Pulls)
|
||||
if err != nil {
|
||||
fmt.Printf(" the checks' builds of %d pull request(s) could not be read, so the walk builds: %v\n", len(r.Pulls), err)
|
||||
return r
|
||||
}
|
||||
read = verdictsRead{at: time.Now(), builds: builds}
|
||||
if walk != "" {
|
||||
walkVerdicts.Lock()
|
||||
for id, w := range walkVerdicts.read {
|
||||
if time.Since(w.at) > 24*time.Hour {
|
||||
delete(walkVerdicts.read, id)
|
||||
}
|
||||
}
|
||||
walkVerdicts.read[walk] = read
|
||||
walkVerdicts.Unlock()
|
||||
}
|
||||
}
|
||||
r.Builds = nil
|
||||
for _, b := range read.builds {
|
||||
if link.SameHead(b.Head, heads[b.Pull]) {
|
||||
r.Builds = append(r.Builds, b)
|
||||
}
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// pullOf is a merge's pull request: owner and repository, the last two parts of however the repository is spelled
|
||||
// (owner/repo, a path on the git seat, a URL), and its number. False when the merge names none.
|
||||
func pullOf(m inventory.PlanMerge) (link.PullRef, bool) {
|
||||
@@ -565,6 +527,13 @@ func askModule(ctx context.Context, p *inventory.Plan, name string, byName map[s
|
||||
p.Note = name + " is no longer in the catalogue"
|
||||
return
|
||||
}
|
||||
// **Every module but the builder's own is taken from the builder's done key, asking no build** (novox/hq ADR 0503
|
||||
// §6, ADR 0501, ADR 0453 §5): the look settles it (settleFromKeys).
|
||||
if !isBuildersOwn(name) {
|
||||
sayBuilding(ctx, p.ID, now)
|
||||
takeFromKey(p, state, e, now)
|
||||
return
|
||||
}
|
||||
source := buildSource{Repository: e.Source.Repository, Seat: e.Source.Seat}
|
||||
fmt.Printf(" tier %d: ", p.Tier)
|
||||
// The branch it follows, never a commit a build once named (novox/hq 04-ISSUES/215): the branch contains every
|
||||
@@ -575,13 +544,10 @@ func askModule(ctx context.Context, p *inventory.Plan, name string, byName map[s
|
||||
if p.Delivery != nil && p.Delivery.Alone && carried != "" {
|
||||
ref = carried
|
||||
}
|
||||
// **The walk takes the check's build of the same key** (novox/hq ADR 0350 §2): its ask says so, with the pull
|
||||
// requests it answers, whose checks name the part of a differing key.
|
||||
// **The plan's key is there before its first build is** (ADR 0483 clause 2): a check taken between the ask and the plan's save
|
||||
// would hold a place for its whole run.
|
||||
sayBuilding(ctx, p.ID, now)
|
||||
id, err := askABuild(withReuse(ctx, reuseWithVerdicts(ctx, p.ID, reuseAskOf(p), mergedHeadsOf(p))), source,
|
||||
e.Source.Path, ref)
|
||||
id, err := askABuild(ctx, source, e.Source.Path, ref)
|
||||
if err != nil {
|
||||
// **A bus that does not answer for a moment is not a failed build** (issue 1334): the module stays unasked and the
|
||||
// tier's next look asks it again. A fault of the module's own fails the plan at once, as does a bus silent too long.
|
||||
@@ -1071,6 +1037,25 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
||||
// the controller in its first tier: the build that produced the new one is recorded, and the
|
||||
// plan never hears it. The record is the fact; a build recorded after the ask is that tier's
|
||||
// outcome, whoever was listening.
|
||||
// **Taken from the builder's keys first** (ADR 0503 §6, issue 1851): every module of the tier but the builder's own.
|
||||
if keyed(p, tier) {
|
||||
keys, closeKeys, err := openPlanKeys()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
was := p.State
|
||||
changed, err := settleFromKeys(ctx, open, keys, p, tier)
|
||||
closeKeys()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if p.State == inventory.PlanFailed && was != inventory.PlanFailed {
|
||||
sayUnsent(p, rollsOut)
|
||||
}
|
||||
if changed {
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
recorded, byID, err := recordsOfAsked(ctx, inv, p, tier)
|
||||
if err != nil {
|
||||
return false, err
|
||||
@@ -2744,7 +2729,7 @@ func settleFromRecords(p *inventory.Plan, tier []string, recorded map[string][]i
|
||||
changed := false
|
||||
for _, m := range tier {
|
||||
s := p.Modules[m]
|
||||
if s == nil || s.State != "asked" || s.AskedAt == nil {
|
||||
if s == nil || s.State != "asked" || s.AskedAt == nil || s.Key {
|
||||
continue
|
||||
}
|
||||
var outcome *inventory.Build
|
||||
@@ -2845,3 +2830,13 @@ func stopByHand(p *inventory.Plan, how, by, why string) {
|
||||
}
|
||||
p.Delivery.Stopped, p.Delivery.StoppedWhy = by, why
|
||||
}
|
||||
|
||||
// keyed says a module of the tier still waits on the builder's key.
|
||||
func keyed(p *inventory.Plan, tier []string) bool {
|
||||
for _, m := range tier {
|
||||
if s := p.Modules[m]; s != nil && s.Key && s.State == "asked" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -27,13 +27,8 @@ func TestReplay325(t *testing.T) {
|
||||
BuiltFrom: "c0", Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
asksWithPaths(t)
|
||||
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "5e450125aa",
|
||||
Paths: []string{"modules/sensors/module.json", "modules/sensors/cmd/sensors/main.go"},
|
||||
ModuleDirs: []string{"modules/sensors"}, ModuleDirsSaid: true}
|
||||
if err := (following{open: open}).SourceMoved(ctx, m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The build the merge asked before R4e (nothing asks one since, ADR 0501 §6), kept as it was.
|
||||
mergeAdding(t, open, "modules/sensors", "5e450125aa")
|
||||
|
||||
said, err := assign(ctx, open, "laptop", "sensors")
|
||||
if err != nil {
|
||||
|
||||
@@ -36,21 +36,14 @@ import (
|
||||
// It runs the whole path: the forge announces each pull request (following.PullUpdated), the controller asks
|
||||
// mesh-delivery which pull requests the group holds, the ask is queued for the build seat, and the build seat's check
|
||||
// (builder.Check, with a container runtime that runs each container's command on this machine) judges it.
|
||||
func TestReplay867(t *testing.T) { replay867(t, false) }
|
||||
|
||||
// TestReplay867Recheck is the same pair re-checked through mesh-delivery (`delivery check` with one member): the forge's
|
||||
// announcement is not repeated, so the pull request reaches the controller without its head branch name.
|
||||
func TestReplay867Recheck(t *testing.T) { replay867(t, true) }
|
||||
|
||||
func replay867(t *testing.T, recheck bool) {
|
||||
if _, err := exec.LookPath("go"); err != nil {
|
||||
t.Skip("no Go toolchain on this machine to build a judge with")
|
||||
}
|
||||
// The check's floor of free space (novox/hq issue 1834) reads the disk the test runs on, which is none of this
|
||||
// replay's business: a build node under the floor failed every mesh-controller check here (issue 1834, note 12).
|
||||
wasFree := builder.FreeBytes
|
||||
builder.FreeBytes = func(string) (uint64, error) { return 500 << 30, nil }
|
||||
t.Cleanup(func() { builder.FreeBytes = wasFree })
|
||||
r := theGroupOfIssue867(t)
|
||||
for _, c := range []struct {
|
||||
what, repo string
|
||||
@@ -330,44 +323,3 @@ exit 1
|
||||
t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH"))
|
||||
t.Setenv("GOTOOLCHAIN", "local")
|
||||
}
|
||||
|
||||
// A recheck runs as a command of its own (`delivery check` with one member), where the controller is not the serving
|
||||
// one and holds no connection: it dials the bus for the moment and asks mesh-delivery's `groups` as the daemon does.
|
||||
// This drives the verb on a test bus with a fake mesh-delivery answering `groups`, with no stub of the seam between.
|
||||
func TestTheRecheckVerbAsksMeshDeliveryOnABusOfItsOwn(t *testing.T) {
|
||||
realAsk := askDeliveryOwner
|
||||
r := theGroupOfIssue867(t)
|
||||
askDeliveryOwner = realAsk
|
||||
js, _ := aBuildQueue(t)
|
||||
if servingBus.Load() != nil {
|
||||
t.Fatal("this test is of a process that is not the serving controller")
|
||||
}
|
||||
asked := 0
|
||||
wrapped, _ := json.Marshal(map[string]any{"content": []map[string]any{{"type": "text", "text": r.groups}}})
|
||||
sub, err := js.Conn().Subscribe(link.SeatToolSubject(catalogue.DeliverySeat, "groups"), func(m *nats.Msg) {
|
||||
asked++
|
||||
body, _ := json.Marshal(link.Answer{Result: wrapped})
|
||||
_ = m.Respond(body)
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
||||
member := `[{"id":"novox/mesh-host@` + r.hostHead[:12] + `","repository":"novox/mesh-host","base":"main","head":"` +
|
||||
r.hostHead + `","number":114,"paths":["notes.txt"],"module_dirs_said":true}]`
|
||||
if err := deliveryCommand(t.Context(), []string{"check", "--members", member}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
q, err := link.ReadQueue(t.Context(), js, link.TheBuildMachine)
|
||||
if err != nil || len(q.Asks) == 0 {
|
||||
t.Fatalf("nothing was asked of the build seat: %v", err)
|
||||
}
|
||||
c := q.Asks[len(q.Asks)-1].Request.Check
|
||||
if c.SiblingsUnknown != "" || len(c.Siblings) != 1 || c.Siblings[0].Repository != "novox/mesh-controller" ||
|
||||
c.Siblings[0].Number != 346 || c.Siblings[0].Head != r.controllerHead {
|
||||
t.Fatalf("the recheck learned siblings %+v, unknown %q", c.Siblings, c.SiblingsUnknown)
|
||||
}
|
||||
if asked != 1 {
|
||||
t.Fatalf("mesh-delivery was asked %d times", asked)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -418,37 +418,6 @@ func TestReplay299AHolderOfASeatWhoseVerbsAreAllOptionalIsNotSilent(t *testing.T
|
||||
}
|
||||
}
|
||||
|
||||
func TestReplay300AMergeAddingAModuleAsksForItsBuild(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
var asked [][3]string
|
||||
was := askABuild
|
||||
askABuild = func(_ context.Context, source buildSource, path, ref string) (string, error) {
|
||||
asked = append(asked, [3]string{source.Repository, path, ref})
|
||||
return "build-" + path, nil
|
||||
}
|
||||
t.Cleanup(func() { askABuild = was })
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "networkmanager", Version: "1"},
|
||||
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/networkmanager", Ref: "main",
|
||||
BuiltFrom: "c0", Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "3da80a4b00",
|
||||
Paths: []string{"modules/systemd-resolved/module.json", "modules/systemd-resolved/cmd/main.go"},
|
||||
ModuleDirs: []string{"modules/systemd-resolved"}, ModuleDirsSaid: true}
|
||||
if err := (following{open: open}).SourceMoved(ctx, m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := [3]string{"novox/mesh-catalog", "modules/systemd-resolved", "main"}
|
||||
if len(asked) != 1 || asked[0] != want {
|
||||
t.Fatalf("the merge adding modules/systemd-resolved asked the build seat for %v, not %v", asked, want)
|
||||
}
|
||||
if plans, err := inv.OpenPlans(ctx); err != nil || len(plans) != 0 {
|
||||
t.Fatalf("a merge moving nothing the mesh holds opened a plan: %+v %v", plans, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReplay301APersonsPushOfAHeldRecordedBuildIsNoRepair(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
|
||||
@@ -69,6 +69,7 @@ func TestARolloutAdvancesOnEventsAndAReplacedControllerResumesIt(t *testing.T) {
|
||||
var mu sync.Mutex
|
||||
var asks []string
|
||||
var sends [][]string
|
||||
asksOfEveryModule(t) // the one ask left is the builder's own (ADR 0453 §5): this test is of what a walk does around it
|
||||
wasAsk := askABuild
|
||||
askABuild = func(_ context.Context, source buildSource, _, _ string) (string, error) {
|
||||
mu.Lock()
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
package main
|
||||
|
||||
import "testing"
|
||||
|
||||
// A held build's commit as recorded and as registered name one commit however each is abbreviated, a short name
|
||||
// included (the fault the try of R4e's controller found: "c0" was never the same as "c0").
|
||||
func TestACommitIsTheSameAsItselfHoweverShort(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
a, b string
|
||||
same bool
|
||||
}{{"c0", "c0", true}, {"c0ffee0123456789", "c0ffee0", true}, {"c0ffee0123", "c0ffee0999", false}, {"", "", false},
|
||||
{"c0", "c1", false}} {
|
||||
if got := sameCommitID(c.a, c.b); got != c.same {
|
||||
t.Errorf("%q %q: %v", c.a, c.b, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -8,45 +8,22 @@ import (
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// **A module moves only when its program changes** (novox/hq issue 721). A merge check builds every module the
|
||||
// delivery plan moves and compares each with the build the mesh holds of it; a walk is cut without a module whose
|
||||
// check found its whole image the same as that build, while the mesh still holds it, and says so among the modules
|
||||
// it left unchanged. A merge of one sentence in a package a program imports but never reaches moves no build of it.
|
||||
// **A module moves only when its source changes** (novox/hq issue 721; from R4e by its fingerprint, ADR 0453 §6, ADR
|
||||
// 0499; issue 1851, A9). A walk is cut without a module whose fingerprint at the commit the walk carries is the
|
||||
// fingerprint of the build the mesh holds of it, and says so among the modules it left unchanged: a merge of a file no
|
||||
// module reads moves nothing. A build the mesh holds of the old form (build-agent's `src1:`) has no fingerprint to
|
||||
// compare: the module stays in the walk, and the rule of the standing build (equal digests, no move) judges it once its
|
||||
// key is taken.
|
||||
|
||||
// sameAsRunning are the modules of a walk its checks found the same as the build the mesh holds, each with what its
|
||||
// check said. **Only when every pull request the walk answers was checked at the head it merged, and every one of
|
||||
// those checks built the module and found it the same as the build the mesh still holds**: a merge the walk answers
|
||||
// without a pull request, a check of another head, a check that built it differently, or a held build that moved
|
||||
// since, leaves the module in the walk.
|
||||
func sameAsRunning(plan *inventory.Plan, builds []link.CheckedBuild, held map[string]string) map[string]string {
|
||||
if plan.Delivery == nil || len(plan.Delivery.Merges) == 0 {
|
||||
return nil
|
||||
}
|
||||
for _, m := range plan.Delivery.Merges {
|
||||
if _, ok := pullOf(m); !ok {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
pulls := reuseAskOf(plan).Pulls
|
||||
heads := mergedHeadsOf(plan)
|
||||
// sameByFingerprint are the modules whose fingerprint now (`src2:`, the indexer's trunk value at the walk's commit) is
|
||||
// the fingerprint of the build the mesh holds, each with why; one whose either is unknown, or of the old form, is not.
|
||||
func sameByFingerprint(now, held map[string]string) map[string]string {
|
||||
out := map[string]string{}
|
||||
for name := range plan.Modules {
|
||||
said := ""
|
||||
for _, pull := range pulls {
|
||||
b, ok := checkedBuildOf(builds, pull, heads[pull], name)
|
||||
if !ok || !stillHeld(b.Built, name, held) {
|
||||
said = ""
|
||||
break
|
||||
}
|
||||
if said == "" {
|
||||
said = b.Built.Same
|
||||
}
|
||||
}
|
||||
if said != "" {
|
||||
out[name] = said
|
||||
for m, fp := range now {
|
||||
if strings.HasPrefix(fp, "src2:") && held[m] == fp {
|
||||
out[m] = fmt.Sprintf("unchanged: its fingerprint %s is the build the mesh holds", short(fp))
|
||||
}
|
||||
}
|
||||
if len(out) == 0 {
|
||||
@@ -55,65 +32,67 @@ func sameAsRunning(plan *inventory.Plan, builds []link.CheckedBuild, held map[st
|
||||
return out
|
||||
}
|
||||
|
||||
// checkedBuildOf is the newest build a check of the pull request at head made of the module; false when none did.
|
||||
func checkedBuildOf(builds []link.CheckedBuild, pull link.PullRef, head, module string) (link.CheckedBuild, bool) {
|
||||
if head == "" {
|
||||
return link.CheckedBuild{}, false
|
||||
// heldFingerprint is the source fingerprint of the build the mesh holds of a module: the newest build of the commit
|
||||
// it is registered at.
|
||||
func heldFingerprint(ctx context.Context, inv *inventory.Inventory, module string) (string, error) {
|
||||
src, err := inv.SourceOf(ctx, module)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
for _, b := range builds {
|
||||
if b.Pull == pull && b.Built.Module == module && link.SameHead(b.Head, head) {
|
||||
return b, true
|
||||
fps, err := inv.BuildSourceFingerprints(ctx, module)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
for commit, fp := range fps {
|
||||
if sameCommitID(commit, src.BuiltFrom) {
|
||||
return fp, nil
|
||||
}
|
||||
}
|
||||
return link.CheckedBuild{}, false
|
||||
return "", nil
|
||||
}
|
||||
|
||||
// stillHeld is whether a check found every artifact of the module the same as a build the mesh still holds.
|
||||
func stillHeld(b link.CheckBuilt, module string, held map[string]string) bool {
|
||||
if b.Same == "" || b.NotBuilt != "" || len(b.Made) == 0 || len(b.SameAs) != len(b.Made) {
|
||||
return false
|
||||
}
|
||||
for _, m := range b.Made {
|
||||
was := pinnedDigest(b.SameAs[m.Name])
|
||||
if was == "" || was != pinnedDigest(held[module+"/"+m.Name]) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// pinnedDigest is a reference's pinned digest, empty when it is not pinned.
|
||||
func pinnedDigest(ref string) string {
|
||||
if i := strings.LastIndex(ref, "@"); i >= 0 {
|
||||
return ref[i+1:]
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// leaveSameAsRunning takes from a walk being cut every module its checks found the same as the build the mesh holds
|
||||
// (sameAsRunning), and every dependent in the walk only for one of those (withDependentsOf): what was taken, with why. **Anything not read leaves the walk as it was**, said.
|
||||
// leaveSameAsRunning takes from a walk being cut every module whose fingerprint is unchanged (sameByFingerprint), and
|
||||
// every dependent in the walk only for one of those (withDependentsOf): what was taken, with why. **Anything not read
|
||||
// leaves the walk as it was**, said.
|
||||
func leaveSameAsRunning(ctx context.Context, inv *inventory.Inventory, plan *inventory.Plan, moved []string,
|
||||
edges []inventory.Edge) map[string]string {
|
||||
if plan.Delivery == nil || len(plan.Modules) == 0 {
|
||||
if len(plan.Modules) == 0 {
|
||||
return nil
|
||||
}
|
||||
pulls := reuseAskOf(plan).Pulls
|
||||
if len(pulls) == 0 {
|
||||
return nil
|
||||
}
|
||||
builds, err := checkedBuildsOf(ctx, pulls)
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
fmt.Printf(" the checks' builds could not be read, so every module moves: %v\n", err)
|
||||
fmt.Printf(" the catalogue could not be read, so every module moves: %v\n", err)
|
||||
// empty-on-error: said above; nothing left out of the walk is the safe answer, every module moves as before
|
||||
return nil
|
||||
}
|
||||
held, err := inv.RegisteredHeld(ctx)
|
||||
keys, closeKeys, err := openPlanKeys()
|
||||
if err != nil {
|
||||
fmt.Printf(" what the mesh holds could not be read, so every module moves: %v\n", err)
|
||||
fmt.Printf(" the indexer's fingerprints could not be read, so every module moves: %v\n", err)
|
||||
// empty-on-error: said above; nothing left out of the walk is the safe answer, every module moves as before
|
||||
return nil
|
||||
}
|
||||
same := withDependentsOf(sameAsRunning(plan, builds, held), plan, moved, edges)
|
||||
defer closeKeys()
|
||||
byName := map[string]inventory.Entry{}
|
||||
for _, e := range entries {
|
||||
byName[e.Manifest.Module] = e
|
||||
}
|
||||
now, held := map[string]string{}, map[string]string{}
|
||||
for m := range plan.Modules {
|
||||
e, known := byName[m]
|
||||
if !known || isBuildersOwn(m) {
|
||||
continue
|
||||
}
|
||||
v, err := keys.ModuleAt(ctx, m, plan.CommitOn(e.Source.Repository, followedBranch(e.Source.Ref)))
|
||||
if err != nil || v.FP2 == "" {
|
||||
continue // not written yet, or none: the module moves, and its key is waited for in the walk
|
||||
}
|
||||
fp, err := heldFingerprint(ctx, inv, m)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
now[m], held[m] = v.FP2, fp
|
||||
}
|
||||
same := withDependentsOf(sameByFingerprint(now, held), plan, moved, edges)
|
||||
if len(same) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -3,45 +3,38 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
func TestAWalkWhoseEveryBuildCameOutTheSameIsDone(t *testing.T) {
|
||||
// R4e A9 (ADR 0453 §6, ADR 0499; issue 1851), against the store and a real bus: a merge of a file no module reads moves
|
||||
// nothing. Its module's fingerprint at the merge, as the indexer wrote it, is the fingerprint of the build the mesh holds,
|
||||
// so the walk is cut done with nothing to walk, and its record names the module unchanged and why — the walk
|
||||
// mesh-delivery reads as delivered, never stranded; no build is asked.
|
||||
func TestAWalkWhoseEveryFingerprintIsUnchangedIsDone(t *testing.T) {
|
||||
open := windowed(t)
|
||||
endsItsLine(t)
|
||||
asksRecorded(t)
|
||||
noBuildAsked(t)
|
||||
b := aBuilderOnTheBus(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-one", Repository: "novox/one", Ref: "main",
|
||||
Module: "one", Commit: "c0", On: "builder", Asked: t0.Add(-time.Hour),
|
||||
Made: []inventory.Artifact{{Name: "one", Kind: "image", Reference: "store/one@sha256:aaaa"}},
|
||||
Sources: []inventory.BuildSource{{Paths: []string{"main.go", "go.mod"}}}}); err != nil {
|
||||
fp := fpOf('5')
|
||||
if err := inv.RecordBuild(ctx, inventory.Build{ID: "key-held", Repository: "novox/one", Ref: "main",
|
||||
Module: "one", Commit: "c0", On: "ace", SourceFingerprint: fp,
|
||||
Made: []inventory.Artifact{{Name: "one", Kind: "image", Reference: "store/one@sha256:aaaa"}}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// What runs: the module registered with that build's image (RegisteredHeld reads the module table).
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "one", Version: "1", Resources: []map[string]any{
|
||||
{"id": "app", "type": "container", "name": "one", "image": "store/one@sha256:aaaa"}}},
|
||||
inventory.Source{Repository: "novox/one", Ref: "main", BuiltFrom: "c0", Asked: time.Now()}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pull := link.PullRef{Owner: "novox", Repo: "one", Number: 7}
|
||||
was := checkedBuildsOf
|
||||
checkedBuildsOf = func(context.Context, []link.PullRef) ([]link.CheckedBuild, error) {
|
||||
return []link.CheckedBuild{{Pull: pull, Head: "headaaaaaaaa", Built: link.CheckBuilt{Module: "one", Key: "k",
|
||||
Made: []link.MadeArtifact{{Name: "one", Kind: "image", Reference: "scratch/one@sha256:bbbb"}},
|
||||
Same: "unchanged: its program is the same as running build one aaaa",
|
||||
SameAs: map[string]string{"one": "10.0.0.1:5000/one@sha256:aaaa"}}}}, nil
|
||||
}
|
||||
t.Cleanup(func() { checkedBuildsOf = was })
|
||||
code := repoMerge("one", "one1aaaaaaaa", t0)
|
||||
code.Number, code.HeadSHA = 7, "headaaaaaaaa1111"
|
||||
b.indexed(t, "one", "1", "novox/one", "", code.Commit, fp)
|
||||
hear(t, open, code, t0.Add(time.Second))
|
||||
cutAt(t, open, t0.Add(2*time.Minute))
|
||||
ws, _ := walks(t, open)
|
||||
@@ -52,7 +45,7 @@ func TestAWalkWhoseEveryBuildCameOutTheSameIsDone(t *testing.T) {
|
||||
if w.State != inventory.PlanDone || len(w.Tiers) != 0 || len(w.Modules) != 0 {
|
||||
t.Fatalf("the walk reads %s, tiers %v, modules %v; want done with nothing to walk", w.State, w.Tiers, w.Modules)
|
||||
}
|
||||
if w.Delivery == nil || !strings.Contains(w.Delivery.Unchanged["one"], "its program is the same as running build") {
|
||||
t.Fatalf("the walk does not say one matched what runs: %+v", w.Delivery)
|
||||
if w.Delivery == nil || !strings.Contains(w.Delivery.Unchanged["one"], "is the build the mesh holds") {
|
||||
t.Fatalf("the walk does not say one's fingerprint is unchanged: %+v", w.Delivery)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,66 +5,26 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// TestAWalkLeavesOutAModuleWhoseProgramDidNotChange is novox/hq issue 721: mesh-controller #335 changed one sentence
|
||||
// that mesh-builder never reaches; its check found build-agent the same as the build the mesh holds, so the walk moves
|
||||
// mesh-controller alone and says build-agent is unchanged.
|
||||
func TestAWalkLeavesOutAModuleWhoseProgramDidNotChange(t *testing.T) {
|
||||
pull := link.PullRef{Owner: "novox", Repo: "mesh-controller", Number: 335}
|
||||
head := "34050dc6aaaabbbbccccdddd"
|
||||
aPlan := func() *inventory.Plan {
|
||||
return &inventory.Plan{Tiers: [][]string{{"mesh-controller", "build-agent"}},
|
||||
Modules: map[string]*inventory.PlanModule{"mesh-controller": {}, "build-agent": {}},
|
||||
Delivery: &inventory.PlanDelivery{Merges: []inventory.PlanMerge{{Repository: "novox/mesh-controller",
|
||||
Commit: "c2ec0a60", Number: 335, Head: head}}}}
|
||||
}
|
||||
held := map[string]string{"build-agent/agent": "store/build-agent@sha256:aaaa",
|
||||
"mesh-controller/controller": "store/mesh-controller@sha256:cccc"}
|
||||
builds := []link.CheckedBuild{
|
||||
{Pull: pull, Head: head[:12], Built: link.CheckBuilt{Module: "build-agent", Key: "k1",
|
||||
Made: []link.MadeArtifact{{Name: "agent", Kind: "image", Reference: "scratch/build-agent@sha256:bbbb"}},
|
||||
Same: "unchanged: its program is the same as running build agent aaaa",
|
||||
SameAs: map[string]string{"agent": "10.0.0.1:5000/build-agent@sha256:aaaa"}}},
|
||||
{Pull: pull, Head: head[:12], Built: link.CheckBuilt{Module: "mesh-controller", Key: "k2",
|
||||
Made: []link.MadeArtifact{{Name: "controller", Kind: "image", Reference: "scratch/mesh-controller@sha256:dddd"}}}},
|
||||
}
|
||||
|
||||
p := aPlan()
|
||||
same := sameAsRunning(p, builds, held)
|
||||
if len(same) != 1 || same["build-agent"] == "" {
|
||||
// TestAWalkLeavesOutAModuleWhoseFingerprintDidNotChange is novox/hq issue 721 on fingerprints (R4e A9, ADR 0453 §6):
|
||||
// mesh-controller #335 changed one sentence that mesh-builder never reaches; the indexer's fingerprint of build-agent at
|
||||
// the merge is the fingerprint of the build the mesh holds, so the walk moves mesh-controller alone and says build-agent
|
||||
// is unchanged. A held build of the old form, or a fingerprint not written, leaves the module in the walk.
|
||||
func TestAWalkLeavesOutAModuleWhoseFingerprintDidNotChange(t *testing.T) {
|
||||
a, b := "src2:"+strings.Repeat("a", 64), "src2:"+strings.Repeat("b", 64)
|
||||
now := map[string]string{"build-agent": a, "mesh-controller": b, "old": a, "unwritten": ""}
|
||||
held := map[string]string{"build-agent": a, "mesh-controller": a, "old": "src1:" + strings.Repeat("a", 64)}
|
||||
same := sameByFingerprint(now, held)
|
||||
if len(same) != 1 || !strings.Contains(same["build-agent"], "is the build the mesh holds") {
|
||||
t.Fatalf("the walk leaves out %v, want build-agent alone", same)
|
||||
}
|
||||
p := &inventory.Plan{Tiers: [][]string{{"mesh-controller", "build-agent"}},
|
||||
Modules: map[string]*inventory.PlanModule{"mesh-controller": {}, "build-agent": {}}}
|
||||
withhold(p, []string{"build-agent"}, nil)
|
||||
if len(p.Tiers) != 1 || len(p.Tiers[0]) != 1 || p.Tiers[0][0] != "mesh-controller" {
|
||||
t.Fatalf("the walk moves %v, want mesh-controller alone", p.Tiers)
|
||||
}
|
||||
|
||||
// **The held build moved since the check**: what it was compared with no longer runs.
|
||||
moved := map[string]string{"build-agent/agent": "store/build-agent@sha256:eeee"}
|
||||
if same := sameAsRunning(aPlan(), builds, moved); len(same) != 0 {
|
||||
t.Fatalf("a module compared with a build the mesh no longer holds is left out: %v", same)
|
||||
}
|
||||
// **A check of another head** says nothing of this merge.
|
||||
other := aPlan()
|
||||
other.Delivery.Merges[0].Head = "ffffffffffffffff"
|
||||
if same := sameAsRunning(other, builds, held); len(same) != 0 {
|
||||
t.Fatalf("a check of another head left out %v", same)
|
||||
}
|
||||
// **Another pull request the walk answers** that did not find it the same leaves it in.
|
||||
two := aPlan()
|
||||
two.Delivery.Merges = append(two.Delivery.Merges, inventory.PlanMerge{Repository: "novox/mesh-catalog",
|
||||
Commit: "1234", Number: 9, Head: "abcdefabcdefabcdef"})
|
||||
if same := sameAsRunning(two, builds, held); len(same) != 0 {
|
||||
t.Fatalf("a module another merge moves is left out: %v", same)
|
||||
}
|
||||
// **A merge with no pull request** leaves the walk as it is.
|
||||
direct := aPlan()
|
||||
direct.Delivery.Merges[0].Number = 0
|
||||
if same := sameAsRunning(direct, builds, held); len(same) != 0 {
|
||||
t.Fatalf("a walk answering a merge without a pull request left out %v", same)
|
||||
}
|
||||
}
|
||||
|
||||
// TestADependentOfABaseThatCameOutTheSameIsLeftOut is ADR 0267 rule 3 (novox/hq issue 607): a dependent is in a walk
|
||||
|
||||
@@ -52,6 +52,17 @@ type verbAnswer struct {
|
||||
// verb without the machine and ran as a push of every machine behind; a verb that answers "I did
|
||||
// not take that" would have stopped it before anything was sent.
|
||||
func argvFor(verb string, args map[string]any) ([]string, error) {
|
||||
// **A removed verb is refused by name, naming its replacement, whatever was asked of it** (novox/hq ADR 0501 §5,
|
||||
// ADR 0503 §3, ADR 0450 §8): before its arguments are read, so an argument a table of before offered is not what the
|
||||
// caller is told.
|
||||
if removedVerbs[verb] {
|
||||
return nil, removedByName(verb)
|
||||
}
|
||||
if verb == "replay" {
|
||||
if v, given := args["register"]; given && v != false && v != "false" {
|
||||
return nil, removedByName("replay --register")
|
||||
}
|
||||
}
|
||||
a, err := readArguments(verb, args)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -86,6 +97,10 @@ type verbArguments struct {
|
||||
misread []string // arguments the command line read that the schema does not declare: a bug here
|
||||
}
|
||||
|
||||
// removedVerbs are the controller seat's verbs R4e removed, kept in its table with no input so a caller is answered
|
||||
// by name (ADR 0501 §5, ADR 0503 §3, ADR 0450 §8).
|
||||
var removedVerbs = map[string]bool{"build": true, "rebuild": true, "forget-checks": true, "delivery-plan": true}
|
||||
|
||||
// controllerVerb is this binary's own definition of a verb: what it runs is what it declares, so the
|
||||
// arguments are checked against the table compiled beside argvFor, not a row a newer or older build
|
||||
// wrote.
|
||||
@@ -403,17 +418,6 @@ func (a *verbArguments) commandLine() ([]string, error) {
|
||||
}
|
||||
return argv, nil
|
||||
// The delivery's owner's verbs (novox/hq ADR 0239).
|
||||
case "delivery-plan":
|
||||
if err := need("repository", "paths"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
argv := []string{"delivery", "plan", "--repository", str("repository"), "--paths", str("paths")}
|
||||
for _, flag := range []string{"head", "base", "module-dirs", "removed"} {
|
||||
if v := str(flag); v != "" {
|
||||
argv = append(argv, "--"+flag, v)
|
||||
}
|
||||
}
|
||||
return argv, nil
|
||||
case "delivery-order":
|
||||
if err := need("members"); err != nil {
|
||||
return nil, err
|
||||
@@ -484,11 +488,6 @@ func (a *verbArguments) commandLine() ([]string, error) {
|
||||
case "queue":
|
||||
// As data, the table its output (novox/hq issue 455).
|
||||
return []string{"queue", "--json"}, nil
|
||||
case "forget-checks":
|
||||
if err := need("owner", "repo", "number"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return []string{"forget-checks", str("owner"), str("repo"), str("number")}, nil
|
||||
case "try":
|
||||
// A branch tried before its pull request (novox/hq issue 686): asked, or read by the id it answered.
|
||||
if id := str("id"); id != "" {
|
||||
@@ -521,23 +520,12 @@ func (a *verbArguments) commandLine() ([]string, error) {
|
||||
return []string{"clear", "--dead"}, nil
|
||||
}
|
||||
return []string{"clear"}, nil
|
||||
case "rebuild":
|
||||
if err := need("what"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return []string{"rebuild", str("what")}, nil
|
||||
case "replay":
|
||||
if err := need("id"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
argv := []string{"replay", str("id")}
|
||||
if on("register") {
|
||||
argv = append(argv, "--register")
|
||||
}
|
||||
if on("older") {
|
||||
argv = append(argv, "--older")
|
||||
}
|
||||
return argv, nil
|
||||
// A dry run alone (ADR 0429 §18): register is refused by name before this (argvFor).
|
||||
return []string{"replay", str("id")}, nil
|
||||
case "pause", "resume":
|
||||
if n := str("node"); n != "" {
|
||||
return []string{verb, n}, nil
|
||||
@@ -1075,35 +1063,6 @@ func (a *verbArguments) commandLine() ([]string, error) {
|
||||
return nil, err
|
||||
}
|
||||
return []string{"module", "issue", str("module"), "--node", str("node")}, nil
|
||||
case "build":
|
||||
// The command's three shapes (`build --on`, `build --behind`, `build <repository>`), one per
|
||||
// call: each branch reads only its own argument, so another given beside it is refused as
|
||||
// passed over rather than dropped. Asked and not waited for, the same as a single build.
|
||||
if b := str("on"); b != "" {
|
||||
return []string{"build", "--on", b, "--wait", "0"}, nil
|
||||
}
|
||||
if on("behind") {
|
||||
return []string{"build", "--behind", "--wait", "0"}, nil
|
||||
}
|
||||
if a.given["repository"] == "" {
|
||||
// No shape named: the command says its usage, which names all three — the answer the
|
||||
// caller needs, and the same as `rotate` given neither of its shapes.
|
||||
return []string{"build"}, nil
|
||||
}
|
||||
// Not waited for: a tool call cannot hold a connection for the minutes a build takes; the
|
||||
// daemon takes the outcome in when it comes and the id follows the build (issue 176). A
|
||||
// repository given without a scheme is a path on the forge holding the git seat.
|
||||
argv := []string{"build", str("repository"), "--wait", "0"}
|
||||
if !strings.Contains(str("repository"), "://") && !strings.HasPrefix(str("repository"), "git@") {
|
||||
argv = append(argv, "--self")
|
||||
}
|
||||
if p := str("path"); p != "" {
|
||||
argv = append(argv, "--path", p)
|
||||
}
|
||||
if r := str("ref"); r != "" {
|
||||
argv = append(argv, "--ref", r)
|
||||
}
|
||||
return argv, nil
|
||||
}
|
||||
return nil, fmt.Errorf("%q is a verb of the %s seat's table that this binary has no command line for",
|
||||
verb, catalogue.ControllerSeatName)
|
||||
@@ -1366,7 +1325,8 @@ func seatToolHandlers() (map[string]link.ToolHandler, []string, error) {
|
||||
continue
|
||||
}
|
||||
var policy *heldAtTheTerminal
|
||||
if _, err := argvFor(verb, sampleArguments(v)); err != nil && !errors.As(err, &policy) {
|
||||
var removed *removedCommand
|
||||
if _, err := argvFor(verb, sampleArguments(v)); err != nil && !errors.As(err, &policy) && !errors.As(err, &removed) {
|
||||
// **A row ahead of this binary is not a reason to go silent.**
|
||||
//
|
||||
// The row is the store's and a control plane follows it (novox/hq ADR 0154), so a verb
|
||||
@@ -1733,6 +1693,8 @@ var commandReadForms = map[string]func(rest []string) bool{
|
||||
"status": flagsOnly, "version": flagsOnly, "help": flagsOnly, "seats": flagsOnly, "healers": flagsOnly,
|
||||
"hand-acts": flagsOnly, "durations": flagsOnly, "collection": flagsOnly, "images": flagsOnly,
|
||||
"artifacts": flagsOnly, "data": flagsOnly, "builds": flagsOnly, "queue": flagsOnly,
|
||||
// The switch's list reads the builder's keys and the builds held, and moves nothing (issue 1851, A13).
|
||||
"switch": flagsOnly,
|
||||
// `plan <node>` previews a node's declaration; it sends nothing.
|
||||
"plan": func([]string) bool { return true },
|
||||
// `plans` lists and `plans <id>` shows one; `plans stop|close|go` acts.
|
||||
@@ -1803,6 +1765,9 @@ var terminalOnlyCommands = map[string]string{
|
||||
"broker": "the bus's accounts",
|
||||
"api": "the controller's API keys",
|
||||
"licence": "the licences' secrets",
|
||||
// The terminal's exception of ADR 0501 §4: a module from a repository the catalogue does not build, registered
|
||||
// from its done key.
|
||||
"register": "a module's registration from a repository the catalogue does not build",
|
||||
}
|
||||
|
||||
// givenAtTheDesk is exactly the line the `give` and `secret-ask` verbs compose, and nothing beside it: `secret ask
|
||||
|
||||
@@ -38,7 +38,7 @@ func servedSchemas(t *testing.T) map[string]catalogue.Verb {
|
||||
|
||||
func TestNoArgumentAVerbIsGivenIsPassedOver(t *testing.T) {
|
||||
for name, v := range servedSchemas(t) {
|
||||
if inProcess[name] {
|
||||
if inProcess[name] || removedVerbs[name] {
|
||||
continue
|
||||
}
|
||||
names, switches := declaredArguments(v)
|
||||
@@ -78,7 +78,7 @@ func TestAnArgumentAVerbDoesNotDeclareIsRefused(t *testing.T) {
|
||||
args := sampleArguments(v)
|
||||
args[stranger] = "x"
|
||||
_, err := argvFor(name, args)
|
||||
if inProcess[name] {
|
||||
if inProcess[name] || removedVerbs[name] {
|
||||
_, err = readArguments(name, args)
|
||||
}
|
||||
if err == nil || !strings.Contains(err.Error(), strconv.Quote(stranger)) {
|
||||
@@ -105,7 +105,7 @@ func TestEveryFlagOfAVerbsCommandIsAnArgumentOrAccountedFor(t *testing.T) {
|
||||
reached := map[string]map[string]bool{} // verb → flag sets its command lines reach
|
||||
served := servedSchemas(t)
|
||||
for name, v := range served {
|
||||
if inProcess[name] || name == "command" {
|
||||
if inProcess[name] || removedVerbs[name] || name == "command" {
|
||||
continue
|
||||
}
|
||||
names, switches := declaredArguments(v)
|
||||
|
||||
@@ -177,11 +177,8 @@ var accountedFlags = map[string]map[string]string{
|
||||
"all": "withheld: the verb naming no machine is --behind, and a push of every machine is said at a " +
|
||||
"shell (novox/hq ADR 0217); `command` reaches it",
|
||||
},
|
||||
"build": {
|
||||
"wait": "set by the verb to 0: the id follows the build (issue 176)",
|
||||
"self": "set by the verb from the repository's form: a path on the forge, or a URL",
|
||||
"dry-run": "withheld: a dry run answers only when the build ends, which a call cannot wait for; `command` reaches it",
|
||||
},
|
||||
// Kept only to be refused by name, naming what replaced it (novox/hq ADR 0501 §5, ADR 0429 §18).
|
||||
"replay": {"register": "withheld: refused by name, a registered replay being removed (ADR 0501 §5)"},
|
||||
"builds": {"n": "=limit"},
|
||||
"plans": {"n": "=limit", "what-if": "=repository"},
|
||||
// The machine's tunnel key, named as the verb's other arguments are (novox/hq ADR 0169).
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
@@ -21,66 +22,6 @@ func TestBuildsWithAnIdReadsThatBuildsLog(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// The build tool takes a repository as a URL or as its path on the forge holding the git seat, and
|
||||
// says which it was given, so the command reads the path as a seat source rather than handing it to
|
||||
// git as written (novox/hq issue 176). And it never waits: the id follows the build.
|
||||
func TestTheBuildToolTellsAForgePathFromAURL(t *testing.T) {
|
||||
argv, _ := argvFor("build", map[string]any{"repository": "novox/mesh-catalog", "path": "modules/x"})
|
||||
if line := strings.Join(argv, " "); !strings.Contains(line, "--self") || !strings.Contains(line, "--wait 0") {
|
||||
t.Fatalf("a forge path is a seat source, not waited for; got %q", line)
|
||||
}
|
||||
argv, _ = argvFor("build", map[string]any{"repository": "https://example.tld/o/r.git"})
|
||||
if line := strings.Join(argv, " "); strings.Contains(line, "--self") {
|
||||
t.Fatalf("a URL is cloned as given; got %q", line)
|
||||
}
|
||||
}
|
||||
|
||||
// The build tool has the command's three shapes: a repository, a base whose dependents are rebuilt
|
||||
// (`--on`), or everything behind its source (`--behind`) — each asked, not waited for, one per call.
|
||||
// Given neither, the command line is what it was: nothing new reaches it.
|
||||
func TestTheBuildToolTakesOnAndBehind(t *testing.T) {
|
||||
cases := []struct {
|
||||
args map[string]any
|
||||
want string
|
||||
}{
|
||||
{map[string]any{"on": "mesh-tools"}, "build --on mesh-tools --wait 0"},
|
||||
{map[string]any{"behind": "true"}, "build --behind --wait 0"},
|
||||
{map[string]any{"behind": true}, "build --behind --wait 0"},
|
||||
{map[string]any{"repository": "novox/x"}, "build novox/x --wait 0 --self"},
|
||||
{map[string]any{"repository": "novox/x", "behind": "false"}, "build novox/x --wait 0 --self"},
|
||||
{map[string]any{"repository": "novox/x", "path": "modules/x", "ref": "main"},
|
||||
"build novox/x --wait 0 --self --path modules/x --ref main"},
|
||||
}
|
||||
for _, c := range cases {
|
||||
argv, err := argvFor("build", c.args)
|
||||
if err != nil {
|
||||
t.Errorf("%v: %v", c.args, err)
|
||||
continue
|
||||
}
|
||||
if got := strings.Join(argv, " "); got != c.want {
|
||||
t.Errorf("%v became %q, not %q", c.args, got, c.want)
|
||||
}
|
||||
}
|
||||
// Naming no shape runs the bare command, which answers its usage naming all three.
|
||||
if argv, err := argvFor("build", map[string]any{}); err != nil || strings.Join(argv, " ") != "build" {
|
||||
t.Errorf("a build naming nothing became %v, %v", argv, err)
|
||||
}
|
||||
// One shape per call: a second beside it is refused, not dropped, and so is half of one.
|
||||
for _, refused := range []map[string]any{
|
||||
{"path": "modules/x"},
|
||||
{"on": "mesh-tools", "repository": "novox/x"},
|
||||
{"on": "mesh-tools", "behind": "true"},
|
||||
{"behind": "true", "repository": "novox/x"},
|
||||
{"behind": "true", "path": "modules/x"},
|
||||
{"on": "mesh-tools", "ref": "main"},
|
||||
{"behind": "yes"},
|
||||
} {
|
||||
if argv, err := argvFor("build", refused); err == nil {
|
||||
t.Errorf("%v was accepted as %v", refused, argv)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// `rotate` is one verb with two shapes (ADR 0114, issue 180): a pair credential by provision, or a
|
||||
// module's own secret by machine, module and name.
|
||||
func TestRotateTakesAProvisionOrAnOwnSecret(t *testing.T) {
|
||||
@@ -171,9 +112,45 @@ func TestActsDoNotBlockTheCall(t *testing.T) {
|
||||
if strings.Join(argv, " ") != "push one --wait 0s --why w" || !answersFirst(argv) {
|
||||
t.Fatalf("a named push with no setting waits, or holds its caller: %v", argv)
|
||||
}
|
||||
argv, _ = argvFor("build", map[string]any{"repository": "novox/x", "path": "modules/x"})
|
||||
if strings.Join(argv, " ") != "build novox/x --wait 0 --self --path modules/x" {
|
||||
t.Fatalf("build: %v", argv)
|
||||
}
|
||||
|
||||
// A11a (ADR 0501 §5; issue 1851): the build, rebuild and replay --register verbs reach a command that refuses them by
|
||||
// name, naming the replacement, whatever a table of before asked of them; the command lines refuse the same.
|
||||
func TestTheRemovedVerbsAreRefusedByName(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
verb string
|
||||
args map[string]any
|
||||
said string
|
||||
}{
|
||||
{"build", map[string]any{"repository": "novox/x", "path": "modules/x"}, "`build` is removed"},
|
||||
{"build", map[string]any{}, "`build` is removed"},
|
||||
{"rebuild", map[string]any{"what": "gitea"}, "`rebuild` is removed"},
|
||||
{"replay", map[string]any{"id": "build-1", "register": "true", "older": "true"}, "`replay --register` is removed"},
|
||||
// 1318: delivery-plan and forget-checks go too.
|
||||
{"delivery-plan", map[string]any{"repository": "novox/a", "paths": "x"}, "`delivery-plan` is removed"},
|
||||
{"forget-checks", map[string]any{"owner": "novox", "repo": "a", "number": "7"}, "`forget-checks` is removed"},
|
||||
} {
|
||||
argv, err := argvFor(c.verb, c.args)
|
||||
if err == nil || !strings.Contains(err.Error(), c.said) {
|
||||
t.Errorf("%s %v: %v %v, want refused saying %q", c.verb, c.args, argv, err, c.said)
|
||||
}
|
||||
}
|
||||
if argv, err := argvFor("replay", map[string]any{"id": "build-1"}); err != nil || strings.Join(argv, " ") != "replay build-1" {
|
||||
t.Errorf("replay as a dry run: %v %v", argv, err)
|
||||
}
|
||||
for _, removed := range []string{"build", "rebuild", "forget-checks", "delivery-plan"} {
|
||||
err := removedByName(removed)
|
||||
if err == nil || !strings.Contains(err.Error(), "`"+removed+"` is removed") || !strings.Contains(err.Error(), "ADR 05") {
|
||||
t.Errorf("%s: %v", removed, err)
|
||||
}
|
||||
}
|
||||
if err := deliveryCommand(context.Background(), []string{"plan", "--repository", "novox/a"}); err == nil ||
|
||||
!strings.Contains(err.Error(), "`delivery-plan` is removed") {
|
||||
t.Errorf("delivery plan: %v", err)
|
||||
}
|
||||
err := replayCommand(context.Background(), []string{"build-1", "--register"})
|
||||
if err == nil || !strings.Contains(err.Error(), "`replay --register` is removed") {
|
||||
t.Errorf("replay --register: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -65,6 +65,7 @@ func TestAPlanPassesATierWhoseMachineNeverReportsAfterTheBoundAndSendsTheNextToT
|
||||
var mu sync.Mutex
|
||||
var asks []string
|
||||
var sends [][]string
|
||||
asksOfEveryModule(t) // the one ask left is the builder's own (ADR 0453 §5): this test is of what a walk does around it
|
||||
wasAsk := askABuild
|
||||
askABuild = func(_ context.Context, source buildSource, _, _ string) (string, error) {
|
||||
mu.Lock()
|
||||
|
||||
@@ -0,0 +1,225 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/announce"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// What each module keeps, and who may read it (novox/hq ADR 0495 clauses 2 to 5, ADR 0513 §3; issue 1790, D1-D3).
|
||||
//
|
||||
// Every module says on the bus which states it keeps and which of them others may read (announce.Subject), from its
|
||||
// code; the controller says its own (D1). The serving controller keeps the last word of each module in its bucket
|
||||
// (D2), and the bus's users are composed from it: a reader's `reads: <owner>.<name>` is granted only when the owner
|
||||
// announced that state readable; an owner that has announced nothing yet is read from `reads` as before (the
|
||||
// transition), and `status` names it. A changed word composes the bus's users again: the machine holding the bus is
|
||||
// sent its new user list (D3). No timer: the words wake it.
|
||||
|
||||
// announceOwnStates says what the controller keeps, on its own event subject (D1).
|
||||
func announceOwnStates(ctx context.Context, js *broker.JetStream) error {
|
||||
host, _ := os.Hostname()
|
||||
body, err := announce.Encode(broker.ControllerAnnouncement(host))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
api, err := jetstream.New(js.Conn())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := api.Publish(ctx, announce.Subject(broker.ControllerSeat), body); err != nil {
|
||||
return fmt.Errorf("saying what the controller keeps: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// keepAnnouncements follows every module's word on what it keeps (D2) and keeps the last of each, from the last each
|
||||
// module said: one module's word in the bucket under its name. A word whose module is not the subject's is refused,
|
||||
// said; a changed word composes the bus's users again (D3).
|
||||
func keepAnnouncements(ctx context.Context, js *broker.JetStream, open *stores) error {
|
||||
api, err := jetstream.New(js.Conn())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
kv, err := api.KeyValue(ctx, broker.StatesAnnouncedBucket)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the bucket of the states announced: %w", err)
|
||||
}
|
||||
stream, err := api.Stream(ctx, broker.EventsStream)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the events stream: %w", err)
|
||||
}
|
||||
consumer, err := stream.OrderedConsumer(ctx, jetstream.OrderedConsumerConfig{
|
||||
FilterSubjects: []string{announce.SubjectAll}, DeliverPolicy: jetstream.DeliverLastPerSubjectPolicy})
|
||||
if err != nil {
|
||||
return fmt.Errorf("following the states announced: %w", err)
|
||||
}
|
||||
consuming, err := consumer.Consume(func(m jetstream.Msg) {
|
||||
changed, err := keepAnnouncement(ctx, kv, m.Subject(), m.Data())
|
||||
if err != nil {
|
||||
fmt.Printf("states: %v\n", err)
|
||||
return
|
||||
}
|
||||
if changed {
|
||||
inBackground(func() { composeAnnouncedReads(context.WithoutCancel(ctx), open) })
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("following the states announced: %w", err)
|
||||
}
|
||||
go func() {
|
||||
<-ctx.Done()
|
||||
consuming.Stop()
|
||||
}()
|
||||
return nil
|
||||
}
|
||||
|
||||
// announcedKeeper is the bucket of the states announced, as keepAnnouncement writes it.
|
||||
type announcedKeeper interface {
|
||||
Get(ctx context.Context, key string) (jetstream.KeyValueEntry, error)
|
||||
Put(ctx context.Context, key string, value []byte) (uint64, error)
|
||||
}
|
||||
|
||||
// keepAnnouncement keeps one module's word, and says whether it changed what the bucket held.
|
||||
func keepAnnouncement(ctx context.Context, kv announcedKeeper, subject string, body []byte) (bool, error) {
|
||||
s, err := announce.Decode(body)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("a word on %s is not read: %w", subject, err)
|
||||
}
|
||||
if subject != announce.Subject(s.Module) {
|
||||
return false, fmt.Errorf("%s says what %s keeps, and a module says only what it keeps itself: refused", subject,
|
||||
s.Module)
|
||||
}
|
||||
key, err := announce.Key(s.Module)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
stored, err := announce.Encode(s)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if was, err := kv.Get(ctx, key); err == nil {
|
||||
if string(was.Value()) == string(stored) {
|
||||
return false, nil
|
||||
}
|
||||
} else if !errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return false, fmt.Errorf("the last word of %s cannot be read: %w", s.Module, err)
|
||||
}
|
||||
if _, err := kv.Put(ctx, key, stored); err != nil {
|
||||
return false, fmt.Errorf("the word of %s cannot be kept: %w", s.Module, err)
|
||||
}
|
||||
fmt.Printf("states: %s keeps %s\n", s.Module, statesWords(s))
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// statesWords is an announcement as a person reads it: each state, readable ones marked.
|
||||
func statesWords(s announce.States) string {
|
||||
if len(s.States) == 0 {
|
||||
return "nothing"
|
||||
}
|
||||
var out []string
|
||||
for _, k := range s.States {
|
||||
if k.Readable {
|
||||
out = append(out, k.Name+" (readable)")
|
||||
} else {
|
||||
out = append(out, k.Name)
|
||||
}
|
||||
}
|
||||
return strings.Join(out, ", ")
|
||||
}
|
||||
|
||||
// readAnnouncements is the last word of every module, by module (what inventory.Announcements reads): none when the
|
||||
// bucket is not there yet, which reads every owner as not yet announced.
|
||||
func readAnnouncements(ctx context.Context) (map[string]announce.States, error) {
|
||||
js, err := aBus()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer js.Close()
|
||||
api, err := jetstream.New(js.Conn())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
kv, err := api.KeyValue(ctx, broker.StatesAnnouncedBucket)
|
||||
if errors.Is(err, jetstream.ErrBucketNotFound) {
|
||||
// empty-on-error: not an error; no module announced yet, so every read is granted from `reads` as before
|
||||
return map[string]announce.States{}, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
keys, err := kv.Keys(ctx)
|
||||
if errors.Is(err, jetstream.ErrNoKeysFound) {
|
||||
// empty-on-error: not an error; no module announced yet
|
||||
return map[string]announce.States{}, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := map[string]announce.States{}
|
||||
for _, key := range keys {
|
||||
e, err := kv.Get(ctx, key)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s, err := announce.Decode(e.Value())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the kept word of %s is not read: %w", key, err)
|
||||
}
|
||||
out[s.Module] = s
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// composeAnnouncedReads sends the machine holding the bus its user list when a changed word changed it (D3): as a new
|
||||
// controller's user list is carried (carryUserList), not when that machine is held.
|
||||
func composeAnnouncedReads(ctx context.Context, open *stores) {
|
||||
holder, behind, err := brokerBehind(ctx, open, nil)
|
||||
if err != nil {
|
||||
fmt.Printf("states: whether the bus's user list is behind the states announced cannot be read: %v\n", err)
|
||||
return
|
||||
}
|
||||
if holder == "" || !behind {
|
||||
return
|
||||
}
|
||||
held, err := heldMachines(ctx, open, []string{holder})
|
||||
if err != nil {
|
||||
fmt.Printf("states: whether %s may be sent its new user list cannot be read: %v\n", holder, err)
|
||||
return
|
||||
}
|
||||
if why, isHeld := held[holder]; isHeld {
|
||||
fmt.Printf("states: the user list the states announced compose is not sent to %s, which holds the bus: %s — "+
|
||||
"`push %s` sends it\n", holder, strings.Join(why, "; "), holder)
|
||||
return
|
||||
}
|
||||
if _, err := sendRollout(ctx, open, []string{holder}); err != nil {
|
||||
fmt.Printf("states: the user list the states announced compose could not be sent to %s: %v\n", holder, err)
|
||||
return
|
||||
}
|
||||
fmt.Printf("states: sent %s, which holds the bus, the user list the states announced compose\n", holder)
|
||||
}
|
||||
|
||||
// notYetAnnounced are the owners some module reads the state of and that have announced nothing yet: what `status`
|
||||
// names while the transition lasts.
|
||||
func notYetAnnounced(ctx context.Context, inv *inventory.Inventory) ([]string, error) {
|
||||
if inventory.Announcements == nil {
|
||||
return nil, nil
|
||||
}
|
||||
records, err := inv.BusRecords(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if records.Announced == nil {
|
||||
return nil, errors.New("the states announced cannot be read")
|
||||
}
|
||||
return broker.NotAnnounced(records), nil
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"slices"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/announce"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// D1-D3 whole against a real bus and the store (ADR 0495 checks §2, §3, §5; ADR 0513 §3; issue 1790): the controller
|
||||
// says what it keeps and keeps what every module says; a reader of a state its owner announced readable is granted it,
|
||||
// one its owner announced and did not make readable is refused and said, and one of an owner that announced nothing
|
||||
// yet is granted as before and named as not yet announced.
|
||||
func TestTheStatesAnnouncedGrantTheReads(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
js, err := broker.Dial(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
if err := broker.AssertMeshStreams(js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := js.EnsureControllerBuckets(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
servingBus.Store(js)
|
||||
t.Cleanup(func() { servingBus.Store(nil) })
|
||||
was := inventory.Announcements
|
||||
inventory.Announcements = readAnnouncements
|
||||
t.Cleanup(func() { inventory.Announcements = was })
|
||||
|
||||
// The owner keeps two states, one readable; the reader reads both and a state of a module that says nothing.
|
||||
register := func(m catalogue.Manifest) {
|
||||
t.Helper()
|
||||
if err := inv.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-catalog", Seat: "git",
|
||||
Path: "modules/" + m.Module, BuiltFrom: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
register(catalogue.Manifest{Module: "owner", Version: "1", State: []catalogue.StateDeclaration{{Name: "open"}, {Name: "closed"}}})
|
||||
register(catalogue.Manifest{Module: "quiet", Version: "1", State: []catalogue.StateDeclaration{{Name: "any"}}})
|
||||
register(catalogue.Manifest{Module: "reader", Version: "1", Reads: []string{"owner.open", "owner.closed", "quiet.any"}})
|
||||
if _, err := inv.AddNode(ctx, "one"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, m := range []string{"owner", "quiet", "reader"} {
|
||||
if _, err := inv.Assign(ctx, "one", m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
watching, stop := context.WithCancel(context.Background())
|
||||
t.Cleanup(stop)
|
||||
if err := announceOwnStates(ctx, js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := keepAnnouncements(watching, js, open); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
api, _ := jetstream.New(js.Conn())
|
||||
word, _ := announce.Encode(announce.States{Module: "owner", Node: "one",
|
||||
States: []announce.State{{Name: "open", Readable: true}, {Name: "closed"}}})
|
||||
if _, err := api.Publish(ctx, announce.Subject("owner"), word); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
becomes(t, 10*time.Second, "both words kept", func() bool {
|
||||
a, err := readAnnouncements(ctx)
|
||||
return err == nil && len(a) == 2 && a["owner"].Readable("open") && a[broker.ControllerSeat].Readable(broker.PlansBuilding)
|
||||
})
|
||||
|
||||
records, err := inv.BusRecords(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var reader broker.Declared
|
||||
for _, d := range records.Assigned["one"] {
|
||||
if d.Module == "reader" {
|
||||
reader = d
|
||||
}
|
||||
}
|
||||
if !slices.Equal(reader.Reads, []string{"owner.open", "quiet.any"}) || !slices.Equal(reader.ReadsRefused, []string{"owner.closed"}) {
|
||||
t.Fatalf("0495 §3: the reader is granted %v and refused %v", reader.Reads, reader.ReadsRefused)
|
||||
}
|
||||
if got := broker.NotAnnounced(records); !slices.Equal(got, []string{"quiet"}) {
|
||||
t.Fatalf("not yet announced: %v", got)
|
||||
}
|
||||
users, err := broker.Users(records)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, u := range users {
|
||||
if u.Kind != broker.KindModule || u.Module != "reader" {
|
||||
continue
|
||||
}
|
||||
perms, err := broker.PermissionsFor(u)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if slices.ContainsFunc(perms.Publish, func(s string) bool { return s == "$JS.API.DIRECT.GET.KV_owner_closed.>" }) {
|
||||
t.Fatal("0495 §3: the reader may read the state its owner did not announce readable")
|
||||
}
|
||||
if !slices.Contains(perms.Publish, "$JS.API.DIRECT.GET.KV_owner_open.>") {
|
||||
t.Fatal("0495 §3: the reader may not read the state its owner announced readable")
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/announce"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
|
||||
// announcedInMemory is the bucket of the states announced, in memory.
|
||||
type announcedInMemory map[string][]byte
|
||||
|
||||
type announcedEntry struct {
|
||||
jetstream.KeyValueEntry
|
||||
value []byte
|
||||
}
|
||||
|
||||
func (e announcedEntry) Value() []byte { return e.value }
|
||||
|
||||
func (k announcedInMemory) Get(_ context.Context, key string) (jetstream.KeyValueEntry, error) {
|
||||
v, ok := k[key]
|
||||
if !ok {
|
||||
return nil, jetstream.ErrKeyNotFound
|
||||
}
|
||||
return announcedEntry{value: v}, nil
|
||||
}
|
||||
|
||||
func (k announcedInMemory) Put(_ context.Context, key string, value []byte) (uint64, error) {
|
||||
k[key] = value
|
||||
return uint64(time.Now().UnixNano()), nil
|
||||
}
|
||||
|
||||
// D2 (ADR 0495 clause 2; issue 1790): a module's word is kept under its name, changed only when it changes, and a word
|
||||
// about another module than its subject's is refused.
|
||||
func TestAModulesWordIsKeptUnderItsNameAndOnlyItsOwn(t *testing.T) {
|
||||
k := announcedInMemory{}
|
||||
word, _ := announce.Encode(announce.States{Module: "keeper", States: []announce.State{{Name: "things", Readable: true}}})
|
||||
changed, err := keepAnnouncement(t.Context(), k, announce.Subject("keeper"), word)
|
||||
if err != nil || !changed || string(k["keeper"]) != string(word) {
|
||||
t.Fatalf("the first word: %v %v %q", changed, err, k["keeper"])
|
||||
}
|
||||
if changed, err := keepAnnouncement(t.Context(), k, announce.Subject("keeper"), word); err != nil || changed {
|
||||
t.Fatalf("the same word again changed the bucket: %v %v", changed, err)
|
||||
}
|
||||
other, _ := announce.Encode(announce.States{Module: "victim", States: []announce.State{{Name: "secrets", Readable: true}}})
|
||||
if _, err := keepAnnouncement(t.Context(), k, announce.Subject("keeper"), other); err == nil ||
|
||||
!strings.Contains(err.Error(), "only what it keeps itself") {
|
||||
t.Fatalf("a word about another module was kept: %v", err)
|
||||
}
|
||||
if _, found := k["victim"]; found {
|
||||
t.Fatal("a word about another module was kept")
|
||||
}
|
||||
}
|
||||
@@ -85,6 +85,13 @@ func printStatus(asked answers) error {
|
||||
// **What is wrong leads** (novox/hq to-be 45 §2): every open condition, urgent first, oldest
|
||||
// first, silenced ones with when their silence ends.
|
||||
printConditions(asked.conditions, asked.conditionsUnread, time.Now())
|
||||
if len(asked.notAnnounced) > 0 {
|
||||
fmt.Printf("%d module(s) others read the state of have not said what they keep, and are read from `reads` as "+
|
||||
"before until they do (ADR 0495): %s\n\n", len(asked.notAnnounced), strings.Join(asked.notAnnounced, ", "))
|
||||
}
|
||||
if asked.notAnnouncedWhy != "" {
|
||||
fmt.Printf("whose states are read without an announcement cannot be said: %s\n\n", asked.notAnnouncedWhy)
|
||||
}
|
||||
|
||||
if len(asked.refused) > 0 {
|
||||
// First of what follows. A machine that cannot be worked out is not running an old
|
||||
@@ -435,6 +442,11 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) {
|
||||
if err != nil {
|
||||
return answers{}, err
|
||||
}
|
||||
// And whose states are still read without an announcement (novox/hq ADR 0495, issue 1790): said, never a reason
|
||||
// for the rest of the answer to fail.
|
||||
if out.notAnnounced, err = notYetAnnounced(ctx, inv); err != nil {
|
||||
out.notAnnounced, out.notAnnouncedWhy = nil, err.Error()
|
||||
}
|
||||
// And what each machine is holding rather than running, by the module that would run it. Read
|
||||
// from what the machine itself last reported, not from what take-time computed: the machine is
|
||||
// the only thing that knows what it found (novox/hq 04-ISSUES/125).
|
||||
|
||||
@@ -0,0 +1,204 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"github.com/novox/mesh-controller/internal/builderkeys"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// The switch's list (novox/hq ADR 0453 §6, ADR 0499; issue 1851, A13): which modules the switch to the builder's keys
|
||||
// moves, read before R4e is proposed and while it is delivered. For each module the catalogue builds, the held build's
|
||||
// artifacts are compared by digest with the builder's done key of the fingerprint the indexer's trunk value names:
|
||||
// every digest the same is no move (the standing build, A9); one that differs moves. An image and a package are named
|
||||
// with why they cannot be compared before the trunk builds them: an image of build-agent and one of mesh-build-image
|
||||
// are not one byte-for-byte build, and a candidate's image is built again for the trunk (ADR 0498 §6); a package is
|
||||
// published by its version, not its digest. It moves nothing.
|
||||
|
||||
// The verdicts of the switch's list.
|
||||
const (
|
||||
switchUnmoved = "unmoved"
|
||||
switchMoves = "moves"
|
||||
switchUncompared = "cannot be compared"
|
||||
switchUnknown = "not known"
|
||||
)
|
||||
|
||||
// switchLine is one module's line of the switch's list.
|
||||
type switchLine struct {
|
||||
Module string `json:"module"`
|
||||
Verdict string `json:"verdict"`
|
||||
Why string `json:"why"`
|
||||
}
|
||||
|
||||
// switchReads is what the switch's list reads of the builder and the indexer.
|
||||
type switchReads interface {
|
||||
ModuleAt(ctx context.Context, module, commit string) (sdkdelivery.Module, error)
|
||||
Build(ctx context.Context, module, fp string) (builderkeys.Key, error)
|
||||
}
|
||||
|
||||
// switchLineOf judges one module: its held build's artifacts against its key, as each artifact's kind allows.
|
||||
func switchLineOf(module string, held []inventory.Artifact, trunk sdkdelivery.Module, k builderkeys.Key) switchLine {
|
||||
line := switchLine{Module: module}
|
||||
switch {
|
||||
case trunk.FP2 == "":
|
||||
line.Verdict, line.Why = switchUnknown, "the indexer wrote no fingerprint of the new form: "+orWords(trunk.None2, "none said")
|
||||
return line
|
||||
case !k.Done():
|
||||
line.Verdict, line.Why = switchUnknown, k.Says()
|
||||
return line
|
||||
case len(held) == 0:
|
||||
line.Verdict, line.Why = switchMoves, "the mesh holds no build of it to compare"
|
||||
return line
|
||||
}
|
||||
digests := map[string]string{}
|
||||
for _, a := range held {
|
||||
digests[a.Name] = digestOfReference(a.Reference)
|
||||
}
|
||||
var moved, uncompared []string
|
||||
for _, a := range k.Build.Artifacts {
|
||||
was, had := digests[a.Name]
|
||||
delete(digests, a.Name)
|
||||
switch {
|
||||
case !had:
|
||||
moved = append(moved, a.Name+" is new")
|
||||
case was == a.Digest:
|
||||
case a.Kind == catalogue.ArtifactImage:
|
||||
uncompared = append(uncompared, a.Name+" is an image: build-agent's and mesh-build-image's are not one "+
|
||||
"byte-for-byte build, and a candidate's image is built again for the trunk (ADR 0498 §6)")
|
||||
case a.Kind == "package":
|
||||
uncompared = append(uncompared, a.Name+" is a package: it is published by its version, not its digest")
|
||||
default:
|
||||
moved = append(moved, fmt.Sprintf("%s %s -> %s", a.Name, shortDigest(was), shortDigest(a.Digest)))
|
||||
}
|
||||
}
|
||||
for name := range digests {
|
||||
moved = append(moved, name+" is gone")
|
||||
}
|
||||
sort.Strings(moved)
|
||||
sort.Strings(uncompared)
|
||||
switch {
|
||||
case len(moved) > 0:
|
||||
line.Verdict, line.Why = switchMoves, strings.Join(append(moved, uncompared...), "; ")
|
||||
case len(uncompared) > 0:
|
||||
line.Verdict, line.Why = switchUncompared, strings.Join(uncompared, "; ")
|
||||
default:
|
||||
line.Verdict, line.Why = switchUnmoved, "every artifact's digest is the held build's: registered with it, no move"
|
||||
}
|
||||
return line
|
||||
}
|
||||
|
||||
// digestOfReference is the `sha256:<hex>` a reference names; empty when it names none.
|
||||
func digestOfReference(reference string) string {
|
||||
for _, marker := range []string{"@sha256:", "/sha256:"} {
|
||||
if _, after, ok := strings.Cut(reference, marker); ok {
|
||||
return "sha256:" + after
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func shortDigest(d string) string {
|
||||
d = strings.TrimPrefix(d, "sha256:")
|
||||
if len(d) > 12 {
|
||||
return d[:12]
|
||||
}
|
||||
if d == "" {
|
||||
return "none"
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
// switchCommand is `switch [--json]`: the switch's list, read-only.
|
||||
func switchCommand(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("switch", flag.ContinueOnError)
|
||||
asJSON := set.Bool("json", false, "answer the list as JSON")
|
||||
if err := set.Parse(args); err != nil {
|
||||
return err
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
keys, closeKeys, err := openBuilderKeys()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer closeKeys()
|
||||
lines, err := switchList(ctx, open.inventory, keys)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if *asJSON {
|
||||
return json.NewEncoder(os.Stdout).Encode(lines)
|
||||
}
|
||||
counts := map[string]int{}
|
||||
for _, l := range lines {
|
||||
counts[l.Verdict]++
|
||||
fmt.Printf("%-28s %-18s %s\n", l.Module, l.Verdict, l.Why)
|
||||
}
|
||||
fmt.Printf("\n%d module(s): %d unmoved, %d move, %d cannot be compared, %d not known\n", len(lines),
|
||||
counts[switchUnmoved], counts[switchMoves], counts[switchUncompared], counts[switchUnknown])
|
||||
return nil
|
||||
}
|
||||
|
||||
// switchList is the switch's line of every module the catalogue builds but the builder's own, by name.
|
||||
func switchList(ctx context.Context, inv *inventory.Inventory, keys switchReads) ([]switchLine, error) {
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []switchLine
|
||||
for _, e := range entries {
|
||||
m := e.Manifest.Module
|
||||
if e.Provided || e.Source.Repository == "" || isBuildersOwn(m) {
|
||||
continue
|
||||
}
|
||||
trunk, err := keys.ModuleAt(ctx, m, "")
|
||||
if errors.Is(err, builderkeys.ErrNoValue) {
|
||||
out = append(out, switchLine{Module: m, Verdict: switchUnknown, Why: "the indexer holds no trunk value of it"})
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var k builderkeys.Key
|
||||
if trunk.FP2 != "" {
|
||||
if k, err = keys.Build(ctx, m, trunk.FP2); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
held, err := heldArtifacts(ctx, inv, m, e.Source.BuiltFrom)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, switchLineOf(m, held, trunk, k))
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].Module < out[j].Module })
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// heldArtifacts are the artifacts of the build the mesh holds of a module: the newest that worked at the commit it is
|
||||
// registered at.
|
||||
func heldArtifacts(ctx context.Context, inv *inventory.Inventory, module, builtFrom string) ([]inventory.Artifact, error) {
|
||||
builds, err := inv.Builds(ctx, module, 20)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, b := range builds {
|
||||
if b.Worked() && sameCommitID(b.Commit, builtFrom) {
|
||||
return b.Made, nil
|
||||
}
|
||||
}
|
||||
// empty-on-error: not an error; the mesh holds no build of that commit on record, which the line says
|
||||
return nil, nil
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"github.com/novox/mesh-controller/internal/builderkeys"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// A13 (ADR 0453 §6; issue 1851): the switch's list names every module moved or unmoved and why: the held build's
|
||||
// digests against the done key's; an image or a package as what cannot be compared; a key not done as not known.
|
||||
func TestTheSwitchsListNamesWhatMovesAndWhy(t *testing.T) {
|
||||
const d1, d2 = "sha256:" + "1111111111111111111111111111111111111111111111111111111111111111",
|
||||
"sha256:" + "2222222222222222222222222222222222222222222222222222222222222222"
|
||||
trunk := sdkdelivery.Module{FP2: "src2:" + strings.Repeat("a", 64)}
|
||||
done := func(arts ...sdkdelivery.BuiltArtifact) builderkeys.Key {
|
||||
return builderkeys.Key{Found: true, Fingerprint: trunk.FP2, Build: sdkdelivery.BuilderBuild{State: sdkdelivery.BuildDone,
|
||||
Artifacts: arts}}
|
||||
}
|
||||
held := []inventory.Artifact{{Name: "tools", Kind: "bundle", Reference: "artifact-store://x/tools/blobs/" + d1}}
|
||||
for _, c := range []struct {
|
||||
what string
|
||||
held []inventory.Artifact
|
||||
trunk sdkdelivery.Module
|
||||
key builderkeys.Key
|
||||
verdict string
|
||||
says string
|
||||
}{
|
||||
{"the same digest", held, trunk, done(sdkdelivery.BuiltArtifact{Name: "tools", Kind: "bundle", Digest: d1}), switchUnmoved, "no move"},
|
||||
{"another digest", held, trunk, done(sdkdelivery.BuiltArtifact{Name: "tools", Kind: "bundle", Digest: d2}), switchMoves, "tools 111111111111 -> 222222222222"},
|
||||
{"an image", []inventory.Artifact{{Name: "img", Kind: "image", Reference: "artifact-store://x/img@" + d1}}, trunk,
|
||||
done(sdkdelivery.BuiltArtifact{Name: "img", Kind: "image", Digest: d2}), switchUncompared, "img is an image"},
|
||||
{"a package", []inventory.Artifact{{Name: "lib", Kind: "package", Reference: "@novox/lib@1.2.0"}}, trunk,
|
||||
done(sdkdelivery.BuiltArtifact{Name: "lib", Kind: "package", Digest: d2}), switchUncompared, "published by its version"},
|
||||
{"a key building", held, trunk, builderkeys.Key{Module: "x", Fingerprint: trunk.FP2, Found: true,
|
||||
Build: sdkdelivery.BuilderBuild{State: sdkdelivery.BuildBuilding}}, switchUnknown, "is building"},
|
||||
{"no fingerprint", held, sdkdelivery.Module{None2: "no lock"}, builderkeys.Key{}, switchUnknown, "no lock"},
|
||||
} {
|
||||
l := switchLineOf("x", c.held, c.trunk, c.key)
|
||||
if l.Verdict != c.verdict || !strings.Contains(l.Why, c.says) {
|
||||
t.Errorf("%s: %+v, want %s saying %q", c.what, l, c.verdict, c.says)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -302,6 +302,7 @@ func TestAModuleItsSendChangedNothingOfIsLeftAsItWasAndRetried(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
asksOfEveryModule(t) // the one ask left is the builder's own (ADR 0453 §5): this test is of what a walk does around it
|
||||
was := askABuild
|
||||
askABuild = func(context.Context, buildSource, string, string) (string, error) { return "build-3", nil }
|
||||
t.Cleanup(func() { askABuild = was })
|
||||
|
||||
+93
-147
@@ -12,41 +12,34 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A branch is tried before its pull request (novox/hq issue 686).
|
||||
// A branch is tried before its pull request (novox/hq issue 686), by the checker (novox/hq ADR 0501 §5; issue 1851).
|
||||
//
|
||||
// A repository's store tests skip without a store, and no agent may start a database on its machine. So a
|
||||
// builder pushed, a reviewer read, and only the pull request's merge check ran those tests for the first time:
|
||||
// each failure there cost a round of fifteen to thirty minutes (mesh-controller #283, #310, #312).
|
||||
// A repository's store tests skip without a store, and no agent may start a database on its machine. So a builder
|
||||
// pushed, a reviewer read, and only the pull request's merge check ran those tests for the first time: each failure
|
||||
// there cost a round of fifteen to thirty minutes (mesh-controller #283, #310, #312).
|
||||
//
|
||||
// `try <owner>/<repository> <ref>` asks the build seat for the repository's own check of a pushed branch or
|
||||
// commit, as the merge check runs it (internal/builder/check.go): the ref merged into the trunk's head, the
|
||||
// trunk's merge-check.sh, each part in its toolchain's container, against a throwaway store and bus. It is a
|
||||
// check and nothing more:
|
||||
// `try <owner>/<repository> <ref>` asks the checker (mesh-delivery-checker's `checker_try`) to judge the commit the ref
|
||||
// names merged onto the trunk, as it judges a pull request's head: the build modules' fetch, digest and test steps,
|
||||
// the unit level and then the integration level, each level's kept verdicts taken. From R4e build-agent is asked no
|
||||
// try. It is a check and nothing more:
|
||||
//
|
||||
// - **no pull request, so no status**: its outcome is never said as `checked`, which is what the forge's
|
||||
// holder sets statuses from (isTry, in Built);
|
||||
// - **nothing registered, nothing built**: it names no module of the graph, so the merge gate does not run and
|
||||
// no build is made for a merge (novox/hq ADR 0350's scratch builds are a pull request's check's alone);
|
||||
// - **a check in the build queue**: after deliveries (link.KindOf, novox/hq issue 558), and bounded by the
|
||||
// check's own bound (checkTimeout);
|
||||
// - **the code a pull request's check would run anyway**: only a repository of the mesh, whose every pull
|
||||
// request the build seat already checks, in containers that hold no runtime socket.
|
||||
// - **no pull request, so no status**: the checker owes the forge nothing for a try;
|
||||
// - **nothing registered, nothing built for delivery**: a try writes no verdict of a group and no preview;
|
||||
// - **bounded**: one try of a head at a time — a second ask answers the first's id — and at most four open, the
|
||||
// checker's own rule; a repository the checker judges no level of (hq) is refused by the checker, and said.
|
||||
//
|
||||
// A check outlasts a seat verb's one answer, so the ask answers its id at once and `try --id <id>` reads what
|
||||
// came of it from the build's log, where the build seat says it whole (link.StepTried): holding its answer up
|
||||
// to tryWait for a verdict, like `plans wait` (novox/hq issue 229). Nothing is kept beside the log.
|
||||
// A check outlasts a seat verb's one answer, so the ask answers its id at once and `try --id <id>` reads what came of
|
||||
// it from the checker (`checker_try_read`), holding its answer up to tryWait for a verdict, like `plans wait`.
|
||||
|
||||
// tryWait is how long reading a try holds its answer for the verdict. A variable so a test need not wait.
|
||||
var tryWait = 20 * time.Second
|
||||
|
||||
// A ref as the forge is asked for it — a branch or a commit, never an option — and a `go test -run` pattern.
|
||||
var (
|
||||
tryID = regexp.MustCompile(`^try-[0-9]+$`)
|
||||
tryRef = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._/-]*$`)
|
||||
tryTests = regexp.MustCompile(`^[A-Za-z0-9_^$|().*+?/\[\]-]+$`)
|
||||
)
|
||||
@@ -77,34 +70,70 @@ func isTry(result link.BuildResult) bool {
|
||||
return c != nil && (c.Try || (c.Number == 0 && c.Group == ""))
|
||||
}
|
||||
|
||||
// asATry makes a check's ask a try's: the repository's own script alone, for no pull request.
|
||||
func asATry(request link.BuildRequest, tests string, storeOnly bool) link.BuildRequest {
|
||||
c := *request.Check
|
||||
c.Number, c.Try, c.Tests, c.StoreOnly = 0, true, tests, storeOnly
|
||||
c.Paths, c.Modules, c.New, c.Dependents, c.Manifests, c.Builds = nil, nil, nil, nil, nil, nil
|
||||
c.Judge, c.Plan, c.Group, c.Members = "", nil, "", nil
|
||||
request.Check = &c
|
||||
return request
|
||||
// checkerModule is the module whose tools judge a try (mesh-delivery-checker).
|
||||
const checkerModule = "mesh-delivery-checker"
|
||||
|
||||
// checkerTry is the checker's answer to a try's ask: the try's id.
|
||||
type checkerTry struct {
|
||||
ID string `json:"id"`
|
||||
}
|
||||
|
||||
// maxTries is how many tries may wait or run at once, mesh-wide: each holds a build machine for minutes.
|
||||
const maxTries = 4
|
||||
// checkerTryRead is the checker's word on a try (`checker_try_read`): its state queued, running, pass, fail or error
|
||||
// (could not run), what it said, the head and trunk judged, how long it took, what failed, and the machines it ran on.
|
||||
type checkerTryRead struct {
|
||||
ID string `json:"id"`
|
||||
Repository string `json:"repository"`
|
||||
Head string `json:"head"`
|
||||
Trunk string `json:"trunk"`
|
||||
Branch string `json:"branch,omitempty"`
|
||||
State string `json:"state"`
|
||||
Said string `json:"said"`
|
||||
Took string `json:"took,omitempty"`
|
||||
Failed []string `json:"failed,omitempty"`
|
||||
Machines []string `json:"machines,omitempty"`
|
||||
}
|
||||
|
||||
// triesIn reads the build queue for a new try of one commit: the id of a try of that commit already waiting or
|
||||
// running — the answer to a second ask — and whether there is room for another.
|
||||
func triesIn(q link.Queue, owner, repo, commit string) (same string, room bool) {
|
||||
n := 0
|
||||
for _, a := range q.Asks {
|
||||
if !link.IsTry(a.Request) || (a.State != link.AskWaiting && a.State != link.AskInFlight) {
|
||||
continue
|
||||
// askChecker asks one of the checker's tools and reads its result into into. A variable so a test needs no checker.
|
||||
var askChecker = func(ctx context.Context, tool string, args map[string]any, into any) error {
|
||||
js, err := aBus()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer js.Close()
|
||||
raw, _ := json.Marshal(args)
|
||||
answer, err := link.Ask(ctx, link.OverNATS{Conn: js.Conn()}, checkerModule, tool, raw, 30*time.Second)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s.%s: %w", checkerModule, tool, err)
|
||||
}
|
||||
if answer.Error != "" {
|
||||
return fmt.Errorf("%s.%s: %s", checkerModule, tool, answer.Error)
|
||||
}
|
||||
var wrapped struct {
|
||||
Result json.RawMessage `json:"result"`
|
||||
}
|
||||
if err := json.Unmarshal(answer.Result, &wrapped); err == nil && len(wrapped.Result) > 0 {
|
||||
return json.Unmarshal(wrapped.Result, into)
|
||||
}
|
||||
return json.Unmarshal(answer.Result, into)
|
||||
}
|
||||
|
||||
// tryOfChecker is the checker's word on a try as `try` answers it, today's answer unchanged (novox/hq issue 686): queued
|
||||
// is waiting, and the machines it ran on are where.
|
||||
func tryOfChecker(r checkerTryRead) tryAnswer {
|
||||
a := tryAnswer{ID: r.ID, State: r.State, Said: r.Said, Repository: r.Repository, Commit: r.Head, Trunk: r.Trunk,
|
||||
Took: r.Took, Failed: r.Failed, On: strings.Join(r.Machines, ", "), Base: r.Branch}
|
||||
switch r.State {
|
||||
case "queued":
|
||||
a.State = "waiting"
|
||||
if a.Said == "" {
|
||||
a.Said = "it waits for the checker; ask again"
|
||||
}
|
||||
n++
|
||||
c := a.Request.Check
|
||||
if same == "" && strings.EqualFold(c.Owner, owner) && strings.EqualFold(c.Repo, repo) && a.Request.Ref == commit {
|
||||
same = a.ID
|
||||
case "running":
|
||||
if a.Said == "" {
|
||||
a.Said = "still running; ask again"
|
||||
}
|
||||
}
|
||||
return same, n < maxTries
|
||||
return a
|
||||
}
|
||||
|
||||
// forgeCommit asks the forge, through its module's tools on the bus, which commit a branch or commit of a
|
||||
@@ -168,9 +197,9 @@ func tryCommand(ctx context.Context, args []string) error {
|
||||
if len(positionals) != 0 {
|
||||
return usage
|
||||
}
|
||||
// Before it is a token of a bus subject: an id, never a pattern.
|
||||
if !link.SafeBuildID(*id) {
|
||||
return fmt.Errorf("%q is not a try's id", *id)
|
||||
// An id, never a pattern: the checker's `try-<n>`.
|
||||
if !tryID.MatchString(*id) {
|
||||
return fmt.Errorf("%q is not a try's id, try-<number>", *id)
|
||||
}
|
||||
return tryRead(ctx, *id)
|
||||
}
|
||||
@@ -220,56 +249,33 @@ func tryCommand(ctx context.Context, args []string) error {
|
||||
return fmt.Errorf("%s/%s is no repository of the mesh — no module is built from it and its owner is not "+
|
||||
"the core's — and only the mesh's are tried on the build seat; nothing was asked", owner, repo)
|
||||
}
|
||||
request, err := checkRequestFor(ctx, open, p, checkScope{Mesh: true, From: mesh.From}, entries)
|
||||
if err != nil {
|
||||
return err
|
||||
// **Asked of the checker, never of build-agent** (ADR 0501 §5): the head merged onto the trunk, as a pull request's.
|
||||
askArgs := map[string]any{"repository": strings.ToLower(owner + "/" + repo), "head": commit, "trunk": trunk}
|
||||
if *tests != "" {
|
||||
askArgs["tests"] = *tests
|
||||
}
|
||||
request = asATry(request, *tests, *storeOnly)
|
||||
seat := buildSeatHeld(ctx)
|
||||
// **Bounded in number** (novox/hq issue 686): one try of a commit at a time, and maxTries in all.
|
||||
js, err := dialTheBus()
|
||||
if err != nil {
|
||||
return err
|
||||
if *storeOnly {
|
||||
askArgs["level"] = "integration"
|
||||
}
|
||||
queue, err := link.ReadQueue(ctx, js, seat)
|
||||
js.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
var asked checkerTry
|
||||
if err := askChecker(ctx, "checker_try", askArgs, &asked); err != nil {
|
||||
return fmt.Errorf("the checker did not take a try of %s/%s at %.12s; nothing was asked: %w", owner, repo, commit, err)
|
||||
}
|
||||
same, room := triesIn(queue, owner, repo, commit)
|
||||
if same != "" {
|
||||
return printTry(tryAnswer{ID: same, State: "asked", Repository: owner + "/" + repo, Ref: ref, Commit: commit,
|
||||
On: seat, Said: fmt.Sprintf("a try of %s/%s at %.12s already waits or runs, as %s: nothing new was asked. "+
|
||||
"Read it with try id=%s", owner, repo, commit, same, same)})
|
||||
}
|
||||
if !room {
|
||||
return fmt.Errorf("%d tries already wait or run on the build seat, the most at once (`queue` lists them); "+
|
||||
"nothing was asked — read one of yours, or ask again when one has ended", maxTries)
|
||||
}
|
||||
ask, err := askOverOn(seat)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer ask.Close()
|
||||
if err := ask.Ask(ctx, request); err != nil {
|
||||
return err
|
||||
}
|
||||
return printTry(tryAnswer{ID: request.ID, State: "asked", Repository: owner + "/" + repo, Ref: ref, Commit: commit,
|
||||
Base: trunk, On: seat, Said: fmt.Sprintf("asked %s to try %s/%s at %.12s (%s) merged into %s: its own "+
|
||||
"merge-check.sh, as a check in the build queue, bounded at %s; no status is set and nothing is registered. "+
|
||||
"Read it with try id=%s — a full run of this kind takes minutes", seat, owner, repo, commit, ref, trunk,
|
||||
checkTimeout, request.ID)})
|
||||
return printTry(tryAnswer{ID: asked.ID, State: "asked", Repository: owner + "/" + repo, Ref: ref, Commit: commit,
|
||||
Base: trunk, On: checkerModule, Said: fmt.Sprintf("asked %s to try %s/%s at %.12s (%s) merged into %s: the head "+
|
||||
"judged as a pull request's, unit level then integration level; no status is set and nothing is registered. "+
|
||||
"Read it with try id=%s", checkerModule, owner, repo, commit, ref, trunk, asked.ID)})
|
||||
}
|
||||
|
||||
// tryRead answers what came of a try, holding the answer up to tryWait for its verdict.
|
||||
func tryRead(ctx context.Context, id string) error {
|
||||
until := time.Now().Add(tryWait)
|
||||
for {
|
||||
lines, err := buildLines(ctx, id)
|
||||
if err != nil {
|
||||
return err
|
||||
var r checkerTryRead
|
||||
if err := askChecker(ctx, "checker_try_read", map[string]any{"id": id}, &r); err != nil {
|
||||
return printTry(tryAnswer{ID: id, State: "error", Said: "the checker's word on it cannot be read: " + err.Error()})
|
||||
}
|
||||
a := tryAnswerOf(id, lines)
|
||||
a := tryOfChecker(r)
|
||||
if (a.State != "waiting" && a.State != "running") || !time.Now().Add(2*time.Second).Before(until) {
|
||||
return printTry(a)
|
||||
}
|
||||
@@ -279,34 +285,6 @@ func tryRead(ctx context.Context, id string) error {
|
||||
}
|
||||
}
|
||||
|
||||
// tryAnswerOf reads a try's answer from what the build seat said of it: its outcome under StepTried; `failed`
|
||||
// for one that could not run — an error, never a pass; else how far it is.
|
||||
func tryAnswerOf(id string, lines []link.BuildLine) tryAnswer {
|
||||
a := tryAnswer{ID: id, State: "waiting", Said: "nothing is said of " + id + " yet: it waits in the build queue " +
|
||||
"(`queue` says where), or no try has that id; ask again"}
|
||||
for _, l := range lines {
|
||||
switch l.Step {
|
||||
case link.StepTried:
|
||||
var t link.Tried
|
||||
if err := json.Unmarshal([]byte(l.Message), &t); err != nil || t.Verdict == "" {
|
||||
return tryAnswer{ID: id, State: "error", Said: "the build seat's word on it cannot be read: " + l.Message}
|
||||
}
|
||||
return tryAnswer{ID: id, State: t.Verdict, Said: t.Summary, Took: t.Took, Commit: t.Head, Trunk: t.Trunk,
|
||||
Failed: t.Failed}
|
||||
case "failed":
|
||||
return tryAnswer{ID: id, State: "error", Said: "it could not run: " + l.Message}
|
||||
case "checked":
|
||||
// A check that is no try, or a build seat from before the verb: its verdict is not a try's.
|
||||
return tryAnswer{ID: id, State: "error", Said: "the build seat checked it and said no try's outcome — " +
|
||||
"it predates the verb (novox/hq issue 686), or " + id + " is no try: " + l.Message + "; `builds` with " +
|
||||
"the id says every line"}
|
||||
}
|
||||
a.State, a.Last = "running", l.Message
|
||||
a.Said = "still running; ask again. `builds` with the id says every line so far"
|
||||
}
|
||||
return a
|
||||
}
|
||||
|
||||
func printTry(a tryAnswer) error {
|
||||
fmt.Fprintln(os.Stderr, a.Said)
|
||||
for _, l := range a.Failed {
|
||||
@@ -319,35 +297,3 @@ func printTry(a tryAnswer) error {
|
||||
fmt.Println(string(out))
|
||||
return nil
|
||||
}
|
||||
|
||||
// buildLines is everything the build seat said of one build, as the bus keeps it, in order.
|
||||
func buildLines(ctx context.Context, id string) ([]link.BuildLine, error) {
|
||||
js, err := aBus()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot reach the bus to read a build's log: %w", err)
|
||||
}
|
||||
defer js.Close()
|
||||
subject := link.BuildLogOf("*", id)
|
||||
sub, err := js.Context().PullSubscribe(subject, "",
|
||||
nats.BindStream(broker.EventsStream), nats.DeliverAll(), nats.AckNone())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot read %s from the bus: %w", subject, err)
|
||||
}
|
||||
defer func() { _ = sub.Unsubscribe() }()
|
||||
var lines []link.BuildLine
|
||||
for {
|
||||
batch, err := sub.Fetch(200, nats.MaxWait(2*time.Second))
|
||||
if err != nil && !errors.Is(err, nats.ErrTimeout) && !errors.Is(err, context.DeadlineExceeded) {
|
||||
return nil, fmt.Errorf("reading a build's log: %w", err)
|
||||
}
|
||||
for _, msg := range batch {
|
||||
var line link.BuildLine
|
||||
if json.Unmarshal(msg.Data, &line) == nil {
|
||||
lines = append(lines, line)
|
||||
}
|
||||
}
|
||||
if len(batch) < 200 {
|
||||
return lines, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,8 +4,13 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
@@ -37,3 +42,37 @@ func TestATrysOutcomeSetsNoStatusAndRegistersNothing(t *testing.T) {
|
||||
t.Fatalf("a pull request's check was said %d time(s)", len(said.events))
|
||||
}
|
||||
}
|
||||
|
||||
// ADR 0501 §5 (issue 1851): a try is asked of the checker, the head merged onto the trunk, never of build-agent:
|
||||
// --store-only is the integration level, --tests passes on, the ask answers the checker's id.
|
||||
func TestATryIsAskedOfTheChecker(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
noBuildAsked(t)
|
||||
if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "sudo", Version: "1"},
|
||||
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/sudo", Ref: "main",
|
||||
BuiltFrom: "c0", Head: "c0"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wasForge, wasChecker := forgeCommit, askChecker
|
||||
forgeCommit = func(context.Context, string, string, string) (string, string, error) {
|
||||
return strings.Repeat("c", 40), "main", nil
|
||||
}
|
||||
var asked []map[string]any
|
||||
askChecker = func(_ context.Context, tool string, args map[string]any, into any) error {
|
||||
if tool != "checker_try" {
|
||||
t.Errorf("asked %s", tool)
|
||||
}
|
||||
asked = append(asked, args)
|
||||
return json.Unmarshal([]byte(`{"id":"try-1"}`), into)
|
||||
}
|
||||
t.Cleanup(func() { forgeCommit, askChecker = wasForge, wasChecker })
|
||||
if err := tryCommand(ctx, []string{"novox/Mesh-Catalog", "fix/x", "--store-only", "--tests", "^TestX$"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := map[string]any{"repository": "novox/mesh-catalog", "head": strings.Repeat("c", 40), "trunk": "main",
|
||||
"level": "integration", "tests": "^TestX$"}
|
||||
if len(asked) != 1 || !reflect.DeepEqual(asked[0], want) {
|
||||
t.Fatalf("the checker was asked %v, want %v", asked, want)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,29 +15,6 @@ import (
|
||||
// A branch is tried before its pull request (novox/hq issue 686): the repository's own check asked for a pushed
|
||||
// branch or commit, with no pull request.
|
||||
|
||||
// What a try asks is a check — after deliveries in the build queue, bounded as one — of the repository's own
|
||||
// script alone: no module of the graph, so no gate; nothing built for a merge, so no reusable image.
|
||||
func TestATryAsksACheckOfTheRepositorysOwnScriptAlone(t *testing.T) {
|
||||
request := link.BuildRequest{ID: "build-1", Ref: "c0ffee", Check: &link.CheckRequest{Owner: "novox", Repo: "a",
|
||||
Number: 7, Base: "main", Modules: []string{"a"}, New: []string{"b"}, Manifests: []string{"module.json"},
|
||||
Dependents: []string{"c"}, Builds: []link.CheckBuild{{Module: "a"}}, Judge: link.JudgeSelf,
|
||||
Plan: &link.ChangePlan{}}}
|
||||
tried := asATry(request, "^TestStore", true)
|
||||
c := tried.Check
|
||||
if !c.Try || c.Number != 0 || c.Tests != "^TestStore" || !c.StoreOnly {
|
||||
t.Fatalf("not marked a try, or it kept a pull request's number: %+v", c)
|
||||
}
|
||||
if len(c.Modules)+len(c.New)+len(c.Manifests)+len(c.Dependents)+len(c.Builds) != 0 || c.Judge != "" || c.Plan != nil {
|
||||
t.Fatalf("a try runs the gate or builds for a merge: %+v", c)
|
||||
}
|
||||
if c.Base != "main" || c.Owner != "novox" || tried.Ref != "c0ffee" {
|
||||
t.Fatalf("a try lost what it merges into what: %+v", tried)
|
||||
}
|
||||
if link.KindOf(tried) != link.KindCheck {
|
||||
t.Fatal("a try does not count as a check in the build queue")
|
||||
}
|
||||
}
|
||||
|
||||
// A try's outcome is heard and left there: no `checked` is said, so the forge's holder sets no status, and
|
||||
// nothing is recorded or registered (the handler holds no store here, and would fail reaching for one).
|
||||
|
||||
@@ -45,10 +22,12 @@ func TestATryAsksACheckOfTheRepositorysOwnScriptAlone(t *testing.T) {
|
||||
type saidOnTheBus struct {
|
||||
link.Bus
|
||||
events []string
|
||||
bodies [][]byte
|
||||
}
|
||||
|
||||
func (s *saidOnTheBus) PublishSeatEvent(_ context.Context, seat, event string, _ []byte) error {
|
||||
func (s *saidOnTheBus) PublishSeatEvent(_ context.Context, seat, event string, body []byte) error {
|
||||
s.events = append(s.events, seat+" "+event)
|
||||
s.bodies = append(s.bodies, body)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -76,58 +55,38 @@ func TestATryOfAnUnknownRefIsRefusedLoudly(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// The answer of a try carries the lines that name what failed (novox/hq issue 460's shape) and how long it took.
|
||||
func TestATrysAnswerCarriesTheFailingLines(t *testing.T) {
|
||||
tried, _ := json.Marshal(link.Tried{Verdict: "fail", Summary: "its merge-check.sh failed: TestX", Took: "5m3s",
|
||||
Head: "c0ffee", Trunk: "ca3f43c", Failed: []string{"--- FAIL: TestX (0.01s)", " x_test.go:12: got 1, want 2",
|
||||
"FAIL\tgithub.com/novox/a/cmd\t3.1s"}})
|
||||
lines := []link.BuildLine{{Seq: 1, Step: "check", Message: "novox/a#0 at c0ffee"},
|
||||
{Seq: 2, Step: "output", Message: "ok github.com/novox/a/internal 1.2s"},
|
||||
{Seq: 3, Step: link.StepTried, Message: string(tried)}, {Seq: 4, Step: "checked", Message: "pass: x"}}
|
||||
a := tryAnswerOf("build-1", lines)
|
||||
if a.State != "fail" || a.Took != "5m3s" || len(a.Failed) != 3 || a.Failed[0] != "--- FAIL: TestX (0.01s)" ||
|
||||
a.Trunk != "ca3f43c" || a.ID != "build-1" {
|
||||
t.Fatalf("the answer lost the verdict, the time or the failing lines: %+v", a)
|
||||
// The answer of a try is the checker's word, in today's shape (ADR 0501 §5; issue 1851): its verdict, the failing
|
||||
// line, how long it took and where; queued is waiting; one that could not run is an error, never a pass.
|
||||
func TestATrysAnswerIsTheCheckersWord(t *testing.T) {
|
||||
var asked []string
|
||||
was := askChecker
|
||||
askChecker = func(_ context.Context, tool string, args map[string]any, into any) error {
|
||||
asked = append(asked, tool)
|
||||
raw, _ := json.Marshal(checkerTryRead{ID: "try-1", Repository: "novox/a", Head: "c0ffee", Trunk: "ca3f43c",
|
||||
State: "fail", Said: "the unit level failed: TestX", Took: "5m3s",
|
||||
Failed: []string{"unit: a/cmd.TestX: got 1, want 2"}, Machines: []string{"ace"}})
|
||||
return json.Unmarshal(raw, into)
|
||||
}
|
||||
// Not answered yet: said as running, with how far it is — never as a pass.
|
||||
if a := tryAnswerOf("build-1", lines[:2]); a.State != "running" || a.Last == "" {
|
||||
t.Fatalf("a try still running: %+v", a)
|
||||
t.Cleanup(func() { askChecker = was })
|
||||
if err := tryRead(t.Context(), "try-1"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if a := tryAnswerOf("build-1", nil); a.State != "waiting" {
|
||||
t.Fatalf("a try nothing was said of: %+v", a)
|
||||
if len(asked) != 1 || asked[0] != "checker_try_read" {
|
||||
t.Fatalf("a try is read from %v, not the checker", asked)
|
||||
}
|
||||
// One that could not run is an error with the build seat's word, never a pass.
|
||||
if a := tryAnswerOf("build-1", append(lines[:1:1], link.BuildLine{Seq: 2, Step: "failed",
|
||||
Message: "cannot clone"})); a.State != "error" || !strings.Contains(a.Said, "cannot clone") {
|
||||
a := tryOfChecker(checkerTryRead{ID: "try-1", State: "fail", Took: "5m3s", Head: "c0ffee", Trunk: "ca3f43c",
|
||||
Failed: []string{"unit: a/cmd.TestX: got 1, want 2"}, Machines: []string{"ace"}})
|
||||
if a.State != "fail" || a.Took != "5m3s" || len(a.Failed) != 1 || a.Commit != "c0ffee" || a.Trunk != "ca3f43c" || a.On != "ace" {
|
||||
t.Fatalf("the answer lost the verdict, the time, the failing line or where: %+v", a)
|
||||
}
|
||||
if a := tryOfChecker(checkerTryRead{ID: "try-2", State: "queued"}); a.State != "waiting" || a.Said == "" {
|
||||
t.Fatalf("a try the checker has not started: %+v", a)
|
||||
}
|
||||
if a := tryOfChecker(checkerTryRead{ID: "try-3", State: "error", Said: "no node holds the go build module"}); a.State != "error" {
|
||||
t.Fatalf("a try that could not run: %+v", a)
|
||||
}
|
||||
}
|
||||
|
||||
// One try of a commit at a time, and four in all (novox/hq issue 686): a second ask is answered the first's id,
|
||||
// a fifth is refused; a try that ended, and a pull request's check, count for nothing.
|
||||
func TestTriesAreBoundedInNumber(t *testing.T) {
|
||||
ask := func(id, repo, ref, state string, try bool) link.QueuedAsk {
|
||||
return link.QueuedAsk{ID: id, State: state, Request: link.BuildRequest{ID: id, Ref: ref,
|
||||
Check: &link.CheckRequest{Owner: "novox", Repo: repo, Try: try}}}
|
||||
}
|
||||
q := link.Queue{Asks: []link.QueuedAsk{
|
||||
ask("t1", "a", "c1", link.AskInFlight, true), ask("t2", "a", "c2", link.AskWaiting, true),
|
||||
ask("t3", "b", "c1", link.AskDead, true), ask("p1", "a", "c9", link.AskWaiting, false)}}
|
||||
if same, room := triesIn(q, "novox", "a", "c2"); same != "t2" || !room {
|
||||
t.Fatalf("a second try of a commit: %q %v", same, room)
|
||||
}
|
||||
if same, room := triesIn(q, "novox", "b", "c1"); same != "" || !room {
|
||||
t.Fatalf("a dead try, or another repository's, was taken for this one: %q %v", same, room)
|
||||
}
|
||||
if same, _ := triesIn(q, "novox", "a", "c9"); same != "" {
|
||||
t.Fatalf("a pull request's check was taken for a try: %q", same)
|
||||
}
|
||||
q.Asks = append(q.Asks, ask("t4", "c", "c3", link.AskWaiting, true), ask("t5", "d", "c4", link.AskWaiting, true))
|
||||
if _, room := triesIn(q, "novox", "e", "c5"); room {
|
||||
t.Fatal("a fifth try was let in")
|
||||
}
|
||||
// An id is checked before it is a token of a bus subject.
|
||||
for _, id := range []string{"x.>", "*", "a b"} {
|
||||
// An id is checked before it is asked of anybody.
|
||||
for _, id := range []string{"x.>", "*", "a b", "build-1"} {
|
||||
if err := tryCommand(context.Background(), []string{"--id", id}); err == nil ||
|
||||
!strings.Contains(err.Error(), "not a try's id") {
|
||||
t.Errorf("try --id %q: %v", id, err)
|
||||
|
||||
+17
-156
@@ -371,12 +371,17 @@ func movesOfMerge(ctx context.Context, inv *inventory.Inventory, m link.SourceMo
|
||||
}
|
||||
// **A new module is built and registered, and sent nowhere** (novox/hq issue 300): the delivery plan
|
||||
// says so, and assigning it is a person's act, which needs it registered first.
|
||||
built := askNewModules(ctx, inv, m, from, added)
|
||||
// Nothing asks its first build (novox/hq ADR 0501 §6): the builder builds every fingerprint the indexer writes, and
|
||||
// the controller registers the module from its first done key (new_modules.go).
|
||||
var built []string
|
||||
if len(from) > 0 {
|
||||
built = added
|
||||
}
|
||||
moved := append(append([]inventory.Entry{}, touched...), packaging...)
|
||||
if len(moved) == 0 {
|
||||
if len(built) > 0 {
|
||||
fmt.Printf("%s/%s merged into %s (%.8s); it changed no module the mesh holds, and adds %s: "+
|
||||
"built, registered when the build lands, and sent nowhere\n", m.Owner, m.Repo, m.Base, m.Commit,
|
||||
"registered from the builder's done key of each, and sent nowhere\n", m.Owner, m.Repo, m.Base, m.Commit,
|
||||
strings.Join(built, ", "))
|
||||
return nil, nil
|
||||
}
|
||||
@@ -399,116 +404,6 @@ func movesOfMerge(ctx context.Context, inv *inventory.Inventory, m link.SourceMo
|
||||
return names, nil
|
||||
}
|
||||
|
||||
// askNewModules asks the build seat for every module a merge adds to the repository — a directory holding a
|
||||
// module.json that no module of the mesh is known from (touchedBy's `added`) — at the branch merged into,
|
||||
// not waited for: the build's take-in registers it, as a hand `build` does, and sends it nowhere, since no
|
||||
// machine is assigned it (novox/hq issue 300). Before this a merge said "it changed nothing any module
|
||||
// the mesh holds is built from", the module was never registered, and `assign` refused it as unknown.
|
||||
//
|
||||
// The repository is spelled, and found on the seat, as the modules already built from it are; `from` is
|
||||
// those of them this merge moves, so a merge read as history, or of a branch nothing follows, builds no
|
||||
// new module either. Outside the plan: the plan walks modules the catalogue holds, and a new one has no
|
||||
// machine to send to and nothing standing on it. What could not be asked is said, and left to `build`.
|
||||
// Returns the directories asked for.
|
||||
func askNewModules(ctx context.Context, inv *inventory.Inventory, m link.SourceMoved, from []inventory.Entry,
|
||||
added []string) []string {
|
||||
if len(added) == 0 || len(from) == 0 {
|
||||
return nil
|
||||
}
|
||||
source := buildSource{Repository: from[0].Source.Repository, Seat: from[0].Source.Seat}
|
||||
// Asked once (novox/hq issue 1617): a module is asked when its merge is heard, and again here when the batch is cut.
|
||||
var requests []inventory.RequestOutcome
|
||||
for _, dir := range added {
|
||||
if got, err := inv.RequestsNamed(ctx, inventory.BuildRequest{Repository: source.Repository, Path: dir}.Name()); err == nil {
|
||||
requests = append(requests, got...)
|
||||
}
|
||||
}
|
||||
return askBuilds(ctx, inv, m, source, unaskedDirs(added, requests, source, time.Now()))
|
||||
}
|
||||
|
||||
// askedInFlightFor is how long a merge's build request stands as the build coming for its module: past it, a lost one
|
||||
// is asked again.
|
||||
const askedInFlightFor = time.Hour
|
||||
|
||||
// unaskedDirs are the directories of a new module no build of a merge is on its way for: a kept request of the same
|
||||
// repository and directory, asked for a merge and handed over, whose outcome has not come and is not given up on,
|
||||
// is the build coming (novox/hq issue 1617).
|
||||
func unaskedDirs(added []string, requests []inventory.RequestOutcome, source buildSource, now time.Time) []string {
|
||||
var out []string
|
||||
for _, dir := range added {
|
||||
path := dir
|
||||
if path == "." {
|
||||
path = ""
|
||||
}
|
||||
coming := false
|
||||
for _, r := range requests {
|
||||
if r.For == "merge" && r.Repository == source.Repository && strings.Trim(r.Path, "/") == path &&
|
||||
r.NotAsked == "" && r.OutcomeUnknown == "" && !r.Heard && now.Sub(r.At) < askedInFlightFor {
|
||||
coming = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !coming {
|
||||
out = append(out, dir)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// askNewModulesAtHearing asks the first build of the modules a merge adds to a repository the mesh builds modules from,
|
||||
// when the merge is heard, not when its batch is cut (novox/hq issue 1617): a batch waits out its window and the walk
|
||||
// before it, and a module added by a merge delivered meanwhile could not be assigned. Outside the batch and its plan.
|
||||
func askNewModulesAtHearing(ctx context.Context, inv *inventory.Inventory, m link.SourceMoved, entries []inventory.Entry,
|
||||
read map[string][]inventory.ReadRepository, from []inventory.Entry) {
|
||||
if len(from) == 0 {
|
||||
return
|
||||
}
|
||||
_, added, _ := touchedBy(from, entries, m, read)
|
||||
if asked := askNewModules(ctx, inv, m, from, added); len(asked) > 0 {
|
||||
fmt.Printf("%s/%s merged into %s (%.8s) adds %s: the first build is asked now, before its batch is cut\n",
|
||||
m.Owner, m.Repo, m.Base, m.Commit, strings.Join(asked, ", "))
|
||||
}
|
||||
}
|
||||
|
||||
// askBuilds asks the first build of each directory of a merge's repository, as askNewModules does, from the source given.
|
||||
func askBuilds(ctx context.Context, inv *inventory.Inventory, m link.SourceMoved, source buildSource,
|
||||
added []string) []string {
|
||||
var asked []string
|
||||
for _, dir := range added {
|
||||
path := dir
|
||||
if path == "." {
|
||||
path = ""
|
||||
}
|
||||
id, err := askABuild(ctx, source, path, m.Base)
|
||||
// Kept, asked or not, so `assign` can tell a build in flight, or a merge that could not ask for one,
|
||||
// from a module nobody ever heard of (novox/hq issue 325).
|
||||
request := inventory.BuildRequest{ID: id, Repository: source.Repository, Seat: source.Seat, Path: path,
|
||||
Ref: m.Base, Commit: m.Commit, For: "merge"}
|
||||
if err != nil {
|
||||
request.ID = notAskedID(m.Commit, dir)
|
||||
request.NotAsked = err.Error()
|
||||
recordBuildRequest(ctx, inv, request)
|
||||
fmt.Printf(" %s is a new module in %s/%s and could not be built: %v — a hand `build` of it "+
|
||||
"asks again\n", dir, m.Owner, m.Repo, err)
|
||||
continue
|
||||
}
|
||||
recordBuildRequest(ctx, inv, request)
|
||||
fmt.Printf(" %s is a new module in %s/%s: build %s asked at %s; registered when it lands, assigned "+
|
||||
"nowhere\n", dir, m.Owner, m.Repo, id, m.Base)
|
||||
asked = append(asked, dir)
|
||||
}
|
||||
return asked
|
||||
}
|
||||
|
||||
// notAskedID is the id a merge's build that could not be asked is kept under: a key the bus takes, so a module at the
|
||||
// repository's root (".") is kept as one in a folder is.
|
||||
func notAskedID(commit, dir string) string {
|
||||
if dir == "." || dir == "" {
|
||||
return "not-asked-" + short(commit)
|
||||
}
|
||||
return fmt.Sprintf("not-asked-%s-%s", short(commit), strings.ReplaceAll(strings.Trim(dir, "/"), "/", "-"))
|
||||
}
|
||||
|
||||
// actingOnMerges keeps one merge acted on at a time (novox/hq issue 266).
|
||||
var actingOnMerges sync.Mutex
|
||||
|
||||
@@ -1181,50 +1076,6 @@ func anyInside(paths []string, dir string) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// orderByBases is the entries with every base before what stands on it: a module whose build stood
|
||||
// on another's artifact comes after that module. Entries outside the set are not waited for — they
|
||||
// are not being rebuilt. Stable for what has no order between it.
|
||||
//
|
||||
// `against` is what each module's newest build stood on (inventory.BuiltAgainst): the edges are
|
||||
// derived from builds, not declared, because a recorded manifest no longer carries `build.on`.
|
||||
func orderByBases(entries []inventory.Entry, against map[string][]string) []inventory.Entry {
|
||||
inSet := map[string]bool{}
|
||||
for _, e := range entries {
|
||||
inSet[e.Manifest.Module] = true
|
||||
}
|
||||
var out []inventory.Entry
|
||||
placed := map[string]bool{}
|
||||
var place func(e inventory.Entry, seen map[string]bool)
|
||||
place = func(e inventory.Entry, seen map[string]bool) {
|
||||
name := e.Manifest.Module
|
||||
if placed[name] || seen[name] {
|
||||
return
|
||||
}
|
||||
seen[name] = true
|
||||
for _, base := range entries {
|
||||
if base.Manifest.Module != name && inSet[base.Manifest.Module] && standsOnModule(e, base.Manifest.Module, against) {
|
||||
place(base, seen)
|
||||
}
|
||||
}
|
||||
placed[name] = true
|
||||
out = append(out, e)
|
||||
}
|
||||
for _, e := range entries {
|
||||
place(e, map[string]bool{})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// standsOn is whether anything in the set is built on the named module's artifacts.
|
||||
func standsOn(entries []inventory.Entry, module string, against map[string][]string) bool {
|
||||
for _, e := range entries {
|
||||
if standsOnModule(e, module, against) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// standsOnModule is whether an entry's build stood on the named module: by what its newest build
|
||||
// recorded it was handed (`artifact-store://<module>/<artifact>@…`, the module's own artifact), or
|
||||
// — for a module registered from a manifest and not yet built — by the base its manifest names.
|
||||
@@ -1248,6 +1099,16 @@ func standsOnModule(e inventory.Entry, module string, against map[string][]strin
|
||||
return false
|
||||
}
|
||||
|
||||
// standsOn is whether anything in the set is built on the named module's artifacts.
|
||||
func standsOn(entries []inventory.Entry, module string, against map[string][]string) bool {
|
||||
for _, e := range entries {
|
||||
if standsOnModule(e, module, against) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// isHistory is whether a merge made at mergedAt predates the last time the source was seen. A merge
|
||||
// with no time on it is taken as news: refusing it would silence a forge that says less.
|
||||
func isHistory(mergedAt string, seen time.Time) bool {
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user