//go:build integration package main import ( "context" "encoding/json" "errors" "fmt" "path" "slices" "strings" "time" "git.novox.be/novox/mesh-sdk/go/catalogue" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" ) // What the tests below take a build in with (R4g): the controller itself registers a module only from the builder's done // key now (takeInKey), but a great many of these tests only need a module registered with a build recorded, in the form // the old outcome carried, so that is kept here and nowhere else. It is the controller's take-in of a build-agent outcome // as it was, less the checks only an outcome needed. Nothing in the controller calls any of it. // buildFrom turns what a builder said into what the mesh keeps. // // **By digest and path, never by where it was pushed** (novox/hq 04-ISSUES/102). The builder // says `://@sha256:…`; the mesh records the artifact-store // reference and composes the store's address back in where a reference is used. `against` — what // the build stood on, the catalogue's edge — is recorded the same way, so an edge names a module's // artifact and not the machine it was pulled from. func buildFrom(result link.BuildResult) inventory.Build { kept := inventory.Build{ ID: result.ID, Repository: result.Repository, Ref: result.Ref, Commit: result.Commit, On: result.On, Failed: result.Failed, // **What the announcement carries, kept rather than discarded** (novox/hq 04-ISSUES/050). // The catalogue turns the manifest into requires/provides edges and `against` into build // edges, and it is not always listening when a build happens — on a fresh mesh it cannot // be, for exactly the modules it needs most. Keeping them is what makes a replay able to // rebuild the graph rather than a list of names. Path: result.Path, // What it was made from (novox/hq issue 280): two builds with one are one build. SourceFingerprint: result.SourceFingerprint, SourceInputs: result.SourceInputs, } // When it was asked, which is what orders it against another build of the same module // (novox/hq 04-ISSUES/219) — not when it was heard. if asked, ok := link.BuildAskedAt(result.ID); ok { kept.Asked = asked } for _, ref := range result.Against { kept.Against = append(kept.Against, catalogue.Recorded(ref)) } for _, ref := range result.Mirrored { kept.Mirrored = append(kept.Mirrored, catalogue.Recorded(ref)) } for _, r := range result.Read { kept.Read = append(kept.Read, inventory.ReadRepository{Repository: r.Repository, Ref: r.Ref}) } // What it was made from, as files (novox/hq ADR 0267): the planner maps the next merge onto it. for _, s := range result.Sources { kept.Sources = append(kept.Sources, inventory.BuildSource{Repository: s.Repository, Ref: s.Ref, Paths: s.Paths}) } var announced []inventory.Artifact for _, made := range result.Made { announced = append(announced, inventory.Artifact{ Name: made.Name, Kind: made.Kind, Reference: made.Reference, }) kept.Made = append(kept.Made, inventory.Artifact{ Name: made.Name, Kind: made.Kind, Reference: catalogue.Recorded(made.Reference), }) } kept.Manifest = recordedManifest(result.Manifest, announced) // The module name comes from the manifest, which only exists when the build got that far. if len(kept.Manifest) > 0 { if list, err := judgedWith(result); err == nil { if m, err := catalogue.ParseManifestWith(kept.Manifest, list); err == nil { kept.Module = m.Module } } } return kept } // recordedManifest is a build's manifest with the store's address taken off every reference the // build itself made. Other references — an image a module runs from a public registry — are what // they were, which is why this rewrites only what `made` names rather than everything that looks // like an address. func recordedManifest(raw json.RawMessage, made []inventory.Artifact) json.RawMessage { announced := map[string]bool{} for _, a := range made { announced[a.Reference] = true } return withReferences(raw, func(ref string) (string, bool) { if !announced[ref] { return ref, false } return catalogue.Recorded(ref), true }) } // takeIn is what the mesh does with a build's outcome, whoever hears it: the waiting command and // the daemon that follows the role's events both come here (novox/hq issue 176), so a build's // result reaches the catalogue whether or not the asker was still listening. // // Kept before it is judged. A failed build that leaves no trace is indistinguishable from one // nobody asked for, and the difference is the whole of whether somebody should be looking at // something. Then parsed with the same parser a hand-written manifest goes through — a second path // would be a second thing to disagree about what a manifest is — and registered with where it came // from: **for a source on a seat, as the path and the seat, never the URL just cloned** (ADR 0111), // which the request carried and the outcome echoes. A definition naming an installation is refused // here, where it would enter the catalogue; the build stays recorded and the refusal says which. // // Idempotent: the same outcome taken in twice registers the same module twice, which is one row // written with the same values. func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResult) ( catalogue.Manifest, inventory.Build, error) { // The list of predefined keys as the store keeps it now: another controller process may have kept a newer one. if err := loadPredefined(ctx, inv); err != nil { return catalogue.Manifest{}, inventory.Build{}, fmt.Errorf("the list of predefined keys kept cannot be "+ "read: %w", err) } kept := buildFrom(result) if err := inv.RecordBuild(ctx, kept); err != nil { return catalogue.Manifest{}, kept, err } if result.Failed != "" { // The builder's own words. Wrapping them in something about the control plane would put // two explanations between a person and a build log. return catalogue.Manifest{}, kept, fmt.Errorf("%s could not build %s:\n%s", result.On, result.Repository, result.Failed) } // Judged first against the list the build was judged against: the catalogue's at the commit built (novox/hq // ADR 0348 §7), so a module naming a key its own commit adds is read; and again below against the list the // mesh publishes, before it is registered. list, err := judgedWith(result) if err != nil { return catalogue.Manifest{}, kept, fmt.Errorf("%s built %s (%s), recorded and not registered: %w", result.On, result.Repository, short(result.Commit), err) } manifest, err := catalogue.ParseManifestWith(kept.Manifest, list) if err != nil { return catalogue.Manifest{}, kept, fmt.Errorf("%s built %s and what came back is not a manifest: %w", result.On, result.Repository, err) } recorded := inventory.Source{ Repository: result.Repository, Path: result.Path, Ref: result.Ref, BuiltFrom: result.Commit, Head: result.Commit, // What it stood on, so registration can judge a built manifest's base (to-be 38 WP2.4). Against: kept.Against, // When it was asked, so an older request heard later does not replace a newer one // (novox/hq 04-ISSUES/219). Asked: kept.Asked, } if result.Source != nil && result.Source.Seat != "" { recorded.Repository, recorded.Seat = result.Source.Repository, result.Source.Seat } // **A build at a commit does not change the branch a module follows** (novox/hq 04-ISSUES/215): // the commit is built and recorded as what it was built from, and the module keeps following // what it followed before — the repository's default branch for one new to the catalogue. if followedBranch(result.Ref) == "" && result.Ref != "" { recorded.Ref = "" if was, err := inv.SourceOf(ctx, manifest.Module); err == nil { recorded.Ref = followedBranch(was.Ref) } } trunk := result.Trunk if was, err := inv.SourceOf(ctx, manifest.Module); err == nil && followedBranch(was.Ref) != "" { trunk = followedBranch(was.Ref) } if trunk == "" { trunk = "main" } j := takenBuild{kept: kept, manifest: manifest, recorded: recorded, ask: registrationAsk{Build: result.ID}, path: result.Path, trunk: trunk, by: result.On, what: result.Repository, commit: result.Commit} // The checks only an outcome of build-agent needs: a key is a trunk build by construction (ADR 0501 §2), in no // scratch namespace, and carries no list of its own. j.published = func() error { // **Only a commit on the trunk is published** (novox/hq ADR 0238): a commit off its repository's // default branch — a pull request's head, a feature branch built by hand, a `rebuild` or `replay // --register` of one — is for checking, and is never a module's version; nothing could then send it. // The branch the module already follows is its trunk — never the branch a build was asked at, or a // build of a feature branch by name would make that branch its trunk. follows := "" if was, err := inv.SourceOf(ctx, manifest.Module); err == nil { follows = followedBranch(was.Ref) } if err := publishable(result, follows); err != nil { return err } return nil } j.carried = func() error { // **The newest trunk build's list of predefined keys is published** (novox/hq ADR 0348 §7): a build of the // catalogue that may be registered keeps its list when its commit is newer than the one kept. carried, err := carriedList(result) if err != nil { return err } if carried != nil { if err := publishPredefined(ctx, inv, result, carried); err != nil { fmt.Printf("%s: the list of predefined keys it carried is not published: %v\n", result.ID, err) } } return nil } manifest, err = registerJudged(ctx, inv, j) return manifest, kept, err } // errOffTheTrunk is a build of a commit off its repository's trunk, which is never published. var errOffTheTrunk = errors.New("the commit is not on its repository's trunk: a commit off the trunk is checked, " + "never published (ADR 0238) — merge it, and the merge builds it") // publishable says whether a build's outcome may become a module's version: its commit on the module's // trunk, as the build seat read the forge at the build — the branch the module follows when its source // names one, else its repository's default branch. A build seat that could not say — one older than the // rule, building its own successor — is let through and said, so the rule can reach the mesh. func publishable(result link.BuildResult, follows string) error { if result.DryRun { return errors.New("a dry run is never published") } if result.Trunk == "" { fmt.Printf("%s: the build seat did not say whether %s is on its repository's trunk (it predates ADR 0238); "+ "registered as before\n", result.ID, short(result.Commit)) return nil } trunk := result.Trunk if follows != "" { trunk = follows } on := slices.Contains(result.Branches, trunk) || (trunk == result.Trunk && result.OnTrunk) if !on { return fmt.Errorf("%w (%s is not on %s)", errOffTheTrunk, short(result.Commit), trunk) } return nil } // fromTheCatalogue is whether a build is of the catalogue's repository, whose list alone the mesh publishes: by // its name, in the URL form or the seat form. func fromTheCatalogue(result link.BuildResult) bool { for _, repository := range []string{result.Repository, sourceRepository(result)} { name := strings.TrimSuffix(path.Base(strings.TrimRight(repository, "/")), ".git") if name == catalogue.CatalogueRepository { return true } } return false } func sourceRepository(result link.BuildResult) string { if result.Source == nil { return "" } return result.Source.Repository } // carriedList is the list a build of the catalogue carried, read strictly; nil for a build of another // repository, or one that carried none. func carriedList(result link.BuildResult) (catalogue.PredefinedList, error) { if result.Predefined == nil || !fromTheCatalogue(result) { return nil, nil } list, err := catalogue.ParsePredefined(result.Predefined.Keys) if err != nil { return nil, fmt.Errorf("%s carried a list of predefined keys that is refused: %w", result.ID, err) } return list, nil } // judgedWith is the list a build's manifest is first judged against at registration: the catalogue's own at the // commit built, which its build judged it against, where it carried one; the list kept otherwise. func judgedWith(result link.BuildResult) (catalogue.PredefinedList, error) { list, err := carriedList(result) if err != nil || list != nil { return list, err } return catalogue.PublishedPredefined(), nil } // publishPredefined keeps a trunk build's list when it is newer than the one kept, says so, and loads what the // store keeps then — this build's, or a newer one another controller process kept. func publishPredefined(ctx context.Context, inv *inventory.Inventory, result link.BuildResult, list catalogue.PredefinedList) error { kept, err := inv.KeepPredefined(ctx, inventory.PredefinedRecord{Keys: list, Repository: result.Repository, Commit: result.Commit, Generation: result.Predefined.Generation, Build: result.ID}) if err != nil { return err } if kept { fmt.Printf("%s: the list of predefined keys at %s (generation %d) is published: %s\n", result.ID, short(result.Commit), result.Predefined.Generation, strings.Join(list.Keys(), ", ")) } else { fmt.Printf("%s: the list of predefined keys at %s (generation %d) is not newer than the one kept, which "+ "stands\n", result.ID, short(result.Commit), result.Predefined.Generation) } return loadPredefined(ctx, inv) } // builds is what the tests of a pending assignment feed an outcome to: the take-in above, then the tick that settles what // waited for it (settlePending), as the daemon's hearing of an outcome did before R4g (it also moved the plans, which the // walk now does from its keys). Nothing in the controller calls it. type builds struct { inv *inventory.Inventory open *stores } func (b builds) Built(ctx context.Context, result link.BuildResult) error { _, _, err := takeIn(ctx, b.inv, result) if b.open != nil { settlePending(ctx, b.open, time.Now()) } if result.Failed != "" { return nil // the build is recorded; whether it registered is what the tests read } return err }