mesh-controller/cmd/mesh-controller/legacy_takein_integration_test.go

//go:build integration

package main

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"path"
	"slices"
	"strings"
	"time"

	"git.novox.be/novox/mesh-sdk/go/catalogue"
	"github.com/novox/mesh-controller/internal/inventory"
	"github.com/novox/mesh-controller/internal/link"
)

// What the tests below take a build in with (R4g): the controller itself registers a module only from the builder's done
// key now (takeInKey), but a great many of these tests only need a module registered with a build recorded, in the form
// the old outcome carried, so that is kept here and nowhere else. It is the controller's take-in of a build-agent outcome
// as it was, less the checks only an outcome needed. Nothing in the controller calls any of it.

// buildFrom turns what a builder said into what the mesh keeps.
//
// **By digest and path, never by where it was pushed** (novox/hq 04-ISSUES/102). The builder
// says `<registry>:<port>/<module>/<artifact>@sha256:…`; the mesh records the artifact-store
// reference and composes the store's address back in where a reference is used. `against` — what
// the build stood on, the catalogue's edge — is recorded the same way, so an edge names a module's
// artifact and not the machine it was pulled from.
func buildFrom(result link.BuildResult) inventory.Build {
	kept := inventory.Build{
		ID: result.ID, Repository: result.Repository, Ref: result.Ref,
		Commit: result.Commit, On: result.On, Failed: result.Failed,
		// **What the announcement carries, kept rather than discarded** (novox/hq 04-ISSUES/050).
		// The catalogue turns the manifest into requires/provides edges and `against` into build
		// edges, and it is not always listening when a build happens — on a fresh mesh it cannot
		// be, for exactly the modules it needs most. Keeping them is what makes a replay able to
		// rebuild the graph rather than a list of names.
		Path: result.Path,
		// What it was made from (novox/hq issue 280): two builds with one are one build.
		SourceFingerprint: result.SourceFingerprint,
		SourceInputs:      result.SourceInputs,
	}
	// When it was asked, which is what orders it against another build of the same module
	// (novox/hq 04-ISSUES/219) — not when it was heard.
	if asked, ok := link.BuildAskedAt(result.ID); ok {
		kept.Asked = asked
	}
	for _, ref := range result.Against {
		kept.Against = append(kept.Against, catalogue.Recorded(ref))
	}
	for _, ref := range result.Mirrored {
		kept.Mirrored = append(kept.Mirrored, catalogue.Recorded(ref))
	}
	for _, r := range result.Read {
		kept.Read = append(kept.Read, inventory.ReadRepository{Repository: r.Repository, Ref: r.Ref})
	}
	// What it was made from, as files (novox/hq ADR 0267): the planner maps the next merge onto it.
	for _, s := range result.Sources {
		kept.Sources = append(kept.Sources, inventory.BuildSource{Repository: s.Repository, Ref: s.Ref, Paths: s.Paths})
	}
	var announced []inventory.Artifact
	for _, made := range result.Made {
		announced = append(announced, inventory.Artifact{
			Name: made.Name, Kind: made.Kind, Reference: made.Reference,
		})
		kept.Made = append(kept.Made, inventory.Artifact{
			Name: made.Name, Kind: made.Kind, Reference: catalogue.Recorded(made.Reference),
		})
	}
	kept.Manifest = recordedManifest(result.Manifest, announced)
	// The module name comes from the manifest, which only exists when the build got that far.
	if len(kept.Manifest) > 0 {
		if list, err := judgedWith(result); err == nil {
			if m, err := catalogue.ParseManifestWith(kept.Manifest, list); err == nil {
				kept.Module = m.Module
			}
		}
	}
	return kept
}

// recordedManifest is a build's manifest with the store's address taken off every reference the
// build itself made. Other references — an image a module runs from a public registry — are what
// they were, which is why this rewrites only what `made` names rather than everything that looks
// like an address.
func recordedManifest(raw json.RawMessage, made []inventory.Artifact) json.RawMessage {
	announced := map[string]bool{}
	for _, a := range made {
		announced[a.Reference] = true
	}
	return withReferences(raw, func(ref string) (string, bool) {
		if !announced[ref] {
			return ref, false
		}
		return catalogue.Recorded(ref), true
	})
}

// takeIn is what the mesh does with a build's outcome, whoever hears it: the waiting command and
// the daemon that follows the role's events both come here (novox/hq issue 176), so a build's
// result reaches the catalogue whether or not the asker was still listening.
//
// Kept before it is judged. A failed build that leaves no trace is indistinguishable from one
// nobody asked for, and the difference is the whole of whether somebody should be looking at
// something. Then parsed with the same parser a hand-written manifest goes through — a second path
// would be a second thing to disagree about what a manifest is — and registered with where it came
// from: **for a source on a seat, as the path and the seat, never the URL just cloned** (ADR 0111),
// which the request carried and the outcome echoes. A definition naming an installation is refused
// here, where it would enter the catalogue; the build stays recorded and the refusal says which.
//
// Idempotent: the same outcome taken in twice registers the same module twice, which is one row
// written with the same values.
func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResult) (
	catalogue.Manifest, inventory.Build, error) {
	// The list of predefined keys as the store keeps it now: another controller process may have kept a newer one.
	if err := loadPredefined(ctx, inv); err != nil {
		return catalogue.Manifest{}, inventory.Build{}, fmt.Errorf("the list of predefined keys kept cannot be "+
			"read: %w", err)
	}
	kept := buildFrom(result)
	if err := inv.RecordBuild(ctx, kept); err != nil {
		return catalogue.Manifest{}, kept, err
	}
	if result.Failed != "" {
		// The builder's own words. Wrapping them in something about the control plane would put
		// two explanations between a person and a build log.
		return catalogue.Manifest{}, kept, fmt.Errorf("%s could not build %s:\n%s",
			result.On, result.Repository, result.Failed)
	}
	// Judged first against the list the build was judged against: the catalogue's at the commit built (novox/hq
	// ADR 0348 §7), so a module naming a key its own commit adds is read; and again below against the list the
	// mesh publishes, before it is registered.
	list, err := judgedWith(result)
	if err != nil {
		return catalogue.Manifest{}, kept, fmt.Errorf("%s built %s (%s), recorded and not registered: %w", result.On,
			result.Repository, short(result.Commit), err)
	}
	manifest, err := catalogue.ParseManifestWith(kept.Manifest, list)
	if err != nil {
		return catalogue.Manifest{}, kept, fmt.Errorf("%s built %s and what came back is not a manifest: %w",
			result.On, result.Repository, err)
	}
	recorded := inventory.Source{
		Repository: result.Repository, Path: result.Path, Ref: result.Ref,
		BuiltFrom: result.Commit, Head: result.Commit,
		// What it stood on, so registration can judge a built manifest's base (to-be 38 WP2.4).
		Against: kept.Against,
		// When it was asked, so an older request heard later does not replace a newer one
		// (novox/hq 04-ISSUES/219).
		Asked: kept.Asked,
	}
	if result.Source != nil && result.Source.Seat != "" {
		recorded.Repository, recorded.Seat = result.Source.Repository, result.Source.Seat
	}
	// **A build at a commit does not change the branch a module follows** (novox/hq 04-ISSUES/215):
	// the commit is built and recorded as what it was built from, and the module keeps following
	// what it followed before — the repository's default branch for one new to the catalogue.
	if followedBranch(result.Ref) == "" && result.Ref != "" {
		recorded.Ref = ""
		if was, err := inv.SourceOf(ctx, manifest.Module); err == nil {
			recorded.Ref = followedBranch(was.Ref)
		}
	}
	trunk := result.Trunk
	if was, err := inv.SourceOf(ctx, manifest.Module); err == nil && followedBranch(was.Ref) != "" {
		trunk = followedBranch(was.Ref)
	}
	if trunk == "" {
		trunk = "main"
	}
	j := takenBuild{kept: kept, manifest: manifest, recorded: recorded, ask: registrationAsk{Build: result.ID}, path: result.Path,
		trunk: trunk, by: result.On, what: result.Repository, commit: result.Commit}
	// The checks only an outcome of build-agent needs: a key is a trunk build by construction (ADR 0501 §2), in no
	// scratch namespace, and carries no list of its own.
	j.published = func() error {
		// **Only a commit on the trunk is published** (novox/hq ADR 0238): a commit off its repository's
		// default branch — a pull request's head, a feature branch built by hand, a `rebuild` or `replay
		// --register` of one — is for checking, and is never a module's version; nothing could then send it.
		// The branch the module already follows is its trunk — never the branch a build was asked at, or a
		// build of a feature branch by name would make that branch its trunk.
		follows := ""
		if was, err := inv.SourceOf(ctx, manifest.Module); err == nil {
			follows = followedBranch(was.Ref)
		}
		if err := publishable(result, follows); err != nil {
			return err
		}
		return nil
	}
	j.carried = func() error {
		// **The newest trunk build's list of predefined keys is published** (novox/hq ADR 0348 §7): a build of the
		// catalogue that may be registered keeps its list when its commit is newer than the one kept.
		carried, err := carriedList(result)
		if err != nil {
			return err
		}
		if carried != nil {
			if err := publishPredefined(ctx, inv, result, carried); err != nil {
				fmt.Printf("%s: the list of predefined keys it carried is not published: %v\n", result.ID, err)
			}
		}
		return nil
	}
	manifest, err = registerJudged(ctx, inv, j)
	return manifest, kept, err
}

// errOffTheTrunk is a build of a commit off its repository's trunk, which is never published.
var errOffTheTrunk = errors.New("the commit is not on its repository's trunk: a commit off the trunk is checked, " +
	"never published (ADR 0238) — merge it, and the merge builds it")

// publishable says whether a build's outcome may become a module's version: its commit on the module's
// trunk, as the build seat read the forge at the build — the branch the module follows when its source
// names one, else its repository's default branch. A build seat that could not say — one older than the
// rule, building its own successor — is let through and said, so the rule can reach the mesh.
func publishable(result link.BuildResult, follows string) error {
	if result.DryRun {
		return errors.New("a dry run is never published")
	}
	if result.Trunk == "" {
		fmt.Printf("%s: the build seat did not say whether %s is on its repository's trunk (it predates ADR 0238); "+
			"registered as before\n", result.ID, short(result.Commit))
		return nil
	}
	trunk := result.Trunk
	if follows != "" {
		trunk = follows
	}
	on := slices.Contains(result.Branches, trunk) || (trunk == result.Trunk && result.OnTrunk)
	if !on {
		return fmt.Errorf("%w (%s is not on %s)", errOffTheTrunk, short(result.Commit), trunk)
	}
	return nil
}

// fromTheCatalogue is whether a build is of the catalogue's repository, whose list alone the mesh publishes: by
// its name, in the URL form or the seat form.
func fromTheCatalogue(result link.BuildResult) bool {
	for _, repository := range []string{result.Repository, sourceRepository(result)} {
		name := strings.TrimSuffix(path.Base(strings.TrimRight(repository, "/")), ".git")
		if name == catalogue.CatalogueRepository {
			return true
		}
	}
	return false
}

func sourceRepository(result link.BuildResult) string {
	if result.Source == nil {
		return ""
	}
	return result.Source.Repository
}

// carriedList is the list a build of the catalogue carried, read strictly; nil for a build of another
// repository, or one that carried none.
func carriedList(result link.BuildResult) (catalogue.PredefinedList, error) {
	if result.Predefined == nil || !fromTheCatalogue(result) {
		return nil, nil
	}
	list, err := catalogue.ParsePredefined(result.Predefined.Keys)
	if err != nil {
		return nil, fmt.Errorf("%s carried a list of predefined keys that is refused: %w", result.ID, err)
	}
	return list, nil
}

// judgedWith is the list a build's manifest is first judged against at registration: the catalogue's own at the
// commit built, which its build judged it against, where it carried one; the list kept otherwise.
func judgedWith(result link.BuildResult) (catalogue.PredefinedList, error) {
	list, err := carriedList(result)
	if err != nil || list != nil {
		return list, err
	}
	return catalogue.PublishedPredefined(), nil
}

// publishPredefined keeps a trunk build's list when it is newer than the one kept, says so, and loads what the
// store keeps then — this build's, or a newer one another controller process kept.
func publishPredefined(ctx context.Context, inv *inventory.Inventory, result link.BuildResult,
	list catalogue.PredefinedList) error {
	kept, err := inv.KeepPredefined(ctx, inventory.PredefinedRecord{Keys: list, Repository: result.Repository,
		Commit: result.Commit, Generation: result.Predefined.Generation, Build: result.ID})
	if err != nil {
		return err
	}
	if kept {
		fmt.Printf("%s: the list of predefined keys at %s (generation %d) is published: %s\n", result.ID,
			short(result.Commit), result.Predefined.Generation, strings.Join(list.Keys(), ", "))
	} else {
		fmt.Printf("%s: the list of predefined keys at %s (generation %d) is not newer than the one kept, which "+
			"stands\n", result.ID, short(result.Commit), result.Predefined.Generation)
	}
	return loadPredefined(ctx, inv)
}

// builds is what the tests of a pending assignment feed an outcome to: the take-in above, then the tick that settles what
// waited for it (settlePending), as the daemon's hearing of an outcome did before R4g (it also moved the plans, which the
// walk now does from its keys). Nothing in the controller calls it.
type builds struct {
	inv  *inventory.Inventory
	open *stores
}

func (b builds) Built(ctx context.Context, result link.BuildResult) error {
	_, _, err := takeIn(ctx, b.inv, result)
	if b.open != nil {
		settlePending(ctx, b.open, time.Now())
	}
	if result.Failed != "" {
		return nil // the build is recorded; whether it registered is what the tests read
	}
	return err
}