mesh-controller/cmd/mesh-controller/build.go

package main

import (
	"context"
	"crypto/rand"
	"encoding/base64"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"os"
	"strings"
	"time"

	"github.com/novox/mesh-controller/internal/broker"
	"github.com/novox/mesh-controller/internal/catalogue"
	"github.com/novox/mesh-controller/internal/inventory"
	"github.com/novox/mesh-controller/internal/link"
)

// asking a build machine for a module, and what came back.
//
// Split out of main.go, which had reached 2,769 lines because appending was always the
// cheapest next step. That is how novox/hq ADR 0001 records `hal/sdk` reaching 34,636:
// nothing in it was wrong, and no one edit was the one that should have been a new file.

// buildCommand builds a module from its source and records what came out.
//
// **Run where there is a container runtime**, which is why it is a command rather than something
// the control plane does on its own: building needs to run things on a machine, and what the
// control plane may send a machine is bounded by the declaration language. This is the shape the
// builder module will take when it is given work over the broker; today a person runs it, and the
// mesh records the result the same way either way.
func buildCommand(ctx context.Context, args []string) error {
	set := flag.NewFlagSet("build", flag.ContinueOnError)
	ref := set.String("ref", "", "the branch, tag or commit to build")
	// A module is a repository and a path within it (novox/hq ADR 0069). Empty is the repository's
	// root, which is the ordinary case and why this is a flag rather than a second argument.
	path := set.String("path", "", "the module's directory inside the repository")
	wait := set.Duration("wait", 10*time.Minute, "how long to wait for a builder to answer")
	dryRun := set.Bool("dry-run", false, "build and print the manifest, recording nothing")
	// Every module whose source has moved, rather than one named repository.
	//
	// **The mirror of `push --behind`, and the same argument** (novox/hq ADR 0010): the mesh
	// already knows which modules are behind their source, so making a person read that list and
	// retype each repository is asking them to be the loop. Naming a repository and asking which
	// ones need building are different requests, so they are not combined.
	behind := set.Bool("behind", false, "every module the mesh holds older than its source has")
	// A repository on the mesh's own forge, named by its path there (novox/hq ADR 0111). Without it
	// the repository is external, cloned exactly as given — see source.go.
	self := set.Bool("self", false, "the repository is a path on the forge holding the git seat")
	positionals, err := parseAround(set, args)
	if err != nil {
		return err
	}
	if *behind {
		if len(positionals) != 0 || *self {
			return errors.New("build <repository> or build --behind, not both: one names a " +
				"repository and the other asks which need building")
		}
		return buildBehind(ctx, *wait)
	}
	if len(positionals) != 1 {
		return errors.New("build <repository> [--self] [--path P] [--ref R] [--wait D] [--dry-run]")
	}
	source := buildSource{Repository: positionals[0]}
	if *self {
		if err := onASeat(source.Repository); err != nil {
			return err
		}
		source.Seat = gitSeat
	}

	if *dryRun {
		return buildAndShow(ctx, source, *path, *ref, *wait)
	}
	return buildOne(ctx, source, *path, *ref, *wait)
}

// buildFrom turns what a builder said into what the mesh keeps.
//
// **By digest and path, never by where it was pushed** (novox/hq 04-ISSUES/102). The builder
// says `<registry>:<port>/<module>/<artifact>@sha256:…`; the mesh records the artifact-store
// reference and composes the store's address back in where a reference is used. `against` is kept
// as announced: it is what the build stood on as the builder saw it, and the catalogue's edge.
func buildFrom(result link.BuildResult) inventory.Build {
	kept := inventory.Build{
		ID: result.ID, Repository: result.Repository, Ref: result.Ref,
		Commit: result.Commit, On: result.On, Failed: result.Failed,
		// **What the announcement carries, kept rather than discarded** (novox/hq 04-ISSUES/050).
		// The catalogue turns the manifest into requires/provides edges and `against` into build
		// edges, and it is not always listening when a build happens — on a fresh mesh it cannot
		// be, for exactly the modules it needs most. Keeping them is what makes a replay able to
		// rebuild the graph rather than a list of names.
		Path: result.Path, Against: result.Against,
	}
	var announced []inventory.Artifact
	for _, made := range result.Made {
		announced = append(announced, inventory.Artifact{
			Name: made.Name, Kind: made.Kind, Reference: made.Reference,
		})
		kept.Made = append(kept.Made, inventory.Artifact{
			Name: made.Name, Kind: made.Kind, Reference: catalogue.Recorded(made.Reference),
		})
	}
	kept.Manifest = recordedManifest(result.Manifest, announced)
	// The module name comes from the manifest, which only exists when the build got that far.
	if len(kept.Manifest) > 0 {
		if m, err := catalogue.ParseManifest(kept.Manifest); err == nil {
			kept.Module = m.Module
		}
	}
	return kept
}

// buildsCommand says what has been built lately.
func buildsCommand(ctx context.Context, args []string) error {
	set := flag.NewFlagSet("builds", flag.ContinueOnError)
	limit := set.Int("n", 20, "how many to show")
	positionals, err := parseAround(set, args)
	if err != nil {
		return err
	}
	module := ""
	if len(positionals) == 1 {
		module = positionals[0]
	} else if len(positionals) > 1 {
		return errors.New("builds [<module>] [-n N]")
	}

	open, err := openStores(ctx)
	if err != nil {
		return err
	}
	defer open.Close()
	inv := open.inventory

	builds, err := inv.Builds(ctx, module, *limit)
	if err != nil {
		return err
	}
	if len(builds) == 0 {
		// Said rather than printed as nothing: an empty list and a failed read must never look
		// the same, and getting here means the store answered.
		if module != "" {
			fmt.Printf("nothing has been built for %s\n", module)
			return nil
		}
		fmt.Println("nothing has been built yet")
		return nil
	}

	for _, b := range builds {
		what := b.Module
		if what == "" {
			// It failed before knowing what it was building, which is most of the interesting
			// failures. The repository is what a person has to go and look at.
			what = "?"
		}
		outcome := "built " + short(b.Commit)
		if !b.Worked() {
			outcome = "failed"
		}
		fmt.Printf("%-18s %-14s %-10s %s\n",
			what, outcome, b.On, b.At.Local().Format("2006-01-02 15:04"))
		fmt.Printf("    %s", b.Repository)
		if b.Ref != "" {
			fmt.Printf(" at %s", b.Ref)
		}
		fmt.Println()
		for _, made := range b.Made {
			fmt.Printf("    %-10s %s\n", made.Kind, made.Reference)
		}
		if !b.Worked() {
			// The builder's own first line. The whole failure is often a build log, and printing
			// it here would bury every other row.
			fmt.Printf("    %s\n", firstLine(b.Failed))
		}
	}
	return nil
}

// builderCommand issues a build machine its own broker credential.
//
// **A build machine is not a node**, and giving it a node's account would let it read another
// machine's declarations. This is narrower and different: read the build queue, write the
// exchange and an asker's reply queue, and nothing else.
//
// Issued rather than assumed, because until this the builder used whatever credential it was
// handed — which in practice meant the broker's own administrative one. A program documented as
// holding its own credential and given somebody else's is worse than one with no story at all.
func builderCommand(ctx context.Context, args []string) error {
	set := flag.NewFlagSet("builder issue", flag.ContinueOnError)
	// Which machine will use it. Given, the credential is delivered by the mesh rather than
	// printed for somebody to carry — which is the difference between the builder being a module
	// and being a program somebody configures.
	forNode := set.String("node", "",
		"the machine that will run it, so the mesh delivers the credential instead of printing it")
	module := set.String("module", "builder", "the module on that machine that will read it")
	positionals, err := parseAround(set, args)
	if err != nil {
		return err
	}
	if len(positionals) != 2 || positionals[0] != "issue" {
		return errors.New("builder issue <name> [--node <machine>]")
	}
	name := positionals[1]

	management, err := broker.ManagementFromEnvironment()
	if err != nil {
		return err
	}

	// The same shape of secret a token carries: enough entropy that guessing is not a strategy,
	// and safe to put in a URL because that is where it goes.
	raw := make([]byte, 32)
	if _, err := rand.Read(raw); err != nil {
		return err
	}
	password := base64.RawURLEncoding.EncodeToString(raw)
	if err := management.CreateBuilderAccount(ctx, name, password); err != nil {
		return err
	}

	fmt.Printf("broker account %s created, scoped to the %s queue and the %s exchange\n\n",
		name, link.BuildQueue, link.Exchange)

	if *forNode != "" {
		known, err := broker.FromEnvironment()
		if err != nil {
			return fmt.Errorf("cannot deliver a credential without knowing where the broker is: %w", err)
		}
		inv, err := openInventory(ctx)
		if err != nil {
			return err
		}
		defer inv.Close()

		brokerAddr, err := brokerReachableAt(ctx, inv, known, *forNode)
		if err != nil {
			return err
		}

		// The URL and what verifies the broker, together. A mesh's broker presents a certificate
		// of the mesh's own, which is in no public trust store — so a URL on its own reaches only
		// a broker somebody else vouches for, and the connection fails at TLS with an error about
		// an unknown authority rather than about a missing pin.
		//
		// **The same two facts a node's token carries** (novox/hq ADR 0004), delivered the same
		// way: out of band relative to the broker, so what is trusted does not come from the thing
		// being trusted.
		held, err := json.Marshal(struct {
			URL         string `json:"url"`
			Fingerprint string `json:"fingerprint,omitempty"`
		}{
			URL:         fmt.Sprintf("amqps://%s:%s@%s/", name, password, brokerAddr),
			Fingerprint: known.Fingerprint,
		})
		if err != nil {
			return err
		}
		if err := inv.AcceptSecretForModule(ctx, *forNode, *module, "broker", string(held)); err != nil {
			return err
		}
		// Not printed. It is sealed to that machine and the mesh cannot read it back, which is
		// the whole point — printing it here would put the one copy that matters on a terminal.
		fmt.Printf("  sealed to %s, for the %s module. It arrives with the next push.\n",
			*forNode, *module)
		fmt.Printf("  run `push %s` to send it\n", *forNode)
		return nil
	}

	// The whole line only when the address is known. A URL with a placeholder where the host
	// should be is a URL somebody pastes and then debugs, and the placeholder is the last thing
	// they look at.
	if known, err := broker.FromEnvironment(); err == nil {
		fmt.Printf("  MESH_BROKER_AMQP=amqps://%s:%s@%s/\n\n", name, password, known.Address)
	} else {
		fmt.Printf("  the password is %s\n\n", password)
		fmt.Printf("  This control plane has no %s, so it cannot say where the broker is.\n"+
			"  Put the password in MESH_BROKER_AMQP on the build machine.\n\n",
			broker.AddressVar)
	}
	// Shown once, like a token, and for the same reason: what is stored is the broker's own hash
	// of it, and a control plane that could show it back would be a control plane that holds it.
	fmt.Println("This is the only time it is shown.")
	return nil
}

// buildBehind builds every module the mesh holds older than its source has.
//
// **This is the loop novox/hq ADR 0010 replaced a pipeline with, closed.** The mesh already
// records where each module came from and what its source last had; until this, a person read
// that list and retyped each repository — which is a person being the loop, and the thing a
// pipeline was doing before it was taken away.
//
// Each is built and recorded on its own. **One failing does not stop the others**, for the same
// reason one broken module no longer blocks a machine's whole declaration: a mesh where one bad
// repository holds back nine good ones is a mesh where nobody dares add the tenth.
func buildBehind(ctx context.Context, wait time.Duration) error {
	open, err := openStores(ctx)
	if err != nil {
		return err
	}
	defer open.Close()
	inv := open.inventory

	held, err := inv.Catalogued(ctx)
	if err != nil {
		return err
	}
	var stale []inventory.Entry
	for _, e := range held {
		if !e.Source.Current() {
			stale = append(stale, e)
		}
	}
	if len(stale) == 0 {
		// Said rather than doing nothing quietly: "nothing needed building" and "this did not
		// run" must never look the same.
		fmt.Println("every module the mesh holds is what its source last had")
		return nil
	}

	fmt.Printf("%d module(s) behind their source:\n", len(stale))
	for _, e := range stale {
		fmt.Printf("  %s  %s < %s\n",
			e.Manifest.Module, short(e.Source.BuiltFrom), short(e.Source.Head))
	}
	fmt.Println()

	var failed []string
	for _, e := range stale {
		fmt.Printf("--- %s\n", e.Manifest.Module)
		// Its own recorded ref, not its head commit: a module tracking a branch should be built
		// from that branch, and pinning to the commit the mesh happened to notice would quietly
		// turn a tracked branch into a pin.
		source := buildSource{Repository: e.Source.Repository, Seat: e.Source.Seat}
		if err := buildOne(ctx, source, e.Source.Path, e.Source.Ref, wait); err != nil {
			fmt.Printf("  %v\n", err)
			failed = append(failed, e.Manifest.Module)
		}
	}

	if len(failed) > 0 {
		return fmt.Errorf("%d of %d could not be built: %s",
			len(failed), len(stale), strings.Join(failed, ", "))
	}
	fmt.Printf("\n%d module(s) built. `push --behind` sends them to the machines running them\n",
		len(stale))
	return nil
}

// buildOne asks a build machine for one repository and records everything that came back.
//
// Separated from the command so `--behind` can walk a list without a second path to the same act.
func buildOne(ctx context.Context, source buildSource, path, ref string, wait time.Duration) error {
	// Before anything is asked of a builder: a source on a seat nobody holds is refused here, with
	// the reason, rather than sent to a machine to fail at `git clone`.
	repository, err := cloneFrom(ctx, source)
	if err != nil {
		return err
	}

	ident, err := openIdentity(ctx)
	if err != nil {
		return err
	}
	defer ident.Close()

	server, err := link.Connect(nil, nil)
	if err != nil {
		return err
	}
	defer server.Close()

	// Correlated by something the control plane makes, not by the module's name: two builds of one
	// module can be in flight, and the second answer is not the first one's.
	request := link.BuildRequest{
		ID:         fmt.Sprintf("%s-%d", "build", time.Now().UnixNano()),
		Repository: repository,
		Path:       path,
		Ref:        ref,
		Held:       heldBy(ctx),
	}
	fmt.Printf("asked for %s", source)
	if source.Seat != "" {
		fmt.Printf(" (%s)", repository)
	}
	if path != "" {
		fmt.Printf(" at %s", path)
	}
	if ref != "" {
		fmt.Printf(" on %s", ref)
	}
	fmt.Println()

	result, err := link.RequestBuild(ctx, server.Channel(), request, wait)
	if err != nil {
		return err
	}

	// Kept before it is judged. A failed build that leaves no trace is indistinguishable from one
	// nobody asked for, and the difference is the whole of whether somebody should be looking at
	// something.
	open, err := openStores(ctx)
	if err != nil {
		return err
	}
	defer open.Close()
	inv := open.inventory
	kept := buildFrom(result)
	if err := inv.RecordBuild(ctx, kept); err != nil {
		return err
	}

	if result.Failed != "" {
		// The builder's own words. Wrapping them in something about the control plane would put
		// two explanations between a person and a build log.
		return fmt.Errorf("%s could not build %s:\n%s", result.On, result.Repository, result.Failed)
	}

	// Said as recorded: what each artifact is, not where this builder happened to push it.
	for _, made := range kept.Made {
		fmt.Printf("  %-12s %s %s\n", made.Name, made.Kind, made.Reference)
	}

	// Parsed with the same parser a hand-written manifest goes through. A second path would be a
	// second thing to disagree about what a manifest is. The manifest as recorded, so the catalogue
	// holds references by digest and path and every declaration composes the store's address in.
	manifest, err := catalogue.ParseManifest(kept.Manifest)
	if err != nil {
		return fmt.Errorf("%s built %s and what came back is not a manifest: %w",
			result.On, result.Repository, err)
	}

	// Recorded with where it came from, so "is this current?" is answerable without building it
	// again (novox/hq ADR 0009). **For a source on a seat, as the path and the seat, never the URL
	// just cloned** (ADR 0111): the URL is where the forge runs today, and recording it would put
	// the forge's address back into every module built from it. The build log above keeps the URL,
	// because that is what was cloned.
	recorded := inventory.Source{
		Repository: result.Repository, Path: result.Path, Ref: result.Ref,
		BuiltFrom: result.Commit, Head: result.Commit,
	}
	if source.Seat != "" {
		recorded.Repository, recorded.Seat = source.Repository, source.Seat
	}
	if err := inv.RegisterModule(ctx, manifest, recorded); err != nil {
		return err
	}
	fmt.Printf("\n%s %s, built on %s from %s\n",
		manifest.Module, manifest.Version, result.On, short(result.Commit))
	fmt.Printf("  run `assign <node> %s` to put it somewhere\n", manifest.Module)
	return nil
}

// buildAndShow builds and prints the manifest without recording anything.
func buildAndShow(ctx context.Context, source buildSource, path, ref string, wait time.Duration) error {
	repository, err := cloneFrom(ctx, source)
	if err != nil {
		return err
	}
	ident, err := openIdentity(ctx)
	if err != nil {
		return err
	}
	defer ident.Close()
	server, err := link.Connect(nil, nil)
	if err != nil {
		return err
	}
	defer server.Close()

	result, err := link.RequestBuild(ctx, server.Channel(), link.BuildRequest{
		ID:         fmt.Sprintf("%s-%d", "build", time.Now().UnixNano()),
		Repository: repository, Path: path, Ref: ref,
		Held: heldBy(ctx),
	}, wait)
	if err != nil {
		return err
	}
	if result.Failed != "" {
		return fmt.Errorf("%s could not build %s:\n%s", result.On, result.Repository, result.Failed)
	}
	manifest, err := catalogue.ParseManifest(result.Manifest)
	if err != nil {
		return fmt.Errorf("%s built %s and what came back is not a manifest: %w",
			result.On, result.Repository, err)
	}
	body, err := json.MarshalIndent(manifest, "", "  ")
	if err != nil {
		return err
	}
	fmt.Println(string(body))
	return nil
}

// answers is what the three questions came back with, read once.
type answers struct {
	wrong   []inventory.Doing
	nodes   []inventory.Node
	quiet   []inventory.Node
	behind  map[string][]string
	sources map[string]inventory.Source
	// waiting is every machine not running what the mesh would send it.
	waiting []inventory.Machine
	// reported is every machine's last word beside when it was last sent a declaration — the
	// pair that answers "has it caught up", which waiting alone cannot (the sent digest is
	// recorded at send, not at apply).
	reported []inventory.Reported
	// refused is why a machine cannot be worked out at all, by name. A different thing from every
	// other answer here: those are about a machine that was told something, and this is about one
	// that cannot be told anything — it never reaches waiting, because nothing was computed for it
	// to compare against, so without this a wholly blocked mesh reads as a well one.
	refused map[string]string
	// network is why the private network could not be computed, when it could not. Almost always
	// a consequence of the refusals above: a node that does not resolve is not on the network, and
	// a mesh whose hub is that node has no hub.
	network string
}

// heldBy is every artifact this mesh has built, for a build that may need one as its base.
//
// **A failure here is not a failure to build.** A module that names no base does not need this at
// all, and one that does gets a refusal naming exactly what is missing — which is a better sentence
// than a build command refusing to start because a query did not run. So the store not opening is
// reported and the build goes ahead without it.
//
// Routed through the artifact store as the network reaches it now (novox/hq 04-ISSUES/102): a
// base is recorded by digest and path, and a build machine needs something it can pull.
func heldBy(ctx context.Context) map[string]string {
	open, err := openStores(ctx)
	if err != nil {
		fmt.Fprintf(os.Stderr, "could not read what this mesh has built, so a module naming a "+
			"base will be told that base is missing: %v\n", err)
		return nil
	}
	defer open.Close()
	held, err := open.inventory.Held(ctx)
	if err != nil {
		fmt.Fprintf(os.Stderr, "could not read what this mesh has built: %v\n", err)
		return nil
	}
	address, err := whereABuilderReachesTheStore(ctx, open.inventory)
	if err != nil {
		fmt.Fprintf(os.Stderr, "could not find the artifact store on this mesh's network, so a "+
			"module naming a base will be handed a reference nothing can fetch: %v\n", err)
		return held
	}
	if address == "" {
		return held
	}
	routed := make(map[string]string, len(held))
	for repository, reference := range held {
		routed[repository] = catalogue.Rerouted(reference, address)
	}
	return routed
}