mesh-host/internal/bootstrap/registry.go

package bootstrap

import (
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"strings"
	"time"
)

// RegistryModule is the module that provides the mesh's artifact store.
//
// **Named for the software, not the job.** The job is `artifact-store`, which is the provision this
// module offers; the software is Distribution, the OCI reference implementation. Calling the module
// `registry` named neither — and promised that any registry could sit there, which is the false
// genericity the naming rule forbids (novox/hq ADR 0040).
const RegistryModule = "distribution"

// registryResource is the id of the resource in that module's manifest that runs the registry.
// What the container is CALLED is read from the manifest rather than assumed, because the name is
// the catalogue's to choose and the installer only has to know which resource to wait for.
const registryResource = "store"

// Registry is what step 7 did.
type Registry struct {
	Installed
	// Container is the container the manifest declares, confirmed running.
	Container string
	// Address is host:port the registry answers on, as this machine reaches it.
	Address string
	// Answered is the status the registry's own `/v2/` gave back.
	Answered int
}

// InstallRegistry gives this mesh somewhere to put images.
//
// **Its image is upstream and it is never built.** novox/hq 04-ISSUES/029 is the whole reason this
// step exists in this position: a module that provides the artifact store cannot be delivered
// through the artifact store, so the registry is the one module whose image is pulled from the
// internet like the store and the broker before it. A manifest carrying the catalogue's
// placeholder digest here would mean somebody had made it buildable, which is the cycle again —
// so it is refused rather than pulled.
//
// **No credentials, and that is deliberate.** The registry is reached over the mesh's own private
// network, which is already the encrypted and authenticated thing; a second layer inside it would
// be certificates to issue and rotate for no property the first does not have (mesh-controller's
// `internal/builder`, which pushes to it the same way). So there is nothing here to configure and
// nothing to seal — which is also why step 8 can push without the mesh having issued anything.
//
// **It is verified by asking it, not by looking at it.** A container that is up is not a registry
// that serves: `/v2/` is the registry API's own "yes, I am one and I am ready", and it is the
// question step 8 depends on the answer to.
func InstallRegistry(ctx context.Context, o Options, d Deps, control controlPlane,
	say func(string)) (Registry, error) {

	out := Registry{Address: o.Registry}

	manifest, err := readManifest(o.Catalogue, RegistryModule)
	if err != nil {
		return out, err
	}
	container, image, err := containerIn(manifest, registryResource)
	if err != nil {
		return out, err
	}
	if strings.Contains(image, placeholderDigest) {
		return out, fmt.Errorf(
			"the %s module's image is %q, which is the catalogue's placeholder for something the "+
				"mesh builds and pushes.\n"+
				"This module is the one that cannot work that way: it PROVIDES the place built "+
				"images go, so it can never be delivered through it (novox/hq 04-ISSUES/029). Its "+
				"image is upstream and pinned in the manifest", RegistryModule, image)
	}
	out.Container = container
	say("  registry image    " + image + " — upstream, never built")

	installed, err := installModule(ctx, o, control, RegistryModule, manifest, say)
	out.Installed = installed
	if err != nil {
		return out, err
	}

	// Read back, in two stages, because they fail differently. A container that never appears is
	// a declaration that did not reach this node or an image that would not pull; a container
	// that is up and does not answer is a registry that started and failed.
	if err := waitForContainer(ctx, control.run, o.Timeout, o.Wait, container, say); err != nil {
		return out, err
	}
	status, err := waitForTheRegistry(ctx, d, o, say)
	if err != nil {
		return out, err
	}
	out.Answered = status
	return out, nil
}

// waitForTheRegistry asks the store's own `/v2/`, on its loopback (StoreLoopback), until it answers: the pushes that
// follow go there, and the store's front is not served yet on a fresh mesh.
func waitForTheRegistry(ctx context.Context, d Deps, o Options, say func(string)) (int, error) {
	loopback, err := StoreLoopback(o.Catalogue)
	if err != nil {
		return 0, err
	}
	where := "http://" + loopback + "/v2/"

	deadline := time.Now().Add(o.Wait)
	var last string
	for {
		asking, cancel := context.WithTimeout(ctx, o.Timeout)
		status, _, err := d.Fetch(asking, where)
		cancel()
		switch {
		case err != nil:
			last = err.Error()
		case status == http.StatusOK:
			say(fmt.Sprintf("  replies           %s answered %d", where, status))
			return status, nil
		default:
			// A registry that answers 401 is one that wants credentials, which this one is
			// configured not to. Reported as what it said rather than retried into a timeout.
			last = fmt.Sprintf("it answered %d", status)
		}

		if time.Now().After(deadline) {
			break
		}
		select {
		case <-ctx.Done():
			return 0, ctx.Err()
		case <-time.After(answerEvery):
		}
	}
	return 0, fmt.Errorf(
		"the registry's container is running and %s does not answer, after waiting %s: %s\n"+
			"Running is not serving. `/v2/` is the registry API saying it is ready, and the next "+
			"step pushes the control plane's image to it — so this is refused here rather than "+
			"discovered inside a `docker push`. `docker logs mesh-registry` says what it did",
		where, o.Wait, last)
}

// waitForContainer waits for a container the mesh was asked to create to be running.
//
// Unlike the foundation's own verify, this one waits: the mesh applies through a node's host, over
// the broker, asynchronously. A push that the control plane accepted has not yet happened on the
// machine, and refusing on the first look would refuse every correct install.
func waitForContainer(ctx context.Context, run Runner, probe, wait time.Duration, name string,
	say func(string)) error {

	deadline := time.Now().Add(wait)
	var last string
	for {
		state, err := containerRunning(ctx, run, probe, name)
		switch {
		case err != nil:
			last = "it is not there at all"
		case state.running:
			say("  running           " + name)
			return nil
		default:
			last = "it is " + state.status
		}

		if time.Now().After(deadline) {
			break
		}
		select {
		case <-ctx.Done():
			return ctx.Err()
		case <-time.After(answerEvery):
		}
	}
	return fmt.Errorf(
		"the mesh accepted the push and %q is not running after %s: %s\n"+
			"The control plane sends a declaration over the broker and this node's host applies "+
			"it, so the two ends fail differently: `mesh-host` on this machine says what it made "+
			"of the declaration, and `status` on the control plane says whether it was collected "+
			"at all", name, wait, last)
}

// containerIn finds one container resource in a module manifest and gives back its name and image.
//
// It reads the manifest as data rather than through the catalogue's own parser, because that
// parser lives in the control plane and the host depends on nothing installed first
// (novox/hq ADR 0041) — importing it would put tier 2 inside tier 0. What is read here is two
// fields of a shape the catalogue owns; the manifest is handed to the control plane unchanged, and
// it is the control plane's `module add` that judges whether it is a manifest at all.
func containerIn(manifest []byte, id string) (name, image string, err error) {
	var m struct {
		Module    string `json:"module"`
		Resources []struct {
			ID    string `json:"id"`
			Type  string `json:"type"`
			Name  string `json:"name"`
			Image string `json:"image"`
		} `json:"resources"`
	}
	if err := json.Unmarshal(manifest, &m); err != nil {
		return "", "", fmt.Errorf("this manifest is not readable as JSON: %w", err)
	}
	var containers []string
	for _, r := range m.Resources {
		if r.Type != "container" {
			continue
		}
		containers = append(containers, r.ID)
		if r.ID == id {
			return r.Name, r.Image, nil
		}
	}
	return "", "", fmt.Errorf(
		"the %s module declares no container %q, so the installer does not know what to wait for. "+
			"It declares: %s", m.Module, id, strings.Join(containers, ", "))
}

// storeContainerPort is the port the registry image listens on inside its container (Distribution's own default).
const storeContainerPort = "5000"

// StoreLoopback is host:port where this machine reaches the store's container itself, read from the port its manifest
// publishes for it: on a fresh mesh the bootstrap pushes the holders' own images there, with no credential, because the
// store's front (distribution's program, a bundle the mesh builds) is not served until the builder has built it (the
// publish contract allows a push on the store's own node to its loopback). A port published on every address is reached
// on 127.0.0.1. What the bootstrap records is named under the seat's reach (Options.Registry), never this address.
func StoreLoopback(catalogue string) (string, error) {
	manifest, err := readManifest(catalogue, RegistryModule)
	if err != nil {
		return "", err
	}
	var m struct {
		Resources []struct {
			ID    string   `json:"id"`
			Ports []string `json:"ports"`
		} `json:"resources"`
	}
	if err := json.Unmarshal(manifest, &m); err != nil {
		return "", fmt.Errorf("the %s manifest is not readable as JSON: %w", RegistryModule, err)
	}
	for _, r := range m.Resources {
		if r.ID != registryResource {
			continue
		}
		for _, p := range r.Ports {
			parts := strings.Split(strings.SplitN(p, "/", 2)[0], ":")
			switch {
			case len(parts) == 3 && parts[2] == storeContainerPort:
				return parts[0] + ":" + parts[1], nil
			case len(parts) == 2 && parts[1] == storeContainerPort:
				return "127.0.0.1:" + parts[0], nil
			}
		}
	}
	return "", fmt.Errorf("the %s module's container %q publishes no port for the store's %s, so the bootstrap cannot "+
		"reach the store to push to it", RegistryModule, registryResource, storeContainerPort)
}