mesh-controller/cmd/mesh-controller/key_takein.go

package main

import (
	"archive/tar"
	"bytes"
	"compress/gzip"
	"context"
	"encoding/base64"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"path"
	"strings"
	"time"

	"git.novox.be/novox/mesh-sdk/go/buildstep"
	"git.novox.be/novox/mesh-sdk/go/catalogue"
	sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
	"github.com/nats-io/nats.go/jetstream"
	"github.com/novox/mesh-controller/internal/artifacts"
	"github.com/novox/mesh-controller/internal/builderkeys"
	"github.com/novox/mesh-controller/internal/buildreads"
	"github.com/novox/mesh-controller/internal/inventory"
	"github.com/novox/mesh-controller/internal/link"
)

// The take-in from the builder's done key (novox/hq ADR 0501, ADR 0498 §8, ADR 0453 §5; issue 1851).
//
// From R4e the controller asks no build but the builder's own module's: a module of a merge is taken from the builder's
// key for the fingerprint the indexer's trunk value of the module names (ADR 0501 §2). Only a done key whose every
// artifact names a reference is taken (ADR 0498 §8); a key not done yet, one waiting for a reference, a templates
// archive the store does not answer, or a list of predefined keys the forge does not answer is waited for and said,
// never failed and never delivered from. A failed key fails, in the builder's own words. The take-in then records the
// key as a build of today's record layout, so a controller of before reads it on a rollback, and judges it as every
// build is judged (registerJudged).

// errKeyWaits says a take-in from a key waits: for the builder, a reference, the store or the forge. Never a failure.
var errKeyWaits = errors.New("waits")

// errKeyFailed says the module's key failed: the builder's own words follow.
var errKeyFailed = errors.New("the builder's key failed")

// keyReads is what a take-in from a key reads of the builder and the indexer (builderkeys.Reader, or a test's).
type keyReads interface {
	Build(ctx context.Context, module, fp string) (builderkeys.Key, error)
	Manifest(ctx context.Context, digest string) (sdkdelivery.Manifest, error)
}

// keyBuildID is the id a key's build is recorded under: the key and its revision, so the same revision recorded twice
// is one record, and a rebuild of the same fingerprint (a new revision, ADR 0429 §18) another.
func keyBuildID(k builderkeys.Key) string {
	part, err := sdkdelivery.FingerprintKeyPart(k.Fingerprint)
	if err != nil {
		part = strings.TrimPrefix(k.Fingerprint, sdkdelivery.SourcePrefix2)
	}
	return fmt.Sprintf("key-%s-%d", part, k.Revision)
}

// readArchive is the archive a key's reference names, read from the artifact store and checked against its digest. A
// variable so a test needs no store.
var readArchive = func(ctx context.Context, inv *inventory.Inventory, reference string) ([]byte, error) {
	shelf, err := inv.Catalogue(ctx)
	if err != nil {
		return nil, err
	}
	address, err := artifactStoreAddress(ctx, inv, shelf, "")
	if err != nil {
		return nil, err
	}
	if address == "" {
		return nil, errors.New("the artifact store is on no network the controller reaches")
	}
	return artifacts.Store{Address: address}.GetBlob(ctx, reference)
}

// predefinedHead is the list of predefined keys at the head of the catalogue repository's trunk, and that head.
type predefinedHead struct {
	List       catalogue.PredefinedList
	Repository string
	Commit     string
	// Generation orders two heads of the trunk: the head's commit time in seconds. A list kept from a build's
	// generation (its commit count) is always older than a head read here.
	Generation int
}

// predefinedAtTrunk is the list of predefined keys at the head of the catalogue repository's trunk (ADR 0501 §8), read
// from the forge. A variable so a test needs no forge.
var predefinedAtTrunk = func(ctx context.Context, inv *inventory.Inventory) (predefinedHead, error) {
	entries, err := inv.Catalogued(ctx)
	if err != nil {
		return predefinedHead{}, err
	}
	repository := ""
	for _, e := range entries {
		if e.Source.Seat == gitSeat && path.Base(strings.Trim(e.Source.Repository, "/")) == catalogue.CatalogueRepository {
			repository = strings.Trim(e.Source.Repository, "/")
			break
		}
	}
	if repository == "" {
		return predefinedHead{}, fmt.Errorf("no module of the catalogue names the repository %s on the forge",
			catalogue.CatalogueRepository)
	}
	var heads []struct {
		SHA    string `json:"sha"`
		Commit struct {
			Committer struct {
				Date time.Time `json:"date"`
			} `json:"committer"`
		} `json:"commit"`
	}
	if err := askGitea(ctx, "gitea_api", map[string]any{"path": "/repos/" + repository +
		"/commits?limit=1&stat=false&verification=false&files=false"}, &heads); err != nil {
		return predefinedHead{}, err
	}
	if len(heads) == 0 || heads[0].SHA == "" {
		return predefinedHead{}, fmt.Errorf("the forge named no head of %s's trunk", repository)
	}
	head := predefinedHead{Repository: repository, Commit: heads[0].SHA,
		Generation: int(heads[0].Commit.Committer.Date.Unix())}
	var file struct {
		Content  string `json:"content"`
		Encoding string `json:"encoding"`
	}
	if err := askGitea(ctx, "gitea_api", map[string]any{"path": "/repos/" + repository + "/contents/" +
		catalogue.PredefinedFile + "?ref=" + head.Commit}, &file); err != nil {
		return predefinedHead{}, err
	}
	if file.Encoding != "base64" {
		return predefinedHead{}, fmt.Errorf("the forge answered %s at %.8s in the encoding %q", catalogue.PredefinedFile,
			head.Commit, file.Encoding)
	}
	raw, err := base64.StdEncoding.DecodeString(file.Content)
	if err != nil {
		return predefinedHead{}, err
	}
	if head.List, err = catalogue.ParsePredefined(raw); err != nil {
		return predefinedHead{}, fmt.Errorf("%s at %.8s is refused: %w", catalogue.PredefinedFile, head.Commit, err)
	}
	return head, nil
}

// askGitea asks the forge's module one tool on the bus and reads its result into into.
func askGitea(ctx context.Context, tool string, args map[string]any, into any) error {
	js, err := aBus()
	if err != nil {
		return err
	}
	defer js.Close()
	raw, _ := json.Marshal(args)
	answer, err := link.Ask(ctx, link.OverNATS{Conn: js.Conn()}, "gitea", tool, raw, 30*time.Second)
	if err != nil {
		return err
	}
	if answer.Error != "" {
		return fmt.Errorf("gitea.%s: %s", tool, answer.Error)
	}
	var wrapped struct {
		Result json.RawMessage `json:"result"`
	}
	if err := json.Unmarshal(answer.Result, &wrapped); err != nil {
		return err
	}
	return json.Unmarshal(wrapped.Result, into)
}

// keyTake is one take-in from a key: the module, the indexer's trunk value naming the key's fingerprint, and the
// registration's warrant.
type keyTake struct {
	Module string
	Trunk  sdkdelivery.Module
	Ask    registrationAsk
}

// takeInKey takes a module in from the builder's key for the fingerprint its trunk value names: records the key as a
// build and registers the module, or says why not. errKeyWaits when it waits, errKeyFailed when the key failed.
func takeInKey(ctx context.Context, inv *inventory.Inventory, keys keyReads, t keyTake) (
	catalogue.Manifest, inventory.Build, error) {
	head, err := predefinedAtTrunk(ctx, inv)
	if err != nil {
		return catalogue.Manifest{}, inventory.Build{}, fmt.Errorf("%w: the list of predefined keys at the catalogue's "+
			"trunk head cannot be read, so %s is not judged: %v", errKeyWaits, t.Module, err)
	}
	keepPredefinedHead(ctx, inv, head)
	previous, err := inv.Builds(ctx, t.Module, 5)
	if err != nil {
		return catalogue.Manifest{}, inventory.Build{}, err
	}
	kept, manifest, err := buildOfKey(ctx, keys, t, keyContext{list: head.List, previous: previous,
		archive: func(ctx context.Context, reference string) ([]byte, error) { return readArchive(ctx, inv, reference) }})
	if err != nil {
		return catalogue.Manifest{}, kept, err
	}
	if err := inv.RecordBuild(ctx, kept); err != nil {
		return catalogue.Manifest{}, kept, err
	}
	recorded := inventory.Source{Repository: t.Trunk.Repository, Seat: gitSeat, Path: keyPath(t.Trunk),
		BuiltFrom: t.Trunk.Commit, Head: t.Trunk.Commit, Against: kept.Against}
	trunk := "main"
	if was, err := inv.SourceOf(ctx, manifest.Module); err == nil {
		recorded.Ref = followedBranch(was.Ref)
		if recorded.Ref != "" {
			trunk = recorded.Ref
		}
	}
	registered, err := registerJudged(ctx, inv, takenBuild{kept: kept, manifest: manifest, recorded: recorded,
		ask: t.Ask, path: recorded.Path, trunk: trunk, by: "the builder", what: t.Trunk.Repository,
		commit: t.Trunk.Commit})
	return registered, kept, err
}

// keyContext is what a key's build is made with besides the key: the list of predefined keys its manifest is judged
// against, the module's recent builds (the bases its last one stood on are carried) and how an archive is read.
type keyContext struct {
	list     catalogue.PredefinedList
	previous []inventory.Build
	archive  func(ctx context.Context, reference string) ([]byte, error)
}

// keyPath is a module's directory as the catalogue records it: empty for a module at its repository's root.
func keyPath(m sdkdelivery.Module) string {
	if m.Path == sdkdelivery.RootPath {
		return ""
	}
	return strings.Trim(m.Path, "/")
}

// buildOfKey reads the key of the trunk value's fingerprint and makes its build record and the manifest it resolves to
// (ADR 0498 §8, ADR 0501 §1, §8): the module's manifest by the key's digest, judged against the list of predefined keys
// at the catalogue's trunk head, resolved with the key's artifacts.
func buildOfKey(ctx context.Context, keys keyReads, t keyTake, kc keyContext) (
	inventory.Build, catalogue.Manifest, error) {
	waits := func(format string, a ...any) (inventory.Build, catalogue.Manifest, error) {
		return inventory.Build{}, catalogue.Manifest{}, fmt.Errorf("%w: "+format, append([]any{errKeyWaits}, a...)...)
	}
	if t.Trunk.FP2 == "" {
		why := t.Trunk.None2
		if why == "" {
			why = "the indexer wrote no fingerprint of the new form"
		}
		return waits("%s at %.8s has no fingerprint to take a build by: %s", t.Module, t.Trunk.Commit, why)
	}
	k, err := keys.Build(ctx, t.Module, t.Trunk.FP2)
	if err != nil {
		return waits("%s's key cannot be read: %v", t.Module, err)
	}
	switch {
	case k.Failed():
		return inventory.Build{}, catalogue.Manifest{}, fmt.Errorf("%w: %s", errKeyFailed, k.Says())
	case !k.Done():
		return waits("%s", k.Says())
	case len(k.Unreferenced()) > 0:
		return waits("%s", k.Says())
	}
	src, err := keys.Manifest(ctx, k.Build.Manifest)
	if err != nil {
		return waits("%s's manifest: %v", t.Module, err)
	}
	manifest, err := catalogue.ParseManifestWith([]byte(src.JSON), kc.list)
	if err != nil {
		// **A word the running controller does not read waits for the controller that reads it** (ADR 0501 §7): the
		// key stays done, and the controller that reads the word registers it with no second build.
		if word := refusedWord(err.Error()); word != "" {
			return waits("%s's manifest uses the word %q, which this controller does not read: it is registered "+
				"when a controller that reads it runs", t.Module, word)
		}
		return inventory.Build{}, catalogue.Manifest{}, fmt.Errorf("the builder built %s and its manifest is "+
			"refused: %w", t.Module, err)
	}
	if manifest.Module != t.Module {
		return inventory.Build{}, catalogue.Manifest{}, fmt.Errorf("the key of %s names the manifest of %s",
			t.Module, manifest.Module)
	}
	built, err := builtOfKey(ctx, kc.archive, manifest, k)
	if err != nil {
		return waits("%v", err)
	}
	resolved, err := manifest.Resolve(built)
	if err != nil {
		return inventory.Build{}, catalogue.Manifest{}, fmt.Errorf("the builder built %s and it does not resolve: %w",
			t.Module, err)
	}
	raw, err := json.Marshal(resolved)
	if err != nil {
		return inventory.Build{}, catalogue.Manifest{}, err
	}
	kept := inventory.Build{ID: keyBuildID(k), Repository: t.Trunk.Repository, Module: t.Module,
		Commit: t.Trunk.Commit, On: k.Build.By, Path: keyPath(t.Trunk), Manifest: raw,
		SourceFingerprint: k.Fingerprint, At: time.Now().UTC()}
	// **Read from the manifest, fresh at every key** (the same rule build-agent says, internal/buildreads). Sources are
	// left unsaid but for a module whose every artifact names its paths: the planner then reads the module's whole
	// directory, over-planning and never under-planning, and a fingerprint that did not change moves nothing (R4e A9).
	// **Its build source where its every artifact names its paths** (novox/hq ADR 0499 §1, §5; issue 1758, B1): the
	// planner reads a merge's changed files against those paths and the manifest alone, so a change elsewhere in the
	// module's directory touches nothing it builds from.
	if own := pathsBuildSource(manifest, keyPath(t.Trunk)); own != nil {
		kept.Sources = []inventory.BuildSource{{Paths: own}}
	}
	for _, c := range buildreads.Of(manifest) {
		kept.Read = append(kept.Read, inventory.ReadRepository{Repository: c.Repository, Ref: c.Ref})
	}
	// **The bases its last build stood on, carried** (R4e, lead decision): what the collector keeps a mirrored base by.
	// A key names none; the module's newest successful build's are carried so the keep set does not change.
	for _, b := range kc.previous {
		if b.Worked() && b.Module == t.Module && b.ID != kept.ID {
			kept.Against = append([]string(nil), b.Against...)
			break
		}
	}
	for _, a := range k.Build.Artifacts {
		kept.Made = append(kept.Made, inventory.Artifact{Name: a.Name, Kind: kindOf(manifest, a),
			Reference: catalogue.Recorded(a.Reference)})
	}
	return kept, resolved, nil
}

// kindOf is an artifact's kind: the key's, else the manifest's.
func kindOf(m catalogue.Manifest, a sdkdelivery.BuiltArtifact) string {
	if a.Kind != "" || m.Build == nil {
		return a.Kind
	}
	for _, d := range m.Build.Artifacts {
		if d.Name == a.Name {
			return d.Kind
		}
	}
	return ""
}

// builtOfKey is each of a done key's artifacts as Resolve reads it: its reference and digest, a TypeScript bundle's
// launchers by the build modules' one rule (mesh-sdk buildstep.Launchers), and, for the archive a module packs its
// templates into, each template's text read back from the archive the store holds under the key's reference.
func builtOfKey(ctx context.Context, archive func(context.Context, string) ([]byte, error), m catalogue.Manifest,
	k builderkeys.Key) (
	[]catalogue.Built, error) {
	declared := map[string]catalogue.Artifact{}
	if m.Build != nil {
		for _, a := range m.Build.Artifacts {
			declared[a.Name] = a
		}
	}
	templates := ""
	if m.Templated() {
		templates = catalogue.TemplatesArtifact(m)
	}
	var out []catalogue.Built
	for _, a := range k.Build.Artifacts {
		d := declared[a.Name]
		b := catalogue.Built{Name: a.Name, Kind: kindOf(m, a), Reference: catalogue.Recorded(a.Reference),
			Digest: a.Digest}
		if launchers := buildstep.Launchers(d.Language, d.Entrypoints); len(launchers) > 0 {
			b.Launchers = launchers
		}
		if a.Name == templates && templates != "" {
			body, err := archive(ctx, b.Reference)
			if err != nil {
				return nil, fmt.Errorf("the templates archive %q of %s cannot be read from the artifact store: %v",
					a.Name, m.Module, err)
			}
			if b.Texts, err = unpackTexts(body); err != nil {
				return nil, fmt.Errorf("the templates archive %q of %s cannot be read back: %v", a.Name, m.Module, err)
			}
		}
		out = append(out, b)
	}
	return out, nil
}

// unpackTexts is every file of a packed templates archive (gzip over tar), by its path inside it.
func unpackTexts(body []byte) (map[string]string, error) {
	zipped, err := gzip.NewReader(bytes.NewReader(body))
	if err != nil {
		return nil, err
	}
	defer zipped.Close()
	reader := tar.NewReader(zipped)
	texts := map[string]string{}
	for {
		header, err := reader.Next()
		if errors.Is(err, io.EOF) {
			return texts, nil
		}
		if err != nil {
			return nil, err
		}
		if header.Typeflag != tar.TypeReg {
			continue
		}
		text, err := io.ReadAll(io.LimitReader(reader, 16<<20))
		if err != nil {
			return nil, err
		}
		texts[strings.TrimPrefix(header.Name, "./")] = string(text)
	}
}

// keepPredefinedHead keeps the list read at the catalogue's trunk head when it is newer than the one kept, as a build of
// the catalogue's did (ADR 0348 §7), and judges every manifest read again against it. Said, never fatal: the list read
// is the one this take-in judges against either way.
func keepPredefinedHead(ctx context.Context, inv *inventory.Inventory, head predefinedHead) {
	kept, err := inv.KeepPredefined(ctx, inventory.PredefinedRecord{Keys: head.List, Repository: head.Repository,
		Commit: head.Commit, Generation: head.Generation, Build: "the trunk head " + short(head.Commit)})
	if err != nil {
		fmt.Printf("the list of predefined keys at %s's trunk head %s is not kept: %v\n", head.Repository,
			short(head.Commit), err)
		return
	}
	if kept {
		fmt.Printf("the list of predefined keys at %s's trunk head %s is published: %s\n", head.Repository,
			short(head.Commit), strings.Join(head.List.Keys(), ", "))
	}
	if err := loadPredefined(ctx, inv); err != nil {
		fmt.Printf("the list of predefined keys kept cannot be read: %v\n", err)
	}
}

// openBuilderKeys is the builder's and the indexer's buckets, read on a connection for the moment.
func openBuilderKeys() (builderkeys.Reader, func(), error) {
	js, err := aBus()
	if err != nil {
		return builderkeys.Reader{}, nil, err
	}
	api, err := jetstream.New(js.Conn())
	if err != nil {
		js.Close()
		return builderkeys.Reader{}, nil, err
	}
	return builderkeys.Reader{JS: api}, js.Close, nil
}

// pathsBuildSource is the build source of a module whose every artifact names the paths it is built from (ADR 0499
// §1): its manifest and each path, relative to its repository's root, a path read as a file or everything below a
// directory of that name (`<path>/**` holds both). Nil when an artifact names none: the module reads its directory.
func pathsBuildSource(m catalogue.Manifest, dir string) []string {
	if m.Build == nil || len(m.Build.Artifacts) == 0 {
		return nil
	}
	in := func(p string) string {
		if dir == "" {
			return p
		}
		return dir + "/" + p
	}
	seen := map[string]bool{}
	out := []string{in("module.json")}
	for _, a := range m.Build.Artifacts {
		if len(a.Paths) == 0 {
			return nil
		}
		for _, p := range a.Paths {
			entry := in(strings.Trim(path.Clean(p), "/")) + "/**"
			if !seen[entry] {
				seen[entry] = true
				out = append(out, entry)
			}
		}
	}
	return out
}