mesh-controller/internal/inventory/chain_builds.go

package inventory

import (
	"bytes"
	"context"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"sort"
	"strings"
	"time"

	"github.com/novox/mesh-controller/internal/chain"
)

// Builds on the bus (novox/hq ADR 0408, issue 1090): once the kind `builds` is moved (chain.go) every build is a key of
// the builds bucket, under its id, and every read below is made from the bucket's records. The tables are read when the
// kind is not moved, by the methods these stand beside.

// toChainBuild is a build as the bucket keeps it.
func toChainBuild(b Build) chain.Build {
	c := chain.Build{ID: b.ID, Repository: b.Repository, Ref: b.Ref, Module: b.Module, Commit: b.Commit, On: b.On,
		Path: b.Path, Against: b.Against, Mirrored: b.Mirrored, SourceFingerprint: b.SourceFingerprint,
		Failed: b.Failed, Asked: b.Asked.UTC(), At: b.At.UTC(), ReuseKey: b.ReuseKey, Published: b.Published}
	if len(b.Manifest) > 0 {
		c.Manifest = json.RawMessage(b.Manifest)
	}
	for _, r := range b.Read {
		c.Read = append(c.Read, chain.ReadRepository{Repository: r.Repository, Ref: r.Ref, Paths: r.Paths, Own: r.Own})
	}
	for _, s := range b.Sources {
		c.Sources = append(c.Sources, chain.Source{Repository: s.Repository, Ref: s.Ref, Paths: s.Paths})
	}
	for _, a := range b.Made {
		c.Made = append(c.Made, chain.Artifact{Name: a.Name, Kind: a.Kind, Reference: a.Reference})
	}
	if c.Published == nil {
		c.Published = publishedOf(b.Made)
	}
	return c
}

// fromChainBuild is a build as the mesh reads it.
func fromChainBuild(c chain.Build) Build {
	b := Build{ID: c.ID, Repository: c.Repository, Ref: c.Ref, Module: c.Module, Commit: c.Commit, On: c.On, Path: c.Path,
		Manifest: []byte(c.Manifest), Against: c.Against, Mirrored: c.Mirrored, SourceFingerprint: c.SourceFingerprint,
		Failed: c.Failed, Asked: c.Asked, At: c.At, ReuseKey: c.ReuseKey, Published: c.Published}
	if len(b.Manifest) == 0 {
		b.Manifest = nil
	}
	for _, r := range c.Read {
		b.Read = append(b.Read, ReadRepository{Repository: r.Repository, Ref: r.Ref, Paths: r.Paths, Own: r.Own})
	}
	for _, s := range c.Sources {
		b.Sources = append(b.Sources, BuildSource{Repository: s.Repository, Ref: s.Ref, Paths: s.Paths})
	}
	for _, a := range c.Made {
		b.Made = append(b.Made, Artifact{Name: a.Name, Kind: a.Kind, Reference: a.Reference})
	}
	return b
}

// publishedOf is what a build published, read from what it made: a package by name and version, and an artifact by the
// digest of the first thing it made that has one (`<reference>@sha256:…`). Nil for a build that published nothing.
func publishedOf(made []Artifact) *chain.Published {
	var out chain.Published
	for _, a := range made {
		switch {
		case a.Kind == "package":
			if name, version, ok := strings.Cut(a.Reference, "@"); ok && name != "" && version != "" && out.Package == nil {
				out.Package = &chain.Package{Name: name, Version: version}
			}
		case out.Artifact == "":
			if _, digest, ok := strings.Cut(a.Reference, "@"); ok && strings.HasPrefix(digest, "sha256:") {
				out.Artifact = digest
			}
		}
	}
	if out.Artifact == "" && out.Package == nil {
		return nil
	}
	return &out
}

// chainBuildKey is a build's key. An id the bus takes no spelling of is refused by the write, loudly.
func chainBuildKey(id string) string { return id }

// chainRecordBuild keeps a build under its id. **Idempotent on the id**, as the table's insert was: a result arrives twice,
// and the first record stands. A request nobody has answered is the record in state asked (issue 1091); the outcome
// takes its place under the same key, and keeps the request it answers.
func (i *Inventory) chainRecordBuild(ctx context.Context, c *chain.Chain, b Build) error {
	if b.At.IsZero() {
		b.At = time.Now().UTC()
	}
	key := chainBuildKey(b.ID)
	for try := 0; try < casTries; try++ {
		next := toChainBuild(b)
		cur, found, err := c.Builds.Get(ctx, key)
		if err != nil {
			return fmt.Errorf("build %s is not kept on the bus: %w", b.ID, err)
		}
		switch {
		case !found:
			_, err = c.Builds.Create(ctx, key, next)
			if errors.Is(err, chain.ErrExists) {
				continue
			}
		case cur.Value.State == chain.BuildAsked:
			next.Request = cur.Value.Request
			_, err = c.Builds.Update(ctx, key, next, cur.Revision)
			if errors.Is(err, chain.ErrMoved) {
				continue
			}
		default:
			// An outcome stands: the first record is the record.
			return i.RecordMirrored(ctx, b.ID, "", b.Mirrored)
		}
		if err != nil {
			return fmt.Errorf("build %s is not kept on the bus: %w", b.ID, err)
		}
		return i.RecordMirrored(ctx, b.ID, "", b.Mirrored)
	}
	return fmt.Errorf("build %s is not kept on the bus: it kept moving under the write", b.ID)
}

// casTries is how often a write that lost a compare-and-set reads the key again and tries once more.
const casTries = 8

// chainBuilds is every build of the bucket, whole, in no order.
func chainBuilds(ctx context.Context, c *chain.Chain) ([]Build, error) {
	all, err := c.Builds.All(ctx)
	if err != nil {
		return nil, fmt.Errorf("the builds cannot be read from the bus: %w", err)
	}
	out := make([]Build, 0, len(all))
	for _, k := range all {
		// A request nobody has answered is no build: it has no outcome to read.
		if k.Value.State == chain.BuildAsked {
			continue
		}
		out = append(out, fromChainBuild(k.Value))
	}
	return out, nil
}

// newestRequestOrder sorts builds as newestRequestFirst does: the newest request first, then the newest record.
func newestRequestOrder(builds []Build) {
	sort.SliceStable(builds, func(a, b int) bool {
		x, y := builds[a].AskedOrAt(), builds[b].AskedOrAt()
		if !x.Equal(y) {
			return x.After(y)
		}
		if !builds[a].At.Equal(builds[b].At) {
			return builds[a].At.After(builds[b].At)
		}
		return builds[a].ID > builds[b].ID
	})
}

// byModule sorts builds by module, then newest request first.
func byModule(builds []Build) {
	newestRequestOrder(builds)
	sort.SliceStable(builds, func(a, b int) bool { return builds[a].Module < builds[b].Module })
}

// worked is the builds that made something for a module: succeeded, and knew what they built.
func worked(all []Build) []Build {
	var out []Build
	for _, b := range all {
		if b.Failed == "" && b.Module != "" {
			out = append(out, b)
		}
	}
	return out
}

func (i *Inventory) chainBuildList(ctx context.Context, c *chain.Chain, module string, limit int) ([]Build, error) {
	all, err := chainBuilds(ctx, c)
	if err != nil {
		return nil, err
	}
	var out []Build
	for _, b := range all {
		if module == "" || b.Module == module {
			out = append(out, Build{ID: b.ID, Repository: b.Repository, Ref: b.Ref, Module: b.Module, Commit: b.Commit,
				On: b.On, Failed: b.Failed, Made: b.Made, Asked: b.Asked, At: b.At, ReuseKey: b.ReuseKey, Published: b.Published})
		}
	}
	sort.SliceStable(out, func(a, b int) bool {
		if !out[a].At.Equal(out[b].At) {
			return out[a].At.After(out[b].At)
		}
		return out[a].ID > out[b].ID
	})
	if len(out) > limit {
		out = out[:limit]
	}
	return out, nil
}

func (i *Inventory) chainBuildByID(ctx context.Context, c *chain.Chain, id string) (Build, bool, error) {
	k, ok, err := c.Builds.Get(ctx, chainBuildKey(id))
	if err != nil || !ok || k.Value.State == chain.BuildAsked {
		// A request nobody has answered has no outcome to read.
		return Build{}, false, err
	}
	b := fromChainBuild(k.Value)
	return Build{ID: b.ID, Repository: b.Repository, Ref: b.Ref, Module: b.Module, Commit: b.Commit, On: b.On,
		Failed: b.Failed, Made: b.Made, Path: b.Path, Asked: b.Asked, At: b.At}, true, nil
}

func (i *Inventory) chainBuiltAgainst(ctx context.Context, c *chain.Chain) (map[string][]string, error) {
	all, err := chainBuilds(ctx, c)
	if err != nil {
		return nil, err
	}
	ok := worked(all)
	byModule(ok)
	out := map[string][]string{}
	seen := map[string]bool{}
	for _, b := range ok {
		if seen[b.Module] {
			continue
		}
		seen[b.Module] = true
		if len(b.Against) > 0 {
			out[b.Module] = b.Against
		}
	}
	return out, nil
}

func (i *Inventory) chainReadRepositories(ctx context.Context, c *chain.Chain) (map[string][]ReadRepository, error) {
	all, err := chainBuilds(ctx, c)
	if err != nil {
		return nil, err
	}
	// The newest build of each module whatever its outcome: a failed one newer than the newest that worked leaves that
	// one's build source stale.
	type newestOf struct {
		at     time.Time
		failed bool
	}
	newest := map[string]newestOf{}
	tried := make([]Build, 0, len(all))
	for _, b := range all {
		if b.Module != "" {
			tried = append(tried, b)
		}
	}
	byModule(tried)
	for _, b := range tried {
		if _, seen := newest[b.Module]; !seen {
			newest[b.Module] = newestOf{b.AskedOrAt(), b.Failed != ""}
		}
	}
	owns, err := i.moduleSources(ctx)
	if err != nil {
		return nil, err
	}
	ok := worked(all)
	byModule(ok)
	read := map[string][]ReadRepository{}
	seen := map[string]bool{}
	for _, b := range ok {
		if seen[b.Module] {
			continue
		}
		seen[b.Module] = true
		built := b.AskedOrAt()
		of := append([]ReadRepository(nil), b.Read...)
		sources := b.Sources
		whole := ""
		if n, known := newest[b.Module]; known && n.failed && n.at.After(built) && len(sources) > 0 {
			sources, whole = nil, "a newer build of it failed"
		}
		of = WithBuildSources(of, sources)
		if own, has := owns[b.Module]; has {
			of = OwnRepositoryAsOwn(of, own)
		}
		if whole != "" {
			of = append(of, ReadRepository{Own: true, Whole: whole})
		}
		if len(of) > 0 {
			for k := range of {
				of[k].Built, of[k].Looked = built, newest[b.Module].at
			}
			read[b.Module] = of
		}
	}
	return read, nil
}

// moduleSources is the repository each registered module is registered from.
func (i *Inventory) moduleSources(ctx context.Context) (map[string]string, error) {
	rows, err := i.store.Pool().Query(ctx, `select name, source from module where source is not null`)
	if err != nil {
		return nil, err
	}
	defer rows.Close()
	out := map[string]string{}
	for rows.Next() {
		var name, source string
		if err := rows.Scan(&name, &source); err != nil {
			return nil, err
		}
		out[name] = source
	}
	return out, rows.Err()
}

func (i *Inventory) chainAnnounceable(ctx context.Context, c *chain.Chain) ([]Build, error) {
	all, err := chainBuilds(ctx, c)
	if err != nil {
		return nil, err
	}
	var ok []Build
	for _, b := range worked(all) {
		if b.Commit != "" {
			ok = append(ok, b)
		}
	}
	newestRequestOrder(ok)
	type key struct{ module, commit string }
	seen := map[key]bool{}
	var out []Build
	for _, b := range ok {
		k := key{b.Module, b.Commit}
		if seen[k] {
			continue
		}
		seen[k] = true
		out = append(out, Build{ID: b.ID, Repository: b.Repository, Ref: b.Ref, Module: b.Module, Commit: b.Commit,
			On: b.On, Failed: b.Failed, Made: b.Made, Path: b.Path, Manifest: b.Manifest, Against: b.Against, At: b.At})
	}
	sort.SliceStable(out, func(a, b int) bool { return out[a].At.Before(out[b].At) })
	return out, nil
}

// failedAtAGate is the builds that failed their gate. The gate's verdicts stay in their table (issue 1097).
func (i *Inventory) failedAtAGate(ctx context.Context) (map[string]bool, error) {
	rows, err := i.store.Pool().Query(ctx, `select build from build_gate where verdict = 'failed'`)
	if err != nil {
		return nil, err
	}
	defer rows.Close()
	out := map[string]bool{}
	for rows.Next() {
		var id string
		if err := rows.Scan(&id); err != nil {
			return nil, err
		}
		out[id] = true
	}
	return out, rows.Err()
}

func (i *Inventory) chainSourceRows(ctx context.Context, c *chain.Chain, module string) (map[string][]sourceRow, []string, error) {
	all, err := chainBuilds(ctx, c)
	if err != nil {
		return nil, nil, err
	}
	failed, err := i.failedAtAGate(ctx)
	if err != nil {
		return nil, nil, err
	}
	ok := worked(all)
	byModule(ok)
	out := map[string][]sourceRow{}
	var order []string
	for _, b := range ok {
		if module != "" && b.Module != module {
			continue
		}
		made, err := json.Marshal(b.Made)
		if err != nil {
			return nil, nil, err
		}
		if _, seen := out[b.Module]; !seen {
			order = append(order, b.Module)
		}
		out[b.Module] = append(out[b.Module], sourceRow{id: b.ID, commit: b.Commit, fingerprint: b.SourceFingerprint,
			made: made, manifest: b.Manifest, failedGate: failed[b.ID]})
	}
	return out, order, nil
}

// canonical is a JSON document encoded the one way (objects' keys sorted, compact), so the same content hashes alike
// however the bucket's record spells it.
func canonicalJSON(raw []byte) []byte {
	if len(bytes.TrimSpace(raw)) == 0 {
		return nil
	}
	out, err := canonical(raw)
	if err != nil {
		return raw
	}
	return out
}

// chainFingerprint is what a build would put on a machine, hashed: its artifacts and its manifest.
func chainFingerprint(b Build) string {
	made, _ := json.Marshal(b.Made)
	sum := sha256.Sum256(append(append(canonicalJSON(made), 0), canonicalJSON(b.Manifest)...))
	return hex.EncodeToString(sum[:])
}

func (i *Inventory) chainBuildFingerprints(ctx context.Context, c *chain.Chain, module string) (map[string]string, error) {
	all, err := chainBuilds(ctx, c)
	if err != nil {
		return nil, err
	}
	var ok []Build
	for _, b := range all {
		if b.Module == module && b.Failed == "" && b.Commit != "" {
			ok = append(ok, b)
		}
	}
	sortByRecorded(ok)
	if len(ok) > 50 {
		ok = ok[:50]
	}
	out := map[string]string{}
	for _, b := range ok {
		if _, seen := out[b.Commit]; !seen {
			out[b.Commit] = chainFingerprint(b)
		}
	}
	return out, nil
}

func (i *Inventory) chainFingerprints(ctx context.Context, c *chain.Chain) (map[string]map[string]string, error) {
	all, err := chainBuilds(ctx, c)
	if err != nil {
		return nil, err
	}
	var ok []Build
	for _, b := range all {
		if b.Module != "" && b.Failed == "" && b.Commit != "" {
			ok = append(ok, b)
		}
	}
	sortByRecorded(ok)
	out := map[string]map[string]string{}
	for _, b := range ok {
		if out[b.Module] == nil {
			out[b.Module] = map[string]string{}
		}
		if _, seen := out[b.Module][b.Commit]; !seen {
			out[b.Module][b.Commit] = chainFingerprint(b)
		}
	}
	return out, nil
}

// sortByRecorded is newest record first, as `order by at desc`.
func sortByRecorded(builds []Build) {
	sort.SliceStable(builds, func(a, b int) bool {
		if !builds[a].At.Equal(builds[b].At) {
			return builds[a].At.After(builds[b].At)
		}
		return builds[a].ID > builds[b].ID
	})
}

func (i *Inventory) chainBuildOf(ctx context.Context, c *chain.Chain, module, commit string) (string, error) {
	all, err := chainBuilds(ctx, c)
	if err != nil {
		return "", err
	}
	var ok []Build
	for _, b := range all {
		if b.Module == module && b.Commit == commit && b.Failed == "" {
			ok = append(ok, b)
		}
	}
	sortByRecorded(ok)
	if len(ok) == 0 {
		return "", nil
	}
	return ok[0].ID, nil
}

// chainManifestBuild is the newest successful build of a module from a commit (or one it abbreviates) that kept a manifest.
func (i *Inventory) chainManifestBuild(ctx context.Context, c *chain.Chain, module, commit string) (string, []byte, bool, error) {
	all, err := chainBuilds(ctx, c)
	if err != nil {
		return "", nil, false, err
	}
	var ok []Build
	for _, b := range all {
		if b.Module == module && b.Failed == "" && len(b.Manifest) > 0 && string(bytes.TrimSpace(b.Manifest)) != "null" &&
			(b.Commit == commit || strings.HasPrefix(b.Commit, commit)) {
			ok = append(ok, b)
		}
	}
	sortByRecorded(ok)
	if len(ok) == 0 {
		return "", nil, false, nil
	}
	return ok[0].ID, ok[0].Manifest, true, nil
}

// chainManifestOf is the manifest a build was recorded with.
func (i *Inventory) chainManifestOf(ctx context.Context, c *chain.Chain, id string) ([]byte, error) {
	k, ok, err := c.Builds.Get(ctx, chainBuildKey(id))
	if err != nil || !ok {
		return nil, err
	}
	return []byte(k.Value.Manifest), nil
}

// chainRecentMade is the artifacts and bases of the KeptBuilds newest successful builds of each registered module.
func (i *Inventory) chainRecentMade(ctx context.Context, c *chain.Chain) ([]Build, error) {
	all, err := chainBuilds(ctx, c)
	if err != nil {
		return nil, err
	}
	rows, err := i.store.Pool().Query(ctx, `select name from module`)
	if err != nil {
		return nil, err
	}
	registered := map[string]bool{}
	for rows.Next() {
		var name string
		if err := rows.Scan(&name); err != nil {
			rows.Close()
			return nil, err
		}
		registered[name] = true
	}
	rows.Close()
	if err := rows.Err(); err != nil {
		return nil, err
	}
	var ok []Build
	for _, b := range worked(all) {
		if registered[b.Module] {
			ok = append(ok, b)
		}
	}
	sortByRecorded(ok)
	back := map[string]int{}
	var out []Build
	for _, b := range ok {
		back[b.Module]++
		if back[b.Module] <= KeptBuilds {
			out = append(out, b)
		}
	}
	return out, nil
}

// chainEveryReferenceMade is every successful build's artifacts, oldest build first.
func (i *Inventory) chainEveryReferenceMade(ctx context.Context, c *chain.Chain) ([]Build, error) {
	all, err := chainBuilds(ctx, c)
	if err != nil {
		return nil, err
	}
	ok := worked(all)
	sort.SliceStable(ok, func(a, b int) bool {
		if !ok[a].At.Equal(ok[b].At) {
			return ok[a].At.Before(ok[b].At)
		}
		return ok[a].ID < ok[b].ID
	})
	return ok, nil
}