mesh-catalog/modules/artifact-store-tools/cmd/artifact-store-tools/definitions.go

package main

import (
	"context"
	"encoding/json"
	"fmt"
	"os"
	"regexp"
	"sort"
	"strings"
	"sync"
	"time"
)

// What the definitions pin in the store (novox/hq issue 323). A definition may pin an image of the store by
// digest that the mesh did not make: one pushed before the mesh built it, pinned under the store's current
// name or under a name the store had before. No record of the controller names such a manifest, and yet a
// machine runs it, so it is said to be named by a definition and never unrecorded.

// NamesFileEnv names the file the mesh writes from the setting older-store-names: a line `older=` with the
// names the store had before, as a definition may still write them — host names, or host:port, separated by
// spaces — or `none`. A catalogue names no installation, so the names are the mesh's setting, not the
// manifest's.
const NamesFileEnv = "MESH_ARTIFACT_STORE_NAMES_FILE"

// CatalogueModules and CatalogueModule are the catalogue's tools these definitions are read through.
const (
	CatalogueModules = "mesh-catalog.catalog_modules"
	CatalogueModule  = "mesh-catalog.catalog_module"
)

// DefinitionsTimeout bounds reading every definition, below the runtime's thirty-second call limit; what
// was read is kept, so a call that runs out reads only the rest the next time.
const DefinitionsTimeout = 20 * time.Second

// definitionReaders is how many definitions are asked at once.
const definitionReaders = 8

// StoreNames are the names a definition may pin an image of the store under: the scheme the mesh writes
// (artifact-store://), the store's address now, and the names it had before.
type StoreNames struct {
	Current []string
	Older   []string
}

// NamesOf is the store's names: its address as the controller answers it, and the older ones given.
func NamesOf(address, older string) StoreNames {
	n := StoreNames{Current: []string{"artifact-store://"}}
	if a := strings.Trim(strings.TrimSpace(address), "/"); a != "" {
		n.Current = append(n.Current, a+"/")
	}
	for _, o := range strings.FieldsFunc(older, func(r rune) bool { return r == ',' || r == ' ' || r == '\n' || r == '\t' }) {
		if o = strings.Trim(o, "/"); o != "" && o != "none" {
			n.Older = append(n.Older, o+"/")
		}
	}
	return n
}

// OlderNames reads the older names from the file the mesh writes, and says why when it cannot.
func OlderNames() (string, string) {
	path := os.Getenv(NamesFileEnv)
	if path == "" {
		return "", NamesFileEnv + " is not set, so no older name of the store was read"
	}
	raw, err := os.ReadFile(path)
	if err != nil {
		return "", "the store's older names could not be read: " + err.Error()
	}
	for _, line := range strings.Split(string(raw), "\n") {
		if v, ok := strings.CutPrefix(strings.TrimSpace(line), "older="); ok {
			return v, ""
		}
	}
	return "", path + " holds no line older=, so no older name of the store was read"
}

// Pin is one definition pinning a manifest of the store by digest.
type Pin struct {
	Module string `json:"module"`
	// Name is the store's name it is pinned under, as written: "artifact-store://", an address, or an older name.
	Name  string `json:"name"`
	Older bool   `json:"older,omitempty"`
}

// pinned is one pin found in a manifest: where in the store, and under which name.
type pinned struct {
	Repository, Digest, Name string
	Older                    bool
}

var pinShapes sync.Map // name -> *regexp.Regexp

// pinShape is `<name><repository>@sha256:<hex>`, the name standing on its own: not the tail of a longer host.
func pinShape(name string) *regexp.Regexp {
	if re, ok := pinShapes.Load(name); ok {
		return re.(*regexp.Regexp)
	}
	re := regexp.MustCompile(`(?:^|[^A-Za-z0-9.-])` + regexp.QuoteMeta(name) +
		`([a-z0-9][a-z0-9._/-]*)@(sha256:[0-9a-f]{64})`)
	pinShapes.Store(name, re)
	return re
}

// PinsIn finds every image of the store a manifest pins by digest, in any string it holds: a resource's
// image, a build's base, a file's content.
func PinsIn(manifest json.RawMessage, names StoreNames) []pinned {
	var doc any
	if json.Unmarshal(manifest, &doc) != nil {
		return nil
	}
	seen := map[string]bool{}
	var out []pinned
	var walk func(any)
	walk = func(x any) {
		switch v := x.(type) {
		case map[string]any:
			for _, e := range v {
				walk(e)
			}
		case []any:
			for _, e := range v {
				walk(e)
			}
		case string:
			for _, group := range []struct {
				names []string
				older bool
			}{{names.Current, false}, {names.Older, true}} {
				for _, name := range group.names {
					for _, m := range pinShape(name).FindAllStringSubmatch(v, -1) {
						p := pinned{Repository: strings.Trim(m[1], "/"), Digest: m[2], Name: name, Older: group.older}
						if k := p.Repository + "@" + p.Digest + " " + p.Name; !seen[k] {
							seen[k] = true
							out = append(out, p)
						}
					}
				}
			}
		}
	}
	walk(doc)
	return out
}

// Definitions is what the catalogue's definitions pin in the store, and what of them could not be read.
type Definitions struct {
	// Pins are by Key(repository, digest).
	Pins map[string][]Pin
	// Read is how many definitions were read; Unread says, per module, why its definition was not.
	Read   int
	Unread map[string]string
	// NotAsked says why the catalogue could not be asked at all, when it could not.
	NotAsked string
	// NamesNotRead says why the store's older names could not be read, when they could not.
	NamesNotRead string
	Names        StoreNames
}

// Catalogue reads the definitions through the catalogue's tools, and keeps each definition it read by its
// module and commit, which never change together.
type Catalogue struct {
	Ask Asking

	mu   sync.Mutex
	kept map[string][]pinned // "<module>@<commit>|<names>" -> its pins
}

type catalogueEntry struct {
	Module string `json:"module"`
	Commit string `json:"commit"`
}

// Read asks the catalogue for every module and reads each definition, bounded by DefinitionsTimeout.
func (c *Catalogue) Read(ctx context.Context, names StoreNames) *Definitions {
	defs := &Definitions{Pins: map[string][]Pin{}, Unread: map[string]string{}, Names: names}
	if c == nil || c.Ask == nil {
		defs.NotAsked = "this bundle has no way to ask the catalogue"
		return defs
	}
	raw, err := c.Ask(CatalogueModules, map[string]any{})
	if err != nil {
		defs.NotAsked = "the catalogue's modules could not be asked: " + err.Error()
		return defs
	}
	answer, output, ok := answerOf(raw)
	if !ok {
		defs.NotAsked = "the catalogue refused its modules: " + lastLine(output)
		return defs
	}
	var list struct {
		Modules []catalogueEntry `json:"modules"`
		Error   string           `json:"error"`
	}
	if err := json.Unmarshal(answer, &list); err != nil || list.Error != "" || list.Modules == nil {
		defs.NotAsked = "the catalogue's modules answer is not readable: " + firstOf(list.Error, errText(err), lastLine(output))
		return defs
	}
	c.mu.Lock()
	if c.kept == nil {
		c.kept = map[string][]pinned{}
	}
	c.mu.Unlock()
	nameKey := strings.Join(names.Current, ",") + ";" + strings.Join(names.Older, ",")

	ctx, cancel := context.WithTimeout(ctx, DefinitionsTimeout)
	defer cancel()
	type result struct {
		module string
		pins   []pinned
		why    string
	}
	results := make(chan result, len(list.Modules))
	work := make(chan catalogueEntry)
	var wg sync.WaitGroup
	for i := 0; i < definitionReaders; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			for m := range work {
				pins, why := c.definition(m, names, nameKey)
				results <- result{m.Module, pins, why}
			}
		}()
	}
	go func() {
		defer close(work)
		for _, m := range list.Modules {
			select {
			case work <- m:
			case <-ctx.Done():
				return
			}
		}
	}()
	go func() { wg.Wait(); close(results) }()

	got := map[string]bool{}
collect:
	for {
		select {
		case r, open := <-results:
			if !open {
				break collect
			}
			got[r.module] = true
			if r.why != "" {
				defs.Unread[r.module] = r.why
				continue
			}
			defs.Read++
			for _, p := range r.pins {
				k := Key(p.Repository, p.Digest)
				defs.Pins[k] = append(defs.Pins[k], Pin{Module: r.module, Name: p.Name, Older: p.Older})
			}
		case <-ctx.Done():
			break collect
		}
	}
	for _, m := range list.Modules {
		if !got[m.Module] {
			defs.Unread[m.Module] = fmt.Sprintf("not read within %d s", int(DefinitionsTimeout/time.Second))
		}
	}
	for k := range defs.Pins {
		sort.Slice(defs.Pins[k], func(i, j int) bool { return defs.Pins[k][i].Module < defs.Pins[k][j].Module })
	}
	return defs
}

// definition reads one module's definition, or answers it from what was read before.
func (c *Catalogue) definition(m catalogueEntry, names StoreNames, nameKey string) ([]pinned, string) {
	key := m.Module + "@" + m.Commit + "|" + nameKey
	if m.Commit != "" {
		c.mu.Lock()
		pins, had := c.kept[key]
		c.mu.Unlock()
		if had {
			return pins, ""
		}
	}
	args := map[string]any{"module": m.Module}
	if m.Commit != "" {
		args["commit"] = m.Commit
	}
	raw, err := c.Ask(CatalogueModule, args)
	if err != nil {
		return nil, "could not be asked: " + err.Error()
	}
	answer, output, ok := answerOf(raw)
	if !ok {
		return nil, "refused: " + lastLine(output)
	}
	var one struct {
		Module *struct {
			Manifest json.RawMessage `json:"manifest"`
		} `json:"module"`
		Error string `json:"error"`
	}
	if err := json.Unmarshal(answer, &one); err != nil || one.Error != "" || one.Module == nil || len(one.Module.Manifest) == 0 {
		return nil, "not readable: " + firstOf(one.Error, errText(err), "no manifest in the answer")
	}
	pins := PinsIn(one.Module.Manifest, names)
	if m.Commit != "" {
		c.mu.Lock()
		c.kept[key] = pins
		c.mu.Unlock()
	}
	return pins, ""
}

// why says which definitions name a manifest, and under which of the store's names.
func (p Pin) why() string {
	if p.Older {
		return fmt.Sprintf("a definition names it: %s, under %s, an older name of the store", p.Module, shownName(p.Name))
	}
	return "a definition names it: " + p.Module
}

func firstOf(s ...string) string {
	for _, x := range s {
		if strings.TrimSpace(x) != "" {
			return x
		}
	}
	return ""
}

func errText(err error) string {
	if err == nil {
		return ""
	}
	return err.Error()
}

// shownName is a name of the store as a person reads it: a host without the slash a pin puts after it, the
// scheme as written.
func shownName(name string) string {
	if strings.HasSuffix(name, "://") {
		return name
	}
	return strings.TrimSuffix(name, "/")
}