Take a module in from the builder's done key of its trunk fingerprint: one record of the key's artifacts, its manifest judged against the catalogue trunk's predefined keys (issue 1851, A3, A4b, A4c)

This commit is contained in:
2026-10-22 19:05:04 +02:00
parent f00b6aff8b
commit d984f0a55d
19 changed files with 1907 additions and 27 deletions
+427
View File
@@ -0,0 +1,427 @@
package main
import (
"archive/tar"
"bytes"
"compress/gzip"
"context"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"io"
"path"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/buildstep"
"git.novox.be/novox/mesh-sdk/go/catalogue"
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
"github.com/nats-io/nats.go/jetstream"
"github.com/novox/mesh-controller/internal/artifacts"
"github.com/novox/mesh-controller/internal/builderkeys"
"github.com/novox/mesh-controller/internal/buildreads"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// The take-in from the builder's done key (novox/hq ADR 0501, ADR 0498 §8, ADR 0453 §5; issue 1851).
//
// From R4e the controller asks no build but the builder's own module's: a module of a merge is taken from the builder's
// key for the fingerprint the indexer's trunk value of the module names (ADR 0501 §2). Only a done key whose every
// artifact names a reference is taken (ADR 0498 §8); a key not done yet, one waiting for a reference, a templates
// archive the store does not answer, or a list of predefined keys the forge does not answer is waited for and said,
// never failed and never delivered from. A failed key fails, in the builder's own words. The take-in then records the
// key as a build of today's record layout, so a controller of before reads it on a rollback, and judges it as every
// build is judged (registerJudged).
// errKeyWaits says a take-in from a key waits: for the builder, a reference, the store or the forge. Never a failure.
var errKeyWaits = errors.New("waits")
// errKeyFailed says the module's key failed: the builder's own words follow.
var errKeyFailed = errors.New("the builder's key failed")
// keyReads is what a take-in from a key reads of the builder and the indexer (builderkeys.Reader, or a test's).
type keyReads interface {
Build(ctx context.Context, module, fp string) (builderkeys.Key, error)
Manifest(ctx context.Context, digest string) (sdkdelivery.Manifest, error)
}
// keyBuildID is the id a key's build is recorded under: the key and its revision, so the same revision recorded twice
// is one record, and a rebuild of the same fingerprint (a new revision, ADR 0429 §18) another.
func keyBuildID(k builderkeys.Key) string {
part, err := sdkdelivery.FingerprintKeyPart(k.Fingerprint)
if err != nil {
part = strings.TrimPrefix(k.Fingerprint, sdkdelivery.SourcePrefix2)
}
return fmt.Sprintf("key-%s-%d", part, k.Revision)
}
// isKeyBuildID says a recorded build was taken from a builder's key.
func isKeyBuildID(id string) bool { return strings.HasPrefix(id, "key-") }
// readArchive is the archive a key's reference names, read from the artifact store and checked against its digest. A
// variable so a test needs no store.
var readArchive = func(ctx context.Context, inv *inventory.Inventory, reference string) ([]byte, error) {
shelf, err := inv.Catalogue(ctx)
if err != nil {
return nil, err
}
address, err := artifactStoreAddress(ctx, inv, shelf, "")
if err != nil {
return nil, err
}
if address == "" {
return nil, errors.New("the artifact store is on no network the controller reaches")
}
return artifacts.Store{Address: address}.GetBlob(ctx, reference)
}
// predefinedHead is the list of predefined keys at the head of the catalogue repository's trunk, and that head.
type predefinedHead struct {
List catalogue.PredefinedList
Repository string
Commit string
// Generation orders two heads of the trunk: the head's commit time in seconds. A list kept from a build's
// generation (its commit count) is always older than a head read here.
Generation int
}
// predefinedAtTrunk is the list of predefined keys at the head of the catalogue repository's trunk (ADR 0501 §8), read
// from the forge. A variable so a test needs no forge.
var predefinedAtTrunk = func(ctx context.Context, inv *inventory.Inventory) (predefinedHead, error) {
entries, err := inv.Catalogued(ctx)
if err != nil {
return predefinedHead{}, err
}
repository := ""
for _, e := range entries {
if e.Source.Seat == gitSeat && path.Base(strings.Trim(e.Source.Repository, "/")) == catalogue.CatalogueRepository {
repository = strings.Trim(e.Source.Repository, "/")
break
}
}
if repository == "" {
return predefinedHead{}, fmt.Errorf("no module of the catalogue names the repository %s on the forge",
catalogue.CatalogueRepository)
}
var heads []struct {
SHA string `json:"sha"`
Commit struct {
Committer struct {
Date time.Time `json:"date"`
} `json:"committer"`
} `json:"commit"`
}
if err := askGitea(ctx, "gitea_api", map[string]any{"path": "/repos/" + repository +
"/commits?limit=1&stat=false&verification=false&files=false"}, &heads); err != nil {
return predefinedHead{}, err
}
if len(heads) == 0 || heads[0].SHA == "" {
return predefinedHead{}, fmt.Errorf("the forge named no head of %s's trunk", repository)
}
head := predefinedHead{Repository: repository, Commit: heads[0].SHA,
Generation: int(heads[0].Commit.Committer.Date.Unix())}
var file struct {
Content string `json:"content"`
Encoding string `json:"encoding"`
}
if err := askGitea(ctx, "gitea_api", map[string]any{"path": "/repos/" + repository + "/contents/" +
catalogue.PredefinedFile + "?ref=" + head.Commit}, &file); err != nil {
return predefinedHead{}, err
}
if file.Encoding != "base64" {
return predefinedHead{}, fmt.Errorf("the forge answered %s at %.8s in the encoding %q", catalogue.PredefinedFile,
head.Commit, file.Encoding)
}
raw, err := base64.StdEncoding.DecodeString(file.Content)
if err != nil {
return predefinedHead{}, err
}
if head.List, err = catalogue.ParsePredefined(raw); err != nil {
return predefinedHead{}, fmt.Errorf("%s at %.8s is refused: %w", catalogue.PredefinedFile, head.Commit, err)
}
return head, nil
}
// askGitea asks the forge's module one tool on the bus and reads its result into into.
func askGitea(ctx context.Context, tool string, args map[string]any, into any) error {
js, err := aBus()
if err != nil {
return err
}
defer js.Close()
raw, _ := json.Marshal(args)
answer, err := link.Ask(ctx, link.OverNATS{Conn: js.Conn()}, "gitea", tool, raw, 30*time.Second)
if err != nil {
return err
}
if answer.Error != "" {
return fmt.Errorf("gitea.%s: %s", tool, answer.Error)
}
var wrapped struct {
Result json.RawMessage `json:"result"`
}
if err := json.Unmarshal(answer.Result, &wrapped); err != nil {
return err
}
return json.Unmarshal(wrapped.Result, into)
}
// keyTake is one take-in from a key: the module, the indexer's trunk value naming the key's fingerprint, and the
// registration's warrant.
type keyTake struct {
Module string
Trunk sdkdelivery.Module
Ask registrationAsk
}
// takeInKey takes a module in from the builder's key for the fingerprint its trunk value names: records the key as a
// build and registers the module, or says why not. errKeyWaits when it waits, errKeyFailed when the key failed.
func takeInKey(ctx context.Context, inv *inventory.Inventory, keys keyReads, t keyTake) (
catalogue.Manifest, inventory.Build, error) {
head, err := predefinedAtTrunk(ctx, inv)
if err != nil {
return catalogue.Manifest{}, inventory.Build{}, fmt.Errorf("%w: the list of predefined keys at the catalogue's "+
"trunk head cannot be read, so %s is not judged: %v", errKeyWaits, t.Module, err)
}
keepPredefinedHead(ctx, inv, head)
previous, err := inv.Builds(ctx, t.Module, 5)
if err != nil {
return catalogue.Manifest{}, inventory.Build{}, err
}
kept, manifest, err := buildOfKey(ctx, keys, t, keyContext{list: head.List, previous: previous,
archive: func(ctx context.Context, reference string) ([]byte, error) { return readArchive(ctx, inv, reference) }})
if err != nil {
return catalogue.Manifest{}, kept, err
}
if err := inv.RecordBuild(ctx, kept); err != nil {
return catalogue.Manifest{}, kept, err
}
recorded := inventory.Source{Repository: t.Trunk.Repository, Seat: gitSeat, Path: keyPath(t.Trunk),
BuiltFrom: t.Trunk.Commit, Head: t.Trunk.Commit, Against: kept.Against}
trunk := "main"
if was, err := inv.SourceOf(ctx, manifest.Module); err == nil {
recorded.Ref = followedBranch(was.Ref)
if recorded.Ref != "" {
trunk = recorded.Ref
}
}
registered, err := registerJudged(ctx, inv, takenBuild{kept: kept, manifest: manifest, recorded: recorded,
ask: t.Ask, path: recorded.Path, trunk: trunk, by: "the builder", what: t.Trunk.Repository,
commit: t.Trunk.Commit})
return registered, kept, err
}
// keyContext is what a key's build is made with besides the key: the list of predefined keys its manifest is judged
// against, the module's recent builds (the bases its last one stood on are carried) and how an archive is read.
type keyContext struct {
list catalogue.PredefinedList
previous []inventory.Build
archive func(ctx context.Context, reference string) ([]byte, error)
}
// keyPath is a module's directory as the catalogue records it: empty for a module at its repository's root.
func keyPath(m sdkdelivery.Module) string {
if m.Path == sdkdelivery.RootPath {
return ""
}
return strings.Trim(m.Path, "/")
}
// buildOfKey reads the key of the trunk value's fingerprint and makes its build record and the manifest it resolves to
// (ADR 0498 §8, ADR 0501 §1, §8): the module's manifest by the key's digest, judged against the list of predefined keys
// at the catalogue's trunk head, resolved with the key's artifacts.
func buildOfKey(ctx context.Context, keys keyReads, t keyTake, kc keyContext) (
inventory.Build, catalogue.Manifest, error) {
waits := func(format string, a ...any) (inventory.Build, catalogue.Manifest, error) {
return inventory.Build{}, catalogue.Manifest{}, fmt.Errorf("%w: "+format, append([]any{errKeyWaits}, a...)...)
}
if t.Trunk.FP2 == "" {
why := t.Trunk.None2
if why == "" {
why = "the indexer wrote no fingerprint of the new form"
}
return waits("%s at %.8s has no fingerprint to take a build by: %s", t.Module, t.Trunk.Commit, why)
}
k, err := keys.Build(ctx, t.Module, t.Trunk.FP2)
if err != nil {
return waits("%s's key cannot be read: %v", t.Module, err)
}
switch {
case k.Failed():
return inventory.Build{}, catalogue.Manifest{}, fmt.Errorf("%w: %s", errKeyFailed, k.Says())
case !k.Done():
return waits("%s", k.Says())
case len(k.Unreferenced()) > 0:
return waits("%s", k.Says())
}
src, err := keys.Manifest(ctx, k.Build.Manifest)
if err != nil {
return waits("%s's manifest: %v", t.Module, err)
}
manifest, err := catalogue.ParseManifestWith([]byte(src.JSON), kc.list)
if err != nil {
// **A word the running controller does not read waits for the controller that reads it** (ADR 0501 §7): the
// key stays done, and the controller that reads the word registers it with no second build.
if word := refusedWord(err.Error()); word != "" {
return waits("%s's manifest uses the word %q, which this controller does not read: it is registered "+
"when a controller that reads it runs", t.Module, word)
}
return inventory.Build{}, catalogue.Manifest{}, fmt.Errorf("the builder built %s and its manifest is "+
"refused: %w", t.Module, err)
}
if manifest.Module != t.Module {
return inventory.Build{}, catalogue.Manifest{}, fmt.Errorf("the key of %s names the manifest of %s",
t.Module, manifest.Module)
}
built, err := builtOfKey(ctx, kc.archive, manifest, k)
if err != nil {
return waits("%v", err)
}
resolved, err := manifest.Resolve(built)
if err != nil {
return inventory.Build{}, catalogue.Manifest{}, fmt.Errorf("the builder built %s and it does not resolve: %w",
t.Module, err)
}
raw, err := json.Marshal(resolved)
if err != nil {
return inventory.Build{}, catalogue.Manifest{}, err
}
kept := inventory.Build{ID: keyBuildID(k), Repository: t.Trunk.Repository, Module: t.Module,
Commit: t.Trunk.Commit, On: k.Build.By, Path: keyPath(t.Trunk), Manifest: raw,
SourceFingerprint: k.Fingerprint, At: time.Now().UTC()}
// **Read from the manifest, fresh at every key** (the same rule build-agent says, internal/buildreads). Sources are
// left unsaid: the planner then reads the module's whole directory, over-planning and never under-planning, and a
// fingerprint that did not change moves nothing (R4e A9).
for _, c := range buildreads.Of(manifest) {
kept.Read = append(kept.Read, inventory.ReadRepository{Repository: c.Repository, Ref: c.Ref})
}
// **The bases its last build stood on, carried** (R4e, lead decision): what the collector keeps a mirrored base by.
// A key names none; the module's newest successful build's are carried so the keep set does not change.
for _, b := range kc.previous {
if b.Worked() && b.Module == t.Module && b.ID != kept.ID {
kept.Against = append([]string(nil), b.Against...)
break
}
}
for _, a := range k.Build.Artifacts {
kept.Made = append(kept.Made, inventory.Artifact{Name: a.Name, Kind: kindOf(manifest, a),
Reference: catalogue.Recorded(a.Reference)})
}
return kept, resolved, nil
}
// kindOf is an artifact's kind: the key's, else the manifest's.
func kindOf(m catalogue.Manifest, a sdkdelivery.BuiltArtifact) string {
if a.Kind != "" || m.Build == nil {
return a.Kind
}
for _, d := range m.Build.Artifacts {
if d.Name == a.Name {
return d.Kind
}
}
return ""
}
// builtOfKey is each of a done key's artifacts as Resolve reads it: its reference and digest, a TypeScript bundle's
// launchers by the build modules' one rule (mesh-sdk buildstep.Launchers), and, for the archive a module packs its
// templates into, each template's text read back from the archive the store holds under the key's reference.
func builtOfKey(ctx context.Context, archive func(context.Context, string) ([]byte, error), m catalogue.Manifest,
k builderkeys.Key) (
[]catalogue.Built, error) {
declared := map[string]catalogue.Artifact{}
if m.Build != nil {
for _, a := range m.Build.Artifacts {
declared[a.Name] = a
}
}
templates := ""
if m.Templated() {
templates = catalogue.TemplatesArtifact(m)
}
var out []catalogue.Built
for _, a := range k.Build.Artifacts {
d := declared[a.Name]
b := catalogue.Built{Name: a.Name, Kind: kindOf(m, a), Reference: catalogue.Recorded(a.Reference),
Digest: a.Digest}
if launchers := buildstep.Launchers(d.Language, d.Entrypoints); len(launchers) > 0 {
b.Launchers = launchers
}
if a.Name == templates && templates != "" {
body, err := archive(ctx, b.Reference)
if err != nil {
return nil, fmt.Errorf("the templates archive %q of %s cannot be read from the artifact store: %v",
a.Name, m.Module, err)
}
if b.Texts, err = unpackTexts(body); err != nil {
return nil, fmt.Errorf("the templates archive %q of %s cannot be read back: %v", a.Name, m.Module, err)
}
}
out = append(out, b)
}
return out, nil
}
// unpackTexts is every file of a packed templates archive (gzip over tar), by its path inside it.
func unpackTexts(body []byte) (map[string]string, error) {
zipped, err := gzip.NewReader(bytes.NewReader(body))
if err != nil {
return nil, err
}
defer zipped.Close()
reader := tar.NewReader(zipped)
texts := map[string]string{}
for {
header, err := reader.Next()
if errors.Is(err, io.EOF) {
return texts, nil
}
if err != nil {
return nil, err
}
if header.Typeflag != tar.TypeReg {
continue
}
text, err := io.ReadAll(io.LimitReader(reader, 16<<20))
if err != nil {
return nil, err
}
texts[strings.TrimPrefix(header.Name, "./")] = string(text)
}
}
// keepPredefinedHead keeps the list read at the catalogue's trunk head when it is newer than the one kept, as a build of
// the catalogue's did (ADR 0348 §7), and judges every manifest read again against it. Said, never fatal: the list read
// is the one this take-in judges against either way.
func keepPredefinedHead(ctx context.Context, inv *inventory.Inventory, head predefinedHead) {
kept, err := inv.KeepPredefined(ctx, inventory.PredefinedRecord{Keys: head.List, Repository: head.Repository,
Commit: head.Commit, Generation: head.Generation, Build: "the trunk head " + short(head.Commit)})
if err != nil {
fmt.Printf("the list of predefined keys at %s's trunk head %s is not kept: %v\n", head.Repository,
short(head.Commit), err)
return
}
if kept {
fmt.Printf("the list of predefined keys at %s's trunk head %s is published: %s\n", head.Repository,
short(head.Commit), strings.Join(head.List.Keys(), ", "))
}
if err := loadPredefined(ctx, inv); err != nil {
fmt.Printf("the list of predefined keys kept cannot be read: %v\n", err)
}
}
// openBuilderKeys is the builder's and the indexer's buckets, read on a connection for the moment.
func openBuilderKeys() (builderkeys.Reader, func(), error) {
js, err := aBus()
if err != nil {
return builderkeys.Reader{}, nil, err
}
api, err := jetstream.New(js.Conn())
if err != nil {
js.Close()
return builderkeys.Reader{}, nil, err
}
return builderkeys.Reader{JS: api}, js.Close, nil
}
@@ -0,0 +1,192 @@
//go:build integration
package main
import (
"context"
"encoding/json"
"errors"
"reflect"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/catalogue"
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
"github.com/novox/mesh-controller/internal/builderkeys"
"github.com/novox/mesh-controller/internal/inventory"
)
// withTrunkList stands in for the catalogue's trunk head: an empty list of predefined keys, readable, or not.
func withTrunkList(t *testing.T, readable bool) {
t.Helper()
was := predefinedAtTrunk
predefinedAtTrunk = func(context.Context, *inventory.Inventory) (predefinedHead, error) {
if !readable {
return predefinedHead{}, errors.New("the forge did not answer")
}
list, err := catalogue.ParsePredefined([]byte(`{}`))
return predefinedHead{List: list, Repository: "novox/mesh-catalog", Commit: "abcdef0123456789",
Generation: 1 << 40}, err
}
t.Cleanup(func() { predefinedAtTrunk = was })
}
// aKeyFor is a done key of module at fp, of one archive with a reference, and the indexer's trunk value naming it at
// commit in repository/path.
func aKeyFor(module, version, fp, repository, path, commit string) (keyBuckets, sdkdelivery.Module) {
raw := `{"module":"` + module + `","version":"` + version + `","build":{"artifacts":[{"name":"a","kind":"archive","from":"files"}]},` +
`"resources":[{"id":"a","type":"archive","path":"/opt/` + module + `","artifact":"a"}]}`
digest := sdkdelivery.ManifestDigest([]byte(raw))
blob := shaOf(module + version)
k := builderkeys.Key{Module: module, Fingerprint: fp, Found: true, Revision: 1,
Build: sdkdelivery.BuilderBuild{State: sdkdelivery.BuildDone, By: "ace", Manifest: digest,
Artifacts: []sdkdelivery.BuiltArtifact{{Name: "a", Kind: "archive", Digest: blob,
Reference: "artifact-store://" + module + "/a/blobs/" + blob}}}}
return keyBuckets{keys: map[string]builderkeys.Key{module: k}, manifests: map[string]string{digest: raw}},
sdkdelivery.Module{FP2: fp, Manifest: digest, Repository: repository, Commit: commit, Path: path}
}
func fpOf(c byte) string { return "src2:" + strings.Repeat(string(c), 64) }
// A3, A4 (ADR 0498 §8, ADR 0501 §1-§3, §8): a done key of a trunk fingerprint records one build of today's layout,
// whose artifacts are the key's, and registers the module under the same rules of who may register as before.
func TestADoneKeyOfTheTrunkRegistersUnderTheSameRules(t *testing.T) {
open := theCatalogue(t) // sudo from novox/mesh-catalog modules/sudo, mesh-host from novox/mesh-host
ctx := t.Context()
inv := open.inventory
withTrunkList(t, true)
// A known module from the repository and path the catalogue builds it from: registered from its key.
keys, trunk := aKeyFor("sudo", "2", fpOf('a'), "novox/mesh-catalog", "modules/sudo", "c0ffee0123456789")
_, kept, err := takeInKey(ctx, inv, keys, keyTake{Module: "sudo", Trunk: trunk, Ask: registrationAsk{Merge: trunk.Commit}})
if err != nil {
t.Fatalf("0501 §1: a done key of sudo's trunk fingerprint was not taken in: %v", err)
}
if shelf, _ := inv.Catalogue(ctx); shelf["sudo"].Version != "2" {
t.Fatalf("0501 §1: sudo is %q", shelf["sudo"].Version)
}
got, found, err := inv.BuildByID(ctx, kept.ID)
if err != nil || !found || len(got.Made) != 1 || got.Made[0].Reference != kept.Made[0].Reference ||
got.SourceFingerprint != fpOf('a') {
t.Fatalf("0498 §8: the recorded build is %+v (%v)", got, err)
}
// Recording the same key again is one record.
if _, _, err := takeInKey(ctx, inv, keys, keyTake{Module: "sudo", Trunk: trunk, Ask: registrationAsk{Merge: trunk.Commit}}); err != nil {
t.Fatal(err)
}
builds, _ := inv.Builds(ctx, "sudo", 20)
n := 0
for _, b := range builds {
if b.ID == kept.ID {
n++
}
}
if n != 1 {
t.Fatalf("0498 §8: one key taken in twice is %d records", n)
}
// A known module from another repository: refused, as a build of it was (ADR 0266).
other, otherTrunk := aKeyFor("sudo", "evil", fpOf('b'), "agent/sudo", "modules/sudo", "d00d0123456789ab")
if _, _, err := takeInKey(ctx, inv, other, keyTake{Module: "sudo", Trunk: otherTrunk,
Ask: registrationAsk{Merge: otherTrunk.Commit}}); !errors.Is(err, errNotItsSource) {
t.Fatalf("0501 §3: a known module from another repository was taken in: %v", err)
}
// A new module from a repository the catalogue builds from: registered.
zram, zramTrunk := aKeyFor("zram", "1", fpOf('c'), "novox/mesh-catalog", "modules/zram", "c0ffee0123456789")
if _, _, err := takeInKey(ctx, inv, zram, keyTake{Module: "zram", Trunk: zramTrunk, Ask: registrationAsk{Merge: zramTrunk.Commit}}); err != nil {
t.Fatalf("0501 §3: a new module from a built repository was refused: %v", err)
}
// A new module from a repository nothing is built from, with no Approve: refused.
tools, toolsTrunk := aKeyFor("agent-tools", "1", fpOf('d'), "agent/tools", "", "e0e0e0e0e0e0e0e0")
if _, _, err := takeInKey(ctx, inv, tools, keyTake{Module: "agent-tools", Trunk: toolsTrunk,
Ask: registrationAsk{Merge: toolsTrunk.Commit}}); !errors.Is(err, errNotItsSource) {
t.Fatalf("0501 §3: a new repository's module was taken in with no warrant: %v", err)
}
// At the terminal the operator registers it (ADR 0501 §4).
if _, _, err := takeInKey(ctx, inv, tools, keyTake{Module: "agent-tools", Trunk: toolsTrunk,
Ask: registrationAsk{Terminal: true}}); err != nil {
t.Fatalf("0501 §4: the terminal's registration was refused: %v", err)
}
}
// ADR 0501 §2: a key only a candidate names registers nothing: the trunk value names another fingerprint, whose key is
// not there, and the take-in waits.
func TestAKeyOnlyACandidateNamesRegistersNothing(t *testing.T) {
open := theCatalogue(t)
ctx := t.Context()
withTrunkList(t, true)
keys, trunk := aKeyFor("sudo", "3", fpOf('e'), "novox/mesh-catalog", "modules/sudo", "c0ffee0123456789")
trunk.FP2 = fpOf('f') // the trunk names another fingerprint than the candidate's done key
_, _, err := takeInKey(ctx, open.inventory, keys, keyTake{Module: "sudo", Trunk: trunk, Ask: registrationAsk{Merge: trunk.Commit}})
if !errors.Is(err, errKeyWaits) {
t.Fatalf("0501 §2: %v", err)
}
if shelf, _ := open.inventory.Catalogue(ctx); shelf["sudo"].Version != "1" {
t.Fatalf("0501 §2: a candidate's key registered sudo %q", shelf["sudo"].Version)
}
}
// ADR 0501 §8: a take-in that cannot read the list of predefined keys at the catalogue's trunk head waits, says so and
// registers nothing.
func TestATakeInWithoutTheTrunksListWaits(t *testing.T) {
open := theCatalogue(t)
ctx := t.Context()
withTrunkList(t, false)
keys, trunk := aKeyFor("sudo", "4", fpOf('1'), "novox/mesh-catalog", "modules/sudo", "c0ffee0123456789")
_, _, err := takeInKey(ctx, open.inventory, keys, keyTake{Module: "sudo", Trunk: trunk, Ask: registrationAsk{Merge: trunk.Commit}})
if !errors.Is(err, errKeyWaits) || !strings.Contains(err.Error(), "predefined keys") {
t.Fatalf("0501 §8: %v", err)
}
if shelf, _ := open.inventory.Catalogue(ctx); shelf["sudo"].Version != "1" {
t.Fatalf("0501 §8: registered %q", shelf["sudo"].Version)
}
}
// A3 (the lead's decision on Against and Read): a key's record keeps the collector's keep set as it was and its read
// repositories are its manifest's, fresh at every key.
func TestAKeysRecordKeepsTheKeepSetAndReadsItsManifest(t *testing.T) {
open := theCatalogue(t)
ctx := t.Context()
inv := open.inventory
withTrunkList(t, true)
before, err := inv.KeepSet(ctx)
if err != nil {
t.Fatal(err)
}
raw := `{"module":"sudo","version":"5","build":{"artifacts":[{"name":"img","kind":"image","from":"Dockerfile",` +
`"context":{"repository":"novox/elsewhere","seat":"git"}}]},"resources":[{"id":"run","type":"container","artifact":"img"}]}`
digest := sdkdelivery.ManifestDigest([]byte(raw))
k := builderkeys.Key{Module: "sudo", Fingerprint: fpOf('2'), Found: true, Revision: 1,
Build: sdkdelivery.BuilderBuild{State: sdkdelivery.BuildDone, By: "g14", Manifest: digest,
Artifacts: []sdkdelivery.BuiltArtifact{{Name: "img", Kind: "image", Digest: shaOf("img5"),
Reference: "artifact-store://sudo/img@" + shaOf("img5")}}}}
keys := keyBuckets{keys: map[string]builderkeys.Key{"sudo": k}, manifests: map[string]string{digest: raw}}
trunk := sdkdelivery.Module{FP2: fpOf('2'), Manifest: digest, Repository: "novox/mesh-catalog",
Commit: "c0ffee0123456789", Path: "modules/sudo"}
if _, _, err := takeInKey(ctx, inv, keys, keyTake{Module: "sudo", Trunk: trunk, Ask: registrationAsk{Merge: trunk.Commit}}); err != nil {
t.Fatal(err)
}
after, err := inv.KeepSet(ctx)
if err != nil {
t.Fatal(err)
}
for reference, why := range before {
if !reflect.DeepEqual(after[reference], why) && !strings.Contains(strings.Join(after[reference], ","), strings.Join(why, ",")) {
t.Errorf("the keep set changed for %s: %v, was %v", reference, after[reference], why)
}
}
read, err := inv.ReadRepositories(ctx)
if err != nil {
t.Fatal(err)
}
found := false
for _, r := range read["sudo"] {
if !r.Own && r.Repository == "novox/elsewhere" {
found = true
}
}
if !found {
text, _ := json.Marshal(read["sudo"])
t.Fatalf("the key's record does not read its manifest's context: %s", text)
}
}
+231
View File
@@ -0,0 +1,231 @@
package main
import (
"archive/tar"
"bytes"
"compress/gzip"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/catalogue"
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
"github.com/novox/mesh-controller/internal/builderkeys"
"github.com/novox/mesh-controller/internal/inventory"
)
// keyBuckets is the builder's and the indexer's buckets as a take-in reads them, for a test: keys by module, manifests by
// digest.
type keyBuckets struct {
keys map[string]builderkeys.Key
manifests map[string]string
}
func (k keyBuckets) Build(_ context.Context, module, fp string) (builderkeys.Key, error) {
if key, ok := k.keys[module]; ok && key.Fingerprint == fp {
return key, nil
}
return builderkeys.Key{Module: module, Fingerprint: fp}, nil
}
func (k keyBuckets) Manifest(_ context.Context, digest string) (sdkdelivery.Manifest, error) {
raw, ok := k.manifests[digest]
if !ok {
return sdkdelivery.Manifest{}, errors.New("the indexer holds no manifest " + digest)
}
return sdkdelivery.Manifest{Module: "x", JSON: raw}, nil
}
const keyFP = "src2:" + "1111111111111111111111111111111111111111111111111111111111111111"
// aKeyOf is a done key of module x at keyFP for the manifest raw, with these artifacts.
func aKeyOf(raw string, revision uint64, made ...sdkdelivery.BuiltArtifact) (keyBuckets, sdkdelivery.Module) {
digest := sdkdelivery.ManifestDigest([]byte(raw))
k := builderkeys.Key{Module: "x", Fingerprint: keyFP, Found: true, Revision: revision,
Build: sdkdelivery.BuilderBuild{State: sdkdelivery.BuildDone, By: "ace", Manifest: digest, Artifacts: made}}
trunk := sdkdelivery.Module{FP2: keyFP, Manifest: digest, Repository: "novox/mesh-catalog", Commit: "c0ffee0123456789",
Path: "modules/x"}
return keyBuckets{keys: map[string]builderkeys.Key{"x": k}, manifests: map[string]string{digest: raw}}, trunk
}
const tsBundle = `{"module":"x","version":"1","tools":["x_read"],
"build":{"artifacts":[{"name":"tools","kind":"bundle","language":"typescript","entrypoints":["dist/index.js"]},
{"name":"img","kind":"image","from":"Dockerfile","context":{"repository":"novox/other","seat":"git"}}]},
"resources":[{"id":"run","type":"container","artifact":"img"}]}`
func shaOf(b string) string {
sum := sha256.Sum256([]byte(b))
return "sha256:" + hex.EncodeToString(sum[:])
}
// A3 (ADR 0498 §8, ADR 0501 §1): a done key whose every artifact names a reference is one recorded build, its artifacts
// the key's, its launchers by the build modules' rule, its reads from its manifest, the bases its last build stood on
// carried; recorded twice it is one record.
func TestADoneKeyIsOneRecordedBuildOfItsArtifacts(t *testing.T) {
keys, trunk := aKeyOf(tsBundle, 3,
sdkdelivery.BuiltArtifact{Name: "tools", Kind: "bundle", Digest: shaOf("t"),
Reference: "artifact-store://x/tools/blobs/" + shaOf("t")},
sdkdelivery.BuiltArtifact{Name: "img", Kind: "image", Digest: shaOf("i"),
Reference: "artifact-store://x/img@" + shaOf("i")})
previous := []inventory.Build{{ID: "build-1", Module: "x", Against: []string{"artifact-store://base@sha256:aa"}}}
take := keyTake{Module: "x", Trunk: trunk}
kept, resolved, err := buildOfKey(t.Context(), keys, take, keyContext{previous: previous})
if err != nil {
t.Fatal(err)
}
if kept.ID != "key-1111111111111111111111111111111111111111111111111111111111111111-3" {
t.Errorf("the record's id is %s", kept.ID)
}
again, _, _ := buildOfKey(t.Context(), keys, take, keyContext{previous: previous})
if again.ID != kept.ID {
t.Errorf("one key recorded twice is two records: %s and %s", kept.ID, again.ID)
}
if kept.SourceFingerprint != keyFP || kept.Commit != trunk.Commit || kept.Path != "modules/x" || kept.On != "ace" ||
kept.Module != "x" || kept.Repository != "novox/mesh-catalog" {
t.Errorf("the record: %+v", kept)
}
if len(kept.Made) != 2 || kept.Made[0].Reference != "artifact-store://x/tools/blobs/"+shaOf("t") ||
kept.Made[1].Reference != "artifact-store://x/img@"+shaOf("i") || kept.Made[1].Kind != "image" {
t.Errorf("Made is not the key's artifacts: %+v", kept.Made)
}
if len(kept.Read) != 1 || kept.Read[0].Repository != "novox/other" {
t.Errorf("Read is not the manifest's contexts: %+v", kept.Read)
}
if kept.Sources != nil || kept.Mirrored != nil {
t.Errorf("a key's record says build sources or mirrored bases it cannot know: %+v %+v", kept.Sources, kept.Mirrored)
}
if len(kept.Against) != 1 || kept.Against[0] != "artifact-store://base@sha256:aa" {
t.Errorf("the bases the last build stood on are not carried: %v", kept.Against)
}
if len(resolved.Bundles) != 1 || resolved.Bundles[0].Launchers["dist/index.js"] != "dist/index.serve.mjs" {
t.Errorf("the bundle's launchers: %+v", resolved.Bundles)
}
var m map[string]any
if err := json.Unmarshal(kept.Manifest, &m); err != nil || !strings.Contains(string(kept.Manifest), shaOf("i")) {
t.Errorf("the recorded manifest names no image digest: %s", kept.Manifest)
}
}
// A3 (ADR 0498 §8): a key with an artifact naming no reference, a key not done and a missing key are waited for, said;
// a failed key fails in the builder's words; a module with no fingerprint of the new form waits, naming why.
func TestAKeyNotReadyWaitsAndAFailedKeyFails(t *testing.T) {
noRef, trunk := aKeyOf(tsBundle, 1,
sdkdelivery.BuiltArtifact{Name: "tools", Kind: "bundle", Digest: shaOf("t"),
Reference: "artifact-store://x/tools/blobs/" + shaOf("t")},
sdkdelivery.BuiltArtifact{Name: "img", Kind: "image", Digest: shaOf("i")})
take := keyTake{Module: "x", Trunk: trunk}
_, _, err := buildOfKey(t.Context(), noRef, take, keyContext{})
if !errors.Is(err, errKeyWaits) || !strings.Contains(err.Error(), "waits for the reference of img") {
t.Errorf("a key without a reference: %v", err)
}
building := noRef
k := building.keys["x"]
k.Build = sdkdelivery.BuilderBuild{State: sdkdelivery.BuildBuilding, By: "g14"}
building.keys = map[string]builderkeys.Key{"x": k}
if _, _, err := buildOfKey(t.Context(), building, take, keyContext{}); !errors.Is(err, errKeyWaits) ||
!strings.Contains(err.Error(), "building on g14") {
t.Errorf("a building key: %v", err)
}
if _, _, err := buildOfKey(t.Context(), keyBuckets{}, take, keyContext{}); !errors.Is(err, errKeyWaits) ||
!strings.Contains(err.Error(), "no build of x") {
t.Errorf("no key: %v", err)
}
k.Build = sdkdelivery.BuilderBuild{State: sdkdelivery.BuildFailed, Error: "digests differ: ace sha256:1, g14 sha256:2"}
failed := keyBuckets{keys: map[string]builderkeys.Key{"x": k}}
if _, _, err := buildOfKey(t.Context(), failed, take, keyContext{}); !errors.Is(err, errKeyFailed) ||
!strings.Contains(err.Error(), "digests differ: ace sha256:1, g14 sha256:2") {
t.Errorf("a failed key: %v", err)
}
none := take
none.Trunk.FP2, none.Trunk.None2 = "", "no lock"
if _, _, err := buildOfKey(t.Context(), noRef, none, keyContext{}); !errors.Is(err, errKeyWaits) ||
!strings.Contains(err.Error(), "no lock") {
t.Errorf("no fingerprint: %v", err)
}
}
// ADR 0501 §7: a manifest word this controller does not read waits for the controller that reads it; never fails.
func TestAManifestWordTheControllerDoesNotReadWaits(t *testing.T) {
raw := `{"module":"x","version":"1","not-a-word-yet":true,"build":{"artifacts":[{"name":"a","kind":"archive","from":"files"}]},
"resources":[{"id":"a","type":"archive","path":"/opt/x","artifact":"a"}]}`
keys, trunk := aKeyOf(raw, 1, sdkdelivery.BuiltArtifact{Name: "a", Kind: "archive", Digest: shaOf("a"),
Reference: "artifact-store://x/a/blobs/" + shaOf("a")})
_, _, err := buildOfKey(t.Context(), keys, keyTake{Module: "x", Trunk: trunk}, keyContext{})
if !errors.Is(err, errKeyWaits) || !strings.Contains(err.Error(), `"not-a-word-yet"`) {
t.Fatalf("an unread word: %v", err)
}
}
func packed(t *testing.T, files map[string]string) []byte {
t.Helper()
var buf bytes.Buffer
z := gzip.NewWriter(&buf)
w := tar.NewWriter(z)
for name, text := range files {
if err := w.WriteHeader(&tar.Header{Name: name, Mode: 0o644, Size: int64(len(text)), Typeflag: tar.TypeReg}); err != nil {
t.Fatal(err)
}
if _, err := w.Write([]byte(text)); err != nil {
t.Fatal(err)
}
}
if err := w.Close(); err != nil {
t.Fatal(err)
}
if err := z.Close(); err != nil {
t.Fatal(err)
}
return buf.Bytes()
}
// A templated module's texts are read back from the archive the store holds under the key's reference; an archive the
// store does not answer is waited for (the lead's Q1b).
func TestATemplatedModulesTextsAreReadFromItsArchive(t *testing.T) {
raw := `{"module":"x","version":"1","templates":"templates",
"build":{"artifacts":[{"name":"templates","kind":"archive","from":"templates"}]},
"resources":[{"id":"main","type":"file","path":"/etc/x.conf","mode":"0644","template":"templates/x.conf"}]}`
body := packed(t, map[string]string{"x.conf": "listen 1\n"})
sum := sha256.Sum256(body)
digest := "sha256:" + hex.EncodeToString(sum[:])
keys, trunk := aKeyOf(raw, 1, sdkdelivery.BuiltArtifact{Name: "templates", Kind: "archive", Digest: digest,
Reference: "artifact-store://x/templates/blobs/" + digest})
asked := ""
archive := func(_ context.Context, reference string) ([]byte, error) { asked = reference; return body, nil }
_, resolved, err := buildOfKey(t.Context(), keys, keyTake{Module: "x", Trunk: trunk}, keyContext{archive: archive})
if err != nil {
t.Fatal(err)
}
if asked != "artifact-store://x/templates/blobs/"+digest {
t.Errorf("the archive asked: %q", asked)
}
if text, _ := json.Marshal(resolved.Resources); !strings.Contains(string(text), "listen 1") {
t.Errorf("the template's text is not in the file: %s", text)
}
gone := func(context.Context, string) ([]byte, error) { return nil, errors.New("the store does not hold it") }
if _, _, err := buildOfKey(t.Context(), keys, keyTake{Module: "x", Trunk: trunk}, keyContext{archive: gone}); !errors.Is(err, errKeyWaits) ||
!strings.Contains(err.Error(), "cannot be read from the artifact store") {
t.Errorf("an archive the store does not answer: %v", err)
}
}
// The launchers a key's bundle is resolved with are mesh-sdk's one rule (buildstep.Launchers), the names
// mesh-build-node writes: a TypeScript bundle's entrypoint x.js runs as x.serve.mjs, a Go bundle has none.
func TestALaunchersRuleIsTheBuildModules(t *testing.T) {
m, err := catalogue.ParseManifest([]byte(tsBundle))
if err != nil {
t.Fatal(err)
}
k := builderkeys.Key{Build: sdkdelivery.BuilderBuild{Artifacts: []sdkdelivery.BuiltArtifact{{Name: "tools", Digest: "sha256:1",
Reference: "artifact-store://x/tools/blobs/sha256:1"}}}}
built, err := builtOfKey(t.Context(), nil, m, k)
if err != nil {
t.Fatal(err)
}
if built[0].Launchers["dist/index.js"] != "dist/index.serve.mjs" || built[0].Kind != "bundle" {
t.Fatalf("built: %+v", built[0])
}
}