Take a module in from the builder's done key of its trunk fingerprint: one record of the key's artifacts, its manifest judged against the catalogue trunk's predefined keys (issue 1851, A3, A4b, A4c)
This commit is contained in:
@@ -0,0 +1,427 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"path"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
"github.com/novox/mesh-controller/internal/artifacts"
|
||||
"github.com/novox/mesh-controller/internal/builderkeys"
|
||||
"github.com/novox/mesh-controller/internal/buildreads"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The take-in from the builder's done key (novox/hq ADR 0501, ADR 0498 §8, ADR 0453 §5; issue 1851).
|
||||
//
|
||||
// From R4e the controller asks no build but the builder's own module's: a module of a merge is taken from the builder's
|
||||
// key for the fingerprint the indexer's trunk value of the module names (ADR 0501 §2). Only a done key whose every
|
||||
// artifact names a reference is taken (ADR 0498 §8); a key not done yet, one waiting for a reference, a templates
|
||||
// archive the store does not answer, or a list of predefined keys the forge does not answer is waited for and said,
|
||||
// never failed and never delivered from. A failed key fails, in the builder's own words. The take-in then records the
|
||||
// key as a build of today's record layout, so a controller of before reads it on a rollback, and judges it as every
|
||||
// build is judged (registerJudged).
|
||||
|
||||
// errKeyWaits says a take-in from a key waits: for the builder, a reference, the store or the forge. Never a failure.
|
||||
var errKeyWaits = errors.New("waits")
|
||||
|
||||
// errKeyFailed says the module's key failed: the builder's own words follow.
|
||||
var errKeyFailed = errors.New("the builder's key failed")
|
||||
|
||||
// keyReads is what a take-in from a key reads of the builder and the indexer (builderkeys.Reader, or a test's).
|
||||
type keyReads interface {
|
||||
Build(ctx context.Context, module, fp string) (builderkeys.Key, error)
|
||||
Manifest(ctx context.Context, digest string) (sdkdelivery.Manifest, error)
|
||||
}
|
||||
|
||||
// keyBuildID is the id a key's build is recorded under: the key and its revision, so the same revision recorded twice
|
||||
// is one record, and a rebuild of the same fingerprint (a new revision, ADR 0429 §18) another.
|
||||
func keyBuildID(k builderkeys.Key) string {
|
||||
part, err := sdkdelivery.FingerprintKeyPart(k.Fingerprint)
|
||||
if err != nil {
|
||||
part = strings.TrimPrefix(k.Fingerprint, sdkdelivery.SourcePrefix2)
|
||||
}
|
||||
return fmt.Sprintf("key-%s-%d", part, k.Revision)
|
||||
}
|
||||
|
||||
// isKeyBuildID says a recorded build was taken from a builder's key.
|
||||
func isKeyBuildID(id string) bool { return strings.HasPrefix(id, "key-") }
|
||||
|
||||
// readArchive is the archive a key's reference names, read from the artifact store and checked against its digest. A
|
||||
// variable so a test needs no store.
|
||||
var readArchive = func(ctx context.Context, inv *inventory.Inventory, reference string) ([]byte, error) {
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
address, err := artifactStoreAddress(ctx, inv, shelf, "")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if address == "" {
|
||||
return nil, errors.New("the artifact store is on no network the controller reaches")
|
||||
}
|
||||
return artifacts.Store{Address: address}.GetBlob(ctx, reference)
|
||||
}
|
||||
|
||||
// predefinedHead is the list of predefined keys at the head of the catalogue repository's trunk, and that head.
|
||||
type predefinedHead struct {
|
||||
List catalogue.PredefinedList
|
||||
Repository string
|
||||
Commit string
|
||||
// Generation orders two heads of the trunk: the head's commit time in seconds. A list kept from a build's
|
||||
// generation (its commit count) is always older than a head read here.
|
||||
Generation int
|
||||
}
|
||||
|
||||
// predefinedAtTrunk is the list of predefined keys at the head of the catalogue repository's trunk (ADR 0501 §8), read
|
||||
// from the forge. A variable so a test needs no forge.
|
||||
var predefinedAtTrunk = func(ctx context.Context, inv *inventory.Inventory) (predefinedHead, error) {
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return predefinedHead{}, err
|
||||
}
|
||||
repository := ""
|
||||
for _, e := range entries {
|
||||
if e.Source.Seat == gitSeat && path.Base(strings.Trim(e.Source.Repository, "/")) == catalogue.CatalogueRepository {
|
||||
repository = strings.Trim(e.Source.Repository, "/")
|
||||
break
|
||||
}
|
||||
}
|
||||
if repository == "" {
|
||||
return predefinedHead{}, fmt.Errorf("no module of the catalogue names the repository %s on the forge",
|
||||
catalogue.CatalogueRepository)
|
||||
}
|
||||
var heads []struct {
|
||||
SHA string `json:"sha"`
|
||||
Commit struct {
|
||||
Committer struct {
|
||||
Date time.Time `json:"date"`
|
||||
} `json:"committer"`
|
||||
} `json:"commit"`
|
||||
}
|
||||
if err := askGitea(ctx, "gitea_api", map[string]any{"path": "/repos/" + repository +
|
||||
"/commits?limit=1&stat=false&verification=false&files=false"}, &heads); err != nil {
|
||||
return predefinedHead{}, err
|
||||
}
|
||||
if len(heads) == 0 || heads[0].SHA == "" {
|
||||
return predefinedHead{}, fmt.Errorf("the forge named no head of %s's trunk", repository)
|
||||
}
|
||||
head := predefinedHead{Repository: repository, Commit: heads[0].SHA,
|
||||
Generation: int(heads[0].Commit.Committer.Date.Unix())}
|
||||
var file struct {
|
||||
Content string `json:"content"`
|
||||
Encoding string `json:"encoding"`
|
||||
}
|
||||
if err := askGitea(ctx, "gitea_api", map[string]any{"path": "/repos/" + repository + "/contents/" +
|
||||
catalogue.PredefinedFile + "?ref=" + head.Commit}, &file); err != nil {
|
||||
return predefinedHead{}, err
|
||||
}
|
||||
if file.Encoding != "base64" {
|
||||
return predefinedHead{}, fmt.Errorf("the forge answered %s at %.8s in the encoding %q", catalogue.PredefinedFile,
|
||||
head.Commit, file.Encoding)
|
||||
}
|
||||
raw, err := base64.StdEncoding.DecodeString(file.Content)
|
||||
if err != nil {
|
||||
return predefinedHead{}, err
|
||||
}
|
||||
if head.List, err = catalogue.ParsePredefined(raw); err != nil {
|
||||
return predefinedHead{}, fmt.Errorf("%s at %.8s is refused: %w", catalogue.PredefinedFile, head.Commit, err)
|
||||
}
|
||||
return head, nil
|
||||
}
|
||||
|
||||
// askGitea asks the forge's module one tool on the bus and reads its result into into.
|
||||
func askGitea(ctx context.Context, tool string, args map[string]any, into any) error {
|
||||
js, err := aBus()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer js.Close()
|
||||
raw, _ := json.Marshal(args)
|
||||
answer, err := link.Ask(ctx, link.OverNATS{Conn: js.Conn()}, "gitea", tool, raw, 30*time.Second)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if answer.Error != "" {
|
||||
return fmt.Errorf("gitea.%s: %s", tool, answer.Error)
|
||||
}
|
||||
var wrapped struct {
|
||||
Result json.RawMessage `json:"result"`
|
||||
}
|
||||
if err := json.Unmarshal(answer.Result, &wrapped); err != nil {
|
||||
return err
|
||||
}
|
||||
return json.Unmarshal(wrapped.Result, into)
|
||||
}
|
||||
|
||||
// keyTake is one take-in from a key: the module, the indexer's trunk value naming the key's fingerprint, and the
|
||||
// registration's warrant.
|
||||
type keyTake struct {
|
||||
Module string
|
||||
Trunk sdkdelivery.Module
|
||||
Ask registrationAsk
|
||||
}
|
||||
|
||||
// takeInKey takes a module in from the builder's key for the fingerprint its trunk value names: records the key as a
|
||||
// build and registers the module, or says why not. errKeyWaits when it waits, errKeyFailed when the key failed.
|
||||
func takeInKey(ctx context.Context, inv *inventory.Inventory, keys keyReads, t keyTake) (
|
||||
catalogue.Manifest, inventory.Build, error) {
|
||||
head, err := predefinedAtTrunk(ctx, inv)
|
||||
if err != nil {
|
||||
return catalogue.Manifest{}, inventory.Build{}, fmt.Errorf("%w: the list of predefined keys at the catalogue's "+
|
||||
"trunk head cannot be read, so %s is not judged: %v", errKeyWaits, t.Module, err)
|
||||
}
|
||||
keepPredefinedHead(ctx, inv, head)
|
||||
previous, err := inv.Builds(ctx, t.Module, 5)
|
||||
if err != nil {
|
||||
return catalogue.Manifest{}, inventory.Build{}, err
|
||||
}
|
||||
kept, manifest, err := buildOfKey(ctx, keys, t, keyContext{list: head.List, previous: previous,
|
||||
archive: func(ctx context.Context, reference string) ([]byte, error) { return readArchive(ctx, inv, reference) }})
|
||||
if err != nil {
|
||||
return catalogue.Manifest{}, kept, err
|
||||
}
|
||||
if err := inv.RecordBuild(ctx, kept); err != nil {
|
||||
return catalogue.Manifest{}, kept, err
|
||||
}
|
||||
recorded := inventory.Source{Repository: t.Trunk.Repository, Seat: gitSeat, Path: keyPath(t.Trunk),
|
||||
BuiltFrom: t.Trunk.Commit, Head: t.Trunk.Commit, Against: kept.Against}
|
||||
trunk := "main"
|
||||
if was, err := inv.SourceOf(ctx, manifest.Module); err == nil {
|
||||
recorded.Ref = followedBranch(was.Ref)
|
||||
if recorded.Ref != "" {
|
||||
trunk = recorded.Ref
|
||||
}
|
||||
}
|
||||
registered, err := registerJudged(ctx, inv, takenBuild{kept: kept, manifest: manifest, recorded: recorded,
|
||||
ask: t.Ask, path: recorded.Path, trunk: trunk, by: "the builder", what: t.Trunk.Repository,
|
||||
commit: t.Trunk.Commit})
|
||||
return registered, kept, err
|
||||
}
|
||||
|
||||
// keyContext is what a key's build is made with besides the key: the list of predefined keys its manifest is judged
|
||||
// against, the module's recent builds (the bases its last one stood on are carried) and how an archive is read.
|
||||
type keyContext struct {
|
||||
list catalogue.PredefinedList
|
||||
previous []inventory.Build
|
||||
archive func(ctx context.Context, reference string) ([]byte, error)
|
||||
}
|
||||
|
||||
// keyPath is a module's directory as the catalogue records it: empty for a module at its repository's root.
|
||||
func keyPath(m sdkdelivery.Module) string {
|
||||
if m.Path == sdkdelivery.RootPath {
|
||||
return ""
|
||||
}
|
||||
return strings.Trim(m.Path, "/")
|
||||
}
|
||||
|
||||
// buildOfKey reads the key of the trunk value's fingerprint and makes its build record and the manifest it resolves to
|
||||
// (ADR 0498 §8, ADR 0501 §1, §8): the module's manifest by the key's digest, judged against the list of predefined keys
|
||||
// at the catalogue's trunk head, resolved with the key's artifacts.
|
||||
func buildOfKey(ctx context.Context, keys keyReads, t keyTake, kc keyContext) (
|
||||
inventory.Build, catalogue.Manifest, error) {
|
||||
waits := func(format string, a ...any) (inventory.Build, catalogue.Manifest, error) {
|
||||
return inventory.Build{}, catalogue.Manifest{}, fmt.Errorf("%w: "+format, append([]any{errKeyWaits}, a...)...)
|
||||
}
|
||||
if t.Trunk.FP2 == "" {
|
||||
why := t.Trunk.None2
|
||||
if why == "" {
|
||||
why = "the indexer wrote no fingerprint of the new form"
|
||||
}
|
||||
return waits("%s at %.8s has no fingerprint to take a build by: %s", t.Module, t.Trunk.Commit, why)
|
||||
}
|
||||
k, err := keys.Build(ctx, t.Module, t.Trunk.FP2)
|
||||
if err != nil {
|
||||
return waits("%s's key cannot be read: %v", t.Module, err)
|
||||
}
|
||||
switch {
|
||||
case k.Failed():
|
||||
return inventory.Build{}, catalogue.Manifest{}, fmt.Errorf("%w: %s", errKeyFailed, k.Says())
|
||||
case !k.Done():
|
||||
return waits("%s", k.Says())
|
||||
case len(k.Unreferenced()) > 0:
|
||||
return waits("%s", k.Says())
|
||||
}
|
||||
src, err := keys.Manifest(ctx, k.Build.Manifest)
|
||||
if err != nil {
|
||||
return waits("%s's manifest: %v", t.Module, err)
|
||||
}
|
||||
manifest, err := catalogue.ParseManifestWith([]byte(src.JSON), kc.list)
|
||||
if err != nil {
|
||||
// **A word the running controller does not read waits for the controller that reads it** (ADR 0501 §7): the
|
||||
// key stays done, and the controller that reads the word registers it with no second build.
|
||||
if word := refusedWord(err.Error()); word != "" {
|
||||
return waits("%s's manifest uses the word %q, which this controller does not read: it is registered "+
|
||||
"when a controller that reads it runs", t.Module, word)
|
||||
}
|
||||
return inventory.Build{}, catalogue.Manifest{}, fmt.Errorf("the builder built %s and its manifest is "+
|
||||
"refused: %w", t.Module, err)
|
||||
}
|
||||
if manifest.Module != t.Module {
|
||||
return inventory.Build{}, catalogue.Manifest{}, fmt.Errorf("the key of %s names the manifest of %s",
|
||||
t.Module, manifest.Module)
|
||||
}
|
||||
built, err := builtOfKey(ctx, kc.archive, manifest, k)
|
||||
if err != nil {
|
||||
return waits("%v", err)
|
||||
}
|
||||
resolved, err := manifest.Resolve(built)
|
||||
if err != nil {
|
||||
return inventory.Build{}, catalogue.Manifest{}, fmt.Errorf("the builder built %s and it does not resolve: %w",
|
||||
t.Module, err)
|
||||
}
|
||||
raw, err := json.Marshal(resolved)
|
||||
if err != nil {
|
||||
return inventory.Build{}, catalogue.Manifest{}, err
|
||||
}
|
||||
kept := inventory.Build{ID: keyBuildID(k), Repository: t.Trunk.Repository, Module: t.Module,
|
||||
Commit: t.Trunk.Commit, On: k.Build.By, Path: keyPath(t.Trunk), Manifest: raw,
|
||||
SourceFingerprint: k.Fingerprint, At: time.Now().UTC()}
|
||||
// **Read from the manifest, fresh at every key** (the same rule build-agent says, internal/buildreads). Sources are
|
||||
// left unsaid: the planner then reads the module's whole directory, over-planning and never under-planning, and a
|
||||
// fingerprint that did not change moves nothing (R4e A9).
|
||||
for _, c := range buildreads.Of(manifest) {
|
||||
kept.Read = append(kept.Read, inventory.ReadRepository{Repository: c.Repository, Ref: c.Ref})
|
||||
}
|
||||
// **The bases its last build stood on, carried** (R4e, lead decision): what the collector keeps a mirrored base by.
|
||||
// A key names none; the module's newest successful build's are carried so the keep set does not change.
|
||||
for _, b := range kc.previous {
|
||||
if b.Worked() && b.Module == t.Module && b.ID != kept.ID {
|
||||
kept.Against = append([]string(nil), b.Against...)
|
||||
break
|
||||
}
|
||||
}
|
||||
for _, a := range k.Build.Artifacts {
|
||||
kept.Made = append(kept.Made, inventory.Artifact{Name: a.Name, Kind: kindOf(manifest, a),
|
||||
Reference: catalogue.Recorded(a.Reference)})
|
||||
}
|
||||
return kept, resolved, nil
|
||||
}
|
||||
|
||||
// kindOf is an artifact's kind: the key's, else the manifest's.
|
||||
func kindOf(m catalogue.Manifest, a sdkdelivery.BuiltArtifact) string {
|
||||
if a.Kind != "" || m.Build == nil {
|
||||
return a.Kind
|
||||
}
|
||||
for _, d := range m.Build.Artifacts {
|
||||
if d.Name == a.Name {
|
||||
return d.Kind
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// builtOfKey is each of a done key's artifacts as Resolve reads it: its reference and digest, a TypeScript bundle's
|
||||
// launchers by the build modules' one rule (mesh-sdk buildstep.Launchers), and, for the archive a module packs its
|
||||
// templates into, each template's text read back from the archive the store holds under the key's reference.
|
||||
func builtOfKey(ctx context.Context, archive func(context.Context, string) ([]byte, error), m catalogue.Manifest,
|
||||
k builderkeys.Key) (
|
||||
[]catalogue.Built, error) {
|
||||
declared := map[string]catalogue.Artifact{}
|
||||
if m.Build != nil {
|
||||
for _, a := range m.Build.Artifacts {
|
||||
declared[a.Name] = a
|
||||
}
|
||||
}
|
||||
templates := ""
|
||||
if m.Templated() {
|
||||
templates = catalogue.TemplatesArtifact(m)
|
||||
}
|
||||
var out []catalogue.Built
|
||||
for _, a := range k.Build.Artifacts {
|
||||
d := declared[a.Name]
|
||||
b := catalogue.Built{Name: a.Name, Kind: kindOf(m, a), Reference: catalogue.Recorded(a.Reference),
|
||||
Digest: a.Digest}
|
||||
if launchers := buildstep.Launchers(d.Language, d.Entrypoints); len(launchers) > 0 {
|
||||
b.Launchers = launchers
|
||||
}
|
||||
if a.Name == templates && templates != "" {
|
||||
body, err := archive(ctx, b.Reference)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the templates archive %q of %s cannot be read from the artifact store: %v",
|
||||
a.Name, m.Module, err)
|
||||
}
|
||||
if b.Texts, err = unpackTexts(body); err != nil {
|
||||
return nil, fmt.Errorf("the templates archive %q of %s cannot be read back: %v", a.Name, m.Module, err)
|
||||
}
|
||||
}
|
||||
out = append(out, b)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// unpackTexts is every file of a packed templates archive (gzip over tar), by its path inside it.
|
||||
func unpackTexts(body []byte) (map[string]string, error) {
|
||||
zipped, err := gzip.NewReader(bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer zipped.Close()
|
||||
reader := tar.NewReader(zipped)
|
||||
texts := map[string]string{}
|
||||
for {
|
||||
header, err := reader.Next()
|
||||
if errors.Is(err, io.EOF) {
|
||||
return texts, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if header.Typeflag != tar.TypeReg {
|
||||
continue
|
||||
}
|
||||
text, err := io.ReadAll(io.LimitReader(reader, 16<<20))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
texts[strings.TrimPrefix(header.Name, "./")] = string(text)
|
||||
}
|
||||
}
|
||||
|
||||
// keepPredefinedHead keeps the list read at the catalogue's trunk head when it is newer than the one kept, as a build of
|
||||
// the catalogue's did (ADR 0348 §7), and judges every manifest read again against it. Said, never fatal: the list read
|
||||
// is the one this take-in judges against either way.
|
||||
func keepPredefinedHead(ctx context.Context, inv *inventory.Inventory, head predefinedHead) {
|
||||
kept, err := inv.KeepPredefined(ctx, inventory.PredefinedRecord{Keys: head.List, Repository: head.Repository,
|
||||
Commit: head.Commit, Generation: head.Generation, Build: "the trunk head " + short(head.Commit)})
|
||||
if err != nil {
|
||||
fmt.Printf("the list of predefined keys at %s's trunk head %s is not kept: %v\n", head.Repository,
|
||||
short(head.Commit), err)
|
||||
return
|
||||
}
|
||||
if kept {
|
||||
fmt.Printf("the list of predefined keys at %s's trunk head %s is published: %s\n", head.Repository,
|
||||
short(head.Commit), strings.Join(head.List.Keys(), ", "))
|
||||
}
|
||||
if err := loadPredefined(ctx, inv); err != nil {
|
||||
fmt.Printf("the list of predefined keys kept cannot be read: %v\n", err)
|
||||
}
|
||||
}
|
||||
|
||||
// openBuilderKeys is the builder's and the indexer's buckets, read on a connection for the moment.
|
||||
func openBuilderKeys() (builderkeys.Reader, func(), error) {
|
||||
js, err := aBus()
|
||||
if err != nil {
|
||||
return builderkeys.Reader{}, nil, err
|
||||
}
|
||||
api, err := jetstream.New(js.Conn())
|
||||
if err != nil {
|
||||
js.Close()
|
||||
return builderkeys.Reader{}, nil, err
|
||||
}
|
||||
return builderkeys.Reader{JS: api}, js.Close, nil
|
||||
}
|
||||
@@ -0,0 +1,192 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"github.com/novox/mesh-controller/internal/builderkeys"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// withTrunkList stands in for the catalogue's trunk head: an empty list of predefined keys, readable, or not.
|
||||
func withTrunkList(t *testing.T, readable bool) {
|
||||
t.Helper()
|
||||
was := predefinedAtTrunk
|
||||
predefinedAtTrunk = func(context.Context, *inventory.Inventory) (predefinedHead, error) {
|
||||
if !readable {
|
||||
return predefinedHead{}, errors.New("the forge did not answer")
|
||||
}
|
||||
list, err := catalogue.ParsePredefined([]byte(`{}`))
|
||||
return predefinedHead{List: list, Repository: "novox/mesh-catalog", Commit: "abcdef0123456789",
|
||||
Generation: 1 << 40}, err
|
||||
}
|
||||
t.Cleanup(func() { predefinedAtTrunk = was })
|
||||
}
|
||||
|
||||
// aKeyFor is a done key of module at fp, of one archive with a reference, and the indexer's trunk value naming it at
|
||||
// commit in repository/path.
|
||||
func aKeyFor(module, version, fp, repository, path, commit string) (keyBuckets, sdkdelivery.Module) {
|
||||
raw := `{"module":"` + module + `","version":"` + version + `","build":{"artifacts":[{"name":"a","kind":"archive","from":"files"}]},` +
|
||||
`"resources":[{"id":"a","type":"archive","path":"/opt/` + module + `","artifact":"a"}]}`
|
||||
digest := sdkdelivery.ManifestDigest([]byte(raw))
|
||||
blob := shaOf(module + version)
|
||||
k := builderkeys.Key{Module: module, Fingerprint: fp, Found: true, Revision: 1,
|
||||
Build: sdkdelivery.BuilderBuild{State: sdkdelivery.BuildDone, By: "ace", Manifest: digest,
|
||||
Artifacts: []sdkdelivery.BuiltArtifact{{Name: "a", Kind: "archive", Digest: blob,
|
||||
Reference: "artifact-store://" + module + "/a/blobs/" + blob}}}}
|
||||
return keyBuckets{keys: map[string]builderkeys.Key{module: k}, manifests: map[string]string{digest: raw}},
|
||||
sdkdelivery.Module{FP2: fp, Manifest: digest, Repository: repository, Commit: commit, Path: path}
|
||||
}
|
||||
|
||||
func fpOf(c byte) string { return "src2:" + strings.Repeat(string(c), 64) }
|
||||
|
||||
// A3, A4 (ADR 0498 §8, ADR 0501 §1-§3, §8): a done key of a trunk fingerprint records one build of today's layout,
|
||||
// whose artifacts are the key's, and registers the module under the same rules of who may register as before.
|
||||
func TestADoneKeyOfTheTrunkRegistersUnderTheSameRules(t *testing.T) {
|
||||
open := theCatalogue(t) // sudo from novox/mesh-catalog modules/sudo, mesh-host from novox/mesh-host
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
withTrunkList(t, true)
|
||||
|
||||
// A known module from the repository and path the catalogue builds it from: registered from its key.
|
||||
keys, trunk := aKeyFor("sudo", "2", fpOf('a'), "novox/mesh-catalog", "modules/sudo", "c0ffee0123456789")
|
||||
_, kept, err := takeInKey(ctx, inv, keys, keyTake{Module: "sudo", Trunk: trunk, Ask: registrationAsk{Merge: trunk.Commit}})
|
||||
if err != nil {
|
||||
t.Fatalf("0501 §1: a done key of sudo's trunk fingerprint was not taken in: %v", err)
|
||||
}
|
||||
if shelf, _ := inv.Catalogue(ctx); shelf["sudo"].Version != "2" {
|
||||
t.Fatalf("0501 §1: sudo is %q", shelf["sudo"].Version)
|
||||
}
|
||||
got, found, err := inv.BuildByID(ctx, kept.ID)
|
||||
if err != nil || !found || len(got.Made) != 1 || got.Made[0].Reference != kept.Made[0].Reference ||
|
||||
got.SourceFingerprint != fpOf('a') {
|
||||
t.Fatalf("0498 §8: the recorded build is %+v (%v)", got, err)
|
||||
}
|
||||
// Recording the same key again is one record.
|
||||
if _, _, err := takeInKey(ctx, inv, keys, keyTake{Module: "sudo", Trunk: trunk, Ask: registrationAsk{Merge: trunk.Commit}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
builds, _ := inv.Builds(ctx, "sudo", 20)
|
||||
n := 0
|
||||
for _, b := range builds {
|
||||
if b.ID == kept.ID {
|
||||
n++
|
||||
}
|
||||
}
|
||||
if n != 1 {
|
||||
t.Fatalf("0498 §8: one key taken in twice is %d records", n)
|
||||
}
|
||||
|
||||
// A known module from another repository: refused, as a build of it was (ADR 0266).
|
||||
other, otherTrunk := aKeyFor("sudo", "evil", fpOf('b'), "agent/sudo", "modules/sudo", "d00d0123456789ab")
|
||||
if _, _, err := takeInKey(ctx, inv, other, keyTake{Module: "sudo", Trunk: otherTrunk,
|
||||
Ask: registrationAsk{Merge: otherTrunk.Commit}}); !errors.Is(err, errNotItsSource) {
|
||||
t.Fatalf("0501 §3: a known module from another repository was taken in: %v", err)
|
||||
}
|
||||
// A new module from a repository the catalogue builds from: registered.
|
||||
zram, zramTrunk := aKeyFor("zram", "1", fpOf('c'), "novox/mesh-catalog", "modules/zram", "c0ffee0123456789")
|
||||
if _, _, err := takeInKey(ctx, inv, zram, keyTake{Module: "zram", Trunk: zramTrunk, Ask: registrationAsk{Merge: zramTrunk.Commit}}); err != nil {
|
||||
t.Fatalf("0501 §3: a new module from a built repository was refused: %v", err)
|
||||
}
|
||||
// A new module from a repository nothing is built from, with no Approve: refused.
|
||||
tools, toolsTrunk := aKeyFor("agent-tools", "1", fpOf('d'), "agent/tools", "", "e0e0e0e0e0e0e0e0")
|
||||
if _, _, err := takeInKey(ctx, inv, tools, keyTake{Module: "agent-tools", Trunk: toolsTrunk,
|
||||
Ask: registrationAsk{Merge: toolsTrunk.Commit}}); !errors.Is(err, errNotItsSource) {
|
||||
t.Fatalf("0501 §3: a new repository's module was taken in with no warrant: %v", err)
|
||||
}
|
||||
// At the terminal the operator registers it (ADR 0501 §4).
|
||||
if _, _, err := takeInKey(ctx, inv, tools, keyTake{Module: "agent-tools", Trunk: toolsTrunk,
|
||||
Ask: registrationAsk{Terminal: true}}); err != nil {
|
||||
t.Fatalf("0501 §4: the terminal's registration was refused: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// ADR 0501 §2: a key only a candidate names registers nothing: the trunk value names another fingerprint, whose key is
|
||||
// not there, and the take-in waits.
|
||||
func TestAKeyOnlyACandidateNamesRegistersNothing(t *testing.T) {
|
||||
open := theCatalogue(t)
|
||||
ctx := t.Context()
|
||||
withTrunkList(t, true)
|
||||
keys, trunk := aKeyFor("sudo", "3", fpOf('e'), "novox/mesh-catalog", "modules/sudo", "c0ffee0123456789")
|
||||
trunk.FP2 = fpOf('f') // the trunk names another fingerprint than the candidate's done key
|
||||
_, _, err := takeInKey(ctx, open.inventory, keys, keyTake{Module: "sudo", Trunk: trunk, Ask: registrationAsk{Merge: trunk.Commit}})
|
||||
if !errors.Is(err, errKeyWaits) {
|
||||
t.Fatalf("0501 §2: %v", err)
|
||||
}
|
||||
if shelf, _ := open.inventory.Catalogue(ctx); shelf["sudo"].Version != "1" {
|
||||
t.Fatalf("0501 §2: a candidate's key registered sudo %q", shelf["sudo"].Version)
|
||||
}
|
||||
}
|
||||
|
||||
// ADR 0501 §8: a take-in that cannot read the list of predefined keys at the catalogue's trunk head waits, says so and
|
||||
// registers nothing.
|
||||
func TestATakeInWithoutTheTrunksListWaits(t *testing.T) {
|
||||
open := theCatalogue(t)
|
||||
ctx := t.Context()
|
||||
withTrunkList(t, false)
|
||||
keys, trunk := aKeyFor("sudo", "4", fpOf('1'), "novox/mesh-catalog", "modules/sudo", "c0ffee0123456789")
|
||||
_, _, err := takeInKey(ctx, open.inventory, keys, keyTake{Module: "sudo", Trunk: trunk, Ask: registrationAsk{Merge: trunk.Commit}})
|
||||
if !errors.Is(err, errKeyWaits) || !strings.Contains(err.Error(), "predefined keys") {
|
||||
t.Fatalf("0501 §8: %v", err)
|
||||
}
|
||||
if shelf, _ := open.inventory.Catalogue(ctx); shelf["sudo"].Version != "1" {
|
||||
t.Fatalf("0501 §8: registered %q", shelf["sudo"].Version)
|
||||
}
|
||||
}
|
||||
|
||||
// A3 (the lead's decision on Against and Read): a key's record keeps the collector's keep set as it was and its read
|
||||
// repositories are its manifest's, fresh at every key.
|
||||
func TestAKeysRecordKeepsTheKeepSetAndReadsItsManifest(t *testing.T) {
|
||||
open := theCatalogue(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
withTrunkList(t, true)
|
||||
before, err := inv.KeepSet(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
raw := `{"module":"sudo","version":"5","build":{"artifacts":[{"name":"img","kind":"image","from":"Dockerfile",` +
|
||||
`"context":{"repository":"novox/elsewhere","seat":"git"}}]},"resources":[{"id":"run","type":"container","artifact":"img"}]}`
|
||||
digest := sdkdelivery.ManifestDigest([]byte(raw))
|
||||
k := builderkeys.Key{Module: "sudo", Fingerprint: fpOf('2'), Found: true, Revision: 1,
|
||||
Build: sdkdelivery.BuilderBuild{State: sdkdelivery.BuildDone, By: "g14", Manifest: digest,
|
||||
Artifacts: []sdkdelivery.BuiltArtifact{{Name: "img", Kind: "image", Digest: shaOf("img5"),
|
||||
Reference: "artifact-store://sudo/img@" + shaOf("img5")}}}}
|
||||
keys := keyBuckets{keys: map[string]builderkeys.Key{"sudo": k}, manifests: map[string]string{digest: raw}}
|
||||
trunk := sdkdelivery.Module{FP2: fpOf('2'), Manifest: digest, Repository: "novox/mesh-catalog",
|
||||
Commit: "c0ffee0123456789", Path: "modules/sudo"}
|
||||
if _, _, err := takeInKey(ctx, inv, keys, keyTake{Module: "sudo", Trunk: trunk, Ask: registrationAsk{Merge: trunk.Commit}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
after, err := inv.KeepSet(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for reference, why := range before {
|
||||
if !reflect.DeepEqual(after[reference], why) && !strings.Contains(strings.Join(after[reference], ","), strings.Join(why, ",")) {
|
||||
t.Errorf("the keep set changed for %s: %v, was %v", reference, after[reference], why)
|
||||
}
|
||||
}
|
||||
read, err := inv.ReadRepositories(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
found := false
|
||||
for _, r := range read["sudo"] {
|
||||
if !r.Own && r.Repository == "novox/elsewhere" {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
text, _ := json.Marshal(read["sudo"])
|
||||
t.Fatalf("the key's record does not read its manifest's context: %s", text)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,231 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"github.com/novox/mesh-controller/internal/builderkeys"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// keyBuckets is the builder's and the indexer's buckets as a take-in reads them, for a test: keys by module, manifests by
|
||||
// digest.
|
||||
type keyBuckets struct {
|
||||
keys map[string]builderkeys.Key
|
||||
manifests map[string]string
|
||||
}
|
||||
|
||||
func (k keyBuckets) Build(_ context.Context, module, fp string) (builderkeys.Key, error) {
|
||||
if key, ok := k.keys[module]; ok && key.Fingerprint == fp {
|
||||
return key, nil
|
||||
}
|
||||
return builderkeys.Key{Module: module, Fingerprint: fp}, nil
|
||||
}
|
||||
|
||||
func (k keyBuckets) Manifest(_ context.Context, digest string) (sdkdelivery.Manifest, error) {
|
||||
raw, ok := k.manifests[digest]
|
||||
if !ok {
|
||||
return sdkdelivery.Manifest{}, errors.New("the indexer holds no manifest " + digest)
|
||||
}
|
||||
return sdkdelivery.Manifest{Module: "x", JSON: raw}, nil
|
||||
}
|
||||
|
||||
const keyFP = "src2:" + "1111111111111111111111111111111111111111111111111111111111111111"
|
||||
|
||||
// aKeyOf is a done key of module x at keyFP for the manifest raw, with these artifacts.
|
||||
func aKeyOf(raw string, revision uint64, made ...sdkdelivery.BuiltArtifact) (keyBuckets, sdkdelivery.Module) {
|
||||
digest := sdkdelivery.ManifestDigest([]byte(raw))
|
||||
k := builderkeys.Key{Module: "x", Fingerprint: keyFP, Found: true, Revision: revision,
|
||||
Build: sdkdelivery.BuilderBuild{State: sdkdelivery.BuildDone, By: "ace", Manifest: digest, Artifacts: made}}
|
||||
trunk := sdkdelivery.Module{FP2: keyFP, Manifest: digest, Repository: "novox/mesh-catalog", Commit: "c0ffee0123456789",
|
||||
Path: "modules/x"}
|
||||
return keyBuckets{keys: map[string]builderkeys.Key{"x": k}, manifests: map[string]string{digest: raw}}, trunk
|
||||
}
|
||||
|
||||
const tsBundle = `{"module":"x","version":"1","tools":["x_read"],
|
||||
"build":{"artifacts":[{"name":"tools","kind":"bundle","language":"typescript","entrypoints":["dist/index.js"]},
|
||||
{"name":"img","kind":"image","from":"Dockerfile","context":{"repository":"novox/other","seat":"git"}}]},
|
||||
"resources":[{"id":"run","type":"container","artifact":"img"}]}`
|
||||
|
||||
func shaOf(b string) string {
|
||||
sum := sha256.Sum256([]byte(b))
|
||||
return "sha256:" + hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
// A3 (ADR 0498 §8, ADR 0501 §1): a done key whose every artifact names a reference is one recorded build, its artifacts
|
||||
// the key's, its launchers by the build modules' rule, its reads from its manifest, the bases its last build stood on
|
||||
// carried; recorded twice it is one record.
|
||||
func TestADoneKeyIsOneRecordedBuildOfItsArtifacts(t *testing.T) {
|
||||
keys, trunk := aKeyOf(tsBundle, 3,
|
||||
sdkdelivery.BuiltArtifact{Name: "tools", Kind: "bundle", Digest: shaOf("t"),
|
||||
Reference: "artifact-store://x/tools/blobs/" + shaOf("t")},
|
||||
sdkdelivery.BuiltArtifact{Name: "img", Kind: "image", Digest: shaOf("i"),
|
||||
Reference: "artifact-store://x/img@" + shaOf("i")})
|
||||
previous := []inventory.Build{{ID: "build-1", Module: "x", Against: []string{"artifact-store://base@sha256:aa"}}}
|
||||
take := keyTake{Module: "x", Trunk: trunk}
|
||||
kept, resolved, err := buildOfKey(t.Context(), keys, take, keyContext{previous: previous})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if kept.ID != "key-1111111111111111111111111111111111111111111111111111111111111111-3" {
|
||||
t.Errorf("the record's id is %s", kept.ID)
|
||||
}
|
||||
again, _, _ := buildOfKey(t.Context(), keys, take, keyContext{previous: previous})
|
||||
if again.ID != kept.ID {
|
||||
t.Errorf("one key recorded twice is two records: %s and %s", kept.ID, again.ID)
|
||||
}
|
||||
if kept.SourceFingerprint != keyFP || kept.Commit != trunk.Commit || kept.Path != "modules/x" || kept.On != "ace" ||
|
||||
kept.Module != "x" || kept.Repository != "novox/mesh-catalog" {
|
||||
t.Errorf("the record: %+v", kept)
|
||||
}
|
||||
if len(kept.Made) != 2 || kept.Made[0].Reference != "artifact-store://x/tools/blobs/"+shaOf("t") ||
|
||||
kept.Made[1].Reference != "artifact-store://x/img@"+shaOf("i") || kept.Made[1].Kind != "image" {
|
||||
t.Errorf("Made is not the key's artifacts: %+v", kept.Made)
|
||||
}
|
||||
if len(kept.Read) != 1 || kept.Read[0].Repository != "novox/other" {
|
||||
t.Errorf("Read is not the manifest's contexts: %+v", kept.Read)
|
||||
}
|
||||
if kept.Sources != nil || kept.Mirrored != nil {
|
||||
t.Errorf("a key's record says build sources or mirrored bases it cannot know: %+v %+v", kept.Sources, kept.Mirrored)
|
||||
}
|
||||
if len(kept.Against) != 1 || kept.Against[0] != "artifact-store://base@sha256:aa" {
|
||||
t.Errorf("the bases the last build stood on are not carried: %v", kept.Against)
|
||||
}
|
||||
if len(resolved.Bundles) != 1 || resolved.Bundles[0].Launchers["dist/index.js"] != "dist/index.serve.mjs" {
|
||||
t.Errorf("the bundle's launchers: %+v", resolved.Bundles)
|
||||
}
|
||||
var m map[string]any
|
||||
if err := json.Unmarshal(kept.Manifest, &m); err != nil || !strings.Contains(string(kept.Manifest), shaOf("i")) {
|
||||
t.Errorf("the recorded manifest names no image digest: %s", kept.Manifest)
|
||||
}
|
||||
}
|
||||
|
||||
// A3 (ADR 0498 §8): a key with an artifact naming no reference, a key not done and a missing key are waited for, said;
|
||||
// a failed key fails in the builder's words; a module with no fingerprint of the new form waits, naming why.
|
||||
func TestAKeyNotReadyWaitsAndAFailedKeyFails(t *testing.T) {
|
||||
noRef, trunk := aKeyOf(tsBundle, 1,
|
||||
sdkdelivery.BuiltArtifact{Name: "tools", Kind: "bundle", Digest: shaOf("t"),
|
||||
Reference: "artifact-store://x/tools/blobs/" + shaOf("t")},
|
||||
sdkdelivery.BuiltArtifact{Name: "img", Kind: "image", Digest: shaOf("i")})
|
||||
take := keyTake{Module: "x", Trunk: trunk}
|
||||
_, _, err := buildOfKey(t.Context(), noRef, take, keyContext{})
|
||||
if !errors.Is(err, errKeyWaits) || !strings.Contains(err.Error(), "waits for the reference of img") {
|
||||
t.Errorf("a key without a reference: %v", err)
|
||||
}
|
||||
building := noRef
|
||||
k := building.keys["x"]
|
||||
k.Build = sdkdelivery.BuilderBuild{State: sdkdelivery.BuildBuilding, By: "g14"}
|
||||
building.keys = map[string]builderkeys.Key{"x": k}
|
||||
if _, _, err := buildOfKey(t.Context(), building, take, keyContext{}); !errors.Is(err, errKeyWaits) ||
|
||||
!strings.Contains(err.Error(), "building on g14") {
|
||||
t.Errorf("a building key: %v", err)
|
||||
}
|
||||
if _, _, err := buildOfKey(t.Context(), keyBuckets{}, take, keyContext{}); !errors.Is(err, errKeyWaits) ||
|
||||
!strings.Contains(err.Error(), "no build of x") {
|
||||
t.Errorf("no key: %v", err)
|
||||
}
|
||||
k.Build = sdkdelivery.BuilderBuild{State: sdkdelivery.BuildFailed, Error: "digests differ: ace sha256:1, g14 sha256:2"}
|
||||
failed := keyBuckets{keys: map[string]builderkeys.Key{"x": k}}
|
||||
if _, _, err := buildOfKey(t.Context(), failed, take, keyContext{}); !errors.Is(err, errKeyFailed) ||
|
||||
!strings.Contains(err.Error(), "digests differ: ace sha256:1, g14 sha256:2") {
|
||||
t.Errorf("a failed key: %v", err)
|
||||
}
|
||||
none := take
|
||||
none.Trunk.FP2, none.Trunk.None2 = "", "no lock"
|
||||
if _, _, err := buildOfKey(t.Context(), noRef, none, keyContext{}); !errors.Is(err, errKeyWaits) ||
|
||||
!strings.Contains(err.Error(), "no lock") {
|
||||
t.Errorf("no fingerprint: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// ADR 0501 §7: a manifest word this controller does not read waits for the controller that reads it; never fails.
|
||||
func TestAManifestWordTheControllerDoesNotReadWaits(t *testing.T) {
|
||||
raw := `{"module":"x","version":"1","not-a-word-yet":true,"build":{"artifacts":[{"name":"a","kind":"archive","from":"files"}]},
|
||||
"resources":[{"id":"a","type":"archive","path":"/opt/x","artifact":"a"}]}`
|
||||
keys, trunk := aKeyOf(raw, 1, sdkdelivery.BuiltArtifact{Name: "a", Kind: "archive", Digest: shaOf("a"),
|
||||
Reference: "artifact-store://x/a/blobs/" + shaOf("a")})
|
||||
_, _, err := buildOfKey(t.Context(), keys, keyTake{Module: "x", Trunk: trunk}, keyContext{})
|
||||
if !errors.Is(err, errKeyWaits) || !strings.Contains(err.Error(), `"not-a-word-yet"`) {
|
||||
t.Fatalf("an unread word: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func packed(t *testing.T, files map[string]string) []byte {
|
||||
t.Helper()
|
||||
var buf bytes.Buffer
|
||||
z := gzip.NewWriter(&buf)
|
||||
w := tar.NewWriter(z)
|
||||
for name, text := range files {
|
||||
if err := w.WriteHeader(&tar.Header{Name: name, Mode: 0o644, Size: int64(len(text)), Typeflag: tar.TypeReg}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := w.Write([]byte(text)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := w.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := z.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
// A templated module's texts are read back from the archive the store holds under the key's reference; an archive the
|
||||
// store does not answer is waited for (the lead's Q1b).
|
||||
func TestATemplatedModulesTextsAreReadFromItsArchive(t *testing.T) {
|
||||
raw := `{"module":"x","version":"1","templates":"templates",
|
||||
"build":{"artifacts":[{"name":"templates","kind":"archive","from":"templates"}]},
|
||||
"resources":[{"id":"main","type":"file","path":"/etc/x.conf","mode":"0644","template":"templates/x.conf"}]}`
|
||||
body := packed(t, map[string]string{"x.conf": "listen 1\n"})
|
||||
sum := sha256.Sum256(body)
|
||||
digest := "sha256:" + hex.EncodeToString(sum[:])
|
||||
keys, trunk := aKeyOf(raw, 1, sdkdelivery.BuiltArtifact{Name: "templates", Kind: "archive", Digest: digest,
|
||||
Reference: "artifact-store://x/templates/blobs/" + digest})
|
||||
asked := ""
|
||||
archive := func(_ context.Context, reference string) ([]byte, error) { asked = reference; return body, nil }
|
||||
_, resolved, err := buildOfKey(t.Context(), keys, keyTake{Module: "x", Trunk: trunk}, keyContext{archive: archive})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if asked != "artifact-store://x/templates/blobs/"+digest {
|
||||
t.Errorf("the archive asked: %q", asked)
|
||||
}
|
||||
if text, _ := json.Marshal(resolved.Resources); !strings.Contains(string(text), "listen 1") {
|
||||
t.Errorf("the template's text is not in the file: %s", text)
|
||||
}
|
||||
gone := func(context.Context, string) ([]byte, error) { return nil, errors.New("the store does not hold it") }
|
||||
if _, _, err := buildOfKey(t.Context(), keys, keyTake{Module: "x", Trunk: trunk}, keyContext{archive: gone}); !errors.Is(err, errKeyWaits) ||
|
||||
!strings.Contains(err.Error(), "cannot be read from the artifact store") {
|
||||
t.Errorf("an archive the store does not answer: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// The launchers a key's bundle is resolved with are mesh-sdk's one rule (buildstep.Launchers), the names
|
||||
// mesh-build-node writes: a TypeScript bundle's entrypoint x.js runs as x.serve.mjs, a Go bundle has none.
|
||||
func TestALaunchersRuleIsTheBuildModules(t *testing.T) {
|
||||
m, err := catalogue.ParseManifest([]byte(tsBundle))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
k := builderkeys.Key{Build: sdkdelivery.BuilderBuild{Artifacts: []sdkdelivery.BuiltArtifact{{Name: "tools", Digest: "sha256:1",
|
||||
Reference: "artifact-store://x/tools/blobs/sha256:1"}}}}
|
||||
built, err := builtOfKey(t.Context(), nil, m, k)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if built[0].Launchers["dist/index.js"] != "dist/index.serve.mjs" || built[0].Kind != "bundle" {
|
||||
t.Fatalf("built: %+v", built[0])
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
package artifacts
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
// blobBound is the most GetBlob reads: a templates archive is a few kilobytes; anything near this is no archive a
|
||||
// take-in reads text from.
|
||||
const blobBound = 64 << 20
|
||||
|
||||
// GetBlob is the archive a recorded reference names (`artifact-store://<repository>/blobs/sha256:<hex>`), read whole
|
||||
// and checked against the digest it names: bytes that hash to another digest are refused, never read (R4e: the
|
||||
// controller reads a templates archive by the reference of a builder's done key). Gone when the store does not hold
|
||||
// it; ErrNotOurs for a reference outside the store or one that is not an archive.
|
||||
func (s Store) GetBlob(ctx context.Context, reference string) ([]byte, error) {
|
||||
repository, digest, archive, err := s.archive(reference)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !archive {
|
||||
return nil, fmt.Errorf("%w: %s is not an archive", ErrNotOurs, reference)
|
||||
}
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodGet, s.url(repository, "blobs", digest), nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
response, err := s.client().Do(request)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot reach the artifact store at %s: %w", s.Address, err)
|
||||
}
|
||||
defer response.Body.Close()
|
||||
switch response.StatusCode {
|
||||
case http.StatusOK:
|
||||
case http.StatusNotFound:
|
||||
return nil, Gone
|
||||
default:
|
||||
return nil, fmt.Errorf("the artifact store answered %s for %s/blobs/%s", response.Status, repository, digest)
|
||||
}
|
||||
body, err := io.ReadAll(io.LimitReader(response.Body, blobBound+1))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading %s/blobs/%s: %w", repository, digest, err)
|
||||
}
|
||||
if len(body) > blobBound {
|
||||
return nil, fmt.Errorf("%s/blobs/%s is larger than %d bytes, which no archive read here is", repository, digest, blobBound)
|
||||
}
|
||||
sum := sha256.Sum256(body)
|
||||
if got := "sha256:" + hex.EncodeToString(sum[:]); got != digest {
|
||||
return nil, fmt.Errorf("the artifact store answered %s/blobs/%s with bytes of the digest %s: refused", repository,
|
||||
digest, got)
|
||||
}
|
||||
return body, nil
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
package artifacts
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A take-in reads a templates archive by its reference, and only bytes of the digest the reference names.
|
||||
func TestGetBlobReadsOnlyTheBytesOfItsDigest(t *testing.T) {
|
||||
good := []byte("the archive")
|
||||
sum := sha256.Sum256(good)
|
||||
digest := "sha256:" + hex.EncodeToString(sum[:])
|
||||
liar := "sha256:" + strings.Repeat("0", 64)
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch {
|
||||
case strings.HasSuffix(r.URL.Path, "/x/files/blobs/"+digest), strings.HasSuffix(r.URL.Path, "/x/files/blobs/"+liar):
|
||||
_, _ = w.Write(good)
|
||||
default:
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
s := Store{Address: strings.TrimPrefix(server.URL, "http://")}
|
||||
ctx := context.Background()
|
||||
body, err := s.GetBlob(ctx, "artifact-store://x/files/blobs/"+digest)
|
||||
if err != nil || string(body) != string(good) {
|
||||
t.Fatalf("the archive: %q %v", body, err)
|
||||
}
|
||||
if _, err := s.GetBlob(ctx, "artifact-store://x/files/blobs/"+liar); err == nil || !strings.Contains(err.Error(), "refused") {
|
||||
t.Fatalf("bytes of another digest are read: %v", err)
|
||||
}
|
||||
if _, err := s.GetBlob(ctx, "artifact-store://x/other/blobs/"+digest); !errors.Is(err, Gone) {
|
||||
t.Fatalf("a blob the store does not hold: %v", err)
|
||||
}
|
||||
if _, err := s.GetBlob(ctx, "registry.example/x@"+digest); !errors.Is(err, ErrNotOurs) {
|
||||
t.Fatalf("a reference outside the store: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -23,6 +23,7 @@ import (
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"github.com/novox/mesh-controller/internal/buildreads"
|
||||
)
|
||||
|
||||
// Turning a repository into artifacts the mesh can pin.
|
||||
@@ -2001,33 +2002,9 @@ func instructions(recipe string) []string {
|
||||
return out
|
||||
}
|
||||
|
||||
// readBy is every repository other than the module's own that this build's recipes read source from,
|
||||
// each once and in a fixed order, so two builds of one commit report the same thing the same way.
|
||||
func readBy(manifest catalogue.Manifest) []catalogue.ArtifactContext {
|
||||
if manifest.Build == nil {
|
||||
return nil
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
var out []catalogue.ArtifactContext
|
||||
for _, a := range manifest.Build.Artifacts {
|
||||
if a.Context == nil || a.Context.Repository == "" {
|
||||
continue
|
||||
}
|
||||
key := a.Context.Repository + "#" + a.Context.Ref
|
||||
if seen[key] {
|
||||
continue
|
||||
}
|
||||
seen[key] = true
|
||||
out = append(out, *a.Context)
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
if out[i].Repository != out[j].Repository {
|
||||
return out[i].Repository < out[j].Repository
|
||||
}
|
||||
return out[i].Ref < out[j].Ref
|
||||
})
|
||||
return out
|
||||
}
|
||||
// readBy is every repository other than the module's own that this build's recipes read source from: the rule
|
||||
// build-agent and the controller share (internal/buildreads).
|
||||
func readBy(manifest catalogue.Manifest) []catalogue.ArtifactContext { return buildreads.Of(manifest) }
|
||||
|
||||
// binaryName is what a compiled bundle's executable is called: what the artifact says, or the name of
|
||||
// the package it is built from, which is what a compiler would have chosen anyway.
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
// Package buildreads is which repositories other than a module's own its build reads source from, as its manifest
|
||||
// names them (novox/hq 04-ISSUES/131): the artifacts' `context` repositories. One rule for build-agent, which says it
|
||||
// in a build's outcome, and for the controller, which reads it from a done key's manifest (R4e, issue 1851): it lives
|
||||
// here, not in internal/builder, so it outlasts build-agent (R4g).
|
||||
package buildreads
|
||||
|
||||
import (
|
||||
"sort"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
)
|
||||
|
||||
// Of is every repository other than the module's own that this build's recipes read source from, each once and in a
|
||||
// fixed order, so two builds of one commit report the same thing the same way.
|
||||
func Of(manifest catalogue.Manifest) []catalogue.ArtifactContext {
|
||||
if manifest.Build == nil {
|
||||
return nil
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
var out []catalogue.ArtifactContext
|
||||
for _, a := range manifest.Build.Artifacts {
|
||||
if a.Context == nil || a.Context.Repository == "" {
|
||||
continue
|
||||
}
|
||||
key := a.Context.Repository + "#" + a.Context.Ref
|
||||
if seen[key] {
|
||||
continue
|
||||
}
|
||||
seen[key] = true
|
||||
out = append(out, *a.Context)
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
if out[i].Repository != out[j].Repository {
|
||||
return out[i].Repository < out[j].Repository
|
||||
}
|
||||
return out[i].Ref < out[j].Ref
|
||||
})
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
// Option changes FetchCommit.
|
||||
type Option func(*fetchOptions)
|
||||
|
||||
type fetchOptions struct{ allowFile bool }
|
||||
|
||||
// AllowFileRemote lets FetchCommit take a file:// remote. It is for a test's local repository; production
|
||||
// code does not pass it, and FetchCommit then refuses anything but ssh://.
|
||||
func AllowFileRemote() Option { return func(o *fetchOptions) { o.allowFile = true } }
|
||||
|
||||
var (
|
||||
commitOK = regexp.MustCompile(`^[0-9a-f]{40}$`)
|
||||
refOK = regexp.MustCompile(`^refs/[A-Za-z0-9._/-]+$`)
|
||||
)
|
||||
|
||||
// FetchCommit fetches one commit of a forge repository into dir, an empty directory, and checks it out detached
|
||||
// (novox/hq ADR 0466 §4, §5; issue 1474). remote is ssh://<ssh name>/<owner>/<repo>.git, the ssh name being the
|
||||
// one the build module's templates set up (ADR 0409 §1); commit is a full 40-hex sha; ref is optional, and names
|
||||
// the pull request's head (refs/pull/<n>/head) when the commit is only there. Every git command runs through run
|
||||
// as `git -C dir ...` with GIT_TERMINAL_PROMPT=0 and no credential helper, so nothing is asked for and no
|
||||
// credential is used. A remote that is not ssh://, or one with user information, and a commit that is not 40
|
||||
// hex digits are refused before anything runs.
|
||||
func FetchCommit(ctx context.Context, run forgekey.Runner, dir, remote, commit, ref string, opts ...Option) error {
|
||||
var o fetchOptions
|
||||
for _, opt := range opts {
|
||||
opt(&o)
|
||||
}
|
||||
if err := checkFetch(o, dir, remote, commit, ref); err != nil {
|
||||
return err
|
||||
}
|
||||
env := []string{"GIT_TERMINAL_PROMPT=0"}
|
||||
git := func(args ...string) (string, error) {
|
||||
full := append([]string{"-c", "credential.helper=", "-C", dir}, args...)
|
||||
r := run(ctx, env, "git", full...)
|
||||
if r.Status != 0 || r.Err != "" {
|
||||
return "", fmt.Errorf("git %s: %s", args[0], firstLine(r.Err+" "+r.Stderr))
|
||||
}
|
||||
return r.Stdout, nil
|
||||
}
|
||||
if _, err := git("init", "-q"); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := git("remote", "add", "origin", remote); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := git("fetch", "--no-tags", "--depth", "1", "origin", commit); err != nil {
|
||||
if ref == "" {
|
||||
return err
|
||||
}
|
||||
if _, rerr := git("fetch", "--no-tags", "origin", ref); rerr != nil {
|
||||
return fmt.Errorf("%v; then the ref %s: %w", err, ref, rerr)
|
||||
}
|
||||
if _, verr := git("cat-file", "-e", commit+"^{commit}"); verr != nil {
|
||||
return fmt.Errorf("the commit %s is not in %s: %w", commit, ref, verr)
|
||||
}
|
||||
}
|
||||
if _, err := git("checkout", "-q", "--detach", commit); err != nil {
|
||||
return err
|
||||
}
|
||||
out, err := git("rev-parse", "HEAD")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if head := strings.TrimSpace(out); head != commit {
|
||||
return fmt.Errorf("the checkout is at %s, not at the commit %s", head, commit)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func checkFetch(o fetchOptions, dir, remote, commit, ref string) error {
|
||||
if dir == "" {
|
||||
return fmt.Errorf("buildstep: no workspace directory")
|
||||
}
|
||||
if !commitOK.MatchString(commit) {
|
||||
return fmt.Errorf("buildstep: %q is not a full commit (40 hex digits)", commit)
|
||||
}
|
||||
if ref != "" && !refOK.MatchString(ref) {
|
||||
return fmt.Errorf("buildstep: %q is not a ref (refs/...)", ref)
|
||||
}
|
||||
u, err := url.Parse(remote)
|
||||
if err != nil {
|
||||
return fmt.Errorf("buildstep: the remote is not a URL: %w", err)
|
||||
}
|
||||
if u.User != nil {
|
||||
return fmt.Errorf("buildstep: the remote carries user information; the key is the credential")
|
||||
}
|
||||
switch {
|
||||
case u.Scheme == "ssh" && u.Host != "":
|
||||
case u.Scheme == "file" && o.allowFile:
|
||||
default:
|
||||
return fmt.Errorf("buildstep: the remote %q is not ssh://", remote)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func firstLine(s string) string {
|
||||
for _, l := range strings.Split(s, "\n") {
|
||||
if l = strings.TrimSpace(l); l != "" {
|
||||
return l
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// Guard is a build module's check that the toolchain installed on its node is the one its recipe builds with
|
||||
// (novox/hq ADR 0469 §1; issue 1473). The same guard serves ADR 0470 §4's postgres major for the integration
|
||||
// level, with its own Probe and Want.
|
||||
type Guard struct {
|
||||
// Language is the toolchain's key part: go or typescript (delivery.ToolchainKey).
|
||||
Language string
|
||||
// Want is the toolchain version the recipe builds with, and Recipe the recipe's own version. The indexer
|
||||
// compares both texts as they are written here.
|
||||
Want, Recipe string
|
||||
// Probe reads the installed version (go version, node --version).
|
||||
Probe func(ctx context.Context) (installed string, err error)
|
||||
Node string
|
||||
// Put is the toolchains state's Put (stdio.State(delivery.ToolchainsState).Put).
|
||||
Put Put
|
||||
}
|
||||
|
||||
// Check probes the installed version, writes it to <node>.<language> of the toolchains state every time it runs,
|
||||
// and answers nil when it is the recipe's. Otherwise it answers a *Refusal naming both versions; the installed
|
||||
// version is written even then, so the indexer sees that builders disagree. A probe that fails is a *Refusal
|
||||
// naming its failure, and nothing is written. A write that fails is an error that is not a Refusal.
|
||||
func (g Guard) Check(ctx context.Context) error {
|
||||
installed, err := g.Probe(ctx)
|
||||
if err != nil {
|
||||
return &Refusal{Text: fmt.Sprintf("the installed %s version could not be read: %v", g.Language, err), Want: g.Want}
|
||||
}
|
||||
key, err := delivery.ToolchainKey(g.Node, g.Language)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
raw, err := delivery.EncodeToolchain(delivery.Toolchain{Language: g.Language, Version: installed, Recipe: g.Recipe, At: now().UTC()})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := g.Put(key, json.RawMessage(raw)); err != nil {
|
||||
return fmt.Errorf("buildstep: writing the toolchain %s: %w", key, err)
|
||||
}
|
||||
if installed != g.Want {
|
||||
return &Refusal{Text: fmt.Sprintf("%s %s installed, the recipe builds with %s %s", g.Language, installed, g.Language, g.Want),
|
||||
Want: g.Want, Have: installed}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
package buildstep
|
||||
|
||||
import "strings"
|
||||
|
||||
// LauncherSuffix is what a TypeScript entrypoint's launcher is called beside it: index.js is started as
|
||||
// index.serve.mjs (novox/hq ADR 0193), an ES module by its own extension whatever the bundle's package.json says.
|
||||
const LauncherSuffix = ".serve.mjs"
|
||||
|
||||
// Launchers are the launchers a bundle's build writes, by entrypoint, relative to the bundle's root: for TypeScript
|
||||
// every entrypoint `x.js` is started by `x.serve.mjs` beside it; an entrypoint that is not JavaScript and every other
|
||||
// language, whose build is already executable, get none. The one rule mesh-build-node, build-agent and the controller
|
||||
// share (R4e), so the name a build writes is the name a reader expects.
|
||||
func Launchers(language string, entrypoints []string) map[string]string {
|
||||
out := map[string]string{}
|
||||
if language != "typescript" {
|
||||
return out
|
||||
}
|
||||
for _, entry := range entrypoints {
|
||||
if strings.HasSuffix(entry, ".js") {
|
||||
out[entry] = strings.TrimSuffix(entry, ".js") + LauncherSuffix
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
// MergeResult is what MergeInto made of a head and a trunk commit: the trunk commit (Base) and the head, whole, and the
|
||||
// merge commit and the paths the merge changes against the trunk commit; or, when the head does not merge cleanly, the
|
||||
// files in conflict, with Merge and Changed empty.
|
||||
type MergeResult struct {
|
||||
Base, Head, Merge string
|
||||
Conflicts []string
|
||||
Changed []string
|
||||
}
|
||||
|
||||
// MergeInto makes, in the workspace dir that FetchCommit left at commit, the merge of commit into the trunk commit
|
||||
// base, as the forge's default merge would (a merge commit, never a fast-forward), and leaves the workspace at the merge
|
||||
// (novox/hq ADR 0484 clause 3, ADR 0323). It is the one merge the build modules' fetch steps make, with the identity
|
||||
// and flags of the controller's check merge (mesh-controller internal/builder/check_merge.go): a name nobody owns, no
|
||||
// signature, no hook, nothing of the machine's git configuration that changes a merge. Nothing is pushed.
|
||||
//
|
||||
// A merge needs both histories, so it makes the fetch whole (the shallow clone FetchCommit made is deepened) and
|
||||
// fetches base by its sha. A head that does not merge cleanly is an answer, not an error: the files in conflict, the
|
||||
// merge undone and the workspace left at base. A head already in base answers a Merge equal to Base and no changed
|
||||
// paths. An error is that the merge could not be tried: a base or commit that is not a full sha, no base, a workspace
|
||||
// not at commit, or a fetch or git command that failed. remote and the options are those of FetchCommit; the runner's
|
||||
// bound must allow a whole fetch.
|
||||
func MergeInto(ctx context.Context, run forgekey.Runner, dir, remote, base, commit string, opts ...Option) (MergeResult, error) {
|
||||
var o fetchOptions
|
||||
for _, opt := range opts {
|
||||
opt(&o)
|
||||
}
|
||||
res := MergeResult{Base: base, Head: commit}
|
||||
if base == "" {
|
||||
return res, fmt.Errorf("buildstep: no base: a merge needs the trunk commit it is made onto")
|
||||
}
|
||||
if err := checkFetch(o, dir, remote, base, ""); err != nil {
|
||||
return res, err
|
||||
}
|
||||
if err := checkFetch(o, dir, remote, commit, ""); err != nil {
|
||||
return res, err
|
||||
}
|
||||
env := []string{"GIT_TERMINAL_PROMPT=0"}
|
||||
git := func(args ...string) (string, error) {
|
||||
full := append([]string{"-c", "credential.helper=", "-C", dir}, args...)
|
||||
r := run(ctx, env, "git", full...)
|
||||
if r.Status != 0 || r.Err != "" {
|
||||
return "", fmt.Errorf("git %s: %s", args[0], firstLine(r.Err+" "+r.Stderr))
|
||||
}
|
||||
return r.Stdout, nil
|
||||
}
|
||||
at, err := git("rev-parse", "HEAD")
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
if head := strings.TrimSpace(at); head != commit {
|
||||
return res, fmt.Errorf("buildstep: the workspace is at %s, not at the commit %s", head, commit)
|
||||
}
|
||||
shallow, err := git("rev-parse", "--is-shallow-repository")
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
fetch := []string{"fetch", "--no-tags", "origin", base}
|
||||
if strings.TrimSpace(shallow) == "true" {
|
||||
fetch = []string{"fetch", "--no-tags", "--unshallow", "origin", base}
|
||||
}
|
||||
if _, err := git(fetch...); err != nil {
|
||||
return res, fmt.Errorf("the trunk commit %s: %w", base, err)
|
||||
}
|
||||
if _, err := git("checkout", "-q", "--detach", base); err != nil {
|
||||
return res, fmt.Errorf("the trunk commit %s cannot be checked out: %w", base, err)
|
||||
}
|
||||
message := fmt.Sprintf("Check the merge of %s into %s (made for a check, never pushed)", short(commit), short(base))
|
||||
args := append(append([]string{}, mergeIdentity...), "merge", "--no-ff", "--no-edit", "--no-verify", "--quiet", "-m", message, commit)
|
||||
if _, err := git(args...); err != nil {
|
||||
out, _ := git("diff", "--name-only", "--diff-filter=U")
|
||||
res.Conflicts = strings.Fields(out)
|
||||
_, _ = git("merge", "--abort")
|
||||
if len(res.Conflicts) == 0 {
|
||||
return res, fmt.Errorf("the head %s could not be merged into the trunk commit %s: %w", short(commit), short(base), err)
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
merge, err := git("rev-parse", "HEAD")
|
||||
if err != nil {
|
||||
return res, fmt.Errorf("the merge cannot be read: %w", err)
|
||||
}
|
||||
res.Merge = strings.TrimSpace(merge)
|
||||
changed, err := git("diff", "--name-only", "--no-renames", base, res.Merge)
|
||||
if err != nil {
|
||||
return res, fmt.Errorf("the paths the merge changes cannot be read: %w", err)
|
||||
}
|
||||
res.Changed = strings.Fields(changed)
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// mergeIdentity is who a check's merge is made as: it is never pushed, so it names nobody's account. No hook of the
|
||||
// clone's or the build machine's git configuration runs, and no setting there changes the merge (the controller's
|
||||
// check merge, internal/builder/check_merge.go).
|
||||
var mergeIdentity = []string{"-c", "user.name=mesh merge check", "-c", "user.email=merge-check@mesh.invalid",
|
||||
"-c", "commit.gpgsign=false", "-c", "core.hooksPath=/dev/null", "-c", "merge.verifySignatures=false"}
|
||||
|
||||
func short(commit string) string {
|
||||
if len(commit) > 12 {
|
||||
return commit[:12]
|
||||
}
|
||||
return commit
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
)
|
||||
|
||||
// Pack tars and gzips a directory the way the build agent does, byte for byte, and answers the archive and its
|
||||
// digest, "sha256:" and the hex of the archive's bytes (issue 1479; novox/hq ADR 0405 §7, ADR 0453 §7). It is a
|
||||
// copy of mesh-controller's internal/builder/builder.go pack (origin/main 073ccc29, :1351-1425), which stays
|
||||
// where it is until the build agent goes: a build module's digest is compared with the build agent's for one
|
||||
// fingerprint, so the bytes must be equal.
|
||||
//
|
||||
// Deterministically: regular files only (a symlink or a device is refused), sorted relative paths with
|
||||
// slashes, each header carrying its name, permission bits and size and nothing else, the gzip default writer.
|
||||
func Pack(root string) (archive []byte, digest string, err error) {
|
||||
raw, err := pack(root)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
sum := sha256.Sum256(raw)
|
||||
return raw, "sha256:" + hex.EncodeToString(sum[:]), nil
|
||||
}
|
||||
|
||||
func pack(root string) ([]byte, error) {
|
||||
info, err := os.Stat(root)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return nil, fmt.Errorf("%s is not a directory", root)
|
||||
}
|
||||
|
||||
var paths []string
|
||||
err = filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if info.IsDir() || !info.Mode().IsRegular() {
|
||||
// Only files. A symlink or a device in an archive is refused by the host that unpacks
|
||||
// it, so putting one in would build something that cannot be applied.
|
||||
if !info.IsDir() && !info.Mode().IsRegular() {
|
||||
return fmt.Errorf("%s is neither a file nor a directory, and an archive carries "+
|
||||
"only those", path)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
paths = append(paths, path)
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// filepath.Walk walks in lexical order; the sort is kept as the build agent keeps it, since an archive
|
||||
// whose digest changes because the traversal did is a silent failure.
|
||||
sort.Strings(paths)
|
||||
|
||||
var out bytes.Buffer
|
||||
zipped := gzip.NewWriter(&out)
|
||||
writer := tar.NewWriter(zipped)
|
||||
for _, path := range paths {
|
||||
body, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
relative, err := filepath.Rel(root, path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
mode := int64(info.Mode().Perm())
|
||||
if err := writer.WriteHeader(&tar.Header{
|
||||
Name: filepath.ToSlash(relative), Mode: mode, Size: int64(len(body)),
|
||||
Typeflag: tar.TypeReg,
|
||||
// Everything else left at its zero value on purpose: no times, owners or names.
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err := writer.Write(body); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if err := writer.Close(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := zipped.Close(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out.Bytes(), nil
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
package buildstep
|
||||
|
||||
// Refusal is a step that is not run on purpose, as against one that failed: the toolchain installed is not the
|
||||
// recipe's, the integration level's postgres major is not the one it is built for (novox/hq ADR 0469 §1, ADR 0470
|
||||
// §4; issue 1473). A step's run returns one and the runner ends the step refused with its text.
|
||||
type Refusal struct {
|
||||
// Text is the reason, in words the operator reads.
|
||||
Text string
|
||||
// Want is the version the recipe asks for and Have the one found; either may be empty.
|
||||
Want, Have string
|
||||
}
|
||||
|
||||
func (r *Refusal) Error() string { return r.Text }
|
||||
@@ -0,0 +1,99 @@
|
||||
// Package buildstep is what both build modules share: the runner of a long step that answers a tool call at once
|
||||
// and ends on its key in the module's `steps` state (novox/hq ADR 0469 §2; issue 1471), the toolchain guard
|
||||
// (issue 1473) and the fetch of one commit over SSH (issue 1474).
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// Put writes one key of a state: stdio.State(delivery.StepsState).Put's shape (go/state.go).
|
||||
type Put func(key string, value any) (uint64, error)
|
||||
|
||||
// EndAttempts and EndRetry bound how a step's end is written: a write that fails is tried again, since the end
|
||||
// is the only way the step's reader learns it is over.
|
||||
var (
|
||||
EndAttempts = 5
|
||||
EndRetry = 2 * time.Second
|
||||
)
|
||||
|
||||
var now = time.Now
|
||||
|
||||
// NewID is a step id: the time in milliseconds in base 36 and six random hex digits, lower-case letters and
|
||||
// digits only, so it is one key token and differs between two steps of one node even in one millisecond.
|
||||
func NewID() string {
|
||||
var b [3]byte
|
||||
_, _ = rand.Read(b[:])
|
||||
return strconv.FormatInt(now().UnixMilli(), 36) + hex.EncodeToString(b[:])
|
||||
}
|
||||
|
||||
// Start begins a step of tool on node and answers its id at once: the key <node>.<id> holds state running when
|
||||
// Start returns. run is then called in a goroutine of its own, under ctx (the process's, not the tool call's: the
|
||||
// call is cut at 30 s, the step is not). When it returns, the key is written passed with run's result, failed with
|
||||
// the error's text, or refused when the error is a *Refusal; a panic in run ends the step failed, naming it, and
|
||||
// the process goes on. If the running write fails, nothing was started and Start answers the error.
|
||||
func Start(ctx context.Context, put Put, node, tool string, run func(ctx context.Context) (result any, err error)) (string, error) {
|
||||
step := delivery.Step{ID: NewID(), Tool: tool, Node: node, State: delivery.StepRunning, Started: now().UTC()}
|
||||
key, err := delivery.StepKey(node, step.ID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := write(put, key, step); err != nil {
|
||||
return "", fmt.Errorf("buildstep: writing the running step %s: %w", key, err)
|
||||
}
|
||||
go func() {
|
||||
result, err := protect(ctx, run)
|
||||
step.Ended = now().UTC()
|
||||
switch ref := (*Refusal)(nil); {
|
||||
case err == nil:
|
||||
step.State = delivery.StepPassed
|
||||
if result != nil {
|
||||
raw, merr := json.Marshal(result)
|
||||
if merr != nil {
|
||||
step.State, step.Reason = delivery.StepFailed, "the result is not JSON: "+merr.Error()
|
||||
} else {
|
||||
step.Result = raw
|
||||
}
|
||||
}
|
||||
case errors.As(err, &ref):
|
||||
step.State, step.Reason = delivery.StepRefused, err.Error()
|
||||
default:
|
||||
step.State, step.Reason = delivery.StepFailed, err.Error()
|
||||
}
|
||||
for attempt := 1; ; attempt++ {
|
||||
if werr := write(put, key, step); werr == nil || attempt >= EndAttempts {
|
||||
return
|
||||
}
|
||||
time.Sleep(EndRetry)
|
||||
}
|
||||
}()
|
||||
return step.ID, nil
|
||||
}
|
||||
|
||||
// protect is run with a panic turned into the error that ends the step.
|
||||
func protect(ctx context.Context, run func(ctx context.Context) (any, error)) (result any, err error) {
|
||||
defer func() {
|
||||
if p := recover(); p != nil {
|
||||
result, err = nil, fmt.Errorf("the step panicked: %v", p)
|
||||
}
|
||||
}()
|
||||
return run(ctx)
|
||||
}
|
||||
|
||||
func write(put Put, key string, step delivery.Step) error {
|
||||
raw, err := delivery.EncodeStep(step)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = put(key, json.RawMessage(raw))
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
package buildstep
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
// ModuleTree is what a merge holds of one module directory: its tree id, the sha256 of its module.json and the sha256
|
||||
// of each lock file it has (go.sum, package-lock.json), from which the checker and the indexer take the module's inputs
|
||||
// (novox/hq ADR 0484 clause 3).
|
||||
type ModuleTree struct {
|
||||
Tree string `json:"tree"`
|
||||
Manifest string `json:"manifest"`
|
||||
Locks map[string]string `json:"locks,omitempty"`
|
||||
}
|
||||
|
||||
var lockFiles = []string{"go.sum", "package-lock.json"}
|
||||
|
||||
// ModuleTrees are the trees, in commit, of every directory of the workspace dir that holds a module.json, by directory
|
||||
// (the repository's root is "."). The workspace is at commit, so the manifests and the lock files are read from it. env is
|
||||
// the environment git runs with (nil for none). Both build modules' fetch steps answer it for a merge.
|
||||
func ModuleTrees(ctx context.Context, run forgekey.Runner, env []string, dir, commit string) (map[string]ModuleTree, error) {
|
||||
r := run(ctx, env, "git", "-C", dir, "ls-files", "--", "module.json", "*/module.json")
|
||||
if r.Status != 0 || r.Err != "" {
|
||||
return nil, fmt.Errorf("git ls-files: %s", firstLine(r.Err+" "+r.Stderr))
|
||||
}
|
||||
trees := map[string]ModuleTree{}
|
||||
for _, l := range strings.Split(r.Stdout, "\n") {
|
||||
if l = strings.TrimSpace(l); l == "" {
|
||||
continue
|
||||
}
|
||||
mdir := filepath.ToSlash(filepath.Dir(l))
|
||||
spec := commit + ":" + mdir
|
||||
if mdir == "." {
|
||||
spec = commit + "^{tree}"
|
||||
}
|
||||
t := run(ctx, env, "git", "-C", dir, "rev-parse", spec)
|
||||
if t.Status != 0 || t.Err != "" {
|
||||
return nil, fmt.Errorf("git rev-parse %s: %s", spec, firstLine(t.Err+" "+t.Stderr))
|
||||
}
|
||||
mt := ModuleTree{Tree: strings.TrimSpace(t.Stdout)}
|
||||
raw, err := os.ReadFile(filepath.Join(dir, filepath.FromSlash(l)))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
mt.Manifest = sha256Hex(raw)
|
||||
for _, lock := range lockFiles {
|
||||
if raw, err := os.ReadFile(filepath.Join(dir, filepath.FromSlash(mdir), lock)); err == nil {
|
||||
if mt.Locks == nil {
|
||||
mt.Locks = map[string]string{}
|
||||
}
|
||||
mt.Locks[lock] = sha256Hex(raw)
|
||||
}
|
||||
}
|
||||
trees[mdir] = mt
|
||||
}
|
||||
return trees, nil
|
||||
}
|
||||
|
||||
func sha256Hex(b []byte) string { h := sha256.Sum256(b); return hex.EncodeToString(h[:]) }
|
||||
@@ -0,0 +1,127 @@
|
||||
// Package forgekey is the key keeper every module that reaches the forge over SSH shares: ssh-client for a
|
||||
// machine's accounts, a build module for its own (design 47 R4c-3, ADR 0466 §2; issue 1469).
|
||||
package forgekey
|
||||
|
||||
// The key this machine's account uses to reach the forge over SSH (novox/hq ADR 0399, ADR 0400; issue 955,
|
||||
// area S1). It is made here, on the machine, once: ssh-keygen writes ~/.ssh/mesh_ed25519 only when it is
|
||||
// missing, so the private half never leaves the machine and an existing key is never replaced. The public
|
||||
// half and its fingerprint go into this module's `keys` state, one entry per account under <node>.<account>,
|
||||
// for the forge's provider to register. The tools bundle is the one process of this module, so the keeper
|
||||
// runs inside it beside the tools (main.go), the way claude-code reports what it holds.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// KeyFile is the key's name under ~/.ssh: never one of the operator's own id_* files.
|
||||
const KeyFile = "mesh_ed25519"
|
||||
|
||||
// The kinds of key (ADR 0400 §1, ADR 0466 §2): a machine's account's own, and a build module's.
|
||||
const (
|
||||
KindMachine = "machine"
|
||||
KindBuilder = "builder"
|
||||
)
|
||||
|
||||
// KeyRetry is how soon it looks again after a failure.
|
||||
const KeyRetry = 30 * time.Second
|
||||
|
||||
// KeyRound is how often the keeper looks again.
|
||||
const KeyRound = 10 * time.Minute
|
||||
|
||||
// KeyEntry is one value of the keys state. It carries the public half only.
|
||||
type KeyEntry struct {
|
||||
Node string `json:"node"`
|
||||
Account string `json:"account"`
|
||||
Kind string `json:"kind"`
|
||||
Public string `json:"public"`
|
||||
Fingerprint string `json:"fingerprint"`
|
||||
At string `json:"at"`
|
||||
}
|
||||
|
||||
// KeyPutter is the keys state, as far as the keeper needs it.
|
||||
type KeyPutter interface {
|
||||
Put(key string, value any) (uint64, error)
|
||||
}
|
||||
|
||||
// KeyKeeper keeps one account's key and its entry.
|
||||
type KeyKeeper struct {
|
||||
Home, Node, Account string
|
||||
Kind string // KindMachine or KindBuilder
|
||||
Run Runner
|
||||
State KeyPutter
|
||||
Now func() time.Time
|
||||
// After waits between rounds; time.After when nil. A test sets it to see which pause was chosen.
|
||||
After func(time.Duration) <-chan time.Time
|
||||
|
||||
reported string // the public half and fingerprint last put
|
||||
}
|
||||
|
||||
func (k *KeyKeeper) path() string { return filepath.Join(k.Home, ".ssh", KeyFile) }
|
||||
|
||||
// Once makes the key when it is missing and puts the entry when it differs from the one last put.
|
||||
func (k *KeyKeeper) Once(ctx context.Context) error {
|
||||
if err := k.ensure(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
raw, err := os.ReadFile(k.path() + ".pub")
|
||||
if err != nil {
|
||||
return fmt.Errorf("the public half: %w", err)
|
||||
}
|
||||
pk, err := ParseKeyLine(string(raw))
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s.pub: %w", k.path(), err)
|
||||
}
|
||||
public := strings.TrimSpace(string(raw))
|
||||
if sig := public + "|" + pk.Fingerprint; sig == k.reported {
|
||||
return nil
|
||||
}
|
||||
entry := KeyEntry{Node: k.Node, Account: k.Account, Kind: k.Kind, Public: public,
|
||||
Fingerprint: pk.Fingerprint, At: k.Now().UTC().Format(time.RFC3339)}
|
||||
if _, err := k.State.Put(k.Node+"."+k.Account, entry); err != nil {
|
||||
return err
|
||||
}
|
||||
k.reported = public + "|" + pk.Fingerprint
|
||||
return nil
|
||||
}
|
||||
|
||||
// ensure makes the pair when the private half is missing. Never over an existing key.
|
||||
func (k *KeyKeeper) ensure(ctx context.Context) error {
|
||||
if _, err := os.Lstat(k.path()); err == nil {
|
||||
return nil
|
||||
} else if !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(k.path()), 0o700); err != nil {
|
||||
return err
|
||||
}
|
||||
r := k.Run(ctx, nil, "ssh-keygen", "-q", "-t", "ed25519", "-N", "", "-C", k.Account+"@"+k.Node+" (mesh)", "-f", k.path())
|
||||
if r.Status != 0 || r.Err != "" {
|
||||
return fmt.Errorf("ssh-keygen: %s", firstLine(r.Err+" "+r.Stderr))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Keep runs Once now and every KeyRound, reporting a failure through say and trying again at the next round.
|
||||
func (k *KeyKeeper) Keep(ctx context.Context, say func(string, ...any)) {
|
||||
for {
|
||||
pause := KeyRound
|
||||
if err := k.Once(ctx); err != nil {
|
||||
say("the forge key: %v", err)
|
||||
pause = KeyRetry // the state or the bus's grant may arrive after the process
|
||||
}
|
||||
after := k.After
|
||||
if after == nil {
|
||||
after = time.After
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-after(pause):
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
package forgekey
|
||||
|
||||
// Keys, by their public half only. A fingerprint is computed here from the public key's bytes, as
|
||||
// ssh-keygen -l does (SHA256 of the key blob, unpadded base64); a private key is never read by this
|
||||
// process — its first line, the PEM header, says it is one, and ssh-keygen is asked about it.
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"math/big"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// KeyTypes are the public key algorithms an authorized_keys or known_hosts line can carry.
|
||||
var KeyTypes = map[string]bool{
|
||||
"ssh-ed25519": true, "ssh-rsa": true, "ssh-dss": true,
|
||||
"ecdsa-sha2-nistp256": true, "ecdsa-sha2-nistp384": true, "ecdsa-sha2-nistp521": true,
|
||||
"sk-ssh-ed25519@openssh.com": true, "sk-ecdsa-sha2-nistp256@openssh.com": true,
|
||||
}
|
||||
|
||||
// PublicKey is one public key as a line carries it.
|
||||
type PublicKey struct {
|
||||
Type string `json:"type"`
|
||||
Bits int `json:"bits"`
|
||||
Fingerprint string `json:"fingerprint"`
|
||||
Comment string `json:"comment"`
|
||||
Options string `json:"options,omitempty"`
|
||||
Weak string `json:"weak,omitempty"`
|
||||
}
|
||||
|
||||
// ParseKeyLine reads one authorized_keys line (options, type, key, comment) or a public key file's.
|
||||
func ParseKeyLine(line string) (PublicKey, error) {
|
||||
fields := fieldsQuoted(strings.TrimSpace(line))
|
||||
for i, f := range fields {
|
||||
if !KeyTypes[f] || i+1 >= len(fields) {
|
||||
continue
|
||||
}
|
||||
k := PublicKey{Type: f, Options: strings.Join(fields[:i], " "), Comment: strings.Join(fields[i+2:], " ")}
|
||||
blob, err := base64.StdEncoding.DecodeString(fields[i+1])
|
||||
if err != nil {
|
||||
return k, fmt.Errorf("the key after %s is not base64", f)
|
||||
}
|
||||
sum := sha256.Sum256(blob)
|
||||
k.Fingerprint = "SHA256:" + base64.RawStdEncoding.EncodeToString(sum[:])
|
||||
k.Bits = bitsOf(f, blob)
|
||||
switch {
|
||||
case f == "ssh-dss":
|
||||
k.Weak = "DSA: refused by current OpenSSH"
|
||||
case f == "ssh-rsa" && k.Bits > 0 && k.Bits < 3072:
|
||||
k.Weak = fmt.Sprintf("RSA of %d bits: below 3072", k.Bits)
|
||||
}
|
||||
return k, nil
|
||||
}
|
||||
return PublicKey{}, fmt.Errorf("no public key on this line")
|
||||
}
|
||||
|
||||
// fieldsQuoted splits on spaces outside double quotes, as sshd reads an options field.
|
||||
func fieldsQuoted(s string) []string {
|
||||
var out []string
|
||||
var cur strings.Builder
|
||||
quoted := false
|
||||
for _, ch := range s {
|
||||
switch {
|
||||
case ch == '"':
|
||||
quoted = !quoted
|
||||
cur.WriteRune(ch)
|
||||
case (ch == ' ' || ch == '\t') && !quoted:
|
||||
if cur.Len() > 0 {
|
||||
out = append(out, cur.String())
|
||||
cur.Reset()
|
||||
}
|
||||
default:
|
||||
cur.WriteRune(ch)
|
||||
}
|
||||
}
|
||||
if cur.Len() > 0 {
|
||||
out = append(out, cur.String())
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// bitsOf reads a key's size from its blob: the RSA modulus, the curve, or ed25519's fixed 256.
|
||||
func bitsOf(kind string, blob []byte) int {
|
||||
strs := [][]byte{}
|
||||
for len(blob) >= 4 {
|
||||
n := binary.BigEndian.Uint32(blob)
|
||||
if int(n) > len(blob)-4 {
|
||||
break
|
||||
}
|
||||
strs = append(strs, blob[4:4+n])
|
||||
blob = blob[4+n:]
|
||||
}
|
||||
switch {
|
||||
case strings.Contains(kind, "ed25519"):
|
||||
return 256
|
||||
case kind == "ssh-rsa" && len(strs) >= 3:
|
||||
return new(big.Int).SetBytes(strs[2]).BitLen()
|
||||
case kind == "ssh-dss" && len(strs) >= 2:
|
||||
return new(big.Int).SetBytes(strs[1]).BitLen()
|
||||
case strings.Contains(kind, "nistp256"):
|
||||
return 256
|
||||
case strings.Contains(kind, "nistp384"):
|
||||
return 384
|
||||
case strings.Contains(kind, "nistp521"):
|
||||
return 521
|
||||
}
|
||||
return 0
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
package forgekey
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Ran is what a command did: its output, its exit status, and why it never ran to an answer.
|
||||
type Ran struct {
|
||||
Stdout string
|
||||
Stderr string
|
||||
Status int
|
||||
// Err is "ENOENT" when the program is not installed, or says it was ended for taking too long.
|
||||
Err string
|
||||
}
|
||||
|
||||
// Runner runs one command with extra environment words, so every tool can be tested without ssh.
|
||||
type Runner func(ctx context.Context, env []string, name string, args ...string) Ran
|
||||
|
||||
// CallTimeout bounds one command of ExecRunner: below the runtime's thirty-second call limit. A variable so a
|
||||
// test can lower it.
|
||||
var CallTimeout = 20 * time.Second
|
||||
|
||||
// ExecRunner runs a command on this machine, with no terminal and nothing on its stdin, bounded by
|
||||
// CallTimeout.
|
||||
func ExecRunner(ctx context.Context, env []string, name string, args ...string) Ran {
|
||||
return ExecRunnerFor(CallTimeout)(ctx, env, name, args...)
|
||||
}
|
||||
|
||||
// ExecRunnerFor is ExecRunner with its own bound (issue 1497): a build module's long step passes 0, so only the
|
||||
// step's own context ends a command, and a fetch may outlast a tool call's thirty seconds.
|
||||
func ExecRunnerFor(bound time.Duration) Runner {
|
||||
return func(ctx context.Context, env []string, name string, args ...string) Ran {
|
||||
return execRun(ctx, bound, false, env, name, args...)
|
||||
}
|
||||
}
|
||||
|
||||
// ExecRunnerClean is ExecRunnerFor whose commands get exactly the env argument as their environment, and nothing of
|
||||
// this process's (issue 1497; novox/hq ADR 0466 §4: no registry password in a fetch's environment). The caller
|
||||
// gives PATH and HOME itself. A directory is the command's own business (`git -C`, `go -C`).
|
||||
func ExecRunnerClean(bound time.Duration) Runner {
|
||||
return func(ctx context.Context, env []string, name string, args ...string) Ran {
|
||||
return execRun(ctx, bound, true, env, name, args...)
|
||||
}
|
||||
}
|
||||
|
||||
func execRun(ctx context.Context, bound time.Duration, clean bool, env []string, name string, args ...string) Ran {
|
||||
if bound > 0 {
|
||||
var cancel context.CancelFunc
|
||||
ctx, cancel = context.WithTimeout(ctx, bound)
|
||||
defer cancel()
|
||||
}
|
||||
cmd := exec.CommandContext(ctx, name, args...)
|
||||
if clean {
|
||||
cmd.Env = append([]string{}, env...) // non-nil, so an empty list is an empty environment
|
||||
} else {
|
||||
cmd.Env = append(os.Environ(), env...)
|
||||
}
|
||||
var out, errb bytes.Buffer
|
||||
cmd.Stdout, cmd.Stderr = &out, &errb
|
||||
err := cmd.Run()
|
||||
r := Ran{Stdout: out.String(), Stderr: errb.String()}
|
||||
var exitErr *exec.ExitError
|
||||
switch {
|
||||
case errors.Is(ctx.Err(), context.DeadlineExceeded):
|
||||
r.Status, r.Err = 124, "ended by its step's deadline"
|
||||
if bound > 0 {
|
||||
r.Err = fmt.Sprintf("no answer within %d s", int(bound/time.Second))
|
||||
}
|
||||
case errors.Is(err, exec.ErrNotFound):
|
||||
r.Status, r.Err = 127, "ENOENT"
|
||||
case errors.As(err, &exitErr):
|
||||
r.Status = exitErr.ExitCode()
|
||||
case err != nil:
|
||||
r.Status, r.Err = 1, err.Error()
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
func firstLine(s string) string {
|
||||
for _, l := range strings.Split(s, "\n") {
|
||||
if l = strings.TrimSpace(l); l != "" {
|
||||
return l
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
Vendored
+2
@@ -2,11 +2,13 @@
|
||||
## explicit; go 1.26.0
|
||||
git.novox.be/novox/mesh-sdk/go
|
||||
git.novox.be/novox/mesh-sdk/go/asks
|
||||
git.novox.be/novox/mesh-sdk/go/buildstep
|
||||
git.novox.be/novox/mesh-sdk/go/catalogue
|
||||
git.novox.be/novox/mesh-sdk/go/compose
|
||||
git.novox.be/novox/mesh-sdk/go/declaration
|
||||
git.novox.be/novox/mesh-sdk/go/delivery
|
||||
git.novox.be/novox/mesh-sdk/go/facts
|
||||
git.novox.be/novox/mesh-sdk/go/forgekey
|
||||
git.novox.be/novox/mesh-sdk/go/membership
|
||||
git.novox.be/novox/mesh-sdk/go/overlay
|
||||
git.novox.be/novox/mesh-sdk/go/process
|
||||
|
||||
Reference in New Issue
Block a user