diff --git a/cmd/mesh-controller/key_takein.go b/cmd/mesh-controller/key_takein.go new file mode 100644 index 00000000..435cfacb --- /dev/null +++ b/cmd/mesh-controller/key_takein.go @@ -0,0 +1,427 @@ +package main + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "path" + "strings" + "time" + + "git.novox.be/novox/mesh-sdk/go/buildstep" + "git.novox.be/novox/mesh-sdk/go/catalogue" + sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" + "github.com/nats-io/nats.go/jetstream" + "github.com/novox/mesh-controller/internal/artifacts" + "github.com/novox/mesh-controller/internal/builderkeys" + "github.com/novox/mesh-controller/internal/buildreads" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +// The take-in from the builder's done key (novox/hq ADR 0501, ADR 0498 §8, ADR 0453 §5; issue 1851). +// +// From R4e the controller asks no build but the builder's own module's: a module of a merge is taken from the builder's +// key for the fingerprint the indexer's trunk value of the module names (ADR 0501 §2). Only a done key whose every +// artifact names a reference is taken (ADR 0498 §8); a key not done yet, one waiting for a reference, a templates +// archive the store does not answer, or a list of predefined keys the forge does not answer is waited for and said, +// never failed and never delivered from. A failed key fails, in the builder's own words. The take-in then records the +// key as a build of today's record layout, so a controller of before reads it on a rollback, and judges it as every +// build is judged (registerJudged). + +// errKeyWaits says a take-in from a key waits: for the builder, a reference, the store or the forge. Never a failure. +var errKeyWaits = errors.New("waits") + +// errKeyFailed says the module's key failed: the builder's own words follow. +var errKeyFailed = errors.New("the builder's key failed") + +// keyReads is what a take-in from a key reads of the builder and the indexer (builderkeys.Reader, or a test's). +type keyReads interface { + Build(ctx context.Context, module, fp string) (builderkeys.Key, error) + Manifest(ctx context.Context, digest string) (sdkdelivery.Manifest, error) +} + +// keyBuildID is the id a key's build is recorded under: the key and its revision, so the same revision recorded twice +// is one record, and a rebuild of the same fingerprint (a new revision, ADR 0429 §18) another. +func keyBuildID(k builderkeys.Key) string { + part, err := sdkdelivery.FingerprintKeyPart(k.Fingerprint) + if err != nil { + part = strings.TrimPrefix(k.Fingerprint, sdkdelivery.SourcePrefix2) + } + return fmt.Sprintf("key-%s-%d", part, k.Revision) +} + +// isKeyBuildID says a recorded build was taken from a builder's key. +func isKeyBuildID(id string) bool { return strings.HasPrefix(id, "key-") } + +// readArchive is the archive a key's reference names, read from the artifact store and checked against its digest. A +// variable so a test needs no store. +var readArchive = func(ctx context.Context, inv *inventory.Inventory, reference string) ([]byte, error) { + shelf, err := inv.Catalogue(ctx) + if err != nil { + return nil, err + } + address, err := artifactStoreAddress(ctx, inv, shelf, "") + if err != nil { + return nil, err + } + if address == "" { + return nil, errors.New("the artifact store is on no network the controller reaches") + } + return artifacts.Store{Address: address}.GetBlob(ctx, reference) +} + +// predefinedHead is the list of predefined keys at the head of the catalogue repository's trunk, and that head. +type predefinedHead struct { + List catalogue.PredefinedList + Repository string + Commit string + // Generation orders two heads of the trunk: the head's commit time in seconds. A list kept from a build's + // generation (its commit count) is always older than a head read here. + Generation int +} + +// predefinedAtTrunk is the list of predefined keys at the head of the catalogue repository's trunk (ADR 0501 §8), read +// from the forge. A variable so a test needs no forge. +var predefinedAtTrunk = func(ctx context.Context, inv *inventory.Inventory) (predefinedHead, error) { + entries, err := inv.Catalogued(ctx) + if err != nil { + return predefinedHead{}, err + } + repository := "" + for _, e := range entries { + if e.Source.Seat == gitSeat && path.Base(strings.Trim(e.Source.Repository, "/")) == catalogue.CatalogueRepository { + repository = strings.Trim(e.Source.Repository, "/") + break + } + } + if repository == "" { + return predefinedHead{}, fmt.Errorf("no module of the catalogue names the repository %s on the forge", + catalogue.CatalogueRepository) + } + var heads []struct { + SHA string `json:"sha"` + Commit struct { + Committer struct { + Date time.Time `json:"date"` + } `json:"committer"` + } `json:"commit"` + } + if err := askGitea(ctx, "gitea_api", map[string]any{"path": "/repos/" + repository + + "/commits?limit=1&stat=false&verification=false&files=false"}, &heads); err != nil { + return predefinedHead{}, err + } + if len(heads) == 0 || heads[0].SHA == "" { + return predefinedHead{}, fmt.Errorf("the forge named no head of %s's trunk", repository) + } + head := predefinedHead{Repository: repository, Commit: heads[0].SHA, + Generation: int(heads[0].Commit.Committer.Date.Unix())} + var file struct { + Content string `json:"content"` + Encoding string `json:"encoding"` + } + if err := askGitea(ctx, "gitea_api", map[string]any{"path": "/repos/" + repository + "/contents/" + + catalogue.PredefinedFile + "?ref=" + head.Commit}, &file); err != nil { + return predefinedHead{}, err + } + if file.Encoding != "base64" { + return predefinedHead{}, fmt.Errorf("the forge answered %s at %.8s in the encoding %q", catalogue.PredefinedFile, + head.Commit, file.Encoding) + } + raw, err := base64.StdEncoding.DecodeString(file.Content) + if err != nil { + return predefinedHead{}, err + } + if head.List, err = catalogue.ParsePredefined(raw); err != nil { + return predefinedHead{}, fmt.Errorf("%s at %.8s is refused: %w", catalogue.PredefinedFile, head.Commit, err) + } + return head, nil +} + +// askGitea asks the forge's module one tool on the bus and reads its result into into. +func askGitea(ctx context.Context, tool string, args map[string]any, into any) error { + js, err := aBus() + if err != nil { + return err + } + defer js.Close() + raw, _ := json.Marshal(args) + answer, err := link.Ask(ctx, link.OverNATS{Conn: js.Conn()}, "gitea", tool, raw, 30*time.Second) + if err != nil { + return err + } + if answer.Error != "" { + return fmt.Errorf("gitea.%s: %s", tool, answer.Error) + } + var wrapped struct { + Result json.RawMessage `json:"result"` + } + if err := json.Unmarshal(answer.Result, &wrapped); err != nil { + return err + } + return json.Unmarshal(wrapped.Result, into) +} + +// keyTake is one take-in from a key: the module, the indexer's trunk value naming the key's fingerprint, and the +// registration's warrant. +type keyTake struct { + Module string + Trunk sdkdelivery.Module + Ask registrationAsk +} + +// takeInKey takes a module in from the builder's key for the fingerprint its trunk value names: records the key as a +// build and registers the module, or says why not. errKeyWaits when it waits, errKeyFailed when the key failed. +func takeInKey(ctx context.Context, inv *inventory.Inventory, keys keyReads, t keyTake) ( + catalogue.Manifest, inventory.Build, error) { + head, err := predefinedAtTrunk(ctx, inv) + if err != nil { + return catalogue.Manifest{}, inventory.Build{}, fmt.Errorf("%w: the list of predefined keys at the catalogue's "+ + "trunk head cannot be read, so %s is not judged: %v", errKeyWaits, t.Module, err) + } + keepPredefinedHead(ctx, inv, head) + previous, err := inv.Builds(ctx, t.Module, 5) + if err != nil { + return catalogue.Manifest{}, inventory.Build{}, err + } + kept, manifest, err := buildOfKey(ctx, keys, t, keyContext{list: head.List, previous: previous, + archive: func(ctx context.Context, reference string) ([]byte, error) { return readArchive(ctx, inv, reference) }}) + if err != nil { + return catalogue.Manifest{}, kept, err + } + if err := inv.RecordBuild(ctx, kept); err != nil { + return catalogue.Manifest{}, kept, err + } + recorded := inventory.Source{Repository: t.Trunk.Repository, Seat: gitSeat, Path: keyPath(t.Trunk), + BuiltFrom: t.Trunk.Commit, Head: t.Trunk.Commit, Against: kept.Against} + trunk := "main" + if was, err := inv.SourceOf(ctx, manifest.Module); err == nil { + recorded.Ref = followedBranch(was.Ref) + if recorded.Ref != "" { + trunk = recorded.Ref + } + } + registered, err := registerJudged(ctx, inv, takenBuild{kept: kept, manifest: manifest, recorded: recorded, + ask: t.Ask, path: recorded.Path, trunk: trunk, by: "the builder", what: t.Trunk.Repository, + commit: t.Trunk.Commit}) + return registered, kept, err +} + +// keyContext is what a key's build is made with besides the key: the list of predefined keys its manifest is judged +// against, the module's recent builds (the bases its last one stood on are carried) and how an archive is read. +type keyContext struct { + list catalogue.PredefinedList + previous []inventory.Build + archive func(ctx context.Context, reference string) ([]byte, error) +} + +// keyPath is a module's directory as the catalogue records it: empty for a module at its repository's root. +func keyPath(m sdkdelivery.Module) string { + if m.Path == sdkdelivery.RootPath { + return "" + } + return strings.Trim(m.Path, "/") +} + +// buildOfKey reads the key of the trunk value's fingerprint and makes its build record and the manifest it resolves to +// (ADR 0498 §8, ADR 0501 §1, §8): the module's manifest by the key's digest, judged against the list of predefined keys +// at the catalogue's trunk head, resolved with the key's artifacts. +func buildOfKey(ctx context.Context, keys keyReads, t keyTake, kc keyContext) ( + inventory.Build, catalogue.Manifest, error) { + waits := func(format string, a ...any) (inventory.Build, catalogue.Manifest, error) { + return inventory.Build{}, catalogue.Manifest{}, fmt.Errorf("%w: "+format, append([]any{errKeyWaits}, a...)...) + } + if t.Trunk.FP2 == "" { + why := t.Trunk.None2 + if why == "" { + why = "the indexer wrote no fingerprint of the new form" + } + return waits("%s at %.8s has no fingerprint to take a build by: %s", t.Module, t.Trunk.Commit, why) + } + k, err := keys.Build(ctx, t.Module, t.Trunk.FP2) + if err != nil { + return waits("%s's key cannot be read: %v", t.Module, err) + } + switch { + case k.Failed(): + return inventory.Build{}, catalogue.Manifest{}, fmt.Errorf("%w: %s", errKeyFailed, k.Says()) + case !k.Done(): + return waits("%s", k.Says()) + case len(k.Unreferenced()) > 0: + return waits("%s", k.Says()) + } + src, err := keys.Manifest(ctx, k.Build.Manifest) + if err != nil { + return waits("%s's manifest: %v", t.Module, err) + } + manifest, err := catalogue.ParseManifestWith([]byte(src.JSON), kc.list) + if err != nil { + // **A word the running controller does not read waits for the controller that reads it** (ADR 0501 §7): the + // key stays done, and the controller that reads the word registers it with no second build. + if word := refusedWord(err.Error()); word != "" { + return waits("%s's manifest uses the word %q, which this controller does not read: it is registered "+ + "when a controller that reads it runs", t.Module, word) + } + return inventory.Build{}, catalogue.Manifest{}, fmt.Errorf("the builder built %s and its manifest is "+ + "refused: %w", t.Module, err) + } + if manifest.Module != t.Module { + return inventory.Build{}, catalogue.Manifest{}, fmt.Errorf("the key of %s names the manifest of %s", + t.Module, manifest.Module) + } + built, err := builtOfKey(ctx, kc.archive, manifest, k) + if err != nil { + return waits("%v", err) + } + resolved, err := manifest.Resolve(built) + if err != nil { + return inventory.Build{}, catalogue.Manifest{}, fmt.Errorf("the builder built %s and it does not resolve: %w", + t.Module, err) + } + raw, err := json.Marshal(resolved) + if err != nil { + return inventory.Build{}, catalogue.Manifest{}, err + } + kept := inventory.Build{ID: keyBuildID(k), Repository: t.Trunk.Repository, Module: t.Module, + Commit: t.Trunk.Commit, On: k.Build.By, Path: keyPath(t.Trunk), Manifest: raw, + SourceFingerprint: k.Fingerprint, At: time.Now().UTC()} + // **Read from the manifest, fresh at every key** (the same rule build-agent says, internal/buildreads). Sources are + // left unsaid: the planner then reads the module's whole directory, over-planning and never under-planning, and a + // fingerprint that did not change moves nothing (R4e A9). + for _, c := range buildreads.Of(manifest) { + kept.Read = append(kept.Read, inventory.ReadRepository{Repository: c.Repository, Ref: c.Ref}) + } + // **The bases its last build stood on, carried** (R4e, lead decision): what the collector keeps a mirrored base by. + // A key names none; the module's newest successful build's are carried so the keep set does not change. + for _, b := range kc.previous { + if b.Worked() && b.Module == t.Module && b.ID != kept.ID { + kept.Against = append([]string(nil), b.Against...) + break + } + } + for _, a := range k.Build.Artifacts { + kept.Made = append(kept.Made, inventory.Artifact{Name: a.Name, Kind: kindOf(manifest, a), + Reference: catalogue.Recorded(a.Reference)}) + } + return kept, resolved, nil +} + +// kindOf is an artifact's kind: the key's, else the manifest's. +func kindOf(m catalogue.Manifest, a sdkdelivery.BuiltArtifact) string { + if a.Kind != "" || m.Build == nil { + return a.Kind + } + for _, d := range m.Build.Artifacts { + if d.Name == a.Name { + return d.Kind + } + } + return "" +} + +// builtOfKey is each of a done key's artifacts as Resolve reads it: its reference and digest, a TypeScript bundle's +// launchers by the build modules' one rule (mesh-sdk buildstep.Launchers), and, for the archive a module packs its +// templates into, each template's text read back from the archive the store holds under the key's reference. +func builtOfKey(ctx context.Context, archive func(context.Context, string) ([]byte, error), m catalogue.Manifest, + k builderkeys.Key) ( + []catalogue.Built, error) { + declared := map[string]catalogue.Artifact{} + if m.Build != nil { + for _, a := range m.Build.Artifacts { + declared[a.Name] = a + } + } + templates := "" + if m.Templated() { + templates = catalogue.TemplatesArtifact(m) + } + var out []catalogue.Built + for _, a := range k.Build.Artifacts { + d := declared[a.Name] + b := catalogue.Built{Name: a.Name, Kind: kindOf(m, a), Reference: catalogue.Recorded(a.Reference), + Digest: a.Digest} + if launchers := buildstep.Launchers(d.Language, d.Entrypoints); len(launchers) > 0 { + b.Launchers = launchers + } + if a.Name == templates && templates != "" { + body, err := archive(ctx, b.Reference) + if err != nil { + return nil, fmt.Errorf("the templates archive %q of %s cannot be read from the artifact store: %v", + a.Name, m.Module, err) + } + if b.Texts, err = unpackTexts(body); err != nil { + return nil, fmt.Errorf("the templates archive %q of %s cannot be read back: %v", a.Name, m.Module, err) + } + } + out = append(out, b) + } + return out, nil +} + +// unpackTexts is every file of a packed templates archive (gzip over tar), by its path inside it. +func unpackTexts(body []byte) (map[string]string, error) { + zipped, err := gzip.NewReader(bytes.NewReader(body)) + if err != nil { + return nil, err + } + defer zipped.Close() + reader := tar.NewReader(zipped) + texts := map[string]string{} + for { + header, err := reader.Next() + if errors.Is(err, io.EOF) { + return texts, nil + } + if err != nil { + return nil, err + } + if header.Typeflag != tar.TypeReg { + continue + } + text, err := io.ReadAll(io.LimitReader(reader, 16<<20)) + if err != nil { + return nil, err + } + texts[strings.TrimPrefix(header.Name, "./")] = string(text) + } +} + +// keepPredefinedHead keeps the list read at the catalogue's trunk head when it is newer than the one kept, as a build of +// the catalogue's did (ADR 0348 §7), and judges every manifest read again against it. Said, never fatal: the list read +// is the one this take-in judges against either way. +func keepPredefinedHead(ctx context.Context, inv *inventory.Inventory, head predefinedHead) { + kept, err := inv.KeepPredefined(ctx, inventory.PredefinedRecord{Keys: head.List, Repository: head.Repository, + Commit: head.Commit, Generation: head.Generation, Build: "the trunk head " + short(head.Commit)}) + if err != nil { + fmt.Printf("the list of predefined keys at %s's trunk head %s is not kept: %v\n", head.Repository, + short(head.Commit), err) + return + } + if kept { + fmt.Printf("the list of predefined keys at %s's trunk head %s is published: %s\n", head.Repository, + short(head.Commit), strings.Join(head.List.Keys(), ", ")) + } + if err := loadPredefined(ctx, inv); err != nil { + fmt.Printf("the list of predefined keys kept cannot be read: %v\n", err) + } +} + +// openBuilderKeys is the builder's and the indexer's buckets, read on a connection for the moment. +func openBuilderKeys() (builderkeys.Reader, func(), error) { + js, err := aBus() + if err != nil { + return builderkeys.Reader{}, nil, err + } + api, err := jetstream.New(js.Conn()) + if err != nil { + js.Close() + return builderkeys.Reader{}, nil, err + } + return builderkeys.Reader{JS: api}, js.Close, nil +} diff --git a/cmd/mesh-controller/key_takein_integration_test.go b/cmd/mesh-controller/key_takein_integration_test.go new file mode 100644 index 00000000..39df46e7 --- /dev/null +++ b/cmd/mesh-controller/key_takein_integration_test.go @@ -0,0 +1,192 @@ +//go:build integration + +package main + +import ( + "context" + "encoding/json" + "errors" + "reflect" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" + "github.com/novox/mesh-controller/internal/builderkeys" + "github.com/novox/mesh-controller/internal/inventory" +) + +// withTrunkList stands in for the catalogue's trunk head: an empty list of predefined keys, readable, or not. +func withTrunkList(t *testing.T, readable bool) { + t.Helper() + was := predefinedAtTrunk + predefinedAtTrunk = func(context.Context, *inventory.Inventory) (predefinedHead, error) { + if !readable { + return predefinedHead{}, errors.New("the forge did not answer") + } + list, err := catalogue.ParsePredefined([]byte(`{}`)) + return predefinedHead{List: list, Repository: "novox/mesh-catalog", Commit: "abcdef0123456789", + Generation: 1 << 40}, err + } + t.Cleanup(func() { predefinedAtTrunk = was }) +} + +// aKeyFor is a done key of module at fp, of one archive with a reference, and the indexer's trunk value naming it at +// commit in repository/path. +func aKeyFor(module, version, fp, repository, path, commit string) (keyBuckets, sdkdelivery.Module) { + raw := `{"module":"` + module + `","version":"` + version + `","build":{"artifacts":[{"name":"a","kind":"archive","from":"files"}]},` + + `"resources":[{"id":"a","type":"archive","path":"/opt/` + module + `","artifact":"a"}]}` + digest := sdkdelivery.ManifestDigest([]byte(raw)) + blob := shaOf(module + version) + k := builderkeys.Key{Module: module, Fingerprint: fp, Found: true, Revision: 1, + Build: sdkdelivery.BuilderBuild{State: sdkdelivery.BuildDone, By: "ace", Manifest: digest, + Artifacts: []sdkdelivery.BuiltArtifact{{Name: "a", Kind: "archive", Digest: blob, + Reference: "artifact-store://" + module + "/a/blobs/" + blob}}}} + return keyBuckets{keys: map[string]builderkeys.Key{module: k}, manifests: map[string]string{digest: raw}}, + sdkdelivery.Module{FP2: fp, Manifest: digest, Repository: repository, Commit: commit, Path: path} +} + +func fpOf(c byte) string { return "src2:" + strings.Repeat(string(c), 64) } + +// A3, A4 (ADR 0498 §8, ADR 0501 §1-§3, §8): a done key of a trunk fingerprint records one build of today's layout, +// whose artifacts are the key's, and registers the module under the same rules of who may register as before. +func TestADoneKeyOfTheTrunkRegistersUnderTheSameRules(t *testing.T) { + open := theCatalogue(t) // sudo from novox/mesh-catalog modules/sudo, mesh-host from novox/mesh-host + ctx := t.Context() + inv := open.inventory + withTrunkList(t, true) + + // A known module from the repository and path the catalogue builds it from: registered from its key. + keys, trunk := aKeyFor("sudo", "2", fpOf('a'), "novox/mesh-catalog", "modules/sudo", "c0ffee0123456789") + _, kept, err := takeInKey(ctx, inv, keys, keyTake{Module: "sudo", Trunk: trunk, Ask: registrationAsk{Merge: trunk.Commit}}) + if err != nil { + t.Fatalf("0501 §1: a done key of sudo's trunk fingerprint was not taken in: %v", err) + } + if shelf, _ := inv.Catalogue(ctx); shelf["sudo"].Version != "2" { + t.Fatalf("0501 §1: sudo is %q", shelf["sudo"].Version) + } + got, found, err := inv.BuildByID(ctx, kept.ID) + if err != nil || !found || len(got.Made) != 1 || got.Made[0].Reference != kept.Made[0].Reference || + got.SourceFingerprint != fpOf('a') { + t.Fatalf("0498 §8: the recorded build is %+v (%v)", got, err) + } + // Recording the same key again is one record. + if _, _, err := takeInKey(ctx, inv, keys, keyTake{Module: "sudo", Trunk: trunk, Ask: registrationAsk{Merge: trunk.Commit}}); err != nil { + t.Fatal(err) + } + builds, _ := inv.Builds(ctx, "sudo", 20) + n := 0 + for _, b := range builds { + if b.ID == kept.ID { + n++ + } + } + if n != 1 { + t.Fatalf("0498 §8: one key taken in twice is %d records", n) + } + + // A known module from another repository: refused, as a build of it was (ADR 0266). + other, otherTrunk := aKeyFor("sudo", "evil", fpOf('b'), "agent/sudo", "modules/sudo", "d00d0123456789ab") + if _, _, err := takeInKey(ctx, inv, other, keyTake{Module: "sudo", Trunk: otherTrunk, + Ask: registrationAsk{Merge: otherTrunk.Commit}}); !errors.Is(err, errNotItsSource) { + t.Fatalf("0501 §3: a known module from another repository was taken in: %v", err) + } + // A new module from a repository the catalogue builds from: registered. + zram, zramTrunk := aKeyFor("zram", "1", fpOf('c'), "novox/mesh-catalog", "modules/zram", "c0ffee0123456789") + if _, _, err := takeInKey(ctx, inv, zram, keyTake{Module: "zram", Trunk: zramTrunk, Ask: registrationAsk{Merge: zramTrunk.Commit}}); err != nil { + t.Fatalf("0501 §3: a new module from a built repository was refused: %v", err) + } + // A new module from a repository nothing is built from, with no Approve: refused. + tools, toolsTrunk := aKeyFor("agent-tools", "1", fpOf('d'), "agent/tools", "", "e0e0e0e0e0e0e0e0") + if _, _, err := takeInKey(ctx, inv, tools, keyTake{Module: "agent-tools", Trunk: toolsTrunk, + Ask: registrationAsk{Merge: toolsTrunk.Commit}}); !errors.Is(err, errNotItsSource) { + t.Fatalf("0501 §3: a new repository's module was taken in with no warrant: %v", err) + } + // At the terminal the operator registers it (ADR 0501 §4). + if _, _, err := takeInKey(ctx, inv, tools, keyTake{Module: "agent-tools", Trunk: toolsTrunk, + Ask: registrationAsk{Terminal: true}}); err != nil { + t.Fatalf("0501 §4: the terminal's registration was refused: %v", err) + } +} + +// ADR 0501 §2: a key only a candidate names registers nothing: the trunk value names another fingerprint, whose key is +// not there, and the take-in waits. +func TestAKeyOnlyACandidateNamesRegistersNothing(t *testing.T) { + open := theCatalogue(t) + ctx := t.Context() + withTrunkList(t, true) + keys, trunk := aKeyFor("sudo", "3", fpOf('e'), "novox/mesh-catalog", "modules/sudo", "c0ffee0123456789") + trunk.FP2 = fpOf('f') // the trunk names another fingerprint than the candidate's done key + _, _, err := takeInKey(ctx, open.inventory, keys, keyTake{Module: "sudo", Trunk: trunk, Ask: registrationAsk{Merge: trunk.Commit}}) + if !errors.Is(err, errKeyWaits) { + t.Fatalf("0501 §2: %v", err) + } + if shelf, _ := open.inventory.Catalogue(ctx); shelf["sudo"].Version != "1" { + t.Fatalf("0501 §2: a candidate's key registered sudo %q", shelf["sudo"].Version) + } +} + +// ADR 0501 §8: a take-in that cannot read the list of predefined keys at the catalogue's trunk head waits, says so and +// registers nothing. +func TestATakeInWithoutTheTrunksListWaits(t *testing.T) { + open := theCatalogue(t) + ctx := t.Context() + withTrunkList(t, false) + keys, trunk := aKeyFor("sudo", "4", fpOf('1'), "novox/mesh-catalog", "modules/sudo", "c0ffee0123456789") + _, _, err := takeInKey(ctx, open.inventory, keys, keyTake{Module: "sudo", Trunk: trunk, Ask: registrationAsk{Merge: trunk.Commit}}) + if !errors.Is(err, errKeyWaits) || !strings.Contains(err.Error(), "predefined keys") { + t.Fatalf("0501 §8: %v", err) + } + if shelf, _ := open.inventory.Catalogue(ctx); shelf["sudo"].Version != "1" { + t.Fatalf("0501 §8: registered %q", shelf["sudo"].Version) + } +} + +// A3 (the lead's decision on Against and Read): a key's record keeps the collector's keep set as it was and its read +// repositories are its manifest's, fresh at every key. +func TestAKeysRecordKeepsTheKeepSetAndReadsItsManifest(t *testing.T) { + open := theCatalogue(t) + ctx := t.Context() + inv := open.inventory + withTrunkList(t, true) + before, err := inv.KeepSet(ctx) + if err != nil { + t.Fatal(err) + } + raw := `{"module":"sudo","version":"5","build":{"artifacts":[{"name":"img","kind":"image","from":"Dockerfile",` + + `"context":{"repository":"novox/elsewhere","seat":"git"}}]},"resources":[{"id":"run","type":"container","artifact":"img"}]}` + digest := sdkdelivery.ManifestDigest([]byte(raw)) + k := builderkeys.Key{Module: "sudo", Fingerprint: fpOf('2'), Found: true, Revision: 1, + Build: sdkdelivery.BuilderBuild{State: sdkdelivery.BuildDone, By: "g14", Manifest: digest, + Artifacts: []sdkdelivery.BuiltArtifact{{Name: "img", Kind: "image", Digest: shaOf("img5"), + Reference: "artifact-store://sudo/img@" + shaOf("img5")}}}} + keys := keyBuckets{keys: map[string]builderkeys.Key{"sudo": k}, manifests: map[string]string{digest: raw}} + trunk := sdkdelivery.Module{FP2: fpOf('2'), Manifest: digest, Repository: "novox/mesh-catalog", + Commit: "c0ffee0123456789", Path: "modules/sudo"} + if _, _, err := takeInKey(ctx, inv, keys, keyTake{Module: "sudo", Trunk: trunk, Ask: registrationAsk{Merge: trunk.Commit}}); err != nil { + t.Fatal(err) + } + after, err := inv.KeepSet(ctx) + if err != nil { + t.Fatal(err) + } + for reference, why := range before { + if !reflect.DeepEqual(after[reference], why) && !strings.Contains(strings.Join(after[reference], ","), strings.Join(why, ",")) { + t.Errorf("the keep set changed for %s: %v, was %v", reference, after[reference], why) + } + } + read, err := inv.ReadRepositories(ctx) + if err != nil { + t.Fatal(err) + } + found := false + for _, r := range read["sudo"] { + if !r.Own && r.Repository == "novox/elsewhere" { + found = true + } + } + if !found { + text, _ := json.Marshal(read["sudo"]) + t.Fatalf("the key's record does not read its manifest's context: %s", text) + } +} diff --git a/cmd/mesh-controller/key_takein_test.go b/cmd/mesh-controller/key_takein_test.go new file mode 100644 index 00000000..37928d49 --- /dev/null +++ b/cmd/mesh-controller/key_takein_test.go @@ -0,0 +1,231 @@ +package main + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" + "github.com/novox/mesh-controller/internal/builderkeys" + "github.com/novox/mesh-controller/internal/inventory" +) + +// keyBuckets is the builder's and the indexer's buckets as a take-in reads them, for a test: keys by module, manifests by +// digest. +type keyBuckets struct { + keys map[string]builderkeys.Key + manifests map[string]string +} + +func (k keyBuckets) Build(_ context.Context, module, fp string) (builderkeys.Key, error) { + if key, ok := k.keys[module]; ok && key.Fingerprint == fp { + return key, nil + } + return builderkeys.Key{Module: module, Fingerprint: fp}, nil +} + +func (k keyBuckets) Manifest(_ context.Context, digest string) (sdkdelivery.Manifest, error) { + raw, ok := k.manifests[digest] + if !ok { + return sdkdelivery.Manifest{}, errors.New("the indexer holds no manifest " + digest) + } + return sdkdelivery.Manifest{Module: "x", JSON: raw}, nil +} + +const keyFP = "src2:" + "1111111111111111111111111111111111111111111111111111111111111111" + +// aKeyOf is a done key of module x at keyFP for the manifest raw, with these artifacts. +func aKeyOf(raw string, revision uint64, made ...sdkdelivery.BuiltArtifact) (keyBuckets, sdkdelivery.Module) { + digest := sdkdelivery.ManifestDigest([]byte(raw)) + k := builderkeys.Key{Module: "x", Fingerprint: keyFP, Found: true, Revision: revision, + Build: sdkdelivery.BuilderBuild{State: sdkdelivery.BuildDone, By: "ace", Manifest: digest, Artifacts: made}} + trunk := sdkdelivery.Module{FP2: keyFP, Manifest: digest, Repository: "novox/mesh-catalog", Commit: "c0ffee0123456789", + Path: "modules/x"} + return keyBuckets{keys: map[string]builderkeys.Key{"x": k}, manifests: map[string]string{digest: raw}}, trunk +} + +const tsBundle = `{"module":"x","version":"1","tools":["x_read"], + "build":{"artifacts":[{"name":"tools","kind":"bundle","language":"typescript","entrypoints":["dist/index.js"]}, + {"name":"img","kind":"image","from":"Dockerfile","context":{"repository":"novox/other","seat":"git"}}]}, + "resources":[{"id":"run","type":"container","artifact":"img"}]}` + +func shaOf(b string) string { + sum := sha256.Sum256([]byte(b)) + return "sha256:" + hex.EncodeToString(sum[:]) +} + +// A3 (ADR 0498 §8, ADR 0501 §1): a done key whose every artifact names a reference is one recorded build, its artifacts +// the key's, its launchers by the build modules' rule, its reads from its manifest, the bases its last build stood on +// carried; recorded twice it is one record. +func TestADoneKeyIsOneRecordedBuildOfItsArtifacts(t *testing.T) { + keys, trunk := aKeyOf(tsBundle, 3, + sdkdelivery.BuiltArtifact{Name: "tools", Kind: "bundle", Digest: shaOf("t"), + Reference: "artifact-store://x/tools/blobs/" + shaOf("t")}, + sdkdelivery.BuiltArtifact{Name: "img", Kind: "image", Digest: shaOf("i"), + Reference: "artifact-store://x/img@" + shaOf("i")}) + previous := []inventory.Build{{ID: "build-1", Module: "x", Against: []string{"artifact-store://base@sha256:aa"}}} + take := keyTake{Module: "x", Trunk: trunk} + kept, resolved, err := buildOfKey(t.Context(), keys, take, keyContext{previous: previous}) + if err != nil { + t.Fatal(err) + } + if kept.ID != "key-1111111111111111111111111111111111111111111111111111111111111111-3" { + t.Errorf("the record's id is %s", kept.ID) + } + again, _, _ := buildOfKey(t.Context(), keys, take, keyContext{previous: previous}) + if again.ID != kept.ID { + t.Errorf("one key recorded twice is two records: %s and %s", kept.ID, again.ID) + } + if kept.SourceFingerprint != keyFP || kept.Commit != trunk.Commit || kept.Path != "modules/x" || kept.On != "ace" || + kept.Module != "x" || kept.Repository != "novox/mesh-catalog" { + t.Errorf("the record: %+v", kept) + } + if len(kept.Made) != 2 || kept.Made[0].Reference != "artifact-store://x/tools/blobs/"+shaOf("t") || + kept.Made[1].Reference != "artifact-store://x/img@"+shaOf("i") || kept.Made[1].Kind != "image" { + t.Errorf("Made is not the key's artifacts: %+v", kept.Made) + } + if len(kept.Read) != 1 || kept.Read[0].Repository != "novox/other" { + t.Errorf("Read is not the manifest's contexts: %+v", kept.Read) + } + if kept.Sources != nil || kept.Mirrored != nil { + t.Errorf("a key's record says build sources or mirrored bases it cannot know: %+v %+v", kept.Sources, kept.Mirrored) + } + if len(kept.Against) != 1 || kept.Against[0] != "artifact-store://base@sha256:aa" { + t.Errorf("the bases the last build stood on are not carried: %v", kept.Against) + } + if len(resolved.Bundles) != 1 || resolved.Bundles[0].Launchers["dist/index.js"] != "dist/index.serve.mjs" { + t.Errorf("the bundle's launchers: %+v", resolved.Bundles) + } + var m map[string]any + if err := json.Unmarshal(kept.Manifest, &m); err != nil || !strings.Contains(string(kept.Manifest), shaOf("i")) { + t.Errorf("the recorded manifest names no image digest: %s", kept.Manifest) + } +} + +// A3 (ADR 0498 §8): a key with an artifact naming no reference, a key not done and a missing key are waited for, said; +// a failed key fails in the builder's words; a module with no fingerprint of the new form waits, naming why. +func TestAKeyNotReadyWaitsAndAFailedKeyFails(t *testing.T) { + noRef, trunk := aKeyOf(tsBundle, 1, + sdkdelivery.BuiltArtifact{Name: "tools", Kind: "bundle", Digest: shaOf("t"), + Reference: "artifact-store://x/tools/blobs/" + shaOf("t")}, + sdkdelivery.BuiltArtifact{Name: "img", Kind: "image", Digest: shaOf("i")}) + take := keyTake{Module: "x", Trunk: trunk} + _, _, err := buildOfKey(t.Context(), noRef, take, keyContext{}) + if !errors.Is(err, errKeyWaits) || !strings.Contains(err.Error(), "waits for the reference of img") { + t.Errorf("a key without a reference: %v", err) + } + building := noRef + k := building.keys["x"] + k.Build = sdkdelivery.BuilderBuild{State: sdkdelivery.BuildBuilding, By: "g14"} + building.keys = map[string]builderkeys.Key{"x": k} + if _, _, err := buildOfKey(t.Context(), building, take, keyContext{}); !errors.Is(err, errKeyWaits) || + !strings.Contains(err.Error(), "building on g14") { + t.Errorf("a building key: %v", err) + } + if _, _, err := buildOfKey(t.Context(), keyBuckets{}, take, keyContext{}); !errors.Is(err, errKeyWaits) || + !strings.Contains(err.Error(), "no build of x") { + t.Errorf("no key: %v", err) + } + k.Build = sdkdelivery.BuilderBuild{State: sdkdelivery.BuildFailed, Error: "digests differ: ace sha256:1, g14 sha256:2"} + failed := keyBuckets{keys: map[string]builderkeys.Key{"x": k}} + if _, _, err := buildOfKey(t.Context(), failed, take, keyContext{}); !errors.Is(err, errKeyFailed) || + !strings.Contains(err.Error(), "digests differ: ace sha256:1, g14 sha256:2") { + t.Errorf("a failed key: %v", err) + } + none := take + none.Trunk.FP2, none.Trunk.None2 = "", "no lock" + if _, _, err := buildOfKey(t.Context(), noRef, none, keyContext{}); !errors.Is(err, errKeyWaits) || + !strings.Contains(err.Error(), "no lock") { + t.Errorf("no fingerprint: %v", err) + } +} + +// ADR 0501 §7: a manifest word this controller does not read waits for the controller that reads it; never fails. +func TestAManifestWordTheControllerDoesNotReadWaits(t *testing.T) { + raw := `{"module":"x","version":"1","not-a-word-yet":true,"build":{"artifacts":[{"name":"a","kind":"archive","from":"files"}]}, + "resources":[{"id":"a","type":"archive","path":"/opt/x","artifact":"a"}]}` + keys, trunk := aKeyOf(raw, 1, sdkdelivery.BuiltArtifact{Name: "a", Kind: "archive", Digest: shaOf("a"), + Reference: "artifact-store://x/a/blobs/" + shaOf("a")}) + _, _, err := buildOfKey(t.Context(), keys, keyTake{Module: "x", Trunk: trunk}, keyContext{}) + if !errors.Is(err, errKeyWaits) || !strings.Contains(err.Error(), `"not-a-word-yet"`) { + t.Fatalf("an unread word: %v", err) + } +} + +func packed(t *testing.T, files map[string]string) []byte { + t.Helper() + var buf bytes.Buffer + z := gzip.NewWriter(&buf) + w := tar.NewWriter(z) + for name, text := range files { + if err := w.WriteHeader(&tar.Header{Name: name, Mode: 0o644, Size: int64(len(text)), Typeflag: tar.TypeReg}); err != nil { + t.Fatal(err) + } + if _, err := w.Write([]byte(text)); err != nil { + t.Fatal(err) + } + } + if err := w.Close(); err != nil { + t.Fatal(err) + } + if err := z.Close(); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} + +// A templated module's texts are read back from the archive the store holds under the key's reference; an archive the +// store does not answer is waited for (the lead's Q1b). +func TestATemplatedModulesTextsAreReadFromItsArchive(t *testing.T) { + raw := `{"module":"x","version":"1","templates":"templates", + "build":{"artifacts":[{"name":"templates","kind":"archive","from":"templates"}]}, + "resources":[{"id":"main","type":"file","path":"/etc/x.conf","mode":"0644","template":"templates/x.conf"}]}` + body := packed(t, map[string]string{"x.conf": "listen 1\n"}) + sum := sha256.Sum256(body) + digest := "sha256:" + hex.EncodeToString(sum[:]) + keys, trunk := aKeyOf(raw, 1, sdkdelivery.BuiltArtifact{Name: "templates", Kind: "archive", Digest: digest, + Reference: "artifact-store://x/templates/blobs/" + digest}) + asked := "" + archive := func(_ context.Context, reference string) ([]byte, error) { asked = reference; return body, nil } + _, resolved, err := buildOfKey(t.Context(), keys, keyTake{Module: "x", Trunk: trunk}, keyContext{archive: archive}) + if err != nil { + t.Fatal(err) + } + if asked != "artifact-store://x/templates/blobs/"+digest { + t.Errorf("the archive asked: %q", asked) + } + if text, _ := json.Marshal(resolved.Resources); !strings.Contains(string(text), "listen 1") { + t.Errorf("the template's text is not in the file: %s", text) + } + gone := func(context.Context, string) ([]byte, error) { return nil, errors.New("the store does not hold it") } + if _, _, err := buildOfKey(t.Context(), keys, keyTake{Module: "x", Trunk: trunk}, keyContext{archive: gone}); !errors.Is(err, errKeyWaits) || + !strings.Contains(err.Error(), "cannot be read from the artifact store") { + t.Errorf("an archive the store does not answer: %v", err) + } +} + +// The launchers a key's bundle is resolved with are mesh-sdk's one rule (buildstep.Launchers), the names +// mesh-build-node writes: a TypeScript bundle's entrypoint x.js runs as x.serve.mjs, a Go bundle has none. +func TestALaunchersRuleIsTheBuildModules(t *testing.T) { + m, err := catalogue.ParseManifest([]byte(tsBundle)) + if err != nil { + t.Fatal(err) + } + k := builderkeys.Key{Build: sdkdelivery.BuilderBuild{Artifacts: []sdkdelivery.BuiltArtifact{{Name: "tools", Digest: "sha256:1", + Reference: "artifact-store://x/tools/blobs/sha256:1"}}}} + built, err := builtOfKey(t.Context(), nil, m, k) + if err != nil { + t.Fatal(err) + } + if built[0].Launchers["dist/index.js"] != "dist/index.serve.mjs" || built[0].Kind != "bundle" { + t.Fatalf("built: %+v", built[0]) + } +} diff --git a/internal/artifacts/blob.go b/internal/artifacts/blob.go new file mode 100644 index 00000000..9f769422 --- /dev/null +++ b/internal/artifacts/blob.go @@ -0,0 +1,57 @@ +package artifacts + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "net/http" +) + +// blobBound is the most GetBlob reads: a templates archive is a few kilobytes; anything near this is no archive a +// take-in reads text from. +const blobBound = 64 << 20 + +// GetBlob is the archive a recorded reference names (`artifact-store:///blobs/sha256:`), read whole +// and checked against the digest it names: bytes that hash to another digest are refused, never read (R4e: the +// controller reads a templates archive by the reference of a builder's done key). Gone when the store does not hold +// it; ErrNotOurs for a reference outside the store or one that is not an archive. +func (s Store) GetBlob(ctx context.Context, reference string) ([]byte, error) { + repository, digest, archive, err := s.archive(reference) + if err != nil { + return nil, err + } + if !archive { + return nil, fmt.Errorf("%w: %s is not an archive", ErrNotOurs, reference) + } + request, err := http.NewRequestWithContext(ctx, http.MethodGet, s.url(repository, "blobs", digest), nil) + if err != nil { + return nil, err + } + response, err := s.client().Do(request) + if err != nil { + return nil, fmt.Errorf("cannot reach the artifact store at %s: %w", s.Address, err) + } + defer response.Body.Close() + switch response.StatusCode { + case http.StatusOK: + case http.StatusNotFound: + return nil, Gone + default: + return nil, fmt.Errorf("the artifact store answered %s for %s/blobs/%s", response.Status, repository, digest) + } + body, err := io.ReadAll(io.LimitReader(response.Body, blobBound+1)) + if err != nil { + return nil, fmt.Errorf("reading %s/blobs/%s: %w", repository, digest, err) + } + if len(body) > blobBound { + return nil, fmt.Errorf("%s/blobs/%s is larger than %d bytes, which no archive read here is", repository, digest, blobBound) + } + sum := sha256.Sum256(body) + if got := "sha256:" + hex.EncodeToString(sum[:]); got != digest { + return nil, fmt.Errorf("the artifact store answered %s/blobs/%s with bytes of the digest %s: refused", repository, + digest, got) + } + return body, nil +} diff --git a/internal/artifacts/blob_test.go b/internal/artifacts/blob_test.go new file mode 100644 index 00000000..94b3b61d --- /dev/null +++ b/internal/artifacts/blob_test.go @@ -0,0 +1,44 @@ +package artifacts + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +// A take-in reads a templates archive by its reference, and only bytes of the digest the reference names. +func TestGetBlobReadsOnlyTheBytesOfItsDigest(t *testing.T) { + good := []byte("the archive") + sum := sha256.Sum256(good) + digest := "sha256:" + hex.EncodeToString(sum[:]) + liar := "sha256:" + strings.Repeat("0", 64) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case strings.HasSuffix(r.URL.Path, "/x/files/blobs/"+digest), strings.HasSuffix(r.URL.Path, "/x/files/blobs/"+liar): + _, _ = w.Write(good) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + defer server.Close() + s := Store{Address: strings.TrimPrefix(server.URL, "http://")} + ctx := context.Background() + body, err := s.GetBlob(ctx, "artifact-store://x/files/blobs/"+digest) + if err != nil || string(body) != string(good) { + t.Fatalf("the archive: %q %v", body, err) + } + if _, err := s.GetBlob(ctx, "artifact-store://x/files/blobs/"+liar); err == nil || !strings.Contains(err.Error(), "refused") { + t.Fatalf("bytes of another digest are read: %v", err) + } + if _, err := s.GetBlob(ctx, "artifact-store://x/other/blobs/"+digest); !errors.Is(err, Gone) { + t.Fatalf("a blob the store does not hold: %v", err) + } + if _, err := s.GetBlob(ctx, "registry.example/x@"+digest); !errors.Is(err, ErrNotOurs) { + t.Fatalf("a reference outside the store: %v", err) + } +} diff --git a/internal/builder/builder.go b/internal/builder/builder.go index eb5b09e3..ba962644 100644 --- a/internal/builder/builder.go +++ b/internal/builder/builder.go @@ -23,6 +23,7 @@ import ( "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/delivery" + "github.com/novox/mesh-controller/internal/buildreads" ) // Turning a repository into artifacts the mesh can pin. @@ -2001,33 +2002,9 @@ func instructions(recipe string) []string { return out } -// readBy is every repository other than the module's own that this build's recipes read source from, -// each once and in a fixed order, so two builds of one commit report the same thing the same way. -func readBy(manifest catalogue.Manifest) []catalogue.ArtifactContext { - if manifest.Build == nil { - return nil - } - seen := map[string]bool{} - var out []catalogue.ArtifactContext - for _, a := range manifest.Build.Artifacts { - if a.Context == nil || a.Context.Repository == "" { - continue - } - key := a.Context.Repository + "#" + a.Context.Ref - if seen[key] { - continue - } - seen[key] = true - out = append(out, *a.Context) - } - sort.Slice(out, func(i, j int) bool { - if out[i].Repository != out[j].Repository { - return out[i].Repository < out[j].Repository - } - return out[i].Ref < out[j].Ref - }) - return out -} +// readBy is every repository other than the module's own that this build's recipes read source from: the rule +// build-agent and the controller share (internal/buildreads). +func readBy(manifest catalogue.Manifest) []catalogue.ArtifactContext { return buildreads.Of(manifest) } // binaryName is what a compiled bundle's executable is called: what the artifact says, or the name of // the package it is built from, which is what a compiler would have chosen anyway. diff --git a/internal/buildreads/buildreads.go b/internal/buildreads/buildreads.go new file mode 100644 index 00000000..6f400bda --- /dev/null +++ b/internal/buildreads/buildreads.go @@ -0,0 +1,39 @@ +// Package buildreads is which repositories other than a module's own its build reads source from, as its manifest +// names them (novox/hq 04-ISSUES/131): the artifacts' `context` repositories. One rule for build-agent, which says it +// in a build's outcome, and for the controller, which reads it from a done key's manifest (R4e, issue 1851): it lives +// here, not in internal/builder, so it outlasts build-agent (R4g). +package buildreads + +import ( + "sort" + + "git.novox.be/novox/mesh-sdk/go/catalogue" +) + +// Of is every repository other than the module's own that this build's recipes read source from, each once and in a +// fixed order, so two builds of one commit report the same thing the same way. +func Of(manifest catalogue.Manifest) []catalogue.ArtifactContext { + if manifest.Build == nil { + return nil + } + seen := map[string]bool{} + var out []catalogue.ArtifactContext + for _, a := range manifest.Build.Artifacts { + if a.Context == nil || a.Context.Repository == "" { + continue + } + key := a.Context.Repository + "#" + a.Context.Ref + if seen[key] { + continue + } + seen[key] = true + out = append(out, *a.Context) + } + sort.Slice(out, func(i, j int) bool { + if out[i].Repository != out[j].Repository { + return out[i].Repository < out[j].Repository + } + return out[i].Ref < out[j].Ref + }) + return out +} diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/fetch.go b/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/fetch.go new file mode 100644 index 00000000..df6a1b01 --- /dev/null +++ b/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/fetch.go @@ -0,0 +1,114 @@ +package buildstep + +import ( + "context" + "fmt" + "net/url" + "regexp" + "strings" + + "git.novox.be/novox/mesh-sdk/go/forgekey" +) + +// Option changes FetchCommit. +type Option func(*fetchOptions) + +type fetchOptions struct{ allowFile bool } + +// AllowFileRemote lets FetchCommit take a file:// remote. It is for a test's local repository; production +// code does not pass it, and FetchCommit then refuses anything but ssh://. +func AllowFileRemote() Option { return func(o *fetchOptions) { o.allowFile = true } } + +var ( + commitOK = regexp.MustCompile(`^[0-9a-f]{40}$`) + refOK = regexp.MustCompile(`^refs/[A-Za-z0-9._/-]+$`) +) + +// FetchCommit fetches one commit of a forge repository into dir, an empty directory, and checks it out detached +// (novox/hq ADR 0466 §4, §5; issue 1474). remote is ssh:////.git, the ssh name being the +// one the build module's templates set up (ADR 0409 §1); commit is a full 40-hex sha; ref is optional, and names +// the pull request's head (refs/pull//head) when the commit is only there. Every git command runs through run +// as `git -C dir ...` with GIT_TERMINAL_PROMPT=0 and no credential helper, so nothing is asked for and no +// credential is used. A remote that is not ssh://, or one with user information, and a commit that is not 40 +// hex digits are refused before anything runs. +func FetchCommit(ctx context.Context, run forgekey.Runner, dir, remote, commit, ref string, opts ...Option) error { + var o fetchOptions + for _, opt := range opts { + opt(&o) + } + if err := checkFetch(o, dir, remote, commit, ref); err != nil { + return err + } + env := []string{"GIT_TERMINAL_PROMPT=0"} + git := func(args ...string) (string, error) { + full := append([]string{"-c", "credential.helper=", "-C", dir}, args...) + r := run(ctx, env, "git", full...) + if r.Status != 0 || r.Err != "" { + return "", fmt.Errorf("git %s: %s", args[0], firstLine(r.Err+" "+r.Stderr)) + } + return r.Stdout, nil + } + if _, err := git("init", "-q"); err != nil { + return err + } + if _, err := git("remote", "add", "origin", remote); err != nil { + return err + } + if _, err := git("fetch", "--no-tags", "--depth", "1", "origin", commit); err != nil { + if ref == "" { + return err + } + if _, rerr := git("fetch", "--no-tags", "origin", ref); rerr != nil { + return fmt.Errorf("%v; then the ref %s: %w", err, ref, rerr) + } + if _, verr := git("cat-file", "-e", commit+"^{commit}"); verr != nil { + return fmt.Errorf("the commit %s is not in %s: %w", commit, ref, verr) + } + } + if _, err := git("checkout", "-q", "--detach", commit); err != nil { + return err + } + out, err := git("rev-parse", "HEAD") + if err != nil { + return err + } + if head := strings.TrimSpace(out); head != commit { + return fmt.Errorf("the checkout is at %s, not at the commit %s", head, commit) + } + return nil +} + +func checkFetch(o fetchOptions, dir, remote, commit, ref string) error { + if dir == "" { + return fmt.Errorf("buildstep: no workspace directory") + } + if !commitOK.MatchString(commit) { + return fmt.Errorf("buildstep: %q is not a full commit (40 hex digits)", commit) + } + if ref != "" && !refOK.MatchString(ref) { + return fmt.Errorf("buildstep: %q is not a ref (refs/...)", ref) + } + u, err := url.Parse(remote) + if err != nil { + return fmt.Errorf("buildstep: the remote is not a URL: %w", err) + } + if u.User != nil { + return fmt.Errorf("buildstep: the remote carries user information; the key is the credential") + } + switch { + case u.Scheme == "ssh" && u.Host != "": + case u.Scheme == "file" && o.allowFile: + default: + return fmt.Errorf("buildstep: the remote %q is not ssh://", remote) + } + return nil +} + +func firstLine(s string) string { + for _, l := range strings.Split(s, "\n") { + if l = strings.TrimSpace(l); l != "" { + return l + } + } + return "" +} diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/guard.go b/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/guard.go new file mode 100644 index 00000000..a9dc49cd --- /dev/null +++ b/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/guard.go @@ -0,0 +1,52 @@ +package buildstep + +import ( + "context" + "encoding/json" + "fmt" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// Guard is a build module's check that the toolchain installed on its node is the one its recipe builds with +// (novox/hq ADR 0469 §1; issue 1473). The same guard serves ADR 0470 §4's postgres major for the integration +// level, with its own Probe and Want. +type Guard struct { + // Language is the toolchain's key part: go or typescript (delivery.ToolchainKey). + Language string + // Want is the toolchain version the recipe builds with, and Recipe the recipe's own version. The indexer + // compares both texts as they are written here. + Want, Recipe string + // Probe reads the installed version (go version, node --version). + Probe func(ctx context.Context) (installed string, err error) + Node string + // Put is the toolchains state's Put (stdio.State(delivery.ToolchainsState).Put). + Put Put +} + +// Check probes the installed version, writes it to . of the toolchains state every time it runs, +// and answers nil when it is the recipe's. Otherwise it answers a *Refusal naming both versions; the installed +// version is written even then, so the indexer sees that builders disagree. A probe that fails is a *Refusal +// naming its failure, and nothing is written. A write that fails is an error that is not a Refusal. +func (g Guard) Check(ctx context.Context) error { + installed, err := g.Probe(ctx) + if err != nil { + return &Refusal{Text: fmt.Sprintf("the installed %s version could not be read: %v", g.Language, err), Want: g.Want} + } + key, err := delivery.ToolchainKey(g.Node, g.Language) + if err != nil { + return err + } + raw, err := delivery.EncodeToolchain(delivery.Toolchain{Language: g.Language, Version: installed, Recipe: g.Recipe, At: now().UTC()}) + if err != nil { + return err + } + if _, err := g.Put(key, json.RawMessage(raw)); err != nil { + return fmt.Errorf("buildstep: writing the toolchain %s: %w", key, err) + } + if installed != g.Want { + return &Refusal{Text: fmt.Sprintf("%s %s installed, the recipe builds with %s %s", g.Language, installed, g.Language, g.Want), + Want: g.Want, Have: installed} + } + return nil +} diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/launchers.go b/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/launchers.go new file mode 100644 index 00000000..4b59f5cd --- /dev/null +++ b/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/launchers.go @@ -0,0 +1,24 @@ +package buildstep + +import "strings" + +// LauncherSuffix is what a TypeScript entrypoint's launcher is called beside it: index.js is started as +// index.serve.mjs (novox/hq ADR 0193), an ES module by its own extension whatever the bundle's package.json says. +const LauncherSuffix = ".serve.mjs" + +// Launchers are the launchers a bundle's build writes, by entrypoint, relative to the bundle's root: for TypeScript +// every entrypoint `x.js` is started by `x.serve.mjs` beside it; an entrypoint that is not JavaScript and every other +// language, whose build is already executable, get none. The one rule mesh-build-node, build-agent and the controller +// share (R4e), so the name a build writes is the name a reader expects. +func Launchers(language string, entrypoints []string) map[string]string { + out := map[string]string{} + if language != "typescript" { + return out + } + for _, entry := range entrypoints { + if strings.HasSuffix(entry, ".js") { + out[entry] = strings.TrimSuffix(entry, ".js") + LauncherSuffix + } + } + return out +} diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/merge.go b/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/merge.go new file mode 100644 index 00000000..6bc8df9e --- /dev/null +++ b/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/merge.go @@ -0,0 +1,112 @@ +package buildstep + +import ( + "context" + "fmt" + "strings" + + "git.novox.be/novox/mesh-sdk/go/forgekey" +) + +// MergeResult is what MergeInto made of a head and a trunk commit: the trunk commit (Base) and the head, whole, and the +// merge commit and the paths the merge changes against the trunk commit; or, when the head does not merge cleanly, the +// files in conflict, with Merge and Changed empty. +type MergeResult struct { + Base, Head, Merge string + Conflicts []string + Changed []string +} + +// MergeInto makes, in the workspace dir that FetchCommit left at commit, the merge of commit into the trunk commit +// base, as the forge's default merge would (a merge commit, never a fast-forward), and leaves the workspace at the merge +// (novox/hq ADR 0484 clause 3, ADR 0323). It is the one merge the build modules' fetch steps make, with the identity +// and flags of the controller's check merge (mesh-controller internal/builder/check_merge.go): a name nobody owns, no +// signature, no hook, nothing of the machine's git configuration that changes a merge. Nothing is pushed. +// +// A merge needs both histories, so it makes the fetch whole (the shallow clone FetchCommit made is deepened) and +// fetches base by its sha. A head that does not merge cleanly is an answer, not an error: the files in conflict, the +// merge undone and the workspace left at base. A head already in base answers a Merge equal to Base and no changed +// paths. An error is that the merge could not be tried: a base or commit that is not a full sha, no base, a workspace +// not at commit, or a fetch or git command that failed. remote and the options are those of FetchCommit; the runner's +// bound must allow a whole fetch. +func MergeInto(ctx context.Context, run forgekey.Runner, dir, remote, base, commit string, opts ...Option) (MergeResult, error) { + var o fetchOptions + for _, opt := range opts { + opt(&o) + } + res := MergeResult{Base: base, Head: commit} + if base == "" { + return res, fmt.Errorf("buildstep: no base: a merge needs the trunk commit it is made onto") + } + if err := checkFetch(o, dir, remote, base, ""); err != nil { + return res, err + } + if err := checkFetch(o, dir, remote, commit, ""); err != nil { + return res, err + } + env := []string{"GIT_TERMINAL_PROMPT=0"} + git := func(args ...string) (string, error) { + full := append([]string{"-c", "credential.helper=", "-C", dir}, args...) + r := run(ctx, env, "git", full...) + if r.Status != 0 || r.Err != "" { + return "", fmt.Errorf("git %s: %s", args[0], firstLine(r.Err+" "+r.Stderr)) + } + return r.Stdout, nil + } + at, err := git("rev-parse", "HEAD") + if err != nil { + return res, err + } + if head := strings.TrimSpace(at); head != commit { + return res, fmt.Errorf("buildstep: the workspace is at %s, not at the commit %s", head, commit) + } + shallow, err := git("rev-parse", "--is-shallow-repository") + if err != nil { + return res, err + } + fetch := []string{"fetch", "--no-tags", "origin", base} + if strings.TrimSpace(shallow) == "true" { + fetch = []string{"fetch", "--no-tags", "--unshallow", "origin", base} + } + if _, err := git(fetch...); err != nil { + return res, fmt.Errorf("the trunk commit %s: %w", base, err) + } + if _, err := git("checkout", "-q", "--detach", base); err != nil { + return res, fmt.Errorf("the trunk commit %s cannot be checked out: %w", base, err) + } + message := fmt.Sprintf("Check the merge of %s into %s (made for a check, never pushed)", short(commit), short(base)) + args := append(append([]string{}, mergeIdentity...), "merge", "--no-ff", "--no-edit", "--no-verify", "--quiet", "-m", message, commit) + if _, err := git(args...); err != nil { + out, _ := git("diff", "--name-only", "--diff-filter=U") + res.Conflicts = strings.Fields(out) + _, _ = git("merge", "--abort") + if len(res.Conflicts) == 0 { + return res, fmt.Errorf("the head %s could not be merged into the trunk commit %s: %w", short(commit), short(base), err) + } + return res, nil + } + merge, err := git("rev-parse", "HEAD") + if err != nil { + return res, fmt.Errorf("the merge cannot be read: %w", err) + } + res.Merge = strings.TrimSpace(merge) + changed, err := git("diff", "--name-only", "--no-renames", base, res.Merge) + if err != nil { + return res, fmt.Errorf("the paths the merge changes cannot be read: %w", err) + } + res.Changed = strings.Fields(changed) + return res, nil +} + +// mergeIdentity is who a check's merge is made as: it is never pushed, so it names nobody's account. No hook of the +// clone's or the build machine's git configuration runs, and no setting there changes the merge (the controller's +// check merge, internal/builder/check_merge.go). +var mergeIdentity = []string{"-c", "user.name=mesh merge check", "-c", "user.email=merge-check@mesh.invalid", + "-c", "commit.gpgsign=false", "-c", "core.hooksPath=/dev/null", "-c", "merge.verifySignatures=false"} + +func short(commit string) string { + if len(commit) > 12 { + return commit[:12] + } + return commit +} diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/pack.go b/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/pack.go new file mode 100644 index 00000000..bab02b2b --- /dev/null +++ b/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/pack.go @@ -0,0 +1,100 @@ +package buildstep + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "crypto/sha256" + "encoding/hex" + "fmt" + "os" + "path/filepath" + "sort" +) + +// Pack tars and gzips a directory the way the build agent does, byte for byte, and answers the archive and its +// digest, "sha256:" and the hex of the archive's bytes (issue 1479; novox/hq ADR 0405 §7, ADR 0453 §7). It is a +// copy of mesh-controller's internal/builder/builder.go pack (origin/main 073ccc29, :1351-1425), which stays +// where it is until the build agent goes: a build module's digest is compared with the build agent's for one +// fingerprint, so the bytes must be equal. +// +// Deterministically: regular files only (a symlink or a device is refused), sorted relative paths with +// slashes, each header carrying its name, permission bits and size and nothing else, the gzip default writer. +func Pack(root string) (archive []byte, digest string, err error) { + raw, err := pack(root) + if err != nil { + return nil, "", err + } + sum := sha256.Sum256(raw) + return raw, "sha256:" + hex.EncodeToString(sum[:]), nil +} + +func pack(root string) ([]byte, error) { + info, err := os.Stat(root) + if err != nil { + return nil, err + } + if !info.IsDir() { + return nil, fmt.Errorf("%s is not a directory", root) + } + + var paths []string + err = filepath.Walk(root, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() || !info.Mode().IsRegular() { + // Only files. A symlink or a device in an archive is refused by the host that unpacks + // it, so putting one in would build something that cannot be applied. + if !info.IsDir() && !info.Mode().IsRegular() { + return fmt.Errorf("%s is neither a file nor a directory, and an archive carries "+ + "only those", path) + } + return nil + } + paths = append(paths, path) + return nil + }) + if err != nil { + return nil, err + } + // filepath.Walk walks in lexical order; the sort is kept as the build agent keeps it, since an archive + // whose digest changes because the traversal did is a silent failure. + sort.Strings(paths) + + var out bytes.Buffer + zipped := gzip.NewWriter(&out) + writer := tar.NewWriter(zipped) + for _, path := range paths { + body, err := os.ReadFile(path) + if err != nil { + return nil, err + } + relative, err := filepath.Rel(root, path) + if err != nil { + return nil, err + } + info, err := os.Stat(path) + if err != nil { + return nil, err + } + mode := int64(info.Mode().Perm()) + if err := writer.WriteHeader(&tar.Header{ + Name: filepath.ToSlash(relative), Mode: mode, Size: int64(len(body)), + Typeflag: tar.TypeReg, + // Everything else left at its zero value on purpose: no times, owners or names. + }); err != nil { + return nil, err + } + if _, err := writer.Write(body); err != nil { + return nil, err + } + } + if err := writer.Close(); err != nil { + return nil, err + } + if err := zipped.Close(); err != nil { + return nil, err + } + return out.Bytes(), nil +} diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/refusal.go b/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/refusal.go new file mode 100644 index 00000000..0f885294 --- /dev/null +++ b/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/refusal.go @@ -0,0 +1,13 @@ +package buildstep + +// Refusal is a step that is not run on purpose, as against one that failed: the toolchain installed is not the +// recipe's, the integration level's postgres major is not the one it is built for (novox/hq ADR 0469 §1, ADR 0470 +// §4; issue 1473). A step's run returns one and the runner ends the step refused with its text. +type Refusal struct { + // Text is the reason, in words the operator reads. + Text string + // Want is the version the recipe asks for and Have the one found; either may be empty. + Want, Have string +} + +func (r *Refusal) Error() string { return r.Text } diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/step.go b/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/step.go new file mode 100644 index 00000000..c5ae0d95 --- /dev/null +++ b/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/step.go @@ -0,0 +1,99 @@ +// Package buildstep is what both build modules share: the runner of a long step that answers a tool call at once +// and ends on its key in the module's `steps` state (novox/hq ADR 0469 §2; issue 1471), the toolchain guard +// (issue 1473) and the fetch of one commit over SSH (issue 1474). +package buildstep + +import ( + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "strconv" + "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// Put writes one key of a state: stdio.State(delivery.StepsState).Put's shape (go/state.go). +type Put func(key string, value any) (uint64, error) + +// EndAttempts and EndRetry bound how a step's end is written: a write that fails is tried again, since the end +// is the only way the step's reader learns it is over. +var ( + EndAttempts = 5 + EndRetry = 2 * time.Second +) + +var now = time.Now + +// NewID is a step id: the time in milliseconds in base 36 and six random hex digits, lower-case letters and +// digits only, so it is one key token and differs between two steps of one node even in one millisecond. +func NewID() string { + var b [3]byte + _, _ = rand.Read(b[:]) + return strconv.FormatInt(now().UnixMilli(), 36) + hex.EncodeToString(b[:]) +} + +// Start begins a step of tool on node and answers its id at once: the key . holds state running when +// Start returns. run is then called in a goroutine of its own, under ctx (the process's, not the tool call's: the +// call is cut at 30 s, the step is not). When it returns, the key is written passed with run's result, failed with +// the error's text, or refused when the error is a *Refusal; a panic in run ends the step failed, naming it, and +// the process goes on. If the running write fails, nothing was started and Start answers the error. +func Start(ctx context.Context, put Put, node, tool string, run func(ctx context.Context) (result any, err error)) (string, error) { + step := delivery.Step{ID: NewID(), Tool: tool, Node: node, State: delivery.StepRunning, Started: now().UTC()} + key, err := delivery.StepKey(node, step.ID) + if err != nil { + return "", err + } + if err := write(put, key, step); err != nil { + return "", fmt.Errorf("buildstep: writing the running step %s: %w", key, err) + } + go func() { + result, err := protect(ctx, run) + step.Ended = now().UTC() + switch ref := (*Refusal)(nil); { + case err == nil: + step.State = delivery.StepPassed + if result != nil { + raw, merr := json.Marshal(result) + if merr != nil { + step.State, step.Reason = delivery.StepFailed, "the result is not JSON: "+merr.Error() + } else { + step.Result = raw + } + } + case errors.As(err, &ref): + step.State, step.Reason = delivery.StepRefused, err.Error() + default: + step.State, step.Reason = delivery.StepFailed, err.Error() + } + for attempt := 1; ; attempt++ { + if werr := write(put, key, step); werr == nil || attempt >= EndAttempts { + return + } + time.Sleep(EndRetry) + } + }() + return step.ID, nil +} + +// protect is run with a panic turned into the error that ends the step. +func protect(ctx context.Context, run func(ctx context.Context) (any, error)) (result any, err error) { + defer func() { + if p := recover(); p != nil { + result, err = nil, fmt.Errorf("the step panicked: %v", p) + } + }() + return run(ctx) +} + +func write(put Put, key string, step delivery.Step) error { + raw, err := delivery.EncodeStep(step) + if err != nil { + return err + } + _, err = put(key, json.RawMessage(raw)) + return err +} diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/trees.go b/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/trees.go new file mode 100644 index 00000000..58b2ccd8 --- /dev/null +++ b/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/trees.go @@ -0,0 +1,67 @@ +package buildstep + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "os" + "path/filepath" + "strings" + + "git.novox.be/novox/mesh-sdk/go/forgekey" +) + +// ModuleTree is what a merge holds of one module directory: its tree id, the sha256 of its module.json and the sha256 +// of each lock file it has (go.sum, package-lock.json), from which the checker and the indexer take the module's inputs +// (novox/hq ADR 0484 clause 3). +type ModuleTree struct { + Tree string `json:"tree"` + Manifest string `json:"manifest"` + Locks map[string]string `json:"locks,omitempty"` +} + +var lockFiles = []string{"go.sum", "package-lock.json"} + +// ModuleTrees are the trees, in commit, of every directory of the workspace dir that holds a module.json, by directory +// (the repository's root is "."). The workspace is at commit, so the manifests and the lock files are read from it. env is +// the environment git runs with (nil for none). Both build modules' fetch steps answer it for a merge. +func ModuleTrees(ctx context.Context, run forgekey.Runner, env []string, dir, commit string) (map[string]ModuleTree, error) { + r := run(ctx, env, "git", "-C", dir, "ls-files", "--", "module.json", "*/module.json") + if r.Status != 0 || r.Err != "" { + return nil, fmt.Errorf("git ls-files: %s", firstLine(r.Err+" "+r.Stderr)) + } + trees := map[string]ModuleTree{} + for _, l := range strings.Split(r.Stdout, "\n") { + if l = strings.TrimSpace(l); l == "" { + continue + } + mdir := filepath.ToSlash(filepath.Dir(l)) + spec := commit + ":" + mdir + if mdir == "." { + spec = commit + "^{tree}" + } + t := run(ctx, env, "git", "-C", dir, "rev-parse", spec) + if t.Status != 0 || t.Err != "" { + return nil, fmt.Errorf("git rev-parse %s: %s", spec, firstLine(t.Err+" "+t.Stderr)) + } + mt := ModuleTree{Tree: strings.TrimSpace(t.Stdout)} + raw, err := os.ReadFile(filepath.Join(dir, filepath.FromSlash(l))) + if err != nil { + return nil, err + } + mt.Manifest = sha256Hex(raw) + for _, lock := range lockFiles { + if raw, err := os.ReadFile(filepath.Join(dir, filepath.FromSlash(mdir), lock)); err == nil { + if mt.Locks == nil { + mt.Locks = map[string]string{} + } + mt.Locks[lock] = sha256Hex(raw) + } + } + trees[mdir] = mt + } + return trees, nil +} + +func sha256Hex(b []byte) string { h := sha256.Sum256(b); return hex.EncodeToString(h[:]) } diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/forgekey/keykeeper.go b/vendor/git.novox.be/novox/mesh-sdk/go/forgekey/keykeeper.go new file mode 100644 index 00000000..cb756685 --- /dev/null +++ b/vendor/git.novox.be/novox/mesh-sdk/go/forgekey/keykeeper.go @@ -0,0 +1,127 @@ +// Package forgekey is the key keeper every module that reaches the forge over SSH shares: ssh-client for a +// machine's accounts, a build module for its own (design 47 R4c-3, ADR 0466 §2; issue 1469). +package forgekey + +// The key this machine's account uses to reach the forge over SSH (novox/hq ADR 0399, ADR 0400; issue 955, +// area S1). It is made here, on the machine, once: ssh-keygen writes ~/.ssh/mesh_ed25519 only when it is +// missing, so the private half never leaves the machine and an existing key is never replaced. The public +// half and its fingerprint go into this module's `keys` state, one entry per account under ., +// for the forge's provider to register. The tools bundle is the one process of this module, so the keeper +// runs inside it beside the tools (main.go), the way claude-code reports what it holds. + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "time" +) + +// KeyFile is the key's name under ~/.ssh: never one of the operator's own id_* files. +const KeyFile = "mesh_ed25519" + +// The kinds of key (ADR 0400 §1, ADR 0466 §2): a machine's account's own, and a build module's. +const ( + KindMachine = "machine" + KindBuilder = "builder" +) + +// KeyRetry is how soon it looks again after a failure. +const KeyRetry = 30 * time.Second + +// KeyRound is how often the keeper looks again. +const KeyRound = 10 * time.Minute + +// KeyEntry is one value of the keys state. It carries the public half only. +type KeyEntry struct { + Node string `json:"node"` + Account string `json:"account"` + Kind string `json:"kind"` + Public string `json:"public"` + Fingerprint string `json:"fingerprint"` + At string `json:"at"` +} + +// KeyPutter is the keys state, as far as the keeper needs it. +type KeyPutter interface { + Put(key string, value any) (uint64, error) +} + +// KeyKeeper keeps one account's key and its entry. +type KeyKeeper struct { + Home, Node, Account string + Kind string // KindMachine or KindBuilder + Run Runner + State KeyPutter + Now func() time.Time + // After waits between rounds; time.After when nil. A test sets it to see which pause was chosen. + After func(time.Duration) <-chan time.Time + + reported string // the public half and fingerprint last put +} + +func (k *KeyKeeper) path() string { return filepath.Join(k.Home, ".ssh", KeyFile) } + +// Once makes the key when it is missing and puts the entry when it differs from the one last put. +func (k *KeyKeeper) Once(ctx context.Context) error { + if err := k.ensure(ctx); err != nil { + return err + } + raw, err := os.ReadFile(k.path() + ".pub") + if err != nil { + return fmt.Errorf("the public half: %w", err) + } + pk, err := ParseKeyLine(string(raw)) + if err != nil { + return fmt.Errorf("%s.pub: %w", k.path(), err) + } + public := strings.TrimSpace(string(raw)) + if sig := public + "|" + pk.Fingerprint; sig == k.reported { + return nil + } + entry := KeyEntry{Node: k.Node, Account: k.Account, Kind: k.Kind, Public: public, + Fingerprint: pk.Fingerprint, At: k.Now().UTC().Format(time.RFC3339)} + if _, err := k.State.Put(k.Node+"."+k.Account, entry); err != nil { + return err + } + k.reported = public + "|" + pk.Fingerprint + return nil +} + +// ensure makes the pair when the private half is missing. Never over an existing key. +func (k *KeyKeeper) ensure(ctx context.Context) error { + if _, err := os.Lstat(k.path()); err == nil { + return nil + } else if !os.IsNotExist(err) { + return err + } + if err := os.MkdirAll(filepath.Dir(k.path()), 0o700); err != nil { + return err + } + r := k.Run(ctx, nil, "ssh-keygen", "-q", "-t", "ed25519", "-N", "", "-C", k.Account+"@"+k.Node+" (mesh)", "-f", k.path()) + if r.Status != 0 || r.Err != "" { + return fmt.Errorf("ssh-keygen: %s", firstLine(r.Err+" "+r.Stderr)) + } + return nil +} + +// Keep runs Once now and every KeyRound, reporting a failure through say and trying again at the next round. +func (k *KeyKeeper) Keep(ctx context.Context, say func(string, ...any)) { + for { + pause := KeyRound + if err := k.Once(ctx); err != nil { + say("the forge key: %v", err) + pause = KeyRetry // the state or the bus's grant may arrive after the process + } + after := k.After + if after == nil { + after = time.After + } + select { + case <-ctx.Done(): + return + case <-after(pause): + } + } +} diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/forgekey/keys.go b/vendor/git.novox.be/novox/mesh-sdk/go/forgekey/keys.go new file mode 100644 index 00000000..4af0499b --- /dev/null +++ b/vendor/git.novox.be/novox/mesh-sdk/go/forgekey/keys.go @@ -0,0 +1,110 @@ +package forgekey + +// Keys, by their public half only. A fingerprint is computed here from the public key's bytes, as +// ssh-keygen -l does (SHA256 of the key blob, unpadded base64); a private key is never read by this +// process — its first line, the PEM header, says it is one, and ssh-keygen is asked about it. + +import ( + "crypto/sha256" + "encoding/base64" + "encoding/binary" + "fmt" + "math/big" + "strings" +) + +// KeyTypes are the public key algorithms an authorized_keys or known_hosts line can carry. +var KeyTypes = map[string]bool{ + "ssh-ed25519": true, "ssh-rsa": true, "ssh-dss": true, + "ecdsa-sha2-nistp256": true, "ecdsa-sha2-nistp384": true, "ecdsa-sha2-nistp521": true, + "sk-ssh-ed25519@openssh.com": true, "sk-ecdsa-sha2-nistp256@openssh.com": true, +} + +// PublicKey is one public key as a line carries it. +type PublicKey struct { + Type string `json:"type"` + Bits int `json:"bits"` + Fingerprint string `json:"fingerprint"` + Comment string `json:"comment"` + Options string `json:"options,omitempty"` + Weak string `json:"weak,omitempty"` +} + +// ParseKeyLine reads one authorized_keys line (options, type, key, comment) or a public key file's. +func ParseKeyLine(line string) (PublicKey, error) { + fields := fieldsQuoted(strings.TrimSpace(line)) + for i, f := range fields { + if !KeyTypes[f] || i+1 >= len(fields) { + continue + } + k := PublicKey{Type: f, Options: strings.Join(fields[:i], " "), Comment: strings.Join(fields[i+2:], " ")} + blob, err := base64.StdEncoding.DecodeString(fields[i+1]) + if err != nil { + return k, fmt.Errorf("the key after %s is not base64", f) + } + sum := sha256.Sum256(blob) + k.Fingerprint = "SHA256:" + base64.RawStdEncoding.EncodeToString(sum[:]) + k.Bits = bitsOf(f, blob) + switch { + case f == "ssh-dss": + k.Weak = "DSA: refused by current OpenSSH" + case f == "ssh-rsa" && k.Bits > 0 && k.Bits < 3072: + k.Weak = fmt.Sprintf("RSA of %d bits: below 3072", k.Bits) + } + return k, nil + } + return PublicKey{}, fmt.Errorf("no public key on this line") +} + +// fieldsQuoted splits on spaces outside double quotes, as sshd reads an options field. +func fieldsQuoted(s string) []string { + var out []string + var cur strings.Builder + quoted := false + for _, ch := range s { + switch { + case ch == '"': + quoted = !quoted + cur.WriteRune(ch) + case (ch == ' ' || ch == '\t') && !quoted: + if cur.Len() > 0 { + out = append(out, cur.String()) + cur.Reset() + } + default: + cur.WriteRune(ch) + } + } + if cur.Len() > 0 { + out = append(out, cur.String()) + } + return out +} + +// bitsOf reads a key's size from its blob: the RSA modulus, the curve, or ed25519's fixed 256. +func bitsOf(kind string, blob []byte) int { + strs := [][]byte{} + for len(blob) >= 4 { + n := binary.BigEndian.Uint32(blob) + if int(n) > len(blob)-4 { + break + } + strs = append(strs, blob[4:4+n]) + blob = blob[4+n:] + } + switch { + case strings.Contains(kind, "ed25519"): + return 256 + case kind == "ssh-rsa" && len(strs) >= 3: + return new(big.Int).SetBytes(strs[2]).BitLen() + case kind == "ssh-dss" && len(strs) >= 2: + return new(big.Int).SetBytes(strs[1]).BitLen() + case strings.Contains(kind, "nistp256"): + return 256 + case strings.Contains(kind, "nistp384"): + return 384 + case strings.Contains(kind, "nistp521"): + return 521 + } + return 0 +} diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/forgekey/runner.go b/vendor/git.novox.be/novox/mesh-sdk/go/forgekey/runner.go new file mode 100644 index 00000000..14bba8af --- /dev/null +++ b/vendor/git.novox.be/novox/mesh-sdk/go/forgekey/runner.go @@ -0,0 +1,93 @@ +package forgekey + +import ( + "bytes" + "context" + "errors" + "fmt" + "os" + "os/exec" + "strings" + "time" +) + +// Ran is what a command did: its output, its exit status, and why it never ran to an answer. +type Ran struct { + Stdout string + Stderr string + Status int + // Err is "ENOENT" when the program is not installed, or says it was ended for taking too long. + Err string +} + +// Runner runs one command with extra environment words, so every tool can be tested without ssh. +type Runner func(ctx context.Context, env []string, name string, args ...string) Ran + +// CallTimeout bounds one command of ExecRunner: below the runtime's thirty-second call limit. A variable so a +// test can lower it. +var CallTimeout = 20 * time.Second + +// ExecRunner runs a command on this machine, with no terminal and nothing on its stdin, bounded by +// CallTimeout. +func ExecRunner(ctx context.Context, env []string, name string, args ...string) Ran { + return ExecRunnerFor(CallTimeout)(ctx, env, name, args...) +} + +// ExecRunnerFor is ExecRunner with its own bound (issue 1497): a build module's long step passes 0, so only the +// step's own context ends a command, and a fetch may outlast a tool call's thirty seconds. +func ExecRunnerFor(bound time.Duration) Runner { + return func(ctx context.Context, env []string, name string, args ...string) Ran { + return execRun(ctx, bound, false, env, name, args...) + } +} + +// ExecRunnerClean is ExecRunnerFor whose commands get exactly the env argument as their environment, and nothing of +// this process's (issue 1497; novox/hq ADR 0466 §4: no registry password in a fetch's environment). The caller +// gives PATH and HOME itself. A directory is the command's own business (`git -C`, `go -C`). +func ExecRunnerClean(bound time.Duration) Runner { + return func(ctx context.Context, env []string, name string, args ...string) Ran { + return execRun(ctx, bound, true, env, name, args...) + } +} + +func execRun(ctx context.Context, bound time.Duration, clean bool, env []string, name string, args ...string) Ran { + if bound > 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, bound) + defer cancel() + } + cmd := exec.CommandContext(ctx, name, args...) + if clean { + cmd.Env = append([]string{}, env...) // non-nil, so an empty list is an empty environment + } else { + cmd.Env = append(os.Environ(), env...) + } + var out, errb bytes.Buffer + cmd.Stdout, cmd.Stderr = &out, &errb + err := cmd.Run() + r := Ran{Stdout: out.String(), Stderr: errb.String()} + var exitErr *exec.ExitError + switch { + case errors.Is(ctx.Err(), context.DeadlineExceeded): + r.Status, r.Err = 124, "ended by its step's deadline" + if bound > 0 { + r.Err = fmt.Sprintf("no answer within %d s", int(bound/time.Second)) + } + case errors.Is(err, exec.ErrNotFound): + r.Status, r.Err = 127, "ENOENT" + case errors.As(err, &exitErr): + r.Status = exitErr.ExitCode() + case err != nil: + r.Status, r.Err = 1, err.Error() + } + return r +} + +func firstLine(s string) string { + for _, l := range strings.Split(s, "\n") { + if l = strings.TrimSpace(l); l != "" { + return l + } + } + return "" +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 30efdfb1..26ad86b3 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -2,11 +2,13 @@ ## explicit; go 1.26.0 git.novox.be/novox/mesh-sdk/go git.novox.be/novox/mesh-sdk/go/asks +git.novox.be/novox/mesh-sdk/go/buildstep git.novox.be/novox/mesh-sdk/go/catalogue git.novox.be/novox/mesh-sdk/go/compose git.novox.be/novox/mesh-sdk/go/declaration git.novox.be/novox/mesh-sdk/go/delivery git.novox.be/novox/mesh-sdk/go/facts +git.novox.be/novox/mesh-sdk/go/forgekey git.novox.be/novox/mesh-sdk/go/membership git.novox.be/novox/mesh-sdk/go/overlay git.novox.be/novox/mesh-sdk/go/process