W12.5 (Go): mesh-build-go keeps the store holder's refused build and serves kept-artifact (issue 1966)
Builds design 47 *The artifact store's holder takes its own new build even when its seat is the one broken* (lines ~1811-1830, test 60): a trunk build of a module claiming oci-artifact-registry that the seat refuses or does not answer is kept as an object by digest and answered kept (BuildAnswer.Kept); kept-artifact answers its store and object, refusing a module that claims no oci-artifact-registry, one not kept here and one under another digest. The store is the template objects (ADR 0666 §4: a part outside the foundation keeps its data in ADR 0595's templates; design 47's "old objects form" contradicts it for a non-foundation part, reported). Whether a module claims the seat is asked of the seat mesh-catalog's verb module (ADR 0596). Callers are left to the seat definition and the grants part (ADR 0666 §20). build_go_kept reads the kept builds, build_go_kept_drop drops one (ADR 0666 §25). Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ (Joined from 5e1cdade.) Builds: design 47 G §271, test 60 (BLD9) Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ
This commit is contained in:
@@ -230,6 +230,10 @@ func (s AskSteps) build(ctx context.Context, job sdkstep.BuildJob) (any, error)
|
||||
if job.Publish {
|
||||
args["publish"] = true
|
||||
}
|
||||
// The fingerprint a build kept for the artifact store's holder is asked by (kept-artifact, milestone 47/G W12.5).
|
||||
if job.Fp != "" {
|
||||
args["fp"] = job.Fp
|
||||
}
|
||||
// The seat it is published through and the version the builder gave it (design 51 *Publishing*, CAT-BG2).
|
||||
if a.Seat != "" {
|
||||
args["seat"] = a.Seat
|
||||
@@ -254,11 +258,15 @@ func (s AskSteps) build(ctx context.Context, job sdkstep.BuildJob) (any, error)
|
||||
var built struct {
|
||||
Digest string `json:"digest"`
|
||||
Reference string `json:"reference"`
|
||||
Kept string `json:"kept"`
|
||||
}
|
||||
if err := json.Unmarshal(r, &built); err != nil || built.Digest == "" {
|
||||
return nil, fmt.Errorf("the build of %s answered no digest: %s", a.Artifact, r)
|
||||
}
|
||||
answer.Artifacts = append(answer.Artifacts, sdkstep.AskBuilt{Artifact: a.Artifact, Digest: built.Digest, Reference: built.Reference})
|
||||
if built.Kept != "" {
|
||||
answer.Kept = strings.TrimPrefix(built.Kept, "sha256:")
|
||||
}
|
||||
}
|
||||
if s.Imports != nil && f.Workspace != "" {
|
||||
out, err := s.Imports(ctx, map[string]any{"workspace": f.Workspace, "module": job.Path})
|
||||
|
||||
@@ -45,9 +45,9 @@ func (m *Module) httpClient() *http.Client {
|
||||
|
||||
// buildArchive packs one archive artifact of the module in dir: a pinned file when it names a url, else the directory
|
||||
// `from` of the module's tree.
|
||||
func (m *Module) buildArchive(ctx context.Context, workspace, module, name, dir string, art catalogue.Artifact) (any, error) {
|
||||
func (m *Module) buildArchive(ctx context.Context, workspace, module, name, dir string, art catalogue.Artifact, keep *keptFor) (any, error) {
|
||||
if art.URL != "" {
|
||||
return m.pinnedArchive(ctx, workspace, module, name, art)
|
||||
return m.pinnedArchive(ctx, workspace, module, name, art, keep)
|
||||
}
|
||||
from := strings.Trim(path.Clean("/"+filepath.ToSlash(art.From)), "/")
|
||||
if strings.TrimSpace(art.From) == "" {
|
||||
@@ -79,12 +79,12 @@ func (m *Module) buildArchive(ctx context.Context, workspace, module, name, dir
|
||||
if !within(realDir, realRoot) {
|
||||
return nil, fmt.Errorf("the archive %s is taken from %q, which lies outside the module: it resolves to %s, and the module is %s", art.Name, art.From, realRoot, realDir)
|
||||
}
|
||||
return m.packInto(ctx, workspace, module, name, art.Name, "archive", root)
|
||||
return m.packInto(ctx, workspace, module, name, art.Name, "archive", root, keep)
|
||||
}
|
||||
|
||||
// pinnedArchive fetches the file an artifact is pinned to, checks it against the pin, lays it out as the artifact says
|
||||
// (kept whole as `file`, or unpacked from `within`) and packs it.
|
||||
func (m *Module) pinnedArchive(ctx context.Context, workspace, module, name string, art catalogue.Artifact) (any, error) {
|
||||
func (m *Module) pinnedArchive(ctx context.Context, workspace, module, name string, art catalogue.Artifact, keep *keptFor) (any, error) {
|
||||
file, err := m.pinnedFile(ctx, workspace, art)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", art.Name, err)
|
||||
@@ -113,7 +113,7 @@ func (m *Module) pinnedArchive(ctx context.Context, workspace, module, name stri
|
||||
} else if err := unpackUpstream(file, laid, art.Within); err != nil {
|
||||
return nil, fmt.Errorf("%s: %s (%s): %w", art.Name, art.URL, art.Version, err)
|
||||
}
|
||||
return m.packInto(ctx, workspace, module, name, art.Name, "archive", laid)
|
||||
return m.packInto(ctx, workspace, module, name, art.Name, "archive", laid, keep)
|
||||
}
|
||||
|
||||
// pinnedFile is the path of the pinned file in the workspace: kept from an earlier build when its bytes are still the
|
||||
|
||||
@@ -105,7 +105,7 @@ func (m *Module) Puller(steps []StepTool, source buildstep.AskSource) *buildstep
|
||||
|
||||
// SeatTools are the seat's verbs on this node (ADR 0525 §10): what it runs, and its pause and resume.
|
||||
func (m *Module) SeatTools(p *buildstep.Puller) []stdio.Tool {
|
||||
return append(buildstep.SeatTools(Seat, "Go", p), m.stepsTool())
|
||||
return append(buildstep.SeatTools(Seat, "Go", p), m.stepsTool(), m.keptArtifactVerb())
|
||||
}
|
||||
|
||||
// StepEndTool ends one step left running (issue 1890); the module's steps state is the one it reads and writes.
|
||||
|
||||
@@ -119,7 +119,7 @@ func mustJSON(v any) []byte { b, _ := json.Marshal(v); return b }
|
||||
// the node's build capacity as a setting, with the builder's default; the seat's verbs are served as its tools.
|
||||
func TestManifestClaimsTheGoBuildSeatAndReadsItsCapacity(t *testing.T) {
|
||||
m := readManifest(t)
|
||||
if len(m.Claims) != 1 || m.Claims[0].Name != Seat || !slices.Equal(m.Claims[0].Serves, []string{"current", "drop", "pause", "resume", "steps"}) {
|
||||
if len(m.Claims) != 1 || m.Claims[0].Name != Seat || !slices.Equal(m.Claims[0].Serves, []string{"current", "drop", "pause", "resume", "steps", "kept-artifact"}) {
|
||||
t.Fatalf("claims = %+v", m.Claims)
|
||||
}
|
||||
if len(m.Reads) != 0 {
|
||||
|
||||
@@ -39,6 +39,9 @@ type BuildResult struct {
|
||||
// §9): oci-artifact-registry/<module>/<artifact>@<digest>. The builder writes the older form beside it while a
|
||||
// counted node-engine reads only that (ADR 0624 §1, §3).
|
||||
Reference string `json:"reference"`
|
||||
// Kept is the archive's digest when the seat did not store it and this node kept it for the artifact store's holder
|
||||
// (keptbuilds.go); Reference is then empty.
|
||||
Kept string `json:"kept,omitempty"`
|
||||
}
|
||||
|
||||
func (m *Module) buildTool() StepTool {
|
||||
@@ -116,12 +119,14 @@ func (m *Module) build(ctx context.Context, args map[string]any) (any, error) {
|
||||
}
|
||||
// The version the build ask carries is the build's argument MESH_VERSION for every artifact (design 47 G §370).
|
||||
version, _ := args["version"].(string)
|
||||
fp, _ := args["fp"].(string)
|
||||
keep := keptOf(publish, manifest, fp)
|
||||
if art.Kind == catalogue.ArtifactPackage && art.Language == "go" {
|
||||
seat, _ := args["seat"].(string)
|
||||
return m.buildGoPackage(ctx, workspace, dir, *art, version, seat)
|
||||
}
|
||||
if art.Kind == "archive" {
|
||||
return m.buildArchive(ctx, workspace, module, manifest.Module, dir, *art)
|
||||
return m.buildArchive(ctx, workspace, module, manifest.Module, dir, *art, keep)
|
||||
}
|
||||
if art.Kind != "bundle" || art.Language != "go" {
|
||||
return nil, fmt.Errorf("the artifact %s of %s is a %s of language %q: this module builds Go bundles and archives", name, manifest.Module, art.Kind, art.Language)
|
||||
@@ -164,7 +169,7 @@ func (m *Module) build(ctx context.Context, args map[string]any) (any, error) {
|
||||
if err := m.insideModule(workspace, dir, out); err != nil {
|
||||
return nil, fmt.Errorf("the output directory of %s: %w", art.Name, err)
|
||||
}
|
||||
res, err := m.packInto(ctx, workspace, module, manifest.Module, art.Name, "bundle", out)
|
||||
res, err := m.packInto(ctx, workspace, module, manifest.Module, art.Name, "bundle", out, keep)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -175,15 +180,22 @@ func (m *Module) build(ctx context.Context, args map[string]any) (any, error) {
|
||||
// packInto packs a directory with mesh-sdk's Pack, as the build agent packs every artifact, and publishes the archive
|
||||
// on the seat oci-artifact-registry from this process, after the build step ended (design 51 *Publishing*; publish.go):
|
||||
// a candidate's as a trunk build's, under <module>/<artifact> by its digest, of kind (bundle or archive). The step
|
||||
// answers the reference; a publish the seat refuses fails the step `not stored:`. The packed bytes never stay on the
|
||||
// node: they are held in this process until the publish answered (the lead's decision L9).
|
||||
func (m *Module) packInto(ctx context.Context, workspace, module, name, artifact, kind, root string) (BuildResult, error) {
|
||||
// answers the reference; a publish the seat refuses fails the step `not stored:`, unless keep names the build of the
|
||||
// artifact store's holder, which this node then keeps and answers as kept (keptbuilds.go). The packed bytes never stay on
|
||||
// the node: they are held in this process until the publish answered (the lead's decision L9).
|
||||
func (m *Module) packInto(ctx context.Context, workspace, module, name, artifact, kind, root string, keep *keptFor) (BuildResult, error) {
|
||||
archive, digest, err := buildstep.Pack(root)
|
||||
if err != nil {
|
||||
return BuildResult{}, fmt.Errorf("packing %s: %w", artifact, err)
|
||||
}
|
||||
if err := m.publishArchive(ctx, name, artifact, kind, digest, archive); err != nil {
|
||||
return BuildResult{}, fmt.Errorf("%s%s: %w", builds.NotStored, artifact, err)
|
||||
if keep == nil {
|
||||
return BuildResult{}, fmt.Errorf("%s%s: %w", builds.NotStored, artifact, err)
|
||||
}
|
||||
if kerr := m.keepBuild(*keep, digest, archive); kerr != nil {
|
||||
return BuildResult{}, fmt.Errorf("%s%s: %w; and it could not be kept: %v", builds.NotStored, artifact, err, kerr)
|
||||
}
|
||||
return BuildResult{Artifact: artifact, Digest: digest, Size: len(archive), Kept: digest}, nil
|
||||
}
|
||||
return BuildResult{Artifact: artifact, Digest: digest, Size: len(archive), Reference: artifactref.SeatReference(ArtifactSeat, name, artifact, digest)}, nil
|
||||
}
|
||||
|
||||
@@ -433,6 +433,7 @@ func (m *Module) goEnvFor(dir string) []string {
|
||||
func (m *Module) CleanWorkspaces(now time.Time) {
|
||||
m.sweepSaid(m.spaces().Sweep(now, buildstep.UnheldBound))
|
||||
m.SweepKept(now, KeptBound)
|
||||
m.sweepKeptBuilds(now, KeptBound)
|
||||
}
|
||||
|
||||
// clock is the time now: Now when a test set it, else the wall clock.
|
||||
|
||||
@@ -148,8 +148,10 @@ func TestManifestRequiresForgeAccountKeepsKeysAndTakesFilesFromTemplates(t *test
|
||||
// node-tools: the module pulls its asks through the tool runner's seat pull (1911).
|
||||
// go-package-registry: the Go seat's read side, the fetch's first source (design 51, CAT-BG3, E2).
|
||||
// detected-steps: the state of the verb steps (design 47 G §326, §414), asked by the template state.
|
||||
// kept-builds: the builds kept for the artifact store's holder (keptbuilds.go), by the template objects (W12.5).
|
||||
kept := map[string]any{"name": "kept-builds", "storage": "object-store", "template": "objects", "options": map[string]any{"max-mib": float64(1024)}}
|
||||
detected := map[string]any{"name": "detected-steps", "storage": "key-value", "template": "state", "options": map[string]any{"history": float64(1)}}
|
||||
if want := []any{detected, "forge-account", "go-toolchain", "go-package-registry", "node-tools"}; !reflect.DeepEqual(m.Requires, want) {
|
||||
if want := []any{detected, kept, "forge-account", "go-toolchain", "go-package-registry", "node-tools"}; !reflect.DeepEqual(m.Requires, want) {
|
||||
t.Errorf("requires = %v, want %v", m.Requires, want)
|
||||
}
|
||||
keeps := false
|
||||
|
||||
@@ -109,19 +109,31 @@ func (m *Module) keptTools() []stdio.Tool {
|
||||
{Name: KeptTool,
|
||||
Description: "The programs this node keeps, built from a commit so each commit is built once: the composers of compose asks and " +
|
||||
"the checking programs of run asks, each with its kind, key, size in bytes and when an ask last used it. One no ask used " +
|
||||
"for 30 days is removed. Read-only.",
|
||||
Run: func(map[string]any) (any, error) { return map[string]any{"kept": m.Kept()}, nil }},
|
||||
"for 30 days is removed. Under builds, the builds it keeps for the artifact store's holder that the seat did not store " +
|
||||
"(served by kept-artifact): each one's module, fingerprint, digest, size and when it was kept; one is removed after 30 days. Read-only.",
|
||||
Run: func(map[string]any) (any, error) {
|
||||
builds, err := m.keptBuilds()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]any{"kept": m.Kept(), "builds": builds}, nil
|
||||
}},
|
||||
{Name: KeptDropTool,
|
||||
Description: "Remove one kept program by its kind (composers or programs) and key, as build_go_kept names them: the next ask " +
|
||||
"for that commit builds it again. One already gone is no error.",
|
||||
"for that commit builds it again. With the kind builds and a build's digest as its key, remove a build kept for the " +
|
||||
"artifact store's holder: its want then waits for the next build. One already gone is no error.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{
|
||||
"kind": map[string]any{"type": "string", "enum": []string{"composers", "programs"}},
|
||||
"key": map[string]any{"type": "string", "description": "the key build_go_kept names"},
|
||||
"kind": map[string]any{"type": "string", "enum": []string{"composers", "programs", keptBuildsKind}},
|
||||
"key": map[string]any{"type": "string", "description": "the key build_go_kept names (a kept build's digest)"},
|
||||
}, "required": []string{"kind", "key"}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
kind, _ := args["kind"].(string)
|
||||
key, _ := args["key"].(string)
|
||||
if err := m.DropKept(kind, key); err != nil {
|
||||
drop := m.DropKept
|
||||
if kind == keptBuildsKind {
|
||||
drop = func(_, key string) error { return m.dropKeptBuild(key) }
|
||||
}
|
||||
if err := drop(kind, key); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]any{"dropped": key, "kind": kind}, nil
|
||||
|
||||
@@ -0,0 +1,214 @@
|
||||
package main
|
||||
|
||||
// **The artifact store's holder takes its own new build even when its seat is the one broken** (novox/hq design 47; test
|
||||
// 60; milestone 47/G W12.5, issue 1966): when the seat oci-artifact-registry refuses or does not answer the store of a
|
||||
// trunk build of a module that claims that seat, this module keeps that build on the bus instead, in its object store
|
||||
// `kept-builds` (asked for in `requires` by the template objects: ADR 0666 §4 keeps a part outside the foundation in
|
||||
// ADR 0595's templates), one object per build named by its digest; the build's answer says it kept it (BuildAnswer.Kept),
|
||||
// so the step key and the builder's done key name it. The node-engine composing the holder asks the seat's verb
|
||||
// kept-artifact of the keeping node for where the bytes are, and reads them from the store itself. The kept builds are
|
||||
// read by build_go_kept, dropped one by one by build_go_kept_drop, and swept with the kept programs after KeptBound.
|
||||
// Who may call the verb is the seat definition's (the node-engines alone), granted by the grants part from it (ADR 0666
|
||||
// §20): this module checks no caller of its own.
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildgo"
|
||||
ociseat "git.novox.be/novox/mesh-delivery/contracts/go/seats/ociartifactregistry"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscalls"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
)
|
||||
|
||||
// KeptBuildsStore is the local name of the object store of kept builds, as the manifest's `requires` asks for it.
|
||||
const KeptBuildsStore = "kept-builds"
|
||||
|
||||
// keptBuildsKind is the kind build_go_kept and build_go_kept_drop name a kept build by, beside composers and programs.
|
||||
const keptBuildsKind = "builds"
|
||||
|
||||
// KeptObjects is the object store of kept builds as the module uses it (busstate.ObjectsOf(KeptBuildsStore)).
|
||||
type KeptObjects interface {
|
||||
Put(name string, data []byte, description string) (buscalls.ObjectInfo, error)
|
||||
List() ([]buscalls.ObjectInfo, error)
|
||||
Delete(name string) error
|
||||
}
|
||||
|
||||
// keptFor is what a build keeps a refused artifact for: the module, which claims the store's seat, and its build's
|
||||
// fingerprint, which kept-artifact is asked by.
|
||||
type keptFor struct{ Module, Fp string }
|
||||
|
||||
// keptOf is the keep of a build, or nil when nothing may be kept: only a trunk build (publish) of a module whose
|
||||
// manifest at the built commit claims oci-artifact-registry, named by its fingerprint.
|
||||
func keptOf(publish bool, manifest catalogue.Manifest, fp string) *keptFor {
|
||||
if !publish || fp == "" || !manifest.ClaimsSeat(ociseat.Seat) {
|
||||
return nil
|
||||
}
|
||||
return &keptFor{Module: manifest.Module, Fp: fp}
|
||||
}
|
||||
|
||||
// KeptBuild is one kept build: the object's description names its module, fingerprint and the node that kept it.
|
||||
type KeptBuild struct {
|
||||
Module string `json:"module"`
|
||||
Fp string `json:"fp"`
|
||||
Node string `json:"node"`
|
||||
Digest string `json:"digest"`
|
||||
Bytes uint64 `json:"bytes"`
|
||||
LastUsed time.Time `json:"last-used"`
|
||||
}
|
||||
|
||||
// keepBuild keeps the archive of digest ("sha256:<hex>") for k on this node; the object is named by the hex digest.
|
||||
func (m *Module) keepBuild(k keptFor, digest string, archive []byte) error {
|
||||
if m.KeptBuilds == nil {
|
||||
return fmt.Errorf("this module keeps no builds (no %s store)", KeptBuildsStore)
|
||||
}
|
||||
hex := strings.TrimPrefix(digest, "sha256:")
|
||||
desc, _ := json.Marshal(KeptBuild{Module: k.Module, Fp: k.Fp, Node: m.Node})
|
||||
if _, err := m.KeptBuilds.Put(hex, archive, string(desc)); err != nil {
|
||||
return fmt.Errorf("keeping the build of %s on %s: %w", k.Module, KeptBuildsStore, err)
|
||||
}
|
||||
if m.Say != nil {
|
||||
m.Say("kept the build %s of %s (%s) on this node: the seat %s did not store it", hex, k.Module, k.Fp, ociseat.Seat)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// keptBuilds are the builds this node keeps, by the store's objects that name it.
|
||||
func (m *Module) keptBuilds() ([]KeptBuild, error) {
|
||||
if m.KeptBuilds == nil {
|
||||
return []KeptBuild{}, nil
|
||||
}
|
||||
infos, err := m.KeptBuilds.List()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := []KeptBuild{}
|
||||
for _, i := range infos {
|
||||
var k KeptBuild
|
||||
if i.Deleted || json.Unmarshal([]byte(i.Description), &k) != nil || k.Node != m.Node {
|
||||
continue
|
||||
}
|
||||
k.Digest, k.Bytes, k.LastUsed = i.Name, i.Size, i.Modified.UTC()
|
||||
out = append(out, k)
|
||||
}
|
||||
slices.SortFunc(out, func(a, b KeptBuild) int { return strings.Compare(a.Digest, b.Digest) })
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// dropKeptBuild removes the kept build of the hex digest this node keeps; one it does not keep is no error.
|
||||
func (m *Module) dropKeptBuild(hex string) error {
|
||||
kept, err := m.keptBuilds()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !slices.ContainsFunc(kept, func(k KeptBuild) bool { return k.Digest == hex }) {
|
||||
return nil
|
||||
}
|
||||
return m.KeptBuilds.Delete(hex)
|
||||
}
|
||||
|
||||
// sweepKeptBuilds drops the kept builds older than bound before now: the want of one waits, and the node-engine says it.
|
||||
func (m *Module) sweepKeptBuilds(now time.Time, bound time.Duration) {
|
||||
kept, err := m.keptBuilds()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
for _, k := range kept {
|
||||
if now.Sub(k.LastUsed) > bound && m.dropKeptBuild(k.Digest) == nil && m.Say != nil {
|
||||
m.Say("removed the kept build %s of %s, kept for %s", k.Digest, k.Module, bound)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// claimsStore asks the seat mesh-catalog whether module's manifest claims oci-artifact-registry.
|
||||
func (m *Module) claimsStore(module string) (bool, error) {
|
||||
ask := m.AskInput
|
||||
if ask == nil {
|
||||
ask = stdio.Ask
|
||||
}
|
||||
raw, err := ask(CatalogModuleVerb, map[string]any{"module": module})
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
var a struct {
|
||||
Module struct {
|
||||
Manifest json.RawMessage `json:"manifest"`
|
||||
} `json:"module"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &a); err != nil {
|
||||
return false, fmt.Errorf("%s answered what is no module: %w", CatalogModuleVerb, err)
|
||||
}
|
||||
if len(a.Module.Manifest) == 0 {
|
||||
return false, nil
|
||||
}
|
||||
man, err := catalogue.Decode(a.Module.Manifest)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("the manifest of %s: %w", module, err)
|
||||
}
|
||||
return man.ClaimsSeat(ociseat.Seat), nil
|
||||
}
|
||||
|
||||
// CatalogModuleVerb is the seat mesh-catalog's verb that answers one module's manifest, as the manifest invokes it.
|
||||
const CatalogModuleVerb = "seat:mesh-catalog.module"
|
||||
|
||||
// keptArtifact is the verb kept-artifact's answer for in.
|
||||
func (m *Module) keptArtifact(in seat.KeptArtifactInput) seat.KeptArtifactAnswer {
|
||||
a := seat.KeptArtifactAnswer{Module: in.Module, Fp: in.Fp}
|
||||
kept, err := m.keptBuilds()
|
||||
if err != nil {
|
||||
a.Error = "reading the kept builds: " + err.Error()
|
||||
return a
|
||||
}
|
||||
var other *KeptBuild
|
||||
for i, k := range kept {
|
||||
if k.Module != in.Module || k.Fp != in.Fp {
|
||||
continue
|
||||
}
|
||||
if k.Digest == in.Digest {
|
||||
a.Digest, a.Size, a.Object = k.Digest, int64(k.Bytes), k.Digest
|
||||
if m.Self == "" {
|
||||
a.Error = "this module runs with no name of its own (MESH_MODULE), so its store's name on the bus is unknown"
|
||||
return a
|
||||
}
|
||||
a.Store = catalogue.ObjectsBucket(m.Self, KeptBuildsStore)
|
||||
return a
|
||||
}
|
||||
other = &kept[i]
|
||||
}
|
||||
claims, err := m.claimsStore(in.Module)
|
||||
switch {
|
||||
case err != nil:
|
||||
a.Error = "asking whether " + in.Module + " claims " + ociseat.Seat + ": " + err.Error()
|
||||
case !claims:
|
||||
a.Refused = seat.KeptArtifactAnswerRefusedClaimsNoOciArtifactRegistry
|
||||
a.Words = fmt.Sprintf("%s claims no %s: a build node keeps builds for that seat's holder alone", in.Module, ociseat.Seat)
|
||||
case other != nil:
|
||||
a.Refused, a.Digest = seat.KeptArtifactAnswerRefusedDigestDiffers, other.Digest
|
||||
a.Words = fmt.Sprintf("this node keeps the build of %s at %s under the digest %s, not %s", in.Module, in.Fp, other.Digest, in.Digest)
|
||||
default:
|
||||
a.Refused = seat.KeptArtifactAnswerRefusedNotKept
|
||||
a.Words = fmt.Sprintf("this node keeps no build of %s at %s (never kept, or swept)", in.Module, in.Fp)
|
||||
}
|
||||
return a
|
||||
}
|
||||
|
||||
// keptArtifactVerb serves `<seat>.kept-artifact`.
|
||||
func (m *Module) keptArtifactVerb() stdio.Tool {
|
||||
return stdio.Tool{Name: Seat + "." + seat.VerbKeptArtifact,
|
||||
Description: "Where the build this node kept for the artifact store's holder is: the new build of the holder of " +
|
||||
ociseat.Seat + " that the seat refused or did not store, as an object of this module's store of kept builds, by its " +
|
||||
"digest. Refuses a module that claims no " + ociseat.Seat + ", a build not kept here (never kept, or swept) and one " +
|
||||
"kept under another digest, naming the module.",
|
||||
Input: map[string]any{"type": "object", "required": []string{"module", "fp", "digest"}, "properties": map[string]any{
|
||||
"module": map[string]any{"type": "string", "description": "the module, the holder of " + ociseat.Seat},
|
||||
"fp": map[string]any{"type": "string", "description": "its build's fingerprint"},
|
||||
"digest": map[string]any{"type": "string", "description": "the artifact's sha256, 64 lowercase hex digits"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
str := func(k string) string { s, _ := args[k].(string); return s }
|
||||
return m.keptArtifact(seat.KeptArtifactInput{Module: str("module"), Fp: str("fp"), Digest: str("digest")}), nil
|
||||
}}
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildgo"
|
||||
sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscalls"
|
||||
)
|
||||
|
||||
// memObjects is an object store in memory, as the tool runner keeps one for the module.
|
||||
type memObjects struct {
|
||||
objects map[string]buscalls.ObjectInfo
|
||||
}
|
||||
|
||||
func (o *memObjects) Put(name string, data []byte, description string) (buscalls.ObjectInfo, error) {
|
||||
i := buscalls.ObjectInfo{Name: name, Size: uint64(len(data)), Description: description, Modified: time.Now()}
|
||||
o.objects[name] = i
|
||||
return i, nil
|
||||
}
|
||||
func (o *memObjects) List() ([]buscalls.ObjectInfo, error) {
|
||||
out := []buscalls.ObjectInfo{}
|
||||
for _, i := range o.objects {
|
||||
out = append(out, i)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
func (o *memObjects) Delete(name string) error { delete(o.objects, name); return nil }
|
||||
|
||||
// Design 47 *The artifact store's holder takes its own new build even when its seat is the one broken* (test 60, W12.5):
|
||||
// a trunk build of a module claiming oci-artifact-registry that the seat refuses is kept on this node and answered kept;
|
||||
// any other build the seat refuses fails not stored; kept-artifact answers the store and object of a kept build, refuses
|
||||
// a module that claims no oci-artifact-registry, one not kept here and one kept under another digest, and a drop or the
|
||||
// sweep removes it.
|
||||
func TestTheStoreHoldersRefusedBuildIsKeptAndServedByKeptArtifact(t *testing.T) {
|
||||
holder := `{"module":"demo","version":"1","claims":[{"name":"oci-artifact-registry","scope":"mesh"}],"build":{"artifacts":[` + goBundle + `]}}`
|
||||
refuse := func(key string, _ any) (json.RawMessage, error) {
|
||||
return nil, errors.New("refused: the store did not take it")
|
||||
}
|
||||
for _, c := range []struct {
|
||||
name, manifest string
|
||||
publish bool
|
||||
kept bool
|
||||
}{
|
||||
{"the store holder's trunk build", holder, true, true},
|
||||
{"another module's trunk build", manifestWith(goBundle), true, false},
|
||||
{"the store holder's candidate build", holder, false, false},
|
||||
} {
|
||||
m, states, _ := buildModule(t, c.manifest, "the program")
|
||||
objects := &memObjects{objects: map[string]buscalls.ObjectInfo{}}
|
||||
m.KeptBuilds, m.Self, m.Node = objects, "mesh-build-go", "ace"
|
||||
m.Call = refuse
|
||||
args := map[string]any{"workspace": "ws1", "module": "modules/demo", "artifact": "tools", "fp": "src2:" + strings.Repeat("a", 64)}
|
||||
if c.publish {
|
||||
args["publish"] = true
|
||||
}
|
||||
for _, tool := range m.Tools(m.StepTools()) {
|
||||
if tool.Name == "build_go_build" {
|
||||
if _, err := tool.Run(args); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
st := states.waitEnd(t)
|
||||
var res BuildResult
|
||||
_ = json.Unmarshal(st.Result, &res)
|
||||
if got := st.State == sdkstep.StepPassed && res.Kept != "" && len(objects.objects) == 1; got != c.kept {
|
||||
t.Fatalf("%s: kept %v (step %s %q, kept %q, %d objects), want %v", c.name, got, st.State, st.Reason, res.Kept, len(objects.objects), c.kept)
|
||||
}
|
||||
if !c.kept {
|
||||
continue
|
||||
}
|
||||
hex := strings.TrimPrefix(res.Kept, "sha256:")
|
||||
m.AskInput = func(key string, body any) (json.RawMessage, error) {
|
||||
if key != CatalogModuleVerb {
|
||||
return nil, errors.New("asked " + key)
|
||||
}
|
||||
mod, _ := body.(map[string]any)["module"].(string)
|
||||
man := `{"module":"` + mod + `"}`
|
||||
if mod == "demo" || mod == "lost" {
|
||||
man = `{"module":"` + mod + `","claims":[{"name":"oci-artifact-registry","scope":"mesh"}]}`
|
||||
}
|
||||
return json.RawMessage(`{"module":{"module":"` + mod + `","manifest":` + man + `}}`), nil
|
||||
}
|
||||
fp := args["fp"].(string)
|
||||
for _, a := range []struct {
|
||||
name string
|
||||
in seat.KeptArtifactInput
|
||||
refused, store string
|
||||
}{
|
||||
{"the kept build", seat.KeptArtifactInput{Module: "demo", Fp: fp, Digest: hex}, "", "mesh-build-go_kept-builds"},
|
||||
{"another digest", seat.KeptArtifactInput{Module: "demo", Fp: fp, Digest: strings.Repeat("b", 64)}, seat.KeptArtifactAnswerRefusedDigestDiffers, ""},
|
||||
{"a module claiming no store", seat.KeptArtifactInput{Module: "other", Fp: fp, Digest: hex}, seat.KeptArtifactAnswerRefusedClaimsNoOciArtifactRegistry, ""},
|
||||
{"a holder not kept here", seat.KeptArtifactInput{Module: "lost", Fp: fp, Digest: hex}, seat.KeptArtifactAnswerRefusedNotKept, ""},
|
||||
} {
|
||||
got := m.keptArtifact(a.in)
|
||||
if got.Refused != a.refused || got.Store != a.store || (a.refused == "" && (got.Object != hex || got.Size == 0)) ||
|
||||
(a.refused != "" && !strings.Contains(got.Words, a.in.Module)) {
|
||||
t.Errorf("%s: %+v", a.name, got)
|
||||
}
|
||||
}
|
||||
// Another node's build in the shared store is not this node's to answer.
|
||||
m.Node = "g14"
|
||||
if got := m.keptArtifact(seat.KeptArtifactInput{Module: "demo", Fp: fp, Digest: hex}); got.Refused != seat.KeptArtifactAnswerRefusedNotKept {
|
||||
t.Errorf("another node's kept build answered %+v", got)
|
||||
}
|
||||
m.Node = "ace"
|
||||
m.sweepKeptBuilds(time.Now(), time.Hour)
|
||||
if len(objects.objects) != 1 {
|
||||
t.Errorf("a build kept within the bound was swept")
|
||||
}
|
||||
if err := m.dropKeptBuild(hex); err != nil || len(objects.objects) != 0 {
|
||||
t.Errorf("drop: %v, %d objects left", err, len(objects.objects))
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -16,6 +16,7 @@ import (
|
||||
seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildgo"
|
||||
sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/busstate"
|
||||
)
|
||||
|
||||
// The directories the module's manifest places outside its home (ADR 0466 §1): the workspaces, and the Go module
|
||||
@@ -63,6 +64,8 @@ func main() {
|
||||
m.Settings = m.Home + "/settings.json"
|
||||
m.ReadManifest = askManifest
|
||||
m.AskInput = stdio.Ask
|
||||
m.KeptBuilds = busstate.ObjectsOf(KeptBuildsStore)
|
||||
m.Self = strings.TrimSpace(os.Getenv("MESH_MODULE"))
|
||||
m.Probe = ProbeGo(m.Home, m.Cache)
|
||||
m.Bubblewrap = ProbeBubblewrap
|
||||
// Outside a launch by the runtime — a build, a check — there is no node and no state to write.
|
||||
|
||||
@@ -112,6 +112,10 @@ type Module struct {
|
||||
// PlaceHead, when set, lays a run's head at a directory instead of fetching it (a test).
|
||||
PlaceHead func(ctx context.Context, h sdkstep.RunHead, dir string) error
|
||||
GoModCache string
|
||||
// KeptBuilds is the object store of the builds kept for the artifact store's holder (keptbuilds.go); nil keeps none.
|
||||
// Self is this module's name as the runtime launched it (MESH_MODULE), the first part of the store's name on the bus.
|
||||
KeptBuilds KeptObjects
|
||||
Self string
|
||||
|
||||
spacesOnce sync.Once
|
||||
workspaces *buildstep.Workspaces
|
||||
|
||||
@@ -21,7 +21,7 @@
|
||||
{
|
||||
"name": "node-build-go",
|
||||
"scope": "node",
|
||||
"serves": ["current", "drop", "pause", "resume", "steps"]
|
||||
"serves": ["current", "drop", "pause", "resume", "steps", "kept-artifact"]
|
||||
}
|
||||
],
|
||||
"consumes": [
|
||||
@@ -36,7 +36,8 @@
|
||||
"seat:mesh-delivery-checker.input",
|
||||
"seat:mesh-catalog.input",
|
||||
"seat:mesh-wants.input",
|
||||
"seat:mesh-module-assigner.assignments"
|
||||
"seat:mesh-module-assigner.assignments",
|
||||
"seat:mesh-catalog.module"
|
||||
],
|
||||
"config": {
|
||||
"capacity": {
|
||||
@@ -88,6 +89,7 @@
|
||||
],
|
||||
"requires": [
|
||||
{"name": "detected-steps", "storage": "key-value", "template": "state", "options": {"history": 1}},
|
||||
{"name": "kept-builds", "storage": "object-store", "template": "objects", "options": {"max-mib": 1024}},
|
||||
"forge-account",
|
||||
"go-toolchain",
|
||||
"go-package-registry",
|
||||
|
||||
Reference in New Issue
Block a user