Merge feat/47-G-fix11-c1: the build modules say steprun, the catalog's buildstep stays gone
Builds: design 47 G §464, §680 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h
This commit is contained in:
@@ -37,7 +37,7 @@ func (m *Module) askSteps(steps []StepTool) steprun.AskSteps {
|
||||
if err != nil {
|
||||
return sdkstep.ToolchainUsed{}, err
|
||||
}
|
||||
return sdkstep.ToolchainUsed{Version: installed, Recipe: m.recipe().Version()}, nil
|
||||
return sdkstep.ToolchainUsed{Version: installed, Recipe: m.recipe().Version(), Digest: m.programDigest(installed)}, nil
|
||||
},
|
||||
Release: func(ws string) error { return m.spaces().Release(ws) },
|
||||
Logf: m.Say,
|
||||
|
||||
@@ -137,6 +137,14 @@ func (m *Module) build(ctx context.Context, args map[string]any) (any, error) {
|
||||
if err := copyPackages(filepath.Join(m.toolchainDir(), "node_modules"), filepath.Join(stage, "node_modules")); err != nil {
|
||||
return nil, fmt.Errorf("providing the toolchain's packages to the build: %w", err)
|
||||
}
|
||||
// A program reads its own version from its package.json `version`, which is the version the ask carries (§458); the
|
||||
// staged copy is the build's own, so nothing is restored. A build with no version, or a module with no package.json,
|
||||
// keeps what it carries.
|
||||
if _, err := os.Stat(filepath.Join(dir, "package.json")); err == nil && version != "" {
|
||||
if _, err := setPackageVersion(dir, version); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
out, final := Out(a.Name), Out(a.Name)+".bundled"
|
||||
if err := os.MkdirAll(filepath.Join(dir, out), 0o755); err != nil {
|
||||
return nil, err
|
||||
|
||||
@@ -328,3 +328,24 @@ func TestTheBundleBuildGetsTheVersionTheAskCarriesAsMeshVersion(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 G §458 (BN8): a Node program reads its version from its package.json `version`, which is the version the
|
||||
// ask carries; a build with no version keeps what the package.json carries.
|
||||
func TestTheBundleCarriesTheAsksVersionInItsPackageJSON(t *testing.T) {
|
||||
for _, c := range []struct{ version, want string }{{"1.4.3", "1.4.3"}, {"", "0.0.0"}} {
|
||||
m, f, work := builtMachine(t, "")
|
||||
workspaceWith(t, work, map[string]string{"modules/m/package.json": `{"name":"m","version":"0.0.0"}`})
|
||||
args := buildArgs()
|
||||
if c.version != "" {
|
||||
args["version"] = c.version
|
||||
}
|
||||
if _, err := m.build(context.Background(), args); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = f
|
||||
raw, err := os.ReadFile(filepath.Join(work, "w1", "stage-tools", "modules", "module", "package.json"))
|
||||
if err != nil || !strings.Contains(string(raw), `"version": "`+c.want+`"`) && !strings.Contains(string(raw), `"version":"`+c.want+`"`) {
|
||||
t.Errorf("ask version %q: the staged package.json is %q (%v), want version %s", c.version, raw, err, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
package main
|
||||
|
||||
// **A group run's package members, laid from their builds** (design 47 G §465, §373): a fetch naming `builds` reads each
|
||||
// member's build (the npm package's packed tarball, package.go's packed) from the artifact store by its digest, checks
|
||||
// it and places it as the local package it names in each TypeScript module's node_modules: nothing is fetched by a
|
||||
// version, and none is reserved. A member that does not match its digest, or a store that does not answer, fails the
|
||||
// step naming the member.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/steprun"
|
||||
sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
|
||||
)
|
||||
|
||||
// storedBuild is one member's build, read and checked.
|
||||
type storedBuild struct {
|
||||
member sdkstep.FetchBuild
|
||||
tgz []byte
|
||||
}
|
||||
|
||||
// readBuilds reads every member of builds from the artifact store.
|
||||
func (m *Module) readBuilds(ctx context.Context, builds []sdkstep.FetchBuild) ([]storedBuild, error) {
|
||||
if len(builds) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
base, err := steprun.StoreURL(m.ArtifactStoreFile)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("a group run's members are read from the artifact store: %w", err)
|
||||
}
|
||||
var out []storedBuild
|
||||
for _, b := range builds {
|
||||
data, err := steprun.ReadStoredBuild(ctx, m.httpClient(), base, b)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, storedBuild{b, data})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// names says whether a member of the builds is the package pkg.
|
||||
func namesPackage(builds []storedBuild, pkg string) bool {
|
||||
for _, b := range builds {
|
||||
if b.member.Package == pkg {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// layBuilds places each member in dir's node_modules as the package it names, in place of any other.
|
||||
func (m *Module) layBuilds(dir string, builds []storedBuild) error {
|
||||
for _, b := range builds {
|
||||
into := filepath.Join(dir, "node_modules", filepath.FromSlash(b.member.Package))
|
||||
if err := os.RemoveAll(into); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := unpackPackage(b.tgz, into); err != nil {
|
||||
return fmt.Errorf("the build of %s (%s %s at %s): %w", b.member.Package, b.member.Module, b.member.Artifact, b.member.Digest, err)
|
||||
}
|
||||
if m.Say != nil {
|
||||
m.Say("laid %s from its build %s (%s %s)", b.member.Package, b.member.Digest, b.member.Module, b.member.Artifact)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
|
||||
)
|
||||
|
||||
// Design 47 G §465, §373 (BN7): a fetch naming builds reads each member from the artifact store by its digest, checks the
|
||||
// bytes and lays it in node_modules as the package named; a mismatch, or a store that does not answer, fails naming it.
|
||||
func TestAFetchLaysAGroupMembersBuildFromTheArtifactStore(t *testing.T) {
|
||||
tgz := tgzOf(t, "@novox/lib", "0.0.0")
|
||||
digest := digestOf(tgz)
|
||||
store := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/v2/lib/pkg/blobs/"+digest {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
w.Write(tgz)
|
||||
}))
|
||||
defer store.Close()
|
||||
m := &Module{ArtifactStoreFile: filepath.Join(t.TempDir(), "artifact-store.json")}
|
||||
os.WriteFile(m.ArtifactStoreFile, []byte(`{"at":"`+strings.TrimPrefix(store.URL, "http://")+`"}`), 0o600)
|
||||
member := sdkstep.FetchBuild{Package: "@novox/lib", Module: "lib", Artifact: "pkg", Digest: digest}
|
||||
|
||||
got, err := m.readBuilds(context.Background(), []sdkstep.FetchBuild{member})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
dir := t.TempDir()
|
||||
if err := m.layBuilds(dir, got); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if b, err := os.ReadFile(filepath.Join(dir, "node_modules", "@novox", "lib", "dist", "index.js")); err != nil || !strings.Contains(string(b), "0.0.0") {
|
||||
t.Fatalf("the member was not laid: %q, %v", b, err)
|
||||
}
|
||||
if !namesPackage(got, "@novox/lib") || namesPackage(got, "@novox/other") {
|
||||
t.Error("namesPackage is wrong")
|
||||
}
|
||||
bad := member
|
||||
bad.Digest = "sha256:" + strings.Repeat("0", 64)
|
||||
for _, c := range []sdkstep.FetchBuild{bad, {Package: "@novox/lib", Module: "lib", Artifact: "missing", Digest: digest}} {
|
||||
if _, err := m.readBuilds(context.Background(), []sdkstep.FetchBuild{c}); err == nil || !strings.Contains(err.Error(), "@novox/lib") {
|
||||
t.Errorf("member %+v: error %v", c, err)
|
||||
}
|
||||
}
|
||||
store.Close()
|
||||
if _, err := m.readBuilds(context.Background(), []sdkstep.FetchBuild{member}); err == nil || !strings.Contains(err.Error(), "@novox/lib") {
|
||||
t.Errorf("a store that does not answer: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -5,51 +5,69 @@ import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Issues 1483 and 1488 (for the indexer, issue 1444): the guard emits toolchain.reported after each write of the
|
||||
// toolchains key, with the node and the language.
|
||||
func TestTheGuardEmitsToolchainReportedAfterEachWrite(t *testing.T) {
|
||||
m := moduleWith(newStates(), "v26.10.0")
|
||||
// Design 47 G §492, §472, §466 (BG5, BG6): the guard says toolchain-reported on the Node build seat's subject of its node,
|
||||
// with the version, the recipe and the digest of the node program, at the first write and again only when one of them
|
||||
// changed; the toolchains key and a build's toolchain carry the digest.
|
||||
func TestTheGuardSaysToolchainReportedOnAChangeWithTheProgramDigest(t *testing.T) {
|
||||
states := newStates()
|
||||
m := moduleWith(states, "v26.10.0")
|
||||
prog := filepath.Join(t.TempDir(), "node")
|
||||
os.WriteFile(prog, []byte("node program"), 0o755)
|
||||
m.ToolchainProgram = func() (string, error) { return prog, nil }
|
||||
type event struct {
|
||||
typ string
|
||||
body map[string]string
|
||||
subject string
|
||||
body map[string]string
|
||||
id string
|
||||
}
|
||||
var got []event
|
||||
m.Emit = func(typ string, body any) error {
|
||||
m.SayToolchain = func(subject string, body any, id string) error {
|
||||
raw, _ := json.Marshal(body)
|
||||
var b map[string]string
|
||||
json.Unmarshal(raw, &b)
|
||||
got = append(got, event{typ, b})
|
||||
got = append(got, event{subject, b, id})
|
||||
return nil
|
||||
}
|
||||
if err := m.Check(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != 1 || got[0].typ != "toolchain.reported" || got[0].body["node"] != "ace" || got[0].body["language"] != "typescript" || len(got[0].body) != 2 {
|
||||
t.Errorf("events = %+v, want one toolchain.reported for ace, typescript", got)
|
||||
if len(got) != 1 || got[0].subject != "mesh.seat.node-build-node.event.toolchain-reported.ace" || got[0].body["node"] != "ace" || got[0].body["language"] != "typescript" ||
|
||||
got[0].body["version"] != "v26.10.0" || got[0].body["recipe"] != m.Recipe.Version() || len(got[0].body["digest"]) != len("sha256:")+64 {
|
||||
t.Fatalf("events = %+v, want one toolchain-reported for ace, go with the program's digest", got)
|
||||
}
|
||||
if tc, _ := states.toolchain("ace.typescript"); tc.Digest != got[0].body["digest"] {
|
||||
t.Errorf("the toolchains key's digest is %q, the event's %q", tc.Digest, got[0].body["digest"])
|
||||
}
|
||||
m.Check(context.Background())
|
||||
if len(got) != 2 {
|
||||
t.Errorf("a second guard pass emitted %d events in all, want 2", len(got))
|
||||
if len(got) != 1 {
|
||||
t.Errorf("a second pass that changed nothing said %d events in all, want 1", len(got))
|
||||
}
|
||||
// A build's toolchain names the same digest (§466).
|
||||
if used, err := m.askSteps(nil).Toolchain(context.Background()); err != nil || used.Digest != got[0].body["digest"] || used.Version != "v26.10.0" {
|
||||
t.Errorf("a build's toolchain = %+v (%v), want the digest %s", used, err, got[0].body["digest"])
|
||||
}
|
||||
os.WriteFile(prog, []byte("another node program"), 0o755)
|
||||
m.Probe = func(context.Context) (string, error) { return "v26.11.0", nil }
|
||||
m.Check(context.Background())
|
||||
if len(got) != 2 || got[1].body["version"] != "v26.11.0" || got[1].body["digest"] == got[0].body["digest"] {
|
||||
t.Errorf("a new version said %+v, want a second event with another digest", got)
|
||||
}
|
||||
}
|
||||
|
||||
// The event is carried by the module's stream, not named in emits (ADR 0363: one place); the validator checks that.
|
||||
func TestTheManifestDeclaresTheToolchainsStream(t *testing.T) {
|
||||
// The seat's events are its definition's: the module declares no stream of its own for the toolchain (§492).
|
||||
func TestTheManifestDeclaresNoToolchainStream(t *testing.T) {
|
||||
raw, err := os.ReadFile(filepath.Join("..", "..", "module.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var m struct {
|
||||
Streams []struct {
|
||||
Name string `json:"name"`
|
||||
Events []string `json:"events"`
|
||||
Name string `json:"name"`
|
||||
} `json:"streams"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &m); err != nil || len(m.Streams) != 1 || m.Streams[0].Name != "toolchains-node" || !slices.Equal(m.Streams[0].Events, []string{"toolchain.**"}) {
|
||||
if err := json.Unmarshal(raw, &m); err != nil || len(m.Streams) != 0 {
|
||||
t.Errorf("streams = %+v (%v)", m.Streams, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -102,6 +102,11 @@ func (m *Module) fetchTool() StepTool {
|
||||
"package": map[string]any{"type": "object", "description": "optional: the group's tagged version of the SDK, {name, version}, placed from the registry in each TypeScript member whatever its lock names (ADR 0607 §3); or the group's package member, {repository, commit, ref} (ADR 0500 §5), packed here, publishing nothing, and served to each TypeScript member whose lock names its version, in place of the registry's",
|
||||
"properties": map[string]any{"name": map[string]any{"type": "string"}, "version": map[string]any{"type": "string"},
|
||||
"repository": map[string]any{"type": "string"}, "commit": map[string]any{"type": "string"}, "ref": map[string]any{"type": "string"}}},
|
||||
"builds": map[string]any{"type": "array", "description": "optional: a group run's package members, each by its build's digest, read from the artifact store and laid as the package it names in each module's node_modules (design 47 G §465)",
|
||||
"items": map[string]any{"type": "object", "properties": map[string]any{
|
||||
"package": map[string]any{"type": "string"}, "module": map[string]any{"type": "string"},
|
||||
"artifact": map[string]any{"type": "string"}, "digest": map[string]any{"type": "string"},
|
||||
}, "required": []string{"package", "module", "artifact", "digest"}}},
|
||||
"base": map[string]any{"type": "string", "description": "optional trunk commit, 40 hex digits: the head is merged onto it as the forge's merge would, and the answer carries the merge commit, the files in conflict (an answer, the workspace is then left at the trunk commit), the changed paths and the tree of each module directory"},
|
||||
},
|
||||
"required": []string{"repository", "commit"},
|
||||
@@ -124,6 +129,10 @@ func (m *Module) fetch(ctx context.Context, args map[string]any) (res any, err e
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
buildsAsked, err := steprun.BuildsArg(args["builds"])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
m.CleanWorkspaces(m.now())
|
||||
id := sdkstep.NewID()
|
||||
done := m.spaces().Hold(id)
|
||||
@@ -135,10 +144,10 @@ func (m *Module) fetch(ctx context.Context, args map[string]any) (res any, err e
|
||||
}
|
||||
}
|
||||
}()
|
||||
return m.fetchInto(ctx, req, group, id)
|
||||
return m.fetchInto(ctx, req, group, id, buildsAsked)
|
||||
}
|
||||
|
||||
func (m *Module) fetchInto(ctx context.Context, req sdkstep.FetchRequest, group *groupPackage, id string) (any, error) {
|
||||
func (m *Module) fetchInto(ctx context.Context, req sdkstep.FetchRequest, group *groupPackage, id string, buildsAsked []sdkstep.FetchBuild) (any, error) {
|
||||
repository, commit, ref, base := req.Repository, req.Commit, req.Ref, req.Base
|
||||
var err error
|
||||
root := filepath.Join(m.Work, id)
|
||||
@@ -181,6 +190,11 @@ func (m *Module) fetchInto(ctx context.Context, req sdkstep.FetchRequest, group
|
||||
}
|
||||
served, res.Group = &p, p.Name+"@"+p.Version+" "+p.Integrity
|
||||
}
|
||||
// A group run's package members, each read from the artifact store by its build's digest (builds.go, §465).
|
||||
members, err := m.readBuilds(ctx, buildsAsked)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
dirs, packages, err := m.typeScriptModules(ctx, tree)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -193,6 +207,9 @@ func (m *Module) fetchInto(ctx context.Context, req sdkstep.FetchRequest, group
|
||||
if err := m.installDev(ctx, root, dir, d); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := m.layBuilds(dir, members); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res.Installed = append(res.Installed, d)
|
||||
continue
|
||||
}
|
||||
@@ -201,8 +218,13 @@ func (m *Module) fetchInto(ctx context.Context, req sdkstep.FetchRequest, group
|
||||
return nil, err
|
||||
}
|
||||
if refused == "" {
|
||||
// The latest published SDK, or the group's (ADR 0573 §2, ADR 0607 §3).
|
||||
if err := m.placeSDK(ctx, dir, d, served, group); err != nil {
|
||||
// The latest published SDK, or the group's (ADR 0573 §2, ADR 0607 §3); the SDK as a member's build is laid from it.
|
||||
if !namesPackage(members, sdkPackage) {
|
||||
if err := m.placeSDK(ctx, dir, d, served, group); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if err := m.layBuilds(dir, members); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if res.SDK == nil {
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
)
|
||||
|
||||
// handActRecorder writes an act by hand in the hand-act log of the seat mesh-hand-acts (design 47 G §480); a log that
|
||||
// does not take it is an error, never a quiet success.
|
||||
func handActRecorder(what, why string) error {
|
||||
_, err := meshhandactsclient.Recorder{Ask: stdio.Ask}.Record(meshhandactsclient.Request{What: what, Why: why, Cause: "build-steps-drop", Verb: "steps drop"})
|
||||
return err
|
||||
}
|
||||
@@ -45,11 +45,13 @@ func main() {
|
||||
Probe: ProbeNode,
|
||||
Toolchain: &Toolchain{Paths: paths, Run: dforgekey.ExecRunnerClean(0), Registry: registry},
|
||||
Toolchains: stdio.State(seat.StateToolchains).Put,
|
||||
Emit: func(t string, b any) error { return stdio.Emit(t, b) },
|
||||
SayToolchain: stdio.SeatPublish,
|
||||
Steps: stdio.State(seat.StateSteps).Put,
|
||||
ReadSteps: readOf(stdio.State(seat.StateSteps)),
|
||||
Detected: stdio.State(seat.StateDetectedSteps).Put,
|
||||
ReadDetected: readOf(stdio.State(seat.StateDetectedSteps)).Get,
|
||||
ForgetDetected: stdio.State(seat.StateDetectedSteps).Delete,
|
||||
RecordAct: handActRecorder,
|
||||
Pauses: stdio.State(sdkstep.PausesState).Put,
|
||||
ReadPauses: readOf(stdio.State(sdkstep.PausesState)),
|
||||
ForgetPause: stdio.State(sdkstep.PausesState).Delete,
|
||||
@@ -63,6 +65,7 @@ func main() {
|
||||
}
|
||||
m.Floor = floorOf(m.Node)
|
||||
m.Settings = filepath.Join(home, "settings.json")
|
||||
m.ArtifactStoreFile = filepath.Join(home, "artifact-store.json")
|
||||
// Outside a launch by the runtime — a build, a check — there is no node and no state to write.
|
||||
if m.Node != "" {
|
||||
go m.Start(ctx)
|
||||
|
||||
@@ -238,8 +238,9 @@ func TestTheModuleInvokesTheRegistrySeatsAndHoldsNoWriteCredential(t *testing.T)
|
||||
}
|
||||
}
|
||||
}
|
||||
if slices.Contains(m.Requires, "artifact-store") {
|
||||
t.Errorf("requires %v still name artifact-store", m.Requires)
|
||||
// It requires the provision to read a group run's package members' builds (design 47 G §465), never to write.
|
||||
if !slices.Contains(m.Requires, "artifact-store") {
|
||||
t.Errorf("requires %v lack artifact-store, which a group run's members are read from", m.Requires)
|
||||
}
|
||||
for _, r := range m.Resources {
|
||||
if r["id"] == "registry" {
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -51,6 +52,12 @@ type Module struct {
|
||||
// Detected and ReadDetected are the Put and the read of the state detected-steps (design 47 G §326, §414).
|
||||
Detected steprun.Put
|
||||
ReadDetected func(key string) (json.RawMessage, error)
|
||||
// ArtifactStoreFile is the binding of the provision artifact-store, read for a group run's package members (§465).
|
||||
ArtifactStoreFile string
|
||||
// ForgetDetected deletes one key of detected-steps; RecordAct writes an act in the hand-act log of the seat
|
||||
// mesh-hand-acts (§463, §480): a drop writes it first.
|
||||
ForgetDetected func(key string) error
|
||||
RecordAct func(what, why string) error
|
||||
// Pauses is the Put of the pauses state, which keeps the node's pause key without an age; ReadPauses reads it back.
|
||||
Pauses steprun.Put
|
||||
ReadPauses StateRead
|
||||
@@ -58,9 +65,15 @@ type Module struct {
|
||||
ForgetPause func(key string) error
|
||||
// WatchPauses follows the drops other holders write in the shared pauses state (stdio.State(...).Watch).
|
||||
WatchPauses func(key string, handle func(stdio.StateChange) error) error
|
||||
// Emit tells the bus an event (stdio.Emit); nil emits nothing. The guard emits toolchain.reported after each write of
|
||||
// the toolchains key, so the indexer, which holds no timer, learns that the bucket exists (issue 1444).
|
||||
Emit func(eventType string, body any) error
|
||||
// SayToolchain says an event on a seat's subject (stdio.SeatPublish; nil says none, a test's); ToolchainProgram finds the
|
||||
// node program whose digest the toolchain's is (exec.LookPath when nil).
|
||||
SayToolchain func(subject string, body any, id string) error
|
||||
ToolchainProgram func() (string, error)
|
||||
reporterOnce sync.Once
|
||||
reporter *toolchainReporter
|
||||
digestMu sync.Mutex
|
||||
digestFor string
|
||||
progDigest string
|
||||
// Ctx is the process's context, which a step runs under (the tool call's is cut at 30 s, the step's is not).
|
||||
Ctx context.Context
|
||||
Say func(format string, a ...any)
|
||||
@@ -260,20 +273,38 @@ func pathOrDefault() string {
|
||||
return defaultPath
|
||||
}
|
||||
|
||||
// ToolchainReported is the event the guard emits after it writes the toolchains key.
|
||||
const ToolchainReported = "toolchain.reported"
|
||||
|
||||
// reportingPut is the toolchains Put that then emits toolchain.reported with the node and the language. A write that
|
||||
// failed emits nothing; an emit that failed is said and does not refuse the step.
|
||||
// reportingPut is the toolchains Put that then says toolchain-reported on the Node build seat's subject of this node when
|
||||
// the toolchain changed (design 47 G §492): at the module's start and on each change of the key. A write that failed
|
||||
// says nothing; an event that failed is said and does not refuse the step.
|
||||
func (m *Module) reportingPut() steprun.Put {
|
||||
return func(key string, value any) (uint64, error) {
|
||||
rev, err := m.Toolchains(key, value)
|
||||
if err != nil || m.Emit == nil {
|
||||
return rev, err
|
||||
m.reporterOnce.Do(func() {
|
||||
m.reporter = &toolchainReporter{AskLanguage: sdkstep.AskNode, Node: m.Node, Publish: m.SayToolchain}
|
||||
})
|
||||
return putReporting(m.Toolchains, m.reporter, m.programDigest, m.Say)
|
||||
}
|
||||
|
||||
// programDigest is the digest of the node program this node's node-toolchain provides, the `node` found on the path: read
|
||||
// again only when the installed version changed (design 47 G §472; the lead's answer A-152).
|
||||
func (m *Module) programDigest(version string) string {
|
||||
m.digestMu.Lock()
|
||||
defer m.digestMu.Unlock()
|
||||
if m.digestFor == version && m.progDigest != "" {
|
||||
return m.progDigest
|
||||
}
|
||||
find := m.ToolchainProgram
|
||||
if find == nil {
|
||||
find = func() (string, error) { return exec.LookPath("node") }
|
||||
}
|
||||
path, err := find()
|
||||
if err == nil {
|
||||
var d string
|
||||
if d, err = programFileDigest(path); err == nil {
|
||||
m.digestFor, m.progDigest = version, d
|
||||
return d
|
||||
}
|
||||
if eerr := m.Emit(ToolchainReported, map[string]string{"node": m.Node, "language": "typescript"}); eerr != nil && m.Say != nil {
|
||||
m.Say("the event %s could not be emitted: %v", ToolchainReported, eerr)
|
||||
}
|
||||
return rev, nil
|
||||
}
|
||||
if m.Say != nil {
|
||||
m.Say("the digest of the node toolchain program could not be read: %v", err)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
@@ -38,6 +38,9 @@ type StepsAnswer struct {
|
||||
Fingerprint string `json:"fingerprint,omitempty"`
|
||||
Steps []DetectedStep `json:"steps"`
|
||||
Kept bool `json:"kept,omitempty"`
|
||||
// Dropped is the artifact whose key a drop deleted; Error why a drop deleted nothing, naming the hand-act log.
|
||||
Dropped string `json:"dropped,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// detectSteps finds the steps of the package directories under dir.
|
||||
@@ -77,26 +80,31 @@ func (m *Module) stepsTool() stdio.Tool {
|
||||
return stdio.Tool{Name: Seat + "." + seat.VerbSteps,
|
||||
Description: "Which pipeline steps a TypeScript artifact can run: the type-check as lint, always; unit tests where a package has a test script; " +
|
||||
"integration tests where one has a test:integration script. A level with no test is not answered.",
|
||||
Input: map[string]any{"type": "object", "required": []string{"module", "artifact", "fetch"}, "properties": map[string]any{
|
||||
Input: map[string]any{"type": "object", "required": []string{"module", "artifact"}, "properties": map[string]any{
|
||||
"module": map[string]any{"type": "string"},
|
||||
"artifact": map[string]any{"type": "string", "description": "the artifact reference, <module>:<artifact>"},
|
||||
"fingerprint": map[string]any{"type": "string"},
|
||||
"fetch": map[string]any{"type": "object", "description": "what the fetch tool is asked: the repository, the full commit and the module's directory"},
|
||||
"again": map[string]any{"type": "boolean", "description": "find them anew, not from the kept answer"},
|
||||
"drop": map[string]any{"type": "boolean", "description": "delete the artifact's kept key instead of detecting; needs why (written in the hand-act log first)"},
|
||||
"why": map[string]any{"type": "string", "description": "with drop, why: a non-empty text"},
|
||||
}},
|
||||
Run: m.Short(m.steps)}
|
||||
}
|
||||
|
||||
func (m *Module) steps(ctx context.Context, args map[string]any) (any, error) {
|
||||
str := func(k string) string { s, _ := args[k].(string); return s }
|
||||
if str("artifact") == "" || str("module") == "" {
|
||||
return nil, fmt.Errorf("steps: the module and the artifact are asked")
|
||||
}
|
||||
if drop, _ := args["drop"].(bool); drop {
|
||||
return m.dropDetected(args, stdio.CallerOf(args))
|
||||
}
|
||||
fetch, _ := args["fetch"].(map[string]any)
|
||||
req, err := sdkstep.ParseFetchRequest(fetch)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if str("artifact") == "" || str("module") == "" {
|
||||
return nil, fmt.Errorf("steps: the module and the artifact are asked")
|
||||
}
|
||||
artifact, fp := str("artifact"), str("fingerprint")
|
||||
name := artifact
|
||||
if _, n, ok := strings.Cut(artifact, ":"); ok {
|
||||
@@ -126,6 +134,27 @@ func (m *Module) steps(ctx context.Context, args map[string]any) (any, error) {
|
||||
return StepsAnswer{Artifact: artifact, Fingerprint: fp, Steps: steps}, nil
|
||||
}
|
||||
|
||||
// dropDetected is `steps` with drop (§463, §480): the act is written in the hand-act log with the caller the bus proved,
|
||||
// then the kept key is deleted; a log that refuses leaves the key.
|
||||
func (m *Module) dropDetected(args map[string]any, caller string) (any, error) {
|
||||
artifact, _ := args["artifact"].(string)
|
||||
why, _ := args["why"].(string)
|
||||
module, _ := args["module"].(string)
|
||||
name := artifact
|
||||
if _, n, ok := strings.Cut(artifact, ":"); ok {
|
||||
name = n
|
||||
}
|
||||
key := seat.DetectedStepsByModuleArtifactKey(module, name)
|
||||
failed, err := steprun.DropDetected(artifact, why, caller, key, m.RecordAct, m.ForgetDetected)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if failed != "" {
|
||||
return StepsAnswer{Artifact: artifact, Error: failed}, nil
|
||||
}
|
||||
return StepsAnswer{Artifact: artifact, Dropped: artifact}, nil
|
||||
}
|
||||
|
||||
// stepsOfTree fetches the commit alone (nothing installed) into a workspace removed at the end.
|
||||
func (m *Module) stepsOfTree(ctx context.Context, req sdkstep.FetchRequest) ([]DetectedStep, error) {
|
||||
if err := os.MkdirAll(m.Work, 0o700); err != nil {
|
||||
|
||||
@@ -2,9 +2,11 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
@@ -51,3 +53,27 @@ func TestTheStepsVerbIsServedAndRefusesAnAskWithoutAnArtifact(t *testing.T) {
|
||||
t.Error("an ask with no artifact was answered")
|
||||
}
|
||||
}
|
||||
|
||||
// Design 47 G §463, §480 (BN9): steps with drop writes the hand-act log first, then deletes the key; a log that
|
||||
// refuses deletes nothing; a drop without why is refused.
|
||||
func TestTheStepsDropWritesTheHandActLogFirst(t *testing.T) {
|
||||
var order []string
|
||||
m := &Module{
|
||||
ForgetDetected: func(key string) error { order = append(order, "delete "+key); return nil },
|
||||
RecordAct: func(what, why string) error { order = append(order, "log "+why); return nil },
|
||||
}
|
||||
args := map[string]any{"module": "mod", "artifact": "mod:web", "drop": true, "why": "stale"}
|
||||
got, err := m.dropDetected(args, "shanks.person")
|
||||
if a, _ := got.(StepsAnswer); err != nil || a.Dropped != "mod:web" || !reflect.DeepEqual(order, []string{"log stale", "delete mod.web"}) {
|
||||
t.Fatalf("drop = %+v, %v, order %v", got, err, order)
|
||||
}
|
||||
if _, err := m.dropDetected(map[string]any{"module": "mod", "artifact": "mod:web", "drop": true}, "shanks.person"); err == nil {
|
||||
t.Error("a drop without why was taken")
|
||||
}
|
||||
order = nil
|
||||
m.RecordAct = func(string, string) error { return errors.New("log down") }
|
||||
got, err = m.dropDetected(args, "shanks.person")
|
||||
if a, _ := got.(StepsAnswer); err != nil || !strings.Contains(a.Error, "hand-act log") || len(order) != 0 {
|
||||
t.Fatalf("a refusing log: %+v, %v, order %v", got, err, order)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,94 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/steprun"
|
||||
"io"
|
||||
"os"
|
||||
"sync"
|
||||
|
||||
sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
|
||||
)
|
||||
|
||||
// toolchainReporter says `toolchain-reported` on the build seat's subject of its node (design 47 G §492): once at the
|
||||
// module's start and each time its toolchains key of the language changes (a write that changes nothing says nothing).
|
||||
// Language is the seat's ask language (go, node, image).
|
||||
type toolchainReporter struct {
|
||||
AskLanguage, Node string
|
||||
// Publish is stdio.SeatPublish: the event is de-duplicated by its id.
|
||||
Publish func(subject string, body any, id string) error
|
||||
|
||||
mu sync.Mutex
|
||||
last map[string]sdkstep.Toolchain
|
||||
}
|
||||
|
||||
// Report says the toolchain written under key at revision rev when it differs from the last one said for the key.
|
||||
func (r *toolchainReporter) Report(key string, rev uint64, t sdkstep.Toolchain) error {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if prev, ok := r.last[key]; ok && prev.Version == t.Version && prev.Recipe == t.Recipe && prev.Digest == t.Digest && prev.Language == t.Language {
|
||||
return nil
|
||||
}
|
||||
seat, err := sdkstep.BuildSeat(r.AskLanguage)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := sdkstep.ToolchainKey(r.Node, t.Language); err != nil {
|
||||
return err
|
||||
}
|
||||
body := map[string]string{"node": r.Node, "language": t.Language, "version": t.Version, "recipe": t.Recipe, "digest": t.Digest}
|
||||
subject := "mesh.seat." + seat + ".event.toolchain-reported." + r.Node
|
||||
if err := r.Publish(subject, body, fmt.Sprintf("toolchain-reported.%s.%d", key, rev)); err != nil {
|
||||
return err
|
||||
}
|
||||
if r.last == nil {
|
||||
r.last = map[string]sdkstep.Toolchain{}
|
||||
}
|
||||
r.last[key] = t
|
||||
return nil
|
||||
}
|
||||
|
||||
// putReporting wraps a toolchains Put so each write that changes the toolchain is then said by r. A write that failed
|
||||
// says nothing; an event that failed is passed to say and does not refuse the write.
|
||||
func putReporting(put steprun.Put, r *toolchainReporter, digest func(version string) string, say func(format string, a ...any)) steprun.Put {
|
||||
return func(key string, value any) (uint64, error) {
|
||||
raw, merr := json.Marshal(value)
|
||||
var t sdkstep.Toolchain
|
||||
if merr != nil || json.Unmarshal(raw, &t) != nil {
|
||||
return put(key, value) // not a toolchain value: written as it is
|
||||
}
|
||||
// The value carries the digest of the toolchain program beside the version and the recipe (design 47 G §472).
|
||||
if t.Digest == "" && digest != nil {
|
||||
t.Digest = digest(t.Version)
|
||||
}
|
||||
enc, eerr := sdkstep.EncodeToolchain(t)
|
||||
if eerr != nil {
|
||||
return put(key, value)
|
||||
}
|
||||
rev, err := put(key, json.RawMessage(enc))
|
||||
if err != nil || r == nil || r.Publish == nil {
|
||||
return rev, err
|
||||
}
|
||||
if eerr := r.Report(key, rev, t); eerr != nil && say != nil {
|
||||
say("toolchain-reported could not be said: %v", eerr)
|
||||
}
|
||||
return rev, nil
|
||||
}
|
||||
}
|
||||
|
||||
// ProgramDigest is the sha256: digest of the file at path, a toolchain program (design 47 G §472).
|
||||
func programFileDigest(path string) (string, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer f.Close()
|
||||
h := sha256.New()
|
||||
if _, err := io.Copy(h, f); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return "sha256:" + hex.EncodeToString(h.Sum(nil)), nil
|
||||
}
|
||||
Reference in New Issue
Block a user