Merge branch 'feat/r4g-asks' into feat/r4g
This commit is contained in:
+3
-21
@@ -20,8 +20,8 @@
|
||||
// writes it (check.ts; ADR 0521, R4e finding R2-1). No tool writes mesh/check (finding K1), and no event: the watch hands
|
||||
// over the latest value of every key whenever this module starts.
|
||||
//
|
||||
// Consumes node-build-agent.started — a build machine took an ask — and says, on the head a pending check is of,
|
||||
// which machine builds it since when (novox/hq issue 446). This module is the one writer of mesh/merge-gate.
|
||||
// This module is the one writer of mesh/merge-gate. It hears no build seat's event: since R4g a check's steps are asks
|
||||
// on the build seats' streams, whose backlog the delivery view shows (novox/hq ADR 0525 §9; issue 1887).
|
||||
//
|
||||
// **A trunk's move puts every open verdict into it out of date** (novox/hq ADR 0323 §4, trunk.ts): each pass reads the
|
||||
// head of every trunk an open pull request merges into, and when one moved, the merge check statuses of every open pull
|
||||
@@ -53,7 +53,7 @@ import { mergeTool } from "./merge.js";
|
||||
import { GiteaAdmin, GiteaClient, movedSince } from "./client.js";
|
||||
import { CHECK_CONTEXT, CHECK_STATUS, REPO_CHECK_CONTEXT, commentFor, composedGate, groupGateStatus, headsToAnnounce, maintained,
|
||||
requiredOf, statusesFor, writesGate, type Announced, type Checked, repoCheckFacts, reposToLook } from "./pulls.js";
|
||||
import { GateWriter, clonedAs, type BuildStarted } from "./gate.js";
|
||||
import { GateWriter } from "./gate.js";
|
||||
import { HOOK_PATH, HOOK_PORT, Pace, claimHead, ensureSystemHook, eventsTool, keptSecret, receiver, type ForgeEvent } from "./webhook.js";
|
||||
import { keepSiblings, noteComposed, noteVerdict, readSiblings, siblingsFile, siblingsOf, weighSiblings } from "./siblings.js";
|
||||
import { bindInBackground } from "./bind.js";
|
||||
@@ -504,23 +504,6 @@ registerModuleTools("gitea", () => [eventsTool(pace), ...(gitea ? [commitStatusT
|
||||
gates,
|
||||
}), mergeTool(gitea, { gates })] : [])]);
|
||||
|
||||
// **A build machine took an ask** (the build seat's `started`, novox/hq ADR 0157): said on the head it checks, when
|
||||
// the head is an open pull request's this module announced. Any other build — a branch, a merge's — says nothing.
|
||||
async function buildStarted(event: { body: unknown }): Promise<void> {
|
||||
const b = (event.body ?? {}) as BuildStarted;
|
||||
const ref = String(b.ref ?? "").toLowerCase();
|
||||
if (!/^[0-9a-f]{40}$/.test(ref)) return;
|
||||
for (const [key, sha] of Object.entries(heads)) {
|
||||
if (sha.toLowerCase() !== ref) continue;
|
||||
const [owner, rest] = key.split("/");
|
||||
const repo = rest?.split("#")[0] ?? "";
|
||||
if (owner && repo && clonedAs(b.repository, owner, repo)) {
|
||||
await gates.started(owner, repo, sha, b);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** What the forge's event asks of the announcer at once (issue 729): a new head is announced, and a merge. */
|
||||
async function actOn(client: GiteaClient, event: ForgeEvent): Promise<void> {
|
||||
const b = event.body as { owner?: string; repo?: string; number?: number };
|
||||
@@ -628,7 +611,6 @@ if (gitea) {
|
||||
}
|
||||
return setVerdict(client, event);
|
||||
})],
|
||||
["node-build-agent.started", () => on("node-build-agent.started", buildStarted)],
|
||||
], () => console.log("[gitea] watching for new repositories and merged pull requests"));
|
||||
// The checker's mesh/check, from its owed state (check.ts; ADR 0521): a read not granted yet is said once and asked again,
|
||||
// apart from the binds above, so neither waits for the other.
|
||||
|
||||
@@ -86,8 +86,7 @@
|
||||
"forge.other"
|
||||
],
|
||||
"consumes": [
|
||||
"mesh-controller.checked",
|
||||
"node-build-agent.started"
|
||||
"mesh-controller.checked"
|
||||
],
|
||||
"listens": [
|
||||
{
|
||||
|
||||
@@ -0,0 +1,110 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// **The module pulls its seat's asks** (novox/hq ADR 0525 §2-§6, ADR 0536 §4; issue 1887 A3go). It holds the seat
|
||||
// node-build-go on its node and pulls the seat's asks through its runtime (buildstep.SeatAsks): delivery first, then
|
||||
// check, then recheck, at most the node's build capacity at once (its setting `capacity`, read at every pull). Each ask
|
||||
// runs with the same step bodies its tools run, behind the same guard, and ends on the key <node>.<ask id> of its steps
|
||||
// state. The tools stay as the hand path (ADR 0525 §8).
|
||||
|
||||
// Seat is the node seat this module holds.
|
||||
const Seat = "node-build-go"
|
||||
|
||||
// KeptState is the checker's kept verdicts, which a check ask takes for the packages whose digest has one (issue
|
||||
// 1887's design limit 1).
|
||||
const KeptState = delivery.CheckerModule + ".kept"
|
||||
|
||||
// askSteps are the module's step bodies for an ask, each holding its workspace while it runs, as a tool's does.
|
||||
func (m *Module) askSteps(steps []StepTool) buildstep.AskSteps {
|
||||
byName := map[string]buildstep.StepFunc{}
|
||||
for _, s := range steps {
|
||||
run := s.Run
|
||||
byName[s.Name] = func(ctx context.Context, args map[string]any) (any, error) {
|
||||
defer m.holdArg(args)()
|
||||
return run(ctx, args)
|
||||
}
|
||||
}
|
||||
return buildstep.AskSteps{
|
||||
Fetch: byName["build_go_fetch"], Build: byName["build_go_build"], Digest: byName["build_go_digest"], Test: byName["build_go_test"],
|
||||
Toolchain: func(ctx context.Context) (delivery.ToolchainUsed, error) {
|
||||
installed, err := m.Probe(ctx)
|
||||
if err != nil {
|
||||
return delivery.ToolchainUsed{}, err
|
||||
}
|
||||
return delivery.ToolchainUsed{Version: installed, Recipe: m.Recipe.Version()}, nil
|
||||
},
|
||||
Kept: m.KeptLookup,
|
||||
Release: func(ws string) error { return m.spaces().Release(ws) },
|
||||
Logf: m.Say,
|
||||
}
|
||||
}
|
||||
|
||||
// keptValue reads one key of the checker's kept state; nil when it is absent. Main sets it to the state's read; a test
|
||||
// its own.
|
||||
func (m *Module) keptValue(key string) (json.RawMessage, error) {
|
||||
if m.ReadKept == nil {
|
||||
return nil, nil
|
||||
}
|
||||
return m.ReadKept(key)
|
||||
}
|
||||
|
||||
// KeptLookup is whether the checker kept a verdict for the level and digest, and how many tests it counted.
|
||||
func (m *Module) KeptLookup(ctx context.Context, level, digest string) (int, bool, error) {
|
||||
key, err := delivery.KeptKey(level, digest)
|
||||
if err != nil {
|
||||
return 0, false, err
|
||||
}
|
||||
raw, err := m.keptValue(key)
|
||||
if err != nil || raw == nil {
|
||||
return 0, false, err
|
||||
}
|
||||
v, err := delivery.DecodeKept(raw)
|
||||
if err != nil {
|
||||
return 0, false, fmt.Errorf("%s holds what is not a kept verdict: %w", key, err)
|
||||
}
|
||||
return v.Tests, true, nil
|
||||
}
|
||||
|
||||
// Puller is the module's pull of its seat's asks, on source (the runtime's seat pull when nil).
|
||||
func (m *Module) Puller(steps []StepTool, source buildstep.AskSource) *buildstep.Puller {
|
||||
if source == nil {
|
||||
source = buildstep.SeatAsks{Language: delivery.AskGo}
|
||||
}
|
||||
capacity := m.Capacity
|
||||
if capacity == nil {
|
||||
capacity = buildstep.NodeCapacity(m.Settings, buildstep.DefaultCapacities, m.Node, m.Say)
|
||||
}
|
||||
ask := m.askSteps(steps)
|
||||
var load func(string) ([]byte, error)
|
||||
if m.ReadSteps.Get != nil {
|
||||
load = func(key string) ([]byte, error) {
|
||||
raw, err := m.ReadSteps.Get(key)
|
||||
return raw, err
|
||||
}
|
||||
}
|
||||
return &buildstep.Puller{Source: source, Language: delivery.AskGo, Node: m.Node, Put: m.Steps, Load: load,
|
||||
Capacity: capacity, Logf: m.Say,
|
||||
Run: func(ctx context.Context, a delivery.StepAsk) (any, error) {
|
||||
if err := m.Check(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := m.room(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return buildstep.RunAsk(ctx, ask, a)
|
||||
}}
|
||||
}
|
||||
|
||||
// SeatTools are the seat's verbs on this node (ADR 0525 §10): what it runs, and its pause and resume.
|
||||
func (m *Module) SeatTools(p *buildstep.Puller) []stdio.Tool {
|
||||
return buildstep.SeatTools(Seat, "Go", p)
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// 1887/AC-order, 1887/AC-pull against the real bus: a check ask published before a delivery ask runs after it, and at
|
||||
// capacity 1 the second waits in the stream until the first ends.
|
||||
func TestTheModuleTakesADeliveryAskBeforeAnEarlierCheck(t *testing.T) {
|
||||
url := os.Getenv("MESH_TEST_NATS")
|
||||
if url == "" {
|
||||
t.Skip("MESH_TEST_NATS names no bus: the integration level starts one")
|
||||
}
|
||||
nc, err := nats.Connect(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer nc.Close()
|
||||
js, err := jetstream.New(nc)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, stop := context.WithCancel(context.Background())
|
||||
defer stop()
|
||||
if err := buildstep.DeclareAskWorkers(ctx, js, delivery.AskGo, 30*time.Second); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = js.DeleteStream(context.Background(), "SEAT_NODE_BUILD_GO") })
|
||||
check := deliveryAsk()
|
||||
check.Kind = delivery.AskCheck
|
||||
first := deliveryAsk()
|
||||
for _, a := range []delivery.StepAsk{check, first} {
|
||||
raw, _ := delivery.EncodeStepAsk(a)
|
||||
subject, _ := delivery.AskSubject(a.Language, a.Kind)
|
||||
if _, err := js.Publish(ctx, subject, raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
states := newStates()
|
||||
m := moduleWith(states, "go1.26.0")
|
||||
m.Capacity = func() int { return 1 }
|
||||
var mu sync.Mutex
|
||||
var order []string
|
||||
steps := fakeSteps(new([]string))
|
||||
for i := range steps {
|
||||
if steps[i].Name == "build_go_fetch" {
|
||||
run := steps[i].Run
|
||||
steps[i].Run = func(ctx context.Context, args map[string]any) (any, error) {
|
||||
mu.Lock()
|
||||
order = append(order, args["commit"].(string))
|
||||
mu.Unlock()
|
||||
return run(ctx, args)
|
||||
}
|
||||
}
|
||||
}
|
||||
p := m.Puller(steps, &buildstep.JetStreamAsks{JS: js, Language: delivery.AskGo})
|
||||
p.PullWait = 200 * time.Millisecond
|
||||
go p.Pull(ctx)
|
||||
for _, a := range []delivery.StepAsk{first, check} {
|
||||
key, _ := delivery.AskStepKey("ace", a.ID)
|
||||
for deadline := time.Now().Add(10 * time.Second); ; time.Sleep(10 * time.Millisecond) {
|
||||
states.mu.Lock()
|
||||
v := states.steps[key]
|
||||
states.mu.Unlock()
|
||||
if len(v) > 0 && v[len(v)-1].State != delivery.StepRunning {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("the ask %s (%s) never ended", a.ID, a.Kind)
|
||||
}
|
||||
}
|
||||
states.mu.Lock()
|
||||
checkKey, _ := delivery.AskStepKey("ace", check.ID)
|
||||
if a.ID == first.ID && len(states.steps[checkKey]) > 0 {
|
||||
states.mu.Unlock()
|
||||
t.Fatal("the check ask started before the delivery ask ended at capacity 1")
|
||||
}
|
||||
states.mu.Unlock()
|
||||
}
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if len(order) != 2 {
|
||||
t.Fatalf("fetched %v", order)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
const askCommit = "0123456789abcdef0123456789abcdef01234567"
|
||||
|
||||
// fakeSteps are step tools that answer at once: a fetch with a workspace, a build with a digest and a reference.
|
||||
func fakeSteps(calls *[]string) []StepTool {
|
||||
answer := func(name string, out any) StepTool {
|
||||
return StepTool{Name: name, Run: func(ctx context.Context, args map[string]any) (any, error) {
|
||||
*calls = append(*calls, name)
|
||||
return out, nil
|
||||
}}
|
||||
}
|
||||
return []StepTool{
|
||||
answer("build_go_fetch", map[string]any{"workspace": "ws1"}),
|
||||
answer("build_go_build", map[string]any{"digest": "sha256:aa", "reference": "artifact-store://mesh-build-go/tools@sha256:aa"}),
|
||||
answer("build_go_digest", map[string]any{"level": "unit", "packages": map[string]string{}}),
|
||||
answer("build_go_test", map[string]any{"passed": true}),
|
||||
}
|
||||
}
|
||||
|
||||
func deliveryAsk() delivery.StepAsk {
|
||||
return delivery.StepAsk{ID: buildstep.NewID(), Kind: delivery.AskDelivery, Language: delivery.AskGo, Asker: "mesh-delivery-builder",
|
||||
At: time.Now(), Build: &delivery.BuildJob{Module: "mesh-delivery-view", Path: "modules/mesh-delivery-view", Publish: true,
|
||||
Fetch: map[string]any{"repository": "novox/mesh-delivery", "commit": askCommit},
|
||||
Artifacts: []delivery.AskArtifact{{Artifact: "program"}}}}
|
||||
}
|
||||
|
||||
// 1887/A3go: an ask pulled runs build_go_build after build_go_fetch, and the step key <node>.<ask id> names the
|
||||
// reference.
|
||||
func TestAPulledAskRunsTheBuildAndItsKeyNamesTheReference(t *testing.T) {
|
||||
states := newStates()
|
||||
m := moduleWith(states, "go1.26.0")
|
||||
m.Capacity = func() int { return 1 }
|
||||
var calls []string
|
||||
src := buildstep.NewFakeAsks()
|
||||
a := deliveryAsk()
|
||||
if err := src.PublishAsk(a); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p := m.Puller(fakeSteps(&calls), src)
|
||||
p.PullWait = 10 * time.Millisecond
|
||||
ctx, stop := context.WithCancel(context.Background())
|
||||
defer stop()
|
||||
go p.Pull(ctx)
|
||||
key, _ := delivery.AskStepKey("ace", a.ID)
|
||||
var step delivery.Step
|
||||
for deadline := time.Now().Add(3 * time.Second); ; time.Sleep(5 * time.Millisecond) {
|
||||
states.mu.Lock()
|
||||
v := states.steps[key]
|
||||
states.mu.Unlock()
|
||||
if len(v) > 0 && v[len(v)-1].State != delivery.StepRunning {
|
||||
step = v[len(v)-1]
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("the ask's step %s never ended: %v", key, v)
|
||||
}
|
||||
}
|
||||
if step.State != delivery.StepPassed || !strings.Contains(string(step.Result), "artifact-store://mesh-build-go/tools@sha256:aa") {
|
||||
t.Fatalf("the step = %+v", step)
|
||||
}
|
||||
if strings.Join(calls, ",") != "build_go_fetch,build_go_build" {
|
||||
t.Errorf("calls = %v", calls)
|
||||
}
|
||||
if acked, _, _ := src.Fates(); len(acked) != 1 || acked[0] != a.ID {
|
||||
t.Errorf("acked %v", acked)
|
||||
}
|
||||
}
|
||||
|
||||
// 1887/AC-zero: at capacity 0 the node's module pulls nothing, and its tools still answer a direct call.
|
||||
func TestAtCapacityZeroNothingIsPulledAndTheToolsAnswer(t *testing.T) {
|
||||
states := newStates()
|
||||
m := moduleWith(states, "go1.26.0")
|
||||
m.Capacity = func() int { return 0 }
|
||||
var calls []string
|
||||
steps := fakeSteps(&calls)
|
||||
src := buildstep.NewFakeAsks()
|
||||
_ = src.PublishAsk(deliveryAsk())
|
||||
p := m.Puller(steps, src)
|
||||
p.PullWait = 10 * time.Millisecond
|
||||
ctx, stop := context.WithCancel(context.Background())
|
||||
defer stop()
|
||||
go p.Pull(ctx)
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
if src.Waiting(delivery.AskDelivery) != 1 || len(calls) != 0 {
|
||||
t.Fatalf("a node at capacity 0 pulled: waiting %d, calls %v", src.Waiting(delivery.AskDelivery), calls)
|
||||
}
|
||||
for _, tool := range m.Tools(steps) {
|
||||
if tool.Name == "build_go_fetch" {
|
||||
out, err := tool.Run(map[string]any{"repository": "novox/x", "commit": askCommit})
|
||||
if err != nil || !strings.Contains(string(mustJSON(out)), `"workspace":"ws1"`) || len(calls) != 1 {
|
||||
t.Fatalf("the direct call answered %v, %v", out, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Fatal("no fetch tool")
|
||||
}
|
||||
|
||||
func mustJSON(v any) []byte { b, _ := json.Marshal(v); return b }
|
||||
|
||||
// 1887/A3go, A7: the manifest claims node-build-go serving its three verbs, reads the checker's kept verdicts and takes
|
||||
// the node's build capacity as a setting, with the builder's default; the seat's verbs are served as its tools.
|
||||
func TestManifestClaimsTheGoBuildSeatAndReadsItsCapacity(t *testing.T) {
|
||||
m := readManifest(t)
|
||||
if len(m.Claims) != 1 || m.Claims[0].Name != Seat || !slices.Equal(m.Claims[0].Serves, []string{"current", "pause", "resume"}) {
|
||||
t.Fatalf("claims = %+v", m.Claims)
|
||||
}
|
||||
if !slices.Contains(m.Reads, KeptState) {
|
||||
t.Errorf("reads = %v lack %s", m.Reads, KeptState)
|
||||
}
|
||||
raw, _ := os.ReadFile(moduleDir + "/module.json")
|
||||
var doc struct {
|
||||
Config map[string]struct {
|
||||
Default string `json:"default"`
|
||||
} `json:"config"`
|
||||
}
|
||||
_ = json.Unmarshal(raw, &doc)
|
||||
if doc.Config[buildstep.CapacitySetting].Default != buildstep.DefaultCapacities {
|
||||
t.Errorf("the capacity's default = %q, want %q", doc.Config[buildstep.CapacitySetting].Default, buildstep.DefaultCapacities)
|
||||
}
|
||||
settings := resource(t, m, "settings")
|
||||
if settings["path"] != "${dir:state}/settings.json" {
|
||||
t.Errorf("the settings file is at %v", settings["path"])
|
||||
}
|
||||
served := map[string]bool{}
|
||||
for _, tool := range m0().SeatTools(m0().Puller(nil, buildstep.NewFakeAsks())) {
|
||||
served[tool.Name] = true
|
||||
}
|
||||
for _, verb := range m.Claims[0].Serves {
|
||||
if !served[Seat+"."+verb] {
|
||||
t.Errorf("%s.%s is not served", Seat, verb)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func m0() *Module { return &Module{Node: "ace", Capacity: func() int { return 0 }} }
|
||||
@@ -0,0 +1,129 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"go/format"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
// **gofmt and go vet are the unit level's** (novox/hq ADR 0529 §1; issue 1887 D7-go): what every repository's script ran
|
||||
// beside its tests, run for every repository by the build module that runs its unit level. In each go.mod directory the
|
||||
// level runs, every Go file outside testdata and vendor must be as gofmt writes it, and `go vet ./...` must pass with no
|
||||
// tag and with every build tag the directory's files name together (integration, isolation, ...), so code behind a tag
|
||||
// nothing else compiles is vetted too. A failure is the level's, naming the files or vet's lines.
|
||||
|
||||
// buildTag is one name in a //go:build line.
|
||||
var buildTag = regexp.MustCompile(`[A-Za-z_][A-Za-z0-9_.]*`)
|
||||
|
||||
// notATag are the names of a //go:build line that are no tag a run sets: the platforms, cgo, ignore, go versions.
|
||||
var notATag = map[string]bool{"ignore": true, "cgo": true, "linux": true, "darwin": true, "windows": true, "freebsd": true,
|
||||
"openbsd": true, "netbsd": true, "unix": true, "plan9": true, "js": true, "wasip1": true, "android": true, "ios": true,
|
||||
"amd64": true, "arm64": true, "386": true, "arm": true, "riscv64": true, "wasm": true, "ppc64le": true, "s390x": true}
|
||||
|
||||
// lintable walks a go.mod directory's own Go files: not those of testdata, vendor, or a directory holding a go.mod of its
|
||||
// own (another directory of the level, linted there).
|
||||
func lintable(dir string, each func(path string) error) error {
|
||||
return filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if d.IsDir() {
|
||||
if path != dir {
|
||||
switch d.Name() {
|
||||
case "testdata", "vendor":
|
||||
return filepath.SkipDir
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(path, "go.mod")); err == nil {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
if strings.HasPrefix(d.Name(), ".") {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if strings.HasSuffix(path, ".go") {
|
||||
return each(path)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// unformatted are the Go files of a directory gofmt would change, relative to the tree, and the build tags they name.
|
||||
func unformatted(tree, d string) (files, tags []string, err error) {
|
||||
seen := map[string]bool{}
|
||||
err = lintable(filepath.Join(tree, d), func(path string) error {
|
||||
src, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if out, ferr := format.Source(src); ferr != nil || !bytes.Equal(out, src) {
|
||||
rel, _ := filepath.Rel(tree, path)
|
||||
files = append(files, filepath.ToSlash(rel))
|
||||
}
|
||||
scan := bufio.NewScanner(bytes.NewReader(src))
|
||||
for scan.Scan() {
|
||||
line := strings.TrimSpace(scan.Text())
|
||||
if strings.HasPrefix(line, "package ") {
|
||||
break
|
||||
}
|
||||
if expr, ok := strings.CutPrefix(line, "//go:build "); ok {
|
||||
for _, tag := range buildTag.FindAllString(expr, -1) {
|
||||
if !notATag[tag] && !strings.HasPrefix(tag, "go1") && !seen[tag] {
|
||||
seen[tag] = true
|
||||
tags = append(tags, tag)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
sort.Strings(files)
|
||||
sort.Strings(tags)
|
||||
return files, tags, err
|
||||
}
|
||||
|
||||
// lint is gofmt and go vet of one go.mod directory d of the tree, run with run in the directory's go environment; a
|
||||
// failure of either is a Failure of the level naming what failed.
|
||||
func (m *Module) lint(ctx context.Context, run forgekey.Runner, tree, d string, env []string) ([]Failure, error) {
|
||||
files, tags, err := unformatted(tree, d)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading the Go files of %s: %w", d, err)
|
||||
}
|
||||
var failed []Failure
|
||||
if len(files) > 0 {
|
||||
failed = append(failed, Failure{Package: d, Test: "gofmt",
|
||||
Output: "not as gofmt writes them (run gofmt -w): " + strings.Join(files, ", ")})
|
||||
}
|
||||
sets := [][]string{nil}
|
||||
if len(tags) > 0 {
|
||||
sets = append(sets, tags)
|
||||
}
|
||||
for _, set := range sets {
|
||||
args := []string{"-C", filepath.Join(tree, d), "vet"}
|
||||
name := "go vet"
|
||||
if len(set) > 0 {
|
||||
args = append(args, "-tags", strings.Join(set, ","))
|
||||
name += " -tags " + strings.Join(set, ",")
|
||||
}
|
||||
args = append(args, "./...")
|
||||
r := run(ctx, env, "go", args...)
|
||||
if r.Err != "" {
|
||||
return nil, fmt.Errorf("%s in %s could not be run: %s", name, d, firstLine(r.Err+"\n"+r.Stderr))
|
||||
}
|
||||
if r.Status != 0 {
|
||||
failed = append(failed, Failure{Package: d, Test: name, Output: strings.TrimSpace(r.Stderr + "\n" + r.Stdout)})
|
||||
}
|
||||
}
|
||||
return failed, nil
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
func writeTree(t *testing.T, files map[string]string) string {
|
||||
t.Helper()
|
||||
tree := t.TempDir()
|
||||
for name, body := range files {
|
||||
p := filepath.Join(tree, name)
|
||||
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(p, []byte(body), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return tree
|
||||
}
|
||||
|
||||
// 1887/D7-go, ADR 0529 §1: an unformatted file fails the unit level naming the file (testdata and vendor are left
|
||||
// alone), and go vet runs with no tag and with every tag the files name together, integration and isolation included, so
|
||||
// a vet error behind isolation fails the level naming the run.
|
||||
func TestTheUnitLevelRunsGofmtAndVetWithEveryTag(t *testing.T) {
|
||||
tree := writeTree(t, map[string]string{
|
||||
"go.mod": "module x\n\ngo 1.26\n",
|
||||
"a.go": "package x\n\nfunc A() {}\n",
|
||||
"b.go": "package x\nfunc B() {}\n",
|
||||
"in_integration_test.go": "//go:build integration\n\npackage x\n",
|
||||
"iso.go": "//go:build integration && isolation && linux\n\npackage x\n",
|
||||
"testdata/ugly.go": "package ugly\n",
|
||||
"vendor/v/v.go": "package v\n",
|
||||
"nested/go.mod": "module x/nested\n",
|
||||
"nested/n.go": "package n\n",
|
||||
})
|
||||
var vetted []string
|
||||
run := func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
|
||||
line := strings.Join(args[2:], " ")
|
||||
vetted = append(vetted, line)
|
||||
if strings.Contains(line, "isolation") {
|
||||
return forgekey.Ran{Status: 1, Stderr: "iso.go:3: unreachable code"}
|
||||
}
|
||||
return forgekey.Ran{}
|
||||
}
|
||||
failed, err := (&Module{}).lint(context.Background(), run, tree, ".", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Join(vetted, " | ") != "vet ./... | vet -tags integration,isolation ./..." {
|
||||
t.Errorf("vet ran %v", vetted)
|
||||
}
|
||||
if len(failed) != 2 || failed[0].Test != "gofmt" || !strings.Contains(failed[0].Output, "b.go") ||
|
||||
strings.Contains(failed[0].Output, "ugly") || strings.Contains(failed[0].Output, "n.go") || strings.Contains(failed[0].Output, "v.go") {
|
||||
t.Fatalf("failed = %+v, want gofmt naming b.go alone", failed)
|
||||
}
|
||||
if failed[1].Test != "go vet -tags integration,isolation" || !strings.Contains(failed[1].Output, "unreachable") {
|
||||
t.Errorf("failed = %+v, want vet with both tags naming its line", failed[1])
|
||||
}
|
||||
// A formatted directory that vets passes.
|
||||
clean := writeTree(t, map[string]string{"go.mod": "module y\n", "a.go": "package y\n"})
|
||||
if failed, err := (&Module{}).lint(context.Background(), func(context.Context, []string, string, ...string) forgekey.Ran { return forgekey.Ran{} }, clean, ".", nil); err != nil || len(failed) != 0 {
|
||||
t.Errorf("a clean directory: %+v, %v", failed, err)
|
||||
}
|
||||
}
|
||||
|
||||
// 1887/D7-go: through the step, the unit level of a tree holding an unformatted file fails, naming the file.
|
||||
func TestAnUnformattedFileFailsTheUnitLevel(t *testing.T) {
|
||||
m, states, ws, _ := workspaceFor(t, "empty")
|
||||
tree := filepath.Join(m.Work, ws, "tree")
|
||||
for name, body := range map[string]string{"go.mod": "module x\n", "x.go": "package x\nfunc X() {}\n"} {
|
||||
if err := os.WriteFile(filepath.Join(tree, name), []byte(body), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
res, _ := runUnit(t, m, states, ws)
|
||||
if res.Passed || len(res.Failed) != 1 || res.Failed[0].Test != "gofmt" || !strings.Contains(res.Failed[0].Output, "x.go") {
|
||||
t.Fatalf("result = %+v, want the level failed naming x.go", res)
|
||||
}
|
||||
}
|
||||
@@ -52,6 +52,8 @@ func main() {
|
||||
}
|
||||
m.Run = StepRunner(m.Home)
|
||||
m.RegistryFile = m.Home + "/registry"
|
||||
m.Settings = m.Home + "/settings.json"
|
||||
m.ReadKept = readOf(stdio.State(KeptState)).Get
|
||||
m.Probe = ProbeGo(m.Home, m.Cache)
|
||||
m.Bubblewrap = ProbeBubblewrap
|
||||
// Outside a launch by the runtime — a build, a check — there is no node and no state to write.
|
||||
@@ -64,7 +66,13 @@ func main() {
|
||||
logf("announcing the states it keeps: %v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if err := stdio.Serve("", m.Tools(m.StepTools())); err != nil {
|
||||
steps := m.StepTools()
|
||||
puller := m.Puller(steps, nil)
|
||||
if m.Node != "" {
|
||||
// The seat's asks, pulled once Serve has the runtime's channel; a pull before it is said and tried again.
|
||||
go func() { _ = puller.Pull(ctx) }()
|
||||
}
|
||||
if err := stdio.Serve("", append(m.Tools(steps), m.SeatTools(puller)...)); err != nil {
|
||||
logf("%v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
@@ -84,9 +84,10 @@ func TestWorkAndCacheDirectoriesLieOutsideTheHome(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// The states are the three the module keeps, no seat is claimed (ADR 0469 §6), and the health is the module's own
|
||||
// tool, which the manifest also lists among its tools.
|
||||
func TestManifestKeepsItsStatesClaimsNoSeatAndDeclaresItsHealthTool(t *testing.T) {
|
||||
// The states are the three the module keeps, the one seat it claims is its language's build seat (ADR 0525 §2, which
|
||||
// changes ADR 0469 §6's "no seat"), and the health is the module's own tool, which the manifest also lists among its
|
||||
// tools.
|
||||
func TestManifestKeepsItsStatesClaimsItsBuildSeatAndDeclaresItsHealthTool(t *testing.T) {
|
||||
m := readManifest(t)
|
||||
states := []string{}
|
||||
for _, s := range m.State {
|
||||
@@ -96,8 +97,8 @@ func TestManifestKeepsItsStatesClaimsNoSeatAndDeclaresItsHealthTool(t *testing.T
|
||||
if want := []string{"keys", "steps", "toolchains"}; !slices.Equal(states, want) {
|
||||
t.Errorf("state = %v, want %v", states, want)
|
||||
}
|
||||
if len(m.Claims) != 0 {
|
||||
t.Errorf("claims = %v, want no seat", m.Claims)
|
||||
if len(m.Claims) != 1 || m.Claims[0].Name != Seat {
|
||||
t.Errorf("claims = %v, want the seat %s alone", m.Claims, Seat)
|
||||
}
|
||||
if !slices.Contains(m.Tools, HealthTool) {
|
||||
t.Errorf("tools = %v lack %s", m.Tools, HealthTool)
|
||||
|
||||
@@ -2,6 +2,7 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"sync"
|
||||
@@ -76,6 +77,12 @@ type Module struct {
|
||||
// floorOf(node), and a zero Floor checks nothing.
|
||||
Floor buildstep.Floor
|
||||
|
||||
// Settings is the module's settings file, which holds the node's build capacity (buildstep.CapacitySetting); Capacity,
|
||||
// when set, answers it instead (a test). ReadKept reads a key of the checker's kept state (KeptState).
|
||||
Settings string
|
||||
Capacity func() int
|
||||
ReadKept func(key string) (json.RawMessage, error)
|
||||
|
||||
spacesOnce sync.Once
|
||||
workspaces *buildstep.Workspaces
|
||||
}
|
||||
|
||||
@@ -253,6 +253,16 @@ func (m *Module) runLevel(ctx context.Context, tree string, o levelOpts) (*TestR
|
||||
if o.wall {
|
||||
res.Elapsed = m.clock().Sub(began).Seconds()
|
||||
}
|
||||
if o.level == LevelUnit {
|
||||
// gofmt and go vet of each directory the level ran are the unit level's (ADR 0529 §1; issue 1887 D7-go).
|
||||
for _, d := range todo {
|
||||
failed, err := m.lint(ctx, run, tree, d, append(m.goEnvFor(filepath.Join(tree, d)), o.env...))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res.Failed = append(res.Failed, failed...)
|
||||
}
|
||||
}
|
||||
res.judge(o.bounds)
|
||||
return res, nil
|
||||
}
|
||||
|
||||
@@ -30,6 +30,8 @@ func capturedRunner(t *testing.T, captured string, calls *[]cmdRecord) forgekey.
|
||||
switch {
|
||||
case name == "git" && args[2] == "ls-files":
|
||||
return forgekey.Ran{Stdout: "go.mod\n"}
|
||||
case name == "go" && len(args) > 2 && args[2] == "vet":
|
||||
return forgekey.Ran{} // vet passes: the fixtures are vetted (lint_test.go)
|
||||
case name == "go":
|
||||
return forgekey.Ran{Stdout: string(raw), Status: status}
|
||||
}
|
||||
@@ -167,7 +169,7 @@ func TestUnitLevelRunsGoTestJsonInTheModulesDirectoryWithTheStepsEnvironment(t *
|
||||
var line string
|
||||
var env string
|
||||
for _, c := range *calls {
|
||||
if strings.HasPrefix(c.line, "go ") {
|
||||
if strings.HasPrefix(c.line, "go ") && strings.Contains(c.line, " test ") {
|
||||
line, env = c.line, strings.Join(c.env, " ")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
{
|
||||
"capacity": "shanks=3,ace=2,g14=0,novox=0"
|
||||
}
|
||||
@@ -12,6 +12,24 @@
|
||||
"keys",
|
||||
"steps"
|
||||
],
|
||||
"claims": [
|
||||
{
|
||||
"name": "node-build-go",
|
||||
"scope": "node",
|
||||
"serves": ["current", "pause", "resume"]
|
||||
}
|
||||
],
|
||||
"reads": [
|
||||
"mesh-delivery-checker.kept"
|
||||
],
|
||||
"config": {
|
||||
"capacity": {
|
||||
"type": "string",
|
||||
"description": "How many asks of the Go build seat this module runs at once on each node, as node=asks pairs separated by commas, such as shanks=3,ace=2 (novox/hq ADR 0525 §5); a node set to 0 pulls nothing, a node not named runs one at a time. The control-node and the laptop build nothing.",
|
||||
"pattern": "|[a-z0-9_-]+=[0-9]+(,[a-z0-9_-]+=[0-9]+)*",
|
||||
"default": "shanks=3,ace=2,g14=0,novox=0"
|
||||
}
|
||||
},
|
||||
"tools": [
|
||||
"build_go_fetch",
|
||||
"build_go_test",
|
||||
@@ -79,6 +97,17 @@
|
||||
"owner": "mesh-build-go",
|
||||
"unknown": "program"
|
||||
},
|
||||
{
|
||||
"id": "settings",
|
||||
"trusted": false,
|
||||
"type": "file",
|
||||
"path": "${dir:state}/settings.json",
|
||||
"mode": "0600",
|
||||
"owner": "mesh-build-go",
|
||||
"takes-config": ["capacity"],
|
||||
"merge": "json",
|
||||
"template": "files/settings.json"
|
||||
},
|
||||
{
|
||||
"id": "ssh-dir",
|
||||
"type": "directory",
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// **The module pulls its seat's asks** (novox/hq ADR 0525 §2-§6; issue 1887 A3image), as mesh-build-go does: the seat
|
||||
// node-build-image, at most the node's build capacity at once (its setting `capacity`); a delivery or check ask is a build
|
||||
// (fetch, build, or copy of a pinned image), never a check of tests. The tools stay as the hand path (ADR 0525 §8).
|
||||
|
||||
// Seat is the node seat this module holds.
|
||||
const Seat = "node-build-image"
|
||||
|
||||
func (m *Module) askSteps(steps []StepTool) buildstep.AskSteps {
|
||||
byName := map[string]buildstep.StepFunc{}
|
||||
for _, s := range steps {
|
||||
run := s.Run
|
||||
byName[s.Name] = func(ctx context.Context, args map[string]any) (any, error) {
|
||||
defer m.holdArg(args)()
|
||||
return run(ctx, args)
|
||||
}
|
||||
}
|
||||
return buildstep.AskSteps{
|
||||
Fetch: byName["build_image_fetch"], Build: byName["build_image_build"], Copy: byName["build_image_copy"],
|
||||
Release: func(ws string) error { return m.spaces().Release(ws) },
|
||||
Logf: m.Say,
|
||||
}
|
||||
}
|
||||
|
||||
// Puller is the module's pull of its seat's asks, on source (the runtime's seat pull when nil).
|
||||
func (m *Module) Puller(steps []StepTool, source buildstep.AskSource) *buildstep.Puller {
|
||||
if source == nil {
|
||||
source = buildstep.SeatAsks{Language: delivery.AskImage}
|
||||
}
|
||||
capacity := m.Capacity
|
||||
if capacity == nil {
|
||||
capacity = buildstep.NodeCapacity(m.Settings, buildstep.DefaultCapacities, m.Node, m.Say)
|
||||
}
|
||||
ask := m.askSteps(steps)
|
||||
var load func(string) ([]byte, error)
|
||||
if m.ReadSteps.Get != nil {
|
||||
load = func(key string) ([]byte, error) { return m.ReadSteps.Get(key) }
|
||||
}
|
||||
return &buildstep.Puller{Source: source, Language: delivery.AskImage, Node: m.Node, Put: m.Steps, Load: load,
|
||||
Capacity: capacity, Logf: m.Say,
|
||||
Run: func(ctx context.Context, a delivery.StepAsk) (any, error) {
|
||||
if err := m.Check(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := m.room(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return buildstep.RunAsk(ctx, ask, a)
|
||||
}}
|
||||
}
|
||||
|
||||
// SeatTools are the seat's verbs on this node (ADR 0525 §10).
|
||||
func (m *Module) SeatTools(p *buildstep.Puller) []stdio.Tool {
|
||||
return buildstep.SeatTools(Seat, "image", p)
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
const askCommit = "0123456789abcdef0123456789abcdef01234567"
|
||||
|
||||
// 1887/A3image: a pulled image ask copies a pinned image with no fetch, then fetches and builds the module's own, and its
|
||||
// key <node>.<ask id> names each reference; the seat's verbs are served.
|
||||
func TestAPulledImageAskCopiesThenBuilds(t *testing.T) {
|
||||
states := newStates()
|
||||
m := moduleWith(t, newFake(t), states)
|
||||
m.Capacity = func() int { return 1 }
|
||||
var calls []string
|
||||
answer := func(name string, out any) StepTool {
|
||||
return StepTool{Name: name, Run: func(ctx context.Context, args map[string]any) (any, error) {
|
||||
calls = append(calls, name)
|
||||
return out, nil
|
||||
}}
|
||||
}
|
||||
steps := []StepTool{
|
||||
answer("build_image_fetch", map[string]any{"workspace": "ws1"}),
|
||||
answer("build_image_build", map[string]any{"digest": "sha256:cc", "reference": "registry/m/server@sha256:cc"}),
|
||||
answer("build_image_copy", map[string]any{"digest": "sha256:dd", "reference": "registry/m/base@sha256:dd"}),
|
||||
}
|
||||
src := buildstep.NewFakeAsks()
|
||||
a := delivery.StepAsk{ID: buildstep.NewID(), Kind: delivery.AskDelivery, Language: delivery.AskImage, Asker: "mesh-delivery-builder",
|
||||
At: time.Now(), Build: &delivery.BuildJob{Module: "m", Path: "modules/m", Publish: true,
|
||||
Fetch: map[string]any{"repository": "novox/mesh-catalog", "commit": askCommit},
|
||||
Artifacts: []delivery.AskArtifact{{Artifact: "base", From: "docker.io/library/alpine@sha256:00"}, {Artifact: "server"}}}}
|
||||
_ = src.PublishAsk(a)
|
||||
p := m.Puller(steps, src)
|
||||
p.PullWait = 10 * time.Millisecond
|
||||
ctx, stop := context.WithCancel(context.Background())
|
||||
defer stop()
|
||||
go p.Pull(ctx)
|
||||
key, _ := delivery.AskStepKey("ace", a.ID)
|
||||
var step delivery.Step
|
||||
for deadline := time.Now().Add(3 * time.Second); ; time.Sleep(5 * time.Millisecond) {
|
||||
states.mu.Lock()
|
||||
v := states.steps[key]
|
||||
states.mu.Unlock()
|
||||
if len(v) > 0 && v[len(v)-1].State != delivery.StepRunning {
|
||||
step = v[len(v)-1]
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("the ask's step never ended: calls %v", calls)
|
||||
}
|
||||
}
|
||||
if step.State != delivery.StepPassed || !strings.Contains(string(step.Result), "registry/m/server@sha256:cc") ||
|
||||
!strings.Contains(string(step.Result), "registry/m/base@sha256:dd") {
|
||||
t.Fatalf("the step = %+v", step)
|
||||
}
|
||||
if got := strings.Join(calls, ","); got != "build_image_copy,build_image_fetch,build_image_build" {
|
||||
t.Errorf("calls = %s", got)
|
||||
}
|
||||
served := map[string]bool{}
|
||||
for _, tool := range m.SeatTools(p) {
|
||||
served[tool.Name] = true
|
||||
}
|
||||
for _, verb := range []string{"current", "pause", "resume"} {
|
||||
if !served[Seat+"."+verb] {
|
||||
t.Errorf("%s.%s is not served", Seat, verb)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -42,6 +42,7 @@ func main() {
|
||||
m.Home, m.Account, m.UID = u.HomeDir, u.Username, u.Uid
|
||||
}
|
||||
m.RegistryFile = m.Home + "/registry"
|
||||
m.Settings = m.Home + "/settings.json"
|
||||
m.Run = StepRunner(m.Home)
|
||||
// Outside a launch by the runtime — a build, a check — there is no node and no state to write.
|
||||
if m.Node != "" {
|
||||
@@ -53,7 +54,13 @@ func main() {
|
||||
logf("announcing the states it keeps: %v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if err := stdio.Serve("", m.Tools(m.StepTools())); err != nil {
|
||||
steps := m.StepTools()
|
||||
puller := m.Puller(steps, nil)
|
||||
if m.Node != "" {
|
||||
// The seat's asks, pulled once Serve has the runtime's channel; a pull before it is said and tried again.
|
||||
go func() { _ = puller.Pull(ctx) }()
|
||||
}
|
||||
if err := stdio.Serve("", append(m.Tools(steps), m.SeatTools(puller)...)); err != nil {
|
||||
logf("%v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
@@ -90,9 +90,9 @@ func TestWorkAndStoreDirectoriesLieOutsideTheHome(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// The states are the two the module keeps, no seat is claimed, the health is its own tool, which the manifest also lists,
|
||||
// The states are the two the module keeps, the one seat it claims is its build seat (ADR 0525 §2), the health is its own tool, which the manifest also lists,
|
||||
// and the registry's address is bound into a file of the home that the program reads.
|
||||
func TestManifestKeepsItsStatesClaimsNoSeatAndDeclaresItsHealthTool(t *testing.T) {
|
||||
func TestManifestKeepsItsStatesClaimsItsBuildSeatAndDeclaresItsHealthTool(t *testing.T) {
|
||||
m := readManifest(t)
|
||||
states := []string{}
|
||||
for _, s := range m.State {
|
||||
@@ -102,8 +102,8 @@ func TestManifestKeepsItsStatesClaimsNoSeatAndDeclaresItsHealthTool(t *testing.T
|
||||
if want := []string{"keys", "steps"}; !slices.Equal(states, want) {
|
||||
t.Errorf("state = %v, want %v", states, want)
|
||||
}
|
||||
if len(m.Claims) != 0 {
|
||||
t.Errorf("claims = %v, want no seat", m.Claims)
|
||||
if len(m.Claims) != 1 || m.Claims[0].Name != Seat || !slices.Equal(m.Claims[0].Serves, []string{"current", "pause", "resume"}) {
|
||||
t.Errorf("claims = %v, want the seat %s alone", m.Claims, Seat)
|
||||
}
|
||||
raw, _ := os.ReadFile(filepath.Join(moduleDir, "module.json"))
|
||||
if !strings.Contains(string(raw), `"tool": "`+HealthTool+`"`) {
|
||||
|
||||
@@ -50,6 +50,11 @@ type Module struct {
|
||||
// 1839); main sets floorOf(node), and a zero Floor checks nothing.
|
||||
Floor buildstep.Floor
|
||||
|
||||
// Settings is the module's settings file, which holds the node's build capacity (buildstep.CapacitySetting); Capacity,
|
||||
// when set, answers it instead (a test).
|
||||
Settings string
|
||||
Capacity func() int
|
||||
|
||||
spacesOnce sync.Once
|
||||
workspaces *buildstep.Workspaces
|
||||
}
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
{
|
||||
"capacity": "shanks=3,ace=2,g14=0,novox=0"
|
||||
}
|
||||
@@ -11,6 +11,21 @@
|
||||
"keys",
|
||||
"steps"
|
||||
],
|
||||
"claims": [
|
||||
{
|
||||
"name": "node-build-image",
|
||||
"scope": "node",
|
||||
"serves": ["current", "pause", "resume"]
|
||||
}
|
||||
],
|
||||
"config": {
|
||||
"capacity": {
|
||||
"type": "string",
|
||||
"description": "How many asks of the image build seat this module runs at once on each node, as node=asks pairs separated by commas, such as shanks=3,ace=2 (novox/hq ADR 0525 §5); a node set to 0 pulls nothing, a node not named runs one at a time. The control-node and the laptop build nothing.",
|
||||
"pattern": "|[a-z0-9_-]+=[0-9]+(,[a-z0-9_-]+=[0-9]+)*",
|
||||
"default": "shanks=3,ace=2,g14=0,novox=0"
|
||||
}
|
||||
},
|
||||
"tools": [
|
||||
"build_image_fetch",
|
||||
"build_image_build",
|
||||
@@ -54,6 +69,17 @@
|
||||
"place": ".",
|
||||
"unknown": "program"
|
||||
},
|
||||
{
|
||||
"id": "settings",
|
||||
"trusted": false,
|
||||
"type": "file",
|
||||
"path": "${dir:state}/settings.json",
|
||||
"mode": "0600",
|
||||
"owner": "mesh-build-image",
|
||||
"takes-config": ["capacity"],
|
||||
"merge": "json",
|
||||
"template": "files/settings.json"
|
||||
},
|
||||
{
|
||||
"id": "work",
|
||||
"type": "directory",
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// **The module pulls its seat's asks** (novox/hq ADR 0525 §2-§6, ADR 0536 §4; issue 1887 A3node), as mesh-build-go
|
||||
// does: the seat node-build-node, delivery first, then check, then recheck, at most the node's build capacity at once
|
||||
// (its setting `capacity`); each ask runs with the step bodies its tools run, behind the same guard, and ends on the key
|
||||
// <node>.<ask id>. The tools stay as the hand path (ADR 0525 §8).
|
||||
|
||||
// Seat is the node seat this module holds.
|
||||
const Seat = "node-build-node"
|
||||
|
||||
func (m *Module) askSteps(steps []StepTool) buildstep.AskSteps {
|
||||
byName := map[string]buildstep.StepFunc{}
|
||||
for _, s := range steps {
|
||||
run := s.Run
|
||||
byName[s.Name] = func(ctx context.Context, args map[string]any) (any, error) {
|
||||
defer m.holdArg(args)()
|
||||
return run(ctx, args)
|
||||
}
|
||||
}
|
||||
return buildstep.AskSteps{
|
||||
Fetch: byName["build_node_fetch"], Build: byName["build_node_build"], Test: byName["build_node_test"],
|
||||
Toolchain: func(ctx context.Context) (delivery.ToolchainUsed, error) {
|
||||
installed, err := m.Probe(ctx)
|
||||
if err != nil {
|
||||
return delivery.ToolchainUsed{}, err
|
||||
}
|
||||
return delivery.ToolchainUsed{Version: installed, Recipe: m.recipe().Version()}, nil
|
||||
},
|
||||
Release: func(ws string) error { return m.spaces().Release(ws) },
|
||||
Logf: m.Say,
|
||||
}
|
||||
}
|
||||
|
||||
// Puller is the module's pull of its seat's asks, on source (the runtime's seat pull when nil).
|
||||
func (m *Module) Puller(steps []StepTool, source buildstep.AskSource) *buildstep.Puller {
|
||||
if source == nil {
|
||||
source = buildstep.SeatAsks{Language: delivery.AskNode}
|
||||
}
|
||||
capacity := m.Capacity
|
||||
if capacity == nil {
|
||||
capacity = buildstep.NodeCapacity(m.Settings, buildstep.DefaultCapacities, m.Node, m.Say)
|
||||
}
|
||||
ask := m.askSteps(steps)
|
||||
var load func(string) ([]byte, error)
|
||||
if m.ReadSteps.Get != nil {
|
||||
load = func(key string) ([]byte, error) { return m.ReadSteps.Get(key) }
|
||||
}
|
||||
return &buildstep.Puller{Source: source, Language: delivery.AskNode, Node: m.Node, Put: m.Steps, Load: load,
|
||||
Capacity: capacity, Logf: m.Say,
|
||||
Run: func(ctx context.Context, a delivery.StepAsk) (any, error) {
|
||||
if err := m.Prepare(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := m.room(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return buildstep.RunAsk(ctx, ask, a)
|
||||
}}
|
||||
}
|
||||
|
||||
// SeatTools are the seat's verbs on this node (ADR 0525 §10).
|
||||
func (m *Module) SeatTools(p *buildstep.Puller) []stdio.Tool {
|
||||
return buildstep.SeatTools(Seat, "Node", p)
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
const askCommit = "0123456789abcdef0123456789abcdef01234567"
|
||||
|
||||
func fakeSteps(calls *[]string) []StepTool {
|
||||
answer := func(name string, out any) StepTool {
|
||||
return StepTool{Name: name, Run: func(ctx context.Context, args map[string]any) (any, error) {
|
||||
*calls = append(*calls, name)
|
||||
return out, nil
|
||||
}}
|
||||
}
|
||||
return []StepTool{
|
||||
answer("build_node_fetch", map[string]any{"workspace": "ws1", "modules": []string{"modules/gitea"}}),
|
||||
answer("build_node_build", map[string]any{"digest": "sha256:bb", "reference": "artifact-store://gitea/code@sha256:bb"}),
|
||||
answer("build_node_test", map[string]any{"passed": true}),
|
||||
}
|
||||
}
|
||||
|
||||
func nodeAsk(kind string) delivery.StepAsk {
|
||||
a := delivery.StepAsk{ID: buildstep.NewID(), Kind: kind, Language: delivery.AskNode, Asker: "mesh-delivery-checker", At: time.Now()}
|
||||
fetch := map[string]any{"repository": "novox/mesh-catalog", "commit": askCommit}
|
||||
if kind == delivery.AskDelivery {
|
||||
a.Asker = "mesh-delivery-builder"
|
||||
a.Build = &delivery.BuildJob{Module: "gitea", Path: "modules/gitea", Fetch: fetch, Artifacts: []delivery.AskArtifact{{Artifact: "code"}}}
|
||||
} else {
|
||||
a.Check = &delivery.CheckJob{Fetch: fetch, Levels: []string{delivery.LevelUnit}}
|
||||
}
|
||||
return a
|
||||
}
|
||||
|
||||
// 1887/A3node: a pulled build ask runs fetch then build and its key names the reference; a check ask tests each module
|
||||
// directory the fetch named.
|
||||
func TestAPulledNodeAskRunsItsJob(t *testing.T) {
|
||||
states := newStates()
|
||||
m := moduleWith(states, "v26.10.0")
|
||||
m.Capacity = func() int { return 1 }
|
||||
var calls []string
|
||||
src := buildstep.NewFakeAsks()
|
||||
build, check := nodeAsk(delivery.AskDelivery), nodeAsk(delivery.AskCheck)
|
||||
_ = src.PublishAsk(check)
|
||||
_ = src.PublishAsk(build)
|
||||
p := m.Puller(fakeSteps(&calls), src)
|
||||
p.PullWait = 10 * time.Millisecond
|
||||
ctx, stop := context.WithCancel(context.Background())
|
||||
defer stop()
|
||||
go p.Pull(ctx)
|
||||
for deadline := time.Now().Add(3 * time.Second); ; time.Sleep(5 * time.Millisecond) {
|
||||
if acked, _, _ := src.Fates(); len(acked) == 2 {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("the asks were not both acked: calls %v", calls)
|
||||
}
|
||||
}
|
||||
if got := strings.Join(calls, ","); got != "build_node_fetch,build_node_build,build_node_fetch,build_node_test" {
|
||||
t.Errorf("calls = %s, want the delivery ask's job first, then the check's", got)
|
||||
}
|
||||
key, _ := delivery.AskStepKey("ace", build.ID)
|
||||
states.mu.Lock()
|
||||
v := states.steps[key]
|
||||
states.mu.Unlock()
|
||||
if len(v) == 0 || !strings.Contains(string(v[len(v)-1].Result), "artifact-store://gitea/code@sha256:bb") {
|
||||
t.Fatalf("the step %s = %v", key, v)
|
||||
}
|
||||
}
|
||||
|
||||
// 1887/A3node, A7: the manifest takes the node's build capacity as a setting, with the builder's default, in a settings
|
||||
// file; the seat's verbs are served.
|
||||
func TestManifestTakesTheBuildCapacity(t *testing.T) {
|
||||
m := readManifest(t)
|
||||
raw, _ := os.ReadFile(moduleDir + "/module.json")
|
||||
var doc struct {
|
||||
Config map[string]struct {
|
||||
Default string `json:"default"`
|
||||
} `json:"config"`
|
||||
}
|
||||
_ = json.Unmarshal(raw, &doc)
|
||||
if doc.Config[buildstep.CapacitySetting].Default != buildstep.DefaultCapacities {
|
||||
t.Errorf("the capacity's default = %q", doc.Config[buildstep.CapacitySetting].Default)
|
||||
}
|
||||
if resource(t, m, "settings")["path"] != "${dir:state}/settings.json" {
|
||||
t.Error("no settings file")
|
||||
}
|
||||
m0 := &Module{Node: "ace", Capacity: func() int { return 0 }}
|
||||
served := map[string]bool{}
|
||||
for _, tool := range m0.SeatTools(m0.Puller(nil, buildstep.NewFakeAsks())) {
|
||||
served[tool.Name] = true
|
||||
}
|
||||
for _, verb := range []string{"current", "pause", "resume"} {
|
||||
if !served[Seat+"."+verb] {
|
||||
t.Errorf("%s.%s is not served", Seat, verb)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -55,6 +55,7 @@ func main() {
|
||||
RegistryFile: filepath.Join(home, "registry"),
|
||||
}
|
||||
m.Floor = floorOf(m.Node)
|
||||
m.Settings = filepath.Join(home, "settings.json")
|
||||
// Outside a launch by the runtime — a build, a check — there is no node and no state to write.
|
||||
if m.Node != "" {
|
||||
go m.Start(ctx)
|
||||
@@ -65,7 +66,13 @@ func main() {
|
||||
logf("announcing the states it keeps: %v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if err := stdio.Serve("", m.Tools(m.StepTools())); err != nil {
|
||||
steps := m.StepTools()
|
||||
puller := m.Puller(steps, nil)
|
||||
if m.Node != "" {
|
||||
// The seat's asks, pulled once Serve has the runtime's channel; a pull before it is said and tried again.
|
||||
go func() { _ = puller.Pull(ctx) }()
|
||||
}
|
||||
if err := stdio.Serve("", append(m.Tools(steps), m.SeatTools(puller)...)); err != nil {
|
||||
logf("%v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
@@ -86,9 +86,10 @@ func TestWorkAndCacheDirectoriesLieOutsideTheHome(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// The states are the three the module keeps, no seat is claimed (ADR 0469 §6), and the health is the module's own
|
||||
// tool, which the manifest also lists among its tools.
|
||||
func TestManifestKeepsItsStatesClaimsNoSeatAndDeclaresItsHealthTool(t *testing.T) {
|
||||
// The states are the three the module keeps, the one seat it claims is its language's build seat (ADR 0525 §2, which
|
||||
// changes ADR 0469 §6's "no seat"), and the health is the module's own tool, which the manifest also lists among its
|
||||
// tools.
|
||||
func TestManifestKeepsItsStatesClaimsItsBuildSeatAndDeclaresItsHealthTool(t *testing.T) {
|
||||
m := readManifest(t)
|
||||
states := []string{}
|
||||
for _, s := range m.State {
|
||||
@@ -98,8 +99,8 @@ func TestManifestKeepsItsStatesClaimsNoSeatAndDeclaresItsHealthTool(t *testing.T
|
||||
if want := []string{"keys", "steps", "toolchains"}; !slices.Equal(states, want) {
|
||||
t.Errorf("state = %v, want %v", states, want)
|
||||
}
|
||||
if len(m.Claims) != 0 {
|
||||
t.Errorf("claims = %v, want no seat", m.Claims)
|
||||
if len(m.Claims) != 1 || m.Claims[0].Name != Seat || !slices.Equal(m.Claims[0].Serves, []string{"current", "pause", "resume"}) {
|
||||
t.Errorf("claims = %v, want the seat %s alone", m.Claims, Seat)
|
||||
}
|
||||
if !slices.Contains(m.Tools, HealthTool) {
|
||||
t.Errorf("tools = %v lack %s", m.Tools, HealthTool)
|
||||
|
||||
@@ -63,6 +63,11 @@ type Module struct {
|
||||
// sets floorOf(node), and a zero Floor checks nothing.
|
||||
Floor buildstep.Floor
|
||||
|
||||
// Settings is the module's settings file, which holds the node's build capacity (buildstep.CapacitySetting); Capacity,
|
||||
// when set, answers it instead (a test).
|
||||
Settings string
|
||||
Capacity func() int
|
||||
|
||||
spacesOnce sync.Once
|
||||
workspaces *buildstep.Workspaces
|
||||
mu sync.Mutex // guards Recipe
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
{
|
||||
"capacity": "shanks=3,ace=2,g14=0,novox=0"
|
||||
}
|
||||
@@ -12,6 +12,21 @@
|
||||
"keys",
|
||||
"steps"
|
||||
],
|
||||
"claims": [
|
||||
{
|
||||
"name": "node-build-node",
|
||||
"scope": "node",
|
||||
"serves": ["current", "pause", "resume"]
|
||||
}
|
||||
],
|
||||
"config": {
|
||||
"capacity": {
|
||||
"type": "string",
|
||||
"description": "How many asks of the Node build seat this module runs at once on each node, as node=asks pairs separated by commas, such as shanks=3,ace=2 (novox/hq ADR 0525 §5); a node set to 0 pulls nothing, a node not named runs one at a time. The control-node and the laptop build nothing.",
|
||||
"pattern": "|[a-z0-9_-]+=[0-9]+(,[a-z0-9_-]+=[0-9]+)*",
|
||||
"default": "shanks=3,ace=2,g14=0,novox=0"
|
||||
}
|
||||
},
|
||||
"tools": [
|
||||
"build_node_fetch",
|
||||
"build_node_test",
|
||||
@@ -76,6 +91,17 @@
|
||||
"place": ".",
|
||||
"unknown": "program"
|
||||
},
|
||||
{
|
||||
"id": "settings",
|
||||
"trusted": false,
|
||||
"type": "file",
|
||||
"path": "${dir:state}/settings.json",
|
||||
"mode": "0600",
|
||||
"owner": "mesh-build-node",
|
||||
"takes-config": ["capacity"],
|
||||
"merge": "json",
|
||||
"template": "files/settings.json"
|
||||
},
|
||||
{
|
||||
"id": "work",
|
||||
"type": "directory",
|
||||
|
||||
@@ -47,12 +47,8 @@ export interface Built {
|
||||
|
||||
/** The events a build outcome reaches this module on: the build seat's, as it happens, and the control plane's replay. */
|
||||
export const BUILD_EVENTS = [
|
||||
// **The build seat that builds now** (novox/hq ADR 0190). Until issue 712 the catalogue heard only the
|
||||
// retired seat below, so every build after the handover reached the graph only through the replay this
|
||||
// module asks for when it starts: a merge built and pushed stayed out of the catalogue until its next start.
|
||||
"node-build-agent.built",
|
||||
// The retired build role, while a machine still serves it (ADR 0190's handover). Goes with the seat row.
|
||||
"mesh-build-machine.built",
|
||||
// The build seat node-build-agent and the retired mesh-build-machine went with build-agent (novox/hq ADR 0524 §5,
|
||||
// ADR 0525 §10): no build seat says `built` since R4g, so what the mesh holds is asked when this module starts.
|
||||
// What the mesh already held, when this module asks what it missed (novox/hq ADR 0134).
|
||||
"mesh-controller.built-before",
|
||||
] as const;
|
||||
|
||||
@@ -26,8 +26,6 @@
|
||||
"postgres-database": "${dir:state}/database.secret"
|
||||
},
|
||||
"consumes": [
|
||||
"node-build-agent.built",
|
||||
"mesh-build-machine.built",
|
||||
"mesh-controller.built-before"
|
||||
],
|
||||
"emits": [
|
||||
|
||||
@@ -20,8 +20,10 @@ const walkBuild = {
|
||||
made: [{ name: "code", kind: "bundle", reference: "artifact-store://mailu/code/blobs/sha256:d041" }],
|
||||
};
|
||||
|
||||
test("the catalogue hears the build seat that builds now, not only the retired one (issue 712)", async () => {
|
||||
assert.ok(BUILD_EVENTS.includes("node-build-agent.built"));
|
||||
test("the catalogue hears no retired build seat, and what the mesh held (issue 712, issue 1887 E2)", async () => {
|
||||
for (const retired of ["node-build-agent.built", "mesh-build-machine.built"]) {
|
||||
assert.ok(!(BUILD_EVENTS as readonly string[]).includes(retired), `${retired} is still heard`);
|
||||
}
|
||||
assert.ok(BUILD_EVENTS.includes("mesh-controller.built-before"));
|
||||
// Subscribing is not enough: the mesh grants and routes only what the manifest consumes.
|
||||
const manifest = JSON.parse(await readFile(new URL("../module.json", import.meta.url), "utf8"));
|
||||
|
||||
@@ -96,9 +96,9 @@ func TestAConsumerWithSeveralFiltersIsHandedEveryMessage(t *testing.T) {
|
||||
FilterSubjects: []string{
|
||||
"mesh.mod.mesh-catalog.event.upgraded",
|
||||
"mesh.mod.mesh-catalog.event.catching-up",
|
||||
"mesh.seat.node-build-agent.event.built",
|
||||
"mesh.seat.node-backup.event.done",
|
||||
"mesh.mod.gitea.event.pull.merged",
|
||||
"mesh.seat.mesh-build-machine.event.built",
|
||||
"mesh.seat.node-power.event.slept",
|
||||
"mesh.mod.*.event.provisioner.failing",
|
||||
"mesh.mod.*.event.provisioner.recovered",
|
||||
},
|
||||
@@ -128,7 +128,7 @@ func TestAConsumerWithSeveralFiltersIsHandedEveryMessage(t *testing.T) {
|
||||
}
|
||||
}()
|
||||
|
||||
followed := []string{"mesh.mod.gitea.event.pull.merged", "mesh.seat.node-build-agent.event.built",
|
||||
followed := []string{"mesh.mod.gitea.event.pull.merged", "mesh.seat.node-backup.event.done",
|
||||
"mesh.mod.postgres.event.provisioner.recovered", "mesh.mod.mesh-catalog.event.catching-up"}
|
||||
var mu sync.Mutex
|
||||
var sent []uint64
|
||||
@@ -153,7 +153,7 @@ func TestAConsumerWithSeveralFiltersIsHandedEveryMessage(t *testing.T) {
|
||||
mu.Unlock()
|
||||
}
|
||||
} else {
|
||||
_, _ = pjs.Publish(fmt.Sprintf("mesh.seat.node-build-agent.event.log.build-%d-%d", w, i/50), make([]byte, 200))
|
||||
_, _ = pjs.Publish(fmt.Sprintf("mesh.seat.node-backup.event.log.run-%d-%d", w, i/50), make([]byte, 200))
|
||||
}
|
||||
if rand.Intn(100) == 0 {
|
||||
time.Sleep(time.Duration(rand.Intn(300)) * time.Millisecond)
|
||||
|
||||
Reference in New Issue
Block a user