Merge main into feat/r4d: keep the image check beside the image builder's two checks (ADR 0487 clause 13)
mesh/merge-gate waits for its group's composed check
mesh/repo-check head 7451e7edf4e5 merged into trunk 529c156f6407 beside mesh-controller#433@499ff5c2 — pass: THE CHANGE ALTERS ITS OWN CHECK (merge-check.s…
mesh/delivery superseded: a newer head of the same pull request
mesh/check failed: unit: modules/gitea.test: ✔ the same merge, head or close announced twice has the same id, and another has another (6.449977ms) | 7…
mesh/merge-gate waits for its group's composed check
mesh/repo-check head 7451e7edf4e5 merged into trunk 529c156f6407 beside mesh-controller#433@499ff5c2 — pass: THE CHANGE ALTERS ITS OWN CHECK (merge-check.s…
mesh/delivery superseded: a newer head of the same pull request
mesh/check failed: unit: modules/gitea.test: ✔ the same merge, head or close announced twice has the same id, and another has another (6.449977ms) | 7…
This commit is contained in:
@@ -27,3 +27,4 @@ modules/mesh-provisioner-client/cmd/mesh-provisioner-client/mesh-provisioner-cli
|
||||
modules/brother/cmd/brother/brother
|
||||
modules/mesh-build-go/cmd/mesh-build-go/mesh-build-go
|
||||
modules/mesh-config/mesh-config
|
||||
modules/mesh-build-image/cmd/mesh-build-image/mesh-build-image
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
module git.novox.be/novox/mesh-catalog/checks/imagebuild
|
||||
|
||||
go 1.22
|
||||
@@ -0,0 +1,92 @@
|
||||
// Command imagebuild refuses any manifest but modules/buildah providing the rootless-image-build provision
|
||||
// (novox/hq ADR 0487 §3, issue 1704): the image builder's tool is buildah alone, and docker, whose daemon holds
|
||||
// the machine's root, never stands in for it. A second provider would let a node compose mesh-build-image over
|
||||
// something that is not the rootless builder it was designed against.
|
||||
//
|
||||
// go run . <catalogue root>
|
||||
//
|
||||
// Exits 1 on a finding.
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
)
|
||||
|
||||
// Provision is the provision only Provider may provide.
|
||||
const Provision = "rootless-image-build"
|
||||
|
||||
// Provider is the one module allowed to provide it.
|
||||
const Provider = "buildah"
|
||||
|
||||
// Finding is one manifest refused: a module other than buildah providing the provision.
|
||||
type Finding struct {
|
||||
Module string
|
||||
}
|
||||
|
||||
// Check reads every manifest under root/modules and answers which ones it refuses, and whether buildah provides.
|
||||
func Check(root string) ([]Finding, error) {
|
||||
manifests, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sort.Strings(manifests)
|
||||
var out []Finding
|
||||
provided := false
|
||||
for _, path := range manifests {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var m struct {
|
||||
Module string `json:"module"`
|
||||
Provides []struct {
|
||||
Name string `json:"name"`
|
||||
} `json:"provides"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", path, err)
|
||||
}
|
||||
for _, p := range m.Provides {
|
||||
if p.Name != Provision {
|
||||
continue
|
||||
}
|
||||
if m.Module == Provider {
|
||||
provided = true
|
||||
} else {
|
||||
out = append(out, Finding{Module: m.Module})
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
if !provided {
|
||||
out = append(out, Finding{Module: Provider})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func main() {
|
||||
root := "."
|
||||
if len(os.Args) > 1 {
|
||||
root = os.Args[1]
|
||||
}
|
||||
found, err := Check(root)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
for _, f := range found {
|
||||
if f.Module == Provider {
|
||||
fmt.Printf("%s does not provide %s (novox/hq ADR 0487 §3)\n", Provider, Provision)
|
||||
continue
|
||||
}
|
||||
fmt.Printf("%s provides %s: only %s may, and docker never (novox/hq ADR 0487 §3)\n", f.Module, Provision, Provider)
|
||||
}
|
||||
if len(found) > 0 {
|
||||
os.Exit(1)
|
||||
}
|
||||
fmt.Println("imagebuild: rootless-image-build is provided only by buildah")
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func catalogue(t *testing.T, manifests map[string]string) string {
|
||||
t.Helper()
|
||||
root := t.TempDir()
|
||||
for name, body := range manifests {
|
||||
dir := filepath.Join(root, "modules", name)
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "module.json"), []byte(body), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return root
|
||||
}
|
||||
|
||||
const buildah = `{"module":"buildah","provides":[{"name":"rootless-image-build","reach":"machine"}]}`
|
||||
|
||||
func TestOnlyBuildahProvidesTheRootlessImageBuild(t *testing.T) {
|
||||
root := catalogue(t, map[string]string{"buildah": buildah, "go": `{"module":"go","provides":[{"name":"go-toolchain"}]}`})
|
||||
got, err := Check(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != 0 {
|
||||
t.Errorf("found %v, want none", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDockerAndAnySecondProviderAreRefused(t *testing.T) {
|
||||
root := catalogue(t, map[string]string{
|
||||
"buildah": buildah,
|
||||
"docker": `{"module":"docker","provides":[{"name":"rootless-image-build"}]}`,
|
||||
"other": `{"module":"other","provides":[{"name":"x"},{"name":"rootless-image-build"}]}`,
|
||||
})
|
||||
got, err := Check(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := []Finding{{Module: "docker"}, {Module: "other"}}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("found %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildahMustProvideIt(t *testing.T) {
|
||||
root := catalogue(t, map[string]string{"buildah": `{"module":"buildah"}`})
|
||||
got, err := Check(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if want := []Finding{{Module: "buildah"}}; !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("found %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// The catalogue as it stands: this is the check itself, run here too.
|
||||
func TestTheCatalogueGivesTheProvisionToBuildahAlone(t *testing.T) {
|
||||
got, err := Check("../..")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != 0 {
|
||||
t.Errorf("the catalogue's providers of rootless-image-build are wrong: %v", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
module git.novox.be/novox/mesh-catalog/checks/imageplace
|
||||
|
||||
go 1.22
|
||||
@@ -0,0 +1,91 @@
|
||||
// Command imageplace refuses an image artifact whose Dockerfile is not a file inside its own module's directory
|
||||
// (novox/hq ADR 0487 §1, issue 1707): the image builder builds a module from that module's directory alone, so
|
||||
// a `from` that leaves it (../other/Dockerfile) or names a file that is not there could never build.
|
||||
//
|
||||
// go run . <catalogue root>
|
||||
//
|
||||
// Exits 1 on a finding.
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Finding is one image refused: Module's artifact Artifact names From, and Why says what is wrong with it.
|
||||
type Finding struct {
|
||||
Module, Artifact, From, Why string
|
||||
}
|
||||
|
||||
// Check reads every manifest under root/modules and answers which images it refuses.
|
||||
func Check(root string) ([]Finding, error) {
|
||||
manifests, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sort.Strings(manifests)
|
||||
var out []Finding
|
||||
for _, path := range manifests {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var m struct {
|
||||
Module string `json:"module"`
|
||||
Build struct {
|
||||
Artifacts []struct {
|
||||
Name string `json:"name"`
|
||||
Kind string `json:"kind"`
|
||||
From string `json:"from"`
|
||||
} `json:"artifacts"`
|
||||
} `json:"build"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", path, err)
|
||||
}
|
||||
dir := filepath.Dir(path)
|
||||
for _, a := range m.Build.Artifacts {
|
||||
if a.Kind != "image" {
|
||||
continue
|
||||
}
|
||||
f := Finding{Module: m.Module, Artifact: a.Name, From: a.From}
|
||||
clean := filepath.Clean(a.From)
|
||||
switch {
|
||||
case a.From == "" || filepath.IsAbs(clean) || clean == ".." || strings.HasPrefix(clean, ".."+string(filepath.Separator)):
|
||||
f.Why = "is not a path inside the module's own directory"
|
||||
default:
|
||||
info, err := os.Stat(filepath.Join(dir, clean))
|
||||
if err != nil || info.IsDir() {
|
||||
f.Why = "is not a file in the module's directory"
|
||||
}
|
||||
}
|
||||
if f.Why != "" {
|
||||
out = append(out, f)
|
||||
}
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func main() {
|
||||
root := "."
|
||||
if len(os.Args) > 1 {
|
||||
root = os.Args[1]
|
||||
}
|
||||
found, err := Check(root)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
for _, f := range found {
|
||||
fmt.Printf("%s: image %s builds from %q, which %s (novox/hq ADR 0487 §1)\n", f.Module, f.Artifact, f.From, f.Why)
|
||||
}
|
||||
if len(found) > 0 {
|
||||
os.Exit(1)
|
||||
}
|
||||
fmt.Println("imageplace: every image builds from a Dockerfile in its own module's directory")
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// module writes modules/<name>/module.json with one image artifact from `from`, and the files named.
|
||||
func module(t *testing.T, root, name, from string, files ...string) {
|
||||
t.Helper()
|
||||
dir := filepath.Join(root, "modules", name)
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body := `{"module":"` + name + `","build":{"artifacts":[{"name":"server","kind":"image","from":"` + from + `"},{"name":"files","kind":"archive","from":"../elsewhere"}]}}`
|
||||
if err := os.WriteFile(filepath.Join(dir, "module.json"), []byte(body), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, f := range files {
|
||||
p := filepath.Join(dir, f)
|
||||
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(p, []byte("FROM scratch\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestADockerfileInTheModulesOwnDirectoryPasses(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
module(t, root, "own", "Dockerfile", "Dockerfile")
|
||||
module(t, root, "nested", "image/Dockerfile", "image/Dockerfile")
|
||||
got, err := Check(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != 0 {
|
||||
t.Errorf("found %v, want none", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestADockerfileOutsideTheModulesDirectoryFails(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
module(t, root, "other", "Dockerfile", "Dockerfile")
|
||||
module(t, root, "leaves", "../other/Dockerfile")
|
||||
module(t, root, "absolute", "/etc/Dockerfile")
|
||||
got, err := Check(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var mods []string
|
||||
for _, f := range got {
|
||||
mods = append(mods, f.Module)
|
||||
}
|
||||
if want := []string{"absolute", "leaves"}; !reflect.DeepEqual(mods, want) {
|
||||
t.Errorf("refused %v (%v), want %v", mods, got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAMissingDockerfileFails(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
module(t, root, "gone", "Dockerfile")
|
||||
got, err := Check(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != 1 || got[0].Module != "gone" {
|
||||
t.Errorf("found %v, want gone refused", got)
|
||||
}
|
||||
}
|
||||
|
||||
// The catalogue as it stands: this is the check itself, run here too.
|
||||
func TestTheCataloguesImagesBuildFromTheirOwnDirectories(t *testing.T) {
|
||||
got, err := Check("../..")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != 0 {
|
||||
t.Errorf("an image builds from outside its module: %v", got)
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
// Command toolchains refuses any manifest but the owner's declaring a language's toolchain package (novox/hq
|
||||
// ADR 0477 clause 4, issue 1573): the package go only in modules/go, and nodejs and npm only in modules/nodejs.
|
||||
// ADR 0477 clause 4, issue 1573): the package go only in modules/go, nodejs and npm only in modules/nodejs, and buildah (issue 1702) only in modules/buildah.
|
||||
// Everything that compiles or runs that language requires the owner's provision instead (go-toolchain,
|
||||
// node-toolchain), so the package is declared once, and a node without the owner refuses the module that
|
||||
// needs it at resolution rather than failing at its first build. The owners must declare their packages.
|
||||
@@ -19,9 +19,10 @@ import (
|
||||
|
||||
// Owners names, for each toolchain package, the one module that declares it.
|
||||
var Owners = map[string]string{
|
||||
"go": "go",
|
||||
"nodejs": "nodejs",
|
||||
"npm": "nodejs",
|
||||
"buildah": "buildah",
|
||||
"go": "go",
|
||||
"nodejs": "nodejs",
|
||||
"npm": "nodejs",
|
||||
}
|
||||
|
||||
// Finding is one refusal: Module declares Package, which Owner alone may, or (Missing) Owner does not declare
|
||||
@@ -100,5 +101,5 @@ func main() {
|
||||
if len(found) > 0 {
|
||||
os.Exit(1)
|
||||
}
|
||||
fmt.Println("toolchains: go, nodejs and npm are declared only by their owners")
|
||||
fmt.Println("toolchains: go, nodejs, npm and buildah are declared only by their owners")
|
||||
}
|
||||
|
||||
@@ -26,6 +26,7 @@ func TestOnlyTheOwnerDeclaresAToolchainPackage(t *testing.T) {
|
||||
root := catalogue(t, map[string]string{
|
||||
"go": `{"module":"go","resources":[{"id":"go","type":"package","package":"go"}]}`,
|
||||
"nodejs": `{"module":"nodejs","resources":[{"id":"nodejs","type":"package","package":"nodejs"},{"id":"npm","type":"package","package":"npm"}]}`,
|
||||
"buildah": `{"module":"buildah","resources":[{"id":"b","type":"package","package":"buildah"}]}`,
|
||||
"lab": `{"module":"lab","resources":[{"id":"git","type":"package","package":"git"},{"id":"go","type":"package","package":"go"}]}`,
|
||||
"builder": `{"module":"builder","resources":[{"id":"n","type":"package","package":"nodejs"},{"id":"dir","type":"directory","path":"/go"}]}`,
|
||||
})
|
||||
@@ -39,6 +40,23 @@ func TestOnlyTheOwnerDeclaresAToolchainPackage(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestASecondDeclarerOfBuildahIsRefused(t *testing.T) {
|
||||
root := catalogue(t, map[string]string{
|
||||
"go": `{"module":"go","resources":[{"id":"go","type":"package","package":"go"}]}`,
|
||||
"nodejs": `{"module":"nodejs","resources":[{"id":"nodejs","type":"package","package":"nodejs"},{"id":"npm","type":"package","package":"npm"}]}`,
|
||||
"buildah": `{"module":"buildah","resources":[{"id":"b","type":"package","package":"buildah"}]}`,
|
||||
"imagery": `{"module":"imagery","resources":[{"id":"b","type":"package","package":"buildah"}]}`,
|
||||
})
|
||||
got, err := Check(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := []Finding{{Module: "imagery", Package: "buildah", Owner: "buildah"}}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("found %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheOwnersMustExistAndDeclareTheirPackages(t *testing.T) {
|
||||
root := catalogue(t, map[string]string{
|
||||
"go": `{"module":"go","resources":[]}`,
|
||||
@@ -47,8 +65,8 @@ func TestTheOwnersMustExistAndDeclareTheirPackages(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != 3 {
|
||||
t.Errorf("an owner missing its packages gave %v, want three findings (go, nodejs, npm)", got)
|
||||
if len(got) != 4 {
|
||||
t.Errorf("an owner missing its packages gave %v, want four findings (buildah, go, nodejs, npm)", got)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -50,6 +50,8 @@
|
||||
# every directory and archive says `unknown` (move-out, module or program) and a valid `knows`; `move-out`
|
||||
# only under a home; and no module declares a path inside a directory of another module.
|
||||
# checks/sshdir (ADR 0415 §6) stays while main runs it; it goes under issue 1035 once main runs checks/owned.
|
||||
# 12. the image builder (novox/hq ADR 0487 §3, issue 1704): only modules/buildah provides rootless-image-build.
|
||||
# 13. the image's place (novox/hq ADR 0487 §1, issue 1707): every image builds from a Dockerfile in its own module's directory.
|
||||
set -eu
|
||||
|
||||
(cd checks/words && go test -count=1 . && go run . ../..)
|
||||
@@ -57,6 +59,10 @@ set -eu
|
||||
(cd checks/toolchains && go test -count=1 . && go run . ../..)
|
||||
# The image (novox/hq ADR 0480 clause 11): no manifest but build-agent's declares an image built from a Dockerfile.
|
||||
(cd checks/images && go test -count=1 . && go run . ../..)
|
||||
# The image builder (novox/hq ADR 0487 §3): only modules/buildah provides rootless-image-build, docker never.
|
||||
(cd checks/imagebuild && go test -count=1 . && go run . ../..)
|
||||
# The image's place (novox/hq ADR 0487 §1): every image builds from a Dockerfile in its own module's directory.
|
||||
(cd checks/imageplace && go test -count=1 . && go run . ../..)
|
||||
# The agent's home (novox/hq ADR 0266): no archive, and no file written into, under it.
|
||||
(cd checks/agenthome && go test -count=1 . && go run . ../..)
|
||||
# The mail-store seat (novox/hq ADR 0378 §3): only mesh-mail may claim it.
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
# buildah
|
||||
|
||||
The rootless image builder as a module (novox/hq ADR 0487, issue 1702).
|
||||
|
||||
- **The only module that declares the package `buildah`** (`checks/toolchains`), and the only one that provides
|
||||
`rootless-image-build` (`checks/imagebuild`; docker never does).
|
||||
- **Provides `rootless-image-build` with the machine's reach**: mesh-build-image requires it, so it composes only on a
|
||||
machine where this module is assigned, and a node without it is refused at resolution naming `buildah`.
|
||||
- **No account and no subordinate-id word.** The node-engine makes every `user` account with `useradd`, which
|
||||
allocates the account 65536 subordinate uids and gids; mesh-build-image's own account gets its ranges that way and
|
||||
its health proves them (it reads /etc/subuid and /etc/subgid).
|
||||
- The helpers (an OCI runtime, passt, containers-common) come as buildah's package dependencies.
|
||||
- Holds no long-running resource, so it adds nothing to `health-undeclared`.
|
||||
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"module": "buildah",
|
||||
"version": "1",
|
||||
"capabilities": [
|
||||
"package-manager",
|
||||
"user-namespaces"
|
||||
],
|
||||
"provides": [
|
||||
{
|
||||
"name": "rootless-image-build",
|
||||
"reach": "machine"
|
||||
}
|
||||
],
|
||||
"resources": [
|
||||
{
|
||||
"id": "buildah",
|
||||
"type": "package",
|
||||
"package": "buildah"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -179,7 +179,7 @@ func newTeamRig(t *testing.T) *teamRig {
|
||||
svc := NewService(Config{Node: "novox", HostFile: host, OriginsFile: origins, Port: 222,
|
||||
Forge: forgehttp.Client{URL: srv.URL, User: "mesh-admin", PasswordFile: pw}, Runner: runner, Log: func(string, ...any) {}})
|
||||
svc.loaded = true
|
||||
svc.loadedFrom["mesh-build-go"], svc.loadedFrom["mesh-build-node"] = true, true
|
||||
svc.loadedFrom["mesh-build-go"], svc.loadedFrom["mesh-build-node"], svc.loadedFrom["mesh-build-image"] = true, true, true
|
||||
return &teamRig{t: t, bus: newBus(), forge: forge, svc: svc, runner: runner}
|
||||
}
|
||||
|
||||
@@ -347,7 +347,7 @@ func TestABuildModulesEntryWaitsForItsOwnBuilderKey(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestTheKindOfAModulesKeyAndItsForgeUser(t *testing.T) {
|
||||
for module, want := range map[string]string{"mesh-build-go": "nox-builder", "mesh-build-node": "nox-builder", "ssh-client": "nox-agent", "anything-else": "nox-agent"} {
|
||||
for module, want := range map[string]string{"mesh-build-go": "nox-builder", "mesh-build-node": "nox-builder", "mesh-build-image": "nox-builder", "ssh-client": "nox-agent", "anything-else": "nox-agent"} {
|
||||
got, err := (&Service{}).valuesForUser("ace", module)
|
||||
if err != nil || got != want {
|
||||
t.Errorf("%s: %q, %v; want %q", module, got, err, want)
|
||||
|
||||
@@ -38,7 +38,7 @@ func newWatched(t *testing.T) *watched {
|
||||
r.svc.OnKeys = func(string) { n++ }
|
||||
r.svc.cfg.Log = func(f string, a ...any) { mu.Lock(); logs = append(logs, fmt.Sprintf(f, a...)); mu.Unlock() }
|
||||
w := &watched{teamRig: r, sources: map[string]*fakeKeys{}, passes: &n, logs: &logs, mu: mu}
|
||||
for _, module := range []string{"ssh-client", "mesh-build-go", "mesh-build-node"} {
|
||||
for _, module := range []string{"ssh-client", "mesh-build-go", "mesh-build-node", "mesh-build-image"} {
|
||||
w.sources[module] = &fakeKeys{}
|
||||
if err := r.svc.WatchSource(context.Background(), module, w.sources[module]); err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -154,7 +154,7 @@ func TestTheManifestReadsTheBuildModulesKeys(t *testing.T) {
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, state := range []string{"mesh-controller.assignments", "ssh-client.keys", "mesh-build-go.keys", "mesh-build-node.keys"} {
|
||||
for _, state := range []string{"mesh-controller.assignments", "ssh-client.keys", "mesh-build-go.keys", "mesh-build-node.keys", "mesh-build-image.keys"} {
|
||||
if !slices.Contains(m.Reads, state) {
|
||||
t.Errorf("gitea's reads %v lack %q", m.Reads, state)
|
||||
}
|
||||
|
||||
@@ -195,7 +195,7 @@ const BuilderUser = "nox-builder"
|
||||
|
||||
// BuildModules are the modules whose keys are the builders' (kind builder), by their name, which is also the name of
|
||||
// the account each runs as (ADR 0466 §1).
|
||||
var BuildModules = []string{"mesh-build-go", "mesh-build-node"}
|
||||
var BuildModules = []string{"mesh-build-go", "mesh-build-node", "mesh-build-image"}
|
||||
|
||||
// KindOfModule is the kind of key a consumer module's entry is made for: builder for a build module, machine for any
|
||||
// other (ssh-client's).
|
||||
|
||||
@@ -12,7 +12,7 @@ func TestTheBuildModulesHaveOneForgeUserAndTheMachinesKeepTheirs(t *testing.T) {
|
||||
if ForgeUser("Not A Node", "builder") != "" {
|
||||
t.Errorf("a name no account could be named for got a user")
|
||||
}
|
||||
for module, want := range map[string]string{"mesh-build-go": "builder", "mesh-build-node": "builder", "ssh-client": "machine", "": "machine"} {
|
||||
for module, want := range map[string]string{"mesh-build-go": "builder", "mesh-build-node": "builder", "mesh-build-image": "builder", "ssh-client": "machine", "": "machine"} {
|
||||
if got := KindOfModule(module); got != want {
|
||||
t.Errorf("KindOfModule(%q) = %s, want %s", module, got, want)
|
||||
}
|
||||
|
||||
@@ -542,6 +542,7 @@
|
||||
"mesh-controller.assignments",
|
||||
"ssh-client.keys",
|
||||
"mesh-build-go.keys",
|
||||
"mesh-build-node.keys"
|
||||
"mesh-build-node.keys",
|
||||
"mesh-build-image.keys"
|
||||
]
|
||||
}
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
# mesh-build-image — the image build module
|
||||
|
||||
Design 47, release R4c-9 (novox/hq ADR 0487, ADR 0466 §1, ADR 0469; issue 1703). The module that builds a module's
|
||||
`image` artifact from the Dockerfile in the module's own directory, on a build machine, as an account of its own
|
||||
(`runs-as`, no login shell, home at mode 0700). It serves every module's image, whatever the base, so it is not a
|
||||
build module for one module. It claims no seat: it is assigned to the build machines and runs dark until the builder
|
||||
uses it.
|
||||
|
||||
## Rootless
|
||||
|
||||
The tool is `buildah`, which the `buildah` module installs and provides as `rootless-image-build` with the machine's
|
||||
reach: a machine without it refuses this module at assignment, naming `buildah`. The build runs as the module's own
|
||||
account in a user namespace (`--isolation oci`), with buildah's image store and runtime state in
|
||||
`/var/lib/mesh-build-image-store` (outside the home), no mount of the host, no container runtime socket, no registry
|
||||
credential and an environment built from scratch.
|
||||
|
||||
The account's subordinate user and group ranges are not declared anywhere: the node-engine makes every `user` account
|
||||
with `useradd`, which allocates them. The module's health reads `/etc/subuid` and `/etc/subgid` and is unhealthy
|
||||
without a range for the account, without buildah, or when a user namespace cannot be made.
|
||||
|
||||
The Dockerfile's steps may use the network (ADR 0487 §6); nothing a step can read holds a key.
|
||||
|
||||
## Steps
|
||||
|
||||
- `build_image_fetch` fetches a commit over SSH into a workspace (a base commit makes the merge preview, as the other
|
||||
build modules' fetch). Long: answers a step id and ends on its key in `steps`.
|
||||
- `build_image_build` builds one image artifact: `buildah bud` with the Dockerfile of the module's directory as file
|
||||
and context, `--target`, and the images `build.on` pins as `--build-arg`. Answers the image's digest. With
|
||||
`publish: true` (a merged trunk commit only) the module's own process then pushes it to the registry under
|
||||
`<module>/<artifact>:<tag of its own>` (the first 12 hex digits of the digest unless the call gives one), waiting out
|
||||
a registry held still, and the key carries the digest the registry serves and the reference pinned by it. Without
|
||||
`publish` (a pull request) nothing is pushed.
|
||||
- It refuses, naming why, an image whose build context is a whole repository (`context`) or whose `build.on` names
|
||||
another module's artifact (`unsupportedImage` in `build.go`): those forms belong to the images build-agent builds
|
||||
until it is removed at R4g.
|
||||
- `build_image_step`, `build_image_steps`, `build_image_status` and `build_image_health` read.
|
||||
|
||||
The registry's address is placed into `<home>/registry` as `${seat:mesh-artifact-store:reach}` (host:port): the provisioner's client fills a moved module's placement, and the artifact store has not moved, so a `${bound:artifact-store:...}` is refused at composition (issue 1703).
|
||||
@@ -0,0 +1,215 @@
|
||||
package main
|
||||
|
||||
// The build step (novox/hq ADR 0487 §1 §2 §5 §6; issue 1703c): one `image` artifact of a module of a fetched workspace,
|
||||
// built from the Dockerfile in the module's own directory with buildah, rootless: as the module's own account, in a
|
||||
// user namespace, with its image store outside the home, no mount of the host, no registry credential and an
|
||||
// environment built from scratch. The step runs the Dockerfile and nothing else; the push of a trunk build is made
|
||||
// after the build has ended (push.go).
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
)
|
||||
|
||||
// BuildResult is what a passed build step says: the image's manifest digest as buildah holds it, and, for a trunk
|
||||
// build that published, the digest the registry serves and the reference pinned by it.
|
||||
type BuildResult struct {
|
||||
Artifact string `json:"artifact"`
|
||||
Digest string `json:"digest"`
|
||||
Reference string `json:"reference,omitempty"`
|
||||
Pushed bool `json:"pushed"`
|
||||
}
|
||||
|
||||
var tagOK = regexp.MustCompile(`^[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}$`)
|
||||
|
||||
func (m *Module) buildTool() StepTool {
|
||||
return StepTool{
|
||||
Name: BuildTool,
|
||||
Description: "Build one image artifact of a module of a fetched workspace from the Dockerfile in the module's own directory, " +
|
||||
"rootless with buildah. Answers a step id at once; the step ends on its key with the image's digest. With publish " +
|
||||
"true (a merged trunk commit) the image is then pushed to the registry from this module's own process and the key " +
|
||||
"carries the digest the registry serves and the reference pinned by it; without it (a pull request) nothing is pushed.",
|
||||
Input: map[string]any{
|
||||
"type": "object",
|
||||
"properties": map[string]any{
|
||||
"workspace": map[string]any{"type": "string", "description": "the workspace id build_image_fetch answered"},
|
||||
"module": map[string]any{"type": "string", "description": "the module's directory in the workspace, such as modules/n8n"},
|
||||
"artifact": map[string]any{"type": "string", "description": "the image artifact's name in the module's manifest"},
|
||||
"publish": map[string]any{"type": "boolean", "description": "push the image to the registry after the build: only for a merged trunk commit"},
|
||||
"tag": map[string]any{"type": "string", "description": "the registry tag of a published image; the first 12 hex digits of the digest when left out"},
|
||||
},
|
||||
"required": []string{"workspace", "module", "artifact"},
|
||||
},
|
||||
Long: true,
|
||||
Run: m.build,
|
||||
}
|
||||
}
|
||||
|
||||
// unsupportedImage says why this module does not build the artifact, or "" when it does. It is the one place that
|
||||
// knows which image forms are out: ADR 0487 §1 takes the Dockerfile's directory as the build context and the base
|
||||
// from the Dockerfile or an image pinned in `build.on`, so a context that is another repository and a base that is
|
||||
// another module's artifact are not built (the lead's decision for R4c-9, which retires them with build-agent at R4g).
|
||||
func unsupportedImage(manifest catalogue.Manifest, a catalogue.Artifact) string {
|
||||
if a.Context != nil {
|
||||
return fmt.Sprintf("the image %s of %s builds with the repository %s as its context; mesh-build-image takes the Dockerfile's directory as the build context (ADR 0487 §1) and does not build a whole repository",
|
||||
a.Name, manifest.Module, a.Context.Repository)
|
||||
}
|
||||
if manifest.Build != nil {
|
||||
for _, on := range manifest.Build.On {
|
||||
if on.Module != "" {
|
||||
return fmt.Sprintf("the image %s of %s stands on the artifact %s of the module %s through the build argument %s; mesh-build-image builds only from the Dockerfile's directory and an image pinned in build.on (ADR 0487 §1)",
|
||||
a.Name, manifest.Module, on.Artifact, on.Module, on.Arg)
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// imageArgs are the --build-arg words of the images the manifest pins in build.on, in its order.
|
||||
func imageArgs(manifest catalogue.Manifest) []string {
|
||||
var out []string
|
||||
if manifest.Build == nil {
|
||||
return out
|
||||
}
|
||||
for _, on := range manifest.Build.On {
|
||||
if on.Image != "" && on.Arg != "" {
|
||||
out = append(out, "--build-arg", on.Arg+"="+on.Image)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (m *Module) build(ctx context.Context, args map[string]any) (any, error) {
|
||||
workspace, _ := args["workspace"].(string)
|
||||
module, _ := args["module"].(string)
|
||||
name, _ := args["artifact"].(string)
|
||||
publish, _ := args["publish"].(bool)
|
||||
tag, _ := args["tag"].(string)
|
||||
if !workspaceOK.MatchString(workspace) {
|
||||
return nil, fmt.Errorf("%q is not a workspace id", workspace)
|
||||
}
|
||||
if tag != "" && !tagOK.MatchString(tag) {
|
||||
return nil, fmt.Errorf("%q is not a registry tag", tag)
|
||||
}
|
||||
tree := filepath.Join(m.Work, workspace, "tree")
|
||||
dir := filepath.Join(tree, filepath.FromSlash(module))
|
||||
if rel, err := filepath.Rel(tree, dir); err != nil || strings.HasPrefix(rel, "..") {
|
||||
return nil, fmt.Errorf("the module directory %q lies outside the workspace", module)
|
||||
}
|
||||
raw, err := os.ReadFile(filepath.Join(dir, "module.json"))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("no module in %q of workspace %s: %w", module, workspace, err)
|
||||
}
|
||||
manifest, err := catalogue.ParseManifest(raw)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the manifest in %q: %w", module, err)
|
||||
}
|
||||
var art *catalogue.Artifact
|
||||
names := []string{}
|
||||
if manifest.Build != nil {
|
||||
for i := range manifest.Build.Artifacts {
|
||||
names = append(names, manifest.Build.Artifacts[i].Name)
|
||||
if manifest.Build.Artifacts[i].Name == name {
|
||||
art = &manifest.Build.Artifacts[i]
|
||||
}
|
||||
}
|
||||
}
|
||||
if art == nil {
|
||||
return nil, fmt.Errorf("the module %s has no artifact %q (it has %s)", manifest.Module, name, strings.Join(names, ", "))
|
||||
}
|
||||
if art.Kind != "image" {
|
||||
return nil, fmt.Errorf("the artifact %s of %s is a %s: this module builds images", name, manifest.Module, art.Kind)
|
||||
}
|
||||
if why := unsupportedImage(manifest, *art); why != "" {
|
||||
return nil, &buildstep.Refusal{Text: why}
|
||||
}
|
||||
dockerfile, contextDir, err := dockerfileOf(dir, art.From)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the image %s of %s: %w", name, manifest.Module, err)
|
||||
}
|
||||
|
||||
local := localName(workspace, manifest.Module, art.Name)
|
||||
argv := append(m.storeFlags(), "bud", "--isolation", "oci", "--tag", local, "--file", dockerfile)
|
||||
if art.Target != "" {
|
||||
argv = append(argv, "--target", art.Target)
|
||||
}
|
||||
argv = append(argv, imageArgs(manifest)...)
|
||||
argv = append(argv, contextDir)
|
||||
if r := m.Run(ctx, m.env(), "buildah", argv...); r.Status != 0 || r.Err != "" {
|
||||
return nil, fmt.Errorf("buildah bud of %s: %s", art.Name, lastLine(r.Err+"\n"+r.Stderr+"\n"+r.Stdout))
|
||||
}
|
||||
defer m.remove(local)
|
||||
digest, err := m.localDigest(ctx, local)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res := BuildResult{Artifact: art.Name, Digest: digest}
|
||||
if !publish {
|
||||
return res, nil
|
||||
}
|
||||
if tag == "" {
|
||||
tag = strings.TrimPrefix(digest, "sha256:")[:12]
|
||||
}
|
||||
return m.push(ctx, workspace, manifest.Module, local, res, tag)
|
||||
}
|
||||
|
||||
// dockerfileOf is the Dockerfile an artifact's `from` names and the build context: the directory it lies in. It must
|
||||
// be a regular file inside the module's own directory (checks/imageplace holds the catalogue to the same).
|
||||
func dockerfileOf(moduleDir, from string) (dockerfile, contextDir string, err error) {
|
||||
clean := filepath.Clean(filepath.FromSlash(strings.TrimSpace(from)))
|
||||
if from == "" || filepath.IsAbs(clean) || clean == ".." || strings.HasPrefix(clean, ".."+string(filepath.Separator)) {
|
||||
return "", "", fmt.Errorf("%q is not a Dockerfile in the module's own directory", from)
|
||||
}
|
||||
dockerfile = filepath.Join(moduleDir, clean)
|
||||
info, err := os.Lstat(dockerfile)
|
||||
if err != nil || !info.Mode().IsRegular() {
|
||||
return "", "", fmt.Errorf("%q is not a file in the module's directory", from)
|
||||
}
|
||||
return dockerfile, filepath.Dir(dockerfile), nil
|
||||
}
|
||||
|
||||
// localName names the image in buildah's store: one name for each artifact of a workspace.
|
||||
func localName(workspace, module, artifact string) string {
|
||||
sum := sha256.Sum256([]byte(module + "/" + artifact))
|
||||
return "mesh-build:" + workspace + "-" + hex.EncodeToString(sum[:])[:8]
|
||||
}
|
||||
|
||||
// localDigest is the manifest digest of the image buildah holds under name.
|
||||
func (m *Module) localDigest(ctx context.Context, name string) (string, error) {
|
||||
r := m.Run(ctx, m.env(), "buildah", append(m.storeFlags(), "images", "--no-trunc", "--format", "{{.Digest}}", name)...)
|
||||
digest := strings.TrimSpace(r.Stdout)
|
||||
if r.Status != 0 || r.Err != "" || !digestOK.MatchString(digest) {
|
||||
return "", fmt.Errorf("the digest of the image %s could not be read: %s", name, lastLine(r.Err+"\n"+r.Stderr+"\n"+r.Stdout))
|
||||
}
|
||||
return digest, nil
|
||||
}
|
||||
|
||||
var digestOK = regexp.MustCompile(`^sha256:[0-9a-f]{64}$`)
|
||||
|
||||
// remove drops the image from buildah's store, so the store does not grow with every build; a failure is said.
|
||||
func (m *Module) remove(name string) {
|
||||
r := m.Run(m.Ctx, m.env(), "buildah", append(m.storeFlags(), "rmi", "--force", name)...)
|
||||
if (r.Status != 0 || r.Err != "") && m.Say != nil {
|
||||
m.Say("the image %s could not be removed from the store: %s", name, lastLine(r.Err+"\n"+r.Stderr))
|
||||
}
|
||||
}
|
||||
|
||||
// lastLine is the last line of a command's output that says something: buildah's error is its last line.
|
||||
func lastLine(s string) string {
|
||||
lines := strings.Split(s, "\n")
|
||||
for i := len(lines) - 1; i >= 0; i-- {
|
||||
if l := strings.TrimSpace(lines[i]); l != "" {
|
||||
return l
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,200 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// The fixture module (testdata/repos/image): a Dockerfile that adds a file to an upstream base the manifest pins in
|
||||
// build.on, as a workspace's tree holds it.
|
||||
func place(t *testing.T, m *Module, edit func(string) string) {
|
||||
t.Helper()
|
||||
src := filepath.Join("testdata", "repos", "image", "modules", "demo")
|
||||
dst := filepath.Join(m.Work, "ws1", "tree", "modules", "demo")
|
||||
if err := os.MkdirAll(dst, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, n := range []string{"Dockerfile", "hello.txt", "module.json.fixture"} {
|
||||
raw, err := os.ReadFile(filepath.Join(src, n))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n == "module.json.fixture" {
|
||||
n = "module.json"
|
||||
raw = []byte(edit(string(raw)))
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dst, n), raw, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func same(s string) string { return s }
|
||||
|
||||
func runBuild(t *testing.T, m *Module, states *keptStates, extra map[string]any) (BuildResult, delivery.Step) {
|
||||
t.Helper()
|
||||
args := map[string]any{"workspace": "ws1", "module": "modules/demo", "artifact": "server"}
|
||||
for k, v := range extra {
|
||||
args[k] = v
|
||||
}
|
||||
if _, err := tool(t, m, BuildTool)(args); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
st := states.waitEnd(t)
|
||||
var res BuildResult
|
||||
if st.State == delivery.StepPassed {
|
||||
if err := json.Unmarshal(st.Result, &res); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return res, st
|
||||
}
|
||||
|
||||
func setup(t *testing.T, edit func(string) string) (*Module, *fake, *keptStates) {
|
||||
t.Helper()
|
||||
f := newFake(t)
|
||||
states := newStates()
|
||||
m := moduleWith(t, f, states)
|
||||
place(t, m, edit)
|
||||
return m, f, states
|
||||
}
|
||||
|
||||
// 1703c/AC: the build runs buildah bud rootless: the store and the runtime state in the store directory, the
|
||||
// Dockerfile of the module's own directory and that directory as the context, the target, the pinned base as a build
|
||||
// argument; no mount, no credential file, a runtime directory of its own, and it answers the image's digest.
|
||||
func TestBuildRunsBuildahRootlessAndAnswersTheDigest(t *testing.T) {
|
||||
m, f, states := setup(t, same)
|
||||
res, st := runBuild(t, m, states, nil)
|
||||
if st.State != delivery.StepPassed {
|
||||
t.Fatalf("step = %+v", st)
|
||||
}
|
||||
if res.Digest != imageDigest || res.Artifact != "server" || res.Pushed || res.Reference != "" {
|
||||
t.Errorf("result = %+v, want the local digest and no push", res)
|
||||
}
|
||||
bud := f.called("bud")
|
||||
if len(bud) != 1 {
|
||||
t.Fatalf("buildah bud ran %d times", len(bud))
|
||||
}
|
||||
dir := filepath.Join(m.Work, "ws1", "tree", "modules", "demo")
|
||||
for _, want := range []string{
|
||||
"--root " + m.Store + "/root", "--runroot " + m.Store + "/run", "--isolation oci",
|
||||
"--file " + dir + "/Dockerfile", "--target final",
|
||||
"--build-arg DEMO_BASE=docker.io/library/busybox@sha256:0000000000000000000000000000000000000000000000000000000000000001",
|
||||
} {
|
||||
if !strings.Contains(bud[0].line, want) {
|
||||
t.Errorf("bud line lacks %q:\n%s", want, bud[0].line)
|
||||
}
|
||||
}
|
||||
if !strings.HasSuffix(bud[0].line, " "+dir) {
|
||||
t.Errorf("the context is not the Dockerfile's directory:\n%s", bud[0].line)
|
||||
}
|
||||
for _, c := range f.calls {
|
||||
if c.name != "buildah" {
|
||||
continue
|
||||
}
|
||||
for _, bad := range []string{" -v ", "--volume", "--mount", "--authfile", "--creds", "--cap-add", "--privileged", "--device"} {
|
||||
if strings.Contains(" "+c.line+" ", bad) {
|
||||
t.Errorf("%q carries %q", c.line, bad)
|
||||
}
|
||||
}
|
||||
env := strings.Join(c.env, " ")
|
||||
if strings.Contains(env, "REGISTRY_AUTH") || strings.Contains(env, "DOCKER_CONFIG") || strings.Contains(env, "MESH_") {
|
||||
t.Errorf("the environment of %q carries a credential or the module's own: %s", c.line, env)
|
||||
}
|
||||
if !strings.Contains(env, "XDG_RUNTIME_DIR="+m.Store+"/xdg") {
|
||||
t.Errorf("the environment of %q lacks the module's own runtime directory: %s", c.line, env)
|
||||
}
|
||||
}
|
||||
// The image is removed from the store once the digest is read, so the store does not grow with every build.
|
||||
if len(f.called("rmi")) != 1 {
|
||||
t.Errorf("rmi ran %d times, want the image removed once", len(f.called("rmi")))
|
||||
}
|
||||
}
|
||||
|
||||
// 1703c: a failing Dockerfile step fails the step, saying buildah's last line.
|
||||
func TestBuildFailsWithBuildahsLastLine(t *testing.T) {
|
||||
m, f, states := setup(t, same)
|
||||
f.failBud = "Error: building at STEP \"RUN false\": exit status 1"
|
||||
_, st := runBuild(t, m, states, nil)
|
||||
if st.State != delivery.StepFailed || !strings.Contains(st.Reason, `STEP "RUN false"`) {
|
||||
t.Errorf("step = %+v", st)
|
||||
}
|
||||
if len(f.called("push")) != 0 {
|
||||
t.Error("a failed build pushed")
|
||||
}
|
||||
}
|
||||
|
||||
// 1703c: an image whose context is a repository, or whose build.on names another module's artifact, is refused
|
||||
// (not failed), the reason saying why; nothing is built.
|
||||
func TestBuildRefusesAnImageWithARepositoryContextOrAModuleBase(t *testing.T) {
|
||||
cases := map[string]struct {
|
||||
edit func(string) string
|
||||
want []string
|
||||
}{
|
||||
"context": {func(s string) string {
|
||||
return strings.Replace(s, `"target": "final"`, `"target": "final", "context": {"seat": "git", "repository": "novox/mesh-controller"}`, 1)
|
||||
}, []string{"novox/mesh-controller", "Dockerfile's directory", "ADR 0487"}},
|
||||
"module base": {func(s string) string {
|
||||
return strings.Replace(s, `"image": "docker.io/library/busybox@sha256:0000000000000000000000000000000000000000000000000000000000000001"`, `"module": "mesh-sdk", "artifact": "lib"`, 1)
|
||||
}, []string{"artifact lib of the module mesh-sdk", "DEMO_BASE"}},
|
||||
}
|
||||
for name, c := range cases {
|
||||
m, f, states := setup(t, c.edit)
|
||||
_, st := runBuild(t, m, states, nil)
|
||||
if st.State != delivery.StepRefused {
|
||||
t.Errorf("%s: step = %+v, want refused", name, st)
|
||||
continue
|
||||
}
|
||||
for _, w := range c.want {
|
||||
if !strings.Contains(st.Reason, w) {
|
||||
t.Errorf("%s: the reason %q lacks %q", name, st.Reason, w)
|
||||
}
|
||||
}
|
||||
if len(f.called("bud")) != 0 {
|
||||
t.Errorf("%s: a refused image was built", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The refusal is one small function: it knows the two forms and passes the rest.
|
||||
func TestUnsupportedImageNamesTheTwoFormsAndPassesTheRest(t *testing.T) {
|
||||
plain := catalogue.Manifest{Module: "x", Build: &catalogue.Build{On: []catalogue.BuildsOn{{Arg: "A", Image: "i@sha256:aa"}}}}
|
||||
if why := unsupportedImage(plain, catalogue.Artifact{Name: "a", Kind: "image"}); why != "" {
|
||||
t.Errorf("a plain image was refused: %s", why)
|
||||
}
|
||||
if why := unsupportedImage(plain, catalogue.Artifact{Name: "a", Kind: "image", Context: &catalogue.ArtifactContext{Repository: "o/r"}}); why == "" {
|
||||
t.Error("a repository context passed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildFailsForWhatIsNotAnImageOrNotThere(t *testing.T) {
|
||||
for name, tc := range map[string]struct {
|
||||
edit func(string) string
|
||||
args map[string]any
|
||||
}{
|
||||
"not an image": {func(s string) string { return strings.Replace(s, `"kind": "image"`, `"kind": "archive"`, 1) }, nil},
|
||||
"no artifact": {same, map[string]any{"artifact": "other"}},
|
||||
"no Dockerfile": {func(s string) string {
|
||||
return strings.Replace(s, `"from": "Dockerfile"`, `"from": "Missing"`, 1)
|
||||
}, nil},
|
||||
"leaves the module": {func(s string) string {
|
||||
return strings.Replace(s, `"from": "Dockerfile"`, `"from": "../x/Dockerfile"`, 1)
|
||||
}, nil},
|
||||
"outside the workspace": {same, map[string]any{"module": "../.."}},
|
||||
} {
|
||||
m, f, states := setup(t, tc.edit)
|
||||
_, st := runBuild(t, m, states, tc.args)
|
||||
if st.State != delivery.StepFailed {
|
||||
t.Errorf("%s: step = %+v, want failed", name, st)
|
||||
}
|
||||
if len(f.called("bud")) != 0 {
|
||||
t.Errorf("%s: buildah bud ran", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,157 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
// ADR 0487 §1: the build step answers the image's digest, through a real bus (issue 1703e). A nats-server from the Go
|
||||
// library, at the version this module's go.mod names, runs with JetStream on in the test's directory (the pattern of
|
||||
// mesh-build-go's bus.go). The module's steps state is a key-value bucket on that bus, and its tools are served on bus
|
||||
// subjects by a small bridge in the test, so a call, the step that runs behind it and the key it ends on all travel over
|
||||
// the bus; the mesh's own tool runner is not started here. buildah and git are the fake runner's, so this runs wherever
|
||||
// the check runs; the real buildah is TestARealBuild... in integration_test.go. What the keys hold is compared with the
|
||||
// captured shapes (testdata/captured, capture_test.go) that the builder's tests copy.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"github.com/nats-io/nats-server/v2/server"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
|
||||
// bus starts the throwaway server and answers a connection to it.
|
||||
func bus(t *testing.T) *nats.Conn {
|
||||
t.Helper()
|
||||
s, err := server.NewServer(&server.Options{Host: "127.0.0.1", Port: server.RANDOM_PORT, JetStream: true, StoreDir: t.TempDir(), NoLog: true, NoSigs: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
go s.Start()
|
||||
if !s.ReadyForConnections(30 * time.Second) {
|
||||
t.Fatal("the bus was not ready")
|
||||
}
|
||||
t.Cleanup(func() { s.Shutdown(); s.WaitForShutdown() })
|
||||
nc, err := nats.Connect(s.ClientURL())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(nc.Close)
|
||||
return nc
|
||||
}
|
||||
|
||||
// onBus wires a module's steps state to a bucket and serves its tools on "test.tool.<name>": the request body is the
|
||||
// arguments, the answer is {"result": ...} or {"error": "..."}.
|
||||
func onBus(t *testing.T, nc *nats.Conn, m *Module) jetstream.KeyValue {
|
||||
t.Helper()
|
||||
js, err := jetstream.New(nc)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx := context.Background()
|
||||
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: "mesh-build-image-steps"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m.Steps = func(key string, value any) (uint64, error) {
|
||||
raw, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return kv.Put(ctx, key, raw)
|
||||
}
|
||||
for _, tl := range m.Tools(m.StepTools()) {
|
||||
tl := tl
|
||||
if _, err := nc.Subscribe("test.tool."+tl.Name, func(msg *nats.Msg) {
|
||||
var args map[string]any
|
||||
_ = json.Unmarshal(msg.Data, &args)
|
||||
res, err := tl.Run(args)
|
||||
out := map[string]any{"result": res}
|
||||
if err != nil {
|
||||
out = map[string]any{"error": err.Error()}
|
||||
}
|
||||
b, _ := json.Marshal(out)
|
||||
_ = msg.Respond(b)
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return kv
|
||||
}
|
||||
|
||||
// call calls a tool over the bus and answers its result, which for a long step is the running step's key.
|
||||
func call(t *testing.T, nc *nats.Conn, tool string, args map[string]any) map[string]any {
|
||||
t.Helper()
|
||||
body, _ := json.Marshal(args)
|
||||
msg, err := nc.Request("test.tool."+tool, body, 10*time.Second)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var out struct {
|
||||
Result map[string]any `json:"result"`
|
||||
Error string `json:"error"`
|
||||
}
|
||||
if err := json.Unmarshal(msg.Data, &out); err != nil || out.Error != "" {
|
||||
t.Fatalf("%s answered %s (%v)", tool, msg.Data, err)
|
||||
}
|
||||
return out.Result
|
||||
}
|
||||
|
||||
// ended waits for the step's key in the bucket to leave running and answers the step as the bucket holds it.
|
||||
func ended(t *testing.T, kv jetstream.KeyValue, key string) delivery.Step {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(30 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
if e, err := kv.Get(context.Background(), key); err == nil {
|
||||
st, err := delivery.DecodeStep(e.Value())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if st.State != delivery.StepRunning {
|
||||
return st
|
||||
}
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
t.Fatalf("the key %s never left running", key)
|
||||
return delivery.Step{}
|
||||
}
|
||||
|
||||
func TestTheStepsAnswerThroughARealBusWithTheCapturedShapes(t *testing.T) {
|
||||
nc := bus(t)
|
||||
f := newFake(t)
|
||||
m := moduleWith(t, f, newStates())
|
||||
kv := onBus(t, nc, m)
|
||||
place(t, m, same)
|
||||
|
||||
// The fetch: a running key at once, then the workspace on the key.
|
||||
started := call(t, nc, FetchTool, map[string]any{"repository": "novox/mesh-catalog", "commit": commitSHA})
|
||||
if started["state"] != delivery.StepRunning || started["key"] == "" {
|
||||
t.Fatalf("the fetch answered %v, want a running step and its key", started)
|
||||
}
|
||||
st := ended(t, kv, started["key"].(string))
|
||||
compareCaptured(t, "fetch.json", normalized(t, st, workspaceOf(t, st), commitSHA, ""))
|
||||
|
||||
// The build of a pull request: the image's digest, nothing pushed. The workspace is the fixture's, "ws1".
|
||||
for name, extra := range map[string]map[string]any{"build-candidate.json": nil, "build-trunk.json": {"publish": true}} {
|
||||
args := map[string]any{"workspace": "ws1", "module": "modules/demo", "artifact": "server"}
|
||||
for k, v := range extra {
|
||||
args[k] = v
|
||||
}
|
||||
started := call(t, nc, BuildTool, args)
|
||||
st := ended(t, kv, started["key"].(string))
|
||||
if st.State != delivery.StepPassed {
|
||||
t.Fatalf("%s: step = %+v", name, st)
|
||||
}
|
||||
var res BuildResult
|
||||
if err := json.Unmarshal(st.Result, &res); err != nil || !digestOK.MatchString(res.Digest) || res.Pushed != (extra != nil) {
|
||||
t.Errorf("%s: result = %s (%v), want the image's digest and pushed %v", name, st.Result, err, extra != nil)
|
||||
}
|
||||
compareCaptured(t, name, normalized(t, st, "", "", f.address()))
|
||||
}
|
||||
if len(f.registry) != 1 || f.pushes != 1 {
|
||||
t.Errorf("the registry holds %v after %d pushes, want only the trunk build's one", f.registry, f.pushes)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
package main
|
||||
|
||||
// The shapes of what the module's steps end on in the steps state (novox/hq ADR 0487 §1, issue 1703e), kept as fixtures in
|
||||
// testdata/captured for the builder's tests (mesh-delivery-builder copies them): the stored step of a fetch, of a
|
||||
// candidate's build (no publish) and of a trunk build (publish). They are the step exactly as buildstep writes it to the
|
||||
// state, with what differs between runs (ids, times, the workspace, the commit, digests, the registry's address) put
|
||||
// to fixed words. This unit test regenerates them from the fake runner and compares (MESH_UPDATE_CAPTURED=1 rewrites
|
||||
// them); the integration test through the real bus (bus_integration_test.go) compares the steps the real bus holds with
|
||||
// the same files, so a change of shape fails in both places.
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
const capturedDir = "testdata/captured"
|
||||
|
||||
var hexDigest = regexp.MustCompile(`sha256:[0-9a-f]{64}`)
|
||||
|
||||
// normalized is a stored step with the words that differ between runs put to fixed ones.
|
||||
func normalized(t *testing.T, st delivery.Step, workspace, commit, registry string) []byte {
|
||||
t.Helper()
|
||||
raw, err := json.Marshal(st)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var v map[string]any
|
||||
if err := json.Unmarshal(raw, &v); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v["id"] = "STEPID"
|
||||
v["started"], v["ended"] = "2026-10-22T08:00:00Z", "2026-10-22T08:00:01Z"
|
||||
out, _ := json.MarshalIndent(v, "", " ")
|
||||
s := string(out)
|
||||
for from, to := range map[string]string{workspace: "WORKSPACE", commit: "COMMIT", registry: "REGISTRY"} {
|
||||
if from != "" {
|
||||
s = strings.ReplaceAll(s, from, to)
|
||||
}
|
||||
}
|
||||
s = hexDigest.ReplaceAllString(s, imageDigest)
|
||||
// The result is stored as a JSON string of an object in some encodings: keep the file readable by decoding it once.
|
||||
var back map[string]any
|
||||
if err := json.Unmarshal([]byte(s), &back); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pretty, _ := json.MarshalIndent(back, "", " ")
|
||||
return append(pretty, '\n')
|
||||
}
|
||||
|
||||
func compareCaptured(t *testing.T, name string, got []byte) {
|
||||
t.Helper()
|
||||
path := filepath.Join(capturedDir, name)
|
||||
if os.Getenv("MESH_UPDATE_CAPTURED") == "1" {
|
||||
if err := os.WriteFile(path, got, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return
|
||||
}
|
||||
want, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v (MESH_UPDATE_CAPTURED=1 writes it)", path, err)
|
||||
}
|
||||
if string(want) != string(got) {
|
||||
t.Errorf("%s differs from what the step ended on:\n--- want\n%s--- got\n%s", name, want, got)
|
||||
}
|
||||
}
|
||||
|
||||
func workspaceOf(t *testing.T, st delivery.Step) string {
|
||||
var r struct {
|
||||
Workspace string `json:"workspace"`
|
||||
}
|
||||
json.Unmarshal(st.Result, &r)
|
||||
return r.Workspace
|
||||
}
|
||||
|
||||
func TestTheStoredStepsHaveTheCapturedShapes(t *testing.T) {
|
||||
_ = time.Now
|
||||
// fetch
|
||||
{
|
||||
f := newFake(t)
|
||||
states := newStates()
|
||||
m := moduleWith(t, f, states)
|
||||
if _, err := tool(t, m, FetchTool)(map[string]any{"repository": "novox/mesh-catalog", "commit": commitSHA}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
st := states.waitEnd(t)
|
||||
compareCaptured(t, "fetch.json", normalized(t, st, workspaceOf(t, st), commitSHA, ""))
|
||||
}
|
||||
// a candidate's build and a trunk build
|
||||
for name, extra := range map[string]map[string]any{"build-candidate.json": nil, "build-trunk.json": {"publish": true}} {
|
||||
m, f, states := setup(t, same)
|
||||
_, st := runBuild(t, m, states, extra)
|
||||
compareCaptured(t, name, normalized(t, st, "", "", f.address()))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
)
|
||||
|
||||
// Issue 1703 (the live failure of the first assignment): the module is moved, so the provisioner's client on the node
|
||||
// fills its templates, and it fills only config, secrets, collections, placement and what a moved provider told it.
|
||||
// The registry's address therefore cannot be a ${bound:artifact-store:...} (the artifact store has not moved): it is the
|
||||
// placement ${seat:mesh-artifact-store:reach}, the form the container runtime's daemon.json uses. This composes the
|
||||
// module on a machine with buildah and the artifact store reachable, handing its files to the client as a node does.
|
||||
func TestTheModuleComposesWhereItsFilesAreHandedToTheProvisionersClient(t *testing.T) {
|
||||
built := func(m catalogue.Manifest) []catalogue.Built {
|
||||
texts := map[string]string{}
|
||||
entries, err := os.ReadDir(filepath.Join(moduleDir, "files"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, e := range entries {
|
||||
raw, err := os.ReadFile(filepath.Join(moduleDir, "files", e.Name()))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
texts[e.Name()] = string(raw)
|
||||
}
|
||||
return []catalogue.Built{
|
||||
{Name: "tools", Kind: "bundle", Reference: catalogue.ArtifactStoreScheme + "mesh-build-image/tools/blobs/sha256:" + strings.Repeat("a", 64), Digest: "sha256:" + strings.Repeat("a", 64)},
|
||||
{Name: "files", Kind: "archive", Reference: catalogue.ArtifactStoreScheme + "mesh-build-image/files/blobs/sha256:" + strings.Repeat("b", 64), Digest: "sha256:" + strings.Repeat("b", 64), Texts: texts},
|
||||
}
|
||||
}
|
||||
read := func(name string) catalogue.Manifest {
|
||||
m, err := catalogue.ReadTreeManifestWritten(catalogueRoot, filepath.Join(catalogueRoot, "modules", name))
|
||||
if err != nil {
|
||||
t.Fatalf("modules/%s: %v", name, err)
|
||||
}
|
||||
return m
|
||||
}
|
||||
image := read("mesh-build-image")
|
||||
resolved, err := image.Resolve(built(image))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
shelf := map[string]catalogue.Manifest{
|
||||
"mesh-build-image": resolved,
|
||||
"buildah": read("buildah"),
|
||||
"forge": {Module: "forge", Version: "1", Provides: []catalogue.Offer{{Name: "forge-account"}}},
|
||||
"store": {Module: "store", Version: "1", Provides: []catalogue.Offer{{Name: "artifact-store"}}},
|
||||
}
|
||||
node := catalogue.Node{Name: "builder", Site: "house",
|
||||
Capabilities: map[string]bool{"package-manager": true, "user-namespaces": true}}
|
||||
r, err := catalogue.Resolve(shelf, []string{"forge", "store", "buildah", "mesh-build-image"}, node, catalogue.World{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r.Account = "op"
|
||||
// What the resolution states for the node: the forge's provider has moved, the artifact store's has not.
|
||||
r.Needs = []catalogue.Needed{
|
||||
{Name: "forge-account", For: "mesh-build-image", Module: "forge", From: "novox", At: "10.7.0.1", ProviderMoved: true,
|
||||
Serves: map[string]any{"known_host": "[novox.internal]:222 ssh-ed25519 AAAA", "host_key": "ssh-ed25519 AAAA", "ssh_host": "novox.internal", "ssh_alias": "mesh-forge", "ssh_port": "222", "ssh_base": "ssh://git@novox.internal:222/", "https_base": "https://git.novox.be/"}},
|
||||
{Name: "artifact-store", For: "mesh-build-image", Module: "store", From: "novox", At: "10.7.0.1", Serves: map[string]any{"port": "5000"}},
|
||||
}
|
||||
with := catalogue.Rendering{Hands: true, Provisioner: "mesh-provisioner-client", ArtifactStore: "store:5000",
|
||||
Needed: map[string]map[string]string{"mesh-build-image": {"broker": "sealed-to-the-node"}}}
|
||||
c, err := r.Compose(with)
|
||||
if err != nil {
|
||||
t.Fatalf("the module does not compose on a machine with buildah and the artifact store: %v", err)
|
||||
}
|
||||
if _, asked := c.Plans["mesh-build-image"].Placement["seat:mesh-artifact-store:reach"]; !asked {
|
||||
t.Errorf("the plan's placement %v does not ask where the registry is reached", c.Plans["mesh-build-image"].Placement)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,159 @@
|
||||
package main
|
||||
|
||||
// The fetch step (novox/hq ADR 0487 §1, ADR 0466 §4 §5, ADR 0469 §2; issue 1703b): a commit of a repository of the
|
||||
// mesh's forge, fetched over SSH as the module's own account into a workspace of its own. Nothing is downloaded
|
||||
// beyond it: an image's Dockerfile fetches what it needs itself, inside the build.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
)
|
||||
|
||||
var (
|
||||
commitOK = regexp.MustCompile(`^[0-9a-f]{40}$`)
|
||||
repositoryOK = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._-]*$`)
|
||||
workspaceOK = regexp.MustCompile(`^[a-z0-9]+$`)
|
||||
)
|
||||
|
||||
// ForgeAlias is the ssh name the forge-account provision sets up in the account's ssh files (ADR 0409 §1); the
|
||||
// remote of a fetch is ssh://<it>/<owner>/<repository>.git.
|
||||
const ForgeAlias = "mesh-forge"
|
||||
|
||||
// WorkspaceAge is how long a workspace is kept: a module removes the workspaces older than a day, each time it
|
||||
// fetches and every round it checks itself. A workspace lies in the module's work directory, never in its home
|
||||
// (ADR 0466 §1).
|
||||
const WorkspaceAge = 24 * time.Hour
|
||||
|
||||
// FetchResult is what a passed fetch step says: the workspace it made and the commit in it, and, only when the call
|
||||
// gave a base, the merge preview of ADR 0484 clause 3 (the trunk commit, the merge commit of the head onto it, the
|
||||
// files in conflict, the paths the merge changes and the tree of each module directory). A head that does not
|
||||
// merge cleanly is an answer: Merge is empty, Conflicts names the files, and the workspace is left at the trunk.
|
||||
type FetchResult struct {
|
||||
Workspace string `json:"workspace"`
|
||||
Commit string `json:"commit"`
|
||||
Base string `json:"base,omitempty"`
|
||||
Merge string `json:"merge,omitempty"`
|
||||
Conflicts []string `json:"conflicts,omitempty"`
|
||||
Changed []string `json:"changed,omitempty"`
|
||||
Trees map[string]buildstep.ModuleTree `json:"trees,omitempty"`
|
||||
}
|
||||
|
||||
func (m *Module) fetchTool() StepTool {
|
||||
return StepTool{
|
||||
Name: FetchTool,
|
||||
Description: "Fetch a commit of a repository of the mesh's forge over SSH, as this module's own account, into a " +
|
||||
"workspace of its own. Answers a step id at once; the step ends on its key in the steps state with the " +
|
||||
"workspace's id.",
|
||||
Input: map[string]any{
|
||||
"type": "object",
|
||||
"properties": map[string]any{
|
||||
"repository": map[string]any{"type": "string", "description": "<owner>/<repository> on the forge"},
|
||||
"commit": map[string]any{"type": "string", "description": "the full commit, 40 hex digits"},
|
||||
"ref": map[string]any{"type": "string", "description": "refs/pull/<n>/head when only a pull request's head carries the commit"},
|
||||
"base": map[string]any{"type": "string", "description": "optional trunk commit, 40 hex digits: the head is merged onto it as the forge's merge would, and the answer carries the merge commit, the files in conflict (an answer, the workspace is then left at the trunk commit), the changed paths and the tree of each module directory"},
|
||||
},
|
||||
"required": []string{"repository", "commit"},
|
||||
},
|
||||
Long: true,
|
||||
Run: m.fetch,
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Module) fetch(ctx context.Context, args map[string]any) (any, error) {
|
||||
repository, _ := args["repository"].(string)
|
||||
commit, _ := args["commit"].(string)
|
||||
ref, _ := args["ref"].(string)
|
||||
base, _ := args["base"].(string)
|
||||
if !repositoryOK.MatchString(repository) || strings.Contains(repository, "..") {
|
||||
return nil, fmt.Errorf("%q is not a repository, <owner>/<name>", repository)
|
||||
}
|
||||
if !commitOK.MatchString(commit) {
|
||||
return nil, fmt.Errorf("%q is not a full commit (40 hex digits)", commit)
|
||||
}
|
||||
if base != "" && !commitOK.MatchString(base) {
|
||||
return nil, fmt.Errorf("%q is not a trunk commit (40 hex digits)", base)
|
||||
}
|
||||
m.CleanWorkspaces(m.clock())
|
||||
id := buildstep.NewID()
|
||||
tree := filepath.Join(m.Work, id, "tree")
|
||||
if err := os.MkdirAll(tree, 0o700); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
remote := m.remote(repository)
|
||||
if err := buildstep.FetchCommit(ctx, m.Run, tree, remote, commit, ref, m.FetchOptions...); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res := FetchResult{Workspace: id, Commit: commit}
|
||||
if base != "" {
|
||||
if err := m.merge(ctx, &res, tree, remote, base, commit); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// remote is the address a repository is fetched from: the forge over ssh as the account, or RemoteBase when a test set it.
|
||||
func (m *Module) remote(repository string) string {
|
||||
base := m.RemoteBase
|
||||
if base == "" {
|
||||
base = "ssh://" + ForgeAlias + "/"
|
||||
}
|
||||
return base + repository + ".git"
|
||||
}
|
||||
|
||||
// merge makes the merge preview of the head onto base (the SDK's buildstep.MergeInto, shared by every build module)
|
||||
// and writes the answer into res.
|
||||
func (m *Module) merge(ctx context.Context, res *FetchResult, tree, remote, base, commit string) error {
|
||||
mr, err := buildstep.MergeInto(ctx, m.Run, tree, remote, base, commit, m.FetchOptions...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
res.Base = mr.Base
|
||||
res.Conflicts, res.Merge, res.Changed = mr.Conflicts, mr.Merge, mr.Changed
|
||||
if res.Merge == "" {
|
||||
return nil
|
||||
}
|
||||
res.Trees, err = buildstep.ModuleTrees(ctx, m.Run, nil, tree, res.Merge)
|
||||
return err
|
||||
}
|
||||
|
||||
// CleanWorkspaces removes the workspaces older than WorkspaceAge from the work directory.
|
||||
func (m *Module) CleanWorkspaces(now time.Time) {
|
||||
entries, err := os.ReadDir(m.Work)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
for _, e := range entries {
|
||||
info, err := e.Info()
|
||||
if err != nil || !e.IsDir() || !workspaceOK.MatchString(e.Name()) || now.Sub(info.ModTime()) < WorkspaceAge {
|
||||
continue
|
||||
}
|
||||
if err := os.RemoveAll(filepath.Join(m.Work, e.Name())); err != nil && m.Say != nil {
|
||||
m.Say("the workspace %s could not be removed: %v", e.Name(), err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// clock is the time now: Now when a test set it, else the wall clock.
|
||||
func (m *Module) clock() time.Time {
|
||||
if m.Now != nil {
|
||||
return m.Now()
|
||||
}
|
||||
return time.Now()
|
||||
}
|
||||
|
||||
func firstLine(s string) string {
|
||||
for _, l := range strings.Split(s, "\n") {
|
||||
if l = strings.TrimSpace(l); l != "" {
|
||||
return l
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// 1703b: the fetch step asks git, through the SDK's FetchCommit, for the ssh remote of the forge as the account, makes
|
||||
// the workspace's tree in the work directory and answers its id and the commit; it downloads nothing else.
|
||||
func TestFetchAsksGitForTheSshRemoteIntoAWorkspace(t *testing.T) {
|
||||
f := newFake(t)
|
||||
states := newStates()
|
||||
m := moduleWith(t, f, states)
|
||||
ans, err := tool(t, m, FetchTool)(map[string]any{"repository": "novox/mesh-catalog", "commit": commitSHA, "ref": "refs/pull/7/head"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if a := ans.(map[string]any); a["state"] != delivery.StepRunning || a["step"] == "" {
|
||||
t.Errorf("the call answered %v, want a running step", a)
|
||||
}
|
||||
st := states.waitEnd(t)
|
||||
if st.State != delivery.StepPassed {
|
||||
t.Fatalf("step = %+v", st)
|
||||
}
|
||||
var res FetchResult
|
||||
if err := json.Unmarshal(st.Result, &res); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if res.Commit != commitSHA || res.Workspace == "" || res.Merge != "" {
|
||||
t.Errorf("result = %+v", res)
|
||||
}
|
||||
if info, err := os.Stat(filepath.Join(m.Work, res.Workspace, "tree")); err != nil || !info.IsDir() {
|
||||
t.Errorf("no tree in the workspace: %v", err)
|
||||
}
|
||||
joined := strings.Join(f.lines(), "\n")
|
||||
if !strings.Contains(joined, "ssh://mesh-forge/novox/mesh-catalog.git") || !strings.Contains(joined, commitSHA) {
|
||||
t.Errorf("git was not asked for the forge's ssh remote and the commit:\n%s", joined)
|
||||
}
|
||||
for _, l := range f.lines() {
|
||||
if strings.Contains(l, "buildah bud") || strings.HasPrefix(l, "go ") {
|
||||
t.Errorf("a fetch ran %q", l)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchRefusesWhatIsNotARepositoryOrACommit(t *testing.T) {
|
||||
for name, args := range map[string]map[string]any{
|
||||
"repository": {"repository": "../etc", "commit": commitSHA},
|
||||
"commit": {"repository": "novox/mesh-catalog", "commit": "main"},
|
||||
"base": {"repository": "novox/mesh-catalog", "commit": commitSHA, "base": "trunk"},
|
||||
} {
|
||||
f := newFake(t)
|
||||
states := newStates()
|
||||
m := moduleWith(t, f, states)
|
||||
if _, err := tool(t, m, FetchTool)(args); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if st := states.waitEnd(t); st.State != delivery.StepFailed {
|
||||
t.Errorf("%s: step = %+v, want failed", name, st)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,258 @@
|
||||
package main
|
||||
|
||||
// The forge side of a build module (novox/hq ADR 0466 §2 and §4, issue 1484). This file is the same for every build
|
||||
// module: mesh-build-node takes a copy and changes the three constants below.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/binary"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
const (
|
||||
forgeModule = "mesh-build-image"
|
||||
forgeHome = "/var/lib/mesh-build-image"
|
||||
forgeModuleD = "../.." // the module's directory, where module.json and files/ are
|
||||
)
|
||||
|
||||
type forgeManifest struct {
|
||||
Moved bool `json:"moved"`
|
||||
Requires []any `json:"requires"`
|
||||
State []any `json:"state"`
|
||||
Resources []map[string]any `json:"resources"`
|
||||
Build struct {
|
||||
Artifacts []map[string]any `json:"artifacts"`
|
||||
} `json:"build"`
|
||||
}
|
||||
|
||||
func readForgeManifest(t *testing.T) forgeManifest {
|
||||
t.Helper()
|
||||
raw, err := os.ReadFile(filepath.Join(forgeModuleD, "module.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var m forgeManifest
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// The forge files are filled on the node by the provisioner's client, never by the controller (ADR 0466 §4, ADR 0400
|
||||
// §5, issue 1537): the forge serves a module's entry only after the module has run and reported its key (ADR 0400
|
||||
// §2), so a controller filling them would refuse the machine until then, and the module could never run.
|
||||
func TestTheForgeFilesAreFilledOnTheNodeSoTheModuleCanRunBeforeItsEntry(t *testing.T) {
|
||||
if !readForgeManifest(t).Moved {
|
||||
t.Errorf("%s's manifest does not say \"moved\": true, so the controller fills its forge files and refuses "+
|
||||
"every machine it is assigned to until the forge serves an entry it makes only after the module ran", forgeModule)
|
||||
}
|
||||
}
|
||||
|
||||
func forgeTemplate(t *testing.T, r map[string]any) string {
|
||||
t.Helper()
|
||||
name, _ := r["template"].(string)
|
||||
if name == "" {
|
||||
t.Fatalf("resource %v names no template", r["id"])
|
||||
}
|
||||
raw, err := os.ReadFile(filepath.Join(forgeModuleD, filepath.FromSlash(name)))
|
||||
if err != nil {
|
||||
t.Fatalf("resource %v names %s: %v", r["id"], name, err)
|
||||
}
|
||||
return string(raw)
|
||||
}
|
||||
|
||||
func forgeResource(t *testing.T, m forgeManifest, id string) map[string]any {
|
||||
t.Helper()
|
||||
for _, r := range m.Resources {
|
||||
if r["id"] == id {
|
||||
if r["owner"] != forgeModule {
|
||||
t.Errorf("%s is owned by %v, not the module's own account", id, r["owner"])
|
||||
}
|
||||
if p, _ := r["path"].(string); !strings.HasPrefix(p, "${dir:") || strings.Contains(p, "account-home") {
|
||||
t.Errorf("%s is at %q: a path of the account's own home (${dir:...}), never the operator's", id, p)
|
||||
}
|
||||
return r
|
||||
}
|
||||
}
|
||||
t.Fatalf("no resource %q", id)
|
||||
return nil
|
||||
}
|
||||
|
||||
var forgeSample = map[string]string{
|
||||
"known_host": "[novox.internal]:222 ssh-ed25519 AAAAC3Nza",
|
||||
"host_key": "ssh-ed25519 AAAAC3Nza",
|
||||
"ssh_host": "novox.internal",
|
||||
"ssh_alias": "mesh-forge",
|
||||
"ssh_port": "222",
|
||||
"ssh_base": "ssh://git@novox.internal:222/",
|
||||
"https_base": "https://git.novox.be/",
|
||||
}
|
||||
|
||||
// forgeRender fills ${bound:forge-account:<key>} with the sample values, as the provisioner client does.
|
||||
func forgeRender(t *testing.T, s string) string {
|
||||
t.Helper()
|
||||
for k, v := range forgeSample {
|
||||
s = strings.ReplaceAll(s, "${bound:forge-account:"+k+"}", v)
|
||||
}
|
||||
if strings.Contains(s, "${") {
|
||||
t.Errorf("unfilled placeholder in %q", s)
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// 1484/AC1: given sample bound values, the ssh config has Host mesh-forge, IdentitiesOnly yes and the key under the
|
||||
// module's own home; known_hosts carries the host key; git's config rewrites https_base to ssh://mesh-forge/; no
|
||||
// ${ is left.
|
||||
func TestForgeFilesRenderFromTheBoundValuesInTheModulesOwnHome(t *testing.T) {
|
||||
m := readForgeManifest(t)
|
||||
|
||||
cfg := forgeResource(t, m, "ssh-config")
|
||||
want := "Host mesh-forge\n HostName novox.internal\n Port 222\n HostKeyAlias mesh-forge\n User git\n IdentityFile ~/.ssh/mesh_ed25519\n IdentitiesOnly yes\n"
|
||||
if got := forgeRender(t, forgeTemplate(t, cfg)); got != want {
|
||||
t.Errorf("ssh config = %q, want %q", got, want)
|
||||
}
|
||||
if cfg["path"] != "${dir:ssh-dir}/config" || cfg["mode"] != "0600" || cfg["into"] != nil {
|
||||
t.Errorf("ssh config resource = %v, want the whole file ${dir:ssh-dir}/config at mode 0600", cfg)
|
||||
}
|
||||
|
||||
kh := forgeResource(t, m, "known-hosts")
|
||||
if got := strings.TrimSpace(forgeRender(t, forgeTemplate(t, kh))); got != "mesh-forge ssh-ed25519 AAAAC3Nza" {
|
||||
t.Errorf("known_hosts = %q", got)
|
||||
}
|
||||
|
||||
g := forgeResource(t, m, "git-config")
|
||||
wantGit := "[url \"ssh://mesh-forge/\"]\n\tinsteadOf = https://git.novox.be/\n"
|
||||
if got := forgeRender(t, forgeTemplate(t, g)); got != wantGit {
|
||||
t.Errorf("git config = %q, want %q", got, wantGit)
|
||||
}
|
||||
|
||||
dir := forgeResource(t, m, "ssh-dir")
|
||||
if dir["mode"] != "0700" || dir["path"] != "${dir:state}/.ssh" {
|
||||
t.Errorf("ssh-dir = %v, want ${dir:state}/.ssh at mode 0700", dir)
|
||||
}
|
||||
}
|
||||
|
||||
// 1484/AC2: the module requires forge-account, keeps the state keys, and every file resource takes its text from a
|
||||
// template in the files/ archive (novox/hq ADR 0349 §2).
|
||||
func TestManifestRequiresForgeAccountKeepsKeysAndTakesFilesFromTemplates(t *testing.T) {
|
||||
m := readForgeManifest(t)
|
||||
if want := []any{"forge-account", "rootless-image-build", "artifact-store"}; !reflect.DeepEqual(m.Requires, want) {
|
||||
t.Errorf("requires = %v, want %v", m.Requires, want)
|
||||
}
|
||||
keeps := false
|
||||
for _, s := range m.State {
|
||||
if s == "keys" {
|
||||
keeps = true
|
||||
}
|
||||
}
|
||||
if !keeps {
|
||||
t.Errorf("state = %v lacks keys", m.State)
|
||||
}
|
||||
files := 0
|
||||
for _, r := range m.Resources {
|
||||
if _, inline := r["content"]; inline {
|
||||
t.Errorf("resource %v keeps its text inline", r["id"])
|
||||
}
|
||||
if r["type"] == "file" {
|
||||
files++
|
||||
forgeTemplate(t, r)
|
||||
}
|
||||
}
|
||||
if files == 0 {
|
||||
t.Errorf("no file resource: the forge's ssh files are missing")
|
||||
}
|
||||
packs := 0
|
||||
for _, a := range m.Build.Artifacts {
|
||||
if a["kind"] == "archive" && a["from"] == "files" && a["name"] == "files" {
|
||||
packs++
|
||||
}
|
||||
}
|
||||
if packs != 1 {
|
||||
t.Errorf("the build packs files/ into %d archives named files, not one", packs)
|
||||
}
|
||||
}
|
||||
|
||||
// forgeEd25519Line is a public key line with a well-formed ed25519 blob.
|
||||
func forgeEd25519Line(comment string) string {
|
||||
blob := []byte{}
|
||||
for _, part := range [][]byte{[]byte("ssh-ed25519"), make([]byte, 32)} {
|
||||
l := make([]byte, 4)
|
||||
binary.BigEndian.PutUint32(l, uint32(len(part)))
|
||||
blob = append(append(blob, l...), part...)
|
||||
}
|
||||
return "ssh-ed25519 " + base64.StdEncoding.EncodeToString(blob) + " " + comment
|
||||
}
|
||||
|
||||
type forgeKeys struct{ puts map[string][]byte }
|
||||
|
||||
func (f *forgeKeys) Put(key string, value any) (uint64, error) {
|
||||
raw, _ := json.Marshal(value)
|
||||
if f.puts == nil {
|
||||
f.puts = map[string][]byte{}
|
||||
}
|
||||
f.puts[key] = raw
|
||||
return uint64(len(f.puts)), nil
|
||||
}
|
||||
|
||||
// forgeKeygen stands in for ssh-keygen: it writes the pair the command names.
|
||||
func forgeKeygen(_ context.Context, _ []string, name string, args ...string) forgekey.Ran {
|
||||
var file, comment string
|
||||
for i, a := range args {
|
||||
switch a {
|
||||
case "-f":
|
||||
file = args[i+1]
|
||||
case "-C":
|
||||
comment = args[i+1]
|
||||
}
|
||||
}
|
||||
if name != "ssh-keygen" || file == "" {
|
||||
return forgekey.Ran{Status: 1, Stderr: "unexpected"}
|
||||
}
|
||||
os.WriteFile(file, []byte("PRIVATE\n"), 0o600)
|
||||
os.WriteFile(file+".pub", []byte(forgeEd25519Line(comment)+"\n"), 0o644)
|
||||
return forgekey.Ran{}
|
||||
}
|
||||
|
||||
// 1484/AC3: given the bundle started with an empty home, the keeper writes <node>.<module> with kind builder, and
|
||||
// the key is in the home's .ssh.
|
||||
func TestTheKeeperWritesTheModulesEntryWithKindBuilder(t *testing.T) {
|
||||
home, st := t.TempDir(), &forgeKeys{}
|
||||
if err := keeperFor(home, "ace", forgeModule, forgeKeygen, st).Once(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(home, ".ssh", "mesh_ed25519")); err != nil {
|
||||
t.Errorf("no key in an empty home: %v", err)
|
||||
}
|
||||
raw, ok := st.puts["ace."+forgeModule]
|
||||
if !ok || len(st.puts) != 1 {
|
||||
t.Fatalf("entries = %v, want one at ace.%s", st.puts, forgeModule)
|
||||
}
|
||||
var e forgekey.KeyEntry
|
||||
if err := json.Unmarshal(raw, &e); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if e.Kind != "builder" || e.Node != "ace" || e.Account != forgeModule || !strings.HasPrefix(e.Public, "ssh-ed25519 ") {
|
||||
t.Errorf("entry = %s", raw)
|
||||
}
|
||||
if strings.Contains(strings.ToUpper(string(raw)), "PRIVATE") {
|
||||
t.Errorf("the entry carries private material: %s", raw)
|
||||
}
|
||||
}
|
||||
|
||||
// The home the manifest places is the one the account is given, so the keeper's home and the templates' agree.
|
||||
func TestTheAccountsHomeIsTheOneTheManifestNames(t *testing.T) {
|
||||
m := readForgeManifest(t)
|
||||
for _, r := range m.Resources {
|
||||
if r["type"] == "user" && r["name"] == forgeModule && r["home"] != forgeHome {
|
||||
t.Errorf("the account's home is %v, want %s", r["home"], forgeHome)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,208 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
const commitSHA = "0123456789abcdef0123456789abcdef01234567"
|
||||
|
||||
// keptStates is the steps state, kept in memory: every write of a key, and the keys that ended.
|
||||
type keptStates struct {
|
||||
mu sync.Mutex
|
||||
steps map[string][]delivery.Step
|
||||
ended chan string
|
||||
}
|
||||
|
||||
func newStates() *keptStates {
|
||||
return &keptStates{steps: map[string][]delivery.Step{}, ended: make(chan string, 16)}
|
||||
}
|
||||
|
||||
func (s *keptStates) putStep(key string, value any) (uint64, error) {
|
||||
st, err := delivery.DecodeStep(value.(json.RawMessage))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
s.mu.Lock()
|
||||
s.steps[key] = append(s.steps[key], st)
|
||||
s.mu.Unlock()
|
||||
if st.State != delivery.StepRunning {
|
||||
s.ended <- key
|
||||
}
|
||||
return 1, nil
|
||||
}
|
||||
|
||||
func (s *keptStates) waitEnd(t *testing.T) delivery.Step {
|
||||
t.Helper()
|
||||
select {
|
||||
case key := <-s.ended:
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
v := s.steps[key]
|
||||
return v[len(v)-1]
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("the step never ended on its key")
|
||||
return delivery.Step{}
|
||||
}
|
||||
}
|
||||
|
||||
// cmdRecord is one command a step ran.
|
||||
type cmdRecord struct {
|
||||
env []string
|
||||
name string
|
||||
line string
|
||||
}
|
||||
|
||||
// fake stands in for git and buildah at the process boundary and for the registry over HTTP.
|
||||
type fake struct {
|
||||
mu sync.Mutex
|
||||
calls []cmdRecord
|
||||
digest string // what `buildah images` prints
|
||||
pushes int
|
||||
refuse int // the first n pushes answer "connection refused"
|
||||
failBud string
|
||||
// digestFile: "" writes the digest, "none" writes no file, "empty" writes an empty one.
|
||||
digestFile string
|
||||
// registry: repository:tag -> digest
|
||||
registry map[string]string
|
||||
server *httptest.Server
|
||||
}
|
||||
|
||||
const imageDigest = "sha256:1111111111111111111111111111111111111111111111111111111111111111"
|
||||
|
||||
func newFake(t *testing.T) *fake {
|
||||
f := &fake{digest: imageDigest, registry: map[string]string{}}
|
||||
f.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
key := strings.TrimPrefix(r.URL.Path, "/v2/")
|
||||
i := strings.Index(key, "/manifests/")
|
||||
if r.Method != http.MethodHead || i < 0 {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
d, ok := f.registry[key[:i]+":"+key[i+len("/manifests/"):]]
|
||||
if !ok {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Docker-Content-Digest", d)
|
||||
}))
|
||||
t.Cleanup(f.server.Close)
|
||||
return f
|
||||
}
|
||||
|
||||
func (f *fake) address() string { return f.server.Listener.Addr().String() }
|
||||
|
||||
func (f *fake) run(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
|
||||
line := name + " " + strings.Join(args, " ")
|
||||
f.mu.Lock()
|
||||
f.calls = append(f.calls, cmdRecord{env: env, name: name, line: line})
|
||||
f.mu.Unlock()
|
||||
switch {
|
||||
case name == "git" && strings.Contains(line, " rev-parse HEAD"):
|
||||
return forgekey.Ran{Stdout: commitSHA + "\n"}
|
||||
case name == "buildah" && strings.Contains(line, "--version"):
|
||||
return forgekey.Ran{Stdout: "buildah version 1.45.1 (image-spec 1.1.0, runtime-spec 1.1.0)\n"}
|
||||
case name == "buildah" && strings.Contains(line, " bud "):
|
||||
if f.failBud != "" {
|
||||
return forgekey.Ran{Status: 1, Stderr: "STEP 1/2: FROM x\n" + f.failBud + "\n"}
|
||||
}
|
||||
case name == "buildah" && strings.Contains(line, " images "):
|
||||
return forgekey.Ran{Stdout: f.digest + "\n"}
|
||||
case name == "buildah" && strings.Contains(line, " push "):
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if f.refuse > 0 {
|
||||
f.refuse--
|
||||
return forgekey.Ran{Status: 125, Stderr: "Error: dial tcp 127.0.0.1:5000: connect: connection refused\n"}
|
||||
}
|
||||
f.pushes++
|
||||
var digestFile, dest string
|
||||
for i, a := range args {
|
||||
if a == "--digestfile" {
|
||||
digestFile = args[i+1]
|
||||
}
|
||||
if strings.HasPrefix(a, "docker://") {
|
||||
dest = strings.TrimPrefix(a, "docker://")
|
||||
}
|
||||
}
|
||||
dest = strings.TrimPrefix(dest, f.address()+"/")
|
||||
f.registry[dest] = f.digest
|
||||
switch f.digestFile {
|
||||
case "":
|
||||
os.WriteFile(digestFile, []byte(f.digest), 0o644)
|
||||
case "empty":
|
||||
os.WriteFile(digestFile, nil, 0o644)
|
||||
}
|
||||
}
|
||||
return forgekey.Ran{}
|
||||
}
|
||||
|
||||
func (f *fake) lines() []string {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
out := []string{}
|
||||
for _, c := range f.calls {
|
||||
out = append(out, c.line)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (f *fake) called(verb string) []cmdRecord {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
var out []cmdRecord
|
||||
for _, c := range f.calls {
|
||||
if c.name == "buildah" && strings.Contains(c.line, " "+verb+" ") {
|
||||
out = append(out, c)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// moduleWith is a module on node ace over the fake, with subordinate ranges for its account, and the registry's
|
||||
// address written where the manifest would bind it.
|
||||
func moduleWith(t *testing.T, f *fake, states *keptStates) *Module {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
write := func(name, body string) string {
|
||||
p := filepath.Join(dir, name)
|
||||
if err := os.WriteFile(p, []byte(body), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return p
|
||||
}
|
||||
m := &Module{
|
||||
Node: "ace", Account: "mesh-build-image", UID: "1234", Home: t.TempDir(), Work: t.TempDir(), Store: t.TempDir(),
|
||||
Steps: states.putStep, Ctx: context.Background(), Run: f.run, HTTP: f.server.Client(),
|
||||
SubUIDFile: write("subuid", "someone:100000:65536\nmesh-build-image:624288:65536\n"),
|
||||
SubGIDFile: write("subgid", "mesh-build-image:624288:65536\n"),
|
||||
RegistryFile: write("registry", f.address()+"\n"),
|
||||
RegistryWait: 2 * time.Second,
|
||||
RegistryPause: 5 * time.Millisecond,
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
func tool(t *testing.T, m *Module, name string) func(map[string]any) (any, error) {
|
||||
t.Helper()
|
||||
for _, tl := range m.Tools(m.StepTools()) {
|
||||
if tl.Name == name {
|
||||
return tl.Run
|
||||
}
|
||||
}
|
||||
t.Fatalf("no tool %s", name)
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,219 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
// The image build module against real git and real buildah (novox/hq ADR 0487 §1, §2, §6; issue 1703e). They start
|
||||
// processes, so they carry the tag; the unit tests prove the commands with a fake runner. A test whose tools are not
|
||||
// installed where the check runs skips, loudly: the mesh's checker has git but not buildah until buildah is delivered
|
||||
// to its node, so the rootless isolation of §2 and §6 is proven live after the delivery (the issue says so).
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"os/exec"
|
||||
"os/user"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
func needTools(t *testing.T, what string, tools ...string) {
|
||||
t.Helper()
|
||||
for _, tool := range tools {
|
||||
if _, err := exec.LookPath(tool); err != nil {
|
||||
t.Skipf("NOT RUN: no %s installed where this check runs (%s); proven live on a build node", tool, what)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func git(t *testing.T, dir string, args ...string) string {
|
||||
t.Helper()
|
||||
cmd := exec.Command("git", append([]string{"-C", dir, "-c", "user.name=t", "-c", "user.email=t@example.invalid", "-c", "commit.gpgsign=false"}, args...)...)
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
t.Fatalf("git %v: %v\n%s", args, err, out)
|
||||
}
|
||||
return strings.TrimSpace(string(out))
|
||||
}
|
||||
|
||||
// realModule is a module over the real runner, a local directory standing in for the forge and the account running the test.
|
||||
func realModule(t *testing.T) (*Module, *keptStates) {
|
||||
t.Helper()
|
||||
states := newStates()
|
||||
u, err := user.Current()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
home := t.TempDir()
|
||||
m := &Module{Node: "ace", Account: u.Username, UID: u.Uid, Home: home, Work: t.TempDir(), Store: t.TempDir(),
|
||||
Steps: states.putStep, Ctx: context.Background(), Say: t.Logf, Run: StepRunner(home),
|
||||
FetchOptions: []buildstep.Option{buildstep.AllowFileRemote()}}
|
||||
return m, states
|
||||
}
|
||||
|
||||
// The fetch step on a real repository: the workspace holds the commit's tree, with the Dockerfile in it, and a base
|
||||
// makes the merge preview.
|
||||
func TestFetchOnARealRepository(t *testing.T) {
|
||||
needTools(t, "the fetch step on a real repository (every step is guarded by buildah and a user namespace)", "git", "buildah", "newuidmap")
|
||||
m, states := realModule(t)
|
||||
root := t.TempDir()
|
||||
bare, work := filepath.Join(root, "forge", "novox", "demo.git"), filepath.Join(root, "work")
|
||||
for _, d := range []string{bare, work} {
|
||||
os.MkdirAll(d, 0o755)
|
||||
}
|
||||
git(t, bare, "init", "-q", "--bare")
|
||||
git(t, work, "init", "-q", "-b", "main")
|
||||
os.MkdirAll(filepath.Join(work, "modules", "demo"), 0o755)
|
||||
os.WriteFile(filepath.Join(work, "modules", "demo", "Dockerfile"), []byte("FROM scratch\n"), 0o644)
|
||||
os.WriteFile(filepath.Join(work, "modules", "demo", "module.json"), []byte(`{"module":"demo","version":"1"}`), 0o644)
|
||||
git(t, work, "add", ".")
|
||||
git(t, work, "commit", "-q", "-m", "one")
|
||||
git(t, work, "push", "-q", bare, "HEAD:refs/heads/main")
|
||||
base := git(t, work, "rev-parse", "HEAD")
|
||||
os.WriteFile(filepath.Join(work, "modules", "demo", "hello.txt"), []byte("hi\n"), 0o644)
|
||||
git(t, work, "add", ".")
|
||||
git(t, work, "commit", "-q", "-m", "two")
|
||||
head := git(t, work, "rev-parse", "HEAD")
|
||||
git(t, work, "push", "-q", bare, "HEAD:refs/pull/1/head")
|
||||
m.RemoteBase = "file://" + filepath.Join(root, "forge") + "/"
|
||||
|
||||
if _, err := tool(t, m, FetchTool)(map[string]any{"repository": "novox/demo", "commit": head, "ref": "refs/pull/1/head", "base": base}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
st := waitLong(t, states)
|
||||
if st.State != delivery.StepPassed {
|
||||
t.Fatalf("step = %+v", st)
|
||||
}
|
||||
var res FetchResult
|
||||
if err := json.Unmarshal(st.Result, &res); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if res.Commit != head || res.Base != base || res.Merge == "" || len(res.Conflicts) != 0 {
|
||||
t.Errorf("result = %+v, want the head merged cleanly onto the base", res)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(m.Work, res.Workspace, "tree", "modules", "demo", "Dockerfile")); err != nil {
|
||||
t.Errorf("the Dockerfile is not in the workspace: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func waitLong(t *testing.T, s *keptStates) delivery.Step {
|
||||
t.Helper()
|
||||
select {
|
||||
case key := <-s.ended:
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
v := s.steps[key]
|
||||
return v[len(v)-1]
|
||||
case <-time.After(5 * time.Minute):
|
||||
t.Fatal("the step never ended on its key")
|
||||
return delivery.Step{}
|
||||
}
|
||||
}
|
||||
|
||||
// imageModule writes a workspace "ws1" holding the module "modules/img" with one image artifact and this Dockerfile.
|
||||
func imageModule(t *testing.T, m *Module, dockerfile string, files map[string]string) {
|
||||
t.Helper()
|
||||
dir := filepath.Join(m.Work, "ws1", "tree", "modules", "img")
|
||||
os.MkdirAll(dir, 0o755)
|
||||
os.WriteFile(filepath.Join(dir, "module.json"), []byte(`{"module":"img","version":"1","build":{"artifacts":[{"name":"server","kind":"image","from":"Dockerfile"}]}}`), 0o644)
|
||||
os.WriteFile(filepath.Join(dir, "Dockerfile"), []byte(dockerfile), 0o644)
|
||||
for n, c := range files {
|
||||
os.MkdirAll(filepath.Dir(filepath.Join(dir, n)), 0o755)
|
||||
os.WriteFile(filepath.Join(dir, n), []byte(c), 0o644)
|
||||
}
|
||||
}
|
||||
|
||||
func buildReal(t *testing.T, m *Module, states *keptStates) (BuildResult, delivery.Step) {
|
||||
t.Helper()
|
||||
if _, err := tool(t, m, BuildTool)(map[string]any{"workspace": "ws1", "module": "modules/img", "artifact": "server"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
st := waitLong(t, states)
|
||||
var res BuildResult
|
||||
if st.State == delivery.StepPassed {
|
||||
if err := json.Unmarshal(st.Result, &res); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return res, st
|
||||
}
|
||||
|
||||
// A Dockerfile that adds a file to scratch builds without any network, and the step answers the manifest digest.
|
||||
func TestARealBuildAnswersTheImagesDigest(t *testing.T) {
|
||||
needTools(t, "a rootless build", "buildah", "newuidmap")
|
||||
m, states := realModule(t)
|
||||
imageModule(t, m, "FROM scratch\nCOPY hello.txt /hello.txt\n", map[string]string{"hello.txt": "hello\n"})
|
||||
res, st := buildReal(t, m, states)
|
||||
if st.State != delivery.StepPassed || !digestOK.MatchString(res.Digest) {
|
||||
t.Fatalf("step = %+v result = %+v", st, res)
|
||||
}
|
||||
}
|
||||
|
||||
// needBase skips unless the build node can pull the small base the isolation Dockerfiles run in.
|
||||
func needBase(t *testing.T, m *Module) string {
|
||||
t.Helper()
|
||||
const base = "docker.io/library/busybox:1.36"
|
||||
if err := os.MkdirAll(filepath.Join(m.Store, "xdg"), 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r := m.Run(context.Background(), m.env(), "buildah", append(m.storeFlags(), "pull", base)...)
|
||||
if r.Status != 0 {
|
||||
t.Skipf("NOT RUN: the base %s cannot be pulled where this check runs (%s); proven live on a build node", base, firstLine(r.Stderr+r.Err))
|
||||
}
|
||||
return base
|
||||
}
|
||||
|
||||
// ADR 0487 §2: the build's process has a non-zero user outside its namespace, no container runtime socket is among its
|
||||
// mounts, and a step that tries to write outside its workspace fails.
|
||||
func TestARealBuildIsRootlessWithNoSocketAndNoWayOut(t *testing.T) {
|
||||
needTools(t, "the rootless isolation", "buildah", "newuidmap")
|
||||
m, states := realModule(t)
|
||||
base := needBase(t, m)
|
||||
|
||||
// The host side: the namespace the module makes maps the namespace's root to an id other than 0.
|
||||
r := m.Run(context.Background(), m.env(), "buildah", append(m.storeFlags(), "unshare", "cat", "/proc/self/uid_map")...)
|
||||
if f := strings.Fields(r.Stdout); r.Status != 0 || len(f) < 3 || f[0] != "0" || f[1] == "0" {
|
||||
t.Errorf("uid_map in the module's namespace = %q (%s), want 0 mapped to a non-zero id", r.Stdout, r.Stderr)
|
||||
}
|
||||
|
||||
// The inside: a RUN step sees root mapped to a non-zero outer id, and no runtime socket anywhere in its mounts.
|
||||
imageModule(t, m, "FROM "+base+"\n"+
|
||||
"RUN awk 'NR==1 { exit ($2 == 0) }' /proc/self/uid_map\n"+
|
||||
"RUN ! ls /var/run/docker.sock /run/docker.sock /run/podman/podman.sock /run/containerd/containerd.sock 2>/dev/null\n"+
|
||||
"RUN ! grep -E 'docker.sock|podman.sock|containerd.sock' /proc/self/mountinfo\n", nil)
|
||||
if _, st := buildReal(t, m, states); st.State != delivery.StepPassed {
|
||||
t.Errorf("the isolation checks inside the build failed: %+v", st)
|
||||
}
|
||||
|
||||
// A step that writes outside its workspace fails: the context cannot be left, and the host's kernel files are not writable.
|
||||
imageModule(t, m, "FROM "+base+"\nCOPY ../outside /outside\n", nil)
|
||||
if _, st := buildReal(t, m, states); st.State != delivery.StepFailed {
|
||||
t.Errorf("a COPY out of the context passed: %+v", st)
|
||||
}
|
||||
imageModule(t, m, "FROM "+base+"\nRUN echo b > /proc/sysrq-trigger\n", nil)
|
||||
if _, st := buildReal(t, m, states); st.State != delivery.StepFailed {
|
||||
t.Errorf("a write to the host's /proc/sysrq-trigger passed: %+v", st)
|
||||
}
|
||||
}
|
||||
|
||||
// ADR 0487 §5, §6: a step that looks for the account's credentials finds nothing: the key in the home's .ssh is in no
|
||||
// mount of the build.
|
||||
func TestARealBuildFindsNoCredentialsOfTheAccount(t *testing.T) {
|
||||
needTools(t, "the credential paths", "buildah", "newuidmap")
|
||||
m, states := realModule(t)
|
||||
base := needBase(t, m)
|
||||
os.MkdirAll(filepath.Join(m.Home, ".ssh"), 0o700)
|
||||
os.WriteFile(filepath.Join(m.Home, ".ssh", "mesh_ed25519"), []byte("PRIVATE\n"), 0o600)
|
||||
imageModule(t, m, "FROM "+base+"\n"+
|
||||
"RUN ! find / -xdev -name mesh_ed25519 2>/dev/null | grep .\n"+
|
||||
"RUN ! test -e /root/.ssh\n"+
|
||||
"RUN ! env | grep -E 'REGISTRY_AUTH|MESH_|SSH_AUTH'\n", nil)
|
||||
if _, st := buildReal(t, m, states); st.State != delivery.StepPassed {
|
||||
t.Errorf("a step found a credential path of the account: %+v", st)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
package main
|
||||
|
||||
// The module's own forge key (novox/hq ADR 0466 §2, issue 1484): mesh-sdk's key keeper, kind builder, for the
|
||||
// module's account in its home. It makes ~/.ssh/mesh_ed25519 when it is missing and writes one entry to this
|
||||
// module's `keys` state at <node>.<account>, which gitea's provider reads to register the key on the forge user
|
||||
// that reads and pushes nothing (ADR 0466 §3).
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/user"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
// keeperFor is this module's keeper: go/forgekey's, of kind builder, for account in home on node, whose entry lies
|
||||
// at <node>.<account> of the keys state.
|
||||
func keeperFor(home, node, account string, run forgekey.Runner, state forgekey.KeyPutter) *forgekey.KeyKeeper {
|
||||
return &forgekey.KeyKeeper{Home: home, Node: node, Account: account, Kind: forgekey.KindBuilder, Run: run, State: state, Now: time.Now}
|
||||
}
|
||||
|
||||
// startKeeper runs the keeper when the runtime launched this process for a node: outside a launch — a build, a
|
||||
// check — there is no state to put the entry in.
|
||||
func startKeeper(ctx context.Context) {
|
||||
node := strings.TrimSpace(os.Getenv("MESH_NODE"))
|
||||
u, err := user.Current()
|
||||
if node == "" || err != nil || u.HomeDir == "" {
|
||||
return
|
||||
}
|
||||
k := keeperFor(u.HomeDir, node, u.Username, forgekey.ExecRunner, keysState{})
|
||||
k.Keep(ctx, func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-build-image] "+f+"\n", a...) })
|
||||
}
|
||||
|
||||
type keysState struct{}
|
||||
|
||||
func (keysState) Put(key string, value any) (uint64, error) {
|
||||
return stdio.State("keys").Put(key, value)
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
// mesh-build-image: the image build module (novox/hq ADR 0487, ADR 0466 §1, ADR 0469; design 47 R4c-9, issue 1703). A
|
||||
// tools bundle the node's runtime launches as the module's own account (runs-as). It builds a module's `image`
|
||||
// artifact from the Dockerfile in the module's own directory with buildah, rootless, in a user namespace of its
|
||||
// account and with no container runtime socket, and pushes a merged trunk commit's image to the registry from its
|
||||
// own process once the build has ended. It refuses a step only when buildah cannot be run or the account has no
|
||||
// subordinate ranges, and its health is then unhealthy. What it says, it says on stderr; stdout is the MCP channel.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/user"
|
||||
"strings"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// The directories the module's manifest places outside its home (ADR 0466 §1): the workspaces, and buildah's image
|
||||
// store, so nothing of the home is in a build.
|
||||
const (
|
||||
WorkDir = "/var/lib/mesh-build-image-work"
|
||||
StoreDir = "/var/lib/mesh-build-image-store"
|
||||
)
|
||||
|
||||
func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-build-image] "+format+"\n", a...) }
|
||||
|
||||
func main() {
|
||||
ctx := context.Background()
|
||||
m := &Module{
|
||||
Node: strings.TrimSpace(os.Getenv("MESH_NODE")),
|
||||
Steps: stdio.State(delivery.StepsState).Put,
|
||||
ReadSteps: readOf(stdio.State(delivery.StepsState)),
|
||||
Ctx: ctx,
|
||||
Say: logf,
|
||||
Work: WorkDir,
|
||||
Store: StoreDir,
|
||||
}
|
||||
if u, err := user.Current(); err == nil {
|
||||
m.Home, m.Account, m.UID = u.HomeDir, u.Username, u.Uid
|
||||
}
|
||||
m.RegistryFile = m.Home + "/registry"
|
||||
m.Run = StepRunner(m.Home)
|
||||
// Outside a launch by the runtime — a build, a check — there is no node and no state to write.
|
||||
if m.Node != "" {
|
||||
go m.Start(ctx)
|
||||
}
|
||||
go startKeeper(ctx)
|
||||
if err := stdio.Serve("", m.Tools(m.StepTools())); err != nil {
|
||||
logf("%v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,191 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
)
|
||||
|
||||
// The catalogue's root and this module's directory.
|
||||
const (
|
||||
catalogueRoot = "../../../.."
|
||||
moduleDir = "../.."
|
||||
)
|
||||
|
||||
func readManifest(t *testing.T) catalogue.Manifest {
|
||||
t.Helper()
|
||||
m, err := catalogue.ReadTreeManifest(catalogueRoot, moduleDir)
|
||||
if err != nil {
|
||||
t.Fatalf("mesh-sdk's validator refuses module.json: %v", err)
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
func resource(t *testing.T, m catalogue.Manifest, id string) map[string]any {
|
||||
t.Helper()
|
||||
for _, r := range m.Resources {
|
||||
if r["id"] == id {
|
||||
return r
|
||||
}
|
||||
}
|
||||
t.Fatalf("module.json declares no resource %q", id)
|
||||
return nil
|
||||
}
|
||||
|
||||
// 1703a: the validator accepts the module (alone on a shelf too), it runs as mesh-build-image with its home at mode
|
||||
// 0700, lists user-namespaces, and declares git and openssh; buildah is not its package (the buildah module's alone).
|
||||
func TestManifestIsValidAndDeclaresTheAccountTheCapabilityAndThePackages(t *testing.T) {
|
||||
m := readManifest(t)
|
||||
if problems := catalogue.CatalogueProblems(catalogue.Shelf{m.Module: m}); len(problems) > 0 {
|
||||
t.Errorf("the catalogue's rules refuse it: %v", problems)
|
||||
}
|
||||
if m.Module != "mesh-build-image" || m.RunsAs != "mesh-build-image" {
|
||||
t.Errorf("module %q runs as %q, want mesh-build-image as itself", m.Module, m.RunsAs)
|
||||
}
|
||||
account := resource(t, m, "account")
|
||||
if account["type"] != "user" || account["name"] != "mesh-build-image" || account["shell"] != "/usr/bin/nologin" || account["home"] != "/var/lib/mesh-build-image" {
|
||||
t.Errorf("account = %v", account)
|
||||
}
|
||||
home := resource(t, m, "state")
|
||||
if home["type"] != "directory" || home["mode"] != "0700" || home["owner"] != "mesh-build-image" || home["place"] != "." {
|
||||
t.Errorf("home = %v, want its directory at mode 0700 owned by the account", home)
|
||||
}
|
||||
for _, c := range []string{"user-namespaces", "package-manager"} {
|
||||
if !slices.Contains(m.Capabilities, c) {
|
||||
t.Errorf("capabilities = %v, want %s", m.Capabilities, c)
|
||||
}
|
||||
}
|
||||
packages := []string{}
|
||||
for _, r := range m.Resources {
|
||||
if r["type"] == "package" {
|
||||
packages = append(packages, r["package"].(string))
|
||||
}
|
||||
}
|
||||
slices.Sort(packages)
|
||||
if want := []string{"git", "openssh"}; !slices.Equal(packages, want) {
|
||||
t.Errorf("packages = %v, want %v (buildah is the buildah module's, ADR 0487 §3)", packages, want)
|
||||
}
|
||||
}
|
||||
|
||||
// The work and store directories are outside the home (ADR 0466 §1), so nothing of the home is in a build; owned by the
|
||||
// account at mode 0700; and the constants of main.go are the manifest's paths.
|
||||
func TestWorkAndStoreDirectoriesLieOutsideTheHome(t *testing.T) {
|
||||
m := readManifest(t)
|
||||
for id, want := range map[string]string{"work": WorkDir, "store": StoreDir} {
|
||||
r := resource(t, m, id)
|
||||
path, _ := r["path"].(string)
|
||||
if path != want {
|
||||
t.Errorf("%s is at %q, the program uses %q", id, path, want)
|
||||
}
|
||||
if path == "" || path == "/var/lib/mesh-build-image" || strings.HasPrefix(path, "/var/lib/mesh-build-image/") {
|
||||
t.Errorf("%s is at %q, inside or at the home", id, path)
|
||||
}
|
||||
if r["owner"] != "mesh-build-image" || r["mode"] != "0700" {
|
||||
t.Errorf("%s = %v, want owned by the account at mode 0700", id, r)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The states are the two the module keeps, no seat is claimed, the health is its own tool, which the manifest also lists,
|
||||
// and the registry's address is bound into a file of the home that the program reads.
|
||||
func TestManifestKeepsItsStatesClaimsNoSeatAndDeclaresItsHealthTool(t *testing.T) {
|
||||
m := readManifest(t)
|
||||
states := []string{}
|
||||
for _, s := range m.State {
|
||||
states = append(states, s.Name)
|
||||
}
|
||||
slices.Sort(states)
|
||||
if want := []string{"keys", "steps"}; !slices.Equal(states, want) {
|
||||
t.Errorf("state = %v, want %v", states, want)
|
||||
}
|
||||
if len(m.Claims) != 0 {
|
||||
t.Errorf("claims = %v, want no seat", m.Claims)
|
||||
}
|
||||
raw, _ := os.ReadFile(filepath.Join(moduleDir, "module.json"))
|
||||
if !strings.Contains(string(raw), `"tool": "`+HealthTool+`"`) {
|
||||
t.Errorf("runtime-health does not name %s", HealthTool)
|
||||
}
|
||||
reg := resource(t, m, "registry")
|
||||
if reg["path"] != "${dir:state}/registry" {
|
||||
t.Errorf("registry file = %v, want ${dir:state}/registry (the program reads <home>/registry)", reg)
|
||||
}
|
||||
tpl, _ := os.ReadFile(filepath.Join(moduleDir, "files", "registry"))
|
||||
if strings.TrimSpace(string(tpl)) != "${seat:mesh-artifact-store:reach}" {
|
||||
t.Errorf("registry template = %q", tpl)
|
||||
}
|
||||
}
|
||||
|
||||
// Every tool the manifest names is served, and every tool served is named.
|
||||
func TestTheToolsServedAreTheToolsTheManifestNames(t *testing.T) {
|
||||
m := readManifest(t)
|
||||
served := []string{}
|
||||
for _, tl := range (&Module{}).Tools((&Module{}).StepTools()) {
|
||||
served = append(served, tl.Name)
|
||||
}
|
||||
slices.Sort(served)
|
||||
named := slices.Clone(m.Tools)
|
||||
slices.Sort(named)
|
||||
if !slices.Equal(served, named) {
|
||||
t.Errorf("served %v, manifest names %v", served, named)
|
||||
}
|
||||
}
|
||||
|
||||
// 1703f: gitea's manifest reads the keys state of mesh-build-image beside the other build modules', and the catalogue's
|
||||
// validator accepts it with the owners present.
|
||||
func TestGiteaReadsTheImageBuildModulesKeys(t *testing.T) {
|
||||
var all []catalogue.Manifest
|
||||
var gitea catalogue.Manifest
|
||||
for _, name := range []string{"gitea", "mesh-build-go", "mesh-build-node", "mesh-build-image"} {
|
||||
m, err := catalogue.ReadTreeManifest(catalogueRoot, filepath.Join(catalogueRoot, "modules", name))
|
||||
if err != nil {
|
||||
t.Fatalf("mesh-sdk's validator refuses modules/%s/module.json: %v", name, err)
|
||||
}
|
||||
all = append(all, m)
|
||||
if name == "gitea" {
|
||||
gitea = m
|
||||
}
|
||||
}
|
||||
for _, want := range []string{"mesh-build-go.keys", "mesh-build-node.keys", "mesh-build-image.keys"} {
|
||||
if !slices.Contains(gitea.Reads, want) {
|
||||
t.Errorf("gitea reads %v, lacks %s", gitea.Reads, want)
|
||||
}
|
||||
}
|
||||
if problems := catalogue.StateReadsNothingDeclares(all); len(problems) > 0 {
|
||||
t.Errorf("a read names a state its owner does not keep: %v", problems)
|
||||
}
|
||||
}
|
||||
|
||||
// 1704b: over the real manifests, mesh-build-image where no module provides rootless-image-build is refused, the error
|
||||
// naming the provision, the module that wants it and the module to assign (buildah); with buildah it composes. The
|
||||
// forge and the registry, which other modules provide for the whole mesh, are stood in for.
|
||||
func TestACompositionWithoutBuildahIsRefusedNamingItAndWithBuildahComposes(t *testing.T) {
|
||||
shelf := map[string]catalogue.Manifest{
|
||||
"forge": {Module: "forge", Version: "1", Provides: []catalogue.Offer{{Name: "forge-account"}}},
|
||||
"store": {Module: "store", Version: "1", Provides: []catalogue.Offer{{Name: "artifact-store"}}},
|
||||
}
|
||||
for _, name := range []string{"mesh-build-image", "buildah", "docker"} {
|
||||
m, err := catalogue.ReadTreeManifest(catalogueRoot, filepath.Join(catalogueRoot, "modules", name))
|
||||
if err != nil {
|
||||
t.Fatalf("modules/%s: %v", name, err)
|
||||
}
|
||||
shelf[m.Module] = m
|
||||
}
|
||||
node := catalogue.Node{Name: "builder", Site: "house",
|
||||
Capabilities: map[string]bool{"package-manager": true, "user-namespaces": true}}
|
||||
_, err := catalogue.Resolve(shelf, []string{"forge", "store", "docker", "mesh-build-image"}, node, catalogue.World{})
|
||||
if err == nil {
|
||||
t.Fatal("mesh-build-image composed on a machine without buildah (docker does not stand in for it)")
|
||||
}
|
||||
for _, want := range []string{`"rootless-image-build" is wanted by mesh-build-image`, "buildah"} {
|
||||
if !strings.Contains(err.Error(), want) {
|
||||
t.Errorf("the refusal does not say %q:\n%v", want, err)
|
||||
}
|
||||
}
|
||||
if _, err := catalogue.Resolve(shelf, []string{"forge", "store", "buildah", "mesh-build-image"}, node, catalogue.World{}); err != nil {
|
||||
t.Errorf("with buildah assigned the composition is refused: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,149 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
)
|
||||
|
||||
// CheckRound is how often the module looks at itself again, beside the look before every step and at every health look.
|
||||
const CheckRound = 10 * time.Minute
|
||||
|
||||
// Module is the image build module's one running state.
|
||||
type Module struct {
|
||||
Node string
|
||||
// Account and UID are the account the module runs as; Home is its home.
|
||||
Account, UID, Home string
|
||||
// Steps is the Put of the steps state; ReadSteps reads it back for the read tools.
|
||||
Steps buildstep.Put
|
||||
ReadSteps StateRead
|
||||
// Ctx is the process's context, which a step runs under (the tool call's is cut at 30 s, the step's is not).
|
||||
Ctx context.Context
|
||||
Say func(format string, a ...any)
|
||||
// Run runs the commands of a step (StepRunner); Work holds the workspaces and Store buildah's image store, both
|
||||
// outside the account's home (ADR 0466 §1).
|
||||
Run forgekey.Runner
|
||||
Work, Store string
|
||||
// SubUIDFile and SubGIDFile are the subordinate id files; /etc/subuid and /etc/subgid when empty.
|
||||
SubUIDFile, SubGIDFile string
|
||||
// RegistryFile holds the registry's address, host:port, which the manifest binds from the artifact-store provision.
|
||||
RegistryFile string
|
||||
// RemoteBase is where repositories are fetched from, "ssh://mesh-forge/" when empty; FetchOptions are the options of
|
||||
// the SDK's fetch and merge. A test points them at a local repository (buildstep.AllowFileRemote).
|
||||
RemoteBase string
|
||||
FetchOptions []buildstep.Option
|
||||
// HTTP talks to the registry; a client of its own when nil. RegistryWait and RegistryPause bound the wait for a
|
||||
// registry held still (RegistryWait and RegistryFirstPause when zero).
|
||||
HTTP *http.Client
|
||||
RegistryWait time.Duration
|
||||
RegistryPause time.Duration
|
||||
// Now is the clock; time.Now when nil.
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
// Look is what the module found out about its machine.
|
||||
type Look struct {
|
||||
Buildah string
|
||||
UIDs, GIDs int
|
||||
Account string
|
||||
}
|
||||
|
||||
// look proves the machine: buildah runs, the account holds ranges, a user namespace can be made. The first thing
|
||||
// missing is a *buildstep.Refusal naming it, so a step ends refused and the health says the same.
|
||||
func (m *Module) look(ctx context.Context) (Look, error) {
|
||||
l := Look{Account: m.Account}
|
||||
var err error
|
||||
if l.Buildah, err = m.buildahVersion(ctx); err != nil {
|
||||
return l, &buildstep.Refusal{Text: err.Error()}
|
||||
}
|
||||
if l.UIDs, l.GIDs, err = m.ranges(); err != nil {
|
||||
return l, &buildstep.Refusal{Text: err.Error(), Have: l.Buildah}
|
||||
}
|
||||
for _, d := range []string{m.Store, filepath.Join(m.Store, "xdg")} {
|
||||
if err := os.MkdirAll(d, 0o700); err != nil {
|
||||
return l, err
|
||||
}
|
||||
}
|
||||
if err := m.userNamespace(ctx); err != nil {
|
||||
return l, &buildstep.Refusal{Text: err.Error(), Have: l.Buildah}
|
||||
}
|
||||
return l, nil
|
||||
}
|
||||
|
||||
// Check is the guard: nil when the machine can build, else the refusal.
|
||||
func (m *Module) Check(ctx context.Context) error {
|
||||
_, err := m.look(ctx)
|
||||
return err
|
||||
}
|
||||
|
||||
// Start looks once, as the module starts, and then every CheckRound until ctx ends; it also removes old workspaces.
|
||||
func (m *Module) Start(ctx context.Context) {
|
||||
for {
|
||||
if err := m.Check(ctx); err != nil && m.Say != nil {
|
||||
m.Say("this machine cannot build images: %v", err)
|
||||
}
|
||||
m.CleanWorkspaces(m.clock())
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(CheckRound):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Health is the module's health (the runtime-health tool, ADR 0366): unhealthy while it cannot build, saying why,
|
||||
// healthy it names the buildah installed and the account's ranges (ADR 0487 §2: the ranges are proven here, not declared).
|
||||
func (m *Module) Health(ctx context.Context) health.Answer {
|
||||
l, err := m.look(ctx)
|
||||
if err != nil {
|
||||
return health.Unhealthy(err.Error())
|
||||
}
|
||||
return health.Healthy(fmt.Sprintf("buildah %s installed; %s holds %d subordinate uids and %d gids; a user namespace can be made", l.Buildah, l.Account, l.UIDs, l.GIDs))
|
||||
}
|
||||
|
||||
// Status is what the module reports of itself.
|
||||
func (m *Module) Status(ctx context.Context) map[string]any {
|
||||
l, err := m.look(ctx)
|
||||
out := map[string]any{"node": m.Node, "account": m.Account, "buildah": l.Buildah, "subuids": l.UIDs, "subgids": l.GIDs, "refusing": err != nil}
|
||||
if err != nil {
|
||||
out["why"] = err.Error()
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Long is a tool that runs a long step (ADR 0469 §2): it answers at once with the step's id, and the step ends on its
|
||||
// key in the steps state. The guard runs inside the step, before run, so a machine that cannot build ends every step
|
||||
// refused with the cause named.
|
||||
func (m *Module) Long(tool string, run func(ctx context.Context, args map[string]any) (any, error)) func(map[string]any) (any, error) {
|
||||
return func(args map[string]any) (any, error) {
|
||||
id, err := buildstep.Start(m.Ctx, m.Steps, m.Node, tool, func(ctx context.Context) (any, error) {
|
||||
if err := m.Check(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return run(ctx, args)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
key, _ := delivery.StepKey(m.Node, id)
|
||||
return map[string]any{"step": id, "key": key, "state": delivery.StepRunning}, nil
|
||||
}
|
||||
}
|
||||
|
||||
// Short is a tool that answers in its own call; the guard runs before it, and a refusal is the call's error.
|
||||
func (m *Module) Short(run func(ctx context.Context, args map[string]any) (any, error)) func(map[string]any) (any, error) {
|
||||
return func(args map[string]any) (any, error) {
|
||||
if err := m.Check(m.Ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return run(m.Ctx, args)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
)
|
||||
|
||||
// 1703a: a machine where buildah runs, the account holds ranges in both files and a namespace can be made is healthy,
|
||||
// and the health names buildah and the ranges (ADR 0487 §2: proven, not declared).
|
||||
func TestHealthIsHealthyWithBuildahRangesAndANamespace(t *testing.T) {
|
||||
f := newFake(t)
|
||||
m := moduleWith(t, f, newStates())
|
||||
a := m.Health(context.Background())
|
||||
if a.State != health.StateHealthy || !a.Healthy {
|
||||
t.Fatalf("health = %+v, want healthy", a)
|
||||
}
|
||||
for _, want := range []string{"buildah 1.45.1", "mesh-build-image holds 65536 subordinate uids and 65536 gids"} {
|
||||
if !strings.Contains(a.Why, want) {
|
||||
t.Errorf("health says %q, lacks %q", a.Why, want)
|
||||
}
|
||||
}
|
||||
if _, err := os.Stat(m.Store + "/xdg"); err != nil {
|
||||
t.Errorf("the runtime directory inside the store was not made: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHealthIsUnhealthyWithoutBuildah(t *testing.T) {
|
||||
f := newFake(t)
|
||||
m := moduleWith(t, f, newStates())
|
||||
m.Run = func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
|
||||
return forgekey.Ran{Status: 127, Err: "exec: \"buildah\": executable file not found in $PATH"}
|
||||
}
|
||||
a := m.Health(context.Background())
|
||||
if a.Healthy || !strings.Contains(a.Why, "buildah") {
|
||||
t.Errorf("health = %+v, want unhealthy naming buildah", a)
|
||||
}
|
||||
}
|
||||
|
||||
// The ranges are read from the two files, one each: without one in /etc/subuid or in /etc/subgid the account has no ids to
|
||||
// map, and the health says which file.
|
||||
func TestHealthIsUnhealthyWithoutARangeInEitherFile(t *testing.T) {
|
||||
for file, set := range map[string]func(*Module, string){
|
||||
"subuid": func(m *Module, p string) { m.SubUIDFile = p },
|
||||
"subgid": func(m *Module, p string) { m.SubGIDFile = p },
|
||||
} {
|
||||
f := newFake(t)
|
||||
m := moduleWith(t, f, newStates())
|
||||
p := t.TempDir() + "/" + file
|
||||
os.WriteFile(p, []byte("someone-else:100000:65536\n"), 0o644)
|
||||
set(m, p)
|
||||
a := m.Health(context.Background())
|
||||
if a.Healthy || !strings.Contains(a.Why, p) || !strings.Contains(a.Why, "mesh-build-image") {
|
||||
t.Errorf("%s: health = %+v, want unhealthy naming %s and the account", file, a, p)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHealthIsUnhealthyWhenANamespaceCannotBeMade(t *testing.T) {
|
||||
f := newFake(t)
|
||||
m := moduleWith(t, f, newStates())
|
||||
run := m.Run
|
||||
m.Run = func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
|
||||
if len(args) >= 2 && args[len(args)-2] == "unshare" {
|
||||
return forgekey.Ran{Status: 125, Stderr: "Error: cannot clone: Operation not permitted\n"}
|
||||
}
|
||||
return run(ctx, env, name, args...)
|
||||
}
|
||||
a := m.Health(context.Background())
|
||||
if a.Healthy || !strings.Contains(a.Why, "user namespace") {
|
||||
t.Errorf("health = %+v, want unhealthy naming the user namespace", a)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubRangeSumsTheAccountsLinesByNameOrUid(t *testing.T) {
|
||||
p := t.TempDir() + "/subuid"
|
||||
os.WriteFile(p, []byte("a:1:10\nmesh-build-image:100:65536\n1234:200000:100\nb:5:5\nbad line\n"), 0o644)
|
||||
if n, err := SubRange(p, "mesh-build-image", "1234"); err != nil || n != 65636 {
|
||||
t.Errorf("range = %d, %v, want 65636", n, err)
|
||||
}
|
||||
if n, _ := SubRange(p, "nobody", ""); n != 0 {
|
||||
t.Errorf("nobody has %d", n)
|
||||
}
|
||||
}
|
||||
|
||||
// A step on a machine that cannot build ends refused with the cause named, and nothing else runs.
|
||||
func TestAStepOnAMachineThatCannotBuildEndsRefused(t *testing.T) {
|
||||
f := newFake(t)
|
||||
states := newStates()
|
||||
m := moduleWith(t, f, states)
|
||||
m.SubGIDFile = "/nonexistent/subgid"
|
||||
if _, err := tool(t, m, FetchTool)(map[string]any{"repository": "novox/mesh-catalog", "commit": commitSHA}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
st := states.waitEnd(t)
|
||||
if st.State != delivery.StepRefused || !strings.Contains(st.Reason, "/nonexistent/subgid") {
|
||||
t.Errorf("step = %+v, want refused naming the file", st)
|
||||
}
|
||||
for _, l := range f.lines() {
|
||||
if strings.HasPrefix(l, "git ") {
|
||||
t.Errorf("a refused step ran %q", l)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestStatusSaysWhatTheMachineHas(t *testing.T) {
|
||||
f := newFake(t)
|
||||
m := moduleWith(t, f, newStates())
|
||||
s := m.Status(context.Background())
|
||||
if s["refusing"] != false || s["buildah"] != "1.45.1" || s["subuids"] != 65536 || s["account"] != "mesh-build-image" {
|
||||
t.Errorf("status = %v", s)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
package main
|
||||
|
||||
// What the module proves about its own machine before it builds (novox/hq ADR 0487 §2, issue 1703a): buildah runs,
|
||||
// the account holds subordinate user and group ranges (the node-engine's useradd allocates them for every `user`
|
||||
// account, so nothing declares them; this reads them), and a user namespace can be made.
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ProbeTimeout bounds each probe: they read the machine and fetch nothing.
|
||||
const ProbeTimeout = 20 * time.Second
|
||||
|
||||
// Default subordinate id files.
|
||||
const (
|
||||
SubUIDFile = "/etc/subuid"
|
||||
SubGIDFile = "/etc/subgid"
|
||||
)
|
||||
|
||||
// buildahVersion runs `buildah --version` and answers the version, the third word of "buildah version 1.45.1 (...)".
|
||||
func (m *Module) buildahVersion(ctx context.Context) (string, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, ProbeTimeout)
|
||||
defer cancel()
|
||||
r := m.Run(ctx, m.env(), "buildah", "--version")
|
||||
if r.Status != 0 || r.Err != "" {
|
||||
return "", fmt.Errorf("`buildah --version` failed (is the module buildah assigned to this machine?): %s", firstLine(r.Err+"\n"+r.Stderr))
|
||||
}
|
||||
f := strings.Fields(r.Stdout)
|
||||
if len(f) < 3 || f[0] != "buildah" || f[1] != "version" {
|
||||
return "", fmt.Errorf("`buildah --version` answered %q", strings.TrimSpace(r.Stdout))
|
||||
}
|
||||
return f[2], nil
|
||||
}
|
||||
|
||||
// userNamespace makes a user namespace as the account does for a build and runs `true` in it.
|
||||
func (m *Module) userNamespace(ctx context.Context) error {
|
||||
ctx, cancel := context.WithTimeout(ctx, ProbeTimeout)
|
||||
defer cancel()
|
||||
r := m.Run(ctx, m.env(), "buildah", append(m.storeFlags(), "unshare", "true")...)
|
||||
if r.Status != 0 || r.Err != "" {
|
||||
return fmt.Errorf("a user namespace cannot be made: %s", firstLine(r.Err+"\n"+r.Stderr))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// SubRange is how many subordinate ids file lists for the account (by name or by numeric uid): the sum of its lines'
|
||||
// counts, zero when it has none.
|
||||
func SubRange(file, account, uid string) (int, error) {
|
||||
f, err := os.Open(file)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer f.Close()
|
||||
total := 0
|
||||
sc := bufio.NewScanner(f)
|
||||
for sc.Scan() {
|
||||
parts := strings.Split(strings.TrimSpace(sc.Text()), ":")
|
||||
if len(parts) != 3 || (parts[0] != account && (uid == "" || parts[0] != uid)) {
|
||||
continue
|
||||
}
|
||||
if n, err := strconv.Atoi(parts[2]); err == nil && n > 0 {
|
||||
total += n
|
||||
}
|
||||
}
|
||||
return total, sc.Err()
|
||||
}
|
||||
|
||||
func (m *Module) subFiles() (uid, gid string) {
|
||||
uid, gid = m.SubUIDFile, m.SubGIDFile
|
||||
if uid == "" {
|
||||
uid = SubUIDFile
|
||||
}
|
||||
if gid == "" {
|
||||
gid = SubGIDFile
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// ranges reads the account's subordinate ranges, or says which file lacks one.
|
||||
func (m *Module) ranges() (uids, gids int, err error) {
|
||||
uf, gf := m.subFiles()
|
||||
for _, c := range []struct {
|
||||
file string
|
||||
n *int
|
||||
}{{uf, &uids}, {gf, &gids}} {
|
||||
n, rerr := SubRange(c.file, m.Account, m.UID)
|
||||
if rerr != nil {
|
||||
return 0, 0, fmt.Errorf("%s cannot be read: %v", c.file, rerr)
|
||||
}
|
||||
if n == 0 {
|
||||
return 0, 0, fmt.Errorf("%s holds no subordinate range for the account %s: a build in a user namespace has no ids to map (the node-engine's useradd allocates them when it makes a user account)", c.file, m.Account)
|
||||
}
|
||||
*c.n = n
|
||||
}
|
||||
return uids, gids, nil
|
||||
}
|
||||
|
||||
// storeFlags are buildah's global flags that put its image store and its runtime state in the module's store
|
||||
// directory, outside the account's home.
|
||||
func (m *Module) storeFlags() []string {
|
||||
return []string{"--root", filepath.Join(m.Store, "root"), "--runroot", filepath.Join(m.Store, "run")}
|
||||
}
|
||||
|
||||
// env is what buildah is given beyond the base environment: a runtime directory of its own inside the store, so
|
||||
// nothing is read from the account's runtime directory, and no registry credential file anywhere (the module has none
|
||||
// and REGISTRY_AUTH_FILE is never set).
|
||||
func (m *Module) env() []string {
|
||||
return []string{"XDG_RUNTIME_DIR=" + filepath.Join(m.Store, "xdg")}
|
||||
}
|
||||
@@ -0,0 +1,157 @@
|
||||
package main
|
||||
|
||||
// The trunk push (novox/hq ADR 0487 §5; issue 1703d): after the build has ended, from the module's own process and not
|
||||
// from a Dockerfile step, the image is pushed to the registry under a tag of its own and the digest is read back from
|
||||
// the registry, which is the only one that can say what it serves for the reference (issue 1742). The push and the
|
||||
// read wait out a registry held still by the store's nightly collection (issue 457).
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
// RegistryWait is how long a call waits for a registry that refuses connections, longer than the nightly collection
|
||||
// holds it still (about a minute and a half); RegistryFirstPause is the first pause, doubling to RegistryMostPause.
|
||||
const (
|
||||
RegistryWait = 5 * time.Minute
|
||||
RegistryFirstPause = time.Second
|
||||
RegistryMostPause = 10 * time.Second
|
||||
)
|
||||
|
||||
// registry is the registry's address, host:port, as the manifest bound it into RegistryFile.
|
||||
func (m *Module) registry() (string, error) {
|
||||
raw, err := os.ReadFile(m.RegistryFile)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("the registry's address cannot be read: %w", err)
|
||||
}
|
||||
addr := strings.TrimSpace(string(raw))
|
||||
if addr == "" || strings.Contains(addr, "${") || strings.HasPrefix(addr, ":") || strings.HasSuffix(addr, ":") {
|
||||
return "", fmt.Errorf("the registry's address in %s is %q, not host:port", m.RegistryFile, addr)
|
||||
}
|
||||
return addr, nil
|
||||
}
|
||||
|
||||
func (m *Module) push(ctx context.Context, workspace, module, local string, res BuildResult, tag string) (any, error) {
|
||||
addr, err := m.registry()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
repository := module + "/" + res.Artifact
|
||||
dest := addr + "/" + repository + ":" + tag
|
||||
digestFile := filepath.Join(m.Work, workspace, "pushed."+tag)
|
||||
argv := append(m.storeFlags(), "push", "--tls-verify=false", "--digestfile", digestFile, local, "docker://"+dest)
|
||||
if err := m.waitForRegistry(ctx, addr, "buildah push "+dest, func() error {
|
||||
r := m.Run(ctx, m.env(), "buildah", argv...)
|
||||
if r.Status != 0 || r.Err != "" {
|
||||
return errors.New(lastLine(r.Err + "\n" + r.Stderr + "\n" + r.Stdout))
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
return nil, fmt.Errorf("pushing %s: %w", dest, err)
|
||||
}
|
||||
var served string
|
||||
if err := m.waitForRegistry(ctx, addr, "reading "+dest, func() error {
|
||||
var err error
|
||||
served, err = m.registryDigest(ctx, addr, repository, tag)
|
||||
return err
|
||||
}); err != nil {
|
||||
return nil, fmt.Errorf("pushed %s and cannot read back what the registry serves for it: %w", dest, err)
|
||||
}
|
||||
// A push that succeeded wrote the digest it made: a file missing, unreadable or empty is a fault, and the registry's
|
||||
// digest is never answered unchecked.
|
||||
raw, err := os.ReadFile(digestFile)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("pushed %s, but the digest buildah pushed cannot be read from %s, so the registry's %s cannot be checked: %w", dest, digestFile, served, err)
|
||||
}
|
||||
pushed := strings.TrimSpace(string(raw))
|
||||
if !digestOK.MatchString(pushed) {
|
||||
return nil, fmt.Errorf("pushed %s, but the digest file %s holds %q, not a digest, so the registry's %s cannot be checked", dest, digestFile, pushed, served)
|
||||
}
|
||||
if pushed != served {
|
||||
return nil, fmt.Errorf("pushed %s as %s, but the registry serves %s for it", dest, pushed, served)
|
||||
}
|
||||
res.Digest, res.Pushed = served, true
|
||||
res.Reference = addr + "/" + repository + "@" + served
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// registryDigest is the digest the registry answers for repository:tag (Docker-Content-Digest of a HEAD).
|
||||
func (m *Module) registryDigest(ctx context.Context, addr, repository, tag string) (string, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodHead, "http://"+addr+"/v2/"+repository+"/manifests/"+tag, nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
req.Header.Set("Accept", strings.Join([]string{
|
||||
"application/vnd.oci.image.manifest.v1+json", "application/vnd.oci.image.index.v1+json",
|
||||
"application/vnd.docker.distribution.manifest.v2+json", "application/vnd.docker.distribution.manifest.list.v2+json"}, ", "))
|
||||
client := m.HTTP
|
||||
if client == nil {
|
||||
client = &http.Client{Timeout: time.Minute}
|
||||
}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return "", fmt.Errorf("the registry answered %s for %s:%s", resp.Status, repository, tag)
|
||||
}
|
||||
d := resp.Header.Get("Docker-Content-Digest")
|
||||
if !digestOK.MatchString(d) {
|
||||
return "", fmt.Errorf("the registry named no digest for %s:%s (Docker-Content-Digest %q)", repository, tag, d)
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
|
||||
// refused is whether an error is a connection refused: from a dial here, or as buildah said it in its output.
|
||||
func refused(err error) bool {
|
||||
return err != nil && (errors.Is(err, syscall.ECONNREFUSED) || strings.Contains(err.Error(), "connection refused"))
|
||||
}
|
||||
|
||||
// waitForRegistry runs try, and while it fails because the registry at addr refuses connections, tries again with a
|
||||
// growing pause until RegistryWait has passed (novox/hq issue 457). A refusal is the one error waited for: nothing
|
||||
// was sent, so trying again does nothing twice. what names the call, for what the module says.
|
||||
func (m *Module) waitForRegistry(ctx context.Context, addr, what string, try func() error) error {
|
||||
err := try()
|
||||
if !refused(err) {
|
||||
return err
|
||||
}
|
||||
limit, pause := m.RegistryWait, m.RegistryPause
|
||||
if limit == 0 {
|
||||
limit = RegistryWait
|
||||
}
|
||||
if pause == 0 {
|
||||
pause = RegistryFirstPause
|
||||
}
|
||||
started := time.Now()
|
||||
m.say("%s: refused; waiting for the registry at %s for up to %s — the store's nightly collection holds it still for about a minute and a half (issue 457)", what, addr, limit)
|
||||
for {
|
||||
left := limit - time.Since(started)
|
||||
if left <= 0 {
|
||||
return fmt.Errorf("the registry at %s refused every connection for %s, longer than its nightly collection holds it still, so it is down, not paused: %w", addr, time.Since(started).Round(time.Millisecond), err)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return fmt.Errorf("stopped while waiting for the registry at %s: %w (last: %v)", addr, ctx.Err(), err)
|
||||
case <-time.After(min(pause, left)):
|
||||
}
|
||||
pause = min(pause*2, RegistryMostPause)
|
||||
if err = try(); !refused(err) {
|
||||
m.say("%s: the registry at %s answers again after %s", what, addr, time.Since(started).Round(time.Millisecond))
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Module) say(format string, a ...any) {
|
||||
if m.Say != nil {
|
||||
m.Say(format, a...)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,156 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
// 1703d/AC (ADR 0487 §5): a candidate's build leaves the registry unchanged: nothing is pushed, nothing is asked of it.
|
||||
func TestACandidatesBuildLeavesTheRegistryUnchanged(t *testing.T) {
|
||||
m, f, states := setup(t, same)
|
||||
res, st := runBuild(t, m, states, nil) // no publish
|
||||
if st.State != delivery.StepPassed || res.Pushed {
|
||||
t.Fatalf("step = %+v result = %+v", st, res)
|
||||
}
|
||||
if len(f.called("push")) != 0 || len(f.registry) != 0 || f.pushes != 0 {
|
||||
t.Errorf("a candidate's build reached the registry: pushes %d, registry %v, calls %v", f.pushes, f.registry, f.lines())
|
||||
}
|
||||
}
|
||||
|
||||
// 1703d/AC: a trunk build adds exactly one digest, under <module>/<artifact> and a tag of its own (the digest's first 12
|
||||
// hex digits), after the build has ended, with TLS verification off (plain HTTP) and no credential; the key carries the
|
||||
// digest the registry serves and the reference pinned by it.
|
||||
func TestATrunkBuildPushesOnceAndAnswersTheRegistrysDigest(t *testing.T) {
|
||||
m, f, states := setup(t, same)
|
||||
res, st := runBuild(t, m, states, map[string]any{"publish": true})
|
||||
if st.State != delivery.StepPassed {
|
||||
t.Fatalf("step = %+v", st)
|
||||
}
|
||||
tag := "111111111111"
|
||||
if len(f.registry) != 1 || f.registry["fixture-image/server:"+tag] != imageDigest || f.pushes != 1 {
|
||||
t.Fatalf("registry = %v after %d pushes, want exactly fixture-image/server:%s", f.registry, f.pushes, tag)
|
||||
}
|
||||
if !res.Pushed || res.Digest != imageDigest || res.Reference != f.address()+"/fixture-image/server@"+imageDigest {
|
||||
t.Errorf("result = %+v", res)
|
||||
}
|
||||
push := f.called("push")
|
||||
if len(push) != 1 || !strings.Contains(push[0].line, "--tls-verify=false") || !strings.Contains(push[0].line, "docker://"+f.address()+"/fixture-image/server:"+tag) {
|
||||
t.Fatalf("push = %v", push)
|
||||
}
|
||||
for _, bad := range []string{"--creds", "--authfile", "--sign"} {
|
||||
if strings.Contains(push[0].line, bad) {
|
||||
t.Errorf("the push carries %s", bad)
|
||||
}
|
||||
}
|
||||
// The push comes after the build has ended: bud, then images, then push.
|
||||
order := []string{}
|
||||
for _, l := range f.lines() {
|
||||
for _, v := range []string{" bud ", " images ", " push ", " rmi "} {
|
||||
if strings.Contains(l, "buildah") && strings.Contains(l, v) {
|
||||
order = append(order, strings.TrimSpace(v))
|
||||
}
|
||||
}
|
||||
}
|
||||
if strings.Join(order, ",") != "bud,images,push,rmi" {
|
||||
t.Errorf("order = %v", order)
|
||||
}
|
||||
}
|
||||
|
||||
// A tag the call gives is the one used; one that is not a tag is refused.
|
||||
func TestATagOfTheCallsOwn(t *testing.T) {
|
||||
m, f, states := setup(t, same)
|
||||
if _, st := runBuild(t, m, states, map[string]any{"publish": true, "tag": "fp-abc123"}); st.State != delivery.StepPassed {
|
||||
t.Fatalf("step = %+v", st)
|
||||
}
|
||||
if _, ok := f.registry["fixture-image/server:fp-abc123"]; !ok || len(f.registry) != 1 {
|
||||
t.Errorf("registry = %v", f.registry)
|
||||
}
|
||||
m2, f2, states2 := setup(t, same)
|
||||
if _, st := runBuild(t, m2, states2, map[string]any{"publish": true, "tag": "a/b:c"}); st.State != delivery.StepFailed || len(f2.called("bud")) != 0 {
|
||||
t.Errorf("step = %+v", st)
|
||||
}
|
||||
}
|
||||
|
||||
// issue 457: a registry held still (connection refused) is waited for, and the push then goes through once.
|
||||
func TestAPushWaitsForARegistryHeldStill(t *testing.T) {
|
||||
m, f, states := setup(t, same)
|
||||
f.refuse = 2
|
||||
res, st := runBuild(t, m, states, map[string]any{"publish": true})
|
||||
if st.State != delivery.StepPassed || !res.Pushed {
|
||||
t.Fatalf("step = %+v result = %+v", st, res)
|
||||
}
|
||||
if f.pushes != 1 || len(f.registry) != 1 || len(f.called("push")) != 3 {
|
||||
t.Errorf("pushes %d, registry %v, push calls %d; want two refused and one made", f.pushes, f.registry, len(f.called("push")))
|
||||
}
|
||||
}
|
||||
|
||||
// A registry that stays down past the wait fails the step saying how long it was refused; nothing is claimed pushed.
|
||||
func TestARegistryDownPastTheWaitFailsTheStep(t *testing.T) {
|
||||
m, f, states := setup(t, same)
|
||||
f.refuse = 1 << 20
|
||||
m.RegistryWait = 50 * time.Millisecond
|
||||
_, st := runBuild(t, m, states, map[string]any{"publish": true})
|
||||
if st.State != delivery.StepFailed || !strings.Contains(st.Reason, "refused every connection") {
|
||||
t.Errorf("step = %+v", st)
|
||||
}
|
||||
if len(f.registry) != 0 {
|
||||
t.Errorf("registry = %v", f.registry)
|
||||
}
|
||||
}
|
||||
|
||||
// The digest the registry serves is the answer: when it differs from the one the push reported, the step fails.
|
||||
func TestAnAnswerThatDisagreesWithThePushFails(t *testing.T) {
|
||||
m, f, states := setup(t, same)
|
||||
run := m.Run
|
||||
m.Run = func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
|
||||
r := run(ctx, env, name, args...)
|
||||
if name == "buildah" && strings.Contains(strings.Join(args, " "), " push ") {
|
||||
f.mu.Lock()
|
||||
for k := range f.registry {
|
||||
f.registry[k] = "sha256:2222222222222222222222222222222222222222222222222222222222222222"
|
||||
}
|
||||
f.mu.Unlock()
|
||||
}
|
||||
return r
|
||||
}
|
||||
_, st := runBuild(t, m, states, map[string]any{"publish": true})
|
||||
if st.State != delivery.StepFailed || !strings.Contains(st.Reason, "the registry serves") {
|
||||
t.Errorf("step = %+v", st)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheRegistrysAddressMustBeBound(t *testing.T) {
|
||||
for _, bad := range []string{"", "${bound:artifact-store:at}:5000", ":5000"} {
|
||||
m, _, _ := setup(t, same)
|
||||
writeFile(t, m.RegistryFile, bad)
|
||||
if _, err := m.registry(); err == nil {
|
||||
t.Errorf("%q was taken as the registry's address", bad)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func writeFile(t *testing.T, path, body string) {
|
||||
t.Helper()
|
||||
if err := os.WriteFile(path, []byte(body), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// A push that succeeded wrote its digest file: one missing, unreadable or empty is a fault, and the registry's digest is
|
||||
// never answered unchecked (review round 1).
|
||||
func TestAPushWithoutAReadableDigestFileFailsTheStep(t *testing.T) {
|
||||
for _, kind := range []string{"none", "empty"} {
|
||||
m, f, states := setup(t, same)
|
||||
f.digestFile = kind
|
||||
_, st := runBuild(t, m, states, map[string]any{"publish": true})
|
||||
if st.State != delivery.StepFailed || !strings.Contains(st.Reason, "digest") {
|
||||
t.Errorf("%s: step = %+v, want failed naming the digest file", kind, st)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,164 @@
|
||||
package main
|
||||
|
||||
// The module's read tools (novox/hq issue 1562): what the module wrote to its own `steps` and `toolchains` states,
|
||||
// answered as stored. A long step ends on its key in the steps state (ADR 0469 §2) and its result, such as the
|
||||
// workspace a fetch made, is only there; a module reading its own bucket needs no grant.
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// The read tools, named as the module is: build_image_ and the verb.
|
||||
const (
|
||||
StepReadTool = "build_image_step"
|
||||
StepsReadTool = "build_image_steps"
|
||||
)
|
||||
|
||||
// StateRead reads one of the module's own states: its keys and a key's stored value (nil for a key with none).
|
||||
type StateRead struct {
|
||||
Keys func() ([]string, error)
|
||||
Get func(key string) (json.RawMessage, error)
|
||||
}
|
||||
|
||||
// readOf is the read of a state the process keeps.
|
||||
func readOf(s stdio.KeptState) StateRead {
|
||||
return StateRead{Keys: s.Keys, Get: func(key string) (json.RawMessage, error) {
|
||||
e, err := s.Get(key)
|
||||
if err != nil || e == nil {
|
||||
return nil, err
|
||||
}
|
||||
return e.Value, nil
|
||||
}}
|
||||
}
|
||||
|
||||
// stepKey is the key of a step of this node: the id alone, or the whole key <node>.<id> as a step tool answered it.
|
||||
func (m *Module) stepKey(id string) (string, error) {
|
||||
id = strings.TrimSpace(id)
|
||||
if node, rest, ok := strings.Cut(id, "."); ok {
|
||||
if node != m.Node {
|
||||
return "", fmt.Errorf("the step %q is of node %q, this module runs on %q", id, node, m.Node)
|
||||
}
|
||||
id = rest
|
||||
}
|
||||
return delivery.StepKey(m.Node, id)
|
||||
}
|
||||
|
||||
// ReadStep is the stored value of this node's step id (state, reason, result).
|
||||
func (m *Module) ReadStep(id string) (delivery.Step, error) {
|
||||
key, err := m.stepKey(id)
|
||||
if err != nil {
|
||||
return delivery.Step{}, err
|
||||
}
|
||||
if m.ReadSteps.Get == nil {
|
||||
return delivery.Step{}, fmt.Errorf("the steps state cannot be read here")
|
||||
}
|
||||
raw, err := m.ReadSteps.Get(key)
|
||||
if err != nil {
|
||||
return delivery.Step{}, err
|
||||
}
|
||||
if len(raw) == 0 {
|
||||
return delivery.Step{}, fmt.Errorf("no step %s in the steps state", key)
|
||||
}
|
||||
return delivery.DecodeStep(raw)
|
||||
}
|
||||
|
||||
// StepLine is a step without its result: the list answers these.
|
||||
type StepLine struct {
|
||||
ID string `json:"id"`
|
||||
Tool string `json:"tool"`
|
||||
State string `json:"state"`
|
||||
Started string `json:"started"`
|
||||
Ended string `json:"ended,omitempty"`
|
||||
Reason string `json:"reason,omitempty"`
|
||||
}
|
||||
|
||||
// RecentSteps are this node's steps, the latest started first, at most limit (ten when limit is not above zero).
|
||||
func (m *Module) RecentSteps(limit int) ([]StepLine, error) {
|
||||
if limit <= 0 {
|
||||
limit = 10
|
||||
}
|
||||
if m.ReadSteps.Keys == nil || m.ReadSteps.Get == nil {
|
||||
return nil, fmt.Errorf("the steps state cannot be read here")
|
||||
}
|
||||
keys, err := m.ReadSteps.Keys()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var steps []delivery.Step
|
||||
for _, k := range keys {
|
||||
if node, _, err := delivery.ParseStepKey(k); err != nil || node != m.Node {
|
||||
continue
|
||||
}
|
||||
raw, err := m.ReadSteps.Get(k)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(raw) == 0 {
|
||||
continue
|
||||
}
|
||||
s, err := delivery.DecodeStep(raw)
|
||||
if err != nil {
|
||||
continue // a value this module cannot read is not one of its own steps
|
||||
}
|
||||
steps = append(steps, s)
|
||||
}
|
||||
sort.SliceStable(steps, func(i, j int) bool { return steps[i].Started.After(steps[j].Started) })
|
||||
if len(steps) > limit {
|
||||
steps = steps[:limit]
|
||||
}
|
||||
out := make([]StepLine, 0, len(steps))
|
||||
for _, s := range steps {
|
||||
l := StepLine{ID: s.ID, Tool: s.Tool, State: s.State, Started: s.Started.UTC().Format("2006-01-02T15:04:05Z"), Reason: s.Reason}
|
||||
if !s.Ended.IsZero() {
|
||||
l.Ended = s.Ended.UTC().Format("2006-01-02T15:04:05Z")
|
||||
}
|
||||
out = append(out, l)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// ReadTools are the module's read tools. All are read-only.
|
||||
func (m *Module) ReadTools() []stdio.Tool {
|
||||
return []stdio.Tool{
|
||||
{Name: StepReadTool,
|
||||
Description: "What one step of this machine ended with, as stored in the module's steps state: its tool, state " +
|
||||
"(running, passed, failed, refused), when it started and ended, why it failed or was refused, and its result " +
|
||||
"(a fetch's workspace id is there). id is the step id a step tool answered, or the whole key <node>.<id>. Read-only.",
|
||||
Input: map[string]any{"type": "object", "required": []string{"id"}, "properties": map[string]any{
|
||||
"id": map[string]any{"type": "string", "description": "the step id, or the key <node>.<id>"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
id, _ := args["id"].(string)
|
||||
return m.ReadStep(id)
|
||||
}},
|
||||
{Name: StepsReadTool,
|
||||
Description: "This machine's latest steps from the module's steps state, newest first, each with its id, tool, state, " +
|
||||
"times and reason but not its result (read one with " + StepReadTool + "). Read-only.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{
|
||||
"limit": map[string]any{"type": "integer", "description": "how many, at most; 10 when left out"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
steps, err := m.RecentSteps(intArg(args["limit"]))
|
||||
return map[string]any{"node": m.Node, "steps": steps}, err
|
||||
}},
|
||||
}
|
||||
}
|
||||
|
||||
// intArg is a tool argument that is a number, or a string of one; zero otherwise.
|
||||
func intArg(v any) int {
|
||||
switch n := v.(type) {
|
||||
case float64:
|
||||
return int(n)
|
||||
case int:
|
||||
return n
|
||||
case string:
|
||||
i, _ := strconv.Atoi(n)
|
||||
return i
|
||||
}
|
||||
return 0
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// The read tools answer what the steps state holds: a finished build's result is there, read by the step's id.
|
||||
func TestTheReadToolsAnswerAStepsResult(t *testing.T) {
|
||||
m, _, states := setup(t, same)
|
||||
res, st := runBuild(t, m, states, nil)
|
||||
if st.State != delivery.StepPassed {
|
||||
t.Fatalf("step = %+v", st)
|
||||
}
|
||||
m.ReadSteps = StateRead{
|
||||
Keys: func() ([]string, error) {
|
||||
states.mu.Lock()
|
||||
defer states.mu.Unlock()
|
||||
var k []string
|
||||
for key := range states.steps {
|
||||
k = append(k, key)
|
||||
}
|
||||
return k, nil
|
||||
},
|
||||
Get: func(key string) (json.RawMessage, error) {
|
||||
states.mu.Lock()
|
||||
defer states.mu.Unlock()
|
||||
v := states.steps[key]
|
||||
return json.Marshal(v[len(v)-1])
|
||||
},
|
||||
}
|
||||
got, err := m.ReadStep(st.ID)
|
||||
if err != nil || got.State != delivery.StepPassed {
|
||||
t.Fatalf("ReadStep = %+v, %v", got, err)
|
||||
}
|
||||
var back BuildResult
|
||||
if err := json.Unmarshal(got.Result, &back); err != nil || back.Digest != res.Digest {
|
||||
t.Errorf("result = %s, %v", got.Result, err)
|
||||
}
|
||||
lines, err := m.RecentSteps(0)
|
||||
if err != nil || len(lines) != 1 || lines[0].Tool != BuildTool {
|
||||
t.Errorf("steps = %+v, %v", lines, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
||||
)
|
||||
|
||||
// StepRunner runs the commands of a build step with an environment built from scratch (novox/hq ADR 0466 §4, issue
|
||||
// 1485): PATH, HOME (the account's own home, where its ssh files and git configuration are) and a fixed locale, plus
|
||||
// the words the caller passes, and nothing else of this process's environment — which holds the module's bus
|
||||
// credentials and might hold a password or a token. It is mesh-sdk's clean runner (ExecRunnerClean, issue 1497) with
|
||||
// that base given; a command is bound by its context alone (the bound 0), so a fetch may outlast a tool call's thirty
|
||||
// seconds (the step has its own context).
|
||||
func StepRunner(home string) forgekey.Runner {
|
||||
run := forgekey.ExecRunnerClean(0)
|
||||
return func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
|
||||
return run(ctx, append(baseEnv(home), env...), name, args...)
|
||||
}
|
||||
}
|
||||
|
||||
// defaultPath is where a step looks for git, ssh and buildah when this process was given no PATH.
|
||||
const defaultPath = "/usr/local/bin:/usr/bin:/bin"
|
||||
|
||||
func baseEnv(home string) []string {
|
||||
path := os.Getenv("PATH")
|
||||
if path == "" {
|
||||
path = defaultPath
|
||||
}
|
||||
return []string{"PATH=" + path, "HOME=" + home, "LC_ALL=C"}
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
The stored steps (what a step ends on in the module's `steps` state) of a fetch (fetch.json), a candidate's build
|
||||
(build-candidate.json, no publish) and a trunk build (build-trunk.json, publish), as buildstep writes them, with ids, times,
|
||||
the workspace, the commit, digests and the registry's address put to fixed words (STEPID, WORKSPACE, COMMIT, REGISTRY, one
|
||||
sha256 digest). Written by capture_test.go (MESH_UPDATE_CAPTURED=1) from the unit tests' fake runner on 2026-10-22, and
|
||||
compared by bus_integration_test.go with the steps a real bus holds. mesh-delivery-builder's tests copy these files.
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"ended": "2026-10-22T08:00:01Z",
|
||||
"id": "STEPID",
|
||||
"node": "ace",
|
||||
"result": {
|
||||
"artifact": "server",
|
||||
"digest": "sha256:1111111111111111111111111111111111111111111111111111111111111111",
|
||||
"pushed": false
|
||||
},
|
||||
"started": "2026-10-22T08:00:00Z",
|
||||
"state": "passed",
|
||||
"tool": "build_image_build",
|
||||
"v": 1
|
||||
}
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"ended": "2026-10-22T08:00:01Z",
|
||||
"id": "STEPID",
|
||||
"node": "ace",
|
||||
"result": {
|
||||
"artifact": "server",
|
||||
"digest": "sha256:1111111111111111111111111111111111111111111111111111111111111111",
|
||||
"pushed": true,
|
||||
"reference": "REGISTRY/fixture-image/server@sha256:1111111111111111111111111111111111111111111111111111111111111111"
|
||||
},
|
||||
"started": "2026-10-22T08:00:00Z",
|
||||
"state": "passed",
|
||||
"tool": "build_image_build",
|
||||
"v": 1
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"ended": "2026-10-22T08:00:01Z",
|
||||
"id": "STEPID",
|
||||
"node": "ace",
|
||||
"result": {
|
||||
"commit": "COMMIT",
|
||||
"workspace": "WORKSPACE"
|
||||
},
|
||||
"started": "2026-10-22T08:00:00Z",
|
||||
"state": "passed",
|
||||
"tool": "build_image_fetch",
|
||||
"v": 1
|
||||
}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
# A fixture module: a file added to an upstream base the manifest pins in build.on.
|
||||
ARG DEMO_BASE
|
||||
FROM ${DEMO_BASE}
|
||||
COPY hello.txt /hello.txt
|
||||
+1
@@ -0,0 +1 @@
|
||||
hello from the fixture module
|
||||
Vendored
+20
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"module": "fixture-image",
|
||||
"version": "1",
|
||||
"build": {
|
||||
"on": [
|
||||
{
|
||||
"arg": "DEMO_BASE",
|
||||
"image": "docker.io/library/busybox@sha256:0000000000000000000000000000000000000000000000000000000000000001"
|
||||
}
|
||||
],
|
||||
"artifacts": [
|
||||
{
|
||||
"name": "server",
|
||||
"kind": "image",
|
||||
"from": "Dockerfile",
|
||||
"target": "final"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
)
|
||||
|
||||
// The tools this module serves. A tool name is the module's as build_image_ and its verb (a tool name has no dot).
|
||||
const (
|
||||
FetchTool = "build_image_fetch"
|
||||
BuildTool = "build_image_build"
|
||||
StatusTool = "build_image_status"
|
||||
HealthTool = "build_image_health"
|
||||
)
|
||||
|
||||
// StepTool is one step tool of the module: fetch or build. Long is a step that answers at once and ends on its key
|
||||
// (ADR 0469 §2); the others answer in their call. Both run behind the readiness guard.
|
||||
type StepTool struct {
|
||||
Name string
|
||||
Description string
|
||||
Input map[string]any
|
||||
Long bool
|
||||
Run func(ctx context.Context, args map[string]any) (any, error)
|
||||
}
|
||||
|
||||
// StepTools are the module's step tools.
|
||||
func (m *Module) StepTools() []StepTool { return []StepTool{m.fetchTool(), m.buildTool()} }
|
||||
|
||||
// Tools is every tool the module serves: its step tools behind the guard, its read tools, its status and its health.
|
||||
func (m *Module) Tools(steps []StepTool) []stdio.Tool {
|
||||
tools := []stdio.Tool{}
|
||||
for _, s := range steps {
|
||||
t := stdio.Tool{Name: s.Name, Description: s.Description, Input: s.Input}
|
||||
if s.Long {
|
||||
t.Run = m.Long(s.Name, s.Run)
|
||||
} else {
|
||||
t.Run = m.Short(s.Run)
|
||||
}
|
||||
tools = append(tools, t)
|
||||
}
|
||||
tools = append(tools, m.ReadTools()...)
|
||||
return append(tools,
|
||||
stdio.Tool{Name: StatusTool,
|
||||
Description: "What the image build module is on this machine: the buildah installed, the account's subordinate " +
|
||||
"user and group ranges, whether a user namespace can be made, and why it refuses to build if it does. Read-only.",
|
||||
Run: func(map[string]any) (any, error) { return m.Status(m.Ctx), nil }},
|
||||
stdio.Tool{Name: HealthTool,
|
||||
Description: "How the image build module is: unhealthy while buildah cannot be run, while its account holds no " +
|
||||
"subordinate user or group range, or while a user namespace cannot be made, saying which; healthy otherwise. Read-only.",
|
||||
Run: func(map[string]any) (any, error) { return m.Health(m.Ctx), nil }},
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
Host ${bound:forge-account:ssh_alias}
|
||||
HostName ${bound:forge-account:ssh_host}
|
||||
Port ${bound:forge-account:ssh_port}
|
||||
HostKeyAlias ${bound:forge-account:ssh_alias}
|
||||
User git
|
||||
IdentityFile ~/.ssh/mesh_ed25519
|
||||
IdentitiesOnly yes
|
||||
@@ -0,0 +1,2 @@
|
||||
[url "ssh://${bound:forge-account:ssh_alias}/"]
|
||||
insteadOf = ${bound:forge-account:https_base}
|
||||
@@ -0,0 +1 @@
|
||||
${bound:forge-account:ssh_alias} ${bound:forge-account:host_key}
|
||||
@@ -0,0 +1 @@
|
||||
${seat:mesh-artifact-store:reach}
|
||||
@@ -0,0 +1,22 @@
|
||||
module meshbuildimage
|
||||
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect
|
||||
github.com/google/go-tpm v0.9.8 // indirect
|
||||
github.com/klauspost/compress v1.20.0 // indirect
|
||||
github.com/minio/highwayhash v1.0.4 // indirect
|
||||
github.com/nats-io/jwt/v2 v2.8.1 // indirect
|
||||
github.com/nats-io/nkeys v0.4.16 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
golang.org/x/crypto v0.57.0 // indirect
|
||||
golang.org/x/sys v0.48.0 // indirect
|
||||
golang.org/x/time v0.15.0 // indirect
|
||||
)
|
||||
@@ -0,0 +1,27 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc h1:ZJScJSnF6LgFl55/VJ9RT/lNroVWun469sXtx0nlVYw=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY=
|
||||
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
|
||||
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4=
|
||||
github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
|
||||
github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU=
|
||||
github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg=
|
||||
github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I=
|
||||
github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24=
|
||||
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
|
||||
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
|
||||
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
|
||||
github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
|
||||
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
|
||||
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
|
||||
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
|
||||
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
|
||||
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
|
||||
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
|
||||
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
|
||||
@@ -0,0 +1,142 @@
|
||||
{
|
||||
"module": "mesh-build-image",
|
||||
"version": "1",
|
||||
"moved": true,
|
||||
"runs-as": "mesh-build-image",
|
||||
"capabilities": [
|
||||
"package-manager",
|
||||
"user-namespaces"
|
||||
],
|
||||
"state": [
|
||||
"keys",
|
||||
"steps"
|
||||
],
|
||||
"tools": [
|
||||
"build_image_fetch",
|
||||
"build_image_build",
|
||||
"build_image_step",
|
||||
"build_image_steps",
|
||||
"build_image_status",
|
||||
"build_image_health"
|
||||
],
|
||||
"requires": [
|
||||
"forge-account",
|
||||
"rootless-image-build",
|
||||
"artifact-store"
|
||||
],
|
||||
"own-secrets": {
|
||||
"broker": "${dir:state}/broker"
|
||||
},
|
||||
"secrets-owner": "mesh-build-image",
|
||||
"runtime-health": {
|
||||
"kind": "tool",
|
||||
"tool": "build_image_health",
|
||||
"interval": "60s",
|
||||
"timeout": "10s",
|
||||
"looks": 2,
|
||||
"grace": "60s"
|
||||
},
|
||||
"resources": [
|
||||
{
|
||||
"id": "account",
|
||||
"type": "user",
|
||||
"name": "mesh-build-image",
|
||||
"shell": "/usr/bin/nologin",
|
||||
"home": "/var/lib/mesh-build-image"
|
||||
},
|
||||
{
|
||||
"id": "state",
|
||||
"type": "directory",
|
||||
"mode": "0700",
|
||||
"owner": "mesh-build-image",
|
||||
"place": ".",
|
||||
"unknown": "program"
|
||||
},
|
||||
{
|
||||
"id": "work",
|
||||
"type": "directory",
|
||||
"path": "/var/lib/mesh-build-image-work",
|
||||
"mode": "0700",
|
||||
"owner": "mesh-build-image",
|
||||
"unknown": "program"
|
||||
},
|
||||
{
|
||||
"id": "store",
|
||||
"type": "directory",
|
||||
"path": "/var/lib/mesh-build-image-store",
|
||||
"mode": "0700",
|
||||
"owner": "mesh-build-image",
|
||||
"unknown": "program"
|
||||
},
|
||||
{
|
||||
"id": "ssh-dir",
|
||||
"type": "directory",
|
||||
"path": "${dir:state}/.ssh",
|
||||
"mode": "0700",
|
||||
"owner": "mesh-build-image",
|
||||
"unknown": "program"
|
||||
},
|
||||
{
|
||||
"id": "ssh-config",
|
||||
"type": "file",
|
||||
"path": "${dir:ssh-dir}/config",
|
||||
"owner": "mesh-build-image",
|
||||
"mode": "0600",
|
||||
"template": "files/forge"
|
||||
},
|
||||
{
|
||||
"id": "known-hosts",
|
||||
"type": "file",
|
||||
"path": "${dir:ssh-dir}/known_hosts",
|
||||
"owner": "mesh-build-image",
|
||||
"mode": "0644",
|
||||
"template": "files/known_hosts"
|
||||
},
|
||||
{
|
||||
"id": "git-config",
|
||||
"type": "file",
|
||||
"path": "${dir:state}/.config/git/config",
|
||||
"owner": "mesh-build-image",
|
||||
"mode": "0644",
|
||||
"template": "files/gitconfig"
|
||||
},
|
||||
{
|
||||
"id": "registry",
|
||||
"type": "file",
|
||||
"path": "${dir:state}/registry",
|
||||
"owner": "mesh-build-image",
|
||||
"mode": "0644",
|
||||
"template": "files/registry"
|
||||
},
|
||||
{
|
||||
"id": "git",
|
||||
"type": "package",
|
||||
"package": "git"
|
||||
},
|
||||
{
|
||||
"id": "openssh",
|
||||
"type": "package",
|
||||
"package": "openssh"
|
||||
}
|
||||
],
|
||||
"build": {
|
||||
"artifacts": [
|
||||
{
|
||||
"name": "tools",
|
||||
"kind": "bundle",
|
||||
"language": "go",
|
||||
"system": "arch",
|
||||
"from": "cmd/mesh-build-image",
|
||||
"binary": "mesh-build-image",
|
||||
"loads": [
|
||||
"mesh-build-image"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "files",
|
||||
"kind": "archive",
|
||||
"from": "files"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -72,6 +72,10 @@ type Change struct {
|
||||
Approved string `json:"approved,omitempty"`
|
||||
At time.Time `json:"at"`
|
||||
Moved []Moved `json:"moved"`
|
||||
// Pull and Head name the pull request a value is held for, dropped from, or applied from at its merge (novox/hq
|
||||
// ADR 0502): the actions hold and drop are always about one, and a set with a pull is its merge.
|
||||
Pull string `json:"pull,omitempty"`
|
||||
Head string `json:"head,omitempty"`
|
||||
}
|
||||
|
||||
// Config is the module: its store, its clock, and one lock, so a set and a resolution never interleave.
|
||||
@@ -86,6 +90,8 @@ type Config struct {
|
||||
|
||||
// Asks is the operator channel as this module asks on it (proposals.go); nil, nothing can be proposed.
|
||||
Asks AskBus
|
||||
// Forge is the forge as pulls are asked of it (pulls.go); nil, a merge or a close is heard from its events alone.
|
||||
Forge PullForge
|
||||
// RouterWithin and RouterEvery bound how long a proposal waits for the router's word on its ask, and how often
|
||||
// it looks; zero is RouterAnswersWithin and RouterAnswersEvery.
|
||||
RouterWithin, RouterEvery time.Duration
|
||||
@@ -109,6 +115,13 @@ type Call struct {
|
||||
// Clear asks a proposal to clear the value instead of setting it; Proposal names one proposal to read.
|
||||
Clear bool
|
||||
Proposal string
|
||||
// Pull names the open pull request that introduces the key, for propose (a held value) and pull-declares; Head is
|
||||
// the commit its declarations are read at; Declares the config declarations the head adds or changes, by module;
|
||||
// Pulls the pull requests held asks about, each at a head.
|
||||
Pull string
|
||||
Head string
|
||||
Declares map[string]any
|
||||
Pulls []PullAt
|
||||
|
||||
// approval is the operator's warrant a set or a clear is made on: given by a decided proposal alone, never by
|
||||
// a call's arguments.
|
||||
@@ -149,8 +162,13 @@ func (c *Config) Do(call Call) (any, error) {
|
||||
return c.Propose(call)
|
||||
case "proposals":
|
||||
return c.Proposals(call)
|
||||
case "pull-declares":
|
||||
return c.PullDeclares(call)
|
||||
case "held":
|
||||
return c.Held(call)
|
||||
}
|
||||
return nil, fmt.Errorf("config does set, clear, list, history, propose or proposals; %q is none of them", call.Action)
|
||||
return nil, fmt.Errorf("config does set, clear, list, history, propose, proposals, pull-declares or held; %q is "+
|
||||
"none of them", call.Action)
|
||||
}
|
||||
|
||||
// level is the level a call names, refused as ADR 0348 §3 says: a module's own value is for one node.
|
||||
@@ -178,6 +196,10 @@ func (c *Config) Change(call Call) (*SetAnswer, error) {
|
||||
if call.Clear || call.Proposal != "" {
|
||||
return nil, fmt.Errorf("clear and proposal are a proposal's: %s takes neither", call.Action)
|
||||
}
|
||||
if call.Pull != "" {
|
||||
return nil, fmt.Errorf("pull names the pull request a value is held for, which only a proposal does: %s takes "+
|
||||
"none", call.Action)
|
||||
}
|
||||
call.approval = nil // only a decided proposal gives one
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
@@ -17,6 +17,8 @@ type Listed struct {
|
||||
Predefined map[string]Published `json:"predefined,omitempty"`
|
||||
// Withheld is why the newest list of predefined keys is not published, when it is not.
|
||||
Withheld string `json:"withheld,omitempty"`
|
||||
// Held is the approved values held for open pull requests, not yet set (novox/hq ADR 0502).
|
||||
Held []HeldValue `json:"held,omitempty"`
|
||||
}
|
||||
|
||||
// SetValue is one value set, at its level.
|
||||
@@ -85,6 +87,10 @@ func (c *Config) List(call Call) (*Listed, error) {
|
||||
if call.Module == "" {
|
||||
out.Predefined = s.Predefined
|
||||
}
|
||||
out.Held = heldListed(s, call)
|
||||
if len(out.Held) == 0 {
|
||||
out.Held = nil
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -94,7 +94,7 @@ func (runtimeAsks) Warranted(ask string) error {
|
||||
}
|
||||
|
||||
func main() {
|
||||
c := &Config{Store: runtimeStore{}, Logf: logf, Asks: runtimeAsks{}}
|
||||
c := &Config{Store: runtimeStore{}, Logf: logf, Asks: runtimeAsks{}, Forge: toolForge{ask: stdio.Ask}}
|
||||
go resolving(c)
|
||||
go hearing(c)
|
||||
if err := stdio.Serve("", Tools(c)); err != nil {
|
||||
@@ -130,7 +130,14 @@ func resolving(c *Config) {
|
||||
tick := time.NewTicker(5 * time.Minute)
|
||||
defer tick.Stop()
|
||||
time.Sleep(time.Second)
|
||||
for {
|
||||
for sweep := true; ; {
|
||||
if sweep {
|
||||
// A merge or a close of a pull request a value is held for, heard nowhere, is found here (ADR 0502).
|
||||
for _, s := range c.SweepPulls() {
|
||||
logf("[mesh-config] %s", s)
|
||||
}
|
||||
}
|
||||
sweep = false
|
||||
r, err := c.Reconcile()
|
||||
switch {
|
||||
case err != nil:
|
||||
@@ -149,6 +156,7 @@ func resolving(c *Config) {
|
||||
default:
|
||||
}
|
||||
case <-tick.C:
|
||||
sweep = true
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -175,6 +183,28 @@ func hearing(c *Config) {
|
||||
logf("[mesh-config] does not hear the operator's answers yet (%v); asking again in %s", err, wait)
|
||||
}
|
||||
}()
|
||||
for _, pattern := range []string{"gitea.pull.merged", "gitea.pull.closed", "gitea.forge.pull-request.closed"} {
|
||||
go func(pattern string) {
|
||||
for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) {
|
||||
time.Sleep(wait)
|
||||
err := stdio.Subscribe(pattern, func(e stdio.Envelope) error {
|
||||
var p PullEvent
|
||||
if json.Unmarshal(e.Body, &p) != nil {
|
||||
logf("[mesh-config] %s is not readable", pattern)
|
||||
return nil
|
||||
}
|
||||
for _, s := range c.HeardPull(p, pattern == "gitea.pull.merged") {
|
||||
logf("[mesh-config] %s", s)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err == nil {
|
||||
return
|
||||
}
|
||||
logf("[mesh-config] does not hear %s yet (%v); asking again in %s", pattern, err, wait)
|
||||
}
|
||||
}(pattern)
|
||||
}
|
||||
tick := time.NewTicker(time.Minute)
|
||||
defer tick.Stop()
|
||||
for range tick.C {
|
||||
@@ -199,10 +229,13 @@ func Tools(c *Config) []stdio.Tool {
|
||||
"who and why. A trusted value is set only at the controller's terminal or on the operator's Approve: " +
|
||||
"propose asks the operator, under mesh-config's own name on a channel that proves who answered, and sets " +
|
||||
"the value on Approve if what it was shown against still stands; proposals lists them, or one with " +
|
||||
"proposal. list and history are narrowed by module, node and key.",
|
||||
"proposal. A proposal naming the open pull request that introduces its key (pull) is judged against what " +
|
||||
"that pull request declares and asked of the operator, and on Approve is held until the pull request " +
|
||||
"merges, then set; list shows the held values. pull-declares and held are the controller's. list and " +
|
||||
"history are narrowed by module, node and key.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{
|
||||
"action": map[string]any{"type": "string",
|
||||
"enum": []string{"list", "set", "clear", "history", "propose", "proposals"},
|
||||
"enum": []string{"list", "set", "clear", "history", "propose", "proposals", "pull-declares", "held"},
|
||||
"description": "list when not given"},
|
||||
"key": map[string]any{"type": "string", "description": "the config key"},
|
||||
"value": map[string]any{"description": "for set: the value, of the key's declared type"},
|
||||
@@ -215,6 +248,14 @@ func Tools(c *Config) []stdio.Tool {
|
||||
"clear": map[string]any{"type": "boolean",
|
||||
"description": "for propose: propose clearing the value at its level instead of setting one"},
|
||||
"proposal": map[string]any{"type": "string", "description": "for proposals: one proposal's id, read whole"},
|
||||
"pull": map[string]any{"type": "string", "description": "owner/repository#number: for propose, the open " +
|
||||
"pull request that introduces the key, so the value is judged against what it declares and held until " +
|
||||
"it merges (a module's own value on one node); for pull-declares, the pull request declared for"},
|
||||
"head": map[string]any{"type": "string", "description": "for pull-declares: the commit the declarations are read at"},
|
||||
"declares": map[string]any{"type": "object", "description": "for pull-declares (the controller's): the " +
|
||||
"config declarations the head adds or changes against its base, by module and key"},
|
||||
"pulls": map[string]any{"type": "array", "description": "for held (the controller's): the pull requests " +
|
||||
"asked about, each {pull, head}", "items": map[string]any{"type": "object"}},
|
||||
}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
call, err := callOf(args)
|
||||
@@ -230,7 +271,8 @@ func Tools(c *Config) []stdio.Tool {
|
||||
// callOf reads a call's arguments strictly: an argument it does not know is refused, not ignored.
|
||||
func callOf(args map[string]any) (Call, error) {
|
||||
known := map[string]bool{"action": true, "key": true, "value": true, "module": true, "node": true, "why": true,
|
||||
"by": true, "limit": true, "clear": true, "proposal": true}
|
||||
"by": true, "limit": true, "clear": true, "proposal": true, "pull": true, "head": true, "declares": true,
|
||||
"pulls": true}
|
||||
var unknown []string
|
||||
for k := range args {
|
||||
if !known[k] {
|
||||
@@ -239,7 +281,7 @@ func callOf(args map[string]any) (Call, error) {
|
||||
}
|
||||
if len(unknown) > 0 {
|
||||
return Call{}, fmt.Errorf("config takes no %s; it takes action, key, value, module, node, why, by, limit, "+
|
||||
"clear and proposal",
|
||||
"clear, proposal, pull, head, declares and pulls",
|
||||
strings.Join(unknown, ", "))
|
||||
}
|
||||
str := func(k string) (string, error) {
|
||||
@@ -256,7 +298,8 @@ func callOf(args map[string]any) (Call, error) {
|
||||
var call Call
|
||||
var problems []error
|
||||
for k, into := range map[string]*string{"action": &call.Action, "key": &call.Key, "module": &call.Module,
|
||||
"node": &call.Node, "why": &call.Why, "by": &call.By, "proposal": &call.Proposal} {
|
||||
"node": &call.Node, "why": &call.Why, "by": &call.By, "proposal": &call.Proposal, "pull": &call.Pull,
|
||||
"head": &call.Head} {
|
||||
s, err := str(k)
|
||||
problems = append(problems, err)
|
||||
*into = s
|
||||
@@ -275,6 +318,19 @@ func callOf(args map[string]any) (Call, error) {
|
||||
}
|
||||
call.Clear = b
|
||||
}
|
||||
if v, ok := args["declares"]; ok && v != nil {
|
||||
m, ok := v.(map[string]any)
|
||||
if !ok {
|
||||
problems = append(problems, errors.New("declares is an object: a module to its keys' declarations"))
|
||||
}
|
||||
call.Declares = m
|
||||
}
|
||||
if v, ok := args["pulls"]; ok && v != nil {
|
||||
raw, _ := json.Marshal(v)
|
||||
if err := json.Unmarshal(raw, &call.Pulls); err != nil {
|
||||
problems = append(problems, errors.New("pulls is a list of {pull, head}"))
|
||||
}
|
||||
}
|
||||
if v, ok := args["value"]; ok {
|
||||
// The value as given, a JSON value: a number for a number, never its text; the validation judges it.
|
||||
call.Value = v
|
||||
|
||||
@@ -97,6 +97,11 @@ type Proposal struct {
|
||||
HadValue bool `json:"had-value"`
|
||||
WasDigest string `json:"was-digest"`
|
||||
Trusted bool `json:"trusted,omitempty"`
|
||||
// Pull is the open pull request the value is held for, when it is (novox/hq ADR 0502): the value is then not set
|
||||
// on Approve but kept apart until the pull request merges, and DeclDigest is the digest of the pull request's
|
||||
// declaration of the key it was judged and shown against.
|
||||
Pull string `json:"pull,omitempty"`
|
||||
DeclDigest string `json:"declaration,omitempty"`
|
||||
// Reaches is each module on each node whose value it moves, as it was judged when proposed.
|
||||
Reaches []string `json:"reaches"`
|
||||
// From is who proposed it, as the bus proved the caller; Why is theirs; At is when.
|
||||
@@ -168,15 +173,24 @@ func (p Proposal) Fingerprint() string {
|
||||
|
||||
// about is what its ask is about: the value's place, so a newer proposal for it replaces an older one still open
|
||||
// (ADR 0277 §10, at a level, a key).
|
||||
func (p Proposal) about() string { return "config." + p.Level.ValueKey(p.Key) }
|
||||
func (p Proposal) about() string {
|
||||
if p.Pull != "" {
|
||||
return "config.held." + pullKey(p.Pull) + "." + p.Level.ValueKey(p.Key)
|
||||
}
|
||||
return "config." + p.Level.ValueKey(p.Key)
|
||||
}
|
||||
|
||||
// Act is what an answer binds: this proposal's level, key, the digest of its exact value and of the value it was
|
||||
// shown against, who proposed it and when, and the answer.
|
||||
func (p Proposal) Act(option string) asks.Act {
|
||||
return asks.Act{"verb": ProposalVerb, "option": option, "proposal": p.ID, "action": p.Action,
|
||||
act := asks.Act{"verb": ProposalVerb, "option": option, "proposal": p.ID, "action": p.Action,
|
||||
"level": p.Level.Kind, "node": p.Level.Node, "module": p.Level.Module, "key": p.Key, "value": p.Digest,
|
||||
"was": p.WasDigest, "had-value": strconv.FormatBool(p.HadValue), "from": p.From,
|
||||
"at": p.At.UTC().Format(time.RFC3339Nano)}
|
||||
if p.Pull != "" {
|
||||
act["pull"], act["declaration"] = p.Pull, p.DeclDigest
|
||||
}
|
||||
return act
|
||||
}
|
||||
|
||||
// ---- what the operator reads --------------------------------------------------------------------------
|
||||
@@ -255,6 +269,16 @@ func (p Proposal) summary() string {
|
||||
if p.Trusted {
|
||||
trusted = ", a trusted value,"
|
||||
}
|
||||
if p.Pull != "" {
|
||||
s := fmt.Sprintf("Holds %s for %s on %s for %s; approved, it takes effect when that pull request merges, "+
|
||||
"and is dropped if it closes unmerged. Nothing reaches a machine before the merge.", p.Key, p.Level.Module,
|
||||
p.Level.Node, p.Pull)
|
||||
if len([]rune(s)) > asks.SummaryLength {
|
||||
s = fmt.Sprintf("Holds %s for %s on %s for %s until it merges; dropped if it closes unmerged.", p.Key,
|
||||
p.Level.Module, p.Level.Node, p.Pull)
|
||||
}
|
||||
return s
|
||||
}
|
||||
reach := p.reaches()
|
||||
s := fmt.Sprintf("%s%s at %s; approved, it takes effect at once on every node it reaches: %s.", verb, trusted,
|
||||
p.Level, reach)
|
||||
@@ -280,6 +304,9 @@ func (p Proposal) headline() string {
|
||||
if p.Action == "clear" {
|
||||
verb = "Clear"
|
||||
}
|
||||
if p.Pull != "" {
|
||||
verb = "Hold"
|
||||
}
|
||||
where := "for the whole mesh"
|
||||
switch p.Level.Kind {
|
||||
case LevelNode:
|
||||
@@ -331,6 +358,9 @@ func (p Proposal) Ask() asks.Ask {
|
||||
a.WholeDetails = w
|
||||
}
|
||||
does := "the value is set at once, and every node it reaches takes it"
|
||||
if p.Pull != "" {
|
||||
does = "the value is held for " + p.Pull + ": it is set when that pull request merges, and dropped if it closes unmerged"
|
||||
}
|
||||
if p.Action == "clear" {
|
||||
does = "the value is cleared at once, and every node it reaches takes what is below it"
|
||||
}
|
||||
@@ -361,6 +391,8 @@ func (p Proposal) problem() string {
|
||||
return "its value is not the value its digest names: it changed after it was asked"
|
||||
case p.Action == "clear" && (len(p.Value) != 0 || p.Digest != valueDigest(nil, false)):
|
||||
return "a clear carries no value"
|
||||
case p.Pull != "" && (p.Action != "set" || p.Level.Kind != LevelModule || p.DeclDigest == ""):
|
||||
return "a value held for a pull request is a set of a module's own value, judged against its declaration"
|
||||
case valueDigest(p.Was, p.HadValue) != p.WasDigest:
|
||||
return "the value it was shown against is not the one its digest names: it changed after it was asked"
|
||||
}
|
||||
@@ -517,16 +549,41 @@ func (c *Config) propose(call Call, action string) (Proposal, asks.Ask, []string
|
||||
return Proposal{}, asks.Ask{}, nil, err
|
||||
}
|
||||
// **Judged as a set judges it, before anyone is asked** (ADR 0277 §4): no ask for a value the verb would refuse.
|
||||
r, trusted, err := c.declared(s, level, call.Key)
|
||||
if err != nil {
|
||||
return fail("%v", err)
|
||||
}
|
||||
var r rule
|
||||
var trusted bool
|
||||
var declDigest string
|
||||
vk := level.ValueKey(call.Key)
|
||||
was, had := s.Values[vk]
|
||||
if call.Pull != "" {
|
||||
// **A value held for the open pull request that introduces its key** (novox/hq ADR 0502): judged against what
|
||||
// the pull request declares at its head, as the controller told it, and asked of the operator whatever the
|
||||
// key's trust.
|
||||
if action != "set" {
|
||||
return fail("a clear is not held for a pull request: only a value to set is")
|
||||
}
|
||||
if _, _, _, err := parsePull(call.Pull); err != nil {
|
||||
return fail("%v", err)
|
||||
}
|
||||
call.Pull = strings.TrimSpace(call.Pull)
|
||||
var herr error
|
||||
r, _, declDigest, herr = heldRule(s, call, level)
|
||||
if herr != nil {
|
||||
return fail("%v", herr)
|
||||
}
|
||||
trusted = false
|
||||
k, kept := s.Kept[keptKey(call.Pull, level.Node, level.Module, call.Key)]
|
||||
was, had = Value{Value: k.Value}, kept
|
||||
} else {
|
||||
var derr error
|
||||
r, trusted, derr = c.declared(s, level, call.Key)
|
||||
if derr != nil {
|
||||
return fail("%v", derr)
|
||||
}
|
||||
}
|
||||
now := c.now()
|
||||
prop := Proposal{ID: newProposalID(), Action: action, Level: level, Key: call.Key, HadValue: had,
|
||||
WasDigest: valueDigest(was.Value, had), Trusted: trusted, From: call.Caller, Why: why, At: now,
|
||||
State: ProposalWaiting, Digest: valueDigest(nil, false)}
|
||||
State: ProposalWaiting, Digest: valueDigest(nil, false), Pull: call.Pull, DeclDigest: declDigest}
|
||||
if had {
|
||||
prop.Was = compact(was.Value)
|
||||
}
|
||||
@@ -553,15 +610,19 @@ func (c *Config) propose(call Call, action string) (Proposal, asks.Ask, []string
|
||||
if had && string(compact(was.Value)) == string(prop.Value) {
|
||||
return fail("%s at %s already holds exactly %s", call.Key, level, prop.Value)
|
||||
}
|
||||
after.Values[vk] = Value{Value: prop.Value}
|
||||
if call.Pull == "" {
|
||||
after.Values[vk] = Value{Value: prop.Value}
|
||||
}
|
||||
case "clear":
|
||||
if !had {
|
||||
return fail("nothing is set for %s at %s, so there is nothing to clear", call.Key, level)
|
||||
}
|
||||
delete(after.Values, vk)
|
||||
}
|
||||
for _, m := range diff(Resolve(s), Resolve(after)) {
|
||||
prop.Reaches = append(prop.Reaches, m.Module+" on "+m.Node)
|
||||
if call.Pull == "" {
|
||||
for _, m := range diff(Resolve(s), Resolve(after)) {
|
||||
prop.Reaches = append(prop.Reaches, m.Module+" on "+m.Node)
|
||||
}
|
||||
}
|
||||
if prop.Reaches == nil {
|
||||
prop.Reaches = []string{}
|
||||
@@ -763,6 +824,14 @@ func (c *Config) decide(p Proposal, w asks.Warrant) string {
|
||||
return fmt.Sprintf("proposal %s: approved by the operator, and not set: the decision could not be kept (%v); "+
|
||||
"it is read again on the next look", p.ID, err)
|
||||
}
|
||||
if p.Pull != "" {
|
||||
// **A held value is kept apart, never set on Approve** (ADR 0502 clause 3).
|
||||
said, err := c.keep(p)
|
||||
if err != nil {
|
||||
return keep(ProposalRefused, "approved by the operator, and refused on Approve, so nothing is held: "+err.Error())
|
||||
}
|
||||
return keep(ProposalApproved, "approved by the operator and held: "+said)
|
||||
}
|
||||
call := Call{Action: p.Action, Key: p.Key, Module: p.Level.Module, Node: p.Level.Node, Why: p.Why, Caller: p.From,
|
||||
approval: &approval{Proposal: p.ID, By: "the operator", Says: p.Approved, Was: p.Was, HadValue: p.HadValue,
|
||||
WasDigest: p.WasDigest}}
|
||||
@@ -865,6 +934,8 @@ type ProposalRow struct {
|
||||
Expires time.Time `json:"expires"`
|
||||
Fingerprint string `json:"fingerprint"`
|
||||
Approved string `json:"approved,omitempty"`
|
||||
// Pull is the pull request a value is held for, when it is.
|
||||
Pull string `json:"pull,omitempty"`
|
||||
// What one proposal read whole says besides.
|
||||
Value json.RawMessage `json:"value,omitempty"`
|
||||
Was json.RawMessage `json:"was,omitempty"`
|
||||
@@ -893,7 +964,8 @@ func (c *Config) Proposals(call Call) ([]ProposalRow, error) {
|
||||
continue
|
||||
}
|
||||
row := ProposalRow{ID: p.ID, Action: p.Action, Level: p.Level, Key: p.Key, State: p.State, Outcome: p.Outcome,
|
||||
From: p.From, At: p.At, Expires: p.At.Add(ProposalLasts), Fingerprint: p.Fingerprint(), Approved: p.Approved}
|
||||
From: p.From, At: p.At, Expires: p.At.Add(ProposalLasts), Fingerprint: p.Fingerprint(), Approved: p.Approved,
|
||||
Pull: p.Pull}
|
||||
if p.State == ProposalWaiting && c.now().After(row.Expires) {
|
||||
row.State = "expired, not yet heard"
|
||||
}
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// toolForge.Pull reads gitea_get_pull_request as the forge answers it (testdata/captured), bare or in the wrappings the
|
||||
// runtime puts around a tool's answer.
|
||||
func TestToolForgePullReadsTheCapturedAnswers(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
file string
|
||||
number int
|
||||
open, merged bool
|
||||
head string
|
||||
}{
|
||||
{"get_pull_request_closed_merged_435.json", 435, false, true, "7b5f7f84acb73aaf8189200548ff69fd9aac04b2"},
|
||||
{"get_pull_request_open_434.json", 434, true, false, "c948f3a61c74257e6e3147effb396d9d6434f491"},
|
||||
} {
|
||||
raw, err := os.ReadFile("testdata/captured/" + tc.file)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
content, _ := json.Marshal(map[string]any{"content": []map[string]any{{"type": "text", "text": string(raw)}}})
|
||||
ok, _ := json.Marshal(map[string]any{"ok": true, "output": "", "answer": json.RawMessage(raw)})
|
||||
for name, answer := range map[string]json.RawMessage{"bare": raw, "content list": content, "ok answer": ok} {
|
||||
f := toolForge{ask: func(key string, body any) (json.RawMessage, error) { return answer, nil }}
|
||||
open, merged, head, err := f.Pull("novox", "mesh-catalog", tc.number)
|
||||
if err != nil || open != tc.open || merged != tc.merged || head != tc.head {
|
||||
t.Errorf("%s (%s): open %v merged %v head %q, err %v", tc.file, name, open, merged, head, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,660 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// A config value held for the open pull request that introduces its key (novox/hq ADR 0502, issue 1826).
|
||||
//
|
||||
// A key a pull request adds is declared by no registered manifest until it merges, so a value for it, which the
|
||||
// pull request's composed check needs, was refused. The controller reads each pull request's manifests from the
|
||||
// forge at its head and tells this module the config declarations the head adds or changes (PullDeclares); mesh-config
|
||||
// reads no manifest itself (ADR 0495 §1). A proposal that names the pull request is judged against those, asked of the
|
||||
// operator whatever the key's trust, and on Approve kept apart (KeptState) from the values it resolves. The
|
||||
// controller asks for the approved held values of a check's pull requests (Held) and hands them to the composed
|
||||
// check. At the merge the value is written as the module's own value on its node (PullMerged), where it resolves
|
||||
// once the trunk declares the key; a pull request closed unmerged drops it (PullClosed). Nothing reaches a machine
|
||||
// before the merge.
|
||||
|
||||
// The states of held values, and the actions their changes carry in the history and in `config-changed`.
|
||||
const (
|
||||
PullsState = "pulls"
|
||||
KeptState = "kept"
|
||||
|
||||
ActionHold = "hold"
|
||||
ActionDrop = "drop"
|
||||
)
|
||||
|
||||
// pullName is a pull request as it is named: owner/repository#number.
|
||||
var pullName = regexp.MustCompile(`^([A-Za-z0-9_.-]+)/([A-Za-z0-9_.-]+)#([0-9]+)$`)
|
||||
|
||||
func parsePull(s string) (owner, repo string, number int, err error) {
|
||||
m := pullName.FindStringSubmatch(strings.TrimSpace(s))
|
||||
if m == nil {
|
||||
return "", "", 0, fmt.Errorf("%q is not a pull request: name it owner/repository#number", s)
|
||||
}
|
||||
n, _ := strconv.Atoi(m[3])
|
||||
if n <= 0 {
|
||||
return "", "", 0, fmt.Errorf("%q is not a pull request: its number is above zero", s)
|
||||
}
|
||||
return m[1], m[2], n, nil
|
||||
}
|
||||
|
||||
// pullKey is a pull request's key in the states.
|
||||
func pullKey(pull string) string {
|
||||
return strings.NewReplacer("/", ".", "#", ".").Replace(strings.TrimSpace(pull))
|
||||
}
|
||||
|
||||
// PullDecl is what the controller told: the config declarations a pull request's head adds or changes, by module.
|
||||
type PullDecl struct {
|
||||
Pull string `json:"pull"`
|
||||
Head string `json:"head"`
|
||||
Declares map[string]map[string]Declaration `json:"declares"`
|
||||
At time.Time `json:"at"`
|
||||
}
|
||||
|
||||
// KeptValue is an approved held value, kept apart from the values mesh-config resolves.
|
||||
type KeptValue struct {
|
||||
Pull string `json:"pull"`
|
||||
Head string `json:"head"`
|
||||
Digest string `json:"declaration"`
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
Key string `json:"key"`
|
||||
Value json.RawMessage `json:"value"`
|
||||
By string `json:"by"`
|
||||
Via string `json:"via"`
|
||||
Why string `json:"why"`
|
||||
// Approved is who approved it, as the proposal kept it.
|
||||
Approved string `json:"approved"`
|
||||
Proposal string `json:"proposal"`
|
||||
At time.Time `json:"at"`
|
||||
}
|
||||
|
||||
func keptKey(pull, node, module, key string) string {
|
||||
return pullKey(pull) + "." + node + "." + module + "." + key
|
||||
}
|
||||
|
||||
func (k KeptValue) stateKey() string { return keptKey(k.Pull, k.Node, k.Module, k.Key) }
|
||||
|
||||
func (k KeptValue) level() Level { return Level{Kind: LevelModule, Node: k.Node, Module: k.Module} }
|
||||
|
||||
// declDigest is the digest of what a declaration holds a value to: a held value stays only while it is the same.
|
||||
func declDigest(d Declaration) string {
|
||||
d.Description, d.Why, d.Kind, d.Runtime = "", "", "", false
|
||||
if !d.HasDefault() {
|
||||
d.Default = nil
|
||||
} else {
|
||||
d.Default = compact(d.Default)
|
||||
}
|
||||
raw, _ := json.Marshal(d)
|
||||
sum := sha256.Sum256(raw)
|
||||
return "sha256:" + hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
// PullForge is the forge as this module asks it of a pull request: its state, and whether it merged.
|
||||
type PullForge interface {
|
||||
Pull(owner, repo string, number int) (open, merged bool, head string, err error)
|
||||
}
|
||||
|
||||
// pulls reads what is kept for pull requests, and the held values.
|
||||
func (c *Config) readPulls(s *Sources) error {
|
||||
s.Pulls, s.Kept = map[string]PullDecl{}, map[string]KeptValue{}
|
||||
if err := c.each(PullsState, func(key string, raw []byte) error {
|
||||
var p PullDecl
|
||||
if err := json.Unmarshal(raw, &p); err != nil || p.Pull == "" {
|
||||
return nil // not one this module wrote
|
||||
}
|
||||
s.Pulls[pullKey(p.Pull)] = p
|
||||
return nil
|
||||
}); err != nil {
|
||||
return fmt.Errorf("the pull requests' declarations cannot be read: %w", err)
|
||||
}
|
||||
if err := c.each(KeptState, func(key string, raw []byte) error {
|
||||
var k KeptValue
|
||||
if err := json.Unmarshal(raw, &k); err != nil || k.Pull == "" {
|
||||
return nil
|
||||
}
|
||||
s.Kept[key] = k
|
||||
return nil
|
||||
}); err != nil {
|
||||
return fmt.Errorf("the held values cannot be read: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ---- pull-declares ---------------------------------------------------------------------------------------
|
||||
|
||||
// PullDeclaresAnswer is what pull-declares answers.
|
||||
type PullDeclaresAnswer struct {
|
||||
Said string `json:"said"`
|
||||
Pull string `json:"pull"`
|
||||
Head string `json:"head"`
|
||||
Modules []string `json:"modules"`
|
||||
Dropped []string `json:"dropped,omitempty"`
|
||||
Unsaid []string `json:"unsaid,omitempty"`
|
||||
}
|
||||
|
||||
// PullDeclares keeps what a pull request's head declares, from the controller alone, and judges again the values
|
||||
// held for it: a value whose key is declared the same (by digest) and still passes is kept at the new head; any
|
||||
// other is dropped, recorded and raised, and must be proposed again (ADR 0502 clause 6).
|
||||
func (c *Config) PullDeclares(call Call) (*PullDeclaresAnswer, error) {
|
||||
if call.Caller != ControllerCaller {
|
||||
return nil, fmt.Errorf("pull-declares is the controller's: it reads a pull request's manifests and tells "+
|
||||
"mesh-config what the head declares; %s is not the controller", call.Caller)
|
||||
}
|
||||
if _, _, _, err := parsePull(call.Pull); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
head := strings.TrimSpace(call.Head)
|
||||
if head == "" {
|
||||
return nil, errors.New("name the head the declarations are read at")
|
||||
}
|
||||
raw, err := json.Marshal(call.Declares)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the declarations cannot be read: %v", err)
|
||||
}
|
||||
declares := map[string]map[string]Declaration{}
|
||||
if call.Declares != nil {
|
||||
if err := json.Unmarshal(raw, &declares); err != nil {
|
||||
return nil, fmt.Errorf("declares is the modules' declarations, a key to a declaration for each module: %v", err)
|
||||
}
|
||||
}
|
||||
for m, keys := range declares {
|
||||
if !nodeOrModule.MatchString(m) {
|
||||
return nil, fmt.Errorf("%q is not a module's name", m)
|
||||
}
|
||||
for k, d := range keys {
|
||||
if !keyName.MatchString(k) {
|
||||
return nil, fmt.Errorf("%q is not a config key", k)
|
||||
}
|
||||
if !d.HasDefault() {
|
||||
d.Default = nil
|
||||
}
|
||||
keys[k] = d
|
||||
}
|
||||
}
|
||||
pull := strings.TrimSpace(call.Pull)
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if err := c.Store.Put(PullsState, pullKey(pull), PullDecl{Pull: pull, Head: head, Declares: declares,
|
||||
At: c.now()}); err != nil {
|
||||
return nil, fmt.Errorf("the declarations were not kept: %v", err)
|
||||
}
|
||||
a := &PullDeclaresAnswer{Pull: pull, Head: head, Modules: sortedKeys(declares)}
|
||||
var s Sources
|
||||
if err := c.readPulls(&s); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, k := range sortedKeys(s.Kept) {
|
||||
kv := s.Kept[k]
|
||||
if pullKey(kv.Pull) != pullKey(pull) {
|
||||
continue
|
||||
}
|
||||
why := ""
|
||||
d, declared := declares[kv.Module][kv.Key]
|
||||
switch {
|
||||
case !declared:
|
||||
why = "the head no longer adds or changes its declaration"
|
||||
case declDigest(d) != kv.Digest:
|
||||
why = "the head declares the key otherwise than the value was approved against"
|
||||
default:
|
||||
var v any
|
||||
_ = json.Unmarshal(kv.Value, &v)
|
||||
if p := d.rule().check(v); len(p) > 0 {
|
||||
why = "the value no longer passes: " + strings.Join(p, "; ")
|
||||
}
|
||||
}
|
||||
if why != "" {
|
||||
a.Unsaid = append(a.Unsaid, c.dropKept(kv, "the new head "+short(head)+": "+why)...)
|
||||
a.Dropped = append(a.Dropped, fmt.Sprintf("%s for %s on %s: %s", kv.Key, kv.Module, kv.Node, why))
|
||||
continue
|
||||
}
|
||||
if kv.Head != head {
|
||||
kv.Head = head
|
||||
if err := c.Store.Put(KeptState, k, kv); err != nil {
|
||||
a.Unsaid = append(a.Unsaid, "the held value "+k+" was not moved to the new head: "+err.Error())
|
||||
}
|
||||
}
|
||||
}
|
||||
a.Said = fmt.Sprintf("keeps what %s declares at %s for %d module(s)", pull, short(head), len(declares))
|
||||
if len(a.Dropped) > 0 {
|
||||
a.Said += fmt.Sprintf("; %d held value(s) dropped", len(a.Dropped))
|
||||
}
|
||||
return a, nil
|
||||
}
|
||||
|
||||
// heldChange records a hold, a drop or a merge's set of a held value in the history, the hand-act log and
|
||||
// config-changed; what could not be done is answered, not dropped.
|
||||
func (c *Config) heldChange(kv KeptValue, action, why, approved string) []string {
|
||||
level := kv.level()
|
||||
ch := Change{Action: action, Level: &level, Key: kv.Key, Value: kv.Value, By: kv.By, Via: kv.Via, Why: why,
|
||||
Approved: approved, Pull: kv.Pull, Head: kv.Head, At: c.now(), Moved: []Moved{}}
|
||||
unsaid := c.record(level.ValueKey(kv.Key), ch)
|
||||
cause := "a value held for " + kv.Pull
|
||||
what := fmt.Sprintf("config %s %s of %s for %s, held for %s", action, kv.Key, kv.Module, kv.Node, kv.Pull)
|
||||
if action != ActionDrop {
|
||||
what += " (" + string(kv.Value) + ")"
|
||||
}
|
||||
if err := c.Store.HandAct(what, why, cause); err != nil {
|
||||
unsaid = append(unsaid, "the hand-act log does not hold it: "+err.Error())
|
||||
}
|
||||
return unsaid
|
||||
}
|
||||
|
||||
// dropKept removes a held value, saying why.
|
||||
func (c *Config) dropKept(kv KeptValue, why string) []string {
|
||||
if err := c.Store.Delete(KeptState, kv.stateKey()); err != nil {
|
||||
return []string{"the held value " + kv.stateKey() + " was not dropped: " + err.Error()}
|
||||
}
|
||||
c.logf("[mesh-config] dropped %s of %s on %s held for %s: %s", kv.Key, kv.Module, kv.Node, kv.Pull, why)
|
||||
return c.heldChange(kv, ActionDrop, "dropped, nothing was set: "+why, "")
|
||||
}
|
||||
|
||||
// ---- held ------------------------------------------------------------------------------------------------
|
||||
|
||||
// HeldAnswer is what held answers: the approved held values of the pull requests asked, at the heads asked.
|
||||
type HeldAnswer struct {
|
||||
Held []HeldValue `json:"held"`
|
||||
Skipped []HeldSkipped `json:"skipped"`
|
||||
}
|
||||
|
||||
// HeldValue is one approved held value as it is handed to a composed check and listed.
|
||||
type HeldValue struct {
|
||||
Pull string `json:"pull"`
|
||||
Head string `json:"head"`
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
Key string `json:"key"`
|
||||
Value json.RawMessage `json:"value"`
|
||||
Approved string `json:"approved"`
|
||||
By string `json:"by,omitempty"`
|
||||
Why string `json:"why,omitempty"`
|
||||
At time.Time `json:"at"`
|
||||
}
|
||||
|
||||
// HeldSkipped is a pull request asked for whose held values are not answered, and why.
|
||||
type HeldSkipped struct {
|
||||
Pull string `json:"pull"`
|
||||
Why string `json:"why"`
|
||||
}
|
||||
|
||||
func heldValueOf(k KeptValue) HeldValue {
|
||||
return HeldValue{Pull: k.Pull, Head: k.Head, Node: k.Node, Module: k.Module, Key: k.Key, Value: compact(k.Value),
|
||||
Approved: k.Approved, By: k.By, Why: k.Why, At: k.At}
|
||||
}
|
||||
|
||||
// Held answers the approved held values for the pull requests asked, at the heads asked, to the controller alone: a
|
||||
// pull request at another head than the one kept is skipped and said (ADR 0502 clause 4).
|
||||
func (c *Config) Held(call Call) (*HeldAnswer, error) {
|
||||
if call.Caller != ControllerCaller {
|
||||
return nil, fmt.Errorf("held is the controller's: it hands the held values to a composed check; %s is not "+
|
||||
"the controller", call.Caller)
|
||||
}
|
||||
asked := map[string]string{}
|
||||
var order []string
|
||||
for _, p := range call.Pulls {
|
||||
if _, _, _, err := parsePull(p.Pull); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if strings.TrimSpace(p.Head) == "" {
|
||||
return nil, fmt.Errorf("name the head of %s", p.Pull)
|
||||
}
|
||||
if _, dup := asked[pullKey(p.Pull)]; !dup {
|
||||
order = append(order, strings.TrimSpace(p.Pull))
|
||||
}
|
||||
asked[pullKey(p.Pull)] = strings.TrimSpace(p.Head)
|
||||
}
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
var s Sources
|
||||
if err := c.readPulls(&s); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := &HeldAnswer{Held: []HeldValue{}, Skipped: []HeldSkipped{}}
|
||||
for _, p := range order {
|
||||
head := asked[pullKey(p)]
|
||||
var mine []KeptValue
|
||||
for _, k := range sortedKeys(s.Kept) {
|
||||
if pullKey(s.Kept[k].Pull) == pullKey(p) {
|
||||
mine = append(mine, s.Kept[k])
|
||||
}
|
||||
}
|
||||
for _, k := range mine {
|
||||
if k.Head != head {
|
||||
out.Skipped = append(out.Skipped, HeldSkipped{Pull: p, Why: fmt.Sprintf("%s for %s on %s is held "+
|
||||
"at head %s, not %s", k.Key, k.Module, k.Node, short(k.Head), short(head))})
|
||||
continue
|
||||
}
|
||||
out.Held = append(out.Held, heldValueOf(k))
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// HeldListed is the held values a listing shows, narrowed by module, node and key.
|
||||
func heldListed(s Sources, call Call) []HeldValue {
|
||||
out := []HeldValue{}
|
||||
for _, k := range sortedKeys(s.Kept) {
|
||||
kv := s.Kept[k]
|
||||
if (call.Module == "" || kv.Module == call.Module) && (call.Node == "" || kv.Node == call.Node) &&
|
||||
(call.Key == "" || kv.Key == call.Key) {
|
||||
out = append(out, heldValueOf(kv))
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// ---- the proposal's rule -------------------------------------------------------------------------------
|
||||
|
||||
// heldRule is the rule a value held for a pull request is judged by: the pull request's declaration of the key for
|
||||
// the module at its head, and the module's assignment to the node. Refused, saying what the pull request does
|
||||
// declare, when it declares none.
|
||||
func heldRule(s Sources, call Call, level Level) (rule, PullDecl, string, error) {
|
||||
if level.Kind != LevelModule {
|
||||
return rule{}, PullDecl{}, "", fmt.Errorf("a value held for a pull request is a module's own value on one "+
|
||||
"node, and %s is not one: a predefined key a pull request adds is set after its merge (novox/hq ADR 0502)",
|
||||
level)
|
||||
}
|
||||
pd, ok := s.Pulls[pullKey(call.Pull)]
|
||||
if !ok {
|
||||
return rule{}, PullDecl{}, "", fmt.Errorf("mesh-config holds no declaration for %s: the controller tells it "+
|
||||
"what a pull request declares when its head is announced, and none has been", strings.TrimSpace(call.Pull))
|
||||
}
|
||||
d, ok := pd.Declares[level.Module][call.Key]
|
||||
if !ok {
|
||||
return rule{}, PullDecl{}, "", fmt.Errorf("%s does not add or change a value %q for %s at %s; it declares "+
|
||||
"for %s: %s", pd.Pull, call.Key, level.Module, short(pd.Head), level.Module, joinKeys(pd.Declares[level.Module]))
|
||||
}
|
||||
if _, reg := s.Declared[level.Module]; !reg {
|
||||
return rule{}, PullDecl{}, "", fmt.Errorf("%s is not a registered module, so a value cannot be held for it", level.Module)
|
||||
}
|
||||
if !assigned(s, level.Node, level.Module) {
|
||||
return rule{}, PullDecl{}, "", fmt.Errorf("%s is not assigned to %s, and a module's own value is set for a node "+
|
||||
"it is assigned to (novox/hq ADR 0348 §3); it is assigned to: %s", level.Module, level.Node,
|
||||
strings.Join(nodesOf(s, level.Module), ", "))
|
||||
}
|
||||
return d.rule(), pd, declDigest(d), nil
|
||||
}
|
||||
|
||||
// keep holds an approved proposal's value apart. The caller holds the lock.
|
||||
func (c *Config) keep(p Proposal) (string, error) {
|
||||
var s Sources
|
||||
if err := c.readPulls(&s); err != nil {
|
||||
return "", err
|
||||
}
|
||||
pd, ok := s.Pulls[pullKey(p.Pull)]
|
||||
if !ok {
|
||||
return "", fmt.Errorf("mesh-config holds no declaration for %s any more", p.Pull)
|
||||
}
|
||||
d, ok := pd.Declares[p.Level.Module][p.Key]
|
||||
if !ok || declDigest(d) != p.DeclDigest {
|
||||
return "", fmt.Errorf("%s declares %s otherwise than the operator was shown, or no longer declares it: "+
|
||||
"propose it again", p.Pull, p.Key)
|
||||
}
|
||||
was, had := s.Kept[keptKey(p.Pull, p.Level.Node, p.Level.Module, p.Key)]
|
||||
if had != p.HadValue || valueDigest(was.Value, had) != p.WasDigest {
|
||||
return "", fmt.Errorf("the value held for %s changed since the operator was shown the change: propose it "+
|
||||
"again", p.Pull)
|
||||
}
|
||||
kv := KeptValue{Pull: p.Pull, Head: pd.Head, Digest: p.DeclDigest, Node: p.Level.Node, Module: p.Level.Module,
|
||||
Key: p.Key, Value: p.Value, By: "the operator", Via: p.From, Why: p.Why, Approved: p.Approved, Proposal: p.ID,
|
||||
At: c.now()}
|
||||
if err := c.Store.Put(KeptState, kv.stateKey(), kv); err != nil {
|
||||
return "", fmt.Errorf("the value was not kept: %v", err)
|
||||
}
|
||||
unsaid := c.heldChange(kv, ActionHold, "held for "+p.Pull+" until it merges: "+p.Why, p.Approved)
|
||||
said := fmt.Sprintf("%s of %s on %s is held at %s for %s until it merges", p.Key, p.Level.Module, p.Level.Node,
|
||||
short(pd.Head), p.Pull)
|
||||
if len(unsaid) > 0 {
|
||||
said += " (" + strings.Join(unsaid, "; ") + ")"
|
||||
}
|
||||
return said, nil
|
||||
}
|
||||
|
||||
// ---- the merge and the close ---------------------------------------------------------------------------
|
||||
|
||||
// PullMerged writes each value held for a pull request as the module's own value on its node, with the approval kept
|
||||
// beside it; it resolves once the trunk declares the key. It answers a line for each.
|
||||
func (c *Config) PullMerged(pull, head string) []string {
|
||||
return c.settlePull(pull, true, head)
|
||||
}
|
||||
|
||||
// PullClosed drops the values held for a pull request closed unmerged, and says that nothing was set.
|
||||
func (c *Config) PullClosed(pull string) []string {
|
||||
return c.settlePull(pull, false, "")
|
||||
}
|
||||
|
||||
func (c *Config) settlePull(pull string, merged bool, head string) []string {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
var s Sources
|
||||
if err := c.readPulls(&s); err != nil {
|
||||
return []string{err.Error()}
|
||||
}
|
||||
var said []string
|
||||
wrote := false
|
||||
for _, k := range sortedKeys(s.Kept) {
|
||||
kv := s.Kept[k]
|
||||
if pullKey(kv.Pull) != pullKey(pull) {
|
||||
continue
|
||||
}
|
||||
if !merged {
|
||||
said = append(said, fmt.Sprintf("%s of %s on %s was held for %s, which closed unmerged: dropped, "+
|
||||
"nothing was set", kv.Key, kv.Module, kv.Node, kv.Pull))
|
||||
said = append(said, c.dropKept(kv, kv.Pull+" closed unmerged")...)
|
||||
continue
|
||||
}
|
||||
// **Only a value held at the merged head, whose key that head declares as it was approved** (ADR 0502 clause 7).
|
||||
why := ""
|
||||
pd, known := s.Pulls[pullKey(pull)]
|
||||
d, declared := pd.Declares[kv.Module][kv.Key]
|
||||
switch {
|
||||
case head == "":
|
||||
why = "the merged head is not known, so the value cannot be held to it"
|
||||
case !strings.EqualFold(head, kv.Head):
|
||||
why = fmt.Sprintf("it is held at head %s and the pull request merged at %s", short(kv.Head), short(head))
|
||||
case !known || !declared || declDigest(d) != kv.Digest:
|
||||
why = "the merged head does not declare the key as the operator was shown"
|
||||
}
|
||||
if why != "" {
|
||||
said = append(said, fmt.Sprintf("%s of %s on %s was held for %s, which merged: dropped, nothing was set (%s)",
|
||||
kv.Key, kv.Module, kv.Node, kv.Pull, why))
|
||||
said = append(said, c.dropKept(kv, kv.Pull+" merged, but "+why)...)
|
||||
continue
|
||||
}
|
||||
vk := kv.level().ValueKey(kv.Key)
|
||||
v := Value{Value: compact(kv.Value), By: kv.By, Via: kv.Via, Why: kv.Why + " (held for " + kv.Pull +
|
||||
", merged)", Approved: kv.Approved, At: c.now()}
|
||||
if err := c.Store.Put(ValuesState, vk, v); err != nil {
|
||||
said = append(said, fmt.Sprintf("%s of %s on %s held for %s is not written yet: %v; it is tried "+
|
||||
"again", kv.Key, kv.Module, kv.Node, kv.Pull, err))
|
||||
continue
|
||||
}
|
||||
if err := c.Store.Delete(KeptState, k); err != nil {
|
||||
said = append(said, "the held value "+k+" is written but not removed: "+err.Error())
|
||||
}
|
||||
wrote = true
|
||||
said = append(said, c.heldChange(kv, "set", kv.Pull+" merged: the held value is the module's own value now",
|
||||
kv.Approved)...)
|
||||
said = append(said, fmt.Sprintf("%s of %s on %s held for %s is set: %s", kv.Key, kv.Module, kv.Node,
|
||||
kv.Pull, kv.Value))
|
||||
}
|
||||
if wrote {
|
||||
if s2, err := c.sources(); err == nil {
|
||||
if _, err := c.writeResolved(s2); err != nil {
|
||||
said = append(said, "the resolved values are not all written yet ("+err.Error()+"); the next "+
|
||||
"resolution writes them")
|
||||
}
|
||||
}
|
||||
}
|
||||
left := 0
|
||||
for _, k := range sortedKeys(s.Kept) {
|
||||
if pullKey(s.Kept[k].Pull) != pullKey(pull) {
|
||||
continue
|
||||
}
|
||||
if e, err := c.Store.Get(KeptState, k); err != nil || e != nil {
|
||||
left++
|
||||
}
|
||||
}
|
||||
if left == 0 {
|
||||
if err := c.Store.Delete(PullsState, pullKey(pull)); err != nil {
|
||||
said = append(said, "the declarations of "+pull+" were not removed: "+err.Error())
|
||||
}
|
||||
}
|
||||
return said
|
||||
}
|
||||
|
||||
// PullEvent is the forge's `pull.merged` or `pull.closed`, as far as this module reads it.
|
||||
type PullEvent struct {
|
||||
Owner string `json:"owner"`
|
||||
Repo string `json:"repo"`
|
||||
Number int `json:"number"`
|
||||
Merged bool `json:"merged,omitempty"`
|
||||
// HeadSHA is the head the pull request merged at (the forge's own word).
|
||||
HeadSHA string `json:"head_sha,omitempty"`
|
||||
}
|
||||
|
||||
func (p PullEvent) name() string { return fmt.Sprintf("%s/%s#%d", p.Owner, p.Repo, p.Number) }
|
||||
|
||||
// HeardPull takes the forge's word that a pull request merged or closed. A close is asked of the forge when it can
|
||||
// be, because a merged pull request may be announced as closed too.
|
||||
func (c *Config) HeardPull(p PullEvent, merged bool) []string {
|
||||
if p.Owner == "" || p.Repo == "" || p.Number <= 0 {
|
||||
return nil
|
||||
}
|
||||
head := p.HeadSHA
|
||||
if !merged && !p.Merged && c.Forge != nil {
|
||||
// A close is told from a merge by the forge; when it cannot be asked nothing is dropped, and the five-minute
|
||||
// look finds it.
|
||||
open, m, h, err := c.Forge.Pull(p.Owner, p.Repo, p.Number)
|
||||
if err != nil {
|
||||
c.logf("[mesh-config] %s closed, and the forge does not say whether it merged (%v); the next look decides", p.name(), err)
|
||||
return nil
|
||||
}
|
||||
if open {
|
||||
return nil
|
||||
}
|
||||
merged = m
|
||||
if head == "" {
|
||||
head = h
|
||||
}
|
||||
}
|
||||
return c.settlePull(p.name(), merged || p.Merged, head)
|
||||
}
|
||||
|
||||
// SweepPulls looks at the forge for every pull request it keeps something for, so a merge or a close it did not hear
|
||||
// is found: run with each five-minute resolution.
|
||||
func (c *Config) SweepPulls() []string {
|
||||
if c.Forge == nil {
|
||||
return nil
|
||||
}
|
||||
c.mu.Lock()
|
||||
var s Sources
|
||||
err := c.readPulls(&s)
|
||||
c.mu.Unlock()
|
||||
if err != nil {
|
||||
return []string{err.Error()}
|
||||
}
|
||||
seen := map[string]string{}
|
||||
for _, p := range s.Pulls {
|
||||
seen[pullKey(p.Pull)] = p.Pull
|
||||
}
|
||||
for _, k := range s.Kept {
|
||||
seen[pullKey(k.Pull)] = k.Pull
|
||||
}
|
||||
var said []string
|
||||
for _, key := range sortedKeys(seen) {
|
||||
pull := seen[key]
|
||||
owner, repo, n, err := parsePull(pull)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
open, merged, head, err := c.Forge.Pull(owner, repo, n)
|
||||
if err != nil {
|
||||
said = append(said, fmt.Sprintf("the forge does not say whether %s is open: %v", pull, err))
|
||||
continue
|
||||
}
|
||||
if !open {
|
||||
said = append(said, c.settlePull(pull, merged, head)...)
|
||||
}
|
||||
}
|
||||
return said
|
||||
}
|
||||
|
||||
// PullAt is a pull request at a head, as held asks about it.
|
||||
type PullAt struct {
|
||||
Pull string `json:"pull"`
|
||||
Head string `json:"head"`
|
||||
}
|
||||
|
||||
// toolForge asks the forge's holder through its tools.
|
||||
type toolForge struct {
|
||||
ask func(key string, body any) (json.RawMessage, error)
|
||||
}
|
||||
|
||||
// Pull asks the forge's gitea_get_pull_request whether a pull request is open, and whether it merged.
|
||||
func (f toolForge) Pull(owner, repo string, number int) (bool, bool, string, error) {
|
||||
raw, err := f.ask("gitea.gitea_get_pull_request", map[string]any{"owner": owner, "repo": repo, "number": number})
|
||||
if err != nil {
|
||||
return false, false, "", err
|
||||
}
|
||||
answer, output, ok := unwrapAnswer(raw)
|
||||
if !ok {
|
||||
return false, false, "", fmt.Errorf("the forge refused gitea_get_pull_request: %s", output)
|
||||
}
|
||||
var a struct {
|
||||
Pull struct {
|
||||
State string `json:"state"`
|
||||
Merged bool `json:"merged"`
|
||||
// head is the branch name; the commit is head_sha (the forge module's mapPull).
|
||||
HeadSHA string `json:"head_sha"`
|
||||
} `json:"pull"`
|
||||
}
|
||||
if err := json.Unmarshal(answer, &a); err != nil || a.Pull.State == "" {
|
||||
return false, false, "", fmt.Errorf("the forge's gitea_get_pull_request is not readable: %s", output)
|
||||
}
|
||||
return a.Pull.State == "open", a.Pull.Merged, a.Pull.HeadSHA, nil
|
||||
}
|
||||
|
||||
// unwrapAnswer reads a tool's answer whatever wraps it: the protocol's content list, or {output, ok, answer}.
|
||||
func unwrapAnswer(raw json.RawMessage) (json.RawMessage, string, bool) {
|
||||
var s string
|
||||
if json.Unmarshal(raw, &s) == nil {
|
||||
return unwrapAnswer(json.RawMessage(s))
|
||||
}
|
||||
var m map[string]json.RawMessage
|
||||
if json.Unmarshal(raw, &m) != nil {
|
||||
return raw, string(raw), true
|
||||
}
|
||||
if content, has := m["content"]; has {
|
||||
var items []struct {
|
||||
Text string `json:"text"`
|
||||
}
|
||||
var e struct {
|
||||
IsError bool `json:"isError"`
|
||||
}
|
||||
_ = json.Unmarshal(raw, &e)
|
||||
if json.Unmarshal(content, &items) == nil && len(items) > 0 {
|
||||
inner, out, ok := unwrapAnswer(json.RawMessage(items[0].Text))
|
||||
return inner, out, ok && !e.IsError
|
||||
}
|
||||
}
|
||||
var v struct {
|
||||
Output string `json:"output"`
|
||||
OK *bool `json:"ok"`
|
||||
Answer json.RawMessage `json:"answer"`
|
||||
}
|
||||
if json.Unmarshal(raw, &v) == nil && v.OK != nil {
|
||||
return v.Answer, v.Output, *v.OK
|
||||
}
|
||||
return raw, string(raw), true
|
||||
}
|
||||
@@ -0,0 +1,495 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Issue 1826 (novox/hq ADR 0502): a value held for the open pull request that introduces its key.
|
||||
|
||||
const thePull = "novox/mesh-catalog#434"
|
||||
|
||||
// n8nPullDeclares is what the controller tells of the pull request: n8n gets a required number media-gid.
|
||||
func n8nPullDeclares() map[string]any {
|
||||
return map[string]any{"n8n": map[string]any{"media-gid": map[string]any{"type": "number",
|
||||
"description": "the media group", "required": true, "minimum": 1, "maximum": 65535}}}
|
||||
}
|
||||
|
||||
// aHoldingMesh is a proposing mesh with n8n assigned to ace and declaring nothing on the trunk (the case of the
|
||||
// issue), and a stand-in forge.
|
||||
func aHoldingMesh(t *testing.T) (*Config, *memStore, *standIn, *standForge) {
|
||||
t.Helper()
|
||||
c, m, r := aProposingMesh(t)
|
||||
m.mirror(t, DeclaredConfigRead, "n8n", `{"module":"n8n","settings":{}}`)
|
||||
m.assign(t, "ace", "n8n")
|
||||
f := &standForge{state: map[string][2]bool{}}
|
||||
c.Forge = f
|
||||
if _, err := c.Reconcile(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m.events, m.handActs = nil, nil
|
||||
return c, m, r, f
|
||||
}
|
||||
|
||||
type standForge struct {
|
||||
state map[string][2]bool // pull -> open, merged
|
||||
head string
|
||||
err error
|
||||
}
|
||||
|
||||
func (f *standForge) Pull(owner, repo string, n int) (bool, bool, string, error) {
|
||||
if f.err != nil {
|
||||
return false, false, "", f.err
|
||||
}
|
||||
st, ok := f.state[(PullEvent{Owner: owner, Repo: repo, Number: n}).name()]
|
||||
if !ok {
|
||||
return true, false, f.head, nil
|
||||
}
|
||||
return st[0], st[1], f.head, nil
|
||||
}
|
||||
|
||||
func tell(t *testing.T, c *Config, pull, head string, declares map[string]any) *PullDeclaresAnswer {
|
||||
t.Helper()
|
||||
a, err := c.Do(Call{Action: "pull-declares", Pull: pull, Head: head, Declares: declares, Caller: ControllerCaller})
|
||||
if err != nil {
|
||||
t.Fatalf("pull-declares: %v", err)
|
||||
}
|
||||
return a.(*PullDeclaresAnswer)
|
||||
}
|
||||
|
||||
// holdIt proposes media-gid=2000 for n8n on ace held for the pull request and has the operator approve it.
|
||||
func holdIt(t *testing.T, c *Config, m *memStore, r *standIn) string {
|
||||
t.Helper()
|
||||
a := propose(t, c, Call{Module: "n8n", Node: "ace", Key: "media-gid", Value: 2000.0, Pull: thePull,
|
||||
Why: "n8n's container joins the media group, held for #434"})
|
||||
r.choose(t, a.Proposal, OptionApprove, nil)
|
||||
said, err := c.Decide()
|
||||
if err != nil || len(said) != 1 || !strings.Contains(said[0], "approved by the operator and held") {
|
||||
t.Fatalf("the Approve decided %v (%v)", said, err)
|
||||
}
|
||||
return a.Proposal
|
||||
}
|
||||
|
||||
func keptCount(m *memStore) int { k, _ := m.Keys(KeptState); return len(k) }
|
||||
|
||||
// 1826/K1: mesh-config takes what a pull request declares from the controller alone, and keeps it per pull request.
|
||||
func TestAPullRequestsDeclarationsAreTakenFromTheControllerAlone(t *testing.T) {
|
||||
c, m, _, _ := aHoldingMesh(t)
|
||||
for _, caller := range []string{"g14.node-tools", Unproven, ""} {
|
||||
if _, err := c.Do(Call{Action: "pull-declares", Pull: thePull, Head: "abc1234", Declares: n8nPullDeclares(),
|
||||
Caller: caller}); err == nil || !strings.Contains(err.Error(), "controller") {
|
||||
t.Errorf("pull-declares from %q: %v", caller, err)
|
||||
}
|
||||
}
|
||||
for _, bad := range []Call{{Pull: "mesh-catalog#434", Head: "a"}, {Pull: thePull}, {Pull: "a/b#0", Head: "a"}} {
|
||||
bad.Action, bad.Caller = "pull-declares", ControllerCaller
|
||||
if _, err := c.Do(bad); err == nil {
|
||||
t.Errorf("pull-declares %+v was taken", bad)
|
||||
}
|
||||
}
|
||||
a := tell(t, c, thePull, "abc1234def", n8nPullDeclares())
|
||||
if len(a.Modules) != 1 || a.Modules[0] != "n8n" {
|
||||
t.Errorf("answered %+v", a)
|
||||
}
|
||||
e, _ := m.Get(PullsState, "novox.mesh-catalog.434")
|
||||
var pd PullDecl
|
||||
if e == nil || json.Unmarshal(e.Value, &pd) != nil || pd.Head != "abc1234def" ||
|
||||
pd.Declares["n8n"]["media-gid"].Type != "number" || !pd.Declares["n8n"]["media-gid"].Required {
|
||||
t.Errorf("kept %s", func() string {
|
||||
if e == nil {
|
||||
return "nothing"
|
||||
}
|
||||
return string(e.Value)
|
||||
}())
|
||||
}
|
||||
// Nothing is resolved from a pull request's declaration.
|
||||
if r, _ := m.Get(ResolvedState, ResolvedKey("ace", "n8n", "media-gid")); r != nil {
|
||||
t.Errorf("a pull request's declaration was resolved: %s", r.Value)
|
||||
}
|
||||
}
|
||||
|
||||
// 1826/K2: a proposal for a key only the pull request declares is refused without its declaration, for a mesh or
|
||||
// node level, for a clear, for an undeclared key, and for a module not assigned to the node; and trunk-only
|
||||
// proposals are judged as before.
|
||||
func TestAProposalNamingAPullRequestIsJudgedAgainstItsDeclaration(t *testing.T) {
|
||||
c, m, _, _ := aHoldingMesh(t)
|
||||
try := func(call Call) error {
|
||||
call.Action, call.Caller, call.Why = "propose", "g14.node-tools", "held for 434"
|
||||
_, err := c.Propose(call)
|
||||
return err
|
||||
}
|
||||
base := Call{Module: "n8n", Node: "ace", Key: "media-gid", Value: 2000.0, Pull: thePull}
|
||||
if err := try(base); err == nil || !strings.Contains(err.Error(), "holds no declaration for "+thePull) {
|
||||
t.Errorf("without the pull request's declaration: %v", err)
|
||||
}
|
||||
// And without the pull argument it is the trunk's refusal, as before.
|
||||
plain := base
|
||||
plain.Pull = ""
|
||||
if err := try(plain); err == nil || !strings.Contains(err.Error(), `n8n declares no value "media-gid"`) {
|
||||
t.Errorf("the trunk's refusal changed: %v", err)
|
||||
}
|
||||
tell(t, c, thePull, "abc1234def", n8nPullDeclares())
|
||||
for name, tc := range map[string]struct {
|
||||
mutate func(*Call)
|
||||
want string
|
||||
}{
|
||||
"a mesh level": {func(c *Call) { c.Module, c.Node = "", "" }, "a module's own value on one node"},
|
||||
"a node level": {func(c *Call) { c.Module = "" }, "a module's own value on one node"},
|
||||
"a clear": {func(c *Call) { c.Clear, c.Value = true, nil }, "a clear is not held"},
|
||||
"a key not added": {func(c *Call) { c.Key = "other" }, "does not add or change a value"},
|
||||
"a bad value": {func(c *Call) { c.Value = 0.0 }, "below the minimum"},
|
||||
"a wrong type": {func(c *Call) { c.Value = "2000" }, "is not a number"},
|
||||
"another node": {func(c *Call) { c.Node = "g14" }, "is not assigned to g14"},
|
||||
"another pull": {func(c *Call) { c.Pull = "novox/mesh-catalog#435" }, "holds no declaration for"},
|
||||
"a malformed pull": {func(c *Call) { c.Pull = "434" }, "owner/repository#number"},
|
||||
} {
|
||||
call := base
|
||||
tc.mutate(&call)
|
||||
if err := try(call); err == nil || !strings.Contains(err.Error(), tc.want) {
|
||||
t.Errorf("%s: %v, want %q", name, err, tc.want)
|
||||
}
|
||||
}
|
||||
// A set or a clear through the verb takes no pull request.
|
||||
if _, err := c.Change(Call{Action: "set", Module: "n8n", Node: "ace", Key: "media-gid", Value: 2000.0, Pull: thePull,
|
||||
Caller: "g14.node-tools", Why: "w"}); err == nil || !strings.Contains(err.Error(), "only a proposal") {
|
||||
t.Errorf("a set naming a pull request: %v", err)
|
||||
}
|
||||
if keptCount(m) != 0 {
|
||||
t.Error("something was held")
|
||||
}
|
||||
}
|
||||
|
||||
// 1826/K2: every held value is asked whatever the key's trust; the ask says it is held and dropped on a close; on
|
||||
// Approve the value is kept apart, never set or resolved; history, event and hand-act say it; held answers it.
|
||||
func TestAHeldValueIsAskedAndKeptApartOnApprove(t *testing.T) {
|
||||
c, m, r, _ := aHoldingMesh(t)
|
||||
tell(t, c, thePull, "abc1234def", n8nPullDeclares())
|
||||
a := propose(t, c, Call{Module: "n8n", Node: "ace", Key: "media-gid", Value: 2000.0, Pull: thePull, Why: "held for #434"})
|
||||
if len(a.Reaches) != 0 {
|
||||
t.Errorf("a held value reaches %v", a.Reaches)
|
||||
}
|
||||
ask := r.asks[a.Proposal]
|
||||
if err := ask.Check(c.now()); err != nil {
|
||||
t.Fatalf("the ask: %v", err)
|
||||
}
|
||||
for _, want := range []string{thePull, "merges", "dropped if it closes unmerged", "media-gid", "n8n", "ace"} {
|
||||
if !strings.Contains(ask.Explanation, want) {
|
||||
t.Errorf("the ask does not say %q:\n%s", want, ask.Explanation)
|
||||
}
|
||||
}
|
||||
if strings.Contains(ask.Explanation, "takes effect at once") || !strings.Contains(ask.Headline, "Hold") {
|
||||
t.Errorf("the ask: %q\n%s", ask.Headline, ask.Explanation)
|
||||
}
|
||||
if ap := ask.Options[0]; !strings.Contains(ap.Does, thePull) || ap.Level == "" {
|
||||
t.Errorf("the approve option does %q", ap.Does)
|
||||
}
|
||||
if keptCount(m) != 0 {
|
||||
t.Fatal("a proposal held something")
|
||||
}
|
||||
r.choose(t, a.Proposal, OptionApprove, nil)
|
||||
said, _ := c.Decide()
|
||||
if len(said) != 1 || !strings.Contains(said[0], "held at abc1234d for "+thePull) {
|
||||
t.Fatalf("decided %v", said)
|
||||
}
|
||||
// Kept apart: not a value, not resolved.
|
||||
if _, set := storedValue(t, m, "module.ace.n8n.media-gid"); set {
|
||||
t.Error("an Approve set the value")
|
||||
}
|
||||
if e, _ := m.Get(ResolvedState, ResolvedKey("ace", "n8n", "media-gid")); e != nil {
|
||||
t.Errorf("an Approve resolved the value: %s", e.Value)
|
||||
}
|
||||
if keptCount(m) != 1 {
|
||||
t.Fatalf("%d values held", keptCount(m))
|
||||
}
|
||||
if len(m.events) != 1 || m.events[0].Action != ActionHold || m.events[0].Pull != thePull ||
|
||||
m.events[0].Key != "media-gid" || m.events[0].Head != "abc1234def" || len(m.events[0].Moved) != 0 ||
|
||||
!strings.Contains(m.events[0].Approved, "approved by the operator") {
|
||||
t.Errorf("raised %+v", m.events)
|
||||
}
|
||||
if len(m.handActs) != 1 || !strings.Contains(m.handActs[0], thePull) {
|
||||
t.Errorf("hand-acts %q", m.handActs)
|
||||
}
|
||||
h, _ := c.History(Call{Module: "n8n", Node: "ace"})
|
||||
if len(h) != 1 || h[0].Action != ActionHold || h[0].Pull != thePull {
|
||||
t.Errorf("history %+v", h)
|
||||
}
|
||||
// held answers it, at the head it is kept at, to the controller alone.
|
||||
if _, err := c.Do(Call{Action: "held", Caller: "g14.node-tools", Pulls: []PullAt{{thePull, "abc1234def"}}}); err == nil {
|
||||
t.Error("held answered a caller that is not the controller")
|
||||
}
|
||||
got, err := c.Do(Call{Action: "held", Caller: ControllerCaller, Pulls: []PullAt{{thePull, "abc1234def"}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ha := got.(*HeldAnswer)
|
||||
if len(ha.Held) != 1 || ha.Held[0].Module != "n8n" || ha.Held[0].Node != "ace" || ha.Held[0].Key != "media-gid" ||
|
||||
string(ha.Held[0].Value) != "2000" || !strings.Contains(ha.Held[0].Approved, "approved by the operator") ||
|
||||
ha.Held[0].Pull != thePull || len(ha.Skipped) != 0 {
|
||||
t.Errorf("held answered %+v", ha)
|
||||
}
|
||||
// Another head, and another pull request, are not answered.
|
||||
got, _ = c.Do(Call{Action: "held", Caller: ControllerCaller,
|
||||
Pulls: []PullAt{{thePull, "ffff0000"}, {"novox/mesh-catalog#435", "abc"}}})
|
||||
if ha = got.(*HeldAnswer); len(ha.Held) != 0 || len(ha.Skipped) != 1 || !strings.Contains(ha.Skipped[0].Why, "held at") {
|
||||
t.Errorf("held at another head answered %+v", ha)
|
||||
}
|
||||
// The listing shows it as held, and the proposal as held for the pull request.
|
||||
l, _ := c.List(Call{Module: "n8n"})
|
||||
if len(l.Held) != 1 || len(l.Set) != 0 {
|
||||
t.Errorf("the listing: held %+v set %+v", l.Held, l.Set)
|
||||
}
|
||||
rows, _ := c.Proposals(Call{})
|
||||
if len(rows) != 1 || rows[0].Pull != thePull || rows[0].State != ProposalApproved {
|
||||
t.Errorf("proposals %+v", rows)
|
||||
}
|
||||
}
|
||||
|
||||
// 1826/K2: an Approve is refused when the pull request's declaration changed after the operator was shown it.
|
||||
func TestAnApproveOfAHeldValueIsRefusedWhenTheDeclarationChanged(t *testing.T) {
|
||||
c, m, r, _ := aHoldingMesh(t)
|
||||
tell(t, c, thePull, "abc1234def", n8nPullDeclares())
|
||||
a := propose(t, c, Call{Module: "n8n", Node: "ace", Key: "media-gid", Value: 2000.0, Pull: thePull, Why: "w"})
|
||||
changed := n8nPullDeclares()
|
||||
changed["n8n"].(map[string]any)["media-gid"].(map[string]any)["maximum"] = 1000.0
|
||||
tell(t, c, thePull, "def5678abc", changed)
|
||||
r.choose(t, a.Proposal, OptionApprove, nil)
|
||||
said, _ := c.Decide()
|
||||
if len(said) != 1 || !strings.Contains(said[0], "nothing is held") || keptCount(m) != 0 {
|
||||
t.Errorf("decided %v; %d held", said, keptCount(m))
|
||||
}
|
||||
}
|
||||
|
||||
// 1826/K1: at a new head a held value is kept if its key is declared the same and it still passes; otherwise it is
|
||||
// dropped, recorded, raised and said.
|
||||
func TestANewHeadKeepsAHeldValueOnlyWhileItsDeclarationIsUnchanged(t *testing.T) {
|
||||
c, m, r, _ := aHoldingMesh(t)
|
||||
tell(t, c, thePull, "abc1234def", n8nPullDeclares())
|
||||
holdIt(t, c, m, r)
|
||||
m.events = nil
|
||||
|
||||
// The same declaration at a new head (the description may differ): kept, moved to the new head.
|
||||
same := n8nPullDeclares()
|
||||
same["n8n"].(map[string]any)["media-gid"].(map[string]any)["description"] = "reworded"
|
||||
if a := tell(t, c, thePull, "1111111111", same); len(a.Dropped) != 0 {
|
||||
t.Errorf("an unchanged declaration dropped %v", a.Dropped)
|
||||
}
|
||||
got, _ := c.Do(Call{Action: "held", Caller: ControllerCaller, Pulls: []PullAt{{thePull, "1111111111"}}})
|
||||
if len(got.(*HeldAnswer).Held) != 1 || len(m.events) != 0 {
|
||||
t.Errorf("after an unchanged head: %+v, events %+v", got, m.events)
|
||||
}
|
||||
|
||||
// A changed declaration: dropped.
|
||||
changed := n8nPullDeclares()
|
||||
changed["n8n"].(map[string]any)["media-gid"].(map[string]any)["maximum"] = 1000.0
|
||||
a := tell(t, c, thePull, "2222222222", changed)
|
||||
if len(a.Dropped) != 1 || !strings.Contains(a.Dropped[0], "declares the key otherwise") || keptCount(m) != 0 {
|
||||
t.Fatalf("a changed declaration: %+v, %d held", a, keptCount(m))
|
||||
}
|
||||
if len(m.events) != 1 || m.events[0].Action != ActionDrop || m.events[0].Pull != thePull {
|
||||
t.Errorf("raised %+v", m.events)
|
||||
}
|
||||
got, _ = c.Do(Call{Action: "held", Caller: ControllerCaller, Pulls: []PullAt{{thePull, "2222222222"}}})
|
||||
if len(got.(*HeldAnswer).Held) != 0 {
|
||||
t.Error("a dropped value was answered")
|
||||
}
|
||||
|
||||
// A head that no longer adds the key drops it too.
|
||||
tell(t, c, thePull, "3333333333", n8nPullDeclares())
|
||||
holdIt(t, c, m, r)
|
||||
if a = tell(t, c, thePull, "4444444444", map[string]any{}); len(a.Dropped) != 1 ||
|
||||
!strings.Contains(a.Dropped[0], "no longer adds or changes") {
|
||||
t.Errorf("a head without the key: %+v", a)
|
||||
}
|
||||
}
|
||||
|
||||
// 1826/K3: the merge writes the held value as the module's own value with its approval kept, before the trunk
|
||||
// declares the key; it resolves right after the declared-config mirror moves.
|
||||
func TestTheMergeAppliesAHeldValueAndItResolvesOnceTheTrunkDeclaresTheKey(t *testing.T) {
|
||||
c, m, r, _ := aHoldingMesh(t)
|
||||
tell(t, c, thePull, "abc1234def", n8nPullDeclares())
|
||||
holdIt(t, c, m, r)
|
||||
approved := func() string {
|
||||
raw, _ := m.Get(KeptState, "novox.mesh-catalog.434.ace.n8n.media-gid")
|
||||
var k KeptValue
|
||||
_ = json.Unmarshal(raw.Value, &k)
|
||||
return k.Approved
|
||||
}()
|
||||
m.events, m.handActs = nil, nil
|
||||
|
||||
said := c.HeardPull(PullEvent{Owner: "novox", Repo: "mesh-catalog", Number: 434, Merged: true, HeadSHA: "abc1234def"}, true)
|
||||
if len(said) == 0 {
|
||||
t.Fatal("the merge said nothing")
|
||||
}
|
||||
v, set := storedValue(t, m, "module.ace.n8n.media-gid")
|
||||
if !set || string(v.Value) != "2000" || v.Approved != approved || v.Approved == "" || v.By != "the operator" ||
|
||||
!strings.Contains(v.Why, thePull) {
|
||||
t.Fatalf("the merge wrote %+v", v)
|
||||
}
|
||||
if keptCount(m) != 0 {
|
||||
t.Error("the value is still held after the merge")
|
||||
}
|
||||
if p, _ := m.Keys(PullsState); len(p) != 0 {
|
||||
t.Errorf("the pull request's declarations are still kept: %v", p)
|
||||
}
|
||||
if len(m.events) != 1 || m.events[0].Action != "set" || m.events[0].Pull != thePull ||
|
||||
m.events[0].Approved != approved || len(m.handActs) != 1 {
|
||||
t.Errorf("raised %+v, recorded %q", m.events, m.handActs)
|
||||
}
|
||||
// Not resolved before the trunk declares it.
|
||||
if e, _ := m.Get(ResolvedState, ResolvedKey("ace", "n8n", "media-gid")); e != nil {
|
||||
t.Errorf("resolved before the trunk declares the key: %s", e.Value)
|
||||
}
|
||||
m.mirror(t, DeclaredConfigRead, "n8n", `{"module":"n8n","settings":{"media-gid":{"kind":"","default":null,"why":"",`+
|
||||
`"type":"number","required":true,"minimum":1,"maximum":65535}}}`)
|
||||
if _, err := c.Reconcile(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if res := m.resolved(t, "ace", "n8n", "media-gid"); string(res.Value) != "2000" || res.Source != SourceModule {
|
||||
t.Errorf("media-gid resolves as %+v", res)
|
||||
}
|
||||
l, _ := c.List(Call{Module: "n8n", Node: "ace"})
|
||||
if len(l.Set) != 1 || l.Set[0].Approved != approved || len(l.Held) != 0 {
|
||||
t.Errorf("the listing: set %+v held %+v", l.Set, l.Held)
|
||||
}
|
||||
// A merge heard twice sets nothing twice.
|
||||
m.events = nil
|
||||
c.HeardPull(PullEvent{Owner: "novox", Repo: "mesh-catalog", Number: 434, Merged: true, HeadSHA: "abc1234def"}, true)
|
||||
if len(m.events) != 0 {
|
||||
t.Errorf("a second merge raised %+v", m.events)
|
||||
}
|
||||
}
|
||||
|
||||
// 1826/K3: a pull request closed unmerged drops its held values and says so; nothing is set or resolved.
|
||||
func TestAClosedPullRequestDropsItsHeldValuesAndSaysSo(t *testing.T) {
|
||||
c, m, r, f := aHoldingMesh(t)
|
||||
tell(t, c, thePull, "abc1234def", n8nPullDeclares())
|
||||
holdIt(t, c, m, r)
|
||||
m.events = nil
|
||||
f.state[thePull] = [2]bool{false, false}
|
||||
said := c.HeardPull(PullEvent{Owner: "novox", Repo: "mesh-catalog", Number: 434}, false)
|
||||
if len(said) == 0 || !strings.Contains(strings.Join(said, "\n"), "closed unmerged: dropped, nothing was set") {
|
||||
t.Errorf("said %v", said)
|
||||
}
|
||||
if _, set := storedValue(t, m, "module.ace.n8n.media-gid"); set || keptCount(m) != 0 {
|
||||
t.Error("a closed pull request left a value")
|
||||
}
|
||||
if len(m.events) != 1 || m.events[0].Action != ActionDrop || m.events[0].Pull != thePull ||
|
||||
!strings.Contains(m.events[0].Why, "closed unmerged") {
|
||||
t.Errorf("raised %+v", m.events)
|
||||
}
|
||||
if p, _ := m.Keys(PullsState); len(p) != 0 {
|
||||
t.Errorf("declarations of a closed pull request kept: %v", p)
|
||||
}
|
||||
}
|
||||
|
||||
// 1826/K3: a close event for a pull request the forge says merged is the merge.
|
||||
func TestACloseEventForAMergedPullRequestIsItsMerge(t *testing.T) {
|
||||
c, m, r, f := aHoldingMesh(t)
|
||||
tell(t, c, thePull, "abc1234def", n8nPullDeclares())
|
||||
holdIt(t, c, m, r)
|
||||
f.state[thePull] = [2]bool{false, true}
|
||||
f.head = "abc1234def"
|
||||
c.HeardPull(PullEvent{Owner: "novox", Repo: "mesh-catalog", Number: 434}, false)
|
||||
if v, set := storedValue(t, m, "module.ace.n8n.media-gid"); !set || string(v.Value) != "2000" {
|
||||
t.Errorf("the merge of a pull request announced closed set %+v", v)
|
||||
}
|
||||
}
|
||||
|
||||
// 1826/K3: the five-minute look finds a merge, a close, and an open pull request, and says a forge that does not
|
||||
// answer.
|
||||
func TestTheLookAtTheForgeFindsAMissedMergeOrCloseAndLeavesAnOpenPullRequest(t *testing.T) {
|
||||
c, m, r, f := aHoldingMesh(t)
|
||||
tell(t, c, thePull, "abc1234def", n8nPullDeclares())
|
||||
holdIt(t, c, m, r)
|
||||
if said := c.SweepPulls(); len(said) != 0 || keptCount(m) != 1 {
|
||||
t.Errorf("an open pull request: said %v, %d held", said, keptCount(m))
|
||||
}
|
||||
f.err = errors.New("the forge is down")
|
||||
if said := c.SweepPulls(); len(said) != 1 || !strings.Contains(said[0], "does not say whether") || keptCount(m) != 1 {
|
||||
t.Errorf("a silent forge: said %v", said)
|
||||
}
|
||||
f.err = nil
|
||||
f.head = "abc1234def"
|
||||
f.state[thePull] = [2]bool{false, true}
|
||||
c.SweepPulls()
|
||||
if v, set := storedValue(t, m, "module.ace.n8n.media-gid"); !set || string(v.Value) != "2000" || keptCount(m) != 0 {
|
||||
t.Errorf("a missed merge: %+v", v)
|
||||
}
|
||||
// A missed close.
|
||||
tell(t, c, "novox/mesh-catalog#440", "abc", n8nPullDeclares())
|
||||
f.state["novox/mesh-catalog#440"] = [2]bool{false, false}
|
||||
c.SweepPulls()
|
||||
if p, _ := m.Keys(PullsState); len(p) != 0 {
|
||||
t.Errorf("a missed close left %v", p)
|
||||
}
|
||||
}
|
||||
|
||||
// 1826/K1: the new arguments are read strictly and the new actions are the verb's.
|
||||
func TestThePullArgumentsAreRead(t *testing.T) {
|
||||
call, err := callOf(map[string]any{"action": "pull-declares", "pull": " " + thePull + " ", "head": "abc",
|
||||
"declares": map[string]any{"n8n": map[string]any{}}, "pulls": []any{map[string]any{"pull": thePull, "head": "h"}}})
|
||||
if err != nil || call.Pull != thePull || call.Head != "abc" || len(call.Declares) != 1 || len(call.Pulls) != 1 ||
|
||||
call.Pulls[0].Head != "h" {
|
||||
t.Errorf("read %+v (%v)", call, err)
|
||||
}
|
||||
if _, err := callOf(map[string]any{"declares": "x"}); err == nil {
|
||||
t.Error("declares that is not an object was read")
|
||||
}
|
||||
if _, err := callOf(map[string]any{"pulls": "x"}); err == nil {
|
||||
t.Error("pulls that is not a list was read")
|
||||
}
|
||||
if _, err := callOf(map[string]any{"bogus": 1}); err == nil || !strings.Contains(err.Error(), "pull") {
|
||||
t.Errorf("an unknown argument: %v", err)
|
||||
}
|
||||
_ = time.Second
|
||||
}
|
||||
|
||||
// 1826 review r1: the merge applies a value only if it is held at the merged head; otherwise it is dropped and said.
|
||||
func TestTheMergeAppliesOnlyAValueHeldAtTheMergedHead(t *testing.T) {
|
||||
for name, head := range map[string]string{"another head": "9999999999", "no head told": ""} {
|
||||
c, m, r, _ := aHoldingMesh(t)
|
||||
tell(t, c, thePull, "abc1234def", n8nPullDeclares())
|
||||
holdIt(t, c, m, r)
|
||||
said := c.HeardPull(PullEvent{Owner: "novox", Repo: "mesh-catalog", Number: 434, Merged: true, HeadSHA: head}, true)
|
||||
if _, set := storedValue(t, m, "module.ace.n8n.media-gid"); set || keptCount(m) != 0 ||
|
||||
!strings.Contains(strings.Join(said, "\n"), "dropped, nothing was set") {
|
||||
t.Errorf("%s: set=%v held=%d said %v", name, set, keptCount(m), said)
|
||||
}
|
||||
if len(m.events) == 0 || m.events[len(m.events)-1].Action != ActionDrop {
|
||||
t.Errorf("%s: raised %+v", name, m.events)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1826 review r1: a later head without the declaration drops the value, so the merge has nothing to apply.
|
||||
func TestALaterHeadWithoutTheDeclarationLeavesNothingToApplyAtTheMerge(t *testing.T) {
|
||||
c, m, r, _ := aHoldingMesh(t)
|
||||
tell(t, c, thePull, "abc1234def", n8nPullDeclares())
|
||||
holdIt(t, c, m, r)
|
||||
tell(t, c, thePull, "bbbbbbbbbb", map[string]any{})
|
||||
c.HeardPull(PullEvent{Owner: "novox", Repo: "mesh-catalog", Number: 434, Merged: true, HeadSHA: "bbbbbbbbbb"}, true)
|
||||
if _, set := storedValue(t, m, "module.ace.n8n.media-gid"); set {
|
||||
t.Error("a value held for a declaration the merged head lacks was set")
|
||||
}
|
||||
}
|
||||
|
||||
// 1826 review r1: a close the forge cannot be asked about drops nothing; the five-minute look decides.
|
||||
func TestACloseThatTheForgeCannotBeAskedAboutDropsNothing(t *testing.T) {
|
||||
c, m, r, f := aHoldingMesh(t)
|
||||
tell(t, c, thePull, "abc1234def", n8nPullDeclares())
|
||||
holdIt(t, c, m, r)
|
||||
f.err = errors.New("the forge is down")
|
||||
c.HeardPull(PullEvent{Owner: "novox", Repo: "mesh-catalog", Number: 434}, false)
|
||||
if keptCount(m) != 1 {
|
||||
t.Fatal("a close the forge could not confirm dropped the value")
|
||||
}
|
||||
f.err = nil
|
||||
f.state[thePull] = [2]bool{false, false}
|
||||
c.SweepPulls()
|
||||
if keptCount(m) != 0 {
|
||||
t.Error("the look did not drop it")
|
||||
}
|
||||
}
|
||||
@@ -127,6 +127,10 @@ type Sources struct {
|
||||
Declared map[string]ModuleDeclarations
|
||||
Predefined map[string]Published
|
||||
Values map[string]Value
|
||||
// Pulls is what the controller told of each open pull request's declarations, and Kept the approved values held
|
||||
// for them (pulls.go); neither is resolved.
|
||||
Pulls map[string]PullDecl
|
||||
Kept map[string]KeptValue
|
||||
}
|
||||
|
||||
// Resolve is every assigned module's every declared value on its node, keyed as mesh-config_resolved is.
|
||||
|
||||
@@ -59,6 +59,9 @@ func (c *Config) sources() (Sources, error) {
|
||||
}); err != nil {
|
||||
return s, fmt.Errorf("the values cannot be read: %w", err)
|
||||
}
|
||||
if err := c.readPulls(&s); err != nil {
|
||||
return s, err
|
||||
}
|
||||
for _, p := range problems {
|
||||
c.logf("[mesh-config] a module's declarations are not read, so its values are not resolved: %s", p)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
Captured answers of the gitea module's `gitea_get_pull_request`, taken 2026-10-22 by fix-1826-catalog@shanks with
|
||||
`mesh_call gitea.gitea_get_pull_request {"owner":"novox","repo":"mesh-catalog","number":N}` through the mesh MCP server.
|
||||
|
||||
- get_pull_request_closed_merged_435.json: pull request 435, closed and merged.
|
||||
- get_pull_request_open_434.json: pull request 434, open.
|
||||
|
||||
Each file is the answer as the MCP server printed it. The runtime's own wrapping of the answer was not visible there
|
||||
(the server prints the answer alone), so the tests also feed the files through the wrappings unwrapAnswer reads.
|
||||
The `body` text is shortened to its first sentence and its Reviewed-by line (long prose of no interest here); every
|
||||
other field is as returned. Note: `head` is the branch name, the commit is `head_sha`.
|
||||
Vendored
+17
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"pull": {
|
||||
"base": "main",
|
||||
"body": "Part of design 47's milestone R4c-9 (ADR 0487): the image build module. A container image becomes one more artifact kind the builder builds through a build module, like Go and Node: mesh-build-image builds rootless with buildah, a candidate's check publishes nothing, a trunk build publishes exactly one digest and its build key names the reference. build-agent keeps building and delivering every image until R4g; nothing the operator sees changes. One delivery group on branch `feat/r4c9-image-build` in mesh-catalog and mesh-delivery. After the delivery the lead assigns buildah and mesh-build-image to shanks and ace.\n\nReviewed-by: reviewer-r4c9@shanks at 7b5f7f84acb73aaf8189200548ff69fd9aac04b2",
|
||||
"head": "feat/r4c9-image-build",
|
||||
"head_sha": "7b5f7f84acb73aaf8189200548ff69fd9aac04b2",
|
||||
"html_url": "http://127.0.0.1:20000/novox/mesh-catalog/pulls/435",
|
||||
"merge_commit_sha": "5a6d082e8c55603e47f074deb96da80fc9240ab4",
|
||||
"merged": true,
|
||||
"merged_at": "2026-10-22T08:57:10Z",
|
||||
"number": 435,
|
||||
"state": "closed",
|
||||
"title": "Add the image build module: buildah and mesh-build-image, rootless, publishing only from the trunk (issues 1702, 1703, 1704, 1707)",
|
||||
"updated_at": "2026-10-22T08:57:10Z",
|
||||
"user": "mesh-tools"
|
||||
}
|
||||
}
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"pull": {
|
||||
"base": "main",
|
||||
"body": "Part of design 47's milestone R4d: n8n and the bus run pinned upstream images (ADR 0480).\n\nReviewed-by: reviewer-r4d@shanks at c948f3a61c74257e6e3147effb396d9d6434f491\n",
|
||||
"head": "feat/r4d",
|
||||
"head_sha": "c948f3a61c74257e6e3147effb396d9d6434f491",
|
||||
"html_url": "http://127.0.0.1:20000/novox/mesh-catalog/pulls/434",
|
||||
"merged": false,
|
||||
"number": 434,
|
||||
"state": "open",
|
||||
"title": "n8n and the bus run their upstream images with the mesh's programs mounted (issues 1639, 1640, 1641, 1642, ADR 0480 §1-§3, §5-§8; ADR 0497 §5)",
|
||||
"updated_at": "2026-10-22T08:57:45Z",
|
||||
"user": "mesh-tools"
|
||||
}
|
||||
}
|
||||
@@ -17,9 +17,15 @@
|
||||
"uses": [
|
||||
"operator-channel"
|
||||
],
|
||||
"consumes": [
|
||||
"gitea.pull.merged",
|
||||
"gitea.pull.closed",
|
||||
"gitea.forge.pull-request.closed"
|
||||
],
|
||||
"invokes": [
|
||||
"seat:mesh-controller.hand-act",
|
||||
"seat:mesh-controller.warranted"
|
||||
"seat:mesh-controller.warranted",
|
||||
"gitea.gitea_get_pull_request"
|
||||
],
|
||||
"reads": [
|
||||
"mesh-controller.assignments",
|
||||
@@ -30,7 +36,9 @@
|
||||
"values",
|
||||
"resolved",
|
||||
"predefined",
|
||||
"proposals"
|
||||
"proposals",
|
||||
"pulls",
|
||||
"kept"
|
||||
],
|
||||
"logs": [
|
||||
"history"
|
||||
|
||||
@@ -0,0 +1,146 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// A config value held or dropped for a pull request asks its check again (novox/hq ADR 0502 clause 5, issue 1826).
|
||||
//
|
||||
// The controller's composed check lays the approved values held for a pull request (mesh-config's `held`) when it
|
||||
// composes, so a check run before a value was held, or after it was dropped, judged another mesh. mesh-config
|
||||
// raises `config-changed` with the action hold or drop and the pull request; this owner then asks that pull
|
||||
// request's check, or its group's composed check, again, the way it asks a moved head: its own path, with the
|
||||
// reason in the delivery's transition, and not the `recheck` verb, which stays an exception (ADR 0421 §12).
|
||||
|
||||
// ConfigChangedEvent is the event this owner follows from mesh-config.
|
||||
const ConfigChangedEvent = "mesh-config.config-changed"
|
||||
|
||||
// ConfigChange is mesh-config's `config-changed` as far as this owner reads it.
|
||||
type ConfigChange struct {
|
||||
Action string `json:"action"`
|
||||
Pull string `json:"pull,omitempty"`
|
||||
Head string `json:"head,omitempty"`
|
||||
Key string `json:"key,omitempty"`
|
||||
Level struct {
|
||||
Node string `json:"node,omitempty"`
|
||||
Module string `json:"module,omitempty"`
|
||||
} `json:"level"`
|
||||
}
|
||||
|
||||
var pullNamed = regexp.MustCompile(`^([A-Za-z0-9_.-]+)/([A-Za-z0-9_.-]+)#([0-9]+)$`)
|
||||
|
||||
// ConfigChanged takes a hold or a drop of a value for a pull request: every live delivery of that pull request has its
|
||||
// check asked again. Anything else mesh-config says (a value set, a resolution, the merge's set) asks nothing.
|
||||
func (h *Holder) ConfigChanged(c ConfigChange) {
|
||||
if c.Action != "hold" && c.Action != "drop" {
|
||||
return
|
||||
}
|
||||
m := pullNamed.FindStringSubmatch(strings.TrimSpace(c.Pull))
|
||||
if m == nil {
|
||||
return
|
||||
}
|
||||
number, _ := strconv.Atoi(m[3])
|
||||
repository := strings.ToLower(m[1] + "/" + m[2])
|
||||
verb := "held for"
|
||||
if c.Action == "drop" {
|
||||
verb = "dropped from"
|
||||
}
|
||||
why := fmt.Sprintf("a config value was %s this pull request: %s of %s on %s (mesh-config)", verb, c.Key,
|
||||
orWord(c.Level.Module, "a module"), orWord(c.Level.Node, "a node"))
|
||||
h.mu.Lock()
|
||||
var ids []string
|
||||
for id, d := range h.deliveries {
|
||||
if d.Number == number && strings.EqualFold(d.Repository, repository) && d.MergedAs == "" && !d.State.Final() {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
}
|
||||
h.mu.Unlock()
|
||||
for _, id := range ids {
|
||||
h.askAgainForConfig(id, why)
|
||||
}
|
||||
}
|
||||
|
||||
func orWord(s, word string) string {
|
||||
if s == "" {
|
||||
return word
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// askAgainForConfig asks one delivery's check again: a rejected or ready one is proposed again, as any judged again
|
||||
// head is; a proposed one has its pending ask replaced. Its group's composed check goes out of date, and is asked
|
||||
// once a member is judged again.
|
||||
func (h *Holder) askAgainForConfig(id, why string) {
|
||||
h.mu.Lock()
|
||||
d := h.deliveries[id]
|
||||
if d == nil {
|
||||
h.mu.Unlock()
|
||||
return
|
||||
}
|
||||
probe := *d
|
||||
probe.Transitions = append([]Transition(nil), d.Transitions...)
|
||||
_, refused := Apply(&probe, EvRecheck, true, Facts{Now: h.Now(), Why: why, By: byOwner})
|
||||
proposed := d.State == Proposed
|
||||
h.mu.Unlock()
|
||||
switch {
|
||||
case refused == nil:
|
||||
if _, err := h.Recheck(id, why, byOwner); err != nil {
|
||||
h.Logf("[mesh-delivery] %s: its check could not be asked again for a config value: %v", id, err)
|
||||
}
|
||||
case proposed:
|
||||
h.askProposedAgain(id, why)
|
||||
}
|
||||
}
|
||||
|
||||
// askProposedAgain asks a proposed delivery's check again, its merge check put back to pending first.
|
||||
func (h *Holder) askProposedAgain(id, why string) {
|
||||
h.mu.Lock()
|
||||
d := h.deliveries[id]
|
||||
if d == nil || d.State != Proposed {
|
||||
h.mu.Unlock()
|
||||
return
|
||||
}
|
||||
owner, repo, commit, page, number := d.Owner(), d.Repo(), d.Commit, d.HTMLURL, d.Number
|
||||
h.mu.Unlock()
|
||||
if err := h.checkingAgain(owner, repo, number, commit, page, why); err != nil {
|
||||
h.Logf("[mesh-delivery] %s: its check is not asked again for a config value: %v", id, err)
|
||||
return
|
||||
}
|
||||
h.mu.Lock()
|
||||
d = h.deliveries[id]
|
||||
if d == nil || d.State != Proposed {
|
||||
h.mu.Unlock()
|
||||
return
|
||||
}
|
||||
d.Check = nil
|
||||
if g := h.groups[d.Group]; g != nil && g.Check != nil {
|
||||
now := h.Now()
|
||||
g.Check.OutOfDate = fmt.Sprintf("composed check out of date: %s was rechecked (%s)", d.ID, why)
|
||||
g.Check.OutOfDateAt = now
|
||||
g.Updated = now
|
||||
h.keepGroup(g)
|
||||
}
|
||||
m := MemberOf(d)
|
||||
h.mu.Unlock()
|
||||
asked, err := h.Controller.Check("", []Member{m})
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
if err != nil {
|
||||
h.Logf("[mesh-delivery] %s's check cannot be asked again yet (%v); it is asked at the next look", id, err)
|
||||
h.again("a check asked again for a config value that could not be asked")
|
||||
return
|
||||
}
|
||||
d = h.deliveries[id]
|
||||
if d == nil || d.State != Proposed {
|
||||
return
|
||||
}
|
||||
d.CheckAsk = &CheckAsk{ID: asked, At: h.Now(), By: byOwner, Why: why}
|
||||
h.Logf("[mesh-delivery] %s: its check asked again as %s — %s", id, asked, why)
|
||||
d.Owed = append(d.Owed, Effect{Kind: EffectNote, Commit: d.Commit, Since: h.Now(),
|
||||
Line: oneLine(fmt.Sprintf("%s delivery %s: its check asked again by %s as %s: %s", h.Now().UTC().Format("2006-01-02T15:04:05Z07:00"),
|
||||
id, byOwner, asked, why))})
|
||||
h.keep(d)
|
||||
}
|
||||
@@ -0,0 +1,145 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
)
|
||||
|
||||
// Issue 1826 (novox/hq ADR 0502 clause 5): when mesh-config holds or drops a value for a pull request that is a
|
||||
// delivery, its check, or its group's composed check, is asked again, and the transition says why.
|
||||
|
||||
func configChanged(action, pull string) ConfigChange {
|
||||
c := ConfigChange{Action: action, Pull: pull, Head: head, Key: "media-gid"}
|
||||
c.Level.Node, c.Level.Module = "ace", "n8n"
|
||||
return c
|
||||
}
|
||||
|
||||
// 1826/K4: a lone delivery that was ready is proposed again with its merge check pending, its check asked, and the
|
||||
// transition says that a config value was held.
|
||||
func TestAHeldValueAsksTheReadyDeliverysCheckAgain(t *testing.T) {
|
||||
w := newWorld(t)
|
||||
id := heldGreen(t, w, "novox/app", 7)
|
||||
asked := len(w.ctl.checks)
|
||||
w.h.ConfigChanged(configChanged("hold", "novox/app#7"))
|
||||
if got := w.ctl.checks[asked:]; len(got) != 1 || got[0] != ":"+id {
|
||||
t.Fatalf("asked %v; wanted its own check once", got)
|
||||
}
|
||||
d := w.delivery(id)
|
||||
last := d.Transitions[len(d.Transitions)-1]
|
||||
if d.State != Proposed || last.Event != EvRecheck || !strings.Contains(last.Why, "media-gid") ||
|
||||
!strings.Contains(last.Why, "held") || last.By != byOwner {
|
||||
t.Errorf("state %s, last transition %+v", d.State, last)
|
||||
}
|
||||
if s := w.forge.statuses["novox/app@"+head+" "+ctxGate]; !strings.HasPrefix(s, "pending checking again") {
|
||||
t.Errorf("the merge check says %q: the old verdict would still let it merge", s)
|
||||
}
|
||||
}
|
||||
|
||||
// 1826/K4: a rejected delivery (the very case: the composed check failed on the missing value) is asked again on a
|
||||
// hold; and a drop asks it again too.
|
||||
func TestAHeldValueAsksARejectedDeliverysCheckAgainAndADropToo(t *testing.T) {
|
||||
w := newWorld(t)
|
||||
w.h.PullUpdated(pr("novox/app", 8, head, "feat/y", "src/a.go"))
|
||||
w.h.Checked(verdict("novox/app", 8, head, "fail"))
|
||||
w.settleAll()
|
||||
id := IDOf("novox/app", head)
|
||||
if w.state(id) != Rejected {
|
||||
t.Fatalf("it is %s", w.state(id))
|
||||
}
|
||||
asked := len(w.ctl.checks)
|
||||
w.h.ConfigChanged(configChanged("hold", "novox/app#8"))
|
||||
if w.state(id) != Proposed || len(w.ctl.checks) != asked+1 {
|
||||
t.Fatalf("a hold left it %s with %v asked", w.state(id), w.ctl.checks[asked:])
|
||||
}
|
||||
w.h.Checked(verdict("novox/app", 8, head, "pass"))
|
||||
w.settleAll()
|
||||
asked = len(w.ctl.checks)
|
||||
w.h.ConfigChanged(configChanged("drop", "novox/app#8"))
|
||||
d := w.delivery(id)
|
||||
if len(w.ctl.checks) != asked+1 || !strings.Contains(d.Transitions[len(d.Transitions)-1].Why, "dropped") {
|
||||
t.Errorf("a drop asked %v; last transition %+v", w.ctl.checks[asked:], d.Transitions[len(d.Transitions)-1])
|
||||
}
|
||||
}
|
||||
|
||||
// 1826/K4: a delivery whose check is on its way is asked again, its old ask replaced, and says why in its note.
|
||||
func TestAHeldValueAsksTheCheckOfAProposedDeliveryAgain(t *testing.T) {
|
||||
w := newWorld(t)
|
||||
w.h.PullUpdated(pr("novox/app", 9, head, "feat/z", "src/a.go"))
|
||||
w.settleAll()
|
||||
id := IDOf("novox/app", head)
|
||||
before := w.delivery(id).CheckAsk
|
||||
asked := len(w.ctl.checks)
|
||||
w.h.ConfigChanged(configChanged("hold", "novox/app#9"))
|
||||
d := w.delivery(id)
|
||||
if len(w.ctl.checks) != asked+1 || d.CheckAsk == nil || (before != nil && d.CheckAsk.ID == before.ID) ||
|
||||
!strings.Contains(d.CheckAsk.Why, "held") {
|
||||
t.Errorf("asked %v; ask %+v (was %+v)", w.ctl.checks[asked:], d.CheckAsk, before)
|
||||
}
|
||||
if d.State != Proposed {
|
||||
t.Errorf("it is %s", d.State)
|
||||
}
|
||||
}
|
||||
|
||||
// 1826/K4: a hold for a member of a group asks the member's check again and then the group's composed check, which
|
||||
// the group waits for.
|
||||
func TestAHeldValueForAGroupMemberAsksTheComposedCheckAgain(t *testing.T) {
|
||||
w := approvedGroup(t)
|
||||
g := w.h.groups["feat/g"]
|
||||
firstComposed := g.Check.Asked
|
||||
asked := len(w.ctl.checks)
|
||||
w.h.ConfigChanged(configChanged("hold", gCat+"#2"))
|
||||
if got := w.ctl.checks[asked:]; len(got) != 1 || got[0] != ":"+IDOf(gCat, gHk) {
|
||||
t.Fatalf("a hold asked %v; wanted the member's own check", got)
|
||||
}
|
||||
if g = w.h.groups["feat/g"]; g.Check.OutOfDate == "" || !strings.Contains(g.Check.OutOfDate, "rechecked") {
|
||||
t.Fatalf("the composed check stands: %+v", g.Check)
|
||||
}
|
||||
w.later(60e9)
|
||||
w.h.Checked(judged(gCat, 2, gHk, "pass", gTrunk0))
|
||||
w.settleAll()
|
||||
last := w.ctl.checks[len(w.ctl.checks)-1]
|
||||
if !strings.HasPrefix(last, "feat/g:") || w.h.groups["feat/g"].Check.Asked == firstComposed {
|
||||
t.Errorf("the composed check was not asked again: %v", w.ctl.checks[asked:])
|
||||
}
|
||||
}
|
||||
|
||||
// 1826/K4: what is not a hold or a drop for a delivery of this owner asks nothing: the merge's set, another action,
|
||||
// another pull request, a pull request with no delivery, and a delivery already merged.
|
||||
func TestOnlyAHoldOrADropForADeliveryAsksACheckAgain(t *testing.T) {
|
||||
w := newWorld(t)
|
||||
heldGreen(t, w, "novox/app", 7)
|
||||
asked := len(w.ctl.checks)
|
||||
for _, c := range []ConfigChange{configChanged("set", "novox/app#7"), configChanged("resolve", ""),
|
||||
configChanged("hold", "novox/app#99"), configChanged("hold", "novox/other#7"), configChanged("hold", ""),
|
||||
configChanged("hold", "garbage")} {
|
||||
w.h.ConfigChanged(c)
|
||||
}
|
||||
if len(w.ctl.checks) != asked || w.state(IDOf("novox/app", head)) != Ready {
|
||||
t.Errorf("asked %v; it is %s", w.ctl.checks[asked:], w.state(IDOf("novox/app", head)))
|
||||
}
|
||||
}
|
||||
|
||||
// 1826/K4: the event is heard as mesh-config's config-changed, and it is no recheck verb.
|
||||
func TestTheConfigChangedEventIsFollowed(t *testing.T) {
|
||||
w := newWorld(t)
|
||||
heldGreen(t, w, "novox/app", 7)
|
||||
found := false
|
||||
for _, f := range followed {
|
||||
found = found || f == "mesh-config.config-changed"
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("not followed: %v", followed)
|
||||
}
|
||||
raw, _ := json.Marshal(map[string]any{"action": "hold", "pull": "novox/app#7", "head": head, "key": "media-gid",
|
||||
"level": map[string]any{"level": "module", "node": "ace", "module": "n8n"}})
|
||||
asked := len(w.ctl.checks)
|
||||
if err := handle(w.h, stdio.Envelope{Key: "mesh-config.config-changed", Body: raw}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(w.ctl.checks) != asked+1 {
|
||||
t.Errorf("the event asked %v", w.ctl.checks[asked:])
|
||||
}
|
||||
}
|
||||
@@ -34,7 +34,7 @@ func (l *listening) get() string { l.mu.Lock(); defer l.mu.Unlock(); return l.n
|
||||
// The forge's status events say that a check answered (events.go).
|
||||
var followed = []string{"gitea.pull.updated", "gitea.pull.merged", "gitea.pull.closed",
|
||||
"gitea.forge.status", "gitea.forge.pull-request.edited", "gitea.forge.pull-request.closed", ControllerSeat + ".checked", ControllerSeat + ".plan-moved",
|
||||
forgePattern}
|
||||
forgePattern, ConfigChangedEvent}
|
||||
|
||||
// forgePattern is the forge bridge's messages on FORGE (novox/hq ADR 0437 §8): the gate follows a merge and the refusal of one.
|
||||
const forgePattern = "mesh-delivery-forge-bridge.forge.**"
|
||||
@@ -187,6 +187,13 @@ func take(h *Holder, e stdio.Envelope) error {
|
||||
}
|
||||
h.Checked(c)
|
||||
h.KeepVerdict(c)
|
||||
case ConfigChangedEvent:
|
||||
var c ConfigChange
|
||||
if err := json.Unmarshal(e.Body, &c); err != nil {
|
||||
h.Refuse(fmt.Errorf("a config change is not readable: %v", err))
|
||||
return nil
|
||||
}
|
||||
h.ConfigChanged(c)
|
||||
case ControllerSeat + ".plan-moved":
|
||||
var w Walk
|
||||
if err := json.Unmarshal(e.Body, &w); err != nil {
|
||||
|
||||
@@ -32,7 +32,8 @@
|
||||
"gitea.forge.pull-request.closed",
|
||||
"mesh-controller.checked",
|
||||
"mesh-controller.plan-moved",
|
||||
"mesh-delivery-forge-bridge.forge.**"
|
||||
"mesh-delivery-forge-bridge.forge.**",
|
||||
"mesh-config.config-changed"
|
||||
],
|
||||
"emits": [
|
||||
"transition",
|
||||
|
||||
@@ -222,3 +222,51 @@ func TestTheOwnersDefaultIsTheTemplatesOwners(t *testing.T) {
|
||||
t.Fatalf("the owners default (%d) and the template's owners (%d) differ", len(m.Config.Owners.Default), len(s.Owners))
|
||||
}
|
||||
}
|
||||
|
||||
// Every module folder of the catalogue that holds code the mesh builds (a go.mod or a package.json) is a
|
||||
// registered owner, so an issue is located where it lives and the next new module cannot be missed
|
||||
// (novox/hq issue 1815: mesh-build-go and mesh-build-node were not on the list).
|
||||
// 1815/AC1
|
||||
func TestEveryCodeModuleOfTheCatalogueIsAnOwner(t *testing.T) {
|
||||
_, _ = readManifest(t)
|
||||
tmpl, err := os.ReadFile(filepath.Join("..", "..", "files", "settings.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var s Settings
|
||||
if err := json.Unmarshal(tmpl, &s); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
modules := filepath.Join("..", "..", "..")
|
||||
dirs, err := os.ReadDir(modules)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
found := 0
|
||||
for _, d := range dirs {
|
||||
if !d.IsDir() {
|
||||
continue
|
||||
}
|
||||
for _, marker := range []string{"go.mod", "package.json"} {
|
||||
if _, err := os.Stat(filepath.Join(modules, d.Name(), marker)); err != nil {
|
||||
continue
|
||||
}
|
||||
found++
|
||||
if !slices.Contains(s.Owners, "mesh-catalog modules/"+d.Name()) {
|
||||
t.Errorf("modules/%s holds code (%s) but %q is not among the owners: add it to module.json and files/settings.json",
|
||||
d.Name(), marker, "mesh-catalog modules/"+d.Name())
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
if found < 10 {
|
||||
t.Fatalf("only %d code modules found under %s: the test looks in the wrong place", found, modules)
|
||||
}
|
||||
// 1815/AC2: the delivery repository's modules cannot ride on its repository entry (owns refuses a folder
|
||||
// of a repository that is registered only as a whole), so each is named.
|
||||
for _, m := range []string{"builder", "checker", "indexer"} {
|
||||
if Owners(s.Owners).owns("mesh-delivery modules/mesh-delivery-"+m) == "" {
|
||||
t.Errorf("mesh-delivery-%s is not an owner", m)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,6 +12,7 @@
|
||||
"mesh-catalog modules/betterbird",
|
||||
"mesh-catalog modules/blueman",
|
||||
"mesh-catalog modules/bluetooth",
|
||||
"mesh-catalog modules/brother",
|
||||
"mesh-catalog modules/ca-trust",
|
||||
"mesh-catalog modules/claude-code",
|
||||
"mesh-catalog modules/claude-licence-manager",
|
||||
@@ -45,6 +46,7 @@
|
||||
"mesh-catalog modules/influxdb",
|
||||
"mesh-catalog modules/invoicing",
|
||||
"mesh-catalog modules/jira",
|
||||
"mesh-catalog modules/kernel",
|
||||
"mesh-catalog modules/keycloak",
|
||||
"mesh-catalog modules/lab",
|
||||
"mesh-catalog modules/lemurs",
|
||||
@@ -56,11 +58,24 @@
|
||||
"mesh-catalog modules/marrytts",
|
||||
"mesh-catalog modules/matrix",
|
||||
"mesh-catalog modules/memory-pressure",
|
||||
"mesh-catalog modules/mesh-build-go",
|
||||
"mesh-catalog modules/mesh-build-image",
|
||||
"mesh-catalog modules/mesh-build-node",
|
||||
"mesh-catalog modules/mesh-catalog",
|
||||
"mesh-catalog modules/mesh-cli",
|
||||
"mesh-catalog modules/mesh-config",
|
||||
"mesh-catalog modules/mesh-delivery",
|
||||
"mesh-catalog modules/mesh-delivery-gate",
|
||||
"mesh-catalog modules/mesh-files",
|
||||
"mesh-catalog modules/mesh-issues",
|
||||
"mesh-catalog modules/mesh-knowledge",
|
||||
"mesh-catalog modules/mesh-mail",
|
||||
"mesh-catalog modules/mesh-mail-notifier",
|
||||
"mesh-catalog modules/mesh-mail-reader",
|
||||
"mesh-catalog modules/mesh-mail-web",
|
||||
"mesh-catalog modules/mesh-provisioner",
|
||||
"mesh-catalog modules/mesh-provisioner-client",
|
||||
"mesh-catalog modules/mesh-reads",
|
||||
"mesh-catalog modules/mesh-vault",
|
||||
"mesh-catalog modules/mesh-watcher",
|
||||
"mesh-catalog modules/messenger",
|
||||
@@ -130,11 +145,19 @@
|
||||
"mesh-controller modules/build-agent",
|
||||
"mesh-controller modules/route-proxy",
|
||||
"mesh-delivery",
|
||||
"mesh-delivery modules/mesh-delivery-builder",
|
||||
"mesh-delivery modules/mesh-delivery-checker",
|
||||
"mesh-delivery modules/mesh-delivery-forge-bridge",
|
||||
"mesh-delivery modules/mesh-delivery-indexer",
|
||||
"mesh-delivery modules/mesh-delivery-teller",
|
||||
"mesh-delivery modules/mesh-delivery-view",
|
||||
"mesh-delivery modules/mesh-delivery-watchman",
|
||||
"mesh-host",
|
||||
"mesh-lab",
|
||||
"mesh-media-catalog",
|
||||
"mesh-sdk",
|
||||
"mesh-tools"
|
||||
"mesh-tools",
|
||||
"okioki"
|
||||
],
|
||||
"duplicate-check": "on",
|
||||
"judge-model": "sonnet",
|
||||
|
||||
@@ -105,6 +105,7 @@
|
||||
"mesh-catalog modules/betterbird",
|
||||
"mesh-catalog modules/blueman",
|
||||
"mesh-catalog modules/bluetooth",
|
||||
"mesh-catalog modules/brother",
|
||||
"mesh-catalog modules/ca-trust",
|
||||
"mesh-catalog modules/claude-code",
|
||||
"mesh-catalog modules/claude-licence-manager",
|
||||
@@ -138,6 +139,7 @@
|
||||
"mesh-catalog modules/influxdb",
|
||||
"mesh-catalog modules/invoicing",
|
||||
"mesh-catalog modules/jira",
|
||||
"mesh-catalog modules/kernel",
|
||||
"mesh-catalog modules/keycloak",
|
||||
"mesh-catalog modules/lab",
|
||||
"mesh-catalog modules/lemurs",
|
||||
@@ -149,11 +151,24 @@
|
||||
"mesh-catalog modules/marrytts",
|
||||
"mesh-catalog modules/matrix",
|
||||
"mesh-catalog modules/memory-pressure",
|
||||
"mesh-catalog modules/mesh-build-go",
|
||||
"mesh-catalog modules/mesh-build-image",
|
||||
"mesh-catalog modules/mesh-build-node",
|
||||
"mesh-catalog modules/mesh-catalog",
|
||||
"mesh-catalog modules/mesh-cli",
|
||||
"mesh-catalog modules/mesh-config",
|
||||
"mesh-catalog modules/mesh-delivery",
|
||||
"mesh-catalog modules/mesh-delivery-gate",
|
||||
"mesh-catalog modules/mesh-files",
|
||||
"mesh-catalog modules/mesh-issues",
|
||||
"mesh-catalog modules/mesh-knowledge",
|
||||
"mesh-catalog modules/mesh-mail",
|
||||
"mesh-catalog modules/mesh-mail-notifier",
|
||||
"mesh-catalog modules/mesh-mail-reader",
|
||||
"mesh-catalog modules/mesh-mail-web",
|
||||
"mesh-catalog modules/mesh-provisioner",
|
||||
"mesh-catalog modules/mesh-provisioner-client",
|
||||
"mesh-catalog modules/mesh-reads",
|
||||
"mesh-catalog modules/mesh-vault",
|
||||
"mesh-catalog modules/mesh-watcher",
|
||||
"mesh-catalog modules/messenger",
|
||||
@@ -223,11 +238,19 @@
|
||||
"mesh-controller modules/build-agent",
|
||||
"mesh-controller modules/route-proxy",
|
||||
"mesh-delivery",
|
||||
"mesh-delivery modules/mesh-delivery-builder",
|
||||
"mesh-delivery modules/mesh-delivery-checker",
|
||||
"mesh-delivery modules/mesh-delivery-forge-bridge",
|
||||
"mesh-delivery modules/mesh-delivery-indexer",
|
||||
"mesh-delivery modules/mesh-delivery-teller",
|
||||
"mesh-delivery modules/mesh-delivery-view",
|
||||
"mesh-delivery modules/mesh-delivery-watchman",
|
||||
"mesh-host",
|
||||
"mesh-lab",
|
||||
"mesh-media-catalog",
|
||||
"mesh-sdk",
|
||||
"mesh-tools"
|
||||
"mesh-tools",
|
||||
"okioki"
|
||||
]
|
||||
},
|
||||
"duplicate-check": {
|
||||
|
||||
Reference in New Issue
Block a user