The delivery repository's contracts and SDK live under modules/ as mesh-delivery-contracts and mesh-delivery-sdk, their Go module paths renamed to match the folders (the TypeScript names already did), and every importer in mesh-delivery, mesh-catalog, mesh-host, mesh-tools and mesh-sdk follows in the same group: a design as it should be, one path per module, checked by the group's check building every importer. Builds: design 47 G §427, §88 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h
380 lines
11 KiB
Go
380 lines
11 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
stdio "git.novox.be/novox/mesh-sdk/go"
|
|
|
|
reg "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/containerimageregistry"
|
|
sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
|
|
"git.novox.be/novox/mesh-sdk/go/forgekey"
|
|
)
|
|
|
|
const commitSHA = "0123456789abcdef0123456789abcdef01234567"
|
|
|
|
// keptStates is the steps state, kept in memory: every write of a key, and the keys that ended.
|
|
type keptStates struct {
|
|
mu sync.Mutex
|
|
steps map[string][]sdkstep.Step
|
|
ended chan string
|
|
}
|
|
|
|
func newStates() *keptStates {
|
|
return &keptStates{steps: map[string][]sdkstep.Step{}, ended: make(chan string, 16)}
|
|
}
|
|
|
|
func (s *keptStates) putStep(key string, value any) (uint64, error) {
|
|
st, err := sdkstep.DecodeStep(value.(json.RawMessage))
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
s.mu.Lock()
|
|
s.steps[key] = append(s.steps[key], st)
|
|
s.mu.Unlock()
|
|
if st.State != sdkstep.StepRunning {
|
|
s.ended <- key
|
|
}
|
|
return 1, nil
|
|
}
|
|
|
|
func (s *keptStates) waitEnd(t *testing.T) sdkstep.Step {
|
|
t.Helper()
|
|
select {
|
|
case key := <-s.ended:
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
v := s.steps[key]
|
|
return v[len(v)-1]
|
|
case <-time.After(5 * time.Second):
|
|
t.Fatal("the step never ended on its key")
|
|
return sdkstep.Step{}
|
|
}
|
|
}
|
|
|
|
// cmdRecord is one command a step ran.
|
|
type cmdRecord struct {
|
|
env []string
|
|
name string
|
|
line string
|
|
}
|
|
|
|
// fake stands in for git and buildah at the process boundary and for the image seat's holder at the seat's verbs.
|
|
type fake struct {
|
|
mu sync.Mutex
|
|
calls []cmdRecord
|
|
digest string // what `buildah images` prints
|
|
pushes int // layouts written
|
|
failBud forgekey.Ran // what a failing buildah bud answers; a zero Status is a passing bud
|
|
// digestFile: "" writes the layout's digest, "none" writes no file, "other" writes another digest.
|
|
digestFile string
|
|
// pushed is the manifest digest of the last layout written; parts its parts' digests, the manifest last.
|
|
pushed string
|
|
parts []string
|
|
// The holder: held are the parts it holds already; refuse fails every publish; published the publishes it was
|
|
// asked, sent the parts it was sent whole; mirrors the mirrors asked, answered by mirrored (published when nil).
|
|
held map[string]bool
|
|
refuse error
|
|
published []reg.PublishInput
|
|
sent []string
|
|
mirrors []reg.MirrorInput
|
|
mirrored func(reg.MirrorInput) (string, reg.Outcome)
|
|
outcomes *Outcomes
|
|
// elsewhere: the outcome is heard by another instance of the module over the same mirror-waits, not by this
|
|
// process; watched: this process's watch of mirror-waits hands that instance's write over; atOnce: the outcome is
|
|
// said before the mirror answers.
|
|
elsewhere, watched, atOnce bool
|
|
// atAsk runs when the mirror is asked.
|
|
atAsk func()
|
|
}
|
|
|
|
// memWaits is the state mirror-waits in memory; watch, when set, is handed every write as the state's watch hands it.
|
|
type memWaits struct {
|
|
mu sync.Mutex
|
|
m map[string]json.RawMessage
|
|
watch func(stdio.StateChange) error
|
|
}
|
|
|
|
func (w *memWaits) Get(key string) (json.RawMessage, error) {
|
|
w.mu.Lock()
|
|
defer w.mu.Unlock()
|
|
return w.m[key], nil
|
|
}
|
|
|
|
func (w *memWaits) Put(key string, value any) (uint64, error) {
|
|
b, err := json.Marshal(value)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
if f := credentialShaped(b); f != "" {
|
|
return 0, fmt.Errorf("the state refuses a field %q, which names a credential", f)
|
|
}
|
|
w.mu.Lock()
|
|
if w.m == nil {
|
|
w.m = map[string]json.RawMessage{}
|
|
}
|
|
w.m[key] = b
|
|
watch := w.watch
|
|
w.mu.Unlock()
|
|
if watch != nil {
|
|
_ = watch(stdio.StateChange{State: MirrorWaitsState, Key: key, Op: "put", Value: b})
|
|
}
|
|
return 1, nil
|
|
}
|
|
|
|
func (w *memWaits) Delete(key string) error {
|
|
w.mu.Lock()
|
|
defer w.mu.Unlock()
|
|
delete(w.m, key)
|
|
return nil
|
|
}
|
|
|
|
func (w *memWaits) Keys() ([]string, error) {
|
|
w.mu.Lock()
|
|
defer w.mu.Unlock()
|
|
keys := []string{}
|
|
for k := range w.m {
|
|
keys = append(keys, k)
|
|
}
|
|
return keys, nil
|
|
}
|
|
|
|
const imageDigest = "sha256:1111111111111111111111111111111111111111111111111111111111111111"
|
|
|
|
func newFake(t *testing.T) *fake {
|
|
return &fake{digest: imageDigest, held: map[string]bool{}}
|
|
}
|
|
|
|
// address is the registry part of the seat's reference: the seat's name.
|
|
func (f *fake) address() string { return reg.Seat }
|
|
|
|
func sha(b []byte) string {
|
|
s := sha256.Sum256(b)
|
|
return "sha256:" + hex.EncodeToString(s[:])
|
|
}
|
|
|
|
// writeLayout writes an OCI layout of one config, one layer and their manifest into dir, as buildah push oci: does.
|
|
func (f *fake) writeLayout(dir, digestFile string) {
|
|
blobs := filepath.Join(dir, "blobs", "sha256")
|
|
_ = os.MkdirAll(blobs, 0o755)
|
|
put := func(b []byte) map[string]any {
|
|
d := sha(b)
|
|
_ = os.WriteFile(filepath.Join(blobs, strings.TrimPrefix(d, "sha256:")), b, 0o644)
|
|
return map[string]any{"digest": d, "size": len(b)}
|
|
}
|
|
config := put([]byte(`{"architecture":"amd64","os":"linux"}`))
|
|
layer := put([]byte("the layer's bytes"))
|
|
config["mediaType"], layer["mediaType"] = "application/vnd.oci.image.config.v1+json", "application/vnd.oci.image.layer.v1.tar+gzip"
|
|
raw, _ := json.Marshal(map[string]any{"schemaVersion": 2, "mediaType": "application/vnd.oci.image.manifest.v1+json",
|
|
"config": config, "layers": []any{layer}})
|
|
manifest := put(raw)
|
|
manifest["mediaType"] = "application/vnd.oci.image.manifest.v1+json"
|
|
index, _ := json.Marshal(map[string]any{"schemaVersion": 2, "manifests": []any{manifest}})
|
|
_ = os.WriteFile(filepath.Join(dir, "index.json"), index, 0o644)
|
|
f.pushed = manifest["digest"].(string)
|
|
f.parts = []string{config["digest"].(string), layer["digest"].(string), f.pushed}
|
|
switch f.digestFile {
|
|
case "":
|
|
_ = os.WriteFile(digestFile, []byte(f.pushed), 0o644)
|
|
case "other":
|
|
_ = os.WriteFile(digestFile, []byte(imageDigest), 0o644)
|
|
}
|
|
}
|
|
|
|
// call is the holder's side of the seat's verbs.
|
|
func (f *fake) call(key string, body any) (json.RawMessage, error) {
|
|
raw, _ := json.Marshal(body)
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
switch key {
|
|
case "seat:" + reg.Seat + "." + reg.VerbPublish:
|
|
var in reg.PublishInput
|
|
_ = json.Unmarshal(raw, &in)
|
|
f.published = append(f.published, in)
|
|
if f.refuse != nil {
|
|
return nil, f.refuse
|
|
}
|
|
a := reg.PublishAnswer{Transfer: in.Transfer}
|
|
for _, p := range in.Parts {
|
|
if !f.held[p.Digest] {
|
|
a.Lacks = append(a.Lacks, p.Digest)
|
|
}
|
|
}
|
|
a.Held = len(a.Lacks) == 0
|
|
return json.Marshal(a)
|
|
case "seat:" + reg.Seat + "." + reg.VerbPublishPart:
|
|
var in reg.PublishPartInput
|
|
_ = json.Unmarshal(raw, &in)
|
|
if in.Last {
|
|
f.sent = append(f.sent, in.Part)
|
|
}
|
|
return json.Marshal(reg.PublishPartAnswer{Transfer: in.Transfer, Part: in.Part, Complete: in.Last})
|
|
case "seat:" + reg.Seat + "." + reg.VerbMirror:
|
|
var in reg.MirrorInput
|
|
_ = json.Unmarshal(raw, &in)
|
|
f.mirrors = append(f.mirrors, in)
|
|
if f.atAsk != nil {
|
|
f.atAsk()
|
|
}
|
|
event, o := reg.EventPublished, reg.Outcome{Kind: "upstream", Name: "upstream/" + in.Name, Digest: in.Digest,
|
|
Holder: "distribution", Reference: reg.Seat + "/upstream/" + in.Name + "@" + in.Digest}
|
|
if f.mirrored != nil {
|
|
event, o = f.mirrored(in)
|
|
}
|
|
if event != "" && f.outcomes != nil {
|
|
hearer := f.outcomes
|
|
if f.elsewhere {
|
|
waits := f.outcomes.Waits.(*memWaits)
|
|
if f.watched {
|
|
waits.watch = f.outcomes.Changed
|
|
}
|
|
hearer = &Outcomes{Waits: waits, Node: "g14"}
|
|
}
|
|
b, _ := json.Marshal(o)
|
|
if f.atOnce {
|
|
_ = hearer.Heard(event, b)
|
|
} else {
|
|
// Said after the answer, on the seat's events, as the holder's watch says it.
|
|
go func() { _ = hearer.Heard(event, b) }()
|
|
}
|
|
}
|
|
return json.Marshal(reg.MirrorAnswer{Digest: in.Digest, Reference: reg.Seat + "/upstream/" + in.Name + "@" + in.Digest})
|
|
}
|
|
return nil, fmt.Errorf("no verb %s", key)
|
|
}
|
|
|
|
func (f *fake) run(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
|
|
line := name + " " + strings.Join(args, " ")
|
|
f.mu.Lock()
|
|
f.calls = append(f.calls, cmdRecord{env: env, name: name, line: line})
|
|
f.mu.Unlock()
|
|
switch {
|
|
case name == "git" && strings.Contains(line, " rev-parse HEAD"):
|
|
return forgekey.Ran{Stdout: commitSHA + "\n"}
|
|
case name == "buildah" && strings.Contains(line, "--version"):
|
|
return forgekey.Ran{Stdout: "buildah version 1.45.1 (image-spec 1.1.0, runtime-spec 1.1.0)\n"}
|
|
case name == "buildah" && strings.Contains(line, " bud "):
|
|
if f.failBud.Status != 0 || f.failBud.Err != "" {
|
|
return f.failBud
|
|
}
|
|
case name == "buildah" && strings.Contains(line, " images "):
|
|
return forgekey.Ran{Stdout: f.digest + "\n"}
|
|
case name == "buildah" && strings.Contains(line, " push "):
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
f.pushes++
|
|
var digestFile, dir string
|
|
for i, a := range args {
|
|
if a == "--digestfile" {
|
|
digestFile = args[i+1]
|
|
}
|
|
if strings.HasPrefix(a, "oci:") {
|
|
dir = strings.TrimPrefix(a, "oci:")
|
|
}
|
|
}
|
|
f.writeLayout(dir, digestFile)
|
|
}
|
|
return forgekey.Ran{}
|
|
}
|
|
|
|
func (f *fake) lines() []string {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
out := []string{}
|
|
for _, c := range f.calls {
|
|
out = append(out, c.line)
|
|
}
|
|
return out
|
|
}
|
|
|
|
func (f *fake) called(verb string) []cmdRecord {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
var out []cmdRecord
|
|
for _, c := range f.calls {
|
|
if c.name == "buildah" && strings.Contains(c.line, " "+verb+" ") {
|
|
out = append(out, c)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// moduleWith is a module on node ace over the fake, with subordinate ranges for its account, and the registry's
|
|
// address written where the manifest would bind it.
|
|
func moduleWith(t *testing.T, f *fake, states *keptStates) *Module {
|
|
t.Helper()
|
|
dir := t.TempDir()
|
|
write := func(name, body string) string {
|
|
p := filepath.Join(dir, name)
|
|
if err := os.WriteFile(p, []byte(body), 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return p
|
|
}
|
|
m := &Module{
|
|
Node: "ace", Account: "mesh-build-image", UID: "1234", Home: t.TempDir(), Work: t.TempDir(), Store: t.TempDir(),
|
|
Steps: states.putStep, Ctx: context.Background(), Run: f.run, Call: f.call, Outcomes: &Outcomes{Waits: &memWaits{}},
|
|
SubUIDFile: write("subuid", "someone:100000:65536\nmesh-build-image:624288:65536\n"),
|
|
SubGIDFile: write("subgid", "mesh-build-image:624288:65536\n"),
|
|
Tun: write("tun", ""),
|
|
MirrorWait: 5 * time.Second,
|
|
}
|
|
f.outcomes = m.Outcomes
|
|
return m
|
|
}
|
|
|
|
func tool(t *testing.T, m *Module, name string) func(map[string]any) (any, error) {
|
|
t.Helper()
|
|
for _, tl := range m.Tools(m.StepTools()) {
|
|
if tl.Name == name {
|
|
return tl.Run
|
|
}
|
|
}
|
|
t.Fatalf("no tool %s", name)
|
|
return nil
|
|
}
|
|
|
|
// credentialShaped is the first field name in a JSON value that ends as the state's guard refuses (the bus's
|
|
// credentialEndings), or "": the in-memory state refuses it as the bus does, so every copy test meets the guard.
|
|
func credentialShaped(value []byte) string {
|
|
var v any
|
|
if json.Unmarshal(value, &v) != nil {
|
|
return ""
|
|
}
|
|
var walk func(any) string
|
|
walk = func(v any) string {
|
|
switch t := v.(type) {
|
|
case map[string]any:
|
|
for k, x := range t {
|
|
norm := strings.NewReplacer("-", "", "_", "", " ", "").Replace(strings.ToLower(k))
|
|
for _, end := range []string{"password", "passwd", "secret", "token", "credential", "credentials",
|
|
"authorization", "apikey", "privatekey", "accesskey", "cookie"} {
|
|
if strings.HasSuffix(norm, end) {
|
|
return k
|
|
}
|
|
}
|
|
if f := walk(x); f != "" {
|
|
return f
|
|
}
|
|
}
|
|
case []any:
|
|
for _, x := range t {
|
|
if f := walk(x); f != "" {
|
|
return f
|
|
}
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
return walk(v)
|
|
}
|