Merge branch 'wip/registry-fcat-b' into feat/registry-milestone
This commit is contained in:
@@ -0,0 +1,124 @@
|
||||
// Package gitignored refuses an ignore pattern that hides a module's source (catalogue review F1, and its kind): the
|
||||
// bare pattern `mesh-build-node` in a module's .gitignore, meant for the binary `go build` leaves, also matched the
|
||||
// directory cmd/mesh-build-node, so git ignored every new file there and publish.go was never committed. A path under
|
||||
// a module's cmd/ or internal/ is source: no .gitignore, the module's or the catalogue's, may ignore one, tracked or
|
||||
// new. A binary is ignored by an anchored pattern (`/mesh-build-node`, `/cmd/mesh-build-node/mesh-build-node`).
|
||||
package gitignored
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Probe is the file name asked of each source directory: a Go file a change might add there.
|
||||
const Probe = "added_by_a_change.go"
|
||||
|
||||
// Hidden is one source path an ignore pattern hides, and the pattern (its file, line and text, as git names it).
|
||||
type Hidden struct {
|
||||
Path, Pattern string
|
||||
}
|
||||
|
||||
// Sources are the source directories of root's modules, relative to root: every directory under modules/*/cmd and
|
||||
// modules/*/internal (node_modules left out).
|
||||
func Sources(root string) ([]string, error) {
|
||||
mods, err := filepath.Glob(filepath.Join(root, "modules", "*"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []string
|
||||
for _, m := range mods {
|
||||
for _, top := range []string{"cmd", "internal"} {
|
||||
base := filepath.Join(m, top)
|
||||
if st, err := os.Stat(base); err != nil || !st.IsDir() {
|
||||
continue
|
||||
}
|
||||
err := filepath.WalkDir(base, func(p string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !d.IsDir() {
|
||||
return nil
|
||||
}
|
||||
if d.Name() == "node_modules" {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
rel, _ := filepath.Rel(root, p)
|
||||
out = append(out, filepath.ToSlash(rel))
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// isSource is whether a file is source a module's build reads: Go and TypeScript files and a Go module's own files.
|
||||
// What a build or a test script leaves beside them (a binary, compiled JavaScript) may be ignored.
|
||||
func isSource(name string) bool {
|
||||
switch filepath.Ext(name) {
|
||||
case ".go", ".ts", ".tsx":
|
||||
return true
|
||||
}
|
||||
return name == "go.mod" || name == "go.sum"
|
||||
}
|
||||
|
||||
// Find asks git which of the source paths root's ignore files hide: each source directory, a Go file added in it, and
|
||||
// every source file there (isSource). It reads the .gitignore files of root's tree through a repository of its own, so it judges a git
|
||||
// checkout and a plain folder alike, and no one's global ignores count.
|
||||
func Find(root string, dirs []string) ([]Hidden, error) {
|
||||
gitDir, err := os.MkdirTemp("", "gitignored-")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer os.RemoveAll(gitDir)
|
||||
if out, err := exec.Command("git", "init", "-q", "--bare", gitDir).CombinedOutput(); err != nil {
|
||||
return nil, fmt.Errorf("git init: %v: %s", err, out)
|
||||
}
|
||||
var ask bytes.Buffer
|
||||
for _, d := range dirs {
|
||||
ask.WriteString(d + "/\x00" + d + "/" + Probe + "\x00")
|
||||
entries, err := os.ReadDir(filepath.Join(root, d))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, e := range entries {
|
||||
if !e.IsDir() && isSource(e.Name()) {
|
||||
ask.WriteString(d + "/" + e.Name() + "\x00")
|
||||
}
|
||||
}
|
||||
}
|
||||
abs, err := filepath.Abs(root)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cmd := exec.Command("git", "--git-dir="+gitDir, "--work-tree="+abs, "-c", "core.excludesFile=/dev/null",
|
||||
"check-ignore", "--no-index", "-v", "-z", "--stdin")
|
||||
cmd.Dir = abs
|
||||
cmd.Stdin = &ask
|
||||
out, err := cmd.Output()
|
||||
if ee, ok := err.(*exec.ExitError); ok && ee.ExitCode() == 1 {
|
||||
return nil, nil // nothing is ignored
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("git check-ignore: %w", err)
|
||||
}
|
||||
// -v -z answers source, line, pattern and path, each ended by NUL.
|
||||
f := strings.Split(strings.TrimRight(string(out), "\x00"), "\x00")
|
||||
var hidden []Hidden
|
||||
for i := 0; i+3 < len(f); i += 4 {
|
||||
if f[i+2] == "" || strings.HasPrefix(f[i+2], "!") {
|
||||
continue // a negation keeps the path
|
||||
}
|
||||
hidden = append(hidden, Hidden{Path: f[i+3], Pattern: fmt.Sprintf("%s:%s: %s", f[i], f[i+1], f[i+2])})
|
||||
}
|
||||
return hidden, nil
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
package gitignored
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The kind of catalogue review F1: a bare pattern hides the command directory it shares a name with; an anchored one
|
||||
// hides the binary alone; a glob that ignores source is refused too; a negation keeps a path.
|
||||
func TestFindNamesEachPatternThatHidesSource(t *testing.T) {
|
||||
for name, c := range map[string]struct {
|
||||
ignore string
|
||||
hidden []string
|
||||
}{
|
||||
"a bare name hides cmd/<name>": {"nfs-server\n", []string{"modules/m/cmd/nfs-server/", "modules/m/cmd/nfs-server/" + Probe, "modules/m/cmd/nfs-server/main.go"}},
|
||||
"anchored to the binary": {"/nfs-server\n/cmd/nfs-server/nfs-server\n", nil},
|
||||
"a glob ignoring Go source": {"*.go\n!main.go\n", []string{"modules/m/cmd/" + Probe, "modules/m/cmd/nfs-server/" + Probe,
|
||||
"modules/m/internal/" + Probe, "modules/m/internal/x/" + Probe, "modules/m/internal/x/x.go"}},
|
||||
"a pattern over internal/": {"internal/\n", []string{"modules/m/internal/", "modules/m/internal/" + Probe, "modules/m/internal/x/",
|
||||
"modules/m/internal/x/" + Probe, "modules/m/internal/x/x.go"}},
|
||||
"nothing but generated JavaScript": {"*.js\n", nil},
|
||||
} {
|
||||
root := t.TempDir()
|
||||
for f, b := range map[string]string{
|
||||
"modules/m/.gitignore": c.ignore,
|
||||
"modules/m/cmd/nfs-server/main.go": "package main\n",
|
||||
"modules/m/cmd/nfs-server/run.js": "",
|
||||
"modules/m/internal/x/x.go": "package x\n",
|
||||
"modules/m/node_modules/y/index.go": "",
|
||||
"modules/other/cmd/other/other.go": "package main\n",
|
||||
} {
|
||||
p := filepath.Join(root, f)
|
||||
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(p, []byte(b), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
dirs, err := Sources(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := Find(root, dirs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var paths []string
|
||||
for _, h := range got {
|
||||
paths = append(paths, h.Path)
|
||||
if !strings.HasPrefix(h.Pattern, "modules/m/.gitignore:") {
|
||||
t.Errorf("%s: %s is named hidden by %q, want the module's .gitignore", name, h.Path, h.Pattern)
|
||||
}
|
||||
}
|
||||
if strings.Join(paths, ",") != strings.Join(c.hidden, ",") {
|
||||
t.Errorf("%s: hidden %v, want %v", name, paths, c.hidden)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The catalogue's own tree: no ignore pattern hides a path under a module's cmd/ or internal/.
|
||||
func TestNoIgnorePatternHidesAModulesSource(t *testing.T) {
|
||||
root := filepath.Join("..", "..")
|
||||
dirs, err := Sources(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(dirs) == 0 {
|
||||
t.Fatal("no module source directory found: the check reads nothing")
|
||||
}
|
||||
hidden, err := Find(root, dirs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, h := range hidden {
|
||||
t.Errorf("%s is hidden by %s: a module's source is never ignored; anchor the pattern to the binary (/<name>, /cmd/<name>/<name>)", h.Path, h.Pattern)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
module git.novox.be/novox/mesh-catalog/checks/gitignored
|
||||
|
||||
go 1.22
|
||||
@@ -0,0 +1,3 @@
|
||||
module git.novox.be/novox/mesh-catalog/checks/gofmt
|
||||
|
||||
go 1.22
|
||||
@@ -0,0 +1,58 @@
|
||||
// Package gofmt refuses a Go file of the catalogue that gofmt would change (the second of a kind in one milestone:
|
||||
// buildstep's generated.go and claude-code's config_test.go reached a join unformatted). A check here holds every
|
||||
// module to it, where a module's own tests hold only themselves.
|
||||
package gofmt
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"go/format"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Files are the Go files under root, relative to it: every .go file but those in .git, node_modules and a testdata
|
||||
// directory (a test's fixture may be anything).
|
||||
func Files(root string) ([]string, error) {
|
||||
var out []string
|
||||
err := filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if d.IsDir() {
|
||||
switch d.Name() {
|
||||
case ".git", "node_modules", "testdata":
|
||||
return filepath.SkipDir
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if strings.HasSuffix(d.Name(), ".go") && d.Type().IsRegular() {
|
||||
rel, _ := filepath.Rel(root, p)
|
||||
out = append(out, filepath.ToSlash(rel))
|
||||
}
|
||||
return nil
|
||||
})
|
||||
sort.Strings(out)
|
||||
return out, err
|
||||
}
|
||||
|
||||
// Unformatted are the files gofmt would change, and why for one that does not parse.
|
||||
func Unformatted(root string, files []string) (map[string]string, error) {
|
||||
out := map[string]string{}
|
||||
for _, f := range files {
|
||||
src, err := os.ReadFile(filepath.Join(root, f))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
got, err := format.Source(src)
|
||||
switch {
|
||||
case err != nil:
|
||||
out[f] = "it does not parse: " + err.Error()
|
||||
case !bytes.Equal(got, src):
|
||||
out[f] = "gofmt changes it: run gofmt -w " + f
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
package gofmt
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestUnformattedNamesWhatGofmtChanges(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
for f, src := range map[string]string{
|
||||
"modules/a/a.go": "package a\n",
|
||||
"modules/b/b.go": "package b\nimport (\n\t\"os\"\n\t\"fmt\"\n)\nvar _, _ = fmt.Sprint, os.Exit\n",
|
||||
"modules/c/testdata/c.go": "package c",
|
||||
"modules/d/node_modules/d.go": "package d",
|
||||
} {
|
||||
p := filepath.Join(root, f)
|
||||
_ = os.MkdirAll(filepath.Dir(p), 0o755)
|
||||
if err := os.WriteFile(p, []byte(src), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
files, err := Files(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := Unformatted(root, files)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(files) != 2 || len(got) != 1 || got["modules/b/b.go"] == "" {
|
||||
t.Errorf("files %v, unformatted %v; want a.go and b.go read, b.go named", files, got)
|
||||
}
|
||||
}
|
||||
|
||||
// The catalogue's own Go files are as gofmt writes them.
|
||||
func TestTheCataloguesGoIsFormatted(t *testing.T) {
|
||||
root := filepath.Join("..", "..")
|
||||
files, err := Files(root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(files) == 0 {
|
||||
t.Fatal("no Go file found: the check reads nothing")
|
||||
}
|
||||
got, err := Unformatted(root, files)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for f, why := range got {
|
||||
t.Errorf("%s: %s", f, why)
|
||||
}
|
||||
}
|
||||
@@ -269,7 +269,7 @@ func TestTheManifestListsEveryToolServed(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var m struct {
|
||||
Tools []string `json:"tools"`
|
||||
Tools []string `json:"tools"`
|
||||
Invokes []string `json:"invokes"`
|
||||
Reads []string `json:"reads"`
|
||||
Consumes []string `json:"consumes"`
|
||||
|
||||
@@ -80,27 +80,6 @@ func finding(kind, subject, words string, raised bool) {
|
||||
}
|
||||
}
|
||||
|
||||
// closeBootstrap asks the program to discard the bootstrap's one-off credential once a publish is answered (design 51
|
||||
// *On a fresh mesh*): the window bootstrap-push closed in the state windows, which the program follows and then removes
|
||||
// the file. The tools bundle runs in the tool runner's account, which cannot remove a file in distribution's state
|
||||
// directory. Once per process; the window's record says it to the next.
|
||||
var closeBootstrap func()
|
||||
|
||||
func bootstrapCloser(windows kept[holder.Window]) func() {
|
||||
return sync.OnceFunc(func() {
|
||||
cur, _, err := windows.Get(holder.BootstrapWindow)
|
||||
if err == nil && !cur.Closed.IsZero() {
|
||||
return
|
||||
}
|
||||
w := holder.Window{Name: holder.BootstrapWindow, Closed: time.Now().UTC()}
|
||||
if err := windows.Put(w); err != nil {
|
||||
logf("the bootstrap's one-off credential could not be asked away: %v", err)
|
||||
return
|
||||
}
|
||||
logf("the bootstrap's one-off credential is asked away: the holders serve publish")
|
||||
})
|
||||
}
|
||||
|
||||
// learned are the proved callers whose build this process recorded as publishing through the seats.
|
||||
var learned sync.Map
|
||||
|
||||
@@ -202,6 +181,28 @@ func (mirrors) Claim(key string, r holder.MirrorRequest, revision uint64) (bool,
|
||||
return err == nil, err
|
||||
}
|
||||
func (mirrors) Delete(key string) error { return stdio.State(holder.MirrorsState).Delete(key) }
|
||||
func (mirrors) List() (map[string]holder.MirrorRequest, error) {
|
||||
keys, err := stdio.State(holder.MirrorsState).Keys()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := map[string]holder.MirrorRequest{}
|
||||
for _, k := range keys {
|
||||
e, err := stdio.State(holder.MirrorsState).Get(k)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var r holder.MirrorRequest
|
||||
if e != nil && json.Unmarshal(e.Value, &r) == nil {
|
||||
out[k] = r
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
func (mirrors) Put(key string, r holder.MirrorRequest) error {
|
||||
_, err := stdio.State(holder.MirrorsState).Put(key, r)
|
||||
return err
|
||||
}
|
||||
|
||||
// followMirrors runs every copy `mirror` took on, on the state's watch: the asks kept when the bundle starts (a copy a
|
||||
// restart cut) and each new one, one at a time. Each is claimed first (Verbs.Take), so a second follower copies none
|
||||
@@ -266,11 +267,7 @@ func tools(v *holder.Verbs, windows kept[holder.Window], callers func() (holder.
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
_, _ = v.DropIdle()
|
||||
a, err := v.Publish(seat, stdio.CallerOf(args), publishOf(in))
|
||||
if err == nil && closeBootstrap != nil {
|
||||
closeBootstrap()
|
||||
}
|
||||
if err == nil {
|
||||
learnPublishing(windows, stdio.CallerOf(args))
|
||||
}
|
||||
@@ -421,7 +418,33 @@ func tools(v *holder.Verbs, windows kept[holder.Window], callers func() (holder.
|
||||
}
|
||||
all, err := v.Transfers.List()
|
||||
used, _ := v.Objects.Used()
|
||||
return map[string]any{"transfers": all, "used": used, "cap": holder.IncomingCap, "dropped_idle": idle}, err
|
||||
return map[string]any{"transfers": all, "used": used, "cap": v.CapNow(), "dropped_idle": idle}, err
|
||||
}},
|
||||
stdio.Tool{Name: "registry_mirrors", Description: "The copies of outside images mirror took on and has not ended (the state mirrors): each by its key, its source, digest, who asked and the follower that claimed it (none: it waits). With again, asks one again by hand, its claim let go so a follower takes it on; with drop, drops one by hand, saying refused on the image seat's events so its waiter ends (each a hand act, why required).",
|
||||
Input: obj(map[string]any{"again": s("a kept copy's key to ask again"), "drop": s("a kept copy's key to drop"), "why": s("why (a hand act)")}),
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
by, why := stdio.CallerOf(args), str(args, "why")
|
||||
switch {
|
||||
case str(args, "again") != "":
|
||||
if why == "" {
|
||||
return nil, errors.New("asking a copy again is a hand act: say why")
|
||||
}
|
||||
r, err := v.MirrorAgain(str(args, "again"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
logf("the copy %s asked again by hand by %s: %s", str(args, "again"), by, why)
|
||||
return map[string]any{"asked_again": str(args, "again"), "copy": r}, nil
|
||||
case str(args, "drop") != "":
|
||||
r, err := v.MirrorDrop(str(args, "drop"), by, why)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
logf("the copy %s dropped by hand by %s: %s", str(args, "drop"), by, why)
|
||||
return map[string]any{"dropped": str(args, "drop"), "copy": r}, nil
|
||||
}
|
||||
all, err := v.MirrorsNow()
|
||||
return map[string]any{"mirrors": all}, err
|
||||
}},
|
||||
stdio.Tool{Name: "registry_holds", Description: "What is held, since when and for whom; with release, lets one go through the seat's release as the operator's session (a hand act).",
|
||||
Input: obj(map[string]any{"seat": s("the seat"), "release": s("the name to release"), "digest": s("its digest"), "why": s("why")}),
|
||||
@@ -465,12 +488,7 @@ func tools(v *holder.Verbs, windows kept[holder.Window], callers func() (holder.
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
b, _, err := windows.Get(holder.BootstrapWindow)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// The program removes the file when this window closes and writes it closed again with its removal's time.
|
||||
return map[string]any{"window": cur, "bootstrap_credential_discarded": !b.Closed.IsZero(), "bootstrap": b}, nil
|
||||
return map[string]any{"window": cur}, nil
|
||||
}})
|
||||
}
|
||||
return out
|
||||
@@ -480,11 +498,17 @@ func main() {
|
||||
base, _ := url.Parse("http://127.0.0.1:5001")
|
||||
store := &holder.OCI{Base: base, Upstream: &holder.Upstream{}}
|
||||
h := &holder.Holder{Objects: parts.BusObjects(holder.IncomingObjects), Builder: delivery.BuilderVersions{Ask: stdio.Ask}, Store: store, Say: say, Finding: finding,
|
||||
Transfers: parts.BusTransfers[holder.Opening](holder.TransfersState)}
|
||||
Transfers: parts.BusTransfers[holder.Opening](holder.TransfersState), Cap: parts.BusCap(holder.IncomingObjects), Idle: parts.BusArm(holder.IdleState)}
|
||||
v := &holder.Verbs{Holder: h, Catalog: store, Holds: kept[holder.Hold]{state: "holds", key: holder.Hold.Key}, Mirrors: mirrors{}}
|
||||
go followMirrors(v)
|
||||
go parts.FollowIdle(holder.IdleState, h.Expired, func(id string, dropped bool, err error) {
|
||||
if err != nil {
|
||||
logf("%v", err)
|
||||
return
|
||||
}
|
||||
logf("transfer %s had no part for %s: dropped with its parts", id, holder.TransferIdle)
|
||||
})
|
||||
windows := kept[holder.Window]{state: "windows", key: func(w holder.Window) string { return w.Name }}
|
||||
closeBootstrap = bootstrapCloser(windows)
|
||||
if err := stdio.Serve("", tools(v, windows, holder.ConfiguredCallers)); err != nil {
|
||||
logf("%v", err)
|
||||
os.Exit(1)
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// Command distribution is distribution's program (novox/hq design 51): it serves the artifact store's port in front of
|
||||
// the registry container, which listens on its node's loopback only. Reads (GET, HEAD) go through to every node's
|
||||
// container runtime; a write is refused but while the direct-write window is open, never over a tag the seat holds
|
||||
// published, or with the bootstrap's one-off credential (holder.Front). The windows are distribution's state windows:
|
||||
// published (holder.Front). The windows are distribution's state windows:
|
||||
// the program moves them (windows.go) and follows the direct-write window's record on each change; it holds no timer.
|
||||
package main
|
||||
|
||||
@@ -10,7 +10,6 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/url"
|
||||
@@ -58,7 +57,6 @@ func run(ctx context.Context, getenv func(string) string) error {
|
||||
windows := process.OptionalBucket(ctx, js, "distribution_windows", "the direct-write window is read as open until the program starts again", logger.Printf)
|
||||
if windows != nil {
|
||||
go follow(ctx, windows, &open, logger)
|
||||
go followBootstrap(ctx, windows, logger)
|
||||
calls, err := process.Connect(cred)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -72,13 +70,6 @@ func run(ctx context.Context, getenv func(string) string) error {
|
||||
Published: func(name, tag string) (bool, error) {
|
||||
d, err := oci.Digest(name, tag)
|
||||
return d != "", err
|
||||
},
|
||||
Bootstrap: func() (string, bool) {
|
||||
b, err := os.ReadFile(holder.BootstrapFile)
|
||||
if err != nil {
|
||||
return "", false
|
||||
}
|
||||
return strings.TrimSpace(string(b)), true
|
||||
}}
|
||||
srv := &http.Server{Addr: or(getenv("MESH_REGISTRY_LISTEN"), ":5000"), Handler: front, ReadHeaderTimeout: 30 * time.Second}
|
||||
errs := make(chan error, 1)
|
||||
@@ -119,42 +110,6 @@ func follow(ctx context.Context, kv jetstream.KeyValue, open *atomic.Bool, logge
|
||||
}
|
||||
}
|
||||
|
||||
// followBootstrap removes the bootstrap's one-off credential (holder.BootstrapFile) when the tools bundle closes the
|
||||
// window bootstrap-push at its first answered publish (design 51 *On a fresh mesh*): the program runs as distribution's
|
||||
// account, which owns the file. A file already gone is no fault; one that cannot be removed is said each time the
|
||||
// window is written.
|
||||
func followBootstrap(ctx context.Context, kv jetstream.KeyValue, logger *log.Logger) {
|
||||
w, err := kv.Watch(ctx, holder.BootstrapWindow)
|
||||
if err != nil {
|
||||
logger.Printf("the bootstrap's credential window cannot be watched (%v): its file stays until the program starts again", err)
|
||||
return
|
||||
}
|
||||
for e := range w.Updates() {
|
||||
if e == nil || e.Operation() != jetstream.KeyValuePut {
|
||||
continue
|
||||
}
|
||||
var win holder.Window
|
||||
if json.Unmarshal(e.Value(), &win) != nil || win.Closed.IsZero() {
|
||||
continue
|
||||
}
|
||||
if why := removeBootstrap(holder.BootstrapFile); why != "" {
|
||||
logger.Printf("%s", why)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// removeBootstrap removes the credential's file; it answers what went wrong, or "" when the file is gone.
|
||||
func removeBootstrap(path string) string {
|
||||
switch err := os.Remove(path); {
|
||||
case err == nil:
|
||||
return "the bootstrap's one-off credential is discarded: the holders serve publish"
|
||||
case errors.Is(err, fs.ErrNotExist):
|
||||
return ""
|
||||
default:
|
||||
return fmt.Sprintf("the bootstrap's one-off credential could not be discarded, so it is still accepted: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func or(v, d string) string {
|
||||
if strings.TrimSpace(v) == "" {
|
||||
return d
|
||||
|
||||
@@ -1,35 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The program, which runs as distribution's account, removes the bootstrap's one-off credential when its window
|
||||
// closes (design 51 *On a fresh mesh*; a3-publish-contract): removed once, a file already gone is no fault, and one
|
||||
// that cannot be removed is said, since the front still accepts it.
|
||||
func TestTheProgramDiscardsTheBootstrapCredential(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
file := filepath.Join(dir, "bootstrap-push")
|
||||
if err := os.WriteFile(file, []byte("token\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if why := removeBootstrap(file); !strings.Contains(why, "discarded") {
|
||||
t.Fatalf("first removal = %q, want it said discarded", why)
|
||||
}
|
||||
if _, err := os.Stat(file); !os.IsNotExist(err) {
|
||||
t.Fatalf("the file is still there: %v", err)
|
||||
}
|
||||
if why := removeBootstrap(file); why != "" {
|
||||
t.Fatalf("a file already gone = %q, want nothing said", why)
|
||||
}
|
||||
stuck := filepath.Join(dir, "stuck")
|
||||
if err := os.MkdirAll(filepath.Join(stuck, "child"), 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if why := removeBootstrap(stuck); !strings.Contains(why, "still accepted") {
|
||||
t.Fatalf("an unremovable file = %q, want it said still accepted", why)
|
||||
}
|
||||
}
|
||||
@@ -91,9 +91,12 @@ func (d busDelivery) Reads(node string) (bool, []string, error) {
|
||||
raw, err := d.c.CallOn(node, readsAsk, map[string]any{})
|
||||
if err != nil {
|
||||
if notServed(err) {
|
||||
return false, nil, nil // a node-engine not serving manifest-reads states nothing (design 51)
|
||||
// A node-engine not serving manifest-reads states nothing (design 51); what the call answered (an away node,
|
||||
// an older node-engine, a timeout) goes with it, so the window's tool says why the node holds it.
|
||||
return false, nil, err
|
||||
}
|
||||
return false, nil, err
|
||||
// Any other failure is a read that did not happen: served true with its error, so the wake judges nothing.
|
||||
return true, nil, err
|
||||
}
|
||||
var a struct {
|
||||
Reads []string `json:"manifest-reads"`
|
||||
|
||||
@@ -3,6 +3,7 @@ package holder
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
@@ -44,7 +45,8 @@ type NodeHave struct {
|
||||
|
||||
// Delivery is what the windows' driver asks of the delivery parts and the node-engine seat, through their tools and
|
||||
// verbs on the bus (ADR 0596): every node's haves, away ones included, and one node-engine's stated reader
|
||||
// capabilities. Reads answers served false when the node-engine does not serve `manifest-reads` (it states nothing).
|
||||
// capabilities. Reads answers served false when the node-engine does not serve `manifest-reads` (it states nothing),
|
||||
// with the call's error, if any, saying why (an away node, an older node-engine, a timeout).
|
||||
type Delivery interface {
|
||||
Haves() ([]NodeHave, error)
|
||||
Reads(node string) (served bool, stated []string, err error)
|
||||
@@ -118,6 +120,9 @@ func (d *Driver) Read() ([]NodeRead, error) {
|
||||
return nil, fmt.Errorf("%s's manifest-reads cannot be read, so no window moves: %w", node, err)
|
||||
}
|
||||
n.Serves, n.Stated = served, stated
|
||||
if !served && err != nil {
|
||||
n.NotServed = err.Error()
|
||||
}
|
||||
out = append(out, *n)
|
||||
}
|
||||
return out, nil
|
||||
@@ -131,13 +136,20 @@ func (d *Driver) Wake(why string) ([]string, error) {
|
||||
return nil, err
|
||||
}
|
||||
now := d.now()
|
||||
named := make([]string, 0, len(nodes))
|
||||
byNode := map[string]NodeRead{}
|
||||
for _, n := range nodes {
|
||||
named = append(named, n.Node)
|
||||
byNode[n.Node] = n
|
||||
}
|
||||
var said, errs []string
|
||||
for _, c := range []struct {
|
||||
name, event, seat string
|
||||
holding func([]NodeRead, []DroppedNode) []string
|
||||
why func(NodeRead) string
|
||||
}{
|
||||
{OldNameWindow, OldNameClosedEvent, OldSeat, HoldingOldName},
|
||||
{DirectWritesWindow, DirectWritesClosedEvent, "", HoldingDirectWrites},
|
||||
{OldNameWindow, OldNameClosedEvent, OldSeat, HoldingOldName, WhyOldName},
|
||||
{DirectWritesWindow, DirectWritesClosedEvent, "", HoldingDirectWrites, WhyDirectWrites},
|
||||
} {
|
||||
cur, err := d.State.Window(c.name)
|
||||
if err != nil {
|
||||
@@ -145,9 +157,16 @@ func (d *Driver) Wake(why string) ([]string, error) {
|
||||
continue
|
||||
}
|
||||
if cur.Name == "" {
|
||||
cur = Window{Name: c.name, Open: true}
|
||||
cur = Window{Name: c.name}
|
||||
cur.Open = true
|
||||
}
|
||||
next, closed, overdue := cur.Step(named, c.holding(nodes, cur.Dropped), now)
|
||||
if next.Open && len(named) > 0 {
|
||||
next.Why = map[string]string{}
|
||||
for _, n := range next.Holding {
|
||||
next.Why[n] = c.why(byNode[n])
|
||||
}
|
||||
}
|
||||
next, closed, overdue := cur.Step(c.holding(nodes, cur.Dropped), now)
|
||||
if !windowsEqual(cur, next) {
|
||||
if err := d.State.PutWindow(next); err != nil {
|
||||
errs = append(errs, fmt.Sprintf("the %s window is not kept: %v", c.name, err))
|
||||
@@ -172,6 +191,8 @@ func (d *Driver) Wake(why string) ([]string, error) {
|
||||
}
|
||||
}
|
||||
said = append(said, fmt.Sprintf("the %s window is closed (%s); it was held last by %v", c.name, why, closed.LastHolding))
|
||||
} else if len(named) == 0 {
|
||||
said = append(said, fmt.Sprintf("the %s window is not judged (%s): the read named no node", c.name, why))
|
||||
} else if next.Open {
|
||||
said = append(said, fmt.Sprintf("the %s window stays open (%s): held by %v", c.name, why, next.Holding))
|
||||
}
|
||||
@@ -191,5 +212,5 @@ func (d *Driver) now() time.Time {
|
||||
|
||||
func windowsEqual(a, b Window) bool {
|
||||
return a.Open == b.Open && slices.Equal(a.Holding, b.Holding) && a.Closed.Equal(b.Closed) &&
|
||||
len(a.Since) == len(b.Since) && len(a.Dropped) == len(b.Dropped)
|
||||
len(a.Since) == len(b.Since) && len(a.Dropped) == len(b.Dropped) && maps.Equal(a.Why, b.Why)
|
||||
}
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package holder
|
||||
|
||||
import (
|
||||
"crypto/subtle"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httputil"
|
||||
@@ -15,7 +14,8 @@ import (
|
||||
// protocol's GET and HEAD) stay open to every node's container runtime; a write reaches the store only from
|
||||
// distribution's own process, which writes to the loopback directly and never through the front. While the direct-write
|
||||
// window is open, a build module's old build still pushes through the front as today, but never to a tag the seat holds
|
||||
// published.
|
||||
// published. No credential opens it: on a fresh mesh the bootstrap pushes the holders' own images to the store on its
|
||||
// loopback, never through the front (design 51 *On a fresh mesh*).
|
||||
type Front struct {
|
||||
// Store is the registry container's loopback address.
|
||||
Store *url.URL
|
||||
@@ -23,9 +23,6 @@ type Front struct {
|
||||
Window *atomic.Bool
|
||||
// Published says whether name holds a tag published through the seat.
|
||||
Published func(name, tag string) (bool, error)
|
||||
// Bootstrap is the bootstrap's one-off credential for the holders' first images (design 51 *On a fresh mesh*):
|
||||
// read on each write, never kept, so the file's removal (BootstrapFile) refuses the next write that carries it.
|
||||
Bootstrap func() (token string, ok bool)
|
||||
Logf func(string, ...any)
|
||||
proxy *httputil.ReverseProxy
|
||||
}
|
||||
@@ -40,14 +37,7 @@ func (f *Front) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
f.proxy.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
bootstrap := false
|
||||
if f.Bootstrap != nil {
|
||||
if token, ok := f.Bootstrap(); ok && token != "" {
|
||||
got, has := strings.CutPrefix(r.Header.Get("Authorization"), "Bearer ")
|
||||
bootstrap = has && subtle.ConstantTimeCompare([]byte(got), []byte(token)) == 1
|
||||
}
|
||||
}
|
||||
if !bootstrap && (f.Window == nil || !f.Window.Load()) {
|
||||
if f.Window == nil || !f.Window.Load() {
|
||||
refuse(w, http.StatusForbidden, "DENIED", "the store takes writes from distribution's own process alone: publish through the seat's publish")
|
||||
return
|
||||
}
|
||||
@@ -62,21 +52,12 @@ func (f *Front) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
}
|
||||
if bootstrap {
|
||||
if f.Logf != nil {
|
||||
f.Logf("a write %s %s from %s with the bootstrap's one-off credential", r.Method, r.URL.Path, r.RemoteAddr)
|
||||
}
|
||||
} else if f.Logf != nil {
|
||||
if f.Logf != nil {
|
||||
f.Logf("a direct write %s %s from %s, accepted while the direct-write window is open", r.Method, r.URL.Path, r.RemoteAddr)
|
||||
}
|
||||
f.proxy.ServeHTTP(w, r)
|
||||
}
|
||||
|
||||
// BootstrapFile is where the node-engine's bootstrap gives distribution its one-off credential, in distribution's
|
||||
// state directory (0600, distribution's account): one line, the token. The tools bundle removes it when it answers its
|
||||
// first publish, so the holders serve publish and the credential is gone (RemoveBootstrap).
|
||||
const BootstrapFile = "/var/lib/mesh/registry/bootstrap-push"
|
||||
|
||||
// manifestPut is a PUT of /v2/<name>/manifests/<tag>, a tag and not a digest.
|
||||
func manifestPut(r *http.Request) (name, tag string, ok bool) {
|
||||
if r.Method != http.MethodPut {
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
@@ -89,6 +90,7 @@ type fakeCatalog struct {
|
||||
tags map[string]string // name:version -> digest
|
||||
removed []string
|
||||
mirrorErr error
|
||||
tagErr error
|
||||
indexes map[string][]string // name@digest -> the platform manifests it names
|
||||
}
|
||||
|
||||
@@ -104,11 +106,27 @@ func (c *fakeCatalog) Versions(name string) ([]string, error) {
|
||||
func (c *fakeCatalog) Digest(name, version string) (string, error) {
|
||||
return c.tags[name+":"+version], nil
|
||||
}
|
||||
|
||||
// Remove deletes by the digest, as the store does: every tag naming the digest goes with it.
|
||||
func (c *fakeCatalog) Remove(name, version string) error {
|
||||
c.removed = append(c.removed, name+":"+version)
|
||||
delete(c.tags, name+":"+version)
|
||||
d := c.tags[name+":"+version]
|
||||
for k, v := range c.tags {
|
||||
if n, _, _ := strings.Cut(k, ":"); n == name && v == d {
|
||||
delete(c.tags, k)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// RemoveTag answers as OCI.RemoveTag over registry 2.8.3 does (fakeRegistry's DELETE by tag): the store cannot
|
||||
// remove a tag alone, and nothing goes. tagErr is any other failure of the store.
|
||||
func (c *fakeCatalog) RemoveTag(name, version string) error {
|
||||
if c.tagErr != nil {
|
||||
return c.tagErr
|
||||
}
|
||||
return fmt.Errorf("%w: DELETE of %s:%s by its tag answered 400 DIGEST_INVALID", ErrNoTagAlone, name, version)
|
||||
}
|
||||
func (c *fakeCatalog) RemoveDigest(name, digest string) error {
|
||||
c.removed = append(c.removed, name+"@"+digest)
|
||||
return nil
|
||||
@@ -143,6 +161,14 @@ func (m memMirrors) Claim(k string, r MirrorRequest, rev uint64) (bool, error) {
|
||||
return true, nil
|
||||
}
|
||||
func (m memMirrors) Delete(k string) error { delete(m, k); return nil }
|
||||
func (m memMirrors) List() (map[string]MirrorRequest, error) {
|
||||
out := map[string]MirrorRequest{}
|
||||
for k, r := range m {
|
||||
out[k] = r
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
func (m memMirrors) Put(k string, r MirrorRequest) error { m[k] = r; return nil }
|
||||
|
||||
// mirrorRev is a kept ask's revision in memMirrors: 1 unclaimed, 2 once claimed.
|
||||
func mirrorRev(r MirrorRequest) uint64 {
|
||||
@@ -178,6 +204,9 @@ func newBed(builderOK bool) *bed {
|
||||
return b
|
||||
}
|
||||
|
||||
// capOf is a binding's cap of n bytes.
|
||||
func capOf(n uint64) func() (uint64, error) { return func() (uint64, error) { return n, nil } }
|
||||
|
||||
func keys[V any](m map[string]V) []string {
|
||||
var out []string
|
||||
for k := range m {
|
||||
@@ -251,7 +280,11 @@ func TestTheHolderOfTheTwoImageSeats(t *testing.T) {
|
||||
_, err := b.h.PublishPart(ArtifactSeat, builder, PartRequest{Transfer: "t", Digest: part.Digest, Data: make([]byte, MaxPart+1)})
|
||||
return err
|
||||
}, "over the"},
|
||||
{"a new transfer at the cap", func(b *bed) error { b.h.Cap = 100; _, err := b.h.Publish(ArtifactSeat, builder, req("t")); return err }, "at their cap"},
|
||||
{"a new transfer at the cap", func(b *bed) error {
|
||||
b.h.Cap = capOf(100)
|
||||
_, err := b.h.Publish(ArtifactSeat, builder, req("t"))
|
||||
return err
|
||||
}, "at their cap"},
|
||||
{"a version the builder did not number", func(b *bed) error {
|
||||
b.h.Builder = fakeBuilder{ok: false}
|
||||
b.store.has[part.Digest] = true // held already: the publish ends at once
|
||||
@@ -316,10 +349,38 @@ func TestTheHolderOfTheTwoImageSeats(t *testing.T) {
|
||||
if dropped, _ := b.h.DropIdle(); !reflect.DeepEqual(dropped, []string{"t2"}) {
|
||||
t.Errorf("DropIdle = %v, want t2 after 24 hours with no part", dropped)
|
||||
}
|
||||
// Running review F2, ADR 0611 §3: the drop follows each transfer's idle key. The open arms it for 24 hours; its
|
||||
// end re-arms a transfer that took a part since for the rest, and drops one idle 24 hours, its chunks with it.
|
||||
b = newBed(true)
|
||||
armed := map[string]time.Duration{}
|
||||
b.h.Idle = func(id string, ttl time.Duration) error { armed[id] = ttl; return nil }
|
||||
if _, err := b.h.Publish(ArtifactSeat, builder, req("t4")); err != nil || armed["t4"] != TransferIdle {
|
||||
t.Fatalf("publish = %v, armed %v; want t4's idle key for %s", err, armed, TransferIdle)
|
||||
}
|
||||
b.now = b.now.Add(time.Hour)
|
||||
_, _ = b.h.PublishPart(ArtifactSeat, builder, PartRequest{Transfer: "t4", Digest: part.Digest, Data: blob[:100]})
|
||||
b.now = b.now.Add(TransferIdle - time.Hour)
|
||||
if dropped, err := b.h.Expired("t4"); err != nil || dropped || armed["t4"] != time.Hour {
|
||||
t.Errorf("the key's end an hour after a part: dropped %v, %v, armed %v; want t4 kept and armed for 1h", dropped, err, armed)
|
||||
}
|
||||
b.now = b.now.Add(time.Hour)
|
||||
if dropped, err := b.h.Expired("t4"); err != nil || !dropped {
|
||||
t.Errorf("the key's end 24 hours after the last part: dropped %v, %v; want t4 dropped", dropped, err)
|
||||
}
|
||||
if _, found, _ := b.h.Transfers.Get("t4"); found || len(b.h.Objects.(memObjects)) != 0 {
|
||||
t.Errorf("t4 after its drop: kept %v, objects %v; want both gone", found, b.h.Objects)
|
||||
}
|
||||
if dropped, err := b.h.Expired("t4"); err != nil || dropped {
|
||||
t.Errorf("the end of a key whose transfer is gone: %v, %v; want nothing done", dropped, err)
|
||||
}
|
||||
b.h.Idle = func(string, time.Duration) error { return errors.New("bus down") }
|
||||
if _, err := b.h.Publish(ArtifactSeat, builder, req("t5")); err == nil || !strings.Contains(err.Error(), "idle bound") {
|
||||
t.Errorf("a publish whose idle key is not written = %v; want it failed loudly", err)
|
||||
}
|
||||
})
|
||||
t.Run("incoming's finding at 80 %, cleared below", func(t *testing.T) {
|
||||
b := newBed(true)
|
||||
b.h.Cap = 3500
|
||||
b.h.Cap = capOf(3500)
|
||||
_, _ = b.h.Publish(ArtifactSeat, builder, req("t3"))
|
||||
_, _ = b.h.PublishPart(ArtifactSeat, builder, PartRequest{Transfer: "t3", Digest: part.Digest, Data: blob[:2800]})
|
||||
if !b.findings["incoming-full incoming"] {
|
||||
@@ -330,12 +391,12 @@ func TestTheHolderOfTheTwoImageSeats(t *testing.T) {
|
||||
t.Error("the finding stayed after the transfer was dropped")
|
||||
}
|
||||
// A refusal for want of room stays loud on incoming until the holder takes a publish again.
|
||||
b.h.Cap = 100
|
||||
b.h.Cap = capOf(100)
|
||||
_, _ = b.h.Publish(ArtifactSeat, builder, req("t4"))
|
||||
if !b.findings["publish-refused incoming"] {
|
||||
t.Errorf("a refusal at the cap raised %v, want publish-refused on incoming", b.findings)
|
||||
}
|
||||
b.h.Cap = 1 << 20
|
||||
b.h.Cap = capOf(1 << 20)
|
||||
b.store.has[part.Digest] = true
|
||||
if _, err := b.h.Publish(ArtifactSeat, builder, req("t5")); err != nil || b.findings["publish-refused incoming"] {
|
||||
t.Errorf("the next publish taken = %v, findings %v: want the refusal at the cap cleared", err, b.findings)
|
||||
@@ -368,6 +429,42 @@ func TestTheHolderOfTheTwoImageSeats(t *testing.T) {
|
||||
if err := b.v.Remove(ArtifactSeat, "novox.mesh-delivery-builder", RemoveRequest{Name: "m/p", Version: "0.1.5-feat-x.1", Reason: "branch merged"}); err != nil || b.cat.tags["m/p:0.1.5-feat-x.1"] != "" {
|
||||
t.Errorf("the builder's removal of a tagged version: %v", err)
|
||||
}
|
||||
// Security review F1 on registry 2.8.3: a reproducible build gives a branch's tagged version and the trunk's
|
||||
// numbered one one digest; the store cannot remove the tag alone, so nothing goes, the numbered one stays
|
||||
// pullable, and the removal is said removed with that reason: no refusal, so the builder raises no retry.
|
||||
b.cat.tags["m/p:0.1.6-feat-y.1"], b.cat.tags["m/p:0.1.6"] = part.Digest, part.Digest
|
||||
if err := b.v.Remove(ArtifactSeat, "novox.mesh-delivery-builder", RemoveRequest{Name: "m/p", Version: "0.1.6-feat-y.1", Reason: "branch merged"}); err != nil {
|
||||
t.Fatalf("the removal of a tagged version sharing its digest: %v; want no error", err)
|
||||
}
|
||||
if b.cat.tags["m/p:0.1.6-feat-y.1"] != part.Digest || b.cat.tags["m/p:0.1.6"] != part.Digest {
|
||||
t.Errorf("tags after the tagged version's removal: %v; want both kept at %s", b.cat.tags, part.Digest)
|
||||
}
|
||||
if last := b.said[len(b.said)-1]; last.event != EventRemoved || last.body.Reason != "branch merged; the store cannot remove a tag alone, so the tag 0.1.6-feat-y.1 is kept and its bytes stay under 0.1.6" {
|
||||
t.Errorf("said %s %q; want removed, saying the tag is kept under 0.1.6", last.event, last.body.Reason)
|
||||
}
|
||||
// Any other failure of the store removes nothing and is refused naming the shared digest.
|
||||
b.cat.tags["m/p:0.1.7-feat-z.1"], b.cat.tags["m/p:0.1.7"] = part.Digest, part.Digest
|
||||
b.cat.tagErr = errors.New("the store answered 500")
|
||||
said := len(b.said)
|
||||
if err := b.v.Remove(ArtifactSeat, "novox.mesh-delivery-builder", RemoveRequest{Name: "m/p", Version: "0.1.7-feat-z.1", Reason: "branch merged"}); err == nil || !strings.Contains(err.Error(), "shares its digest") || b.cat.tags["m/p:0.1.7"] != part.Digest || len(b.said) != said {
|
||||
t.Errorf("a store failing otherwise: %v, tags %v; want refused, 0.1.7 kept, nothing said", err, b.cat.tags)
|
||||
}
|
||||
b.cat.tagErr = nil
|
||||
// The store's own answers, as registry 2.8.3 gives them: DELETE by tag removes nothing (ErrNoTagAlone); by
|
||||
// digest the manifest goes with every tag naming it.
|
||||
r283 := newFakeRegistry()
|
||||
sts := httptest.NewServer(r283)
|
||||
defer sts.Close()
|
||||
u, _ := url.Parse(sts.URL)
|
||||
o := &OCI{Base: u}
|
||||
manifest := []byte(`{"schemaVersion":2}`)
|
||||
r283.manifests["m/p:0.1.8-feat-w.1"], r283.manifests["m/p:0.1.8"] = manifest, manifest
|
||||
if err := o.RemoveTag("m/p", "0.1.8-feat-w.1"); !errors.Is(err, ErrNoTagAlone) || r283.manifests["m/p:0.1.8-feat-w.1"] == nil {
|
||||
t.Errorf("RemoveTag on 2.8.3 = %v; want ErrNoTagAlone and the tag kept", err)
|
||||
}
|
||||
if err := o.Remove("m/p", "0.1.8"); err != nil || len(r283.manifests) != 0 {
|
||||
t.Errorf("Remove by digest on 2.8.3 = %v, %d manifests left; want both tags gone", err, len(r283.manifests))
|
||||
}
|
||||
})
|
||||
// CAT-AS1: what the sweep did over the store's door is the holder's: a removal for retention of an index removes
|
||||
// its platform manifests first, but those a tagged or held index of the repository names; a manifest such an index
|
||||
@@ -437,9 +534,7 @@ func TestTheHolderOfTheTwoImageSeats(t *testing.T) {
|
||||
defer store.Close()
|
||||
u, _ := url.Parse(store.URL)
|
||||
var open atomic.Bool
|
||||
token, kept := "s3cret", true
|
||||
f := &Front{Store: u, Window: &open, Published: func(name, tag string) (bool, error) { return name == "m/i" && tag == "0.1.3", nil },
|
||||
Bootstrap: func() (string, bool) { return token, kept }}
|
||||
f := &Front{Store: u, Window: &open, Published: func(name, tag string) (bool, error) { return name == "m/i" && tag == "0.1.3", nil }}
|
||||
auth := ""
|
||||
do := func(method, path string) int {
|
||||
rec := httptest.NewRecorder()
|
||||
@@ -468,24 +563,12 @@ func TestTheHolderOfTheTwoImageSeats(t *testing.T) {
|
||||
t.Errorf("window open %v: %s %s = %d, want %d", c.open, c.method, c.path, got, c.want)
|
||||
}
|
||||
}
|
||||
// The bootstrap's one-off credential writes with the window closed, never over a published tag, and not once
|
||||
// its file is gone; a wrong token writes nothing.
|
||||
// No credential opens the front (the lead's ENG-5 ruling): a write carrying a token, with the window closed, is
|
||||
// refused as any other.
|
||||
open.Store(false)
|
||||
for _, c := range []struct {
|
||||
auth string
|
||||
kept bool
|
||||
path string
|
||||
want int
|
||||
}{
|
||||
{"s3cret", true, "/v2/gitea/program/manifests/0.1.0", http.StatusCreated},
|
||||
{"s3cret", true, "/v2/m/i/manifests/0.1.3", http.StatusForbidden},
|
||||
{"wrong", true, "/v2/gitea/program/manifests/0.1.0", http.StatusForbidden},
|
||||
{"s3cret", false, "/v2/gitea/program/manifests/0.1.0", http.StatusForbidden},
|
||||
} {
|
||||
auth, kept = c.auth, c.kept
|
||||
if got := do(http.MethodPut, c.path); got != c.want {
|
||||
t.Errorf("bootstrap %q kept %v: PUT %s = %d, want %d", c.auth, c.kept, c.path, got, c.want)
|
||||
}
|
||||
auth = "s3cret"
|
||||
if got := do(http.MethodPut, "/v2/gitea/program/manifests/0.1.0"); got != http.StatusForbidden {
|
||||
t.Errorf("a write with a bearer token and the window closed = %d, want %d", got, http.StatusForbidden)
|
||||
}
|
||||
})
|
||||
t.Run("mirror copies an outside image by its pinned digest, every platform, only the parts the store lacks", func(t *testing.T) {
|
||||
@@ -593,6 +676,24 @@ func TestTheHolderOfTheTwoImageSeats(t *testing.T) {
|
||||
t.Fatalf("two followers on one ask said %+v (ask id %q), asks %v: want one copy, said once under the ask's id", b.said[copies:], ask.ID, asked)
|
||||
}
|
||||
b.said = b.said[:copies]
|
||||
// registry_mirrors (seams review note 1): a kept copy is read, asked again with its claim let go, and dropped
|
||||
// by hand, which says refused under the ask's id with who and why.
|
||||
_, _ = b.v.Mirror(ImageSeat, "novox.mesh-build-image", MirrorRequest{Source: "nats", Digest: d})
|
||||
asked[key] = MirrorRequest{Source: asked[key].Source, Digest: d, ID: asked[key].ID, Claim: "cut-follower"}
|
||||
if now, err := b.v.MirrorsNow(); err != nil || now[key].Claim != "cut-follower" {
|
||||
t.Fatalf("MirrorsNow = %v, %v; want the claimed ask", now, err)
|
||||
}
|
||||
if r, err := b.v.MirrorAgain(key); err != nil || r.Claim != "" || asked[key].Claim != "" {
|
||||
t.Errorf("MirrorAgain = %+v, %v (kept %+v); want the claim let go", r, err, asked[key])
|
||||
}
|
||||
if _, err := b.v.MirrorDrop(key, "person.jochen", ""); err == nil {
|
||||
t.Error("a drop with no why is taken")
|
||||
}
|
||||
if _, err := b.v.MirrorDrop(key, "person.jochen", "upstream gone"); err != nil || len(asked) != 0 ||
|
||||
b.said[copies].event != EventRefused || b.said[copies].body.ID == "" || !strings.Contains(b.said[copies].body.Reason, "dropped by hand by person.jochen: upstream gone") {
|
||||
t.Errorf("MirrorDrop = %v, said %+v, asks %v; want refused said under the ask's id and the ask gone", err, b.said[copies:], asked)
|
||||
}
|
||||
b.said = b.said[:copies]
|
||||
b.cat.mirrorErr = errors.New("the upstream said 404")
|
||||
_, _ = b.v.Mirror(ImageSeat, "novox.mesh-build-image", MirrorRequest{Source: "nats", Digest: d})
|
||||
if err := b.v.Mirrored(asked[key]); err != nil || b.said[1].event != EventRefused || !strings.Contains(b.said[1].body.Reason, "404") || len(asked) != 0 {
|
||||
@@ -612,20 +713,26 @@ func TestTheHolderOfTheTwoImageSeats(t *testing.T) {
|
||||
t.Errorf("direct writes held by %v, want g14", got)
|
||||
}
|
||||
t0 := time.Unix(0, 0)
|
||||
w, closed, _ := Window{Name: OldNameWindow}.Step([]string{"g14"}, t0)
|
||||
all := []string{"ace", "g14", "novox"}
|
||||
w, closed, _ := Window{Name: OldNameWindow}.Step(all, []string{"g14"}, t0)
|
||||
if closed != nil || !w.Open {
|
||||
t.Fatalf("Step = %+v, %+v", w, closed)
|
||||
}
|
||||
if _, _, overdue := w.Step([]string{"g14"}, t0.Add(WindowFindingAfter)); !reflect.DeepEqual(overdue, []string{"g14"}) {
|
||||
if _, _, overdue := w.Step(all, []string{"g14"}, t0.Add(WindowFindingAfter)); !reflect.DeepEqual(overdue, []string{"g14"}) {
|
||||
t.Errorf("seven days: overdue %v", overdue)
|
||||
}
|
||||
// Security review F3: a read naming no node (an empty node_haves) judges nothing; the window stays open.
|
||||
if same, shut, _ := w.Step(nil, nil, t0.Add(time.Minute)); shut != nil || !same.Open || !reflect.DeepEqual(same.Holding, []string{"g14"}) {
|
||||
t.Errorf("a read naming no node: %+v, closed %+v; want the window open as it was", same, shut)
|
||||
}
|
||||
w, _ = w.Drop("g14", "person.jochen", "retired", t0)
|
||||
w, closed, _ = w.Step(HoldingOldName(nodes, w.Dropped), t0.Add(time.Hour))
|
||||
w, closed, _ = w.Step(all, HoldingOldName(nodes, w.Dropped), t0.Add(time.Hour))
|
||||
if closed == nil || closed.Window != OldNameWindow || !reflect.DeepEqual(closed.LastHolding, []string{"g14"}) || len(closed.Dropped) != 1 {
|
||||
t.Errorf("closing = %+v", closed)
|
||||
}
|
||||
if _, again, _ := w.Step([]string{"g14"}, t0.Add(2*time.Hour)); again != nil {
|
||||
t.Error("a closed window closed again")
|
||||
// Once closed, it stays closed: a node holding it later opens nothing.
|
||||
if again, shut, _ := w.Step(all, []string{"g14"}, t0.Add(2*time.Hour)); shut != nil || again.Open {
|
||||
t.Errorf("a closed window stepped on a holder: %+v, closed %+v; want it closed, said once", again, shut)
|
||||
}
|
||||
})
|
||||
t.Run("the windows' driver: every node read on each wake, closed once, a paused build agent's same build counted", func(t *testing.T) {
|
||||
@@ -647,6 +754,17 @@ func TestTheHolderOfTheTwoImageSeats(t *testing.T) {
|
||||
if !reflect.DeepEqual(st.w[OldNameWindow].Holding, []string{"g14"}) || !reflect.DeepEqual(st.w[DirectWritesWindow].Holding, []string{"g14"}) {
|
||||
t.Errorf("windows = %+v, want g14 holding both (away: states nothing; runs the old build)", st.w)
|
||||
}
|
||||
// The window's record says why g14 holds it: its manifest-reads timed out, counted as stating nothing.
|
||||
if why := st.w[OldNameWindow].Why["g14"]; !strings.Contains(why, "did not answer") || !strings.Contains(why, "timeout") {
|
||||
t.Errorf("why g14 holds the old name = %q, want the timeout said", why)
|
||||
}
|
||||
// Security review F3: a wake whose node_haves names no node closes neither window.
|
||||
haves := del.haves
|
||||
del.haves = nil
|
||||
if _, err := d.Wake("delivered"); err != nil || !st.w[OldNameWindow].Open || !st.w[DirectWritesWindow].Open || len(closed) != 0 {
|
||||
t.Fatalf("a wake on an empty node_haves: %v, windows %+v, closed %v; want both open, none said", err, st.w, closed)
|
||||
}
|
||||
del.haves = haves
|
||||
now = now.Add(WindowFindingAfter)
|
||||
_, _ = d.Wake("delivered")
|
||||
if !found[FindingWindowHeld+" "+OldNameWindow+".g14"] {
|
||||
@@ -729,5 +847,28 @@ func TestTheHolderOfTheTwoImageSeats(t *testing.T) {
|
||||
if store < 0 || program < 0 || store > program {
|
||||
t.Errorf("store at %d, program at %d: want the store listed first", store, program)
|
||||
}
|
||||
// Running review F2: incoming asks the bus 8 GiB, the most the objects template allows, equal to IncomingCap
|
||||
// (said only where no binding is read); transfers keeps for good, the holder dropping an idle one by its key
|
||||
// in idle, a state of keys with their own time to live.
|
||||
var req struct {
|
||||
Requires []struct {
|
||||
Name string `json:"name"`
|
||||
Options map[string]any `json:"options"`
|
||||
} `json:"requires"`
|
||||
}
|
||||
_ = json.Unmarshal(raw, &req)
|
||||
opts := map[string]map[string]any{}
|
||||
for _, r := range req.Requires {
|
||||
opts[r.Name] = r.Options
|
||||
}
|
||||
if mib, _ := opts[IncomingObjects]["max-mib"].(float64); uint64(mib)<<20 != IncomingCap {
|
||||
t.Errorf("incoming asks %v MiB, IncomingCap is %d bytes: want them equal", opts[IncomingObjects]["max-mib"], uint64(IncomingCap))
|
||||
}
|
||||
if _, ok := opts[TransfersState]["ttl-seconds"]; ok {
|
||||
t.Errorf("transfers has a lifetime %v: the bus would drop a transfer's key and leave its parts", opts[TransfersState])
|
||||
}
|
||||
if opts[IdleState]["key-ttl"] != true {
|
||||
t.Errorf("idle's options %v: want key-ttl, each transfer's key ending on its own", opts[IdleState])
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -10,7 +10,10 @@ import (
|
||||
)
|
||||
|
||||
// fakeRegistry is an OCI registry in memory, enough for a mirror's copy: blobs, uploads and manifests by
|
||||
// reference. With token set it answers 401 with a Bearer challenge until a request carries it.
|
||||
// reference. With token set it answers 401 with a Bearer challenge until a request carries it. A DELETE of a manifest
|
||||
// answers as registry 2.8.3, the store's pinned image, does (distribution v2.8.3 registry/handlers/manifests.go,
|
||||
// DeleteManifest): by digest it deletes the manifest and every tag naming it; by a tag it deletes nothing and answers
|
||||
// 400 DIGEST_INVALID, the check of the empty digest it deletes instead failing.
|
||||
type fakeRegistry struct {
|
||||
mu sync.Mutex
|
||||
blobs map[string][]byte // name@digest
|
||||
@@ -69,6 +72,27 @@ func (f *fakeRegistry) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
case strings.Contains(p, "/manifests/"):
|
||||
i := strings.Index(p, "/manifests/")
|
||||
key := p[:i] + ":" + p[i+11:]
|
||||
if r.Method == http.MethodDelete {
|
||||
ref := p[i+11:]
|
||||
if !strings.HasPrefix(ref, "sha256:") {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
_, _ = io.WriteString(w, `{"errors":[{"code":"DIGEST_INVALID","message":"provided digest did not match uploaded content"}]}`)
|
||||
return
|
||||
}
|
||||
found := false
|
||||
for k, b := range f.manifests {
|
||||
if strings.HasPrefix(k, p[:i]+":") && digestOf(b) == ref {
|
||||
delete(f.manifests, k)
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusAccepted)
|
||||
return
|
||||
}
|
||||
if r.Method == http.MethodPut {
|
||||
b, _ := io.ReadAll(r.Body)
|
||||
f.manifests[key] = b
|
||||
@@ -98,7 +122,10 @@ type fakeDelivery struct {
|
||||
func (f *fakeDelivery) Haves() ([]NodeHave, error) { return f.haves, nil }
|
||||
func (f *fakeDelivery) Reads(node string) (bool, []string, error) {
|
||||
s, ok := f.stated[node]
|
||||
return ok, s, nil
|
||||
if !ok {
|
||||
return false, nil, errors.New("nats: timeout")
|
||||
}
|
||||
return true, s, nil
|
||||
}
|
||||
|
||||
type memWindows struct {
|
||||
|
||||
@@ -3,6 +3,7 @@ package holder
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
@@ -253,7 +254,29 @@ func (o *OCI) Digest(name, version string) (string, error) {
|
||||
return resp.Header.Get("Docker-Content-Digest"), nil
|
||||
}
|
||||
|
||||
// Remove deletes the manifest name holds at version.
|
||||
// ErrNoTagAlone is a store that cannot delete a tag alone. registry 2.8.3, the store's pinned image, is one: its
|
||||
// DeleteManifest (distribution v2.8.3 registry/handlers/manifests.go) ignores a tag reference and deletes the empty
|
||||
// digest, whose check fails, answered 400 DIGEST_INVALID; registry 3 untags alone.
|
||||
var ErrNoTagAlone = errors.New("the store cannot remove a tag alone")
|
||||
|
||||
// RemoveTag deletes the tag version of name alone (the OCI distribution protocol's DELETE of a manifest by its tag);
|
||||
// the manifest stays under every other tag. A store that cannot delete a tag answers ErrNoTagAlone, any other failure
|
||||
// an error of its own; either removes nothing.
|
||||
func (o *OCI) RemoveTag(name, version string) error {
|
||||
resp, b, err := o.do(http.MethodDelete, "/v2/"+name+"/manifests/"+version, "", nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if resp.StatusCode == http.StatusBadRequest && bytes.Contains(b, []byte(`"DIGEST_INVALID"`)) {
|
||||
return fmt.Errorf("%w: DELETE of %s:%s by its tag answered 400 DIGEST_INVALID", ErrNoTagAlone, name, version)
|
||||
}
|
||||
if resp.StatusCode != http.StatusAccepted && resp.StatusCode != http.StatusNotFound {
|
||||
return fmt.Errorf("the store did not delete the tag %s:%s alone (%d): %s", name, version, resp.StatusCode, b)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Remove deletes the manifest name holds at version, by its digest: every tag naming it goes with it.
|
||||
func (o *OCI) Remove(name, version string) error {
|
||||
d, err := o.Digest(name, version)
|
||||
if err != nil || d == "" {
|
||||
|
||||
@@ -12,16 +12,22 @@ import (
|
||||
"git.novox.be/novox/mesh-sdk/go/parts"
|
||||
)
|
||||
|
||||
// The bounds of incoming (design 51 *Publishing*, ADR 0611): a part is at most 1 MiB, the store 20 GiB, a transfer
|
||||
// with no part for 24 hours is dropped, and past 80 % of the cap the holder writes a finding.
|
||||
// The bounds of incoming (design 51 *Publishing*, ADR 0611): a part is at most 1 MiB, a transfer with no part for 24
|
||||
// hours is dropped, and past 80 % of the cap the holder writes a finding. The cap is the one incoming's binding carries
|
||||
// (the bus's own bound of the object store, read through the runtime); IncomingCap is the manifest's 8 GiB, the most
|
||||
// the objects template allows, said only where no binding is read (a test), and the manifest's test holds the two equal.
|
||||
const (
|
||||
MaxPart = 1 << 20
|
||||
IncomingCap = 20 << 30
|
||||
IncomingCap = 8 << 30
|
||||
TransferIdle = 24 * time.Hour
|
||||
IncomingFullPct = 80
|
||||
// IncomingObjects and TransfersState are the object store and the state distribution asks as bus state.
|
||||
IncomingObjects = "incoming"
|
||||
TransfersState = "transfers"
|
||||
// IdleState is the state of one key per open transfer with its own time to live (a state declared key-ttl): the
|
||||
// bus's marker of its end is the event that drops a transfer left with no part for TransferIdle (ADR 0611 §3), never
|
||||
// a timer.
|
||||
IdleState = "idle"
|
||||
)
|
||||
|
||||
// The kinds a seat holds.
|
||||
@@ -156,8 +162,11 @@ type Holder struct {
|
||||
Say Say
|
||||
Finding Finding
|
||||
Now func() time.Time
|
||||
// Cap is incoming's cap; IncomingCap when zero.
|
||||
Cap uint64
|
||||
// Cap reads incoming's cap from its binding (parts.BusCap); IncomingCap when nil or when the binding names none.
|
||||
Cap func() (uint64, error)
|
||||
// Idle writes a transfer's idle key with its own time to live, once (parts.BusArm over IdleState); the receiver
|
||||
// arms it on each open that leaves a transfer open. Nil arms nothing.
|
||||
Idle func(id string, ttl time.Duration) error
|
||||
}
|
||||
|
||||
func (h *Holder) now() time.Time {
|
||||
@@ -167,18 +176,14 @@ func (h *Holder) now() time.Time {
|
||||
return time.Now().UTC()
|
||||
}
|
||||
|
||||
func (h *Holder) cap() uint64 {
|
||||
if h.Cap > 0 {
|
||||
return h.Cap
|
||||
}
|
||||
return IncomingCap
|
||||
}
|
||||
// CapNow is incoming's cap as its binding carries it.
|
||||
func (h *Holder) CapNow() uint64 { return h.receiver().CapNow() }
|
||||
|
||||
// receiver binds mesh-sdk's receiving side to distribution's incoming, its transfers state, the builder's check, its
|
||||
// store and its seats' events.
|
||||
func (h *Holder) receiver() *parts.Receiver[Opening] {
|
||||
return &parts.Receiver[Opening]{Objects: h.Objects, Transfers: h.Transfers, Cap: h.cap(), MaxChunk: MaxPart,
|
||||
Idle: TransferIdle, Full: IncomingFullPct, Now: h.now,
|
||||
return &parts.Receiver[Opening]{Objects: h.Objects, Transfers: h.Transfers, Cap: IncomingCap, Bound: h.Cap,
|
||||
MaxChunk: MaxPart, Idle: TransferIdle, Arm: h.Idle, Full: IncomingFullPct, Now: h.now,
|
||||
// Only the missing layers are sent: a part the store already holds under the name counts as held.
|
||||
Holds: func(t Transfer, digest string) bool {
|
||||
ok, err := h.Store.Has(t.Shape.Name, digest)
|
||||
@@ -283,10 +288,15 @@ func (h *Holder) eventOf(seat string, r PublishRequest, ref, reason string) Even
|
||||
// Drop drops a transfer by hand (registry_incoming) or because it was idle for TransferIdle: its chunks and its key.
|
||||
func (h *Holder) Drop(id string) (Transfer, error) { return h.receiver().Drop(id) }
|
||||
|
||||
// DropIdle drops every transfer with no part for TransferIdle, answering them; it runs on each publish's open and each
|
||||
// registry_incoming call, never on a timer (the object store's own lifetime of 24 hours drops the bytes too).
|
||||
// DropIdle drops every transfer with no part for TransferIdle, answering them: registry_incoming's look, by hand. The
|
||||
// drop itself follows each transfer's idle key (Expired); the objects template keeps for good, so nothing else does.
|
||||
func (h *Holder) DropIdle() ([]string, error) { return h.receiver().DropIdle() }
|
||||
|
||||
// Expired is told that a transfer's idle key ended (the bus's marker, parts.FollowIdle): the receiver drops a transfer
|
||||
// with no part for TransferIdle, its chunks with it, arms one that took a part since again for the rest, and leaves one
|
||||
// already gone. It answers whether it dropped the transfer.
|
||||
func (h *Holder) Expired(id string) (bool, error) { return h.receiver().Expired(id) }
|
||||
|
||||
func checkPublish(seat string, r PublishRequest) error {
|
||||
switch {
|
||||
case !slices.Contains(Seats, seat):
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -18,8 +19,12 @@ type Catalog interface {
|
||||
Versions(name string) ([]string, error)
|
||||
// Digest is the digest name holds at version, empty when none.
|
||||
Digest(name, version string) (string, error)
|
||||
// Remove deletes name at version (its manifest); the store's collector frees its blobs.
|
||||
// Remove deletes name at version by its manifest's digest, every tag naming that digest with it; the store's
|
||||
// collector frees its blobs.
|
||||
Remove(name, version string) error
|
||||
// RemoveTag deletes the tag version of name alone, the manifest staying under every other tag naming it; a store
|
||||
// that cannot delete a tag alone answers ErrNoTagAlone and removes nothing.
|
||||
RemoveTag(name, version string) error
|
||||
// RemoveDigest deletes an untagged manifest of name by its digest (retention).
|
||||
RemoveDigest(name, digest string) error
|
||||
// Platforms are the manifests an index of name names; none for a manifest that is not an index or is not held.
|
||||
@@ -42,6 +47,10 @@ type Mirrors interface {
|
||||
// writer wrote the key first.
|
||||
Claim(key string, r MirrorRequest, revision uint64) (bool, error)
|
||||
Delete(key string) error
|
||||
// List is every ask kept, by its key: what registry_mirrors reads.
|
||||
List() (map[string]MirrorRequest, error)
|
||||
// Put writes r under key whatever it held: an ask asked again by hand, its claim let go.
|
||||
Put(key string, r MirrorRequest) error
|
||||
}
|
||||
|
||||
// HoldState is distribution's own state holds (bus state, template state; ADR 0617 §2).
|
||||
@@ -149,7 +158,23 @@ func (v *Verbs) Remove(seat, caller string, r RemoveRequest) error {
|
||||
return fmt.Errorf("remove: the store cannot read %s@%s: %w", r.Name, r.Version, err)
|
||||
}
|
||||
if d != "" {
|
||||
if err := v.Catalog.Remove(r.Name, r.Version); err != nil {
|
||||
// Reproducible builds share a digest between a branch's tagged version and the trunk's numbered one: a
|
||||
// manifest deleted by its digest would take the trunk's version with it (security review F1). When another
|
||||
// tag names the digest, the tag alone goes. A store that cannot remove a tag alone (registry 2.8.3) keeps the
|
||||
// tag: its bytes are kept under the other tags anyway, so the removal is said removed with that reason and
|
||||
// the builder lets the version go from its keeping, no refusal and no retry. Any other failure is refused.
|
||||
others, err := v.otherTags(r.Name, r.Version, d)
|
||||
if err != nil {
|
||||
return fmt.Errorf("remove: the store cannot say which other tags of %s name %s, so nothing is removed: %w", r.Name, d, err)
|
||||
}
|
||||
if len(others) > 0 {
|
||||
if err := v.Catalog.RemoveTag(r.Name, r.Version); errors.Is(err, ErrNoTagAlone) {
|
||||
sort.Strings(others)
|
||||
ev.Reason = fmt.Sprintf("%s; the store cannot remove a tag alone, so the tag %s is kept and its bytes stay under %s", r.Reason, r.Version, strings.Join(others, ", "))
|
||||
} else if err != nil {
|
||||
return fmt.Errorf("remove: refused: %s@%s shares its digest %s with %v, and the store did not remove the tag alone, so nothing is removed: %w", r.Name, r.Version, d, others, err)
|
||||
}
|
||||
} else if err := v.Catalog.Remove(r.Name, r.Version); err != nil {
|
||||
return fmt.Errorf("remove: the store did not remove %s@%s: %w", r.Name, r.Version, err)
|
||||
}
|
||||
}
|
||||
@@ -158,6 +183,28 @@ func (v *Verbs) Remove(seat, caller string, r RemoveRequest) error {
|
||||
return v.Say(seat, EventRemoved, ev)
|
||||
}
|
||||
|
||||
// otherTags are the tags of name but version that name digest.
|
||||
func (v *Verbs) otherTags(name, version, digest string) ([]string, error) {
|
||||
tags, err := v.Catalog.Versions(name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []string
|
||||
for _, t := range tags {
|
||||
if t == version {
|
||||
continue
|
||||
}
|
||||
d, err := v.Catalog.Digest(name, t)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if d == digest {
|
||||
out = append(out, t)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// HoldRequest is `hold` and `release`.
|
||||
type HoldRequest struct {
|
||||
Name string `json:"name"`
|
||||
@@ -316,6 +363,55 @@ func (v *Verbs) Take(key string, r MirrorRequest, revision uint64, current bool,
|
||||
return v.Mirrored(r)
|
||||
}
|
||||
|
||||
// MirrorsNow lists every copy `mirror` took on and has not ended, by its key, for registry_mirrors: what waits (no
|
||||
// claim) and what a follower took on (its claim).
|
||||
func (v *Verbs) MirrorsNow() (map[string]MirrorRequest, error) { return v.Mirrors.List() }
|
||||
|
||||
// MirrorAgain asks a kept copy again by hand (registry_mirrors, a hand act): its claim is let go, so the state's watch
|
||||
// hands it to a follower, which takes it on as a new ask. A copy whose follower was cut, or that a follower holds and
|
||||
// does not end, goes on this way without a restart.
|
||||
func (v *Verbs) MirrorAgain(key string) (MirrorRequest, error) {
|
||||
all, err := v.Mirrors.List()
|
||||
if err != nil {
|
||||
return MirrorRequest{}, fmt.Errorf("mirror: the asks cannot be read: %w", err)
|
||||
}
|
||||
r, ok := all[key]
|
||||
if !ok {
|
||||
return MirrorRequest{}, fmt.Errorf("mirror: no copy is kept under %s", key)
|
||||
}
|
||||
r.Claim = ""
|
||||
if err := v.Mirrors.Put(key, r); err != nil {
|
||||
return MirrorRequest{}, fmt.Errorf("mirror: %s is not asked again: %w", key, err)
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// MirrorDrop drops a kept copy by hand (registry_mirrors, a hand act with who and why): it says refused on the image
|
||||
// seat's events with that reason, under the ask's id, so a waiter on the copy ends, and the ask leaves the state.
|
||||
func (v *Verbs) MirrorDrop(key, by, why string) (MirrorRequest, error) {
|
||||
if strings.TrimSpace(why) == "" {
|
||||
return MirrorRequest{}, errors.New("mirror: a drop is a hand act: say why")
|
||||
}
|
||||
all, err := v.Mirrors.List()
|
||||
if err != nil {
|
||||
return MirrorRequest{}, fmt.Errorf("mirror: the asks cannot be read: %w", err)
|
||||
}
|
||||
r, ok := all[key]
|
||||
if !ok {
|
||||
return MirrorRequest{}, fmt.Errorf("mirror: no copy is kept under %s", key)
|
||||
}
|
||||
src, err := ParseSource(r.Source)
|
||||
if err != nil {
|
||||
return MirrorRequest{}, err
|
||||
}
|
||||
e := Event{Name: src.Name(), Kind: KindUpstream, Digest: r.Digest, Holder: Self, By: by, ID: r.ID,
|
||||
Reason: fmt.Sprintf("%s@%s was not copied: dropped by hand by %s: %s", r.Source, r.Digest, by, why)}
|
||||
if err := v.Say(ImageSeat, EventRefused, e); err != nil {
|
||||
return MirrorRequest{}, fmt.Errorf("mirror: the drop of %s cannot be said, so the ask stays: %w", key, err)
|
||||
}
|
||||
return r, v.Mirrors.Delete(key)
|
||||
}
|
||||
|
||||
// HeldNow lists the holds, for registry_holds.
|
||||
func (v *Verbs) HeldNow() ([]Hold, error) {
|
||||
all, err := v.Holds.List()
|
||||
|
||||
@@ -2,6 +2,8 @@ package holder
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"slices"
|
||||
"sort"
|
||||
"time"
|
||||
@@ -16,10 +18,6 @@ import (
|
||||
const (
|
||||
OldNameWindow = "old-name"
|
||||
DirectWritesWindow = "direct-writes"
|
||||
// BootstrapWindow is the bootstrap's one-off credential (BootstrapFile): open while the file is there. The tools
|
||||
// bundle closes it at its first answered publish; the program, which runs as distribution's account and alone may
|
||||
// remove the file in distribution's state directory, removes it on that close.
|
||||
BootstrapWindow = "bootstrap-push"
|
||||
// WindowFindingAfter is how long a node holds a window open before it is distribution's finding naming the node.
|
||||
WindowFindingAfter = 7 * 24 * time.Hour
|
||||
// The events a window's close says on distribution's own family distribution.events (7 days, 16 MiB, drop
|
||||
@@ -31,16 +29,16 @@ const (
|
||||
)
|
||||
|
||||
// Window is one window's record, kept in distribution's state windows: what registry_old_name and
|
||||
// registry_direct_writes answer.
|
||||
// registry_direct_writes answer. Its open, holding, since, closed and last holding are mesh-sdk's delivery.WindowHeld,
|
||||
// the part every window keeper shares and steps by one rule (delivery.StepWindow).
|
||||
type Window struct {
|
||||
Name string `json:"name"`
|
||||
Open bool `json:"open"`
|
||||
Holding []string `json:"holding,omitempty"`
|
||||
Since map[string]time.Time `json:"since,omitempty"`
|
||||
Dropped []DroppedNode `json:"dropped,omitempty"`
|
||||
Closed time.Time `json:"closed,omitzero"`
|
||||
// LastHolding are the nodes that held it last, said when it closes.
|
||||
LastHolding []string `json:"last_holding,omitempty"`
|
||||
Name string `json:"name"`
|
||||
delivery.WindowHeld
|
||||
Dropped []DroppedNode `json:"dropped,omitempty"`
|
||||
// Why says, of each node holding the window, why it holds it: what its read answered, a node-engine whose
|
||||
// manifest-reads did not answer (an away node, an older node-engine, a timeout) included, so a timeout taken as
|
||||
// stating nothing is told apart from a node-engine that states the old names.
|
||||
Why map[string]string `json:"why,omitempty"`
|
||||
}
|
||||
|
||||
// DroppedNode is one retired node dropped from a window by hand: a hand act.
|
||||
@@ -61,16 +59,42 @@ type Closed struct {
|
||||
}
|
||||
|
||||
// NodeRead is what distribution read of one node: the reader capabilities its node-engine states through
|
||||
// `manifest-reads` (Serves false while it does not serve the verb: it states nothing), whether its declaration names an
|
||||
// artifact, and its build modules' haves: whether each runs a build that publishes through the seats.
|
||||
// `manifest-reads` (Serves false while it does not serve the verb: it states nothing; NotServed then says what the
|
||||
// call answered), whether its declaration names an artifact, and its build modules' haves: whether each runs a build
|
||||
// that publishes through the seats.
|
||||
type NodeRead struct {
|
||||
Node string `json:"node"`
|
||||
Serves bool `json:"serves"`
|
||||
NotServed string `json:"not_served,omitempty"`
|
||||
Stated []string `json:"stated,omitempty"`
|
||||
NamesArtifact bool `json:"names_artifact"`
|
||||
BuildModules map[string]bool `json:"build_modules,omitempty"`
|
||||
}
|
||||
|
||||
// WhyOldName says why a node holds the old name open.
|
||||
func WhyOldName(n NodeRead) string {
|
||||
if !n.Serves {
|
||||
if n.NotServed != "" {
|
||||
return "its node-engine's manifest-reads did not answer, so it counts as stating nothing: " + n.NotServed
|
||||
}
|
||||
return "its node-engine does not serve manifest-reads, so it states nothing"
|
||||
}
|
||||
return "its node-engine does not state " + NewNamesCapability
|
||||
}
|
||||
|
||||
// WhyDirectWrites says why a node holds the direct writes open: its build modules running a build not heard
|
||||
// publishing through the seats.
|
||||
func WhyDirectWrites(n NodeRead) string {
|
||||
var old []string
|
||||
for m, publishes := range n.BuildModules {
|
||||
if !publishes {
|
||||
old = append(old, m)
|
||||
}
|
||||
}
|
||||
sort.Strings(old)
|
||||
return fmt.Sprintf("its %v run a build not heard publishing through the seats", old)
|
||||
}
|
||||
|
||||
// HoldingOldName are the nodes holding the old name open: a node whose declaration names an artifact and whose
|
||||
// node-engine does not state the new names (one not serving manifest-reads states nothing).
|
||||
func HoldingOldName(nodes []NodeRead, dropped []DroppedNode) []string {
|
||||
@@ -110,33 +134,25 @@ func isDropped(dropped []DroppedNode, node string) bool {
|
||||
return slices.ContainsFunc(dropped, func(d DroppedNode) bool { return d.Node == node })
|
||||
}
|
||||
|
||||
// Step moves the window to the nodes holding it at now. It answers the next record, the closing event when it closed in
|
||||
// this step (said once), and the nodes holding it past WindowFindingAfter (the finding; cleared when it closes). A
|
||||
// window once closed stays closed: a node that holds it later is no reason to serve the old name again.
|
||||
func (w Window) Step(holding []string, now time.Time) (next Window, closed *Closed, overdue []string) {
|
||||
next = Window{Name: w.Name, Dropped: w.Dropped, Closed: w.Closed, LastHolding: w.LastHolding}
|
||||
// Step moves the window by mesh-sdk's one rule (delivery.StepWindow): named is every node the read named, dropped ones
|
||||
// included, holding those holding the window at now. A read naming no node judges nothing. It answers the next record,
|
||||
// the closing event when it closed in this step (said once), and the nodes holding it past WindowFindingAfter (the
|
||||
// finding; cleared when it closes). A window once closed stays closed: a node that holds it later is no reason to serve
|
||||
// the old name again or to take direct writes again.
|
||||
func (w Window) Step(named, holding []string, now time.Time) (next Window, closed *Closed, overdue []string) {
|
||||
next = w
|
||||
if !w.Open && !w.Closed.IsZero() {
|
||||
return next, nil, nil
|
||||
}
|
||||
if len(holding) == 0 {
|
||||
next.Closed, next.LastHolding = now, w.Holding
|
||||
if next.LastHolding == nil {
|
||||
next.LastHolding = []string{}
|
||||
}
|
||||
return next, &Closed{Window: w.Name, At: now, LastHolding: next.LastHolding, Dropped: w.Dropped}, nil
|
||||
held, shut, overdue := delivery.StepWindow(w.WindowHeld, named, holding, now, WindowFindingAfter)
|
||||
next.WindowHeld = held
|
||||
if !held.Open {
|
||||
next.Why = nil
|
||||
}
|
||||
next.Open, next.Holding, next.Since = true, holding, map[string]time.Time{}
|
||||
for _, n := range holding {
|
||||
since, had := w.Since[n]
|
||||
if !had {
|
||||
since = now
|
||||
}
|
||||
next.Since[n] = since
|
||||
if now.Sub(since) >= WindowFindingAfter {
|
||||
overdue = append(overdue, n)
|
||||
}
|
||||
if !shut {
|
||||
return next, nil, overdue
|
||||
}
|
||||
return next, nil, overdue
|
||||
return next, &Closed{Window: w.Name, At: held.Closed, LastHolding: held.LastHolding, Dropped: w.Dropped}, nil
|
||||
}
|
||||
|
||||
// Drop drops a retired node from the window by hand, a hand act with who and why.
|
||||
|
||||
@@ -48,6 +48,7 @@
|
||||
"registry_health",
|
||||
"registry_holds",
|
||||
"registry_incoming",
|
||||
"registry_mirrors",
|
||||
"registry_old_name"
|
||||
],
|
||||
"invokes": [
|
||||
@@ -83,9 +84,14 @@
|
||||
{
|
||||
"name": "transfers",
|
||||
"storage": "key-value",
|
||||
"template": "state"
|
||||
},
|
||||
{
|
||||
"name": "idle",
|
||||
"storage": "key-value",
|
||||
"template": "state",
|
||||
"options": {
|
||||
"ttl-seconds": 86400
|
||||
"key-ttl": true
|
||||
}
|
||||
},
|
||||
{
|
||||
|
||||
@@ -13,8 +13,10 @@ trusts that root. On a machine with no public domain the key is left out, and th
|
||||
|
||||
**Both package provisions carry no credential** (`credential.form` none, design 51 *Publishing*): reads of the
|
||||
organisation's packages are anonymous, and every write goes through the seats' verbs, so no consumer is given an
|
||||
account of the forge. The registry accounts the TypeScript pair provisioner made before stay on the forge until
|
||||
removed by hand; nothing hands their passwords out any more.
|
||||
account of the forge. The registry accounts the TypeScript pair provisioner made before (`mesh_<node>_<consumer>`)
|
||||
and its two teams `packages` and `readers` are removed by `cmd/agent-teams` at each of its runs (ADR 0605 §5 revokes the
|
||||
npm write token): both teams go, and each account of that form whose only teams they are; one the forge refuses to
|
||||
delete because it owns something keeps its data and has its login prohibited.
|
||||
|
||||
## Announcing pull requests to the merge check (novox/hq to-be 45 §9, issue 347)
|
||||
|
||||
@@ -142,7 +144,8 @@ teams exist and who is in them is the operator's, and it is one trusted config v
|
||||
whenever that file changes. For each team it makes `agent-read-<team>` in the organisation of the first repository's
|
||||
owner, with permission read and no other right, its repositories exactly the listed ones and its members exactly the
|
||||
listed accounts (an account that does not exist yet is skipped and said). A team named `agent-read-*` that has left
|
||||
the value is emptied, never deleted. A team without that prefix is never touched. There are no verbs: a change is a
|
||||
the value is emptied, never deleted. A team without that prefix is never touched, but the two the module made once
|
||||
and declares no more, `packages` and `readers` (above). There are no verbs: a change is a
|
||||
change of the value. It says each change in one line on stdout, and never a secret. `go test ./cmd/agent-teams`
|
||||
runs it against a fake forge.
|
||||
|
||||
@@ -208,6 +211,8 @@ deletes the versions nothing keeps.
|
||||
| `npm_retention` | a | the versions retention would delete, and the bytes. **A dry run unless `dry_run` is false**; a real run needs `why` |
|
||||
| `go_packages` | r | every Go module of the registry's owner (the Go seat's), each version newest first with its publish time, its size and why it is kept: a `go.mod` on a default branch requires it, or it is among the newest `keep` |
|
||||
| `go_retention` | a | the Go module versions retention would delete, and the bytes; as `npm_retention`, and a `go.mod` that could not be read stops a real run. Each deletion says `removed` on the Go seat |
|
||||
| `npm_incoming` | a | the npm seat's open transfers in the module's object store `incoming`, its fill and the cap its binding carries; with `drop` and `why`, drops one by hand. A transfer with no chunk for 24 hours is dropped on its own, when its key in the state `idle` ends |
|
||||
| `go_incoming` | a | the same for the Go seat |
|
||||
|
||||
**A version is kept when:**
|
||||
|
||||
|
||||
@@ -2,7 +2,9 @@
|
||||
// runtime runs once when the module starts and again whenever the module's config value `agent-read-teams` changes:
|
||||
// it reads the file the mesh renders from that value, and makes the forge's teams named agent-read-* match it. Each
|
||||
// team reads only; its repositories and its members (agent accounts, agent-<node>) are exactly the value's. It
|
||||
// touches nothing outside the value: no team without the prefix, no repository or account of another kind.
|
||||
// touches nothing else of the value's kind: no team without the prefix, no repository or account of another kind. Beside
|
||||
// the value it removes what the module made once and declares no more (retire.go): the npm pair's teams packages and
|
||||
// readers and the accounts in them alone.
|
||||
//
|
||||
// It says each change on stdout in one line, and never a secret.
|
||||
package main
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"gitea/internal/forgehttp"
|
||||
)
|
||||
|
||||
// The forge accounts and teams the module made once and declares no more (novox/hq ADR 0605 §5: no build module holds
|
||||
// a registry's write credential; the npm write token is revoked). The npm pair provisioner, removed when the package
|
||||
// seats came, made each consumer of npm-package-registry an account under the mesh's derived login
|
||||
// (mesh_<node>_<consumer>) and put it in the organisation's team packages (read and write) or readers (read only).
|
||||
// Reads of the packages are anonymous and every write goes through the seats, so both teams go, and each account of
|
||||
// that form in no team but them goes with them. Nothing else is touched: a team of another name, an account of
|
||||
// another form or one still in a team the module does not retire stays as it is.
|
||||
var RetiredTeams = []string{"packages", "readers"}
|
||||
|
||||
// retiredAccount is the login form the npm pair provisioner made its accounts under.
|
||||
var retiredAccount = regexp.MustCompile(`^mesh_[a-z0-9-]+_[a-z0-9_]+$`)
|
||||
|
||||
// Retire removes the retired teams in every organisation, and each account of the pair's form whose only teams they
|
||||
// are. An account the forge refuses to delete (it owns a repository or an organisation, so a delete would take what
|
||||
// it owns) has its login prohibited instead, said, so its password no longer reaches anything. Each removal is said
|
||||
// in one line; a run with nothing left to retire says nothing.
|
||||
func (s *Syncer) Retire(ctx context.Context, orgs []string) error {
|
||||
teamsOf := map[string][]string{} // login -> every team it is in, as org/team
|
||||
type retired struct {
|
||||
org string
|
||||
team forgeTeam
|
||||
}
|
||||
var gone []retired
|
||||
for _, org := range orgs {
|
||||
teams, err := s.teams(ctx, org)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, t := range teams {
|
||||
var members []forgeMember
|
||||
if err := s.list(ctx, fmt.Sprintf("/api/v1/teams/%d/members", t.ID), &members); err != nil {
|
||||
return fmt.Errorf("the members of %s in %s could not be read, so nothing is retired: %w", t.Name, org, err)
|
||||
}
|
||||
for _, m := range members {
|
||||
teamsOf[m.Login] = append(teamsOf[m.Login], org+"/"+t.Name)
|
||||
}
|
||||
if slices.Contains(RetiredTeams, t.Name) {
|
||||
gone = append(gone, retired{org, t})
|
||||
}
|
||||
}
|
||||
}
|
||||
isRetired := func(orgTeam string) bool {
|
||||
_, team, _ := strings.Cut(orgTeam, "/")
|
||||
return slices.Contains(RetiredTeams, team)
|
||||
}
|
||||
var accounts []string
|
||||
for login, in := range teamsOf {
|
||||
if retiredAccount.MatchString(login) && slices.ContainsFunc(in, isRetired) && !slices.ContainsFunc(in, func(t string) bool { return !isRetired(t) }) {
|
||||
accounts = append(accounts, login)
|
||||
}
|
||||
}
|
||||
sort.Strings(accounts)
|
||||
var errs []string
|
||||
for _, a := range accounts {
|
||||
if err := s.removeAccount(ctx, a); err != nil {
|
||||
errs = append(errs, err.Error())
|
||||
}
|
||||
}
|
||||
for _, r := range gone {
|
||||
if err := s.Forge.SendJSON(ctx, http.MethodDelete, fmt.Sprintf("/api/v1/teams/%d", r.team.ID), nil, nil); err != nil && !errors.Is(err, forgehttp.ErrNotFound) {
|
||||
errs = append(errs, fmt.Sprintf("the retired team %s in %s could not be removed: %v", r.team.Name, r.org, err))
|
||||
continue
|
||||
}
|
||||
s.say("removed the retired team %s in %s (the npm pair's, ADR 0605 §5)", r.team.Name, r.org)
|
||||
}
|
||||
if len(errs) > 0 {
|
||||
return errors.New(strings.Join(errs, "; "))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// removeAccount deletes a retired account, never purging what it owns; one the forge refuses to delete has its login
|
||||
// prohibited instead.
|
||||
func (s *Syncer) removeAccount(ctx context.Context, login string) error {
|
||||
err := s.Forge.SendJSON(ctx, http.MethodDelete, "/api/v1/admin/users/"+url.PathEscape(login), nil, nil)
|
||||
if err == nil || errors.Is(err, forgehttp.ErrNotFound) {
|
||||
s.say("removed the retired account %s (the npm pair's, ADR 0605 §5)", login)
|
||||
return nil
|
||||
}
|
||||
var se *forgehttp.StatusError
|
||||
if !errors.As(err, &se) || se.Status != http.StatusUnprocessableEntity {
|
||||
return fmt.Errorf("the retired account %s could not be removed: %w", login, err)
|
||||
}
|
||||
if perr := s.Forge.SendJSON(ctx, http.MethodPatch, "/api/v1/admin/users/"+url.PathEscape(login), map[string]any{
|
||||
"login_name": login, "source_id": 0, "prohibit_login": true,
|
||||
}, nil); perr != nil {
|
||||
return fmt.Errorf("the retired account %s could not be removed (%v) nor its login prohibited: %w", login, err, perr)
|
||||
}
|
||||
s.say("the retired account %s owns something, so the forge keeps it (%v): its login is prohibited instead", login, err)
|
||||
return nil
|
||||
}
|
||||
@@ -135,6 +135,10 @@ func (s *Syncer) Sync(ctx context.Context, v Value) error {
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := s.Retire(ctx, orgs); err != nil {
|
||||
s.say("not retired: %v", err)
|
||||
failed = append(failed, "the retired teams and accounts")
|
||||
}
|
||||
if len(failed) > 0 {
|
||||
return fmt.Errorf("not applied: %s", strings.Join(failed, ", "))
|
||||
}
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"slices"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -34,6 +35,9 @@ type fakeForge struct {
|
||||
users map[string]bool
|
||||
teams []*fTeam
|
||||
next int64
|
||||
// owners are accounts that own a repository: the forge refuses to delete them (422); prohibited, whose login
|
||||
// was prohibited.
|
||||
owners, prohibited map[string]bool
|
||||
}
|
||||
|
||||
func newFake(orgs []string, users ...string) *fakeForge {
|
||||
@@ -122,6 +126,26 @@ func (f *fakeForge) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
out = nil
|
||||
}
|
||||
reply(200, out)
|
||||
case strings.HasPrefix(p, "/api/v1/admin/users/"):
|
||||
u := strings.TrimPrefix(p, "/api/v1/admin/users/")
|
||||
switch {
|
||||
case !f.users[u]:
|
||||
reply(404, nil)
|
||||
case r.Method == http.MethodDelete && f.owners[u]:
|
||||
reply(422, map[string]string{"message": "user still owns one or more repositories"})
|
||||
case r.Method == http.MethodDelete:
|
||||
delete(f.users, u)
|
||||
for _, t := range f.teams {
|
||||
delete(t.Members, u)
|
||||
}
|
||||
reply(204, nil)
|
||||
case r.Method == http.MethodPatch && in["prohibit_login"] == true:
|
||||
if f.prohibited == nil {
|
||||
f.prohibited = map[string]bool{}
|
||||
}
|
||||
f.prohibited[u] = true
|
||||
reply(200, map[string]string{})
|
||||
}
|
||||
case strings.HasPrefix(p, "/api/v1/users/"):
|
||||
if !f.users[strings.TrimPrefix(p, "/api/v1/users/")] {
|
||||
reply(404, nil)
|
||||
@@ -155,6 +179,10 @@ func (f *fakeForge) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
}
|
||||
reply(200, out)
|
||||
case teamPath.MatchString(p) && r.Method == http.MethodDelete:
|
||||
t := f.find(teamPath.FindStringSubmatch(p)[1])
|
||||
f.teams = slices.DeleteFunc(f.teams, func(x *fTeam) bool { return x == t })
|
||||
reply(204, nil)
|
||||
case teamPath.MatchString(p):
|
||||
t := f.find(teamPath.FindStringSubmatch(p)[1])
|
||||
t.Perm = in["permission"].(string)
|
||||
@@ -331,3 +359,39 @@ func TestReadValue(t *testing.T) {
|
||||
t.Fatal("an unknown field was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
// Security review F5 (ADR 0605 §5): the npm pair's teams packages and readers go, and each account of the pair's form
|
||||
// whose only teams they are; one that owns a repository is kept with its login prohibited; an account of the pair's
|
||||
// form still in another team, an account of another form and every other team stay; a second run changes nothing.
|
||||
func TestTheNpmPairsTeamsAndAccountsAreRetired(t *testing.T) {
|
||||
f := newFake([]string{"novox"}, "mesh_ace_mesh_build_node", "mesh_shanks_lab", "mesh_g14_agent", "mesh_novox_owner",
|
||||
"mesh-packages", "agent-ace", "jschoubben")
|
||||
f.owners = map[string]bool{"mesh_novox_owner": true}
|
||||
f.add("novox", "packages", "none", nil, []string{"mesh_ace_mesh_build_node", "mesh_g14_agent", "mesh_novox_owner", "jschoubben"})
|
||||
f.add("novox", "readers", "none", nil, []string{"mesh_ace_mesh_build_node", "mesh_shanks_lab"})
|
||||
f.add("novox", "agents", "none", nil, []string{"mesh_g14_agent", "agent-ace"})
|
||||
f.add("novox", "mesh-packages", "none", nil, []string{"mesh-packages"})
|
||||
s, said := syncer(t, f)
|
||||
if err := s.Sync(context.Background(), Value{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f.byName("novox", "packages") != nil || f.byName("novox", "readers") != nil {
|
||||
t.Errorf("the retired teams stay: %+v", f.teams)
|
||||
}
|
||||
if f.byName("novox", "agents") == nil || f.byName("novox", "mesh-packages") == nil {
|
||||
t.Errorf("a team the module did not retire was removed: %+v", f.teams)
|
||||
}
|
||||
for u, want := range map[string]bool{"mesh_ace_mesh_build_node": false, "mesh_shanks_lab": false, "mesh_g14_agent": true,
|
||||
"mesh_novox_owner": true, "mesh-packages": true, "agent-ace": true, "jschoubben": true} {
|
||||
if f.users[u] != want {
|
||||
t.Errorf("account %s kept %v, want %v", u, f.users[u], want)
|
||||
}
|
||||
}
|
||||
if !f.prohibited["mesh_novox_owner"] || len(f.prohibited) != 1 {
|
||||
t.Errorf("prohibited %v, want mesh_novox_owner alone", f.prohibited)
|
||||
}
|
||||
before := len(*said)
|
||||
if err := s.Sync(context.Background(), Value{}); err != nil || len(*said) != before {
|
||||
t.Errorf("a second run: %v, said %v; want nothing", err, (*said)[before:])
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
package main
|
||||
|
||||
// The package seats' incoming, read and steered (design 51 *Publishing*; tools review F4): npm_incoming and
|
||||
// go_incoming list a seat's open transfers, incoming's fill and the cap its binding carries, and drop one by hand.
|
||||
// The holder counts against that cap, never a number of its own, and drops a transfer left with no chunk for 24 hours
|
||||
// when its idle key ends (the bus's marker), never on a timer.
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/parts"
|
||||
|
||||
"gitea/internal/packageseats"
|
||||
)
|
||||
|
||||
// incomingTool is a seat's <type>_incoming.
|
||||
func incomingTool(typ, seat string, holder func() (*packageseats.Holder, error)) stdio.Tool {
|
||||
str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) }
|
||||
return stdio.Tool{
|
||||
Name: typ + "_incoming",
|
||||
Description: "The " + seat + " seat's open transfers in the forge module's object store incoming (each with its " +
|
||||
"opener, its parts and when it last took a chunk), incoming's fill and the cap its binding carries; a " +
|
||||
"transfer with no chunk for 24 hours is dropped on its own. With drop, drops one by hand, its chunks with " +
|
||||
"it (a hand act, why required). (a)",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{
|
||||
"drop": map[string]any{"type": "string", "description": "a transfer id (the caller's own name) to drop"},
|
||||
"why": map[string]any{"type": "string", "description": "why it is dropped"},
|
||||
}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
h, err := holder()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if id := str(args, "drop"); id != "" {
|
||||
if str(args, "why") == "" {
|
||||
return nil, errors.New("a drop is a hand act: say why")
|
||||
}
|
||||
t, err := h.Drop(seat, id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "transfer %s of %s dropped by hand by %s: %s\n", id, seat, stdio.CallerOf(args), str(args, "why"))
|
||||
return map[string]any{"dropped": t.ID}, nil
|
||||
}
|
||||
return h.IncomingOf(seat)
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// followIdle drops a transfer when its idle key ends (packageseats.Holder.Expired): the bus's marker is the event.
|
||||
func followIdle(holder func() (*packageseats.Holder, error)) {
|
||||
parts.FollowIdle(packageseats.IdleState, func(id string) (bool, error) {
|
||||
h, err := holder()
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("transfer %s's idle bound ended, but the holder cannot be made: %w", id, err)
|
||||
}
|
||||
return h.Expired(id)
|
||||
}, func(id string, dropped bool, err error) {
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
return
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "transfer %s had no chunk for %s: dropped with its chunks\n", id, packageseats.TransferIdle)
|
||||
})
|
||||
}
|
||||
@@ -21,11 +21,13 @@ import (
|
||||
// ToolNames are the tools this bundle serves. The module's manifest lists no `tools`: its TypeScript
|
||||
// bundle's tools are served without one, and a claim without `serves` would offer a `tools` list to both
|
||||
// of the module's seats as their verbs. The README names these two, and a test holds the two together.
|
||||
var ToolNames = []string{"go_packages", "go_retention", "npm_packages", "npm_retention"}
|
||||
var ToolNames = []string{"go_packages", "go_retention", "go_incoming", "npm_packages", "npm_retention", "npm_incoming"}
|
||||
|
||||
func main() {
|
||||
forge := func() (*Forge, error) { return ForgeFromEnv() }
|
||||
tools := append(Tools(forge), SeatTools(func() (*packageseats.Holder, error) { return busHolder(forge) })...)
|
||||
holder := func() (*packageseats.Holder, error) { return busHolder(forge) }
|
||||
go followIdle(holder)
|
||||
tools := append(Tools(forge, holder), SeatTools(holder)...)
|
||||
if err := stdio.Serve("", tools); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
@@ -34,8 +36,9 @@ func main() {
|
||||
|
||||
// Tools are the bundle's tools over the forge the function gives; asked per call, so a manifest that
|
||||
// does not set the environment is a refusal naming what is missing, not a bundle that will not start.
|
||||
func Tools(forge func() (*Forge, error)) []stdio.Tool {
|
||||
return append(typeTools(TypeGo, forge), typeTools(TypeNpm, forge)...)
|
||||
func Tools(forge func() (*Forge, error), holder func() (*packageseats.Holder, error)) []stdio.Tool {
|
||||
out := append(typeTools(TypeGo, forge), incomingTool(TypeGo, packageseats.GoSeat, holder))
|
||||
return append(append(out, typeTools(TypeNpm, forge)...), incomingTool(TypeNpm, packageseats.NpmSeat, holder))
|
||||
}
|
||||
|
||||
// typeTools are one package type's two tools, <type>_packages and <type>_retention, over the forge reading that type.
|
||||
|
||||
@@ -402,12 +402,12 @@ func TestAWrongPasswordIsAFailureThatDoesNotSayThePassword(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestToolsAreTheOnesTheReadmeNamesAndSayWhatTheyDo(t *testing.T) {
|
||||
tools := Tools(func() (*Forge, error) { return nil, fmt.Errorf("unset") })
|
||||
tools := Tools(func() (*Forge, error) { return nil, fmt.Errorf("unset") }, func() (*packageseats.Holder, error) { return nil, fmt.Errorf("unset") })
|
||||
var served []string
|
||||
for _, tl := range tools {
|
||||
served = append(served, tl.Name)
|
||||
suffix := "(r)"
|
||||
if strings.HasSuffix(tl.Name, "_retention") {
|
||||
if strings.HasSuffix(tl.Name, "_retention") || strings.HasSuffix(tl.Name, "_incoming") {
|
||||
suffix = "(a)"
|
||||
}
|
||||
if !strings.HasSuffix(tl.Description, suffix) {
|
||||
@@ -468,11 +468,15 @@ func TestToolsAreTheOnesTheReadmeNamesAndSayWhatTheyDo(t *testing.T) {
|
||||
t.Errorf("the seat tools are %v", names)
|
||||
}
|
||||
req, _ := json.Marshal(m.Requires)
|
||||
for _, want := range []string{`"name":"incoming"`, `"name":"transfers"`, `"name":"findings"`} {
|
||||
for _, want := range []string{`"name":"incoming"`, `"name":"transfers"`, `"name":"findings"`, `"name":"idle","options":{"key-ttl":true}`} {
|
||||
if !strings.Contains(string(req), want) {
|
||||
t.Errorf("the manifest does not require %s", want)
|
||||
t.Errorf("the manifest does not require %s (%s)", want, req)
|
||||
}
|
||||
}
|
||||
// transfers keeps for good: a lifetime of its own would drop a transfer's key and leave its chunks in incoming.
|
||||
if strings.Contains(string(req), `"name":"transfers","options"`) {
|
||||
t.Errorf("transfers has options (a lifetime): %s", req)
|
||||
}
|
||||
// The retention asks the builder's versions; the forge's other bundles ask the indexer's modules and record a re-read.
|
||||
if strings.Join(m.Invokes, ",") != "mesh-delivery-indexer.indexer_read_modules,seat:mesh-hand-acts.hand-act,mesh-delivery-builder.builder_versions" {
|
||||
t.Errorf("the manifest invokes %v", m.Invokes)
|
||||
|
||||
@@ -263,6 +263,9 @@ func finding(kind, subject, words string, raised bool) {
|
||||
}
|
||||
}
|
||||
|
||||
// boundCap is incoming's cap from its binding, read once per bundle.
|
||||
var boundCap = parts.BusCap(packageseats.IncomingObjects)
|
||||
|
||||
// busHolder is the holder over the runtime's bus state, the builder's numbering and the forge.
|
||||
func busHolder(forge func() (*Forge, error)) (*packageseats.Holder, error) {
|
||||
callers, err := packageseats.ConfiguredCallers()
|
||||
@@ -271,7 +274,8 @@ func busHolder(forge func() (*Forge, error)) (*packageseats.Holder, error) {
|
||||
}
|
||||
return &packageseats.Holder{Callers: callers, Objects: parts.BusObjects(packageseats.IncomingObjects),
|
||||
Transfers: parts.BusTransfers[packageseats.Opening](packageseats.TransfersState),
|
||||
Builder: delivery.BuilderVersions{Ask: stdio.Ask}, Store: seatStore{forge}, Say: say, Finding: finding}, nil
|
||||
Builder: delivery.BuilderVersions{Ask: stdio.Ask}, Store: seatStore{forge}, Say: say, Finding: finding,
|
||||
Cap: boundCap, Idle: parts.BusArm(packageseats.IdleState)}, nil
|
||||
}
|
||||
|
||||
// SeatTools are both seats' verbs as tools <seat>.<verb>.
|
||||
|
||||
@@ -114,6 +114,7 @@ func (p *player) handle(m wire) {
|
||||
Subject string `json:"subject"`
|
||||
Body json.RawMessage `json:"body"`
|
||||
Kind string `json:"kind"`
|
||||
TTL int `json:"ttl_seconds"`
|
||||
}
|
||||
_ = json.Unmarshal(m.Params, &a)
|
||||
switch {
|
||||
@@ -131,7 +132,7 @@ func (p *player) handle(m wire) {
|
||||
case strings.HasPrefix(m.Method, "mesh/objects."):
|
||||
p.objects(m, a.Objects, a.Name, a.Data)
|
||||
case strings.HasPrefix(m.Method, "mesh/state."):
|
||||
p.state(m, a.State, a.Key, a.Value)
|
||||
p.state(m, a.State, a.Key, a.Value, a.TTL)
|
||||
default:
|
||||
p.answer(m.ID, nil, errors.New("not played here: "+m.Method))
|
||||
}
|
||||
@@ -172,7 +173,7 @@ func (p *player) objects(m wire, local, name string, data []byte) {
|
||||
}
|
||||
}
|
||||
|
||||
func (p *player) state(m wire, state, key string, value json.RawMessage) {
|
||||
func (p *player) state(m wire, state, key string, value json.RawMessage, ttl int) {
|
||||
kv, err := p.js.KeyValue(p.ctx, "gitea_"+state)
|
||||
if err != nil {
|
||||
p.answer(m.ID, nil, err)
|
||||
@@ -193,6 +194,13 @@ func (p *player) state(m wire, state, key string, value json.RawMessage) {
|
||||
case "mesh/state.put":
|
||||
rev, err := kv.Put(p.ctx, key, value)
|
||||
p.answer(m.ID, map[string]any{"revision": rev}, err)
|
||||
case "mesh/state.create":
|
||||
var opts []jetstream.KVCreateOpt
|
||||
if ttl > 0 {
|
||||
opts = append(opts, jetstream.KeyTTL(time.Duration(ttl)*time.Second))
|
||||
}
|
||||
rev, err := kv.Create(p.ctx, key, value, opts...)
|
||||
p.answer(m.ID, map[string]any{"revision": rev}, err)
|
||||
case "mesh/state.delete":
|
||||
p.answer(m.ID, map[string]any{}, kv.Delete(p.ctx, key))
|
||||
case "mesh/state.keys":
|
||||
@@ -268,6 +276,10 @@ func TestThePackageSeatsWhole(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// idle takes keys with their own time to live (key-ttl), as the bus module makes it.
|
||||
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: "gitea_" + packageseats.IdleState, LimitMarkerTTL: time.Minute}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := js.CreateObjectStore(ctx, jetstream.ObjectStoreConfig{Bucket: "gitea_" + packageseats.IncomingObjects}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -364,6 +376,11 @@ func TestThePackageSeatsWhole(t *testing.T) {
|
||||
if transfer == "" || len(lacks) != 1 || lacks[0] != digest {
|
||||
t.Fatalf("publish answered transfer %q lacks %v", transfer, lacks)
|
||||
}
|
||||
// The open transfer is bounded by its idle key, written through the runtime with its own time to live.
|
||||
idle, _ := js.KeyValue(ctx, "gitea_"+packageseats.IdleState)
|
||||
if _, err := idle.Get(ctx, packageseats.GoSeat+"_"+transfer); err != nil {
|
||||
t.Errorf("the open transfer %s has no idle key: %v", transfer, err)
|
||||
}
|
||||
if err := send(transfer, digest, zipped); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -44,8 +44,8 @@ const (
|
||||
)
|
||||
|
||||
// The bounds of incoming (design 51 *Publishing*, ADR 0611): a chunk at most 1 MiB, a transfer with no chunk for 24
|
||||
// hours dropped, a finding past 80 % of the cap. A package is one file, far smaller than an image, so the cap is the
|
||||
// manifest's 2 GiB.
|
||||
// hours dropped, a finding past 80 % of the cap. The cap is the one incoming's binding carries; IncomingCap is the
|
||||
// manifest's 2 GiB (a package is one file, far smaller than an image), said only where no binding is read.
|
||||
const (
|
||||
MaxChunk = 1 << 20
|
||||
IncomingCap = 2 << 30
|
||||
@@ -53,6 +53,9 @@ const (
|
||||
IncomingFullPct = 80
|
||||
IncomingObjects = "incoming"
|
||||
TransfersState = "transfers"
|
||||
// IdleState is the state of one key per open transfer with its own time to live (a state declared key-ttl): its
|
||||
// end is the event that drops a transfer left with no chunk for TransferIdle, never a timer.
|
||||
IdleState = "idle"
|
||||
// KindPackage is the one kind a package seat holds.
|
||||
KindPackage = "package"
|
||||
// Self is the forge module, the holder every outcome names.
|
||||
@@ -151,9 +154,16 @@ type Holder struct {
|
||||
// Finding writes (raised) or clears the module's finding of a kind for a subject.
|
||||
Finding func(kind, subject, words string, raised bool)
|
||||
Now func() time.Time
|
||||
Cap uint64
|
||||
// Cap reads incoming's cap from its binding (parts.BusCap); IncomingCap when nil or when the binding names none.
|
||||
Cap func() (uint64, error)
|
||||
// Idle writes a transfer's idle key with its own time to live, once (parts.BusArm over IdleState); the receiver
|
||||
// arms it on each open that leaves a transfer open. Nil arms nothing.
|
||||
Idle func(key string, ttl time.Duration) error
|
||||
}
|
||||
|
||||
// CapNow is incoming's cap as its binding carries it.
|
||||
func (h *Holder) CapNow() uint64 { return h.receiver().CapNow() }
|
||||
|
||||
func (h *Holder) now() time.Time {
|
||||
if h.Now != nil {
|
||||
return h.Now().UTC()
|
||||
@@ -168,12 +178,8 @@ func Reference(name, version string) string { return name + "@" + version }
|
||||
func key(seat, transfer string) string { return seat + "_" + transfer }
|
||||
|
||||
func (h *Holder) receiver() *parts.Receiver[Opening] {
|
||||
cap := h.Cap
|
||||
if cap == 0 {
|
||||
cap = IncomingCap
|
||||
}
|
||||
return &parts.Receiver[Opening]{Objects: h.Objects, Transfers: h.Transfers, Cap: cap, MaxChunk: MaxChunk,
|
||||
Idle: TransferIdle, Full: IncomingFullPct, Now: h.now,
|
||||
return &parts.Receiver[Opening]{Objects: h.Objects, Transfers: h.Transfers, Cap: IncomingCap, Bound: h.Cap,
|
||||
MaxChunk: MaxChunk, Idle: TransferIdle, Arm: h.Idle, Full: IncomingFullPct, Now: h.now,
|
||||
// The builder's numbering is asked before anything is uploaded (ADR 0605 §4).
|
||||
Check: func(t Transfer) error {
|
||||
ok, why, err := h.Builder.Recorded(t.Shape.Name, t.Shape.Version, t.Shape.Digest)
|
||||
@@ -230,7 +236,6 @@ func (h *Holder) Publish(seat, caller string, p Publish) (Answer, error) {
|
||||
if err := checkPublish(seat, p); err != nil {
|
||||
return Answer{}, err
|
||||
}
|
||||
_, _ = h.receiver().DropIdle()
|
||||
// A rebuild of a published version answers it held and uploads nothing; the same version with other content is
|
||||
// refused, as the registry would refuse it.
|
||||
if d, ok, err := h.Store.Held(seat, p.Name, p.Version); err != nil {
|
||||
@@ -324,11 +329,46 @@ func (h *Holder) Retained(seat, name, version, digest string) error {
|
||||
return h.Say(seat, EventRemoved, Outcome{Name: name, Version: version, Kind: KindPackage, Digest: digest, Reason: ReasonRetention})
|
||||
}
|
||||
|
||||
// Drop drops a transfer by hand; DropIdle every transfer idle for TransferIdle.
|
||||
// Drop drops a transfer by hand.
|
||||
func (h *Holder) Drop(seat, transfer string) (Transfer, error) {
|
||||
return h.receiver().Drop(key(seat, transfer))
|
||||
}
|
||||
|
||||
// Incoming is what npm_incoming and go_incoming answer of a seat: its open transfers, incoming's fill (both seats
|
||||
// share the store) and the cap its binding carries.
|
||||
type Incoming struct {
|
||||
Seat string `json:"seat"`
|
||||
Transfers []Transfer `json:"transfers"`
|
||||
Used uint64 `json:"used"`
|
||||
Cap uint64 `json:"cap"`
|
||||
}
|
||||
|
||||
// IncomingOf reads a seat's open transfers and incoming's fill.
|
||||
func (h *Holder) IncomingOf(seat string) (Incoming, error) {
|
||||
if _, ok := Seats[seat]; !ok {
|
||||
return Incoming{}, fmt.Errorf("the forge holds no seat %s", seat)
|
||||
}
|
||||
all, err := h.Transfers.List()
|
||||
if err != nil {
|
||||
return Incoming{}, fmt.Errorf("the transfers cannot be read: %w", err)
|
||||
}
|
||||
out := Incoming{Seat: seat, Transfers: []Transfer{}, Cap: h.CapNow()}
|
||||
for _, t := range all {
|
||||
if t.Shape.Seat == seat {
|
||||
out.Transfers = append(out.Transfers, t)
|
||||
}
|
||||
}
|
||||
if out.Used, err = h.Objects.Used(); err != nil {
|
||||
return out, fmt.Errorf("incoming's fill cannot be read: %w", err)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Expired is told that a transfer's idle key (its key in the state transfers) ended (the bus's marker,
|
||||
// parts.FollowIdle): the receiver drops a transfer with no chunk for TransferIdle with its chunks, arms one that took a
|
||||
// chunk since again for the rest, and leaves one gone. It answers whether it dropped it.
|
||||
func (h *Holder) Expired(id string) (bool, error) { return h.receiver().Expired(id) }
|
||||
|
||||
func checkPublish(seat string, p Publish) error {
|
||||
switch {
|
||||
case Seats[seat] == "":
|
||||
|
||||
@@ -149,6 +149,48 @@ func TestTheForgeHoldsBothPackageSeats(t *testing.T) {
|
||||
}
|
||||
}
|
||||
})
|
||||
// Tools review F4 and running review F2's kind: a seat's incoming is read per seat with the cap its binding
|
||||
// carries; an open transfer is armed with its idle key, re-armed for the rest when it took a chunk since, and
|
||||
// dropped with its chunks when it had none for 24 hours; a cap from the binding refuses a transfer past it.
|
||||
t.Run("incoming: read per seat, the binding's cap, an idle transfer dropped when its key ends", func(t *testing.T) {
|
||||
b := newBed(true)
|
||||
now := time.Unix(1e9, 0)
|
||||
b.h.Now = func() time.Time { return now }
|
||||
armed := map[string]time.Duration{}
|
||||
b.h.Idle = func(k string, ttl time.Duration) error { armed[k] = ttl; return nil }
|
||||
b.h.Cap = func() (uint64, error) { return 1000, nil }
|
||||
big := []byte(strings.Repeat("x", 600))
|
||||
if _, err := b.h.Publish(NpmSeat, nodeBuild, pub("a", "0.1.0", big)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := b.h.Publish(GoSeat, goBuild, pub("b", "0.1.0", big)); err != nil || armed["go-package-registry_t-b"] != TransferIdle {
|
||||
t.Fatalf("publish = %v, armed %v; want the Go transfer's idle key for %s", err, armed, TransferIdle)
|
||||
}
|
||||
_, _ = b.h.PublishPart(NpmSeat, nodeBuild, "t-a", digest(big), 0, false, big[:500])
|
||||
in, err := b.h.IncomingOf(NpmSeat)
|
||||
if err != nil || len(in.Transfers) != 1 || in.Transfers[0].Shape.Name != "a" || in.Cap != 1000 || in.Used != 500 {
|
||||
t.Fatalf("IncomingOf(npm) = %+v, %v; want a's transfer alone, 500 used of the binding's 1000", in, err)
|
||||
}
|
||||
if _, err := b.h.Publish(NpmSeat, nodeBuild, pub("c", "0.1.0", big)); err == nil || !strings.Contains(err.Error(), "cap") {
|
||||
t.Errorf("a transfer past the binding's cap = %v; want refused at the cap", err)
|
||||
}
|
||||
now = now.Add(time.Hour)
|
||||
_, _ = b.h.PublishPart(GoSeat, goBuild, "t-b", digest(big), 0, false, big[:10])
|
||||
now = now.Add(TransferIdle - time.Hour)
|
||||
if dropped, err := b.h.Expired("go-package-registry_t-b"); err != nil || dropped || armed["go-package-registry_t-b"] != time.Hour {
|
||||
t.Errorf("b's key ends an hour after a chunk: %v, %v, armed %v; want kept, armed 1h", dropped, err, armed)
|
||||
}
|
||||
if dropped, err := b.h.Expired("npm-package-registry_t-a"); err != nil || !dropped {
|
||||
t.Errorf("a's key ends 24 hours after its last chunk: %v, %v; want dropped", dropped, err)
|
||||
}
|
||||
if in, _ := b.h.IncomingOf(NpmSeat); len(in.Transfers) != 0 || in.Used != 10 {
|
||||
t.Errorf("after a's drop: %+v; want no npm transfer and only b's 10 bytes used", in)
|
||||
}
|
||||
b.h.Idle = func(string, time.Duration) error { return errors.New("bus down") }
|
||||
if _, err := b.h.Publish(NpmSeat, nodeBuild, pub("d", "0.1.0", file)); err == nil || !strings.Contains(err.Error(), "idle bound") {
|
||||
t.Errorf("a publish whose idle key is not written = %v; want it failed loudly", err)
|
||||
}
|
||||
})
|
||||
t.Run("refusals", func(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
|
||||
@@ -17,9 +17,14 @@
|
||||
{
|
||||
"name": "transfers",
|
||||
"storage": "key-value",
|
||||
"template": "state"
|
||||
},
|
||||
{
|
||||
"name": "idle",
|
||||
"storage": "key-value",
|
||||
"template": "state",
|
||||
"options": {
|
||||
"ttl-seconds": 86400
|
||||
"key-ttl": true
|
||||
}
|
||||
},
|
||||
{
|
||||
|
||||
@@ -20,8 +20,8 @@ import (
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep/seatgen"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
)
|
||||
|
||||
// GeneratedRootGo and GeneratedRootTS are the generated roots of a contracts module, one per build module.
|
||||
|
||||
@@ -10,8 +10,8 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep/seatgen"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// contracts writes a contracts module of one seat, its generated files as the generator writes them, and answers its
|
||||
|
||||
@@ -1 +1,3 @@
|
||||
nfs-server
|
||||
# the binary, built at the module or in its command directory; never the command directory itself
|
||||
/nfs-server
|
||||
/cmd/nfs-server/nfs-server
|
||||
|
||||
@@ -1 +1,2 @@
|
||||
restic-backups
|
||||
# the binary, built in this command directory
|
||||
/restic-backups
|
||||
|
||||
Reference in New Issue
Block a user