novox/mesh-catalog
18
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

F-catalog-B: distribution's front takes no bootstrap credential

Browse Source 
The lead's ENG-5 ruling: on a fresh mesh the bootstrap pushes the holders' own images to the store on its
loopback with no credential, so the front's bootstrap token, its file, the bootstrap-push window and the
program's removal of the file go. Fewer secrets; nothing to refuse once a window closes.

Claude-Session: https://claude.ai/code/session_01JG73KJNrH5EgFXMKKYSrig
This commit is contained in:
jschoubben
2026-11-04 17:05:24 +01:00
parent 1bdca045c2
commit 1b485eb2aa
6 changed files with 12 additions and 159 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/distribution/cmd/distribution-tools/main.go
+1 -31
View File
@@ -80,27 +80,6 @@ func finding(kind, subject, words string, raised bool) {
}
}
// closeBootstrap asks the program to discard the bootstrap's one-off credential once a publish is answered (design 51
// *On a fresh mesh*): the window bootstrap-push closed in the state windows, which the program follows and then removes
// the file. The tools bundle runs in the tool runner's account, which cannot remove a file in distribution's state
// directory. Once per process; the window's record says it to the next.
var closeBootstrap func()
func bootstrapCloser(windows kept[holder.Window]) func() {
return sync.OnceFunc(func() {
cur, _, err := windows.Get(holder.BootstrapWindow)
if err == nil && !cur.Closed.IsZero() {
return
}
w := holder.Window{Name: holder.BootstrapWindow, Closed: time.Now().UTC()}
if err := windows.Put(w); err != nil {
logf("the bootstrap's one-off credential could not be asked away: %v", err)
return
}
logf("the bootstrap's one-off credential is asked away: the holders serve publish")
})
}
// learned are the proved callers whose build this process recorded as publishing through the seats.
var learned sync.Map
@@ -268,9 +247,6 @@ func tools(v *holder.Verbs, windows kept[holder.Window], callers func() (holder.
}
_, _ = v.DropIdle()
a, err := v.Publish(seat, stdio.CallerOf(args), publishOf(in))
if err == nil && closeBootstrap != nil {
closeBootstrap()
}
if err == nil {
learnPublishing(windows, stdio.CallerOf(args))
}
@@ -465,12 +441,7 @@ func tools(v *holder.Verbs, windows kept[holder.Window], callers func() (holder.
return nil, err
}
}
b, _, err := windows.Get(holder.BootstrapWindow)
if err != nil {
return nil, err
}
// The program removes the file when this window closes and writes it closed again with its removal's time.
return map[string]any{"window": cur, "bootstrap_credential_discarded": !b.Closed.IsZero(), "bootstrap": b}, nil
return map[string]any{"window": cur}, nil
}})
}
return out
@@ -484,7 +455,6 @@ func main() {
v := &holder.Verbs{Holder: h, Catalog: store, Holds: kept[holder.Hold]{state: "holds", key: holder.Hold.Key}, Mirrors: mirrors{}}
go followMirrors(v)
windows := kept[holder.Window]{state: "windows", key: func(w holder.Window) string { return w.Name }}
closeBootstrap = bootstrapCloser(windows)
if err := stdio.Serve("", tools(v, windows, holder.ConfiguredCallers)); err != nil {
logf("%v", err)
os.Exit(1)
modules/distribution/cmd/distribution/main.go
+1 -46
View File
@@ -1,7 +1,7 @@
// Command distribution is distribution's program (novox/hq design 51): it serves the artifact store's port in front of
// the registry container, which listens on its node's loopback only. Reads (GET, HEAD) go through to every node's
// container runtime; a write is refused but while the direct-write window is open, never over a tag the seat holds
// published, or with the bootstrap's one-off credential (holder.Front). The windows are distribution's state windows:
// published (holder.Front). The windows are distribution's state windows:
// the program moves them (windows.go) and follows the direct-write window's record on each change; it holds no timer.
package main
@@ -10,7 +10,6 @@ import (
"encoding/json"
"errors"
"fmt"
"io/fs"
"log"
"net/http"
"net/url"
@@ -58,7 +57,6 @@ func run(ctx context.Context, getenv func(string) string) error {
windows := process.OptionalBucket(ctx, js, "distribution_windows", "the direct-write window is read as open until the program starts again", logger.Printf)
if windows != nil {
go follow(ctx, windows, &open, logger)
go followBootstrap(ctx, windows, logger)
calls, err := process.Connect(cred)
if err != nil {
return err
@@ -72,13 +70,6 @@ func run(ctx context.Context, getenv func(string) string) error {
Published: func(name, tag string) (bool, error) {
d, err := oci.Digest(name, tag)
return d != "", err
},
Bootstrap: func() (string, bool) {
b, err := os.ReadFile(holder.BootstrapFile)
if err != nil {
return "", false
}
return strings.TrimSpace(string(b)), true
}}
srv := &http.Server{Addr: or(getenv("MESH_REGISTRY_LISTEN"), ":5000"), Handler: front, ReadHeaderTimeout: 30 * time.Second}
errs := make(chan error, 1)
@@ -119,42 +110,6 @@ func follow(ctx context.Context, kv jetstream.KeyValue, open *atomic.Bool, logge
}
}
// followBootstrap removes the bootstrap's one-off credential (holder.BootstrapFile) when the tools bundle closes the
// window bootstrap-push at its first answered publish (design 51 *On a fresh mesh*): the program runs as distribution's
// account, which owns the file. A file already gone is no fault; one that cannot be removed is said each time the
// window is written.
func followBootstrap(ctx context.Context, kv jetstream.KeyValue, logger *log.Logger) {
w, err := kv.Watch(ctx, holder.BootstrapWindow)
if err != nil {
logger.Printf("the bootstrap's credential window cannot be watched (%v): its file stays until the program starts again", err)
return
}
for e := range w.Updates() {
if e == nil || e.Operation() != jetstream.KeyValuePut {
continue
}
var win holder.Window
if json.Unmarshal(e.Value(), &win) != nil || win.Closed.IsZero() {
continue
}
if why := removeBootstrap(holder.BootstrapFile); why != "" {
logger.Printf("%s", why)
}
}
}
// removeBootstrap removes the credential's file; it answers what went wrong, or "" when the file is gone.
func removeBootstrap(path string) string {
switch err := os.Remove(path); {
case err == nil:
return "the bootstrap's one-off credential is discarded: the holders serve publish"
case errors.Is(err, fs.ErrNotExist):
return ""
default:
return fmt.Sprintf("the bootstrap's one-off credential could not be discarded, so it is still accepted: %v", err)
}
}
func or(v, d string) string {
if strings.TrimSpace(v) == "" {
return d
modules/distribution/cmd/distribution/main_test.go
-35
View File
@@ -1,35 +0,0 @@
package main
import (
"os"
"path/filepath"
"strings"
"testing"
)
// The program, which runs as distribution's account, removes the bootstrap's one-off credential when its window
// closes (design 51 *On a fresh mesh*; a3-publish-contract): removed once, a file already gone is no fault, and one
// that cannot be removed is said, since the front still accepts it.
func TestTheProgramDiscardsTheBootstrapCredential(t *testing.T) {
dir := t.TempDir()
file := filepath.Join(dir, "bootstrap-push")
if err := os.WriteFile(file, []byte("token\n"), 0o600); err != nil {
t.Fatal(err)
}
if why := removeBootstrap(file); !strings.Contains(why, "discarded") {
t.Fatalf("first removal = %q, want it said discarded", why)
}
if _, err := os.Stat(file); !os.IsNotExist(err) {
t.Fatalf("the file is still there: %v", err)
}
if why := removeBootstrap(file); why != "" {
t.Fatalf("a file already gone = %q, want nothing said", why)
}
stuck := filepath.Join(dir, "stuck")
if err := os.MkdirAll(filepath.Join(stuck, "child"), 0o700); err != nil {
t.Fatal(err)
}
if why := removeBootstrap(stuck); !strings.Contains(why, "still accepted") {
t.Fatalf("an unremovable file = %q, want it said still accepted", why)
}
}
modules/distribution/internal/holder/front.go
+4 -23
View File
@@ -1,7 +1,6 @@
package holder
import (
"crypto/subtle"
"fmt"
"net/http"
"net/http/httputil"
@@ -15,7 +14,8 @@ import (
// protocol's GET and HEAD) stay open to every node's container runtime; a write reaches the store only from
// distribution's own process, which writes to the loopback directly and never through the front. While the direct-write
// window is open, a build module's old build still pushes through the front as today, but never to a tag the seat holds
// published.
// published. No credential opens it: on a fresh mesh the bootstrap pushes the holders' own images to the store on its
// loopback, never through the front (design 51 *On a fresh mesh*).
type Front struct {
// Store is the registry container's loopback address.
Store *url.URL
@@ -23,9 +23,6 @@ type Front struct {
Window *atomic.Bool
// Published says whether name holds a tag published through the seat.
Published func(name, tag string) (bool, error)
// Bootstrap is the bootstrap's one-off credential for the holders' first images (design 51 *On a fresh mesh*):
// read on each write, never kept, so the file's removal (BootstrapFile) refuses the next write that carries it.
Bootstrap func() (token string, ok bool)
Logf func(string, ...any)
proxy *httputil.ReverseProxy
}
@@ -40,14 +37,7 @@ func (f *Front) ServeHTTP(w http.ResponseWriter, r *http.Request) {
f.proxy.ServeHTTP(w, r)
return
}
bootstrap := false
if f.Bootstrap != nil {
if token, ok := f.Bootstrap(); ok && token != "" {
got, has := strings.CutPrefix(r.Header.Get("Authorization"), "Bearer ")
bootstrap = has && subtle.ConstantTimeCompare([]byte(got), []byte(token)) == 1
}
}
if !bootstrap && (f.Window == nil || !f.Window.Load()) {
if f.Window == nil || !f.Window.Load() {
refuse(w, http.StatusForbidden, "DENIED", "the store takes writes from distribution's own process alone: publish through the seat's publish")
return
}
@@ -62,21 +52,12 @@ func (f *Front) ServeHTTP(w http.ResponseWriter, r *http.Request) {
return
}
}
if bootstrap {
if f.Logf != nil {
f.Logf("a write %s %s from %s with the bootstrap's one-off credential", r.Method, r.URL.Path, r.RemoteAddr)
}
} else if f.Logf != nil {
if f.Logf != nil {
f.Logf("a direct write %s %s from %s, accepted while the direct-write window is open", r.Method, r.URL.Path, r.RemoteAddr)
}
f.proxy.ServeHTTP(w, r)
}
// BootstrapFile is where the node-engine's bootstrap gives distribution its one-off credential, in distribution's
// state directory (0600, distribution's account): one line, the token. The tools bundle removes it when it answers its
// first publish, so the holders serve publish and the credential is gone (RemoveBootstrap).
const BootstrapFile = "/var/lib/mesh/registry/bootstrap-push"
// manifestPut is a PUT of /v2/<name>/manifests/<tag>, a tag and not a digest.
func manifestPut(r *http.Request) (name, tag string, ok bool) {
if r.Method != http.MethodPut {
modules/distribution/internal/holder/holder_test.go
+6 -20
View File
@@ -469,9 +469,7 @@ func TestTheHolderOfTheTwoImageSeats(t *testing.T) {
defer store.Close()
u, _ := url.Parse(store.URL)
var open atomic.Bool
token, kept := "s3cret", true
f := &Front{Store: u, Window: &open, Published: func(name, tag string) (bool, error) { return name == "m/i" && tag == "0.1.3", nil },
Bootstrap: func() (string, bool) { return token, kept }}
f := &Front{Store: u, Window: &open, Published: func(name, tag string) (bool, error) { return name == "m/i" && tag == "0.1.3", nil }}
auth := ""
do := func(method, path string) int {
rec := httptest.NewRecorder()
@@ -500,24 +498,12 @@ func TestTheHolderOfTheTwoImageSeats(t *testing.T) {
t.Errorf("window open %v: %s %s = %d, want %d", c.open, c.method, c.path, got, c.want)
}
}
// The bootstrap's one-off credential writes with the window closed, never over a published tag, and not once
// its file is gone; a wrong token writes nothing.
// No credential opens the front (the lead's ENG-5 ruling): a write carrying a token, with the window closed, is
// refused as any other.
open.Store(false)
for _, c := range []struct {
auth string
kept bool
path string
want int
}{
{"s3cret", true, "/v2/gitea/program/manifests/0.1.0", http.StatusCreated},
{"s3cret", true, "/v2/m/i/manifests/0.1.3", http.StatusForbidden},
{"wrong", true, "/v2/gitea/program/manifests/0.1.0", http.StatusForbidden},
{"s3cret", false, "/v2/gitea/program/manifests/0.1.0", http.StatusForbidden},
} {
auth, kept = c.auth, c.kept
if got := do(http.MethodPut, c.path); got != c.want {
t.Errorf("bootstrap %q kept %v: PUT %s = %d, want %d", c.auth, c.kept, c.path, got, c.want)
}
auth = "s3cret"
if got := do(http.MethodPut, "/v2/gitea/program/manifests/0.1.0"); got != http.StatusForbidden {
t.Errorf("a write with a bearer token and the window closed = %d, want %d", got, http.StatusForbidden)
}
})
t.Run("mirror copies an outside image by its pinned digest, every platform, only the parts the store lacks", func(t *testing.T) {
modules/distribution/internal/holder/window.go
-4
View File
@@ -16,10 +16,6 @@ import (
const (
OldNameWindow = "old-name"
DirectWritesWindow = "direct-writes"
// BootstrapWindow is the bootstrap's one-off credential (BootstrapFile): open while the file is there. The tools
// bundle closes it at its first answered publish; the program, which runs as distribution's account and alone may
// remove the file in distribution's state directory, removes it on that close.
BootstrapWindow = "bootstrap-push"
// WindowFindingAfter is how long a node holds a window open before it is distribution's finding naming the node.
WindowFindingAfter = 7 * 24 * time.Hour
// The events a window's close says on distribution's own family distribution.events (7 days, 16 MiB, drop