From d3ee49d6f37fbc8b743d194a7179afb28e56e2ad Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 4 Nov 2026 16:29:56 +0100 Subject: [PATCH] F-host 2, ENG-5: on a fresh mesh the bootstrap pushes the holders' own images to the store on its loopback with no credential, and names them under the seat's reach (lead's design limit, review of issue 1966) The store's front is distribution's program, a bundle the mesh builds, so nothing serves it before the builder: the bootstrap's /v2/ wait and its pushes now go to the loopback port the store's manifest publishes, as the publish contract allows on the store's own node. The one-off credential and its file go: the runtime could not carry it on a push, and no push needs it. The tags list is asked for the repository being published, not always the control plane's. Claude-Session: https://claude.ai/code/session_01JG73KJNrH5EgFXMKKYSrig --- cmd/mesh-bootstrap/main.go | 4 +- internal/bootstrap/bootstrap.go | 27 +----- internal/bootstrap/publish.go | 37 +++++---- internal/bootstrap/publish_test.go | 3 +- internal/bootstrap/push_credential.go | 114 -------------------------- internal/bootstrap/registry.go | 49 ++++++++++- internal/bootstrap/trust_test.go | 68 ++++++--------- 7 files changed, 102 insertions(+), 200 deletions(-) delete mode 100644 internal/bootstrap/push_credential.go diff --git a/cmd/mesh-bootstrap/main.go b/cmd/mesh-bootstrap/main.go index 46ccad0..369a744 100644 --- a/cmd/mesh-bootstrap/main.go +++ b/cmd/mesh-bootstrap/main.go @@ -188,9 +188,7 @@ func parseArgs(args []string) (string, bootstrap.Options, bool, error) { Out: defaultOut, State: store.DefaultPath, Registry: defaultRegistry, - // Where the store's holder reads the one-off credential the holders' first images are pushed with (design 51, ENG-5). - PushCredentialFile: "/var/lib/mesh/registry/bootstrap-push", - Host: defaultHost, + Host: defaultHost, // The machine's own name, because that is what a person already calls it and an installer // inventing a different one would leave the mesh naming a machine nobody recognises. It is // read here rather than inside the bootstrap so that --node overrides a fact rather than a diff --git a/internal/bootstrap/bootstrap.go b/internal/bootstrap/bootstrap.go index b95905c..5835bc5 100644 --- a/internal/bootstrap/bootstrap.go +++ b/internal/bootstrap/bootstrap.go @@ -157,11 +157,6 @@ type Options struct { // an address the others can reach. Registry string - // PushCredentialFile is where the bootstrap writes its one-off push credential for the store's holder to find, the - // holder's state file `bootstrap-push` on the node that holds it (design 51 *The move*, issue 1966 ENG-5); empty writes - // none. The file goes to the account that owns that directory in the holder's manifest, once the node-engine made both. - PushCredentialFile string - // Host is the `mesh-host` binary on this machine — the program that enrols and then holds the // machine to what the mesh says. The installer runs it; it does not contain it. Host string @@ -301,12 +296,10 @@ type Result struct { Agent string `json:"host-agent,omitempty"` // Registry is the mesh's own artifact store, once it answers. - Registry string `json:"registry,omitempty"` - // PushCredentialFile is where this run left its one-off push credential (never the credential itself). - PushCredentialFile string `json:"push-credential-file,omitempty"` - RegistryReplied int `json:"registry-replied,omitempty"` - RegistryKnown bool `json:"registry-already-registered,omitempty"` - RegistryRunning string `json:"registry-container,omitempty"` + Registry string `json:"registry,omitempty"` + RegistryReplied int `json:"registry-replied,omitempty"` + RegistryKnown bool `json:"registry-already-registered,omitempty"` + RegistryRunning string `json:"registry-container,omitempty"` // RegistryTrusted are the names the runtime was told to trust the registry by, before the push. RegistryTrusted []string `json:"registry-trusted,omitempty"` PublishedAs string `json:"control-plane-image,omitempty"` @@ -822,18 +815,6 @@ func RegistryAndPublish(ctx context.Context, o Options, d Deps, sys system.Syste if err != nil { return Published{}, failed(StepRegistry, err) } - // The one-off credential the holders' first images are pushed to the store's front with, written for its holder to find - // and forgotten when this run ends; the holder discards the file once it answers its first publish (ENG-5). - if o.PushCredentialFile != "" { - cred, err := WritePushCredential(o.Catalogue, o.PushCredentialFile) - if err != nil { - return Published{}, failed(StepRegistry, err) - } - defer cred.Forget() - result.PushCredentialFile = cred.File - say(" a one-off credential for the holders' first images is at " + cred.File + "; the store's holder discards it once it serves publish") - } - // ---- 8. publish ----------------------------------------------------------------------- say("publish — the control plane's image gets its first manifest digest") // **The image this mesh built, not the one the installer carried.** The carried one is the diff --git a/internal/bootstrap/publish.go b/internal/bootstrap/publish.go index 561ce92..97dbd13 100644 --- a/internal/bootstrap/publish.go +++ b/internal/bootstrap/publish.go @@ -69,17 +69,24 @@ func PublishControlPlane(ctx context.Context, o Options, d Deps, imageID string, func publishAs(ctx context.Context, o Options, d Deps, imageID, repository string, say func(string)) (Published, error) { - remote := o.Registry + "/" + repository + // Pushed to the store's own loopback with no credential (StoreLoopback); recorded under the seat's reach, the + // address every node pulls by, with the digest the store gave. + loopback, err := StoreLoopback(o.Catalogue) + if err != nil { + return Published{}, err + } + remote := loopback + "/" + repository out := Published{Tagged: remote + ":" + genesisTag} + reach := func(digest string) string { return o.Registry + "/" + repository + strings.TrimPrefix(digest, remote) } // Asked first. A digest already served is a fact about the registry, and re-pushing an image // the registry already holds is asking it to store what it already has under the name it // already has. - if held, err := digestOf(ctx, o, d, remote); err != nil { + if held, err := digestOf(ctx, o, d, loopback, repository); err != nil { return out, err } else if held != "" { - out.Reference, out.Already = held, true - say(" already published " + held) + out.Reference, out.Already = reach(held), true + say(" already published " + out.Reference) return out, nil } @@ -93,12 +100,12 @@ func publishAs(ctx context.Context, o Options, d Deps, imageID, repository strin "over the mesh's own network, which is already the encrypted and authenticated "+ "thing. A runtime refusing it for being insecure is refusing a registry on %s, "+ "which it does not do for a loopback address", - out.Tagged, err, o.Registry) + out.Tagged, err, loopback) } // Read back, from the registry's own answer rather than computed here. What matters is what // the registry will serve for that reference, and only it can say (novox/hq ADR 0018). - pinned, err := digestOf(ctx, o, d, remote) + pinned, err := digestOf(ctx, o, d, loopback, repository) if err != nil { return out, err } @@ -107,10 +114,10 @@ func publishAs(ctx context.Context, o Options, d Deps, imageID, repository strin "%s was pushed and the registry does not serve it.\n"+ "The next step names this module by the digest this was supposed to produce, so "+ "there is nothing to name. Check `docker push` and "+ - "http://%s/v2/%s/tags/list", out.Tagged, o.Registry, repository) + "http://%s/v2/%s/tags/list", out.Tagged, loopback, repository) } - out.Reference = pinned - say(" published " + pinned) + out.Reference = reach(pinned) + say(" published " + out.Reference) return out, nil } @@ -121,13 +128,13 @@ func publishAs(ctx context.Context, o Options, d Deps, imageID, repository strin // whether the registry still has it — a registry whose volume was recreated would leave the // runtime remembering a digest nothing serves, and the module registered against it would pin the // mesh to an image that cannot be pulled. -func digestOf(ctx context.Context, o Options, d Deps, remote string) (string, error) { +func digestOf(ctx context.Context, o Options, d Deps, store, repository string) (string, error) { + remote := store + "/" + repository asking, cancel := context.WithTimeout(ctx, o.Timeout) - status, body, err := d.Fetch(asking, - "http://"+o.Registry+"/v2/"+ControlPlaneRepository+"/tags/list") + status, body, err := d.Fetch(asking, "http://"+store+"/v2/"+repository+"/tags/list") cancel() if err != nil { - return "", fmt.Errorf("cannot ask the registry at %s what it holds: %w", o.Registry, err) + return "", fmt.Errorf("cannot ask the registry at %s what it holds: %w", store, err) } if status == http.StatusNotFound { // Nothing has ever been pushed under this name. An answer, not a failure. @@ -135,14 +142,14 @@ func digestOf(ctx context.Context, o Options, d Deps, remote string) (string, er } if status != http.StatusOK { return "", fmt.Errorf("the registry answered %d when asked what it holds for %s", - status, ControlPlaneRepository) + status, repository) } var listed struct { Tags []string `json:"tags"` } if err := json.Unmarshal([]byte(body), &listed); err != nil { return "", fmt.Errorf("the registry's answer about %s is not readable: %w", - ControlPlaneRepository, err) + repository, err) } if !contains(listed.Tags, genesisTag) { return "", nil diff --git a/internal/bootstrap/publish_test.go b/internal/bootstrap/publish_test.go index adeaf95..6120136 100644 --- a/internal/bootstrap/publish_test.go +++ b/internal/bootstrap/publish_test.go @@ -22,7 +22,8 @@ func publishing(t *testing.T, fetch func(string) (int, string, error), // Each value named before it is returned: a return of several composite literals spanning lines is laid // out differently by the gofmt of one Go release and the next, and the build seat's is the one that // judges (novox/hq issue 286). - options := Options{Registry: "127.0.0.1:5000", Timeout: time.Second, Wait: 0} + options := Options{Registry: "127.0.0.1:5000", Timeout: time.Second, Wait: 0, + Catalogue: catalogueWith(t, RegistryModule, upstreamRegistryManifest)} deps := Deps{ Run: runtime.run, Fetch: func(_ context.Context, url string) (int, string, error) { diff --git a/internal/bootstrap/push_credential.go b/internal/bootstrap/push_credential.go deleted file mode 100644 index 9134836..0000000 --- a/internal/bootstrap/push_credential.go +++ /dev/null @@ -1,114 +0,0 @@ -package bootstrap - -import ( - "crypto/rand" - "encoding/hex" - "encoding/json" - "fmt" - "os" - "os/user" - "path/filepath" - "strconv" - "syscall" -) - -// PushCredential is the bootstrap's one-off credential for the holders' first images (design 51 *The move*; issue 1966 ENG-5): -// a bearer token of 32 random bytes, hex, on one line, made for one fresh-mesh run. No vault and no bus holds it: this run -// holds it and writes it once, to the file the store's holder reads on each write (`Authorization: Bearer ` on an OCI -// write to the store's port, from any node); the holder removes the file when it answers its first `publish`, and this run -// forgets the token when it ends. On the holder's own node the bootstrap may push to the registry container's loopback with -// no credential at all. -type PushCredential struct { - // File is where the token was written. - File string - token string -} - -// Token is the bearer token, for a push through the store's front from another node. -func (c *PushCredential) Token() string { return c.token } - -// Forget drops the token from this run's memory. The file is the holder's to remove. -func (c *PushCredential) Forget() { c.token = "" } - -// lookupAccount gives an account's user id and group id. A variable so a test can name an account this machine has. -var lookupAccount = func(name string) (uid, gid int, err error) { - u, err := user.Lookup(name) - if err != nil { - return 0, 0, err - } - if uid, err = strconv.Atoi(u.Uid); err != nil { - return 0, 0, err - } - gid, err = strconv.Atoi(u.Gid) - return uid, gid, err -} - -// WritePushCredential writes the one-off credential for the store's holder, read from its manifest in catalogue: the file -// goes into the holder's directory resource that holds file's path, given to the account that directory names as owner, -// and only once the node-engine has made that account and that directory, owned by it. Until then the holder's program, -// which runs as that account, could not read it, so it is refused rather than written as root's. -func WritePushCredential(catalogue, file string) (*PushCredential, error) { - manifest, err := readManifest(catalogue, RegistryModule) - if err != nil { - return nil, err - } - var m struct { - Resources []struct { - Type string `json:"type"` - Name string `json:"name"` - Path string `json:"path"` - Owner string `json:"owner"` - } `json:"resources"` - } - if err := json.Unmarshal(manifest, &m); err != nil { - return nil, fmt.Errorf("the %s manifest is not readable as JSON: %w", RegistryModule, err) - } - dir, owner, account := filepath.Dir(file), "", false - for _, r := range m.Resources { - if r.Type == "directory" && filepath.Clean(r.Path) == dir { - owner = r.Owner - } - } - for _, r := range m.Resources { - account = account || (owner != "" && r.Type == "user" && r.Name == owner) - } - if !account { - return nil, fmt.Errorf("the push credential %s has no account to go to: the %s manifest declares no directory %s "+ - "owned by an account it declares, so its program could not read the file", file, RegistryModule, dir) - } - uid, gid, err := lookupAccount(owner) - if err != nil { - return nil, fmt.Errorf("the push credential waits on the account %s, which the node-engine has not made: %w", owner, err) - } - st, err := os.Stat(dir) - if err != nil { - return nil, fmt.Errorf("the push credential waits on %s's state directory %s, which the node-engine has not made: %w", owner, dir, err) - } - if sys, ok := st.Sys().(*syscall.Stat_t); !ok || int(sys.Uid) != uid { - return nil, fmt.Errorf("the push credential waits on %s owning its state directory %s, which the node-engine has not yet given it", owner, dir) - } - return MintPushCredential(file, uid, gid) -} - -// MintPushCredential makes a token and writes it to file, mode 0600, owned by the user id and group id given, into a -// directory that must be there. A file that is there is replaced: a credential left by an earlier run is never reused. -func MintPushCredential(file string, uid, gid int) (*PushCredential, error) { - raw := make([]byte, 32) - if _, err := rand.Read(raw); err != nil { - return nil, fmt.Errorf("a push credential cannot be made: %w", err) - } - token := hex.EncodeToString(raw) - tmp := file + ".new" - if err := os.WriteFile(tmp, []byte(token+"\n"), 0o600); err != nil { - return nil, fmt.Errorf("the push credential cannot be written to %s: %w", file, err) - } - if err := os.Chown(tmp, uid, gid); err != nil { - os.Remove(tmp) - return nil, fmt.Errorf("the push credential cannot be given to the store's holder: %w", err) - } - if err := os.Rename(tmp, file); err != nil { - os.Remove(tmp) - return nil, fmt.Errorf("the push credential cannot be put at %s: %w", file, err) - } - return &PushCredential{File: file, token: token}, nil -} diff --git a/internal/bootstrap/registry.go b/internal/bootstrap/registry.go index 359007f..966d612 100644 --- a/internal/bootstrap/registry.go +++ b/internal/bootstrap/registry.go @@ -95,9 +95,14 @@ func InstallRegistry(ctx context.Context, o Options, d Deps, control controlPlan return out, nil } -// waitForTheRegistry asks `/v2/` until it answers. +// waitForTheRegistry asks the store's own `/v2/`, on its loopback (StoreLoopback), until it answers: the pushes that +// follow go there, and the store's front is not served yet on a fresh mesh. func waitForTheRegistry(ctx context.Context, d Deps, o Options, say func(string)) (int, error) { - where := "http://" + o.Registry + "/v2/" + loopback, err := StoreLoopback(o.Catalogue) + if err != nil { + return 0, err + } + where := "http://" + loopback + "/v2/" deadline := time.Now().Add(o.Wait) var last string @@ -207,3 +212,43 @@ func containerIn(manifest []byte, id string) (name, image string, err error) { "the %s module declares no container %q, so the installer does not know what to wait for. "+ "It declares: %s", m.Module, id, strings.Join(containers, ", ")) } + +// storeContainerPort is the port the registry image listens on inside its container (Distribution's own default). +const storeContainerPort = "5000" + +// StoreLoopback is host:port where this machine reaches the store's container itself, read from the port its manifest +// publishes for it: on a fresh mesh the bootstrap pushes the holders' own images there, with no credential, because the +// store's front (distribution's program, a bundle the mesh builds) is not served until the builder has built it (the +// publish contract allows a push on the store's own node to its loopback). A port published on every address is reached +// on 127.0.0.1. What the bootstrap records is named under the seat's reach (Options.Registry), never this address. +func StoreLoopback(catalogue string) (string, error) { + manifest, err := readManifest(catalogue, RegistryModule) + if err != nil { + return "", err + } + var m struct { + Resources []struct { + ID string `json:"id"` + Ports []string `json:"ports"` + } `json:"resources"` + } + if err := json.Unmarshal(manifest, &m); err != nil { + return "", fmt.Errorf("the %s manifest is not readable as JSON: %w", RegistryModule, err) + } + for _, r := range m.Resources { + if r.ID != registryResource { + continue + } + for _, p := range r.Ports { + parts := strings.Split(strings.SplitN(p, "/", 2)[0], ":") + switch { + case len(parts) == 3 && parts[2] == storeContainerPort: + return parts[0] + ":" + parts[1], nil + case len(parts) == 2 && parts[1] == storeContainerPort: + return "127.0.0.1:" + parts[0], nil + } + } + } + return "", fmt.Errorf("the %s module's container %q publishes no port for the store's %s, so the bootstrap cannot "+ + "reach the store to push to it", RegistryModule, registryResource, storeContainerPort) +} diff --git a/internal/bootstrap/trust_test.go b/internal/bootstrap/trust_test.go index 6fd6533..9ec16d4 100644 --- a/internal/bootstrap/trust_test.go +++ b/internal/bootstrap/trust_test.go @@ -7,9 +7,7 @@ import ( "net/http" "os" "path/filepath" - "regexp" "strings" - "syscall" "testing" "time" @@ -228,29 +226,15 @@ func TestARangeTheRuntimeTrustsCoversAnAddressInIt(t *testing.T) { // moving the trust after the push, or dropping it, fails this test. func TestNothingIsPushedToTheRegistryBeforeTheRuntimeTrustsIt(t *testing.T) { o, runtime, sys := trustSetup(t, `{"data-root":"/srv/docker"}`) - // The holder's state directory and its account, as distribution's manifest declares them (ENG-5). - stateDir := filepath.Join(t.TempDir(), "registry") - o.Catalogue = catalogueWith(t, RegistryModule, strings.Replace(upstreamRegistryManifest, - `{"id": "state", "type": "directory", "path": "/var/lib/mesh/registry", "mode": "0700"}`, - `{"id": "account", "type": "user", "name": "distribution"}, - {"id": "state", "type": "directory", "path": "`+stateDir+`", "mode": "0700", "owner": "distribution"}`, 1)) + // The store as distribution's manifest declares it: its container on this machine's loopback, its front not yet served, + // and its state directory, where no credential is ever written (design 51 *On a fresh mesh*, a design limit of issue 1966). + stateDir := t.TempDir() + o.Catalogue = catalogueWith(t, RegistryModule, strings.NewReplacer( + `"path": "/var/lib/mesh/registry"`, `"path": "`+stateDir+`"`, + `"ports": ["5000:5000"]`, `"ports": ["127.0.0.1:5001:5000"]`).Replace(upstreamRegistryManifest)) o.Registry = "192.0.2.10:5100" - asked, was := []string{}, lookupAccount - lookupAccount = func(name string) (int, int, error) { - asked = append(asked, name) - return os.Getuid(), os.Getgid(), nil - } - t.Cleanup(func() { lookupAccount = was }) - o.PushCredentialFile = filepath.Join(stateDir, "bootstrap-push") - // Before the node-engine made distribution's state directory, nothing is written: the program could not read it. - if _, err := WritePushCredential(o.Catalogue, o.PushCredentialFile); err == nil { - t.Errorf("the push credential was written before its holder's state directory existed") - } else if _, err := os.Stat(stateDir); err == nil { - t.Errorf("the bootstrap made the holder's state directory itself") - } - if err := os.Mkdir(stateDir, 0o700); err != nil { - t.Fatal(err) - } + const loopback = "127.0.0.1:5001" + var fetched []string pushed := false mesh := aMeshThatAgrees(nil) @@ -264,7 +248,8 @@ func TestNothingIsPushedToTheRegistryBeforeTheRuntimeTrustsIt(t *testing.T) { case "tag": return "", nil case "push": - trusted := false + // The runtime trusts a loopback registry with no configuration; any other only once told. + trusted := strings.HasPrefix(args[1], "127.0.0.1:") for _, n := range runtime.trusted { trusted = trusted || strings.HasPrefix(args[1], n+"/") } @@ -274,20 +259,19 @@ func TestNothingIsPushedToTheRegistryBeforeTheRuntimeTrustsIt(t *testing.T) { pushed = true return "", nil case "image": - return `["` + o.Registry + `/mesh-controller@sha256:` + strings.Repeat("a", 64) + `"]`, nil + return `["` + loopback + `/mesh-controller@sha256:` + strings.Repeat("a", 64) + `"]`, nil } } return mesh(name, args) } d := Deps{Run: run, Fetch: func(_ context.Context, url string) (int, string, error) { + fetched = append(fetched, url) if strings.HasSuffix(url, "/v2/") || pushed { return http.StatusOK, `{"name":"mesh-controller","tags":["genesis"]}`, nil } return http.StatusNotFound, "", nil }} var result Result - // And the one-off credential for the holders' first images (ENG-5): written once, mode 0600, one line of 64 hex digits, - // named in the result and never put in it. published, err := RegistryAndPublish(context.Background(), o, d, sys, controlPlane{container: "temp-mesh-controller", run: run, timeout: time.Second}, held, &result, func(string) {}) if err != nil { @@ -312,22 +296,22 @@ func TestNothingIsPushedToTheRegistryBeforeTheRuntimeTrustsIt(t *testing.T) { t.Errorf("the trust was not written (%d) and confirmed (%d) before the first tag or push (%d): %v", reloaded, confirmed, firstPush, runtime.commands) } - if raw, err := os.ReadFile(o.PushCredentialFile); err != nil || !regexp.MustCompile(`^[0-9a-f]{64}\n$`).Match(raw) { - t.Errorf("the push credential is %q (%v)", raw, err) - } else if st, _ := os.Stat(o.PushCredentialFile); st.Mode().Perm() != 0o600 || - int(st.Sys().(*syscall.Stat_t).Uid) != os.Getuid() || int(st.Sys().(*syscall.Stat_t).Gid) != os.Getgid() { - t.Errorf("the push credential is mode %v, owner %d:%d", st.Mode().Perm(), st.Sys().(*syscall.Stat_t).Uid, st.Sys().(*syscall.Stat_t).Gid) + // The push went to the store on its loopback, and nothing the run asked or pushed reached the seat's reach; the + // reference is named under the reach all the same, and no credential file was written for the store's holder. + for _, c := range runtime.commands { + if strings.HasPrefix(c, "docker tag") || strings.HasPrefix(c, "docker push") { + if !strings.Contains(c, " "+loopback+"/mesh-controller:") { + t.Errorf("a push not to the loopback store: %q", c) + } + } } - if fmt.Sprint(asked) != "[distribution distribution]" { - t.Errorf("the push credential's owner was asked as %v, not the account distribution's state directory names", asked) + for _, u := range fetched { + if !strings.HasPrefix(u, "http://"+loopback+"/v2/") { + t.Errorf("the run asked %s, not the loopback store", u) + } } - if result.PushCredentialFile != o.PushCredentialFile { - t.Errorf("the result names %q", result.PushCredentialFile) - } - if again, err := MintPushCredential(o.PushCredentialFile, os.Getuid(), os.Getgid()); err != nil || again.Token() == "" { - t.Errorf("a credential of an earlier run was not replaced: %v", err) - } else if again.Forget(); again.Token() != "" { - t.Errorf("a forgotten credential still holds its token") + if entries, err := os.ReadDir(stateDir); err != nil || len(entries) != 0 { + t.Errorf("the run wrote into the store's state directory: %v (%v)", entries, err) } if fmt.Sprint(result.RegistryTrusted) != "[192.0.2.10:5100 anchor.internal:5100]" { t.Errorf("the result says it trusted %v", result.RegistryTrusted)