diff --git a/cmd/mesh-bootstrap/main.go b/cmd/mesh-bootstrap/main.go index 46ccad0..369a744 100644 --- a/cmd/mesh-bootstrap/main.go +++ b/cmd/mesh-bootstrap/main.go @@ -188,9 +188,7 @@ func parseArgs(args []string) (string, bootstrap.Options, bool, error) { Out: defaultOut, State: store.DefaultPath, Registry: defaultRegistry, - // Where the store's holder reads the one-off credential the holders' first images are pushed with (design 51, ENG-5). - PushCredentialFile: "/var/lib/mesh/registry/bootstrap-push", - Host: defaultHost, + Host: defaultHost, // The machine's own name, because that is what a person already calls it and an installer // inventing a different one would leave the mesh naming a machine nobody recognises. It is // read here rather than inside the bootstrap so that --node overrides a fact rather than a diff --git a/internal/bootstrap/bootstrap.go b/internal/bootstrap/bootstrap.go index b95905c..5835bc5 100644 --- a/internal/bootstrap/bootstrap.go +++ b/internal/bootstrap/bootstrap.go @@ -157,11 +157,6 @@ type Options struct { // an address the others can reach. Registry string - // PushCredentialFile is where the bootstrap writes its one-off push credential for the store's holder to find, the - // holder's state file `bootstrap-push` on the node that holds it (design 51 *The move*, issue 1966 ENG-5); empty writes - // none. The file goes to the account that owns that directory in the holder's manifest, once the node-engine made both. - PushCredentialFile string - // Host is the `mesh-host` binary on this machine — the program that enrols and then holds the // machine to what the mesh says. The installer runs it; it does not contain it. Host string @@ -301,12 +296,10 @@ type Result struct { Agent string `json:"host-agent,omitempty"` // Registry is the mesh's own artifact store, once it answers. - Registry string `json:"registry,omitempty"` - // PushCredentialFile is where this run left its one-off push credential (never the credential itself). - PushCredentialFile string `json:"push-credential-file,omitempty"` - RegistryReplied int `json:"registry-replied,omitempty"` - RegistryKnown bool `json:"registry-already-registered,omitempty"` - RegistryRunning string `json:"registry-container,omitempty"` + Registry string `json:"registry,omitempty"` + RegistryReplied int `json:"registry-replied,omitempty"` + RegistryKnown bool `json:"registry-already-registered,omitempty"` + RegistryRunning string `json:"registry-container,omitempty"` // RegistryTrusted are the names the runtime was told to trust the registry by, before the push. RegistryTrusted []string `json:"registry-trusted,omitempty"` PublishedAs string `json:"control-plane-image,omitempty"` @@ -822,18 +815,6 @@ func RegistryAndPublish(ctx context.Context, o Options, d Deps, sys system.Syste if err != nil { return Published{}, failed(StepRegistry, err) } - // The one-off credential the holders' first images are pushed to the store's front with, written for its holder to find - // and forgotten when this run ends; the holder discards the file once it answers its first publish (ENG-5). - if o.PushCredentialFile != "" { - cred, err := WritePushCredential(o.Catalogue, o.PushCredentialFile) - if err != nil { - return Published{}, failed(StepRegistry, err) - } - defer cred.Forget() - result.PushCredentialFile = cred.File - say(" a one-off credential for the holders' first images is at " + cred.File + "; the store's holder discards it once it serves publish") - } - // ---- 8. publish ----------------------------------------------------------------------- say("publish — the control plane's image gets its first manifest digest") // **The image this mesh built, not the one the installer carried.** The carried one is the diff --git a/internal/bootstrap/publish.go b/internal/bootstrap/publish.go index 561ce92..97dbd13 100644 --- a/internal/bootstrap/publish.go +++ b/internal/bootstrap/publish.go @@ -69,17 +69,24 @@ func PublishControlPlane(ctx context.Context, o Options, d Deps, imageID string, func publishAs(ctx context.Context, o Options, d Deps, imageID, repository string, say func(string)) (Published, error) { - remote := o.Registry + "/" + repository + // Pushed to the store's own loopback with no credential (StoreLoopback); recorded under the seat's reach, the + // address every node pulls by, with the digest the store gave. + loopback, err := StoreLoopback(o.Catalogue) + if err != nil { + return Published{}, err + } + remote := loopback + "/" + repository out := Published{Tagged: remote + ":" + genesisTag} + reach := func(digest string) string { return o.Registry + "/" + repository + strings.TrimPrefix(digest, remote) } // Asked first. A digest already served is a fact about the registry, and re-pushing an image // the registry already holds is asking it to store what it already has under the name it // already has. - if held, err := digestOf(ctx, o, d, remote); err != nil { + if held, err := digestOf(ctx, o, d, loopback, repository); err != nil { return out, err } else if held != "" { - out.Reference, out.Already = held, true - say(" already published " + held) + out.Reference, out.Already = reach(held), true + say(" already published " + out.Reference) return out, nil } @@ -93,12 +100,12 @@ func publishAs(ctx context.Context, o Options, d Deps, imageID, repository strin "over the mesh's own network, which is already the encrypted and authenticated "+ "thing. A runtime refusing it for being insecure is refusing a registry on %s, "+ "which it does not do for a loopback address", - out.Tagged, err, o.Registry) + out.Tagged, err, loopback) } // Read back, from the registry's own answer rather than computed here. What matters is what // the registry will serve for that reference, and only it can say (novox/hq ADR 0018). - pinned, err := digestOf(ctx, o, d, remote) + pinned, err := digestOf(ctx, o, d, loopback, repository) if err != nil { return out, err } @@ -107,10 +114,10 @@ func publishAs(ctx context.Context, o Options, d Deps, imageID, repository strin "%s was pushed and the registry does not serve it.\n"+ "The next step names this module by the digest this was supposed to produce, so "+ "there is nothing to name. Check `docker push` and "+ - "http://%s/v2/%s/tags/list", out.Tagged, o.Registry, repository) + "http://%s/v2/%s/tags/list", out.Tagged, loopback, repository) } - out.Reference = pinned - say(" published " + pinned) + out.Reference = reach(pinned) + say(" published " + out.Reference) return out, nil } @@ -121,13 +128,13 @@ func publishAs(ctx context.Context, o Options, d Deps, imageID, repository strin // whether the registry still has it — a registry whose volume was recreated would leave the // runtime remembering a digest nothing serves, and the module registered against it would pin the // mesh to an image that cannot be pulled. -func digestOf(ctx context.Context, o Options, d Deps, remote string) (string, error) { +func digestOf(ctx context.Context, o Options, d Deps, store, repository string) (string, error) { + remote := store + "/" + repository asking, cancel := context.WithTimeout(ctx, o.Timeout) - status, body, err := d.Fetch(asking, - "http://"+o.Registry+"/v2/"+ControlPlaneRepository+"/tags/list") + status, body, err := d.Fetch(asking, "http://"+store+"/v2/"+repository+"/tags/list") cancel() if err != nil { - return "", fmt.Errorf("cannot ask the registry at %s what it holds: %w", o.Registry, err) + return "", fmt.Errorf("cannot ask the registry at %s what it holds: %w", store, err) } if status == http.StatusNotFound { // Nothing has ever been pushed under this name. An answer, not a failure. @@ -135,14 +142,14 @@ func digestOf(ctx context.Context, o Options, d Deps, remote string) (string, er } if status != http.StatusOK { return "", fmt.Errorf("the registry answered %d when asked what it holds for %s", - status, ControlPlaneRepository) + status, repository) } var listed struct { Tags []string `json:"tags"` } if err := json.Unmarshal([]byte(body), &listed); err != nil { return "", fmt.Errorf("the registry's answer about %s is not readable: %w", - ControlPlaneRepository, err) + repository, err) } if !contains(listed.Tags, genesisTag) { return "", nil diff --git a/internal/bootstrap/publish_test.go b/internal/bootstrap/publish_test.go index adeaf95..6120136 100644 --- a/internal/bootstrap/publish_test.go +++ b/internal/bootstrap/publish_test.go @@ -22,7 +22,8 @@ func publishing(t *testing.T, fetch func(string) (int, string, error), // Each value named before it is returned: a return of several composite literals spanning lines is laid // out differently by the gofmt of one Go release and the next, and the build seat's is the one that // judges (novox/hq issue 286). - options := Options{Registry: "127.0.0.1:5000", Timeout: time.Second, Wait: 0} + options := Options{Registry: "127.0.0.1:5000", Timeout: time.Second, Wait: 0, + Catalogue: catalogueWith(t, RegistryModule, upstreamRegistryManifest)} deps := Deps{ Run: runtime.run, Fetch: func(_ context.Context, url string) (int, string, error) { diff --git a/internal/bootstrap/push_credential.go b/internal/bootstrap/push_credential.go deleted file mode 100644 index 9134836..0000000 --- a/internal/bootstrap/push_credential.go +++ /dev/null @@ -1,114 +0,0 @@ -package bootstrap - -import ( - "crypto/rand" - "encoding/hex" - "encoding/json" - "fmt" - "os" - "os/user" - "path/filepath" - "strconv" - "syscall" -) - -// PushCredential is the bootstrap's one-off credential for the holders' first images (design 51 *The move*; issue 1966 ENG-5): -// a bearer token of 32 random bytes, hex, on one line, made for one fresh-mesh run. No vault and no bus holds it: this run -// holds it and writes it once, to the file the store's holder reads on each write (`Authorization: Bearer ` on an OCI -// write to the store's port, from any node); the holder removes the file when it answers its first `publish`, and this run -// forgets the token when it ends. On the holder's own node the bootstrap may push to the registry container's loopback with -// no credential at all. -type PushCredential struct { - // File is where the token was written. - File string - token string -} - -// Token is the bearer token, for a push through the store's front from another node. -func (c *PushCredential) Token() string { return c.token } - -// Forget drops the token from this run's memory. The file is the holder's to remove. -func (c *PushCredential) Forget() { c.token = "" } - -// lookupAccount gives an account's user id and group id. A variable so a test can name an account this machine has. -var lookupAccount = func(name string) (uid, gid int, err error) { - u, err := user.Lookup(name) - if err != nil { - return 0, 0, err - } - if uid, err = strconv.Atoi(u.Uid); err != nil { - return 0, 0, err - } - gid, err = strconv.Atoi(u.Gid) - return uid, gid, err -} - -// WritePushCredential writes the one-off credential for the store's holder, read from its manifest in catalogue: the file -// goes into the holder's directory resource that holds file's path, given to the account that directory names as owner, -// and only once the node-engine has made that account and that directory, owned by it. Until then the holder's program, -// which runs as that account, could not read it, so it is refused rather than written as root's. -func WritePushCredential(catalogue, file string) (*PushCredential, error) { - manifest, err := readManifest(catalogue, RegistryModule) - if err != nil { - return nil, err - } - var m struct { - Resources []struct { - Type string `json:"type"` - Name string `json:"name"` - Path string `json:"path"` - Owner string `json:"owner"` - } `json:"resources"` - } - if err := json.Unmarshal(manifest, &m); err != nil { - return nil, fmt.Errorf("the %s manifest is not readable as JSON: %w", RegistryModule, err) - } - dir, owner, account := filepath.Dir(file), "", false - for _, r := range m.Resources { - if r.Type == "directory" && filepath.Clean(r.Path) == dir { - owner = r.Owner - } - } - for _, r := range m.Resources { - account = account || (owner != "" && r.Type == "user" && r.Name == owner) - } - if !account { - return nil, fmt.Errorf("the push credential %s has no account to go to: the %s manifest declares no directory %s "+ - "owned by an account it declares, so its program could not read the file", file, RegistryModule, dir) - } - uid, gid, err := lookupAccount(owner) - if err != nil { - return nil, fmt.Errorf("the push credential waits on the account %s, which the node-engine has not made: %w", owner, err) - } - st, err := os.Stat(dir) - if err != nil { - return nil, fmt.Errorf("the push credential waits on %s's state directory %s, which the node-engine has not made: %w", owner, dir, err) - } - if sys, ok := st.Sys().(*syscall.Stat_t); !ok || int(sys.Uid) != uid { - return nil, fmt.Errorf("the push credential waits on %s owning its state directory %s, which the node-engine has not yet given it", owner, dir) - } - return MintPushCredential(file, uid, gid) -} - -// MintPushCredential makes a token and writes it to file, mode 0600, owned by the user id and group id given, into a -// directory that must be there. A file that is there is replaced: a credential left by an earlier run is never reused. -func MintPushCredential(file string, uid, gid int) (*PushCredential, error) { - raw := make([]byte, 32) - if _, err := rand.Read(raw); err != nil { - return nil, fmt.Errorf("a push credential cannot be made: %w", err) - } - token := hex.EncodeToString(raw) - tmp := file + ".new" - if err := os.WriteFile(tmp, []byte(token+"\n"), 0o600); err != nil { - return nil, fmt.Errorf("the push credential cannot be written to %s: %w", file, err) - } - if err := os.Chown(tmp, uid, gid); err != nil { - os.Remove(tmp) - return nil, fmt.Errorf("the push credential cannot be given to the store's holder: %w", err) - } - if err := os.Rename(tmp, file); err != nil { - os.Remove(tmp) - return nil, fmt.Errorf("the push credential cannot be put at %s: %w", file, err) - } - return &PushCredential{File: file, token: token}, nil -} diff --git a/internal/bootstrap/registry.go b/internal/bootstrap/registry.go index 359007f..966d612 100644 --- a/internal/bootstrap/registry.go +++ b/internal/bootstrap/registry.go @@ -95,9 +95,14 @@ func InstallRegistry(ctx context.Context, o Options, d Deps, control controlPlan return out, nil } -// waitForTheRegistry asks `/v2/` until it answers. +// waitForTheRegistry asks the store's own `/v2/`, on its loopback (StoreLoopback), until it answers: the pushes that +// follow go there, and the store's front is not served yet on a fresh mesh. func waitForTheRegistry(ctx context.Context, d Deps, o Options, say func(string)) (int, error) { - where := "http://" + o.Registry + "/v2/" + loopback, err := StoreLoopback(o.Catalogue) + if err != nil { + return 0, err + } + where := "http://" + loopback + "/v2/" deadline := time.Now().Add(o.Wait) var last string @@ -207,3 +212,43 @@ func containerIn(manifest []byte, id string) (name, image string, err error) { "the %s module declares no container %q, so the installer does not know what to wait for. "+ "It declares: %s", m.Module, id, strings.Join(containers, ", ")) } + +// storeContainerPort is the port the registry image listens on inside its container (Distribution's own default). +const storeContainerPort = "5000" + +// StoreLoopback is host:port where this machine reaches the store's container itself, read from the port its manifest +// publishes for it: on a fresh mesh the bootstrap pushes the holders' own images there, with no credential, because the +// store's front (distribution's program, a bundle the mesh builds) is not served until the builder has built it (the +// publish contract allows a push on the store's own node to its loopback). A port published on every address is reached +// on 127.0.0.1. What the bootstrap records is named under the seat's reach (Options.Registry), never this address. +func StoreLoopback(catalogue string) (string, error) { + manifest, err := readManifest(catalogue, RegistryModule) + if err != nil { + return "", err + } + var m struct { + Resources []struct { + ID string `json:"id"` + Ports []string `json:"ports"` + } `json:"resources"` + } + if err := json.Unmarshal(manifest, &m); err != nil { + return "", fmt.Errorf("the %s manifest is not readable as JSON: %w", RegistryModule, err) + } + for _, r := range m.Resources { + if r.ID != registryResource { + continue + } + for _, p := range r.Ports { + parts := strings.Split(strings.SplitN(p, "/", 2)[0], ":") + switch { + case len(parts) == 3 && parts[2] == storeContainerPort: + return parts[0] + ":" + parts[1], nil + case len(parts) == 2 && parts[1] == storeContainerPort: + return "127.0.0.1:" + parts[0], nil + } + } + } + return "", fmt.Errorf("the %s module's container %q publishes no port for the store's %s, so the bootstrap cannot "+ + "reach the store to push to it", RegistryModule, registryResource, storeContainerPort) +} diff --git a/internal/bootstrap/trust_test.go b/internal/bootstrap/trust_test.go index 6fd6533..9ec16d4 100644 --- a/internal/bootstrap/trust_test.go +++ b/internal/bootstrap/trust_test.go @@ -7,9 +7,7 @@ import ( "net/http" "os" "path/filepath" - "regexp" "strings" - "syscall" "testing" "time" @@ -228,29 +226,15 @@ func TestARangeTheRuntimeTrustsCoversAnAddressInIt(t *testing.T) { // moving the trust after the push, or dropping it, fails this test. func TestNothingIsPushedToTheRegistryBeforeTheRuntimeTrustsIt(t *testing.T) { o, runtime, sys := trustSetup(t, `{"data-root":"/srv/docker"}`) - // The holder's state directory and its account, as distribution's manifest declares them (ENG-5). - stateDir := filepath.Join(t.TempDir(), "registry") - o.Catalogue = catalogueWith(t, RegistryModule, strings.Replace(upstreamRegistryManifest, - `{"id": "state", "type": "directory", "path": "/var/lib/mesh/registry", "mode": "0700"}`, - `{"id": "account", "type": "user", "name": "distribution"}, - {"id": "state", "type": "directory", "path": "`+stateDir+`", "mode": "0700", "owner": "distribution"}`, 1)) + // The store as distribution's manifest declares it: its container on this machine's loopback, its front not yet served, + // and its state directory, where no credential is ever written (design 51 *On a fresh mesh*, a design limit of issue 1966). + stateDir := t.TempDir() + o.Catalogue = catalogueWith(t, RegistryModule, strings.NewReplacer( + `"path": "/var/lib/mesh/registry"`, `"path": "`+stateDir+`"`, + `"ports": ["5000:5000"]`, `"ports": ["127.0.0.1:5001:5000"]`).Replace(upstreamRegistryManifest)) o.Registry = "192.0.2.10:5100" - asked, was := []string{}, lookupAccount - lookupAccount = func(name string) (int, int, error) { - asked = append(asked, name) - return os.Getuid(), os.Getgid(), nil - } - t.Cleanup(func() { lookupAccount = was }) - o.PushCredentialFile = filepath.Join(stateDir, "bootstrap-push") - // Before the node-engine made distribution's state directory, nothing is written: the program could not read it. - if _, err := WritePushCredential(o.Catalogue, o.PushCredentialFile); err == nil { - t.Errorf("the push credential was written before its holder's state directory existed") - } else if _, err := os.Stat(stateDir); err == nil { - t.Errorf("the bootstrap made the holder's state directory itself") - } - if err := os.Mkdir(stateDir, 0o700); err != nil { - t.Fatal(err) - } + const loopback = "127.0.0.1:5001" + var fetched []string pushed := false mesh := aMeshThatAgrees(nil) @@ -264,7 +248,8 @@ func TestNothingIsPushedToTheRegistryBeforeTheRuntimeTrustsIt(t *testing.T) { case "tag": return "", nil case "push": - trusted := false + // The runtime trusts a loopback registry with no configuration; any other only once told. + trusted := strings.HasPrefix(args[1], "127.0.0.1:") for _, n := range runtime.trusted { trusted = trusted || strings.HasPrefix(args[1], n+"/") } @@ -274,20 +259,19 @@ func TestNothingIsPushedToTheRegistryBeforeTheRuntimeTrustsIt(t *testing.T) { pushed = true return "", nil case "image": - return `["` + o.Registry + `/mesh-controller@sha256:` + strings.Repeat("a", 64) + `"]`, nil + return `["` + loopback + `/mesh-controller@sha256:` + strings.Repeat("a", 64) + `"]`, nil } } return mesh(name, args) } d := Deps{Run: run, Fetch: func(_ context.Context, url string) (int, string, error) { + fetched = append(fetched, url) if strings.HasSuffix(url, "/v2/") || pushed { return http.StatusOK, `{"name":"mesh-controller","tags":["genesis"]}`, nil } return http.StatusNotFound, "", nil }} var result Result - // And the one-off credential for the holders' first images (ENG-5): written once, mode 0600, one line of 64 hex digits, - // named in the result and never put in it. published, err := RegistryAndPublish(context.Background(), o, d, sys, controlPlane{container: "temp-mesh-controller", run: run, timeout: time.Second}, held, &result, func(string) {}) if err != nil { @@ -312,22 +296,22 @@ func TestNothingIsPushedToTheRegistryBeforeTheRuntimeTrustsIt(t *testing.T) { t.Errorf("the trust was not written (%d) and confirmed (%d) before the first tag or push (%d): %v", reloaded, confirmed, firstPush, runtime.commands) } - if raw, err := os.ReadFile(o.PushCredentialFile); err != nil || !regexp.MustCompile(`^[0-9a-f]{64}\n$`).Match(raw) { - t.Errorf("the push credential is %q (%v)", raw, err) - } else if st, _ := os.Stat(o.PushCredentialFile); st.Mode().Perm() != 0o600 || - int(st.Sys().(*syscall.Stat_t).Uid) != os.Getuid() || int(st.Sys().(*syscall.Stat_t).Gid) != os.Getgid() { - t.Errorf("the push credential is mode %v, owner %d:%d", st.Mode().Perm(), st.Sys().(*syscall.Stat_t).Uid, st.Sys().(*syscall.Stat_t).Gid) + // The push went to the store on its loopback, and nothing the run asked or pushed reached the seat's reach; the + // reference is named under the reach all the same, and no credential file was written for the store's holder. + for _, c := range runtime.commands { + if strings.HasPrefix(c, "docker tag") || strings.HasPrefix(c, "docker push") { + if !strings.Contains(c, " "+loopback+"/mesh-controller:") { + t.Errorf("a push not to the loopback store: %q", c) + } + } } - if fmt.Sprint(asked) != "[distribution distribution]" { - t.Errorf("the push credential's owner was asked as %v, not the account distribution's state directory names", asked) + for _, u := range fetched { + if !strings.HasPrefix(u, "http://"+loopback+"/v2/") { + t.Errorf("the run asked %s, not the loopback store", u) + } } - if result.PushCredentialFile != o.PushCredentialFile { - t.Errorf("the result names %q", result.PushCredentialFile) - } - if again, err := MintPushCredential(o.PushCredentialFile, os.Getuid(), os.Getgid()); err != nil || again.Token() == "" { - t.Errorf("a credential of an earlier run was not replaced: %v", err) - } else if again.Forget(); again.Token() != "" { - t.Errorf("a forgotten credential still holds its token") + if entries, err := os.ReadDir(stateDir); err != nil || len(entries) != 0 { + t.Errorf("the run wrote into the store's state directory: %v (%v)", entries, err) } if fmt.Sprint(result.RegistryTrusted) != "[192.0.2.10:5100 anchor.internal:5100]" { t.Errorf("the result says it trusted %v", result.RegistryTrusted)