From ab2ca6e3a7e6dd43a03d492f993c5fc5e6fa6fb5 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 17:59:14 +0200 Subject: [PATCH] =?UTF-8?q?Push=20a=20done=20build's=20bundle=20or=20archi?= =?UTF-8?q?ve=20to=20the=20artifact=20store=20from=20the=20build=20module,?= =?UTF-8?q?=20in=20one=20shared=20client=20(issue=201587,=20ADR=200498=20?= =?UTF-8?q?=C2=A71-=C2=A72)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit buildstep.PushBlob uploads by digest unless the store holds it, waits out a store refusing connections as mesh-build-image does, and names the store's own words; BlobReference is the form the key and the controller check. --- go/buildstep/blob.go | 170 ++++++++++++++++++++++++++++++++++++++ go/buildstep/blob_test.go | 151 +++++++++++++++++++++++++++++++++ 2 files changed, 321 insertions(+) create mode 100644 go/buildstep/blob.go create mode 100644 go/buildstep/blob_test.go diff --git a/go/buildstep/blob.go b/go/buildstep/blob.go new file mode 100644 index 0000000..127c6ac --- /dev/null +++ b/go/buildstep/blob.go @@ -0,0 +1,170 @@ +package buildstep + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "syscall" + "time" +) + +// RegistryWait is how long a push waits for an artifact store that refuses connections, longer than its nightly +// collection holds it still (about a minute and a half, issue 457); RegistryFirstPause is the first pause, doubling +// to RegistryMostPause. Variables so a test shortens them. +var ( + RegistryWait = 5 * time.Minute + RegistryFirstPause = time.Second + RegistryMostPause = 10 * time.Second +) + +// BlobReference is where a pushed bundle or archive is, as a done key names it and the controller delivers from +// (novox/hq ADR 0498 clause 1): artifact-store:////blobs/, the store's address left out +// (issue 102). +func BlobReference(module, artifact, digest string) string { + return "artifact-store://" + module + "/" + artifact + "/blobs/" + digest +} + +// PushBlob puts body into the artifact store at registry (host:port) as a blob of repository (/) by +// its digest (`sha256:` and hex), from the build module's own process, after the build step ended (novox/hq ADR 0498 +// clauses 1 and 2): a blob the store already holds is done without an upload; otherwise it is uploaded (POST +// blobs/uploads/, then PUT with the digest). A store refusing connections is waited out up to RegistryWait; any other +// refusal fails, naming the store's status and its own words, for the caller's `not stored:` error. Bytes that are not +// their digest are refused before anything is sent. +func PushBlob(ctx context.Context, client *http.Client, registry, repository, digest string, body []byte) error { + sum := sha256.Sum256(body) + if got := "sha256:" + hex.EncodeToString(sum[:]); got != digest { + return fmt.Errorf("the blob is %s, not the digest %s it was to be pushed as", got, digest) + } + if client == nil { + client = http.DefaultClient + } + base := "http://" + registry + "/v2/" + repository + "/blobs/" + var present bool + if err := waitForStore(ctx, registry, func() error { + var err error + present, err = blobPresent(ctx, client, base+digest) + return err + }); err != nil { + return err + } + if present { + return nil + } + var location string + if err := waitForStore(ctx, registry, func() error { + var err error + location, err = startUpload(ctx, client, base+"uploads/") + return err + }); err != nil { + return err + } + target, err := url.Parse(base) + if err != nil { + return err + } + loc, err := target.Parse(location) + if err != nil { + return fmt.Errorf("the store answered the upload's location %q, which is not a URL: %w", location, err) + } + q := loc.Query() + q.Set("digest", digest) + loc.RawQuery = q.Encode() + req, err := http.NewRequestWithContext(ctx, http.MethodPut, loc.String(), bytes.NewReader(body)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/octet-stream") + resp, err := client.Do(req) + if err != nil { + return fmt.Errorf("putting %s of %s: %w", digest, repository, err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusCreated { + return storeSaid(resp, "putting "+digest+" of "+repository) + } + return nil +} + +func blobPresent(ctx context.Context, client *http.Client, at string) (bool, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodHead, at, nil) + if err != nil { + return false, err + } + resp, err := client.Do(req) + if err != nil { + return false, err + } + resp.Body.Close() + switch resp.StatusCode { + case http.StatusOK: + return true, nil + case http.StatusNotFound: + return false, nil + } + return false, fmt.Errorf("asking the store for %s: it answered %s", at, resp.Status) +} + +func startUpload(ctx context.Context, client *http.Client, at string) (string, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodPost, at, nil) + if err != nil { + return "", err + } + resp, err := client.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusAccepted { + return "", storeSaid(resp, "starting an upload at "+at) + } + location := resp.Header.Get("Location") + if location == "" { + return "", fmt.Errorf("starting an upload at %s: the store named no location", at) + } + return location, nil +} + +// storeSaid is the store's refusal: its status and the first of its own words. +func storeSaid(resp *http.Response, what string) error { + words, _ := io.ReadAll(io.LimitReader(resp.Body, 2048)) + return fmt.Errorf("%s: the store answered %s: %s", what, resp.Status, strings.TrimSpace(string(words))) +} + +// refused is whether an error is a connection refused. +func refused(err error) bool { + return err != nil && (errors.Is(err, syscall.ECONNREFUSED) || strings.Contains(err.Error(), "connection refused")) +} + +// waitForStore runs try, and while it fails because the store refuses connections, tries again with a growing pause +// until RegistryWait has passed. A refusal is the one error waited for: nothing was sent, so trying again does nothing +// twice. +func waitForStore(ctx context.Context, registry string, try func() error) error { + err := try() + if !refused(err) { + return err + } + started, pause := time.Now(), RegistryFirstPause + for { + left := RegistryWait - time.Since(started) + if left <= 0 { + return fmt.Errorf("the artifact store at %s refused every connection for %s, longer than its nightly collection holds it still, so it is down, not paused: %w", + registry, time.Since(started).Round(time.Millisecond), err) + } + select { + case <-ctx.Done(): + return fmt.Errorf("stopped while waiting for the artifact store at %s: %w (last: %v)", registry, ctx.Err(), err) + case <-time.After(min(pause, left)): + } + pause = min(pause*2, RegistryMostPause) + if err = try(); !refused(err) { + return err + } + } +} diff --git a/go/buildstep/blob_test.go b/go/buildstep/blob_test.go new file mode 100644 index 0000000..b77afb0 --- /dev/null +++ b/go/buildstep/blob_test.go @@ -0,0 +1,151 @@ +package buildstep + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "io" + "net" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +func digestOfBytes(b []byte) string { + h := sha256.Sum256(b) + return "sha256:" + hex.EncodeToString(h[:]) +} + +// registry is an artifact store as far as a blob push needs one: HEAD a blob, start an upload, PUT it with its digest. +type registry struct { + mu sync.Mutex + blobs map[string][]byte + calls []string + put int // the status a PUT answers, 201 when 0 +} + +func (r *registry) ServeHTTP(w http.ResponseWriter, req *http.Request) { + r.mu.Lock() + defer r.mu.Unlock() + r.calls = append(r.calls, req.Method+" "+req.URL.Path) + switch { + case req.Method == http.MethodHead && strings.Contains(req.URL.Path, "/blobs/sha256:"): + if _, ok := r.blobs[req.URL.Path[strings.LastIndex(req.URL.Path, "/")+1:]]; ok { + w.WriteHeader(http.StatusOK) + return + } + w.WriteHeader(http.StatusNotFound) + case req.Method == http.MethodPost && strings.HasSuffix(req.URL.Path, "/blobs/uploads/"): + w.Header().Set("Location", strings.TrimSuffix(req.URL.Path, "/")+"/u1?state=x") + w.WriteHeader(http.StatusAccepted) + case req.Method == http.MethodPut && strings.Contains(req.URL.Path, "/blobs/uploads/u1"): + if r.put != 0 { + w.WriteHeader(r.put) + _, _ = io.WriteString(w, `{"errors":[{"code":"DIGEST_INVALID","message":"provided digest did not match uploaded content"}]}`) + return + } + body, _ := io.ReadAll(req.Body) + d := req.URL.Query().Get("digest") + if req.URL.Query().Get("state") != "x" || d != digestOfBytes(body) { + w.WriteHeader(http.StatusBadRequest) + return + } + r.blobs[d] = body + w.WriteHeader(http.StatusCreated) + default: + w.WriteHeader(http.StatusMethodNotAllowed) + } +} + +// A done build's bundle or archive is pushed to the artifact store by its build module (novox/hq ADR 0498 clauses 1 +// and 2): a blob the store already holds is done with no upload; one it lacks is uploaded by digest; its reference is +// the artifact-store form the builder's key and the controller check. +func TestABlobIsPushedOnceByItsDigest(t *testing.T) { + reg := ®istry{blobs: map[string][]byte{}} + srv := httptest.NewServer(reg) + defer srv.Close() + addr := strings.TrimPrefix(srv.URL, "http://") + body := []byte("the packed bundle") + d := digestOfBytes(body) + if err := PushBlob(context.Background(), srv.Client(), addr, "mail/code", d, body); err != nil { + t.Fatal(err) + } + if string(reg.blobs[d]) != string(body) { + t.Fatalf("the store holds %q", reg.blobs[d]) + } + reg.calls = nil + if err := PushBlob(context.Background(), srv.Client(), addr, "mail/code", d, body); err != nil { + t.Fatal(err) + } + if len(reg.calls) != 1 || !strings.HasPrefix(reg.calls[0], "HEAD ") { + t.Fatalf("a blob the store holds was uploaded again: %v", reg.calls) + } + ref := BlobReference("mail", "code", d) + if ref != "artifact-store://mail/code/blobs/"+d { + t.Fatalf("BlobReference = %q", ref) + } + b := delivery.BuilderBuild{State: delivery.BuildDone, Manifest: strings.Repeat("4", 64), + Artifacts: []delivery.BuiltArtifact{{Name: "code", Kind: "bundle", Digest: d, Reference: ref}}} + if _, err := delivery.EncodeBuilderBuild(b); err != nil { + t.Fatalf("the key refuses the reference BlobReference gives: %v", err) + } +} + +// A store that refuses the upload fails the push naming the store's own words; bytes that are not the digest are +// refused before anything is sent. +func TestAPushTheStoreRefusesSaysTheStoresWords(t *testing.T) { + reg := ®istry{blobs: map[string][]byte{}, put: http.StatusBadRequest} + srv := httptest.NewServer(reg) + defer srv.Close() + addr := strings.TrimPrefix(srv.URL, "http://") + body := []byte("x") + err := PushBlob(context.Background(), srv.Client(), addr, "mail/code", digestOfBytes(body), body) + if err == nil || !strings.Contains(err.Error(), "provided digest did not match") || !strings.Contains(err.Error(), "400") { + t.Fatalf("a refused upload said %v", err) + } + reg.calls = nil + err = PushBlob(context.Background(), srv.Client(), addr, "mail/code", digestOfBytes([]byte("other")), body) + if err == nil || len(reg.calls) != 0 { + t.Fatalf("bytes that are not their digest: %v, calls %v", err, reg.calls) + } +} + +// A store refusing connections (its nightly collection holds it still) is waited out, up to RegistryWait (issue 457). +func TestAPushWaitsOutAStoreRefusingConnections(t *testing.T) { + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + addr := l.Addr().String() + l.Close() // refusing now + reg := ®istry{blobs: map[string][]byte{}} + go func() { + time.Sleep(300 * time.Millisecond) + l2, err := net.Listen("tcp", addr) + if err != nil { + return + } + _ = http.Serve(l2, reg) + }() + beforeWait, beforePause := RegistryWait, RegistryFirstPause + RegistryWait, RegistryFirstPause = 3*time.Second, 100*time.Millisecond + defer func() { RegistryWait, RegistryFirstPause = beforeWait, beforePause }() + body := []byte("late") + if err := PushBlob(context.Background(), http.DefaultClient, addr, "mail/code", digestOfBytes(body), body); err != nil { + t.Fatalf("a store that came back was not waited for: %v", err) + } + + l3, _ := net.Listen("tcp", "127.0.0.1:0") + gone := l3.Addr().String() + l3.Close() + RegistryWait = 300 * time.Millisecond + err = PushBlob(context.Background(), http.DefaultClient, gone, "mail/code", digestOfBytes(body), body) + if err == nil || !strings.Contains(err.Error(), "refused every connection") { + t.Fatalf("a store that stayed down said %v", err) + } +}