Push a done build's bundle or archive to the artifact store from the build module, in one shared client (issue 1587, ADR 0498 §1-§2)

buildstep.PushBlob uploads by digest unless the store holds it, waits out a store refusing connections as mesh-build-image does, and names the store's own words; BlobReference is the form the key and the controller check.
2026-10-22 17:59:14 +02:00
parent a90f523edc
commit ab2ca6e3a7
2 changed files with 321 additions and 0 deletions
@@ -0,0 +1,170 @@
+package buildstep
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strings"
+	"syscall"
+	"time"
+)
+
+// RegistryWait is how long a push waits for an artifact store that refuses connections, longer than its nightly
+// collection holds it still (about a minute and a half, issue 457); RegistryFirstPause is the first pause, doubling
+// to RegistryMostPause. Variables so a test shortens them.
+var (
+	RegistryWait       = 5 * time.Minute
+	RegistryFirstPause = time.Second
+	RegistryMostPause  = 10 * time.Second
+)
+
+// BlobReference is where a pushed bundle or archive is, as a done key names it and the controller delivers from
+// (novox/hq ADR 0498 clause 1): artifact-store://<module>/<artifact>/blobs/<digest>, the store's address left out
+// (issue 102).
+func BlobReference(module, artifact, digest string) string {
+	return "artifact-store://" + module + "/" + artifact + "/blobs/" + digest
+}
+
+// PushBlob puts body into the artifact store at registry (host:port) as a blob of repository (<module>/<artifact>) by
+// its digest (`sha256:` and hex), from the build module's own process, after the build step ended (novox/hq ADR 0498
+// clauses 1 and 2): a blob the store already holds is done without an upload; otherwise it is uploaded (POST
+// blobs/uploads/, then PUT with the digest). A store refusing connections is waited out up to RegistryWait; any other
+// refusal fails, naming the store's status and its own words, for the caller's `not stored:` error. Bytes that are not
+// their digest are refused before anything is sent.
+func PushBlob(ctx context.Context, client *http.Client, registry, repository, digest string, body []byte) error {
+	sum := sha256.Sum256(body)
+	if got := "sha256:" + hex.EncodeToString(sum[:]); got != digest {
+		return fmt.Errorf("the blob is %s, not the digest %s it was to be pushed as", got, digest)
+	}
+	if client == nil {
+		client = http.DefaultClient
+	}
+	base := "http://" + registry + "/v2/" + repository + "/blobs/"
+	var present bool
+	if err := waitForStore(ctx, registry, func() error {
+		var err error
+		present, err = blobPresent(ctx, client, base+digest)
+		return err
+	}); err != nil {
+		return err
+	}
+	if present {
+		return nil
+	}
+	var location string
+	if err := waitForStore(ctx, registry, func() error {
+		var err error
+		location, err = startUpload(ctx, client, base+"uploads/")
+		return err
+	}); err != nil {
+		return err
+	}
+	target, err := url.Parse(base)
+	if err != nil {
+		return err
+	}
+	loc, err := target.Parse(location)
+	if err != nil {
+		return fmt.Errorf("the store answered the upload's location %q, which is not a URL: %w", location, err)
+	}
+	q := loc.Query()
+	q.Set("digest", digest)
+	loc.RawQuery = q.Encode()
+	req, err := http.NewRequestWithContext(ctx, http.MethodPut, loc.String(), bytes.NewReader(body))
+	if err != nil {
+		return err
+	}
+	req.Header.Set("Content-Type", "application/octet-stream")
+	resp, err := client.Do(req)
+	if err != nil {
+		return fmt.Errorf("putting %s of %s: %w", digest, repository, err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusCreated {
+		return storeSaid(resp, "putting "+digest+" of "+repository)
+	}
+	return nil
+}
+
+func blobPresent(ctx context.Context, client *http.Client, at string) (bool, error) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodHead, at, nil)
+	if err != nil {
+		return false, err
+	}
+	resp, err := client.Do(req)
+	if err != nil {
+		return false, err
+	}
+	resp.Body.Close()
+	switch resp.StatusCode {
+	case http.StatusOK:
+		return true, nil
+	case http.StatusNotFound:
+		return false, nil
+	}
+	return false, fmt.Errorf("asking the store for %s: it answered %s", at, resp.Status)
+}
+
+func startUpload(ctx context.Context, client *http.Client, at string) (string, error) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, at, nil)
+	if err != nil {
+		return "", err
+	}
+	resp, err := client.Do(req)
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusAccepted {
+		return "", storeSaid(resp, "starting an upload at "+at)
+	}
+	location := resp.Header.Get("Location")
+	if location == "" {
+		return "", fmt.Errorf("starting an upload at %s: the store named no location", at)
+	}
+	return location, nil
+}
+
+// storeSaid is the store's refusal: its status and the first of its own words.
+func storeSaid(resp *http.Response, what string) error {
+	words, _ := io.ReadAll(io.LimitReader(resp.Body, 2048))
+	return fmt.Errorf("%s: the store answered %s: %s", what, resp.Status, strings.TrimSpace(string(words)))
+}
+
+// refused is whether an error is a connection refused.
+func refused(err error) bool {
+	return err != nil && (errors.Is(err, syscall.ECONNREFUSED) || strings.Contains(err.Error(), "connection refused"))
+}
+
+// waitForStore runs try, and while it fails because the store refuses connections, tries again with a growing pause
+// until RegistryWait has passed. A refusal is the one error waited for: nothing was sent, so trying again does nothing
+// twice.
+func waitForStore(ctx context.Context, registry string, try func() error) error {
+	err := try()
+	if !refused(err) {
+		return err
+	}
+	started, pause := time.Now(), RegistryFirstPause
+	for {
+		left := RegistryWait - time.Since(started)
+		if left <= 0 {
+			return fmt.Errorf("the artifact store at %s refused every connection for %s, longer than its nightly collection holds it still, so it is down, not paused: %w",
+				registry, time.Since(started).Round(time.Millisecond), err)
+		}
+		select {
+		case <-ctx.Done():
+			return fmt.Errorf("stopped while waiting for the artifact store at %s: %w (last: %v)", registry, ctx.Err(), err)
+		case <-time.After(min(pause, left)):
+		}
+		pause = min(pause*2, RegistryMostPause)
+		if err = try(); !refused(err) {
+			return err
+		}
+	}
+}
@@ -0,0 +1,151 @@
+package buildstep
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"io"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"git.novox.be/novox/mesh-sdk/go/delivery"
+)
+
+func digestOfBytes(b []byte) string {
+	h := sha256.Sum256(b)
+	return "sha256:" + hex.EncodeToString(h[:])
+}
+
+// registry is an artifact store as far as a blob push needs one: HEAD a blob, start an upload, PUT it with its digest.
+type registry struct {
+	mu    sync.Mutex
+	blobs map[string][]byte
+	calls []string
+	put   int // the status a PUT answers, 201 when 0
+}
+
+func (r *registry) ServeHTTP(w http.ResponseWriter, req *http.Request) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	r.calls = append(r.calls, req.Method+" "+req.URL.Path)
+	switch {
+	case req.Method == http.MethodHead && strings.Contains(req.URL.Path, "/blobs/sha256:"):
+		if _, ok := r.blobs[req.URL.Path[strings.LastIndex(req.URL.Path, "/")+1:]]; ok {
+			w.WriteHeader(http.StatusOK)
+			return
+		}
+		w.WriteHeader(http.StatusNotFound)
+	case req.Method == http.MethodPost && strings.HasSuffix(req.URL.Path, "/blobs/uploads/"):
+		w.Header().Set("Location", strings.TrimSuffix(req.URL.Path, "/")+"/u1?state=x")
+		w.WriteHeader(http.StatusAccepted)
+	case req.Method == http.MethodPut && strings.Contains(req.URL.Path, "/blobs/uploads/u1"):
+		if r.put != 0 {
+			w.WriteHeader(r.put)
+			_, _ = io.WriteString(w, `{"errors":[{"code":"DIGEST_INVALID","message":"provided digest did not match uploaded content"}]}`)
+			return
+		}
+		body, _ := io.ReadAll(req.Body)
+		d := req.URL.Query().Get("digest")
+		if req.URL.Query().Get("state") != "x" || d != digestOfBytes(body) {
+			w.WriteHeader(http.StatusBadRequest)
+			return
+		}
+		r.blobs[d] = body
+		w.WriteHeader(http.StatusCreated)
+	default:
+		w.WriteHeader(http.StatusMethodNotAllowed)
+	}
+}
+
+// A done build's bundle or archive is pushed to the artifact store by its build module (novox/hq ADR 0498 clauses 1
+// and 2): a blob the store already holds is done with no upload; one it lacks is uploaded by digest; its reference is
+// the artifact-store form the builder's key and the controller check.
+func TestABlobIsPushedOnceByItsDigest(t *testing.T) {
+	reg := &registry{blobs: map[string][]byte{}}
+	srv := httptest.NewServer(reg)
+	defer srv.Close()
+	addr := strings.TrimPrefix(srv.URL, "http://")
+	body := []byte("the packed bundle")
+	d := digestOfBytes(body)
+	if err := PushBlob(context.Background(), srv.Client(), addr, "mail/code", d, body); err != nil {
+		t.Fatal(err)
+	}
+	if string(reg.blobs[d]) != string(body) {
+		t.Fatalf("the store holds %q", reg.blobs[d])
+	}
+	reg.calls = nil
+	if err := PushBlob(context.Background(), srv.Client(), addr, "mail/code", d, body); err != nil {
+		t.Fatal(err)
+	}
+	if len(reg.calls) != 1 || !strings.HasPrefix(reg.calls[0], "HEAD ") {
+		t.Fatalf("a blob the store holds was uploaded again: %v", reg.calls)
+	}
+	ref := BlobReference("mail", "code", d)
+	if ref != "artifact-store://mail/code/blobs/"+d {
+		t.Fatalf("BlobReference = %q", ref)
+	}
+	b := delivery.BuilderBuild{State: delivery.BuildDone, Manifest: strings.Repeat("4", 64),
+		Artifacts: []delivery.BuiltArtifact{{Name: "code", Kind: "bundle", Digest: d, Reference: ref}}}
+	if _, err := delivery.EncodeBuilderBuild(b); err != nil {
+		t.Fatalf("the key refuses the reference BlobReference gives: %v", err)
+	}
+}
+
+// A store that refuses the upload fails the push naming the store's own words; bytes that are not the digest are
+// refused before anything is sent.
+func TestAPushTheStoreRefusesSaysTheStoresWords(t *testing.T) {
+	reg := &registry{blobs: map[string][]byte{}, put: http.StatusBadRequest}
+	srv := httptest.NewServer(reg)
+	defer srv.Close()
+	addr := strings.TrimPrefix(srv.URL, "http://")
+	body := []byte("x")
+	err := PushBlob(context.Background(), srv.Client(), addr, "mail/code", digestOfBytes(body), body)
+	if err == nil || !strings.Contains(err.Error(), "provided digest did not match") || !strings.Contains(err.Error(), "400") {
+		t.Fatalf("a refused upload said %v", err)
+	}
+	reg.calls = nil
+	err = PushBlob(context.Background(), srv.Client(), addr, "mail/code", digestOfBytes([]byte("other")), body)
+	if err == nil || len(reg.calls) != 0 {
+		t.Fatalf("bytes that are not their digest: %v, calls %v", err, reg.calls)
+	}
+}
+
+// A store refusing connections (its nightly collection holds it still) is waited out, up to RegistryWait (issue 457).
+func TestAPushWaitsOutAStoreRefusingConnections(t *testing.T) {
+	l, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	addr := l.Addr().String()
+	l.Close() // refusing now
+	reg := &registry{blobs: map[string][]byte{}}
+	go func() {
+		time.Sleep(300 * time.Millisecond)
+		l2, err := net.Listen("tcp", addr)
+		if err != nil {
+			return
+		}
+		_ = http.Serve(l2, reg)
+	}()
+	beforeWait, beforePause := RegistryWait, RegistryFirstPause
+	RegistryWait, RegistryFirstPause = 3*time.Second, 100*time.Millisecond
+	defer func() { RegistryWait, RegistryFirstPause = beforeWait, beforePause }()
+	body := []byte("late")
+	if err := PushBlob(context.Background(), http.DefaultClient, addr, "mail/code", digestOfBytes(body), body); err != nil {
+		t.Fatalf("a store that came back was not waited for: %v", err)
+	}
+
+	l3, _ := net.Listen("tcp", "127.0.0.1:0")
+	gone := l3.Addr().String()
+	l3.Close()
+	RegistryWait = 300 * time.Millisecond
+	err = PushBlob(context.Background(), http.DefaultClient, gone, "mail/code", digestOfBytes(body), body)
+	if err == nil || !strings.Contains(err.Error(), "refused every connection") {
+		t.Fatalf("a store that stayed down said %v", err)
+	}
+}