Merge main: hold a moving machine back, then deliver the rest grants first
Main sends every machine through deliver (memberships before declarations, hq ADR 0218); this branch holds back a machine whose running module's data would move (ADR 0217). Both stand: the held machines are filtered out before delivery, in the named push and its cascade alike.
This commit is contained in:
@@ -0,0 +1,345 @@
|
||||
package artifacts
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// Every archive the store keeps is held by a manifest (novox/hq issue 253, ADR 0189).
|
||||
//
|
||||
// **The store's collector marks only from manifests.** The mesh's store is a stock registry, and
|
||||
// its nightly `registry garbage-collect` walks every manifest in every repository, marks the blobs
|
||||
// those manifests name, and deletes every blob it did not mark. An image is a manifest, so what
|
||||
// the mesh keeps of an image survives. An archive was not: the builder put it in the store as a
|
||||
// bare blob — upload, then `PUT ?digest=` — and nothing in the store names it. To the collector a
|
||||
// bare blob is unreferenced, so the first real collection would have deleted every archive the
|
||||
// mesh holds, kept or not, and every machine pinning a bundle would have found it gone. The
|
||||
// collector runs `--dry-run` until this is true.
|
||||
//
|
||||
// **So each archive gets a holder**: the smallest OCI image manifest that names it — the empty
|
||||
// config, one layer, nothing else — put in the archive's own repository, by digest, untagged. The
|
||||
// collector marks it and so keeps the archive; the sweep lets go of an archive by deleting its
|
||||
// holder first, which is what lets the bytes go at the next collection.
|
||||
//
|
||||
// **Nothing a machine reads changes.** The recorded reference stays
|
||||
// `artifact-store://<module>/<artifact>/blobs/sha256:…`, and machines fetch the blob exactly as
|
||||
// before. The holder is the store's bookkeeping, not a second way to reach anything.
|
||||
//
|
||||
// **Deterministic, so it never needs recording.** The holder is composed from the archive's digest
|
||||
// and size alone, in a fixed field order with no timestamps or annotations, so the sweep can
|
||||
// compute which manifest holds any archive from the reference it already has plus one HEAD for the
|
||||
// size. No schema change, no second record that could disagree with the store.
|
||||
|
||||
const (
|
||||
// mediaManifest is the type a holder is put and asked for as.
|
||||
mediaManifest = "application/vnd.oci.image.manifest.v1+json"
|
||||
// mediaEmpty is the OCI empty descriptor's type: a config that says nothing, for a manifest
|
||||
// whose only purpose is to name its layer.
|
||||
mediaEmpty = "application/vnd.oci.empty.v1+json"
|
||||
// emptyDigest is the digest of `{}`, the empty config's content, fixed by the OCI spec.
|
||||
emptyDigest = "sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a"
|
||||
// mediaArchive is the layer type an archive is held as. Every archive the builder publishes is
|
||||
// `pack`'s gzipped tar, so this is the true type and not a placeholder — and it is a constant,
|
||||
// not read from anywhere, because the holder must be recomputable from the reference alone.
|
||||
mediaArchive = "application/vnd.oci.image.layer.v1.tar+gzip"
|
||||
)
|
||||
|
||||
// emptyConfig is the content emptyDigest names.
|
||||
var emptyConfig = []byte("{}")
|
||||
|
||||
// manifestAccept is what a manifest is asked for as. A registry answers a manifest HEAD only in a
|
||||
// type the caller named, and answers 404 to a bare one for a manifest it holds perfectly well
|
||||
// (measured 2026-09-28; internal/builder/registry.go says how that was found).
|
||||
var manifestAccept = []string{
|
||||
mediaManifest,
|
||||
"application/vnd.docker.distribution.manifest.v2+json",
|
||||
}
|
||||
|
||||
type descriptor struct {
|
||||
MediaType string `json:"mediaType"`
|
||||
Digest string `json:"digest"`
|
||||
Size int64 `json:"size"`
|
||||
}
|
||||
|
||||
type holderManifest struct {
|
||||
SchemaVersion int `json:"schemaVersion"`
|
||||
MediaType string `json:"mediaType"`
|
||||
Config descriptor `json:"config"`
|
||||
Layers []descriptor `json:"layers"`
|
||||
}
|
||||
|
||||
// Holder is the manifest that holds an archive in the store, and its digest.
|
||||
//
|
||||
// A pure function of the archive's digest and size: the same two in give the same bytes out,
|
||||
// always, because `encoding/json` writes a struct's fields in their declared order and there is
|
||||
// nothing here that varies by when or where it was composed.
|
||||
func Holder(digest string, size int64) (body []byte, holder string) {
|
||||
body, err := json.Marshal(holderManifest{
|
||||
SchemaVersion: 2,
|
||||
MediaType: mediaManifest,
|
||||
Config: descriptor{MediaType: mediaEmpty, Digest: emptyDigest, Size: int64(len(emptyConfig))},
|
||||
Layers: []descriptor{{MediaType: mediaArchive, Digest: digest, Size: size}},
|
||||
})
|
||||
if err != nil {
|
||||
// Marshalling a struct of strings and integers cannot fail.
|
||||
panic(err)
|
||||
}
|
||||
sum := sha256.Sum256(body)
|
||||
return body, "sha256:" + hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
// Hold makes sure the store holds this archive by a manifest, and says whether it had to write one.
|
||||
//
|
||||
// Takes a reference as the mesh records it. An image is its own manifest and needs no holder, so
|
||||
// it answers false and nothing is asked. Idempotent: a holder already there is left alone, which
|
||||
// is what lets the sweep run it over every kept archive on every build and so backfill the bare
|
||||
// blobs published before holders existed (novox/hq issue 253).
|
||||
//
|
||||
// Gone when the store does not hold the archive at all: there is nothing to hold, and that is a
|
||||
// fact the caller reports rather than one this invents a remedy for.
|
||||
func (s Store) Hold(ctx context.Context, reference string) (bool, error) {
|
||||
repository, digest, archive, err := s.archive(reference)
|
||||
if err != nil || !archive {
|
||||
return false, err
|
||||
}
|
||||
size, err := s.blobSize(ctx, repository, digest)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return s.HoldBlob(ctx, repository, digest, size)
|
||||
}
|
||||
|
||||
// Held is whether the store holds this archive by its manifest. Asks and changes nothing — the
|
||||
// question an operator needs answered with "none unheld" before the collector is let loose.
|
||||
//
|
||||
// An image answers true: it is its own manifest. An archive the store does not have answers Gone.
|
||||
func (s Store) Held(ctx context.Context, reference string) (bool, error) {
|
||||
repository, digest, archive, err := s.archive(reference)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if !archive {
|
||||
return true, nil
|
||||
}
|
||||
size, err := s.blobSize(ctx, repository, digest)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
_, holder := Holder(digest, size)
|
||||
return s.has(ctx, s.url(repository, "manifests", holder), manifestAccept...)
|
||||
}
|
||||
|
||||
// HoldBlob puts the holder for a blob of this digest and size into its repository, unless it is
|
||||
// there already. Answers whether it wrote one.
|
||||
//
|
||||
// The builder calls this with the size it has just uploaded; the sweep, through Hold, with the size
|
||||
// the store reports. Both arrive at the same holder, which is the point of composing it.
|
||||
func (s Store) HoldBlob(ctx context.Context, repository, digest string, size int64) (bool, error) {
|
||||
if s.Address == "" {
|
||||
return false, fmt.Errorf("this mesh has no artifact store on its network to hold %s/%s in", repository, digest)
|
||||
}
|
||||
body, holder := Holder(digest, size)
|
||||
there, err := s.has(ctx, s.url(repository, "manifests", holder), manifestAccept...)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if there {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// The config must be in the repository before a manifest naming it is accepted: a registry
|
||||
// refuses a manifest whose blobs it cannot find there, which is the property that makes a
|
||||
// holder mean something.
|
||||
if err := s.putBlob(ctx, repository, emptyDigest, emptyConfig); err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
// **By digest, never by tag.** A tag would be one more name to move and one more thing the
|
||||
// collector's `--delete-untagged` would read as meaningful; the mesh names nothing by tag that
|
||||
// it pins by digest, and an untagged manifest is kept by plain collection.
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodPut,
|
||||
s.url(repository, "manifests", holder), bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
request.Header.Set("Content-Type", mediaManifest)
|
||||
response, err := s.client().Do(request)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer response.Body.Close()
|
||||
if response.StatusCode != http.StatusCreated {
|
||||
said, _ := io.ReadAll(io.LimitReader(response.Body, 4096))
|
||||
return false, fmt.Errorf("the artifact store refused to hold %s/%s: %s %s",
|
||||
repository, digest, response.Status, strings.TrimSpace(string(said)))
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// letGoOfHolder deletes the manifest holding an archive, before the archive's own link goes.
|
||||
//
|
||||
// **Holder first.** Deleting the blob link alone leaves a manifest still naming the blob, and the
|
||||
// collector would keep its bytes for ever on the strength of it — the sweep would record the
|
||||
// archive collected while the disk said otherwise. Deleting the holder first and failing before
|
||||
// the link goes leaves an unheld archive that the next sweep still offers, which is safe.
|
||||
//
|
||||
// A store that no longer has the blob answers Gone: without its size the holder cannot be named,
|
||||
// and without the blob there is nothing left for a holder to keep. A store that never had a holder
|
||||
// for it — an archive published before holders, never backfilled — answers 404 to the delete, and
|
||||
// that is the outcome wanted.
|
||||
func (s Store) letGoOfHolder(ctx context.Context, repository, digest string) error {
|
||||
size, err := s.blobSize(ctx, repository, digest)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, holder := Holder(digest, size)
|
||||
err = s.remove(ctx, s.url(repository, "manifests", holder), repository+"/manifests/"+holder)
|
||||
if err == Gone {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// archive reads a recorded reference into its repository and digest, and whether it is an archive
|
||||
// at all. Refuses as ErrNotOurs anything the mesh did not put in its own store.
|
||||
func (s Store) archive(reference string) (repository, digest string, archive bool, err error) {
|
||||
path, kept := catalogue.InArtifactStore(reference)
|
||||
if !kept {
|
||||
return "", "", false, fmt.Errorf("%w: %s", ErrNotOurs, reference)
|
||||
}
|
||||
if s.Address == "" {
|
||||
return "", "", false, fmt.Errorf("this mesh has no artifact store on its network to ask about %s", reference)
|
||||
}
|
||||
repository, kind, digest, err := split(path)
|
||||
if err != nil {
|
||||
return "", "", false, err
|
||||
}
|
||||
return repository, digest, kind == "blobs", nil
|
||||
}
|
||||
|
||||
// blobSize is how large the store says a blob is; Gone when it does not have it.
|
||||
func (s Store) blobSize(ctx context.Context, repository, digest string) (int64, error) {
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodHead, s.url(repository, "blobs", digest), nil)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
response, err := s.client().Do(request)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("cannot reach the artifact store at %s: %w", s.Address, err)
|
||||
}
|
||||
defer response.Body.Close()
|
||||
switch response.StatusCode {
|
||||
case http.StatusOK:
|
||||
case http.StatusNotFound:
|
||||
return 0, Gone
|
||||
default:
|
||||
return 0, fmt.Errorf("the artifact store answered %s for %s/blobs/%s", response.Status, repository, digest)
|
||||
}
|
||||
// Read from the header rather than ContentLength: a HEAD's ContentLength is what the response
|
||||
// says it would have sent, which Go reports faithfully, but a proxy in between is free to drop
|
||||
// it, and the header is what the registry itself wrote.
|
||||
if length := response.Header.Get("Content-Length"); length != "" {
|
||||
if n, err := strconv.ParseInt(length, 10, 64); err == nil && n >= 0 {
|
||||
return n, nil
|
||||
}
|
||||
}
|
||||
if response.ContentLength >= 0 {
|
||||
return response.ContentLength, nil
|
||||
}
|
||||
return 0, fmt.Errorf("the artifact store holds %s/blobs/%s and will not say how large it is", repository, digest)
|
||||
}
|
||||
|
||||
// putBlob uploads a small blob unless the repository already has it: ask where, then put it there
|
||||
// naming the digest — the registry's own two steps, the same the builder takes for an archive.
|
||||
func (s Store) putBlob(ctx context.Context, repository, digest string, body []byte) error {
|
||||
if there, err := s.has(ctx, s.url(repository, "blobs", digest)); err != nil {
|
||||
return err
|
||||
} else if there {
|
||||
return nil
|
||||
}
|
||||
start, err := http.NewRequestWithContext(ctx, http.MethodPost,
|
||||
"http://"+s.Address+"/v2/"+repository+"/blobs/uploads/", nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
begun, err := s.client().Do(start)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot start an upload to %s: %w", repository, err)
|
||||
}
|
||||
begun.Body.Close()
|
||||
if begun.StatusCode != http.StatusAccepted {
|
||||
return fmt.Errorf("the artifact store answered %s when asked where to put a blob in %s", begun.Status, repository)
|
||||
}
|
||||
where := begun.Header.Get("Location")
|
||||
if where == "" {
|
||||
return fmt.Errorf("the artifact store accepted an upload to %s and said nowhere to put it", repository)
|
||||
}
|
||||
if strings.HasPrefix(where, "/") {
|
||||
where = "http://" + s.Address + where
|
||||
}
|
||||
separator := "?"
|
||||
if strings.Contains(where, "?") {
|
||||
separator = "&"
|
||||
}
|
||||
put, err := http.NewRequestWithContext(ctx, http.MethodPut, where+separator+"digest="+digest, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
put.Header.Set("Content-Type", "application/octet-stream")
|
||||
done, err := s.client().Do(put)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer done.Body.Close()
|
||||
if done.StatusCode != http.StatusCreated {
|
||||
said, _ := io.ReadAll(io.LimitReader(done.Body, 4096))
|
||||
return fmt.Errorf("the artifact store refused a blob in %s: %s %s", repository, done.Status, strings.TrimSpace(string(said)))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// has is whether the store answers 200 for a HEAD at that URL.
|
||||
func (s Store) has(ctx context.Context, url string, accept ...string) (bool, error) {
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodHead, url, nil)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
for _, media := range accept {
|
||||
request.Header.Add("Accept", media)
|
||||
}
|
||||
response, err := s.client().Do(request)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("cannot reach the artifact store at %s: %w", s.Address, err)
|
||||
}
|
||||
defer response.Body.Close()
|
||||
switch response.StatusCode {
|
||||
case http.StatusOK:
|
||||
return true, nil
|
||||
case http.StatusNotFound:
|
||||
return false, nil
|
||||
default:
|
||||
return false, fmt.Errorf("the artifact store answered %s for %s", response.Status, url)
|
||||
}
|
||||
}
|
||||
|
||||
func (s Store) url(repository, kind, digest string) string {
|
||||
return "http://" + s.Address + "/v2/" + repository + "/" + kind + "/" + digest
|
||||
}
|
||||
|
||||
func (s Store) client() *http.Client {
|
||||
if s.HTTP != nil {
|
||||
return s.HTTP
|
||||
}
|
||||
return &http.Client{Timeout: 30 * time.Second}
|
||||
}
|
||||
@@ -0,0 +1,268 @@
|
||||
package artifacts
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// Every kept archive is held by a manifest (novox/hq issue 253, ADR 0189).
|
||||
//
|
||||
// Against an in-memory registry that keeps blobs and manifests per repository and refuses what a
|
||||
// registry refuses — a blob whose digest does not match, a manifest whose digest does not match
|
||||
// or whose blobs the repository does not have, a manifest asked for without an Accept naming its
|
||||
// type. What is asserted is this side's decisions; the live test below asserts the registry's.
|
||||
|
||||
type memRegistry struct {
|
||||
mu sync.Mutex
|
||||
blobs map[string][]byte // repository + "@" + digest
|
||||
manifests map[string][]byte // repository + "@" + digest
|
||||
writes []string // every PUT and DELETE, as "METHOD path"
|
||||
}
|
||||
|
||||
func digestOf(body []byte) string {
|
||||
sum := sha256.Sum256(body)
|
||||
return "sha256:" + hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
func (m *memRegistry) serve(t *testing.T) Store {
|
||||
t.Helper()
|
||||
m.blobs = map[string][]byte{}
|
||||
m.manifests = map[string][]byte{}
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
path := strings.TrimPrefix(r.URL.Path, "/v2/")
|
||||
if r.Method == http.MethodPut || r.Method == http.MethodDelete {
|
||||
m.writes = append(m.writes, r.Method+" "+r.URL.Path)
|
||||
}
|
||||
switch {
|
||||
case r.Method == http.MethodPost && strings.HasSuffix(path, "/blobs/uploads/"):
|
||||
repository := strings.TrimSuffix(path, "/blobs/uploads/")
|
||||
w.Header().Set("Location", "/upload/"+repository+"?state=x")
|
||||
w.WriteHeader(http.StatusAccepted)
|
||||
case r.Method == http.MethodPut && strings.HasPrefix(r.URL.Path, "/upload/"):
|
||||
repository := strings.TrimPrefix(r.URL.Path, "/upload/")
|
||||
body, _ := io.ReadAll(r.Body)
|
||||
digest := r.URL.Query().Get("digest")
|
||||
if digest != digestOf(body) {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
m.blobs[repository+"@"+digest] = body
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
case strings.Contains(path, "/blobs/"):
|
||||
repository, digest, _ := strings.Cut(path, "/blobs/")
|
||||
key := repository + "@" + digest
|
||||
body, ok := m.blobs[key]
|
||||
if !ok {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
switch r.Method {
|
||||
case http.MethodHead:
|
||||
w.Header().Set("Content-Length", strconv.Itoa(len(body)))
|
||||
w.WriteHeader(http.StatusOK)
|
||||
case http.MethodDelete:
|
||||
delete(m.blobs, key)
|
||||
w.WriteHeader(http.StatusAccepted)
|
||||
default:
|
||||
w.WriteHeader(http.StatusMethodNotAllowed)
|
||||
}
|
||||
case strings.Contains(path, "/manifests/"):
|
||||
repository, digest, _ := strings.Cut(path, "/manifests/")
|
||||
key := repository + "@" + digest
|
||||
switch r.Method {
|
||||
case http.MethodHead:
|
||||
if _, ok := m.manifests[key]; !ok || !strings.Contains(r.Header.Get("Accept"), mediaManifest) {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
case http.MethodPut:
|
||||
body, _ := io.ReadAll(r.Body)
|
||||
if digest != digestOf(body) {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
var named holderManifest
|
||||
if err := json.Unmarshal(body, &named); err != nil {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
for _, d := range append([]descriptor{named.Config}, named.Layers...) {
|
||||
if _, ok := m.blobs[repository+"@"+d.Digest]; !ok {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
fmt.Fprintf(w, "MANIFEST_BLOB_UNKNOWN %s", d.Digest)
|
||||
return
|
||||
}
|
||||
}
|
||||
m.manifests[key] = body
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
case http.MethodDelete:
|
||||
if _, ok := m.manifests[key]; !ok {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
delete(m.manifests, key)
|
||||
w.WriteHeader(http.StatusAccepted)
|
||||
}
|
||||
default:
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
return Store{Address: strings.TrimPrefix(server.URL, "http://")}
|
||||
}
|
||||
|
||||
// bare puts an archive in the store the way the builder did before holders: a blob, nothing more.
|
||||
func (m *memRegistry) bare(repository string, body []byte) string {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
digest := digestOf(body)
|
||||
m.blobs[repository+"@"+digest] = body
|
||||
return catalogue.ArtifactStoreScheme + repository + "/blobs/" + digest
|
||||
}
|
||||
|
||||
func TestTheHolderIsComposedFromTheDigestAndSizeAlone(t *testing.T) {
|
||||
// The sweep must arrive at the very manifest the builder wrote, with nothing recorded between
|
||||
// them. Same inputs, same bytes — and a different size is a different holder, so a holder can
|
||||
// never be mistaken for one of a different blob.
|
||||
digest := "sha256:" + strings.Repeat("a", 64)
|
||||
one, first := Holder(digest, 42)
|
||||
two, second := Holder(digest, 42)
|
||||
if !bytes.Equal(one, two) || first != second {
|
||||
t.Fatalf("the same archive composed two holders:\n%s\n%s", one, two)
|
||||
}
|
||||
if _, other := Holder(digest, 43); other == first {
|
||||
t.Fatal("a different size composed the same holder")
|
||||
}
|
||||
want := `{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json",` +
|
||||
`"config":{"mediaType":"application/vnd.oci.empty.v1+json",` +
|
||||
`"digest":"sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a","size":2},` +
|
||||
`"layers":[{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"` + digest + `","size":42}]}`
|
||||
if string(one) != want {
|
||||
t.Fatalf("the holder is\n%s\nwant\n%s", one, want)
|
||||
}
|
||||
if digestOf(emptyConfig) != emptyDigest {
|
||||
t.Fatalf("the empty config's digest is %s, not %s", digestOf(emptyConfig), emptyDigest)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHoldBackfillsABareArchiveAndIsIdempotent(t *testing.T) {
|
||||
// The archives published before this have no holder. Hold, run over every kept archive on
|
||||
// every sweep, writes one the first time and nothing after.
|
||||
m := &memRegistry{}
|
||||
store := m.serve(t)
|
||||
ctx := context.Background()
|
||||
body := []byte("a theme")
|
||||
reference := m.bare("shell/config", body)
|
||||
|
||||
if held, err := store.Held(ctx, reference); err != nil || held {
|
||||
t.Fatalf("a bare blob reads as held=%v (%v)", held, err)
|
||||
}
|
||||
wrote, err := store.Hold(ctx, reference)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !wrote {
|
||||
t.Fatal("holding a bare archive wrote nothing")
|
||||
}
|
||||
_, holder := Holder(digestOf(body), int64(len(body)))
|
||||
if _, ok := m.manifests["shell/config@"+holder]; !ok {
|
||||
t.Fatalf("the store holds manifests %v; want %s", m.manifests, holder)
|
||||
}
|
||||
if held, err := store.Held(ctx, reference); err != nil || !held {
|
||||
t.Fatalf("after holding, held=%v (%v)", held, err)
|
||||
}
|
||||
|
||||
writes := len(m.writes)
|
||||
wrote, err = store.Hold(ctx, reference)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if wrote || len(m.writes) != writes {
|
||||
t.Fatalf("holding again wrote %v", m.writes[writes:])
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnImageNeedsNoHolderAndAMissingArchiveIsGone(t *testing.T) {
|
||||
m := &memRegistry{}
|
||||
store := m.serve(t)
|
||||
ctx := context.Background()
|
||||
|
||||
// An image is its own manifest: nothing is asked.
|
||||
wrote, err := store.Hold(ctx, catalogue.ArtifactStoreScheme+"web/app@sha256:"+strings.Repeat("b", 64))
|
||||
if err != nil || wrote || len(m.writes) != 0 {
|
||||
t.Fatalf("holding an image wrote=%v err=%v writes=%v", wrote, err, m.writes)
|
||||
}
|
||||
// An archive the store does not have is a fact to report, not something to invent a holder for.
|
||||
_, err = store.Hold(ctx, catalogue.ArtifactStoreScheme+"web/config/blobs/sha256:"+strings.Repeat("c", 64))
|
||||
if !errors.Is(err, Gone) {
|
||||
t.Fatalf("holding a missing archive answered %v, want Gone", err)
|
||||
}
|
||||
// And a reference that is not the mesh's is refused as such.
|
||||
if _, err := store.Hold(ctx, "docker.io/library/registry@sha256:abc"); !errors.Is(err, ErrNotOurs) {
|
||||
t.Fatalf("holding a vendor's image answered %v, want ErrNotOurs", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLettingGoOfAnArchiveDeletesItsHolderFirst(t *testing.T) {
|
||||
// A holder left behind would keep the bytes through every collection while the record said
|
||||
// collected; the link deleted first and the holder failing after would be that exactly.
|
||||
m := &memRegistry{}
|
||||
store := m.serve(t)
|
||||
ctx := context.Background()
|
||||
body := []byte("an old theme")
|
||||
reference := m.bare("shell/config", body)
|
||||
if _, err := store.Hold(ctx, reference); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m.writes = nil
|
||||
|
||||
if err := store.LetGo(ctx, reference); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, holder := Holder(digestOf(body), int64(len(body)))
|
||||
want := []string{
|
||||
"DELETE /v2/shell/config/manifests/" + holder,
|
||||
"DELETE /v2/shell/config/blobs/" + digestOf(body),
|
||||
}
|
||||
if strings.Join(m.writes, "\n") != strings.Join(want, "\n") {
|
||||
t.Fatalf("the store was asked\n%s\nwant\n%s", strings.Join(m.writes, "\n"), strings.Join(want, "\n"))
|
||||
}
|
||||
if len(m.manifests) != 0 {
|
||||
t.Fatalf("a holder survived: %v", m.manifests)
|
||||
}
|
||||
// Asked again, the archive is already gone, which is the outcome wanted.
|
||||
if err := store.LetGo(ctx, reference); !errors.Is(err, Gone) {
|
||||
t.Fatalf("letting go twice answered %v, want Gone", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLettingGoOfAnUnheldArchiveStillDeletesIt(t *testing.T) {
|
||||
// An archive published before holders and let go of before any sweep held it: the holder's
|
||||
// delete answers 404, which is the outcome wanted, and the blob still goes.
|
||||
m := &memRegistry{}
|
||||
store := m.serve(t)
|
||||
reference := m.bare("shell/config", []byte("never held"))
|
||||
if err := store.LetGo(context.Background(), reference); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(m.blobs) != 0 {
|
||||
t.Fatalf("the blob survived: %v", m.blobs)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,130 @@
|
||||
package artifacts
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// The registry's own collector keeps a held archive and takes a bare one (novox/hq issue 253,
|
||||
// ADR 0189).
|
||||
//
|
||||
// Everything else here is asserted against a fake, which can only say what this side asks. This is
|
||||
// the one question a fake cannot answer — what `registry garbage-collect` actually does with what
|
||||
// this side wrote — and it is the whole of whether the store's nightly step may stop being a dry
|
||||
// run. Against the very image the mesh's store runs:
|
||||
//
|
||||
// docker run -d --rm --name mesh-controller-registry -p 15000:5000 \
|
||||
// -e REGISTRY_STORAGE_DELETE_ENABLED=true registry:2.8.3
|
||||
// MESH_TEST_REGISTRY=127.0.0.1:15000 MESH_TEST_REGISTRY_CONTAINER=mesh-controller-registry \
|
||||
// go test -run Live ./internal/artifacts/
|
||||
// docker stop mesh-controller-registry
|
||||
//
|
||||
// Skipped without both variables: it needs a registry it may write to and collect, and a container
|
||||
// to run the collector in.
|
||||
func TestLiveTheRegistrysCollectorKeepsWhatIsHeldAndTakesWhatIsNot(t *testing.T) {
|
||||
address := os.Getenv("MESH_TEST_REGISTRY")
|
||||
container := os.Getenv("MESH_TEST_REGISTRY_CONTAINER")
|
||||
if address == "" || container == "" {
|
||||
t.Skip("no MESH_TEST_REGISTRY / MESH_TEST_REGISTRY_CONTAINER; see this test's comment for the registry to raise")
|
||||
}
|
||||
ctx := context.Background()
|
||||
store := Store{Address: address}
|
||||
run := time.Now().UnixNano()
|
||||
|
||||
// Four archives in four repositories, each a different story. Distinct bytes per run, so a
|
||||
// registry reused across runs cannot answer for an earlier one.
|
||||
put := func(name string) (repository, digest string, body []byte) {
|
||||
repository = fmt.Sprintf("live-%d/%s", run, name)
|
||||
body = []byte(fmt.Sprintf("%s archive of run %d", name, run))
|
||||
digest = digestOf(body)
|
||||
if err := store.putBlob(ctx, repository, digest, body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return repository, digest, body
|
||||
}
|
||||
reference := func(repository, digest string) string {
|
||||
return catalogue.ArtifactStoreScheme + repository + "/blobs/" + digest
|
||||
}
|
||||
|
||||
// Published held — what PublishArchive now does.
|
||||
heldRepo, heldDigest, heldBody := put("held")
|
||||
if _, err := store.HoldBlob(ctx, heldRepo, heldDigest, int64(len(heldBody))); err != nil {
|
||||
t.Fatalf("the registry refused a holder: %v", err)
|
||||
}
|
||||
// Published bare, as before, and never held: what the collector must take.
|
||||
_, bareDigest, _ := put("bare")
|
||||
// Published bare and then held by the sweep: the backfill.
|
||||
backRepo, backDigest, backBody := put("backfilled")
|
||||
if wrote, err := store.Hold(ctx, reference(backRepo, backDigest)); err != nil || !wrote {
|
||||
t.Fatalf("backfilling wrote=%v: %v", wrote, err)
|
||||
}
|
||||
if held, err := store.Held(ctx, reference(backRepo, backDigest)); err != nil || !held {
|
||||
t.Fatalf("after backfilling, held=%v: %v", held, err)
|
||||
}
|
||||
// Held, and then let go of by the sweep: holder first, then the link.
|
||||
goneRepo, goneDigest, _ := put("let-go")
|
||||
if _, err := store.Hold(ctx, reference(goneRepo, goneDigest)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.LetGo(ctx, reference(goneRepo, goneDigest)); err != nil {
|
||||
t.Fatalf("letting go of a held archive: %v", err)
|
||||
}
|
||||
|
||||
collected, err := exec.CommandContext(ctx, "docker", "exec", container,
|
||||
"registry", "garbage-collect", "/etc/docker/registry/config.yml").CombinedOutput()
|
||||
if err != nil {
|
||||
t.Fatalf("the collector failed: %v\n%s", err, collected)
|
||||
}
|
||||
t.Logf("the collector said:\n%s", lastLines(string(collected), 12))
|
||||
|
||||
// What is asserted is the bytes on the store's disk, not what the running server answers: the
|
||||
// server caches blob descriptors in memory and can answer for a blob the collector removed.
|
||||
onDisk := func(digest string) bool {
|
||||
hex := strings.TrimPrefix(digest, "sha256:")
|
||||
path := "/var/lib/registry/docker/registry/v2/blobs/sha256/" + hex[:2] + "/" + hex + "/data"
|
||||
return exec.CommandContext(ctx, "docker", "exec", container, "test", "-f", path).Run() == nil
|
||||
}
|
||||
if !onDisk(heldDigest) {
|
||||
t.Error("the collector took an archive published held")
|
||||
}
|
||||
if !onDisk(backDigest) {
|
||||
t.Error("the collector took an archive the sweep backfilled a holder for")
|
||||
}
|
||||
if onDisk(bareDigest) {
|
||||
t.Error("the collector kept a bare archive — then the holders prove nothing, and this test is wrong")
|
||||
}
|
||||
if onDisk(goneDigest) {
|
||||
t.Error("the collector kept an archive the sweep let go of: its holder outlived its link")
|
||||
}
|
||||
// And what survived is still fetched exactly as machines fetch it: the blob, by digest.
|
||||
for repository, want := range map[string][]byte{heldRepo: heldBody, backRepo: backBody} {
|
||||
digest := digestOf(want)
|
||||
response, err := http.Get(catalogue.Routed(reference(repository, digest), address))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, _ := io.ReadAll(response.Body)
|
||||
response.Body.Close()
|
||||
if response.StatusCode != http.StatusOK || !bytes.Equal(got, want) {
|
||||
t.Errorf("%s answered %s with %q after collection", repository, response.Status, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func lastLines(s string, n int) string {
|
||||
lines := strings.Split(strings.TrimSpace(s), "\n")
|
||||
if len(lines) > n {
|
||||
lines = lines[len(lines)-n:]
|
||||
}
|
||||
return strings.Join(lines, "\n")
|
||||
}
|
||||
+20
-10
@@ -1,7 +1,8 @@
|
||||
// Package artifacts speaks to the mesh's artifact store over its own door.
|
||||
//
|
||||
// Only what the mesh needs that nothing else does: letting go of something it put there
|
||||
// (novox/hq ADR 0189, issue 108). Pushing is the builder's, through the container runtime; reading
|
||||
// (novox/hq ADR 0189, issue 108), and holding every archive it keeps by a manifest so the store's
|
||||
// own collector does not take it (novox/hq issue 253). Pushing is the builder's, through the container runtime; reading
|
||||
// is every machine's, through its runtime. This is the one operation that belongs to the thing
|
||||
// holding the records, because it is the only one that is a decision rather than a transfer.
|
||||
package artifacts
|
||||
@@ -12,7 +13,6 @@ import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
@@ -43,6 +43,9 @@ var ErrNotOurs = errors.New("not a reference into the mesh's artifact store")
|
||||
// an image, `…/blobs/sha256:…` for an archive — because that is the identity every record uses,
|
||||
// and composes the address here at the moment of use.
|
||||
//
|
||||
// An archive is let go of in two deletes, its holder manifest and then the blob's link (novox/hq
|
||||
// issue 253); an image in one.
|
||||
//
|
||||
// Returns Gone when the store answers that it does not have it. That is not a failure: the sweep
|
||||
// wants the artifact absent, and it is. It is distinguished from success only so a caller can say
|
||||
// which of the two happened.
|
||||
@@ -66,17 +69,24 @@ func (s Store) LetGo(ctx context.Context, reference string) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
url := "http://" + s.Address + "/v2/" + repository + "/" + kind + "/" + digest
|
||||
if kind == "blobs" {
|
||||
// **An archive's holder goes before the archive** (novox/hq issue 253): a manifest left
|
||||
// naming the blob would keep its bytes through every collection while the record said
|
||||
// collected. Gone here means the store has no such blob, so there is nothing to let go.
|
||||
if err := s.letGoOfHolder(ctx, repository, digest); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return s.remove(ctx, s.url(repository, kind, digest), reference)
|
||||
}
|
||||
|
||||
// remove asks the store to delete what is at url. Gone when it has no such thing.
|
||||
func (s Store) remove(ctx context.Context, url, what string) error {
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodDelete, url, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
client := s.HTTP
|
||||
if client == nil {
|
||||
client = &http.Client{Timeout: 30 * time.Second}
|
||||
}
|
||||
response, err := client.Do(request)
|
||||
response, err := s.client().Do(request)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -92,9 +102,9 @@ func (s Store) LetGo(ctx context.Context, reference string) error {
|
||||
return fmt.Errorf(
|
||||
"the artifact store refuses deletion: its server was started without it enabled "+
|
||||
"(REGISTRY_STORAGE_DELETE_ENABLED), so nothing can be collected until the store "+
|
||||
"module is applied again (novox/hq ADR 0189). Asking about %s", reference)
|
||||
"module is applied again (novox/hq ADR 0189). Asking about %s", what)
|
||||
default:
|
||||
return fmt.Errorf("the artifact store answered %s for %s", response.Status, reference)
|
||||
return fmt.Errorf("the artifact store answered %s for %s", response.Status, what)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -20,6 +20,17 @@ func fakeStore(t *testing.T, answer int) (Store, *[]string) {
|
||||
t.Helper()
|
||||
var asked []string
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method == http.MethodHead && strings.Contains(r.URL.Path, "/blobs/") {
|
||||
// An archive's size, asked so its holder can be named (novox/hq issue 253). A store
|
||||
// that does not have the thing does not have its blob either.
|
||||
if answer == http.StatusNotFound {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Length", "7")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
return
|
||||
}
|
||||
if r.Method != http.MethodDelete {
|
||||
t.Errorf("the store was asked %s %s; collecting is a delete", r.Method, r.URL.Path)
|
||||
}
|
||||
@@ -45,8 +56,14 @@ func TestAnImageAndAnArchiveAreAskedForAtTheirOwnEndpoints(t *testing.T) {
|
||||
if err := store.LetGo(ctx, archive); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := []string{"/v2/web/app/manifests/sha256:abc123", "/v2/web/config/blobs/sha256:def456"}
|
||||
if len(*asked) != 2 || (*asked)[0] != want[0] || (*asked)[1] != want[1] {
|
||||
// The archive's holder goes first, then the archive (novox/hq issue 253).
|
||||
_, holder := Holder("sha256:def456", 7)
|
||||
want := []string{
|
||||
"/v2/web/app/manifests/sha256:abc123",
|
||||
"/v2/web/config/manifests/" + holder,
|
||||
"/v2/web/config/blobs/sha256:def456",
|
||||
}
|
||||
if strings.Join(*asked, " ") != strings.Join(want, " ") {
|
||||
t.Fatalf("the store was asked %v; want %v", *asked, want)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
package broker
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
)
|
||||
|
||||
// A consumer that reacts to announcements, made on a stream that keeps a week of them, starts from now:
|
||||
// made from the start it replays every merge and build of that week (novox/hq issue 248). And a reset
|
||||
// re-makes a stuck one from now, its configuration otherwise kept, and refuses a work queue.
|
||||
//
|
||||
// docker run -d --rm --name t -p 14231:4222 nats:2.10-alpine -js
|
||||
// MESH_TEST_NATS=nats://127.0.0.1:14231 go test ./internal/broker/ -run TestAConsumerMadeFromNow
|
||||
func TestAConsumerMadeFromNowNeverReplaysTheStreamsHistory(t *testing.T) {
|
||||
url := os.Getenv("MESH_TEST_NATS")
|
||||
if url == "" {
|
||||
t.Skip("MESH_TEST_NATS unset")
|
||||
}
|
||||
js, err := Dial(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer js.Close()
|
||||
const stream = "HISTORY_TEST"
|
||||
_ = js.js.DeleteStream(stream)
|
||||
if _, err := js.js.AddStream(&nats.StreamConfig{Name: stream, Subjects: []string{"history.>"}, Storage: nats.MemoryStorage}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = js.js.DeleteStream(stream) }()
|
||||
for i := 0; i < 5; i++ {
|
||||
if _, err := js.js.Publish("history.merged", []byte(fmt.Sprint(i))); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
fresh := Consumer{Name: "fresh", Stream: stream, Filters: []string{"history.merged"}, Push: true,
|
||||
AckWaitSeconds: 30, MaxDeliver: 5, MaxAckPending: 1, FromNow: true}
|
||||
if err := js.EnsureConsumer(fresh); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
info, _ := js.js.ConsumerInfo(stream, "fresh")
|
||||
if info.NumPending != 0 || info.Config.DeliverPolicy != nats.DeliverNewPolicy {
|
||||
t.Fatalf("a consumer made from now holds %d of the stream's past (policy %v)", info.NumPending, info.Config.DeliverPolicy)
|
||||
}
|
||||
_, _ = js.js.Publish("history.merged", []byte("new"))
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
if info, _ = js.js.ConsumerInfo(stream, "fresh"); info.NumPending != 1 {
|
||||
t.Fatalf("a new announcement is not pending: %d", info.NumPending)
|
||||
}
|
||||
// Asserted again, it keeps where it is.
|
||||
if err := js.EnsureConsumer(fresh); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// One made from the start, as the server's default makes it, and stuck behind its history.
|
||||
old := fresh
|
||||
old.Name, old.FromNow = "old", false
|
||||
if err := js.EnsureConsumer(old); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if info, _ = js.js.ConsumerInfo(stream, "old"); info.NumPending != 6 {
|
||||
t.Fatalf("the default should replay all six: %d", info.NumPending)
|
||||
}
|
||||
before, after, err := js.ResetConsumer(stream, "old")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
info, _ = js.js.ConsumerInfo(stream, "old")
|
||||
if before.Pending != 6 || after.Pending != 0 || info.Config.MaxAckPending != 1 || info.Config.MaxDeliver != 5 ||
|
||||
info.Config.AckWait != 30*time.Second || info.Config.DeliverSubject == "" {
|
||||
t.Fatalf("before %+v after %+v config %+v", before, after, info.Config)
|
||||
}
|
||||
|
||||
// Never a work queue: what is pending there is work.
|
||||
const queue = "QUEUE_TEST"
|
||||
_ = js.js.DeleteStream(queue)
|
||||
if _, err := js.js.AddStream(&nats.StreamConfig{Name: queue, Subjects: []string{"queue.>"}, Retention: nats.WorkQueuePolicy, Storage: nats.MemoryStorage}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = js.js.DeleteStream(queue) }()
|
||||
if _, err := js.js.AddConsumer(queue, &nats.ConsumerConfig{Durable: "w", AckPolicy: nats.AckExplicitPolicy}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, err := js.ResetConsumer(queue, "w"); err == nil {
|
||||
t.Fatal("a work queue's consumer was reset")
|
||||
}
|
||||
}
|
||||
@@ -47,7 +47,13 @@ type Consumer struct {
|
||||
// and a merge that came back rebuilt what it had just built, five times over on 2026-09-30.
|
||||
// With one outstanding, the server holds the rest, and the heartbeat is keeping the message.
|
||||
MaxAckPending int
|
||||
Why string
|
||||
// FromNow makes a consumer that does not exist yet start at the stream's end rather than its
|
||||
// beginning (novox/hq issue 248). For a consumer that reacts to announcements — a merge, a build's
|
||||
// outcome — on a stream that keeps a week of them: the server's default, everything the stream
|
||||
// holds, replays every merge and every build of that week as if it had just happened. A consumer
|
||||
// that exists keeps where it is, whatever this says; only its making is decided here.
|
||||
FromNow bool
|
||||
Why string
|
||||
}
|
||||
|
||||
// seatStreamName is the stream holding a seat's inbound work. Named after the seat rather than
|
||||
|
||||
@@ -308,6 +308,9 @@ func (j *JetStream) EnsureConsumer(c Consumer) error {
|
||||
}
|
||||
return nil
|
||||
case errors.Is(err, nats.ErrConsumerNotFound):
|
||||
if c.FromNow {
|
||||
want.DeliverPolicy = nats.DeliverNewPolicy
|
||||
}
|
||||
if _, err := j.js.AddConsumer(c.Stream, want); err != nil {
|
||||
return fmt.Errorf("creating consumer %s on %s: %w", c.Name, c.Stream, err)
|
||||
}
|
||||
@@ -317,6 +320,51 @@ func (j *JetStream) EnsureConsumer(c Consumer) error {
|
||||
}
|
||||
}
|
||||
|
||||
// ConsumerState is what a reset says about a consumer, before and after.
|
||||
type ConsumerState struct {
|
||||
DeliverPolicy string
|
||||
Delivered uint64
|
||||
AckFloor uint64
|
||||
Pending uint64
|
||||
AckPending int
|
||||
}
|
||||
|
||||
func stateOf(info *nats.ConsumerInfo) ConsumerState {
|
||||
policy, _ := info.Config.DeliverPolicy.MarshalJSON()
|
||||
return ConsumerState{DeliverPolicy: strings.Trim(string(policy), `"`), Delivered: info.Delivered.Stream,
|
||||
AckFloor: info.AckFloor.Stream, Pending: info.NumPending, AckPending: info.NumAckPending}
|
||||
}
|
||||
|
||||
// ResetConsumer re-makes a consumer to start from now, its configuration otherwise unchanged (novox/hq
|
||||
// issue 248): what it had not yet delivered or acknowledged is dropped, which is the point — on a stream
|
||||
// that keeps history, a consumer replaying a week of announcements does nothing anyone wants. Refused on
|
||||
// a work queue, where what is pending is work nobody else will do.
|
||||
func (j *JetStream) ResetConsumer(stream, name string) (before, after ConsumerState, err error) {
|
||||
info, err := j.js.StreamInfo(stream)
|
||||
if err != nil {
|
||||
return before, after, fmt.Errorf("asking about stream %s: %w", stream, err)
|
||||
}
|
||||
if info.Config.Retention == nats.WorkQueuePolicy {
|
||||
return before, after, fmt.Errorf("%s is a work queue: what its consumer has pending is work, and a reset would drop it", stream)
|
||||
}
|
||||
have, err := j.js.ConsumerInfo(stream, name)
|
||||
if err != nil {
|
||||
return before, after, fmt.Errorf("asking about consumer %s on %s: %w", name, stream, err)
|
||||
}
|
||||
before = stateOf(have)
|
||||
want := have.Config
|
||||
want.DeliverPolicy = nats.DeliverNewPolicy
|
||||
want.OptStartSeq, want.OptStartTime = 0, nil
|
||||
if err := j.js.DeleteConsumer(stream, name); err != nil {
|
||||
return before, after, fmt.Errorf("removing consumer %s on %s: %w", name, stream, err)
|
||||
}
|
||||
made, err := j.js.AddConsumer(stream, &want)
|
||||
if err != nil {
|
||||
return before, after, fmt.Errorf("re-making consumer %s on %s — it is gone until the controller asserts it at its next start: %w", name, stream, err)
|
||||
}
|
||||
return before, stateOf(made), nil
|
||||
}
|
||||
|
||||
func retentionOf(r Retention) nats.RetentionPolicy {
|
||||
switch r {
|
||||
case RetentionWorkQueue:
|
||||
|
||||
@@ -270,6 +270,7 @@ func MeshConsumers() []Consumer {
|
||||
// announcement handed over behind it must wait on the server, not time out on the
|
||||
// client and come back to be acted on again.
|
||||
MaxAckPending: 1,
|
||||
FromNow: true,
|
||||
Why: "the two events the mesh's own controller reacts to, one at a time; after " +
|
||||
"max-deliver it dead-letters, because an announcement it cannot act on will not " +
|
||||
"become actionable",
|
||||
|
||||
@@ -7,6 +7,8 @@ import (
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/artifacts"
|
||||
)
|
||||
|
||||
// Where built artifacts go.
|
||||
@@ -66,22 +68,32 @@ func (r Registry) PublishImage(ctx context.Context, localTag, repository string)
|
||||
return pinned, nil
|
||||
}
|
||||
|
||||
// PublishArchive stores bytes as a blob and returns where to fetch them from.
|
||||
// PublishArchive stores bytes as a blob, holds it by a manifest, and returns where to fetch them
|
||||
// from.
|
||||
//
|
||||
// Two steps, which is the registry's own protocol: ask for somewhere to put it, then put it there
|
||||
// naming the digest. The registry verifies the digest itself, so a blob that arrived corrupted is
|
||||
// refused by the thing storing it rather than by the machine unpacking it a week later.
|
||||
// Two steps for the blob, which is the registry's own protocol: ask for somewhere to put it, then
|
||||
// put it there naming the digest. The registry verifies the digest itself, so a blob that arrived
|
||||
// corrupted is refused by the thing storing it rather than by the machine unpacking it a week
|
||||
// later.
|
||||
//
|
||||
// **Then a manifest that names it** (novox/hq issue 253, ADR 0189). The store's own collector
|
||||
// marks only from manifests, and a blob no manifest names is collected however much the mesh
|
||||
// means to keep it — so an archive published bare is an archive the first nightly collection
|
||||
// deletes. The holder is composed from the digest and size alone (artifacts.Holder), which is
|
||||
// what lets the sweep recompute it to backfill or let go without anything being recorded here.
|
||||
// What a machine is told to fetch is the blob, exactly as before.
|
||||
func (r Registry) PublishArchive(ctx context.Context, repository string, body []byte, digest string) (string, error) {
|
||||
base := "http://" + r.Address + "/v2/" + repository
|
||||
final := base + "/blobs/" + digest
|
||||
|
||||
// Already there. Blobs are immutable and named by their content, so this is not an
|
||||
// optimisation — re-uploading would be asking the registry to store what it already has under
|
||||
// the name it already has.
|
||||
// the name it already has. It is still held: a blob published before holders existed is
|
||||
// exactly the one a rebuild of the same source finds already there.
|
||||
if there, err := r.has(ctx, final); err != nil {
|
||||
return "", err
|
||||
} else if there {
|
||||
return final, nil
|
||||
return final, r.hold(ctx, repository, digest, len(body))
|
||||
}
|
||||
|
||||
start, err := http.NewRequestWithContext(ctx, http.MethodPost, base+"/blobs/uploads/", nil)
|
||||
@@ -119,7 +131,17 @@ func (r Registry) PublishArchive(ctx context.Context, repository string, body []
|
||||
said, _ := io.ReadAll(io.LimitReader(done.Body, 4096))
|
||||
return "", fmt.Errorf("%s refused the blob: %s %s", base, done.Status, strings.TrimSpace(string(said)))
|
||||
}
|
||||
return final, nil
|
||||
return final, r.hold(ctx, repository, digest, len(body))
|
||||
}
|
||||
|
||||
// hold puts the manifest holding an archive beside it. A build whose archive could not be held is
|
||||
// a failed build: recorded as published, it would be an archive the store's collector takes.
|
||||
func (r Registry) hold(ctx context.Context, repository, digest string, size int) error {
|
||||
store := artifacts.Store{Address: r.Address, HTTP: r.client()}
|
||||
if _, err := store.HoldBlob(ctx, repository, digest, int64(size)); err != nil {
|
||||
return fmt.Errorf("published %s/blobs/%s and could not hold it by a manifest: %w", repository, digest, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// has is whether this registry already holds what is at that URL.
|
||||
|
||||
@@ -9,6 +9,8 @@ import (
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/artifacts"
|
||||
)
|
||||
|
||||
// An OCI registry as a content-addressed blob store, which is what it is.
|
||||
@@ -18,9 +20,10 @@ import (
|
||||
// there is not sent again, and that a tag is never accepted as a pin.
|
||||
|
||||
type fakeRegistry struct {
|
||||
blobs map[string][]byte
|
||||
uploads int
|
||||
location string
|
||||
blobs map[string][]byte
|
||||
manifests map[string][]byte // "<repository>@<digest>"
|
||||
puts map[string]int // blob uploads, by digest
|
||||
location string
|
||||
}
|
||||
|
||||
func (f *fakeRegistry) serve(t *testing.T) *httptest.Server {
|
||||
@@ -28,6 +31,8 @@ func (f *fakeRegistry) serve(t *testing.T) *httptest.Server {
|
||||
if f.blobs == nil {
|
||||
f.blobs = map[string][]byte{}
|
||||
}
|
||||
f.manifests = map[string][]byte{}
|
||||
f.puts = map[string]int{}
|
||||
mux := http.NewServeMux()
|
||||
server := httptest.NewServer(mux)
|
||||
mux.HandleFunc("/v2/", func(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -39,8 +44,28 @@ func (f *fakeRegistry) serve(t *testing.T) *httptest.Server {
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
case strings.Contains(r.URL.Path, "/manifests/"):
|
||||
// The archive's holder (novox/hq issue 253): asked for with an Accept, put by digest.
|
||||
repository, digest, _ := strings.Cut(strings.TrimPrefix(r.URL.Path, "/v2/"), "/manifests/")
|
||||
switch r.Method {
|
||||
case http.MethodHead:
|
||||
if _, ok := f.manifests[repository+"@"+digest]; ok &&
|
||||
strings.Contains(r.Header.Get("Accept"), "application/vnd.oci.image.manifest.v1+json") {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
case http.MethodPut:
|
||||
body, _ := io.ReadAll(r.Body)
|
||||
sum := sha256.Sum256(body)
|
||||
if digest != "sha256:"+hex.EncodeToString(sum[:]) {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
f.manifests[repository+"@"+digest] = body
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
}
|
||||
case r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/blobs/uploads/"):
|
||||
f.uploads++
|
||||
where := f.location
|
||||
if where == "" {
|
||||
where = "/v2/upload/" + hex.EncodeToString([]byte("session"))
|
||||
@@ -58,6 +83,7 @@ func (f *fakeRegistry) serve(t *testing.T) *httptest.Server {
|
||||
return
|
||||
}
|
||||
f.blobs[digest] = body
|
||||
f.puts[digest]++
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
default:
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
@@ -92,6 +118,57 @@ func TestAnArchiveIsStoredAndFetchableByItsDigest(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnArchiveIsPublishedWithAManifestHoldingIt(t *testing.T) {
|
||||
// The store's collector marks only from manifests, so a bare blob is one the first collection
|
||||
// deletes, kept or not (novox/hq issue 253, ADR 0189). Every archive goes out held, by the
|
||||
// holder the sweep can compute for itself from the digest and size.
|
||||
f := &fakeRegistry{}
|
||||
r := registryFor(t, f)
|
||||
body := []byte("a theme")
|
||||
sum := sha256.Sum256(body)
|
||||
digest := "sha256:" + hex.EncodeToString(sum[:])
|
||||
|
||||
where, err := r.PublishArchive(context.Background(), "shell/config", body, digest)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.HasSuffix(where, "/v2/shell/config/blobs/"+digest) {
|
||||
t.Fatalf("a machine is told to fetch %q; the blob is still what is fetched", where)
|
||||
}
|
||||
manifest, holder := artifacts.Holder(digest, int64(len(body)))
|
||||
if got := f.manifests["shell/config@"+holder]; string(got) != string(manifest) {
|
||||
t.Fatalf("the store holds %v; want the holder %s in the archive's own repository", f.manifests, holder)
|
||||
}
|
||||
if !strings.Contains(string(manifest), `"digest":"`+digest+`"`) {
|
||||
t.Fatalf("the holder does not name the archive: %s", manifest)
|
||||
}
|
||||
empty := sha256.Sum256([]byte("{}"))
|
||||
if _, ok := f.blobs["sha256:"+hex.EncodeToString(empty[:])]; !ok {
|
||||
t.Fatal("the holder's empty config was never put, and a registry refuses a manifest without it")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnArchiveAlreadyThereIsStillHeld(t *testing.T) {
|
||||
// A rebuild of the same source finds its archive already there — often one published bare,
|
||||
// before holders. It is not sent again, and it is held.
|
||||
f := &fakeRegistry{}
|
||||
r := registryFor(t, f)
|
||||
body := []byte("published bare")
|
||||
sum := sha256.Sum256(body)
|
||||
digest := "sha256:" + hex.EncodeToString(sum[:])
|
||||
f.blobs[digest] = body
|
||||
|
||||
if _, err := r.PublishArchive(context.Background(), "shell/config", body, digest); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f.puts[digest] != 0 {
|
||||
t.Fatal("a blob already there was sent again")
|
||||
}
|
||||
if _, holder := artifacts.Holder(digest, int64(len(body))); f.manifests["shell/config@"+holder] == nil {
|
||||
t.Fatal("a blob already there was left bare")
|
||||
}
|
||||
}
|
||||
|
||||
func TestABlobAlreadyThereIsNotSentAgain(t *testing.T) {
|
||||
// Not an optimisation: blobs are named by their content, so re-uploading is asking the
|
||||
// registry to store what it already has under the name it already has.
|
||||
@@ -107,8 +184,11 @@ func TestABlobAlreadyThereIsNotSentAgain(t *testing.T) {
|
||||
if _, err := r.PublishArchive(context.Background(), "shell/config", body, digest); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f.uploads != 1 {
|
||||
t.Fatalf("the blob was uploaded %d times", f.uploads)
|
||||
if f.puts[digest] != 1 {
|
||||
t.Fatalf("the blob was uploaded %d times", f.puts[digest])
|
||||
}
|
||||
if len(f.manifests) != 1 {
|
||||
t.Fatalf("publishing twice left %d holders; want the one", len(f.manifests))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -96,6 +96,7 @@ var ControllerVerbs = []Verb{
|
||||
Input: schema(map[string]string{
|
||||
"id": "a plan's id (as `plans` lists them): that plan, tier by tier",
|
||||
"stop": "a plan's id: stop it — what was asked still builds, nothing further is asked",
|
||||
"close": "a plan's id: close a plan that will not move again, as failed by hand (novox/hq issue 254)",
|
||||
"repository": "owner/repository: the plan a merge there would produce, saving nothing (what-if); with paths or modules",
|
||||
"paths": "with repository: the files the merge would change, comma-separated, from the repository's root",
|
||||
"modules": "with repository: or the modules it would change, comma-separated",
|
||||
|
||||
@@ -42,26 +42,37 @@ func theSeatDeclarer() catalogue.Manifest {
|
||||
// A module assigned to a machine becomes a user with the authority its manifest declared — and the
|
||||
// protocol of a seat declared by a *different* module, which is the whole reason a seat exists.
|
||||
func TestAnAssignedModuleBecomesAUserWithWhatItDeclared(t *testing.T) {
|
||||
// It declares where its account is delivered: a module with no own secret named broker can never
|
||||
// be issued one, and is no user at all (novox/hq issue 195) — the case asserted below.
|
||||
shop := catalogue.Manifest{
|
||||
Module: "shop", Version: "1",
|
||||
Emits: []string{"order.placed"}, Tools: []string{"price"},
|
||||
Uses: []string{"telegram-sender"},
|
||||
Uses: []string{"telegram-sender"},
|
||||
OwnSecrets: catalogue.OwnSecrets{"broker": {Path: "/run/broker"}},
|
||||
}
|
||||
inv, ctx := aMeshWith(t, theSeatDeclarer(), shop)
|
||||
quiet := catalogue.Manifest{Module: "quiet", Version: "1", Emits: []string{"thing.happened"}}
|
||||
inv, ctx := aMeshWith(t, theSeatDeclarer(), shop, quiet)
|
||||
if _, err := inv.AddNode(ctx, "one"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := inv.Assign(ctx, "one", "shop"); err != nil {
|
||||
t.Fatal(err)
|
||||
for _, module := range []string{"shop", "quiet"} {
|
||||
if _, err := inv.Assign(ctx, "one", module); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
records, err := inv.BusRecords(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
on := records.Assigned["one"]
|
||||
if len(on) != 1 || on[0].Module != "shop" {
|
||||
t.Fatalf("the machine's modules read as %+v", on)
|
||||
var on []broker.Declared
|
||||
for _, d := range records.Assigned["one"] {
|
||||
if d.Module == "shop" {
|
||||
on = append(on, d)
|
||||
}
|
||||
}
|
||||
if len(on) != 1 || on[0].NoAccount {
|
||||
t.Fatalf("the machine's modules read as %+v", records.Assigned["one"])
|
||||
}
|
||||
if len(on[0].Uses) != 1 || on[0].Uses[0].Accepts[0] != "send" {
|
||||
t.Fatalf("the seat it uses carries no protocol: %+v — so it would be granted nothing on a "+
|
||||
@@ -98,6 +109,11 @@ func TestAnAssignedModuleBecomesAUserWithWhatItDeclared(t *testing.T) {
|
||||
if !found {
|
||||
t.Fatal("no user was derived for the assigned module")
|
||||
}
|
||||
for _, u := range users {
|
||||
if u.Username() == "one.quiet" {
|
||||
t.Fatal("a module with nowhere to read an account was made a user (novox/hq issue 195)")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A machine holding a live token gets an enrolment user; one whose token is spent or expired does
|
||||
|
||||
@@ -3,6 +3,7 @@ package inventory
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
@@ -33,7 +34,8 @@ const KeptBuilds = 5
|
||||
// - **a definition names it** — the reference appears in a module's recorded manifest, which is
|
||||
// what the mesh would hand a machine now. No age limit: this is the floor;
|
||||
// - **the mesh can still go back to it** — it is an artifact of one of the KeptBuilds most
|
||||
// recent successful builds of its module;
|
||||
// recent successful builds of a module the mesh still holds. A forgotten module keeps
|
||||
// nothing beyond what a held definition names (novox/hq issue 253);
|
||||
// - it was already collected, in which case there is nothing left to do.
|
||||
//
|
||||
// Returned in a stated order so two runs over the same records ask for the same things in the
|
||||
@@ -109,12 +111,19 @@ func (i *Inventory) keptReferences(ctx context.Context) (map[string]bool, error)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// The KeptBuilds most recent successful builds of each module, whole.
|
||||
// The KeptBuilds most recent successful builds of each module the mesh still holds, whole.
|
||||
//
|
||||
// **Only a module the mesh still holds can be gone back to** (novox/hq issue 253). "Somewhere
|
||||
// to return to" is a reason about a module's releases; a module that has been forgotten has
|
||||
// no releases left to return between, and its build rows stay only as history. Without the
|
||||
// join every module ever built kept five builds' artifacts for ever — and once the store's
|
||||
// collector runs for real, what the keep set says is what the disk holds.
|
||||
recent, err := i.store.Pool().Query(ctx,
|
||||
`select made from (
|
||||
select made, row_number() over (partition by module order by at desc, id desc) as back
|
||||
from build
|
||||
where failed = '' and module is not null and module <> ''
|
||||
select b.made, row_number() over (partition by b.module order by b.at desc, b.id desc) as back
|
||||
from build b
|
||||
join module m on m.name = b.module
|
||||
where b.failed = '' and b.module is not null and b.module <> ''
|
||||
) ranked where back <= $1`, KeptBuilds)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -169,6 +178,28 @@ func (i *Inventory) keptReferences(ctx context.Context) (map[string]bool, error)
|
||||
return keep, nil
|
||||
}
|
||||
|
||||
// KeptArchives is every archive the mesh keeps, in a stated order: the references the sweep must
|
||||
// hold by a manifest before it lets anything go, and the ones an operator needs to read as all
|
||||
// held before the store's collector is let loose (novox/hq issue 253, ADR 0189).
|
||||
//
|
||||
// Only references into the mesh's own store, and only blobs: an image is its own manifest, and a
|
||||
// reference that is kept because nothing here can speak for it is not one the store can be asked
|
||||
// about.
|
||||
func (i *Inventory) KeptArchives(ctx context.Context) ([]string, error) {
|
||||
keep, err := i.keptReferences(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []string
|
||||
for reference := range keep {
|
||||
if path, ours := catalogue.InArtifactStore(reference); ours && strings.Contains(path, "/blobs/sha256:") {
|
||||
out = append(out, reference)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// everyReferenceMade is every artifact reference any successful build recorded.
|
||||
func (i *Inventory) everyReferenceMade(ctx context.Context) ([]string, error) {
|
||||
rows, err := i.store.Pool().Query(ctx,
|
||||
|
||||
@@ -20,6 +20,19 @@ func ref(module, artifact string, n int) string {
|
||||
return fmt.Sprintf("%s%s/%s@sha256:%064x", catalogue.ArtifactStoreScheme, module, artifact, n)
|
||||
}
|
||||
|
||||
// holding registers a definition for each module that names no artifact, so the mesh holds the
|
||||
// module and its recent builds are somewhere it can go back to — and nothing more.
|
||||
func holding(t *testing.T, inv *Inventory, modules ...string) {
|
||||
t.Helper()
|
||||
for _, module := range modules {
|
||||
m := catalogue.Manifest{Module: module, Version: "1"}
|
||||
if err := inv.RegisterModule(context.Background(), m,
|
||||
Source{Repository: "https://forge.invalid/" + module + ".git"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// built records one successful build of a module publishing one image.
|
||||
func built(t *testing.T, inv *Inventory, id, module string, n int) string {
|
||||
t.Helper()
|
||||
@@ -35,6 +48,7 @@ func built(t *testing.T, inv *Inventory, id, module string, n int) string {
|
||||
func TestTheStoreKeepsTheRecentBuildsAndLetsGoOfTheRest(t *testing.T) {
|
||||
inv := fresh(t)
|
||||
ctx := context.Background()
|
||||
holding(t, inv, "web")
|
||||
|
||||
// Eight builds of one module, oldest first. Five are kept — the newest, and the four a
|
||||
// release that turns out wrong can be taken back to.
|
||||
@@ -98,6 +112,7 @@ func TestWhatHasBeenCollectedIsNotOfferedAgain(t *testing.T) {
|
||||
// time it runs, for ever — a number of requests that grows with the mesh's whole history.
|
||||
inv := fresh(t)
|
||||
ctx := context.Background()
|
||||
holding(t, inv, "web")
|
||||
for i := 1; i <= 7; i++ {
|
||||
built(t, inv, fmt.Sprintf("b%02d", i), "web", i)
|
||||
}
|
||||
@@ -123,6 +138,7 @@ func TestWhatHasBeenCollectedIsNotOfferedAgain(t *testing.T) {
|
||||
func TestAFailedBuildNamesNothingToCollectAndEachModuleIsCountedOnItsOwn(t *testing.T) {
|
||||
inv := fresh(t)
|
||||
ctx := context.Background()
|
||||
holding(t, inv, "web", "db")
|
||||
|
||||
// A failed build published nothing, so it is neither kept nor collected — and it must not
|
||||
// count against the module's five.
|
||||
@@ -156,6 +172,7 @@ func TestAFailedBuildNamesNothingToCollectAndEachModuleIsCountedOnItsOwn(t *test
|
||||
func TestAnArtifactRecordedWithAnAddressIsOfferedAsTheMeshRecordsOne(t *testing.T) {
|
||||
inv := fresh(t)
|
||||
ctx := context.Background()
|
||||
holding(t, inv, "tools")
|
||||
|
||||
// The oldest build published the old way; five newer ones fill the module's five.
|
||||
old := aBuild("a00", "tools", "")
|
||||
@@ -190,3 +207,82 @@ func TestAnArtifactRecordedWithAnAddressIsOfferedAsTheMeshRecordsOne(t *testing.
|
||||
t.Fatalf("offered %v again after collecting it", again)
|
||||
}
|
||||
}
|
||||
|
||||
// A forgotten module keeps nothing beyond what a held definition names (novox/hq issue 253).
|
||||
//
|
||||
// "Somewhere to go back to" is a reason about a module's releases, and a module the mesh no
|
||||
// longer holds has none. Its build rows stay as history; its artifacts go — except one a module
|
||||
// the mesh still holds names, which is the floor whatever built it.
|
||||
func TestAForgottenModuleKeepsNothingAHeldDefinitionDoesNotName(t *testing.T) {
|
||||
inv := fresh(t)
|
||||
ctx := context.Background()
|
||||
|
||||
// Three builds of a module that was never held, or was held and then forgotten: within its
|
||||
// five, and kept for that reason until now.
|
||||
var gone []string
|
||||
for i := 1; i <= 3; i++ {
|
||||
gone = append(gone, built(t, inv, fmt.Sprintf("o%02d", i), "old", 200+i))
|
||||
}
|
||||
// A module the mesh holds, whose definition runs the forgotten module's newest image.
|
||||
named := gone[2]
|
||||
m := catalogue.Manifest{Module: "web", Version: "1", Resources: []map[string]any{{
|
||||
"id": "app", "type": "container", "name": "web", "image": named,
|
||||
}}}
|
||||
if err := inv.RegisterModule(ctx, m, Source{Repository: "https://forge.invalid/web.git"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
go_, err := inv.ToCollect(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(go_) != 2 || go_[0] != gone[0] || go_[1] != gone[1] {
|
||||
t.Fatalf("offered %v; want %v — a forgotten module's builds are no release to go back to, "+
|
||||
"and only what a held definition names stays", go_, gone[:2])
|
||||
}
|
||||
|
||||
// And once the module is held again, its five are kept again.
|
||||
holding(t, inv, "old")
|
||||
again, err := inv.ToCollect(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(again) != 0 {
|
||||
t.Fatalf("offered %v for a module the mesh holds, within its five", again)
|
||||
}
|
||||
}
|
||||
|
||||
// The archives the mesh keeps are what the sweep holds before it lets anything go, and what an
|
||||
// operator reads as all held before the store's collector is let loose (novox/hq issue 253).
|
||||
func TestKeptArchivesAreTheKeptBlobsOnly(t *testing.T) {
|
||||
inv := fresh(t)
|
||||
ctx := context.Background()
|
||||
holding(t, inv, "shell")
|
||||
|
||||
archive := func(n int) string {
|
||||
return fmt.Sprintf("%sshell/config/blobs/sha256:%064x", catalogue.ArtifactStoreScheme, n)
|
||||
}
|
||||
for i := 1; i <= 6; i++ {
|
||||
b := aBuild(fmt.Sprintf("s%02d", i), "shell", "")
|
||||
b.Made = []Artifact{
|
||||
{Name: "app", Kind: "image", Reference: ref("shell", "app", i)},
|
||||
{Name: "config", Kind: "archive", Reference: archive(i)},
|
||||
}
|
||||
if err := inv.RecordBuild(ctx, b); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
kept, err := inv.KeptArchives(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := []string{archive(2), archive(3), archive(4), archive(5), archive(6)}
|
||||
if len(kept) != len(want) {
|
||||
t.Fatalf("kept archives %v; want the five recent ones and no images", kept)
|
||||
}
|
||||
for i := range want {
|
||||
if kept[i] != want[i] {
|
||||
t.Fatalf("kept archives %v; want %v", kept, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
-- A newer plan supersedes the older open plans of the same repository and branch (novox/hq issue 254,
|
||||
-- ADR 0218).
|
||||
--
|
||||
-- A merge produced a plan without looking at the plans still open, so two merges a few minutes apart
|
||||
-- were two plans working the same modules, and a plan stuck waiting on something that would never
|
||||
-- come stayed open for ever beside the newer ones. The newer plan now takes over what the older had
|
||||
-- not yet built and the older is closed as `superseded` — a state of its own, so `plans` can say
|
||||
-- which plan replaced it rather than reading as a failure.
|
||||
--
|
||||
-- `branch` is the branch the merge went into, so only a plan of the same branch is superseded. Empty
|
||||
-- for every plan from before this was kept: which branch it answered is not known, and such a plan
|
||||
-- is superseded by the next plan of its repository, whichever branch — nothing is lost by it, since
|
||||
-- what it had not built is folded into the plan that supersedes it.
|
||||
alter table release_plan add column branch text not null default '';
|
||||
|
||||
-- And the bus's user list the machine holding the bus was last sent, as a digest (novox/hq issue
|
||||
-- 249). A module's new grants are refused by the bus until its user list says them, so that machine
|
||||
-- is sent first whenever the list it would be sent differs from the one it was. Read from its whole
|
||||
-- declaration, every pending change on it — a recorded upgrade the operator chose not to roll out —
|
||||
-- went with every send anywhere. A digest and never the list (ADR 0043: the list is composed on each
|
||||
-- push, never kept). Empty for a machine never sent one, which reads as behind once.
|
||||
alter table node add column sent_bus_users text not null default '';
|
||||
@@ -921,6 +921,26 @@ func (i *Inventory) RecordSent(ctx context.Context, node, digest string) error {
|
||||
return err
|
||||
}
|
||||
|
||||
// RecordSentBusUsers keeps a digest of the bus's user list a machine was just sent, by its name
|
||||
// (novox/hq issue 249): whether the machine holding the bus must go first is whether this differs
|
||||
// from the list composed now.
|
||||
func (i *Inventory) RecordSentBusUsers(ctx context.Context, name, digest string) error {
|
||||
_, err := i.store.Pool().Exec(ctx,
|
||||
`update node set sent_bus_users = $2 where name = $1`, name, digest)
|
||||
return err
|
||||
}
|
||||
|
||||
// SentBusUsers is the digest of the bus's user list a machine was last sent, empty for none.
|
||||
func (i *Inventory) SentBusUsers(ctx context.Context, name string) (string, error) {
|
||||
var sent string
|
||||
err := i.store.Pool().QueryRow(ctx,
|
||||
`select sent_bus_users from node where name = $1`, name).Scan(&sent)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return "", nil
|
||||
}
|
||||
return sent, err
|
||||
}
|
||||
|
||||
// Outstanding is the digest of the declaration a machine was last sent, by its name, and empty
|
||||
// for one that has never been sent anything.
|
||||
//
|
||||
|
||||
+31
-18
@@ -15,16 +15,19 @@ import (
|
||||
// the store so a controller replaced mid-plan resumes it, and so `status` can say what a merge
|
||||
// still waits for.
|
||||
type Plan struct {
|
||||
ID string `json:"id"`
|
||||
Repository string `json:"repository"`
|
||||
Commit string `json:"commit"`
|
||||
Created time.Time `json:"created"`
|
||||
Updated time.Time `json:"updated"`
|
||||
State string `json:"state"`
|
||||
Tier int `json:"tier"`
|
||||
Tiers [][]string `json:"tiers"`
|
||||
Modules map[string]*PlanModule `json:"modules"`
|
||||
Note string `json:"note,omitempty"`
|
||||
ID string `json:"id"`
|
||||
Repository string `json:"repository"`
|
||||
// Branch is the branch the merge went into (novox/hq issue 254): a newer plan supersedes the open
|
||||
// ones of the same repository and branch. Empty for a plan from before it was kept.
|
||||
Branch string `json:"branch,omitempty"`
|
||||
Commit string `json:"commit"`
|
||||
Created time.Time `json:"created"`
|
||||
Updated time.Time `json:"updated"`
|
||||
State string `json:"state"`
|
||||
Tier int `json:"tier"`
|
||||
Tiers [][]string `json:"tiers"`
|
||||
Modules map[string]*PlanModule `json:"modules"`
|
||||
Note string `json:"note,omitempty"`
|
||||
}
|
||||
|
||||
// PlanModule is one module's state within a plan.
|
||||
@@ -37,8 +40,15 @@ type PlanModule struct {
|
||||
// later tier is built by it (ADR 0163's gate): the reports that open the gate are the ones
|
||||
// after this.
|
||||
SentAt *time.Time `json:"sent_at,omitempty"`
|
||||
Commit string `json:"commit,omitempty"`
|
||||
Why string `json:"why,omitempty"`
|
||||
// First is the machines the plan sent the new build to first, and FirstAt when (novox/hq issue
|
||||
// 249, ADR 0218): unless the module's policy rolls it out together, one machine takes it before
|
||||
// the rest, and the rest are sent once that one reports it applied. Kept so a controller
|
||||
// replaced while the plan waits on that report resumes the wait rather than sending again. The
|
||||
// machine holding the bus is among them when its user list had to go first.
|
||||
First []string `json:"first,omitempty"`
|
||||
FirstAt *time.Time `json:"first_at,omitempty"`
|
||||
Commit string `json:"commit,omitempty"`
|
||||
Why string `json:"why,omitempty"`
|
||||
}
|
||||
|
||||
// The states a plan passes through.
|
||||
@@ -47,6 +57,9 @@ const (
|
||||
PlanRolling = "rolling"
|
||||
PlanDone = "done"
|
||||
PlanFailed = "failed"
|
||||
// PlanSuperseded is a plan a newer merge of the same repository and branch took over (novox/hq
|
||||
// issue 254, ADR 0218): what it had not built is in the newer plan, and its note names it.
|
||||
PlanSuperseded = "superseded"
|
||||
)
|
||||
|
||||
// Open says whether the plan is still being worked.
|
||||
@@ -63,11 +76,11 @@ func (i *Inventory) SavePlan(ctx context.Context, p Plan) error {
|
||||
return err
|
||||
}
|
||||
_, err = i.store.Pool().Exec(ctx,
|
||||
`insert into release_plan (id, repository, commit_hash, created, updated, state, tier, tiers, modules, note)
|
||||
values ($1, $2, $3, $4, now(), $5, $6, $7, $8, $9)
|
||||
`insert into release_plan (id, repository, commit_hash, created, updated, state, tier, tiers, modules, note, branch)
|
||||
values ($1, $2, $3, $4, now(), $5, $6, $7, $8, $9, $10)
|
||||
on conflict (id) do update set updated = now(), state = excluded.state, tier = excluded.tier,
|
||||
tiers = excluded.tiers, modules = excluded.modules, note = excluded.note`,
|
||||
p.ID, p.Repository, p.Commit, p.Created, p.State, p.Tier, tiers, modules, p.Note)
|
||||
tiers = excluded.tiers, modules = excluded.modules, note = excluded.note, branch = excluded.branch`,
|
||||
p.ID, p.Repository, p.Commit, p.Created, p.State, p.Tier, tiers, modules, p.Note, p.Branch)
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -95,7 +108,7 @@ func (i *Inventory) PlanByID(ctx context.Context, id string) (Plan, error) {
|
||||
|
||||
func (i *Inventory) plans(ctx context.Context, tail string) ([]Plan, error) {
|
||||
rows, err := i.store.Pool().Query(ctx,
|
||||
`select id, repository, commit_hash, created, updated, state, tier, tiers, modules, note
|
||||
`select id, repository, commit_hash, created, updated, state, tier, tiers, modules, note, branch
|
||||
from release_plan `+tail)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -106,7 +119,7 @@ func (i *Inventory) plans(ctx context.Context, tail string) ([]Plan, error) {
|
||||
var p Plan
|
||||
var tiers, modules []byte
|
||||
if err := rows.Scan(&p.ID, &p.Repository, &p.Commit, &p.Created, &p.Updated, &p.State,
|
||||
&p.Tier, &tiers, &modules, &p.Note); err != nil {
|
||||
&p.Tier, &tiers, &modules, &p.Note, &p.Branch); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := json.Unmarshal(tiers, &p.Tiers); err != nil {
|
||||
|
||||
@@ -46,3 +46,30 @@ func TestAPlanIsKeptAdvancedAndResumedFromTheStore(t *testing.T) {
|
||||
t.Fatalf("a done plan is still among the recent ones: %+v", recent)
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq issue 254: a plan keeps the branch its merge went into, and a superseded plan is not open.
|
||||
func TestASupersededPlanIsNotOpen(t *testing.T) {
|
||||
inv := ForTest(t)
|
||||
ctx := t.Context()
|
||||
p := Plan{ID: "plan-1", Repository: "novox/mesh-catalog", Branch: "main", Commit: "abc",
|
||||
Created: time.Now().UTC(), State: PlanBuilding, Tiers: [][]string{{"gitea"}},
|
||||
Modules: map[string]*PlanModule{"gitea": {}}}
|
||||
if err := inv.SavePlan(ctx, p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kept, err := inv.PlanByID(ctx, "plan-1")
|
||||
if err != nil || kept.Branch != "main" {
|
||||
t.Fatalf("the branch was not kept: %v %+v", err, kept)
|
||||
}
|
||||
kept.State = PlanSuperseded
|
||||
kept.Note = "superseded at tier 0 by plan-2"
|
||||
if err := inv.SavePlan(ctx, kept); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if open, err := inv.OpenPlans(ctx); err != nil || len(open) != 0 {
|
||||
t.Fatalf("a superseded plan is still open: %v %+v", err, open)
|
||||
}
|
||||
if recent, _ := inv.RecentPlans(ctx, 5); len(recent) != 1 || recent[0].State != PlanSuperseded {
|
||||
t.Fatalf("a superseded plan is not among the recent ones as superseded: %+v", recent)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
package inventory
|
||||
|
||||
import "testing"
|
||||
|
||||
// novox/hq issue 249: the digest of the user list a machine was last sent is kept, by its name, and
|
||||
// is empty for a machine never sent one.
|
||||
func TestTheUserListAMachineWasSentIsKept(t *testing.T) {
|
||||
inv := ForTest(t)
|
||||
ctx := t.Context()
|
||||
if _, err := inv.AddNode(ctx, "anchor"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if sent, err := inv.SentBusUsers(ctx, "anchor"); err != nil || sent != "" {
|
||||
t.Fatalf("a machine never sent a list has %q: %v", sent, err)
|
||||
}
|
||||
if err := inv.RecordSentBusUsers(ctx, "anchor", "abc"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if sent, err := inv.SentBusUsers(ctx, "anchor"); err != nil || sent != "abc" {
|
||||
t.Fatalf("the list sent was not kept: %q %v", sent, err)
|
||||
}
|
||||
if sent, err := inv.SentBusUsers(ctx, "nobody"); err != nil || sent != "" {
|
||||
t.Fatalf("a machine the mesh does not know: %q %v", sent, err)
|
||||
}
|
||||
}
|
||||
@@ -315,6 +315,12 @@ func TestNatsTheEventsTheControllerFollowsArriveAndAreAcknowledged(t *testing.T)
|
||||
ctx, stop := context.WithCancel(context.Background())
|
||||
defer stop()
|
||||
go func() { _ = s.Serve(ctx) }()
|
||||
// The controller's event consumer is made from now (novox/hq issue 248): what was published before
|
||||
// it existed is history it never replays. So the announcements are made once it is there.
|
||||
eventually(t, "the controller's event consumer being made", func() bool {
|
||||
_, err := js.Context().ConsumerInfo("EVENTS", broker.ControllerName)
|
||||
return err == nil
|
||||
})
|
||||
|
||||
moved, _ := json.Marshal(Upgraded{Module: "gitea", Commit: "abcdef0123"})
|
||||
if _, err := js.Context().Publish(broker.ControllerFollows[0], moved); err != nil {
|
||||
|
||||
Reference in New Issue
Block a user