Merge pull request 'Let a declared file travel by its sha256 digest instead of its content, so a node's declaration is no longer bounded by one bus message (issue 1694, ADR 0486 step 1)' (#59) from fix/1694-file-references into main
This commit was merged in pull request #59.
This commit is contained in:
@@ -0,0 +1,10 @@
|
||||
package catalogue
|
||||
|
||||
import "testing"
|
||||
|
||||
// 1694/AC5: the capability a node-engine reports for files by digest (novox/hq ADR 0486 §5) is a known one.
|
||||
func TestFileReferencesIsAKnownCapability(t *testing.T) {
|
||||
if !oneOf(KnownCapabilities, "file-references") {
|
||||
t.Fatal("file-references is not a known capability")
|
||||
}
|
||||
}
|
||||
@@ -66,7 +66,7 @@ var capabilityName = regexp.MustCompile(`^[a-z0-9][a-z0-9-]*$`)
|
||||
// names an `if-capability` may give (novox/hq ADR 0255). A name nothing detects would leave its
|
||||
// contribution out on every machine, silently, so it is refused. A detector added there is added here.
|
||||
var KnownCapabilities = []string{
|
||||
"battery", "container-runtime", "firewall", "graphical-session", "overlay", "package-manager",
|
||||
"battery", "container-runtime", "file-references", "firewall", "graphical-session", "overlay", "package-manager",
|
||||
"power-meter", "privileged", "seat", "service-manager", "uplink-dhcpcd", "uplink-networkmanager",
|
||||
"uplink-systemd-networkd", "user-namespaces", "virtualisation",
|
||||
}
|
||||
|
||||
@@ -324,6 +324,15 @@ type File struct {
|
||||
// depend on a field somewhere else.
|
||||
Bytes string `json:"bytes,omitempty"`
|
||||
|
||||
// Digest and Size say the content is not carried but fetched: a file over FileReferenceThreshold travels as
|
||||
// the sha256 digest of its content, "sha256:<64 hex>", and its size in bytes, in place of Content or Bytes
|
||||
// (novox/hq ADR 0486 §1). The node-engine fetches the object of that name from FileBucket, checks what it
|
||||
// received against Digest and Size, and only then writes the file. Exclusive with Content, Sealed, Bytes,
|
||||
// VaultSealed and Secrets; sent only to an engine that reports FileReferencesCapability, as an older engine
|
||||
// refuses a field it does not know, whole.
|
||||
Digest string `json:"digest,omitempty"`
|
||||
Size int64 `json:"size,omitempty"`
|
||||
|
||||
// Owner is the user this belongs to, by name. Absent means root.
|
||||
Owner string `json:"owner,omitempty"`
|
||||
|
||||
@@ -465,6 +474,10 @@ func (f *File) validate(where string, _ bool) []string {
|
||||
problems = append(problems, where+": a file the vault sealed whole has no holes for secrets")
|
||||
}
|
||||
}
|
||||
problems = append(problems, f.checkReference(where)...)
|
||||
if f.Digest != "" || f.Size != 0 {
|
||||
said = append(said, "digest")
|
||||
}
|
||||
if len(said) > 1 {
|
||||
sort.Strings(said)
|
||||
problems = append(problems, where+
|
||||
|
||||
@@ -0,0 +1,125 @@
|
||||
package declaration
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"regexp"
|
||||
)
|
||||
|
||||
// The three names of a file travelling by digest (novox/hq ADR 0486 §1, §2, §5), shared by the node-engine that
|
||||
// reads them and the controller that writes them.
|
||||
const (
|
||||
// FileReferenceThreshold is the size above which a file travels by digest: a file of this many bytes or fewer
|
||||
// stays in the declaration as it is.
|
||||
FileReferenceThreshold = 64 * 1024
|
||||
// FileBucket is the object bucket of the bus that holds the contents, each under the hex of its sha256 digest
|
||||
// (ObjectName). The controller alone writes it; a node-engine reads an object by name.
|
||||
FileBucket = "declaration-files"
|
||||
// FileReferencesCapability is what a node-engine reports when it reads a file given as a digest and size.
|
||||
FileReferencesCapability = "file-references"
|
||||
)
|
||||
|
||||
var digestForm = regexp.MustCompile(`^sha256:[0-9a-f]{64}$`)
|
||||
|
||||
// DigestOf is the digest a reference names for a content: "sha256:<64 lower-case hex>".
|
||||
func DigestOf(content []byte) string {
|
||||
sum := sha256.Sum256(content)
|
||||
return "sha256:" + hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
// ObjectName is the name of the object holding the content a digest names: the hex without its "sha256:" prefix.
|
||||
func ObjectName(digest string) (string, error) {
|
||||
if !digestForm.MatchString(digest) {
|
||||
return "", fmt.Errorf("digest %q is not sha256:<64 lower-case hex characters>", digest)
|
||||
}
|
||||
return digest[len("sha256:"):], nil
|
||||
}
|
||||
|
||||
// IsReference reports whether the file gives its content as a digest and size instead of carrying it.
|
||||
func (f *File) IsReference() bool { return f.Digest != "" }
|
||||
|
||||
// checkReference judges the digest and size, and what a reference may not be given with.
|
||||
func (f *File) checkReference(where string) []string {
|
||||
if f.Digest == "" && f.Size == 0 {
|
||||
return nil
|
||||
}
|
||||
var problems []string
|
||||
if !digestForm.MatchString(f.Digest) {
|
||||
problems = append(problems, fmt.Sprintf("%s: digest %q; a file by reference names its content as "+
|
||||
"sha256:<64 lower-case hex characters>", where, f.Digest))
|
||||
}
|
||||
if f.Size <= 0 {
|
||||
problems = append(problems, fmt.Sprintf("%s: size %d; a file by reference states its size in bytes, "+
|
||||
"more than zero", where, f.Size))
|
||||
}
|
||||
if f.Content != "" || f.Bytes != "" || f.Sealed != "" || f.VaultSealed != nil || len(f.Secrets) > 0 {
|
||||
problems = append(problems, where+": a file by reference carries its content in the object its digest "+
|
||||
"names, so it says no content, bytes, sealed, vault-sealed or secrets")
|
||||
}
|
||||
if f.Into != "" {
|
||||
problems = append(problems, where+": a file by reference is written whole; a file written into says its "+
|
||||
"part in content")
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
// Body is the bytes the file's content stands for: its content, or its bytes decoded.
|
||||
func (f *File) Body() ([]byte, error) {
|
||||
if f.Bytes == "" {
|
||||
return []byte(f.Content), nil
|
||||
}
|
||||
b, err := base64.StdEncoding.DecodeString(f.Bytes)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("file %s: bytes are not base64: %w", f.ID, err)
|
||||
}
|
||||
return b, nil
|
||||
}
|
||||
|
||||
// TravelsByDigest reports whether the controller may put this file's content in the bucket: it is plain content
|
||||
// or bytes (not sealed, not filled with secrets, not written into another file) of more than FileReferenceThreshold
|
||||
// bytes.
|
||||
func (f *File) TravelsByDigest() bool {
|
||||
if f.IsReference() || f.Sealed != "" || f.VaultSealed != nil || len(f.Secrets) > 0 || f.Into != "" {
|
||||
return false
|
||||
}
|
||||
if f.Bytes != "" {
|
||||
// DecodedLen counts padding, so the exact length is taken off the decoded bytes.
|
||||
body, err := f.Body()
|
||||
return err == nil && len(body) > FileReferenceThreshold
|
||||
}
|
||||
return len(f.Content) > FileReferenceThreshold
|
||||
}
|
||||
|
||||
// AsReference is the file as it is sent to an engine that reports FileReferencesCapability: a copy with the digest
|
||||
// and size in place of the content, and the content the controller puts into the bucket under ObjectName(digest).
|
||||
// A file that does not travel by digest is refused, not passed through, so a caller cannot send a small file empty.
|
||||
func (f *File) AsReference() (File, []byte, error) {
|
||||
if !f.TravelsByDigest() {
|
||||
return File{}, nil, fmt.Errorf("file %s does not travel by digest", f.ID)
|
||||
}
|
||||
body, err := f.Body()
|
||||
if err != nil {
|
||||
return File{}, nil, err
|
||||
}
|
||||
ref := *f
|
||||
ref.Content, ref.Bytes = "", ""
|
||||
ref.Digest, ref.Size = DigestOf(body), int64(len(body))
|
||||
return ref, body, nil
|
||||
}
|
||||
|
||||
// Verify checks what was fetched against the digest and size the file names; the node-engine writes the file only
|
||||
// after it returns nil. A mismatch is the apply's failure, never a partial apply (ADR 0486 §1).
|
||||
func (f *File) Verify(body []byte) error {
|
||||
if !f.IsReference() {
|
||||
return fmt.Errorf("file %s is not given by reference", f.ID)
|
||||
}
|
||||
if int64(len(body)) != f.Size {
|
||||
return fmt.Errorf("file %s: the object is %d bytes and the declaration says %d", f.ID, len(body), f.Size)
|
||||
}
|
||||
if got := DigestOf(body); got != f.Digest {
|
||||
return fmt.Errorf("file %s: the object hashes to %s and the declaration names %s", f.ID, got, f.Digest)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
package declaration
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func refFile(extra string) string {
|
||||
return `{"declaration":1,"resources":[{"id":"m.f","type":"file","path":"/etc/m","digest":"sha256:` +
|
||||
strings.Repeat("a", 64) + `","size":70000` + extra + `}]}`
|
||||
}
|
||||
|
||||
// 1694/AC1: a reference round trips, and an invalid one is refused naming the field.
|
||||
func TestAFileReferenceRoundTripsAndABadOneNamesItsField(t *testing.T) {
|
||||
d, err := Parse([]byte(refFile("")))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f := d.Resources[0].(*File)
|
||||
if !f.IsReference() || f.Size != 70000 || f.Digest != "sha256:"+strings.Repeat("a", 64) {
|
||||
t.Fatalf("read: %+v", f)
|
||||
}
|
||||
raw, _ := json.Marshal(d.Resources[0])
|
||||
d2, err := Parse([]byte(`{"declaration":1,"resources":[` + string(raw) + `]}`))
|
||||
if err != nil || !reflect.DeepEqual(d2.Resources[0], d.Resources[0]) {
|
||||
t.Fatalf("round trip: %v %s", err, raw)
|
||||
}
|
||||
for want, raw := range map[string]string{
|
||||
"digest": strings.Replace(refFile(""), strings.Repeat("a", 64), "abc", 1),
|
||||
"size": strings.Replace(refFile(""), `"size":70000`, `"size":0`, 1),
|
||||
"lower-case": strings.Replace(refFile(""), strings.Repeat("a", 64), strings.Repeat("A", 64), 1),
|
||||
"content": refFile(`,"content":"x"`),
|
||||
"bytes": refFile(`,"bytes":"eA=="`),
|
||||
"whole": refFile(`,"into":"block"`),
|
||||
} {
|
||||
_, err := Parse([]byte(raw))
|
||||
if err == nil || !strings.Contains(err.Error(), want) {
|
||||
t.Errorf("want a refusal naming %q, got %v", want, err)
|
||||
}
|
||||
}
|
||||
// a size alone is not a reference
|
||||
if _, err := Parse([]byte(`{"declaration":1,"resources":[{"id":"m.f","type":"file","path":"/e","size":5}]}`)); err == nil {
|
||||
t.Error("a size without a digest was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
// 1694/AC2: a declaration as today's SDK writes it reads unchanged and its bytes carry no new key.
|
||||
func TestAnInlineFileIsReadAndWrittenAsBefore(t *testing.T) {
|
||||
const in = `{"id":"m.f","type":"file","path":"/e","content":"hello"}`
|
||||
d, err := Parse([]byte(`{"declaration":1,"resources":[` + in + `]}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out, _ := json.Marshal(d.Resources[0])
|
||||
if string(out) != in {
|
||||
t.Fatalf("signed bytes changed: %s", out)
|
||||
}
|
||||
if d.Resources[0].(*File).IsReference() {
|
||||
t.Fatal("inline read as a reference")
|
||||
}
|
||||
}
|
||||
|
||||
// 1694/AC3: the controller's helper makes a reference of a big file and the engine's check refuses a changed byte.
|
||||
func TestABigFileBecomesAReferenceAndAChangedByteFailsTheCheck(t *testing.T) {
|
||||
big := strings.Repeat("x", FileReferenceThreshold+1)
|
||||
f := &File{ID: "m.f", Type: TypeFile, Path: "/e", Content: big}
|
||||
if !f.TravelsByDigest() {
|
||||
t.Fatal("a file over the threshold stays inline")
|
||||
}
|
||||
ref, body, err := f.AsReference()
|
||||
if err != nil || ref.Content != "" || ref.Size != int64(len(big)) || string(body) != big {
|
||||
t.Fatalf("%v %+v", err, ref)
|
||||
}
|
||||
if p := ref.validate("f", false); len(p) > 0 {
|
||||
t.Fatalf("the reference is refused: %v", p)
|
||||
}
|
||||
if name, err := ObjectName(ref.Digest); err != nil || len(name) != 64 {
|
||||
t.Fatalf("object name %q %v", name, err)
|
||||
}
|
||||
if err := ref.Verify(body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
bad := bytes.Clone(body)
|
||||
bad[7] ^= 1
|
||||
if err := ref.Verify(bad); err == nil || !strings.Contains(err.Error(), "hashes to") {
|
||||
t.Fatalf("a changed byte: %v", err)
|
||||
}
|
||||
if err := ref.Verify(body[1:]); err == nil || !strings.Contains(err.Error(), "bytes") {
|
||||
t.Fatalf("a short object: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// 1694/AC4: what stays inline, and bytes files travel decoded.
|
||||
func TestWhichFilesTravelByDigest(t *testing.T) {
|
||||
small := strings.Repeat("x", FileReferenceThreshold)
|
||||
big := strings.Repeat("x", FileReferenceThreshold+1)
|
||||
for name, f := range map[string]File{
|
||||
"at the threshold": {Content: small},
|
||||
"sealed": {Sealed: "s"},
|
||||
"secrets": {Content: big, Secrets: map[string]string{"a": "b"}},
|
||||
"into": {Content: big, Into: IntoBlock},
|
||||
} {
|
||||
if f.TravelsByDigest() {
|
||||
t.Errorf("%s travels by digest", name)
|
||||
}
|
||||
if _, _, err := f.AsReference(); err == nil {
|
||||
t.Errorf("%s was made a reference", name)
|
||||
}
|
||||
}
|
||||
exact := File{ID: "m.e", Bytes: base64.StdEncoding.EncodeToString(bytes.Repeat([]byte{7}, FileReferenceThreshold))}
|
||||
if exact.TravelsByDigest() {
|
||||
t.Error("a bytes file of exactly the threshold travels by digest")
|
||||
}
|
||||
if _, _, err := exact.AsReference(); err == nil {
|
||||
t.Error("a bytes file of exactly the threshold was made a reference")
|
||||
}
|
||||
raw := bytes.Repeat([]byte{0, 1, 2}, FileReferenceThreshold)
|
||||
f := File{ID: "m.b", Bytes: base64.StdEncoding.EncodeToString(raw)}
|
||||
ref, body, err := f.AsReference()
|
||||
if err != nil || !bytes.Equal(body, raw) || ref.Size != int64(len(raw)) || ref.Digest != DigestOf(raw) {
|
||||
t.Fatalf("bytes file: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheSharedNames(t *testing.T) {
|
||||
if FileBucket != "declaration-files" || FileReferencesCapability != "file-references" || FileReferenceThreshold != 65536 {
|
||||
t.Fatal("a name changed")
|
||||
}
|
||||
if _, err := ObjectName("sha256:zz"); err == nil {
|
||||
t.Fatal("bad digest named an object")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user