From f8a9420952b6706f1bf29e4413e7fb704127941d Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 21 Oct 2026 14:12:19 +0200 Subject: [PATCH 1/2] A declaration's file can be a reference by digest: layout, bucket and capability names, helpers (issues 1694, ADR 0486) --- go/catalogue/file_references_test.go | 10 +++ go/catalogue/seat_data.go | 2 +- go/declaration/declaration.go | 13 +++ go/declaration/filereference.go | 123 +++++++++++++++++++++++++ go/declaration/filereference_test.go | 129 +++++++++++++++++++++++++++ 5 files changed, 276 insertions(+), 1 deletion(-) create mode 100644 go/catalogue/file_references_test.go create mode 100644 go/declaration/filereference.go create mode 100644 go/declaration/filereference_test.go diff --git a/go/catalogue/file_references_test.go b/go/catalogue/file_references_test.go new file mode 100644 index 0000000..62f726d --- /dev/null +++ b/go/catalogue/file_references_test.go @@ -0,0 +1,10 @@ +package catalogue + +import "testing" + +// 1694/AC5: the capability a node-engine reports for files by digest (novox/hq ADR 0486 §5) is a known one. +func TestFileReferencesIsAKnownCapability(t *testing.T) { + if !oneOf(KnownCapabilities, "file-references") { + t.Fatal("file-references is not a known capability") + } +} diff --git a/go/catalogue/seat_data.go b/go/catalogue/seat_data.go index bea4fc7..3a4fc8b 100644 --- a/go/catalogue/seat_data.go +++ b/go/catalogue/seat_data.go @@ -66,7 +66,7 @@ var capabilityName = regexp.MustCompile(`^[a-z0-9][a-z0-9-]*$`) // names an `if-capability` may give (novox/hq ADR 0255). A name nothing detects would leave its // contribution out on every machine, silently, so it is refused. A detector added there is added here. var KnownCapabilities = []string{ - "battery", "container-runtime", "firewall", "graphical-session", "overlay", "package-manager", + "battery", "container-runtime", "file-references", "firewall", "graphical-session", "overlay", "package-manager", "power-meter", "privileged", "seat", "service-manager", "uplink-dhcpcd", "uplink-networkmanager", "uplink-systemd-networkd", "user-namespaces", "virtualisation", } diff --git a/go/declaration/declaration.go b/go/declaration/declaration.go index ea16b91..85fe147 100644 --- a/go/declaration/declaration.go +++ b/go/declaration/declaration.go @@ -324,6 +324,15 @@ type File struct { // depend on a field somewhere else. Bytes string `json:"bytes,omitempty"` + // Digest and Size say the content is not carried but fetched: a file over FileReferenceThreshold travels as + // the sha256 digest of its content, "sha256:<64 hex>", and its size in bytes, in place of Content or Bytes + // (novox/hq ADR 0486 §1). The node-engine fetches the object of that name from FileBucket, checks what it + // received against Digest and Size, and only then writes the file. Exclusive with Content, Sealed, Bytes, + // VaultSealed and Secrets; sent only to an engine that reports FileReferencesCapability, as an older engine + // refuses a field it does not know, whole. + Digest string `json:"digest,omitempty"` + Size int64 `json:"size,omitempty"` + // Owner is the user this belongs to, by name. Absent means root. Owner string `json:"owner,omitempty"` @@ -465,6 +474,10 @@ func (f *File) validate(where string, _ bool) []string { problems = append(problems, where+": a file the vault sealed whole has no holes for secrets") } } + problems = append(problems, f.checkReference(where)...) + if f.Digest != "" || f.Size != 0 { + said = append(said, "digest") + } if len(said) > 1 { sort.Strings(said) problems = append(problems, where+ diff --git a/go/declaration/filereference.go b/go/declaration/filereference.go new file mode 100644 index 0000000..1bbdcdd --- /dev/null +++ b/go/declaration/filereference.go @@ -0,0 +1,123 @@ +package declaration + +import ( + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "fmt" + "regexp" +) + +// The three names of a file travelling by digest (novox/hq ADR 0486 §1, §2, §5), shared by the node-engine that +// reads them and the controller that writes them. +const ( + // FileReferenceThreshold is the size above which a file travels by digest: a file of this many bytes or fewer + // stays in the declaration as it is. + FileReferenceThreshold = 64 * 1024 + // FileBucket is the object bucket of the bus that holds the contents, each under the hex of its sha256 digest + // (ObjectName). The controller alone writes it; a node-engine reads an object by name. + FileBucket = "declaration-files" + // FileReferencesCapability is what a node-engine reports when it reads a file given as a digest and size. + FileReferencesCapability = "file-references" +) + +var digestForm = regexp.MustCompile(`^sha256:[0-9a-f]{64}$`) + +// DigestOf is the digest a reference names for a content: "sha256:<64 lower-case hex>". +func DigestOf(content []byte) string { + sum := sha256.Sum256(content) + return "sha256:" + hex.EncodeToString(sum[:]) +} + +// ObjectName is the name of the object holding the content a digest names: the hex without its "sha256:" prefix. +func ObjectName(digest string) (string, error) { + if !digestForm.MatchString(digest) { + return "", fmt.Errorf("digest %q is not sha256:<64 lower-case hex characters>", digest) + } + return digest[len("sha256:"):], nil +} + +// IsReference reports whether the file gives its content as a digest and size instead of carrying it. +func (f *File) IsReference() bool { return f.Digest != "" } + +// checkReference judges the digest and size, and what a reference may not be given with. +func (f *File) checkReference(where string) []string { + if f.Digest == "" && f.Size == 0 { + return nil + } + var problems []string + if !digestForm.MatchString(f.Digest) { + problems = append(problems, fmt.Sprintf("%s: digest %q; a file by reference names its content as "+ + "sha256:<64 lower-case hex characters>", where, f.Digest)) + } + if f.Size <= 0 { + problems = append(problems, fmt.Sprintf("%s: size %d; a file by reference states its size in bytes, "+ + "more than zero", where, f.Size)) + } + if f.Content != "" || f.Bytes != "" || f.Sealed != "" || f.VaultSealed != nil || len(f.Secrets) > 0 { + problems = append(problems, where+": a file by reference carries its content in the object its digest "+ + "names, so it says no content, bytes, sealed, vault-sealed or secrets") + } + if f.Into != "" { + problems = append(problems, where+": a file by reference is written whole; a file written into says its "+ + "part in content") + } + return problems +} + +// Body is the bytes the file's content stands for: its content, or its bytes decoded. +func (f *File) Body() ([]byte, error) { + if f.Bytes == "" { + return []byte(f.Content), nil + } + b, err := base64.StdEncoding.DecodeString(f.Bytes) + if err != nil { + return nil, fmt.Errorf("file %s: bytes are not base64: %w", f.ID, err) + } + return b, nil +} + +// TravelsByDigest reports whether the controller may put this file's content in the bucket: it is plain content +// or bytes (not sealed, not filled with secrets, not written into another file) of more than FileReferenceThreshold +// bytes. +func (f *File) TravelsByDigest() bool { + if f.IsReference() || f.Sealed != "" || f.VaultSealed != nil || len(f.Secrets) > 0 || f.Into != "" { + return false + } + if f.Bytes != "" { + return base64.StdEncoding.DecodedLen(len(f.Bytes)) > FileReferenceThreshold + } + return len(f.Content) > FileReferenceThreshold +} + +// AsReference is the file as it is sent to an engine that reports FileReferencesCapability: a copy with the digest +// and size in place of the content, and the content the controller puts into the bucket under ObjectName(digest). +// A file that does not travel by digest is refused, not passed through, so a caller cannot send a small file empty. +func (f *File) AsReference() (File, []byte, error) { + if !f.TravelsByDigest() { + return File{}, nil, fmt.Errorf("file %s does not travel by digest", f.ID) + } + body, err := f.Body() + if err != nil { + return File{}, nil, err + } + ref := *f + ref.Content, ref.Bytes = "", "" + ref.Digest, ref.Size = DigestOf(body), int64(len(body)) + return ref, body, nil +} + +// Verify checks what was fetched against the digest and size the file names; the node-engine writes the file only +// after it returns nil. A mismatch is the apply's failure, never a partial apply (ADR 0486 §1). +func (f *File) Verify(body []byte) error { + if !f.IsReference() { + return fmt.Errorf("file %s is not given by reference", f.ID) + } + if int64(len(body)) != f.Size { + return fmt.Errorf("file %s: the object is %d bytes and the declaration says %d", f.ID, len(body), f.Size) + } + if got := DigestOf(body); got != f.Digest { + return fmt.Errorf("file %s: the object hashes to %s and the declaration names %s", f.ID, got, f.Digest) + } + return nil +} diff --git a/go/declaration/filereference_test.go b/go/declaration/filereference_test.go new file mode 100644 index 0000000..957a054 --- /dev/null +++ b/go/declaration/filereference_test.go @@ -0,0 +1,129 @@ +package declaration + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "reflect" + "strings" + "testing" +) + +func refFile(extra string) string { + return `{"declaration":1,"resources":[{"id":"m.f","type":"file","path":"/etc/m","digest":"sha256:` + + strings.Repeat("a", 64) + `","size":70000` + extra + `}]}` +} + +// 1694/AC1: a reference round trips, and an invalid one is refused naming the field. +func TestAFileReferenceRoundTripsAndABadOneNamesItsField(t *testing.T) { + d, err := Parse([]byte(refFile(""))) + if err != nil { + t.Fatal(err) + } + f := d.Resources[0].(*File) + if !f.IsReference() || f.Size != 70000 || f.Digest != "sha256:"+strings.Repeat("a", 64) { + t.Fatalf("read: %+v", f) + } + raw, _ := json.Marshal(d.Resources[0]) + d2, err := Parse([]byte(`{"declaration":1,"resources":[` + string(raw) + `]}`)) + if err != nil || !reflect.DeepEqual(d2.Resources[0], d.Resources[0]) { + t.Fatalf("round trip: %v %s", err, raw) + } + for want, raw := range map[string]string{ + "digest": strings.Replace(refFile(""), strings.Repeat("a", 64), "abc", 1), + "size": strings.Replace(refFile(""), `"size":70000`, `"size":0`, 1), + "lower-case": strings.Replace(refFile(""), strings.Repeat("a", 64), strings.Repeat("A", 64), 1), + "content": refFile(`,"content":"x"`), + "bytes": refFile(`,"bytes":"eA=="`), + "whole": refFile(`,"into":"block"`), + } { + _, err := Parse([]byte(raw)) + if err == nil || !strings.Contains(err.Error(), want) { + t.Errorf("want a refusal naming %q, got %v", want, err) + } + } + // a size alone is not a reference + if _, err := Parse([]byte(`{"declaration":1,"resources":[{"id":"m.f","type":"file","path":"/e","size":5}]}`)); err == nil { + t.Error("a size without a digest was accepted") + } +} + +// 1694/AC2: a declaration as today's SDK writes it reads unchanged and its bytes carry no new key. +func TestAnInlineFileIsReadAndWrittenAsBefore(t *testing.T) { + const in = `{"id":"m.f","type":"file","path":"/e","content":"hello"}` + d, err := Parse([]byte(`{"declaration":1,"resources":[` + in + `]}`)) + if err != nil { + t.Fatal(err) + } + out, _ := json.Marshal(d.Resources[0]) + if string(out) != in { + t.Fatalf("signed bytes changed: %s", out) + } + if d.Resources[0].(*File).IsReference() { + t.Fatal("inline read as a reference") + } +} + +// 1694/AC3: the controller's helper makes a reference of a big file and the engine's check refuses a changed byte. +func TestABigFileBecomesAReferenceAndAChangedByteFailsTheCheck(t *testing.T) { + big := strings.Repeat("x", FileReferenceThreshold+1) + f := &File{ID: "m.f", Type: TypeFile, Path: "/e", Content: big} + if !f.TravelsByDigest() { + t.Fatal("a file over the threshold stays inline") + } + ref, body, err := f.AsReference() + if err != nil || ref.Content != "" || ref.Size != int64(len(big)) || string(body) != big { + t.Fatalf("%v %+v", err, ref) + } + if p := ref.validate("f", false); len(p) > 0 { + t.Fatalf("the reference is refused: %v", p) + } + if name, err := ObjectName(ref.Digest); err != nil || len(name) != 64 { + t.Fatalf("object name %q %v", name, err) + } + if err := ref.Verify(body); err != nil { + t.Fatal(err) + } + bad := bytes.Clone(body) + bad[7] ^= 1 + if err := ref.Verify(bad); err == nil || !strings.Contains(err.Error(), "hashes to") { + t.Fatalf("a changed byte: %v", err) + } + if err := ref.Verify(body[1:]); err == nil || !strings.Contains(err.Error(), "bytes") { + t.Fatalf("a short object: %v", err) + } +} + +// 1694/AC4: what stays inline, and bytes files travel decoded. +func TestWhichFilesTravelByDigest(t *testing.T) { + small := strings.Repeat("x", FileReferenceThreshold) + big := strings.Repeat("x", FileReferenceThreshold+1) + for name, f := range map[string]File{ + "at the threshold": {Content: small}, + "sealed": {Sealed: "s"}, + "secrets": {Content: big, Secrets: map[string]string{"a": "b"}}, + "into": {Content: big, Into: IntoBlock}, + } { + if f.TravelsByDigest() { + t.Errorf("%s travels by digest", name) + } + if _, _, err := f.AsReference(); err == nil { + t.Errorf("%s was made a reference", name) + } + } + raw := bytes.Repeat([]byte{0, 1, 2}, FileReferenceThreshold) + f := File{ID: "m.b", Bytes: base64.StdEncoding.EncodeToString(raw)} + ref, body, err := f.AsReference() + if err != nil || !bytes.Equal(body, raw) || ref.Size != int64(len(raw)) || ref.Digest != DigestOf(raw) { + t.Fatalf("bytes file: %v", err) + } +} + +func TestTheSharedNames(t *testing.T) { + if FileBucket != "declaration-files" || FileReferencesCapability != "file-references" || FileReferenceThreshold != 65536 { + t.Fatal("a name changed") + } + if _, err := ObjectName("sha256:zz"); err == nil { + t.Fatal("bad digest named an object") + } +} -- 2.54.0 From 6c857e1c025ff6cd1c0eddb80af1382da2095264 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 21 Oct 2026 14:23:48 +0200 Subject: [PATCH 2/2] A bytes file of exactly 64 KiB stays inline: take its exact decoded length (issue 1694) --- go/declaration/filereference.go | 4 +++- go/declaration/filereference_test.go | 7 +++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/go/declaration/filereference.go b/go/declaration/filereference.go index 1bbdcdd..166022a 100644 --- a/go/declaration/filereference.go +++ b/go/declaration/filereference.go @@ -85,7 +85,9 @@ func (f *File) TravelsByDigest() bool { return false } if f.Bytes != "" { - return base64.StdEncoding.DecodedLen(len(f.Bytes)) > FileReferenceThreshold + // DecodedLen counts padding, so the exact length is taken off the decoded bytes. + body, err := f.Body() + return err == nil && len(body) > FileReferenceThreshold } return len(f.Content) > FileReferenceThreshold } diff --git a/go/declaration/filereference_test.go b/go/declaration/filereference_test.go index 957a054..6bf9be9 100644 --- a/go/declaration/filereference_test.go +++ b/go/declaration/filereference_test.go @@ -111,6 +111,13 @@ func TestWhichFilesTravelByDigest(t *testing.T) { t.Errorf("%s was made a reference", name) } } + exact := File{ID: "m.e", Bytes: base64.StdEncoding.EncodeToString(bytes.Repeat([]byte{7}, FileReferenceThreshold))} + if exact.TravelsByDigest() { + t.Error("a bytes file of exactly the threshold travels by digest") + } + if _, _, err := exact.AsReference(); err == nil { + t.Error("a bytes file of exactly the threshold was made a reference") + } raw := bytes.Repeat([]byte{0, 1, 2}, FileReferenceThreshold) f := File{ID: "m.b", Bytes: base64.StdEncoding.EncodeToString(raw)} ref, body, err := f.AsReference() -- 2.54.0