File a scan from the store by the link the device process signs, not by parts over the bus (issue 1601)

This commit is contained in:
2026-10-21 15:01:09 +02:00
parent 278014fd79
commit cb226c6fef
11 changed files with 261 additions and 76 deletions
+2 -2
View File
@@ -122,8 +122,8 @@ the login page again. The logout follows a logout link or form on the page; a pa
## The event
`scan.added`, said by this module, on the wire `mesh.mod.brother.event.scan.added`; a consumer declares `brother.scan.added`. Its body:
`path`, `node`, `bytes`, `sha256`, `width-mm`, `height-mm`, `resolution` (dpi, as the scanner gave it),
`scan.added`, said by the device process once the PDF is stored, before the file is written (the tool side files it afterwards, from the store, by a link the device signed; the verb's answer adds the file's `path`), on the wire `mesh.mod.brother.event.scan.added`; a consumer declares `brother.scan.added`. Its body:
`object` (the key of the stored PDF), `node`, `bytes`, `sha256`, `width-mm`, `height-mm`, `resolution` (dpi, as the scanner gave it),
`color-mode`, `cropped`, `pages` (1 for the glass), `scanned-at` (RFC 3339, UTC).
## Files it does not know
+17 -5
View File
@@ -11,6 +11,7 @@ import (
"encoding/json"
"errors"
"fmt"
"net/http"
"os"
"os/signal"
"strings"
@@ -19,6 +20,7 @@ import (
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/objectlink"
"git.novox.be/novox/mesh-sdk/go/process"
"github.com/nats-io/nats.go/jetstream"
@@ -247,10 +249,14 @@ func (w *deviceWorker) keep(ctx context.Context, objects Objects, set Settings,
// keepOne keeps one captured PDF and announces it.
func (w *deviceWorker) keepOne(ctx context.Context, objects Objects, set Settings, c Capture) (map[string]any, error) {
// A scan the tool could not take is not kept nor announced: an announced scan is always filed (issue 1539).
if most := MostAnswerParts * AnswerPartSize; len(c.PDF) > most {
return nil, fmt.Errorf("the scan's PDF is %d bytes, over the %d the tool can fetch (%d parts of %d): it is not kept; scan with a lower resolution or fewer sheets",
len(c.PDF), most, MostAnswerParts, AnswerPartSize)
// A scan over a document's limit is not kept nor announced: an announced scan is always filed (issue 1539).
if len(c.PDF) > int(objectlink.MaxDocument) {
return nil, fmt.Errorf("the scan's PDF is %d bytes, over the %d a document may be (novox/hq ADR 0468 §3): it is not kept; scan with a lower resolution or fewer sheets",
len(c.PDF), objectlink.MaxDocument)
}
signer, ok := objects.(Presigner)
if !ok {
return nil, errors.New("the bucket cannot sign a link: the scan is not kept")
}
body := c.Body(w.Node, set.ColorMode)
key := body["object"].(string)
@@ -260,7 +266,13 @@ func (w *deviceWorker) keepOne(ctx context.Context, objects Objects, set Setting
if err := w.Emit(body, c.SHA256); err != nil {
return nil, fmt.Errorf("the scan is kept as %s, but %s could not be said on the bus: %w", key, EventAdded, err)
}
answer := map[string]any{"key": key, "size": len(c.PDF), "parts": answerPartCount(len(c.PDF))}
// The answer carries a link to read the stored PDF, signed with the binding's key (novox/hq ADR 0468 §4): the tool
// side reads the scan by it and files it; no byte of the document is in the answer.
link, err := signer.Link(http.MethodGet, key, objectlink.MaxExpires)
if err != nil {
return nil, fmt.Errorf("the scan is kept as %s, but a link to it could not be signed: %w", key, err)
}
answer := map[string]any{"key": key, "size": len(c.PDF), "link": link, "link-seconds": int(objectlink.MaxExpires / time.Second)}
for k, v := range body {
answer[k] = v
}
@@ -115,7 +115,7 @@ func TestScannerDocumentGivesEachPartOfAKeptScan(t *testing.T) {
}
}
// 1453/AC10: a scan of more than one part is fetched whole by scanner.scan, joined, checked and filed.
// 1453/AC10, 1601: a scan the size of several answer parts is read whole from the store by scanner.scan, checked and filed.
func TestScanNowFetchesAScanOfSeveralParts(t *testing.T) {
f := newFakeScanner(t, fixture(t, "scan.jpg"))
r, _ := scanRig(t, f, Settings{Crop: false})
+19
View File
@@ -24,6 +24,8 @@ import (
"sort"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/objectlink"
)
// Objects is the module's bucket.
@@ -55,6 +57,13 @@ func Store(ctx context.Context, o Objects, key, mediaType string, data []byte) e
return o.Put(ctx, key, mediaType, data)
}
// Presigner is a store that signs a link to one of its objects (novox/hq ADR 0468 §3): the device process's, with the
// binding's key, which reaches this module's bucket only.
type Presigner interface {
// Link is a link for GET or PUT of one key that lives for the given time (at most ten minutes).
Link(method, key string, expires time.Duration) (string, error)
}
// StoreFacts is the file the mesh writes from the s3-bucket binding and its secret.
type StoreFacts struct {
Scheme string `json:"scheme"`
@@ -114,6 +123,16 @@ func NewS3(f StoreFacts) *S3 {
func (s *S3) Bucket() string { return s.Name }
// Link signs a link for one object of the bucket with the binding's key, for the host the binding gives.
func (s *S3) Link(method, key string, expires time.Duration) (string, error) {
now := time.Now
if s.Now != nil {
now = s.Now
}
return objectlink.Presign(objectlink.Request{Method: method, Endpoint: s.Endpoint, Bucket: s.Name, Key: key,
AccessKey: s.Access, SecretKey: s.Secret, Region: s.Region, Expires: expires, Now: now()})
}
func (s *S3) do(ctx context.Context, method, key, mediaType string, body []byte) (*http.Response, error) {
u, err := url.Parse(s.Endpoint + "/" + s.Name + "/" + key)
if err != nil {
@@ -1,6 +1,7 @@
package main
import (
"bytes"
"context"
"fmt"
"io"
@@ -9,9 +10,11 @@ import (
"os"
"path/filepath"
"sort"
"strconv"
"strings"
"sync"
"testing"
"time"
)
// memObjects is the bucket in memory, for the tests of what the device process keeps.
@@ -19,6 +22,8 @@ type memObjects struct {
mu sync.Mutex
kept map[string][]byte
puts int
// corrupt, when set, makes a GET of a link answer bytes that are not the kept ones.
corrupt bool
}
func newMemObjects() *memObjects { return &memObjects{kept: map[string][]byte{}} }
@@ -68,6 +73,29 @@ func (m *memObjects) Delete(_ context.Context, key string) error {
return nil
}
// Link is a link the bucket in memory serves itself (RoundTrip): the key in the path and the life in the query, as a
// presigned link says them (X-Amz-Expires).
func (m *memObjects) Link(method, key string, expires time.Duration) (string, error) {
return "http://" + m.Bucket() + ".objects.test/" + key + "?X-Amz-Expires=" + strconv.Itoa(int(expires/time.Second)), nil
}
// RoundTrip answers a GET of a link made by Link with the object kept under its key.
func (m *memObjects) RoundTrip(r *http.Request) (*http.Response, error) {
m.mu.Lock()
b, ok := m.kept[strings.TrimPrefix(r.URL.Path, "/")]
corrupt := m.corrupt
m.mu.Unlock()
if corrupt && ok {
b = append(append([]byte(nil), b...), 0)
}
status := 200
if !ok {
status = 404
}
return &http.Response{StatusCode: status, Body: io.NopCloser(bytes.NewReader(b)), ContentLength: int64(len(b)),
Header: http.Header{}, Request: r}, nil
}
func (m *memObjects) keys() []string {
m.mu.Lock()
defer m.mu.Unlock()
+7 -1
View File
@@ -4,6 +4,7 @@ import (
"encoding/json"
"fmt"
"github.com/nats-io/nats.go/jetstream"
"net/http"
"os"
"path/filepath"
"sync"
@@ -144,7 +145,12 @@ func newRig(t *testing.T, settings, password string) *printerRig {
bus := &fakeAskBus{worker: w, answers: answers}
old, oldWait, oldScan, oldPoll := askBus, answerWait, scanWait, answerPoll
askBus, answerWait, scanWait, answerPoll = bus, 5*time.Second, 5*time.Second, 5*time.Millisecond
t.Cleanup(func() { askBus, answerWait, scanWait, answerPoll = old, oldWait, oldScan, oldPoll })
oldFetch := fetchClient
fetchClient = &http.Client{Transport: objects}
t.Cleanup(func() {
askBus, answerWait, scanWait, answerPoll = old, oldWait, oldScan, oldPoll
fetchClient = oldFetch
})
tl := tools(file, func() time.Time { return fixedNow })
return &printerRig{tl: tl, run: tool(t, tl, "printer.settings").Run, bus: bus, worker: w, answers: answers, objects: objects, said: &out, mu: &mu}
}
+27 -31
View File
@@ -1,12 +1,15 @@
package main
import (
"bytes"
"context"
"encoding/base64"
"fmt"
"net/http"
"strings"
"sync"
"time"
"git.novox.be/novox/mesh-sdk/go/objectlink"
)
// EventAdded is the module's event: a scan was kept in the bucket (ADR 0464, ADR 0468 clause 2), said by the device
@@ -280,26 +283,19 @@ func (s *Service) fileAnswer(ctx context.Context, ans map[string]any) (map[strin
func (s *Service) fileOne(ctx context.Context, ans map[string]any) (map[string]any, error) {
key, _ := ans["key"].(string)
sum, _ := ans["sha256"].(string)
parts, _ := ans["parts"].(float64)
if key == "" || sum == "" || parts < 1 || parts > MostAnswerParts {
return nil, fmt.Errorf("the device process's answer to the scan names no stored scan: %v", ans)
link, _ := ans["link"].(string)
if key == "" || sum == "" || link == "" {
return nil, fmt.Errorf("the device process's answer to the scan names no stored scan or no link to it: %v", withoutLink(ans))
}
got := make([][]byte, 0, int(parts))
for i := 1; i <= int(parts); i++ {
p, err := s.fetchPart(ctx, key, i)
if err != nil && ctx.Err() == nil {
// A part fetch is a read, safe to repeat: one retry rides out a short bus outage (issue 1536).
p, err = s.fetchPart(ctx, key, i)
}
if err != nil {
return nil, fmt.Errorf("the scan is kept as %s, but its part %d could not be fetched: %w", key, i, err)
}
got = append(got, p)
}
pdf, err := joinParts(got, sum)
if err != nil {
return nil, fmt.Errorf("the scan is kept as %s: %w", key, err)
// The scan is read from the store by the link the device process signed, never through the bus (ADR 0468 §4), and
// must hash to the scan's SHA-256 before anything is written.
var buf bytes.Buffer
gctx, cancel := context.WithTimeout(ctx, 5*time.Minute)
defer cancel()
if _, err := objectlink.Get(gctx, fetchClient, link, &buf, objectlink.MaxDocument, sum); err != nil {
return nil, fmt.Errorf("the scan is kept as %s, but it could not be read from the store: %w", key, err)
}
pdf := buf.Bytes()
at, err := time.Parse(time.RFC3339, fmt.Sprint(ans["scanned-at"]))
if err != nil {
at = s.Now()
@@ -310,7 +306,7 @@ func (s *Service) fileOne(ctx context.Context, ans map[string]any) (map[string]a
}
body := map[string]any{}
for k, v := range ans {
if k != "key" && k != "size" && k != "parts" {
if k != "key" && k != "size" && k != "link" && k != "link-seconds" {
body[k] = v
}
}
@@ -318,20 +314,20 @@ func (s *Service) fileOne(ctx context.Context, ans map[string]any) (map[string]a
return body, nil
}
// fetchPart is one part of a stored object, asked of the device process and decoded.
func (s *Service) fetchPart(ctx context.Context, object string, part int) ([]byte, error) {
ans, err := s.call(ctx, Ask{Op: OpDocument, Object: object, Part: part}, s.AnswerWait)
if err != nil {
return nil, err
// withoutLink is an answer said in an error without its link, which carries a signature.
func withoutLink(ans map[string]any) map[string]any {
out := map[string]any{}
for k, v := range ans {
if k != "link" {
out[k] = v
}
}
data, _ := ans["data"].(string)
raw, err := base64.StdEncoding.DecodeString(data)
if err != nil {
return nil, fmt.Errorf("the part is not base64: %w", err)
}
return raw, nil
return out
}
// fetchClient reads the links the device process signs; a test puts a client of its own in its place.
var fetchClient = &http.Client{}
// ScanStatus says whether the scanner answers, and what it is doing. A scanner that does not answer is an
// answer, not an error: that is how the module's health is read.
func (s *Service) ScanStatus(ctx context.Context) (map[string]any, error) {
@@ -0,0 +1,157 @@
package main
import (
"encoding/json"
"net/url"
"os"
"path/filepath"
"strconv"
"strings"
"testing"
"time"
)
// 1601/AC1: scanner.scan reads the stored PDF by the GET link the device's answer carries (never by parts) and writes it
// into the scan folder; the answer is the event's body plus the path.
func TestTheScanIsReadFromTheStoreByTheLinkAndFiled(t *testing.T) {
f := newFakeScanner(t, fixture(t, "scan.jpg"))
r, folder := scanRig(t, f, Settings{Crop: false})
body, err := r.scan(t)
if err != nil {
t.Fatal(err)
}
if len(*r.said) != 1 {
t.Fatalf("%d events", len(*r.said))
}
stored, err := r.objects.Get(t.Context(), (*r.said)[0].body["object"].(string))
if err != nil {
t.Fatal(err)
}
filed, err := os.ReadFile(body["path"].(string))
if err != nil || string(filed) != string(stored) {
t.Fatalf("the filed scan is not the stored one: %v", err)
}
if filepath.Dir(body["path"].(string)) != folder {
t.Errorf("filed in %s, want %s", filepath.Dir(body["path"].(string)), folder)
}
for _, a := range r.bus.published {
if a.Op == OpDocument {
t.Errorf("a part of the scan was asked of the device process over the bus")
}
}
for k := range (*r.said)[0].body {
if body[k] == nil && k != "path" {
t.Errorf("the event's %s is not in the answer", k)
}
}
if body["link"] != nil || body["parts"] != nil || body["key"] != nil {
t.Errorf("the answer carries the transport's own fields: %v", body)
}
}
// 1601/AC2: the device's answer carries a link that expires within ten minutes and no bytes of the document; the real
// store signs it with the binding's key for the host the binding gives.
func TestTheDeviceAnswerCarriesALinkAndNoBytes(t *testing.T) {
f := newFakeScanner(t, fixture(t, "scan.jpg"))
r, _ := scanRig(t, f, Settings{Crop: false})
if _, err := r.scan(t); err != nil {
t.Fatal(err)
}
var ans map[string]any
for _, raw := range r.answers.m {
json.Unmarshal(raw, &ans)
}
link, _ := ans["link"].(string)
u, err := url.Parse(link)
if err != nil || link == "" {
t.Fatalf("answer %v", ans)
}
secs, _ := strconv.Atoi(u.Query().Get("X-Amz-Expires"))
if secs < 1 || time.Duration(secs)*time.Second > 10*time.Minute {
t.Errorf("the link lives %d s", secs)
}
raw, _ := json.Marshal(ans)
if ans["data"] != nil || len(raw) > 4096 {
t.Errorf("the answer is %d bytes and carries data %v", len(raw), ans["data"] != nil)
}
// The real store: the binding's key signs, for the binding's host and port, in the module's bucket.
s3 := NewS3(StoreFacts{Scheme: "http", At: "novox.internal", Port: "20001", Bucket: "mesh-shanks-brother", Region: "eu-west",
Key: "binding-key", Secret: "binding-secret"})
s3.Now = func() time.Time { return fixedNow }
got, err := s3.Link("GET", "abc.pdf", 10*time.Minute)
if err != nil {
t.Fatal(err)
}
gu, _ := url.Parse(got)
if gu.Host != "novox.internal:20001" || gu.Path != "/mesh-shanks-brother/abc.pdf" ||
!strings.HasPrefix(gu.Query().Get("X-Amz-Credential"), "binding-key/") || gu.Query().Get("X-Amz-Expires") != "600" {
t.Errorf("link %s", got)
}
if _, err := s3.Link("GET", "abc.pdf", 11*time.Minute); err == nil {
t.Error("a link over ten minutes was signed")
}
}
// 1601/AC3: a body whose SHA-256 differs from the scan's writes no file and answers the mismatch.
func TestABodyOfAnotherHashWritesNoFile(t *testing.T) {
f := newFakeScanner(t, fixture(t, "scan.jpg"))
r, folder := scanRig(t, f, Settings{Crop: false})
r.objects.corrupt = true
_, err := r.scan(t)
if err == nil || !strings.Contains(err.Error(), "hashes to") {
t.Fatalf("error = %v", err)
}
if e, _ := os.ReadDir(folder); len(e) != 0 {
t.Errorf("a scan of the wrong hash was filed: %v", e)
}
}
// 1601/AC4: brother.scan.added is said before the file is written, once the scan is stored, and is under 4 KiB.
func TestTheEventIsSaidBeforeTheFileIsWritten(t *testing.T) {
f := newFakeScanner(t, fixture(t, "scan.jpg"))
r, folder := scanRig(t, f, Settings{Crop: false})
var filedWhenSaid = -1
emit := r.worker.Emit
r.worker.Emit = func(body map[string]any, id string) error {
e, _ := os.ReadDir(folder)
filedWhenSaid = len(e)
return emit(body, id)
}
if _, err := r.scan(t); err != nil {
t.Fatal(err)
}
if filedWhenSaid != 0 {
t.Errorf("%d files in the folder when the event was said, want 0", filedWhenSaid)
}
said := (*r.said)[0]
if !said.seen {
t.Error("the event was said before the scan was stored")
}
raw, _ := json.Marshal(said.body)
if len(raw) >= 4096 || said.body["path"] != nil {
t.Errorf("the event is %d bytes, path %v", len(raw), said.body["path"])
}
}
// 1601/AC5: no code of the device process writes a file outside its state directory: the only writes to the disk in the
// module's code are the filing of a scan (file.go), made by the tool side (scan.go).
func TestOnlyTheFilingWritesFiles(t *testing.T) {
writes := []string{"os.WriteFile", "os.Create(", "os.OpenFile", "os.MkdirAll", "os.Mkdir(", "os.Rename", "os.CreateTemp", "File(s.Settings.Folder"}
files, _ := filepath.Glob("*.go")
for _, name := range files {
if strings.HasSuffix(name, "_test.go") || name == "file.go" {
continue
}
raw, _ := os.ReadFile(name)
for _, w := range writes {
if strings.Contains(string(raw), w) && !(name == "scan.go" && w == "File(s.Settings.Folder") {
t.Errorf("%s writes with %s: only file.go does, and scan.go (the tool side) calls it", name, w)
}
}
}
raw, _ := os.ReadFile("deviceproc.go")
if strings.Contains(string(raw), "File(") {
t.Error("deviceproc.go files a scan")
}
}
@@ -362,38 +362,3 @@ func (b *flakyBus) Publish(id string, body any) error {
}
return b.fakeAskBus.Publish(id, body)
}
// 1536: a part fetch that fails once is asked again, and the scan is filed; one that fails twice is a failure
// naming the stored object, so it can be filed later.
func TestAFailedPartFetchIsAskedOnceMoreBeforeTheScanFails(t *testing.T) {
f := newFakeScanner(t, fixture(t, "scan.jpg"))
r, _ := scanRig(t, f, Settings{Crop: false})
r.bus = nil
flaky := &flakyBus{fakeAskBus: &fakeAskBus{worker: r.worker, answers: r.answers}, fails: 1}
askBus = flaky
body, err := r.scan(t)
if err != nil {
t.Fatalf("one failed part fetch ended the scan: %v", err)
}
if _, err := os.Stat(body["path"].(string)); err != nil {
t.Errorf("the scan is not filed: %v", err)
}
f2 := newFakeScanner(t, fixture(t, "scan.jpg"))
r2, folder := scanRig(t, f2, Settings{Crop: false})
askBus = &flakyBus{fakeAskBus: &fakeAskBus{worker: r2.worker, answers: r2.answers}, fails: 2}
_, err = r2.scan(t)
if err == nil {
t.Fatal("two failed part fetches did not fail the scan")
}
key := ""
if got := *r2.said; len(got) == 1 {
key, _ = got[0].body["object"].(string)
}
if key == "" || !strings.Contains(err.Error(), key) {
t.Errorf("the failure %q does not name the stored object %q", err, key)
}
if entries, _ := os.ReadDir(folder); len(entries) != 0 {
t.Errorf("a scan whose parts failed was filed: %v", entries)
}
}
+1 -1
View File
@@ -3,7 +3,7 @@ module brother
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020060201-88581d0acaef
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021122348-6c857e1c025f
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
+2
View File
@@ -1,5 +1,7 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020060201-88581d0acaef h1:xy3wuGWJpqMAXYdXolocYG6w9xnWqBltVUffoaziIKo=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020060201-88581d0acaef/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021122348-6c857e1c025f h1:7J/9jOfSfjfYpf8MOj96HwQRViQUkBNMb71RHP2mjn8=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021122348-6c857e1c025f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=