From d4967f186e9f240ae667b1dc2b58a5e64dc5bfb3 Mon Sep 17 00:00:00 2001 From: mesh-tools Date: Tue, 20 Oct 2026 01:44:22 +0000 Subject: [PATCH] Test the indexer's manifests, the new fields of modules and candidates, and the fpset (issue 1357) --- go/delivery/indexer2_test.go | 321 +++++++++++++++++++++++++++++++++++ 1 file changed, 321 insertions(+) create mode 100644 go/delivery/indexer2_test.go diff --git a/go/delivery/indexer2_test.go b/go/delivery/indexer2_test.go new file mode 100644 index 0000000..4aa1fe0 --- /dev/null +++ b/go/delivery/indexer2_test.go @@ -0,0 +1,321 @@ +package delivery + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "strings" + "testing" +) + +// The values below that today's SDK wrote were produced by EncodeCandidate(sampleCandidate()) and EncodeModule at +// mesh-sdk 5786ba19 (source_test.go's samples), and are pasted as the bytes it wrote. +const ( + oldCandidateJSON = `{"v":1,"group":"feat/x","base":12,"heads":"4b5491557853","pulls":[{"repository":"novox/mesh-sdk","number":36,"head":"aaaa","modules":["mesh-mail"]},{"repository":"novox/mesh-control","number":7,"head":"bbbb","modules":["mesh-controller"]}]}` + oldModuleJSON = `{"v":1,"fp":"src1:abc","src":5,"repository":"novox/mesh-sdk","commit":"cccc"}` + oldNoneJSON = `{"v":1,"none":"a build that resolves packages when built","repository":"novox/mesh-sdk","commit":"cccc"}` +) + +// The reader structs of today's shape (source.go at 5786ba19): what the gate and the view are built on until they move +// to this SDK. They decode with plain json.Unmarshal after the version check, as decode does. +type oldModule struct { + V int `json:"v"` + FP string `json:"fp,omitempty"` + None string `json:"none,omitempty"` + Src uint64 `json:"src,omitempty"` + Repository string `json:"repository"` + Commit string `json:"commit"` +} + +type oldCandidate struct { + V int `json:"v"` + Group string `json:"group"` + Base uint64 `json:"base"` + HeadsHash string `json:"heads"` + Pulls []struct { + Head + Modules []string `json:"modules"` + } `json:"pulls"` +} + +func fp2of(c byte) string { return SourcePrefix2 + strings.Repeat(string(c), 64) } + +func digestOf(c byte) string { return strings.Repeat(string(c), 64) } + +func newCandidate() Candidate { + c := sampleCandidate() + c.Pulls[0].Fps = map[string]CandidateModule{"mesh-mail": {FP2: fp2of('a'), Manifest: digestOf('1')}} + c.Pulls[1].Fps = map[string]CandidateModule{"mesh-controller": {None2: "no lock", Manifest: digestOf('2')}} + c.FPSet = FingerprintSet(c.Pulls) + return c +} + +// 1357/AC1: the digest is the hex sha256 of exactly the bytes; the key is that hex; a key that is not 64 lowercase hex +// is refused. +func TestManifestKeys(t *testing.T) { + raw := []byte("{\n \"module\": \"x\"\n}\n") + sum := sha256.Sum256(raw) + want := hex.EncodeToString(sum[:]) + if got := ManifestDigest(raw); got != want { + t.Errorf("ManifestDigest = %s, want %s", got, want) + } + key, err := ManifestsKey(want) + if err != nil || key != want { + t.Errorf("ManifestsKey = %q, %v", key, err) + } + back, err := ParseManifestsKey(key) + if err != nil || back != want { + t.Errorf("ParseManifestsKey = %q, %v", back, err) + } + for _, bad := range []string{"", "abc", strings.ToUpper(want), want + "0", want[:63], "sha256:" + want, strings.Repeat("g", 64)} { + if _, err := ManifestsKey(bad); err == nil || !strings.Contains(err.Error(), "\""+bad+"\"") { + t.Errorf("ManifestsKey(%q) = %v, want a refusal naming it", bad, err) + } + if _, err := ParseManifestsKey(bad); err == nil || !strings.Contains(err.Error(), "\""+bad+"\"") { + t.Errorf("ParseManifestsKey(%q) = %v, want a refusal naming it", bad, err) + } + } + if ManifestsBucket != "mesh-delivery-indexer_manifests" || ManifestsHistory != 1 { + t.Errorf("bucket %s history %d", ManifestsBucket, ManifestsHistory) + } +} + +// 1357/AC2 (the shape of it; the 21 KB case is in manifests_mailu_test.go): a manifests value encodes and decodes with +// its manifest byte for byte, whatever its spacing, escapes and characters, and its digest is the key. +func TestManifestRoundTripBytesForByte(t *testing.T) { + // odd spacing, a tab, an escaped newline and quote, an em dash and an HTML-looking character, a trailing newline: + // what a JSON re-encoder would change. + raw := []byte("{\n\t\"module\": \"x\",\n \"why\": \"a — b & \\n \\\" d é\",\n\n \"list\": [ \"a\",\"b\" ]\n}\n\n") + m := Manifest{Module: "x", JSON: string(raw)} + data, err := EncodeManifest(m) + if err != nil { + t.Fatal(err) + } + got, err := DecodeManifest(data) + if err != nil { + t.Fatal(err) + } + if got.JSON != string(raw) || ManifestDigest([]byte(got.JSON)) != ManifestDigest(raw) || got.Module != "x" || got.V != Version { + t.Errorf("the manifest changed on the way: %q", got.JSON) + } + for name, bad := range map[string]Manifest{ + "no module": {JSON: string(raw)}, + "a dotted module": {Module: "a.b", JSON: string(raw)}, + "no manifest": {Module: "x"}, + "not JSON": {Module: "x", JSON: "{nope"}, + "not UTF-8": {Module: "x", JSON: "{\"a\":\"\xff\"}"}, + } { + if _, err := EncodeManifest(bad); err == nil { + t.Errorf("a manifest with %s was encoded", name) + } + } + if _, err := DecodeManifest([]byte(`{"v":2,"module":"m","manifest":"{}"}`)); !errors.Is(err, ErrUnknownVersion) { + t.Errorf("version 2: %v", err) + } +} + +// 1357/AC3: a Module and a Candidate written by today's SDK decode and check under the new one, new fields empty. +func TestOldValuesDecodeUnderTheNewSDK(t *testing.T) { + c, err := DecodeCandidate([]byte(oldCandidateJSON)) + if err != nil { + t.Fatal(err) + } + if c.FPSet != "" || c.Pulls[0].Fps != nil || c.Pulls[1].Fps != nil || c.Group != "feat/x" { + t.Errorf("a candidate of today grew fields: %+v", c) + } + for _, in := range []string{oldModuleJSON, oldNoneJSON} { + m, err := DecodeModule([]byte(in)) + if err != nil { + t.Fatal(err) + } + if m.FP2 != "" || m.None2 != "" || m.Manifest != "" { + t.Errorf("a module of today grew fields: %+v", m) + } + } + // And the new SDK writes today's bytes for today's value: nothing new appears when nothing new is set. + data, err := EncodeCandidate(sampleCandidate()) + if err != nil || string(data) != oldCandidateJSON { + t.Errorf("EncodeCandidate of an old value = %s, %v", data, err) + } + data, err = EncodeModule(Module{FP: "src1:abc", Src: 5, Repository: "novox/mesh-sdk", Commit: "cccc"}) + if err != nil || string(data) != oldModuleJSON { + t.Errorf("EncodeModule of an old value = %s, %v", data, err) + } +} + +// 1357/AC4: a value with the new fields decodes, under a reader struct of today's shape, with no error. +func TestNewValuesDecodeUnderTheOldReader(t *testing.T) { + m := Module{FP: "src1:abc", FP2: fp2of('b'), Manifest: digestOf('3'), Src: 5, Repository: "novox/mesh-sdk", Commit: "cccc"} + data, err := EncodeModule(m) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(data), `"fp2"`) || !strings.Contains(string(data), `"manifest"`) { + t.Fatalf("the new fields are not in %s", data) + } + if err := checkVersion(data); err != nil { + t.Fatal(err) + } + var om oldModule + if err := json.Unmarshal(data, &om); err != nil || om.FP != "src1:abc" || om.Commit != "cccc" { + t.Errorf("the old reader of a module: %+v %v", om, err) + } + data, err = EncodeCandidate(newCandidate()) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(data), `"fpset"`) || !strings.Contains(string(data), `"fp2"`) { + t.Fatalf("the new fields are not in %s", data) + } + var oc oldCandidate + if err := json.Unmarshal(data, &oc); err != nil || oc.Group != "feat/x" || len(oc.Pulls) != 2 || oc.Pulls[0].Modules[0] != "mesh-mail" { + t.Errorf("the old reader of a candidate: %+v %v", oc, err) + } + // and the new reader reads it back whole + got, err := DecodeCandidate(data) + want := newCandidate() + want.V = Version + if err != nil || !SameCandidate(got, want) { + t.Errorf("round trip: %+v %v", got, err) + } + gm, err := DecodeModule(mustJSON(t, m)) + m.V = Version + if err != nil || !SameModule(gm, m) { + t.Errorf("module round trip: %+v %v", gm, err) + } +} + +func mustJSON(t *testing.T, m Module) []byte { + t.Helper() + data, err := EncodeModule(m) + if err != nil { + t.Fatal(err) + } + return data +} + +// 1357/AC5: a Module with an fp2 not of the new form's prefix, or with both fp2 and none2, is refused naming it; a +// Candidate whose fpset is not the hash of its fingerprints is refused. +func TestNewFieldsAreChecked(t *testing.T) { + wantErr := func(name string, err error, words ...string) { + t.Helper() + if err == nil { + t.Errorf("%s accepted", name) + return + } + for _, w := range words { + if !strings.Contains(err.Error(), w) { + t.Errorf("%s: error %q does not name %q", name, err, w) + } + } + } + base := Module{FP: "src1:abc", Repository: "a/b", Commit: "c"} + m := base + m.FP2 = "src1:" + strings.Repeat("a", 64) + _, err := EncodeModule(m) + wantErr("fp2 of the old form", err, "src1:"+strings.Repeat("a", 64)) + m.FP2 = "src2:xyz" + _, err = EncodeModule(m) + wantErr("fp2 malformed", err, "src2:xyz") + m = base + m.FP2, m.None2 = fp2of('a'), "no lock" + _, err = EncodeModule(m) + wantErr("fp2 and none2", err, "both", "fp2", "none2") + m = base + m.Manifest = "abc" + _, err = EncodeModule(m) + wantErr("a manifest digest malformed", err, "abc") + m = base + m.None2 = "no lock" + if _, err = EncodeModule(m); err != nil { + t.Errorf("none2 alone refused: %v", err) + } + // decode refuses what encode refuses + _, err = DecodeModule([]byte(`{"v":1,"fp":"src1:abc","fp2":"src2:zz","repository":"a/b","commit":"c"}`)) + wantErr("decoded fp2 malformed", err, "src2:zz") + + c := newCandidate() + c.FPSet = "000000000000" + _, err = EncodeCandidate(c) + wantErr("a stale fpset", err, "fpset", "000000000000") + c = newCandidate() + c.FPSet = "" + _, err = EncodeCandidate(c) + wantErr("fingerprints and no fpset", err, "fpset") + c = sampleCandidate() + c.FPSet = FingerprintSet(newCandidate().Pulls) + _, err = EncodeCandidate(c) + wantErr("an fpset and no fingerprints", err, "fpset") + c = newCandidate() + c.Pulls[0].Fps["mesh-mail"] = CandidateModule{FP2: "src1:abc"} + c.FPSet = FingerprintSet(c.Pulls) + _, err = EncodeCandidate(c) + wantErr("a candidate's fp2 of the old form", err, "src1:abc") + c = newCandidate() + c.Pulls[0].Fps["mesh-mail"] = CandidateModule{FP2: fp2of('a'), None2: "x"} + c.FPSet = FingerprintSet(c.Pulls) + _, err = EncodeCandidate(c) + wantErr("a candidate's module with both", err, "both", "mesh-mail") + c = newCandidate() + c.Pulls[0].Fps["other"] = CandidateModule{FP2: fp2of('a')} + c.FPSet = FingerprintSet(c.Pulls) + _, err = EncodeCandidate(c) + wantErr("a fingerprint of a module the pull does not touch", err, "other") +} + +// The fpset is the same for any order and changes with any fingerprint or reason (research 063 section 01: +// verdicts..check. is keyed by it). +func TestFingerprintSet(t *testing.T) { + c := newCandidate() + want := FingerprintSet(c.Pulls) + if len(want) != 12 || strings.Trim(want, "0123456789abcdef") != "" { + t.Errorf("fpset %q is not 12 lowercase hex digits", want) + } + c.Pulls[0], c.Pulls[1] = c.Pulls[1], c.Pulls[0] + if got := FingerprintSet(c.Pulls); got != want { + t.Errorf("the order of pull requests changes it: %s", got) + } + c = newCandidate() + c.Pulls[0].Fps["mesh-mail"] = CandidateModule{FP2: fp2of('b')} + if got := FingerprintSet(c.Pulls); got == want { + t.Error("a changed fingerprint keeps it") + } + c = newCandidate() + c.Pulls[1].Fps["mesh-controller"] = CandidateModule{None2: "no tree"} + if got := FingerprintSet(c.Pulls); got == want { + t.Error("a changed reason keeps it") + } + c = newCandidate() + c.Pulls[0].Fps["mesh-mail"] = CandidateModule{FP2: fp2of('a'), Manifest: digestOf('9')} + if got := FingerprintSet(c.Pulls); got != want { + t.Error("a manifest digest, which the fingerprint already covers, changes it") + } +} + +// 1357/AC6: both values keep version 1, and a value of another version is refused with ErrUnknownVersion. +func TestVersionsStayOne(t *testing.T) { + if Version != 1 { + t.Fatalf("Version = %d", Version) + } + for _, data := range [][]byte{mustJSON(t, Module{FP: "src1:a", FP2: fp2of('a'), Repository: "a/b", Commit: "c"})} { + if !strings.HasPrefix(string(data), `{"v":1,`) { + t.Errorf("module at %s", data) + } + } + data, _ := EncodeCandidate(newCandidate()) + if !strings.HasPrefix(string(data), `{"v":1,`) { + t.Errorf("candidate at %s", data) + } + for _, in := range []string{ + `{"v":2,"fp":"src1:a","fp2":"` + fp2of('a') + `","repository":"a/b","commit":"c"}`, + `{"fp2":"` + fp2of('a') + `","repository":"a/b","commit":"c"}`, + } { + if _, err := DecodeModule([]byte(in)); !errors.Is(err, ErrUnknownVersion) { + t.Errorf("DecodeModule(%s) = %v", in, err) + } + } + if _, err := DecodeCandidate([]byte(`{"v":2,"group":"g","fpset":"abc"}`)); !errors.Is(err, ErrUnknownVersion) { + t.Errorf("candidate version 2: %v", err) + } +}