package main import ( "bytes" "context" "encoding/json" "errors" "io/fs" "os" "path" "path/filepath" "reflect" "strings" "testing" "mesh-delivery-checker/rules" "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliverycheckerclient" "git.novox.be/novox/mesh-delivery/sdk/buildstep" "git.novox.be/novox/mesh-delivery/sdk/seatdefs" "git.novox.be/novox/mesh-sdk/go/seatdef" ) // 1921 (B0, D2): the checking program answers, on its last line, the verdict the rules give the facts in its input, and // refuses facts of a version it does not read, an input it is not given, and a question it is not asked, naming each. // 1921 (B0, D3): asked -seats, it answers the seat list of the heads laid for it: a definition a head adds, resolved with // the parts it includes; what it left out, and why; and an empty list for heads with no definition. func TestTheCheckingProgramAnswersTheVerdictOfItsFacts(t *testing.T) { failing, _ := json.Marshal(rules.Tested{Level: buildstep.LevelUnit, Tests: 3, Failed: []rules.Failure{{Package: "x/a", Test: "TestA", Cause: "a_test.go:9: boom"}}}) facts := rules.Facts{V: rules.FactsVersion, Group: "feat/x", Heads: "0123456789ab", Members: []rules.Member{{Repository: "novox/x", Number: 3, Head: "h"}}, Steps: []rules.Step{ {Repository: "novox/x", Number: 3, Module: "mesh-build-go", Node: "laptop", Level: &buildstep.LevelAnswer{Level: buildstep.LevelUnit, Tests: []json.RawMessage{failing}}}, {Repository: "novox/x", Number: 3, Failed: &rules.FailedFirst{Machine: "laptop", Words: "novox/x#3 has files in conflict: go.mod"}}, }} dir := t.TempDir() write := func(name string, v any) string { data, _ := json.Marshal(v) p := filepath.Join(dir, name) if err := os.WriteFile(p, data, 0o600); err != nil { t.Fatal(err) } return p } v1 := write("v1.json", facts) later := facts later.V = rules.FactsVersion + 1 v2 := write("v2.json", later) // The heads as the build node lays them, each at its repository's path: a contracts module of its own defining a seat // with a part, and the mesh's own contracts module defining a seat of a domain that is not its (ADR 0636 §5). lay := func(name string, files map[string]string) string { root := filepath.Join(dir, name) for p, body := range files { if err := os.MkdirAll(filepath.Dir(filepath.Join(root, p)), 0o755); err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Join(root, p), []byte(body), 0o600); err != nil { t.Fatal(err) } } return root } contracts := func(module string) string { return `{"module": "` + module + `", "build": {"artifacts": [{"name": "go", "kind": "package", "language": "go", "generated-from": "."}]}}` } heads := lay("heads", map[string]string{ "novox/x/x-contracts/module.json": contracts("x-contracts"), "novox/x/x-contracts/seats/node-build-x/seat.json": `{"format":1,"seat":"node-build-x","scope":"node","grain":"whole","domain":"Delivery","decision":"ADR 1","description":"kept","includes":["build"],"verbs":[{"name":"own","input":{"type":"object","properties":{"credentials":{"$ref":"#/defs/Sealed"}}},"answer":{"type":"object"}}],"defs":{"Sealed":{"type":"object"}}}`, "novox/x/x-contracts/parts/build.json": `{"format":1,"part":"build","verbs":[{"name":"current","input":{"type":"object"},"answer":{"type":"object"}}],"events":[{"name":"ended","body":{"type":"object"}}]}`, "novox/x/contracts/module.json": contracts("mesh-contracts"), "novox/x/contracts/seats/mesh-catalog/seat.json": `{"format":1,"seat":"mesh-catalog","scope":"mesh","grain":"whole","domain":"Delivery","decision":"ADR 1"}`, }) broken := lay("broken", map[string]string{ "novox/x/x-contracts/module.json": contracts("x-contracts"), "novox/x/x-contracts/seats/broken/seat.json": `{"format":1,"seat":"broken","scope":"nowhere"}`, }) none := filepath.Join(dir, "no-heads") if err := os.MkdirAll(filepath.Join(none, "novox/x"), 0o755); err != nil { t.Fatal(err) } for name, c := range map[string]struct { args []string input, heads string exit int said string }{ "the seats": {args: []string{"-seats"}, input: v1, heads: heads}, "no seats": {args: []string{"-seats"}, input: v1, heads: none}, "a seat unparsed": {args: []string{"-seats"}, input: v1, heads: broken, exit: 2, said: "the seat definition of broken (module x-contracts) at the head of novox/x is not taken"}, "no heads laid": {args: []string{"-seats"}, exit: 2, said: buildstep.RunHeadsEnv + " names no directory"}, "the facts": {args: []string{"-verdict"}, input: v1}, "a later version": {args: []string{"-verdict"}, input: v2, exit: 2, said: "facts of version 2, this checking program reads version 1"}, "no input": {args: []string{"-verdict"}, exit: 2, said: buildstep.RunInputEnv + " names no file"}, "nothing asked": {input: v1, exit: 2, said: "nothing asked"}, "an unknown asking": {args: []string{"-judge"}, input: v1, exit: 2, said: "-judge"}, } { var out, errs bytes.Buffer exit := run(c.args, func(k string) string { switch k { case buildstep.RunInputEnv: return c.input case buildstep.RunHeadsEnv: return c.heads } return "" }, &out, &errs) if exit != c.exit || !strings.Contains(errs.String(), c.said) { t.Errorf("%s: exit %d, said %q; want %d and %q", name, exit, errs.String(), c.exit, c.said) continue } if c.exit != 0 { continue } lines := strings.Split(strings.TrimSpace(out.String()), "\n") if c.heads != "" { var sa rules.SeatsAnswer if err := json.Unmarshal([]byte(lines[len(lines)-1]), &sa); err != nil || sa.V != rules.SeatsVersion || sa.Seats == nil { t.Fatalf("%s: the last line %q: %v", name, lines[len(lines)-1], err) } if c.heads == none { if len(sa.Seats) != 0 || len(sa.Left) != 0 { t.Errorf("%s: %+v, want an empty list", name, sa) } continue } var d struct { Seat string `json:"seat"` Verbs []struct { Name string `json:"name"` } `json:"verbs"` } if len(sa.Seats) == 1 { _ = json.Unmarshal(sa.Seats[0], &d) } if d.Seat != "node-build-x" || len(d.Verbs) != 2 || len(sa.Left) != 1 || !strings.Contains(sa.Left[0].Why, "whose seats mesh-delivery-contracts defines") { t.Errorf("%s: seats %s, left %+v; want node-build-x with its part's verb, and mesh-catalog left out", name, sa.Seats, sa.Left) } // The group check of 616c42434ac2: a checker from before G (no digests in these facts) writes the answer into the // build seat's step state, whose writer refuses a value with a field named like a credential, here a schema's // property name; the answer carries no verb's or message's schema and no type only they referred to. if len(sa.Seats) == 1 && (bytes.Contains(sa.Seats[0], []byte("credentials")) || bytes.Contains(sa.Seats[0], []byte("Sealed"))) { t.Errorf("%s: %s carries a verb's schema to a checker that hands no digests", name, sa.Seats[0]) } continue } var a rules.Answer if err := json.Unmarshal([]byte(lines[len(lines)-1]), &a); err != nil || a.V != rules.AnswerVersion { t.Fatalf("%s: the last line %q: %v", name, lines[len(lines)-1], err) } if want := rules.Judge(facts); !reflect.DeepEqual(a.Verdict, want) || a.Verdict.State != meshdeliverycheckerclient.VerdictFail || len(a.Verdict.Failures) != 2 || a.Verdict.Failures[1].Words != "novox/x#3 has files in conflict: go.mod" || a.Verdict.Machines[0] != "laptop" || a.Verdict.Levels[buildstep.LevelUnit].Run != 3 { t.Errorf("%s: the verdict %+v, want %+v", name, a.Verdict, want) } } // Review finding 3 of milestone 47/G: the heads of a real group carry every seat definition the reader holds, unchanged // (this repository's own contracts, laid as its head): the answer leaves them all to the checker's list and stays // within the bound a run's answer is written under (buildstep.RunInputBound), where the whole list would not. t.Run("heads carrying every definition the reader holds unchanged answer none of them, within the bound", func(t *testing.T) { real := under{prefix: "novox/mesh-delivery", fs: os.DirFS("../../../..")} group := rules.Facts{V: rules.FactsVersion, Members: []rules.Member{{Repository: "novox/mesh-delivery", Head: "h"}}} // An empty list of digests answers every definition whole (a checker that hands its list's digests, holding none). whole, err := rules.SeatsOf(real, rules.Facts{V: rules.FactsVersion, Members: group.Members, SeatDigests: map[string]string{}}) if err != nil || len(whole.Seats) < 10 { t.Fatalf("the repository's own definitions: %d, %v", len(whole.Seats), err) } // The reader's digests are of each definition whole, as the seat mesh-catalog answers it; the answer is its judging view. group.SeatDigests = wholeDigests(t, real, "novox/mesh-delivery") same, err := rules.SeatsOf(real, group) answer, _ := json.Marshal(same) if err != nil || len(same.Seats) != 0 || same.Same != len(whole.Seats) || len(answer) > buildstep.RunInputBound { t.Errorf("unchanged heads answer %d definitions (%d the same), %d bytes; want none, all %d the same, within %d: %v", len(same.Seats), same.Same, len(answer), len(whole.Seats), buildstep.RunInputBound, err) } // Review round 2 of G: a checker from before G (B0's, which makes G's own check) hands no digests, and the answer // then keeps each seat's own description only, so a group's whole list stays within the bound; each still parses. b0, err := rules.SeatsOf(real, rules.Facts{V: rules.FactsVersion, Members: group.Members}) if err != nil || len(b0.Seats) != len(whole.Seats) { t.Fatalf("B0-shaped facts: %d definitions, %v", len(b0.Seats), err) } for i, raw := range b0.Seats { var own struct { Description string `json:"description"` } _ = json.Unmarshal(whole.Seats[i], &own) var d struct { Description string `json:"description"` Verbs []json.RawMessage `json:"verbs"` Events []json.RawMessage `json:"events"` } _ = json.Unmarshal(raw, &d) inner := 0 for _, m := range append(d.Verbs, d.Events...) { inner += strings.Count(string(m), `"description":"`) } parsed, perr := seatdef.Parse(raw) for _, v := range parsed.Verbs { if len(v.Input) != 0 || len(v.Answer) != 0 { t.Errorf("B0-shaped facts answer %s's verb %s with its schemas; want none", parsed.Seat, v.Name) } } for name := range parsed.Defs { if !bytes.Contains(raw, []byte(`"#/defs/`+name+`"`)) { t.Errorf("B0-shaped facts answer %s with the type %s, which nothing it keeps refers to", parsed.Seat, name) } } if inner > 0 || d.Description != own.Description || perr != nil { t.Errorf("B0-shaped facts answer %s with %d inner descriptions, its own %q: %v; want only its own", raw[:min(len(raw), 80)], inner, d.Description, perr) } } if answer, _ := json.Marshal(b0); len(answer) > buildstep.RunInputBound || len(answer) >= len(mustJSON(t, whole))*3/4 { t.Errorf("B0-shaped facts answer %d bytes (whole %d), want within %d and far below the whole", len(answer), len(mustJSON(t, whole)), buildstep.RunInputBound) } // One definition the heads change is answered alone. for seat := range group.SeatDigests { group.SeatDigests[seat] = "changed" break } if one, err := rules.SeatsOf(real, group); err != nil || len(one.Seats) != 1 { t.Errorf("one changed definition: answered %d, %v", len(one.Seats), err) } }) // The lead's ruling after G's group check of 616c42434ac2: with digests given too, a changed definition whose verb input // has a property named credentials is answered as the validator and the composer read it, with no such field. changed, err := rules.SeatsOf(os.DirFS(heads), rules.Facts{V: rules.FactsVersion, Members: facts.Members, SeatDigests: map[string]string{"node-build-x": "changed"}}) if err != nil || len(changed.Seats) != 1 || bytes.Contains(changed.Seats[0], []byte("credentials")) || bytes.Contains(changed.Seats[0], []byte("Sealed")) { t.Errorf("a changed definition with digests given: %d answered, %s, %v; want it alone, with no verb's schema", len(changed.Seats), changed.Seats, err) } // Review round 1 of milestone 47/B0: the module graph's cycle rule is the program's (ADR 0603 §2, ADR 0656 §5), over the // edges the running checker gathers. A cycle the change makes fails; one the trunk has among the same modules is noted. web, mail := rules.Edge{From: "web", To: "mail", Kind: rules.EdgeAwaits}, rules.Edge{From: "mail", To: "web", Kind: rules.EdgeAwaits} only := func(e rules.Edge, where string) rules.Edge { e.Only = where; return e } for _, row := range []struct { name string graph rules.Graph fails string note string }{ {name: "no cycle", graph: rules.Graph{Trunk: true, Edges: []rules.Edge{web}}, note: "has no cycle"}, {name: "a cycle the change makes", graph: rules.Graph{Trunk: true, Edges: []rules.Edge{mail, only(web, rules.EdgeChange)}}, fails: "has the cycle mail -> web -> mail"}, {name: "a cycle the trunk has", graph: rules.Graph{Trunk: true, Edges: []rules.Edge{mail, web}}, note: "which the trunk already has"}, {name: "the trunk unread", graph: rules.Graph{Edges: []rules.Edge{mail, web}}, fails: "has the cycle mail -> web -> mail"}, {name: "a base on itself", graph: rules.Graph{Trunk: true, Edges: []rules.Edge{only(rules.Edge{From: "web", To: "web", Kind: rules.EdgeBase}, rules.EdgeChange)}}, fails: "has the cycle web -> web"}, {name: "an edge only the trunk has", graph: rules.Graph{Trunk: true, Edges: []rules.Edge{mail, only(web, rules.EdgeTrunk)}}, note: "has no cycle"}, } { g := row.graph a := rules.Judged(rules.Facts{V: rules.FactsVersion, Steps: []rules.Step{{Graph: &g}}}) if row.fails != "" && (a.Verdict.State != meshdeliverycheckerclient.VerdictFail || !strings.Contains(a.Verdict.Failed.Words, row.fails)) { t.Errorf("%s: verdict %+v, want a fail saying %q", row.name, a.Verdict, row.fails) } if row.note != "" && (a.Verdict.State != meshdeliverycheckerclient.VerdictPass || !strings.Contains(strings.Join(a.Notes, "\n"), row.note)) { t.Errorf("%s: verdict %s, notes %q, want a pass noting %q", row.name, a.Verdict.State, a.Notes, row.note) } } } // under is a directory laid at a repository's path of the heads, as the build node lays a head. type under struct { prefix string fs fs.FS } func (u under) Open(name string) (fs.File, error) { if name == u.prefix { return u.fs.Open(".") } if rest, ok := strings.CutPrefix(name, u.prefix+"/"); ok { return u.fs.Open(rest) } return nil, &fs.PathError{Op: "open", Path: name, Err: fs.ErrNotExist} } func mustJSON(t *testing.T, v any) []byte { t.Helper() out, err := json.Marshal(v) if err != nil { t.Fatal(err) } return out } // wholeDigests are the digests of a head's seat definitions whole (rules.SeatDigest), read by the domain SDK as the // indexer reads them: what a checker's reader holds for heads that change none. func wholeDigests(t *testing.T, root fs.FS, repository string) map[string]string { t.Helper() head, err := fs.Sub(root, repository) if err != nil { t.Fatal(err) } manifests := map[string]string{} _ = fs.WalkDir(head, ".", func(p string, d fs.DirEntry, err error) error { if err != nil { return err } if d.IsDir() && (d.Name() == ".git" || d.Name() == "testdata" || d.Name() == "node_modules") { return fs.SkipDir } if !d.IsDir() && d.Name() == "module.json" { raw, _ := fs.ReadFile(head, p) manifests[path.Dir(p)] = string(raw) } return nil }) forge := seatdefs.Forge{ Files: func(_ context.Context, _, _ string, dirs []string) (map[string]map[string]string, error) { out := map[string]map[string]string{} for _, dir := range dirs { files := map[string]string{} if err := fs.WalkDir(head, dir, func(p string, d fs.DirEntry, err error) error { if err == nil && !d.IsDir() { files[strings.TrimPrefix(p, dir+"/")] = "" } return err }); err == nil { out[dir] = files } } return out, nil }, File: func(_ context.Context, _, _, file string) ([]byte, error) { raw, err := fs.ReadFile(head, file) if errors.Is(err, fs.ErrNotExist) { return nil, nil } return raw, err }, } out := map[string]string{} if len(manifests) == 0 { t.Fatalf("no manifest at the head of %s", repository) } for _, c := range seatdefs.ContractsModules(manifests) { defs, _, err := forge.Read(context.Background(), repository, "h", c) if err != nil { t.Fatal(err) } for seat, d := range defs { raw, _ := json.Marshal(d) out[seat] = rules.SeatDigest(raw) } } return out }