package catalogue import ( "slices" "strings" "testing" ) // A store as a module now declares it (novox/hq ADR 0233): its own data by directory with a class and a // dump, and what it keeps for its consumers. const declaredStore = `{"module":"pg","version":"1", "provides":[{"name":"postgres-database","scope":"mesh"}], "grants":{"postgres-database":"${dir:grants}"}, "data":{ "own":[ {"id":"store","path":"${dir:store}","class":"irreplaceable","backup":{"dump":"dump-all > ${dir:dumps}/all","into":"dumps"},"active":"1d"}, {"id":"dumps","path":"${dir:dumps}","class":"rebuildable","backup":"none"}], "consumers":{"postgres-database":{"class":"irreplaceable","in":"store"}}}, "resources":[ {"id":"grants","type":"directory","mode":"0700"}, {"id":"store","type":"directory","path":"/srv/store","mode":"0700"}, {"id":"dumps","type":"directory","path":"${dir:store}/dumps","mode":"0700"}, {"id":"server","type":"container","image":"x@sha256:aa","volumes":["${dir:store}:/var/lib/postgresql/data"]}]}` func mustParse(t *testing.T, raw string) Manifest { t.Helper() m, err := ParseManifest([]byte(raw)) if err != nil { t.Fatalf("refused: %v", err) } return m } func TestADeclaredStoreParsesAndPassesTheCheck(t *testing.T) { m := mustParse(t, declaredStore) if problems := DataProblems(Shelf{"pg": m}); len(problems) > 0 { t.Fatalf("a store declaring its data was refused: %v", problems) } if !m.KeepsConsumerData("postgres-database") { t.Fatal("an irreplaceable consumer class does not keep the consumer's data") } if it, _ := m.DataItem("store"); !it.BackedUp() || it.BackupWithin() != DefaultBackupWithin || it.ActiveWithin().Hours() != 24 { t.Fatalf("the store item reads %+v", it) } if it, _ := m.DataItem("dumps"); it.BackedUp() { t.Fatal("a rebuildable item that says none is backed up") } } // Every rule of the section itself, refused at parse in the words of the module. func TestTheDataSectionIsRefusedWhereItIsWrong(t *testing.T) { for _, c := range []struct{ name, data, want string }{ {"a class it does not know", `{"own":[{"id":"a","path":"${dir:d}","class":"precious"}]}`, "not one the mesh protects by"}, {"an access it does not have", `{"own":[{"id":"a","path":"${access:media}","class":"irreplaceable","redundancy":"an array"}]}`, "declares no access"}, {"irreplaceable and protected by nothing", `{"own":[{"id":"a","path":"${dir:d}","class":"irreplaceable","backup":"none"}]}`, "protected one way or the other"}, {"a cache on redundancy", `{"own":[{"id":"a","path":"${dir:d}","class":"cache","redundancy":"an array"}]}`, "a cache is not protected"}, {"kept with something not required", `{"kept-by":{"s3-bucket":{"class":"irreplaceable"}}}`, "does not require"}, {"a measure it does not know", `{"own":[{"id":"a","path":"${dir:d}","class":"cache","measure":"du"}]}`, "walk, dataset or shallow"}, {"a cache expecting writes", `{"own":[{"id":"a","path":"${dir:d}","class":"cache","active":"1d"}]}`, "only irreplaceable and valuable"}, {"a machine path", `{"own":[{"id":"a","path":"/srv/a","class":"cache"}]}`, "names no machine path"}, {"a directory it does not declare", `{"own":[{"id":"a","path":"${dir:nowhere}","class":"cache"}]}`, "declares no directory"}, {"a cache backed up", `{"own":[{"id":"a","path":"${dir:d}","class":"cache","backup":"copy"}]}`, "disposable"}, {"a dump into nothing", `{"own":[{"id":"a","path":"${dir:d}","class":"irreplaceable","backup":{"dump":"x","into":"b"}}]}`, "not one of the module's data items"}, {"a dump into itself", `{"own":[{"id":"a","path":"${dir:d}","class":"irreplaceable","backup":{"dump":"x","into":"a"}}]}`, "into itself"}, {"a bad duration", `{"own":[{"id":"a","path":"${dir:d}","class":"irreplaceable","within":"soon"}]}`, "not a duration"}, {"the same id twice", `{"own":[{"id":"a","path":"${dir:d}","class":"cache"},{"id":"a","path":"${dir:d}/x","class":"cache"}]}`, "declared twice"}, {"own data of class none", `{"own":[{"id":"a","path":"${dir:d}","class":"none"}]}`, "only a provision"}, {"consumers of something not provided", `{"consumers":{"s3-bucket":{"class":"cache"}}}`, "does not provide"}, {"consumer data that lives nowhere", `{"consumers":{"q":{"class":"irreplaceable"}}}`, "says nowhere it lives"}, {"consumer data in an item less precious", `{"own":[{"id":"a","path":"${dir:d}","class":"rebuildable"}],"consumers":{"q":{"class":"irreplaceable","in":"a"}}}`, "which is only rebuildable"}, {"consumer data in something unknown", `{"consumers":{"q":{"class":"rebuildable","in":"elsewhere"}}}`, "neither one of its data items"}, } { raw := `{"module":"m","version":"1","provides":["q"],"resources":[{"id":"d","type":"directory","mode":"0700"}],"data":` + c.data + `}` _, err := ParseManifest([]byte(raw)) if err == nil || !strings.Contains(err.Error(), c.want) { t.Errorf("%s: %v, want %q", c.name, err, c.want) } } } // The catalogue check: a granting provider says what it keeps, says it in one place, writes no backup // line by hand, and every directory a container writes is declared (novox/hq ADR 0233). func TestTheCatalogueCheckAsksEveryModuleForItsData(t *testing.T) { unsaid := mustParse(t, `{"module":"q","version":"1","provides":[{"name":"queue","scope":"mesh"}], "grants":{"queue":"${dir:g}"},"resources":[{"id":"g","type":"directory","mode":"0700"}]}`) onTheOffer := mustParse(t, `{"module":"r","version":"1","provides":[{"name":"resolver","scope":"mesh","keeps-consumer-data":false}]}`) byHand := mustParse(t, `{"module":"h","version":"1","resources":[{"id":"d","type":"directory","mode":"0700"}], "contributions":[{"seat":"node-backup","kind":"backup","content":"path ${dir:d}"}]}`) writes := mustParse(t, `{"module":"w","version":"1","resources":[{"id":"d","type":"directory","mode":"0700"}, {"id":"c","type":"directory","mode":"0700"}, {"id":"app","type":"container","image":"x@sha256:aa","volumes":["${dir:d}:/data","${dir:c}:/etc/app:ro","${dir:d}/f:/f"]}]}`) problems := strings.Join(DataProblems(Shelf{"q": unsaid, "r": onTheOffer, "h": byHand, "w": writes}), "\n") for _, want := range []string{ "q grants queue and does not say what it keeps", "r says keeps-consumer-data on its offer", "h's contribution 1 is a backup line written by hand", `w mounts its directory "d" into a container to be written`, } { if !strings.Contains(problems, want) { t.Errorf("the check does not say %q:\n%s", want, problems) } } if strings.Contains(problems, `"c"`) { t.Errorf("a read-only mount was taken for written data:\n%s", problems) } // The same module declaring the directory, as a cache, passes. declared := mustParse(t, `{"module":"w","version":"1","data":{"own":[{"id":"d","path":"${dir:d}","class":"cache"}]}, "resources":[{"id":"d","type":"directory","mode":"0700"}, {"id":"app","type":"container","image":"x@sha256:aa","volumes":["${dir:d}:/data"]}]}`) if p := DataProblems(Shelf{"w": declared}); len(p) > 0 { t.Fatalf("a declared directory is still refused: %v", p) } } // Consumer data said to live in a provision the module requires is protected only as well as that // provision's provider protects its consumers' data. func TestConsumerDataInARequiredProvisionIsJudgedAgainstItsProvider(t *testing.T) { idp := mustParse(t, `{"module":"idp","version":"1","requires":["postgres-database"], "provides":[{"name":"oidc-client","scope":"mesh"}],"grants":{"oidc-client":"${dir:g}"}, "resources":[{"id":"g","type":"directory","mode":"0700"}], "data":{"consumers":{"oidc-client":{"class":"irreplaceable","in":"postgres-database"}}}}`) cachy := mustParse(t, strings.Replace(declaredStore, `"consumers":{"postgres-database":{"class":"irreplaceable","in":"store"}}`, `"consumers":{"postgres-database":{"class":"cache"}}`, 1)) if p := strings.Join(DataProblems(Shelf{"idp": idp, "pg": cachy}), "\n"); !strings.Contains(p, "not protected as irreplaceable") { t.Fatalf("irreplaceable data kept in a cache passed: %s", p) } if p := DataProblems(Shelf{"idp": idp, "pg": mustParse(t, declaredStore)}); len(p) > 0 { t.Fatalf("refused against a provider that keeps its consumers' data: %v", p) } } // What a binding is sticky by follows the class (ADR 0232, generalised by 0233): a cache's consumers // move freely, a store's do not; an older definition saying nothing still follows its grant. func TestKeepingConsumerDataFollowsTheClass(t *testing.T) { for class, keeps := range map[string]bool{"irreplaceable": true, "valuable": true, "rebuildable": true, "cache": false, "none": false} { in := `,"in":"d"` own := `"own":[{"id":"d","path":"${dir:d}","class":"irreplaceable"}],` if !keeps { in, own = "", "" } m := mustParse(t, `{"module":"p","version":"1","provides":[{"name":"q","scope":"mesh"}],"grants":{"q":"${dir:d}"}, "resources":[{"id":"d","type":"directory","mode":"0700"}], "data":{`+own+`"consumers":{"q":{"class":"`+class+`"`+in+`}}}}`) if m.KeepsConsumerData("q") != keeps { t.Errorf("class %s keeps consumer data: %v, want %v", class, !keeps, keeps) } shelf := map[string]Manifest{"p": m} if KeepsConsumerData(shelf, "q") != keeps { t.Errorf("class %s: the provision by name keeps: %v", class, !keeps) } } older := mustParse(t, `{"module":"p","version":"1","provides":[{"name":"q","scope":"mesh"}],"grants":{"q":"${dir:d}"}, "resources":[{"id":"d","type":"directory","mode":"0700"}]}`) if !older.KeepsConsumerData("q") { t.Fatal("an older definition that grants no longer keeps its consumers' data") } } // The backup holder's lines are derived: the dump runs and the directory it writes into is kept, // anything but a cache is copied by default — the standard nightly plan — an item on redundancy is not // copied, and every item is listed with its class and protection for the holder to measure and watch: // directories filled, a home directory filled from the machine, an operator's path from where the // assignment placed it. A backup line still written by hand beside a data section is not placed. func TestTheBackupHoldersLinesAreDerivedFromTheData(t *testing.T) { pg := mustParse(t, declaredStore) pg.Contributions = append(pg.Contributions, SeatContribution{Seat: BackupSeat, Kind: "backup", Content: "path /stale"}) agent := mustParse(t, `{"module":"agent","version":"1", "data":{"own":[{"id":"home","path":"${dir:home}","class":"valuable"},{"id":"tmp","path":"${dir:home}/tmp","class":"cache"}]}, "resources":[{"id":"home","type":"directory","path":"${machine:account-home}/.agent","mode":"0700"}]}`) media := mustParse(t, `{"module":"media","version":"1", "accesses":[{"id":"films","mode":"read"}], "data":{"own":[{"id":"films","path":"${access:films}","class":"irreplaceable","redundancy":"on a redundant array; no room for a copy","measure":"dataset"}, {"id":"meta","path":"${dir:meta}","class":"rebuildable"}]}, "resources":[{"id":"meta","type":"directory","mode":"0700"}]}`) facts := map[string]string{"account-home": "/home/op"} with := Rendering{Settings: SettingsBy{"media": {{From: "node", Values: map[string]any{AccessesSetting: map[string]any{"films": "/tank/films"}}}}}} backup, err := seatContributions([]Manifest{pg, agent, media}, BackupSeat, BackupKindBackup, with, facts) if err != nil { t.Fatal(err) } want := "# agent\npath /home/op/.agent\n# media\npath /var/lib/media/meta\n# pg\nrun dump-all > /srv/store/dumps/all\npath /srv/store/dumps\n" if backup != want { t.Fatalf("backup lines:\n%s\nwant:\n%s", backup, want) } data, err := seatContributions([]Manifest{pg, agent, media}, BackupSeat, BackupKindData, with, facts) if err != nil { t.Fatal(err) } want = "# agent\nitem home valuable /home/op/.agent /home/op/.agent backup walk\nitem tmp cache /home/op/.agent/tmp - none walk\n" + "# media\nitem films irreplaceable /tank/films - redundancy dataset\nitem meta rebuildable /var/lib/media/meta /var/lib/media/meta backup walk\n" + "# pg\nitem store irreplaceable /srv/store /srv/store/dumps backup walk\nitem dumps rebuildable /srv/store/dumps - none walk\n" if data != want { t.Fatalf("data lines:\n%s\nwant:\n%s", data, want) } if _, err := seatContributions([]Manifest{agent}, BackupSeat, BackupKindData, Rendering{}, nil); err == nil { t.Fatal("a home directory with no account on the machine reached the holder as a placeholder") } // What keeps something irreplaceable depends on the machine's backup holder — it backs it up or // watches its array; valuable data alone is backed up where a holder is, and refused nowhere. if !slices.Contains(DependsOn(media), BackupSeat) || !slices.Contains(DependsOn(pg), BackupSeat) || slices.Contains(DependsOn(agent), BackupSeat) { t.Fatalf("depends: media %v, pg %v, agent %v", DependsOn(media), DependsOn(pg), DependsOn(agent)) } if it, _ := media.DataItem("films"); it.OwnedByModule() || it.Protection() != "redundancy" { t.Fatalf("an operator's path reads %+v", it) } } // Data a consumer keeps with a provider as irreplaceable must be protected there: the check refuses a // provider that keeps its consumers' data as a cache, or in an item with no protection. func TestIrreplaceableDataKeptWithAProviderMustBeProtectedThere(t *testing.T) { photos := mustParse(t, `{"module":"photos","version":"1","requires":["s3-bucket"], "data":{"kept-by":{"s3-bucket":{"class":"irreplaceable"}}}}`) store := func(backup string) Manifest { return mustParse(t, `{"module":"objects","version":"1","provides":[{"name":"s3-bucket","scope":"mesh"}], "grants":{"s3-bucket":"${dir:g}"}, "data":{"own":[{"id":"data","path":"${dir:data}","class":"valuable","backup":"`+backup+`"}], "consumers":{"s3-bucket":{"class":"valuable","in":"data"}}}, "resources":[{"id":"g","type":"directory","mode":"0700"},{"id":"data","type":"directory","mode":"0700"}]}`) } if p := DataProblems(Shelf{"photos": photos, "objects": store("copy")}); len(p) > 0 { t.Fatalf("a provider backing its consumers' data up was refused: %v", p) } if p := strings.Join(DataProblems(Shelf{"photos": photos, "objects": store("none")}), "\n"); !strings.Contains(p, "neither backed up") { t.Fatalf("irreplaceable photos in an unprotected store passed: %s", p) } }