jochen
abf9125689
A walk over a large library every hour loads the array that protects it. An item now says how it is measured — a bounded daily walk, a dataset's counters, or its top level only — and a size that is a lower bound is kept as such and never read as a shrink.
228 lines
14 KiB
Go
228 lines
14 KiB
Go
package catalogue
|
|
|
|
import (
|
|
"slices"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
// A store as a module now declares it (novox/hq ADR 0233): its own data by directory with a class and a
|
|
// dump, and what it keeps for its consumers.
|
|
const declaredStore = `{"module":"pg","version":"1",
|
|
"provides":[{"name":"postgres-database","scope":"mesh"}],
|
|
"grants":{"postgres-database":"${dir:grants}"},
|
|
"data":{
|
|
"own":[
|
|
{"id":"store","path":"${dir:store}","class":"irreplaceable","backup":{"dump":"dump-all > ${dir:dumps}/all","into":"dumps"},"active":"1d"},
|
|
{"id":"dumps","path":"${dir:dumps}","class":"rebuildable","backup":"none"}],
|
|
"consumers":{"postgres-database":{"class":"irreplaceable","in":"store"}}},
|
|
"resources":[
|
|
{"id":"grants","type":"directory","mode":"0700"},
|
|
{"id":"store","type":"directory","path":"/srv/store","mode":"0700"},
|
|
{"id":"dumps","type":"directory","path":"${dir:store}/dumps","mode":"0700"},
|
|
{"id":"server","type":"container","image":"x@sha256:aa","volumes":["${dir:store}:/var/lib/postgresql/data"]}]}`
|
|
|
|
func mustParse(t *testing.T, raw string) Manifest {
|
|
t.Helper()
|
|
m, err := ParseManifest([]byte(raw))
|
|
if err != nil {
|
|
t.Fatalf("refused: %v", err)
|
|
}
|
|
return m
|
|
}
|
|
|
|
func TestADeclaredStoreParsesAndPassesTheCheck(t *testing.T) {
|
|
m := mustParse(t, declaredStore)
|
|
if problems := DataProblems(Shelf{"pg": m}); len(problems) > 0 {
|
|
t.Fatalf("a store declaring its data was refused: %v", problems)
|
|
}
|
|
if !m.KeepsConsumerData("postgres-database") {
|
|
t.Fatal("an irreplaceable consumer class does not keep the consumer's data")
|
|
}
|
|
if it, _ := m.DataItem("store"); !it.BackedUp() || it.BackupWithin() != DefaultBackupWithin || it.ActiveWithin().Hours() != 24 {
|
|
t.Fatalf("the store item reads %+v", it)
|
|
}
|
|
if it, _ := m.DataItem("dumps"); it.BackedUp() {
|
|
t.Fatal("a rebuildable item that says none is backed up")
|
|
}
|
|
}
|
|
|
|
// Every rule of the section itself, refused at parse in the words of the module.
|
|
func TestTheDataSectionIsRefusedWhereItIsWrong(t *testing.T) {
|
|
for _, c := range []struct{ name, data, want string }{
|
|
{"a class it does not know", `{"own":[{"id":"a","path":"${dir:d}","class":"precious"}]}`, "not one the mesh protects by"},
|
|
{"an access it does not have", `{"own":[{"id":"a","path":"${access:media}","class":"irreplaceable","redundancy":"an array"}]}`, "declares no access"},
|
|
{"irreplaceable and protected by nothing", `{"own":[{"id":"a","path":"${dir:d}","class":"irreplaceable","backup":"none"}]}`, "protected one way or the other"},
|
|
{"a cache on redundancy", `{"own":[{"id":"a","path":"${dir:d}","class":"cache","redundancy":"an array"}]}`, "a cache is not protected"},
|
|
{"kept with something not required", `{"kept-by":{"s3-bucket":{"class":"irreplaceable"}}}`, "does not require"},
|
|
{"a measure it does not know", `{"own":[{"id":"a","path":"${dir:d}","class":"cache","measure":"du"}]}`, "walk, dataset or shallow"},
|
|
{"a cache expecting writes", `{"own":[{"id":"a","path":"${dir:d}","class":"cache","active":"1d"}]}`, "only irreplaceable and valuable"},
|
|
{"a machine path", `{"own":[{"id":"a","path":"/srv/a","class":"cache"}]}`, "names no machine path"},
|
|
{"a directory it does not declare", `{"own":[{"id":"a","path":"${dir:nowhere}","class":"cache"}]}`, "declares no directory"},
|
|
{"a cache backed up", `{"own":[{"id":"a","path":"${dir:d}","class":"cache","backup":"copy"}]}`, "disposable"},
|
|
{"a dump into nothing", `{"own":[{"id":"a","path":"${dir:d}","class":"irreplaceable","backup":{"dump":"x","into":"b"}}]}`, "not one of the module's data items"},
|
|
{"a dump into itself", `{"own":[{"id":"a","path":"${dir:d}","class":"irreplaceable","backup":{"dump":"x","into":"a"}}]}`, "into itself"},
|
|
{"a bad duration", `{"own":[{"id":"a","path":"${dir:d}","class":"irreplaceable","within":"soon"}]}`, "not a duration"},
|
|
{"the same id twice", `{"own":[{"id":"a","path":"${dir:d}","class":"cache"},{"id":"a","path":"${dir:d}/x","class":"cache"}]}`, "declared twice"},
|
|
{"own data of class none", `{"own":[{"id":"a","path":"${dir:d}","class":"none"}]}`, "only a provision"},
|
|
{"consumers of something not provided", `{"consumers":{"s3-bucket":{"class":"cache"}}}`, "does not provide"},
|
|
{"consumer data that lives nowhere", `{"consumers":{"q":{"class":"irreplaceable"}}}`, "says nowhere it lives"},
|
|
{"consumer data in an item less precious", `{"own":[{"id":"a","path":"${dir:d}","class":"rebuildable"}],"consumers":{"q":{"class":"irreplaceable","in":"a"}}}`, "which is only rebuildable"},
|
|
{"consumer data in something unknown", `{"consumers":{"q":{"class":"rebuildable","in":"elsewhere"}}}`, "neither one of its data items"},
|
|
} {
|
|
raw := `{"module":"m","version":"1","provides":["q"],"resources":[{"id":"d","type":"directory","mode":"0700"}],"data":` + c.data + `}`
|
|
_, err := ParseManifest([]byte(raw))
|
|
if err == nil || !strings.Contains(err.Error(), c.want) {
|
|
t.Errorf("%s: %v, want %q", c.name, err, c.want)
|
|
}
|
|
}
|
|
}
|
|
|
|
// The catalogue check: a granting provider says what it keeps, says it in one place, writes no backup
|
|
// line by hand, and every directory a container writes is declared (novox/hq ADR 0233).
|
|
func TestTheCatalogueCheckAsksEveryModuleForItsData(t *testing.T) {
|
|
unsaid := mustParse(t, `{"module":"q","version":"1","provides":[{"name":"queue","scope":"mesh"}],
|
|
"grants":{"queue":"${dir:g}"},"resources":[{"id":"g","type":"directory","mode":"0700"}]}`)
|
|
onTheOffer := mustParse(t, `{"module":"r","version":"1","provides":[{"name":"resolver","scope":"mesh","keeps-consumer-data":false}]}`)
|
|
byHand := mustParse(t, `{"module":"h","version":"1","resources":[{"id":"d","type":"directory","mode":"0700"}],
|
|
"contributions":[{"seat":"node-backup","kind":"backup","content":"path ${dir:d}"}]}`)
|
|
writes := mustParse(t, `{"module":"w","version":"1","resources":[{"id":"d","type":"directory","mode":"0700"},
|
|
{"id":"c","type":"directory","mode":"0700"},
|
|
{"id":"app","type":"container","image":"x@sha256:aa","volumes":["${dir:d}:/data","${dir:c}:/etc/app:ro","${dir:d}/f:/f"]}]}`)
|
|
problems := strings.Join(DataProblems(Shelf{"q": unsaid, "r": onTheOffer, "h": byHand, "w": writes}), "\n")
|
|
for _, want := range []string{
|
|
"q grants queue and does not say what it keeps",
|
|
"r says keeps-consumer-data on its offer",
|
|
"h's contribution 1 is a backup line written by hand",
|
|
`w mounts its directory "d" into a container to be written`,
|
|
} {
|
|
if !strings.Contains(problems, want) {
|
|
t.Errorf("the check does not say %q:\n%s", want, problems)
|
|
}
|
|
}
|
|
if strings.Contains(problems, `"c"`) {
|
|
t.Errorf("a read-only mount was taken for written data:\n%s", problems)
|
|
}
|
|
// The same module declaring the directory, as a cache, passes.
|
|
declared := mustParse(t, `{"module":"w","version":"1","data":{"own":[{"id":"d","path":"${dir:d}","class":"cache"}]},
|
|
"resources":[{"id":"d","type":"directory","mode":"0700"},
|
|
{"id":"app","type":"container","image":"x@sha256:aa","volumes":["${dir:d}:/data"]}]}`)
|
|
if p := DataProblems(Shelf{"w": declared}); len(p) > 0 {
|
|
t.Fatalf("a declared directory is still refused: %v", p)
|
|
}
|
|
}
|
|
|
|
// Consumer data said to live in a provision the module requires is protected only as well as that
|
|
// provision's provider protects its consumers' data.
|
|
func TestConsumerDataInARequiredProvisionIsJudgedAgainstItsProvider(t *testing.T) {
|
|
idp := mustParse(t, `{"module":"idp","version":"1","requires":["postgres-database"],
|
|
"provides":[{"name":"oidc-client","scope":"mesh"}],"grants":{"oidc-client":"${dir:g}"},
|
|
"resources":[{"id":"g","type":"directory","mode":"0700"}],
|
|
"data":{"consumers":{"oidc-client":{"class":"irreplaceable","in":"postgres-database"}}}}`)
|
|
cachy := mustParse(t, strings.Replace(declaredStore, `"consumers":{"postgres-database":{"class":"irreplaceable","in":"store"}}`,
|
|
`"consumers":{"postgres-database":{"class":"cache"}}`, 1))
|
|
if p := strings.Join(DataProblems(Shelf{"idp": idp, "pg": cachy}), "\n"); !strings.Contains(p, "not protected as irreplaceable") {
|
|
t.Fatalf("irreplaceable data kept in a cache passed: %s", p)
|
|
}
|
|
if p := DataProblems(Shelf{"idp": idp, "pg": mustParse(t, declaredStore)}); len(p) > 0 {
|
|
t.Fatalf("refused against a provider that keeps its consumers' data: %v", p)
|
|
}
|
|
}
|
|
|
|
// What a binding is sticky by follows the class (ADR 0232, generalised by 0233): a cache's consumers
|
|
// move freely, a store's do not; an older definition saying nothing still follows its grant.
|
|
func TestKeepingConsumerDataFollowsTheClass(t *testing.T) {
|
|
for class, keeps := range map[string]bool{"irreplaceable": true, "valuable": true, "rebuildable": true, "cache": false, "none": false} {
|
|
in := `,"in":"d"`
|
|
own := `"own":[{"id":"d","path":"${dir:d}","class":"irreplaceable"}],`
|
|
if !keeps {
|
|
in, own = "", ""
|
|
}
|
|
m := mustParse(t, `{"module":"p","version":"1","provides":[{"name":"q","scope":"mesh"}],"grants":{"q":"${dir:d}"},
|
|
"resources":[{"id":"d","type":"directory","mode":"0700"}],
|
|
"data":{`+own+`"consumers":{"q":{"class":"`+class+`"`+in+`}}}}`)
|
|
if m.KeepsConsumerData("q") != keeps {
|
|
t.Errorf("class %s keeps consumer data: %v, want %v", class, !keeps, keeps)
|
|
}
|
|
shelf := map[string]Manifest{"p": m}
|
|
if KeepsConsumerData(shelf, "q") != keeps {
|
|
t.Errorf("class %s: the provision by name keeps: %v", class, !keeps)
|
|
}
|
|
}
|
|
older := mustParse(t, `{"module":"p","version":"1","provides":[{"name":"q","scope":"mesh"}],"grants":{"q":"${dir:d}"},
|
|
"resources":[{"id":"d","type":"directory","mode":"0700"}]}`)
|
|
if !older.KeepsConsumerData("q") {
|
|
t.Fatal("an older definition that grants no longer keeps its consumers' data")
|
|
}
|
|
}
|
|
|
|
// The backup holder's lines are derived: the dump runs and the directory it writes into is kept,
|
|
// anything but a cache is copied by default — the standard nightly plan — an item on redundancy is not
|
|
// copied, and every item is listed with its class and protection for the holder to measure and watch:
|
|
// directories filled, a home directory filled from the machine, an operator's path from where the
|
|
// assignment placed it. A backup line still written by hand beside a data section is not placed.
|
|
func TestTheBackupHoldersLinesAreDerivedFromTheData(t *testing.T) {
|
|
pg := mustParse(t, declaredStore)
|
|
pg.Contributions = append(pg.Contributions, SeatContribution{Seat: BackupSeat, Kind: "backup", Content: "path /stale"})
|
|
agent := mustParse(t, `{"module":"agent","version":"1",
|
|
"data":{"own":[{"id":"home","path":"${dir:home}","class":"valuable"},{"id":"tmp","path":"${dir:home}/tmp","class":"cache"}]},
|
|
"resources":[{"id":"home","type":"directory","path":"${machine:account-home}/.agent","mode":"0700"}]}`)
|
|
media := mustParse(t, `{"module":"media","version":"1",
|
|
"accesses":[{"id":"films","mode":"read"}],
|
|
"data":{"own":[{"id":"films","path":"${access:films}","class":"irreplaceable","redundancy":"on a redundant array; no room for a copy","measure":"dataset"},
|
|
{"id":"meta","path":"${dir:meta}","class":"rebuildable"}]},
|
|
"resources":[{"id":"meta","type":"directory","mode":"0700"}]}`)
|
|
facts := map[string]string{"account-home": "/home/op"}
|
|
with := Rendering{Settings: SettingsBy{"media": {{From: "node", Values: map[string]any{AccessesSetting: map[string]any{"films": "/tank/films"}}}}}}
|
|
backup, err := seatContributions([]Manifest{pg, agent, media}, BackupSeat, BackupKindBackup, with, facts)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
want := "# agent\npath /home/op/.agent\n# media\npath /var/lib/media/meta\n# pg\nrun dump-all > /srv/store/dumps/all\npath /srv/store/dumps\n"
|
|
if backup != want {
|
|
t.Fatalf("backup lines:\n%s\nwant:\n%s", backup, want)
|
|
}
|
|
data, err := seatContributions([]Manifest{pg, agent, media}, BackupSeat, BackupKindData, with, facts)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
want = "# agent\nitem home valuable /home/op/.agent /home/op/.agent backup walk\nitem tmp cache /home/op/.agent/tmp - none walk\n" +
|
|
"# media\nitem films irreplaceable /tank/films - redundancy dataset\nitem meta rebuildable /var/lib/media/meta /var/lib/media/meta backup walk\n" +
|
|
"# pg\nitem store irreplaceable /srv/store /srv/store/dumps backup walk\nitem dumps rebuildable /srv/store/dumps - none walk\n"
|
|
if data != want {
|
|
t.Fatalf("data lines:\n%s\nwant:\n%s", data, want)
|
|
}
|
|
if _, err := seatContributions([]Manifest{agent}, BackupSeat, BackupKindData, Rendering{}, nil); err == nil {
|
|
t.Fatal("a home directory with no account on the machine reached the holder as a placeholder")
|
|
}
|
|
// What keeps something irreplaceable depends on the machine's backup holder — it backs it up or
|
|
// watches its array; valuable data alone is backed up where a holder is, and refused nowhere.
|
|
if !slices.Contains(DependsOn(media), BackupSeat) || !slices.Contains(DependsOn(pg), BackupSeat) ||
|
|
slices.Contains(DependsOn(agent), BackupSeat) {
|
|
t.Fatalf("depends: media %v, pg %v, agent %v", DependsOn(media), DependsOn(pg), DependsOn(agent))
|
|
}
|
|
if it, _ := media.DataItem("films"); it.OwnedByModule() || it.Protection() != "redundancy" {
|
|
t.Fatalf("an operator's path reads %+v", it)
|
|
}
|
|
}
|
|
|
|
// Data a consumer keeps with a provider as irreplaceable must be protected there: the check refuses a
|
|
// provider that keeps its consumers' data as a cache, or in an item with no protection.
|
|
func TestIrreplaceableDataKeptWithAProviderMustBeProtectedThere(t *testing.T) {
|
|
photos := mustParse(t, `{"module":"photos","version":"1","requires":["s3-bucket"],
|
|
"data":{"kept-by":{"s3-bucket":{"class":"irreplaceable"}}}}`)
|
|
store := func(backup string) Manifest {
|
|
return mustParse(t, `{"module":"objects","version":"1","provides":[{"name":"s3-bucket","scope":"mesh"}],
|
|
"grants":{"s3-bucket":"${dir:g}"},
|
|
"data":{"own":[{"id":"data","path":"${dir:data}","class":"valuable","backup":"`+backup+`"}],
|
|
"consumers":{"s3-bucket":{"class":"valuable","in":"data"}}},
|
|
"resources":[{"id":"g","type":"directory","mode":"0700"},{"id":"data","type":"directory","mode":"0700"}]}`)
|
|
}
|
|
if p := DataProblems(Shelf{"photos": photos, "objects": store("copy")}); len(p) > 0 {
|
|
t.Fatalf("a provider backing its consumers' data up was refused: %v", p)
|
|
}
|
|
if p := strings.Join(DataProblems(Shelf{"photos": photos, "objects": store("none")}), "\n"); !strings.Contains(p, "neither backed up") {
|
|
t.Fatalf("irreplaceable photos in an unprotected store passed: %s", p)
|
|
}
|
|
}
|