jochen
52af210e47
A module's data section says what it keeps and how precious it is; the backup holder's lines, binding stickiness, retirement on unassign and D13's conditions follow from it, so issue 273's empty replacement is said and an unassigned module's data is remembered, not forgotten.
227 lines
14 KiB
Go
227 lines
14 KiB
Go
package catalogue
|
|
|
|
import (
|
|
"slices"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
// A store as a module now declares it (novox/hq ADR 0233): its own data by directory with a class and a
|
|
// dump, and what it keeps for its consumers.
|
|
const declaredStore = `{"module":"pg","version":"1",
|
|
"provides":[{"name":"postgres-database","scope":"mesh"}],
|
|
"grants":{"postgres-database":"${dir:grants}"},
|
|
"data":{
|
|
"own":[
|
|
{"id":"store","path":"${dir:store}","class":"irreplaceable","backup":{"dump":"dump-all > ${dir:dumps}/all","into":"dumps"},"active":"1d"},
|
|
{"id":"dumps","path":"${dir:dumps}","class":"rebuildable","backup":"none"}],
|
|
"consumers":{"postgres-database":{"class":"irreplaceable","in":"store"}}},
|
|
"resources":[
|
|
{"id":"grants","type":"directory","mode":"0700"},
|
|
{"id":"store","type":"directory","path":"/srv/store","mode":"0700"},
|
|
{"id":"dumps","type":"directory","path":"${dir:store}/dumps","mode":"0700"},
|
|
{"id":"server","type":"container","image":"x@sha256:aa","volumes":["${dir:store}:/var/lib/postgresql/data"]}]}`
|
|
|
|
func mustParse(t *testing.T, raw string) Manifest {
|
|
t.Helper()
|
|
m, err := ParseManifest([]byte(raw))
|
|
if err != nil {
|
|
t.Fatalf("refused: %v", err)
|
|
}
|
|
return m
|
|
}
|
|
|
|
func TestADeclaredStoreParsesAndPassesTheCheck(t *testing.T) {
|
|
m := mustParse(t, declaredStore)
|
|
if problems := DataProblems(Shelf{"pg": m}); len(problems) > 0 {
|
|
t.Fatalf("a store declaring its data was refused: %v", problems)
|
|
}
|
|
if !m.KeepsConsumerData("postgres-database") {
|
|
t.Fatal("an irreplaceable consumer class does not keep the consumer's data")
|
|
}
|
|
if it, _ := m.DataItem("store"); !it.BackedUp() || it.BackupWithin() != DefaultBackupWithin || it.ActiveWithin().Hours() != 24 {
|
|
t.Fatalf("the store item reads %+v", it)
|
|
}
|
|
if it, _ := m.DataItem("dumps"); it.BackedUp() {
|
|
t.Fatal("a rebuildable item that says none is backed up")
|
|
}
|
|
}
|
|
|
|
// Every rule of the section itself, refused at parse in the words of the module.
|
|
func TestTheDataSectionIsRefusedWhereItIsWrong(t *testing.T) {
|
|
for _, c := range []struct{ name, data, want string }{
|
|
{"a class it does not know", `{"own":[{"id":"a","path":"${dir:d}","class":"precious"}]}`, "not one the mesh protects by"},
|
|
{"an access it does not have", `{"own":[{"id":"a","path":"${access:media}","class":"irreplaceable","redundancy":"an array"}]}`, "declares no access"},
|
|
{"irreplaceable and protected by nothing", `{"own":[{"id":"a","path":"${dir:d}","class":"irreplaceable","backup":"none"}]}`, "protected one way or the other"},
|
|
{"a cache on redundancy", `{"own":[{"id":"a","path":"${dir:d}","class":"cache","redundancy":"an array"}]}`, "a cache is not protected"},
|
|
{"kept with something not required", `{"kept-by":{"s3-bucket":{"class":"irreplaceable"}}}`, "does not require"},
|
|
{"a cache expecting writes", `{"own":[{"id":"a","path":"${dir:d}","class":"cache","active":"1d"}]}`, "only irreplaceable and valuable"},
|
|
{"a machine path", `{"own":[{"id":"a","path":"/srv/a","class":"cache"}]}`, "names no machine path"},
|
|
{"a directory it does not declare", `{"own":[{"id":"a","path":"${dir:nowhere}","class":"cache"}]}`, "declares no directory"},
|
|
{"a cache backed up", `{"own":[{"id":"a","path":"${dir:d}","class":"cache","backup":"copy"}]}`, "disposable"},
|
|
{"a dump into nothing", `{"own":[{"id":"a","path":"${dir:d}","class":"irreplaceable","backup":{"dump":"x","into":"b"}}]}`, "not one of the module's data items"},
|
|
{"a dump into itself", `{"own":[{"id":"a","path":"${dir:d}","class":"irreplaceable","backup":{"dump":"x","into":"a"}}]}`, "into itself"},
|
|
{"a bad duration", `{"own":[{"id":"a","path":"${dir:d}","class":"irreplaceable","within":"soon"}]}`, "not a duration"},
|
|
{"the same id twice", `{"own":[{"id":"a","path":"${dir:d}","class":"cache"},{"id":"a","path":"${dir:d}/x","class":"cache"}]}`, "declared twice"},
|
|
{"own data of class none", `{"own":[{"id":"a","path":"${dir:d}","class":"none"}]}`, "only a provision"},
|
|
{"consumers of something not provided", `{"consumers":{"s3-bucket":{"class":"cache"}}}`, "does not provide"},
|
|
{"consumer data that lives nowhere", `{"consumers":{"q":{"class":"irreplaceable"}}}`, "says nowhere it lives"},
|
|
{"consumer data in an item less precious", `{"own":[{"id":"a","path":"${dir:d}","class":"rebuildable"}],"consumers":{"q":{"class":"irreplaceable","in":"a"}}}`, "which is only rebuildable"},
|
|
{"consumer data in something unknown", `{"consumers":{"q":{"class":"rebuildable","in":"elsewhere"}}}`, "neither one of its data items"},
|
|
} {
|
|
raw := `{"module":"m","version":"1","provides":["q"],"resources":[{"id":"d","type":"directory","mode":"0700"}],"data":` + c.data + `}`
|
|
_, err := ParseManifest([]byte(raw))
|
|
if err == nil || !strings.Contains(err.Error(), c.want) {
|
|
t.Errorf("%s: %v, want %q", c.name, err, c.want)
|
|
}
|
|
}
|
|
}
|
|
|
|
// The catalogue check: a granting provider says what it keeps, says it in one place, writes no backup
|
|
// line by hand, and every directory a container writes is declared (novox/hq ADR 0233).
|
|
func TestTheCatalogueCheckAsksEveryModuleForItsData(t *testing.T) {
|
|
unsaid := mustParse(t, `{"module":"q","version":"1","provides":[{"name":"queue","scope":"mesh"}],
|
|
"grants":{"queue":"${dir:g}"},"resources":[{"id":"g","type":"directory","mode":"0700"}]}`)
|
|
onTheOffer := mustParse(t, `{"module":"r","version":"1","provides":[{"name":"resolver","scope":"mesh","keeps-consumer-data":false}]}`)
|
|
byHand := mustParse(t, `{"module":"h","version":"1","resources":[{"id":"d","type":"directory","mode":"0700"}],
|
|
"contributions":[{"seat":"node-backup","kind":"backup","content":"path ${dir:d}"}]}`)
|
|
writes := mustParse(t, `{"module":"w","version":"1","resources":[{"id":"d","type":"directory","mode":"0700"},
|
|
{"id":"c","type":"directory","mode":"0700"},
|
|
{"id":"app","type":"container","image":"x@sha256:aa","volumes":["${dir:d}:/data","${dir:c}:/etc/app:ro","${dir:d}/f:/f"]}]}`)
|
|
problems := strings.Join(DataProblems(Shelf{"q": unsaid, "r": onTheOffer, "h": byHand, "w": writes}), "\n")
|
|
for _, want := range []string{
|
|
"q grants queue and does not say what it keeps",
|
|
"r says keeps-consumer-data on its offer",
|
|
"h's contribution 1 is a backup line written by hand",
|
|
`w mounts its directory "d" into a container to be written`,
|
|
} {
|
|
if !strings.Contains(problems, want) {
|
|
t.Errorf("the check does not say %q:\n%s", want, problems)
|
|
}
|
|
}
|
|
if strings.Contains(problems, `"c"`) {
|
|
t.Errorf("a read-only mount was taken for written data:\n%s", problems)
|
|
}
|
|
// The same module declaring the directory, as a cache, passes.
|
|
declared := mustParse(t, `{"module":"w","version":"1","data":{"own":[{"id":"d","path":"${dir:d}","class":"cache"}]},
|
|
"resources":[{"id":"d","type":"directory","mode":"0700"},
|
|
{"id":"app","type":"container","image":"x@sha256:aa","volumes":["${dir:d}:/data"]}]}`)
|
|
if p := DataProblems(Shelf{"w": declared}); len(p) > 0 {
|
|
t.Fatalf("a declared directory is still refused: %v", p)
|
|
}
|
|
}
|
|
|
|
// Consumer data said to live in a provision the module requires is protected only as well as that
|
|
// provision's provider protects its consumers' data.
|
|
func TestConsumerDataInARequiredProvisionIsJudgedAgainstItsProvider(t *testing.T) {
|
|
idp := mustParse(t, `{"module":"idp","version":"1","requires":["postgres-database"],
|
|
"provides":[{"name":"oidc-client","scope":"mesh"}],"grants":{"oidc-client":"${dir:g}"},
|
|
"resources":[{"id":"g","type":"directory","mode":"0700"}],
|
|
"data":{"consumers":{"oidc-client":{"class":"irreplaceable","in":"postgres-database"}}}}`)
|
|
cachy := mustParse(t, strings.Replace(declaredStore, `"consumers":{"postgres-database":{"class":"irreplaceable","in":"store"}}`,
|
|
`"consumers":{"postgres-database":{"class":"cache"}}`, 1))
|
|
if p := strings.Join(DataProblems(Shelf{"idp": idp, "pg": cachy}), "\n"); !strings.Contains(p, "not protected as irreplaceable") {
|
|
t.Fatalf("irreplaceable data kept in a cache passed: %s", p)
|
|
}
|
|
if p := DataProblems(Shelf{"idp": idp, "pg": mustParse(t, declaredStore)}); len(p) > 0 {
|
|
t.Fatalf("refused against a provider that keeps its consumers' data: %v", p)
|
|
}
|
|
}
|
|
|
|
// What a binding is sticky by follows the class (ADR 0232, generalised by 0233): a cache's consumers
|
|
// move freely, a store's do not; an older definition saying nothing still follows its grant.
|
|
func TestKeepingConsumerDataFollowsTheClass(t *testing.T) {
|
|
for class, keeps := range map[string]bool{"irreplaceable": true, "valuable": true, "rebuildable": true, "cache": false, "none": false} {
|
|
in := `,"in":"d"`
|
|
own := `"own":[{"id":"d","path":"${dir:d}","class":"irreplaceable"}],`
|
|
if !keeps {
|
|
in, own = "", ""
|
|
}
|
|
m := mustParse(t, `{"module":"p","version":"1","provides":[{"name":"q","scope":"mesh"}],"grants":{"q":"${dir:d}"},
|
|
"resources":[{"id":"d","type":"directory","mode":"0700"}],
|
|
"data":{`+own+`"consumers":{"q":{"class":"`+class+`"`+in+`}}}}`)
|
|
if m.KeepsConsumerData("q") != keeps {
|
|
t.Errorf("class %s keeps consumer data: %v, want %v", class, !keeps, keeps)
|
|
}
|
|
shelf := map[string]Manifest{"p": m}
|
|
if KeepsConsumerData(shelf, "q") != keeps {
|
|
t.Errorf("class %s: the provision by name keeps: %v", class, !keeps)
|
|
}
|
|
}
|
|
older := mustParse(t, `{"module":"p","version":"1","provides":[{"name":"q","scope":"mesh"}],"grants":{"q":"${dir:d}"},
|
|
"resources":[{"id":"d","type":"directory","mode":"0700"}]}`)
|
|
if !older.KeepsConsumerData("q") {
|
|
t.Fatal("an older definition that grants no longer keeps its consumers' data")
|
|
}
|
|
}
|
|
|
|
// The backup holder's lines are derived: the dump runs and the directory it writes into is kept,
|
|
// anything but a cache is copied by default — the standard nightly plan — an item on redundancy is not
|
|
// copied, and every item is listed with its class and protection for the holder to measure and watch:
|
|
// directories filled, a home directory filled from the machine, an operator's path from where the
|
|
// assignment placed it. A backup line still written by hand beside a data section is not placed.
|
|
func TestTheBackupHoldersLinesAreDerivedFromTheData(t *testing.T) {
|
|
pg := mustParse(t, declaredStore)
|
|
pg.Contributions = append(pg.Contributions, SeatContribution{Seat: BackupSeat, Kind: "backup", Content: "path /stale"})
|
|
agent := mustParse(t, `{"module":"agent","version":"1",
|
|
"data":{"own":[{"id":"home","path":"${dir:home}","class":"valuable"},{"id":"tmp","path":"${dir:home}/tmp","class":"cache"}]},
|
|
"resources":[{"id":"home","type":"directory","path":"${machine:account-home}/.agent","mode":"0700"}]}`)
|
|
media := mustParse(t, `{"module":"media","version":"1",
|
|
"accesses":[{"id":"films","mode":"read"}],
|
|
"data":{"own":[{"id":"films","path":"${access:films}","class":"irreplaceable","redundancy":"on a redundant array; no room for a copy"},
|
|
{"id":"meta","path":"${dir:meta}","class":"rebuildable"}]},
|
|
"resources":[{"id":"meta","type":"directory","mode":"0700"}]}`)
|
|
facts := map[string]string{"account-home": "/home/op"}
|
|
with := Rendering{Settings: SettingsBy{"media": {{From: "node", Values: map[string]any{AccessesSetting: map[string]any{"films": "/tank/films"}}}}}}
|
|
backup, err := seatContributions([]Manifest{pg, agent, media}, BackupSeat, BackupKindBackup, with, facts)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
want := "# agent\npath /home/op/.agent\n# media\npath /var/lib/media/meta\n# pg\nrun dump-all > /srv/store/dumps/all\npath /srv/store/dumps\n"
|
|
if backup != want {
|
|
t.Fatalf("backup lines:\n%s\nwant:\n%s", backup, want)
|
|
}
|
|
data, err := seatContributions([]Manifest{pg, agent, media}, BackupSeat, BackupKindData, with, facts)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
want = "# agent\nitem home valuable /home/op/.agent /home/op/.agent backup\nitem tmp cache /home/op/.agent/tmp - none\n" +
|
|
"# media\nitem films irreplaceable /tank/films - redundancy\nitem meta rebuildable /var/lib/media/meta /var/lib/media/meta backup\n" +
|
|
"# pg\nitem store irreplaceable /srv/store /srv/store/dumps backup\nitem dumps rebuildable /srv/store/dumps - none\n"
|
|
if data != want {
|
|
t.Fatalf("data lines:\n%s\nwant:\n%s", data, want)
|
|
}
|
|
if _, err := seatContributions([]Manifest{agent}, BackupSeat, BackupKindData, Rendering{}, nil); err == nil {
|
|
t.Fatal("a home directory with no account on the machine reached the holder as a placeholder")
|
|
}
|
|
// What keeps something irreplaceable depends on the machine's backup holder — it backs it up or
|
|
// watches its array; valuable data alone is backed up where a holder is, and refused nowhere.
|
|
if !slices.Contains(DependsOn(media), BackupSeat) || !slices.Contains(DependsOn(pg), BackupSeat) ||
|
|
slices.Contains(DependsOn(agent), BackupSeat) {
|
|
t.Fatalf("depends: media %v, pg %v, agent %v", DependsOn(media), DependsOn(pg), DependsOn(agent))
|
|
}
|
|
if it, _ := media.DataItem("films"); it.OwnedByModule() || it.Protection() != "redundancy" {
|
|
t.Fatalf("an operator's path reads %+v", it)
|
|
}
|
|
}
|
|
|
|
// Data a consumer keeps with a provider as irreplaceable must be protected there: the check refuses a
|
|
// provider that keeps its consumers' data as a cache, or in an item with no protection.
|
|
func TestIrreplaceableDataKeptWithAProviderMustBeProtectedThere(t *testing.T) {
|
|
photos := mustParse(t, `{"module":"photos","version":"1","requires":["s3-bucket"],
|
|
"data":{"kept-by":{"s3-bucket":{"class":"irreplaceable"}}}}`)
|
|
store := func(backup string) Manifest {
|
|
return mustParse(t, `{"module":"objects","version":"1","provides":[{"name":"s3-bucket","scope":"mesh"}],
|
|
"grants":{"s3-bucket":"${dir:g}"},
|
|
"data":{"own":[{"id":"data","path":"${dir:data}","class":"valuable","backup":"`+backup+`"}],
|
|
"consumers":{"s3-bucket":{"class":"valuable","in":"data"}}},
|
|
"resources":[{"id":"g","type":"directory","mode":"0700"},{"id":"data","type":"directory","mode":"0700"}]}`)
|
|
}
|
|
if p := DataProblems(Shelf{"photos": photos, "objects": store("copy")}); len(p) > 0 {
|
|
t.Fatalf("a provider backing its consumers' data up was refused: %v", p)
|
|
}
|
|
if p := strings.Join(DataProblems(Shelf{"photos": photos, "objects": store("none")}), "\n"); !strings.Contains(p, "neither backed up") {
|
|
t.Fatalf("irreplaceable photos in an unprotected store passed: %s", p)
|
|
}
|
|
}
|