jschoubben
78c9ec8817
mesh/delivery merged: source.main moved to novox/mesh-sdk@044ead7db83e
mesh/delivery-group group fix/774-new-words-first merged: every member is merged
mesh/check pass 78c9ec881771 on 307b6234e6bb | unit 9174 run, 546 kept, 51 skipped | integration 9952 run, 659 kept, 60 skipped | over bound: unit, in…
254 lines
16 KiB
Go
254 lines
16 KiB
Go
package catalogue
|
|
|
|
import (
|
|
"encoding/json"
|
|
"slices"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
// A store as a module now declares it (novox/hq ADR 0233): its own data by directory with a class and a
|
|
// dump, and what it keeps for its consumers.
|
|
const declaredStore = `{"module":"pg","version":"1",
|
|
"provides":[{"name":"postgres-database","scope":"mesh"}],
|
|
"grants":{"postgres-database":"${dir:grants}"},
|
|
"data":{
|
|
"own":[
|
|
{"id":"store","path":"${dir:store}","class":"irreplaceable","backup":{"dump":"dump-all > ${dir:dumps}/all","into":"dumps"},"active":"1d"},
|
|
{"id":"dumps","path":"${dir:dumps}","class":"rebuildable","backup":"none"}],
|
|
"consumers":{"postgres-database":{"class":"irreplaceable","in":"store"}}},
|
|
"resources":[
|
|
{"id":"grants","type":"directory","mode":"0700"},
|
|
{"id":"store","type":"directory","path":"/srv/store","mode":"0700"},
|
|
{"id":"dumps","type":"directory","path":"${dir:store}/dumps","mode":"0700"},
|
|
{"id":"server","type":"container","image":"x@sha256:aa","volumes":["${dir:store}:/var/lib/postgresql/data"]}]}`
|
|
|
|
func mustParse(t *testing.T, raw string) Manifest {
|
|
t.Helper()
|
|
m, err := ParseManifest([]byte(raw))
|
|
if err != nil {
|
|
t.Fatalf("refused: %v", err)
|
|
}
|
|
return m
|
|
}
|
|
|
|
func TestADeclaredStoreParsesAndPassesTheCheck(t *testing.T) {
|
|
m := mustParse(t, declaredStore)
|
|
if problems := DataProblems(Shelf{"pg": m}); len(problems) > 0 {
|
|
t.Fatalf("a store declaring its data was refused: %v", problems)
|
|
}
|
|
if !m.KeepsConsumerData("postgres-database") {
|
|
t.Fatal("an irreplaceable consumer class does not keep the consumer's data")
|
|
}
|
|
if it, _ := m.DataItem("store"); !it.BackedUp() || it.BackupWithin() != DefaultBackupWithin || it.ActiveWithin().Hours() != 24 {
|
|
t.Fatalf("the store item reads %+v", it)
|
|
}
|
|
if it, _ := m.DataItem("dumps"); it.BackedUp() {
|
|
t.Fatal("a rebuildable item that says none is backed up")
|
|
}
|
|
// Issue 774's new words first: an offer's credential form and an artifact's bus are accepted and change nothing: the
|
|
// manifest that says them composes byte-equal to the one that does not, and no form still means a pair.
|
|
plain := strings.Replace(declaredStore, `"resources":[`, `"build":{"artifacts":[{"name":"tools","kind":"bundle","language":"go","system":"arch","from":"cmd/x","binary":"x","loads":["x"]}]},"resources":[`, 1)
|
|
for _, form := range []string{"pair", "none"} {
|
|
worded := strings.Replace(plain, `"scope":"mesh"}]`, `"scope":"mesh","credential":{"form":"`+form+`"}}]`, 1)
|
|
worded = strings.Replace(worded, `"loads":["x"]}`, `"loads":["x"],"bus":{"state":["acts"],"removes":["engine-acts"]}}`, 1)
|
|
a, b := mustParse(t, plain), mustParse(t, worded)
|
|
if b.Provides[0].Credential != nil || b.CredentialFormOf("postgres-database") != a.CredentialFormOf("postgres-database") ||
|
|
a.CredentialFormOf("postgres-database") != CredentialFormPair || len(b.Build.Artifacts[0].Bus) == 0 {
|
|
t.Fatalf("form %s: offer %+v, form %q against %q", form, b.Provides[0], b.CredentialFormOf("postgres-database"),
|
|
a.CredentialFormOf("postgres-database"))
|
|
}
|
|
b.Build.Artifacts[0].Bus = nil
|
|
ra, erra := (Resolution{Node: "anchor", Modules: []Manifest{a}}).Compose(Rendering{})
|
|
rb, errb := (Resolution{Node: "anchor", Modules: []Manifest{b}}).Compose(Rendering{})
|
|
ja, _ := json.Marshal(ra.Resources)
|
|
jb, _ := json.Marshal(rb.Resources)
|
|
if erra != nil || errb != nil || string(ja) != string(jb) || len(ra.Resources) == 0 {
|
|
t.Fatalf("form %s composes %s against %s (%v, %v)", form, jb, ja, errb, erra)
|
|
}
|
|
}
|
|
if _, err := ParseManifest([]byte(strings.Replace(declaredStore, `"scope":"mesh"}]`, `"scope":"mesh","credential":{"form":"shared"}}]`, 1))); err == nil ||
|
|
!strings.Contains(err.Error(), `the form "shared", not "pair" or "none"`) {
|
|
t.Fatalf("a form the release does not know: %v", err)
|
|
}
|
|
}
|
|
|
|
// Every rule of the section itself, refused at parse in the words of the module.
|
|
func TestTheDataSectionIsRefusedWhereItIsWrong(t *testing.T) {
|
|
for _, c := range []struct{ name, data, want string }{
|
|
{"a class it does not know", `{"own":[{"id":"a","path":"${dir:d}","class":"precious"}]}`, "not one the mesh protects by"},
|
|
{"an access it does not have", `{"own":[{"id":"a","path":"${access:media}","class":"irreplaceable","redundancy":"an array"}]}`, "declares no access"},
|
|
{"irreplaceable and protected by nothing", `{"own":[{"id":"a","path":"${dir:d}","class":"irreplaceable","backup":"none"}]}`, "protected one way or the other"},
|
|
{"a cache on redundancy", `{"own":[{"id":"a","path":"${dir:d}","class":"cache","redundancy":"an array"}]}`, "a cache is not protected"},
|
|
{"kept with something not required", `{"kept-by":{"s3-bucket":{"class":"irreplaceable"}}}`, "does not require"},
|
|
{"a measure it does not know", `{"own":[{"id":"a","path":"${dir:d}","class":"cache","measure":"du"}]}`, "walk, dataset or shallow"},
|
|
{"a cache expecting writes", `{"own":[{"id":"a","path":"${dir:d}","class":"cache","active":"1d"}]}`, "only irreplaceable and valuable"},
|
|
{"a machine path", `{"own":[{"id":"a","path":"/srv/a","class":"cache"}]}`, "names no machine path"},
|
|
{"a directory it does not declare", `{"own":[{"id":"a","path":"${dir:nowhere}","class":"cache"}]}`, "declares no directory"},
|
|
{"a cache backed up", `{"own":[{"id":"a","path":"${dir:d}","class":"cache","backup":"copy"}]}`, "disposable"},
|
|
{"a dump into nothing", `{"own":[{"id":"a","path":"${dir:d}","class":"irreplaceable","backup":{"dump":"x","into":"b"}}]}`, "not one of the module's data items"},
|
|
{"a dump into itself", `{"own":[{"id":"a","path":"${dir:d}","class":"irreplaceable","backup":{"dump":"x","into":"a"}}]}`, "into itself"},
|
|
{"a bad duration", `{"own":[{"id":"a","path":"${dir:d}","class":"irreplaceable","within":"soon"}]}`, "not a duration"},
|
|
{"the same id twice", `{"own":[{"id":"a","path":"${dir:d}","class":"cache"},{"id":"a","path":"${dir:d}/x","class":"cache"}]}`, "declared twice"},
|
|
{"own data of class none", `{"own":[{"id":"a","path":"${dir:d}","class":"none"}]}`, "only a provision"},
|
|
{"consumers of something not provided", `{"consumers":{"s3-bucket":{"class":"cache"}}}`, "does not provide"},
|
|
{"consumer data that lives nowhere", `{"consumers":{"q":{"class":"irreplaceable"}}}`, "says nowhere it lives"},
|
|
{"consumer data in an item less precious", `{"own":[{"id":"a","path":"${dir:d}","class":"rebuildable"}],"consumers":{"q":{"class":"irreplaceable","in":"a"}}}`, "which is only rebuildable"},
|
|
{"consumer data in something unknown", `{"consumers":{"q":{"class":"rebuildable","in":"elsewhere"}}}`, "neither one of its data items"},
|
|
} {
|
|
raw := `{"module":"m","version":"1","provides":["q"],"resources":[{"id":"d","type":"directory","mode":"0700"}],"data":` + c.data + `}`
|
|
_, err := ParseManifest([]byte(raw))
|
|
if err == nil || !strings.Contains(err.Error(), c.want) {
|
|
t.Errorf("%s: %v, want %q", c.name, err, c.want)
|
|
}
|
|
}
|
|
}
|
|
|
|
// The catalogue check: a granting provider says what it keeps, says it in one place, writes no backup
|
|
// line by hand, and every directory a container writes is declared (novox/hq ADR 0233).
|
|
func TestTheCatalogueCheckAsksEveryModuleForItsData(t *testing.T) {
|
|
unsaid := mustParse(t, `{"module":"q","version":"1","provides":[{"name":"queue","scope":"mesh"}],
|
|
"grants":{"queue":"${dir:g}"},"resources":[{"id":"g","type":"directory","mode":"0700"}]}`)
|
|
onTheOffer := mustParse(t, `{"module":"r","version":"1","provides":[{"name":"resolver","scope":"mesh","keeps-consumer-data":false}]}`)
|
|
byHand := mustParse(t, `{"module":"h","version":"1","resources":[{"id":"d","type":"directory","mode":"0700"}],
|
|
"contributions":[{"seat":"node-backup","kind":"backup","content":"path ${dir:d}"}]}`)
|
|
writes := mustParse(t, `{"module":"w","version":"1","resources":[{"id":"d","type":"directory","mode":"0700"},
|
|
{"id":"c","type":"directory","mode":"0700"},
|
|
{"id":"app","type":"container","image":"x@sha256:aa","volumes":["${dir:d}:/data","${dir:c}:/etc/app:ro","${dir:d}/f:/f"]}]}`)
|
|
problems := strings.Join(DataProblems(Shelf{"q": unsaid, "r": onTheOffer, "h": byHand, "w": writes}), "\n")
|
|
for _, want := range []string{
|
|
"q grants queue and does not say what it keeps",
|
|
"r says keeps-consumer-data on its offer",
|
|
"h's contribution 1 is a backup line written by hand",
|
|
`w mounts its directory "d" into a container to be written`,
|
|
} {
|
|
if !strings.Contains(problems, want) {
|
|
t.Errorf("the check does not say %q:\n%s", want, problems)
|
|
}
|
|
}
|
|
if strings.Contains(problems, `"c"`) {
|
|
t.Errorf("a read-only mount was taken for written data:\n%s", problems)
|
|
}
|
|
// The same module declaring the directory, as a cache, passes.
|
|
declared := mustParse(t, `{"module":"w","version":"1","data":{"own":[{"id":"d","path":"${dir:d}","class":"cache"}]},
|
|
"resources":[{"id":"d","type":"directory","mode":"0700"},
|
|
{"id":"app","type":"container","image":"x@sha256:aa","volumes":["${dir:d}:/data"]}]}`)
|
|
if p := DataProblems(Shelf{"w": declared}); len(p) > 0 {
|
|
t.Fatalf("a declared directory is still refused: %v", p)
|
|
}
|
|
}
|
|
|
|
// Consumer data said to live in a provision the module requires is protected only as well as that
|
|
// provision's provider protects its consumers' data.
|
|
func TestConsumerDataInARequiredProvisionIsJudgedAgainstItsProvider(t *testing.T) {
|
|
idp := mustParse(t, `{"module":"idp","version":"1","requires":["postgres-database"],
|
|
"provides":[{"name":"oidc-client","scope":"mesh"}],"grants":{"oidc-client":"${dir:g}"},
|
|
"resources":[{"id":"g","type":"directory","mode":"0700"}],
|
|
"data":{"consumers":{"oidc-client":{"class":"irreplaceable","in":"postgres-database"}}}}`)
|
|
cachy := mustParse(t, strings.Replace(declaredStore, `"consumers":{"postgres-database":{"class":"irreplaceable","in":"store"}}`,
|
|
`"consumers":{"postgres-database":{"class":"cache"}}`, 1))
|
|
if p := strings.Join(DataProblems(Shelf{"idp": idp, "pg": cachy}), "\n"); !strings.Contains(p, "not protected as irreplaceable") {
|
|
t.Fatalf("irreplaceable data kept in a cache passed: %s", p)
|
|
}
|
|
if p := DataProblems(Shelf{"idp": idp, "pg": mustParse(t, declaredStore)}); len(p) > 0 {
|
|
t.Fatalf("refused against a provider that keeps its consumers' data: %v", p)
|
|
}
|
|
}
|
|
|
|
// What a binding is sticky by follows the class (ADR 0232, generalised by 0233): a cache's consumers
|
|
// move freely, a store's do not; an older definition saying nothing still follows its grant.
|
|
func TestKeepingConsumerDataFollowsTheClass(t *testing.T) {
|
|
for class, keeps := range map[string]bool{"irreplaceable": true, "valuable": true, "rebuildable": true, "cache": false, "none": false} {
|
|
in := `,"in":"d"`
|
|
own := `"own":[{"id":"d","path":"${dir:d}","class":"irreplaceable"}],`
|
|
if !keeps {
|
|
in, own = "", ""
|
|
}
|
|
m := mustParse(t, `{"module":"p","version":"1","provides":[{"name":"q","scope":"mesh"}],"grants":{"q":"${dir:d}"},
|
|
"resources":[{"id":"d","type":"directory","mode":"0700"}],
|
|
"data":{`+own+`"consumers":{"q":{"class":"`+class+`"`+in+`}}}}`)
|
|
if m.KeepsConsumerData("q") != keeps {
|
|
t.Errorf("class %s keeps consumer data: %v, want %v", class, !keeps, keeps)
|
|
}
|
|
shelf := map[string]Manifest{"p": m}
|
|
if KeepsConsumerData(shelf, "q") != keeps {
|
|
t.Errorf("class %s: the provision by name keeps: %v", class, !keeps)
|
|
}
|
|
}
|
|
older := mustParse(t, `{"module":"p","version":"1","provides":[{"name":"q","scope":"mesh"}],"grants":{"q":"${dir:d}"},
|
|
"resources":[{"id":"d","type":"directory","mode":"0700"}]}`)
|
|
if !older.KeepsConsumerData("q") {
|
|
t.Fatal("an older definition that grants no longer keeps its consumers' data")
|
|
}
|
|
}
|
|
|
|
// The backup holder's lines are derived: the dump runs and the directory it writes into is kept,
|
|
// anything but a cache is copied by default — the standard nightly plan — an item on redundancy is not
|
|
// copied, and every item is listed with its class and protection for the holder to measure and watch:
|
|
// directories filled, a home directory filled from the machine, an operator's path from where the
|
|
// assignment placed it. A backup line still written by hand beside a data section is not placed.
|
|
func TestTheBackupHoldersLinesAreDerivedFromTheData(t *testing.T) {
|
|
pg := mustParse(t, declaredStore)
|
|
pg.Contributions = append(pg.Contributions, SeatContribution{Seat: BackupSeat, Kind: "backup", Content: "path /stale"})
|
|
agent := mustParse(t, `{"module":"agent","version":"1",
|
|
"data":{"own":[{"id":"home","path":"${dir:home}","class":"valuable"},{"id":"tmp","path":"${dir:home}/tmp","class":"cache"}]},
|
|
"resources":[{"id":"home","type":"directory","path":"${machine:account-home}/.agent","mode":"0700"}]}`)
|
|
media := mustParse(t, `{"module":"media","version":"1",
|
|
"accesses":[{"id":"films","mode":"read"}],
|
|
"data":{"own":[{"id":"films","path":"${access:films}","class":"irreplaceable","redundancy":"on a redundant array; no room for a copy","measure":"dataset"},
|
|
{"id":"meta","path":"${dir:meta}","class":"rebuildable"}]},
|
|
"resources":[{"id":"meta","type":"directory","mode":"0700"}]}`)
|
|
facts := map[string]string{"account-home": "/home/op"}
|
|
with := Rendering{Settings: SettingsBy{"media": {LayerOf("node", map[string]any{AccessesSetting: map[string]any{"films": "/tank/films"}})}}}
|
|
backup, _, err := seatContributions([]Manifest{pg, agent, media}, Manifest{}, BackupSeat+":"+BackupKindBackup, with, facts, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
want := "# agent\npath /home/op/.agent\n# media\npath /var/lib/media/meta\n# pg\nrun dump-all > /srv/store/dumps/all\npath /srv/store/dumps\n"
|
|
if backup != want {
|
|
t.Fatalf("backup lines:\n%s\nwant:\n%s", backup, want)
|
|
}
|
|
data, _, err := seatContributions([]Manifest{pg, agent, media}, Manifest{}, BackupSeat+":"+BackupKindData, with, facts, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
want = "# agent\nitem home valuable /home/op/.agent /home/op/.agent backup walk\nitem tmp cache /home/op/.agent/tmp - none walk\n" +
|
|
"# media\nitem films irreplaceable /tank/films - redundancy dataset\nitem meta rebuildable /var/lib/media/meta /var/lib/media/meta backup walk\n" +
|
|
"# pg\nitem store irreplaceable /srv/store /srv/store/dumps backup walk\nitem dumps rebuildable /srv/store/dumps - none walk\n"
|
|
if data != want {
|
|
t.Fatalf("data lines:\n%s\nwant:\n%s", data, want)
|
|
}
|
|
if _, _, err := seatContributions([]Manifest{agent}, Manifest{}, BackupSeat+":"+BackupKindData, Rendering{}, nil, nil); err == nil {
|
|
t.Fatal("a home directory with no account on the machine reached the holder as a placeholder")
|
|
}
|
|
// What keeps something irreplaceable depends on the machine's backup holder — it backs it up or
|
|
// watches its array; valuable data alone is backed up where a holder is, and refused nowhere.
|
|
if !slices.Contains(DependsOn(nil, media), BackupSeat) || !slices.Contains(DependsOn(nil, pg), BackupSeat) ||
|
|
slices.Contains(DependsOn(nil, agent), BackupSeat) {
|
|
t.Fatalf("depends: media %v, pg %v, agent %v", DependsOn(nil, media), DependsOn(nil, pg), DependsOn(nil, agent))
|
|
}
|
|
if it, _ := media.DataItem("films"); it.OwnedByModule() || it.Protection() != "redundancy" {
|
|
t.Fatalf("an operator's path reads %+v", it)
|
|
}
|
|
}
|
|
|
|
// Data a consumer keeps with a provider as irreplaceable must be protected there: the check refuses a
|
|
// provider that keeps its consumers' data as a cache, or in an item with no protection.
|
|
func TestIrreplaceableDataKeptWithAProviderMustBeProtectedThere(t *testing.T) {
|
|
photos := mustParse(t, `{"module":"photos","version":"1","requires":["s3-bucket"],
|
|
"data":{"kept-by":{"s3-bucket":{"class":"irreplaceable"}}}}`)
|
|
store := func(backup string) Manifest {
|
|
return mustParse(t, `{"module":"objects","version":"1","provides":[{"name":"s3-bucket","scope":"mesh"}],
|
|
"grants":{"s3-bucket":"${dir:g}"},
|
|
"data":{"own":[{"id":"data","path":"${dir:data}","class":"valuable","backup":"`+backup+`"}],
|
|
"consumers":{"s3-bucket":{"class":"valuable","in":"data"}}},
|
|
"resources":[{"id":"g","type":"directory","mode":"0700"},{"id":"data","type":"directory","mode":"0700"}]}`)
|
|
}
|
|
if p := DataProblems(Shelf{"photos": photos, "objects": store("copy")}); len(p) > 0 {
|
|
t.Fatalf("a provider backing its consumers' data up was refused: %v", p)
|
|
}
|
|
if p := strings.Join(DataProblems(Shelf{"photos": photos, "objects": store("none")}), "\n"); !strings.Contains(p, "neither backed up") {
|
|
t.Fatalf("irreplaceable photos in an unprotected store passed: %s", p)
|
|
}
|
|
}
|