diff --git a/cmd/mesh-compose/main_test.go b/cmd/mesh-compose/main_test.go index f70cf66..2ce1355 100644 --- a/cmd/mesh-compose/main_test.go +++ b/cmd/mesh-compose/main_test.go @@ -3,6 +3,9 @@ package main import ( "bytes" "encoding/json" + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + "github.com/novox/mesh-host/internal/buckets/bucketstest" "go/build" "os" "os/exec" @@ -14,7 +17,6 @@ import ( "testing" "git.novox.be/novox/mesh-sdk/go/catalogue" - "git.novox.be/novox/mesh-sdk/go/delivery" "github.com/novox/mesh-host/internal/beside" "github.com/novox/mesh-host/internal/compose" "github.com/novox/mesh-host/internal/composer" @@ -94,7 +96,7 @@ func TestTheCommandWritesOneVerdictPerManifestAndMachine(t *testing.T) { r := request(t, composer.DepthResolve) r.Cases[0].Manifests["later"] = composer.Manifest{Raw: json.RawMessage(`{"module":"later","version":"1","shares":["x"]}`)} r.Cases[0].Manifests["refused"] = composer.Manifest{Raw: json.RawMessage(`{"module":"refused","version":"1"}`)} - r.Cases[0].Verdicts = map[string]delivery.ManifestVerdict{"refused": {State: delivery.VerdictRefused, Problems: []string{"it is wrong"}}} + r.Cases[0].Verdicts = bucketstest.Raw(map[string]meshcatalog.VerdictsByDigestValue{"refused": {V: 1, Module: "refused", Judge: "src2:j", State: meshcatalogclient.VerdictRefused, Problems: []string{"it is wrong"}}}) code, v, stderr := runJSON(t, nil, encode(t, r)) if code != 0 { t.Fatalf("exit %d: %s", code, stderr) @@ -280,7 +282,7 @@ func ownPackages(t *testing.T) map[string][]string { } // 1621/AC2 and AC4: the command imports, through every package of this repository it reaches, only the standard -// library, mesh-sdk, the contracts modules' pure packages (seats' types, clients' codecs), the private network's overlay package (mesh-wireguard's own pure package: names, graph and +// library, mesh-sdk, the contracts modules' pure packages (seats' types, clients' codecs), the delivery SDK's composer (the command's own request and verdict types), the private network's overlay package (mesh-wireguard's own pure package: names, graph and // generator, no bus) and this repository's own packages, and none that reaches a bus, a store, a network service or // another process. func TestTheCommandImportsNoBusStoreNetworkOrProcess(t *testing.T) { @@ -292,7 +294,7 @@ func TestTheCommandImportsNoBusStoreNetworkOrProcess(t *testing.T) { for path, imports := range own { for _, imp := range imports { std := !strings.Contains(strings.SplitN(imp, "/", 2)[0], ".") - if !std && !strings.HasPrefix(imp, "git.novox.be/novox/mesh-networking/modules/mesh-wireguard/go/overlay") && !strings.HasPrefix(imp, "git.novox.be/novox/mesh-sdk/go") && !strings.HasPrefix(imp, "git.novox.be/novox/mesh-contracts/") && !strings.HasPrefix(imp, "git.novox.be/novox/mesh-delivery/contracts/") && !strings.HasPrefix(imp, "github.com/novox/mesh-host/") { + if !std && !strings.HasPrefix(imp, "git.novox.be/novox/mesh-networking/modules/mesh-wireguard/go/overlay") && !strings.HasPrefix(imp, "git.novox.be/novox/mesh-sdk/go") && !strings.HasPrefix(imp, "git.novox.be/novox/mesh-contracts/") && !strings.HasPrefix(imp, "git.novox.be/novox/mesh-delivery/contracts/") && !strings.HasPrefix(imp, "git.novox.be/novox/mesh-delivery/sdk/composer") && !strings.HasPrefix(imp, "github.com/novox/mesh-host/") { t.Errorf("%s imports %s, which is neither the standard library, mesh-sdk nor this repository", path, imp) } for _, f := range forbidden { diff --git a/cmd/mesh-host/nodeagent_integration_test.go b/cmd/mesh-host/nodeagent_integration_test.go index 9e6e94f..eda3474 100644 --- a/cmd/mesh-host/nodeagent_integration_test.go +++ b/cmd/mesh-host/nodeagent_integration_test.go @@ -11,6 +11,9 @@ import ( "git.novox.be/novox/mesh-contracts/placement/go/clients/meshmoduleassignerclient" "git.novox.be/novox/mesh-contracts/placement/go/seats/meshmoduleassigner" "git.novox.be/novox/mesh-contracts/placement/go/seats/meshwants" + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" "github.com/novox/mesh-host/internal/buckets" "github.com/novox/mesh-host/internal/buckets/bucketstest" engcomp "github.com/novox/mesh-host/internal/compositions" @@ -184,7 +187,7 @@ func (b *itBed) say(line string) { b.t.Log(line) } -func fpIt(c string) string { return delivery.SourcePrefix2 + strings.Repeat(c, 64) } +func fpIt(c string) string { return meshcatalogclient.SourcePrefix2 + strings.Repeat(c, 64) } func fpDigest(c string) string { return "sha256:" + strings.Repeat(c, 64) } @@ -411,7 +414,7 @@ func h8(t *testing.T, b *itBed, checkBuild bool) (func(), h8state) { } return kv } - placement, modules, manifests := kvOf(buckets.Placement), kvOf(delivery.ModulesBucket), kvOf(delivery.ManifestsBucket) + placement, modules, manifests := kvOf(buckets.Placement), kvOf(buckets.Modules), kvOf(buckets.Manifests) resolved := kvOf(selfcompose.ResolvedBucket) builds := kvOf(delivery.BuilderBuildsBucket) candidates := kvOf(buckets.PlacementCandidates) @@ -421,11 +424,11 @@ func h8(t *testing.T, b *itBed, checkBuild bool) (func(), h8state) { `"resources":[{"id":"conf","type":"file","path":"` + path + `","mode":"0644","content":"greeting=${config:greeting}\n"},` + `{"id":"tools","type":"package","package":"web-tools"}],` + `"build":{"artifacts":[{"name":"files","kind":"archive","from":"files"}]}}` - digest := delivery.ManifestDigest([]byte(man)) - mraw, _ := delivery.EncodeManifest(delivery.Manifest{Module: "web", JSON: man}) - mkey, _ := delivery.ManifestsKey(digest) + digest := meshcatalogclient.ManifestDigest([]byte(man)) + mraw, _ := meshcatalogclient.EncodeManifest(meshcatalog.ManifestsByDigestValue{Module: "web", Manifest: man}) + mkey, _ := meshcatalogclient.ManifestsKey(digest) fp := fpIt("a") - modv, _ := delivery.EncodeModule(delivery.Module{FP2: fp, Manifest: digest, Repository: "novox/web", Commit: strings.Repeat("1", 40)}) + modv, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fp, Manifest: digest, Repository: "novox/web", Commit: strings.Repeat("1", 40)}) place, _ := meshmoduleassignerclient.EncodePlacement(meshmoduleassigner.Placement{Node: itNode, Modules: []meshmoduleassigner.PlacedModule{{Module: "web"}}}) rec, _ := nec.EncodeHaveNode(nodeengine.HaveByNodeNodeValue{Record: &nodeengine.NodeRecord{Node: itNode, Capabilities: []string{"service-manager", "package-manager"}}}) // it1 in one key per module and node (research 072 row 37), it2 in the form before: a node reads both. @@ -436,10 +439,10 @@ func h8(t *testing.T, b *itBed, checkBuild bool) (func(), h8state) { // A provider on it2 serving from its own config value: a mail domain, which it1's composition settles. mailer := `{"module":"mailer","version":"1","config":{"domain":{"type":"string","description":"the mail domain","required":true}},` + `"provides":[{"name":"smtp","scope":"mesh"}],"serves":{"smtp":{"port":587,"domain":"${config:domain}"}}}` - mdigest := delivery.ManifestDigest([]byte(mailer)) - mlraw, _ := delivery.EncodeManifest(delivery.Manifest{Module: "mailer", JSON: mailer}) - mlkey, _ := delivery.ManifestsKey(mdigest) - mlmod, _ := delivery.EncodeModule(delivery.Module{FP2: fpIt("b"), Manifest: mdigest, Repository: "novox/mailer", Commit: strings.Repeat("2", 40)}) + mdigest := meshcatalogclient.ManifestDigest([]byte(mailer)) + mlraw, _ := meshcatalogclient.EncodeManifest(meshcatalog.ManifestsByDigestValue{Module: "mailer", Manifest: mailer}) + mlkey, _ := meshcatalogclient.ManifestsKey(mdigest) + mlmod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fpIt("b"), Manifest: mdigest, Repository: "novox/mailer", Commit: strings.Repeat("2", 40)}) place2, _ := meshmoduleassignerclient.EncodePlacement(meshmoduleassigner.Placement{Node: "it2", Modules: []meshmoduleassigner.PlacedModule{{Module: "mailer"}}}) rec2, _ := nec.EncodeHaveNode(nodeengine.HaveByNodeNodeValue{Record: &nodeengine.NodeRecord{Node: "it2", Capabilities: []string{"service-manager"}}}) pkey2, _ := meshmoduleassignerclient.LegacyKey("it2") @@ -582,8 +585,8 @@ func h8candidate(t *testing.T, b *itBed, fp, running string) { } return kv } - cands, _ := kvOf(delivery.CandidatesBucket), kvOf(delivery.VerdictsBucket) - manifests, err := b.js.KeyValue(delivery.ManifestsBucket) + cands, _ := kvOf(buckets.Candidates), kvOf(buckets.Verdicts) + manifests, err := b.js.KeyValue(buckets.Manifests) if err != nil { t.Fatal(err) } @@ -591,25 +594,25 @@ func h8candidate(t *testing.T, b *itBed, fp, running string) { man := `{"module":"web","version":"2","config":{"greeting":{"type":"string","description":"what the file says","default":"hello"}},` + `"resources":[{"id":"conf","type":"file","path":"` + path + `","mode":"0640","content":"greeting=${config:greeting}\n"},` + `{"id":"tools","type":"package","package":"web-tools"}]}` - digest := delivery.ManifestDigest([]byte(man)) - mraw, _ := delivery.EncodeManifest(delivery.Manifest{Module: "web", JSON: man}) - mkey, _ := delivery.ManifestsKey(digest) + digest := meshcatalogclient.ManifestDigest([]byte(man)) + mraw, _ := meshcatalogclient.EncodeManifest(meshcatalog.ManifestsByDigestValue{Module: "web", Manifest: man}) + mkey, _ := meshcatalogclient.ManifestsKey(digest) if _, err := manifests.Put(mkey, mraw); err != nil { t.Fatal(err) } - head := delivery.Head{Repository: "novox/web", Number: 3, Head: strings.Repeat("3", 40)} - heads, _ := delivery.HeadsHash([]delivery.Head{head}) - pulls := []delivery.CandidatePull{{Head: head, Modules: []string{"web"}, - Fps: map[string]delivery.CandidateModule{"web": {FP2: fpIt("c"), Manifest: digest}}}} + head := meshdeliveryclient.Head{Repository: "novox/web", Number: 3, Head: strings.Repeat("3", 40)} + heads, _ := meshdeliveryclient.HeadsHash([]meshdeliveryclient.Head{head}) + pulls := []meshcatalog.CandidatesByGroupValuePullsItem{bucketstest.Pull(head, []string{"web"}, + map[string]meshcatalog.CandidatesByGroupValuePullsItemFpsValue{"web": {Fp2: fpIt("c"), Manifest: digest}})} // The indexer keys the candidate by its group (a branch name, slash allowed); the checker asks by the composition key. const group = "fix/3-web" - craw, err := delivery.EncodeCandidate(delivery.Candidate{Group: group, HeadsHash: heads, Pulls: pulls, - FPSet: delivery.FingerprintSet(pulls)}) + craw, err := bucketstest.GroupCandidate(meshcatalog.CandidatesByGroupValue{Group: group, Heads: heads, Pulls: pulls, + Fpset: meshcatalogclient.FingerprintSet(pulls)}) if err != nil { t.Fatal(err) } - ckey, _ := delivery.CandidatesKey(group) - key, _ := delivery.CompositionKey(group, heads) + ckey, _ := buckets.CandidateKey(group) + key, _ := nec.CompositionKey(group, heads) if _, err := cands.Put(ckey, craw); err != nil { t.Fatal(err) } @@ -765,7 +768,7 @@ func TestNatsTheBusServerIsTakenOverAndReplacedOnlyWhenItsBuildChangesIt(t *test } return kv } - placement, modules, manifests := kvOf(buckets.Placement), kvOf(delivery.ModulesBucket), kvOf(delivery.ManifestsBucket) + placement, modules, manifests := kvOf(buckets.Placement), kvOf(buckets.Modules), kvOf(buckets.Manifests) kvOf(selfcompose.ResolvedBucket) builds := kvOf(delivery.BuilderBuildsBucket) kvOf(buckets.PlacementCandidates) @@ -776,10 +779,10 @@ func TestNatsTheBusServerIsTakenOverAndReplacedOnlyWhenItsBuildChangesIt(t *test `"data":{"own":[{"id":"jetstream","path":"` + b.dir + `","class":"valuable","why":"the bus's streams"}]},"resources":[` + `{"id":"server-conf","type":"file","path":"` + conf + `","mode":"0644","content":"listen: new\n"},` + `{"id":"server","type":"container","name":"` + server + `","image":"` + image + `"` + env + `}]}` - digest := delivery.ManifestDigest([]byte(man)) - mraw, _ := delivery.EncodeManifest(delivery.Manifest{Module: "nats", JSON: man}) - mkey, _ := delivery.ManifestsKey(digest) - modv, _ := delivery.EncodeModule(delivery.Module{FP2: fp, Manifest: digest, Repository: "novox/mesh-catalog", Commit: strings.Repeat("3", 40)}) + digest := meshcatalogclient.ManifestDigest([]byte(man)) + mraw, _ := meshcatalogclient.EncodeManifest(meshcatalog.ManifestsByDigestValue{Module: "nats", Manifest: man}) + mkey, _ := meshcatalogclient.ManifestsKey(digest) + modv, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fp, Manifest: digest, Repository: "novox/mesh-catalog", Commit: strings.Repeat("3", 40)}) if _, err := manifests.Put(mkey, mraw); err != nil { t.Fatal(err) } diff --git a/cmd/mesh-host/trust.go b/cmd/mesh-host/trust.go index 74f7f10..4eb1e7d 100644 --- a/cmd/mesh-host/trust.go +++ b/cmd/mesh-host/trust.go @@ -11,6 +11,7 @@ import ( "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" "git.novox.be/novox/mesh-contracts/placement/go/clients/meshmoduleassignerclient" "git.novox.be/novox/mesh-contracts/placement/go/seats/meshmoduleassigner" + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" "github.com/novox/mesh-host/internal/buckets" "os" "path/filepath" @@ -244,8 +245,8 @@ func allHave(p meshmoduleassigner.Placement, has func(module string) (nodeengine return false, m.Module + " has no have yet" } // A module the mesh provides (mesh-wireguard) never has a fingerprint: the node's own composes it from the manifest - // mesh-sdk provides (delivery.ProvidedManifest), never from the kept declaration, so it keeps no read. - if h.Phase != nodeengineclient.PhaseRemoved && !strings.HasPrefix(h.Fp, delivery.SourcePrefix2) && !delivery.ProvidedByMesh(m.Module) { + // mesh-sdk provides (overlay.ProvidedManifest), never from the kept declaration, so it keeps no read. + if h.Phase != nodeengineclient.PhaseRemoved && !strings.HasPrefix(h.Fp, meshcatalogclient.SourcePrefix2) && !overlay.ProvidedByMesh(m.Module) { return false, m.Module + " runs at a build of the old form (" + h.Build + "), which only the kept declaration records" } } diff --git a/cmd/mesh-host/whatruns_test.go b/cmd/mesh-host/whatruns_test.go index 17067bc..79bbda2 100644 --- a/cmd/mesh-host/whatruns_test.go +++ b/cmd/mesh-host/whatruns_test.go @@ -8,6 +8,7 @@ import ( "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" "git.novox.be/novox/mesh-contracts/placement/go/seats/meshmoduleassigner" + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" engcomp "github.com/novox/mesh-host/internal/compositions" "path/filepath" "sort" @@ -16,7 +17,6 @@ import ( "git.novox.be/novox/mesh-networking/modules/mesh-wireguard/go/overlay" "git.novox.be/novox/mesh-sdk/go/declaration" - "git.novox.be/novox/mesh-sdk/go/delivery" "github.com/novox/mesh-host/internal/agent" "github.com/novox/mesh-host/internal/liveness" @@ -123,7 +123,7 @@ func TestDeclaredNowReadsWhatRunsWhenTheAgentIsRunning(t *testing.T) { placed := meshmoduleassigner.Placement{Node: "n1", Modules: []meshmoduleassigner.PlacedModule{{Module: "zsh"}, {Module: "lemurs"}}} // Issue 774, 2026-10-30 19:43: a have naming only a build of the old form (systemd-resolved on shanks, folded in the // node's own) keeps the kept declaration read, since it is the one record of what runs of it. - newForm := nodeengine.HaveByNodeModuleValue{Fp: delivery.SourcePrefix2 + strings.Repeat("a", 64), Phase: nodeengineclient.PhaseRunning} + newForm := nodeengine.HaveByNodeModuleValue{Fp: meshcatalogclient.SourcePrefix2 + strings.Repeat("a", 64), Phase: nodeengineclient.PhaseRunning} oldForm := nodeengine.HaveByNodeModuleValue{Build: "build-1791404191702719760", Phase: nodeengineclient.PhaseRunning} for _, row := range []struct { name string diff --git a/go.mod b/go.mod index cbda85e..46b4065 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,9 @@ module github.com/novox/mesh-host go 1.26.0 require ( + git.novox.be/novox/mesh-delivery/sdk v0.0.0-20261105000000-000000000000 + git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient v0.0.0-20261105000000-000000000000 + git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery v0.0.0-20261105000000-000000000000 git.novox.be/novox/mesh-contracts/placement/go/clients/meshmoduleassignerclient v0.0.0-20261105000000-000000000000 git.novox.be/novox/mesh-contracts/placement/go/seats/meshmoduleassigner v0.0.0-20261105000000-000000000000 git.novox.be/novox/mesh-contracts/placement/go/seats/meshwants v0.0.0-20261105000000-000000000000 diff --git a/internal/agent/agent_test.go b/internal/agent/agent_test.go index 8627176..46155a0 100644 --- a/internal/agent/agent_test.go +++ b/internal/agent/agent_test.go @@ -7,6 +7,7 @@ import ( "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" "git.novox.be/novox/mesh-contracts/placement/go/seats/meshwants" + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" "github.com/novox/mesh-host/internal/buckets/bucketstest" "github.com/novox/mesh-host/internal/compositions" "maps" @@ -24,7 +25,7 @@ import ( const node = "n1" -func fpOf(c string) string { return delivery.SourcePrefix2 + strings.Repeat(c, 64) } +func fpOf(c string) string { return meshcatalogclient.SourcePrefix2 + strings.Repeat(c, 64) } // A watcher the test writes to; it says Caught once the keys written before the watch began are out. type feed struct { diff --git a/internal/buckets/buckets.go b/internal/buckets/buckets.go index 00214c2..53b0479 100644 --- a/internal/buckets/buckets.go +++ b/internal/buckets/buckets.go @@ -4,9 +4,13 @@ package buckets import ( + "errors" + assigner "git.novox.be/novox/mesh-contracts/placement/go/clients/meshmoduleassignerclient" seat "git.novox.be/novox/mesh-contracts/placement/go/seats/meshmoduleassigner" wants "git.novox.be/novox/mesh-contracts/placement/go/seats/meshwants" + catalog "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-sdk/go/keys" ) // The node-engine's own states and the have state, written by every node's engine for its own keys. @@ -32,3 +36,24 @@ const ( // Wants holds what each node should run, `want..` and `want..node`. Wants = assigner.Module + "_" + wants.StateWants ) + +// The states of the catalogue seat's holder that the engine reads, named by the holder's module (ADR 0634 §1, finding F4). +const ( + // Modules holds a module's source, ``; Manifests a manifest by its digest; Verdicts the judge's verdict of a + // manifest by its digest; Candidates a group's candidate, by the group's key part (CandidateKey). + Modules = catalogHolder + "_modules" + Manifests = catalogHolder + "_manifests" + Verdicts = catalogHolder + "_verdicts" + Candidates = catalogHolder + "_candidates" + + catalogHolder = "mesh-delivery-indexer" +) + +// CandidateKey is the key of a group's candidate in Candidates: the group as a key part, a branch such as `fix/1.2` +// escaped so it keys. +func CandidateKey(group string) (string, error) { + if group == "" { + return "", errors.New("buckets: the group name is empty") + } + return catalog.CandidatesByGroupKey(keys.EscapeName(group)), nil +} diff --git a/internal/buckets/buckets_test.go b/internal/buckets/buckets_test.go index db981d6..d24750b 100644 --- a/internal/buckets/buckets_test.go +++ b/internal/buckets/buckets_test.go @@ -1,7 +1,10 @@ package buckets import ( + "strings" + "encoding/json" + "git.novox.be/novox/mesh-sdk/go/keys" "os" "testing" ) @@ -32,3 +35,22 @@ func TestTheOwnStatesAreTheManifestsStates(t *testing.T) { } } } + +// A group's candidate key is its name as a key part: a branch with a slash keeps it, a dot is escaped, and an empty group does not. +func TestACandidateKeyEscapesTheGroup(t *testing.T) { + for _, group := range []string{"mesh-sdk#7", "fix/940-no-retire", "fix/1.2"} { + key, err := CandidateKey(group) + if err != nil { + t.Fatalf("%s: %v", group, err) + } + if strings.Contains(key, ".") { + t.Errorf("the key %q of %q holds a dot", key, group) + } + if back, err := keys.UnescapeName(key); err != nil || back != group { + t.Errorf("%q came back as %q (%v)", group, back, err) + } + } + if _, err := CandidateKey(""); err == nil { + t.Error("a key was made for no group") + } +} diff --git a/internal/buckets/bucketstest/bucketstest.go b/internal/buckets/bucketstest/bucketstest.go index 9848e23..03912cc 100644 --- a/internal/buckets/bucketstest/bucketstest.go +++ b/internal/buckets/bucketstest/bucketstest.go @@ -4,6 +4,8 @@ package bucketstest import ( "encoding/json" + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-contracts/placement/go/seats/meshmoduleassigner" "git.novox.be/novox/mesh-contracts/placement/go/seats/meshwants" @@ -34,3 +36,27 @@ func Candidate(c meshmoduleassigner.CandidatesByProposalNodeValue) ([]byte, erro } return json.Marshal(c) } + +// GroupCandidate is a group's candidate as the catalogue seat's holder writes it. +func GroupCandidate(c meshcatalog.CandidatesByGroupValue) ([]byte, error) { + c.V = Version + return json.Marshal(c) +} + +// Pull is one pull request of a candidate, from its head. +func Pull(head meshdeliveryclient.Head, modules []string, fps map[string]meshcatalog.CandidatesByGroupValuePullsItemFpsValue) meshcatalog.CandidatesByGroupValuePullsItem { + return meshcatalog.CandidatesByGroupValuePullsItem{Repository: head.Repository, Number: int64(head.Number), Head: head.Head, Modules: modules, Fps: fps} +} + +// Raw is each value as JSON, as a composer case carries verdicts and placements. +func Raw[V any](in map[string]V) map[string]json.RawMessage { + out := map[string]json.RawMessage{} + for k, v := range in { + raw, err := json.Marshal(v) + if err != nil { + panic(err) + } + out[k] = raw + } + return out +} diff --git a/internal/compose/composed.go b/internal/compose/composed.go index c30b539..a045795 100644 --- a/internal/compose/composed.go +++ b/internal/compose/composed.go @@ -4,6 +4,7 @@ import ( "encoding/json" "errors" "fmt" + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" engcomp "github.com/novox/mesh-host/internal/compositions" "slices" "sort" @@ -12,7 +13,6 @@ import ( "git.novox.be/novox/mesh-networking/modules/mesh-wireguard/go/overlay" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/declaration" - "git.novox.be/novox/mesh-sdk/go/delivery" "github.com/novox/mesh-host/internal/compose/resolution" ) @@ -70,7 +70,7 @@ func CompositionsIn(in Input, node string, fps map[string]string, mode Mode) (ou // not folded (issue 1585: nats on novox, folded by the reader and composed at src2: here, refused for its server // artifact no build had published). func Folds(inOwn bool, fp string) bool { - return inOwn && !strings.HasPrefix(fp, delivery.SourcePrefix2) + return inOwn && !strings.HasPrefix(fp, meshcatalogclient.SourcePrefix2) } func compositions(in Input, node string, fps map[string]string, mode Mode, applied bool) (out map[string]engcomp.Composed, skipped, stood []string, err error) { diff --git a/internal/compose/composed_test.go b/internal/compose/composed_test.go index ccbb3d4..c33c749 100644 --- a/internal/compose/composed_test.go +++ b/internal/compose/composed_test.go @@ -2,6 +2,7 @@ package compose import ( "encoding/json" + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" engcomp "github.com/novox/mesh-host/internal/compositions" "sort" "strings" @@ -9,11 +10,10 @@ import ( "git.novox.be/novox/mesh-networking/modules/mesh-wireguard/go/overlay" "git.novox.be/novox/mesh-sdk/go/catalogue" - "git.novox.be/novox/mesh-sdk/go/delivery" "github.com/novox/mesh-host/internal/compose/resolution" ) -func fpFor(c byte) string { return delivery.SourcePrefix2 + strings.Repeat(string(c), 64) } +func fpFor(c byte) string { return meshcatalogclient.SourcePrefix2 + strings.Repeat(string(c), 64) } // 1943/AC5: a node's compositions are its declaration as Compose gives it, split by the module each resource belongs // to: every resource once, each in its module's composition at the fingerprint given, the module's ownership said on @@ -186,18 +186,3 @@ func TestCompositionsAreTheDeclarationSplitByModule(t *testing.T) { } } } - -// 1943/AC1: the upgrade policies and sources a composition's value may name are the catalogue's own words. -func TestTheComposedPoliciesAreTheCatalogues(t *testing.T) { - for _, p := range []string{catalogue.PolicyRoll, catalogue.PolicyTogether, catalogue.PolicyRecord} { - for _, from := range []string{catalogue.FromPerson, catalogue.FromModule, "the bus", catalogue.FromData, catalogue.FromDefault} { // "the bus": an older reader's word, still read - v := delivery.ComposedDigest{Digest: "sha256:" + strings.Repeat("a", 64), Judgement: delivery.JudgedNeither, Policy: p, PolicyFrom: from} - if _, err := delivery.EncodeComposedDigest(v); err != nil { - t.Errorf("%s from %s: %v", p, from, err) - } - } - } - if delivery.PolicyRecord != catalogue.PolicyRecord { - t.Errorf("delivery.PolicyRecord = %q, the catalogue's %q", delivery.PolicyRecord, catalogue.PolicyRecord) - } -} diff --git a/internal/compose/frombus/answer_rows_test.go b/internal/compose/frombus/answer_rows_test.go index e1e2747..ebaaf31 100644 --- a/internal/compose/frombus/answer_rows_test.go +++ b/internal/compose/frombus/answer_rows_test.go @@ -7,6 +7,8 @@ import ( "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" nodeengine "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" "git.novox.be/novox/mesh-contracts/placement/go/seats/meshmoduleassigner" + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" engcomp "github.com/novox/mesh-host/internal/compositions" "os" "reflect" @@ -45,24 +47,24 @@ func answerNamesEveryRefusalAndLeavesAnOldBuildToTheNode(t *testing.T) { } b := &bus{placements: map[string]meshmoduleassigner.Placement{"n1": p}, records: map[string]nodeengine.NodeRecord{"n1": {Node: "n1", TunnelKey: publicKey(t), SealingKey: "sk", Capabilities: []string{"systemd", "container-runtime"}}}, - modules: map[string]delivery.Module{"lemurs": {Manifest: delivery.ManifestDigest(lemurs)}}, - manifests: map[string][]byte{delivery.ManifestDigest(lemurs): lemurs}, + modules: map[string]meshcatalog.ModulesByModuleValue{"lemurs": {Manifest: meshcatalogclient.ManifestDigest(lemurs)}}, + manifests: map[string][]byte{meshcatalogclient.ManifestDigest(lemurs): lemurs}, builds: map[string]*delivery.BuilderBuild{}} for _, m := range []string{"web", "api", "spiegel"} { b.own = append(b.own, compose.OwnSecretRow{Node: "n1", Module: m, Name: "token", Origin: "made", SealedTo: "sk", Sealed: `"box"`}) } for _, m := range []string{"web", "api", "spiegel"} { raw := manifest(m) - b.manifests[delivery.ManifestDigest(raw)] = raw - b.modules[m] = delivery.Module{Manifest: delivery.ManifestDigest(raw)} + b.manifests[meshcatalogclient.ManifestDigest(raw)] = raw + b.modules[m] = meshcatalog.ModulesByModuleValue{Manifest: meshcatalogclient.ManifestDigest(raw)} } // web and api are built, their keys done and waiting for the reference of their image; spiegel has no build of // the new form at all. for i, m := range []string{"web", "api"} { mod := b.modules[m] - mod.FP2 = fp(string(rune('1' + i))) + mod.Fp2 = fp(string(rune('1' + i))) b.modules[m] = mod - b.builds[m+" "+mod.FP2] = &delivery.BuilderBuild{State: delivery.BuildDone, Manifest: mod.Manifest, + b.builds[m+" "+mod.Fp2] = &delivery.BuilderBuild{State: delivery.BuildDone, Manifest: mod.Manifest, Artifacts: []delivery.BuiltArtifact{{Name: "site", Kind: "image", Digest: "sha256:" + strings.Repeat("b", 64)}}} } return b @@ -204,16 +206,16 @@ func answerNamesEveryRefusalAndLeavesAnOldBuildToTheNode(t *testing.T) { bp.Decided.Endpoint = "n1.example:51820" bb.placements["n1"] = bp sp := bb.modules["spiegel"] - sp.FP2 = fp("3") + sp.Fp2 = fp("3") bb.modules["spiegel"] = sp image := "registry.example:5100/spiegel/site@sha256:" + strings.Repeat("7", 64) - bb.builds["spiegel "+sp.FP2] = &delivery.BuilderBuild{State: delivery.BuildDone, Manifest: sp.Manifest, + bb.builds["spiegel "+sp.Fp2] = &delivery.BuilderBuild{State: delivery.BuildDone, Manifest: sp.Manifest, Artifacts: []delivery.BuiltArtifact{{Name: "site", Kind: "upstream", Reference: image, Digest: "sha256:" + strings.Repeat("7", 64)}}} read, err := ComposingInput(ctx, bb, Mesh{}, Scope{Node: "n1", InOwn: []string{"spiegel"}}, - map[string]string{"lemurs": fp("e"), "spiegel": sp.FP2}) + map[string]string{"lemurs": fp("e"), "spiegel": sp.Fp2}) raw, _ := json.Marshal(read.Catalogue["spiegel"]) if err != nil || read.ComposedHow.Held["spiegel"] != "" || !strings.Contains(string(raw), strings.Repeat("7", 64)) { - t.Errorf("spiegel in InOwn wanted at %s: %v, held %v, read as %s; want its build's image", sp.FP2, err, + t.Errorf("spiegel in InOwn wanted at %s: %v, held %v, read as %s; want its build's image", sp.Fp2, err, read.ComposedHow, raw) } } @@ -232,8 +234,8 @@ func answerNamesEveryRefusalAndLeavesAnOldBuildToTheNode(t *testing.T) { `"resources":[{"id":"service","type":"service","unit":"vault.service","state":"running","boot":"enabled"}]}`) b = mk("lemurs", overlay.Name, "spiegel", "nft", "vault") for m, raw := range map[string][]byte{"spiegel": listening, "nft": nft, "vault": vault} { - b.manifests[delivery.ManifestDigest(raw)] = raw - b.modules[m] = delivery.Module{Manifest: delivery.ManifestDigest(raw)} + b.manifests[meshcatalogclient.ManifestDigest(raw)] = raw + b.modules[m] = meshcatalog.ModulesByModuleValue{Manifest: meshcatalogclient.ManifestDigest(raw)} } p = b.placements["n1"] p.Decided.Endpoint = "n1.example:51820" @@ -370,8 +372,8 @@ func answerNamesEveryRefusalAndLeavesAnOldBuildToTheNode(t *testing.T) { `"resources":[{"id":"service","type":"service","unit":"mailu.service","state":"running","boot":"enabled"}]}`) b = mk("lemurs", overlay.Name, "mailu", "proxy", "de-spiegel") for m, raw := range map[string][]byte{"de-spiegel": spiegel, "proxy": proxy, "mailu": mailu} { - b.manifests[delivery.ManifestDigest(raw)] = raw - b.modules[m] = delivery.Module{Manifest: delivery.ManifestDigest(raw)} + b.manifests[meshcatalogclient.ManifestDigest(raw)] = raw + b.modules[m] = meshcatalog.ModulesByModuleValue{Manifest: meshcatalogclient.ManifestDigest(raw)} } p = b.placements["n1"] p.Decided.Endpoint = "n1.example:51820" @@ -470,8 +472,8 @@ func answerNamesEveryRefusalAndLeavesAnOldBuildToTheNode(t *testing.T) { `{"id":"server","type":"container","name":"site","image":"registry.example/site@sha256:` + strings.Repeat("c", 64) + `","ports":["8080"]}]}`) b = mk("lemurs", overlay.Name, "site", "nft") for m, raw := range map[string][]byte{"site": site, "nft": nft} { - b.manifests[delivery.ManifestDigest(raw)] = raw - b.modules[m] = delivery.Module{Manifest: delivery.ManifestDigest(raw)} + b.manifests[meshcatalogclient.ManifestDigest(raw)] = raw + b.modules[m] = meshcatalog.ModulesByModuleValue{Manifest: meshcatalogclient.ManifestDigest(raw)} } p = b.placements["n1"] p.Decided.Endpoint = "n1.example:51820" @@ -511,8 +513,8 @@ func answerNamesEveryRefusalAndLeavesAnOldBuildToTheNode(t *testing.T) { `"resources":[{"id":"service","type":"service","unit":"app.service","state":"running","boot":"enabled"}]}`) b2 := mk("lemurs", overlay.Name, "site", "nft", "app") for m, raw := range map[string][]byte{"site": provider, "nft": nft, "app": app} { - b2.manifests[delivery.ManifestDigest(raw)] = raw - b2.modules[m] = delivery.Module{Manifest: delivery.ManifestDigest(raw)} + b2.manifests[meshcatalogclient.ManifestDigest(raw)] = raw + b2.modules[m] = meshcatalog.ModulesByModuleValue{Manifest: meshcatalogclient.ManifestDigest(raw)} } p2 := b2.placements["n1"] p2.Decided.Endpoint = "n1.example:51820" @@ -554,16 +556,16 @@ func answerNamesEveryRefusalAndLeavesAnOldBuildToTheNode(t *testing.T) { // build that runs (7) listens on 8080. The filter composes with what site last offered, its manifest at the build // that runs (issue 1960: a hold never spreads), so it opens 8080 and nothing applied opens 9090. moved := []byte(strings.Replace(string(site), `"port":8080`, `"port":9090`, 1)) - b.manifests[delivery.ManifestDigest(moved)] = moved - b.modules["site"] = delivery.Module{Manifest: delivery.ManifestDigest(moved)} + b.manifests[meshcatalogclient.ManifestDigest(moved)] = moved + b.modules["site"] = meshcatalog.ModulesByModuleValue{Manifest: meshcatalogclient.ManifestDigest(moved)} for i := range p.Modules { p.Modules[i].Ports = nil } b.placements["n1"] = p siteRuns.Resources[1] = json.RawMessage(`{"id":"site.server","module":"site","type":"container","name":"site","image":"registry.example/site@sha256:` + strings.Repeat("c", 64) + `","ports":["8080"]}`) siteRuns.FP = fp("7") - b.manifests[delivery.ManifestDigest(site)] = site - b.builds["site "+fp("7")] = &delivery.BuilderBuild{State: delivery.BuildDone, Manifest: delivery.ManifestDigest(site)} + b.manifests[meshcatalogclient.ManifestDigest(site)] = site + b.builds["site "+fp("7")] = &delivery.BuilderBuild{State: delivery.BuildDone, Manifest: meshcatalogclient.ManifestDigest(site)} got, _, held, err = Compose(ctx, b, Mesh{}, Scope{Node: "n1", Running: []engcomp.Composed{siteRuns}}, fps) if err != nil || held["site"] == "" || held["nft"] != "" { t.Errorf("site moved to 9090 in its new manifest: %v, held %v; want site held alone", err, held) @@ -593,8 +595,8 @@ func otherNodeNotResolvedIsNamed(t *testing.T) { "n1": {Node: "n1", Decided: meshmoduleassigner.NodeDecided{Hub: true, Address: "10.77.0.1", Endpoint: "n1.example:51820"}, Modules: []meshmoduleassigner.PlacedModule{{Module: "cons"}, {Module: overlay.Name}}}, "n2": {Node: "n2", Decided: meshmoduleassigner.NodeDecided{Address: "10.77.0.2"}, Modules: []meshmoduleassigner.PlacedModule{{Module: "prov"}, {Module: overlay.Name}}}}, records: map[string]nodeengine.NodeRecord{"n1": {Node: "n1", TunnelKey: publicKey(t)}, "n2": {Node: "n2", TunnelKey: publicKey(t)}}, - modules: map[string]delivery.Module{"prov": {Manifest: delivery.ManifestDigest(prov)}, "cons": {Manifest: delivery.ManifestDigest(cons)}}, - manifests: map[string][]byte{delivery.ManifestDigest(prov): prov, delivery.ManifestDigest(cons): cons}} + modules: map[string]meshcatalog.ModulesByModuleValue{"prov": {Manifest: meshcatalogclient.ManifestDigest(prov)}, "cons": {Manifest: meshcatalogclient.ManifestDigest(cons)}}, + manifests: map[string][]byte{meshcatalogclient.ManifestDigest(prov): prov, meshcatalogclient.ManifestDigest(cons): cons}} given := map[string]*meshmoduleassigner.Placement{} for n, p := range b.placements { p := p diff --git a/internal/compose/frombus/frombus.go b/internal/compose/frombus/frombus.go index 27030e9..899e057 100644 --- a/internal/compose/frombus/frombus.go +++ b/internal/compose/frombus/frombus.go @@ -38,6 +38,8 @@ import ( "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" "git.novox.be/novox/mesh-contracts/placement/go/clients/meshmoduleassignerclient" "git.novox.be/novox/mesh-contracts/placement/go/seats/meshmoduleassigner" + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" engcomp "github.com/novox/mesh-host/internal/compositions" "maps" "net/netip" @@ -62,9 +64,9 @@ type Reader interface { // Records is every node's record, by node: HaveNode.Record of each node's own have key, `.node` of // buckets.Have (ADR 0555 §3). Records(ctx context.Context) (map[string]nodeengine.NodeRecord, error) - // Modules is every module the indexer holds, by module (delivery.ModulesBucket). - Modules(ctx context.Context) (map[string]delivery.Module, error) - // Manifest is the raw manifest of a digest (delivery.ManifestsBucket). + // Modules is every module the indexer holds, by module (buckets.Modules). + Modules(ctx context.Context) (map[string]meshcatalog.ModulesByModuleValue, error) + // Manifest is the raw manifest of a digest (buckets.Manifests). Manifest(ctx context.Context, digest string) ([]byte, error) // Values is one module's config values on one node as mesh-config resolved them (mesh-config's state `resolved`, // keys `..`), decoded; nil when it has none. @@ -198,7 +200,7 @@ func InOwn(running []engcomp.Composed, p meshmoduleassigner.Placement) []string // Input is the composition input for s, read through r. // // **A refused manifest never ends the read** (novox/hq ADR 0578 §3): each manifest is decoded leniently -// (catalogue.Decode) and the indexer's verdict on it taken from its modules value (delivery.TakeManifestVerdict, a missing +// (catalogue.Decode) and the indexer's verdict on it taken from its modules value (meshcatalogclient.TakeVerdict, a missing // one accepted); a module whose manifest the indexer's judge refused, or whose bytes do not decode, stands in the // catalogue as its stub (unusable), so what it provides and requires still shapes the rest, and is named with why in // the input's Refused; a field set aside is named in its SetAside. Every other module of the read goes on. @@ -209,10 +211,10 @@ func Input(ctx context.Context, r Reader, mesh Mesh, s Scope) (compose.Input, er // taken is a module's manifest as a reader takes it (novox/hq ADR 0578 §3): decoded leniently, with the indexer's verdict // from its modules value. A refused one, or bytes that do not decode, stands as its stub (unusable) and why says so; a // manifest not read past a field it set aside holds its name alone and is held the same. -func taken(name string, raw []byte, m delivery.Module) (man catalogue.Manifest, why string) { - if v := m.VerdictOf(name); v != nil { - if refused, _ := delivery.TakeManifestVerdict(v); refused { - return unusable(name, raw), v.Why() +func taken(name string, raw []byte, m meshcatalog.ModulesByModuleValue) (man catalogue.Manifest, why string) { + if v := meshcatalogclient.ModuleVerdict(name, m); v != nil { + if refused, _ := meshcatalogclient.TakeVerdict(v); refused { + return unusable(name, raw), meshcatalogclient.VerdictWhy(*v) } } man, err := catalogue.Decode(raw) @@ -256,7 +258,7 @@ type base struct { in compose.Input placements map[string]meshmoduleassigner.Placement records map[string]nodeengine.NodeRecord - modules map[string]delivery.Module + modules map[string]meshcatalog.ModulesByModuleValue // ownRefused is each module placed on the composing node whose manifest the indexer refused, with why, in the order // read: every scope holds it. ownRefused [][2]string @@ -383,7 +385,7 @@ func readBase(ctx context.Context, r Reader, mesh Mesh, s Scope) (base, error) { // The module the mesh provides itself (the private network) has no repository the indexer reads: its manifest comes // with mesh-sdk and always wins, as the controller's providedModules and the grants part have it. - for _, name := range delivery.ProvidedModules() { + for _, name := range overlay.ProvidedModules() { raw, err := json.Marshal(overlay.Manifest()) if err != nil { return b, err @@ -794,7 +796,7 @@ func ranOf(ctx context.Context, r Reader, mesh Mesh, running []engcomp.Composed, // by the one take-in rule (takein.Manifest), a templates archive read in the artifact store as the node reaches it // (issue 1126). func fromBuilds(ctx context.Context, r Reader, build func(ctx context.Context, module, fp string) (*delivery.BuilderBuild, error), - in *compose.Input, s Scope, p meshmoduleassigner.Placement, modules map[string]delivery.Module, + in *compose.Input, s Scope, p meshmoduleassigner.Placement, modules map[string]meshcatalog.ModulesByModuleValue, read func(module, digest string) (catalogue.Manifest, error), parse func([]byte) (catalogue.Manifest, error), refuse func(module string, err error), wanted map[string]string) error { // stub keeps a refused module in the composition as a stub for this node alone (compose.Stub): what it provides @@ -881,7 +883,7 @@ func fromBuilds(ctx context.Context, r Reader, build func(ctx context.Context, m // the scope keeps at another build is. keptAt := maps.Clone(s.KeptAt) for _, pm := range p.Modules { - if m, ok := modules[pm.Module]; ok && m.FP2 != "" && wanted[pm.Module] != "" && wanted[pm.Module] != m.FP2 && + if m, ok := modules[pm.Module]; ok && m.Fp2 != "" && wanted[pm.Module] != "" && wanted[pm.Module] != m.Fp2 && keptAt[pm.Module] == "" && !compose.Folds(slices.Contains(s.InOwn, pm.Module), wanted[pm.Module]) { if keptAt == nil { keptAt = map[string]string{} @@ -897,7 +899,7 @@ func fromBuilds(ctx context.Context, r Reader, build func(ctx context.Context, m } for _, pm := range p.Modules { m, ok := modules[pm.Module] - if !ok || m.FP2 == "" || keptAt[pm.Module] != "" || compose.Folds(slices.Contains(s.InOwn, pm.Module), wanted[pm.Module]) { + if !ok || m.Fp2 == "" || keptAt[pm.Module] != "" || compose.Folds(slices.Contains(s.InOwn, pm.Module), wanted[pm.Module]) { // A module it runs inside its own composition is composed there as before while it is folded: no build of the // new form is what it runs (Scope.InOwn). Wanted at a build of the new form it is no longer folded (compose.Folds, // the composition's own rule), so its build is taken in (issue 1585). @@ -909,9 +911,9 @@ func fromBuilds(ctx context.Context, r Reader, build func(ctx context.Context, m if man, ok := in.Catalogue[pm.Module]; !ok || (man.Build == nil && !man.Templated()) { continue // nothing is built of it: its manifest is what it composes from } - b, err := build(ctx, pm.Module, m.FP2) + b, err := build(ctx, pm.Module, m.Fp2) if err != nil { - err = fmt.Errorf("frombus: the build of %s at %s: %w", pm.Module, m.FP2, err) + err = fmt.Errorf("frombus: the build of %s at %s: %w", pm.Module, m.Fp2, err) if refuse == nil { return err } @@ -924,11 +926,11 @@ func fromBuilds(ctx context.Context, r Reader, build func(ctx context.Context, m // runtime's bundle): it is held, keeping what runs, until its build lands (issue 774, 2026-10-30 19:30: // node-tools on ace, g14 and shanks, "its build produced 0 bundle(s)", and every module refused). stub(pm.Module, fmt.Errorf("frombus: %s is wanted at %s, of which the builder holds no done build yet; it "+ - "keeps running what it runs", pm.Module, m.FP2)) + "keeps running what it runs", pm.Module, m.Fp2)) } continue // nothing of it runs: composed from the indexer's manifest, as before } - resolved, err := built(pm.Module, m.FP2, b) + resolved, err := built(pm.Module, m.Fp2, b) if err != nil { if refuse == nil { return err diff --git a/internal/compose/frombus/frombus_test.go b/internal/compose/frombus/frombus_test.go index 8b112ba..33a75fe 100644 --- a/internal/compose/frombus/frombus_test.go +++ b/internal/compose/frombus/frombus_test.go @@ -15,6 +15,8 @@ import ( nodeengine "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" "git.novox.be/novox/mesh-contracts/placement/go/clients/meshmoduleassignerclient" "git.novox.be/novox/mesh-contracts/placement/go/seats/meshmoduleassigner" + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" "github.com/novox/mesh-host/internal/compositions" "os" "reflect" @@ -35,7 +37,7 @@ type bus struct { handed map[string]bool placements map[string]meshmoduleassigner.Placement records map[string]nodeengine.NodeRecord - modules map[string]delivery.Module + modules map[string]meshcatalog.ModulesByModuleValue manifests map[string][]byte values map[string]map[string]any // by node.module valuesRead []string @@ -89,7 +91,9 @@ func (b *bus) Handed(context.Context, string) map[string]bool { return b.handed func (b *bus) Records(context.Context) (map[string]nodeengine.NodeRecord, error) { return b.records, nil } -func (b *bus) Modules(context.Context) (map[string]delivery.Module, error) { return b.modules, nil } +func (b *bus) Modules(context.Context) (map[string]meshcatalog.ModulesByModuleValue, error) { + return b.modules, nil +} func (b *bus) Manifest(_ context.Context, digest string) ([]byte, error) { raw, ok := b.manifests[digest] if !ok { @@ -161,13 +165,13 @@ func TestInput(t *testing.T) { if err != nil { t.Fatal(err) } - digest := delivery.ManifestDigest(xterm) + digest := meshcatalogclient.ManifestDigest(xterm) // A module in the settings form (plex and the media modules on one node, 2026-10-26): its config values are in its // layers, not mesh-config, so the placement carries them (MeshValues, Values) and mesh-config's are not read for it. media := []byte(`{"module":"media","version":"1","settings":{"lines":{"kind":"preference","default":0,"why":"kept"},` + `"name":{"kind":"preference","default":"x","why":"shown"}},"own-secrets":{"token":"/srv/media/token"},"resources":[{"id":"conf","type":"file",` + `"path":"/srv/media/conf","mode":"0644","content":"uid=${setting:lines} name=${setting:name}\n"}]}`) - mediaDigest := delivery.ManifestDigest(media) + mediaDigest := meshcatalogclient.ManifestDigest(media) keys := map[string]string{"anchor": publicKey(t), "laptop": publicKey(t)} caps := []string{"container-runtime", "systemd", "wireguard"} tunnel := &meshmoduleassigner.FoundTunnel{Interface: "wg0", PublicKey: keys["anchor"], Address: "10.77.0.1/24", @@ -189,7 +193,7 @@ func TestInput(t *testing.T) { "anchor": {Node: "anchor", TunnelKey: keys["anchor"], Capabilities: caps, Contract: 2}, "laptop": {Node: "laptop", TunnelKey: keys["laptop"], Capabilities: caps, Authority: "the mesh's certificate"}, }, - modules: map[string]delivery.Module{"xterm": {Manifest: digest}, "media": {Manifest: mediaDigest}, "unread": {}}, + modules: map[string]meshcatalog.ModulesByModuleValue{"xterm": {Manifest: digest}, "media": {Manifest: mediaDigest}, "unread": {}}, manifests: map[string][]byte{digest: xterm, mediaDigest: media}, values: map[string]map[string]any{"laptop.xterm": {"font": "mono"}, "anchor.xterm": {"font": "serif"}, "laptop.media": {"lines": "not mesh-config's"}}, @@ -337,10 +341,10 @@ func TestInput(t *testing.T) { } return &bus{placements: map[string]meshmoduleassigner.Placement{"n1": p}, records: map[string]nodeengine.NodeRecord{"n1": {Node: "n1", TunnelKey: keys["anchor"], SealingKey: "sk", Capabilities: []string{"systemd"}}}, - modules: map[string]delivery.Module{"keeper": {Manifest: delivery.ManifestDigest(keeper)}, "lemurs": {Manifest: delivery.ManifestDigest(lemurs)}, - "objects": {Manifest: delivery.ManifestDigest(objects)}, "files": {Manifest: delivery.ManifestDigest(files)}}, - manifests: map[string][]byte{delivery.ManifestDigest(keeper): keeper, delivery.ManifestDigest(lemurs): lemurs, - delivery.ManifestDigest(objects): objects, delivery.ManifestDigest(files): files}, + modules: map[string]meshcatalog.ModulesByModuleValue{"keeper": {Manifest: meshcatalogclient.ManifestDigest(keeper)}, "lemurs": {Manifest: meshcatalogclient.ManifestDigest(lemurs)}, + "objects": {Manifest: meshcatalogclient.ManifestDigest(objects)}, "files": {Manifest: meshcatalogclient.ManifestDigest(files)}}, + manifests: map[string][]byte{meshcatalogclient.ManifestDigest(keeper): keeper, meshcatalogclient.ManifestDigest(lemurs): lemurs, + meshcatalogclient.ManifestDigest(objects): objects, meshcatalogclient.ManifestDigest(files): files}, own: []compose.OwnSecretRow{{Node: "n1", Module: "keeper", Name: "password", Origin: "made", SealedTo: "sk", Sealed: `"box"`}}} } oneFps := map[string]string{"keeper": "src2:" + strings.Repeat("d", 64), "lemurs": "src2:" + strings.Repeat("e", 64), @@ -609,7 +613,7 @@ func TestInput(t *testing.T) { if err := zw.Close(); err != nil { t.Fatal(err) } - archiveDigest := delivery.ManifestDigest(packed.Bytes()) + archiveDigest := meshcatalogclient.ManifestDigest(packed.Bytes()) digestOf := func(c string) string { return "sha256:" + strings.Repeat(c, 64) } movedFps := map[string]string{"restic": "src2:" + strings.Repeat("3", 64), "mesh-provisioner-client": "src2:" + strings.Repeat("4", 64), "store": "src2:" + strings.Repeat("5", 64)} @@ -625,19 +629,19 @@ func TestInput(t *testing.T) { rec := b.records["n1"] rec.Contract, rec.Capabilities = compose.HandedContract, []string{"systemd", "package-manager"} b.records["n1"] = rec - b.modules = map[string]delivery.Module{} + b.modules = map[string]meshcatalog.ModulesByModuleValue{} b.manifests = map[string][]byte{} for module, raw := range map[string][]byte{"restic": resticRaw, "mesh-provisioner-client": clientRaw, "store": store} { - b.modules[module] = delivery.Module{FP2: movedFps[module], Manifest: delivery.ManifestDigest(raw)} - b.manifests[delivery.ManifestDigest(raw)] = raw + b.modules[module] = meshcatalog.ModulesByModuleValue{Fp2: movedFps[module], Manifest: meshcatalogclient.ManifestDigest(raw)} + b.manifests[meshcatalogclient.ManifestDigest(raw)] = raw } // The vault's own answer: the client's own secret; restic's is the client's to ask, so own leaves it out. b.own = []compose.OwnSecretRow{{Node: "n1", Module: "mesh-provisioner-client", Name: "provisioner-broker", Origin: "made", SealedTo: "sk", Sealed: `"box"`}} if built { b.builds = map[string]*delivery.BuilderBuild{ - "restic " + movedFps["restic"]: {State: delivery.BuildDone, Manifest: delivery.ManifestDigest(resticRaw), Artifacts: resticArtifacts}, - "mesh-provisioner-client " + movedFps["mesh-provisioner-client"]: {State: delivery.BuildDone, Manifest: delivery.ManifestDigest(clientRaw), Artifacts: clientArtifacts}, - "store " + movedFps["store"]: {State: delivery.BuildDone, Manifest: delivery.ManifestDigest(store)}, + "restic " + movedFps["restic"]: {State: delivery.BuildDone, Manifest: meshcatalogclient.ManifestDigest(resticRaw), Artifacts: resticArtifacts}, + "mesh-provisioner-client " + movedFps["mesh-provisioner-client"]: {State: delivery.BuildDone, Manifest: meshcatalogclient.ManifestDigest(clientRaw), Artifacts: clientArtifacts}, + "store " + movedFps["store"]: {State: delivery.BuildDone, Manifest: meshcatalogclient.ManifestDigest(store)}, } b.archives = map[string][]byte{archiveDigest: packed.Bytes()} } @@ -770,8 +774,8 @@ func TestInput(t *testing.T) { p := b.placements["anchor"] p.Modules = append(p.Modules, meshmoduleassigner.PlacedModule{Module: "mailer"}) b.placements["anchor"] = p - b.modules["mailer"] = delivery.Module{Manifest: delivery.ManifestDigest(mailer)} - b.manifests[delivery.ManifestDigest(mailer)] = mailer + b.modules["mailer"] = meshcatalog.ModulesByModuleValue{Manifest: meshcatalogclient.ManifestDigest(mailer)} + b.manifests[meshcatalogclient.ManifestDigest(mailer)] = mailer if set { b.served = map[string]map[string]any{"anchor.mailer": {"domain": "example.org"}} } @@ -852,12 +856,12 @@ func statesOf(t *testing.T, ctx context.Context, b *bus) mapStates { } modules, _ := b.Modules(ctx) for name, m := range modules { - raw, err := delivery.EncodeModule(m) + raw, err := meshcatalogclient.EncodeModule(m) put(ModulesRead, name, raw, err) var mf struct{ Module string } _ = json.Unmarshal(b.manifests[m.Manifest], &mf) - k, _ := delivery.ManifestsKey(m.Manifest) - raw, err = delivery.EncodeManifest(delivery.Manifest{Module: mf.Module, JSON: string(b.manifests[m.Manifest])}) + k, _ := meshcatalogclient.ManifestsKey(m.Manifest) + raw, err = meshcatalogclient.EncodeManifest(meshcatalog.ManifestsByDigestValue{Module: mf.Module, Manifest: string(b.manifests[m.Manifest])}) put(ManifestsRead, k, raw, err) } for key, build := range b.builds { diff --git a/internal/compose/frombus/frombustest/frombustest.go b/internal/compose/frombus/frombustest/frombustest.go index beda4ff..e0016d0 100644 --- a/internal/compose/frombus/frombustest/frombustest.go +++ b/internal/compose/frombus/frombustest/frombustest.go @@ -14,6 +14,8 @@ import ( "errors" "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" "git.novox.be/novox/mesh-contracts/placement/go/seats/meshmoduleassigner" + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" "strings" "git.novox.be/novox/mesh-sdk/go/delivery" @@ -28,7 +30,7 @@ var testdata embed.FS type Bus struct { Placement map[string]meshmoduleassigner.Placement Record map[string]nodeengine.NodeRecord - Module map[string]delivery.Module + Module map[string]meshcatalog.ModulesByModuleValue Manifests map[string][]byte Value map[string]map[string]any // by node.module Own []compose.OwnSecretRow @@ -46,8 +48,10 @@ func (b *Bus) Placements(context.Context) (map[string]meshmoduleassigner.Placeme func (b *Bus) Records(context.Context) (map[string]nodeengine.NodeRecord, error) { return b.Record, nil } -func (b *Bus) Modules(context.Context) (map[string]delivery.Module, error) { return b.Module, nil } -func (b *Bus) VaultAuthority(context.Context) string { return "" } +func (b *Bus) Modules(context.Context) (map[string]meshcatalog.ModulesByModuleValue, error) { + return b.Module, nil +} +func (b *Bus) VaultAuthority(context.Context) string { return "" } func (b *Bus) Manifest(_ context.Context, digest string) ([]byte, error) { raw, ok := b.Manifests[digest] @@ -149,7 +153,7 @@ func Provisioned() (*Bus, error) { if err := zw.Close(); err != nil { return nil, err } - archive := delivery.ManifestDigest(packed.Bytes()) + archive := meshcatalogclient.ManifestDigest(packed.Bytes()) digestOf := func(c string) string { return "sha256:" + strings.Repeat(c, 64) } fps := FPs() b := &Bus{ @@ -158,24 +162,24 @@ func Provisioned() (*Bus, error) { Modules: []meshmoduleassigner.PlacedModule{{Module: "store"}, {Module: "postgres"}, {Module: "mesh-provisioner-client"}, {Module: "restic"}}}}, Record: map[string]nodeengine.NodeRecord{Node: {Node: Node, TunnelKey: "dGVzdGtleXRlc3RrZXl0ZXN0a2V5dGVzdGtleXRlc3Q=", SealingKey: "sk", Contract: compose.HandedContract, Capabilities: []string{"systemd", "package-manager"}}}, - Module: map[string]delivery.Module{}, Manifests: map[string][]byte{}, + Module: map[string]meshcatalog.ModulesByModuleValue{}, Manifests: map[string][]byte{}, Own: []compose.OwnSecretRow{{Node: Node, Module: "mesh-provisioner-client", Name: "provisioner-broker", Origin: "made", SealedTo: "sk", Sealed: `"box"`}}, Builds: map[string]*delivery.BuilderBuild{ - "restic " + fps["restic"]: {State: delivery.BuildDone, Manifest: delivery.ManifestDigest(resticRaw), Artifacts: []delivery.BuiltArtifact{ + "restic " + fps["restic"]: {State: delivery.BuildDone, Manifest: meshcatalogclient.ManifestDigest(resticRaw), Artifacts: []delivery.BuiltArtifact{ {Name: "tools", Kind: "bundle", Reference: "artifact-store://restic/tools/blobs/" + digestOf("6"), Digest: digestOf("6")}, {Name: "files", Kind: "archive", Reference: "artifact-store://restic/files/blobs/" + archive, Digest: archive}}}, - "mesh-provisioner-client " + fps["mesh-provisioner-client"]: {State: delivery.BuildDone, Manifest: delivery.ManifestDigest(clientRaw), + "mesh-provisioner-client " + fps["mesh-provisioner-client"]: {State: delivery.BuildDone, Manifest: meshcatalogclient.ManifestDigest(clientRaw), Artifacts: []delivery.BuiltArtifact{{Name: "client", Kind: "bundle", Reference: "artifact-store://mesh-provisioner-client/client/blobs/" + digestOf("7"), Digest: digestOf("7")}}}, - "store " + fps["store"]: {State: delivery.BuildDone, Manifest: delivery.ManifestDigest(store)}, - "postgres " + fps["postgres"]: {State: delivery.BuildDone, Manifest: delivery.ManifestDigest(postgres)}, + "store " + fps["store"]: {State: delivery.BuildDone, Manifest: meshcatalogclient.ManifestDigest(store)}, + "postgres " + fps["postgres"]: {State: delivery.BuildDone, Manifest: meshcatalogclient.ManifestDigest(postgres)}, }, Archives: map[string][]byte{archive: packed.Bytes()}, Held: true, } for module, raw := range map[string][]byte{"restic": resticRaw, "mesh-provisioner-client": clientRaw, "store": store, "postgres": postgres} { - b.Module[module] = delivery.Module{FP2: fps[module], Manifest: delivery.ManifestDigest(raw)} - b.Manifests[delivery.ManifestDigest(raw)] = raw + b.Module[module] = meshcatalog.ModulesByModuleValue{Fp2: fps[module], Manifest: meshcatalogclient.ManifestDigest(raw)} + b.Manifests[meshcatalogclient.ManifestDigest(raw)] = raw } return b, nil } diff --git a/internal/compose/frombus/frombustest/frombustest_test.go b/internal/compose/frombus/frombustest/frombustest_test.go index a071e3b..e0ac451 100644 --- a/internal/compose/frombus/frombustest/frombustest_test.go +++ b/internal/compose/frombus/frombustest/frombustest_test.go @@ -7,6 +7,8 @@ import ( "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" "git.novox.be/novox/mesh-contracts/placement/go/clients/meshmoduleassignerclient" "git.novox.be/novox/mesh-contracts/placement/go/seats/meshmoduleassigner" + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" engcomp "github.com/novox/mesh-host/internal/compositions" "reflect" "slices" @@ -72,15 +74,15 @@ func TestTheProvisionedMeshHandsAPlanWithItsProvision(t *testing.T) { p.Modules = append(p.Modules, meshmoduleassigner.PlacedModule{Module: "backup"}) b.Placement[Node] = p } - digest := delivery.ManifestDigest([]byte(row.bad)) + digest := meshcatalogclient.ManifestDigest([]byte(row.bad)) b.Manifests[digest] = []byte(row.bad) - b.Module["backup"] = delivery.Module{FP2: fp, Manifest: digest} + b.Module["backup"] = meshcatalog.ModulesByModuleValue{Fp2: fp, Manifest: digest} if row.refused != "" { - b.Module["backup"] = delivery.Module{FP2: fp, Manifest: digest, Verdict: delivery.VerdictRefused, + b.Module["backup"] = meshcatalog.ModulesByModuleValue{Fp2: fp, Manifest: digest, Verdict: meshcatalogclient.VerdictRefused, Problems: []string{row.refused}} } if row.built != "" { - builtDigest := delivery.ManifestDigest([]byte(row.built)) + builtDigest := meshcatalogclient.ManifestDigest([]byte(row.built)) b.Manifests[builtDigest] = []byte(row.built) b.Builds["backup "+fp] = &delivery.BuilderBuild{State: delivery.BuildDone, Manifest: builtDigest, Artifacts: []delivery.BuiltArtifact{{Name: "files", Kind: "archive", Reference: "artifact-store://backup/files/blobs/x", Digest: "sha256:x"}}} @@ -265,7 +267,7 @@ func TestTheProvisionedMeshHandsAPlanWithItsProvision(t *testing.T) { delete(r.(map[string]any), "user") } raw, _ := json.Marshal(m) - mod.Manifest = delivery.ManifestDigest(raw) + mod.Manifest = meshcatalogclient.ManifestDigest(raw) bb.Module["mesh-provisioner-client"], bb.Manifests[mod.Manifest] = mod, raw built := *bb.Builds["mesh-provisioner-client "+fps["mesh-provisioner-client"]] built.Manifest = mod.Manifest @@ -321,13 +323,13 @@ func statesOf(t *testing.T, b *Bus) frombus.States { put(frombus.HaveRead, node+".node", raw, err) } for module, m := range b.Module { - key, _ := delivery.ModulesKey(module) - raw, err := delivery.EncodeModule(m) + key, _ := meshcatalogclient.ModulesKey(module) + raw, err := meshcatalogclient.EncodeModule(m) put(frombus.ModulesRead, key, raw, err) } for digest, man := range b.Manifests { - key, _ := delivery.ManifestsKey(digest) - raw, err := delivery.EncodeManifest(delivery.Manifest{Module: "m", JSON: string(man)}) + key, _ := meshcatalogclient.ManifestsKey(digest) + raw, err := meshcatalogclient.EncodeManifest(meshcatalog.ManifestsByDigestValue{Module: "m", Manifest: string(man)}) put(frombus.ManifestsRead, key, raw, err) } for at, build := range b.Builds { diff --git a/internal/compose/frombus/held_rows_test.go b/internal/compose/frombus/held_rows_test.go index 09ab4c8..a60ec47 100644 --- a/internal/compose/frombus/held_rows_test.go +++ b/internal/compose/frombus/held_rows_test.go @@ -7,6 +7,8 @@ import ( "fmt" "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" "git.novox.be/novox/mesh-contracts/placement/go/seats/meshmoduleassigner" + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" engcomp "github.com/novox/mesh-host/internal/compositions" "strings" "testing" @@ -21,13 +23,13 @@ func aNode(t *testing.T, manifests ...string) (*bus, map[string]string) { t.Helper() p := meshmoduleassigner.Placement{Node: "n1", Decided: meshmoduleassigner.NodeDecided{Hub: true, Address: "10.77.0.1"}} b := &bus{records: map[string]nodeengine.NodeRecord{"n1": {Node: "n1", TunnelKey: publicKey(t), SealingKey: "sk", Capabilities: []string{"systemd"}}}, - modules: map[string]delivery.Module{}, manifests: map[string][]byte{}, builds: map[string]*delivery.BuilderBuild{}} + modules: map[string]meshcatalog.ModulesByModuleValue{}, manifests: map[string][]byte{}, builds: map[string]*delivery.BuilderBuild{}} fps := map[string]string{} for i, raw := range manifests { module := strings.SplitN(strings.TrimPrefix(raw, `{"module":"`), `"`, 2)[0] p.Modules = append(p.Modules, meshmoduleassigner.PlacedModule{Module: module}) - b.modules[module] = delivery.Module{Manifest: delivery.ManifestDigest([]byte(raw))} - b.manifests[delivery.ManifestDigest([]byte(raw))] = []byte(raw) + b.modules[module] = meshcatalog.ModulesByModuleValue{Manifest: meshcatalogclient.ManifestDigest([]byte(raw))} + b.manifests[meshcatalogclient.ManifestDigest([]byte(raw))] = []byte(raw) fps[module] = "src2:" + strings.Repeat(string(rune('1'+i)), 64) } b.placements = map[string]meshmoduleassigner.Placement{"n1": p} @@ -51,7 +53,7 @@ func aNodeRefusedWholeSaysWhatItHeld(t *testing.T) { // refuse writes the indexer's refusal of module's manifest into its modules value (novox/hq ADR 0578 §1). func refuse(b *bus, module string, problems ...string) { m := b.modules[module] - m.Verdict, m.Problems = delivery.VerdictRefused, problems + m.Verdict, m.Problems = meshcatalogclient.VerdictRefused, problems b.modules[module] = m } @@ -144,7 +146,7 @@ func aRunningModuleWaitsForItsBuild(t *testing.T) { b.held = true // every grant held, so a contribution's credential stands in for nothing b.own = []compose.OwnSecretRow{{Node: "n1", Module: "node-tools", Name: "broker", Origin: "made", SealedTo: "sk", Sealed: `"box"`}} mod := b.modules["node-tools"] - mod.FP2 = next + mod.Fp2 = next b.modules["node-tools"] = mod b.builds["node-tools "+old] = &delivery.BuilderBuild{State: delivery.BuildDone, Manifest: mod.Manifest, Artifacts: []delivery.BuiltArtifact{{Name: "runtime", Kind: "bundle", Reference: "artifact-store://node-tools/runtime/blobs/" + dg("1"), Digest: dg("1")}}} @@ -254,7 +256,7 @@ func aHeldModuleOfItsOwnAccountNeverRefusesTheNode(t *testing.T) { name = "runtime" } mod := b.modules[m] - mod.FP2 = fps[m] + mod.Fp2 = fps[m] b.modules[m] = mod b.builds[m+" "+fps[m]] = &delivery.BuilderBuild{State: delivery.BuildDone, Manifest: mod.Manifest, Artifacts: []delivery.BuiltArtifact{{Name: name, Kind: "bundle", Reference: "artifact-store://" + m + "/" + name + "/blobs/" + dg, Digest: dg}}} diff --git a/internal/compose/frombus/states.go b/internal/compose/frombus/states.go index 2c5d38b..501005a 100644 --- a/internal/compose/frombus/states.go +++ b/internal/compose/frombus/states.go @@ -16,6 +16,8 @@ import ( "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" "git.novox.be/novox/mesh-contracts/placement/go/clients/meshmoduleassignerclient" "git.novox.be/novox/mesh-contracts/placement/go/seats/meshmoduleassigner" + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" "strings" "time" @@ -102,11 +104,11 @@ func (r statesReader) Records(context.Context) (map[string]nodeengine.NodeRecord return out, err } -func (r statesReader) Modules(context.Context) (map[string]delivery.Module, error) { - out := map[string]delivery.Module{} +func (r statesReader) Modules(context.Context) (map[string]meshcatalog.ModulesByModuleValue, error) { + out := map[string]meshcatalog.ModulesByModuleValue{} err := r.each(ModulesRead, func(k string, v []byte) { - if module, err := delivery.ParseModulesKey(k); err == nil { - if m, err := delivery.DecodeModule(v); err == nil { + if module, err := meshcatalogclient.ParseModulesKey(k); err == nil { + if m, err := meshcatalogclient.DecodeModule(v); err == nil { out[module] = m } } @@ -115,7 +117,7 @@ func (r statesReader) Modules(context.Context) (map[string]delivery.Module, erro } func (r statesReader) Manifest(_ context.Context, digest string) ([]byte, error) { - key, err := delivery.ManifestsKey(digest) + key, err := meshcatalogclient.ManifestsKey(digest) if err != nil { return nil, err } @@ -126,11 +128,11 @@ func (r statesReader) Manifest(_ context.Context, digest string) ([]byte, error) if !found { return nil, fmt.Errorf("the indexer holds no manifest %s", digest) } - m, err := delivery.DecodeManifest(v) + m, err := meshcatalogclient.DecodeManifest(v) if err != nil { return nil, err } - return []byte(m.JSON), nil + return []byte(m.Manifest), nil } func (r statesReader) Values(_ context.Context, node, module string) (map[string]any, error) { diff --git a/internal/composer/composer.go b/internal/composer/composer.go index 984bbcd..3e5f3f2 100644 --- a/internal/composer/composer.go +++ b/internal/composer/composer.go @@ -10,14 +10,16 @@ package composer import ( "encoding/json" "fmt" + "git.novox.be/novox/mesh-contracts/placement/go/clients/meshmoduleassignerclient" "git.novox.be/novox/mesh-contracts/placement/go/seats/meshmoduleassigner" + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-networking/modules/mesh-wireguard/go/overlay" "slices" "sort" "strings" + sdk "git.novox.be/novox/mesh-delivery/sdk/composer" "git.novox.be/novox/mesh-sdk/go/catalogue" - sdk "git.novox.be/novox/mesh-sdk/go/composer" - "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/sdkversion" "github.com/novox/mesh-host/internal/compose" @@ -92,7 +94,7 @@ func Run(r Request) (Verdict, error) { // parse decodes every manifest leniently and takes its verdict (novox/hq ADR 0578 §3, §5): the composer judges none. A // manifest refused by its verdict is held, composed as its stub without its items (as the bus reader holds one); bytes // that do not decode are left out. Both are named in reads. -func parse(m map[string]Manifest, reads map[string]ManifestRead, verdicts map[string]delivery.ManifestVerdict) (map[string]catalogue.Manifest, error) { +func parse(m map[string]Manifest, reads map[string]ManifestRead, verdicts map[string]json.RawMessage) (map[string]catalogue.Manifest, error) { out := map[string]catalogue.Manifest{} for name, man := range m { if len(man.Raw) == 0 { @@ -102,12 +104,16 @@ func parse(m map[string]Manifest, reads map[string]ManifestRead, verdicts map[st read := ManifestRead{Accepted: err == nil, SetAside: decoded.SetAside()} if err != nil { read.Refused = fmt.Sprintf("the manifest of %s does not decode: %v", name, err) - } else if v, ok := verdicts[name]; ok { - if refused, _ := delivery.TakeManifestVerdict(&v); refused { + } else if raw, ok := verdicts[name]; ok { + v, verr := meshcatalogclient.DecodeVerdict(raw) + if verr != nil { + return nil, fmt.Errorf("composer: the verdict of %s is not one: %w", name, verr) + } + if refused, _ := meshcatalogclient.TakeVerdict(&v); refused { if v.Module == "" { v.Module = name } - read = ManifestRead{Refused: v.Why()} + read = ManifestRead{Refused: meshcatalogclient.VerdictWhy(v)} decoded.Items = nil decoded = compose.Stub(decoded) } @@ -132,9 +138,17 @@ func runCase(r Request, c Case, depth string) (CaseVerdict, error) { if err != nil { return CaseVerdict{}, err } - placements, err := interimPlacements(c.Placements) - if err != nil { - return CaseVerdict{}, fmt.Errorf("composer: the case %q: %w", c.Name, err) + placements := map[string]*meshmoduleassigner.Placement{} + for node, raw := range c.Placements { + if string(raw) == "null" { + placements[node] = nil + continue + } + p, perr := meshmoduleassignerclient.DecodePlacement(raw) + if perr != nil { + return CaseVerdict{}, fmt.Errorf("composer: the case %q: the placement of %s is not one: %w", c.Name, node, perr) + } + placements[node] = &p } in, err := frombus.Placed(base, placements) if err != nil { @@ -143,8 +157,8 @@ func runCase(r Request, c Case, depth string) (CaseVerdict, error) { // The module the mesh provides itself (the private network) has no repository the indexer reads, so no case carries // it: its manifest comes with mesh-sdk and always wins, as the bus reader has it (frombus readBase). Without it a node // it is assigned to composed nothing (issue 1126, check of mesh-host #200: "nothing provides mesh-wireguard"). - for _, name := range delivery.ProvidedModules() { - m, _, err := delivery.ProvidedManifest(name) + for _, name := range overlay.ProvidedModules() { + m, _, err := overlay.ProvidedManifest(name) if err != nil { return CaseVerdict{}, fmt.Errorf("composer: %w", err) } @@ -345,20 +359,3 @@ func SentInput(in compose.Input, node string) compose.Input { } return in } - -// interimPlacements carries the case's placements, typed as mesh-sdk's composer still types them, over to the seat's -// type through JSON. It goes when this package takes the case from the delivery SDK's composer, which holds them raw. -func interimPlacements(in map[string]*delivery.Placement) (map[string]*meshmoduleassigner.Placement, error) { - if in == nil { - return nil, nil - } - raw, err := json.Marshal(in) - if err != nil { - return nil, err - } - var out map[string]*meshmoduleassigner.Placement - if err := json.Unmarshal(raw, &out); err != nil { - return nil, err - } - return out, nil -} diff --git a/internal/composer/composer_test.go b/internal/composer/composer_test.go index a2a2101..c9ff7e1 100644 --- a/internal/composer/composer_test.go +++ b/internal/composer/composer_test.go @@ -3,6 +3,11 @@ package composer import ( "context" "encoding/json" + "git.novox.be/novox/mesh-contracts/placement/go/seats/meshmoduleassigner" + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-networking/modules/mesh-wireguard/go/overlay" + "github.com/novox/mesh-host/internal/buckets/bucketstest" engcomp "github.com/novox/mesh-host/internal/compositions" "os" "reflect" @@ -10,7 +15,6 @@ import ( "testing" "git.novox.be/novox/mesh-sdk/go/catalogue" - "git.novox.be/novox/mesh-sdk/go/delivery" "github.com/novox/mesh-host/internal/compose" "github.com/novox/mesh-host/internal/compose/frombus" "github.com/novox/mesh-host/internal/compose/frombus/frombustest" @@ -55,20 +59,20 @@ func TestARequestItCannotRunIsRefused(t *testing.T) { placed := withInput(t, Request{V: Version, Machines: []string{"a"}, Depth: DepthResolve, Cases: []Case{{Name: "trunk", Manifests: map[string]Manifest{"demo": {Raw: json.RawMessage(demo)}}}, {Name: "change", Manifests: map[string]Manifest{"demo": {Raw: json.RawMessage(demo)}}, - Placements: map[string]*delivery.Placement{"a": {Node: "a", Modules: []delivery.PlacedModule{{Module: "ghost"}}}}}}}, + Placements: bucketstest.Raw(map[string]meshmoduleassigner.Placement{"a": {V: 1, Node: "a", Modules: []meshmoduleassigner.PlacedModule{{Module: "ghost"}}}})}}}, compose.Input{Nodes: []compose.Node{{Name: "a"}}}) if v, err := Run(placed); err != nil || !v.Cases[0].Machines["a"].Composes || v.Cases[1].Machines["a"].Composes || !strings.Contains(v.Cases[1].Machines["a"].Reason, "ghost") { t.Errorf("a case composes its own placement: %+v, %v", v, err) } - placed.Cases[1].Placements = map[string]*delivery.Placement{"z": {Node: "z"}} + placed.Cases[1].Placements = bucketstest.Raw(map[string]meshmoduleassigner.Placement{"z": {V: 1, Node: "z"}}) if _, err := Run(placed); err == nil || !strings.Contains(err.Error(), "holds no node for") { t.Errorf("a case placing a node the input lacks: %v", err) } // Issue 1126 (check of mesh-host #200): the module the mesh provides itself (the private network) comes from mesh-sdk, // a second source beside the indexer's manifests the cases carry, and the input carries no catalogue: a node it is // assigned to still composes, and what it provides is provided. - wg := delivery.ProvidedModules()[0] + wg := overlay.ProvidedModules()[0] provided := withInput(t, Request{V: Version, Machines: []string{"a"}, Depth: DepthResolve, Cases: []Case{{Name: "change", Manifests: map[string]Manifest{"demo": {Raw: json.RawMessage(demo)}}}}}, compose.Input{Nodes: []compose.Node{{Name: "a", Assigned: []string{wg, "demo"}}}}) @@ -245,8 +249,8 @@ func TestTheComposerTakesTheVerdictsOfItsInput(t *testing.T) { if got := v.Cases[0].Manifests["xterm"]; !got.Accepted { t.Fatalf("xterm: %+v, want accepted", got) } - r.Cases[0].Verdicts = map[string]delivery.ManifestVerdict{"xterm": {Module: "xterm", State: delivery.VerdictRefused, - Problems: []string{"falls back to a key not listed"}, Judge: "src2:j"}} + r.Cases[0].Verdicts = bucketstest.Raw(map[string]meshcatalog.VerdictsByDigestValue{"xterm": {V: 1, Module: "xterm", State: meshcatalogclient.VerdictRefused, + Problems: []string{"falls back to a key not listed"}, Judge: "src2:j"}}) shelf, err := parse(r.Cases[0].Manifests, map[string]ManifestRead{}, r.Cases[0].Verdicts) if err != nil || !shelf["xterm"].Stubbed { t.Fatalf("the refused module is not held as its stub: %+v %v", shelf["xterm"], err) diff --git a/internal/judge/answer_test.go b/internal/judge/answer_test.go index bdd7d9f..70e0cc7 100644 --- a/internal/judge/answer_test.go +++ b/internal/judge/answer_test.go @@ -2,16 +2,15 @@ package judge import ( nodeengine "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" + "github.com/novox/mesh-host/internal/compositions" "strings" "testing" - - "git.novox.be/novox/mesh-sdk/go/delivery" ) // An answer too large for the node's key is fitted: the texts go first, the verdict and the paths stay, and the answer // given is not changed (moved from mesh-sdk's placement test with FitAnswer, novox/hq ADR 0593 §7). func TestFitAnswer(t *testing.T) { - long := strings.Repeat("x", delivery.MaxBody) + long := strings.Repeat("x", compositions.MaxBody) big := CandidateAnswer{Proposal: "p-x1", Revision: 1, Verdict: nodeengine.CandidateAnswerVerdictDiffers, Differs: []ModuleDiffers{{ Module: "nats", Why: "differs", Bodies: []ResourceBodies{{ID: "conf", Why: "changed", Paths: []string{"content.links.1", "content.password"}, Composed: long, Running: long}}}}} diff --git a/internal/judge/judge_test.go b/internal/judge/judge_test.go index 2a5b7e3..12829b9 100644 --- a/internal/judge/judge_test.go +++ b/internal/judge/judge_test.go @@ -3,14 +3,13 @@ package judge import ( "encoding/json" "fmt" + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" "strings" "testing" - - "git.novox.be/novox/mesh-sdk/go/delivery" ) func composedOf(module string, c byte, data bool, resources ...string) Composed { - out := Composed{Node: "ace", Module: module, FP: delivery.SourcePrefix2 + strings.Repeat(string(c), 64), Data: data} + out := Composed{Node: "ace", Module: module, FP: meshcatalogclient.SourcePrefix2 + strings.Repeat(string(c), 64), Data: data} for _, r := range resources { out.Resources = append(out.Resources, json.RawMessage(r)) } diff --git a/internal/selfcompose/candidate.go b/internal/selfcompose/candidate.go index c56f6b4..023db96 100644 --- a/internal/selfcompose/candidate.go +++ b/internal/selfcompose/candidate.go @@ -7,6 +7,9 @@ import ( "errors" "fmt" "git.novox.be/novox/mesh-contracts/placement/go/seats/meshwants" + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + "github.com/novox/mesh-host/internal/buckets" engcomp "github.com/novox/mesh-host/internal/compositions" "maps" "slices" @@ -16,7 +19,6 @@ import ( nec "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" - "git.novox.be/novox/mesh-sdk/go/delivery" "github.com/novox/mesh-host/internal/agent" "github.com/novox/mesh-host/internal/compose/frombus" @@ -25,7 +27,7 @@ import ( // **A check's candidate, composed in shadow on a call** (novox/hq ADR 0593 §7): the node seat's verb compose-candidate. // Given the check's composition key (`.`), it reads the group's candidate from the indexer -// (delivery.CandidatesBucket, keyed by the group alone, delivery.CandidatesKey) at those heads, and composes itself +// (buckets.Candidates, keyed by the group alone, buckets.CandidateKey) at those heads, and composes itself // twice from what its shadow holds (placement, records, the indexer's modules, its config values), with its own sealed // values: once as it is, and once with the candidate's modules laid over the indexer's (each at the candidate's // fingerprint, its manifest by digest and the indexer's verdict on it, ADR 0578 §3; a module the candidate removes left @@ -141,13 +143,13 @@ func (s *Shadow) composeCandidate(ctx context.Context, c *Composer, ask nodeengi a.Verdict = nec.Unready return a } - group, heads, err := delivery.ParseCompositionKey(ask.Candidate) + group, heads, err := nec.ParseCompositionKey(ask.Candidate) if err != nil { return refuse(err.Error()) } // The ask names the check (the composition key, `.`); the indexer keys the group's candidate by - // the group alone (delivery.CandidatesKey) and holds its heads hash in the value. - ckey, err := delivery.CandidatesKey(group) + // the group alone (buckets.CandidateKey) and holds its heads hash in the value. + ckey, err := buckets.CandidateKey(group) if err != nil { return refuse(err.Error()) } @@ -155,21 +157,21 @@ func (s *Shadow) composeCandidate(ctx context.Context, c *Composer, ask nodeengi return unready(errNotReady.Error()) } base := run.add(c.scratch()) - raw, err := base.get(ctx, delivery.CandidatesBucket, ckey) + raw, err := base.get(ctx, buckets.Candidates, ckey) if err != nil { return unready(fmt.Sprintf("the indexer's candidate %s cannot be read: %v", ask.Candidate, err)) } if raw == nil { return refuse("the indexer holds no candidate " + ask.Candidate) } - cand, err := delivery.DecodeCandidate(raw) + cand, err := meshcatalog.DecodeCandidatesByGroupValue(raw) if err != nil { return refuse(fmt.Sprintf("the indexer's candidate %s cannot be read: %v", ask.Candidate, err)) } - if cand.HeadsHash != heads { + if cand.Heads != heads { // The group moved to other heads: the check asked is replaced, and this node composes not a candidate it was // not asked about. No answer (ADR 0593 §9), never a verdict on the wrong heads. - return unready(fmt.Sprintf("the indexer's candidate of the group %s is at heads %s, not %s: the check asked is replaced", group, cand.HeadsHash, heads)) + return unready(fmt.Sprintf("the indexer's candidate of the group %s is at heads %s, not %s: the check asked is replaced", group, cand.Heads, heads)) } // Every module's manifest is read before either composition, several at once, into the shadow's own: a composition // reads every manifest in the mesh one by one, which off the control-node took about 35 s each (issue 1126). @@ -332,7 +334,7 @@ func (c *Composer) scratch() *Composer { // warm reads, several at once, each manifest the indexer's modules and the candidate name that the shadow (origin) does // not hold yet, keeping each in origin and in c; one not read is left to the composition, which reads it and says why. // Its reads are not noted as the bus failing: the composition that reads again notes its own. -func (c *Composer) warm(ctx context.Context, origin *Composer, cand delivery.Candidate) { +func (c *Composer) warm(ctx context.Context, origin *Composer, cand meshcatalog.CandidatesByGroupValue) { origin.mu.Lock() want := map[string]bool{} for _, m := range origin.modules { @@ -351,11 +353,11 @@ func (c *Composer) warm(ctx context.Context, origin *Composer, cand delivery.Can if len(want) == 0 { return } - defer c.waiting(fmt.Sprintf("%d of the indexer's manifests (%s)", len(want), delivery.ManifestsBucket))() + defer c.waiting(fmt.Sprintf("%d of the indexer's manifests (%s)", len(want), buckets.Manifests))() slots := make(chan struct{}, warmAtOnce) var wg sync.WaitGroup for digest := range want { - key, err := delivery.ManifestsKey(digest) + key, err := meshcatalogclient.ManifestsKey(digest) if err != nil { continue } @@ -368,15 +370,15 @@ func (c *Composer) warm(ctx context.Context, origin *Composer, cand delivery.Can return } defer func() { <-slots }() - value, err := c.env.Get(ctx, delivery.ManifestsBucket, key) + value, err := c.env.Get(ctx, buckets.Manifests, key) if err != nil || value == nil { return } - m, err := delivery.DecodeManifest(value) + m, err := meshcatalogclient.DecodeManifest(value) if err != nil { return } - c.keepRead(func(k *Composer) { k.manifests[digest] = []byte(m.JSON) }) + c.keepRead(func(k *Composer) { k.manifests[digest] = []byte(m.Manifest) }) }() } wg.Wait() @@ -385,31 +387,31 @@ func (c *Composer) warm(ctx context.Context, origin *Composer, cand delivery.Can // layCandidate lays the candidate's modules over the indexer's: each at its fingerprint, with its manifest's digest and // the indexer's verdict on it (read from its verdicts, none read taken as accepted); a module the candidate removes goes. // It answers the modules laid. -func (c *Composer) layCandidate(ctx context.Context, cand delivery.Candidate) (map[string]bool, error) { +func (c *Composer) layCandidate(ctx context.Context, cand meshcatalog.CandidatesByGroupValue) (map[string]bool, error) { touched := map[string]bool{} for _, pull := range cand.Pulls { for module, cm := range pull.Fps { - if cm.None2 == delivery.CandidateNoneNoModule { + if cm.None2 == candidateNoneNoModule { c.mu.Lock() delete(c.modules, module) c.mu.Unlock() continue } - if cm.FP2 == "" { + if cm.Fp2 == "" { continue // no fingerprint at the preview: composed as the trunk holds it } - m := delivery.Module{FP2: cm.FP2, Manifest: cm.Manifest, Repository: pull.Repository, Commit: pull.Head.Head} + m := meshcatalog.ModulesByModuleValue{Fp2: cm.Fp2, Manifest: cm.Manifest, Repository: pull.Repository, Commit: pull.Head} if cm.Manifest != "" { - key, err := delivery.VerdictsKey(cm.Manifest) + key, err := meshcatalogclient.VerdictsKey(cm.Manifest) if err != nil { return nil, err } - raw, err := c.get(ctx, delivery.VerdictsBucket, key) + raw, err := c.get(ctx, buckets.Verdicts, key) if err != nil { return nil, fmt.Errorf("the indexer's verdict on %s's manifest cannot be read: %w", module, err) } if raw != nil { - v, err := delivery.DecodeManifestVerdict(raw) + v, err := meshcatalogclient.DecodeVerdict(raw) if err != nil { return nil, fmt.Errorf("the indexer's verdict on %s's manifest cannot be read: %w", module, err) } @@ -442,12 +444,12 @@ func (c *Composer) composeShadow(ctx context.Context, running []engcomp.Composed // A module the laid candidate changes is composed at the candidate's build, though the node runs another. for module := range fps { if mod, ok := modules[module]; ok && laid[module] { - fps[module] = mod.FP2 + fps[module] = mod.Fp2 } } kept := map[string]string{} for module, fp := range fps { - if mod, ok := modules[module]; ok && mod.FP2 == fp { + if mod, ok := modules[module]; ok && mod.Fp2 == fp { continue } kept[module] = fp @@ -472,3 +474,7 @@ func (c *Composer) composeShadow(ctx context.Context, running []engcomp.Composed } return comps, held, nil } + +// candidateNoneNoModule is the words the indexer writes in a candidate's module that the merge preview names no manifest +// for: the change removes the module. It is the indexer's text, kept here until the catalogue client names it. +const candidateNoneNoModule = "the merge preview names no manifest for the module's directory" diff --git a/internal/selfcompose/candidate_test.go b/internal/selfcompose/candidate_test.go index 1d481a0..e659478 100644 --- a/internal/selfcompose/candidate_test.go +++ b/internal/selfcompose/candidate_test.go @@ -5,7 +5,11 @@ import ( "encoding/json" "git.novox.be/novox/mesh-contracts/placement/go/clients/meshmoduleassignerclient" "git.novox.be/novox/mesh-contracts/placement/go/seats/meshmoduleassigner" + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" "github.com/novox/mesh-host/internal/buckets" + "github.com/novox/mesh-host/internal/buckets/bucketstest" "strings" "sync/atomic" "testing" @@ -13,7 +17,6 @@ import ( nec "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" - "git.novox.be/novox/mesh-sdk/go/delivery" "github.com/novox/mesh-host/internal/agent" "github.com/novox/mesh-host/internal/compose" @@ -26,48 +29,48 @@ import ( // within the bound; a shadow not ready and an ended bound answer unready at once, which the checker takes as no answer. func TestTheNodeComposesACandidateOnAsk(t *testing.T) { b := newFakeBus() - fp := delivery.SourcePrefix2 + strings.Repeat("a", 64) - fpNew := delivery.SourcePrefix2 + strings.Repeat("b", 64) - digest := delivery.ManifestDigest([]byte(lemurs)) + fp := meshcatalogclient.SourcePrefix2 + strings.Repeat("a", 64) + fpNew := meshcatalogclient.SourcePrefix2 + strings.Repeat("b", 64) + digest := meshcatalogclient.ManifestDigest([]byte(lemurs)) lemurs2 := strings.Replace(lemurs, `"version":"1"`, `"version":"2"`, 1) - digest2 := delivery.ManifestDigest([]byte(lemurs2)) + digest2 := meshcatalogclient.ManifestDigest([]byte(lemurs2)) for d, body := range map[string]string{digest: lemurs, digest2: lemurs2} { - man, _ := delivery.EncodeManifest(delivery.Manifest{Module: "lemurs", JSON: body}) - key, _ := delivery.ManifestsKey(d) - b.kv[delivery.ManifestsBucket+"/"+key] = man + man, _ := meshcatalogclient.EncodeManifest(meshcatalog.ManifestsByDigestValue{Module: "lemurs", Manifest: body}) + key, _ := meshcatalogclient.ManifestsKey(d) + b.kv[buckets.Manifests+"/"+key] = man } - // The indexer keys a candidate by its group alone (delivery.CandidatesKey, the branch name escaped, a slash + // The indexer keys a candidate by its group alone (buckets.CandidateKey, the branch name escaped, a slash // allowed), its heads hash inside the value; the checker asks by the composition key, `.`. candidateOf := func(group, manifest string) string { - head := delivery.Head{Repository: "novox/x", Number: 7, Head: "c0ffee" + manifest[len(manifest)-6:]} - heads, _ := delivery.HeadsHash([]delivery.Head{head}) - pulls := []delivery.CandidatePull{{Head: head, Modules: []string{"lemurs"}, - Fps: map[string]delivery.CandidateModule{"lemurs": {FP2: fpNew, Manifest: manifest}}}} - raw, err := delivery.EncodeCandidate(delivery.Candidate{Group: group, HeadsHash: heads, Pulls: pulls, - FPSet: delivery.FingerprintSet(pulls)}) + head := meshdeliveryclient.Head{Repository: "novox/x", Number: 7, Head: "c0ffee" + manifest[len(manifest)-6:]} + heads, _ := meshdeliveryclient.HeadsHash([]meshdeliveryclient.Head{head}) + pulls := []meshcatalog.CandidatesByGroupValuePullsItem{bucketstest.Pull(head, []string{"lemurs"}, + map[string]meshcatalog.CandidatesByGroupValuePullsItemFpsValue{"lemurs": {Fp2: fpNew, Manifest: manifest}})} + raw, err := bucketstest.GroupCandidate(meshcatalog.CandidatesByGroupValue{Group: group, Heads: heads, Pulls: pulls, + Fpset: meshcatalogclient.FingerprintSet(pulls)}) if err != nil { t.Fatal(err) } - ckey, _ := delivery.CandidatesKey(group) - b.kv[delivery.CandidatesBucket+"/"+ckey] = raw - key, _ := delivery.CompositionKey(group, heads) + ckey, _ := buckets.CandidateKey(group) + b.kv[buckets.Candidates+"/"+ckey] = raw + key, _ := nec.CompositionKey(group, heads) return key } accepted, refusedKey := candidateOf("fix/940-no-retire-on-partial-read", digest2), "" { // The same change, its manifest refused by the indexer's judge. refusedMan := strings.Replace(lemurs2, `"version":"2"`, `"version":"3"`, 1) - d3 := delivery.ManifestDigest([]byte(refusedMan)) - man, _ := delivery.EncodeManifest(delivery.Manifest{Module: "lemurs", JSON: refusedMan}) - key, _ := delivery.ManifestsKey(d3) - b.kv[delivery.ManifestsBucket+"/"+key] = man - vkey, _ := delivery.VerdictsKey(d3) - v, err := delivery.EncodeManifestVerdict(delivery.ManifestVerdict{Module: "lemurs", State: delivery.VerdictRefused, - Problems: []string{"a field the judge refuses"}, Judge: "src2:j", At: time.Now()}) + d3 := meshcatalogclient.ManifestDigest([]byte(refusedMan)) + man, _ := meshcatalogclient.EncodeManifest(meshcatalog.ManifestsByDigestValue{Module: "lemurs", Manifest: refusedMan}) + key, _ := meshcatalogclient.ManifestsKey(d3) + b.kv[buckets.Manifests+"/"+key] = man + vkey, _ := meshcatalogclient.VerdictsKey(d3) + v, err := meshcatalogclient.EncodeVerdict(meshcatalog.VerdictsByDigestValue{Module: "lemurs", State: meshcatalogclient.VerdictRefused, + Problems: []string{"a field the judge refuses"}, Judge: "src2:j", At: func(t time.Time) *time.Time { return &t }(time.Now())}) if err != nil { t.Fatal(err) } - b.kv[delivery.VerdictsBucket+"/"+vkey] = v + b.kv[buckets.Verdicts+"/"+vkey] = v refusedKey = candidateOf("fix/941-refused", d3) } @@ -87,7 +90,7 @@ func TestTheNodeComposesACandidateOnAsk(t *testing.T) { place, _ := meshmoduleassignerclient.EncodePlacement(meshmoduleassigner.Placement{Node: "n1", Decided: meshmoduleassigner.NodeDecided{Hub: true, Address: "10.77.0.1"}, Modules: []meshmoduleassigner.PlacedModule{{Module: "lemurs"}}}) have, _ := nec.EncodeHaveNode(nodeengine.HaveByNodeNodeValue{Record: &nodeengine.NodeRecord{Node: "n1", TunnelKey: "k", SealingKey: "sk", Capabilities: []string{"systemd"}}}) - mod, _ := delivery.EncodeModule(delivery.Module{FP2: fp, Manifest: digest, Repository: "novox/x", Commit: "c"}) + mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fp, Manifest: digest, Repository: "novox/x", Commit: "c"}) deadline := time.Now().Add(5 * time.Second) watching := func() int { b.mu.Lock(); defer b.mu.Unlock(); return len(b.watches) } for watching() < 6 && time.Now().Before(deadline) { @@ -95,8 +98,8 @@ func TestTheNodeComposesACandidateOnAsk(t *testing.T) { } b.put(t, buckets.Placement, "placement.n1", place) b.put(t, buckets.Have, "n1.node", have) - b.put(t, delivery.ModulesBucket, "lemurs", mod) - for _, bucket := range []string{buckets.PlacementCandidates, buckets.Placement, buckets.Have, delivery.ModulesBucket, ResolvedBucket} { + b.put(t, buckets.Modules, "lemurs", mod) + for _, bucket := range []string{buckets.PlacementCandidates, buckets.Placement, buckets.Have, buckets.Modules, ResolvedBucket} { b.send(bucket, agent.Update{Caught: true}) } for !s.ready.Load() && time.Now().Before(deadline) { @@ -129,13 +132,13 @@ func TestTheNodeComposesACandidateOnAsk(t *testing.T) { if _, err := nec.EncodeAnswer(r); err != nil { t.Errorf("the refused answer does not fit the seat's shape: %v", err) } - missing, _ := delivery.CompositionKey("novox/x#8", "0123456789ab") + missing, _ := nec.CompositionKey("novox/x#8", "0123456789ab") if m := s.ComposeCandidate(ctx, nodeengine.ComposeCandidateInput{Candidate: missing}); !strings.Contains(m.Refused, "holds no candidate") { t.Errorf("a candidate the indexer does not hold answered %+v", m) } // The group's candidate moved to other heads since the ask: the node composes not the one asked, and answers // unready (no answer), naming the heads it holds. - moved, _ := delivery.CompositionKey("fix/940-no-retire-on-partial-read", "0123456789ab") + moved, _ := nec.CompositionKey("fix/940-no-retire-on-partial-read", "0123456789ab") if m := s.ComposeCandidate(ctx, nodeengine.ComposeCandidateInput{Candidate: moved}); m.Verdict != nec.Unready || !strings.Contains(m.Refused, "heads") { t.Errorf("a candidate at other heads answered %+v", m) @@ -154,30 +157,30 @@ func TestTheNodeComposesACandidateOnAsk(t *testing.T) { // composition, into the shadow, and a later ask reads none; an ask whose read does not answer is answered unready at // its bound, naming what the node waits on; and every answer is said in one line with what it waited on. t.Run("each manifest is read once, and the answer comes within its bound", func(t *testing.T) { - fp := delivery.SourcePrefix2 + strings.Repeat("a", 64) - fpNew := delivery.SourcePrefix2 + strings.Repeat("b", 64) + fp := meshcatalogclient.SourcePrefix2 + strings.Repeat("a", 64) + fpNew := meshcatalogclient.SourcePrefix2 + strings.Repeat("b", 64) lemurs2 := strings.Replace(lemurs, `"version":"1"`, `"version":"2"`, 1) other := strings.NewReplacer(`"module":"lemurs"`, `"module":"other"`, "lemurs.service", "other.service").Replace(lemurs) - digest, digest2, digestOther := delivery.ManifestDigest([]byte(lemurs)), delivery.ManifestDigest([]byte(lemurs2)), - delivery.ManifestDigest([]byte(other)) - otherKey, _ := delivery.ManifestsKey(digestOther) + digest, digest2, digestOther := meshcatalogclient.ManifestDigest([]byte(lemurs)), meshcatalogclient.ManifestDigest([]byte(lemurs2)), + meshcatalogclient.ManifestDigest([]byte(other)) + otherKey, _ := meshcatalogclient.ManifestsKey(digestOther) // shadow is a ready shadow of n1 over b: lemurs placed on it, other placed nowhere (its manifest read for the mesh's // catalogue alone), and a candidate laying lemurs at a new build; its clock moves a second at each reading. shadow := func(b *fakeBus) (*Shadow, string) { for d, body := range map[string]string{digest: lemurs, digest2: lemurs2, digestOther: other} { - man, _ := delivery.EncodeManifest(delivery.Manifest{Module: "x", JSON: body}) - key, _ := delivery.ManifestsKey(d) - b.kv[delivery.ManifestsBucket+"/"+key] = man + man, _ := meshcatalogclient.EncodeManifest(meshcatalog.ManifestsByDigestValue{Module: "x", Manifest: body}) + key, _ := meshcatalogclient.ManifestsKey(d) + b.kv[buckets.Manifests+"/"+key] = man } - head := delivery.Head{Repository: "novox/x", Number: 7, Head: "c0ffee"} - heads, _ := delivery.HeadsHash([]delivery.Head{head}) - pulls := []delivery.CandidatePull{{Head: head, Modules: []string{"lemurs"}, - Fps: map[string]delivery.CandidateModule{"lemurs": {FP2: fpNew, Manifest: digest2}}}} - raw, _ := delivery.EncodeCandidate(delivery.Candidate{Group: "fix/1126", HeadsHash: heads, Pulls: pulls, - FPSet: delivery.FingerprintSet(pulls)}) - ckey, _ := delivery.CandidatesKey("fix/1126") - b.kv[delivery.CandidatesBucket+"/"+ckey] = raw + head := meshdeliveryclient.Head{Repository: "novox/x", Number: 7, Head: "c0ffee"} + heads, _ := meshdeliveryclient.HeadsHash([]meshdeliveryclient.Head{head}) + pulls := []meshcatalog.CandidatesByGroupValuePullsItem{bucketstest.Pull(head, []string{"lemurs"}, + map[string]meshcatalog.CandidatesByGroupValuePullsItemFpsValue{"lemurs": {Fp2: fpNew, Manifest: digest2}})} + raw, _ := bucketstest.GroupCandidate(meshcatalog.CandidatesByGroupValue{Group: "fix/1126", Heads: heads, Pulls: pulls, + Fpset: meshcatalogclient.FingerprintSet(pulls)}) + ckey, _ := buckets.CandidateKey("fix/1126") + b.kv[buckets.Candidates+"/"+ckey] = raw env := b.env() var tick atomic.Int64 env.Now = func() time.Time { return time.Unix(0, 0).Add(time.Duration(tick.Add(1)) * time.Second) } @@ -185,17 +188,17 @@ func TestTheNodeComposesACandidateOnAsk(t *testing.T) { s.c.placements["n1"] = meshmoduleassigner.Placement{Node: "n1", Decided: meshmoduleassigner.NodeDecided{Hub: true, Address: "10.77.0.1"}, Modules: []meshmoduleassigner.PlacedModule{{Module: "lemurs"}}} s.c.records["n1"] = nodeengine.NodeRecord{Node: "n1", TunnelKey: "k", SealingKey: "sk", Capabilities: []string{"systemd"}} - s.c.modules["lemurs"] = delivery.Module{FP2: fp, Manifest: digest, Repository: "novox/x", Commit: "c"} - s.c.modules["other"] = delivery.Module{FP2: fp, Manifest: digestOther, Repository: "novox/x", Commit: "c"} + s.c.modules["lemurs"] = meshcatalog.ModulesByModuleValue{Fp2: fp, Manifest: digest, Repository: "novox/x", Commit: "c"} + s.c.modules["other"] = meshcatalog.ModulesByModuleValue{Fp2: fp, Manifest: digestOther, Repository: "novox/x", Commit: "c"} s.ready.Store(true) - key, _ := delivery.CompositionKey("fix/1126", heads) + key, _ := nec.CompositionKey("fix/1126", heads) return s, key } readsOf := func(b *fakeBus, key string) (n int) { b.mu.Lock() defer b.mu.Unlock() for _, r := range b.reads { - if r == [2]string{delivery.ManifestsBucket, key} { + if r == [2]string{buckets.Manifests, key} { n++ } } @@ -235,7 +238,7 @@ func TestTheNodeComposesACandidateOnAsk(t *testing.T) { if tc.stuck { get := s.c.env.Get s.c.env.Get = func(rctx context.Context, bucket, k string) ([]byte, error) { - if bucket == delivery.ManifestsBucket && k == otherKey { + if bucket == buckets.Manifests && k == otherKey { b.mu.Lock() b.reads = append(b.reads, [2]string{bucket, k}) b.mu.Unlock() diff --git a/internal/selfcompose/composedstate.go b/internal/selfcompose/composedstate.go index af2c1f4..5693f90 100644 --- a/internal/selfcompose/composedstate.go +++ b/internal/selfcompose/composedstate.go @@ -90,7 +90,7 @@ func (c *Composer) newestFPs(ctx context.Context, at *published) map[string]stri r := reader{c: c} for _, module := range at.placed { c.mu.Lock() - fp2 := c.modules[module].FP2 + fp2 := c.modules[module].Fp2 c.mu.Unlock() if fp2 != "" { if b, err := r.Build(ctx, module, fp2); err == nil && b != nil && b.State == delivery.BuildDone { @@ -184,7 +184,7 @@ func (c *Composer) composedValues(ctx context.Context, at *published) (map[strin var inOwn []string c.mu.Lock() for module, fp := range newest { - if fp != c.modules[module].FP2 { + if fp != c.modules[module].Fp2 { scope.KeptAt[module] = fp } } diff --git a/internal/selfcompose/composedstate_test.go b/internal/selfcompose/composedstate_test.go index d971e0e..4cf6e82 100644 --- a/internal/selfcompose/composedstate_test.go +++ b/internal/selfcompose/composedstate_test.go @@ -7,6 +7,8 @@ import ( "git.novox.be/novox/mesh-contracts/placement/go/clients/meshmoduleassignerclient" "git.novox.be/novox/mesh-contracts/placement/go/seats/meshmoduleassigner" "git.novox.be/novox/mesh-contracts/placement/go/seats/meshwants" + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" "github.com/novox/mesh-host/internal/buckets" "github.com/novox/mesh-host/internal/buckets/bucketstest" engcomp "github.com/novox/mesh-host/internal/compositions" @@ -70,15 +72,15 @@ func (f *fakeComposed) written() []string { // by a want, and deleted otherwise. func TestTheNodePublishesItsCompositions(t *testing.T) { b := newFakeBus() - fp := delivery.SourcePrefix2 + strings.Repeat("a", 64) - ofp := delivery.SourcePrefix2 + strings.Repeat("c", 64) - digest := delivery.ManifestDigest([]byte(lemurs)) + fp := meshcatalogclient.SourcePrefix2 + strings.Repeat("a", 64) + ofp := meshcatalogclient.SourcePrefix2 + strings.Repeat("c", 64) + digest := meshcatalogclient.ManifestDigest([]byte(lemurs)) oldie := `{"module":"oldie","version":"1","capabilities":["systemd"],"resources":[{"id":"service","type":"service","unit":"oldie.service","state":"running","boot":"enabled"}]}` - odigest := delivery.ManifestDigest([]byte(oldie)) + odigest := meshcatalogclient.ManifestDigest([]byte(oldie)) for d, json := range map[string]string{digest: lemurs, odigest: oldie} { - man, _ := delivery.EncodeManifest(delivery.Manifest{Module: map[string]string{digest: "lemurs", odigest: "oldie"}[d], JSON: json}) - key, _ := delivery.ManifestsKey(d) - b.kv[delivery.ManifestsBucket+"/"+key] = man + man, _ := meshcatalogclient.EncodeManifest(meshcatalog.ManifestsByDigestValue{Module: map[string]string{digest: "lemurs", odigest: "oldie"}[d], Manifest: json}) + key, _ := meshcatalogclient.ManifestsKey(d) + b.kv[buckets.Manifests+"/"+key] = man } state := &fakeComposed{kv: map[string]ComposedEntry{}} env := b.env() @@ -93,17 +95,17 @@ func TestTheNodePublishesItsCompositions(t *testing.T) { place, _ := meshmoduleassignerclient.EncodePlacement(meshmoduleassigner.Placement{Node: "n1", Decided: meshmoduleassigner.NodeDecided{Hub: true, Address: "10.77.0.1"}, Modules: []meshmoduleassigner.PlacedModule{{Module: "lemurs"}, {Module: "oldie"}}}) have, _ := nodeengineclient.EncodeHaveNode(nodeengine.HaveByNodeNodeValue{Record: &nodeengine.NodeRecord{Node: "n1", TunnelKey: "k", SealingKey: "sk", Capabilities: []string{"systemd"}}}) - mod, _ := delivery.EncodeModule(delivery.Module{FP2: fp, Manifest: digest, Repository: "novox/x", Commit: "c"}) - omod, _ := delivery.EncodeModule(delivery.Module{FP2: ofp, Manifest: odigest, Repository: "novox/o", Commit: "c"}) + mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fp, Manifest: digest, Repository: "novox/x", Commit: "c"}) + omod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: ofp, Manifest: odigest, Repository: "novox/o", Commit: "c"}) want, _ := bucketstest.Want(meshwants.WantsByWantNodeModuleValue{Fp: fp, Self: true}) nodeWant, _ := bucketstest.NodeWant(meshwants.WantsByWantNodeNodeValue{Self: true}) b.put(t, buckets.Placement, "placement.n1", place) b.put(t, buckets.Have, "n1.node", have) - b.put(t, delivery.ModulesBucket, "lemurs", mod) - b.put(t, delivery.ModulesBucket, "oldie", omod) + b.put(t, buckets.Modules, "lemurs", mod) + b.put(t, buckets.Modules, "oldie", omod) b.put(t, buckets.Wants, "want.n1.lemurs", want) b.put(t, buckets.Wants, "want.n1.node", nodeWant) - for _, bucket := range []string{buckets.Wants, buckets.Placement, buckets.Have, delivery.ModulesBucket, ResolvedBucket, delivery.BuilderBuildsBucket} { + for _, bucket := range []string{buckets.Wants, buckets.Placement, buckets.Have, buckets.Modules, ResolvedBucket, delivery.BuilderBuildsBucket} { b.send(bucket, agent.Update{Caught: true}) } for u := next(t, out); !u.Caught; u = next(t, out) { @@ -166,7 +168,7 @@ func TestTheNodePublishesItsCompositions(t *testing.T) { keep.pubKept = map[string]ComposedEntry{} var keys []string for i, ch := range "123456" { - key, _ := engcomp.Key("n1", "lemurs", delivery.SourcePrefix2+strings.Repeat(string(ch), 64)) + key, _ := engcomp.Key("n1", "lemurs", meshcatalogclient.SourcePrefix2+strings.Repeat(string(ch), 64)) keys = append(keys, key) keep.pubKept[key] = ComposedEntry{Revision: uint64(10 + i)} } @@ -174,7 +176,7 @@ func TestTheNodePublishesItsCompositions(t *testing.T) { if past := keep.pastKeeping(); !slices.Equal(past, []string{keys[0]}) { t.Errorf("past keeping: %v, want the oldest of six, %s", past, keys[0]) } - keep.wants = map[string]meshwants.WantsByWantNodeModuleValue{"lemurs": {Fp: delivery.SourcePrefix2 + strings.Repeat("1", 64)}} + keep.wants = map[string]meshwants.WantsByWantNodeModuleValue{"lemurs": {Fp: meshcatalogclient.SourcePrefix2 + strings.Repeat("1", 64)}} if past := keep.pastKeeping(); len(past) != 0 { t.Errorf("past keeping with a want naming the oldest: %v, want none", past) } diff --git a/internal/selfcompose/data_test.go b/internal/selfcompose/data_test.go index dc32133..d2273d3 100644 --- a/internal/selfcompose/data_test.go +++ b/internal/selfcompose/data_test.go @@ -5,6 +5,7 @@ import ( "errors" "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" "github.com/novox/mesh-host/internal/buckets" engcomp "github.com/novox/mesh-host/internal/compositions" "slices" @@ -93,7 +94,7 @@ func TestTheEngineKeepsEachPlacedModulesDataItems(t *testing.T) { t.Error("the node's grant reads other keys of the engine's state data than its own") } c := New(env) - webKey, _ := engcomp.Key("n1", "web", delivery.SourcePrefix2+strings.Repeat("a", 64)) + webKey, _ := engcomp.Key("n1", "web", meshcatalogclient.SourcePrefix2+strings.Repeat("a", 64)) nodeKey, _ := engcomp.NodeKey("n1") for _, step := range []struct { name string @@ -234,9 +235,9 @@ func TestTheEngineKeepsEachPlacedModulesDataItems(t *testing.T) { // The items are worked out from what the pass read: a done build is read over the bus once, not again for the data // items after the pass (a module in a running build is read again, since it may land). b := newFakeBus() - done, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, Manifest: delivery.ManifestDigest([]byte(lemurs))}) + done, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, Manifest: meshcatalogclient.ManifestDigest([]byte(lemurs))}) running, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: "building"}) - fp := delivery.SourcePrefix2 + strings.Repeat("b", 64) + fp := meshcatalogclient.SourcePrefix2 + strings.Repeat("b", 64) doneKey, _ := delivery.BuilderBuildKey("web", fp) runningKey, _ := delivery.BuilderBuildKey("pg", fp) b.kv[delivery.BuilderBuildsBucket+"/"+doneKey], b.kv[delivery.BuilderBuildsBucket+"/"+runningKey] = done, running @@ -257,12 +258,12 @@ func TestTheEngineKeepsEachPlacedModulesDataItems(t *testing.T) { t.Errorf("the builds were read %v times over the bus; want the done one once, the running one each time", reads) } // A done key the builder claims again (a gone step, issue 1937) and lands done again is read again, not served kept. - again, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, Manifest: delivery.ManifestDigest([]byte("{}"))}) + again, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, Manifest: meshcatalogclient.ManifestDigest([]byte("{}"))}) b.mu.Lock() b.kv[delivery.BuilderBuildsBucket+"/"+doneKey] = again b.mu.Unlock() r.c.takeBuildDone(agent.Update{Key: doneKey, Value: running}) - if got, err := r.Build(context.Background(), "web", fp); err != nil || got.Manifest != delivery.ManifestDigest([]byte("{}")) { + if got, err := r.Build(context.Background(), "web", fp); err != nil || got.Manifest != meshcatalogclient.ManifestDigest([]byte("{}")) { t.Errorf("a done build written again is served as %+v (%v), not read again", got, err) } } diff --git a/internal/selfcompose/selfcompose.go b/internal/selfcompose/selfcompose.go index 3f39ef9..ce02dfd 100644 --- a/internal/selfcompose/selfcompose.go +++ b/internal/selfcompose/selfcompose.go @@ -24,6 +24,8 @@ import ( "git.novox.be/novox/mesh-contracts/placement/go/clients/meshmoduleassignerclient" "git.novox.be/novox/mesh-contracts/placement/go/seats/meshmoduleassigner" "git.novox.be/novox/mesh-contracts/placement/go/seats/meshwants" + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" "github.com/novox/mesh-host/internal/buckets" "github.com/novox/mesh-host/internal/compositions" engcomp "github.com/novox/mesh-host/internal/compositions" @@ -141,7 +143,7 @@ type Composer struct { // inOwn is, by module, whether this node's have of it names no fingerprint of the new form (mesh-wireguard, a module // at a build of the old form): the node runs it inside its own composition, and composes it there again (issue 1126). inOwn map[string]bool - modules map[string]delivery.Module + modules map[string]meshcatalog.ModulesByModuleValue told map[string]bool // lines said once each (sayOnceEach) values map[string]map[string]any // by module, this node's manifests map[string][]byte // by digest, read once @@ -304,7 +306,7 @@ func New(env Env) *Composer { env.Say = func(string) {} } return &Composer{env: env, wants: map[string]meshwants.WantsByWantNodeModuleValue{}, placements: map[string]meshmoduleassigner.Placement{}, placementKeys: meshmoduleassignerclient.NewKeys(), - records: map[string]nodeengine.NodeRecord{}, inOwn: map[string]bool{}, modules: map[string]delivery.Module{}, values: map[string]map[string]any{}, + records: map[string]nodeengine.NodeRecord{}, inOwn: map[string]bool{}, modules: map[string]meshcatalog.ModulesByModuleValue{}, values: map[string]map[string]any{}, manifests: map[string][]byte{}, archives: map[string][]byte{}, builds: map[string]delivery.BuilderBuild{}, handed: map[string]string{}, sealedOwn: map[string]ownSealed{}, pkgs: map[string]compositions.Composed{}, asked: make(chan chan Pass, 8)} } @@ -365,7 +367,7 @@ func (c *Composer) inputs() []input { // A watch of every key is refused by the bus (issue 1126, 2026-10-27 09:21: the fold's `>` stopped every agent). {buckets.Have, "*." + nodeengineclient.NodeWord, nil, (*Composer).takeRecordChanged}, {buckets.Have, node + ".>", nil, (*Composer).takeModuleHave}, - {delivery.ModulesBucket, ">", (*Composer).takeModule, nil}, + {buckets.Modules, ">", (*Composer).takeModule, nil}, {ResolvedBucket, node + ".>", (*Composer).takeValue, nil}, // The builder's builds: a pass that held a module for a build not yet done is made again when it lands (issue // 774, 2026-10-30 19:30: node-tools' build landed and ace, g14 and shanks stayed refused until another input moved). @@ -398,7 +400,7 @@ func (c *Composer) takeBuildDone(u agent.Update) bool { c.mu.Unlock() // A module placed here is composed at its trunk's build once that is done, for the composition state, before its // want moves (ADR 0593 §6). - if !(wanted && w.Fp == fp) && !((wanted || placedHere) && m.FP2 == fp) { + if !(wanted && w.Fp == fp) && !((wanted || placedHere) && m.Fp2 == fp) { return false } b, err := delivery.DecodeBuilderBuild(u.Value) @@ -628,8 +630,8 @@ func (c *Composer) entry(bucket, key string) (v any, has, known bool) { } p, ok := c.placements[node] return p, ok, true - case delivery.ModulesBucket: - name, err := delivery.ParseModulesKey(key) + case buckets.Modules: + name, err := meshcatalogclient.ParseModulesKey(key) if err != nil { return nil, false, true } @@ -737,7 +739,7 @@ func (c *Composer) takeModuleHave(u agent.Update) bool { // (waiting for a yes inside the node's own stays in). interrupted := h.Phase == nodeengineclient.PhaseHeld && h.Detail == agent.BackupInterrupted runsNothing := (h.Phase == nodeengineclient.PhaseFailed || interrupted) && h.Fp == "" && h.Composed == "" - in = h.Phase != nodeengineclient.PhaseRemoved && !runsNothing && !strings.HasPrefix(h.Fp, delivery.SourcePrefix2) + in = h.Phase != nodeengineclient.PhaseRemoved && !runsNothing && !strings.HasPrefix(h.Fp, meshcatalogclient.SourcePrefix2) } } c.mu.Lock() @@ -784,13 +786,13 @@ func (c *Composer) inOwnOf(p meshmoduleassigner.Placement) []string { } func (c *Composer) takeModule(u agent.Update) { - name, err := delivery.ParseModulesKey(u.Key) + name, err := meshcatalogclient.ParseModulesKey(u.Key) if err != nil { return } c.mu.Lock() defer c.mu.Unlock() - m, err := delivery.DecodeModule(u.Value) + m, err := meshcatalogclient.DecodeModule(u.Value) if u.Deleted || err != nil { delete(c.modules, name) return @@ -863,7 +865,7 @@ func (c *Composer) compose(ctx context.Context) []agent.Update { continue } fps[module] = w.Fp - if m, ok := c.modules[module]; ok && m.FP2 != w.Fp { + if m, ok := c.modules[module]; ok && m.Fp2 != w.Fp { behind = append(behind, module) } } @@ -1263,10 +1265,10 @@ func (c *Composer) HoldsData(ctx context.Context, module string) (data, known bo // the indexer's verdict from the module's value (a missing one is accepted). held is why the module is held, "" when it is // not: a manifest the indexer refused, or one not read past a field this engine does not know. A field set aside is said // once, and an error is returned only for bytes that are no manifest, which the caller says in its own words. -func (c *Composer) taken(module string, m delivery.Module, raw []byte) (man catalogue.Manifest, held string, err error) { - if v := m.VerdictOf(module); v != nil { - if refused, _ := delivery.TakeManifestVerdict(v); refused { - return catalogue.Manifest{}, v.Why(), nil +func (c *Composer) taken(module string, m meshcatalog.ModulesByModuleValue, raw []byte) (man catalogue.Manifest, held string, err error) { + if v := meshcatalogclient.ModuleVerdict(module, m); v != nil { + if refused, _ := meshcatalogclient.TakeVerdict(v); refused { + return catalogue.Manifest{}, meshcatalogclient.VerdictWhy(*v), nil } } man, err = catalogue.Decode(raw) @@ -1390,10 +1392,10 @@ func (r reader) Records(context.Context) (map[string]nodeengine.NodeRecord, erro return out, nil } -func (r reader) Modules(context.Context) (map[string]delivery.Module, error) { +func (r reader) Modules(context.Context) (map[string]meshcatalog.ModulesByModuleValue, error) { r.c.mu.Lock() defer r.c.mu.Unlock() - out := make(map[string]delivery.Module, len(r.c.modules)) + out := make(map[string]meshcatalog.ModulesByModuleValue, len(r.c.modules)) for k, v := range r.c.modules { out[k] = v } @@ -1407,24 +1409,24 @@ func (r reader) Manifest(ctx context.Context, digest string) ([]byte, error) { if ok { return raw, nil } - key, err := delivery.ManifestsKey(digest) + key, err := meshcatalogclient.ManifestsKey(digest) if err != nil { return nil, err } - defer r.c.waiting("the manifest " + digest + " (" + delivery.ManifestsBucket + ")")() - value, err := r.c.get(ctx, delivery.ManifestsBucket, key) + defer r.c.waiting("the manifest " + digest + " (" + buckets.Manifests + ")")() + value, err := r.c.get(ctx, buckets.Manifests, key) if err != nil { return nil, err } if value == nil { return nil, fmt.Errorf("the indexer holds no manifest %s", digest) } - m, err := delivery.DecodeManifest(value) + m, err := meshcatalogclient.DecodeManifest(value) if err != nil { return nil, err } - r.c.keepRead(func(k *Composer) { k.manifests[digest] = []byte(m.JSON) }) - return []byte(m.JSON), nil + r.c.keepRead(func(k *Composer) { k.manifests[digest] = []byte(m.Manifest) }) + return []byte(m.Manifest), nil } func (r reader) Build(ctx context.Context, module, fp string) (*delivery.BuilderBuild, error) { @@ -1723,13 +1725,13 @@ func (c *Composer) ComposedToday(ctx context.Context, module, fp string) (string continue } fps[m] = w.Fp - if mm, ok := c.modules[m]; ok && mm.FP2 != w.Fp { + if mm, ok := c.modules[m]; ok && mm.Fp2 != w.Fp { kept[m] = w.Fp } } fps[module] = fp delete(kept, module) - if mm, ok := c.modules[module]; !ok || mm.FP2 != fp { + if mm, ok := c.modules[module]; !ok || mm.Fp2 != fp { kept[module] = fp } own, placed := c.placements[c.env.Node] diff --git a/internal/selfcompose/selfcompose_test.go b/internal/selfcompose/selfcompose_test.go index d029b67..dcbaa2c 100644 --- a/internal/selfcompose/selfcompose_test.go +++ b/internal/selfcompose/selfcompose_test.go @@ -13,6 +13,8 @@ import ( "git.novox.be/novox/mesh-contracts/placement/go/clients/meshmoduleassignerclient" "git.novox.be/novox/mesh-contracts/placement/go/seats/meshmoduleassigner" "git.novox.be/novox/mesh-contracts/placement/go/seats/meshwants" + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" "github.com/novox/mesh-host/internal/buckets" "github.com/novox/mesh-host/internal/buckets/bucketstest" "github.com/novox/mesh-host/internal/compositions" @@ -151,11 +153,11 @@ func quiet(t *testing.T, out <-chan agent.Update) { // names any more is taken back; a node that cannot compose says so once and hands nothing. func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { b := newFakeBus() - fp := delivery.SourcePrefix2 + strings.Repeat("a", 64) - digest := delivery.ManifestDigest([]byte(lemurs)) - man, _ := delivery.EncodeManifest(delivery.Manifest{Module: "lemurs", JSON: lemurs}) - key, _ := delivery.ManifestsKey(digest) - b.kv[delivery.ManifestsBucket+"/"+key] = man + fp := meshcatalogclient.SourcePrefix2 + strings.Repeat("a", 64) + digest := meshcatalogclient.ManifestDigest([]byte(lemurs)) + man, _ := meshcatalogclient.EncodeManifest(meshcatalog.ManifestsByDigestValue{Module: "lemurs", Manifest: lemurs}) + key, _ := meshcatalogclient.ManifestsKey(digest) + b.kv[buckets.Manifests+"/"+key] = man ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -188,26 +190,26 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { // oldie is placed and no want names it (a module at a build of the old form, nats on novox): the node runs it inside // its own composition, and composes it there again, even before the agent has applied anything (Running empty). oldie := `{"module":"oldie","version":"1","capabilities":["systemd"],"resources":[{"id":"service","type":"service","unit":"oldie.service","state":"running","boot":"enabled"}]}` - odigest := delivery.ManifestDigest([]byte(oldie)) - oman, _ := delivery.EncodeManifest(delivery.Manifest{Module: "oldie", JSON: oldie}) - okey, _ := delivery.ManifestsKey(odigest) - b.kv[delivery.ManifestsBucket+"/"+okey] = oman - omod, _ := delivery.EncodeModule(delivery.Module{FP2: delivery.SourcePrefix2 + strings.Repeat("c", 64), Manifest: odigest, Repository: "novox/o", Commit: "c"}) + odigest := meshcatalogclient.ManifestDigest([]byte(oldie)) + oman, _ := meshcatalogclient.EncodeManifest(meshcatalog.ManifestsByDigestValue{Module: "oldie", Manifest: oldie}) + okey, _ := meshcatalogclient.ManifestsKey(odigest) + b.kv[buckets.Manifests+"/"+okey] = oman + omod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: meshcatalogclient.SourcePrefix2 + strings.Repeat("c", 64), Manifest: odigest, Repository: "novox/o", Commit: "c"}) place, _ := meshmoduleassignerclient.EncodePlacement(meshmoduleassigner.Placement{Node: "n1", Decided: meshmoduleassigner.NodeDecided{Hub: true, Address: "10.77.0.1"}, Modules: []meshmoduleassigner.PlacedModule{{Module: "lemurs"}, {Module: "oldie"}}}) have, _ := nodeengineclient.EncodeHaveNode(nodeengine.HaveByNodeNodeValue{Record: &nodeengine.NodeRecord{Node: "n1", TunnelKey: "k", SealingKey: "sk", Capabilities: []string{"systemd"}}}) - mod, _ := delivery.EncodeModule(delivery.Module{FP2: fp, Manifest: digest, Repository: "novox/x", Commit: "c"}) + mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fp, Manifest: digest, Repository: "novox/x", Commit: "c"}) want, _ := bucketstest.Want(meshwants.WantsByWantNodeModuleValue{Fp: fp, Self: true}) nodeWant, _ := bucketstest.NodeWant(meshwants.WantsByWantNodeNodeValue{Self: true}) b.put(t, buckets.Placement, "placement.n1", place) b.put(t, buckets.Have, "n1.node", have) oldHave, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Build: "build-1", Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}) b.put(t, buckets.Have, "n1.oldie", oldHave) - b.put(t, delivery.ModulesBucket, "lemurs", mod) - b.put(t, delivery.ModulesBucket, "oldie", omod) + b.put(t, buckets.Modules, "lemurs", mod) + b.put(t, buckets.Modules, "oldie", omod) b.put(t, buckets.Wants, "want.n1.lemurs", want) b.put(t, buckets.Wants, "want.n1.node", nodeWant) - for _, bucket := range []string{buckets.Wants, buckets.Placement, buckets.Have, delivery.ModulesBucket, ResolvedBucket, delivery.BuilderBuildsBucket} { + for _, bucket := range []string{buckets.Wants, buckets.Placement, buckets.Have, buckets.Modules, ResolvedBucket, delivery.BuilderBuildsBucket} { b.send(bucket, agent.Update{Caught: true}) } got := map[string]bool{} @@ -285,17 +287,17 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { if dig, ok := c.ComposedToday(ctx, "lemurs", fp); !ok || dig != handedDig { t.Errorf("composed today at the build it runs: %s %v, want %s", dig, ok, handedDig) } - if dig, ok := c.ComposedToday(ctx, "lemurs", delivery.SourcePrefix2+strings.Repeat("9", 64)); ok { + if dig, ok := c.ComposedToday(ctx, "lemurs", meshcatalogclient.SourcePrefix2+strings.Repeat("9", 64)); ok { t.Errorf("a build that cannot be composed was composed today: %s", dig) } chgJSON := strings.Replace(lemurs, `"state":"running"`, `"state":"stopped"`, 1) - chgDigest := delivery.ManifestDigest([]byte(chgJSON)) - chgMan, _ := delivery.EncodeManifest(delivery.Manifest{Module: "lemurs", JSON: chgJSON}) - chgKey, _ := delivery.ManifestsKey(chgDigest) + chgDigest := meshcatalogclient.ManifestDigest([]byte(chgJSON)) + chgMan, _ := meshcatalogclient.EncodeManifest(meshcatalog.ManifestsByDigestValue{Module: "lemurs", Manifest: chgJSON}) + chgKey, _ := meshcatalogclient.ManifestsKey(chgDigest) b.mu.Lock() - b.kv[delivery.ManifestsBucket+"/"+chgKey] = chgMan + b.kv[buckets.Manifests+"/"+chgKey] = chgMan b.mu.Unlock() - chgMod, _ := delivery.EncodeModule(delivery.Module{FP2: fp, Manifest: chgDigest, Repository: "novox/x", Commit: "c"}) + chgMod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fp, Manifest: chgDigest, Repository: "novox/x", Commit: "c"}) c.takeModule(agent.Update{Key: "lemurs", Value: chgMod}) if dig, ok := c.ComposedToday(ctx, "lemurs", fp); !ok || dig == handedDig { t.Errorf("composed today with a changed input: %s %v, want another digest than %s", dig, ok, handedDig) @@ -427,14 +429,14 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { `{"id":"pkg","type":"package","package":"haproxy"}]}` c3, told := fresh(env.Get) for m, raw := range map[string]string{"de-spiegel": string(spiegel), "proxy": proxy} { - dig := delivery.ManifestDigest([]byte(raw)) - enc, _ := delivery.EncodeManifest(delivery.Manifest{Module: m, JSON: raw}) - k, _ := delivery.ManifestsKey(dig) + dig := meshcatalogclient.ManifestDigest([]byte(raw)) + enc, _ := meshcatalogclient.EncodeManifest(meshcatalog.ManifestsByDigestValue{Module: m, Manifest: raw}) + k, _ := meshcatalogclient.ManifestsKey(dig) b.mu.Lock() - b.kv[delivery.ManifestsBucket+"/"+k] = enc + b.kv[buckets.Manifests+"/"+k] = enc b.mu.Unlock() - mfp := delivery.SourcePrefix2 + strings.Repeat(map[string]string{"de-spiegel": "d", "proxy": "e"}[m], 64) - c3.modules[m] = delivery.Module{FP2: mfp, Manifest: dig, Repository: "novox/" + m, Commit: "c"} + mfp := meshcatalogclient.SourcePrefix2 + strings.Repeat(map[string]string{"de-spiegel": "d", "proxy": "e"}[m], 64) + c3.modules[m] = meshcatalog.ModulesByModuleValue{Fp2: mfp, Manifest: dig, Repository: "novox/" + m, Commit: "c"} c3.wants[m] = meshwants.WantsByWantNodeModuleValue{Fp: mfp, Self: true} } p3 := c3.placements["n1"] @@ -480,16 +482,16 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { t.Fatal(err) } c4, told4 := fresh(env.Get) - powerDig := delivery.ManifestDigest(power) - powerEnc, _ := delivery.EncodeManifest(delivery.Manifest{Module: "power", JSON: string(power)}) - powerKey, _ := delivery.ManifestsKey(powerDig) + powerDig := meshcatalogclient.ManifestDigest(power) + powerEnc, _ := meshcatalogclient.EncodeManifest(meshcatalog.ManifestsByDigestValue{Module: "power", Manifest: string(power)}) + powerKey, _ := meshcatalogclient.ManifestsKey(powerDig) b.mu.Lock() - b.kv[delivery.ManifestsBucket+"/"+powerKey] = powerEnc + b.kv[buckets.Manifests+"/"+powerKey] = powerEnc b.mu.Unlock() - powerFP := delivery.SourcePrefix2 + strings.Repeat("c", 64) - c4.modules["power"] = delivery.Module{FP2: powerFP, Manifest: powerDig, Repository: "novox/mesh-catalog", Commit: "ca8d6f33", + powerFP := meshcatalogclient.SourcePrefix2 + strings.Repeat("c", 64) + c4.modules["power"] = meshcatalog.ModulesByModuleValue{Fp2: powerFP, Manifest: powerDig, Repository: "novox/mesh-catalog", Commit: "ca8d6f33", // The indexer's verdict on the captured manifest (ADR 0578): the engine judges nothing, it takes this. - Verdict: delivery.VerdictRefused, Problems: []string{`bar item "shows" is none of them`}} + Verdict: meshcatalogclient.VerdictRefused, Problems: []string{`bar item "shows" is none of them`}} c4.wants["power"] = meshwants.WantsByWantNodeModuleValue{Fp: powerFP, Self: true} p4 := c4.placements["n1"] p4.Modules = append(append([]meshmoduleassigner.PlacedModule{}, p4.Modules...), meshmoduleassigner.PlacedModule{Module: "power"}) @@ -505,7 +507,7 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { t.Errorf("the agent was told %q, want power held alone, naming its item", *told4) } // Review of issue 774 (the kept read): mesh-wireguard, which the mesh provides and which never has a fingerprint, is - // folded in the node's own and composed from the manifest mesh-sdk provides (delivery.ProvidedManifest), not from + // folded in the node's own and composed from the manifest mesh-sdk provides (overlay.ProvidedManifest), not from // what runs nor the kept declaration: with nothing running (the kept read retired, Running empty) the node's own // still carries its resources, so retiring the read removes nothing of it. c5, told5 := fresh(env.Get) @@ -569,9 +571,9 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { }{ {"lemurs's wanted build, not done", "lemurs", fp, "building", false}, {"lemurs's wanted build lands", "lemurs", fp, delivery.BuildDone, true}, - {"another build of lemurs", "lemurs", delivery.SourcePrefix2 + strings.Repeat("e", 64), delivery.BuildDone, false}, - {"a placed module no want names, at its trunk's build", "oldie", delivery.SourcePrefix2 + strings.Repeat("c", 64), delivery.BuildDone, true}, - {"a module neither wanted nor placed here", "ghost", delivery.SourcePrefix2 + strings.Repeat("c", 64), delivery.BuildDone, false}, + {"another build of lemurs", "lemurs", meshcatalogclient.SourcePrefix2 + strings.Repeat("e", 64), delivery.BuildDone, false}, + {"a placed module no want names, at its trunk's build", "oldie", meshcatalogclient.SourcePrefix2 + strings.Repeat("c", 64), delivery.BuildDone, true}, + {"a module neither wanted nor placed here", "ghost", meshcatalogclient.SourcePrefix2 + strings.Repeat("c", 64), delivery.BuildDone, false}, } { before := passes() key, _ := delivery.BuilderBuildKey(row.module, row.fp) @@ -592,7 +594,7 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { // Issue 774, 2026-10-31: every put under want.n1.> made a pass, and each pass asked the vault for the node's own // secrets. A key written again with what it held, or a want whose words moved and not its build, makes none. worded, _ := bucketstest.Want(meshwants.WantsByWantNodeModuleValue{Fp: fp, Self: true, Words: "lemurs is held at aaaaaaaaaaaa"}) - elsewhere, _ := bucketstest.Want(meshwants.WantsByWantNodeModuleValue{Fp: delivery.SourcePrefix2 + strings.Repeat("e", 64), Self: true}) + elsewhere, _ := bucketstest.Want(meshwants.WantsByWantNodeModuleValue{Fp: meshcatalogclient.SourcePrefix2 + strings.Repeat("e", 64), Self: true}) // Research 072 row 37: the placement converted to one key per module and node moves nothing composed; a write in // keys makes its pass at its node key, the commit, and not at a module key written before it nor at the mirror. placeBase := meshmoduleassigner.Placement{Node: "n1", Decided: meshmoduleassigner.NodeDecided{Hub: true, Address: "10.77.0.1"}, @@ -645,7 +647,7 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { {"the node's own want again", buckets.Wants, "want.n1.node", nodeWant, false}, {"lemurs's want with other words", buckets.Wants, "want.n1.lemurs", worded, false}, {"the placement again", buckets.Placement, "placement.n1", place, false}, - {"lemurs's module again", delivery.ModulesBucket, "lemurs", mod, false}, + {"lemurs's module again", buckets.Modules, "lemurs", mod, false}, {"lemurs's want at another build", buckets.Wants, "want.n1.lemurs", elsewhere, true}, {"lemurs's want back at its build", buckets.Wants, "want.n1.lemurs", want, true}, }, placementRows...) { @@ -672,11 +674,11 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { tb := newFakeBus() tool := `{"module":"tool","version":"1","capabilities":["systemd"],"build":{"artifacts":[{"name":"files","kind":"archive","from":"files"}]},` + `"resources":[{"id":"service","type":"service","unit":"tool.service","state":"running","boot":"enabled"}]}` - tdigest := delivery.ManifestDigest([]byte(tool)) - tman, _ := delivery.EncodeManifest(delivery.Manifest{Module: "tool", JSON: tool}) - tkey, _ := delivery.ManifestsKey(tdigest) - tb.kv[delivery.ManifestsBucket+"/"+tkey] = tman - old, next2 := delivery.SourcePrefix2+strings.Repeat("1", 64), delivery.SourcePrefix2+strings.Repeat("2", 64) + tdigest := meshcatalogclient.ManifestDigest([]byte(tool)) + tman, _ := meshcatalogclient.EncodeManifest(meshcatalog.ManifestsByDigestValue{Module: "tool", Manifest: tool}) + tkey, _ := meshcatalogclient.ManifestsKey(tdigest) + tb.kv[buckets.Manifests+"/"+tkey] = tman + old, next2 := meshcatalogclient.SourcePrefix2+strings.Repeat("1", 64), meshcatalogclient.SourcePrefix2+strings.Repeat("2", 64) tenv := tb.env() var held []string var hmu sync.Mutex @@ -696,16 +698,16 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { } tplace, _ := meshmoduleassignerclient.EncodePlacement(meshmoduleassigner.Placement{Node: "n1", Decided: meshmoduleassigner.NodeDecided{Hub: true, Address: "10.77.0.1"}, Modules: []meshmoduleassigner.PlacedModule{{Module: "tool"}}}) - tmod, _ := delivery.EncodeModule(delivery.Module{FP2: next2, Manifest: tdigest, Repository: "novox/t", Commit: "c"}) + tmod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: next2, Manifest: tdigest, Repository: "novox/t", Commit: "c"}) twant, _ := bucketstest.Want(meshwants.WantsByWantNodeModuleValue{Fp: next2, Self: true}) building, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: "building"}) bkey, _ := delivery.BuilderBuildKey("tool", next2) tb.kv[delivery.BuilderBuildsBucket+"/"+bkey] = building tb.put(t, buckets.Placement, "placement.n1", tplace) tb.put(t, buckets.Have, "n1.node", have) - tb.put(t, delivery.ModulesBucket, "tool", tmod) + tb.put(t, buckets.Modules, "tool", tmod) tb.put(t, buckets.Wants, "want.n1.tool", twant) - for _, bucket := range []string{buckets.Wants, buckets.Placement, buckets.Have, delivery.ModulesBucket, + for _, bucket := range []string{buckets.Wants, buckets.Placement, buckets.Have, buckets.Modules, ResolvedBucket, delivery.BuilderBuildsBucket} { tb.send(bucket, agent.Update{Caught: true}) } @@ -740,7 +742,7 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { } // A want of a module whose manifest cannot be read: said once, nothing handed, what was handed stands. - gone := delivery.SourcePrefix2 + strings.Repeat("b", 64) + gone := meshcatalogclient.SourcePrefix2 + strings.Repeat("b", 64) missing, _ := bucketstest.Want(meshwants.WantsByWantNodeModuleValue{Fp: gone, Self: true}) b.put(t, buckets.Wants, "want.n1.lemurs", missing) quiet(t, out) @@ -766,16 +768,16 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { } keeper := `{"module":"keeper","version":"1","own-secrets":{"password":"/var/lib/mesh/keeper/password"},` + `"resources":[{"id":"state","type":"directory","path":"/var/lib/mesh/keeper","mode":"0700"}]}` - kdigest := delivery.ManifestDigest([]byte(keeper)) - kman, _ := delivery.EncodeManifest(delivery.Manifest{Module: "keeper", JSON: keeper}) - kkey, _ := delivery.ManifestsKey(kdigest) + kdigest := meshcatalogclient.ManifestDigest([]byte(keeper)) + kman, _ := meshcatalogclient.EncodeManifest(meshcatalog.ManifestsByDigestValue{Module: "keeper", Manifest: keeper}) + kkey, _ := meshcatalogclient.ManifestsKey(kdigest) b.mu.Lock() - b.kv[delivery.ManifestsBucket+"/"+kkey] = kman + b.kv[buckets.Manifests+"/"+kkey] = kman b.mu.Unlock() - kmod, _ := delivery.EncodeModule(delivery.Module{FP2: fp, Manifest: kdigest, Repository: "novox/k", Commit: "c"}) + kmod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fp, Manifest: kdigest, Repository: "novox/k", Commit: "c"}) both, _ := meshmoduleassignerclient.EncodePlacement(meshmoduleassigner.Placement{Node: "n1", Decided: meshmoduleassigner.NodeDecided{Hub: true, Address: "10.77.0.1"}, Modules: []meshmoduleassigner.PlacedModule{{Module: "lemurs"}, {Module: "oldie"}, {Module: "keeper"}}}) - b.put(t, delivery.ModulesBucket, "keeper", kmod) + b.put(t, buckets.Modules, "keeper", kmod) // n1 is held in keys per module and node since the rows above, which a legacy key no longer moves. bothPlaced, _ := meshmoduleassignerclient.DecodePlacement(both) bothWrites, err := placeKeys.Writes("n1", &bothPlaced) @@ -1041,9 +1043,9 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { } sb := newPatternBus() trunk := meshmoduleassigner.Placement{Node: "n1", Decided: meshmoduleassigner.NodeDecided{Hub: true, Address: "10.77.0.1"}, Modules: []meshmoduleassigner.PlacedModule{{Module: "lemurs"}}} - mods := map[string]delivery.Module{"lemurs": {FP2: fp, Manifest: digest, Repository: "novox/x", Commit: "c"}, - "keeper": {FP2: fp, Manifest: kdigest, Repository: "novox/k", Commit: "c"}} - readOnly := &fakeBus{kv: map[string][]byte{delivery.ManifestsBucket + "/" + key: man, delivery.ManifestsBucket + "/" + kkey: kman}} + mods := map[string]meshcatalog.ModulesByModuleValue{"lemurs": {Fp2: fp, Manifest: digest, Repository: "novox/x", Commit: "c"}, + "keeper": {Fp2: fp, Manifest: kdigest, Repository: "novox/k", Commit: "c"}} + readOnly := &fakeBus{kv: map[string][]byte{buckets.Manifests + "/" + key: man, buckets.Manifests + "/" + kkey: kman}} senv := readOnly.env() senv.Watch = sb.watch var ownFails atomic.Bool // the vault's own on a bus that does not answer (2026-10-28, ace's answer to seed-7) @@ -1096,8 +1098,8 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { sb.put(buckets.Placement, "placement.n1", tp, 1) sb.put(buckets.Have, "n1.node", rec, 1) for name, m := range mods { - raw, _ := delivery.EncodeModule(m) - sb.put(delivery.ModulesBucket, name, raw, 1) + raw, _ := meshcatalogclient.EncodeModule(m) + sb.put(buckets.Modules, name, raw, 1) } wantLemurs := meshwants.WantsByWantNodeModuleKey("n1", "lemurs") var lemursDigest string @@ -1252,8 +1254,8 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { lemursHave("its tool lemurs_health: no answer within 10s", 11) lemursHave("its tool lemurs_health: no answer within 20s", 12) sb.put(buckets.Placement, "placement.n1", tp, 2) - lraw, _ := delivery.EncodeModule(mods["lemurs"]) - sb.put(delivery.ModulesBucket, "lemurs", lraw, 2) + lraw, _ := meshcatalogclient.EncodeModule(mods["lemurs"]) + sb.put(buckets.Modules, "lemurs", lraw, 2) drain() if got := answering(); got != settled { t.Errorf("answered %d more time(s) with no candidate revision and no input of the node changed", got-settled) @@ -1311,7 +1313,7 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { // Review of L2b, round 2: a watch ends while the link stays; a group's key is deleted before the shadow watches // again. The new run reads afresh, so that group is no longer answered. sb.missing(buckets.PlacementCandidates, true) - sb.end(delivery.ModulesBucket) + sb.end(buckets.Modules) dropsKey := meshmoduleassigner.CandidatesByProposalNodeKey("drops", "n1") sb.del(buckets.PlacementCandidates, dropsKey) time.Sleep(30 * time.Millisecond) @@ -1358,8 +1360,8 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { nb.put(buckets.Placement, "placement.n1", tp, 1) nb.put(buckets.Have, "n1.node", rec, 1) for name, m := range mods { - raw, _ := delivery.EncodeModule(m) - nb.put(delivery.ModulesBucket, name, raw, 1) + raw, _ := meshcatalogclient.EncodeModule(m) + nb.put(buckets.Modules, name, raw, 1) } sameRaw, _ := bucketstest.Candidate(meshmoduleassigner.CandidatesByProposalNodeValue{Proposal: "same", Placement: &trunk}) sameKey := meshmoduleassigner.CandidatesByProposalNodeKey("same", "n1") @@ -1368,7 +1370,7 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { oldRaw, _ := bucketstest.Candidate(meshmoduleassigner.CandidatesByProposalNodeValue{Proposal: "older", Placement: &trunk}) oldKey := meshmoduleassigner.CandidatesByProposalNodeKey("older", "n1") nb.put(buckets.PlacementCandidates, oldKey, oldRaw, 2) - for _, bucket := range []string{buckets.PlacementCandidates, buckets.Placement, buckets.Have, delivery.ModulesBucket, delivery.BuilderBuildsBucket} { + for _, bucket := range []string{buckets.PlacementCandidates, buckets.Placement, buckets.Have, buckets.Modules, delivery.BuilderBuildsBucket} { nb.send(bucket, agent.Update{Caught: true}) } select { @@ -1451,7 +1453,7 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { inOwnRunning := []compositions.Composed{{Node: "n1", Module: "lemurs", FP: fp}} other := "src2:" + strings.Repeat("9", 64) gotFps := shadowBuilds(inOwnRunning, meshmoduleassigner.Placement{Node: "n1", Modules: []meshmoduleassigner.PlacedModule{{Module: "lemurs"}, {Module: "nats"}, {Module: "keeper"}}}, - map[string]delivery.Module{"lemurs": {FP2: other}, "nats": {FP2: other}, "keeper": {FP2: other}}, []string{"nats"}) + map[string]meshcatalog.ModulesByModuleValue{"lemurs": {Fp2: other}, "nats": {Fp2: other}, "keeper": {Fp2: other}}, []string{"nats"}) if fmt.Sprint(gotFps) != fmt.Sprint(map[string]string{"lemurs": fp, "keeper": other}) { t.Errorf("shadow builds %v; want lemurs at what runs, keeper at the trunk's, nats at none (folded)", gotFps) } @@ -1510,10 +1512,10 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { certEnv.VaultAuthority = func() string { return "" } cc := New(certEnv) withCert := `{"module":"web","version":"1","certificate":{"into":"/etc/web/tls.crt"}}` - cdigest := delivery.ManifestDigest([]byte(withCert)) + cdigest := meshcatalogclient.ManifestDigest([]byte(withCert)) cc.manifests[digest] = []byte(lemurs) cc.manifests[cdigest] = []byte(withCert) - cc.modules = map[string]delivery.Module{"lemurs": {Manifest: digest}, "web": {Manifest: cdigest}} + cc.modules = map[string]meshcatalog.ModulesByModuleValue{"lemurs": {Manifest: digest}, "web": {Manifest: cdigest}} cc.records["n1"] = nodeengine.NodeRecord{Node: "n1", ServingKey: base64.StdEncoding.EncodeToString(make([]byte, 32))} onlyLemurs := meshmoduleassigner.Placement{Node: "n1", Modules: []meshmoduleassigner.PlacedModule{{Module: "lemurs"}}} cc.certifyIfWanted(context.Background(), reader{c: cc}, onlyLemurs) @@ -1663,7 +1665,7 @@ func (b *patternBus) caught() { type fixedReader struct { placements map[string]meshmoduleassigner.Placement records map[string]nodeengine.NodeRecord - modules map[string]delivery.Module + modules map[string]meshcatalog.ModulesByModuleValue manifests map[string][]byte } @@ -1673,7 +1675,7 @@ func (r fixedReader) Placements(context.Context) (map[string]meshmoduleassigner. func (r fixedReader) Records(context.Context) (map[string]nodeengine.NodeRecord, error) { return r.records, nil } -func (r fixedReader) Modules(context.Context) (map[string]delivery.Module, error) { +func (r fixedReader) Modules(context.Context) (map[string]meshcatalog.ModulesByModuleValue, error) { return r.modules, nil } func (r fixedReader) Manifest(_ context.Context, d string) ([]byte, error) { @@ -1717,7 +1719,7 @@ func TestAHaveSaysWhetherItsModuleRunsInsideTheNodesOwn(t *testing.T) { Health: nodeengineclient.HealthFailed, Detail: "the node cannot compose it, and keeps running what it ran: no build"}, true, true}, // The live have of novox's de-spiegel (2026-10-28): a build of the old form, named by `build`, and no `fp`. {"running a build of the old form", nodeengine.HaveByNodeModuleValue{Build: "build-1790368817085312877", Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}, true, true}, - {"running a build of the new form", nodeengine.HaveByNodeModuleValue{Fp: delivery.SourcePrefix2 + strings.Repeat("a", 64), Composed: "sha256:" + strings.Repeat("c", 64), + {"running a build of the new form", nodeengine.HaveByNodeModuleValue{Fp: meshcatalogclient.SourcePrefix2 + strings.Repeat("a", 64), Composed: "sha256:" + strings.Repeat("c", 64), Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}, false, false}, {"running with no fingerprint (mesh-wireguard)", nodeengine.HaveByNodeModuleValue{Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}, true, false}, {"held today", nodeengine.HaveByNodeModuleValue{Fp: "build-0123abcd", Composed: "sha256:" + strings.Repeat("c", 64), Phase: nodeengineclient.PhaseHeld, @@ -1768,11 +1770,11 @@ func (g *gate) MayAct(...string) bool { return g.open.Load() } // already runs with the same composition is not held. func TestAnUnreadSeatListHoldsWhatIsNewAndHandsItAtTheRead(t *testing.T) { b := newFakeBus() - fp := delivery.SourcePrefix2 + strings.Repeat("a", 64) - digest := delivery.ManifestDigest([]byte(lemurs)) - man, _ := delivery.EncodeManifest(delivery.Manifest{Module: "lemurs", JSON: lemurs}) - key, _ := delivery.ManifestsKey(digest) - b.kv[delivery.ManifestsBucket+"/"+key] = man + fp := meshcatalogclient.SourcePrefix2 + strings.Repeat("a", 64) + digest := meshcatalogclient.ManifestDigest([]byte(lemurs)) + man, _ := meshcatalogclient.EncodeManifest(meshcatalog.ManifestsByDigestValue{Module: "lemurs", Manifest: lemurs}) + key, _ := meshcatalogclient.ManifestsKey(digest) + b.kv[buckets.Manifests+"/"+key] = man ctx, cancel := context.WithCancel(context.Background()) defer cancel() env := b.env() @@ -1790,15 +1792,15 @@ func TestAnUnreadSeatListHoldsWhatIsNewAndHandsItAtTheRead(t *testing.T) { place, _ := meshmoduleassignerclient.EncodePlacement(meshmoduleassigner.Placement{Node: "n1", Decided: meshmoduleassigner.NodeDecided{Hub: true, Address: "10.77.0.1"}, Modules: []meshmoduleassigner.PlacedModule{{Module: "lemurs"}}}) have, _ := nodeengineclient.EncodeHaveNode(nodeengine.HaveByNodeNodeValue{Record: &nodeengine.NodeRecord{Node: "n1", TunnelKey: "k", SealingKey: "sk", Capabilities: []string{"systemd"}}}) - mod, _ := delivery.EncodeModule(delivery.Module{FP2: fp, Manifest: digest, Repository: "novox/x", Commit: "c"}) + mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fp, Manifest: digest, Repository: "novox/x", Commit: "c"}) want, _ := bucketstest.Want(meshwants.WantsByWantNodeModuleValue{Fp: fp, Self: true}) nodeWant, _ := bucketstest.NodeWant(meshwants.WantsByWantNodeNodeValue{Self: true}) b.put(t, buckets.Placement, "placement.n1", place) b.put(t, buckets.Have, "n1.node", have) - b.put(t, delivery.ModulesBucket, "lemurs", mod) + b.put(t, buckets.Modules, "lemurs", mod) b.put(t, buckets.Wants, "want.n1.lemurs", want) b.put(t, buckets.Wants, "want.n1.node", nodeWant) - for _, bucket := range []string{buckets.Wants, buckets.Placement, buckets.Have, delivery.ModulesBucket, ResolvedBucket, delivery.BuilderBuildsBucket} { + for _, bucket := range []string{buckets.Wants, buckets.Placement, buckets.Have, buckets.Modules, ResolvedBucket, delivery.BuilderBuildsBucket} { b.send(bucket, agent.Update{Caught: true}) } handed := map[string]bool{} diff --git a/internal/selfcompose/shadow.go b/internal/selfcompose/shadow.go index 4ca4646..abc73a1 100644 --- a/internal/selfcompose/shadow.go +++ b/internal/selfcompose/shadow.go @@ -9,18 +9,17 @@ import ( nodeengine "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" "git.novox.be/novox/mesh-contracts/placement/go/seats/meshmoduleassigner" "git.novox.be/novox/mesh-contracts/placement/go/seats/meshwants" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" "github.com/novox/mesh-host/internal/buckets" + "github.com/novox/mesh-host/internal/compose/frombus" engcomp "github.com/novox/mesh-host/internal/compositions" + "github.com/novox/mesh-host/internal/judge" "sort" "strings" "sync" "sync/atomic" "time" - "git.novox.be/novox/mesh-sdk/go/delivery" - "github.com/novox/mesh-host/internal/compose/frombus" - "github.com/novox/mesh-host/internal/judge" - "github.com/novox/mesh-host/internal/agent" ) @@ -165,7 +164,7 @@ func (s *Shadow) Run(ctx context.Context) error { {buckets.PlacementCandidates, ">", nil, nil}, {buckets.Placement, ">", (*Composer).takePlacement, nil}, {buckets.Have, "*." + nodeengineclient.NodeWord, nil, nil}, - {delivery.ModulesBucket, ">", (*Composer).takeModule, nil}, + {buckets.Modules, ">", (*Composer).takeModule, nil}, {ResolvedBucket, node + ".>", (*Composer).takeValue, nil}, // What this node runs moved: an apply ends in its module's have. {buckets.Have, node + ".>", nil, nil}, @@ -521,7 +520,7 @@ func (s *Shadow) answerProposal(ctx context.Context, proposal string) (nodeengin s.mu.Unlock() c.mu.Lock() - modules := make(map[string]delivery.Module, len(c.modules)) + modules := make(map[string]meshcatalog.ModulesByModuleValue, len(c.modules)) for k, v := range c.modules { modules[k] = v } @@ -557,7 +556,7 @@ func (s *Shadow) answerProposal(ctx context.Context, proposal string) (nodeengin fps := shadowBuilds(running, *placement, modules, inOwn) kept := map[string]string{} for module, fp := range fps { - if mod, ok := modules[module]; ok && mod.FP2 == fp { + if mod, ok := modules[module]; ok && mod.Fp2 == fp { continue } kept[module] = fp // composed from that build (frombus.Scope.KeptAt, issue 1126) @@ -665,7 +664,7 @@ func asHavesSay(a nodeengine.CandidateAnswer, ran map[string]string) nodeengine. // of the old form) is given none, so it is folded there again, as the node's own composer folds it, rather than // composed at the trunk's build as if the candidate added it (issue 1126: nats and photos have a trunk build that no // want names). -func shadowBuilds(running []engcomp.Composed, placement meshmoduleassigner.Placement, modules map[string]delivery.Module, inOwn []string) map[string]string { +func shadowBuilds(running []engcomp.Composed, placement meshmoduleassigner.Placement, modules map[string]meshcatalog.ModulesByModuleValue, inOwn []string) map[string]string { fps := map[string]string{} for _, r := range running { if r.Module != "" && r.FP != "" { @@ -678,8 +677,8 @@ func shadowBuilds(running []engcomp.Composed, placement meshmoduleassigner.Place } for _, m := range placement.Modules { if _, runs := fps[m.Module]; !runs && !folded[m.Module] { - if mod, ok := modules[m.Module]; ok && mod.FP2 != "" { - fps[m.Module] = mod.FP2 + if mod, ok := modules[m.Module]; ok && mod.Fp2 != "" { + fps[m.Module] = mod.Fp2 } } } diff --git a/internal/userlist/userlist_test.go b/internal/userlist/userlist_test.go index 7aad01e..3fdecc0 100644 --- a/internal/userlist/userlist_test.go +++ b/internal/userlist/userlist_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "errors" + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" engcomp "github.com/novox/mesh-host/internal/compositions" "strings" "sync" @@ -17,7 +18,7 @@ import ( const node = "novox" -func fpOf(c string) string { return delivery.SourcePrefix2 + strings.Repeat(c, 64) } +func fpOf(c string) string { return meshcatalogclient.SourcePrefix2 + strings.Repeat(c, 64) } // feed is a watch the test writes to: the keys there before the watch, Caught, then each put. watching is closed once // the watch began, so a put after it is never handed before the keys there before it.