From 4bfa94dfefcd07b3c0c330a45328ecd6813777d6 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 5 Nov 2026 15:44:31 +0100 Subject: [PATCH] =?UTF-8?q?Node-engine:=20a=20builder's=20build,=20its=20k?= =?UTF-8?q?eys=20and=20codec=20come=20from=20the=20builder's=20builds=20pa?= =?UTF-8?q?ckage=20(ADR=200634=20=C2=A71,=20=C2=A74;=20issue=201966)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Claude-Session: https://claude.ai/code/session_01NcbgfFSnn24M3GcqttoU7i --- cmd/mesh-compose/main_test.go | 4 +- cmd/mesh-host/nodeagent_integration_test.go | 16 +++--- go.mod | 1 + internal/compose/frombus/answer_rows_test.go | 14 +++--- internal/compose/frombus/frombus.go | 17 ++++--- internal/compose/frombus/frombus_test.go | 28 +++++------ .../frombus/frombustest/frombustest.go | 25 +++++----- .../frombus/frombustest/frombustest_test.go | 14 +++--- internal/compose/frombus/held_rows_test.go | 25 +++++----- internal/compose/frombus/states.go | 13 +++-- internal/compose/frombus/twice.go | 10 ++-- internal/selfcompose/composedstate.go | 4 +- internal/selfcompose/composedstate_test.go | 15 +++--- internal/selfcompose/data_test.go | 18 +++---- internal/selfcompose/selfcompose.go | 25 +++++----- internal/selfcompose/selfcompose_test.go | 50 +++++++++---------- internal/takein/takein.go | 8 +-- 17 files changed, 143 insertions(+), 144 deletions(-) diff --git a/cmd/mesh-compose/main_test.go b/cmd/mesh-compose/main_test.go index 2ce1355..0b69063 100644 --- a/cmd/mesh-compose/main_test.go +++ b/cmd/mesh-compose/main_test.go @@ -282,7 +282,7 @@ func ownPackages(t *testing.T) map[string][]string { } // 1621/AC2 and AC4: the command imports, through every package of this repository it reaches, only the standard -// library, mesh-sdk, the contracts modules' pure packages (seats' types, clients' codecs), the delivery SDK's composer (the command's own request and verdict types), the private network's overlay package (mesh-wireguard's own pure package: names, graph and +// library, mesh-sdk, the contracts modules' pure packages (seats' types, clients' codecs), the delivery SDK's composer (the command's own request and verdict types), the builder's builds package (a build's value and its keys), the private network's overlay package (mesh-wireguard's own pure package: names, graph and // generator, no bus) and this repository's own packages, and none that reaches a bus, a store, a network service or // another process. func TestTheCommandImportsNoBusStoreNetworkOrProcess(t *testing.T) { @@ -294,7 +294,7 @@ func TestTheCommandImportsNoBusStoreNetworkOrProcess(t *testing.T) { for path, imports := range own { for _, imp := range imports { std := !strings.Contains(strings.SplitN(imp, "/", 2)[0], ".") - if !std && !strings.HasPrefix(imp, "git.novox.be/novox/mesh-networking/modules/mesh-wireguard/go/overlay") && !strings.HasPrefix(imp, "git.novox.be/novox/mesh-sdk/go") && !strings.HasPrefix(imp, "git.novox.be/novox/mesh-contracts/") && !strings.HasPrefix(imp, "git.novox.be/novox/mesh-delivery/contracts/") && !strings.HasPrefix(imp, "git.novox.be/novox/mesh-delivery/sdk/composer") && !strings.HasPrefix(imp, "github.com/novox/mesh-host/") { + if !std && !strings.HasPrefix(imp, "git.novox.be/novox/mesh-networking/modules/mesh-wireguard/go/overlay") && !strings.HasPrefix(imp, "git.novox.be/novox/mesh-sdk/go") && !strings.HasPrefix(imp, "git.novox.be/novox/mesh-contracts/") && !strings.HasPrefix(imp, "git.novox.be/novox/mesh-delivery/contracts/") && !strings.HasPrefix(imp, "git.novox.be/novox/mesh-delivery/sdk/composer") && !strings.HasPrefix(imp, "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds") && !strings.HasPrefix(imp, "github.com/novox/mesh-host/") { t.Errorf("%s imports %s, which is neither the standard library, mesh-sdk nor this repository", path, imp) } for _, f := range forbidden { diff --git a/cmd/mesh-host/nodeagent_integration_test.go b/cmd/mesh-host/nodeagent_integration_test.go index eda3474..f8fcfbc 100644 --- a/cmd/mesh-host/nodeagent_integration_test.go +++ b/cmd/mesh-host/nodeagent_integration_test.go @@ -14,6 +14,7 @@ import ( "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + builderbuilds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "github.com/novox/mesh-host/internal/buckets" "github.com/novox/mesh-host/internal/buckets/bucketstest" engcomp "github.com/novox/mesh-host/internal/compositions" @@ -28,7 +29,6 @@ import ( nec "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" "git.novox.be/novox/mesh-sdk/go/buscompose" "git.novox.be/novox/mesh-sdk/go/declaration" - "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/seatlist" "github.com/nats-io/nats-server/v2/server" "github.com/nats-io/nats.go" @@ -416,7 +416,7 @@ func h8(t *testing.T, b *itBed, checkBuild bool) (func(), h8state) { } placement, modules, manifests := kvOf(buckets.Placement), kvOf(buckets.Modules), kvOf(buckets.Manifests) resolved := kvOf(selfcompose.ResolvedBucket) - builds := kvOf(delivery.BuilderBuildsBucket) + builds := kvOf(builderbuilds.BuilderBuildsBucket) candidates := kvOf(buckets.PlacementCandidates) packages := kvOf(link.PackagesBucket) path := filepath.Join(b.dir, "web.conf") @@ -447,9 +447,9 @@ func h8(t *testing.T, b *itBed, checkBuild bool) (func(), h8state) { rec2, _ := nec.EncodeHaveNode(nodeengine.HaveByNodeNodeValue{Record: &nodeengine.NodeRecord{Node: "it2", Capabilities: []string{"service-manager"}}}) pkey2, _ := meshmoduleassignerclient.LegacyKey("it2") hkey2, _ := nec.HaveNodeKey("it2") - buildKey, _ := delivery.BuilderBuildKey("web", fp) - done, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, Manifest: digest, - Artifacts: []delivery.BuiltArtifact{{Name: "files", Kind: "archive", Reference: "artifact-store://web/files/blobs/" + fpDigest("f"), Digest: fpDigest("f")}}}) + buildKey, _ := builderbuilds.BuilderBuildKey("web", fp) + done, _ := builderbuilds.EncodeBuilderBuild(builderbuilds.BuilderBuild{State: builderbuilds.BuildDone, Manifest: digest, + Artifacts: []builderbuilds.BuiltArtifact{{Name: "files", Kind: "archive", Reference: "artifact-store://web/files/blobs/" + fpDigest("f"), Digest: fpDigest("f")}}}) firstBuild := done if checkBuild { firstBuild = []byte(`{"v":1,"state":"no such state"}`) @@ -770,7 +770,7 @@ func TestNatsTheBusServerIsTakenOverAndReplacedOnlyWhenItsBuildChangesIt(t *test } placement, modules, manifests := kvOf(buckets.Placement), kvOf(buckets.Modules), kvOf(buckets.Manifests) kvOf(selfcompose.ResolvedBucket) - builds := kvOf(delivery.BuilderBuildsBucket) + builds := kvOf(builderbuilds.BuilderBuildsBucket) kvOf(buckets.PlacementCandidates) kvOf(link.PackagesBucket) // The bus module at a build, done: its manifest written and the indexer's module naming it at fp. @@ -786,8 +786,8 @@ func TestNatsTheBusServerIsTakenOverAndReplacedOnlyWhenItsBuildChangesIt(t *test if _, err := manifests.Put(mkey, mraw); err != nil { t.Fatal(err) } - bkey, _ := delivery.BuilderBuildKey("nats", fp) - done, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, Manifest: digest}) + bkey, _ := builderbuilds.BuilderBuildKey("nats", fp) + done, _ := builderbuilds.EncodeBuilderBuild(builderbuilds.BuilderBuild{State: builderbuilds.BuildDone, Manifest: digest}) if _, err := builds.Put(bkey, done); err != nil { t.Fatal(err) } diff --git a/go.mod b/go.mod index 46b4065..bf5e482 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/novox/mesh-host go 1.26.0 require ( + git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds v0.0.0 git.novox.be/novox/mesh-delivery/sdk v0.0.0-20261105000000-000000000000 git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient v0.0.0-20261105000000-000000000000 git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery v0.0.0-20261105000000-000000000000 diff --git a/internal/compose/frombus/answer_rows_test.go b/internal/compose/frombus/answer_rows_test.go index ebaaf31..261f490 100644 --- a/internal/compose/frombus/answer_rows_test.go +++ b/internal/compose/frombus/answer_rows_test.go @@ -9,6 +9,7 @@ import ( "git.novox.be/novox/mesh-contracts/placement/go/seats/meshmoduleassigner" "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" engcomp "github.com/novox/mesh-host/internal/compositions" "os" "reflect" @@ -18,7 +19,6 @@ import ( "git.novox.be/novox/mesh-networking/modules/mesh-wireguard/go/overlay" "git.novox.be/novox/mesh-sdk/go/catalogue" - "git.novox.be/novox/mesh-sdk/go/delivery" "github.com/novox/mesh-host/internal/compose" "github.com/novox/mesh-host/internal/compose/resolution" ) @@ -49,7 +49,7 @@ func answerNamesEveryRefusalAndLeavesAnOldBuildToTheNode(t *testing.T) { records: map[string]nodeengine.NodeRecord{"n1": {Node: "n1", TunnelKey: publicKey(t), SealingKey: "sk", Capabilities: []string{"systemd", "container-runtime"}}}, modules: map[string]meshcatalog.ModulesByModuleValue{"lemurs": {Manifest: meshcatalogclient.ManifestDigest(lemurs)}}, manifests: map[string][]byte{meshcatalogclient.ManifestDigest(lemurs): lemurs}, - builds: map[string]*delivery.BuilderBuild{}} + builds: map[string]*builds.BuilderBuild{}} for _, m := range []string{"web", "api", "spiegel"} { b.own = append(b.own, compose.OwnSecretRow{Node: "n1", Module: m, Name: "token", Origin: "made", SealedTo: "sk", Sealed: `"box"`}) } @@ -64,8 +64,8 @@ func answerNamesEveryRefusalAndLeavesAnOldBuildToTheNode(t *testing.T) { mod := b.modules[m] mod.Fp2 = fp(string(rune('1' + i))) b.modules[m] = mod - b.builds[m+" "+mod.Fp2] = &delivery.BuilderBuild{State: delivery.BuildDone, Manifest: mod.Manifest, - Artifacts: []delivery.BuiltArtifact{{Name: "site", Kind: "image", Digest: "sha256:" + strings.Repeat("b", 64)}}} + b.builds[m+" "+mod.Fp2] = &builds.BuilderBuild{State: builds.BuildDone, Manifest: mod.Manifest, + Artifacts: []builds.BuiltArtifact{{Name: "site", Kind: "image", Digest: "sha256:" + strings.Repeat("b", 64)}}} } return b } @@ -209,8 +209,8 @@ func answerNamesEveryRefusalAndLeavesAnOldBuildToTheNode(t *testing.T) { sp.Fp2 = fp("3") bb.modules["spiegel"] = sp image := "registry.example:5100/spiegel/site@sha256:" + strings.Repeat("7", 64) - bb.builds["spiegel "+sp.Fp2] = &delivery.BuilderBuild{State: delivery.BuildDone, Manifest: sp.Manifest, - Artifacts: []delivery.BuiltArtifact{{Name: "site", Kind: "upstream", Reference: image, Digest: "sha256:" + strings.Repeat("7", 64)}}} + bb.builds["spiegel "+sp.Fp2] = &builds.BuilderBuild{State: builds.BuildDone, Manifest: sp.Manifest, + Artifacts: []builds.BuiltArtifact{{Name: "site", Kind: "upstream", Reference: image, Digest: "sha256:" + strings.Repeat("7", 64)}}} read, err := ComposingInput(ctx, bb, Mesh{}, Scope{Node: "n1", InOwn: []string{"spiegel"}}, map[string]string{"lemurs": fp("e"), "spiegel": sp.Fp2}) raw, _ := json.Marshal(read.Catalogue["spiegel"]) @@ -565,7 +565,7 @@ func answerNamesEveryRefusalAndLeavesAnOldBuildToTheNode(t *testing.T) { siteRuns.Resources[1] = json.RawMessage(`{"id":"site.server","module":"site","type":"container","name":"site","image":"registry.example/site@sha256:` + strings.Repeat("c", 64) + `","ports":["8080"]}`) siteRuns.FP = fp("7") b.manifests[meshcatalogclient.ManifestDigest(site)] = site - b.builds["site "+fp("7")] = &delivery.BuilderBuild{State: delivery.BuildDone, Manifest: meshcatalogclient.ManifestDigest(site)} + b.builds["site "+fp("7")] = &builds.BuilderBuild{State: builds.BuildDone, Manifest: meshcatalogclient.ManifestDigest(site)} got, _, held, err = Compose(ctx, b, Mesh{}, Scope{Node: "n1", Running: []engcomp.Composed{siteRuns}}, fps) if err != nil || held["site"] == "" || held["nft"] != "" { t.Errorf("site moved to 9090 in its new manifest: %v, held %v; want site held alone", err, held) diff --git a/internal/compose/frombus/frombus.go b/internal/compose/frombus/frombus.go index 899e057..9e7b302 100644 --- a/internal/compose/frombus/frombus.go +++ b/internal/compose/frombus/frombus.go @@ -40,6 +40,7 @@ import ( "git.novox.be/novox/mesh-contracts/placement/go/seats/meshmoduleassigner" "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" engcomp "github.com/novox/mesh-host/internal/compositions" "maps" "net/netip" @@ -78,9 +79,9 @@ type Reader interface { // and each it grants as a provider, with the copy sealed to it (the vault's `own` with the keys, ADR 0550 §5). A key // the vault holds no value for is left out; the composition then stands in for it, and Compose refuses it. Credentials(ctx context.Context, node string, asConsumer, asProvider []compose.CredentialKey) ([]compose.Credential, []compose.Issued, error) - // Build is a module's build at a fingerprint, the builder's key (delivery.BuilderBuildsBucket, BuilderBuildKey); nil + // Build is a module's build at a fingerprint, the builder's key (builds.BuilderBuildsBucket, BuilderBuildKey); nil // when the builder holds none. The composing node composes each of its modules from its build (issue 1126). - Build(ctx context.Context, module, fp string) (*delivery.BuilderBuild, error) + Build(ctx context.Context, module, fp string) (*builds.BuilderBuild, error) // Archive is the archive at url, a kept reference routed to the artifact store as the composing node reaches it // (catalogue.Routed), checked against digest: a moved module's templates archive, read back for its texts as the // controller's take-in reads it (takein.Built). @@ -501,7 +502,7 @@ func readBase(ctx context.Context, r Reader, mesh Mesh, s Scope) (base, error) { // finish is input from its base on: the composing node's modules from their builds (build reads one), its own secrets. // in is the base's input, or a copy of it (base.cloned) where one base is finished more than once. -func (b base) finish(ctx context.Context, r Reader, build func(ctx context.Context, module, fp string) (*delivery.BuilderBuild, error), +func (b base) finish(ctx context.Context, r Reader, build func(ctx context.Context, module, fp string) (*builds.BuilderBuild, error), mesh Mesh, s Scope, in compose.Input, refuse func(module string, err error), wanted map[string]string) (compose.Input, error) { var err error parse := parser(mesh) @@ -777,7 +778,7 @@ func ranOf(ctx context.Context, r Reader, mesh Mesh, running []engcomp.Composed, continue } b, err := r.Build(ctx, module, c.FP) - if err != nil || b == nil || b.State != delivery.BuildDone || b.Manifest == "" { + if err != nil || b == nil || b.State != builds.BuildDone || b.Manifest == "" { return catalogue.Manifest{}, false } raw, err := r.Manifest(ctx, b.Manifest) @@ -795,7 +796,7 @@ func ranOf(ctx context.Context, r Reader, mesh Mesh, running []engcomp.Composed, // module it keeps at another build (Scope.KeptAt, into its node's Kept), read from the builder's done key and resolved // by the one take-in rule (takein.Manifest), a templates archive read in the artifact store as the node reaches it // (issue 1126). -func fromBuilds(ctx context.Context, r Reader, build func(ctx context.Context, module, fp string) (*delivery.BuilderBuild, error), +func fromBuilds(ctx context.Context, r Reader, build func(ctx context.Context, module, fp string) (*builds.BuilderBuild, error), in *compose.Input, s Scope, p meshmoduleassigner.Placement, modules map[string]meshcatalog.ModulesByModuleValue, read func(module, digest string) (catalogue.Manifest, error), parse func([]byte) (catalogue.Manifest, error), refuse func(module string, err error), wanted map[string]string) error { @@ -835,7 +836,7 @@ func fromBuilds(ctx context.Context, r Reader, build func(ctx context.Context, m } return r.Archive(ctx, catalogue.Routed(reference, address), digest) } - built := func(module, fp string, b *delivery.BuilderBuild) (catalogue.Manifest, error) { + built := func(module, fp string, b *builds.BuilderBuild) (catalogue.Manifest, error) { if b.Manifest == "" { return catalogue.Manifest{}, fmt.Errorf("frombus: the build of %s at %s names no manifest", module, fp) } @@ -920,7 +921,7 @@ func fromBuilds(ctx context.Context, r Reader, build func(ctx context.Context, m stub(pm.Module, err) continue } - if b == nil || b.State != delivery.BuildDone { + if b == nil || b.State != builds.BuildDone { if refuse != nil && runs[pm.Module] { // A module that runs is never composed from the indexer's manifest, which names no build's artifacts (no // runtime's bundle): it is held, keeping what runs, until its build lands (issue 774, 2026-10-30 19:30: @@ -950,7 +951,7 @@ func fromBuilds(ctx context.Context, r Reader, build func(ctx context.Context, m switch { case err != nil: err = fmt.Errorf("frombus: the build of %s at %s: %w", module, fp, err) - case b == nil || b.State != delivery.BuildDone: + case b == nil || b.State != builds.BuildDone: err = fmt.Errorf("frombus: %s is wanted at %s, of which the builder holds no done build", module, fp) } var resolved catalogue.Manifest diff --git a/internal/compose/frombus/frombus_test.go b/internal/compose/frombus/frombus_test.go index 33a75fe..c0a4cae 100644 --- a/internal/compose/frombus/frombus_test.go +++ b/internal/compose/frombus/frombus_test.go @@ -17,6 +17,7 @@ import ( "git.novox.be/novox/mesh-contracts/placement/go/seats/meshmoduleassigner" "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "github.com/novox/mesh-host/internal/compositions" "os" "reflect" @@ -27,7 +28,6 @@ import ( "git.novox.be/novox/mesh-networking/modules/mesh-wireguard/go/overlay" "git.novox.be/novox/mesh-sdk/go/catalogue" - "git.novox.be/novox/mesh-sdk/go/delivery" "github.com/novox/mesh-host/internal/compose" ) @@ -48,7 +48,7 @@ type bus struct { asked []compose.CredentialKey // builds are the builder's keys, by " "; archives the artifact store's blobs, by digest, and fetched // every URL asked; served another node's values by ".", and servedRead every ask with its keys. - builds map[string]*delivery.BuilderBuild + builds map[string]*builds.BuilderBuild archives map[string][]byte fetched []string served map[string]map[string]any @@ -58,7 +58,7 @@ type bus struct { held bool } -func (b *bus) Build(_ context.Context, module, fp string) (*delivery.BuilderBuild, error) { +func (b *bus) Build(_ context.Context, module, fp string) (*builds.BuilderBuild, error) { b.buildsRead = append(b.buildsRead, module) return b.builds[module+" "+fp], nil } @@ -560,8 +560,8 @@ func TestInput(t *testing.T) { kept.manifests["old"] = oldXterm oldFP := "src2:" + strings.Repeat("9", 64) toolsRef := "artifact-store://xterm/tools/blobs/sha256:" + strings.Repeat("1", 64) - kept.builds = map[string]*delivery.BuilderBuild{"xterm " + oldFP: {State: delivery.BuildDone, Manifest: "old", - Artifacts: []delivery.BuiltArtifact{{Name: "tools", Kind: "bundle", Reference: toolsRef, Digest: "sha256:" + strings.Repeat("1", 64)}}}} + kept.builds = map[string]*builds.BuilderBuild{"xterm " + oldFP: {State: builds.BuildDone, Manifest: "old", + Artifacts: []builds.BuiltArtifact{{Name: "tools", Kind: "bundle", Reference: toolsRef, Digest: "sha256:" + strings.Repeat("1", 64)}}}} parsedOld, err := catalogue.Decode(oldXterm) if err != nil { t.Fatal(err) @@ -617,9 +617,9 @@ func TestInput(t *testing.T) { digestOf := func(c string) string { return "sha256:" + strings.Repeat(c, 64) } movedFps := map[string]string{"restic": "src2:" + strings.Repeat("3", 64), "mesh-provisioner-client": "src2:" + strings.Repeat("4", 64), "store": "src2:" + strings.Repeat("5", 64)} - resticArtifacts := []delivery.BuiltArtifact{{Name: "tools", Kind: "bundle", Reference: "artifact-store://restic/tools/blobs/" + digestOf("6"), Digest: digestOf("6")}, + resticArtifacts := []builds.BuiltArtifact{{Name: "tools", Kind: "bundle", Reference: "artifact-store://restic/tools/blobs/" + digestOf("6"), Digest: digestOf("6")}, {Name: "files", Kind: "archive", Reference: "artifact-store://restic/files/blobs/" + archiveDigest, Digest: archiveDigest}} - clientArtifacts := []delivery.BuiltArtifact{{Name: "client", Kind: "bundle", Reference: "artifact-store://mesh-provisioner-client/client/blobs/" + digestOf("7"), Digest: digestOf("7")}} + clientArtifacts := []builds.BuiltArtifact{{Name: "client", Kind: "bundle", Reference: "artifact-store://mesh-provisioner-client/client/blobs/" + digestOf("7"), Digest: digestOf("7")}} movedMesh := func(built bool) *bus { b := one() p := b.placements["n1"] @@ -638,10 +638,10 @@ func TestInput(t *testing.T) { // The vault's own answer: the client's own secret; restic's is the client's to ask, so own leaves it out. b.own = []compose.OwnSecretRow{{Node: "n1", Module: "mesh-provisioner-client", Name: "provisioner-broker", Origin: "made", SealedTo: "sk", Sealed: `"box"`}} if built { - b.builds = map[string]*delivery.BuilderBuild{ - "restic " + movedFps["restic"]: {State: delivery.BuildDone, Manifest: meshcatalogclient.ManifestDigest(resticRaw), Artifacts: resticArtifacts}, - "mesh-provisioner-client " + movedFps["mesh-provisioner-client"]: {State: delivery.BuildDone, Manifest: meshcatalogclient.ManifestDigest(clientRaw), Artifacts: clientArtifacts}, - "store " + movedFps["store"]: {State: delivery.BuildDone, Manifest: meshcatalogclient.ManifestDigest(store)}, + b.builds = map[string]*builds.BuilderBuild{ + "restic " + movedFps["restic"]: {State: builds.BuildDone, Manifest: meshcatalogclient.ManifestDigest(resticRaw), Artifacts: resticArtifacts}, + "mesh-provisioner-client " + movedFps["mesh-provisioner-client"]: {State: builds.BuildDone, Manifest: meshcatalogclient.ManifestDigest(clientRaw), Artifacts: clientArtifacts}, + "store " + movedFps["store"]: {State: builds.BuildDone, Manifest: meshcatalogclient.ManifestDigest(store)}, } b.archives = map[string][]byte{archiveDigest: packed.Bytes()} } @@ -674,7 +674,7 @@ func TestInput(t *testing.T) { } // A done key an artifact of which names no reference is waited for, in the controller's words (ADR 0498 §8). unreferenced := movedMesh(true) - unreferenced.builds["restic "+movedFps["restic"]].Artifacts = []delivery.BuiltArtifact{{Name: "tools", Kind: "bundle", Digest: digestOf("6")}, resticArtifacts[1]} + unreferenced.builds["restic "+movedFps["restic"]].Artifacts = []builds.BuiltArtifact{{Name: "tools", Kind: "bundle", Digest: digestOf("6")}, resticArtifacts[1]} if _, _, held, err := Compose(ctx, unreferenced, Mesh{}, Scope{Node: "n1"}, movedFps); err != nil || !strings.Contains(held["restic"], "the build of restic is done and waits for the reference of tools") { t.Errorf("a done key with an unreferenced artifact: %v %v, want restic held alone", err, held) } @@ -866,8 +866,8 @@ func statesOf(t *testing.T, ctx context.Context, b *bus) mapStates { } for key, build := range b.builds { module, fp, _ := strings.Cut(key, " ") - k, _ := delivery.BuilderBuildKey(module, fp) - raw, err := delivery.EncodeBuilderBuild(*build) + k, _ := builds.BuilderBuildKey(module, fp) + raw, err := builds.EncodeBuilderBuild(*build) put(BuildsRead, k, raw, err) } for nm, values := range b.values { diff --git a/internal/compose/frombus/frombustest/frombustest.go b/internal/compose/frombus/frombustest/frombustest.go index e0016d0..70a7a56 100644 --- a/internal/compose/frombus/frombustest/frombustest.go +++ b/internal/compose/frombus/frombustest/frombustest.go @@ -16,11 +16,10 @@ import ( "git.novox.be/novox/mesh-contracts/placement/go/seats/meshmoduleassigner" "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" - "strings" - - "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "github.com/novox/mesh-host/internal/compose" "github.com/novox/mesh-host/internal/compose/frombus" + "strings" ) //go:embed testdata @@ -34,9 +33,9 @@ type Bus struct { Manifests map[string][]byte Value map[string]map[string]any // by node.module Own []compose.OwnSecretRow - Builds map[string]*delivery.BuilderBuild // by " " - Archives map[string][]byte // by digest - Serves map[string]map[string]any // another node's values, by node.module + Builds map[string]*builds.BuilderBuild // by " " + Archives map[string][]byte // by digest + Serves map[string]map[string]any // another node's values, by node.module Held bool } @@ -81,7 +80,7 @@ func (b *Bus) Credentials(_ context.Context, _ string, asConsumer, asProvider [] return out, issued, nil } -func (b *Bus) Build(_ context.Context, module, fp string) (*delivery.BuilderBuild, error) { +func (b *Bus) Build(_ context.Context, module, fp string) (*builds.BuilderBuild, error) { return b.Builds[module+" "+fp], nil } @@ -164,15 +163,15 @@ func Provisioned() (*Bus, error) { SealingKey: "sk", Contract: compose.HandedContract, Capabilities: []string{"systemd", "package-manager"}}}, Module: map[string]meshcatalog.ModulesByModuleValue{}, Manifests: map[string][]byte{}, Own: []compose.OwnSecretRow{{Node: Node, Module: "mesh-provisioner-client", Name: "provisioner-broker", Origin: "made", SealedTo: "sk", Sealed: `"box"`}}, - Builds: map[string]*delivery.BuilderBuild{ - "restic " + fps["restic"]: {State: delivery.BuildDone, Manifest: meshcatalogclient.ManifestDigest(resticRaw), Artifacts: []delivery.BuiltArtifact{ + Builds: map[string]*builds.BuilderBuild{ + "restic " + fps["restic"]: {State: builds.BuildDone, Manifest: meshcatalogclient.ManifestDigest(resticRaw), Artifacts: []builds.BuiltArtifact{ {Name: "tools", Kind: "bundle", Reference: "artifact-store://restic/tools/blobs/" + digestOf("6"), Digest: digestOf("6")}, {Name: "files", Kind: "archive", Reference: "artifact-store://restic/files/blobs/" + archive, Digest: archive}}}, - "mesh-provisioner-client " + fps["mesh-provisioner-client"]: {State: delivery.BuildDone, Manifest: meshcatalogclient.ManifestDigest(clientRaw), - Artifacts: []delivery.BuiltArtifact{{Name: "client", Kind: "bundle", + "mesh-provisioner-client " + fps["mesh-provisioner-client"]: {State: builds.BuildDone, Manifest: meshcatalogclient.ManifestDigest(clientRaw), + Artifacts: []builds.BuiltArtifact{{Name: "client", Kind: "bundle", Reference: "artifact-store://mesh-provisioner-client/client/blobs/" + digestOf("7"), Digest: digestOf("7")}}}, - "store " + fps["store"]: {State: delivery.BuildDone, Manifest: meshcatalogclient.ManifestDigest(store)}, - "postgres " + fps["postgres"]: {State: delivery.BuildDone, Manifest: meshcatalogclient.ManifestDigest(postgres)}, + "store " + fps["store"]: {State: builds.BuildDone, Manifest: meshcatalogclient.ManifestDigest(store)}, + "postgres " + fps["postgres"]: {State: builds.BuildDone, Manifest: meshcatalogclient.ManifestDigest(postgres)}, }, Archives: map[string][]byte{archive: packed.Bytes()}, Held: true, diff --git a/internal/compose/frombus/frombustest/frombustest_test.go b/internal/compose/frombus/frombustest/frombustest_test.go index e0ac451..3593057 100644 --- a/internal/compose/frombus/frombustest/frombustest_test.go +++ b/internal/compose/frombus/frombustest/frombustest_test.go @@ -9,6 +9,7 @@ import ( "git.novox.be/novox/mesh-contracts/placement/go/seats/meshmoduleassigner" "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" engcomp "github.com/novox/mesh-host/internal/compositions" "reflect" "slices" @@ -17,7 +18,6 @@ import ( stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/catalogue" - "git.novox.be/novox/mesh-sdk/go/delivery" "github.com/novox/mesh-host/internal/compose/frombus" "github.com/novox/mesh-host/internal/compose/fromruntime" ) @@ -84,8 +84,8 @@ func TestTheProvisionedMeshHandsAPlanWithItsProvision(t *testing.T) { if row.built != "" { builtDigest := meshcatalogclient.ManifestDigest([]byte(row.built)) b.Manifests[builtDigest] = []byte(row.built) - b.Builds["backup "+fp] = &delivery.BuilderBuild{State: delivery.BuildDone, Manifest: builtDigest, - Artifacts: []delivery.BuiltArtifact{{Name: "files", Kind: "archive", Reference: "artifact-store://backup/files/blobs/x", Digest: "sha256:x"}}} + b.Builds["backup "+fp] = &builds.BuilderBuild{State: builds.BuildDone, Manifest: builtDigest, + Artifacts: []builds.BuiltArtifact{{Name: "files", Kind: "archive", Reference: "artifact-store://backup/files/blobs/x", Digest: "sha256:x"}}} } if row.dataOnly { items, _, iheld, err := frombus.DataItemsHeld(context.Background(), b, frombus.Mesh{}, frombus.Scope{Node: Node}) @@ -274,10 +274,10 @@ func TestTheProvisionedMeshHandsAPlanWithItsProvision(t *testing.T) { bb.Builds["mesh-provisioner-client "+fps["mesh-provisioner-client"]] = &built want = "" } else if row.filledHeld { - bb.Builds["restic "+fps["restic"]] = &delivery.BuilderBuild{State: delivery.BuildBuilding} + bb.Builds["restic "+fps["restic"]] = &builds.BuilderBuild{State: builds.BuildBuilding} } else if row.placed { // held: its build not done while it runs - bb.Builds["mesh-provisioner-client "+fps["mesh-provisioner-client"]] = &delivery.BuilderBuild{State: delivery.BuildBuilding} + bb.Builds["mesh-provisioner-client "+fps["mesh-provisioner-client"]] = &builds.BuilderBuild{State: builds.BuildBuilding} } else { // taken out of the node's placement while what runs still names it: the account goes with it, never kept stale p := bb.Placement[Node] @@ -334,8 +334,8 @@ func statesOf(t *testing.T, b *Bus) frombus.States { } for at, build := range b.Builds { module, fp, _ := strings.Cut(at, " ") - key, _ := delivery.BuilderBuildKey(module, fp) - raw, err := delivery.EncodeBuilderBuild(*build) + key, _ := builds.BuilderBuildKey(module, fp) + raw, err := builds.EncodeBuilderBuild(*build) put(frombus.BuildsRead, key, raw, err) } return s diff --git a/internal/compose/frombus/held_rows_test.go b/internal/compose/frombus/held_rows_test.go index a60ec47..4936bb1 100644 --- a/internal/compose/frombus/held_rows_test.go +++ b/internal/compose/frombus/held_rows_test.go @@ -9,13 +9,12 @@ import ( "git.novox.be/novox/mesh-contracts/placement/go/seats/meshmoduleassigner" "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" + "github.com/novox/mesh-host/internal/compose" + "github.com/novox/mesh-host/internal/compose/resolution" engcomp "github.com/novox/mesh-host/internal/compositions" "strings" "testing" - - "git.novox.be/novox/mesh-sdk/go/delivery" - "github.com/novox/mesh-host/internal/compose" - "github.com/novox/mesh-host/internal/compose/resolution" ) // aNode is a bus with n1 the hub, each of manifests placed on it and held at the indexer, in the order given. @@ -23,7 +22,7 @@ func aNode(t *testing.T, manifests ...string) (*bus, map[string]string) { t.Helper() p := meshmoduleassigner.Placement{Node: "n1", Decided: meshmoduleassigner.NodeDecided{Hub: true, Address: "10.77.0.1"}} b := &bus{records: map[string]nodeengine.NodeRecord{"n1": {Node: "n1", TunnelKey: publicKey(t), SealingKey: "sk", Capabilities: []string{"systemd"}}}, - modules: map[string]meshcatalog.ModulesByModuleValue{}, manifests: map[string][]byte{}, builds: map[string]*delivery.BuilderBuild{}} + modules: map[string]meshcatalog.ModulesByModuleValue{}, manifests: map[string][]byte{}, builds: map[string]*builds.BuilderBuild{}} fps := map[string]string{} for i, raw := range manifests { module := strings.SplitN(strings.TrimPrefix(raw, `{"module":"`), `"`, 2)[0] @@ -148,12 +147,12 @@ func aRunningModuleWaitsForItsBuild(t *testing.T) { mod := b.modules["node-tools"] mod.Fp2 = next b.modules["node-tools"] = mod - b.builds["node-tools "+old] = &delivery.BuilderBuild{State: delivery.BuildDone, Manifest: mod.Manifest, - Artifacts: []delivery.BuiltArtifact{{Name: "runtime", Kind: "bundle", Reference: "artifact-store://node-tools/runtime/blobs/" + dg("1"), Digest: dg("1")}}} - b.builds["node-tools "+next] = &delivery.BuilderBuild{State: "building", Manifest: mod.Manifest} + b.builds["node-tools "+old] = &builds.BuilderBuild{State: builds.BuildDone, Manifest: mod.Manifest, + Artifacts: []builds.BuiltArtifact{{Name: "runtime", Kind: "bundle", Reference: "artifact-store://node-tools/runtime/blobs/" + dg("1"), Digest: dg("1")}}} + b.builds["node-tools "+next] = &builds.BuilderBuild{State: "building", Manifest: mod.Manifest} if row.done { - b.builds["node-tools "+next] = &delivery.BuilderBuild{State: delivery.BuildDone, Manifest: mod.Manifest, - Artifacts: []delivery.BuiltArtifact{{Name: "runtime", Kind: "bundle", Reference: "artifact-store://node-tools/runtime/blobs/" + dg("2"), Digest: dg("2")}}} + b.builds["node-tools "+next] = &builds.BuilderBuild{State: builds.BuildDone, Manifest: mod.Manifest, + Artifacts: []builds.BuiltArtifact{{Name: "runtime", Kind: "bundle", Reference: "artifact-store://node-tools/runtime/blobs/" + dg("2"), Digest: dg("2")}}} } fps["node-tools"] = row.want out, _, held, err := Compose(context.Background(), b, Mesh{}, Scope{Node: "n1", Running: running}, fps) @@ -258,10 +257,10 @@ func aHeldModuleOfItsOwnAccountNeverRefusesTheNode(t *testing.T) { mod := b.modules[m] mod.Fp2 = fps[m] b.modules[m] = mod - b.builds[m+" "+fps[m]] = &delivery.BuilderBuild{State: delivery.BuildDone, Manifest: mod.Manifest, - Artifacts: []delivery.BuiltArtifact{{Name: name, Kind: "bundle", Reference: "artifact-store://" + m + "/" + name + "/blobs/" + dg, Digest: dg}}} + b.builds[m+" "+fps[m]] = &builds.BuilderBuild{State: builds.BuildDone, Manifest: mod.Manifest, + Artifacts: []builds.BuiltArtifact{{Name: name, Kind: "bundle", Reference: "artifact-store://" + m + "/" + name + "/blobs/" + dg, Digest: dg}}} if m == "node-tools" && row.building { - b.builds[m+" "+fps[m]] = &delivery.BuilderBuild{State: "building", Manifest: mod.Manifest} + b.builds[m+" "+fps[m]] = &builds.BuilderBuild{State: "building", Manifest: mod.Manifest} } } out, _, held, err := Compose(context.Background(), b, Mesh{}, Scope{Node: "n1"}, fps) diff --git a/internal/compose/frombus/states.go b/internal/compose/frombus/states.go index 501005a..2ae2480 100644 --- a/internal/compose/frombus/states.go +++ b/internal/compose/frombus/states.go @@ -18,12 +18,11 @@ import ( "git.novox.be/novox/mesh-contracts/placement/go/seats/meshmoduleassigner" "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" - "strings" - "time" - - "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "github.com/novox/mesh-host/internal/compose" "github.com/novox/mesh-host/internal/compose/resolution" + "strings" + "time" ) // The states a module names in its manifest's `reads` to compose through StatesReader. @@ -161,8 +160,8 @@ func (statesReader) Credentials(context.Context, string, []compose.CredentialKey return nil, nil, nil } -func (r statesReader) Build(_ context.Context, module, fp string) (*delivery.BuilderBuild, error) { - key, err := delivery.BuilderBuildKey(module, fp) +func (r statesReader) Build(_ context.Context, module, fp string) (*builds.BuilderBuild, error) { + key, err := builds.BuilderBuildKey(module, fp) if err != nil { return nil, err } @@ -170,7 +169,7 @@ func (r statesReader) Build(_ context.Context, module, fp string) (*delivery.Bui if err != nil || !found { return nil, err } - b, err := delivery.DecodeBuilderBuild(v) + b, err := builds.DecodeBuilderBuild(v) if err != nil { return nil, err } diff --git a/internal/compose/frombus/twice.go b/internal/compose/frombus/twice.go index e9738dd..4411c51 100644 --- a/internal/compose/frombus/twice.go +++ b/internal/compose/frombus/twice.go @@ -12,11 +12,11 @@ import ( "encoding/hex" "encoding/json" "fmt" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "sync" "time" "git.novox.be/novox/mesh-sdk/go/catalogue" - "git.novox.be/novox/mesh-sdk/go/delivery" ) // HeldPlans is one scope's plans, as PlansHeld answers them: err a refusal that is no single module's. @@ -75,13 +75,13 @@ func PlansHeldTwice(ctx context.Context, r Reader, mesh Mesh, s Scope) (at, newe // onceBuilds is r's Build, each module's build at a fingerprint read once: both scopes of one composition read the // builds of the same moment. -func onceBuilds(r Reader) func(ctx context.Context, module, fp string) (*delivery.BuilderBuild, error) { +func onceBuilds(r Reader) func(ctx context.Context, module, fp string) (*builds.BuilderBuild, error) { type answer struct { - b *delivery.BuilderBuild + b *builds.BuilderBuild err error } read := map[string]answer{} - return func(ctx context.Context, module, fp string) (*delivery.BuilderBuild, error) { + return func(ctx context.Context, module, fp string) (*builds.BuilderBuild, error) { key := module + " " + fp if a, ok := read[key]; ok { return a.b, a.err @@ -101,7 +101,7 @@ type takenInKeeper interface { // takenInKey is what a take-in is made from: the artifact store's address and the build's record (its manifest's digest, its artifacts by digest), read // whole; an archive is checked against its digest on every read, so the same record takes in to the same bytes. -func takenInKey(module, fp, address string, b delivery.BuilderBuild) (string, bool) { +func takenInKey(module, fp, address string, b builds.BuilderBuild) (string, bool) { raw, err := json.Marshal(b) if err != nil { return "", false diff --git a/internal/selfcompose/composedstate.go b/internal/selfcompose/composedstate.go index 5693f90..5c4dab7 100644 --- a/internal/selfcompose/composedstate.go +++ b/internal/selfcompose/composedstate.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "fmt" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" engcomp "github.com/novox/mesh-host/internal/compositions" "maps" "sort" @@ -11,7 +12,6 @@ import ( "time" "git.novox.be/novox/mesh-sdk/go/catalogue" - "git.novox.be/novox/mesh-sdk/go/delivery" "github.com/novox/mesh-host/internal/agent" "github.com/novox/mesh-host/internal/compose/frombus" @@ -93,7 +93,7 @@ func (c *Composer) newestFPs(ctx context.Context, at *published) map[string]stri fp2 := c.modules[module].Fp2 c.mu.Unlock() if fp2 != "" { - if b, err := r.Build(ctx, module, fp2); err == nil && b != nil && b.State == delivery.BuildDone { + if b, err := r.Build(ctx, module, fp2); err == nil && b != nil && b.State == builds.BuildDone { out[module] = fp2 continue } diff --git a/internal/selfcompose/composedstate_test.go b/internal/selfcompose/composedstate_test.go index 4cf6e82..9f57406 100644 --- a/internal/selfcompose/composedstate_test.go +++ b/internal/selfcompose/composedstate_test.go @@ -9,6 +9,7 @@ import ( "git.novox.be/novox/mesh-contracts/placement/go/seats/meshwants" "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "github.com/novox/mesh-host/internal/buckets" "github.com/novox/mesh-host/internal/buckets/bucketstest" engcomp "github.com/novox/mesh-host/internal/compositions" @@ -18,8 +19,6 @@ import ( "testing" "time" - "git.novox.be/novox/mesh-sdk/go/delivery" - "github.com/novox/mesh-host/internal/agent" ) @@ -105,7 +104,7 @@ func TestTheNodePublishesItsCompositions(t *testing.T) { b.put(t, buckets.Modules, "oldie", omod) b.put(t, buckets.Wants, "want.n1.lemurs", want) b.put(t, buckets.Wants, "want.n1.node", nodeWant) - for _, bucket := range []string{buckets.Wants, buckets.Placement, buckets.Have, buckets.Modules, ResolvedBucket, delivery.BuilderBuildsBucket} { + for _, bucket := range []string{buckets.Wants, buckets.Placement, buckets.Have, buckets.Modules, ResolvedBucket, builds.BuilderBuildsBucket} { b.send(bucket, agent.Update{Caught: true}) } for u := next(t, out); !u.Caught; u = next(t, out) { @@ -147,17 +146,17 @@ func TestTheNodePublishesItsCompositions(t *testing.T) { t.Errorf("a pass that changed nothing wrote %v", got[2:]) } // oldie's trunk build is done: its key at that build is written once, though no want names it. - build, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, Manifest: odigest}) - bkey, _ := delivery.BuilderBuildKey("oldie", ofp) + build, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildDone, Manifest: odigest}) + bkey, _ := builds.BuilderBuildKey("oldie", ofp) b.mu.Lock() - b.kv[delivery.BuilderBuildsBucket+"/"+bkey] = build + b.kv[builds.BuilderBuildsBucket+"/"+bkey] = build b.mu.Unlock() - b.put(t, delivery.BuilderBuildsBucket, bkey, build) + b.put(t, builds.BuilderBuildsBucket, bkey, build) got := waitFor("oldie's done build", func(got []string) bool { return len(got) >= 3 }) if got[2] != oldieKey { t.Errorf("the done build of oldie wrote %v, want %s", got[2:], oldieKey) } - b.put(t, delivery.BuilderBuildsBucket, bkey, build) + b.put(t, builds.BuilderBuildsBucket, bkey, build) time.Sleep(100 * time.Millisecond) if got := state.written(); len(got) != 3 { t.Errorf("the same build landing again wrote %v", got[3:]) diff --git a/internal/selfcompose/data_test.go b/internal/selfcompose/data_test.go index d2273d3..04bca2b 100644 --- a/internal/selfcompose/data_test.go +++ b/internal/selfcompose/data_test.go @@ -6,6 +6,7 @@ import ( "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "github.com/novox/mesh-host/internal/buckets" engcomp "github.com/novox/mesh-host/internal/compositions" "slices" @@ -15,7 +16,6 @@ import ( "time" "git.novox.be/novox/mesh-sdk/go/catalogue" - "git.novox.be/novox/mesh-sdk/go/delivery" "github.com/novox/mesh-host/internal/compose/frombus" "github.com/novox/mesh-host/internal/agent" @@ -235,15 +235,15 @@ func TestTheEngineKeepsEachPlacedModulesDataItems(t *testing.T) { // The items are worked out from what the pass read: a done build is read over the bus once, not again for the data // items after the pass (a module in a running build is read again, since it may land). b := newFakeBus() - done, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, Manifest: meshcatalogclient.ManifestDigest([]byte(lemurs))}) - running, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: "building"}) + done, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildDone, Manifest: meshcatalogclient.ManifestDigest([]byte(lemurs))}) + running, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: "building"}) fp := meshcatalogclient.SourcePrefix2 + strings.Repeat("b", 64) - doneKey, _ := delivery.BuilderBuildKey("web", fp) - runningKey, _ := delivery.BuilderBuildKey("pg", fp) - b.kv[delivery.BuilderBuildsBucket+"/"+doneKey], b.kv[delivery.BuilderBuildsBucket+"/"+runningKey] = done, running + doneKey, _ := builds.BuilderBuildKey("web", fp) + runningKey, _ := builds.BuilderBuildKey("pg", fp) + b.kv[builds.BuilderBuildsBucket+"/"+doneKey], b.kv[builds.BuilderBuildsBucket+"/"+runningKey] = done, running r := reader{c: New(b.env())} for range 3 { - if got, err := r.Build(context.Background(), "web", fp); err != nil || got.State != delivery.BuildDone { + if got, err := r.Build(context.Background(), "web", fp); err != nil || got.State != builds.BuildDone { t.Fatalf("the done build: %+v, %v", got, err) } if got, err := r.Build(context.Background(), "pg", fp); err != nil || got.State != "building" { @@ -258,9 +258,9 @@ func TestTheEngineKeepsEachPlacedModulesDataItems(t *testing.T) { t.Errorf("the builds were read %v times over the bus; want the done one once, the running one each time", reads) } // A done key the builder claims again (a gone step, issue 1937) and lands done again is read again, not served kept. - again, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, Manifest: meshcatalogclient.ManifestDigest([]byte("{}"))}) + again, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildDone, Manifest: meshcatalogclient.ManifestDigest([]byte("{}"))}) b.mu.Lock() - b.kv[delivery.BuilderBuildsBucket+"/"+doneKey] = again + b.kv[builds.BuilderBuildsBucket+"/"+doneKey] = again b.mu.Unlock() r.c.takeBuildDone(agent.Update{Key: doneKey, Value: running}) if got, err := r.Build(context.Background(), "web", fp); err != nil || got.Manifest != meshcatalogclient.ManifestDigest([]byte("{}")) { diff --git a/internal/selfcompose/selfcompose.go b/internal/selfcompose/selfcompose.go index ce02dfd..cee73ed 100644 --- a/internal/selfcompose/selfcompose.go +++ b/internal/selfcompose/selfcompose.go @@ -26,6 +26,7 @@ import ( "git.novox.be/novox/mesh-contracts/placement/go/seats/meshwants" "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "github.com/novox/mesh-host/internal/buckets" "github.com/novox/mesh-host/internal/compositions" engcomp "github.com/novox/mesh-host/internal/compositions" @@ -150,7 +151,7 @@ type Composer struct { archives map[string][]byte // templates archives by digest, read once // builds are the builder's done builds by key, read once: a done build at a fingerprint never changes, so a pass and // the data items after it (publishData) do not read each placed module's build again over the bus (ADR 0577). - builds map[string]delivery.BuilderBuild + builds map[string]builds.BuilderBuild handed map[string]string // composed key to the digest handed // sealedOwn is each own secret's sealed value as first handed, by module/name and generation: a value sealed again // is other bytes, so the one first had is kept while the generation stands, and the composition stays the same. @@ -307,7 +308,7 @@ func New(env Env) *Composer { } return &Composer{env: env, wants: map[string]meshwants.WantsByWantNodeModuleValue{}, placements: map[string]meshmoduleassigner.Placement{}, placementKeys: meshmoduleassignerclient.NewKeys(), records: map[string]nodeengine.NodeRecord{}, inOwn: map[string]bool{}, modules: map[string]meshcatalog.ModulesByModuleValue{}, values: map[string]map[string]any{}, - manifests: map[string][]byte{}, archives: map[string][]byte{}, builds: map[string]delivery.BuilderBuild{}, handed: map[string]string{}, sealedOwn: map[string]ownSealed{}, + manifests: map[string][]byte{}, archives: map[string][]byte{}, builds: map[string]builds.BuilderBuild{}, handed: map[string]string{}, sealedOwn: map[string]ownSealed{}, pkgs: map[string]compositions.Composed{}, asked: make(chan chan Pass, 8)} } @@ -371,7 +372,7 @@ func (c *Composer) inputs() []input { {ResolvedBucket, node + ".>", (*Composer).takeValue, nil}, // The builder's builds: a pass that held a module for a build not yet done is made again when it lands (issue // 774, 2026-10-30 19:30: node-tools' build landed and ace, g14 and shanks stayed refused until another input moved). - {delivery.BuilderBuildsBucket, ">", nil, (*Composer).takeBuildDone}, + {builds.BuilderBuildsBucket, ">", nil, (*Composer).takeBuildDone}, } } @@ -386,7 +387,7 @@ func (c *Composer) takeBuildDone(u agent.Update) bool { if u.Deleted { return false } - module, fp, err := delivery.ParseBuilderBuildKey(u.Key) + module, fp, err := builds.ParseBuilderBuildKey(u.Key) if err != nil { return false } @@ -403,8 +404,8 @@ func (c *Composer) takeBuildDone(u agent.Update) bool { if !(wanted && w.Fp == fp) && !((wanted || placedHere) && m.Fp2 == fp) { return false } - b, err := delivery.DecodeBuilderBuild(u.Value) - return err == nil && b.State == delivery.BuildDone + b, err := builds.DecodeBuilderBuild(u.Value) + return err == nil && b.State == builds.BuildDone } // Watch starts the composer and answers the compositions it makes: each as a key of `composed-` would be, then a @@ -1429,8 +1430,8 @@ func (r reader) Manifest(ctx context.Context, digest string) ([]byte, error) { return []byte(m.Manifest), nil } -func (r reader) Build(ctx context.Context, module, fp string) (*delivery.BuilderBuild, error) { - key, err := delivery.BuilderBuildKey(module, fp) +func (r reader) Build(ctx context.Context, module, fp string) (*builds.BuilderBuild, error) { + key, err := builds.BuilderBuildKey(module, fp) if err != nil { return nil, err } @@ -1440,16 +1441,16 @@ func (r reader) Build(ctx context.Context, module, fp string) (*delivery.Builder if ok { return &done, nil } - defer r.c.waiting("the build of " + module + " at " + fp + " (" + delivery.BuilderBuildsBucket + ")")() - raw, err := r.c.get(ctx, delivery.BuilderBuildsBucket, key) + defer r.c.waiting("the build of " + module + " at " + fp + " (" + builds.BuilderBuildsBucket + ")")() + raw, err := r.c.get(ctx, builds.BuilderBuildsBucket, key) if err != nil || raw == nil { return nil, err } - b, err := delivery.DecodeBuilderBuild(raw) + b, err := builds.DecodeBuilderBuild(raw) if err != nil { return nil, fmt.Errorf("the builder's %s: %w", key, err) } - if b.State == delivery.BuildDone { + if b.State == builds.BuildDone { r.c.keepRead(func(k *Composer) { k.builds[key] = b }) } return &b, nil diff --git a/internal/selfcompose/selfcompose_test.go b/internal/selfcompose/selfcompose_test.go index dcbaa2c..ecce6b7 100644 --- a/internal/selfcompose/selfcompose_test.go +++ b/internal/selfcompose/selfcompose_test.go @@ -15,6 +15,7 @@ import ( "git.novox.be/novox/mesh-contracts/placement/go/seats/meshwants" "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "github.com/novox/mesh-host/internal/buckets" "github.com/novox/mesh-host/internal/buckets/bucketstest" "github.com/novox/mesh-host/internal/compositions" @@ -29,7 +30,6 @@ import ( "time" "git.novox.be/novox/mesh-networking/modules/mesh-wireguard/go/overlay" - "git.novox.be/novox/mesh-sdk/go/delivery" "github.com/nats-io/nats.go" "github.com/novox/mesh-host/internal/compose" "github.com/novox/mesh-host/internal/compose/frombus" @@ -209,7 +209,7 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { b.put(t, buckets.Modules, "oldie", omod) b.put(t, buckets.Wants, "want.n1.lemurs", want) b.put(t, buckets.Wants, "want.n1.node", nodeWant) - for _, bucket := range []string{buckets.Wants, buckets.Placement, buckets.Have, buckets.Modules, ResolvedBucket, delivery.BuilderBuildsBucket} { + for _, bucket := range []string{buckets.Wants, buckets.Placement, buckets.Have, buckets.Modules, ResolvedBucket, builds.BuilderBuildsBucket} { b.send(bucket, agent.Update{Caught: true}) } got := map[string]bool{} @@ -570,22 +570,22 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { pass bool }{ {"lemurs's wanted build, not done", "lemurs", fp, "building", false}, - {"lemurs's wanted build lands", "lemurs", fp, delivery.BuildDone, true}, - {"another build of lemurs", "lemurs", meshcatalogclient.SourcePrefix2 + strings.Repeat("e", 64), delivery.BuildDone, false}, - {"a placed module no want names, at its trunk's build", "oldie", meshcatalogclient.SourcePrefix2 + strings.Repeat("c", 64), delivery.BuildDone, true}, - {"a module neither wanted nor placed here", "ghost", meshcatalogclient.SourcePrefix2 + strings.Repeat("c", 64), delivery.BuildDone, false}, + {"lemurs's wanted build lands", "lemurs", fp, builds.BuildDone, true}, + {"another build of lemurs", "lemurs", meshcatalogclient.SourcePrefix2 + strings.Repeat("e", 64), builds.BuildDone, false}, + {"a placed module no want names, at its trunk's build", "oldie", meshcatalogclient.SourcePrefix2 + strings.Repeat("c", 64), builds.BuildDone, true}, + {"a module neither wanted nor placed here", "ghost", meshcatalogclient.SourcePrefix2 + strings.Repeat("c", 64), builds.BuildDone, false}, } { before := passes() - key, _ := delivery.BuilderBuildKey(row.module, row.fp) - build := delivery.BuilderBuild{State: row.state} - if row.state == delivery.BuildDone { + key, _ := builds.BuilderBuildKey(row.module, row.fp) + build := builds.BuilderBuild{State: row.state} + if row.state == builds.BuildDone { build.Manifest = digest } - raw, err := delivery.EncodeBuilderBuild(build) + raw, err := builds.EncodeBuilderBuild(build) if err != nil { t.Fatal(err) } - b.put(t, delivery.BuilderBuildsBucket, key, raw) + b.put(t, builds.BuilderBuildsBucket, key, raw) quiet(t, out) if got := passes() > before; got != row.pass { t.Errorf("%s: a pass made %v, want %v", row.name, got, row.pass) @@ -700,15 +700,15 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { Modules: []meshmoduleassigner.PlacedModule{{Module: "tool"}}}) tmod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: next2, Manifest: tdigest, Repository: "novox/t", Commit: "c"}) twant, _ := bucketstest.Want(meshwants.WantsByWantNodeModuleValue{Fp: next2, Self: true}) - building, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: "building"}) - bkey, _ := delivery.BuilderBuildKey("tool", next2) - tb.kv[delivery.BuilderBuildsBucket+"/"+bkey] = building + building, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: "building"}) + bkey, _ := builds.BuilderBuildKey("tool", next2) + tb.kv[builds.BuilderBuildsBucket+"/"+bkey] = building tb.put(t, buckets.Placement, "placement.n1", tplace) tb.put(t, buckets.Have, "n1.node", have) tb.put(t, buckets.Modules, "tool", tmod) tb.put(t, buckets.Wants, "want.n1.tool", twant) for _, bucket := range []string{buckets.Wants, buckets.Placement, buckets.Have, buckets.Modules, - ResolvedBucket, delivery.BuilderBuildsBucket} { + ResolvedBucket, builds.BuilderBuildsBucket} { tb.send(bucket, agent.Update{Caught: true}) } toolKey, _ := compositions.Key("n1", "tool", next2) @@ -722,13 +722,13 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { t.Errorf("the node was told %q, want tool held for its build not yet done", held) } hmu.Unlock() - done, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, Manifest: tdigest, - Artifacts: []delivery.BuiltArtifact{{Name: "files", Kind: "archive", Reference: "artifact-store://tool/files/blobs/sha256:" + strings.Repeat("9", 64), + done, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildDone, Manifest: tdigest, + Artifacts: []builds.BuiltArtifact{{Name: "files", Kind: "archive", Reference: "artifact-store://tool/files/blobs/sha256:" + strings.Repeat("9", 64), Digest: "sha256:" + strings.Repeat("9", 64)}}}) tb.mu.Lock() - tb.kv[delivery.BuilderBuildsBucket+"/"+bkey] = done + tb.kv[builds.BuilderBuildsBucket+"/"+bkey] = done tb.mu.Unlock() - tb.put(t, delivery.BuilderBuildsBucket, bkey, done) + tb.put(t, builds.BuilderBuildsBucket, bkey, done) if u := next(t, tout); u.Key != toolKey || u.Deleted { t.Errorf("the build landed and the node handed %s, want tool at its new build", u.Key) } @@ -1370,7 +1370,7 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { oldRaw, _ := bucketstest.Candidate(meshmoduleassigner.CandidatesByProposalNodeValue{Proposal: "older", Placement: &trunk}) oldKey := meshmoduleassigner.CandidatesByProposalNodeKey("older", "n1") nb.put(buckets.PlacementCandidates, oldKey, oldRaw, 2) - for _, bucket := range []string{buckets.PlacementCandidates, buckets.Placement, buckets.Have, buckets.Modules, delivery.BuilderBuildsBucket} { + for _, bucket := range []string{buckets.PlacementCandidates, buckets.Placement, buckets.Have, buckets.Modules, builds.BuilderBuildsBucket} { nb.send(bucket, agent.Update{Caught: true}) } select { @@ -1461,10 +1461,10 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { // Issue 1126, the seed's refusals: what a node composes from its builds and reads of a provider elsewhere. A build // is the builder's key; a templates archive is fetched once per digest and refused when it is not that digest; another // node's served key is read alone, as the node's grant allows it (buscompose Principal.ServedReads). - built, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, Manifest: digest}) - buildKey, _ := delivery.BuilderBuildKey("lemurs", fp) + built, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildDone, Manifest: digest}) + buildKey, _ := builds.BuilderBuildKey("lemurs", fp) b.mu.Lock() - b.kv[delivery.BuilderBuildsBucket+"/"+buildKey] = built + b.kv[builds.BuilderBuildsBucket+"/"+buildKey] = built b.kv[ResolvedBucket+"/n2.mailer.domain"] = []byte(`{"value":"example.org","source":"module"}`) b.reads = nil b.mu.Unlock() @@ -1691,7 +1691,7 @@ func (r fixedReader) OwnSecrets(context.Context, string) ([]compose.OwnSecretRow func (r fixedReader) Credentials(context.Context, string, []compose.CredentialKey, []compose.CredentialKey) ([]compose.Credential, []compose.Issued, error) { return nil, nil, nil } -func (r fixedReader) Build(context.Context, string, string) (*delivery.BuilderBuild, error) { +func (r fixedReader) Build(context.Context, string, string) (*builds.BuilderBuild, error) { return nil, nil } func (r fixedReader) Archive(_ context.Context, url, _ string) ([]byte, error) { @@ -1800,7 +1800,7 @@ func TestAnUnreadSeatListHoldsWhatIsNewAndHandsItAtTheRead(t *testing.T) { b.put(t, buckets.Modules, "lemurs", mod) b.put(t, buckets.Wants, "want.n1.lemurs", want) b.put(t, buckets.Wants, "want.n1.node", nodeWant) - for _, bucket := range []string{buckets.Wants, buckets.Placement, buckets.Have, buckets.Modules, ResolvedBucket, delivery.BuilderBuildsBucket} { + for _, bucket := range []string{buckets.Wants, buckets.Placement, buckets.Have, buckets.Modules, ResolvedBucket, builds.BuilderBuildsBucket} { b.send(bucket, agent.Update{Caught: true}) } handed := map[string]bool{} diff --git a/internal/takein/takein.go b/internal/takein/takein.go index 657ab45..a47bb27 100644 --- a/internal/takein/takein.go +++ b/internal/takein/takein.go @@ -18,11 +18,11 @@ import ( "context" "errors" "fmt" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "io" "strings" "git.novox.be/novox/mesh-sdk/go/catalogue" - "git.novox.be/novox/mesh-sdk/go/delivery" ) // Archive reads the archive a recorded reference names (catalogue.Recorded: the artifact store's address left out), @@ -33,8 +33,8 @@ type Archive func(ctx context.Context, reference, digest string) ([]byte, error) // and every artifact must name its reference. // A templated module's templates archive is read through archive; a module without templates reads none, and archive // may then be nil. -func Manifest(ctx context.Context, m catalogue.Manifest, b delivery.BuilderBuild, archive Archive) (catalogue.Manifest, error) { - if b.State != delivery.BuildDone { +func Manifest(ctx context.Context, m catalogue.Manifest, b builds.BuilderBuild, archive Archive) (catalogue.Manifest, error) { + if b.State != builds.BuildDone { return catalogue.Manifest{}, fmt.Errorf("takein: the build of %s is %s, not done: nothing to resolve it with", m.Module, b.State) } // **A done key whose artifact names no reference is waited for, never composed from** (novox/hq ADR 0498 §8), in the @@ -64,7 +64,7 @@ func Manifest(ctx context.Context, m catalogue.Manifest, b delivery.BuilderBuild // Built is each of a done key's artifacts as catalogue.Manifest.Resolve reads it: its recorded reference and digest, a // TypeScript bundle's launchers by the build modules' one rule (catalogue.Launchers), and, for the archive a module packs // its templates into, each template's text read back from that archive. -func Built(ctx context.Context, m catalogue.Manifest, artifacts []delivery.BuiltArtifact, archive Archive) ([]catalogue.Built, error) { +func Built(ctx context.Context, m catalogue.Manifest, artifacts []builds.BuiltArtifact, archive Archive) ([]catalogue.Built, error) { declared := map[string]catalogue.Artifact{} if m.Build != nil { for _, a := range m.Build.Artifacts {