Merge branch 'feat/47-G-builder6y' into feat/47-G-join8
# Conflicts: # modules/mesh-delivery-builder/buildrun/asks_test.go # modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go # modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go # modules/mesh-delivery-builder/internal/build/numbers.go # modules/mesh-delivery-builder/internal/build/run.go # modules/mesh-delivery-builder/internal/build/stuck.go # modules/mesh-delivery-builder/internal/versions/publish.go # modules/mesh-delivery-builder/internal/versions/reserve.go # modules/mesh-delivery-builder/manifest_test.go # modules/mesh-delivery-builder/module.json
This commit is contained in:
@@ -15,10 +15,15 @@ import (
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds"
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
|
||||
"git.novox.be/novox/mesh-delivery/sdk/buildstep"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/seatdef"
|
||||
"git.novox.be/novox/mesh-sdk/go/seatlist"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient/enginereads"
|
||||
@@ -35,8 +40,11 @@ type queueAsker func(key string, value []byte) (json.RawMessage, error)
|
||||
|
||||
// Tools are the builder's tools: stuck reads, askAgain writes the reasks value, queue writes a queues value and waits for
|
||||
// the builder's read, buildAgain writes a reasks value of mode build and waits for the builder's answer.
|
||||
//
|
||||
// builder_versions, asked by a name, answers also what the verb reserved answers for it (reserve; design 47 G §572): the
|
||||
// bundle calls the seat mesh-builds through its invokes, as any caller does. Nil answers none.
|
||||
func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, queue, buildAgain queueAsker, since build.LogSince, now func() time.Time, vs versions.Reader,
|
||||
fw, of versions.Reader) []stdio.Tool {
|
||||
fw, of versions.Reader, reserve func(name, version string) (meshbuilds.ReservedAnswer, error)) []stdio.Tool {
|
||||
str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) }
|
||||
// handAct writes a reasks value of a versions hand act and answers the builder's answer, or its refusal as an error.
|
||||
handAct := func(key string, value []byte, err error) (any, error) {
|
||||
@@ -83,7 +91,16 @@ func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, q
|
||||
"version": map[string]any{"type": "string", "description": "one version, to ask whether it is recorded"},
|
||||
"digest": map[string]any{"type": "string", "description": "with version: the digest the holder received"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
return versions.AnswerVersions(context.Background(), vs, str(args, "name"), str(args, "version"), str(args, "digest"))
|
||||
out, err := versions.AnswerVersions(context.Background(), vs, str(args, "name"), str(args, "version"), str(args, "digest"))
|
||||
if err != nil || reserve == nil {
|
||||
return out, err
|
||||
}
|
||||
ra, err := reserve(out.Name, str(args, "version"))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the seat mesh-builds' reserved did not answer for %s: %w", out.Name, err)
|
||||
}
|
||||
out.Reserved = &ra
|
||||
return out, nil
|
||||
}},
|
||||
{Name: build.StuckTool,
|
||||
Description: "Every build key that waits (building or queued), oldest first: its module, fingerprint, step words and since when, and each ask it " +
|
||||
@@ -307,13 +324,15 @@ func written(s stdio.KeptState, key string, revision uint64) time.Time {
|
||||
return time.Time{}
|
||||
}
|
||||
|
||||
// The seat verbs the bundle calls (the manifest's invokes).
|
||||
const (
|
||||
catalogVersionsVerb = "seat:mesh-catalog.versions"
|
||||
reservedVerb = "seat:mesh-builds.reserved"
|
||||
)
|
||||
|
||||
func main() {
|
||||
steps := map[string]build.StateRead{}
|
||||
for _, bm := range build.StuckBuildModules {
|
||||
steps[bm] = read(stdio.State(bm + ".steps"))
|
||||
}
|
||||
reasks := stdio.State(build.ReAsksState)
|
||||
r := build.StuckReader{Logf: func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-delivery-builder-tools] "+f+"\n", a...) }, Builds: read(stdio.State("builds")), ReAsks: read(reasks), Steps: steps,
|
||||
r := build.StuckReader{Logf: func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-delivery-builder-tools] "+f+"\n", a...) }, Builds: read(stdio.State("builds")), ReAsks: read(reasks), StepsOf: func(seat string) build.StateRead { return read(stdio.State(seat + ".steps")) },
|
||||
Hearing: read(stdio.State(buildrun.HearingState))}
|
||||
put := func(key string, value []byte) error {
|
||||
_, err := reasks.Put(key, json.RawMessage(value))
|
||||
@@ -363,10 +382,52 @@ func main() {
|
||||
}
|
||||
return e.Value, true, nil
|
||||
}}
|
||||
tools := Tools(r, put, askQueue(stdio.State(build.QueuesState), 25*time.Second), buildAgain, stdio.Log(builds.BuilderLog).Since, time.Now, vr, fr, ofr)
|
||||
// The catalog half's view of the packages the bundle follows itself (design 47 G §572, A-70), as other bundles follow views.
|
||||
pk := &views.Reader{Module: builds.BuilderModule, Logf: func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-delivery-builder-tools] "+f+"\n", a...) },
|
||||
Acts: meshhandactsclient.Recorder{Ask: stdio.Ask}}
|
||||
branches, err := meshcatalogclient.FollowBranches(pk, stdio.Ask)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, "mesh-delivery-builder-tools:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
pk.Start(context.Background())
|
||||
// The build modules are no names (design 47 G A-82): the build seats on the seat list, each held by the module the catalog
|
||||
// half's manifests say; the seat list and its two tools wired once by mesh-sdk (§204).
|
||||
gate := seatlist.NewPart("builder", []string{stdio.AnyPerson, "node-tools", "mesh-healers"})
|
||||
go func() {
|
||||
<-stdio.Served()
|
||||
gate.Follow(seatlist.StdioBus(), nil, func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-delivery-builder-tools] "+f+"\n", a...) })
|
||||
}()
|
||||
r.Seats = buildrun.NewBuildSeats(func() map[string]seatdef.Definition {
|
||||
if rd := gate.Reader(); rd != nil {
|
||||
return rd.Definitions()
|
||||
}
|
||||
return nil
|
||||
}, versions.HolderOf(branches))
|
||||
carriers := versions.CarriersFrom(versions.EntriesOf(branches))
|
||||
released := func(_ context.Context, module string) ([]meshcatalog.VersionEntry, error) {
|
||||
raw, err := stdio.Ask(catalogVersionsVerb, meshcatalog.VersionsInput{Module: module})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var a meshcatalog.VersionsAnswer
|
||||
if err := json.Unmarshal(raw, &a); err != nil {
|
||||
return nil, fmt.Errorf("the seat mesh-catalog's versions answered what does not read: %w", err)
|
||||
}
|
||||
return a.Versions, nil
|
||||
}
|
||||
reserve := func(name, version string) (meshbuilds.ReservedAnswer, error) {
|
||||
raw, err := stdio.Ask(reservedVerb, meshbuilds.ReservedInput{Name: name, Version: version})
|
||||
if err != nil {
|
||||
return meshbuilds.ReservedAnswer{}, err
|
||||
}
|
||||
var a meshbuilds.ReservedAnswer
|
||||
return a, json.Unmarshal(raw, &a)
|
||||
}
|
||||
tools := Tools(r, put, askQueue(stdio.State(build.QueuesState), 25*time.Second), buildAgain, stdio.Log(builds.BuilderLog).Since, time.Now, vr, fr, ofr, reserve)
|
||||
tools = append(tools, buildTool(&build.BuildReader{Builds: read(stdio.State("builds")), Entry: entryOf(stdio.State("builds")), Asked: read(stdio.State("asked")),
|
||||
Newest: newestVia(stdio.Ask), Steps: r}))
|
||||
tools = append(tools, reservedTools(rr, buildAgain, time.Now, meshhandactsclient.Recorder{Ask: stdio.Ask})...)
|
||||
tools = append(tools, reservedTools(rr, buildAgain, time.Now, meshhandactsclient.Recorder{Ask: stdio.Ask}, carriers, released)...)
|
||||
qs, as := stdio.State(versions.PublishQueueState), stdio.State(versions.ActedState)
|
||||
tools = append(tools, republishTools(readerOf(qs), readerOf(as), buildAgain, time.Now, meshhandactsclient.Recorder{Ask: stdio.Ask})...)
|
||||
ws := stdio.State(build.WaitsState)
|
||||
@@ -392,6 +453,7 @@ func main() {
|
||||
// builder_first_views and builder_first_view_again: where the builder's first views of the indexer's and the assigner's
|
||||
// states stand, and asking one again by hand (ADR 0615, DLV-M1).
|
||||
tools = append(tools, views.StatusTools(builds.BuilderModule, "builder_first_views", "builder_first_view_again", stdio.State(views.StatusState), meshhandactsclient.Recorder{Ask: stdio.Ask}, time.Now)...)
|
||||
tools = append(tools, gate.Tools()...)
|
||||
if err := stdio.Serve("", tools); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "[mesh-delivery-builder-tools] %v\n", err)
|
||||
os.Exit(1)
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
@@ -16,7 +17,10 @@ import (
|
||||
// reservedTools are builder_reserved, builder_reserved_drop and the seat mesh-builds' verb reserved (design 47 G section
|
||||
// 369), served as tools of the bundle. The drop is a hand act: it is recorded in the hand-act log first, then written to
|
||||
// reasks for the builder, which owns the reservations, to answer.
|
||||
func reservedTools(rr versions.Reader, ask queueAsker, now func() time.Time, acts meshhandactsclient.Recorder) []stdio.Tool {
|
||||
//
|
||||
// The verb reserved given a name finds the module from the catalog half's view the bundle follows (carriers) and, for a version
|
||||
// no reservation holds, asks the seat mesh-catalog's verb versions (released); the builder's process serves no call (A-70).
|
||||
func reservedTools(rr versions.Reader, ask queueAsker, now func() time.Time, acts meshhandactsclient.Recorder, carriers func(context.Context, string) ([]versions.Built, error), released versions.CatalogVersions) []stdio.Tool {
|
||||
str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) }
|
||||
return []stdio.Tool{
|
||||
{Name: versions.ReservedTool, Description: versions.ReservedDescription,
|
||||
@@ -25,10 +29,14 @@ func reservedTools(rr versions.Reader, ask queueAsker, now func() time.Time, act
|
||||
Run: func(args map[string]any) (any, error) { return versions.AnswerReserved(rr, str(args, "module")) }},
|
||||
{Name: versions.ReservedVerb, Description: "The seat mesh-builds' verb reserved: a module's reserved version with each artifact's digest once built " +
|
||||
"(design 47 G section 369), for a registry holder deciding whether to publish. Read-only.",
|
||||
Input: map[string]any{"type": "object", "required": []string{"module"}, "properties": map[string]any{
|
||||
"module": map[string]any{"type": "string", "description": "the module"},
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{
|
||||
"module": map[string]any{"type": "string", "description": "the module; give this or name"},
|
||||
"name": map[string]any{"type": "string", "description": "a package's or an image's name in its ecosystem, in place of module"},
|
||||
"version": map[string]any{"type": "string", "description": "the version, when asked for a particular one"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
if name := str(args, "name"); name != "" {
|
||||
return versions.AnswerReservedByName(context.Background(), rr, carriers, released, name, str(args, "version"))
|
||||
}
|
||||
return versions.AnswerReservedVerb(rr, str(args, "module"), str(args, "version"))
|
||||
}},
|
||||
{Name: versions.ReservedDropTool, Description: versions.ReservedDropDescription,
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient"
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds"
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
|
||||
|
||||
"mesh-delivery-builder/internal/build"
|
||||
"mesh-delivery-builder/internal/versions"
|
||||
)
|
||||
|
||||
// Design 47 G §572, §582, A-70: the bundle's verb reserved takes a name and answers from the view it follows and the catalog's
|
||||
// versions, and builder_versions, asked by a name, calls reserved and answers what it answers.
|
||||
func TestReservedByNameIsTheBundlesAndBuilderVersionsCallsIt(t *testing.T) {
|
||||
none := versions.Reader{Keys: func() ([]string, error) { return nil, nil }, Get: func(string) (json.RawMessage, bool, error) { return nil, false, nil }}
|
||||
carriers := func(_ context.Context, name string) ([]versions.Built, error) {
|
||||
if name == "@novox/mesh-sdk" {
|
||||
return []versions.Built{{Module: "mesh-sdk", Artifact: "ts"}}, nil
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
released := func(_ context.Context, module string) ([]meshcatalog.VersionEntry, error) {
|
||||
return []meshcatalog.VersionEntry{{Module: module, Version: "0.1.2", Artifacts: []meshcatalog.VersionArtifact{{Artifact: "mesh-sdk:ts", Digest: "sha256:dd"}}}}, nil
|
||||
}
|
||||
acts := meshhandactsclient.Recorder{Ask: func(string, any) (json.RawMessage, error) { return json.RawMessage(`{}`), nil }}
|
||||
byName := map[string]func(map[string]any) (any, error){}
|
||||
for _, tl := range reservedTools(none, nil, time.Now, acts, carriers, released) {
|
||||
byName[tl.Name] = tl.Run
|
||||
}
|
||||
got, err := byName[versions.ReservedVerb](map[string]any{"name": "@novox/mesh-sdk", "version": "0.1.2"})
|
||||
want := meshbuilds.ReservedAnswer{Module: "mesh-sdk", Version: "0.1.2", Released: true, Artifacts: []meshbuilds.ReservedAnswerArtifactsItem{{Artifact: "mesh-sdk:ts", Digest: "sha256:dd"}}}
|
||||
if err != nil || got.(meshbuilds.ReservedAnswer).Released != want.Released || got.(meshbuilds.ReservedAnswer).Artifacts[0] != want.Artifacts[0] {
|
||||
t.Fatalf("reserved by name = %+v (%v), want %+v", got, err, want)
|
||||
}
|
||||
var called []string
|
||||
reserve := func(name, version string) (meshbuilds.ReservedAnswer, error) {
|
||||
called = append(called, name+"@"+version)
|
||||
return want, nil
|
||||
}
|
||||
tools := Tools(build.StuckReader{}, nil, nil, nil, nil, time.Now, none, none, none, reserve)
|
||||
for _, tl := range tools {
|
||||
if tl.Name == versions.VersionsTool {
|
||||
out, err := tl.Run(map[string]any{"name": "@novox/mesh-sdk", "version": "0.1.2"})
|
||||
ans, ok := out.(versions.VersionsAnswer)
|
||||
if err != nil || !ok || ans.Reserved == nil || ans.Reserved.Module != "mesh-sdk" || !ans.Reserved.Released {
|
||||
t.Fatalf("builder_versions by name = %+v (%v), want reserved's answer in it", out, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(called) != 1 || called[0] != "@novox/mesh-sdk@0.1.2" {
|
||||
t.Errorf("reserved was called %v, want once, by the name and version", called)
|
||||
}
|
||||
}
|
||||
@@ -3,13 +3,9 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
|
||||
"mesh-delivery-builder/internal/build"
|
||||
)
|
||||
|
||||
@@ -106,23 +102,3 @@ func follow(ctx context.Context, w jetstream.KeyWatcher, each func(jetstream.Key
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// manifestReader reads manifests.<digest> of the indexer and parses the manifest it holds.
|
||||
type manifestReader struct{ kv jetstream.KeyValue }
|
||||
|
||||
func (m manifestReader) Manifest(ctx context.Context, digest string) (catalogue.Manifest, error) {
|
||||
key, err := meshcatalogclient.ManifestsKey(digest)
|
||||
if err != nil {
|
||||
return catalogue.Manifest{}, err
|
||||
}
|
||||
e, err := m.kv.Get(ctx, key)
|
||||
if err != nil {
|
||||
return catalogue.Manifest{}, fmt.Errorf("manifests.%s: %w", key, err)
|
||||
}
|
||||
kept, err := meshcatalogclient.DecodeManifest(e.Value())
|
||||
if err != nil {
|
||||
return catalogue.Manifest{}, err
|
||||
}
|
||||
// Decoded, never judged (novox/hq ADR 0578 §3): what a build needs of it is read, a field it does not know set aside.
|
||||
return catalogue.Decode([]byte(kept.Manifest))
|
||||
}
|
||||
|
||||
@@ -0,0 +1,202 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
"mesh-delivery-builder/internal/build"
|
||||
)
|
||||
|
||||
// **The builder reads the catalog half per watched branch** (design 47 G §348, §479, §614; A-5, A-15): the shared branch follower
|
||||
// of meshcatalogclient, adopted on the builder's views reader, gives each watched branch's entries; the manifests, the packages
|
||||
// and each package's name in its ecosystem are read from the entries, never the indexer's states.
|
||||
|
||||
// entryManifests keeps the manifest of each entry heard, by the entry's manifest digest (the entry holds the manifest whole).
|
||||
type entryManifests struct {
|
||||
mu sync.Mutex
|
||||
byFp map[string]catalogue.Manifest
|
||||
}
|
||||
|
||||
// keep takes the manifest of e in; an entry with no manifest or digest keeps none.
|
||||
func (m *entryManifests) keep(e meshcatalog.CatalogEntry) error {
|
||||
if e.ManifestDigest == "" || len(e.Manifest) == 0 {
|
||||
return nil
|
||||
}
|
||||
raw, err := json.Marshal(e.Manifest)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Decoded, never judged (novox/hq ADR 0578 §3): what a build needs of it is read, a field it does not know set aside.
|
||||
mf, err := catalogue.Decode(raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the manifest of the entry of %s does not decode: %w", e.Module, err)
|
||||
}
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if m.byFp == nil {
|
||||
m.byFp = map[string]catalogue.Manifest{}
|
||||
}
|
||||
m.byFp[e.ManifestDigest] = mf
|
||||
return nil
|
||||
}
|
||||
|
||||
// Manifest is the manifest the entry heard with that digest holds.
|
||||
func (m *entryManifests) Manifest(_ context.Context, digest string) (catalogue.Manifest, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
mf, ok := m.byFp[digest]
|
||||
if !ok {
|
||||
return catalogue.Manifest{}, fmt.Errorf("no entry of the catalog half holds the manifest %s", digest)
|
||||
}
|
||||
return mf, nil
|
||||
}
|
||||
|
||||
// catalogHalf is the builder's part over the branch follower: what changes in a branch's half goes to the builder, and the
|
||||
// entries of every followed half are what the package names are read from.
|
||||
type catalogHalf struct {
|
||||
branches *meshcatalogclient.Branches
|
||||
b *build.Builder
|
||||
manifests *entryManifests
|
||||
logf func(string, ...any)
|
||||
// named is, by module, the package names of its first entry last heard, so a name newly heard looks at its module again.
|
||||
mu sync.Mutex
|
||||
named map[string]string
|
||||
// ready is told when every followed branch's first view is in, once; halves are the views adopted (sweepWiring).
|
||||
ready func()
|
||||
halves map[string]*views.Follower
|
||||
once sync.Once
|
||||
ctx context.Context
|
||||
// holders is told each entry change: a claim of a build seat may have moved (design 47 G A-82).
|
||||
holders func()
|
||||
}
|
||||
|
||||
// first says whether branch is the first watched branch of module's registration (design 47 G §371); a module with no
|
||||
// registration heard is taken as watched on this branch, the first one.
|
||||
func (c *catalogHalf) first(branch, module string) bool {
|
||||
reg, ok := c.branches.Registrations.Registration(module)
|
||||
return !ok || len(reg.Branches) == 0 || reg.Branches[0] == branch
|
||||
}
|
||||
|
||||
// Changed is the follower's Changed: one module's entry of a branch came, or went.
|
||||
func (c *catalogHalf) Changed(ctx context.Context) func(branch, module string, entry json.RawMessage, deleted bool, at time.Time) {
|
||||
return func(branch, module string, entry json.RawMessage, deleted bool, _ time.Time) {
|
||||
if c.holders != nil {
|
||||
defer c.holders()
|
||||
}
|
||||
first := c.first(branch, module)
|
||||
if deleted {
|
||||
c.b.EntryGone(branch, first, module)
|
||||
return
|
||||
}
|
||||
origin := build.Origin{Branch: branch, First: first}
|
||||
if cat, ok := c.branches.Catalog(branch); ok {
|
||||
if half, ok := cat.View(); ok {
|
||||
origin.Source = half.Revision.Source
|
||||
}
|
||||
}
|
||||
var e meshcatalog.CatalogEntry
|
||||
if err := json.Unmarshal(entry, &e); err != nil || e.Module != module {
|
||||
c.logf("the catalog half of the branch %q holds what is not the entry of %s, left: %v", branch, module, err)
|
||||
return
|
||||
}
|
||||
if err := c.manifests.keep(e); err != nil {
|
||||
c.logf("%v", err)
|
||||
}
|
||||
c.b.Entry(ctx, origin, e)
|
||||
c.mu.Lock()
|
||||
was := c.named[module]
|
||||
now := namesOf(e)
|
||||
if c.named == nil {
|
||||
c.named = map[string]string{}
|
||||
}
|
||||
c.named[module] = now
|
||||
c.mu.Unlock()
|
||||
if first && now != "" && now != was {
|
||||
c.logf("the catalog half's entry names a package of %s: its build is looked at again", module)
|
||||
c.b.PackageNamed(ctx, module)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// namesOf are the package names an entry's artifacts carry, in one line.
|
||||
func namesOf(e meshcatalog.CatalogEntry) string {
|
||||
var names []string
|
||||
for _, a := range e.Artifacts {
|
||||
if a.Package != nil && a.Package.Name != "" {
|
||||
names = append(names, a.Ref+"="+a.Package.Name)
|
||||
}
|
||||
}
|
||||
slices.Sort(names)
|
||||
return fmt.Sprint(names)
|
||||
}
|
||||
|
||||
// sweepWiring is the follower's wiring on the builder's reader that learns each catalog half view adopted, so ready is told once
|
||||
// when the first view of every branch followed is in (a branch with no revision yet is ready and empty at once).
|
||||
type sweepWiring struct {
|
||||
meshcatalogclient.Wiring
|
||||
half *catalogHalf
|
||||
}
|
||||
|
||||
func (w sweepWiring) Adopt(f *views.Follower, seatName, event string, hear func(context.Context, []byte) error) error {
|
||||
if err := w.Wiring.Adopt(f, seatName, event, hear); err != nil {
|
||||
return err
|
||||
}
|
||||
if seatName == meshcatalog.Seat && event == meshcatalog.EventCatalogRevision {
|
||||
w.half.adopted(f)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w sweepWiring) Drop(owner, state string) error {
|
||||
err := w.Wiring.Drop(owner, state)
|
||||
w.half.dropped(state)
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *catalogHalf) adopted(f *views.Follower) {
|
||||
c.mu.Lock()
|
||||
if c.halves == nil {
|
||||
c.halves = map[string]*views.Follower{}
|
||||
}
|
||||
c.halves[f.State] = f
|
||||
c.mu.Unlock()
|
||||
go func() {
|
||||
select {
|
||||
case <-c.ctx.Done():
|
||||
case <-f.Ready():
|
||||
c.checkReady()
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func (c *catalogHalf) dropped(state string) {
|
||||
c.mu.Lock()
|
||||
delete(c.halves, state)
|
||||
c.mu.Unlock()
|
||||
c.checkReady()
|
||||
}
|
||||
|
||||
// checkReady calls ready once every adopted half's first view is in.
|
||||
func (c *catalogHalf) checkReady() {
|
||||
c.mu.Lock()
|
||||
all := len(c.halves) > 0
|
||||
for _, f := range c.halves {
|
||||
select {
|
||||
case <-f.Ready():
|
||||
default:
|
||||
all = false
|
||||
}
|
||||
}
|
||||
c.mu.Unlock()
|
||||
if all && c.ready != nil {
|
||||
c.once.Do(c.ready)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
|
||||
)
|
||||
|
||||
// TestTheManifestOfAnEntryIsReadByItsDigest: the builder reads a manifest from the entry that holds it, by the entry's manifest
|
||||
// digest (design 47 G §348), and an entry with no manifest keeps none.
|
||||
func TestTheManifestOfAnEntryIsReadByItsDigest(t *testing.T) {
|
||||
m := &entryManifests{}
|
||||
e := meshcatalog.CatalogEntry{Module: "netcheck", ManifestDigest: "sha256:aa", Manifest: map[string]any{"module": "netcheck", "version": "1"}}
|
||||
if err := m.keep(e); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if mf, err := m.Manifest(context.Background(), "sha256:aa"); err != nil || mf.Module != "netcheck" {
|
||||
t.Errorf("Manifest(sha256:aa) = %+v, %v", mf, err)
|
||||
}
|
||||
if _, err := m.Manifest(context.Background(), "sha256:bb"); err == nil {
|
||||
t.Error("Manifest of a digest no entry holds: want an error")
|
||||
}
|
||||
if err := m.keep(meshcatalog.CatalogEntry{Module: "bare"}); err != nil || len(m.byFp) != 1 {
|
||||
t.Errorf("an entry with no manifest kept one: %v, %v", err, m.byFp)
|
||||
}
|
||||
}
|
||||
+153
@@ -0,0 +1,153 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/seatdef"
|
||||
"git.novox.be/novox/mesh-sdk/go/seatlist"
|
||||
"git.novox.be/novox/mesh-sdk/go/seatlist/seatlisttest"
|
||||
)
|
||||
|
||||
// catalogHolder is the holder of the seat mesh-catalog as the bed needs it (design 47 G §348, §614): it answers `revision` for the
|
||||
// branch main with the entries put so far, `registrations` with none (so the follower follows the branches the gate's `source`
|
||||
// answers) and `entry`, and says `catalog-revision` at each put, as the follower and the builder hear it.
|
||||
type catalogHolder struct {
|
||||
ctx context.Context
|
||||
js jetstream.JetStream
|
||||
|
||||
mu sync.Mutex
|
||||
entries map[string]meshcatalog.CatalogEntry
|
||||
revision int64
|
||||
}
|
||||
|
||||
func newCatalogHolder(t *testing.T, ctx context.Context, js jetstream.JetStream, nc *nats.Conn) *catalogHolder {
|
||||
t.Helper()
|
||||
h := &catalogHolder{ctx: ctx, js: js, entries: map[string]meshcatalog.CatalogEntry{}}
|
||||
answer := func(subject string, with func(in []byte) any) {
|
||||
s, err := nc.Subscribe(subject, func(m *nats.Msg) {
|
||||
reply, _ := json.Marshal(map[string]any{"result": with(m.Data), "node": "ctl"})
|
||||
_ = m.Respond(reply)
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = s.Unsubscribe() })
|
||||
}
|
||||
answer("mesh.seat."+meshcatalog.Seat+".call."+meshcatalog.VerbRevision+".>", func([]byte) any { return h.whole() })
|
||||
answer("mesh.seat."+meshcatalog.Seat+".call."+meshcatalog.VerbRegistrations+".>", func([]byte) any {
|
||||
return meshcatalog.RegistrationsAnswer{Registrations: []meshcatalog.Registration{}}
|
||||
})
|
||||
answer("mesh.seat."+meshcatalog.Seat+".call."+meshcatalog.VerbEntry+".>", func(in []byte) any {
|
||||
var q meshcatalog.EntryInput
|
||||
_ = json.Unmarshal(in, &q)
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
for _, e := range h.entries {
|
||||
if e.Fp == q.Fingerprint {
|
||||
return meshcatalog.EntryAnswer{Entry: e}
|
||||
}
|
||||
}
|
||||
return map[string]any{"error": "no entry of that fingerprint"}
|
||||
})
|
||||
answer("mesh.seat."+meshdelivery.Seat+".call."+meshdelivery.VerbSource+".>", func([]byte) any {
|
||||
return meshdelivery.SourceAnswer{Branches: []string{"main"}}
|
||||
})
|
||||
return h
|
||||
}
|
||||
|
||||
// whole is the newest trunk revision of main, every entry in one page; the branch has none yet before the first put.
|
||||
func (h *catalogHolder) whole() meshcatalog.RevisionAnswer {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
if h.revision == 0 {
|
||||
return meshcatalog.RevisionAnswer{NoneYet: true}
|
||||
}
|
||||
var refs []meshcatalog.EntryRef
|
||||
var all []meshcatalog.CatalogEntry
|
||||
for m, e := range h.entries {
|
||||
refs = append(refs, meshcatalog.EntryRef{Module: m, Digest: "sha256:" + e.Fp})
|
||||
all = append(all, e)
|
||||
}
|
||||
at := time.Now()
|
||||
return meshcatalog.RevisionAnswer{Revision: &meshcatalog.CatalogRevision{Revision: h.revision, Kind: "trunk", Branch: "main", Entries: refs, Source: 1, At: &at}, Entries: all}
|
||||
}
|
||||
|
||||
// put sets one module's entry in a new trunk revision of main and says catalog-revision.
|
||||
func (h *catalogHolder) put(t *testing.T, e meshcatalog.CatalogEntry) { h.putAll(t, e) }
|
||||
|
||||
// putAll sets several modules' entries in one new trunk revision of main.
|
||||
func (h *catalogHolder) putAll(t *testing.T, es ...meshcatalog.CatalogEntry) {
|
||||
t.Helper()
|
||||
h.mu.Lock()
|
||||
h.revision++
|
||||
var changed []string
|
||||
for _, e := range es {
|
||||
h.entries[e.Module] = e
|
||||
changed = append(changed, e.Module)
|
||||
}
|
||||
ev, _ := json.Marshal(meshcatalog.CatalogRevisionEvent{Branch: "main", Kind: "trunk", Revision: h.revision, Source: 1, Entries: int64(len(h.entries)), Changed: changed, V: 1})
|
||||
h.mu.Unlock()
|
||||
if _, err := h.js.Publish(h.ctx, meshcatalog.SubjectCatalogRevision("main"), ev); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// buildSeats is the seat list and the catalog half as the builder finds its build modules from them (design 47 G A-82): the seat
|
||||
// mesh-catalog's verb seat-definitions answers the three build seats' definitions, each acting on its language (the image seat on
|
||||
// kind:image), and an entry of each holder's manifest claims its seat.
|
||||
func (h *catalogHolder) buildSeats(t *testing.T, nc *nats.Conn, holders map[string]string) {
|
||||
t.Helper()
|
||||
defs := map[string]seatdef.Definition{}
|
||||
for _, d := range seatlisttest.Definitions() {
|
||||
if _, ok := holders[d.Seat]; ok {
|
||||
defs[d.Seat] = d
|
||||
}
|
||||
}
|
||||
for seat, on := range map[string]string{"node-build-go": "go", "node-build-node": "typescript", "node-build-image": seatdef.OnKindImage} {
|
||||
d := defs[seat]
|
||||
d.Acts = []seatdef.ActingRule{{Step: seatdef.StepBuild, On: on}}
|
||||
defs[seat] = d
|
||||
}
|
||||
s, err := nc.Subscribe("mesh.seat."+meshcatalog.Seat+".call."+seatlist.Verb+".>", func(m *nats.Msg) {
|
||||
var in struct {
|
||||
Seat string `json:"seat"`
|
||||
}
|
||||
_ = json.Unmarshal(m.Data, &in)
|
||||
var result any
|
||||
if in.Seat == "" {
|
||||
l := seatlist.Listing{Revision: 1}
|
||||
for seat := range defs {
|
||||
l.Seats = append(l.Seats, seatlist.Entry{Seat: seat, Source: seatlist.SourceTrunk, Digest: "sha256:" + seat})
|
||||
}
|
||||
result = l
|
||||
} else {
|
||||
raw, _ := json.Marshal(defs[in.Seat])
|
||||
result = seatlist.Answer{Seat: in.Seat, Found: true, Revision: 1, Entry: &seatlist.Entry{Seat: in.Seat, Source: seatlist.SourceTrunk, Digest: "sha256:" + in.Seat, Definition: raw}}
|
||||
}
|
||||
reply, _ := json.Marshal(map[string]any{"result": result, "node": "ctl"})
|
||||
_ = m.Respond(reply)
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = s.Unsubscribe() })
|
||||
var entries []meshcatalog.CatalogEntry
|
||||
for seat, module := range holders {
|
||||
entries = append(entries, meshcatalog.CatalogEntry{Module: module, Repository: "novox/mesh-catalog", Commit: "c0", Fp: "src2:" + strings.Repeat(fmt.Sprintf("%x", 10+len(seat)%6), 64), ManifestDigest: "sha256:" + fmt.Sprintf("%064x", len(module)*7919+int(module[len(module)-1])),
|
||||
Manifest: map[string]any{"module": module, "claims": []any{map[string]any{"name": seat}}}})
|
||||
}
|
||||
h.putAll(t, entries...)
|
||||
}
|
||||
@@ -19,7 +19,10 @@ import (
|
||||
// heldOnTheBus says whether an ask a build key names is on its build seat's stream, or a build node wrote a step for it
|
||||
// (issue 1910), read once at the take-over of a key at start; and, at the builder's check bound, whether an ask can still
|
||||
// be answered (issue 1945). Never polled.
|
||||
type heldOnTheBus struct{ js jetstream.JetStream }
|
||||
type heldOnTheBus struct {
|
||||
js jetstream.JetStream
|
||||
seats *buildrun.BuildSeats // the build modules (design 47 G A-82)
|
||||
}
|
||||
|
||||
// Held is build.Builder's Held: a step key `<node>.<ask id>` in the build module's steps, or a message of the stream under
|
||||
// the ask's subject whose Nats-Msg-Id is the ask's id.
|
||||
@@ -51,10 +54,11 @@ const (
|
||||
// the ask is on its stream.
|
||||
func (h heldOnTheBus) seen(ctx context.Context, a buildstep.StepAsk, later bool) (build.AskSeen, error) {
|
||||
seen := build.AskSeen{Kind: a.Kind, At: build.AskedAt(a.ID)}
|
||||
module, err := buildstep.BuildModuleOf(a.Language)
|
||||
bm, err := h.seats.OfAskLanguage(a.Language)
|
||||
if err != nil {
|
||||
return seen, err
|
||||
}
|
||||
module := bm.Module
|
||||
steps, err := h.js.KeyValue(ctx, buildrun.StepsBucket(module))
|
||||
if err != nil {
|
||||
return seen, fmt.Errorf("the bucket %s: %w", buildrun.StepsBucket(module), err)
|
||||
@@ -259,10 +263,11 @@ func (h heldOnTheBus) Drop(ctx context.Context, a buildstep.StepAsk) error {
|
||||
// every kind, oldest first and at most limit of each kind, each with its step's state and whether it still waits.
|
||||
func (h heldOnTheBus) Queue(ctx context.Context, language string, limit int) (build.SeatQueue, error) {
|
||||
q := build.SeatQueue{Language: language, Note: build.QueueNote}
|
||||
module, err := buildstep.BuildModuleOf(language)
|
||||
bm, err := h.seats.OfAskLanguage(language)
|
||||
if err != nil {
|
||||
return q, err
|
||||
}
|
||||
module := bm.Module
|
||||
stream, name, _, err := h.stream(ctx, language, buildstep.AskDelivery)
|
||||
q.Stream = name
|
||||
if errors.Is(err, jetstream.ErrStreamNotFound) {
|
||||
|
||||
@@ -28,15 +28,12 @@ import (
|
||||
runner "git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep"
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient"
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery"
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/ociartifactregistry"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
|
||||
sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"git.novox.be/novox/mesh-sdk/go/views/viewstest"
|
||||
|
||||
@@ -201,38 +198,22 @@ func (f *fakeBuildModules) serve(t *testing.T, module string, nodes ...string) {
|
||||
}
|
||||
}
|
||||
|
||||
// trunkHead is the trunk head of novox/mesh-catalog the bed's source revision names: newer than the module values' commit.
|
||||
// trunkHead is a trunk head of novox/mesh-catalog.
|
||||
const trunkHead = "3333333333333333333333333333333333333333"
|
||||
|
||||
// trunk writes the gate's source.main naming head as novox/mesh-catalog's trunk head, and answers its revision.
|
||||
func (b *bed) trunk(t *testing.T, head string) uint64 {
|
||||
t.Helper()
|
||||
data, err := meshdeliveryclient.EncodeSource(meshdelivery.SourceByBranchValue{V: keys.Version, Group: "g", Heads: map[string]string{"novox/mesh-catalog": head}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rev, err := b.source.Put(b.ctx, meshdeliveryclient.SourceKey("main"), data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return rev
|
||||
}
|
||||
|
||||
type bed struct {
|
||||
ctx context.Context
|
||||
js jetstream.JetStream
|
||||
modules jetstream.KeyValue
|
||||
candidates jetstream.KeyValue
|
||||
source jetstream.KeyValue
|
||||
builds jetstream.KeyValue
|
||||
heard jetstream.KeyValue // the state toolchains-heard
|
||||
instances jetstream.KeyValue
|
||||
packages jetstream.KeyValue // the indexer's packages, served by its read tool and views family
|
||||
out *saidLog // the builder's process log, every instance's
|
||||
stop func() // stops the builder started last
|
||||
fake *fakeBuildModules
|
||||
srvURL string
|
||||
dir string
|
||||
ctx context.Context
|
||||
js jetstream.JetStream
|
||||
catalog *catalogHolder // the seat mesh-catalog, as its holder answers the branch follower
|
||||
source jetstream.KeyValue
|
||||
builds jetstream.KeyValue
|
||||
heard jetstream.KeyValue // the state toolchains-heard
|
||||
instances jetstream.KeyValue
|
||||
out *saidLog // the builder's process log, every instance's
|
||||
stop func() // stops the builder started last
|
||||
fake *fakeBuildModules
|
||||
srvURL string
|
||||
dir string
|
||||
}
|
||||
|
||||
// startBed starts the bus, the buckets the controller makes from the builder's and the build modules' manifests, the
|
||||
@@ -300,17 +281,13 @@ func startBedWith(t *testing.T, closed bool) *bed {
|
||||
}
|
||||
return kv
|
||||
}
|
||||
b.modules = mk(indexerBucket(modulesState), modulesHistory)
|
||||
b.candidates = mk(indexerBucket(candidatesState), candidatesHistory)
|
||||
b.source = mk(sourceBucket, sourceHistory)
|
||||
mk(indexerBucket(manifestsState), manifestsHistory)
|
||||
// The indexer and the assigner as the owners of the states the builder follows: their read tools and views families
|
||||
// (ADR 0615, DLV-M1); n1 and n2 each have their durable consumer.
|
||||
// The seat mesh-catalog's holder and the assigner as the owners of what the builder follows: the catalog half through the seat's
|
||||
// verbs and events (design 47 G §348, §614), the assignments by their read tool and views family (ADR 0615, DLV-M1); n1 and n2
|
||||
// each have their durable consumer.
|
||||
b.catalog = newCatalogHolder(t, ctx, js, ctl)
|
||||
b.catalog.buildSeats(t, ctl, map[string]string{"node-build-go": testGo, "node-build-node": testNode, "node-build-image": testImage,
|
||||
"git": "mesh-delivery-forge-bridge", "mesh-catalog": "mesh-delivery-indexer"}) // the floor's sources are the holders of git and mesh-catalog
|
||||
readers := []string{"n1_" + builds.BuilderModule, "n2_" + builds.BuilderModule}
|
||||
for _, st := range []string{modulesState, manifestsState, candidatesState} {
|
||||
viewstest.Serve(t, ctx, js, ctl, indexerModule, st, indexerModule+".indexer_read_"+st, readers...)
|
||||
}
|
||||
b.packages = viewstest.Serve(t, ctx, js, ctl, indexerModule, meshcatalog.StatePackages, indexerModule+".indexer_read_packages", readers...)
|
||||
viewstest.Serve(t, ctx, js, ctl, assignerModule, "assignments", assignerModule+".assigner_read_assignments", readers...)
|
||||
b.builds = mk(builds.BuilderBuildsBucket, builds.BuilderBuildsHistory)
|
||||
b.instances = mk(instancesBucket, 1)
|
||||
@@ -404,28 +381,34 @@ func (b *bed) moduleFP(t *testing.T, name, fixture, path string, toolchains map[
|
||||
t.Fatal(err)
|
||||
}
|
||||
digest := meshcatalogclient.ManifestDigest(raw)
|
||||
manifests, err := b.js.KeyValue(b.ctx, indexerBucket(manifestsState))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mv, err := meshcatalogclient.EncodeManifest(meshcatalog.ManifestsByDigestValue{Module: name, Manifest: string(raw)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := manifests.Put(b.ctx, digest, mv); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
data, err := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-catalog", Commit: at, Fp: "src1:" + strings.Repeat("f", 64), Fp2: fp2,
|
||||
Manifest: digest, Path: path, Toolchains: build.CatalogToolchains(toolchains), Src: int64(src)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := b.modules.Put(b.ctx, name, data); err != nil {
|
||||
var manifest map[string]any
|
||||
if err := json.Unmarshal(raw, &manifest); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.catalog.put(t, b.entry(name, path, at, fp2, digest, manifest, toolchains, src))
|
||||
return digest
|
||||
}
|
||||
|
||||
// manifestOf is a fixture manifest as the entry holds it.
|
||||
func (b *bed) manifestOf(t *testing.T, fixture string) map[string]any {
|
||||
t.Helper()
|
||||
raw, err := os.ReadFile(filepath.Join("testdata", fixture))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var m map[string]any
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// entry is the catalog graph entry of a module as the holder answers it.
|
||||
func (b *bed) entry(name, path, at, fp2, digest string, manifest map[string]any, toolchains map[string]builds.ToolchainUsed, src uint64) meshcatalog.CatalogEntry {
|
||||
return meshcatalog.CatalogEntry{Module: name, Repository: "novox/mesh-catalog", Commit: at, Fp: fp2, Path: path, ManifestDigest: digest, Manifest: manifest,
|
||||
Inputs: &meshcatalog.EntryInputs{Toolchains: build.CatalogToolchains(toolchains), Src: int64(src)}}
|
||||
}
|
||||
|
||||
// archive writes modules.<name> as the real indexer does for a module of an archive only: its fingerprint names no
|
||||
// toolchain (mesh-delivery-indexer internal/index inputs.go languagesOf gives an archive no language), so Toolchains is empty.
|
||||
func (b *bed) archive(t *testing.T, name, path, at string) string {
|
||||
@@ -459,6 +442,12 @@ type saidLog struct {
|
||||
buf strings.Builder
|
||||
}
|
||||
|
||||
func (l *saidLog) String() string {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
return l.buf.String()
|
||||
}
|
||||
|
||||
func (l *saidLog) Write(p []byte) (int, error) {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
@@ -588,11 +577,12 @@ func TestBuilderMakesADoneKeyWithTheDigests(t *testing.T) { // 1607/AC7
|
||||
t.Errorf("%s called %d times, want %d (on one node, ADR 0478 clause 4)", tool, got, want)
|
||||
}
|
||||
}
|
||||
// Design 32, ADR 0615 §1: a package's name the indexer writes later reaches the builder by its packages' views family
|
||||
// (never their bucket), and looks at the module again; the done bundles are kept, not numbered twice.
|
||||
if _, err := b.packages.Put(b.ctx, "netcheck.0.1", []byte(`{"v":1,"name":"netcheck","package":"netcheck/lib","series":"0.1"}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Design 32, design 47 G §348: a package's name the catalog half's entry of the module gives later reaches the builder by the
|
||||
// branch follower's view (never the indexer's state), and looks at the module again; the done bundles are kept, not numbered twice.
|
||||
named := b.entry("netcheck", "modules/netcheck", commit, fp, digest, nil, bothToolchains, 0)
|
||||
named.Manifest = b.manifestOf(t, "catalogue-a67f0196-netcheck.module.json")
|
||||
named.Artifacts = []meshcatalog.Artifact{{Ref: "netcheck:lib", Language: "go", Package: &meshcatalog.ArtifactPackage{Ecosystem: "go", Name: "netcheck"}}}
|
||||
b.catalog.put(t, named)
|
||||
b.awaitSaid(t, "names a package of netcheck")
|
||||
// Design 47 G §349: version.numbered is no longer said, for a number or for a late name.
|
||||
if got := reg.numberedOf("netcheck/tools-go"); len(got) != 0 {
|
||||
@@ -647,7 +637,7 @@ func TestBuilderMakesADoneKeyWithTheDigests(t *testing.T) { // 1607/AC7
|
||||
reg.awaitFinding(t, failed, true)
|
||||
b.fake.hang.Store(nil)
|
||||
close(release)
|
||||
b.moduleFP(t, "netcheck", netcheck, "modules/netcheck", bothToolchains, commit, third, 3) // the indexer writes it again
|
||||
b.moduleFP(t, "netcheck", netcheck, "modules/netcheck", bothToolchains, "89abcdef0123456789abcdef0123456789abcdef", third, 3) // the entry moves to a later commit, same fingerprint
|
||||
b.awaitFP(t, "netcheck", third, isState(builds.BuildDone))
|
||||
b.awaitSaid(t, "kept: fingerprint "+third+" is numbered under it")
|
||||
if !slices.ContainsFunc(b.logged(t, "netcheck"), func(e builds.BuildLogEntry) bool {
|
||||
@@ -747,7 +737,7 @@ func TestAToolchainChangeDoesNotBuildTheOldFingerprint(t *testing.T) { // 1615/A
|
||||
map[string]builds.ToolchainUsed{"go": {Version: "go1.27.2", Recipe: "1"}, "typescript": {Version: "tsc5", Recipe: "1"}}, commit)
|
||||
b.await(t, "netcheck2", isState(builds.BuildDone))
|
||||
if _, err := b.builds.Get(b.ctx, mustKey(t, "netcheck")); err == nil {
|
||||
t.Error("the old fingerprint, naming go1.27.1, was built with go1.27.2")
|
||||
t.Error("the old fingerprint, naming go1.27.1, was built with go1.27.2; the log: " + b.out.String())
|
||||
}
|
||||
if n := b.fake.count("mesh-build-go.build_go_build"); n != 1 {
|
||||
t.Errorf("build_go_build was called %d times, want 1 (the new fingerprint on one node only)", n)
|
||||
@@ -786,77 +776,14 @@ func TestAKeyLeftBuildingForAModuleThatIsGoneIsClosedAtStart(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestBuilderBuildsACandidatesFingerprintFromTheHeadMergedOntoTheTrunk: the trunk's record of netcheck carries a fingerprint of its
|
||||
// own; a candidate the indexer wrote gives netcheck the fingerprint fp at a pull request's head; the program writes
|
||||
// builds.netcheck.<fp> done, its fetch given the head, the ref that carries it and the trunk head of the candidate's source
|
||||
// revision, newer than the module value's commit (issue 1684; the operator 2026-10-25: a value names the commit its
|
||||
// fingerprint first appeared at).
|
||||
func TestBuilderBuildsACandidatesFingerprintFromTheHeadMergedOntoTheTrunk(t *testing.T) { // N7/AC1
|
||||
// TestAnImageIsBuiltOnOneNodeAndEveryFingerprintOfTheCatalogHalfPublishes: the catalog half's entry of imaged (one image
|
||||
// artifact) is built on n1, where mesh-build-image runs: its key is done with the digest and the reference, its build call carrying
|
||||
// publish; the entry's next fingerprint is built again with publish and ends with a reference of its own (ADR 0487 clauses 5 and
|
||||
// 12; a run's candidate is no input of the builder: design 47 G A-84).
|
||||
func TestAnImageIsBuiltOnOneNodeAndEveryFingerprintOfTheCatalogHalfPublishes(t *testing.T) { // 1705
|
||||
b := startBed(t)
|
||||
other := "src2:" + strings.Repeat("e", 64)
|
||||
digest := b.module(t, "netcheck", "catalogue-a67f0196-netcheck.module.json", "modules/netcheck")
|
||||
data, err := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-catalog", Commit: commit, Fp: "src1:" + strings.Repeat("f", 64), Fp2: other,
|
||||
Manifest: digest, Path: "modules/netcheck", Toolchains: build.CatalogToolchains(map[string]builds.ToolchainUsed{"go": {Version: "go1.27.1", Recipe: "1"}, "typescript": {Version: "tsc5", Recipe: "1"}})})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := b.modules.Put(b.ctx, "netcheck", data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
const head = "1111111111111111111111111111111111111111"
|
||||
pulls := []meshcatalog.CandidatesByGroupValuePullsItem{{Repository: "novox/mesh-catalog", Number: 7, Head: head, Modules: []string{"netcheck"},
|
||||
Fps: map[string]meshcatalog.CandidatesByGroupValuePullsItemFpsValue{"netcheck": {Fp2: fp, Manifest: digest}}}}
|
||||
hash, _ := meshdeliveryclient.HeadsHash([]meshdeliveryclient.Head{{Repository: pulls[0].Repository, Number: int(pulls[0].Number), Head: pulls[0].Head}})
|
||||
cand, err := json.Marshal(meshcatalog.CandidatesByGroupValue{V: keys.Version, Group: "feat/x", Base: int64(b.trunk(t, trunkHead)), Heads: hash, Pulls: pulls, Fpset: meshcatalogclient.FingerprintSet(pulls)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ckey, err := keys.EscapeName("feat/x"), error(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := b.candidates.Put(b.ctx, ckey, cand); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.start(t, "n1", nil)
|
||||
v := b.await(t, "netcheck", isState(builds.BuildDone))
|
||||
if len(v.Artifacts) != 2 {
|
||||
t.Fatalf("build = %+v", v)
|
||||
}
|
||||
b.fake.mu.Lock()
|
||||
defer b.fake.mu.Unlock()
|
||||
found := false
|
||||
for _, f := range b.fake.fetches {
|
||||
if f["commit"] == head && f["ref"] == "refs/pull/7/head" && f["base"] == trunkHead {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Errorf("fetches = %v, want one of the candidate's head with its ref and the trunk head it was previewed onto, not the module value's commit", b.fake.fetches)
|
||||
}
|
||||
}
|
||||
|
||||
// TestAnImageIsBuiltOnOneNodeAndOnlyATrunkKeyPublishes: modules.imaged (one image artifact) is built on n1, where
|
||||
// mesh-build-image runs: a trunk record's key is done with the digest and the reference, its build call carrying publish; a
|
||||
// candidate's key for the same module is built first with no publish and no reference; when the pull request is merged and the
|
||||
// trunk's record names the candidate's fingerprint, the key is built again with publish and ends with a reference
|
||||
// (ADR 0487 clauses 5 and 12).
|
||||
func TestAnImageIsBuiltOnOneNodeAndOnlyATrunkKeyPublishes(t *testing.T) { // 1705
|
||||
b := startBed(t)
|
||||
other := "src2:" + strings.Repeat("e", 64)
|
||||
digest := b.moduleCommit(t, "imaged", "catalogue-a67f0196-imaged.module.json", "modules/imaged", nil, commit)
|
||||
// The trunk carries `other`; its key is built and published.
|
||||
trunk := func(fp2 string) {
|
||||
data, err := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Repository: "novox/mesh-catalog", Commit: commit, Fp: "src1:" + strings.Repeat("f", 64), Fp2: fp2,
|
||||
Manifest: digest, Path: "modules/imaged"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := b.modules.Put(b.ctx, "imaged", data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
trunk(other)
|
||||
digest := b.moduleFP(t, "imaged", "catalogue-a67f0196-imaged.module.json", "modules/imaged", nil, commit, other, 0)
|
||||
b.start(t, "n1", nil)
|
||||
okey, _ := builds.BuilderBuildKey("imaged", other)
|
||||
awaitKey := func(key string, pred func(builds.BuilderBuild) bool) builds.BuilderBuild {
|
||||
@@ -868,42 +795,23 @@ func TestAnImageIsBuiltOnOneNodeAndOnlyATrunkKeyPublishes(t *testing.T) { // 170
|
||||
}
|
||||
}
|
||||
}
|
||||
t.Fatalf("builds.%s = %+v within 20 s", key, last)
|
||||
t.Fatalf("builds.%s = %+v within 20 s; the log: %s", key, last, b.out.String())
|
||||
return last
|
||||
}
|
||||
v := awaitKey(okey, isState(builds.BuildDone))
|
||||
if len(v.Artifacts) != 1 || v.Artifacts[0].Digest != digestOf("server") || !strings.HasSuffix(v.Artifacts[0].Reference, "@"+digestOf("server")) {
|
||||
t.Fatalf("trunk build = %+v, want the digest and the reference", v)
|
||||
t.Fatalf("build = %+v, want the digest and the reference", v)
|
||||
}
|
||||
// The candidate carries fp: built, not published.
|
||||
const head = "1111111111111111111111111111111111111111"
|
||||
pulls := []meshcatalog.CandidatesByGroupValuePullsItem{{Repository: "novox/mesh-catalog", Number: 7, Head: head, Modules: []string{"imaged"},
|
||||
Fps: map[string]meshcatalog.CandidatesByGroupValuePullsItemFpsValue{"imaged": {Fp2: fp, Manifest: digest}}}}
|
||||
hash, _ := meshdeliveryclient.HeadsHash([]meshdeliveryclient.Head{{Repository: pulls[0].Repository, Number: int(pulls[0].Number), Head: pulls[0].Head}})
|
||||
cand, err := json.Marshal(meshcatalog.CandidatesByGroupValue{V: keys.Version, Group: "feat/x", Base: int64(b.trunk(t, trunkHead)), Heads: hash, Pulls: pulls, Fpset: meshcatalogclient.FingerprintSet(pulls)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
// The module's next fingerprint: built once more, published.
|
||||
if got := b.moduleFP(t, "imaged", "catalogue-a67f0196-imaged.module.json", "modules/imaged", nil, commit, fp, 0); got != digest {
|
||||
t.Fatalf("manifest digest = %s, want %s", got, digest)
|
||||
}
|
||||
ckey := keys.EscapeName("feat/x")
|
||||
if _, err := b.candidates.Put(b.ctx, ckey, cand); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v = b.await(t, "imaged", isState(builds.BuildDone))
|
||||
if len(v.Artifacts) != 1 || v.Artifacts[0].Reference != "" {
|
||||
t.Fatalf("candidate build = %+v, want done with no reference", v)
|
||||
}
|
||||
b.fake.mu.Lock()
|
||||
if len(b.fake.builds) != 2 || b.fake.builds[0]["publish"] != true || b.fake.builds[1]["publish"] != nil {
|
||||
t.Errorf("builds = %v, want the trunk's with publish, then the candidate's with none", b.fake.builds)
|
||||
}
|
||||
b.fake.mu.Unlock()
|
||||
// The pull request is merged: the trunk's record names fp, and its key ends published, built once more with publish.
|
||||
trunk(fp)
|
||||
v = b.await(t, "imaged", func(v builds.BuilderBuild) bool {
|
||||
fkey, _ := builds.BuilderBuildKey("imaged", fp)
|
||||
v = awaitKey(fkey, func(v builds.BuilderBuild) bool {
|
||||
return v.State == builds.BuildDone && len(v.Artifacts) == 1 && v.Artifacts[0].Reference != ""
|
||||
})
|
||||
if v.Artifacts[0].Digest != digestOf("server") {
|
||||
t.Errorf("merged build = %+v", v)
|
||||
t.Errorf("second build = %+v", v)
|
||||
}
|
||||
b.fake.mu.Lock()
|
||||
defer b.fake.mu.Unlock()
|
||||
@@ -913,8 +821,8 @@ func TestAnImageIsBuiltOnOneNodeAndOnlyATrunkKeyPublishes(t *testing.T) { // 170
|
||||
published++
|
||||
}
|
||||
}
|
||||
if len(b.fake.builds) != 3 || published != 2 {
|
||||
t.Errorf("builds = %v, want three, two of them with publish (the trunk's two fingerprints)", b.fake.builds)
|
||||
if len(b.fake.builds) != 2 || published != 2 {
|
||||
t.Errorf("builds = %v, want two, both with publish", b.fake.builds)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1085,7 +993,7 @@ func TestARestartedBuilderPutsNoSecondAsk(t *testing.T) {
|
||||
}
|
||||
// 1945 note 62, on the real bus: the stream read as builder_seat_queue reads it shows the Go ask waiting; the ask can
|
||||
// still be answered (nothing was taken since); and a message dropped as an ask asked again is gone from the stream.
|
||||
held := heldOnTheBus{b.js}
|
||||
held := heldOnTheBus{js: b.js, seats: testSeats()}
|
||||
goAsk := sdkstep.StepAsk{ID: v.Asks[sdkstep.AskGo], Language: sdkstep.AskGo, Kind: sdkstep.AskDelivery}
|
||||
stale := sdkstep.StepAsk{ID: "mvo53jcm70b5f7", Language: sdkstep.AskGo, Kind: sdkstep.AskDelivery}
|
||||
subject, _ := sdkstep.AskSubjectBy(sdkstep.AskGo, sdkstep.AskDelivery, sdkstep.AskerBuilder)
|
||||
|
||||
@@ -12,13 +12,10 @@ import (
|
||||
"io"
|
||||
"io/fs"
|
||||
"log"
|
||||
"maps"
|
||||
"os"
|
||||
"os/signal"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
@@ -34,14 +31,13 @@ import (
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient"
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
|
||||
sweep "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshartifactsweep"
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds"
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
|
||||
"git.novox.be/novox/mesh-delivery/sdk/buildstep"
|
||||
"git.novox.be/novox/mesh-delivery/sdk/trunk"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"git.novox.be/novox/mesh-sdk/go/seatlist"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
|
||||
"mesh-delivery-builder/internal/build"
|
||||
@@ -82,10 +78,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// The indexer's modules, manifests, candidates and packages and the assigner's assignments are their owners' states, read by the
|
||||
// first view of their read tools and then their views families, never their buckets (ADR 0615 §1, DLV-M1). An owner that
|
||||
// is down never holds the start: each is read on a goroutine once its first view is answered, and until then the
|
||||
// unanswered first view is the builder's finding.
|
||||
// The assigner's assignments are its state, read by the first view of its read tool and then its views family, never its
|
||||
// bucket (ADR 0615 §1, DLV-M1); the catalog half is read through the seat mesh-catalog. An owner that is down never holds the
|
||||
// start: each is read on a goroutine once its first view is answered, and until then the unanswered first view is the
|
||||
// builder's finding.
|
||||
calls, err := process.Connect(cred, nats.Name(cred.Node+"."+cred.Module+".calls"), nats.RetryOnFailedConnect(true))
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -109,24 +105,20 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
}
|
||||
})
|
||||
reader.Findings = func() views.WireWriter { return fw }
|
||||
modules, err := reader.Follow(indexerModule, modulesState, indexerModule+".indexer_read_modules")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
manifests, err := reader.Follow(indexerModule, manifestsState, indexerModule+".indexer_read_manifests")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// The candidates' fingerprints are built too (issue 1684).
|
||||
candidates, err := reader.Follow(indexerModule, candidatesState, indexerModule+".indexer_read_candidates")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// The indexer's packages, whose names a package is numbered under (design 32).
|
||||
packages, err := reader.Follow(indexerModule, meshcatalog.StatePackages, indexerModule+".indexer_read_packages")
|
||||
// The catalog half per watched branch (design 47 G §348, §479, §614; A-5, A-15): the shared branch follower of
|
||||
// meshcatalogclient on the builder's reader, so the modules, their manifests, their packages and each package's name in its
|
||||
// ecosystem are read from the entries, never the indexer's states.
|
||||
half := &catalogHalf{manifests: &entryManifests{}, logf: logger.Printf, ctx: ctx}
|
||||
branches, err := meshcatalogclient.NewBranches(sweepWiring{meshcatalogclient.ProcessWiring(reader), half}, calls.Call)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
branches.Logf = logger.Printf
|
||||
half.branches = branches
|
||||
// The build modules are no names (design 47 G A-82): the holders of the build seats, the seat list's definitions and the
|
||||
// catalog half's claims, found at each use.
|
||||
seatGate := newSeatGate()
|
||||
seats := buildrun.NewBuildSeats(seatGate.definitions, versions.HolderOf(branches))
|
||||
assignments, err := reader.Follow(assignerModule, "assignments", assignerModule+".assigner_read_assignments")
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -137,13 +129,14 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
return fmt.Errorf("MESH_BUILDER_STEP_BOUND: %w", err)
|
||||
}
|
||||
}
|
||||
tools := &buildrun.ToolchainSet{}
|
||||
tools := &buildrun.ToolchainSet{Seats: seats}
|
||||
// Each build module's part of a build is an ask on its build seat's stream, pulled by a build node with a free slot
|
||||
// (novox/hq ADR 0525): the builder chooses no node, so it reads neither presence nor have and holds no capacity.
|
||||
// The read of an ask's steps is a kept call (design 47 G §568): a fault ends no key, and past five minutes unanswered it is a
|
||||
// finding; its bound is a key of the state bounds, bound once the bus holds it (set below, with the findings).
|
||||
stepReads := &stepsRead{bounds: func() jetstream.KeyValue { return nil }, bound: meshconditionsclient.CallUnansweredAfter, now: time.Now, logf: logger.Printf}
|
||||
asks := buildrun.JetStreamAsks{JS: js, Unanswered: stepReads.unanswered, Answered: stepReads.answered}
|
||||
// The steps of each build module are watched once the seat list and the catalog half name it (seatsChanged).
|
||||
// The asks the build seats gave up on (issue 734): heard from DEAD_LETTERS, each ends the wait for its step at once,
|
||||
// and the lane asks its part again (Builder.waitOnce) instead of waiting for the check bound.
|
||||
// Heard in the background, and again at a reconnect or an ask while it is not; whether it is heard is kept in the
|
||||
@@ -157,13 +150,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
defer hearing.Stop()
|
||||
// Each ask's step is heard by its own watch and, at every step end of its build module and every reconnect, read
|
||||
// (buildrun.HearEnds): a watch that goes deaf loses no end (the lost check of 2026-10-28).
|
||||
var buildModules []string
|
||||
for _, l := range buildstep.AskLanguages {
|
||||
if m, err := buildstep.BuildModuleOf(l); err == nil {
|
||||
buildModules = append(buildModules, m)
|
||||
}
|
||||
}
|
||||
heard := buildrun.HearEnds(ctx, asks, asks, buildModules, logger.Printf)
|
||||
heard := buildrun.HearEnds(ctx, asks, asks, seats.Modules(), logger.Printf)
|
||||
conn.SetReconnectHandler(func(*nats.Conn) { hearing.Again(); heard.Nudge("") })
|
||||
// The builder asks the controller, as its own account, whether the artifact store still holds a done key's blobs
|
||||
// (the seat verb held, novox/hq issue 1937) before it leaves the key done, on its calls connection.
|
||||
@@ -171,31 +158,39 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// the bus holds it; the floor's three sources asked as its own account.
|
||||
versionsKV := process.AwaitBucket(ctx, js, conn, versions.Bucket, "no trunk build is numbered: each is built unnumbered and published by its digest", logger.Printf)
|
||||
found := &findings{w: fw, logf: logger.Printf, now: time.Now}
|
||||
floorWaits := &versions.FloorWaits{Logf: logger.Printf, Said: found.floorSaid(context.WithoutCancel(ctx))}
|
||||
floorWaits := &versions.FloorWaits{Holder: versions.HolderOf(branches), Logf: logger.Printf, Said: found.floorSaid(context.WithoutCancel(ctx))}
|
||||
// The names waiting on their floor are kept in the state floor-waits (DLV-B4): a restarted builder takes them in, so
|
||||
// their silent sources' events still wake them; until the state is bound they are kept here only.
|
||||
process.AwaitBucket(ctx, js, conn, versions.FloorWaitsBucket, "the names waiting on their floor are kept in memory only, and builder_floor_waits shows none", logger.Printf).
|
||||
Ready(func(kv jetstream.KeyValue) {
|
||||
floorWaits.Keep(kvKeeper{ctx: ctx, kv: kv}, heldFloorWaits(ctx, kv, logger.Printf))
|
||||
})
|
||||
// A package is numbered under its name in its ecosystem, read from the indexer's packages state (design 32).
|
||||
numberer := &versions.Numberer{Versions: versions.Versions{Store: awaitedVersions{b: versionsKV}}, Floor: floorVia(calls.Call), Waits: floorWaits,
|
||||
Names: versions.NamesIn(indexerPackages{packages})}
|
||||
// A package is numbered under its name in its ecosystem, read from the catalog half's entries (design 32, design 47 G §348).
|
||||
numberer := &versions.Numberer{Versions: versions.Versions{Store: awaitedVersions{b: versionsKV}}, Floor: floorVia(calls.Call, versions.HolderOf(branches)), Waits: floorWaits,
|
||||
Names: versions.NamesFrom(versions.EntriesOf(branches))}
|
||||
// A publish that ends badly is a finding (design 51 *Health and conditions*): a holder's refused, a version no holder
|
||||
// answered within its bound (a key of the state bounds with its own time to live), a seat's outcome that does not read.
|
||||
pubBoundsKV := process.AwaitBucket(ctx, js, conn, versions.BoundsBucket, "a publish no holder answers, and a node holding the old form seven days, are not raised", logger.Printf)
|
||||
stepReads.bounds, stepReads.findings = func() jetstream.KeyValue { return pubBoundsKV.KV() }, found
|
||||
pubs := &publishes{bounds: pubBoundsKV.KV, versions: numberer.Versions, findings: found, bound: versions.PublishBound, logf: logger.Printf}
|
||||
numberer.Publishing = pubs.arm
|
||||
b := &build.Builder{
|
||||
var b *build.Builder
|
||||
seatsChanged := func(ctx context.Context) {
|
||||
heard.Watch(seats.Modules())
|
||||
if b != nil {
|
||||
b.Reconsider(ctx)
|
||||
}
|
||||
}
|
||||
b = &build.Builder{
|
||||
Claimer: build.Claimer{Store: kvStore{buildsKV}, By: cred.Node + "." + cred.Module},
|
||||
Asks: buildrun.Asker{Publisher: asks, Waiter: buildrun.LostAskWaiter{Inner: heard, Dead: dead, Hearing: hearing}, Bound: bound, Dropper: buildrun.SeatDrops{Caller: calls}},
|
||||
Held: heldOnTheBus{js},
|
||||
Seats: seats,
|
||||
Asks: buildrun.Asker{Seats: seats, Publisher: asks, Waiter: buildrun.LostAskWaiter{Inner: heard, Dead: dead, Hearing: hearing}, Bound: bound, Dropper: buildrun.SeatDrops{Caller: calls}},
|
||||
Held: heldOnTheBus{js: js, seats: seats},
|
||||
// An ask that can no longer be answered is asked again (issue 1945), read at the step bound: a context's bound.
|
||||
Gone: heldOnTheBus{js},
|
||||
Gone: heldOnTheBus{js: js, seats: seats},
|
||||
CheckAfter: bound,
|
||||
Toolchains: tools,
|
||||
Manifests: manifestReader{manifests},
|
||||
Manifests: half.manifests,
|
||||
Logf: logger.Printf,
|
||||
Stored: storedVia(calls.Call),
|
||||
Node: cred.Node,
|
||||
@@ -221,6 +216,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// and version-released heard; nothing is reserved before the fill.
|
||||
reservedKV := process.AwaitBucket(ctx, js, conn, versions.ReservedBucket, "no version is reserved: builds are made unreserved", logger.Printf)
|
||||
reserved := newReservations(awaitedReserved{awaitedVersions{b: reservedKV, bucket: versions.ReservedBucket}}, calls.Call, pubBoundsKV.KV, found, b.Reconsider, logger.Printf)
|
||||
// Each artifact the holder says published at a reserved version is said as the seat mesh-builds' package-published (design 47 G §52).
|
||||
reserved.say = func(ctx context.Context, p meshbuilds.PackagePublished) error {
|
||||
return emitSeatEvent(context.WithoutCancel(ctx), js, cred, meshbuilds.SubjectPackagePublished, "package-published-"+p.Module+"-"+p.Version+"-"+p.Artifact, p)
|
||||
}
|
||||
b.Reserved = reserved
|
||||
// A state bound later numbers the builds looked at without it.
|
||||
versionsKV.Ready(func(jetstream.KeyValue) { b.Reconsider(ctx) })
|
||||
@@ -251,18 +250,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
retake := &retaking{rt: &build.Retaker{Builder: b, Acted: retakeActed, Entry: entryVia(calls.Call), Ask: operator.Ask, Logf: logger.Printf}}
|
||||
queueKV.Ready(again.bound(ctx, js, logger.Printf))
|
||||
actedKV.Ready(again.bound(ctx, js, logger.Printf))
|
||||
// A candidate's preview is merged onto the trunk head the indexer previewed it onto: source.main at the candidate's base
|
||||
// revision (the gate's source state). The source's revisions are watched with their history (a direct get of one revision
|
||||
// is not what a read grants): a revision heard after the candidate that names it builds that candidate then. Where it
|
||||
// cannot be read the trunk's keys are built and the candidates wait (issue 1945).
|
||||
errs := make(chan error, 10)
|
||||
trunks := &trunkSource{ctx: ctx, js: js, heads: &sourceHeads{}, moved: func(ctx context.Context) {
|
||||
// A source revision heard is the forge's merge: it wakes a floor whose bridge_tags was silent (design 51 *The floor*).
|
||||
(&floorWake{b: b, waits: floorWaits, logf: logger.Printf}).wake(ctx, versions.WakeForge, "", "")
|
||||
b.TrunkMoved(ctx)
|
||||
}, errs: errs, logf: logger.Printf}
|
||||
trunks.start()
|
||||
b.Trunk = trunks.head
|
||||
b.MemberBounds = &memberBounds{bounds: pubBoundsKV.KV, now: time.Now, logf: logger.Printf}
|
||||
b.Members = versions.ModuleFrom(versions.EntriesOf(branches)) // the build of a group's package member (design 47 G §373, §465)
|
||||
// The builder's log (ADR 0571 §1): each build's start and end in its module log `build-log`, read by the view, the
|
||||
// indexer and the assigner through their reads, and by the operator's command nox builds through the tool builder_builds_since.
|
||||
b.Log = func(e builds.BuildLogEntry) {
|
||||
@@ -290,7 +280,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
switch kv := process.OptionalBucket(ctx, js, instancesBucket, "the controller is not told that this builder writes references until it starts again", logger.Printf); {
|
||||
case kv == nil:
|
||||
default:
|
||||
says := newSayer(ctx, kv, cred.Node, []string{buildrun.GoBuildModule, buildrun.NodeBuildModule, build.ImageBuildModule}, logger.Printf)
|
||||
says := newSayer(ctx, kv, cred.Node, seats.Modules, logger.Printf)
|
||||
b.Answered = says.Answered
|
||||
defer func() {
|
||||
stopping, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
|
||||
@@ -393,7 +383,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// condition keeper's condition-cleared.
|
||||
floor := floorWake{b: b, waits: floorWaits, logf: logger.Printf}
|
||||
// And the four registry seats' published, refused and removed (DLV-B4), which move a numbered version.
|
||||
seats := seatHearer{n: numberer, b: b, logf: logger.Printf, pub: pubs, res: reserved}
|
||||
seatHeard := seatHearer{n: numberer, b: b, logf: logger.Printf, pub: pubs, res: reserved}
|
||||
events := func(m jetstream.Msg) {
|
||||
if again.heard(ctx, m, logger.Printf) { // the publish again's own subjects, left for the bus to give again when its state cannot take them
|
||||
return
|
||||
@@ -404,17 +394,29 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
if retake.heard(ctx, m, logger.Printf) { // a key failed on a transport fault, built again when its cause passed (§568)
|
||||
return
|
||||
}
|
||||
if reads.heard(m, logger.Printf) || heardTools.Reported(ctx, m) || heardTools.Running(ctx, m) || floor.heard(ctx, m) || reserved.heard(ctx, m) || seats.heard(ctx, m) { // reads: the node-engine seat's manifest-reads-changed (ADR 0618 §4)
|
||||
if reads.heard(m, logger.Printf) || heardTools.Reported(ctx, m) || heardTools.Running(ctx, m) || floor.heard(ctx, m) || reserved.heard(ctx, m) || seatHeard.heard(ctx, m) { // reads: the node-engine seat's manifest-reads-changed (ADR 0618 §4)
|
||||
return
|
||||
}
|
||||
heardGone(ctx, b, m, logger.Printf)
|
||||
}
|
||||
// The seats' events (artifacts-gone, the registry seats' words, manifest-accepted and manifest-reads-changed and the
|
||||
// seat mesh-conditions' condition-cleared, which this pattern covers: a consumer takes no two overlapping filters); the
|
||||
// owners' views events are the views reader's.
|
||||
if err := hub.Handle("mesh.seat.*.event.>", events); err != nil {
|
||||
// The seats' events (artifacts-gone, the registry seats' words, the build seats' toolchain-reported, manifest-accepted and
|
||||
// manifest-reads-changed and the seat mesh-conditions' condition-cleared: seatEventPatterns); the owners' views events and the
|
||||
// catalog half's catalog-revision and registration-changed and the gate's source-moved are the views reader's (a message has
|
||||
// one taker, so no pattern here covers them).
|
||||
// The seat list over the connection and heard on the hub: a change of it, as of the catalog half's holders, names the build
|
||||
// modules again (design 47 G A-82, §204).
|
||||
// The runner's build step ask, the builder's one entry point for a candidate (design 47 G A-97).
|
||||
b.StepEntry = newEntryWaits(branches, half.manifests).entry
|
||||
if err := serveSteps(ctx, conn, js, cred, b, logger.Printf); err != nil {
|
||||
return err
|
||||
}
|
||||
seatGate.Follow(seatlist.ProcessBus(calls.Call, hub), func() { seatsChanged(ctx) }, logger.Printf)
|
||||
half.holders = func() { seatsChanged(ctx) }
|
||||
for _, pattern := range seatEventPatterns {
|
||||
if err := hub.Handle(pattern, events); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
hub.OnBound(reads.atBind) // what a node's engine said before the consumer was bound (ADR 0668 §4)
|
||||
if stopped != nil {
|
||||
return stopped
|
||||
@@ -426,10 +428,16 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
return hub.Start(ctx, js, cred)
|
||||
})
|
||||
go func() { <-ctx.Done(); hub.Stop() }()
|
||||
// The modules, once their first views and the manifests' are answered: what a module builds is read from the manifests.
|
||||
go func() { errs <- buildFromModules(ctx, b, modules, manifests, candidates) }()
|
||||
// A package's name heard in the indexer's packages looks at its module again, so a done build is numbered then.
|
||||
go func() { errs <- followPackages(ctx, packages, b, logger.Printf) }()
|
||||
// The catalog half's entries as they come: a module's entry of a branch goes to the builder, and a key left building for a
|
||||
// fingerprint no branch carries is closed (issue 1660) once the first view of every followed branch is in (never at the start:
|
||||
// a holder of the seat that is down holds no other part of it).
|
||||
half.b = b
|
||||
half.ready = func() {
|
||||
if err := b.Sweep(ctx); err != nil {
|
||||
errs <- err
|
||||
}
|
||||
}
|
||||
branches.Changed = half.Changed(ctx)
|
||||
// A person asks a key's build again with builder_ask_again, which writes the reasks state (issue 1945): watched, as
|
||||
// every other input. The builder asks again by itself without it.
|
||||
// A state not on the bus yet at this start is watched once it appears (issue 1790).
|
||||
@@ -454,6 +462,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
stepReads.expired(ctx, e)
|
||||
return
|
||||
}
|
||||
if key, ok := versions.MemberBoundOf(e.Key()); ok {
|
||||
if e.Operation() == jetstream.KeyValuePurge { // its 60 minutes ended: the dependent's step fails naming the member
|
||||
b.MemberBoundEnded(ctx, key)
|
||||
}
|
||||
return
|
||||
}
|
||||
if node, ok := enginereads.BoundOf(e.Key()); ok {
|
||||
if e.Operation() == jetstream.KeyValuePurge { // its five minutes ended
|
||||
reads.expiredCall(node)
|
||||
@@ -473,7 +487,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// A person reads a build seat's stream with builder_seat_queue, which writes the queues state (issue 1945 note 62): the
|
||||
// builder alone may read the stream, so it answers there.
|
||||
process.AwaitBucket(ctx, js, conn, build.QueuesBucket, "builder_seat_queue is not answered", logger.Printf).Ready(func(kv jetstream.KeyValue) {
|
||||
eachRead := func(e jetstream.KeyValueEntry) { queueRead(ctx, heldOnTheBus{js}, kv, e, logger.Printf) }
|
||||
eachRead := func(e jetstream.KeyValueEntry) {
|
||||
queueRead(ctx, heldOnTheBus{js: js, seats: seats}, kv, e, logger.Printf)
|
||||
}
|
||||
if qw, err := settle(ctx, kv, eachRead); err != nil {
|
||||
process.NotWatched(logger.Printf, build.QueuesBucket, err, "builder_seat_queue is not answered until the builder starts again")
|
||||
} else {
|
||||
@@ -509,102 +525,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// keeper raises a first view unanswered past its bound (ADR 0615 §6).
|
||||
var firstViewsBucket = builds.BuilderModule + "_" + views.StatusState
|
||||
|
||||
// The indexer's states the builder follows (indexerBucket(modulesState), ManifestsBucket, CandidatesBucket), and the owners'
|
||||
// views families as their manifests' `streams` name them.
|
||||
const (
|
||||
modulesState = "modules"
|
||||
manifestsState = "manifests"
|
||||
candidatesState = "candidates"
|
||||
)
|
||||
|
||||
// buildFromModules builds from the indexer's modules and candidates once the first views of the modules, the manifests and
|
||||
// the candidates are answered (never at the start: an indexer that is down holds no other part of it), and follows them.
|
||||
func buildFromModules(ctx context.Context, b *build.Builder, modules, manifests, candidates *views.Picture) error {
|
||||
for _, p := range []*views.Picture{modules, manifests, candidates} {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
case <-p.Follower().Ready():
|
||||
}
|
||||
}
|
||||
each := func(e jetstream.KeyValueEntry) {
|
||||
if opOf(e) {
|
||||
b.Module(ctx, e.Key(), e.Value())
|
||||
} else {
|
||||
b.Forget(e.Key())
|
||||
}
|
||||
}
|
||||
w, err := settle(ctx, modules, each)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// A key left building for a fingerprint its module no longer has is closed (issue 1660); the modules are all known now.
|
||||
if err := b.Sweep(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
// The fingerprints the indexer gives the candidates' modules are built too (issue 1684), once the trunk's modules are known.
|
||||
eachCandidate := func(e jetstream.KeyValueEntry) {
|
||||
if opOf(e) {
|
||||
b.Candidate(ctx, e.Key(), e.Value())
|
||||
} else {
|
||||
b.ForgetCandidate(e.Key())
|
||||
}
|
||||
}
|
||||
cw, err := settle(ctx, candidates, eachCandidate)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
go func() { _ = follow(ctx, cw, eachCandidate) }()
|
||||
return follow(ctx, w, each)
|
||||
}
|
||||
|
||||
// sourceHeads are the trunk heads of each revision of the gate's source.main the watch delivered, the newest
|
||||
// sourceHistory of them: what a candidate's base revision names.
|
||||
type sourceHeads struct {
|
||||
mu sync.Mutex
|
||||
byRev map[uint64]map[string]string
|
||||
}
|
||||
|
||||
// put takes one revision in and answers whether it is new.
|
||||
func (s *sourceHeads) put(e jetstream.KeyValueEntry) bool {
|
||||
if !opOf(e) {
|
||||
return false
|
||||
}
|
||||
src, err := meshdeliveryclient.DecodeSource(e.Value())
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.byRev == nil {
|
||||
s.byRev = map[uint64]map[string]string{}
|
||||
}
|
||||
if _, had := s.byRev[e.Revision()]; had {
|
||||
return false
|
||||
}
|
||||
s.byRev[e.Revision()] = src.Heads
|
||||
for len(s.byRev) > sourceHistory {
|
||||
delete(s.byRev, slices.Min(slices.Collect(maps.Keys(s.byRev))))
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// head is the head of repository in revision rev; an error when the revision is not heard (yet, or any more).
|
||||
func (s *sourceHeads) head(_ context.Context, rev uint64, repository string) (string, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
heads, ok := s.byRev[rev]
|
||||
if !ok {
|
||||
return "", fmt.Errorf("source revision %d is not heard (it is built once it is)", rev)
|
||||
}
|
||||
for r, head := range heads {
|
||||
if trunk.SameRepository(r, repository) {
|
||||
return head, nil
|
||||
}
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
|
||||
// toolchainWaitOf reads the toolchain wait's setting, toolchain-wait-minutes, from the file the manifest declares with the
|
||||
// content ${config:toolchain-wait-minutes} (design 47 G §672, A-71), at the builder's start: no file, or an empty one, is
|
||||
// the default; a content that is no number of minutes is an error naming the file.
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"mesh-delivery-builder/internal/versions"
|
||||
)
|
||||
|
||||
// memberBounds keeps the bound of a dependent's wait for a package member of a group's run (design 47 G §373, A-77): a key of
|
||||
// the state bounds with its own time to live, whose expiry the watch of the state bounds hears (Builder.MemberBoundEnded).
|
||||
// No timer.
|
||||
type memberBounds struct {
|
||||
bounds func() jetstream.KeyValue
|
||||
now func() time.Time
|
||||
logf func(string, ...any)
|
||||
}
|
||||
|
||||
// Arm is build.MemberBounds' Arm.
|
||||
func (m *memberBounds) Arm(ctx context.Context, key, why string, bound time.Duration) error {
|
||||
kv := m.bounds()
|
||||
if kv == nil {
|
||||
return fmt.Errorf("%s is not on the bus yet", versions.BoundsBucket)
|
||||
}
|
||||
if _, err := kv.Create(ctx, versions.MemberBoundKey(key), versions.EncodeBound(why, m.now().Add(bound)), jetstream.KeyTTL(bound)); err != nil && !errors.Is(err, jetstream.ErrKeyExists) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// End is build.MemberBounds' End.
|
||||
func (m *memberBounds) End(ctx context.Context, key string) {
|
||||
if kv := m.bounds(); kv != nil {
|
||||
if err := kv.Delete(ctx, versions.MemberBoundKey(key)); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
m.logf("the member bound of %s is not deleted: %v", key, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -18,20 +18,24 @@ const (
|
||||
indexerModule = "mesh-delivery-indexer"
|
||||
// assignerModule is the holder of the state assignments.
|
||||
assignerModule = buildrun.AssignerModule
|
||||
|
||||
// sourceBucket is the gate's state source (reads: mesh-delivery-gate.source), the heads a candidate's base names.
|
||||
sourceBucket = gateModule + "_source"
|
||||
// packagesBucket is the indexer's state packages.
|
||||
packagesBucket = indexerModule + "_" + meshcatalog.StatePackages
|
||||
|
||||
// The revisions each keeps per key (the beds of the integration test create them alike).
|
||||
sourceHistory = 64
|
||||
candidatesHistory = 10
|
||||
modulesHistory = 20
|
||||
manifestsHistory = 1
|
||||
)
|
||||
|
||||
func indexerBucket(state string) string { return indexerModule + "_" + state }
|
||||
|
||||
// namedTreePrefix opens the tree input of a module whose every artifact names its paths.
|
||||
const namedTreePrefix = "named:"
|
||||
|
||||
// seatEventPatterns are the seats' events the builder's own handler takes off its durable consumer (its manifest's `consumes` of
|
||||
// them): the registry seats' words, artifacts-gone, the build seats' toolchain-reported, condition-cleared, the node-engine
|
||||
// seat's events and, of the seat mesh-catalog, manifest-accepted, version-released and versions-filled. The seat mesh-catalog's
|
||||
// catalog-revision and registration-changed and the seat mesh-delivery's source-moved are the views reader's (the branch
|
||||
// follower's), so no pattern here covers them: a message has one taker.
|
||||
var seatEventPatterns = func() []string {
|
||||
var out []string
|
||||
for _, seat := range []string{"container-image-registry", "go-package-registry", "npm-package-registry", "oci-artifact-registry",
|
||||
"mesh-artifact-sweep", "mesh-conditions", "node-engine", "node-build-go", "node-build-image", "node-build-node"} {
|
||||
out = append(out, "mesh.seat."+seat+".event.>")
|
||||
}
|
||||
for _, event := range []string{meshcatalog.EventManifestAccepted, meshcatalog.EventVersionReleased, meshcatalog.EventVersionsFilled} {
|
||||
out = append(out, "mesh.seat."+meshcatalog.Seat+".event."+event, "mesh.seat."+meshcatalog.Seat+".event."+event+".>")
|
||||
}
|
||||
return out
|
||||
}()
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
|
||||
@@ -25,22 +26,26 @@ func TestTheNamesAreTheManifests(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, read := range []string{
|
||||
gateModule + ".source",
|
||||
"mesh-build-go.steps", "mesh-build-image.steps", "mesh-build-node.steps",
|
||||
"node-build-go.steps", "node-build-image.steps", "node-build-node.steps",
|
||||
} {
|
||||
if !slices.Contains(m.Reads, read) {
|
||||
t.Errorf("the builder reads %s, its manifest's reads are %v", read, m.Reads)
|
||||
}
|
||||
}
|
||||
for _, view := range []string{indexerModule + ".views.modules", indexerModule + ".views.manifests", indexerModule + ".views.candidates",
|
||||
indexerModule + ".views.packages", assignerModule + ".views.assignments"} {
|
||||
// The catalog half (design 47 G §348, §614) is followed through the seat, and no state of the indexer is.
|
||||
for _, c := range m.Consumes {
|
||||
if strings.HasPrefix(c, indexerModule+".") {
|
||||
t.Errorf("the builder follows %s, an indexer state, its manifest consumes it", c)
|
||||
}
|
||||
}
|
||||
for _, view := range []string{"mesh-catalog.catalog-revision", "mesh-catalog.registration-changed", "mesh-delivery.source-moved", assignerModule + ".views.assignments"} {
|
||||
if !slices.Contains(m.Consumes, view) {
|
||||
t.Errorf("the builder follows %s, its manifest consumes %v", view, m.Consumes)
|
||||
}
|
||||
}
|
||||
for want, got := range map[string]string{
|
||||
"mesh-delivery-gate_source": sourceBucket, "mesh-build-go_steps": buildrun.StepsBucket("mesh-build-go"),
|
||||
buildrun.ToolchainsHeardBucket: "mesh-delivery-builder_toolchains-heard", "mesh-delivery-indexer_packages": packagesBucket,
|
||||
"mesh-build-go_steps": buildrun.StepsBucket("mesh-build-go"),
|
||||
buildrun.ToolchainsHeardBucket: "mesh-delivery-builder_toolchains-heard",
|
||||
"mesh-delivery-assigner_assignments": buildrun.AssignmentsBucket,
|
||||
} {
|
||||
if want != got {
|
||||
|
||||
@@ -1,76 +1,8 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
)
|
||||
|
||||
// **A part does not stop for a bucket it can work without** (mesh-sdk process.EssentialBucket, OptionalBucket) (novox/hq issue 1945, met twice on 2026-10-25: the builder of
|
||||
// mesh-delivery #48 exited on a refused read of the gate's source, and its reasks bucket would have done the same). A grant
|
||||
// reaches the bus with a delivery, so a bucket the bus refuses, or times out on, at start is often only not granted yet:
|
||||
// exiting for it stops every build until a person's hand. A bucket the builder cannot build at all without (its builds,
|
||||
// the indexer's modules and manifests) still stops the start, naming the grant to look at; every other is said and left,
|
||||
// and opened again at the next event that needs it where there is one.
|
||||
|
||||
// trunkSource is the gate's source history the candidates' previews are merged onto (sourceHeads), opened at start and,
|
||||
// while it cannot be, again at the next candidate that asks a trunk head: a builder without it builds the trunk's keys,
|
||||
// and a candidate's preview waits, saying why.
|
||||
type trunkSource struct {
|
||||
ctx context.Context
|
||||
js bucketOpener
|
||||
heads *sourceHeads
|
||||
moved func(context.Context)
|
||||
errs chan<- error
|
||||
logf func(string, ...any)
|
||||
|
||||
mu sync.Mutex
|
||||
opened bool
|
||||
}
|
||||
|
||||
// ensure opens and settles the source's watch once, and follows it; an error leaves it closed for the next ask.
|
||||
func (t *trunkSource) ensure() error {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
if t.opened {
|
||||
return nil
|
||||
}
|
||||
kv, err := t.js.KeyValue(t.ctx, sourceBucket)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the bucket %s cannot be read (%v)", sourceBucket, err)
|
||||
}
|
||||
w, err := kv.WatchAll(t.ctx, jetstream.IncludeHistory()) // every watched branch's key: a revision names its head whichever branch it is of (design 47 G §15)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the bucket %s cannot be watched (%v)", sourceBucket, err)
|
||||
}
|
||||
sw, err := settled(t.ctx, w, func(e jetstream.KeyValueEntry) { t.heads.put(e) })
|
||||
if err != nil {
|
||||
return fmt.Errorf("the bucket %s: %w", sourceBucket, err)
|
||||
}
|
||||
t.opened = true
|
||||
go func() {
|
||||
t.errs <- follow(t.ctx, sw, func(e jetstream.KeyValueEntry) {
|
||||
if t.heads.put(e) && t.moved != nil {
|
||||
t.moved(t.ctx)
|
||||
}
|
||||
})
|
||||
}()
|
||||
return nil
|
||||
}
|
||||
|
||||
// start opens the source at start; one that cannot be is said, and the builder goes on.
|
||||
func (t *trunkSource) start() {
|
||||
if err := t.ensure(); err != nil {
|
||||
t.logf("%v: the trunk's keys are built, and a candidate's preview waits until the source can be read, tried again at its next candidate", err)
|
||||
}
|
||||
}
|
||||
|
||||
// head is Builder.Trunk: the head of repository at source revision rev, opening the source first if it is not yet.
|
||||
func (t *trunkSource) head(ctx context.Context, rev uint64, repository string) (string, error) {
|
||||
if err := t.ensure(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return t.heads.head(ctx, rev, repository)
|
||||
}
|
||||
|
||||
@@ -43,23 +43,11 @@ func TestARefusedBucketIsSaidAndThePartGoesOn(t *testing.T) {
|
||||
t.Run("a floor's wake is heard on the events consumer (design 51 The floor)", aFloorsWakeIsHeard)
|
||||
t.Run("a failed trunk build is a finding until a later done one (DLV-B7)", aFailedTrunkBuildIsAFinding)
|
||||
t.Run("a branch's end is acknowledged once its tagged versions are removed (design 51 Tagged versions)", aBranchsEndIsHeard)
|
||||
var logs bytes.Buffer
|
||||
logger := log.New(&logs, "", 0)
|
||||
refused := refusingBus{err: errors.New("nats: permissions violation for publish to $JS.API.CONSUMER.CREATE.KV_mesh-delivery-indexer_modules")}
|
||||
if _, err := process.EssentialBucket(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, "builder", "mesh-delivery-indexer_modules", "work"); err == nil ||
|
||||
!strings.Contains(err.Error(), "grant of mesh-delivery-indexer_modules") {
|
||||
t.Errorf("a refused bucket the part needs: %v, want an error naming its grant", err)
|
||||
}
|
||||
// The gate's source (the previews' trunk heads): refused at start, the builder goes on, and a candidate asking a trunk
|
||||
// head is told why and opens it again then.
|
||||
src := &trunkSource{ctx: context.Background(), js: refusingBus{refuseOpen: true, err: refused.err}, heads: &sourceHeads{}, errs: make(chan error, 1), logf: logger.Printf}
|
||||
src.start()
|
||||
if _, err := src.head(context.Background(), 7, "novox/mesh-catalog"); err == nil || !strings.Contains(err.Error(), sourceBucket) {
|
||||
t.Errorf("a trunk head with the source refused: %v, want why", err)
|
||||
}
|
||||
if src.opened {
|
||||
t.Error("a refused source counts as opened, so it is never opened again")
|
||||
}
|
||||
t.Run("ADR 0624: the counted nodes' engine reads", theCountedNodesEngineReads)
|
||||
}
|
||||
|
||||
@@ -258,7 +246,8 @@ func (m *wakeMsg) Headers() nats.Header {
|
||||
|
||||
func aFloorsWakeIsHeard(t *testing.T) {
|
||||
var logs bytes.Buffer
|
||||
waits := &versions.FloorWaits{}
|
||||
holders := map[string]string{"git": "mesh-delivery-forge-bridge", "mesh-catalog": "mesh-delivery-indexer"}
|
||||
waits := &versions.FloorWaits{Holder: func(seat string) (string, bool) { h, ok := holders[seat]; return h, ok }}
|
||||
waits.Wait("m/lib", versions.SeatGoPackages, "", versions.Floor{Silent: []string{versions.SourceRequirements}}, nil, time.Unix(0, 0))
|
||||
waits.Wait("m/img", versions.SeatImages, "", versions.Floor{Silent: []string{versions.SourceTags}}, nil, time.Unix(0, 0))
|
||||
f := &floorWake{b: &build.Builder{}, waits: waits, logf: log.New(&logs, "", 0).Printf}
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
@@ -48,6 +49,8 @@ type reservations struct {
|
||||
logf func(string, ...any)
|
||||
// call asks a seat's verb: the git seat's tag (design 47 G §370).
|
||||
call toolCaller
|
||||
// say says package-published on the seat mesh-builds; nil says none.
|
||||
say func(context.Context, meshbuilds.PackagePublished) error
|
||||
}
|
||||
|
||||
// newReservations makes the reserving over store; the released versions of a module are asked of the seat mesh-catalog's
|
||||
@@ -82,9 +85,9 @@ func (rv *reservations) Reserve(ctx context.Context, module, series string, arti
|
||||
|
||||
// Built records the digests; once every artifact has one, the publish's bound starts (a key of the state bounds with its
|
||||
// own time to live): its end with the reservation standing is version-unreleased.
|
||||
func (rv *reservations) Built(ctx context.Context, module, version string, digests map[string]string) error {
|
||||
func (rv *reservations) Built(ctx context.Context, module, version string, digests map[string]string, deps map[string][]meshbuilds.Dependency) error {
|
||||
for ref, d := range digests {
|
||||
if _, err := rv.r.Built(ctx, module, version, ref, d); err != nil {
|
||||
if _, err := rv.r.Built(ctx, module, version, ref, d, deps[ref]); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
@@ -203,9 +206,19 @@ func (rv *reservations) heard(ctx context.Context, m jetstream.Msg) bool {
|
||||
|
||||
// published marks the artifacts of the reservation a registry seat's published names (its version and digest).
|
||||
func (rv *reservations) published(ctx context.Context, o versions.Outcome) {
|
||||
if _, err := rv.r.Published(ctx, o.Version, o.Digest); err != nil {
|
||||
marked, err := rv.r.Published(ctx, o.Version, o.Digest)
|
||||
if err != nil {
|
||||
rv.logf("a published of %s@%s is not recorded in the reservations: %v", o.Name, o.Version, err)
|
||||
}
|
||||
// Each artifact the holder's published newly marked is said as the seat mesh-builds' package-published (design 47 G §52, §334).
|
||||
for _, m := range marked {
|
||||
if rv.say == nil {
|
||||
continue
|
||||
}
|
||||
if err := rv.say(ctx, m.PackagePublished(o.Digest, time.Now())); err != nil {
|
||||
rv.logf("the package-published of %s at %s is not said: %v", m.Artifact, m.Reserved.Version, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// dropped is builder_reserved_drop: the reservation is deleted and its bound ended.
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/git"
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds"
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
|
||||
|
||||
"mesh-delivery-builder/internal/versions"
|
||||
@@ -93,11 +94,13 @@ func TestTheCatalogsWordsMoveTheReservations(t *testing.T) {
|
||||
if !rv.heard(ctx, filled) || !filled.acked || !rv.Filled() || woke != 1 {
|
||||
t.Fatalf("versions-filled: acked %v, filled %v, woke %d", filled.acked, rv.Filled(), woke)
|
||||
}
|
||||
res, err := rv.Reserve(ctx, "m", "1.4", []string{"m:a", "m:b"}, versions.Source{})
|
||||
var said []meshbuilds.PackagePublished
|
||||
rv.say = func(_ context.Context, p meshbuilds.PackagePublished) error { said = append(said, p); return nil }
|
||||
res, err := rv.Reserve(ctx, "m", "1.4", []string{"m:a", "m:b"}, versions.Source{Branch: "main", Revision: 41})
|
||||
if err != nil || res.Version != "1.4.3" {
|
||||
t.Fatalf("reserved %q (%v), want 1.4.3 after the catalog's 1.4.2", res.Version, err)
|
||||
}
|
||||
if err := rv.Built(ctx, "m", "1.4.3", map[string]string{"m:a": "sha256:a", "m:b": "sha256:b"}); err != nil {
|
||||
if err := rv.Built(ctx, "m", "1.4.3", map[string]string{"m:a": "sha256:a", "m:b": "sha256:b"}, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
boundKey := versions.ReservedBoundKey("m", "1.4")
|
||||
@@ -105,6 +108,12 @@ func TestTheCatalogsWordsMoveTheReservations(t *testing.T) {
|
||||
t.Fatalf("the publish's bound is not armed: %v", bounds.keys)
|
||||
}
|
||||
rv.published(ctx, versions.Outcome{Name: "m-a", Version: "1.4.3", Digest: "sha256:a"})
|
||||
// Design 47 G §52, A-57: the holder's published is said once, per artifact, as the seat mesh-builds' package-published with the
|
||||
// branch and source revision of the reservation; a second published of the same artifact says nothing.
|
||||
rv.published(ctx, versions.Outcome{Name: "m-a", Version: "1.4.3", Digest: "sha256:a"})
|
||||
if len(said) != 1 || said[0].Artifact != "m:a" || said[0].Version != "1.4.3" || said[0].Digest != "sha256:a" || said[0].Branch != "main" || said[0].Source != 41 {
|
||||
t.Fatalf("package-published said %+v, want one of m:a at 1.4.3 for main at source 41", said)
|
||||
}
|
||||
if !rv.ended(ctx, purged{key: boundKey}) {
|
||||
t.Fatal("a reservation's bound was not taken in")
|
||||
}
|
||||
|
||||
@@ -31,7 +31,7 @@ type sayer struct {
|
||||
ctx context.Context
|
||||
store sayStore
|
||||
node string
|
||||
need []string
|
||||
need func() []string
|
||||
logf func(string, ...any)
|
||||
|
||||
mu sync.Mutex
|
||||
@@ -39,7 +39,7 @@ type sayer struct {
|
||||
said bool
|
||||
}
|
||||
|
||||
func newSayer(ctx context.Context, store sayStore, node string, need []string, logf func(string, ...any)) *sayer {
|
||||
func newSayer(ctx context.Context, store sayStore, node string, need func() []string, logf func(string, ...any)) *sayer {
|
||||
return &sayer{ctx: ctx, store: store, node: node, need: need, logf: logf, answered: map[string]bool{}}
|
||||
}
|
||||
|
||||
@@ -51,7 +51,7 @@ func (s *sayer) Answered(module string) {
|
||||
if s.said {
|
||||
return
|
||||
}
|
||||
for _, m := range s.need {
|
||||
for _, m := range s.need() {
|
||||
if !s.answered[m] {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -24,7 +24,7 @@ func (s saysOf) Delete(_ context.Context, key string, _ ...jetstream.KVDeleteOpt
|
||||
// one; stopped, it says nothing any more.
|
||||
func TestTheBuilderSaysItWritesReferencesOnceEveryBuildModuleAnsweredOne(t *testing.T) {
|
||||
s := saysOf{}
|
||||
say := newSayer(t.Context(), s, "shanks", []string{"mesh-build-go", "mesh-build-node", "mesh-build-image"}, t.Logf)
|
||||
say := newSayer(t.Context(), s, "shanks", func() []string { return []string{"mesh-build-go", "mesh-build-node", "mesh-build-image"} }, t.Logf)
|
||||
say.Answered("mesh-build-go")
|
||||
say.Answered("mesh-build-node")
|
||||
if v, ok := s["shanks"]; ok {
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"git.novox.be/novox/mesh-sdk/go/seatdef"
|
||||
"git.novox.be/novox/mesh-sdk/go/seatlist"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
)
|
||||
|
||||
// The builder reads the seat list as every part does, through mesh-sdk's one wiring (go/seatlist.Part; design 47 G §204): its
|
||||
// definitions name the build seats (the seats whose acting rule is the step build), its tools `builder_seat_list` and
|
||||
// `builder_seat_list_again`. A call through the mesh MCP server is proved only as its machine's tool runner (issue 627).
|
||||
var seatListAgainCallers = []string{stdio.AnyPerson, "node-tools", "mesh-healers"}
|
||||
|
||||
// seatGate is the builder's wiring of the seat list.
|
||||
type seatGate struct{ *seatlist.Part }
|
||||
|
||||
func newSeatGate() *seatGate { return &seatGate{seatlist.NewPart("builder", seatListAgainCallers)} }
|
||||
|
||||
// definitions are the seat definitions the list holds, none before it is read.
|
||||
func (g *seatGate) definitions() map[string]seatdef.Definition {
|
||||
if r := g.Reader(); r != nil {
|
||||
return r.Definitions()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
|
||||
"git.novox.be/novox/mesh-sdk/go/seatdef"
|
||||
)
|
||||
|
||||
// The build modules of the fixture seat list: the holders of the three build seats (design 47 G A-82).
|
||||
const (
|
||||
testGo = "mesh-build-go"
|
||||
testNode = "mesh-build-node"
|
||||
testImage = "mesh-build-image"
|
||||
)
|
||||
|
||||
// testSeats is the finder over a seat list whose build seats act on go, typescript and kind:image, each held by its module.
|
||||
func testSeats() *buildrun.BuildSeats {
|
||||
rule := func(on string) seatdef.Definition {
|
||||
return seatdef.Definition{Acts: []seatdef.ActingRule{{Step: seatdef.StepBuild, On: on}}}
|
||||
}
|
||||
defs := map[string]seatdef.Definition{"node-build-go": rule("go"), "node-build-node": rule("typescript"), "node-build-image": rule(seatdef.OnKindImage)}
|
||||
holders := map[string]string{"node-build-go": testGo, "node-build-node": testNode, "node-build-image": testImage}
|
||||
return buildrun.NewBuildSeats(func() map[string]seatdef.Definition { return defs }, func(seat string) (string, bool) { h, ok := holders[seat]; return h, ok })
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sync"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds"
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"mesh-delivery-builder/internal/build"
|
||||
)
|
||||
|
||||
// **The runner's build step ask is the builder's one entry point for a candidate** (design 47 G A-97, A-84, A-99): the seat
|
||||
// mesh-builds' accepted kind `step`, heard on its subject, names the entry of one module by fingerprint; the builder reads it through
|
||||
// the seat mesh-catalog's `entry`, builds it (build.Builder.BuildStep) and says the end as the seat's event step-ended.<node>, the
|
||||
// ask's id as the event's id, which the runner holds the end to.
|
||||
|
||||
// entryWaits answers an entry by fingerprint from the catalog half or the seat's `entry`, waiting for the answer the entries
|
||||
// keeper is told of (meshcatalogclient.Entries.Changed), never a timer.
|
||||
type entryWaits struct {
|
||||
branches *meshcatalogclient.Branches
|
||||
manifest *entryManifests
|
||||
mu sync.Mutex
|
||||
waiting map[string][]chan meshcatalogclient.EntryRow
|
||||
}
|
||||
|
||||
func newEntryWaits(branches *meshcatalogclient.Branches, manifests *entryManifests) *entryWaits {
|
||||
w := &entryWaits{branches: branches, manifest: manifests, waiting: map[string][]chan meshcatalogclient.EntryRow{}}
|
||||
prior := branches.Entries.Changed
|
||||
branches.Entries.Changed = func(row meshcatalogclient.EntryRow) {
|
||||
w.mu.Lock()
|
||||
chans := w.waiting[row.Fingerprint]
|
||||
delete(w.waiting, row.Fingerprint)
|
||||
w.mu.Unlock()
|
||||
for _, ch := range chans {
|
||||
ch <- row
|
||||
}
|
||||
if prior != nil {
|
||||
prior(row)
|
||||
}
|
||||
}
|
||||
return w
|
||||
}
|
||||
|
||||
// entry is build.Builder.StepEntry.
|
||||
func (w *entryWaits) entry(ctx context.Context, fingerprint string) (meshcatalog.CatalogEntry, error) {
|
||||
ch := make(chan meshcatalogclient.EntryRow, 1)
|
||||
w.mu.Lock()
|
||||
w.waiting[fingerprint] = append(w.waiting[fingerprint], ch)
|
||||
w.mu.Unlock()
|
||||
e, state := w.branches.Resolve(ctx, fingerprint)
|
||||
if state == meshcatalogclient.EntryAsked {
|
||||
select {
|
||||
case row := <-ch:
|
||||
if row.Entry == nil {
|
||||
return meshcatalog.CatalogEntry{}, fmt.Errorf("the seat mesh-catalog holds no entry of it (%s)", row.State)
|
||||
}
|
||||
e, state = *row.Entry, row.State
|
||||
case <-ctx.Done():
|
||||
return meshcatalog.CatalogEntry{}, ctx.Err()
|
||||
}
|
||||
}
|
||||
if state != meshcatalogclient.EntryAnswered {
|
||||
return meshcatalog.CatalogEntry{}, fmt.Errorf("the seat mesh-catalog's entry of it is %s", state)
|
||||
}
|
||||
return e, w.manifest.keep(e) // the manifest the build is planned from
|
||||
}
|
||||
|
||||
// serveSteps hears the step asks on the seat's accept subject and answers each one's end, in its own goroutine.
|
||||
func serveSteps(ctx context.Context, conn *nats.Conn, js jetstream.JetStream, cred process.Account, b *build.Builder, logf func(string, ...any)) error {
|
||||
sub, err := conn.Subscribe(meshbuilds.AcceptSubjectStep, func(m *nats.Msg) {
|
||||
id := m.Header.Get("x-event-id")
|
||||
if id == "" {
|
||||
id = m.Header.Get(nats.MsgIdHdr)
|
||||
}
|
||||
var run meshbuilds.StepRun
|
||||
if err := json.Unmarshal(m.Data, &run); err != nil || id == "" || run.Step != meshbuilds.StepRunStepBuild {
|
||||
logf("a step ask on %s that is no build step with an id is left: %v", m.Subject, err)
|
||||
return
|
||||
}
|
||||
go func() {
|
||||
end := b.BuildStep(ctx, run)
|
||||
end.ID = id
|
||||
if err := emitSeatEvent(context.WithoutCancel(ctx), js, cred, meshbuilds.SubjectStepEnded(cred.Node), id, end); err != nil {
|
||||
logf("saying the end of the build step %s: %v", id, err)
|
||||
}
|
||||
}()
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
go func() { <-ctx.Done(); _ = sub.Drain() }()
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
|
||||
)
|
||||
|
||||
// The runner's build step ask on the seat mesh-builds' accept subject, naming an entry by its fingerprint, is built through the
|
||||
// module's one claim and answered as the seat's step-ended.<node> under the ask's id (design 47 G A-97, A-99).
|
||||
func TestTheRunnersBuildStepAskIsAnsweredAsStepEnded(t *testing.T) {
|
||||
b := startBed(t)
|
||||
b.module(t, "netcheck", "catalogue-a67f0196-netcheck.module.json", "modules/netcheck")
|
||||
sub, err := nats.Connect(b.srvURL, nats.UserInfo("ctl", "pw"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer sub.Close()
|
||||
ended := make(chan *nats.Msg, 4)
|
||||
s, err := sub.ChanSubscribe(meshbuilds.SubjectStepEndedAll, ended)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer s.Unsubscribe()
|
||||
b.start(t, "n1", nil)
|
||||
b.await(t, "netcheck", isState(builds.BuildDone))
|
||||
run := meshbuilds.StepRun{Pipeline: "main.1.netcheck", Step: meshbuilds.StepRunStepBuild, Branch: "main", Revision: 1,
|
||||
Entries: []meshbuilds.BuildEntry{{Module: "netcheck", Fingerprint: fp}}}
|
||||
data, _ := json.Marshal(run)
|
||||
msg := nats.NewMsg(meshbuilds.AcceptSubjectStep)
|
||||
msg.Data = data
|
||||
msg.Header.Set("x-event-id", "ask-1")
|
||||
if err := sub.PublishMsg(msg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case m := <-ended:
|
||||
var end meshbuilds.StepEnded
|
||||
if err := json.Unmarshal(m.Data, &end); err != nil || end.ID != "ask-1" || end.Outcome != meshbuilds.StepEndedOutcomePassed || end.Pipeline != "main.1.netcheck" || m.Subject != meshbuilds.SubjectStepEnded("n1") {
|
||||
t.Fatalf("step-ended on %s = %+v, %v; want the ask ask-1 passed on n1", m.Subject, end, err)
|
||||
}
|
||||
case <-time.After(20 * time.Second):
|
||||
t.Fatal("no step-ended within 20 s")
|
||||
}
|
||||
}
|
||||
@@ -104,7 +104,7 @@ func (h *toolchainsHeard) bind(ctx context.Context, state heardState) {
|
||||
h.mu.Lock()
|
||||
h.kept = state
|
||||
h.mu.Unlock()
|
||||
for _, language := range buildrun.LanguageOf {
|
||||
for _, language := range h.set.Seats.Languages() {
|
||||
for _, e := range h.set.Entries(language) {
|
||||
h.keep(e.Node, e.Toolchain)
|
||||
}
|
||||
@@ -192,7 +192,7 @@ func (h *toolchainsHeard) running(ctx context.Context, data []byte) {
|
||||
h.runAt[id] = r.At
|
||||
kept := h.kept
|
||||
h.mu.Unlock()
|
||||
language, build := buildrun.LanguageOf[r.Module]
|
||||
language, build := h.set.Seats.LanguageOf(r.Module)
|
||||
if !build || r.Fingerprint != "" {
|
||||
return
|
||||
}
|
||||
@@ -224,7 +224,7 @@ func (h *toolchainsHeard) Dropped(ctx context.Context, key string) {
|
||||
// agree says toolchain-agreed for each language whose agreed version is not the one last said (design 47 G §472: said
|
||||
// only when the agreed version changes, ADR 0584 §1).
|
||||
func (h *toolchainsHeard) agree(ctx context.Context) {
|
||||
for _, language := range buildrun.LanguageOf {
|
||||
for _, language := range h.set.Seats.Languages() {
|
||||
entries := h.set.Entries(language)
|
||||
tc, none := buildstep.AgreedToolchain(entries)
|
||||
if none != "" {
|
||||
|
||||
@@ -75,7 +75,7 @@ type heardRig struct {
|
||||
func newHeardRig(t *testing.T, state *memState) *heardRig {
|
||||
t.Helper()
|
||||
r := &heardRig{state: state}
|
||||
r.h = newToolchainsHeard(&buildrun.ToolchainSet{}, func(context.Context) { r.changed++ },
|
||||
r.h = newToolchainsHeard(&buildrun.ToolchainSet{Seats: testSeats()}, func(context.Context) { r.changed++ },
|
||||
func(_ context.Context, e meshbuilds.ToolchainAgreed) error { r.said = append(r.said, e); return nil }, log.New(io.Discard, "", 0))
|
||||
r.h.bind(context.Background(), state)
|
||||
return r
|
||||
@@ -130,18 +130,18 @@ func TestTheBuilderKeepsTheToolchainsItHeardAndForgetsThemOnStopped(t *testing.T
|
||||
t.Fatalf("a changed agreed version was not said: %+v", again.said)
|
||||
}
|
||||
at := time.Now()
|
||||
again.running(t, "ace", buildrun.GoBuildModule, "src2:x", at) // running: nothing is forgotten
|
||||
again.running(t, "ace", testGo, "src2:x", at) // running: nothing is forgotten
|
||||
if k, _ := state.Keys(); len(k) != 3 {
|
||||
t.Fatalf("a running forgot a key: %v", k)
|
||||
}
|
||||
again.running(t, "ace", buildrun.GoBuildModule, "", at.Add(time.Second)) // stopped: no fingerprint
|
||||
again.running(t, "ace", testGo, "", at.Add(time.Second)) // stopped: no fingerprint
|
||||
if k, _ := state.Keys(); len(k) != 2 || k[0] != "g14.go" {
|
||||
t.Fatalf("stopped on ace left %v, want g14.go and novox.go", k)
|
||||
}
|
||||
// An older running read after (the start's read of the stream kept last per subject) changes nothing.
|
||||
before := again.changed
|
||||
again.running(t, "g14", buildrun.GoBuildModule, "", at)
|
||||
again.running(t, "g14", buildrun.GoBuildModule, "src2:y", at.Add(-time.Hour))
|
||||
again.running(t, "g14", testGo, "", at)
|
||||
again.running(t, "g14", testGo, "src2:y", at.Add(-time.Hour))
|
||||
if k, _ := state.Keys(); len(k) != 1 || k[0] != "novox.go" || again.changed != before+1 {
|
||||
t.Fatalf("after g14 stopped and an older running: %v", k)
|
||||
}
|
||||
|
||||
@@ -21,8 +21,8 @@ import (
|
||||
|
||||
// The floor's two module tools (novox/hq design 51 *The floor*); the seat's is `seat:<registry>.versions`.
|
||||
const (
|
||||
bridgeTagsTool = "mesh-delivery-forge-bridge.bridge_tags"
|
||||
requirementsTool = "mesh-delivery-indexer.indexer_requirements"
|
||||
bridgeTagsTool = "bridge_tags"
|
||||
requirementsTool = "indexer_requirements"
|
||||
)
|
||||
|
||||
// awaitedVersions is the state versions as versions.Store: bound once the bus holds it (the grants part makes it, issue
|
||||
@@ -60,55 +60,6 @@ func (s awaitedVersions) Get(ctx context.Context, key string) ([]byte, uint64, b
|
||||
return e.Value(), e.Revision(), true, nil
|
||||
}
|
||||
|
||||
// indexerPackages is the indexer's packages state as the builder reads a package's name in its ecosystem from it (design
|
||||
// 32): the picture of its read tool's first view and its views family (ADR 0615 §1), never its bucket. Until the first
|
||||
// view is answered a package is built unnumbered and said (versions.Numberer.Names), and numbered once its name is heard
|
||||
// (followPackages).
|
||||
type indexerPackages struct{ kv jetstream.KeyValue }
|
||||
|
||||
func (s indexerPackages) Keys(ctx context.Context) ([]string, error) {
|
||||
keys, err := s.kv.Keys(ctx)
|
||||
if errors.Is(err, jetstream.ErrNoKeysFound) {
|
||||
return nil, nil
|
||||
}
|
||||
return keys, err
|
||||
}
|
||||
|
||||
func (s indexerPackages) Get(ctx context.Context, key string) ([]byte, bool, error) {
|
||||
e, err := s.kv.Get(ctx, key)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) || errors.Is(err, jetstream.ErrKeyDeleted) {
|
||||
return nil, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
return e.Value(), true, nil
|
||||
}
|
||||
|
||||
// followPackages watches the picture of the indexer's packages state: each series key heard that names a package's name
|
||||
// (versions.NamedModule), its first view's included, looks at its module again (build.Builder.PackageNamed), so a done
|
||||
// build that left the package unnumbered is numbered then with its digests. Woken by the views family, never a timer; a
|
||||
// key heard again naming the name and module it named (the indexer writes a series at each publish) looks at nothing.
|
||||
func followPackages(ctx context.Context, kv jetstream.KeyValue, b *build.Builder, logf func(string, ...any)) error {
|
||||
w, err := kv.WatchAll(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
heard := map[string]string{}
|
||||
return follow(ctx, w, func(e jetstream.KeyValueEntry) {
|
||||
if e.Operation() != jetstream.KeyValuePut {
|
||||
delete(heard, e.Key())
|
||||
return
|
||||
}
|
||||
module, name, named := versions.NamedModule(e.Key(), e.Value())
|
||||
if named && heard[e.Key()] != module+" "+name {
|
||||
heard[e.Key()] = module + " " + name
|
||||
logf("%s.%s names a package of %s: its build is looked at again", packagesBucket, e.Key(), module)
|
||||
b.PackageNamed(ctx, module)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func versionsConflict(err error) error {
|
||||
if errors.Is(err, jetstream.ErrKeyExists) || errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
|
||||
return versions.ErrConflict
|
||||
@@ -154,7 +105,15 @@ func (s awaitedVersions) KeysOf(ctx context.Context, filter string) ([]string, e
|
||||
// floorVia is the floor's three sources as calls of the builder's own account (design 51 *The floor*): the seat's verb
|
||||
// versions, the forge bridge's bridge_tags and the indexer's indexer_requirements. A call that fails is that source
|
||||
// silent.
|
||||
func floorVia(call toolCaller) versions.FloorSources {
|
||||
func floorVia(call toolCaller, holder func(seat string) (string, bool)) versions.FloorSources {
|
||||
sourceTool := func(source, tool string) (string, error) {
|
||||
seat := versions.SourceSeats[source]
|
||||
module, ok := holder(seat)
|
||||
if !ok {
|
||||
return "", fmt.Errorf("no module is known to hold the seat %s, the source of %s", seat, tool)
|
||||
}
|
||||
return module + "." + tool, nil
|
||||
}
|
||||
return versions.FloorSources{
|
||||
Seat: func(_ context.Context, seat, name string) ([]string, error) {
|
||||
raw, err := call("seat:"+seat+".versions", map[string]any{"name": name})
|
||||
@@ -178,7 +137,11 @@ func floorVia(call toolCaller) versions.FloorSources {
|
||||
return out, nil
|
||||
},
|
||||
Tags: func(_ context.Context, repository, prefix string) ([]string, error) {
|
||||
raw, err := call(bridgeTagsTool, map[string]any{"repository": repository, "prefix": prefix})
|
||||
tool, err := sourceTool(versions.SourceTags, bridgeTagsTool)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
raw, err := call(tool, map[string]any{"repository": repository, "prefix": prefix})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -191,7 +154,11 @@ func floorVia(call toolCaller) versions.FloorSources {
|
||||
return a.Tags, nil
|
||||
},
|
||||
Requirements: func(_ context.Context, name string) ([]string, error) {
|
||||
raw, err := call(requirementsTool, map[string]any{"package": name})
|
||||
tool, err := sourceTool(versions.SourceRequirements, requirementsTool)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
raw, err := call(tool, map[string]any{"package": name})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user